import streamlit as st
import preprocessor, helper
import matplotlib.pyplot as plt

st.sidebar.title("Whatsapp Chat Analyzer")
uploaded_file = st.sidebar.file_uploader("Choose a file")
if uploaded_file is not None:
    bytes_data = uploaded_file.getvalue()
    data=bytes_data.decode("utf-8")
    df = preprocessor.preprocess(data)

    st.dataframe(df)
    #to extract users
    users_list=df['user'].unique().tolist()
    users_list.remove('group_notification')
    users_list.sort()
    users_list.insert(0,"Overall")

    selected_user = st.sidebar.selectbox("show Analysis wrt",users_list)

    if st.sidebar.button("Show Analysis"):
        col1, col2, col3, col4 = st.columns(4)
        num_message, words, num_media_msg,links = helper.fetch_stats(selected_user,df)
        with col1:
            st.header("Total Messages")
            st.title(num_message)
        with col2:
            st.header("Total Words")
            st.title(words)
        with col3:
            st.header("Media")
            st.title(num_media_msg)
        with col4:
            st.header("Links")
            st.title(links)

        #finding the busiest users in the group
        if selected_user == 'Overall':
            st.title('Busiest User')
            x, new_df =helper.busiest_user(df)
            fig, ax = plt.subplots()
            col1, col2 = st.columns(2)
            with col1:
                ax.bar(x.index,x.values,color='yellow')
                plt.xticks(rotation='vertical')
                st.pyplot(fig)
            with col2:
                st.dataframe(new_df)

        #WordCloud
        st.title("Word Cloud")
        df_wc =helper.create_wordcloud(selected_user,df)
        fig,ax = plt.subplots()
        ax.imshow(df_wc)
        st.pyplot(fig)


import matplotlib.pylab as plt

days = [1,2,3,4,5]
sleeping = [7,8,6,11,7]
eating = [2,3,4,3,2]
working = [7,8,7,2,2]
playing = [8,5,7,8,13]

slices = [7,2,2,13]
activities = ['sleeping','eating','working','playing']

cols = ['r','g','y','b']

plt.pie(slices,
        labels = activities ,
        colors = cols,
        startangle=90,
        shadow=True,
        explode=(0,0.1,0,0.1),
        autopct='%1.1f%%')

#plt.xlabel('X')
#plt.ylabel('Y')

plt.legend()

plt.title('Intresting Graph\nCheck it Out')
plt.show()


import aeidon
import contextlib
import inspect
import locale
import os
import random
import re
import shutil
import subprocess
import stat
import sys
import urllib.parse

def affirm(value):
    """Raise :exc:`aeidon.AffirmationError` if value evaluates to ``False``."""
    if not value:
        raise aeidon.AffirmationError

def atomic_open_require(path, mode="w", *args, **kwargs):
    assert "w" in mode

@contextlib.contextmanager
@aeidon.deco.contractual
def atomic_open(path, mode="w", *args, **kwargs):
    """
    A context manager for atomically writing a file.

    The file is written to a temporary file on the same filesystem, flushed and
    fsynced and then renamed to replace the existing file. This should
    (probably) be atomic on any Unix system. On Windows, it should (probably)
    be atomic if using Python 3.3 or greater.
    """
    path = os.path.realpath(path)
    chars = list("abcdefghijklmnopqrstuvwxyz0123456789")
    directory = os.path.dirname(path)
    basename = os.path.basename(path)
    while True:
        # Let's use a hidden temporary file to avoid a file
        # flickering in a possibly open file browser window.
        suffix = "".join(random.sample(chars, 8))
        temp_basename = ".{}.tmp{}".format(basename, suffix)
        temp_path = os.path.join(directory, temp_basename)
        if not os.path.isfile(temp_path): break
    try:
        if os.path.isfile(path):
            # If the file exists, use the same permissions.
            # Note that all other file metadata, including
            # owner and group, is not preserved.
            with open(temp_path, "w") as fobj: pass
            st = os.stat(path)
            os.chmod(temp_path, stat.S_IMODE(st.st_mode))
        with open(temp_path, mode, *args, **kwargs) as fobj:
            yield fobj
            fobj.flush()
            os.fsync(fobj.fileno())
        try:
            if hasattr(os, "replace"):
                # os.replace was added in Python 3.3.
                # This should be atomic on Windows too.
                os.replace(temp_path, path)
            else:
                # os.rename is atomic on Unix, but fails
                # on Windows if the file exists.
                os.rename(temp_path, path)
            # os.rename and os.replace will fail if path
            # and temp_path are not on the same device,
            # for instance they can be on two separate
            # branches of a union mount. Atomicity is not
            # possible in this case.
        except OSError:
            # Fall back to a non-atomic operation using
            # shutil.move. On Windows this requires that
            # the destination file does not exist.
            if sys.platform == "win32":
                if os.path.isfile(path):
                    os.remove(path)
            shutil.move(temp_path, path)
    finally:
        try:
            os.remove(temp_path)
        except Exception:
            pass

@aeidon.deco.once
def chardet_available():
    """Return ``True`` if :mod:`chardet` module is available."""
    try:
        import chardet
        return True
    except Exception:
        return False

def compare_versions_require(x, y):
    re_version = re.compile(r"^(\d+\.)+\d+$")
    assert re_version.match(x) is not None
    assert re_version.match(y) is not None

@aeidon.deco.contractual
def compare_versions(x, y):
    """
    Compare version strings `x` and `y`.

    Used version number formats are ``MAJOR.MINOR``, ``MAJOR.MINOR.PATCH`` and
    ``MAJOR.MINOR.PATCH.DATE/REVISION``, where all items are integers.
    Return 1 if `x` newer, 0 if equal or -1 if `y` newer.
    """
    compare = lambda a, b: (a > b) - (a < b)
    return compare(list(map(int, x.split("."))),
                   list(map(int, y.split("."))))

def connect_require(observer, observable, signal, *args):
    if observer is not observable:
        assert hasattr(observer, observable)

@aeidon.deco.contractual
def connect(observer, observable, signal, *args):
    """
    Connect `observable`'s `signal` to `observer`'s callback method.

    If `observable` is a string, it should be an attribute of `observer`.
    If `observable` is not a string it should be the same as `observer`.
    """
    method_name = signal.replace("-", "_").replace("::", "_")
    if observer is not observable:
        method_name = "_".join((observable, method_name))
    method_name = ("_on_{}".format(method_name)).replace("__", "_")
    if not hasattr(observer, method_name):
        method_name = method_name[1:]
    method = getattr(observer, method_name)
    if observer is not observable:
        observable = getattr(observer, observable)
    return observable.connect(signal, method, *args)

def copy_dict_ensure(src, value):
    assert src == value
    assert src is not value

@aeidon.deco.contractual
def copy_dict(src):
    """Copy `src` dictionary recursively and return copy."""
    dst = src.copy()
    for key, value in src.items():
        if isinstance(value, dict):
            dst[key] = copy_dict(value)
        if isinstance(value, list):
            dst[key] = copy_list(value)
        if isinstance(value, set):
            dst[key] = set(value)
    return dst

def copy_list_ensure(src, value):
    assert src == value
    assert src is not value

@aeidon.deco.contractual
def copy_list(src):
    """Copy `src` list recursively and return copy."""
    dst = list(src)
    for i, value in enumerate(src):
        if isinstance(value, dict):
            dst[i] = copy_dict(value)
        if isinstance(value, list):
            dst[i] = copy_list(value)
        if isinstance(value, set):
            dst[i] = set(value)
    return dst

def detect_format_require(path, encoding):
    assert aeidon.encodings.is_valid_code(encoding)

@aeidon.deco.contractual
def detect_format(path, encoding):
    """
    Detect and return format of subtitle file at `path`.

    Raise :exc:`IOError` if reading fails.
    Raise :exc:`UnicodeError` if decoding fails.
    Raise :exc:`aeidon.FormatError` if unable to detect format.
    Return an :attr:`aeidon.formats` enumeration item.
    """
    re_ids = [(x, re.compile(x.identifier)) for x in aeidon.formats]
    with open(path, "r", encoding=encoding) as fobj:
        for line in fobj:
            for format, re_id in re_ids:
                if re_id.search(line) is not None:
                    return format
    raise aeidon.FormatError("Failed to detect format of file {}"
                             .format(repr(path)))

def detect_newlines(path):
    """Detect and return the newline type of file at `path` or ``None``."""
    try:
        fobj = open(path, "r", newline="")
        fobj.read()
        chars = fobj.newlines
    except Exception:
        return None
    if chars is None:
        return None
    if isinstance(chars, str):
        return aeidon.newlines.find_item("value", chars)
    if isinstance(chars, tuple):
        if len(chars) == 1:
            return aeidon.newlines.find_item("value", chars[0])
        # This is not actually correct. If both CR and LF are detected,
        # it could mean a mixture of Mac and Unix newlines on separate
        # lines or one Windows newline in a mostly something else file.
        # We could count the frequencies, but it's probably not worth
        # the effort.
        return aeidon.newlines.WINDOWS
    return None

@aeidon.deco.once
def enchant_available():
    """Return ``True`` if :mod:`enchant` module is available."""
    try:
        import enchant
        return True
    except Exception:
        return False

@aeidon.deco.once
def enchant_and_dicts_available():
    """Return ``True`` if :mod:`enchant` and dictionaries are available."""
    try:
        import enchant
        return bool(enchant.list_languages())
    except Exception:
        return False

def flatten(lst):
    """
    Return a shallow version of `lst`.

    >>> aeidon.util.flatten([1, 2, 3, [4, 5, [6]]])
    [1, 2, 3, 4, 5, 6]
    """
    flat_lst = []
    for item in lst:
        if isinstance(item, list):
            flat_lst.extend(flatten(item))
        else: # Non-list item.
            flat_lst.append(item)
    return flat_lst

def get_chardet_version_ensure(value):
    if value is not None:
        assert isinstance(value, str)

@aeidon.deco.contractual
def get_chardet_version():
    """Return :mod:`chardet` version number as string or ``None``."""
    try:
        import chardet
        return chardet.__version__
    except Exception:
        return None

def get_default_encoding_ensure(value):
    assert aeidon.encodings.is_valid_code(value)

@aeidon.deco.once
@aeidon.deco.contractual
def get_default_encoding(fallback="utf_8"):
    """Return the locale encoding or `fallback`."""
    encoding = locale.getpreferredencoding()
    encoding = encoding or fallback
    re_illegal = re.compile(r"[^a-z0-9_]")
    encoding = re_illegal.sub("_", encoding.lower())
    encoding = get_encoding_alias(encoding)
    return encoding

@aeidon.deco.once
def get_default_newline():
    """Return system default newline as :attr:`aeidon.newlines` item."""
    return aeidon.newlines.find_item("value", os.linesep)

def get_enchant_version_ensure(value):
    if value is not None:
        assert isinstance(value, str)

@aeidon.deco.contractual
def get_enchant_version():
    """Return :mod:`enchant` version number as string or ``None``."""
    try:
        import enchant
        return enchant.__version__
    except Exception:
        return None

def get_encoding_alias(encoding):
    """Return proper Python alias for `encoding`."""
    from encodings.aliases import aliases
    if encoding in aliases:
        return aliases[encoding]
    return encoding

def get_ranges_require(lst):
    for item in lst:
        assert isinstance(item, int)

def get_ranges_ensure(value, lst):
    for item in value:
        assert item == list(range(item[0], item[-1] + 1))

@aeidon.deco.contractual
def get_ranges(lst):
    """
    Return a list of ranges in list of integers.

    >>> aeidon.util.get_ranges([1, 2, 3, 5, 6, 7, 9, 11, 12])
    [[1, 2, 3], [5, 6, 7], [9], [11, 12]]
    """
    if not lst: return []
    lst = get_sorted_unique(lst)
    ranges = [[lst.pop(0)]]
    for item in lst:
        if item == ranges[-1][-1] + 1:
            ranges[-1].append(item)
        else: ranges.append([item])
    return ranges

def get_sorted_unique_ensure(value, lst):
    for item in value:
        assert value.count(item) == 1
    assert sorted(value) == value

@aeidon.deco.contractual
def get_sorted_unique(lst):
    """
    Return sorted `lst` with duplicates removed.

    >>> aeidon.util.get_sorted_unique([3, 2, 1, 2, 4, 4])
    [1, 2, 3, 4]
    """
    lst = sorted(lst)
    for i in reversed(range(1, len(lst))):
        if lst[i] == lst[i - 1]:
            lst.pop(i)
    return lst

@aeidon.deco.memoize(100)
def get_template_header(format):
    """
    Read and return the template header for `format`.

    Raise :exc:`IOError` if reading global header file fails.
    Raise :exc:`UnicodeError` if decoding global header file fails.
    """
    header = None
    directory = os.path.join(aeidon.DATA_HOME_DIR, "headers")
    path = os.path.join(directory, format.name.lower())
    if os.path.isfile(path):
        try: header = read(path, None).rstrip()
        except IOError:
            print_read_io(sys.exc_info(), path)
        except UnicodeError:
            print_read_unicode(sys.exc_info(),
                               path,
                               get_default_encoding())

    if header is None:
        directory = os.path.join(aeidon.DATA_DIR, "headers")
        path = os.path.join(directory, format.name.lower())
        header = read(path, "ascii").rstrip()
    return normalize_newlines(header)

def get_unique_ensure(value, lst, keep_last=False):
    for item in value:
        assert value.count(item) == 1

@aeidon.deco.contractual
def get_unique(lst, keep_last=False):
    """
    Return `lst` with duplicates removed.

    Keep the last duplicate if `keep_last` is ``True``, else keep first.

    >>> aeidon.util.get_unique([3, 2, 1, 2, 4, 4])
    [3, 2, 1, 4]
    >>> aeidon.util.get_unique([3, 2, 1, 2, 4, 4], keep_last=True)
    [3, 1, 2, 4]
    """
    lst = lst[:]
    if keep_last:
        lst.reverse()
    for i in reversed(range(len(lst))):
        for j in range(0, i):
            if lst[j] == lst[i]:
                lst.pop(i)
                break
    if keep_last:
        lst.reverse()
    return lst

def install_module(name, obj):
    """
    Install `obj`'s module into the :mod:`aeidon` namespace.

    Typical call is of form::

        aeidon.util.install_module("foo", lambda: None)
    """
    aeidon.__dict__[name] = inspect.getmodule(obj)

def is_command(command):
    """Return ``True`` if `command` exists as a file in ``$PATH``."""
    dirs = os.environ.get("PATH", "").split(os.pathsep)
    paths = [os.path.join(x, command) for x in dirs]
    return any(map(os.path.isfile, paths))

def last(iterator):
    """Return the last value from `iterator` or ``None``."""
    value = None
    for value in iterator: pass
    return value

def makedirs_ensure(value, directory):
    assert os.path.isdir(directory)

@aeidon.deco.contractual
def makedirs(directory):
    """
    Recursively make `directory` if it does not exist.

    Raise :exc:`OSError` if unsuccessful.
    """
    if not os.path.isdir(directory):
        os.makedirs(directory)

def normalize_newlines(text):
    """
    Convert all newlines in `text` to Unix newlines.

    >>> aeidon.util.normalize_newlines("one\\r\\ntwo")
    'one\\ntwo'
    """
    re_newline_char = re.compile(r"\r\n?")
    return re_newline_char.sub("\n", text)

def path_to_uri(path):
    """Convert local filepath to URI."""
    if sys.platform == "win32":
        path = "/{}".format(path.replace("\\", "/"))
    return "file://{}".format(urllib.parse.quote(path))

def print_read_io(exc_info, path):
    """Print :exc:`IOError` message to standard error."""
    print("Failed to read file '{}': {}"
          .format(path, exc_info[1].args[1]),
          file=sys.stderr)

def print_read_unicode(exc_info, path, encoding):
    """Print :exc:`UnicodeError` message to standard error."""
    encoding = encoding or get_default_encoding()
    print("Failed to decode file '{}' with codec '{}'"
          .format(path, encoding),
          file=sys.stderr)

def print_remove_os(exc_info, path):
    """Print :exc:`OSError` message to standard error."""
    print("Failed to remove file '{}': {}"
          .format(path, exc_info[1].args[1]),
          file=sys.stderr)

def print_write_io(exc_info, path):
    """Print :exc:`IOError` message to standard error."""
    print("Failed to write file '{}': {}"
          .format(path, exc_info[1].args[1]),
          file=sys.stderr)

def print_write_unicode(exc_info, path, encoding):
    """Print :exc:`UnicodeError` message to standard error."""
    encoding = encoding or get_default_encoding()
    print("Failed to encode file '{}' with codec '{}'"
          .format(path, encoding),
          file=sys.stderr)

def read_require(path, encoding=None, fallback="utf_8"):
    if encoding is not None:
        assert aeidon.encodings.is_valid_code(encoding)
    if fallback is not None:
        assert aeidon.encodings.is_valid_code(fallback)

@aeidon.deco.contractual
def read(path, encoding=None, fallback="utf_8"):
    """
    Read file at `path` and return text.

    `fallback` should be ``None`` to try only `encoding`.
    Raise :exc:`IOError` if reading fails.
    Raise :exc:`UnicodeError` if decoding fails.
    """
    encoding = encoding or get_default_encoding()
    try:
        with open(path, "r", encoding=encoding) as fobj:
            return fobj.read().strip()
    except UnicodeError:
        if not fallback in (encoding, None, ""):
            return read(path, fallback, None)
        raise # UnicodeError

def readlines_require(path, encoding=None, fallback="utf_8"):
    if encoding is not None:
        assert aeidon.encodings.is_valid_code(encoding)
    if fallback is not None:
        assert aeidon.encodings.is_valid_code(fallback)

@aeidon.deco.contractual
def readlines(path, encoding=None, fallback="utf_8"):
    """
    Read file at `path` and return lines.

    `fallback` should be ``None`` to try only `encoding`.
    Raise :exc:`IOError` if reading fails.
    Raise :exc:`UnicodeError` if decoding fails.
    """
    text = read(path, encoding, fallback)
    text = normalize_newlines(text)
    return text.split("\n")

def replace_extension(path, format):
    """Replace possible extension in `path` with that of `format`."""
    extensions = [x.extension for x in aeidon.formats]
    if path.endswith(tuple(extensions)):
        path = path[:path.rfind(".")]
    return "".join((path, format.extension))

def shell_quote(path):
    """Quote and escape `path` for shell use."""
    if sys.platform != "win32":
        # Windows filenames can contain backslashes only as
        # directory separators and cannot contain double quotes.
        path = path.replace("\\", "\\\\")
        path = path.replace('"', '\\"')
    return '"{}"'.format(path)

def start_process(command, **kwargs):
    """
    Start `command` as a new background subprocess.

    `command` and `kwargs` are passed to :class:`subprocess.Popen`.
    Raise :exc:`aeidon.ProcessError` if something goes wrong.
    Return :class:`subprocess.Popen` instance.
    """
    # Use no environment on Windows due to a subprocess bug.
    # https://bugzilla.gnome.org/show_bug.cgi?id=605805
    try:
        return subprocess.Popen(command,
                                shell=(sys.platform != "win32"),
                                cwd=os.getcwd(),
                                env=(os.environ.copy()
                                     if sys.platform != "win32"
                                     else None),

                                universal_newlines=True,
                                **kwargs)

    except OSError as error:
        raise aeidon.ProcessError(str(error.args))

def title_to_lower_case_ensure(value, title_name):
    assert value.islower()

@aeidon.deco.memoize(100)
@aeidon.deco.contractual
def title_to_lower_case(title_name):
    """
    Convert title case name to lower case with underscores.

    >>> aeidon.util.title_to_lower_case('TitleCase')
    'title_case'
    """
    lower_name = ""
    for char in title_name:
        if char.isupper() and lower_name:
            lower_name += "_"
        lower_name += char.lower()
    return lower_name

def uri_to_path(uri):
    """Convert `uri` to local filepath."""
    uri = urllib.parse.unquote(uri)
    if sys.platform == "win32":
        path = urllib.parse.urlsplit(uri)[2]
        while path.startswith("/"):
            path = path[1:]
        return path.replace("/", "\\")
    return urllib.parse.urlsplit(uri)[2]

def write_require(path, text, encoding=None, fallback="utf_8"):
    if encoding is not None:
        assert aeidon.encodings.is_valid_code(encoding)
    if fallback is not None:
        assert aeidon.encodings.is_valid_code(fallback)

def write_ensure(value, path, text, encoding=None, fallback="utf_8"):
    assert os.path.isfile(path)

@aeidon.deco.contractual
def write(path, text, encoding=None, fallback="utf_8"):
    """
    Write `text` to file at `path`.

    `fallback` should be ``None`` to try only `encoding`.
    Raise :exc:`IOError` if writing fails.
    Raise :exc:`UnicodeError` if encoding fails.
    """
    encoding = encoding or get_default_encoding()
    try:
        with open(path, "w", encoding=encoding) as fobj:
            return fobj.write(text)
    except UnicodeError:
        if not fallback in (encoding, None, ""):
            return write(path, text, fallback, None)
        raise # UnicodeError

def writelines_require(path, lines, encoding=None, fallback="utf_8"):
    if encoding is not None:
        assert aeidon.encodings.is_valid_code(encoding)
    if fallback is not None:
        assert aeidon.encodings.is_valid_code(fallback)

def writelines_ensure(value, path, lines, encoding=None, fallback="utf_8"):
    assert os.path.isfile(path)

@aeidon.deco.contractual
def writelines(path, lines, encoding=None, fallback="utf_8"):
    """
    Write `lines` of text to file at `path`.

    `fallback` should be ``None`` to try only `encoding`.
    Raise :exc:`IOError` if writing fails.
    Raise :exc:`UnicodeError` if encoding fails.
    """
    text = "\n".join(lines) + "\n"
    return write(path, text, encoding, fallback)


import numpy as np
import matplotlib.pyplot as plt
import pandas as pd
import numpy as np
import os
import lob_for_futures as lobFut
from lob_for_futures import *
import os
import matplotlib.pyplot as plt

from dateutil.parser import parse
from pandas.tseries.offsets import BDay
import pickle as pkl
import fnmatch
import pickle
import datetime as dt
from datetime import timedelta
import glob
import fathon
from fathon import fathonUtils as fu
import itertools
import pyinform as pyinf
import pingouin as pig
from scipy.stats import entropy
from scipy.spatial.distance import jensenshannon
from scipy.special import kl_div
import time
dataFolder = lobFut.dataFolder
quotes =[f for f in os.listdir(dataFolder) if str('_quotes') in f]
trades =[f for f in os.listdir(dataFolder) if str('_trades') in f]

symbolIdx = 2

symbols = [f.split("_")[0] for f in quotes]

symbol = sorted(symbols)[symbolIdx]
print(symbol)
quotesFileCh = os.path.join(dataFolder, quotes[symbolIdx])
tradesFileCh = os.path.join(dataFolder, trades[symbolIdx])

# get common Dates
quotesDates = sorted([f.split(".csv")[0] for f in os.listdir(quotesFileCh)])
tradesDates = sorted([f.split(".csv")[0] for f in os.listdir(tradesFileCh)])
intersectionDates = list(set(quotesDates).intersection(tradesDates))

trades_cols = ['size', 'time', 'type', 'value']
experimentsDestination = '/media/ak/T7/MFDFA Experiments'

if __name__ == '__main__':
    # params i need for fathon
    winSizes = fu.linRangeByStep(5, 60)
    revSeg = True
    polOrd = 1
    # load all teh dataframes at once with this
    start = time.time()
    print(start)
    # go through all the dataframes and take out n, f and h, h_ intercept
    # somehow one list comprehension blows up in memory

    h_dict = defaultdict(dict)

    symbolIdx = 0

    symbol = sorted(symbols)[symbolIdx]
    print(symbol)
    quotesFileCh = os.path.join(dataFolder, quotes[symbolIdx])
    tradesFileCh = os.path.join(dataFolder, trades[symbolIdx])

    # get common Dates
    quotesDates = sorted([f.split(".csv")[0] for f in os.listdir(quotesFileCh)])
    tradesDates = sorted([f.split(".csv")[0] for f in os.listdir(tradesFileCh)])
    intersectionDates = list(set(quotesDates).intersection(tradesDates))

    trades_cols = ['size', 'time', 'type', 'value']

    # params for the clocks

    calendar_resample_freq = "300S"
    trade_volume_width = 100
    ticks_width = 100
    usd_volume_width = 100
    dates_choice = intersectionDates

    testClass = DataLoader(data_location=dataFolder,
                           symbol=symbol,
                           dates=dates_choice,
                           use_columns=trades_cols,
                           calendar_resample=calendar_resample_freq,
                           trade_volume_width=trade_volume_width,
                           ticks_width=ticks_width,
                           usd_volume_width=usd_volume_width)

    hash_of_file = "_".join(
        (str(symbol), "volume_width", str(trade_volume_width), "calendar_resample", str(calendar_resample_freq)))

    # load data
    input_dict = testClass.load_and_format_data()

    tick_bar_dfs = []
    volume_bar_dfs = []
    usd_volume_bar_dfs = []
    calendar_bar_dfs = []
    dates = list(input_dict.keys())

    bar_returns = dict()
    for date in dates:
        df = testClass.load_and_format_data()[str(date)]
        input_dict = testClass.get_bars(df)
        tick_bar_df = testClass.get_concat_data(testClass._bars_dict)['tick_bars']
        volume_bar_df = testClass.get_concat_data(testClass._bars_dict)['volume_bars']
        usd_volume_bar_df = testClass.get_concat_data(testClass._bars_dict)['usd_volume_bars']
        calendar_bar_df = testClass.get_concat_data(testClass._bars_dict)['calendar_bars']
        vr = returns(volume_bar_df.micro_price_close).replace([np.inf, -np.inf], 0)  # volume
        tr = returns(tick_bar_df.micro_price_close).replace([np.inf, -np.inf], 0)  # tick
        dr = returns(usd_volume_bar_df.micro_price_close).dropna().replace([np.inf, -np.inf], 0)  # usd volume
        df_ret = returns(calendar_bar_df.micro_price_close).dropna().replace([np.inf, -np.inf], 0)  # calendar
        bar_returns[date] = {'tick': tr,
                             'volume': vr,
                             'dollar': dr,
                             'calendar': df_ret}
    dict_dfa = defaultdict(dict)
    for i, j in itertools.product(['tick', 'volume', 'dollar', 'calendar'], dates):
        data = (bar_returns[j][i])
        a = fu.toAggregated(np.asanyarray(data))
        pydfa = fathon.DFA(a)
        pymfdfa = fathon.MFDFA(a)
        n, F = pydfa.computeFlucVec(winSizes, revSeg=revSeg, polOrd=polOrd)
        dict_dfa[j][i] = dict(zip(n, F))  # dictionary to match all the n and F values

        H, H_intercept = pydfa.fitFlucVec()  # same for H values
        h_dict[j][i] = [H, H_intercept]

        print('for date', j, ' and bar type', i, 'you get', H)

        # everything below here is to store

    pickle_out_filename = os.path.join(experimentsDestination, symbol, 'DFA_' +
                                       'n_f_values' +"volume_width"+ str(trade_volume_width)
                                       + "calendar_resample"+ str(calendar_resample_freq)+
                                       '_bar_' + str(i) + '_' + str(j) + '.pkl')
    pickle.dump(dict_dfa, open(pickle_out_filename, 'wb'), protocol=pickle.HIGHEST_PROTOCOL)
    print('just saved: ', pickle_out_filename)

    pickle_out_filename_two = os.path.join(experimentsDestination, symbol, 'DFA_' +
                                       'n_f_values' +"volume_width"+ str(trade_volume_width)
                                       + "calendar_resample"+ str(calendar_resample_freq)+
                                       '_bar_' +
                                       'H_values' + '_bar_' + str(i) + '_' + str(j) + '.pkl')
    pickle.dump(h_dict, open(pickle_out_filename_two, 'wb'), protocol=pickle.HIGHEST_PROTOCOL)
    print('just saved: ', pickle_out_filename_two)


import traceback
from datetime import datetime
from typing import List, Union

from google.protobuf import timestamp_pb2, wrappers_pb2

import durabletask.internal.orchestrator_service_pb2 as pb

# TODO: The new_xxx_event methods are only used by test code and should be moved elsewhere

def new_orchestrator_started_event(timestamp: Union[datetime, None] = None) -> pb.HistoryEvent:
    ts = timestamp_pb2.Timestamp()
    if timestamp is not None:
        ts.FromDatetime(timestamp)
    return pb.HistoryEvent(eventId=-1, timestamp=ts, orchestratorStarted=pb.OrchestratorStartedEvent())

def new_execution_started_event(name: str, instance_id: str, encoded_input: Union[str, None] = None) -> pb.HistoryEvent:
    return pb.HistoryEvent(
        eventId=-1,
        timestamp=timestamp_pb2.Timestamp(),
        executionStarted=pb.ExecutionStartedEvent(
            name=name,
            input=get_string_value(encoded_input),
            orchestrationInstance=pb.OrchestrationInstance(instanceId=instance_id)))

def new_timer_created_event(timer_id: int, fire_at: datetime) -> pb.HistoryEvent:
    ts = timestamp_pb2.Timestamp()
    ts.FromDatetime(fire_at)
    return pb.HistoryEvent(
        eventId=timer_id,
        timestamp=timestamp_pb2.Timestamp(),
        timerCreated=pb.TimerCreatedEvent(fireAt=ts)
    )

def new_timer_fired_event(timer_id: int, fire_at: datetime) -> pb.HistoryEvent:
    ts = timestamp_pb2.Timestamp()
    ts.FromDatetime(fire_at)
    return pb.HistoryEvent(
        eventId=-1,
        timestamp=timestamp_pb2.Timestamp(),
        timerFired=pb.TimerFiredEvent(fireAt=ts, timerId=timer_id)
    )

def new_task_scheduled_event(event_id: int, name: str, encoded_input: Union[str, None] = None) -> pb.HistoryEvent:
    return pb.HistoryEvent(
        eventId=event_id,
        timestamp=timestamp_pb2.Timestamp(),
        taskScheduled=pb.TaskScheduledEvent(name=name, input=get_string_value(encoded_input))
    )

def new_task_completed_event(event_id: int, encoded_output: Union[str, None] = None) -> pb.HistoryEvent:
    return pb.HistoryEvent(
        eventId=-1,
        timestamp=timestamp_pb2.Timestamp(),
        taskCompleted=pb.TaskCompletedEvent(taskScheduledId=event_id, result=get_string_value(encoded_output))
    )

def new_task_failed_event(event_id: int, ex: Exception) -> pb.HistoryEvent:
    return pb.HistoryEvent(
        eventId=-1,
        timestamp=timestamp_pb2.Timestamp(),
        taskFailed=pb.TaskFailedEvent(taskScheduledId=event_id, failureDetails=new_failure_details(ex))
    )

def new_sub_orchestration_created_event(
        event_id: int,
        name: str,
        instance_id: str,
        encoded_input: Union[str, None] = None) -> pb.HistoryEvent:
    return pb.HistoryEvent(
        eventId=event_id,
        timestamp=timestamp_pb2.Timestamp(),
        subOrchestrationInstanceCreated=pb.SubOrchestrationInstanceCreatedEvent(
            name=name,
            input=get_string_value(encoded_input),
            instanceId=instance_id)
    )

def new_sub_orchestration_completed_event(event_id: int, encoded_output: Union[str, None] = None) -> pb.HistoryEvent:
    return pb.HistoryEvent(
        eventId=-1,
        timestamp=timestamp_pb2.Timestamp(),
        subOrchestrationInstanceCompleted=pb.SubOrchestrationInstanceCompletedEvent(
            result=get_string_value(encoded_output),
            taskScheduledId=event_id)
    )

def new_sub_orchestration_failed_event(event_id: int, ex: Exception) -> pb.HistoryEvent:
    return pb.HistoryEvent(
        eventId=-1,
        timestamp=timestamp_pb2.Timestamp(),
        subOrchestrationInstanceFailed=pb.SubOrchestrationInstanceFailedEvent(
            failureDetails=new_failure_details(ex),
            taskScheduledId=event_id)
    )

def new_failure_details(ex: Exception) -> pb.TaskFailureDetails:
    return pb.TaskFailureDetails(
        errorType=type(ex).__name__,
        errorMessage=str(ex),
        stackTrace=wrappers_pb2.StringValue(value=''.join(traceback.format_tb(ex.__traceback__)))
    )

def new_event_raised_event(name: str, encoded_input: Union[str, None] = None) -> pb.HistoryEvent:
    return pb.HistoryEvent(
        eventId=-1,
        timestamp=timestamp_pb2.Timestamp(),
        eventRaised=pb.EventRaisedEvent(name=name, input=get_string_value(encoded_input))
    )

def new_suspend_event() -> pb.HistoryEvent:
    return pb.HistoryEvent(
        eventId=-1,
        timestamp=timestamp_pb2.Timestamp(),
        executionSuspended=pb.ExecutionSuspendedEvent()
    )

def new_resume_event() -> pb.HistoryEvent:
    return pb.HistoryEvent(
        eventId=-1,
        timestamp=timestamp_pb2.Timestamp(),
        executionResumed=pb.ExecutionResumedEvent()
    )

def new_terminated_event(*, encoded_output: Union[str, None] = None) -> pb.HistoryEvent:
    return pb.HistoryEvent(
        eventId=-1,
        timestamp=timestamp_pb2.Timestamp(),
        executionTerminated=pb.ExecutionTerminatedEvent(
            input=get_string_value(encoded_output)
        )
    )

def get_string_value(val: Union[str, None]) -> Union[wrappers_pb2.StringValue, None]:
    if val is None:
        return None
    else:
        return wrappers_pb2.StringValue(value=val)

def new_complete_orchestration_action(
        id: int,
        status: pb.OrchestrationStatus,
        result: Union[str, None] = None,
        failure_details: Union[pb.TaskFailureDetails, None] = None,
        carryover_events: Union[List[pb.HistoryEvent], None] = None) -> pb.OrchestratorAction:
    completeOrchestrationAction = pb.CompleteOrchestrationAction(
        orchestrationStatus=status,
        result=get_string_value(result),
        failureDetails=failure_details,
        carryoverEvents=carryover_events)

    return pb.OrchestratorAction(id=id, completeOrchestration=completeOrchestrationAction)

def new_create_timer_action(id: int, fire_at: datetime) -> pb.OrchestratorAction:
    timestamp = timestamp_pb2.Timestamp()
    timestamp.FromDatetime(fire_at)
    return pb.OrchestratorAction(id=id, createTimer=pb.CreateTimerAction(fireAt=timestamp))

def new_schedule_task_action(id: int, name: str, encoded_input: Union[str, None]) -> pb.OrchestratorAction:
    return pb.OrchestratorAction(id=id, scheduleTask=pb.ScheduleTaskAction(
        name=name,
        input=get_string_value(encoded_input)
    ))

def new_timestamp(dt: datetime) -> timestamp_pb2.Timestamp:
    ts = timestamp_pb2.Timestamp()
    ts.FromDatetime(dt)
    return ts

def new_create_sub_orchestration_action(
        id: int,
        name: str,
        instance_id: Union[str, None],
        encoded_input: Union[str, None]) -> pb.OrchestratorAction:
    return pb.OrchestratorAction(id=id, createSubOrchestration=pb.CreateSubOrchestrationAction(
        name=name,
        instanceId=instance_id,
        input=get_string_value(encoded_input)
    ))

def is_empty(v: wrappers_pb2.StringValue):
    return v is None or v.value == ''

def get_orchestration_status_str(status: pb.OrchestrationStatus):
    try:
        const_name = pb.OrchestrationStatus.Name(status)
        if const_name.startswith('ORCHESTRATION_STATUS_'):
            return const_name[len('ORCHESTRATION_STATUS_'):]
    except Exception:
        return "UNKNOWN"


from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
import pandas as pd
import argparse
import sys,os
import tensorflow as tf
from PIL import Image
from tqdm import tqdm
import csv

t=tqdm(pd.read_csv('/home/DATA/guoming/food_recog/raw_data/list/food_meituan_val.csv').values)
val_list=[]
val_labels=[]

i=0
for tt in t:
    val_list.append(tt[0])
    val_labels.append(tt[1])
    i+=1

def load_image(filename):
    #Read in the image_data to be classified."""
    return tf.gfile.FastGFile(filename, 'rb').read()

def load_labels(filename):
    #Read in labels, one label per line."""
    return [line.rstrip() for line in tf.gfile.GFile(filename)]

def load_graph(filename):
    #Unpersists graph from file as default graph."""
    with tf.gfile.FastGFile(filename, 'rb') as f:
        graph_def = tf.GraphDef()
        graph_def.ParseFromString(f.read())
        tf.import_graph_def(graph_def, name='')
def read_tensor_from_image_file(file_name,
                                input_height=224,
                                input_width=224,
                                input_mean=127.5,
                                input_std=127.5):
    input_name = "file_reader"
    output_name = "normalized"
    file_reader = tf.read_file(file_name, input_name)
    if file_name.endswith(".png"):
        image_reader = tf.io.decode_png(
            file_reader, channels=3, name="png_reader")
    elif file_name.endswith(".gif"):
        image_reader = tf.squeeze(
            tf.io.decode_gif(file_reader, name="gif_reader"))
    elif file_name.endswith(".bmp"):
        image_reader = tf.io.decode_bmp(file_reader, name="bmp_reader")
    else:
        image_reader = tf.image.decode_jpeg(
            file_reader, channels=3, name="jpeg_reader")
    float_caster = tf.cast(image_reader, tf.float32)
    dims_expander = tf.expand_dims(float_caster, 0)
    resized = tf.image.resize_bilinear(
        dims_expander, [input_height, input_width])
    normalized = tf.divide(tf.subtract(resized, [input_mean]), [input_std])
    sess = tf.Session()
    result = sess.run(normalized)

    return result

def run_graph(src, dest, labels, input_layer_name, output_layer_name, num_top_predictions,val_img_list,output_csv):
    config = tf.ConfigProto(gpu_options=tf.GPUOptions(allow_growth=True))
    #config.gpu_options.per_process_gpu_memory_fraction = 0.1
    with tf.Session(config=config) as sess:
        # Feed the image_data as input to the graph.
        # predictions  will contain a two-dimensional array, where one
        # dimension represents the input image count, and the other has
        # predictions per class
        i=0
        f_csv = open(output_csv,'w',encoding='utf-8',newline="")
        csv_writer = csv.writer(f_csv)
        csv_writer.writerow(["id","predicted"])
        #with open('submit.csv','w') as outfile:
        right_num=0
        for f in tqdm(val_img_list):
            # im=Image.open(os.path.join(src,f))
            # im = im.resize((224,224),Image.ANTIALIAS)
            # img=im.convert('RGB')
            # img.save(os.path.join(dest,f))
            # image_data=load_image(os.path.join(dest,f))

            # img_data_jpg = tf.image.decode_jpeg(image_data) #图像解码
            # img_data_jpg = tf.image.convert_image_dtype(img_data_jpg, dtype=tf.float32)
            file_name = os.path.join(src,f)
            t = read_tensor_from_image_file(
            file_name)
            softmax_tensor=sess.graph.get_tensor_by_name(output_layer_name)
            predictions,=sess.run(softmax_tensor, {input_layer_name: t})

            # Sort to show labels in order of confidence
            top_k = predictions.argsort()[-num_top_predictions:][::-1]
            result=[]
            for node_id in top_k:
                score = predictions[node_id]
                result.append(int(labels[node_id]))
            if val_labels[i] in result:
                right_num+=1
            csv_writer.writerow([f,result])
            i+=1
            if i%100==1:
                print("processing %d"%i)
                accuracy=float(right_num)/float(i)
                print("accuracy %.4f"%accuracy)

    accuracy=float(right_num)/float(len(val_img_list))
    print("accuracy %.4f"%accuracy)

os.environ['CUDA_VISIBLE_DEVICES']='0'
src="/home/DATA/guoming/food_recog/raw_data/val"
dest="/home/DATA/guoming/food_recog/raw_data/val2"
if not os.path.exists(dest):
    os.makedirs(dest)
labels='./output/output_labels.txt'
predictions_csv="./output/guoming_val.csv"
graph='./output/intermediate_graph/intermediate_30000.pb'
input_layer='DecodeJpeg/contents:0'
input_layer='input:0'

output_layer='final_result:0'
num_top_predictions=3
labels = load_labels(labels)
print("labels")
print(labels)
load_graph(graph)
run_graph(src,dest,labels,input_layer,output_layer,num_top_predictions,val_list,predictions_csv)


# oracle_helpers

from typing import List

from primitives.resources import Function
from primitives.actions import Action

from .simple_functions import requires

def build_oracle_requirements(oracle) -> List[Action]:
    requirements = []
    # Require interface exist
    requirements.append(
        Action(
            identifier=f'requires_{oracle.interface.identifier}',
            actor=oracle,
            resources=[oracle.interface],
            function = Function(
                identifier=f'requires_{oracle.interface.identifier}',
                kwargs={'resource_a': oracle, 'resource_b': oracle.interface},
                fn=requires,
            )
        )
    )
    return requirements


#!/usr/bin/env python3
import tkinter as tk
from tkinter import ttk
from tkinter import messagebox as mb
import json
import re

with open('karttunen.json', 'r') as k:
    karttunen = json.load(k)

print('Nr. entries in Karttunen:', len(karttunen), '\n==============================')

class Application(tk.Frame):
    def __init__(self, master=None):
        super().__init__(master)
        self.master = master
        self.master.title("E-Karttunen")
        self.master.geometry('1000x700')
       # self.pack()
       # self.grid(sticky='nsew')
        self.place()

        self.create_widgets()

    def create_widgets(self):
        self.first = tk.Label(self.master,
                                width=115,
                                height=40,
                                bg='black')
        self.first.place(relx=.5, rely=.5, anchor='c')

        ###################
        ##
        ##   TOP BAR
        ##
        ###################
        self.top_bar = tk.Label(self.first,
                                height=3,
                                width=105,
                                bg='grey',
                              #  text='TOP BAR',
                                anchor='n')
        self.top_bar.place(relx=.5, rely=.01, anchor='n')

        self.search_lab = tk.Label(self.top_bar,
                                height=1,
                                width=39,
                                bg='grey',
                                text='Search String:',
                                padx=1,
                                anchor='c')
        self.search_lab.place(relx=.195, rely=.1, anchor='n')

        self.search_entry = tk.Entry(self.top_bar,
                                width=39)
                                #bg='pink')
        self.search_entry.place(relx=.195, rely=.5, anchor='n')

        self.search = tk.Button(self.top_bar,
                                width=13,
                                text="Search",
                                command=self.init_search)
        self.search.place(relx=.485, rely=.35, anchor='n')

        self.clear = tk.Button(self.top_bar,
                                width=13,
                                text="Clear Search",
                                command=self.clear_search)
        self.clear.place(relx=.65, rely=.35, anchor='n')

        self.search_target = tk.StringVar()
        self.search_in = ttk.Combobox(self.top_bar,
                                width=13,
                                textvariable=self.search_target)
        self.search_in["values"] = ('Nahuatl', 'English', 'Spanish')
        self.search_in.current(0)
        self.search_in.place(relx=.92, rely=.1, anchor='n')

        self.search_in_lab = tk.Label(self.top_bar,
                                text="Searching in:",
                                bg='grey')
        self.search_in_lab.place(relx=.795, rely=.1, anchor='n')

        self.search_type = tk.StringVar()
        self.search_like = ttk.Combobox(self.top_bar,
                                width=13,
                                textvariable=self.search_type)
        self.search_like["values"] = ('Substring','Exact Match')
        self.search_like.current(0)
        self.search_like.place(relx=.92, rely=.5, anchor='n')

        self.search_like_lab = tk.Label(self.top_bar,
                                text="Search type:",
                                bg='grey')
        self.search_like_lab.place(relx=.796, rely=.5, anchor='n')

        ###################
        ##
        ##   LEFT COLUMN
        ##
        ###################
        self.left_col = tk.Label(self.first,
                                width=40,
                                height=35,
                                bg='#282828')
        self.left_col.place(relx=.217, rely=.1, anchor='n')

        self.result_hws = tk.Listbox(self.left_col,
                            #    width=39,
                            #    height=33,
                                bg='grey')
        self.result_hws.place(relheight=1, relwidth=1)#relx=.5, rely=.5, anchor='c')
        self.result_hws.bind('<<ListboxSelect>>', self.show_hw_res)

        self.L_scrollbar = tk.Scrollbar(self.result_hws,
                                orient='vertical')
        self.L_scrollbar.place(relx=.97, relheight=1)

        ###################
        ##
        ##   RIGHT COLUMN
        ##
        ###################
        self.right_col = tk.Canvas(self.first,
                                width=500,#62,
                                height=596,#35,
                               # bg='yellow',
                                bg='#282828') #,
  #                              text='RIGHT COL')
        self.right_col.place(relx=.687, rely=.1, anchor='n')

        self.hw_details = tk.Canvas(self.right_col,
                            #    width=39,
                            #    height=20,
                                bg='grey')
        self.hw_details.place(relheight=1, relwidth=1)#.975)

       # self.R_scrollbar = tk.Scrollbar(self.right_col,
       #                         orient='vertical')
       # self.R_scrollbar.place(relx=.975, relheight=1)

        ###################
        ##
        ##   FUNCTIONS
        ##
        ###################

    def init_search(self):
        self.search_term = self.search_entry.get().lower()
        self.result_hws.delete(0,'end')
        if len(self.search_term) > 0 :
            print('You searched for:', self.search_term)

            # IF NAHUATL
            self.search_target = self.search_in.get()
            self.search_in.current(self.search_in['values'].index(self.search_target))
            if self.search_target == 'Nahuatl':
                self.search_type = self.search_like.get()
                self.search_like.current(self.search_like['values'].index(self.search_type))
                print("Searching in: Nahuatl")

                print('Search Type:', self.search_type)
                self.L_scrollbar.config(command=self.result_hws.yview)
                self.result_hws.config(yscrollcommand=self.L_scrollbar.set)

                if self.search_type == 'Substring':
                    self.regex_term = re.compile('.*(%s).*'%self.search_term).match
                    self.sub_dict = {k:v for k,v in karttunen.items() if self.regex_term(k)}
                    print('Hits:', len(self.sub_dict))
                    for k,v in self.sub_dict.items():
                        self.result_hws.insert('end', k)

                if self.search_type == 'Exact Match':
                    if self.search_term in karttunen:
                        self.result_hws.insert('end', self.search_term)
                    mb.showinfo(title="Search Tip", message="Karttunen's headwords are sometimes broken by hyphens to indicate morphology (e.g., and esp. absolutive endings on nouns) and parentheses to indicate optional / variation in phonemes. If these 'special characters' are not included in the search, but appear in the entry, there will be no hit.")
            else:
                mb.showerror(title='Error', message="English and Spanish don't work yet. Search in Nahuatl.")
        else:
            print('Oi! Enter a search term, mate.')
            mb.showerror(title="Hold up!", message="Lookie here mang... \nYou need to enter a search term if you expect to find anything.")

    def show_hw_res(self, evt):
        self.selected_hw = self.result_hws.get(self.result_hws.curselection())
        print(karttunen[self.selected_hw])

        slaves = self.hw_details.pack_slaves()
        for slave in slaves:
            slave.destroy()

     #   self.R_scrollbar.config(command=self.hw_details.yview)
      #  self.hw_details.config(yscrollcommand=self.R_scrollbar.set)

        self.hw_self = tk.Label(self.hw_details,
                            text=self.selected_hw,
                            bg='grey',
                            pady=10,
                            font=('times', 24))
        self.hw_self.pack()

        if 'Grammar' in karttunen[self.selected_hw]:
            self.hw_grammar_lab = tk.Label(self.hw_details,
                                    text = "Grammar information:",
                                    height=2,
                                    padx=10, pady=5,
                                    anchor='sw',
                                    bg='grey',
                                    font=('times', 10))
            self.hw_grammar_lab.pack(fill='x')
            self.hw_grammar = tk.Label(self.hw_details,
                                    text = karttunen[self.selected_hw]['Grammar'],
                                    padx=10,
                                    anchor='w',
                                    wraplength=475, justify='left',
                                    bg='grey',
                                    font=('times', 14))
            self.hw_grammar.pack(fill='x')

        if 'English' in karttunen[self.selected_hw]:
            self.hw_english_lab = tk.Label(self.hw_details,
                                    text = "English:",
                                    height=2,
                                    padx=10, pady=5,
                                    anchor='sw',
                                    bg='grey',
                                    font=('times', 10))
            self.hw_english_lab.pack(fill='x')
            self.hw_english = tk.Label(self.hw_details,
                                    text = karttunen[self.selected_hw]['English'],
                                    padx=10,
                                    anchor='w',
                                    wraplength=475, justify='left',
                                    bg='grey',
                                    font=('times', 14))
            self.hw_english.pack(fill='x')

        if 'Spanish' in karttunen[self.selected_hw]:
            self.hw_spanish_lab = tk.Label(self.hw_details,
                                    text = "Spanish:",
                                    height=2,
                                    padx=10, pady=5,
                                    anchor='sw',
                                    bg='grey',
                                    font=('times', 10))
            self.hw_spanish_lab.pack(fill='x')
            self.hw_spanish = tk.Label(self.hw_details,
                                    text = karttunen[self.selected_hw]['Spanish'],
                                    padx=10,
                                    anchor='w',
                                    wraplength=475, justify='left',
                                    bg='grey',
                                    font=('times', 14))
            self.hw_spanish.pack(fill='x')

        if 'Comment' in karttunen[self.selected_hw]:
            self.hw_comment_lab = tk.Label(self.hw_details,
                                    text = "Comment:",
                                    height=2,
                                    padx=10, pady=5,
                                    anchor='sw',
                                    bg='grey',
                                    font=('times', 10))
            self.hw_comment_lab.pack(fill='x')
            self.hw_comment = tk.Label(self.hw_details,
                                    text = karttunen[self.selected_hw]['Comment'],
                                    padx=10,
                                    anchor='w',
                                    wraplength=475, justify='left',
                                    bg='grey',
                                    font=('times', 14))
            self.hw_comment.pack(fill='x')

        if 'Lexical Reference' in karttunen[self.selected_hw]:
            self.hw_lexref_lab = tk.Label(self.hw_details,
                                    text = "See also:",
                                    height=2,
                                    padx=10, pady=5,
                                    anchor='sw',
                                    bg='grey',
                                    font=('times', 10))
            self.hw_lexref_lab.pack(fill='x')
            self.hw_lexref = tk.Label(self.hw_details,
                                    text = karttunen[self.selected_hw]['Lexical Reference'],
                                    padx=10,
                                    anchor='w',
                                    wraplength=475, justify='left',
                                    bg='grey',
                                    font=('times', 14))
            self.hw_lexref.pack(fill='x')

        if "Source" in karttunen[self.selected_hw]:
            self.hw_src_lab = tk.Label(self.hw_details,
                                    text = "Karttunen's source:",
                                    height=2,
                                    padx=10, pady=5,
                                    anchor='sw',
                                    bg='grey',
                                    font=('times', 10))
            self.hw_src_lab.pack(fill='x')
            self.hw_src = tk.Label(self.hw_details,
                                    text = karttunen[self.selected_hw]['Source'],
                                    padx=10,
                                    anchor='w',
                                    wraplength=475, justify='left',
                                    bg='grey',
                                    font=('times', 14))
            self.hw_src.pack(fill='x')

    def clear_search(self):
        self.search_entry.delete(0, 'end')
        self.result_hws.delete(0,'end')
        self.search_like.current(self.search_like['values'].index(self.search_type))
        slaves = self.hw_details.pack_slaves()
        for slave in slaves:
            slave.destroy()

def main():
    root = tk.Tk()
    app = Application(master=root)
    app.mainloop()

if __name__ == '__main__':
	main()


import argparse
import csv
import re

from openpyxl import load_workbook

parser = argparse.ArgumentParser(
    description='Converts the first sheet of an .xlsx-file to a .csv-file'
)
parser.add_argument('xlsxfile', help='The .xlsx file to be converted')
parser.add_argument('-o,--output', dest='output', help='Name of the output file')
parser.add_argument(
    '-s,--sheet',
    dest='sheet',
    help='Name of sheet to convert to .xlsx. By default the first sheet will be converted',
)

def main():
    args = parser.parse_args()

    csv_rows = []

    wb = load_workbook(args.xlsxfile)

    sheet_index = 0
    if args.sheet:
        assert args.sheet in wb.sheetnames
        sheet_index = wb.sheetnames.index(args.sheet)

    ws = None
    for sheet in wb:
        if args.sheet:
            if sheet.title == args.sheet:
                ws = sheet
                break
        else:
            ws = sheet

    if ws is None:
        print(f'Sheet with name "{args.name}" not found')
        exit(1)

    for xls_row in ws:
        csv_row = []
        for xls_cell in xls_row:
            csv_row.append(xls_cell.value)
        csv_rows.append(csv_row)

    if args.output:
        output_filename = args.output
    else:
        output_filename = re.sub(r'\.xlsx$', '.csv', args.xlsxfile)
    with open(output_filename, 'w') as output_file:
        writer = csv.writer(output_file)
        writer.writerows(csv_rows)


import json

import requests

from automatic_walk_time_tables.utils import path

def fetch_map_numbers(path_: path.Path) -> str:
    url = "http://awt-swiss-tml-api:1848/map_numbers"

    # Convert path to LV95 and save the coordinates in an array
    coordinates = [pkt.point.to_LV95() for pkt in path_.way_points]
    coordinates = [[pkt.lat, pkt.lon] for pkt in coordinates]

    payload = json.dumps(coordinates)
    headers = {'Content-Type': 'application/json'}
    req = requests.request("GET", url, headers=headers, data=payload)

    return req.text


import time
import sys
import logging
from common.common import Coords, ClicPoint, DragAndDropsRecorder

class ForgeStrategy:
    def __init__(self, first_runes: ClicPoint, second_runes: ClicPoint, n_first_runes: int, n_second_runes: int, dry_run: bool):
        self.first_runes: ClicPoint = first_runes
        self.second_runes: ClicPoint = second_runes
        self.n_first_runes = n_first_runes
        self.n_second_runes = n_second_runes
        self.dry_run = dry_run

    def use_runes(self, runes: ClicPoint, times: int):
        if self.dry_run:
            runes.point_at()
        else:
            for _ in range(times):
                runes.clic()

    def forge(self):
        while True:
            self.use_runes(self.first_runes, self.n_first_runes)
            self.use_runes(self.second_runes, self.n_second_runes)

if __name__ == "__main__":
    n_first_runes: int = int(sys.argv[1])
    n_second_runes: int = int(sys.argv[2])
    dry_run = "--dry-run" in sys.argv

    ClicPoint.actions_delay_seconds = 0.3
    dd_recorder = DragAndDropsRecorder()
    dd_recorder.start()
    time.sleep(6)
    dd_recorder.stop()

    forge_strat = ForgeStrategy(
        ClicPoint(Coords(*dd_recorder.clicks[-2])),
        ClicPoint(Coords(*dd_recorder.clicks[-1])),
        n_first_runes,
        n_second_runes,
        dry_run
    )

    logging.getLogger().setLevel("INFO")
    logging.info(f"Entering forge with last vector={dd_recorder.last_vector().tuple}, dry_run= {dry_run}")
    forge_strat.forge()


# Import dependencies
from bs4 import BeautifulSoup as bs
from splinter import Browser
import pandas as pd
from webdriver_manager.chrome import ChromeDriverManager

def scrape_all():
    browser = Browser('chrome', **executable_path, headless=False)
    news_title, news_paragraph = mars_news(browser)
    data = {
        "news_title": news_title,
        "news_paragraph": news_paragraph,
        "featured_image": featured_image(browser),
        "facts": mars_facts(),
        "last_modified": dt.datetime.now()
    }
    browser.quit()
    return data

def scrape():
    executable_path = {'executable_path': ChromeDriverManager().install()}
    browser = Browser('chrome', **executable_path, headless=False)
    mars={}

    # Get the URL for NASA latest news
    url = 'https://redplanetscience.com/'
    # Open the URL
    browser.visit(url)

    # Create the html
    html = browser.html
    soup = bs(html, 'html.parser')

    # Get the latest news
    news_latest = soup.find("div", class_="content_title").text
    news_latest_p = soup.find("div", class_="article_teaser_body").text
    mars["news_latest"]=news_latest
    mars["news_latest_p"]=news_latest_p

    # Get the images in the JPL Mars Space Image - Featured Image
    url = 'https://spaceimages-mars.com/'
    browser.visit(url)
    browser.find_by_tag("button")[1].click()

    html = browser.html
    jpl_image = bs(html, "html.parser")
    image = jpl_image.find('img', class_="fancybox-image").get('src')
    # image = jpl_image.find('img', class_="fancybox-image")
    feature_image = "https://spaceimages-mars.com/"+image
    mars["featured_image_url"]=feature_image

    # Mars facts
    url = "https://galaxyfacts-mars.com/"
    mars_df = pd.read_html(url)
    mars_df=mars_df[0]

    mars_df.columns=["Description", "Mars", "Earth"]
    mars_df.set_index("Description", inplace=True)
    mars_df
    mars_df_html = mars_df.to_html()
    mars_df_html
    mars["facts"]=mars_df_html

    # Mars hemisphere
    url = "https://marshemispheres.com/"
    browser.visit(url)
    result = browser.find_by_css("a.product-item img")
    hemisphere_img_url = []
    for i in range(len(result)):
        hemisphere = {}
        browser.find_by_css("a.product-item img")[i].click()
        element = browser.links.find_by_text('Sample').first
        img_url = element["href"]
        hemisphere["img_url"]=img_url
        hemisphere["title"]=browser.find_by_css("h2.title").text
        hemisphere_img_url.append(hemisphere)
        browser.back()
    hemisphere_img_url

    return mars

if __name__ == "__main__":
        print(scrape())


import sys

from PySide6.QtWidgets import QApplication, QWidget, QPushButton, QVBoxLayout, QLabel
from PySide6.QtGui import QIcon

class MainWindow(QWidget):
    def __init__(self, parent=None):
        super().__init__(parent)

        self.setWindowTitle('UI Tutorial')
        self.setGeometry(450, 250, 400, 350)
        self.setWindowIcon(QIcon(r"C:\WORK\2023_februar_ui\icon\test.png"))
        # vertikális elrendezés
        vbox = QVBoxLayout()

        # signal kezelés -> figyelni kell azokat a jeleket, amelyekkel a műveletet akarom végezni
        btn = QPushButton('Change text')
        self.label = QLabel('Hello szia')

        vbox.addWidget(btn)
        vbox.addWidget(self.label)

        self.setLayout(vbox)

        btn.clicked.connect(self.change_text)

    def change_text(self):
        import random
        colors = {
            1: 'purple',
            2: "green",
            3: 'red'
        }
        self.label.setText(f"I changed this: {random.randint(10, 20)}")
        self.label.setStyleSheet(f"color: {colors[random.randint(1, 3)]}")

if __name__ == '__main__':
    app = QApplication([])
    window = MainWindow()
    window.show()
    sys.exit(app.exec())


import os
import sys
#下面三行代码可以解决没有模块的报错
curPath = os.path.abspath(os.path.dirname(__file__))
rootPath = os.path.split(curPath)[0]
sys.path.append(rootPath)
import  unittest
import  logging
from test_case.read_API import readData
import  logging.config

from  test_request.test_re import requ
path11=os.path.dirname(os.path.dirname(__file__))
logging.config.fileConfig(path11+'/test_log/log.conf')
logging = logging.getLogger()

class test_weater(unittest.TestCase):

    @classmethod
    def setUpClass(cls):
        logging.info('Start')

    @classmethod
    def tearDownClass(cls):
        logging.info('End')

    def setUp(self):
        self.data = readData(r'D:\soft\github1\web_api\test_excel\API.xls')
        self.ex = self.data.excelApi()
        #print(self.ex)

    def test_a(self):
        #print(self.ex[0]['result']['status'],type(self.ex[0]['result']['status']))
        if self.ex[0]['result']['status']==1000:
            tag=True
        self.assertTrue(tag)

    def test_b(self):
        if self.ex[1]['result']['status']==1002:
            tag=True
        self.assertTrue(tag)

if __name__ == '__main__':
    suite=unittest.TestSuite()
    tests=[test_weater('test_a'),test_weater('test_b')]
    suite.addTests(tests)
    runner=unittest.TextTestRunner(verbosity=2)
    runner.run(suite)


#!/usr/bin/env python3

import os
import logging
import traceback
import inspect
import datetime
import functools

logs_dir_name = 'logs'
__dir__ = os.path.dirname(os.path.abspath(__file__))

if logs_dir_name not in os.listdir(os.path.join(__dir__, '../')):
    os.mkdir(''.join((os.path.join(__dir__, '../'), logs_dir_name)))

_now = datetime.datetime.now().strftime('%Y-%m-%d_%H_%M_%S')
_logs_dir_path = os.path.join(__dir__, ''.join(('../', logs_dir_name, '/')))
log_fp = ''.join((_logs_dir_path, _now, ".log"))

logging.basicConfig(level=logging.DEBUG,
                    format='%(asctime)s %(filename)s[line:%(lineno)d] %(levelname)s %(message)s',
                    datefmt='%a, %d %b %Y %H:%M:%S',
                    filename=log_fp,
                    filemode='w')

_console = logging.StreamHandler()
_console.setLevel(logging.INFO)
_formatter = logging.Formatter('%(name)-12s: %(levelname)-8s %(message)s')
_console.setFormatter(_formatter)
LOGGER = logging.getLogger('czb test')
LOGGER.addHandler(_console)

def logged(func):
    """创建一个日志装饰器，它会记录所装饰函数的入参和
    """
    result = None

    @functools.wraps(func)
    def inner(*args, **kwargs):

        try:
            nonlocal result
            result = func(*args, **kwargs)
            LOGGER.info('模块:{}\n 调用函数 {} 传入参数: {},{}\n 返回结果: {}'
                        .format(inspect.getmodule(func), func.__name__,
                                args, kwargs, result))
            return result
        except Exception as Ex:
            LOGGER.error("{}方法入参:{},{}".format(func.__name__, args, kwargs))
            e = traceback.format_exc()
            LOGGER.error('Exception：{}'.format(e))
            raise Ex

    return inner


import cv2
from PIL import Image
import numpy as np
from sklearn import preprocessing
import matplotlib.pyplot as plt

class PCAWhitening:
    def __init__(self, epsilon=1E-6):
        self.epsilon = epsilon
        # self.mean = None
        self.eigenvalue = None
        self.eigenvector = None
        self.pca = None

    def fit(self, x):
        print("Fit IN")
        print("Fit self : " + str(self))
        print("Fit x : " + str(x))
        """"
        self.mean = np.mean(x, axis=0) #列に沿って平均化
        # CIFA-10の場合は画像を複数取り込む
        各列の平均を元から引く
        print("Fit self.mean " + str(self.mean))
        x_ = x - self.mean  # 平均値で引く
        """
        x_ = x
        print("Fit OVR 2" + "shape:¥n" + str(x_.shape) + str(x_))

        cov = np.dot(x_.T, x_)  # 転置と元の積
        print("Fit OVR 3" + "shape:¥n" + str(cov.shape) + str(cov))

        E, D, _ = np.linalg.svd(cov)  # 特異値分解
        print("Fit OVR 4 E: " + str(E))
        print(",D : " + str(D))

        D = np.sqrt(D) + self.epsilon
        self.eigenvalue = D
        self.eigenvector = E
        print("Fit OVR 5")

        self.pca = np.dot(np.diag(1. / D), E.T)
        print("Fit END")
        return self

    def transform(self, x):
        # x_ = x - self.mean
        x_ = x
        return np.dot(x_, self.pca.T)

def normalizeMinMax(x, axis=0, epsilon=1E-5):
    vmin = np.min(x, axis)
    vmax = np.max(x, axis)
    return (x - vmin) / (vmax - vmin + epsilon)

def normalizeImage(x):
    img = x.reshape(x.shape[0] * x.shape[1], x.shape[2])
    img = normalizeMinMax(img, axis=0)
    return img.reshape(x.shape)

def normalizeImage2(x, epsilon=1E-6):
    vmin = np.min(x)
    vmax = np.max(x)
    return (x - vmin) / (vmax - vmin + epsilon)

if __name__ == '__main__':

    # 画像の読み込み
    img = cv2.imread("cat9.png", 1)  # ndarray形式?
    # print(img)

    # 基本表示
    # cv2.imshow('frame',img )
    print("shape 0 :" + str(img.shape[0]))  # たて
    print("shape 1 :" + str(img.shape[1]))  # 横
    print("shape 2 :" + str(img.shape[2]))  # 3がでる　RGBってことか

    r=(img[:,:,0].reshape(1,-1))
    g=(img[:, :, 1].reshape(1, -1))
    b=(img[:, :, 2].reshape(1, -1))
    print(r)
    print(g)
    print(b)

    rgb = np.append(r, g, axis=0)
    rgb = np.append(rgb, b, axis=0)
    print(rgb)
    # print(img_pcaw)


import os
import argparse
import numpy as np
import pandas as pd
from env import custor_Env

parser = argparse.ArgumentParser()
parser.add_argument("--order_path", default="data/order.csv")
parser.add_argument("--stock_path", default="data/stock.csv")
args = parser.parse_args()

env = custor_Env(args)

# hyper-parameters for RL training
N_epiosde = 1

if __name__ == "__main__":

    for eps in range(N_epiosde):
        state = env.reset()

        for st in range(env.max_steps):
            action = np.random.randint(10, size=(24, 8))
            state, reward, done, info = env.step(action)
            # print(state)
            print(reward)
            # print(f"submission : {info['submission']} ")


import pandas as pd
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import LabelBinarizer
import keras as K
import numpy as np
import matplotlib.pyplot as plt

def load_data(excel_file_path):
    classification = pd.read_excel(excel_file_path, sheet_name=0, header=0)
    target_var = "label"
    features = list(classification.columns)
    features.remove(target_var)
    class_type = classification[target_var].unique()
    class_dict = dict(zip(class_type, range(len(class_type))))
    classification['target'] = classification[target_var].apply(lambda x: class_dict[x])
    lb = LabelBinarizer()
    lb.fit(list(class_dict.values()))
    transformed_labers = lb.transform(classification['target'])
    y_bin_labels = []
    for i in range(transformed_labers.shape[1]):
        y_bin_labels.append('y' + str(i))
        classification['y' + str(i)] = transformed_labers[:, i]

    train_x, test_x, train_y, test_y = train_test_split(classification[features], classification[y_bin_labels],
                                                        train_size=0.7, test_size=0.3, random_state=0)
    return train_x, test_x, train_y, test_y, class_dict

train_x, test_x, train_y, test_y, class_dict = load_data(r'C:\Users\Asher\Desktop\self_\SVM_three_data.xlsx')

init = K.initializers.glorot_uniform(seed=1)
simple_adam = K.optimizers.Adam()
model = K.models.Sequential()
model.add(K.layers.Dense(units=5, input_dim=3, kernel_initializer=init, activation='relu'))
model.add(K.layers.Dense(units=6, kernel_initializer=init, activation="relu"))
model.add(K.layers.Dense(units=3,kernel_initializer=init, activation="softmax"))
model.compile(loss="categorical_crossentropy", optimizer=simple_adam, metrics=["accuracy"])

batch_size = 1
epochs = 1
print("startring training")
history = model.fit(train_x, train_y, batch_size=batch_size, epochs=epochs, shuffle=True, verbose=1,
                    validation_data=(test_x, test_y))
print("training finished \n")

eval = model.evaluate(test_x, test_y, verbose=0)
unknown = np.array([[0.6024, 0.03579, 0.7795]], dtype=np.float32)
predicted = model.predict(unknown)
print("predicted softmax vector is:", predicted)
specied_dict = {v:k for k,v in class_dict.items()}
print("predicted species is:", specied_dict[np.argmax(predicted)])


import marshmallow
from flask import Blueprint
from flask import render_template, request, url_for, redirect

from app.iris_engine.utils.tracker import track_activity
from app.models.models import AssetsType, CaseAssets
from app.forms import AddAssetForm
from app import db
from app.schema.marshables import AssetSchema

from app.util import response_success, response_error, login_required, admin_required, api_admin_required, \
    api_login_required

manage_assets_blueprint = Blueprint('manage_assets',
                                    __name__,
                                    template_folder='templates')

@manage_assets_blueprint.route('/manage/asset-type/list')
@api_login_required
def list_assets(caseid):
    # Get all assets
    assets = AssetsType.query.with_entities(
        AssetsType.asset_name,
        AssetsType.asset_description,
        AssetsType.asset_id
    ).all()

    data = [row._asdict() for row in assets]

    # Return the assets
    return response_success("", data=data)

@manage_assets_blueprint.route('/manage/asset-type/<int:cur_id>', methods=['GET'])
@api_login_required
def view_asset_api(cur_id, caseid):
    # Get all assets
    asset_type = AssetsType.query.with_entities(
        AssetsType.asset_name,
        AssetsType.asset_description,
        AssetsType.asset_id
    ).filter(
        AssetsType.asset_id == cur_id
    ).first()

    if not asset_type:
        return response_error(f'Invalid asset type ID {cur_id}')

    # Return the assets
    return response_success("", data=asset_type._asdict())

@manage_assets_blueprint.route('/manage/asset-type/update/<int:cur_id>/modal', methods=['GET'])
@login_required
def view_assets_modal(cur_id, caseid, url_redir):
    if url_redir:
        return redirect(url_for('manage_assets.manage_assets', cid=caseid))

    form = AddAssetForm()
    asset = AssetsType.query.filter(AssetsType.asset_id == cur_id).first()
    if not asset:
        return response_error("Invalid asset type ID")

    form.asset_name.render_kw = {'value': asset.asset_name}
    form.asset_description.render_kw = {'value': asset.asset_description}

    return render_template("modal_add_asset_type.html", form=form, assettype=asset)

@manage_assets_blueprint.route('/manage/asset-type/update/<int:cur_id>', methods=['POST'])
@api_admin_required
def view_assets(cur_id, caseid):
    if not request.is_json:
        return response_error("Invalid request")

    asset = AssetsType.query.filter(AssetsType.asset_id == cur_id).first()
    if not asset:
        return response_error("Invalid asset type ID")

    asset_schema = AssetSchema()

    try:

        asset_sc = asset_schema.load(request.get_json(), instance=asset)

        if asset_sc:
            track_activity("updated asset type {}".format(asset_sc.asset_name), caseid=caseid)
            return response_success("Asset type updated", asset_sc)

    except marshmallow.exceptions.ValidationError as e:
        return response_error(msg="Data error", data=e.messages, status=400)

    return response_success("Unexpected error server-side. Nothing updated", data=asset)

@manage_assets_blueprint.route('/manage/asset-type/add/modal', methods=['GET'])
@admin_required
def add_assets_modal(caseid, url_redir):
    if url_redir:
        return redirect(url_for('manage_assets.manage_assets', cid=caseid))
    form = AddAssetForm()

    return render_template("modal_add_asset_type.html", form=form, assettype=None)

@manage_assets_blueprint.route('/manage/asset-type/add', methods=['POST'])
@api_admin_required
def add_assets(caseid):
    if not request.is_json:
        return response_error("Invalid request")

    asset_schema = AssetSchema()

    try:

        asset_sc = asset_schema.load(request.get_json())
        db.session.add(asset_sc)
        db.session.commit()

    except marshmallow.exceptions.ValidationError as e:
        return response_error(msg="Data error", data=e.messages, status=400)

    track_activity("Added asset type {asset_name}".format(asset_name=asset_sc.asset_name), caseid=caseid, ctx_less=True)
    # Return the assets
    return response_success("Added successfully", data=asset_sc)

@manage_assets_blueprint.route('/manage/asset-type/delete/<int:cur_id>', methods=['GET'])
@api_admin_required
def delete_assets(cur_id, caseid):
    asset = AssetsType.query.filter(AssetsType.asset_id == cur_id).first()
    if not asset:
        return response_error("Invalid asset ID")

    case_linked = CaseAssets.query.filter(CaseAssets.asset_type_id == cur_id).first()
    if case_linked:
        return response_error("Cannot delete a referenced asset type. Please delete any assets of this type first.")

    db.session.delete(asset)
    track_activity("Deleted asset type ID {asset_id}".format(asset_id=cur_id), caseid=caseid, ctx_less=True)

    return response_success("Deleted asset type ID {cur_id} successfully".format(cur_id=cur_id))


import torch
import numpy as np
from torch.nn import functional as F

def batch_rodrigues(rot_vecs, epsilon=1e-8, dtype=torch.float32):
    """
    Taken from https://github.com/mkocabas/VIBE/blob/master/lib/utils/geometry.py
    Calculates the rotation matrices for a batch of rotation vectors
    - param rot_vecs: torch.tensor (N, 3) array of N axis-angle vectors
    - returns R: torch.tensor (N, 3, 3) rotation matrices
    """
    batch_size = rot_vecs.shape[0]
    device = rot_vecs.device

    angle = torch.norm(rot_vecs + 1e-8, dim=1, keepdim=True)
    rot_dir = rot_vecs / angle

    cos = torch.unsqueeze(torch.cos(angle), dim=1)
    sin = torch.unsqueeze(torch.sin(angle), dim=1)

    # Bx1 arrays
    rx, ry, rz = torch.split(rot_dir, 1, dim=1)
    K = torch.zeros((batch_size, 3, 3), dtype=dtype, device=device)

    zeros = torch.zeros((batch_size, 1), dtype=dtype, device=device)
    K = torch.cat([zeros, -rz, ry, rz, zeros, -rx, -ry, rx, zeros], dim=1).view(
        (batch_size, 3, 3)
    )

    ident = torch.eye(3, dtype=dtype, device=device).unsqueeze(dim=0)
    rot_mat = ident + sin * K + (1 - cos) * torch.bmm(K, K)
    return rot_mat

def quaternion_mul(q0, q1):
    """
    EXPECTS WXYZ
    :param q0 (*, 4)
    :param q1 (*, 4)
    """
    r0, r1 = q0[..., :1], q1[..., :1]
    v0, v1 = q0[..., 1:], q1[..., 1:]
    r = r0 * r1 - (v0 * v1).sum(dim=-1, keepdim=True)
    v = r0 * v1 + r1 * v0 + torch.linalg.cross(v0, v1)
    return torch.cat([r, v], dim=-1)

def quaternion_inverse(q, eps=1e-8):
    """
    EXPECTS WXYZ
    :param q (*, 4)
    """
    conj = torch.cat([q[..., :1], -q[..., 1:]], dim=-1)
    mag = torch.square(q).sum(dim=-1, keepdim=True) + eps
    return conj / mag

def quaternion_slerp(t, q0, q1, eps=1e-8):
    """
    :param t (*, 1)  must be between 0 and 1
    :param q0 (*, 4)
    :param q1 (*, 4)
    """
    dims = q0.shape[:-1]
    t = t.view(*dims, 1)

    q0 = F.normalize(q0, p=2, dim=-1)
    q1 = F.normalize(q1, p=2, dim=-1)
    dot = (q0 * q1).sum(dim=-1, keepdim=True)

    # make sure we give the shortest rotation path (< 180d)
    neg = dot < 0
    q1 = torch.where(neg, -q1, q1)
    dot = torch.where(neg, -dot, dot)
    angle = torch.acos(dot)

    # if angle is too small, just do linear interpolation
    collin = torch.abs(dot) > 1 - eps
    fac = 1 / torch.sin(angle)
    w0 = torch.where(collin, 1 - t, torch.sin((1 - t) * angle) * fac)
    w1 = torch.where(collin, t, torch.sin(t * angle) * fac)
    slerp = q0 * w0 + q1 * w1
    return slerp

def rotation_matrix_to_angle_axis(rotation_matrix):
    """
    This function is borrowed from https://github.com/kornia/kornia

    Convert rotation matrix to Rodrigues vector
    """
    quaternion = rotation_matrix_to_quaternion(rotation_matrix)
    aa = quaternion_to_angle_axis(quaternion)
    aa[torch.isnan(aa)] = 0.0
    return aa

def quaternion_to_angle_axis(quaternion):
    """
    This function is borrowed from https://github.com/kornia/kornia

    Convert quaternion vector to angle axis of rotation.
    Adapted from ceres C++ library: ceres-solver/include/ceres/rotation.h

    :param quaternion (*, 4) expects WXYZ
    :returns angle_axis (*, 3)
    """
    # unpack input and compute conversion
    q1 = quaternion[..., 1]
    q2 = quaternion[..., 2]
    q3 = quaternion[..., 3]
    sin_squared_theta = q1 * q1 + q2 * q2 + q3 * q3

    sin_theta = torch.sqrt(sin_squared_theta)
    cos_theta = quaternion[..., 0]
    two_theta = 2.0 * torch.where(
        cos_theta < 0.0,
        torch.atan2(-sin_theta, -cos_theta),
        torch.atan2(sin_theta, cos_theta),
    )

    k_pos = two_theta / sin_theta
    k_neg = 2.0 * torch.ones_like(sin_theta)
    k = torch.where(sin_squared_theta > 0.0, k_pos, k_neg)

    angle_axis = torch.zeros_like(quaternion)[..., :3]
    angle_axis[..., 0] += q1 * k
    angle_axis[..., 1] += q2 * k
    angle_axis[..., 2] += q3 * k
    return angle_axis

def quaternion_to_rotation_matrix(quaternion):
    """
    Convert a quaternion to a rotation matrix.
    Taken from https://github.com/kornia/kornia, based on
    https://github.com/matthew-brett/transforms3d/blob/8965c48401d9e8e66b6a8c37c65f2fc200a076fa/transforms3d/quaternions.py#L101
    https://github.com/tensorflow/graphics/blob/master/tensorflow_graphics/geometry/transformation/rotation_matrix_3d.py#L247
    :param quaternion (N, 4) expects WXYZ order
    returns rotation matrix (N, 3, 3)
    """
    # normalize the input quaternion
    quaternion_norm = F.normalize(quaternion, p=2, dim=-1, eps=1e-12)
    *dims, _ = quaternion_norm.shape

    # unpack the normalized quaternion components
    w, x, y, z = torch.chunk(quaternion_norm, chunks=4, dim=-1)

    # compute the actual conversion
    tx = 2.0 * x
    ty = 2.0 * y
    tz = 2.0 * z
    twx = tx * w
    twy = ty * w
    twz = tz * w
    txx = tx * x
    txy = ty * x
    txz = tz * x
    tyy = ty * y
    tyz = tz * y
    tzz = tz * z
    one = torch.tensor(1.0)

    matrix = torch.stack(
        (
            one - (tyy + tzz),
            txy - twz,
            txz + twy,
            txy + twz,
            one - (txx + tzz),
            tyz - twx,
            txz - twy,
            tyz + twx,
            one - (txx + tyy),
        ),
        dim=-1,
    ).view(*dims, 3, 3)
    return matrix

def angle_axis_to_quaternion(angle_axis):
    """
    This function is borrowed from https://github.com/kornia/kornia
    Convert angle axis to quaternion in WXYZ order
    :param angle_axis (*, 3)
    :returns quaternion (*, 4) WXYZ order
    """
    theta_sq = torch.sum(angle_axis**2, dim=-1, keepdim=True)  # (*, 1)
    # need to handle the zero rotation case
    valid = theta_sq > 0
    theta = torch.sqrt(theta_sq)
    half_theta = 0.5 * theta
    ones = torch.ones_like(half_theta)
    # fill zero with the limit of sin ax / x -> a
    k = torch.where(valid, torch.sin(half_theta) / theta, 0.5 * ones)
    w = torch.where(valid, torch.cos(half_theta), ones)
    quat = torch.cat([w, k * angle_axis], dim=-1)
    return quat

def rotation_matrix_to_quaternion(rotation_matrix, eps=1e-6):
    """
    This function is borrowed from https://github.com/kornia/kornia
    Convert rotation matrix to 4d quaternion vector
    This algorithm is based on algorithm described in
    https://github.com/KieranWynn/pyquaternion/blob/master/pyquaternion/quaternion.py#L201

    :param rotation_matrix (N, 3, 3)
    """
    *dims, m, n = rotation_matrix.shape
    rmat_t = torch.transpose(rotation_matrix.reshape(-1, m, n), -1, -2)

    mask_d2 = rmat_t[:, 2, 2] < eps

    mask_d0_d1 = rmat_t[:, 0, 0] > rmat_t[:, 1, 1]
    mask_d0_nd1 = rmat_t[:, 0, 0] < -rmat_t[:, 1, 1]

    t0 = 1 + rmat_t[:, 0, 0] - rmat_t[:, 1, 1] - rmat_t[:, 2, 2]
    q0 = torch.stack(
        [
            rmat_t[:, 1, 2] - rmat_t[:, 2, 1],
            t0,
            rmat_t[:, 0, 1] + rmat_t[:, 1, 0],
            rmat_t[:, 2, 0] + rmat_t[:, 0, 2],
        ],
        -1,
    )
    t0_rep = t0.repeat(4, 1).t()

    t1 = 1 - rmat_t[:, 0, 0] + rmat_t[:, 1, 1] - rmat_t[:, 2, 2]
    q1 = torch.stack(
        [
            rmat_t[:, 2, 0] - rmat_t[:, 0, 2],
            rmat_t[:, 0, 1] + rmat_t[:, 1, 0],
            t1,
            rmat_t[:, 1, 2] + rmat_t[:, 2, 1],
        ],
        -1,
    )
    t1_rep = t1.repeat(4, 1).t()

    t2 = 1 - rmat_t[:, 0, 0] - rmat_t[:, 1, 1] + rmat_t[:, 2, 2]
    q2 = torch.stack(
        [
            rmat_t[:, 0, 1] - rmat_t[:, 1, 0],
            rmat_t[:, 2, 0] + rmat_t[:, 0, 2],
            rmat_t[:, 1, 2] + rmat_t[:, 2, 1],
            t2,
        ],
        -1,
    )
    t2_rep = t2.repeat(4, 1).t()

    t3 = 1 + rmat_t[:, 0, 0] + rmat_t[:, 1, 1] + rmat_t[:, 2, 2]
    q3 = torch.stack(
        [
            t3,
            rmat_t[:, 1, 2] - rmat_t[:, 2, 1],
            rmat_t[:, 2, 0] - rmat_t[:, 0, 2],
            rmat_t[:, 0, 1] - rmat_t[:, 1, 0],
        ],
        -1,
    )
    t3_rep = t3.repeat(4, 1).t()

    mask_c0 = mask_d2 * mask_d0_d1
    mask_c1 = mask_d2 * ~mask_d0_d1
    mask_c2 = ~mask_d2 * mask_d0_nd1
    mask_c3 = ~mask_d2 * ~mask_d0_nd1
    mask_c0 = mask_c0.view(-1, 1).type_as(q0)
    mask_c1 = mask_c1.view(-1, 1).type_as(q1)
    mask_c2 = mask_c2.view(-1, 1).type_as(q2)
    mask_c3 = mask_c3.view(-1, 1).type_as(q3)

    q = q0 * mask_c0 + q1 * mask_c1 + q2 * mask_c2 + q3 * mask_c3
    q /= torch.sqrt(
        t0_rep * mask_c0
        + t1_rep * mask_c1
        + t2_rep * mask_c2  # noqa
        + t3_rep * mask_c3
    )  # noqa
    q *= 0.5
    return q.reshape(*dims, 4)


import os.path
import unittest

from hamcrest import assert_that, contains_string

from heimer_tools.convert import illustrated_dot_data
from heimer_tools.convert import simple_dot_data

DATA_DIR = '/home/romilly/git/active/heimer-tools/data'

def path_to(relative_path: str):
    return os.path.join(DATA_DIR, relative_path)

def read(file_name):
    with open(file_name) as f:
        contents = f.read()
    return contents

class IllustratedDotFileTestCase(unittest.TestCase):
    def test_small_map(self):
        expected = read(path_to('test.dot'))
        df = illustrated_dot_data(path_to('test.alz'))
        self.assertEqual(expected, df)

    def test_big_map(self):
        dot_data = illustrated_dot_data(path_to('exponential.alz'))
        assert_that(dot_data, contains_string('digraph G {'))
        assert_that(dot_data, contains_string('Becoming an Expert'))

class SimpleDotFileTestCase(unittest.TestCase):
    def test_small_map(self):
        expected = read(path_to('simple.dot'))
        df = simple_dot_data(path_to('test.alz'))
        self.assertEqual(expected, df)

if __name__ == '__main__':
    unittest.main()


from pkg_resources import resource_exists, resource_listdir, resource_string, resource_stream,resource_filename
import xml.etree.ElementTree as ET
import numpy
from sklearn.datasets import fetch_20newsgroups

def load_n2c2_2006_train_dev_split():
    train = list(load_n2c2_2006(partition='train'))
    numpy.random.seed(0)
    numpy.random.shuffle(train)

    labels = {}
    for id, doc, label in train:
        if label not in labels:
            labels[label] = []
        labels[label].append(tuple((id,doc,label)))

    dev = []
    train = []
    for label in labels.keys():
        dev += labels[label][:int(len(labels[label])*.2)]
        train += labels[label][int(len(labels[label])*.2):]

    return train,dev

def load_n2c2_2006(partition='train'):
    """
    Yields a generator of id, doc, label tuples.
    :param partition:
    :return:
    """
    assert partition in ['train', 'test']

    if partition == 'train':
        with open("data/smokers_surrogate_%s_all_version2.xml" % partition) as raw:
            file = raw.read().strip()

    elif partition == 'test':
        with open("data/smokers_surrogate_%s_all_groundtruth_version2.xml" % partition) as raw:
            file = raw.read().strip()

    # file = resource_string('clinical_data', 'phenotyping/n2c2_2006/smokers_surrogate_%s_all_version2.xml' % partition).decode('utf-8').strip()
    root = ET.fromstring(file)
    ids = []
    notes = []
    labels = []
    documents = root.findall("./RECORD")
    for document in documents:
        ids.append(document.attrib['ID'])
        notes.append(document.findall('./TEXT')[0].text)
        labels.append(document.findall('./SMOKING')[0].attrib['STATUS'])

    for id, note, label in zip(ids,notes,labels):
        yield (id,note,label)

def load_n2c2_2008_train_dev_split():
    train = list(load_n2c2_2008(partition='train'))

    return train[:int(len(train)*.8)], train[int(len(train)*.8):]

def load_n2c2_2008(partition='train'):
    assert partition in ['train', 'test']
    documents = {} #id : text
    all_diseases = set()
    notes = tuple()
    if partition == 'train':
        with open('data/obesity_patient_records_training.xml') as t1, \
                open('data/obesity_patient_records_training2.xml') as t2:
            notes1 = t1.read().strip()
            notes2 = t2.read().strip()
        notes = (notes1,notes2)
    elif partition == 'test':
        with open('data/obesity_patient_records_test.xml') as t1:
            notes1 = t1.read().strip()
        notes = (notes1,)

    for file in notes:
        root = ET.fromstring(file)
        root = root.findall("./docs")[0]
        for document in root.findall("./doc"):
            assert document.attrib['id'] not in documents
            documents[document.attrib['id']] = {}
            documents[document.attrib['id']]['text'] = document.findall("./text")[0].text

    annotation_files = tuple()
    if partition == 'train':
        with open('data/obesity_standoff_annotations_training.xml') as t1, \
                open('data/obesity_standoff_annotations_training_addendum.xml') as t2, \
                open('data/obesity_standoff_annotations_training_addendum2.xml') as t3, \
                open('data/obesity_standoff_annotations_training_addendum3.xml') as t4:
            train1 = t1.read().strip()
            train2 = t2.read().strip()
            train3 = t3.read().strip()
            train4 = t4.read().strip()
        # train1 = resource_string('clinical_data', 'phenotyping/n2c2_2008/train/obesity_standoff_annotations_training.xml').decode('utf-8').strip()
        # train2 = resource_string('clinical_data', 'phenotyping/n2c2_2008/train/obesity_standoff_annotations_training_addendum.xml').decode('utf-8').strip()
        # train3 = resource_string('clinical_data', 'phenotyping/n2c2_2008/train/obesity_standoff_annotations_training_addendum2.xml').decode('utf-8').strip()
        # train4 = resource_string('clinical_data', 'phenotyping/n2c2_2008/train/obesity_standoff_annotations_training_addendum3.xml').decode('utf-8').strip()
        annotation_files = (train1,train2,train3,train4)
    elif partition == 'test':
        with open('data/obesity_standoff_annotations_test.xml') as t1:
            test1 = t1.read().strip()
        # test1 = resource_string('clinical_data','phenotyping/n2c2_2008/test/obesity_standoff_annotations_test.xml').decode('utf-8').strip()
        annotation_files = (test1,)

    for file in annotation_files:
        root = ET.fromstring(file)
        for diseases_annotation in root.findall("./diseases"):

            annotation_source = diseases_annotation.attrib['source']
            assert isinstance(annotation_source, str)
            for disease in diseases_annotation.findall("./disease"):
                disease_name = disease.attrib['name']
                all_diseases.add(disease_name)
                for annotation in disease.findall("./doc"):
                    doc_id = annotation.attrib['id']
                    if not annotation_source in documents[doc_id]:
                        documents[doc_id][annotation_source] = {}
                    assert doc_id in documents
                    judgment = annotation.attrib['judgment']
                    documents[doc_id][annotation_source][disease_name] = judgment

    all_diseases = list(all_diseases)
    #print(all_diseases)

    for id in documents: #set unlabeled instances to None
        for annotation_type in ('textual', 'intuitive'):
            for disease in all_diseases:
                if not annotation_type in documents[id]:
                    documents[id][annotation_type] = {}
                if not disease in documents[id][annotation_type]:
                    #print(id, annotation_type, disease)
                    documents[id][annotation_type][disease] = None

    for id in documents:
        yield id, documents[id]['text'], documents[id]['textual'], documents[id]['intuitive']
    from pprint import pprint
    #pprint(documents[list(documents.keys())[1]])

def load_newstest( random_state=42 , categories = ['alt.atheism', 'talk.religion.misc', 'comp.graphics', 'sci.space']):

    remove=()

    data_train = fetch_20newsgroups(subset='train', categories=categories,
                                    shuffle=True, random_state=42,
                                    remove=remove)

    data_test = fetch_20newsgroups(subset='test', categories=categories,
                                   shuffle=True, random_state=42,
                                   remove=remove)

    target_names = data_train.target_names
    print(  'labels for newstest dataset:', target_names )

    return data_train, data_test, target_names

def generator_newstest( data, target_names   ):
    """
    Yields a generator of id, doc, label tuples.
    :param dict of newstest data , target_names/labels:
    :return:
    """
    ids=[]
    documents=[]
    labels=[]
    for index, (text, nr_label) in enumerate(zip( data.data, data.target )):
        ids.append( index )
        documents.append( text )
        labels.append( target_names[ nr_label  ]   )

    for id, text, label in zip(ids,documents,labels):
        yield (id,text,label)


import serial

class Interface:

    def __init__(self):

        self.port = None
        self.sensor_state = None

    def initSensor(self, com_port='COM9', baud_rate=9600):
        try:
            self.port = serial.Serial(com_port, baud_rate)
            print("Receiving data from " + self.port.name)
            self.port.flushOutput()
            return True
        except:
            print ("Cannot open " + com_port)
            return False

    def updateSensor(self):
        # read data stream from port
        try:
            x = self.port.readline()
        except serial.portNotOpenError:
            print("Port closed")
        if x[-1] == '\n':
            x = x[0:-1]
        # tokenizing the string to sub-strings
        data = x.split(',')

        # converting them to numerical data
        try:
            self.sensor_state = [int(data[1]), int(data[2])]
       #     print(self.sensor_state)
        except:
            print(x)

        try:
            self.port.flushOutput()
        except serial.portNotOpenError:
            print("Port closed")

    def closeSensor(self):
        print(self.port.name + " closed")
        self.port.close()

    def getSensorState(self):
        return self.sensor_state


from src.utils.bam import addHPtag
import pandas as pd
import sys,argparse

parser = argparse.ArgumentParser(description='Add HP cluster tag and YC color tag to reads given readname-->cluster(int) pairing. Default setting for use with read_info.txt output from hifi amplicon clustering')
parser.add_argument('inbam',help='bam contaning hifi reads to tag',
                    type=str)
parser.add_argument('readInfo',help='read_info.txt output from hifi amplicon cluster',
                    type=str)
parser.add_argument('outbam',help='output bam',type=str)
parser.add_argument('--names',help='zero-indexed column number with read names. default 0',
                    type=int,default=0)
parser.add_argument('--cluster',help='zero-indexed column number with cluster number. default 8',
                    type=int,default=8)
parser.add_argument('--sep',help='column separator. default <space>',
                    type=str,default=' ')
parser.add_argument('--header',action='store_true',help='treat first row as column headers. default no header',
                    default=False)

args = parser.parse_args()

class arghandle:
    def __init__(self,inbam):
        self.inBAM    = inbam
        self.drop     = False
        self.splitBam = False
        self.prog     ='foo'
        self.region   = None

table = pd.read_csv(args.readInfo,sep=args.sep,header=0 if args.header else None)

print('making clustermap')
clustermap = dict(table[[args.names,args.cluster]].values)

print('tagging bam')
addHPtag(arghandle(args.inbam),args.outbam,clustermap)
print('Done')


import pydotplus
import collections
from IPython.display import Image
from sklearn.tree import export_graphviz
from sklearn import tree
import pandas as pd

titanic_train = pd.read_csv('titanic_train.csv')
titanic_test = pd.read_csv('titanic_test.csv')

#Ajout 0 sur passagers sans canines
titanic_train['Cabin'] = titanic_train['Cabin'].fillna(0)
titanic_test['Cabin'] = titanic_test['Cabin'].fillna(0)

#Suppression passagers sans ages
titanic_train = titanic_train.dropna(axis=0)
titanic_test = titanic_test.dropna(axis=0)

#Ajout colonne Child
titanic_train['Child'] = titanic_train["Age"].apply(lambda x: 1 if x<18 else 0)
titanic_test['Child'] = titanic_test["Age"].apply(lambda x: 1 if x<18 else 0)

#Ajout colonne Sex_cat
titanic_train['Sex_cat'] = titanic_train["Sex"].apply(lambda sex: 1 if str(sex)=='female' else 0)
titanic_test['Sex_cat'] = titanic_test["Sex"].apply(lambda sex: 1 if str(sex)=='female' else 0)

#Ajout colonne Fare2
def fareCategory(fare):
    if (fare < 10):
        return 1
    elif (fare < 20):
        return 2
    elif (fare < 30):
        return 3
    else:
        return 4

titanic_train['Fare2'] = titanic_train["Fare"].apply(lambda fare: fareCategory(fare))
titanic_test['Fare2'] = titanic_test["Fare"].apply(lambda fare: fareCategory(fare))

#Ajout colonne Has cabin
titanic_test['cabin2'] = titanic_test["Cabin"].apply(lambda x: 0 if x==0 else 1)
titanic_train['cabin2'] = titanic_train["Cabin"].apply(lambda x: 0 if x==0 else 1)

#Ajout colonne Has cabin
titanic_test['cabin2'] = titanic_test["Cabin"].apply(lambda x: 0 if x==0 else 1)
titanic_train['cabin2'] = titanic_train["Cabin"].apply(lambda x: 0 if x==0 else 1)

#Ajout colonne
titanic_test['closeFamily'] = titanic_test["SibSp"] + titanic_test["Parch"]
titanic_train['closeFamily'] = titanic_train["SibSp"] + titanic_train["Parch"]

clf=tree.DecisionTreeClassifier(max_depth=3)
data_feature_name = ['Child','Sex_cat','Fare2','cabin2','closeFamily', 'Pclass']

clf=clf.fit(titanic_train[data_feature_name], titanic_train['Survived'])

dot_data = tree.export_graphviz(clf, out_file=None, feature_names=data_feature_name,class_names=['Dead', 'Survived'],filled=True,rounded=True)

graph = pydotplus.graph_from_dot_data(dot_data)

Image(graph.create_png())#toplotatree
Image(graph.write_png('./tree.png'))#tosavetheplot

acc_decision_tree = round(clf.score(titanic_train[data_feature_name], titanic_train['Survived']) * 100, 2)
print("Accuracy of the decision tree: ",acc_decision_tree)


import json
import pandas as pd
from . import preprocessor

@preprocessor('likes')
def _get_likes_data(zip_file, folders):
    like_table = pd.DataFrame({'time': [], 'reaction': []})

    paths = ['likes_and_reactions/posts_and_comments.json', 'likes_and_reactions/pages.json']
    for path in paths:
        # Likes on posts
        with zip_file.open('likes_and_reactions/posts_and_comments.json') as f:
            jdata = json.loads(f.read())

            likes = []
            times = []

            for i in jdata['reactions']:
                times.append(i['timestamp'])
                likes.append(i['data'][0]['reaction']['reaction'])

            temp_table = pd.DataFrame({'time': times, 'type': likes})
            temp_table.time = pd.to_datetime(temp_table.time, unit='s')
            like_table = pd.concat([like_table, temp_table])

    return like_table