# first add mnist_original.mat to mldata, otherwise error
from sklearn.datasets import fetch_mldata
import matplotlib.pyplot as plt

# download and read mnist
mnist = fetch_mldata('MNIST original', data_home='./')

# 'mnist.data' is 70k x 784 array, each row represents the pixels from a 28x28=784 image
# 'mnist.target' is 70k x 1 array, each row represents the target class of the corresponding image
images=mnist.data
targets=mnist.target

# make the value of pixels from [0, 255] to [0, 1] for further process
X = mnist.data / 255.
Y = mnist.target

# print the first image of the dataset
img1 = X[0].reshape(28, 28)
plt.subplot(131)
plt.imshow(img1, cmap='gray')

# print the images after simple transformation
img2 = 1 - img1
plt.subplot(132)
plt.imshow(img2, cmap='gray')

img3 = img1.transpose()     #转置
plt.subplot(133)
plt.imshow(img3, cmap='gray')
# plt.show()

# split data to train and test(for faster calculation, just use 1/10 data)
from sklearn.model_selection import train_test_split
X_train,X_test,Y_train,Y_test=train_test_split(X[::10],Y[::10],test_size=1000)

# Q1:
# Please use the logistic regression(default parameters) in sklearn to classify the data above, and print the training accuracy and test accuracy.

# TODO:use logistic regression  Logistic回归模型
from sklearn.linear_model import LogisticRegression
from sklearn import metrics

lr_model=LogisticRegression()
lr_model.fit(X_train,Y_train)
train_accuracy=lr_model.score(X_train,Y_train)
test_accuracy=lr_model.score(X_test,Y_test)

print('LogisticRegression')
print('Training accuracy:%0.2f%%' % (train_accuracy*100))
print('Testing accuracy:%0.2f%%' % (test_accuracy*100))

# TODO:use naive bayes
from sklearn.naive_bayes import BernoulliNB

Bnl_model=BernoulliNB()
Bnl_model.fit(X_train,Y_train)
train_accuracy=Bnl_model.score(X_train,Y_train)
test_accuracy=Bnl_model.score(X_test,Y_test)

print('BernoulliNB')
print('Training accuracy:%0.2f%%' % (train_accuracy*100))
print('Testing accuracy:%0.2f%%' % (test_accuracy*100))

# TODO:use support vector machine
from sklearn.svm import LinearSVC

SVC_model=LinearSVC()
SVC_model.fit(X_train,Y_train)
train_accuracy=SVC_model.score(X_train,Y_train)
test_accuracy=SVC_model.score(X_test,Y_test)

print('LinearSVC_1')
print('Training accuracy:%0.2f%%' % (train_accuracy*100))
print('Testing accuracy:%0.2f%%' % (test_accuracy*100))

# TODO:use SVM with another group of parameters
SVC_model=LinearSVC(penalty='l2', loss='squared_hinge', dual=True, tol=0.0001, C=0.1, multi_class='ovr', fit_intercept=True, intercept_scaling=1, class_weight=None, verbose=0, random_state=None, max_iter=5000)
SVC_model.fit(X_train,Y_train)
train_accuracy=SVC_model.score(X_train,Y_train)
test_accuracy=SVC_model.score(X_test,Y_test)

print('LinearSVC_2')
print('Training accuracy:%0.2f%%' % (train_accuracy*100))
print('Testing accuracy:%0.2f%%' % (test_accuracy*100))

# LogisticRegression
# Training accuracy:95.58%
# Testing accuracy:90.90%
# BernoulliNB
# Training accuracy:83.75%
# Testing accuracy:84.20%
# LinearSVC_1
# Training accuracy:98.15%
# Testing accuracy:87.10%
# LinearSVC_2
# Training accuracy:95.77%
# Testing accuracy:90.20%


# python3 -m pip install isort autoflake astpretty black
# requires python3.9 to run
import os
import argparse
import ast
import subprocess
import multiprocessing
from pathlib import Path
import astpretty
import sys

parser = argparse.ArgumentParser()
parser.add_argument(
    "--out_dir", type=str, default="python",
)
parser.add_argument("--verbose", "-v", action="store_true")
parser.add_argument("--debug", "-d", action="store_true")
parser.add_argument("--autoflake", "-a", action="store_true")
parser.add_argument("--black", "-b", action="store_true")
parser.add_argument("--isort", "-i", action="store_true")
parser.add_argument("--license", "-l", action="store_true")
parser.add_argument("--ast", action="store_true")
args = parser.parse_args()

OUT_PATH = Path(args.out_dir)
SHOULD_SAVE_AST = args.ast
COMPATIBLE_MODULE = "oneflow.compatible.single_client"

def dumpprint(node):
    astpretty.pprint(node)

def is_decorator(d, name=None):
    return (isinstance(d, ast.Name) and d.id == name) or (
        isinstance(d, ast.Call) and isinstance(d.func, ast.Name) and d.func.id == name
    )

def is_stable(node: ast.AST):
    for d in node.decorator_list:
        if is_decorator(d, "stable_api"):
            return True
    return False

def is_experimental(node: ast.AST):
    for d in node.decorator_list:
        if is_decorator(d, "experimental_api"):
            return True
    return False

def get_parent_module(value):
    return ".".join(value.split(".")[0:-1])

def join_module(*args):
    return ".".join([m for m in args if m])

def path_from_module(module, is_init=False):
    if is_init:
        return Path("/".join(module.split(".") + ["__init__.py"]))
    else:
        return Path("/".join(module.split(".")) + ".py")

def module_from_path(path: Path):
    assert path.name.endswith(".py")
    parts = path.parts
    if parts[-1] == "__init__.py":
        return ".".join(path.parts[0:-1])
    else:
        return ".".join(path.parts)[0:-3]

def is_compatible_root_module(module: str):
    if module == COMPATIBLE_MODULE:
        return True
    assert module == "oneflow"
    return False

class ReservedKeywordsVisitor(ast.NodeVisitor):
    def __init__(self, keywords=None) -> None:
        self.keywords = keywords
        self.has_reserved_keyword = False

    def visit_Name(self, node: ast.Name):
        if node.id in self.keywords:
            self.has_reserved_keyword = True

def replace_filename(name: str):
    return name.replace("name_scope", "namescope")

def replace_str(name: str):
    name = replace_filename(name)
    name = name.replace("lib.core", "support")
    name = name.replace("compatible.single_client.core", "core")
    name = name.replace("enable_typing_check", "typing_check")
    if name.startswith("oneflow.python."):
        return name.replace("oneflow.python.", "oneflow.")
    elif name == "oneflow.python":
        return "oneflow"
    elif "single_client.python." in name or name.endswith("single_client.python"):
        return name.replace("single_client.python", "single_client")
    else:
        return name

class ExportVisitor(ast.NodeTransformer):
    def __init__(self, root_module="oneflow", src_target_module: str = None) -> None:
        super().__init__()
        self.staging_decorators = []
        self.root_module = root_module
        self.export_modules = {}
        self.top_imports = []
        self.src_target_module = src_target_module

    def append_export(self, target_module=None, node=None):
        if target_module not in self.export_modules:
            module = ast.Module(body=[], type_ignores=[])
            self.export_modules[target_module] = module
        else:
            module = self.export_modules[target_module]
        if isinstance(node, list):
            module.body += node
        else:
            module.body.append(node)

    def visit_Expr(self, node):
        if isinstance(node.value, ast.Constant) and isinstance(node.value.value, str):
            if "Copyright 2020 The OneFlow Authors" in node.value.value:
                return None
        return node

    def visit_ImportFrom(self, node):
        for name in node.names:
            if isinstance(name, ast.alias) and name.name == "oneflow_deprecate":
                return ast.ImportFrom(
                    module="oneflow",
                    names=[ast.alias(name="oneflow_deprecate")],
                    level=0,
                )
        for name in node.names:
            if not self.visit(name):
                return None
        if node.module:
            if node.module == "__future__" or "oneflow_export" in node.module:
                return None
            node.module = replace_str(node.module)
        self.top_imports.append(node)
        return node

    def visit_Import(self, node):
        for name in node.names:
            if not super().visit(name):
                return None
        self.top_imports.append(node)
        return node

    def visit_alias(self, node: ast.alias) -> ast.alias:
        if node.name.startswith("oneflow.python."):
            node.name = replace_str(node.name)
            return node
        elif node.name == "oneflow_export":
            return None
        elif "__export_symbols__" in node.name:
            return None
        else:
            node.name = replace_str(node.name)
            return node

    def visit_Name(self, node: ast.AST):
        if node.id == "oneflow_export":
            return None
        if node.id == "stable_api":
            return None
        if node.id == "experimental_api":
            return None
        return node

    def visit_Call(self, node: ast.AST):
        if not self.visit(node.func):
            return None
        return node

    def visit_ClassDef(self, node):
        node.body = [self.visit(n) for n in node.body]
        return self.visit_FunctionDef(node)

    def visit_FunctionDef(self, node):
        is_compatible_and_experimental = is_compatible_root_module(
            self.root_module
        ) and is_experimental(node)
        if not is_compatible_root_module(self.root_module) and is_stable(node):
            return None
        compact_decorator_list = [self.visit(d) for d in node.decorator_list]
        compact_decorator_list = [d for d in compact_decorator_list if d]
        node.body = [self.visit(n) for n in node.body]
        rkv = ReservedKeywordsVisitor(keywords=set({"int", "float"}))
        rkv.visit(node)
        has_reserved_keyword = rkv.has_reserved_keyword
        is_deprecated = False
        for d in node.decorator_list:
            if is_decorator(d, name="oneflow_deprecate"):
                is_deprecated = True
        for d in node.decorator_list:
            if is_decorator(d, name="register_tensor_op"):
                import_src = ast.parse(f"import {self.src_target_module}")
                self.append_export(target_module=self.root_module, node=import_src)
            if is_decorator(d, name="oneflow_export"):
                is_kept_in_src = (
                    True
                    or has_reserved_keyword
                    or self.src_target_module == target_module
                    or target_module in ["oneflow", "oneflow.scope", COMPATIBLE_MODULE]
                )
                arg0 = d.args[0]
                experimental_module = None
                if is_compatible_and_experimental:
                    experimental_module = "experimental"
                target_module0 = join_module(
                    self.root_module, experimental_module, get_parent_module(arg0.value)
                )
                target_symbol0 = arg0.value.split(".")[-1]

                if ".".join([target_module0, target_symbol0]) == self.src_target_module:
                    raise ValueError(
                        "[colition][both func and module]", self.src_target_module
                    )
                if is_kept_in_src:
                    target_module = self.src_target_module
                    target_symbol = node.name
                else:
                    target_module = target_module0
                    target_symbol = target_symbol0
                # nth export: import from first export
                for argN in d.args[1::]:
                    target_moduleN = join_module(
                        self.root_module,
                        experimental_module,
                        get_parent_module(argN.value),
                    )
                    target_nameN = argN.value.split(".")[-1]
                    assert arg0 != argN, {"arg0": arg0, "argN": argN}
                    import_from_first_export = ast.ImportFrom(
                        module=target_module,
                        names=[ast.alias(name=target_symbol, asname=target_nameN),],
                        level=0,
                    )
                    self.append_export(
                        target_module=target_moduleN, node=import_from_first_export
                    )

                if is_deprecated:
                    import_oneflow_deprecate = ast.ImportFrom(
                        module="oneflow",
                        names=[ast.alias(name="oneflow_deprecate")],
                        level=0,
                    )

                node.decorator_list = compact_decorator_list
                if is_kept_in_src:
                    asname = target_symbol0
                    if node.name == target_symbol0:
                        asname = None
                    if target_module0 == target_module and node.name == target_symbol0:
                        # print("[skip]", target_module0, target_symbol0)
                        pass
                    else:
                        import_from_src = ast.ImportFrom(
                            module=self.src_target_module,
                            names=[ast.alias(name=node.name, asname=asname),],
                            level=0,
                        )
                        self.append_export(
                            target_module=target_module0, node=import_from_src
                        )
                    if is_deprecated:
                        return [import_oneflow_deprecate, node]
                    else:
                        return node
                else:
                    if is_deprecated:
                        self.append_export(
                            target_module=target_module, node=import_oneflow_deprecate
                        )
                    # prepend imports in target module
                    self.append_export(
                        target_module=target_module, node=self.top_imports
                    )
                    if target_module != "oneflow":
                        import_star_from_src = ast.ImportFrom(
                            module=self.src_target_module,
                            names=[ast.alias(name="*")],
                            level=0,
                        )
                        # node.body.insert(0, import_star_from_src)
                        self.append_export(
                            target_module=target_module, node=import_star_from_src
                        )
                    # save func name for src import as before modifing node.name
                    src_asname = None
                    if node.name != target_symbol:
                        src_asname = node.name
                    # save first export in target module
                    node.name = target_symbol
                    self.append_export(target_module=target_module, node=node)

                    # src: import from first export
                    return ast.ImportFrom(
                        module=target_module,
                        names=[ast.alias(name=target_symbol, asname=src_asname),],
                        level=0,
                    )
            if is_decorator(d, name="oneflow_export_value"):
                assert len(node.body) == 2
                assert len(d.args) == 1
                target_module = join_module(
                    self.root_module, get_parent_module(d.args[0].value)
                )
                call = node.body[1].value
                assign = ast.Assign(
                    targets=[
                        ast.Name(id=d.args[0].value.split(".")[-1], ctx=ast.Store())
                    ],
                    value=call,
                )
                self.append_export(target_module=target_module, node=assign)
                # TODO: the doc is not dumped properly
                # doc = node.body[0]
                # self.append_export(target_module=target_module, node=doc)
                return None
        node.decorator_list = compact_decorator_list
        return node

class SrcFile:
    def __init__(self, spec) -> None:
        is_test = "is_test" in spec and spec["is_test"]
        self.export_visitor = None
        self.tree = None
        self.dst = Path(spec["dst"])
        self.src: Path = spec["src"]
        self.target_module = module_from_path(self.dst)
        self.target_module = replace_str(self.target_module)
        if is_test and args.verbose:
            print("[skip test]", self.src)
        else:
            txt = self.src.read_text()
            self.tree = ast.parse(txt)
            root_module = "oneflow"
            if (
                "compatible_single_client_python" in self.src.parts
                or self.src.name == "single_client_init.py"
                or self.src.name == "single_client_main.py"
            ):
                root_module = COMPATIBLE_MODULE
            self.export_visitor = ExportVisitor(
                root_module=root_module, src_target_module=self.target_module
            )
            self.export_visitor.visit(self.tree)
            if self.target_module == root_module:
                self.export_visitor.append_export(
                    target_module=root_module,
                    node=ast.parse(f"from . import distributed"),
                )
                if self.target_module == "oneflow":
                    self.export_visitor.append_export(
                        target_module=root_module,
                        node=ast.parse(f"from . import saved_model"),
                    )
                else:
                    self.export_visitor.append_export(
                        target_module="oneflow.compatible.single_client",
                        node=ast.parse(
                            f"from . import env, scope, math, optimizer, losses, config, layers, summary, random, typing, train, data, profiler, sysconfig, checkpoint, distribute, util, model, image, tensorrt, saved_model, regularizers"
                        ),
                    )
                    self.export_visitor.append_export(
                        target_module="oneflow.compatible.single_client.experimental",
                        node=ast.parse(
                            f"""from . import scope
from oneflow.compatible.single_client import unittest
"""
                        ),
                    )
                    self.export_visitor.append_export(
                        target_module="oneflow.compatible.single_client.deprecated",
                        node=ast.parse(f"from . import nn"),
                    )
                    self.export_visitor.append_export(
                        target_module="oneflow.compatible.single_client.config",
                        node=ast.parse(f"from . import collective_boxing"),
                    )
                    self.export_visitor.append_export(
                        target_module="oneflow.compatible.single_client.unittest",
                        node=ast.parse(f"from . import env"),
                    )
                    self.export_visitor.append_export(
                        target_module="oneflow.compatible.single_client.optimizer",
                        node=ast.parse(
                            f"from . import warmup, loss_scale, grad_clipping"
                        ),
                    )
            #     self.export_visitor.append_export(
            #         target_module=".".join([root_module, "lib.core"]), node=ast.parse(f"from . import async_util")
            #     )

def get_specs_under_python(python_path=None, dst_path=None):
    specs = []
    for p in Path(python_path).rglob("*.py"):
        if p.name == "version.py":
            continue
        rel = p.relative_to(python_path)
        dst = Path(dst_path).joinpath(rel)
        dst = Path(replace_filename(str(dst)))
        spec = {"src": p, "dst": dst}
        if rel.parts[0] == "test":
            spec["is_test"] = True
        specs.append(spec)
    return specs

def get_files():
    srcs = (
        get_specs_under_python(python_path="oneflow/python", dst_path="oneflow")
        + get_specs_under_python(
            python_path="oneflow/compatible_single_client_python",
            dst_path="oneflow/compatible/single_client",
        )
        + [
            {"src": Path("oneflow/init.py"), "dst": "oneflow/__init__.py"},
            {"src": Path("oneflow/__main__.py"), "dst": "oneflow/__main__.py"},
            {
                "src": Path("oneflow/single_client_init.py"),
                "dst": "oneflow/compatible/single_client/__init__.py",
            },
            {
                "src": Path("oneflow/single_client_main.py"),
                "dst": "oneflow/compatible/single_client/__main__.py",
            },
        ]
    )
    srcs = list(filter(lambda x: ("oneflow_export" not in x["src"].name), srcs))
    if args.debug:
        srcs = [
            {
                "src": Path("oneflow/python/ops/nn_ops.py"),
                "dst": "oneflow/ops/nn_ops.py",
            },
            {
                "src": Path("oneflow/python/advanced/distribute_ops.py"),
                "dst": "oneflow/advanced/distribute_ops.py",
            },
        ]
    pool = multiprocessing.Pool()
    srcs = pool.map(SrcFile, srcs,)
    pool.close()
    return srcs

class ModuleNode:
    def __init__(self, name=None, parent=None) -> None:
        self.children = dict()
        self.parent = parent
        self.level = 0
        if parent:
            self.level = parent.level + 1
        self.name = name

    def add_or_get_child(self, name):
        if name in self.children:
            return self.children[name]
        else:
            self.children[name] = ModuleNode(name=name, parent=self)
            return self.children[name]

    @property
    def is_leaf(self):
        return len(self.children.keys()) == 0

    def walk(self, cb):
        cb(self)
        for child in self.children.values():
            child.walk(cb)

    @property
    def leafs(self):
        ret = []

        def add_leafs(node: ModuleNode):
            if node.is_leaf:
                ret.append(node)

        self.walk(add_leafs)
        return ret

    @property
    def full_name(self):
        current_parent = self
        ret = self.name
        while current_parent.parent:
            current_parent = current_parent.parent
            ret = current_parent.name + "." + ret
        return ret

    def __str__(self) -> str:
        return "\n".join(
            [f"{self.full_name}"]
            + [child.__str__() for child in self.children.values()]
        )

    @staticmethod
    def add_sub_module(root=None, module=None):
        parts = module.split(".")
        current_node = root
        assert current_node.name == parts[0]
        for part in parts[1::]:
            current_node = current_node.add_or_get_child(part)

def save_trees(args=None):
    dst: Path = args["dst"]
    trees = args["trees"]
    dst_full = OUT_PATH.joinpath(dst)
    dst_full.parent.mkdir(parents=True, exist_ok=True)
    dst_full.touch(exist_ok=False)
    # TODO: append "doctest.testmod(raise_on_error=True)"
    trees = [ast.fix_missing_locations(tree) for tree in trees]
    if SHOULD_SAVE_AST:
        new_txt = "\n".join([str(astpretty.pformat(tree)) for tree in trees])
        new_txt = f"""from ast import *
{new_txt}
"""
        dst_full.with_suffix(".ast.py").write_text(new_txt)
    new_txt = ""
    if dst.name.startswith("test_"):
        if "compatible" in str(dst):
            new_txt += f"""
import {COMPATIBLE_MODULE}.unittest
"""
        else:
            new_txt += """
import oneflow.unittest
"""
    new_txt += "\n".join([ast.unparse(tree) for tree in trees])
    dst_full.write_text(new_txt)

def append_trees(tree_dict: dict, module: str, tree: ast.AST):
    tree_dict[module] = tree_dict.get(module, [])
    tree_dict[module].append(tree)

if __name__ == "__main__":
    out_oneflow_dir = os.path.join(args.out_dir, "oneflow")
    assert args.out_dir
    assert args.out_dir != "~"
    assert args.out_dir != "/"
    subprocess.check_call(f"mkdir -p {OUT_PATH}", shell=True)

    for py_f in Path(out_oneflow_dir).glob("**/*.py"):
        if py_f.name != "version.py":
            py_f.unlink()
    for pyc in Path(out_oneflow_dir).glob("**/*.pyc"):
        pyc.unlink()
    for d in reversed(
        [
            Path(dirpath)
            for dirpath, dirnames, files in os.walk(out_oneflow_dir)
            if not files and not dirnames
        ]
    ):
        if "include" not in str(d) and "core" not in str(d):
            d.rmdir()

    srcs = get_files()
    final_trees = {}

    root_module = ModuleNode(name="oneflow")
    src_module_added = {}
    for s in srcs:
        # src
        append_trees(tree_dict=final_trees, module=s.target_module, tree=s.tree)
        if (
            str(s.src) == "oneflow/python/__init__.py"
            or str(s.src) == "oneflow/compatible_single_client_python/__init__.py"
        ):
            assert not s.src.read_text()
            continue
        assert s.target_module not in src_module_added, {
            "target_module": s.target_module,
            "new": str(s.src),
            "exist": str(src_module_added[s.target_module]),
        }
        src_module_added[s.target_module] = s.src
        ModuleNode.add_sub_module(root=root_module, module=s.target_module)
    for s in srcs:
        # exports
        for export_path, export_tree in s.export_visitor.export_modules.items():
            append_trees(tree_dict=final_trees, module=export_path, tree=export_tree)
            ModuleNode.add_sub_module(root=root_module, module=export_path)
    leaf_modules = set([leaf.full_name for leaf in root_module.leafs])
    pool = multiprocessing.Pool()

    def is_init(module: str):
        is_leaf = module in leaf_modules
        is_magic = module.endswith("__")
        return is_leaf == False and is_magic == False

    srcs = pool.map(
        save_trees,
        [
            {"dst": path_from_module(module, is_init=is_init(module)), "trees": trees,}
            for module, trees in final_trees.items()
        ],
    )
    pool.close()
    # TODO: touch __init__.py, oneflow/F/__init__.py
    Path(os.path.join(OUT_PATH, "oneflow", "F")).mkdir(exist_ok=True)
    Path(os.path.join(OUT_PATH, "oneflow", "F/__init__.py")).touch()
    Path(os.path.join(OUT_PATH, COMPATIBLE_MODULE.replace(".", "/"), "F")).mkdir(
        parents=True, exist_ok=True
    )
    Path(
        os.path.join(OUT_PATH, COMPATIBLE_MODULE.replace(".", "/"), "F/__init__.py")
    ).touch()
    Path(
        os.path.join(OUT_PATH, COMPATIBLE_MODULE.replace(".", "/"), "experimental/F")
    ).mkdir(parents=True, exist_ok=True)
    Path(
        os.path.join(
            OUT_PATH, COMPATIBLE_MODULE.replace(".", "/"), "experimental/F/__init__.py"
        )
    ).touch()
    Path(os.path.join(OUT_PATH, f"oneflow/compatible/__init__.py")).touch()
    # step 1: extract all exports
    # step 2: merge exports into src in python/
    # step 4: finalize __all__, if it is imported by another module or wrapped in 'oneflow.export', it should appears in __all__
    # step 5: save file and post process (sort imports, format, etc)
    extra_arg = ""
    if args.verbose == False:
        extra_arg += "--quiet"
    if args.autoflake:
        print("[postprocess]", "autoflake")
        subprocess.check_call(
            f"{sys.executable} -m autoflake --in-place --remove-all-unused-imports --exclude '**/*.ast.py' --recursive .",
            shell=True,
            cwd=args.out_dir,
        )
    if args.isort:
        print("[postprocess]", "isort")
        subprocess.check_call(
            f"{sys.executable} -m isort --skip oneflow/utils/data/__init__.py . {extra_arg}",
            shell=True,
            cwd=args.out_dir,
        )
    if args.license:
        print("[postprocess]", "license")
        subprocess.check_call(
            f"`which python3` ci/check/run_license_format.py -i {OUT_PATH} --fix --silent",
            shell=True,
        )
    if args.black:
        print("[postprocess]", "black")
        subprocess.check_call(
            f"`which python3` -m black --exclude '\\.ast\\.py' . {extra_arg}",
            shell=True,
            cwd=args.out_dir,
        )


import deact
import argparse
from datetime import datetime
from pyspark.sql import SparkSession

# Parse arguments
def date_str(x):
	try:
		datetime.strftime(datetime.strptime(x, '%Y-%m-%d'), '%Y-%m-%d')
	except Exception:
		raise argparse.ArgumentTypeError('Argument must be a string formatted as "yyyy-mm-dd"')
	return x

def percent(x):
	try:
		x = float(x)
	except Exception:
		raise argparse.ArgumentTypeError('Argument must be a float in the range [0.0, 1.0]')
	if x < 0.0 or x > 1.0:
		raise argparse.ArgumentTypeError('Argument must be in the range [0.0, 1.0]')
	return x

parser = argparse.ArgumentParser()
parser.add_argument('--train_date', type=date_str, default='2018-11-01',
	help='String of format "yyyy-mm-dd" representing the feature date for the model\'s training data')
parser.add_argument('--score_date', type=date_str, default=None,
	help='String of format "yyyy-mm-dd" representing the feature date for the active customers being scored by the model')
parser.add_argument('--train_pct', type=percent, default=0.10,
	help='Float between 0.0 and 1.0 giving the proportion of the training data to use when training the model')
parser.add_argument('--calibrate_pct', type=percent, default=0.0,
	help='Float between 0.0 and 1.0 giving the proportion of the training data to use when calibrating the model')
parser.add_argument('--validate_pct', type=percent, default=0.0,
	help='Float between 0.0 and 1.0 giving the proportion of the training data to use when validating the model')
parser.add_argument('--calibrate_model', action='store_true', default=False,
	help='Makes model predictions on calibration data and fits an isotonic regression to adjust predictions')
parser.add_argument('--validate_model', action='store_true', default=False,
	help='Makes model predictions on validation data and evaluates the predictions')
parser.add_argument('--score_active_users', action='store_true', default=False,
	help='Makes model predictions on active customers')
parser.add_argument('--mask_prob', action='store_true', default=False,
	help='Drops the predicted deact probabilities for customers with large recency values and assigns probability = 1.0')
parser.add_argument('--mask_days', type=int, default=0,
	help='Integer giving the number of days to apply the probability mask.' +
		'E.g. a value of 3 masks customers who will deact in the next 3 days.')
args = parser.parse_args()

train_date = args.train_date
score_date = args.score_date
train_pct = args.train_pct
calibrate_pct = args.calibrate_pct
validate_pct = args.validate_pct
calibrate_model = args.calibrate_model
validate_model = args.validate_model
score_active_users = args.score_active_users
mask_prob = args.mask_prob
mask_days = args.mask_days

# Validate arguments
if train_pct + calibrate_pct + validate_pct > 1.0:
	raise parser.error('Argument Validation Error: train_pct + calibrate_pct + validate_pct cannot exceed 1.0')

if calibrate_model and calibrate_pct == 0.0:
	raise parser.error('Argument Validation Error: calibrate_pct > 0.0 is required when calibrate_model is True')

if validate_model and validate_pct == 0.0:
	raise parser.error('Argument Validation Error: validate_pct > 0.0 is required when validate_model is True')

if score_active_users and score_date is None:
	raise parser.error('Argument Validation Error: score_date is required when score_active_users is True')

if not calibrate_model and calibrate_pct > 0.0:
	print('[ {0} ] : Overwriting calibrate_pct to 0.0% since calibrate_model is False'.format(datetime.utcnow()))
	calibrate_pct = 0.

if not validate_model and validate_pct > 0.0:
	print('[ {0} ] : Overwriting validate_pct to 0.0% since validate_model is False'.format(datetime.utcnow()))
	validate_model = 0.

if not score_active_users and score_date is not None:
	print('[ {0} ] : score_date will be ignored since score_active_users is False'.format(datetime.utcnow()))
score_date = score_date if score_active_users else 'N/A'

# Coalesce some variables
not_used_pct = 1 - train_pct - calibrate_pct - validate_pct

# Start SparkSession
# warehouse_location = 'hdfs://cerebro-namenode-vip.snc1/user/grp_gdoop_clv/grp_gdoop_clv_hiveDB.db'
# python_location = './ANACONDA/anaconda2_env/bin/python'
# archives_location = 'hdfs:////user/grp_gdoop_admin/anaconda/anaconda2_env.zip#ANACONDA'

# spark = SparkSession\
# 	.builder\
# 	.master('yarn')\
# 	.appName('deact-model')\
# 	.config('spark.submit.deployMode','client')\
# 	.config('spark.executor.instances', 50)\
# 	.config('spark.yarn.appMasterEnv.PYSPARK_PYTHON',python_location)\
# 	.config('spark.yarn.dist.archives',archives_location)\
# 	.config('spark.sql.warehouse.dir',warehouse_location)\
# 	.config('spark.yarn.queue','public')\
# 	.enableHiveSupport()\
# 	.getOrCreate()

spark = SparkSession\
	.builder\
	.enableHiveSupport()\
	.getOrCreate()

spark.sparkContext.setLogLevel('WARN')

########################
# DEACT MODEL WORKFLOW #
########################

print('''
[ {0} ] : BEGINNING DEACT MODEL PIPELINE

	PARAMETERS:
	Training date = {1}
	Proportion of training data used to:
		Train model = {2:0.0%}
		Calibrate model = {3:0.0%}
		Validate model = {4:0.0%}
		Not used = {5:0.0%}
	Scoring date = {6}
	Masking deact probabilities for {7:d} days
'''.format(datetime.utcnow(), train_date, train_pct, calibrate_pct, validate_pct, not_used_pct, score_date, mask_days))

# Save locations
hdfs_loc = 'hdfs://cerebro-namenode-vip.snc1/user/grp_gdoop_clv/deact-model/'
eval_path = '/home/svc_clv/Consumer-Intelligence/Models/Deact-Model/model_evaluation/'

# Run data pipeline
pl = deact.Pipeline(spark, train_date, score_date, train_pct, calibrate_pct, validate_pct)
dfs = pl.run(calibrate_model, validate_model, score_active_users)

# Train model
model = deact.Model(train_date, score_date, pl.feature_list)
model.train(dfs['training'])

# Calibrate model probabilities
if calibrate_model:
	model.calibrate(dfs['calibration'])

# Make predictions on validation data, evaluate them, and plot results
if validate_model:
	model.predict(dfs['validation'], 'validation', calibrate_model, mask_prob, False, mask_days)
	evaluator = deact.Evaluator(model.pred_df, calibrate_model, mask_prob, True, eval_path)
	evaluator.class_metrics()
	evaluator.calibration()
	evaluator.decision_boundary_curves()
	evaluator.recency_aggregation()
	evaluator.feature_importances(model.feat_imp, min_imp=0.001)

# Make predictions on scoring data and save them. Check that predictions are stable over time.
pct_change = None
if score_active_users:
	model.predict(dfs['scoring'], 'scoring', calibrate_model, mask_prob, True, mask_days, hdfs_loc)
	model.total_deacts(calibrate_model, 'scoring', True, hdfs_loc)
	pct_change = model.compare_to_prev_day(spark, calibrate_model, hdfs_loc, max_change=0.05)

print('\n[ {0} ] : COMPLETED DEACT MODEL PIPELINE\n'.format(datetime.utcnow()))


from opengever.propertysheets.annotation import CustomPropertiesStorage
from opengever.propertysheets.annotation import CustomPropertiesStorageImpl
from opengever.propertysheets.assignment import DOCUMENT_DEFAULT_ASSIGNMENT_SLOT
from opengever.propertysheets.assignment import DOCUMENT_TYPE_ASSIGNMENT_SLOT_PREFIX
from opengever.propertysheets.exceptions import BadCustomPropertiesFactoryConfiguration
from opengever.propertysheets.field import PropertySheetField
from opengever.testing.test_case import FunctionalTestCase
from persistent.dict import PersistentDict
from plone.supermodel import model
from zope import schema
from zope.annotation.interfaces import IAnnotations

def fixture_assignment_factory():
    return [
        u"IDocumentMetadata.document_type.contract",
        u"IDocumentMetadata.document_type.question",
    ]

class ITestFixtureWithCustomProperties(model.Schema):

    custom_properties = PropertySheetField(
        request_key='some_request_key',
        attribute_name='some_attribute',
        assignemnt_prefix=DOCUMENT_TYPE_ASSIGNMENT_SLOT_PREFIX,
        valid_assignment_slots_factory=fixture_assignment_factory,
        default_slot=DOCUMENT_DEFAULT_ASSIGNMENT_SLOT
    )

class TestCustomPropertiesStorage(FunctionalTestCase):

    def setUp(self):
        super(TestCustomPropertiesStorage, self).setUp()
        self.factory = CustomPropertiesStorage(
            ITestFixtureWithCustomProperties
        )
        # any IAnnotatable will serve as a test-fixture
        self.context = self.portal
        self.props = self.factory(self.context)
        self.annotations = IAnnotations(self.context)
        self.key = (
            'opengever.propertysheets.tests.test_annotation.'
            'ITestFixtureWithCustomProperties.custom_properties'
        )

    def assert_custom_properties_equal(self, expected):
        self.assertEqual(expected, self.props.custom_properties)
        self.assertEqual(expected, self.annotations[self.key])

    def test_custom_property_storage_is_none_per_default(self):
        self.assertIsNone(self.props.custom_properties)

    def test_custom_property_storage_initialzes_none_in_annotations(self):
        self.assertNotIn(self.key, self.annotations)

        self.props.custom_properties = None
        self.assertIn(self.key, self.annotations)
        self.assertIsNone(self.annotations[self.key])

        self.assertIsNone(self.props.custom_properties)

    def test_custom_property_storage_initialzes_empty_dict_in_annotations(self):
        self.assertNotIn(self.key, self.annotations)

        self.props.custom_properties = {}
        self.assertIn(self.key, self.annotations)
        self.assertIsInstance(self.annotations[self.key], PersistentDict)
        self.assertEqual(PersistentDict(), self.annotations[self.key])

    def test_custom_property_storage_raises_attribute_error_for_unknowns(self):
        with self.assertRaises(AttributeError):
            self.props.foo

    def test_custom_property_storage_converts_values_to_persistent(self):
        self.props.custom_properties = {"foo": 123}
        self.assertIsInstance(self.annotations[self.key], PersistentDict)

    def test_custom_property_storage_updates_existing_values(self):
        self.props.custom_properties = {"foo": 123}
        self.props.custom_properties = {"bar": "qux"}

        expected = {"foo": 123, "bar": "qux"}
        self.assert_custom_properties_equal(expected)

    def test_custom_property_storage_keeps_existing_when_set_to_empty(self):
        self.props.custom_properties = {"foo": 123}
        self.props.custom_properties = {}

        expected = {"foo": 123}
        self.assert_custom_properties_equal(expected)

    def test_custom_property_storage_keeps_existing_when_set_to_none(self):
        self.props.custom_properties = {"foo": 123}
        self.props.custom_properties = None

        expected = {"foo": 123}
        self.assert_custom_properties_equal(expected)

    def test_custom_property_storage_allows_clearing(self):
        self.props.custom_properties = {"foo": 123}
        self.assertEqual({"foo": 123}, self.props.custom_properties)

        self.props.clear()
        self.assertIsNone(self.props.custom_properties)
        self.assertNotIn(self.key, self.annotations)

    def test_custom_property_storage_allows_clearing_of_empty_storage(self):
        self.assertIsNone(self.props.custom_properties)

        self.props.clear()
        self.assertIsNone(self.props.custom_properties)
        self.assertNotIn(self.key, self.annotations)

    def test_custom_property_storage_prevents_setting_wrong_iterable_type(self):
        with self.assertRaises(TypeError):
            self.props.custom_properties = []

        with self.assertRaises(TypeError):
            self.props.custom_properties = ['foo']

        with self.assertRaises(TypeError):
            self.props.custom_properties = tuple()

        with self.assertRaises(TypeError):
            self.props.custom_properties = ('foo', 'bar')

    def test_custom_property_storage_prevents_setting_basic_type(self):
        with self.assertRaises(TypeError):
            self.props.custom_properties = 0

        with self.assertRaises(TypeError):
            self.props.custom_properties = ''

        with self.assertRaises(TypeError):
            self.props.custom_properties = 'blub'

        with self.assertRaises(TypeError):
            self.props.custom_properties = 123

        with self.assertRaises(TypeError):
            self.props.custom_properties = 1.3

        with self.assertRaises(TypeError):
            self.props.custom_properties = True

        with self.assertRaises(TypeError):
            self.props.custom_properties = False

    def test_factory_validates_schema_field_name(self):
        class SchemaWithMissingField(model.Schema):
            pass

        context = None
        with self.assertRaises(BadCustomPropertiesFactoryConfiguration) as cm:
            CustomPropertiesStorageImpl(context, SchemaWithMissingField)

        self.assertEqual(
            u"Custom properties factory must be assigned to a "
            u"schema with only a 'custom_properties' field.",
            cm.exception.message
        )

    def test_factory_validates_schema_field_type(self):
        class SchemaWithWrongField(model.Schema):
            custom_properties = schema.Text()

        context = None
        with self.assertRaises(BadCustomPropertiesFactoryConfiguration) as cm:
            CustomPropertiesStorageImpl(context, SchemaWithWrongField)

        self.assertEqual(
            u"The schema must contain a field providing 'IPropertySheetField'",
            cm.exception.message
        )


import torch
import numpy as np
from torch import nn
import torch.nn.functional as F
from torchaudioclip import stft_and_db

class Net(nn.Module):
    def __init__(self, layer_num, state_shape, action_shape=0, device='cpu',
                 softmax=False):
        super().__init__()
        self.device = device
        self.model = [
            nn.Linear(np.prod(state_shape), 128),
            nn.ReLU(inplace=True)]
        for i in range(layer_num):
            self.model += [nn.Linear(128, 128), nn.ReLU(inplace=True)]
        if action_shape:
            self.model += [nn.Linear(128, np.prod(action_shape))]
        if softmax:
            self.model += [nn.Softmax(dim=-1)]
        self.model = nn.Sequential(*self.model)

    def forward(self, s, state=None, info={}):
        if not isinstance(s, torch.Tensor):
            s = torch.tensor(s, device=self.device, dtype=torch.float)
        batch = s.shape[0]
        s = s.view(batch, -1)
        logits = self.model(s)
        return logits, state

class Actor(nn.Module):
    def __init__(self, preprocess_net, action_shape):
        super().__init__()
        self.preprocess = preprocess_net
        self.last = nn.Linear(128, np.prod(action_shape))

    def forward(self, s, state=None, info={}):
        logits, h = self.preprocess(s, state)
        logits = F.softmax(self.last(logits), dim=-1)
        # print(logits)
        return logits, h

class Critic(nn.Module):
    def __init__(self, preprocess_net):
        super().__init__()
        self.preprocess = preprocess_net
        self.last = nn.Linear(128, 1)

    def forward(self, s, **kwargs):
        logits, h = self.preprocess(s, state=kwargs.get('state', None))
        logits = self.last(logits)
        return logits

class BandoriMusicNet(nn.Module):
    def __init__(self, input_size, layer_num, mid_size=256, output_size=256, device='cuda'):
        super().__init__()
        self.device = device
        self.model = [
            nn.Linear(input_size, mid_size), nn.ReLU(inplace=True),
        ]
        for _ in range(layer_num):
            self.model += [nn.Linear(mid_size, mid_size), nn.ReLU(inplace=True)]
        self.model += [nn.Linear(mid_size, output_size)]
        self.model = nn.Sequential(*self.model)

    def forward(self, s, state=None, info={}):
        if not isinstance(s, torch.Tensor):
            s = torch.tensor(s, device=self.device, dtype=torch.float32)
        batch = s.shape[0]
        s = s.view(batch, -1)
        music_out = self.model(s)
        return music_out

class BandoriPPO(nn.Module):
    def __init__(self, is_actor, height=72, input_channel=3, interval=3,
                 audio_state=True, audio_input_size=257, audio_net_layer=2, audio_mid_size=256, audio_output_size=256, device='cuda'):
        super().__init__()
        self.device = device
        self.input_channel = input_channel
        self.interval = interval
        self._is_actor = is_actor
        self.one_frame_shape = height>>1, (height<<4)//9  # height is attr of full screen
        # music model
        self.audio_state = audio_state
        if self.audio_state:
            self.music_model = BandoriMusicNet(
                audio_input_size, audio_net_layer, audio_mid_size, audio_output_size, device)
        # graph model, input shape = [batch, input_channel, interval, height_ width]
        self.conv1 = nn.Conv3d(in_channels=self.input_channel, out_channels=32, kernel_size=(1,3,15), stride=(1,2,8))
        self.relu1 = nn.ReLU(inplace=True)
        # input = 32, 3, 18?, 16
        self.conv2 = nn.Conv3d(in_channels=32, out_channels=64, kernel_size=(1,3,3))
        self.relu2 = nn.ReLU(inplace=True)
        # input = 32, 3, 18?, 16
        self.conv3 = nn.Conv3d(in_channels=64, out_channels=64, kernel_size=(1,3,3))
        self.relu3 = nn.ReLU(inplace=True)
        # graph: FC -> 256
        self.fc_graph = None
        self.total_relu = nn.ReLU(inplace=True)
        self.total_FC = nn.Linear(512 if self.audio_state else 256, 21 if self._is_actor else 1)

    def forward(self, s, state=None, info={}):
        # s.shape = [batch, len([height, width, channel]*3, audio_shape) ]
        batch = s.shape[0]
        graph_batch = torch.tensor(list(s[:, 0:self.interval]), dtype=torch.float32).to(self.device)
        # if not isinstance(s, torch.Tensor):
        #     s = torch.tensor(s, device=self.device, dtype=torch.float32)
        # shape=[batch, 3-png-shape] -> batch, channel, interval, height, width, verified in small rand tensor
        graph_batch = (graph_batch/255.).squeeze(dim=0).view([batch, self.input_channel, self.interval, *self.one_frame_shape])
        x = self.conv1(graph_batch)
        x = self.relu1(x)
        x = self.conv2(x)
        x = self.relu2(x)
        x = self.conv3(x)
        x = self.relu3(x)
        x = x.view(batch,-1)
        self.fc_graph = nn.Linear(int(torch.tensor(x[0].shape, dtype=torch.int).prod()), 256).to(self.device)
        x = self.fc_graph(x)
        if self.audio_state:
            audio_input = stft_and_db(list(s[:, -1]))
            y = self.music_model(audio_input)
            x = torch.cat((x,y), dim=-1)
        x = self.total_relu(x)
        logits = self.total_FC(x)
        if self._is_actor:
            return F.softmax(logits, dim=-1), state
        else:
            return logits


#Bhandari algorithm
import math
import sys
import copy
import itertools
import topology

#networks topology

map1 = topology.USbackbone

#topology's copy
map2 = topology.USbackbone

g = copy.deepcopy(map1)
g2 = copy.deepcopy(map2)

#the number of nodes
node_num = len(map1)

nodes =[a for a in range(node_num)]
s_d = list(itertools.combinations(nodes,2))
s_d = [(14,4)]

def dijkstra(s,d):
  S_bar =[] #list of neighbor nodes
  predecessor = [None] * node_num

  # step 1

  distance = [math.inf] * node_num #distance of i from s
  distance[s] = 0 #source node
  for index in nodes:
    if g[index][s] > 0:
      distance[index] = g[index][s]
      predecessor[index] = s
      S_bar.append(index)
  print(S_bar)
  while True:

    # step 2

    j = get_j(S_bar,distance)
    if j == None:
      print('Not found')
      j = d
    if j == d:
      break
    print('j =',j+1)
    S_bar.remove(j)
    print(S_bar)

    # step 3
    # print(g[j])
    for index in nodes:
      if g[j][index] != 0:
        # print(distance[index],distance[j],g[])
        if distance[j] + g[j][index] < distance[index]:
          distance[index] = distance[j] + g[j][index]
          predecessor[index] = j
          if (index in S_bar) == False:
            print('reseach it')
            print('i=',index+1)
            S_bar.append(index)
            print(S_bar)

  return distance,predecessor

def get_j(S_bar,distance):
  min_dis = math.inf
  min_i = None
  for index in S_bar:
    if min_dis > distance[index]:
      min_dis = distance[index]
      min_i = index
  return min_i

def modulation_decision(distance):
  if distance <= 250:#32-QAM
    return 62.5
  elif distance <= 500:#16-QAM
    return 50
  elif distance <= 1000:#8-QAM
    return 37.5
  elif distance <= 2000:#QPSK
    return 25
  elif distance <= 4000:#BPSK
    return 12.5
  else:#over reach
    return 0

#Bhandari
def Bhandari(s,d):

  #first dijkstra

  distance,predecessor = dijkstra(s,d)
  # print(predecessor)
  p = d
  while p != None:
    pre = predecessor[p]
    if p != s:
      g[p][pre] = - g[p][pre]
      g[pre][p] = 0
    p = pre

  #N-1 times dijkstra for the topology including negative arcs

  count = 1
  while count < N:
    print('N=',count)
    distance,predecessor = dijkstra(s,d)
    print(distance)
    print(g)
    # print(predecessor)
    p = d
    while p != None:
      pre = predecessor[p]
      if p != s and pre != None:

        #replace the overlapping links with the original links

        if g[pre][p] < 0:
          g[pre][p] = g2[pre][p]
          g[p][pre] = g2[p][pre]

        #make the length of each edges negative

        else:
          g[p][pre] = - g[p][pre]
          g[pre][p] = 0
      p = pre

    count += 1

  #replace links we should use (negative arcs will be used)
  # print(g)
  for i in nodes:
    for j in nodes:
      if g[i][j] > 0:
        g[i][j] = 0
  for i in nodes:
    for j in nodes:
      if g[i][j] < 0:
        g[i][j] = g2[i][j]
        g[j][i] = g2[j][i]

  print(g)
  #get N disjoint paths

  for a in range(N):

    distance,predecessor = dijkstra(s,d)
    # print('distance = ',distance)
    hop_count = 0

    p = d
    # print(predecessor)
    while p != None:
      pre = predecessor[p]
      if p != s and pre != None:
        g[p][pre] = 0
        g[pre][p] = 0

        print(str(p+1) + " <- ", end='')
        hop_count += 1
      else:
        print(str(p+1))

      p = pre

    # print(g)
    hop.append(hop_count)

    eta.append(modulation_decision(distance[d]))

if __name__ == "__main__":
  for (s,d) in s_d:

    print("(s,d)=", (s+1,d+1))
    N = 2
    while True:

      hop = []
      eta = []
      Bhandari(s,d)
      g = copy.deepcopy(map1)
      g2 = copy.deepcopy(map2)

      if 0 in hop:
        break
      print(str(N) + "本")
      print(hop,eta)

      N += 1
    print("-------------")


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Fri Jun  7 16:10:07 2019

@author: shoichi
"""

#!/usr/bin/env python
# -*- coding: utf-8 -*-
import os
import sys
import time
from keras import backend as K
K.set_image_data_format('channels_last')
from keras.models import Sequential
from keras.layers import Flatten, Dense, Dropout, Conv2D,Activation,MaxPooling2D
from sklearn.metrics import confusion_matrix
from keras.optimizers import Adam
import train1000

def model():
    model = Sequential()
    model.add(Conv2D(32, (3, 3), padding="same", input_shape=X_train.shape[1:]))
    model.add(Activation('relu'))
    model.add(MaxPooling2D(pool_size=(2, 2)))
    model.add(Dropout(0.5))
    model.add(Flatten())
    model.add(Dense(512))
    model.add(Activation('relu'))
    model.add(Dropout(0.7))
    model.add(Dense(10))
    model.add(Activation('softmax'))
    model.compile(loss='categorical_crossentropy', optimizer=Adam(), metrics=['categorical_crossentropy', 'accuracy'])
    model.summary()
    return model

if __name__ == '__main__':
    epochs = 50
    batch_size = 10
    (X_train, Y_train), (X_test, Y_test) = train1000.cifar10()
    nb_classes = 10
    model = model()
    result = model.fit(X_train, Y_train, batch_size=batch_size, epochs=epochs, validation_split=0.1,verbose=1)
    print('train data:')
    eva=model.evaluate(X_train,Y_train,verbose=1)
    for i in range(1,len(model.metrics_names)):
        print( model.metrics_names[i] + ' : ', eva[i] )
    print()
    print('test data:')
    eva = model.evaluate(X_test,Y_test,verbose=1)
    for i in range(1,len(model.metrics_names)):
        print( model.metrics_names[i] + ' : ', eva[i] )


# -*- coding: utf-8 -*-
"""
Created on Tue Nov  3 14:35:36 2020

@author: Kang Gu
"""
from tensorflow import keras
from tensorflow.keras.utils import to_categorical
from data_loader import load_MTS
from NFS import build_model, feature_selection, dstmodel, Agnos
from sklearn.model_selection import train_test_split
from sklearn.metrics import accuracy_score
import numpy as np

def run_NFS(dataset,K=32, perform_fs=True, dst_dim=32, dst_model="LSTM", selected_feature=None):
    X, Y, x_test, y_test = load_MTS(dataset)
    x_train, x_val, y_train, y_val = train_test_split(X, Y, test_size=0.25)
    y_train = to_categorical(y_train)
    y_val = to_categorical(y_val)
    y_test = to_categorical(y_test)

    model = build_model([x_train.shape[1], x_train.shape[2]], [x_train.shape[1], dst_dim],downstream=dst_model, feature_subset=selected_feature, enable_fs=perform_fs)
    print("train stage I...")
    model.fit( x_train, y_train,
                    epochs=100,
                    batch_size=128, # Default is 32, we can also change it smaller
                    validation_data=(x_val, y_val),
                    verbose=1,
                    shuffle=True,
                    callbacks = [keras.callbacks.EarlyStopping(monitor='val_loss',
                                   min_delta=0.1,
                                   patience=1,
                                   verbose=0,
                                   mode='min',
                                   baseline=None,
                                   restore_best_weights=True)]
                    )

    score = model.evaluate(x_test,y_test, verbose=0)

    if perform_fs:
        print("train stage II...")
        selected_feature = feature_selection(model, K)
        new_model =  build_model([x_train.shape[1], x_train.shape[2]], [x_train.shape[1], dst_dim],downstream=dst_model, feature_subset=selected_feature)
        new_model.fit( x_train, y_train,
                    epochs=100,
                    batch_size=128, # Default is 32, we can also change it smaller
                    validation_data=(x_val, y_val),
                    verbose=1,
                    shuffle=True,
                    callbacks = [keras.callbacks.EarlyStopping(monitor='val_loss',
                                   min_delta=0.1,
                                   patience=3,
                                   verbose=0,
                                   mode='min',
                                   baseline=None,
                                   restore_best_weights=True)]
                    )
        newscore = new_model.evaluate(x_test, y_test, verbose=0)

        print('Test accuracy:', score[1], newscore[1])
    else:

        print('Test accuracy:', score[1])
    return

def run_baseline(dataset,  dst_model="LSTM"):
    name = dst_model.lower()
    X, Y, x_test, y_test = load_MTS(dataset)
    x_train, x_val, y_train, y_val = train_test_split(X, Y, test_size=0.25)
    y_train = to_categorical(y_train)
    y_val = to_categorical(y_val)
    #y_test = to_categorical(y_test)

    model = dstmodel(name, [x_train.shape[1], x_train.shape[2]])
    model.compile(loss = "categorical_crossentropy", optimizer="adam", metrics=["categorical_accuracy"])
    model.fit( x_train, y_train,
                    epochs=100,
                    batch_size=128, # Default is 32, we can also change it smaller
                    validation_data=(x_val, y_val),
                    verbose=1,
                    shuffle=True,
                    callbacks = [keras.callbacks.EarlyStopping(monitor='val_loss',
                                   min_delta=0,
                                   patience=2,
                                   verbose=0,
                                   mode='min',
                                   baseline=None,
                                   restore_best_weights=True)])

    #score = model.evaluate(x_test, y_test, verbose=0)
    y_pred = model.predict(x_test)
    y_pred = np.argmax(y_pred, axis=1)

    accuracy =  accuracy_score(y_test, y_pred)
    print("Test Accuracy:", accuracy )

    return

def run_autoencoder(dataset, k):
    X, Y, x_test, y_test = load_MTS(dataset)
    x_train, x_val, y_train, y_val = train_test_split(X, Y, test_size=0.25)
    y_train = to_categorical(y_train)
    y_val = to_categorical(y_val)

    model = Agnos([x_train.shape[1], x_train.shape[2]])
    model.fit( x_train, x_train,
                    epochs=100,
                    batch_size=128, # Default is 32, we can also change it smaller
                    validation_data=(x_val, x_val),
                    verbose=1,
                    shuffle=True,
                    callbacks = [keras.callbacks.EarlyStopping(monitor='val_loss',
                                   min_delta=0,
                                   patience=2,
                                   verbose=0,
                                   mode='min',
                                   baseline=None,
                                   restore_best_weights=True)])
    weights = model.layers[1].get_weights()[0]
    score = np.amax(weights, axis=1)
    indices = np.argsort(score)[-k:]
    indices.sort()
    print(indices)
    return indices

if __name__ =="__main__":
    #Downstream models include "CNNRNN", "CNNGRU", "CNNLSTM", "RESNET", "MCNN","MCDCNN" and "TLENET"

    dataset = "FaceDetection"
    perform_fs = True
    dst_dim = 32
    k = 60 ##parameter for feature selection
    downstream_model = "TLENET"
    autoencoder_feature = [0,1,2,3,4,5,7,10,11,13,17,19,20,21,26,27,28,30,31,33,39,40,41,42,
    48,49,50,54,55,56,57,58]

    #run_baseline(dataset, dst_model=downstream_model)
    run_NFS(dataset,K=k, perform_fs=perform_fs,dst_model="MCDCNN")
    #run_autoencoder(dataset, dst_dim)
    #run_NFS(dataset, False, dst_model="MCDCNN",selected_feature=autoencoder_feature)


import pytest

from .. import create_application
from ..config import TestConfig
from ..db import Session, configure_engine, create_all, drop_all

pytest.register_assert_rewrite("conbench.tests.api._asserts")
pytest.register_assert_rewrite("conbench.tests.app._asserts")

@pytest.fixture(scope="session", autouse=True)
def create_db():
    configure_engine(TestConfig.SQLALCHEMY_DATABASE_URI)
    drop_all()
    create_all()

@pytest.fixture
def application():
    application = create_application(TestConfig)

    with application.app_context():
        pass

    yield application

    with application.app_context():
        Session.remove()

@pytest.fixture
def client(application):
    return application.test_client()

@pytest.fixture
def runner(application):
    return application.test_cli_runner()

def pytest_addoption(parser):
    parser.addoption(
        "--run-slow",
        action="store_true",
        default=False,
        help="run slow tests",
    )

def pytest_configure(config):
    config.addinivalue_line(
        "markers",
        "slow: mark test as slow to run",
    )

def pytest_collection_modifyitems(config, items):
    if config.getoption("--run-slow"):
        return
    reason = "need --run-slow option to run"
    skip_slow = pytest.mark.skip(reason=reason)
    for item in items:
        if "slow" in item.keywords:
            item.add_marker(skip_slow)


#! /usr/bin/env python

from __future__ import print_function
import time
import soco
import json
from flask import jsonify
""" IP Addresses for Zones """

projector_ip = "10.31.71.126"
lounge_left_ip = "10.31.69.64"
lounge_right_ip = "10.31.69.63"
pin_up_ip = "10.31.71.133"
bench_ip = "10.31.68.78"

projector = soco.core.SoCo(projector_ip)
pin_up = soco.core.SoCo(pin_up_ip)
bench = soco.core.SoCo(bench_ip)
lounge_left = soco.core.SoCo(lounge_left_ip)
lounge_right = soco.core.SoCo(lounge_right_ip)

while True:
	controller_ip = lounge_left.group.coordinator.ip_address
	controller = soco.core.SoCo(controller_ip)

	#print("left speaker coordinator ip: ", lounge_left.group.coordinator.ip_address)
	#print("left speaker coordinator name: ", lounge_left.group.coordinator.player_name)

	json_string = json.dumps(controller.get_current_transport_info())
	parsed_json = json.loads(json_string)
	playback_state = parsed_json['current_transport_state']

	log = open('/home/pi/playbackstate.txt', 'w')
	print(playback_state.lower(), file = log, end="")
#	print(playback_state.lower())

	time.sleep(0.5)

	#print("current transport info: ", json.dumps(controller.get_current_transport_info()))


from micro.service.utool import SpiderService

from thrift.transport import TSocket
from thrift.transport import TTransport
from thrift.protocol import TBinaryProtocol
from thrift.server import TServer

class SpyNewsServiceHandler:

    def spy_news_service(self, id):
        print(id)

if __name__ == '__main__':
    handler = SpyNewsServiceHandler()
    processor = SpiderService.Processor(handler)
    transport = TSocket.TServerSocket(host='127.0.0.1', port=6000)
    tfactory = TTransport.TBufferedTransportFactory()
    pfactory = TBinaryProtocol.TBinaryProtocolFactory()

    server = TServer.TSimpleServer(processor, transport, tfactory, pfactory)
    print("py thrift server start")
    server.serve()
    print("py thrift server stop")


"""Tests for the redis orm"""

import unittest
from datetime import date

from pydantic_redis.config import RedisConfig
from pydantic_redis.model import Model
from pydantic_redis.store import Store

class Book(Model):
    _primary_key_field: str = 'title'
    title: str
    author: str
    published_on: date
    in_stock: bool = True

books = [
    Book(title="Oliver Twist", author='Charles Dickens', published_on=date(year=1215, month=4, day=4),
         in_stock=False),
    Book(title="Great Expectations", author='Charles Dickens', published_on=date(year=1220, month=4, day=4)),
    Book(title="Jane Eyre", author='Charles Dickens', published_on=date(year=1225, month=6, day=4), in_stock=False),
    Book(title="Wuthering Heights", author='Jane Austen', published_on=date(year=1600, month=4, day=4)),
]

class ModelWithoutPrimaryKey(Model):
    title: str

class TestRedisOrm(unittest.TestCase):
    """
    Tests for Redis Orm
    """

    @classmethod
    def setUpClass(cls) -> None:
        cls.store = Store(name='sample', redis_config=RedisConfig(db=5), life_span_in_seconds=3600)
        cls.store.register_model(Book)

    def tearDown(self) -> None:
        """Tear down a few things"""
        keys = [f"book_%&_{book.title}" for book in books]
        self.store.redis_store.delete(*keys)

    def test_register_model_without_primary_key(self):
        """Throws error when a model without the _primary_key_field class variable set is registered"""
        self.assertRaisesRegex(AttributeError, '_primary_key_field', self.store.register_model,
                               ModelWithoutPrimaryKey)
        ModelWithoutPrimaryKey._primary_key_field = None
        self.assertRaisesRegex(Exception, 'should have a _primary_key_field', self.store.register_model,
                               ModelWithoutPrimaryKey)

    def test_bulk_insert(self):
        """Providing a list of Model instances to the insert method inserts the records in redis"""
        keys = [f"book_%&_{book.title}" for book in books]
        self.store.redis_store.delete(*keys)

        for key in keys:
            book_in_redis = self.store.redis_store.hgetall(name=key)
            self.assertEqual(book_in_redis, {})

        Book.insert(books)

        pipeline = self.store.redis_store.pipeline()
        for key in keys:
            pipeline.hgetall(name=key)
        books_in_redis = pipeline.execute()
        books_in_redis_as_models = [Book(**Book.deserialize_partially(book)) for book in books_in_redis]
        self.assertEqual(books, books_in_redis_as_models)

    def test_insert_single(self):
        """
        Providing a single Model instance
        """
        key = f"book_%&_{books[0].title}"
        book = self.store.redis_store.hgetall(name=key)
        self.assertEqual(book, {})

        Book.insert(books[0])

        book = self.store.redis_store.hgetall(name=key)
        book_as_model = Book(**Book.deserialize_partially(book))
        self.assertEqual(books[0], book_as_model)

    def test_select_default(self):
        """Selecting without arguments returns all the book models"""
        Book.insert(books)
        response = Book.select()
        sorted_books = sorted(books, key=lambda x: x.title)
        sorted_response = sorted(response, key=lambda x: x.title)
        self.assertEqual(sorted_books, sorted_response)

    def test_select_some_columns(self):
        """
        Selecting some columns returns a list of dictionaries of all books models with only those columns
        """
        Book.insert(books)
        books_dict = {book.title: book for book in books}
        columns = ['title', 'author', 'in_stock']
        response = Book.select(columns=['title', 'author', 'in_stock'])
        response_dict = {book['title']: book for book in response}

        for title, book in books_dict.items():
            book_in_response = response_dict[title]
            self.assertIsInstance(book_in_response, dict)
            self.assertEqual(sorted(book_in_response.keys()), sorted(columns))
            for column in columns:
                self.assertEqual(f"{book_in_response[column]}", f"{getattr(book, column)}")

    def test_select_some_ids(self):
        """
        Selecting some ids returns only those elements with the given ids
        """
        Book.insert(books)
        ids = [book.title for book in books[:2]]
        response = Book.select(ids=ids)
        self.assertEqual(response, books[:2])

    def test_update(self):
        """
        Updating an item of a given primary key updates it in redis
        """
        Book.insert(books)
        title = books[0].title
        new_author = 'John Doe'
        key = f"book_%&_{title}"
        old_book_data = self.store.redis_store.hgetall(name=key)
        old_book = Book(**Book.deserialize_partially(old_book_data))
        self.assertEqual(old_book, books[0])
        self.assertNotEqual(old_book.author, new_author)

        Book.update(_id=title, data={"author": "John Doe"})

        book_data = self.store.redis_store.hgetall(name=key)
        book = Book(**Book.deserialize_partially(book_data))
        self.assertEqual(book.author, new_author)
        self.assertEqual(book.title, old_book.title)
        self.assertEqual(book.in_stock, old_book.in_stock)
        self.assertEqual(book.published_on, old_book.published_on)

    def test_delete_multiple(self):
        """
        Providing a list of ids to the delete function will remove the items from redis
        """
        Book.insert(books)
        books_to_delete = books[:2]
        books_left_in_db = books[2:]

        ids_to_delete = [book.title for book in books_to_delete]
        ids_to_leave_intact = [book.title for book in books_left_in_db]

        keys_to_delete = [f"book_%&_{_id}" for _id in ids_to_delete]
        keys_to_leave_intact = [f"book_%&_{_id}" for _id in ids_to_leave_intact]

        Book.delete(ids=ids_to_delete)

        for key in keys_to_delete:
            deleted_book_in_redis = self.store.redis_store.hgetall(name=key)
            self.assertEqual(deleted_book_in_redis, {})

        pipeline = self.store.redis_store.pipeline()
        for key in keys_to_leave_intact:
            pipeline.hgetall(name=key)
        books_in_redis = pipeline.execute()
        books_in_redis_as_models = [Book(**Book.deserialize_partially(book)) for book in books_in_redis]
        self.assertEqual(books_left_in_db, books_in_redis_as_models)

if __name__ == '__main__':
    unittest.main()


from streamsx.topology.topology import Topology
import streamsx.topology.context
from sources import sklearn_sources
from estimator.estimator_model import Estimator
from sklearn.neighbors import KNeighborsClassifier

def main():
    """
    The 'Estimator' model accepts a tuple with these elements: (type, X, y), where:
       'type':  't' (for training), 'd' (for data), '' (empty string, same as 'd')
       'X':     is the data
       'y':     is the actual class of the data (only used to train the model)
    """
    training_size = 100
    num_centers = 2
    num_features = 2

    t = Topology("Estimator_Sample")
    trainingStream = t.source(sklearn_sources.Blobs(iterations=training_size, isTraining=True, centers=num_centers, n_features=num_features))
    dataStream = t.source(sklearn_sources.Blobs(centers=num_centers, n_features=num_features))
    combinedStreams = trainingStream.union({dataStream})
    predictionStream = combinedStreams.transform(Estimator(training_size, KNeighborsClassifier()))
    predictionStream.sink(print)

    streamsx.topology.context.submit("STANDALONE", t.graph)

if __name__ == '__main__':
    main()


from django.shortcuts import render
from django.urls import reverse
from tutorial.auth_helper import get_sign_in_url, get_token_from_code
import dateutil.parser

def initialize_context(request):
  context = {}

  # Check for any errors in the session
  error = request.session.pop('flash_error', None)

  if error != None:
    context['errors'] = []
    context['errors'].append(error)

  # Check for user in the session
  context['user'] = request.session.get('user', {'is_authenticated': False})
  return context

# Create your views here.
from django.shortcuts import render
from django.http import HttpResponse, HttpResponseRedirect
from tutorial.graph_helper import get_user, get_calendar_events, get_planner_tasks
from tutorial.auth_helper import get_sign_in_url, get_token_from_code, store_token, store_user, remove_user_and_token, get_token

def home(request):
  context = initialize_context(request)

  return render(request, 'tutorial/home.html', context)

def sign_in(request):
  # Get the sign-in URL
  sign_in_url, state = get_sign_in_url()
  # Save the expected state so we can validate in the callback
  request.session['auth_state'] = state
  # Redirect to the Azure sign-in page
  return HttpResponseRedirect(sign_in_url)

def callback(request):
  # Get the state saved in session
  expected_state = request.session.pop('auth_state', '')
  # Make the token request
  token = get_token_from_code(request.get_full_path(), expected_state)

  # Get the user's profile
  user = get_user(token)

  # Save token and user
  store_token(request, token)
  store_user(request, user)

  return HttpResponseRedirect(reverse('home'))

def sign_out(request):
  # Clear out the user and token
  remove_user_and_token(request)

  return HttpResponseRedirect(reverse('home'))

def calendar(request):
  context = initialize_context(request)

  token = get_token(request)

  events = get_calendar_events(token)

  if events:
    # Convert the ISO 8601 date times to a datetime object
    # This allows the Django template to format the value nicely
    for event in events['value']:
      event['start']['dateTime'] = dateutil.parser.parse(event['start']['dateTime'])
      event['end']['dateTime'] = dateutil.parser.parse(event['end']['dateTime'])

    context['events'] = events['value']

  return render(request, 'tutorial/calendar.html', context)

def tasks(request):
  context = initialize_context(request)

  token = get_token(request)

  tasks = get_planner_tasks(token)

  if tasks:
    # Convert the ISO 8601 date times to a datetime object
    # This allows the Django template to format the value nicely
    for task in tasks['value']:
      if task['startDateTime'] != None:
        task['startDateTime'] = dateutil.parser.parse(task['startDateTime'])
      if task['dueDateTime'] != None:
        task['dueDateTime'] = dateutil.parser.parse(task['dueDateTime'])

    context['tasks'] = tasks['value']

  return render(request, 'tutorial/tasks.html', context)


import itertools
import pandas as pd
import glob
from os import listdir
from os.path import isfile, join
from sklearn.preprocessing import normalize
from scipy.stats import norm
import numpy as np
import math
import sys
import warnings
# this function iterates through all files in a folder

warnings.filterwarnings("ignore")
def get_csv_files(directory):
    only_csv_files = [f for f in listdir(directory) if isfile(
        join(directory, f)) if f.endswith('.csv')]
    return only_csv_files

def normalize_csv(csvfile):
    minrows = csvfile.min(axis=1)
    maxrows = csvfile.max(axis=1)
    csv_opened2 = csvfile.sub(minrows, axis='rows')
    csv_opened3 = csv_opened2.div(maxrows-minrows, axis='rows')
    return csv_opened3*2-1

# long code at https://moonbooks.org/Articles/How-to-integrate-a-normal-distribution-in-python-/

def denominator():
    a = norm(0, 0.25).cdf(-1)
    b = norm(0, 0.25).cdf(1)
    return b-a

def find_lim1(i, r):
    return (i-r)/r

def find_lim2(i, r):
    return (i-r-1)/r

def find_quant_levels(i, r):
    lim1 = find_lim1(i, r)
    lim2 = find_lim2(i, r)
    l1a = norm(0, 0.25).cdf(lim1)
    l1b = norm(0, 0.25).cdf(lim2)
    return l1a-l1b/denominator()

def populate_quant_lengths(r):
    quant_levels = []
    for i in range(1, 2*r+1):
        quant_levels.append(find_quant_levels(i, r))
    return quant_levels

def populate_quant_levels(quant_lengths):
    quant_levels = []
    sums = -1
    for i in quant_lengths:
        sums += 2*i
        quant_levels.append(sums)
    # print(quant_levels)
    return quant_levels

# quantize code
# pd.np.digitize( a.col4, bins = [0.3,0.6,0.9 ]  )

def quantize(normalized, quant_levels):
    ans = np.digitize(normalized, bins=quant_levels, right=True)
    return ans

def rolling_win_gen_starts(shift, n):
    ans = [i for i in range(0, n, shift)]
    return ans

def rolling_win_shift(shift, win, data, i):
    start = 0  # it's the index of your 1st valid value.
    # print(data.iloc[i])
    # res = data.iloc[i].rolling(win).mean()[start::shift]
    # print(data.size)
    output = []
    while(start+win<data.size):
    # if(start+shift+1<data.size):
        res = data.iloc[i][start:start+win]
        start+=shift
        resstr  = ""
        for index, value in res.items():
            resstr+=str(value)
        if(len(resstr) == win):
            output.append(resstr)
        elif(len(resstr) < win and len(resstr)>0):
            output.append("nan")

    # res=""
    # for j in range(start, start+shift+1):
    #     # print(i)
    #     print(type(data))
    #     print(type(data.iloc[i,j]))
    #     print(data.iloc[i,j])
    #     res += str(data.iloc[i,j])
    #     break
    # for i in data
    # res =
    # print("res is")
    # print(res)
    # print("done")

    # print(resstr)
    # print(output)
    # print(len(output))
    return output
    # return np.array(res), start

def rolling_all_rows(data, window, shift):
    output = []
    for i in range(len(data)):
        tempout = rolling_win_shift(data=data, win=window, shift=shift, i=i)
        # print("tempout is")
        # print(tempout)
        output.append(rolling_win_shift(data=data, win=window, shift=shift, i=i))

    # output = np.array(output)
    # print("output is")
    # print(output)
    return output

def fill_sensor_id(data, shift):
    #     print(data)
    rows = data.shape[0]
    cols = math.ceil(data.shape[1]/shift)
    # print(cols)
    sensor_id = [[y]*cols for y in range(0, rows)]
#     sensor_id.flatten()
    return sensor_id

def flatten(data):
    return list(itertools.chain(*data))

def process_rolling(rolling):
    # ans = []
    # for i in rolling:
    #     ans.append(i[0].tolist())
    # ans = flatten(ans)
    # return ans
    return rolling

def round_rolling(rolling):
    # ans = []
    # for i in rolling:
    #     if(not math.isnan(i)):
    #         ans.append(math.ceil(i))
    #     else:
    #         ans.append(float("nan"))
    # return ans
    return rolling
# a = fill_sensor_id(csv10)

def process(filename, csv_file, shift, resolution, window_len):
    # normalized the data
    normalized = normalize_csv(csv_file)
    # find the quant lengths
    # print(normalized)
    r = resolution
    quant_lengths = populate_quant_lengths(r)
    # find the quant levels
    quant_levels = populate_quant_levels(quant_lengths)
    quant_levels.insert(0, -1.1)
    quant_levels[-1]=1.1
    # print(quant_levels)
    # quantized the values based on bins
    quantized = quantize(normalized, quant_levels)
    quantized = pd.DataFrame(data=quantized)
    rolling = rolling_all_rows(quantized, window_len, shift)
    rolling = process_rolling(rolling)
    rolling = round_rolling(rolling)
    rolling = np.array(rolling)
    rolling = rolling.ravel()
    #rolling = rolling[0]
#     print(rolling[0])
#     print(rolling)
    # print("no of rows is")
    rows = csv_file.shape[0]
    # print(rows)
    # print("no of cols is")
    cols = csv_file.shape[1]
    # print(cols)
    starts = (rolling_win_gen_starts(shift, n=normalized.shape[1]))*rows
    files = [filename]*len(starts)
    # print(len(rolling))
    # print(len(starts))
    # print(len(files))
    sensor_id = fill_sensor_id(csv_file, shift)
    sensor_id = flatten(sensor_id)
    # print(len(sensor_id))
    # print(sensor_id)
    filename = filename.replace('.csv', '')
    f = open("output/"+filename+".wrd", "w")
    # print(len(files))
    # print(len(sensor_id))
    # print(len(starts))
    # print(len(rolling))
    for i in range(len(files)):
        # print(i)
        # print(rolling[i])
        # if(not math.isnan(rolling[i])):
        if(not rolling[i] == "nan"):
            f.write(str(files[i]) + " ")
            f.write(str(sensor_id[i]+1)+" ")
            f.write(str(starts[i])+" ")
            f.write(str(rolling[i]))
            f.write("\n")
    f.close()

# w -> windows length
# s -> shift
# r -> resolution
# ans = []

def gesture_words(directory, shift, window_len, resolution):
    files = get_csv_files(directory)
    for file in files:
        csv_opened = pd.read_csv(directory+file, header=None)
        process(csv_file=csv_opened, shift=shift, filename=file, resolution=resolution, window_len = window_len)

# directory = "../Z/"
# gesture_words(directory)

if __name__ == "__main__":
    directory = sys.argv[1]
    shift = int(sys.argv[2])
    window_len = int(sys.argv[3])
    resolution = int(sys.argv[4])
    print("wait a few seconds")
    gesture_words(directory, shift, window_len, resolution)
    print("done, check your gesture files now")
# command to run  - python task1.py ../Z/ 2 3 3


try:
    from flask import Flask, request
    from flask_restful import Resource, Api, reqparse
    from flask import app,Flask, request
    from flask_restful import Resource, Api, reqparse
    import datetime
    import json
    import os
    import sys
    import numpy as np
    import ssl
    import elasticsearch
    from elasticsearch import Elasticsearch
    import tensorflow as tf
    import tensorflow_hub as hub
except Exception as e:
    print("Some Modules are Missing {}".format(e))

app = Flask(__name__)
api = Api(app)

# ------------------------------------------------------------
global ELK_ENDPOINT
global ELK_INDEX
global es

ELK_ENDPOINT =  "http://localhost:9200/"
ELK_INDEX    =  "final_project"

es = Elasticsearch(ELK_ENDPOINT, timeout=300)
ssl._create_default_https_context = ssl._create_unverified_context
# ------------------------------------------------------------

class Controller(Resource):

    def __init__(self):
        self.what = parser.parse_args().get('what', None)
        self.name =  parser.parse_args().get('name', None)
        self.categories = parser.parse_args().get('categories', None)

    def get(self):
        """
        Return the JSON Response fROM ELK
        :return:
        """
        print("IN")
        try:
            # Step 1: Create a instance of class
            helper  = ElasticSearchQuery(size=100, BucketName="MyBuckets")

            print("WHAT ")

            if self.what is not None:
                if len(self.what) >= 2:
                    query = helper.match(field="name", value=self.what, operation='must')

            if self.name is not None:
                if len(self.name) >= 2:
                    query = helper.match_phrase(field="name", value=self.name, operation='filter')

            if self.categories is not None:
                if len(self.categories) >= 2:
                    query = helper.match_phrase(field="categories", value=self.categories, operation='filter')

            # Aggreation Query in ELK
            helper.add_aggreation(aggregate_name="Name", field="name.keyword",type='terms',sort='desc', size=5)
            helper.add_aggreation(aggregate_name="PrimaryCategories", field="primaryCategories.keyword",type='terms',sort='desc', size=5)
            helper.add_aggreation(aggregate_name="Categories", field="categories.keyword",type='terms',sort='desc', size=5)

            query = helper.complete_aggreation()
            es = Elasticsearch(timeout=600, hosts=ELK_ENDPOINT)
            res = es.search(index=ELK_INDEX,
                            size=4,
                            body=query,
                            request_timeout=55)

            # -----------------------------------------------------
            # convert what into vector
            helper_token = Tokens(word=self.what)
            token = helper_token.token()

            similar_query = {
                "size":30,
                "query":{
                    "script_score":{
                        "query":{
                            "match_all":{

                            }
                        },
                        "script":{
                            "source":"cosineSimilarity(params.query_vector, 'name_vector') + 1.0",
                            "params":{
                                "query_vector":token
                            }
                        }
                    }
                }
            }

            res_similar = es.search(index=ELK_INDEX,
                            size=30,
                            body=similar_query,
                            request_timeout=55)

            return {
                       "result":res,
                        "similar":res_similar
                   },200

        except Exception as e:
            print(e)
            return {"Message":"Error : {} ".format(e)},500

class AutoComplete(Resource):
    def __init__(self):
        self.what =  parser.parse_args().get('what', None)
        self.baseQuery ={
            "_source": [],
            "size": 0,
            "min_score": 0.5,
            "query": {
                "bool": {
                    "must": [
                        {
                            "match_phrase_prefix": {
                                "name": {
                                    "query": "{}".format(self.what)
                                }
                            }
                        }
                    ],
                    "filter": [],
                    "should": [],
                    "must_not": []
                }
            },
            "aggs": {
                "auto_complete": {
                    "terms": {
                        "field": "name.keyword",
                        "order": {
                            "_count": "desc"
                        },
                        "size": 25
                    }
                }
            }
        }

    def get(self):

        """
        This is Auto Complete
        :return: Json
        """

        es = Elasticsearch(timeout=600, hosts=ELK_ENDPOINT)
        res = es.search(index=ELK_INDEX, size=0, body=self.baseQuery)
        return res

parser = reqparse.RequestParser()
parser.add_argument("what", type=str, required=False, help="This is Required Parameters ")
parser.add_argument("name", type=str, required=False, help="This is filter name ")
parser.add_argument("categories", type=str, required=False, help="Filter for Categories")

api.add_resource(Controller, '/search')
api.add_resource(AutoComplete, '/autocomplete')

# -------- DONT TOUCH-------
class ElasticSearchQuery(object):

    def __init__(self, size=10, BucketName=None, source=[], min_score=0.5):

        """Constructor """

        self.size = size
        self.BucketName = BucketName
        self.min_score = min_score
        self.source = source
        self.baseQuery = {
            "_source":source,
            "size":self.size,
            "min_score": self.min_score,
            "query": {
                "bool": {
                    "must": [],
                    "filter": [],
                    "should": [],
                    "must_not": []
                }
            }
        }
        self.GeoBaseQuery =  {
            "_source":self.source,
            "size":self.size,
            "query": {
                "bool" : {
                    "must":{ "match_all":{}},
                    "should":[],
                    "filter": {}
                }
            }
        }
        self.aggtem = []
        self.base_higghlight = {
            "pre_tags":[
                "<em>"
            ],
            "post_tags":[
                "</em>"
            ],
            "tags_schema":"styled",
            "fields":{

            }
        }

    def match(self,field=None, value=None, boost=None, operation='should',analyzer=None):

        _ = {
            "match": {
                field: {
                    "query": value
                }
            }
        }
        if boost is None:
            self.baseQuery["query"]["bool"][operation].append(_)

        if boost is not None:
            _["match"][field]["boost"] = boost

        if analyzer is not None:
            _["match"][field]["analyzer"] = analyzer

        self.baseQuery["query"]["bool"][operation].append(_)

        return self.baseQuery

    def match_phrase(self, field=None, value=None, boost=None, operation='should',analyzer=None):
        _ = {
            "match_phrase": {
                field: {
                    "query": value
                }
            }
        }

        if boost is None:
            self.baseQuery["query"]["bool"][operation].append(_)

        if boost is not None:
            _["match_phrase"][field]["boost"] = boost

        if analyzer is not None:
            _["match_phrase"][field]["analyzer"] = analyzer

        self.baseQuery["query"]["bool"][operation].append(_)

        return self.baseQuery

    def terms(self,field=None, value=None, boost=None, operation='should'):

        _ = {"term" :{
            field : value
        }
        }
        self.baseQuery["query"]["bool"][operation].append(_)

    def add_aggreation(self, aggregate_name=None,
                       field=None,
                       type='terms',
                       sort='desc',
                       size = 10):

        _ = {
            aggregate_name:{
                type: {
                    "field": field,
                    "order" :
                        {"_count" :
                             sort
                         },
                    "size": size

                }
            }
        }
        self.aggtem.append(_)

    def complete_aggreation(self):
        _ = {
            "aggs":{

            }
        }
        for item in self.aggtem:
            for key,value in item.items():
                _["aggs"][key] = value
        self.baseQuery["aggs"] = _["aggs"]
        return self.baseQuery

    def add_geoqueries(self, radius=None, lat=None, lon=None, field=None, operation='filter'):
        radius = str(radius) + "mi"
        _ = {
            "geo_distance" : {
                "distance": radius,
                field : {
                    "lat": lat,
                    "lon": lon
                }
            }}
        self.baseQuery["query"]["bool"][operation].append(_)
        return self.baseQuery

    def wildcard(self,field=None, value=None, boost=None, operation=None):
        _ =  {
            "wildcard":{
                field:{
                    "value":value

                }
            }
        }
        if boost is None:
            self.baseQuery["query"]["bool"][operation].append(_)
            return self.baseQuery
        else:
            _["wildcard"][field]["boost"] = boost
            self.baseQuery["query"]["bool"][operation].append(_)
            return self.baseQuery

    def exists(self,field=None, operation="must"):

        _ = {
            "exists": {
                "field": field
            }
        }
        self.baseQuery["query"]["bool"][operation].append(_)
        return  self.baseQuery

    def query_string(self, default_field=None, query=None, operation="should"):
        _ = {
            "query_string":{
                "default_field": default_field,
                "query":"{}".format(query)
            }
        }
        self.baseQuery["query"]["bool"][operation].append(_)
        return self.baseQuery

    def add_geo_aggreation(self, field=None,lat=None, lon=None, aggregate_name='distance'):
        self.baseQuery.get("aggs")[aggregate_name] = {
            "geo_distance" : {
                "field" : field,
                "origin" : "{},{}".format(lat, lon),
                "unit" : "mi",
                "ranges" : [
                    { "to" : 0 },
                    { "from" : 0, "to" : 25 },
                    { "from" : 25, "to" : 50 },
                    { "from" : 50, "to" : 75 },
                    { "from" : 75, "to" : 100 },
                    { "from" : 100 }
                ]
            }}
        return self.baseQuery

    def match_phrase_prefix(self, field=None, value=None, boost=None, operation='should',analyzer=None):
        _ = {
            "match_phrase_prefix": {
                field: {
                    "query": value
                }
            }
        }

        if boost is not None:
            _["match_phrase_prefix"][field]["boost"] = boost
        if analyzer is not None:
            _["match_phrase_prefix"][field]["analyzer"] = analyzer
        self.baseQuery["query"]["bool"][operation].append(_)
        return self.baseQuery

    def autocomplete_1(self, field=None,size=25, value=None, sort='des', operation='must'):
        query = self.match_phrase_prefix(field=field,value=value, operation=operation)
        query  =self.add_aggreation(field=field, size=size, sort=sort,aggregate_name='auto_complete' )
        query = self.complete_aggreation()
        return query

class Tokens(object):

    def __init__(self, word):
        self.word = word

    def token(self):
        module_url = os.getcwd()
        #path = os.path.join(module_url, "API/Compute")
        embed = hub.KerasLayer(module_url)
        embeddings = embed([self.word])
        x = np.asarray(embeddings)
        x = x[0].tolist()
        return x
# ------------

if __name__ == '__main__':
    app.run(debug=True)


# Imports
import asyncio

import discord
from discord.ext import commands

import Config

class Misc(commands.Cog):

    def __init__(self, bot):
        self.bot = bot

    async def get_prefix(self, ctx):
        if ctx.message.guild == None:
            return Config.PREFIX
        else:
            prefixes = await Config.CLUSTER["servers"]["prefixes"].find_one({"_id": ctx.message.guild.id})
            if prefixes == None:
                return Config.PREFIX
            else:
                return prefixes["prefix"]

    @commands.group()
    async def help(self, ctx):
        if ctx.invoked_subcommand == None:
            embed = discord.Embed(
                title = "Help",
                color = Config.MAINCOLOR
            )
            prefix = await self.get_prefix(ctx)
            embed.add_field(name = f"**{prefix}help [Command]**", value = "View this menu or information about a command.", inline = False)
            embed.add_field(name = f"**{prefix}invite**", value = "Get a link to invite me.", inline = False)
            embed.add_field(name = f"**{prefix}support**", value = "Get a link to the support server.", inline = False)
            embed.add_field(name = f"**{prefix}vote**", value = "Get a link to vote for the bot.", inline = False)
            embed.add_field(name = f"**{prefix}ping**", value = "View the bot's latency.", inline = False)
            embed.add_field(name = f"**{prefix}stop**", value = "Have the bot stop responding to you.", inline = False)
            embed.add_field(name = f"**{prefix}suggest <Suggestion>**", value = "Suggest a trigger for the Positive Peter community to review.", inline = False)
            if ctx.guild != None:
                if ctx.author.guild_permissions.manage_guild == True:
                    embed.add_field(name = f"**{prefix}config <Configuration> <Value>**", value = "Edit a server side configuration value.", inline = False)
                    embed.add_field(name = f"**{prefix}prefix <New Prefix>**", value = "Edit the server side prefix.", inline = False)
            if ctx.author.id in Config.OWNERIDS:
                embed.add_field(name = f"**{prefix}restart [Cog]**", value = "Restart the entire bot or a singular cog.", inline = False)
                embed.add_field(name = f"**{prefix}block**", value = "Block a user from sending suggestions.", inline = False)
                embed.add_field(name = f"**{prefix}unblock**", value = "Unblock a user from sending suggestions.", inline = False)
            embed.add_field(name = "\u200b", value = "**[Invite](https://discordapp.com/oauth2/authorize?client_id=649535694145847301&permissions=26688&scope=bot)\n[Vote](https://top.gg/bot/649535694145847301/vote)\n[Support](https://discord.gg/VwMWj2B)**", inline = False)
            await ctx.send(embed = embed)

    @help.command(aliases = ["help"])
    async def _help(self, ctx):
        prefix = await self.get_prefix(ctx)
        embed = discord.Embed(
            title = f"Help [Command]",
            description = "View this menu or information about a command.",
            color = Config.MAINCOLOR
        )
        embed.set_author(name = "Help")
        embed.add_field(name = "**Example Usage**", value = f"`{prefix}help suggest`")
        await ctx.send(embed = embed)

    @help.command()
    async def invite(self, ctx):
        embed = discord.Embed(
            title = "Invite",
            description = "Get a link to invite me.",
            color = Config.MAINCOLOR
        )
        embed.set_author(name = "Help")
        await ctx.send(embed = embed)

    @help.command()
    async def support(self, ctx):
        embed = discord.Embed(
            title = "Support",
            description = "Get a link to the support server.",
            color = Config.MAINCOLOR
        )
        embed.set_author(name = "Help")
        await ctx.send(embed = embed)

    @help.command()
    async def vote(self, ctx):
        embed = discord.Embed(
            title = "Vote",
            description = "Get a link to vote for the bot.",
            color = Config.MAINCOLOR
        )
        embed.set_author(name = "Help")
        await ctx.send(embed = embed)

    @help.command()
    async def ping(self, ctx):
        embed = discord.Embed(
            title = "Ping",
            description = "View the bot's latency.",
            color = Config.MAINCOLOR
        )
        embed.set_author(name = "Help")
        await ctx.send(embed = embed)

    @help.command()
    async def stop(self, ctx):
        embed = discord.Embed(
            title = "Stop",
            description = "Have the bot stop responding to you.",
            color = Config.MAINCOLOR
        )
        embed.set_author(name = "Help")
        await ctx.send(embed = embed)

    @help.command()
    async def suggest(self, ctx):
        embed = discord.Embed(
            title = "Suggest <Suggestion>",
            description = "Suggest a trigger for the Positive Peter community to review.",
            color = Config.MAINCOLOR
        )
        embed.set_author(name = "Help")
        prefix = await self.get_prefix(ctx)
        embed.add_field(name = "**Example Usage**", value = f"`{prefix}suggest I want to die`")
        await ctx.send(embed = embed)

    @help.command()
    @commands.has_permissions(manage_guild = True)
    async def config(self, ctx):
        embed = discord.Embed(
            title = "Config <Configuration> <Value / Reset>",
            description = "Edit a server side configuration value.",
            color = Config.MAINCOLOR
        )
        embed.set_author(name = "Help")
        prefix = await self.get_prefix(ctx)
        embed.add_field(name = "**Example Usage**", value = f"`{prefix}config hotline 123-456-7890`")
        await ctx.send(embed = embed)

    @help.command()
    @commands.has_permissions(manage_guild = True)
    async def prefix(self, ctx):
        embed = discord.Embed(
            title = "Prefix <New Prefix / Reset>",
            description = "Edit a server side configuration value.",
            color = Config.MAINCOLOR
        )
        embed.set_author(name = "Help")
        prefix = await self.get_prefix(ctx)
        embed.add_field(name = "**Example Usage**", value = f"`{prefix}prefix !`")
        await ctx.send(embed = embed)

def setup(bot):
    bot.add_cog(Misc(bot))


import matplotlib.pyplot as plt

def Serie_Leibniz_pi(n):
    return ((-1)**n) / (2*n+1)

n_termini = []
pi_termini = []

Somma_n = 0

for n in range(0, 30):
    a_n = Serie_Leibniz_pi(n)
    Somma_n += a_n
    pi_n = 4 * a_n

    n_termini.append(n)
    pi_termini.append(pi_n)

plt.plot(n_termini, pi_termini)
plt.grid()
plt.title("Serie di Leibniz per π")
plt.xlabel("n")
plt.ylabel("π(n)")
plt.show()