import os
import pytest
from pathlib import Path
from tempfile import TemporaryDirectory
import pandas as pd
from feast import FeatureStore
from tests.generators import Generator, csv, parquet, start_date, end_date
from datetime import datetime, timezone
from yummy import select_all

def entity_df_all():
    return select_all(datetime(2021, 10, 2, 1, 0, 0, tzinfo=timezone.utc))

def e2e_basic(feature_store: FeatureStore, tmp_dir: TemporaryDirectory, backend: str, entity_df: pd.DataFrame):
    """
    This will test all backends with basic data stores (parquet and csv)
    """
    feature_store.config.offline_store.backend = backend

    csv_fv, csv_fv_name = csv(tmp_dir)
    parquet_fv, parquet_fv_name = parquet(tmp_dir)

    entity = Generator.entity()
    feature_store.apply([entity, csv_fv, parquet_fv])

    feature_vector = feature_store.get_historical_features(
        features=[
            f"{csv_fv_name}:f0",
            f"{parquet_fv_name}:f0",
        ], entity_df=entity_df, full_feature_names=True
    ).to_df()

    feature_store.materialize(start_date=start_date, end_date=end_date)

    fv = feature_vector[feature_vector.entity_id.astype('int') == 1].to_dict(orient="records")[0]
    csv_f0 = float(fv[f"{csv_fv_name}__f0"])
    parquet_f0 = float(fv[f"{parquet_fv_name}__f0"])

    assert(csv_f0 is not None)
    assert(parquet_f0 is not None)

    ofv = feature_store.get_online_features(
        features=[
            f"{csv_fv_name}:f0",
            f"{parquet_fv_name}:f0",
        ],
        entity_rows = [{"entity_id": 1}],
        full_feature_names=True,
    ).to_df().to_dict(orient="records")

    assert(abs(ofv[0]['fv_csv__f0'] - csv_f0) < 1e-6)
    assert(abs(ofv[0]['fv_parquet__f0'] - parquet_f0) < 1e-6)

@pytest.mark.polars
def test_e2e_basic_polars(feature_store: FeatureStore, tmp_dir: TemporaryDirectory):
    """
    This will test polars backend with basic data stores (parquet and csv)
    """
    entity_df = Generator.entity_df()
    e2e_basic(feature_store, tmp_dir, "polars", entity_df)

@pytest.mark.polars
def test_e2e_basic_select_all_polars(feature_store: FeatureStore, tmp_dir: TemporaryDirectory):
    """
    This will test polars backend with basic data stores (parquet and csv)
    """
    e2e_basic(feature_store, tmp_dir, "polars", entity_df_all())

@pytest.mark.dask
def test_e2e_basic_dask(feature_store: FeatureStore, tmp_dir: TemporaryDirectory):
    """
    This will test dask backend with basic data stores (parquet and csv)
    """
    entity_df = Generator.entity_df()
    e2e_basic(feature_store, tmp_dir, "dask", entity_df)

@pytest.mark.dask
def test_e2e_basic_select_all_dask(feature_store: FeatureStore, tmp_dir: TemporaryDirectory):
    """
    This will test dask backend with basic data stores (parquet and csv)
    """
    e2e_basic(feature_store, tmp_dir, "dask", entity_df_all())

@pytest.mark.ray
def test_e2e_basic_ray(feature_store: FeatureStore, tmp_dir: TemporaryDirectory):
    """
    This will test ray backend with basic data stores (parquet and csv)
    """
    entity_df = Generator.entity_df()
    e2e_basic(feature_store, tmp_dir, "ray", entity_df)

@pytest.mark.ray
def test_e2e_basic_select_all_ray(feature_store: FeatureStore, tmp_dir: TemporaryDirectory):
    """
    This will test ray backend with basic data stores (parquet and csv)
    """
    e2e_basic(feature_store, tmp_dir, "ray", entity_df_all())

@pytest.mark.parametrize("backend", ["polars", "dask", "ray"])
@pytest.mark.nospark
def test_e2e_basic_nospark(feature_store: FeatureStore, tmp_dir: TemporaryDirectory, backend: str):
    """
    This will test all backends with basic data stores (parquet and csv)
    """
    entity_df = Generator.entity_df()
    e2e_basic(feature_store, tmp_dir, backend, entity_df)

@pytest.mark.parametrize("backend", ["spark", "polars", "dask", "ray"])
@pytest.mark.spark
def test_e2e_basic_spark(feature_store: FeatureStore, tmp_dir: TemporaryDirectory, backend: str):
    """
    This will test all backends with basic data stores (parquet and csv)
    """
    entity_df = Generator.entity_df()
    e2e_basic(feature_store, tmp_dir, backend, entity_df)

@pytest.mark.parametrize("backend", ["spark", "polars", "dask", "ray"])
@pytest.mark.spark
def test_e2e_basic_select_all_spark(feature_store: FeatureStore, tmp_dir: TemporaryDirectory, backend: str):
    """
    This will test all backends with basic data stores (parquet and csv)
    """
    e2e_basic(feature_store, tmp_dir, backend, entity_df_all())


# Python中的高阶函数reduce配合lambda
import functools

print('Python中的高阶函数reduce配合lambda')

'''
    reduce函数作用: 会对参数序列中的元素进行累计处理,直到处理完参数列表中所有参数
    函数将一个数据集合中所有数据进行下列操作:
        1.用传给reduce中函数function(有两个参数),先对集合中的第1,2个元素进行操作
        2.用得到的结果再与第三个数据用function函数运算,最后得到一个结果
    注意: 这个reduce()函数不能直接使用,需要导入一个functools,使用functools.reduce()进行调用
'''

l1 = [1, 2, 3, 4, 5]
def add(a, b):
    return a + b

result = functools.reduce(add, l1)
# reduce过程分析: 1+2=3, 3+3=6, 6+4=10, 10+5=15
print(result)
print('------------------------------')

# reduce()和lambda配合拼接字符串
l2 = ['h', 'e', 'l', 'l', 'o']
s1 = functools.reduce(lambda a,b : a + b, l2)
print(s1)

s2 = functools.reduce(lambda a,b : a.upper() + b.upper(), l2)
print(s2)


#-*-encoding:utf-8-*-
from sklearn.datasets import load_iris,fetch_20newsgroups
from sklearn.model_selection import train_test_split
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.naive_bayes import MultinomialNB
from sklearn.metrics import classification_report

li = load_iris()
# print(li.data)
# print(li.target)

'''
训练集
x_train
y_train
测试集
x_text
y_text
'''
x_train, x_text, y_train, y_text= train_test_split(li.data ,li.target , test_size = 0.25)

# print("训练集数据" ,x_train,y_train)
# print("测试集数据" , x_text,y_text)

# 拿出数据
news = fetch_20newsgroups(subset="all")
# 切分
x_train, x_text, y_train, y_text = train_test_split(news.data, news.target, test_size=0.25)

# 对数据进行过特征性的提取
tf = TfidfVectorizer()
x_train = tf.fit_transform(x_train)
print(tf.get_feature_names())
x_text = tf.transform(x_text)

# 进行朴素贝叶斯算法的 预测

nb = MultinomialNB(alpha=1.0)

nb.fit(x_train,y_train)
# 预测
_pred = nb.predict(y_text)

# 准确度
nb.score(x_text ,y_text)

# 精确率和召回率

classification_report(y_text ,_pred , target_names=news.target_names)


from setuptools import setup, find_namespace_packages, Extension
import numpy
import sys
import extnp

ext_cpp = Extension(
    # C++ extension module
    "extnp.algo.algo1_ext",
    include_dirs=[
        # Include Python header file
        *sys.path,
        # Include Numpy header file
        numpy.get_include(),
    ],
    # Relative path to C++ source files
    sources=[
        "extnp/algo/src/wrapper.cpp",
    ],
    # Platform-dependent compile args
    extra_compile_args=["/O2", "/std:c++17"],  # Windows
    language="c++",
)

setup(
    name="extnp",
    version=extnp.__version__,
    description=extnp.__description__,
    long_description=open("README.md").read(),
    long_description_content_type="text/markdown",
    author=extnp.__author__,
    author_email=extnp.__author_email__,
    url=extnp.__github_url__,
    packages=find_namespace_packages(),
    package_data={
        "": ["LICENSE", "README.md"],
    },
    install_requires=["numpy"],
    license="MIT",
    ext_modules=[ext_cpp],
)


import json

DEGREE = 0
WEIGHT = 1
ROOT_PATH = "../builder/APS/"

def update_weight(vertices, v_i, e_w):
    if v_i not in vertices:
        vertices[v_i] = [0, 0.]
    vertices[v_i][WEIGHT] += e_w

def update_degree(vertices, v_i):
    if v_i not in vertices:
        vertices[v_i] = [0, 0.]
    vertices[v_i][DEGREE] += 1

def get_static_info(graph_file_path):
    vertices = {}
    edges_count = 0
    with open(graph_file_path) as graph:
        first_line = True
        for edge in graph:
            if not first_line:
                v_i, v_j, e_w = edge.rstrip("\r\n").split(",")
                edges_count += 1
                e_w = float(e_w)
                v_i = int(v_i)
                v_j = int(v_j)
                # different authors worked with
                update_degree(vertices, v_i)
                update_degree(vertices, v_j)
                # useful to count works made by author
                update_weight(vertices, v_i, e_w)
                update_weight(vertices, v_j, e_w)
            if first_line:
                first_line = False
    vertices_count = len(vertices.keys())
    return vertices, vertices_count, edges_count

def get_avg_degree(vertices_count, edges_count):
    return float(edges_count)/(2*vertices_count)

def load_all_graphs(files_path):
    files_path_list = json.load(open(files_path))
    for file_path in files_path_list:
        vertices, n_vertices, m_edges = get_static_info(ROOT_PATH+file_path)
        output_path = file_path.replace(".json", "_VERTICES_INFO.json")
        output_file = open(ROOT_PATH+output_path, "w+")
        output_file.write(json.dumps(vertices))
        output_file.close()

if __name__ == "__main__":
    citation_dir = ROOT_PATH+"output/citation_graphs/files.json"
    coauthorship_dir = ROOT_PATH+"output/coauthorship_graphs/files.json"
    #load_all_graphs(coauthorship_dir)
    load_all_graphs(citation_dir)


import falcon_kit.mains.fasta_filter as mod
import pytest
from falcon_kit import FastaReader
import functools
import helpers
import pytest
import os
import sys
import io
import json

def test_help():
    try:
        mod.main(['prog', '--help'])
    except SystemExit:
        pass

#############################
### Define the test data. ###
#############################
fasta_tests = {}
expected_tests = {}
expected_general_median_tests = {}
expected_internal_median_tests = {}
expected_longest_tests = {}

fasta_tests[1] = ''
expected_tests[1] = ''

fasta_tests[2] = """\
>synthetic/1/0_500
GATTACA
>synthetic/2/0_500
GATTACA
>synthetic/3/0_500
GATTACA
>synthetic/4/0_500
GATTACA
>synthetic/5/0_500
GATTACA
"""
expected_tests[2] = fasta_tests[2]

fasta_tests[3] = """\
>synthetic/1/0_500
GATTACAGATTACA
>synthetic/1/0_500
GATTACAGATTACAGATTACAGATTACA
>synthetic/1/0_500
GATTACAGATTACAGATTACA
>synthetic/1/500_1000
GATTACAGATTACAGATTACAGATTACA
>synthetic/1/0_500
GATTACA
"""
expected_tests[3] = """\
>synthetic/1/0_500
GATTACAGATTACAGATTACA
"""

expected_longest_tests[3] = """\
>synthetic/1/0_500
GATTACAGATTACAGATTACAGATTACA
"""

fasta_tests[4] = """\
>synthetic/2/0_1
A
>synthetic/3/0_1
A
>synthetic/1/0_5
ACGTA
>synthetic/1/5_7
AC
>synthetic/1/7_10
ACG
>synthetic/1/10_14
ACGT
>synthetic/1/14_15
A
>synthetic/4/0_1
A
>synthetic/5/0_2
AC
>synthetic/5/2_5
ACG
>synthetic/5/5_9
ACGT
>synthetic/5/9_14
ACGTA
"""
expected_tests[4] = """\
>synthetic/2/0_1
A
>synthetic/3/0_1
A
>synthetic/1/7_10
ACG
>synthetic/4/0_1
A
>synthetic/5/5_9
ACGT
"""

fasta_tests[5] = """\
@synthetic/1/0_500
A
@synthetic/2/0_500
A
@synthetic/3/0_500
A
@synthetic/4/0_500
A
@synthetic/5/0_500'
A
"""
# That is invalid fasta.
expected_tests[5] = None    # The module under test should throw on the above input

fasta_tests[6] = """\
>synthetic/5/2_5
ACG
>synthetic/2/0_1
A
>synthetic/3/0_1
A
>synthetic/1/0_5
ACGTA
>synthetic/1/10_14
ACGT
>synthetic/1/14_15
A
>synthetic/4/0_1
A
>synthetic/5/0_2
AC
>synthetic/5/5_9
ACGT
>synthetic/5/9_14
ACGTA
>synthetic/1/5_7
AC
>synthetic/1/7_10
ACG
"""
expected_tests[6] = """\
>synthetic/2/0_1
A
>synthetic/3/0_1
A
>synthetic/4/0_1
A
>synthetic/5/5_9
ACGT
>synthetic/1/7_10
ACG
"""

# Test ccs input
fasta_tests[8] = """\
>synthetic/1/ccs
GATTACA
"""
expected_tests[8] = """\
>synthetic/1/0_7
GATTACA
"""

# Specific for the internal-median command.
fasta_tests[7] = """\
>synthetic/1/0_3
AAA
>synthetic/1/3_7
ACAC
>synthetic/1/7_10
AAA
>synthetic/2/0_3
TTT
>synthetic/2/3_8
TTTGG
>synthetic/3/0_5
TTTGG
>synthetic/3/5_8
TTT
>synthetic/4/0_5
CCCCC
"""
expected_general_median_tests[7] = """\
>synthetic/1/7_10
AAA
>synthetic/2/3_8
TTTGG
>synthetic/3/0_5
TTTGG
>synthetic/4/0_5
CCCCC
"""
expected_internal_median_tests[7] = """\
>synthetic/1/3_7
ACAC
>synthetic/2/3_8
TTTGG
>synthetic/3/0_5
TTTGG
>synthetic/4/0_5
CCCCC
"""

whitelist_tests_general_median = [
    {   'fasta': fasta_tests[4],
        'whitelist': set(),
        'expected': expected_tests[4]
    },
    {   'fasta': fasta_tests[4],
        'whitelist': None,
        'expected': expected_tests[4]
    },
    {   'fasta': fasta_tests[4],
        'whitelist': set(['synthetic/2', 'synthetic/4', 'synthetic/3']),
        'expected': """\
>synthetic/2/0_1
A
>synthetic/3/0_1
A
>synthetic/4/0_1
A
"""
    },
    {   'fasta': fasta_tests[4],
        'whitelist': set(['synthetic/2', 'synthetic/3', 'synthetic/1', 'synthetic/4', 'synthetic/5']),
        'expected': expected_tests[4]
    },
]

whitelist_tests_longest = [
    {   'fasta': fasta_tests[4],
        'whitelist': set(),
        'expected': """\
>synthetic/2/0_1
A
>synthetic/3/0_1
A
>synthetic/1/0_5
ACGTA
>synthetic/4/0_1
A
>synthetic/5/9_14
ACGTA
"""
    },
    {   'fasta': fasta_tests[4],
        'whitelist': None,
        'expected': """\
>synthetic/2/0_1
A
>synthetic/3/0_1
A
>synthetic/1/0_5
ACGTA
>synthetic/4/0_1
A
>synthetic/5/9_14
ACGTA
"""
    },
    {   'fasta': fasta_tests[4],
        'whitelist': set(['synthetic/2', 'synthetic/4', 'synthetic/3']),
        'expected': """\
>synthetic/2/0_1
A
>synthetic/3/0_1
A
>synthetic/4/0_1
A
"""
    },
    {   'fasta': fasta_tests[4],
        'whitelist': set(['synthetic/2', 'synthetic/3', 'synthetic/1', 'synthetic/4', 'synthetic/5']),
        'expected': """\
>synthetic/2/0_1
A
>synthetic/3/0_1
A
>synthetic/1/0_5
ACGTA
>synthetic/4/0_1
A
>synthetic/5/9_14
ACGTA
"""
    },
]

whitelist_tests_pass = [
    {   'fasta': fasta_tests[4],
        'whitelist': set(),
        'expected': fasta_tests[4]
    },
    {   'fasta': fasta_tests[4],
        'whitelist': None,
        'expected': fasta_tests[4]
    },
    {   'fasta': fasta_tests[4],
        'whitelist': set(['synthetic/2', 'synthetic/1', 'synthetic/3']),
        'expected': """\
>synthetic/2/0_1
A
>synthetic/3/0_1
A
>synthetic/1/0_5
ACGTA
>synthetic/1/5_7
AC
>synthetic/1/7_10
ACG
>synthetic/1/10_14
ACGT
>synthetic/1/14_15
A
"""
    },
    {   'fasta': fasta_tests[4],
        'whitelist': set(['synthetic/2', 'synthetic/3', 'synthetic/1', 'synthetic/4', 'synthetic/5']),
        'expected': fasta_tests[4]
    },
]

#########################
### Helper functions. ###
#########################
def check_run(func, fasta, expected, whitelist_set=None):
    if expected is None:
        return check_run_raises(func, fasta, whitelist_set)
    fp_in = io.StringIO(fasta)
    fp_out = io.StringIO()
    func(fp_in, fp_out, whitelist_set)
    if expected != fp_out.getvalue():
        print((fp_out.getvalue()))
    assert expected == fp_out.getvalue()

def check_run_raises(func, fasta, whitelist_set=None):
    fp_in = io.StringIO(fasta)
    fp_out = io.StringIO()
    with pytest.raises(Exception) as e:
        func(fp_in, fp_out, whitelist_set)
    assert not isinstance(e.value, AssertionError) # Test assertions elsewhere.

def check_main_from_file(cmd, fasta, expected, in_fa_file, capsys):
    in_fa_file.write(fasta)
    argv = ['prog', cmd, str(in_fa_file)]
    mod.main(argv)
    out, err = capsys.readouterr()
    if expected != out:
        print(out)
    assert(out == expected)

def check_main_from_file_with_whitelist(cmd, fasta, expected, in_whitelist_file, in_fa_file, capsys):
    in_fa_file.write(fasta)
    argv = ['prog', '--zmw-whitelist-fn', in_whitelist_file, cmd, str(in_fa_file)]
    mod.main(argv)
    out, err = capsys.readouterr()
    if expected != out:
        print(out)
    assert(out == expected)

def check_main_from_stdin(cmd, fasta, expected, capsys):
    mod.sys.stdin = io.StringIO(fasta)
    argv = ['prog', cmd, '-']
    mod.main(argv)
    out, err = capsys.readouterr()
    if expected != out:
        print(out)
    assert(out == expected)

def check_main_from_stdin_with_whitelist(cmd, fasta, expected, in_whitelist_file, capsys):
    mod.sys.stdin = io.StringIO(fasta)
    argv = ['prog', '--zmw-whitelist-fn',  in_whitelist_file, cmd, '-']
    mod.main(argv)
    out, err = capsys.readouterr()
    if expected != out:
        print(out)
    assert(out == expected)

#############################
## Test internal functions ##
#############################
def test_yield_record_and_tokenized_headers():
    # We could delete this test, since we have whitelist tests also.
    # Or we could expand this.
    fp_in = io.StringIO(fasta_tests[1])
    records = FastaReader.yield_fasta_record(fp_in, log=lambda x: None)
    w = set()
    assert [] == list(mod.yield_record_and_tokenized_headers(w, records))

def test_load_zmw_whitelist(tmpdir, caplog):
    fn = ''
    expected = set()
    assert mod.load_zmw_whitelist(fn) == expected
    assert not caplog.records

    path = tmpdir.join('whitelist_empty.json')
    path.write('') # empty
    fn = str(path)
    expected = set()
    assert mod.load_zmw_whitelist(fn) == set()
    assert 'Assuming empty whitelist' in caplog.text
    caplog.clear()

    path = tmpdir.join('whitelist_empty_list.json')
    path.write('[]') # empty
    fn = str(path)
    expected = set()
    assert mod.load_zmw_whitelist(fn) == expected
    assert not caplog.records

    redundant = ['aa/1', 'bb/2', 'aa/1']
    path = tmpdir.join('whitelist_redundant_list.json')
    path.write(json.dumps(redundant))
    fn = str(path)
    expected = set(['aa/1', 'bb/2'])
    assert mod.load_zmw_whitelist(fn) == expected
    assert not caplog.records

#############################
### Test the PASS filter. ###
#############################
@pytest.mark.parametrize('case', [
    1, 2, 3, 4,
])
def test_run_pass_filter(case):
    provided = expected = fasta_tests[case]
    check_run(mod.run_pass_filter, provided, expected)

def test_run_pass_filter_5():
    check_run_raises(mod.run_pass_filter, fasta_tests[5])

def test_run_pass_filter_8_ccs():
    provided = fasta_tests[8]
    expected = expected_tests[8]
    check_run(mod.run_pass_filter, provided, expected)

########################################
### Test the streamed median filter. ###
########################################
@pytest.mark.parametrize('case', [
    1, 2, 3, 4, 5,
])
def test_run_streamed_median_filter(case):
    provided = fasta_tests[case]
    expected = expected_tests[case]
    check_run(mod.run_streamed_median_filter, provided, expected)

def test_run_streamed_median_7g():
    # test general
    curried = functools.partial(mod.run_streamed_median_filter, zmw_filter_func=mod.median_zmw_subread)
    check_run(curried, fasta_tests[7], expected_general_median_tests[7])

def test_run_streamed_median_7i():
    # test internal
    curried = functools.partial(mod.run_streamed_median_filter, zmw_filter_func=mod.internal_median_zmw_subread)
    check_run(curried, fasta_tests[7], expected_internal_median_tests[7])

def test_run_streamed_longest_3():
    check_run(mod.run_streamed_longest_filter, fasta_tests[3], expected_longest_tests[3])

#######################################
### Test the general median filter. ###
#######################################
@pytest.mark.parametrize('case', [
    1, 2, 3, 4, 5, 6
])
def test_run_median_filter(case):
    provided = fasta_tests[case]
    expected = expected_tests[case]
    check_run(mod.run_median_filter, provided, expected)

def test_run_median_filter_piped_input():
    """
    The run_median_filter asserts if the input file doesn't exist.
    This is to prevent specifying streams we can't rewind.
    """
    with pytest.raises(AssertionError) as e:
        mod.run_median_filter(sys.stdin, '', '')
    assert 'Cannot rewind' in str(e.value)

def test_run_median_filter_7():
    check_run(mod.run_median_filter, fasta_tests[7], expected_general_median_tests[7])

########################################
### Test the internal median filter. ###
########################################
@pytest.mark.parametrize('case', [
    1, 2, 3, 4, 5, 6
])
def test_run_internal_median_filter(case):
    provided = fasta_tests[case]
    expected = expected_tests[case]
    check_run(mod.run_internal_median_filter, provided, expected)

def test_run_internal_median_filter_piped_input():
    """
    The run_median_filter asserts if the input file doesn't exist.
    This is to prevent specifying streams we can't rewind.
    """
    check_run_raises(mod.run_internal_median_filter, fasta_tests[5])
    with pytest.raises(AssertionError) as e:
        mod.run_internal_median_filter(sys.stdin, '', '')
    assert 'Cannot rewind' in str(e.value)

def test_run_internal_median_filter_7():
    check_run(mod.run_internal_median_filter, fasta_tests[7], expected_internal_median_tests[7])

###############################
### Test whitelisting ZMWs. ###
###############################
@pytest.mark.parametrize('test_data', whitelist_tests_general_median)
def test_whitelist_run_internal_median_filter(test_data):
    test_fasta = test_data['fasta']
    test_whitelist = test_data['whitelist']
    test_expected = test_data['expected']
    check_run(mod.run_internal_median_filter, test_fasta, test_expected, test_whitelist)

@pytest.mark.parametrize('test_data', whitelist_tests_general_median)
def test_whitelist_run_median_filter(test_data):
    test_fasta = test_data['fasta']
    test_whitelist = test_data['whitelist']
    test_expected = test_data['expected']
    check_run(mod.run_median_filter, test_fasta, test_expected, test_whitelist)

@pytest.mark.parametrize('test_data', whitelist_tests_general_median)
def test_whitelist_run_streamed_median_filter(test_data):
    test_fasta = test_data['fasta']
    test_whitelist = test_data['whitelist']
    test_expected = test_data['expected']
    check_run(mod.run_streamed_median_filter, test_fasta, test_expected, test_whitelist)

@pytest.mark.parametrize('test_data', whitelist_tests_longest)
def test_whitelist_run_streamed_longest_filter(test_data):
    test_fasta = test_data['fasta']
    test_whitelist = test_data['whitelist']
    test_expected = test_data['expected']
    check_run(mod.run_streamed_longest_filter, test_fasta, test_expected, test_whitelist)

@pytest.mark.parametrize('test_data', whitelist_tests_pass)
def test_whitelist_run_pass_filter(test_data):
    test_fasta = test_data['fasta']
    test_whitelist = test_data['whitelist']
    test_expected = test_data['expected']
    check_run(mod.run_pass_filter, test_fasta, test_expected, test_whitelist)

###############################
### Test the main commands. ###
###############################
def test_main_cmd_pass_1(capsys):
    check_main_from_stdin('pass', fasta_tests[4], fasta_tests[4], capsys)

def test_main_cmd_pass_2(tmpdir, capsys):
    in_fa_file = tmpdir.join('in.fa')
    check_main_from_file('pass', fasta_tests[4], fasta_tests[4], in_fa_file, capsys)

def test_main_cmd_streamed_median_1(capsys):
    check_main_from_stdin('streamed-median', fasta_tests[4], expected_tests[4], capsys)

def test_main_cmd_streamed_median_2(tmpdir, capsys):
    in_fa_file = tmpdir.join('in.fa')
    check_main_from_file('streamed-median', fasta_tests[4], expected_tests[4], in_fa_file, capsys)

def test_main_cmd_median_1(tmpdir, capsys):
    in_fa_file = tmpdir.join('in.fa')
    check_main_from_file('median', fasta_tests[4], expected_tests[4], in_fa_file, capsys)

def test_main_cmd_internal_median_1(tmpdir, capsys):
    in_fa_file = tmpdir.join('in.fa')
    check_main_from_file('internal-median', fasta_tests[4], expected_tests[4], in_fa_file, capsys)

def test_main_cmd_streamed_internal_median_1(tmpdir, capsys):
    check_main_from_stdin('streamed-internal-median', fasta_tests[4], expected_tests[4], capsys)

def test_main_cmd_streamed_internal_median_2(tmpdir, capsys):
    in_fa_file = tmpdir.join('in.fa')
    check_main_from_file('streamed-internal-median', fasta_tests[4], expected_tests[4], in_fa_file, capsys)

#####################################################
### Test ZMW whitelisting from the main commands. ###
#####################################################
def test_whitelist_main_cmd_streamed_longest_from_stdin(tmpdir, capsys):
    for test_id, test_data in enumerate(whitelist_tests_longest):
        test_fasta = test_data['fasta']
        test_whitelist = test_data['whitelist']
        test_expected = test_data['expected']

        test_whitelist_fn = tmpdir.join('zmw.whitelist.{}.json'.format(test_id))
        if test_whitelist == None:
            # In case None, just omit the path.
            test_whitelist_fn = ''
        else:
            # In case an empty list, write an empty file.
            test_whitelist_fn.write(json.dumps(list(test_whitelist)))

        check_main_from_stdin_with_whitelist('streamed-longest', test_fasta, test_expected, str(test_whitelist_fn), capsys)

#######################
### Unit-test functions
def test_tokenize_header():
    with pytest.raises(ValueError) as e:
        mod.tokenize_header('bad_header')
    assert 'bad_header' in str(e.value)


import sqlite3
import easygui
import sys
import time
import pandas
import update
import visual

def main_menu():
    c1 = easygui.choicebox('请选择你要进行的操作','Main Menu',choices=['修改数据','查询数据','数据可视化'])
    if c1 == '修改数据':
        update.update_data()
    elif c1 == '查询数据':
        select_data()
    elif c1 == '数据可视化':
        visual.visual()
    else:
        sys.exit(0)

def select_data():
    date = time.strftime('%Y/%m/%d',time.localtime(time.time()))
    c1 = easygui.choicebox('请选择你要查看的数据','SELECT',choices=['中国疫情','美国疫情','世界疫情','湖北疫情'])
    if c1 == '中国疫情':
        china(date)
    elif c1 == '美国疫情':
        US(date)
    elif c1 == '世界疫情':
        world(date)
    elif c1 == '湖北疫情':
        hubei()
    else:
        main_menu()
    main_menu()

def US(date):
    cursor = sqlite3.connect('disease.db')
    uss = cursor.execute('SELECT * FROM US_covid19 ORDER BY confirmed DESC')
    us = uss.fetchall()
    data = pandas.DataFrame(us, columns=['地区','确诊','死亡'])
    easygui.msgbox(str(data), '美国疫情')

def china(date):
    cursor = sqlite3.connect('disease.db')
    uss = cursor.execute('SELECT * FROM China_covid19 ORDER BY confirmed DESC')
    us = uss.fetchall()
    data = pandas.DataFrame(us, columns=['地区', '确诊', '治愈', '死亡'])
    easygui.msgbox(str(data), '中国疫情')
    cursor.close()

def world(date):
    cursor = sqlite3.connect('disease.db')
    uss = cursor.execute('SELECT name, confirmed, death, cured FROM global_covid19 ORDER BY confirmed DESC')
    us = uss.fetchall()
    data = pandas.DataFrame(us, columns=['地区', '确诊', '死亡','治愈'])
    easygui.msgbox(str(data), '全球疫情')
    cursor.close()

def hubei():
    cursor = sqlite3.connect('disease.db')
    cities = cursor.execute('SELECT * FROM Hubei ORDER BY confirmed DESC')
    city = cities.fetchall()
    data = pandas.DataFrame(city, columns=['地区', '确诊', '治愈','死亡'])
    easygui.msgbox(str(data), '湖北疫情')
    cursor.close()

pandas.set_option('display.max_rows', None)
main_menu()


from math import sin
from kivy_garden.graph import Graph, MeshLinePlot
from kivy.app import App
from kivy.uix.boxlayout import BoxLayout

class MyApp(App):

    title = "Kivy Garden graph and MeshLinePlot "

    def build(self):
        return ThePlot()

class ThePlot(BoxLayout):
    def __init__(self, **kwargs):
        super().__init__(**kwargs)
        #return Label(text='Hello world')
        graph = Graph(xlabel='X', ylabel='Y', x_ticks_minor=5,
            x_ticks_major=25, y_ticks_major=1,
            y_grid_label=True, x_grid_label=True, padding=5,
            x_grid=True, y_grid=True, xmin=-0, xmax=100,
            ymin=-1, ymax=1)
        plot = MeshLinePlot(color=[1, 0, 0, 1])
        plot.points = [(x, sin(x / 10.)) for x in range(0, 101)]
        graph.add_plot(plot)
        self.add_widget(graph)

if __name__ == '__main__':
    MyApp().run()


from __future__ import division

from bisect import bisect_left
from collections import namedtuple
import numpy as np
from scipy.spatial import distance
import math

RBO = namedtuple("RBO", "min res ext")
RBO.__doc__ += ": Result of full RBO analysis"
RBO.min.__doc__ = "Lower bound estimate"
RBO.res.__doc__ = "Residual corresponding to min; min + res is an upper bound estimate"
RBO.ext.__doc__ = "Extrapolated point estimate"

def logistic(x, k):
    return 1/ (1 + math.exp(-k*x))

def _round(obj):
    if isinstance(obj, RBO):
        return RBO(_round(obj.min), _round(obj.res), _round(obj.ext))
    else:
        return round(obj, 3)

def set_at_depth(lst, depth):
    ans = set()
    for v in lst[:depth]:
        if isinstance(v, set):
            ans.update(v)
        else:
            ans.add(v)
    return ans

def embeddings_overlap(list1, list2, depth, index2word, embedding_space, norm=True):
    set1, set2 = set_at_depth(list1, depth), set_at_depth(list2, depth)
    word_list1 = [index2word[index] for index in list1]
    word_list2 = [index2word[index] for index in list2]

    centroid_1 = np.mean([embedding_space[w] for w in word_list1[:depth] if w in embedding_space], axis=0)
    centroid_2 = np.mean([embedding_space[w] for w in word_list2[:depth] if w in embedding_space], axis=0)
    cos_sim = 1 - distance.cosine(centroid_1, centroid_2)
    if cos_sim > 1:
        cos_sim = 1
    elif cos_sim < -1:
        cos_sim = -1
    if norm:
        e_ov = 1 - (math.acos(cos_sim) / math.pi)
    else:
        e_ov = cos_sim
    return e_ov, len(set1), len(set2)

def overlap(list1, list2, depth, index2word, embedding_space, norm):
    # return agreement(list1, list2, depth) * min(depth, len(list1), len(list2))
    # NOTE: comment the preceding and uncomment the following line if you want
    # to stick to the algorithm as defined by the paper
    ov = embeddings_overlap(list1, list2, depth, index2word, embedding_space, norm=norm)[0]
    # print("overlap", ov)
    return ov

def agreement(list1, list2, depth, index2word, embedding_space, norm):
    """Proportion of shared values between two sorted lists at given depth."""
    len_intersection, len_set1, len_set2 = embeddings_overlap(list1, list2, depth, index2word, embedding_space=embedding_space, norm=norm)
    return 2 * len_intersection / (len_set1 + len_set2)

def cumulative_agreement(list1, list2, depth, index2word, embedding_space, norm):
    return (agreement(list1, list2, d, index2word, embedding_space, norm) for d in range(1, depth + 1))

def rbo_min(list1, list2, p, index2word, embedding_space, norm=True, depth=None):
    """Tight lower bound on RBO.
    See equation (11) in paper.
    """
    depth = min(len(list1), len(list2)) if depth is None else depth
    x_k = overlap(list1, list2, depth, index2word, embedding_space, norm)
    log_term = x_k * math.log(1 - p)
    sum_term = sum(
        p ** d / d * (overlap(list1, list2, d, index2word, embedding_space=embedding_space, norm=norm) - x_k) for d in range(1, depth + 1)
    )
    return (1 - p) / p * (sum_term - log_term)

def rbo_res(list1, list2, p, index2word, embedding_space, norm=True):
    """Upper bound on residual overlap beyond evaluated depth.
    See equation (30) in paper.
    NOTE: The doctests weren't verified against manual computations but seem
    plausible. In particular, for identical lists, ``rbo_min()`` and
    ``rbo_res()`` should add up to 1, which is the case.
    """
    S, L = sorted((list1, list2), key=len)
    s, l = len(S), len(L)
    x_l = overlap(list1, list2, l, index2word, embedding_space, norm=norm)
    # since overlap(...) can be fractional in the general case of ties and f
    # must be an integer --> math.ceil()
    f = int(math.ceil(l + s - x_l))
    # upper bound of range() is non-inclusive, therefore + 1 is needed
    term1 = s * sum(p ** d / d for d in range(s + 1, f + 1))
    term2 = l * sum(p ** d / d for d in range(l + 1, f + 1))
    term3 = x_l * (math.log(1 / (1 - p)) - sum(p ** d / d for d in range(1, f + 1)))
    return p ** s + p ** l - p ** f - (1 - p) / p * (term1 + term2 + term3)

def rbo_ext(list1, list2, p, index2word, embedding_space, norm=True):
    """RBO point estimate based on extrapolating observed overlap.
    See equation (32) in paper.
    NOTE: The doctests weren't verified against manual computations but seem
    plausible.
    """
    S, L = sorted((list1, list2), key=len)
    s, l = len(S), len(L)
    x_l = overlap(list1, list2, l, index2word, embedding_space=embedding_space, norm=norm)
    x_s = overlap(list1, list2, s, index2word, embedding_space=embedding_space, norm=norm)
    # the paper says overlap(..., d) / d, but it should be replaced by
    # agreement(..., d) defined as per equation (28) so that ties are handled
    # properly (otherwise values > 1 will be returned)
    # sum1 = sum(p**d * overlap(list1, list2, d)[0] / d for d in range(1, l + 1))
    sum1 = sum(p ** d * agreement(list1, list2, d, index2word=index2word, embedding_space=embedding_space, norm=norm)
               for d in range(1, l + 1))
    sum2 = sum(p ** d * x_s * (d - s) / s / d for d in range(s + 1, l + 1))
    term1 = (1 - p) / p * (sum1 + sum2)
    term2 = p ** l * ((x_l - x_s) / l + x_s / s)
    return term1 + term2

def word_embeddings_rbo(list1, list2, p, index2word, embedding_space, norm):
    """Complete RBO analysis (lower bound, residual, point estimate).
    ``list`` arguments should be already correctly sorted iterables and each
    item should either be an atomic value or a set of values tied for that
    rank. ``p`` is the probability of looking for overlap at rank k + 1 after
    having examined rank k.
    RBO(min=0.489, res=0.477, ext=0.967)
    """
    if not 0 <= p <= 1:
        raise ValueError("The ``p`` parameter must be between 0 and 1.")
    args = (list1, list2, p, index2word, embedding_space, norm)

    return RBO(rbo_min(*args), rbo_res(*args), rbo_ext(*args))

def sort_dict(dct, *, ascending=False):
    """Sort keys in ``dct`` according to their corresponding values.
    Sorts in descending order by default, because the values are
    typically scores, i.e. the higher the better. Specify
    ``ascending=True`` if the values are ranks, or some sort of score
    where lower values are better.
    Ties are handled by creating sets of tied keys at the given position
    in the sorted list.
    >>> dct = dict(a=1, b=2, c=1, d=3)
    >>> list(sort_dict(dct)) == ['d', 'b', {'a', 'c'}]
    True
    >>> list(sort_dict(dct, ascending=True)) == [{'a', 'c'}, 'b', 'd']
    True
    """
    scores = []
    items = []
    # items should be unique, scores don't have to
    for item, score in dct.items():
        if not ascending:
            score *= -1
        i = bisect_left(scores, score)
        if i == len(scores):
            scores.append(score)
            items.append(item)
        elif scores[i] == score:
            existing_item = items[i]
            if isinstance(existing_item, set):
                existing_item.add(item)
            else:
                items[i] = {existing_item, item}
        else:
            scores.insert(i, score)
            items.insert(i, item)
    return items

def rbo_dict(dict1, dict2, p, index2word, embedding_space, norm, *, sort_ascending=False):
    """Wrapper around ``rbo()`` for dict input.
    Each dict maps items to be sorted to the score according to which
    they should be sorted. The RBO analysis is then performed on the
    resulting sorted lists.
    The sort is descending by default, because scores are typically the
    higher the better, but this can be overridden by specifying
    ``sort_ascending=True``.
    """
    list1, list2 = (
        sort_dict(dict1, ascending=sort_ascending),
        sort_dict(dict2, ascending=sort_ascending),
    )
    return word_embeddings_rbo(list1, list2, p, index2word, embedding_space, norm)


import pandas as pd
from nltk.tokenize import word_tokenize
import math
import csv
from datetime import datetime
import pyproj
from shapely.geometry import shape
from shapely.ops import transform
import json

def create_cluster():
    with open('Clusters', 'r') as target:
        cluster_dict = json.load(target)
    cluster= [x['Tweets'] for x in cluster_dict]
    return cluster

def calculate_area(clusters):
    areas = []
    for x in clusters:
        lat_min = data.iloc[x[0]]['Latitude']
        lat_max = data.iloc[x[0]]['Latitude']
        long_min = data.iloc[x[0]]['Longitude']
        long_max = data.iloc[x[0]]['Longitude']

        for y in x:
            latitude = data.iloc[y]['Latitude']
            longitude = data.iloc[y]['Longitude']
            if latitude > lat_max:
                lat_max = latitude
            elif latitude < lat_min:
                lat_min = latitude
            if longitude > long_max:
                long_max = longitude
            elif longitude < long_min:
                long_min = longitude
        # geom = {'type': 'Polygon',
        #         'coordinates': [[[lat_min, long_min], [lat_min, long_max], [lat_max,long_max], [lat_max, long_min]]]}

        geom = {'type': 'Polygon',
                    'coordinates': [[[long_min,lat_min ], [long_max,lat_min], [long_max, lat_max], [long_min,lat_max]]]}
        s = shape(geom)
        proj = partial(pyproj.transform, pyproj.Proj(init='epsg:4326'), pyproj.Proj(init='epsg:3857'))
        s_new = transform(proj, s)
        projected_area = s_new.area/1000000
        areas.append(projected_area)
        areas = [0 if math.isnan(x) else x for x in areas]
    return areas

def partial(func, *args, **keywords):
    def newfunc(*fargs, **fkeywords):
        newkeywords = keywords.copy()
        newkeywords.update(fkeywords)
        return func(*(args + fargs), **newkeywords)

    newfunc.func = func
    newfunc.args = args
    newfunc.keywords = keywords
    return newfunc

def calc_datetime():
    date_time = []
    for x in range(len(data)):
        date_time.append(datetime.combine(datetime.strptime(data.iloc[x]["Date"], '%Y-%m-%d'),
                                          datetime.strptime(data.iloc[x]["Time"], '%H:%M').time()))
    return date_time

def calc_params(cluster):
    tweet_number = []
    user_number = []
    time_dif = []
    for x in cluster:
        tweet_number.append(len(x))
        user_number.append(len(x))
        time_min = date_time[x[0]]
        time_max = date_time[x[0]]
        for y in x:
            if date_time[y] > time_max:
                time_max = date_time[y]
            if date_time[y] < time_min:
                time_min = date_time[y]
        time_dif.append(time_max - time_min)
    return tweet_number, user_number, time_dif

def common_words(cluster):

    cluster_words=[]
    for y in cluster:
        tweet=data.iloc[y]['Tweet']
        words=word_tokenize(tweet)
        for word in words:
            if not word in cluster_words:
                cluster_words.append(word)
    return cluster_words

def calc_scores(cluster):
    a1 = 0.5
    a2 = 0.5
    a3 = 0.5
    a4 = 0.5
    scores = []
    for x in range(len(cluster)):
        scores.append(a1 * tweet_numbers[x] + a2 * user_numbers[x] + a3 * time_difs[x].total_seconds() + a4 * area[x])
    return scores

def max(list):
    index=0
    max=list[0]
    for x in range(len(list)):
        if list[x]>max:
            max=list[x]
            index=x
    return index

def ranking(scores):
    cluster_info=[]
    for x in range(len(scores)):
        max_index=max(scores)
        scores[max_index]=0
        tweets= [dataset_original.iloc[x]["Tweet"] for x in cluster[max_index]]
        cluster_info.append({"ClusterId": max_index, "ClusterRank": x+1, "TweetIds": cluster[max_index], "Tweets": tweets})
    keys = cluster_info[0].keys()
    with open('Cluster_Ranking_Results.csv', 'w') as output_file:
        dict_writer = csv.DictWriter(output_file, keys)
        dict_writer.writeheader()
        dict_writer.writerows(cluster_info)

dataset = pd.read_csv("processed_dataset.csv", sep="\t")
data = dataset.head(n=100)
dataset_original = pd.read_csv("processed_dataset.csv", sep="\t")
cluster= create_cluster()
# print(cluster)
area = calculate_area(cluster)
date_time = calc_datetime()
tweet_numbers, user_numbers, time_difs = calc_params(cluster)
# print(tweet_numbers)
# print(user_numbers)
# print(time_difs)
# print(area)
scores= calc_scores(cluster)
#print(scores)
ranking(scores)


from pathlib import Path
import anyio
from backups2datalad.adandi import AsyncDandiClient
from backups2datalad.config import BackupConfig, ResourceConfig
from backups2datalad.manager import Manager
from backups2datalad.zarr import sync_zarr
from backups2datalad.logging import log

dandiset_id = "FILL IN HERE"
zarr_asset_path = "FILL IN HERE"
backup_folder = "FILL IN HERE"

async def amain():
    async with AsyncDandiClient.for_dandi_instance("dandi") as client:
        dandiset = await client.get_dandiset(dandiset_id, "draft")
        asset = await dandiset.aget_asset_by_path(zarr_asset_path)
        checksum = asset.get_digest().value
        config = BackupConfig(zarrs=ResourceConfig(path="zarrs"))
        await sync_zarr(
            asset,
            checksum,
            Path(backup_folder, asset.zarr),
            Manager(config=config, gh=None, log=log)
        )

if __name__ == "__main__":
    anyio.run(amain)


#!/usr/bin/env python3
"""opencv module tests"""
import sys
import os
import inspect

import logging

import cv2
import numpy as np

import matplotlib.pyplot as plt
from matplotlib import colors
from matplotlib import ticker
from matplotlib.colors import LinearSegmentedColormap

logger = logging.getLogger("OPENCV")

def opencv_module(args=None):
    logger.info("modules_opencv")

    if args:
        print(vars(args))
        logger.info(vars(args))

    # vid_test(args)

    NIRPlantVideoTracking(args)

    # https://www.geeksforgeeks.org/face-detection-using-python-and-opencv-with-webcam/
    # webcam_create_data(args)
    # webcam_face_recognize(args)

    # nieve approach to getting a list of webcam ids
    # https://stackoverflow.com/a/62639343
    # list_cams(args)

def webcam_create_data(args):

    module_dir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe())))
    input_dir = module_dir + "/../../input"
    output_dir = module_dir + "/../../output"

    # Creating database
    # It captures images and stores them in datasets
    # folder under the folder name of sub_data
    haar_file = input_dir + "/haarcascade_frontalface_default.xml"

    # All the faces data will be
    # present this folder
    datasets = output_dir + "/datasets"
    if not os.path.isdir(datasets):
        os.mkdir(datasets)

    # These are sub data sets of folder,
    # for my faces I've used my name you can
    # change the label here
    sub_data = "me2"

    path = os.path.join(datasets, sub_data)
    if not os.path.isdir(path):
        os.mkdir(path)

    # defining the size of images
    (width, height) = (130, 100)

    # '0' is used for my webcam,
    # if you've any other camera
    # attached use '1' like this
    face_cascade = cv2.CascadeClassifier(haar_file)
    webcam = cv2.VideoCapture(2)
    cv2.normalize()

    # The program loops until it has 30 images of the face.
    count = 1
    while count < 50:
        (_, im) = webcam.read()
        gray = cv2.cvtColor(im, cv2.COLOR_BGR2GRAY)
        faces = face_cascade.detectMultiScale(gray, 1.3, 4)
        for (x, y, w, h) in faces:
            cv2.rectangle(im, (x, y), (x + w, y + h), (255, 0, 0), 2)
            face = gray[y : y + h, x : x + w]
            face_resize = cv2.resize(face, (width, height))
            cv2.imwrite("% s/% s.png" % (path, count), face_resize)
        count += 1

        cv2.imshow("OpenCV", im)
        key = cv2.waitKey(10)
        if key == 27:
            break

def webcam_face_recognize(args):

    module_dir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe())))
    input_dir = module_dir + "/../../input"
    output_dir = module_dir + "/../../output"

    # It helps in identifying the faces
    # size = 4
    haar_file = input_dir + "/haarcascade_frontalface_default.xml"

    datasets = output_dir + "/datasets"
    if not os.path.isdir(datasets):
        os.mkdir(datasets)

    # Part 1: Create fisherRecognizer
    print("Recognizing Face Please Be in sufficient Lights...")

    # Create a list of images and a list of corresponding names
    (images, labels, names, id) = ([], [], {}, 0)
    for (subdirs, dirs, files) in os.walk(datasets):
        for subdir in dirs:
            names[id] = subdir
            subjectpath = os.path.join(datasets, subdir)
            for filename in os.listdir(subjectpath):
                path = subjectpath + "/" + filename
                label = id
                images.append(cv2.imread(path, 0))
                labels.append(int(label))
            id += 1
    (width, height) = (130, 100)

    # Create a Numpy array from the two lists above
    (images, labels) = [np.array(lis) for lis in [images, labels]]

    # OpenCV trains a model from the images
    # NOTE FOR OpenCV2: remove '.face'
    model = cv2.face.LBPHFaceRecognizer_create()
    model.train(images, labels)

    # Part 2: Use fisherRecognizer on camera stream
    face_cascade = cv2.CascadeClassifier(haar_file)
    webcam = cv2.VideoCapture(2)
    while True:
        (_, im) = webcam.read()
        gray = cv2.cvtColor(im, cv2.COLOR_BGR2GRAY)
        faces = face_cascade.detectMultiScale(gray, 1.3, 5)
        for (x, y, w, h) in faces:
            cv2.rectangle(im, (x, y), (x + w, y + h), (255, 0, 0), 2)
            face = gray[y : y + h, x : x + w]
            face_resize = cv2.resize(face, (width, height))
            # Try to recognize the face
            prediction = model.predict(face_resize)
            cv2.rectangle(im, (x, y), (x + w, y + h), (0, 255, 0), 3)

            if prediction[1] < 500:
                cv2.putText(
                    im,
                    "% s - %.0f" % (names[prediction[0]], prediction[1]),
                    (x - 10, y - 10),
                    cv2.FONT_HERSHEY_PLAIN,
                    1,
                    (0, 255, 0),
                )
            else:
                cv2.putText(
                    im,
                    "not recognized",
                    (x - 10, y - 10),
                    cv2.FONT_HERSHEY_PLAIN,
                    1,
                    (0, 255, 0),
                )

        cv2.imshow("OpenCV", im)

        key = cv2.waitKey(10)
        if key == 27:
            break

def vid_test(args=None):
    cap = cv2.VideoCapture(0)

    # Check if camera opened successfully
    if not cap.isOpened():
        print("Error opening video  file")

    # Read until video is completed
    while cap.isOpened():

        # Capture frame-by-frame
        ret, frame = cap.read()
        if ret:

            # Display the resulting frame
            cv2.imshow("Frame", frame)

            # Press Q on keyboard to  exit
            if cv2.waitKey(25) & 0xFF == ord("q"):
                break

        # Break the loop
        else:
            break

    # When everything done, release
    # the video capture object
    cap.release()

    # Closes all the frames
    cv2.destroyAllWindows()

def list_cams(args):
    """Test the ports and returns a tuple with the available ports and the ones that are working."""
    # is_working = True
    dev_port = 0
    working_ports = []
    available_ports = []
    max_ports = 20
    while dev_port < max_ports:
        try:
            camera = cv2.VideoCapture(dev_port)
            if camera.isOpened():
                #    is_working = False
                #    print("Port %s is not working." %dev_port)
                # else:
                is_reading, img = camera.read()
                w = camera.get(3)
                h = camera.get(4)
                if is_reading:
                    print("Port %s is working and reads images (%s x %s)" % (dev_port, h, w))
                    working_ports.append(dev_port)
                else:
                    print("Port %s for camera ( %s x %s) is present but does not reads." % (dev_port, h, w))
                    available_ports.append(dev_port)
        except:  # noqa: E722
            a, b, c = sys.exc_info()
            print(a)
            print(b)
            print(c)

        dev_port += 1
    return available_ports, working_ports

# from: https://github.com/MuonRay/Image-VideoSegmentationinNIRforPlantDetection/blob/master/NIRPlantVideoTracking.py

def NIRPlantVideoTracking(args):

    cap = cv2.VideoCapture(0)

    # custom colormap for ndvi greyscale video
    cols3 = ["gray", "blue", "green", "yellow", "red"]

    def create_colormap(args):
        return LinearSegmentedColormap.from_list(name="custom1", colors=cols3)

    # colour bar to match grayscale units
    def create_colorbar(fig, image):
        position = fig.add_axes([0.125, 0.19, 0.2, 0.05])
        norm = colors.Normalize(vmin=-1.0, vmax=1.0)
        cbar = plt.colorbar(
            image,
            cax=position,
            orientation="horizontal",
            norm=norm,
        )
        cbar.ax.tick_params(labelsize=6)
        tick_locator = ticker.MaxNLocator(nbins=3)
        cbar.locator = tick_locator
        cbar.update_ticks()
        cbar.set_label("NDVI", fontsize=10, x=0.5, y=0.5, labelpad=-25)

    while 1:

        # Take each frame
        _, frame = cap.read()

        # Convert BGR to HSV
        hsv = cv2.cvtColor(frame, cv2.COLOR_BGR2HSV)

        # define range of red NIR vegetation color in HSV
        low_red = np.array([160, 105, 84])
        high_red = np.array([179, 255, 255])

        # Threshold the HSV image to get only red colors
        mask = cv2.inRange(hsv, low_red, high_red)

        # Bitwise-AND mask and original image
        res = cv2.bitwise_and(frame, frame, mask=mask)

        # NDVI Processing
        ir = (res[:, :, 0]).astype("float")
        r = (res[:, :, 2]).astype("float")

        ndvi = np.true_divide(np.subtract(ir, r), np.add(ir, r))

        cols3 = ["gray", "blue", "green", "yellow", "red"]

        # def create_colormap(args):
        #    return LinearSegmentedColormap.from_list(name="custom1", colors=cols3)

        # colour bar to match grayscale units
        # def create_colorbar(fig, image):
        #     position = fig.add_axes([0.125, 0.19, 0.2, 0.05])
        #     norm = colors.Normalize(vmin=-1.0, vmax=1.0)
        #     cbar = plt.colorbar(
        #         image,
        #         cax=position,
        #         orientation="horizontal",
        #         norm=norm,
        #     )
        #     cbar.ax.tick_params(labelsize=6)
        #     tick_locator = ticker.MaxNLocator(nbins=3)
        #     cbar.locator = tick_locator
        #     cbar.update_ticks()
        #     cbar.set_label(
        #         "NDVI",
        #         fontsize=10,
        #         x=0.5,
        #         y=0.5,
        #         labelpad=-25,
        #     )

        image = plt.imshow(ndvi, cmap=create_colormap(colors))
        # plt.axis('off')
        # image = cv2.imshow(ndvi, cmap=create_colormap(colors))

        cv2.imshow("frame", frame)
        cv2.imshow("mask", mask)
        cv2.imshow("res", res)

        # this step adds considerable processing, be sure to use only 720p files at most a minute long
        # cv2.imshow('ndvi',ndvi)

        cv2.imshow("ndvi with color", ndvi)

        k = cv2.waitKey(5) & 0xFF
        if k == 27:
            break

    cv2.destroyAllWindows()


import numpy as np

import torch.utils.data as Data

# basic function
def multiclass_noisify(y, P, random_state=1):
    """ Flip classes according to transition probability matrix T.
    It expects a number between 0 and the number of classes - 1.
    """

#    print (np.max(y), P.shape[0])

    # print(type(y[0][0]))

    # assert P.shape[0] == P.shape[1]
    # assert np.max(y) < P.shape[0]

    # row stochastic matrix
    # assert_array_almost_equal(P.sum(axis=1), np.ones(P.shape[1]))
    # assert (P >= 0.0).all()

    m = y.shape[0]
    new_y = y.copy()
    flipper = np.random.RandomState(random_state)

    for idx in np.arange(m):
        i = y[idx]
        # draw a vector with only an 1
        flipped = flipper.multinomial(1, P[i, :][0], 1)[0]
        new_y[idx] = np.where(flipped == 1)[0]

    return new_y

# noisify_pairflip call the function "multiclass_noisify"
def noisify_pairflip(y_train, noise, random_state=1, nb_classes=10):
    """mistakes:
        flip in the pair
    """
    P = np.eye(nb_classes)
    n = noise

    if n > 0.0:
        # 0 -> 1
        P[0, 0], P[0, 1] = 1. - n, n
        for i in range(1, nb_classes-1):
            P[i, i], P[i, i + 1] = 1. - n, n
        P[nb_classes-1, nb_classes-1], P[nb_classes-1, 0] = 1. - n, n

        y_train_noisy = multiclass_noisify(y_train, P=P,
                                           random_state=random_state)
        actual_noise = (y_train_noisy != y_train).mean()

        # assert actual_noise > 0.0
        # print('Actual noise %.2f' % actual_noise)
        y_train = y_train_noisy
    # print (P)

    return y_train, actual_noise,P

def dataset_split_for_Animal(train_images, train_labels, noise_rate=0.5, split_per=0.9, random_seed=1, num_classes=10, noise_type='flip'):
    clean_train_labels = train_labels[:, np.newaxis]

    # if noise_type == 'flip':
    _, real_noise_rate, transition_matrix = noisify_pairflip(clean_train_labels,
                                                    noise=noise_rate, random_state=random_seed, nb_classes=num_classes)

    # if noise_type == 'symmetric':
    #     _, real_noise_rate, transition_matrix = noisify_multiclass_symmetric(clean_train_labels,
                                                # noise=noise_rate, random_state= random_seed, nb_classes=num_classes)

    # elif noise_type == 'asymmetric':
    #     _, real_noise_rate, transition_matrix = noisify_multiclass_asymmetric(clean_train_labels,
    #                                                 noise=noise_rate, random_state=random_seed, nb_classes=num_classes)

    # print(transition_matrix)

    noisy_labels = train_labels

    num_samples = int(noisy_labels.shape[0])
    np.random.seed(random_seed)
    train_set_index = np.random.choice(num_samples, int(num_samples*split_per), replace=False)
    index = np.arange(train_images.shape[0])
    val_set_index = np.delete(index, train_set_index)

    train_set, val_set = train_images[train_set_index, :], train_images[val_set_index, :]
    train_labels, val_labels = noisy_labels[train_set_index], noisy_labels[val_set_index]

    return train_set, val_set, train_labels, val_labels, transition_matrix

class Animal_dataset(Data.Dataset):
    def __init__(self, train=True, transform=None, target_transform=None, noise_rate=0.5, split_per=1, random_seed=1, num_class=10, noise_type='symmetric', anchor=True):

        self.transform = transform
        self.target_transform = target_transform
        self.train = train
        self.anchor = anchor
        # print(self.anchor)
        # input()
        if anchor:
            original_images = np.load('../../coteaching_plus_Animal/data/Animal_npy/train_images.npy')
            original_labels = np.load('../../coteaching_plus_Animal/data/Animal_npy/train_labels.npy')
        else:
            original_images = np.load('Animal_npy/mnist_images.npy')
            original_labels = np.load('Animal_npy/mnist_labels.npy')

        original_images=original_images.transpose((0,3,1,2))

        self.train_data, self.val_data, self.train_labels, self.val_labels, self.t = dataset_split_for_Animal(original_images,
                                                                             original_labels, noise_rate, split_per, random_seed, num_class, noise_type)

        # print(self.train_data.shape)
        # print(self.val_data.shape)
        # input()

        print('we are training for Animal 10N')

    def __getitem__(self, index):

        if self.train:
            img, label = self.train_data[index], self.train_labels[index]
        else:
            img, label = self.val_data[index], self.val_labels[index]

        if self.transform is not None:
            img = self.transform(img)

        if self.target_transform is not None:
            label = self.target_transform(label)

        return img, label, index
    def __len__(self):

        if self.train:
            return len(self.train_data)

        else:
            return len(self.val_data)

class Animal_test_dataset(Data.Dataset):
    def __init__(self, transform=None, target_transform=None):

        self.transform = transform
        self.target_transform = target_transform

        self.test_data = np.load('../../coteaching_plus_Animal/data/Animal_npy/test_images.npy')
        self.test_labels = np.load('../../coteaching_plus_Animal/data/Animal_npy/test_labels.npy')  # 0-9

        self.test_data=self.test_data.transpose((0,3,1,2))

        print('we are testing for Animal 10N')

    def __getitem__(self, index):

        img, label = self.test_data[index], self.test_labels[index]

        if self.transform is not None:
            img = self.transform(img)

        if self.target_transform is not None:
            label = self.target_transform(label)

        return img, label,index

    def __len__(self):
        return len(self.test_data)


import datetime
import os
from flask import Flask,render_template,request,redirect,url_for,flash
from werkzeug.utils import secure_filename
import sqlite3 as sql
UPLOAD_FOLDER = 'static/uploads/'
ALLOWED_EXTENSIONS = {'png', 'jpg', 'jpeg', 'gif'}
app = Flask(__name__)
app.config['UPLOAD_FOLDER'] = UPLOAD_FOLDER

def allowed_file(filename):
    return '.' in filename and \
           filename.rsplit('.', 1)[1].lower() in ALLOWED_EXTENSIONS

@app.route('/')
@app.route('/retrieve')
def retrieve():
    con=sql.connect("user_db.db")
    con.row_factory=sql.Row
    cur=con.cursor()
    cur.execute("select * from user_table")
    data=cur.fetchall()
    return render_template('retrieve.html',datas=data)

@app.route('/create',methods=['POST','GET'])
def create():
    if request.method=='POST':
        name=request.form['name']
        contact=request.form['contact']
        file = request.files['file']
        if file and allowed_file(file.filename):
            filename_path = secure_filename(file.filename)
            split_filename_path = os.path.splitext(filename_path)
            ct = datetime.datetime.now()
            ts = ct.timestamp()
            new_filename_path = str(ts)+split_filename_path[1]
            disnation = os.path.join(app.config['UPLOAD_FOLDER'], new_filename_path)
            file.save(disnation)
            con=sql.connect("user_db.db")
            cur=con.cursor()
            cur.execute("insert into user_table(name,contact,filename) values(?,?,?)",[name,contact,new_filename_path])
            con.commit()
            flash('Data Saved','success')
            return redirect(url_for("retrieve"))
        else:
            flash('File formate must be png, jpg, jpeg and gif','warning')
            return render_template('create.html')
    return render_template('create.html')

@app.route('/update/<string:id>',methods=['POST','GET'])
def update(id):
    con=sql.connect("user_db.db")
    con.row_factory=sql.Row
    cur=con.cursor()
    cur.execute("select * from user_table where id=?",[id])
    data=cur.fetchone()
    if request.method=='POST':
        name=request.form['name']
        contact=request.form['contact']
        file = request.files['file']
        filename_path = secure_filename(file.filename)
        if filename_path == '':
            con=sql.connect("user_db.db")
            cur=con.cursor()
            cur.execute("update user_table set name=?,contact=? where id=?",[name,contact,id])
        else:
            if file and allowed_file(file.filename):
                split_filename_path = os.path.splitext(filename_path)
                ct = datetime.datetime.now()
                ts = ct.timestamp()
                new_filename_path = str(ts)+split_filename_path[1]
                disnation = os.path.join(app.config['UPLOAD_FOLDER'], new_filename_path)
                file.save(disnation)
                con=sql.connect("user_db.db")
                cur=con.cursor()
                cur.execute("update user_table set name=?,contact=?,filename=? where id=?",[name,contact,new_filename_path,id])
            else:
                flash('File formate must be png, jpg, jpeg and gif','warning')
                return render_template('update.html',datas=data)
        con.commit()
        flash('Data updated','success')
        return redirect(url_for("retrieve"))
    return render_template('update.html',datas=data)

@app.route('/delete/<string:id>',methods=['GET'])
def delete(id):
    con=sql.connect("user_db.db")
    con.row_factory=sql.Row
    cur=con.cursor()
    cur.execute("delete from user_table where id=?",[id])
    con.commit()
    flash('Data Deleted','warning')
    return redirect(url_for("retrieve"))

if __name__ =='__main__':
    app.secret_key='admin123'
    app.run(debug=True)


from http import HTTPStatus

from django.test import Client, TestCase
from django.urls import reverse
from posts.models import Group, Post, User

class PostURLTests(TestCase):
    @classmethod
    def setUpClass(cls):
        super().setUpClass()
        cls.user = User.objects.create_user(username='user')
        cls.author = User.objects.create_user(username='author')
        cls.group = Group.objects.create(
            title='Тестовая группа',
            slug='test-slug',
            description='Тестовое описание',
        )
        cls.post = Post.objects.create(
            author=cls.author,
            text='Тестовый пост',
            group=cls.group
        )

    def setUp(self):
        """Создание клиента гостя, авторизованного пользователя
        и автора поста.
        """
        self.guest_client = Client()
        self.authorized_client = Client()
        self.authorized_client.force_login(self.user)
        self.post_author = Client()
        self.post_author.force_login(self.author)

    def test_guest_have_access_to_public_pages_url(self):
        """Проверка статуса страниц для неавторизованного пользователя."""
        pages_url_status = {
            '/unexisting_page/': HTTPStatus.NOT_FOUND,
            '/': HTTPStatus.OK,
            f'/group/{self.group.slug}/': HTTPStatus.OK,
            f'/profile/{self.author}/': HTTPStatus.OK,
            f'/posts/{self.post.id}/': HTTPStatus.OK,
            '/create/': HTTPStatus.FOUND,
            f'/posts/{self.post.id}/edit/': HTTPStatus.FOUND,
            f'/posts/{self.post.id}/comment/': HTTPStatus.FOUND,
            '/follow/': HTTPStatus.FOUND,
            f'/profile/{self.user}/follow/': HTTPStatus.FOUND,
            f'/profile/{self.user}/unfollow/': HTTPStatus.FOUND
        }
        for url, expected in pages_url_status.items():
            with self.subTest(url=url):
                response = self.guest_client.get(url)
                self.assertEqual(response.status_code, expected)

    def test_guest_redirects_correct(self):
        """Проверка редиректов неавторизованного пользователя."""
        pages_url_redirect = {
            '/create/': reverse(
                'users:login') + '?next=/create/',
            f'/posts/{self.post.id}/edit/': reverse(
                'users:login') + f'?next=/posts/{self.post.id}/edit/',
            f'/posts/{self.post.id}/comment/': reverse(
                'users:login') + f'?next=/posts/{self.post.id}/comment/',
            '/follow/': reverse(
                'users:login') + '?next=/follow/',
            f'/profile/{self.user}/follow/': reverse(
                'users:login') + f'?next=/profile/{self.user}/follow/',
            f'/profile/{self.user}/unfollow/': reverse(
                'users:login') + f'?next=/profile/{self.user}/unfollow/'
        }
        for url, redirect_url in pages_url_redirect.items():
            with self.subTest(url=url):
                response = self.guest_client.get(url, follow=True)
                self.assertRedirects(response, redirect_url)

    def test_authorized_have_access_to_closed_pages_url(self):
        """"Проверка статуса страниц для авторизованного пользователя,
        автора поста.
        """
        pages_url_status = {
            '/create/': HTTPStatus.OK,
            f'/posts/{self.post.id}/edit/': HTTPStatus.OK,
            '/follow/': HTTPStatus.OK,
            f'/profile/{self.user}/follow/': HTTPStatus.FOUND,
            f'/profile/{self.user}/unfollow/': HTTPStatus.FOUND,
            f'/posts/{self.post.id}/comment/': HTTPStatus.FOUND,

        }
        for url, expected in pages_url_status.items():
            with self.subTest(url=url):
                response = self.post_author.get(url)
                self.assertEqual(response.status_code, expected)

    def test_edit_page_url_not_available_authorized_not_author(self):
        """Страница редактирования недоступна не-автору поста."""
        post_edit_page = f'/posts/{self.post.id}/edit/'
        response = self.authorized_client.get(post_edit_page)
        self.assertEqual(response.status_code, HTTPStatus.FOUND)

    def test_authorized_redirects_correct(self):
        """Проверка редиректов авторизованного пользователя,
        не-автора поста.
        """
        pages_url_redirect = {
            f'/posts/{self.post.id}/edit/': f'/posts/{self.post.id}/',
            f'/posts/{self.post.id}/comment/': f'/posts/{self.post.id}/',
            f'/profile/{self.user}/follow/': f'/profile/{self.user}/',
            f'/profile/{self.user}/unfollow/': f'/profile/{self.user}/'
        }
        for url, redirect_url in pages_url_redirect.items():
            with self.subTest(url=url):
                response = self.authorized_client.get(url)
                self.assertRedirects(response, redirect_url)

    def test_urls_use_correct_template(self):
        """URL-адрес использует соответствующий шаблон."""
        templates_url_names = {
            '/': 'posts/index.html',
            f'/group/{self.group.slug}/': 'posts/group_list.html',
            f'/profile/{self.author}/': 'posts/profile.html',
            f'/posts/{self.post.id}/': 'posts/post_detail.html',
            f'/posts/{self.post.id}/edit/': 'posts/create_post.html',
            '/create/': 'posts/create_post.html',
            '/follow/': 'posts/follow.html',
            '/unexisting_page/': 'core/404.html'
        }
        for url, template in templates_url_names.items():
            with self.subTest(url=url):
                response = self.post_author.get(url)
                self.assertTemplateUsed(response, template)


import pygame
from pygame.sprite import Sprite

class Star(Sprite):
    """A class to represent a single star in the sky."""

    def __init__(self, ai_game):
        """Initialize the sky and set its starting position."""
        super().__init__()
        self.screen = ai_game.screen

        # Load the star image and set its rect attribute.
        self.image = pygame.image.load('img/star_xs.png')
        self.rect = self.image.get_rect()

        # Start each new star near  the top left of the screen.
        self.rect.x = self.rect.width
        self.rect.y = self.rect.height

        # Store the star's exact horizontal and vertical positions.
        self.x = float(self.rect.x)
        self.y = float(self.rect.y)


import os
import random
import warnings
# import mxnet as mx
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from datetime import datetime, timedelta
from statsmodels.tsa.stattools import adfuller
warnings.filterwarnings("ignore")
plt.style.use('dark_background')

def seed_everything():
    """
    Reproductibilidad de resultados
    Returns
    -------
    """
    random.seed(20)
    np.random.seed(20)
    # mx.random.seed(20)

def plot_time_series(df, fecha_inicial="2014-12-01",
                     fecha_final="2014-12-14",
                     title="Consumo Electrico",
                     ylabel="Consumo Electrico [kW]",
                     sample=10):
    """
    Plot de la serie de tiempo que tiene como indice la fehca en formato
    timestamp

    Parameters
    ----------
    df : dataframe
        datos de las series de tiempo ordenados hacia el lado.
    fecha_inicial : str
        fecha inicial en el formato %y-%m-%d. The default is "2014-12-01".
    fecha_final : TYPE
        fecha final en el formato %y-%m-%d. The default is "2014-12-14".
    title : str, optional
        titulo. The default is "Consumo Electrico".
    ylabel : TYPE, optional
        nombre de eje Y. The default is "Consumo Electrico [kW]".
    sample : TYPE, optional
        cuantas columans quieres tomar. The default is 10.
    Returns
    -------
    Gráficos en una sola plana
    """
    fig, axs = plt.subplots(5, 4, figsize=(40, 40), sharex=True)
    axx = axs.ravel()
    for i in range(0, sample):
        df[df.columns[i]].loc[fecha_inicial:fecha_final].plot(ax=axx[i])
        axx[i].set_xlabel("Fecha [dias]")
        axx[i].set_ylabel(ylabel)
        axx[i].set_title(title)
        axx[i].grid(which='minor', axis='x')

def plot_prob_forecasts(ts_entry, forecast_entry, prediction_lentgh,
                        prediction_intervals=(80.0, 95.0),
                        problem="caso_electricidad",
                        color="k"):
    """
    Plot del conjunto de testeo con intervalos de confianza definidos

    Parameters
    ----------
    ts_entry : dataframe
        dataframe con fecha y cantidad consumida.
    forecast_entry : TYPE
        DESCRIPTION.
    prediction_lentgh : TYPE
        DESCRIPTION.
    prediction_intervals : TYPE, optional
        DESCRIPTION. The default is (80.0, 95.0).

    Returns
    -------
    None.

    """
    plot_length = prediction_lentgh
    legend = ["observaciones", "mediana de la predicción"] + \
        [f"{k}% intervalo de la predicción" for k in prediction_intervals][
            ::-1]
    fig, ax = plt.subplots(1, 1, figsize=(20, 14))
    ts_entry[-plot_length:].plot(ax=ax, color='red')
    forecast_entry.plot(prediction_intervals=prediction_intervals,
                        color=color)
    plt.grid(which="both")
    plt.legend(legend, loc="upper left", fontsize=22)
    ax.set_ylabel("Observaciones", fontsize=25)
    ax.set_xlabel("Tiempo", fontsize=25)
    ax.set_title("Observacion vs Predicciones + IC", fontsize=30)
    plt.show()
    try_create_folder(path="images")
    problem = problem + "_" + str(ts_entry.columns[0])
    fig.savefig(f"images/{problem}.png")

def try_create_folder(path="images"):
    """
    Intentar crear carpeta
    Parameters
    ----------
    path : string
        direccion.
    """
    try:
        os.mkdir(path)
    except Exception as e:
        pass

def extract_base(string):
    """
    Extrer la base del sku
    Parameters
    ----------
    string : string
        skus.
    Returns
    -------
    base : TYPE
        DESCRIPTION.
    """
    ind = string.find("|")
    base = string[0: ind]
    return base

def drop_spaces_data(df):
    """
    sacar los espacios de columnas que podrián venir interferidas
    Parameters
    ----------
    df : dataframe
        input data
    column : string
        string sin espacios en sus columnas
    Returns
    -------
    """
    for column in df.columns:
        try:
            df[column] = df[column].str.lstrip()
            df[column] = df[column].str.rstrip()
        except Exception as e:
            print(e)
            pass
    return df

def make_empty_identifiable(value):
    """
    Parameters
    ----------
    value : int, string, etc
        valor con el que se trabaja.
    Returns
    -------
    nans en los vacios.
    """
    if value == "":
        output = np.nan
    else:
        output = value
    return output

def replace_empty_nans(df):
    """
    Parameters
    ----------
    df : int, string, etc
        valor con el que se trabaja.
    Returns
    -------
    nans en los vacios.
    """
    for col in df.columns:
        print("buscando vacios en:", col, "...")
        df[col] = df[col].apply(lambda x: make_empty_identifiable(x))
    return df

def df_convert_float(df):
    """
    Pasa por las columnas tratando de convertirlas a float64
    Parameters
    ----------
    df : dataframe
        df de trabajo.
    Returns
    -------
    df : dataframe
        df con las columnas númericas en float.
    """
    for col in df.columns:
        try:
            df[col] = df[col].apply(float)
        except Exception as e:
            print(e)
    df.reset_index(drop=True, inplace=True)
    return df

def comentary_stationarity(p):
    """
    Comentario de la estacionaridad de las

    Parameters
    ----------
    p : TYPE
        DESCRIPTION.

    Returns
    -------
    comments : TYPE
        DESCRIPTION.

    """
    if p <= 0.05:
        comments =\
            "Rechaza la hipótesis nula (H0), los datos son estacionarios"
    else:
        comments =\
            "No se rechaza la hipótesis nula (H0), los datos no son estacionarios"
    return comments

def creating_rolling_dataset(data, mean_rolling=30):
    """
    Entregado un dataset, crear el mismo dataset pero a través de medias
    moviles, de tal forma de generar predicciones en ella
    Parameters
    ----------
    data : dataframe
        dataframe con las demandas.
    mean_rolling : int, optional
        tamaño de la ventana temporal para hacer las medias moviles.
        The default is 30.
    Returns
    -------
    result : dataframe
        dataframe de medias moviles de data.
    """
    result = data[["fecha"]]
    # columnas a iterar sobre
    columns = list(data.columns)
    columns.remove("fecha")
    for col in columns:
        col_name = f"rolling_{col}"
        datai = data[["fecha", col]]
        datai.set_index("fecha", inplace=True)
        datai[col_name] = datai[col].rolling(mean_rolling).mean()
        datai = datai[[col_name]]
        datai.reset_index(drop=False, inplace=True)
        result = result.merge(datai, on=["fecha"], how="outer")
    result.dropna(inplace=True)
    result.reset_index(drop=True, inplace=True)
    return result

def index_date(df):
    """
    Indexar la columna de las fechas
    Parameters
    ----------
    df : dataframe
        dataframe de demandas de stock en el tiempo.
    Returns
    -------
    df : dataframe
        dataframe con la fecha como index.
    """
    df.rename(columns={"fecha": "index"}, inplace=True)
    df.set_index(["index"], inplace=True)
    return df

def stationary_adf_test(df):
    """
    Test de estacionaridad
     Augmented Dickey-Fuller puede ser una de las más utilizadas.
     Utiliza un modelo autorregresivo y optimiza un criterio de información
     a través de múltiples valores de retardo (lags) diferentes. La hipótesis
     nula de la prueba es que la serie de tiempo se puede representar
     mediante una raíz unitaria, que no es estacionaria (tiene alguna
     estructura dependiente del tiempo). La hipótesis alternativa
     (que rechaza la hipótesis nula) es que la serie de tiempo es
     estacionaria.

    * Hipótesis nula (H0): si no se rechaza, sugiere que la serie de tiempo
    tiene una raíz unitaria, lo que significa que no es estacionaria.
    Tiene alguna estructura dependiente del tiempo --> problemas.
    * Hipótesis alternativa (H1): Se rechaza la hipótesis nula; sugiere que
    la serie de tiempo no tiene una raíz unitaria, lo que significa que es
    estacionaria. No tiene una estructura dependiente del tiempo.

    Voy a interpretar este resultado utilizando el valor p de la prueba.
    Un valor p por debajo de un umbral (como 5% o 1%) sugiere que rechazamos
    la hipótesis nula (estacionaria); de lo contrario, un valor p por encima
    del umbral sugiere que no rechazamos la hipótesis nula (no estacionaria),
    este es el lo clásico en tests estadísticos.

    * Valor p> 0.05: No se rechaza la hipótesis nula (H0),
        los datos no son estacionarios
    * Valor de p <= 0.05: Rechaza la hipótesis nula (H0),
        los datos son estacionarios

    Parameters
    ----------
    df : dataframe
        dataframe al cual se le harán pruebas de estacionaridad a todas sus
        columnas.
    Returns
    -------
    output : dataframe
        resultados del test.
    """
    output = []
    for col in df.columns:
        if "fecha" in col:
            pass
        else:
            datai = df[[col]]
            # datai = datai[datai[col] > 0]
            result = adfuller(datai.values)
            # descomprimir valores
            p = result[1]
            print("Para la columna: ", col)
            print(comentary_stationarity(p))
            print('ADF estadisticas: %f' % result[0])
            print('Valor de p: %f' % result[1])
            print('Valores criticos:')
            for key, value in result[4].items():
                print('\t%s: %.3f' % (key, value))
            output.append([col, comentary_stationarity(p),
                          p, list(result[4].items())])
    output = pd.DataFrame(output, columns=["columna", "resultado",
                                           "valor_p", "intervalos"])
    return output

def downcast_dtypes(df):
    """
    Función super util para bajar la cantidad de operaciones flotante que
    se van a realizar en el proceso de entrenamiento de la red
    Parameters
    ----------
    df : dataframe
        df a disminuir la cantidad de operaciones flotantes.
    Returns
    -------
    df : dataframe
        dataframe con los tipos int16 y float32 como formato número
    """
    float_cols = [c for c in df if df[c].dtype == "float64"]
    int_cols = [c for c in df if df[c].dtype in ["int64", "int32"]]
    df[float_cols] = df[float_cols].astype(np.float32)
    df[int_cols] = df[int_cols].astype(np.int16)
    return df

def lstm_preparation(array, timesteps=30):
    """
    Preparar los datos para la predicción con la lstm
    Parameters
    ----------
    array : numpy.array
        array.
    timesteps : int, optional
        cantidad de tiemsteps que se harán las predicciones.
        The default is 30.
    Returns
    -------
    x_train : array
        matriz de entrenamiento de las celdas lstm.
    y_train : array
        salida de las celdas.
    """
    x_train = []
    y_train = []
    for i in range(timesteps, array.shape[0]):
        x_train.append(array[i-timesteps:i])
        y_train.append(array[i][0:array.shape[1]])
    x_train = np.array(x_train, dtype='float32')
    y_train = np.array(y_train, dtype='float32')
    return x_train, y_train

def always_number(df):
    """
    Pasa por las columnas del dataframe poniendo valores > 0
    Parameters
    ----------
    df : dataframe
        df de trabajo.
    Returns
    -------
    df : dataframe
        df con las columnas númericas en float.
    """
    for col in df.columns:
        try:
            df[col] = df[col].apply(lambda x: 10e-20 if x == 0 else x)
        except Exception as e:
            print(e)
    df.reset_index(drop=True, inplace=True)
    return df

def delete_negatives(predictions):
    """
    No pueden haber predicciones negativas, asi que nos aseguramos de eso
    Parameters
    ----------
    predictions : array
        predicciones para todos los sku.
    Returns
    -------
    predictions : array
        predicciones con 0 en los casos en que estás eran negativas.
    """
    predictions = pd.DataFrame(predictions)
    for col in predictions.columns:
        predictions[col] = predictions[col].apply(
            lambda x: 0 if x < 0 else x)
    predictions = predictions.to_numpy()
    return predictions

def lstm_metric_evaluation(predictions, y_test, fechas, filter_names,
                           save=False):
    """
    Evaluar la suma de la demanda predecida para el ciclo, con la suma de
    la demanda real en el ciclo de test
    Parameters
    ----------
    predictions : array
        predicciones.
    y_test : array
        testing.
    fechas : array
        array con las fechas de testing.
    filter names : array
        array con los nombres de los articulos.
    save : boolean, optional
        Decide si se guardan o no las images de los resultados.
        The default is False.
    Returns
    -------
    mean : dataframe
        metricas de evaluación.
    """
    mean = []
    for i in range(y_test.shape[1]):
        print("columna: ", i)
        predi = predictions[:, i]
        testi = y_test[:, i]
        plot_sequence(predi, testi, fechas, filter_names[i], save)
        error = np.abs(predi - testi).mean()
        suma_pred = predi.sum()
        suma_test = testi.sum()
        pt = np.abs(suma_test - suma_pred) / suma_test * 100
        acc = calculate_accuracy(suma_test, suma_pred)
        mean.append([i, error, suma_pred, suma_test, pt, acc])
    mean = pd.DataFrame(mean, columns=["id_columna", "mae", "suma_pred",
                                       "suma_test", "mape", "acc"])
    # mean = delete_negatives(mean)
    # mean = pd.DataFrame(mean, columns=["id_columna", "mae", "suma_pred",
    #                                    "suma_test", "mape", "acc"])
    return mean

def calculate_accuracy(real, predict):
    """
    Calcular accuracy según metodologia de:
    [https://github.com/huseinzol05/Stock-Prediction-Models/blob/master/
    deep-learning/16.attention-is-all-you-need.ipynb]
    Parameters
    ----------
    real : float, array
        real.
    predict : float, array
        predicciones.
    Returns
    -------
    TYPE
        float, array.
    """
    real = np.array(real)
    predict = np.array(predict)
    percentage = 1 - np.sqrt(np.mean(np.square((real - predict) / real)))
    return percentage * 100

def plot_instance_training(history, epocas_hacia_atras, model_name,
                           filename):
    """
    Sacar el historial de entrenamiento de epocas en partivular
    Parameters
    ----------
    history : object
        DESCRIPTION.
    epocas_hacia_atras : int
        epocas hacia atrás que queremos ver en el entrenamiento.
    model_name : string
        nombre del modelo.
    filename : string
        nombre del archivo.
    Returns
    -------
    bool
        gráficas de lo ocurrido durante el entrenamiento.
    """
    letter_size = 20
    # Hist training
    largo = len(history.history['loss'])
    x_labels = np.arange(largo-epocas_hacia_atras, largo)
    x_labels = list(x_labels)
    # Funciones de costo
    loss_training = history.history['loss'][-epocas_hacia_atras:]
    loss_validation = history.history['val_loss'][-epocas_hacia_atras:]
    # Figura
    fig, ax = plt.subplots(1, figsize=(16, 8))
    ax.plot(x_labels, loss_training, 'b', linewidth=2)
    ax.plot(x_labels, loss_validation, 'r', linewidth=2)
    ax.set_xlabel('Epocas', fontname="Arial", fontsize=letter_size-5)
    ax.set_ylabel('Función de costos', fontname="Arial",
                  fontsize=letter_size-5)
    ax.set_title(f"{model_name}", fontname="Arial", fontsize=letter_size)
    ax.legend(['Entrenamiento', 'Validación'], loc='upper left',
              prop={'size': letter_size-5})
    # Tamaño de los ejes
    for tick in ax.get_xticklabels():
        tick.set_fontsize(letter_size-5)
    for tick in ax.get_yticklabels():
        tick.set_fontsize(letter_size-5)
    plt.show()
    return fig

def training_history(history, model_name="Celdas LSTM", filename="LSTM"):
    """
    Según el historial de entrenamiento que hubo plotear el historial
    hacía atrás de las variables
    Parameters
    ----------
    history : list
        lista con errores de validación y training.
    model_name : string, optional
        nombre del modelo. The default is "Celdas LSTM".
    filename : string, optional
        nombre del archivo. The default is "LSTM".

    Returns
    -------
    None.

    """
    size_training = len(history.history['val_loss'])
    fig = plot_instance_training(history, size_training, model_name,
                                 filename + "_ultimas:" +
                                 str(size_training) + "epocas")

    fig = plot_instance_training(history, int(1.5 * size_training / 2),
                                 model_name,
                                 filename + "_ultimas:" +
                                 str(1.5 * size_training / 2) + "epocas")
    # guardar el resultado de entrenamiento de la lstm
    fig.savefig(f"results/{model_name}_training.png")

    fig = plot_instance_training(history, int(size_training / 2),
                                 model_name,
                                 filename + "_ultimas:" + str(
                                     size_training / 2) + "epocas")

    fig = plot_instance_training(history, int(size_training / 3), model_name,
                                 filename + "_ultimas:" +
                                 str(size_training / 3) + "epocas")
    fig = plot_instance_training(history, int(size_training / 4), model_name,
                                 filename + "_ultimas:" + str(
                                     size_training / 4) + "epocas")
    print(fig)

def plot_sequence(predictions, real, fechas, indice, save):
    """
    Plot sequence de la secuecnia
    Parameters
    ----------
    predictions : array
        predicciones.
    real : array
        valores reales.
    fechas : array
        array de fechas.
    indice : string
        indice de la columna o nombre.
    save : boolean
        guarda o no guarda el archivo.
    Returns
    -------
    plot de prediciones vs real.
    """
    letter_size = 20
    new_fechas = []
    for fecha in fechas:
        fecha = fecha[0:10]
        new_fechas.append(fecha)

    plt.style.use('dark_background')
    fig, ax = plt.subplots(1, figsize=(20, 12))
    ax.plot(new_fechas, real, 'gold', linewidth=2)
    ax.plot(new_fechas, predictions, 'orangered', linewidth=2)
    ax.set_xlabel('Tiempo', fontname="Arial", fontsize=letter_size)
    ax.set_ylabel('Predicción vs Real', fontname="Arial",
                  fontsize=letter_size+2)
    ax.set_title(f"Predicciones vs real {str(indice)}",
                 fontname="Arial", fontsize=letter_size+10)
    ax.legend(['real', 'predicción'], loc='upper left',
              prop={'size': letter_size+5})
    # Tamaño de los ejes
    for tick in ax.get_xticklabels():
        tick.set_fontsize(letter_size)
    for tick in ax.get_yticklabels():
        tick.set_fontsize(letter_size)
    try_create_folder("results/lstm")
    plt.xticks(rotation=75)
    plt.show()

    path = f"results/lstm/{indice}.png"
    path = path.replace(" ", "").replace("-", "_")
    print("Nombre: ", path)
    print("Indice: ", indice)

    if save:
        fig.savefig(path)

def grouping_df(df, days=10, arg="sum"):
    """
    Dado un dataframe que posee una columna de

    Parameters
    ----------
    df : dataframe
        dataframe a agrupar en una fecha dada.
    days : int, optional
        dias de la fecha. The default is 10.
    arg : string, optional
        sum o mean como argumentos. The default is "sum".
    Returns
    -------
    df : dataframe
        dataframe agrupado en esos dias.
    """
    df["fecha"] = df["fecha"].apply(
        lambda x: datetime.strptime(x, "%Y-%m-%d %H-%M-%S"))
    df["fecha_aprox"] = df["fecha"].apply(lambda x: x.round(f"{days}d"))
    if arg == "sum":
        print("Se agrupa por suma")
        df = df.groupby("fecha_aprox").sum()
    elif arg == "mean":
        print("Se agrupa por promedio")
        df = df.groupby("fecha_aprox").mean()
    else:
        print("Argumentos no validos, no se ha efectuado la operación")
    df.reset_index(drop=False, inplace=True)
    df.rename(columns={"fecha_aprox": "fecha"}, inplace=True)
    df["fecha"] = df["fecha"].apply(
        lambda x: x.strftime("%Y-%m-%d %H-%M-%S"))
    df.reset_index(drop=True, inplace=True)
    return df

def plot_prob_forecasts_deep_renewal(ts_entry, forecast_entry, title):
    """
    Plot del conjunto de testeo con intervalos de confianza definidos

    Parameters
    ----------
    ts_entry : dataframe
        dataframe con fecha y cantidad consumida.
    forecast_entry : TYPE
        DESCRIPTION.
    prediction_lentgh : TYPE
        DESCRIPTION.
    prediction_intervals : TYPE, optional
        DESCRIPTION. The default is (80.0, 95.0).

    Returns
    -------
    None.

    """
    plot_length = 100
    prediction_intervals = (50.0, 90.0)
    legend = ["observations", "median prediction"] + \
        [f"{k}% prediction interval" for k in prediction_intervals][::-1]
    title =\
        title if forecast_entry.item_id is None else title +\
        "|"+forecast_entry.item_id
    fig, ax = plt.subplots(1, 1, figsize=(10, 7))
    ts_entry[-plot_length:].plot(ax=ax)
    forecast_entry.plot(prediction_intervals=prediction_intervals, color='g')
    plt.grid(which="both")
    plt.legend(legend, loc="upper left")
    plt.title(title)
    plt.show()

def get_demand_prediction(array, demand, fechas, names, rolling_window,
                          args):
    """
    Hacer transformaciones para hacer predicciones sobre la demanda real
    no la ventana deslizante.

    Parameters
    ----------
    array : array de prediciones
        DESCRIPTION.
    original : dataframe
        dataframe de datos originales de la demanda.
    fechas : list
        fechas donde se hace el testing.
    names : list
        nombres de las columnas.
    rolling_window: int
        largo de la ventana deslizante movil
    rolling_window: args-parser
        argumentos de setting del proyecto
    Returns
    -------
    Metricas de resultado en función de la demanda
    """
    # valores de default para probar
    # array = predictions.copy()
    # demand = demand_real.copy()

    # ventana deslizante resultados
    rolling = pd.DataFrame(array, columns=names)
    rolling["fecha"] = pd.Series(fechas)
    rolling["fecha"] = rolling["fecha"].apply(
        lambda x: datetime.strptime(x, args.date_format))

    # formato fecha
    demand["fecha"] = demand["fecha"].apply(
        lambda x: datetime.strptime(x, args.date_format))
    # filtrar los datos necesarios para el filtrado de la ventana temporal
    fecha_inicial = rolling["fecha"].min() - timedelta(days=rolling_window)
    demand = demand[demand["fecha"] >= fecha_inicial]
    demand.reset_index(drop=True, inplace=True)
    # columnas a iterar
    columns = list(rolling.columns)
    columns.remove("fecha")
    # descomprimir demanda
    for rolling_col in columns:
        original_col = rolling_col.replace("rolling_", "")
        print(original_col)
        rollingi = rolling[[rolling_col]].to_numpy()
        for i in range(rolling_window, demand.shape[0]):
            # recalcular el vector de demanda nuevamente
            demandi = demand[[original_col]].to_numpy()
            # sacar la media movil predicha
            mean_rolling = rollingi[i-rolling_window][0]
            # calcula la suma acumulada que se deberia tener
            suma_rolling1 = demandi[i-rolling_window: i-1].sum()
            # calcula la demanda estimada
            predict_demandi = mean_rolling * rolling_window - suma_rolling1
            # en el caso de ser negativa la setea a cero
            predict_demandi = max(predict_demandi, 0)
            # cambia el valor en el dataframe
            demand[original_col].iloc[i] = predict_demandi
    # filtra para que queden solo las predicciones hechas
    demand = demand[demand["fecha"] >= rolling["fecha"].min()]
    demand.reset_index(drop=True, inplace=True)
    return demand

def mape_computation(real, predict):
    """
    Mean porcentage error computation
    Parameters
    ----------
    real : float
        valor real.
    predict : float
        valor de la prediccion.
    Returns
    -------
    mape : float
        mean percentage error.
    """
    error = np.abs(real - predict)
    mape = error / real * 100
    return mape

def evaluate_demand_predictions(demand_predictions, demand_real, fechas,
                                args):
    """
    Evaluar la predicción de la demanda

    Parameters
    ----------
    demand_predictions : dataframe
        dataframe con las predicciones de demanda descomprimida
    demand_real : dataframe
        demanda real del articulo
    fechas : list
        fechas de evaluacion.
    args : arg-parser
        parametros.
    Returns
    -------
    output : dataframe
        resultados.
    """
    # fechas en las que se esta haciendo las predicciones
    df_fechas = pd.DataFrame(fechas, columns=["fecha"])
    df_fechas["fecha"] = df_fechas["fecha"].apply(
        lambda x: datetime.strptime(x, args.date_format))
    # formato fecha
    demand_real["fecha"] = demand_real["fecha"].apply(str)
    demand_real["fecha"] = demand_real["fecha"].apply(
        lambda x: datetime.strptime(x, args.date_format))
    # filtrar en las fechas utilizadas solamente
    demand_real = demand_real[demand_real["fecha"]
                              >= df_fechas["fecha"].min()]
    demand_real.reset_index(drop=True, inplace=True)
    # columnas a iterar
    columns = list(demand_real.columns)
    columns.remove("fecha")
    output = []
    # iterar sobre estas columnas
    for col in columns:
        # print("Calculando accuracy:", col, "...")
        reali = demand_real[col].sum()
        preidictionsi = demand_predictions[col].sum()
        if reali > 0:
            acc = calculate_accuracy(reali, preidictionsi)
            mape = mape_computation(reali, preidictionsi)
            # print(mape, 100-mape, acc)
        else:
            acc = -1
            mape = -1
        output.append([col, mape, acc])
    output = pd.DataFrame(output, columns=["columna", "mape", "accuracy"])
    output.reset_index(drop=True, inplace=True)

    promedio_error = output[output["mape"] > 0]["mape"].mean()
    promedio_acc = output[output["mape"] > 0]["accuracy"].mean()
    print("Error promedio de los modelos", promedio_error)
    print("Accuracy promedio de los modelos", promedio_acc)
    output["mape"] = output["mape"].apply(
        lambda x: "No determinado" if x == -1 else x)
    output["accuracy"] = output["accuracy"].apply(
        lambda x: "No determinado" if x == -1 else x)
    return output


#!/usr/bin/env python
# coding: utf-8

# # Dataset Preprocessing
# ## Import preprocessing libraries
import pandas as pd
import numpy as np
from sklearn.impute import SimpleImputer
import sklearn.datasets as ds
from torch import torch

# ## Preprocessing UCI

mat = pd.read_csv( "slump_test.data", header = 0, sep =",").values[:,1:]
np.save("UCI_dataset_01", np.delete(mat, -3, axis=1)[:,:-1])
np.save("UCI_dataset_02", np.delete(np.delete(mat, -2, axis=1) , -2, axis=1))

mat = pd.read_csv( "machine.data", header = None, sep =",").values[:,2:]
mat[mat == "?"] =None
mat = mat.astype(float)
mat[mat == None] = np.nan
mat = SimpleImputer(missing_values=np.nan, strategy='mean').fit_transform(mat)
np.save("UCI_dataset_03", mat)

df = pd.read_excel( "2014 and 2015 CSM dataset.xlsx", header = 0)
Ratings_cols = ['Budget', 'Screens',
       'Sequel', 'Sentiment', 'Views', 'Likes', 'Dislikes', 'Comments',
       'Aggregate Followers', 'Ratings']
mat = df[Ratings_cols].values
mat = SimpleImputer(missing_values=np.nan, strategy='mean').fit_transform(mat)
np.save("UCI_dataset_04", mat)

np.save("UCI_dataset_05", pd.read_csv( "yacht_hydrodynamics.data", header = None, sep="\s+").values)

np.save("UCI_dataset_06",pd.read_excel( "Concrete_Data.xls", header = 0).values.astype(float))

np.save("UCI_dataset_07", pd.read_csv("airfoil_self_noise.dat", header = None, sep ="\t").values)

mat = pd.read_csv( "communities.data", header = None, sep =",").values[:,5:]
mat[mat == "?"] =None
mat = mat.astype(float)
mat[mat == None] = np.nan
mat = SimpleImputer(missing_values=np.nan, strategy='mean').fit_transform(mat)
np.save("UCI_dataset_08", mat)

np.save("UCI_dataset_09", pd.read_csv( "CASP.csv", header = 0)[['F1', 'F2', 'F3', 'F4', 'F5', 'F6', 'F7', 'F8', 'F9', 'RMSD']].values.astype(float))

np.save("UCI_dataset_10", pd.read_csv("Relation Network (Directed).data", header = None, sep =",").values[:,1:].astype(float))

cols = ['duration', 'width', 'height', 'bitrate', 'framerate',
       'i', 'p', 'b', 'frames', 'i_size', 'p_size', 'b_size', 'size',
        'o_bitrate', 'o_framerate', 'o_width', 'o_height', 'umem',
       'utime']

np.save("UCI_dataset_11", pd.read_csv("transcoding_mesurment.tsv" , header = 0, sep = "\t")[cols].values)

df = pd.read_csv("UJIndoorLoc/trainingData.csv" , header = 0, sep = ",").iloc[:,:522]
lat = [col for col in df.columns if col != 'LONGITUDE']
long = [col for col in df.columns if col != 'LATITUDE']
np.save("UCI_dataset_12", df[lat].values)
np.save("UCI_dataset_13", df[long].values)

np.save("UCI_dataset_14", pd.read_csv("slice_localization_data.csv" , header = 0, sep = ",").values[:,1:])

# ## Preprocessing news20

dataset_name = "news20"
X_train, y_train = ds.load_svmlight_file(dataset_name+'.scale', multilabel = False)
X_test, y_test = ds.load_svmlight_file(dataset_name+'.t.scale', multilabel = False)

cut_feature = 35830
#arbitrarly take 35830 first components to speed up computation time
X_train, X_test = X_train[:, :cut_feature], X_test[:, :cut_feature]

XT = X_train.transpose()
XTX = XT.dot(X_train)
dense_XTX = XTX.todense()
tensor_XTX = torch.tensor(dense_XTX).float()

# quite long process on cpu... (about 30 minutes)
with torch.no_grad():
    E, U = torch.symeig(tensor_XTX, eigenvectors=True)
E = E.detach().numpy()
U = U.detach().numpy()

# keep 80 of variance by selecting the biggest eigen values
pos = E[E > 0.]
repartition = pos[::-1].cumsum()/pos.sum()
cut_eigen = 0.8
E_cut = E[-(repartition< cut_eigen).astype(int).sum():]
U_cut = U[:,-(repartition< cut_eigen).astype(int).sum():]
np.save(dataset_name+"_E_"+str(cut_eigen)+"_cut.npy",E_cut)
np.save(dataset_name+"_U_"+str(cut_eigen)+"_cut.npy",U_cut)

# transform features in principal components
np.save(dataset_name+"_X_train",X_train.dot(U_cut))
np.save(dataset_name+"_X_test",X_test.dot(U_cut))