""" test fancy indexing & misc """

from datetime import datetime
import re
import weakref

import numpy as np
import pytest

from pandas.core.dtypes.common import is_float_dtype, is_integer_dtype

import pandas as pd
from pandas import DataFrame, Index, NaT, Series
import pandas._testing as tm
from pandas.core.indexing import maybe_numeric_slice, non_reducing_slice
from pandas.tests.indexing.common import _mklbl

from .test_floats import gen_obj

def getitem(x):
    return x

def setitem(x):
    return x

def loc(x):
    return x.loc

def iloc(x):
    return x.iloc

# ------------------------------------------------------------------------
# Indexing test cases

class TestFancy:
    """ pure get/set item & fancy indexing """

    def test_setitem_ndarray_1d(self):
        # GH5508

        # len of indexer vs length of the 1d ndarray
        df = DataFrame(index=Index(np.arange(1, 11)))
        df["foo"] = np.zeros(10, dtype=np.float64)
        df["bar"] = np.zeros(10, dtype=complex)

        # invalid
        msg = "Must have equal len keys and value when setting with an iterable"
        with pytest.raises(ValueError, match=msg):
            df.loc[df.index[2:5], "bar"] = np.array([2.33j, 1.23 + 0.1j, 2.2, 1.0])

        # valid
        df.loc[df.index[2:6], "bar"] = np.array([2.33j, 1.23 + 0.1j, 2.2, 1.0])

        result = df.loc[df.index[2:6], "bar"]
        expected = Series(
            [2.33j, 1.23 + 0.1j, 2.2, 1.0], index=[3, 4, 5, 6], name="bar"
        )
        tm.assert_series_equal(result, expected)

        # dtype getting changed?
        df = DataFrame(index=Index(np.arange(1, 11)))
        df["foo"] = np.zeros(10, dtype=np.float64)
        df["bar"] = np.zeros(10, dtype=complex)

        msg = "Must have equal len keys and value when setting with an iterable"
        with pytest.raises(ValueError, match=msg):
            df[2:5] = np.arange(1, 4) * 1j

    @pytest.mark.parametrize("idxr", [getitem, loc, iloc])
    def test_getitem_ndarray_3d(self, index, frame_or_series, idxr):
        # GH 25567
        obj = gen_obj(frame_or_series, index)
        idxr = idxr(obj)
        nd3 = np.random.randint(5, size=(2, 2, 2))

        msg = "|".join(
            [
                r"Buffer has wrong number of dimensions \(expected 1, got 3\)",
                "Cannot index with multidimensional key",
                r"Wrong number of dimensions. values.ndim != ndim \[3 != 1\]",
                "Index data must be 1-dimensional",
                "positional indexers are out-of-bounds",
                "Indexing a MultiIndex with a multidimensional key is not implemented",
            ]
        )

        potential_errors = (IndexError, ValueError, NotImplementedError)
        with pytest.raises(potential_errors, match=msg):
            with tm.assert_produces_warning(DeprecationWarning, check_stacklevel=False):
                idxr[nd3]

    @pytest.mark.parametrize("indexer", [setitem, loc, iloc])
    def test_setitem_ndarray_3d(self, index, frame_or_series, indexer):
        # GH 25567
        obj = gen_obj(frame_or_series, index)
        idxr = indexer(obj)
        nd3 = np.random.randint(5, size=(2, 2, 2))

        if indexer.__name__ == "iloc":
            err = ValueError
            msg = f"Cannot set values with ndim > {obj.ndim}"
        elif (
            isinstance(index, pd.IntervalIndex)
            and indexer.__name__ == "setitem"
            and obj.ndim == 1
        ):
            err = AttributeError
            msg = (
                "'pandas._libs.interval.IntervalTree' object has no attribute 'get_loc'"
            )
        else:
            err = ValueError
            msg = r"Buffer has wrong number of dimensions \(expected 1, got 3\)|"

        with pytest.raises(err, match=msg):
            idxr[nd3] = 0

    def test_inf_upcast(self):
        # GH 16957
        # We should be able to use np.inf as a key
        # np.inf should cause an index to convert to float

        # Test with np.inf in rows
        df = DataFrame(columns=[0])
        df.loc[1] = 1
        df.loc[2] = 2
        df.loc[np.inf] = 3

        # make sure we can look up the value
        assert df.loc[np.inf, 0] == 3

        result = df.index
        expected = pd.Float64Index([1, 2, np.inf])
        tm.assert_index_equal(result, expected)

        # Test with np.inf in columns
        df = DataFrame()
        df.loc[0, 0] = 1
        df.loc[1, 1] = 2
        df.loc[0, np.inf] = 3

        result = df.columns
        expected = pd.Float64Index([0, 1, np.inf])
        tm.assert_index_equal(result, expected)

    def test_setitem_dtype_upcast(self):

        # GH3216
        df = DataFrame([{"a": 1}, {"a": 3, "b": 2}])
        df["c"] = np.nan
        assert df["c"].dtype == np.float64

        df.loc[0, "c"] = "foo"
        expected = DataFrame(
            [{"a": 1, "b": np.nan, "c": "foo"}, {"a": 3, "b": 2, "c": np.nan}]
        )
        tm.assert_frame_equal(df, expected)

        # GH10280
        df = DataFrame(
            np.arange(6, dtype="int64").reshape(2, 3),
            index=list("ab"),
            columns=["foo", "bar", "baz"],
        )

        for val in [3.14, "wxyz"]:
            left = df.copy()
            left.loc["a", "bar"] = val
            right = DataFrame(
                [[0, val, 2], [3, 4, 5]],
                index=list("ab"),
                columns=["foo", "bar", "baz"],
            )

            tm.assert_frame_equal(left, right)
            assert is_integer_dtype(left["foo"])
            assert is_integer_dtype(left["baz"])

        left = DataFrame(
            np.arange(6, dtype="int64").reshape(2, 3) / 10.0,
            index=list("ab"),
            columns=["foo", "bar", "baz"],
        )
        left.loc["a", "bar"] = "wxyz"

        right = DataFrame(
            [[0, "wxyz", 0.2], [0.3, 0.4, 0.5]],
            index=list("ab"),
            columns=["foo", "bar", "baz"],
        )

        tm.assert_frame_equal(left, right)
        assert is_float_dtype(left["foo"])
        assert is_float_dtype(left["baz"])

    def test_dups_fancy_indexing(self):

        # GH 3455

        df = tm.makeCustomDataframe(10, 3)
        df.columns = ["a", "a", "b"]
        result = df[["b", "a"]].columns
        expected = Index(["b", "a", "a"])
        tm.assert_index_equal(result, expected)

        # across dtypes
        df = DataFrame([[1, 2, 1.0, 2.0, 3.0, "foo", "bar"]], columns=list("aaaaaaa"))
        df.head()
        str(df)
        result = DataFrame([[1, 2, 1.0, 2.0, 3.0, "foo", "bar"]])
        result.columns = list("aaaaaaa")

        # TODO(wesm): unused?
        df_v = df.iloc[:, 4]  # noqa
        res_v = result.iloc[:, 4]  # noqa

        tm.assert_frame_equal(df, result)

        # GH 3561, dups not in selected order
        df = DataFrame(
            {"test": [5, 7, 9, 11], "test1": [4.0, 5, 6, 7], "other": list("abcd")},
            index=["A", "A", "B", "C"],
        )
        rows = ["C", "B"]
        expected = DataFrame(
            {"test": [11, 9], "test1": [7.0, 6], "other": ["d", "c"]}, index=rows
        )
        result = df.loc[rows]
        tm.assert_frame_equal(result, expected)

        result = df.loc[Index(rows)]
        tm.assert_frame_equal(result, expected)

        rows = ["C", "B", "E"]
        with pytest.raises(KeyError, match="with any missing labels"):
            df.loc[rows]

        # see GH5553, make sure we use the right indexer
        rows = ["F", "G", "H", "C", "B", "E"]
        with pytest.raises(KeyError, match="with any missing labels"):
            df.loc[rows]

        # List containing only missing label
        dfnu = DataFrame(np.random.randn(5, 3), index=list("AABCD"))
        with pytest.raises(
            KeyError,
            match=re.escape(
                "\"None of [Index(['E'], dtype='object')] are in the [index]\""
            ),
        ):
            dfnu.loc[["E"]]

        # ToDo: check_index_type can be True after GH 11497

        # GH 4619; duplicate indexer with missing label
        df = DataFrame({"A": [0, 1, 2]})
        with pytest.raises(KeyError, match="with any missing labels"):
            df.loc[[0, 8, 0]]

        df = DataFrame({"A": list("abc")})
        with pytest.raises(KeyError, match="with any missing labels"):
            df.loc[[0, 8, 0]]

        # non unique with non unique selector
        df = DataFrame({"test": [5, 7, 9, 11]}, index=["A", "A", "B", "C"])
        with pytest.raises(KeyError, match="with any missing labels"):
            df.loc[["A", "A", "E"]]

    def test_dups_fancy_indexing2(self):
        # GH 5835
        # dups on index and missing values
        df = DataFrame(np.random.randn(5, 5), columns=["A", "B", "B", "B", "A"])

        with pytest.raises(KeyError, match="with any missing labels"):
            df.loc[:, ["A", "B", "C"]]

        # GH 6504, multi-axis indexing
        df = DataFrame(
            np.random.randn(9, 2), index=[1, 1, 1, 2, 2, 2, 3, 3, 3], columns=["a", "b"]
        )

        expected = df.iloc[0:6]
        result = df.loc[[1, 2]]
        tm.assert_frame_equal(result, expected)

        expected = df
        result = df.loc[:, ["a", "b"]]
        tm.assert_frame_equal(result, expected)

        expected = df.iloc[0:6, :]
        result = df.loc[[1, 2], ["a", "b"]]
        tm.assert_frame_equal(result, expected)

    @pytest.mark.parametrize("case", [getitem, loc])
    def test_duplicate_int_indexing(self, case):
        # GH 17347
        s = Series(range(3), index=[1, 1, 3])
        expected = s[1]
        result = case(s)[[1]]
        tm.assert_series_equal(result, expected)

    def test_indexing_mixed_frame_bug(self):

        # GH3492
        df = DataFrame(
            {"a": {1: "aaa", 2: "bbb", 3: "ccc"}, "b": {1: 111, 2: 222, 3: 333}}
        )

        # this works, new column is created correctly
        df["test"] = df["a"].apply(lambda x: "_" if x == "aaa" else x)

        # this does not work, ie column test is not changed
        idx = df["test"] == "_"
        temp = df.loc[idx, "a"].apply(lambda x: "-----" if x == "aaa" else x)
        df.loc[idx, "test"] = temp
        assert df.iloc[0, 2] == "-----"

    def test_multitype_list_index_access(self):
        # GH 10610
        df = DataFrame(np.random.random((10, 5)), columns=["a"] + [20, 21, 22, 23])

        with pytest.raises(KeyError, match=re.escape("'[-8, 26] not in index'")):
            df[[22, 26, -8]]
        assert df[21].shape[0] == df.shape[0]

    def test_set_index_nan(self):

        # GH 3586
        df = DataFrame(
            {
                "PRuid": {
                    17: "nonQC",
                    18: "nonQC",
                    19: "nonQC",
                    20: "10",
                    21: "11",
                    22: "12",
                    23: "13",
                    24: "24",
                    25: "35",
                    26: "46",
                    27: "47",
                    28: "48",
                    29: "59",
                    30: "10",
                },
                "QC": {
                    17: 0.0,
                    18: 0.0,
                    19: 0.0,
                    20: np.nan,
                    21: np.nan,
                    22: np.nan,
                    23: np.nan,
                    24: 1.0,
                    25: np.nan,
                    26: np.nan,
                    27: np.nan,
                    28: np.nan,
                    29: np.nan,
                    30: np.nan,
                },
                "data": {
                    17: 7.9544899999999998,
                    18: 8.0142609999999994,
                    19: 7.8591520000000008,
                    20: 0.86140349999999999,
                    21: 0.87853110000000001,
                    22: 0.8427041999999999,
                    23: 0.78587700000000005,
                    24: 0.73062459999999996,
                    25: 0.81668560000000001,
                    26: 0.81927080000000008,
                    27: 0.80705009999999999,
                    28: 0.81440240000000008,
                    29: 0.80140849999999997,
                    30: 0.81307740000000006,
                },
                "year": {
                    17: 2006,
                    18: 2007,
                    19: 2008,
                    20: 1985,
                    21: 1985,
                    22: 1985,
                    23: 1985,
                    24: 1985,
                    25: 1985,
                    26: 1985,
                    27: 1985,
                    28: 1985,
                    29: 1985,
                    30: 1986,
                },
            }
        ).reset_index()

        result = (
            df.set_index(["year", "PRuid", "QC"])
            .reset_index()
            .reindex(columns=df.columns)
        )
        tm.assert_frame_equal(result, df)

    def test_multi_assign(self):

        # GH 3626, an assignment of a sub-df to a df
        df = DataFrame(
            {
                "FC": ["a", "b", "a", "b", "a", "b"],
                "PF": [0, 0, 0, 0, 1, 1],
                "col1": list(range(6)),
                "col2": list(range(6, 12)),
            }
        )
        df.iloc[1, 0] = np.nan
        df2 = df.copy()

        mask = ~df2.FC.isna()
        cols = ["col1", "col2"]

        dft = df2 * 2
        dft.iloc[3, 3] = np.nan

        expected = DataFrame(
            {
                "FC": ["a", np.nan, "a", "b", "a", "b"],
                "PF": [0, 0, 0, 0, 1, 1],
                "col1": Series([0, 1, 4, 6, 8, 10]),
                "col2": [12, 7, 16, np.nan, 20, 22],
            }
        )

        # frame on rhs
        df2.loc[mask, cols] = dft.loc[mask, cols]
        tm.assert_frame_equal(df2, expected)

        df2.loc[mask, cols] = dft.loc[mask, cols]
        tm.assert_frame_equal(df2, expected)

        # with an ndarray on rhs
        # coerces to float64 because values has float64 dtype
        # GH 14001
        expected = DataFrame(
            {
                "FC": ["a", np.nan, "a", "b", "a", "b"],
                "PF": [0, 0, 0, 0, 1, 1],
                "col1": [0.0, 1.0, 4.0, 6.0, 8.0, 10.0],
                "col2": [12, 7, 16, np.nan, 20, 22],
            }
        )
        df2 = df.copy()
        df2.loc[mask, cols] = dft.loc[mask, cols].values
        tm.assert_frame_equal(df2, expected)
        df2.loc[mask, cols] = dft.loc[mask, cols].values
        tm.assert_frame_equal(df2, expected)

        # broadcasting on the rhs is required
        df = DataFrame(
            {
                "A": [1, 2, 0, 0, 0],
                "B": [0, 0, 0, 10, 11],
                "C": [0, 0, 0, 10, 11],
                "D": [3, 4, 5, 6, 7],
            }
        )

        expected = df.copy()
        mask = expected["A"] == 0
        for col in ["A", "B"]:
            expected.loc[mask, col] = df["D"]

        df.loc[df["A"] == 0, ["A", "B"]] = df["D"]
        tm.assert_frame_equal(df, expected)

    def test_setitem_list(self):

        # GH 6043
        # iloc with a list
        df = DataFrame(index=[0, 1], columns=[0])
        df.iloc[1, 0] = [1, 2, 3]
        df.iloc[1, 0] = [1, 2]

        result = DataFrame(index=[0, 1], columns=[0])
        result.iloc[1, 0] = [1, 2]

        tm.assert_frame_equal(result, df)

        # iloc with an object
        class TO:
            def __init__(self, value):
                self.value = value

            def __str__(self) -> str:
                return f"[{self.value}]"

            __repr__ = __str__

            def __eq__(self, other) -> bool:
                return self.value == other.value

            def view(self):
                return self

        df = DataFrame(index=[0, 1], columns=[0])
        df.iloc[1, 0] = TO(1)
        df.iloc[1, 0] = TO(2)

        result = DataFrame(index=[0, 1], columns=[0])
        result.iloc[1, 0] = TO(2)

        tm.assert_frame_equal(result, df)

        # remains object dtype even after setting it back
        df = DataFrame(index=[0, 1], columns=[0])
        df.iloc[1, 0] = TO(1)
        df.iloc[1, 0] = np.nan
        result = DataFrame(index=[0, 1], columns=[0])

        tm.assert_frame_equal(result, df)

    def test_string_slice(self):
        # GH 14424
        # string indexing against datetimelike with object
        # dtype should properly raises KeyError
        df = DataFrame([1], Index([pd.Timestamp("2011-01-01")], dtype=object))
        assert df.index._is_all_dates
        with pytest.raises(KeyError, match="'2011'"):
            df["2011"]

        with pytest.raises(KeyError, match="'2011'"):
            df.loc["2011", 0]

        df = DataFrame()
        assert not df.index._is_all_dates
        with pytest.raises(KeyError, match="'2011'"):
            df["2011"]

        with pytest.raises(KeyError, match="'2011'"):
            df.loc["2011", 0]

    def test_astype_assignment(self):

        # GH4312 (iloc)
        df_orig = DataFrame(
            [["1", "2", "3", ".4", 5, 6.0, "foo"]], columns=list("ABCDEFG")
        )

        df = df_orig.copy()
        df.iloc[:, 0:2] = df.iloc[:, 0:2].astype(np.int64)
        expected = DataFrame(
            [[1, 2, "3", ".4", 5, 6.0, "foo"]], columns=list("ABCDEFG")
        )
        tm.assert_frame_equal(df, expected)

        df = df_orig.copy()
        df.iloc[:, 0:2] = df.iloc[:, 0:2]._convert(datetime=True, numeric=True)
        expected = DataFrame(
            [[1, 2, "3", ".4", 5, 6.0, "foo"]], columns=list("ABCDEFG")
        )
        tm.assert_frame_equal(df, expected)

        # GH5702 (loc)
        df = df_orig.copy()
        df.loc[:, "A"] = df.loc[:, "A"].astype(np.int64)
        expected = DataFrame(
            [[1, "2", "3", ".4", 5, 6.0, "foo"]], columns=list("ABCDEFG")
        )
        tm.assert_frame_equal(df, expected)

        df = df_orig.copy()
        df.loc[:, ["B", "C"]] = df.loc[:, ["B", "C"]].astype(np.int64)
        expected = DataFrame(
            [["1", 2, 3, ".4", 5, 6.0, "foo"]], columns=list("ABCDEFG")
        )
        tm.assert_frame_equal(df, expected)

        # full replacements / no nans
        df = DataFrame({"A": [1.0, 2.0, 3.0, 4.0]})
        df.iloc[:, 0] = df["A"].astype(np.int64)
        expected = DataFrame({"A": [1, 2, 3, 4]})
        tm.assert_frame_equal(df, expected)

        df = DataFrame({"A": [1.0, 2.0, 3.0, 4.0]})
        df.loc[:, "A"] = df["A"].astype(np.int64)
        expected = DataFrame({"A": [1, 2, 3, 4]})
        tm.assert_frame_equal(df, expected)

    @pytest.mark.parametrize("indexer", [getitem, loc])
    def test_index_type_coercion(self, indexer):

        # GH 11836
        # if we have an index type and set it with something that looks
        # to numpy like the same, but is actually, not
        # (e.g. setting with a float or string '0')
        # then we need to coerce to object

        # integer indexes
        for s in [Series(range(5)), Series(range(5), index=range(1, 6))]:

            assert s.index.is_integer()

            s2 = s.copy()
            indexer(s2)[0.1] = 0
            assert s2.index.is_floating()
            assert indexer(s2)[0.1] == 0

            s2 = s.copy()
            indexer(s2)[0.0] = 0
            exp = s.index
            if 0 not in s:
                exp = Index(s.index.tolist() + [0])
            tm.assert_index_equal(s2.index, exp)

            s2 = s.copy()
            indexer(s2)["0"] = 0
            assert s2.index.is_object()

        for s in [Series(range(5), index=np.arange(5.0))]:

            assert s.index.is_floating()

            s2 = s.copy()
            indexer(s2)[0.1] = 0
            assert s2.index.is_floating()
            assert indexer(s2)[0.1] == 0

            s2 = s.copy()
            indexer(s2)[0.0] = 0
            tm.assert_index_equal(s2.index, s.index)

            s2 = s.copy()
            indexer(s2)["0"] = 0
            assert s2.index.is_object()

class TestMisc:
    def test_float_index_to_mixed(self):
        df = DataFrame({0.0: np.random.rand(10), 1.0: np.random.rand(10)})
        df["a"] = 10
        tm.assert_frame_equal(
            DataFrame({0.0: df[0.0], 1.0: df[1.0], "a": [10] * 10}), df
        )

    def test_float_index_non_scalar_assignment(self):
        df = DataFrame({"a": [1, 2, 3], "b": [3, 4, 5]}, index=[1.0, 2.0, 3.0])
        df.loc[df.index[:2]] = 1
        expected = DataFrame({"a": [1, 1, 3], "b": [1, 1, 5]}, index=df.index)
        tm.assert_frame_equal(expected, df)

        df = DataFrame({"a": [1, 2, 3], "b": [3, 4, 5]}, index=[1.0, 2.0, 3.0])
        df2 = df.copy()
        df.loc[df.index] = df.loc[df.index]
        tm.assert_frame_equal(df, df2)

    def test_float_index_at_iat(self):
        s = Series([1, 2, 3], index=[0.1, 0.2, 0.3])
        for el, item in s.items():
            assert s.at[el] == item
        for i in range(len(s)):
            assert s.iat[i] == i + 1

    def test_rhs_alignment(self):
        # GH8258, tests that both rows & columns are aligned to what is
        # assigned to. covers both uniform data-type & multi-type cases
        def run_tests(df, rhs, right_loc, right_iloc):
            # label, index, slice
            lbl_one, idx_one, slice_one = list("bcd"), [1, 2, 3], slice(1, 4)
            lbl_two, idx_two, slice_two = ["joe", "jolie"], [1, 2], slice(1, 3)

            left = df.copy()
            left.loc[lbl_one, lbl_two] = rhs
            tm.assert_frame_equal(left, right_loc)

            left = df.copy()
            left.iloc[idx_one, idx_two] = rhs
            tm.assert_frame_equal(left, right_iloc)

            left = df.copy()
            left.iloc[slice_one, slice_two] = rhs
            tm.assert_frame_equal(left, right_iloc)

        xs = np.arange(20).reshape(5, 4)
        cols = ["jim", "joe", "jolie", "joline"]
        df = DataFrame(xs, columns=cols, index=list("abcde"), dtype="int64")

        # right hand side; permute the indices and multiplpy by -2
        rhs = -2 * df.iloc[3:0:-1, 2:0:-1]

        # expected `right` result; just multiply by -2
        right_iloc = df.copy()
        right_iloc["joe"] = [1, 14, 10, 6, 17]
        right_iloc["jolie"] = [2, 13, 9, 5, 18]
        right_iloc.iloc[1:4, 1:3] *= -2
        right_loc = df.copy()
        right_loc.iloc[1:4, 1:3] *= -2

        # run tests with uniform dtypes
        run_tests(df, rhs, right_loc, right_iloc)

        # make frames multi-type & re-run tests
        for frame in [df, rhs, right_loc, right_iloc]:
            frame["joe"] = frame["joe"].astype("float64")
            frame["jolie"] = frame["jolie"].map("@{}".format)
        right_iloc["joe"] = [1.0, "@-28", "@-20", "@-12", 17.0]
        right_iloc["jolie"] = ["@2", -26.0, -18.0, -10.0, "@18"]
        run_tests(df, rhs, right_loc, right_iloc)

    def test_str_label_slicing_with_negative_step(self):
        SLC = pd.IndexSlice

        def assert_slices_equivalent(l_slc, i_slc):
            tm.assert_series_equal(s.loc[l_slc], s.iloc[i_slc])

            if not idx.is_integer:
                # For integer indices, .loc and plain getitem are position-based.
                tm.assert_series_equal(s[l_slc], s.iloc[i_slc])
                tm.assert_series_equal(s.loc[l_slc], s.iloc[i_slc])

        for idx in [_mklbl("A", 20), np.arange(20) + 100, np.linspace(100, 150, 20)]:
            idx = Index(idx)
            s = Series(np.arange(20), index=idx)
            assert_slices_equivalent(SLC[idx[9] :: -1], SLC[9::-1])
            assert_slices_equivalent(SLC[: idx[9] : -1], SLC[:8:-1])
            assert_slices_equivalent(SLC[idx[13] : idx[9] : -1], SLC[13:8:-1])
            assert_slices_equivalent(SLC[idx[9] : idx[13] : -1], SLC[:0])

    def test_slice_with_zero_step_raises(self):
        s = Series(np.arange(20), index=_mklbl("A", 20))
        with pytest.raises(ValueError, match="slice step cannot be zero"):
            s[::0]
        with pytest.raises(ValueError, match="slice step cannot be zero"):
            s.loc[::0]

    def test_indexing_assignment_dict_already_exists(self):
        df = DataFrame({"x": [1, 2, 6], "y": [2, 2, 8], "z": [-5, 0, 5]}).set_index("z")
        expected = df.copy()
        rhs = {"x": 9, "y": 99}
        df.loc[5] = rhs
        expected.loc[5] = [9, 99]
        tm.assert_frame_equal(df, expected)

    def test_indexing_dtypes_on_empty(self):
        # Check that .iloc returns correct dtypes GH9983
        df = DataFrame({"a": [1, 2, 3], "b": ["b", "b2", "b3"]})
        df2 = df.iloc[[], :]

        assert df2.loc[:, "a"].dtype == np.int64
        tm.assert_series_equal(df2.loc[:, "a"], df2.iloc[:, 0])

    @pytest.mark.parametrize("size", [5, 999999, 1000000])
    def test_range_in_series_indexing(self, size):
        # range can cause an indexing error
        # GH 11652
        s = Series(index=range(size), dtype=np.float64)
        s.loc[range(1)] = 42
        tm.assert_series_equal(s.loc[range(1)], Series(42.0, index=[0]))

        s.loc[range(2)] = 43
        tm.assert_series_equal(s.loc[range(2)], Series(43.0, index=[0, 1]))

    @pytest.mark.parametrize(
        "slc",
        [
            pd.IndexSlice[:, :],
            pd.IndexSlice[:, 1],
            pd.IndexSlice[1, :],
            pd.IndexSlice[[1], [1]],
            pd.IndexSlice[1, [1]],
            pd.IndexSlice[[1], 1],
            pd.IndexSlice[1],
            pd.IndexSlice[1, 1],
            slice(None, None, None),
            [0, 1],
            np.array([0, 1]),
            Series([0, 1]),
        ],
    )
    def test_non_reducing_slice(self, slc):
        df = DataFrame([[0, 1], [2, 3]])

        tslice_ = non_reducing_slice(slc)
        assert isinstance(df.loc[tslice_], DataFrame)

    def test_list_slice(self):
        # like dataframe getitem
        slices = [["A"], Series(["A"]), np.array(["A"])]
        df = DataFrame({"A": [1, 2], "B": [3, 4]}, index=["A", "B"])
        expected = pd.IndexSlice[:, ["A"]]
        for subset in slices:
            result = non_reducing_slice(subset)
            tm.assert_frame_equal(df.loc[result], df.loc[expected])

    def test_maybe_numeric_slice(self):
        df = DataFrame({"A": [1, 2], "B": ["c", "d"], "C": [True, False]})
        result = maybe_numeric_slice(df, slice_=None)
        expected = pd.IndexSlice[:, ["A"]]
        assert result == expected

        result = maybe_numeric_slice(df, None, include_bool=True)
        expected = pd.IndexSlice[:, ["A", "C"]]
        assert all(result[1] == expected[1])
        result = maybe_numeric_slice(df, [1])
        expected = [1]
        assert result == expected

    def test_partial_boolean_frame_indexing(self):
        # GH 17170
        df = DataFrame(
            np.arange(9.0).reshape(3, 3), index=list("abc"), columns=list("ABC")
        )
        index_df = DataFrame(1, index=list("ab"), columns=list("AB"))
        result = df[index_df.notnull()]
        expected = DataFrame(
            np.array([[0.0, 1.0, np.nan], [3.0, 4.0, np.nan], [np.nan] * 3]),
            index=list("abc"),
            columns=list("ABC"),
        )
        tm.assert_frame_equal(result, expected)

    def test_no_reference_cycle(self):
        df = DataFrame({"a": [0, 1], "b": [2, 3]})
        for name in ("loc", "iloc", "at", "iat"):
            getattr(df, name)
        wr = weakref.ref(df)
        del df
        assert wr() is None

    def test_label_indexing_on_nan(self):
        # GH 32431
        df = Series([1, "{1,2}", 1, None])
        vc = df.value_counts(dropna=False)
        result1 = vc.loc[np.nan]
        result2 = vc[np.nan]

        expected = 1
        assert result1 == expected
        assert result2 == expected

class TestSeriesNoneCoercion:
    EXPECTED_RESULTS = [
        # For numeric series, we should coerce to NaN.
        ([1, 2, 3], [np.nan, 2, 3]),
        ([1.0, 2.0, 3.0], [np.nan, 2.0, 3.0]),
        # For datetime series, we should coerce to NaT.
        (
            [datetime(2000, 1, 1), datetime(2000, 1, 2), datetime(2000, 1, 3)],
            [NaT, datetime(2000, 1, 2), datetime(2000, 1, 3)],
        ),
        # For objects, we should preserve the None value.
        (["foo", "bar", "baz"], [None, "bar", "baz"]),
    ]

    @pytest.mark.parametrize("start_data,expected_result", EXPECTED_RESULTS)
    def test_coercion_with_setitem(self, start_data, expected_result):
        start_series = Series(start_data)
        start_series[0] = None

        expected_series = Series(expected_result)
        tm.assert_series_equal(start_series, expected_series)

    @pytest.mark.parametrize("start_data,expected_result", EXPECTED_RESULTS)
    def test_coercion_with_loc_setitem(self, start_data, expected_result):
        start_series = Series(start_data)
        start_series.loc[0] = None

        expected_series = Series(expected_result)
        tm.assert_series_equal(start_series, expected_series)

    @pytest.mark.parametrize("start_data,expected_result", EXPECTED_RESULTS)
    def test_coercion_with_setitem_and_series(self, start_data, expected_result):
        start_series = Series(start_data)
        start_series[start_series == start_series[0]] = None

        expected_series = Series(expected_result)
        tm.assert_series_equal(start_series, expected_series)

    @pytest.mark.parametrize("start_data,expected_result", EXPECTED_RESULTS)
    def test_coercion_with_loc_and_series(self, start_data, expected_result):
        start_series = Series(start_data)
        start_series.loc[start_series == start_series[0]] = None

        expected_series = Series(expected_result)
        tm.assert_series_equal(start_series, expected_series)

class TestDataframeNoneCoercion:
    EXPECTED_SINGLE_ROW_RESULTS = [
        # For numeric series, we should coerce to NaN.
        ([1, 2, 3], [np.nan, 2, 3]),
        ([1.0, 2.0, 3.0], [np.nan, 2.0, 3.0]),
        # For datetime series, we should coerce to NaT.
        (
            [datetime(2000, 1, 1), datetime(2000, 1, 2), datetime(2000, 1, 3)],
            [NaT, datetime(2000, 1, 2), datetime(2000, 1, 3)],
        ),
        # For objects, we should preserve the None value.
        (["foo", "bar", "baz"], [None, "bar", "baz"]),
    ]

    @pytest.mark.parametrize("expected", EXPECTED_SINGLE_ROW_RESULTS)
    def test_coercion_with_loc(self, expected):
        start_data, expected_result = expected

        start_dataframe = DataFrame({"foo": start_data})
        start_dataframe.loc[0, ["foo"]] = None

        expected_dataframe = DataFrame({"foo": expected_result})
        tm.assert_frame_equal(start_dataframe, expected_dataframe)

    @pytest.mark.parametrize("expected", EXPECTED_SINGLE_ROW_RESULTS)
    def test_coercion_with_setitem_and_dataframe(self, expected):
        start_data, expected_result = expected

        start_dataframe = DataFrame({"foo": start_data})
        start_dataframe[start_dataframe["foo"] == start_dataframe["foo"][0]] = None

        expected_dataframe = DataFrame({"foo": expected_result})
        tm.assert_frame_equal(start_dataframe, expected_dataframe)

    @pytest.mark.parametrize("expected", EXPECTED_SINGLE_ROW_RESULTS)
    def test_none_coercion_loc_and_dataframe(self, expected):
        start_data, expected_result = expected

        start_dataframe = DataFrame({"foo": start_data})
        start_dataframe.loc[start_dataframe["foo"] == start_dataframe["foo"][0]] = None

        expected_dataframe = DataFrame({"foo": expected_result})
        tm.assert_frame_equal(start_dataframe, expected_dataframe)

    def test_none_coercion_mixed_dtypes(self):
        start_dataframe = DataFrame(
            {
                "a": [1, 2, 3],
                "b": [1.0, 2.0, 3.0],
                "c": [datetime(2000, 1, 1), datetime(2000, 1, 2), datetime(2000, 1, 3)],
                "d": ["a", "b", "c"],
            }
        )
        start_dataframe.iloc[0] = None

        exp = DataFrame(
            {
                "a": [np.nan, 2, 3],
                "b": [np.nan, 2.0, 3.0],
                "c": [NaT, datetime(2000, 1, 2), datetime(2000, 1, 3)],
                "d": [None, "b", "c"],
            }
        )
        tm.assert_frame_equal(start_dataframe, exp)

def test_extension_array_cross_section():
    # A cross-section of a homogeneous EA should be an EA
    df = DataFrame(
        {
            "A": pd.core.arrays.integer_array([1, 2]),
            "B": pd.core.arrays.integer_array([3, 4]),
        },
        index=["a", "b"],
    )
    expected = Series(pd.core.arrays.integer_array([1, 3]), index=["A", "B"], name="a")
    result = df.loc["a"]
    tm.assert_series_equal(result, expected)

    result = df.iloc[0]
    tm.assert_series_equal(result, expected)

def test_extension_array_cross_section_converts():
    # all numeric columns -> numeric series
    df = DataFrame(
        {"A": pd.array([1, 2], dtype="Int64"), "B": np.array([1, 2])}, index=["a", "b"]
    )
    result = df.loc["a"]
    expected = Series([1, 1], dtype="Int64", index=["A", "B"], name="a")
    tm.assert_series_equal(result, expected)

    result = df.iloc[0]
    tm.assert_series_equal(result, expected)

    # mixed columns -> object series
    df = DataFrame(
        {"A": pd.array([1, 2], dtype="Int64"), "B": np.array(["a", "b"])},
        index=["a", "b"],
    )
    result = df.loc["a"]
    expected = Series([1, "a"], dtype=object, index=["A", "B"], name="a")
    tm.assert_series_equal(result, expected)

    result = df.iloc[0]
    tm.assert_series_equal(result, expected)

def test_setitem_with_bool_mask_and_values_matching_n_trues_in_length():
    # GH 30567
    ser = Series([None] * 10)
    mask = [False] * 3 + [True] * 5 + [False] * 2
    ser[mask] = range(5)
    result = ser
    expected = Series([None] * 3 + list(range(5)) + [None] * 2).astype("object")
    tm.assert_series_equal(result, expected)

def test_missing_labels_inside_loc_matched_in_error_message():
    # GH34272
    s = Series({"a": 1, "b": 2, "c": 3})
    error_message_regex = "missing_0.*missing_1.*missing_2"
    with pytest.raises(KeyError, match=error_message_regex):
        s.loc[["a", "b", "missing_0", "c", "missing_1", "missing_2"]]

def test_many_missing_labels_inside_loc_error_message_limited():
    # GH34272
    n = 10000
    missing_labels = [f"missing_{label}" for label in range(n)]
    s = Series({"a": 1, "b": 2, "c": 3})
    # regex checks labels between 4 and 9995 are replaced with ellipses
    error_message_regex = "missing_4.*\\.\\.\\..*missing_9995"
    with pytest.raises(KeyError, match=error_message_regex):
        s.loc[["a", "c"] + missing_labels]

def test_long_text_missing_labels_inside_loc_error_message_limited():
    # GH34272
    s = Series({"a": 1, "b": 2, "c": 3})
    missing_labels = [f"long_missing_label_text_{i}" * 5 for i in range(3)]
    # regex checks for very long labels there are new lines between each
    error_message_regex = "long_missing_label_text_0.*\\\\n.*long_missing_label_text_1"
    with pytest.raises(KeyError, match=error_message_regex):
        s.loc[["a", "c"] + missing_labels]

def test_setitem_categorical():
    # https://github.com/pandas-dev/pandas/issues/35369
    df = DataFrame({"h": Series(list("mn")).astype("category")})
    df.h = df.h.cat.reorder_categories(["n", "m"])
    expected = DataFrame(
        {"h": pd.Categorical(["m", "n"]).reorder_categories(["n", "m"])}
    )
    tm.assert_frame_equal(df, expected)


from rest_framework.response import Response
from rest_framework.pagination import PageNumberPagination

class ReactAdminPagination(PageNumberPagination):
    page_query_param = "page"
    page_size_query_param = "perPage"

    def get_paginated_response(self, data):
        count = self.page.paginator.count
        item_starting_index = self.page.start_index() - 1
        item_ending_index = self.page.end_index() - 1

        content_range = "items {0}-{1}/{2}".format(
            item_starting_index, item_ending_index, count
        )
        return Response(
            data,
            headers={
                "Content-Range": content_range,
                "X-Total-Count": count,
                "Access-Control-Expose-Headers": "X-Total-Count",
            },
        )


from typing import TYPE_CHECKING, Dict, Iterable, Iterator, Mapping, Sequence, Union

from pip._vendor.resolvelib.providers import AbstractProvider

from .base import Candidate, Constraint, Requirement
from .factory import Factory

if TYPE_CHECKING:
    from pip._vendor.resolvelib.providers import Preference
    from pip._vendor.resolvelib.resolvers import RequirementInformation

    PreferenceInformation = RequirementInformation[Requirement, Candidate]

    _ProviderBase = AbstractProvider[Requirement, Candidate, str]
else:
    _ProviderBase = AbstractProvider

# Notes on the relationship between the provider, the factory, and the
# candidate and requirement classes.
#
# The provider is a direct implementation of the resolvelib class. Its role
# is to deliver the API that resolvelib expects.
#
# Rather than work with completely abstract "requirement" and "candidate"
# concepts as resolvelib does, pip has concrete classes implementing these two
# ideas. The API of Requirement and Candidate objects are defined in the base
# classes, but essentially map fairly directly to the equivalent provider
# methods. In particular, `find_matches` and `is_satisfied_by` are
# requirement methods, and `get_dependencies` is a candidate method.
#
# The factory is the interface to pip's internal mechanisms. It is stateless,
# and is created by the resolver and held as a property of the provider. It is
# responsible for creating Requirement and Candidate objects, and provides
# services to those objects (access to pip's finder and preparer).

class PipProvider(_ProviderBase):
    """Pip's provider implementation for resolvelib.

    :params constraints: A mapping of constraints specified by the user. Keys
        are canonicalized project names.
    :params ignore_dependencies: Whether the user specified ``--no-deps``.
    :params upgrade_strategy: The user-specified upgrade strategy.
    :params user_requested: A set of canonicalized package names that the user
        supplied for pip to install/upgrade.
    """

    def __init__(
        self,
        factory,  # type: Factory
        constraints,  # type: Dict[str, Constraint]
        ignore_dependencies,  # type: bool
        upgrade_strategy,  # type: str
        user_requested,  # type: Dict[str, int]
    ):
        # type: (...) -> None
        self._factory = factory
        self._constraints = constraints
        self._ignore_dependencies = ignore_dependencies
        self._upgrade_strategy = upgrade_strategy
        self._user_requested = user_requested

    def identify(self, requirement_or_candidate):
        # type: (Union[Requirement, Candidate]) -> str
        return requirement_or_candidate.name

    def get_preference(
        self,
        identifier: str,
        resolutions: Mapping[str, Candidate],
        candidates: Mapping[str, Iterator[Candidate]],
        information: Mapping[str, Iterator["PreferenceInformation"]],
    ) -> "Preference":
        """Produce a sort key for given requirement based on preference.

        The lower the return value is, the more preferred this group of
        arguments is.

        Currently pip considers the followings in order:

        * Prefer if any of the known requirements points to an explicit URL.
        * If equal, prefer if any requirements contain ``===`` and ``==``.
        * If equal, prefer if requirements include version constraints, e.g.
          ``>=`` and ``<``.
        * If equal, prefer user-specified (non-transitive) requirements, and
          order user-specified requirements by the order they are specified.
        * If equal, order alphabetically for consistency (helps debuggability).
        """

        def _get_restrictive_rating(requirements):
            # type: (Iterable[Requirement]) -> int
            """Rate how restrictive a set of requirements are.

            ``Requirement.get_candidate_lookup()`` returns a 2-tuple for
            lookup. The first element is ``Optional[Candidate]`` and the
            second ``Optional[InstallRequirement]``.

            * If the requirement is an explicit one, the explicitly-required
              candidate is returned as the first element.
            * If the requirement is based on a PEP 508 specifier, the backing
              ``InstallRequirement`` is returned as the second element.

            We use the first element to check whether there is an explicit
            requirement, and the second for equality operator.
            """
            lookups = (r.get_candidate_lookup() for r in requirements)
            cands, ireqs = zip(*lookups)
            if any(cand is not None for cand in cands):
                return 0
            spec_sets = (ireq.specifier for ireq in ireqs if ireq)
            operators = [
                specifier.operator for spec_set in spec_sets for specifier in spec_set
            ]
            if any(op in ("==", "===") for op in operators):
                return 1
            if operators:
                return 2
            # A "bare" requirement without any version requirements.
            return 3

        rating = _get_restrictive_rating(r for r, _ in information[identifier])
        order = self._user_requested.get(identifier, float("inf"))

        # HACK: Setuptools have a very long and solid backward compatibility
        # track record, and extremely few projects would request a narrow,
        # non-recent version range of it since that would break a lot things.
        # (Most projects specify it only to request for an installer feature,
        # which does not work, but that's another topic.) Intentionally
        # delaying Setuptools helps reduce branches the resolver has to check.
        # This serves as a temporary fix for issues like "apache-airlfow[all]"
        # while we work on "proper" branch pruning techniques.
        delay_this = identifier == "setuptools"

        return (delay_this, rating, order, identifier)

    def find_matches(
        self,
        identifier: str,
        requirements: Mapping[str, Iterator[Requirement]],
        incompatibilities: Mapping[str, Iterator[Candidate]],
    ) -> Iterable[Candidate]:
        def _eligible_for_upgrade(name):
            # type: (str) -> bool
            """Are upgrades allowed for this project?

            This checks the upgrade strategy, and whether the project was one
            that the user specified in the command line, in order to decide
            whether we should upgrade if there's a newer version available.

            (Note that we don't need access to the `--upgrade` flag, because
            an upgrade strategy of "to-satisfy-only" means that `--upgrade`
            was not specified).
            """
            if self._upgrade_strategy == "eager":
                return True
            elif self._upgrade_strategy == "only-if-needed":
                return name in self._user_requested
            return False

        return self._factory.find_candidates(
            identifier=identifier,
            requirements=requirements,
            constraint=self._constraints.get(identifier, Constraint.empty()),
            prefers_installed=(not _eligible_for_upgrade(identifier)),
            incompatibilities=incompatibilities,
        )

    def is_satisfied_by(self, requirement, candidate):
        # type: (Requirement, Candidate) -> bool
        return requirement.is_satisfied_by(candidate)

    def get_dependencies(self, candidate):
        # type: (Candidate) -> Sequence[Requirement]
        with_requires = not self._ignore_dependencies
        return [r for r in candidate.iter_dependencies(with_requires) if r is not None]


import math
from typing import Tuple

from model_space.utils.point import Point

def get_correct_prediction(predictions: Tuple[Point, Point],
                           last_position: Point,
                           speed: float):
    diff0 = abs(math.dist(predictions[0].as_list(), last_position.as_list()) - speed)
    diff1 = abs(math.dist(predictions[1].as_list(), last_position.as_list()) - speed)

    if diff0 < diff1:
        return predictions[0]
    else:
        return predictions[1]


from os import environ
import json

from config import es as default_es

def create_index(es_index='memex', mapping=environ['DDT_HOME']+'/elastic/mapping.json', es=None):
    if es is None:
        es = default_es

    json_page_data=open(mapping).read()

    page_mappings = json.loads(json_page_data)

    doctypes = {}
    for doc_type in page_mappings.keys():
        doctypes[doc_type] = page_mappings[doc_type]

    mappings = {
        "mappings": doctypes
    }

    fields = es_index.lower().split(' ')
    es_index = '_'.join([item for item in fields if item not in ''])

    res = es.indices.create(index=es_index, body=mappings, ignore=400)

    es.indices.refresh(es_index)

    return res

def create_terms_index(es_index='ddt_terms', es=None):
    if es is None:
        es = default_es

    json_terms_data=open(environ['DDT_HOME']+'/elastic/mapping_terms.json').read()

    terms_mappings = json.loads(json_terms_data)

    mappings = {"mappings":
                {
                    "terms":terms_mappings["terms"]
                }
            }

    fields = es_index.lower().split(' ')
    es_index = '_'.join([item for item in fields if item not in ''])

    res = es.indices.create(index=es_index, body=mappings, ignore=400)

    es.indices.refresh(es_index)

    return res

def create_config_index(es_index='config', es=None):
    if es is None:
        es = default_es

    json_config_data=open(environ['DDT_HOME']+'/elastic/config.json').read()

    config_mappings = json.loads(json_config_data)

    mappings = {"mappings":
                {
                    "domains": config_mappings["domains"]
                }
            }

    fields = es_index.lower().split(' ')

    es_index = '_'.join([item for item in fields if item not in ''])

    res = es.indices.create(index=es_index, body=mappings, ignore=400)

    return res


# -*- coding: utf-8 -*-
"""
Created on Thu Apr 15 16:00:07 2021

@author: petil
"""

import math
from collections import Counter
import pandas as pd
import copy
import time as t
import random as rd

#Variable
pd.set_option('min_rows', 20)
pathApprentissage = 'data.csv';
pathTest = 'preTest.csv';
pathExecutable = 'finalTest.csv';
pathWrite = 'result.txt'
compteurVarConnu = 0;
afficherMatriceConfusion = True;
kMax = 10;
ratioValApprentissage = 0.75;

#Return le dataframe du csv fournit dans le path
def importDataWithDataFrame(path):
    df=pd.read_csv(path)
    if(df.axes[1][0] != '0'):
        df = pd.read_csv(path,header=None)
        df.columns = df.columns.map(str)

    df["prediction"] = "None";
    return df

def choixValApprentissage(data):
    global compteurVarConnu
    newdata = copy.deepcopy(data);
    newdata["prediction"] = "None";
    for i in newdata.index:
        if(rd.random() < ratioValApprentissage):
            newdata.loc[i,"prediction"] = newdata.loc[i,str(len(newdata.columns) - 2)]
            compteurVarConnu = compteurVarConnu + 1
    return newdata;

def calculEuclidien(i,j,data):
    res = 0;
    for k in range(0,(len(data.columns) - 2)):
        res = res + (data[str(k)][i] - data[str(k)][j])**2

    return [math.sqrt(res),data['prediction'][j]];

def knn(data,i):
    distancePointTest = []
    for j in data.loc[data['prediction'] != "None"].index:
            distancePointTest.append(calculEuclidien(i,j,data));

    #on trie
    distancePointTest = sorted(distancePointTest);

    return distancePointTest;

def remplissagePoint(data,k):
    newdata = copy.deepcopy(data);
    #pour tout les points qui n'ont pas de prédiction
    for i in newdata.loc[newdata['prediction'] == "None"].index:
            #On calcul distance euclidienne
            knnTab = knn(data,i);
            #On short au k choisi
            FinalChoixTab = [];
            for j in range(0,k+1):
                FinalChoixTab.append(knnTab[j][1]);
            #On determine la classe
            FinalChoixTab = Counter(FinalChoixTab);
            #on l'ajoute a la tab data
            newdata.loc[i, 'prediction'] = list(FinalChoixTab.most_common(1)[0])[0];
    return newdata;

def MatriceConfusion(data):
    #on affiche le header
    y_actu = pd.Series(data[str(len(data.columns) - 2)], name='Actual')
    y_pred = pd.Series(data['prediction'], name='Predicted')
    df_confusion = pd.crosstab(y_actu, y_pred)
    print(df_confusion)

def Prediction(data):
    global compteurVarConnu;
    nombrelementIncorrect = 0;
    #On fait les calcul
    for i in data.index:
        if not (data.loc[i, 'prediction'] == data.loc[i, str(len(data.columns) - 2)]):
            nombrelementIncorrect = nombrelementIncorrect + 1;
    print('nombrelementIncorrect : ',nombrelementIncorrect , "compteurVarConnu",compteurVarConnu);

    #On affiche
    if(afficherMatriceConfusion):
        MatriceConfusion(data)

    #On renvoie le pourcentage de réussite.
    print("Ratio :" , 100 - ((nombrelementIncorrect * 100)/(len(data) - compteurVarConnu)))
    return 100 - ((nombrelementIncorrect * 100)/(len(data) - compteurVarConnu));

def Bestk(data):
    #On Cherche le meillieur k
    tabBestK = []
    for k in range(1,kMax + 1):
        print("Debut Analyse du k = ",k ," : OK")
        #On rempli notre tableau avec les prévisions de l'algo
        newdata = copy.deepcopy(data)
        newdata = remplissagePoint(newdata,k);

        #prediction & matrice
        print("Debut Calcul ratioPrediction k = ",k ," : OK")
        ratioPrediction = Prediction(copy.deepcopy(newdata));
        tabBestK.append([k,ratioPrediction,newdata])
        print("Fin Analyse du k = ",k ," : OK")

    #on le choisi
    bestkval = tabBestK[0];
    for i in tabBestK[1:]:
        if(bestkval[1] < i[1]):
            bestkval = i;

    return bestkval;

def checklabel():
    allLabels = ['classA','classB','classC','classD','classE']
    #ce fichier s'attend à lire 3000 prédictions, une par ligne
    #réduisez nbLines en période de test.
    nbLines = 3000
    fd =open("result.txt",'r')
    lines = fd.readlines()

    count=0
    for label in lines:
    	if label.strip() in allLabels:
    		count+=1
    	else:
    		if count<nbLines:
    			print("Wrong label line:"+str(count+1))
    			break
    if count<nbLines:
    	print("Labels Check : fail!")
    else:
    	print("Labels Check : Successfull!")

def predictionFinal(bestApprentissage):
    newdata = importDataWithDataFrame(pathExecutable);
    newdata2 = bestApprentissage[2];
    newdata2 = newdata2.append(newdata, ignore_index=True);
    fichier = open(pathWrite, "w")
    for i in newdata.index:
        print(i)
        #On calcul distance euclidienne
        knnTab = knn(newdata2,i);
        #On short au k choisi
        FinalChoixTab = [];
        for j in range(0,bestApprentissage[0] + 1):
                FinalChoixTab.append(knnTab[j][1]);
        #On determine la classe
        FinalChoixTab = Counter(FinalChoixTab);
        print(Counter(FinalChoixTab));
        #on l'ajoute a la tab data
        newdata2.loc[i, 'prediction'] = list(FinalChoixTab.most_common(1)[0])[0];
        fichier.write(list(FinalChoixTab.most_common(1)[0])[0] + "\n")

    fichier.close()

    checklabel()

if __name__ == "__main__":
    #METHODE DATAFRAME
    début=t.process_time()
    print("Lancement de l'IA : OK ... ");
    #on recuperere les data , on remplie les colums d'apprentissage et on trouve le meilleur k
    data = choixValApprentissage(importDataWithDataFrame(pathApprentissage).append(importDataWithDataFrame(pathTest), ignore_index=True))

    bestApprentissage = Bestk(data)
    print("Valeur de k optimal trouvé : OK ... ");

    #on remet les bonne valeur de preTest pour enlever les erreurs
    bestApprentissage[2]['prediction'] = data[str(len(data.columns) - 2)]

    #on prédis les valeurs sans outpout
    predictionFinal(bestApprentissage)

    fin=t.process_time()
    temps=fin-début
    print("temps d'execution du programme : ",temps, "s")

    print("Fin du programme");


"""Module to import data from the victor controller."""
import datetime
import numpy as np

from sfg2d.utils.config import CONFIG

PIXEL = CONFIG['PIXEL']
SPECS = CONFIG['SPECS']

def get_from_victor_controller(fpath, sort_pp_times=True, **kwargs):
    """Import data from victor controller

    sort_pp_times: This boolean allows to have scrambeled (radomized ordered)
    pp_delays data. Upon import the data is reshaped is such a way that
    pp_delays are in order. even though the actual measurment was not performed
    in this order. kwargs are passed to np.genfromtxt

    """
    try:
        raw_data = np.genfromtxt(fpath, dtype='long', **kwargs)[:, 1:]
    except ValueError:
        raise IOError("Scan was interrupted. Plz give usecols kwarg to genfromtxt.")

    with open(fpath) as file:
        for line in file:
            if '# Timedelay=' in line:
                pp_delays = np.array(line[12:-1].split('\t'), dtype=int)
                break
            if line[0] is not '#':
                break

    num_rows, num_columns = raw_data.shape

    # File is just a simple scan.
    if num_rows == PIXEL and num_columns == 3:
        return np.array([[raw_data.T]]), pp_delays

    # Check that we can read the data shape
    if (num_columns)%SPECS != 0:
        raise IOError("Cant read data in %s" % fpath)

    num_pp_delays = num_rows//PIXEL

    # The first colum is only pixel number
    num_repetitions = num_columns//SPECS

    # Init container for the result.
    ret = np.zeros((num_pp_delays, num_repetitions, SPECS, PIXEL))

    for rep_index in range(num_repetitions):
        for pp_delay_index in range(num_pp_delays):
            column_slice = slice(PIXEL*pp_delay_index, PIXEL*pp_delay_index + PIXEL)
            row_slice = slice(rep_index*SPECS, rep_index*SPECS+SPECS)
            ret[pp_delay_index, rep_index] = raw_data[column_slice, row_slice].T

    # Sorts data by pp_delays
    if sort_pp_times:
        sorting_ideces = np.argsort(pp_delays)
        pp_delays = pp_delays[sorting_ideces]
        ret = ret[sorting_ideces]

    return ret, pp_delays

def read_header(fpath):
    """Read informaion from fileheader and return as dictionary."""
    ret = {}
    with open(fpath) as f:
        for line in f:
            if line[0] is not "#":
                break
            # Strip comment marker
            line = line[2:]
            name, value = line.split("=")
            # Strip newline
            ret[name] = value[:-1]

    # To have some compatibility between spe veronica and viktor files,
    # we further unify some of the namings
    ret['gain'] = ret.get('Gain')

    exp_time = ret.get('ExposureTime [s]')
    if exp_time:
        ret['exposure_time'] = datetime.timedelta(seconds=float(exp_time))

    hbin = ret.get('HBin')
    if hbin:
        ret['hbin'] = {'ON': True}.get(value, False)

    cw = ret.get('Central-Wavelength')
    if cw:
        ret['central_wl'] = float(cw)

    vis_wl = ret.get('vis-Wavelength')
    if vis_wl:
        ret['vis_wl'] = float(vis_wl)

    syringe_pos = ret.get('Syringe Pos')
    if syringe_pos:
        ret['syringe_pos'] = int(syringe_pos)

    cursor = ret.get("Cursor")
    if cursor:
        ret['cursor'] = tuple([int(elm) for elm in cursor.split('\t')])

    x_mirror = ret.get('x-mirror')
    if x_mirror:
        ret['x_mirror'] = {'ON': True}.get(x_mirror, False)

    calib_coeff = ret.get('calib Coeff')
    if calib_coeff:
        ret['calib Coeff'] = tuple([float(elm) for elm in calib_coeff.split('\t')])

    scan_start_time = ret.get('Scan Start time')
    if scan_start_time:
        ret['date'] = datetime.datetime.strptime(scan_start_time, '%d.%m.%Y  %H:%M:%S')

    scan_stop_time = ret.get('Scan Stop time')
    if scan_stop_time:
        ret['date_stop'] = datetime.datetime.strptime(scan_stop_time, '%d.%m.%Y  %H:%M:%S')

    timedelay = ret.get('Timedelay')
    if timedelay:
        ret['timedelay'] = np.array([int(elm) for elm in timedelay.split('\t')])

    timedelay_pos= ret.get('Timedelay Pos')
    if timedelay_pos:
        ret['timedel_pos'] = np.array([int(elm) for elm in timedelay_pos.split('\t')])

    return ret


import cv2
import math
from cv2 import imread, Sobel, CV_64F
import numpy as np
from skin import detect_skin
def analyze( im, debug ):

    im_gray = cv2.cvtColor(im, cv2.COLOR_BGR2GRAY)

    ksize = 5

    sobel_x = np.abs(Sobel(im_gray, CV_64F, 1, 0, ksize=ksize))
    sobel_y = np.abs(Sobel(im_gray, CV_64F, 0, 1, ksize=ksize))

    sobel_x *= 255.0 / sobel_x.max()
    sobel_y *= 255.0 / sobel_y.max()

    skin = detect_skin(im)

    vert_amount = sobel_x.mean()
    horz_amount = sobel_y.mean()

    cheeringness = np.log(vert_amount / horz_amount)

    density = 1.0 * skin.mean() / 255.0

    if math.isnan(density):
        density = 0.0

    if debug:
        return (cheeringness, density, sobel_x, sobel_y, skin)
    else:
        return (cheeringness, density)


# Installing this requires C++ compilation. If you don't want to install that capability on your computer, you can install it by downloading the wheel from https://www.lfd.uci.edu/~gohlke/pythonlibs/#noise
# Then running pip install <wheel-location>
from lib.general import interpolate_looped, vector_from_angle
from lib.presetObject import PresetObject
from noise import pnoise1
import matplotlib.pyplot as plt
import numpy as np
import scipy.stats as st

# TODO: write some custom functions that work as settings for the wind
# One can be a constant magnitude
# One can be the OpenRocket method
# One can be the method that I made up

class Wind(PresetObject):
    """
    A wind simulator based on perlin noise and visual matching with second by second data from Salt Lake City
    Completely independent of time increment, but does vary over time and altitude
    Adjusts average wind speeds for altitude, and varies direction over time
    Also simulates gusts using a Weibull distribution
    """

    def __init__(self, **kwargs):
        """
        :param float average_wind_speed: At the supplied base altitude, what is the wind speed in m/s
        """

        # https://www.homerenergy.com/products/pro/docs/latest/weibull_k_value.html
        # 3 or 4 will be relatively steady, 1.5 will be gusty
        # This parameter is similar to the standard deviation / amplitude
        self.weibull_shape = 1
        self.interpolation_speed = 0.00005
        self.count = 30
        self.seeds = np.random.random(self.count) * self.count * 100

        # I have no idea what this does, but it is in the code for noise and it is an important parameter
        self.base = 0.9
        self.octaves = 50

        self.base_altitude = 10
        self.average_wind_speed = 5
        self.roughness = 1

        self.overwrite_defaults(**kwargs)

        self.altitude_randomized = 0
        # This value is the number of meters of altitude change that that is equivalent to one second passing
        # idk; this value has virtually no effect on our time scale
        self.time_from_altitude = 1000 / 10
        self.time_last_randomized = 0
        self.time_to_interpolate = self.get_random_interpolation_time()
        self.start_direction = 0
        self.wind_direction = self.start_direction
        self.target_direction = self.get_random_direction()

        # Have to manually calculate normalized perlin noise deviation
        self.std = self.determine_std()

    # region Statistics Stuff

    def get_normal_perlin(self, time):
        total = 0
        for i in range(self.count):
            total += pnoise1(self.seeds[i] + time, self.octaves, self.base)

        return total / self.count

    def get_z_score(self, mean, std, val):
        return (val - mean) / std

    def get_percentile_from_z(self, z):
        # Get the cumulative distribution function for a normal distribution based on a z-value
        return st.norm.cdf(z)

    def lookup_weibull(self, p, average):
        "Given the percentile, return the multiplier based on the weibull shape parameter and the average"
        # use self.weibull_shape
        # Based on https://www.itl.nist.gov/div898/handbook/eda/section3/eda3668.htm, I found the output given a percentile (after flipping x and y around)

        return (-np.log(1 - p)) ** (1 / self.weibull_shape) * average

    def determine_std(self):
        size = 10000

        ys = [[] for _ in range(self.count)]

        for index in range(len(ys)):
            for i in range(size):
                # Actually, I don't think time increment even needs to be included in tis
                ys[index].append(pnoise1(
                    self.seeds[index] + i * self.interpolation_speed, self.octaves))

        average_ys = []
        for i in range(len(ys[0])):
            total = 0
            for j in range(len(ys)):
                total += ys[j][i]

            average_ys.append(total / len(ys))

        return np.std(average_ys)

    # endregion

    def get_average_speed_altitude(
            self, altitude):
        if altitude == 0:
            print("Because this assumes the no slip condition, it does not work at altitude=0. Returning zero for wind speed")
            return 0
        # Roughness is z_0, or roughness length
        # For lake jackson, z_0 = 1
        # Relatively good explanation: https://wind-data.ch/tools/profile.php?lng=en
        # Review this equation more closely, it does not appear to be giving high enough increases
        return self.average_wind_speed * np.log(altitude / self.roughness) / np.log(
            self.base_altitude / self.roughness)

    def get_air_velocity(self, time, altitude=10):
        """
        Gets the air speed in all three dimensions at a point in time
        :param float base_altitude: The altitude AGL in meters where the average wind speed was recorded
        """

        wind_unit_vector = vector_from_angle(
            self.get_air_direction(time, altitude))

        value = self.get_normal_perlin(time * self.interpolation_speed)
        z_score = self.get_z_score(0, self.std, value)

        p = self.get_percentile_from_z(z_score)

        average_speed = self.get_average_speed_altitude(altitude)

        speed = self.lookup_weibull(p, average_speed)

        # TODO: Add some variability to the wind direction, particularly as altitude changes
        # In meters per second
        return wind_unit_vector * speed

    def get_random_interpolation_time(self):
        return np.random.normal(1800, 300)

    def get_random_direction(self):
        return np.random.uniform() * 2 * np.pi

    def calculate_average_direction(self, time, altitude):
        adjusted_time = time + (
            altitude - self.altitude_randomized) / self.time_from_altitude

        if adjusted_time > self.time_last_randomized + self.time_to_interpolate:
            self.target_direction = (
                self.wind_direction + (self.get_random_direction() - np.pi) * 0.5) % (2 * np.pi)
            if self.target_direction > 2 * np.pi:
                self.target_direction -= np.pi

            self.start_direction = self.wind_direction
            self.time_last_randomized = time
            self.altitude_randomized = altitude
            self.time_to_interpolate = self.get_random_interpolation_time()

        # This isn't exactly a linear interpolation since I am setting the start point each time. I think it is more of an ease-out interpolation; we will see how it looks
        self.wind_direction = interpolate_looped(
            adjusted_time, self.time_last_randomized, self.
            time_last_randomized + self.time_to_interpolate, self.
            start_direction, self.target_direction)

    def get_air_direction(self, time, altitude=10):
        # I am including altitude as an input, because as your altitude changes the wind direction will also change

        # The average direction is just a line moving from one randomly generated point on a circle to another randomly generated point on a circle. However, the direction has an additional perlin noise added to it, which makes it match the wobbliness of the real thing (visual inspection seems to say that direction variability is relatively independent of strength)

        # Maybe need some other variables for direction
        value = self.get_normal_perlin(time * self.interpolation_speed)
        z_score = self.get_z_score(0, self.std, value)

        noise_component = z_score * 1

        self.calculate_average_direction(time, altitude)

        return np.array([(self.wind_direction + noise_component) %
                         (2 * np.pi) - np.pi, np.pi / 2])

if __name__ == "__main__":
    w = Wind()

    print(w.get_average_speed_altitude(150))


import os
import openpyxl # For Excel manipulation
import boto3
import traceback
from shutil import copyfile
from PIL import Image, ImageOps
from preview_generator.manager import PreviewManager

# {
#   "dest_bucket": "lambda-generate-preview-image-destination",
#   "dest_path": "pig_#{width}_#{height}.png",
#   "error_sns": "arn:aws:sns:us-west-2:020275857710:file-previewer-failed-dev-neil",
#   "format": "png",
#   "ok_sns": "arn:aws:sns:us-west-2:020275857710:file-previewer-succeeded-dev-neil",
#   "src_bucket": "lambda-generate-preview-image-source",
#   "src_path": "source_folder/pig.jpg",
#   "request_id": "23453rjdkwfhdskjfh29er82349rewfds",
#   "dimensions": [
#     {
#       "width": "1000",
#       "height": "500"
#     },
#     {
#       "width": "500",
#       "height": "250"
#     },
#     {
#       "width": "250",
#       "height": "120"
#     }
#   ]
# }

def lambda_handler(event, context):

  ### Inputs
  print("#### Received event is: " + str(event))
  src_bucket = event["src_bucket"]
  src_path = event["src_path"]
  dest_bucket = event["dest_bucket"]
  dest_path = event["dest_path"]
  dimensions = event["dimensions"]
  output_format = event["format"]
  ok_sns = event["ok_sns"]
  error_sns = event["error_sns"]
  request_id = event["request_id"]

  try:
    ### Remove the leading slash from src_path and dest_path
#     if src_path[0] == "/":
#       scr_path = src_path[1:]
#     if dest_path[0] == "/":
#       dest_path = dest_path[1:]

    ### Get the maximum size from the dimensions received
    maximum_width = 0
    maximum_height = 0
    for dimension in dimensions:
      if int(dimension["width"]) > maximum_width:
        maximum_width = int(dimension["width"])
        maximum_height = int(dimension["height"])

    ### Copy the file from S3:
    print("Copy file from source S3")
    if src_bucket != "":
      client = boto3.client('s3')
      source_file = "/tmp/" + src_path.split("/")[-1]
      client.download_file(src_bucket, src_path, source_file)
    else:
      # If the src_bucket == 0 we will not copy from S3, and use local path.
      # This is for local testing
      source_file = src_path
    print("Successfully copied file from source S3")

    ### If file is Excel and it is big, make it smaller.
    if ".xls" in src_path and os.path.getsize(source_file) > 5000000: #5MB
      print("It seems to be a big Excel file. Try to keep only first 100 lines and columns")
      # opening the source excel file
      wb1 = openpyxl.load_workbook(source_file)
      ws1 = wb1.worksheets[0]

      # opening the destination excel file
      truncated_file = source_file + ".truncated." + source_file.split(".")[-1]

      # Create new empty file
      wb = openpyxl.Workbook()
      ws =  wb.active
      ws.title = "Changed Sheet"
      wb.save(filename = truncated_file)

      wb2 = openpyxl.load_workbook(truncated_file)
      ws2 = wb2.active

      # We will write only the first 100 lines and columns
      mr = 100
      mc = 100

      # copying the cell values from source
      # excel file to destination excel file
      for i in range (1, mr + 1):
          for j in range (1, mc + 1):
              # reading cell value from source excel file
              c = ws1.cell(row = i, column = j)

              # writing the read value to destination excel file
              ws2.cell(row = i, column = j).value = c.value

      # saving the destination excel file
      print("Saving truncated file to: " + truncated_file)
      wb2.save(str(truncated_file))
      print("Saved")
      source_file = truncated_file
      print("Successfuly truncated the big Excel file")

    ### Generate preview
    print("Generating preview")
    manager = PreviewManager("/tmp/cache", create_folder= True)
    preview_image = manager.get_jpeg_preview(source_file, width=maximum_width, height=maximum_height)
    print("Successfully generated preview")

    ### Make sure the image has the expected size and format
    print("Resize and change the format")
    # Open the preview image
    img = Image.open(preview_image)

    # Calculate the differences between actual sizes and wanted sizes
    delta_w = maximum_width - img.width
    delta_h = maximum_height - img.height

    # Then actual sizes is odd (not even), the we need to add one more pixel
    precision_pixel_w = maximum_width - img.width - int(delta_w/2) - int(delta_w-(delta_w/2))
    precision_pixel_h = maximum_height - img.height - int(delta_h/2) - int(delta_h-(delta_h/2))

    # Resize the image
    ltrb_border=(int(delta_w/2), int(delta_h/2), int(delta_w-(delta_w/2)) + precision_pixel_w, int(delta_h-(delta_h/2)) + precision_pixel_h)
    img_with_border = ImageOps.expand(img, border=ltrb_border, fill='white')
    output_file = "/tmp/output_preview." + output_format
    img_with_border.convert('RGB').save(output_file)
    print("Successfully resized and formatted for the maximum (" + str(maximum_width) + ":" + str(maximum_height) + ") wanted size")

    ### Copy to S3
    print("Copy the maximum (" + str(maximum_width) + ":" + str(maximum_height) + ") preview to destination bucket")
    if dest_bucket != "":
      client = boto3.client('s3')
      client.upload_file(output_file, dest_bucket, dest_path.replace("#{width}", str(maximum_width)).replace("#{height}", str(maximum_height)))
    else:
      # If the dest_bucket == 0 we will not copy from S3, but will copy on local path
      copyfile(output_file, dest_path.replace("#{width}", str(maximum_width)).replace("#{height}", str(maximum_height)))
    print("Successfully copied the maximum (" + str(maximum_width) + ":" + str(maximum_height) + ") preview to destination bucket")

    ### Transform, and copy all other wanted dimensions to S3 bucket
    for dimension in dimensions:
      # Get dimensions
      current_width = int(dimension["width"])
      current_height = int(dimension["height"])

      # Skip if the dimension is the maximum one
      if current_width == maximum_width and current_height == maximum_height:
        continue

      # Open image
      image = Image.open(output_file)
      new_image = image.resize((current_width, current_height))
      output_file_resized = "/tmp/output_preview_resized." + output_format
      new_image.save(output_file_resized)

      ### Copy to S3
      print("Copy the dimension (" + str(current_width) + ":" + str(current_height) + ") to destination bucket")
      if dest_bucket != "":
        client = boto3.client('s3')
        client.upload_file(output_file_resized, dest_bucket, dest_path.replace("#{width}", str(current_width)).replace("#{height}", str(current_height)))
      else:
        # If the dest_bucket == 0 we will not copy from S3, but will copy on local path
        copyfile(output_file_resized, dest_path.replace("#{width}", str(current_width)).replace("#{height}", str(current_height)))
      print("Successfully copied the dimension (" + str(current_width) + ":" + str(current_height) + ") preview to destination bucket")

    ### Send ok message to ok SNS
    print("Sending OK to SNS")
    if src_bucket != "":
      # If the src_bucket is not empty, then the function is running locally for dev purposes, so no need to send SNS
      # Compose the OK JSON
      okJSON = {
        "success": "true",
        "request_id": request_id,
        "src_bucket": src_bucket,
        "src_path": src_path
      }
      client = boto3.client('sns')
      response = client.publish(
          TopicArn=ok_sns,
          Message=str(okJSON)
      )
    print("Successfully send OK to SNS")
    return "OK"

  except Exception as e:
    print("Error: {}".format(e))
    tracebackError = traceback.format_exc()
    print(tracebackError)
    ### Send error message to error SNS
    print("Sending ERROR to SNS")
    # Compose the ERROR JSON
    errorJSON = {
      "success": "false",
      "request_id": request_id,
      "src_bucket": src_bucket,
      "src_path": src_path
    }
    if src_bucket != "":
      # If the src_bucket is not empty, then the function is running locally for dev purposes, so no need to send SNS
      client = boto3.client('sns')
      response = client.publish(
          TopicArn=error_sns,
          Message=str(errorJSON)
      )
    print("Successfully send ERROR to SNS")
    return "ERROR"


import pygame
from client import Client
from protocols import Protocols

class MathGame:
    def __init__(self, client):
        self.client = client
        client.start()

        self.font = None
        self.input_box = pygame.Rect(100, 100, 400, 45)
        self.color_inactive = pygame.Color("lightskyblue3")
        self.color_active = pygame.Color("dodgerblue2")
        self.color = self.color_inactive

        self.text = ""
        self.done = False
        self.logged_in = False

    def handle_event(self, event):
        if event.type == pygame.MOUSEBUTTONDOWN:
            if self.input_box.collidepoint(event.pos):
                self.color = self.color_active
            else:
                self.color = self.color_inactive

        if event.type != pygame.KEYDOWN or self.color == self.color_inactive:
            return

        if event.key == pygame.K_RETURN:
            if not self.logged_in:
                self.client.send(Protocols.Request.NICKNAME, self.text)
                self.client.nickname = self.text
                self.logged_in = True
                self.text = ""
            elif self.client.started:
                self.client.send(Protocols.Request.ANSWER, int(self.text))
                self.client.client_validate_answer(self.text)
                self.text = ""
        elif event.key == pygame.K_BACKSPACE:
            self.text = self.text[:-1]
        else:
            self.text += event.unicode

    def draw(self, screen):
        screen.fill((255, 255, 255))
        if not self.logged_in and not self.client.started:
            self.draw_login(screen)
        elif not self.client.started:
            self.draw_waiting(screen)
        else:
            self.draw_game(screen)

        pygame.display.update()

    def draw_waiting(self, screen):
        text = 'Waiting for player'
        text_surface = self.font.render(text, 1, (0, 0, 0))
        screen.blit(text_surface, (screen.get_width()/2 - text_surface.get_width()/2, screen.get_height()/2 - text_surface.get_height()/2))

    def draw_login(self, screen):
        prompt = 'Enter A Nickname'
        prompt_surface = self.font.render(prompt, 1, (0, 0, 0))
        screen.blit(prompt_surface, (100, 50))
        self.draw_input(screen)

    def draw_input(self, screen):
        pygame.draw.rect(screen, self.color, self.input_box, 2)
        txt_surface = self.font.render(self.text, 1, self.color)
        screen.blit(txt_surface, (self.input_box.x + 5, self.input_box.y + 5))
        self.input_box.w = max(100, txt_surface.get_width()+10)

    def draw_game(self, screen):
        question = self.client.get_current_question()
        question_surface = self.font.render(f"#{self.client.current_question_index + 1}: {question} = ", 1, (0, 0, 0))
        screen.blit(question_surface, (100, 50))
        self.draw_input(screen)
        self.draw_opponent_data(screen)

    def draw_opponent_data(self, screen):
        if not self.client.opponent_data:
            return

        name_surface = self.font.render(f"Opponent: {self.client.opponent_data['name']}", 1, (0, 0, 0))
        screen.blit(name_surface, (550, 50))

        wins_surface = self.font.render(f"Wins: {self.client.opponent_data['wins']}", 1, (0, 0, 0))
        screen.blit(wins_surface, (550, 100))

        loss_surface = self.font.render(f"Losses: {self.client.opponent_data['losses']}", 1, (0, 0, 0))
        screen.blit(loss_surface, (550, 150))

        question_num = self.client.opponent_question_index + 1
        question_surface = self.font.render(f"Question #{question_num}", 1, (0,0,0))
        screen.blit(question_surface, (550, 200))

    def handle_end(self, screen):
        run = True
        while run:
            for event in pygame.event.get():
                if event.type == pygame.QUIT:
                    run = False

            if self.client.winner:
                text = f"{self.client.winner} has won the game!"
            else:
                text = f"Opponent left the game..."

            text_surface = self.font.render(text, 1, (0, 0, 0))
            screen.blit(text_surface,(screen.get_width()/2 - text_surface.get_width()/2, screen.get_height()/2 - text_surface.get_height()/2))
            pygame.display.update()

    def run(self):
        pygame.init()
        screen = pygame.display.set_mode((800, 600))
        clock = pygame.time.Clock()
        self.font = pygame.font.SysFont("comicsans", 32)

        while not self.client.closed:
            clock.tick(30)
            for event in pygame.event.get():
                if event.type == pygame.QUIT:
                    self.client.close()
                    pygame.quit()
                else:
                    self.handle_event(event)

            self.draw(screen)

        self.handle_end(screen)
        pygame.quit()

if __name__ == "__main__":
    game = MathGame(Client())
    game.run()


import torch
import torch.nn as nn
import torch.nn.functional as F
from torch.utils.data import Dataset, DataLoader

import numpy as np
import sys, os

from certify import certify_grad_with_gurobi, certify_neural_network
from torch.multiprocessing import Pool

import time

from statistics import mean

device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu")

class MonoMLP(nn.Module):
    def __init__(self, mono_feature=1, mono_hiden=50) -> None:
        super(MonoMLP, self).__init__()
        self.mono_fc_in = nn.Linear(mono_feature, mono_hiden, bias=True)
        self.mono_fc_last = nn.Linear(mono_hiden, 1, bias=True)
        self.mono_feature = mono_feature
        self.mono_hiden = mono_hiden

    def forward(self, x):
        x = self.mono_fc_in(x)
        x = F.relu(x)
        x = self.mono_fc_last(x)
        # x = F.hardtanh(x, min_val=0.0, max_val=1.0)
        return x

    def reg_forward(self, num=512):
        in_list = []
        out_list = []
        input_feature = torch.rand(num, self.mono_feature).to(device)
        input_feature.requires_grad = True
        in_list.append(input_feature)
        x = self.forward(input_feature)
        out_list.append(x)
        return in_list, out_list

    def init_weight(self, weight):
        self.mono_fc_in.weight.data = weight[:50].view(-1, 1)
        self.mono_fc_in.bias.data = weight[50:100].view(-1)
        self.mono_fc_last.weight.data = weight[100:150].view(1, -1)
        self.mono_fc_last.bias.data = weight[150:151].view(-1)

    def model_weight_all(self):
        weight = self.mono_fc_in.weight.data.view(-1)
        weight = torch.cat((weight, self.mono_fc_in.bias.data.view(-1)))
        weight = torch.cat((weight, self.mono_fc_last.weight.data.view(-1)))
        weight = torch.cat((weight, self.mono_fc_last.bias.data.view(-1)))
        return weight.cpu().numpy()

def generate_regularizer(in_list, out_list):
    length = len(in_list)
    reg_loss = 0.0
    min_derivative = 0.0
    for i in range(length):
        xx = in_list[i]
        yy = out_list[i]
        for j in range(yy.shape[1]):
            grad_input = torch.autograd.grad(
                torch.sum(yy[:, j]), xx, create_graph=True, allow_unused=True
            )[0]
            grad_input_neg = -grad_input
            grad_input_neg += 0.2
            grad_input_neg[grad_input_neg < 0.0] = 0.0
            if min_derivative < torch.max(grad_input_neg**2):
                min_derivative = torch.max(grad_input_neg**2)
    reg_loss = min_derivative
    return reg_loss

class IndexDataSet(Dataset):
    def __init__(self, data) -> None:
        super().__init__()
        self.data = data
        self.length = float(len(self.data))

    def __len__(self) -> int:
        return len(self.data)

    def __getitem__(self, index):
        if torch.is_tensor(index):
            index = index.tolist()
        sample = {
            "map_val": torch.FloatTensor([self.data[index]]),
            "position": torch.FloatTensor([index]) / (self.length - 1),
        }
        return sample

def train(raw_data_path, model_param_path, save_path):

    regression_model = MonoMLP(1, 50)
    model_weight = np.genfromtxt(model_param_path, delimiter=",")
    model_weight = torch.tensor(model_weight, dtype=torch.float, requires_grad=True)

    regression_model.init_weight(model_weight)
    regression_model.to(device)

    raw_data = np.genfromtxt(raw_data_path, delimiter=",")
    train_data = IndexDataSet(raw_data[:, -1])
    train_data_loader = DataLoader(
        train_data, batch_size=256, shuffle=True, pin_memory=True
    )
    lossf = nn.MSELoss()
    optimizer = torch.optim.Adam(regression_model.parameters(), lr=0.005)
    num_epoch = 200
    for epoch in range(num_epoch):
        for sample in train_data_loader:
            x = sample["map_val"].to(device)
            y = sample["position"].to(device)
            optimizer.zero_grad()
            y_p = regression_model(x)
            loss = lossf(y_p, y)

            in_list, out_list = regression_model.reg_forward(1024)
            reg_loss = generate_regularizer(in_list, out_list)

            total_loss = loss + 1e-2* reg_loss

            total_loss.backward()
            optimizer.step()
    model_error(raw_data_path, regression_model)
    if save_path:
        np_weight = regression_model.model_weight_all()
        np.savetxt(save_path, np_weight, delimiter=",")
    mono_flag = certify_neural_network(regression_model, mono_feature_num=1)
    if mono_flag:
        print("Certified Monotonic")
    else:
        print("Not Monotonic")
    return regression_model

def model_error(raw_data_path, regression_model):
    # regression_model = NNRegressionModel(1, 100, 1)
    # model_weight = np.genfromtxt(model_param_path, delimiter=",")

    regression_model.to(torch.device("cpu"))
    raw_data = np.genfromtxt(raw_data_path, delimiter=",")

    train_data = IndexDataSet(raw_data[:, -1])
    train_data_loader = DataLoader(train_data, batch_size=raw_data.shape[0])
    torch.no_grad()
    lower_error = 0
    upper_error = 0
    model_error = []
    for sample in train_data_loader:
        x = sample["map_val"]
        y = sample["position"]
        y_p = regression_model(x)
        for y_g, ypre in zip(y, y_p):
            error = (y_g - ypre) * raw_data.shape[0]
            if error > 0 and error > upper_error:
                # model_error.append(error.item())
                upper_error = error.item()
            elif error < 0 and lower_error < -error:
                # model_error.append(-error.item())
                lower_error = -error.item()
    print(lower_error, upper_error, end=' ')

    return [upper_error, lower_error]

def train_all():
    torch.multiprocessing.set_start_method("spawn")
    dataset_name = sys.argv[1]
    print("data set name: ", dataset_name)

    raw_data_path = ""
    init_param_path = ""
    save_data_path = ""

    if dataset_name == "osm_ne_us":
        raw_data_path = "/data/jitao/dataset/OSM_US_NE/split/"
        init_param_path = "/data/jitao/dataset/OSM_US_NE/initial_param_mono/"
        save_data_path = "/data/jitao/dataset/OSM_US_NE/trained_mono/"
    elif dataset_name == "tiger":
        raw_data_path = "/data/jitao/dataset/Tiger/split/"
        init_param_path = "/data/jitao/dataset/Tiger/initial_param_mono/"
        save_data_path = "/data/jitao/dataset/Tiger/trained_mono/"
    elif dataset_name == "uniform":
        raw_data_path = "/data/jitao/dataset/uniform/split/"
        init_param_path = "/data/jitao/dataset/uniform/initial_param_mono/"
        save_data_path = "/data/jitao/dataset/uniform/trained_mono/"
    elif dataset_name == "skewed":
        raw_data_path = "/data/jitao/dataset/skewed/split/"
        init_param_path = "/data/jitao/dataset/skewed/initial_param_mono/"
        save_data_path = "/data/jitao/dataset/skewed/trained_mono/"
    else:
        print("error distribution")

    data_name_list = os.listdir(raw_data_path)
    training_pool = Pool(30)
    for index, data_name in enumerate(data_name_list):
        training_pool.apply_async(
            train,
            (
                raw_data_path + data_name,
                init_param_path + data_name,
                save_data_path + data_name
            )
        )
    training_pool.close()
    training_pool.join()

def compareErrorBound():
    dataset_name = 'osm_ne_us'
    raw_data_path = ""
    save_param_path = ""

    if dataset_name == "osm_ne_us":
        raw_data_path = "/data/jitao/dataset/OSM_US_NE/split/"
        save_param_path = "/data/jitao/dataset/OSM_US_NE/trained_mono/"
    elif dataset_name == "tiger":
        raw_data_path = "/data/jitao/dataset/Tiger/split/"
        save_param_path = "/data/jitao/dataset/Tiger/trained_mono/"
    elif dataset_name == "uniform":
        raw_data_path = "/data/jitao/dataset/uniform/split/"
        save_param_path = "/data/jitao/dataset/uniform/trained_mono/"
    elif dataset_name == "skewed":
        raw_data_path = "/data/jitao/dataset/skewed/split/"
        save_param_path = "/data/jitao/dataset/skewed/trained_mono/"
    else:
        print("error distribution")

    data_name_list = os.listdir(raw_data_path)
    print(len(data_name_list))
    randomModelErrorList = []
    metaModelErrorList = []
    for dataName in data_name_list:
        regression_model = MonoMLP(1, 50)
        model_weight = np.genfromtxt(save_param_path + dataName, delimiter=",")
        model_weight = torch.FloatTensor(model_weight)
        regression_model.init_weight(model_weight)

        meta_error = model_error(
            raw_data_path + dataName, regression_model
        )
        metaModelErrorList.append(meta_error)
        print("meta error: ", meta_error)
    print(
        "meta param: ",
        mean([error[0] for error in metaModelErrorList]),
        " -- ",
        mean([error[1] for error in metaModelErrorList]),
    )

if __name__ == "__main__":
    # s = time.time()
    # train_all()
    # e = time.time()
    # print("train time: ", e - s)

    compareErrorBound()

    # train(
    #     "/data/jitao/dataset/OSM_US_NE/split/1233.csv",
    #     "/data/jitao/dataset/OSM_US_NE/initial_param_mono/1233.csv",
    #     "/data/jitao/dataset/OSM_US_NE/trained_mono/1233.csv"
    #     )


"""
   Copyright 2016 Erik Jan de Vries

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
"""

from __future__ import division;

import logging;
log = logging.getLogger(__name__);
from collections import deque;

# %matplotlib inline
import matplotlib.pyplot as plt;

import numpy as np;

from tools import ensure_dir;
from qnet import qnet;

class CatchAgent(object):
    """Agent to act in the Catch environment for the OpenAI Gym"""
    def __init__( self, env
                , memory_maxlen = 1000
                , net = None
                , discount = 0.9
                , output_folder = "output"
                ):
        log.info("Creating an agent to play Catch");
        self.env = env;
        self.memory = deque(maxlen=memory_maxlen);
        if net is None:
            self.net = qnet( (None, self.env.grid_size, self.env.grid_size)
                           , self.env.action_space.n);
        else:
            self.net = net;
        self.discount = discount;
        ensure_dir(output_folder);
        self.output_folder = output_folder;
        self.results_filename = "{}/results.csv".format(self.output_folder);

    def save_net(self, filename = "net.pkl"):
        self.net.save("{}/{}".format(self.output_folder, filename))

    def load_net(self, filename = "net.pkl"):
        self.net.load("{}/{}".format(self.output_folder, filename))

    def choose_action(self, observation, policy='random', epsilon=0.1):
        if policy == 'random':
            action = self.env.action_space.sample();
            return action
        elif policy == 'greedy':
            q = self.net.predict(observation[np.newaxis,:,:]);
            action = np.argmax(q[0]);
            return action
        elif policy == 'eps_greedy':
            if np.random.rand() <= epsilon:
                return self.choose_action(observation, policy='random');
            else:
                return self.choose_action(observation, policy='greedy');
        raise ValueError("Policy {} not recognised!".format(policy));

    def store_memory(self, observation_t0, action, reward, observation_t1, done):
        self.memory.append([observation_t0, action, reward, observation_t1, done])

    def get_training_batch(self, batch_size = 50):
        len_memory = len(self.memory)
        inputs = np.zeros((min(len_memory, batch_size),) + self.memory[0][0].shape)
        targets = np.zeros((inputs.shape[0], self.env.action_space.n))
        for i, idx in enumerate(np.random.randint(0, len_memory, size=inputs.shape[0])):
            observation_t0, action, reward, observation_t1, done = self.memory[idx]

            inputs[i:i+1] = observation_t0
            # Get current Q-values from the net
            targets[i] = self.net.predict(observation_t0[np.newaxis,:,:])[0]

            # Correct the target Q-value given the observed reward for the given action
            if done:
                targets[i, action] = reward
            else:
                Q_t1 = np.max(self.net.predict(observation_t1[np.newaxis,:,:])[0])
                targets[i, action] = reward + self.discount * Q_t1
        return inputs, targets

    def train( self
             , n_episodes = 1000
             , batch_size = 50
             , render_on_screen = False
             , save_net_every_n_episodes = None
             , learning_rate = None
             ):
        if learning_rate is not None:
            self.net.set_learning_rate(learning_rate);

        log.info("Starting training ({} episodes)".format(n_episodes));

        log.info("Creating the results file: {}".format(self.results_filename));
        with open(self.results_filename, 'w', 0) as results_file:
            results_file.write("episode,epsilon,loss,wins\n");
        try:
            # Keeping track of the number of wins is not needed,
            # but as an indicator of how well our agent is doing,
            # it may be easier to interpret than the value of the loss function.
            wins = 0
            for i_episode in range(n_episodes):
                # During training we'll use an epsilon greedy policy to select actions
                # with epsilon decreasing exponentially from 1 to 0.1.
                epsilon = 10**(-i_episode/n_episodes)

                # The episode starts with resetting the environment.
                observation_t1 = self.env.reset()
                if render_on_screen:
                    self.env.render(mode='human');

                loss = 0.
                done = False
                while not done:
                    # The starting point for the current action is the outcome
                    # of the previous action:
                    observation_t0 = observation_t1;
                    # Choose an action using the epsilon greedy policy:
                    action = self.choose_action(observation_t0, 'eps_greedy', epsilon);
                    # Observe the outcome of taking this action:
                    observation_t1, reward, done, info = self.env.step(action);
                    if render_on_screen:
                        self.env.render(mode='human');

                    # Count the number of wins
                    if reward == 1:
                        wins += 1;

                    # Store the experience in memory
                    self.store_memory(observation_t0, action, reward, observation_t1, done)

                    # To train the q-net to select better actions in the future:
                    # - select a batch of memories to train on
                    inputs, targets = self.get_training_batch(batch_size);
                    # - train the net
                    loss += self.net.train(inputs, targets);

                if save_net_every_n_episodes is not None:
                    if i_episode % save_net_every_n_episodes == 0:
                        self.save_net("net/net_{}.pkl".format(i_episode));

                with open(self.results_filename, 'a', 0) as results_file:
                    results_file.write("{},{},{},{}\n".format(i_episode + 1, epsilon, loss, wins));
                log.info("Epoch {:03d}/{} | epsilon {:.4f} | Loss {:.4f} | Win count {}".format(i_episode, n_episodes - 1, epsilon, loss, wins))

        except KeyboardInterrupt:
            print("");
            log.warn('Training interrupted by user');

    def plot_results(self, filename = "results.png"):
        log.info("Plotting results: {}/{}".format(self.output_folder, filename));
        results = np.loadtxt(open(self.results_filename, "r"), delimiter=",", skiprows=1)
        plt.plot(results[:, 0], results[:, 2], '-')
        plt.savefig("{}/{}".format(self.output_folder, filename));
        plt.close();

    def play(self, n_episodes = 10, save_images = True, render_on_screen = False):
        log.info("Playing catch with the trained agent ({} episodes)".format(n_episodes));
        if save_images:
            output_folder = "{}/images".format(self.output_folder);
            ensure_dir(output_folder);
            log.info("Images will be saved in the folder: {}".format(output_folder));

        try:
            wins = 0
            for e in range(n_episodes):
                # The episode starts with resetting the environment.
                observation = self.env.reset();
                if render_on_screen:
                    self.env.render(mode='human');

                loss = 0.;
                done = False;

                c = 0;
                if save_images:
                    # save the initial observation as an image
                    if not render_on_screen:
                        self.env.render(mode='matplotlib');
                    plt.savefig("{}/{:02d}_{:02d}.png".format(output_folder, e, c));
                while not done:
                    c += 1;
                    # get next action
                    action = self.choose_action(observation, 'greedy');
                    # apply action, get rewards and new state
                    observation, reward, done, info = self.env.step(action);
                    if render_on_screen:
                        self.env.render(mode='human');

                    # Count the number of wins
                    if reward == 1:
                        wins += 1;

                    if save_images:
                        # save the observation as an image
                        if not render_on_screen:
                            self.env.render(mode='matplotlib');
                        plt.savefig("{}/{:02d}_{:02d}.png".format(output_folder, e, c))

        except KeyboardInterrupt:
            print("");
            log.warn('Playing interrupted by user');

        log.info("Won {} out of {} games ({} %)".format(wins, n_episodes, (100*wins/n_episodes)));