import pytest

from typing import AsyncIterator

from distributed import LocalCluster
from dask_ctl.discovery import (
    discover_cluster_names,
    discover_clusters,
    list_discovery_methods,
)
from dask_ctl.proxy import ProxyCluster

def test_discovery_methods():
    assert "proxycluster" in list_discovery_methods()

@pytest.mark.asyncio
async def test_discover_cluster_names():
    assert isinstance(discover_cluster_names(), AsyncIterator)
    async with LocalCluster(scheduler_port=8786, asynchronous=True) as _:
        count = 0
        async for _ in discover_cluster_names():
            count += 1
        assert count == 1

@pytest.mark.asyncio
async def test_cluster_client():
    from dask.distributed import Client

    port = 8786
    async with LocalCluster(scheduler_port=port, asynchronous=True) as _:
        async with Client(
            f"tcp://localhost:{port}", asynchronous=True, timeout=1
        ) as client:
            assert int(client.scheduler.address.split(":")[-1]) == port

@pytest.mark.asyncio
async def test_discovery_list():
    from dask_ctl.proxy import discover

    port = 8786
    async with LocalCluster(scheduler_port=port, asynchronous=True) as _:
        async for name, _ in discover():
            assert str(port) in name

@pytest.mark.asyncio
async def test_discover_clusters():
    with LocalCluster() as cluster:
        async for discovered_cluster in discover_clusters():
            if isinstance(discovered_cluster, ProxyCluster):
                assert cluster.scheduler_info == discovered_cluster.scheduler_info


"""empty message

Revision ID: 3aea127c3537
Revises: c3afd655f340
Create Date: 2020-09-25 00:03:08.342014

"""
from alembic import op
import sqlalchemy as sa

# revision identifiers, used by Alembic.
revision = '3aea127c3537'
down_revision = 'c3afd655f340'
branch_labels = None
depends_on = None

def upgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.create_table('post_tags',
    sa.Column('post_id', sa.Integer(), nullable=True),
    sa.Column('tag_id', sa.Integer(), nullable=True),
    sa.ForeignKeyConstraint(['post_id'], ['posts.id'], ),
    sa.ForeignKeyConstraint(['tag_id'], ['tag.id'], )
    )
    # ### end Alembic commands ###

def downgrade():
    # ### commands auto generated by Alembic - please adjust! ###
    op.drop_table('post_tags')
    # ### end Alembic commands ###


import pytest
import numpy as np
import torch
import torch.nn as nn
import io
import os
import random
import string

dtypes = [
    np.float64,
    np.float32,
    np.int8,
    np.int16,
    np.int32,
    np.int64,
    np.uint8,
    np.uint16,
]

metadata_scalar_dtypes = [
    np.float32,
    np.float64,
    np.int32,
    np.int64,
    np.uint32,
    np.uint64,
]

@pytest.fixture
def use_cluster():
    return os.getenv('SMARTREDIS_TEST_CLUSTER', "").lower() == 'true'

@pytest.fixture
def mock_data():
    return MockTestData

@pytest.fixture
def mock_model():
    return MockTestModel

class MockTestData:

    @staticmethod
    def create_data(shape):
        """Helper for creating numpy data"""

        data = []
        for dtype in dtypes:
            array = np.random.randint(-10, 10, size=shape).astype(dtype)
            data.append(array)
        return data

    @staticmethod
    def create_metadata_scalars(length):
        """Helper for creating numpy data"""

        data = []
        for dtype in metadata_scalar_dtypes:
            array = np.random.randint(-10, 10, size=length).astype(dtype)
            data.append(array)
        return data

    @staticmethod
    def create_metadata_strings(length):
        """Helper for creating list of strings"""

        data = []
        for _ in range(length):
            meta_string = ''.join(random.choices(string.ascii_uppercase + string.digits, k=20))
            data.append(meta_string)
        return data

# taken from https://pytorch.org/docs/master/generated/torch.jit.trace.html
class Net(nn.Module):
    def __init__(self):
        super(Net, self).__init__()
        self.conv = nn.Conv2d(1, 1, 3)

    def forward(self, x):
        return self.conv(x)

class MockTestModel:

    @staticmethod
    def create_torch_cnn(filepath=None):
        """Create a torch CNN for testing purposes

        Jit traces the torch Module for storage in RedisAI
        Function either saves to a file or returns a byte string
        """
        n = Net()
        example_forward_input = torch.rand(1, 1, 3, 3)

        # Trace a module (implicitly traces `forward`) and construct a
        # `ScriptModule` with a single `forward` method
        module = torch.jit.trace(n, example_forward_input)

        if filepath:
            torch.jit.save(module, filepath)
            return
        else:
            # save model into an in-memory buffer then string
            buffer = io.BytesIO()
            torch.jit.save(module, buffer)
            str_model = buffer.getvalue()
            return str_model


# This file is part of MaixUI
# Copyright (c) sipeed.com
#
# Licensed under the MIT license:
#   http://www.opensource.org/licenses/mit-license.php
#

import time, gc, math, random, sensor, audio

from fpioa_manager import fm
from machine import I2C, SPI
from Maix import I2S, GPIO, FFT

from led import cube_led
from button import sipeed_button, button_io
from pmu_axp173 import AXP173, AXP173_ADDR
from sound import CubeAudio
#from es8374 import ES8374
from msa301 import MSA301, _MSA301_I2CADDR_DEFAULT
from shtxx import SHT3x, SHT3x_ADDR, SHT31_ADDR
from bme280 import BME280, BME280_I2CADDR
from qmcx983 import QMCX983, QMCX983_I2CADDR

from ui_catch import catch
from ui_canvas import ui
from ui_sample import sample_page
from core import agent
from wdt import protect

protect.keep()

class Report():

    Key_Test = False
    Led_Test = False
    Touch_Test = False
    Power_Test = False
    Audio_Test = False
    FrontSensor_Test = False
    RearSensor_Test = False
    Msa301_Test = False
    Grove_Test = False
    Spmod_Test = False

    def __init__(self):
        self.is_load = False

    def load(self):
        if self.is_load == False:
            #print(case.load)
            self.is_load = True

    def work(self):
        #print(case.work)
        y = 0
        ui.canvas.draw_string(100, y, "Power", (127, 255, 255), scale=2)
        ui.canvas.draw_string(10, y, "1  " + str(Report.Power_Test), (0, 255, 0) if (Report.Power_Test) else (255, 0, 0), scale=2)
        ui.canvas.draw_line(10, y + 25, 240, y + 25, color=(255, 255, 255))
        y += 30
        ui.canvas.draw_string(100, y, "Msa301", (127, 255, 255), scale=2)
        ui.canvas.draw_string(10, y, "2  " + str(Report.Msa301_Test), (0, 255, 0) if (Report.Msa301_Test) else (255, 0, 0), scale=2)
        ui.canvas.draw_line(10, y + 25, 240, y + 25, color=(255, 255, 255))
        y += 30
        ui.canvas.draw_string(100, y, "Grove", (127, 255, 255), scale=2)
        ui.canvas.draw_string(10, y, "3  " + str(Report.Grove_Test), (0, 255, 0) if (Report.Grove_Test) else (255, 0, 0), scale=2)
        ui.canvas.draw_line(10, y + 25, 240, y + 25, color=(255, 255, 255))
        y += 30
        ui.canvas.draw_string(100, y, "Spmod", (127, 255, 255), scale=2)
        ui.canvas.draw_string(10, y, "4  " + str(Report.Spmod_Test), (0, 255, 0) if (Report.Spmod_Test) else (255, 0, 0), scale=2)
        ui.canvas.draw_line(10, y + 25, 240, y + 25, color=(255, 255, 255))
        y += 30
        ui.canvas.draw_string(100, y, "Key + RGB", (127, 255, 255), scale=2)
        ui.canvas.draw_string(10, y, "5  " + str(Report.Key_Test), (0, 255, 0) if (Report.Key_Test) else (255, 0, 0), scale=2)
        ui.canvas.draw_line(10, y + 25, 240, y + 25, color=(255, 255, 255))
        y += 30
        #ui.canvas.draw_string(100, y, "Touch", (127, 255, 255), scale=2)
        #ui.canvas.draw_string(10, y, "6  " + str(Report.Touch_Test), (0, 255, 0) if (Report.Touch_Test) else (255, 0, 0), scale=2)
        #ui.canvas.draw_line(10, y + 25, 240, y + 25, color=(255, 255, 255))
        #y += 30
        #ui.canvas.draw_string(100, y, "FrontSensor", (127, 255, 255), scale=2)
        #ui.canvas.draw_string(10, y, "7  " + str(Report.FrontSensor_Test), (0, 255, 0) if (Report.FrontSensor_Test) else (255, 0, 0), scale=2)
        #ui.canvas.draw_line(10, y + 25, 240, y + 25, color=(255, 255, 255))
        #y += 30
        ui.canvas.draw_string(100, y, "RearSensor", (127, 255, 255), scale=2)
        ui.canvas.draw_string(10, y, "8  " + str(Report.RearSensor_Test), (0, 255, 0) if (Report.RearSensor_Test) else (255, 0, 0), scale=2)
        ui.canvas.draw_line(10, y + 25, 240, y + 25, color=(255, 255, 255))
        y += 30
        ui.canvas.draw_string(100, y, "Audio", (127, 255, 255), scale=2)
        ui.canvas.draw_string(10, y, "9  " + str(Report.Audio_Test), (0, 255, 0) if (Report.Audio_Test) else (255, 0, 0), scale=2)
        ui.canvas.draw_line(10, y + 25, 240, y + 25, color=(255, 255, 255))
        y += 30
        ui.canvas.draw_string(100, y, "SdCard & Lcd", (127, 255, 255), scale=2)
        ui.canvas.draw_string(10, y, "*  " + str(True), (0, 255, 0) if (True) else (255, 0, 0), scale=2)
        ui.canvas.draw_line(10, y + 25, 240, y + 25, color=(255, 255, 255))
        y += 30

    def free(self):
        if self.is_load:
            #print(sample.free)
            self.is_load = False

class PowerTest():

    def __init__(self):
        self.is_load = False
        self.i2c = I2C(I2C.I2C1, freq=100*1000)
        #fm.register(30, fm.fpioa.I2C1_SCLK, force=True)
        #fm.register(31, fm.fpioa.I2C1_SDA, force=True)
        #self.load()

    def test_event(self):
        if self.isconnected and self.vbat_voltage > 0 and self.usb_voltage:
            Report.Power_Test = True
        if Report.Power_Test:
            self.agent.event(2500, self.test_event)
            sample_page.next()

    def load(self):
        if Report.Power_Test:
            sample_page.next()
        if self.is_load == False:
            # i2c init()
            sample_page.btn.enable = False
            fm.register(30, fm.fpioa.I2C1_SCLK, force=True)
            fm.register(31, fm.fpioa.I2C1_SDA, force=True)
            self.isconnected = False
            self.isError = None
            self.work_info = []
            self.agent = agent()
            self.agent.event(500, self.check)
            self.agent.event(2500, self.test_event)
            self.is_load = True

    def free(self):
        if self.is_load:
            # i2c deinit()
            sample_page.btn.enable = True
            self.is_load = False

    def check(self):
        try:
            if self.isconnected == False:
                if AXP173_ADDR in self.i2c.scan():
                    self.axp173 = AXP173(self.i2c)
                    self.isconnected = True
                    self.axp173.enable_adc(True)
                    # 默认充电限制在 4.2V, 190mA 档位
                    self.axp173.setEnterChargingControl(True)
                    self.axp173.exten_output_enable()
                    ## amigo sensor config.
                    #self.axp173.writeREG(0x27, 0x20)
                    #self.axp173.writeREG(0x28, 0x0C)
            else:
                tmp = []
                self.work_mode = self.axp173.getPowerWorkMode()
                tmp.append("WorkMode:" + hex(self.work_mode))

                # 检测 电池电压
                self.vbat_voltage = self.axp173.getVbatVoltage()
                tmp.append("vbat_voltage: {0} V".format(self.vbat_voltage))

                # 检测 电池充电电流
                self.BatteryChargeCurrent = self.axp173.getBatteryChargeCurrent()
                tmp.append("BatChargeCurrent: {0:>4.1f}mA".format(
                    self.BatteryChargeCurrent))

                # 检测 USB-ACIN 电压
                self.usb_voltage = self.axp173.getUSBVoltage()
                tmp.append("usb_voltage: {0:>4}mV".format(self.usb_voltage))

                # 检测 USB-ACIN 电流
                self.USBInputCurrent = self.axp173.getUSBInputCurrent()
                tmp.append("USBInputCurrent: {0:>4.1f}mA".format(self.USBInputCurrent))

                ### 检测 VBUS 电压
                #usb_voltage = self.axp173.getConnextVoltage()
                #print("6 VUBS_voltage: " + str(usb_voltage))

                ### 检测 VBUS 电流
                #USBInputCurrent = self.axp173.getConnextInputCurrent()
                #print("7 VUBSInputCurrent: " + str(USBInputCurrent) + "mA")

                self.getChargingControl = self.axp173.getChargingControl()
                tmp.append("ChargingControl: {}".format(hex(self.getChargingControl)))

                # 检测 是否正在充电
                if self.axp173.is_charging() == True:
                    tmp.append("Charging....")
                else:
                    tmp.append("Not Charging")
                tmp.append(self.axp173.is_charging())

                # 检测 USB 是否连接
                if self.axp173.is_usb_plugged_in() == 1:
                    tmp.append("USB plugged ....")
                else:
                    tmp.append("USB is not plugged in")

                self.work_info = tmp
        except Exception as e:
            Report.Power_Test = False
            Report.isError = str(e)
            print(e)

    def work(self):
        self.agent.parallel_cycle()

        ui.canvas.draw_string(10, 10, "1 Power Test", (127, 127, 255), scale=3)
        ui.canvas.draw_string(10, 50, "isconnected: %s" % (
            str)(self.isconnected), (255, 127, 0), scale=2)
        #if self.isconnected:
            #for i in range(len(self.work_info)):
                #ui.canvas.draw_string(
                    #20, 20*i + 80, "{0}".format(str(self.work_info[i])), mono_space=2)
        if self.isError != None:
            ui.canvas.draw_string(40, 80, self.isError, (255, 255, 255), scale=2)
            sample_page.next()

class PowerReport():

    def __init__(self):
        self.is_load = False

    def load(self):
        if self.is_load == False:
            self.is_load = True
            self.agent = agent()
            self.agent.event(2500, sample_page.next)
        else:
            sample_page.next()

    def work(self):
        self.agent.cycle()
        ui.canvas.draw_string(10, 20, "1 PowerReport", (127, 255, 255), scale=3)
        ui.canvas.draw_string(30, 120, "Pass" if (Report.Power_Test) else "Fail", (0, 255, 0) if (Report.Power_Test) else (255, 0, 0), scale=8)

    def free(self):
        if self.is_load:
            pass
            #self.is_load = False

class Msa301Test():

    def __init__(self, scl=30, sda=31):
        self.is_load = False
        self.scl = scl
        self.sda = sda
        self.i2c = I2C(I2C.I2C1, freq=100*1000, scl=self.scl, sda=self.sda)
        #fm.register(30, fm.fpioa.I2C1_SCLK, force=True)
        #fm.register(31, fm.fpioa.I2C1_SDA, force=True)

    def test_event(self):
        if self.isconnected and self.acceleration[0] != 0 and self.acceleration[1] != 0 and self.acceleration[2] != 0:
            Report.Msa301_Test = True
        sample_page.next()

    def load(self):
        if Report.Msa301_Test:
            sample_page.next()
        if self.is_load == False:
            # i2c init()
            sample_page.btn.enable = False
            fm.register(self.scl, fm.fpioa.I2C1_SCLK, force=True)
            fm.register(self.sda, fm.fpioa.I2C1_SDA, force=True)
            self.isconnected = False
            self.isError = None
            self.tapped = False
            self.acceleration = (0, 0, 0)
            self.agent = agent()
            self.agent.event(500, self.check)
            self.agent.event(3000, self.test_event)
            self.is_load = True

    def free(self):
        if self.is_load:
            # i2c deinit()
            sample_page.btn.enable = True
            self.is_load = False

    def check(self):
        try:
            if self.isconnected == False:
                if _MSA301_I2CADDR_DEFAULT in self.i2c.scan():
                    self.msa301 = MSA301(self.i2c)
                    self.isconnected = True
            else:
                self.tapped = self.msa301.tapped
                self.acceleration = self.msa301.acceleration
        except Exception as e:
            Report.Msa301_Test = False
            Report.isError = str(e)
            print(e)

    def work(self):
        self.agent.parallel_cycle()

        ui.canvas.draw_string(10, 30, "2 Msa301Test", (127, 127, 255), scale=3)
        ui.canvas.draw_string(10, 80, "isconnected: %s" % (
            str)(self.isconnected), (255, 127, 0), scale=2)
        #if self.isconnected:
            #ui.canvas.draw_string(10, 120, "tapped: %s" % (
                #str)(self.tapped), (0, 214, 126), scale=2)

            #ui.canvas.draw_string(10, 140, "x", (255, 0, 0), scale=2)
            #ui.canvas.draw_line(120, 150, 120 + int(self.acceleration[0] * 8), 150, color=(41, 131, 255))
            #ui.canvas.draw_string(10, 160, "y", (0, 255, 0), scale=2)
            #ui.canvas.draw_line(120, 170, 120 + int(self.acceleration[1] * 8), 170, color=(141, 31, 255))
            #ui.canvas.draw_string(10, 180, "z", (0, 0, 255), scale=2)
            #ui.canvas.draw_line(120, 190, 120 + int(self.acceleration[2] * 8), 190, color=(241, 131, 55))

            #ui.canvas.draw_string(40, 210,
                #str(("%-02.2f %-02.2f %-02.2f" % self.acceleration)), (127, 255, 255), scale=2)

        if self.isError != None:
            ui.canvas.draw_string(40, 80, self.isError, (255, 255, 255), scale=2)
            sample_page.next()

class Msa301Report():

    def __init__(self):
        self.is_load = False

    def load(self):
        if self.is_load == False:
            self.is_load = True
            self.agent = agent()
            self.agent.event(2000, sample_page.next)
        else:
            sample_page.next()

    def work(self):
        self.agent.cycle()
        ui.canvas.draw_string(10, 20, "2 Msa301Report", (127, 255, 255), scale=3)
        ui.canvas.draw_string(30, 120, "Pass" if (Report.Msa301_Test) else "Fail", (0, 255, 0) if (Report.Msa301_Test) else (255, 0, 0), scale=8)

    def free(self):
        if self.is_load:
            pass
            #self.is_load = False

class GroveTest():

    def __init__(self, scl=24, sda=25):
        self.is_load = False
        self.scl = scl
        self.sda = sda
        self.i2c = I2C(I2C.I2C0, freq=100*1000, scl=self.scl, sda=self.sda)
        #fm.register(30, fm.fpioa.I2C1_SCLK, force=True)
        #fm.register(31, fm.fpioa.I2C1_SDA, force=True)

    def test_event(self):
        if self.isconnected and self.work_data != None and self.work_data[0] > 0 and self.work_data[1] > 1:
            Report.Grove_Test = True
        sample_page.next()

    def load(self):
        if Report.Grove_Test:
            sample_page.next()
        if self.is_load == False:
            # i2c init()
            sample_page.btn.enable = False
            fm.register(self.scl, fm.fpioa.I2C0_SCLK, force=True)
            fm.register(self.sda, fm.fpioa.I2C0_SDA, force=True)
            self.isconnected = False
            self.isError = None
            self.work_info = []
            self.work_data = None
            self.agent = agent()
            self.agent.event(250, self.check)
            self.agent.event(3000, self.test_event)
            self.is_load = True

    def free(self):
        if self.is_load:
            # i2c deinit()
            sample_page.btn.enable = True
            self.is_load = False

    def check(self):
        try:
            if self.isconnected == False:
                # print(self.i2c.scan())
                if SHT3x_ADDR in self.i2c.scan():
                    self.sht3x = SHT3x(self.i2c, SHT3x_ADDR)
                    self.isconnected = True
                if SHT31_ADDR in self.i2c.scan():
                    self.sht3x = SHT3x(self.i2c, SHT31_ADDR)
                    self.isconnected = True
            else:
                tmp = []
                self.work_data = self.sht3x.read_temp_humd()
                tmp.append("data:" + str(self.work_data))
                self.work_info = tmp
        except Exception as e:
            Report.Grove_Test = False
            Report.isError = str(e)
            print(e)

    def work(self):
        self.agent.parallel_cycle()

        ui.canvas.draw_string(10, 10, "3 Grove Test SHT3X", (0, 255, 127), scale=2)
        ui.canvas.draw_string(10, 50, "isconnected: %s" % (
            str)(self.isconnected), (255, 127, 0), scale=2)
        #if self.isconnected:
            #for i in range(len(self.work_info)):
                #ui.canvas.draw_string(
                    #20, 20*i + 90, "{0}".format(str(self.work_info[i])), scale=2)
        if self.isError != None:
            ui.canvas.draw_string(40, 80, self.isError, (255, 255, 255), scale=2)
            sample_page.next()

class GroveReport():

    def __init__(self):
        self.is_load = False

    def load(self):
        if self.is_load == False:
            self.is_load = True
            self.agent = agent()
            self.agent.event(2000, sample_page.next)
        else:
            sample_page.next()

    def work(self):
        self.agent.cycle()
        ui.canvas.draw_string(10, 20, "3 GroveReport", (127, 255, 255), scale=3)
        ui.canvas.draw_string(30, 120, "Pass" if (Report.Grove_Test) else "Fail", (0, 255, 0) if (Report.Grove_Test) else (255, 0, 0), scale=8)

    def free(self):
        if self.is_load:
            pass
            #self.is_load = False

class SpmodTest():

    test_conut = 0

    def __init__(self, mosi=8, miso=15, cs=20, clk=21):
        self.is_load = False
        self.spi = SPI(SPI.SPI1, mode=SPI.MODE_MASTER, baudrate=400*1000,
            polarity=0, phase=0, bits=8, firstbit=SPI.MSB, sck=clk, mosi=mosi, miso=miso)
        fm.register(cs, fm.fpioa.GPIO6, force=True)
        self.cs = GPIO(GPIO.GPIO6, GPIO.OUT)

    def test_event(self):
        if self.work_data != None and self.work_data == b'\x0b\x17':
            Report.Spmod_Test = True
        sample_page.next()

    def load(self):
        if Report.Spmod_Test and SpmodTest.test_conut == 2:
            sample_page.next()
        if self.is_load == False:
            # i2c init()
            sample_page.btn.enable = False
            self.isError = None
            self.work_info = []
            self.work_data = None
            self.agent = agent()
            self.agent.event(250, self.check)
            self.agent.event(2500, self.test_event)
            self.is_load = True

    def free(self):
        if self.is_load:
            # i2c deinit()
            sample_page.btn.enable = True
            self.is_load = False
            # recover sdcard
            fm.register(27, fm.fpioa.SPI1_SCLK, force=True)
            fm.register(28, fm.fpioa.SPI1_D0, force=True)
            fm.register(26, fm.fpioa.SPI1_D1, force=True)

    def check(self):
        try:
            tmp = []
            self.cs.value(0)
            write_data = bytearray([0x90, 0x00, 0x00, 0x00])
            self.spi.write(write_data)
            id_buf = bytearray(2)
            self.spi.readinto(id_buf, write=0xff)
            self.work_data = id_buf
            self.cs.value(1)
            tmp.append("Flash ReadID\n\n" + str(self.work_data))
            self.work_info = tmp
        except Exception as e:
            Report.Spmod_Test = False
            Report.isError = str(e)
            print(e)

    def work(self):
        self.agent.parallel_cycle()

        ui.canvas.draw_string(10, 10, "4 Spmod Test", (0, 255, 127), scale=2)
        #if self.work_data:
            #for i in range(len(self.work_info)):
                #ui.canvas.draw_string(
                    #20, 20*i + 90, "{0}".format(str(self.work_info[i])), scale=2)
        if self.isError != None:
            ui.canvas.draw_string(40, 80, self.isError, (255, 255, 255), scale=2)
            sample_page.next()

class SpmodReport():

    def __init__(self):
        self.is_load = False

    def load(self):
        if self.is_load == False:
            self.is_load = True
            self.agent = agent()
            self.agent.event(2000, sample_page.next)
        else:
            sample_page.next()

    def work(self):
        self.agent.cycle()
        ui.canvas.draw_string(10, 20, "4 SpmodReport", (127, 255, 255), scale=3)
        ui.canvas.draw_string(30, 120, "Pass" if (Report.Spmod_Test) else "Fail", (0, 255, 0) if (Report.Spmod_Test) else (255, 0, 0), scale=8)

    def free(self):
        if self.is_load:
            pass
            #self.is_load = False

class WaitTestStart():

    def __init__(self):
        self.is_load = False

    def key_event(self):
        self.btn.expand_event()

        if self.btn.back() == 2:
            sample_page.next()
        elif self.btn.next() == 2:
            sample_page.next()
        elif self.btn.home() == 2:
            sample_page.next()

    def load(self):
        if self.is_load == False:
            self.is_load = True
            sample_page.btn.enable = False
            self.btn = sipeed_button()
            # self.btn.config(23, 20, 31)
            self.agent = agent()
            self.agent.event(150, self.key_event)
            #self.agent.event(2000, sample_page.next)
        else:
            if Report.Key_Test:
                sample_page.next()

    def work(self):
        self.agent.cycle()
        ui.canvas.draw_string(10, 20, "Press \n\n Any-key \n\n Start Test", (127, 255, 255), scale=3)

    def free(self):
        if self.is_load:
            pass
            #self.is_load = False
        sample_page.btn.enable = True

class KeyTest():

    home_click = 0
    back_click = 0
    next_click = 0

    def __init__(self):
        self.is_load = False
        #self.load()

    def load(self):
        if Report.Key_Test:
            sample_page.next()
        if self.is_load == False:
            #print(case.load)
            self.is_load = True
            sample_page.btn.enable = False
            cube_led.init(13, 12, 14, 32)
            self.btn = sipeed_button()
            # self.btn.config(23, 20, 31)
            self.agent = agent()
            self.agent.event(150, self.key_event)
            self.agent.event(16000, lambda :sample_page.next())
            KeyTest.home_click = 0
            KeyTest.back_click = 0
            KeyTest.next_click = 0

    def key_event(self):
        self.btn.expand_event()

        if self.btn.back() == 2:
            KeyTest.back_click += 1
            cube_led.r.value(0)
            cube_led.g.value(1)
            cube_led.b.value(1)
        elif self.btn.next() == 2:
            KeyTest.next_click += 1
            cube_led.r.value(1)
            cube_led.g.value(1)
            cube_led.b.value(0)
        elif self.btn.home() == 2:
            KeyTest.home_click += 1
            cube_led.r.value(1)
            cube_led.g.value(0)
            cube_led.b.value(1)
            if self.btn.interval() > 2000: # long press
                sample_page.next()
        if KeyTest.home_click > 1 and KeyTest.back_click > 1 and KeyTest.next_click > 1:
            Report.Key_Test = True
            sample_page.next()

    def work(self):
        self.agent.parallel_cycle()
        y = 20
        ui.canvas.draw_string(10, y,
            '5 KeyTest', (255, 255, 255), scale=3)
        y += 60
        ui.canvas.draw_string(20, y,
            'home click %d' % self.home_click, (255, 0, 0), scale=3)
        y += 60
        ui.canvas.draw_string(20, y,
            'back click %d' % self.back_click, (0, 255, 0), scale=3)
        y += 60
        ui.canvas.draw_string(20, y,
            'next click %d' % self.next_click, (0, 0, 255), scale=3)

    def free(self):
        if self.is_load:
            #print(sample.free)
            self.is_load = False
            sample_page.btn.enable = True
            if self.home_click > 0 and self.back_click > 0 and self.next_click > 0:
                Report.Key_Test = True
            cube_led.r.value(1)
            cube_led.g.value(1)
            cube_led.b.value(1)

class KeyReport():

    def __init__(self):
        self.is_load = False

    def key_event(self):
        self.btn.expand_event()

        if self.btn.back() == 2:
            sample_page.next()
        elif self.btn.next() == 2:
            sample_page.next()
        elif self.btn.home() == 2:
            sample_page.next()

    def load(self):
        if self.is_load == False:
            self.is_load = True
            sample_page.btn.enable = False
            self.btn = sipeed_button()
            # self.btn.config(23, 20, 31)
            self.agent = agent()
            self.agent.event(150, self.key_event)
            #self.agent.event(2000, sample_page.next)
        else:
            if Report.Key_Test:
                sample_page.next()

    def work(self):
        self.agent.cycle()
        y = 20
        ui.canvas.draw_string(10, y, "5 KeyReport", (127, 255, 255), scale=3)
        y += 40
        ui.canvas.draw_string(10, y, "Home " + ("Pass" if (KeyTest.home_click) else "Fail"), (0, 255, 0) if (KeyTest.home_click) else (255, 0, 0), scale=3)
        y += 40
        ui.canvas.draw_string(10, y, "Back " + ("Pass" if (KeyTest.back_click) else "Fail"), (0, 255, 0) if (KeyTest.back_click) else (255, 0, 0), scale=3)
        y += 40
        ui.canvas.draw_string(10, y, "Next " + ("Pass" if (KeyTest.next_click) else "Fail"), (0, 255, 0) if (KeyTest.next_click) else (255, 0, 0), scale=3)
        y += 40
        ui.canvas.draw_string(10, y, "KeyTest " + ("Pass" if (Report.Key_Test) else "Fail"), (0, 255, 0) if (Report.Key_Test) else (255, 0, 0), scale=3)
        y += 40
        ui.canvas.draw_string(10, y, "Press Any-Key Continue", (255, 255, 255), scale=2)

    def free(self):
        if self.is_load:
            pass
        sample_page.btn.enable = True

class RearSensorTest():

    def __init__(self):
        self.is_load = False
        self.isconnected = False

    def test_event(self):
        if self.get_image != None:
            Report.RearSensor_Test = True
        sample_page.next()

    def check(self):
        try:
            self.btn.expand_event()

            if self.btn.home() == 2:
                cube_led.w.value(0)
                Report.RearSensor_Test = True
                sample_page.next()

            if self.isconnected == False:
                try:
                    sensor.reset()
                    sensor.set_pixformat(sensor.YUV422)
                    sensor.set_framesize(sensor.QVGA)
                    sensor.set_hmirror(1)
                    sensor.set_vflip(1)
                    sensor.run(1)
                    sensor.skip_frames()
                    self.isconnected = True
                    cube_led.w.value(0)
                except Exception as e:
                    Report.RearSensor_Test = False
                    Report.isError = str(e)
                    print(e)
        except Exception as e:
            Report.RearSensor_Test = False
            Report.isError = str(e)
            print(e)

    def load(self):
        if Report.RearSensor_Test:
            sample_page.next()
        if self.is_load == False:
            cube_led.init(13, 12, 14, 32)
            cube_led.w.value(1)
            sample_page.btn.enable = False
            self.btn = sipeed_button()
            # self.btn.config(23, 20, 31)
            self.get_image = None
            self.isError = None
            self.agent = agent()
            self.agent.event(150, self.check)
            self.agent.event(8000, self.test_event)
            self.is_load = True

    def free(self):
        if self.is_load:
            sample_page.btn.enable = True
            self.is_load = False
            cube_led.w.value(1)

    def work(self):
        self.agent.parallel_cycle()
        if self.isconnected:
            try:
                self.get_image = sensor.snapshot()
                #ui.canvas.draw_image(self.get_image, 0, 0)
                ui.canvas = (self.get_image)
            except Exception as e:
                print(e)
        ui.canvas.draw_string(10, 30, "8 RearSensor Test", (127, 127, 255), scale=3)
        ui.canvas.draw_string(10, 70, "isconnected: %s" % (
            str)(self.isconnected), (255, 127, 0), scale=2)
        if self.isError != None:
            ui.canvas.draw_string(40, 80, self.isError, (255, 255, 255), scale=2)
            sample_page.next()

class RearSensorReport():

    def __init__(self):
        self.is_load = False

    def key_event(self):
        self.btn.expand_event()

        if self.btn.back() == 2:
            sample_page.next()
        elif self.btn.next() == 2:
            sample_page.next()
        elif self.btn.home() == 2:
            sample_page.next()

    def load(self):
        if self.is_load == False:
            self.is_load = True
            sample_page.btn.enable = False
            self.btn = sipeed_button()
            # self.btn.config(23, 20, 31)
            self.agent = agent()
            self.agent.event(150, self.key_event)
            #self.agent.event(2000, sample_page.next)
        elif Report.RearSensor_Test:
            sample_page.next()

    def work(self):
        self.agent.cycle()
        y = 20
        ui.canvas.draw_string(10, y, "8 RearSensorReport", (127, 255, 255), scale=3)
        y += 50
        ui.canvas.draw_string(10, y, "RearSensor " + ("Pass" if (Report.RearSensor_Test) else "Fail"), (0, 255, 0) if (Report.RearSensor_Test) else (255, 0, 0), scale=3)
        y += 50
        ui.canvas.draw_string(10, y, "Press Any-Key Continue", (255, 255, 255), scale=2)

    def free(self):
        if self.is_load:
            pass
            #self.is_load = False
        sample_page.btn.enable = True

class AudioTest():

    PlayTest = False
    RecordTest = False

    def __init__(self, scl=30, sda=31):
        self.is_load = False
        self.scl = scl
        self.sda = sda
        self.i2c = I2C(I2C.I2C1, freq=100*1000)
        self.count = 0

    def load(self):
        if Report.Audio_Test:
            sample_page.next()
        if self.is_load == False:
            # i2c init()
            sample_page.btn.enable = False
            from fpioa_manager import fm
            fm.register(self.scl,fm.fpioa.I2C1_SCLK, force=True)
            fm.register(self.sda,fm.fpioa.I2C1_SDA, force=True)
            self.isconnected = False
            self.isError = None
            self.is_play = False
            self.is_record = False
            self.state = 0
            self.fft_amp = None
            self.btn = sipeed_button()
            # self.btn.config(23, 20, 31)
            self.count += 1
            self.agent = agent()
            self.agent.event(150, self.key_event)
            self.agent.event(500, self.check)
            self.agent.event(16000, self.test_event)
            self.is_load = True

    def key_event(self):
        self.btn.expand_event()

        if self.btn.back() == 2:
            if self.state == 0:
                AudioTest.PlayTest = False
            if self.state == 2:
                AudioTest.RecordTest = False
            self.state += 1
        elif self.btn.next() == 2 or self.btn.home() == 2:
            if self.state == 0:
                AudioTest.PlayTest = True
            if self.state == 2:
                AudioTest.RecordTest = True
            self.state += 1
        if self.state > 2:
            sample_page.next()

    def test_event(self):
        if self.state == 0 or self.state == 2:
            self.state += 1

    def free(self):
        if self.is_load:
            # i2c deinit()
            sample_page.btn.enable = True
            self.is_load = False

    def check(self):
        try:
            if self.isconnected == False:
                self.isconnected = CubeAudio.check()
            else:
                if self.state == 0 and self.is_play == False:
                    self.is_play = True
                    CubeAudio.i2c = self.i2c
                    CubeAudio.ready()
                    from fpioa_manager import fm
                    fm.register(19, fm.fpioa.I2S0_MCLK, force=True)
                    fm.register(35, fm.fpioa.I2S0_SCLK, force=True)
                    fm.register(33, fm.fpioa.I2S0_WS, force=True)
                    fm.register(34, fm.fpioa.I2S0_IN_D0, force=True)
                    fm.register(18, fm.fpioa.I2S0_OUT_D2, force=True)
                    # CubeAudio.i2s.set_sample_rate(22050)
                elif self.state == 1 and self.is_record == False:
                    self.is_record = True
                    CubeAudio.ready(True)
                    CubeAudio.i2s.set_sample_rate(22050)
        except Exception as e:
            #Report.Audio_Test = False
            Report.isError = str(e)
            print(e)

    def work(self):
        self.agent.parallel_cycle()

        ui.canvas.draw_string(10, 30, "9 Audio Test", (127, 127, 255), scale=3)
        ui.canvas.draw_string(10, 70, "isconnected: %s" % (
            str)(self.isconnected), (255, 127, 0), scale=2)

        ui.canvas.draw_string(10, 100, "Test: %s" %
            ('play' if self.state == 0 else 'record'), (255, 127, 0), scale=3)

        #print(time.ticks_ms())

        if self.isconnected:
            if self.state == 0 and self.is_play:
                if CubeAudio.event() == False:
                    CubeAudio.load(os.getcwd() + "/res/sound/loop.wav", 100)
                    #print('self.count', self.count)
                    if self.count > 1:
                        CubeAudio.i2s.set_sample_rate(22050)
                    else:
                        # pass
                        CubeAudio.i2s.set_sample_rate(22050)
            elif self.state == 1:
                ui.canvas.draw_string(10, 200, "Press Any-Key \n Start", (255, 127, 0), scale=3)
            elif self.state == 2 and self.is_record:
                tmp = CubeAudio.i2s.record(1024)
                fft_res = FFT.run(tmp.to_bytes(), 512)
                fft_amp = FFT.amplitude(fft_res)
                if fft_amp[50] > 100 and fft_amp[100] > 100:
                    AudioTest.RecordTest = True
                    sample_page.next()
                for x_shift in range(240):
                    hist_height = fft_amp[x_shift]
                    ui.canvas.draw_rectangle((x_shift, 0, 1, hist_height), [255,255,255], 1, True)
                    #print((x_shift, 0, 1, hist_height))

        if self.isError != None:
            ui.canvas.draw_string(40, 80, self.isError, (255, 255, 255), scale=2)
            sample_page.next()

class AudioReport():

    def __init__(self):
        self.is_load = False

    def key_event(self):
        self.btn.expand_event()

        if self.btn.back() == 2:
            sample_page.next()
        elif self.btn.next() == 2:
            sample_page.next()
        elif self.btn.home() == 2:
            sample_page.next()

    def load(self):
        if self.is_load == False:
            self.is_load = True
            sample_page.btn.enable = False
            self.btn = sipeed_button()
            # self.btn.config(23, 20, 31)
            self.agent = agent()
            self.agent.event(150, self.key_event)
            #self.agent.event(2000, sample_page.next)
        elif Report.Audio_Test:
            sample_page.next()

        Report.Audio_Test = False
        if AudioTest.PlayTest and AudioTest.RecordTest:
            Report.Audio_Test = True

    def work(self):
        self.agent.cycle()
        y = 20
        ui.canvas.draw_string(10, y, "9 AudioReport", (127, 255, 255), scale=3)
        y += 50
        ui.canvas.draw_string(10, y, "PlayTest " + ("Pass" if (AudioTest.PlayTest) else "Fail"), (0, 255, 0) if (AudioTest.PlayTest) else (255, 0, 0), scale=3)
        y += 50
        ui.canvas.draw_string(10, y, "Record " + ("Pass" if (AudioTest.RecordTest) else "Fail"), (0, 255, 0) if (AudioTest.RecordTest) else (255, 0, 0), scale=3)
        y += 50
        ui.canvas.draw_string(10, y, "Audio " + ("Pass" if (Report.Audio_Test) else "Fail"), (0, 255, 0) if (Report.Audio_Test) else (255, 0, 0), scale=3)
        y += 50
        ui.canvas.draw_string(10, y, "Press Any-Key Continue", (255, 255, 255), scale=2)

    def free(self):
        if self.is_load:
            pass
            #self.is_load = False
        sample_page.btn.enable = True

class SdcardTest():

    def __init__(self):
        self.is_load = False
        self.load()

    def load(self):
        if self.is_load == False:
            self.is_load = True
            self.result = os.getcwd() == '/sd' # and len(os.listdir('/sd')) > 0
            self.agent = agent()
            self.agent.event(2000, sample_page.next)
        else:
            sample_page.next()

    def work(self):
        self.agent.cycle()
        ui.canvas.draw_string(10, 20, "SdCardTest", (127, 255, 255), scale=3)
        ui.canvas.draw_string(30, 120, "Pass" if (self.result) else "Fail", (0, 255, 0) if (self.result) else (255, 0, 0), scale=8)

        ui.canvas.draw_string(30, 260, "Start Test", (0, 0, 255), scale=3)

    def free(self):
        if self.is_load:
            pass
            #self.is_load = False

if __name__ == "__main__":

    protect.keep()

    import time, gc
    # gc.collect()
    # gc.collect()
    if len(sample_page.samples) > 0:
        sample_page.samples = []
        # gc.collect()

    button_io.config(10, 11, 16)
    sample_page.key_init()

    sample_page.add_sample(Report()) # keep

    sample_page.add_sample(AudioReport())
    sample_page.add_sample(AudioTest())

    sample_page.add_sample(RearSensorReport())
    sample_page.add_sample(RearSensorTest())

    sample_page.add_sample(KeyReport())
    sample_page.add_sample(KeyTest())

    sample_page.add_sample(WaitTestStart())

    sample_page.add_sample(SpmodReport())
    sample_page.add_sample(SpmodTest())

    sample_page.add_sample(GroveReport())
    sample_page.add_sample(GroveTest())

    sample_page.add_sample(Msa301Report())
    sample_page.add_sample(Msa301Test())

    sample_page.add_sample(PowerReport())
    sample_page.add_sample(PowerTest())

    sample_page.add_sample(SdcardTest()) # keep

    #ui.height, ui.weight = int(lcd.width() / 2), int(lcd.height())

    ui.height, ui.weight = 240, 240

    @ui.warp_template(ui.blank_draw)
    #@ui.warp_template(ui.grey_draw)
    @ui.warp_template(sample_page.sample_draw)
    @catch
    def app_main():
        ui.display()

    last = time.ticks_ms()
    while True:
        #app_main()
        #import time
        # last = time.ticks_ms() - 1
        while True:
            #app_main()
            #protect.keep()
            #continue
            try:
                #print((int)(1000 / (time.ticks_ms() - last)), 'fps')
                last = time.ticks_ms()
                app_main()
                protect.keep()
                #print(time.ticks_ms(), 'ram total : ' + str(gc.mem_free() / 1024) + ' kb')
                #time.sleep(0.1)
            except KeyboardInterrupt:
                protect.stop()
                raise KeyboardInterrupt()
            except MemoryError as e:
                print(time.ticks_ms(), 'ram total : ' + str(gc.mem_free() / 1024) + ' kb')
                #print(e)
            except Exception as e:
                print(e)


from model_helper import *
from sklearn import preprocessing
from sklearn.model_selection import train_test_split

def map_reopening_plan(s):
    if s == "Fully online":
        return -1
    elif s == "Primarily online":
        return -0.5
    elif s == "Hybrid":
        return 0
    elif s == "Primarily in person":
        return 0.5
    elif s == "Fully in person":
        return 1
    else:
        return float('nan')

def map_covid_cases(n):
    if n < 10:
        return 0
    elif n < 200:
        return 1
    elif n >= 200:
        return 2
    else:
        return float('nan')

def load_and_clean_data():
    # Load dataset
    df = pd.read_csv("./data/all-data.csv")

    # Keep only relevant columns
    df = df[['UGDS', 'LOCALE', 'ADM_RATE', 'REGION', 'CCSIZSET', 'TUITFTE', 'reopening_plan', 'covid_cases']]

    # Map reopening plan to float from -1 to 1
    df['reopening_plan'] = df['reopening_plan'].apply(map_reopening_plan)

    # Map covid cases to three categories
    df['covid_cases'] = df['covid_cases'].apply(map_covid_cases)

    # Drop rows with NaN
    df = df.dropna()

    # Encode LOCALE, REGION, and CCSIZSET as one-hot vectors
    locale_col = df['LOCALE'].tolist()
    df = df.drop(columns=['LOCALE'])
    locale_options = [11, 12, 13, 21, 22, 23, 31, 32, 33, 41, 42, 43]
    locale_option_names = ["Locale_" + str(xi) for xi in locale_options]
    locale_matrix = convert_to_one_hot(locale_options, locale_col)

    region_col = df['REGION'].tolist()
    df = df.drop(columns=['REGION'])
    region_options = list(range(10))
    region_option_names = ["Region_" + str(xi) for xi in region_options]
    region_matrix = convert_to_one_hot(region_options, region_col)

    carnegie_col = df['CCSIZSET'].tolist()
    df = df.drop(columns=['CCSIZSET'])
    carnegie_options = list(range(18))
    carnegie_options.append(-2)
    carnegie_option_names = ["Carnegie_" + str(xi) for xi in carnegie_options]
    carnegie_matrix = convert_to_one_hot(carnegie_options, carnegie_col)

    # Remove covid_cases from data so that it is not normalized
    covid_col = df['covid_cases'].tolist()
    df = df.drop(columns=['covid_cases'])

    # Normalize the values in the dataset
    df = normalize(df)
    df = reset(df)

    # Add locale, carnegie, region, and covid back to the datase
    locale_df = pd.DataFrame(data=locale_matrix, columns=locale_option_names)
    carnegie_df = pd.DataFrame(data=carnegie_matrix, columns=carnegie_option_names)
    region_df = pd.DataFrame(data=region_matrix, columns=region_option_names)
    df = pd.concat([df, locale_df, carnegie_df, region_df], axis=1, sort=False)
    df['covid_cases'] = covid_col

    return df

def test_model(test, model):
    # Evaluate the precision of the model for each row in the test set
    test_clean = test.drop(columns=["covid_cases"])
    correct = 0
    total = 0
    correct_by_bucket = {0: 0, 1: 0, 2: 0} # True positives
    predictions_by_bucket = {0: 0, 1: 0, 2: 0} # True positives + false positives
    total_by_bucket = {0: 0, 1: 0, 2: 0} # True positives + false negatives
    for index, row_clean in test_clean.iterrows():
        actual = test.loc[index]["covid_cases"]
        numpy_batch = np.array([row_clean.values])
        prediction_list = model.predict_on_batch(x=numpy_batch)[0].tolist()
        prediction = prediction_list.index(max(prediction_list))

        if int(actual) == int(prediction):
            correct += 1
            correct_by_bucket[actual] += 1
        total += 1
        total_by_bucket[actual] += 1
        predictions_by_bucket[prediction] += 1

    # Print the results
    print("\n" + ("-" * 50))
    print("TEST SET RESULTS")
    print("=" * 50)
    print()
    print("Overall Precision: %0.3f\n" % (correct/total))
    for k in correct_by_bucket.keys():
        print("Bucket %d:" % (k))
        print("\tPrecision: %0.3f" % (correct_by_bucket[k]/predictions_by_bucket[k]))
        print("\tActual Count: %d" % (total_by_bucket[k]))
        print("\tNum Predictions: %d" % (predictions_by_bucket[k]))
        print()
    print("-" * 50)

def main():
    tf.get_logger().setLevel('ERROR')

    df = load_and_clean_data()

    # Split into test and train sets
    train, test = train_test_split(df, test_size=0.25)
    x_train = train.drop(columns=['covid_cases']).values
    y_train = train['covid_cases'].values
    y_train = tf.keras.utils.to_categorical(y_train)

    # Train the model
    epochs = 2000
    batch_size = 20
    my_model = create_model()
    epochs, mse = train_model(my_model, x_train, y_train, epochs, batch_size)

    # Test the model
    test_model(test, my_model)

    # Plot the loss curve
    plot_the_loss_curve(epochs, mse)

if __name__=="__main__":
    main()


from ..regex import RegexDispatcher
from ..append import append
from .csv import CSV
import os
import datashape
import sqlalchemy
import subprocess

copy_command = RegexDispatcher('copy_command')
execute_copy = RegexDispatcher('execute_copy')

@copy_command.register('.*sqlite')
def copy_sqlite(dialect, tbl, csv):
    abspath = os.path.abspath(csv.path)
    tblname = tbl.name
    dbpath = str(tbl.bind.url).split('///')[-1]

    statement = """
        (echo '.mode csv'; echo '.import {abspath} {tblname}';) | sqlite3 {dbpath}
        """

    return statement.format(**locals())

@execute_copy.register('sqlite')
def execute_copy_sqlite(dialect, engine, statement):
    ps = subprocess.Popen(statement, shell=True, stdout=subprocess.PIPE)
    return ps.stdout.read()

@copy_command.register('postgresql')
def copy_postgres(dialect, tbl, csv):
    abspath = os.path.abspath(csv.path)
    tblname = tbl.name
    format_str = 'csv'
    delimiter = csv.dialect.get('delimiter', ',')
    na_value = ''
    quotechar = csv.dialect.get('quotechar', '"')
    escapechar = csv.dialect.get('escapechar', '\\')
    header = not not csv.has_header
    encoding = csv.encoding or 'utf-8'

    statement = """
        COPY {tblname} FROM '{abspath}'
            (FORMAT {format_str},
             DELIMITER E'{delimiter}',
             NULL '{na_value}',
             QUOTE '{quotechar}',
             ESCAPE '{escapechar}',
             HEADER {header},
             ENCODING '{encoding}');"""

    return statement.format(**locals())

@copy_command.register('mysql.*')
def copy_mysql(dialect, tbl, csv):
    mysql_local = ''
    abspath = os.path.abspath(csv.path)
    tblname = tbl.name
    delimiter = csv.dialect.get('delimiter', ',')
    quotechar = csv.dialect.get('quotechar', '"')
    escapechar = csv.dialect.get('escapechar', '\\')
    lineterminator = csv.dialect.get('lineterminator', r'\n\r')
    skiprows = 1 if csv.has_header else 0
    encoding = csv.encoding or 'utf-8'

    statement = u"""
        LOAD DATA {mysql_local} INFILE '{abspath}'
        INTO TABLE {tblname}
        CHARACTER SET {encoding}
        FIELDS
            TERMINATED BY '{delimiter}'
            ENCLOSED BY '{quotechar}'
            ESCAPED BY '{escapechar}'
        LINES TERMINATED by '{lineterminator}'
        IGNORE {skiprows} LINES;
    """

    return statement.format(**locals())

@execute_copy.register('.*', priority=9)
def execute_copy_all(dialect, engine, statement):
    conn = engine.raw_connection()
    cursor = conn.cursor()
    cursor.execute(statement)
    conn.commit()

@append.register(sqlalchemy.Table, CSV)
def append_csv_to_sql_table(tbl, csv, **kwargs):
    statement = copy_command(tbl.bind.dialect.name, tbl, csv)
    execute_copy(tbl.bind.dialect.name, tbl.bind, statement)
    return tbl


from core.handler.base import AdminHandler, route, \
    noblock, success, failure, authenticated, \
    permission
from core.service.serv_admin_log import admin_log_serv
from core.lib.model import AdminLog

@route(r'/admin/log/list')
class LogListHandler(AdminHandler):
    '''
    查询管理员登录日志
    :url: /admin/admin/log/list
    :methods: GET
    :param title: <str> 标题
    :param url: <str> 请求地址
    :param admin: <str> 管理员账号
    :param client_ip: <str> 客户端ip
    :param log_type: <int> 日志类型 1查询日志 2登录日志 3修改日志 4接口日志， 默认None
    :param error: <int> 是否错误 1否 0是 默认None
    :param start_time: <str> 开始时间 yyyy-mm-dd
    :param end_time: <str> 结束时间 yyyy-mm-dd，结束时间必须大于等于开始时间

    '''
    @authenticated
    @permission('adminlog:view')
    @noblock
    async def get(self):
        kwargs = {
            'title': self.get_arg('title', ''),
            'url': self.get_arg('url', ''),
            'admin': self.get_arg('admin', ''),
            'client_ip': self.get_arg('client_ip', ''),
            'log_type': self.get_int_arg('log_type', 0),
            'error': self.get_int_arg('error', -1),
            'start_time': self.get_arg('start_time', ''),
            'end_time': self.get_arg('end_time', '')
        }
        order = (-AdminLog.create_time,)
        data = admin_log_serv.find_log_by_page(
            self.get_page(), order=order, **kwargs)
        return success(data=data, url='admin/admin/log_list.htm')


# Common characters in n strings
# Given n strings, find the common characters in all the strings. In simple words, find characters that appear in all the strings and display them in alphabetical order or lexicographical order.

def CommonChars(ary):

    fnl_lst=[]

    lst=[0]*26

    for l in ary:
        tmp=set(l)
        for k in tmp:
            n=ord(k)-97
            lst[n]=lst[n]+1

    for i in range(0,len(lst)):
        if lst[i]==len(ary):
            fnl_lst.append(chr(i+97))

    fnl_lst.sort()

    return fnl_lst

def main():

    ary=['geeksforgeeks', 'gemkstones', 'acknowledges', 'aguelikes']
    print(CommonChars(ary))

    ary = ['apple', 'orange']
    print(CommonChars(ary))

if __name__=='__main__':
    main()


import random
import time

_COLOR_OF_PEGS = ['RED', 'GREEN', 'BLUE', 'CYAN', 'MAGENTA', 'YELLOW']
_NUMBER_OF_PEGS = 4

class Game():
	''' class that represents the role of the codemaker'''

	def __init__(self, id):
		''' build empty game'''
		self.history = []
		self.id = id
		self.code = [random.choice(_COLOR_OF_PEGS) for _ in range(_NUMBER_OF_PEGS)]
		self.timestamp = time.time()

	def guess(self, code):
		''' give feedback to codebreaker'''
		if len(code) != _NUMBER_OF_PEGS:
			return False, 'Incorrect number of pegs, expected {expected}, got {got}'.format(expected=_NUMBER_OF_PEGS, got=len(code))
		for index, color in enumerate(code):
			if color not in _COLOR_OF_PEGS:
				return False, 'Invalid color \'{color}\' for peg #{index}. Expected colors are: {color_list}'.format(color=color, index=index, color_list=', '.join(_COLOR_OF_PEGS))

		# colors remaining after black pegs
		filtered_code = {}
		filtered_guess = {}

		good_pos_color = 0
		for index, color in enumerate(code):
			if color == self.code[index]:
				good_pos_color += 1
			else:
				filtered_code[self.code[index]] = filtered_code.get(self.code[index], 0) + 1
				filtered_guess[color] = filtered_code.get(color, 0) + 1

		# compute the cardinal of the intersection
		good_color = 0
		for color, number in filtered_code.items():
			good_color += min(number, filtered_guess.get(color, 0))

		# save feedback to history and successfully return it
		result = {
			'timestamp': time.time(),
			'code': code,
			'good_position_and_color': good_pos_color,
			'good_color': good_color
		}
		self.history.append(result)
		return True, result

	def info(self):
		''' return the state of the game in JSON format'''
		return {
			'code': self.code,
			'history': self.history,
			'id': self.id,
			'timestamp': self.timestamp
		}


import pandas as pd
import numpy as np
from time import time
import os
import sys

from utils.helper import timeit_context

def determine_split_lens(seq_len: int, split=(0.9, 0.05, 0.05)):

    tr_pc, val_pc, ts_pc = split

    # Remove one item for label
    X_len = seq_len - 1

    # Lengths of outputs for a given user
    ts_len = int(np.floor(X_len * ts_pc))
    val_len = int(np.floor(X_len * val_pc))
    tr_len = X_len - val_len - ts_len + 1

    # Add one more item for label if any items are at all added to the output df
    ts_len += 1 if ts_len > 0 else 0
    val_len += 1 if val_len > 0 else 0

    tr_start = 0
    tr_end = tr_start + tr_len
    val_start = tr_end - 1
    val_end = val_start + val_len
    ts_start = val_end - 1 if val_len > 0 else val_end
    ts_end = ts_start + ts_len

    assert tr_len + val_len + ts_len - 1 - (val_len > 0) - (ts_len > 0) == X_len

    return (tr_len, val_len, ts_len), ((tr_start, tr_end), (val_start, val_end), (ts_start, ts_end))

def train_val_test_split_train_overlapping(df, col_names=["uid", "iid", "time"], split=[0.9, 0.05, 0.05]):

    """

    Generates three dataframes for train, val, test. The last item from train is also
    the first item in val as the item is used as a label in train but a set of
    features in val. The split is also performed in a way that items are added only to
    train until val_pc (or test_pc) would result a new full item - so in case of 20 items
    minus one label (so 19) all items go to train. But if we have 21-1 items for features
    they are split 18-1-1 (resulting df's are 19-2-2 due to a final label)

    Parameters
    ----------
    df
    split

    Returns
    -------

    """

    assert np.sum(split) == 1

    train_lengths = {}
    val_lengths = {}
    test_lengths = {}

    train_idx = {}
    val_idx = {}
    test_idx = {}

    grouped = df.groupby("uid")
    # iterate over users and determine lengths for each user (efficiency)
    for uid, items in grouped:
        lens, idx = determine_split_lens(len(items), split=split)
        train_lengths[uid], val_lengths[uid], test_lengths[uid] = lens
        train_idx[uid], val_idx[uid], test_idx[uid] = idx

    # Generate appropriate Df's
    train_size = int(sum(train_lengths.values()))
    val_size = int(sum(val_lengths.values()))
    test_size = int(sum(test_lengths.values()))

    train_np = np.empty((train_size, 3), dtype=np.int32)
    val_np = np.empty((val_size, 3), dtype=np.int32)
    test_np = np.empty((test_size, 3), dtype=np.int32)

    train_seen = 0
    val_seen = 0
    test_seen = 0

    for uid, items in grouped:

        train_start, train_end = train_idx[uid]
        val_start, val_end = val_idx[uid]
        test_start, test_end = test_idx[uid]

        train_len = train_lengths[uid]
        val_len = val_lengths[uid]
        test_len = test_lengths[uid]

        u_train_slice = items.iloc[train_start:train_end]
        u_val_slice = items.iloc[val_start:val_end]
        u_test_slice = items.iloc[test_start:test_end]

        train_np[train_seen:train_seen+train_len] = u_train_slice
        val_np[val_seen:val_seen+val_len] = u_val_slice
        test_np[test_seen:test_seen+test_len] = u_test_slice

        train_seen += train_len
        val_seen += val_len
        test_seen += test_len

    train_df = pd.DataFrame(train_np, columns=col_names)
    val_df = pd.DataFrame(val_np, columns=col_names)
    test_df = pd.DataFrame(test_np, columns=col_names)

    return train_df, val_df, test_df

def calculate_num_seqs_for_user_big_hop(num_interactions, hop_size, seq_len):

    if seq_len < hop_size:
        print("SEQ_LEN < HOP_SIZE CURRENTLY UNSUPPORTED")
        sys.exit()

    tail = num_interactions % hop_size

    num_seqs = num_interactions / hop_size

    num_seqs = np.floor(num_seqs) if tail < 2 else np.ceil(num_seqs)

    # Needed as hs:1, ni:5 should result in ns:4 not 5
    if hop_size == 1:
        num_seqs -= 1

    return int(num_seqs)

def pad_subseq(seq, predictions_per_seq):
    dim = len(seq.shape)
    padding_size = predictions_per_seq - len(seq)

    if dim == 2:
        return np.pad(seq, ((0, padding_size), (0, 0)), "constant", constant_values=0)
    elif dim == 1:
        return np.pad(seq, ((0, padding_size)), "constant", constant_values=0)
    else:
        sys.exit()

def generate_big_hop_numpy_files(df,
                                 X_file_path,
                                 Y_file_path,
                                 lens_file_path,
                                 seq_len=21,
                                 hop_size=20,
                                 features=["uid", "iid"]):

    # Number of predictions made for 1 subsequence
    predictions_per_seq = seq_len - 1

    grouped_per_user = df.groupby("uid")

    num_interactions_per_user = grouped_per_user.size()

    # Calculate number of subsequences for each user
    num_subseqs_per_user = {u: calculate_num_seqs_for_user_big_hop(count,
                                                                   hop_size,
                                                                   seq_len)
                            for u, count in num_interactions_per_user.items()}

    # Number of subseqs in epoch
    total_seqs = int(sum(num_subseqs_per_user.values()))

    # Empty placeholders
    epoch_X = np.empty((total_seqs, predictions_per_seq, len(features)), dtype=np.int32)
    epoch_Y = np.empty((total_seqs, predictions_per_seq), dtype=np.int32)

    # Unpadded lenghts
    epoch_lens = np.empty((total_seqs), dtype=np.int32)

    # Current row in epoch
    pos_in_epoch = 0

    # Iterate over users
    for uid, num_interactions in num_subseqs_per_user.items():

        if uid % 100 == 0:
            print("Processing user number: {}".format(uid))

        # Extract user sequence
        user_seq = grouped_per_user.get_group(uid)

        # Iterate over the number of subseqs by the user
        for uid_iter in np.arange(0, num_interactions):

            # Find the starting index for the subseq
            user_start = uid_iter * hop_size
            user_end = user_start + seq_len
            if len(user_seq) < user_end:
                user_end = len(user_seq)
            seq = user_seq[user_start:user_end]

            seq_X = seq[:-1][features].values
            seq_Y = seq[1:]["iid"].values

            if len(seq_X) == 0 or len(seq_Y) == 0:
                print('ok')

            epoch_lens[pos_in_epoch] = predictions_per_seq

            # Unpadded subsequence length
            actual_num_predictions = len(seq_X)

            # If sequence needs padding
            if actual_num_predictions != predictions_per_seq:

                # Amend unpadded sequence length
                epoch_lens[pos_in_epoch] = len(seq_X)
                # Apply padding with the last element in the sequence
                seq_X = pad_subseq(seq_X, predictions_per_seq)
                seq_Y = pad_subseq(seq_Y, predictions_per_seq)

            # Appropriately shaped numpy array
            seq_X = seq_X.reshape((1, predictions_per_seq, len(features)))
            seq_Y = seq_Y.reshape((1, predictions_per_seq))

            epoch_X[pos_in_epoch] = seq_X
            epoch_Y[pos_in_epoch] = seq_Y

            pos_in_epoch += 1

    np.save(X_file_path, epoch_X, allow_pickle=False)
    np.save(Y_file_path, epoch_Y, allow_pickle=False)
    np.save(lens_file_path, epoch_lens, allow_pickle=False)
    return epoch_X, epoch_Y, epoch_lens

def big_hop_numpy_arrays(df, path, train_or_test, dataset_name, seq_len=21,
                         hop_size=20, features=["uid", "iid"]):
    assert seq_len >= 2

    if train_or_test == "Mid_Epoch_Test":
        train_or_test = "Test"

    X_file_name = generate_numpy_filename(dataset_name, features, train_or_test, df, seq_len, hop_size, "X")
    X_file_path = path + X_file_name

    Y_file_name = generate_numpy_filename(dataset_name, features, train_or_test, df, seq_len, hop_size, "Y")
    Y_file_path = path + Y_file_name

    lens_file_name = generate_numpy_filename(dataset_name, features, train_or_test, df, seq_len, hop_size, "lens")
    lens_file_path = path + lens_file_name

    files_exist = os.path.isfile(X_file_path) and \
                  os.path.isfile(Y_file_path) and \
                  os.path.isfile(lens_file_path)

    t = time()
    if files_exist:
        epoch_X = np.load(X_file_path, allow_pickle=False).astype(np.float32)
        epoch_Y = np.load(Y_file_path, allow_pickle=False)
        epoch_lens = np.load(lens_file_path, allow_pickle=False)
        print("{} arrays loaded. Elapsed {} seconds".format(train_or_test, round(time() - t, 2)))

    else:

        epoch_X, epoch_Y, epoch_lens = generate_big_hop_numpy_files(df=df,
                                                                    X_file_path=X_file_path,
                                                                    Y_file_path=Y_file_path,
                                                                    lens_file_path=lens_file_path,
                                                                    seq_len=seq_len,
                                                                    hop_size=hop_size,
                                                                    features=features)
        epoch_X = epoch_X.astype(np.float32)
        print("{} arrays generated and saved. Elapsed {} seconds".format(train_or_test, round(time() - t, 2)))

    epoch_user_ids = epoch_X[:,0,0]
    return epoch_X, epoch_Y, epoch_lens, epoch_user_ids

def big_hop_generator(df,
                      path,
                      train_or_test,
                      dataset_name,
                      seq_len=21,
                      batch_size=1000,
                      hop_size = 20,
                      features=["uid", "iid"],
                      min_seq_len=None,
                      num_samples=None,
                      randomize=False):
    """
    Generator with ability to save numpy arrays for x, y, lens
    Designed for hop size 20 and seq_len 21 (20 features + 20 predictions)
    Outputs bs x timesteps (padded) x 2 for both X and Y

    Parameters
    ----------
    df
    path
    train_or_test
    dataset_name
    seq_len
    batch_size
    hop_size
    features
    min_seq_len
    num_samples
    randomize

    Returns
    -------

    """

    assert seq_len >= 2

    if train_or_test == "Mid_Epoch_Test":
        train_or_test = "Test"

    X_file_name = generate_numpy_filename(dataset_name, train_or_test, df, seq_len, hop_size, "X")
    X_file_path = path + X_file_name

    Y_file_name = generate_numpy_filename(dataset_name, train_or_test, df, seq_len, hop_size, "Y")
    Y_file_path = path + Y_file_name

    lens_file_name = generate_numpy_filename(dataset_name, train_or_test, df, seq_len, hop_size, "lens")
    lens_file_path = path + lens_file_name
    #
    # try:
    #     os.remove(X_file_path)
    # except FileNotFoundError as e:
    #     pass

    files_exist = os.path.isfile(X_file_path) and \
                  os.path.isfile(Y_file_path) and \
                  os.path.isfile(lens_file_path)

    t = time()
    if files_exist:
        epoch_X = np.load(X_file_path, allow_pickle=False)
        epoch_Y = np.load(Y_file_path, allow_pickle=False)
        epoch_lens = np.load(lens_file_path, allow_pickle=False)
        print("{} arrays loaded. Elapsed {} seconds".format(train_or_test, round(time() - t, 2)))

    else:

        epoch_X, epoch_Y, epoch_lens = generate_big_hop_numpy_files(df=df,
                                                                    X_file_path=X_file_path,
                                                                    Y_file_path=Y_file_path,
                                                                    lens_file_path=lens_file_path,
                                                                    seq_len=seq_len,
                                                                    hop_size=hop_size,
                                                                    features=features)
        print("{} arrays generated and saved. Elapsed {} seconds".format(train_or_test, round(time() - t, 2)))

    num_seqs = len(epoch_X)
    seq_order = np.fromiter(range(num_seqs), dtype=np.int32)

    if randomize:
        np.random.shuffle(seq_order)

    no_batches = int(np.ceil(num_seqs / batch_size))
    batch_start = 0
    for batch_id in range(no_batches):
        with timeit_context("Generation"):
            batch_end = batch_start + batch_size
            if batch_id == no_batches - 1:
                batch_end = num_seqs
            seq_ids = seq_order[batch_start:batch_end]
            batch_X = epoch_X[seq_ids]
            batch_Y = epoch_Y[seq_ids]
            batch_lens = epoch_lens[seq_ids]
            batch_start += batch_size
        user_ids = batch_X[:,0,0]
        yield batch_id, batch_X, batch_Y, batch_lens, user_ids

###
# Input: Pandas DataFrame with cols "uid", "iid", "time", percentages for data split e.g. [0.9, 0.05, 0.05]
# Return: 3 Dataframes: one for train, one for validation and one for testing

#### ASSUMES THE SPLITTING STRATEGY IS SIMPLE - TRAIN VAL TEST follow each other without gaps

def train_val_test_split(df, split=[0.9, 0.05, 0.05]):

    # Ensure percentages add up to 1
    assert np.sum(split) == 1

    # Output dataframes
    train_df = pd.DataFrame(columns=["uid", "iid", "time"])
    val_df = pd.DataFrame(columns=["uid", "iid", "time"])
    test_df = pd.DataFrame(columns=["uid", "iid", "time"])

    # Get a list of users
    users = df["uid"].unique()

    # Group users by uid
    grouped = df.groupby("uid")

    # Iterate over users
    for user in users:

        # Get all items for one user
        items = grouped.get_group(user)

        # Get item count
        count = len(items)

        # Get starts and splits
        train_indices = range(0, int(np.floor(split[0]*count)))
        val_indices = range(int(np.floor(split[0]*count)), int(np.floor(split[0]*count)) + int(np.floor(split[1]*count)))
        test_indices = range(int(np.floor(split[0]*count)) + int(np.floor(split[1]*count)), count)

        # Make temporary DataFrames with slices selected by indices
        train_items = items.iloc[train_indices]
        val_items = items.iloc[val_indices]
        test_items = items.iloc[test_indices]

        # Add these slices to the output
        train_df = pd.concat([train_df, train_items])
        val_df = pd.concat([val_df, val_items])
        test_df = pd.concat([test_df, test_items])

    return([train_df, val_df, test_df])

def get_number_seqs(ints, seq_len, min_seq_len=10):

    """

    Parameters
    ----------
    ints: int, number of interactions of a given user
    seq_len: int, how many interactions should be at most in a sequence
    min_seq_len: int, discard any remaining tails if they are below a given length

    Returns
    -------
    int, number of subsequences that we get from one user sequence
    """

    # How many sequences of len seq_len OR sequences shorter than seq_len there are
    seqs_int = np.ceil(ints / seq_len)

    # The length of the last seq (or 0 if all batches of len seq_len)
    short_last_batch = ints % seq_len

    # If the last seq too short, don't include the starting index for that seq
    if short_last_batch < min_seq_len and short_last_batch != 0:
        seqs_int -= 1

    return seqs_int

def sliding_window_random_sample_generator(df,
                                           seq_len=20,
                                           batch_size=1000,
                                           num_samples=1,
                                           hop_size=1,
                                           features=["uid", "iid"],
                                           min_seq_len=None,
                                           randomize=False,
                                           path=None,
                                           name=None):

    """

    Similar in function to the sliding window generator but only produces
    num_samples batches. Batches are generated by selecting a random user
    and then selecting a random starting point in their sequence. Sequences
    are added into the batch using a sliding window with the hop size
    hop_size. When a batch is still not of size batch_size but the user has
    run out of subsequences another random user is chosen. During any user
    selection if a user has fewer than 20 interactions then a new user is
    selected. The same user can be rolled multiple times and thus in theory
    there may be overlap between subsequences in one batch. The approach
    also first selects the user and thus users with longer histories are
    not preferred over shorter users.

    Parameters
    ----------
    df
    seq_len
    batch_size
    num_samples
    hop_size
    features
    min_seq_len

    Returns
    -------

    """

    seq_lens = np.array([seq_len-1]*batch_size)
    grouped_per_user = df.groupby("uid")
    user_ids = np.fromiter(grouped_per_user.groups.keys(), np.int32)
    for sample_number in range(num_samples):
        yielded = False
        batch_X, batch_Y = np.empty((0, seq_len - 1, 2)), np.empty((0))
        while len(batch_X) < batch_size:
            user_selected = False

            # Handle test users that have fewer than seq_len interactions in total
            while not user_selected:
                uid = np.random.choice(np.array(user_ids), 1)[0]
                user_seq = grouped_per_user.get_group(uid)

                # Number of interactions the user has had in this DataFrame
                user_len = len(user_seq)
                if user_len >= seq_len:
                    user_selected = True

            user_start = np.random.randint(0, user_len - seq_len + 1, 1)[0]
            while user_start + seq_len <= user_len:
                # Extract the seq_len x features DataFrame
                seq = user_seq[user_start:user_start + seq_len]

                # 1 x timesteps - 1 x num_features
                seq_X = seq[:-1][features].values.reshape((1, seq_len - 1, len(features)))
                # 1
                seq_Y = seq.iloc[-1]["iid"]

                # growing until batch_size x timesteps x num_features
                batch_X = np.append(batch_X, seq_X, 0)
                # growing until batch_size
                batch_Y = np.append(batch_Y, seq_Y)
                # Move the sliding window
                user_start += hop_size

                # If batch has batch_size sequences yield it and reset the DataFrame
                if len(batch_X) == batch_size:
                    yield batch_X, batch_Y, seq_lens
                    yielded = True
                # Break out of both loops and start a new batch
                if yielded:
                    break

            if yielded:
                break

def calculate_num_seqs_for_user_sliding_window(num_interactions: int, seq_len: int):
    return num_interactions - seq_len + 1

def generate_sliding_window_sequence_starts(num_seqs: int, uid: int):
    return [(uid, x) for x in np.arange(num_seqs)]

def sliding_window_generator_with_sequence_mixing(df,
                                                  seq_len=20,
                                                  batch_size=1000,
                                                  hop_size = 1,
                                                  features=["uid", "iid"],
                                                  min_seq_len=None,
                                                  num_samples=None,
                                                  randomize=False,
                                                  path=None,
                                                  name=None):
    # Need at least 1 value for X and 1 value for y
    assert seq_len >= 2

    # Return the original lengths of each of the seqs
    # Required as other generators may produce padded sequences
    # some sequences may be shorter than seq_len
    seq_lens = np.array([seq_len-1]*batch_size)

    grouped_per_user = df.groupby("uid")

    # Number of interactions for each user in the DataFrame
    num_interactions_per_user = grouped_per_user.size()

    # [uid, int] - int is the number of subsequences of len seq_len that can be produced for each user
    seqs_per_user = [(u, calculate_num_seqs_for_user_sliding_window(
        num_interactions_per_user[u], seq_len)) for u in num_interactions_per_user.keys()
                     ]
    # [(uid, seq_id)...] where seq_id is every value between 0 and num_seqs
    permutations = []
    for uid, num_seqs in seqs_per_user:
        permutations.extend(generate_sliding_window_sequence_starts(num_seqs, uid))

    # Shuffle the order
    if randomize:
        np.random.shuffle(permutations)

    # Number of batches
    no_batches = np.ceil(len(permutations) / batch_size)

    # Get the size of the final batch
    final_batch_size = len(permutations) % batch_size

    # Number of subsequences yielded so far
    permutations_start = 0

    start_times = []
    end_times = []

    for batch_id in np.arange(no_batches):

        start_times.append(time())

        # Generally batch_size except for the last batch
        current_batch_size = batch_size if batch_id + 1 < no_batches else final_batch_size

        # Initialize full-sized empty placeholder for batch outputs
        batch_X, batch_Y = np.empty((current_batch_size, seq_len-1, 2)), np.empty((current_batch_size))

        # Range for the (shuffled) subsequences
        permutations_end = permutations_start + current_batch_size

        # Select subsequences to add to batch
        batch_permutations = permutations[permutations_start:permutations_end]

        for seq_id in range(len(batch_permutations)):

            uid, user_start = batch_permutations[seq_id]

            user_end = user_start + seq_len

            # DataFrame with only 1 user's interactions
            user_seq = grouped_per_user.get_group(uid)

            # Extract relevant subsequence
            seq = user_seq[user_start:user_end]

            # 1 x timesteps - 1 x num_features; 1
            seq_X = seq[:-1][features].values.reshape((1, seq_len - 1, len(features)))
            seq_Y = seq.iloc[-1]["iid"]

            batch_X[seq_id] = seq_X
            batch_Y[seq_id] = seq_Y

        permutations_start += current_batch_size
        end_times.append(time())
        print("Generation time: {} seconds".format(round(end_times[-1] - start_times[-1], 2)))
        yield batch_X, batch_Y, seq_lens[:current_batch_size]

def generate_numpy_filename(dataset_name, features, train_or_test, df, seq_len, hop_size, type):
    # return "/" + str(len(df)) + "_" + name + "_" + str(seq_len) + "_" + str(hop_size) + "_" + type + ".npy"
    return "/" + dataset_name + "_" + \
           "_".join(features) + "_" + \
           str(len(df)) + "_" + \
           train_or_test + "_" + \
           str(seq_len) + "_" + \
           str(hop_size) + "_" +\
           type + ".npy"

def generate_numpy_files(df,
                         X_file_path,
                         Y_file_path,
                         seq_len=20,
                         hop_size=1,
                         features=["uid", "iid"]):
    grouped_per_user = df.groupby("uid")

    user_ids = np.fromiter(grouped_per_user.groups.keys(), np.int32)

    num_interactions_per_user = grouped_per_user.size()

    num_interactions_per_user = {u: calculate_num_seqs_for_user_sliding_window(count,seq_len) for u, count in num_interactions_per_user.items() if count >= seq_len}

    total_seqs = sum(num_interactions_per_user.values())

    epoch_X = np.empty((total_seqs, seq_len - 1, len(features)), dtype=np.int32)
    epoch_Y = np.empty((total_seqs), dtype=np.int32)

    pos_in_epoch = 0

    for uid in num_interactions_per_user.keys():

        print(uid)
        user_seq = grouped_per_user.get_group(uid)

        user_len = num_interactions_per_user[uid]

        for user_start in np.arange(user_len, step=hop_size):
            seq = user_seq[user_start:user_start + seq_len]

            # 1 x timesteps - 1 x num_features
            seq_X = seq[:-1][features].values.reshape((1, seq_len - 1, len(features)))
            # 1
            seq_Y = seq.iloc[-1]["iid"]

            epoch_X[pos_in_epoch] = seq_X
            epoch_Y[pos_in_epoch] = seq_Y
            pos_in_epoch += 1

    np.save(X_file_path, epoch_X, allow_pickle=False)
    np.save(Y_file_path, epoch_Y, allow_pickle=False)
    return epoch_X, epoch_Y

def window_gen_w_save_and_seq_mix(df,
                                  path,
                                  train_or_test,
                                  dataset_name,
                                  seq_len=20,
                                  batch_size=1000,
                                  hop_size = 1,
                                  features=["uid", "iid"],
                                  min_seq_len=None,
                                  num_samples=None,
                                  randomize=False):
    assert seq_len >= 2

    if train_or_test == "Mid_Epoch_Test":
        train_or_test = "Test"

    X_file_name = generate_numpy_filename(dataset_name, features, train_or_test, df, seq_len, hop_size, "X")
    X_file_path = path + X_file_name

    Y_file_name = generate_numpy_filename(dataset_name, features, train_or_test, df, seq_len, hop_size, "Y")
    Y_file_path = path + Y_file_name

    files_exist = os.path.isfile(X_file_path) and os.path.isfile(Y_file_path)

    t = time()
    if files_exist:
        epoch_X = np.load(X_file_path, allow_pickle=False)
        epoch_Y = np.load(Y_file_path, allow_pickle=False)
        print("{} arrays loaded. Elapsed {} seconds".format(train_or_test, round(time() - t, 2)))

    else:
        epoch_X, epoch_Y = generate_numpy_files(df=df,
                                                X_file_path=X_file_path,
                                                Y_file_path=Y_file_path,
                                                seq_len=seq_len,
                                                hop_size=hop_size,
                                                features=features)
        print("{} arrays generated and saved. Elapsed {} seconds".format(train_or_test, round(time() - t, 2)))

    seq_lens = np.array([seq_len-1]*batch_size)

    num_seqs = len(epoch_X)
    seq_order = np.fromiter(range(num_seqs), dtype=np.int32)

    if randomize:
        np.random.shuffle(seq_order)

    no_batches = int(np.ceil(num_seqs / batch_size))
    batch_start = 0
    start_times = []
    end_times = []
    for batch_id in range(no_batches):
        start_times.append(time())
        batch_end = batch_start + batch_size
        if batch_id == no_batches - 1:
            batch_end = num_seqs
            seq_lens = seq_lens[:batch_end-batch_start]
        seq_ids = seq_order[batch_start:batch_end]
        batch_X = epoch_X[seq_ids]
        batch_Y = epoch_Y[seq_ids]
        end_times.append(time())
        print("Generation time: {} seconds".format(round(end_times[-1] - start_times[-1], 2)))
        batch_start += batch_size
        yield batch_X, batch_Y, seq_lens

def sliding_window_generator(df,
                             seq_len=20,
                             batch_size=1000,
                             hop_size = 1,
                             features=["uid", "iid"],
                             min_seq_len=None,
                             num_samples=None,
                             randomize=False,
                             path=None,
                             name=None):

    """
    Takes in a DataFrame with columns "uid" and other specified by features
    Yields an appropriate X, y tensors as well as tensor with lengths of each
    of the sequences in the batch. Sequences are added to a batch in sliding
    window fashion - window sliding depends on hop_size. Each sequence appears
    only in one batch. When we run of a particular user's sequences before the
    end of the batch current batch is continued with the next user until the
    total batch size adds up to batch_size or in case of last user until
    their last sequence.

    Parameters
    ----------
    df: pandas.DataFrame
    seq_len
    batch_size
    hop_size
    features
    min_seq_len: placeholder to unify both generators

    Returns
    -------
    tf.Tensor, a tensor of shape batch_size* x seq_len - 1 x num_features representing
               the measurements on the basis of which predictions are made.
    tf.Tensor, a tensor of shape batch_size* representing the target indices
    tf.Tensor, a tensor of shape batch_size* representing the length of each of
               subsequences

    * batch_size or between 1 and batch_size for the last batch
    """

    # Need at least 1 value for X and 1 value for y
    assert seq_len >= 2

    # Return the original lengths of each of the seqs
    # Required as other generators may produce padded sequences
    # some sequences may be shorter than seq_len
    seq_lens = np.array([seq_len-1]*batch_size)

    grouped_per_user = df.groupby("uid")

    # X: batch_size x timesteps x (iid+uid=2)
    # Y: batch_size
    batch_X, batch_Y = np.empty((0,seq_len-1,2)), np.empty((0))

    start_times = [time()]
    end_times = []

    times_yielded = 0

    user_ids = np.fromiter(grouped_per_user.groups.keys(), np.int32)

    if randomize:
        np.random.shuffle(user_ids)

    # Iterate over all users
    for uid in user_ids:

        # DataFrame of this user's interactions
        user_seq = grouped_per_user.get_group(uid)
        # Number of interactions the user has had in this DataFrame
        user_len = len(user_seq)
        # How many of user's sequences have already been processed
        user_start = 0

        # Iterate until user cannot produce any more sequence of length seq_len
        # Keep filling the batch until it's of size batch_size
        while user_start + seq_len <= user_len:

            # Extract the seq_len x features DataFrame
            seq = user_seq[user_start:user_start+seq_len]

            # 1 x timesteps - 1 x num_features
            seq_X = seq[:-1][features].values.reshape((1, seq_len-1, len(features)))
            # 1
            seq_Y = seq.iloc[-1]["iid"]

            # growing until batch_size x timesteps x num_features
            batch_X = np.append(batch_X, seq_X, 0)
            # growing until batch_size
            batch_Y = np.append(batch_Y, seq_Y)
            # Move the sliding window
            user_start += hop_size

            # If batch has batch_size sequences yield it and reset the DataFrame
            if len(batch_X) == batch_size:
                end_times.append(time())
                print("Generation time: {} seconds".format(round(end_times[-1] - start_times[-1], 2)))
                yield batch_X, batch_Y, seq_lens
                times_yielded += 1
                start_times.append(time())
                batch_X, batch_Y = np.empty((0, seq_len-1,2)), np.empty((0))

    # If reached the last user and the final batch didn't add up to
    # batch_size - yield the final batch and sequence_length tensor
    # of appropriate length
    if len(batch_X) > 0:
        yield batch_X, batch_Y, seq_lens[:len(batch_X)]

def time_insensitive_batch_generator(df,
                                     seq_len=20,
                                     batch_size=1000,
                                     min_seq_len=10,
                                     hop_size=1,
                                     features = ["uid", "iid"],
                                     num_samples=None,
                                     randomize=False):

    """

    Parameters
    ----------
    df: pandas.DataFrame, columns ["uid", "iid", "time"]
    seq_len: int, how many interactions should be at most in a sequence
    batch_size: int: how many sequences in a batch
    min_seq_len: int, discard any remaining subsequences if they are below a given length
    hop_size: int, NOT USED - only to unify inputs for generators

    Yields
    -------
    List([seq_len - 1, 3])
    List([seq_len - 1, None])

    """

    # Dictionary k="uid", v=number_of_interactions_by_uid
    grouped_per_user = df.groupby("uid")
    ints_per_batch_per_user = dict(grouped_per_user.size())

    # Dictionary k="uid", v=number_of_seqs_of_len_batch_size_by_uid
    seqs_per_batch_per_user = dict([(x, get_number_seqs(ints_per_batch_per_user[x], seq_len, min_seq_len))
                                    for x in ints_per_batch_per_user.keys()])

    # Make indices for every uid for every seq_id
    permutations = [(k, x) for k in seqs_per_batch_per_user.keys()
                    for x in np.arange(seqs_per_batch_per_user[k])]
    # Shuffle the order
    if randomize:
        np.random.shuffle(permutations)

    # Number of batches
    no_batches = np.ceil(len(permutations) / batch_size)

    # Get the size of the final batch
    final_batch_size = len(permutations) % batch_size

    # Make and yield batches of X and y
    for batch_id in np.arange(no_batches):

        # Empty batch
        batch_X, batch_y = [], []

        # If last batch then get the remaining seqs < batch_size
        batch_limit = (batch_id + 1) * batch_size
        if batch_id == no_batches - 1:
            batch_limit = batch_id * batch_size + final_batch_size

        # Get the indices of sequences in the batch (tuple of uid and which seq of len seq_len it is)
        batch_seq_idx = np.arange(batch_id * batch_size, batch_limit)
        for uid, seq_id in [permutations[int(x)] for x in batch_seq_idx]:

            # Interactions of one user
            u_ints = grouped_per_user.get_group(uid)

            # Subsequence from all user's interactions
            seq_start = int(seq_id * seq_len)
            seq_end = seq_start + seq_len

            # If the last few interactions of the user
            if len(u_ints) < seq_end:
                seq_end = len(u_ints)
            seq = u_ints[seq_start:seq_end]

            # Get first 19 interactions
            X = seq[:-1]

            # Get 2-20th items
            y = seq[1:]["iid"]

            # Add to the batch list
            batch_X.append(X)
            batch_y.append(y)

        batch_X, seq_lens = batchX_to_tensor(batch_X, timesteps=seq_len - 1, features=features)
        batch_y = batchY_to_tensor(batch_y, get_all=True, timesteps=seq_len - 1)[:,-1]
        yield batch_X, batch_y, seq_lens

def seq_to_tensor(seq, features = ["uid", "iid"]):

    """

    Helper method, called by batchX_to_tensor.
    Transforms one pd.Dataframe into a tf.Tensor (numpy array) of timesteps x features

    Parameters
    ----------
    seq: pd.DataFrame with colums "uid", "iid" and "time"
    features: list, columns to return in the tensor. The tensor's third dimension
                    is of size len(features)

    Returns
    -------
    np.array, dimensionality timesteps x features

    """
    selected_cols = seq[features]
    tensor = selected_cols.as_matrix()
    return tensor

def batchX_to_tensor(batch: list, features = ["uid", "iid"], timesteps = None):

    """

    Converts a list of DF's representing the batch in the output of
    time_insensitive_batch_generator into a batch_size x timesteps
    x features tf.Tensor (numpy array)
    Additionally returns the lengths of the unpadded part of each
    sequence in numpy array of length = batch_size (seq_padding)
    Seq_padding is an empty array if number of timesteps not given

    Parameters
    ----------
    batch: list, contains num_batches DataFrames of length timesteps (minus one)
                 every DF contains "uid", "iid" and "time"
    features: list, columns to return in the tensor. The tensor's third dimension
                    is of size len(features)
    timesteps: int or None, if value provided then all sequences in the batch are
                            padded with their last known values (last u-i pair)

    Returns
    -------
    np.array, dimensionality batch_size x max_timesteps/timesteps x features
    np.array, dimensionality batch_size x None

    """
    output_tensor = []
    seq_padding = []

    for seq in batch:
        output_tensor.append(seq_to_tensor(seq, features))

    # If max seq len provided then pad the sequence with its last value until
    # the sequence is of length = timesteps
    # Used for padding variable sequence sizes
    if timesteps:
        for s in range(len(output_tensor)):
            seq = output_tensor[s]
            if len(seq) < timesteps:
                padding_size = timesteps - len(seq)
                output_tensor[s] = np.pad(seq, ((0, padding_size), (0, 0)), "edge")
            seq_padding.append(len(seq))

    output_tensor = np.array(output_tensor)
    seq_padding = np.array(seq_padding)
    return output_tensor, seq_padding

def mask_all_but_one(padding, timesteps):
    batch_mask = np.empty([len(padding), timesteps], dtype=np.float32)

    for x in range(len(padding)):
        zeros = np.zeros([timesteps], dtype=np.float32)
        zeros[padding[x] - 1] = 1.
        batch_mask[x] = zeros
    return batch_mask

def batchY_to_tensor(batch: list, get_all = False, timesteps = None):

    """

    Transform list of DF's into the last item at the end of the sequence
    joined together into a batch_size x None tensor (np.array)
    If get_all is True then all the items are returned per sequence
    and the output is then of shape batch_size x timesteps (minus one)

    Parameters
    ----------
    batch: list, contains DFs with col "iid" of len timesteps (minus one)
    get_all: bool, If True - the output array contains labels for each next
                   item for every timestep. If False, only the last timesteps
                   next item is returned.
    Returns
    -------
    np.array, labels of the item purchased after the last timestep for each
              sequence in the batch if get_all is False (batch_size x None)
              and all items purchased after every timestep if get_all
              is True (batch_size x timesteps)
    """
    if not get_all:
        return np.array([x.iloc[-1] for x in batch])
    if timesteps:
        for s in range(len(batch)):
            seq = batch[s]
            if len(seq) < timesteps:
                padding_size = timesteps - len(seq)
                # batch[s] = np.reshape(np.pad(np.reshape(seq, (len(seq), 1)), \
                #                              ((0, padding_size), (0, 0)), \
                #                              "edge"), (timesteps,))
                batch[s] = np.reshape(np.pad(seq.values.reshape((len(seq), 1)),((0, padding_size), (0, 0)),"edge"),(timesteps,))
    return np.array(batch)

def remove_unseen_items_in_train(train_df, test_df):

    '''

    Parameters
    ----------
    train_df: pd.DataFrame, training set DataFrame with cols ["uid", "iid", "time"]
    test_df:  pd.DataFrame, test set DataFrame with cols ["uid", "iid", "time"]

    Returns
    -------
    pd.DataFrame, test set DataFrame with cols ["uid", "iid", "time"] - all
                  items present in test are present at least once in train

    '''

    # Find unique items in train and test
    tr_items = set(train_df["iid"])
    ts_items = set(test_df["iid"])

    # Find iid's in test that are also present in train
    ts_items.intersection_update(tr_items)

    # Retain only test's columns that are present in train
    short_test_df = test_df[test_df["iid"].isin(ts_items)]

    return short_test_df

if __name__ == "__main__":
    input_filename = "/Users/nknyazev/Documents/Delft/Thesis/temporal/data/processed/lastfm_10_pc_per_user_incr.csv"

    df = pd.read_csv(input_filename, header=None, names=["uid", "iid", "time"])
    tr, val, ts = train_val_test_split_train_overlapping(df)
    g = big_hop_generator(df=ts,
                          path="/Users/nknyazev/Documents/Delft/Thesis/temporal/data/numpy",
                          train_or_test="Test",
                          dataset_name="lastfm_10_pc",
                          randomize=False
                          )
    for a,b,c in g:
        print("ok")
        break


#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Object Oriented Python to Technical Acoustics example file

Created on Fri Aug 31 11:56:54 2018
Last modified on Sun Oct 28 09:56:32 2018

@author: Matheus Lazarin Albert, João Vitor Gutkoski Paes

This file shows a few ways to use the PyTTa package and it''s modules
Consider running this file section by section, without the runfile() method

"""
#%%
import pytta # importing Object Oriented Python in Technical Acoustics

#%% Default properties and device selecting (not to be run inside a script)

## To run this example file using runfile(), consider commenting this section.
#pytta.properties.default # show all the defaul properties and its values
#pytta.properties.list_devices() # list available audio in/out devices and identify the default ones
#
## change the device and sampling rate values within the "default" dict
#pytta.properties.set_default(device = [1,2], samplingRate=51200)

#%% generate excitation signal
x1 = pytta.generate.sweep() # with default settings
x2 = pytta.generate.sweep(50,16000) # chosing bandwidth
x3 = pytta.generate.sweep(fftDeg = 16,
                          startmargin=0.1,
                          stopmargin=0.9) # chosing some parameters, without ordering
x4 = pytta.generate.sweep(100,10000,48000,17,0.5,0.5) # choosing all parameters, ordered input

#%% creating the frfmeasure object
x = x1 # define signalObj for the measurement example
recTime = pytta.generate.measurement('rec','time',10)
recDeg = pytta.generate.measurement('rec','samples',17)
m1 = pytta.generate.measurement('frf',x) #initializing with selected signalobj
m2 = pytta.generate.measurement('frf') #initializing with default settings, empty excitation
m2.excitation = x #declaring excitation origin, or
# m2.excitation.t = x.t[:] # declaring excitation value
#m3 = pytta.FRFMeasure(x,
#                      device = 'audiobox',
#                      outch = [3,4],
#                      inch = [1,2] ) # initializing with different properties

#%% running the measurement
m = m1 # define measurement object for run example
y = m.run()

#%% plot stuff
y.plot_freq()
y.plot_time()