import logging
import itertools

import numpy as np
from scipy.optimize import OptimizeResult, minimize_scalar
import scipy.constants

from .util import find_vertex_x_of_positive_parabola

def scalar_discrete_gap_filling_minimizer(
        fun, bracket, args=(), tol=1.0, maxfev=None, maxiter=100, callback=None, verbose=False,
        parabolic_method=False, golden_section_method=False, best_x_aggregator=None, **options):
    """Find a local minimum of a scalar function of a single integer variable.
    The domain of the function is all integers between, and including, the bracket.
    The function may have flat spots where f(a) == f(b) for a != b and this method will
    attempt to search around and within the flat spots.
    The function must have exactly one local minimum in the bracket.
    This method maintains a left and right bracket, where the function value is greater than the best known minimum.
    It also maintains a list of best x values, and the function values at all of these x values equals the best known
    minimum.
    At each iteration, it finds the largest gap in these x values (including the brackets) and selects
    the point in the center of the largest gap.
    It will then either adjust the bracket or add to the list of best x values.
    The method terminates when the largest gap is less than or equal to tol.

    Parameters
    ----------
    bracket : array_like
        A tuple of the bounds of the function (x_min, x_max).
        Optionally, a 3-tuple can be specified and the middle point will be the initial best point.
    tol : float
        The method terminates when the largest gap is less than or equal to this value.

    Returns
    -------
    OptimizeResult
        The result of the minimization.

    """
    # bestx is a list.
    # besty is a scalar and equals f(x) for all x in bestx.

    funcalls = 0

    # print('parabolic_method=%s,golden_section_method=%s' % (parabolic_method,golden_section_method))

    if len(bracket) == 2:
        bracket_left_x = bracket[0]
        bracket_right_x = bracket[1]
        bestx = [np.round(np.mean([bracket_left_x, bracket_right_x]))]
        a = bracket_left_x
        b = bracket_right_x
        if golden_section_method:
            bestx = [np.round(b - (b - a) / scipy.constants.golden)]
        else:
            bestx = [np.round(np.mean([a, b]))]
    elif len(bracket) == 3:
        bracket_left_x = bracket[0]
        bracket_right_x = bracket[2]
        bestx = [bracket[1]]
    else:
        raise ValueError('Invalid bracket')

    assert isinstance(bestx, list)

    if not (bracket_left_x <= bestx[0] <= bracket_right_x):
        raise ValueError('Invalid bracket')

    if best_x_aggregator is None:
        best_x_aggregator = lambda x: x[int((len(x)-1)/2)]

    # Evaluate function at bestx.
    besty = fun(bestx[0])
    funcalls += 1
    assert np.isscalar(besty)

    # Evaluate function at brackets to determine if they are better than the initial bestx.
    bracket_left_y = fun(bracket_left_x, *args)
    bracket_right_y = fun(bracket_right_x, *args)
    funcalls += 2
    if bracket_left_y < besty:
        bestx = [bracket_left_x]
        besty = bracket_left_y
    if bracket_right_y < besty:
        bestx = [bracket_right_x]
        besty = bracket_right_y

    if verbose: logging.info('bracket=(%f,%s,%f); besty=%f' % (bracket_left_x, str(bestx), bracket_right_x, besty))

    niter = 0
    while niter < maxiter:
        niter += 1

        X = np.array([bracket_left_x] +  bestx               + [bracket_right_x])
        Y = np.array([bracket_left_y] + [besty] * len(bestx) + [bracket_right_y])

        # if verbose:
        #     logging.info('X=%s' % str(X))
        #     logging.info('Y=%s' % str(Y))

        testx = None
        testx_index = None

        #
        #  Step 1: Determine the value of x to test next (testx).
        #

        # If we have exactly one bestx, then fit a parabola to the 3 points and test the vertex.
        if parabolic_method and len(bestx) == 1:
            if verbose: logging.info('Attempting parabolic method')
            try:
                # Attempt to fit a parabola to the 3 points and find the vertex.
                testx = find_vertex_x_of_positive_parabola(X, Y)
                if verbose: logging.info('Parabolic method returned testx=%f' % testx)
                testx = np.round(testx)
                if testx <= bracket_left_x or testx >= bracket_right_x or testx == bestx[0]:
                    testx = None
                elif testx <= bestx[0]:
                    testx_index = 0
                else:
                    testx_index = 1
            except:
                # This will happen if a parabola can't be fit through the 3 points.
                # Ignore error and use the gap method below.
                testx = None
        if testx is None:
            # Measure gaps in brackets and bestx and find the largest one.
            if verbose: logging.info('Attempting gap method')
            gaps = np.diff(X)
            testx_index = np.argmax(gaps)
            gapsize = gaps[testx_index]
            if gapsize <= tol:
                if verbose: logging.info('Achieved gap size tol')
                break

            # Pick a point between the largest gap.
            a = X[testx_index]
            b = X[testx_index + 1]
            if golden_section_method:
                golden_distance = (b - a) / scipy.constants.golden
                if bool(np.random.randint(low=0, high=2)):
                    testx = np.round(b - golden_distance)
                else:
                    testx = np.round(a + golden_distance)
            else:
                testx = np.round(np.mean([a, b]))

            if verbose: logging.info('gapsize=%f, len(bestx)=%d, testx=%f' % (gapsize, len(bestx), testx))

        assert(testx is not None)
        assert(testx_index is not None)
        assert(bracket_left_x <= testx <= bracket_right_x)

        #
        #  Step 2: Evaluate function at testx.
        #

        testy = fun(testx, *args)
        funcalls += 1

        #
        #  Step 3: Update bracket, etc. based on function value testy at testx.
        #

        add_to_bestx = False

        if testy < besty:
            # Found a point better than all others so far.
            # The new bracket will be the points to the immediate left and right of the test point.
            bestx = [testx]
            besty = testy
            bracket_left_x = X[testx_index]
            bracket_left_y = Y[testx_index]
            bracket_right_x = X[testx_index + 1]
            bracket_right_y = Y[testx_index + 1]
        elif testy > besty:
            # Point is worse than best. Reduce bracket.
            if testx_index == 0:
                # Test point was adjacent to left bracket.
                bracket_left_x = testx
                bracket_left_y = testy
            elif testx_index == len(X) - 2:
                # Test point was adjacent to right bracket.
                bracket_right_x = testx
                bracket_right_y = testy
            else:
                # Test point was inside the set of bestx points but is worse than besty.
                # This indicates more than one local minima or a round off error.
                # We will assume a round off error and handle it as if it had the same besty.
                add_to_bestx = True
        else:
            # Point is same as best. Add it to the bestx list.
            add_to_bestx = True

        if add_to_bestx:
            bestx = sorted(bestx + [testx])

        if verbose: logging.info('bracket=(%f,%s,%f); besty=%f' % (bracket_left_x, str(bestx), bracket_right_x, besty))
        if callback is not None:
            callback(bestx)
        if maxfev is not None and funcalls >= maxfev:
            break

    # Return the x that is in the median of bestx.
    bestx = best_x_aggregator(np.array(bestx))

    return OptimizeResult(fun=besty, x=bestx, nit=niter, nfev=funcalls, success=(niter > 1))

def multivariate_discrete_gap_filling_minimizer(
        fun, x0, bounds, args=(), tol=1.0, maxfev=None, maxiter=2, callback=None, verbose=False,
        scalar_options={}, axes=None, **options):
    """It is assumed that there is exactly one local minimum in the domain.
    For each dimension, the domain of the function consists of all integers between, and including, the bounds.
    The function may have flat spots where f(a) == f(b) for a != b and this method will
    attempt to search around and within the flat spots.
    This multivariate method uses `scalar_gap_filling_minimizer` repeatedly along each dimension
    for a fixed number of iterations. There is currently no other stopping criteria.

    Parameters
    ----------
    fun: Function of a single variable of a list-type.
    x0 : array_like
        Initial guess.
    bounds
        List-type of (min, max) pairs for each element in x, defining the bounds in that dimension.
    tol
        See `scalar_discrete_gap_filling_minimizer`.
    axes : array_like
        Number of columns must equal length of x0.
        The rows will determine the set of axes that this function will optimize along.
        Leave as None to use unit axes along each dimension.

    Returns
    -------
    OptimizeResult
        The result of the minimization.

    """

    ndims = len(x0)

    bounds = np.array(bounds)
    if bounds.shape != (ndims, 2):
        raise ValueError()

    if axes is None:
        axes = np.eye(ndims)

    if axes.shape[1] != ndims:
        raise ValueError()

    naxes = len(axes)
    if naxes <= 0:
        raise ValueError()

    bestx = x0
    besty = np.inf
    niter = 0
    funcalls = 0

    while niter < maxiter:
        niter += 1
        for i in range(naxes):
            axis = axes[i]
            if verbose:
                logging.info('multivariate_discrete_gap_filling_minimizer: axis %d, %s' % (i, str(axis)))

            def transform(t):
                return bestx + t*axis

            # Function of single variable (t) that we will optimize during this iteration.
            def scalar_fun(t):
                testx = transform(t)
                return fun(testx)

            # logging.info(transform(0.0))
            # logging.info(scalar_fun(0.0))

            # Determine bracket along optimization axis (for t).
            # All axes must remain within their respective bounds.
            bracket = np.array([-np.inf, 0.0, np.inf])
            for j in range(ndims):
                if axis[j] != 0.0:
                    btj = np.sort((bounds[j] - bestx[j]) / axis[j])
                    if bracket[0] < btj[0]:
                        bracket[0] = btj[0]
                    if bracket[2] > btj[1]:
                        bracket[2] = btj[1]
            # if verbose:
            #     logging.info('multivariate_discrete_gap_filling_minimizer: bracket=%s' % str(bracket))

            optresult = minimize_scalar(
                scalar_fun, bracket=bracket, tol=tol, method=scalar_discrete_gap_filling_minimizer,
                options=scalar_options)
            if verbose:
                logging.info('minimize_scalar returned t=%f, y=%f' % (optresult.x, optresult.fun))
            bestx = transform(optresult.x)
            besty = optresult.fun
            if verbose:
                logging.info(
                    'multivariate_gap_filling_minimizer: niter=%d, axis=%d, best f(%s) = %f'
                    % (niter, i, str(bestx), besty))
            funcalls += optresult.nfev
        if maxfev is not None and funcalls >= maxfev:
            break

    return OptimizeResult(fun=besty, x=bestx, nit=niter, nfev=funcalls, success=(niter > 1))

def simple_global_minimizer_spark(
        fun, x0, bounds, sc=None, verbose=True, **options):
    """Exhaustive global minimizer with same calling convention as `multivariate_discrete_gap_filling_minimizer`.

    Parameters
    ----------
    fun
        Function of a single variable of a list-type.
    x0
        Unused but kept for compatibility.
    bounds
        List-type of (min, max) pairs for each element in x, defining the bounds in that dimension.
    sc (SparkContext)
        The SparkContext.

    Returns
    -------
    OptimizeResult
        The result of the minimization.

    """
    axis_domains = [range(a, b+1) for a, b in bounds]
    domain_size = np.product([len(d) for d in axis_domains])
    if verbose:
        logging.info('simple_global_minimizer_spark: domain_size=%d' % domain_size)
    domain = itertools.product(*axis_domains)
    domain_rdd = sc.parallelize(domain)

    # Evaluate function at each point in parallel using Spark.
    fun_eval_rdd = domain_rdd.map(lambda x: (fun(x), x))

    # Find the minimum y value. Secondary sort on the x value for tie breaking.
    best_y, best_x = fun_eval_rdd.min(lambda yx: (yx[0][0], yx[1]) if isinstance(yx[0], tuple) else (yx[0], yx[1]))

    return OptimizeResult(fun=best_y, x=best_x, nfev=domain_size, success=True)


import socket
import threading

socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
socket.settimeout(5)
socket.connect(('192.168.0.142', 5577))

msg = bytearray([0x41])

r = 0
g = 255
b = 0

msg.append(int(r))
msg.append(int(g))
msg.append(int(b))

msg.append(int(0))

write_mask = 0x00

write_mask |= 0xf0
msg.append(write_mask)
msg.append(0x0f)

csum = sum(msg) & 0xFF
msg.append(csum)

lock = threading.Lock()

with lock:
    socket.send(msg)


import dataclasses
from pathlib import Path
from types import MappingProxyType

import pytest
from ruamel.yaml import YAML

from satellite.config import InvalidConfigError, configure

DEFAULT_CONFIG_VALUES = MappingProxyType(
    {
        'db_path': str(Path.home() / '.vgs-satellite' / 'db.sqlite'),
        'debug': False,
        'forward_proxy_port': 9099,
        'log_path': None,
        'reverse_proxy_port': 9098,
        'routes_path': None,
        'silent': False,
        'volatile_aliases_ttl': 3600,
        'web_server_port': 8089,
    }
)

def _write_config(config_path: Path, raw_data: str = None, **kwargs):
    with open(config_path, 'w') as stream:
        if raw_data:
            stream.write(raw_data)
        else:
            YAML().dump(kwargs, stream)

def test_defaults(monkeypatch, tmp_path):
    monkeypatch.setattr(
        'satellite.config.DEFAULT_CONFIG_PATH',
        tmp_path / 'config.yml',
    )
    config = configure()
    assert dataclasses.asdict(config) == DEFAULT_CONFIG_VALUES

def test_from_default_config_file(monkeypatch, tmp_path):
    config_path = tmp_path / 'config.yml'
    _write_config(config_path, web_server_port=1)
    monkeypatch.setattr('satellite.config.DEFAULT_CONFIG_PATH', config_path)
    config = configure()
    assert dataclasses.asdict(config) == {
        **DEFAULT_CONFIG_VALUES,
        'web_server_port': 1,
    }

def test_from_custom_config_file(monkeypatch, tmp_path):
    default_config_path = tmp_path / 'config.yml'
    _write_config(default_config_path, web_server_port=1)
    monkeypatch.setattr(
        'satellite.config.DEFAULT_CONFIG_PATH',
        default_config_path,
    )

    custom_config_path = tmp_path / 'custom-config.yml'
    _write_config(custom_config_path, web_server_port=2)

    config = configure(config_path=custom_config_path)
    assert dataclasses.asdict(config) == {
        **DEFAULT_CONFIG_VALUES,
        'web_server_port': 2,
    }

def test_invalid_config_syntax(monkeypatch, tmp_path):
    config_path = tmp_path / 'config.yml'
    _write_config(config_path, raw_data='{invalid')
    monkeypatch.setattr('satellite.config.DEFAULT_CONFIG_PATH', config_path)
    with pytest.raises(InvalidConfigError):
        configure()

def test_invalid_config_data(monkeypatch, tmp_path):
    config_path = tmp_path / 'config.yml'
    _write_config(config_path, web_server_port='invalid')
    monkeypatch.setattr('satellite.config.DEFAULT_CONFIG_PATH', config_path)
    with pytest.raises(InvalidConfigError):
        configure()

def test_invalid_config_args(monkeypatch, tmp_path):
    config_path = tmp_path / 'config.yml'
    monkeypatch.setattr('satellite.config.DEFAULT_CONFIG_PATH', config_path)
    with pytest.raises(InvalidConfigError):
        configure(web_server_port='invalid')


import tensorflow as tf
import numpy as np
import cv2

class CONV(tf.keras.Model):
    def __init__(self,filter,size,stride):
        super(CONV, self).__init__()
        self.filter = filter
        self.size = size
        self.stride = stride
        self.conv = tf.keras.layers.Conv2D(self.filter,self.size,self.stride,padding='same')
        self.bn1 = tf.keras.layers.BatchNormalization()

    def call(self, inputs, **kwargs):
        x = self.conv(inputs)
        x = self.bn1(x,training = None)
        x = tf.nn.leaky_relu(x)
        return x

class Res(tf.keras.Model):
    def __init__(self,filter):
        super(Res, self).__init__()
        self.filter = filter
        self.conv1 = CONV(self.filter/2,1,1)
        self.conv2 = CONV(self.filter,3,1)

    def call(self, inputs, training=None):
        residual = inputs
        x = self.conv1(inputs)
        x = self.conv2(x)
        x = x + residual
        return x

class Darknet(tf.keras.Model):
    def __init__(self):
        super(Darknet, self).__init__()
        self.conv1 = CONV(32,3,1)
        self.conv2 = CONV(64,3,2)
        self.res3 = Res(64)
        self.conv4 = CONV(128,3,2)
        # self.res5 = Res(128)
        self.blocks5 = tf.keras.models.Sequential()
        self.conv6 = CONV(256,3,2)
        # self.res7 = Res(256)
        self.blocks7 = tf.keras.models.Sequential()
        self.conv8 = CONV(512,3,2)
        # self.res9 = Res(512)
        self.blocks9 = tf.keras.models.Sequential()
        self.conv10 = CONV(1024,3,2)
        # self.res11 = Res(1024)
        self.blocks11 = tf.keras.models.Sequential()
        for i in range(2):
            self.blocks5.add(Res(128))
        for i in range(8):
            self.blocks7.add(Res(256))
        for i in range(8):
            self.blocks9.add(Res(512))
        for i in range(4):
            self.blocks11.add(Res(1024))

        self.conv12 = CONV(512,1,1)
        self.conv13 = CONV(1024,3,1)
        self.conv14 = CONV(512,1,1)
        self.conv15 = CONV(1024,3,1)
        self.conv16 = CONV(512,1,1)
        self.conv17 = CONV(1024,3,1)
        self.conv18 = CONV(255,1,1)

        self.conv19 = CONV(256,1,1)
        self.upsample20 = tf.keras.layers.UpSampling2D(size=[2,2])

        self.conv21 = CONV(256,1,1)
        self.conv22 = CONV(512,3,1)
        self.conv23 = CONV(256,1,1)
        self.conv24 = CONV(512,3,1)
        self.conv25 = CONV(256,1,1)
        self.conv26 = CONV(512,3,1)
        self.conv27 = CONV(255,1,1)

        self.conv28 = CONV(128,1,1)
        self.upsample29 = tf.keras.layers.UpSampling2D(size=[2,2])

        self.conv30 = CONV(128,1,1)
        self.conv31 = CONV(256,3,1)
        self.conv32 = CONV(128,1,1)
        self.conv33 = CONV(256,3,1)
        self.conv34 = CONV(128,1,1)
        self.conv35 = CONV(256,3,1)
        self.conv36 = CONV(255,1,1)

    def call(self, inputs, training=None):
        x = self.conv1(inputs)
        x = self.conv2(x)
        x = self.res3(x)
        x = self.conv4(x)
        x = self.blocks5(x)
        x = self.conv6(x)
        x = self.blocks7(x)
        x_52_2 = x
        x = self.conv8(x)
        x = self.blocks9(x)
        x_26_2 = x
        x = self.conv10(x)
        x = self.blocks11(x)

        x = self.conv12(x)
        x = self.conv13(x)
        x = self.conv14(x)
        x = self.conv15(x)
        x = self.conv16(x)
        x_26_1 = x
        x = self.conv17(x)
        x = self.conv18(x)
        #   -1x13x13x255
        y_pred_13 = x

        x_26_1 = self.conv19(x_26_1)
        x_26_1 = self.upsample20(x_26_1)
        x_26 = tf.concat([x_26_1,x_26_2],axis=-1)
        x_26 = self.conv21(x_26)
        x_26 = self.conv22(x_26)
        x_26 = self.conv23(x_26)
        x_26 = self.conv24(x_26)
        x_26 = self.conv25(x_26)
        x_52_1 = x_26
        x_26 = self.conv26(x_26)
        x_26 = self.conv27(x_26)
        #   -1x26x26x255
        y_pred_26 = x_26

        x_52_1 = self.conv28(x_52_1)
        x_52_1 = self.upsample29(x_52_1)
        x_52 = tf.concat([x_52_1,x_52_2],axis=-1)
        x_52 = self.conv30(x_52)
        x_52 = self.conv31(x_52)
        x_52 = self.conv32(x_52)
        x_52 = self.conv33(x_52)
        x_52 = self.conv34(x_52)
        x_52 = self.conv35(x_52)
        x_52 = self.conv36(x_52)
        #   -1x52x52x255
        y_pred_52 = x_52

        return y_pred_13,y_pred_26,y_pred_52


import numpy as np
import cv2
import sys
import os
import glob
import argparse
import traceback
from tqdm import tqdm
import pdb
cur_dir = os.getcwd()

parser = argparse.ArgumentParser(description="recognize musics and create scores")
group1 = parser.add_mutually_exclusive_group()
group2 = parser.add_mutually_exclusive_group()
group1.add_argument("-d", "--add", action="store_true", help="add scores")
group1.add_argument("-w", "--write", action="store_true", help="overwrite scores")
group2.add_argument("-a", "--all", action="store_true", help="recognize all musics")
group2.add_argument("-s", "--select", nargs="*", help="recognize selected musics (you need to input music names with extension)")
args = parser.parse_args()

mdir = "music"
sdir = "score"

mdir_path = os.path.join(cur_dir, mdir)
if args.all:
    mnames = os.listdir(mdir_path)
elif args.select:
    mnames = args.select
else:
    raise RuntimeError("option or music names should be set")
    traceback.print_exc()

mpaths = []
spaths = []
sdir_path = os.path.join(cur_dir, sdir)
for mname in mnames:
    sname = mname.replace(".mp4", ".txt")
    spath = os.path.join(sdir_path, sname)
    if os.path.isfile(spath) and args.write:
        os.remove(spath)
    if (not os.path.isfile(spath)) or args.write:
        mpath = os.path.join(mdir_path, mname)
        mpaths.append(mpath)
        spaths.append(spath)

print("{} music data entry".format(len(mpaths)))

tdir = "template"
tdir_path = os.path.join(cur_dir, tdir)
tpaths = glob.glob(os.path.join(tdir_path,"*"))

icons = ["single", "long", "flick", "slide", "slidem", "none"]
slides = ["slide", "slidem"]
icon_avr1 = []
icon_avr2 = []
icon_max = []
icon_min = []

#get icon data
idir = "icondata"
ifile = "icondata.txt"
idir_path = os.path.join(cur_dir, idir)
idata_path = os.path.join(idir_path, ifile)
ipaths1 = []
ipaths2 = []
for icon in icons:
    iname = icon + ".txt"
    ipaths1.append(os.path.join(idir_path, *["icondata1", iname]))
    ipaths2.append(os.path.join(idir_path, *["icondata2", iname]))

itmp = []
with open(idata_path) as f:
    for s_line in f:
        itmp.append(s_line.split())

inumt = len(itmp)
idatat = np.zeros((inumt, 3), dtype = np.float64)
iconst = []

for i in range(inumt):
    iconst.append(itmp[i][0])
    idatat[i,0] = np.array(itmp[i][1]).astype(np.float64)
    idatat[i,1] = np.array(itmp[i][2]).astype(np.float64)
    idatat[i,2] = np.array(itmp[i][3]).astype(np.float64)

for ipath in ipaths1:
    itmp = []
    with open(ipath) as f:
        for s_line in f:
            itmp.append(s_line.split())

    icon_avr1.append(np.array(itmp[1]).astype(np.float64))

for ipath in ipaths2:
    itmp = []
    with open(ipath) as f:
        for s_line in f:
            itmp.append(s_line.split())

    icon_avr2.append(np.array(itmp[1]).astype(np.float64))

# set thresholds
th1 = 25.
th2 = 25.
ths1 = np.array([th1, th1, th1], dtype=np.float64)
ths2 = np.array([th2, th2, th2], dtype=np.float64)
ths = []

for idx, icon in enumerate(icons):
        if (icon == "slide") or (icon == "slidem"):
            ths.append(ths2)
        else:
            ths.append(ths1)

# set functions
def start_predict(count, i, mean):
    last_icons[i] = now_icons[i]
    th = 20.

    if np.sum(abs(icon_avr2[icons.index("none")] - np.array(mean)) < np.array([th, th, th])) == 3:
        now_icons[i] = "none"

    else:
        now_icons[i] = "null"

def icon_classify_a(count, i, mean):
    th = 10.
    ths = np.array([th, th, th], dtype=np.float64)
    for idx, icon in enumerate(slides):
        if (np.abs(mean - idatat[3 + idx]) < ths).all():
            now_icons[i] = icon
            sflag[i] = True
            break

def icon_classify_b(count, i, mean1, mean2, path):
    last_icons2[i] = last_icons[i]
    last_icons[i] = now_icons[i]

    for idx, icon in enumerate(icons):
        if sflag[i]:
                sflag[i] = False
#                now_icons[i] = last_icons[i]
#                last_icons[i] = last_icons2[i]
                break

        if ((np.abs(mean2 - icon_avr2[idx]) < ths[idx]).all() or (np.abs(mean1 - icon_avr1[idx]) < ths[idx]).all()):
            if ((icon != "slide") and (icon != "slidem")):
                now_icons[i] = icon
                break

        else:
            now_icons[i] = "null"

    if (not(last_icons[i] == "none" or last_icons[i] == "null")) and (now_icons[i] != last_icons[i]):
        mdata = [count, i, last_icons[i]]
#        print([count, i, last_icons[i]])
        with open(path, mode = "a") as f:
            f.write(" ".join(map(str,mdata)) + "\n")

def non_max_suppression_slow(w, h, overlapThresh, *boxes):
    if len(boxes) == 0:
        return []

    pick = []

    x1 = boxes[0]
    y1 = boxes[1]
    x2 = x1 + w
    y2 = y1 + h

    area = (x2 - x1 + 1) * (y2 - y1 + 1)
    if H < W:
        idxs = np.argsort(-x2)
    else:
        idxs = np.argsort(y2)

    while len(idxs) > 0:
        last = len(idxs) - 1
        i = idxs[last]
        pick.append(i)
        suppress = [last]

        for pos in range(0, last):
            j = idxs[pos]

            xx1 = max(x1[i], x1[j])
            yy1 = max(y1[i], y1[j])
            xx2 = min(x2[i], x2[j])
            yy2 = min(y2[i], y2[j])

            ww = max(0, xx2 - xx1 + 1)
            hh = max(0, yy2 - yy1 + 1)

            overlap = float(ww * hh) / float(area[j])

            if overlap > overlapThresh:
                suppress.append(pos)

        idxs = np.delete(idxs, suppress)

    nb0 = []
    nb1 = []
    for k in pick:
        nb0.append(boxes[0][k])
        nb1.append(boxes[1][k])

    new_boxes = tuple([np.array(nb0), np.array(nb1)])
    return new_boxes

for mpath in mpaths:
    print("music name : {}\n".format(os.path.basename(mpath)))
    spath = spaths[mpaths.index(mpath)]

    # get the video's information
    cap = cv2.VideoCapture(mpath)
    W = cap.get(cv2.CAP_PROP_FRAME_WIDTH)
    W_stand = 1920
    H = cap.get(cv2.CAP_PROP_FRAME_HEIGHT)
    H_stand = 886
    fps = cap.get(cv2.CAP_PROP_FPS)
    fnum = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))

    # get the template image for template matching
    tmp = cv2.imread(tpaths[0])
    tmp = cv2.cvtColor(tmp, cv2.COLOR_BGR2GRAY)
    w, h = tmp.shape[::-1]
    overlapThresh = 0.3

    tmp2 = cv2.imread(tpaths[1])
    tmp2 = cv2.cvtColor(tmp2, cv2.COLOR_BGR2GRAY)
    w2, h2 = tmp2.shape[::-1]

    # set variables
    count = 0  # to record the frame number
    start = 0  # to decide the strat frame
    mdata = [] # to record the music score as a list of string
    flag = 0   #
    sflag = [False] * 5
    scount = [0] * 5
    nones = np.array(["none", "none", "none", "none", "none"], dtype = "<U6")
    last_icons2 = np.array(["null", "null", "null", "null", "null"], dtype = "<U6")
    last_icons = np.array(["null", "null", "null", "null", "null"], dtype = "<U6")
    now_icons = np.array(["null", "null", "null", "null", "null"], dtype = "<U6")

    with open(spath, mode = "a") as f:
        f.write(str(fps) + "\n")

    # resize the template size
    if H_stand != H:
        tmp = cv2.resize(tmp, dsize=None, fx=H/H_stand, fy=H/H_stand, interpolation=cv2.INTER_CUBIC)
        w, h = tmp.shape[::-1]

    for _ in tqdm(range(fnum)):
        ret, img = cap.read()

        if ret:
            if flag == 0:
                gray = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
                res =  cv2.matchTemplate(gray, tmp, cv2.TM_CCOEFF_NORMED)
                threshold = 0.89
                loc = np.where(res > threshold)
                boxes = non_max_suppression_slow(w, h, overlapThresh, *loc[::-1])

                for pt in zip(*boxes):
                    cv2.rectangle(img, pt, (pt[0] + w, pt[1] + h), (0, 0, 255), 2)

                if len(boxes[0]) == 5 & len(boxes[1]) == 5:
                    flag = 1
                    bloc = boxes[0] + w - boxes[0][0]

            elif flag == 1:
                for i in range(5):
                    b0, b1 = boxes[0][i], boxes[1][i]
                    part = img[b1 + int(1/10*h) : b1 + int(9/10*h), b0 + int(1/10*w)  : b0 + int(9/10*w), :]
                    mean = np.mean(np.mean(part, axis = 0), axis = 0)
                    start_predict(count, i, mean)
                    if (now_icons == nones).all():
                        count = -36
                        flag = 2

            else:
                ba, br, bl = boxes[1][0], boxes[0][4], boxes[0][0]
                imgc = img[int(ba-h*20/100):int(ba+h*8/10), int(bl-w/2):int(br+w*3/2), :]
                Wc, Hc, Cc = imgc.shape
                gray2 = cv2.cvtColor(imgc, cv2.COLOR_BGR2GRAY)
                res2 =  cv2.matchTemplate(gray2, tmp2, cv2.TM_CCOEFF_NORMED)
                threshold2 = 0.60
                loc2 = np.where(res2 > threshold2)
                boxes2 = non_max_suppression_slow(w2, h2, overlapThresh, *loc2[::-1])

                for i in range(len(boxes2[0])):
                    b20, b21 = boxes2[0][i], boxes2[1][i]
                    part = imgc[b21: b21 + h2, b20 : b20 + w2, :]
                    mean = np.mean(np.mean(part, axis = 0), axis = 0)
                    idx = np.argmin(np.abs((b20+w2/2)-bloc))
                    icon_classify_a(count, idx, mean)

                for i in range(5):
                    b0, b1 = boxes[0][i], boxes[1][i]
                    part1 = img[b1 : b1 + h, b0 : b0 + w, :]
                    mean1 = np.mean(np.mean(part1, axis = 0), axis = 0)
                    part2 = img[b1 + int(1/10*h) : b1 + int(9/10*h), b0 + int(1/10*w)  : b0 + int(9/10*w), :]
                    mean2 = np.mean(np.mean(part2, axis = 0), axis = 0)
                    icon_classify_b(count, i, mean1, mean2, spath)

            count += 1
#            cv2.imshow('frame',img)
            if cv2.waitKey(1) & 0xFF == ord('q'):
                break

        else:
            break

    cap.release()
    cv2.destroyAllWindows()


#! /usr/bin/env python
# encoding: utf-8
import rospy
import serial
import Queue
import threading
import struct
from geometry_msgs.msg import Twist
from sensor_msgs.msg import Imu,MagneticField
## rosrun teleop_twist_keyboard teleop_twist_keyboard.py

def do_parse(ext_type,ext_len,ext_data):
    # 将ext_data 封装成完整的数据帧
    ext_data.insert(0, ext_len);
    ext_data.insert(0, ext_type);
    ext_data.insert(0, 0xfa);
    ext_data.insert(0,0xce);
    # CE FA 03 18    76 0E    5C 03 F0 FF 88 3C      E0 FF A2 00 84 00    00 A2 FF E0 00 84      F4 01     20 03
    # 根据数据帧的类型的类型来做对应的解析  0x01 线速度  0x02 电池
    if ext_type == 0x03:
        # 对数据进行拆包
        # 温度
        temperature = struct.unpack('h',bytearray(ext_data[4:6]))[0]

        # 加速度
        ax = struct.unpack('h',bytearray(ext_data[6:8]))[0]
        ay = struct.unpack('h',bytearray(ext_data[8:10]))[0]
        az = struct.unpack('h',bytearray(ext_data[10:12]))[0]
        # 角速度
        gx = struct.unpack('h', bytearray(ext_data[12:14]))[0]
        gy = struct.unpack('h', bytearray(ext_data[14:16]))[0]
        gz = struct.unpack('h', bytearray(ext_data[16:18]))[0]
        # 地磁
        mx = struct.unpack('h', bytearray(ext_data[18:20]))[0]
        my = struct.unpack('h', bytearray(ext_data[20:22]))[0]
        mz = struct.unpack('h', bytearray(ext_data[22:24]))[0]
        # 速度
        velocity = struct.unpack('h', bytearray(ext_data[24:26]))[0]
        angular = struct.unpack('h', bytearray(ext_data[26:28]))[0]

        # 发布陀螺仪的数据
        imu = Imu()
        # 根据芯片手册设定缩放系数
        accel_ratio = 1/16384.0
        imu.linear_acceleration.x = ax*accel_ratio;
        imu.linear_acceleration.y = ay*accel_ratio;
        imu.linear_acceleration.z = az*accel_ratio;

        # °/s 转成弧度/s
        gyro_ratio = 1/65.5/(180/3.1415926);
        imu.angular_velocity.x = gx*gyro_ratio;
        imu.angular_velocity.y = gy*gyro_ratio;
        imu.angular_velocity.z = gz*gyro_ratio;

        imuPublisher.publish(imu)

        # 发布地磁的数据
        mag = MagneticField();
        mag.magnetic_field.x = mx;
        mag.magnetic_field.y = my;
        mag.magnetic_field.z = mz;

        magPublisher.publish(mag);

        # print(velocity,angular)
        # 将小车当前的线速度和角速度发布出去
        twist = Twist()
        twist.linear.x = velocity/1000.0
        twist.angular.z = angular/1000.0
        velPublisher.publish(twist);

def do_recv():
    """
        找到一帧完整数据 从recv_queue中查找
        1. 查找帧头0
        2. 查找帧头1
        3. 查找帧类型
        4. 查找帧的长度
        5. 根据帧长度,读取帧数据
        完整的帧数据
    """
    while True:
        buff = recv_queue.get();
        value = bytearray(buff)[0]

        if value == 0xce: # 找到帧头0
            # 若找到帧头0,则继续判断帧头1
            value = bytearray(recv_queue.get())[0]
            if value == 0xfa: # 若判断成功,则head0和head1匹配成功,意味着帧头完全匹配
                # 开始读类型
                ext_type = bytearray(recv_queue.get())[0]
                # 读帧数据长度
                ext_len = bytearray(recv_queue.get())[0]
                # 读数据
                ext_data = []
                while len(ext_data) < ext_len:
                    # 不断往后读取数据
                    value = bytearray(recv_queue.get())[0]
                    ext_data.append(value);

                if ext_data[-1]== 0xad:
                   # 开始解析数据
                   do_parse(ext_type,ext_len,ext_data);

def cmd_vel_callback(msg):
    print("msg：",msg.linear.x,msg.angular.z);
    #     帧头0  帧头1  帧类型
    cmd = [0xce,0xfa,0x05]
    # 添加帧长度
    cmd.append(0x04);

    # 下发线速度和角速度
    velocity = msg.linear.x;
    angular = msg.angular.z;

    # 对数据进行拆解
    velocity_params = bytearray(struct.pack('h',int(velocity*1000)))
    angular_params = bytearray(struct.pack('h',int(angular*1000)))

    cmd.append(velocity_params[0])
    cmd.append(velocity_params[1])

    cmd.append(angular_params[0]);
    cmd.append(angular_params[1]);

    # [206, 250, 5, 4, 250, 0, 244, 1]
    # 将数据发送给串口
    ser.write(cmd)

##rosrun teleop_twist_keyboard teleop_twist_keyboard.py
if __name__ == '__main__':
    # 创建ros节点
    node_name="car_driver_node";
    rospy.init_node(node_name);

    # 订阅/cmd_vel
    rospy.Subscriber("/cmd_vel",Twist,cmd_vel_callback);
    # 发布者
    velPublisher = rospy.Publisher("/zxcar/get_vel",Twist,queue_size=50);
    # 发布陀螺仪的数据
    imuPublisher = rospy.Publisher("/zxcar/imu",Imu,queue_size=100);
    # 地磁的数据
    magPublisher = rospy.Publisher("/zxcar/magnetic",MagneticField,queue_size=50)

    port = rospy.get_param("~port", "/dev/ttyUSB0")

    # 创建serial 串口通讯对象
    ser = serial.Serial(port=port,baudrate=115200)

    # 判断串口是否打开成功
    if not ser.isOpen():
        ser.open();

    # 创建消息队列,专门用于存储读到的数据
    recv_queue = Queue.Queue();

    # 开启线程,专门用于取队列中查找帧数据
    threading.Thread(target=do_recv).start();
    try:
        # 读取串口中的数据
        while not rospy.is_shutdown():
            buff = ser.read();
            # 读到数据,立马存起来
            recv_queue.put(buff);
        rospy.spin();
    except Exception as e:
        rospy.logerr("car_driver:{}".format(e))
        ser.close();
        exit();


#!/usr/bin/env python
# -*- coding: utf-8 -*-

import io
import sys
import argparse
import requests
from requests.exceptions import HTTPError
import six
import json
import re
import email
import logging
from datetime import datetime
from os import path
from abc import ABCMeta, abstractmethod
try:
    from urllib.parse import urlencode
    from urllib.parse import urlunparse
except ImportError:
    from urllib import urlencode
    from urlparse import urlunparse
from bs4 import BeautifulSoup
from time import sleep, time, mktime
from fake_useragent import UserAgent, settings as fake_useragent_settings

__author__ = 'Tom Dickinson, Flavio Martins, David Semedo'

logger = logging.getLogger(__name__)
logging.basicConfig(format='%(asctime)s : %(levelname)s : %(message)s', level=logging.INFO)

# Only needed in case user additional details are required
# If that is the case, the library requires the package python-twitter
TWITTER_REST_API_CONSUMER_KEY = ""
TWITTER_REST_API_CONSUMER_SECRET = ""
TWITTER_REST_API_ACCESS_TOKEN = ""
TWITTER_REST_API_ACCESS_TOKEN_SECRET = ""

DEFAULT_RATE_DELAY = 0
DEFAULT_ERROR_DELAY = 5
DEFAULT_LIMIT = None
MAX_RETRIES_SESSION = 5
MAX_RETRIES = MAX_RETRIES_SESSION*5
PROGRESS_PER = 100
DEFAULT_TARGET_TYPE = "tweets"
DATE_FORMAT = "%a %b %d %H:%M:%S +0000 %Y"  # "Fri Mar 29 11:03:41 +0000 2013";

class TwitterSearch:
    __metaclass__ = ABCMeta

    def __init__(self, session, rate_delay, error_delay=5, useragent_cache_path=fake_useragent_settings.DB):
        """
        :param rate_delay: How long to pause between calls to Twitter
        :param error_delay: How long to pause when an error occurs
        """
        self.session = session
        self.rate_delay = rate_delay
        self.error_delay = error_delay

        self.UA = UserAgent(fallback='Mozilla/5.0 (Windows NT 6.1; WOW64; rv:33.0) Gecko/20100101 Firefox/33.0',
                       path=useragent_cache_path)

    def search(self, query, target_type, **kwargs):
        """
        Scrape items from twitter
        :param query:   Query to search Twitter with. Takes form of queries constructed with using Twitters
                        advanced search: https://twitter.com/search-advanced
        :param target_type:    Can be "tweets" or "users"
        """
        url = self.construct_url(query, target_type=target_type, language=kwargs['language'])
        continue_search = True
        min_item = None

        # Initialize search function wrapper according to the target type
        parse_tweets_fn = self.parse_tweets if target_type == DEFAULT_TARGET_TYPE else self.parse_users

        response = self.execute_search(url)
        while response is not None and continue_search and response['items_html'] is not None:
            items = parse_tweets_fn(response['items_html'])

            # Check if we should collect additional user details
            if target_type == "users" and kwargs["user_stats"]:
                self.retrieve_user_details(items)

            # If we have no items, then we can break the loop early
            if len(items) == 0:
                break

            continue_search = self.save_items(items)

            max_item = response["min_position"]

            if min_item is not max_item:
                url = self.construct_url(query, target_type=target_type, max_position=max_item,
                                         language=kwargs['language'])
                # Sleep for our rate_delay
                sleep(self.rate_delay)
                response = self.execute_search(url)
                min_item = max_item

    def execute_search(self, url, retry_num=0):
        """
        Executes a search to Twitter for the given URL
        :param url: URL to search twitter with
        :param retry_num: Retry number of current function call
        :return: A JSON object with data from Twitter
        """
        try:
            logger.info("URL: " + url)
            response = self.session.get(url)
            response.raise_for_status()  # raise on any HTTPError
            data = response.json()
            return data
        # If we get a HTTPError exception due to a request timing out, we sleep for our error delay, then make
        # another attempt
        except HTTPError as e:
            # 400 Bad Request
            if e.response.status_code == 400:
                logger.debug("HTTP 400 - Bad request")
                return e.response.json()
            elif e.response.status_code == 429:
                now_ts = datetime.utcnow().timestamp()
                logger.debug("HTTP 429 - Too many requests")
                logger.debug(e.response.headers)
                utc_reset_ts = int(e.response.headers['x-rate-limit-reset'])
                reset = utc_reset_ts - now_ts
                logger.debug("Reset time: %s", str(reset))
                # Multiply by error delay for paranoid reasons.
                seconds = reset * self.error_delay
                logger.debug("Going to sleep for %s seconds.", str(seconds))
                sleep(seconds)
            else:
                retry_after = e.response.headers.get('retry-after', None)
                reset_seconds = 1

                if retry_after is not None:
                    if re.match("([0-9])+", retry_after):
                        reset_seconds = int(retry_after)
                    else:
                        retry_after_tuple = email.utils.parsedate(retry_after)
                        if retry_after_tuple is None:
                            logger.error("Invalid Retry-After header: %s" % retry_after)
                        retry_date = mktime(retry_after_tuple)
                        reset_seconds = retry_date - time()

                logger.error(e.response.headers)
                logger.error(e.response.message)
                total_sleep = reset_seconds * self.error_delay
                logger.info("Sleeping for %i", total_sleep)
                sleep(total_sleep)

                if retry_num % MAX_RETRIES_SESSION == 0 and retry_num > 0:
                    headers = {'User-Agent': self.UA.random}
                    self.session.headers.update(headers)
                elif retry_num == MAX_RETRIES:
                    return None
            return self.execute_search(url, retry_num + 1)

    @staticmethod
    def parse_tweets(items_html):
        """
        Parses Tweets from the given HTML
        :param items_html: The HTML block with tweets
        :return: A JSON list of tweets
        """
        soup = BeautifulSoup(items_html, 'lxml')

        comma = ','
        dot = '.'
        tweets = []
        for tweet in soup.find_all(class_='stream-item'):
            if 'data-item-id' not in tweet.attrs:
                continue

            id_str = tweet.attrs['data-item-id']
            id = int(id_str)

            tweet_text = tweet.find(class_="tweet-text")
            if tweet_text is None:
                continue

            timestamp = int(tweet.find(class_="_timestamp").attrs['data-time'])
            created_at = datetime.utcfromtimestamp(timestamp).strftime(DATE_FORMAT)

            tweet_div = tweet.find(class_="tweet")
            user = {
                'id_str': tweet_div.attrs['data-user-id'],
                'id': int(tweet_div.attrs['data-user-id']),
                'screen_name': tweet_div.attrs['data-screen-name'],
                'name': tweet_div.attrs['data-name'],
            }

            interactions = [x.get_text() for x in tweet.find_all(class_='ProfileTweet-actionCount')]
            replies = int(interactions[0].split(" ")[0].replace(comma, "").replace(dot, ""))
            retweets = int(interactions[1].split(" ")[
                               0].replace(comma, "").replace(dot, ""))
            likes = int(interactions[2].split(" ")[0].replace(comma, "").replace(dot, ""))
            hashtags = [hashtag_node.get_text() for hashtag_node in tweet.find_all(class_='twitter-hashtag')]
            urls = [url_node.attrs['data-expanded-url']
                    for url_node in tweet.find_all('a', class_='twitter-timeline-link')
                    if 'data-expanded-url' in url_node.attrs]
            photos = [photo_node.attrs['data-image-url']
                      for photo_node in tweet.find_all(class_='AdaptiveMedia-photoContainer')
                      if 'data-image-url' in photo_node.attrs]

            videos = []
            native_video_node = tweet.find(class_='PlayableMedia-player')
            if native_video_node is not None:
                videos.append({
                    'expanded_url': 'https://twitter.com/i/videos/tweet/%s' % id_str
                })

            cards = []
            card_nodes = tweet.find_all(class_='card2')
            for node in card_nodes:
                card_type = node.get('data-card2-name', None)
                if card_type is not None:  # Only care about media. Ignore Tweet Quotes, etc.
                    if 'summary' in card_type:  # Expanded URL w/ image
                        iframe_container = node.find(class_='js-macaw-cards-iframe-container')
                        if iframe_container is not None:
                            timeline_link = tweet_text.find('a', class_='twitter-timeline-link')
                            if timeline_link is not None and 'data-expanded-url' in timeline_link.attrs:
                                cards.append({
                                    'card_url': 'https://twitter.com' + iframe_container.attrs['data-src'],
                                    'expanded_url': timeline_link.attrs['data-expanded-url']
                                })
                            else:
                                logger.error("BAD CARD: %s", id_str)
                    elif 'player' in card_type:  # Embedded video
                        timeline_link = tweet_text.find('a', class_='twitter-timeline-link')
                        if timeline_link is not None and 'data-expanded-url' in timeline_link.attrs:
                            videos.append({
                                'expanded_url': timeline_link.attrs['data-expanded-url']
                            })
                        else:
                            logger.error("BAD CARD: %s", id_str)

            # remove u-hidden links, etc
            for hidden_child in tweet_text.find_all(class_='u-hidden'):
                hidden_child.decompose()
            text = tweet_text.get_text()

            tweets.append({
                'created_at': created_at,
                'text': text,
                'id': id,
                'id_str': id_str,
                'epoch': timestamp,
                'reply_count': replies,
                'retweet_count': retweets,
                'favorite_count': likes,
                'hashtags': hashtags,
                'cards': cards,
                'urls': urls,
                'photos': photos,
                'videos': videos,
                'user': user,
            })
        return tweets

    @staticmethod
    def parse_users(items_html):
        """
        Parses Users from the given HTML
        :param items_html: The HTML block with items
        :return: A JSON list of items
        """
        soup = BeautifulSoup(items_html, 'lxml')
        items = []
        for div in soup.find_all("div", class_='js-stream-item'):

            # If our li doesn't have a tweet-id, we skip it as it's not going to be a tweet.
            if 'data-item-id' not in div.attrs:
                continue

            user = {
                'bio': None,
                'id_str': div['data-item-id'],
                'id': int(div['data-item-id']),
                'screen_name': None,
                'name': None,
            }

            # User Bio
            text_p = div.find("p", class_="ProfileCard-bio")
            if text_p is not None:
                user['bio'] = text_p.get_text()

            # Tweet User ID, User Screen Name, User Name
            user_details_div = div.find("div", class_="user-actions")
            if user_details_div is not None:
                user['screen_name'] = user_details_div['data-screen-name']
                user['name'] = user_details_div['data-name']

            user_fields_div = div.find("div", class_="ProfileCard-userFields")
            user_verified_span = user_fields_div.find("span", class_="Icon--verified")
            user['verified'] = True if user_verified_span else False

            items.append(user)
        return items

    @staticmethod
    def construct_url(query, target_type, max_position=None, language=None):
        """
        For a given query, will construct a URL to search Twitter with
        :param query: The query term used to search twitter
        :param target_type:    Can be "tweets" or "users"
        :param max_position: The max_position value to select the next pagination of items
        :param language: Specifies a language to filter search results
        :return: A string URL
        """
        params = {
            'f': target_type,
            'vertical': 'default',
            'include_available_features': 1,
            'include_entities': 1,
            'include_new_items_bar': 'true',
            'reset_error_state': 'false',
            'src': 'typd',
            'max_position': max_position,
            'q': query,
        }

        if language:
            params['l'] = language

        url_tupple = ('https', 'twitter.com', '/i/search/timeline', '', urlencode(params), '')
        return urlunparse(url_tupple)

    @staticmethod
    def construct_user_url(query, target_type, max_position=None):
        """
        For a given query, will construct a URL to search Twitter with
        :param query: The query term used to search twitter
        :param target_type:    Can be "tweets" or "users"
        :param max_position: The max_position value to select the next pagination of items
        :return: A string URL
        """
        params = {
            'f': target_type,
            'vertical': 'default',
            'q': query,
            'src': 'typd',
            'include_available_features': 1,
            'include_entities': 1,
            'max_position': max_position,
            'reset_error_state': 'false',
            'include_new_items_bar': 'true',
        }

        url_tupple = ('https', 'twitter.com', '/i/search/timeline', '', urlencode(params), '')
        return urlunparse(url_tupple)

    @abstractmethod
    def save_items(self, items):
        """
        An abstract method that's called with a list of items.
        When implementing this class, you can do whatever you want with these items.
        """

    def retrieve_user_details(self, items):
        """
        For a given set of crawled users, retrieves additional information using the Twitter REST API
        :param items: A list of user dictionarities
        :return: An updated list with additional fields of User dictionaries
        """
        # The user lookup API limit per request is 100
        step = 100

        import twitter
        api = twitter.Api(consumer_key=TWITTER_REST_API_CONSUMER_KEY,
                          consumer_secret=TWITTER_REST_API_CONSUMER_SECRET,
                          access_token_key=TWITTER_REST_API_ACCESS_TOKEN,
                          access_token_secret=TWITTER_REST_API_ACCESS_TOKEN_SECRET)

        for i in range(0, len(items), step):
            statuses = api.UsersLookup(screen_name=[item["screen_name"] for item in items[i:step]])
            for j in range(min(len(statuses), step)):
                items[i + j] = {**items[i + j], **statuses[j].AsDict()}

        return items

class TwitterSearchImpl(TwitterSearch):
    def __init__(self, session, rate_delay, error_delay, max_items, filepath, useragent_cache_path=fake_useragent_settings.DB):
        """
        :param rate_delay: How long to pause between calls to Twitter
        :param error_delay: How long to pause when an error occurs
        :param max_items: Maximum number of items to collect for this example
        """
        super(TwitterSearchImpl, self).__init__(session, rate_delay, error_delay, useragent_cache_path)
        self.max_items = max_items
        self.counter = 0
        self.filepath = filepath
        self.jsonl_file = None

    def search(self, query, target_type, **kwargs):
        # Specify a user agent to prevent Twitter from returning a profile card
        headers = {
            'Accept': 'application/json, text/javascript, */*; q=0.01',
            'Referer': 'https://twitter.com/search',
            'User-Agent': self.UA.random,
            'X-Twitter-Active-User': 'yes',
            'X-Requested-With': 'XMLHttpRequest'
        }
        self.session.headers.update(headers)

        self.jsonl_file = io.open(self.filepath, 'w', encoding='utf-8')
        super(TwitterSearchImpl, self).search(query, target_type=target_type, **kwargs)
        self.jsonl_file.close()

    def save_items(self, items):
        """
        Just prints out items
        :return:
        """
        for item in items:
            # Lets add a counter so we only collect a max number of items
            self.counter += 1

            if six.PY2:
                data = json.dumps(item, ensure_ascii=False, encoding='utf-8')
            else:
                data = json.dumps(item, ensure_ascii=False)

            self.jsonl_file.write(data + '\n')

            if self.counter % PROGRESS_PER == 0:
                logger.info("%s : %i items saved to file.", self.filepath, self.counter)

            # When we've reached our max limit, return False so collection stops
            if self.max_items is not None and self.counter >= self.max_items:
                return False

        return True

def twitter_search(search_terms=None, since=None, until=None, language=None, accounts=None, search_filter=None,
                   target_type=DEFAULT_TARGET_TYPE,
                   rate_delay=DEFAULT_RATE_DELAY, error_delay=DEFAULT_ERROR_DELAY, user_stats=False,
                   limit=DEFAULT_LIMIT,
                   output_dir=".", output_file=None, useragent_cache_path=fake_useragent_settings.DB):
    session = requests.Session()

    search_str = ""

    if search_terms:
        search_str = " ".join(search_terms)

    if since:
        search_str += " since:" + since

    if until:
        search_str += " until:" + until

    if search_filter:
        search_str += " filter:" + search_filter

    if not accounts:
        if not search_terms:
            logger.error("Nothing to search")
            sys.exit(1)
        elif not output_file:
            logger.error("No output_file specified")
            sys.exit(1)
        else:
            filepath = path.join(output_dir, output_file)
            twit = TwitterSearchImpl(session, rate_delay, error_delay,
                                     limit, filepath, useragent_cache_path=useragent_cache_path)
            logger.info("Search : %s", search_str)
            twit.search(search_str, target_type=target_type, user_stats=user_stats, language=language)
    else:
        if not path.isdir(output_dir) and not output_file:
            logger.error('Output directory does not exist.')
            sys.exit(1)

        for act in accounts:
            if output_file:
                filepath = output_file
            else:
                filepath = path.join(output_dir, act + '.jsonl')
                # do not overwrite existing files in output directory
                try:
                    if path.getsize(filepath) > 0:
                        logger.error('%s : File already has content.', filepath)
                        continue
                except OSError:
                    pass

            twit = TwitterSearchImpl(session, rate_delay, error_delay,
                                     limit, filepath, useragent_cache_path=useragent_cache_path)
            search_str_from = search_str + " from:" + act
            logger.info("Search : %s", search_str_from)
            twit.search(search_str_from, target_type=DEFAULT_TARGET_TYPE, language=language)

def main():
    parser = argparse.ArgumentParser()
    parser.add_argument("--search", default=[], nargs='+')
    parser.add_argument("-f", default=DEFAULT_TARGET_TYPE, type=str)
    parser.add_argument("--user_stats", action="store_true", default=False, required=False)
    parser.add_argument('--accounts', nargs='+', required=False)
    parser.add_argument('-l', type=str, required=False)
    parser.add_argument("--filter", type=str)
    parser.add_argument("--since", type=str)
    parser.add_argument("--until", type=str)
    parser.add_argument("--rate_delay", type=int, default=DEFAULT_RATE_DELAY)
    parser.add_argument("--error_delay", type=int, default=DEFAULT_ERROR_DELAY)
    parser.add_argument("--limit", type=int, default=DEFAULT_LIMIT)
    parser.add_argument("--output_dir", type=str, default='.')
    parser.add_argument("--output_file", type=str)
    parser.add_argument("--fake_useragent_cache_path", type=str, default=fake_useragent_settings.DB)
    args = parser.parse_args()

    twitter_search(target_type=args.f, search_terms=args.search, since=args.since, until=args.until, language=args.l,
                   accounts=args.accounts, search_filter=args.filter, rate_delay=args.rate_delay,
                   error_delay=args.error_delay, limit=args.limit,
                   output_dir=args.output_dir, output_file=args.output_file, user_stats=args.user_stats,
                   useragent_cache_path=args.fake_useragent_cache_path)

if __name__ == '__main__':
    main()


"""
Neural network models for the regression experiments
"""
import math

import torch
import torch.nn.functional as F
from torch import nn

class TransformerNet(nn.Module):
     def __init__(self, n_inputs, n_hidden, size_hidden, n_o, device):
        super(TransformerNet,self).__init__()
        #self.task_context = torch.zeros(size_hidden).to(device)
       # self.task_context.requires_grad = True #maybe break net into two, similar to CAVIA
        self.net=nn.Sequential(OrderedDict([
            ('l1',nn.Linear(n_inputs, size_hidden)),
            ('relu1',nn.ReLU()),
            ('l2',nn.Linear(size_hidden, size_hidden)),
            ('relu2',nn.ReLU()),
            ('l3',nn.Linear(size_hidden, size_hidden)),
        ]))

     def forward(self,x):
     #   if len(self.task_context) != 0:
      #      x = torch.cat((x, self.task_context.expand(x.shape[0], -1)), dim=1)
       # else:
        #    x = torch.cat((x, self.task_context))
        return self.net(x)

class FCNet(nn.Module):
     def __init__(self, n_inputs, n_hidden, size_hidden, n_out):
        super(FCNet, self).__init__()

        # fully connected layers
        self.fc_layers = nn.ModuleList()
        self.fc_layers.append(nn.Linear(n_inputs, size_hidden))
        for k in range(n_hidden - 2):
            self.fc_layers.append(nn.Linear(size_hidden, size_hidden))
        self.fc_layers.append(nn.Linear(size_hidden, n_out))

        # context parameters (note that these are *not* registered parameters of the model!)
        #self.num_context_params = num_context_params
        #self.context_params = None
        #self.reset_context_params()

     def forward(self, x):

        # concatenate input with context parameters
        #x = torch.cat((x, self.context_params.expand(x.shape[0], -1)), dim=1)

        for k in range(len(self.fc_layers) - 1):
            x = F.relu(self.fc_layers[k](x))
        y = self.fc_layers[-1](x)

        return y

def transformer_loss(L0, L1, model, device):
    params = [w for w in model.weights] + [b for b in model.biases] + [model.task_context]
    gradients0 = torch.autograd.grad(L0, params, retain_graph=True, create_graph=True)
    gradients1 = torch.autograd.grad(L1, params, retain_graph=True, create_graph=True)
    loss = torch.zeros(1).to(device)
    for i in range(len(gradients0)):
        a = torch.flatten(gradients0[i].clamp_(-10, 10)).to(device)
        b = torch.flatten(gradients1[i].clamp_(-10, 10)).to(device)
        product = torch.dot(a, b)
        loss = loss - torch.sum(product)
    divisor = norm_tensorlist(gradients0, device)*norm_tensorlist(gradients1, device)
    loss = torch.div(loss , divisor)
    return loss

def cosine_loss(L0, L1, model, device):
    params = [w for w in model.weights] + [b for b in model.biases] + [model.task_context]
    gradients0 = torch.autograd.grad(L0, params, retain_graph=True, create_graph=True)
    gradients1 = torch.autograd.grad(L1, params, retain_graph=True, create_graph=True)
    criterion = nn.CosineEmbeddingLoss().to(device)
    mask = torch.ones(1).to(device)
    views1 =[]
    views2 = []
    for i in range(len(gradients0)):
        view1 = gradients0[i].clamp_(-10, 10).view(-1, 1).to(device)
        view2 = gradients1[i].clamp_(-10, 10).view(-1, 1).to(device)
        views1.append(view1)
        views2.append(view2)
    grad0 = torch.cat(views1, 0).to(device)
    grad1 = torch.cat(views2, 0).to(device)
    loss = criterion(grad0, grad1, mask)
    return loss

def phi_gradients(model, device):
    listOfGradients = []
    for p in model.parameters():
        listOfGradients.append(torch.zeros(list(p.size())).to(device))
    return listOfGradients

def norm_tensorlist(T1, device):
    sum_powers_T1 = torch.zeros(1).to(device)
    for i in range(len(T1)):
        sum_powers_T1 = sum_powers_T1 + torch.sum(torch.pow(T1[i], 2))
    return torch.sqrt(sum_powers_T1)

#######----------------------------------------------------------------------------------------------------
##
##           Luisa's stuff
##
######-----------------------------------------------------------------------------------------------------
class MamlModel(nn.Module):
    def __init__(self,
                 n_inputs,
                 n_outputs,
                 n_weights,
                 num_context_params,
                 device
                 ):
        """
        :param n_inputs:            the number of inputs to the network
        :param n_outputs:           the number of outputs of the network
        :param n_weights:           for each hidden layer the number of weights
        :param num_context_params:  number of additional inputs (trained together with rest)
        """
        super(MamlModel, self).__init__()

        # initialise lists for biases and fully connected layers
        self.weights = []
        self.biases = []

        # add one
        self.nodes_per_layer = n_weights + [n_outputs]

        # additional biases
        self.task_context = torch.zeros(num_context_params).to(device)
        self.task_context.requires_grad = True

        # set up the shared parts of the layers
        prev_n_weight = n_inputs + num_context_params
        for i in range(len(self.nodes_per_layer)):
            w = torch.Tensor(size=(prev_n_weight, self.nodes_per_layer[i])).to(device)
            w.requires_grad = True
            self.weights.append(w)
            b = torch.Tensor(size=[self.nodes_per_layer[i]]).to(device)
            b.requires_grad = True
            self.biases.append(b)
            prev_n_weight = self.nodes_per_layer[i]

        self._reset_parameters()

    def _reset_parameters(self):
        for i in range(len(self.nodes_per_layer)):
            stdv = 1. / math.sqrt(self.nodes_per_layer[i])
            self.weights[i].data.uniform_(-stdv, stdv)
            self.biases[i].data.uniform_(-stdv, stdv)

    def forward(self, x):

        if len(self.task_context) != 0:
            x = torch.cat((x, self.task_context.expand(x.shape[0], -1)), dim=1)
        else:
            x = torch.cat((x, self.task_context))

        for i in range(len(self.weights) - 1):
            x = F.relu(F.linear(x, self.weights[i].t(), self.biases[i]))
        y = F.linear(x, self.weights[-1].t(), self.biases[-1])

        return y


from rest_framework.serializers import ModelSerializer,HyperlinkedIdentityField,SerializerMethodField,ValidationError
from django.contrib.contenttypes.models import ContentType

from comments.models import Comment

def create_comment_serializer(model_type='post',slug=None,parent_id=None):
	class CommentCreateSerializer(ModelSerializer):
		class Meta:
			model=Comment
			fields=[
				'email',
				'content'
			]
		def __init__(self,*args,**kwargs):
			self.model_type=model_type
			self.slug=slug
			self.parent_obj=None
			if parent_id:
				parent_qs=Comment.objects.filter(id=parent_id)
				if parent_qs.exists() and parent_qs.count() == 1:
					self.parent_obj=parent_qs.first()

			return super(CommentCreateSerializer,self).__init__(*args,**kwargs)
		def validate(self,data):
			model_type=self.model_type
			model_qs=ContentType.objects.filter(model=model_type)
			if not model_qs.exists():
				raise ValidationError('This is not a valid content type.')
			someModel=model_qs.first().model_class()
			obj_qs=someModel.objects.filter(slug=self.slug)
			if not obj_qs.exists() or obj_qs.count() != 1:
				raise ValidationError('This is not a valid slug.')
			return data
		def create(self,validated_data):
			content=validated_data.get('content')
			email=validated_data.get('email')
			model_type=self.model_type
			slug=self.slug
			parent_obj=self.parent_obj
			comment=Comment.objects.create_by_model_type(
				model_type=model_type,
				slug=slug,
				email=email,
				content=content,
				parent_obj=parent_obj)
			return comment
	return CommentCreateSerializer

class CommentListSerializer(ModelSerializer):
	replycount=SerializerMethodField()
	url=HyperlinkedIdentityField(
			view_name='comments-api:detail',
			lookup_field='pk'
		)
	class Meta:
		model=Comment
		fields=[
			'url',
			'email',
			'post',
			'content',
			'replycount'
		]
	def get_replycount(self,obj):
		if obj.is_parent:
			return obj.children().count()
		return 0

class CommentChildDetailSerializer(ModelSerializer):
	class Meta:
		model=Comment
		fields=[
			'email',
			'content',
			'timestamp'
		]
class CommentDetailSerializer(ModelSerializer):
	replies=SerializerMethodField()
	post_url=SerializerMethodField()
	class Meta:
		model=Comment
		fields=[
			'email',
			'post_url',
			'content',
			'replies',
			'timestamp'
		]
		read_only_fields=[
			'email',
			'replies',
			'timestamp'
		]
	def get_replies(self,obj):
		if obj.is_parent:
			return CommentChildDetailSerializer(obj.children(),many=True).data
		return None
	def get_post_url(self,obj):
		return obj.post.get_api_absolute_url()


from results import Analysis
import plotly.express as px

class Plot:
    def __init__(self, budget, genre, n, v, l, plot):
        self.budget = budget
        self.genre = genre
        self.nudity = n
        self.violence = v
        self.language = l
        self.plot = plot

    def figure(self):
        og_data = Analysis(self.budget * 1000000, list(self.genre), self.nudity
                           , self.violence, self.language, self.plot)
        og_data1 = og_data.data()
        imdb_rating = og_data.rating()
        fig = px.scatter(data_frame=og_data1, x='Index', y='IMDB Rating', size='Budget',
                         hover_name='Name', hover_data=['IMDB Rating', 'Budget'])
        fig.add_scatter(x=[800], y=[float(imdb_rating)], mode='markers', name='OUR MOVIE',
                        marker=dict(size=20, color='Red', symbol='cross'),
                        hovertemplate=
                        'OUR MOVIE
'
                        '
' +
                        'IMDB Rating=%{y}
')
        fig.update_layout(plot_bgcolor='#00bd81',
                          paper_bgcolor='#00bd81',
                          font_color='Black', )
        fig.update_xaxes(gridcolor='Black')
        fig.update_yaxes(gridcolor='Black')
        return fig


import re
import requests

url = 'http://goo.gl/U7mSQl'
rsp = requests.get(url)
html = rsp.text
results = re.findall(r'[A-Za-z0-9]+\*\*\*', html)

for id in results:
    print(id)


_list = []

def isPrime(val):
    for n in _list:
        if ( 0 == val % n):
            return False

    return True

result = 0
for i in range(2, 2000000):

    if ( 0 == i%10000):
        print("progress : " + str(i))

    if ( isPrime(i)):
        _list.append(i)
        result += i

print(result)


def add_new_id(table):
    new_id = [0]
    for row in table:
            new_id.append(int(row[0]))
    new_id = max(new_id) + 1
    return str(new_id)

def find_line(table, question_id, id_number):

    line = 0
    if line < len(table) -1 and table[line][id_number] != str(question_id):
        line += 1
    return table[line]


'''
DP problem.
dp(i) will be minimum number of slices to partition s[i:]
# O(n2)
optimization: if n^2 is required overall,
create a new dp memoization structure that returns true if s[i:j] is
palindrome according to problem 5.
'''

class Solution:
    def minCut(self, s: str) -> int:
        # acceleration
        # this heavily improve my algorithm wtf?
        if s == s[::-1]: return 0
        for i in range(1, len(s)):
            if s[:i] == s[:i][::-1] and s[i:] == s[i:][::-1]:
                return 1
        if not s:
            return 0
        mem = [-1 for i in range(len(s) + 1)]
        memm = {} # [(start,end)]
        # dp problem: if s[start] == s[end], is_palin = is_palin[start + 1,end - 1]
        def is_palin(i,j ):
            if i > j:
                return True
            if i >= len(s):
                return True
            if j < 0:
                return True
            if (i,j) not in memm:
                memm[(i,j)] = s[i] == s[j] and is_palin(i+1, j - 1)
            return memm[(i,j )]
        for each_start in range(len(mem) - 2,-1, -1 ):
            min_slice = float('inf')
            for each_end in range(each_start, len(mem) - 1):
                # s[each_start: each_end + 1]
                if is_palin(each_start,each_end ):
                    current_slice = 1 + mem[each_end + 1]
                    min_slice = min(current_slice, min_slice)
            mem[each_start] = min_slice
        return mem[0]

a = 'aab'
print(Solution().minCut(a))


import datetime
import random
import re
import socket
import urllib.error
import urllib.request

from ircbot import log
import ircbot.plugin

class YouPornComment():
	_last_fetch = None
	THROTTLE_SECS = 15

	@classmethod
	def get_random(cls, include_url=False):
		now = datetime.datetime.now()
		if cls._last_fetch is not None:
			diff = now - cls._last_fetch
			if diff.seconds < (cls.THROTTLE_SECS):
				log.debug('YouPorn comment less than {secs} seconds old, blocking'.format(
					secs=cls.THROTTLE_SECS))
				return False

		cls._last_fetch = now
		result = cls._get_random(include_url)
		# try a 2nd time
		if not result:
			log.debug('Retrying')
			result = cls._get_random(include_url)

		return result

	@staticmethod
	def _get_random(include_url=False):
		try:
			response = urllib.request.urlopen('http://www.youporn.com/random/video/',
				timeout=2)
			result = response.read().decode()
			response.close()

			result = re.findall('
((?:.|\\n)*?)
', result)

			if not result:
				log.debug('No comments found in '+response.url)
				return None

			result = random.choice(result).strip().replace('\r', '').replace('\n', ' ')

			if ' ' not in result and '+' in result:
				result = result.replace('+', ' ')

			if include_url:
				# remove the long slug at the end of the URL
				url = '/'.join(response.url.split('/')[:-2])

				result = result + ' (' + url + ')'

			return result
		except (socket.timeout, urllib.error.URLError, UnicodeDecodeError):
			log.debug('HTTP request failed')
			pass

		return None

class Bitcoin:
	currencies = (
		'USD', 'EUR', 'hamburgers', 'farts', 'Razielcoins', 'BTC', 'salmons',
		'marble eggs in a shitty condom', 'typematrix keyboards', 'clean teeth',
		'dead Palestinian children', 'cmd.exe resizes', 'warp-in staplers',
		'mutalisks on creep', 'mutalisks off creep', 'floating cars',
		'burned rice', 'wordpress conference tickets', 'ice creams',
		'base64 encoded o\'reilly books', 'rolls of vitamin E toilet paper',
		'WISPY BEARDED POT SMOKING FAT FAGCUNT BITCOIN WORSHIPPERS WHO OBSESS OVER ME AND MAKE A SPORT OUT OF DRIVING ME INSANE AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA'
	)

	@classmethod
	def get_worth(cls):
		num = random.randint(100,100000) / 100
		currency = random.choice(cls.currencies)
		return '%.2f %s' % (num, currency)

class Raziel:
	nicks = ('radio', 'brazier', 'easel', 'raIel', 'easiek', 'ramen', 'russell')

	@classmethod
	def get_random_nick(cls):
		return random.choice(cls.nicks)

class RedditeuPlugin(ircbot.plugin.Plugin):
	"""#redditeu plugin."""
	@ircbot.plugin.command('btc')
	def get_btc_worth(self, cmd):
		return '1 bitcoin is currently worth ' + Bitcoin.get_worth()

	@ircbot.plugin.command('random')
	def get_yp_comment(self, cmd):
		result = YouPornComment.get_random(True)
		if result is False:
			return None

		if result:
			return result
		else:
			return 'No comment found, try again later!'

	@ircbot.plugin.command('michael')
	def who_is_michael(self, cmd):
		channel = self.bot.conn.channels.get(cmd.message.target)
		if not channel:
			return
		for (host,nick) in channel.host_map.items():
			if 'nevzetz' in host or 'ip51cc146b.speed.planet.nl' in host:
				return 'Michael is ' + nick
		return 'Michael not found!'

	@ircbot.plugin.join
	def welcome(self, user, channel):
		if 'happy0' in user.nick.lower():
			return 'ypyotootp hippy 0'
		if user.nick.lower().startswith('raziel'):
			return 'hello ' + Raziel.get_random_nick()


# 2018.07.16 Y.Mizoguchi
#  testing AI player for model1
#
import os, sys
os.environ['KERAS_BACKEND'] = 'theano'
import keras as ks
from keras.models import Sequential, load_model
from keras.layers import Dense, Dropout, Flatten, Conv2D, MaxPooling2D, Reshape
from keras.utils import np_utils
from keras.callbacks import CSVLogger, TensorBoard
from keras.utils.vis_utils import plot_model
from sklearn.datasets import fetch_mldata
import pandas as pd
import numpy as np
import matplotlib
import matplotlib.pyplot as plt
import pickle
from countinglib import vretrieve,int2board,boardprint,rotatecheck,transpose,\
                        board2int,turncheck,boardtex,texprint
import random
#
if len(sys.argv)==1:
    modelname='model1'
    epochs = 10
elif len(sys.argv)==2:
    modelname=sys.argv[1]
    epochs = 10
else:
    modelname=sys.argv[1]
    epochs = int(sys.argv[2])
#
perfectdata='./counting0.csv'
print('epochs = ',epochs)
perfectdata='./counting0.csv'
print('input perfectdata = '+perfectdata)
print('modelname = '+modelname)
#
df = pd.read_csv(perfectdata, header=None)
data = df.values.tolist()
#
Xy = list(map(lambda x:[int2board(x[0]),x[1]],data))
X = list(map(lambda x:x[0],Xy))
y = list(map(lambda x:x[1],Xy))
no = len(X)
no0 = int(no / 10)
#
Xy_test = random.sample(Xy,no0)
X_test = np.array(list(map(lambda x:np.array(x[0]),Xy_test)))
X_test = X_test.reshape(X_test.shape[0],16)
X_test = X_test.astype('float32')
y_test = np.array(list(map(lambda x:x[1],Xy_test)))
y_test_backup = y_test
y_test = np_utils.to_categorical(y_test, 3)
print('X_test.shape,y_test.shape')
print(X_test.shape,y_test.shape)
#
fp='./dump/'+modelname+"_{0:05d}".format(epochs)+'.h5'
print('Loading model structure from '+fp)
model=load_model(fp)
#
fcsv='./dump/'+modelname+"_{0:05d}".format(epochs)+'.csv'
fpdf='./dump/'+modelname+"_{0:05d}".format(epochs)+'.pdf'
df = pd.read_csv(fcsv)
data = df.values.tolist()
epochs = [x[0] for x in data]
acc =  [x[1] for x in data]
loss = [x[2] for x in data]
fig = plt.figure()
ax1 = fig.add_subplot(1,1,1)
p1 = ax1.plot(epochs, acc)
p2 = ax1.plot(epochs, loss)
plt.legend((p1[0], p2[0]), ("accuracy", "loss"), loc=3)
plt.title(modelname+' ('+str(epochs)+' epochs learning)')
ax1.set_xlabel('epochs')
ax1.set_ylabel('rate')
ax1.set_ylim(0,1)
ax1.grid(True)
# fig.show()
fig.savefig(fpdf)
print('Saving learning logs graph to '+fpdf)
#
print('Testing...')
loss_and_metrics = model.evaluate(X_test, y_test, batch_size=128)
print('Results: ',loss_and_metrics)


#! /usr/bin/env python
###############################################################################
#
# Copyright (C) 2015
# ASTRON (Netherlands Institute for Radio Astronomy) <http://www.astron.nl/>
# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program.  If not, see <http://www.gnu.org/licenses/>.
#
###############################################################################

"""Test case for unb1_ddr3_reorder

Description:
  Tx seq --> Reorder_transpose --> Rx seq
               |         |
             WR DDR    RD DDR
Usage:
  > simulate tb_unb1_ddr3_reorder.vhd
  > python tc_unb1_ddr3_reorder_seq.py --sim --unb 0 --bn 3 --rep 2 -v 5
  to run the test again (eg. also with other verbosity):
  > python tc_unb1_ddr3_reorder_seq.py --sim --unb 0 --bn 3 --rep 2 -v 3 --read     # continue rx seq checking
  > python tc_unb1_ddr3_reorder_seq.py --sim --unb 0 --bn 3 --rep 2 -v 5 --rerun    # restart rx seq checking
"""

###############################################################################
# System imports
import test_case
import node_io
import pi_diag_block_gen
import pi_diag_tx_seq
import pi_diag_data_buffer
import pi_diag_rx_seq
import pi_io_ddr
import pi_ss_ss_wide
import pi_bsn_monitor
import mem_init_file

from tools import *
from common import *
from pi_common import *

##################################################################################################################

if __name__ == "__main__":

    # Create a test case object

    tc = test_case.Testcase('TB - ', '')
    tc.set_result('PASSED')
    tc.append_log(3, '>>>')
    tc.append_log(1, '>>> Title : Test case for the unb1_ddr3_reorder design on %s' % tc.unb_nodes_string())
    tc.append_log(3, '>>>')
    tc.append_log(3, '>>> Testcase run mode: %s' % tc.runmode)
    tc.append_log(3, '')

    c_ena_pre_transpose = True
    c_nof_streams   = 1
    c_data_w_ratio  = 4
    c_wr_chunksize  = 176       # must be a factor of c_data_w_ratio=4
    c_rd_chunksize  = 16        # must be a factor of c_data_w_ratio=4
    c_rd_nof_chunks = 11        # must be c_wr_chunksize = c_rd_nof_chunks*c_rd_chunksize
    c_rd_interval   = c_rd_chunksize
    c_gapsize       = 0         # This specifies the gapsize in the sequencer and NOT in the blockgenerator
    c_blocksize     = c_wr_chunksize + c_gapsize
    c_nof_blocks    = sel_a_b(tc.sim, 16, 800000)  # must be multiple of c_rd_chunksize
    c_pagesize      = c_nof_blocks*c_blocksize

    if c_ena_pre_transpose:
      c_bg_blocksize = 256
      c_bg_gapsize   = 0
    else:
      c_bg_blocksize = c_wr_chunksize
      c_bg_gapsize    = 256 - c_wr_chunksize

    tc.append_log(3, 'Script transpose parameters, these must match the compile parameters of the design:')
    tc.append_log(3, '   . c_ena_pre_transpose  = %s ' % c_ena_pre_transpose)
    tc.append_log(3, '   . c_nof_streams        = %d ' % c_nof_streams)
    tc.append_log(3, '   . c_data_w_ratio       = %d ' % c_data_w_ratio)
    tc.append_log(3, '   . c_wr_chunksize       = %d ' % c_wr_chunksize)
    tc.append_log(3, '   . c_rd_chunksize       = %d ' % c_rd_chunksize)
    tc.append_log(3, '   . c_rd_nof_chunks      = %d ' % c_rd_nof_chunks)
    tc.append_log(3, '   . c_rd_interval        = %d ' % c_rd_interval)
    tc.append_log(3, '   . c_gapsize            = %d ' % c_gapsize)
    tc.append_log(3, '   . c_blocksize          = %d ' % c_blocksize)
    tc.append_log(3, '   . c_nof_blocks         = %d ' % c_nof_blocks)
    tc.append_log(3, '   . c_pagesize           = %d ' % c_pagesize)
    tc.append_log(3, '')

    # Create access object for all nodes
    io = node_io.NodeIO(tc.nodeImages, tc.base_ip)

    # Create instances for the periperals
    bg = pi_diag_block_gen.PiDiagBlockGen(tc, io, nofChannels=c_nof_streams, ramSizePerChannel=2**14)
    db = pi_diag_data_buffer.PiDiagDataBuffer(tc, io, nofStreams=c_nof_streams)
    tx_seq = pi_diag_tx_seq.PiDiagTxSeq(tc, io, nof_inst=c_nof_streams)
    rx_seq = pi_diag_rx_seq.PiDiagRxSeq(tc, io, nof_inst=c_nof_streams)

    # BSN monitors
    bsn_bg = 0   # bg_sosi_arr
    bsn_wr = 1   # to_mem_sosi
    bsn_rd = 2   # from_mem_sosi
    bsn_db = 3   # db_sosi_arr
    nof_bsn = 4

    bsn = pi_bsn_monitor.PiBsnMonitor(tc, io, nofStreams=nof_bsn)

    # Create object for DDR register map
    ddr_status = pi_io_ddr.PiIoDdr(tc, io, nof_inst=1)

    # Create subandselect instance for pre-transpose.
    ss = pi_ss_ss_wide.PiSsSsWide (tc, io, c_wr_chunksize*c_rd_chunksize, c_nof_streams)

    ################################################################################
    # Wait for
    ################################################################################

    # Wait for power up (reset release)
    io.wait_for_time(hw_time=0.01, sim_time=(1, 'us'))

    # Wait for DDR memory available
    do_until_eq(ddr_status.read_init_done, ms_retry=3000, val=1, s_timeout=3600)

    if tc.run:
        ################################################################################
        # Write initialisations
        ################################################################################

        # Init pre-transpose reorder
        if c_ena_pre_transpose:
            ss_list = []
            for i in range(c_wr_chunksize):
                for j in range(c_rd_chunksize):
                   ss_list.append(i + j*c_bg_blocksize)
            ss.write_selects(ss_list)

        # Init rx seq based on description in reorder_sequencer.vhd
        if c_ena_pre_transpose:
            step_0 = 1
            step_1 = c_bg_blocksize  #
            step_2 = 1 + c_bg_blocksize - c_bg_blocksize * c_nof_blocks
            step_3 = 1 + (c_rd_chunksize - c_rd_nof_chunks) * c_rd_chunksize
        else:
            step_0 = 1
            step_1 = step_0 + c_blocksize * c_rd_interval - c_rd_chunksize  # Step due to next burst access
            step_2 = step_0 + c_blocksize * c_rd_interval - c_pagesize      # step due to page swap
            step_3 = 1

        tc.append_log(3, 'Rx seq step sizes:')
        tc.append_log(3, '   . step_0  = %d ' % step_0)
        tc.append_log(3, '   . step_1  = %d ' % step_1)
        tc.append_log(3, '   . step_2  = %d ' % step_2)
        tc.append_log(3, '   . step_3  = %d ' % step_3)
        tc.append_log(3, '')

        # use to_unsigned() to pass on negative values to MM reg
        rx_seq.write_step_3(StepSize=to_unsigned(step_3, c_word_w))
        rx_seq.write_step_2(StepSize=to_unsigned(step_2, c_word_w))
        rx_seq.write_step_1(StepSize=to_unsigned(step_1, c_word_w))
        rx_seq.write_step_0(StepSize=to_unsigned(step_0, c_word_w))

        # Init bg
        bg.write_block_gen_settings(samplesPerPacket=c_bg_blocksize, blocksPerSync=c_nof_blocks, gapSize=c_bg_gapsize, BSNInit=0)

        ################################################################################
        # Block sequence start
        ################################################################################
        rx_seq.write_enable_cntr()
        tx_seq.write_enable_cntr()
        bg.write_enable()

    if tc.rerun:
        # Restart Rx verification
        rx_seq.write_disable()
        io.wait_for_time(hw_time=0.01, sim_time=(1, 'us'))
        rx_seq.write_enable_cntr()

    if tc.read:
        # Continue Rx verification
        pass

    ################################################################################
    # Monitor results
    ################################################################################

    # Control defaults
    nof_mon = sel_a_b(tc.sim, 1, 2)

    # Tx sequence monitor
    for mon in range(nof_mon):
        io.wait_for_time(hw_time=0.01, sim_time=(1, 'us'))
        tx_seq.read_cnt()

    # Wait for rx data
    do_until_gt(rx_seq.read_cnt, ms_retry=3000, val=0, s_timeout=3600)
    io.wait_for_time(hw_time=0.1, sim_time=(1, 'us'))   # wait some more to ensure all nodes are busy with rx data

    for rep in range(tc.repeat):
        tc.append_log(5, '')
        tc.append_log(3, '>>> Rep-%d' % rep)

        # Monitor
        for mon in range(nof_mon):
            io.wait_for_time(hw_time=0.01, sim_time=(1, 'us'))
            rx_seq.read_cnt()

        bsn.read_bsn_monitor(streamNr=bsn_db, bsnInit=0, vLevel=3)

        # Verify Rx result
        rx_seq.read_result()
        if tc.get_result() != "PASSED":
            break

        # Wait some interval until to verify Rx result again
        io.wait_for_time(hw_time=3, sim_time=(10, 'us'))

    # End
    tc.set_section_id('')
    tc.append_log(3, '')
    tc.append_log(3, '>>>')
    tc.append_log(0, '>>> Test bench result: %s' % tc.get_result())
    tc.append_log(0, '>>> Runtime=%f seconds (%f hours)' % (tc.get_run_time(),tc.get_run_time()/3600))
    tc.append_log(3, '>>>')


import numpy as np
import pandas as pd
import copy

class TaggedString():
    def __init__(self, string, tag=None):
        self.string = string
        self.tag = tag
    def intersects(self, other):
        return self.string == other.string
        # return self.string in other.string or other.string in self.string
    def __getitem__(self, item):
        raise Exception("no get item")
    def __lt__(self, other):
        return self.string < other.string
    def __eq__(self, other):
        return self.string == other.string
    def __hash__(self):
        return 1
    def __repr__(self):
        return self.string

class VectorRange():
    def __init__(self, start_vector, end_vector, tag=None):
        assert len(start_vector) == len(end_vector)
        self.numdims = len(start_vector)
        for i in range(len(start_vector)):
            assert start_vector[i] <= end_vector[i]
        self.start_vector = start_vector
        self.end_vector = end_vector
        self.tag = tag
    def intersects(self, other):
        for i in range(self.numdims):
            if self.start_vector[i] > other.end_vector[i] or self.end_vector[i] < other.start_vector[i]:
                return False
        return True
    def __getitem__(self, item):
        return self.start_vector[item]
    def __lt__(self, other):
        return self.start_vector[0] < other.start_vector[0]
    def __eq__(self, other):
        assert self.numdims == other.numdims
        if self.intersects(other):
            for i in range(self.numdims):
                self.start_vector[i] = min(self.start_vector[i], other.start_vector[i])
                self.end_vector[i] = max(self.end_vector[i], other.end_vector[i])
                other.start_vector[i] = self.start_vector[i]
                other.end_vector[i] = self.end_vector[i]
            return True
        else:
            return False
    def __hash__(self):
        return 1
    def __repr__(self):
        return str((list(self.start_vector), list(self.end_vector)))

class SeqRange(VectorRange):
    def __init__(self, startend, tag=None):
        super().__init__([startend[0]], [startend[1]], tag)
        self.startend = startend
    def __getitem__(self, item):
        return self.startend[item]

def unionize_vectorrange_sequence(vectorranges):
    vectorranges = copy.deepcopy(vectorranges)
    for dim in range(vectorranges[0].numdims):
        sortedvectorranges = sorted(vectorranges, key=lambda x:x[dim])
        vectorranges = sorted(list(set(sortedvectorranges)))
    return vectorranges

def unionize_range_sequence(ranges):
    return sorted(list(set([SeqRange(x) for x in sorted(ranges)])))

def merge_vectorranges(annotations):
    non_empty = [anno for anno in annotations if len(anno) > 0]
    frac_non_empty = len(non_empty) / len(annotations)
    if frac_non_empty < 0.5:
        return []
    else:
        vrs = [vr for annotation in non_empty for vr in annotation]
        start = [int(x) for x in np.round(np.array([vr.start_vector for vr in vrs]).mean(axis=0))]
        end = [int(x) for x in np.round(np.array([vr.end_vector for vr in vrs]).mean(axis=0))]
        if isinstance(vrs[0], SeqRange):
            return [SeqRange((start, end))]
        else:
            return [VectorRange(start, end)]

def fragment_by_overlaps(experiment, use_oracle=False):
    return _fragment_by_overlaps(experiment.annodf,
                                 experiment.uid_colname,
                                 experiment.item_colname,
                                 experiment.label_colname,
                                 experiment.golddict if use_oracle else None)

def _fragment_by_overlaps(annodf, uid_colname, item_colname, label_colname, oracle_golddict=None):
    resultdfs = []
    for item_id in annodf[item_colname].unique():
        idf = annodf[annodf[item_colname] == item_id]
        vectorranges = [vr for annotation in idf[label_colname].values for vr in annotation]
        if oracle_golddict is not None:
            unranges = []
            try:
                gold_vrs = [vr for vr in oracle_golddict.get(item_id)]
                orbuckets = {}
                for vr in vectorranges:
                    vr_orbucket = np.argmin([dist_center(vr, gold_vr) for gold_vr in gold_vrs])
                    orbuckets.setdefault(vr_orbucket, []).append(vr)
                for orbucket in orbuckets.values():
                    minstart = np.min([vr.start_vector for vr in orbucket], axis=0)
                    maxend = np.max([vr.end_vector for vr in orbucket], axis=0)
                    unranges.append(VectorRange(minstart, maxend))
            except:
                pass
        else:
            unranges = unionize_vectorrange_sequence(vectorranges)
        origItemID = []
        newItemID = []
        newItemVR = []
        uid = []
        label = []
        gold = []
        for unrange in unranges:
            for i, row in idf.iterrows():
                origItemID.append(idf[item_colname].values[0])
                newItemID.append(F"{idf[item_colname].values[0]}-{unrange}")
                newItemVR.append(unrange)
                uid.append(row[uid_colname])
                label.append([vr for vr in row[label_colname] if unrange.intersects(vr)])
                gold.append(None)
        resultdfs.append(pd.DataFrame({"origItemID":origItemID, "newItemID":newItemID, "newItemVR":newItemVR, uid_colname:uid, label_colname:label, "gold":gold}))
    return pd.concat(resultdfs)

def dist_center(vr1, vr2):
    vr1c = (np.array(vr1.start_vector) + np.array(vr1.end_vector)) / 2
    vr2c = (np.array(vr2.start_vector) + np.array(vr2.end_vector)) / 2
    return np.sum(np.abs(vr1c - vr2c))