# -*- coding: utf-8 -*-
"""
Created on Tue Mar 20 16:48:04 2018

@author: so041e
"""

# XB boost + Neural network model
#
#
#

import numpy as np
import pandas as pd
import xgboost as xgb
import zipfile,os,sys

from datetime import datetime
from sklearn.preprocessing import LabelEncoder
from sklearn import preprocessing
from sklearn.preprocessing import OneHotEncoder

from xgboost.sklearn import XGBClassifier
#from xgboost.sklearn import LGBMClassifier
#from sklearn.grid_search import GridSearchCV
from sklearn.model_selection import GridSearchCV
from sklearn.preprocessing import StandardScaler

if sys.platform=='win32':
	Data_path="d:/tmp"
elif sys.platform=='linux2':
	Data_path='.'
elif sys.platform=='darwin':
	Data_path="~/tmp"
else:
	print (os.name)

#with zipfile.ZipFile(Data_path+"/zillow/properties_2016.csv.zip") as z:
#	properties=pd.read_csv(z.open('Properties_2016.csv'))
otrain = pd.read_csv('train_data.csv',skiprows=1,names=['imsi','et','byte','man','model','pro','rt','enb2','freq','bw','rsrp','rsrq','lt','uen','thr'])
otest = pd.read_csv('test_data_without_throughput.csv',skiprows=1,names=['imsi','et','byte','man','model','pro','rt','enb2','freq','bw','rsrp','rsrq','lt','uen'])
train=otrain.copy()
test=otest.copy()
cols=train.columns
#all_cols.sort()

#[(0, 'imsi'),             too many 	 not sure
# (1, 'event_time'),       all 0,      no
# (2, 'bytes'),            int         y
# (3, 'manufacturer'),     cat         y
# (4, 'model'),            cat         not sure
# (5, 'protocol'),         cat         y
# (6, 'rrc_time'),         all 0,      no
# (7, 'EUTRANCELL_NAME'),  just cell   y
# (8, 'FREQUENCY'),        cat         y
# (9, 'BANDWIDTH'),        int         y
# (10, 'rsrp'),            int         y
# (11, 'rsrq'),            int         y
# (12, 'load_time'),                   no
# (13, 'num_ue'),          int         y
# (14, 'throughput')]      float       TARGET
EXP=False

if EXP:
	train[cols[11]].value_counts().sort_index()
	train[cols[13]].value_counts().sort_index().plot()

train['enb']=train.enb2.apply(lambda x: x.split('_')[0])
test['enb']=test.enb2.apply(lambda x: x.split('_')[0])

# Exclude columns
#nouse_cols=['latitude','longitude','rawcensustractandblock','propertycountylandusecode','unitcnt','roomcnt']
#use_cols=[x for x in all_cols if x not in nouse_cols]
#ndf=properties[use_cols]

#train['et']=train.event_time.apply(lambda x: x.split()[1].split(':')[0])
#train_tar=np.log(train['thr'])  # LOG
train_tar=train['thr']
del train['thr']

#%% Handle nan values
#ndf[col]=properties[col].astype(float)
#ndf[col]=properties[col].astype(float).fillna(0)

#['imsi', 'et', 'byte', 'man', 'model', 'pro', 'rt', 'enb2', 'freq', 'bw','rsrp', 'rsrq', 'lt', 'uen', 'thr']
num_col=['byte','rsrp','rsrq','uen']
#lab_col=['man','model','enb','freq','bw']
lab_col=['imsi','freq','bw'] #'enb','model','man',

for col in lab_col:
	lbl = LabelEncoder()
	vals=train[col].astype('str')
	lbl.fit(vals)
	train[col] = lbl.transform(vals)

	test_vals=test[col].astype('str')
	test[col] = lbl.transform(test_vals)

#OneHotEncoder(categorical_features='all', dtype=<'numpy.float64'>, handle_unknown='error', n_values='auto', sparse=True)
lab_train=[]
lab_test=[]
for col in lab_col:
	ncol=pd.get_dummies(train[col],col)
	lab_train.append(ncol)
	ncol=pd.get_dummies(test[col],col)
	lab_test.append(ncol)
train_lab=pd.concat(lab_train,1)
test_lab=pd.concat(lab_test,1)

SCALER=True
if SCALER:
	scalerX = StandardScaler().fit(train[num_col])  # fit y scaler
	train_num=scalerX.transform(train[num_col])  # fit your pipeline to scaled Y
	test_num=scalerX.transform(test[num_col])  # fit your pipeline to scaled Y
else:
	train_num=train[num_col]
	test_num=test[num_col]

train_data=pd.concat([pd.DataFrame(train_num),train_lab],1)
test_data=pd.concat([pd.DataFrame(test_num),test_lab],1)

dtrain = xgb.DMatrix(train_data, train_tar)
dtest = xgb.DMatrix(test_data)

#%% Merge data and trim outliers
OUTLIER=False
if OUTLIER:
	train_df = train.merge(nnndf, how='left', on='parcelid')
	x_train=train_df.drop(['parcelid', 'logerror','transactiondate'], axis=1)
	y_train = train_df["logerror"].values.astype(np.float32)

	# Trim outliers for training
	train_df_trim=train_df[ (train_df.logerror > -0.4) & (train_df.logerror < 0.42) ]
	x_train_trim=train_df_trim.drop(['parcelid', 'logerror','transactiondate'], axis=1)
	y_train_trim = train_df_trim["logerror"].values.astype(np.float32)
	x_test = nnndf.drop(['parcelid'], axis=1)

	print('After removing outliers:')
	print('Shape train: {}\nShape test: {}'.format(x_train.shape, x_test.shape))

#%% Parameter tuning
xgb_params = {
		'eta': 0.11,
		'max_depth': 4,
		'subsample': 0.95,
		'objective': 'reg:linear',
		'eval_metric': 'mae',  #mae
		#'feval':evalerror,
		'base_score': train_data.mean(),
		#'nthread':8,
		'min_child_weight':2, #default 1
		'alpha':3.0, #default 0 - L1 regularize
		'lambda':1.0,
		#'gamma':0,
		#'max_leaves':0,
		#'grow_policy':'lossguide',
		#'nthread':8,
		'silent': 0
}
#%%
param = {'alpha':4.9,'max_depth':15, 'eta':0.8, 'silent':0, 'objective':'reg:linear'} #'min_child_weight':10,
#num_round = 5
#bst = xgb.train(param, dtrain, num_round)
CrossVal=True
if CrossVal:
	cv_result = xgb.cv(param, dtrain, nfold=10, num_boost_round=40)
	cv_result[['test-rmse-mean','train-rmse-mean']].plot()
	print (cv_result.tail(1))

#%%
TRAIN_WHOLE=True
if TRAIN_WHOLE:
	param = {'alpha':4.9, 'max_depth':25, 'eta':0.8, 'silent':0, 'objective':'reg:linear'} #'min_child_weight':10,
	model = xgb.train(param, dtrain, num_boost_round=10)
	pred = model.predict(dtest)

otest['Throughput (in Kbps)']=pred #np.exp(pred) #LOG
otest[['imsi','Throughput (in Kbps)']].to_csv('gv1.csv',index=False)

#%%
sys.exit()

xgb_params[parameter]=vv
cv_result = xgb.cv(xgb_params, dtrain, nfold=5, num_boost_round=500, early_stopping_rounds=250,  verbose_eval=10, show_stdv=False)
cv_results[(parameter,vv)]=cv_result.tail(1)
cv_result[['test-mae-mean','train-mae-mean']].plot()

CV=False
if CV:
	cv_results={}
	parameter='alpha'
	values=[3]
	for vv in values:
		xgb_params[parameter]=vv
		cv_result = xgb.cv(xgb_params, dtrain, nfold=5, num_boost_round=500, early_stopping_rounds=250,  verbose_eval=10, show_stdv=False)
		cv_results[(parameter,vv)]=cv_result.tail(1)
		cv_result[['test-mae-mean','train-mae-mean']].plot()
		print(cv_results)

	cvr=pd.concat(cv_results)
	cvr[['test-mae-mean','train-mae-mean']].plot()
	print (cvr)

#%%
TRAIN_WHOLE=True
if TRAIN_WHOLE:
	model = xgb.train(dict(xgb_params), dtrain, num_boost_round=NUM_ROUND)

	# Validate the model ------------------------------------------------------
	y_train_pre = model.predict(xgb.DMatrix(x_train_sc)).round(4)
	y_train_trim_pre = model.predict(xgb.DMatrix(x_train_trim_sc)).round(4)

	train_loss=y_train-y_train_pre
	train_trim_loss=y_train_trim-y_train_trim_pre
	print ("\nUnTrain Loss:\t", sum(abs(y_train)).round(),sum(y_train).round())
	print ("\nTrain Loss:\t",  sum(abs(train_loss)).round(),sum(train_loss).round())
	print ("\nTrim Tr Loss:\t",  sum(abs(train_trim_loss)).round(),sum(train_trim_loss).round())

#%%
TEST = True

if TEST:
	# Test -----------------------------------------------------------------------		x_test_sc=scalerX.transform(x_test.fillna(-1))  # fit your pipeline to scaled Y
	x_test_sc=scalerX.transform(x_test.fillna(-1))  # fit your pipeline to scaled Y
	y_pred = model.predict(xgb.DMatrix(x_test_sc))

#%%
sys.exit()


from flask import Blueprint, render_template, jsonify, redirect

viewer = Blueprint("viewer", __name__, template_folder="templates")

# @viewer.route("/view/<db>/<dataset_id>")
def view_s3(public_db, dataset_id):

    if public_db not in ("sra", "img"):
        return "Database not supported", 404

    import boto3

    conn = boto3.client("s3")

    key = f"sigs/{dataset_id}.sig"

    params = {
        "Bucket": f"wort-{public_db}",
        "Key": key,
        "ResponseContentType": "application/json",
        "ResponseContentEncoding": "gzip",
    }

    url = conn.generate_presigned_url("get_object", Params=params, ExpiresIn=100)

    return redirect(url)


from functools import partial, singledispatch
from typing import Any, Callable, List, Optional, Tuple, Union

import numba
import numpy as np
import pyarrow as pa
from numba import njit
from pandas.core import nanops

def _calculate_chunk_offsets(chunked_array: pa.ChunkedArray) -> np.ndarray:
    """Return an array holding the indices pointing to the first element of each chunk."""
    offset = 0
    offsets = []
    for chunk in chunked_array.iterchunks():
        offsets.append(offset)
        offset += len(chunk)
    return np.array(offsets)

def _extract_data_buffer_as_np_array(array: pa.Array) -> np.ndarray:
    """Extract the data buffer of a numeric-typed pyarrow.Array as an np.ndarray."""
    dtype = array.type.to_pandas_dtype()
    start = array.offset
    end = array.offset + len(array)
    return np.asanyarray(array.buffers()[1]).view(dtype)[start:end]

@njit
def _extract_isnull_bytemap(
    bitmap: bytes,
    bitmap_length: int,
    bitmap_offset: int,
    dst_offset: int,
    dst: np.ndarray,
) -> None:
    """Write the values of a valid bitmap as bytes to a pre-allocatored isnull bytemap.

    Parameters
    ----------
    bitmap: pyarrow.Buffer
        bitmap where a set bit indicates that a value is valid
    bitmap_length: int
        Number of bits to read from the bitmap
    bitmap_offset: int
        Number of bits to skip from the beginning of the bitmap.
    dst_offset: int
        Number of bytes to skip from the beginning of the output
    dst: numpy.array(dtype=bool)
        Pre-allocated numpy array where a byte is set when a value is null
    """
    for i in range(bitmap_length):
        idx = bitmap_offset + i
        byte_idx = idx // 8
        bit_mask = 1 << (idx % 8)
        dst[dst_offset + i] = (bitmap[byte_idx] & bit_mask) == 0

def extract_isnull_bytemap(array: Union[pa.ChunkedArray, pa.Array]) -> np.ndarray:
    """
    Extract the valid bitmaps of a (chunked) array into numpy isnull bytemaps.

    Parameters
    ----------
    array
        Array from which we extract the validity bits as bytes

    Returns
    -------
    valid_bytemap
    """
    if array.null_count == len(array):
        return np.ones(len(array), dtype=bool)

    if isinstance(array, pa.ChunkedArray):
        result = np.zeros(len(array), dtype=bool)
        if array.null_count == 0:
            return result

        offset = 0
        for chunk in array.chunks:
            if chunk.null_count > 0:
                _extract_isnull_bytemap(
                    chunk.buffers()[0], len(chunk), chunk.offset, offset, result
                )
            offset += len(chunk)
    else:
        valid_bitmap = array.buffers()[0]
        if valid_bitmap:
            # TODO: Can we use np.empty here to improve performance?
            result = np.zeros(len(array), dtype=bool)
            # TODO(ARROW-2664): We only need to following line to support
            #   executing the code in disabled-JIT mode.
            buf = memoryview(valid_bitmap)
            _extract_isnull_bytemap(buf, len(array), array.offset, 0, result)
        else:
            result = np.full(len(array), False)

    return result

@njit
def isnull(sa):
    result = np.empty(sa.size, np.uint8)
    _isnull(sa, 0, result)
    return result

@njit
def _isnull(sa, offset, out):
    for i in range(sa.size):
        out[offset + i] = sa.isnull(i)

@njit
def _startswith(sa, needle, na, offset, out):
    for i in range(sa.size):
        if sa.isnull(i):
            out[offset + i] = na
            continue

        if sa.byte_length(i) < needle.length:
            out[offset + i] = 0
            continue

        for j in range(needle.length):
            if sa.get_byte(i, j) != needle.get_byte(j):
                out[offset + i] = 0
                break

        else:
            out[offset + i] = 1

@njit
def _endswith(sa, needle, na, offset, out):
    for i in range(sa.size):
        if sa.isnull(i):
            out[offset + i] = na
            continue

        string_length = sa.byte_length(i)
        needle_length = needle.length
        if string_length < needle.length:
            out[offset + i] = 0
            continue

        for j in range(needle_length):
            if sa.get_byte(i, string_length - needle_length + j) != needle.get_byte(j):
                out[offset + i] = 0
                break

        else:
            out[offset + i] = 1

@njit
def str_length(sa):
    result = np.empty(sa.size, np.uint32)

    for i in range(sa.size):
        result[i] = sa.length(i)

    return result

@njit(locals={"valid": numba.bool_, "value": numba.bool_})
def _any_op(length: int, valid_bits: bytes, data: bytes) -> int:
    for i in range(length):
        byte_offset = i // 8
        bit_offset = i % 8
        mask = np.uint8(1 << bit_offset)
        valid = valid_bits[byte_offset] & mask
        value = data[byte_offset] & mask
        if (valid and value) or (not valid):
            return True

    return False

@njit(locals={"valid": numba.bool_, "value": numba.bool_})
def _any_op_skipna(length: int, valid_bits: bytes, data: bytes) -> bool:
    for i in range(length):
        byte_offset = i // 8
        bit_offset = i % 8
        mask = np.uint8(1 << bit_offset)
        valid = valid_bits[byte_offset] & mask
        value = data[byte_offset] & mask
        if valid and value:
            return True

    return False

@njit(locals={"value": numba.bool_})
def _any_op_nonnull(length: int, data: bytes) -> bool:
    for i in range(length):
        byte_offset = i // 8
        bit_offset = i % 8
        mask = np.uint8(1 << bit_offset)
        value = data[byte_offset] & mask
        if value:
            return True

    return False

def any_op(arr: Union[pa.ChunkedArray, pa.Array], skipna: bool) -> bool:
    if isinstance(arr, pa.ChunkedArray):
        return any(any_op(chunk, skipna) for chunk in arr.chunks)

    if arr.null_count == 0:
        return _any_op_nonnull(len(arr), arr.buffers()[1])
    if skipna:
        return _any_op_skipna(len(arr), *arr.buffers())
    return _any_op(len(arr), *arr.buffers())

@njit(locals={"valid": numba.bool_, "value": numba.bool_})
def _all_op(length: int, valid_bits: bytes, data: bytes) -> bool:
    # This may be specific to Pandas but we return True as long as there is not False in the data.
    for i in range(length):
        byte_offset = i // 8
        bit_offset = i % 8
        mask = np.uint8(1 << bit_offset)
        valid = valid_bits[byte_offset] & mask
        value = data[byte_offset] & mask
        if valid and not value:
            return False
    return True

@njit(locals={"value": numba.bool_})
def _all_op_nonnull(length: int, data: bytes) -> bool:
    for i in range(length):
        byte_offset = i // 8
        bit_offset = i % 8
        mask = np.uint8(1 << bit_offset)
        value = data[byte_offset] & mask
        if not value:
            return False
    return True

def all_op(arr: Union[pa.ChunkedArray, pa.Array], skipna: bool) -> bool:
    if isinstance(arr, pa.ChunkedArray):
        return all(all_op(chunk, skipna) for chunk in arr.chunks)

    if arr.null_count == 0:
        return _all_op_nonnull(len(arr), arr.buffers()[1])
    # skipna is not relevant in the Pandas behaviour
    return _all_op(len(arr), *arr.buffers())

def np_reduce_op(
    npop: Callable,
    arr: Union[pa.ChunkedArray, pa.Array],
    skipna: bool = True,
    identity: Optional[int] = None,
):
    """Use numpy operations to provide a reduction."""
    if not skipna and arr.null_count > 0:
        return pa.NA
    elif isinstance(arr, pa.ChunkedArray):
        valid_chunks = [
            chunk for chunk in arr.iterchunks() if chunk.null_count != len(chunk)
        ]
        results = pa.array(
            [
                np_reduce_op(npop, chunk, skipna=skipna, identity=identity)
                for chunk in valid_chunks
            ]
        )
        return np_reduce_op(npop, results, skipna=skipna, identity=identity)
    else:
        if len(arr) == 0:
            if identity is None:
                raise ValueError("zero-size reduction on operation with no identity")
            else:
                return identity
        np_arr = _extract_data_buffer_as_np_array(arr)
        if arr.null_count > 0:
            mask = extract_isnull_bytemap(arr)
            return npop(np_arr[~mask])
        else:
            return npop(np_arr)

sum_op = partial(np_reduce_op, np.sum, identity=0)
max_op = partial(np_reduce_op, np.amax)
min_op = partial(np_reduce_op, np.amin)
prod_op = partial(np_reduce_op, np.prod, identity=1)

def pd_nanop(nanop: Callable, arr: Union[pa.ChunkedArray, pa.Array], skipna: bool):
    """Use pandas.core.nanops to provide a reduction."""
    if isinstance(arr, pa.ChunkedArray):
        data = pa.concat_arrays(arr.iterchunks())
    else:
        data = arr
    np_arr = _extract_data_buffer_as_np_array(data)
    mask = extract_isnull_bytemap(data)

    return nanop(np_arr, skipna=skipna, mask=mask)

std_op = partial(pd_nanop, nanops.nanstd)
skew_op = partial(pd_nanop, nanops.nanskew)
kurt_op = partial(pd_nanop, nanops.nankurt)
var_op = partial(pd_nanop, nanops.nanvar)
median_op = partial(pd_nanop, nanops.nanmedian)

def _in_chunk_offsets(
    arr: pa.ChunkedArray, offsets: List[int]
) -> List[Tuple[int, int, int]]:
    """Calculate the access ranges for a given list of offsets.

    All chunk start indices must be included as offsets and the offsets must be
    unique.

    Returns a list of tuples that contain:
     * The index of the given chunk
     * The position inside the chunk
     * The length of the current range
    """
    new_offsets = []
    pos = 0
    chunk = 0
    chunk_pos = 0
    for offset, offset_next in zip(offsets, offsets[1:] + [len(arr)]):
        diff = offset - pos
        chunk_remains = len(arr.chunk(chunk)) - chunk_pos
        step = offset_next - offset
        if diff == 0:  # The first offset
            new_offsets.append((chunk, chunk_pos, step))
        elif diff == chunk_remains:
            chunk += 1
            chunk_pos = 0
            pos += chunk_remains
            new_offsets.append((chunk, chunk_pos, step))
        else:  # diff < chunk_remains
            chunk_pos += diff
            pos += diff
            new_offsets.append((chunk, chunk_pos, step))
    return new_offsets

def _combined_in_chunk_offsets(
    a: pa.ChunkedArray, b: pa.ChunkedArray
) -> Tuple[List[Tuple[int, int, int]], List[Tuple[int, int, int]]]:
    offsets_a = _calculate_chunk_offsets(a)
    offsets_b = _calculate_chunk_offsets(b)
    offsets = sorted(set(list(offsets_a) + list(offsets_b)))
    in_a_offsets = _in_chunk_offsets(a, offsets)
    in_b_offsets = _in_chunk_offsets(b, offsets)
    return in_a_offsets, in_b_offsets

@singledispatch
def np_ufunc_op(a: Any, b: Any, op: Callable):
    """Apply a NumPy ufunc where at least one of the arguments is an Arrow structure."""
    # a is neither a pa.Array nor a pa.ChunkedArray, we expect only numpy.ndarray or scalars.
    if isinstance(b, pa.ChunkedArray):
        if np.isscalar(a):
            new_chunks = []
            for chunk in b.iterchunks():
                new_chunks.append(np_ufunc_op(a, chunk, op))
            return pa.chunked_array(new_chunks)
        else:
            new_chunks = []
            offsets = _calculate_chunk_offsets(b)
            for chunk, offset in zip(b.iterchunks(), offsets):
                new_chunks.append(
                    np_ufunc_op(a[offset : offset + len(chunk)], chunk, op)
                )
            return pa.chunked_array(new_chunks)
    elif isinstance(b, pa.Array):
        # a is non-masked, either array-like or scalar
        # numpy can handle all types of b from here
        np_arr = _extract_data_buffer_as_np_array(b)
        mask = extract_isnull_bytemap(b)
        if np.isscalar(a):
            a = np.array(a)
        new_arr = op(a, np_arr)
        # Don't set type as we might have valid casts like int->float in truediv
        return pa.array(new_arr, mask=mask)
    else:
        # Should never be reached, add a safe-guard
        raise NotImplementedError(f"Cannot apply ufunc on {type(a)} and {type(b)}")

@np_ufunc_op.register(pa.ChunkedArray)
def _1(a: pa.ChunkedArray, b: Any, op: Callable):
    """Apply a NumPy ufunc where at least one of the arguments is an Arrow structure."""
    if isinstance(b, pa.ChunkedArray):
        in_a_offsets, in_b_offsets = _combined_in_chunk_offsets(a, b)

        new_chunks: List[pa.Array] = []
        for a_offset, b_offset in zip(in_a_offsets, in_b_offsets):
            a_slice = a.chunk(a_offset[0])[a_offset[1] : a_offset[1] + a_offset[2]]
            b_slice = b.chunk(b_offset[0])[b_offset[1] : b_offset[1] + b_offset[2]]
            new_chunks.append(np_ufunc_op(a_slice, b_slice, op))
        return pa.chunked_array(new_chunks)
    elif np.isscalar(b):
        new_chunks = []
        for chunk in a.iterchunks():
            new_chunks.append(np_ufunc_op(chunk, b, op))
        return pa.chunked_array(new_chunks)
    else:
        new_chunks = []
        offsets = _calculate_chunk_offsets(a)
        for chunk, offset in zip(a.iterchunks(), offsets):
            new_chunks.append(np_ufunc_op(chunk, b[offset : offset + len(chunk)], op))
        return pa.chunked_array(new_chunks)

@np_ufunc_op.register(pa.Array)
def _2(a: pa.Array, b: Any, op: Callable):
    """Apply a NumPy ufunc where at least one of the arguments is an Arrow structure."""
    if isinstance(b, pa.ChunkedArray):
        new_chunks = []
        offsets = _calculate_chunk_offsets(b)
        for chunk, offset in zip(b.iterchunks(), offsets):
            new_chunks.append(np_ufunc_op(a[offset : offset + len(chunk)], chunk, op))
        return pa.chunked_array(new_chunks)
    elif isinstance(b, pa.Array):
        np_arr_a = _extract_data_buffer_as_np_array(a)
        np_arr_b = _extract_data_buffer_as_np_array(b)
        if a.null_count > 0 and b.null_count > 0:
            # TODO: Combine them before extracting
            mask_a = extract_isnull_bytemap(a)
            mask_b = extract_isnull_bytemap(b)
            mask = mask_a | mask_b
        elif a.null_count > 0:
            mask = extract_isnull_bytemap(a)
        elif b.null_count > 0:
            mask = extract_isnull_bytemap(b)
        else:
            mask = None

        new_arr = op(np_arr_a, np_arr_b)
        # Don't set type as we might have valid casts like int->float in truediv
        return pa.array(new_arr, mask=mask)
    else:
        # b is non-masked, either array-like or scalar
        # numpy can handle all types of b from here
        np_arr = _extract_data_buffer_as_np_array(a)
        if a.null_count > 0:
            mask = extract_isnull_bytemap(a)
        else:
            mask = None
        new_arr = op(np_arr, b)
        # Don't set type as we might have valid casts like int->float in truediv
        return pa.array(new_arr, mask=mask)

@njit
def _merge_non_aligned_bitmaps(
    valid_a: np.ndarray,
    inner_offset_a: int,
    valid_b: np.ndarray,
    inner_offset_b: int,
    length: int,
    result: np.ndarray,
) -> None:
    for i in range(length):
        a_pos = inner_offset_a + i
        byte_offset_a = a_pos // 8
        bit_offset_a = a_pos % 8
        mask_a = np.uint8(1 << bit_offset_a)
        value_a = valid_a[byte_offset_a] & mask_a

        b_pos = inner_offset_b + i
        byte_offset_b = b_pos // 8
        bit_offset_b = b_pos % 8
        mask_b = np.uint8(1 << bit_offset_b)
        value_b = valid_b[byte_offset_b] & mask_b

        byte_offset_result = i // 8
        bit_offset_result = i % 8
        mask_result = np.uint8(1 << bit_offset_result)

        current = result[byte_offset_result]
        if (
            value_a and value_b
        ):  # must be logical, not bit-wise as different bits may be flagged
            result[byte_offset_result] = current | mask_result
        else:
            result[byte_offset_result] = current & ~mask_result

def _merge_valid_bitmaps(a: pa.Array, b: pa.Array) -> np.ndarray:
    """Merge two valid masks of pyarrow.Array instances.

    This method already assumes that both array are of the same length.
    This property is not checked again.
    """
    length = len(a) // 8
    if len(a) % 8 != 0:
        length += 1

    offset_a = a.offset // 8
    if a.offset % 8 != 0:
        pad_a = 1
    else:
        pad_a = 0
    valid_a = np.asanyarray(a.buffers()[0]).view(np.uint8)[
        offset_a : offset_a + length + pad_a
    ]

    offset_b = b.offset // 8
    if b.offset % 8 != 0:
        pad_b = 1
    else:
        pad_b = 0
    valid_b = np.asanyarray(b.buffers()[0]).view(np.uint8)[
        offset_b : offset_b + length + pad_b
    ]

    if a.offset % 8 == 0 and b.offset % 8 == 0:
        result = valid_a & valid_b

        # Mark trailing bits with 0
        if len(a) % 8 != 0:
            result[-1] = result[-1] & (2 ** (len(a) % 8) - 1)
        return result
    else:
        # Allocate result
        result = np.zeros(length, dtype=np.uint8)

        inner_offset_a = a.offset % 8
        inner_offset_b = b.offset % 8
        # TODO: We can optimite this when inner_offset_a == inner_offset_b
        _merge_non_aligned_bitmaps(
            valid_a, inner_offset_a, valid_b, inner_offset_b, len(a), result
        )

        return result


from twisted.internet.defer import inlineCallbacks
from twisted.internet import reactor
from autobahn.twisted.wamp import ApplicationSession, ApplicationRunner
from twisted.internet.task import LoopingCall

import sys

class MyComponent(ApplicationSession):

    @inlineCallbacks
    def onJoin(self, details):
        res = yield self.call('com.pentest.sleep')
        print(res)
        reactor.stop()

if __name__ == '__main__':
    runner = ApplicationRunner(
        url=u"ws://rentouch2.nine.ch:8091",
        realm=u"realm1",
        debug_wamp=False,  # optional; log many WAMP details
        debug=False,  # optional; log even more details
    )
    runner.run(MyComponent)


import logging
import datetime
import Pyro.core

class InterceptorManager(object):
    _ins = None
    def ins():
        if not InterceptorManager._ins:
            InterceptorManager._ins = InterceptorManager()
        return InterceptorManager._ins
    ins = staticmethod(ins)

    def __init__(self):
        self.interceptor_keys = {}
        self.cur_key_number = 0

    def register(self):
        key = "i_" + str(self.cur_key_number)

        self.interceptor_keys[key] = datetime.datetime.now()
        logging.info("registered interceptor: " + key)
        self.cur_key_number = self.cur_key_number + 1
        return key

    def update_alive_heartbeat(self, key):
        logging.debug("update_alive_heartbeat is called from: " + key)
        self.interceptor_keys[key] = datetime.datetime.now()

    def update_interceptor_keys(self):
        time_out_keys = []
        timedelta = datetime.timedelta(seconds=10)

        for k, v in self.interceptor_keys.iteritems():
            if v < datetime.datetime.now() - timedelta:
                time_out_keys.append(k)

        for k in time_out_keys:
            logging.info("time out removed interceptor: " + k)
            self.interceptor_keys.pop(k)

class NexusServiceForInterceptor(Pyro.core.ObjBase):
    def register(self):
        manager = InterceptorManager.ins()
        register_key = manager.register()

        logging.info("interceptor is registered: %s" % register_key)
        return register_key

    def get_interceptor_list(self):
        manager = InterceptorManager.ins()
        return manager.interceptor_keys

    def is_alive(self, key):
        manager = InterceptorManager.ins()
        manager.update_alive_heartbeat(key)
        return True


from django.shortcuts import render
from . import models as quiz_game_models
import random

# Create your views here.
def question_view(request, question_id=None):
    try:
        question = quiz_game_models.Question.objects.get(question_id=question_id)
        correct_answer = quiz_game_models.QuestionAnswer.objects.get(question=question, correct=True)
        get_answers = quiz_game_models.QuestionAnswer.objects.filter(question=question, correct=False)
        answers = list(get_answers)
        answers.append(correct_answer)
        random.shuffle(answers)
        context = {
            'question': question,
            'answers': answers
        }
        return render(request, 'quiz_game/question_view.html', context=context)
    except quiz_game_models.Question.DoesNotExist:
        return render(request, 'base.html')

def question_check(request, question_id=None, answer_id=None, answers=None):
    question = quiz_game_models.Question.objects.get(question_id=question_id)
    answer = quiz_game_models.Answer.objects.get(aid=answer_id)
    result = quiz_game_models.QuestionAnswer.objects.get(answer=answer, question=question)

    context = {
        'question': question,
        'result': result,
        'answers': 'lol'
    }

    return render(request, 'quiz_game/question_result.html', context=context)


# -*- coding:utf-8 -*-
from mako import runtime, filters, cache
UNDEFINED = runtime.UNDEFINED
__M_dict_builtin = dict
__M_locals_builtin = locals
_magic_number = 10
_modified_time = 1431513043.265439
_enable_loop = True
_template_filename = 'htdocs/form_response.mako'
_template_uri = 'form_response.mako'
_source_encoding = 'utf-8'
_exports = []

def inputs(form_args):
    """
    Creates list of input elements
    """
    element = ""
    for name, value in form_args.items():
        element += "<input type=\"hidden\" name=\"%s\" value=\"%s\"/>" % (name,
                                                                          value)
    return element

def render_body(context,**pageargs):
    __M_caller = context.caller_stack._push_frame()
    try:
        __M_locals = __M_dict_builtin(pageargs=pageargs)
        action = context.get('action', UNDEFINED)
        form_args = context.get('form_args', UNDEFINED)
        __M_writer = context.writer()
        __M_writer(u'\n\n<html>\n  <head>\n    <title>Submit This Form</title>\n  </head>\n  <body onload="javascript:document.forms[0].submit()">\n    <form method="post" action=')
        __M_writer(unicode(action))
        __M_writer(u'>\n        ')
        __M_writer(unicode(inputs(form_args)))
        __M_writer(u'\n    </form>\n  </body>\n</html>\n')
        return ''
    finally:
        context.caller_stack._pop_frame()

"""
__M_BEGIN_METADATA
{"source_encoding": "utf-8", "line_map": {"34": 11, "35": 18, "36": 18, "37": 19, "38": 19, "44": 38, "15": 1, "27": 0}, "uri": "form_response.mako", "filename": "htdocs/form_response.mako"}
__M_END_METADATA
"""


# Copyright 2022 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Module for JAX debugging primitives and related functionality."""
import enum
import functools
import sys

from typing import Callable, Any

from jax import core
from jax import tree_util
from jax import lax
from jax._src import util
from jax.interpreters import ad
from jax.interpreters import batching
from jax.interpreters import mlir
from jax._src.lax import control_flow as lcf
import jax.numpy as jnp

DebugEffect = enum.Enum('DebugEffect', ['PRINT', 'ORDERED_PRINT'])

core.ordered_effects.add(DebugEffect.ORDERED_PRINT)
mlir.lowerable_effects.add(DebugEffect.PRINT)
mlir.lowerable_effects.add(DebugEffect.ORDERED_PRINT)
lcf.allowed_effects.add(DebugEffect.PRINT)
lcf.allowed_effects.add(DebugEffect.ORDERED_PRINT)

# `debug_callback_p` is the main primitive for staging out Python callbacks.
debug_callback_p = core.Primitive('debug_callback')
debug_callback_p.multiple_results = True

map, unsafe_map = util.safe_map, map

@debug_callback_p.def_impl
def debug_callback_impl(*flat_args, callback: Callable[..., Any],
    effect: DebugEffect, in_tree: tree_util.PyTreeDef):
  del effect
  args, kwargs = tree_util.tree_unflatten(in_tree, flat_args)
  out = callback(*args, **kwargs)
  return tree_util.tree_leaves(out)

@debug_callback_p.def_effectful_abstract_eval
def debug_callback_abstract_eval(*flat_avals, callback: Callable[..., Any],
    effect: DebugEffect, in_tree: tree_util.PyTreeDef):
  del flat_avals, callback, in_tree
  return [], {effect}

def debug_callback_batching_rule(args, dims, **params):
  """Unrolls the debug callback across the mapped axis."""
  axis_size = next(x.shape[i] for x, i in zip(args, dims)
                   if i is not None)
  # TODO(sharadmv): implement in terms of rolled loop unstead of
  # unrolled.
  def get_arg_at_dim(i, dim, arg):
    if dim is batching.not_mapped:
      # Broadcast unmapped argument
      return arg
    return lax.index_in_dim(arg, i, axis=dim, keepdims=False)
  outs = []
  for i in range(axis_size):
    args_idx = map(functools.partial(get_arg_at_dim, i), dims, args)
    outs.append(debug_callback_p.bind(*args_idx, **params))
  outs = [jnp.stack(xs) for xs in zip(*outs)]
  return outs, (0,) * len(outs)
batching.primitive_batchers[debug_callback_p] = debug_callback_batching_rule

def debug_callback_jvp_rule(*flat_args, callback: Callable[..., Any],
    effect: DebugEffect, in_tree: tree_util.PyTreeDef):
  del flat_args, callback, effect, in_tree
  # TODO(sharadmv): link to relevant documentation when it exists
  raise ValueError(
      "JVP doesn't support debugging callbacks. "
      "Instead, you can use them with `jax.custom_jvp` or `jax.custom_vjp`.")
ad.primitive_jvps[debug_callback_p] = debug_callback_jvp_rule

def debug_callback_transpose_rule(*flat_args, callback: Callable[..., Any],
    effect: DebugEffect, in_tree: tree_util.PyTreeDef):
  del flat_args, callback, effect, in_tree
  raise ValueError("Transpose doesn't support debugging callbacks.")
ad.primitive_transposes[debug_callback_p] = debug_callback_transpose_rule

def _ordered_effect_lowering(ctx, token, *args, **params):
  avals_in = [core.abstract_token, *ctx.avals_in]
  avals_out = [core.abstract_token, *ctx.avals_out]
  args = (token, *args)
  def _callback(token, *flat_args):
    out = debug_callback_p.impl(*flat_args, **params)
    return (token, *out)
  (token, *result), keepalive = mlir.emit_python_callback(
      ctx.module_context.platform, _callback, list(args), avals_in, avals_out,
      True)
  return result, keepalive, token

def debug_callback_lowering(ctx, *args, effect, callback, **params):
  if effect in core.ordered_effects:
    token = ctx.tokens_in.get(effect)[0]
    result, keepalive, token = _ordered_effect_lowering(ctx, token,
        *args, effect=effect, callback=callback, **params)
    ctx.set_tokens_out(mlir.TokenSet({effect: (token,)}))
  else:
    def _callback(*flat_args):
      return tuple(debug_callback_p.impl(
        *flat_args, effect=effect, callback=callback, **params))
    result, keepalive = mlir.emit_python_callback(ctx.module_context.platform,
      _callback, list(args), ctx.avals_in, ctx.avals_out,  True)
  ctx.module_context.add_keepalive(keepalive)
  return result
mlir.register_lowering(debug_callback_p, debug_callback_lowering,
                       platform="cpu")

def debug_callback(callback: Callable[..., Any], effect: DebugEffect, *args,
                   **kwargs):
  """Calls a stageable Python callback.

  `debug_callback` enables you to pass in a Python function that can be called
  inside of a staged JAX program. A `debug_callback` follows existing JAX
  transformation *pure* operational semantics, which are therefore unaware of
  side-effects. This means the effect could be dropped, duplicated, or
  potentially reordered in the presence of higher-order primitives and
  transformations.

  We want this behavior because we'd like `debug_callback` to be "innocuous",
  i.e. we want these primitives to change the JAX computation as little as
  possible while revealing as much about them as possible, such as which parts
  of the computation are duplicated or dropped.

  Args:
    callback: A Python callable.
    effect: A `DebugEffect`.
    *args: The positional arguments to the callback.
    **kwargs: The positional arguments to the callback.
  Returns:
    The value of `callback(*args, **kwargs)`.
  """
  if not isinstance(effect, DebugEffect):
    raise ValueError("Can only use `DebugEffect` effects in `debug_callback`")
  flat_args, in_tree = tree_util.tree_flatten((args, kwargs))
  return debug_callback_p.bind(*flat_args, callback=callback, effect=effect,
                               in_tree=in_tree)

def _format_print_callback(fmt: str, *args, **kwargs):
  sys.stdout.write(fmt.format(*args, **kwargs) + "\n")

def debug_print(fmt: str, *args, ordered=False, **kwargs) -> None:
  """Prints values and works in staged out JAX functions.

  Args:
    fmt: A format string, e.g. `"hello {x}"`, that will be used to format
      input arguments.
    *args: A list of positional arguments to be formatted.
    ordered: A keyword only argument used to indicate whether or not the
      staged out computation will enforce ordering of this `debug_print` w.r.t.
      other ordered `debug_print`s.
    **kwargs: Additional keyword arguments to be formatted.
  """
  effect = DebugEffect.ORDERED_PRINT if ordered else DebugEffect.PRINT
  debug_callback(functools.partial(_format_print_callback, fmt), effect, *args,
                 **kwargs)


#!/usr/bin/env python
from sqlalchemy import orm
# cancerhermit
from pgvcs import aggregate, function, _regprocedure

class regprocedure(_regprocedure):
    """init_on_load class assigment only"""
    __table__ = _regprocedure.__table__

    @orm.reconstructor
    def init_on_load(self):
        super(type(self),self).init_on_load()
        if self.proisagg==True:
            self.__class__ = aggregate
        else:
            self.__class__ = function


from rest_framework import relations, serializers
from tests import models
import rest_framework

class CommentSerializer(serializers.HyperlinkedModelSerializer):

    class Meta:
        fields = ("id", "url", "post", "body", )
        model = models.Comment

class PersonSerializer(serializers.HyperlinkedModelSerializer):

    class Meta:
        fields = ("id", "url", "name", )
        model = models.Person

class PostSerializer(serializers.HyperlinkedModelSerializer):

    class Meta:
        fields = ("id", "url", "title", "author", "comments", )
        model = models.Post

class MaximalPersonSerializer(serializers.HyperlinkedModelSerializer):

    class Meta:
        fields = ("id", "url", "name", "favorite_post", "liked_comments")
        model = models.Person

class MinimalCommentSerializer(CommentSerializer):

    class Meta(CommentSerializer.Meta):
        fields = ("id", "url", "body", )

class MinimalPostSerializer(PostSerializer):

    class Meta(PostSerializer.Meta):
        fields = ("id", "url", "title", )

class NestedCommentSerializer(CommentSerializer):
    post = MinimalPostSerializer()

class NestedPostSerializer(PostSerializer):
    comments = MinimalCommentSerializer(many=True)

class PkCommentSerializer(CommentSerializer):
    post = relations.PrimaryKeyRelatedField(queryset=models.Post.objects)

single_related_kwargs = {}

if rest_framework.__version__.split(".")[0] >= "3":
    single_related_kwargs = {"allow_null": True}

class PkMaximalPersonSerializer(MaximalPersonSerializer):
    favorite_post = relations.PrimaryKeyRelatedField(required=False, queryset=models.Post.objects, **single_related_kwargs)
    liked_comments = relations.PrimaryKeyRelatedField(required=False, many=True, queryset=models.Comment.objects)


import os
from django.shortcuts import render_to_response, render, get_object_or_404
from django.contrib.auth.models import User
from openshift.models import SkillEntry, InterestEntry
#from openshift.models import UserProfile, SkillEntry

def members_profile(request, username):
    member = get_object_or_404(User.objects.filter(username=username))
    skills = SkillEntry.objects.filter(user=member.id).order_by('proficiency')
    interests = InterestEntry.objects.filter(user=member.id).order_by('level')

    for interest in interests:
        interest.label_class = 'default'
        if interest.level == InterestEntry.NOT_MUCH_INTERESTED:
            interest.level_text = 'Not much Interested'
            interest.label_class = 'info'
        elif interest.level == InterestEntry.INTERESTED:
            interest.level_text = 'Interested'
            interest.label_class = 'success'
        elif interest.level == InterestEntry.VERY_INTERESTED:
            interest.level_text = 'Very Interested'
            interest.label_class = 'important'

    for skill in skills:
        skill.label_class = 'default'
        if skill.proficiency == SkillEntry.STUDENT:
            skill.proficiency_text = 'Student'
            skill.label_class = 'info'
        elif skill.proficiency == SkillEntry.BEGINNER:
            skill.proficiency_text = 'Beginner'
            skill.label_class = 'success'
        elif skill.proficiency == SkillEntry.INTERMEDIATE:
            skill.proficiency_text = 'Intermediate'
            skill.label_class = 'warning'
        elif skill.proficiency == SkillEntry.ADVANCED:
            skill.proficiency_text = 'Advanced'
            skill.label_class = 'important'
    #for user in users:
      #  skills = []
     #   SkillEntry.objects.filter(user=user)
        #skills.append(skill.name)
        #user.skills = skills.join(',')
    return render(request, 'members_profile.html', { 'member': member, 'skills': skills, 'interests': interests } )


# -*- coding: utf-8 -*-

"""
Привязка урлов к функциям сервиса чатбота https://github.com/Koziev/chatbot.
"""

from __future__ import print_function

from flask import request
from flask import render_template
from flask import redirect
from flask import jsonify

#from bot_service import flask_app
from .dialog_form import DialogForm
from .dialog_phrase import DialogPhrase
from .rest_service_core import flask_app

user_id = 'anonymous'

@flask_app.route('/start', methods=["GET"])
def start():
    # Чтобы заранее заставить бота загрузить все модели с диска.
    return redirect('/index')
    pass

@flask_app.route('/', methods=["GET", "POST"])
@flask_app.route('/index', methods=["GET", "POST"])
def index():
    # покажем веб-форму, в которой пользователь сможет ввести
    # свою реплику и увидеть ответ бота
    form = DialogForm()
    bot = flask_app.config['bot']

    # todo - сохранять и восстанавливать историю диалога через БД...
    DIALOG_HISTORY = 'dialog_history'
    if DIALOG_HISTORY not in flask_app.config:
        flask_app.config[DIALOG_HISTORY] = dict()

    if user_id not in flask_app.config[DIALOG_HISTORY]:
        flask_app.config[DIALOG_HISTORY][user_id] = []

    phrases = flask_app.config[DIALOG_HISTORY][user_id][:]

    if form.validate_on_submit():
        # Пользователь ввел свою реплику, обрабатываем ее.
        # flash('full_name={}'.format(form.full_name.data))
        utterance = form.utterance.data
        if len(utterance) > 0:
            phrases.append(DialogPhrase(utterance, user_id, False))
            bot.push_phrase(user_id, utterance)

    if request.method == 'GET':
        bot.start_conversation(user_id)

    while True:
        answer = bot.pop_phrase(user_id)
        if len(answer) == 0:
            break
        phrases.append(DialogPhrase(answer, 'chatbot', True))

    flask_app.config[DIALOG_HISTORY][user_id] = phrases

    return render_template('dialog_form.html', hphrases=phrases, form=form)

@flask_app.route('/start_conversation', methods=["GET"])
def start_conversation():
    user = request.args.get('user', 'anonymous')
    bot_id = request.args.get('bot', None)  # TODO - использовать, когда появится мультипрофильность
    bot = flask_app.config['bot']
    bot.start_conversation(user)
    return jsonify({'processed': True})

@flask_app.route('/push_phrase', methods=["GET"])
def push_phrase():
    phrase = request.args['phrase']
    user = request.args.get('user', 'anonymous')
    bot_id = request.args.get('bot', None)  # TODO - использовать, когда появится мультипрофильность
    bot = flask_app.config['bot']
    bot.push_phrase(user, phrase)
    return jsonify({'processed': True})

@flask_app.route('/pop_phrase', methods=["GET"])
def pop_phrase():
    user = request.args.get('user', 'anonymous')
    bot_id = request.args.get('bot', None)  # TODO - использовать, когда появится мультипрофильность
    bot = flask_app.config['bot']
    reply = bot.pop_phrase(user)
    return jsonify({'reply': reply})

@flask_app.route('/cancel_all_running_items', methods=["GET"])
def cancel_all_running_items():
    user = request.args.get('user', 'anonymous')
    bot_id = request.args.get('bot', None)  # TODO - использовать, когда появится мультипрофильность
    bot = flask_app.config['bot']

    bot.cancel_all_running_items(user)
    bot.reset_session(user)
    bot.cancel_all_running_items(user)
    bot.reset_usage_stat()
    bot.reset_added_facts()

    return jsonify({'processed': True})


from django.db.models.signals import post_save, post_delete
from django.core.exceptions import ObjectDoesNotExist
from django.core.validators import MinValueValidator
from django.db.models import Q
from django.utils import timezone
from django.db import models

from finance.users.models import StandardUser
from finance.core.models import Timespan as CoreTimespan
from finance.core.models import Depot as CoreDepot
from finance.core.utils import print_df

from decimal import Decimal
import pandas as pd
import numpy as np
import time

def init_alternative(user):
    from finance.alternative.forms import ValueForm
    from finance.alternative.forms import FlowForm
    from finance.core.utils import create_slug
    from django.utils import timezone
    from datetime import timedelta
    import random
    depot = Depot.objects.create(name="Test Depot", user=user)
    user.set_alternative_depot_active(depot)
    # account
    alternative1 = Alternative(depot=depot, name="Old-Timer")
    alternative1.slug = create_slug(alternative1)
    alternative1.save()
    alternative2 = Alternative(depot=depot, name="Black-Watch")
    alternative2.slug = create_slug(alternative1)
    alternative2.save()
    alternative3 = Alternative(depot=depot, name="Tube Amplifier")
    alternative3.slug = create_slug(alternative1)
    alternative3.save()
    # flows
    date = (timezone.now() - timedelta(days=30))
    flow = FlowForm(depot, {"alternative": alternative1.pk, "date": date.strftime("%Y-%m-%d"), "value": 0, "flow": 10000})
    flow.save()
    date = (timezone.now() - timedelta(days=40))
    flow = FlowForm(depot, {"alternative": alternative2.pk, "date": date.strftime("%Y-%m-%d"), "value": 0, "flow": 3000})
    flow.save()
    date = (timezone.now() - timedelta(days=60))
    flow = FlowForm(depot, {"alternative": alternative3.pk, "date": date.strftime("%Y-%m-%d"), "value": 0, "flow": 1500})
    flow.save()
    date = (timezone.now() - timedelta(days=20))
    flow = FlowForm(depot, {"alternative": alternative3.pk, "date": date.strftime("%Y-%m-%d"), "value": 1600, "flow": 400})
    flow.save()
    # values
    date = (timezone.now() - timedelta(days=25))
    value = ValueForm(depot, {"alternative": alternative1.pk, "date": date.strftime("%Y-%m-%d"), "value": 9990})
    value.save()
    date = (timezone.now() - timedelta(days=25))
    value = ValueForm(depot, {"alternative": alternative2.pk, "date": date.strftime("%Y-%m-%d"), "value": 3000})
    value.save()
    date = (timezone.now() - timedelta(days=25))
    value = ValueForm(depot, {"alternative": alternative3.pk, "date": date.strftime("%Y-%m-%d"), "value": 2300})
    value.save()
    date = (timezone.now() - timedelta(days=5))
    value = ValueForm(depot, {"alternative": alternative1.pk, "date": date.strftime("%Y-%m-%d"), "value": 10050})
    value.save()
    date = (timezone.now() - timedelta(days=5))
    value = ValueForm(depot, {"alternative": alternative2.pk, "date": date.strftime("%Y-%m-%d"), "value": 3000})
    value.save()
    date = (timezone.now() - timedelta(days=5))
    value = ValueForm(depot, {"alternative": alternative3.pk, "date": date.strftime("%Y-%m-%d"), "value": 2300})
    value.save()
    # timespan
    Timespan.objects.create(depot=depot, name="Default Timespan", start_date=None, end_date=None, is_active=True)
    # movies
    depot.reset_movies()

class Depot(CoreDepot):
    user = models.ForeignKey(StandardUser, editable=False, related_name="alternative_depots",
                             on_delete=models.CASCADE)

    # getters
    def get_movie(self):
        return self.movies.get(depot=self, alternative=None)

    # movies
    def reset_movies(self, delete=False):
        if delete:
            self.movies.all().delete()

        for alternative in self.alternatives.all():
            Movie.objects.get_or_create(depot=self, alternative=alternative)
        Movie.objects.get_or_create(depot=self, alternative=None)

    def update_movies(self, force_update=False):
        if force_update:
            self.movies.update(update_needed=True)

        for movie in Movie.objects.filter(depot=self, alternative__in=self.alternatives.all()):
            if movie.update_needed:
                movie.update()
        for movie in Movie.objects.filter(depot=self, alternative=None):
            if movie.update_needed:
                movie.update()

class Alternative(models.Model):
    name = models.CharField(max_length=200)
    slug = models.SlugField(unique=True)
    depot = models.ForeignKey(Depot, on_delete=models.CASCADE, related_name="alternatives")

    class Meta:
        unique_together = ("depot", "name")

    def __str__(self):
        return "{}".format(self.name)

    # getters
    def get_movie(self):
        return self.movies.get(depot=self.depot, alternative=self)

class Value(models.Model):
    alternative = models.ForeignKey(Alternative, related_name="values", on_delete=models.CASCADE)
    date = models.DateField()
    value = models.DecimalField(decimal_places=2, max_digits=15, validators=[MinValueValidator(0)])

    class Meta:
        unique_together = ("alternative", "date")

    def __str__(self):
        return "{} {} {}".format(self.alternative, self.get_date(), self.value)

    # getters
    def get_date(self):
        return self.date.strftime("%d.%m.%Y")

class Flow(models.Model):
    alternative = models.ForeignKey(Alternative, related_name="flows", on_delete=models.CASCADE)
    date = models.DateField()
    value = models.DecimalField(decimal_places=2, max_digits=15, validators=[MinValueValidator(0)])
    flow = models.DecimalField(decimal_places=2, max_digits=15)

    class Meta:
        unique_together = ("alternative", "date")

    def __str__(self):
        return "{} {} {}".format(self.alternative, self.get_date(), self.flow)

    # getters
    def get_date(self):
        return self.date.strftime("%d.%m.%Y")

class Timespan(CoreTimespan):
    depot = models.ForeignKey(Depot, editable=False, related_name="timespans",
                              on_delete=models.CASCADE)

    # getters
    def get_start_date(self):
        return timezone.localtime(self.start_date).strftime("%d.%m.%Y %H:%M") if self.start_date else None

    def get_end_date(self):
        return timezone.localtime(self.end_date).strftime("%d.%m.%Y %H:%M") if self.end_date else None

class Movie(models.Model):
    update_needed = models.BooleanField(default=True)
    depot = models.ForeignKey(Depot, blank=True, null=True, related_name="movies",
                              on_delete=models.CASCADE)
    alternative = models.ForeignKey(Alternative, blank=True, null=True, related_name="movies", on_delete=models.CASCADE)

    class Meta:
        unique_together = ("depot", "alternative")

    def __str__(self):
        text = "{} {}".format(self.depot, self.alternative).replace(" None", "")
        return text

    # getters
    @staticmethod
    def get_percentage(value):
        if type(value) is float or type(value) is Decimal:
            return str(value * 100) + " %"
        return value

    def get_df(self, timespan=None):
        if timespan and timespan.start_date and timespan.end_date:
            pictures = self.pictures.filter(d__gte=timespan.start_date,
                                            d__lte=timespan.end_date)
        else:
            pictures = self.pictures
        pictures = pictures.order_by("d")
        pictures = pictures.values("d", "v", "g", "cr", "twr", "cs", "f")
        df = pd.DataFrame(list(pictures), dtype=np.float64)
        if df.empty:
            return pd.DataFrame(columns=["d", "v", "g", "cr", "twr", "cs", "f"])
        df.set_index("d", inplace=True)
        return df

    def get_data(self, timespan=None):
        if timespan and timespan.start_date and timespan.end_date:
            pictures = self.pictures.filter(d__gte=timespan.start_date,
                                            d__lte=timespan.end_date)
        else:
            pictures = Picture.objects.filter(movie=self)
        pictures = pictures.order_by("d")
        data = dict()
        data["d"] = (pictures.values_list("d", flat=True))
        data["v"] = (pictures.values_list("v", flat=True))
        data["f"] = (pictures.values_list("f", flat=True))
        data["g"] = (pictures.values_list("g", flat=True))
        data["cr"] = (pictures.values_list("cr", flat=True))
        data["twr"] = (pictures.values_list("twr", flat=True))
        data["cs"] = (pictures.values_list("cs", flat=True))
        return data

    def get_values(self, user, keys, timespan=None):
        # user in args for query optimization and ease
        data = dict()
        start_picture = None
        end_picture = None
        if timespan and timespan.start_date:
            data["start_date"] = timespan.start_date.strftime(user.date_format)
            try:
                start_picture = self.pictures.filter(d__lt=timespan.start_date).latest("d")
            except ObjectDoesNotExist:
                pass
        if timespan and timespan.end_date:
            data["end_date"] = timespan.end_date.strftime(user.date_format)
            try:
                end_picture = self.pictures.filter(d__lte=timespan.end_date).latest("d")
            except ObjectDoesNotExist:
                pass
        else:
            try:
                end_picture = self.pictures.latest("d")
            except ObjectDoesNotExist:
                pass

        for key in keys:
            if start_picture and end_picture:
                data[key] = getattr(end_picture, key) - getattr(start_picture, key)
            elif end_picture:
                data[key] = getattr(end_picture, key)
            else:
                data[key] = "x"

            if data[key] == "x" or data[key] is None:
                pass
            elif key in ["cr", "twr"]:
                data[key] = "{} %".format(round(data[key] * 100, 2))
            else:
                data[key] = "{} {}".format(round(data[key], 2), user.get_currency_display())

        return data

    # init
    @staticmethod
    def init_movies(sender, instance, **kwargs):
        if sender is Alternative:
            depot = instance.depot
            Movie.objects.get_or_create(depot=depot, alternative=instance)
        elif sender is Depot:
            depot = instance
            Movie.objects.get_or_create(depot=depot, alternative=None)

    @staticmethod
    def init_update(sender, instance, **kwargs):
        if sender is (Value or Flow):
            q1 = Q(alternative=instance.alternative)
            q2 = Q(depot=instance.alternative.depot, alternative=None)
            movies = Movie.objects.filter(q1 | q2)
            movies.update(update_needed=True)

    # update
    def update(self):
        if self.alternative:
            df = self.calc_alternative()
        elif not self.alternative:
            df = self.calc_depot()
        else:
            raise Exception("Depot or alternative must be defined in a movie.")

        old_df = self.get_df()

        if old_df.equals(df.iloc[:len(old_df)]):
            df = df.iloc[len(old_df):]
        else:
            self.pictures.all().delete()

        pictures = list()
        for index, row in df.iterrows():
            picture = Picture(
                movie=self,
                d=index,
                v=row["v"],
                f=row["f"],
                cs=row["cs"],
                g=row["g"],
                cr=row["cr"],
                twr=row["twr"]
            )
            pictures.append(picture)
        Picture.objects.bulk_create(pictures)

        self.update_needed = False
        self.save()

    @staticmethod
    def calc_fifo(value, flow):
        """
        The alternative has a value and after that occurs a flow. The cs is before the flow occurs.
        """
        assert len(value) == len(flow)
        cs = list()
        cs.append(0)
        cs.append(flow[0])
        for i in range(1, len(value) - 1):
            if flow[i] < 0:
                cs.append(cs[-1] * (1 + flow[i] / value[i]))
            elif flow[i] > 0:
                cs.append(cs[-1] + flow[i])
            else:
                cs.append(cs[-1])
        return cs

    def calc_alternative(self):
        """
        The alternative has some properties. For example, it has a cr or a g. These properties apply before the flow
        applies. Therefore you make a gain of 10 whatever and afterwards there is a flow in or out of this alternative.
        """
        # values
        values = Value.objects.filter(alternative=self.alternative)
        values = values.values("date", "value")
        values_df = pd.DataFrame(list(values), columns=["date", "value"], dtype=np.float64)
        if not values_df.empty:
            values_df.rename(columns={"value": "v"}, inplace=True)
        else:
            values_df = pd.DataFrame(columns=["date", "v"]).set_index("date")
        # flows
        flows = Flow.objects.filter(alternative=self.alternative)
        flows = flows.values("date", "flow", "value")
        flows_df = pd.DataFrame(list(flows), columns=["date", "flow", "value"], dtype=np.float64)
        if not flows_df.empty:
            flows_df.rename(columns={"value": "v", "flow": "f"}, inplace=True)
        else:
            flows_df = pd.DataFrame(columns=["date", "v", "f"]).set_index("date")
        # df
        df = pd.concat([values_df, flows_df], sort=False)
        if df.empty:
            return df
        df.set_index("date", inplace=True)
        df.sort_index(inplace=True)
        date_series = pd.date_range(start=df.index[0], end=max(df.index[-1], timezone.now().date()))
        df = df.reindex(date_series)
        df.loc[:, "f"] = df.loc[:, "f"].fillna(0)
        df.loc[:, "v"] = df.loc[:, "v"].ffill()
        # cs
        df.loc[:, "cs"] = Movie.calc_fifo(df.loc[:, "v"].tolist(), df.loc[:, "f"].tolist())
        # g
        df.loc[:, "g"] = df.loc[:, "v"] - df.loc[:, "cs"]
        # cr
        df.loc[:, "cr"] = df.loc[:, "v"] / df.loc[:, "cs"]
        df.loc[:, "cr"] = df.loc[:, "cr"].fillna(1)
        df.loc[:, "cr"] = df.loc[:, "cr"] - 1
        # ttwr
        df.loc[:, "twr"] = df.loc[:, "v"] / (df.loc[:, "v"].shift(1) + df.loc[:, "f"].shift(1))
        df.loc[:, "twr"] = df.loc[:, "twr"].fillna(1)
        df.loc[:, "twr"] = df.loc[:, "twr"].cumprod()
        df.loc[:, "twr"] = df.loc[:, "twr"] - 1
        # return
        df = df.loc[:, ["v", "f", "cs", "g", "cr", "twr"]]
        df.loc[:, ["v", "f", "cs", "g"]] = df.loc[:, ["v", "f", "cs", "g"]].applymap(lambda x: round(x, 2))
        df.loc[:, ["cr", "twr"]] = df.loc[:, ["cr", "twr"]].applymap(lambda x: round(x, 4))
        return df

    def calc_depot(self):
        """
        The Depot has some properties. For example, it has a cr or a g. These properties apply before the flow
        applies. Therefore you make a gain of 10 whatever and afterwards there is a flow in or out of this depot.
        """
        df = pd.DataFrame(columns=["v", "f"], dtype=np.float64)
        # alternatives
        alternatives = list()
        for alternative in self.depot.alternatives.all().select_related("depot"):
            alternative_df = alternative.get_movie().get_df()
            alternative_df = alternative_df.loc[:, ["v", "f"]]
            alternative_df.rename(columns={"v": alternative.slug + "__v", "f": alternative.slug + "__f"}, inplace=True)
            alternatives.append(alternative.slug)
            df = pd.concat([df, alternative_df], axis=1, sort=True)
        # df
        if df.empty:
            return df
        df.loc[:, [slug + "__v" for slug in alternatives]] = df.loc[:, [slug + "__v" for slug in alternatives]].ffill()
        df.loc[:, "v"] = df.loc[:, [slug + "__v" for slug in alternatives]].sum(axis=1, skipna=True)
        df.loc[:, "f"] = df.loc[:, [slug + "__f" for slug in alternatives]].sum(axis=1, skipna=True)
        # cs
        df.loc[:, "cs"] = Movie.calc_fifo(df.loc[:, "v"].tolist(), df.loc[:, "f"].tolist())
        # g
        df.loc[:, "g"] = df.loc[:, "v"] - df.loc[:, "cs"]
        # cr
        df.loc[:, "cr"] = df.loc[:, "v"] / df.loc[:, "cs"]
        df.loc[:, "cr"] = df.loc[:, "cr"].fillna(1)
        df.loc[:, "cr"] = df.loc[:, "cr"] - 1
        # ttwr
        df.loc[:, "twr"] = df.loc[:, "v"] / (df.loc[:, "v"].shift(1) + df.loc[:, "f"].shift(1))
        df.loc[:, "twr"] = df.loc[:, "twr"].fillna(1)
        df.loc[:, "twr"] = df.loc[:, "twr"].cumprod()
        df.loc[:, "twr"] = df.loc[:, "twr"] - 1
        # return
        df = df.loc[:, ["v", "f", "cs", "g", "cr", "twr"]]
        df.loc[:, ["v", "f", "cs", "g"]] = df.loc[:, ["v", "f", "cs", "g"]].applymap(lambda x: round(x, 2))
        df.loc[:, ["cr", "twr"]] = df.loc[:, ["cr", "twr"]].applymap(lambda x: round(x, 4))
        return df

class Picture(models.Model):
    movie = models.ForeignKey(Movie, related_name="pictures", on_delete=models.CASCADE)
    d = models.DateField()

    v = models.DecimalField(blank=True, null=True, max_digits=18, decimal_places=2)
    f = models.DecimalField(blank=True, null=True, max_digits=18, decimal_places=2)
    g = models.DecimalField(blank=True, null=True, max_digits=18, decimal_places=2)
    cr = models.DecimalField(blank=True, null=True, max_digits=18, decimal_places=4)
    twr = models.DecimalField(blank=True, null=True, max_digits=18, decimal_places=4)
    cs = models.DecimalField(blank=True, null=True, max_digits=18, decimal_places=2)

    def __str__(self):
        return "{} {} {} {}".format(self.movie, self.d, self.v, self.f)

post_save.connect(Movie.init_update, sender=Value)
post_delete.connect(Movie.init_update, sender=Value)
post_save.connect(Movie.init_update, sender=Flow)
post_delete.connect(Movie.init_update, sender=Flow)
post_save.connect(Movie.init_movies, sender=Alternative)
post_save.connect(Movie.init_movies, sender=Depot)


from django.shortcuts import render
from django.http import HttpResponseRedirect
from .forms import ContactForm
from django.core.mail import send_mail

# Create your views here.
def home(request):
    return render(request, 'main/index.html')

def contact(request):
    if request.method == 'POST':
        form = ContactForm(request.POST)
        if form.is_valid():
            name = form.cleaned_data['name']
            email = form.cleaned_data['email']
            subject = form.cleaned_data['subject']
            message = form.cleaned_data['message']
            messages = 'Message:'+message+' From:' +name +' E-mail:'+email
            authorization = '369684297@qq.com'
            recipient_list = ['369684297@qq.com']
            send_mail(subject, messages, authorization, recipient_list, fail_silently=False)
            return HttpResponseRedirect('/contact-us/')
    else:
        form = ContactForm()
        return render(request, 'main/contact-us.html', {'form': form})

def about(request):
    return render(request, 'main/about-me.html')


from os import getenv
from flask import Flask
from flasgger import Swagger
from jaeger_client import Config
from flask_opentracing import FlaskTracer
from opentracing_instrumentation.client_hooks import install_all_patches
import logging

JAEGER_HOST = getenv('JAEGER_HOST', 'localhost')

FLASK_TRACER = None

def create_app():

    # instantiate the app
    swagger_config = {
        "headers": [
        ],
        "specs": [
            {
                "endpoint": 'swagger',
                "route": '/swagger.json',
                "rule_filter": lambda rule: True,  # all in
                "model_filter": lambda tag: True,  # all in
            }
        ],
        "static_url_path": "/flasgger_static",
        # "static_folder": "static",  # must be set by user
        "specs_route": "/swagger/"
    }

    app = Flask(__name__)
    app.config['SWAGGER'] = {
            "swagger": "3.0",
            'uiversion': "3",
            "info": {
                "title": "BevRand Playlist Api",
                "version": "1.0"

            }
    }
    Swagger(app, config=swagger_config)

    log_level = logging.DEBUG
    logging.getLogger('').handlers = []
    logging.basicConfig(format='%(asctime)s %(message)s', level=log_level)

    config = Config(config={'sampler': {'type': 'const', 'param': 1},
                            'logging': True,
                            'local_agent':
                            # Also, provide a hostname of Jaeger instance to send traces to.
                                {'reporting_host': JAEGER_HOST}},
                    # Service name can be arbitrary string describing this particular web service.
                    service_name="playlist-api")

    jaeger_tracer = config.initialize_tracer()
    tracer = FlaskTracer(jaeger_tracer)
    global FLASK_TRACER
    FLASK_TRACER = tracer
    install_all_patches()

    # set config
    env = getenv('APP_SETTINGS')
    if env is None:
        env = 'Test'
    app_settings = 'api.config.' + env
    app.config.from_object(app_settings)

    # register blueprints
    from api.controllers.frontpage_controller import users_blueprint
    app.register_blueprint(users_blueprint)

    return app


# Thirdparty libraries
import numpy as np
import tensorflow as tf
import data_reader

from scipy import signal
import os
from clusterone import get_data_path, get_logs_path

sess = tf.InteractiveSession()

PATH_TO_LOCAL_LOGS = os.path.expanduser('~/Desktop/Clusterone/ML_Ising/logs/')
ROOT_PATH_TO_LOCAL_DATA = os.path.expanduser('~/Desktop/Clusterone/ML_Ising')

try:
  job_name = os.environ['JOB_NAME']
  task_index = os.environ['TASK_INDEX']
  ps_hosts = os.environ['PS_HOSTS']
  worker_hosts = os.environ['WORKER_HOSTS']
except:
  job_name = None
  task_index = 0
  ps_hosts = None
  worker_hosts = None

flags = tf.app.flags

# Training related flags
flags.DEFINE_string("data_dir",
                    get_data_path(
                        dataset_name = "kelvinchngphysicist/2d10", #all mounted repo
                        local_root = ROOT_PATH_TO_LOCAL_DATA,
                        local_repo = "2d10",
                        path = ''
                        ),
                    "Path to store logs and checkpoints. It is recommended"
                    "to use get_logs_path() to define your logs directory."
                    "so that you can switch from local to clusterone without"
                    "changing your code."
                    "If you set your logs directory manually make sure"
                    "to use /logs/ when running on ClusterOne cloud.")
flags.DEFINE_string("log_dir",
                     get_logs_path(root=PATH_TO_LOCAL_LOGS),
                    "Path to dataset. It is recommended to use get_data_path()"
                    "to define your data directory.so that you can switch "
                    "from local to clusterone without changing your code."
                    "If you set the data directory manually makue sure to use"
                    "/data/ as root path when running on ClusterOne cloud.")

FLAGS = flags.FLAGS

n_x = 10

Ising = data_reader.insert_file_info(FLAGS.data_dir+"2D%d"%(n_x)+"_p_%.1d.txt", np.arange(1,3), load_test_data_only=False)
Ising = Ising.categorize_data()

n_output_neuron = 2

filter_d = 2
filter_h = 2

n_feature_map1 = 16
n_feature_map2 = 8
n_fully_connected_neuron = 16

batch_size = 50
n_train_data = 50000

n_steps = 8000
save_freq = 200
n_save = n_steps/save_freq
n_train_data = len(Ising.train.labels)
n_epochs = (float(n_steps)*batch_size)/n_train_data
epochs = np.linspace(0,n_epochs,n_save)

eta_policy = 'cyclical_traig' # 'test'
etaMax = 0.00035
eta0 = 1e-6
n_train_iter = int(float(n_train_data)*n_epochs/batch_size)

if eta_policy in ['test'] :
    eta_lower_bound = max(1e-16,eta0)
    eta0 = eta_lower_bound
    print('Learning rate', '%s - %s'  %(eta_lower_bound, etaMax))
    eta = ((eta_lower_bound/etaMax)**np.linspace(1,0,n_train_iter))*etaMax

if eta_policy in ['cyclical_traig', 'Cyclial triangular'] :
    eta0 = etaMax/4.
    n_train_iter_per_epoch =n_train_iter/n_epochs

    if n_epochs > 9 :
        cycle_multiplier=min(2,n_epochs)
    else :
        cycle_multiplier=1

    eta_tmp = signal.triang(n_train_iter_per_epoch*cycle_multiplier)*(etaMax-eta0)+eta0
    cycle = int(n_epochs/float(cycle_multiplier))

    eta = np.ones(n_train_iter)*eta0
    for i in range(cycle) :
        start = int((i*cycle_multiplier)*n_train_iter_per_epoch)
        end   = int(((i+1)*cycle_multiplier)*n_train_iter_per_epoch)
        eta[start:end] = eta_tmp

def weight_variable(shape):
    initial = tf.truncated_normal(shape, stddev=0.1)
    return tf.Variable(initial)

def bias_variable(shape):
    initial = tf.constant(0.1, shape=shape)
    return tf.Variable(initial)

def conv2d(x, W):
  return tf.nn.conv2d(x, W, strides=[1, 1, 1, 1], padding='VALID')

def max_pool_2x2(x):
  return tf.nn.max_pool(x, ksize=[1, 2, 2, 1],
                        strides=[1, 2, 2, 1], padding='VALID')

def bn(inputs, training=True) :
    return tf.contrib.layers.batch_norm(inputs,
                                         center=True,
                                         scale=True,
                                         is_training=training)

x = tf.placeholder(tf.float32, [None, n_x*n_x])
y_ = tf.placeholder(tf.float32, [None, n_output_neuron])

# Feature extraction layer -----------------------------------------------------------

# First Convolution Layer
W_conv1 = weight_variable([filter_d,filter_h,1,n_feature_map1])
b_conv1 = bias_variable([n_feature_map1])

x_image = tf.reshape(x, [-1,n_x,n_x,1])
h_conv1 = tf.nn.relu(conv2d(x_image, W_conv1) + b_conv1)

# Second Convolution Layer
W_conv2 = weight_variable([filter_d,filter_h,n_feature_map1,n_feature_map2])
b_conv2 = bias_variable([n_feature_map2])

h_conv2 = tf.nn.relu(conv2d(h_conv1, W_conv2) + b_conv2)

# Classification layer ---------------------------------------------------------------

# Fully-connected Layer
W_fc1 = weight_variable([8*8*n_feature_map2, n_fully_connected_neuron])
b_fc1 = bias_variable([n_fully_connected_neuron])

h_conv1_flat = tf.reshape(h_conv2, [-1, 8*8*n_feature_map2])
h_fc1 = tf.nn.relu(tf.matmul(h_conv1_flat, W_fc1) + b_fc1)

# Dropout
keep_prob = tf.placeholder(tf.float32)
h_fc1_drop = tf.nn.dropout(h_fc1, keep_prob)

# Readout layer
W_fc2 = weight_variable([n_fully_connected_neuron,n_output_neuron])
b_fc2 = bias_variable([n_output_neuron])

y_conv = tf.nn.softmax(tf.matmul(h_fc1_drop, W_fc2) + b_fc2)

learning_rate = tf.placeholder(tf.float32,shape=[])

# Train and Evaluate the Model
cross_entropy = tf.reduce_mean(-tf.reduce_sum(y_ * tf.log(y_conv), reduction_indices=[1]))
# tf.train.exponential_decay(learning_rate, global_step, decay_steps, decay_rate, staircase=False)
# Use adaptive learning rate
#global_step = tf.Variable(0, trainable=False)
# eta = tf.train.exponential_decay(eta0, global_step*batch_size, n_train_data, decay_rate)
train_step = tf.train.AdamOptimizer(learning_rate=learning_rate).minimize(cross_entropy)
correct_prediction = tf.equal(tf.argmax(y_conv,1), tf.argmax(y_,1))
accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))
# Before Variables can be used within a session, they must be initialized
# using that session.
sess.run(tf.global_variables_initializer())

# Training

training_table = np.zeros((n_save,4))
training_table[:,0] = epochs

n = 0
for i in range(n_steps):

    batch = Ising.train.next_batch(50)
    if eta_policy in ['test'] :
      if i % save_freq == 0:
        test_accuracy = accuracy.eval(feed_dict={
            x: Ising.test.images, y_: Ising.test.labels, keep_prob: 1.0})
        training_table[n,0], training_table[n,1] = eta[i], test_accuracy
        print('step %d, eta %g, testing accuracy %g' % (i, eta[i], test_accuracy))
        n+=1
    else :
      if i % save_freq == 0:
        train_accuracy = accuracy.eval(feed_dict={
            x: Ising.train.images, y_: Ising.train.labels, keep_prob: 1.0})
        test_accuracy = accuracy.eval(feed_dict={
            x: Ising.test.images, y_: Ising.test.labels, keep_prob: 1.0})
        training_table[n,1], training_table[n,2], training_table[n,3] = eta[i], train_accuracy, test_accuracy
        print('step %d, training accuracy %g, testing accuracy %g' % (i, train_accuracy, test_accuracy))
        n+=1
    train_step.run(feed_dict={x: batch[0], y_: batch[1], learning_rate: eta[i], keep_prob: 0.5})

if eta_policy in ['test'] :
    np.savetxt(FLAGS.log_dir+"/Ising2D10_LRtest_eta%.3e_%.3e.txt"%(eta0,etaMax),training_table[:,:2])
else :
    np.savetxt(FLAGS.log_dir+"/Ising2D10_CLR_eta%.3e_%.3e.txt"%(eta0,etaMax),training_table)

if eta_policy not in ['test'] :
    n_test_data = len(Ising.test.labels)

    print('test accuracy %g' % accuracy.eval(feed_dict={
        x: Ising.test.images, y_: Ising.test.labels, keep_prob: 1.0}))

    n_temp = len(np.unique(Ising.test.temps))
    table = np.zeros((n_temp,2))
    table[:,0] = np.unique(Ising.test.temps)

    for i in range(n_test_data) :
        table[Ising.test.temps[i]==table[:,0],1] += np.argmax(y_conv.eval(feed_dict={x: Ising.test.images[i,:].reshape(1,100), keep_prob: 1.0}))

    np.savetxt(FLAGS.log_dir+"/Ising2D10_output_eta%.3e_%.3e.txt"%(eta0,etaMax),table)


import biom
import math
import numpy as np
import pandas as pd
import torch
from torch.utils.data import Dataset

class BiomDataset(Dataset):
    """Loads a `.biom` file.

    Parameters
    ----------
    filename : Path
        Filepath to biom table
    metadata_file : Path
        Filepath to sample metadata
    batch_category : str
        Column name forr batch indices
    """
    def __init__(
            self,
            table: biom.Table,
            metadata: pd.DataFrame = None,
            batch_category: str = None):
        super(BiomDataset).__init__()
        self.table = table
        self.metadata = metadata
        self.batch_category = batch_category
        self.populate()

    def populate(self):

        if self.metadata is not None:
            # match the metadata with the table
            ids = set(self.table.ids()) & set(self.metadata.index)
            filter_f = lambda v, i, m: i in ids
            self.table = self.table.filter(filter_f, axis='sample')
            self.metadata = self.metadata.loc[self.table.ids()]
            if self.metadata.index.name is None:
                raise ValueError('`Index` must have a name either'
                                 '`sampleid`, `sample-id` or #SampleID')
            self.index_name = self.metadata.index.name
            self.metadata = self.metadata.reset_index()

        self.batch_indices = None
        if self.batch_category is not None and self.metadata is not None:
            batch_cats = np.unique(self.metadata[self.batch_category].values)
            self.batch_cats = pd.Series(
                np.arange(len(batch_cats)), index=batch_cats)
            self.batch_indices = np.array(
                list(map(lambda x: self.batch_cats.loc[x],
                         self.metadata[self.batch_category].values)))

    def __len__(self) -> int:
        return len(self.table.ids())

    def __getitem__(self, i):
        """ Returns all of the samples for a given subject

        Returns
        -------
        counts : np.array
            OTU counts for specified samples.
        batch_indices : np.array
            Membership ids for batch samples. If not specified, return None.
        """
        sample_idx = self.table.ids()[i]
        if self.batch_indices is not None:
            batch_indices = self.batch_indices[i]
        else:
            batch_indices = None
        counts = self.table.data(id=sample_idx, axis='sample')
        return counts, batch_indices

    def __iter__(self):
        worker_info = torch.utils.data.get_worker_info()
        start = 0
        end = self.__len__()

        if worker_info is None:  # single-process data loading
            for i in range(end):
                yield self.__getitem__(i)
        else:
            worker_id = worker_info.id
            w = float(worker_info.num_workers)
            t = (end - start)
            w = float(worker_info.num_workers)
            per_worker = int(math.ceil(t / w))
            worker_id = worker_info.id
            iter_start = start + worker_id * per_worker
            iter_end = min(iter_start + per_worker, end)
            for i in range(iter_start, iter_end):
                yield self.__getitem__(i)

def _sample2dict(feature_data, feature_ids, i, sample_ids=None):
    # This is taking code from here
    # https://github.com/qiime2/q2-sample-classifier/
    # blob/master/q2_sample_classifier/utilities.py#L64
    features = np.empty(1, dtype=dict)
    row = feature_data[i]
    features = {feature_ids[ix]: d
                for ix, d in zip(row.indices, row.data)}
    return features, i

class Q2BiomDataset(BiomDataset):
    """ This is specific to q2 sample classifiers. """
    def __init__(self, table: biom.Table):
        super(Q2BiomDataset).__init__()
        self.feature_ids = table.ids('observation')
        self.feature_data = table.matrix_data.T.tocsr()

    def __getitem__(self, i):
        return _sample2dict(self.feature_data, self.feature_ids, i)

def collate_single_f(batch):
    counts_list = np.vstack([b[0] for b in batch])
    counts = torch.from_numpy(counts_list).float()
    return counts

def collate_batch_f(batch):
    counts_list = np.vstack([b[0] for b in batch])
    batch_ids = np.vstack([b[1] for b in batch])
    counts = torch.from_numpy(counts_list).float()
    batch_ids = torch.from_numpy(batch_ids).long()
    return counts, batch_ids.squeeze()

def collate_q2_f(batch):
    features = [b[0] for b in batch]
    sample_idx = np.vstack([b[1] for b in batch])
    return features, sample_idx.squeeze()