# Copyright 2015 Leon Sixt
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#    http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

import theano
import numpy as np
import copy

from keras.models import Sequential, Graph
from keras.objectives import mse, binary_crossentropy
import keras.initializations
from keras.layers.core import Dense, Flatten, Reshape, Activation, \
    Layer, Merge, Dropout, Lambda
from keras.layers.convolutional import Convolution2D, MaxPooling2D, UpSampling2D
from keras.layers.advanced_activations import LeakyReLU
from keras.layers.normalization import BatchNormalization
from keras.layers.noise import GaussianNoise
from keras.regularizers import Regularizer
from keras.engine.topology import merge
import keras.backend as K

from beras.layers.attention import RotationTransformer
from beras.gan import GAN, gan_outputs
from beras.util import sequential, concat, collect_layers, load_weights, \
    namespace
from beras.regularizers import ActivityInBoundsRegularizer, \
    SumOfActivityBelowRegularizer, with_regularizer
from beras.layers.core import Split, SplitAt, ZeroGradient, LinearInBounds, Switch

from beesgrid import NUM_MIDDLE_CELLS, TAG_SIZE

from deepdecoder.deconv import Deconvolution2D
from deepdecoder.keras_fix import Convolution2D as TheanoConvolution2D
from deepdecoder.utils import binary_mask, rotate_by_multiple_of_90
from deepdecoder.data import nb_normalized_params
from deepdecoder.mask_loss import pyramid_loss
from deepdecoder.transform import PyramidBlending, PyramidReduce, \
    AddLighting, Selection, GaussianBlur, HighFrequencies, \
    LowFrequenciesRegularizer

def get_decoder_model(nb_output=NUM_MIDDLE_CELLS, inputs=['input'],
                      add_output=True, batch_size=None):

    def conv_bn_relu(model, nb_filter, kernel_size=(3, 3), **kwargs):
        model.add(Convolution2D(nb_filter, *kernel_size, **kwargs))
        model.add(BatchNormalization(axis=1))
        model.add(Activation('relu'))

    size = 64
    rot_model = Sequential()
    conv_bn_relu(rot_model, 16, batch_input_shape=(batch_size, 1, size, size))
    rot_model.add(MaxPooling2D((3, 3)))
    conv_bn_relu(rot_model, 16)
    rot_model.add(MaxPooling2D((3, 3)))
    conv_bn_relu(rot_model, 16)
    rot_model.add(MaxPooling2D((3, 3)))
    rot_model.add(Flatten())
    rot_model.add(Dense(64))
    rot_model.add(BatchNormalization(axis=1))
    rot_model.add(Activation('relu'))

    rot_model.add(Dense(1, activation='linear'))

    model = Sequential()
    conv_bn_relu(model, 32, batch_input_shape=(batch_size, 1, size, size))
    model.add(MaxPooling2D(pool_size=(2, 2)))
    conv_bn_relu(model, 64)
    model.add(MaxPooling2D(pool_size=(2, 2)))
    conv_bn_relu(model, 128)
    model.add(MaxPooling2D(pool_size=(2, 2)))
    conv_bn_relu(model, 256)
    model.add(Flatten())

    g = Graph()
    for input in inputs:
        g.add_input(input, batch_input_shape=(batch_size, 1, size, size))

    g.add_node(rot_model, "rot", inputs=inputs,
               concat_axis=0)
    g.add_node(RotationTransformer(rot_model), "rot_input", inputs=inputs,
               concat_axis=0)
    g.add_node(model, "conv_encoder", input="rot_input")
    g.add_node(Dense(1024, activation='relu'), "dense1",
               inputs=["conv_encoder", "rot"])
    g.add_node(Dense(nb_output, activation='linear'), "encoder",
               input="dense1")
    if add_output:
        g.add_output("output", input="encoder")
    return g

def autoencoder_mask_decoder(nb_output=21, batch_size=128,
                             inputs=['mask', 'real']):

    def conv_bn_relu(model, nb_filter, kernel_size=(3, 3), **kwargs):
        model.add(Convolution2D(nb_filter, *kernel_size, **kwargs))
        model.add(BatchNormalization(axis=1))
        model.add(Activation('relu'))
    size = 64
    shape = (len(inputs)*batch_size, 1, 64, 64)
    pre_st = Sequential()
    pre_st.add(Convolution2D(48, 3, 3, activation='relu',
               batch_input_shape=shape))
    conv_bn_relu(pre_st, 48, border_mode='same')
    pre_st.add(MaxPooling2D())
    pre_st.add(Convolution2D(96, 3, 3, activation='relu'))
    conv_bn_relu(pre_st, 96, border_mode='same')

    print(pre_st.output_shape)
    st_model = Sequential()
    st_model.add(MaxPooling2D((3, 3),
                 batch_input_shape=pre_st.output_shape))
    conv_bn_relu(st_model, 48)
    st_model.add(Flatten())
    st_model.add(Dense(128))
    st_model.add(BatchNormalization(axis=1))
    st_model.add(Activation('relu'))
    st_model.add(Dense(6, activation='linear'))

    post_st = Sequential()
    post_st.add(MaxPooling2D(batch_input_shape=pre_st.output_shape))
    post_st.add(Convolution2D(96, 3, 3, activation='relu'))
    conv_bn_relu(post_st, 96, border_mode='same')
    post_st.add(MaxPooling2D())
    post_st.add(Convolution2D(128, 3, 3, activation='relu'))
    conv_bn_relu(post_st, 128, border_mode='same')
    post_st.add(Flatten())

    g = Graph()
    for input in inputs:
        g.add_input(input, batch_input_shape=(batch_size, 1, size, size))

    g.add_node(pre_st, "pre_st", inputs=inputs,
               concat_axis=0)
    g.add_node(st_model, "st", input="pre_st",
               concat_axis=0)
    g.add_node(GraphSpatialTransformer(st_model), "st_input", input="pre_st",
               concat_axis=0)

    g.add_node(post_st, "post_st", input="st_input")
    g.add_node(Dense(512, activation='relu'), "dense1",
               inputs=["post_st", "st"])
    g.add_node(Dense(512, activation='relu'), "dense2",
               input="dense1")
    g.add_node(Dense(nb_output, activation='linear'), "encoder",
               input="dense2")
    return g

def batch_norm():
    mode = 0
    bn = BatchNormalization(mode, axis=1)
    return bn

def normal(scale=0.02):
    def normal_wrapper(shape, name=None):
        return keras.initializations.normal(shape, scale, name)
    return normal_wrapper

def deconv_bn(model, nb_filter, h=3, w=3, init=normal(0.02),
              activation='relu', upsample=True, **kwargs):
    if upsample:
        subsample = (2, 2)
    else:
        subsample = (1, 1)

    model.add(Deconvolution2D(nb_filter, h, w, subsample=subsample,
                              border_mode=(h//2, w//2), init=init, **kwargs))
    model.add(BatchNormalization(axis=1))
    if activation is not None:
        model.add(Activation(activation))

def mask_generator_with_conv(nb_units=64, input_dim=20, init=normal(0.02),
                             nb_output_channels=1, filter_size=3):
    n = nb_units
    f = filter_size
    def deconv(nb_filter, h, w):
        return Deconvolution2D(nb_filter, h, w, subsample=(2, 2),
                               border_mode=(h//2, w//2), init=init)

    model = Sequential()
    model.add(Dense(5*input_dim, input_dim=input_dim, init=init))
    model.add(batch_norm())
    model.add(Activation('relu'))

    model.add(Dense(8*n*4*4, input_dim=input_dim, init=init))
    model.add(batch_norm())
    model.add(Activation('relu'))
    model.add(Reshape((8*n, 4, 4,)))

    model.add(deconv(4*n, f, f))
    model.add(batch_norm())
    model.add(Activation('relu'))

    model.add(Convolution2D(4*n, f, f, border_mode='same'))
    model.add(batch_norm())
    model.add(Activation('relu'))

    model.add(deconv(2*n, f, f))
    model.add(batch_norm())
    model.add(Activation('relu'))

    model.add(Convolution2D(2*n, f, f, border_mode='same'))
    model.add(batch_norm())
    model.add(Activation('relu'))

    model.add(deconv(n, f, f))
    model.add(batch_norm())
    model.add(Activation('relu'))

    model.add(Convolution2D(n, f, f, border_mode='same'))
    model.add(batch_norm())
    model.add(Activation('relu'))

    model.add(Deconvolution2D(nb_output_channels,
                              f, f, border_mode=(1, 1), init=init))
    model.add(Activation('linear'))
    return model

class MinCoveredRegularizer(Regularizer):
    def __init__(self, mask_size=32, min_covered=1/8, max_loss=4):
        self.mask_size = mask_size
        self.min_covered = min_covered
        self.max_loss = max_loss
        self.uses_learning_phase = True

    def set_layer(self, layer):
        self.layer = layer

    def __call__(self, loss):
        if not hasattr(self, 'layer'):
            raise Exception('Need to call `set_layer` on '
                            'MaskRegularizer instance '
                            'before calling the instance.')
        min_tag_size = self.mask_size**2 * self.min_covered
        factor = min_tag_size / self.max_loss
        out = self.layer.output
        out_sum = out.sum(axis=(1, 2, 3))
        reg_loss = K.switch(out_sum <= min_tag_size,
                            factor*(out_sum - min_tag_size)**2, 0)
        return K.in_train_phase(loss + reg_loss.mean(), loss)

def mask_generator(input, nb_units=64, dense_factor=3, nb_dense_layers=2,
                   trainable=True):
    n = nb_units

    def conv(n):
        return [
            Convolution2D(n, 3, 3, border_mode='same', init='he_normal'),
            Activation('relu'),
        ]

    dense_layers = [
        Dense(dense_factor*nb_units, activation='relu')
        for _ in range(nb_dense_layers)]
    return sequential(dense_layers + [
        Dense(8*n*4*4),
        Activation('relu'),
        Reshape((8*n, 4, 4,)),
        UpSampling2D(),  # 8x8
        conv(4*n),
        conv(4*n),
        UpSampling2D(),  # 16x16
        conv(2*n),
        conv(2*n),
        UpSampling2D(),  # 32x32
        conv(n),
        UpSampling2D(),  # 64x64
        conv(n),
        Convolution2D(1, 3, 3, border_mode='same', init='he_normal'),
        Activation('linear'),
    ], ns='mask_gen', trainable=trainable)(input)

def dcgan_small_generator(nb_units=64, input_dim=20, init=normal(0.02),
                          dense_factor=2,
                          nb_dense_layers=2,
                          nb_output_channels=1, filter_size=3,
                          deconv_layers=False,
                          output_size=32):
    n = nb_units
    f = filter_size

    def deconv_up(nb_filter, h, w):
        return Deconvolution2D(nb_filter, h, w, subsample=(2, 2),
                               border_mode=(h//2, w//2), init=init)

    def deconv(nb_filter, h, w):
        return Deconvolution2D(nb_filter, h, w, subsample=(1, 1),
                               border_mode=(h//2, w//2), init=init)
    model = Sequential()
    model.add(Layer(input_shape=(input_dim,)))
    for _ in range(nb_dense_layers):
        model.add(Dense(dense_factor*nb_units, input_dim=input_dim,
                        activation='relu'))

    model.add(Dense(8*n*4*4, input_dim=input_dim))
    model.add(Activation('relu'))
    model.add(Reshape((8*n, 4, 4,)))

    if deconv_layers:
        model.add(deconv(8*n, f, f))
        model.add(Activation('relu'))

    model.add(deconv_up(4*n, f, f))
    model.add(Activation('relu'))

    if deconv_layers:
        model.add(deconv(4*n, f, f))
        model.add(Activation('relu'))

    if output_size >= 16:
        model.add(deconv_up(2*n, f, f))
        model.add(Activation('relu'))

        if deconv_layers:
            model.add(deconv(2*n, f, f))
            model.add(Activation('relu'))

    if output_size == 32:
        model.add(deconv_up(n, f, f))
        model.add(Activation('relu'))

    if deconv_layers:
        model.add(deconv(n, f, f))
        model.add(Activation('relu'))

    model.add(Deconvolution2D(nb_output_channels,
                              f, f, border_mode=(1, 1), init=init))

    model.add(Activation('linear'))
    return model

def dcgan_pyramid_mse_generator(generator, input_dim=40):
    g = Graph()
    g.add_input("input", input_shape=(input_dim, ))
    g.add_node(generator, "generator", input="input")
    g.add_node(Split(-2, None, axis=1), "mean", input="input")
    g.add_output("output", input="generator")
    return g

def autoencoder(encoder, decoder):
    encoder.add_node(decoder, "decoder", input='encoder')
    encoder.add_output("output", input="decoder")
    return encoder

def autoencoder_with_mask_loss(encoder, decoder, batch_size=128):
    half = batch_size // 2
    encoder.add_node(Split(0, half), 'encoder_real', input='encoder')
    encoder.add_node(Split(half, batch_size), 'encoder_mask', input='encoder',
                     create_output=True)
    encoder.add_node(decoder, "decoder", input='encoder_real',
                     create_output=True)
    return encoder

def get_mask_driver(x, nb_units, nb_output_units):
    n = nb_units
    driver = sequential([
        Dense(n),
        BatchNormalization(),
        Dropout(0.25),
        Activation('relu'),
        Dense(n),
        BatchNormalization(),
        Dropout(0.25),
        Activation('relu'),
        Dense(nb_output_units),
        BatchNormalization(),
        LinearInBounds(-1, 1),
    ], ns='driver')
    return driver(x)

def get_offset_front(inputs, nb_units):
    n = nb_units
    input = concat(inputs)

    return sequential([
        Dense(8*n*4*4),
        BatchNormalization(),
        Activation('relu'),
        Reshape((8*n, 4, 4)),
        UpSampling2D(),  # 8x8
        conv(4*n, 3, 3),
        conv(4*n, 3, 3),
        UpSampling2D(),  # 16x16
        conv(2*n, 3, 3),
        conv(2*n, 3, 3),
    ], ns='offset.front')(input)

def get_offset_middle(inputs, nb_units):
    n = nb_units
    input = concat(inputs)
    return sequential([
        UpSampling2D(),  # 32x32
        conv(2*n, 3, 3),
        conv(2*n, 3, 3),
        conv(2*n, 3, 3),
    ], ns='offset.middle')(input)

def get_offset_back(inputs, nb_units):
    n = nb_units
    input = concat(inputs)
    back_feature_map = sequential([
        UpSampling2D(),  # 64x64
        conv(n, 3, 3),
        conv(n, 3, 3),
    ], ns='offset.back')(input)

    return back_feature_map, sequential([
        Convolution2D(1, 3, 3, border_mode='same'),
        LinearInBounds(-1, 1, clip=True),
    ], ns='offset.back_out')(back_feature_map)

def get_mask_weight_blending(inputs, min=0, max=2):
    input = concat(inputs)
    return sequential([
        Convolution2D(1, 3, 3),
        Flatten(),
        Dense(1),
        LinearInBounds(K.variable(min), K.variable(2), clip=True),
    ], ns='mask_weight_blending')(input)

def get_lighting_generator(inputs, nb_units):
    n = nb_units
    input = concat(inputs)
    light_conv = sequential([
        conv(n, 5, 5),
        conv(n, 5, 5),
        conv(n, 3, 3),
        UpSampling2D(),  # 32x32
        conv(n, 5, 5),
        Convolution2D(2, 1, 1, border_mode='same'),
        UpSampling2D(),  # 64x64
        LinearInBounds(-1, 1, clip=True),
        GaussianBlur(sigma=4),
    ], ns='lighting')(input)

    shift = Split(0, 1, axis=1)(light_conv)
    scale = Split(1, 2, axis=1)(light_conv)

    return shift, scale

def get_mask_postprocess(inputs, nb_units):
    n = nb_units
    return sequential([
        conv(n, 3, 3),
        conv(n, 3, 3),
        Deconvolution2D(1, 5, 5, border_mode=(2, 2)),
        LinearInBounds(-1, 1, clip=True),
    ], ns='mask_post')(concat(inputs))

def get_offset_merge_mask(input, nb_units, nb_conv_layers, poolings=None, ns=None):
    def conv_layers(units, pooling):
        layers = [Convolution2D(units, 3, 3, border_mode='same')]
        if pooling:
            layers.append(MaxPooling2D())
        layers.extend([
            BatchNormalization(axis=1),
            Activation('relu'),
        ])
        return layers

    if poolings is None:
        poolings = [False] * nb_conv_layers
    if type(nb_units) == int:
        nb_units = [nb_units] * nb_conv_layers
    layers = []
    for i, (units, pooling) in enumerate(zip(nb_units, poolings)):
        layers.extend(conv_layers(units, pooling))
    return sequential(layers, ns=ns)(input)

def get_bits(z):
    return Lambda(lambda x: 2*K.cast(x > 0, K.floatx()) - 1)(z)

class NormSinCosAngle(Layer):
    def __init__(self, idx, **kwargs):
        self.sin_idx = idx
        self.cos_idx = idx + 1
        super().__init__(**kwargs)

    def get_output(self, x, mask=None):
        sin = x[:, self.sin_idx]
        cos = x[:, self.cos_idx]
        eps = 1e7
        scale = K.sqrt(1/(eps + sin**2 + cos**2))
        sin = K.clip(scale*sin, -1, 1)
        cos = K.clip(scale*cos, -1, 1)
        return K.concatenate([x[:, :self.sin_idx], scale*sin, scale*cos,
                              x[:, self.cos_idx+1:]], axis=1)

def mask_blending_generator(
        mask_driver,
        mask_generator,
        light_merge_mask16,
        offset_merge_mask16,
        offset_merge_mask32,
        offset_merge_light16,
        lighting_generator,
        offset_front,
        offset_middle,
        offset_back,
        mask_weight_blending32,
        mask_weight_blending64,
        mask_postprocess,
        z_for_bits,
        z_for_driver,
        z_for_offset,
        mask_generator_weights=None,
        ):

    def generator(inputs):
        z, = inputs
        z_driver = Split(*z_for_driver, axis=1)(z)
        z_offset = Split(*z_for_offset, axis=1)(z)
        z_bits = Split(*z_for_bits, axis=1)(z)
        bits = get_bits(z_bits)
        driver = mask_driver(z_driver)
        driver_norm = NormSinCosAngle(0)(driver)
        mask_input = concat([bits, driver_norm], name='mask_gen_in')
        mask = mask_generator(mask_input)

        if mask_generator_weights:
            mask_layers = collect_layers(mask_input, mask)
            load_weights(mask_layers, mask_generator_weights)

        selection = with_regularizer(Selection(threshold=-0.08,
                                               smooth_threshold=0.2,
                                               sigma=1.5, name='selection'),
                                     MinCoveredRegularizer())

        mask_down = PyramidReduce()(mask)
        mask_selection = selection(mask)
        mask_selection_down = PyramidReduce(scale=4)(mask_selection)
        out_offset_front = offset_front([z_offset,
                                         ZeroGradient()(driver_norm)])

        light_scale64, light_shift64 = \
            lighting_generator([out_offset_front,
                                light_merge_mask16(mask_selection_down)])

        mask_with_lighting = AddLighting(
            scale_factor=0.5, shift_factor=0.75)(
                [mask, light_scale64, light_shift64])

        out_offset_middle = offset_middle(
            [out_offset_front, offset_merge_mask16(mask_selection_down),
             offset_merge_light16(concat(light_scale64, light_shift64))
             ])

        offset_back_feature_map, out_offset_back = offset_back(
            [out_offset_middle, offset_merge_mask32(mask_down)])

        mask_weight32 = mask_weight_blending32(out_offset_middle)
        mask_weight64 = mask_weight_blending64(out_offset_middle)

        blending = PyramidBlending(offset_pyramid_layers=3,
                                   mask_pyramid_layers=3,
                                   mask_weights=['variable', 'variable', 1],
                                   offset_weights=[1, 1, 1],
                                   use_selection=[True, True, True],
                                   name='blending')(
                [out_offset_back, mask_with_lighting, mask_selection,
                 mask_weight32, mask_weight64])

        mask_post = mask_postprocess([
            blending, mask_selection, light_scale64, light_shift64, mask,
            out_offset_back, offset_back_feature_map
        ])
        mask_post_high = HighFrequencies(4, nb_steps=5,
                                         name='mask_post_high')(mask_post)
        blending_post = merge([mask_post_high, blending], mode='sum',
                              name='blending_post')
        return LinearInBounds(-1.2, 1.2)(blending_post)

    return generator

def conv(nb_filter, h=3, w=3, activation='relu'):
    if issubclass(type(activation), Layer):
        activation_layer = activation
    else:
        activation_layer = Activation(activation)

    return [
        Convolution2D(nb_filter, h, w, border_mode='same'),
        BatchNormalization(axis=1),
        activation_layer
    ]

def deconv(model, nb_filter, h, w, activation='relu'):
    def deconv_init(shape, name=None):
        w = np.random.normal(0, 0.02, shape).astype(np.float32)
        nb_filter = shape[1]
        mask = np.random.binomial(1, 10/nb_filter, (1, nb_filter, 1, 1))
        w *= mask
        return K.variable(w, name=name)

    model.add(Deconvolution2D(nb_filter, h, w, border_mode=(1, 1),
                              init=deconv_init))
    model.add(BatchNormalization(axis=1))
    if issubclass(type(activation), Layer):
        model.add(activation)
    else:
        model.add(Activation(activation))

def get_offset_generator(n=32, batch_input_shape=50, nb_output_channels=1,
                         init=normal(0.02), merge_layer=None):
    def middle_input_shape():
        input_shape = list(front.output_shape)
        if merge_layer is not None:
            input_shape[1] += merge_layer.output_shape[1]
        return tuple(input_shape)

    front = Sequential()
    front.add(Dense(8*n*4*4, batch_input_shape=batch_input_shape, init=init))
    front.add(BatchNormalization())
    front.add(Activation('relu'))
    front.add(Reshape((8*n, 4, 4,)))

    up(front)  # 8x8
    conv(front, 4*n, 3, 3)

    up(front)  # 16x16
    conv(front, 2*n, 3, 3)

    middle = Sequential()
    middle.add(Layer(batch_input_shape=middle_input_shape()))
    up(middle)
    conv(middle, n, 3, 3)
    deconv(front, n, 3, 3)
    conv(middle, n, 3, 3)
    return front, middle

def dcgan_generator(n=32, input_dim=50, nb_output_channels=1, use_dct=False,
                    init=normal(0.02)):

    def deconv(nb_filter, h, w):
        return Deconvolution2D(nb_filter, h, w, subsample=(2, 2),
                               border_mode=(2, 2), init=init)
    model = Sequential()
    model.add(Dense(8*n*4*4, input_dim=input_dim, init=init))
    model.add(batch_norm())
    model.add(Reshape((8*n, 4, 4,)))
    if use_dct:
        model.add(iDCT())
    model.add(Activation('relu'))

    model.add(deconv(4*n, 5, 5))
    model.add(batch_norm())
    model.add(Activation('relu'))

    model.add(deconv(2*n, 5, 5))
    model.add(batch_norm())
    model.add(Activation('relu'))

    model.add(deconv(n, 5, 5))
    model.add(batch_norm())
    model.add(Activation('relu'))

    model.add(Deconvolution2D(nb_output_channels, 5, 5, subsample=(2, 2),
                              border_mode=(2, 2), init=init))
    model.add(LinearInBounds(-1, 1))
    return model

def dcgan_generator_conv(n=32, input_dim=50, nb_output_channels=1,
                         init=normal(0.02)):
    def conv(nb_filter, h, w):
        model.add(Convolution2D(nb_filter, h, w, border_mode='same',
                                init=init))
        model.add(BatchNormalization(axis=1))
        model.add(Activation('relu'))

    def deconv(nb_filter, h, w):
        deconv_layer = Deconvolution2D(nb_filter, h, w, border_mode=(1, 1),
                                       init=init)
        model.add(deconv_layer)

        w = np.random.normal(0, 0.02, deconv_layer.W_shape).astype(np.float32)
        w *= np.random.uniform(0, 1, (1, w.shape[1], 1, 1))
        deconv_layer.W.set_value(w)
        model.add(BatchNormalization(axis=1))
        model.add(Activation('relu'))

    def up():
        model.add(UpSampling2D())

    z = Layer(input_shape=(input_dim,))
    model = Sequential()
    model.add(z)
    model.add(Dense(8*n*4*4, init=init))
    model.add(batch_norm())
    model.add(Reshape((8*n, 4, 4,)))
    model.add(Activation('relu'))

    up()  # 8
    conv(4*n, 3, 3)
    up()  # 16
    conv(2*n, 3, 3)
    conv(2*n, 3, 3)
    up()  # 32
    conv(n, 3, 3)
    conv(n, 3, 3)
    up()  # 64
    conv(n, 3, 3)

    model.add(Deconvolution2D(nb_output_channels, 3, 3, border_mode=(1, 1),
                              init=init))
    model.add(LinearInBounds(-1, 1))
    return model

def mask_blending_discriminator(x, n=32, out_activation='sigmoid'):
    def conv(n):
        return [
            Convolution2D(n, 5, 5, subsample=(2, 2), border_mode='same'),
            BatchNormalization(),
            LeakyReLU(0.2),
        ]

    return sequential([
        Convolution2D(n, 5, 5, subsample=(2, 2), border_mode='same'),
        LeakyReLU(0.2),
        conv(2*n),
        conv(4*n),
        conv(8*n),
        Flatten(),
        Dense(1, activation=out_activation)
    ], ns='dis')(concat(x, axis=0, name='concat_fake_real'))

def mask_blending_gan(offset_generator, mask_generator, discriminator,
                      nb_fake=64, nb_real=32):
    assert len(mask_generator.input_shape) == 2
    assert len(offset_generator.input_shape) == 2

    g = Graph()
    mask_input_dim = mask_generator.input_shape[1]
    z_shape = (nb_fake, offset_generator.input_shape[1] - mask_input_dim)
    g.add_input(GAN.z_name, batch_input_shape=z_shape)

    g.add_node(Dense(mask_input_dim, activation='relu'), 'dense_mask',
               input=GAN.z_name)
    g.add_node(mask_generator, 'mask_generator', input='dense_mask')
    g.add_node(offset_generator, 'gen_offset',
               inputs=[GAN.z_name, 'dense_mask'])
    g.add_node(PyramidBlending(mask_generator, input_pyramid_layers=3,
                               mask_pyramid_layers=2),
               GAN.generator_name,
               input='gen_offset')

    real_shape = (nb_real,) + g.nodes[GAN.generator_name].output_shape[1:]
    g.add_input(GAN.real_name, batch_input_shape=real_shape)
    g.add_node(discriminator, "discriminator",
               inputs=[GAN.generator_name, "real"], concat_axis=0)
    gan_outputs(g, fake_for_gen=(0, nb_fake),
                    fake_for_dis=(nb_fake//2, nb_fake),
                    real=(nb_fake, nb_fake+nb_real))
    return g

def mask_blending_gan_new(offset_generator, mask_generator, discriminator,
                          nb_fake=64, nb_real=32):
    assert len(mask_generator.input_shape) == 2
    assert len(offset_generator.input_shape) == 2

    g = Graph()
    mask_input_dim = mask_generator.input_shape[1]
    z_shape = (nb_fake, offset_generator.input_shape[1] - mask_input_dim)

    g.add_input(GAN.z_name, batch_input_shape=z_shape)

    g.add_node(Dense(32), 'gen_driver_dense_1', input=GAN.z_name)
    g.add_node(BatchNormalization(), 'gen_driver_bn_1',
               input='gen_driver_dense_1')
    g.add_node(Activation('relu'), 'gen_driver_act_1',
               input='gen_driver_bn_1')

    g.add_node(Dense(mask_input_dim), 'gen_driver_dense_2',
               input='gen_driver_act_1')
    g.add_node(BatchNormalization(), 'gen_driver_bn_2',
               input='gen_driver_dense_2')
    g.add_node(Layer(), 'driver', input='gen_driver_bn_2')
    # g.add_node(ZeroGradient(), 'gen_driver_zero_grad', input='driver')

    g.add_node(mask_generator, 'mask_generator', input='driver')
    g.add_node(offset_generator, 'gen_offset',
               input=GAN.z_name)
    g.add_node(PyramidBlending(mask_generator, input_pyramid_layers=3,
                               mask_pyramid_layers=2),
               'blending', input='gen_offset')
    reg_layer = Layer()
    act = ActivityInBoundsRegularizer(-1, 1)
    act.set_layer(reg_layer)
    reg_layer.regularizers = [act]
    g.add_node(reg_layer, GAN.generator_name, input='blending')

    real_shape = (nb_real,) + g.nodes[GAN.generator_name].output_shape[1:]
    g.add_input(GAN.real_name, batch_input_shape=real_shape)
    g.add_node(discriminator, "discriminator",
               inputs=[GAN.generator_name, "real"], concat_axis=0)
    gan_outputs(g, fake_for_gen=(0, nb_fake - nb_real),
                    fake_for_dis=(nb_fake - nb_real, nb_fake),
                    real=(nb_fake, nb_fake+nb_real))
    return g

def dcgan_discriminator(n=32, image_views=1, extra_dense_layer=False,
                        out_activation='sigmoid'):
    model = Sequential()
    model.add(TheanoConvolution2D(n, 5, 5, subsample=(2, 2),
                                  border_mode='full',
                                  input_shape=(1, 64, 64*image_views)))
    model.add(LeakyReLU(0.2))

    model.add(TheanoConvolution2D(2*n, 5, 5, subsample=(2, 2),
                                  border_mode='full'))
    model.add(batch_norm())
    model.add(LeakyReLU(0.2))

    model.add(TheanoConvolution2D(4*n, 5, 5, subsample=(2, 2),
                                  border_mode='full'))
    model.add(batch_norm())
    model.add(LeakyReLU(0.2))

    model.add(TheanoConvolution2D(8*n, 5, 5, subsample=(2, 2),
                                  border_mode='full'))
    model.add(batch_norm())
    model.add(LeakyReLU(0.2))

    model.add(Flatten())
    if extra_dense_layer:
        model.add(Dense(4*n))
        model.add(LeakyReLU(0.2))

    model.add(Dense(1, activation=out_activation))
    return model

def dcgan_generator_add_preprocess_network(g, input_dim=50):
    p = Sequential()
    p.add(Dense(g.layers[0].input_shape[1],
                activation='relu', input_dim=input_dim))
    g.trainable = False
    p.add(g)
    return p

def dcgan_seperated_generator(input_dim=40, model_generator=None,
                              variation_generator=None):
    if model_generator is None:
        model_generator = dcgan_generator(input_dim)
    if variation_generator is None:
        variation_generator = dcgan_generator(input_dim, include_last_layer=False)

    g = Graph()
    g.add_input('input', input_shape=(input_dim,))
    g.add_node(model_generator, 'model', input='input')
    g.add_node(variation_generator, 'variation', input='input')
    g.add_node(Deconvolution2D(1, 5, 5, subsample=(2, 2), border_mode=(2, 2)),
              'variation_deconv', input='variation')
    g.add_node(Activation('sigmoid'), 'variation_out', input='variation_deconv')
    g.add_output('output', inputs=['model', 'variation_out'], concat_axis=1)
    return g

def dcgan_variational_add_generator(nb_z=40, n=16):
    g = Graph()

    class Disconnected(Layer):
        def get_output(self, train=True):
            input = self.get_input(train)
            return theano.gradient.disconnected_grad(input)

    class Builder:
        def __init__(self, g):
            self.g = g
            self.outputs = ['input']
            self.next_input = ''
            self.join_cnt = 0
            self.deconv_filter = n*4
            self.vi = 0
            self.mi = 0

        def v(self):
            name = 'var_{}'.format(self.vi)
            self.vi += 1
            return name

        def m(self):
            name = 'model_{}'.format(self.mi)
            self.mi += 1
            return name

        def add_output(self, name):
            self.outputs.append(name)
            return name

        def get_input(self):
            if self.next_input:
                tmp = copy.copy(self.next_input)
                self.next_input = ''
                return tmp
            else:
                return self.outputs[-1]

        def set_input(self, name):
            self.next_input = name

        def project(self, name):
            dense_name = name + '_dense'
            self.g.add_node(Dense(8*n*4*4), dense_name,
                            input=self.get_input())
            reshape_name = name + '_reshape'
            self.g.add_node(Reshape((8*n, 4, 4,)), reshape_name,
                            input=dense_name)
            return self.add_output(reshape_name)

        def deconv(self, name, batch_norm=True, activation='relu'):
            deconv = name + "_deconv"
            batch = name + "_batch_norm"
            act = name + "_" + activation
            self.g.add_node(Deconvolution2D(
                self.deconv_filter, 5, 5, subsample=(2, 2),
                border_mode=(2, 2)),
                            deconv, input=self.get_input())
            if batch_norm:
                self.g.add_node(BatchNormalization(), batch, input=deconv)
                act_in = batch
            else:
                act_in = deconv
            self.g.add_node(Activation(activation), act, input=act_in)
            return self.add_output(act)

        def join(self, mo, vo):
            name = 'join_{}'.format(self.join_cnt)
            self.join_cnt += 1
            disconn = mo + '_disconn'
            self.g.add_node(Disconnected(), disconn, input=mo)
            self.g.add_node(Layer(), name, inputs=[disconn, vo],
                            concat_axis=1)
            return self.add_output(name)

    g.add_input('input', input_shape=(nb_z,))
    b = Builder(g)
    vo = b.project(b.v())
    b.set_input('input')
    mo = b.project(b.m())
    b.join(mo, vo)

    vo = b.deconv(b.v())
    b.set_input(mo)
    mo = b.deconv(b.m())

    b.deconv_filter //= 2

    b.join(mo, vo)
    vo = b.deconv(b.v())
    b.set_input(mo)
    mo = b.deconv(b.m())

    b.deconv_filter //= 2

    b.join(mo, vo)
    vo = b.deconv(b.v())
    b.set_input(mo)
    mo = b.deconv(b.m())

    b.deconv_filter = 1

    b.join(mo, vo)
    v_out = b.deconv(b.v(), batch_norm=False, activation='sigmoid')
    b.set_input(mo)
    m_out = b.deconv(b.m(), batch_norm=False, activation='sigmoid')

    g.add_output('output', inputs=[m_out, v_out], concat_axis=1)
    return g

def gan_with_z_rot90_grid_idx(generator, discriminator,
                              batch_size=128, nb_z=20, reconstruct_fn=None):
    nb_grid_params = nb_normalized_params()
    z_shape = (batch_size, nb_z)
    grid_shape = (1, TAG_SIZE, TAG_SIZE)
    grid_params_shape = (nb_grid_params, )

    g_graph = Graph()
    g_graph.add_input('z', input_shape=z_shape[1:])
    g_graph.add_input('grid_params', input_shape=grid_params_shape)
    g_graph.add_input('grid_idx', input_shape=grid_shape)
    g_graph.add_input('z_rot90', input_shape=(1, ))
    g_graph.add_node(generator, 'generator', inputs=['grid_params', 'z'])
    g_graph.add_output('output', input='generator')
    g_graph.add_output('z_rot90', input='z_rot90')
    g_graph.add_output('grid_idx', input='grid_idx')
    d_graph = asgraph(discriminator, input_name=GAN.d_input)
    return GAN(g_graph, d_graph, z_shape, reconstruct_fn=reconstruct_fn)

def add_diff_rot90(g_outmap):
    g_out = g_outmap["output"]
    grid_idx = g_outmap['grid_idx']
    z_rot90 = g_outmap['z_rot90']
    alphas = binary_mask(grid_idx, black=0.5, ignore=1.0,  white=0.5)
    bw_mask = binary_mask(grid_idx, black=0., ignore=0,  white=0.5)
    combined = (alphas * g_out) + bw_mask
    return rotate_by_multiple_of_90(combined, z_rot90)

def gan_add_bw_grid(generator, discriminator, batch_size=128, nb_z=20):
    return gan_with_z_rot90_grid_idx(
        generator, discriminator, batch_size=batch_size,
        nb_z=nb_z, reconstruct_fn=add_diff_rot90)

def dcgan_pyramid_generator(nb_dcgan_units=64, z_dim=50, nb_fake=64):
    nb_grid_params = nb_normalized_params()
    g_input_dim = nb_grid_params + z_dim
    g = Graph()
    g.add_input(GAN.z_name, batch_input_shape=(nb_fake, z_dim))
    g.add_input("gen_grid_params",
                batch_input_shape=(nb_fake, nb_grid_params))
    g.add_input("gen_grid_idx", batch_input_shape=(nb_fake, 1, 64, 64))
    tag_mean = Dense(2, activation='relu', init=normal(0.005))
    g.add_node(tag_mean, "tag_mean", inputs=[GAN.z_name, "gen_grid_params"])
    g.add_node(dcgan_generator(nb_dcgan_units, input_dim=g_input_dim),
               "dcgan_generator",
               inputs=[GAN.z_name, "gen_grid_params"])
    g.add_node(PyramidBlending(tag_mean), GAN.generator_name,
               inputs=["dcgan_generator", "gen_grid_idx"], concat_axis=1)
    return g

def mogan_learn_bw_grid(generator, discriminator, optimizer_fn,
                        batch_size=128, nb_z=20):
    variation_weight = 0.3

    def reconstruct(g_outmap):
        g_out = g_outmap["output"]
        grid_idx = g_outmap["grid_idx"]
        z_rot90 = g_outmap['z_rot90']
        alphas = binary_mask(grid_idx, black=variation_weight,
                             ignore=1.0, white=variation_weight)
        m = theano.gradient.disconnected_grad(g_out[:, :1])
        v = g_out[:, 1:]
        combined = v  # T.clip(m + alphas*v, 0., 1.)
        return rotate_by_multiple_of_90(combined, z_rot90)

    grid_loss_weight = theano.shared(np.cast[np.float32](1))

    def grid_loss(g_outmap):
        g_out = g_outmap['output']
        grid_idx = g_outmap["grid_idx"]
        m = g_out[:, :1]
        b = binary_mask(grid_idx, ignore=0, black=0,
                        white=1 - variation_weight)
        return grid_loss_weight*mse(b, m)

    gan = gan_with_z_rot90_grid_idx(generator, discriminator,
                                    batch_size=batch_size, nb_z=nb_z,
                                    reconstruct_fn=reconstruct)
    # FIXME
    mogan = MOGAN(gan, grid_loss, optimizer_fn,
                  gan_regulizer=GAN.L2Regularizer())
    return mogan, grid_loss_weight

def dummy_dcgan_generator(n=32, input_dim=50, nb_output_channels=1,
                          include_last_layer=True):
    model = Sequential()
    model.add(Dense(64*64, input_dim=input_dim, activation='relu'))
    model.add(Reshape((1, 64, 64)))
    return model

def gan_grid_idx(generator, discriminator,
                 batch_size=128, nb_z=20, reconstruct_fn=None):
    nb_grid_params = nb_normalized_params()
    z_shape = (batch_size, nb_z)
    grid_params_shape = (nb_grid_params, )
    g_graph = Graph()
    g_graph.add_input('z', input_shape=z_shape[1:])
    g_graph.add_input('grid_params', input_shape=grid_params_shape)
    g_graph.add_node(generator, 'generator', inputs=['grid_params', 'z'])
    g_graph.add_output('output', input='generator')
    d_graph = asgraph(discriminator, input_name=GAN.d_input)
    return GAN(g_graph, d_graph, z_shape, reconstruct_fn=reconstruct_fn)

def mogan_pyramid(generator, discriminator, optimizer_fn,
                  batch_size=128, nb_z=20,
                  gan_objective=binary_crossentropy,
                  d_loss_grad_weight=0):
    def tag_loss(cond_true, g_out_dict):
        g_out = g_out_dict['output']
        grid_idx = cond_true
        return pyramid_loss(grid_idx, g_out).loss

    gan = gan_grid_idx(generator, discriminator, batch_size, nb_z)
    # FIXME
    mogan = MOGAN(gan, tag_loss, optimizer_fn,
                  gan_regulizer=GAN.L2Regularizer(),
                  gan_objective=gan_objective,
                  d_loss_grad_weight=d_loss_grad_weight)
    return mogan


'''
Created on 03.07.2019

@author: RalphG
'''

from pyspark import SparkConf, SparkContext
from pyspark.tests import SPARK_HOME

# from pyspark.sql import SparkSession

if __name__ == "__main__":
    # Initialize Spark
    spark_cfg = SparkConf().setAppName("WordCounts")
    spark_ctx = SparkContext(conf = spark_cfg)
    spark_ctx.setLogLevel("warn")

    log4jLogger = spark_ctx._jvm.org.apache.log4j
    log = log4jLogger.LogManager.getLogger(__name__)

    log.warn("%-20s: %s" % ("version: ", spark_ctx.version))
    log.warn("%-20s: %s" % ("spark.master: ", spark_cfg.get("spark.master")))
    log.warn("%-20s: %s" % ("spark.app.master: ", spark_cfg.get("spark.app.name")))
    log.warn("%-20s: %s" % ("master: ", spark_ctx.master))
    log.warn("%-20s: %s" % ("spartctx.appName: ", spark_ctx.appName))
    log.warn("%-20s: %s" % ("spartctx.sparkHome: ", spark_ctx.sparkHome))

    log.warn("%-20s: %s" % ("SPARK_HOME", SPARK_HOME))

    FILENAME="../README.md"
    log.warn("%-20s: %s" % ("FILENAME", FILENAME))

    textFile = spark_ctx.textFile(FILENAME)
    log.warn("%-20s: %s" % ("name of file", textFile.name()))

    log.warn("%-20s: %r" % ("DO word_counts", ""))

    try:
        word_counts = textFile           \
                        .flatMap(lambda line: line.split()) \
                        .map(lambda word: (word, 1)) \
                        .reduceByKey(lambda a, b: a+b)
    except Exception as ex:
        word_counts=None
        print(ex)

    log.warn("%-20s: %r" % ("word_counts", word_counts))

    log.warn("%-20s: %r" % ("DO word_collect", ""))

    try:
        word_collect = word_counts.collect()
    except Exception as ex:
        word_collect = None
        print(ex)

    log.warn("%-20s: %r" % ("word_collect", word_collect))

    if (word_collect == None):
        pass
    else:
        for wc in word_collect():
            try:
                print(wc)
            except Exception as ex:
                print(ex)


from celery import Celery

app = Celery('celery_app', broker='redis://redis')
app.conf.update(
    beat_scheduler = 'redbeat.RedBeatScheduler',
    beat_max_loop_interval=5,  # 5 seconds
    beat_schedule = {
        'check-scrapyd-jobs': {
            'task': 'server.tasks.scrapyd.listjobs',
            'schedule': 30,  # 30 seconds
        },
    },
    redbeat_lock_key = None,
    redbeat_lock_timeout = 5,
)


#!/usr/bin/env python
# -*- coding: utf-8 -*-

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals

import datetime
import cStringIO as StringIO

from django.db import models as db
from django.http import HttpResponseForbidden, HttpResponse, Http404
from django.conf import settings
from django.db.models import Q
from django.shortcuts import get_list_or_404
from django.utils.safestring import mark_safe
from django.utils.html import escape

from bob.menu import MenuItem
from dj.choices import Choices

from ralph.account.models import Perm
from ralph.business.models import Venture, VentureExtraCostType
from ralph.cmdb.models_ci import (
    CI,
    CIRelation,
    CI_STATE_TYPES,
    CI_RELATION_TYPES,
    CI_TYPES
)
from ralph.deployment.models import DeploymentStatus
from ralph.discovery.models_component import (
    Component,
    ComponentType,
    FibreChannel,
    GenericComponent,
    Memory,
    OperatingSystem,
    Processor,
    Software,
    Storage,
)
from ralph.discovery.models_device import (
    Device,
    DeviceModelGroup,
    DeviceType,
    MarginKind,
)
from ralph.discovery.models_history import HistoryCost
from ralph.ui.forms import DateRangeForm, MarginsReportForm
from ralph.ui.reports import (
    get_total_cores,
    get_total_cost,
    get_total_count,
    get_total_virtual_cores,
)
from ralph.ui.views.common import Base, DeviceDetailView, _get_details
from ralph.ui.views.devices import DEVICE_SORT_COLUMNS
from ralph.ui.forms.reports import (

    DeprecationRangeReportForm,
    DevicesChoiceReportForm,
    SupportRangeReportForm,
    ReportVentureCost,
    ReportDeviceListForm,
    WarrantyRangeReportForm,
)
from ralph.util import csvutil
from ralph.util.pricing import (
    get_device_auto_price,
    get_device_chassis_price,
    get_device_components_price,
    get_device_cpu_price,
    get_device_fc_price,
    get_device_local_storage_price,
    get_device_memory_price,
    get_device_operatingsystem_price,
    get_device_software_price,
    is_deprecated,
)

def threshold(days):
    return datetime.date.today() + datetime.timedelta(days=days)

class ReportType(Choices):
    _ = Choices.Choice

    no_ping1 = _('No ping since 1 day').extra(
        filter=lambda device_list: device_list.filter(
            ipaddress__last_seen__lte=threshold(-1)),
        columns=['venture', 'position', 'lastseen', 'remarks', 'lastping'],
    )
    no_ping3 = _('No ping since 3 days').extra(
        filter=lambda device_list: device_list.filter(
            ipaddress__last_seen__lte=threshold(-3)),
        columns=['venture', 'position', 'lastseen', 'remarks', 'lastping'],
    )
    no_ping7 = _('No ping since 7 days').extra(
        filter=lambda device_list: device_list.filter(
            ipaddress__last_seen__lte=threshold(-7)),
        columns=['venture', 'position', 'lastseen', 'remarks', 'lastping'],
    )
    no_purchase_date = _('No purchase date').extra(
        filter=lambda device_list: device_list.filter(
            purchase_date=None),
        columns=['venture', 'position', 'barcode', 'cost', 'price', 'remarks'],
    )
    no_venture_role = _('No venture and role').extra(
        filter=lambda device_list: device_list.filter(
            venture_role=None),
        columns=[
            'venture', 'position', 'barcode', 'cost', 'lastseen', 'remarks'
        ],
    )
    deactivated_support = _('Deactivated support').extra(
        filter=lambda device_list: device_list.filter(
            support_expiration_date__lte=datetime.date.today()
        ),
        columns=['venture', 'model', 'position', 'barcode',
                 'serial_number', 'remarks', 'support'],
    )
    support_expires30 = _('Support expires in 30 days').extra(
        filter=lambda device_list: device_list.filter(
            support_expiration_date__lte=threshold(30)),
        columns=[
            'venture', 'model', 'position', 'barcode', 'serial_number',
            'remarks', 'support'
        ],
    )
    support_expires60 = _('Support expires in 60 days').extra(
        filter=lambda device_list: device_list.filter(
            support_expiration_date__lte=threshold(60)),
        columns=[
            'venture', 'model', 'position', 'barcode', 'serial_number',
            'remarks', 'support'
        ],
    )
    support_expires90 = _('Support expires in 90 days').extra(
        filter=lambda device_list: device_list.filter(
            support_expiration_date__lte=threshold(90)),
        columns=['venture', 'model', 'position', 'barcode',
                 'serial_number', 'remarks', 'support'],
    )
    verified = _('Verified venture and role').extra(
        filter=lambda device_list: device_list.filter(verified=True),
        columns=['venture', 'remarks', 'barcode', 'serial_number']
    )
    deployment_open = _('Deployment open').extra(
        filter=lambda device_list: device_list.filter(
            deployment__status=DeploymentStatus.open
        ),
        columns=['venture', 'remarks', 'position', 'barcode'],
    )
    deployment_in_progress = _('Deployment in progress').extra(
        filter=lambda device_list: device_list.filter(
            deployment__status=DeploymentStatus.in_progress
        ),
        columns=['venture', 'remarks', 'position', 'barcode']
    )
    deprecation_devices = _('Deprecation devices').extra(
        filter=lambda device_list: device_list.filter(
            deprecation_date__lte=datetime.date.today()
        ),
        columns=[
            'venture', 'purchase', 'deprecation', 'deprecation_date',
            'remarks', 'barcode'
        ]
    )
    deprecation_devices30 = _('Deprecation devices in 30').extra(
        filter=lambda device_list: device_list.filter(
            deprecation_date__lte=threshold(30)
        ).filter(
            deprecation_date__gte=datetime.date.today()
        ),
        columns=[
            'venture', 'purchase', 'deprecation', 'deprecation_date',
            'remarks', 'barcode'
        ]
    )
    deprecation_devices60 = _('Deprecation devices in 60').extra(
        filter=lambda device_list: device_list.filter(
            deprecation_date__lte=threshold(60)
        ).filter(
            deprecation_date__gte=datetime.date.today()
        ),
        columns=[
            'venture', 'purchase', 'deprecation', 'deprecation_date',
            'remarks', 'barcode'
        ]
    )
    deprecation_devices90 = _('Deprecation devices in 90').extra(
        filter=lambda device_list: device_list.filter(
            deprecation_date__lte=threshold(90)
        ).filter(
            deprecation_date__gte=datetime.date.today()
        ),
        columns=[
            'venture', 'purchase', 'deprecation', 'deprecation_date',
            'remarks', 'barcode'
        ]
    )

class Reports(DeviceDetailView):
    template_name = 'ui/device_reports.html'
    read_perm = Perm.read_device_info_history

    def get_context_data(self, **kwargs):
        result = super(Reports, self).get_context_data(**kwargs)
        return result

class SidebarReports(object):
    section = 'reports'
    subsection = ''

    def get_context_data(self, **kwargs):
        context = super(SidebarReports, self).get_context_data(**kwargs)
        sidebar_items = [
            MenuItem(
                "Devices",
                fugue_icon='fugue-computer',
                view_name='reports_devices'
            ),
            MenuItem(
                "Margins",
                fugue_icon='fugue-piggy-bank',
                view_name='reports_margins'
            ),
            MenuItem(
                "Services",
                fugue_icon='fugue-disc-share',
                view_name='reports_services'
            ),
            MenuItem(
                "Ventures",
                fugue_icon='fugue-store',
                view_name='reports_ventures'
            ),
            MenuItem(
                "Device prices per venture",
                fugue_icon='fugue-computer',
                view_name='device_prices_per_venture'
            ),
        ]
        context.update({
            'sidebar_items': sidebar_items,
            'sidebar_selected': self.subsection,
            'section': 'reports',
            'subsection': self.subsection,
        })
        return context

class ReportMargins(SidebarReports, Base):
    template_name = 'ui/report_margins.html'
    subsection = 'margins'

    def get(self, *args, **kwargs):
        profile = self.request.user.get_profile()
        has_perm = profile.has_perm
        if not has_perm(Perm.read_device_info_reports):
            return HttpResponseForbidden(
                "You don't have permission to see reports."
            )
        self.margin_kinds = MarginKind.objects.all()
        if 'start' in self.request.GET:
            self.form = MarginsReportForm(self.margin_kinds, self.request.GET)
        else:
            self.form = MarginsReportForm(self.margin_kinds, initial={
                'start': datetime.date.today() - datetime.timedelta(days=30),
                'end': datetime.date.today(),
            })
        return super(ReportMargins, self).get(*args, **kwargs)

    def post(self, *args, **kwargs):
        return self.get(*args, **kwargs)

    def get_context_data(self, **kwargs):
        context = super(ReportMargins, self).get_context_data(**kwargs)
        if self.form.is_valid():
            venture = Venture.objects.get(
                id=self.form.cleaned_data['margin_venture']
            )
            query = HistoryCost.objects.filter(
                db.Q(venture=venture) |
                db.Q(venture__parent=venture) |
                db.Q(venture__parent__parent=venture) |
                db.Q(venture__parent__parent__parent=venture) |
                db.Q(venture__parent__parent__parent__parent=venture)
            ).exclude(device__deleted=True)
            total_cost = 0
            total_sim = 0
            total_count = 0
            start = self.form.cleaned_data['start']
            end = self.form.cleaned_data['end']
            for mk in self.margin_kinds:
                q = query.filter(
                    db.Q(device__margin_kind=mk) |
                    db.Q(
                        db.Q(device__margin_kind=None) &
                        db.Q(
                            db.Q(device__venture__margin_kind=mk) |
                            db.Q(device__venture__margin_kind=None,
                                 device__venture__parent__margin_kind=mk) |
                            db.Q(device__venture__margin_kind=None,
                                 device__venture__parent__margin_kind=None,
                                 device__venture__parent__parent__margin_kind=mk) |
                            db.Q(
                                device__venture__margin_kind=None,
                                device__venture__parent__margin_kind=None,
                                device__venture__parent__parent__margin_kind=None,
                                device__venture__parent__parent__parent__margin_kind=mk
                            )
                        )
                    )
                )
                mk.total = get_total_cost(q, start, end)
                mk.count, mk.count_now, devices = get_total_count(
                    q, start, end
                )
                mk.sim_margin = self.form.get('m_%d' % mk.id) or 0
                mk.sim_cost = (
                    (mk.total or 0) / (1 + mk.margin / 100) *
                    (1 + mk.sim_margin / 100)
                )
                total_sim += mk.sim_cost
                total_cost += mk.total or 0
                total_count += mk.count or 0
            context.update({
                'venture': venture,
                'total_cost': total_cost,
                'total_sim': total_sim,
                'total_count': total_count,
            })
        context.update({
            'form': self.form,
            'margin_kinds': self.margin_kinds,
            'zip_margin_kinds_form': zip(
                [f for f in self.form if not f.label], self.margin_kinds
            ),
        })
        return context

def _currency(value):
    return '{:,.2f} {}'.format(value or 0, settings.CURRENCY).replace(',', ' ')

class ReportVentures(SidebarReports, Base):
    template_name = 'ui/report_ventures.html'
    subsection = 'ventures'

    def export_csv(self):
        def iter_rows():
            yield [
                'Venture ID',
                'Venture',
                'Path',
                'Department',
                'Default margin',
                'Device count',
                'Core count',
                'Virtual core count',
                'Cloud use',
                'Cloud cost',
            ] + [extra_type.name for extra_type in self.extra_types] + [
                'Hardware cost',
                'Total cost',
            ]
            for data in self.venture_data:
                yield [
                    '%d' % data['id'],
                    data['name'],
                    data['path'],
                    data['department'],
                    '%d%%' % data['margin'],
                    '%d' % (data['count'] or 0),
                    '%d' % (data['core_count'] or 0),
                    '%d' % (data['virtual_core_count'] or 0),
                    '%f' % (data['cloud_use'] or 0),
                    _currency(data['cloud_cost']),
                ] + [_currency(v) for v in data['extras']] + [
                    _currency(data['hardware_cost']),
                    _currency(data['total']),
                ]
        f = StringIO.StringIO()
        csvutil.UnicodeWriter(f).writerows(iter_rows())
        response = HttpResponse(f.getvalue(), content_type='application/csv')
        response['Content-Disposition'] = 'attachment; filename=ventures.csv'
        return response

    def _get_totals(self, start, end, query, extra_types):
        venture_total = get_total_cost(query, start, end)
        venture_count, venture_count_now, devices = get_total_count(
            # Exclude all non-physical devices.
            query.exclude(
                device__model__type__in=(
                    DeviceType.cloud_server,
                    DeviceType.virtual_server,
                    DeviceType.unknown,
                    DeviceType.data_center,
                    DeviceType.rack,
                    DeviceType.management,
                ),
            ),
            start,
            end,
        )
        venture_core_count = get_total_cores(devices, start, end)
        venture_virtual_core_count = get_total_virtual_cores(
            devices, start, end
        )
        q = query.filter(extra=None)
        venture_hardware_cost = get_total_cost(q, start, end)
        cloud_cost = get_total_cost(
            query.filter(
                device__model__type=DeviceType.cloud_server.id
            ), start, end
        )
        venture_extras = []
        for extra_type in extra_types:
            cost = None
            for extra_cost in extra_type.ventureextracost_set.all():
                q = query.filter(extra=extra_cost)
                c = get_total_cost(q, start, end)
                cost = cost + (c or 0) if cost else c
            venture_extras.append(cost)
        return {
            'count': venture_count,
            'count_now': venture_count_now,
            'core_count': venture_core_count,
            'virtual_core_count': venture_virtual_core_count,
            'hardware_cost': venture_hardware_cost,
            'cloud_cost': cloud_cost,
            'extras': venture_extras,
            'total': venture_total,
        }

    def _get_venture_data(self, start, end, ventures, extra_types):
        total_cloud_cost = get_total_cost(
            HistoryCost.objects.filter(
                device__model__type=DeviceType.cloud_server.id
            ), start, end
        )
        for venture in ventures:
            query = HistoryCost.objects.filter(
                db.Q(venture=venture) |
                db.Q(venture__parent=venture) |
                db.Q(venture__parent__parent=venture) |
                db.Q(venture__parent__parent__parent=venture) |
                db.Q(venture__parent__parent__parent__parent=venture)
            ).exclude(device__deleted=True)
            data = self._get_totals(start, end, query, extra_types)
            data.update({
                'id': venture.id,
                'name': venture.name,
                'symbol': venture.symbol,
                'path': venture.path,
                'department': unicode(venture.department or ''),
                'margin': venture.get_margin(),
                'top_level': venture.parent is None,
                'venture': venture,
                'cloud_use': (
                    (data['cloud_cost'] or 0) / total_cloud_cost
                ) if total_cloud_cost else 0,
            })
            yield data
            if venture.parent is not None:
                continue
            if not venture.child_set.exists():
                continue
            query = HistoryCost.objects.filter(venture=venture)
            data = self._get_totals(start, end, query, extra_types)
            data.update({
                'id': venture.id,
                'name': '-',
                'symbol': venture.symbol,
                'path': venture.path,
                'department': unicode(venture.department or ''),
                'margin': venture.get_margin(),
                'top_level': False,
                'venture': venture,
                'cloud_use': (
                    (data['cloud_cost'] or 0) / total_cloud_cost
                ) if total_cloud_cost else 0,
            })
            yield data

    def get(self, *args, **kwargs):
        profile = self.request.user.get_profile()
        has_perm = profile.has_perm
        if not has_perm(Perm.read_device_info_reports):
            return HttpResponseForbidden(
                "You don't have permission to see reports."
            )
        if 'start' in self.request.GET:
            self.form = DateRangeForm(self.request.GET)
        else:
            self.form = DateRangeForm(initial={
                'start': datetime.date.today() - datetime.timedelta(days=30),
                'end': datetime.date.today(),
            })
        self.extra_types = list(VentureExtraCostType.objects.annotate(
            cost_count=db.Count('ventureextracost')
        ).filter(cost_count__gt=0).order_by('name'))
        if self.form.is_valid():
            self.ventures = profile.perm_ventures(
                Perm.read_device_info_reports
            ).filter(
                db.Q(parent=None) |
                db.Q(parent__parent=None),
                show_in_ralph=True
            ).order_by('path')
            start = self.form.cleaned_data['start']
            end = self.form.cleaned_data['end']
            self.venture_data = self._get_venture_data(
                start,
                end,
                self.ventures,
                self.extra_types,
            )
        else:
            self.ventures = Venture.objects.none()
            self.venture_data = []
        if self.request.GET.get('export') == 'csv':
            return self.export_csv()
        return super(ReportVentures, self).get(*args, **kwargs)

    def get_context_data(self, **kwargs):
        context = super(ReportVentures, self).get_context_data(**kwargs)
        context.update({
            'form': self.form,
            'ventures': self.ventures,
            'venture_data': self.venture_data,
            'profile': self.request.user.get_profile(),
            'extra_types': self.extra_types,
        })
        return context

class ReportServices(SidebarReports, Base):
    template_name = 'ui/report_services.html'
    subsection = 'services'

    def get(self, *args, **kwargs):
        profile = self.request.user.get_profile()
        has_perm = profile.has_perm
        if not has_perm(Perm.read_device_info_reports):
            return HttpResponseForbidden(
                "You don't have permission to see reports.",
            )
        self.perm_to_edit = has_perm(Perm.edit_configuration_item_relations)
        services = CI.objects.filter(type=CI_TYPES.SERVICE.id)
        relations = CIRelation.objects.filter(
            child__type=CI_TYPES.SERVICE.id,
            parent__type=CI_TYPES.VENTURE.id,
            type=CI_RELATION_TYPES.CONTAINS.id,
        )
        self.invalid_relation = []
        for relation in relations:
            child = relation.child
            child.state = CI_STATE_TYPES.NameFromID(child.state)
            child.venture_id = relation.parent.id
            child.venture = relation.parent.name
            child.relation_type = CI_RELATION_TYPES.NameFromID(relation.type)
            child.relation_type_id = relation.type
            self.invalid_relation.append(child)

        self.services_without_venture = []
        for service in services:
            if CIRelation.objects.filter(
                parent=service,
                type=CI_RELATION_TYPES.CONTAINS,
                child__type=CI_TYPES.VENTURE
            ).exists():
                service.state = CI_STATE_TYPES.NameFromID(service.state)
                self.services_without_venture.append(service)
        return super(ReportServices, self).get(*args, **kwargs)

    def get_context_data(self, **kwargs):
        context = super(ReportServices, self).get_context_data(**kwargs)
        context.update({
            'invalid_relation': self.invalid_relation,
            'services_without_venture': self.services_without_venture,
            'perm_to_edit': self.perm_to_edit,
        })
        return context

class ReportDeviceList(object):
    template_name = 'ui/device_report_list.html'

    def get_report_type(self):
        try:
            return ReportType.from_name(self.kwargs['report'])
        except (KeyError, ValueError):
            return ReportType.no_ping1

    def get_context_data(self, **kwargs):
        result = super(ReportDeviceList, self).get_context_data(**kwargs)
        report_menu_items = (
            MenuItem(desc, href=name) for name, desc in
            ReportType(item=lambda v: (v.name, v.desc))
        )
        report_type = self.get_report_type()
        result.update({
            'report_menu_items': report_menu_items,
            'report_selected': report_type.desc.lower(),
            'columns': report_type.columns,
        })
        return result

    def get_queryset(self, queryset=None):
        if queryset is None:
            queryset = super(ReportDeviceList, self).get_queryset()
        queryset = self.get_report_type().filter(queryset).distinct()
        return self.sort_queryset(queryset, columns=DEVICE_SORT_COLUMNS)

class ReportDevices(SidebarReports, Base):
    template_name = 'ui/report_devices.html'
    subsection = 'devices'

    def export_csv(self, data, fname):
        export = []
        for item in data:
            export.append([unicode(x) for x in item])
        f = StringIO.StringIO()
        csvutil.UnicodeWriter(f).writerows(export)
        response = HttpResponse(f.getvalue(), content_type='application/csv')
        response['Content-Disposition'] = 'attachment; filename=%s.csv' % fname
        return response

    def get_name(self, name, id):
        id = escape(id)
        name = escape(name)
        html = '<a href="/ui/search/info/%s">%s</a> (%s)' % (id, name, id)
        return mark_safe(html)

    def get(self, *args, **kwargs):
        profile = self.request.user.get_profile()
        has_perm = profile.has_perm
        if not has_perm(Perm.read_device_info_reports):
            return HttpResponseForbidden(
                "You don't have permission to see reports.")
        self.perm_edit = False
        if has_perm(Perm.edit_device_info_financial):
            self.perm_edit = True
        request = self.request.GET
        csv_conf = {
            'name': 'report_devices',
            'url': None,
        }
        # Filtering of the cross
        self.form_choice = DevicesChoiceReportForm(request)
        queries = {Q()}
        headers = ['Name']
        dep = self.request.GET.get('deprecation')
        no_dep = self.request.GET.get('no_deprecation')
        no_mar = self.request.GET.get('no_margin')
        no_sup = self.request.GET.get('no_support')
        no_pur = self.request.GET.get('no_purchase')
        no_ven = self.request.GET.get('no_venture')
        no_rol = self.request.GET.get('no_role')
        no_par = self.request.GET.get('no_parent')
        rows = []
        if dep:
            headers.append('Depreciation date')
            queries.update({Q(deprecation_date__lte=datetime.date.today())})
        if no_dep:
            headers.append('No depreciation date')
            queries.update({Q(deprecation_date=None)})
        if no_mar:
            headers.append('No depreciation kind')
            queries.update({Q(deprecation_kind=None)})
        if no_sup:
            headers.append('No support')
            queries.update({Q(support_expiration_date=None)})
        if no_pur:
            headers.append('No purchase')
            queries.update({Q(purchase_date=None)})
        if no_ven:
            headers.append('No venture')
            queries.update({Q(venture=None)})
        if no_rol:
            headers.append('No venture role')
            queries.update({Q(venture_role=None)})
        if no_par:
            headers.append('No parent')
            p_deleted = parent.deleted
            queries.update({Q(parent=None) | Q(p_deleted=True)})
        if len(queries) > 1:
            devices = Device.objects.filter(*queries)
            for dev in devices:
                row = []
                row.append(self.get_name(dev.name, dev.id))
                if dep:
                    row.append(dev.deprecation_date)
                if no_dep:
                    row.append(dev.deprecation_date)
                if no_mar:
                    row.append(dev.deprecation_kind)
                if no_sup:
                    row.append(dev.support_expiration_date)
                if no_pur:
                    row.append(dev.purchase_date)
                if no_ven:
                    row.append(dev.venture)
                if no_rol:
                    row.append(dev.venture_role)
                rows.append(row)
        # Filtering of th range
        # Support Range
        s_start = self.request.GET.get('s_start', None)
        s_end = self.request.GET.get('s_end', None)
        if s_start and s_end:
            self.form_support_range = SupportRangeReportForm(request)
            devices = Device.objects.all()
            devs = devices.filter(
                support_expiration_date__gte=s_start,
                support_expiration_date__lte=s_end,
            )
            headers = ('Name', 'Support expiration date')
            for dev in devs:
                name = self.get_name(dev.name, dev.id)
                rows.append([name, dev.support_expiration_date])
        else:
            self.form_support_range = SupportRangeReportForm(initial={
                's_start': datetime.date.today() - datetime.timedelta(days=30),
                's_end': datetime.date.today(),
                })
        # Deprecation Range
        d_start = self.request.GET.get('d_start', None)
        d_end = self.request.GET.get('d_end', None)
        if d_start and d_end:
            self.form_deprecation_range = DeprecationRangeReportForm(request)
            devices = Device.objects.all()
            devs = devices.filter(
                deprecation_date__gte=d_start,
                deprecation_date__lte=d_end,
            )
            headers = ('Name', 'Depreciation date')
            for dev in devs:
                name = self.get_name(dev.name, dev.id)
                rows.append([name, dev.deprecation_date])
        else:
            self.form_deprecation_range = DeprecationRangeReportForm(initial={
                'd_start': datetime.date.today() - datetime.timedelta(days=30),
                'd_end': datetime.date.today(),
                })
        # warranty_expiration_date Range
        w_start = self.request.GET.get('w_start', None)
        w_end = self.request.GET.get('w_end', None)
        if w_start and w_end:
            self.form_warranty_range = WarrantyRangeReportForm(request)
            devices = Device.objects.all()
            devs = devices.filter(
                warranty_expiration_date__gte=w_start,
                warranty_expiration_date__lte=w_end,
            )
            headers = ('Name', 'Warranty expiration date')
            for dev in devs:
                name = self.get_name(dev.name, dev.id)
                rows.append([name, dev.warranty_expiration_date])
        else:
            self.form_warranty_range = WarrantyRangeReportForm(initial={
                'w_start': datetime.date.today() - datetime.timedelta(days=30),
                'w_end': datetime.date.today(),
            })
        # Show devices active or / and deleted
        self.device_list = ReportDeviceListForm(request)
        all_devices = request.get('show_all_devices')
        all_deleted_devices = request.get('show_all_deleted_devices')
        if all_devices or all_deleted_devices:
            show_devices = None
            if all_devices and all_deleted_devices:
                show_devices = Device.admin_objects.all()
                csv_conf = {
                    'title': 'Show all devices (active and deleted)',
                    'name': 'report_all_devices',
                    'url': '?show_all_devices=on&show_all_deleted_devices=on&export=csv',
                    }
            elif all_devices:
                show_devices = Device.objects.all()
                csv_conf = {
                    'title': 'Show all active devices',
                    'name': 'report_all_active_devices',
                    'url': '?show_all_devices=on&export=csv',
                    }
            elif all_deleted_devices:
                show_devices = Device.admin_objects.filter(deleted=True)
                csv_conf = {
                    'title': 'Show all deleted devices',
                    'name': 'report_deleted_devices',
                    'url': '?show_all_deleted_devices=on&export=csv',
                    }
            headers = [
                'Device', 'Model', 'SN', 'Barcode', 'Auto price', 'Venture',
                'Venture ID', 'Role', 'Remarks', 'Verified', 'Deleted'
            ]
            for dev in show_devices:
                rows.append([
                    dev.name,
                    dev.model,
                    dev.sn,
                    dev.barcode,
                    dev.cached_price,
                    dev.venture,
                    dev.role,
                    dev.remarks,
                    dev.verified,
                    dev.deleted,
                ])
        if request.get('export') == 'csv':
            rows.insert(0, headers)
            return self.export_csv(rows, csv_conf.get('name'))
        self.headers = headers
        self.rows = rows
        self.csv_url = csv_conf.get('url')
        self.title = csv_conf.get('title')
        return super(ReportDevices, self).get(*args, **kwargs)

    def get_context_data(self, **kwargs):
        context = super(ReportDevices, self).get_context_data(**kwargs)
        context.update(
            {
                'title': self.title,
                'form_choice': self.form_choice,
                'form_support_range': self.form_support_range,
                'form_deprecation_range': self.form_deprecation_range,
                'form_warranty_range': self.form_warranty_range,
                'form_device_list': self.device_list,
                'csv_url': self.csv_url,
                'tabele_header': self.headers,
                'rows': self.rows,
                'perm_to_edit': self.perm_edit,
            }
        )
        return context

def is_bladesystem(component):
    """ Check if component is bladesystem - be careful BladeSystem and
    RAM have the same type_id. If component type is RAM runs except """
    model = component.get('model')
    if model in component:
        type = model.type
        bladeservers_models = DeviceModelGroup.objects.filter(
            type=type
        )
        group = model.group
        if (bladeservers_models and group in bladeservers_models
            and type == DeviceType.blade_system):
            return True
    return False

class ReportDevicePricesPerVenture(SidebarReports, Base):
    template_name = 'ui/report_venture_costs.html'
    subsection = 'venture_costs'

    def generate_csv(self, data, view_components=True):
        rows = []
        max = 0
        for item in data:
            details = []
            dev = item.get('device')
            price = item.get('price')
            components = item.get('components')
            row = [
                dev.venture.symbol,
                dev.name,
                dev.role,
                dev.sn,
                dev.barcode or '',
                dev.cached_price or 'N/A',
                price or 'N/A',
            ]
            row = [unicode(c) for c in row]
            if view_components:
                max = len(components) if max < len(components) else max
                for detail in components:
                    details.extend([
                        detail.get('name'),
                        detail.get('count'),
                        detail.get('price') or 'N/A',
                        detail.get('total_component') or 'N/A',
                    ])
                    details = [unicode(d) for d in details]
                row.extend(details)
            rows.append(row)
        headers = [
            'Venture', 'Device', 'Role', 'SN', 'Barcode', 'Quoted price (PLN)',
            'Total component (PLN)',
        ]
        if view_components:
            for i in range(max):
                headers.extend([
                    'Component name',
                    'Component count',
                    'Component price (PLN)',
                    'Component total (PLN)',
                ])
        rows.insert(0, headers)
        return rows

    def export_csv(self, data, all_devices=False):
        if all_devices:
            rows = self.generate_csv(data, view_components=False)
            filename = 'report_all_devices_prices-%s.csv' % (
                datetime.date.today()
            )
        else:
            rows = self.generate_csv(data)
            venture = Venture.objects.get(id=self.venture_id)
            filename = 'report_devices_prices_per_venture-%s-%s.csv' % (
                venture.symbol, datetime.date.today()
            )
        f = StringIO.StringIO()
        csvutil.UnicodeWriter(f).writerows(rows)
        response = HttpResponse(f.getvalue(), content_type='application/csv')
        disposition = 'attachment; filename=%s' % filename
        response['Content-Disposition'] = disposition
        return response

    def get_device_with_components(self, venture_devices, blacklist):
        devices = []
        for device in venture_devices:
            all_components_price = 0
            components = []
            for component in _get_details(device, ignore_deprecation=True):
                count = 1
                model = component.get('model')
                if not isinstance(model, basestring):
                    component_type = model.type
                    component_group = model.group_id
                    model_group = model.group
                else:
                    component_group = None
                    component_type = None
                    model_group = None
                act_components = [x.get('name') for x in components]
                if (model not in act_components and
                    component_type not in blacklist):
                    if is_bladesystem(component):
                        bs_count = device.child_set.filter(
                            deleted=False).count() or 1
                        chassis_price = get_device_chassis_price(device)
                        auto_price = get_device_auto_price(device)
                        bs_price = 0
                        if device.price != 0:
                            bs_price = device.price / bs_count
                        elif chassis_price != 0:
                            bs_price = chassis_price
                        elif auto_price != 0:
                            bs_price = auto_price / bs_count
                        components.append({
                            'icon': component.get('icon'),
                            'name': model,
                            'price': bs_price,
                            'count': count,
                            'bs_count': bs_count,
                        })
                    elif component_type == ComponentType.share:
                        components.append({
                            'icon': component.get('icon'),
                            'name': model,
                            'price': component.get('price') or 0,
                            'count': component.get('count') or 1,
                        })
                    else:
                        d_price = 0
                        if device.price and device.price != 0:
                            d_price = device.price
                        else:
                            d_price = component.get('price') or 0
                        components.append({
                            'icon': component.get('icon'),
                            'name': model,
                            'price': d_price,
                            'model_group': model_group,
                            'count': count,
                        })
                else:
                    for component in components:
                        if component.get('name') == model:
                            count = component.get('count')
                            component.update(count=count + 1)
            for component in components:
                count = component.get('count')
                price = component.get('price')
                total_component = price * count
                component['total_component'] = total_component
                all_components_price += total_component
            devices.append({
                'device': device,
                'deprecated': is_deprecated(device),
                'price': all_components_price,
                'components': components
            })
        return devices

    def get(self, *args, **kwargs):
        profile = self.request.user.get_profile()
        has_perm = profile.has_perm
        if not has_perm(Perm.read_device_info_reports):
            return HttpResponseForbidden(
                "You don't have permission to see reports.")
        self.perm_edit = False
        if has_perm(Perm.edit_device_info_financial):
            self.perm_edit = True
        self.form = ReportVentureCost()
        self.venture_id = self.request.GET.get('venture')
        venture_devices = None
        if self.venture_id not in ['', None] and not self.venture_id.isdigit():
            raise Http404
        if self.venture_id:
            venture_devices = Device.objects.filter(venture_id=self.venture_id)
            self.form = ReportVentureCost(initial={'venture': self.venture_id})
        if venture_devices:
            self.devices = self.get_device_with_components(
                venture_devices,
                blacklist=[
                    ComponentType.software,
                    ComponentType.os,
                ]
            )
        else:
            self.devices = None
        if self.request.GET.get('export') == 'csv':
            return self.export_csv(self.devices)
        if self.request.GET.get('export-all') == 'csv':
            devices = Device.objects.all()
            csv = self.get_device_with_components(devices, blacklist=[
                    ComponentType.software,
                    ComponentType.os,
                ])
            return self.export_csv(csv, all_devices=True)
        return super(ReportDevicePricesPerVenture, self).get(*args, **kwargs)

    def get_context_data(self, **kwargs):
        context = super(ReportDevicePricesPerVenture, self).get_context_data(
            **kwargs)

        context.update({
            'form': self.form,
        })
        if self.venture_id:
            context.update({
                'rows': self.devices,
                'venture': self.venture_id,
            })
        return context


# Copyright 2020 DeepMind Technologies Limited.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     https://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Tests for substrate."""

from absl.testing import absltest
from absl.testing import parameterized
import dm_env
import numpy as np

from meltingpot.python import substrate

REWARD_SPEC = dm_env.specs.Array(shape=[], dtype=np.float64, name='REWARD')
ACTION_SPEC = dm_env.specs.DiscreteArray(
    num_values=1, dtype=np.int32, name='action')

class SubstrateTest(parameterized.TestCase):

  @parameterized.parameters(
      [[42], [123], [1337], [12481632]])
  def test_seed_causes_determinism(self, seed):
    config = substrate.get_config('allelopathic_harvest')
    with config.unlocked():
      config.env_seed = seed

    env = substrate.build(config)
    prev_obs = env.reset().observation[0]
    obs = []
    for _ in range(10):
      env = substrate.build(config)
      obs.append(env.reset().observation[0])

    np.testing.assert_equal(obs, [prev_obs] * 10)

  @parameterized.named_parameters(
      (name, name) for name in substrate.AVAILABLE_SUBSTRATES)
  def test_substrate_creation(self, substrate_name):
    config = substrate.get_config(substrate_name)
    with substrate.build(config) as env:
      reset_timestep = env.reset()
      action_spec = env.action_spec()
      observation_spec = env.observation_spec()
      reward_spec = env.reward_spec()

    with self.subTest('reset_reward'):
      self.assertNotEqual(reset_timestep.reward, None)
    with self.subTest('reset_discount'):
      self.assertNotEqual(reset_timestep.discount, None)
    with self.subTest('observation_spec'):
      self.assertLen(observation_spec, config.num_players)
    with self.subTest('action_spec'):
      spec = ACTION_SPEC.replace(num_values=len(config.action_set))
      self.assertEqual(action_spec, (spec,) * config.num_players)
    with self.subTest('reward_spec'):
      self.assertEqual(reward_spec, [REWARD_SPEC] * config.num_players)

if __name__ == '__main__':
  absltest.main()


import sys
from setuptools import setup
from gap_statistic import __version__

try:
    from setuptools_rust import RustExtension, Binding
except ImportError:
    import subprocess

    errno = subprocess.call(
        [sys.executable, "-m", "pip", "install", "setuptools-rust>=0.9.2"]
    )
    if errno:
        print("Please install the 'setuptools-rust>=0.9.2' package")
        raise SystemExit(errno)
    else:
        from setuptools_rust import RustExtension, Binding

install_requires = ["numpy", "pandas", "scipy"]

setup_requires = [
    "setuptools-rust>=0.9.2",
    "pytest-runner",
    "pytest",
    "scikit-learn",
    "joblib",
    "scipy",
    "pandas",
]
tests_require = ["scikit-learn", "pytest", "joblib", "black", "click"]

setup(
    name="gap-stat",
    version=__version__,
    author="Miles Granger",
    maintainer="Miles Granger",
    author_email="miles59923@gmail.com",
    maintainer_email="miles59923@gmail.com",
    keywords="kmeans unsupervised learning machine-learning clustering",
    description="Python implementation of the gap statistic with Rust optimizations.",
    long_description="Uses the gap statistic method by Tibshirani, Walther, Hastie to suggest n_clusters.",
    packages=["gap_statistic"],
        license="MIT",
    url="https://github.com/milesgranger/gap_statistic",
    zip_safe=False,
    setup_requires=setup_requires,
    install_requires=install_requires,
    tests_require=tests_require,
    test_suite="tests",
    include_package_data=True,
    classifiers=[
        "Development Status :: 5 - Production/Stable",
        "Intended Audience :: Developers",
        "Intended Audience :: Financial and Insurance Industry",
        "Intended Audience :: Information Technology",
        "Intended Audience :: Science/Research",
        "Programming Language :: Python",
        "Programming Language :: Python :: 3.5",
        "Programming Language :: Python :: 3.6",
        "Programming Language :: Python :: 3 :: Only",
        "Programming Language :: Rust",
        "Operating System :: Microsoft :: Windows",
        "Operating System :: POSIX",
        "Operating System :: Unix",
        "Operating System :: MacOS :: MacOS X",
    ],
)


import time

def partition(arr, low, high):
    i = low
    pivot = arr[low]

    for j in range(low, high+1):
        if arr[j] < pivot:
            # Increment i
            i += 1
            # Swap i and j
            arr[i], arr[j] = arr[j], arr[i]

    # Swap the i index value with the pivot
    arr[i], arr[low] = arr[low], arr[i]

    # Return partition split index
    return i
def partition2(arr, low, high):
    i = ( low-1 )         # index of smaller element
    pivot = arr[high]     # pivot

    for j in range(low , high):

        # If current element is smaller than the pivot
        if   arr[j] < pivot:

            # increment index of smaller element
            i = i+1
            arr[i],arr[j] = arr[j],arr[i]

    arr[i+1],arr[high] = arr[high],arr[i+1]
    return ( i+1 )

def quickSort(arr, low, high):
    try:
        if low < high:
            partitionSplitIndex = partition2(arr, low, high)

            quickSort(arr, low, partitionSplitIndex-1)
            quickSort(arr, partitionSplitIndex+1, high)
    except RecursionError as _rec_:
        print("RECURSION ERROR")

    return arr

RANGE_STOP = eval(input("RANGE > "))
n = list(range(1, RANGE_STOP+1))[::-1]

print("Array before sort : {0}".format(n))
start = time.perf_counter()
quickSort(n, 0, len(n)-1)
print("Time needed : {0}".format(time.perf_counter() - start))
print("Array after sort : {0}".format(n))


from core.TaxonClass import TaxonClass
from core.Ref import Ref
from Wpp.WppDictionary import WppDictionary

class WppClass(TaxonClass, WppDictionary):

	def readHead(self, context):
		self._location = context.createLocation()
		words = context.currentLine.split()
		if len(words) < 2:
			self.throwError('Required class name')
		self.name = words[-1]
		self.attrs = set(words[1:-1])
		# check attributes
		self.checkAttributes()
		self.checkAttributes(['abstract', 'static'])
		if 'protected' in self.attrs:
			context.throwError('Access level "protected" cannot be applied to a class')

	def readBody(self, context):
		from Wpp.WppVar import WppField, WppReadonly
		from Wpp.WppFunc import WppMethod, WppConstructor, WppOperator
		from Wpp.WppTypedef import WppTypedef
		# from Wpp.WppCast import WppCast
		word = context.getFirstWord()
		line = context.currentLine
		if word == 'extends':
			chunks = line.split()
			if len(chunks) != 2:
				context.throwError('Expected single name of class after "extends"')
			self.parent = Ref(chunks[1])
			return None
		if word == 'implements':
			chunks = line.split()
			self.implements = [Ref(name) for name in chunks[1:]]
			return None
		if word == 'field':
			return WppField()
		if word == 'readonly':
			return WppReadonly()
		if word == WppMethod.keyWord:
			return WppMethod()
		if word == WppConstructor.keyWord:
			return WppConstructor()
		if word == WppOperator.keyWord:
			return WppOperator(True)
		if word == 'typedef':
			return WppTypedef()
		# if word == 'cast':
		# 	return WppCast()
		return super().readBody(context)

	def addTaxon(self, taxon):
		if 'static' in self.attrs and taxon.canBeStatic:
			# Если класс статическмй, то элементы автоматически получают атрибут static
			taxon.attrs.add('static')
		if hasattr(taxon, 'classMember'):
			taxon.addFuncToOwner(self)
			return taxon
		return super().addTaxon(taxon)

	def onUpdate(self):
		result = super().onUpdate()
		# parent must be a Class
		parent = self.getParent()
		if parent and parent.type != 'Class':
			self.throwError('Invalid parent %s:%s' % (parent.getPath(), parent.type))
		# implements - interfaces only!
		for i in self.implements:
			if i.target.type != 'Interface':
				self.throwError('Invalid interface %s:%s' % (i.target.getPath(), i.target.type))
		return result

	def export(self, outContext):
		# head of class
		chunks = ['class'] + self.getExportAttrs() + [self.name]
		outContext.writeln(' '.join(chunks))
		outContext.push()
		# comment
		self.exportComment(outContext)
		# extends
		parent = self.getParent()
		if parent:
			outContext.writeln('extends '+parent.getName(self))
		# implements
		if self.implements:
			outContext.writeln('implements ' + ' '.join([i.name for i in self.implements]))
		for item in self.items:
			item.export(outContext)
		outContext.pop()

	def validate(self, valueTaxon, attrs):
		""" Можно ли переменной данного класса присвоить указанное значение """
		if valueTaxon.type == 'Null':
			# Если класс объявлен с атрибутом simple, то null можно присваивать только указателю
			if 'simple' in self.attrs and 'ptr' not in attrs:
				return 'Cannot use null for simple class "%s" without pointer' % self.getName(self)
			return ''
		if valueTaxon.type == 'New' or valueTaxon.type == 'Call':
			sourceClass = valueTaxon.getCaller().getQuasiType()
			if sourceClass.canUpcastTo(self):
				return ''
		if valueTaxon.type == 'IdExpr':
			sourceClass = valueTaxon.getQuasiType()
			if sourceClass.canUpcastTo(self):
				return ''

		# self.throwError('Invalid value for class %s = %s:%s' % (self.name, valueTaxon.type, valueTaxon.exportString()))
		return 'NotFound'


#!/usr/bin/python
# Program:
# 	This program will split the original lttng log file into pure-log and
#   summary-log two parts.  It should be done before parser.py program.
# History:
# 	2011/9/8  Hsiao-Mei Lin

def ltt_split(filename, PATH):
# create purelog and summery file
	origFile = open(filename)
	pureFile = open(PATH+'/_pure.log', 'w+')
#	smryFile = open(PATH+'/_smry.log', 'w+')

# start to process
	i = 1
	count = 0
	origFile.readline()
	origFile.readline()

	while 1:
		line = origFile.readline()
		if not line: break
		if 'End trace set\n' == line:
			break
		else:
			pureFile.write(line)

#	while 1:
#		line = origFile.readline()
#		if not line: break
#		smryFile.write(line)

	origFile.close()
	pureFile.close()
#	smryFile.close()

if __name__=='__main__':
	import sys
	ltt_split(sys.argv[1], sys.argv[2])