#ifdef CMAKE_BUILD
#include "caffe_config.h"
#endif
#ifdef USE_INTEL_SPATIAL
#include <sstream>
#include <string>
#include <vector>
#include "caffe/filler.hpp"
#include "caffe/layer.hpp"
#include "caffe/layers/conv_spatial_layer.hpp"
#include "caffe/util/benchmark.hpp"
#include "caffe/util/im2col.hpp"
#include "caffe/util/math_functions.hpp"

#ifdef USE_GREENTEA
#include "caffe/greentea/cl_kernels.hpp"
#include "caffe/greentea/greentea.hpp"
#include "caffe/greentea/greentea_im2col.hpp"
#include "caffe/greentea/greentea_math_functions.hpp"
#include "viennacl/tools/sha1.hpp"
#endif

#include <boost/filesystem.hpp>

// #define TEST_ALL_KERNELS
namespace caffe {

#define ALIGN(val, N) (((val) + (N) - 1) & ~((N) - 1))

template<typename Dtype>
void ConvolutionLayerSpatial<Dtype>::compute_output_shape() {
  const int_tp* kernel_shape_data = this->kernel_shape_.cpu_data();
  const int_tp* stride_data = this->stride_.cpu_data();
  const int_tp* pad_data = this->pad_.cpu_data();
  const int_tp* dilation_data = this->dilation_.cpu_data();
  this->output_shape_.clear();
  for (int_tp i = 0; i < this->num_spatial_axes_; ++i) {
    // i + 1 to skip channel axis
    const int_tp input_dim = this->input_shape(i + 1);
    const int_tp kernel_extent = dilation_data[i] * (kernel_shape_data[i] - 1)
        + 1;
    const int_tp output_dim = (input_dim + 2 * pad_data[i]
        - kernel_extent) / stride_data[i] + 1;
    this->output_shape_.push_back(output_dim);
  }
}

template<typename Dtype>
void ConvolutionLayerSpatial<Dtype>::LayerSetUp(
    const vector<Blob<Dtype>*>& bottom, const vector<Blob<Dtype>*>& top) {
  BaseConvolutionLayer<Dtype>::LayerSetUp(bottom, top);
  tuned_ = 0;
  // Calculate variables used for kernel generation
  const int_tp* kernel_shape_data = this->kernel_shape_.cpu_data();
  kernel_h_ = kernel_shape_data[0];
  kernel_w_ = kernel_shape_data[1];
  const int_tp* pad_data = this->pad_.cpu_data();
  pad_h_ = pad_data[0];
  pad_w_ = pad_data[1];
  const int_tp* stride_data = this->stride_.cpu_data();
  stride_h_ = stride_data[0];
  stride_w_ = stride_data[1];
  const int_tp* dilation_data = this->dilation_.cpu_data();
  dilation_h_ = dilation_data[0];
  dilation_w_ = dilation_data[1];
  M_ = this->num_output_ / this->group_;
  K_ = this->channels_ * kernel_h_ * kernel_w_ / this->group_;
  swizzled_weights_blob_.Reshape((this->num_output_ + 15) & ~15,
                            this->channels_,
                            kernel_h_, (kernel_w_ + 1) & ~1);
  swizzled_weights_ = NULL;
  bias_ = NULL;
  if (std::getenv("CLCAFFE_CACHE_PATH"))
    cache_path_ << std::getenv("CLCAFFE_CACHE_PATH");
  else if (std::getenv("VIENNACL_CACHE_PATH"))
    cache_path_ << std::getenv("VIENNACL_CACHE_PATH") << "/clCaffe";
  else if (std::getenv("HOME")) {
    cache_path_ << std::getenv("HOME") << "/.cache/clCaffe";
  }
  cache_path_ << "/spatialkernels/";
  const boost::filesystem::path& path = cache_path_.str();
  const boost::filesystem::path& dir =
                 boost::filesystem::unique_path(path).string();
  bool hasCacheDir = false;
  if (!boost::filesystem::exists(dir))
    hasCacheDir = boost::filesystem::create_directories(dir);
  else
    hasCacheDir = boost::filesystem::is_directory(dir);

  if (hasCacheDir != true) {
    std::cout << "Failed to create cache directory,"
              << "will tune again for next running" << std::endl;
    return;
  }
}

template<typename Dtype>
void ConvolutionLayerSpatial<Dtype>::Reshape(const vector<Blob<Dtype>*>& bottom,
                                             const vector<Blob<Dtype>*>& top) {
  BaseConvolutionLayer<Dtype>::Reshape(bottom, top);
  height_ = bottom[0]->shape(this->channel_axis_ + 1);
  width_ = bottom[0]->shape(this->channel_axis_ + 2);
  const int_tp kernel_extent_h = dilation_h_ * (kernel_h_ - 1) + 1;
  const int_tp kernel_extent_w = dilation_w_ * (kernel_w_ - 1) + 1;
  output_h_ = (height_ + 2 * pad_h_ - kernel_extent_h) / stride_h_ + 1;
  output_w_ = (width_ + 2 * pad_w_ - kernel_extent_w) / stride_w_ + 1;

  // Shape the tops.
  vector<int_tp> top_shape(bottom[0]->shape().begin(),
                           bottom[0]->shape().begin() + this->channel_axis_);
  top_shape.push_back(this->num_output_);
  for (int_tp i = 0; i < this->num_spatial_axes_; ++i) {
    top_shape.push_back(this->output_shape_[i]);
  }

  for (int_tp top_id = 0; top_id < top.size(); ++top_id) {
    top[top_id]->Reshape(top_shape);
  }

  CHECK_EQ(2, this->num_spatial_axes_)
    << "ConvolutionSpatial input must have 2 spatial axes "
    << "(e.g., height and width). ";

  const int_tp height_out = top[0]->shape(this->channel_axis_ + 1);
  const int_tp width_out = top[0]->shape(this->channel_axis_ + 2);
  N_ = height_out * width_out;
  // Set up the all ones "bias multiplier" for adding biases by BLAS
  if (this->bias_term_) {
    bias_multiplier_.Reshape(1, 1, 1, N_);
    caffe_set(N_, Dtype(1), bias_multiplier_.mutable_cpu_data());
  }

  if (std::is_same<Dtype, float>::value) {
    this->num_ = bottom[0]->count(0, this->channel_axis_);
    SetUp(bottom, top, Caffe::GetDefaultDevice()->backend());
  }
}

template<typename Dtype>
void ConvolutionLayerSpatial<Dtype>::Forward_cpu(
    const vector<Blob<Dtype>*>& bottom, const vector<Blob<Dtype>*>& top) {
  const Dtype* weight = this->blobs_[0]->cpu_data();
  for (int_tp i = 0; i < bottom.size(); ++i) {
    const Dtype* bottom_data = bottom[i]->cpu_data();
    Dtype* top_data = top[i]->mutable_cpu_data();
    for (int_tp n = 0; n < this->num_; ++n) {
      this->forward_cpu_gemm(bottom_data + n * this->bottom_dim_, weight,
                             top_data + n * this->top_dim_);
      if (this->bias_term_) {
        const Dtype* bias = this->blobs_[1]->cpu_data();
        this->forward_cpu_bias(top_data + n * this->top_dim_, bias);
      }
    }
  }
}

template<typename Dtype>
void ConvolutionLayerSpatial<Dtype>::Backward_cpu(
    const vector<Blob<Dtype>*>& top, const vector<bool>& propagate_down,
    const vector<Blob<Dtype>*>& bottom) {
  const Dtype* weight = this->blobs_[0]->cpu_data();
  Dtype* weight_diff = this->blobs_[0]->mutable_cpu_diff();
  for (int_tp i = 0; i < top.size(); ++i) {
    const Dtype* top_diff = top[i]->cpu_diff();
    const Dtype* bottom_data = bottom[i]->cpu_data();
    Dtype* bottom_diff = bottom[i]->mutable_cpu_diff();
    // Bias gradient, if necessary.
    if (this->bias_term_ && this->param_propagate_down_[1]) {
      Dtype* bias_diff = this->blobs_[1]->mutable_cpu_diff();
      for (int_tp n = 0; n < this->num_; ++n) {
        this->backward_cpu_bias(bias_diff, top_diff + n * this->top_dim_);
      }
    }
    if (this->param_propagate_down_[0] || propagate_down[i]) {
      for (int_tp n = 0; n < this->num_; ++n) {
        // gradient w.r.t. weight. Note that we will accumulate diffs.
        if (this->param_propagate_down_[0]) {
          this->weight_cpu_gemm(bottom_data + n * this->bottom_dim_,
                                top_diff + n * this->top_dim_, weight_diff);
        }
        // gradient w.r.t. bottom data, if necessary.
        if (propagate_down[i]) {
          this->backward_cpu_gemm(top_diff + n * this->top_dim_, weight,
                                  bottom_diff + n * this->bottom_dim_);
        }
      }
    }
  }
}

#ifndef CPU_ONLY
#ifdef USE_GREENTEA

//  #define dbg
#ifdef dbg
#define dbgPrint(x) (x)
#else
#define dbgPrint(x)
#endif

// For large enough input size, we do not need to tune kernels for different
// size. The reason is with large input size, there will be enough work items
// to feed al the EUs.
// FIXME for the gemm like convolution, switch back to eaxct image size.

#define TUNING_SIZE(x) ((x) > 256 ? 256 : (ALIGN(x, 16)))

template<>
void ConvolutionLayerSpatial<float>::generate_key() {
  std::stringstream keyBuilder;
  keyBuilder << kernel_w_ << "_"
             << kernel_h_ << "_"
             << channels_ << "_"
             << group_ << "_"
             << stride_h_ << "_"
             << stride_w_ << "_"
             << dilation_h_ << "_"
             << dilation_w_ << "_"
             << bias_term_ << "_"
             << TUNING_SIZE(width_) << "_"
             << TUNING_SIZE(height_) << "_"
             << pad_w_ << "_"
             << pad_h_ << "_"
             << num_ << "_"
             << M_;

  viennacl::ocl::context &ctx = viennacl::ocl::get_context
                                (this->device_->id());
  std::string prefix = ctx.current_device().name()
                  + ctx.current_device().vendor()
                  + ctx.current_device().driver_version()
                  + std::to_string(ctx.current_device().max_compute_units());
  key_ = viennacl::tools::sha1(prefix + keyBuilder.str());
  short_key_ = keyBuilder.str();
}

template<>
std::string ConvolutionLayerSpatial<float>::generate_specific_key(
    int_tp type, int_tp blockWidth, int_tp blockHeight, int_tp blockDepth) {
  std::stringstream keyBuilder;
  keyBuilder << short_key_
             << "_" << type
             << "_" << blockWidth
             << "_" << blockHeight
             << "_" << blockDepth;
  return keyBuilder.str();
}

template<typename Dtype>
void interleaveMatrix(
         Dtype* mem_dst, const Dtype *mem,
         int r, int c, int interleavedRows, int nonInterleavedRows,
         int blockWidth, int rowAlignment ) {
  CHECK_EQ(interleavedRows % 2, 0) <<
      "interleaveMatrix only supports even values for interleavedRows.";

  size_t memSize = r * c * sizeof(float);
  size_t dstSize = memSize *
            (interleavedRows + nonInterleavedRows * 2) /
            (interleavedRows + nonInterleavedRows);
  memset(mem_dst, 0, dstSize);    // NOLINT

  const int xStride = blockWidth;
  const int yStride = c * 2;
  const Dtype *pSrc = mem;
  Dtype* pDst = mem_dst;
  for (int y = 0; y < r;) {
    for (int rows = 0; rows < interleavedRows; rows += 2) {
      if ( y >= r ) break;
      if ((c % xStride) == 0) {
        for (int x = 0; x < c / xStride; x++) {
          memcpy( pDst + x * xStride * 2,                         // NOLINT
                  pSrc + x * xStride,     xStride * sizeof(Dtype));
          memcpy( pDst + x * xStride * 2 + xStride,               // NOLINT
                  pSrc + x * xStride + c, xStride * sizeof(Dtype));
        }
      } else {
        const int count = c / xStride;
        int x = 0;
        for (; x < count - 1; x++) {
          memcpy(pDst + x * xStride * 2,                          // NOLINT
                 pSrc + x * xStride, xStride * sizeof(Dtype));
          memcpy(pDst + x * xStride * 2 + xStride,                // NOLINT
                 pSrc + x * xStride + c, xStride * sizeof(Dtype));
        }
        memcpy(pDst + x * xStride * 2,                            // NOLINT
               pSrc + x * xStride, xStride * sizeof(Dtype));
      }
      pSrc += yStride;
      pDst += yStride;
      y += 2;
    }

    for (int rows = 0; rows < nonInterleavedRows; rows++) {
      if (y >= r) break;
      const int stride = rowAlignment;
      int remaining = c;
      for (int x = 0; x < c; x += stride) {
        if (remaining >= stride) {
          memcpy( pDst + x * 2, pSrc + x, stride * sizeof(Dtype));    // NOLINT
          remaining -=stride;
        } else {
          memcpy(pDst + x * 2, pSrc + x, remaining * sizeof(Dtype));  // NOLINT
        }
      }
      pSrc += yStride / 2;
      pDst += yStride;
      y++;
    }
  }
}

template<typename Dtype>
void ConvolutionLayerSpatial<Dtype>::swizzleWeights(
    const vector<Blob<Dtype>*>& bottom,
    const vector<Blob<Dtype>*>& top,
    int_tp swizzled_factor,
    bool interleave) {

  // Simply skip the weight swizzle if we already got a swizzled_weights_
  // in test phase and not in auto tuning
  // This requires we always call convolve again with the winner configuration
  // during the auto tuning stage.
  if (tuned_ &&
      swizzled_weights_ != NULL &&
      this->phase_ == TEST)
    return;

  swizzled_weights_ = swizzled_weights_blob_.mutable_gpu_data();

  if (!interleave) {
    viennacl::ocl::context &ctx = viennacl::ocl::get_context(
        this->device_->id());
    viennacl::ocl::program &program = this->device_->program();
    viennacl::ocl::kernel &oclk_copy_weight = program.get_kernel(
        CL_KERNEL_SELECT("copyWeightsSwizzled"));
    cl_uint argIdx = 0;

    int_tp channels = this->channels_ / this->group_;
    oclk_copy_weight.arg(argIdx++, WrapHandle((cl_mem) weight, &ctx));
    oclk_copy_weight.arg(argIdx++, WrapHandle((cl_mem) swizzled_weights_,
                         &ctx));
    oclk_copy_weight.arg(argIdx++, kernel_w_);
    oclk_copy_weight.arg(argIdx++, kernel_h_);
    oclk_copy_weight.arg(argIdx++, channels);
    oclk_copy_weight.arg(argIdx++, this->num_output_);
    oclk_copy_weight.arg(argIdx++, swizzled_factor);
    const size_t global_work_size_Copy[3] = {
        (size_t) (ALIGN(this->num_output_, swizzled_factor)
        * channels * kernel_w_ * kernel_h_), 1, 1 };

    OCL_CHECK(clEnqueueNDRangeKernel(ctx.get_queue().handle().get(),
                                     oclk_copy_weight.handle().get(), 3, NULL,
                                     global_work_size_Copy, NULL, 0, NULL,
                                     NULL));
  } else {
    Dtype *cpu_swizzled_weight = swizzled_weights_blob_.mutable_cpu_data();
    int interleavedRows = (kernel_w_ / 2) * 2;
    int nonInterleavedRows = kernel_w_ % 2;
    int blockWidth = swizzled_factor;  // should equal to SIMD size.
    int rowAlignment = 32;
    size_t interleaved_filter_size = M_ * kernel_w_ * kernel_h_ *
                                     this->channels_ * sizeof(Dtype);
    Dtype * tmpSwizzledWeight = static_cast<Dtype*>(
                                  malloc(interleaved_filter_size));
    CHECK_EQ(tmpSwizzledWeight != NULL, true)
      << "Failed to allocate temporary swizzled weight";
    for ( int od = 0; od < M_; od++)
      for ( int id = 0; id < this->channels_; id++)
        for ( int r = 0; r < kernel_h_; r++)
          for ( int c = 0; c < kernel_w_; c++)
            tmpSwizzledWeight[((id * kernel_h_ + r)
                * kernel_w_ + c) * M_ + od]
                = weight_cpu[((od * this->channels_ + id)
                * kernel_h_ + r) * kernel_w_ + c ];
    interleaveMatrix(cpu_swizzled_weight, tmpSwizzledWeight,
              kernel_w_ * kernel_h_ * this->channels_, M_,
              interleavedRows, nonInterleavedRows, blockWidth, rowAlignment);
    free(tmpSwizzledWeight);
  }
}

template<>
void ConvolutionLayerSpatial<float>::calculate_global_size(int_tp batch,
                                  int_tp* wio,  // work item output size
                                  size_t* lSize,  // local size
                                  size_t* gSize) {  // global size
  gSize[0] = ceil(
      (fmax(static_cast<float>(output_w_) / wio[0], 1.0)) / lSize[0])
      * lSize[0];
  gSize[1] = ceil(
      (fmax(static_cast<float>(output_h_) / wio[1], 1.0)) / lSize[1])
      * lSize[1];
  gSize[2] = ceil(
      static_cast<float>((ceil(static_cast<float>(M_) * batch / wio[2])))
          / lSize[2]) * lSize[2];
}

template<>
bool ConvolutionLayerSpatial<float>::create_basic_kernel(
    const vector<Blob<float>*>& bottom, const vector<Blob<float>*>& top,
    int_tp blockWidth,
    int_tp blockHeight, int_tp blockDepth) {
  // Standard spatial setup is done here
  std::stringstream keyBuilder;
  std::stringstream multFunctionBuilder;
  std::string stringBuilder;
  std::stringstream optionsString;
  std::string kernelDef = "MULTI";
  std::string kernelUKey = generate_specific_key(4, blockWidth, blockHeight,
                                                 blockDepth);
  int_tp workItemOutput[3];
  workItemOutput[0] = 1;
  workItemOutput[1] = 1;
  workItemOutput[2] = 1;

  kernel_name_ = "U";
  kernel_name_ += kernelUKey.c_str();
  kernel_name_ += "_BASIC";

  // Build list of options and defines
  optionsString.str("");
  optionsString << "-cl-fast-relaxed-math " << " -D KERNELSIZE="
                << kernel_w_ * kernel_h_ << " -D KERNEL_W=" << kernel_w_
                << " -D KERNEL_H=" << kernel_h_ << " -D CHANNELS="
                << channels_ / group_ << " -D STRIDE_H=" << stride_h_
                << " -DDILATION_X=" << dilation_w_
                << " -DDILATION_Y=" << dilation_h_
                << " -D STRIDE_W=" << stride_w_ << " -D APPLY_BIAS="
                << bias_term_ << " -D OUTPUT_Z=" << M_
                << " -D XPAR=" << workItemOutput[0] << " -D YPAR="
                << workItemOutput[1] << " -D ZPAR=" << workItemOutput[2]
                << " -D " << kernelDef.c_str() << " -D CFMultiNoPadding="
                << kernel_name_;

  viennacl::ocl::context &ctx = viennacl::ocl::get_context(this->device_->id());
  if (IsBeignet(&ctx))
    optionsString << " -D__BEIGNET__";
  string options = optionsString.str();
  try {
    submit_conv_spatial_program(&ctx, kernel_name_, options);
  } catch (std::exception& e) {
    dbgPrint(std::cout << "Basic kernel generation failed" << std::endl);
    return false;
  }

  size_t localSize[3] = { 1, 1, 1 };
  size_t globalSize[3];
  calculate_global_size(1, workItemOutput, localSize, globalSize);

  kernelQueue.push_back(
      new kernelConfig(kernel_name_, globalSize, localSize, workItemOutput,
                       false, false, true, 4));

  return true;
}

template<typename Dtype>
void ConvolutionLayerSpatial<Dtype>::setBufferKernelArg(
    const vector<Blob<Dtype>*>& bottom, const vector<Blob<Dtype>*>& top,
    viennacl::ocl::kernel *kernel,
    const cl_uint &argIdx,
    viennacl::ocl::context *ctx,
    cl_mem buffer, size_t offset,
    size_t size, bool readOnly,
    bool preserved) {

  if (offset == 0) {
    kernel->arg(argIdx, WrapHandle((cl_mem) buffer, ctx));
    return;
  }

  if (preserved &&
    subBufferMap.find(std::make_tuple(buffer, offset, size))
      != subBufferMap.end()) {
    kernel->arg(argIdx,
      WrapHandle(subBufferMap.find
                   (std::make_tuple(buffer, offset, size))->second, ctx));
    return;
  }
  cl_buffer_region region;
  region.origin = offset * sizeof(Dtype);
  region.size = size * sizeof(Dtype);
  cl_mem_flags memFlags = readOnly ? CL_MEM_READ_ONLY : CL_MEM_READ_WRITE;
  cl_int error;
  cl_mem sub_buffer = clCreateSubBuffer(buffer, memFlags,
                        CL_BUFFER_CREATE_TYPE_REGION,
                        &region, &error);
  if (error != CL_SUCCESS) {
    dbgPrint(std::cout << "Failed to create sub buffer ("
                        << error << ")." << std::endl);
    throw(error);
  }
  kernel->arg(argIdx, WrapHandle(sub_buffer, ctx));
  if (preserved)
    subBufferMap.insert(std::make_pair(std::make_tuple(buffer, offset, size),
                        sub_buffer));
  else
    tmpSubBuffers.push_back(sub_buffer);
}

template<typename Dtype>
void ConvolutionLayerSpatial<Dtype>::cleanTmpSubBuffers(
    const vector<Blob<Dtype>*>& bottom, const vector<Blob<Dtype>*>& top) {
  for (auto &buffer : tmpSubBuffers)
    clReleaseMemObject(buffer);
  tmpSubBuffers.clear();
}

template<>
cl_int ConvolutionLayerSpatial<float>::convolve(
    const vector<Blob<float>*>& bottom, const vector<Blob<float>*>& top,
    int_tp index,
    int_tp numImages, kernelConfig* config) {
  viennacl::ocl::context &ctx = viennacl::ocl::get_context(this->device_->id());
  viennacl::ocl::program & program = ctx.get_program(config->kernelName);
  viennacl::ocl::kernel &kernel = program.get_kernel(config->kernelName);
  cl_int err = CL_SUCCESS;
  if (config->kernelType == 2) {
    swizzleWeights(bottom, top, config->workItem_output[2], false);
    size_t total_bottom_size = bottom_dim_ * numImages;
    size_t total_kernel_size = kernel_h_ * kernel_w_ * channels_ * M_;
    size_t total_bias_size = M_ * group_;
    size_t total_top_size = top_dim_ * numImages;
    for (int_tp g = 0; g < group_; ++g) {
      bias_offset_ = M_ * g;
      int_tp image_offset = width_ * height_ * (channels_ / group_) * g;
      int_tp output_image_offset = output_w_ * output_h_ * M_ * g;

      int_tp kernel_offset = kernel_h_ * kernel_w_
                             * (channels_ / group_) * M_ * g;
      cl_uint argIdx = 0;
      try {
        setBufferKernelArg(bottom, top, &kernel, argIdx++, &ctx,
                           (cl_mem) bottom_data,
                           image_offset,
                           total_bottom_size - image_offset,
                           true, false);
        setBufferKernelArg(bottom, top, &kernel, argIdx++, &ctx,
                           (cl_mem) swizzled_weights_,
                           kernel_offset,
                           total_kernel_size - kernel_offset,
                           true, true);
        setBufferKernelArg(bottom, top, &kernel, argIdx++, &ctx,
                           (cl_mem) bias_,
                           bias_offset_,
                           total_bias_size - bias_offset_,
                           true, true);
        setBufferKernelArg(bottom, top, &kernel, argIdx++, &ctx,
                           (cl_mem) top_data,
                           output_image_offset,
                           total_top_size - output_image_offset,
                           false, false);
      } catch (int e) {
        err = e;
      }

      if (err == CL_SUCCESS) {
        kernel.arg(argIdx++, (uint16_t)width_);
        kernel.arg(argIdx++, (uint16_t)height_);
        kernel.arg(argIdx++, (uint16_t)output_w_);
        kernel.arg(argIdx++, (uint16_t)output_h_);
        const int_tp output_block_w = config->workItem_output[0];
        const int_tp output_block_h = config->workItem_output[1];
        size_t global_size[3] = { (size_t) (output_w_ + output_block_w - 1)
             / output_block_w, (size_t) (output_h_ + output_block_h - 1)
             / output_block_h, (size_t) config->global_work_size[2]};

        err = clEnqueueNDRangeKernel(ctx.get_queue().handle().get(),
                                     kernel.handle().get(), 3,
                                     NULL,
                                     global_size,
                                     config->local_work_size, 0, NULL,
                                     NULL);
      }
      if (err != CL_SUCCESS)
        break;
    }

    if (group_ > 1) {
      cleanTmpSubBuffers(bottom, top);
    }
    if (err != CL_SUCCESS)
      return err;
  } else if (config->kernelType == 5) {
    swizzleWeights(bottom, top, config->workItem_output[1], true);
    size_t total_bottom_size = bottom_dim_ * numImages;
    size_t total_kernel_size = kernel_h_ * kernel_w_ * channels_ * M_;
    size_t total_bias_size = M_ * group_;
    size_t total_top_size = top_dim_ * numImages;
    for (int_tp g = 0; g < group_; ++g) {
      bias_offset_ = M_ * g;
      int_tp image_offset = width_ * height_ * (channels_ / group_) * g;
      int_tp output_image_offset = output_w_ * output_h_ * M_ * g;

      cl_uint argIdx = 0;
      int_tp kernel_offset = kernel_h_ * kernel_w_
                             * (channels_ / group_) * M_ * g;
      try {
        setBufferKernelArg(bottom, top, &kernel, argIdx++, &ctx,
                           (cl_mem) bottom_data,
                           image_offset,
                           total_bottom_size - image_offset,
                           true, false);
        setBufferKernelArg(bottom, top, &kernel, argIdx++, &ctx,
                           (cl_mem) swizzled_weights_,
                           kernel_offset,
                           total_kernel_size - kernel_offset,
                           true, true);
        setBufferKernelArg(bottom, top, &kernel, argIdx++, &ctx,
                           (cl_mem) bias_,
                           bias_offset_,
                           total_bias_size - bias_offset_,
                           true, true);
        setBufferKernelArg(bottom, top, &kernel, argIdx++, &ctx,
                           (cl_mem) top_data,
                           output_image_offset,
                           total_top_size - output_image_offset,
                           false, false);
      } catch (int e) {
        err = e;
      }

      if (err == CL_SUCCESS) {
        kernel.arg(argIdx++, (uint16_t)width_);
        kernel.arg(argIdx++, (uint16_t)height_);
        kernel.arg(argIdx++, (uint16_t)output_w_);
        kernel.arg(argIdx++, (uint16_t)output_h_);
        viennacl::ocl::context &ctx =
          viennacl::ocl::get_context(this->device_->id());
        int out_pitch_y = output_w_ * output_h_;
        int out_pitch_z = out_pitch_y * M_;
        int aligned_input_size = height_ * width_ * channels_ / group_;
        int slice_pitch = width_ * height_;
        kernel.arg(argIdx++, (uint32_t)out_pitch_y);
        kernel.arg(argIdx++, (uint32_t)out_pitch_z);
        kernel.arg(argIdx++, (uint32_t)aligned_input_size);
        kernel.arg(argIdx++, (uint32_t)slice_pitch);

        int blockM = config->workItem_output[0];
        int blockK = config->workItem_output[1];
        int blockN = config->workItem_output[2];
        int_tp alignedFilterWidth = ALIGN(M_, blockN);
        int_tp alignedExpandHeight = ALIGN(output_w_ * output_h_, blockM);
        int_tp globalWorkSizeDX = blockN;
        int_tp globalWorkSizeDY = blockM;
        size_t sgemm_m = alignedExpandHeight;
        size_t sgemm_n = alignedFilterWidth;
        size_t gx = (size_t) ceil(static_cast<float>(sgemm_n)
                             / static_cast<float>(globalWorkSizeDX));
        size_t gy = (size_t) ceil(static_cast<float>(sgemm_m)
                             / static_cast<float>(globalWorkSizeDY));
        gy = ALIGN(gy, blockK);
        size_t global_size[3] = { gx, gy, config->global_work_size[2] };

        err = clEnqueueNDRangeKernel(ctx.get_queue().handle().get(),
                                     kernel.handle().get(), 3,
                                     NULL,
                                     global_size,
                                     config->local_work_size, 0, NULL,
                                     NULL);
        OCL_CHECK(err);
      }
      if (err != CL_SUCCESS)
        break;
    }

    if (group_ > 1) {
      cleanTmpSubBuffers(bottom, top);
    }
    if (err != CL_SUCCESS)
      return err;
  } else {
    for (int_tp n = 0; n < numImages; ++n) {
      for (int_tp g = 0; g < group_; ++g) {
        bias_offset_ = M_ * g;
        int_tp image_offset = n * this->bottom_dim_
            + width_ * height_ * (channels_ / group_) * g;
        int_tp output_image_offset = n * this->top_dim_
            + output_w_ * output_h_ * M_ * g;

        cl_uint argIdx = 0;
        int_tp kernel_offset = kernel_h_ * kernel_w_
                               * (channels_ / group_) * M_ * g;

        kernel.arg(argIdx++, WrapHandle((cl_mem) bottom_data, &ctx));
        kernel.arg(argIdx++, image_offset);
        kernel.arg(argIdx++, WrapHandle((cl_mem) weight, &ctx));
        kernel.arg(argIdx++, kernel_offset);
        kernel.arg(argIdx++, WrapHandle((cl_mem) bias_, &ctx));
        kernel.arg(argIdx++, bias_offset_);
        kernel.arg(argIdx++, WrapHandle((cl_mem) top_data, &ctx));
        kernel.arg(argIdx++, output_image_offset);
        kernel.arg(argIdx++, (uint16_t)width_);
        kernel.arg(argIdx++, (uint16_t)height_);
        kernel.arg(argIdx++, (uint16_t)output_w_);
        kernel.arg(argIdx++, (uint16_t)output_h_);
        kernel.arg(argIdx++, (uint16_t)pad_w_);
        kernel.arg(argIdx++, (uint16_t)pad_h_);

        int_tp workItemOutput[3] = { 1, 1, 1 };
        size_t localSize[3] = { 1, 1, 1 };
        size_t globalSize[3];
        calculate_global_size(1, workItemOutput, localSize, globalSize);
        if (config->use_null_local) {
          err = clEnqueueNDRangeKernel(ctx.get_queue().handle().get(),
                                       kernel.handle().get(), 3,
                                       NULL,
                                       config->global_work_size, NULL, 0, NULL,
                                       NULL);
        } else {
          err = clEnqueueNDRangeKernel(ctx.get_queue().handle().get(),
                                       kernel.handle().get(), 3,
                                       NULL,
                                       config->global_work_size,
                                       config->local_work_size, 0, NULL,
                                       NULL);
        }

        if (err != CL_SUCCESS)
          return err;
      }
    }
  }

  return err;
}

template<>
float ConvolutionLayerSpatial<float>::timed_convolve(
    const vector<Blob<float>*>& bottom, const vector<Blob<float>*>& top,
    int_tp index,
    int_tp numImages, kernelConfig* config) {
  // warm up.
  bool saved_tuned = tuned_;
  tuned_ = false;
  convolve(bottom, top, index, num_, config);
  Timer timer;
  timer.initted();
  timer.Start();
  cl_int err;
  dbgPrint(std::cout << "Bechmarking kernel: " << config->kernelName
           << std::endl);
  tuned_ = true;
  int loop_cnt = 4;
  for (int i = 0; i < loop_cnt; i++) {
    err = convolve(bottom, top, index, num_, config);
    if (err != CL_SUCCESS)
      break;
  }
  tuned_ = saved_tuned;
  timer.Stop();
  if (err != CL_SUCCESS) {
    config->tested = true;
    config->verified = false;
    dbgPrint(std::cout << "convolution failed with error code "
             << err << std::endl);
    return 1e5;
  }

  float elapsedTime = timer.MilliSeconds() / loop_cnt;
#ifdef dbg
  double out_w = output_w_;
  double out_h = output_h_;
  double out_z = M_;
  double k_w = kernel_w_;
  double k_h = kernel_h_;
  double k_z = channels_;
  double totalFlops = ((k_w*k_h*k_z -1)*2)*(out_w*out_h*out_z)*num_;
  std::cout << "\tEstimated Gflops:" << ((totalFlops/1000)/1000)/1000
  << std::endl;
  std::cout << "\tEstimated GFLOPS/S: " <<
  (((totalFlops/1000)/1000)/1000)*(1000.0/elapsedTime) << std::endl;
#if 0
  std::cout << "Estimated utilization: " <<
  ((((totalFlops/1000)/1000)/1000)*(1000.0/elapsedTime))/880.0
  << std::endl;
#endif
#endif
  return elapsedTime;
}

template<>
bool ConvolutionLayerSpatial<float>::verify_result(
    const vector<Blob<float>*>& bottom, const vector<Blob<float>*>& top,
    int_tp index,
    int_tp numImages, const Blob<float> &verify_blob, kernelConfig* config) {

  uint_tp verificationFail = 0;

  if (config->verified)
    return true;
  else if (config->tested)
    return false;
  greentea_memset(this->device_->id(), top[index]->count(), 0,
                  (cl_mem)top[index]->mutable_gpu_data(), 0);
  config->executionTime = timed_convolve(bottom, top, index, numImages,
                                         config);
  const float *verify_data = verify_blob.cpu_data();
  const float *data = top[index]->cpu_data();

  for (int_tp n = 0; n < numImages; ++n) {
    for (int_tp g = 0; g < group_; ++g) {
      int_tp output_image_offset = n * this->top_dim_
          + output_w_ * output_h_ * M_ * g;
      for (int out_ch = 0; out_ch < M_ && !verificationFail; out_ch++)
        for (int h = 0; h < output_h_ && !verificationFail; h++)
          for (int w = 0; w < output_w_; w++) {
            size_t offset = output_image_offset + out_ch * output_w_ * output_h_
                            + h * output_w_ + w;
            if (fabs(data[offset] - verify_data[offset]) >
                       0.1 * fabs(verify_data[offset]) &&
                !(fabs(verify_data[offset]) < 1.e-3
                  && fabs(data[offset] - verify_data[offset]) < 1.e-4)) {
              dbgPrint(printf("test verification failed @ image %d group %d"
                              "out_ch %d h %d w %d got %G expected %G\n",
                      n, g, out_ch, h, w, data[offset], verify_data[offset]));
              verificationFail = 1;
              break;
            }
          }
      if (verificationFail)
        return false;
    }
  }
  return true;
}

template<>
bool ConvolutionLayerSpatial<float>::create_gemm_like_conv_kernel(
    const vector<Blob<float>*>& bottom, const vector<Blob<float>*>& top,
    int_tp blockM,
    int_tp blockK, int_tp blockN) {
  std::stringstream multFunctionBuilder;
  std::string stringBuilder;
  std::stringstream optionsString;
  std::string kernelUKey = generate_specific_key(5, blockM, blockK,
                                                 blockN);
  int_tp workItemOutput[3] = { blockM, blockK, blockN };

  int_tp simd_size = blockK;
  int_tp num_batches = num_;
  int_tp globalWorkSizeDX = blockN;
  int_tp globalWorkSizeDY = blockM;

  kernel_name_ = "U_GEMM_LIKE_CONV_";
  kernel_name_ += kernelUKey.c_str();
  if (blockK == 8)
    kernel_name_ += "_SIMD8";
  else
    kernel_name_ += "_SIMD16";
  std::stringstream kernelDef;
  kernelDef << "GEMM_LIKE_CONV_" << blockN << "_" << blockM;
  if (blockK == 16)
    kernelDef << "_SIMD16";

  // Build list of options and defines
  optionsString.str("");
  optionsString << "-cl-fast-relaxed-math " << " -D " << kernelDef.str()
                << " -D Conv_Interleaved=" << kernel_name_.c_str();

  optionsString <<
        " -cl-mad-enable" <<
        " -DKERNEL_WIDTH=" << kernel_w_ <<
        " -DKERNEL_HEIGHT=" << kernel_h_ <<
        " -DSTRIDE_X=" << stride_w_ <<
        " -DSTRIDE_Y=" << stride_h_ <<
        " -DDILATION_X=" << dilation_w_ <<
        " -DDILATION_Y=" << dilation_h_ <<
        " -DINPUT_DEPTH=" << channels_ <<
        " -DWIDTH1=" << M_ <<
        " -DOUT_PADDING_LEFT=" << 0 <<
        " -DOUT_PADDING_HEIGHT=" << 0 <<
        " -DOUT_DEPTH=" << M_ <<
        " -DNUM_BATCHES=" << num_ <<
        " -DDY=" << globalWorkSizeDY <<
        " -DDX=" << globalWorkSizeDX <<
        " -DKERNEL_WIDTH_DIV2=" << kernel_w_ / 2 <<
        " -DKERNEL_SLICE_DIV2=" << (kernel_w_ * kernel_h_) / 2 <<
        " -DTILE_N_LAST=" << M_ % 32 <<
        " -DTILE_N_LAST_DIV8=" << (M_ % 32) / 8;

  optionsString << " -DINPUT_PAD_W=" << pad_w_ << " -DINPUT_PAD_H=" << pad_h_;
  size_t gz = num_batches;
  size_t global_size[3] = { 0, 0, gz };

  size_t local_size[3] = { 1, static_cast<size_t>(simd_size), 1 };
  viennacl::ocl::context &ctx = viennacl::ocl::get_context(this->device_->id());
  if (IsBeignet(&ctx))
    optionsString << " -D__BEIGNET__";
  else
    optionsString <<
        " -cl-no-subgroup-ifp ";
  string options = optionsString.str();

  viennacl::ocl::program & program = submit_conv_spatial_program(&ctx,
                                                                 kernel_name_,
                                                                 options);
  size_t workgroupSize_used;
  viennacl::ocl::kernel & kernel = program.get_kernel(kernel_name_);
  cl_int err = clGetKernelWorkGroupInfo(
      kernel.handle().get(), viennacl::ocl::current_device().id(),
      CL_KERNEL_PREFERRED_WORK_GROUP_SIZE_MULTIPLE,
      sizeof(size_t), &workgroupSize_used,
      NULL);

  if (workgroupSize_used != simd_size) {
    ctx.delete_program(kernel_name_);
    return false;
  }

  if (err == CL_SUCCESS || err == true) {
    kernelQueue.push_back(
        new kernelConfig(kernel_name_, global_size, local_size, workItemOutput,
                         false, true, false, 5));
    return true;
  } else {
    ctx.delete_program(kernel_name_);
    return false;
  }
}

template<>
bool ConvolutionLayerSpatial<float>::setup_IDLF(
    const vector<Blob<float>*>& bottom, const vector<Blob<float>*>& top,
    int_tp blockWidth,
    int_tp blockHeight, int_tp simd_size) {
  std::stringstream multFunctionBuilder;
  std::string stringBuilder;
  std::stringstream optionsString;
  const int_tp blockDepth = 1;
  std::string kernelUKey = generate_specific_key(2, blockWidth, blockHeight,
                                                 blockDepth);
  int_tp workItemOutput[3] = { blockWidth, blockHeight, simd_size };
  const int_tp num_output_maps = M_;
  int_tp output_block_width = blockWidth;
  int_tp output_block_height = blockHeight;
  int_tp num_batches = num_;

  kernel_name_ = "U";
  kernel_name_ += kernelUKey.c_str();

  if (simd_size == 16)
    kernel_name_ += "_SIMD16";
  else
    kernel_name_ += "_SIMD8";

  // Build list of options and defines
  optionsString.str("");
  optionsString << "-cl-fast-relaxed-math " << " -D IDLF"
                << " -D convolve_simd="
                << kernel_name_;

  size_t global_size[3] = { 0, 0,
                (size_t) num_batches * ALIGN(num_output_maps, simd_size) };

  size_t local_size[3] = { 1, 1, static_cast<size_t>(simd_size) };
  int tile_x = (((output_block_width - 1) * stride_w_
               + kernel_w_ * dilation_w_) + 3) & ~3;
  int tile_y = (output_block_height -1) * stride_h_ + kernel_h_ * dilation_h_;
  int tile_y_stride = (4 * simd_size) / tile_x;
  int invec_size = (tile_y + tile_y_stride - 1) / tile_y_stride;

  optionsString << " -D SIMD_SIZE=" << simd_size
                << " -D filter_qualifier=__global" << " -D OUT_BLOCK_WIDTH="
                << output_block_width << " -D OUT_BLOCK_HEIGHT="
                << output_block_height
                << " -D INPUT_DEPTH=" << channels_ / group_
                << " -DTOTAL_INPUT_DEPTH_SIZE=" << channels_
                << " -DTOTAL_OUTPUT_DEPTH=" << num_output_
                << " -DINPUT_START_X=" << 0 << " -DINPUT_START_Y=" << 0
                << " -DINPUT_START_Z=" << 0
                << " -DKERNEL_WIDTH=" << kernel_w_
                << " -DKERNEL_HEIGHT=" << kernel_h_
                << " -DNUM_FILTERS=" << M_ << " -DSTRIDEX=" << stride_w_
                << " -DSTRIDEY=" << stride_h_ << " -DDILATION_X=" << dilation_w_
                << " -DDILATION_Y=" << dilation_h_
                << " -DOWPAD=" << 0 << " -DOHPAD="
                << 0 << " -DOUT_BUFF_OFFSET=" << 0
                << " -DTILE_X=" << tile_x
                << " -DTILE_Y=" << tile_y
                << " -DTILE_Y_STRIDE=" << tile_y_stride
                << " -DINVEC_SIZE=" << invec_size
                << " -DALIGNED_NUM_FILTERS=" << ALIGN(M_, simd_size);

  optionsString << " -DINPUT_PAD_W=" << pad_w_ << " -DINPUT_PAD_H=" << pad_h_;

  string options = optionsString.str();
  viennacl::ocl::context &ctx = viennacl::ocl::get_context(this->device_->id());
  if (IsBeignet(&ctx))
    optionsString << " -D__BEIGNET__";
  viennacl::ocl::program & program = submit_conv_spatial_program(&ctx,
                                                                 kernel_name_,
                                                                 options);

  // ClKernel kernel;
  size_t workgroupSize_used;
  viennacl::ocl::kernel & kernel = program.get_kernel(kernel_name_);
  cl_int err = clGetKernelWorkGroupInfo(
      kernel.handle().get(), viennacl::ocl::current_device().id(),
      CL_KERNEL_PREFERRED_WORK_GROUP_SIZE_MULTIPLE,
      sizeof(size_t), &workgroupSize_used,
      NULL);

  if (workgroupSize_used != simd_size) {
    ctx.delete_program(kernel_name_);
    return false;
  }

  if (err == CL_SUCCESS || err == true) {
    kernelQueue.push_back(
        new kernelConfig(kernel_name_, global_size, local_size, workItemOutput,
                         false, true, false, 2));
    return true;
  } else {
    ctx.delete_program(kernel_name_);
    return false;
  }
}

template<>
bool ConvolutionLayerSpatial<float>::tune_local_size(
    const vector<Blob<float>*>& bottom, const vector<Blob<float>*>& top,
    kernelConfig* config) {
  if (config->use_null_local || !config->autoTune)
    return true;

  float fastestTime = 999999990000000000000000000.0f;
  uint_tp multiplier = 4;
  uint_tp localSize[3] = { 1, 1, 1 };

  int_tp skip = 0;
  Timer timer;
  timer.initted();
  bool allFailed = true;
  for (int_tp z = 0; z <= 16; z++) {
    for (int_tp y = 0; y <= 16; y++) {
      for (int_tp x = 1; x <= 16; x++) {
        timer.Start();
        skip = 0;

        if (config->autoTune) {
          config->local_work_size[0] =
              (multiplier * x == 0) ? 1 : multiplier * x;
          config->local_work_size[1] =
              (multiplier * y == 0) ? 1 : multiplier * y;
          config->local_work_size[2] =
              (multiplier * z == 0) ? 1 : multiplier * z;

          calculate_global_size(1, config->workItem_output,
                                config->local_work_size,
                                config->global_work_size);
        }
        if (config->workItem_output[2] *
            config->global_work_size[2] != M_)
          break;

        if (config->swizzle_weights)
          z = 32;

        int_tp err = 0;
        err = convolve(bottom, top, 0, 1, config);

        if (err != CL_SUCCESS)
          skip = 1;

        if (skip) {
          timer.Stop();
          break;
        }
        timer.Stop();
        allFailed = false;
        float elapsedTime = timer.MilliSeconds();

        if (elapsedTime < fastestTime) {
          fastestTime = elapsedTime;
          localSize[0] = config->local_work_size[0];
          localSize[1] = config->local_work_size[1];
          localSize[2] = config->local_work_size[2];
        }
      }
    }
  }
  if (allFailed) {
    // 1,1,1 is never a good local size and no need to test at all.
    dbgPrint(std::cout << "Can't find good local size for "
                       << config->kernelName << std::endl);
    return false;
  }

  dbgPrint(std::cout << "Best local size[" << localSize[0] << "][" <<
      localSize[1] << "]["<< localSize[2] << "]: " << fastestTime <<
      " Kernel_h: " << kernel_h_ << " kernel_w_: " << kernel_w_ <<
      " stride_w: " << stride_w_ << " pad_w_: " << pad_w_ << std::endl);

  if (config->autoTune) {
    for (int_tp li = 0; li < 3; li++)
      config->local_work_size[li] = localSize[li];

    calculate_global_size(1, config->workItem_output, config->local_work_size,
                          config->global_work_size);
  }
  return true;
}

template<>
void ConvolutionLayerSpatial<float>::create_convolution_kernel(
    const vector<Blob<float>*>& bottom, const vector<Blob<float>*>& top,
    int_tp kernelType,
    int_tp blockWidth, int_tp blockHeight,
    int_tp blockDepth) {
  if (kernelType == 2)
    setup_IDLF(bottom, top, blockWidth, blockHeight, blockDepth);
  else if (kernelType == 4)
    create_basic_kernel(bottom, top, blockWidth, blockHeight, blockDepth);
  else if (kernelType == 5)
    create_gemm_like_conv_kernel(
        bottom, top, blockWidth, blockHeight, blockDepth);
  else
    assert(0);
}

template<>
void ConvolutionLayerSpatial<float>::setup_convolution(
    const vector<Blob<float>*>& bottom, const vector<Blob<float>*>& top,
    const Blob<float> &verify_blob) {
  // Initializes unique kernel ID
  kernel_uid_ = 0;
  std::string viennacl_cache_path;

  viennacl::ocl::context &ctx = viennacl::ocl::get_context
                                    (this->device_->id());
  if (std::getenv("VIENNACL_CACHE_PATH")) {
    viennacl_cache_path = std::getenv("VIENNACL_CACHE_PATH");
    // Disable viennacl cache mechanism during tuning phase.
    ctx.cache_path("");
  } else {
    viennacl_cache_path = "";
  }

  if (this->device_->CheckCapability("cl_intel_subgroups")) {
    /* IDLF kernels are using Intel specific extension which make
       them intel only. */
    // Generates static key_
    int max_compute_units = ctx.current_device().max_compute_units();
    int kernelCnt = 0;
    if (this->group_ == 1 && ((M_ % 8 == 0) && (M_ % 32 != 24))) {
      create_convolution_kernel(bottom, top, 5, 1, 8, 32);
      create_convolution_kernel(bottom, top, 5, 2, 8, 32);
      if (kernel_w_ < 4 && M_ % 32 == 0)
        create_convolution_kernel(bottom, top, 5, 1, 16, 32);
    }

    for (int simd_size = 8; simd_size <= 16; simd_size += 8) {
      if (simd_size == 8
          && !((this->group_ == 1 || M_ % 8 == 0)))
        continue;
      if (simd_size == 16
          && !(this->group_ == 1 || M_ % 16 == 0))
        continue;
      int width_max, height_max, block_size_max;
      if (simd_size == 8) {
        width_max = 16;
        height_max = 16;
        block_size_max = 64;
      } else {
        width_max = 14;
        height_max = 14;
        block_size_max = 32;
      }
      for (uint32_t width = width_max; width > 0; width--) {
        int candidate = 0;
        if (width > output_w_)
          continue;
        for (uint32_t height = height_max; height > 0; height--) {
          if (width * height > block_size_max || height > output_h_)
            continue;
          // Only when the work items count is less than the device
          // max work items or the M_ is less than 16, we will tune
          // for simd 8.
          if (simd_size == 8
              && M_ >= 16
              && ((num_ * M_ * output_w_ * output_h_ /
                   static_cast<float>(width * height))
                 >= max_compute_units * 7 * 16))
            continue;
          int tile_x = (kernel_w_ * dilation_w_
                       + (width - 1) * stride_w_ + 3) & ~3;
          int tile_y = kernel_h_ * dilation_h_ + (height - 1) * stride_h_;
          if (tile_x > (4 * simd_size))
            continue;
          int tile_y_stride = (4 * simd_size) / tile_x;

          if ((tile_y + tile_y_stride - 1) / tile_y_stride < 4) {
            create_convolution_kernel(bottom, top, 2, width, height, simd_size);
            candidate++;
          }
          if (candidate >= 4 && height == 2)
            break;
        }
        kernelCnt += candidate;
        if (kernelCnt >= 12 && width == 2)
          break;
      }
    }
  }
  for (int_tp x = 0; x < kernelQueue.size(); x++) {
    if (tune_local_size(bottom, top, kernelQueue[x])) {
      kernelQueue[x]->executionTime = timed_convolve(bottom, top, bottom_index_,
                                                     num_, kernelQueue[x]);
    } else {
      // skip those kernels without a good local size.
      kernelQueue[x]->verified = false;
      kernelQueue[x]->tested = true;
    }
#ifdef TEST_ALL_KERNELS
    if (kernelQueue[x]->tested == false) {
      bool verified = verify_result(bottom, top, bottom_index_, num_,
                                      verify_blob, kernelQueue[x]);
      if (verified == false) {
        dbgPrint(std::cout << "Kernel "
                             << kernelQueue[x]->kernelName
                             << " failed verification" << std::endl);
        dbgPrint(std::cout << "kernelQueue[x]->workItem_output[0]: "
                       << kernelQueue[x]->workItem_output[0] << " "
                       << "kernelQueue[x]->workItem_output[1]: "
                       << kernelQueue[x]->workItem_output[1] << " "
                       << "kernelQueue[x]->workItem_output[2]: "
                       << kernelQueue[x]->workItem_output[2] << " "
                       << "kernelQueue[x]->kernelType: "
                       << kernelQueue[x]->kernelType << " "
                       << "kernelQueue[x]->global_work_size[0]: "
                       << kernelQueue[x]->global_work_size[0] << " "
                       << "kernelQueue[x]->global_work_size[1]: "
                       << kernelQueue[x]->global_work_size[1] << " "
                       << "kernelQueue[x]->global_work_size[2]: "
                       << kernelQueue[x]->global_work_size[2] << " "
                       << "kernelQueue[x]->local_work_size[0]: "
                       << kernelQueue[x]->local_work_size[0] << " "
                       << "kernelQueue[x]->local_work_size[1]: "
                       << kernelQueue[x]->local_work_size[1] << " "
                       << "kernelQueue[x]->local_work_size[2]: "
                       << kernelQueue[x]->local_work_size[2] << " "
                       << kernelQueue[x]->swizzle_weights << " "
                       << kernelQueue[x]->use_null_local << std::endl);
      } else {
        dbgPrint(std::cout << "Kernel "
                           << kernelQueue[x]->kernelName
                           << " pass verification" << std::endl);
      }
    }
#endif
  }
  int_tp failures = 0;
  bool verification = false;
  if (kernelQueue.size()) {
    while (failures < kernelQueue.size()) {
      int_tp fastestKernel = -1;
      float fastestTime = 999999990000000000000000000.0f;

      for (int_tp x = 0; x < kernelQueue.size(); x++) {
        if (kernelQueue[x]->executionTime < fastestTime
            && kernelQueue[x]->tested == false) {
          fastestKernel = x;
          fastestTime = kernelQueue[x]->executionTime;
        }
      }
      if (fastestKernel < 0) break;
      // Test fastest kernel
      bool verified = verify_result(bottom, top, bottom_index_, num_,
                                    verify_blob, kernelQueue[fastestKernel]);
      if (verified == true) {
        kernelQueue[fastestKernel]->verified = true;
        kernel_index_ = fastestKernel;
        verification = true;
        break;
      } else {
        kernelQueue[fastestKernel]->tested = true;
        dbgPrint(std::cout << "Kernel "
                           << kernelQueue[fastestKernel]->kernelName
                           << " failed verification" << std::endl);
        failures++;
      }
    }
  }
  if (verification) {
    dbgPrint(std::cout << "Kernel <" << kernelQueue[kernel_index_]->kernelName
                       << "> passed verification" << std::endl);
  } else {
    dbgPrint(std::cout << "Verification was not successful, "
                       << "fallback to basic kernel" << std::endl);
    create_basic_kernel(bottom, top, 1, 1, 1);
    kernel_index_ = kernelQueue.size() - 1;
    verification = verify_result(bottom, top, bottom_index_, num_,
                                 verify_blob, kernelQueue[kernel_index_]);
    CHECK_EQ(verification, true) << "Basic kernel failed verification."
                                 << std::endl;
  }
  this->bestKernelConfig = kernelQueue[kernel_index_];

  dbgPrint(std::cout << "Convolution Time:"
                     << kernelQueue[kernel_index_]->executionTime << std::endl);

  if (bestKernelConfig->kernelType != 2 && bestKernelConfig->kernelType != 5)
    swizzled_weights_ = NULL;

  for (int_tp x = 0; x < kernelQueue.size(); x++) {
    if (x != kernel_index_) {
      viennacl::ocl::current_context().delete_program(
          kernelQueue[x]->kernelName);
      delete kernelQueue[x];
    }
  }
  kernelQueue.clear();

  tuned_ = true;

  string outputFile;
  outputFile = cache_path_.str() + key_;
  std::ifstream cachedKernel(outputFile.c_str());
  std::ofstream outputKernel;
  outputKernel.open(outputFile.c_str());
  outputKernel << bestKernelConfig->workItem_output[0] << " "
               << bestKernelConfig->workItem_output[1] << " "
               << bestKernelConfig->workItem_output[2] << " "
               << bestKernelConfig->kernelType << " "
               << bestKernelConfig->global_work_size[0] << " "
               << bestKernelConfig->global_work_size[1] << " "
               << bestKernelConfig->global_work_size[2] << " "
               << bestKernelConfig->local_work_size[0] << " "
               << bestKernelConfig->local_work_size[1] << " "
               << bestKernelConfig->local_work_size[2] << " "
               << bestKernelConfig->swizzle_weights << " "
               << 0 << " "  // deprecated
               << bestKernelConfig->use_null_local << " ";
  outputKernel.close();
  ctx.cache_path(viennacl_cache_path);
}

template<>
void ConvolutionLayerSpatial<float>::Forward_gpu(
    const vector<Blob<float>*>& bottom, const vector<Blob<float>*>& top) {
  weight = this->blobs_[0]->gpu_data();
  weight_cpu = static_cast<const float*>(this->blobs_[0]->cpu_data());
  if (bias_term_)
    bias_ = this->blobs_[1]->gpu_data();

  for (int_tp i = 0; i < bottom.size(); ++i) {
    bottom_index_ = i;
    bottom_data = bottom[i]->gpu_data();
    top_data = top[i]->mutable_gpu_data();
    weight_offset = M_ * K_;
    col_offset = K_ * N_;
    top_offset = M_ * N_;
    bias_offset_ = 0;

    if (!tuned_) {
      Blob<float> verify_blob;
      verify_blob.ReshapeLike(*top[i]);
      float *verify_data = verify_blob.mutable_gpu_data();
      const float *weight_gpu_data = this->blobs_[0]->gpu_data();
      const float *bottom_gpu_data = bottom[i]->gpu_data();
      for (int_tp n = 0; n < this->num_; ++n) {
        this->forward_gpu_gemm(bottom_gpu_data, n * this->bottom_dim_,
                               weight_gpu_data, verify_data,
                               n * this->top_dim_);
        if (this->bias_term_) {
          const float* bias = this->blobs_[1]->gpu_data();
          this->forward_gpu_bias(verify_data, n * this->top_dim_, bias);
        }
      }
      generate_key();
      setup_convolution(bottom, top, verify_blob);
      CHECK_EQ(tuned_, true) << "Spatial convolution auto-tuning failed.";
    }

    convolve(bottom, top, i, num_, bestKernelConfig);
  }
}

template<>
void ConvolutionLayerSpatial<float>::Backward_gpu(
    const vector<Blob<float>*>& top, const vector<bool>& propagate_down,
    const vector<Blob<float>*>& bottom) {
  const float* weight = this->blobs_[0]->gpu_data();
  float* weight_diff = this->blobs_[0]->mutable_gpu_diff();
  for (int_tp i = 0; i < top.size(); ++i) {
    const float* top_diff = top[i]->gpu_diff();
    // Bias gradient, if necessary.
    if (this->bias_term_ && this->param_propagate_down_[1]) {
      float* bias_diff = this->blobs_[1]->mutable_gpu_diff();
      for (int_tp n = 0; n < this->num_; ++n) {
        this->backward_gpu_bias(bias_diff, top_diff, n * this->top_dim_);
      }
    }
    if (this->param_propagate_down_[0] || propagate_down[i]) {
      const float* bottom_data = bottom[i]->gpu_data();
      float* bottom_diff = bottom[i]->mutable_gpu_diff();
      for (int_tp n = 0; n < this->num_; ++n) {
        // gradient w.r.t. weight. Note that we will accumulate diffs.
        if (this->param_propagate_down_[0]) {
          this->weight_gpu_gemm(bottom_data, n * this->bottom_dim_,
              top_diff, n * this->top_dim_, weight_diff);
        }
      }
      // gradient w.r.t. bottom data, if necessary.
      if (propagate_down[i]) {
        // Multi queue execution, all previous work needs to be done first
        this->device_->FinishQueues();
        for (int_tp n = 0; n < this->num_; ++n) {
          // Multi queue execution, go through work queues
          this->device_->SwitchQueue(n);
          this->backward_gpu_gemm(top_diff, n * this->top_dim_, weight,
                                  bottom_diff, n * this->bottom_dim_);
        }
        // Multi queue execution, finish all queues
        this->device_->FinishQueues();
      }
    }
  }
}

template<typename Dtype>
void ConvolutionLayerSpatial<Dtype>::load_cached_kernels(
    const vector<Blob<Dtype>*>& bottom, const vector<Blob<Dtype>*>& top) {
  // Generates static key_
  std::string previous_key = key_;
  generate_key();
  int prev_kernel_type = 0;
  if (tuned_) {
    if (key_.compare(previous_key) == 0)
      return;
    tuned_ = false;
    prev_kernel_type = bestKernelConfig->kernelType;
    viennacl::ocl::current_context().
      delete_program(bestKernelConfig->kernelName);
    delete bestKernelConfig;
    bestKernelConfig = NULL;
  }
  // Initializes unique kernel ID
  kernel_uid_ = 0;

  // Find cached kernel configuration
  string outputFile;
  outputFile = cache_path_.str() + key_;
  std::ifstream cachedKernel(outputFile.c_str());
  if (cachedKernel) {
    int_tp x, y, z, type;
    cachedKernel >> x;
    cachedKernel >> y;
    cachedKernel >> z;
    cachedKernel >> type;
    if (type == 2) {
      if (z == 1)
        z = 16;
      CHECK_EQ(z == 16 || z == 8, true) << "invalid SIMD size" << std::endl;
    }
    create_convolution_kernel(bottom, top, type, x, y, z);
    kernel_index_ = kernelQueue.size() - 1;
    if (kernel_index_ == -1) {
      std::cerr << "Failed to get kernel from cached configurations."
                << std::endl;
      std::cerr << "Deleting broken cache file and try tuning again..."
                << std::endl;
      string bakFile = outputFile + ".bak";
      std::rename(outputFile.c_str(), bakFile.c_str());
      return;
    }
    bestKernelConfig = kernelQueue[kernel_index_];
    kernelQueue.clear();
    // As we are using varying image size kernels now, let's skip the
    // cached work group size and local group size here, and we already
    // get correct work/local group size at the create_convolution kernel stage.
    // To not break the previous trained record, for now just skipping them.
    // Will use a totally different cache mechanism in the future.
    size_t foo;  // for deprecated parameters.
    cachedKernel >> foo;
    cachedKernel >> foo;
    cachedKernel >> foo;
    cachedKernel >> bestKernelConfig->local_work_size[0];
    cachedKernel >> bestKernelConfig->local_work_size[1];
    cachedKernel >> bestKernelConfig->local_work_size[2];
    if (bestKernelConfig->kernelType == 1)
      calculate_global_size(1, bestKernelConfig->workItem_output,
                            bestKernelConfig->local_work_size,
                            bestKernelConfig->global_work_size);
    cachedKernel >> bestKernelConfig->swizzle_weights;
    cachedKernel >> foo;
    cachedKernel >> bestKernelConfig->use_null_local;
    tuned_ = true;
    // If kernel type changed to type 2 or 4, we need to reset the swizzled
    // weights pointer to invalidate the previous swizzled weights data.
    if (prev_kernel_type != bestKernelConfig->kernelType &&
        (bestKernelConfig->kernelType == 2 ||
         bestKernelConfig->kernelType == 5))
      swizzled_weights_ = NULL;
  }
  return;
}

template<typename Dtype>
void ConvolutionLayerSpatial<Dtype>::SetUp(
    const vector<Blob<Dtype>*>& bottom, const vector<Blob<Dtype>*>& top,
    caffe::Backend backend) {
  if (backend == caffe::BACKEND_OpenCL) {
    load_cached_kernels(bottom, top);
  }
}

template void ConvolutionLayerSpatial<float>::SetUp(
    const vector<Blob<float>*>& bottom, const vector<Blob<float>*>& top,
    caffe::Backend backend);

template void ConvolutionLayerSpatial<double>::SetUp(
    const vector<Blob<double>*>& bottom, const vector<Blob<double>*>& top,
    caffe::Backend backend);

template void ConvolutionLayerSpatial<float>::swizzleWeights(
    const vector<Blob<float>*>& bottom,
    const vector<Blob<float>*>& top,
    int_tp swizzle_factor,
    bool interleave = false);
template void ConvolutionLayerSpatial<double>::swizzleWeights(
    const vector<Blob<double>*>& bottom,
    const vector<Blob<double>*>& top,
    int_tp swizzle_factor,
    bool interleave = false);

template<>
void ConvolutionLayerSpatial<double>::create_convolution_kernel(
    const vector<Blob<double>*>& bottom, const vector<Blob<double>*>& top,
    int_tp kernelType,
    int_tp blockWidth, int_tp blockHeight,
    int_tp blockDepth) {
  NOT_IMPLEMENTED;
  return;
}

template<>
bool ConvolutionLayerSpatial<double>::setup_IDLF(
    const vector<Blob<double>*>& bottom, const vector<Blob<double>*>& top,
    int_tp blockWidth,
    int_tp blockHeight, int_tp blockDepth) {
  NOT_IMPLEMENTED;
  return false;
}

template<>
bool ConvolutionLayerSpatial<double>::create_gemm_like_conv_kernel(
    const vector<Blob<double>*>& bottom, const vector<Blob<double>*>& top,
    int_tp blockWidth,
    int_tp blockHeight, int_tp blockDepth) {
  NOT_IMPLEMENTED;
  return false;
}

template<>
bool ConvolutionLayerSpatial<double>::verify_result(
    const vector<Blob<double>*>& bottom, const vector<Blob<double>*>& top,
    int_tp index,
    int_tp numImages, const Blob<double> &verify_blob, kernelConfig* config) {
  NOT_IMPLEMENTED;
  return false;
}

template<>
bool ConvolutionLayerSpatial<double>::create_basic_kernel(
    const vector<Blob<double>*>& bottom, const vector<Blob<double>*>& top,
    int_tp blockWidth,
    int_tp blockHeight, int_tp blockDepth) {
  NOT_IMPLEMENTED;
  return false;
}

template<>
bool ConvolutionLayerSpatial<double>::tune_local_size(
    const vector<Blob<double>*>& bottom, const vector<Blob<double>*>& top,
    kernelConfig* config) {
  NOT_IMPLEMENTED;
  return false;
}

template<>
cl_int ConvolutionLayerSpatial<double>::convolve(
    const vector<Blob<double>*>& bottom, const vector<Blob<double>*>& top,
    int_tp index,
    int_tp numImages, kernelConfig* config) {
  NOT_IMPLEMENTED;
  return false;
}

template<>
float ConvolutionLayerSpatial<double>::timed_convolve(
    const vector<Blob<double>*>& bottom, const vector<Blob<double>*>& top,
    int_tp index,
    int_tp numImages, kernelConfig* config) {
  NOT_IMPLEMENTED;
  return 0.f;
}

template<>
void ConvolutionLayerSpatial<double>::setup_convolution(
    const vector<Blob<double>*>& bottom, const vector<Blob<double>*>& top,
    const Blob<double> &verify_blob) {
  NOT_IMPLEMENTED;
}

template<>
void ConvolutionLayerSpatial<double>::calculate_global_size(
    int_tp batch,
    int_tp* workItemOutput,
    size_t* localSizes, size_t* globalSizes) {
  NOT_IMPLEMENTED;
}

template<>
void ConvolutionLayerSpatial<double>::generate_key() {
  NOT_IMPLEMENTED;
}

template<>
std::string ConvolutionLayerSpatial<double>::generate_specific_key(
    int_tp type, int_tp blockWidth, int_tp blockHeight, int_tp blockDepth) {
  NOT_IMPLEMENTED;
  return "";
}

template<>
void ConvolutionLayerSpatial<double>::Forward_gpu(
    const vector<Blob<double>*>& bottom, const vector<Blob<double>*>& top) {
  NOT_IMPLEMENTED;
}

template<>
void ConvolutionLayerSpatial<double>::Backward_gpu(
    const vector<Blob<double>*>& top, const vector<bool>& propagate_down,
    const vector<Blob<double>*>& bottom) {
  NOT_IMPLEMENTED;
}
#else
template<typename Dtype>
void ConvolutionLayerSpatial<Dtype>::Forward_gpu(
    const vector<Blob<Dtype>*>& bottom, const vector<Blob<Dtype>*>& top) {
  NOT_IMPLEMENTED;
}

template<typename Dtype>
void ConvolutionLayerSpatial<Dtype>::Backward_gpu(
    const vector<Blob<Dtype>*>& top, const vector<bool>& propagate_down,
    const vector<Blob<Dtype>*>& bottom) {
  NOT_IMPLEMENTED;
}
#endif
INSTANTIATE_LAYER_GPU_FUNCS(ConvolutionLayerSpatial);
#endif

#ifdef CPU_ONLY
STUB_GPU(ConvolutionLayerSpatial);
#endif

INSTANTIATE_CLASS(ConvolutionLayerSpatial);

}  // namespace caffe
#endif  // USE_INTEL_SPATIAL


#ifndef ASLAM_BACKEND_GENERIC_MATRIX_EXPRESSION_HPP
#define ASLAM_BACKEND_GENERIC_MATRIX_EXPRESSION_HPP

#include <set>
#include <type_traits>
#include <boost/shared_ptr.hpp>
#include <Eigen/Core>
#include <aslam/backend/JacobianContainer.hpp>
#include "ScalarExpression.hpp"
#include "ScalarExpressionNode.hpp"
#include "GenericMatrixExpressionNode.hpp"
#include "OperationResultNodes.hpp"
#include "Differential.hpp"

namespace aslam {
namespace backend {

namespace internal {
template<typename TNode>
struct GenericMatrixNodeTraits {
  typedef TNode node_t;
  typedef typename boost::shared_ptr<node_t> node_ptr_t;
  typedef typename node_t::matrix_t matrix_t;
  typedef typename node_t::value_t value_t;
  typedef typename node_t::tangent_vector_t tangent_vector_t;
  typedef typename node_t::differential_t differential_t;
  typedef typename node_t::constant_t constant_t;
};
}

template<int IRows, int ICols, typename TScalar = double, typename TNode = GenericMatrixExpressionNode<IRows, ICols, TScalar> >
class GenericMatrixExpression {
 public:
  enum { Dimension = IRows*ICols };
  typedef GenericMatrixExpression self_t;
  typedef GenericMatrixExpressionNode<IRows, ICols, TScalar> default_node_t;
  typedef TNode node_t;
  typedef GenericMatrixExpression<IRows, ICols, TScalar, default_node_t> default_self_t;

  typedef TScalar scalar_t;
  typedef internal::GenericMatrixNodeTraits<node_t> node_traits_t;
  typedef typename node_traits_t::node_ptr_t node_ptr_t;
  typedef typename node_traits_t::matrix_t matrix_t;
  typedef typename node_traits_t::value_t value_t;
  typedef typename node_traits_t::tangent_vector_t tangent_vector_t;
  typedef typename node_traits_t::differential_t differential_t;
  typedef typename node_traits_t::constant_t constant_t;

  /// \brief initialize from an existing node.
  GenericMatrixExpression(node_ptr_t root);

  /// \brief Initialize from an existing node. The node will not be deleted.
  GenericMatrixExpression(node_t * root);

  /// \brief Initialize a constant expression from an matrix. The matrix will be copied.
  template<typename DERIVED>
  GenericMatrixExpression(const Eigen::MatrixBase<DERIVED> & mat);

  /// \brief Initialize with a TScalar in case of 1 x 1 matrix (allows implizit cast from TScalar)
  template<typename dummy = int>
  GenericMatrixExpression(TScalar v, dummy = typename std::enable_if<(IRows == 1 && ICols == 1), int>::type(0));

  /// \brief Initialize from a ScalarExpression in case of 1 x 1 matrix
  template<typename dummy = int>
  GenericMatrixExpression(ScalarExpression v, dummy = typename std::enable_if<(IRows == 1 && ICols == 1), int>::type(0));

  /// \brief Support cast to ScalarExpression in case of 1 x 1 matrix
  template<typename dummy = int, typename = typename std::enable_if<(IRows == 1 && ICols == 1), dummy>::type>
  operator ScalarExpression const (){ return toScalarExpression(); }

  virtual ~GenericMatrixExpression() {
  }

  /// \brief Evaluate the full Eigen matrix.
  matrix_t toFullMatrix() const;

  /// \brief Evaluate the full Eigen matrix.
  matrix_t evaluate() const {
    return toFullMatrix();
  }

  /// \brief return the expression representing the Moore-Penrose-Inverse of this expression.
  GenericMatrixExpression<ICols, IRows, TScalar> inverse() const;

  /// \brief return the transposed matrix expression.
  GenericMatrixExpression<ICols, IRows, TScalar> transpose() const;

  /// \brief return the squared norm expression (only supported for column vectors)
  ScalarExpression squaredNorm() const;

  template<int IColsOther, typename TOtherNode>
  GenericMatrixExpression<IRows, IColsOther, TScalar> operator*(const GenericMatrixExpression<ICols, IColsOther, TScalar, TOtherNode> & other) const;

  template<typename Derived>
  GenericMatrixExpression<IRows, Derived::ColsAtCompileTime, TScalar> operator*(const Eigen::MatrixBase<Derived> & otherMatrix) const {
    return *this * GenericMatrixExpression<ICols, Derived::ColsAtCompileTime, TScalar>(otherMatrix);
  }

  inline operator GenericMatrixExpression<IRows, ICols, TScalar>() const;

  default_self_t operator*(const ScalarExpression & scalarExpression) const;
  default_self_t operator*(TScalar scalar) const;
  default_self_t operator-() const;
  template<typename TOtherNode>
  default_self_t operator+(const GenericMatrixExpression<IRows, ICols, TScalar, TOtherNode> & other) const;
  template <typename Derived>
  default_self_t operator+(const Eigen::MatrixBase<Derived> & m) const { return *this + default_self_t(m); }
  template<typename TOtherNode>
  default_self_t operator-(const GenericMatrixExpression<IRows, ICols, TScalar, TOtherNode> & other) const;
  template <typename Derived>
  default_self_t operator-(const Eigen::MatrixBase<Derived> & m) const { return *this - default_self_t(m); }

  template<int IColsOther, typename TOtherNode, typename std::enable_if<IRows == 3 && (ICols == 1 || IColsOther == 1), int>::type = 0>
  GenericMatrixExpression<3, (ICols == 1 ? IColsOther : ICols), TScalar> cross(const GenericMatrixExpression<3, IColsOther, TScalar, TOtherNode> & other) const;

  /// \brief Evaluate the Jacobians
  void evaluateJacobians(JacobianContainer & outJacobians) const;
  void evaluateJacobians(JacobianContainer & outJacobians, const Eigen::MatrixXd & applyChainRule) const;
  void evaluateJacobians(JacobianContainer & outJacobians, const differential_t & diff) const;

  void getDesignVariables(DesignVariable::set_t & designVariables) const;

  ScalarExpression toScalarExpression() const;
  template <int RowIndex, int ColIndex>
  ScalarExpression toScalarExpression() const;

  ScalarExpression toScalarExpression(int row, int col = 0) const;

  ScalarExpression operator () (int row, int col = 0) const {
    return toScalarExpression(row, col);
  }
  ScalarExpression operator [] (int row) const {
    return toScalarExpression(row);
  }

  inline node_ptr_t root() const {
    return _root;
  }

  inline static self_t constant(int rows = IRows, int cols = ICols) {
    return GenericMatrixExpression(new constant_t(rows, cols));
  }
 protected:
  GenericMatrixExpression() {
  }
 private:
  node_ptr_t _root;

 public:
  template<typename TDerived, typename TOperand>
  class UnaryOperationResult : public UnaryOperationResultNode<TDerived, TOperand, self_t, typename node_t::differential_t::domain_t, scalar_t> {
  };

  template<typename TDerived, typename TLhs, typename TRhs>
  class BinaryOperationResult : public BinaryOperationResultNode<TDerived, TLhs, TRhs, self_t, typename node_t::differential_t::domain_t, scalar_t> {
  };
};

template<int ICols, int IColsOther, typename TScalar, typename TNode, typename Derived>
GenericMatrixExpression<Derived::RowsAtCompileTime, IColsOther, TScalar> operator*(const Eigen::MatrixBase<Derived> & matrix, GenericMatrixExpression<ICols, IColsOther, TScalar, TNode> otherMatrix) {
  return  GenericMatrixExpression<Derived::RowsAtCompileTime, ICols, TScalar>(matrix) * otherMatrix;
}

template<int IRows, int ICols, typename TScalar, typename TNode, typename Derived>
GenericMatrixExpression<IRows, ICols, TScalar> operator+(const Eigen::MatrixBase<Derived> & matrix, GenericMatrixExpression<IRows, ICols, TScalar, TNode> otherMatrix) {
  return  GenericMatrixExpression<IRows, ICols, TScalar>(matrix) + otherMatrix;
}

template<int IRows, int ICols, typename TScalar, typename TNode, typename Derived>
GenericMatrixExpression<IRows, ICols, TScalar> operator-(const Eigen::MatrixBase<Derived> & matrix, GenericMatrixExpression<IRows, ICols, TScalar, TNode> otherMatrix) {
  return  GenericMatrixExpression<IRows, ICols, TScalar>(matrix) - otherMatrix;
}

}  // namespace backend
}  // namespace aslam

#include "implementation/GenericMatrixExpression.hpp"

#endif /* ASLAM_BACKEND_MATRIX_EXPRESSION_HPP */


/** This is free and unencumbered software released into the public domain.

The authors of ISIS do not claim copyright on the contents of this file.
For more details about the LICENSE terms and the AUTHORS, you will
find files of those names at the top level of this repository. **/

/* SPDX-License-Identifier: CC0-1.0 */

#include <boost/foreach.hpp>
#include "opencv2/opencv.hpp"
#include "opencv2/xfeatures2d.hpp"

#include "SIFTAlgorithm.h"
#include <QVariant>

namespace Isis {

  /**
   * Constructs an empty SiftAlgorithm with default variables.
   */
  SIFTAlgorithm::SIFTAlgorithm() :
                 Feature2DAlgorithm( "SIFT", "Feature2D",
                                      SIFTType::create() ) {
    setupParameters();
  }

  /**
   * Constructs a SIFTAlgorithm with input variables.
   *
   * @param cvars Variables that are not included will be set to their default.
   * @param config The config string used to construct cvars.
   */
  SIFTAlgorithm::SIFTAlgorithm(const PvlFlatMap &cvars, const QString &config) :
                 Feature2DAlgorithm("SIFT", "Feature2D",
                                     SIFTType::create(), cvars) {
    setConfig(config);
    PvlFlatMap variables = setupParameters();
    variables.merge(cvars);
    const int nfeatures = toInt(variables.get("nfeatures"));
    const int nOctaveLayers = toInt(variables.get("nOctaveLayers"));
    const double contrastThreshold = toDouble(variables.get("constrastThreshold"));
    const double edgeThreshold = toDouble(variables.get("edgeThreshold"));
    const double sigma = toDouble(variables.get("sigma"));

    m_algorithm = SIFTType::create(nfeatures, nOctaveLayers, contrastThreshold,
                                   edgeThreshold, sigma);

    m_variables.merge(variables);
  }

  /**
   * Destroys the algorithm
   */
  SIFTAlgorithm::~SIFTAlgorithm() { }

  /**
   * Sets up the algorithm parameters with default values.
   *
   * @return PvlFlatMap Algorithm parameters and their default values.
   */
  PvlFlatMap SIFTAlgorithm::setupParameters() {
    PvlFlatMap variables;
    variables.add("nfeatures",          "0");
    variables.add("nOctaveLayers",      "3");
    variables.add("constrastThreshold", "0.04");
    variables.add("edgeThreshold",      "10");
    variables.add("sigma",              "1.6");
    m_variables = variables;
    return (m_variables);
  }

  /**
   * Returns a description of the algorithm.
   *
   * @return @b QString A description of the algorithm.
   */
  QString SIFTAlgorithm::description() const {
    QString desc = "The OpenCV SIFT Feature2D detector/extractor algorithm."
                   " See the documentation at "
     "http://docs.opencv.org/3.1.0/d5/d3c/classcv_1_1xfeatures2d_1_1SIFT.html";
    return (desc);
  }

  /**
   * Creates and returns an intance of SIFTAlgorithm.
   *
   * @param vars PvlFlatMap containing algorithm parameters and their values
   * @param config A configuration string input by the user
   *
   * @return @b Feature2DAlgorithm An instance of SIFTAlgorithm
   */
  Feature2DAlgorithm *SIFTAlgorithm::create(const PvlFlatMap &vars, const QString &config) {
    return ( new SIFTAlgorithm(vars, config) );
  }

  /**
   * Returns true if the algorithm has a detector.
   *
   * @return @b true if the algorithm has a detector.
   */
  bool SIFTAlgorithm::hasDetector() const {
    return true;
  }

  /**
   * Returns true if the algorithm has an extractor.
   *
   * @return @b true if the algorithm has an extractor.
   */
  bool SIFTAlgorithm::hasExtractor() const {
    return true;
  }

  /**
   * Returns true if the algorithm has a matcher.
   *
   * @return @b true if the algorithm has a matcher.
   */
  bool SIFTAlgorithm::hasMatcher() const {
    return false;
  }

/**
 * Get and return SIFT's parameters and what they're set to as a PvlFlatMap.
 *
 * @return @b PvlFlatMap A PvlFlatMap of SIFT's currently set variables and their values.
 */
  PvlFlatMap SIFTAlgorithm::getAlgorithmVariables( ) const {
    return ( variables() );
  }

/**
 * @brief Set parameters as provided by the variables, does not work for the
 * SIFT algorithm in OpenCV3, so calling this will throw an exception.
 * Always returns -1.
 *
 * @param variables Container of parameters to set
 *
 * @throws IException::Programmer "SIFT does not have the ability to set algorithm parameters.";
 *
 * @return @b int -1 Cannot set the Algorithm Variables
 */
  int SIFTAlgorithm::setAlgorithmVariables(const PvlFlatMap &variables) {
    QString message = "SIFT does not have the ability to set algorithm parameters.";
    throw IException(IException::Programmer, message, _FILEINFO_);

    return (-1);
  }

};  // namespace Isis


/* -*- c++ -*- */
/*
 * Copyright 2013-2014 Free Software Foundation, Inc.
 *
 * This file is part of GNU Radio
 *
 * GNU Radio is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 3, or (at your option)
 * any later version.
 *
 * GNU Radio is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with GNU Radio; see the file COPYING.  If not, write to
 * the Free Software Foundation, Inc., 51 Franklin Street,
 * Boston, MA 02110-1301, USA.
 */

#ifdef HAVE_CONFIG_H
#include "config.h"
#endif

#include "puncture_ff_impl.h"
#include <gnuradio/io_signature.h>
#include <volk/volk.h>
#include <boost/bind.hpp>
#include <pmt/pmt.h>
#include <string>
#include <stdio.h>

namespace gr {
  namespace fec {

    puncture_ff::sptr
    puncture_ff::make(int puncsize, int puncpat, int delay)
    {
      return gnuradio::get_initial_sptr
        (new puncture_ff_impl(puncsize, puncpat, delay));
    }

    puncture_ff_impl::puncture_ff_impl(int puncsize, int puncpat, int delay)
      : block("puncture_ff",
              io_signature::make(1, 1, sizeof(float)),
              io_signature::make(1, 1, sizeof(float))),
        d_puncsize(puncsize), d_delay(delay)
    {
      // Create a mask of all 1's of puncsize length
      int mask = 0;
      for(int i = 0; i < d_puncsize; i++)
        mask |= 1 << i;

      // Rotate the pattern for the delay value; then mask it if there
      // are any excess 1's in the pattern.
      for(int i = 0; i < d_delay; ++i) {
	puncpat = ((puncpat & 1) << (d_puncsize - 1)) + (puncpat >> 1);
      }
      d_puncpat = puncpat & mask;

      // Calculate the number of holes in the pattern. The mask is all
      // 1's given puncsize and puncpat is a pattern with >= puncsize
      // 0's (masked to ensure this). The difference between the
      // number of 1's in the mask and the puncpat is the number of
      // holes.
      uint32_t count_mask=0, count_pat=0;
      volk_32u_popcnt(&count_mask, static_cast<uint32_t>(mask));
      volk_32u_popcnt(&count_pat, static_cast<uint32_t>(d_puncpat));
      d_puncholes = count_mask - count_pat;

      set_fixed_rate(true);
      set_relative_rate((double)(d_puncsize - d_puncholes)/d_puncsize);
      set_output_multiple(d_puncsize - d_puncholes);
      //set_msg_handler(boost::bind(&puncture_ff_impl::catch_msg, this, _1));
    }

    puncture_ff_impl::~puncture_ff_impl()
    {
    }

    int
    puncture_ff_impl::fixed_rate_ninput_to_noutput(int ninput)
    {
      return (int)((((d_puncsize - d_puncholes)/(double)(d_puncsize)) * ninput) + .5);
    }

    int
    puncture_ff_impl::fixed_rate_noutput_to_ninput(int noutput)
    {
      return (int)(((d_puncsize/(double)(d_puncsize-d_puncholes)) * noutput) + .5);
    }

    void
    puncture_ff_impl::forecast(int noutput_items,
                               gr_vector_int& ninput_items_required)
    {
      ninput_items_required[0] = (int)(((d_puncsize/(double)(d_puncsize-d_puncholes)) * noutput_items) + .5);
    }

    /*
    void
    puncture_ff_impl::catch_msg(pmt::pmt_t msg)
    {
      long mlong = pmt::pmt_to_long(msg);
      for(int i = 0; i < mlong; ++i) {
        d_puncholes = (d_puncholes >> 1) | ((d_puncholes & 1) << (d_puncsize - 1));
      }
    }
    */

    int
    puncture_ff_impl::general_work(int noutput_items,
                                   gr_vector_int& ninput_items,
                                   gr_vector_const_void_star &input_items,
                                   gr_vector_void_star &output_items)
    {
      const float *in = (const float *)input_items[0];
      float *out = (float *)output_items[0];

      for(int i=0, k=0; i < noutput_items/output_multiple(); ++i) {
        for(int j = 0; j < d_puncsize; ++j) {
	  if((d_puncpat >> (d_puncsize - 1 - j)) & 1) {
            out[k++] = in[i*d_puncsize + j];
	  }
        }
      }

      /*
      GR_LOG_DEBUG(d_debug_logger, ">>>>>> start");
      for(int i = 0, k=0; i < noutput_items; ++i) {
        if((d_puncpat >> (d_puncsize - 1 - (i % d_puncsize))) & 1) {
	  GR_LOG_DEBUG(d_debug_logger, boost::format("%1%...%2%") \
                       % out[k++] % in[i]);
        }
        else {
	  GR_LOG_DEBUG(d_debug_logger, boost::format("snit %1%") % in[i]);
        }
      }

      GR_LOG_DEBUG(d_debug_logger, boost::format("comp: %1%, %2%\n") \
                   % noutput_items % ninput_items[0]);
      GR_LOG_DEBUG(d_debug_logger, boost::format("consuming %1%") \
                   % ((int)(((1.0/relative_rate()) * noutput_items) + .5)));
      */

      consume_each((int)(((1.0/relative_rate()) * noutput_items) + .5));
      return noutput_items;
    }

  } /* namespace fec */
}/* namespace gr */


#include <iostream>
#include <fstream>
#include <queue>
#include <string>
#include <limits>

#include <boost/random.hpp>
#include <boost/generator_iterator.hpp>

#include <glog/logging.h>

using boost::variate_generator;
using boost::mt19937;
using boost::exponential_distribution;

#define ONLY_EVENT_TYPE     0
#define NUMBER_EVENT_TYPES  3   //NEED TO MAKE SURE THIS IS 1 MORE THAN THE LAST DEFINED EVENT

const std::string event_names[NUMBER_EVENT_TYPES] = {
    "ARRIVE",
    "DEPART",
    "CLOSE"
};

bool closed = false;

std::vector<bool> servers_idle;
std::vector< std::queue<double>* > servers_queue;

// Stats vectors
std::vector<double> times_after_close;
std::vector<double> average_times_in_queue;
std::vector< std::vector<double>* > average_server_utilizations;
std::vector< std::vector<double>* > average_length_of_queues;

// Output File
std::ofstream outfile;

/**
 *  A class for an event, which holds an int for the event type, a double for simulation time and possibly other data.
 *  We may want to subclass this with our own events
 */
class Event {
    public:
        const double time;
        const double server_start_time;
        const int server_index;
        const int type;

        Event(double time, int type) : time(time), server_start_time(0), server_index(-1), type(type) {
            //cout << "created an event with simulation time: " << this->time << endl;
            //cout << "created an event with event type: " << this->type << endl;
        }

        Event(double time, double server_start_time, int server_index, int type) : time(time), server_start_time(server_start_time), server_index(server_index), type(type) {
            //cout << "created an event with simulation time: " << this->time << endl;
            //cout << "created an event with event type: " << this->type << endl;
        }

        bool operator<( const Event& e ) const {
            return time < e.time;
        }

        bool operator>( const Event& e ) const {
            return time > e.time;
        }

        bool less( const Event& e) const {
            return time < e.time;
        }

        friend std::ostream& operator<< (std::ostream& out, Event& event);
};

class CompareEvent {
    public:
        bool operator()(Event* &e1, Event* &e2) {
            return e1->time > e2->time;
            //return false;
        }
};

// Print event
std::ostream& operator<< ( std::ostream& out, Event& event) {
    out << "[sim_time: " << std::setw(10) << std::setprecision(3) << std::fixed << event.time << ", type: " << std::setw(4) << event.type;
    if (event.type < NUMBER_EVENT_TYPES) {
        out << " - " << std::left << std::setw(50) << event_names[event.type];
    } else {
        out << std::left << std::setw(50) << " - UNKNOWN";
    }
    out << std::right << "]";
    return out;
}

bool server_idle(int &idle_index) {
    for (int i = 0; i < servers_idle.size(); i++) {
        if (servers_idle[i] == true) {
            idle_index = i;
            return true;
        }
    }
    return false;
}

void addToQueue(double enqueue_time) {
    std::queue<double> *shortest = NULL;
    for (int i = 0; i < servers_queue.size(); i++) {
        if (shortest == NULL || servers_queue[i]->size() < shortest->size()) {
            shortest = servers_queue[i];
        }
    }
    shortest->push(enqueue_time);
}

void jockey(const int &departed_queue) {
    for (int i = 1; i < servers_queue.size(); i++) {
        int pos = departed_queue - i;
        if (pos < 0) pos = servers_queue.size() + pos;
        if (servers_queue[departed_queue]->size() < servers_queue[pos]->size()) {
            servers_queue[departed_queue]->push(servers_queue[pos]->front());
            servers_queue[pos]->pop();
        }
    }
}

void run_simulation(
        const double close_time,
        variate_generator< mt19937, exponential_distribution<> > &arrival_generator,
        variate_generator< mt19937, exponential_distribution<> > &departure_generator) {
    double simulation_time_s = 0;
    double previous_time_s = 0;

    std::vector<double> servers_work_time;
    std::vector<double> servers_time_queue_length;
    for (int i = 0; i < servers_idle.size(); i++) {
        servers_work_time.push_back(0);
        servers_time_queue_length.push_back(0);
    }

    double total_queue_time = 0;
    double total_departures = 0;

    std::priority_queue<Event*, std::vector<Event*>, CompareEvent> heap;

    // Put initial event in the heap
    heap.push(new Event(simulation_time_s + arrival_generator(), 0));
    heap.push(new Event(close_time, 2));

    VLOG(2) << "Start Simulation...";

    while (!heap.empty()) {
        Event *current_event = heap.top();
        heap.pop();

        if (current_event == NULL) {
            LOG(ERROR) << "Simulation not complete and there are no events in the min heap.";
            LOG(ERROR) << "\tsimulation time: " << simulation_time_s;
            exit(0);
        }

        previous_time_s = simulation_time_s;
        simulation_time_s = current_event->time;

        // Calculate average time stuff
        double time_since_last = simulation_time_s - previous_time_s;
        // Calculate sum of time queue length for all servers
        for (int i = 0; i < servers_idle.size(); i++) {
            servers_time_queue_length[i] += servers_queue[i]->size() * time_since_last;
        }

        int idle_index = -1;
        switch (current_event->type) {
            case 0: // ARRIVE
                // If the server is busy add new arrival to waiting queue
                // otherwise set the server as busy and add new departure time.
                VLOG(2) << "Arrival Begin";
                if (!server_idle(idle_index)) {
                    addToQueue(simulation_time_s);
                } else {
                    VLOG(2) << "Idle Index: " << idle_index;
                    servers_idle[idle_index] = false;
                    heap.push(new Event(simulation_time_s + departure_generator(), simulation_time_s, idle_index, 1));
                }
                if (!closed) {
                    // Add next arrival event
                    heap.push(new Event(simulation_time_s + arrival_generator(), 0));
                }
                VLOG(2) << "Arrival End";
                break;
            case 1: // DEPART
                // Add server busy time.
                // Which server is busy?
                servers_work_time[current_event->server_index] += simulation_time_s - current_event->server_start_time;
                jockey(current_event->server_index);
                // If the queue is empty set the server to idle otherwise
                // get the next person from the queue and set their departure
                // time.
                VLOG(2) << "Depart Begin";
                if (servers_queue[current_event->server_index]->empty()) {
                    servers_idle[current_event->server_index] = true;
                } else {
                    // Add current simulation time minus time stored in the
                    // queue to the total queue time.
                    total_queue_time += simulation_time_s - servers_queue[current_event->server_index]->front();
                    servers_queue[current_event->server_index]->pop();
                    // Add a new depature to the heap.
                    heap.push(new Event(simulation_time_s + departure_generator(), simulation_time_s, current_event->server_index, 1));
                }
                total_departures++;
                VLOG(2) << "Depart End";
                break;
            case 2: // CLOSE
                // Set flag to stop arrivals
                VLOG(2) << "Closed";
                closed = true;
                break;
            default:
                LOG(ERROR) << "Simulation had an event with an unknown type: " << current_event->type;
                LOG(ERROR) << "\tsimulation time: " << simulation_time_s;
                exit(0);
        }

        VLOG(3) << *current_event << ", h: " << heap.size();

        delete current_event; // Event's are created with new, so we need to delete them when we're done with them
    }

    assert(heap.empty());

    VLOG(1) << "The simulation ended at time: " << simulation_time_s;
    times_after_close.push_back(simulation_time_s - close_time);
    VLOG(1) << "The simulation ended " << simulation_time_s - close_time << " after close.";
    average_server_utilizations.push_back(new std::vector<double>());
    average_length_of_queues.push_back(new std::vector<double>());
    VLOG(1) << "Server util vector size: " << average_server_utilizations.size();
    for (int i = 0; i < servers_idle.size(); i++) {
        VLOG(1) << "The server " << i+1 << " was busy for time: " << servers_work_time[i];
        average_server_utilizations.back()->push_back(servers_work_time[i] / simulation_time_s);
        VLOG(1) << "The server " << i+1 << " utilization was: " << servers_work_time[i] / simulation_time_s;
        average_length_of_queues.back()->push_back(servers_time_queue_length[i] / simulation_time_s);
        VLOG(1) << "The average length of server " << i+1 << " queue was: " << servers_time_queue_length[i] / simulation_time_s;
        VLOG(1) << "Server util size: " << average_server_utilizations.back()->size();
    }
    VLOG(1) << "Total time spent in the queue: " << total_queue_time;
    VLOG(1) << "Total departures: " << total_departures;
    average_times_in_queue.push_back(total_queue_time / total_departures);
    VLOG(1) << "The average time spent in queue: " << total_queue_time / total_departures;
}

int main(int argc, char **argv) {
    // Initialize Google Logging
    google::InitGoogleLogging(argv[0]);
    // Log to Stderr
    FLAGS_logtostderr = 1;

    int seed = time(0);
    double arrival_mean = 1.0;
    double departure_mean = 4.5;
    variate_generator< mt19937, exponential_distribution<> > arrival_generator(mt19937(seed), exponential_distribution<>(1/arrival_mean));
    variate_generator< mt19937, exponential_distribution<> > departure_generator(mt19937(seed+1), exponential_distribution<>(1/departure_mean));

    // Intiate
    int duration = 480;
    int min_servers = 4;
    int max_servers = 7;
    int num_iterations = 10;

    // Open files
    std::stringstream filename;
    filename << "servers_stats" << ".dat";
    outfile.open(filename.str());

    for (int sev = min_servers; sev <= max_servers; sev++) {
        int num_servers = sev;
        for (int j = 0; j < num_iterations; j++) {
            closed = false;
            servers_idle.clear();
            servers_queue.clear();
            for (int k = 0; k < num_servers; k++) {
                servers_idle.push_back(true);
                servers_queue.push_back(new std::queue<double>());
            }
            run_simulation(duration, arrival_generator, departure_generator);

            for (int j = 0; j < num_servers; j++) {
                delete servers_queue[j];
            }
        }

        // Collect and print statistics.

        double sum_average_queue_time = std::accumulate(average_times_in_queue.begin(), average_times_in_queue.end(), 0.0);
        double min_average_queue_time = *std::min_element(average_times_in_queue.begin(), average_times_in_queue.end());
        double max_average_queue_time = *std::max_element(average_times_in_queue.begin(), average_times_in_queue.end());
        double avg_average_queue_time = sum_average_queue_time / average_times_in_queue.size();
        double sqr_average_queue_time = std::inner_product(average_times_in_queue.begin(), average_times_in_queue.end(), average_times_in_queue.begin(), 0.0);
        double std_average_queue_time = std::sqrt(sqr_average_queue_time / average_times_in_queue.size() - avg_average_queue_time * avg_average_queue_time);

        double sum_times_after_close = std::accumulate(times_after_close.begin(), times_after_close.end(), 0.0);
        double min_times_after_close = *std::min_element(times_after_close.begin(), times_after_close.end());
        double max_times_after_close = *std::max_element(times_after_close.begin(), times_after_close.end());
        double avg_times_after_close = sum_times_after_close / average_times_in_queue.size();
        double sqr_times_after_close = std::inner_product(times_after_close.begin(), times_after_close.end(), times_after_close.begin(), 0.0);
        double std_times_after_close = std::sqrt(sqr_times_after_close / times_after_close.size() - avg_times_after_close * avg_times_after_close);

        double min_server_utilization = std::numeric_limits<double>::max();
        double max_server_utilization = std::numeric_limits<double>::min();
        double sum_server_utilization = 0;
        double sqr_server_utilization = 0;

        double min_average_length = std::numeric_limits<double>::max();
        double max_average_length = std::numeric_limits<double>::min();
        double sum_average_length = 0;
        double sqr_average_length = 0;

        for (int i = 0; i < num_iterations; i++) {
            double temp_avg_server_utilization = 0;
            double temp_avg_average_length = 0;

            // Find the average server utilization of the queues;
            for (int j = 0; j < num_servers; j++) {
                double current_utilization = average_server_utilizations.at(i)->at(j);
                VLOG(2) << "Server " << j+1 << " utilization: " << current_utilization;
                temp_avg_server_utilization += current_utilization;

                double current_average_length = average_length_of_queues.at(i)->at(j);
                VLOG(2) << "Server " << j+1 << " average length: " << current_average_length;
                temp_avg_average_length += current_average_length;
                temp_avg_average_length = temp_avg_average_length / num_servers;
            }
            delete average_server_utilizations.at(i);
            delete average_length_of_queues.at(i);

            temp_avg_server_utilization = temp_avg_server_utilization / num_servers;
            sum_server_utilization += temp_avg_server_utilization;
            if (temp_avg_server_utilization > max_server_utilization) max_server_utilization = temp_avg_server_utilization;
            if (temp_avg_server_utilization < min_server_utilization) min_server_utilization = temp_avg_server_utilization;
            sqr_server_utilization += temp_avg_server_utilization * temp_avg_server_utilization;

            temp_avg_average_length = temp_avg_average_length / num_servers;
            sum_average_length += temp_avg_average_length;
            if (temp_avg_average_length > max_average_length) max_average_length = temp_avg_average_length;
            if (temp_avg_average_length < min_average_length) min_average_length = temp_avg_average_length;
            sqr_average_length += temp_avg_average_length * temp_avg_average_length;
        }
        average_server_utilizations.clear();
        average_length_of_queues.clear();

        double avg_server_utilization = sum_server_utilization / num_iterations;
        double std_server_utilization = std::sqrt(sqr_server_utilization / num_iterations - avg_server_utilization * avg_server_utilization);

        double avg_average_length = sum_average_length / (num_servers * num_iterations);
        double std_average_length = std::sqrt(sqr_average_length / (num_servers * num_iterations) - avg_average_length * avg_average_length);

        outfile << "------------------------------------------" << std::endl;
        outfile << "---------------- " << num_servers << " SERVERS" << " ---------------" << std::endl;
        outfile << "------------------------------------------" << std::endl;
        outfile << std::endl;

        outfile << "-----------------------------------------" << std::endl;
        outfile << "------- AVERAGE SERVER UTILIZATION ------" << std::endl;
        outfile << "-----------------------------------------" << std::endl;
        outfile << "Min server utiliation: " << min_server_utilization << std::endl;
        outfile << "Max server utiliation: " << max_server_utilization << std::endl;
        outfile << "Average server utilization: " << avg_server_utilization << std::endl;
        outfile << "Standard deviation of server utilization: " << std_server_utilization << std::endl;
        outfile << std::endl;

        outfile << "-----------------------------------------" << std::endl;
        outfile << "---------- AVERAGE QUEUE LENGTH ---------" << std::endl;
        outfile << "-----------------------------------------" << std::endl;
        outfile << "Min queue length: " << min_average_length << std::endl;
        outfile << "Max queue length: " << max_average_length << std::endl;
        outfile << "Average queue length: " << avg_average_length << std::endl;
        outfile << "Standard deviation of queue length: " << std_average_length << std::endl;
        outfile << std::endl;

        outfile << "-----------------------------------------" << std::endl;
        outfile << "---------- AVERAGE QUEUE TIMES ----------" << std::endl;
        outfile << "-----------------------------------------" << std::endl;
        outfile << "Min of average queue times: " << min_average_queue_time << std::endl;
        outfile << "Max of average queue times: " << max_average_queue_time << std::endl;
        outfile << "Mean of average queue times: " << avg_average_queue_time << std::endl;
        outfile << "Standard deviation of average queue times: " << std_average_queue_time << std::endl;
        outfile << std::endl;

        outfile << "------------------------------------------" << std::endl;
        outfile << "-------- AVERAGE TIME AFTER CLOSE --------" << std::endl;
        outfile << "------------------------------------------" << std::endl;
        outfile << "Min of times after close: " << min_times_after_close << std::endl;
        outfile << "Max of times after close: " << max_times_after_close << std::endl;
        outfile << "Mean of times after close: " << avg_times_after_close << std::endl;
        outfile << "Standard deviation of times after close: " << std_times_after_close << std::endl;
        outfile << std::endl;
        outfile << std::endl;
        outfile << std::endl;

    }
    outfile.close();
}