// Copyright 2014 BVLC and contributors.

#include <boost/math/special_functions/next.hpp>
#include <boost/random.hpp>

#include <limits>

#include "caffe/common.hpp"
#include "caffe/util/math_functions.hpp"
#include "caffe/util/rng.hpp"

static const clblasOrder order = clblasColumnMajor;
#define pi 3.1415926

namespace caffe {

template<>
void caffe_cpu_gemm<float>(const CBLAS_TRANSPOSE TransA,
    const CBLAS_TRANSPOSE TransB, const int M, const int N, const int K,
    const float alpha, const float* A, const float* B, const float beta,
    float* C) {
  int lda = (TransA == CblasNoTrans) ? K : M;
  int ldb = (TransB == CblasNoTrans) ? N : K;
  cblas_sgemm(CblasRowMajor, TransA, TransB, M, N, K, alpha, A, lda, B,
      ldb, beta, C, N);
}

template<>
void caffe_cpu_gemm<double>(const CBLAS_TRANSPOSE TransA,
    const CBLAS_TRANSPOSE TransB, const int M, const int N, const int K,
    const double alpha, const double* A, const double* B, const double beta,
    double* C) {
  int lda = (TransA == CblasNoTrans) ? K : M;
  int ldb = (TransB == CblasNoTrans) ? N : K;
  cblas_dgemm(CblasRowMajor, TransA, TransB, M, N, K, alpha, A, lda, B,
      ldb, beta, C, N);
}

template <>
void caffe_gpu_gemm<float>(const CBLAS_TRANSPOSE TransA,
    const CBLAS_TRANSPOSE TransB, const int M, const int N, const int K,
    const float alpha, const float* A, const float* B, const float beta,
    float* C) {
    clblasTranspose transA = (TransA == CblasNoTrans)? clblasNoTrans : clblasTrans;
    clblasTranspose transB = (TransB == CblasNoTrans)? clblasNoTrans : clblasTrans;
    int lda = (TransA == CblasNoTrans) ? K : M;
    int ldb = (TransB == CblasNoTrans) ? N : K;
    int ldc = N;
    //AMDBLAS_CHECK( clAmdBlasSgemm(amdDevice.col, transB, transA, N, M, K, (cl_float)alpha, (cl_mem)B, ldb, (cl_mem)A, lda, (cl_float)beta, (cl_mem)C, ldc, 1, &(amdDevice.CommandQueue), 0, NULL, NULL) );
    AMDBLAS_CHECK( clblasSgemm(amdDevice.col, transB, transA, N, M, K, (cl_float)alpha, (cl_mem)B, 0, ldb, (cl_mem)A, 0, lda, (cl_float)beta, (cl_mem)C, 0, ldc, 1, &(amdDevice.CommandQueue), 0, NULL, NULL) );
}

template <>
void caffe_gpu_gemm<double>(const CBLAS_TRANSPOSE TransA,
    const CBLAS_TRANSPOSE TransB, const int M, const int N, const int K,
    const double alpha, const double* A, const double* B, const double beta,
    double* C) {
    clblasTranspose transA = (TransA == CblasNoTrans)? clblasNoTrans : clblasTrans;
    clblasTranspose transB = (TransB == CblasNoTrans)? clblasNoTrans : clblasTrans;
    int lda = (TransA == CblasNoTrans) ? K : M;
    int ldb = (TransB == CblasNoTrans) ? N : K;
    int ldc = N;
    AMDBLAS_CHECK( clblasDgemm(amdDevice.col, transB, transA, N, M, K, (cl_float)alpha, (cl_mem)B, 0, ldb, (cl_mem)A, 0, lda, (cl_float)beta, (cl_mem)C, 0, ldc, 1, &(amdDevice.CommandQueue), 0, NULL, NULL) );
}

template <>
cl_event caffe_gpu_gemm_ex<float>(const CBLAS_TRANSPOSE TransA,
    const CBLAS_TRANSPOSE TransB, const int M, const int N, const int K,
    const float alpha, const float* A,const int offA, const float* B, const int offB, const float beta, float* C, const int offC) {
    cl_event event;
    clblasTranspose transA = (TransA == CblasNoTrans)? clblasNoTrans : clblasTrans;
    clblasTranspose transB = (TransB == CblasNoTrans)? clblasNoTrans : clblasTrans;
    int lda = (TransA == CblasNoTrans) ? K : M;
    int ldb = (TransB == CblasNoTrans) ? N : K;
    int ldc = N;
    AMDBLAS_CHECK( clblasSgemm(amdDevice.col, transB, transA, N, M, K, (cl_float)alpha, (cl_mem)B, offB, ldb, (cl_mem)A, offA, lda, (cl_float)beta, (cl_mem)C, offC, ldc, 1, &(amdDevice.CommandQueue), 0, NULL, &event) );
    return event;
}

template <>
cl_event caffe_gpu_gemm_ex<double>(const CBLAS_TRANSPOSE TransA,
    const CBLAS_TRANSPOSE TransB, const int M, const int N, const int K,
    const double alpha, const double* A,const int offA, const double* B, const int offB, const double beta, double* C, const int offC) {
    cl_event event;
    clblasTranspose transA = (TransA == CblasNoTrans)? clblasNoTrans : clblasTrans;
    clblasTranspose transB = (TransB == CblasNoTrans)? clblasNoTrans : clblasTrans;
    int lda = (TransA == CblasNoTrans) ? K : M;
    int ldb = (TransB == CblasNoTrans) ? N : K;
    int ldc = N;
    AMDBLAS_CHECK( clblasDgemm(amdDevice.col, transB, transA, N, M, K, (cl_float)alpha, (cl_mem)B, offB, ldb, (cl_mem)A, offA, lda, (cl_float)beta, (cl_mem)C, offC, ldc, 1, &(amdDevice.CommandQueue), 0, NULL, &event) );
    return event;
}

template <>
cl_event caffe_gpu_gemmex<float>(cl_command_queue *queue, const CBLAS_TRANSPOSE TransA,
    const CBLAS_TRANSPOSE TransB, const int M, const int N, const int K,
    const float alpha, const float* A,const int offA, const float* B, const int offB, const float beta, float* C, const int offC) {
    cl_event event;
    clblasTranspose transA = (TransA == CblasNoTrans)? clblasNoTrans : clblasTrans;
    clblasTranspose transB = (TransB == CblasNoTrans)? clblasNoTrans : clblasTrans;
    int lda = (TransA == CblasNoTrans) ? K : M;
    int ldb = (TransB == CblasNoTrans) ? N : K;
    int ldc = N;
    //AMDBLAS_CHECK( clAmdBlasSgemmEx(amdDevice.col, transB, transA, N, M, K, (cl_float)alpha, (cl_mem)B, offB, ldb, (cl_mem)A, offA, lda, (cl_float)beta, (cl_mem)C, offC, ldc, 1, queue, 0, NULL, NULL) );
    AMDBLAS_CHECK( clblasSgemm(amdDevice.col, transB, transA, N, M, K, (cl_float)alpha, (cl_mem)B, offB, ldb, (cl_mem)A, offA, lda, (cl_float)beta, (cl_mem)C, offC, ldc, 1, queue, 0, NULL, &event) );
    return event;
 }

template <>
cl_event caffe_gpu_gemmex<double>(cl_command_queue *queue, const CBLAS_TRANSPOSE TransA,
    const CBLAS_TRANSPOSE TransB, const int M, const int N, const int K,
    const double alpha, const double* A,const int offA, const double* B, const int offB, const double beta, double* C, const int offC) {
    cl_event event;
    clblasTranspose transA = (TransA == CblasNoTrans)? clblasNoTrans : clblasTrans;
    clblasTranspose transB = (TransB == CblasNoTrans)? clblasNoTrans : clblasTrans;
    int lda = (TransA == CblasNoTrans) ? K : M;
    int ldb = (TransB == CblasNoTrans) ? N : K;
    int ldc = N;
    //AMDBLAS_CHECK( clAmdBlasSgemmEx(amdDevice.col, transB, transA, N, M, K, (cl_float)alpha, (cl_mem)B, offB, ldb, (cl_mem)A, offA, lda, (cl_float)beta, (cl_mem)C, offC, ldc, 1, queue, 0, NULL, NULL) );
    AMDBLAS_CHECK( clblasDgemm(amdDevice.col, transB, transA, N, M, K, (cl_float)alpha, (cl_mem)B, offB, ldb, (cl_mem)A, offA, lda, (cl_float)beta, (cl_mem)C, offC, ldc, 1, queue, 0, NULL, &event) );
    return event;
}

template <>
void caffe_cpu_gemv<float>(const CBLAS_TRANSPOSE TransA, const int M,
    const int N, const float alpha, const float* A, const float* x,
    const float beta, float* y) {
  cblas_sgemv(CblasRowMajor, TransA, M, N, alpha, A, N, x, 1, beta, y, 1);
}

template <>
void caffe_cpu_gemv<double>(const CBLAS_TRANSPOSE TransA, const int M,
    const int N, const double alpha, const double* A, const double* x,
    const double beta, double* y) {
  cblas_dgemv(CblasRowMajor, TransA, M, N, alpha, A, N, x, 1, beta, y, 1);
}

template <>
void caffe_gpu_gemvv<float>(const CBLAS_TRANSPOSE TransA, const int M,
    const int N, const float alpha, const float* A, size_t offA, int lda,
    const float* x, size_t offx, const float beta, int incx,
    float* y, size_t offy, int incy) {
    clblasTranspose transA = (TransA == CblasNoTrans)? clblasNoTrans : clblasTrans;
    AMDBLAS_CHECK( clblasSgemv(amdDevice.row, transA,
                                  M, N, (cl_float)alpha, (cl_mem)A, offA, lda,
                                  (cl_mem)x, offx, incx, (cl_float)beta,
                                  (cl_mem)y, offy, incy,
                                  1, &(amdDevice.CommandQueue), 0, NULL, NULL) );
}

template <>
void caffe_gpu_gemvv<double>(const CBLAS_TRANSPOSE TransA, const int M,
    const int N, const double alpha, const double* A, size_t offA, int lda,
    const double* x, size_t offx, const double beta, int incx,
    double* y, size_t offy, int incy) {
    clblasTranspose transA = (TransA == CblasNoTrans)? clblasNoTrans : clblasTrans;
    AMDBLAS_CHECK( clblasSgemv(amdDevice.row, transA, M, N, (cl_double)alpha, (cl_mem)A, offA, lda, (cl_mem)x, offx, incx, (cl_double)beta, (cl_mem)y, offy, incy, 1, &(amdDevice.CommandQueue), 0, NULL, NULL) );

}

template <>
void caffe_gpu_gemv<float>(const CBLAS_TRANSPOSE TransA, const int M,
    const int N, const float alpha, const float* A, const float* x,
    const float beta, float* y) {
}

template <>
void caffe_gpu_gemv<double>(const CBLAS_TRANSPOSE TransA, const int M,
    const int N, const double alpha, const double* A, const double* x,
    const double beta, double* y) {
}

template <>
void caffe_axpy<float>(const int N, const float alpha, const float* X,
    float* Y) { cblas_saxpy(N, alpha, X, 1, Y, 1); }

template <>
void caffe_axpy<double>(const int N, const double alpha, const double* X,
    double* Y) { cblas_daxpy(N, alpha, X, 1, Y, 1); }

template <>
void caffe_gpu_axpy<float>(const int N, const float alpha, const float* X,
    float* Y) {
    AMDBLAS_CHECK( clblasSaxpy(N, alpha, (cl_mem)X, 0, 1, (cl_mem)Y, 0, 1, 1, &(amdDevice.CommandQueue),0, NULL, NULL) );
}

template <>
void caffe_gpu_axpy<double>(const int N, const double alpha, const double* X,
    double* Y) {
    AMDBLAS_CHECK( clblasDaxpy(N, alpha, (cl_mem)X, 0, 1, (cl_mem)Y, 0, 1, 1, &(amdDevice.CommandQueue),0, NULL, NULL) );
}

template <>
void caffe_set(const int N, const float alpha, float* Y) {
  if (alpha == 0) {
    memset(Y, 0, sizeof(float) * N);
    return;
  }
  for (int i = 0; i < N; ++i) {
    Y[i] = alpha;
  }
}

template <>
void caffe_set(const int N, const double alpha, double* Y) {
  if (alpha == 0) {
    memset(Y, 0, sizeof(double) * N);
    return;
  }
  for (int i = 0; i < N; ++i) {
    Y[i] = alpha;
  }
}

template <>
void caffe_add_scalar(const int N, const float alpha, float* Y) {
  for (int i = 0; i < N; ++i) {
    Y[i] += alpha;
  }
}

template <>
void caffe_add_scalar(const int N, const double alpha, double* Y) {
  for (int i = 0; i < N; ++i) {
    Y[i] += alpha;
  }
}

template <>
void caffe_copy<float>(const int N, const float* X, float* Y) {
  cblas_scopy(N, X, 1, Y, 1);
}

template <>
void caffe_copy<double>(const int N, const double* X, double* Y) {
  cblas_dcopy(N, X, 1, Y, 1);
}

template <>
void caffe_gpu_copy<float>(const int N, const float* X, float* Y) {
  AMDBLAS_CHECK( clblasScopy( N, (cl_mem)X, 0,1, (cl_mem)Y, 0, 1, 1, &(amdDevice.CommandQueue), 0, NULL, NULL) );

}

template <>
void caffe_gpu_copy<double>(const int N, const double* X, double* Y) {
  AMDBLAS_CHECK( clblasDcopy( N, (cl_mem)X, 0,1, (cl_mem)Y, 0, 1, 1, &(amdDevice.CommandQueue), 0, NULL, NULL) );
}

template <>
void caffe_scal<float>(const int N, const float alpha, float *X) {
  cblas_sscal(N, alpha, X, 1);
}

template <>
void caffe_scal<double>(const int N, const double alpha, double *X) {
  cblas_dscal(N, alpha, X, 1);
}

template <>
void caffe_gpu_scal<float>(const int N, const float alpha, float *X) {
   AMDBLAS_CHECK(clblasSscal(N, alpha, (cl_mem)X, 0, 1, 1, &(amdDevice.CommandQueue), 0, NULL, NULL));
}

template <>
void caffe_gpu_scal<double>(const int N, const double alpha, double *X) {
  AMDBLAS_CHECK(clblasDscal(N, alpha, (cl_mem)X, 0, 1, 1, &(amdDevice.CommandQueue), 0, NULL, NULL));
}

template <>
void caffe_gpu_axpby<float>(const int N, const float alpha, const float* X,
    const float beta, float* Y) {
  caffe_gpu_scal<float>(N, beta, Y);
  caffe_gpu_axpy<float>(N, alpha, X, Y);
}

template <>
void caffe_gpu_axpby<double>(const int N, const double alpha, const double* X,
    const double beta, double* Y) {
  caffe_gpu_scal<double>(N, beta, Y);
  caffe_gpu_axpy<double>(N, alpha, X, Y);
}

template <>
void caffe_cpu_axpby<float>(const int N, const float alpha, const float* X,
                            const float beta, float* Y) {
  cblas_saxpby(N, alpha, X, 1, beta, Y, 1);
}

template <>
void caffe_cpu_axpby<double>(const int N, const double alpha, const double* X,
                             const double beta, double* Y) {
  cblas_daxpby(N, alpha, X, 1, beta, Y, 1);
}

template <>
void caffe_add<float>(const int n, const float* a, const float* b,
    float* y) {
  vsAdd(n, a, b, y);
}

template <>
void caffe_add<double>(const int n, const double* a, const double* b,
    double* y) {
  vdAdd(n, a, b, y);
}

template <>
void caffe_sub<float>(const int n, const float* a, const float* b,
    float* y) {
  vsSub(n, a, b, y);
}

template <>
void caffe_sub<double>(const int n, const double* a, const double* b,
    double* y) {
  vdSub(n, a, b, y);
}

template <>
void caffe_mul<float>(const int n, const float* a, const float* b,
    float* y) {
  vsMul(n, a, b, y);
}

template <>
void caffe_mul<double>(const int n, const double* a, const double* b,
    double* y) {
  vdMul(n, a, b, y);
}

template <>
void caffe_div<float>(const int n, const float* a, const float* b,
    float* y) {
  vsDiv(n, a, b, y);
}

template <>
void caffe_div<double>(const int n, const double* a, const double* b,
    double* y) {
  vdDiv(n, a, b, y);
}

template <>
void caffe_powx<float>(const int n, const float* a, const float b,
    float* y) {
  vsPowx(n, a, b, y);
}

template <>
void caffe_powx<double>(const int n, const double* a, const double b,
    double* y) {
  vdPowx(n, a, b, y);
}

template <>
void caffe_sqr<float>(const int n, const float* a, float* y) {
  vsSqr(n, a, y);
}

template <>
void caffe_sqr<double>(const int n, const double* a, double* y) {
  vdSqr(n, a, y);
}

template <>
void caffe_exp<float>(const int n, const float* a, float* y) {
  vsExp(n, a, y);
}

template <>
void caffe_exp<double>(const int n, const double* a, double* y) {
  vdExp(n, a, y);
}

unsigned int caffe_rng_rand() {
  return (*caffe_rng())();
}

template <typename Dtype>
Dtype caffe_nextafter(const Dtype b) {
  return boost::math::nextafter<Dtype>(
      b, std::numeric_limits<Dtype>::max());
}

template
float caffe_nextafter(const float b);

template
double caffe_nextafter(const double b);

template <typename Dtype>
void caffe_rng_uniform(const int n, const Dtype a, const Dtype b, Dtype* r) {
  CHECK_GE(n, 0);
  CHECK(r);
  CHECK_LE(a, b);
  boost::uniform_real<Dtype> random_distribution(a, caffe_nextafter<Dtype>(b));
  boost::variate_generator<caffe::rng_t*, boost::uniform_real<Dtype> >
      variate_generator(caffe_rng(), random_distribution);
  for (int i = 0; i < n; ++i) {
    r[i] = variate_generator();
  }

  //LOG(INFO) << "caffe_rng_uniform";
}

template
void caffe_rng_uniform<float>(const int n, const float a, const float b,
                              float* r);

template
void caffe_rng_uniform<double>(const int n, const double a, const double b,
                               double* r);

template <typename Dtype>
void caffe_rng_gaussian(const int n, const Dtype a,
                        const Dtype sigma, Dtype* r) {
  CHECK_GE(n, 0);
  CHECK(r);
  CHECK_GT(sigma, 0);
  boost::normal_distribution<Dtype> random_distribution(a, sigma);
  boost::variate_generator<caffe::rng_t*, boost::normal_distribution<Dtype> >
      variate_generator(caffe_rng(), random_distribution);
      //variate_generator(37, random_distribution);
  for (int i = 0; i < n; ++i) {
    r[i] = variate_generator();
  }
  //LOG(INFO) << "caffe_rng_guassian";
}

template
void caffe_rng_gaussian<float>(const int n, const float mu,
                               const float sigma, float* r);

template
void caffe_rng_gaussian<double>(const int n, const double mu,
                                const double sigma, double* r);

template <typename Dtype>
void caffe_rng_bernoulli(const int n, const Dtype p, int* r) {
  CHECK_GE(n, 0);
  CHECK(r);
  CHECK_GE(p, 0);
  CHECK_LE(p, 1);
  boost::bernoulli_distribution<Dtype> random_distribution(p);
  boost::variate_generator<caffe::rng_t*, boost::bernoulli_distribution<Dtype> >
      variate_generator(caffe_rng(), random_distribution);
  for (int i = 0; i < n; ++i) {
    r[i] = variate_generator();
  }
  //LOG(INFO) << "caffe_rng_bernoulli";
}

template
void caffe_rng_bernoulli<double>(const int n, const double p, int* r);

template
void caffe_rng_bernoulli<float>(const int n, const float p, int* r);
//
template <>
float caffe_cpu_dot<float>(const int n, const float* x, const float* y) {
  return cblas_sdot(n, x, 1, y, 1);
}

template <>
double caffe_cpu_dot<double>(const int n, const double* x, const double* y) {
  return cblas_ddot(n, x, 1, y, 1);
}

template <>
void caffe_gpu_dot<float>(const int n, const float* x, const float* y,
    float* out) {
  //need to pass in scratchBuff
  //AMDBLAS_CHECK(clAmdBlasSdot(n, out, 0, x, 0, 1, y, 0, 1, scratch_buf, 1, &(amdDevice.CommandQueue), 0, NULL, NULL));
}

template <>
void caffe_gpu_dot<double>(const int n, const double* x, const double* y,
    double * out) {
  //need to pass in scratchBuff
  //AMDBLAS_CHECK(clAmdBlasDdot(n, out, 0, x, 0, 1, y, 0, 1, scratch_buf, 1, &(amdDevice.CommandQueue), 0, NULL, NULL));
}

template <>
int caffe_cpu_hamming_distance<float>(const int n, const float* x,
                                  const float* y) {
  int dist = 0;
  for (int i = 0; i < n; ++i) {
    dist += __builtin_popcount(static_cast<uint32_t>(x[i]) ^
                               static_cast<uint32_t>(y[i]));
  }
  return dist;
}

template <>
int caffe_cpu_hamming_distance<double>(const int n, const double* x,
                                   const double* y) {
  int dist = 0;
  for (int i = 0; i < n; ++i) {
    dist += __builtin_popcountl(static_cast<uint64_t>(x[i]) ^
                                static_cast<uint64_t>(y[i]));
  }
  return dist;
}

template <>
float caffe_cpu_asum<float>(const int n, const float* x) {
  return cblas_sasum(n, x, 1);
}

template <>
double caffe_cpu_asum<double>(const int n, const double* x) {
  return cblas_dasum(n, x, 1);
}

template <>
void caffe_gpu_asum<float>(const int n, const float* x, float* y) {
}

template <>
void caffe_gpu_asum<double>(const int n, const double* x, double* y) {
}

INSTANTIATE_CAFFE_CPU_UNARY_FUNC(sign);
INSTANTIATE_CAFFE_CPU_UNARY_FUNC(sgnbit);
INSTANTIATE_CAFFE_CPU_UNARY_FUNC(fabs);

template <>
void caffe_cpu_scale<float>(const int n, const float alpha, const float *x,
                            float* y) {
  cblas_scopy(n, x, 1, y, 1);
  cblas_sscal(n, alpha, y, 1);
}

template <>
void caffe_cpu_scale<double>(const int n, const double alpha, const double *x,
                             double* y) {
  cblas_dcopy(n, x, 1, y, 1);
  cblas_dscal(n, alpha, y, 1);
}

template <>
void caffe_gpu_scale<float>(const int n, const float alpha, const float *x,
                            float* y) {
}

template <>
void caffe_gpu_scale<double>(const int n, const double alpha, const double *x,
                             double* y) {
}

template <typename Dtype>
void set_kernel(const int n, const Dtype alpha, Dtype* y) {
}

template <>
void caffe_gpu_set(const int N, const float alpha, float* Y) {
  if (alpha == 0) {
    return;
  }
}

template <>
void caffe_gpu_set(const int N, const double alpha, double* Y) {
  if (alpha == 0) {
    return;
  }
}

template <typename Dtype>
void add_scalar_kernel(const int n, const Dtype alpha, Dtype* y) {
}

template <>
void caffe_gpu_add_scalar(const int N, const float alpha, float* Y) {
}

template <>
void caffe_gpu_add_scalar(const int N, const double alpha, double* Y) {
}

template <typename Dtype>
void mul_kernel(const int n, const Dtype* a,
    const Dtype* b, Dtype* y) {
}

template <>
void caffe_gpu_mul<float>(const int N, const float* a,
    const float* b, float* y) {
}

template <>
void caffe_gpu_mul<double>(const int N, const double* a,
    const double* b, double* y) {
}

template <typename Dtype>
void div_kernel(const int n, const Dtype* a,
    const Dtype* b, Dtype* y) {
}

template <>
void caffe_gpu_div<float>(const int N, const float* a,
    const float* b, float* y) {
}

template <>
void caffe_gpu_div<double>(const int N, const double* a,
    const double* b, double* y) {
}

template <typename Dtype>
void powx_kernel(const int n, const Dtype* a,
    const Dtype alpha, Dtype* y) {
}

template <>
void caffe_gpu_powx<float>(const int N, const float* a,
    const float alpha, float* y) {
}

template <>
void caffe_gpu_powx<double>(const int N, const double* a,
    const double alpha, double* y) {
}

void popc_kernel(const int n, const float* a,
    const float* b, uint8_t* y) {
}

void popcll_kernel(const int n, const double* a,
    const double* b, uint8_t* y) {
}

template <>
uint32_t caffe_gpu_hamming_distance<float>(const int n, const float* x,
                                  const float* y) {
}

template <>
uint32_t caffe_gpu_hamming_distance<double>(const int n, const double* x,
                                   const double* y) {
}

void caffe_gpu_rng_uniform(const int n, unsigned int* r) {
}

template <>
void caffe_gpu_rng_uniform<float>(const int n, const float a, const float b,
                                  float* r) {
}
template <>
void caffe_gpu_rng_uniform<double>(const int n, const double a, const double b,
                                   double* r) {
}

template <>
void caffe_gpu_rng_gaussian(const int n, const float mu, const float sigma,
                            float* r) {
}

template <>
void caffe_gpu_rng_gaussian(const int n, const double mu, const double sigma,
                            double* r) {
}
}  // namespace caffe


/*
Brian Staber (brian.staber@gmail.com)
*/

#include "Epetra_ConfigDefs.h"
#ifdef HAVE_MPI
#include "mpi.h"
#include "Epetra_MpiComm.h"
#else
#include "Epetra_SerialComm.h"
#endif

#include "Teuchos_CommandLineProcessor.hpp"
#include "Teuchos_StandardCatchMacros.hpp"
#include "Teuchos_ParameterList.hpp"
#include "Teuchos_XMLParameterListCoreHelpers.hpp"

#include "neumannInnerSurface_StochasticPolyconvexHGO.hpp"
#include <boost/math/special_functions/gamma.hpp>

#include "shinozukapp.hpp"

int main(int argc, char *argv[]){

    std::string    xmlInFileName = "";
    std::string    extraXmlFile = "";
    std::string    xmlOutFileName = "paramList.out";

    Teuchos::CommandLineProcessor  clp(false);
    clp.setOption("xml-in-file",&xmlInFileName,"The XML file to read into a parameter list");
    clp.setDocString("TO DO.");

    Teuchos::CommandLineProcessor::EParseCommandLineReturn
    parse_return = clp.parse(argc,argv);
    if( parse_return != Teuchos::CommandLineProcessor::PARSE_SUCCESSFUL ) {
        std::cout << "\nEnd Result: TEST FAILED" << std::endl;
        return parse_return;
    }

#ifdef HAVE_MPI
    MPI_Init(&argc, &argv);
    Epetra_MpiComm Comm(MPI_COMM_WORLD);
#else
    Epetra_SerialComm Comm;
#endif

    Teuchos::RCP<Teuchos::ParameterList> paramList = Teuchos::rcp(new Teuchos::ParameterList);
    if(xmlInFileName.length()) {
        Teuchos::updateParametersFromXmlFile(xmlInFileName, inoutArg(*paramList));
    }

    if (Comm.MyPID()==0){
        paramList->print(std::cout,2,true,true);
    }

    Teuchos::RCP<neumannInnerSurface_StochasticPolyconvexHGO> interface
    = Teuchos::rcp(new neumannInnerSurface_StochasticPolyconvexHGO(Comm,*paramList));

    std::ifstream parameters_file_1, parameters_file_2, parameters_file_3, parameters_file_4;

    std::string path1 = "/Users/Brian/Documents/Thesis/Trilinos_results/arteries/gmrf_neumann/a100_gamma3_delta010/";
    parameters_file_1.open(path1+"w1.txt");
    parameters_file_2.open(path1+"w2.txt");
    parameters_file_3.open(path1+"w3.txt");
    parameters_file_4.open(path1+"w4.txt");

    unsigned int n_cells_p1_med = 297828;
    unsigned int n_nodes_p1_med = 58464;

    std::string path = "/Users/Brian/Documents/Thesis/Trilinos/arteries/mesh/connectivity_p1_media.txt";
    interface->get_media(n_cells_p1_med,n_nodes_p1_med,path);

    if (parameters_file_1.is_open() && parameters_file_2.is_open() && parameters_file_3.is_open() && parameters_file_4.is_open()){

        for (unsigned nmc=0; nmc<1; ++nmc){
            for (int i=0; i<n_nodes_p1_med; ++i){
                parameters_file_1 >> interface->w1_gmrf(i);
                parameters_file_2 >> interface->w2_gmrf(i);
                parameters_file_3 >> interface->w3_gmrf(i);
                parameters_file_4 >> interface->w4_gmrf(i);
            }
        }
        Comm.Barrier();
        parameters_file_1.close();
        parameters_file_2.close();
        parameters_file_3.close();
        parameters_file_4.close();
    }
    else{
        std::cout << "Couldn't open one of the parameters_file.\n";
    }

    int e_gid;
    int n_local_cells = interface->Mesh->n_local_cells;
    int n_gauss_cells = interface->Mesh->n_gauss_cells;
    std::vector<int> local_gauss_points(n_local_cells*n_gauss_cells);
    for (unsigned int e_lid=0; e_lid<n_local_cells; ++e_lid){
        e_gid = interface->Mesh->local_cells[e_lid];
        for (unsigned int gp=0; gp<n_gauss_cells; ++gp){
            local_gauss_points[e_lid*n_gauss_cells+gp] = e_gid*n_gauss_cells+gp;
        }

    }
    Epetra_Map GaussMap(-1,n_local_cells*n_gauss_cells,&local_gauss_points[0],0,Comm);

    Epetra_Vector mu1_gmrf(GaussMap);
    Epetra_Vector mu2_gmrf(GaussMap);
    Epetra_Vector mu3_gmrf(GaussMap);
    Epetra_Vector mu4_gmrf(GaussMap);
    Epetra_Vector x_coord(GaussMap);
    Epetra_Vector y_coord(GaussMap);
    Epetra_Vector z_coord(GaussMap);

    int node;
    double xi, eta, zeta;
    Epetra_SerialDenseMatrix matrix_X(3,interface->Mesh->el_type);
    Epetra_SerialDenseVector vector_X(3);
    Epetra_SerialDenseVector N(interface->Mesh->el_type);
    for (unsigned int e_lid=0; e_lid<n_local_cells; ++e_lid){
        e_gid = interface->Mesh->local_cells[e_lid];
        for (int inode=0; inode<interface->Mesh->el_type; ++inode){
            node = interface->Mesh->cells_nodes[interface->Mesh->el_type*e_gid+inode];
            matrix_X(0,inode) = interface->Mesh->nodes_coord[3*node+0];
            matrix_X(1,inode) = interface->Mesh->nodes_coord[3*node+1];
            matrix_X(2,inode) = interface->Mesh->nodes_coord[3*node+2];
        }
        for (unsigned int gp=0; gp<n_gauss_cells; ++gp){
            xi   = interface->Mesh->xi_cells[gp];
            eta  = interface->Mesh->eta_cells[gp];
            zeta = interface->Mesh->zeta_cells[gp];
            tetra10::shape_functions(N,xi,eta,zeta);
            vector_X.Multiply('N','N',1.0,matrix_X,N,0.0);
            interface->get_material_parameters(e_lid,gp);
            mu1_gmrf[GaussMap.LID(int(e_gid*n_gauss_cells+gp))] = interface->mu1;
            mu2_gmrf[GaussMap.LID(int(e_gid*n_gauss_cells+gp))] = interface->mu2;
            mu3_gmrf[GaussMap.LID(int(e_gid*n_gauss_cells+gp))] = interface->mu3;
            mu4_gmrf[GaussMap.LID(int(e_gid*n_gauss_cells+gp))] = interface->mu4;
            x_coord[GaussMap.LID(int(e_gid*n_gauss_cells+gp))] = vector_X(0);
            y_coord[GaussMap.LID(int(e_gid*n_gauss_cells+gp))] = vector_X(1);
            z_coord[GaussMap.LID(int(e_gid*n_gauss_cells+gp))] = vector_X(2);
         }
    }

    int error;
    int NumTargetElements = 0;
    if (Comm.MyPID()==0){
        NumTargetElements = interface->Mesh->n_cells*n_gauss_cells;
    }
    Epetra_Map MapOnRoot(-1,NumTargetElements,0,Comm);
    Epetra_Export ExportOnRoot(GaussMap,MapOnRoot);
    Epetra_MultiVector lhs_root(MapOnRoot,true);
    lhs_root.Export(mu1_gmrf,ExportOnRoot,Insert);
    std::string filename = "/Users/Brian/Documents/Thesis/Trilinos_results/arteries/boost_gmrf/gmrf_gauss/mu1_gmrf.mtx";
    error = EpetraExt::MultiVectorToMatrixMarketFile(filename.c_str(),lhs_root,0,0,false);

    lhs_root.PutScalar(0.0);
    lhs_root.Export(mu2_gmrf,ExportOnRoot,Insert);
    filename = "/Users/Brian/Documents/Thesis/Trilinos_results/arteries/boost_gmrf/gmrf_gauss/mu2_gmrf.mtx";
    error = EpetraExt::MultiVectorToMatrixMarketFile(filename.c_str(),lhs_root,0,0,false);

    lhs_root.PutScalar(0.0);
    lhs_root.Export(mu3_gmrf,ExportOnRoot,Insert);
    filename = "/Users/Brian/Documents/Thesis/Trilinos_results/arteries/boost_gmrf/gmrf_gauss/mu3_gmrf.mtx";
    error = EpetraExt::MultiVectorToMatrixMarketFile(filename.c_str(),lhs_root,0,0,false);

    lhs_root.PutScalar(0.0);
    lhs_root.Export(mu4_gmrf,ExportOnRoot,Insert);
    filename = "/Users/Brian/Documents/Thesis/Trilinos_results/arteries/boost_gmrf/gmrf_gauss/mu4_gmrf.mtx";
    error = EpetraExt::MultiVectorToMatrixMarketFile(filename.c_str(),lhs_root,0,0,false);

    lhs_root.PutScalar(0.0);
    lhs_root.Export(x_coord,ExportOnRoot,Insert);
    filename = "/Users/Brian/Documents/Thesis/Trilinos_results/arteries/boost_gmrf/gmrf_gauss/x_coord.mtx";
    error = EpetraExt::MultiVectorToMatrixMarketFile(filename.c_str(),lhs_root,0,0,false);

    lhs_root.PutScalar(0.0);
    lhs_root.Export(y_coord,ExportOnRoot,Insert);
    filename = "/Users/Brian/Documents/Thesis/Trilinos_results/arteries/boost_gmrf/gmrf_gauss/y_coord.mtx";
    error = EpetraExt::MultiVectorToMatrixMarketFile(filename.c_str(),lhs_root,0,0,false);

    lhs_root.PutScalar(0.0);
    lhs_root.Export(z_coord,ExportOnRoot,Insert);
    filename = "/Users/Brian/Documents/Thesis/Trilinos_results/arteries/boost_gmrf/gmrf_gauss/z_coord.mtx";
    error = EpetraExt::MultiVectorToMatrixMarketFile(filename.c_str(),lhs_root,0,0,false);

#ifdef HAVE_MPI
    MPI_Finalize();
#endif
    return 0;

}


/*
 * Rbpf.hpp
 *
 *  Rao Blackwellized Particle Filter (RBPF)
 *  We use the method of [1] where a RBPF is used to track a ball
 *  using a particle filter over the non-linear portions of
 *  the state (the models; e.g., was ball kicked?, is ball rolling?) and an
 *  Extended Kalman Filter (EKF) over the 'linear' portions (ball dynamics.)
 *
 *  This differs slightly from a traditional RBPF because the particles are
 *  over a discrete state space (the model graph).
 *
 *  For a visual understanding of the model graph, see [1], pg.4, fig.1
 *  The algorithm here follows the algorithm in [1], pg.8, table.1 with the
 *  exception that there is no particle filter over the robot positions.
 *
 *  How to use:
 *  To create a Rbpf, you'll need to instantiate this class and determine the
 *  structure of the discrete space (the model graph). Create models and add
 *  them to the internal modelGraph using addModel(), and set the transition
 *  probabilities using setTransProb(A,B,prob).  Note that the initial state
 *  and covariance sizes you choose when creating the Rbpf should correspond to
 *  the state sizes used in your model graph.
 *
 *  Note:
 *    When possible the notation here is based on:
 *      http://en.wikipedia.org/wiki/Kalman_filter
 *      http://en.wikipedia.org/wiki/Extended_Kalman_filter
 *      http://en.wikipedia.org/wiki/Particle_filter
 *      [1]: Map-Based Multiple Model Tracking of a Moving Object
 *           Authors: Cody Kwok and Dieter Fox.
 *           DOI: 10.1007/b106671
 *
 *  Author: Philip Rogers, Nov 19th 2009
 */

#ifndef RBPF_HPP_
#define RBPF_HPP_

#include <stdlib.h>
#include <iostream>
#include <boost/ptr_container/ptr_vector.hpp>
#include <LinearAlgebra.hpp>
#include "RbpfState.hpp"
#include "RbpfModel.hpp"
#include "RbpfModelGraph.hpp"

class Rbpf {
public:
    typedef boost::ptr_vector<RbpfState> ParticleVector;

    // X: initial state, (n x 1)
    // P: initial state covariance, (n x n)
    // k: the number of particles to be initialized.
    // Where n = size of Kalman Filter state
    Rbpf(const rbpf::VectorNd& _X, const rbpf::MatrixNNd& _P, size_t _k);

    ~Rbpf();

    // updates the filter with 2d observation [x;y], change in time dt, and no
    // control input
    void update(double x, double y, double dt);

    // updates the filter for observation Z, control input U, and change in time
    // dt
    // U: control input, (m x 1)
    // Z: measurement, (s x 1)
    // dt: change in time
    // Where m and s = size of the control and measurement input
    void update(const rbpf::VectorNd& U, const rbpf::VectorSd& Z, double dt);

    // reinitialize the parameters from the config files - should be called each
    // frame
    void initParams();

    void updateMultipleObs(double x[], double y[], double dt[], int numObs);

    // adds a model to the internal modelGraph
    void addModel(RbpfModel* model);

    // sets a transition probability in the modelGraph from model A to model B
    void setTransProb(int AIdx, int BIdx, double weight);

    // returns a pointer to the best particle state
    RbpfState* getBestFilterState();

    //   Used for printing a Rbpf to a stream
    friend std::ostream& operator<<(std::ostream& out, const Rbpf& rbpf);

    int k;                      // number of particles
    RbpfModelGraph modelGraph;  // Graph of models
    int n;                      // size of state vector

protected:
    ParticleVector particleVector;     // vector of particles, (k x 1)
    ParticleVector tmpParticleVector;  // vector of temp particles, (k*j x 1)

    // Evaluates the multivariate PDF of a centered (mean=0,0) 2D Gaussian dist.
    // X: point to be evaluated (2 x 1)
    // Sigma: covariance matrix (2 x 2)
    inline double gaussianPDF2D(const rbpf::VectorSd& X,
                                const rbpf::MatrixSSd& Sigma);

    // resample k particles from out, with respect to their weights
    // NOTE: in vector has its weight changed
    void resampleParticles(ParticleVector& in, ParticleVector& out, int);

public:
    EIGEN_MAKE_ALIGNED_OPERATOR_NEW
};

#endif /* RBPF_HPP_ */


#include <ctime>
#include <cmath>

#include <iostream>
#include <fstream>
#include <vector>
#include <algorithm>

#include <boost/unordered_map.hpp>
#include <boost/functional.hpp>
#include <boost/lexical_cast.hpp>
#include <boost/random/mersenne_twister.hpp>
#include <boost/algorithm/string/join.hpp>
# include <boost/interprocess/managed_shared_memory.hpp>
# include <boost/interprocess/allocators/allocator.hpp>
# include <boost/interprocess/managed_mapped_file.hpp>
#include <boost/interprocess/containers/vector.hpp>
#include <boost/math/distributions/binomial.hpp>

#include <Eigen/Dense>
#include <Eigen/Sparse>
#include "maybe_omp.h"
#include <tclap/CmdLine.h>

#include "model.h"
#include "propagator.h"
#include "param.h"
#include "neuralClasses.h"
#include "graphClasses.h"
#include "util.h"
#include "multinomial.h"
//#include "gradientCheck.h"

//#define EIGEN_DONT_PARALLELIZE

using namespace std;
using namespace TCLAP;
using namespace Eigen;
using namespace boost;
using namespace boost::random;

using namespace nplm;

namespace ip = boost::interprocess;
typedef unordered_map<Matrix<int,Dynamic,1>, double> vector_map;

typedef ip::allocator<int, ip::managed_mapped_file::segment_manager> intAllocator;
typedef ip::vector<int, intAllocator> vec;
typedef ip::allocator<vec, ip::managed_mapped_file::segment_manager> vecAllocator;

typedef long long int data_size_t; // training data can easily exceed 2G instances

int main(int argc, char** argv)
{
    ios::sync_with_stdio(false);
    bool use_mmap_file, randomize;
    param myParam;
    try {
      // program options //
      CmdLine cmd("Trains a two-layer neural probabilistic language model.", ' ' , "0.2\n",
          "Changes since V0.1: Addition of biases");

      // The options are printed in reverse order

      ValueArg<string> unigram_probs_file("", "unigram_probs_file", "Unigram model (deprecated and ignored)." , false, "", "string", cmd);

      ValueArg<int> num_threads("", "num_threads", "Number of threads. Default: maximum.", false, 0, "int", cmd);

      ValueArg<double> final_momentum("", "final_momentum", "Final value of momentum. Default: 0.9.", false, 0.9, "double", cmd);
      ValueArg<double> initial_momentum("", "initial_momentum", "Initial value of momentum. Default: 0.9.", false, 0.9, "double", cmd);
      ValueArg<bool> use_momentum("", "use_momentum", "Use momentum (hidden layer weights only). 1 = yes, 0 = no. Default: 0.", false, 0, "bool", cmd);

      ValueArg<double> normalization_init("", "normalization_init", "Initial normalization parameter. Default: 0.", false, 0.0, "double", cmd);
      ValueArg<bool> normalization("", "normalization", "Learn individual normalization factors during training. 1 = yes, 0 = no. Default: 0.", false, 0, "bool", cmd);

      ValueArg<bool> mmap_file("", "mmap_file", "Use memory mapped files. This is useful if the entire data cannot fit in memory. prepareNeuralLM can generate memory mapped files", false, 0, "bool", cmd);

      ValueArg<bool> arg_randomize("", "randomize", "Randomize training instances for better training. 1 = yes, 0 = no. Default: 1.", false, true, "bool", cmd);

      ValueArg<int> num_noise_samples("", "num_noise_samples", "Number of noise samples for noise-contrastive estimation. Default: 100.", false, 100, "int", cmd);

      ValueArg<double> L2_reg("", "L2_reg", "L2 regularization strength (hidden layer weights only). Default: 0.", false, 0.0, "double", cmd);

      ValueArg<double> learning_rate("", "learning_rate", "Learning rate for stochastic gradient ascent. Default: 1.", false, 1., "double", cmd);

      ValueArg<double> conditioning_constant("", "conditioning_constant", "Constant to condition the RMS of the expected square of the gradient in ADADELTA. Default: 10E-3.", false, 10E-3, "double", cmd);

      ValueArg<double> decay("", "decay", "Decay for ADADELTA. Default: 0.95", false, 0.95, "double", cmd);
      ValueArg<double> adagrad_epsilon("", "adagrad_epsilon", "Constant to initialize the L2 squared norm of the gradients with.\
          Default: 10E-3", false, 10E-3, "double", cmd);
      ValueArg<int> validation_minibatch_size("", "validation_minibatch_size", "Minibatch size for validation. Default: 64.", false, 64, "int", cmd);
      ValueArg<int> minibatch_size("", "minibatch_size", "Minibatch size (for training). Default: 1000.", false, 1000, "int", cmd);

      ValueArg<int> num_epochs("", "num_epochs", "Number of epochs. Default: 10.", false, 10, "int", cmd);

      ValueArg<double> init_range("", "init_range", "Maximum (of uniform) or standard deviation (of normal) for initialization. Default: 0.01", false, 0.01, "double", cmd);
      ValueArg<bool> init_normal("", "init_normal", "Initialize parameters from a normal distribution. 1 = normal, 0 = uniform. Default: 0.", false, 0, "bool", cmd);

      ValueArg<string> loss_function("", "loss_function", "Loss function (log, nce). Default: nce.", false, "nce", "string", cmd);
      ValueArg<string> activation_function("", "activation_function", "Activation function (identity, rectifier, tanh, hardtanh). Default: rectifier.", false, "rectifier", "string", cmd);
      ValueArg<int> num_hidden("", "num_hidden", "Number of hidden nodes. Default: 100.", false, 100, "int", cmd);
	  ValueArg<int> num_second_hidden("", "num_second_hidden", "Number of hidden nodes in the second hidden layer. Default: 100.", false, 100, "int", cmd);

      ValueArg<bool> share_embeddings("", "share_embeddings", "Share input and output embeddings. 1 = yes, 0 = no. Default: 0.", false, 0, "bool", cmd);
      ValueArg<int> output_embedding_dimension("", "output_embedding_dimension", "Number of output embedding dimensions. Default: 50.", false, 50, "int", cmd);
      ValueArg<int> input_embedding_dimension("", "input_embedding_dimension", "Number of input embedding dimensions. Default: 50.", false, 50, "int", cmd);
      ValueArg<int> embedding_dimension("", "embedding_dimension", "Number of input and output embedding dimensions. Default: none.", false, -1, "int", cmd);
	  ValueArg<string> input_embeddings_file("","input_embeddings_file", "Read the input embeddings from the specified file. Default: none", false,"","string",cmd);
	  ValueArg<int> context_vector_size("", "context_vector_size", "Size of the context vector. Default: 128.", false, 128, "int", cmd);
      ValueArg<int> vocab_size("", "vocab_size", "Vocabulary size. Default: auto.", false, 0, "int", cmd);
      ValueArg<int> input_vocab_size("", "input_vocab_size", "Vocabulary size. Default: auto.", false, 0, "int", cmd);
      ValueArg<int> output_vocab_size("", "output_vocab_size", "Vocabulary size. Default: auto.", false, 0, "int", cmd);
      ValueArg<int> ngram_size("", "ngram_size", "Size of n-grams. Default: auto.", false, 0, "int", cmd);

      ValueArg<string> model_prefix("", "model_prefix", "Prefix for output model files." , false, "", "string", cmd);
      ValueArg<string> words_file("", "words_file", "Vocabulary." , false, "", "string", cmd);
      ValueArg<string> parameter_update("", "parameter_update", "parameter update type.\n Stochastic Gradient Descent(SGD)\n \
          ADAGRAD(ADA)\n \
          ADADELTA(ADAD)" , false, "SGD", "string", cmd);
      ValueArg<string> input_words_file("", "input_words_file", "Vocabulary." , false, "", "string", cmd);
      ValueArg<string> output_words_file("", "output_words_file", "Vocabulary." , false, "", "string", cmd);
      ValueArg<string> validation_file("", "validation_file", "Validation data (one numberized example per line)." , false, "", "string", cmd);
      ValueArg<string> train_file("", "train_file", "Training data (one numberized example per line)." , true, "", "string", cmd);

      ValueArg<string> model_file("", "model_file", "Model file.", false, "", "string", cmd);
      ValueArg<double> hidden1_dropout_value("", "hidden1_dropout_value", "The probability of drop out for hidden layer 1.\
          Default: 0.9", false, 0.9, "double", cmd);
      ValueArg<double> hidden2_dropout_value("", "hidden2_dropout_value", "The probability of drop out for hidden layer 2.\
          Default: 0.9", false, 0.9, "double", cmd);
      ValueArg<double> hidden3_dropout_value("", "hidden3_dropout_value", "The probability of drop out for hidden layer 3.\
          Default: 0.9", false, 0.9, "double", cmd);
	  ValueArg<string> train_context_vectors_file("", "train_context_vectors_file", "Training context vectors file."
		  , false, "", "string", cmd);
	  ValueArg<string> validation_context_vectors_file("", "validation_context_vectors_file", "Validation context vectors file."
		  , false, "", "string", cmd);

      cmd.parse(argc, argv);

      // define program parameters //
      use_mmap_file = mmap_file.getValue();
      randomize = arg_randomize.getValue();
      myParam.model_file = model_file.getValue();
      myParam.train_file = train_file.getValue();
      myParam.validation_file = validation_file.getValue();
      myParam.input_words_file = input_words_file.getValue();
      myParam.output_words_file = output_words_file.getValue();
      if (words_file.getValue() != "")
	      myParam.input_words_file = myParam.output_words_file = words_file.getValue();

      myParam.model_prefix = model_prefix.getValue();

      myParam.ngram_size = ngram_size.getValue();
      myParam.vocab_size = vocab_size.getValue();
      myParam.input_vocab_size = input_vocab_size.getValue();
      myParam.output_vocab_size = output_vocab_size.getValue();
      if (vocab_size.getValue() >= 0) {
	      myParam.input_vocab_size = myParam.output_vocab_size = vocab_size.getValue();
      }
      myParam.num_hidden = num_hidden.getValue();
	  myParam.num_second_hidden = num_second_hidden.getValue();
      myParam.activation_function = activation_function.getValue();
      myParam.loss_function = loss_function.getValue();

      myParam.num_threads = num_threads.getValue();

      myParam.num_noise_samples = num_noise_samples.getValue();

      myParam.input_embedding_dimension = input_embedding_dimension.getValue();
      myParam.output_embedding_dimension = output_embedding_dimension.getValue();
      if (embedding_dimension.getValue() >= 0) {
	      myParam.input_embedding_dimension = myParam.output_embedding_dimension = embedding_dimension.getValue();
      }
	  myParam.input_embeddings_file = input_embeddings_file.getValue();

	  myParam.context_vector_size = context_vector_size.getValue();

      myParam.minibatch_size = minibatch_size.getValue();
      myParam.validation_minibatch_size = validation_minibatch_size.getValue();
      myParam.num_epochs= num_epochs.getValue();
      myParam.learning_rate = learning_rate.getValue();
      myParam.conditioning_constant = conditioning_constant.getValue();
      myParam.decay = decay.getValue();
      myParam.adagrad_epsilon = adagrad_epsilon.getValue();
      myParam.use_momentum = use_momentum.getValue();
      myParam.share_embeddings = share_embeddings.getValue();
      myParam.normalization = normalization.getValue();
      myParam.initial_momentum = initial_momentum.getValue();
      myParam.final_momentum = final_momentum.getValue();
      myParam.L2_reg = L2_reg.getValue();
      myParam.init_normal= init_normal.getValue();
      myParam.init_range = init_range.getValue();
      myParam.normalization_init = normalization_init.getValue();
      myParam.parameter_update = parameter_update.getValue();
	  myParam.hidden1_dropout_value = hidden1_dropout_value.getValue();
	  myParam.hidden2_dropout_value = hidden2_dropout_value.getValue();
	  myParam.hidden3_dropout_value = hidden3_dropout_value.getValue();

	  myParam.train_context_vectors_file = train_context_vectors_file.getValue();
	  myParam.validation_context_vectors_file = validation_context_vectors_file.getValue();

      cerr << "Command line: " << endl;
      cerr << boost::algorithm::join(vector<string>(argv, argv+argc), " ") << endl;

      const string sep(" Value: ");
      cerr << train_file.getDescription() << sep << train_file.getValue() << endl;
      cerr << validation_file.getDescription() << sep << validation_file.getValue() << endl;
      cerr << input_words_file.getDescription() << sep << input_words_file.getValue() << endl;
      cerr << output_words_file.getDescription() << sep << output_words_file.getValue() << endl;
      cerr << model_prefix.getDescription() << sep << model_prefix.getValue() << endl;

      cerr << ngram_size.getDescription() << sep << ngram_size.getValue() << endl;
      cerr << input_vocab_size.getDescription() << sep << input_vocab_size.getValue() << endl;
      cerr << output_vocab_size.getDescription() << sep << output_vocab_size.getValue() << endl;
      cerr << mmap_file.getDescription() << sep << mmap_file.getValue() << endl;

      if (embedding_dimension.getValue() >= 0)
      {
	      cerr << embedding_dimension.getDescription() << sep << embedding_dimension.getValue() << endl;
      }
      else
      {
	      cerr << input_embedding_dimension.getDescription() << sep << input_embedding_dimension.getValue() << endl;
	      cerr << output_embedding_dimension.getDescription() << sep << output_embedding_dimension.getValue() << endl;
      }
      cerr << share_embeddings.getDescription() << sep << share_embeddings.getValue() << endl;
      if (share_embeddings.getValue() && input_embedding_dimension.getValue() != output_embedding_dimension.getValue())
      {
	      cerr << "error: sharing input and output embeddings requires that input and output embeddings have same dimension" << endl;
	      exit(1);
      }

      cerr << num_hidden.getDescription() << sep << num_hidden.getValue() << endl;

      if (string_to_activation_function(activation_function.getValue()) == InvalidFunction)
      {
	      cerr << "error: invalid activation function: " << activation_function.getValue() << endl;
	      exit(1);
      }
      cerr << activation_function.getDescription() << sep << activation_function.getValue() << endl;

      if (string_to_loss_function(loss_function.getValue()) == InvalidLoss)
      {
	      cerr << "error: invalid loss function: " << loss_function.getValue() << endl;
	      exit(1);
      }
      cerr << loss_function.getDescription() << sep << loss_function.getValue() << endl;

      cerr << init_normal.getDescription() << sep << init_normal.getValue() << endl;
      cerr << init_range.getDescription() << sep << init_range.getValue() << endl;

      cerr << num_epochs.getDescription() << sep << num_epochs.getValue() << endl;
      cerr << minibatch_size.getDescription() << sep << minibatch_size.getValue() << endl;
      if (myParam.validation_file != "") {
	     cerr << validation_minibatch_size.getDescription() << sep << validation_minibatch_size.getValue() << endl;
      }
      cerr << learning_rate.getDescription() << sep << learning_rate.getValue() << endl;
      cerr << L2_reg.getDescription() << sep << L2_reg.getValue() << endl;

      cerr << num_noise_samples.getDescription() << sep << num_noise_samples.getValue() << endl;

      cerr << normalization.getDescription() << sep << normalization.getValue() << endl;
      if (myParam.normalization){
	      cerr << normalization_init.getDescription() << sep << normalization_init.getValue() << endl;
      }

      cerr << use_momentum.getDescription() << sep << use_momentum.getValue() << endl;
      if (myParam.use_momentum)
      {
        cerr << initial_momentum.getDescription() << sep << initial_momentum.getValue() << endl;
        cerr << final_momentum.getDescription() << sep << final_momentum.getValue() << endl;
      }

      cerr << num_threads.getDescription() << sep << num_threads.getValue() << endl;

      if (unigram_probs_file.getValue() != "")
      {
	      cerr << "Note: --unigram_probs_file is deprecated and ignored." << endl;
      }
    }
    catch (TCLAP::ArgException &e)
    {
      cerr << "error: " << e.error() <<  " for arg " << e.argId() << endl;
      exit(1);
    }

    myParam.num_threads = setup_threads(myParam.num_threads);
    int save_threads;

    //unsigned seed = std::time(0);
	//cerr<<"Seed is "<<seed<<endl;
    unsigned seed = 1234; //for testing only
    mt19937 rng(seed);

    /////////////////////////READING IN THE TRAINING AND VALIDATION DATA///////////////////
    /////////////////////////////////////////////////////////////////////////////////////

    // Read training data

    vector<int> training_data_flat;
    vec * training_data_flat_mmap;
    data_size_t training_data_size; //num_tokens;
    ip::managed_mapped_file mmap_file;
    if (use_mmap_file == false) {
      cerr<<"Reading data from regular text file "<<endl;
      readDataFile(myParam.train_file, myParam.ngram_size, training_data_flat, myParam.minibatch_size);
      training_data_size = training_data_flat.size()/myParam.ngram_size;
    } else {
      cerr<<"Using mmaped file"<<endl;
      mmap_file = ip::managed_mapped_file(ip::open_only,myParam.train_file.c_str());
      training_data_flat_mmap = mmap_file.find<vec>("vector").first;
      cerr<<"Size of mmaped vector is "<<training_data_flat_mmap->size()<<endl;
      training_data_size = training_data_flat_mmap->size()/myParam.ngram_size;
      //randomly shuffle the data for better learning. The shuffling will
      //be different for a standard stl vector
      // Randomly shuffle training data to improve learning
      if (randomize == true) {
        cerr<<"Randomly shuffling data...";
        data_size_t counter =0;
        while (counter < training_data_size) {
          data_size_t upper_limit = counter+5000000;
          long int vector_size = 5000000;
          if (counter + 10000000 >= training_data_size) {
            upper_limit = training_data_size;
            vector_size = training_data_size - counter;
          }
          vector<int> temp(vector_size*myParam.ngram_size,0);
          for (int i=0;i<vector_size;i++){
           for (int k=0;k<myParam.ngram_size;k++) {
             temp[i*myParam.ngram_size+k] = training_data_flat_mmap->at((i+counter)*myParam.ngram_size+k);
           }
          }
          /*
          for (data_size_t i=upper_limit; i>counter; i--)
          {
            if (i %500000 == 0) {
              cerr<<"Shuffled "<<training_data_size-1<<" instances...";
            }
            data_size_t j = uniform_int_distribution<data_size_t>(0, i-1)(rng);
            for (int k=0;k<myParam.ngram_size;k++) {
              int temp_val = training_data_flat_mmap->at(i*myParam.ngram_size+k);
              training_data_flat_mmap->at(i*myParam.ngram_size+k) =
                training_data_flat_mmap->at(j*myParam.ngram_size+k);
              training_data_flat_mmap->at(j*myParam.ngram_size+k) = temp_val;
            }
          }
          */
          for (data_size_t i=vector_size-1; i>0; i--)
          {
            if (i %500000 == 0) {
              cerr<<"Shuffled "<<training_data_size-1<<" instances...";
            }
            data_size_t j = uniform_int_distribution<data_size_t>(0, i-1)(rng);
            for (int k=0;k<myParam.ngram_size;k++) {
              int temp_val = temp.at(i*myParam.ngram_size+k);
              temp.at(i*myParam.ngram_size+k) =
                temp.at(j*myParam.ngram_size+k);
              temp.at(j*myParam.ngram_size+k) = temp_val;
            }
          }
          //Putting it back
          for (int i=0;i<vector_size;i++){
           for (int k=0;k<myParam.ngram_size;k++) {
             training_data_flat_mmap->at((i+counter)*myParam.ngram_size+k) = temp[i*myParam.ngram_size+k];
           }
          }
          counter = upper_limit;
        }
        /*
        for (data_size_t i=training_data_size-1; i>0; i--)
        {
          if (i %500000 == 0) {
            cerr<<"Shuffled "<<training_data_size-1<<" instances...";
          }
          data_size_t j = uniform_int_distribution<data_size_t>(0, i-1)(rng);
          for (int k=0;k<myParam.ngram_size;k++) {
            int temp_val = training_data_flat_mmap->at(i*myParam.ngram_size+k);
            training_data_flat_mmap->at(i*myParam.ngram_size+k) =
              training_data_flat_mmap->at(j*myParam.ngram_size+k);
            training_data_flat_mmap->at(j*myParam.ngram_size+k) = temp_val;
          }
        }
        */
      cerr<<endl;
      }
    }
    //cerr<<"Num tokens "<<num_tokens<<endl;
    //data_size_t training_data_size = num_tokens / myParam.ngram_size;
    cerr << "Number of training instances: "<< training_data_size << endl;

    Matrix<int,Dynamic,Dynamic> training_data;
    //(training_data_flat.data(), myParam.ngram_size, training_data_size);

    #ifdef MAP
    cerr<<"Setting up eigen map"<<endl;
    if (use_mmap_file == false) {
      training_data = Map< Matrix<int,Dynamic,Dynamic> >(training_data_flat.data(), myParam.ngram_size, training_data_size);
    } else {
      training_data = Map< Matrix<int,Dynamic,Dynamic> >(training_data_flat_mmap->data().get(), myParam.ngram_size, training_data_size);
    }
    cerr<<"Created eigen map"<<endl;
    #else
    if (use_mmap_file == false) {
      training_data = Map< Matrix<int,Dynamic,Dynamic> >(training_data_flat.data(), myParam.ngram_size, training_data_size);
    }
    #endif
    // If neither --input_vocab_size nor --input_words_file is given, set input_vocab_size to the maximum word index
    if (myParam.input_vocab_size == 0 and myParam.input_words_file == "")
    {
        myParam.input_vocab_size = training_data.topRows(myParam.ngram_size-1).maxCoeff()+1;
    }

    // If neither --output_vocab_size nor --output_words_file is given, set output_vocab_size to the maximum word index
    if (myParam.output_vocab_size == 0 and myParam.output_words_file == "")
    {
        myParam.output_vocab_size = training_data.row(myParam.ngram_size-1).maxCoeff()+1;
    }

    // Read validation data
    vector<int> validation_data_flat;
    int validation_data_size = 0;

    if (myParam.validation_file != "")
    {
      readDataFile(myParam.validation_file, myParam.ngram_size, validation_data_flat);
      validation_data_size = validation_data_flat.size() / myParam.ngram_size;
      cerr << "Number of validation instances: " << validation_data_size << endl;
    }

	// Read train context vectors
	vector<double> training_context_vectors_flat;
	//int context_size = 128;
    if (myParam.train_context_vectors_file != "")
    {
      readContextVectorsFile(myParam.train_context_vectors_file, myParam.context_vector_size, training_context_vectors_flat);
      int train_context_vectors_flat_size  = training_context_vectors_flat.size() / myParam.context_vector_size;
	  if (train_context_vectors_flat_size != training_data_size){
		  cerr<<"The nubmer of training context vectors was not equal to the training data!"<<endl;
		  exit(1);
	  }
    }

	// Read validation context vectors
	vector<double> validation_context_vectors_flat;
    if (myParam.validation_context_vectors_file != "")
    {
      readContextVectorsFile(myParam.validation_context_vectors_file, myParam.context_vector_size, validation_context_vectors_flat);
      int validation_context_vectors_flat_size  = validation_context_vectors_flat.size() / myParam.context_vector_size;
	  if (validation_context_vectors_flat_size != validation_data_size){
		  cerr<<"The nubmer of validation context vectors was not equal to the validationing data!"<<endl;
		  exit(1);
	  }
    }

	Map< Matrix<double,Dynamic,Dynamic> > validation_context_vectors(validation_context_vectors_flat.data(),
																	myParam.context_vector_size, validation_data_size);

	Map< Matrix<double,Dynamic,Dynamic> > training_context_vectors(training_context_vectors_flat.data(),
																	myParam.context_vector_size, training_data_size);

    Map< Matrix<int,Dynamic,Dynamic> > validation_data(validation_data_flat.data(), myParam.ngram_size, validation_data_size);

    if (use_mmap_file == false && randomize == true) {
      cerr<<"Randomly shuffling data..."<<endl;
      // Randomly shuffle training data to improve learning. Shuffling the context vectors as well
      for (data_size_t i=training_data_size-1; i>0; i--)
      {
        data_size_t j = uniform_int_distribution<data_size_t>(0, i-1)(rng);
        training_data.col(i).swap(training_data.col(j));
		//training_context_vectors.col(i).swap(training_context_vectors.col(j));
      }
    }

    ///// Read in vocabulary file. We don't actually use it; it just gets reproduced in the output file

    vector<string> input_words;
    if (myParam.input_words_file != "")
    {
        readWordsFile(myParam.input_words_file, input_words);
	if (myParam.input_vocab_size == 0)
	    myParam.input_vocab_size = input_words.size();
    }

    vector<string> output_words;
    if (myParam.output_words_file != "")
    {
        readWordsFile(myParam.output_words_file, output_words);
	if (myParam.output_vocab_size == 0)
	    myParam.output_vocab_size = output_words.size();
    }

    ///// Construct unigram model and sampler that will be used for NCE

    vector<data_size_t> unigram_counts(myParam.output_vocab_size);
    for (data_size_t train_id=0; train_id < training_data_size; train_id++)
    {
        int output_word;
        if (use_mmap_file == false) {
          output_word = training_data(myParam.ngram_size-1, train_id);
        } else {
	      //cerr<<"mmap word is "<<training_data_flat_mmap->at((train_id+1)*myParam.ngram_size - 1)<<endl;
          output_word = training_data_flat_mmap->at((train_id+1)*myParam.ngram_size - 1);
        }
		//cerr<<"output word is "<<output_word<<endl;
	    unigram_counts[output_word] += 1;
    }
    multinomial<data_size_t> unigram (unigram_counts);

    ///// Create and initialize the neural network and associated propagators.
    model nn;
    // IF THE MODEL FILE HAS BEEN DEFINED, THEN
    // LOAD THE NEURAL NETWORK MODEL
    //if (myParam.model_file != ""){
    //nn.read(myParam.model_file);
    // cerr<<"reading the model"<<endl;

	//} else {
      nn.resize(myParam.ngram_size,
          myParam.input_vocab_size,
          myParam.output_vocab_size,
          myParam.input_embedding_dimension,
          myParam.num_hidden,
		  myParam.num_second_hidden,
          myParam.output_embedding_dimension,
		  myParam.context_vector_size);

      nn.initialize(rng,
          myParam.init_normal,
          myParam.init_range,
          -log(myParam.output_vocab_size),
          myParam.parameter_update,
          myParam.adagrad_epsilon);
      nn.set_activation_function(string_to_activation_function(myParam.activation_function));
	  // If the input embeddings file has been specified then read from the input embeddings files
	  if (myParam.input_embeddings_file != ""){
		  cerr<<" Reading the input embeddings from file"<<myParam.input_embeddings_file<<endl;
		  nn.input_layer.read(myParam.input_embeddings_file);
	  }
      if (myParam.model_file != ""){
        nn.read(myParam.model_file);
        cerr<<"reading the model"<<endl;
      }
    //}
    loss_function_type loss_function = string_to_loss_function(myParam.loss_function);

    propagator prop(nn, myParam.minibatch_size);
    propagator prop_validation(nn, myParam.validation_minibatch_size);
    SoftmaxNCELoss<multinomial<data_size_t> > softmax_loss(unigram);
    // normalization parameters
    vector_map c_h, c_h_running_gradient;

    ///////////////////////TRAINING THE NEURAL NETWORK////////////////////////////////////
    /////////////////////////////////////////////////////////////////////////////////////

    data_size_t num_batches = (training_data_size-1)/myParam.minibatch_size + 1;
    cerr<<"Number of training minibatches: "<<num_batches<<endl;

    int num_validation_batches = 0;
    if (validation_data_size > 0)
    {
        num_validation_batches = (validation_data_size-1)/myParam.validation_minibatch_size+1;
	cerr<<"Number of validation minibatches: "<<num_validation_batches<<endl;
    }

    double current_momentum = myParam.initial_momentum;
    double momentum_delta = (myParam.final_momentum - myParam.initial_momentum)/(myParam.num_epochs-1);
    double current_learning_rate = myParam.learning_rate;
    double current_validation_ll = 0.0;
	double current_validation_accuracy = 0.0;

    int ngram_size = myParam.ngram_size;
    int input_vocab_size = myParam.input_vocab_size;
    int output_vocab_size = myParam.output_vocab_size;
    int minibatch_size = myParam.minibatch_size;
    int validation_minibatch_size = myParam.validation_minibatch_size;
    int num_noise_samples = myParam.num_noise_samples;

    if (myParam.normalization)
    {
      for (data_size_t i=0;i<training_data_size;i++)
      {
          Matrix<int,Dynamic,1> context = training_data.block(0,i,ngram_size-1,1);
          if (c_h.find(context) == c_h.end())
          {
              c_h[context] = -myParam.normalization_init;
          }
      }
    }

	//Setting up the dropout sampler
	boost::random::uniform_real_distribution<> real_01(0, 1);

    for (int epoch=0; epoch<myParam.num_epochs; epoch++)
    {
		double train_correct_labels = 0.;
        cerr << "Epoch " << epoch+1 << endl;
        cerr << "Current learning rate: " << current_learning_rate << endl;

        if (myParam.use_momentum)
	    cerr << "Current momentum: " << current_momentum << endl;
	else
            current_momentum = -1;

	cerr << "Training minibatches: ";

	double log_likelihood = 0.0;

	int num_samples = 0;
	if (loss_function == LogLoss)
	    num_samples = output_vocab_size;
	else if (loss_function == NCELoss)
	    num_samples = 1+num_noise_samples;

	Matrix<double,Dynamic,Dynamic> minibatch_weights(num_samples, minibatch_size);
	Matrix<int,Dynamic,Dynamic> minibatch_samples(num_samples, minibatch_size);
	Matrix<double,Dynamic,Dynamic> scores(num_samples, minibatch_size);
	Matrix<double,Dynamic,Dynamic> probs(num_samples, minibatch_size);
	Matrix<double,Dynamic,Dynamic> first_hidden_dropout_mask(myParam.num_hidden,minibatch_size);
	Matrix<double,Dynamic,Dynamic> second_hidden_dropout_mask(myParam.output_embedding_dimension,minibatch_size);

  for(data_size_t batch=0;batch<num_batches;batch++)
        {
            if (batch > 0 && batch % 10000 == 0)
            {
	        cerr << batch <<"...";
            }

            data_size_t minibatch_start_index = minibatch_size * batch;

      int current_minibatch_size = min(static_cast<data_size_t>(minibatch_size), training_data_size - minibatch_start_index);
      #ifdef MAP
	    Matrix<int,Dynamic,Dynamic> minibatch = training_data.middleCols(minibatch_start_index, current_minibatch_size);
      #else
      //ALTERNATIVE OPTION IF YOU'RE NOT USING eigen map interface on the mmapped file
	    Matrix<int,Dynamic,Dynamic> minibatch;// = training_data.middleCols(minibatch_start_index, current_minibatch_size);
		//cerr<<"Minibatch start index "<<minibatch_start_index<<endl;
		//cerr<<"Minibatch size "<<current_minibatch_size<<endl;
            if (use_mmap_file == true) {
            minibatch.setZero(ngram_size,current_minibatch_size);
            //now reading the ngrams from the mmaped file
              for (int k=0; k<ngram_size; k++){
                for (data_size_t index = 0 ; index<current_minibatch_size; index++) {
				  data_size_t current_index = index + minibatch_start_index;
				  //cerr<<"the value in the mmap file "<<index<<" "<<k<<" is "<<training_data_flat_mmap->at(current_index*ngram_size+k)<<endl;
                  minibatch(k,index) = training_data_flat_mmap->at(current_index*ngram_size+k);
                }
              }
            } else {
              minibatch = training_data.middleCols(minibatch_start_index, current_minibatch_size);
            }
      #endif
            double adjusted_learning_rate = current_learning_rate/current_minibatch_size;
            //cerr<<"Adjusted learning rate: "<<adjusted_learning_rate<<endl;

            /*
            if (batch == rand() % num_batches)
            {
                cerr<<"we are checking the gradient in batch "<<batch<<endl;
                /////////////////////////CHECKING GRADIENTS////////////////////////////////////////
                gradientChecking(myParam,minibatch_start_index,current_minibatch_size,word_nodes,context_nodes,hidden_layer_node,hidden_layer_to_output_node,
                              shuffled_training_data,c_h,unif_real_vector,eng_real_vector,unif_int_vector,eng_int_vector,unigram_probs_vector,
                              q_vector,J_vector,D_prime);
            }
            */

		// Generating the dropout mask
		for(int q=0; q<current_minibatch_size; q++){
			for (int p=0; p<myParam.num_hidden; p++){
				first_hidden_dropout_mask(p,q) = ( real_01(rng) <1.0 - myParam.hidden1_dropout_value ) ? 0: 1;
			}
			for (int p=0; p<myParam.output_embedding_dimension; p++){
				second_hidden_dropout_mask(p,q) = ( real_01(rng) < 1.0 - myParam.hidden2_dropout_value) ? 0: 1;
			}

		}
		//cerr<<"Current minibatch size is "<<current_minibatch_size<<endl;
		//cerr<<"Shape of dropout mask is "<<second_hidden_dropout_mask.rows()<<" "<<second_hidden_dropout_mask.cols()<<endl;
		//cerr<<"Shape of dropout mask is "<<first_hidden_dropout_mask.rows()<<" "<<first_hidden_dropout_mask.cols()<<endl;
		//cerr<<"First hidden dropout mask is "<<first_hidden_dropout_mask<<endl;
		//cerr<<"dropout mas is "<<second_hidden_dropout_mask<<endl;
		//getchar();

        ///// Forward propagation
        prop.fPropDropout(minibatch.topRows(ngram_size-1),
					first_hidden_dropout_mask,
					second_hidden_dropout_mask);

	    if (loss_function == NCELoss)
	    {
	      ///// Noise-contrastive estimation

	      // Generate noise samples. Gather positive and negative samples into matrix.

	      start_timer(3);

        minibatch_samples.block(0, 0, 1, current_minibatch_size) = minibatch.bottomRows(1);

        for (int sample_id = 1; sample_id < num_noise_samples+1; sample_id++)
            for (int train_id = 0; train_id < current_minibatch_size; train_id++)
                minibatch_samples(sample_id, train_id) = unigram.sample(rng);

        stop_timer(3);

        // Final forward propagation step (sparse)
        start_timer(4);
        prop.output_layer_node.param->fProp(prop.second_hidden_activation_node.fProp_matrix,
                    minibatch_samples, scores);
        stop_timer(4);

        // Apply normalization parameters
        if (myParam.normalization)
        {
            for (int train_id = 0;train_id < current_minibatch_size;train_id++)
            {
          Matrix<int,Dynamic,1> context = minibatch.block(0, train_id, ngram_size-1, 1);
          scores.col(train_id).array() += c_h[context];
            }
        }

        double minibatch_log_likelihood;
        start_timer(5);
        softmax_loss.fProp(scores.leftCols(current_minibatch_size),
               minibatch_samples,
               probs,
			   minibatch_log_likelihood);
        stop_timer(5);
        log_likelihood += minibatch_log_likelihood;

        ///// Backward propagation

        start_timer(6);
        softmax_loss.bProp(probs, minibatch_weights);
        stop_timer(6);

        // Update the normalization parameters

        if (myParam.normalization)
        {
          for (int train_id = 0;train_id < current_minibatch_size;train_id++)
          {
            Matrix<int,Dynamic,1> context = minibatch.block(0, train_id, ngram_size-1, 1);
            c_h[context] += adjusted_learning_rate * minibatch_weights.col(train_id).sum();
          }
        }

        // Be careful of short minibatch
        prop.bProp(minibatch.topRows(ngram_size-1),
             minibatch_samples.leftCols(current_minibatch_size),
             minibatch_weights.leftCols(current_minibatch_size),
			 first_hidden_dropout_mask.leftCols(current_minibatch_size),
			 second_hidden_dropout_mask.leftCols(current_minibatch_size),
             adjusted_learning_rate,
             current_momentum,
             myParam.L2_reg,
             myParam.parameter_update,
             myParam.conditioning_constant,
             myParam.decay);
	    }
	    else if (loss_function == LogLoss)
	    {
	      ///// Standard log-likelihood
	      start_timer(4);
        prop.output_layer_node.param->fProp(prop.second_hidden_activation_node.fProp_matrix, scores);
        stop_timer(4);

        double minibatch_log_likelihood;
        start_timer(5);
        SoftmaxLogLoss().fProp(scores.leftCols(current_minibatch_size),
                   minibatch.row(ngram_size-1),
                   probs,
                   minibatch_log_likelihood);
        stop_timer(5);
        log_likelihood += minibatch_log_likelihood;

        ///// Backward propagation

        start_timer(6);
        SoftmaxLogLoss().bProp(minibatch.row(ngram_size-1).leftCols(current_minibatch_size),
                   probs.leftCols(current_minibatch_size),
                   minibatch_weights);
        stop_timer(6);
        //Computing the training accuracy
   		for (int minibatch_instance=0; minibatch_instance < current_minibatch_size; minibatch_instance++){
   			Matrix<double,1,Dynamic>::Index max_index;
   			probs.col(minibatch_instance).maxCoeff(&max_index);
   			if (max_index ==
   				minibatch.row(ngram_size-1).leftCols(current_minibatch_size)(minibatch_instance)){
   					train_correct_labels += 1;
   				}
   		}

        prop.bProp(minibatch.topRows(ngram_size-1).leftCols(current_minibatch_size),
             minibatch_weights,
			 first_hidden_dropout_mask,
			 second_hidden_dropout_mask,
             adjusted_learning_rate,
             current_momentum,
             myParam.L2_reg,
             myParam.parameter_update,
             myParam.conditioning_constant,
             myParam.decay);
          }
      }

	cerr << "done." << endl;

	if (loss_function == LogLoss)
	{
	    cerr << "Training log-likelihood: " << log_likelihood << endl;
        cerr << "         perplexity:     "<< exp(-log_likelihood/training_data_size) << endl;
		cerr << "         accuracy:       "<< train_correct_labels/training_data_size <<endl;
	}
	else if (loss_function == NCELoss)
	    cerr << "Training NCE log-likelihood: " << log_likelihood << endl;

        current_momentum += momentum_delta;

	#ifdef USE_CHRONO
	cerr << "Propagation times:";
	for (int i=0; i<timer.size(); i++)
	  cerr << " " << timer.get(i);
	cerr << endl;
	#endif

	//Scaled the model before writing
	nn.scale(myParam.hidden1_dropout_value,
			myParam.hidden2_dropout_value,
			myParam.hidden3_dropout_value);
	if (myParam.model_prefix != "")
	{
	    cerr << "Writing model" << endl;
	    if (myParam.input_words_file != "")
	        nn.write(myParam.model_prefix + "." + lexical_cast<string>(epoch+1), input_words, output_words);
	    else
	        nn.write(myParam.model_prefix + "." + lexical_cast<string>(epoch+1));
	}

        if (epoch % 1 == 0 && validation_data_size > 0)
        {
		    //Matrix<int,Dynamic,1> validation_argmaxes;
			//validation_argmaxes.setZero(validation_data_size);
			vector<int> validation_argmaxes;
			string argmax_file = "argmax."+lexical_cast<string>(epoch+1);
            //////COMPUTING VALIDATION SET PERPLEXITY///////////////////////
            ////////////////////////////////////////////////////////////////

            double log_likelihood = 0.0;

	    Matrix<double,Dynamic,Dynamic> scores(output_vocab_size, validation_minibatch_size);
	    Matrix<double,Dynamic,Dynamic> output_probs(output_vocab_size, validation_minibatch_size);
	    Matrix<int,Dynamic,Dynamic> minibatch(ngram_size, validation_minibatch_size);
		double validation_correct_labels  =0;
            for (int validation_batch =0;validation_batch < num_validation_batches;validation_batch++)
            {
                int validation_minibatch_start_index = validation_minibatch_size * validation_batch;
		int current_minibatch_size = min(validation_minibatch_size,
						 validation_data_size - validation_minibatch_start_index);
		minibatch.leftCols(current_minibatch_size) = validation_data.middleCols(validation_minibatch_start_index,
											current_minibatch_size);
		prop_validation.fPropContext(minibatch.topRows(ngram_size-1));

		// Do full forward prop through output word embedding layer
		start_timer(4);
		prop_validation.output_layer_node.param->fProp(prop_validation.second_hidden_activation_node.fProp_matrix, scores);
		stop_timer(4);

		// And softmax and loss. Be careful of short minibatch
		double minibatch_log_likelihood;
		start_timer(5);
		SoftmaxLogLoss().fProp(scores.leftCols(current_minibatch_size),
				       minibatch.row(ngram_size-1),
				       output_probs,
				       minibatch_log_likelihood);
		stop_timer(5);
		log_likelihood += minibatch_log_likelihood;
	   		for (int minibatch_instance=0; minibatch_instance < current_minibatch_size; minibatch_instance++){
	   			Matrix<double,1,Dynamic>::Index max_index;
	   			output_probs.col(minibatch_instance).maxCoeff(&max_index);
				validation_argmaxes.push_back(max_index);
	   			if (max_index ==
	   				minibatch.row(ngram_size-1)(minibatch_instance)){
	   					validation_correct_labels += 1;
	   				}
	   		}
	    }
		   /*
           //CREATING AN EIGEN MATRIX OUT OF THE ARGMAXES
		   Matrix<int,Dynamic,1> argmaxes_vector(validation_data_size);
		   for (data_size_t index=0; index<=validation_data_size; validation_data_size++){
		     argmaxes_vector(index) = validation_argmaxes[index];
		   }
		   */
		   //argmaxes_vector = Map< Matrix<int,Dynamic,1> >(validation_argmaxes.data(),validation_data_size,1);
            cerr << "Validation log-likelihood: "<< log_likelihood << endl;
            cerr << "           perplexity:     "<< exp(-log_likelihood/validation_data_size) << endl;
			cerr << "           accuracy:       "<< validation_correct_labels/validation_data_size <<endl;
            writeVector(validation_argmaxes,argmax_file);
	    // If the validation perplexity decreases, halve the learning rate.

            //if (epoch > 0 && validation_correct_labels/validation_data_size < current_validation_accuracy && myParam.parameter_update != "ADA" && current_momentum < 0.)
			//
            if (epoch > 0 && log_likelihood < current_validation_ll && myParam.parameter_update != "ADA" && current_momentum < 0.)
            {
                current_learning_rate /= 2;
            }
            current_validation_ll = log_likelihood;
			current_validation_accuracy = validation_correct_labels/validation_data_size;

		}
		//Scaled the model back after validation
		nn.scale(1.0/myParam.hidden1_dropout_value,
			1.0/myParam.hidden2_dropout_value,
			1.0/myParam.hidden3_dropout_value);
    }
    return 0;
}


//==============================================================================
//         Copyright 2003 - 2011 LASMEA UMR 6602 CNRS/Univ. Clermont II
//         Copyright 2009 - 2011 LRI    UMR 8623 CNRS/Univ Paris Sud XI
//         Copyright 2012 - 2014 MetaScale SAS
//
//          Distributed under the Boost Software License, Version 1.0.
//                 See accompanying file LICENSE.txt or copy at
//                     http://www.boost.org/LICENSE_1_0.txt
//==============================================================================
#ifndef BOOST_SIMD_ARITHMETIC_FUNCTIONS_SIMD_COMMON_DIVS_HPP_INCLUDED
#define BOOST_SIMD_ARITHMETIC_FUNCTIONS_SIMD_COMMON_DIVS_HPP_INCLUDED

#include <boost/simd/arithmetic/functions/divs.hpp>
#include <boost/simd/include/functions/simd/is_eqz.hpp>
#include <boost/simd/include/functions/simd/is_nez.hpp>
#include <boost/simd/include/functions/simd/divides.hpp>
#include <boost/simd/include/functions/simd/shift_right.hpp>
#include <boost/simd/include/functions/simd/plus.hpp>
#include <boost/simd/include/functions/simd/if_zero_else_one.hpp>
#include <boost/simd/include/functions/simd/bitwise_or.hpp>
#include <boost/simd/include/functions/simd/bitwise_xor.hpp>
#include <boost/simd/include/functions/simd/if_else.hpp>
#include <boost/simd/include/functions/simd/genmask.hpp>
#include <boost/simd/include/functions/simd/logical_and.hpp>
#include <boost/simd/include/constants/one.hpp>
#include <boost/simd/include/constants/valmin.hpp>
#include <boost/simd/include/constants/valmax.hpp>
#include <boost/simd/sdk/meta/scalar_of.hpp>
#include <boost/simd/sdk/meta/as_logical.hpp>
#include <boost/dispatch/attributes.hpp>

// perhaps divs for signed integral types must invoke correct Valmin entry and invoke  divfix
// also call simply divfix for unsigned
// also rdivide has to be defined as divs for float and as divfix for integers
namespace boost { namespace simd { namespace ext
{
  BOOST_DISPATCH_IMPLEMENT          ( divs_, tag::cpu_, (A0)(X)
                                    , ((simd_<floating_<A0>,X>))
                                      ((simd_<floating_<A0>,X>))
                                    )
  {
    typedef A0 result_type;
    BOOST_FORCEINLINE BOOST_SIMD_FUNCTOR_CALL_REPEAT(2)
    {
      return a0/a1;
    }
  };

  BOOST_DISPATCH_IMPLEMENT          ( divs_, tag::cpu_, (A0)(X)
                                    , ((simd_<uint_<A0>,X>))
                                      ((simd_<uint_<A0>,X>))
                                    )
  {
    typedef A0 result_type;
    BOOST_FORCEINLINE BOOST_SIMD_FUNCTOR_CALL_REPEAT(2)
    {
      typedef typename meta::as_logical<A0>::type bA0;

      const bA0 iseqza1 = is_eqz(a1);
      const A0 aa1 = if_else(iseqza1, One<A0>(), a1);
      const A0 aa0 = if_else(iseqza1, genmask(a0), a0);
      return aa0/aa1;
    }
  };

  BOOST_DISPATCH_IMPLEMENT          ( divs_, tag::cpu_, (A0)(X)
                                    , ((simd_<int_<A0>,X>))
                                      ((simd_<int_<A0>,X>))
                                    )
  {
    typedef A0 result_type;

    BOOST_FORCEINLINE BOOST_SIMD_FUNCTOR_CALL_REPEAT(2)
    {
      typedef typename meta::as_logical<A0>::type bA0;
      typedef typename meta::scalar_of<A0>::type sA0;

      const bA0 iseqza1 = is_eqz(a1);

      // replace valmin/-1 by (valmin+1)/-1
      A0 x = a0 + if_zero_else_one((a1 + One<A0>()) | (a0 + Valmin<A0>()));
      // negative -> valmin
      // positive -> valmax
      const A0 x2 = bitwise_xor(Valmax<A0>(), shrai(x, sizeof(sA0)*CHAR_BIT-1));

      x = if_else(logical_and(iseqza1, is_nez(x)), x2, x);
      const A0 y = if_else(iseqza1, One<A0>(), a1);
      return x/y;
    }
  };
} } }

#endif