#include "als.h"
#include "mpi.h"
#include <stdio.h>
#include <stdlib.h>
#include <unistd.h>

#include <gsl/gsl_blas.h>
#include <gsl/gsl_linalg.h>
#include <gsl/gsl_matrix.h>

#define MAX_NAME_LEN 10

/* Define global variables */
global_info info;

feature_t *movieMatrix; // feature matrix of movies
feature_t *userMatrix;  // feature matrix of users

// store rating matrix in a compressed way
int userNum;
int movieNum;
int ratingNum;

int* userStartIdx;
int* movieId;
double* movieRating;

int* movieStartIdx;
int* userId;
double* userRating;

struct movie* movie_hashtable; // to record the number of rating for each movie
struct user* user_hashtable;   // to record the number of rating for each user

void computePredictionRMSE(gsl_matrix * M, gsl_matrix * U, gsl_matrix * R);

//---------------------------------------------
//------- helper function for hashtable--------
//---------------------------------------------
void add_movie(int id, double rating) {
    struct movie* res;
    HASH_FIND_INT(movie_hashtable, &id, res);
    if (res == NULL) {
        struct movie* new_movie = (struct movie*)malloc(sizeof(struct movie));
        new_movie -> id = id;
        new_movie -> rating_num = 1;
        new_movie -> num = 1;
        new_movie -> total_rating = rating;
        HASH_ADD_INT(movie_hashtable, id, new_movie);
    } else {
        res -> rating_num ++;
        res -> num ++;
        res -> total_rating += rating;
    }
}

void delete_movie(int id) {
    struct movie* res;
    HASH_FIND_INT(movie_hashtable, &id, res);
    res -> rating_num --;
}

int find_movie(int id) {
    struct movie *result;
    HASH_FIND_INT(movie_hashtable, &id, result);
    if (result == NULL) {
        return 0;
    } else {
        return result -> rating_num;
    }
}

double get_movie_average(int id) {
    struct movie *result;
    HASH_FIND_INT(movie_hashtable, &id, result);
    if (result == NULL) {
        return 0;
    } else {
        return result -> total_rating / ((double)result -> num);
    }
}

void add_user(int id) {
    struct user* res;
    HASH_FIND_INT(user_hashtable, &id, res);
    if (res == NULL) {
        struct user* new_user = (struct user*)malloc(sizeof(struct user));
        new_user -> id = id;
        new_user -> rating_num = 1;
        HASH_ADD_INT(user_hashtable, id, new_user);
    } else {
        res -> rating_num ++;
    }
}

void delete_user(int id) {
    struct user* res;
    HASH_FIND_INT(user_hashtable, &id, res);
    res -> rating_num --;
}

int find_user(int id) {
    struct user *result;
    HASH_FIND_INT(user_hashtable, &id, result);
    if (result == NULL) {
        return 0;
    } else {
        return result -> rating_num;
    }
}

//------------------------------------------------
//-----------process input data-------------------
//------------------------------------------------

/*
 * get the input relevant stat
 */
void getInputStat(char* inputFilename) {
    userNum = 0;
    movieNum = 0;
    ratingNum = 0;

    FILE* fp = fopen(inputFilename, "r");
    if (fp == NULL) {
        fprintf(stderr, "Failed to read input file %s\n", inputFilename);
        exit(-1);
    }

    ssize_t read;
    size_t len = 0;
    char* line = NULL;
    int uid = 0;
    int mid = 0;
    double rating = 0;
    if ((read = getline(&line, &len, fp)) != -1) {
        printf("Start reading file, get input stat\n");
    }

    while ((read = getline(&line, &len, fp)) != -1) {
        ratingNum++;

        // split the data line with comma
        char* tmp = line;
        char* last_pos = line;
        int word_len = 0;
        int comma_cnt = 0;

        while (comma_cnt < 3) {
            if (*tmp == ',') {
                word_len = tmp - last_pos;
                char tmpbuf[MAX_NAME_LEN] = "";
                strncpy(tmpbuf, last_pos, word_len);
                if (comma_cnt == 0) {
                    uid = atoi(tmpbuf);
                } else if (comma_cnt == 1) {
                    mid = atoi(tmpbuf);
                } else {
                    rating = atof(tmpbuf);
                }
                comma_cnt++;
                last_pos = tmp + 1;
            }
            tmp++;
        }
        add_movie(mid, rating);  // keep record of rating number for each movie
        add_user(uid);           // keep record of rating number of each user
        userNum = max(userNum, uid);
        movieNum = max(movieNum, mid);
    }
    printf("There are %d ratings, %d users, %d movies\n", ratingNum, userNum, movieNum);
}

void initUserStartIndex() {
    int i;
    int sum = 0;
    for (i = 1; i <= userNum; ++i) {
        int user_rating = find_user(i);
        userStartIdx[i - 1] = sum;
        sum += user_rating;
    }
    userStartIdx[userNum] = ratingNum;
}

void initMovieStartIndex() {
    int i;
    int sum = 0;
    for (i = 1; i <= movieNum; ++i) {
        int movie_rating = find_movie(i);
        movieStartIdx[i - 1] = sum;
        sum += movie_rating;
    }
    movieStartIdx[movieNum] = ratingNum;
}

void initRatingMatrix(int uid, int mid, double rating) {
    int movieIdx = movieStartIdx[mid] - find_movie(mid);
    int userIdx = userStartIdx[uid] - find_user(uid);
    userId[movieIdx] = uid;
    userRating[movieIdx] = rating;
    movieId[userIdx] = mid;
    movieRating[userIdx] = rating;
    delete_movie(mid);
    delete_user(uid);
}

void initMatrix() {
    printf("Init movie matrix\n");
    int i, j;
    for (i = 0; i < movieNum; ++i) {
        int start_idx = i * info.numFeatures;
        movieMatrix[start_idx] = get_movie_average(i + 1);
        // initialize the random small value
        for (j = 1; j < info.numFeatures; ++j) {
            movieMatrix[start_idx + j] = (rand() % 1000) / 1000.0;
        }
    }
}

// read input file
// initializa the rating matrix & movie matrix
void readInput(char* inputFilename) {
    FILE* fp = fopen(inputFilename, "r");
    if (fp == NULL) {
        fprintf(stderr, "Failed to read input file %s\n", inputFilename);
        exit(-1);
    }

    ssize_t read;
    size_t len = 0;
    char* line = NULL;
    int uid = 0;
    int mid = 0;
    double rating = 0;
    if ((read = getline(&line, &len, fp)) != -1) {
        printf("Start reading file, get rating details\n");
    }

    while ((read = getline(&line, &len, fp)) != -1) {
        // split the data line with comma
        char* tmp = line;
        char* last_pos = line;
        int word_len = 0;
        int comma_cnt = 0;

        while (comma_cnt < 3) {
            if (*tmp == ',') {
                word_len = tmp - last_pos;
                char tmpbuf[MAX_NAME_LEN] = "";
                strncpy(tmpbuf, last_pos, word_len);
                if (comma_cnt == 0) {
                    uid = atoi(tmpbuf);
                } else if (comma_cnt == 1) {
                    mid = atoi(tmpbuf);
                } else {
                    rating = atof(tmpbuf);
                }
                comma_cnt++;
                last_pos = tmp + 1;
            }
            tmp++;
        }
        // initialize the rating matrix (use compression)
        initRatingMatrix(uid, mid, rating);
    }
}

// init basic arg
void init(int numFeatures, int numIter, double lambda) {
    info.numIter = numIter;
    info.numFeatures = numFeatures;
    info.lambda = lambda;
}

/*
 * allocate data structure memory
 */
void init_data(int userNum, int movieNum, int ratingNum, int nproc) {
    printf("allocating data memory\n");

    int movieSpan = (movieNum + nproc - 1) / nproc;
    int userSpan = (userNum + nproc - 1) / nproc;

    userStartIdx = (int*)calloc(userNum + 1, sizeof(int));
    movieId =      (int*)calloc(ratingNum + 1, sizeof(int));
    movieRating =  (double*)calloc(ratingNum + 1, sizeof(double));

    movieStartIdx = (int*)calloc(movieNum + 1, sizeof(int));
    userId =        (int*)calloc(ratingNum + 1, sizeof(int));
    userRating =    (double*)calloc(ratingNum + 1, sizeof(double));

    movieMatrix = (feature_t *)calloc(info.numFeatures * movieSpan * nproc, sizeof(feature_t));
    userMatrix =  (feature_t *)calloc(info.numFeatures * userSpan * nproc, sizeof(feature_t));
}

//-----------------------------------------------------------------
//---------------------real computation----------------------------
//-----------------------------------------------------------------
void compute(int procID, int nproc, char* inputFilename,
             int numFeatures, int numIterations, double lambda)
{
    const int root = 0; // set the rank 0 processor as the root
    int span;

    // initialize
    init(numFeatures, numIterations, lambda);

    /* Read the input file and initialization */
    if (procID == root) {
        getInputStat(inputFilename);
    }

    /* broadcast the information of basic stat*/
    MPI_Bcast(&userNum, 1, MPI_INT, root, MPI_COMM_WORLD);
    MPI_Bcast(&movieNum, 1, MPI_INT, root, MPI_COMM_WORLD);
    MPI_Bcast(&ratingNum, 1, MPI_INT, root, MPI_COMM_WORLD);

    // initialize the data structure
    init_data(userNum, movieNum, ratingNum, nproc);

    if (procID == root) {
        initUserStartIndex();
        initMovieStartIndex();
    }

    /* broadcast the information about start index */
    MPI_Bcast(userStartIdx, userNum + 1, MPI_INT, root, MPI_COMM_WORLD);
    MPI_Bcast(movieStartIdx, movieNum + 1, MPI_INT, root, MPI_COMM_WORLD);

    /* Read the input file and get detailed rating */
    if (procID == root) {
        readInput(inputFilename);
    }

    MPI_Bcast(movieId, ratingNum + 1, MPI_INT, root, MPI_COMM_WORLD);
    MPI_Bcast(movieRating, ratingNum + 1, MPI_DOUBLE, root, MPI_COMM_WORLD);
    MPI_Bcast(userId, ratingNum + 1, MPI_INT, root, MPI_COMM_WORLD);
    MPI_Bcast(userRating, ratingNum + 1, MPI_DOUBLE, root, MPI_COMM_WORLD);

    // initialize movie/ user matrix
    if (procID == root) {
        initMatrix();
    }

    MPI_Bcast(movieMatrix, numFeatures * movieNum, MPI_DOUBLE, root, MPI_COMM_WORLD);

    // initialize n_user and n_movie
    int* n_user =  (int *)malloc(sizeof(int) * userNum);
    int* n_movie = (int *)malloc(sizeof(int) * movieNum);
    int i, j;
    for (i = 0; i < userNum; ++i) {
        n_user[i] = userStartIdx[i+1] - userStartIdx[i];
    }
    for (i = 0; i < movieNum; ++i) {
        n_movie[i] = movieStartIdx[i+1] - movieStartIdx[i];
    }

    // create gsl objects
    gsl_matrix * M = gsl_matrix_alloc (numFeatures, movieNum);
    gsl_matrix * U = gsl_matrix_alloc (numFeatures, userNum);
    gsl_matrix * R = gsl_matrix_alloc (userNum, movieNum);

    gsl_matrix * A = gsl_matrix_alloc (numFeatures, numFeatures);
    gsl_matrix * Ainv = gsl_matrix_alloc (numFeatures, numFeatures);

    gsl_matrix * E = gsl_matrix_alloc (numFeatures, numFeatures);
    for (i = 0; i < numFeatures; ++i)
        gsl_matrix_set(E, i, i, 1);
    gsl_matrix * F = gsl_matrix_alloc (numFeatures, numFeatures);

    gsl_vector * V = gsl_vector_alloc (numFeatures);

    gsl_vector * Rm = gsl_vector_alloc (movieNum);
    gsl_vector * Ru = gsl_vector_alloc (userNum);

    gsl_permutation *p = gsl_permutation_alloc(numFeatures);
    gsl_vector * O = gsl_vector_alloc (numFeatures);

    /* Start */
    // start iteration
    if (procID == root) {
        printf("Start iteration\n");
        computePredictionRMSE(M, U, R);
    }

    int iter = 0;
    for (iter = 0; iter < numIterations; ++iter) {

        // Each processor solve user feature
        if (procID == root) {
            printf("Updating user (iter = %d)\n", iter);
        }

        int user_idx;
        int user_num = 0;

        // check span
        // NOTE THE NUMBER SHOULD BE DIVISIBLE
        span = (userNum + nproc - 1) / nproc;
        int user_start_idx = min(procID * span, userNum);
        int user_end_idx = min(user_start_idx + span, userNum);

        for (user_idx = user_start_idx; user_idx < user_end_idx; ++user_idx) {
            user_num++;

            gsl_matrix_set_zero (M);
            for (i = userStartIdx[user_idx]; i < userStartIdx[user_idx+1]; ++i)
                for (j = 0; j < numFeatures; ++j)
                    gsl_matrix_set(M, j, i - userStartIdx[user_idx], movieMatrix[(movieId[i]-1) * numFeatures + j]);

            gsl_blas_dgemm(CblasNoTrans, CblasTrans, 1, M, M, 0, A);
            gsl_matrix_memcpy(F, E);
            gsl_matrix_scale(F, lambda * n_user[user_idx]);
            gsl_matrix_add(A, F); // A = M M^T + lambda * n * E

            gsl_vector_set_zero (Rm);
            for (i = userStartIdx[user_idx]; i < userStartIdx[user_idx+1]; ++i)
                gsl_vector_set(Rm, i - userStartIdx[user_idx], movieRating[i]);

            gsl_blas_dgemv(CblasNoTrans, 1, M, Rm, 0, V); // V = M R^T

            int s;
            gsl_linalg_LU_decomp(A, p, &s);
            gsl_linalg_LU_invert(A, p, Ainv);

            gsl_blas_dgemv(CblasNoTrans, 1, Ainv, V, 0, O); // O = A^-1 V

            for (j = 0; j < numFeatures; ++j)
                userMatrix[user_idx * numFeatures + j] = gsl_vector_get(O, j);
        }

        // allgather (everyone gets a local copy of U)
        MPI_Allgather(&userMatrix[user_start_idx * numFeatures],
                      span * numFeatures, MPI_DOUBLE,
                      userMatrix, span * numFeatures,
                      MPI_DOUBLE, MPI_COMM_WORLD);

        // solver movie feature matrix
        if (procID == root) {
            printf("Updating movie (iter = %d)\n", iter);
        }

        int movie_idx;
        int movie_num = 0;

        span = (movieNum + nproc - 1) / nproc;
        int movie_start_idx = min(procID * span, movieNum);
        int movie_end_idx = min(movie_start_idx + span, movieNum);

        for (movie_idx = movie_start_idx; movie_idx < movie_end_idx; ++movie_idx) {
            if (movieStartIdx[movie_idx] == movieStartIdx[movie_idx+1])
                continue;

            movie_num++;
            gsl_matrix_set_zero (U);
            for (i = movieStartIdx[movie_idx]; i < movieStartIdx[movie_idx+1]; ++i)
                for (j = 0; j < numFeatures; ++j)
                    gsl_matrix_set(U, j, i - movieStartIdx[movie_idx], userMatrix[(userId[i]-1) * numFeatures + j]);

            gsl_blas_dgemm(CblasNoTrans, CblasTrans, 1, U, U, 0, A);
            gsl_matrix_memcpy(F, E);
            gsl_matrix_scale(F, lambda * n_movie[movie_idx]);
            gsl_matrix_add(A, F); // A = U U^T + lambda * n * E

            gsl_vector_set_zero (Ru);
            for (i = movieStartIdx[movie_idx]; i < movieStartIdx[movie_idx+1]; ++i)
                gsl_vector_set(Ru, i - movieStartIdx[movie_idx], userRating[i]);

            gsl_blas_dgemv(CblasNoTrans, 1, U, Ru, 0, V); // V = U R^T

            int s;
            gsl_linalg_LU_decomp(A, p, &s);
            gsl_linalg_LU_invert(A, p, Ainv);

            gsl_blas_dgemv(CblasNoTrans, 1, Ainv, V, 0, O); // O = A^-1 V

            for (j = 0; j < numFeatures; ++j)
                movieMatrix[movie_idx * numFeatures + j] = gsl_vector_get(O, j);
        }

        // allgather (everyone gets a local copy of M)
        MPI_Allgather(&movieMatrix[movie_start_idx * numFeatures],
                      span * numFeatures, MPI_DOUBLE,
                      movieMatrix, span * numFeatures,
                      MPI_DOUBLE, MPI_COMM_WORLD);

        // compute prediction and rmse
        if (procID == root) {
            computePredictionRMSE(M, U, R);
        }

    }

    gsl_matrix_free (M);
    gsl_matrix_free (U);
    gsl_matrix_free (R);
    gsl_matrix_free (A);
    gsl_matrix_free (Ainv);
    gsl_matrix_free (E);
    gsl_matrix_free (F);

    gsl_vector_free (V);
    gsl_vector_free (Rm);
    gsl_vector_free (Ru);
    gsl_vector_free (O);

    gsl_permutation_free (p);

    free(n_user);
    free(n_movie);

    free(userStartIdx);
    free(movieId);
    free(movieRating);
    free(movieStartIdx);
    free(userId);
    free(userRating);
    free(movieMatrix);
    free(userMatrix);
}

void computePredictionRMSE(gsl_matrix * M, gsl_matrix * U, gsl_matrix * R) {
    int i, j, user_idx;

    for (i = 0; i < movieNum; ++i)
        for (j = 0; j < info.numFeatures; ++j)
            gsl_matrix_set(M, j, i, movieMatrix[i * info.numFeatures + j]);

    for (i = 0; i < userNum; ++i)
        for (j = 0; j < info.numFeatures; ++j)
            gsl_matrix_set(U, j, i, userMatrix[i * info.numFeatures + j]);

    gsl_blas_dgemm(CblasTrans, CblasNoTrans, 1, U, M, 0, R);

    double rmse = 0;

    for (user_idx = 0; user_idx < userNum; ++user_idx)
        for (i = userStartIdx[user_idx]; i < userStartIdx[user_idx+1]; ++i) {
            double diff = movieRating[i] - gsl_matrix_get(R, user_idx, movieId[i] - 1);
            rmse += diff * diff;
        }

    printf("RMSE = %f\n", rmse / ratingNum);
}


#include <gsl/gsl_combination.h>
#include <gsl/gsl_errno.h>
#include <stdio.h>

int mgsl_combination_fwrite(const char *filename, const gsl_combination *p)
{
  FILE *fp;
  if((fp = fopen(filename, "w")) == NULL) return GSL_EFAILED;
  if(gsl_combination_fwrite(fp, p) != GSL_SUCCESS) return GSL_EFAILED;
  fclose(fp);
  return GSL_SUCCESS;
}

int mgsl_combination_fread(const char *filename, gsl_combination *p)
{
  FILE *fp;
  if((fp = fopen(filename, "r")) == NULL) return GSL_EFAILED;
  if(gsl_combination_fread(fp, p) != GSL_SUCCESS) return GSL_EFAILED;
  fclose(fp);
  return GSL_SUCCESS;
}

int mgsl_combination_fprintf(const char *filename, const gsl_combination *p, const char *format)
{
  FILE *fp;
  if((fp = fopen(filename, "w")) == NULL) return GSL_EFAILED;
  if(gsl_combination_fprintf(fp, p, format) != GSL_SUCCESS) return GSL_EFAILED;
  fclose(fp);
  return GSL_SUCCESS;
}

int mgsl_combination_fscanf(const char *filename, gsl_combination *p)
{
  FILE *fp;
  if((fp = fopen(filename, "r")) == NULL) return GSL_EFAILED;
  if(gsl_combination_fscanf(fp, p) != GSL_SUCCESS) return GSL_EFAILED;
  fclose(fp);
  return GSL_SUCCESS;
}


#ifndef BANDED_FLOAT_IMAGE_H
#define BANDED_FLOAT_IMAGE_H

#include <stdio.h>
#include <sys/types.h>

#include <glib.h>
#include <gsl/gsl_matrix.h>
#include <gsl/gsl_histogram.h>

#include "float_image.h"

typedef struct
{
        int nbands;
        FloatImage **images;
} BandedFloatImage;

BandedFloatImage *
banded_float_image_new(int nbands, size_t size_x, size_t size_y);

BandedFloatImage *
banded_float_image_new_with_value(int nBands, ssize_t size_x, ssize_t size_y,
				  float value);

int
banded_float_image_store (BandedFloatImage *self, const char *file,
			  float_image_byte_order_t byte_order);

void
banded_float_image_free(BandedFloatImage *self);

float
banded_float_image_get_pixel(BandedFloatImage *self, int nband,
                             ssize_t x, ssize_t y);

void
banded_float_image_set_pixel(BandedFloatImage *self, int nband,
                             ssize_t x, ssize_t y, float value);

FloatImage *
banded_float_image_get_band(BandedFloatImage *self, int nband);

BandedFloatImage *
banded_float_image_new_from_model_scaled (BandedFloatImage *model,
                                          ssize_t scale_factor);

void
banded_float_image_export_as_jpeg(BandedFloatImage *self,
                                  const char *output_name);

#endif


#pragma once

#include <gsl\gsl>
#include <winrt\Windows.Foundation.h>
#include <d3d11.h>
#include "DrawableGameComponent.h"
#include "MatrixHelper.h"
#include "DirectionalLight.h"

namespace Library
{
	class Texture2D;
	class ProxyModel;
}

namespace Rendering
{
	class NormalMappingMaterial;

	class NormalMappingDemo final : public Library::DrawableGameComponent
	{
	public:
		NormalMappingDemo(Library::Game& game, const std::shared_ptr<Library::Camera>& camera);
		NormalMappingDemo(const NormalMappingDemo&) = delete;
		NormalMappingDemo(NormalMappingDemo&&) = default;
		NormalMappingDemo& operator=(const NormalMappingDemo&) = default;
		NormalMappingDemo& operator=(NormalMappingDemo&&) = default;
		~NormalMappingDemo();

		bool RealNormalMapEnabled() const;
		void SetRealNormalMapEnabled(bool enabled);
		void ToggleRealNormalMap();

		float AmbientLightIntensity() const;
		void SetAmbientLightIntensity(float intensity);

		float DirectionalLightIntensity() const;
		void SetDirectionalLightIntensity(float intensity);

		const DirectX::XMFLOAT3& LightDirection() const;
		void RotateDirectionalLight(DirectX::XMFLOAT2 amount);

		virtual void Initialize() override;
		virtual void Update(const Library::GameTime& gameTime) override;
		virtual void Draw(const Library::GameTime& gameTime) override;

	private:
		std::shared_ptr<NormalMappingMaterial> mMaterial;
		DirectX::XMFLOAT4X4 mWorldMatrix{ Library::MatrixHelper::Identity };
		winrt::com_ptr<ID3D11Buffer> mVertexBuffer;
		std::uint32_t mVertexCount{ 0 };
		Library::DirectionalLight mDirectionalLight;
		std::unique_ptr<Library::ProxyModel> mProxyModel;
		std::shared_ptr<Library::Texture2D> mRealNormalMap;
		std::shared_ptr<Library::Texture2D> mDefaultNormalMap;
		bool mUpdateMaterial{ true };
		bool mRealNormalMapEnabled{ true };
	};
}


#pragma once

#include <gsl\gsl>
#include <winrt\Windows.Foundation.h>
#include <d3d11.h>
#include "DrawableGameComponent.h"
#include "MatrixHelper.h"
#include "DirectionalLight.h"

namespace Library
{
	class ProxyModel;
}

namespace Rendering
{
	class DiffuseLightingMaterial;

	class DiffuseLightingDemo final : public Library::DrawableGameComponent
	{
	public:
		DiffuseLightingDemo(Library::Game& game, const std::shared_ptr<Library::Camera>& camera);
		DiffuseLightingDemo(const DiffuseLightingDemo&) = delete;
		DiffuseLightingDemo(DiffuseLightingDemo&&) = default;
		DiffuseLightingDemo& operator=(const DiffuseLightingDemo&) = default;
		DiffuseLightingDemo& operator=(DiffuseLightingDemo&&) = default;
		~DiffuseLightingDemo();

		bool AnimationEnabled() const;
		void SetAnimationEnabled(bool enabled);
		void ToggleAnimation();

		float AmbientLightIntensity() const;
		void SetAmbientLightIntensity(float intensity);

		float DirectionalLightIntensity() const;
		void SetDirectionalLightIntensity(float intensity);

		const DirectX::XMFLOAT3& LightDirection() const;
		void RotateDirectionalLight(DirectX::XMFLOAT2 amount);

		virtual void Initialize() override;
		virtual void Update(const Library::GameTime& gameTime) override;
		virtual void Draw(const Library::GameTime& gameTime) override;

	private:
		inline static const float RotationRate{ DirectX::XM_PI };

		std::shared_ptr<DiffuseLightingMaterial> mMaterial;
		DirectX::XMFLOAT4X4 mWorldMatrix{ Library::MatrixHelper::Identity };
		winrt::com_ptr<ID3D11Buffer> mVertexBuffer;
		winrt::com_ptr<ID3D11Buffer> mIndexBuffer;
		std::uint32_t mIndexCount{ 0 };
		Library::DirectionalLight mDirectionalLight;
		std::unique_ptr<Library::ProxyModel> mProxyModel;
		float mModelRotationAngle{ 0.0f };
		bool mAnimationEnabled{ false };
		bool mUpdateMaterial{ true };
	};
}


#include <stdlib.h>
#include <stdio.h>
#include <stdint.h>
#include <dc1394/dc1394.h>
#include <math.h>
#include <fitsio.h>
#include <gsl/gsl_statistics.h>
#include <time.h>
#include <sys/time.h>
#include <string.h>
#include <inttypes.h>

#define REG_CAMERA_ABS_MIN                  0x000U
#define REG_CAMERA_ABS_MAX                  0x004U
#define REG_CAMERA_ABS_VALUE                0x008U

int grab_frame(dc1394camera_t *c, unsigned char *buf, int nbytes) {
  dc1394video_frame_t *frame=NULL;
  dc1394error_t err;

  err = dc1394_capture_dequeue(c, DC1394_CAPTURE_POLICY_WAIT, &frame);
  if (err != DC1394_SUCCESS) {
    dc1394_log_error("Unable to capture.");
    dc1394_capture_stop(c);
    dc1394_camera_free(c);
    exit(1);
  }

  memcpy(buf, frame->image, nbytes);
  dc1394_capture_enqueue(c, frame);
  return 1;
}

int main(int argc, char *argv[]) {

  long nelements, naxes[2];
  dc1394_t * dc;
  dc1394camera_t * camera;
  dc1394camera_list_t * list;
  dc1394error_t err;
  dc1394video_mode_t mode;
  unsigned int max_height, max_width, winleft, wintop;

  unsigned char *buffer, *buffer2;
  double *average, *diff;
  int j, f, status, nimages, xsize, ysize;
  double mean, var, exp;
  struct timeval start_time, end_time;
  time_t start_sec, end_sec;
  suseconds_t start_usec, end_usec;
  float elapsed_time, fps;

  stderr = freopen("noise_test.log", "w", stderr);

  srand48((unsigned)time(NULL));

  nimages = 0;
  status = 0;
  xsize = 320;
  ysize = 240;
  naxes[0] = xsize;
  naxes[1] = ysize;

  nelements = naxes[0]*naxes[1];

  mode = DC1394_VIDEO_MODE_FORMAT7_1;
  dc = dc1394_new();
  if (!dc)
    return -1;
  err = dc1394_camera_enumerate(dc, &list);
  DC1394_ERR_RTN(err, "Failed to enumerate cameras.");

  if (list->num == 0) {
    dc1394_log_error("No cameras found.");
    return -1;
  }

  camera = dc1394_camera_new(dc, list->ids[0].guid);
  if (!camera) {
    dc1394_log_error("Failed to initialize camera with guid %"PRIx64".",
                     list->ids[0].guid);
    return -1;
  }
  dc1394_camera_free_list(list);

  printf("Using camera with GUID %"PRIx64"\n", camera->guid);

  // need to use legacy firewire400 mode for now.  800 not quite reliable.
  dc1394_video_set_iso_speed(camera, DC1394_ISO_SPEED_400);

  // configure camera for format7
  err = dc1394_video_set_mode(camera, mode);
  DC1394_ERR_CLN_RTN(err, dc1394_camera_free(camera), "Can't choose video mode.");

  err = dc1394_format7_get_max_image_size(camera, mode, &max_width, &max_height);
  DC1394_ERR_CLN_RTN(err,dc1394_camera_free (camera),"cannot get max image size.");

  winleft = 0;
  wintop = 0;

  err = dc1394_format7_set_roi(camera,
                               mode,
                               DC1394_COLOR_CODING_MONO8,
                               DC1394_USE_MAX_AVAIL,
                               //				 packet,
                               winleft, wintop, // left, top
                               xsize, ysize);
  DC1394_ERR_CLN_RTN(err, dc1394_camera_free(camera), "Can't set ROI.");

  err=dc1394_video_set_framerate(camera, DC1394_FRAMERATE_MAX);
  DC1394_ERR_CLN_RTN(err,dc1394_camera_free (camera),"cannot set framerate");

  err = dc1394_feature_set_mode (camera, DC1394_FEATURE_SHUTTER, DC1394_FEATURE_MODE_MANUAL);
  DC1394_ERR_CLN_RTN(err,dc1394_camera_free (camera),"cannot set shutter to manual");

  err = dc1394_feature_set_value (camera, DC1394_FEATURE_SHUTTER, 61);
  DC1394_ERR_CLN_RTN(err,dc1394_camera_free (camera),"cannot set shutter");

  err = dc1394_feature_set_mode (camera, DC1394_FEATURE_GAIN, DC1394_FEATURE_MODE_MANUAL);
  DC1394_ERR_CLN_RTN(err,dc1394_camera_free (camera),"cannot set gain to manual");

  err = dc1394_feature_set_value (camera, DC1394_FEATURE_GAIN, 48);
  DC1394_ERR_CLN_RTN(err,dc1394_camera_free (camera),"cannot set gain");

  // set brightness manually.  use relative value in range 0 to 1023.
  err = dc1394_feature_set_mode(camera, DC1394_FEATURE_BRIGHTNESS, DC1394_FEATURE_MODE_MANUAL);
  DC1394_ERR_CLN_RTN(err,dc1394_camera_free (camera),"cannot set brightness to manual");
  err = dc1394_feature_set_value(camera, DC1394_FEATURE_BRIGHTNESS, 500);
  DC1394_ERR_CLN_RTN(err,dc1394_camera_free (camera),"cannot set brightness");
  printf ("I: brightness is %d\n", 500);

  err = dc1394_capture_setup(camera, 16, DC1394_CAPTURE_FLAGS_DEFAULT);
  DC1394_ERR_CLN_RTN(err, dc1394_camera_free(camera), "Error capturing.");

  // start the camera up
  err = dc1394_video_set_transmission(camera, DC1394_ON);
  if (err != DC1394_SUCCESS) {
    dc1394_log_error("Unable to start camera iso transmission.");
    dc1394_capture_stop(camera);
    dc1394_camera_free(camera);
    return -1;
  }

  printf("Camera successfully initialized.\n");

  if (!(buffer = malloc(nelements*sizeof(char)))) {
    printf("Couldn't Allocate Image Buffer\n");
    exit(-1);
  }

  if (!(buffer2 = malloc(nelements*sizeof(char)))) {
    printf("Couldn't Allocate Image2 Buffer\n");
    exit(-1);
  }

  if (!(average = calloc(nelements, sizeof(double)))) {
    printf("Couldn't Allocate Average Image Buffer\n");
    exit(-1);
  }

  if (!(diff = calloc(nelements, sizeof(double)))) {
    printf("Couldn't Allocate Diff Image Buffer\n");
    exit(-1);
  }

  gettimeofday(&start_time, NULL);

  f = 0;

  err = dc1394_feature_set_absolute_control(camera, DC1394_FEATURE_SHUTTER, DC1394_TRUE);
  DC1394_ERR_CLN_RTN(err,dc1394_camera_free (camera),"cannot set shutter to absolute mode");
  exp = 8.0*1.0e-5;
  err = dc1394_feature_set_absolute_value(camera, DC1394_FEATURE_SHUTTER, exp);
  grab_frame(camera, buffer, nelements*sizeof(char));
  grab_frame(camera, buffer, nelements*sizeof(char));

  for (f=8; f<320; f++) {
    exp = f*1.0e-5;
    err = dc1394_feature_set_absolute_value(camera, DC1394_FEATURE_SHUTTER, exp);
    DC1394_ERR_CLN_RTN(err,dc1394_camera_free (camera),"cannot set shutter");

    grab_frame(camera, buffer, nelements*sizeof(char));
    grab_frame(camera, buffer2, nelements*sizeof(char));
    for (j=0; j<nelements; j++) {
      average[j] = (buffer[j]+buffer2[j])/2.0;
    }
    for (j=0; j<nelements; j++) {
      diff[j] = buffer[j] - buffer2[j];
    }

    var = gsl_stats_variance(diff, 1, nelements);
    mean = gsl_stats_mean(average, 1, nelements);

    var = var/2.0;

    printf("%e %10.2f %10.3f\n", exp, mean, var);

  }

  gettimeofday(&end_time, NULL);
  printf("End capture.\n");

  start_sec = start_time.tv_sec;
  start_usec = start_time.tv_usec;
  end_sec = end_time.tv_sec;
  end_usec = end_time.tv_usec;

  elapsed_time = (float)((end_sec + 1.0e-6*end_usec) - (start_sec + 1.0e-6*start_usec));
  fps = nimages/elapsed_time;
  printf("Elapsed time = %g seconds.\n", elapsed_time);
  printf("Framerate = %g fps.\n", fps);

  /*-----------------------------------------------------------------------
   *  stop data transmission
   *-----------------------------------------------------------------------*/
  err=dc1394_video_set_transmission(camera,DC1394_OFF);
  DC1394_ERR_RTN(err,"couldn't stop the camera?");

  return (status);
}


#include <api/components.h>
#include <api/kvstore_itf.h>
#include <common/string_view.h>
#include <common/utils.h> /* MiB */
#include <gsl/pointers> /* not_null */

#include <cstddef> /* size_t */
#include <memory> /* unique_ptr */
#include <stdexcept> /* runtime_error */
#include <string>

namespace
{
	/* things which differ depending on the type of store used */
	struct custom_store
	{
		virtual ~custom_store() {}
		virtual std::size_t minimum_size(std::size_t s) const { return s; }
		virtual component::uuid_t factory() const = 0;
		virtual std::size_t presumed_allocation() const = 0;
		virtual bool uses_numa_nodes() const { return false; }
		virtual status_t rc_unknown_attribute() const { return E_NOT_SUPPORTED; }
		virtual status_t rc_percent_used() const { return S_OK; }
		virtual status_t rc_resize_locked() const { return E_LOCKED; }
		virtual status_t rc_attribute_key_null_ptr() const { return E_BAD_PARAM; }
		virtual status_t rc_attribute_key_not_found() const { return component::IKVStore::E_KEY_NOT_FOUND; }
		virtual status_t rc_attribute_hashtable_expansion() const { return S_OK; }
		virtual status_t rc_out_of_memory() const { return component::IKVStore::E_TOO_LARGE; }
		virtual status_t rc_atomic_update() const { return S_OK; }
		virtual status_t rc_allocate_pool_memory_size_0() const { return S_OK; }
		virtual bool swap_updates_timestamp() const { return false; }
	};

	struct custom_mapstore
		: public custom_store
	{
		virtual component::uuid_t factory() const override { return component::mapstore_factory; }
		std::size_t minimum_size(std::size_t s) const override { return std::max(std::size_t(8), s); }
		std::size_t presumed_allocation() const override { return 1ULL << DM_REGION_LOG_GRAIN_SIZE; }
		bool uses_numa_nodes() const override { return true; }
		status_t rc_unknown_attribute() const override { return E_INVALID_ARG; }
		status_t rc_percent_used() const override { return rc_unknown_attribute(); }
		status_t rc_resize_locked() const override { return E_INVAL; }
		status_t rc_attribute_key_null_ptr() const override { return E_INVALID_ARG; }
		status_t rc_attribute_key_not_found() const override { return rc_unknown_attribute(); }
		status_t rc_attribute_hashtable_expansion() const override { return rc_unknown_attribute(); }
		status_t rc_out_of_memory() const override { return E_INVAL; }
		status_t rc_atomic_update() const override { return E_NOT_SUPPORTED; }
		status_t rc_allocate_pool_memory_size_0() const override { return E_INVAL; }
	};

	struct custom_hstore
		: public custom_store
	{
		virtual component::uuid_t factory() const override { return component::hstore_factory; }
		std::size_t presumed_allocation() const override { return MiB(32); }
		bool swap_updates_timestamp() const override { return true; }
	};

	struct custom_hstore_cc
		: public custom_hstore
	{
		std::size_t presumed_allocation() const override { return MiB(32); }
	};

	custom_mapstore custom_mapstore_i{};
	custom_hstore custom_hstore_i{};
	custom_hstore_cc custom_hstore_cc_i{};

	const std::map<std::string, gsl::not_null<custom_store *>, std::less<>> custom_map =
	{
		{ "mapstore", &custom_mapstore_i },
		{ "hstore", &custom_hstore_i },
		{ "hstore-cc", &custom_hstore_cc_i },
		{ "hstore-mc", &custom_hstore_i },
		{ "hstore-mr", &custom_hstore_i },
		{ "hstore-mm", &custom_hstore_i },
	};
}

gsl::not_null<custom_store *> locate_custom_store(common::string_view store)
{
	const auto c_it = custom_map.find(store);
	if ( c_it == custom_map.end() )
    {
		throw std::runtime_error(common::format("store {} not recognized", store));
    }
	return c_it->second;
}

auto make_kvstore(
	common::string_view store
	, gsl::not_null<custom_store *> c
	, const component::IKVStore_factory::map_create & mc
) -> std::unique_ptr<component::IKVStore>
{
	using IKVStore_factory = component::IKVStore_factory;
	const std::string store_lib = "libcomponent-" + std::string(store) + ".so";
	auto *comp = component::load_component(store_lib.c_str(), c->factory());
	if ( ! comp )
    {
		throw std::runtime_error(common::format("failed to load component {}", store_lib));
    }

	const auto fact = make_itf_ref(static_cast<IKVStore_factory*>(comp->query_interface(IKVStore_factory::iid())));
	const auto kvstore = fact->create(0, mc);

	return std::unique_ptr<component::IKVStore>(kvstore);
}


#pragma once
#include <array>
#include <variant>
#include <gsl/gsl-lite.hpp>

#include <clrie/method_info.h>
#include <clrie/instruction_factory.h>

#include <corhdr.h>

#include "cil.h"
#include "signature.h"

namespace appmap {
    template <typename T>
    constexpr COR_SIGNATURE type_signature() {
        if constexpr (std::is_same_v<T, void>) {
            return ELEMENT_TYPE_VOID;
        } else if constexpr (std::is_same_v<T, int32_t>) {
            return ELEMENT_TYPE_I4;
        } else if constexpr (std::is_same_v<T, int64_t>) {
            return ELEMENT_TYPE_I8;
        } else if constexpr (std::is_same_v<T, uint32_t>) {
            return ELEMENT_TYPE_U4;
        } else if constexpr (std::is_same_v<T, uint64_t>) {
            return ELEMENT_TYPE_U8;
        } else if constexpr (std::is_same_v<T, char *> || std::is_same_v<T, const char *>) {
            return ELEMENT_TYPE_STRING;
        } else if constexpr (std::is_pointer_v<T>) {
            static_assert(sizeof(void *) == 8);
            return ELEMENT_TYPE_I8;
        } else if constexpr (std::is_same_v<T, bool>) {
            return ELEMENT_TYPE_BOOLEAN;
        } else {
            static_assert(std::is_same_v<T, void>, "unhandled native type");
        }
    }

    template <typename F>
    struct func_traits;

    template <typename Ret, typename... Args>
    struct func_traits<Ret(*)(Args...)>
    {
        static constexpr std::size_t arity = sizeof...(Args);
        static constexpr std::array<COR_SIGNATURE, 3 + arity> signature = {
            IMAGE_CEE_UNMANAGED_CALLCONV_STDCALL,
            arity,
            type_signature<Ret>(),
            type_signature<Args>()...
        };
    };

    template <typename C, typename Ret, typename... Args>
    struct func_traits<Ret(C::*)(Args...)>
    {
        using instance_type = C;
        static constexpr std::size_t arity = sizeof...(Args);
        static constexpr std::array<COR_SIGNATURE, 4 + arity> signature = {
            IMAGE_CEE_UNMANAGED_CALLCONV_STDCALL,
            arity + 1,
            type_signature<Ret>,
            type_signature<C*>,
            type_signature<Args>...
        };
    };

    struct instrumentation : public clrie::instruction_factory {
        instrumentation(clrie::method_info method_info) :
            clrie::instruction_factory(method_info.instruction_factory()),
            method(method_info),
            module(method.module_info()),
            module_id(module.module_id()),
            metadata(module.meta_data_emit()),
            type_factory(module.create_type_factory()),
            locals(method.get(&IMethodInfo::GetLocalVariables))
        {}

        clrie::method_info method;
        clrie::module_info module;
        const ModuleID module_id;
        com::ptr<IMetaDataEmit> metadata;
        com::ptr<ITypeCreator> type_factory;
        com::ptr<ILocalVariableCollection> locals;

        static com::ptr<ISignatureBuilder> signature_builder;

        template <class F>
        instruction_sequence make_call(F f) const {
            return make_call_sig(reinterpret_cast<void *>(f), gsl::make_span(func_traits<F>::signature));
        }

        instruction_sequence create_call_to_string(const clrie::type &type) const noexcept;

        // Note capture_value takes a reference; in case of a composite type, it dereferences
        // it to a primitive that can be then passed onto the correct native function.
        // The argument is updated to reflect the resulting simple type.
        instruction_sequence capture_value(clrie::type &type) const noexcept;

        template <typename T>
        uint64_t add_local()
        {
            auto t = type_factory.get(&ITypeCreator::FromCorElement, static_cast<CorElementType>(type_signature<T>()));
            return locals.get(&ILocalVariableCollection::AddLocal, t);
        }

        // emit metadata and return reference tokens

        mdAssemblyRef assembly_reference(const char16_t *assembly);

        mdTypeRef type_reference(mdAssemblyRef assembly, const char16_t *type);
        mdTypeRef type_reference(const char16_t *assembly, const char16_t *type) {
            return type_reference(assembly_reference(assembly), type);
        }

        mdMemberRef member_reference(mdTypeRef type, const char16_t *member,
            gsl::span<const COR_SIGNATURE> signature);

        mdMemberRef member_reference(mdAssemblyRef assembly, const char16_t *type,
            const char16_t *member, gsl::span<const COR_SIGNATURE> signature) {
            return member_reference(type_reference(assembly, type), member, signature);
        }

        mdMemberRef member_reference(const char16_t *assembly, const char16_t *type,
            const char16_t *member, gsl::span<const COR_SIGNATURE> signature) {
            return member_reference(type_reference(assembly, type), member, signature);
        }

        mdTypeDef define_type(const char16_t *name);
        mdFieldDef define_field(mdTypeDef type, const char16_t *name, gsl::span<const COR_SIGNATURE> signature);

        mdMethodDef define_method(
            mdTypeDef type,
            const char16_t *name,
            gsl::span<const COR_SIGNATURE> signature,
            std::initializer_list<appmap::signature::type> locals,
            std::vector<cil::instruction> code
        );

        mdMethodDef define_method(mdTypeDef type, const char16_t *name, gsl::span<const COR_SIGNATURE> signature, std::vector<cil::instruction> code) {
            return define_method(type, name, signature, {}, std::move(code));
        }

        template <class Ret, class... Args>
        mdTypeSpec native_type(Ret (*)(Args...)) {
            constexpr auto sig = func_traits<Ret(*)(Args...)>::signature;
            return metadata.get(&IMetaDataEmit::GetTokenFromSig, sig.data(), sig.size());
        }

        mdTypeSpec type_token(gsl::span<const COR_SIGNATURE> sig) {
            return metadata.get(&IMetaDataEmit::GetTokenFromTypeSpec, sig.data(), sig.size());
        }

    protected:
        instruction_sequence make_call_sig(void *fn, gsl::span<const COR_SIGNATURE> signature) const;
    };
}


/*
* Copyright 2019 Virtru Corporation
*
* SPDX - License Identifier: BSD-3-Clause-Clear
*
*/
//
//  TDF SDK
//
//  Created by Sujan Reddy on 2019/04/07.
//

#ifndef VIRTRU_BYTES_H
#define VIRTRU_BYTES_H

#include <gsl/span>
#include <array>

/// This provides a simple wrapper around gsl to work with bytes.
namespace virtru::crypto {

    /// Constants
    constexpr auto kKeyLength = 32;
    constexpr auto kIVLength = 16;

   ///
   /// Define the bytes(Read only).
   ///
   template <std::ptrdiff_t Extent = gsl :: dynamic_extent>
   using BytesT = gsl::span <const gsl::byte, Extent>;
   using Bytes = BytesT<>;

   ///
   /// Wrapper around std::array
   ///
   template <std::size_t N>
   using ByteArray = std::array <gsl :: byte, N>;
   using WrappedKey = ByteArray<kKeyLength>;
   using IV = ByteArray<kIVLength>;

    ///
    /// Define writeable bytes
    ///
    template <std::ptrdiff_t Extent = gsl::dynamic_extent>
    using WriteableBytesT = gsl::span <gsl::byte, Extent>;
    using WriteableBytes = WriteableBytesT<>;

    template <std :: ptrdiff_t Extent >
    constexpr auto toBytes(BytesT < Extent > data) noexcept {
        return data;
    }

    template <std::ptrdiff_t Extent>
    constexpr auto toWriteableBytes(WriteableBytesT <Extent> data) noexcept {
        return data;
    }

    /// Adding constness
    template <std::ptrdiff_t Extent>
    constexpr auto toBytes (gsl::span <gsl::byte, Extent> data) noexcept {
        return BytesT <Extent> {data};
    }

    /// Any containers inherited from basic_string<ElementType> to bytes(Read only).
    template <typename ElementType,
              std::ptrdiff_t Extent,
              typename = std::enable_if_t <std::has_unique_object_representations_v <ElementType>> >
    auto toBytes (gsl::span <ElementType, Extent> data) noexcept {
        return as_bytes(data);
    }

    /// Any containers inherited from basic_string<ElementType> to writable bytes.
    template <typename ElementType,
              std::ptrdiff_t Extent,
              typename = std::enable_if_t <std::has_unique_object_representations_v <ElementType>> >
    auto toWriteableBytes(gsl::span <ElementType, Extent> data) noexcept {
        return as_writeable_bytes(data);
    }

    /// Form std::array to bytes(Read only).
    template <typename ElementType,
              std::size_t N,
              typename = std::enable_if_t <std::has_unique_object_representations_v <ElementType>> >
    constexpr auto toBytes(const std::array <ElementType, N> & data) noexcept {
        return toBytes(gsl::span <const ElementType, N > { data });
    }

    /// Form std::array to writable bytes.
    template <typename ElementType,
              std::size_t N,
              typename = std::enable_if_t <std::has_unique_object_representations_v <ElementType>> >
    constexpr auto toWriteableBytes(std::array <ElementType, N> & data) noexcept {
        return toWriteableBytes(gsl::span <ElementType, N> { data });
    }

    // Form Plain old C array to bytes(Read only).
    template <typename ElementType,
              std::size_t N,
              typename = std::enable_if_t <std::has_unique_object_representations_v <ElementType>> >
    constexpr auto toBytes(ElementType (& arr)[N]) noexcept {
        return toBytes(gsl::make_span(arr));
    }

    // Form Plain old C array to writable bytes.
    template <typename ElementType,
              std::size_t N,
              typename = std::enable_if_t <std::has_unique_object_representations_v <ElementType>> >
    constexpr auto toWriteableBytes(ElementType(&arr)[N]) noexcept {
        return toWriteableBytes(gsl::make_span(arr));
    }

    // From container(std::string/std::vector<char>) to bytes(Read only).
    template <class Cont,
              typename = std::enable_if_t <std::has_unique_object_representations_v <std::remove_reference_t <
                                           decltype(* std::declval <const Cont&> ().data ())>> > >
    constexpr auto toBytes(const Cont & cont) noexcept {
        return toBytes(gsl::make_span(cont));
    }

    // From container(std::string/std::vector<char>) to to writable bytes.
    template <class Cont,
              typename = std::enable_if_t <std::has_unique_object_representations_v <std::remove_reference_t <
                                           decltype(* std::declval <Cont &> ().data ())>> > >
    constexpr auto toWriteableBytes(Cont & cont) noexcept {
        return toWriteableBytes(gsl::make_span(cont));
    }

    ///
    /// Simple conversions.
    ///
    template <typename T>
    using ConvertibleToBytes = decltype(toBytes(std::declval <const T&> ()));

    template <typename T, std::ptrdiff_t Extent = gsl::dynamic_extent>
    inline constexpr bool isBytes = std::is_convertible_v <const T&, BytesT <Extent>>;

    template <typename T>
    using ExplicitlyConvertibleToBytes = std::enable_if_t <! isBytes <T>, ConvertibleToBytes <T>>;

    template <typename T, typename = ConvertibleToBytes <T>>
    auto toDynamicBytes (const T& t) {
        return Bytes { toBytes(t) };
    }

    template <typename T, typename = ExplicitlyConvertibleToBytes <T>>
    auto toDynamicBytesExplicitly (const T& t) {
        return toDynamicBytes(t);
    }

    inline unsigned char* toUchar(gsl::byte* p) {
        return reinterpret_cast <unsigned char *> (p);
    }

    inline const unsigned char *toUchar (const gsl::byte *p) {
        return reinterpret_cast <const unsigned char *> (p);
    }

    template <std::ptrdiff_t Extent>
    auto toUchar(BytesT<Extent> src) {
        return gsl::span <const unsigned char, Extent> {reinterpret_cast <const unsigned char *> (src.data ()), src.size()};
    }

    inline char* toChar(gsl::byte *p) {
        return reinterpret_cast <char*> (p);
    }

    inline const char* toChar(const gsl::byte* p) {
        return reinterpret_cast <const char *> (p);
    }

    template <std::ptrdiff_t Extent>
    auto toChar(BytesT <Extent> src) {
        return gsl :: span <const char, Extent> { reinterpret_cast <const char*> (src.data()), src.size ()};
    }

    template <std::ptrdiff_t Extent>
    auto toChar(WriteableBytesT <Extent> src) {
        return gsl::span <const char, Extent> { reinterpret_cast <const char *> (src.data()), src.size ()};
    }

    inline auto finalizeSize(WriteableBytes& buffer, const int& size) {
        return gsl::finally( [&buffer, &size ] { buffer = buffer.first(size); });
    }
} // namespace virtru::crypto

#endif //VIRTRU_BYTES_H


#include "ccv_nnc.h"
#include "ccv_nnc_easy.h"
#include "ccv_nnc_internal.h"
#include "ccv_internal.h"
#include "3rdparty/khash/khash.h"
#ifdef HAVE_GSL
#include <gsl/gsl_rng.h>
#include <gsl/gsl_randist.h>
#else
#include "3rdparty/sfmt/SFMT.h"
#endif

KHASH_MAP_INIT_INT64(ctx, ccv_array_t*)

struct ccv_cnnp_dataframe_s {
	int row_count;
	int column_size;
	int* shuffled_idx;
#ifdef HAVE_GSL
	gsl_rng* rng;
#else
	sfmt_t sfmt;
#endif
	khash_t(ctx)* data_ctx; // The stream context based cache for data entity of columns. This helps us to avoid allocations when iterate through data.
	ccv_array_t* derived_column_data;
	ccv_cnnp_column_data_t column_data[1];
};

typedef struct {
	int stream_type;
	int column_idx_size;
	int* column_idxs;
	ccv_cnnp_column_data_enum_f data_enum;
	ccv_cnnp_column_data_deinit_f data_deinit;
	void* context;
	ccv_cnnp_column_data_context_deinit_f context_deinit;
	ccv_cnnp_column_data_map_f map;
} ccv_cnnp_derived_column_data_t;

ccv_cnnp_dataframe_t* ccv_cnnp_dataframe_new(const ccv_cnnp_column_data_t* const column_data, const int column_size, const int row_count)
{
	assert(column_size >= 0);
	ccv_cnnp_dataframe_t* const dataframe = (ccv_cnnp_dataframe_t*)cccalloc(1, sizeof(ccv_cnnp_dataframe_t) + sizeof(ccv_cnnp_column_data_t) * (column_size - 1));
	dataframe->row_count = row_count;
	dataframe->column_size = column_size;
	dataframe->data_ctx = kh_init(ctx);
	if (column_size > 0)
		memcpy(dataframe->column_data, column_data, sizeof(ccv_cnnp_column_data_t) * column_size);
	return dataframe;
}

void ccv_cnnp_dataframe_shuffle(ccv_cnnp_dataframe_t* const dataframe)
{
	assert(dataframe->row_count);
	int i;
	if (!dataframe->shuffled_idx)
	{
		dataframe->shuffled_idx = (int*)ccmalloc(sizeof(int) * dataframe->row_count);
		for (i = 0; i < dataframe->row_count; i++)
			dataframe->shuffled_idx[i] = i;
#ifdef HAVE_GSL
		assert(!dataframe->rng);
		gsl_rng_env_setup();
		dataframe->rng = gsl_rng_alloc(gsl_rng_default);
		gsl_rng_set(dataframe->rng, (unsigned long int)dataframe);
#else
		sfmt_init_gen_rand(&dataframe->sfmt, (uint32_t)dataframe);
#endif
	}
#ifdef HAVE_GSL
	gsl_ran_shuffle(dataframe->rng, dataframe->shuffled_idx, dataframe->row_count, sizeof(int));
#else
	sfmt_genrand_shuffle(&dataframe->sfmt, dataframe->shuffled_idx, dataframe->row_count, sizeof(int));
#endif
}

int ccv_cnnp_dataframe_row_count(ccv_cnnp_dataframe_t* const dataframe)
{
	return dataframe->row_count;
}

int ccv_cnnp_dataframe_add(ccv_cnnp_dataframe_t* const dataframe, ccv_cnnp_column_data_enum_f data_enum, const int stream_type, ccv_cnnp_column_data_deinit_f data_deinit, void* const context, ccv_cnnp_column_data_context_deinit_f context_deinit)
{
	if (!dataframe->derived_column_data)
		dataframe->derived_column_data = ccv_array_new(sizeof(ccv_cnnp_derived_column_data_t), 1, 0);
	ccv_cnnp_derived_column_data_t column_data = {
		.stream_type = stream_type,
		.data_enum = data_enum,
		.data_deinit = data_deinit,
		.context = context,
		.context_deinit = context_deinit,
	};
	ccv_array_push(dataframe->derived_column_data, &column_data);
	return dataframe->column_size + dataframe->derived_column_data->rnum - 1;
}

int ccv_cnnp_dataframe_map(ccv_cnnp_dataframe_t* const dataframe, ccv_cnnp_column_data_map_f map, const int stream_type, ccv_cnnp_column_data_deinit_f data_deinit, const int* const column_idxs, const int column_idx_size, void* const context, ccv_cnnp_column_data_context_deinit_f context_deinit)
{
	assert(column_idx_size > 0);
	if (!dataframe->derived_column_data)
		dataframe->derived_column_data = ccv_array_new(sizeof(ccv_cnnp_derived_column_data_t), 1, 0);
	const int column_size = dataframe->column_size + dataframe->derived_column_data->rnum;
	int i;
	for (i = 0; i < column_idx_size; i++)
		{ assert(column_idxs[i] < column_size); }
	ccv_cnnp_derived_column_data_t column_data = {
		.stream_type = stream_type,
		.column_idx_size = column_idx_size,
		.column_idxs = (int*)ccmalloc(sizeof(int) * column_idx_size),
		.map = map,
		.data_deinit = data_deinit,
		.context = context,
		.context_deinit = context_deinit,
	};
	memcpy(column_data.column_idxs, column_idxs, sizeof(int) * column_idx_size);
	ccv_array_push(dataframe->derived_column_data, &column_data);
	return dataframe->column_size + dataframe->derived_column_data->rnum - 1;
}

void* ccv_cnnp_dataframe_column_context(const ccv_cnnp_dataframe_t* const dataframe, const int column_idx)
{
	assert(column_idx >= 0);
	const int column_size = dataframe->column_size + (dataframe->derived_column_data ? dataframe->derived_column_data->rnum : 0);
	assert(column_idx < column_size);
	if (column_idx < dataframe->column_size)
		return dataframe->column_data[column_idx].context;
	assert(dataframe->derived_column_data);
	ccv_cnnp_derived_column_data_t* const derived_column_data = (ccv_cnnp_derived_column_data_t*)ccv_array_get(dataframe->derived_column_data, column_idx - dataframe->column_size);
	return derived_column_data->context;
}

typedef struct {
	int flag; // Mark this as cached or not.
	uint64_t ctx; // The stream context.
	void* data;
} ccv_cnnp_dataframe_data_item_t;

typedef struct {
	ccv_nnc_stream_context_t* stream_context;
	ccv_nnc_stream_signal_t* signal;
} ccv_cnnp_dataframe_column_ctx_t;

KHASH_MAP_INIT_INT64(iter_ctx, ccv_cnnp_dataframe_column_ctx_t*)

struct ccv_cnnp_dataframe_iter_s {
	int idx;
	int prefetch_head;
	int prefetch_tail;
	int column_idx_size;
	int fetched_size; // The size of fetched data.
	ccv_cnnp_dataframe_t* dataframe;
	void**** derived_data; // This is ridiculous, but it is true.
	void** fetched_data; // The cache to store fetched data.
	khash_t(iter_ctx)* column_ctx; // Column context specific to a stream context. The key will be a parent stream context and value will be child stream context + signal.
	ccv_array_t* prefetches; // The prefetch contents.
	int* column_idxs;
	ccv_cnnp_dataframe_data_item_t cached_data[1]; // The data cached when deriving data.
};

#define INDEX_DATA(iter) ((int*)((iter)->fetched_data))
#define FETCHED_DATA(iter, idx) ((iter)->fetched_data + ((idx) + 1) * (iter)->fetched_size)

ccv_cnnp_dataframe_iter_t* ccv_cnnp_dataframe_iter_new(ccv_cnnp_dataframe_t* const dataframe, const int* const column_idxs, const int column_idx_size)
{
	assert(column_idx_size > 0);
	const int column_size = dataframe->column_size + (dataframe->derived_column_data ? dataframe->derived_column_data->rnum : 0);
	int i;
	for (i = 0; i < column_idx_size; i++)
		{ assert(column_idxs[i] < column_size); }
	ccv_cnnp_dataframe_iter_t* const iter = (ccv_cnnp_dataframe_iter_t*)cccalloc(1, sizeof(ccv_cnnp_dataframe_iter_t) + sizeof(ccv_cnnp_dataframe_data_item_t) * column_size + sizeof(void*) * (column_idx_size - 1) + sizeof(int) * column_idx_size);
	iter->dataframe = dataframe;
	iter->prefetch_tail = -1;
	iter->column_idx_size = column_idx_size;
	iter->column_idxs = (int*)(iter->cached_data + column_size);
	memcpy(iter->column_idxs, column_idxs, sizeof(int) * column_idx_size);
	// Preallocate fetched data.
	iter->fetched_size = 1;
	iter->fetched_data = (void**)ccmalloc(sizeof(void*) * (column_size + 1));
	return iter;
}

static void _ccv_cnnp_dataframe_enqueue_data(ccv_cnnp_dataframe_t* const dataframe, void* const data, const int column_idx, const uint64_t ctx)
{
	if (!data)
		return;
	khash_t(ctx)* const data_ctx = dataframe->data_ctx;
	int ret = 0;
	khiter_t k = kh_put(ctx, data_ctx, ctx, &ret);
	assert(ret >= 0);
	const int column_size = dataframe->column_size + (dataframe->derived_column_data ? dataframe->derived_column_data->rnum : 0);
	assert(column_idx < column_size);
	// If ret == 0, the key already exist, we can get the columns directly, otherwise, create and assign back.
	ccv_array_t* const columns = (ret == 0) ? kh_val(data_ctx, k) : ccv_array_new(sizeof(ccv_array_t*), column_size, 0);
	if (ret != 0)
		kh_val(data_ctx, k) = columns;
	if (columns->rnum < column_size)
		ccv_array_resize(columns, column_size);
	ccv_array_t* column = *(ccv_array_t**)ccv_array_get(columns, column_idx);
	if (!column)
	{
		column = ccv_array_new(sizeof(void*), 1, 0);
		*(ccv_array_t**)ccv_array_get(columns, column_idx) = column;
	}
	ccv_array_push(column, &data);
}

static void* _ccv_cnnp_dataframe_dequeue_data(ccv_cnnp_dataframe_t* const dataframe, const int column_idx, ccv_nnc_stream_context_t* const stream_context)
{
	const uint64_t ctx = (uint64_t)(uintptr_t)stream_context;
	khash_t(ctx)* const data_ctx = dataframe->data_ctx;
	khiter_t k = kh_get(ctx, data_ctx, ctx);
	if (k == kh_end(data_ctx))
		return 0;
	ccv_array_t* const columns = kh_val(data_ctx, k);
	if (column_idx >= columns->rnum)
		return 0;
	ccv_array_t* const column = *(ccv_array_t**)ccv_array_get(columns, column_idx);
	if (!column || column->rnum == 0)
		return 0;
	void* const data = *(void**)ccv_array_get(column, column->rnum - 1);
	--column->rnum;
	return data;
}

static ccv_cnnp_dataframe_column_ctx_t _ccv_cnnp_child_column_ctx_for_stream_type(ccv_cnnp_dataframe_t* const dataframe, ccv_cnnp_dataframe_iter_t* const iter, const int column_idx, ccv_nnc_stream_context_t* const stream_context, const int stream_type)
{
	ccv_cnnp_dataframe_column_ctx_t child_ctx = {
		.stream_context = stream_context,
	};
	if (stream_context && ccv_nnc_stream_context_type(stream_context) != stream_type && stream_type != 0)
	{
		if (!iter->column_ctx)
			iter->column_ctx = kh_init(iter_ctx);
		khash_t(iter_ctx)* const column_ctx = iter->column_ctx;
		int ret = 0;
		khiter_t k = kh_put(iter_ctx, column_ctx, (uint64_t)(uintptr_t)stream_context, &ret);
		assert(ret >= 0);
		const int column_size = dataframe->column_size + (dataframe->derived_column_data ? dataframe->derived_column_data->rnum : 0);
		ccv_cnnp_dataframe_column_ctx_t* const ctx = (ret == 0) ? kh_val(column_ctx, k) : cccalloc(column_size, sizeof(ccv_cnnp_dataframe_column_ctx_t));
		if (ret != 0)
			kh_val(column_ctx, k) = ctx;
		if (!ctx[column_idx].stream_context)
			ctx[column_idx].stream_context = ccv_nnc_stream_context_new(stream_type);
		if (!ctx[column_idx].signal)
			ctx[column_idx].signal = ccv_nnc_stream_signal_new(stream_type);
		child_ctx = ctx[column_idx];
	}
	return child_ctx;
}

static void _ccv_cnnp_dataframe_column_data(ccv_cnnp_dataframe_t* const dataframe, ccv_cnnp_dataframe_iter_t* const iter, ccv_cnnp_dataframe_data_item_t* const cached_data, void** const fetched_data, const int* const row_idxs, const int row_size, const int column_idx, const int cached_step, ccv_nnc_stream_context_t* const stream_context)
{
	int i;
	if (cached_data[column_idx * cached_step].flag)
	{
		for (i = 1; i < row_size; i++)
			{ assert(cached_data[i + column_idx * cached_step].flag); }
		for (i = 0; i < row_size; i++)
			fetched_data[i] = cached_data[i + column_idx * cached_step].data;
		return;
	} else {
		for (i = 1; i < row_size; i++)
			{ assert(!cached_data[i + column_idx * cached_step].flag); }
		for (i = 0; i < row_size; i++)
			fetched_data[i] = _ccv_cnnp_dataframe_dequeue_data(dataframe, column_idx, stream_context);
	}
	if (column_idx >= dataframe->column_size)
	{
		assert(dataframe->derived_column_data);
		const int derived_column_idx = column_idx - dataframe->column_size;
		const ccv_cnnp_derived_column_data_t* const derived_column_data = (ccv_cnnp_derived_column_data_t*)ccv_array_get(dataframe->derived_column_data, derived_column_idx);
		ccv_cnnp_dataframe_column_ctx_t child_ctx = _ccv_cnnp_child_column_ctx_for_stream_type(dataframe, iter, column_idx, stream_context, derived_column_data->stream_type);
		const int column_idx_size = derived_column_data->column_idx_size;
		if (derived_column_data->map)
		{
			int i;
			if (!iter->derived_data)
				iter->derived_data = (void****)cccalloc(dataframe->derived_column_data->rnum, sizeof(void***));
			if (!iter->derived_data[derived_column_idx])
				iter->derived_data[derived_column_idx] = (void***)cccalloc(derived_column_data->column_idx_size, sizeof(void**));
			void*** const derived_data = iter->derived_data[derived_column_idx];
			for (i = 0; i < column_idx_size; i++)
			{
				derived_data[i] = FETCHED_DATA(iter, derived_column_data->column_idxs[i]);
				_ccv_cnnp_dataframe_column_data(dataframe, iter, cached_data, derived_data[i], row_idxs, row_size, derived_column_data->column_idxs[i], cached_step, stream_context);
			}
			derived_column_data->map(derived_data, derived_column_data->column_idx_size, row_size, fetched_data, derived_column_data->context, child_ctx.stream_context);
		} else
			derived_column_data->data_enum(column_idx, row_idxs, row_size, fetched_data, derived_column_data->context, child_ctx.stream_context);
		if (child_ctx.stream_context != stream_context)
		{
			ccv_nnc_stream_context_emit_signal(child_ctx.stream_context, child_ctx.signal);
			ccv_nnc_stream_context_wait_signal(stream_context, child_ctx.signal);
		}
	} else {
		const ccv_cnnp_column_data_t* const column_data = dataframe->column_data + column_idx;
		ccv_cnnp_dataframe_column_ctx_t child_ctx = _ccv_cnnp_child_column_ctx_for_stream_type(dataframe, iter, column_idx, stream_context, column_data->stream_type);
		column_data->data_enum(column_idx, row_idxs, row_size, fetched_data, column_data->context, child_ctx.stream_context);
		if (child_ctx.stream_context != stream_context)
		{
			ccv_nnc_stream_context_emit_signal(child_ctx.stream_context, child_ctx.signal);
			ccv_nnc_stream_context_wait_signal(stream_context, child_ctx.signal);
		}
	}
	for (i = 0; i < row_size; i++)
	{
		cached_data[i + column_idx * cached_step].flag = 1;
		cached_data[i + column_idx * cached_step].ctx = (uint64_t)(uintptr_t)stream_context;
		cached_data[i + column_idx * cached_step].data = fetched_data[i];
	}
}

int ccv_cnnp_dataframe_iter_next(ccv_cnnp_dataframe_iter_t* const iter, void** const data_ref, const int column_idx_size, ccv_nnc_stream_context_t* const stream_context)
{
	ccv_cnnp_dataframe_t* const dataframe = iter->dataframe;
	assert(column_idx_size <= iter->column_idx_size);
	const int column_size = dataframe->column_size + (dataframe->derived_column_data ? dataframe->derived_column_data->rnum : 0);
	int i;
	// Push existing data back to reusable state (note, these may not be reused immediately because they may be on a different stream context).
	for (i = 0; i < column_size; i++)
		if (iter->cached_data[i].flag)
		{
			_ccv_cnnp_dataframe_enqueue_data(dataframe, iter->cached_data[i].data, i, iter->cached_data[i].ctx);
			iter->cached_data[i].flag = 0;
			iter->cached_data[i].data = 0;
			iter->cached_data[i].ctx = 0;
		}
	const int idx = iter->idx;
	if (idx == dataframe->row_count)
		return -1;
	if (iter->prefetch_tail != -1) // If there is something in prefetch log.
	{
		ccv_array_t* const prefetches = iter->prefetches;
		assert(prefetches);
		const int lines = prefetches->rnum / column_size;
		if (iter->prefetch_head == iter->prefetch_tail) // Only one item.
			iter->prefetch_tail = -1;
		ccv_cnnp_dataframe_data_item_t* const cached_data = (ccv_cnnp_dataframe_data_item_t*)ccv_array_get(iter->prefetches, iter->prefetch_head);
		for (i = 0; i < column_size; i++)
		{
			if (!cached_data[i * lines].flag)
				continue;
			if (cached_data[i * lines].ctx == (uint64_t)(uintptr_t)stream_context) // If match existing stream context.
				iter->cached_data[i] = cached_data[i * lines];
			else // Recycle
				_ccv_cnnp_dataframe_enqueue_data(dataframe, cached_data[i * lines].data, i, cached_data[i * lines].ctx);
		}
		++iter->prefetch_head;
		assert(prefetches->rnum % column_size == 0);
		if (iter->prefetch_head >= lines)
			iter->prefetch_head = 0;
	}
	for (i = 0; i < column_idx_size; i++)
		_ccv_cnnp_dataframe_column_data(dataframe, iter, iter->cached_data, data_ref + i, dataframe->shuffled_idx ? dataframe->shuffled_idx + idx : &idx, 1, iter->column_idxs[i], 1, stream_context);
	++iter->idx;
	return 0;
}

static void _ccv_cnnp_null_prefetches(ccv_cnnp_dataframe_iter_t* const iter)
{
	ccv_cnnp_dataframe_t* const dataframe = iter->dataframe;
	assert(dataframe);
	int i, j;
	const int column_size = dataframe->column_size + (dataframe->derived_column_data ? dataframe->derived_column_data->rnum : 0);
	if (iter->prefetch_head <= iter->prefetch_tail)
	{
		assert(iter->prefetches);
		const int lines = iter->prefetches->rnum / column_size;
		for (i = iter->prefetch_head; i <= iter->prefetch_tail; i++)
		{
			ccv_cnnp_dataframe_data_item_t* const cached_data = ccv_array_get(iter->prefetches, i);
			for (j = 0; j < column_size; j++)
				if (cached_data[j * lines].flag)
					_ccv_cnnp_dataframe_enqueue_data(dataframe, cached_data[j * lines].data, j, cached_data[j * lines].ctx);
		}
	} else if (iter->prefetch_tail >= 0) { // -1 means no item.
		assert(iter->prefetches);
		const int lines = iter->prefetches->rnum / column_size;
		for (i = iter->prefetch_head; i < lines; i++)
		{
			ccv_cnnp_dataframe_data_item_t* const cached_data = ccv_array_get(iter->prefetches, i);
			for (j = 0; j < column_size; j++)
				if (cached_data[j * lines].flag)
					_ccv_cnnp_dataframe_enqueue_data(dataframe, cached_data[j * lines].data, j, cached_data[j * lines].ctx);
		}
		for (i = 0; i <= iter->prefetch_tail; i++)
		{
			ccv_cnnp_dataframe_data_item_t* const cached_data = ccv_array_get(iter->prefetches, i);
			for (j = 0; j < column_size; j++)
				if (cached_data[j * lines].flag)
					_ccv_cnnp_dataframe_enqueue_data(dataframe, cached_data[j * lines].data, j, cached_data[j * lines].ctx);
		}
	}
	iter->prefetch_head = 0;
	iter->prefetch_tail = -1;
}

static void _ccv_cnnp_prefetch_cached_data(ccv_cnnp_dataframe_iter_t* const iter, ccv_cnnp_dataframe_data_item_t* const cached_data, const int idx, const int max_to_prefetch, ccv_nnc_stream_context_t* const stream_context)
{
	ccv_cnnp_dataframe_t* const dataframe = iter->dataframe;
	assert(dataframe);
	const int column_size = dataframe->column_size + (dataframe->derived_column_data ? dataframe->derived_column_data->rnum : 0);
	assert(iter->prefetches);
	const int lines = iter->prefetches->rnum / column_size;
	int i, j;
	// Reset
	for (i = 0; i < column_size; i++)
		for (j = 0; j < max_to_prefetch; j++)
		{
			cached_data[j + i * lines].flag = 0;
			cached_data[j + i * lines].data = 0;
			cached_data[j + i * lines].ctx = 0;
		}
	if (iter->fetched_size < max_to_prefetch)
	{
		iter->fetched_data = ccrealloc(iter->fetched_data, sizeof(void*) * max_to_prefetch * (column_size + 1));
		iter->fetched_size = max_to_prefetch;
	}
	if (dataframe->shuffled_idx)
		for (i = 0; i < iter->column_idx_size; i++)
			_ccv_cnnp_dataframe_column_data(dataframe, iter, cached_data, FETCHED_DATA(iter, iter->column_idxs[i]), dataframe->shuffled_idx + idx, max_to_prefetch, iter->column_idxs[i], lines, stream_context);
	else {
		for (i = 0; i < max_to_prefetch; i++)
			INDEX_DATA(iter)[i] = idx + i;
		for (i = 0; i < iter->column_idx_size; i++)
			_ccv_cnnp_dataframe_column_data(dataframe, iter, cached_data, FETCHED_DATA(iter, iter->column_idxs[i]), INDEX_DATA(iter), max_to_prefetch, iter->column_idxs[i], lines, stream_context);
	}
}

int ccv_cnnp_dataframe_iter_prefetch(ccv_cnnp_dataframe_iter_t* const iter, const int prefetch_count, ccv_nnc_stream_context_t* const stream_context)
{
	ccv_cnnp_dataframe_t* const dataframe = iter->dataframe;
	assert(dataframe);
	const int column_size = dataframe->column_size + (dataframe->derived_column_data ? dataframe->derived_column_data->rnum : 0);
	int i, j;
	assert(iter->idx <= dataframe->row_count);
	int lines, next, max_to_prefetch;
	if (iter->prefetch_tail == -1)
	{
		if (iter->idx == dataframe->row_count)
			return -1; // Cannot be done.
		max_to_prefetch = ccv_min(dataframe->row_count - iter->idx, prefetch_count);
		if (!iter->prefetches)
		{
			iter->prefetches = ccv_array_new(sizeof(ccv_cnnp_dataframe_data_item_t), max_to_prefetch * column_size, 0);
			ccv_array_resize(iter->prefetches, max_to_prefetch * column_size);
		}
		iter->prefetch_tail = iter->prefetch_head = 0; // Advance!
		next = iter->idx;
		lines = iter->prefetches->rnum / column_size;
		// Reset to enough space.
		if (lines < max_to_prefetch)
		{
			ccv_array_resize(iter->prefetches, max_to_prefetch * column_size);
			lines = max_to_prefetch;
		}
	} else {
		assert(iter->prefetches);
		ccv_array_t* const prefetches = iter->prefetches;
		assert(prefetches->rnum % column_size == 0);
		lines = prefetches->rnum / column_size;
		const int prefetched = iter->prefetch_tail >= iter->prefetch_head ? iter->prefetch_tail - iter->prefetch_head + 1: lines - iter->prefetch_head + iter->prefetch_tail + 1;
		if (iter->idx + prefetched == dataframe->row_count) // Nothing to prefetch.
			return -1;
		max_to_prefetch = ccv_min(dataframe->row_count - (iter->idx + prefetched), prefetch_count);
		// Not enough space, need to resize.
		if (prefetched + max_to_prefetch > lines)
		{
			const int new_lines = prefetched + max_to_prefetch;
			ccv_array_resize(prefetches, new_lines * column_size);
			// These are overlap moves, have to make sure start from the end and move it up to the beginning.
			if (iter->prefetch_head > iter->prefetch_tail)
			{
				const int offset = new_lines - lines;
				for (i = column_size - 1; i >= 0; i--)
				{
					for (j = lines - 1; j >= iter->prefetch_head; j--)
						*(ccv_cnnp_dataframe_data_item_t*)ccv_array_get(prefetches, j + offset + i * new_lines) = *(ccv_cnnp_dataframe_data_item_t*)ccv_array_get(prefetches, j + i * lines);
					for (j = iter->prefetch_tail; j >= 0; j--)
						*(ccv_cnnp_dataframe_data_item_t*)ccv_array_get(prefetches, j + i * new_lines) = *(ccv_cnnp_dataframe_data_item_t*)ccv_array_get(prefetches, j + i * lines);
				}
				iter->prefetch_head += offset;
			} else {
				for (i = column_size - 1; i >= 0; i--)
					for (j = iter->prefetch_tail; j >= iter->prefetch_head; j--)
						*(ccv_cnnp_dataframe_data_item_t*)ccv_array_get(prefetches, j + i * new_lines) = *(ccv_cnnp_dataframe_data_item_t*)ccv_array_get(prefetches, j + i * lines);
			}
			lines = new_lines;
		}
		++iter->prefetch_tail; // Move to the next ready tail.
		if (iter->prefetch_tail >= lines)
			iter->prefetch_tail = 0;
		next = iter->idx + prefetched;
	}
	ccv_array_t* const prefetches = iter->prefetches;
	ccv_cnnp_dataframe_data_item_t* const cached_data = (ccv_cnnp_dataframe_data_item_t*)ccv_array_get(prefetches, iter->prefetch_tail);
	// If the tail is before the head, we must have enough space for the max_to_prefetch
	if (iter->prefetch_tail < iter->prefetch_head)
	{
		assert(iter->prefetch_tail + max_to_prefetch - 1 < iter->prefetch_head);
		_ccv_cnnp_prefetch_cached_data(iter, cached_data, next, max_to_prefetch, stream_context);
		iter->prefetch_tail += max_to_prefetch - 1;
	} else {
		// First, fetch to the end.
		const int fetch_to_end = ccv_min(max_to_prefetch, lines - iter->prefetch_tail);
		_ccv_cnnp_prefetch_cached_data(iter, cached_data, next, fetch_to_end, stream_context);
		if (fetch_to_end == max_to_prefetch)
			iter->prefetch_tail += fetch_to_end - 1;
		else {
			// Need to fetch more.
			ccv_cnnp_dataframe_data_item_t* const more_data = (ccv_cnnp_dataframe_data_item_t*)ccv_array_get(prefetches, 0);
			assert(max_to_prefetch > fetch_to_end);
			_ccv_cnnp_prefetch_cached_data(iter, more_data, next + fetch_to_end, max_to_prefetch - fetch_to_end, stream_context);
			iter->prefetch_tail = max_to_prefetch - fetch_to_end - 1;
		}
	}
	return 0;
}

int ccv_cnnp_dataframe_iter_set_cursor(ccv_cnnp_dataframe_iter_t* const iter, const int idx)
{
	ccv_cnnp_dataframe_t* const dataframe = iter->dataframe;
	assert(dataframe);
	if (idx >= dataframe->row_count)
		return -1;
	if (idx == iter->idx)
		return 0;
	iter->idx = idx;
	_ccv_cnnp_null_prefetches(iter);
	return 0;
}

void ccv_cnnp_dataframe_iter_free(ccv_cnnp_dataframe_iter_t* const iter)
{
	ccv_cnnp_dataframe_t* const dataframe = iter->dataframe;
	const int column_size = dataframe->column_size + (dataframe->derived_column_data ? dataframe->derived_column_data->rnum : 0);
	int i;
	// Push existing data back to reusable state (note, these may not be reused immediately because they may be on a different stream context).
	for (i = 0; i < column_size; i++)
		if (iter->cached_data[i].flag)
			_ccv_cnnp_dataframe_enqueue_data(dataframe, iter->cached_data[i].data, i, iter->cached_data[i].ctx);
	// Push prefetches back to reusable state.
	_ccv_cnnp_null_prefetches(iter);
	if (iter->prefetches)
		ccv_array_free(iter->prefetches);
	if (iter->derived_data)
	{
		assert(dataframe->derived_column_data);
		for (i = 0; i < dataframe->derived_column_data->rnum; i++)
			if (iter->derived_data[i])
				ccfree(iter->derived_data[i]);
		ccfree(iter->derived_data);
	}
	ccfree(iter->fetched_data);
	if (iter->column_ctx)
	{
		khash_t(iter_ctx)* const column_ctx = iter->column_ctx;
		khiter_t k;
		for (k = kh_begin(column_ctx); k != kh_end(column_ctx); ++k)
		{
			if (!kh_exist(column_ctx, k))
				continue;
			ccv_cnnp_dataframe_column_ctx_t* const ctx = kh_val(column_ctx, k);
			for (i = 0; i < column_size; i++)
			{
				if (ctx[i].stream_context)
					ccv_nnc_stream_context_free(ctx[i].stream_context);
				if (ctx[i].signal)
					ccv_nnc_stream_signal_free(ctx[i].signal);
			}
		}
		kh_destroy(iter_ctx, column_ctx);
	}
	ccfree(iter);
}

void ccv_cnnp_dataframe_free(ccv_cnnp_dataframe_t* const dataframe)
{
	int i, j;
	khash_t(ctx)* const data_ctx = dataframe->data_ctx;
	khiter_t k;
	const int column_size = dataframe->column_size + (dataframe->derived_column_data ? dataframe->derived_column_data->rnum : 0);
	for (k = kh_begin(data_ctx); k != kh_end(data_ctx); ++k)
	{
		if (!kh_exist(data_ctx, k))
			continue;
		ccv_array_t* const columns = kh_val(data_ctx, k);
		assert(columns->rnum <= column_size);
		for (i = 0; i < columns->rnum; i++)
		{
			ccv_array_t* const column = *(ccv_array_t**)ccv_array_get(columns, i);
			void* context;
			ccv_cnnp_column_data_deinit_f data_deinit;
			if (i < dataframe->column_size)
			{
				data_deinit = dataframe->column_data[i].data_deinit;
				context = dataframe->column_data[i].context;
			} else {
				assert(dataframe->derived_column_data);
				ccv_cnnp_derived_column_data_t* const derived_column_data = (ccv_cnnp_derived_column_data_t*)ccv_array_get(dataframe->derived_column_data, i - dataframe->column_size);
				data_deinit = derived_column_data->data_deinit;
				context = derived_column_data->context;
			}
			if (data_deinit)
				for (j = 0; j < column->rnum; j++)
					data_deinit(*(void**)ccv_array_get(column, j), context);
			ccv_array_free(column);
		}
		ccv_array_free(columns);
	}
	kh_destroy(ctx, data_ctx);
	if (dataframe->derived_column_data)
	{
		for (i = 0; i < dataframe->derived_column_data->rnum; i++)
		{
			ccv_cnnp_derived_column_data_t* const derived_column_data = (ccv_cnnp_derived_column_data_t*)ccv_array_get(dataframe->derived_column_data, i);
			if (derived_column_data->context_deinit)
				derived_column_data->context_deinit(derived_column_data->context);
			ccfree(derived_column_data->column_idxs);
		}
		ccv_array_free(dataframe->derived_column_data);
	}
	for (i = 0; i < dataframe->column_size; i++)
		if (dataframe->column_data[i].context_deinit)
			dataframe->column_data[i].context_deinit(dataframe->column_data[i].context);
	if (dataframe->shuffled_idx)
		ccfree(dataframe->shuffled_idx);
#ifdef HAVE_GSL
	if (dataframe->rng)
		gsl_rng_free(dataframe->rng);
#endif
	ccfree(dataframe);
}


/* matrix/gsl_matrix_double.h
 *
 * Copyright (C) 1996, 1997, 1998, 1999, 2000, 2007 Gerard Jungman, Brian Gough
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 3 of the License, or (at
 * your option) any later version.
 *
 * This program is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
 */

#ifndef __GSL_MATRIX_DOUBLE_H__
#define __GSL_MATRIX_DOUBLE_H__

#if !defined( GSL_FUN )
#  if !defined( GSL_DLL )
#    define GSL_FUN extern
#  elif defined( BUILD_GSL_DLL )
#    define GSL_FUN extern __declspec(dllexport)
#  else
#    define GSL_FUN extern __declspec(dllimport)
#  endif
#endif

#include <stdlib.h>
#include <gsl/gsl_types.h>
#include <gsl/gsl_errno.h>
#include <gsl/gsl_inline.h>
#include <gsl/gsl_check_range.h>
#include <gsl/gsl_vector_double.h>
#include <gsl/gsl_blas_types.h>

#undef __BEGIN_DECLS
#undef __END_DECLS
#ifdef __cplusplus
# define __BEGIN_DECLS extern "C" {
# define __END_DECLS }
#else
# define __BEGIN_DECLS /* empty */
# define __END_DECLS /* empty */
#endif

__BEGIN_DECLS

typedef struct
{
  size_t size1;
  size_t size2;
  size_t tda;
  double * data;
  gsl_block * block;
  int owner;
} gsl_matrix;

typedef struct
{
  gsl_matrix matrix;
} _gsl_matrix_view;

typedef _gsl_matrix_view gsl_matrix_view;

typedef struct
{
  gsl_matrix matrix;
} _gsl_matrix_const_view;

typedef const _gsl_matrix_const_view gsl_matrix_const_view;

/* Allocation */

GSL_FUN gsl_matrix *
gsl_matrix_alloc (const size_t n1, const size_t n2);

GSL_FUN gsl_matrix *
gsl_matrix_calloc (const size_t n1, const size_t n2);

GSL_FUN gsl_matrix *
gsl_matrix_alloc_from_block (gsl_block * b,
                                   const size_t offset,
                                   const size_t n1,
                                   const size_t n2,
                                   const size_t d2);

GSL_FUN gsl_matrix *
gsl_matrix_alloc_from_matrix (gsl_matrix * m,
                                    const size_t k1,
                                    const size_t k2,
                                    const size_t n1,
                                    const size_t n2);

GSL_FUN gsl_vector *
gsl_vector_alloc_row_from_matrix (gsl_matrix * m,
                                        const size_t i);

GSL_FUN gsl_vector *
gsl_vector_alloc_col_from_matrix (gsl_matrix * m,
                                        const size_t j);

GSL_FUN void gsl_matrix_free (gsl_matrix * m);

/* Views */

GSL_FUN _gsl_matrix_view
gsl_matrix_submatrix (gsl_matrix * m,
                            const size_t i, const size_t j,
                            const size_t n1, const size_t n2);

GSL_FUN _gsl_vector_view
gsl_matrix_row (gsl_matrix * m, const size_t i);

GSL_FUN _gsl_vector_view
gsl_matrix_column (gsl_matrix * m, const size_t j);

GSL_FUN _gsl_vector_view
gsl_matrix_diagonal (gsl_matrix * m);

GSL_FUN _gsl_vector_view
gsl_matrix_subdiagonal (gsl_matrix * m, const size_t k);

GSL_FUN _gsl_vector_view
gsl_matrix_superdiagonal (gsl_matrix * m, const size_t k);

GSL_FUN _gsl_vector_view
gsl_matrix_subrow (gsl_matrix * m, const size_t i,
                         const size_t offset, const size_t n);

GSL_FUN _gsl_vector_view
gsl_matrix_subcolumn (gsl_matrix * m, const size_t j,
                            const size_t offset, const size_t n);

GSL_FUN _gsl_matrix_view
gsl_matrix_view_array (double * base,
                             const size_t n1,
                             const size_t n2);

GSL_FUN _gsl_matrix_view
gsl_matrix_view_array_with_tda (double * base,
                                      const size_t n1,
                                      const size_t n2,
                                      const size_t tda);

GSL_FUN _gsl_matrix_view
gsl_matrix_view_vector (gsl_vector * v,
                              const size_t n1,
                              const size_t n2);

GSL_FUN _gsl_matrix_view
gsl_matrix_view_vector_with_tda (gsl_vector * v,
                                       const size_t n1,
                                       const size_t n2,
                                       const size_t tda);

GSL_FUN _gsl_matrix_const_view
gsl_matrix_const_submatrix (const gsl_matrix * m,
                                  const size_t i, const size_t j,
                                  const size_t n1, const size_t n2);

GSL_FUN _gsl_vector_const_view
gsl_matrix_const_row (const gsl_matrix * m,
                            const size_t i);

GSL_FUN _gsl_vector_const_view
gsl_matrix_const_column (const gsl_matrix * m,
                               const size_t j);

GSL_FUN _gsl_vector_const_view
gsl_matrix_const_diagonal (const gsl_matrix * m);

GSL_FUN _gsl_vector_const_view
gsl_matrix_const_subdiagonal (const gsl_matrix * m,
                                    const size_t k);

GSL_FUN _gsl_vector_const_view
gsl_matrix_const_superdiagonal (const gsl_matrix * m,
                                      const size_t k);

GSL_FUN _gsl_vector_const_view
gsl_matrix_const_subrow (const gsl_matrix * m, const size_t i,
                               const size_t offset, const size_t n);

GSL_FUN _gsl_vector_const_view
gsl_matrix_const_subcolumn (const gsl_matrix * m, const size_t j,
                                  const size_t offset, const size_t n);

GSL_FUN _gsl_matrix_const_view
gsl_matrix_const_view_array (const double * base,
                                   const size_t n1,
                                   const size_t n2);

GSL_FUN _gsl_matrix_const_view
gsl_matrix_const_view_array_with_tda (const double * base,
                                            const size_t n1,
                                            const size_t n2,
                                            const size_t tda);

GSL_FUN _gsl_matrix_const_view
gsl_matrix_const_view_vector (const gsl_vector * v,
                                    const size_t n1,
                                    const size_t n2);

GSL_FUN _gsl_matrix_const_view
gsl_matrix_const_view_vector_with_tda (const gsl_vector * v,
                                             const size_t n1,
                                             const size_t n2,
                                             const size_t tda);

/* Operations */

GSL_FUN void gsl_matrix_set_zero (gsl_matrix * m);
GSL_FUN void gsl_matrix_set_identity (gsl_matrix * m);
GSL_FUN void gsl_matrix_set_all (gsl_matrix * m, double x);

GSL_FUN int gsl_matrix_fread (FILE * stream, gsl_matrix * m) ;
GSL_FUN int gsl_matrix_fwrite (FILE * stream, const gsl_matrix * m) ;
GSL_FUN int gsl_matrix_fscanf (FILE * stream, gsl_matrix * m);
GSL_FUN int gsl_matrix_fprintf (FILE * stream, const gsl_matrix * m, const char * format);

GSL_FUN int gsl_matrix_memcpy(gsl_matrix * dest, const gsl_matrix * src);
GSL_FUN int gsl_matrix_swap(gsl_matrix * m1, gsl_matrix * m2);
GSL_FUN int gsl_matrix_tricpy(CBLAS_UPLO_t Uplo, CBLAS_DIAG_t Diag, gsl_matrix * dest, const gsl_matrix * src);

GSL_FUN int gsl_matrix_swap_rows(gsl_matrix * m, const size_t i, const size_t j);
GSL_FUN int gsl_matrix_swap_columns(gsl_matrix * m, const size_t i, const size_t j);
GSL_FUN int gsl_matrix_swap_rowcol(gsl_matrix * m, const size_t i, const size_t j);
GSL_FUN int gsl_matrix_transpose (gsl_matrix * m);
GSL_FUN int gsl_matrix_transpose_memcpy (gsl_matrix * dest, const gsl_matrix * src);
GSL_FUN int gsl_matrix_transpose_tricpy (CBLAS_UPLO_t Uplo_src, CBLAS_DIAG_t Diag, gsl_matrix * dest, const gsl_matrix * src);

GSL_FUN double gsl_matrix_max (const gsl_matrix * m);
GSL_FUN double gsl_matrix_min (const gsl_matrix * m);
GSL_FUN void gsl_matrix_minmax (const gsl_matrix * m, double * min_out, double * max_out);

GSL_FUN void gsl_matrix_max_index (const gsl_matrix * m, size_t * imax, size_t *jmax);
GSL_FUN void gsl_matrix_min_index (const gsl_matrix * m, size_t * imin, size_t *jmin);
GSL_FUN void gsl_matrix_minmax_index (const gsl_matrix * m, size_t * imin, size_t * jmin, size_t * imax, size_t * jmax);

GSL_FUN int gsl_matrix_equal (const gsl_matrix * a, const gsl_matrix * b);

GSL_FUN int gsl_matrix_isnull (const gsl_matrix * m);
GSL_FUN int gsl_matrix_ispos (const gsl_matrix * m);
GSL_FUN int gsl_matrix_isneg (const gsl_matrix * m);
GSL_FUN int gsl_matrix_isnonneg (const gsl_matrix * m);

GSL_FUN int gsl_matrix_add (gsl_matrix * a, const gsl_matrix * b);
GSL_FUN int gsl_matrix_sub (gsl_matrix * a, const gsl_matrix * b);
GSL_FUN int gsl_matrix_mul_elements (gsl_matrix * a, const gsl_matrix * b);
GSL_FUN int gsl_matrix_div_elements (gsl_matrix * a, const gsl_matrix * b);
GSL_FUN int gsl_matrix_scale (gsl_matrix * a, const double x);
GSL_FUN int gsl_matrix_scale_rows (gsl_matrix * a, const gsl_vector * x);
GSL_FUN int gsl_matrix_scale_columns (gsl_matrix * a, const gsl_vector * x);
GSL_FUN int gsl_matrix_add_constant (gsl_matrix * a, const double x);
GSL_FUN int gsl_matrix_add_diagonal (gsl_matrix * a, const double x);

/***********************************************************************/
/* The functions below are obsolete                                    */
/***********************************************************************/
GSL_FUN int gsl_matrix_get_row(gsl_vector * v, const gsl_matrix * m, const size_t i);
GSL_FUN int gsl_matrix_get_col(gsl_vector * v, const gsl_matrix * m, const size_t j);
GSL_FUN int gsl_matrix_set_row(gsl_matrix * m, const size_t i, const gsl_vector * v);
GSL_FUN int gsl_matrix_set_col(gsl_matrix * m, const size_t j, const gsl_vector * v);
/***********************************************************************/

/* inline functions if you are using GCC */

GSL_FUN INLINE_DECL double   gsl_matrix_get(const gsl_matrix * m, const size_t i, const size_t j);
GSL_FUN INLINE_DECL void    gsl_matrix_set(gsl_matrix * m, const size_t i, const size_t j, const double x);
GSL_FUN INLINE_DECL double * gsl_matrix_ptr(gsl_matrix * m, const size_t i, const size_t j);
GSL_FUN INLINE_DECL const double * gsl_matrix_const_ptr(const gsl_matrix * m, const size_t i, const size_t j);

#ifdef HAVE_INLINE
INLINE_FUN
double
gsl_matrix_get(const gsl_matrix * m, const size_t i, const size_t j)
{
#if GSL_RANGE_CHECK
  if (GSL_RANGE_COND(1))
    {
      if (i >= m->size1)
        {
          GSL_ERROR_VAL("first index out of range", GSL_EINVAL, 0) ;
        }
      else if (j >= m->size2)
        {
          GSL_ERROR_VAL("second index out of range", GSL_EINVAL, 0) ;
        }
    }
#endif
  return m->data[i * m->tda + j] ;
}

INLINE_FUN
void
gsl_matrix_set(gsl_matrix * m, const size_t i, const size_t j, const double x)
{
#if GSL_RANGE_CHECK
  if (GSL_RANGE_COND(1))
    {
      if (i >= m->size1)
        {
          GSL_ERROR_VOID("first index out of range", GSL_EINVAL) ;
        }
      else if (j >= m->size2)
        {
          GSL_ERROR_VOID("second index out of range", GSL_EINVAL) ;
        }
    }
#endif
  m->data[i * m->tda + j] = x ;
}

INLINE_FUN
double *
gsl_matrix_ptr(gsl_matrix * m, const size_t i, const size_t j)
{
#if GSL_RANGE_CHECK
  if (GSL_RANGE_COND(1))
    {
      if (i >= m->size1)
        {
          GSL_ERROR_NULL("first index out of range", GSL_EINVAL) ;
        }
      else if (j >= m->size2)
        {
          GSL_ERROR_NULL("second index out of range", GSL_EINVAL) ;
        }
    }
#endif
  return (double *) (m->data + (i * m->tda + j)) ;
}

INLINE_FUN
const double *
gsl_matrix_const_ptr(const gsl_matrix * m, const size_t i, const size_t j)
{
#if GSL_RANGE_CHECK
  if (GSL_RANGE_COND(1))
    {
      if (i >= m->size1)
        {
          GSL_ERROR_NULL("first index out of range", GSL_EINVAL) ;
        }
      else if (j >= m->size2)
        {
          GSL_ERROR_NULL("second index out of range", GSL_EINVAL) ;
        }
    }
#endif
  return (const double *) (m->data + (i * m->tda + j)) ;
}

#endif

__END_DECLS

#endif /* __GSL_MATRIX_DOUBLE_H__ */


/**
 */
#ifndef _DRZ2PET_UTILS_H
#define _DRZ2PET_UTILS_H

#include <gsl/gsl_vector.h>
#include <gsl/gsl_matrix.h>
#include "aXe_grism.h"
#include <math.h>
#include <string.h>

ap_pixel *
make_spc_drztable(observation * const obs, observation * const wobs,
                  const object *ob, const double lambda0,
                  const double dlambda);

extern char *
get_ID_num(char *grism_image,char *extname);

extern void
normalize_weight(observation * wobs, object *ob,
                 const int opt_extr);

extern gsl_matrix *
comp_equ_weight(gsl_matrix *exp_map, const object *ob);

gsl_matrix *
comp_exp_weight(gsl_matrix *exp_map, const object *ob);

gsl_matrix *
comp_opt_weight(gsl_matrix *mod_map,
                gsl_matrix *var_map, const object *ob);
#endif /* !_DRZ2PET_UTILS_H */


#pragma once

#include <algorithm>
#include <cstddef>
#include <cstdint>
#include <tuple>
#include <type_traits>

#include <gsl/gsl>

#include "chainerx/array.h"
#include "chainerx/axes.h"
#include "chainerx/backend_util.h"
#include "chainerx/constant.h"
#include "chainerx/dtype.h"
#include "chainerx/indexer.h"
#include "chainerx/macro.h"
#include "chainerx/shape.h"
#include "chainerx/strides.h"

namespace chainerx {
namespace indexable_array_detail {

template <typename To, typename From>
using WithConstnessOf = dtype_detail::WithConstnessOf<To, From>;

static inline std::tuple<const uint8_t*, const uint8_t*> GetDataRange(const Array& a) {
    std::tuple<int64_t, int64_t> range = chainerx::GetDataRange(a.shape(), a.strides(), a.GetItemSize());
    int64_t lower = std::get<0>(range);
    int64_t upper = std::get<1>(range);
    const uint8_t* base = static_cast<const uint8_t*>(internal::GetRawOffsetData(a));
    return std::tuple<const uint8_t*, const uint8_t*>{base + lower, base + upper};
}

}  // namespace indexable_array_detail

template <typename T, int8_t kNdim = kDynamicNdim>
class IndexableArray {
public:
    using ElementType = T;

private:
    template <typename U>
    using WithConstnessOfT = indexable_array_detail::WithConstnessOf<U, T>;
    using VoidType = WithConstnessOfT<void>;
    using DeviceStorageType = TypeToDeviceStorageType<T>;

public:
    IndexableArray(VoidType* data, const Strides& strides) : data_{data} { std::copy(strides.begin(), strides.end(), strides_); }

    IndexableArray(const Array& array, const Strides& strides) : IndexableArray{internal::GetRawOffsetData(array), strides} {
        CHAINERX_ASSERT(TypeToDtype<T> == array.dtype());

#if CHAINERX_DEBUG
        std::tie(first_, last_) = indexable_array_detail::GetDataRange(array);
#endif  // CHAINERX_DEBUG
    }

    explicit IndexableArray(const Array& array) : IndexableArray{array, array.strides()} {}

    CHAINERX_HOST_DEVICE int8_t ndim() const { return kNdim; }

    CHAINERX_HOST_DEVICE const int64_t* strides() const { return strides_; }

    CHAINERX_HOST_DEVICE VoidType* data() const { return data_; }

    CHAINERX_HOST_DEVICE DeviceStorageType& operator[](const int64_t* index) const {
        auto data_ptr = static_cast<WithConstnessOfT<uint8_t>*>(data_);
        for (int8_t dim = 0; dim < kNdim; ++dim) {
            data_ptr += strides_[dim] * index[dim];
        }
#if CHAINERX_DEBUG
        CHAINERX_ASSERT(first_ == nullptr || first_ <= data_ptr);
        CHAINERX_ASSERT(last_ == nullptr || data_ptr <= last_ - sizeof(T));
#endif  // CHAINERX_DEBUG
        return *static_cast<DeviceStorageType*>(static_cast<VoidType*>(data_ptr));
    }

    CHAINERX_HOST_DEVICE WithConstnessOfT<DeviceStorageType>& operator[](const IndexIterator<kNdim>& it) const {
        return operator[](it.index());
    }

    // Permutes the axes.
    //
    // It is the caller's responsibility to ensure validity of permutation.
    // If the permutation is invalid, the behavior is undefined.
    IndexableArray<T, kNdim>& Permute(const Axes& axes) {
        CHAINERX_ASSERT(axes.size() == static_cast<size_t>(kNdim));
        int64_t c[kNdim]{};
        std::copy(std::begin(strides_), std::end(strides_), c);
        for (size_t i = 0; i < kNdim; ++i) {
            strides_[i] = c[axes[i]];
        }
        return *this;
    }

private:
    WithConstnessOfT<void>* data_;
#if CHAINERX_DEBUG
    const uint8_t* first_{nullptr};
    const uint8_t* last_{nullptr};
#endif  // CHAINERX_DEBUG
    int64_t strides_[kNdim];
};

// Static 0-dimensional specialization.
template <typename T>
class IndexableArray<T, 0> {
public:
    using ElementType = T;

private:
    template <typename U>
    using WithConstnessOfT = indexable_array_detail::WithConstnessOf<U, T>;
    using VoidType = WithConstnessOfT<void>;
    using DeviceStorageType = TypeToDeviceStorageType<T>;

public:
    IndexableArray(VoidType* data, const Strides& strides) : data_{data} { CHAINERX_ASSERT(0 == strides.ndim()); }

    IndexableArray(const Array& array, const Strides& strides) : IndexableArray{internal::GetRawOffsetData(array), strides} {
        CHAINERX_ASSERT(TypeToDtype<T> == array.dtype());
    }

    explicit IndexableArray(const Array& array) : IndexableArray{array, array.strides()} {}

    CHAINERX_HOST_DEVICE constexpr int8_t ndim() const { return 0; }

    CHAINERX_HOST_DEVICE constexpr const int64_t* strides() const { return nullptr; }

    CHAINERX_HOST_DEVICE VoidType* data() const { return data_; }

    CHAINERX_HOST_DEVICE DeviceStorageType& operator[](const int64_t* index) const {
        CHAINERX_ASSERT(index == nullptr || index[0] == 0);
        return *static_cast<WithConstnessOfT<DeviceStorageType>*>(data_);
    }

    CHAINERX_HOST_DEVICE DeviceStorageType& operator[](const IndexIterator<0>& it) const { return operator[](it.index()); }

    IndexableArray<T, 0>& Permute(const Axes& /*axes*/) {
        // NOOP for 1-dimensional array.
        return *this;
    }

private:
    WithConstnessOfT<void>* data_;
};

// Static 1-dimensional specialization.
template <typename T>
class IndexableArray<T, 1> {
public:
    using ElementType = T;

private:
    template <typename U>
    using WithConstnessOfT = indexable_array_detail::WithConstnessOf<U, T>;
    using VoidType = WithConstnessOfT<void>;
    using DeviceStorageType = TypeToDeviceStorageType<T>;

public:
    IndexableArray(VoidType* data, const Strides& strides) : data_{data}, stride_{strides[0]} { CHAINERX_ASSERT(1 == strides.ndim()); }

    IndexableArray(const Array& array, const Strides& strides) : IndexableArray{internal::GetRawOffsetData(array), strides} {
        CHAINERX_ASSERT(TypeToDtype<T> == array.dtype());

#if CHAINERX_DEBUG
        std::tie(first_, last_) = indexable_array_detail::GetDataRange(array);
#endif  // CHAINERX_DEBUG
    }

    explicit IndexableArray(const Array& array) : IndexableArray{array, array.strides()} {}

    CHAINERX_HOST_DEVICE constexpr int8_t ndim() const { return 1; }

    CHAINERX_HOST_DEVICE const int64_t* strides() const { return &stride_; }

    CHAINERX_HOST_DEVICE VoidType* data() const { return data_; }

    CHAINERX_HOST_DEVICE DeviceStorageType& operator[](const int64_t* index) const {
        auto data_ptr = reinterpret_cast<WithConstnessOfT<uint8_t>*>(data_) +
                        stride_ * index[0];  // NOLINT(cppcoreguidelines-pro-type-reinterpret-cast)
#if CHAINERX_DEBUG
        CHAINERX_ASSERT(first_ == nullptr || first_ <= data_ptr);
        CHAINERX_ASSERT(last_ == nullptr || data_ptr <= last_ - sizeof(T));
#endif  // CHAINERX_DEBUG
        return *static_cast<DeviceStorageType*>(static_cast<VoidType*>(data_ptr));
    }

    CHAINERX_HOST_DEVICE DeviceStorageType& operator[](const IndexIterator<1>& it) const { return operator[](it.index()); }

    IndexableArray<T, 1>& Permute(const Axes& /*axes*/) {
        // NOOP for 1-dimensional array.
        return *this;
    }

private:
    WithConstnessOfT<void>* data_;
#if CHAINERX_DEBUG
    const uint8_t* first_{nullptr};
    const uint8_t* last_{nullptr};
#endif  // CHAINERX_DEBUG
    int64_t stride_{};
};

// Runtime determined dynamic dimension specialization.
template <typename T>
class IndexableArray<T, kDynamicNdim> {
public:
    using ElementType = T;

private:
    template <typename U>
    using WithConstnessOfT = indexable_array_detail::WithConstnessOf<U, T>;
    using VoidType = WithConstnessOfT<void>;
    using DeviceStorageType = TypeToDeviceStorageType<T>;

public:
    IndexableArray(WithConstnessOfT<void>* data, const Strides& strides) : data_{data}, ndim_{strides.ndim()} {
        std::copy(strides.begin(), strides.end(), strides_);
    }

    IndexableArray(const Array& array, const Strides& strides) : IndexableArray{internal::GetRawOffsetData(array), strides} {
        CHAINERX_ASSERT(TypeToDtype<T> == array.dtype());

#if CHAINERX_DEBUG
        std::tie(first_, last_) = indexable_array_detail::GetDataRange(array);
#endif  // CHAINERX_DEBUG
    }

    explicit IndexableArray(const Array& array) : IndexableArray{array, array.strides()} {}

    CHAINERX_HOST_DEVICE int8_t ndim() const { return ndim_; }

    CHAINERX_HOST_DEVICE const int64_t* strides() const { return strides_; }

    CHAINERX_HOST_DEVICE VoidType* data() const { return data_; }

    CHAINERX_HOST_DEVICE DeviceStorageType& operator[](const int64_t* index) const {
        auto data_ptr = static_cast<WithConstnessOfT<uint8_t>*>(data_);
        for (int8_t dim = 0; dim < ndim_; ++dim) {
            data_ptr += strides_[dim] * index[dim];
        }
#if CHAINERX_DEBUG
        CHAINERX_ASSERT(first_ == nullptr || first_ <= data_ptr);
        CHAINERX_ASSERT(last_ == nullptr || data_ptr <= last_ - sizeof(T));
#endif  // CHAINERX_DEBUG
        return *static_cast<DeviceStorageType*>(static_cast<VoidType*>(data_ptr));
    }

    CHAINERX_HOST_DEVICE DeviceStorageType& operator[](const IndexIterator<kDynamicNdim>& it) const { return operator[](it.index()); }

    // Permutes the axes.
    //
    // Given axes may be fewer than that held by the array.
    // In that case, the axes in the array will be reduced.
    //
    // It is the caller's responsibility to ensure validity of permutation.
    // If the permutation is invalid, the behavior is undefined.
    IndexableArray<T, kDynamicNdim>& Permute(const Axes& axes) {
        CHAINERX_ASSERT(axes.size() <= static_cast<size_t>(ndim_));
        int64_t c[kMaxNdim]{};
        std::copy(std::begin(strides_), std::end(strides_), c);
        for (size_t i = 0; i < axes.size(); ++i) {
            strides_[i] = c[axes[i]];
        }
        ndim_ = static_cast<int8_t>(axes.size());
        return *this;
    }

private:
    WithConstnessOfT<void>* data_;
#if CHAINERX_DEBUG
    const uint8_t* first_{nullptr};
    const uint8_t* last_{nullptr};
#endif  // CHAINERX_DEBUG
    int64_t strides_[kMaxNdim];
    int8_t ndim_;
};

}  // namespace chainerx


#pragma once
#include <Hypo/Config.h>
#include <Hypo/System/Exports.h>
#include <gsl/span>
#include <Hypo/System/Streams/MemoryStream.h>
#include <Hypo/System/Buffer/Buffer.h>

namespace Hypo
{
	class HYPO_SYSTEM_API BinaryReader
	{
	public:
		BinaryReader(std::istream& stream);

		BinaryReader& operator >>(uInt8& value);
		BinaryReader& operator >>(uInt16& value);
		BinaryReader& operator >>(uInt32& value);
		BinaryReader& operator >>(uInt64& value);
		BinaryReader& operator >>(Int8& value );
		BinaryReader& operator >>(Int16& value);
		BinaryReader& operator >>(Int32& value);
		BinaryReader& operator >>(Int64& value);

		BinaryReader& operator >>(std::string& value);

		template<typename T>
		BinaryReader& operator >>(std::vector<T>& value)
		{
			uInt32 size = 0;
			*this >> size;

			for (int i = 0; i < size; i++)
			{
				T elem;
				*this >> elem;
				value.push_back(elem);
			}
			return *this;
		}

		std::vector<unsigned char> ReadRaw(int size);
		void ReadRaw(void* buffer, int size);

		void ReadBOM();

		void Read7BitEncoded(uInt32& value);
		void Read7BitEncoded(uInt64& value);

		bool Good()const { return m_Stream.good(); }
		bool Bad()const { return m_Stream.bad(); }
		bool Fail() const { return m_Stream.fail(); }
		bool Eof() const { return m_Stream.eof(); }

		std::istream& GetStream() const { return m_Stream; }

	private:
		std::istream& m_Stream;
	};

	template <typename T>
	class BasicMemoryBinaryReader : public BinaryReader
		/// A convenient wrapper for using Buffer and MemoryStream with BinaryReader.
	{
	public:
		BasicMemoryBinaryReader(const Buffer<T>& data) :
			BinaryReader(_istr),
			_data(data),
			_istr(data.begin(), data.Capacity())
		{
		}

		~BasicMemoryBinaryReader()
		{
		}

		const Buffer<T>& data() const
		{
			return _data;
		}

		const MemoryInputStream& stream() const
		{
			return _istr;
		}

		MemoryInputStream& stream()
		{
			return _istr;
		}

	private:
		const Buffer<T>& _data;
		MemoryInputStream _istr;
	};

	typedef BasicMemoryBinaryReader<char> MemoryBinaryReader;

}