/**
 * @file pflow.h
 * @brief Public header file for power flow application
 */

#ifndef PFLOW_H
#define PFLOW_H

#include <petsc.h>
#include "cJSON.h"

typedef struct _p_PFLOW *PFLOW;

PETSC_EXTERN PetscErrorCode PFLOWCreate(MPI_Comm,PFLOW*);
PETSC_EXTERN PetscErrorCode PFLOWDestroy(PFLOW*);
PETSC_EXTERN PetscErrorCode PFLOWReadMatPowerData(PFLOW,const char[]);
PETSC_EXTERN PetscErrorCode PFLOWSolve(PFLOW);
PETSC_EXTERN PetscErrorCode PFLOWPostSolve(PFLOW);

/* FUNCTIONS FOR PARSING DATA */
int PFLOWParserGetVal(cJSON *, const char *, double *, int);
double PFLOWParserGet(PFLOW,int,char *, const char *);
void PFLOWParserSet(PFLOW,int,char *, PetscScalar, const char *);

#endif


/**
 * \copyright
 * Copyright (c) 2012-2018, OpenGeoSys Community (http://www.opengeosys.org)
 *            Distributed under a Modified BSD License.
 *              See accompanying file LICENSE.txt or
 *              http://www.opengeosys.org/project/license
 *
 */

#pragma once

/// The LinearSolverLibrarySetup takes care of proper initialization and
/// shutting down of an external linear solver library. The concrete
/// implementation is chosen by the build system.
/// An object of this class must be created at the begining of the scope where
/// it is used. When the scope closes (or the object is destroyed explicitly)
/// library shutting down functions are automatically called.
/// The default implementation is empty providing polymorphic behaviour when
/// using this class.

#include "NumLib/DOF/GlobalMatrixProviders.h"

#if defined(USE_PETSC)
#include <petsc.h>
#include <mpi.h>
namespace ApplicationsLib
{
struct LinearSolverLibrarySetup final
{
    LinearSolverLibrarySetup(int argc, char* argv[])
    {
        MPI_Init(&argc, &argv);
        char help[] = "ogs6 with PETSc \n";
        PetscInitialize(&argc, &argv, nullptr, help);
    }

    ~LinearSolverLibrarySetup()
    {
        NumLib::cleanupGlobalMatrixProviders();
        PetscFinalize();
        MPI_Finalize();
    }
};
}    // ApplicationsLib
#elif defined(USE_LIS)
#include <lis.h>
namespace ApplicationsLib
{
struct LinearSolverLibrarySetup final
{
    LinearSolverLibrarySetup(int argc, char* argv[])
    {
        lis_initialize(&argc, &argv);
    }

    ~LinearSolverLibrarySetup()
    {
        NumLib::cleanupGlobalMatrixProviders();
        lis_finalize();
    }
};
}    // ApplicationsLib
#else
namespace ApplicationsLib
{
struct LinearSolverLibrarySetup final
{
    LinearSolverLibrarySetup(int /*argc*/, char* /*argv*/[]) {}
    ~LinearSolverLibrarySetup()
    {
        NumLib::cleanupGlobalMatrixProviders();
    }
};
}    // ApplicationsLib
#endif


#pragma once

#include <gsl/span>
#include <atomic>

namespace Halley {
	// This is a lock-free, wait-free ring buffer
	// It is only thread safe for one producer and one consumer at any given time
	// Loosely based on implementation from Game Audio Programming 2

    template <typename T>
    class RingBuffer {
    public:
    	explicit RingBuffer(size_t capacity)
	        : numEntries(0)
    	{
            entries.resize(capacity);
    	}

    	size_t availableToRead() const
    	{
            return numEntries.load();
    	}

        size_t availableToWrite() const
        {
            return entries.size() - numEntries.load();
        }

    	bool canWrite(size_t n) const
    	{
            return numEntries.load() + n <= entries.size();
    	}

    	bool canRead(size_t n) const
    	{
            return numEntries.load() >= n;
    	}

    	bool empty() const
    	{
            return numEntries.load() == 0;
    	}

        void writeOne(T e)
        {
            Expects(canWrite(1));
            entries[writePos] = std::move(e);
            writePos = (writePos + 1) % entries.size();
            ++numEntries;
        }

        void write(gsl::span<T> es)
        {
            const size_t numToWrite = size_t(es.size());
            Expects(canWrite(numToWrite));
            const size_t spaceToEnd = entries.size() - writePos;
            const size_t nToWrite1 = std::min(spaceToEnd, numToWrite);

            for (size_t i = 0; i < nToWrite1; ++i) {
                entries[writePos + i] = std::move(es[i]);
            }

            const size_t nToWrite2 = numToWrite - nToWrite1;
            for (size_t i = 0; i < nToWrite2; ++i) {
                entries[i] = std::move(es[i + nToWrite1]);
            }

            writePos = (writePos + numToWrite) % entries.size();
            numEntries.fetch_add(numToWrite);
        }

    	void write(gsl::span<const T> es)
    	{
            const size_t numToWrite = size_t(es.size());
            Expects(canWrite(numToWrite));
            const size_t spaceToEnd = entries.size() - writePos;
            const size_t nToWrite1 = std::min(spaceToEnd, numToWrite);

    		for (size_t i = 0; i < nToWrite1; ++i) {
                entries[writePos + i] = es[i];
    		}

            const size_t nToWrite2 = numToWrite - nToWrite1;
    		for (size_t i = 0; i < nToWrite2; ++i) {
                entries[i] = es[i + nToWrite1];
    		}

            writePos = (writePos + numToWrite) % entries.size();
            numEntries.fetch_add(numToWrite);
    	}

    	T readOne()
    	{
            Expects(canRead(1));
            T v = std::move(entries[readPos]);
    		entries[readPos] = T();
            readPos = (readPos + 1) % entries.size();
            --numEntries;
            return v;
    	}

    	void read(gsl::span<T> es)
    	{
            const size_t numToRead = size_t(es.size());
            Expects(canRead(numToRead));
            const size_t spaceToEnd = entries.size() - readPos;
            const size_t nToRead1 = std::min(spaceToEnd, numToRead);

            for (size_t i = 0; i < nToRead1; ++i) {
                es[i] = std::move(entries[readPos + i]);
            	entries[readPos + i] = T();
            }

            const size_t nToRead2 = numToRead - nToRead1;
            for (size_t i = 0; i < nToRead2; ++i) {
                es[i + nToRead1] = std::move(entries[i]);
            	entries[i] = T();
            }

            readPos = (readPos + numToRead) % entries.size();
            numEntries.fetch_sub(numToRead);
    	}

    private:
        size_t readPos = 0;
        size_t writePos = 0;
        std::atomic<size_t> numEntries;
        Vector<T> entries;
    };
}


#ifndef MATRIX_H_
#define MATRIX_H_

#include "matrix_funcs.h"
#ifdef NUMPY_INTERFACE
#include <Python.h>
#include <arrayobject.h>
#endif
#include <limits>
#include <assert.h>
#include <stdio.h>
#include <string.h>

#ifdef USE_MKL
#include <mkl.h>
#include <mkl_cblas.h>
#include <mkl_vsl.h>
#include <mkl_vml.h>

#define IS_MKL true

#ifdef DOUBLE_PRECISION
#define MKL_UNIFORM vdRngUniform
#define MKL_NORMAL vdRngGaussian
#define MKL_UNIFORM_RND_METHOD VSL_METHOD_DUNIFORM_STD_ACCURATE
#define MKL_GAUSSIAN_RND_METHOD VSL_METHOD_DGAUSSIAN_BOXMULLER
#define MKL_EXP vdExp
#define MKL_RECIP vdInv
#define MKL_SQUARE vdSqr
#define MKL_TANH vdTanh
#define MKL_LOG vdLn
#define MKL_VECMUL vdMul
#define MKL_VECDIV vdDiv
#else
#define MKL_UNIFORM vsRngUniform
#define MKL_NORMAL vsRngGaussian
#define MKL_UNIFORM_RND_METHOD VSL_METHOD_SUNIFORM_STD_ACCURATE
#define MKL_GAUSSIAN_RND_METHOD VSL_METHOD_SGAUSSIAN_BOXMULLER
#define MKL_EXP vsExp
#define MKL_RECIP vsInv
#define MKL_SQUARE vsSqr
#define MKL_TANH vsTanh
#define MKL_LOG vsLn
#define MKL_VECMUL vsMul
#define MKL_VECDIV vsDiv
#endif /* DOUBLE_PRECISION */

#else
#include <cblas.h>
#define IS_MKL false
#endif /* USE_MKL */

#ifdef DOUBLE_PRECISION
#define CBLAS_GEMM cblas_dgemm
#define CBLAS_SCAL cblas_dscal
#define CBLAS_AXPY cblas_daxpy
#else
#define CBLAS_GEMM cblas_sgemm
#define CBLAS_SCAL cblas_sscal
#define CBLAS_AXPY cblas_saxpy
#endif /* DOUBLE_PRECISION */

#define MTYPE_MAX numeric_limits<MTYPE>::max()

class Matrix {
private:
    MTYPE* _data;
    bool _ownsData;
    int _numRows, _numCols;
    int _numElements;
    int _numDataBytes;
    CBLAS_TRANSPOSE _trans;

    void _init(MTYPE* data, int numRows, int numCols, bool transpose, bool ownsData);
    void _tileTo2(Matrix& target) const;
    void _copyAllTo(Matrix& target) const;
    MTYPE _sum_column(int col) const;
    MTYPE _sum_row(int row) const;
    MTYPE _aggregate(MTYPE(*agg_func)(MTYPE, MTYPE), MTYPE initialValue) const;
    void _aggregate(int axis, Matrix& target, MTYPE(*agg_func)(MTYPE, MTYPE), MTYPE initialValue) const;
    MTYPE _aggregateRow(int row, MTYPE(*agg_func)(MTYPE, MTYPE), MTYPE initialValue) const;
    MTYPE _aggregateCol(int row, MTYPE(*agg_func)(MTYPE, MTYPE), MTYPE initialValue) const;
    void _updateDims(int numRows, int numCols);
    void _applyLoop(MTYPE(*func)(MTYPE));
    void _applyLoop(MTYPE (*func)(MTYPE), Matrix& target);
    void _applyLoop2(const Matrix& a, MTYPE(*func)(MTYPE, MTYPE), Matrix& target) const;
    void _applyLoop2(const Matrix& a, MTYPE (*func)(MTYPE,MTYPE, MTYPE), MTYPE scalar, Matrix& target) const;
    void _applyLoopScalar(const MTYPE scalar, MTYPE(*func)(MTYPE, MTYPE), Matrix& target) const;
    void _checkBounds(int startRow, int endRow, int startCol, int endCol) const;
    void _divideByVector(const Matrix& vec, Matrix& target);
    inline int _getNumColsBackEnd() const {
        return _trans == CblasNoTrans ? _numCols : _numRows;
    }
public:
    enum FUNCTION {
        TANH, RECIPROCAL, SQUARE, ABS, EXP, LOG, ZERO, ONE, LOGISTIC1, LOGISTIC2
    };
    Matrix();
    Matrix(int numRows, int numCols);
#ifdef NUMPY_INTERFACE
    Matrix(const PyArrayObject *src);
#endif
    Matrix(const Matrix &like);
    Matrix(MTYPE* data, int numRows, int numCols);
    Matrix(MTYPE* data, int numRows, int numCols, bool transpose);
    ~Matrix();

    inline MTYPE& getCell(int i, int j) const {
//        assert(i >= 0 && i < _numRows);
//        assert(j >= 0 && j < _numCols);
        if (_trans == CblasTrans) {
            return _data[j * _numRows + i];
        }
        return _data[i * _numCols + j];
    }

    MTYPE& operator()(int i, int j) const {
        return getCell(i, j);
    }

    inline MTYPE* getData() const {
        return _data;
    }

    inline bool isView() const {
        return !_ownsData;
    }

    inline int getNumRows() const {
        return _numRows;
    }

    inline int getNumCols() const {
        return _numCols;
    }

    inline int getNumDataBytes() const {
        return _numDataBytes;
    }

    inline int getNumElements() const {
        return _numElements;
    }

    inline CBLAS_TRANSPOSE getBLASTrans() const {
        return _trans;
    }

    inline bool isSameDims(const Matrix& a) const {
        return a.getNumRows() == getNumRows() && a.getNumCols() == getNumCols();
    }

    inline bool isTrans() const {
        return _trans == CblasTrans;
    }

    /*
     * Only use if you know what you're doing!
     * Does not update any dimensions. Just flips the _trans flag.
     *
     * Use transpose() if you want to get the transpose of this matrix.
     */
    inline void setTrans(bool trans) {
        _trans = trans ? CblasTrans : CblasNoTrans;
    }

    void apply(FUNCTION f);
    void apply(Matrix::FUNCTION f, Matrix& target);
    void subtractFromScalar(MTYPE scalar);
    void subtractFromScalar(MTYPE scalar, Matrix &target) const;
    void biggerThanScalar(MTYPE scalar);
    void smallerThanScalar(MTYPE scalar);
    void equalsScalar(MTYPE scalar);
    void biggerThanScalar(MTYPE scalar, Matrix& target) const;
    void smallerThanScalar(MTYPE scalar, Matrix& target) const;
    void equalsScalar(MTYPE scalar, Matrix& target) const;
    void biggerThan(Matrix& a);
    void biggerThan(Matrix& a, Matrix& target) const;
    void smallerThan(Matrix& a);
    void smallerThan(Matrix& a, Matrix& target) const;
    void minWith(Matrix &a);
    void minWith(Matrix &a, Matrix &target) const;
    void maxWith(Matrix &a);
    void maxWith(Matrix &a, Matrix &target) const;
    void equals(Matrix& a);
    void equals(Matrix& a, Matrix& target) const;
    void notEquals(Matrix& a) ;
    void notEquals(Matrix& a, Matrix& target) const;
    void add(const Matrix &m);
    void add(const Matrix &m, MTYPE scale);
    void add(const Matrix &m, Matrix& target);
    void add(const Matrix &m, MTYPE scale, Matrix& target);
    void subtract(const Matrix &m);
    void subtract(const Matrix &m, Matrix& target);
    void subtract(const Matrix &m, MTYPE scale);
    void subtract(const Matrix &m, MTYPE scale, Matrix& target);
    void addVector(const Matrix& vec, MTYPE scale);
    void addVector(const Matrix& vec, MTYPE scale, Matrix& target);
    void addVector(const Matrix& vec);
    void addVector(const Matrix& vec, Matrix& target);
    void addScalar(MTYPE scalar);
    void addScalar(MTYPE scalar, Matrix& target) const;
    void maxWithScalar(MTYPE scalar);
    void maxWithScalar(MTYPE scalar, Matrix &target) const;
    void minWithScalar(MTYPE scalar);
    void minWithScalar(MTYPE scalar, Matrix &target) const;
    void eltWiseMultByVector(const Matrix& vec);
    void eltWiseMultByVector(const Matrix& vec, Matrix& target);
    void eltWiseDivideByVector(const Matrix& vec);
    void eltWiseDivideByVector(const Matrix& vec, Matrix& target);
    void resize(int newNumRows, int newNumCols);
    void resize(const Matrix& like);
    Matrix& slice(int startRow, int endRow, int startCol, int endCol) const;
    void slice(int startRow, int endRow, int startCol, int endCol, Matrix &target) const;
    Matrix& sliceRows(int startRow, int endRow) const;
    void sliceRows(int startRow, int endRow, Matrix& target) const;
    Matrix& sliceCols(int startCol, int endCol) const;
    void sliceCols(int startCol, int endCol, Matrix& target) const;
    void rightMult(const Matrix &b, MTYPE scale);
    void rightMult(const Matrix &b, Matrix &target) const;
    void rightMult(const Matrix &b);
    void rightMult(const Matrix &b, MTYPE scaleAB, Matrix &target) const;
    void addProduct(const Matrix &a, const Matrix &b, MTYPE scaleAB, MTYPE scaleThis);
    void addProduct(const Matrix& a, const Matrix& b);
    void eltWiseMult(const Matrix& a);
    void eltWiseMult(const Matrix& a, Matrix& target) const;
    void eltWiseDivide(const Matrix& a);
    void eltWiseDivide(const Matrix& a, Matrix &target) const;
    Matrix& transpose() const;
    Matrix& transpose(bool hard) const;
    Matrix& tile(int timesY, int timesX) const;
    void tile(int timesY, int timesX, Matrix& target) const;
    void copy(Matrix &dest, int srcStartRow, int srcEndRow, int srcStartCol, int srcEndCol, int destStartRow, int destStartCol) const;
    Matrix& copy() const;
    void copy(Matrix& target) const;
    Matrix& sum(int axis) const;
    void sum(int axis, Matrix &target) const;
    MTYPE sum() const;
    MTYPE max() const;
    Matrix& max(int axis) const;
    void max(int axis, Matrix& target) const;
    MTYPE min() const;
    Matrix& min(int axis) const;
    void min(int axis, Matrix& target) const;
    void scale(MTYPE scale);
#ifdef USE_MKL
    void randomizeNormal(VSLStreamStatePtr stream, MTYPE mean, MTYPE stdev);
    void randomizeUniform(VSLStreamStatePtr stream);
    void randomizeNormal(VSLStreamStatePtr stream);
#else
    void randomizeNormal(MTYPE mean, MTYPE stdev);
    void randomizeUniform();
    void randomizeNormal();
#endif
    void print() const;
    void print(int startRow,int rows, int startCol,int cols) const;
    void print(int rows, int cols) const;
};

#endif /* MATRIX_H_ */


/* vector/gsl_vector_double.h
 *
 * Copyright (C) 1996, 1997, 1998, 1999, 2000, 2007 Gerard Jungman, Brian Gough
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 3 of the License, or (at
 * your option) any later version.
 *
 * This program is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
 */

#ifndef __GSL_VECTOR_DOUBLE_H__
#define __GSL_VECTOR_DOUBLE_H__

#if !defined( GSL_FUN )
#  if !defined( GSL_DLL )
#    define GSL_FUN extern
#  elif defined( BUILD_GSL_DLL )
#    define GSL_FUN extern __declspec(dllexport)
#  else
#    define GSL_FUN extern __declspec(dllimport)
#  endif
#endif

#include <stdlib.h>
#include <gsl/gsl_types.h>
#include <gsl/gsl_errno.h>
#include <gsl/gsl_inline.h>
#include <gsl/gsl_check_range.h>
#include <gsl/gsl_block_double.h>

#undef __BEGIN_DECLS
#undef __END_DECLS
#ifdef __cplusplus
# define __BEGIN_DECLS extern "C" {
# define __END_DECLS }
#else
# define __BEGIN_DECLS /* empty */
# define __END_DECLS /* empty */
#endif

__BEGIN_DECLS

typedef struct
{
  size_t size;
  size_t stride;
  double *data;
  gsl_block *block;
  int owner;
}
gsl_vector;

typedef struct
{
  gsl_vector vector;
} _gsl_vector_view;

typedef _gsl_vector_view gsl_vector_view;

typedef struct
{
  gsl_vector vector;
} _gsl_vector_const_view;

typedef const _gsl_vector_const_view gsl_vector_const_view;

/* Allocation */

GSL_FUN gsl_vector *gsl_vector_alloc (const size_t n);
GSL_FUN gsl_vector *gsl_vector_calloc (const size_t n);

GSL_FUN gsl_vector *gsl_vector_alloc_from_block (gsl_block * b,
                                                     const size_t offset,
                                                     const size_t n,
                                                     const size_t stride);

GSL_FUN gsl_vector *gsl_vector_alloc_from_vector (gsl_vector * v,
                                                      const size_t offset,
                                                      const size_t n,
                                                      const size_t stride);

GSL_FUN void gsl_vector_free (gsl_vector * v);

/* Views */

GSL_FUN _gsl_vector_view
gsl_vector_view_array (double *v, size_t n);

GSL_FUN _gsl_vector_view
gsl_vector_view_array_with_stride (double *base,
                                         size_t stride,
                                         size_t n);

GSL_FUN _gsl_vector_const_view
gsl_vector_const_view_array (const double *v, size_t n);

GSL_FUN _gsl_vector_const_view
gsl_vector_const_view_array_with_stride (const double *base,
                                               size_t stride,
                                               size_t n);

GSL_FUN _gsl_vector_view
gsl_vector_subvector (gsl_vector *v,
                            size_t i,
                            size_t n);

GSL_FUN _gsl_vector_view
gsl_vector_subvector_with_stride (gsl_vector *v,
                                        size_t i,
                                        size_t stride,
                                        size_t n);

GSL_FUN _gsl_vector_const_view
gsl_vector_const_subvector (const gsl_vector *v,
                                  size_t i,
                                  size_t n);

GSL_FUN _gsl_vector_const_view
gsl_vector_const_subvector_with_stride (const gsl_vector *v,
                                              size_t i,
                                              size_t stride,
                                              size_t n);

/* Operations */

GSL_FUN void gsl_vector_set_zero (gsl_vector * v);
GSL_FUN void gsl_vector_set_all (gsl_vector * v, double x);
GSL_FUN int gsl_vector_set_basis (gsl_vector * v, size_t i);

GSL_FUN int gsl_vector_fread (FILE * stream, gsl_vector * v);
GSL_FUN int gsl_vector_fwrite (FILE * stream, const gsl_vector * v);
GSL_FUN int gsl_vector_fscanf (FILE * stream, gsl_vector * v);
GSL_FUN int gsl_vector_fprintf (FILE * stream, const gsl_vector * v,
                              const char *format);

GSL_FUN int gsl_vector_memcpy (gsl_vector * dest, const gsl_vector * src);

GSL_FUN int gsl_vector_reverse (gsl_vector * v);

GSL_FUN int gsl_vector_swap (gsl_vector * v, gsl_vector * w);
GSL_FUN int gsl_vector_swap_elements (gsl_vector * v, const size_t i, const size_t j);

GSL_FUN double gsl_vector_max (const gsl_vector * v);
GSL_FUN double gsl_vector_min (const gsl_vector * v);
GSL_FUN void gsl_vector_minmax (const gsl_vector * v, double * min_out, double * max_out);

GSL_FUN size_t gsl_vector_max_index (const gsl_vector * v);
GSL_FUN size_t gsl_vector_min_index (const gsl_vector * v);
GSL_FUN void gsl_vector_minmax_index (const gsl_vector * v, size_t * imin, size_t * imax);

GSL_FUN int gsl_vector_add (gsl_vector * a, const gsl_vector * b);
GSL_FUN int gsl_vector_sub (gsl_vector * a, const gsl_vector * b);
GSL_FUN int gsl_vector_mul (gsl_vector * a, const gsl_vector * b);
GSL_FUN int gsl_vector_div (gsl_vector * a, const gsl_vector * b);
GSL_FUN int gsl_vector_scale (gsl_vector * a, const double x);
GSL_FUN int gsl_vector_add_constant (gsl_vector * a, const double x);
GSL_FUN int gsl_vector_axpby (const double alpha, const gsl_vector * x, const double beta, gsl_vector * y);
GSL_FUN double gsl_vector_sum (const gsl_vector * a);

GSL_FUN int gsl_vector_equal (const gsl_vector * u,
                            const gsl_vector * v);

GSL_FUN int gsl_vector_isnull (const gsl_vector * v);
GSL_FUN int gsl_vector_ispos (const gsl_vector * v);
GSL_FUN int gsl_vector_isneg (const gsl_vector * v);
GSL_FUN int gsl_vector_isnonneg (const gsl_vector * v);

GSL_FUN INLINE_DECL double gsl_vector_get (const gsl_vector * v, const size_t i);
GSL_FUN INLINE_DECL void gsl_vector_set (gsl_vector * v, const size_t i, double x);
GSL_FUN INLINE_DECL double * gsl_vector_ptr (gsl_vector * v, const size_t i);
GSL_FUN INLINE_DECL const double * gsl_vector_const_ptr (const gsl_vector * v, const size_t i);

#ifdef HAVE_INLINE

INLINE_FUN
double
gsl_vector_get (const gsl_vector * v, const size_t i)
{
#if GSL_RANGE_CHECK
  if (GSL_RANGE_COND(i >= v->size))
    {
      GSL_ERROR_VAL ("index out of range", GSL_EINVAL, 0);
    }
#endif
  return v->data[i * v->stride];
}

INLINE_FUN
void
gsl_vector_set (gsl_vector * v, const size_t i, double x)
{
#if GSL_RANGE_CHECK
  if (GSL_RANGE_COND(i >= v->size))
    {
      GSL_ERROR_VOID ("index out of range", GSL_EINVAL);
    }
#endif
  v->data[i * v->stride] = x;
}

INLINE_FUN
double *
gsl_vector_ptr (gsl_vector * v, const size_t i)
{
#if GSL_RANGE_CHECK
  if (GSL_RANGE_COND(i >= v->size))
    {
      GSL_ERROR_NULL ("index out of range", GSL_EINVAL);
    }
#endif
  return (double *) (v->data + i * v->stride);
}

INLINE_FUN
const double *
gsl_vector_const_ptr (const gsl_vector * v, const size_t i)
{
#if GSL_RANGE_CHECK
  if (GSL_RANGE_COND(i >= v->size))
    {
      GSL_ERROR_NULL ("index out of range", GSL_EINVAL);
    }
#endif
  return (const double *) (v->data + i * v->stride);
}
#endif /* HAVE_INLINE */

__END_DECLS

#endif /* __GSL_VECTOR_DOUBLE_H__ */


/* monte/gsl_monte_plain.h
 *
 * Copyright (C) 1996, 1997, 1998, 1999, 2000 Michael Booth
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 3 of the License, or (at
 * your option) any later version.
 *
 * This program is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
 */

/* Plain Monte-Carlo. */

/* Author: MJB */

#ifndef __GSL_MONTE_PLAIN_H__
#define __GSL_MONTE_PLAIN_H__

#if !defined( GSL_FUN )
#  if !defined( GSL_DLL )
#    define GSL_FUN extern
#  elif defined( BUILD_GSL_DLL )
#    define GSL_FUN extern __declspec(dllexport)
#  else
#    define GSL_FUN extern __declspec(dllimport)
#  endif
#endif

#include <stdio.h>
#include <gsl/gsl_monte.h>
#include <gsl/gsl_rng.h>

#undef __BEGIN_DECLS
#undef __END_DECLS
#ifdef __cplusplus
# define __BEGIN_DECLS extern "C" {
# define __END_DECLS }
#else
# define __BEGIN_DECLS /* empty */
# define __END_DECLS /* empty */
#endif

__BEGIN_DECLS

typedef struct {
  size_t dim;
  double *x;
} gsl_monte_plain_state;

GSL_FUN int
gsl_monte_plain_integrate (const gsl_monte_function * f,
                           const double xl[], const double xu[],
                           const size_t dim,
                           const size_t calls,
                           gsl_rng * r,
                           gsl_monte_plain_state * state,
                           double *result, double *abserr);

GSL_FUN gsl_monte_plain_state* gsl_monte_plain_alloc(size_t dim);

GSL_FUN int gsl_monte_plain_init(gsl_monte_plain_state* state);

GSL_FUN void gsl_monte_plain_free (gsl_monte_plain_state* state);

__END_DECLS

#endif /* __GSL_MONTE_PLAIN_H__ */


/*
 *
 * Copyright (c) Kresimir Fresl 2002
 *
 * Distributed under the Boost Software License, Version 1.0.
 * (See accompanying file LICENSE_1_0.txt or copy at
 * http://www.boost.org/LICENSE_1_0.txt)
 *
 * Author acknowledges the support of the Faculty of Civil Engineering,
 * University of Zagreb, Croatia.
 *
 */

#ifndef BOOST_NUMERIC_BINDINGS_BLAS_DETAIL_CBLAS_H
#define BOOST_NUMERIC_BINDINGS_BLAS_DETAIL_CBLAS_H

//
// MKL-specific CBLAS include
//
#if defined BOOST_NUMERIC_BINDINGS_BLAS_MKL

extern "C" {
#include <mkl_cblas.h>
//#include <mkl_service.h>
//
// mkl_types.h defines P4 macro which breaks MPL, undefine it here.
//
#undef P4
}

//
// Default CBLAS include
//
#else

extern "C" {
#include <cblas.h>
}

#endif
#endif


/* ieee-utils/fp-tru64.c
 *
 * Copyright (C) 1996, 1997, 1998, 1999, 2000 Tim Mooney
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 3 of the License, or (at
 * your option) any later version.
 *
 * This program is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
 */

/*
 * Under Compaq's Unix with the silly name, read the man pages for read_rnd,
 * write_rnd, and ieee(3) for more information on the functions used here.
 *
 * Note that enabling control of dynamic rounding mode (via write_rnd) requires
 * that you pass a special flag to your C compiler.  For Compaq's C compiler
 * the flag is `-fprm d', for gcc it's `-mfp-rounding-mode=d'.
 *
 * Enabling the trap control (via ieee_set_fp_control) also requires a
 * flag be passed to the C compiler.  The flag for Compaq's C compiler
 * is `-ieee' and for gcc it's `-mieee'.

 * We have not implemented the `inexact' case, since it is rarely used
 * and requires the library being built with an additional compiler
 * flag that can degrade performance for everything else. If you need
 * to add support for `inexact' the relevant flag for Compaq's
 * compiler is `-ieee_with_inexact', and the flag for gcc is
 * `-mieee-with-inexact'.
 *
 * Problem have been reported with the "fixed" float.h installed with
 * gcc-2.95 lacking some of the definitions in the system float.h (the
 * symptoms are errors like: `FP_RND_RN' undeclared). To work around
 * this we can include the system float.h before the gcc version, e.g.
 *
 *  #include "/usr/include/float.h"
 *  #include <float.h>
 */

#include <float.h>

#ifndef FP_RND_RN
#  undef _FLOAT_H_
#  include "/usr/include/float.h"
#  undef _FLOAT_H_
#  include <float.h>
#endif

#include <machine/fpu.h>
#include <stdio.h>
#include <gsl/gsl_ieee_utils.h>
#include <gsl/gsl_errno.h>

int
gsl_ieee_set_mode (int precision, int rounding, int exception_mask)
{
  unsigned long int mode = 0 ;
  unsigned int    rnd  = 0 ;

/* I'm actually not completely sure that the alpha only supports default
 * precisions rounding, but I couldn't find any information regarding this, so
 * it seems safe to assume this for now until it's proven otherwise.
 */

  switch (precision)
    {
    case GSL_IEEE_SINGLE_PRECISION:
      GSL_ERROR ("Tru64 Unix on the alpha only supports default precision rounding",
                 GSL_EUNSUP) ;
      break ;
    case GSL_IEEE_DOUBLE_PRECISION:
      GSL_ERROR ("Tru64 Unix on the alpha only supports default precision rounding",
                 GSL_EUNSUP) ;
      break ;
    case GSL_IEEE_EXTENDED_PRECISION:
      GSL_ERROR ("Tru64 Unix on the alpha only supports default precision rounding",
                 GSL_EUNSUP) ;
      break ;
    }

  switch (rounding)
    {
    case GSL_IEEE_ROUND_TO_NEAREST:
      rnd = FP_RND_RN ;
      write_rnd (rnd) ;
      break ;
    case GSL_IEEE_ROUND_DOWN:
      rnd = FP_RND_RM ;
      write_rnd (rnd) ;
      break ;
    case GSL_IEEE_ROUND_UP:
      rnd = FP_RND_RP ;
      write_rnd (rnd) ;
      break ;
    case GSL_IEEE_ROUND_TO_ZERO:
      rnd = FP_RND_RZ ;
      write_rnd (rnd) ;
      break ;
    default:
      rnd = FP_RND_RN ;
      write_rnd (rnd) ;
    }

  /* Turn on all the exceptions apart from 'inexact' */

  /* from the ieee(3) man page:
   * IEEE_TRAP_ENABLE_INV       ->      Invalid operation
   * IEEE_TRAP_ENABLE_DZE       ->      Divide by 0
   * IEEE_TRAP_ENABLE_OVF       ->      Overflow
   * IEEE_TRAP_ENABLE_UNF       ->      Underflow
   * IEEE_TRAP_ENABLE_INE       ->      Inexact (requires special option to C compiler)
   * IEEE_TRAP_ENABLE_DNO       ->      denormal operand
   * Note: IEEE_TRAP_ENABLE_DNO is not supported on OSF 3.x or Digital Unix
   * 4.0 - 4.0d(?).
   * IEEE_TRAP_ENABLE_MASK      ->      mask of all the trap enables
   * IEEE_MAP_DMZ                       ->      map denormal inputs to zero
   * IEEE_MAP_UMZ                       ->      map underflow results to zero
   */

  mode = IEEE_TRAP_ENABLE_INV | IEEE_TRAP_ENABLE_DZE | IEEE_TRAP_ENABLE_OVF
                | IEEE_TRAP_ENABLE_UNF ;

  if (exception_mask & GSL_IEEE_MASK_INVALID)
    mode &= ~ IEEE_TRAP_ENABLE_INV ;

  if (exception_mask & GSL_IEEE_MASK_DENORMALIZED)
    {
#ifdef IEEE_TRAP_ENABLE_DNO
     mode &= ~ IEEE_TRAP_ENABLE_DNO ;
#else
     GSL_ERROR ("Sorry, this version of Digital Unix does not support denormalized operands", GSL_EUNSUP) ;
#endif
    }

  if (exception_mask & GSL_IEEE_MASK_DIVISION_BY_ZERO)
    mode &= ~ IEEE_TRAP_ENABLE_DZE ;

  if (exception_mask & GSL_IEEE_MASK_OVERFLOW)
    mode &= ~ IEEE_TRAP_ENABLE_OVF ;

  if (exception_mask & GSL_IEEE_MASK_UNDERFLOW)
    mode &=  ~ IEEE_TRAP_ENABLE_UNF ;

  if (exception_mask & GSL_IEEE_TRAP_INEXACT)
    {
      /* To implement this would require a special flag to the C
       compiler which can cause degraded performance */

      GSL_ERROR ("Sorry, GSL does not implement trap-inexact for Tru64 Unix on the alpha - see fp-tru64.c for details", GSL_EUNSUP) ;

      /* In case you need to add it, the appropriate line would be
       *
       *  mode |= IEEE_TRAP_ENABLE_INE ;
       *
       */

    }
  else
    {
      mode &= ~ IEEE_TRAP_ENABLE_INE ;
    }

  ieee_set_fp_control (mode) ;

  return GSL_SUCCESS ;
}


#include <mpi.h>
#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#include <math.h>
#include <sys/time.h>
#include <sys/resource.h>
#include <unistd.h>
#include <signal.h>
#include <gsl/gsl_rng.h>

#include "allvars.h"
#include "proto.h"

#define TAG_DATAIN  100
#define TAG_DATAOUT 101

static void serial_sort(char *base, size_t nmemb, size_t size, int (*compar) (const void *, const void *));
static void msort_serial_with_tmp(char *base, size_t n, size_t s, int (*compar) (const void *, const void *),
				  char *t);
static void parallel_merge(int master, int ncpu, size_t * nlist, size_t nmax, char *base, size_t nmemb,
			   size_t size, int (*compar) (const void *, const void *));

void parallel_sort(void *base, size_t nmemb, size_t size, int (*compar) (const void *, const void *))
{
  int i, ncpu_in_group, master, groupnr;
  size_t nmax, *nlist;

  serial_sort((char *) base, nmemb, size, compar);

  nlist = (size_t *) mymalloc(NTask * sizeof(size_t));

  MPI_Allgather(&nmemb, sizeof(size_t), MPI_BYTE, nlist, sizeof(size_t), MPI_BYTE, MPI_COMM_WORLD);

  for(i = 0, nmax = 0; i < NTask; i++)
    if(nlist[i] > nmax)
      nmax = nlist[i];

  for(ncpu_in_group = 2; ncpu_in_group <= (1 << PTask); ncpu_in_group *= 2)
    {
      groupnr = ThisTask / ncpu_in_group;

      master = ncpu_in_group * groupnr;

      parallel_merge(master, ncpu_in_group, nlist, nmax, (char *) base, nmemb, size, compar);
    }

  myfree(nlist);
}

void parallel_merge(int master, int ncpu, size_t * nlist, size_t nmax,
		    char *base, size_t nmemb, size_t size, int (*compar) (const void *, const void *))
{
  size_t na, nb, nr;
  int cpua, cpub, cpur;
  char *list_a, *list_b, *list_r;

  if(master + ncpu / 2 >= NTask)	/* nothing to do */
    return;

  if(ThisTask != master)
    {
      if(nmemb)
	{
	  list_r = (char *) mymalloc(nmemb * size);

	  MPI_Request requests[2];

	  MPI_Isend(base, nmemb * size, MPI_BYTE, master, TAG_DATAIN, MPI_COMM_WORLD, &requests[0]);
	  MPI_Irecv(list_r, nmemb * size, MPI_BYTE, master, TAG_DATAOUT, MPI_COMM_WORLD, &requests[1]);
	  MPI_Waitall(2, requests, MPI_STATUSES_IGNORE);

	  memcpy(base, list_r, nmemb * size);
	  myfree(list_r);
	}
    }
  else
    {
      list_a = (char *) mymalloc(nmax * size);
      list_b = (char *) mymalloc(nmax * size);
      list_r = (char *) mymalloc(nmax * size);

      cpua = master;
      cpub = master + ncpu / 2;
      cpur = master;

      na = 0;
      nb = 0;
      nr = 0;

      memcpy(list_a, base, nmemb * size);
      if(nlist[cpub])
	MPI_Recv(list_b, nlist[cpub] * size, MPI_BYTE, cpub, TAG_DATAIN, MPI_COMM_WORLD, MPI_STATUS_IGNORE);

      while(cpur < master + ncpu && cpur < NTask)
	{
	  while(na >= nlist[cpua] && cpua < master + ncpu / 2 - 1)
	    {
	      cpua++;
	      if(nlist[cpua])
		MPI_Recv(list_a, nlist[cpua] * size, MPI_BYTE, cpua, TAG_DATAIN, MPI_COMM_WORLD,
			 MPI_STATUS_IGNORE);
	      na = 0;
	    }
	  while(nb >= nlist[cpub] && cpub < master + ncpu - 1 && cpub < NTask - 1)
	    {
	      cpub++;
	      if(nlist[cpub])
		MPI_Recv(list_b, nlist[cpub] * size, MPI_BYTE, cpub, TAG_DATAIN, MPI_COMM_WORLD,
			 MPI_STATUS_IGNORE);
	      nb = 0;
	    }

	  while(nr >= nlist[cpur])
	    {
	      if(cpur == master)
		memcpy(base, list_r, nr * size);
	      else
		{
		  if(nlist[cpur])
		    MPI_Send(list_r, nlist[cpur] * size, MPI_BYTE, cpur, TAG_DATAOUT, MPI_COMM_WORLD);
		}
	      nr = 0;
	      cpur++;
	    }

	  if(na < nlist[cpua] && nb < nlist[cpub])
	    {
	      if(compar(list_a + na * size, list_b + nb * size) < 0)
		{
		  memcpy(list_r + nr * size, list_a + na * size, size);
		  na++;
		  nr++;
		}
	      else
		{
		  memcpy(list_r + nr * size, list_b + nb * size, size);
		  nb++;
		  nr++;
		}
	    }
	  else if(na < nlist[cpua])
	    {
	      memcpy(list_r + nr * size, list_a + na * size, size);
	      na++;
	      nr++;
	    }
	  else if(nb < nlist[cpub])
	    {
	      memcpy(list_r + nr * size, list_b + nb * size, size);
	      nb++;
	      nr++;
	    }

	}

      myfree(list_r);
      myfree(list_b);
      myfree(list_a);
    }
}

static void serial_sort(char *base, size_t nmemb, size_t size, int (*compar) (const void *, const void *))
{
  const size_t storage = nmemb * size;

  char *tmp = (char *) mymalloc(storage);

  msort_serial_with_tmp(base, nmemb, size, compar, tmp);

  myfree(tmp);
}

static void msort_serial_with_tmp(char *base, size_t n, size_t s, int (*compar) (const void *, const void *),
				  char *t)
{
  char *tmp;
  char *b1, *b2;
  size_t n1, n2;

  if(n <= 1)
    return;

  n1 = n / 2;
  n2 = n - n1;
  b1 = base;
  b2 = base + n1 * s;

  msort_serial_with_tmp(b1, n1, s, compar, t);
  msort_serial_with_tmp(b2, n2, s, compar, t);

  tmp = t;

  while(n1 > 0 && n2 > 0)
    {
      if(compar(b1, b2) < 0)
	{
	  --n1;
	  memcpy(tmp, b1, s);
	  tmp += s;
	  b1 += s;
	}
      else
	{
	  --n2;
	  memcpy(tmp, b2, s);
	  tmp += s;
	  b2 += s;
	}
    }

  if(n1 > 0)
    memcpy(tmp, b1, n1 * s);

  memcpy(base, t, (n - n2) * s);
}


#pragma once

#include <memory_resource>
#include <vector>

#include <thrust/execution_policy.h>
#include <thrust/functional.h>
#include <thrust/inner_product.h>
#include <thrust/iterator/zip_iterator.h>
#include <thrust/merge.h>
#include <thrust/reduce.h>
#include <thrust/sort.h>
#include <thrust/tuple.h>

#include <gsl-lite/gsl-lite.hpp>

#include <cuda/define_specifiers.hpp>

#include <thrustshift/copy.h>
#include <thrustshift/equal.h>
#include <thrustshift/managed-vector.h>
#include <thrustshift/math.h>
#include <thrustshift/memory-resource.h>
#include <thrustshift/not-a-vector.h>
#include <thrustshift/transform.h>

namespace thrustshift {

enum class storage_order_t { row_major, col_major, none };

template <typename DataType, typename IndexType>
class COO {

   public:
	using value_type = DataType;
	using index_type = IndexType;

	COO() : num_cols_(0), num_rows_(0) {
	}

	template <class MemoryResource>
	COO(size_t nnz,
	    size_t num_rows,
	    size_t num_cols,
	    MemoryResource& memory_resource)
	    : values_(nnz, &memory_resource),
	      row_indices_(nnz, &memory_resource),
	      col_indices_(nnz, &memory_resource),
	      num_rows_(num_rows),
	      num_cols_(num_cols),
	      storage_order_(storage_order_t::none) {
	}

	COO(size_t nnz, size_t num_rows, size_t num_cols)
	    : COO(nnz, num_rows, num_cols, pmr::default_resource) {
	}

	template <class DataRange,
	          class RowIndRange,
	          class ColIndRange,
	          class MemoryResource>
	COO(DataRange&& values,
	    RowIndRange&& row_indices,
	    ColIndRange&& col_indices,
	    size_t num_rows,
	    size_t num_cols,
	    storage_order_t storage_order,
	    MemoryResource& memory_resource)
	    : values_(values.begin(), values.end(), &memory_resource),
	      row_indices_(row_indices.begin(),
	                   row_indices.end(),
	                   &memory_resource),
	      col_indices_(col_indices.begin(),
	                   col_indices.end(),
	                   &memory_resource),
	      num_rows_(num_rows),
	      num_cols_(num_cols),
	      storage_order_(storage_order) {
		gsl_Expects(values.size() == row_indices.size());
		gsl_Expects(values.size() == col_indices.size());
	}

	template <class DataRange,
	          class RowIndRange,
	          class ColIndRange,
	          class MemoryResource>
	COO(DataRange&& values,
	    RowIndRange&& row_indices,
	    ColIndRange&& col_indices,
	    size_t num_rows,
	    size_t num_cols,
	    MemoryResource& memory_resource)
	    : COO(std::forward<DataRange>(values),
	          std::forward<RowIndRange>(row_indices),
	          std::forward<ColIndRange>(col_indices),
	          num_rows,
	          num_cols,
	          storage_order_t::none,
	          memory_resource) {
	}

	template <class DataRange, class ColIndRange, class RowIndRange>
	COO(DataRange&& values,
	    RowIndRange&& row_indices,
	    ColIndRange&& col_indices,
	    size_t num_rows,
	    size_t num_cols)
	    : COO(std::forward<DataRange>(values),
	          std::forward<RowIndRange>(row_indices),
	          std::forward<ColIndRange>(col_indices),
	          num_rows,
	          num_cols,
	          pmr::default_resource) {
	}

	// The copy constructor is declared explicitly to ensure
	// managed memory is used per default.
	COO(const COO& other)
	    : COO(other.values(),
	          other.row_indices(),
	          other.col_indices(),
	          other.num_rows(),
	          other.num_cols(),
	          other.get_storage_order(),
	          pmr::default_resource) {
	}

	COO(COO&& other) = default;

	void change_storage_order(storage_order_t new_storage_order) {
		IndexType* primary_keys_first;
		IndexType* secondary_keys_first;

		if (new_storage_order == storage_order_) {
			return;
		}

		switch (new_storage_order) {
			case storage_order_t::row_major:
				primary_keys_first = row_indices_.data();
				secondary_keys_first = col_indices_.data();
				break;
			case storage_order_t::col_major:
				primary_keys_first = col_indices_.data();
				secondary_keys_first = row_indices_.data();
				break;
			case storage_order_t::none:
				return;
		}

		// Thrust's relational operators are overloaded for thrust::pair.
		// This ensures that we sort with respect to the second key if the first key is equal.
		auto key_it = thrust::make_zip_iterator(
		    thrust::make_tuple(primary_keys_first, secondary_keys_first));
		thrust::sort_by_key(
		    thrust::cuda::par, key_it, key_it + values_.size(), values_.data());
		storage_order_ = new_storage_order;
	}

	// If the storage order is changed externally
	void set_storage_order(storage_order_t new_storage_order) {
		storage_order_ = new_storage_order;
	}

	void transpose() {
		std::swap(col_indices_, row_indices_);
		std::swap(num_rows_, num_cols_);
		switch (storage_order_) {
			case storage_order_t::none:
				break;
			case storage_order_t::row_major:
				storage_order_ = storage_order_t::col_major;
				break;
			case storage_order_t::col_major:
				storage_order_ = storage_order_t::row_major;
				break;
		}
	}

	gsl_lite::span<DataType> values() {
		return gsl_lite::make_span(values_);
	}

	gsl_lite::span<const DataType> values() const {
		return gsl_lite::make_span(values_);
	}

	gsl_lite::span<IndexType> row_indices() {
		return gsl_lite::make_span(row_indices_);
	}

	gsl_lite::span<const IndexType> row_indices() const {
		return gsl_lite::make_span(row_indices_);
	}

	gsl_lite::span<IndexType> col_indices() {
		return gsl_lite::make_span(col_indices_);
	}

	gsl_lite::span<const IndexType> col_indices() const {
		return gsl_lite::make_span(col_indices_);
	}

	size_t num_rows() const {
		return num_rows_;
	}

	size_t num_cols() const {
		return num_cols_;
	}

	storage_order_t get_storage_order() const {
		return storage_order_;
	}

	//! Return row_ptrs or col_ptrs depending on the current storage order
	thrustshift::managed_vector<index_type> get_ptrs() const {
		gsl_Expects(storage_order_ != storage_order_t::none);
		auto indices = storage_order_ == storage_order_t::row_major
		                   ? row_indices()
		                   : col_indices();
		auto size = storage_order_ == storage_order_t::row_major ? num_rows()
		                                                         : num_cols();

		if (indices.size() > 0) {
			thrustshift::managed_vector<index_type> ptrs(size + 1, -1);
			for (int nns_id = gsl_lite::narrow<int>(indices.size()) - 1;
			     nns_id >= 0;
			     --nns_id) {
				ptrs[indices[nns_id]] = nns_id;
			}
			ptrs[0] = 0;
			ptrs.back() = indices.size();
			index_type k = indices.size();
			for (int id = gsl_lite::narrow<int>(ptrs.size()) - 1; id > 0;
			     --id) {
				if (ptrs[id] == -1) {
					ptrs[id] = k;
				}
				else {
					k = ptrs[id];
				}
			}
			return ptrs;
		}
		else {
			return {0};
		}
	}

   private:
	std::pmr::vector<DataType> values_;
	std::pmr::vector<IndexType> row_indices_;
	std::pmr::vector<IndexType> col_indices_;
	size_t num_rows_;
	size_t num_cols_;
	storage_order_t storage_order_;
};

//! Storage order does not affect comparison operator. Therefore this operator is defined explicitly.
template <typename DataType, typename IndexType>
bool operator==(const COO<DataType, IndexType>& a,
                const COO<DataType, IndexType>& b) {
	return equal(a.values(), b.values()) &&
	       equal(a.row_indices(), b.row_indices()) &&
	       equal(a.col_indices(), b.col_indices()) &&
	       a.num_rows() == b.num_rows() && a.num_cols() == b.num_cols();
}

template <typename DataType, typename IndexType>
class COO_view {

   public:
	using value_type = DataType;
	using index_type = IndexType;

	template <typename OtherDataType, typename OtherIndexType>
	COO_view(COO<OtherDataType, OtherIndexType>& owner)
	    : values_(owner.values()),
	      row_indices_(owner.row_indices()),
	      col_indices_(owner.col_indices()),
	      num_rows_(owner.num_rows()),
	      num_cols_(owner.num_cols()) {
	}

	template <typename OtherDataType, typename OtherIndexType>
	COO_view(const COO<OtherDataType, OtherIndexType>& owner)
	    : values_(owner.values()),
	      row_indices_(owner.row_indices()),
	      col_indices_(owner.col_indices()),
	      num_rows_(owner.num_rows()),
	      num_cols_(owner.num_cols()) {
	}

	template <class ValueRange, class ColIndRange, class RowIndRange>
	COO_view(ValueRange&& values,
	         RowIndRange&& row_indices,
	         ColIndRange&& col_indices,
	         size_t num_rows,
	         size_t num_cols)
	    : values_(values),
	      row_indices_(row_indices),
	      col_indices_(col_indices),
	      num_rows_(num_rows),
	      num_cols_(num_cols) {
		const auto nnz = values.size();
		gsl_Expects(col_indices.size() == nnz);
		gsl_Expects(row_indices.size() == nnz);
	}

	COO_view(const COO_view& other) = default;

	CUDA_FHD gsl_lite::span<DataType> values() {
		return values_;
	}

	CUDA_FHD gsl_lite::span<const DataType> values() const {
		return values_;
	}

	CUDA_FHD gsl_lite::span<IndexType> row_indices() {
		return row_indices_;
	}

	CUDA_FHD gsl_lite::span<const IndexType> row_indices() const {
		return row_indices_;
	}

	CUDA_FHD gsl_lite::span<IndexType> col_indices() {
		return col_indices_;
	}

	CUDA_FHD gsl_lite::span<const IndexType> col_indices() const {
		return col_indices_;
	}

	CUDA_FHD size_t num_rows() const {
		return num_rows_;
	}

	CUDA_FHD size_t num_cols() const {
		return num_cols_;
	}

   private:
	gsl_lite::span<DataType> values_;
	gsl_lite::span<IndexType> row_indices_;
	gsl_lite::span<IndexType> col_indices_;
	size_t num_rows_;
	size_t num_cols_;
};

namespace kernel {

template <typename T, typename T0, typename I0, typename T1, typename I1>
__global__ void diagmm(gsl_lite::span<const T> diag,
                       COO_view<const T0, const I0> mtx,
                       COO_view<T1, I1> result_mtx) {

	const auto nnz = mtx.values().size();
	const auto gtid = threadIdx.x + blockIdx.x * blockDim.x;
	if (gtid < nnz) {
		result_mtx.values()[gtid] =
		    diag[mtx.row_indices()[gtid]] * mtx.values()[gtid];
	}
}

} // namespace kernel

namespace async {

//! Calculate the product `result_coo_mtx = (diag_mtx * coo_mtx)`
//! result_coo_mtx can be equal to coo_mtx. Then the multiplication is in place.
template <class Range, class COO_C0, class COO_C1>
void diagmm(cuda::stream_t& stream,
            Range&& diag_mtx,
            COO_C0&& coo_mtx,
            COO_C1&& result_coo_mtx) {

	gsl_Expects(coo_mtx.values().size() == result_coo_mtx.values().size());
	gsl_Expects(coo_mtx.num_cols() == result_coo_mtx.num_cols());
	gsl_Expects(coo_mtx.num_rows() == result_coo_mtx.num_rows());
	gsl_Expects(diag_mtx.size() == coo_mtx.num_rows());
	// Only square matrices are allowed
	gsl_Expects(coo_mtx.num_rows() == coo_mtx.num_cols());

	using RangeT = typename std::remove_reference<Range>::type::value_type;
	using COO_T0 = typename std::remove_reference<COO_C0>::type::value_type;
	using COO_I0 = typename std::remove_reference<COO_C0>::type::index_type;
	using COO_T1 = typename std::remove_reference<COO_C1>::type::value_type;
	using COO_I1 = typename std::remove_reference<COO_C1>::type::index_type;

	const auto nnz = coo_mtx.values().size();
	constexpr cuda::grid::block_dimension_t block_dim = 128;
	const cuda::grid::dimension_t grid_dim = ceil_divide(nnz, block_dim);
	cuda::enqueue_launch(kernel::diagmm<RangeT, COO_T0, COO_I0, COO_T1, COO_I1>,
	                     stream,
	                     cuda::make_launch_config(grid_dim, block_dim),
	                     diag_mtx,
	                     coo_mtx,
	                     result_coo_mtx);
}

} // namespace async

namespace kernel {

template <typename T0, typename I, typename T1>
__global__ void get_diagonal(COO_view<const T0, const I> mtx,
                             gsl_lite::span<T1> diag) {

	const auto gtid = threadIdx.x + blockIdx.x * blockDim.x;
	const auto values = mtx.values();
	const auto nnz = values.size();
	if (gtid < nnz) {
		const auto row_id = mtx.row_indices()[gtid];
		const auto col_id = mtx.col_indices()[gtid];
		if (row_id == col_id) {
			diag[row_id] = values[gtid];
		}
	}
}

} // namespace kernel

namespace async {

template <typename T, class COO>
void get_diagonal(cuda::stream_t& stream, COO&& mtx, gsl_lite::span<T> diag) {

	gsl_Expects(diag.size() == std::min(mtx.num_rows(), mtx.num_cols()));
	using I = typename std::remove_reference<COO>::type::index_type;
	using T0 = typename std::remove_reference<COO>::type::value_type;
	const auto nnz = mtx.values().size();
	constexpr cuda::grid::block_dimension_t block_dim = 128;
	const cuda::grid::dimension_t grid_dim =
	    ceil_divide(gsl_lite::narrow<cuda::grid::dimension_t>(nnz),
	                gsl_lite::narrow<cuda::grid::dimension_t>(block_dim));
	fill(stream, diag, 0);
	if (nnz != 0) {
		cuda::enqueue_launch(kernel::get_diagonal<T0, I, T>,
		                     stream,
		                     cuda::make_launch_config(grid_dim, block_dim),
		                     mtx,
		                     diag);
	}
}

} // namespace async

/* \brief Transform two sparse COO matrices coefficient-wise `op(op_a(a), op_b(b))`.
 * \param a first sparse COO matrix.
 * \param b second sparse COO matrix.
 * \param op Binary operator how coefficients at the same positions are combined.
 * \param op_a Unary operator which is applied to all non-zero coefficients of `a`.
 * \param op_b Unary operator which is applied to all non-zero coefficients of `b`.
 * \return COO matrix allocated with `memory_resource`
 * \note It might be misleading that the binary operator is **not** applied to coefficients, which
 *   only appear in one of the two matrices. E.g. `op=minus`, `op_a=identity`, `op_b=identity` and `a` is a zero matrix the result
 *   is `result != a - b = -b`. Also note that it cannot be ensured that the left operand of the binary operator
 *   is always an element of `a`. Therefore the unary operators are required.
 */
template <typename DataType,
          typename IndexType,
          class BinaryOperator,
          class UnaryOperatorA,
          class UnaryOperatorB,
          class MemoryResource>
thrustshift::COO<DataType, IndexType> transform(
    thrustshift::COO_view<const DataType, const IndexType> a,
    thrustshift::COO_view<const DataType, const IndexType> b,
    BinaryOperator&& op,
    UnaryOperatorA&& op_a,
    UnaryOperatorB&& op_b,
    MemoryResource& memory_resource) {

	const auto nnz_a = a.values().size();
	const auto nnz_b = b.values().size();
	const auto num_rows = a.num_rows();
	const auto num_cols = b.num_cols();
	gsl_Expects(b.num_rows() == num_rows);
	gsl_Expects(b.num_cols() == num_cols);

	auto [tmp0, values] =
	    make_not_a_vector_and_span<DataType>(nnz_a + nnz_b, memory_resource);
	auto [tmp1, row_indices] =
	    make_not_a_vector_and_span<IndexType>(nnz_a + nnz_b, memory_resource);
	auto [tmp2, col_indices] =
	    make_not_a_vector_and_span<IndexType>(nnz_a + nnz_b, memory_resource);

	auto device = cuda::device::current::get();
	auto stream = device.default_stream();

	async::transform(stream, a.values(), values.first(nnz_a), op_a);
	async::copy(stream, a.row_indices(), row_indices.first(nnz_a));
	async::copy(stream, a.col_indices(), col_indices.first(nnz_a));

	async::transform(stream, b.values(), values.subspan(nnz_a), op_b);
	async::copy(stream, b.row_indices(), row_indices.subspan(nnz_a));
	async::copy(stream, b.col_indices(), col_indices.subspan(nnz_a));

	using KeyT = thrust::tuple<IndexType, IndexType>;
	auto keys_begin = thrust::make_zip_iterator(
	    thrust::make_tuple(row_indices.begin(), col_indices.begin()));
	auto keys_end = keys_begin + nnz_a + nnz_b;

	std::pmr::polymorphic_allocator<KeyT> alloc(&memory_resource);
	thrust::sort_by_key(
	    thrust::cuda::par(alloc), keys_begin, keys_end, values.begin());

	const std::size_t nnz_result =
	    thrust::inner_product(thrust::cuda::par(alloc),
	                          keys_begin,
	                          keys_end - 1,
	                          keys_begin + 1,
	                          std::size_t(0),
	                          thrust::plus<std::size_t>(),
	                          thrust::not_equal_to<KeyT>()) +
	    std::size_t(1);

	// ```cpp
	//   auto memory_resource = ...;
	//   auto res = transform(..., memory_resource);
	//
	// ```
	// should work fine because the dtor of `res` is called before the dtor of `memory_resource`
	COO<DataType, IndexType> result(
	    nnz_result, num_rows, num_cols, memory_resource);
	auto keys_result_begin = thrust::make_zip_iterator(thrust::make_tuple(
	    result.row_indices().begin(), result.col_indices().begin()));
	thrust::reduce_by_key(thrust::cuda::par(alloc),
	                      keys_begin,
	                      keys_end,
	                      values.begin(),
	                      keys_result_begin,
	                      result.values().begin(),
	                      thrust::equal_to<KeyT>(),
	                      op);
	result.set_storage_order(storage_order_t::row_major);
	return result;
}

template <typename DataType, typename IndexType, class MemoryResource>
COO<DataType, IndexType> symmetrize_abs(
    COO_view<const DataType, const IndexType> mtx,
    MemoryResource& memory_resource) {

	if (mtx.values().empty()) {
		return COO<DataType, IndexType>(
		    0, mtx.num_rows(), mtx.num_cols(), memory_resource);
	}

	COO_view<const DataType, const IndexType> mtx_trans(mtx.values(),
	                                                    mtx.col_indices(),
	                                                    mtx.row_indices(),
	                                                    mtx.num_rows(),
	                                                    mtx.num_cols());

	auto abs = [] __device__(DataType x) { return std::abs(x); };

	return transform(
	    mtx,
	    mtx_trans,
	    [] __device__(DataType x, DataType y) { return x + y; },
	    abs,
	    abs,
	    memory_resource);
}

template <typename DataType, typename IndexType, class MemoryResource>
COO<DataType, IndexType> make_pattern_symmetric(
    COO_view<const DataType, const IndexType> mtx,
    MemoryResource& memory_resource) {

	if (mtx.values().empty()) {
		return COO<DataType, IndexType>(
		    0, mtx.num_rows(), mtx.num_cols(), memory_resource);
	}

	COO_view<const DataType, const IndexType> mtx_trans(mtx.values(),
	                                                    mtx.col_indices(),
	                                                    mtx.row_indices(),
	                                                    mtx.num_rows(),
	                                                    mtx.num_cols());

	auto identity = [] __device__(DataType x) { return x; };
	auto make_zero = [] __device__(DataType x) { return DataType(0); };

	return transform(
	    mtx,
	    mtx_trans,
	    [] __device__(DataType x, DataType y) { return x + y; },
	    identity,
	    make_zero,
	    memory_resource);
}

} // namespace thrustshift


#pragma once

#include <petsc.h>
#include <memory>

#include "partition/rank_subset.h"

namespace TimeSteppingScheme
{

namespace NestedMatVecUtility
{

//! from a Petsc Vec with nested type (nestedVec) create a new Petsc Vec (singleVec) that contains all values at once. If the singleVec already exists, do not create again, only copy the values.
void createVecFromNestedVec(Vec nestedVec, Vec &singleVec, std::shared_ptr<Partition::RankSubset> rankSubset);

//! copy the values from a singleVec back to the nested Petsc Vec (nestedVec)
void fillNestedVec(Vec singleVec, Vec nestedVec);

//! from a Petsc Mat with nested type (nestedMat) create a new Petsc Mat (singleMat) that contains all values at once. If the singleMat already exists, do not create again, only copy the values.
void createMatFromNestedMat(Mat nestedMat, Mat &singleMat, std::shared_ptr<Partition::RankSubset> rankSubset);

}
}  // namespace


/*
 * \file base.h
 * \brief Header file with base requirements for codes
 *
 *
 */

#ifndef BASE_H
#define BASE_H

#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <math.h>
#include <getopt.h>
#include <unistd.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <time.h>

#include <bzlib.h>
#include <hdf5.h>
#include <gsl/gsl_math.h>
#include <gsl/gsl_eigen.h>

#define NAME_LENGTH 256
#define LONG_LENGTH 3000

#endif    /*  BASE_H  */


/* rb.c
 *
 * Copyright (C) 1998-2002, 2004 Free Software Foundation, Inc.
 * Copyright (C) 2018 Patrick Alken
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 3 of the License, or (at
 * your option) any later version.
 *
 * This program is distributed in the hope that it will be useful, but
 * WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
 */

/* This code is originally from GNU libavl, with some modifications */

#include <config.h>
#include <stdlib.h>
#include <stddef.h>
#include <string.h>

#include <gsl/gsl_math.h>
#include <gsl/gsl_bst.h>
#include <gsl/gsl_errno.h>

typedef struct gsl_bst_rb_node rb_node;
typedef gsl_bst_rb_table rb_table;
typedef gsl_bst_rb_traverser rb_traverser;

enum rb_color
{
  RB_BLACK, /* black */
  RB_RED    /* red */
};

#ifndef RB_MAX_HEIGHT
#define RB_MAX_HEIGHT GSL_BST_RB_MAX_HEIGHT
#endif

/* tree functions */
static int rb_init(const gsl_bst_allocator * allocator, gsl_bst_cmp_function * compare,
                   void * params, void * vtable);
static size_t rb_nodes (const void * vtable);
static int rb_empty (void * vtable);
static void ** rb_probe (void * item, rb_table * table);
static void * rb_insert (void * item, void * vtable);
static void * rb_find (const void * item, const void * vtable);
static void * rb_remove (const void * item, void * vtable);

/* traverser functions */
static int rb_t_init (void * vtrav, const void * vtable);
static void * rb_t_first (void * vtrav, const void * vtable);
static void * rb_t_last (void * vtrav, const void * vtable);
static void * rb_t_find (const void * item, void * vtrav, const void * vtable);
static void * rb_t_insert (void * item, void * vtrav, void * vtable);
static void * rb_t_copy (void * vtrav, const void * vsrc);
static void * rb_t_next (void * vtrav);
static void * rb_t_prev (void * vtrav);
static void * rb_t_cur (const void * vtrav);
static void * rb_t_replace (void * vtrav, void * new_item);
static void rb_trav_refresh (rb_traverser *trav);

static int
rb_init(const gsl_bst_allocator * allocator, gsl_bst_cmp_function * compare,
        void * params, void * vtable)
{
  rb_table * table = (rb_table *) vtable;

  table->rb_alloc = allocator;
  table->rb_compare = compare;
  table->rb_param = params;

  table->rb_root = NULL;
  table->rb_count = 0;
  table->rb_generation = 0;

  return GSL_SUCCESS;
}

static size_t
rb_nodes (const void * vtable)
{
  const rb_table * table = (const rb_table *) vtable;
  return table->rb_count;
}

/* empty tree (delete all nodes) but do not free the tree itself */
static int
rb_empty (void * vtable)
{
  rb_table * table = (rb_table *) vtable;
  rb_node *p, *q;

  for (p = table->rb_root; p != NULL; p = q)
    {
      if (p->rb_link[0] == NULL)
        {
          q = p->rb_link[1];
          table->rb_alloc->free (p, table->rb_param);
        }
      else
        {
          q = p->rb_link[0];
          p->rb_link[0] = q->rb_link[1];
          q->rb_link[1] = p;
        }
    }

  table->rb_root = NULL;
  table->rb_count = 0;
  table->rb_generation = 0;

  return GSL_SUCCESS;
}

/* Inserts |item| into |table| and returns a pointer to |item|'s address.
   If a duplicate item is found in the tree,
   returns a pointer to the duplicate without inserting |item|.
   Returns |NULL| in case of memory allocation failure. */

static void **
rb_probe (void * item, rb_table * table)
{
  rb_node *pa[RB_MAX_HEIGHT];      /* nodes on stack */
  unsigned char da[RB_MAX_HEIGHT]; /* directions moved from stack nodes */
  int k;                           /* stack height */

  rb_node *p; /* traverses tree looking for insertion point */
  rb_node *n; /* newly inserted node */

  pa[0] = (rb_node *) &table->rb_root;
  da[0] = 0;
  k = 1;
  for (p = table->rb_root; p != NULL; p = p->rb_link[da[k - 1]])
    {
      int cmp = table->rb_compare (item, p->rb_data, table->rb_param);
      if (cmp == 0)
        return &p->rb_data;

      pa[k] = p;
      da[k++] = cmp > 0;
    }

  n = pa[k - 1]->rb_link[da[k - 1]] =
    table->rb_alloc->alloc (sizeof *n, table->rb_param);
  if (n == NULL)
    return NULL;

  n->rb_data = item;
  n->rb_link[0] = n->rb_link[1] = NULL;
  n->rb_color = RB_RED;
  table->rb_count++;
  table->rb_generation++;

  while (k >= 3 && pa[k - 1]->rb_color == RB_RED)
    {
      if (da[k - 2] == 0)
        {
          rb_node *y = pa[k - 2]->rb_link[1];
          if (y != NULL && y->rb_color == RB_RED)
            {
              pa[k - 1]->rb_color = y->rb_color = RB_BLACK;
              pa[k - 2]->rb_color = RB_RED;
              k -= 2;
            }
          else
            {
              rb_node *x;

              if (da[k - 1] == 0)
                y = pa[k - 1];
              else
                {
                  x = pa[k - 1];
                  y = x->rb_link[1];
                  x->rb_link[1] = y->rb_link[0];
                  y->rb_link[0] = x;
                  pa[k - 2]->rb_link[0] = y;
                }

              x = pa[k - 2];
              x->rb_color = RB_RED;
              y->rb_color = RB_BLACK;

              x->rb_link[0] = y->rb_link[1];
              y->rb_link[1] = x;
              pa[k - 3]->rb_link[da[k - 3]] = y;
              break;
            }
        }
      else
        {
          rb_node *y = pa[k - 2]->rb_link[0];
          if (y != NULL && y->rb_color == RB_RED)
            {
              pa[k - 1]->rb_color = y->rb_color = RB_BLACK;
              pa[k - 2]->rb_color = RB_RED;
              k -= 2;
            }
          else
            {
              rb_node *x;

              if (da[k - 1] == 1)
                y = pa[k - 1];
              else
                {
                  x = pa[k - 1];
                  y = x->rb_link[0];
                  x->rb_link[0] = y->rb_link[1];
                  y->rb_link[1] = x;
                  pa[k - 2]->rb_link[1] = y;
                }

              x = pa[k - 2];
              x->rb_color = RB_RED;
              y->rb_color = RB_BLACK;

              x->rb_link[1] = y->rb_link[0];
              y->rb_link[0] = x;
              pa[k - 3]->rb_link[da[k - 3]] = y;
              break;
            }
        }
    }

  table->rb_root->rb_color = RB_BLACK;

  return &n->rb_data;
}

/* Inserts |item| into |table|.
   Returns |NULL| if |item| was successfully inserted
   or if a memory allocation error occurred.
   Otherwise, returns the duplicate item. */

static void *
rb_insert (void * item, void * vtable)
{
  void **p = rb_probe (item, vtable);
  return p == NULL || *p == item ? NULL : *p;
}

/* Search |table| for an item matching |item|, and return it if found.
   Otherwise return |NULL|. */
static void *
rb_find (const void * item, const void * vtable)
{
  const rb_table * table = (const rb_table *) vtable;
  const rb_node *p;

  for (p = table->rb_root; p != NULL; )
    {
      int cmp = table->rb_compare (item, p->rb_data, table->rb_param);

      if (cmp < 0)
        p = p->rb_link[0];
      else if (cmp > 0)
        p = p->rb_link[1];
      else /* |cmp == 0| */
        return p->rb_data;
    }

  return NULL;
}

#if 0 /*XXX*/

/* Inserts |item| into |table|, replacing any duplicate item.
   Returns |NULL| if |item| was inserted without replacing a duplicate,
   or if a memory allocation error occurred.
   Otherwise, returns the item that was replaced. */
void *
rb_replace (struct rb_table *table, void *item)
{
  void **p = rb_probe (table, item);
  if (p == NULL || *p == item)
    return NULL;
  else
    {
      void *r = *p;
      *p = item;
      return r;
    }
}

#endif

/* Deletes from |table| and returns an item matching |item|.
   Returns a null pointer if no matching item found. */

static void *
rb_remove (const void * item, void * vtable)
{
  rb_table * table = (rb_table *) vtable;
  rb_node *pa[RB_MAX_HEIGHT];      /* nodes on stack */
  unsigned char da[RB_MAX_HEIGHT]; /* directions moved from stack nodes */
  int k;                           /* stack height */

  rb_node *p;    /* the node to delete, or a node part way to it */
  int cmp;       /* result of comparison between |item| and |p| */

  k = 0;
  p = (rb_node *) &table->rb_root;
  for (cmp = -1; cmp != 0;
       cmp = table->rb_compare (item, p->rb_data, table->rb_param))
    {
      int dir = cmp > 0;

      pa[k] = p;
      da[k++] = dir;

      p = p->rb_link[dir];
      if (p == NULL)
        return NULL;
    }
  item = p->rb_data;

  if (p->rb_link[1] == NULL)
    pa[k - 1]->rb_link[da[k - 1]] = p->rb_link[0];
  else
    {
      enum rb_color t;
      rb_node *r = p->rb_link[1];

      if (r->rb_link[0] == NULL)
        {
          r->rb_link[0] = p->rb_link[0];
          t = r->rb_color;
          r->rb_color = p->rb_color;
          p->rb_color = t;
          pa[k - 1]->rb_link[da[k - 1]] = r;
          da[k] = 1;
          pa[k++] = r;
        }
      else
        {
          rb_node *s;
          int j = k++;

          for (;;)
            {
              da[k] = 0;
              pa[k++] = r;
              s = r->rb_link[0];
              if (s->rb_link[0] == NULL)
                break;

              r = s;
            }

          da[j] = 1;
          pa[j] = s;
          pa[j - 1]->rb_link[da[j - 1]] = s;

          s->rb_link[0] = p->rb_link[0];
          r->rb_link[0] = s->rb_link[1];
          s->rb_link[1] = p->rb_link[1];

          t = s->rb_color;
          s->rb_color = p->rb_color;
          p->rb_color = t;
        }
    }

  if (p->rb_color == RB_BLACK)
    {
      for (;;)
        {
          rb_node *x = pa[k - 1]->rb_link[da[k - 1]];
          if (x != NULL && x->rb_color == RB_RED)
            {
              x->rb_color = RB_BLACK;
              break;
            }
          if (k < 2)
            break;

          if (da[k - 1] == 0)
            {
              rb_node *w = pa[k - 1]->rb_link[1];

              if (w->rb_color == RB_RED)
                {
                  w->rb_color = RB_BLACK;
                  pa[k - 1]->rb_color = RB_RED;

                  pa[k - 1]->rb_link[1] = w->rb_link[0];
                  w->rb_link[0] = pa[k - 1];
                  pa[k - 2]->rb_link[da[k - 2]] = w;

                  pa[k] = pa[k - 1];
                  da[k] = 0;
                  pa[k - 1] = w;
                  k++;

                  w = pa[k - 1]->rb_link[1];
                }

              if ((w->rb_link[0] == NULL
                   || w->rb_link[0]->rb_color == RB_BLACK)
                  && (w->rb_link[1] == NULL
                      || w->rb_link[1]->rb_color == RB_BLACK))
                w->rb_color = RB_RED;
              else
                {
                  if (w->rb_link[1] == NULL
                      || w->rb_link[1]->rb_color == RB_BLACK)
                    {
                      rb_node *y = w->rb_link[0];
                      y->rb_color = RB_BLACK;
                      w->rb_color = RB_RED;
                      w->rb_link[0] = y->rb_link[1];
                      y->rb_link[1] = w;
                      w = pa[k - 1]->rb_link[1] = y;
                    }

                  w->rb_color = pa[k - 1]->rb_color;
                  pa[k - 1]->rb_color = RB_BLACK;
                  w->rb_link[1]->rb_color = RB_BLACK;

                  pa[k - 1]->rb_link[1] = w->rb_link[0];
                  w->rb_link[0] = pa[k - 1];
                  pa[k - 2]->rb_link[da[k - 2]] = w;
                  break;
                }
            }
          else
            {
              rb_node *w = pa[k - 1]->rb_link[0];

              if (w->rb_color == RB_RED)
                {
                  w->rb_color = RB_BLACK;
                  pa[k - 1]->rb_color = RB_RED;

                  pa[k - 1]->rb_link[0] = w->rb_link[1];
                  w->rb_link[1] = pa[k - 1];
                  pa[k - 2]->rb_link[da[k - 2]] = w;

                  pa[k] = pa[k - 1];
                  da[k] = 1;
                  pa[k - 1] = w;
                  k++;

                  w = pa[k - 1]->rb_link[0];
                }

              if ((w->rb_link[0] == NULL
                   || w->rb_link[0]->rb_color == RB_BLACK)
                  && (w->rb_link[1] == NULL
                      || w->rb_link[1]->rb_color == RB_BLACK))
                w->rb_color = RB_RED;
              else
                {
                  if (w->rb_link[0] == NULL
                      || w->rb_link[0]->rb_color == RB_BLACK)
                    {
                      rb_node *y = w->rb_link[1];
                      y->rb_color = RB_BLACK;
                      w->rb_color = RB_RED;
                      w->rb_link[1] = y->rb_link[0];
                      y->rb_link[0] = w;
                      w = pa[k - 1]->rb_link[0] = y;
                    }

                  w->rb_color = pa[k - 1]->rb_color;
                  pa[k - 1]->rb_color = RB_BLACK;
                  w->rb_link[0]->rb_color = RB_BLACK;

                  pa[k - 1]->rb_link[0] = w->rb_link[1];
                  w->rb_link[1] = pa[k - 1];
                  pa[k - 2]->rb_link[da[k - 2]] = w;
                  break;
                }
            }

          k--;
        }

    }

  table->rb_alloc->free (p, table->rb_param);
  table->rb_count--;
  table->rb_generation++;

  return (void *) item;
}

/* Initializes |trav| for use with |tree| and selects the null node. */
static int
rb_t_init (void * vtrav, const void * vtable)
{
  rb_traverser * trav = (rb_traverser *) vtrav;
  const rb_table * table = (const rb_table *) vtable;

  trav->rb_table = table;
  trav->rb_node = NULL;
  trav->rb_height = 0;
  trav->rb_generation = table->rb_generation;

  return GSL_SUCCESS;
}

/* Initializes |trav| for |table|
   and selects and returns a pointer to its least-valued item.
   Returns |NULL| if |table| contains no nodes. */
static void *
rb_t_first (void * vtrav, const void * vtable)
{
  const rb_table * table = (const rb_table *) vtable;
  rb_traverser * trav = (rb_traverser *) vtrav;
  rb_node *x;

  trav->rb_table = table;
  trav->rb_height = 0;
  trav->rb_generation = table->rb_generation;

  x = table->rb_root;
  if (x != NULL)
    {
      while (x->rb_link[0] != NULL)
        {
          if (trav->rb_height >= RB_MAX_HEIGHT)
            {
              GSL_ERROR_NULL ("traverser height exceeds maximum", GSL_ETABLE);
            }

          trav->rb_stack[trav->rb_height++] = x;
          x = x->rb_link[0];
        }
    }

  trav->rb_node = x;

  return x != NULL ? x->rb_data : NULL;
}

/* Initializes |trav| for |table|
   and selects and returns a pointer to its greatest-valued item.
   Returns |NULL| if |table| contains no nodes. */
static void *
rb_t_last (void * vtrav, const void * vtable)
{
  const rb_table * table = (const rb_table *) vtable;
  rb_traverser * trav = (rb_traverser *) vtrav;
  rb_node *x;

  trav->rb_table = table;
  trav->rb_height = 0;
  trav->rb_generation = table->rb_generation;

  x = table->rb_root;
  if (x != NULL)
    {
      while (x->rb_link[1] != NULL)
        {
          if (trav->rb_height >= RB_MAX_HEIGHT)
            {
              GSL_ERROR_NULL ("traverser height exceeds maximum", GSL_ETABLE);
            }

          trav->rb_stack[trav->rb_height++] = x;
          x = x->rb_link[1];
        }
    }

  trav->rb_node = x;

  return x != NULL ? x->rb_data : NULL;
}

/* Searches for |item| in |table|.
   If found, initializes |trav| to the item found and returns the item
   as well.
   If there is no matching item, initializes |trav| to the null item
   and returns |NULL|. */
static void *
rb_t_find (const void * item, void * vtrav, const void * vtable)
{
  const rb_table * table = (const rb_table *) vtable;
  rb_traverser * trav = (rb_traverser *) vtrav;
  rb_node *p, *q;

  trav->rb_table = table;
  trav->rb_height = 0;
  trav->rb_generation = table->rb_generation;
  for (p = table->rb_root; p != NULL; p = q)
    {
      int cmp = table->rb_compare (item, p->rb_data, table->rb_param);

      if (cmp < 0)
        q = p->rb_link[0];
      else if (cmp > 0)
        q = p->rb_link[1];
      else /* |cmp == 0| */
        {
          trav->rb_node = p;
          return p->rb_data;
        }

      if (trav->rb_height >= RB_MAX_HEIGHT)
        {
          GSL_ERROR_NULL ("traverser height exceeds maximum", GSL_ETABLE);
        }

      trav->rb_stack[trav->rb_height++] = p;
    }

  trav->rb_height = 0;
  trav->rb_node = NULL;

  return NULL;
}

/* Attempts to insert |item| into |table|.
   If |item| is inserted successfully, it is returned and |trav| is
   initialized to its location.
   If a duplicate is found, it is returned and |trav| is initialized to
   its location.  No replacement of the item occurs.
   If a memory allocation failure occurs, |NULL| is returned and |trav|
   is initialized to the null item. */
static void *
rb_t_insert (void * item, void * vtrav, void * vtable)
{
  rb_table * table = (rb_table *) vtable;
  rb_traverser * trav = (rb_traverser *) vtrav;
  void **p;

  p = rb_probe (item, table);
  if (p != NULL)
    {
      trav->rb_table = table;
      trav->rb_node = ((rb_node *) ((char *) p - offsetof (rb_node, rb_data)));
      trav->rb_generation = table->rb_generation - 1;
      return *p;
    }
  else
    {
      rb_t_init (vtrav, vtable);
      return NULL;
    }
}

/* Initializes |trav| to have the same current node as |src|. */
static void *
rb_t_copy (void * vtrav, const void * vsrc)
{
  const rb_traverser * src = (const rb_traverser *) vsrc;
  rb_traverser * trav = (rb_traverser *) vtrav;

  if (trav != src)
    {
      trav->rb_table = src->rb_table;
      trav->rb_node = src->rb_node;
      trav->rb_generation = src->rb_generation;
      if (trav->rb_generation == trav->rb_table->rb_generation)
        {
          trav->rb_height = src->rb_height;
          memcpy (trav->rb_stack, (const void *) src->rb_stack,
                  sizeof *trav->rb_stack * trav->rb_height);
        }
    }

  return trav->rb_node != NULL ? trav->rb_node->rb_data : NULL;
}

/* Returns the next data item in inorder
   within the tree being traversed with |trav|,
   or if there are no more data items returns |NULL|. */
static void *
rb_t_next (void * vtrav)
{
  rb_traverser * trav = (rb_traverser *) vtrav;
  rb_node *x;

  if (trav->rb_generation != trav->rb_table->rb_generation)
    rb_trav_refresh (trav);

  x = trav->rb_node;
  if (x == NULL)
    {
      return rb_t_first (vtrav, trav->rb_table);
    }
  else if (x->rb_link[1] != NULL)
    {
      if (trav->rb_height >= RB_MAX_HEIGHT)
        {
          GSL_ERROR_NULL ("traverser height exceeds maximum", GSL_ETABLE);
        }

      trav->rb_stack[trav->rb_height++] = x;
      x = x->rb_link[1];

      while (x->rb_link[0] != NULL)
        {
          if (trav->rb_height >= RB_MAX_HEIGHT)
            {
              GSL_ERROR_NULL ("traverser height exceeds maximum", GSL_ETABLE);
            }

          trav->rb_stack[trav->rb_height++] = x;
          x = x->rb_link[0];
        }
    }
  else
    {
      rb_node *y;

      do
        {
          if (trav->rb_height == 0)
            {
              trav->rb_node = NULL;
              return NULL;
            }

          y = x;
          x = trav->rb_stack[--trav->rb_height];
        }
      while (y == x->rb_link[1]);
    }

  trav->rb_node = x;

  return x->rb_data;
}

/* Returns the previous data item in inorder
   within the tree being traversed with |trav|,
   or if there are no more data items returns |NULL|. */
static void *
rb_t_prev (void * vtrav)
{
  rb_traverser * trav = (rb_traverser *) vtrav;
  rb_node *x;

  if (trav->rb_generation != trav->rb_table->rb_generation)
    rb_trav_refresh (trav);

  x = trav->rb_node;
  if (x == NULL)
    {
      return rb_t_last (vtrav, trav->rb_table);
    }
  else if (x->rb_link[0] != NULL)
    {
      if (trav->rb_height >= RB_MAX_HEIGHT)
        {
          GSL_ERROR_NULL ("traverser height exceeds maximum", GSL_ETABLE);
        }

      trav->rb_stack[trav->rb_height++] = x;
      x = x->rb_link[0];

      while (x->rb_link[1] != NULL)
        {
          if (trav->rb_height >= RB_MAX_HEIGHT)
            {
              GSL_ERROR_NULL ("traverser height exceeds maximum", GSL_ETABLE);
            }

          trav->rb_stack[trav->rb_height++] = x;
          x = x->rb_link[1];
        }
    }
  else
    {
      rb_node *y;

      do
        {
          if (trav->rb_height == 0)
            {
              trav->rb_node = NULL;
              return NULL;
            }

          y = x;
          x = trav->rb_stack[--trav->rb_height];
        }
      while (y == x->rb_link[0]);
    }

  trav->rb_node = x;

  return x->rb_data;
}

/* Returns |trav|'s current item. */
static void *
rb_t_cur (const void * vtrav)
{
  const rb_traverser * trav = (const rb_traverser *) vtrav;
  return trav->rb_node != NULL ? trav->rb_node->rb_data : NULL;
}

/* Replaces the current item in |trav| by |new| and returns the item replaced.
   |trav| must not have the null item selected.
   The new item must not upset the ordering of the tree. */
static void *
rb_t_replace (void * vtrav, void * new_item)
{
  rb_traverser * trav = (rb_traverser *) vtrav;
  void *old;

  old = trav->rb_node->rb_data;
  trav->rb_node->rb_data = new_item;

  return old;
}

#if 0 /*XXX*/
/* Destroys |new| with |rb_destroy (new, destroy)|,
   first setting right links of nodes in |stack| within |new|
   to null pointers to avoid touching uninitialized data. */
static void
copy_error_recovery (rb_node **stack, int height,
                     struct rb_table *new, rb_item_func *destroy)
{
  assert (stack != NULL && height >= 0 && new != NULL);

  for (; height > 2; height -= 2)
    stack[height - 1]->rb_link[1] = NULL;
  rb_destroy (new, destroy);
}

/* Copies |org| to a newly created tree, which is returned.
   If |copy != NULL|, each data item in |org| is first passed to |copy|,
   and the return values are inserted into the tree,
   with |NULL| return values taken as indications of failure.
   On failure, destroys the partially created new tree,
   applying |destroy|, if non-null, to each item in the new tree so far,
   and returns |NULL|.
   If |allocator != NULL|, it is used for allocation in the new tree.
   Otherwise, the same allocator used for |org| is used. */
struct rb_table *
rb_copy (const struct rb_table *org, rb_copy_func *copy,
          rb_item_func *destroy, struct libavl_allocator *allocator)
{
  rb_node *stack[2 * (RB_MAX_HEIGHT + 1)];
  int height = 0;

  struct rb_table *new;
  const rb_node *x;
  rb_node *y;

  assert (org != NULL);
  new = rb_create (org->rb_compare, org->rb_param,
                    allocator != NULL ? allocator : org->rb_alloc);
  if (new == NULL)
    return NULL;
  new->rb_count = org->rb_count;
  if (new->rb_count == 0)
    return new;

  x = (const rb_node *) &org->rb_root;
  y = (rb_node *) &new->rb_root;
  for (;;)
    {
      while (x->rb_link[0] != NULL)
        {
          assert (height < 2 * (RB_MAX_HEIGHT + 1));

          y->rb_link[0] =
            new->rb_alloc->libavl_malloc (new->rb_alloc,
                                           sizeof *y->rb_link[0]);
          if (y->rb_link[0] == NULL)
            {
              if (y != (rb_node *) &new->rb_root)
                {
                  y->rb_data = NULL;
                  y->rb_link[1] = NULL;
                }

              copy_error_recovery (stack, height, new, destroy);
              return NULL;
            }

          stack[height++] = (rb_node *) x;
          stack[height++] = y;
          x = x->rb_link[0];
          y = y->rb_link[0];
        }
      y->rb_link[0] = NULL;

      for (;;)
        {
          y->rb_color = x->rb_color;
          if (copy == NULL)
            y->rb_data = x->rb_data;
          else
            {
              y->rb_data = copy (x->rb_data, org->rb_param);
              if (y->rb_data == NULL)
                {
                  y->rb_link[1] = NULL;
                  copy_error_recovery (stack, height, new, destroy);
                  return NULL;
                }
            }

          if (x->rb_link[1] != NULL)
            {
              y->rb_link[1] =
                new->rb_alloc->libavl_malloc (new->rb_alloc,
                                               sizeof *y->rb_link[1]);
              if (y->rb_link[1] == NULL)
                {
                  copy_error_recovery (stack, height, new, destroy);
                  return NULL;
                }

              x = x->rb_link[1];
              y = y->rb_link[1];
              break;
            }
          else
            y->rb_link[1] = NULL;

          if (height <= 2)
            return new;

          y = stack[--height];
          x = stack[--height];
        }
    }
}

#endif

/* Refreshes the stack of parent pointers in |trav|
   and updates its generation number. */
static void
rb_trav_refresh (rb_traverser *trav)
{
  trav->rb_generation = trav->rb_table->rb_generation;

  if (trav->rb_node != NULL)
    {
      gsl_bst_cmp_function *cmp = trav->rb_table->rb_compare;
      void *param = trav->rb_table->rb_param;
      rb_node *node = trav->rb_node;
      rb_node *i;

      trav->rb_height = 0;
      for (i = trav->rb_table->rb_root; i != node; )
        {
          if (trav->rb_height >= RB_MAX_HEIGHT)
            {
              GSL_ERROR_VOID ("traverser height exceeds maximum", GSL_ETABLE);
            }

          trav->rb_stack[trav->rb_height++] = i;
          i = i->rb_link[cmp (node->rb_data, i->rb_data, param) > 0];
        }
    }
}

static const gsl_bst_type rb_tree_type =
{
  "red-black",
  sizeof(rb_node),
  rb_init,
  rb_nodes,
  rb_insert,
  rb_find,
  rb_remove,
  rb_empty,

  rb_t_init,
  rb_t_first,
  rb_t_last,
  rb_t_find,
  rb_t_insert,
  rb_t_copy,
  rb_t_next,
  rb_t_prev,
  rb_t_cur,
  rb_t_replace
};

const gsl_bst_type * gsl_bst_rb = &rb_tree_type;


#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <math.h>
#ifndef NOMPI
#include <mpi.h>
#endif

#include <fcntl.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <sys/file.h>
#include <unistd.h>
#include <gsl/gsl_rng.h>

#include "allvars.h"
#include "proto.h"

/*! \file restart.c
 *  \brief Code for reading and writing restart files
 */

static FILE *fd;

static void in(int *x, int modus);
static void byten(void *x, size_t n, int modus);

/*! This function reads or writes the restart files.  Each processor writes
 *  its own restart file, with the I/O being done in parallel. To avoid
 *  congestion of the disks you can tell the program to restrict the number of
 *  files that are simultaneously written to NumFilesWrittenInParallel.
 *
 *  If modus>0 the restart()-routine reads, if modus==0 it writes a restart
 *  file.
 */
void restart(int modus)
{
  char buf[200], buf_bak[200], buf_mv[500];
  double save_PartAllocFactor, save_TreeAllocFactor;
  int i, nprocgroup, masterTask, groupTask, old_MaxPart, old_MaxNodes;
  struct global_data_all_processes all_task0;

  sprintf(buf, "%s%s.%d", All.OutputDir, All.RestartFile, ThisTask);
  sprintf(buf_bak, "%s%s.%d.bak", All.OutputDir, All.RestartFile, ThisTask);
  sprintf(buf_mv, "mv %s %s", buf, buf_bak);

  if((NTask < All.NumFilesWrittenInParallel))
    {
      printf
	("Fatal error.\nNumber of processors must be a smaller or equal than `NumFilesWrittenInParallel'.\n");
      endrun(2131);
    }

  nprocgroup = NTask / All.NumFilesWrittenInParallel;

  if((NTask % All.NumFilesWrittenInParallel))
    {
      nprocgroup++;
    }

  masterTask = (ThisTask / nprocgroup) * nprocgroup;

  for(groupTask = 0; groupTask < nprocgroup; groupTask++)
    {
      if(ThisTask == (masterTask + groupTask))	/* ok, it's this processor's turn */
	{
	  if(modus)
	    {
	      if(!(fd = fopen(buf, "r")))
		{
		  printf("Restart file '%s' not found.\n", buf);
		  endrun(7870);
		}
	    }
	  else
	    {
	      system(buf_mv);	/* move old restart files to .bak files */

	      if(!(fd = fopen(buf, "w")))
		{
		  printf("Restart file '%s' cannot be opened.\n", buf);
		  endrun(7878);
		}
	    }

	  save_PartAllocFactor = All.PartAllocFactor;
	  save_TreeAllocFactor = All.TreeAllocFactor;

	  /* common data  */
	  byten(&All, sizeof(struct global_data_all_processes), modus);

	  if(ThisTask == 0 && modus > 0)
	    all_task0 = All;

#ifndef NOMPI
	  if(modus > 0 && groupTask == 0)	/* read */
	    {
	      MPI_Bcast(&all_task0, sizeof(struct global_data_all_processes), MPI_BYTE, 0, GADGET_WORLD);
	    }
#endif
	  old_MaxPart = All.MaxPart;
	  old_MaxNodes = All.TreeAllocFactor * All.MaxPart;

	  if(modus)		/* read */
	    {
	      if(All.PartAllocFactor != save_PartAllocFactor)
		{
		  All.PartAllocFactor = save_PartAllocFactor;
		  All.MaxPart = All.PartAllocFactor * (All.TotNumPart / NTask);
		  All.MaxPartSph = All.PartAllocFactor * (All.TotN_gas / NTask);
		  save_PartAllocFactor = -1;
		}

	      if(All.TreeAllocFactor != save_TreeAllocFactor)
		{
		  All.TreeAllocFactor = save_TreeAllocFactor;
		  save_TreeAllocFactor = -1;
		}

	      if(all_task0.Time != All.Time)
		{
		  printf("The restart file on task=%d is not consistent with the one on task=0\n", ThisTask);
		  fflush(stdout);
		  endrun(16);
		}

	      allocate_memory();
	    }

	  in(&NumPart, modus);

	  if(NumPart > All.MaxPart)
	    {
	      printf
		("it seems you have reduced(!) 'PartAllocFactor' below the value of %g needed to load the restart file.\n",
		 NumPart / (((double) All.TotNumPart) / NTask));
	      printf("fatal error\n");
	      endrun(22);
	    }

	  /* Particle data  */
	  byten(&P[0], NumPart * sizeof(struct particle_data), modus);

	  in(&N_gas, modus);

	  if(N_gas > 0)
	    {
	      if(N_gas > All.MaxPartSph)
		{
		  printf
		    ("SPH: it seems you have reduced(!) 'PartAllocFactor' below the value of %g needed to load the restart file.\n",
		     N_gas / (((double) All.TotN_gas) / NTask));
		  printf("fatal error\n");
		  endrun(222);
		}
	      /* Sph-Particle data  */
	      byten(&SphP[0], N_gas * sizeof(struct sph_particle_data), modus);
	    }

	  /* write state of random number generator */
	  byten(gsl_rng_state(random_generator), gsl_rng_size(random_generator), modus);

	  /* now store relevant data for tree */

	  if(modus)		/* read */
	    {
	      ngb_treeallocate(MAX_NGB);

	      force_treeallocate(All.TreeAllocFactor * All.MaxPart, All.MaxPart);
	    }

	  in(&Numnodestree, modus);

	  if(Numnodestree > MaxNodes)
	    {
	      printf
		("Tree storage: it seems you have reduced(!) 'PartAllocFactor' below the value needed to load the restart file (task=%d). "
		 "Numnodestree=%d  MaxNodes=%d\n", ThisTask, Numnodestree, MaxNodes);
	      endrun(221);
	    }

	  byten(Nodes_base, Numnodestree * sizeof(struct NODE), modus);
	  byten(Extnodes_base, Numnodestree * sizeof(struct extNODE), modus);

	  byten(Father, NumPart * sizeof(int), modus);

	  byten(Nextnode, NumPart * sizeof(int), modus);
	  byten(Nextnode + All.MaxPart, MAXTOPNODES * sizeof(int), modus);

	  byten(DomainStartList, NTask * sizeof(int), modus);
	  byten(DomainEndList, NTask * sizeof(int), modus);
	  byten(DomainTask, MAXTOPNODES * sizeof(int), modus);
	  byten(DomainNodeIndex, MAXTOPNODES * sizeof(int), modus);
	  byten(DomainTreeNodeLen, MAXTOPNODES * sizeof(FLOAT), modus);
	  byten(DomainHmax, MAXTOPNODES * sizeof(FLOAT), modus);
	  byten(DomainMoment, MAXTOPNODES * sizeof(struct DomainNODE), modus);

	  byten(DomainCorner, 3 * sizeof(double), modus);
	  byten(DomainCenter, 3 * sizeof(double), modus);
	  byten(&DomainLen, sizeof(double), modus);
	  byten(&DomainFac, sizeof(double), modus);
	  byten(&DomainMyStart, sizeof(int), modus);
	  byten(&DomainMyLast, sizeof(int), modus);

	  if(modus)		/* read */
	    if(All.PartAllocFactor != save_PartAllocFactor || All.TreeAllocFactor != save_TreeAllocFactor)
	      {
		for(i = 0; i < NumPart; i++)
		  Father[i] += (All.MaxPart - old_MaxPart);

		for(i = 0; i < NumPart; i++)
		  if(Nextnode[i] >= old_MaxPart)
		    {
		      if(Nextnode[i] >= old_MaxPart + old_MaxNodes)
			Nextnode[i] += (All.MaxPart - old_MaxPart) + (MaxNodes - old_MaxPart);
		      else
			Nextnode[i] += (All.MaxPart - old_MaxPart);
		    }

		for(i = 0; i < Numnodestree; i++)
		  {
		    if(Nodes_base[i].u.d.sibling >= old_MaxPart)
		      {
			if(Nodes_base[i].u.d.sibling >= old_MaxPart + old_MaxNodes)
			  Nodes_base[i].u.d.sibling +=
			    (All.MaxPart - old_MaxPart) + (MaxNodes - old_MaxNodes);
			else
			  Nodes_base[i].u.d.sibling += (All.MaxPart - old_MaxPart);
		      }

		    if(Nodes_base[i].u.d.father >= old_MaxPart)
		      {
			if(Nodes_base[i].u.d.father >= old_MaxPart + old_MaxNodes)
			  Nodes_base[i].u.d.father += (All.MaxPart - old_MaxPart) + (MaxNodes - old_MaxNodes);
			else
			  Nodes_base[i].u.d.father += (All.MaxPart - old_MaxPart);
		      }

		    if(Nodes_base[i].u.d.nextnode >= old_MaxPart)
		      {
			if(Nodes_base[i].u.d.nextnode >= old_MaxPart + old_MaxNodes)
			  Nodes_base[i].u.d.nextnode +=
			    (All.MaxPart - old_MaxPart) + (MaxNodes - old_MaxNodes);
			else
			  Nodes_base[i].u.d.nextnode += (All.MaxPart - old_MaxPart);
		      }
		  }

		for(i = 0; i < MAXTOPNODES; i++)
		  if(Nextnode[i + All.MaxPart] >= old_MaxPart)
		    {
		      if(Nextnode[i + All.MaxPart] >= old_MaxPart + old_MaxNodes)
			Nextnode[i + All.MaxPart] += (All.MaxPart - old_MaxPart) + (MaxNodes - old_MaxNodes);
		      else
			Nextnode[i + All.MaxPart] += (All.MaxPart - old_MaxPart);
		    }

		for(i = 0; i < MAXTOPNODES; i++)
		  if(DomainNodeIndex[i] >= old_MaxPart)
		    {
		      if(DomainNodeIndex[i] >= old_MaxPart + old_MaxNodes)
			DomainNodeIndex[i] += (All.MaxPart - old_MaxPart) + (MaxNodes - old_MaxNodes);
		      else
			DomainNodeIndex[i] += (All.MaxPart - old_MaxPart);
		    }
	      }

	  fclose(fd);
	}
      else			/* wait inside the group */
	{
#ifndef NOMPI
	  if(modus > 0 && groupTask == 0)	/* read */
	    {
	      MPI_Bcast(&all_task0, sizeof(struct global_data_all_processes), MPI_BYTE, 0, GADGET_WORLD);
	    }
#endif
	}

#ifndef NOMPI
	      MPI_Barrier(GADGET_WORLD);
#endif
    }
}

/*! reads/writes n bytes in restart routine
 */
void byten(void *x, size_t n, int modus)
{
  if(modus)
    my_fread(x, n, 1, fd);
  else
    my_fwrite(x, n, 1, fd);
}

/*! reads/writes one `int' variable in restart routine
 */
void in(int *x, int modus)
{
  if(modus)
    my_fread(x, 1, sizeof(int), fd);
  else
    my_fwrite(x, 1, sizeof(int), fd);
}


/**
 * @file cblas.h
 * @brief Header file to handle Intel MKL, OpenBLAS, or Netlib includes.
 *
 * `npypacke` can be linked with a reference CBLAS implementation, OpenBLAS, or
 * Intel MKL, but Intel MKL uses a different header file, `mkl.h`, while
 * OpenBLAS uses the CBLAS header `cblas.h`. This file includes the correct
 * header file given an appropriate preprocessor macro is defined, and defines
 * some types appropriately so that the user can simply write in terms of the
 * Intel MKL interface, whether or not Intel MKL is actually linked.
 */

#ifndef NPY_LPK_CBLAS_H
#define NPY_LPK_CBLAS_H
// if linking with Netlib CBLAS
#if defined(CBLAS_INCLUDE)
#include <cblas.h>
// define MKL_INT used in extension modules as in mkl.h
#ifndef MKL_INT
#define MKL_INT int
#endif /* MKL_INT */
// else if linking with OpenBLAS CBLAS
#elif defined(OPENBLAS_INCLUDE)
#include <cblas.h>
// OpenBLAS has blasint typedef, so define MKL_INT using blasint
#ifndef MKL_INT
#define MKL_INT blasint
#endif /* MKL_INT */
// else if linking to Intel MKL
#elif defined(MKL_INCLUDE)
#include <mkl.h>
// else error, no LAPACKE includes specified
#else
// silence error squiggles in VS Code (__INTELLISENSE__ always defined)
#ifndef __INTELLISENSE__
#error "no CBLAS includes specified. try -D(CBLAS|OPENBLAS|MKL)_INCLUDE"
#endif /* __INTELLISENSE__ */
#endif

// silence error squiggles in VS Code. use mkl.h since it also define MKL types
#ifdef __INTELLISENSE__
#include <mkl.h>
#endif /* __INTELLISENSE__ */

#endif /* NPY_LPK_CBLAS_H */