#include "script_visitor.hpp"

#include <program_ast.hpp>
#include <intvar.hpp>
#include <realvar.hpp>
#include <strvar.hpp>
#include <listvar.hpp>
#include <reply.hpp>
#include <op_package.hpp>
#include <compiler.hpp>

#include <iostream>
#include <boost/iterator/indirect_iterator.hpp>

using namespace Moove;

template<class T, class VariantType>
struct FixedCast {
   T operator () (const Variant& var)const
   { return dynamic_cast<const VariantType&>(var).value(); }
};

template<class T, class VariantType1, class VariantType2>
struct BinaryCast {
   T operator () (const Variant& var)const
   {
      if(const VariantType1* v1 = dynamic_cast<const VariantType1*>(&var))
	 return v1->value();
      else if(const VariantType2* v2 = dynamic_cast<const VariantType2*>(&var))
	 return v2->value();

      throw std::bad_cast();
      return T();
   }
};

template<class ResultVariant, class Cast>
class ScriptOperatorPackage : public OperatorPackage {
private:
   Cast m_cast;

   Reply doBinaryMathOp(Opcode op, const Variant& left, const Variant& right)const
   {
      boost::shared_ptr<ResultVariant> resultVar(dynamic_cast<ResultVariant*>(ResultVariant::classFactory().create()));
      assert(resultVar.get() != 0);

      switch(op) {
	 case OP_ADD:
	    resultVar->setValue(m_cast(left) + m_cast(right));
	    break;

	 case OP_SUB:
	    resultVar->setValue(m_cast(left) - m_cast(right));
	    break;

	 case OP_MUL:
	    resultVar->setValue(m_cast(left) * m_cast(right));
	    break;

	 case OP_DIV:
	    resultVar->setValue(m_cast(left) / m_cast(right));
	    break;

	 default:
	    return OperatorPackage::doBinaryOp(op, left, right);
      }

      return Reply(Reply::NORMAL, resultVar);
   }

   Reply doBinaryCompareOp(Opcode op, const Variant& left, const Variant& right)const
   {
      boost::shared_ptr<DefaultIntVar> resultVar(DefaultIntVar::classFactory().create());
      assert(resultVar.get() != 0);

      switch(op) {
	 case OP_LT:
	    resultVar->setValue(m_cast(left) < m_cast(right));
	    break;

	 case OP_LE:
	    resultVar->setValue(m_cast(left) <= m_cast(right));
	    break;

	 case OP_EQ:
	    resultVar->setValue(m_cast(left) == m_cast(right));
	    break;

	 case OP_NE:
	    resultVar->setValue(m_cast(left) != m_cast(right));
	    break;

	 case OP_GE:
	    resultVar->setValue(m_cast(left) >= m_cast(right));
	    break;

	 case OP_GT:
	    resultVar->setValue(m_cast(left) > m_cast(right));
	    break;

	 default:
	    return OperatorPackage::doBinaryOp(op, left, right);
      }

      return Reply(Reply::NORMAL, resultVar);
   }

public:
   Reply doBinaryOp(Opcode op, const Variant& left, const Variant& right)const
   {
      switch(op) {
	 case OP_ADD:
	 case OP_SUB:
	 case OP_MUL:
	 case OP_DIV:
	    return doBinaryMathOp(op, left, right);

	 case OP_LT:
	 case OP_LE:
	 case OP_EQ:
	 case OP_NE:
	 case OP_GE:
	 case OP_GT:
	    return doBinaryCompareOp(op, left, right);
      }

      return OperatorPackage::doBinaryOp(op, left, right);
   }
};

template<>
Reply
ScriptOperatorPackage<DefaultStrVar,
		      FixedCast<const std::string&, StrVar> >::doBinaryMathOp(Opcode op, const Variant& left, const Variant& right)const
{
   if(op == OP_ADD) {
      boost::shared_ptr<DefaultStrVar> resultVar(DefaultStrVar::classFactory().create());
      assert(resultVar.get() != 0);

      resultVar->setValue(m_cast(left) + m_cast(right));
      return Reply(Reply::NORMAL, resultVar);
   }

   return OperatorPackage::doBinaryOp(op, left, right);
}

typedef ScriptOperatorPackage<DefaultIntVar, FixedCast<int, IntVar> > IntOpPkg;
typedef ScriptOperatorPackage<DefaultRealVar, FixedCast<double, RealVar> > RealOpPkg;
typedef ScriptOperatorPackage<DefaultRealVar, BinaryCast<double, IntVar, RealVar> > IntRealOpPkg;
typedef ScriptOperatorPackage<DefaultStrVar, FixedCast<const std::string&, StrVar> > StrOpPkg;

namespace {

IntOpPkg	pkgInt;
RealOpPkg	pkgReal;
IntRealOpPkg	pkgIntReal;
StrOpPkg	pkgStr;

}

void ScriptVisitor::visitStmts(const Stmt::Block& block)
{
   Stmt::Block::const_iterator end = block.end();
   for(Stmt::Block::const_iterator stmt = block.begin(); stmt != end; ++stmt)
      (*stmt)->accept(*this);
}

void ScriptVisitor::doBinaryOp(Opcode op, const Expr::BinaryExpr& binExpr)
{
   binExpr.left().accept(*this);
   binExpr.right().accept(*this);
   m_vm.doBinaryOp(op);
}

void ScriptVisitor::assignTo(const Expr::Expr& expr)
{
   if(const Expr::Variable* var = dynamic_cast<const Expr::Variable*>(&expr))
      m_vars[var->id()] = m_vm.stackTop();
   else
      throw InvalidAssignment("Invalid Assignment");
}

void ScriptVisitor::acceptArgs(const Expr::ArgList& args)
{
   ListVar::Container contents;

   Expr::ArgList::const_iterator end = args.end();
   for(Expr::ArgList::const_iterator elem = args.begin(); elem != end; ++elem) {
      m_spliceFlag = false;
      (*elem)->accept(*this);

      if(m_spliceFlag) {
	 boost::shared_ptr<ListVar> splice = boost::dynamic_pointer_cast<ListVar>(m_vm.popOffStack());
	 assert(splice);
	 contents.insert(contents.end(), splice->contents().begin(), splice->contents().end());
      } else {
	 contents.push_back(m_vm.popOffStack());
      }
   }

   m_spliceFlag = false;
   m_vm.pushOnStack(boost::shared_ptr<Variant>(DefaultListVar::classFactory().createList(contents)));
}

bool ScriptVisitor::runIf(const Expr::Expr& test, const Stmt::Block& stmts)
{
   test.accept(*this);
   boost::shared_ptr<Variant> result = m_vm.popOffStack();

   bool shouldRun = false;

   if(boost::shared_ptr<IntVar> intVar = boost::dynamic_pointer_cast<IntVar>(result))
      shouldRun = intVar->value() != 0;
   else if(boost::shared_ptr<RealVar> realVar = boost::dynamic_pointer_cast<RealVar>(result))
      shouldRun = realVar->value() != 0;
   else if(boost::shared_ptr<StrVar> strVar = boost::dynamic_pointer_cast<StrVar>(result))
      shouldRun = !strVar->value().empty();

   if(shouldRun)
      visitStmts(stmts);

   return shouldRun;
}

ScriptVisitor::ScriptVisitor() : m_spliceFlag(false), m_vm(m_opMap)
{
   const TypeRegistry::TypeEntry& intType = m_typeReg.registerType("int", DefaultIntVar::classFactory());
   const TypeRegistry::TypeEntry& realType = m_typeReg.registerType("real", DefaultRealVar::classFactory());
   const TypeRegistry::TypeEntry& strType = m_typeReg.registerType("str", DefaultStrVar::classFactory());

   m_opMap.registerUnary(intType, pkgInt);
   m_opMap.registerUnary(realType, pkgReal);
   m_opMap.registerUnary(strType, pkgStr);

   m_opMap.registerBinary(intType, intType, pkgInt);
   m_opMap.registerBinary(realType, realType, pkgReal);
   m_opMap.registerBinary(intType, realType, pkgIntReal);
   m_opMap.registerBinary(realType, intType, pkgIntReal);
   m_opMap.registerBinary(strType, strType, pkgStr);
}

void ScriptVisitor::visit(const Program& program)
{
   m_vars.clear();
   m_vm.reset();
   m_program = &program;

   visitStmts(program.stmts());
}

void ScriptVisitor::visit(const Expr::Integer& intExpr)
{
   m_vm.pushOnStack(boost::shared_ptr<Variant>(DefaultIntVar::classFactory().createInt(intExpr.value())));
}

void ScriptVisitor::visit(const Expr::Real& realExpr)
{
   m_vm.pushOnStack(boost::shared_ptr<Variant>(DefaultRealVar::classFactory().createReal(realExpr.value())));
}

void ScriptVisitor::visit(const Expr::Str& strExpr)
{
   m_vm.pushOnStack(boost::shared_ptr<Variant>(DefaultStrVar::classFactory().createStr(strExpr.str())));
}

void ScriptVisitor::visit(const Expr::List& listExpr)
{
   acceptArgs(listExpr.elements());
}

void ScriptVisitor::visit(const Expr::Variable& varExpr)
{
   VariableTable::const_iterator var = m_vars.find(varExpr.id());
   if(var == m_vars.end())
      throw UndefinedVariable("Undefined Variable");

   m_vm.pushOnStack(var->second);
}

void ScriptVisitor::visit(const Expr::Splice& spliceExpr)
{
   spliceExpr.operand().accept(*this);
   m_spliceFlag = true;
}

void ScriptVisitor::visit(const Expr::PreInc& preIncExpr)
{
   preIncExpr.operand().accept(*this);
   m_vm.pushOnStack(boost::shared_ptr<Variant>(DefaultIntVar::classFactory().createInt(1)));
   m_vm.doBinaryOp(OP_ADD);
   assignTo(preIncExpr.operand());
}

void ScriptVisitor::visit(const Expr::PreDec& preDecExpr)
{
   preDecExpr.operand().accept(*this);
   m_vm.pushOnStack(boost::shared_ptr<Variant>(DefaultIntVar::classFactory().createInt(-1)));
   m_vm.doBinaryOp(OP_ADD);
   assignTo(preDecExpr.operand());
}

void ScriptVisitor::visit(const Expr::PostInc& postIncExpr)
{
   postIncExpr.operand().accept(*this);
   m_vm.pushOnStack(m_vm.stackTop());
   m_vm.pushOnStack(boost::shared_ptr<Variant>(DefaultIntVar::classFactory().createInt(1)));
   m_vm.doBinaryOp(OP_ADD);
   assignTo(postIncExpr.operand());
   m_vm.popOffStack();
}

void ScriptVisitor::visit(const Expr::PostDec& postDecExpr)
{
   postDecExpr.operand().accept(*this);
   m_vm.pushOnStack(m_vm.stackTop());
   m_vm.pushOnStack(boost::shared_ptr<Variant>(DefaultIntVar::classFactory().createInt(-1)));
   m_vm.doBinaryOp(OP_ADD);
   assignTo(postDecExpr.operand());
   m_vm.popOffStack();
}

void ScriptVisitor::visit(const Expr::Assign& assignExpr)
{
   assignExpr.right().accept(*this);
   assignTo(assignExpr.left());
}

void ScriptVisitor::visit(const Expr::AddEqual& addEqExpr)
{
   doBinaryOp(OP_ADD, addEqExpr);
   assignTo(addEqExpr.left());
}

void ScriptVisitor::visit(const Expr::SubEqual& subEqExpr)
{
   doBinaryOp(OP_SUB, subEqExpr);
   assignTo(subEqExpr.left());
}

void ScriptVisitor::visit(const Expr::MulEqual& mulEqExpr)
{
   doBinaryOp(OP_MUL, mulEqExpr);
   assignTo(mulEqExpr.left());
}

void ScriptVisitor::visit(const Expr::DivEqual& divEqExpr)
{
   doBinaryOp(OP_DIV, divEqExpr);
   assignTo(divEqExpr.left());
}

void ScriptVisitor::visit(const Expr::Conditional& condExpr)
{
   condExpr.test().accept(*this);
   boost::shared_ptr<IntVar> result = boost::dynamic_pointer_cast<IntVar>(m_vm.popOffStack());
   assert(result);

   if(result->value())
      condExpr.trueExpr().accept(*this);
   else
      condExpr.falseExpr().accept(*this);
}

void ScriptVisitor::visit(const Expr::Index& expr)
{
   expr.expr().accept(*this);
   expr.index().accept(*this);

   boost::shared_ptr<IntVar> index = boost::dynamic_pointer_cast<IntVar>(m_vm.popOffStack());
   boost::shared_ptr<ListVar> value = boost::dynamic_pointer_cast<ListVar>(m_vm.popOffStack());
   assert(index && value);

   if(index->value() >= 1 && index->value() <= value->contents().size())
      m_vm.pushOnStack(value->contents()[index->value() - 1]);
   else
      throw OutOfRange("Index value out of range");
}

void ScriptVisitor::printVariant(const Variant& var)
{
   if(const IntVar* intValue = dynamic_cast<const IntVar*>(&var)) {
      std::cout << intValue->value();
   } else if(const RealVar* realValue = dynamic_cast<const RealVar*>(&var)) {
      std::cout << realValue->value();
   } else if(const StrVar* strValue = dynamic_cast<const StrVar*>(&var)) {
      std::cout << strValue->value();
   } else if(const ListVar* listValue = dynamic_cast<const ListVar*>(&var)) {
      std::cout << "{";

      ListVar::Container::const_iterator end = listValue->contents().end();
      for(ListVar::Container::const_iterator elem = listValue->contents().begin(); elem != end; ++elem) {
	 printVariant(**elem);

	 if(elem + 1 != end)
	    std::cout << ", ";
      }

      std::cout << "}";
   } else {
      std::cout << "(unknown type: " << var.factory().regEntry().name() << ")";
   }
}

void ScriptVisitor::visit(const Expr::Builtin& builtinExpr)
{
   typedef boost::indirect_iterator<ListVar::Container::const_iterator> ArgIterator;

   boost::shared_ptr<Variant> resultVar;
   boost::shared_ptr<ListVar> argsVar;

   acceptArgs(builtinExpr.args());
   if(!(argsVar = boost::dynamic_pointer_cast<ListVar>(m_vm.popOffStack())))
      throw InvalidType("Invalid Type");

   if(builtinExpr.name() == "print") {
      std::for_each(ArgIterator(argsVar->contents().begin()), ArgIterator(argsVar->contents().end()), &printVariant);
      std::cout << std::endl;
   } else if(builtinExpr.name() == "input") {
      assert(argsVar->contents().size() == 1);

      boost::shared_ptr<StrVar> inputType = boost::dynamic_pointer_cast<StrVar>(argsVar->contents()[0]);
      assert(inputType);

      std::cout << "Enter value: ";

      if(inputType->value() == "int") {
	 int value;

	 assert(std::cin >> value);
	 std::cin.ignore();
	 resultVar.reset(DefaultIntVar::classFactory().createInt(value));
      } else if(inputType->value() == "real") {
	 double value;

	 assert(std::cin >> value);
	 std::cin.ignore();
	 resultVar.reset(DefaultRealVar::classFactory().createReal(value));
      } else if(inputType->value() == "str") {
	 std::string value;

	 assert(getline(std::cin, value));
	 resultVar.reset(DefaultStrVar::classFactory().createStr(value));
      } else {
	 throw InvalidType("Invalid Type");
      }
   }

   // this is an expression, so it needs a return value
   m_vm.pushOnStack(resultVar ? resultVar : boost::shared_ptr<Variant>(DefaultIntVar::classFactory().createInt(0)));
}

void ScriptVisitor::visit(const Expr::Length&)
{
   boost::shared_ptr<ListVar> listValue = boost::dynamic_pointer_cast<ListVar>(m_vm.stackTop());
   m_vm.pushOnStack(boost::shared_ptr<Variant>(DefaultIntVar::classFactory().createInt(listValue->contents().size())));
}

void ScriptVisitor::visit(const Stmt::If& ifStmt)
{
   if(!runIf(ifStmt.test(), ifStmt.body())) {
      Stmt::If::ElseList::const_iterator clause, end = ifStmt.elseList().end();
      for(clause = ifStmt.elseList().begin(); clause != end; ++clause) {
	 m_ranIf = false;
	 (*clause)->accept(*this);

	 if(m_ranIf)
	    break;
      }
   }
}

void ScriptVisitor::visit(const Stmt::If::Else& elseClause)
{
   if(elseClause.hasTest())		// elseif
      m_ranIf = runIf(elseClause.test(), elseClause.body());
   else {				// else
      visitStmts(elseClause.body());
      m_ranIf = true;
   }
}

void ScriptVisitor::visit(const Stmt::While& whileStmt)
{
   for(;;) {
      whileStmt.test().accept(*this);
      boost::shared_ptr<IntVar> testResult = boost::dynamic_pointer_cast<IntVar>(m_vm.popOffStack());
      assert(testResult.get() != 0);

      if(!testResult->value())
	 break;

      visitStmts(whileStmt.body());
   }
}

void ScriptVisitor::visit(const Stmt::ForRange& forRangeStmt)
{
   forRangeStmt.start().accept(*this);
   boost::shared_ptr<Variant> indexVar = m_vm.stackTop();

   forRangeStmt.end().accept(*this);
   boost::shared_ptr<Variant> endVar = m_vm.stackTop();

   for(;;) {
      m_vm.doBinaryOp(OP_LE);
      boost::shared_ptr<IntVar> compResult = boost::dynamic_pointer_cast<IntVar>(m_vm.popOffStack());
      assert(compResult);

      if(compResult->value()) {
	 m_vars[forRangeStmt.id()] = indexVar;
	 visitStmts(forRangeStmt.body());

	 m_vm.pushOnStack(indexVar);
	 m_vm.pushOnStack(boost::shared_ptr<DefaultIntVar>(DefaultIntVar::classFactory().createInt(1)));
	 m_vm.doBinaryOp(OP_ADD);
	 indexVar = m_vm.stackTop();
	 m_vm.pushOnStack(endVar);
      } else {
	 break;
      }
   }
}

void ScriptVisitor::visit(const Stmt::Expr& exprStmt)
{
   exprStmt.expr().accept(*this);
   m_vm.popOffStack();
}


//---------------------------------------------------------------------------//
// Copyright (c) 2018-2021 Mikhail Komarov <nemo@nil.foundation>
// Copyright (c) 2020-2021 Nikita Kaskov <nbering@nil.foundation>
// Copyright (c) 2020-2021 Ilias Khairullin <ilias@nil.foundation>
//
// MIT License
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in all
// copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
// SOFTWARE.
//---------------------------------------------------------------------------//

#ifndef CRYPTO3_R1CS_GG_PPZKSNARK_IPP2_PROVE_HPP
#define CRYPTO3_R1CS_GG_PPZKSNARK_IPP2_PROVE_HPP

#include <algorithm>
#include <vector>
#include <tuple>
#include <string>

#include <boost/iterator/zip_iterator.hpp>

#include <nil/crypto3/detail/pack_numeric.hpp>

#include <nil/crypto3/hash/algorithm/hash.hpp>
#include <nil/crypto3/hash/sha2.hpp>

#include <nil/crypto3/math/polynomial/basic_operations.hpp>

#include <nil/crypto3/algebra/multiexp/multiexp.hpp>
#include <nil/crypto3/algebra/multiexp/policies.hpp>
#include <nil/crypto3/algebra/algorithms/pair.hpp>

#include <nil/crypto3/zk/snark/schemes/ppzksnark/r1cs_gg_ppzksnark/detail/basic_policy.hpp>
#include <nil/crypto3/zk/snark/schemes/ppzksnark/r1cs_gg_ppzksnark/ipp2/proof.hpp>
#include <nil/crypto3/zk/snark/schemes/ppzksnark/r1cs_gg_ppzksnark/ipp2/srs.hpp>
#include <nil/crypto3/zk/snark/schemes/ppzksnark/r1cs_gg_ppzksnark/ipp2/transcript.hpp>
#include <nil/crypto3/zk/snark/schemes/ppzksnark/r1cs_gg_ppzksnark/proof.hpp>

namespace nil {
    namespace crypto3 {
        namespace zk {
            namespace snark {
                /// Returns the vector used for the linear combination fo the inner pairing product
                /// between A and B for the Groth16 aggregation: A^r * B. It is required as it
                /// is not enough to simply prove the ipp of A*B, we need a random linear
                /// combination of those.
                template<typename FieldType>
                std::vector<typename FieldType::value_type>
                    structured_scalar_power(std::size_t num, const typename FieldType::value_type &s) {
                    std::vector<typename FieldType::value_type> powers = {FieldType::value_type::one()};
                    for (int i = 1; i < num; i++) {
                        powers.emplace_back(powers.back() * s);
                    }
                    return powers;
                }

                /// compress is similar to commit::{V,W}KEY::compress: it modifies the `vec`
                /// vector by setting the value at index $i:0 -> split$  $vec[i] = vec[i] +
                /// vec[i+split]^scaler$. The `vec` vector is half of its size after this call.
                template<typename CurveType, typename InputRange,
                         typename ValueType = typename std::iterator_traits<typename InputRange::iterator>::value_type>
                typename std::enable_if<
                    std::is_same<typename CurveType::template g1_type<>::value_type, ValueType>::value ||
                    std::is_same<typename CurveType::template g2_type<>::value_type, ValueType>::value ||
                    std::is_same<typename CurveType::scalar_field_type::value_type, ValueType>::value>::type
                    compress(InputRange &vec, std::size_t split,
                             const typename CurveType::scalar_field_type::value_type &scalar) {
                    std::for_each(boost::make_zip_iterator(boost::make_tuple(vec.begin(), vec.begin() + split)),
                                  boost::make_zip_iterator(boost::make_tuple(vec.begin() + split, vec.end())),
                                  [&](const boost::tuple<ValueType &, ValueType &> &t) {
                                      t.template get<0>() = t.template get<0>() + t.template get<1>() * scalar;
                                  });
                    vec.resize(split);
                }

                /// It returns the evaluation of the polynomial $\prod (1 + x_{l-j}(rX)^{2j}$ at
                /// the point z, where transcript contains the reversed order of all challenges (the x).
                /// The challenges must be in reversed order for the correct evaluation of the
                /// polynomial in O(logn)
                template<typename FieldType, typename InputFieldValueIterator>
                typename std::enable_if<std::is_same<typename std::iterator_traits<InputFieldValueIterator>::value_type,
                                                     typename FieldType::value_type>::value,
                                        typename FieldType::value_type>::type
                    polynomial_evaluation_product_form_from_transcript(InputFieldValueIterator transcript_first,
                                                                       InputFieldValueIterator transcript_last,
                                                                       const typename FieldType::value_type &z,
                                                                       const typename FieldType::value_type &r_shift) {
                    // this is the term (rz) that will get squared at each step to produce the
                    // $(rz)^{2j}$ of the formula
                    typename FieldType::value_type power_zr = z;
                    power_zr = power_zr * r_shift;

                    // 0 iteration
                    InputFieldValueIterator transcript_iter = transcript_first;
                    typename FieldType::value_type res = FieldType::value_type::one() + (*transcript_iter * power_zr);
                    power_zr = power_zr * power_zr;
                    ++transcript_iter;

                    // the rest
                    while (transcript_iter != transcript_last) {
                        res = res * (FieldType::value_type::one() + (*transcript_iter * power_zr));
                        power_zr = power_zr * power_zr;
                        ++transcript_iter;
                    }

                    return res;
                }

                // Compute the coefficients of the polynomial $\prod_{j=0}^{l-1} (1 + x_{l-j}(rX)^{2j})$
                // It does this in logarithmic time directly; here is an example with 2
                // challenges:
                //
                //     We wish to compute $(1+x_1ra)(1+x_0(ra)^2) = 1 +  x_1ra + x_0(ra)^2 + x_0x_1(ra)^3$
                //     Algorithm: $c_{-1} = [1]$; $c_j = c_{i-1} \| (x_{l-j} * c_{i-1})$; $r = r*r$
                //     $c_0 = c_{-1} \| (x_1 * r * c_{-1}) = [1] \| [rx_1] = [1, rx_1]$, $r = r^2$
                //     $c_1 = c_0 \| (x_0 * r^2c_0) = [1, rx_1] \| [x_0r^2, x_0x_1r^3] = [1, x_1r, x_0r^2, x_0x_1r^3]$
                //     which is equivalent to $f(a) = 1 + x_1ra + x_0(ra)^2 + x_0x_1r^2a^3$
                //
                // This method expects the coefficients in reverse order so transcript[i] =
                // x_{l-j}.
                template<typename FieldType, typename InputFieldValueIterator>
                typename std::enable_if<std::is_same<typename std::iterator_traits<InputFieldValueIterator>::value_type,
                                                     typename FieldType::value_type>::value,
                                        std::vector<typename FieldType::value_type>>::type
                    polynomial_coefficients_from_transcript(InputFieldValueIterator transcript_first,
                                                            InputFieldValueIterator transcript_last,
                                                            const typename FieldType::value_type &r_shift) {
                    std::vector<typename FieldType::value_type> coefficients = {FieldType::value_type::one()};
                    typename FieldType::value_type power_2_r = r_shift;

                    InputFieldValueIterator transcript_iter = transcript_first;
                    while (transcript_iter != transcript_last) {
                        std::size_t n = coefficients.size();
                        for (int j = 0; j < n; j++) {
                            coefficients.emplace_back(coefficients[j] * (*transcript_iter * power_2_r));
                        }
                        power_2_r = power_2_r * power_2_r;

                        ++transcript_iter;
                    }

                    return coefficients;
                }

                /// Returns the KZG opening proof for the given commitment key. Specifically, it
                /// returns $g^{f(alpha) - f(z) / (alpha - z)}$ for $a$ and $b$.
                template<typename GroupType, typename InputGroupIterator, typename InputScalarRange>
                typename std::enable_if<
                    std::is_same<typename GroupType::value_type,
                                 typename std::iterator_traits<InputGroupIterator>::value_type>::value &&
                        std::is_same<
                            typename GroupType::curve_type::scalar_field_type::value_type,
                            typename std::iterator_traits<typename InputScalarRange::iterator>::value_type>::value,
                    kzg_opening<GroupType>>::type
                    prove_commitment_key_kzg_opening(
                        InputGroupIterator srs_powers_alpha_first, InputGroupIterator srs_powers_alpha_last,
                        InputGroupIterator srs_powers_beta_first, InputGroupIterator srs_powers_beta_last,
                        const InputScalarRange &poly,
                        const typename GroupType::curve_type::scalar_field_type::value_type &eval_poly,
                        const typename GroupType::curve_type::scalar_field_type::value_type &kzg_challenge) {
                    typename GroupType::curve_type::scalar_field_type::value_type neg_kzg_challenge = -kzg_challenge;

                    BOOST_ASSERT(poly.size() == std::distance(srs_powers_alpha_first, srs_powers_alpha_last));
                    BOOST_ASSERT(poly.size() == std::distance(srs_powers_beta_first, srs_powers_beta_last));

                    // f_v(X) - f_v(z) / (X - z)
                    std::vector<typename GroupType::curve_type::scalar_field_type::value_type> f_vX_sub_f_vZ;
                    math::_polynomial_subtraction(f_vX_sub_f_vZ,
                                                  poly,
                                                  {{
                                                      eval_poly,
                                                  }});
                    std::vector<typename GroupType::curve_type::scalar_field_type::value_type> quotient_polynomial,
                        remainder_polynomial;
                    math::_polynomial_division<typename GroupType::curve_type::scalar_field_type>(
                        quotient_polynomial, remainder_polynomial, f_vX_sub_f_vZ,
                        {{
                            neg_kzg_challenge,
                            GroupType::curve_type::scalar_field_type::value_type::one(),
                        }});

                    if (quotient_polynomial.size() < poly.size()) {
                        quotient_polynomial.resize(poly.size(),
                                                   GroupType::curve_type::scalar_field_type::value_type::zero());
                    }
                    BOOST_ASSERT(quotient_polynomial.size() == poly.size());

                    // we do one proof over h^a and one proof over h^b (or g^a and g^b depending
                    // on the curve we are on). that's the extra cost of the commitment scheme
                    // used which is compatible with Groth16 CRS insteaf of the original paper
                    // of Bunz'19
                    return kzg_opening<GroupType> {algebra::multiexp<algebra::policies::multiexp_method_bos_coster>(
                                                       srs_powers_alpha_first, srs_powers_alpha_last,
                                                       quotient_polynomial.begin(), quotient_polynomial.end(), 1),
                                                   algebra::multiexp<algebra::policies::multiexp_method_bos_coster>(
                                                       srs_powers_beta_first, srs_powers_beta_last,
                                                       quotient_polynomial.begin(), quotient_polynomial.end(), 1)};
                }

                template<typename CurveType, typename InputG2Iterator, typename InputScalarIterator>
                typename std::enable_if<
                    std::is_same<typename CurveType::template g2_type<>::value_type,
                                 typename std::iterator_traits<InputG2Iterator>::value_type>::value &&
                        std::is_same<typename CurveType::scalar_field_type::value_type,
                                     typename std::iterator_traits<InputScalarIterator>::value_type>::value,
                    kzg_opening<typename CurveType::template g2_type<>>>::type
                    prove_commitment_v(InputG2Iterator srs_powers_alpha_first, InputG2Iterator srs_powers_alpha_last,
                                       InputG2Iterator srs_powers_beta_first, InputG2Iterator srs_powers_beta_last,
                                       InputScalarIterator transcript_first, InputScalarIterator transcript_last,
                                       const typename CurveType::scalar_field_type::value_type &kzg_challenge) {
                    std::vector<typename CurveType::scalar_field_type::value_type> vkey_poly =
                        polynomial_coefficients_from_transcript<typename CurveType::scalar_field_type>(
                            transcript_first, transcript_last, CurveType::scalar_field_type::value_type::one());
                    math::_condense(vkey_poly);
                    BOOST_ASSERT(!math::_is_zero(vkey_poly));

                    typename CurveType::scalar_field_type::value_type vkey_poly_z =
                        polynomial_evaluation_product_form_from_transcript<typename CurveType::scalar_field_type>(
                            transcript_first, transcript_last, kzg_challenge,
                            CurveType::scalar_field_type::value_type::one());

                    return prove_commitment_key_kzg_opening<typename CurveType::template g2_type<>>(
                        srs_powers_alpha_first, srs_powers_alpha_last, srs_powers_beta_first, srs_powers_beta_last,
                        vkey_poly, vkey_poly_z, kzg_challenge);
                }

                template<typename CurveType, typename InputG1Iterator, typename InputScalarIterator>
                typename std::enable_if<
                    std::is_same<typename CurveType::template g1_type<>::value_type,
                                 typename std::iterator_traits<InputG1Iterator>::value_type>::value &&
                        std::is_same<typename CurveType::scalar_field_type::value_type,
                                     typename std::iterator_traits<InputScalarIterator>::value_type>::value,
                    kzg_opening<typename CurveType::template g1_type<>>>::type
                    prove_commitment_w(InputG1Iterator srs_powers_alpha_first, InputG1Iterator srs_powers_alpha_last,
                                       InputG1Iterator srs_powers_beta_first, InputG1Iterator srs_powers_beta_last,
                                       InputScalarIterator transcript_first, InputScalarIterator transcript_last,
                                       typename CurveType::scalar_field_type::value_type r_shift,
                                       const typename CurveType::scalar_field_type::value_type &kzg_challenge) {
                    std::size_t n = std::distance(srs_powers_beta_first, srs_powers_beta_last) / 2;
                    BOOST_ASSERT(2 * n == std::distance(srs_powers_alpha_first, srs_powers_alpha_last));

                    // this computes f(X) = \prod (1 + x (rX)^{2^j})
                    std::vector<typename CurveType::scalar_field_type::value_type> fcoeffs =
                        polynomial_coefficients_from_transcript<typename CurveType::scalar_field_type>(
                            transcript_first, transcript_last, r_shift);
                    // this computes f_w(X) = X^n * f(X) - it simply shifts all coefficients to by n
                    fcoeffs.insert(fcoeffs.begin(), n, CurveType::scalar_field_type::value_type::zero());

                    // this computes f(z)
                    typename CurveType::scalar_field_type::value_type fz =
                        polynomial_evaluation_product_form_from_transcript<typename CurveType::scalar_field_type>(
                            transcript_first, transcript_last, kzg_challenge, r_shift);
                    // this computes the "shift" z^n
                    typename CurveType::scalar_field_type::value_type zn = kzg_challenge.pow(n);
                    // this computes f_w(z) by multiplying by zn
                    typename CurveType::scalar_field_type::value_type fwz = fz * zn;

                    return prove_commitment_key_kzg_opening<typename CurveType::template g1_type<>>(
                        srs_powers_alpha_first, srs_powers_alpha_last, srs_powers_beta_first, srs_powers_beta_last,
                        fcoeffs, fwz, kzg_challenge);
                }

                /// gipa_tipp_mipp peforms the recursion of the GIPA protocol for TIPP and MIPP.
                /// It returns a proof containing all intermdiate committed values, as well as
                /// the challenges generated necessary to do the polynomial commitment proof
                /// later in TIPP.
                template<typename CurveType, typename Hash = hashes::sha2<256>, typename InputG1Iterator1,
                         typename InputG2Iterator, typename InputG1Iterator2, typename InputScalarIterator>
                typename std::enable_if<
                    std::is_same<typename CurveType::template g1_type<>::value_type,
                                 typename std::iterator_traits<InputG1Iterator1>::value_type>::value &&
                        std::is_same<typename CurveType::template g2_type<>::value_type,
                                     typename std::iterator_traits<InputG2Iterator>::value_type>::value &&
                        std::is_same<typename CurveType::scalar_field_type::value_type,
                                     typename std::iterator_traits<InputScalarIterator>::value_type>::value &&
                        std::is_same<typename CurveType::template g1_type<>::value_type,
                                     typename std::iterator_traits<InputG1Iterator2>::value_type>::value,
                    std::tuple<gipa_proof<CurveType>, std::vector<typename CurveType::scalar_field_type::value_type>,
                               std::vector<typename CurveType::scalar_field_type::value_type>>>::type
                    gipa_tipp_mipp(transcript<CurveType, Hash> &tr, InputG1Iterator1 a_first, InputG1Iterator1 a_last,
                                   InputG2Iterator b_first, InputG2Iterator b_last, InputG1Iterator2 c_first,
                                   InputG1Iterator2 c_last, const r1cs_gg_ppzksnark_ipp2_vkey<CurveType> &vkey_input,
                                   const r1cs_gg_ppzksnark_ipp2_wkey<CurveType> &wkey_input,
                                   InputScalarIterator r_first, InputScalarIterator r_last) {
                    std::size_t input_len = std::distance(a_first, a_last);
                    BOOST_ASSERT(input_len >= 2);
                    BOOST_ASSERT((input_len & (input_len - 1)) == 0);
                    BOOST_ASSERT(input_len == std::distance(b_first, b_last));
                    BOOST_ASSERT(input_len == std::distance(r_first, r_last));
                    BOOST_ASSERT(input_len == std::distance(c_first, c_last));

                    // the values of vectors A and B rescaled at each step of the loop
                    // the values of vectors C and r rescaled at each step of the loop
                    std::vector<typename CurveType::template g1_type<>::value_type> m_a {a_first, a_last}, m_c {c_first, c_last};
                    std::vector<typename CurveType::template g2_type<>::value_type> m_b {b_first, b_last};
                    std::vector<typename CurveType::scalar_field_type::value_type> m_r {r_first, r_last};

                    // the values of the commitment keys rescaled at each step of the loop
                    r1cs_gg_ppzksnark_ipp2_vkey<CurveType> vkey = vkey_input;
                    r1cs_gg_ppzksnark_ipp2_wkey<CurveType> wkey = wkey_input;

                    // storing the values for including in the proof
                    std::vector<std::pair<typename r1cs_gg_ppzksnark_ipp2_commitment<CurveType>::output_type,
                                          typename r1cs_gg_ppzksnark_ipp2_commitment<CurveType>::output_type>>
                        comms_ab;
                    std::vector<std::pair<typename r1cs_gg_ppzksnark_ipp2_commitment<CurveType>::output_type,
                                          typename r1cs_gg_ppzksnark_ipp2_commitment<CurveType>::output_type>>
                        comms_c;
                    std::vector<
                        std::pair<typename CurveType::gt_type::value_type, typename CurveType::gt_type::value_type>>
                        z_ab;
                    std::vector<
                        std::pair<typename CurveType::template g1_type<>::value_type, typename CurveType::template g1_type<>::value_type>>
                        z_c;
                    std::vector<typename CurveType::scalar_field_type::value_type> challenges, challenges_inv;

                    constexpr std::array<std::uint8_t, 4> domain_separator {'g', 'i', 'p', 'a'};
                    tr.write_domain_separator(domain_separator.begin(), domain_separator.end());
                    typename CurveType::scalar_field_type::value_type _i = tr.read_challenge();

                    while (m_a.size() > 1) {
                        // recursive step
                        // Recurse with problem of half size
                        std::size_t split = m_a.size() / 2;

                        auto [vk_left, vk_right] = vkey.split(split);
                        auto [wk_left, wk_right] = wkey.split(split);

                        // TODO: parallel
                        // See section 3.3 for paper version with equivalent names
                        // TIPP part
                        typename r1cs_gg_ppzksnark_ipp2_commitment<CurveType>::output_type tab_l =
                            r1cs_gg_ppzksnark_ipp2_commitment<CurveType>::pair(
                                vk_left, wk_right, m_a.begin() + split, m_a.end(), m_b.begin(), m_b.begin() + split);
                        typename r1cs_gg_ppzksnark_ipp2_commitment<CurveType>::output_type tab_r =
                            r1cs_gg_ppzksnark_ipp2_commitment<CurveType>::pair(
                                vk_right, wk_left, m_a.begin(), m_a.begin() + split, m_b.begin() + split, m_b.end());

                        // \prod e(A_right,B_left)
                        typename CurveType::gt_type::value_type zab_l = CurveType::gt_type::value_type::one();
                        std::for_each(boost::make_zip_iterator(boost::make_tuple(m_a.begin() + split, m_b.begin())),
                                      boost::make_zip_iterator(boost::make_tuple(m_a.end(), m_b.begin() + split)),
                                      [&](const boost::tuple<const typename CurveType::template g1_type<>::value_type &,
                                                             const typename CurveType::template g2_type<>::value_type &> &t) {
                                          zab_l = zab_l *
                                                  algebra::pair<CurveType>(t.template get<0>(), t.template get<1>());
                                      });
                        zab_l = algebra::final_exponentiation<CurveType>(zab_l);
                        typename CurveType::gt_type::value_type zab_r = CurveType::gt_type::value_type::one();
                        std::for_each(boost::make_zip_iterator(boost::make_tuple(m_a.begin(), m_b.begin() + split)),
                                      boost::make_zip_iterator(boost::make_tuple(m_a.begin() + split, m_b.end())),
                                      [&](const boost::tuple<const typename CurveType::template g1_type<>::value_type &,
                                                             const typename CurveType::template g2_type<>::value_type &> &t) {
                                          zab_r = zab_r *
                                                  algebra::pair<CurveType>(t.template get<0>(), t.template get<1>());
                                      });
                        zab_r = algebra::final_exponentiation<CurveType>(zab_r);

                        // MIPP part
                        // z_l = c[n':] ^ r[:n']
                        typename CurveType::template g1_type<>::value_type zc_l =
                            algebra::multiexp<algebra::policies::multiexp_method_bos_coster>(
                                m_c.begin() + split, m_c.end(), m_r.begin(), m_r.begin() + split, 1);
                        // Z_r = c[:n'] ^ r[n':]
                        typename CurveType::template g1_type<>::value_type zc_r =
                            algebra::multiexp<algebra::policies::multiexp_method_bos_coster>(
                                m_c.begin(), m_c.begin() + split, m_r.begin() + split, m_r.end(), 1);
                        // u_l = c[n':] * v[:n']
                        typename r1cs_gg_ppzksnark_ipp2_commitment<CurveType>::output_type tuc_l =
                            r1cs_gg_ppzksnark_ipp2_commitment<CurveType>::single(vk_left, m_c.begin() + split,
                                                                                m_c.end());
                        // u_r = c[:n'] * v[n':]
                        typename r1cs_gg_ppzksnark_ipp2_commitment<CurveType>::output_type tuc_r =
                            r1cs_gg_ppzksnark_ipp2_commitment<CurveType>::single(vk_right, m_c.begin(),
                                                                                m_c.begin() + split);

                        // Fiat-Shamir challenge
                        // combine both TIPP and MIPP transcript
                        tr.template write<typename CurveType::gt_type>(zab_l);
                        tr.template write<typename CurveType::gt_type>(zab_r);
                        tr.template write<typename CurveType::template g1_type<>>(zc_l);
                        tr.template write<typename CurveType::template g1_type<>>(zc_r);
                        tr.template write<typename CurveType::gt_type>(tab_l.first);
                        tr.template write<typename CurveType::gt_type>(tab_l.second);
                        tr.template write<typename CurveType::gt_type>(tab_r.first);
                        tr.template write<typename CurveType::gt_type>(tab_r.second);
                        tr.template write<typename CurveType::gt_type>(tuc_l.first);
                        tr.template write<typename CurveType::gt_type>(tuc_l.second);
                        tr.template write<typename CurveType::gt_type>(tuc_r.first);
                        tr.template write<typename CurveType::gt_type>(tuc_r.second);
                        typename CurveType::scalar_field_type::value_type c_inv = tr.read_challenge();

                        // Optimization for multiexponentiation to rescale G2 elements with
                        // 128-bit challenge Swap 'c' and 'c_inv' since can't control bit size
                        // of c_inv
                        typename CurveType::scalar_field_type::value_type c = c_inv.inversed();

                        // Set up values for next step of recursion
                        // A[:n'] + A[n':] ^ x
                        compress<CurveType>(m_a, split, c);
                        // B[:n'] + B[n':] ^ x^-1
                        compress<CurveType>(m_b, split, c_inv);
                        // c[:n'] + c[n':]^x
                        compress<CurveType>(m_c, split, c);
                        // r[:n'] + r[n':]^x^-1
                        compress<CurveType>(m_r, split, c_inv);

                        // v_left + v_right^x^-1
                        vkey = vk_left.compress(vk_right, c_inv);
                        // w_left + w_right^x
                        wkey = wk_left.compress(wk_right, c);

                        comms_ab.emplace_back(std::make_pair(tab_l, tab_r));
                        comms_c.emplace_back(std::make_pair(tuc_l, tuc_r));
                        z_ab.emplace_back(std::make_pair(zab_l, zab_r));
                        z_c.emplace_back(std::make_pair(zc_l, zc_r));
                        challenges.emplace_back(c);
                        challenges_inv.emplace_back(c_inv);
                    }

                    BOOST_ASSERT(m_a.size() == 1 && m_b.size() == 1);
                    BOOST_ASSERT(m_c.size() == 1 && m_r.size() == 1);
                    BOOST_ASSERT(vkey.a.size() == 1 && vkey.b.size() == 1);
                    BOOST_ASSERT(wkey.a.size() == 1 && wkey.b.size() == 1);

                    return std::make_tuple(gipa_proof<CurveType> {input_len, comms_ab, comms_c, z_ab, z_c, m_a[0],
                                                                  m_b[0], m_c[0], vkey.first(), wkey.first()},
                                           challenges, challenges_inv);
                }

                /// Proves a TIPP relation between A and B as well as a MIPP relation with C and
                /// r. Commitment keys must be of size of A, B and C. In the context of Groth16
                /// aggregation, we have that B = B^r and wkey is scaled by r^{-1}. The
                /// commitment key v is used to commit to A and C recursively in GIPA such that
                /// only one KZG proof is needed for v. In the original paper version, since the
                /// challenges of GIPA would be different, two KZG proofs would be needed.
                template<typename CurveType, typename Hash = hashes::sha2<256>, typename InputG1Iterator1,
                         typename InputG2Iterator, typename InputG1Iterator2, typename InputScalarIterator>
                typename std::enable_if<
                    std::is_same<typename CurveType::template g1_type<>::value_type,
                                 typename std::iterator_traits<InputG1Iterator1>::value_type>::value &&
                        std::is_same<typename CurveType::template g2_type<>::value_type,
                                     typename std::iterator_traits<InputG2Iterator>::value_type>::value &&
                        std::is_same<typename CurveType::template g1_type<>::value_type,
                                     typename std::iterator_traits<InputG1Iterator2>::value_type>::value &&
                        std::is_same<typename CurveType::scalar_field_type::value_type,
                                     typename std::iterator_traits<InputScalarIterator>::value_type>::value,
                    tipp_mipp_proof<CurveType>>::type
                    prove_tipp_mipp(const r1cs_gg_ppzksnark_aggregate_proving_srs<CurveType> &srs,
                                    transcript<CurveType, Hash> &tr, InputG1Iterator1 a_first, InputG1Iterator1 a_last,
                                    InputG2Iterator b_first, InputG2Iterator b_last, InputG1Iterator2 c_first,
                                    InputG1Iterator2 c_last, const r1cs_gg_ppzksnark_ipp2_wkey<CurveType> &wkey,
                                    InputScalarIterator r_first, InputScalarIterator r_last) {
                    typename CurveType::scalar_field_type::value_type r_shift = *(r_first + 1);
                    // Run GIPA
                    auto [proof, challenges, challenges_inv] = gipa_tipp_mipp<CurveType>(
                        tr, a_first, a_last, b_first, b_last, c_first, c_last, srs.vkey, wkey, r_first, r_last);

                    // Prove final commitment keys are wellformed
                    // we reverse the transcript so the polynomial in kzg opening is constructed
                    // correctly - the formula indicates x_{l-j}. Also for deriving KZG
                    // challenge point, input must be the last challenge.
                    std::reverse(challenges.begin(), challenges.end());
                    std::reverse(challenges_inv.begin(), challenges_inv.end());
                    typename CurveType::scalar_field_type::value_type r_inverse = r_shift.inversed();

                    // KZG challenge point
                    constexpr std::array<std::uint8_t, 8> domain_separator {'r', 'a', 'n', 'd', 'o', 'm', '-', 'z'};
                    tr.write_domain_separator(domain_separator.begin(), domain_separator.end());
                    tr.template write<typename CurveType::scalar_field_type>(challenges[0]);
                    tr.template write<typename CurveType::template g2_type<>>(proof.final_vkey.first);
                    tr.template write<typename CurveType::template g2_type<>>(proof.final_vkey.second);
                    tr.template write<typename CurveType::template g1_type<>>(proof.final_wkey.first);
                    tr.template write<typename CurveType::template g1_type<>>(proof.final_wkey.second);
                    typename CurveType::scalar_field_type::value_type z = tr.read_challenge();

                    // Complete KZG proofs
                    return tipp_mipp_proof<CurveType> {
                        proof,
                        prove_commitment_v<CurveType>(srs.h_alpha_powers.begin(), srs.h_alpha_powers.end(),
                                                      srs.h_beta_powers.begin(), srs.h_beta_powers.end(),
                                                      challenges_inv.begin(), challenges_inv.end(), z),
                        prove_commitment_w<CurveType>(srs.g_alpha_powers.begin(), srs.g_alpha_powers.end(),
                                                      srs.g_beta_powers.begin(), srs.g_beta_powers.end(),
                                                      challenges.begin(), challenges.end(), r_inverse, z)};
                }

                /// Aggregate `n` zkSnark proofs, where `n` must be a power of two.
                template<typename CurveType, typename Hash = hashes::sha2<256>, typename InputTranscriptIncludeIterator,
                         typename InputProofIterator>
                typename std::enable_if<
                    std::is_same<std::uint8_t,
                                 typename std::iterator_traits<InputTranscriptIncludeIterator>::value_type>::value &&
                        std::is_same<typename std::iterator_traits<InputProofIterator>::value_type,
                                     r1cs_gg_ppzksnark_proof<CurveType>>::value,
                    r1cs_gg_ppzksnark_aggregate_proof<CurveType>>::type
                    aggregate_proofs(const r1cs_gg_ppzksnark_aggregate_proving_srs<CurveType> &srs,
                                     InputTranscriptIncludeIterator tr_include_first,
                                     InputTranscriptIncludeIterator tr_include_last, InputProofIterator proofs_first,
                                     InputProofIterator proofs_last) {
                    std::size_t nproofs = std::distance(proofs_first, proofs_last);
                    BOOST_ASSERT(nproofs >= 2);
                    BOOST_ASSERT((nproofs & (nproofs - 1)) == 0);
                    BOOST_ASSERT(srs.has_correct_len(nproofs));

                    // TODO: parallel
                    // We first commit to A B and C - these commitments are what the verifier
                    // will use later to verify the TIPP and MIPP proofs
                    std::vector<typename CurveType::template g1_type<>::value_type> a, c;
                    std::vector<typename CurveType::template g2_type<>::value_type> b;
                    auto proofs_it = proofs_first;
                    while (proofs_it != proofs_last) {
                        a.emplace_back(proofs_it->g_A);
                        b.emplace_back(proofs_it->g_B);
                        c.emplace_back(proofs_it->g_C);
                        ++proofs_it;
                    }

                    // A and B are committed together in this scheme
                    // we need to take the reference so the macro doesn't consume the value
                    // first
                    typename r1cs_gg_ppzksnark_ipp2_commitment<CurveType>::output_type com_ab =
                        r1cs_gg_ppzksnark_ipp2_commitment<CurveType>::pair(srs.vkey, srs.wkey, a.begin(), a.end(),
                                                                           b.begin(), b.end());
                    typename r1cs_gg_ppzksnark_ipp2_commitment<CurveType>::output_type com_c =
                        r1cs_gg_ppzksnark_ipp2_commitment<CurveType>::single(srs.vkey, c.begin(), c.end());

                    // Derive a random scalar to perform a linear combination of proofs
                    constexpr std::array<std::uint8_t, 9> application_tag = {'s', 'n', 'a', 'r', 'k',
                                                                             'p', 'a', 'c', 'k'};
                    constexpr std::array<std::uint8_t, 8> domain_separator {'r', 'a', 'n', 'd', 'o', 'm', '-', 'r'};
                    transcript<CurveType, Hash> tr(application_tag.begin(), application_tag.end());
                    tr.write_domain_separator(domain_separator.begin(), domain_separator.end());
                    tr.template write<typename CurveType::gt_type>(com_ab.first);
                    tr.template write<typename CurveType::gt_type>(com_ab.second);
                    tr.template write<typename CurveType::gt_type>(com_c.first);
                    tr.template write<typename CurveType::gt_type>(com_c.second);
                    tr.write(tr_include_first, tr_include_last);
                    typename CurveType::scalar_field_type::value_type r = tr.read_challenge();

                    // 1,r, r^2, r^3, r^4 ...
                    std::vector<typename CurveType::scalar_field_type::value_type> r_vec =
                        structured_scalar_power<typename CurveType::scalar_field_type>(
                            std::distance(proofs_first, proofs_last), r);
                    // 1,r^-1, r^-2, r^-3
                    std::vector<typename CurveType::scalar_field_type::value_type> r_inv;
                    std::transform(r_vec.begin(), r_vec.end(), std::back_inserter(r_inv),
                                   [](const auto &r_i) { return r_i.inversed(); });

                    // B^{r}
                    std::vector<typename CurveType::template g2_type<>::value_type> b_r;
                    std::for_each(
                        boost::make_zip_iterator(boost::make_tuple(b.begin(), r_vec.begin())),
                        boost::make_zip_iterator(boost::make_tuple(b.end(), r_vec.end())),
                        [&](const boost::tuple<const typename CurveType::template g2_type<>::value_type &,
                                               const typename CurveType::scalar_field_type::value_type &> &t) {
                            b_r.emplace_back((t.template get<0>() * t.template get<1>()));
                        });
                    // TODO: parallel
                    // compute A * B^r for the verifier
                    // auto ip_ab = algebra::pair<CurveType>(a, b_r);
                    typename CurveType::gt_type::value_type ip_ab = CurveType::gt_type::value_type::one();
                    std::for_each(boost::make_zip_iterator(boost::make_tuple(a.begin(), b_r.begin())),
                                  boost::make_zip_iterator(boost::make_tuple(a.end(), b_r.end())),
                                  [&](const boost::tuple<const typename CurveType::template g1_type<>::value_type &,
                                                         const typename CurveType::template g2_type<>::value_type &> &t) {
                                      ip_ab =
                                          ip_ab * algebra::pair<CurveType>(t.template get<0>(), t.template get<1>());
                                  });
                    ip_ab = algebra::final_exponentiation<CurveType>(ip_ab);
                    // compute C^r for the verifier
                    typename CurveType::template g1_type<>::value_type agg_c =
                        algebra::multiexp<algebra::policies::multiexp_method_bos_coster>(c.begin(), c.end(),
                                                                                r_vec.begin(), r_vec.end(), 1);
                    tr.template write<typename CurveType::gt_type>(ip_ab);
                    tr.template write<typename CurveType::template g1_type<>>(agg_c);

                    // w^{r^{-1}}
                    r1cs_gg_ppzksnark_ipp2_commitment_key<typename CurveType::template g1_type<>> wkey_r_inv =
                        srs.wkey.scale(r_inv.begin(), r_inv.end());

                    // we prove tipp and mipp using the same recursive loop
                    tipp_mipp_proof<CurveType> proof =
                        prove_tipp_mipp(srs, tr, a.begin(), a.end(), b_r.begin(), b_r.end(), c.begin(), c.end(),
                                        wkey_r_inv, r_vec.begin(), r_vec.end());

                    // debug assert
                    BOOST_ASSERT(com_ab == r1cs_gg_ppzksnark_ipp2_commitment<CurveType>::pair(
                                               srs.vkey, wkey_r_inv, a.begin(), a.end(), b_r.begin(), b_r.end()));

                    return {com_ab, com_c, ip_ab, agg_c, proof};
                }

                template<typename CurveType, typename BasicProver>
                class r1cs_gg_ppzksnark_aggregate_prover {
                    typedef detail::r1cs_gg_ppzksnark_basic_policy<CurveType, ProvingMode::Aggregate> policy_type;

                public:

                    typedef BasicProver basic_prover;

                    typedef typename policy_type::primary_input_type primary_input_type;
                    typedef typename policy_type::auxiliary_input_type auxiliary_input_type;
                    typedef typename policy_type::proving_key_type proving_key_type;
                    typedef typename policy_type::proving_srs_type proving_srs_type;
                    typedef typename policy_type::proof_type proof_type;
                    typedef typename policy_type::aggregate_proof_type aggregate_proof_type;

                    // Aggregate prove
                    template<typename Hash, typename InputTranscriptIncludeIterator, typename InputProofIterator>
                    static inline aggregate_proof_type process(const proving_srs_type &srs,
                                                               InputTranscriptIncludeIterator transcript_include_first,
                                                               InputTranscriptIncludeIterator transcript_include_last,
                                                               InputProofIterator proofs_first,
                                                               InputProofIterator proofs_last) {
                        return aggregate_proofs<CurveType, Hash>(srs, transcript_include_first, transcript_include_last,
                                                                 proofs_first, proofs_last);
                    }

                    // Basic prove
                    static inline proof_type process(const proving_key_type &pk,
                                                     const primary_input_type &primary_input,
                                                     const auxiliary_input_type &auxiliary_input) {

                        return BasicProver::process(pk, primary_input, auxiliary_input);
                    }
                };
            }    // namespace snark
        }        // namespace zk
    }            // namespace crypto3
}    // namespace nil

#endif    // CRYPTO3_R1CS_GG_PPZKSNARK_TYPES_POLICY_HPP


#include <chrono>
#include <iostream>
#include <memory>
#include <random>
// uncomment to disable assert()
// #define NDEBUG
#include <cassert>

#include <clipper/constants.hpp>
#include <clipper/containers.hpp>
#include <clipper/logging.hpp>
#include <clipper/metrics.hpp>
#include <clipper/util.hpp>

#include <boost/circular_buffer.hpp>
#include <boost/thread.hpp>

namespace clipper {

const std::string LOGGING_TAG_CONTAINERS = "CONTAINERS";

ModelContainer::ModelContainer(VersionedModelId model, int container_id,
                               int replica_id, InputType input_type)
    : model_(model),
      container_id_(container_id),
      replica_id_(replica_id),
      input_type_(input_type),
      avg_throughput_per_milli_(0),
      throughput_buffer_(THROUGHPUT_BUFFER_CAPACITY) {
  std::string model_str = versioned_model_to_str(model);
  log_info_formatted(LOGGING_TAG_CONTAINERS,
                     "Creating new ModelContainer for model {}, id: {}",
                     model_str, std::to_string(container_id));
}

void ModelContainer::update_throughput(size_t batch_size,
                                       long total_latency_micros) {
  if (batch_size <= 0 || total_latency_micros <= 0) {
    throw std::invalid_argument(
        "Batch size and latency must be positive for throughput updates!");
  }
  boost::unique_lock<boost::shared_mutex> lock(throughput_mutex_);
  double new_throughput = 1000 * (static_cast<double>(batch_size) /
                                  static_cast<double>(total_latency_micros));
  double old_total_throughput =
      avg_throughput_per_milli_ * throughput_buffer_.size();
  if (throughput_buffer_.size() == throughput_buffer_.capacity()) {
    // If the throughput buffer is already at maximum capacity,
    // we replace the oldest throughput sample with
    // the latest throughput and recalculate the average
    double oldest_throughput = throughput_buffer_.front();
    double new_total_throughput =
        (old_total_throughput - oldest_throughput + new_throughput);
    avg_throughput_per_milli_ =
        new_total_throughput / static_cast<double>(throughput_buffer_.size());
  } else {
    // If the throughput buffer is not yet at capacity,
    // we add the latest throughput sample to the buffer
    // and incorporate it into the average
    avg_throughput_per_milli_ =
        (old_total_throughput + new_throughput) /
        static_cast<double>(throughput_buffer_.size() + 1);
  }
  throughput_buffer_.push_back(new_throughput);
}

double ModelContainer::get_average_throughput_per_millisecond() {
  boost::shared_lock<boost::shared_mutex> lock(throughput_mutex_);
  return avg_throughput_per_milli_;
}

size_t ModelContainer::get_batch_size(Deadline deadline) {
  double current_time_millis =
      std::chrono::duration_cast<std::chrono::milliseconds>(
          std::chrono::system_clock::now().time_since_epoch())
          .count();
  double deadline_millis =
      std::chrono::duration_cast<std::chrono::milliseconds>(
          deadline.time_since_epoch())
          .count();
  double remaining_time_millis = deadline_millis - current_time_millis;
  boost::shared_lock<boost::shared_mutex> lock(throughput_mutex_);
  int batch_size =
      static_cast<int>(avg_throughput_per_milli_ * remaining_time_millis);
  if (batch_size < 1) {
    batch_size = 1;
  }
  return batch_size;
}

ActiveContainers::ActiveContainers()
    : containers_(
          std::unordered_map<VersionedModelId,
                             std::map<int, std::shared_ptr<ModelContainer>>,
                             decltype(&versioned_model_hash)>(
              100, &versioned_model_hash)) {}

void ActiveContainers::add_container(VersionedModelId model, int connection_id,
                                     int replica_id, InputType input_type) {
  log_info_formatted(LOGGING_TAG_CONTAINERS,
                     "Adding new container - model: {}, version: {}, "
                     "connection ID: {}, replica ID: {}, input_type: {}",
                     model.first, model.second, connection_id, replica_id,
                     get_readable_input_type(input_type));
  boost::unique_lock<boost::shared_mutex> l{m_};
  auto new_container = std::make_shared<ModelContainer>(model, connection_id,
                                                        replica_id, input_type);
  auto entry = containers_[new_container->model_];
  entry.emplace(replica_id, new_container);
  containers_[new_container->model_] = entry;
  assert(containers_[new_container->model_].size() > 0);
  std::stringstream log_msg;
  log_msg << "\nActive containers:\n";
  for (auto model : containers_) {
    log_msg << "\tModel: " << versioned_model_to_str(model.first) << "\n";
    for (auto r : model.second) {
      log_msg << "\t\trep_id: " << r.first
              << ", container_id: " << r.second->container_id_ << "\n";
    }
  }
  log_info(LOGGING_TAG_CONTAINERS, log_msg.str());
}

std::shared_ptr<ModelContainer> ActiveContainers::get_model_replica(
    const VersionedModelId &model, const int replica_id) {
  boost::shared_lock<boost::shared_mutex> l{m_};

  auto replicas_map_entry = containers_.find(model);
  if (replicas_map_entry == containers_.end()) {
    log_error_formatted(LOGGING_TAG_CONTAINERS,
                        "Requested replica {} for model {} NOT FOUND",
                        replica_id, versioned_model_to_str(model));
    return nullptr;
  }

  std::map<int, std::shared_ptr<ModelContainer>> replicas_map =
      replicas_map_entry->second;
  auto replica_entry = replicas_map.find(replica_id);
  if (replica_entry != replicas_map.end()) {
    return replica_entry->second;
  } else {
    log_error_formatted(LOGGING_TAG_CONTAINERS,
                        "Requested replica {} for model {} NOT FOUND",
                        replica_id, versioned_model_to_str(model));
    return nullptr;
  }
}

std::vector<VersionedModelId> ActiveContainers::get_known_models() {
  boost::shared_lock<boost::shared_mutex> l{m_};
  std::vector<VersionedModelId> keys;
  for (auto m : containers_) {
    keys.push_back(m.first);
  }
  return keys;
}
}


// Copyright (c) 2013 Spotify AB
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not
// use this file except in compliance with the License. You may obtain a copy of
// the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
// License for the specific language governing permissions and limitations under
// the License.

#include "Python.h"
#include <stdio.h>
#include <string>
#include <boost/python.hpp>
#include <sys/stat.h>
#include <unistd.h>
#include <stdio.h>
#include <stdlib.h>
#include <sys/types.h>
#include <fcntl.h>
#include <sys/mman.h>
#include <string.h>
#include <math.h>
#include <vector>
#include <queue>
#include <limits>
#include <boost/random.hpp>
#include <boost/random/normal_distribution.hpp>
#include <boost/random/uniform_01.hpp>
#include <boost/random/bernoulli_distribution.hpp>

#ifndef NO_PACKED_STRUCTS
#define PACKED_STRUCTS_EXTRA __attribute__((__packed__))
// TODO: this is turned on by default, but may not work for all architectures! Need to investigate.
#endif

using namespace std;
using namespace boost;

template<typename T>
struct Randomness {
  // Just a dummy class to avoid code repetition.
  // Owned by the AnnoyIndex, passed around to the distance metrics
  mt19937 _rng;
  normal_distribution<T> _nd;
  variate_generator<mt19937&,
		    normal_distribution<T> > _var_nor;
  uniform_01<T> _ud;
  variate_generator<mt19937&,
		    uniform_01<T> > _var_uni;
  bernoulli_distribution<T> _bd;
  variate_generator<mt19937&,
		    bernoulli_distribution<T> > _var_ber;
  Randomness() : _rng(), _nd(), _var_nor(_rng, _nd), _ud(), _var_uni(_rng, _ud), _bd(), _var_ber(_rng, _bd) {}
  inline T gaussian() {
    return _var_nor();
  }
  inline int flip() {
    return _var_ber();
  }
  inline T uniform(T min, T max) {
    return _var_uni() * (max - min) + min;
  }
};

template<typename T>
inline void normalize(T* v, int f) {
  T sq_norm = 0;
  for (int z = 0; z < f; z++)
    sq_norm += v[z] * v[z];
  T norm = sqrt(sq_norm);
  for (int z = 0; z < f; z++)
    v[z] /= norm;
}

template<typename T>
struct Angular {
  struct PACKED_STRUCTS_EXTRA node {
    /*
     * We store a binary tree where each node has two things
     * - A vector associated with it
     * - Two children
     * All nodes occupy the same amount of memory
     * All nodes with n_descendants == 1 are leaf nodes.
     * A memory optimization is that for nodes with 2 <= n_descendants <= K,
     * we skip the vector. Instead we store a list of all descendants. K is
     * determined by the number of items that fits in the same space.
     * For nodes with n_descendants == 1 or > K, there is always a
     * corresponding vector.
     * Note that we can't really do sizeof(node<T>) because we cheat and allocate
     * more memory to be able to fit the vector outside
     */
    int n_descendants;
    int children[2]; // Will possibly store more than 2
    T v[0]; // Hack. We just allocate as much memory as we need and let this array overflow
  };
  static inline T distance(const T* x, const T* y, int f) {
    // want to calculate (a/|a| - b/|b|)^2
    // = a^2 / a^2 + b^2 / b^2 - 2ab/|a||b|
    // = 2 - 2cos
    T pp = 0, qq = 0, pq = 0;
    for (int z = 0; z < f; z++) {
      pp += x[z] * x[z];
      qq += y[z] * y[z];
      pq += x[z] * y[z];
    }
    T ppqq = pp * qq;
    if (ppqq > 0) return 2.0 - 2.0 * pq / sqrt(ppqq);
    else return 2.0; // cos is 0
  }
  static inline T margin(const node* n, const T* y, int f) {
    T dot = 0;
    for (int z = 0; z < f; z++)
      dot += n->v[z] * y[z];
    return dot;
  }
  static inline bool side(const node* n, const T* y, int f, Randomness<T>* random) {
    T dot = margin(n, y, f);
    if (dot != 0)
      return (dot > 0);
    else
      return random->flip();
  }
  static inline void create_split(const vector<node*>& nodes, int f, Randomness<T>* random, node* n) {
    for (int z = 0; z < f; z++)
      n->v[z] = random->gaussian();
    normalize(n->v, f);
  }
};

template<typename T>
struct Euclidean {
  struct __attribute__((__packed__)) node {
    int n_descendants;
    T a; // need an extra constant term to determine the offset of the plane
    int children[2];
    T v[0];
  };
  static inline T distance(const T* x, const T* y, int f) {
    T d = 0.0;
    for (int i = 0; i < f; i++)
      d += (x[i] - y[i]) * (x[i] - y[i]);
    return d;
  }
  static inline T margin(const node* n, const T* y, int f) {
    T dot = n->a;
    for (int z = 0; z < f; z++)
      dot += n->v[z] * y[z];
    return dot;
  }
  static inline bool side(const node* n, const T* y, int f, Randomness<T>* random) {
    T dot = margin(n, y, f);
    if (dot != 0)
      return (dot > 0);
    else
      return random->flip();
  }
  static inline void create_split(const vector<node*>& nodes, int f, Randomness<T>* random, node* n) {
    // See http://en.wikipedia.org/wiki/Bertrand_paradox_(probability)
    // We want to sample a random hyperplane out of all hyperplanes that cut the convex hull.
    // The probability of each angle is in proportion to the extent of the projection.
    // This is good because it means we try to split in the longest direction.
    // Doing this using Metropolis-Hastings sampling using 10 steps
    T* v = (T*)malloc(sizeof(T) * f); // TODO: would be really nice to get rid of this allocation
    double max_proj = 0.0;
    for (int step = 0; step < 10; step++) {
      for (int z = 0; z < f; z++)
	v[z] = random->gaussian();
      normalize(v, f);
      // Project the nodes onto the vector and calculate max and min
      T min = INFINITY, max = -INFINITY;
      for (size_t i = 0; i < nodes.size(); i++) {
	T dot = 0;
	for (int z = 0; z < f; z++)
	  dot += nodes[i]->v[z] * v[z];
	if (dot > max)
	  max = dot;
	if (dot < min)
	  min = dot;
      }
      if (max - min > random->uniform(0, max_proj)) {
	max_proj = max - min;
	memcpy(n->v, v, sizeof(T) * f);
	n->a = -random->uniform(min, max); // Take a random split along this axis
      }
    }
    free(v);
  }
};

template<typename T, typename Distance>
class AnnoyIndex {
  /*
   * We use random projection to build a forest of binary trees of all items.
   * Basically just split the hyperspace into two sides by a hyperplane,
   * then recursively split each of those subtrees etc.
   * We create a tree like this q times. The default q is determined automatically
   * in such a way that we at most use 2x as much memory as the vectors take.
   */
protected:
  int _f;
  size_t _s;
  int _n_items;
  Randomness<T> _random;
  void* _nodes; // Could either be mmapped, or point to a memory buffer that we reallocate
  int _n_nodes;
  int _nodes_size;
  vector<int> _roots;
  int _K;
  bool _loaded;
public:
  AnnoyIndex(int f) : _random() {
    _f = f;
    _s = sizeof(typename Distance::node) + sizeof(T) * f; // Size of each node
    _n_items = 0;
    _n_nodes = 0;
    _nodes_size = 0;
    _nodes = NULL;
    _loaded = false;

    _K = (sizeof(T) * f + sizeof(int) * 2) / sizeof(int);
  }
  ~AnnoyIndex() {
    if (_loaded) {
      unload();
    } else if(_nodes) {
      free(_nodes);
    }
  }

  void add_item(int item, const T* w) {
    _allocate_size(item+1);
    typename Distance::node* n = _get(item);

    n->children[0] = 0;
    n->children[1] = 0;
    n->n_descendants = 1;

    for (int z = 0; z < _f; z++)
      n->v[z] = w[z];

    if (item >= _n_items)
      _n_items = item + 1;
  }

  void build(int q) {
    _n_nodes = _n_items;
    while (1) {
      if (q == -1 && _n_nodes >= _n_items * 2)
	break;
      if (q != -1 && _roots.size() >= (size_t)q)
	break;
      fprintf(stderr, "pass %zd...\n", _roots.size());

      vector<int> indices;
      for (int i = 0; i < _n_items; i++)
	indices.push_back(i);

      _roots.push_back(_make_tree(indices));
    }
    // Also, copy the roots into the last segment of the array
    // This way we can load them faster without reading the whole file
    _allocate_size(_n_nodes + _roots.size());
    for (size_t i = 0; i < _roots.size(); i++)
      memcpy(_get(_n_nodes + i), _get(_roots[i]), _s);
    _n_nodes += _roots.size();

    fprintf(stderr, "has %d nodes\n", _n_nodes);
  }

  void save(const string& filename) {
    FILE *f = fopen(filename.c_str(), "w");

    fwrite(_nodes, _s, _n_nodes, f);

    fclose(f);

    free(_nodes);
    _n_items = 0;
    _n_nodes = 0;
    _nodes_size = 0;
    _nodes = NULL;
    _roots.clear();
    load(filename);
  }

  void reinitialize() {
    _nodes = NULL;
    _loaded = false;
    _n_items = 0;
    _n_nodes = 0;
    _nodes_size = 0;
    _roots.clear();
  }

  void unload() {
    off_t size = _n_nodes * _s;
    munmap(_nodes, size);
    reinitialize();
    fprintf(stderr, "unloaded\n");
  }

  void load(const string& filename) {
    struct stat buf;
    stat(filename.c_str(), &buf);
    off_t size = buf.st_size;
    int fd = open(filename.c_str(), O_RDONLY, (mode_t)0400);
#ifdef MAP_POPULATE
    _nodes = (typename Distance::node*)mmap(
        0, size, PROT_READ, MAP_SHARED | MAP_POPULATE, fd, 0);
#else
    _nodes = (typename Distance::node*)mmap(
        0, size, PROT_READ, MAP_SHARED, fd, 0);
#endif

    _n_nodes = size / _s;

    // Find the roots by scanning the end of the file and taking the nodes with most descendants
    int m = -1;
    for (int i = _n_nodes - 1; i >= 0; i--) {
      int k = _get(i)->n_descendants;
      if (m == -1 || k == m) {
	_roots.push_back(i);
	m = k;
      } else {
	break;
      }
    }
    _loaded = true;
    _n_items = m;
    fprintf(stderr, "found %lu roots with degree %d\n", _roots.size(), m);
  }

  inline T get_distance(int i, int j) {
    const T* x = _get(i)->v;
    const T* y = _get(j)->v;
    return Distance::distance(x, y, _f);
  }

  void get_nns_by_item(int item, size_t n, vector<int>* result) {
    const typename Distance::node* m = _get(item);
    _get_all_nns(m->v, n, result);
  }

  void get_nns_by_vector(const T* w, size_t n, vector<int>* result) {
    _get_all_nns(w, n, result);
  }
  int get_n_items() {
    return _n_items;
  }
protected:
  void _allocate_size(int n) {
    if (n > _nodes_size) {
      int new_nodes_size = (_nodes_size + 1) * 2;
      if (n > new_nodes_size)
	new_nodes_size = n;
      _nodes = realloc(_nodes, _s * new_nodes_size);
      memset((char *)_nodes + (_nodes_size * _s)/sizeof(char), 0, (new_nodes_size - _nodes_size) * _s);
      _nodes_size = new_nodes_size;
    }
  }

  inline typename Distance::node* _get(int i) {
    return (typename Distance::node*)((char *)_nodes + (_s * i)/sizeof(char));
  }

  int _make_tree(const vector<int >& indices) {
    if (indices.size() == 1)
      return indices[0];

    _allocate_size(_n_nodes + 1);
    int item = _n_nodes++;
    typename Distance::node* m = _get(item);
    m->n_descendants = indices.size();

    if (indices.size() <= (size_t)_K) {
      for (size_t i = 0; i < indices.size(); i++)
	m->children[i] = indices[i];
      return item;
    }

    vector<int> children_indices[2];
    for (int attempt = 0; attempt < 20; attempt ++) {
      /*
       * Create a random hyperplane.
       * If all points end up on the same time, we try again.
       * We could in principle *construct* a plane so that we split
       * all items evenly, but I think that could violate the guarantees
       * given by just picking a hyperplane at random
       */
      vector<typename Distance::node*> children;

      for (size_t i = 0; i < indices.size(); i++) {
	// TODO: this loop isn't needed for the angular distance, because
	// we can just split by a random vector and it's fine. For Euclidean
	// distance we need it to calculate the offset
	int j = indices[i];
	typename Distance::node* n = _get(j);
	if (n)
	  children.push_back(n);
      }

      Distance::create_split(children, _f, &_random, m);

      children_indices[0].clear();
      children_indices[1].clear();

      for (size_t i = 0; i < indices.size(); i++) {
	int j = indices[i];
	typename Distance::node* n = _get(j);
	if (n) {
	  bool side = Distance::side(m, n->v, _f, &_random);
	  children_indices[side].push_back(j);
	}
      }

      if (children_indices[0].size() > 0 && children_indices[1].size() > 0) {
	break;
      }
    }

    while (children_indices[0].size() == 0 || children_indices[1].size() == 0) {
      // If we didn't find a hyperplane, just randomize sides as a last option
      if (indices.size() > 100000)
	fprintf(stderr, "Failed splitting %lu items\n", indices.size());

      children_indices[0].clear();
      children_indices[1].clear();

      // Set the vector to 0.0
      for (int z = 0; z < _f; z++)
	m->v[z] = 0.0;

      for (size_t i = 0; i < indices.size(); i++) {
	int j = indices[i];
	// Just randomize...
	children_indices[_random.flip()].push_back(j);
      }
    }

    int children_0 = _make_tree(children_indices[0]);
    int children_1 = _make_tree(children_indices[1]);

    // We need to fetch m again because it might have been reallocated
    m = _get(item);
    m->children[0] = children_0;
    m->children[1] = children_1;

    return item;
  }

  void _get_nns(const T* v, int i, vector<int>* result, int limit) {
    const typename Distance::node* n = _get(i);

    if (n->n_descendants == 0) {
      // unknown item, nothing to do...
    } else if (n->n_descendants == 1) {
      result->push_back(i);
    } else if (n->n_descendants <= _K) {
      for (int j = 0; j < n->n_descendants; j++) {
	result->push_back(n->children[j]);
      }
    } else {
      bool side = Distance::side(n, v, _f, &_random);

      _get_nns(v, n->children[side], result, limit);
      if (result->size() < (size_t)limit)
	_get_nns(v, n->children[!side], result, limit);
    }
  }

  void _get_all_nns(const T* v, size_t n, vector<int>* result) {
    std::priority_queue<pair<T, int> > q;

    for (size_t i = 0; i < _roots.size(); i++) {
      q.push(make_pair(numeric_limits<T>::infinity(), _roots[i]));
    }

    vector<int> nns;
    while (nns.size() < n * _roots.size() && !q.empty()) {
      const pair<T, int>& top = q.top();
      int i = top.second;
      const typename Distance::node* n = _get(top.second);
      q.pop();
      if (n->n_descendants == 1) {
	nns.push_back(i);
      } else if (n->n_descendants <= _K) {
	for (int x = 0; x < n->n_descendants; x++) {
	  int j = n->children[x];
	  nns.push_back(j);
	}
      } else{
	T margin = Distance::margin(n, v, _f);
	q.push(make_pair(+margin, n->children[1]));
	q.push(make_pair(-margin, n->children[0]));
      }
    }

    sort(nns.begin(), nns.end());
    vector<pair<T, int> > nns_dist;
    int last = -1;
    for (size_t i = 0; i < nns.size(); i++) {
      int j = nns[i];
      if (j == last)
	continue;
      last = j;
      nns_dist.push_back(make_pair(Distance::distance(v, _get(j)->v, _f), j));
    }

    sort(nns_dist.begin(), nns_dist.end());
    for (size_t i = 0; i < nns_dist.size() && result->size() < n; i++) {
      result->push_back(nns_dist[i].second);
    }
  }
};

template<typename T, typename Distance>
class AnnoyIndexPython : public AnnoyIndex<T, Distance > {
public:
  AnnoyIndexPython(int f): AnnoyIndex<T, Distance>(f) {}
  void add_item_py(int item, const python::list& v) {
    vector<T> w;
    for (int z = 0; z < this->_f; z++)
      w.push_back(python::extract<T>(v[z]));

    this->add_item(item, &w[0]);
  }
  python::list get_nns_by_item_py(int item, size_t n) {
    vector<int> result;
    this->get_nns_by_item(item, n, &result);
    python::list l;
    for (size_t i = 0; i < result.size(); i++)
      l.append(result[i]);
    return l;
  }
  python::list get_nns_by_vector_py(python::list v, size_t n) {
    vector<T> w(this->_f);
    for (int z = 0; z < this->_f; z++)
      w[z] = python::extract<T>(v[z]);
    vector<int> result;
    this->get_nns_by_vector(&w[0], n, &result);
    python::list l;
    for (size_t i = 0; i < result.size(); i++)
      l.append(result[i]);
    return l;
  }
  python::list get_item_vector_py(int item) {
    const typename Distance::node* m = this->_get(item);
    const T* v = m->v;
    python::list l;
    for (int z = 0; z < this->_f; z++) {
      l.append(v[z]);
    }
    return l;
  }
};

template<typename C>
void expose_methods(python::class_<C> c) {
  c.def("add_item",          &C::add_item_py)
    .def("build",             &C::build)
    .def("save",              &C::save)
    .def("load",              &C::load)
    .def("unload",            &C::unload)
    .def("get_distance",      &C::get_distance)
    .def("get_nns_by_item",   &C::get_nns_by_item_py)
    .def("get_nns_by_vector", &C::get_nns_by_vector_py)
    .def("get_item_vector",   &C::get_item_vector_py)
    .def("get_n_items",       &C::get_n_items);
}

BOOST_PYTHON_MODULE(annoylib)
{
  expose_methods(python::class_<AnnoyIndexPython<float, Angular<float> > >("AnnoyIndexAngular", python::init<int>()));
  expose_methods(python::class_<AnnoyIndexPython<float, Euclidean<float> > >("AnnoyIndexEuclidean", python::init<int>()));
}


//==================================================================================================
/**
  Copyright 2016 NumScale SAS

  Distributed under the Boost Software License, Version 1.0.
  (See accompanying file LICENSE.md or copy at http://boost.org/LICENSE_1_0.txt)
**/
//==================================================================================================
#ifndef BOOST_SIMD_ARCH_COMMON_SIMD_FUNCTION_BITWISE_XOR_HPP_INCLUDED
#define BOOST_SIMD_ARCH_COMMON_SIMD_FUNCTION_BITWISE_XOR_HPP_INCLUDED

#include <boost/simd/function/bitwise_cast.hpp>
#include <boost/simd/detail/overload.hpp>
#include <boost/simd/detail/traits.hpp>
#include <boost/simd/detail/nsm.hpp>

namespace boost { namespace simd { namespace ext
{
  namespace bs = boost::simd;
  namespace bd = boost::dispatch;

  BOOST_DISPATCH_OVERLOAD_IF( bitwise_xor_
                            , (typename A0,typename A1,typename X, typename Y)
                            , (nsm::and_< nsm::not_ < std::is_same<A0,A1> >
                                            , nsm::and_ < detail::is_native<X>
                                                            , detail::is_native<Y>
                                                            >
                                            >
                              )
                            , bd::cpu_
                            , bs::pack_<bd::arithmetic_<A0>,X>
                            , bs::pack_<bd::arithmetic_<A1>,Y>
                            )
  {
    BOOST_FORCEINLINE A0 operator()(const A0& a0, const A1& a1) const BOOST_NOEXCEPT
    {
      return bitwise_xor(a0, bitwise_cast<A0>(a1));
    }
  };
} } }

#endif