<commit_before>// Copyright (c) Ilia Kravets, 2015. All rights reserved. PROVIDED "AS IS"
// WITHOUT ANY WARRANTY, EXPRESS OR IMPLIED. See LICENSE file for details.

package rec

import (
	"bytes"
	"encoding/binary"
	"fmt"
	"io"

	"github.com/ikravets/errs"

	"my/ev/packet"
	"my/ev/packet/bats"
	"my/ev/packet/miax"
	"my/ev/packet/nasdaq"
	"my/ev/sim"
)

type SimLoggerConfig struct {
	EfhLoggerConfig
	SupernodeLevels int
}

type SimLogger struct {
	w               io.Writer
	tobOld, tobNew  []sim.PriceLevel
	efhLogger       EfhLogger
	supernodeLevels int
}

const SimLoggerDefaultSupernodeLevels = 256
const SimLoggerUpperSupernodeLevels = 8

func NewSimLogger(c SimLoggerConfig) *SimLogger {
	s := &SimLogger{
		w:               c.Writer,
		supernodeLevels: c.SupernodeLevels,
	}
	if s.supernodeLevels == 0 {
		s.supernodeLevels = SimLoggerDefaultSupernodeLevels
	}
	elc := c.EfhLoggerConfig
	elc.Printer = s
	elc.Writer = nil
	s.efhLogger = *NewEfhLogger(elc)
	return s
}
func (s *SimLogger) SetOutputMode(mode EfhLoggerOutputMode) {
	s.efhLogger.SetOutputMode(mode)
}
func (s *SimLogger) SetSupernodeLevels(levels int) {
	errs.Check(levels > 0)
	s.supernodeLevels = levels
}

func (s *SimLogger) printf(format string, vs ...interface{}) {
	_, err := fmt.Fprintf(s.w, format, vs...)
	errs.CheckE(err)
}
func (s *SimLogger) printfln(format string, vs ...interface{}) {
	f := format + "\n"
	s.printf(f, vs...)
}
func (s *SimLogger) MessageArrived(idm *sim.SimMessage) {
	outItto := func(name string, typ nasdaq.IttoMessageType, f string, vs ...interface{}) {
		s.printf("NORM %s %c ", name, typ)
		s.printfln(f, vs...)
	}
	outBats := func(f string, vs ...interface{}) {
		s.printf("NORM ORDER %02x ", idm.Pam.Layer().(bats.PitchMessage).Base().Type.ToInt())
		s.printfln(f, vs...)
	}
	outMiax := func(f string, vs ...interface{}) {
		s.printf("NORM TOM %02x ", idm.Pam.Layer().(miax.TomMessage).Base().Type.ToInt())
		s.printfln(f, vs...)
	}
	sideChar := func(s packet.MarketSide) byte {
		if s == packet.MarketSideAsk {
			return 'S'
		}
		return byte(s)
	}
	switch im := idm.Pam.Layer().(type) {
	case *nasdaq.IttoMessageAddOrder:
		outItto("ORDER", im.Type, "%c %012x %016x %08x %08x", sideChar(im.Side), im.OId.ToUint64(), im.RefNumD.ToUint32(), im.Size, im.Price)
	case *nasdaq.IttoMessageAddQuote:
		outItto("QBID", im.Type, "%012x %016x %08x %08x", im.OId.ToUint64(), im.Bid.RefNumD.ToUint32(), im.Bid.Size, im.Bid.Price)
		outItto("QASK", im.Type, "%012x %016x %08x %08x", im.OId.ToUint64(), im.Ask.RefNumD.ToUint32(), im.Ask.Size, im.Ask.Price)
	case *nasdaq.IttoMessageSingleSideExecuted:
		outItto("ORDER", im.Type, "%016x %08x", im.OrigRefNumD.ToUint32(), im.Size)
	case *nasdaq.IttoMessageSingleSideExecutedWithPrice:
		outItto("ORDER", im.Type, "%016x %08x", im.OrigRefNumD.ToUint32(), im.Size)
	case *nasdaq.IttoMessageOrderCancel:
		outItto("ORDER", im.Type, "%016x %08x", im.OrigRefNumD.ToUint32(), im.Size)
	case *nasdaq.IttoMessageSingleSideReplace:
		outItto("ORDER", im.Type, "%016x %016x %08x %08x", im.RefNumD.ToUint32(), im.OrigRefNumD.ToUint32(), im.Size, im.Price)
	case *nasdaq.IttoMessageSingleSideDelete:
		outItto("ORDER", im.Type, "%016x", im.OrigRefNumD.ToUint32())
	case *nasdaq.IttoMessageSingleSideUpdate:
		outItto("ORDER", im.Type, "%016x %08x %08x", im.RefNumD.ToUint32(), im.Size, im.Price)
	case *nasdaq.IttoMessageQuoteReplace:
		outItto("QBID", im.Type, "%016x %016x %08x %08x", im.Bid.RefNumD.ToUint32(), im.Bid.OrigRefNumD.ToUint32(), im.Bid.Size, im.Bid.Price)
		outItto("QASK", im.Type, "%016x %016x %08x %08x", im.Ask.RefNumD.ToUint32(), im.Ask.OrigRefNumD.ToUint32(), im.Ask.Size, im.Ask.Price)
	case *nasdaq.IttoMessageQuoteDelete:
		outItto("QBID", im.Type, "%016x", im.BidOrigRefNumD.ToUint32())
		outItto("QASK", im.Type, "%016x", im.AskOrigRefNumD.ToUint32())
	case *nasdaq.IttoMessageBlockSingleSideDelete:
		for _, r := range im.RefNumDs {
			outItto("ORDER", im.Type, "%016x", r.ToUint32())
		}

	case *bats.PitchMessageAddOrder:
		outBats("%c %012x %016x %08x %08x", sideChar(im.Side), im.Symbol.ToUint64(), im.OrderId.ToUint64(), im.Size, packet.PriceTo4Dec(im.Price))
	case *bats.PitchMessageDeleteOrder:
		outBats("%016x", im.OrderId.ToUint64())
	case *bats.PitchMessageOrderExecuted:
		outBats("%016x %08x", im.OrderId.ToUint64(), im.Size)
	case *bats.PitchMessageOrderExecutedAtPriceSize:
		outBats("%016x %08x", im.OrderId.ToUint64(), im.Size)
	case *bats.PitchMessageReduceSize:
		outBats("%016x %08x", im.OrderId.ToUint64(), im.Size)
	case *bats.PitchMessageModifyOrder:
		outBats("%016x %08x %08x", im.OrderId.ToUint64(), im.Size, packet.PriceTo4Dec(im.Price))
	case *miax.TomMessageTom:
		outMiax("%c %08x %08x %08x %08x", sideChar(im.Side), im.ProductId.ToUint32(), packet.PriceTo4Dec(im.Price), im.Size, im.PriorityCustomerSize)
	}
	s.efhLogger.MessageArrived(idm)
}
func (s *SimLogger) OperationAppliedToOrders(operation sim.SimOperation) {
	type ordrespLogInfo struct {
		notFound, addOp   int
		orderId           packet.OrderId
		optionId          packet.OptionId
		side, price, size int
		ordlSuffix        string
	}
	type orduLogInfo struct {
		orderId           packet.OrderId
		optionId          packet.OptionId
		side, price, size int
	}

	var or ordrespLogInfo
	var ou orduLogInfo
	if _, ok := operation.(*sim.OperationTop); ok {
		return
	} else if op, ok := operation.(*sim.OperationAdd); ok {
		var oid packet.OptionId
		if op.Independent() {
			oid = op.GetOptionId()
		}
		or = ordrespLogInfo{
			addOp:      1,
			orderId:    op.OrderId,
			optionId:   oid,
			ordlSuffix: fmt.Sprintf(" %012x", oid.ToUint64()),
		}
		ou = orduLogInfo{
			orderId:  or.orderId,
			optionId: op.GetOptionId(),
			price:    op.GetPrice(),
			size:     op.GetNewSize(sim.SizeKindDefault),
		}
		if op.GetSide() == packet.MarketSideAsk {
			ou.side = 1
		}
	} else {
		if operation.GetOptionId().Invalid() {
			or = ordrespLogInfo{notFound: 1}
		} else {
			newSize := operation.GetNewSize(sim.SizeKindDefault)
			or = ordrespLogInfo{
				optionId: operation.GetOptionId(),
				price:    operation.GetPrice(),
				size:     newSize - operation.GetDefaultSizeDelta(),
			}
			if operation.GetSide() == packet.MarketSideAsk {
				or.side = 1
			}
			if newSize != 0 {
				ou = orduLogInfo{
					optionId: or.optionId,
					side:     or.side,
					price:    or.price,
					size:     newSize,
				}
			}
		}
		or.orderId = operation.GetOrigOrderId()
		ou.orderId = or.orderId
	}
	s.printfln("ORDL %d %016x%s", or.addOp, or.orderId.ToUint64(), or.ordlSuffix)
	s.printfln("ORDRESP %d %d %d %08x %08x %012x %016x", or.notFound, or.addOp, or.side, or.size, or.price, or.optionId.ToUint64(), or.orderId.ToUint64())
	if operation.GetOptionId().Valid() {
		s.printfln("ORDU %016x %012x %d %08x %08x", ou.orderId.ToUint64(), ou.optionId.ToUint64(), ou.side, ou.price, ou.size)
	}
}
func (s *SimLogger) BeforeBookUpdate(book sim.Book, operation sim.SimOperation) {
	tobOld := book.GetTop(operation.GetOptionId(), operation.GetSide(), s.supernodeLevels)
	s.tobOld = make([]sim.PriceLevel, len(tobOld))
	for i, pl := range tobOld {
		s.tobOld[i] = pl.Clone()
	}
	s.efhLogger.BeforeBookUpdate(book, operation)
}
func (s *SimLogger) AfterBookUpdate(book sim.Book, operation sim.SimOperation) {
	if operation.GetOptionId().Valid() && s.supernodeLevels > 1 {
		var emptyPrice uint32
		if operation.GetSide() == packet.MarketSideAsk {
			emptyPrice -= 1
		}
		printablePriceLevel := func(pls []sim.PriceLevel, pos int) (price uint32, size int) {
			if pos < len(pls) {
				price = uint32(pls[pos].Price())
				size = pls[pos].Size(sim.SizeKindDefault)
			} else if operation.GetSide() == packet.MarketSideAsk {
				price = emptyPrice
			}
			return
		}
		s.tobNew = book.GetTop(operation.GetOptionId(), operation.GetSide(), s.supernodeLevels)
		for i := 0; i < s.accessedLevels(operation); i++ {
			priceOld, sizeOld := printablePriceLevel(s.tobOld, i)
			priceNew, sizeNew := printablePriceLevel(s.tobNew, i)
			s.printfln("SN_OLD_NEW %02d %08x %08x  %08x %08x", i, sizeOld, priceOld, sizeNew, priceNew)
		}
	}
	s.efhLogger.AfterBookUpdate(book, operation)
}
func (s *SimLogger) accessedLevels(operation sim.SimOperation) (levels int) {
	levels = SimLoggerUpperSupernodeLevels
	if s.supernodeLevels <= levels {
		return s.supernodeLevels
	}
	if operation.GetPrice() == 0 {
		// TODO hw can skip SN access at all
		return
	}
	lenOld, lenNew := len(s.tobOld), len(s.tobNew)
	if lenOld < levels {
		return
	}
	if lenOld == lenNew {
		for i := 0; i < levels; i++ {
			if !s.tobOld[i].Equals(s.tobNew[i]) {
				return
			}
		}
	}
	return s.supernodeLevels
}

func (s *SimLogger) PrintMessage(m efhMessage) (err error) {
	defer errs.PassE(&err)
	var bb bytes.Buffer
	errs.CheckE(binary.Write(&bb, binary.LittleEndian, m))
	if r := bb.Len() % 8; r > 0 {
		// pad to  multiple of 8 bytes
		z := make([]byte, 8)
		_, err = bb.Write(z[0 : 8-r])
		errs.CheckE(err)
	}

	for {
		var qw uint64
		if err := binary.Read(&bb, binary.LittleEndian, &qw); err != nil {
			if err == io.EOF {
				break
			}
			errs.CheckE(err)
		} else {
			s.printfln("DMATOHOST_DATA %016x", qw)
		}
	}
	s.printfln("DMATOHOST_TRAILER 00656e696c616b45")
	return
}
<commit_msg>rec:SimLogger: remove SetOutputMode()<commit_after>// Copyright (c) Ilia Kravets, 2015. All rights reserved. PROVIDED "AS IS"
// WITHOUT ANY WARRANTY, EXPRESS OR IMPLIED. See LICENSE file for details.

package rec

import (
	"bytes"
	"encoding/binary"
	"fmt"
	"io"

	"github.com/ikravets/errs"

	"my/ev/packet"
	"my/ev/packet/bats"
	"my/ev/packet/miax"
	"my/ev/packet/nasdaq"
	"my/ev/sim"
)

type SimLoggerConfig struct {
	EfhLoggerConfig
	SupernodeLevels int
}

type SimLogger struct {
	w               io.Writer
	tobOld, tobNew  []sim.PriceLevel
	efhLogger       EfhLogger
	supernodeLevels int
}

const SimLoggerDefaultSupernodeLevels = 256
const SimLoggerUpperSupernodeLevels = 8

func NewSimLogger(c SimLoggerConfig) *SimLogger {
	s := &SimLogger{
		w:               c.Writer,
		supernodeLevels: c.SupernodeLevels,
	}
	if s.supernodeLevels == 0 {
		s.supernodeLevels = SimLoggerDefaultSupernodeLevels
	}
	elc := c.EfhLoggerConfig
	elc.Printer = s
	elc.Writer = nil
	s.efhLogger = *NewEfhLogger(elc)
	return s
}
func (s *SimLogger) SetSupernodeLevels(levels int) {
	errs.Check(levels > 0)
	s.supernodeLevels = levels
}

func (s *SimLogger) printf(format string, vs ...interface{}) {
	_, err := fmt.Fprintf(s.w, format, vs...)
	errs.CheckE(err)
}
func (s *SimLogger) printfln(format string, vs ...interface{}) {
	f := format + "\n"
	s.printf(f, vs...)
}
func (s *SimLogger) MessageArrived(idm *sim.SimMessage) {
	outItto := func(name string, typ nasdaq.IttoMessageType, f string, vs ...interface{}) {
		s.printf("NORM %s %c ", name, typ)
		s.printfln(f, vs...)
	}
	outBats := func(f string, vs ...interface{}) {
		s.printf("NORM ORDER %02x ", idm.Pam.Layer().(bats.PitchMessage).Base().Type.ToInt())
		s.printfln(f, vs...)
	}
	outMiax := func(f string, vs ...interface{}) {
		s.printf("NORM TOM %02x ", idm.Pam.Layer().(miax.TomMessage).Base().Type.ToInt())
		s.printfln(f, vs...)
	}
	sideChar := func(s packet.MarketSide) byte {
		if s == packet.MarketSideAsk {
			return 'S'
		}
		return byte(s)
	}
	switch im := idm.Pam.Layer().(type) {
	case *nasdaq.IttoMessageAddOrder:
		outItto("ORDER", im.Type, "%c %012x %016x %08x %08x", sideChar(im.Side), im.OId.ToUint64(), im.RefNumD.ToUint32(), im.Size, im.Price)
	case *nasdaq.IttoMessageAddQuote:
		outItto("QBID", im.Type, "%012x %016x %08x %08x", im.OId.ToUint64(), im.Bid.RefNumD.ToUint32(), im.Bid.Size, im.Bid.Price)
		outItto("QASK", im.Type, "%012x %016x %08x %08x", im.OId.ToUint64(), im.Ask.RefNumD.ToUint32(), im.Ask.Size, im.Ask.Price)
	case *nasdaq.IttoMessageSingleSideExecuted:
		outItto("ORDER", im.Type, "%016x %08x", im.OrigRefNumD.ToUint32(), im.Size)
	case *nasdaq.IttoMessageSingleSideExecutedWithPrice:
		outItto("ORDER", im.Type, "%016x %08x", im.OrigRefNumD.ToUint32(), im.Size)
	case *nasdaq.IttoMessageOrderCancel:
		outItto("ORDER", im.Type, "%016x %08x", im.OrigRefNumD.ToUint32(), im.Size)
	case *nasdaq.IttoMessageSingleSideReplace:
		outItto("ORDER", im.Type, "%016x %016x %08x %08x", im.RefNumD.ToUint32(), im.OrigRefNumD.ToUint32(), im.Size, im.Price)
	case *nasdaq.IttoMessageSingleSideDelete:
		outItto("ORDER", im.Type, "%016x", im.OrigRefNumD.ToUint32())
	case *nasdaq.IttoMessageSingleSideUpdate:
		outItto("ORDER", im.Type, "%016x %08x %08x", im.RefNumD.ToUint32(), im.Size, im.Price)
	case *nasdaq.IttoMessageQuoteReplace:
		outItto("QBID", im.Type, "%016x %016x %08x %08x", im.Bid.RefNumD.ToUint32(), im.Bid.OrigRefNumD.ToUint32(), im.Bid.Size, im.Bid.Price)
		outItto("QASK", im.Type, "%016x %016x %08x %08x", im.Ask.RefNumD.ToUint32(), im.Ask.OrigRefNumD.ToUint32(), im.Ask.Size, im.Ask.Price)
	case *nasdaq.IttoMessageQuoteDelete:
		outItto("QBID", im.Type, "%016x", im.BidOrigRefNumD.ToUint32())
		outItto("QASK", im.Type, "%016x", im.AskOrigRefNumD.ToUint32())
	case *nasdaq.IttoMessageBlockSingleSideDelete:
		for _, r := range im.RefNumDs {
			outItto("ORDER", im.Type, "%016x", r.ToUint32())
		}

	case *bats.PitchMessageAddOrder:
		outBats("%c %012x %016x %08x %08x", sideChar(im.Side), im.Symbol.ToUint64(), im.OrderId.ToUint64(), im.Size, packet.PriceTo4Dec(im.Price))
	case *bats.PitchMessageDeleteOrder:
		outBats("%016x", im.OrderId.ToUint64())
	case *bats.PitchMessageOrderExecuted:
		outBats("%016x %08x", im.OrderId.ToUint64(), im.Size)
	case *bats.PitchMessageOrderExecutedAtPriceSize:
		outBats("%016x %08x", im.OrderId.ToUint64(), im.Size)
	case *bats.PitchMessageReduceSize:
		outBats("%016x %08x", im.OrderId.ToUint64(), im.Size)
	case *bats.PitchMessageModifyOrder:
		outBats("%016x %08x %08x", im.OrderId.ToUint64(), im.Size, packet.PriceTo4Dec(im.Price))
	case *miax.TomMessageTom:
		outMiax("%c %08x %08x %08x %08x", sideChar(im.Side), im.ProductId.ToUint32(), packet.PriceTo4Dec(im.Price), im.Size, im.PriorityCustomerSize)
	}
	s.efhLogger.MessageArrived(idm)
}
func (s *SimLogger) OperationAppliedToOrders(operation sim.SimOperation) {
	type ordrespLogInfo struct {
		notFound, addOp   int
		orderId           packet.OrderId
		optionId          packet.OptionId
		side, price, size int
		ordlSuffix        string
	}
	type orduLogInfo struct {
		orderId           packet.OrderId
		optionId          packet.OptionId
		side, price, size int
	}

	var or ordrespLogInfo
	var ou orduLogInfo
	if _, ok := operation.(*sim.OperationTop); ok {
		return
	} else if op, ok := operation.(*sim.OperationAdd); ok {
		var oid packet.OptionId
		if op.Independent() {
			oid = op.GetOptionId()
		}
		or = ordrespLogInfo{
			addOp:      1,
			orderId:    op.OrderId,
			optionId:   oid,
			ordlSuffix: fmt.Sprintf(" %012x", oid.ToUint64()),
		}
		ou = orduLogInfo{
			orderId:  or.orderId,
			optionId: op.GetOptionId(),
			price:    op.GetPrice(),
			size:     op.GetNewSize(sim.SizeKindDefault),
		}
		if op.GetSide() == packet.MarketSideAsk {
			ou.side = 1
		}
	} else {
		if operation.GetOptionId().Invalid() {
			or = ordrespLogInfo{notFound: 1}
		} else {
			newSize := operation.GetNewSize(sim.SizeKindDefault)
			or = ordrespLogInfo{
				optionId: operation.GetOptionId(),
				price:    operation.GetPrice(),
				size:     newSize - operation.GetDefaultSizeDelta(),
			}
			if operation.GetSide() == packet.MarketSideAsk {
				or.side = 1
			}
			if newSize != 0 {
				ou = orduLogInfo{
					optionId: or.optionId,
					side:     or.side,
					price:    or.price,
					size:     newSize,
				}
			}
		}
		or.orderId = operation.GetOrigOrderId()
		ou.orderId = or.orderId
	}
	s.printfln("ORDL %d %016x%s", or.addOp, or.orderId.ToUint64(), or.ordlSuffix)
	s.printfln("ORDRESP %d %d %d %08x %08x %012x %016x", or.notFound, or.addOp, or.side, or.size, or.price, or.optionId.ToUint64(), or.orderId.ToUint64())
	if operation.GetOptionId().Valid() {
		s.printfln("ORDU %016x %012x %d %08x %08x", ou.orderId.ToUint64(), ou.optionId.ToUint64(), ou.side, ou.price, ou.size)
	}
}
func (s *SimLogger) BeforeBookUpdate(book sim.Book, operation sim.SimOperation) {
	tobOld := book.GetTop(operation.GetOptionId(), operation.GetSide(), s.supernodeLevels)
	s.tobOld = make([]sim.PriceLevel, len(tobOld))
	for i, pl := range tobOld {
		s.tobOld[i] = pl.Clone()
	}
	s.efhLogger.BeforeBookUpdate(book, operation)
}
func (s *SimLogger) AfterBookUpdate(book sim.Book, operation sim.SimOperation) {
	if operation.GetOptionId().Valid() && s.supernodeLevels > 1 {
		var emptyPrice uint32
		if operation.GetSide() == packet.MarketSideAsk {
			emptyPrice -= 1
		}
		printablePriceLevel := func(pls []sim.PriceLevel, pos int) (price uint32, size int) {
			if pos < len(pls) {
				price = uint32(pls[pos].Price())
				size = pls[pos].Size(sim.SizeKindDefault)
			} else if operation.GetSide() == packet.MarketSideAsk {
				price = emptyPrice
			}
			return
		}
		s.tobNew = book.GetTop(operation.GetOptionId(), operation.GetSide(), s.supernodeLevels)
		for i := 0; i < s.accessedLevels(operation); i++ {
			priceOld, sizeOld := printablePriceLevel(s.tobOld, i)
			priceNew, sizeNew := printablePriceLevel(s.tobNew, i)
			s.printfln("SN_OLD_NEW %02d %08x %08x  %08x %08x", i, sizeOld, priceOld, sizeNew, priceNew)
		}
	}
	s.efhLogger.AfterBookUpdate(book, operation)
}
func (s *SimLogger) accessedLevels(operation sim.SimOperation) (levels int) {
	levels = SimLoggerUpperSupernodeLevels
	if s.supernodeLevels <= levels {
		return s.supernodeLevels
	}
	if operation.GetPrice() == 0 {
		// TODO hw can skip SN access at all
		return
	}
	lenOld, lenNew := len(s.tobOld), len(s.tobNew)
	if lenOld < levels {
		return
	}
	if lenOld == lenNew {
		for i := 0; i < levels; i++ {
			if !s.tobOld[i].Equals(s.tobNew[i]) {
				return
			}
		}
	}
	return s.supernodeLevels
}

func (s *SimLogger) PrintMessage(m efhMessage) (err error) {
	defer errs.PassE(&err)
	var bb bytes.Buffer
	errs.CheckE(binary.Write(&bb, binary.LittleEndian, m))
	if r := bb.Len() % 8; r > 0 {
		// pad to  multiple of 8 bytes
		z := make([]byte, 8)
		_, err = bb.Write(z[0 : 8-r])
		errs.CheckE(err)
	}

	for {
		var qw uint64
		if err := binary.Read(&bb, binary.LittleEndian, &qw); err != nil {
			if err == io.EOF {
				break
			}
			errs.CheckE(err)
		} else {
			s.printfln("DMATOHOST_DATA %016x", qw)
		}
	}
	s.printfln("DMATOHOST_TRAILER 00656e696c616b45")
	return
}
<|endoftext|>


<commit_before>//
// Copyright (c) 2016 Snowplow Analytics Ltd. All rights reserved.
//
// This program is licensed to you under the Apache License Version 2.0,
// and you may not use this file except in compliance with the Apache License Version 2.0.
// You may obtain a copy of the Apache License Version 2.0 at http://www.apache.org/licenses/LICENSE-2.0.
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the Apache License Version 2.0 is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the Apache License Version 2.0 for the specific language governing permissions and limitations there under.
//

package tracker

import (
  "log"
  "database/sql"
  _ "github.com/mattn/go-sqlite3"
)

const (
  DB_DRIVER       = "sqlite3"
  DB_TABLE_NAME   = "events"
  DB_COLUMN_ID    = "id"
  DB_COLUMN_EVENT = "event"
)

type Storage struct {
  DbName   string
  DbConn   *sql.DB
  AddStmt  *sql.Stmt
}

type RawEventRow struct {
  id    int
  event []byte
}

type EventRow struct {
  id    int
  event Payload
}

func InitStorage(dbName string) *Storage {
  db, err := sql.Open(DB_DRIVER, dbName)
  checkErr(err)
  db.SetMaxOpenConns(1)

  // Enable Write-Ahead-Logging for concurrent read and write
  _, err1 := db.Exec("PRAGMA journal_mode=WAL;")
  checkErr(err1)

  // Create the Events Table
  query :=
    "CREATE TABLE IF NOT EXISTS " + DB_TABLE_NAME + "(" +
      DB_COLUMN_ID + " INTEGER PRIMARY KEY, " +
      DB_COLUMN_EVENT + " BLOB" +
    ");"
  _, err2 := db.Exec(query)
  checkErr(err2)

  // Prepare Add Statement
  query =
    "INSERT INTO " + DB_TABLE_NAME + "(" +
      DB_COLUMN_EVENT +
    ") values(?);"
  addStmt, err3 := db.Prepare(query)
  checkErr(err3)

  return &Storage{ DbName: dbName, DbConn: db, AddStmt: addStmt }
}

// --- ADD

// Add stores an event payload in the database.
func (s Storage) AddEventRow(payload Payload) bool {
  byteBuffer := SerializeMap(payload.Get())
  return execAddStatement(s.AddStmt, byteBuffer)
}

// execAddStatement executes the add statement passed to it.
func execAddStatement(stmt *sql.Stmt, byteBuffer []byte) bool {
  defer func() {
    if err := recover(); err != nil {
      log.Println(err)
    }
  }()

  res, err := stmt.Exec(byteBuffer); checkErr(err)
  affected, err2 := res.RowsAffected(); checkErr(err2)

  return affected == 1
}

// --- DELETE

// DeleteAllEventRows removes all events from the database.
func (s Storage) DeleteAllEventRows() int64 {
  query := "DELETE FROM " + DB_TABLE_NAME + ";"
  return execDeleteQuery(s.DbConn, query)
}

// DeleteEventRows removes a range of ids from the database.
func (s Storage) DeleteEventRows(ids []int) int64 {
  if len(ids) > 0 {
    query :=
      "DELETE FROM " + DB_TABLE_NAME + " " +
      "WHERE " + DB_COLUMN_ID + " in(" + IntArrayToString(ids, ",") + ");"
    return execDeleteQuery(s.DbConn, query)
  } else {
    return 0
  }
}

// execDeleteQuery is used to run queries which removed event rows from the database.
func execDeleteQuery(db *sql.DB, query string) int64 {
  defer func() {
    if err := recover(); err != nil {
      log.Println(err)
    }
  }()

  stmt, err := db.Prepare(query); checkErr(err)
  defer stmt.Close()
  res, err2 := stmt.Exec(); checkErr(err2)
  affected, err3 := res.RowsAffected(); checkErr(err3)

  return affected
}

// --- GET

// GetAllEventRows returns all events in the database.
func (s Storage) GetAllEventRows() []EventRow {
  query := "SELECT " + DB_COLUMN_ID + ", " + DB_COLUMN_EVENT + " FROM " + DB_TABLE_NAME + ";"
  return execGetQuery(s.DbConn, query)
}

// GetEventRowsWithinRange returns a specified range of events from the database.
func (s Storage) GetEventRowsWithinRange(eventRange int) []EventRow {
  query :=
    "SELECT " + DB_COLUMN_ID + ", " + DB_COLUMN_EVENT + " FROM " + DB_TABLE_NAME + " " +
    "ORDER BY " + DB_COLUMN_ID + " DESC LIMIT " + IntToString(eventRange) + ";"
  return execGetQuery(s.DbConn, query)
}

// execGetQuery is used to run queries to fetch event rows from the database.
func execGetQuery(db *sql.DB, query string) []EventRow {
  defer func() {
    if err := recover(); err != nil {
      log.Println(err)
    }
  }()

  eventItems := []EventRow{}
  rows, err := db.Query(query); checkErr(err)
  defer rows.Close()

  for rows.Next() {
    item := RawEventRow{}
    rows.Scan(&item.id, &item.event)
    eventMap, _ := DeserializeMap(item.event)
    eventItems = append(eventItems, EventRow{ item.id, Payload{ eventMap }})
  }

  return eventItems
}

// --- Helpers

// checkErr throws a panic for all non-nil errors passed to it.
func checkErr(err error) {
  if err != nil {
    panic(err.Error())
  }
}
<commit_msg>Ensure SQLite3 database cannot be left in a bad state (closes #15)<commit_after>//
// Copyright (c) 2016 Snowplow Analytics Ltd. All rights reserved.
//
// This program is licensed to you under the Apache License Version 2.0,
// and you may not use this file except in compliance with the Apache License Version 2.0.
// You may obtain a copy of the Apache License Version 2.0 at http://www.apache.org/licenses/LICENSE-2.0.
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the Apache License Version 2.0 is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the Apache License Version 2.0 for the specific language governing permissions and limitations there under.
//

package tracker

import (
  "log"
  "database/sql"
  _ "github.com/mattn/go-sqlite3"
)

const (
  DB_DRIVER       = "sqlite3"
  DB_TABLE_NAME   = "events"
  DB_COLUMN_ID    = "id"
  DB_COLUMN_EVENT = "event"
)

type Storage struct {
  DbName string
}

type RawEventRow struct {
  id    int
  event []byte
}

type EventRow struct {
  id    int
  event Payload
}

func InitStorage(dbName string) *Storage {
  db, err := getDbConn(dbName)
  checkErr(err)
  defer db.Close()

  db.SetMaxOpenConns(1)

  // Enable Write-Ahead-Logging for concurrent read and write
  _, err1 := db.Exec("PRAGMA journal_mode=WAL;")
  checkErr(err1)

  // Create the Events Table
  query :=
    "CREATE TABLE IF NOT EXISTS " + DB_TABLE_NAME + "(" +
      DB_COLUMN_ID + " INTEGER PRIMARY KEY, " +
      DB_COLUMN_EVENT + " BLOB" +
    ");"
  _, err2 := db.Exec(query)
  checkErr(err2)

  return &Storage{ DbName: dbName }
}

func getDbConn(dbName string) (*sql.DB, error) {
  return sql.Open(DB_DRIVER, dbName)
}

// --- ADD

// Add stores an event payload in the database.
func (s Storage) AddEventRow(payload Payload) bool {
  db, err := getDbConn(s.DbName)
  checkErr(err)
  defer db.Close()

  // Prepare Add Statement
  query :=
    "INSERT INTO " + DB_TABLE_NAME + "(" +
      DB_COLUMN_EVENT +
    ") values(?);"
  addStmt, err1 := db.Prepare(query)
  checkErr(err1)

  byteBuffer := SerializeMap(payload.Get())
  return execAddStatement(addStmt, byteBuffer)
}

// execAddStatement executes the add statement passed to it.
func execAddStatement(stmt *sql.Stmt, byteBuffer []byte) bool {
  defer func() {
    if err := recover(); err != nil {
      log.Println(err)
    }
  }()

  res, err := stmt.Exec(byteBuffer); checkErr(err)
  affected, err2 := res.RowsAffected(); checkErr(err2)

  return affected == 1
}

// --- DELETE

// DeleteAllEventRows removes all events from the database.
func (s Storage) DeleteAllEventRows() int64 {
  db, err := getDbConn(s.DbName)
  checkErr(err)
  defer db.Close()

  query := "DELETE FROM " + DB_TABLE_NAME + ";"
  return execDeleteQuery(db, query)
}

// DeleteEventRows removes a range of ids from the database.
func (s Storage) DeleteEventRows(ids []int) int64 {
  db, err := getDbConn(s.DbName)
  checkErr(err)
  defer db.Close()

  if len(ids) > 0 {
    query :=
      "DELETE FROM " + DB_TABLE_NAME + " " +
      "WHERE " + DB_COLUMN_ID + " in(" + IntArrayToString(ids, ",") + ");"
    return execDeleteQuery(db, query)
  } else {
    return 0
  }
}

// execDeleteQuery is used to run queries which removed event rows from the database.
func execDeleteQuery(db *sql.DB, query string) int64 {
  defer func() {
    if err := recover(); err != nil {
      log.Println(err)
    }
  }()

  stmt, err := db.Prepare(query); checkErr(err)
  defer stmt.Close()
  res, err2 := stmt.Exec(); checkErr(err2)
  affected, err3 := res.RowsAffected(); checkErr(err3)

  return affected
}

// --- GET

// GetAllEventRows returns all events in the database.
func (s Storage) GetAllEventRows() []EventRow {
  db, err := getDbConn(s.DbName)
  checkErr(err)
  defer db.Close()

  query := "SELECT " + DB_COLUMN_ID + ", " + DB_COLUMN_EVENT + " FROM " + DB_TABLE_NAME + ";"
  return execGetQuery(db, query)
}

// GetEventRowsWithinRange returns a specified range of events from the database.
func (s Storage) GetEventRowsWithinRange(eventRange int) []EventRow {
  db, err := getDbConn(s.DbName)
  checkErr(err)
  defer db.Close()

  query :=
    "SELECT " + DB_COLUMN_ID + ", " + DB_COLUMN_EVENT + " FROM " + DB_TABLE_NAME + " " +
    "ORDER BY " + DB_COLUMN_ID + " DESC LIMIT " + IntToString(eventRange) + ";"
  return execGetQuery(db, query)
}

// execGetQuery is used to run queries to fetch event rows from the database.
func execGetQuery(db *sql.DB, query string) []EventRow {
  defer func() {
    if err := recover(); err != nil {
      log.Println(err)
    }
  }()

  eventItems := []EventRow{}
  rows, err := db.Query(query); checkErr(err)
  defer rows.Close()

  for rows.Next() {
    item := RawEventRow{}
    rows.Scan(&item.id, &item.event)
    eventMap, _ := DeserializeMap(item.event)
    eventItems = append(eventItems, EventRow{ item.id, Payload{ eventMap }})
  }

  return eventItems
}

// --- Helpers

// checkErr throws a panic for all non-nil errors passed to it.
func checkErr(err error) {
  if err != nil {
    panic(err.Error())
  }
}
<|endoftext|>


<commit_before>/*
 *
 * Copyright IBM Corp. All Rights Reserved.
 *
 * SPDX-License-Identifier: Apache-2.0
 * /
 *
 */

package gossip

import (
	"fmt"
	"io/ioutil"
	"os"
	"syscall"
	"time"

	"github.com/fsouza/go-dockerclient"
	"github.com/hyperledger/fabric/integration/nwo"
	"github.com/hyperledger/fabric/integration/nwo/commands"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"github.com/onsi/gomega/gbytes"
	"github.com/onsi/gomega/gexec"
	"github.com/tedsuo/ifrit"
)

var _ = Describe("Gossip Test", func() {
	var (
		testDir   string
		client    *docker.Client
		network   *nwo.Network
		chaincode nwo.Chaincode
		process   ifrit.Process
	)

	BeforeEach(func() {
		var err error
		testDir, err = ioutil.TempDir("", "e2e")
		Expect(err).NotTo(HaveOccurred())

		client, err = docker.NewClientFromEnv()
		Expect(err).NotTo(HaveOccurred())

		chaincode = nwo.Chaincode{
			Name:    "mycc",
			Version: "0.0",
			Path:    "github.com/hyperledger/fabric/integration/chaincode/simple/cmd",
			Ctor:    `{"Args":["init","a","100","b","200"]}`,
			Policy:  `OR ('Org1MSP.member','Org2MSP.member')`,
		}
	})

	AfterEach(func() {
		if process != nil {
			process.Signal(syscall.SIGTERM)
			Eventually(process.Wait(), network.EventuallyTimeout).Should(Receive())
		}
		if network != nil {
			network.Cleanup()
		}
		os.RemoveAll(testDir)
	})

	Describe("State transfer test", func() {
		var (
			ordererProcess ifrit.Process
			peerProcesses  = map[string]ifrit.Process{}
		)

		BeforeEach(func() {
			network = nwo.New(nwo.BasicSolo(), testDir, client, StartPort(), components)

			network.GenerateConfigTree()
			network.Bootstrap()
		})

		AfterEach(func() {
			if ordererProcess != nil {
				ordererProcess.Signal(syscall.SIGTERM)
				Eventually(ordererProcess.Wait(), network.EventuallyTimeout).Should(Receive())
			}

			for _, process := range peerProcesses {
				process.Signal(syscall.SIGTERM)
				Eventually(process.Wait(), network.EventuallyTimeout).Should(Receive())
			}
		})

		It("solo network with 2 orgs, 2 peers each, should sync from the peer if no orderer available", func() {
			orderer := network.Orderer("orderer")
			ordererRunner := network.OrdererRunner(orderer)
			ordererProcess = ifrit.Invoke(ordererRunner)

			peer0Org1, peer1Org1 := network.Peer("Org1", "peer0"), network.Peer("Org1", "peer1")
			peer0Org2, peer1Org2 := network.Peer("Org2", "peer0"), network.Peer("Org2", "peer1")

			for _, peer := range []*nwo.Peer{peer0Org1, peer1Org1, peer0Org2, peer1Org2} {
				runner := network.PeerRunner(peer)
				peerProcesses[peer.ID()] = ifrit.Invoke(runner)
			}

			channelName := "testchannel"
			network.CreateChannel(channelName, orderer, peer0Org1)
			network.JoinChannel(channelName, orderer, peer0Org1, peer1Org1, peer0Org2, peer1Org2)

			nwo.DeployChaincode(network, channelName, orderer, chaincode, peer0Org1)
			network.UpdateChannelAnchors(orderer, channelName)

			for _, peer := range []*nwo.Peer{peer0Org1, peer1Org1, peer0Org2, peer1Org2} {
				Eventually(func() int {
					return nwo.GetLedgerHeight(network, peer, channelName)
				}).Should(BeNumerically(">=", 2))
			}

			By("stop peers to make sure they cannot get blocks from orderer")
			for id, proc := range peerProcesses {
				if id == peer0Org1.ID() {
					continue
				}
				proc.Signal(syscall.SIGTERM)
				Eventually(proc.Wait(), network.EventuallyTimeout).Should(Receive())
				delete(peerProcesses, id)
			}

			By("create transactions")
			for i := 0; i < 10; i++ {
				sess, err := network.PeerUserSession(peer0Org1, "User1", commands.ChaincodeInvoke{
					ChannelID: channelName,
					Orderer:   network.OrdererAddress(orderer, nwo.ListenPort),
					Name:      "mycc",
					Ctor:      `{"Args":["invoke","a","b","10"]}`,
					PeerAddresses: []string{
						network.PeerAddress(peer0Org1, nwo.ListenPort),
					},
					WaitForEvent: true,
				})
				Expect(err).NotTo(HaveOccurred())
				Eventually(sess, network.EventuallyTimeout).Should(gexec.Exit(0))
				Expect(sess.Err).To(gbytes.Say("Chaincode invoke successful. result: status:200"))
			}

			peer0LedgerHeight := nwo.GetLedgerHeight(network, peer0Org1, channelName)

			By("turning down ordering service")
			ordererProcess.Signal(syscall.SIGTERM)
			Eventually(ordererProcess.Wait(), network.EventuallyTimeout).Should(Receive())
			ordererProcess = nil

			By("wait for rest of the peers to join and startup")
			for _, peer := range []*nwo.Peer{peer1Org1, peer0Org2, peer1Org2} {
				runner := network.PeerRunner(peer, fmt.Sprint("CORE_PEER_GOSSIP_STATE_CHECKINTERVAL=200ms"),
					fmt.Sprint("FABRIC_LOGGING_SPEC=info:gossip.state=debug"),
				)
				peerProcesses[peer.ID()] = ifrit.Invoke(runner)
			}

			By("Make sure peers are synced up")
			for _, peer := range []*nwo.Peer{peer1Org1, peer0Org2, peer1Org2} {
				Eventually(func() int {
					return nwo.GetLedgerHeight(network, peer, channelName)
				}, time.Second*10).Should(Equal(peer0LedgerHeight))
			}
		})
	})
})
<commit_msg>[FAB-15225] Disable brittle state IT<commit_after>/*
 *
 * Copyright IBM Corp. All Rights Reserved.
 *
 * SPDX-License-Identifier: Apache-2.0
 * /
 *
 */

package gossip

import (
	"fmt"
	"io/ioutil"
	"os"
	"syscall"
	"time"

	"github.com/fsouza/go-dockerclient"
	"github.com/hyperledger/fabric/integration/nwo"
	"github.com/hyperledger/fabric/integration/nwo/commands"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"github.com/onsi/gomega/gbytes"
	"github.com/onsi/gomega/gexec"
	"github.com/tedsuo/ifrit"
)

var _ = Describe("Gossip Test", func() {
	var (
		testDir   string
		client    *docker.Client
		network   *nwo.Network
		chaincode nwo.Chaincode
		process   ifrit.Process
	)

	BeforeEach(func() {
		var err error
		testDir, err = ioutil.TempDir("", "e2e")
		Expect(err).NotTo(HaveOccurred())

		client, err = docker.NewClientFromEnv()
		Expect(err).NotTo(HaveOccurred())

		chaincode = nwo.Chaincode{
			Name:    "mycc",
			Version: "0.0",
			Path:    "github.com/hyperledger/fabric/integration/chaincode/simple/cmd",
			Ctor:    `{"Args":["init","a","100","b","200"]}`,
			Policy:  `OR ('Org1MSP.member','Org2MSP.member')`,
		}
	})

	AfterEach(func() {
		if process != nil {
			process.Signal(syscall.SIGTERM)
			Eventually(process.Wait(), network.EventuallyTimeout).Should(Receive())
		}
		if network != nil {
			network.Cleanup()
		}
		os.RemoveAll(testDir)
	})

	PDescribe("State transfer test", func() {
		var (
			ordererProcess ifrit.Process
			peerProcesses  = map[string]ifrit.Process{}
		)

		BeforeEach(func() {
			network = nwo.New(nwo.BasicSolo(), testDir, client, StartPort(), components)

			network.GenerateConfigTree()
			network.Bootstrap()
		})

		AfterEach(func() {
			if ordererProcess != nil {
				ordererProcess.Signal(syscall.SIGTERM)
				Eventually(ordererProcess.Wait(), network.EventuallyTimeout).Should(Receive())
			}

			for _, process := range peerProcesses {
				process.Signal(syscall.SIGTERM)
				Eventually(process.Wait(), network.EventuallyTimeout).Should(Receive())
			}
		})

		It("solo network with 2 orgs, 2 peers each, should sync from the peer if no orderer available", func() {
			orderer := network.Orderer("orderer")
			ordererRunner := network.OrdererRunner(orderer)
			ordererProcess = ifrit.Invoke(ordererRunner)

			peer0Org1, peer1Org1 := network.Peer("Org1", "peer0"), network.Peer("Org1", "peer1")
			peer0Org2, peer1Org2 := network.Peer("Org2", "peer0"), network.Peer("Org2", "peer1")

			for _, peer := range []*nwo.Peer{peer0Org1, peer1Org1, peer0Org2, peer1Org2} {
				runner := network.PeerRunner(peer)
				peerProcesses[peer.ID()] = ifrit.Invoke(runner)
			}

			channelName := "testchannel"
			network.CreateChannel(channelName, orderer, peer0Org1)
			network.JoinChannel(channelName, orderer, peer0Org1, peer1Org1, peer0Org2, peer1Org2)

			nwo.DeployChaincode(network, channelName, orderer, chaincode, peer0Org1)
			network.UpdateChannelAnchors(orderer, channelName)

			for _, peer := range []*nwo.Peer{peer0Org1, peer1Org1, peer0Org2, peer1Org2} {
				Eventually(func() int {
					return nwo.GetLedgerHeight(network, peer, channelName)
				}).Should(BeNumerically(">=", 2))
			}

			By("stop peers to make sure they cannot get blocks from orderer")
			for id, proc := range peerProcesses {
				if id == peer0Org1.ID() {
					continue
				}
				proc.Signal(syscall.SIGTERM)
				Eventually(proc.Wait(), network.EventuallyTimeout).Should(Receive())
				delete(peerProcesses, id)
			}

			By("create transactions")
			for i := 0; i < 10; i++ {
				sess, err := network.PeerUserSession(peer0Org1, "User1", commands.ChaincodeInvoke{
					ChannelID: channelName,
					Orderer:   network.OrdererAddress(orderer, nwo.ListenPort),
					Name:      "mycc",
					Ctor:      `{"Args":["invoke","a","b","10"]}`,
					PeerAddresses: []string{
						network.PeerAddress(peer0Org1, nwo.ListenPort),
					},
					WaitForEvent: true,
				})
				Expect(err).NotTo(HaveOccurred())
				Eventually(sess, network.EventuallyTimeout).Should(gexec.Exit(0))
				Expect(sess.Err).To(gbytes.Say("Chaincode invoke successful. result: status:200"))
			}

			peer0LedgerHeight := nwo.GetLedgerHeight(network, peer0Org1, channelName)

			By("turning down ordering service")
			ordererProcess.Signal(syscall.SIGTERM)
			Eventually(ordererProcess.Wait(), network.EventuallyTimeout).Should(Receive())
			ordererProcess = nil

			By("wait for rest of the peers to join and startup")
			for _, peer := range []*nwo.Peer{peer1Org1, peer0Org2, peer1Org2} {
				runner := network.PeerRunner(peer, fmt.Sprint("CORE_PEER_GOSSIP_STATE_CHECKINTERVAL=200ms"),
					fmt.Sprint("FABRIC_LOGGING_SPEC=info:gossip.state=debug"),
				)
				peerProcesses[peer.ID()] = ifrit.Invoke(runner)
			}

			By("Make sure peers are synced up")
			for _, peer := range []*nwo.Peer{peer1Org1, peer0Org2, peer1Org2} {
				Eventually(func() int {
					return nwo.GetLedgerHeight(network, peer, channelName)
				}, time.Second*10).Should(Equal(peer0LedgerHeight))
			}
		})
	})
})
<|endoftext|>


<commit_before>package commontmpl

import (
	"html/template"
	"strings"
	"time"

	"golang.org/x/text/language"
	"golang.org/x/text/language/display"

	"github.com/Debian/debiman/internal/bundled"
)

const iso8601Format = "2006-01-02T15:04:05Z"

func MustParseCommonTmpls() *template.Template {
	t := template.New("root")
	t = template.Must(t.New("header").Parse(bundled.Asset("header.tmpl")))
	t = template.Must(t.New("footer").
		Funcs(map[string]interface{}{
			"DisplayLang": func(tag language.Tag) string {
				lang := display.Self.Name(tag)
				// Some languages are not present in the Unicode CLDR,
				// so we cannot express their name in their own
				// language. Fall back to English.
				if lang == "" {
					return display.English.Languages().Name(tag)
				}
				return lang

			},
			"EnglishLang": func(tag language.Tag) string {
				return display.English.Languages().Name(tag)
			},
			"HasSuffix": func(s, suffix string) bool {
				return strings.HasSuffix(s, suffix)
			},
			"Now": func() string {
				return time.Now().UTC().Format(iso8601Format)
			}}).
		Parse(bundled.Asset("footer.tmpl")))
	t = template.Must(t.New("style").Parse(bundled.Asset("style.css")))
	return t
}
<commit_msg>disambiguate languages where necessary<commit_after>package commontmpl

import (
	"html/template"
	"strings"
	"time"

	"golang.org/x/text/language"
	"golang.org/x/text/language/display"

	"github.com/Debian/debiman/internal/bundled"
)

const iso8601Format = "2006-01-02T15:04:05Z"

var ambiguousLangs = map[string]bool{
	"cat": true, // català (ca, ca@valencia)
	"por": true, // português (pt, pt_BR)
	"zho": true, // 繁體中文 (zh_HK, zh_TW)
}

func MustParseCommonTmpls() *template.Template {
	t := template.New("root")
	t = template.Must(t.New("header").Parse(bundled.Asset("header.tmpl")))
	t = template.Must(t.New("footer").
		Funcs(map[string]interface{}{
			"DisplayLang": func(tag language.Tag) string {
				lang := display.Self.Name(tag)
				// Some languages are not present in the Unicode CLDR,
				// so we cannot express their name in their own
				// language. Fall back to English.
				if lang == "" {
					return display.English.Languages().Name(tag)
				}
				base, _ := tag.Base()
				if ambiguousLangs[base.ISO3()] {
					return lang + " (" + tag.String() + ")"
				}
				return lang

			},
			"EnglishLang": func(tag language.Tag) string {
				return display.English.Languages().Name(tag)
			},
			"HasSuffix": func(s, suffix string) bool {
				return strings.HasSuffix(s, suffix)
			},
			"Now": func() string {
				return time.Now().UTC().Format(iso8601Format)
			}}).
		Parse(bundled.Asset("footer.tmpl")))
	t = template.Must(t.New("style").Parse(bundled.Asset("style.css")))
	return t
}
<|endoftext|>


<commit_before>package aws

import (
	"fmt"
	"reflect"
	"testing"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/awserr"
	"github.com/aws/aws-sdk-go/service/ses"
	"github.com/hashicorp/terraform/helper/acctest"
	"github.com/hashicorp/terraform/helper/resource"
	"github.com/hashicorp/terraform/terraform"
)

func TestAccAWSSESReceiptRule_basic(t *testing.T) {
	resource.Test(t, resource.TestCase{
		PreCheck: func() {
			testAccPreCheck(t)
		},
		Providers:    testAccProviders,
		CheckDestroy: testAccCheckSESReceiptRuleDestroy,
		Steps: []resource.TestStep{
			resource.TestStep{
				Config: testAccAWSSESReceiptRuleBasicConfig,
				Check: resource.ComposeTestCheckFunc(
					testAccCheckAwsSESReceiptRuleExists("aws_ses_receipt_rule.basic"),
				),
			},
		},
	})
}

func TestAccAWSSESReceiptRule_order(t *testing.T) {
	resource.Test(t, resource.TestCase{
		PreCheck: func() {
			testAccPreCheck(t)
		},
		Providers:    testAccProviders,
		CheckDestroy: testAccCheckSESReceiptRuleDestroy,
		Steps: []resource.TestStep{
			resource.TestStep{
				Config: testAccAWSSESReceiptRuleOrderConfig,
				Check: resource.ComposeTestCheckFunc(
					testAccCheckAwsSESReceiptRuleOrder("aws_ses_receipt_rule.second"),
				),
			},
		},
	})
}

func TestAccAWSSESReceiptRule_actions(t *testing.T) {
	resource.Test(t, resource.TestCase{
		PreCheck: func() {
			testAccPreCheck(t)
		},
		Providers:    testAccProviders,
		CheckDestroy: testAccCheckSESReceiptRuleDestroy,
		Steps: []resource.TestStep{
			resource.TestStep{
				Config: testAccAWSSESReceiptRuleActionsConfig,
				Check: resource.ComposeTestCheckFunc(
					testAccCheckAwsSESReceiptRuleActions("aws_ses_receipt_rule.actions"),
				),
			},
		},
	})
}

func testAccCheckSESReceiptRuleDestroy(s *terraform.State) error {
	conn := testAccProvider.Meta().(*AWSClient).sesConn

	for _, rs := range s.RootModule().Resources {
		if rs.Type != "aws_ses_receipt_rule" {
			continue
		}

		params := &ses.DescribeReceiptRuleInput{
			RuleName:    aws.String(rs.Primary.Attributes["name"]),
			RuleSetName: aws.String(rs.Primary.Attributes["rule_set_name"]),
		}

		_, err := conn.DescribeReceiptRule(params)
		if err == nil {
			return fmt.Errorf("Receipt rule %s still exists. Failing!", rs.Primary.ID)
		}

		// Verify the error is what we want
		_, ok := err.(awserr.Error)
		if !ok {
			return err
		}

	}

	return nil

}

func testAccCheckAwsSESReceiptRuleExists(n string) resource.TestCheckFunc {
	return func(s *terraform.State) error {
		rs, ok := s.RootModule().Resources[n]
		if !ok {
			return fmt.Errorf("SES Receipt Rule not found: %s", n)
		}

		if rs.Primary.ID == "" {
			return fmt.Errorf("SES Receipt Rule name not set")
		}

		conn := testAccProvider.Meta().(*AWSClient).sesConn

		params := &ses.DescribeReceiptRuleInput{
			RuleName:    aws.String("basic"),
			RuleSetName: aws.String(fmt.Sprintf("test-me-%d", srrsRandomInt)),
		}

		response, err := conn.DescribeReceiptRule(params)
		if err != nil {
			return err
		}

		if !*response.Rule.Enabled {
			return fmt.Errorf("Enabled (%v) was not set to true", *response.Rule.Enabled)
		}

		if !reflect.DeepEqual(response.Rule.Recipients, []*string{aws.String("test@example.com")}) {
			return fmt.Errorf("Recipients (%v) was not set to [test@example.com]", response.Rule.Recipients)
		}

		if !*response.Rule.ScanEnabled {
			return fmt.Errorf("ScanEnabled (%v) was not set to true", *response.Rule.ScanEnabled)
		}

		if *response.Rule.TlsPolicy != "Require" {
			return fmt.Errorf("TLS Policy (%s) was not set to Require", *response.Rule.TlsPolicy)
		}

		return nil
	}
}

func testAccCheckAwsSESReceiptRuleOrder(n string) resource.TestCheckFunc {
	return func(s *terraform.State) error {
		rs, ok := s.RootModule().Resources[n]
		if !ok {
			return fmt.Errorf("SES Receipt Rule not found: %s", n)
		}

		if rs.Primary.ID == "" {
			return fmt.Errorf("SES Receipt Rule name not set")
		}

		conn := testAccProvider.Meta().(*AWSClient).sesConn

		params := &ses.DescribeReceiptRuleSetInput{
			RuleSetName: aws.String(fmt.Sprintf("test-me-%d", srrsRandomInt)),
		}

		response, err := conn.DescribeReceiptRuleSet(params)
		if err != nil {
			return err
		}

		if len(response.Rules) != 2 {
			return fmt.Errorf("Number of rules (%d) was not equal to 2", len(response.Rules))
		} else if *response.Rules[0].Name != "first" || *response.Rules[1].Name != "second" {
			return fmt.Errorf("Order of rules (%v) was incorrect", response.Rules)
		}

		return nil
	}
}

func testAccCheckAwsSESReceiptRuleActions(n string) resource.TestCheckFunc {
	return func(s *terraform.State) error {
		rs, ok := s.RootModule().Resources[n]
		if !ok {
			return fmt.Errorf("SES Receipt Rule not found: %s", n)
		}

		if rs.Primary.ID == "" {
			return fmt.Errorf("SES Receipt Rule name not set")
		}

		conn := testAccProvider.Meta().(*AWSClient).sesConn

		params := &ses.DescribeReceiptRuleInput{
			RuleName:    aws.String("actions"),
			RuleSetName: aws.String(fmt.Sprintf("test-me")),
		}

		response, err := conn.DescribeReceiptRule(params)
		if err != nil {
			return err
		}

		actions := response.Rule.Actions

		if len(actions) != 3 {
			return fmt.Errorf("Number of rules (%d) was not equal to 3", len(actions))
		}

		addHeaderAction := actions[0].AddHeaderAction
		if *addHeaderAction.HeaderName != "Another-Header" {
			return fmt.Errorf("Header Name (%s) was not equal to Another-Header", *addHeaderAction.HeaderName)
		}

		if *addHeaderAction.HeaderValue != "First" {
			return fmt.Errorf("Header Value (%s) was not equal to First", *addHeaderAction.HeaderValue)
		}

		secondAddHeaderAction := actions[1].AddHeaderAction
		if *secondAddHeaderAction.HeaderName != "Added-By" {
			return fmt.Errorf("Header Name (%s) was not equal to Added-By", *secondAddHeaderAction.HeaderName)
		}

		if *secondAddHeaderAction.HeaderValue != "Terraform" {
			return fmt.Errorf("Header Value (%s) was not equal to Terraform", *secondAddHeaderAction.HeaderValue)
		}

		stopAction := actions[2].StopAction
		if *stopAction.Scope != "RuleSet" {
			return fmt.Errorf("Scope (%s) was not equal to RuleSet", *stopAction.Scope)
		}

		return nil
	}
}

var srrsRandomInt = acctest.RandInt()
var testAccAWSSESReceiptRuleBasicConfig = fmt.Sprintf(`
resource "aws_ses_receipt_rule_set" "test" {
    rule_set_name = "test-me-%d"
}

resource "aws_ses_receipt_rule" "basic" {
    name = "basic"
    rule_set_name = "${aws_ses_receipt_rule_set.test.rule_set_name}"
    recipients = ["test@example.com"]
    enabled = true
    scan_enabled = true
    tls_policy = "Require"
}
`, srrsRandomInt)

var testAccAWSSESReceiptRuleOrderConfig = fmt.Sprintf(`
resource "aws_ses_receipt_rule_set" "test" {
    rule_set_name = "test-me-%d"
}

resource "aws_ses_receipt_rule" "second" {
    name = "second"
    rule_set_name = "${aws_ses_receipt_rule_set.test.rule_set_name}"
    after = "${aws_ses_receipt_rule.first.name}"
}

resource "aws_ses_receipt_rule" "first" {
    name = "first"
    rule_set_name = "${aws_ses_receipt_rule_set.test.rule_set_name}"
}
`, srrsRandomInt)

var testAccAWSSESReceiptRuleActionsConfig = fmt.Sprintf(`
resource "aws_s3_bucket" "emails" {
    bucket = "ses-terraform-emails"
}

resource "aws_ses_receipt_rule_set" "test" {
    rule_set_name = "test-me"
}

resource "aws_ses_receipt_rule" "actions" {
    name = "actions"
    rule_set_name = "${aws_ses_receipt_rule_set.test.rule_set_name}"

    add_header_action {
	header_name = "Added-By"
	header_value = "Terraform"
	position = 1
    }

    add_header_action {
	header_name = "Another-Header"
	header_value = "First"
	position = 0
    }

    stop_action {
	scope = "RuleSet"
	position = 2
    }
}
`)
<commit_msg>Randomize test name<commit_after>package aws

import (
	"fmt"
	"reflect"
	"testing"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/awserr"
	"github.com/aws/aws-sdk-go/service/ses"
	"github.com/hashicorp/terraform/helper/acctest"
	"github.com/hashicorp/terraform/helper/resource"
	"github.com/hashicorp/terraform/terraform"
)

func TestAccAWSSESReceiptRule_basic(t *testing.T) {
	resource.Test(t, resource.TestCase{
		PreCheck: func() {
			testAccPreCheck(t)
		},
		Providers:    testAccProviders,
		CheckDestroy: testAccCheckSESReceiptRuleDestroy,
		Steps: []resource.TestStep{
			resource.TestStep{
				Config: testAccAWSSESReceiptRuleBasicConfig,
				Check: resource.ComposeTestCheckFunc(
					testAccCheckAwsSESReceiptRuleExists("aws_ses_receipt_rule.basic"),
				),
			},
		},
	})
}

func TestAccAWSSESReceiptRule_order(t *testing.T) {
	resource.Test(t, resource.TestCase{
		PreCheck: func() {
			testAccPreCheck(t)
		},
		Providers:    testAccProviders,
		CheckDestroy: testAccCheckSESReceiptRuleDestroy,
		Steps: []resource.TestStep{
			resource.TestStep{
				Config: testAccAWSSESReceiptRuleOrderConfig,
				Check: resource.ComposeTestCheckFunc(
					testAccCheckAwsSESReceiptRuleOrder("aws_ses_receipt_rule.second"),
				),
			},
		},
	})
}

func TestAccAWSSESReceiptRule_actions(t *testing.T) {
	resource.Test(t, resource.TestCase{
		PreCheck: func() {
			testAccPreCheck(t)
		},
		Providers:    testAccProviders,
		CheckDestroy: testAccCheckSESReceiptRuleDestroy,
		Steps: []resource.TestStep{
			resource.TestStep{
				Config: testAccAWSSESReceiptRuleActionsConfig,
				Check: resource.ComposeTestCheckFunc(
					testAccCheckAwsSESReceiptRuleActions("aws_ses_receipt_rule.actions"),
				),
			},
		},
	})
}

func testAccCheckSESReceiptRuleDestroy(s *terraform.State) error {
	conn := testAccProvider.Meta().(*AWSClient).sesConn

	for _, rs := range s.RootModule().Resources {
		if rs.Type != "aws_ses_receipt_rule" {
			continue
		}

		params := &ses.DescribeReceiptRuleInput{
			RuleName:    aws.String(rs.Primary.Attributes["name"]),
			RuleSetName: aws.String(rs.Primary.Attributes["rule_set_name"]),
		}

		_, err := conn.DescribeReceiptRule(params)
		if err == nil {
			return fmt.Errorf("Receipt rule %s still exists. Failing!", rs.Primary.ID)
		}

		// Verify the error is what we want
		_, ok := err.(awserr.Error)
		if !ok {
			return err
		}

	}

	return nil

}

func testAccCheckAwsSESReceiptRuleExists(n string) resource.TestCheckFunc {
	return func(s *terraform.State) error {
		rs, ok := s.RootModule().Resources[n]
		if !ok {
			return fmt.Errorf("SES Receipt Rule not found: %s", n)
		}

		if rs.Primary.ID == "" {
			return fmt.Errorf("SES Receipt Rule name not set")
		}

		conn := testAccProvider.Meta().(*AWSClient).sesConn

		params := &ses.DescribeReceiptRuleInput{
			RuleName:    aws.String("basic"),
			RuleSetName: aws.String(fmt.Sprintf("test-me-%d", srrsRandomInt)),
		}

		response, err := conn.DescribeReceiptRule(params)
		if err != nil {
			return err
		}

		if !*response.Rule.Enabled {
			return fmt.Errorf("Enabled (%v) was not set to true", *response.Rule.Enabled)
		}

		if !reflect.DeepEqual(response.Rule.Recipients, []*string{aws.String("test@example.com")}) {
			return fmt.Errorf("Recipients (%v) was not set to [test@example.com]", response.Rule.Recipients)
		}

		if !*response.Rule.ScanEnabled {
			return fmt.Errorf("ScanEnabled (%v) was not set to true", *response.Rule.ScanEnabled)
		}

		if *response.Rule.TlsPolicy != "Require" {
			return fmt.Errorf("TLS Policy (%s) was not set to Require", *response.Rule.TlsPolicy)
		}

		return nil
	}
}

func testAccCheckAwsSESReceiptRuleOrder(n string) resource.TestCheckFunc {
	return func(s *terraform.State) error {
		rs, ok := s.RootModule().Resources[n]
		if !ok {
			return fmt.Errorf("SES Receipt Rule not found: %s", n)
		}

		if rs.Primary.ID == "" {
			return fmt.Errorf("SES Receipt Rule name not set")
		}

		conn := testAccProvider.Meta().(*AWSClient).sesConn

		params := &ses.DescribeReceiptRuleSetInput{
			RuleSetName: aws.String(fmt.Sprintf("test-me-%d", srrsRandomInt)),
		}

		response, err := conn.DescribeReceiptRuleSet(params)
		if err != nil {
			return err
		}

		if len(response.Rules) != 2 {
			return fmt.Errorf("Number of rules (%d) was not equal to 2", len(response.Rules))
		} else if *response.Rules[0].Name != "first" || *response.Rules[1].Name != "second" {
			return fmt.Errorf("Order of rules (%v) was incorrect", response.Rules)
		}

		return nil
	}
}

func testAccCheckAwsSESReceiptRuleActions(n string) resource.TestCheckFunc {
	return func(s *terraform.State) error {
		rs, ok := s.RootModule().Resources[n]
		if !ok {
			return fmt.Errorf("SES Receipt Rule not found: %s", n)
		}

		if rs.Primary.ID == "" {
			return fmt.Errorf("SES Receipt Rule name not set")
		}

		conn := testAccProvider.Meta().(*AWSClient).sesConn

		params := &ses.DescribeReceiptRuleInput{
			RuleName:    aws.String("actions4"),
			RuleSetName: aws.String(fmt.Sprintf("test-me-%d", srrsRandomInt)),
		}

		response, err := conn.DescribeReceiptRule(params)
		if err != nil {
			return err
		}

		actions := response.Rule.Actions

		if len(actions) != 3 {
			return fmt.Errorf("Number of rules (%d) was not equal to 3", len(actions))
		}

		addHeaderAction := actions[0].AddHeaderAction
		if *addHeaderAction.HeaderName != "Another-Header" {
			return fmt.Errorf("Header Name (%s) was not equal to Another-Header", *addHeaderAction.HeaderName)
		}

		if *addHeaderAction.HeaderValue != "First" {
			return fmt.Errorf("Header Value (%s) was not equal to First", *addHeaderAction.HeaderValue)
		}

		secondAddHeaderAction := actions[1].AddHeaderAction
		if *secondAddHeaderAction.HeaderName != "Added-By" {
			return fmt.Errorf("Header Name (%s) was not equal to Added-By", *secondAddHeaderAction.HeaderName)
		}

		if *secondAddHeaderAction.HeaderValue != "Terraform" {
			return fmt.Errorf("Header Value (%s) was not equal to Terraform", *secondAddHeaderAction.HeaderValue)
		}

		stopAction := actions[2].StopAction
		if *stopAction.Scope != "RuleSet" {
			return fmt.Errorf("Scope (%s) was not equal to RuleSet", *stopAction.Scope)
		}

		return nil
	}
}

var srrsRandomInt = acctest.RandInt()
var testAccAWSSESReceiptRuleBasicConfig = fmt.Sprintf(`
resource "aws_ses_receipt_rule_set" "test" {
    rule_set_name = "test-me-%d"
}

resource "aws_ses_receipt_rule" "basic" {
    name = "basic"
    rule_set_name = "${aws_ses_receipt_rule_set.test.rule_set_name}"
    recipients = ["test@example.com"]
    enabled = true
    scan_enabled = true
    tls_policy = "Require"
}
`, srrsRandomInt)

var testAccAWSSESReceiptRuleOrderConfig = fmt.Sprintf(`
resource "aws_ses_receipt_rule_set" "test" {
    rule_set_name = "test-me-%d"
}

resource "aws_ses_receipt_rule" "second" {
    name = "second"
    rule_set_name = "${aws_ses_receipt_rule_set.test.rule_set_name}"
    after = "${aws_ses_receipt_rule.first.name}"
}

resource "aws_ses_receipt_rule" "first" {
    name = "first"
    rule_set_name = "${aws_ses_receipt_rule_set.test.rule_set_name}"
}
`, srrsRandomInt)

var testAccAWSSESReceiptRuleActionsConfig = fmt.Sprintf(`
resource "aws_s3_bucket" "emails" {
    bucket = "ses-terraform-emails"
}

resource "aws_ses_receipt_rule_set" "test" {
    rule_set_name = "test-me-%d"
}

resource "aws_ses_receipt_rule" "actions" {
    name = "actions4"
    rule_set_name = "${aws_ses_receipt_rule_set.test.rule_set_name}"

    add_header_action {
			header_name = "Added-By"
			header_value = "Terraform"
			position = 1
    }

    add_header_action {
			header_name = "Another-Header"
			header_value = "First"
			position = 0
    }

    stop_action {
			scope = "RuleSet"
			position = 2
    }
}
`, srrsRandomInt)
<|endoftext|>


<commit_before>// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

package newton

import (
	"github.com/szabba/md/src/vect"
)

type Body struct {
	Xs, Vs []vect.Vector
	mass   float64
	currAt int
}

// Constructs a body of specified mass suitable for working with the integrator
func NewBody(algo Integrator, mass float64) *Body {

	b := new(Body)

	b.Xs = make([]vect.Vector, algo.StateLen())
	b.Vs = make([]vect.Vector, algo.StateLen())

	b.mass = mass
	b.currAt = algo.CurrentAt()

	return b
}

// Give a body's mass
func (b *Body) Mass() float64 {

	return b.mass
}

// Put new values of x and v a the beginning of the remembered values
//
// The oldest values get discarded
func (b *Body) Shift(x, v vect.Vector) {

	Shift(b.Xs, x)
	Shift(b.Vs, v)
}

// Current positon and velocity
func (b *Body) Now() (x, v vect.Vector) {

	return b.XNow(), b.VNow()
}

// Current position
func (b *Body) XNow() vect.Vector {

	return b.Xs[b.currAt]
}

// Current velocity
func (b *Body) VNow() vect.Vector {

	return b.Vs[b.currAt]
}

// Position and velocity delta steps before now
func (b *Body) Before(delta int) (x, v vect.Vector) {

	return b.XBefore(delta), b.VBefore(delta)
}

// Position delta steps before now
func (b *Body) XBefore(delta int) vect.Vector {

	return b.Xs[b.currAt+delta]
}

// Velocity delta steps before now
func (b *Body) VBefore(delta int) vect.Vector {

	return b.Vs[b.currAt+delta]
}

// Shifts all the values in xs by one and puts x at the beginning.
func Shift(xs []vect.Vector, x vect.Vector) {

	for i := len(xs) - 1; i > 0; i-- {

		xs[i] = xs[i-1]
	}

	xs[0] = x
}
<commit_msg>Add newton.Body.{,X,V}After<commit_after>// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.

package newton

import (
	"github.com/szabba/md/src/vect"
)

type Body struct {
	Xs, Vs []vect.Vector
	mass   float64
	currAt int
}

// Constructs a body of specified mass suitable for working with the integrator
func NewBody(algo Integrator, mass float64) *Body {

	b := new(Body)

	b.Xs = make([]vect.Vector, algo.StateLen())
	b.Vs = make([]vect.Vector, algo.StateLen())

	b.mass = mass
	b.currAt = algo.CurrentAt()

	return b
}

// Give a body's mass
func (b *Body) Mass() float64 {

	return b.mass
}

// Put new values of x and v a the beginning of the remembered values
//
// The oldest values get discarded
func (b *Body) Shift(x, v vect.Vector) {

	Shift(b.Xs, x)
	Shift(b.Vs, v)
}

// Current positon and velocity
func (b *Body) Now() (x, v vect.Vector) {

	return b.XNow(), b.VNow()
}

// Current position
func (b *Body) XNow() vect.Vector {

	return b.Xs[b.currAt]
}

// Current velocity
func (b *Body) VNow() vect.Vector {

	return b.Vs[b.currAt]
}

// Position and velocity delta steps before now
func (b *Body) Before(delta int) (x, v vect.Vector) {

	return b.XBefore(delta), b.VBefore(delta)
}

// Position delta steps before now
func (b *Body) XBefore(delta int) vect.Vector {

	return b.Xs[b.currAt+delta]
}

// Velocity delta steps before now
func (b *Body) VBefore(delta int) vect.Vector {

	return b.Vs[b.currAt+delta]
}

// Position and velocity delta steps after now
func (b *Body) After(delta int) (x, v vect.Vector) {

	return b.XAfter(delta), b.VAfter(delta)
}

// Position delta steps after now
func (b *Body) XAfter(delta int) vect.Vector {

	return b.Xs[b.currAt+delta]
}

// Velocity delta steps after now
func (b *Body) VAfter(delta int) vect.Vector {

	return b.Vs[b.currAt+delta]
}

// Shifts all the values in xs by one and puts x at the beginning.
func Shift(xs []vect.Vector, x vect.Vector) {

	for i := len(xs) - 1; i > 0; i-- {

		xs[i] = xs[i-1]
	}

	xs[0] = x
}
<|endoftext|>


<commit_before>package add

import (
	"github.com/BytemarkHosting/bytemark-client/cmd/bytemark/app"
	"github.com/BytemarkHosting/bytemark-client/cmd/bytemark/app/args"
	"github.com/BytemarkHosting/bytemark-client/cmd/bytemark/app/with"
	"github.com/urfave/cli"
)

func init() {
	Commands = append(Commands, cli.Command{
		Name:    "api key",
		Aliases: []string{"apikey"},
		Flags: []cli.Flag{
			cli.StringFlag{
				Name:  "expires-at",
				Usage: "Date the API key should expire. Leave unset for keys that never expire",
			},
			cli.StringSliceFlag{
				Name:  "group",
				Usage: "Group to grant the API key administrative privilege over",
			},
			cli.StringFlag{
				Name:  "label",
				Usage: "user-friendly label for the API key",
			},
			cli.StringSliceFlag{
				Name:  "server",
				Usage: "Server to grant the API key administrative privilege over",
			},
			cli.StringFlag{
				Name:  "user",
				Usage: "User the API key will be attached to. Defaults to the user you log in as",
			},
		},
		Usage:     "add an API key to your Bytemark Cloud Servers user",
		UsageText: "add api key [--server <cloud server>]... [--group <group name>]... [--user <user>] <label>",
		Description: `--expires-at may be set to any date format the Brain
accepts, but we generally recommend ISO8601 format.

Servers and groups will be searched for on the default account for the user
you are logged in as. This may trip up cluster administrators, so
bytemark-client will refuse to create an API key whose access is not a subset
of the access the specified user normally has. To create such an API key you
can either add the necessary privileges to ensure that the API key privileges
are a subset, or create the API key without privileges and add them via the
grant command, which does not have this limitation.

Note that the API key will only currently be able to access the Bytemark Cloud
Servers API - to manage servers and groups.

Multiple --group and --server flags (and combinations thereof) can be supplied,
and the API key will be have privileges over each that is supplied.`,
		Action: app.Action(args.Optional("label"), with.Auth, func(ctx *app.Context) error {
			servers, serverIDsMap, err := findServers(ctx)
			if err != nil {
				return err
			}
			groups, groupIDsMap := findGroups(ctx)
			if err != nil {
				return err
			}

			apiKey, err := brainRequests.CreateAPIKey(ctx.Client(),keySpec)
			if err != nil {
				return err
			}
			privs := makeServerPrivs(servers, apiKey)
			privs = append(privs, makeGroupPrivs(groups, apiKey))
			privErrs := []error{}
			for _, priv := range privs {
				err = ctx.Client().GrantPrivilege(privs)
				if err != nil {
					privErrs
			}
		}),
	})
}
<commit_msg>mostly finish implementing add api key. I hope :\<commit_after>package add

import (
	"errors"
	"fmt"
	"strings"

	"github.com/BytemarkHosting/bytemark-client/cmd/bytemark/app"
	"github.com/BytemarkHosting/bytemark-client/cmd/bytemark/app/args"
	"github.com/BytemarkHosting/bytemark-client/cmd/bytemark/app/flags"
	"github.com/BytemarkHosting/bytemark-client/cmd/bytemark/app/with"
	"github.com/BytemarkHosting/bytemark-client/lib/brain"
	"github.com/BytemarkHosting/bytemark-client/lib/output/morestrings"
	"github.com/BytemarkHosting/bytemark-client/lib/output/prettyprint"
	brainRequests "github.com/BytemarkHosting/bytemark-client/lib/requests/brain"
	"github.com/urfave/cli"
)

func init() {
	Commands = append(Commands, cli.Command{
		Name:    "api key",
		Aliases: []string{"apikey"},
		Flags: []cli.Flag{
			cli.StringFlag{
				Name:  "expires-at",
				Usage: "Date the API key should expire. Leave unset for keys that never expire",
			},
			cli.GenericFlag{
				Name:  "account-admin",
				Usage: "Account to grant the API key administrative privilege over",
				Value: &flags.GroupNameSliceFlag{},
			},
			cli.GenericFlag{
				Name:  "group",
				Usage: "Group to grant the API key administrative privilege over",
				Value: &flags.GroupNameSliceFlag{},
			},
			cli.StringFlag{
				Name:  "label",
				Usage: "user-friendly label for the API key",
			},
			cli.GenericFlag{
				Name:  "server",
				Usage: "Server to grant the API key administrative privilege over",
				Value: &flags.VirtualMachineNameSliceFlag{},
			},
			cli.StringFlag{
				Name:  "user",
				Usage: "User the API key will be attached to. Defaults to the user you log in as",
			},
		},
		Usage:     "add an API key to your Bytemark Cloud Servers user",
		UsageText: "add api key [--server <cloud server>]... [--group <group name>]... [--account-admin <account name>]... [--user <user>] <label>",
		Description: `--expires-at may be set to any date format the Brain
accepts, but we generally recommend ISO8601 format.

--label or <label> is just for your reference to make it easier to discern keys
from each other at a glance, without having to compare lists of privileges.

Servers and groups will be searched for on the default account for the user
you are logged in as - not the user specified in --user. This may trip up
cluster administrators - use impersonation if you're not granting account_admin
access.

Only cluster administrators can create API keys with account-admin support. Pop
and email over in to Bytemark support and we'll do it for you.

Multiple --account-admin, --group and --server flags (and combinations thereof)
can be supplied, and the API key will have privileges over each that is
supplied.

Note that the API key will only be able to access the Bytemark Cloud Servers API
- to manage your cloud servers, IPs, discs and backups, not billing or domains.

In future we may implement API key support on more of our API, but the
privileges to access those parts of the API will not be added automatically to
any existing API keys. In other words - an API key with account admin access now
will not be able to suddenly access your invoices/account owner address in
future if and when we add that feature to API keys.

EXAMPLES

To create an API key for yourself without any privileges:

  bytemark add api key currently-useless-api-key

You can always add privileges later to make it a non-useless API key with the
'grant' command using the --api-key flag. See 'bytemark help grant' for more
details.

To create an API key for yourself which can create, delete, and alter any server
in the 'kube' and 'swarm' groups on your default account:

  bytemark add api key --group kube --group swarm container-swarms-key

To create a key for yourself with access to rescale the database servers in the
'internal' group on an account 'big-data-services' which is not your default
account:

  bytemark add api key --server db1.internal.big-data-services \
                       --server db2.internal.big-data-services \
					   --label auto-rescale-db
`,
		Action: app.Action(args.Optional("label"), with.User("user"), func(ctx *app.Context) error {
			keySpec, err := makeAPIKeySpec(ctx)
			if err != nil {
				ctx.LogErr("Couldn't make a specification for the API key")
				return err
			}

			apiKey, err := brainRequests.CreateAPIKey(ctx.Client(), "", keySpec)
			if err != nil {
				return err
			}
			keySpec.APIKey = apiKey.APIKey
			keySpec.ID = apiKey.ID

			for i := range keySpec.Privileges {
				keySpec.Privileges[i].APIKeyID = apiKey.ID
			}

			if len(keySpec.Privileges) == 0 {
				ctx.LogErr("Successfully created an api key:")
				err = apiKey.PrettyPrint(ctx.Writer(), prettyprint.Full)
				return err
			}

			ctx.LogErr("Successfully created api key, now creating %d privileges...", len(keySpec.Privileges))

			// keep this error separate cause we check it later
			privsErr := addAPIKeyPrivileges(ctx, keySpec)

			done := "done."
			if privsErr != nil {
				done = "done, with some errors. Here's the API key we created with all the privileges we managed to create:"
			}

			ctx.LogErr(done)
			apiKey.PrettyPrint(ctx.Writer(), prettyprint.Full)

			if privsErr != nil {
				return privsErr
			}

			return nil

		}),
	})
}

func makeAPIKeySpec(ctx *app.Context) (spec brain.APIKey, err error) {
	spec.Label = ctx.String("label")
	spec.UserID = ctx.User.ID

	typesWithErrs := []string{}
	spec.Privileges, err = addAPIKeySpecifyServerPrivileges(ctx, spec.Privileges)
	if err != nil {
		typesWithErrs = append(typesWithErrs, "servers")
	}
	spec.Privileges, err = addAPIKeySpecifyGroupPrivileges(ctx, spec.Privileges)
	if err != nil {
		typesWithErrs = append(typesWithErrs, "groups")
	}
	spec.Privileges, err = addAPIKeySpecifyAccountPrivileges(ctx, spec.Privileges)
	if err != nil {
		typesWithErrs = append(typesWithErrs, "accounts")
	}
	if len(typesWithErrs) > 0 {
		err = fmt.Errorf("Some %s could not be looked up", morestrings.JoinWithSpecialLast(", ", " and ", typesWithErrs))
	}

	return
}

// creates a privilege spec for every --server flag
func addAPIKeySpecifyServerPrivileges(ctx *app.Context, privileges brain.Privileges) (brain.Privileges, error) {
	var serverErr error
	for _, serverFlag := range flags.VirtualMachineNameSlice(ctx, "server") {
		serverName := serverFlag.VirtualMachineName
		ctx.LogErr("Looking up %s", serverName)

		server, err := ctx.Client().GetVirtualMachine(serverName)
		if err != nil {
			ctx.LogErr(err.Error())
			if serverErr == nil {
				serverErr = errors.New("Some servers could not be looked up - see above for errors")
			}
			continue
		}

		privileges = append(privileges, brain.Privilege{
			Username:         ctx.User.Username,
			VirtualMachineID: server.ID,
		})
	}
	return privileges, nil
}

// creates a privilege spec for every --group flag
func addAPIKeySpecifyGroupPrivileges(ctx *app.Context, privileges brain.Privileges) (brain.Privileges, error) {
	var groupErr error
	for _, groupFlag := range flags.GroupNameSlice(ctx, "group") {
		groupName := groupFlag.GroupName
		ctx.LogErr("Looking up %s", groupName)

		group, err := ctx.Client().GetGroup(groupName)
		if err != nil {
			ctx.LogErr(err.Error())
			if groupErr == nil {
				groupErr = errors.New("Some groups could not be looked up - see above for errors")
			}
			continue
		}

		privileges = append(privileges, brain.Privilege{
			Username: ctx.User.Username,
			GroupID:  group.ID,
		})
	}
	return privileges, nil
}

// creates a privilege spec for every --account-admin flag
func addAPIKeySpecifyAccountPrivileges(ctx *app.Context, privileges brain.Privileges) (brain.Privileges, error) {
	var accountErr error
	for _, accountFlag := range flags.AccountNameSlice(ctx, "account-admin") {
		accountName := accountFlag.AccountName
		ctx.LogErr("Looking up %s", accountName)

		account, err := ctx.Client().GetAccount(accountName)
		if err != nil {
			ctx.LogErr(err.Error())
			if accountErr == nil {
				accountErr = errors.New("Some accounts could not be looked up - see above for errors")
			}
			continue
		}

		privileges = append(privileges, brain.Privilege{
			Username:  ctx.User.Username,
			AccountID: account.BrainID,
		})
	}
	return privileges, nil
}

// addAPIKeyPrivileges goes over the privilege specs and grants them to the api
// key.
func addAPIKeyPrivileges(ctx *app.Context, apiKey brain.APIKey) error {
	// collect up all the errors to output a nice list at the end, so
	// that we still output the complete API key as it actually stands
	succeededPrivs := make(brain.Privileges, 0, len(apiKey.Privileges))
	privErrs := []addAPIKeyPrivErr{}

	for i, priv := range apiKey.Privileges {
		priv.APIKeyID = apiKey.ID

		err := ctx.Client().GrantPrivilege(priv)
		if err != nil {
			privErrs = append(privErrs, addAPIKeyPrivErr{
				idx:  i,
				priv: priv,
				err:  err,
			})
		} else {
			succeededPrivs = append(succeededPrivs, priv)
		}
	}
	apiKey.Privileges = succeededPrivs

	if len(privErrs) > 0 {
		// catalogue the failures
		lines := make([]string, len(privErrs))
		for i, privErr := range privErrs {
			lines[i] = fmt.Sprintf("  • %s: %s", privErr.priv, privErr.err)
		}
		intro := fmt.Sprintf("Couldn't create %d/%d privileges requested:", len(privErrs), len(apiKey.Privileges))

		return errors.New(intro + "\n" + strings.Join(lines, "\n"))
	}
	return nil
}

type addAPIKeyPrivErr struct {
	idx  int
	priv brain.Privilege
	err  error
}
<|endoftext|>


<commit_before>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package features

import (
	"fmt"
	"sort"
	"strconv"
	"strings"

	"github.com/pkg/errors"
	"k8s.io/apimachinery/pkg/util/version"
	"k8s.io/component-base/featuregate"
)

const (
	// IPv6DualStack is expected to be beta in v1.21
	IPv6DualStack = "IPv6DualStack"
	// PublicKeysECDSA is expected to be alpha in v1.19
	PublicKeysECDSA = "PublicKeysECDSA"
)

// InitFeatureGates are the default feature gates for the init command
var InitFeatureGates = FeatureList{
	IPv6DualStack:   {FeatureSpec: featuregate.FeatureSpec{Default: true, PreRelease: featuregate.Beta}},
	PublicKeysECDSA: {FeatureSpec: featuregate.FeatureSpec{Default: false, PreRelease: featuregate.Alpha}},
}

// Feature represents a feature being gated
type Feature struct {
	featuregate.FeatureSpec
	MinimumVersion     *version.Version
	HiddenInHelpText   bool
	DeprecationMessage string
}

// FeatureList represents a list of feature gates
type FeatureList map[string]Feature

// ValidateVersion ensures that a feature gate list is compatible with the chosen Kubernetes version
func ValidateVersion(allFeatures FeatureList, requestedFeatures map[string]bool, requestedVersion string) error {
	if requestedVersion == "" {
		return nil
	}
	parsedExpVersion, err := version.ParseSemantic(requestedVersion)
	if err != nil {
		return errors.Wrapf(err, "error parsing version %s", requestedVersion)
	}
	for k := range requestedFeatures {
		if minVersion := allFeatures[k].MinimumVersion; minVersion != nil {
			if !parsedExpVersion.AtLeast(minVersion) {
				return errors.Errorf(
					"the requested Kubernetes version (%s) is incompatible with the %s feature gate, which needs %s as a minimum",
					requestedVersion, k, minVersion)
			}
		}
	}
	return nil
}

// Enabled indicates whether a feature name has been enabled
func Enabled(featureList map[string]bool, featureName string) bool {
	if enabled, ok := featureList[string(featureName)]; ok {
		return enabled
	}
	return InitFeatureGates[string(featureName)].Default
}

// Supports indicates whether a feature name is supported on the given
// feature set
func Supports(featureList FeatureList, featureName string) bool {
	for k, v := range featureList {
		if featureName == string(k) {
			return v.PreRelease != featuregate.Deprecated
		}
	}
	return false
}

// Keys returns a slice of feature names for a given feature set
func Keys(featureList FeatureList) []string {
	var list []string
	for k := range featureList {
		list = append(list, string(k))
	}
	return list
}

// KnownFeatures returns a slice of strings describing the FeatureList features.
func KnownFeatures(f *FeatureList) []string {
	var known []string
	for k, v := range *f {
		if v.HiddenInHelpText {
			continue
		}

		pre := ""
		if v.PreRelease != featuregate.GA {
			pre = fmt.Sprintf("%s - ", v.PreRelease)
		}
		known = append(known, fmt.Sprintf("%s=true|false (%sdefault=%t)", k, pre, v.Default))
	}
	sort.Strings(known)
	return known
}

// NewFeatureGate parses a string of the form "key1=value1,key2=value2,..." into a
// map[string]bool of known keys or returns an error.
func NewFeatureGate(f *FeatureList, value string) (map[string]bool, error) {
	featureGate := map[string]bool{}
	for _, s := range strings.Split(value, ",") {
		if len(s) == 0 {
			continue
		}

		arr := strings.SplitN(s, "=", 2)
		if len(arr) != 2 {
			return nil, errors.Errorf("missing bool value for feature-gate key:%s", s)
		}

		k := strings.TrimSpace(arr[0])
		v := strings.TrimSpace(arr[1])

		featureSpec, ok := (*f)[k]
		if !ok {
			return nil, errors.Errorf("unrecognized feature-gate key: %s", k)
		}

		if featureSpec.PreRelease == featuregate.Deprecated {
			return nil, errors.Errorf("feature-gate key is deprecated: %s", k)
		}

		boolValue, err := strconv.ParseBool(v)
		if err != nil {
			return nil, errors.Errorf("invalid value %v for feature-gate key: %s, use true|false instead", v, k)
		}
		featureGate[k] = boolValue
	}

	return featureGate, nil
}

// CheckDeprecatedFlags takes a list of existing feature gate flags and validates against the current feature flag set.
// It used during upgrades for ensuring consistency of feature gates used in an existing cluster, that might
// be created with a previous version of kubeadm, with the set of features currently supported by kubeadm
func CheckDeprecatedFlags(f *FeatureList, features map[string]bool) map[string]string {
	deprecatedMsg := map[string]string{}
	for k := range features {
		featureSpec, ok := (*f)[k]
		if !ok {
			// This case should never happen, it is implemented only as a sentinel
			// for removal of flags executed when flags are still in use (always before deprecate, then after one cycle remove)
			deprecatedMsg[k] = fmt.Sprintf("Unknown feature gate flag: %s", k)
		}

		if featureSpec.PreRelease == featuregate.Deprecated {
			if _, ok := deprecatedMsg[k]; !ok {
				deprecatedMsg[k] = featureSpec.DeprecationMessage
			}
		}
	}

	return deprecatedMsg
}
<commit_msg>Add a feature-gate to kubeadm to enable/disable Rootless control-plane.<commit_after>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package features

import (
	"fmt"
	"sort"
	"strconv"
	"strings"

	"github.com/pkg/errors"
	"k8s.io/apimachinery/pkg/util/version"
	"k8s.io/component-base/featuregate"
)

const (
	// IPv6DualStack is expected to be beta in v1.21
	IPv6DualStack = "IPv6DualStack"
	// PublicKeysECDSA is expected to be alpha in v1.19
	PublicKeysECDSA = "PublicKeysECDSA"
	// RootlessControlPlane is expected to be in alpha in v1.22
	RootlessControlPlane = "RootlessControlPlane"
)

// InitFeatureGates are the default feature gates for the init command
var InitFeatureGates = FeatureList{
	IPv6DualStack:        {FeatureSpec: featuregate.FeatureSpec{Default: true, PreRelease: featuregate.Beta}},
	PublicKeysECDSA:      {FeatureSpec: featuregate.FeatureSpec{Default: false, PreRelease: featuregate.Alpha}},
	RootlessControlPlane: {FeatureSpec: featuregate.FeatureSpec{Default: false, PreRelease: featuregate.Alpha}},
}

// Feature represents a feature being gated
type Feature struct {
	featuregate.FeatureSpec
	MinimumVersion     *version.Version
	HiddenInHelpText   bool
	DeprecationMessage string
}

// FeatureList represents a list of feature gates
type FeatureList map[string]Feature

// ValidateVersion ensures that a feature gate list is compatible with the chosen Kubernetes version
func ValidateVersion(allFeatures FeatureList, requestedFeatures map[string]bool, requestedVersion string) error {
	if requestedVersion == "" {
		return nil
	}
	parsedExpVersion, err := version.ParseSemantic(requestedVersion)
	if err != nil {
		return errors.Wrapf(err, "error parsing version %s", requestedVersion)
	}
	for k := range requestedFeatures {
		if minVersion := allFeatures[k].MinimumVersion; minVersion != nil {
			if !parsedExpVersion.AtLeast(minVersion) {
				return errors.Errorf(
					"the requested Kubernetes version (%s) is incompatible with the %s feature gate, which needs %s as a minimum",
					requestedVersion, k, minVersion)
			}
		}
	}
	return nil
}

// Enabled indicates whether a feature name has been enabled
func Enabled(featureList map[string]bool, featureName string) bool {
	if enabled, ok := featureList[string(featureName)]; ok {
		return enabled
	}
	return InitFeatureGates[string(featureName)].Default
}

// Supports indicates whether a feature name is supported on the given
// feature set
func Supports(featureList FeatureList, featureName string) bool {
	for k, v := range featureList {
		if featureName == string(k) {
			return v.PreRelease != featuregate.Deprecated
		}
	}
	return false
}

// Keys returns a slice of feature names for a given feature set
func Keys(featureList FeatureList) []string {
	var list []string
	for k := range featureList {
		list = append(list, string(k))
	}
	return list
}

// KnownFeatures returns a slice of strings describing the FeatureList features.
func KnownFeatures(f *FeatureList) []string {
	var known []string
	for k, v := range *f {
		if v.HiddenInHelpText {
			continue
		}

		pre := ""
		if v.PreRelease != featuregate.GA {
			pre = fmt.Sprintf("%s - ", v.PreRelease)
		}
		known = append(known, fmt.Sprintf("%s=true|false (%sdefault=%t)", k, pre, v.Default))
	}
	sort.Strings(known)
	return known
}

// NewFeatureGate parses a string of the form "key1=value1,key2=value2,..." into a
// map[string]bool of known keys or returns an error.
func NewFeatureGate(f *FeatureList, value string) (map[string]bool, error) {
	featureGate := map[string]bool{}
	for _, s := range strings.Split(value, ",") {
		if len(s) == 0 {
			continue
		}

		arr := strings.SplitN(s, "=", 2)
		if len(arr) != 2 {
			return nil, errors.Errorf("missing bool value for feature-gate key:%s", s)
		}

		k := strings.TrimSpace(arr[0])
		v := strings.TrimSpace(arr[1])

		featureSpec, ok := (*f)[k]
		if !ok {
			return nil, errors.Errorf("unrecognized feature-gate key: %s", k)
		}

		if featureSpec.PreRelease == featuregate.Deprecated {
			return nil, errors.Errorf("feature-gate key is deprecated: %s", k)
		}

		boolValue, err := strconv.ParseBool(v)
		if err != nil {
			return nil, errors.Errorf("invalid value %v for feature-gate key: %s, use true|false instead", v, k)
		}
		featureGate[k] = boolValue
	}

	return featureGate, nil
}

// CheckDeprecatedFlags takes a list of existing feature gate flags and validates against the current feature flag set.
// It used during upgrades for ensuring consistency of feature gates used in an existing cluster, that might
// be created with a previous version of kubeadm, with the set of features currently supported by kubeadm
func CheckDeprecatedFlags(f *FeatureList, features map[string]bool) map[string]string {
	deprecatedMsg := map[string]string{}
	for k := range features {
		featureSpec, ok := (*f)[k]
		if !ok {
			// This case should never happen, it is implemented only as a sentinel
			// for removal of flags executed when flags are still in use (always before deprecate, then after one cycle remove)
			deprecatedMsg[k] = fmt.Sprintf("Unknown feature gate flag: %s", k)
		}

		if featureSpec.PreRelease == featuregate.Deprecated {
			if _, ok := deprecatedMsg[k]; !ok {
				deprecatedMsg[k] = featureSpec.DeprecationMessage
			}
		}
	}

	return deprecatedMsg
}
<|endoftext|>