<commit_before>package goridge

import (
	"bytes"
	"io"
	"net/rpc"
	"sync"

	j "github.com/json-iterator/go"
	"github.com/spiral/errors"
	"go.uber.org/multierr"
)

var json = j.ConfigCompatibleWithStandardLibrary

// Codec represent net/rpc bridge over Goridge socket relay.
type Codec struct {
	relay  Relay
	closed bool
	frame  *Frame
	codec  sync.Map
}

// NewCodec initiates new server rpc codec over socket connection.
func NewCodec(rwc io.ReadWriteCloser) *Codec {
	return &Codec{
		relay: NewSocketRelay(rwc),
		codec: sync.Map{},
	}
}

// NewCodecWithRelay initiates new server rpc codec with a relay of choice.
func NewCodecWithRelay(relay Relay) *Codec {
	return &Codec{relay: relay}
}

// WriteResponse marshals response, byte slice or error to remote party.
func (c *Codec) WriteResponse(r *rpc.Response, body interface{}) error {
	const op = errors.Op("codec: write response")
	frame := NewFrame()

	// SEQ_ID + METHOD_NAME_LEN
	frame.WriteOptions(uint32(r.Seq), uint32(len(r.ServiceMethod)))
	// Write protocol version
	frame.WriteVersion(VERSION_1)

	// load and delete associated codec to not waste memory
	// because we write it to the frame and don't need more information about it
	// as for go.14, Load and Delete are separate methods
	codec, ok := c.codec.Load(r.Seq)
	if !ok {
		// fallback codec
		frame.WriteFlags(CODEC_GOB)
	} else {
		frame.WriteFlags(codec.(FrameFlag))
	}

	// delete the key
	c.codec.Delete(r.Seq)

	// initialize buffer
	buf := new(bytes.Buffer)
	// writeServiceMethod to the buffer
	buf.WriteString(r.ServiceMethod)

	// if error returned, we sending it via relay and return error from WriteResponse
	if r.Error != "" {
		// Append error flag
		return c.handleError(r, frame, buf, errors.Str(r.Error))
	}

	// read flag previously written
	// TODO might be better to save it to local variable
	flags := frame.ReadFlags()

	switch {
	case flags&byte(CODEC_JSON) != 0:
		err := encodeJSON(buf, body)
		if err != nil {
			return c.handleError(r, frame, buf, err)
		}
		// send buffer
		return c.sendBuf(frame, buf)
	case flags&byte(CODEC_MSGPACK) != 0:
		err := encodeMsgPack(buf, body)
		if err != nil {
			return c.handleError(r, frame, buf, err)
		}
		// send buffer
		return c.sendBuf(frame, buf)
	case flags&byte(CODEC_RAW) != 0:
		err := encodeRaw(buf, body)
		if err != nil {
			return c.handleError(r, frame, buf, err)
		}
		// send buffer
		return c.sendBuf(frame, buf)
	case flags&byte(CODEC_GOB) != 0:
		err := encodeGob(buf, body)
		if err != nil {
			return c.handleError(r, frame, buf, err)
		}
		// send buffer
		return c.sendBuf(frame, buf)
	default:
		return c.handleError(r, frame, buf, errors.E(op, errors.Str("unknown codec")))
	}
}

func (c *Codec) sendBuf(frame *Frame, buf *bytes.Buffer) error {
	frame.WritePayloadLen(uint32(buf.Len()))
	frame.WritePayload(buf.Bytes())

	frame.WriteCRC()
	return c.relay.Send(frame)
}

func (c *Codec) handleError(r *rpc.Response, frame *Frame, buf *bytes.Buffer, err error) error {
	// just to be sure, remove all data from buffer and write new
	buf.Truncate(0)
	// write all possible errors
	err = multierr.Append(err, errors.Str(r.Error))
	buf.WriteString(r.ServiceMethod)

	const op = errors.Op("handle codec error")
	frame.WriteFlags(ERROR)
	// error should be here
	if err != nil {
		buf.WriteString(err.Error())
	}
	frame.WritePayloadLen(uint32(buf.Len()))
	frame.WritePayload(buf.Bytes())

	frame.WriteCRC()
	_ = c.relay.Send(frame)
	return errors.E(op, errors.Str(r.Error))
}

// ReadRequestHeader receives frame with options
// options should have 2 values
// [0] - integer, sequence ID
// [1] - integer, offset for method name
// For example:
// 15Test.Payload
// SEQ_ID: 15
// METHOD_LEN: 11 and we take 11bytes from the payload as method name
func (c *Codec) ReadRequestHeader(r *rpc.Request) error {
	const op = errors.Op("codec: read request header")
	frame := NewFrame()
	err := c.relay.Receive(frame)
	if err != nil {
		return err
	}

	// opts[0] sequence ID
	// opts[1] service method name offset from payload in bytes
	opts := frame.ReadOptions()
	if len(opts) != 2 {
		return errors.E(op, errors.Str("should be 2 options. SEQ_ID and METHOD_LEN"))
	}

	r.Seq = uint64(opts[0])
	r.ServiceMethod = string(frame.Payload()[:opts[1]])
	c.frame = frame
	return c.storeCodec(r, frame.ReadFlags())
}

func (c *Codec) storeCodec(r *rpc.Request, flag byte) error {
	switch {
	case flag&byte(CODEC_JSON) != 0:
		c.codec.Store(r.Seq, CODEC_JSON)
	case flag&byte(CODEC_RAW) != 0:
		c.codec.Store(r.Seq, CODEC_RAW)
	case flag&byte(CODEC_MSGPACK) != 0:
		c.codec.Store(r.Seq, CODEC_MSGPACK)
	case flag&byte(CODEC_GOB) != 0:
		c.codec.Store(r.Seq, CODEC_GOB)
	default:
		c.codec.Store(r.Seq, CODEC_GOB)
	}

	return nil
}

// ReadRequestBody fetches prefixed body data and automatically unmarshal it as json. RawBody flag will populate
// []byte lice argument for rpc method.
func (c *Codec) ReadRequestBody(out interface{}) error {
	const op = errors.Op("codec read request body")
	if out == nil {
		return nil
	}

	defer func() {
		c.frame = nil
	}()

	flags := c.frame.ReadFlags()

	switch {
	case flags&byte(CODEC_JSON) != 0:
		return decodeJSON(out, c.frame)
	case flags&byte(CODEC_GOB) != 0:
		return decodeGob(out, c.frame)
	case flags&byte(CODEC_RAW) != 0:
		return decodeRaw(out, c.frame)
	case flags&byte(CODEC_MSGPACK) != 0:
		return decodeMsgPack(out, c.frame)
	default:
		return errors.E(op, errors.Str("unknown decoder used in frame"))
	}
}

// Close underlying socket.
func (c *Codec) Close() error {
	if c.closed {
		return nil
	}

	c.closed = true
	return c.relay.Close()
}
<commit_msg>Add comments<commit_after>package goridge

import (
	"bytes"
	"io"
	"net/rpc"
	"sync"

	j "github.com/json-iterator/go"
	"github.com/spiral/errors"
	"go.uber.org/multierr"
)

var json = j.ConfigCompatibleWithStandardLibrary

// Codec represent net/rpc bridge over Goridge socket relay.
type Codec struct {
	relay  Relay
	closed bool
	frame  *Frame
	codec  sync.Map
}

// NewCodec initiates new server rpc codec over socket connection.
func NewCodec(rwc io.ReadWriteCloser) *Codec {
	return &Codec{
		relay: NewSocketRelay(rwc),
		codec: sync.Map{},
	}
}

// NewCodecWithRelay initiates new server rpc codec with a relay of choice.
func NewCodecWithRelay(relay Relay) *Codec {
	return &Codec{relay: relay}
}

// WriteResponse marshals response, byte slice or error to remote party.
func (c *Codec) WriteResponse(r *rpc.Response, body interface{}) error {
	const op = errors.Op("codec: write response")
	frame := NewFrame()

	// SEQ_ID + METHOD_NAME_LEN
	frame.WriteOptions(uint32(r.Seq), uint32(len(r.ServiceMethod)))
	// Write protocol version
	frame.WriteVersion(VERSION_1)

	// load and delete associated codec to not waste memory
	// because we write it to the frame and don't need more information about it
	// as for go.14, Load and Delete are separate methods
	codec, ok := c.codec.Load(r.Seq)
	if !ok {
		// fallback codec
		frame.WriteFlags(CODEC_GOB)
	} else {
		frame.WriteFlags(codec.(FrameFlag))
	}

	// delete the key
	c.codec.Delete(r.Seq)

	// initialize buffer
	buf := new(bytes.Buffer)
	// writeServiceMethod to the buffer
	buf.WriteString(r.ServiceMethod)

	// if error returned, we sending it via relay and return error from WriteResponse
	if r.Error != "" {
		// Append error flag
		return c.handleError(r, frame, buf, errors.Str(r.Error))
	}

	// read flag previously written
	// TODO might be better to save it to local variable
	flags := frame.ReadFlags()

	switch {
	case flags&byte(CODEC_JSON) != 0:
		err := encodeJSON(buf, body)
		if err != nil {
			return c.handleError(r, frame, buf, err)
		}
		// send buffer
		return c.sendBuf(frame, buf)
	case flags&byte(CODEC_MSGPACK) != 0:
		err := encodeMsgPack(buf, body)
		if err != nil {
			return c.handleError(r, frame, buf, err)
		}
		// send buffer
		return c.sendBuf(frame, buf)
	case flags&byte(CODEC_RAW) != 0:
		err := encodeRaw(buf, body)
		if err != nil {
			return c.handleError(r, frame, buf, err)
		}
		// send buffer
		return c.sendBuf(frame, buf)
	case flags&byte(CODEC_GOB) != 0:
		err := encodeGob(buf, body)
		if err != nil {
			return c.handleError(r, frame, buf, err)
		}
		// send buffer
		return c.sendBuf(frame, buf)
	default:
		return c.handleError(r, frame, buf, errors.E(op, errors.Str("unknown codec")))
	}
}

func (c *Codec) sendBuf(frame *Frame, buf *bytes.Buffer) error {
	frame.WritePayloadLen(uint32(buf.Len()))
	frame.WritePayload(buf.Bytes())

	frame.WriteCRC()
	return c.relay.Send(frame)
}

func (c *Codec) handleError(r *rpc.Response, frame *Frame, buf *bytes.Buffer, err error) error {
	// just to be sure, remove all data from buffer and write new
	buf.Truncate(0)
	// write all possible errors
	err = multierr.Append(err, errors.Str(r.Error))
	buf.WriteString(r.ServiceMethod)

	const op = errors.Op("handle codec error")
	frame.WriteFlags(ERROR)
	// error should be here
	if err != nil {
		buf.WriteString(err.Error())
	}
	frame.WritePayloadLen(uint32(buf.Len()))
	frame.WritePayload(buf.Bytes())

	frame.WriteCRC()
	_ = c.relay.Send(frame)
	return errors.E(op, errors.Str(r.Error))
}

// ReadRequestHeader receives frame with options
// options should have 2 values
// [0] - integer, sequence ID
// [1] - integer, offset for method name
// For example:
// 15Test.Payload
// SEQ_ID: 15
// METHOD_LEN: 12 and we take 12 bytes from the payload as method name
func (c *Codec) ReadRequestHeader(r *rpc.Request) error {
	const op = errors.Op("codec: read request header")
	frame := NewFrame()
	err := c.relay.Receive(frame)
	if err != nil {
		return err
	}

	// opts[0] sequence ID
	// opts[1] service method name offset from payload in bytes
	opts := frame.ReadOptions()
	if len(opts) != 2 {
		return errors.E(op, errors.Str("should be 2 options. SEQ_ID and METHOD_LEN"))
	}

	r.Seq = uint64(opts[0])
	r.ServiceMethod = string(frame.Payload()[:opts[1]])
	c.frame = frame
	return c.storeCodec(r, frame.ReadFlags())
}

func (c *Codec) storeCodec(r *rpc.Request, flag byte) error {
	switch {
	case flag&byte(CODEC_JSON) != 0:
		c.codec.Store(r.Seq, CODEC_JSON)
	case flag&byte(CODEC_RAW) != 0:
		c.codec.Store(r.Seq, CODEC_RAW)
	case flag&byte(CODEC_MSGPACK) != 0:
		c.codec.Store(r.Seq, CODEC_MSGPACK)
	case flag&byte(CODEC_GOB) != 0:
		c.codec.Store(r.Seq, CODEC_GOB)
	default:
		c.codec.Store(r.Seq, CODEC_GOB)
	}

	return nil
}

// ReadRequestBody fetches prefixed body data and automatically unmarshal it as json. RawBody flag will populate
// []byte lice argument for rpc method.
func (c *Codec) ReadRequestBody(out interface{}) error {
	const op = errors.Op("codec read request body")
	if out == nil {
		return nil
	}

	defer func() {
		c.frame = nil
	}()

	flags := c.frame.ReadFlags()

	switch {
	case flags&byte(CODEC_JSON) != 0:
		return decodeJSON(out, c.frame)
	case flags&byte(CODEC_GOB) != 0:
		return decodeGob(out, c.frame)
	case flags&byte(CODEC_RAW) != 0:
		return decodeRaw(out, c.frame)
	case flags&byte(CODEC_MSGPACK) != 0:
		return decodeMsgPack(out, c.frame)
	default:
		return errors.E(op, errors.Str("unknown decoder used in frame"))
	}
}

// Close underlying socket.
func (c *Codec) Close() error {
	if c.closed {
		return nil
	}

	c.closed = true
	return c.relay.Close()
}
<|endoftext|>


<commit_before>package sphinx

import (
	"crypto/elliptic"
	"math/big"
	"crypto/aes"
	"crypto/cipher"
	"strings"
	"anonymous-messaging/config"
	"bytes"
	"errors"

	"github.com/protobuf/proto"
)

const (
	K = 16
	R = 5
	HEADERLENGTH = 192
	LAST_HOP_FLAG = "\xF0"
	RELAY_FLAG = "\xF1"

)

var curve = elliptic.P224()

func (p *SphinxPacket) Bytes() ([]byte, error) {
	b, err := proto.Marshal(p)
	if err != nil {
		return nil, err
	}
	return b, nil
}

func PacketFromBytes(bytes []byte) (SphinxPacket, error) {
	var packet SphinxPacket
	err := proto.Unmarshal(bytes, &packet)

	if err != nil {
		return SphinxPacket{}, err
	}

	return packet, nil
}

func (r *RoutingInfo) Bytes() ([]byte, error) {
	b, err := proto.Marshal(r)
	if err != nil{
		return nil, err
	}
	return b, nil
}

func RoutingInfoFromBytes(bytes []byte) (RoutingInfo, error) {

	var finalHopReconstruct RoutingInfo

	err := proto.Unmarshal(bytes, &finalHopReconstruct)

	if err != nil {
		return RoutingInfo{}, err
	}

	return finalHopReconstruct, nil
}

func PackForwardMessage(curve elliptic.Curve, path config.E2EPath, delays []float64, message string) (SphinxPacket, error){
	nodes := []config.MixPubs{path.IngressProvider}
	nodes = append(nodes, path.Mixes...)
	nodes = append(nodes, path.EgressProvider)
	dest := path.Recipient

	asb, header, err := createHeader(curve, nodes, delays, dest)
	if err != nil{
		return SphinxPacket{}, err
	}

	payload, err := encapsulateContent(asb, message)
	if err != nil{
		return SphinxPacket{}, err
	}

	return SphinxPacket{Hdr: &header, Pld: payload}, nil
}

func createHeader(curve elliptic.Curve, nodes []config.MixPubs, delays []float64, dest config.ClientPubs) ([]HeaderInitials, Header, error){

	x, err := randomBigInt(curve.Params())

	if err != nil {
		return nil, Header{}, err
	}

	asb, err := getSharedSecrets(curve, nodes, x)
	if err != nil{
		return nil, Header{}, err
	}

	var commands []Commands
	for i, _ := range nodes {
		var c Commands
		if i == len(nodes) - 1 {
			c = Commands{Delay: delays[i], Flag: LAST_HOP_FLAG}
		} else {
			c = Commands{Delay: delays[i], Flag: RELAY_FLAG}
		}
		commands = append(commands, c)
	}

	header, err := encapsulateHeader(asb, nodes, commands, dest)
	if err !=nil{
		return nil, Header{}, err
	}
	return asb, header, nil

}

func encapsulateContent(asb []HeaderInitials, message string) ([]byte, error) {

	enc := []byte(message)
	err := error(nil)
	for i := len(asb) - 1; i >= 0; i-- {
		sharedKey := KDF(asb[i].SecretHash)
		enc, err = AES_CTR(sharedKey, enc)
		if err != nil{
			return nil, err
		}

	}
	return enc, nil
}

func encapsulateHeader(asb []HeaderInitials, nodes []config.MixPubs, commands []Commands, destination config.ClientPubs) (Header, error){

	finalHop := RoutingInfo{NextHop: &Hop{Id: destination.Id, Address: destination.Host + ":" + destination.Port, PubKey: []byte{}}, RoutingCommands: &commands[len(commands) - 1], NextHopMetaData: []byte{}, Mac: []byte{}}

	finalHopBytes, err := finalHop.Bytes()
	if err != nil{
		return Header{}, err
	}

	encFinalHop, err := AES_CTR(KDF(asb[len(asb)-1].SecretHash), finalHopBytes)
	if err != nil{
		return Header{}, err
	}

	mac := computeMac(KDF(asb[len(asb)-1].SecretHash) , encFinalHop)

	routingCommands := [][]byte{encFinalHop}

	var encRouting []byte
	for i := len(nodes)-2; i >= 0; i-- {
		nextNode := nodes[i+1]
		routing := RoutingInfo{NextHop: &Hop{Id: nextNode.Id, Address: nextNode.Host+":"+nextNode.Port, PubKey: nodes[i+1].PubKey}, RoutingCommands: &commands[i], NextHopMetaData: routingCommands[len(routingCommands)-1], Mac: mac}

		encKey := KDF(asb[i].SecretHash)
		routingBytes, err := routing.Bytes()

		if err != nil{
			return Header{}, err
		}

		encRouting, err = AES_CTR(encKey, routingBytes)
		if err != nil{
			return Header{}, err
		}

		routingCommands = append(routingCommands, encRouting)
		mac = computeMac(KDF(asb[i].SecretHash) , encRouting)

	}
	return Header{Alpha: asb[0].Alpha, Beta: encRouting, Mac : mac}, nil

}

func getSharedSecrets(curve elliptic.Curve, nodes []config.MixPubs, initialVal big.Int) ([]HeaderInitials, error){

	blindFactors := []big.Int{initialVal}
	var tuples []HeaderInitials

	for _, n := range nodes {

		alpha := expo_group_base(curve, blindFactors)

		s := expo(n.PubKey, blindFactors)
		aes_s := KDF(s)

		blinder, err := computeBlindingFactor(curve, aes_s)
		if err != nil{
			return nil, err
		}

		blindFactors = append(blindFactors, *blinder)
		tuples = append(tuples, HeaderInitials{Alpha:alpha, Secret: s, Blinder: blinder.Bytes(), SecretHash: aes_s})
	}
	return tuples, nil

}

func computeFillers(nodes []config.MixPubs, tuples []HeaderInitials) (string, error) {

	filler := ""
	minLen := HEADERLENGTH - 32
	for i := 1; i < len(nodes); i++ {
		base := filler + strings.Repeat("\x00", K)
		kx, err := computeSharedSecretHash(tuples[i-1].SecretHash, []byte("hrhohrhohrhohrho"))
		if err != nil{
			return "", err
		}
		mx := strings.Repeat("\x00", minLen) + base

		xorVal, err := AES_CTR([]byte(kx), []byte(mx))
		if err != nil{
			return "", err
		}

		filler = BytesToString(xorVal)
		filler = filler[minLen:]

		minLen = minLen - K
	}

	return filler, nil

}

func computeBlindingFactor(curve elliptic.Curve, key []byte) (*big.Int, error) {
	iv := []byte("initialvector000")
	blinderBytes, err := computeSharedSecretHash(key, iv)

	if err != nil{
		return &big.Int{}, err
	}

	return bytesToBigNum(curve, blinderBytes), nil
}

func computeSharedSecretHash(key []byte, iv []byte) ([]byte, error) {
	aesCipher, err := aes.NewCipher(key)

	if err != nil {
		return nil, err
	}

	stream := cipher.NewCTR(aesCipher, iv)
	plaintext := []byte("0000000000000000")

	ciphertext := make([]byte, len(plaintext))
	stream.XORKeyStream(ciphertext, plaintext)

	return ciphertext, nil
}

func ProcessSphinxPacket(packetBytes []byte, privKey []byte) (Hop, Commands, []byte, error) {

	packet, err := PacketFromBytes(packetBytes)

	if err != nil {
		return Hop{}, Commands{}, nil, err
	}

	hop, commands, newHeader, err := ProcessSphinxHeader(*packet.Hdr, privKey)
	if err != nil {
		return Hop{}, Commands{}, nil, err
	}

	newPayload, err := ProcessSphinxPayload(packet.Hdr.Alpha, packet.Pld, privKey)
	if err != nil {
		return Hop{}, Commands{}, nil, err
	}

	newPacket := SphinxPacket{Hdr: &newHeader, Pld: newPayload}
	newPacketBytes, err := newPacket.Bytes()
	if err != nil{
		return Hop{}, Commands{}, nil, err
	}

	return hop, commands, newPacketBytes, nil
}

func ProcessSphinxHeader(packet Header, privKey []byte) (Hop, Commands, Header, error) {

	alpha := packet.Alpha
	beta := packet.Beta
	mac := packet.Mac

	curve := elliptic.P224()
	alphaX, alphaY := elliptic.Unmarshal(curve, alpha)
	sharedSecretX, sharedSecretY:= curve.Params().ScalarMult(alphaX, alphaY, privKey)
	sharedSecret := elliptic.Marshal(curve, sharedSecretX, sharedSecretY)

	aes_s := KDF(sharedSecret)
	encKey := KDF(aes_s)

	recomputedMac := computeMac(KDF(aes_s) , beta)

	if bytes.Compare(recomputedMac, mac) != 0 {
		return Hop{}, Commands{}, Header{}, errors.New("Packet processing error: MACs are not matching")
	}

	blinder, err := computeBlindingFactor(curve, aes_s)
	if err != nil{
		return Hop{}, Commands{}, Header{}, err
	}

	newAlphaX, newAlphaY := curve.Params().ScalarMult(alphaX, alphaY, blinder.Bytes())
	newAlpha := elliptic.Marshal(curve, newAlphaX, newAlphaY)

	decBeta, err := AES_CTR(encKey, beta)
	if err != nil{
		return Hop{}, Commands{}, Header{}, err
	}

	routingInfo, err := RoutingInfoFromBytes(decBeta)
	if err != nil{
		return Hop{}, Commands{}, Header{}, err
	}
	nextHop, commands, nextBeta, nextMac := readBeta(routingInfo)

	return nextHop, commands, Header{Alpha: newAlpha, Beta: nextBeta, Mac: nextMac}, nil
}

func readBeta(beta RoutingInfo) (Hop, Commands, []byte, []byte){
	nextHop := *beta.NextHop
	commands := *beta.RoutingCommands
	nextBeta := beta.NextHopMetaData
	nextMac := beta.Mac

	return nextHop, commands, nextBeta, nextMac
}

func ProcessSphinxPayload(alpha []byte, payload []byte, privKey []byte) ([]byte, error) {

	curve := elliptic.P224()
	alphaX, alphaY := elliptic.Unmarshal(curve, alpha)
	sharedSecretX, sharedSecretY:= curve.Params().ScalarMult(alphaX, alphaY, privKey)
	sharedSecret := elliptic.Marshal(curve, sharedSecretX, sharedSecretY)

	aes_s := KDF(sharedSecret)
	decKey := KDF(aes_s)

	decPayload, err := AES_CTR(decKey, payload)
	if err != nil{
		return nil, err
	}

	return decPayload, nil
}<commit_msg>Added documentation for sphinx<commit_after>/*	Package sphinx implements the library of a cryptographic packet format,
	which can be used to secure the content as well as the metadata of the transported
    messages.
 */

package sphinx

import (
	"crypto/elliptic"
	"math/big"
	"crypto/aes"
	"crypto/cipher"
	"strings"
	"anonymous-messaging/config"
	"bytes"
	"errors"

	"github.com/protobuf/proto"
)

const (
	K = 16
	R = 5
	HEADERLENGTH = 192
	LAST_HOP_FLAG = "\xF0"
	RELAY_FLAG = "\xF1"

)

var curve = elliptic.P224()

// Bytes parses the sphinx packet into byte array using the proto Marshal serialization.
func (p *SphinxPacket) Bytes() ([]byte, error) {
	b, err := proto.Marshal(p)
	if err != nil {
		return nil, err
	}
	return b, nil
}

/* PacketFromBytes converts the byte representation of a Sphinx packet,
serialized by Bytes, into the Sphinx packet struct. If the byte representation
does not match the struct an error is returned.
*/
func PacketFromBytes(bytes []byte) (SphinxPacket, error) {
	var packet SphinxPacket
	err := proto.Unmarshal(bytes, &packet)

	if err != nil {
		return SphinxPacket{}, err
	}

	return packet, nil
}

// Bytes parses the RoutingInfo struct into byte array using the proto Marshal serialization.
func (r *RoutingInfo) Bytes() ([]byte, error) {
	b, err := proto.Marshal(r)
	if err != nil{
		return nil, err
	}
	return b, nil
}

/* RoutingInfoFromBytes converts the byte representation of a RoutingInformation,
serialized by Bytes, into the RoutingInformation struct. If the byte representation
does not match the struct an error is returned.
*/
func RoutingInfoFromBytes(bytes []byte) (RoutingInfo, error) {

	var finalHopReconstruct RoutingInfo

	err := proto.Unmarshal(bytes, &finalHopReconstruct)

	if err != nil {
		return RoutingInfo{}, err
	}

	return finalHopReconstruct, nil
}

/*
	PackForwardMessage encapsulates the given message into the cryptographic Sphinx packet format.
	As arguments the function takes the path, consisting of the sequence of nodes the packet should traverse
	and the destination of the message, a set of delays and the information about the curve used to perform cryptographic
	operations.
    In order to encapsulate the message PackForwardMessage computes two parts of the packet - the header and
	the encrypted payload. If creating of any of the packet block failed, an error is returned. Otherwise,
	a Sphinx packet format is returned.
 */
func PackForwardMessage(curve elliptic.Curve, path config.E2EPath, delays []float64, message string) (SphinxPacket, error){
	nodes := []config.MixPubs{path.IngressProvider}
	nodes = append(nodes, path.Mixes...)
	nodes = append(nodes, path.EgressProvider)
	dest := path.Recipient

	asb, header, err := createHeader(curve, nodes, delays, dest)
	if err != nil{
		return SphinxPacket{}, err
	}

	payload, err := encapsulateContent(asb, message)
	if err != nil{
		return SphinxPacket{}, err
	}

	return SphinxPacket{Hdr: &header, Pld: payload}, nil
}

/*
	createHeader builds the Sphinx packet header, consisting of three parts: the public element, the encapsulated routing information
	and the message authentication code. createHeader layer encapsulates the routing information for each given node. The routing information
    contains information where the packet should be forwarded next, how long it should be delayed by the node, and if relevant additional
    auxiliary information. The message authentication code allows to detect tagging attacks.
	createHeader computes the secret shared key between sender and the nodes and destination, which are used as keys for encryption.

	createHeader returns the header and a list of the initial elements, used for creating the header. If any operation was unsuccessful
	createHeader returns an error.
 */
func createHeader(curve elliptic.Curve, nodes []config.MixPubs, delays []float64, dest config.ClientPubs) ([]HeaderInitials, Header, error){

	x, err := randomBigInt(curve.Params())

	if err != nil {
		return nil, Header{}, err
	}

	asb, err := getSharedSecrets(curve, nodes, x)
	if err != nil{
		return nil, Header{}, err
	}

	var commands []Commands
	for i, _ := range nodes {
		var c Commands
		if i == len(nodes) - 1 {
			c = Commands{Delay: delays[i], Flag: LAST_HOP_FLAG}
		} else {
			c = Commands{Delay: delays[i], Flag: RELAY_FLAG}
		}
		commands = append(commands, c)
	}

	header, err := encapsulateHeader(asb, nodes, commands, dest)
	if err !=nil{
		return nil, Header{}, err
	}
	return asb, header, nil

}

/*
	encapsulateHeader layer encrypts the meta-data of the packet, containing information about the
    sequence of nodes the packet should traverse before reaching the destination, and message authentication codes,
	given the pre-computed shared keys which are used for encryption.

	encapsulateHeader returns the Header, or an error if any internal cryptographic of parsing operation failed.
 */

func encapsulateHeader(asb []HeaderInitials, nodes []config.MixPubs, commands []Commands, destination config.ClientPubs) (Header, error){

	finalHop := RoutingInfo{NextHop: &Hop{Id: destination.Id, Address: destination.Host + ":" + destination.Port, PubKey: []byte{}}, RoutingCommands: &commands[len(commands) - 1], NextHopMetaData: []byte{}, Mac: []byte{}}

	finalHopBytes, err := finalHop.Bytes()
	if err != nil{
		return Header{}, err
	}

	encFinalHop, err := AES_CTR(KDF(asb[len(asb)-1].SecretHash), finalHopBytes)
	if err != nil{
		return Header{}, err
	}

	mac := computeMac(KDF(asb[len(asb)-1].SecretHash) , encFinalHop)

	routingCommands := [][]byte{encFinalHop}

	var encRouting []byte
	for i := len(nodes)-2; i >= 0; i-- {
		nextNode := nodes[i+1]
		routing := RoutingInfo{NextHop: &Hop{Id: nextNode.Id, Address: nextNode.Host+":"+nextNode.Port, PubKey: nodes[i+1].PubKey}, RoutingCommands: &commands[i], NextHopMetaData: routingCommands[len(routingCommands)-1], Mac: mac}

		encKey := KDF(asb[i].SecretHash)
		routingBytes, err := routing.Bytes()

		if err != nil{
			return Header{}, err
		}

		encRouting, err = AES_CTR(encKey, routingBytes)
		if err != nil{
			return Header{}, err
		}

		routingCommands = append(routingCommands, encRouting)
		mac = computeMac(KDF(asb[i].SecretHash) , encRouting)

	}
	return Header{Alpha: asb[0].Alpha, Beta: encRouting, Mac : mac}, nil

}

/*
	encapsulateContent layer encrypts the given messages using a set of shared keys
	and the AES_CTR encryption.
	encapsulateContent returns the encrypted payload in byte representation. If the AES_CTR
	encryption failed encapsulateContent returns an error.
 */
func encapsulateContent(asb []HeaderInitials, message string) ([]byte, error) {

	enc := []byte(message)
	err := error(nil)
	for i := len(asb) - 1; i >= 0; i-- {
		sharedKey := KDF(asb[i].SecretHash)
		enc, err = AES_CTR(sharedKey, enc)
		if err != nil{
			return nil, err
		}

	}
	return enc, nil
}

/*
	getSharedSecrets computes a sequence of HeaderInitial values, containing the initial elements,
    shared secrets and blinding factors for each node on the path. As input getSharedSecrets takes the initial
	secret value, the list of nodes, and the curve in which the cryptographic operations are performed.
    getSharedSecrets returns the list of computed HeaderInitials or an error.
 */
func getSharedSecrets(curve elliptic.Curve, nodes []config.MixPubs, initialVal big.Int) ([]HeaderInitials, error){

	blindFactors := []big.Int{initialVal}
	var tuples []HeaderInitials

	for _, n := range nodes {

		alpha := expo_group_base(curve, blindFactors)

		s := expo(n.PubKey, blindFactors)
		aes_s := KDF(s)

		blinder, err := computeBlindingFactor(curve, aes_s)
		if err != nil{
			return nil, err
		}

		blindFactors = append(blindFactors, *blinder)
		tuples = append(tuples, HeaderInitials{Alpha:alpha, Secret: s, Blinder: blinder.Bytes(), SecretHash: aes_s})
	}
	return tuples, nil

}

/*
	TO DO: computeFillers needs to be fixed
 */
func computeFillers(nodes []config.MixPubs, tuples []HeaderInitials) (string, error) {

	filler := ""
	minLen := HEADERLENGTH - 32
	for i := 1; i < len(nodes); i++ {
		base := filler + strings.Repeat("\x00", K)
		kx, err := computeSharedSecretHash(tuples[i-1].SecretHash, []byte("hrhohrhohrhohrho"))
		if err != nil{
			return "", err
		}
		mx := strings.Repeat("\x00", minLen) + base

		xorVal, err := AES_CTR([]byte(kx), []byte(mx))
		if err != nil{
			return "", err
		}

		filler = BytesToString(xorVal)
		filler = filler[minLen:]

		minLen = minLen - K
	}

	return filler, nil

}

/*
	computeBlindingFactor computes the blinding factor extracted from the
	shared secrets. Blinding factors allow both the sender and intermediate nodes
	recompute the shared keys used at each hop of the message processing.

	computeBlindingFactor returns a value of a blinding factor or an error.
 */

func computeBlindingFactor(curve elliptic.Curve, key []byte) (*big.Int, error) {
	iv := []byte("initialvector000")
	blinderBytes, err := computeSharedSecretHash(key, iv)

	if err != nil{
		return &big.Int{}, err
	}

	return bytesToBigNum(curve, blinderBytes), nil
}

/*

 */
func computeSharedSecretHash(key []byte, iv []byte) ([]byte, error) {
	aesCipher, err := aes.NewCipher(key)

	if err != nil {
		return nil, err
	}

	stream := cipher.NewCTR(aesCipher, iv)
	plaintext := []byte("0000000000000000")

	ciphertext := make([]byte, len(plaintext))
	stream.XORKeyStream(ciphertext, plaintext)

	return ciphertext, nil
}

/*
	ProcessSphinxPacket processes the sphinx packet using the given private key.
	ProcessSphinxPacket unwraps one layer of both the header and the payload encryption.
	ProcessSphinxPacket returns a new packet and the routing information which should
    be used by the processing node. If any cryptographic or parsing operation failed ProcessSphinxPacket
	returns an error.
 */
func ProcessSphinxPacket(packetBytes []byte, privKey []byte) (Hop, Commands, []byte, error) {

	packet, err := PacketFromBytes(packetBytes)

	if err != nil {
		return Hop{}, Commands{}, nil, err
	}

	hop, commands, newHeader, err := ProcessSphinxHeader(*packet.Hdr, privKey)
	if err != nil {
		return Hop{}, Commands{}, nil, err
	}

	newPayload, err := ProcessSphinxPayload(packet.Hdr.Alpha, packet.Pld, privKey)
	if err != nil {
		return Hop{}, Commands{}, nil, err
	}

	newPacket := SphinxPacket{Hdr: &newHeader, Pld: newPayload}
	newPacketBytes, err := newPacket.Bytes()
	if err != nil{
		return Hop{}, Commands{}, nil, err
	}

	return hop, commands, newPacketBytes, nil
}

/*
	ProcessSphinxHeader unwraps one layer of encryption from the header of a sphinx packet.
    ProcessSphinxHeader recomputes the shared key and checks whether the message authentication code is valid.
	If not, the packet is dropped and error is returned. If MAC checking was passed successfully ProcessSphinxHeader
	performs the AES_CTR decryption, recomputes the blinding factor and updates the init public element from the header.
	Next, ProcessSphinxHeader extracts the routing information from the decrypted packet and returns it, together with the
    updated init public element. If any crypto or parsing operation failed ProcessSphinxHeader returns an error.
 */

func ProcessSphinxHeader(packet Header, privKey []byte) (Hop, Commands, Header, error) {

	alpha := packet.Alpha
	beta := packet.Beta
	mac := packet.Mac

	curve := elliptic.P224()
	alphaX, alphaY := elliptic.Unmarshal(curve, alpha)
	sharedSecretX, sharedSecretY:= curve.Params().ScalarMult(alphaX, alphaY, privKey)
	sharedSecret := elliptic.Marshal(curve, sharedSecretX, sharedSecretY)

	aes_s := KDF(sharedSecret)
	encKey := KDF(aes_s)

	recomputedMac := computeMac(KDF(aes_s) , beta)

	if bytes.Compare(recomputedMac, mac) != 0 {
		return Hop{}, Commands{}, Header{}, errors.New("Packet processing error: MACs are not matching")
	}

	blinder, err := computeBlindingFactor(curve, aes_s)
	if err != nil{
		return Hop{}, Commands{}, Header{}, err
	}

	newAlphaX, newAlphaY := curve.Params().ScalarMult(alphaX, alphaY, blinder.Bytes())
	newAlpha := elliptic.Marshal(curve, newAlphaX, newAlphaY)

	decBeta, err := AES_CTR(encKey, beta)
	if err != nil{
		return Hop{}, Commands{}, Header{}, err
	}

	routingInfo, err := RoutingInfoFromBytes(decBeta)
	if err != nil{
		return Hop{}, Commands{}, Header{}, err
	}
	nextHop, commands, nextBeta, nextMac := readBeta(routingInfo)

	return nextHop, commands, Header{Alpha: newAlpha, Beta: nextBeta, Mac: nextMac}, nil
}

/*
	readBeta extracts all the fields from the RoutingInfo structure
 */
func readBeta(beta RoutingInfo) (Hop, Commands, []byte, []byte){
	nextHop := *beta.NextHop
	commands := *beta.RoutingCommands
	nextBeta := beta.NextHopMetaData
	nextMac := beta.Mac

	return nextHop, commands, nextBeta, nextMac
}

/*
	ProcessSphinxPayload unwraps a single layer of the encryption from the sphinx packet payload.
	ProcessSphinxPayload first recomputes the shared secret which is used to perform the AES_CTR decryption.
	ProcessSphinxPayload returnes the new packet payload or an error if the decryption failed.
 */

func ProcessSphinxPayload(alpha []byte, payload []byte, privKey []byte) ([]byte, error) {

	curve := elliptic.P224()
	alphaX, alphaY := elliptic.Unmarshal(curve, alpha)
	sharedSecretX, sharedSecretY:= curve.Params().ScalarMult(alphaX, alphaY, privKey)
	sharedSecret := elliptic.Marshal(curve, sharedSecretX, sharedSecretY)

	aes_s := KDF(sharedSecret)
	decKey := KDF(aes_s)

	decPayload, err := AES_CTR(decKey, payload)
	if err != nil{
		return nil, err
	}

	return decPayload, nil
}<|endoftext|>


<commit_before>// Copyright © 2015 Steve Francia <spf@spf13.com>.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package cmd

import (
	"fmt"
	"os"
	"path/filepath"
	"unicode"

	"github.com/spf13/cobra"
)

func init() {
	addCmd.Flags().StringVarP(&packageName, "package", "t", "", "target package name (e.g. github.com/spf13/hugo)")
	addCmd.Flags().StringVarP(&parentName, "parent", "p", "RootCmd", "name of parent command for this command")
}

var packageName, parentName string

var addCmd = &cobra.Command{
	Use:     "add [command name]",
	Aliases: []string{"command"},
	Short:   "Add a command to a Cobra Application",
	Long: `Add (cobra add) will create a new command, with a license and
the appropriate structure for a Cobra-based CLI application,
and register it to its parent (default RootCmd).

If you want your command to be public, pass in the command name
with an initial uppercase letter.

Example: cobra add server -> resulting in a new cmd/server.go`,

	Run: func(cmd *cobra.Command, args []string) {
		if len(args) < 1 {
			er("add needs a name for the command")
		}

		var project *Project
		if packageName != "" {
			project = NewProject(packageName)
		} else {
			wd, err := os.Getwd()
			if err != nil {
				er(err)
			}
			project = NewProjectFromPath(wd)
		}

		cmdName := validateCmdName(args[0])
		cmdPath := filepath.Join(project.CmdPath(), cmdName+".go")
		createCmdFile(project.License(), cmdPath, cmdName)

		fmt.Fprintln(cmd.OutOrStdout(), cmdName, "created at", cmdPath)
	},
}

// validateCmdName returns source without any dashes and underscore.
// If there will be dash or underscore, next letter will be uppered.
// It supports only ASCII (1-byte character) strings.
// https://github.com/spf13/cobra/issues/269
func validateCmdName(source string) string {
	i := 0
	l := len(source)
	// The output is initialized on demand, then first dash or underscore
	// occurs.
	var output string

	for i < l {
		if source[i] == '-' || source[i] == '_' {
			if output == "" {
				output = source[:i]
			}

			// If it's last rune and it's dash or underscore,
			// don't add it output and break the loop.
			if i == l-1 {
				break
			}

			// If next character is dash or underscore,
			// just skip the current character.
			if source[i+1] == '-' || source[i+1] == '_' {
				i++
				continue
			}

			// If the current character is dash or underscore,
			// upper next letter and add to output.
			output += string(unicode.ToUpper(rune(source[i+1])))
			// We know, what source[i] is dash or underscore and source[i+1] is
			// uppered character, so make i = i+2.
			i += 2
			continue
		}

		// If the current character isn't dash or underscore,
		// just add it.
		if output != "" {
			output += string(source[i])
		}
		i++
	}

	if output == "" {
		return source // source is initially valid name.
	}
	return output
}

func createCmdFile(license License, path, cmdName string) {
	template := `{{comment .copyright}}
{{if .license}}{{comment .license}}{{end}}

package {{.cmdPackage}}

import (
	"fmt"

	"github.com/spf13/cobra"
)

// {{.cmdName}}Cmd represents the {{.cmdName}} command
var {{.cmdName}}Cmd = &cobra.Command{
	Use:   "{{.cmdName}}",
	Short: "A brief description of your command",
	Long: ` + "`" + `A longer description that spans multiple lines and likely contains examples
and usage of using your command. For example:

Cobra is a CLI library for Go that empowers applications.
This application is a tool to generate the needed files
to quickly create a Cobra application.` + "`" + `,
	Run: func(cmd *cobra.Command, args []string) {
		fmt.Println("{{.cmdName}} called")
	},
}

func init() {
	{{.parentName}}.AddCommand({{.cmdName}}Cmd)

	// Here you will define your flags and configuration settings.

	// Cobra supports Persistent Flags which will work for this command
	// and all subcommands, e.g.:
	// {{.cmdName}}Cmd.PersistentFlags().String("foo", "", "A help for foo")

	// Cobra supports local flags which will only run when this command
	// is called directly, e.g.:
	// {{.cmdName}}Cmd.Flags().BoolP("toggle", "t", false, "Help message for toggle")
}
`

	data := make(map[string]interface{})
	data["copyright"] = copyrightLine()
	data["license"] = license.Header
	data["cmdPackage"] = filepath.Base(filepath.Dir(path)) // last dir of path
	data["parentName"] = parentName
	data["cmdName"] = cmdName

	cmdScript, err := executeTemplate(template, data)
	if err != nil {
		er(err)
	}
	err = writeStringToFile(path, cmdScript)
	if err != nil {
		er(err)
	}
}
<commit_msg>Specify that parent's name is a variable for clarity. #514<commit_after>// Copyright © 2015 Steve Francia <spf@spf13.com>.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package cmd

import (
	"fmt"
	"os"
	"path/filepath"
	"unicode"

	"github.com/spf13/cobra"
)

func init() {
	addCmd.Flags().StringVarP(&packageName, "package", "t", "", "target package name (e.g. github.com/spf13/hugo)")
	addCmd.Flags().StringVarP(&parentName, "parent", "p", "RootCmd", "variable name of parent command for this command")
}

var packageName, parentName string

var addCmd = &cobra.Command{
	Use:     "add [command name]",
	Aliases: []string{"command"},
	Short:   "Add a command to a Cobra Application",
	Long: `Add (cobra add) will create a new command, with a license and
the appropriate structure for a Cobra-based CLI application,
and register it to its parent (default RootCmd).

If you want your command to be public, pass in the command name
with an initial uppercase letter.

Example: cobra add server -> resulting in a new cmd/server.go`,

	Run: func(cmd *cobra.Command, args []string) {
		if len(args) < 1 {
			er("add needs a name for the command")
		}

		var project *Project
		if packageName != "" {
			project = NewProject(packageName)
		} else {
			wd, err := os.Getwd()
			if err != nil {
				er(err)
			}
			project = NewProjectFromPath(wd)
		}

		cmdName := validateCmdName(args[0])
		cmdPath := filepath.Join(project.CmdPath(), cmdName+".go")
		createCmdFile(project.License(), cmdPath, cmdName)

		fmt.Fprintln(cmd.OutOrStdout(), cmdName, "created at", cmdPath)
	},
}

// validateCmdName returns source without any dashes and underscore.
// If there will be dash or underscore, next letter will be uppered.
// It supports only ASCII (1-byte character) strings.
// https://github.com/spf13/cobra/issues/269
func validateCmdName(source string) string {
	i := 0
	l := len(source)
	// The output is initialized on demand, then first dash or underscore
	// occurs.
	var output string

	for i < l {
		if source[i] == '-' || source[i] == '_' {
			if output == "" {
				output = source[:i]
			}

			// If it's last rune and it's dash or underscore,
			// don't add it output and break the loop.
			if i == l-1 {
				break
			}

			// If next character is dash or underscore,
			// just skip the current character.
			if source[i+1] == '-' || source[i+1] == '_' {
				i++
				continue
			}

			// If the current character is dash or underscore,
			// upper next letter and add to output.
			output += string(unicode.ToUpper(rune(source[i+1])))
			// We know, what source[i] is dash or underscore and source[i+1] is
			// uppered character, so make i = i+2.
			i += 2
			continue
		}

		// If the current character isn't dash or underscore,
		// just add it.
		if output != "" {
			output += string(source[i])
		}
		i++
	}

	if output == "" {
		return source // source is initially valid name.
	}
	return output
}

func createCmdFile(license License, path, cmdName string) {
	template := `{{comment .copyright}}
{{if .license}}{{comment .license}}{{end}}

package {{.cmdPackage}}

import (
	"fmt"

	"github.com/spf13/cobra"
)

// {{.cmdName}}Cmd represents the {{.cmdName}} command
var {{.cmdName}}Cmd = &cobra.Command{
	Use:   "{{.cmdName}}",
	Short: "A brief description of your command",
	Long: ` + "`" + `A longer description that spans multiple lines and likely contains examples
and usage of using your command. For example:

Cobra is a CLI library for Go that empowers applications.
This application is a tool to generate the needed files
to quickly create a Cobra application.` + "`" + `,
	Run: func(cmd *cobra.Command, args []string) {
		fmt.Println("{{.cmdName}} called")
	},
}

func init() {
	{{.parentName}}.AddCommand({{.cmdName}}Cmd)

	// Here you will define your flags and configuration settings.

	// Cobra supports Persistent Flags which will work for this command
	// and all subcommands, e.g.:
	// {{.cmdName}}Cmd.PersistentFlags().String("foo", "", "A help for foo")

	// Cobra supports local flags which will only run when this command
	// is called directly, e.g.:
	// {{.cmdName}}Cmd.Flags().BoolP("toggle", "t", false, "Help message for toggle")
}
`

	data := make(map[string]interface{})
	data["copyright"] = copyrightLine()
	data["license"] = license.Header
	data["cmdPackage"] = filepath.Base(filepath.Dir(path)) // last dir of path
	data["parentName"] = parentName
	data["cmdName"] = cmdName

	cmdScript, err := executeTemplate(template, data)
	if err != nil {
		er(err)
	}
	err = writeStringToFile(path, cmdScript)
	if err != nil {
		er(err)
	}
}
<|endoftext|>


<commit_before>package models

import (
	"errors"
	"fmt"
	"log"
	"reflect"
	"strings"

	"github.com/astaxie/beego/orm"
	"github.com/astaxie/beego/validation"
)

type Isla struct {
	Id          int        `orm:"column(id);pk;auto"`
	Ocupado     bool       `orm:"column(ocupado)"`
	Geometria   string     `orm:"column(geometria)"`
	IdGrupoIsla *GrupoIsla `orm:"column(id_grupo_isla);rel(fk)"`
}

func (t *Isla) TableName() string {
	return "isla"
}

func init() {
	orm.RegisterModel(new(Isla))
}

// AddIsla insert a new Isla into database and returns
// last inserted Id on success.
func AddIsla(m *Isla) (id int64, err error) {
	o := orm.NewOrm()
	valid := validation.Validation{}
	valid.Required(m.Ocupado, "Ocupado")
	valid.Required(m.Geometria, "Geometria")
	if valid.HasErrors() {
		for _, err := range valid.Errors {
			log.Println(err.Key, err.Message)
		}
	} else {
		log.Println("Insert New Register")
		m.IdGrupoIsla, _ = GetGrupoIslaById(m.IdGrupoIsla.Id)
		id, err = o.Insert(m)
	}
	return
}

// GetIslaById retrieves Isla by Id. Returns error if
// Id doesn't exist
func GetIslaById(id int) (v *Isla, err error) {
	o := orm.NewOrm()
	v = &Isla{Id: id}
	if err = o.Read(v); err == nil {
		v.IdGrupoIsla, _ = GetGrupoIslaById(v.IdGrupoIsla.Id)
		return v, nil
	}
	return nil, err
}

// GetAllIsla retrieves all Isla matches certain condition. Returns empty list if
// no records exist
func GetAllIsla(query map[string]string, fields []string, sortby []string, order []string,
	offset int64, limit int64) (ml []interface{}, err error) {
	o := orm.NewOrm()
	qs := o.QueryTable(new(Isla))
	// query k=v
	for k, v := range query {
		// rewrite dot-notation to Object__Attribute
		k = strings.Replace(k, ".", "__", -1)
		qs = qs.Filter(k, v)
	}
	// order by:
	var sortFields []string
	if len(sortby) != 0 {
		if len(sortby) == len(order) {
			// 1) for each sort field, there is an associated order
			for i, v := range sortby {
				orderby := ""
				if order[i] == "desc" {
					orderby = "-" + v
				} else if order[i] == "asc" {
					orderby = v
				} else {
					return nil, errors.New("Error: Invalid order. Must be either [asc|desc]")
				}
				sortFields = append(sortFields, orderby)
			}
			qs = qs.OrderBy(sortFields...)
		} else if len(sortby) != len(order) && len(order) == 1 {
			// 2) there is exactly one order, all the sorted fields will be sorted by this order
			for _, v := range sortby {
				orderby := ""
				if order[0] == "desc" {
					orderby = "-" + v
				} else if order[0] == "asc" {
					orderby = v
				} else {
					return nil, errors.New("Error: Invalid order. Must be either [asc|desc]")
				}
				sortFields = append(sortFields, orderby)
			}
		} else if len(sortby) != len(order) && len(order) != 1 {
			return nil, errors.New("Error: 'sortby', 'order' sizes mismatch or 'order' size is not 1")
		}
	} else {
		if len(order) != 0 {
			return nil, errors.New("Error: unused 'order' fields")
		}
	}

	var l []Isla
	qs = qs.OrderBy("id")
	if _, err := qs.Limit(limit, offset).All(&l, fields...); err == nil {
		if len(fields) == 0 {
			for _, v := range l {
				v.IdGrupoIsla, _ = GetGrupoIslaById(v.IdGrupoIsla.Id)
				ml = append(ml, v)
			}
		} else {
			// trim unused fields
			for _, v := range l {
				m := make(map[string]interface{})
				val := reflect.ValueOf(v)
				for _, fname := range fields {
					m[fname] = val.FieldByName(fname).Interface()
				}
				ml = append(ml, m)
			}
		}
		return ml, nil
	}
	return nil, err
}

// UpdateIsla updates Isla by Id and returns error if
// the record to be updated doesn't exist
func UpdateIslaById(m *Isla) (err error) {
	o := orm.NewOrm()
	v := Isla{Id: m.Id}
	// ascertain id exists in the database
	if err = o.Read(&v); err == nil {
		var num int64
		if num, err = o.Update(m); err == nil {
			fmt.Println("Number of records updated in database:", num)
		}
	}
	return
}

// DeleteIsla deletes Isla by Id and returns error if
// the record to be deleted doesn't exist
func DeleteIsla(id int) (err error) {
	o := orm.NewOrm()
	v := Isla{Id: id}
	// ascertain id exists in the database
	if err = o.Read(&v); err == nil {
		var num int64
		if num, err = o.Delete(&Isla{Id: id}); err == nil {
			fmt.Println("Number of records deleted in database:", num)
		}
	}
	return
}
<commit_msg>Se crea el metodo para busqueda por isla en estado desocupado<commit_after>package models

import (
	"errors"
	"fmt"
	"log"
	"reflect"
	"strings"

	"github.com/astaxie/beego/orm"
	"github.com/astaxie/beego/validation"
)

type Isla struct {
	Id          int        `orm:"column(id);pk;auto"`
	Ocupado     bool       `orm:"column(ocupado)"`
	Geometria   string     `orm:"column(geometria)"`
	IdGrupoIsla *GrupoIsla `orm:"column(id_grupo_isla);rel(fk)"`
}

func (t *Isla) TableName() string {
	return "isla"
}

func init() {
	orm.RegisterModel(new(Isla))
}

// AddIsla insert a new Isla into database and returns
// last inserted Id on success.
func AddIsla(m *Isla) (id int64, err error) {
	o := orm.NewOrm()
	valid := validation.Validation{}
	valid.Required(m.Ocupado, "Ocupado")
	valid.Required(m.Geometria, "Geometria")
	if valid.HasErrors() {
		for _, err := range valid.Errors {
			log.Println(err.Key, err.Message)
		}
	} else {
		log.Println("Insert New Register")
		m.IdGrupoIsla, _ = GetGrupoIslaById(m.IdGrupoIsla.Id)
		id, err = o.Insert(m)
	}
	return
}

// GetIslaById retrieves Isla by Id. Returns error if
// Id doesn't exist
func GetIslaById(id int) (v *Isla, err error) {
	o := orm.NewOrm()
	v = &Isla{Id: id}
	if err = o.Read(v); err == nil {
		v.IdGrupoIsla, _ = GetGrupoIslaById(v.IdGrupoIsla.Id)
		return v, nil
	}
	return nil, err
}

// GetIslaByOcu retrieves Isla by state Ocupado. Returns error if
// Id doesn't exist
func GetIslaByOcu(id bool) (v *Isla, err error) {
	//func GetIslaByOcu(id bool) (ml []interface{}, err error) {
	o := orm.NewOrm()

	v = &Isla{Ocupado: id}
	if err = o.Read(v, "Ocupado"); err == nil {
		v.IdGrupoIsla, _ = GetGrupoIslaById(v.IdGrupoIsla.Id)
		return v, nil
	}
	return nil, err
	// De la segunda manera se obtiene todas las islas desocupadas
	// qs := o.QueryTable(new(Isla))
	// var l []Isla
	// if _, err := qs.Filter("Ocupado", id).All(&l); err == nil {
	// 	for _, v := range l {
	// 		v.IdGrupoIsla, _ = GetGrupoIslaById(v.IdGrupoIsla.Id)
	// 		ml = append(ml, v)
	// 	}
	// 	return ml, nil
	// }
	// return nil, err
}

// GetAllIsla retrieves all Isla matches certain condition. Returns empty list if
// no records exist
func GetAllIsla(query map[string]string, fields []string, sortby []string, order []string,
	offset int64, limit int64) (ml []interface{}, err error) {
	o := orm.NewOrm()
	qs := o.QueryTable(new(Isla))
	// query k=v
	for k, v := range query {
		// rewrite dot-notation to Object__Attribute
		k = strings.Replace(k, ".", "__", -1)
		qs = qs.Filter(k, v)
	}
	// order by:
	var sortFields []string
	if len(sortby) != 0 {
		if len(sortby) == len(order) {
			// 1) for each sort field, there is an associated order
			for i, v := range sortby {
				orderby := ""
				if order[i] == "desc" {
					orderby = "-" + v
				} else if order[i] == "asc" {
					orderby = v
				} else {
					return nil, errors.New("Error: Invalid order. Must be either [asc|desc]")
				}
				sortFields = append(sortFields, orderby)
			}
			qs = qs.OrderBy(sortFields...)
		} else if len(sortby) != len(order) && len(order) == 1 {
			// 2) there is exactly one order, all the sorted fields will be sorted by this order
			for _, v := range sortby {
				orderby := ""
				if order[0] == "desc" {
					orderby = "-" + v
				} else if order[0] == "asc" {
					orderby = v
				} else {
					return nil, errors.New("Error: Invalid order. Must be either [asc|desc]")
				}
				sortFields = append(sortFields, orderby)
			}
		} else if len(sortby) != len(order) && len(order) != 1 {
			return nil, errors.New("Error: 'sortby', 'order' sizes mismatch or 'order' size is not 1")
		}
	} else {
		if len(order) != 0 {
			return nil, errors.New("Error: unused 'order' fields")
		}
	}

	var l []Isla
	qs = qs.OrderBy("id")
	if _, err := qs.Limit(limit, offset).All(&l, fields...); err == nil {
		if len(fields) == 0 {
			for _, v := range l {
				v.IdGrupoIsla, _ = GetGrupoIslaById(v.IdGrupoIsla.Id)
				ml = append(ml, v)
			}
		} else {
			// trim unused fields
			for _, v := range l {
				m := make(map[string]interface{})
				val := reflect.ValueOf(v)
				for _, fname := range fields {
					m[fname] = val.FieldByName(fname).Interface()
				}
				ml = append(ml, m)
			}
		}
		return ml, nil
	}
	return nil, err
}

// UpdateIsla updates Isla by Id and returns error if
// the record to be updated doesn't exist
func UpdateIslaById(m *Isla) (err error) {
	o := orm.NewOrm()
	v := Isla{Id: m.Id}
	// ascertain id exists in the database
	if err = o.Read(&v); err == nil {
		var num int64
		if num, err = o.Update(m); err == nil {
			fmt.Println("Number of records updated in database:", num)
		}
	}
	return
}

// DeleteIsla deletes Isla by Id and returns error if
// the record to be deleted doesn't exist
func DeleteIsla(id int) (err error) {
	o := orm.NewOrm()
	v := Isla{Id: id}
	// ascertain id exists in the database
	if err = o.Read(&v); err == nil {
		var num int64
		if num, err = o.Delete(&Isla{Id: id}); err == nil {
			fmt.Println("Number of records deleted in database:", num)
		}
	}
	return
}
<|endoftext|>


<commit_before>package main

import (
	"context"
	"crypto/tls"
	"crypto/x509"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"net/http/pprof"
	"os"
	"time"

	"cloud.google.com/go/pubsub"

	"golang.org/x/net/trace"
	"golang.org/x/oauth2"

	"code.cloudfoundry.org/clock"
	"code.cloudfoundry.org/lager"
	"github.com/google/go-github/github"
	flags "github.com/jessevdk/go-flags"
	"github.com/tedsuo/ifrit"
	"github.com/tedsuo/ifrit/grouper"
	"github.com/tedsuo/ifrit/http_server"
	"github.com/tedsuo/ifrit/sigmon"

	"cred-alert/config"
	"cred-alert/crypto"
	"cred-alert/db"
	"cred-alert/db/migrations"
	"cred-alert/gitclient"
	"cred-alert/metrics"
	"cred-alert/notifications"
	"cred-alert/queue"
	"cred-alert/revok"
	"cred-alert/revok/stats"
	"cred-alert/sniff"
)

func main() {
	var cfg *config.WorkerConfig
	var flagOpts config.WorkerOpts

	logger := lager.NewLogger("revok-worker")
	logger.RegisterSink(lager.NewWriterSink(os.Stdout, lager.DEBUG))

	logger.Debug("starting")

	_, err := flags.Parse(&flagOpts)
	if err != nil {
		os.Exit(1)
	}

	if flagOpts.ConfigFile != "" {
		bs, err := ioutil.ReadFile(string(flagOpts.ConfigFile))
		if err != nil {
			logger.Error("failed-opening-config-file", err)
			os.Exit(1)
		}

		cfg, err = config.LoadWorkerConfig(bs)
		cfg.Merge(flagOpts.WorkerConfig)
	} else {
		cfg = flagOpts.WorkerConfig
	}

	errs := cfg.Validate()
	if errs != nil {
		for _, err := range errs {
			fmt.Println(err.Error())
		}
		os.Exit(1)
	}

	if cfg.Metrics.SentryDSN != "" {
		logger.RegisterSink(revok.NewSentrySink(cfg.Metrics.SentryDSN, cfg.Metrics.Environment))
	}

	workdir := cfg.WorkDir
	_, err = os.Lstat(workdir)
	if err != nil {
		log.Fatalf("workdir error: %s", err)
	}

	dbURI := db.NewDSN(cfg.MySQL.Username, cfg.MySQL.Password, cfg.MySQL.DBName, cfg.MySQL.Hostname, int(cfg.MySQL.Port))
	database, err := migrations.LockDBAndMigrate(logger, "mysql", dbURI)
	if err != nil {
		log.Fatalf("db error: %s", err)
	}

	database.LogMode(false)

	clock := clock.NewClock()

	cloneMsgCh := make(chan revok.CloneMsg)

	scanRepository := db.NewScanRepository(database, clock)
	repositoryRepository := db.NewRepositoryRepository(database)
	fetchRepository := db.NewFetchRepository(database)
	credentialRepository := db.NewCredentialRepository(database)
	emitter := metrics.BuildEmitter(cfg.Metrics.DatadogAPIKey, cfg.Metrics.Environment)
	gitClient := gitclient.New(string(cfg.GitHub.PrivateKeyPath), string(cfg.GitHub.PublicKeyPath))
	repoWhitelist := notifications.BuildWhitelist(cfg.Whitelist...)

	var notifier notifications.Notifier
	if cfg.Slack.WebhookURL != "" {
		notifier = notifications.NewSlackNotifier(cfg.Slack.WebhookURL, clock, repoWhitelist)
	} else {
		notifier = notifications.NewNullNotifier()
	}

	sniffer := sniff.NewDefaultSniffer()
	ancestryScanner := revok.NewScanner(
		gitClient,
		repositoryRepository,
		scanRepository,
		credentialRepository,
		sniffer,
		notifier,
		emitter,
	)

	changeFetcher := revok.NewChangeFetcher(
		logger,
		gitClient,
		ancestryScanner,
		repositoryRepository,
		fetchRepository,
		emitter,
	)

	changeScheduleRunner := revok.NewScheduleRunner(logger)

	changeScheduler := revok.NewChangeScheduler(
		logger,
		repositoryRepository,
		changeScheduleRunner,
		changeFetcher,
	)

	cloner := revok.NewCloner(
		logger,
		workdir,
		cloneMsgCh,
		gitClient,
		repositoryRepository,
		ancestryScanner,
		emitter,
		changeScheduler,
	)

	dirscanUpdater := revok.NewRescanner(
		logger,
		scanRepository,
		credentialRepository,
		ancestryScanner,
		notifier,
		emitter,
	)

	statsReporter := stats.NewReporter(
		logger,
		clock,
		60*time.Second,
		db.NewStatsRepository(database),
		emitter,
	)

	headCredentialCounter := revok.NewHeadCredentialCounter(
		logger,
		repositoryRepository,
		clock,
		cfg.CredentialCounterInterval,
		gitClient,
		sniffer,
	)

	members := []grouper.Member{
		{"cloner", cloner},
		{"dirscan-updater", dirscanUpdater},
		{"stats-reporter", statsReporter},
		{"head-credential-counter", headCredentialCounter},
		{"change-schedule-runner", changeScheduleRunner},
		{"debug", http_server.New("127.0.0.1:6060", debugHandler())},
	}

	if cfg.IsRPCConfigured() {
		certificate, err := tls.LoadX509KeyPair(
			string(cfg.RPC.CertificatePath),
			string(cfg.RPC.PrivateKeyPath),
		)

		clientCertPool := x509.NewCertPool()
		bs, err := ioutil.ReadFile(string(cfg.RPC.ClientCACertificatePath))
		if err != nil {
			log.Fatalf("failed to read client ca certificate: %s", err.Error())
		}

		ok := clientCertPool.AppendCertsFromPEM(bs)
		if !ok {
			log.Fatalf("failed to append client certs from pem: %s", err.Error())
		}

		grpcServer := revok.NewGRPCServer(
			logger,
			fmt.Sprintf("%s:%d", cfg.RPC.BindIP, cfg.RPC.BindPort),
			revok.NewServer(logger, repositoryRepository),
			&tls.Config{
				ClientAuth:   tls.RequireAndVerifyClientCert,
				Certificates: []tls.Certificate{certificate},
				ClientCAs:    clientCertPool,
			},
		)

		members = append(members, grouper.Member{
			Name:   "grpc-server",
			Runner: grpcServer,
		})
	}

	if cfg.IsPubSubConfigured() {
		pubSubClient, err := pubsub.NewClient(context.Background(), cfg.PubSub.ProjectName)
		if err != nil {
			logger.Fatal("failed", err)
			os.Exit(1)
		}

		subscription := pubSubClient.Subscription(cfg.PubSub.FetchHint.Subscription)

		publicKey, err := crypto.ReadRSAPublicKey(string(cfg.PubSub.PublicKeyPath))
		if err != nil {
			logger.Fatal("failed", err)
			os.Exit(1)
		}
		pushEventProcessor := queue.NewPushEventProcessor(
			changeFetcher,
			crypto.NewRSAVerifier(publicKey),
			emitter,
		)

		members = append(members, grouper.Member{
			Name:   "github-hint-handler",
			Runner: queue.NewPubSubSubscriber(logger, subscription, pushEventProcessor),
		})
	}

	if cfg.GitHub.AccessToken != "" {
		githubHTTPClient := &http.Client{
			Timeout: 30 * time.Second,
			Transport: &oauth2.Transport{
				Source: oauth2.StaticTokenSource(
					&oauth2.Token{AccessToken: cfg.GitHub.AccessToken},
				),
				Base: &http.Transport{
					DisableKeepAlives: true,
				},
			},
		}

		ghClient := revok.NewGitHubClient(github.NewClient(githubHTTPClient))

		repoDiscoverer := revok.NewRepoDiscoverer(
			logger,
			workdir,
			cloneMsgCh,
			ghClient,
			clock,
			cfg.RepositoryDiscoveryInterval,
			repositoryRepository,
		)

		members = append(members, grouper.Member{
			Name:   "repo-discoverer",
			Runner: repoDiscoverer,
		})
	}

	startupTasks := []grouper.Member{
		{
			Name:   "schedule-fetches",
			Runner: changeScheduler,
		},
	}

	system := []grouper.Member{
		{
			Name:   "servers",
			Runner: grouper.NewParallel(os.Interrupt, members),
		},
		{
			Name:   "startup-tasks",
			Runner: grouper.NewParallel(os.Interrupt, startupTasks),
		},
	}

	runner := sigmon.New(grouper.NewOrdered(os.Interrupt, system))

	err = <-ifrit.Invoke(runner).Wait()
	if err != nil {
		log.Fatalf("failed-to-start: %s", err)
	}
}

func debugHandler() http.Handler {
	debugRouter := http.NewServeMux()
	debugRouter.Handle("/debug/pprof/", http.HandlerFunc(pprof.Index))
	debugRouter.Handle("/debug/pprof/cmdline", http.HandlerFunc(pprof.Cmdline))
	debugRouter.Handle("/debug/pprof/profile", http.HandlerFunc(pprof.Profile))
	debugRouter.Handle("/debug/pprof/symbol", http.HandlerFunc(pprof.Symbol))
	debugRouter.Handle("/debug/pprof/trace", http.HandlerFunc(pprof.Trace))

	debugRouter.HandleFunc("/debug/requests", func(w http.ResponseWriter, req *http.Request) {
		any, sensitive := trace.AuthRequest(req)
		if !any {
			http.Error(w, "not allowed", http.StatusUnauthorized)
			return
		}
		w.Header().Set("Content-Type", "text/html; charset=utf-8")
		trace.Render(w, req, sensitive)
	})

	debugRouter.HandleFunc("/debug/events", func(w http.ResponseWriter, req *http.Request) {
		any, sensitive := trace.AuthRequest(req)
		if !any {
			http.Error(w, "not allowed", http.StatusUnauthorized)
			return
		}
		w.Header().Set("Content-Type", "text/html; charset=utf-8")
		trace.RenderEvents(w, req, sensitive)
	})

	return debugRouter
}
<commit_msg>de-noise logs<commit_after>package main

import (
	"context"
	"crypto/tls"
	"crypto/x509"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"net/http/pprof"
	"os"
	"time"

	"cloud.google.com/go/pubsub"

	"golang.org/x/net/trace"
	"golang.org/x/oauth2"

	"code.cloudfoundry.org/clock"
	"code.cloudfoundry.org/lager"
	"github.com/google/go-github/github"
	flags "github.com/jessevdk/go-flags"
	"github.com/tedsuo/ifrit"
	"github.com/tedsuo/ifrit/grouper"
	"github.com/tedsuo/ifrit/http_server"
	"github.com/tedsuo/ifrit/sigmon"

	"cred-alert/config"
	"cred-alert/crypto"
	"cred-alert/db"
	"cred-alert/db/migrations"
	"cred-alert/gitclient"
	"cred-alert/metrics"
	"cred-alert/notifications"
	"cred-alert/queue"
	"cred-alert/revok"
	"cred-alert/revok/stats"
	"cred-alert/sniff"
)

func main() {
	var cfg *config.WorkerConfig
	var flagOpts config.WorkerOpts

	logger := lager.NewLogger("revok-worker")
	logger.RegisterSink(lager.NewWriterSink(os.Stdout, lager.INFO))

	logger.Debug("starting")

	_, err := flags.Parse(&flagOpts)
	if err != nil {
		os.Exit(1)
	}

	if flagOpts.ConfigFile != "" {
		bs, err := ioutil.ReadFile(string(flagOpts.ConfigFile))
		if err != nil {
			logger.Error("failed-opening-config-file", err)
			os.Exit(1)
		}

		cfg, err = config.LoadWorkerConfig(bs)
		cfg.Merge(flagOpts.WorkerConfig)
	} else {
		cfg = flagOpts.WorkerConfig
	}

	errs := cfg.Validate()
	if errs != nil {
		for _, err := range errs {
			fmt.Println(err.Error())
		}
		os.Exit(1)
	}

	if cfg.Metrics.SentryDSN != "" {
		logger.RegisterSink(revok.NewSentrySink(cfg.Metrics.SentryDSN, cfg.Metrics.Environment))
	}

	workdir := cfg.WorkDir
	_, err = os.Lstat(workdir)
	if err != nil {
		log.Fatalf("workdir error: %s", err)
	}

	dbURI := db.NewDSN(cfg.MySQL.Username, cfg.MySQL.Password, cfg.MySQL.DBName, cfg.MySQL.Hostname, int(cfg.MySQL.Port))
	database, err := migrations.LockDBAndMigrate(logger, "mysql", dbURI)
	if err != nil {
		log.Fatalf("db error: %s", err)
	}

	database.LogMode(false)

	clock := clock.NewClock()

	cloneMsgCh := make(chan revok.CloneMsg)

	scanRepository := db.NewScanRepository(database, clock)
	repositoryRepository := db.NewRepositoryRepository(database)
	fetchRepository := db.NewFetchRepository(database)
	credentialRepository := db.NewCredentialRepository(database)
	emitter := metrics.BuildEmitter(cfg.Metrics.DatadogAPIKey, cfg.Metrics.Environment)
	gitClient := gitclient.New(string(cfg.GitHub.PrivateKeyPath), string(cfg.GitHub.PublicKeyPath))
	repoWhitelist := notifications.BuildWhitelist(cfg.Whitelist...)

	var notifier notifications.Notifier
	if cfg.Slack.WebhookURL != "" {
		notifier = notifications.NewSlackNotifier(cfg.Slack.WebhookURL, clock, repoWhitelist)
	} else {
		notifier = notifications.NewNullNotifier()
	}

	sniffer := sniff.NewDefaultSniffer()
	ancestryScanner := revok.NewScanner(
		gitClient,
		repositoryRepository,
		scanRepository,
		credentialRepository,
		sniffer,
		notifier,
		emitter,
	)

	changeFetcher := revok.NewChangeFetcher(
		logger,
		gitClient,
		ancestryScanner,
		repositoryRepository,
		fetchRepository,
		emitter,
	)

	changeScheduleRunner := revok.NewScheduleRunner(logger)

	changeScheduler := revok.NewChangeScheduler(
		logger,
		repositoryRepository,
		changeScheduleRunner,
		changeFetcher,
	)

	cloner := revok.NewCloner(
		logger,
		workdir,
		cloneMsgCh,
		gitClient,
		repositoryRepository,
		ancestryScanner,
		emitter,
		changeScheduler,
	)

	dirscanUpdater := revok.NewRescanner(
		logger,
		scanRepository,
		credentialRepository,
		ancestryScanner,
		notifier,
		emitter,
	)

	statsReporter := stats.NewReporter(
		logger,
		clock,
		60*time.Second,
		db.NewStatsRepository(database),
		emitter,
	)

	headCredentialCounter := revok.NewHeadCredentialCounter(
		logger,
		repositoryRepository,
		clock,
		cfg.CredentialCounterInterval,
		gitClient,
		sniffer,
	)

	members := []grouper.Member{
		{"cloner", cloner},
		{"dirscan-updater", dirscanUpdater},
		{"stats-reporter", statsReporter},
		{"head-credential-counter", headCredentialCounter},
		{"change-schedule-runner", changeScheduleRunner},
		{"debug", http_server.New("127.0.0.1:6060", debugHandler())},
	}

	if cfg.IsRPCConfigured() {
		certificate, err := tls.LoadX509KeyPair(
			string(cfg.RPC.CertificatePath),
			string(cfg.RPC.PrivateKeyPath),
		)

		clientCertPool := x509.NewCertPool()
		bs, err := ioutil.ReadFile(string(cfg.RPC.ClientCACertificatePath))
		if err != nil {
			log.Fatalf("failed to read client ca certificate: %s", err.Error())
		}

		ok := clientCertPool.AppendCertsFromPEM(bs)
		if !ok {
			log.Fatalf("failed to append client certs from pem: %s", err.Error())
		}

		grpcServer := revok.NewGRPCServer(
			logger,
			fmt.Sprintf("%s:%d", cfg.RPC.BindIP, cfg.RPC.BindPort),
			revok.NewServer(logger, repositoryRepository),
			&tls.Config{
				ClientAuth:   tls.RequireAndVerifyClientCert,
				Certificates: []tls.Certificate{certificate},
				ClientCAs:    clientCertPool,
			},
		)

		members = append(members, grouper.Member{
			Name:   "grpc-server",
			Runner: grpcServer,
		})
	}

	if cfg.IsPubSubConfigured() {
		pubSubClient, err := pubsub.NewClient(context.Background(), cfg.PubSub.ProjectName)
		if err != nil {
			logger.Fatal("failed", err)
			os.Exit(1)
		}

		subscription := pubSubClient.Subscription(cfg.PubSub.FetchHint.Subscription)

		publicKey, err := crypto.ReadRSAPublicKey(string(cfg.PubSub.PublicKeyPath))
		if err != nil {
			logger.Fatal("failed", err)
			os.Exit(1)
		}
		pushEventProcessor := queue.NewPushEventProcessor(
			changeFetcher,
			crypto.NewRSAVerifier(publicKey),
			emitter,
		)

		members = append(members, grouper.Member{
			Name:   "github-hint-handler",
			Runner: queue.NewPubSubSubscriber(logger, subscription, pushEventProcessor),
		})
	}

	if cfg.GitHub.AccessToken != "" {
		githubHTTPClient := &http.Client{
			Timeout: 30 * time.Second,
			Transport: &oauth2.Transport{
				Source: oauth2.StaticTokenSource(
					&oauth2.Token{AccessToken: cfg.GitHub.AccessToken},
				),
				Base: &http.Transport{
					DisableKeepAlives: true,
				},
			},
		}

		ghClient := revok.NewGitHubClient(github.NewClient(githubHTTPClient))

		repoDiscoverer := revok.NewRepoDiscoverer(
			logger,
			workdir,
			cloneMsgCh,
			ghClient,
			clock,
			cfg.RepositoryDiscoveryInterval,
			repositoryRepository,
		)

		members = append(members, grouper.Member{
			Name:   "repo-discoverer",
			Runner: repoDiscoverer,
		})
	}

	startupTasks := []grouper.Member{
		{
			Name:   "schedule-fetches",
			Runner: changeScheduler,
		},
	}

	system := []grouper.Member{
		{
			Name:   "servers",
			Runner: grouper.NewParallel(os.Interrupt, members),
		},
		{
			Name:   "startup-tasks",
			Runner: grouper.NewParallel(os.Interrupt, startupTasks),
		},
	}

	runner := sigmon.New(grouper.NewOrdered(os.Interrupt, system))

	err = <-ifrit.Invoke(runner).Wait()
	if err != nil {
		log.Fatalf("failed-to-start: %s", err)
	}
}

func debugHandler() http.Handler {
	debugRouter := http.NewServeMux()
	debugRouter.Handle("/debug/pprof/", http.HandlerFunc(pprof.Index))
	debugRouter.Handle("/debug/pprof/cmdline", http.HandlerFunc(pprof.Cmdline))
	debugRouter.Handle("/debug/pprof/profile", http.HandlerFunc(pprof.Profile))
	debugRouter.Handle("/debug/pprof/symbol", http.HandlerFunc(pprof.Symbol))
	debugRouter.Handle("/debug/pprof/trace", http.HandlerFunc(pprof.Trace))

	debugRouter.HandleFunc("/debug/requests", func(w http.ResponseWriter, req *http.Request) {
		any, sensitive := trace.AuthRequest(req)
		if !any {
			http.Error(w, "not allowed", http.StatusUnauthorized)
			return
		}
		w.Header().Set("Content-Type", "text/html; charset=utf-8")
		trace.Render(w, req, sensitive)
	})

	debugRouter.HandleFunc("/debug/events", func(w http.ResponseWriter, req *http.Request) {
		any, sensitive := trace.AuthRequest(req)
		if !any {
			http.Error(w, "not allowed", http.StatusUnauthorized)
			return
		}
		w.Header().Set("Content-Type", "text/html; charset=utf-8")
		trace.RenderEvents(w, req, sensitive)
	})

	return debugRouter
}
<|endoftext|>


<commit_before>package db

// This file defined interfaces for key-value caches
// and some simple implementations

import (
    "os"
    "sync"
)

type kvdata struct {
    key string
    value interface{}
}

// This should be some database-related struct
// to flush dirty data into database
// The implementation could put the real removal and insertion action
// in either the Flush() or Add()/Remove() functions.
// The Cache will always call Flush() after a bunch of Add()/Remove()
type KeyValueFlusher interface {
    Set(key string, value interface{}) os.Error
    Remove(key string, value interface{}) os.Error
    Flush() os.Error
}

type KeyValueCacheIf interface {
    Show(key string, value interface{}) os.Error
    Get(key string) (interface{}, os.Error)
    Modify(key string, value interface{}) os.Error
    Flush() os.Error
    Keys() ([]string, os.Error)
    Remove(key string, value interface{}) os.Error
}

// A key-value cache is like a lazy person:
// You want him to remember something (for example, building his vocabulary)
// You provide the data in key-value forms (word - meaning)
//
// Suppose this man holds a dictionary which has all word. But he
// will not remember all of them in his mind.
//
// At first, you pick up a word from dictionary, and show it to that person.
// (Using Show() method)
// You don't know if he has remembered this word. He has his own strategy to
// decide which word should remember.
//
// Then you ask him about the word by calling Get(). You may or may not get
// an answer. This person will never look up the word in dictionary. He only
// tells you the word he remembered. In general, most strategies will try to
// remember words you frequently asked (using Get() *not* Show()).
//
// If a word's meaning changed, which means the value changed, you call
// Modify() to tell the person this change and ask him to update his
// dictionary also. However, because updating dictionary is really expensive,
// and it could be cheaper if you update more words at same time, moreover,
// this person is really lazy, he may wait for a while and update the
// dictionary later hopefully could get more word need to be updated.
//
// Similarly, if you remove some word by calling Remove(), this person
// may not update the dictionary immediately. Instead, he will keep a list
// of words need to be removed and remove them all once.
//
// I hope this analogue could help someone to understand the cache. Very
// simple idea. Think some database as the dictionary and the cache as the
// lazy guy.
//
// MFAQ (May be Frequently Asked Questions, because nobody asked them now)
//
// Why not let the cache find value from the
// database on cache miss? After all, we only need to define a new interface
// and the cache could call some method, say Get(), in that interface and then
// return the value. It helps a lot, doesn't it? And actually, we already
// has such interface, the KeyValueStorage could be used.
//
// Well, here is the reason: we may not use a key-value database behind the
// cache. The possible Get() method takes a string and return some value.
// If it is not a key-value database, we have to parse the string and find
// what we really want in the implementation of Get(). Parsing string
// may take a lot of time and more important: we need to write some tedious
// code to split the string, compare it, or some regular expressions, etc.
// If we want to add support such Get() feature, then we need either 1)
// to do the tedious string parsing work, or 2) to have more database- or
// even application-specific knowledge to get the value we want from the
// database. For example, we could have some function like db.GetUser()
// to get the user data. However, it is definitely not a good design
//
// In short: we don't want the cache has too much information about the
// database or application.
//
// Then another question may be in your mind: then why we need KeyValueFlusher
// to flush the data? It involves database anyway, right?
//
// The answer is simple: bacause we provide more than a key to the methods
// in KeyValueFlusher. The Set() and Remove() method in KeyValueFlusher
// take both key and value as their parameters. This means the database- or
// even application-specific data could be stored in the value and the
// implementation of KeyValueFlusher could take such data from value, then
// flush them without even taking a look at the key.
//
type KeyValueCache struct {
    storage KeyValueStorage
    strategy KeyValueCacheStrategy
    flusher KeyValueFlusher
    dirty_list []kvdata
    rm_list []kvdata
    rwlock sync.RWMutex
}

const (
    default_dirty_list_size = 50
)

func NewKeyValueCache(storage KeyValueStorage,
                      strategy KeyValueCacheStrategy,
                      flusher KeyValueFlusher) KeyValueCacheIf {
    c := new(KeyValueCache)
    c.storage = storage
    c.strategy = strategy
    c.dirty_list = make([]kvdata, 0, default_dirty_list_size)
    c.rm_list = make([]kvdata, 0, 10)
    c.flusher = flusher
    return c
}

func (c *KeyValueCache) remove() os.Error {
    c.rwlock.Lock()
    defer c.rwlock.Unlock()

    rmlist := c.strategy.GetObsoleted()
    var err os.Error
    for _, k := range rmlist {
        _, err = c.storage.Remove(k)
        c.strategy.Removed(k)
        if err != nil {
            return err
        }
    }
    return nil
}

func (c *KeyValueCache) flush() os.Error {
    c.rwlock.Lock()
    defer c.rwlock.Unlock()

    if need_flush := c.strategy.ShouldFlush(); need_flush {
        for _, d := range c.dirty_list {
            err := c.flusher.Set(d.key, d.value)
            if err != nil {
                return err
            }
        }

        c.dirty_list = make([]kvdata, 0, len(c.dirty_list))

        for _, d := range c.rm_list {
            err := c.flusher.Remove(d.key, d.value)
            if err != nil {
                return err
            }
        }

        c.rm_list = make([]kvdata, 0, len(c.rm_list))
        c.flusher.Flush()
        c.strategy.Flushed()
    }
    return nil
}

func (c *KeyValueCache) Flush() os.Error {
    return c.flush()
}

// The caller could Show a key value pair to a cache,
// and let the cache decide if it want to add this pair into the cache.
// A cache make this decision based on its strategy
// Note: if the key is already in the cache, Show() will not increase
// the cache hit rate; If the key is not in the cache, it will call
// Added() to strategy
func (c *KeyValueCache) Show(key string, v interface{}) os.Error {
    var err os.Error
    if should_add := c.strategy.ShouldAdd(key); should_add {
        c.rwlock.Lock()
        oldv, err := c.storage.Set(key, v)
        if oldv == nil {
            c.strategy.Added(key)
        }
        c.rwlock.Unlock()

        if err != nil {
            return err
        }

    }
    err = c.remove()
    err = c.flush()
    if err != nil {
        return err
    }
    return nil
}

func (c *KeyValueCache) Modify(key string, v interface{}) os.Error {
    c.rwlock.Lock()
    defer c.rwlock.Unlock()

    // First, this data is dirty
    c.dirty_list = append(c.dirty_list, kvdata{key, v})
    c.strategy.Dirty(key)

    oldv, err := c.storage.Get(key)
    if err != nil {
        v = nil
        c.rwlock.RUnlock()
        return err
    }

    if oldv == nil {
        // This data is not in cache. So a cache-miss occur
        // We need to know if we should add this value into cache
        c.strategy.Miss(key)
        if should_add := c.strategy.ShouldAdd(key); should_add {
            _, err := c.storage.Set(key, v)
            c.strategy.Added(key)
            return err
        }
        return nil
    }
    // Cache hit. We need to update to the latest value
    c.strategy.Hit(key)
    _, err = c.storage.Set(key, v)
    return nil
}

func (c *KeyValueCache) Get(key string) (v interface{}, err os.Error) {
    c.rwlock.RLock()
    v, err = c.storage.Get(key)
    if err != nil {
        v = nil
        c.rwlock.RUnlock()
        return
    }

    if v == nil {
        // Cache miss
        c.strategy.Miss(key)
    } else {
        // Cache hit
        c.strategy.Hit(key)
    }
    c.rwlock.RUnlock()

    err = c.remove()
    err = c.flush()
    return
}

// Remove records the key need to be removed. And perform the
// actual removal in next flush
func (c *KeyValueCache) Remove(key string, value interface{}) (err os.Error) {
    c.rwlock.Lock()

    _, err = c.storage.Remove(key)
    if err != nil {
        return err
    }
    c.strategy.Removed(key)

    c.rm_list = append(c.rm_list, kvdata{key, value})
    c.strategy.Dirty(key)
    c.rwlock.Unlock()

    err = c.remove()
    err = c.flush()
    return
}

func (c *KeyValueCache) Keys() (keys []string, err os.Error) {
    keys, err = c.storage.Keys()
    return
}

<commit_msg>associated changes<commit_after>package db

// This file defined interfaces for key-value caches
// and some simple implementations

import (
    "os"
    "sync"
)

type kvdata struct {
    key string
    value interface{}
}

// This should be some database-related struct
// to flush dirty data into database
// The implementation could put the real removal and insertion action
// in either the Flush() or Add()/Remove() functions.
// The Cache will always call Flush() after a bunch of Add()/Remove()
type KeyValueFlusher interface {
    Set(key string, value interface{}) os.Error
    Remove(key string, value interface{}) os.Error
    Flush() os.Error
}

type KeyValueCacheIf interface {
    Show(key string, value interface{}) os.Error
    Get(key string) (interface{}, os.Error)
    Modify(key string, value interface{}) os.Error
    Flush() os.Error
    Keys() ([]string, os.Error)
    Remove(key string, value interface{}) os.Error
}

// A key-value cache is like a lazy person:
// You want him to remember something (for example, building his vocabulary)
// You provide the data in key-value forms (word - meaning)
//
// Suppose this man holds a dictionary which has all word. But he
// will not remember all of them in his mind.
//
// At first, you pick up a word from dictionary, and show it to that person.
// (Using Show() method)
// You don't know if he has remembered this word. He has his own strategy to
// decide which word should remember.
//
// Then you ask him about the word by calling Get(). You may or may not get
// an answer. This person will never look up the word in dictionary. He only
// tells you the word he remembered. In general, most strategies will try to
// remember words you frequently asked (using Get() *not* Show()).
//
// If a word's meaning changed, which means the value changed, you call
// Modify() to tell the person this change and ask him to update his
// dictionary also. However, because updating dictionary is really expensive,
// and it could be cheaper if you update more words at same time, moreover,
// this person is really lazy, he may wait for a while and update the
// dictionary later hopefully could get more word need to be updated.
//
// Similarly, if you remove some word by calling Remove(), this person
// may not update the dictionary immediately. Instead, he will keep a list
// of words need to be removed and remove them all once.
//
// I hope this analogue could help someone to understand the cache. Very
// simple idea. Think some database as the dictionary and the cache as the
// lazy guy.
//
// MFAQ (May be Frequently Asked Questions, because nobody asked them now)
//
// Why not let the cache find value from the
// database on cache miss? After all, we only need to define a new interface
// and the cache could call some method, say Get(), in that interface and then
// return the value. It helps a lot, doesn't it? And actually, we already
// has such interface, the KeyValueStorage could be used.
//
// Well, here is the reason: we may not use a key-value database behind the
// cache. The possible Get() method takes a string and return some value.
// If it is not a key-value database, we have to parse the string and find
// what we really want in the implementation of Get(). Parsing string
// may take a lot of time and more important: we need to write some tedious
// code to split the string, compare it, or some regular expressions, etc.
// If we want to add support such Get() feature, then we need either 1)
// to do the tedious string parsing work, or 2) to have more database- or
// even application-specific knowledge to get the value we want from the
// database. For example, we could have some function like db.GetUser()
// to get the user data. However, it is definitely not a good design
//
// In short: we don't want the cache has too much information about the
// database or application.
//
// Then another question may be in your mind: then why we need KeyValueFlusher
// to flush the data? It involves database anyway, right?
//
// The answer is simple: bacause we provide more than a key to the methods
// in KeyValueFlusher. The Set() and Remove() method in KeyValueFlusher
// take both key and value as their parameters. This means the database- or
// even application-specific data could be stored in the value and the
// implementation of KeyValueFlusher could take such data from value, then
// flush them without even taking a look at the key.
//
type KeyValueCache struct {
    storage KeyValueStorage
    strategy KeyValueCacheStrategy
    flusher KeyValueFlusher
    dirty_list []kvdata
    rm_list []kvdata
    rwlock sync.RWMutex
}

const (
    default_dirty_list_size = 50
)

func NewKeyValueCache(storage KeyValueStorage,
                      strategy KeyValueCacheStrategy,
                      flusher KeyValueFlusher) KeyValueCacheIf {
    c := new(KeyValueCache)
    c.storage = storage
    c.strategy = strategy
    c.dirty_list = make([]kvdata, 0, default_dirty_list_size)
    c.rm_list = make([]kvdata, 0, 10)
    c.flusher = flusher
    return c
}

func (c *KeyValueCache) remove() os.Error {
    c.rwlock.Lock()
    defer c.rwlock.Unlock()

    rmlist := c.strategy.GetObsoleted()
    var err os.Error
    for _, k := range rmlist {
        _, err = c.storage.Remove(k)
        c.strategy.Removed(k)
        if err != nil {
            return err
        }
    }
    return nil
}

func (c *KeyValueCache) flush() os.Error {
    c.rwlock.Lock()
    defer c.rwlock.Unlock()

    if need_flush := c.strategy.ShouldFlush(); need_flush {
        for _, d := range c.dirty_list {
            err := c.flusher.Set(d.key, d.value)
            if err != nil {
                return err
            }
        }

        c.dirty_list = make([]kvdata, 0, len(c.dirty_list))

        for _, d := range c.rm_list {
            _, err := c.storage.Remove(d.key)
            if err != nil {
                return err
            }
            c.strategy.Removed(d.key)

            err = c.flusher.Remove(d.key, d.value)
            if err != nil {
                return err
            }
        }

        c.rm_list = make([]kvdata, 0, len(c.rm_list))
        c.flusher.Flush()
        c.strategy.Flushed()
    }
    return nil
}

func (c *KeyValueCache) Flush() os.Error {
    return c.flush()
}

// The caller could Show a key value pair to a cache,
// and let the cache decide if it want to add this pair into the cache.
// A cache make this decision based on its strategy
// Note: if the key is already in the cache, Show() will not increase
// the cache hit rate; If the key is not in the cache, it will call
// Added() to strategy
func (c *KeyValueCache) Show(key string, v interface{}) os.Error {
    var err os.Error
    if should_add := c.strategy.ShouldAdd(key); should_add {
        c.rwlock.Lock()
        oldv, err := c.storage.Set(key, v)
        if oldv == nil {
            c.strategy.Added(key)
        }
        c.rwlock.Unlock()

        if err != nil {
            return err
        }

    }
    err = c.remove()
    err = c.flush()
    if err != nil {
        return err
    }
    return nil
}

func (c *KeyValueCache) Modify(key string, v interface{}) os.Error {
    c.rwlock.Lock()
    defer c.rwlock.Unlock()

    // First, this data is dirty
    c.dirty_list = append(c.dirty_list, kvdata{key, v})
    c.strategy.Dirty(key)

    oldv, err := c.storage.Get(key)
    if err != nil {
        v = nil
        c.rwlock.RUnlock()
        return err
    }

    if oldv == nil {
        // This data is not in cache. So a cache-miss occur
        // We need to know if we should add this value into cache
        c.strategy.Miss(key)
        if should_add := c.strategy.ShouldAdd(key); should_add {
            _, err := c.storage.Set(key, v)
            c.strategy.Added(key)
            return err
        }
        return nil
    }
    // Cache hit. We need to update to the latest value
    c.strategy.Hit(key)
    _, err = c.storage.Set(key, v)
    return nil
}

func (c *KeyValueCache) Get(key string) (v interface{}, err os.Error) {
    c.rwlock.RLock()
    v, err = c.storage.Get(key)
    if err != nil {
        v = nil
        c.rwlock.RUnlock()
        return
    }

    if v == nil {
        // Cache miss
        c.strategy.Miss(key)
    } else {
        // Cache hit
        c.strategy.Hit(key)
    }
    c.rwlock.RUnlock()

    err = c.remove()
    err = c.flush()
    return
}

// Remove records the key need to be removed. And perform the
// actual removal in next flush
func (c *KeyValueCache) Remove(key string, value interface{}) (err os.Error) {
    c.rwlock.Lock()
    c.rm_list = append(c.rm_list, kvdata{key, value})
    c.strategy.Dirty(key)
    c.rwlock.Unlock()

    err = c.remove()
    err = c.flush()
    return
}

func (c *KeyValueCache) Keys() (keys []string, err os.Error) {
    keys, err = c.storage.Keys()
    return
}

<|endoftext|>


<commit_before>package db

// This file defined interfaces for key-value caches
// and some simple implementations

import (
    "os"
    "sync"
    "time"
)

// This should be some database-related struct
// to flush dirty data into database
type KeyValueFlusher interface {
    Flush(key string, value *interface{}) os.Error
}

// This interface defines bahaviors of a cache
// Like: if we should store this data; if we need to remove some data
type KeyValueCacheStrategy interface {
    // This should be called on cache hit.
    Hit(key string)

    // This should be called by cache when a key is inserted
    Added(key string)

    // This should be called when a key is removed
    Removed(key string)

    // This should be called on cache miss.
    Miss(key string)

    // This should return all keys which need to be removed
    GetObsoleted() []string

    // This should tell the cache if add the key into cache
    ShouldAdd(key string) bool

    // Nothing fancy
    ShouldFlush() bool

    Flushed()
}

// This is the interface to represent a key-value storage
type KeyValueStorage interface {
    Add(key string, v *interface{}) os.Error
    Remove(key string) os.Error
    Get(key string) (*interface{}, os.Error)
    Len() (int, os.Error)
}

/**** In Memory Storage ****/

type InMemoryKeyValueStorage struct {
    data map[string]*interface{}
}

const (
    default_cache_size int = 100
)

func NewInMemoryKeyValueStorage(size int) *InMemoryKeyValueStorage {
    s := new(InMemoryKeyValueStorage)
    if size <= 0 {
        size = default_cache_size
    }
    s.data = make(map[string]*interface{}, size)
    return s
}

func (s *InMemoryKeyValueStorage) Remove(key string) os.Error {
    s.data[key] = nil, false
    return nil
}

func (s *InMemoryKeyValueStorage) Get(key string) (v *interface{}, err os.Error) {
    var has bool
    v, has = s.data[key]
    if !has {
        return nil, nil
    }
    return v, nil
}

func (s *InMemoryKeyValueStorage) Len() (l int, err os.Error) {
    return len(s.data), nil
}

/******* Key Value Cache **************/

type kvdata struct {
    key string
    value *interface{}
}

type KeyValueCache struct {
    storage KeyValueStorage
    strategy KeyValueCacheStrategy
    flusher KeyValueFlusher
    dirty_list []kvdata
    rwlock sync.RWMutex
}

const (
    default_dirty_list_size = 50
)

func NewKeyValueCache(storage KeyValueStorage,
                      strategy KeyValueCacheStrategy,
                      flusher KeyValueFlusher) *KeyValueCache {
    c := new(KeyValueCache)
    c.storage = storage
    c.strategy = strategy
    c.dirty_list = make([]kvdata, 0, default_dirty_list_size)
    c.flusher = flusher
    return c
}

func (c *KeyValueCache) remove() os.Error {
    c.rwlock.Lock()
    defer c.rwlock.Unlock()

    rmlist := c.strategy.GetObsoleted()
    var err os.Error
    for _, k := range rmlist {
        err = c.storage.Remove(k)
        c.strategy.Removed(k)
        if err != nil {
            return err
        }
    }
    return nil
}

func (c *KeyValueCache) flush() os.Error {
    c.rwlock.Lock()
    defer c.rwlock.Unlock()

    if need_flush := c.strategy.ShouldFlush(); need_flush {
        for _, d := range c.dirty_list {
            err := c.flusher.Flush(d.key, d.value)
            if err != nil {
                return err
            }
        }
        c.strategy.Flushed()
    }
    return nil
}

// The caller could Show a key value pair to a cache,
// and let the cache decide if it want to add this pair into the cache.
// A cache make this decision based on its strategy
func (c *KeyValueCache) Show(key string, v *interface{}) os.Error {
    var err os.Error
    if should_add := c.strategy.ShouldAdd(key); should_add {

        c.rwlock.Lock()
        c.strategy.Added(key)
        err := c.storage.Add(key, v)
        c.dirty_list = append(c.dirty_list, kvdata{key, v})
        c.rwlock.Unlock()

        if err != nil {
            return err
        }

    }
    err = c.remove()
    err = c.flush()
    if err != nil {
        return err
    }
    return nil
}

func (c *KeyValueCache) Get(key string) (v *interface{}, err os.Error) {
    c.rwlock.RLock()
    v, err = c.storage.Get(key)
    if err != nil {
        v = nil
        c.rwlock.RUnlock()
        return
    }

    // Cache miss
    if v == nil {
        c.strategy.Miss(key)
    }

    // Cache hit
    c.strategy.Hit(key)
    c.rwlock.RUnlock()

    err = c.remove()
    err = c.flush()
    return
}

/*************** Strategies ************************/

type PeriodFlushStrategy struct {
    last_flush_time int64
    period int64
}

func (s *PeriodFlushStrategy) ShouldFlush() bool {
    current_time := time.Seconds()
    if current_time - s.last_flush_time >= s.period {
        return true
    }
    return false
}

func (s *PeriodFlushStrategy) Flushed() {
    s.last_flush_time = time.Seconds()
}

type AlwaysInCacheStrategy struct {
}

func (s *AlwaysInCacheStrategy) Hit(string) {
    return
}

func (s *AlwaysInCacheStrategy) Added(string) {
    return
}

func (s *AlwaysInCacheStrategy) Removed(string) {
    return
}

func (s *AlwaysInCacheStrategy) Miss(string) {
    return
}

func (s *AlwaysInCacheStrategy) ShouldAdd(key string) bool {
    return true
}

func (s *AlwaysInCacheStrategy) GetObsoleted() []string {
    return nil
}

type AlwaysInCachePeriodFlushStrategy struct {
    AlwaysInCacheStrategy
    PeriodFlushStrategy
}

func NewAlwaysInCachePeriodFlush(period int64) KeyValueCacheStrategy {
    s := new(AlwaysInCachePeriodFlushStrategy)
    s.last_flush_time = time.Seconds()
    s.period = period
    var ret KeyValueCacheStrategy
    ret = s
    return ret
}

type LRUStrategy struct {
    q []string
    max int
}

func (s *LRUStrategy) Hit(key string) {
    return
}

func (s *LRUStrategy) Miss(key string) {
    return
}

func (s *LRUStrategy) Added(key string) {
    s.q = append(s.q, key)
}

func (s *LRUStrategy) Removed(key string) {
}
<commit_msg>Added the remove method in cache<commit_after>package db

// This file defined interfaces for key-value caches
// and some simple implementations

import (
    "os"
    "sync"
    "time"
)

// This interface defines bahaviors of a cache
// Like: if we should store this data; if we need to remove some data
type KeyValueCacheStrategy interface {
    // This should be called on cache hit.
    Hit(key string)

    // This should be called by cache when a key is inserted
    Added(key string)

    // This should be called when a key is removed
    Removed(key string)

    // This should be called on cache miss.
    Miss(key string)

    // This should return all keys which need to be removed
    GetObsoleted() []string

    // This should tell the cache if add the key into cache
    ShouldAdd(key string) bool

    // Nothing fancy
    ShouldFlush() bool

    Flushed()
}

// This is the interface to represent a key-value storage
type KeyValueStorage interface {
    Add(key string, v interface{}) os.Error
    Remove(key string) (interface{}, os.Error)
    Get(key string) (interface{}, os.Error)
    Len() (int, os.Error)
}

// This should be some database-related struct
// to flush dirty data into database
// The implementation could put the real removal and insertion action
// in either the Flush() or Add()/Remove() functions.
// The Cache will always call Flush() after a bunch of Add()/Remove()
type KeyValueFlusher interface {
    Add(key string, value interface{}) os.Error
    Remove(key string) os.Error
    Flush() os.Error
}

/**** In Memory Storage ****/

type InMemoryKeyValueStorage struct {
    data map[string]interface{}
}

const (
    default_cache_size int = 100
)

func NewInMemoryKeyValueStorage(size int) *InMemoryKeyValueStorage {
    s := new(InMemoryKeyValueStorage)
    if size <= 0 {
        size = default_cache_size
    }
    s.data = make(map[string]interface{}, size)
    return s
}

func (s *InMemoryKeyValueStorage) Remove(key string) os.Error {
    s.data[key] = nil, false
    return nil
}

func (s *InMemoryKeyValueStorage) Get(key string) (v interface{}, err os.Error) {
    var has bool
    v, has = s.data[key]
    if !has {
        return nil, nil
    }
    return v, nil
}

func (s *InMemoryKeyValueStorage) Len() (l int, err os.Error) {
    return len(s.data), nil
}

/******* Key Value Cache **************/

type kvdata struct {
    key string
    value interface{}
}

type KeyValueCache struct {
    storage KeyValueStorage
    strategy KeyValueCacheStrategy
    flusher KeyValueFlusher
    dirty_list []kvdata
    rm_list []string
    rwlock sync.RWMutex
}

const (
    default_dirty_list_size = 50
)

func NewKeyValueCache(storage KeyValueStorage,
                      strategy KeyValueCacheStrategy,
                      flusher KeyValueFlusher) *KeyValueCache {
    c := new(KeyValueCache)
    c.storage = storage
    c.strategy = strategy
    c.dirty_list = make([]kvdata, 0, default_dirty_list_size)
    c.rm_list = make([]string, 0, 10)
    c.flusher = flusher
    return c
}

func (c *KeyValueCache) remove() os.Error {
    c.rwlock.Lock()
    defer c.rwlock.Unlock()

    rmlist := c.strategy.GetObsoleted()
    var err os.Error
    for _, k := range rmlist {
        _, err = c.storage.Remove(k)
        c.strategy.Removed(k)
        if err != nil {
            return err
        }
    }
    return nil
}

func (c *KeyValueCache) flush() os.Error {
    c.rwlock.Lock()
    defer c.rwlock.Unlock()

    if need_flush := c.strategy.ShouldFlush(); need_flush {
        for _, d := range c.dirty_list {
            err := c.flusher.Add(d.key, d.value)
            if err != nil {
                return err
            }
        }

        c.dirty_list = make([]kvdata, 0, len(c.dirty_list))

        for _, d := range c.rm_list {
            err := c.flusher.Remove(d)
            if err != nil {
                return err
            }
        }

        c.rm_list = make([]string, 0, len(c.rm_list))
        c.flusher.Flush()
        c.strategy.Flushed()
    }
    return nil
}

// The caller could Show a key value pair to a cache,
// and let the cache decide if it want to add this pair into the cache.
// A cache make this decision based on its strategy
func (c *KeyValueCache) Show(key string, v interface{}) os.Error {
    var err os.Error
    if should_add := c.strategy.ShouldAdd(key); should_add {

        c.rwlock.Lock()
        c.strategy.Added(key)
        err := c.storage.Add(key, v)
        c.dirty_list = append(c.dirty_list, kvdata{key, v})
        c.rwlock.Unlock()

        if err != nil {
            return err
        }

    }
    err = c.remove()
    err = c.flush()
    if err != nil {
        return err
    }
    return nil
}

func (c *KeyValueCache) Get(key string) (v interface{}, err os.Error) {
    c.rwlock.RLock()
    v, err = c.storage.Get(key)
    if err != nil {
        v = nil
        c.rwlock.RUnlock()
        return
    }

    // Cache miss
    if v == nil {
        c.strategy.Miss(key)
    }

    // Cache hit
    c.strategy.Hit(key)
    c.rwlock.RUnlock()

    err = c.remove()
    err = c.flush()
    return
}

// Remove records the key need to be removed. And perform the
// actual removal in next flush
// Why not return the value? because the item may not in the cache.
func (c *KeyValueCache) Remove(key string) (err os.Error) {
    c.rwlock.Lock()

    _, err = c.storage.Remove(key)
    if err != nil {
        return err
    }
    c.strategy.Removed(key)

    c.rm_list = append(c.rm_list, key)
    c.rwlock.Unlock()

    err = c.remove()
    err = c.flush()
    return
}

/*************** Strategies ************************/

type PeriodFlushStrategy struct {
    last_flush_time int64
    period int64
}

func (s *PeriodFlushStrategy) ShouldFlush() bool {
    current_time := time.Seconds()
    if current_time - s.last_flush_time >= s.period {
        return true
    }
    return false
}

func (s *PeriodFlushStrategy) Flushed() {
    s.last_flush_time = time.Seconds()
}

type AlwaysInCacheStrategy struct {
}

func (s *AlwaysInCacheStrategy) Hit(string) {
    return
}

func (s *AlwaysInCacheStrategy) Added(string) {
    return
}

func (s *AlwaysInCacheStrategy) Removed(string) {
    return
}

func (s *AlwaysInCacheStrategy) Miss(string) {
    return
}

func (s *AlwaysInCacheStrategy) ShouldAdd(key string) bool {
    return true
}

func (s *AlwaysInCacheStrategy) GetObsoleted() []string {
    return nil
}

type AlwaysInCachePeriodFlushStrategy struct {
    AlwaysInCacheStrategy
    PeriodFlushStrategy
}

func NewAlwaysInCachePeriodFlush(period int64) KeyValueCacheStrategy {
    s := new(AlwaysInCachePeriodFlushStrategy)
    s.last_flush_time = time.Seconds()
    s.period = period
    var ret KeyValueCacheStrategy
    ret = s
    return ret
}

type LRUStrategy struct {
    q []string
    max int
}

func (s *LRUStrategy) Hit(key string) {
    return
}

func (s *LRUStrategy) Miss(key string) {
    return
}

func (s *LRUStrategy) Added(key string) {
    s.q = append(s.q, key)
}

func (s *LRUStrategy) Removed(key string) {
}
<|endoftext|>