<commit_before>// Copyright 2017 PingCAP, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// See the License for the specific language governing permissions and
// limitations under the License.

package tikv

import (
	"bytes"
	"context"
	"math"

	"github.com/pingcap/errors"
	"github.com/pingcap/failpoint"
	"github.com/pingcap/kvproto/pkg/kvrpcpb"
	"github.com/pingcap/kvproto/pkg/metapb"
	"github.com/pingcap/kvproto/pkg/pdpb"
	"github.com/pingcap/tidb/kv"
	"github.com/pingcap/tidb/store/tikv/tikvrpc"
	"github.com/pingcap/tidb/util"
	"github.com/pingcap/tidb/util/logutil"
	"github.com/pingcap/tidb/util/stringutil"
	"go.uber.org/zap"
)

func equalRegionStartKey(key, regionStartKey []byte) bool {
	return bytes.Equal(key, regionStartKey)
}

func (s *tikvStore) splitBatchRegionsReq(bo *Backoffer, keys [][]byte, scatter bool) (*tikvrpc.Response, error) {
	// equalRegionStartKey is used to filter split keys.
	// If the split key is equal to the start key of the region, then the key has been split, we need to skip the split key.
	groups, _, err := s.regionCache.GroupKeysByRegion(bo, keys, equalRegionStartKey)
	if err != nil {
		return nil, errors.Trace(err)
	}

	var batches []batch
	for regionID, groupKeys := range groups {
		batches = appendKeyBatches(batches, regionID, groupKeys, rawBatchPutSize)
	}

	if len(batches) == 0 {
		return nil, nil
	}
	// The first time it enters this function.
	if bo.totalSleep == 0 {
		logutil.BgLogger().Info("split batch regions request",
			zap.Int("split key count", len(keys)),
			zap.Int("batch count", len(batches)),
			zap.Uint64("first batch, region ID", batches[0].regionID.id),
			zap.Stringer("first split key", kv.Key(batches[0].keys[0])))
	}
	if len(batches) == 1 {
		resp := s.batchSendSingleRegion(bo, batches[0], scatter)
		return resp.resp, errors.Trace(resp.err)
	}
	ch := make(chan singleBatchResp, len(batches))
	for _, batch1 := range batches {
		go func(b batch) {
			backoffer, cancel := bo.Fork()
			defer cancel()

			util.WithRecovery(func() {
				select {
				case ch <- s.batchSendSingleRegion(backoffer, b, scatter):
				case <-bo.ctx.Done():
					ch <- singleBatchResp{err: bo.ctx.Err()}
				}
			}, func(r interface{}) {
				if r != nil {
					ch <- singleBatchResp{err: errors.Errorf("%v", r)}
				}
			})
		}(batch1)
	}

	srResp := &kvrpcpb.SplitRegionResponse{Regions: make([]*metapb.Region, 0, len(keys)*2)}
	for i := 0; i < len(batches); i++ {
		batchResp := <-ch
		if batchResp.err != nil {
			logutil.BgLogger().Info("batch split regions failed", zap.Error(batchResp.err))
			if err == nil {
				err = batchResp.err
			}
		}

		// If the split succeeds and the scatter fails, we also need to add the region IDs.
		if batchResp.resp != nil {
			spResp := batchResp.resp.Resp.(*kvrpcpb.SplitRegionResponse)
			regions := spResp.GetRegions()
			srResp.Regions = append(srResp.Regions, regions...)
		}
	}
	return &tikvrpc.Response{Resp: srResp}, errors.Trace(err)
}

func (s *tikvStore) batchSendSingleRegion(bo *Backoffer, batch batch, scatter bool) singleBatchResp {
	failpoint.Inject("MockSplitRegionTimeout", func(val failpoint.Value) {
		if val.(bool) {
			if _, ok := bo.ctx.Deadline(); ok {
				<-bo.ctx.Done()
			}
		}
	})

	req := tikvrpc.NewRequest(tikvrpc.CmdSplitRegion, &kvrpcpb.SplitRegionRequest{
		SplitKeys: batch.keys,
	}, kvrpcpb.Context{
		Priority: kvrpcpb.CommandPri_Normal,
	})

	sender := NewRegionRequestSender(s.regionCache, s.client)
	resp, err := sender.SendReq(bo, req, batch.regionID, readTimeoutShort)

	batchResp := singleBatchResp{resp: resp}
	if err != nil {
		batchResp.err = errors.Trace(err)
		return batchResp
	}
	regionErr, err := resp.GetRegionError()
	if err != nil {
		batchResp.err = errors.Trace(err)
		return batchResp
	}
	if regionErr != nil {
		err := bo.Backoff(BoRegionMiss, errors.New(regionErr.String()))
		if err != nil {
			batchResp.err = errors.Trace(err)
			return batchResp
		}
		resp, err = s.splitBatchRegionsReq(bo, batch.keys, scatter)
		batchResp.resp = resp
		batchResp.err = err
		return batchResp
	}

	spResp := resp.Resp.(*kvrpcpb.SplitRegionResponse)
	regions := spResp.GetRegions()
	if len(regions) > 0 {
		// Divide a region into n, one of them may not need to be scattered,
		// so n-1 needs to be scattered to other stores.
		spResp.Regions = regions[:len(regions)-1]
	}
	logutil.BgLogger().Info("batch split regions complete",
		zap.Uint64("batch region ID", batch.regionID.id),
		zap.Stringer("first at", kv.Key(batch.keys[0])),
		zap.Stringer("first new region left", stringutil.MemoizeStr(func() string {
			if len(spResp.Regions) == 0 {
				return ""
			}
			return logutil.Hex(spResp.Regions[0]).String()
		})),
		zap.Int("new region count", len(spResp.Regions)))

	if !scatter {
		if len(spResp.Regions) == 0 {
			return batchResp
		}
		return batchResp
	}

	for i, r := range spResp.Regions {
		if err = s.scatterRegion(bo.ctx, r.Id); err == nil {
			logutil.BgLogger().Info("batch split regions, scatter region complete",
				zap.Uint64("batch region ID", batch.regionID.id),
				zap.Stringer("at", kv.Key(batch.keys[i])),
				zap.Stringer("new region left", logutil.Hex(r)))
			continue
		}

		logutil.BgLogger().Info("batch split regions, scatter region failed",
			zap.Uint64("batch region ID", batch.regionID.id),
			zap.Stringer("at", kv.Key(batch.keys[i])),
			zap.Stringer("new region left", logutil.Hex(r)),
			zap.Error(err))
		if batchResp.err == nil {
			batchResp.err = err
		}
		if ErrPDServerTimeout.Equal(err) {
			break
		}
	}
	return batchResp
}

// SplitRegions splits regions by splitKeys.
func (s *tikvStore) SplitRegions(ctx context.Context, splitKeys [][]byte, scatter bool) (regionIDs []uint64, err error) {
	bo := NewBackoffer(ctx, int(math.Min(float64(len(splitKeys))*splitRegionBackoff, maxSplitRegionsBackoff)))
	resp, err := s.splitBatchRegionsReq(bo, splitKeys, scatter)
	regionIDs = make([]uint64, 0, len(splitKeys))
	if resp != nil && resp.Resp != nil {
		spResp := resp.Resp.(*kvrpcpb.SplitRegionResponse)
		for _, r := range spResp.Regions {
			regionIDs = append(regionIDs, r.Id)
		}
		logutil.BgLogger().Info("split regions complete", zap.Int("region count", len(regionIDs)), zap.Uint64s("region IDs", regionIDs))
	}
	return regionIDs, errors.Trace(err)
}

func (s *tikvStore) scatterRegion(ctx context.Context, regionID uint64) error {
	logutil.BgLogger().Info("start scatter region",
		zap.Uint64("regionID", regionID))
	bo := NewBackoffer(ctx, scatterRegionBackoff)
	for {
		err := s.pdClient.ScatterRegion(ctx, regionID)

		failpoint.Inject("MockScatterRegionTimeout", func(val failpoint.Value) {
			if val.(bool) {
				err = ErrPDServerTimeout
			}
		})

		if err == nil {
			break
		}
		err = bo.Backoff(BoPDRPC, errors.New(err.Error()))
		if err != nil {
			return errors.Trace(err)
		}
	}
	logutil.BgLogger().Debug("scatter region complete",
		zap.Uint64("regionID", regionID))
	return nil
}

// WaitScatterRegionFinish implements SplittableStore interface.
// backOff is the back off time of the wait scatter region.(Milliseconds)
// if backOff <= 0, the default wait scatter back off time will be used.
func (s *tikvStore) WaitScatterRegionFinish(ctx context.Context, regionID uint64, backOff int) error {
	if backOff <= 0 {
		backOff = waitScatterRegionFinishBackoff
	}
	logutil.BgLogger().Info("wait scatter region",
		zap.Uint64("regionID", regionID), zap.Int("backoff(ms)", backOff))

	bo := NewBackoffer(ctx, backOff)
	logFreq := 0
	for {
		resp, err := s.pdClient.GetOperator(ctx, regionID)
		if err == nil && resp != nil {
			if !bytes.Equal(resp.Desc, []byte("scatter-region")) || resp.Status != pdpb.OperatorStatus_RUNNING {
				logutil.BgLogger().Info("wait scatter region finished",
					zap.Uint64("regionID", regionID))
				return nil
			}
			if logFreq%10 == 0 {
				logutil.BgLogger().Info("wait scatter region",
					zap.Uint64("regionID", regionID),
					zap.String("reverse", string(resp.Desc)),
					zap.String("status", pdpb.OperatorStatus_name[int32(resp.Status)]))
			}
			logFreq++
		}
		if err != nil {
			err = bo.Backoff(BoRegionMiss, errors.New(err.Error()))
		} else {
			err = bo.Backoff(BoRegionMiss, errors.New("wait scatter region timeout"))
		}
		if err != nil {
			return errors.Trace(err)
		}
	}
}

// CheckRegionInScattering uses to check whether scatter region finished.
func (s *tikvStore) CheckRegionInScattering(regionID uint64) (bool, error) {
	bo := NewBackoffer(context.Background(), locateRegionMaxBackoff)
	for {
		resp, err := s.pdClient.GetOperator(context.Background(), regionID)
		if err == nil && resp != nil {
			if !bytes.Equal(resp.Desc, []byte("scatter-region")) || resp.Status != pdpb.OperatorStatus_RUNNING {
				return false, nil
			}
		}
		if err != nil {
			err = bo.Backoff(BoRegionMiss, errors.New(err.Error()))
		} else {
			return true, nil
		}
		if err != nil {
			return true, errors.Trace(err)
		}
	}
}
<commit_msg>tikv: use Split Backoffer to backoff Scatter error (#18376)<commit_after>// Copyright 2017 PingCAP, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// See the License for the specific language governing permissions and
// limitations under the License.

package tikv

import (
	"bytes"
	"context"
	"math"

	"github.com/pingcap/errors"
	"github.com/pingcap/failpoint"
	"github.com/pingcap/kvproto/pkg/kvrpcpb"
	"github.com/pingcap/kvproto/pkg/metapb"
	"github.com/pingcap/kvproto/pkg/pdpb"
	"github.com/pingcap/tidb/kv"
	"github.com/pingcap/tidb/store/tikv/tikvrpc"
	"github.com/pingcap/tidb/util"
	"github.com/pingcap/tidb/util/logutil"
	"github.com/pingcap/tidb/util/stringutil"
	"go.uber.org/zap"
)

func equalRegionStartKey(key, regionStartKey []byte) bool {
	return bytes.Equal(key, regionStartKey)
}

func (s *tikvStore) splitBatchRegionsReq(bo *Backoffer, keys [][]byte, scatter bool) (*tikvrpc.Response, error) {
	// equalRegionStartKey is used to filter split keys.
	// If the split key is equal to the start key of the region, then the key has been split, we need to skip the split key.
	groups, _, err := s.regionCache.GroupKeysByRegion(bo, keys, equalRegionStartKey)
	if err != nil {
		return nil, errors.Trace(err)
	}

	var batches []batch
	for regionID, groupKeys := range groups {
		batches = appendKeyBatches(batches, regionID, groupKeys, rawBatchPutSize)
	}

	if len(batches) == 0 {
		return nil, nil
	}
	// The first time it enters this function.
	if bo.totalSleep == 0 {
		logutil.BgLogger().Info("split batch regions request",
			zap.Int("split key count", len(keys)),
			zap.Int("batch count", len(batches)),
			zap.Uint64("first batch, region ID", batches[0].regionID.id),
			zap.Stringer("first split key", kv.Key(batches[0].keys[0])))
	}
	if len(batches) == 1 {
		resp := s.batchSendSingleRegion(bo, batches[0], scatter)
		return resp.resp, errors.Trace(resp.err)
	}
	ch := make(chan singleBatchResp, len(batches))
	for _, batch1 := range batches {
		go func(b batch) {
			backoffer, cancel := bo.Fork()
			defer cancel()

			util.WithRecovery(func() {
				select {
				case ch <- s.batchSendSingleRegion(backoffer, b, scatter):
				case <-bo.ctx.Done():
					ch <- singleBatchResp{err: bo.ctx.Err()}
				}
			}, func(r interface{}) {
				if r != nil {
					ch <- singleBatchResp{err: errors.Errorf("%v", r)}
				}
			})
		}(batch1)
	}

	srResp := &kvrpcpb.SplitRegionResponse{Regions: make([]*metapb.Region, 0, len(keys)*2)}
	for i := 0; i < len(batches); i++ {
		batchResp := <-ch
		if batchResp.err != nil {
			logutil.BgLogger().Info("batch split regions failed", zap.Error(batchResp.err))
			if err == nil {
				err = batchResp.err
			}
		}

		// If the split succeeds and the scatter fails, we also need to add the region IDs.
		if batchResp.resp != nil {
			spResp := batchResp.resp.Resp.(*kvrpcpb.SplitRegionResponse)
			regions := spResp.GetRegions()
			srResp.Regions = append(srResp.Regions, regions...)
		}
	}
	return &tikvrpc.Response{Resp: srResp}, errors.Trace(err)
}

func (s *tikvStore) batchSendSingleRegion(bo *Backoffer, batch batch, scatter bool) singleBatchResp {
	failpoint.Inject("MockSplitRegionTimeout", func(val failpoint.Value) {
		if val.(bool) {
			if _, ok := bo.ctx.Deadline(); ok {
				<-bo.ctx.Done()
			}
		}
	})

	req := tikvrpc.NewRequest(tikvrpc.CmdSplitRegion, &kvrpcpb.SplitRegionRequest{
		SplitKeys: batch.keys,
	}, kvrpcpb.Context{
		Priority: kvrpcpb.CommandPri_Normal,
	})

	sender := NewRegionRequestSender(s.regionCache, s.client)
	resp, err := sender.SendReq(bo, req, batch.regionID, readTimeoutShort)

	batchResp := singleBatchResp{resp: resp}
	if err != nil {
		batchResp.err = errors.Trace(err)
		return batchResp
	}
	regionErr, err := resp.GetRegionError()
	if err != nil {
		batchResp.err = errors.Trace(err)
		return batchResp
	}
	if regionErr != nil {
		err := bo.Backoff(BoRegionMiss, errors.New(regionErr.String()))
		if err != nil {
			batchResp.err = errors.Trace(err)
			return batchResp
		}
		resp, err = s.splitBatchRegionsReq(bo, batch.keys, scatter)
		batchResp.resp = resp
		batchResp.err = err
		return batchResp
	}

	spResp := resp.Resp.(*kvrpcpb.SplitRegionResponse)
	regions := spResp.GetRegions()
	if len(regions) > 0 {
		// Divide a region into n, one of them may not need to be scattered,
		// so n-1 needs to be scattered to other stores.
		spResp.Regions = regions[:len(regions)-1]
	}
	logutil.BgLogger().Info("batch split regions complete",
		zap.Uint64("batch region ID", batch.regionID.id),
		zap.Stringer("first at", kv.Key(batch.keys[0])),
		zap.Stringer("first new region left", stringutil.MemoizeStr(func() string {
			if len(spResp.Regions) == 0 {
				return ""
			}
			return logutil.Hex(spResp.Regions[0]).String()
		})),
		zap.Int("new region count", len(spResp.Regions)))

	if !scatter {
		if len(spResp.Regions) == 0 {
			return batchResp
		}
		return batchResp
	}

	for i, r := range spResp.Regions {
		if err = s.scatterRegion(bo, r.Id); err == nil {
			logutil.BgLogger().Info("batch split regions, scatter region complete",
				zap.Uint64("batch region ID", batch.regionID.id),
				zap.Stringer("at", kv.Key(batch.keys[i])),
				zap.Stringer("new region left", logutil.Hex(r)))
			continue
		}

		logutil.BgLogger().Info("batch split regions, scatter region failed",
			zap.Uint64("batch region ID", batch.regionID.id),
			zap.Stringer("at", kv.Key(batch.keys[i])),
			zap.Stringer("new region left", logutil.Hex(r)),
			zap.Error(err))
		if batchResp.err == nil {
			batchResp.err = err
		}
		if ErrPDServerTimeout.Equal(err) {
			break
		}
	}
	return batchResp
}

// SplitRegions splits regions by splitKeys.
func (s *tikvStore) SplitRegions(ctx context.Context, splitKeys [][]byte, scatter bool) (regionIDs []uint64, err error) {
	bo := NewBackoffer(ctx, int(math.Min(float64(len(splitKeys))*splitRegionBackoff, maxSplitRegionsBackoff)))
	resp, err := s.splitBatchRegionsReq(bo, splitKeys, scatter)
	regionIDs = make([]uint64, 0, len(splitKeys))
	if resp != nil && resp.Resp != nil {
		spResp := resp.Resp.(*kvrpcpb.SplitRegionResponse)
		for _, r := range spResp.Regions {
			regionIDs = append(regionIDs, r.Id)
		}
		logutil.BgLogger().Info("split regions complete", zap.Int("region count", len(regionIDs)), zap.Uint64s("region IDs", regionIDs))
	}
	return regionIDs, errors.Trace(err)
}

func (s *tikvStore) scatterRegion(bo *Backoffer, regionID uint64) error {
	logutil.BgLogger().Info("start scatter region",
		zap.Uint64("regionID", regionID))
	for {
		err := s.pdClient.ScatterRegion(bo.ctx, regionID)

		failpoint.Inject("MockScatterRegionTimeout", func(val failpoint.Value) {
			if val.(bool) {
				err = ErrPDServerTimeout
			}
		})

		if err == nil {
			break
		}
		err = bo.Backoff(BoPDRPC, errors.New(err.Error()))
		if err != nil {
			return errors.Trace(err)
		}
	}
	logutil.BgLogger().Debug("scatter region complete",
		zap.Uint64("regionID", regionID))
	return nil
}

// WaitScatterRegionFinish implements SplittableStore interface.
// backOff is the back off time of the wait scatter region.(Milliseconds)
// if backOff <= 0, the default wait scatter back off time will be used.
func (s *tikvStore) WaitScatterRegionFinish(ctx context.Context, regionID uint64, backOff int) error {
	if backOff <= 0 {
		backOff = waitScatterRegionFinishBackoff
	}
	logutil.BgLogger().Info("wait scatter region",
		zap.Uint64("regionID", regionID), zap.Int("backoff(ms)", backOff))

	bo := NewBackoffer(ctx, backOff)
	logFreq := 0
	for {
		resp, err := s.pdClient.GetOperator(ctx, regionID)
		if err == nil && resp != nil {
			if !bytes.Equal(resp.Desc, []byte("scatter-region")) || resp.Status != pdpb.OperatorStatus_RUNNING {
				logutil.BgLogger().Info("wait scatter region finished",
					zap.Uint64("regionID", regionID))
				return nil
			}
			if logFreq%10 == 0 {
				logutil.BgLogger().Info("wait scatter region",
					zap.Uint64("regionID", regionID),
					zap.String("reverse", string(resp.Desc)),
					zap.String("status", pdpb.OperatorStatus_name[int32(resp.Status)]))
			}
			logFreq++
		}
		if err != nil {
			err = bo.Backoff(BoRegionMiss, errors.New(err.Error()))
		} else {
			err = bo.Backoff(BoRegionMiss, errors.New("wait scatter region timeout"))
		}
		if err != nil {
			return errors.Trace(err)
		}
	}
}

// CheckRegionInScattering uses to check whether scatter region finished.
func (s *tikvStore) CheckRegionInScattering(regionID uint64) (bool, error) {
	bo := NewBackoffer(context.Background(), locateRegionMaxBackoff)
	for {
		resp, err := s.pdClient.GetOperator(context.Background(), regionID)
		if err == nil && resp != nil {
			if !bytes.Equal(resp.Desc, []byte("scatter-region")) || resp.Status != pdpb.OperatorStatus_RUNNING {
				return false, nil
			}
		}
		if err != nil {
			err = bo.Backoff(BoRegionMiss, errors.New(err.Error()))
		} else {
			return true, nil
		}
		if err != nil {
			return true, errors.Trace(err)
		}
	}
}
<|endoftext|>


<commit_before>package stun

import (
	"encoding/binary"
	"net"

	"io"

	"github.com/pkg/errors"
)

// Clipped from net.IP
// Is p all zeros?
func isZeros(p net.IP) bool {
	for i := 0; i < len(p); i++ {
		if p[i] != 0 {
			return false
		}
	}
	return true
}

// Clipped from net.IP
// To4 converts the IPv4 address ip to a 4-byte representation.
// If ip is not an IPv4 address, To4 returns nil.
func toIPv4(ip net.IP) net.IP {
	if len(ip) == net.IPv4len {
		return ip
	}
	if len(ip) == net.IPv6len &&
		isZeros(ip[0:10]) &&
		ip[10] == 0xff &&
		ip[11] == 0xff {
		return ip[12:16]
	}
	return nil
}

func xor(dst, l, r []byte) {
	n := len(l)
	if len(r) < n {
		n = len(r)
	}
	for i := 0; i < n; i++ {
		dst[i] = l[i] ^ r[i]
	}
}

//  0                   1                   2                   3
//  0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
// |x x x x x x x x|    Family     |         X-Port                |
// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
// |                X-Address (Variable)
// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+

// XorMappedAddress represents the XOR-MAPPED-ADDRESS attribute and
// provides the Pack and Unpack methods of Attribute
// https://tools.ietf.org/html/rfc5389#section-15.2
type XorMappedAddress struct {
	IP   net.IP
	Port int
}

// https://tools.ietf.org/html/rfc5389#section-15.1
// The address family can take on the following values:
//   0x01:IPv4
//   0x02:IPv6
const (
	familyIPv4 uint16 = 0x01
	familyIPv6 uint16 = 0x02
)

const (
	familyStart = 0
	// family length is actually 1 byte, but the preceding byte in the header must be all zero according
	// to https://tools.ietf.org/html/rfc5389#section-15.1
	familyLength = 2
	portStart    = 2
	portLength   = 2
	addressStart = 4
)

func getIPAndFamily(unkIP net.IP) (ip net.IP, family uint16, err error) {
	ip = unkIP
	family = familyIPv4

	if len(ip) == net.IPv6len {
		ipv4 := toIPv4(ip)
		if ipv4 != nil {
			ip = ipv4
		} else {
			family = familyIPv6
		}
	} else if len(ip) != net.IPv4len {
		err = errors.Errorf("invalid IP length %d", len(ip))
	}

	return ip, family, err
}
func (x *XorMappedAddress) Pack(message *Message) (*RawAttribute, error) {
	ra := &RawAttribute{}
	ra.Type = AttrXORMappedAddress

	ip, family, err := getIPAndFamily(x.IP)
	if err != nil {
		return nil, errors.Wrap(err, "unable to get IP and family")
	}

	len := familyLength + portLength + len(ip)
	pad := getPadding(len)
	v := make([]byte, len+pad)

	// Family
	binary.BigEndian.PutUint16(v[familyStart:familyStart+familyLength], family)
	// Port
	binary.BigEndian.PutUint16(v[portStart:portStart+portLength], uint16(x.Port))
	xor(v[portStart:portStart+portLength], v[portStart:portStart+portLength], message.TransactionID[0:2])
	// Address
	xor(v[addressStart:], v[addressStart:], message.TransactionID)

	ra.Value = v
	ra.Length = uint16(len)
	// Always a multiple of 4 due to addrs being 4 or 16 bytes
	ra.Pad = uint16(pad)

	return ra, nil
}

func (x *XorMappedAddress) Unpack(message *Message, rawAttribute *RawAttribute) error {
	v := rawAttribute.Value

	if len(v) < familyStart+familyLength {
		return io.ErrUnexpectedEOF
	}

	family := binary.BigEndian.Uint16(v[familyStart : familyStart+familyLength])

	if family != familyIPv4 && family != familyIPv6 {
		return errors.Errorf("invalid family %d (expected IPv4(%d) or IPv6(%d)", family, familyIPv4, familyIPv6)
	}

	if len(v[portStart:]) < portLength {
		return io.ErrUnexpectedEOF
	}

	var p [2]byte
	// Transaction ID [0,2] is top half of magic cookie
	xor(p[:], v[portStart:portStart+portLength], message.TransactionID[0:2])

	x.Port = int(binary.BigEndian.Uint16(p[:]))

	al := net.IPv4len
	if family == familyIPv6 {
		al = net.IPv6len
	}

	if len(v[addressStart:]) < al {
		return io.ErrUnexpectedEOF
	}

	if len(v[addressStart:]) > al {
		return errors.Errorf("invalid length for %d family address (%d)", family, len(v[4:]))
	}

	x.IP = make([]byte, al)
	xor(x.IP[:], v[4:], message.TransactionID)

	return nil
}
<commit_msg>turn: Copy IP into xor addr attribute value<commit_after>package stun

import (
	"encoding/binary"
	"net"

	"io"

	"github.com/pkg/errors"
)

// Clipped from net.IP
// Is p all zeros?
func isZeros(p net.IP) bool {
	for i := 0; i < len(p); i++ {
		if p[i] != 0 {
			return false
		}
	}
	return true
}

// Clipped from net.IP
// To4 converts the IPv4 address ip to a 4-byte representation.
// If ip is not an IPv4 address, To4 returns nil.
func toIPv4(ip net.IP) net.IP {
	if len(ip) == net.IPv4len {
		return ip
	}
	if len(ip) == net.IPv6len &&
		isZeros(ip[0:10]) &&
		ip[10] == 0xff &&
		ip[11] == 0xff {
		return ip[12:16]
	}
	return nil
}

func xor(dst, l, r []byte) {
	n := len(l)
	if len(r) < n {
		n = len(r)
	}
	for i := 0; i < n; i++ {
		dst[i] = l[i] ^ r[i]
	}
}

//  0                   1                   2                   3
//  0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
// |x x x x x x x x|    Family     |         X-Port                |
// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
// |                X-Address (Variable)
// +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+

// XorMappedAddress represents the XOR-MAPPED-ADDRESS attribute and
// provides the Pack and Unpack methods of Attribute
// https://tools.ietf.org/html/rfc5389#section-15.2
type XorMappedAddress struct {
	IP   net.IP
	Port int
}

// https://tools.ietf.org/html/rfc5389#section-15.1
// The address family can take on the following values:
//   0x01:IPv4
//   0x02:IPv6
const (
	familyIPv4 uint16 = 0x01
	familyIPv6 uint16 = 0x02
)

const (
	familyStart = 0
	// family length is actually 1 byte, but the preceding byte in the header must be all zero according
	// to https://tools.ietf.org/html/rfc5389#section-15.1
	familyLength = 2
	portStart    = 2
	portLength   = 2
	addressStart = 4
)

func getIPAndFamily(unkIP net.IP) (ip net.IP, family uint16, err error) {
	ip = unkIP
	family = familyIPv4

	if len(ip) == net.IPv6len {
		ipv4 := toIPv4(ip)
		if ipv4 != nil {
			ip = ipv4
		} else {
			family = familyIPv6
		}
	} else if len(ip) != net.IPv4len {
		err = errors.Errorf("invalid IP length %d", len(ip))
	}

	return ip, family, err
}
func (x *XorMappedAddress) Pack(message *Message) (*RawAttribute, error) {
	ra := &RawAttribute{}
	ra.Type = AttrXORMappedAddress

	ip, family, err := getIPAndFamily(x.IP)
	if err != nil {
		return nil, errors.Wrap(err, "unable to get IP and family")
	}

	len := familyLength + portLength + len(ip)
	pad := getPadding(len)
	v := make([]byte, len+pad)

	// Family
	binary.BigEndian.PutUint16(v[familyStart:familyStart+familyLength], family)
	// Port
	binary.BigEndian.PutUint16(v[portStart:portStart+portLength], uint16(x.Port))
	xor(v[portStart:portStart+portLength], v[portStart:portStart+portLength], message.TransactionID[0:2])
	// Address
	copy(v[addressStart:], ip)
	xor(v[addressStart:], v[addressStart:], message.TransactionID)

	ra.Value = v
	ra.Length = uint16(len)
	// Always a multiple of 4 due to addrs being 4 or 16 bytes
	ra.Pad = uint16(pad)

	return ra, nil
}

func (x *XorMappedAddress) Unpack(message *Message, rawAttribute *RawAttribute) error {
	v := rawAttribute.Value

	if len(v) < familyStart+familyLength {
		return io.ErrUnexpectedEOF
	}

	family := binary.BigEndian.Uint16(v[familyStart : familyStart+familyLength])

	if family != familyIPv4 && family != familyIPv6 {
		return errors.Errorf("invalid family %d (expected IPv4(%d) or IPv6(%d)", family, familyIPv4, familyIPv6)
	}

	if len(v[portStart:]) < portLength {
		return io.ErrUnexpectedEOF
	}

	var p [2]byte
	// Transaction ID [0,2] is top half of magic cookie
	xor(p[:], v[portStart:portStart+portLength], message.TransactionID[0:2])

	x.Port = int(binary.BigEndian.Uint16(p[:]))

	al := net.IPv4len
	if family == familyIPv6 {
		al = net.IPv6len
	}

	if len(v[addressStart:]) < al {
		return io.ErrUnexpectedEOF
	}

	if len(v[addressStart:]) > al {
		return errors.Errorf("invalid length for %d family address (%d)", family, len(v[4:]))
	}

	x.IP = make([]byte, al)
	xor(x.IP[:], v[4:], message.TransactionID)

	return nil
}
<|endoftext|>


<commit_before>package postal

/*
#cgo pkg-config: libpostal
#include <libpostal/libpostal.h>
#include <stdlib.h>
*/
import "C"

import (
    "log"
    "unsafe"
    "unicode/utf8"
)

func init() {
    if (!bool(C.libpostal_setup()) || !bool(C.libpostal_setup_parser())) {
        log.Fatal("Could not load libpostal")
    }
}

type ParserOptions struct {
    Language string
    Country string
}

func getDefaultParserOptions() ParserOptions {
    return ParserOptions {
        Language: "",
        Country: "",
    }
}

var parserDefaultOptions = getDefaultParserOptions()

type ParsedComponent struct {
    Label string `json:"label"`
    Value string `json:"value"`
}

func ParseAddressOptions(address string, options ParserOptions) []ParsedComponent {
    if !utf8.ValidString(address) {
        return nil
    }

    cAddress := C.CString(address)
    defer C.free(unsafe.Pointer(cAddress))

    //var b *C.char
    //ptr_size := unsafe.Sizeof(b)

    cOptions := C.libpostal_get_address_parser_default_options()
    if options.Language != "" {
        cLanguage := C.CString(options.Language)
        defer C.free(unsafe.Pointer(cLanguage))

        cOptions.language = cLanguage
    }

    if options.Country != "" {
        cCountry := C.CString(options.Country)
        defer C.free(unsafe.Pointer(cCountry))

        cOptions.country = cCountry
    }

    cAddressParserResponsePtr := C.libpostal_parse_address(cAddress, cOptions)

    if cAddressParserResponsePtr == nil {
        return nil
    }

    cAddressParserResponse := *cAddressParserResponsePtr

    cNumComponents := cAddressParserResponse.num_components
    cComponents := cAddressParserResponse.components
    cLabels := cAddressParserResponse.labels

    numComponents := uint64(cNumComponents)

    parsedComponents := make([]ParsedComponent, numComponents)

    // Accessing a C array
    cComponentsPtr := (*[1<<30](*C.char))(unsafe.Pointer(cComponents))
    cLabelsPtr := (*[1<<30](*C.char))(unsafe.Pointer(cLabels))

    var i uint64
    for i = 0; i < numComponents; i++ {
        parsedComponents[i] = ParsedComponent{
            Label: C.GoString(cLabelsPtr[i]),
            Value: C.GoString(cComponentsPtr[i]),
        }
    }

    C.libpostal_address_parser_response_destroy(cAddressParserResponsePtr)

    return parsedComponents
}

func ParseAddress(address string) []ParsedComponent {
    return ParseAddressOptions(address, parserDefaultOptions)
}
<commit_msg>[concurrency] adding a mutex around parser CGo calls<commit_after>package postal

/*
#cgo pkg-config: libpostal
#include <libpostal/libpostal.h>
#include <stdlib.h>
*/
import "C"

import (
    "log"
    "sync"
    "unsafe"
    "unicode/utf8"
)

var mu sync.Mutex

func init() {
    if (!bool(C.libpostal_setup()) || !bool(C.libpostal_setup_parser())) {
        log.Fatal("Could not load libpostal")
    }
}

type ParserOptions struct {
    Language string
    Country string
}

func getDefaultParserOptions() ParserOptions {
    return ParserOptions {
        Language: "",
        Country: "",
    }
}

var parserDefaultOptions = getDefaultParserOptions()

type ParsedComponent struct {
    Label string `json:"label"`
    Value string `json:"value"`
}

func ParseAddressOptions(address string, options ParserOptions) []ParsedComponent {
    if !utf8.ValidString(address) {
        return nil
    }

    mu.Lock()
    defer mu.Unlock()

    cAddress := C.CString(address)
    defer C.free(unsafe.Pointer(cAddress))

    cOptions := C.libpostal_get_address_parser_default_options()
    if options.Language != "" {
        cLanguage := C.CString(options.Language)
        defer C.free(unsafe.Pointer(cLanguage))

        cOptions.language = cLanguage
    }

    if options.Country != "" {
        cCountry := C.CString(options.Country)
        defer C.free(unsafe.Pointer(cCountry))

        cOptions.country = cCountry
    }

    cAddressParserResponsePtr := C.libpostal_parse_address(cAddress, cOptions)

    if cAddressParserResponsePtr == nil {
        return nil
    }

    cAddressParserResponse := *cAddressParserResponsePtr

    cNumComponents := cAddressParserResponse.num_components
    cComponents := cAddressParserResponse.components
    cLabels := cAddressParserResponse.labels

    numComponents := uint64(cNumComponents)

    parsedComponents := make([]ParsedComponent, numComponents)

    // Accessing a C array
    cComponentsPtr := (*[1<<30](*C.char))(unsafe.Pointer(cComponents))
    cLabelsPtr := (*[1<<30](*C.char))(unsafe.Pointer(cLabels))

    var i uint64
    for i = 0; i < numComponents; i++ {
        parsedComponents[i] = ParsedComponent{
            Label: C.GoString(cLabelsPtr[i]),
            Value: C.GoString(cComponentsPtr[i]),
        }
    }

    C.libpostal_address_parser_response_destroy(cAddressParserResponsePtr)

    return parsedComponents
}

func ParseAddress(address string) []ParsedComponent {
    return ParseAddressOptions(address, parserDefaultOptions)
}
<|endoftext|>


<commit_before>// Package parser provides a parsing function for GoCaml.
package parser

import (
	"bytes"
	"fmt"
	"github.com/rhysd/gocaml/ast"
	"github.com/rhysd/gocaml/token"
)

type pseudoLexer struct {
	tokens       chan token.Token
	errorCount   int
	errorMessage bytes.Buffer
	result       ast.Expr
}

func (l *pseudoLexer) Lex(lval *yySymType) int {
	for {
		select {
		case t := <-l.tokens:
			lval.token = &t

			switch t.Kind {
			case token.EOF:
				// Zero means input ends
				// (see golang.org/x/tools/cmd/goyacc/testdata/expr/expr.y)
				return 0
			case token.COMMENT:
				continue
			case token.ILLEGAL:
				return -1
			}

			// XXX:
			// Converting token value into yacc's token.
			// This conversion requires that token order must the same as
			// yacc's token order. EOF is a first token. So we can use it
			// to make an offset between token value and yacc's token value.
			return int(t.Kind) + ILLEGAL
		}
	}
}

func (l *pseudoLexer) Error(msg string) {
	l.errorCount++
	l.errorMessage.WriteString(fmt.Sprintf("  * %s\n", msg))
}

func (l *pseudoLexer) getError() error {
	return fmt.Errorf("%d error(s) while parsing\n%s", l.errorCount, l.errorMessage.String())
}

// Parse parses given tokens and returns parsed AST.
// Tokens are passed via channel.
func Parse(tokens chan token.Token) (ast.Expr, error) {
	yyErrorVerbose = true

	l := &pseudoLexer{tokens: tokens}
	ret := yyParse(l)

	if ret != 0 || l.errorCount != 0 {
		return nil, l.getError()
	}

	root := l.result
	if root == nil {
		return nil, fmt.Errorf("Parsing failed")
	}

	return root, nil
}
<commit_msg>fix return value of pseudo lexer on error<commit_after>// Package parser provides a parsing function for GoCaml.
package parser

import (
	"bytes"
	"fmt"
	"github.com/rhysd/gocaml/ast"
	"github.com/rhysd/gocaml/token"
)

type pseudoLexer struct {
	tokens       chan token.Token
	errorCount   int
	errorMessage bytes.Buffer
	result       ast.Expr
}

func (l *pseudoLexer) Lex(lval *yySymType) int {
	for {
		select {
		case t := <-l.tokens:
			lval.token = &t

			switch t.Kind {
			case token.EOF, token.ILLEGAL:
				// Zero means input ends
				// (see golang.org/x/tools/cmd/goyacc/testdata/expr/expr.y)
				return 0
			case token.COMMENT:
				continue
			}

			// XXX:
			// Converting token value into yacc's token.
			// This conversion requires that token order must the same as
			// yacc's token order. EOF is a first token. So we can use it
			// to make an offset between token value and yacc's token value.
			return int(t.Kind) + ILLEGAL
		}
	}
}

func (l *pseudoLexer) Error(msg string) {
	l.errorCount++
	l.errorMessage.WriteString(fmt.Sprintf("  * %s\n", msg))
}

func (l *pseudoLexer) getError() error {
	return fmt.Errorf("%d error(s) while parsing\n%s", l.errorCount, l.errorMessage.String())
}

// Parse parses given tokens and returns parsed AST.
// Tokens are passed via channel.
func Parse(tokens chan token.Token) (ast.Expr, error) {
	yyErrorVerbose = true

	l := &pseudoLexer{tokens: tokens}
	ret := yyParse(l)

	if ret != 0 || l.errorCount != 0 {
		return nil, l.getError()
	}

	root := l.result
	if root == nil {
		return nil, fmt.Errorf("Parsing failed")
	}

	return root, nil
}
<|endoftext|>


<commit_before>package parser

import (
	"regexp"
	"sort"
	"strings"
)

// Result is a Parse result, returning the matched repo, issue, etc. as applicable
type Result struct {
	Owner string
	Name  string
	Match string // the matched shorthand value, if applicable
	Query string // the remainder of the input
}

// HasRepo checks if the result has a repo, either from a matched repo shorthand,
// or from an explicit owner/name.
func (r *Result) HasRepo() bool {
	return len(r.Name) > 0
}

// Repo returns the repo defined in the result, either from a matched repo
// shorthand or from an explicit owner/name.
func (r *Result) Repo() string {
	if r.HasRepo() {
		return r.Owner + "/" + r.Name
	}
	return ""
}

// SetRepo overrides owner and name on the result from an `owner/name` string.
func (r *Result) SetRepo(repo string) error {
	parts := strings.SplitN(repo, "/", 2)
	if len(parts) > 1 {
		r.Owner = parts[0]
		r.Name = parts[1]
	}
	return nil
}

// HasIssue checks to see if the result's query looks like an issue reference.
func (r *Result) HasIssue() bool {
	return issueRegexp.MatchString(r.Query)
}

// Issue returns the issue reference from the query, if applicable.
func (r *Result) Issue() string {
	match := issueRegexp.FindStringSubmatch(r.Query)
	if len(match) > 0 {
		return match[1]
	}
	return ""
}

// HasPath checks to see if the query looks like a path (leading `/` followed by non-whitespace)
func (r *Result) HasPath() bool {
	return pathRegexp.MatchString(r.Query)
}

// Path returns the defined path from the query, if applicable.
func (r *Result) Path() string {
	match := pathRegexp.FindStringSubmatch(r.Query)
	if len(match) > 0 {
		return match[1]
	}
	return ""
}

// Annotation is a helper for displaying details about a match. Returns a string
// with a leading space, noting the matched shorthand and issue if applicable.
func (r *Result) Annotation() (ann string) {
	if len(r.Match) > 0 {
		ann += " (" + r.Match
		if r.HasIssue() {
			ann += "#" + r.Issue()
		}
		ann += ")"
	}
	return
}

// EmptyQuery returns true if no query has been provided.
func (r *Result) EmptyQuery() bool {
	return len(r.Query) == 0
}

// RepoAnnotation is a helper for displaying details about a match, but only
// for user/repo matches, excluding issue.
func (r *Result) RepoAnnotation() (ann string) {
	if len(r.Match) > 0 {
		ann += " (" + r.Match + ")"
	}
	return
}

// Parse takes a repo mapping and input string and attempts to extract a repo,
// issue, etc. from the input using the repo map for shorthand expansion.
func Parse(repoMap, userMap map[string]string, input string) *Result {
	owner, name, match, query := extractRepo(repoMap, input)
	if len(name) == 0 {
		owner, match, query = extractUser(userMap, input)
	}
	query = strings.Trim(query, " ")
	return &Result{
		Owner: owner,
		Name:  name,
		Match: match,
		Query: query,
	}
}

var (
	userRepoRegexp = regexp.MustCompile(`^([A-Za-z0-9][-A-Za-z0-9]*)/([\w\.\-]+)\b`) // user/repo
	issueRegexp    = regexp.MustCompile(`^#?([1-9]\d*)$`)
	pathRegexp     = regexp.MustCompile(`^(/\S*)$`)
)

func extractRepo(repoMap map[string]string, input string) (owner, name, match, query string) {
	var keys []string
	for k := range repoMap {
		keys = append(keys, k)
	}

	// sort the keys in reverse so the longest is matched first
	sort.Sort(sort.Reverse(sort.StringSlice(keys)))

	for _, k := range keys {
		if strings.HasPrefix(input, k) {
			parts := strings.SplitN(repoMap[k], "/", 2)
			if len(parts) > 1 {
				return parts[0], parts[1], k, strings.TrimLeft(input[len(k):], " ")
			}
		}
	}

	result := userRepoRegexp.FindStringSubmatch(input)
	if len(result) > 0 {
		repo, owner, name := result[0], result[1], result[2]
		return owner, name, "", strings.TrimLeft(input[len(repo):], " ")
	}
	return "", "", "", input
}

func extractUser(userMap map[string]string, input string) (user, match, query string) {
	var keys []string
	for k := range userMap {
		keys = append(keys, k)
	}

	// sort the keys in reverse so the longest is matched first
	sort.Sort(sort.Reverse(sort.StringSlice(keys)))

	for _, k := range keys {
		if strings.HasPrefix(input, k) {
			return userMap[k], k, strings.TrimLeft(input[len(k):], " ")
		}
	}

	return "", "", input
}
<commit_msg>Refill docs for Result struct<commit_after>package parser

import (
	"regexp"
	"sort"
	"strings"
)

// Result is a Parse result, returning the matched repo, issue, etc. as applicable
type Result struct {
	Owner string // the repository owner, if present
	Name  string // the repository name, if present
	Match string // the matched shorthand value, if shorthand was expanded
	Query string // the remainder of the input
}

// HasRepo checks if the result has a repo, either from a matched repo shorthand,
// or from an explicit owner/name.
func (r *Result) HasRepo() bool {
	return len(r.Name) > 0
}

// Repo returns the repo defined in the result, either from a matched repo
// shorthand or from an explicit owner/name.
func (r *Result) Repo() string {
	if r.HasRepo() {
		return r.Owner + "/" + r.Name
	}
	return ""
}

// SetRepo overrides owner and name on the result from an `owner/name` string.
func (r *Result) SetRepo(repo string) error {
	parts := strings.SplitN(repo, "/", 2)
	if len(parts) > 1 {
		r.Owner = parts[0]
		r.Name = parts[1]
	}
	return nil
}

// HasIssue checks to see if the result's query looks like an issue reference.
func (r *Result) HasIssue() bool {
	return issueRegexp.MatchString(r.Query)
}

// Issue returns the issue reference from the query, if applicable.
func (r *Result) Issue() string {
	match := issueRegexp.FindStringSubmatch(r.Query)
	if len(match) > 0 {
		return match[1]
	}
	return ""
}

// HasPath checks to see if the query looks like a path (leading `/` followed by non-whitespace)
func (r *Result) HasPath() bool {
	return pathRegexp.MatchString(r.Query)
}

// Path returns the defined path from the query, if applicable.
func (r *Result) Path() string {
	match := pathRegexp.FindStringSubmatch(r.Query)
	if len(match) > 0 {
		return match[1]
	}
	return ""
}

// Annotation is a helper for displaying details about a match. Returns a string
// with a leading space, noting the matched shorthand and issue if applicable.
func (r *Result) Annotation() (ann string) {
	if len(r.Match) > 0 {
		ann += " (" + r.Match
		if r.HasIssue() {
			ann += "#" + r.Issue()
		}
		ann += ")"
	}
	return
}

// EmptyQuery returns true if no query has been provided.
func (r *Result) EmptyQuery() bool {
	return len(r.Query) == 0
}

// RepoAnnotation is a helper for displaying details about a match, but only
// for user/repo matches, excluding issue.
func (r *Result) RepoAnnotation() (ann string) {
	if len(r.Match) > 0 {
		ann += " (" + r.Match + ")"
	}
	return
}

// Parse takes a repo mapping and input string and attempts to extract a repo,
// issue, etc. from the input using the repo map for shorthand expansion.
func Parse(repoMap, userMap map[string]string, input string) *Result {
	owner, name, match, query := extractRepo(repoMap, input)
	if len(name) == 0 {
		owner, match, query = extractUser(userMap, input)
	}
	query = strings.Trim(query, " ")
	return &Result{
		Owner: owner,
		Name:  name,
		Match: match,
		Query: query,
	}
}

var (
	userRepoRegexp = regexp.MustCompile(`^([A-Za-z0-9][-A-Za-z0-9]*)/([\w\.\-]+)\b`) // user/repo
	issueRegexp    = regexp.MustCompile(`^#?([1-9]\d*)$`)
	pathRegexp     = regexp.MustCompile(`^(/\S*)$`)
)

func extractRepo(repoMap map[string]string, input string) (owner, name, match, query string) {
	var keys []string
	for k := range repoMap {
		keys = append(keys, k)
	}

	// sort the keys in reverse so the longest is matched first
	sort.Sort(sort.Reverse(sort.StringSlice(keys)))

	for _, k := range keys {
		if strings.HasPrefix(input, k) {
			parts := strings.SplitN(repoMap[k], "/", 2)
			if len(parts) > 1 {
				return parts[0], parts[1], k, strings.TrimLeft(input[len(k):], " ")
			}
		}
	}

	result := userRepoRegexp.FindStringSubmatch(input)
	if len(result) > 0 {
		repo, owner, name := result[0], result[1], result[2]
		return owner, name, "", strings.TrimLeft(input[len(repo):], " ")
	}
	return "", "", "", input
}

func extractUser(userMap map[string]string, input string) (user, match, query string) {
	var keys []string
	for k := range userMap {
		keys = append(keys, k)
	}

	// sort the keys in reverse so the longest is matched first
	sort.Sort(sort.Reverse(sort.StringSlice(keys)))

	for _, k := range keys {
		if strings.HasPrefix(input, k) {
			return userMap[k], k, strings.TrimLeft(input[len(k):], " ")
		}
	}

	return "", "", input
}
<|endoftext|>


<commit_before>package parser

import (
	"fmt"

	"github.com/Tamrin007/monkey/ast"
	"github.com/Tamrin007/monkey/lexer"
	"github.com/Tamrin007/monkey/token"
)

const (
	_ int = iota
	LOWEST
	EQUALS
	LESSGREATER
	SUM
	PRODUCT
	PREFIX
	CALL
)

type (
	prefixParseFn func() ast.Expression
	infixParseFn  func(ast.Expression) ast.Expression
)

type Parser struct {
	l         *lexer.Lexer
	errors    []string
	curToken  token.Token
	peekToken token.Token

	prefixParseFns map[token.TokenType]prefixParseFn
	infixParseFns  map[token.TokenType]infixParseFn
}

func New(l *lexer.Lexer) *Parser {
	p := &Parser{
		l:      l,
		errors: []string{},
	}

	p.prefixParseFns = make(map[token.TokenType]prefixParseFn)
	p.registerPrefix(token.IDENT, p.parseIdentifier)

	p.nextToken()
	p.nextToken()

	return p
}

func (p *Parser) Errors() []string {
	return p.errors
}

func (p *Parser) peekError(t token.TokenType) {
	msg := fmt.Sprintf("expected next token to be %s, got %s instead", t, p.peekToken.Type)
	p.errors = append(p.errors, msg)
}

func (p *Parser) nextToken() {
	p.curToken = p.peekToken
	p.peekToken = p.l.NextToken()
}

func (p *Parser) ParseProgram() *ast.Program {
	program := &ast.Program{}
	program.Statements = []ast.Statement{}

	for p.curToken.Type != token.EOF {
		stmt := p.parseStatement()
		if stmt != nil {
			program.Statements = append(program.Statements, stmt)
		}
		p.nextToken()
	}
	return program
}

func (p *Parser) parseStatement() ast.Statement {
	switch p.curToken.Type {
	case token.LET:
		return p.parseLetStatement()
	case token.RETURN:
		return p.parseReturnStatement()
	default:
		return p.parseExpressionStatement()
	}
}

func (p *Parser) parseLetStatement() *ast.LetStatement {
	stmt := &ast.LetStatement{Token: p.curToken}

	if !p.expectPeek(token.IDENT) {
		return nil
	}

	stmt.Name = &ast.Identifier{Token: p.curToken, Value: p.curToken.Literal}

	if !p.expectPeek(token.ASSIGN) {
		return nil
	}

	// TODO: We're skiping the expressions until we encounter a semicolon
	for !p.curTokenIs(token.SEMICOLON) {
		p.nextToken()
	}

	return stmt
}

func (p *Parser) parseReturnStatement() *ast.ReturnStatement {
	stmt := &ast.ReturnStatement{Token: p.curToken}

	p.nextToken()

	// TODO: We're skipping the expressions until we encounter a semicolon
	for !p.curTokenIs(token.SEMICOLON) {
		p.nextToken()
	}

	return stmt
}

func (p *Parser) parseExpressionStatement() *ast.ExpressionStatement {
	stmt := &ast.ExpressionStatement{Token: p.curToken}

	stmt.Expression = p.parseExpression(LOWEST)

	if p.peekTokenIs(token.SEMICOLON) {
		p.nextToken()
	}

	return stmt
}

func (p *Parser) parseExpression(precedence int) ast.Expression {
	prefix := p.prefixParseFns[p.curToken.Type]
	if prefix == nil {
		return nil
	}
	leftExp := prefix()

	return leftExp
}

func (p *Parser) parseIdentifier() ast.Expression {
	return &ast.Identifier{Token: p.curToken, Value: p.curToken.Literal}
}

func (p *Parser) curTokenIs(t token.TokenType) bool {
	return p.curToken.Type == t
}

func (p *Parser) peekTokenIs(t token.TokenType) bool {
	return p.peekToken.Type == t
}

func (p *Parser) expectPeek(t token.TokenType) bool {
	if p.peekTokenIs(t) {
		p.nextToken()
		return true
	}
	p.peekError(t)
	return false
}

func (p *Parser) registerPrefix(tokenType token.TokenType, fn prefixParseFn) {
	p.prefixParseFns[tokenType] = fn
}

func (p *Parser) registerInfix(tokenType token.TokenType, fn infixParseFn) {
	p.infixParseFns[tokenType] = fn
}
<commit_msg>implement a parser for integer literal<commit_after>package parser

import (
	"fmt"
	"strconv"

	"github.com/Tamrin007/monkey/ast"
	"github.com/Tamrin007/monkey/lexer"
	"github.com/Tamrin007/monkey/token"
)

const (
	_ int = iota
	LOWEST
	EQUALS
	LESSGREATER
	SUM
	PRODUCT
	PREFIX
	CALL
)

type (
	prefixParseFn func() ast.Expression
	infixParseFn  func(ast.Expression) ast.Expression
)

type Parser struct {
	l         *lexer.Lexer
	errors    []string
	curToken  token.Token
	peekToken token.Token

	prefixParseFns map[token.TokenType]prefixParseFn
	infixParseFns  map[token.TokenType]infixParseFn
}

func New(l *lexer.Lexer) *Parser {
	p := &Parser{
		l:      l,
		errors: []string{},
	}

	p.prefixParseFns = make(map[token.TokenType]prefixParseFn)
	p.registerPrefix(token.IDENT, p.parseIdentifier)
	p.registerPrefix(token.INT, p.parseIntegerLiteral)

	p.nextToken()
	p.nextToken()

	return p
}

func (p *Parser) Errors() []string {
	return p.errors
}

func (p *Parser) peekError(t token.TokenType) {
	msg := fmt.Sprintf("expected next token to be %s, got %s instead", t, p.peekToken.Type)
	p.errors = append(p.errors, msg)
}

func (p *Parser) nextToken() {
	p.curToken = p.peekToken
	p.peekToken = p.l.NextToken()
}

func (p *Parser) ParseProgram() *ast.Program {
	program := &ast.Program{}
	program.Statements = []ast.Statement{}

	for p.curToken.Type != token.EOF {
		stmt := p.parseStatement()
		if stmt != nil {
			program.Statements = append(program.Statements, stmt)
		}
		p.nextToken()
	}
	return program
}

func (p *Parser) parseStatement() ast.Statement {
	switch p.curToken.Type {
	case token.LET:
		return p.parseLetStatement()
	case token.RETURN:
		return p.parseReturnStatement()
	default:
		return p.parseExpressionStatement()
	}
}

func (p *Parser) parseLetStatement() *ast.LetStatement {
	stmt := &ast.LetStatement{Token: p.curToken}

	if !p.expectPeek(token.IDENT) {
		return nil
	}

	stmt.Name = &ast.Identifier{Token: p.curToken, Value: p.curToken.Literal}

	if !p.expectPeek(token.ASSIGN) {
		return nil
	}

	// TODO: We're skiping the expressions until we encounter a semicolon
	for !p.curTokenIs(token.SEMICOLON) {
		p.nextToken()
	}

	return stmt
}

func (p *Parser) parseReturnStatement() *ast.ReturnStatement {
	stmt := &ast.ReturnStatement{Token: p.curToken}

	p.nextToken()

	// TODO: We're skipping the expressions until we encounter a semicolon
	for !p.curTokenIs(token.SEMICOLON) {
		p.nextToken()
	}

	return stmt
}

func (p *Parser) parseExpressionStatement() *ast.ExpressionStatement {
	stmt := &ast.ExpressionStatement{Token: p.curToken}

	stmt.Expression = p.parseExpression(LOWEST)

	if p.peekTokenIs(token.SEMICOLON) {
		p.nextToken()
	}

	return stmt
}

func (p *Parser) parseExpression(precedence int) ast.Expression {
	prefix := p.prefixParseFns[p.curToken.Type]
	if prefix == nil {
		return nil
	}
	leftExp := prefix()

	return leftExp
}

func (p *Parser) parseIdentifier() ast.Expression {
	return &ast.Identifier{Token: p.curToken, Value: p.curToken.Literal}
}

func (p *Parser) curTokenIs(t token.TokenType) bool {
	return p.curToken.Type == t
}

func (p *Parser) peekTokenIs(t token.TokenType) bool {
	return p.peekToken.Type == t
}

func (p *Parser) expectPeek(t token.TokenType) bool {
	if p.peekTokenIs(t) {
		p.nextToken()
		return true
	}
	p.peekError(t)
	return false
}

func (p *Parser) registerPrefix(tokenType token.TokenType, fn prefixParseFn) {
	p.prefixParseFns[tokenType] = fn
}

func (p *Parser) registerInfix(tokenType token.TokenType, fn infixParseFn) {
	p.infixParseFns[tokenType] = fn
}

func (p *Parser) parseIntegerLiteral() ast.Expression {
	lit := &ast.IntegerLiteral{Token: p.curToken}

	value, err := strconv.ParseInt(p.curToken.Literal, 0, 64)
	if err != nil {
		msg := fmt.Sprintf("could not parse %q as integer", p.curToken.Literal)
		p.errors = append(p.errors, msg)
		return nil
	}

	lit.Value = value
	return lit
}
<|endoftext|>


<commit_before>// Package github.com/haisum/recaptcha is google golang module for google re-captcha.
//
// Installation
//
//   go get github.com/haisum/recaptcha
//
// Usage
//
// Usage example can be found in example/main.go file.
//
//
// Source code
//
// Available on github: http://github.com/haisum/recaptcha
//
// Author: Haisum (haisumbhatti@gmail.com)
package recaptcha

import (
	"encoding/json"
	"io/ioutil"
	"net/http"
	"net/url"
	"time"
)

// recaptcha.R type represents an object of Recaptcha and has public property Secret,
// which is secret obtained from google recaptcha tool admin interface
type R struct {
	Secret    string
	lastError []string
}

// Struct for parsing json in google's response
type googleResponse struct {
	Success    bool
	ErrorCodes []string `json:"error-codes"`
}

// url to post submitted re-captcha response to
var postUrl string = "https://www.google.com/recaptcha/api/siteverify"

// Verify method, verifies if current request have valid re-captcha response and returns true or false
// This method also records any errors in validation.
// These errors can be received by calling LastError() method.
func (r *R) Verify(req http.Request) bool {
	r.lastError = make([]string, 1)
	response := req.FormValue("g-recaptcha-response")
	client := &http.Client{Timeout: 20 * time.Second}
	resp, err := client.PostForm(postUrl,
		url.Values{"secret": {r.Secret}, "response": {response}})
	if err != nil {
		r.lastError = append(r.lastError, err.Error())
		return false
	}
	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		r.lastError = append(r.lastError, err.Error())
		return false
	} else {
		gr := new(googleResponse)
		err := json.Unmarshal(body, gr)
		if err != nil {
			r.lastError = append(r.lastError, err.Error())
			return false
		}
		if !gr.Success {
			r.lastError = append(r.lastError, gr.ErrorCodes...)
		}
		return gr.Success
	}
}

// Returns errors occurred in last re-captcha validation attempt
func (r R) LastError() []string {
	return r.lastError
}
<commit_msg>Doing refactoring based on golint's suggestions. Fixes issue#2<commit_after>// Package recaptcha is google golang module for google re-captcha.
//
// Installation
//
//   go get github.com/haisum/recaptcha
//
// Usage
//
// Usage example can be found in example/main.go file.
//
//
// Source code
//
// Available on github: http://github.com/haisum/recaptcha
//
// Author: Haisum (haisumbhatti@gmail.com)
package recaptcha

import (
	"encoding/json"
	"io/ioutil"
	"net/http"
	"net/url"
	"time"
)

// R type represents an object of Recaptcha and has public property Secret,
// which is secret obtained from google recaptcha tool admin interface
type R struct {
	Secret    string
	lastError []string
}

// Struct for parsing json in google's response
type googleResponse struct {
	Success    bool
	ErrorCodes []string `json:"error-codes"`
}

// url to post submitted re-captcha response to
var postURL = "https://www.google.com/recaptcha/api/siteverify"

// Verify method, verifies if current request have valid re-captcha response and returns true or false
// This method also records any errors in validation.
// These errors can be received by calling LastError() method.
func (r *R) Verify(req http.Request) bool {
	r.lastError = make([]string, 1)
	response := req.FormValue("g-recaptcha-response")
	client := &http.Client{Timeout: 20 * time.Second}
	resp, err := client.PostForm(postURL,
		url.Values{"secret": {r.Secret}, "response": {response}})
	if err != nil {
		r.lastError = append(r.lastError, err.Error())
		return false
	}
	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		r.lastError = append(r.lastError, err.Error())
		return false
	}
	gr := new(googleResponse)
	err = json.Unmarshal(body, gr)
	if err != nil {
		r.lastError = append(r.lastError, err.Error())
		return false
	}
	if !gr.Success {
		r.lastError = append(r.lastError, gr.ErrorCodes...)
	}
	return gr.Success
}

// LastError returns errors occurred in last re-captcha validation attempt
func (r R) LastError() []string {
	return r.lastError
}
<|endoftext|>


<commit_before>package emulator

import (
	"fmt"
	"k8s.io/kubernetes/pkg/api"
	"k8s.io/kubernetes/pkg/api/meta"
	"k8s.io/kubernetes/pkg/client/cache"
	"k8s.io/kubernetes/pkg/fields"
	"k8s.io/kubernetes/pkg/watch"
)

// TODO(jchaloup,hodovska): currently, only scheduler caches are considered.
// Later, include cache for each object.
type caches struct {
	// Pod cache modifed by emulation strategy
	PodCache cache.Store

	// Node cache modifed by emulation strategy
	NodeCache cache.Store

	// PVC cache modifed by emulation strategy
	PVCCache cache.Store

	// PV cache modifed by emulation strategy
	PVCache cache.Store

	// Service cache modifed by emulation strategy
	ServiceCache cache.Store

	// RC cache modifed by emulation strategy
	ReplicationControllerCache cache.Store

	// RS cache modifed by emulation strategy
	ReplicaSetCache cache.Store
}

type Strategy interface {
	// Add new objects
	Add(objs []interface{}) error

	// Update objects
	Update(objs []interface{}) error

	// Delete objects
	Delete(objs []interface{}) error
}

type predictiveStrategy struct {
	caches *caches
}

func NewPredictiveStrategy(c *caches) Strategy {
	return &predictiveStrategy{
		caches: c,
	}
}

func NewClientEmulator() ClientEmulator {
	caches := &caches{
		PodCache:                   cache.NewStore(cache.MetaNamespaceKeyFunc),
		NodeCache:                  cache.NewStore(cache.MetaNamespaceKeyFunc),
		PVCache:                    cache.NewStore(cache.MetaNamespaceKeyFunc),
		PVCCache:                   cache.NewStore(cache.MetaNamespaceKeyFunc),
		ServiceCache:               cache.NewStore(cache.MetaNamespaceKeyFunc),
		ReplicaSetCache:            cache.NewStore(cache.MetaNamespaceKeyFunc),
		ReplicationControllerCache: cache.NewStore(cache.MetaNamespaceKeyFunc),
	}

	resourceToCache := map[string]cache.Store{
		"pods":                   caches.PodCache,
		//"node":                   caches.NodeCache,
		//"persistentVolumes":      caches.PVCache,
		//"persistentVolumeClaims": caches.PVCCache,
		"services":               caches.ServiceCache,
		//"replicasets":            caches.ServiceCache,
		"replicationControllers": caches.ReplicationControllerCache,
	}
	return ClientEmulator{
		caches:          caches,
		resourceToCache: resourceToCache,
	}
}
func (*predictiveStrategy) Add(obj []interface{}) error {
	return fmt.Errorf("Not implemented yet")
}

func (*predictiveStrategy) Update(objs []interface{}) error {
	return fmt.Errorf("Not implemented yet")
}

func (*predictiveStrategy) Delete(objs []interface{}) error {
	return fmt.Errorf("Not implemented yet")
}

type ClientEmulator struct {
	// caches modified by emulation strategy
	*caches

	// watch events emulator
	watcher *watch.FakeWatcher

	// emulation strategy
	strategy *Strategy

	resourceToCache map[string]cache.Store
}

func storeItems(lw *cache.ListWatch, store cache.Store) error {
	options := api.ListOptions{ResourceVersion: "0"}
	list, err := lw.List(options)
	if err != nil {
		return fmt.Errorf("Failed to list objects: %v", err)
	}

	listMetaInterface, err := meta.ListAccessor(list)
	if err != nil {
		return fmt.Errorf("Unable to understand list result %#v: %v", list, err)
	}
	resourceVersion := listMetaInterface.GetResourceVersion()

	items, err := meta.ExtractList(list)
	if err != nil {
		return fmt.Errorf("Unable to understand list result %#v (%v)", list, err)
	}
	found := make([]interface{}, 0, len(items))
	for _, item := range items {
		found = append(found, item)
	}
	err = store.Replace(found, resourceVersion)
	if err != nil {
		return fmt.Errorf("Unable to store list result: %v", err)
	}
	return nil
}

func (c *ClientEmulator) sync(client cache.Getter) error {
	for resource, objectCache := range c.resourceToCache {
		listWatcher := cache.NewListWatchFromClient(client, resource, api.NamespaceAll, fields.ParseSelectorOrDie(""))
		if err := storeItems(listWatcher, objectCache); err != nil {
			return fmt.Errorf("Unable to sync %s: %v", resource, err)
		}
	}
	return nil
}
<commit_msg>provide some List() methods for resource ListerWatchers<commit_after>package emulator

import (
	"fmt"
	"k8s.io/kubernetes/pkg/api"
	"k8s.io/kubernetes/pkg/api/meta"
	"k8s.io/kubernetes/pkg/client/cache"
	"k8s.io/kubernetes/pkg/fields"
	"k8s.io/kubernetes/pkg/watch"
	"k8s.io/kubernetes/pkg/api/testapi"
	"k8s.io/kubernetes/pkg/api/unversioned"
)

// TODO(jchaloup,hodovska): currently, only scheduler caches are considered.
// Later, include cache for each object.
type caches struct {
	// Pod cache modifed by emulation strategy
	PodCache cache.Store

	// Node cache modifed by emulation strategy
	NodeCache cache.Store

	// PVC cache modifed by emulation strategy
	PVCCache cache.Store

	// PV cache modifed by emulation strategy
	PVCache cache.Store

	// Service cache modifed by emulation strategy
	ServiceCache cache.Store

	// RC cache modifed by emulation strategy
	ReplicationControllerCache cache.Store

	// RS cache modifed by emulation strategy
	ReplicaSetCache cache.Store
}

type Strategy interface {
	// Add new objects
	Add(objs []interface{}) error

	// Update objects
	Update(objs []interface{}) error

	// Delete objects
	Delete(objs []interface{}) error
}

type predictiveStrategy struct {
	caches *caches
}

func NewPredictiveStrategy(c *caches) Strategy {
	return &predictiveStrategy{
		caches: c,
	}
}

func NewClientEmulator() ClientEmulator {
	caches := &caches{
		PodCache:                   cache.NewStore(cache.MetaNamespaceKeyFunc),
		NodeCache:                  cache.NewStore(cache.MetaNamespaceKeyFunc),
		PVCache:                    cache.NewStore(cache.MetaNamespaceKeyFunc),
		PVCCache:                   cache.NewStore(cache.MetaNamespaceKeyFunc),
		ServiceCache:               cache.NewStore(cache.MetaNamespaceKeyFunc),
		ReplicaSetCache:            cache.NewStore(cache.MetaNamespaceKeyFunc),
		ReplicationControllerCache: cache.NewStore(cache.MetaNamespaceKeyFunc),
	}

	resourceToCache := map[string]cache.Store{
		"pods":                   caches.PodCache,
		//"node":                   caches.NodeCache,
		//"persistentVolumes":      caches.PVCache,
		//"persistentVolumeClaims": caches.PVCCache,
		"services":               caches.ServiceCache,
		//"replicasets":            caches.ServiceCache,
		"replicationControllers": caches.ReplicationControllerCache,
	}
	return ClientEmulator{
		caches:          caches,
		resourceToCache: resourceToCache,
	}
}
func (*predictiveStrategy) Add(obj []interface{}) error {
	return fmt.Errorf("Not implemented yet")
}

func (*predictiveStrategy) Update(objs []interface{}) error {
	return fmt.Errorf("Not implemented yet")
}

func (*predictiveStrategy) Delete(objs []interface{}) error {
	return fmt.Errorf("Not implemented yet")
}

type ClientEmulator struct {
	// caches modified by emulation strategy
	*caches

	// watch events emulator
	watcher *watch.FakeWatcher

	// emulation strategy
	strategy *Strategy

	resourceToCache map[string]cache.Store
}

func storeItems(lw *cache.ListWatch, store cache.Store) error {
	options := api.ListOptions{ResourceVersion: "0"}
	list, err := lw.List(options)
	if err != nil {
		return fmt.Errorf("Failed to list objects: %v", err)
	}

	listMetaInterface, err := meta.ListAccessor(list)
	if err != nil {
		return fmt.Errorf("Unable to understand list result %#v: %v", list, err)
	}
	resourceVersion := listMetaInterface.GetResourceVersion()

	items, err := meta.ExtractList(list)
	if err != nil {
		return fmt.Errorf("Unable to understand list result %#v (%v)", list, err)
	}
	found := make([]interface{}, 0, len(items))
	for _, item := range items {
		found = append(found, item)
	}
	err = store.Replace(found, resourceVersion)
	if err != nil {
		return fmt.Errorf("Unable to store list result: %v", err)
	}
	return nil
}

func (c *ClientEmulator) sync(client cache.Getter) error {
	for resource, objectCache := range c.resourceToCache {
		listWatcher := cache.NewListWatchFromClient(client, resource, api.NamespaceAll, fields.ParseSelectorOrDie(""))
		if err := storeItems(listWatcher, objectCache); err != nil {
			return fmt.Errorf("Unable to sync %s: %v", resource, err)
		}
	}
	return nil
}

func (c *ClientEmulator) RESTClient() *RESTClient {
	return &RESTClient{
		NegotiatedSerializer: testapi.Default.NegotiatedSerializer(),
		// define methods for ListerWatcher's List()
		podsDataSource: func() *api.PodList {
			items := c.caches.PodCache.List()
			podItems := make([]api.Pod, 0, len(items))
			for _, item := range items {
				podItems = append(podItems, item.(api.Pod))
			}

			return &api.PodList{
				ListMeta: unversioned.ListMeta{
					// choose arbitrary value as the cache does not store the ResourceVersion
					ResourceVersion: "0",
				},
				Items: podItems,
			}
		},
		servicesDataSource: func() *api.ServiceList {
			items := c.caches.ServiceCache.List()
			serviceItems := make([]api.Service, 0, len(items))
			for _, item := range items {
				serviceItems = append(serviceItems, item.(api.Service))
			}

			return &api.ServiceList{
				ListMeta: unversioned.ListMeta{
					// choose arbitrary value as the cache does not store the ResourceVersion
					ResourceVersion: "0",
				},
				Items: serviceItems,
			}

		},
		replicationControllersDataSource: func() *api.ReplicationControllerList {
			items := c.caches.ReplicationControllerCache.List()
			rcItems := make([]api.ReplicationController, 0, len(items))
			for _, item := range items {
				rcItems = append(rcItems, item.(api.ReplicationController))
			}

			return &api.ReplicationControllerList{
				ListMeta: unversioned.ListMeta{
					// choose arbitrary value as the cache does not store the ResourceVersion
					ResourceVersion: "0",
				},
				Items: rcItems,
			}
		},
		// TODO(jchaloup): add methods for missing resources
	}
}
<|endoftext|>


<commit_before>package redisence

import (
	"fmt"
	"strconv"
	"sync"
	"time"

	gredis "github.com/garyburd/redigo/redis"
	"github.com/koding/redis"
)

type Status int

const (
	Offline Status = iota
	Online
	Closed
)

// Event is the data type for
// occuring events in the system
type Event struct {
	// Id is the given key by the application
	Id string

	// Status holds the changing type of event
	Status Status
}

// Prefix for redisence package
const RedisencePrefix = "redisence"

// Session holds the required connection data for redis
type Session struct {
	// main redis connection
	redis *redis.RedisSession

	// inactiveDuration specifies no-probe allowance time
	inactiveDuration time.Duration

	// receiving offline events pattern
	becameOfflinePattern string

	// receiving online events pattern
	becameOnlinePattern string
}

func New(server string, db int, inactiveDuration time.Duration) (*Session, error) {
	redis, err := redis.NewRedisSession(&redis.RedisConf{Server: server, DB: db})
	if err != nil {
		return nil, err
	}
	redis.SetPrefix(RedisencePrefix)

	return &Session{
		redis:                redis,
		becameOfflinePattern: fmt.Sprintf("__keyevent@%d__:expired", db),
		becameOnlinePattern:  fmt.Sprintf("__keyevent@%d__:set", db),
		inactiveDuration:     inactiveDuration,
	}, nil
}

// Ping resets the expiration time for any given key
// if key doesnt exists, it means user is now online
// Whenever application gets any prob from a client
// should call this function
func (s *Session) Ping(ids ...string) error {
	if len(ids) == 1 {
		// if member exits increase ttl
		if s.redis.Expire(ids[0], s.inactiveDuration) == nil {
			return nil
		}

		// if member doesnt exist set it
		return s.redis.Setex(ids[0], s.inactiveDuration, ids[0])
	}

	existance, err := s.sendMultiExpire(ids)
	if err != nil {
		return err
	}

	return s.sendMultiSetIfRequired(ids, existance)
}

func (s *Session) sendMultiSetIfRequired(ids []string, existance []int) error {
	if len(ids) != len(existance) {
		return fmt.Errorf("Length is not same Ids: %d Existance: %d", len(ids), len(existance))
	}

	// cache inactive duration as string
	seconds := strconv.Itoa(int(s.inactiveDuration.Seconds()))

	// get one connection from pool
	c := s.redis.Pool().Get()

	// item count for non-existent members
	notExistsCount := 0

	for i, exists := range existance {
		// `0` means, member doesnt exists in presence system
		if exists != 0 {
			continue
		}

		// init multi command lazily
		if notExistsCount == 0 {
			c.Send("MULTI")
		}

		notExistsCount++
		c.Send("SETEX", s.redis.AddPrefix(ids[i]), seconds, ids[i])
	}

	// execute multi command if only we flushed some to connection
	if notExistsCount != 0 {
		// ignore values
		if _, err := c.Do("EXEC"); err != nil {
			return err
		}
	}

	// do not forget to close the connection
	if err := c.Close(); err != nil {
		return err
	}

	return nil
}

func (s *Session) sendMultiExpire(ids []string) ([]int, error) {
	// cache inactive duration as string
	seconds := strconv.Itoa(int(s.inactiveDuration.Seconds()))

	// get one connection from pool
	c := s.redis.Pool().Get()

	// init multi command
	c.Send("MULTI")

	// send expire command for all members
	for _, id := range ids {
		c.Send("EXPIRE", s.redis.AddPrefix(id), seconds)
	}

	// execute command
	r, err := c.Do("EXEC")
	if err != nil {
		return make([]int, 0), err
	}

	// close connection
	if err := c.Close(); err != nil {
		return make([]int, 0), err
	}

	values, err := s.redis.Values(r)
	if err != nil {
		return make([]int, 0), err
	}

	res := make([]int, len(values))
	for i, value := range values {
		res[i], err = s.redis.Int(value)
		if err != nil {
			return make([]int, 0), err
		}

	}

	return res, nil
}

// Status returns the current status a key from system
func (s *Session) Status(ids ...string) ([]Event, error) {
	// get one connection from pool
	c := s.redis.Pool().Get()

	// init multi command
	c.Send("MULTI")

	// send expire command for all members
	for _, id := range ids {
		c.Send("EXISTS", s.redis.AddPrefix(id))
	}

	// execute command
	r, err := c.Do("EXEC")
	if err != nil {
		return make([]Event, 0), err
	}

	// close connection
	if err := c.Close(); err != nil {
		return make([]Event, 0), err
	}

	values, err := s.redis.Values(r)
	if err != nil {
		return make([]Event, 0), err
	}

	res := make([]Event, len(values))
	for i, value := range values {
		status, err := s.redis.Int(value)
		if err != nil {
			return make([]Event, 0), err
		}

		res[i] = Event{
			Id:     ids[i],
			Status: Status(status),
		}
	}

	return res, nil
}

// createEvent Creates the event with the required properties
func (s *Session) createEvent(n gredis.PMessage) Event {
	e := Event{}

	switch n.Pattern {
	case s.becameOfflinePattern:
		e.Id = string(n.Data[len(RedisencePrefix)+1:])
		e.Status = Offline
	case s.becameOnlinePattern:
		e.Id = string(n.Data[len(RedisencePrefix)+1:])
		e.Status = Online
	default:
		//ignore other events
	}

	return e
}

// ListenStatusChanges pubscribes to the redis and
// gets online and offline status changes from it
func (s *Session) ListenStatusChanges(events chan Event) {
	psc := s.redis.CreatePubSubConn()

	psc.PSubscribe(s.becameOnlinePattern, s.becameOfflinePattern)

	var wg sync.WaitGroup
	wg.Add(1)

	go func() {
		defer wg.Done()
		for {
			switch n := psc.Receive().(type) {
			case gredis.PMessage:
				events <- s.createEvent(n)
			case error:
				fmt.Printf("error: %v\n", n)
				return
			}
		}
	}()

	// avoid lock
	go func() {
		wg.Wait()
		psc.PUnsubscribe(s.becameOfflinePattern, s.becameOnlinePattern)
		psc.Close()
		events <- Event{Status: Closed}
	}()
}
<commit_msg>Redisence: add documentation onto functions<commit_after>// Package redisence provides simple user presence system
package redisence

import (
	"fmt"
	"strconv"
	"time"

	gredis "github.com/garyburd/redigo/redis"
	"github.com/koding/redis"
)

// Status defines what is the current status of a user
// in presence system
type Status int

const (
	Offline Status = iota
	Online
	Closed
)

// Event is the data type for
// occuring events in the system
type Event struct {
	// Id is the given key by the application
	Id string

	// Status holds the changing type of event
	Status Status
}

// Prefix for redisence package
const RedisencePrefix = "redisence"

// Session holds the required connection data for redis
type Session struct {
	// main redis connection
	redis *redis.RedisSession

	// inactiveDuration specifies no-probe allowance time
	inactiveDuration time.Duration

	// receiving offline events pattern
	becameOfflinePattern string

	// receiving online events pattern
	becameOnlinePattern string
}

// New creates a session for any broker system that is architected to use,
// communicate, forward events to the presence system
func New(server string, db int, inactiveDuration time.Duration) (*Session, error) {
	redis, err := redis.NewRedisSession(&redis.RedisConf{Server: server, DB: db})
	if err != nil {
		return nil, err
	}
	redis.SetPrefix(RedisencePrefix)

	return &Session{
		redis:                redis,
		becameOfflinePattern: fmt.Sprintf("__keyevent@%d__:expired", db),
		becameOnlinePattern:  fmt.Sprintf("__keyevent@%d__:set", db),
		inactiveDuration:     inactiveDuration,
	}, nil
}

// Ping resets the expiration time for any given key
// if key doesnt exists, it means user is now online and should be set as online
// Whenever application gets any prob from a client
// should call this function
func (s *Session) Online(ids ...string) error {
	if len(ids) == 1 {
		// if member exits increase ttl
		if s.redis.Expire(ids[0], s.inactiveDuration) == nil {
			return nil
		}

		// if member doesnt exist set it
		return s.redis.Setex(ids[0], s.inactiveDuration, ids[0])
	}

	existance, err := s.sendMultiExpire(ids)
	if err != nil {
		return err
	}

	return s.sendMultiSetIfRequired(ids, existance)
}

// sendMultiSetIfRequired accepts set of ids and their existtance status
// traverse over them and any key is not exists in db, set them in a multi/exec
// request
func (s *Session) sendMultiSetIfRequired(ids []string, existance []int) error {
	if len(ids) != len(existance) {
		return fmt.Errorf("Length is not same Ids: %d Existance: %d", len(ids), len(existance))
	}

	// cache inactive duration as string
	seconds := strconv.Itoa(int(s.inactiveDuration.Seconds()))

	// get one connection from pool
	c := s.redis.Pool().Get()

	// item count for non-existent members
	notExistsCount := 0

	for i, exists := range existance {
		// `0` means, member doesnt exists in presence system
		if exists != 0 {
			continue
		}

		// init multi command lazily
		if notExistsCount == 0 {
			c.Send("MULTI")
		}

		notExistsCount++
		c.Send("SETEX", s.redis.AddPrefix(ids[i]), seconds, ids[i])
	}

	// execute multi command if only we flushed some to connection
	if notExistsCount != 0 {
		// ignore values
		if _, err := c.Do("EXEC"); err != nil {
			return err
		}
	}

	// do not forget to close the connection
	if err := c.Close(); err != nil {
		return err
	}

	return nil
}

// sendMultiExpire if the system tries to update more than one key at a time
// inorder to leverage rtt, send multi expire
func (s *Session) sendMultiExpire(ids []string, duration string) ([]int, error) {
	// get one connection from pool
	c := s.redis.Pool().Get()

	// init multi command
	c.Send("MULTI")

	// send expire command for all members
	for _, id := range ids {
		c.Send("EXPIRE", s.redis.AddPrefix(id), seconds)
	}

	// execute command
	r, err := c.Do("EXEC")
	if err != nil {
		return make([]int, 0), err
	}

	// close connection
	if err := c.Close(); err != nil {
		return make([]int, 0), err
	}

	values, err := s.redis.Values(r)
	if err != nil {
		return make([]int, 0), err
	}

	res := make([]int, len(values))
	for i, value := range values {
		res[i], err = s.redis.Int(value)
		if err != nil {
			// what about returning half-generated slice?
			// instead of an empty one
			return make([]int, 0), err
		}

	}

	return res, nil
}

// MultipleStatus returns the current status multiple keys from system
func (s *Session) MultipleStatus(ids []string) ([]Event, error) {
	// get one connection from pool
	c := s.redis.Pool().Get()

	// init multi command
	c.Send("MULTI")

	// send expire command for all members
	for _, id := range ids {
		c.Send("EXISTS", s.redis.AddPrefix(id))
	}

	// execute command
	r, err := c.Do("EXEC")
	if err != nil {
		return make([]Event, 0), err
	}

	// close connection
	if err := c.Close(); err != nil {
		return make([]Event, 0), err
	}

	values, err := s.redis.Values(r)
	if err != nil {
		return make([]Event, 0), err
	}

	res := make([]Event, len(values))
	for i, value := range values {
		status, err := s.redis.Int(value)
		if err != nil {
			return make([]Event, 0), err
		}

		res[i] = Event{
			Id: ids[i],
			// cast redis response to Status
			Status: Status(status),
		}
	}

	return res, nil
}

// createEvent Creates the event with the required properties
func (s *Session) createEvent(n gredis.PMessage) Event {
	e := Event{}

	switch n.Pattern {
	case s.becameOfflinePattern:
		e.Id = string(n.Data[len(RedisencePrefix)+1:])
		e.Status = Offline
	case s.becameOnlinePattern:
		e.Id = string(n.Data[len(RedisencePrefix)+1:])
		e.Status = Online
	default:
		//ignore other events
	}

	return e
}

// ListenStatusChanges pubscribes to the redis and
// gets online and offline status changes from it
func (s *Session) ListenStatusChanges(events chan Event) {
	psc := s.redis.CreatePubSubConn()

	psc.PSubscribe(s.becameOnlinePattern, s.becameOfflinePattern)

	var wg sync.WaitGroup
	wg.Add(1)

	go func() {
		defer wg.Done()
		for {
			switch n := psc.Receive().(type) {
			case gredis.PMessage:
				events <- s.createEvent(n)
			case error:
				fmt.Printf("error: %v\n", n)
				return
			}
		}
	}()

	// avoid lock
	go func() {
		wg.Wait()
		psc.PUnsubscribe(s.becameOfflinePattern, s.becameOnlinePattern)
		psc.Close()
		events <- Event{Status: Closed}
	}()
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"fmt"
	"strings"
	"sync"

	"a4.io/blobstash/pkg/backend/s3/s3util"
	"a4.io/blobstash/pkg/blob"
	bcache "a4.io/blobstash/pkg/cache"
	"a4.io/blobstash/pkg/client/blobstore"
	"a4.io/blobstash/pkg/hashutil"
	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/service/s3"
	"golang.org/x/net/context"
)

// cache implements the blobStore interface with a local disk-backed LRU cache
type cache struct {
	fs         *FS
	bs         *blobstore.BlobStore
	mu         sync.Mutex
	remoteRefs map[string]string

	blobsCache *bcache.Cache
}

// newCache initializes a new cache instance
func newCache(fs *FS, bs *blobstore.BlobStore, path string) (*cache, error) {
	blobsCache, err := bcache.New(path, "blobs.cache", (5*1024)<<20) // 5GB on-disk LRU cache TODO(tsileo): make it configurable
	if err != nil {
		return nil, err
	}

	return &cache{
		fs:         fs,
		bs:         bs,
		blobsCache: blobsCache,
		remoteRefs: map[string]string{},
	}, nil
}

func (c *cache) RemoteRefs() map[string]string {
	c.mu.Lock()
	defer c.mu.Unlock()
	return c.remoteRefs
}

// Close implements the io.Closer interface
func (c *cache) Close() error {
	return c.blobsCache.Close()
}

// Stat implements the blobStore interface
func (c *cache) Stat(ctx context.Context, hash string) (bool, error) {
	c.mu.Lock()
	defer c.mu.Unlock()

	// Check if the blob has already been uploaded to the remote storage
	if _, ok := c.remoteRefs[hash]; ok {
		return true, nil
	}

	stat, err := c.bs.Stat(context.TODO(), hash)
	if err != nil {
		return false, err
	}

	return stat, nil
}

// Put implements the blobStore interface for filereader.File
func (c *cache) Put(ctx context.Context, hash string, data []byte) error {
	c.mu.Lock()
	defer c.mu.Unlock()

	if err := c.bs.Put(ctx, hash, data); err != nil {
		return err
	}

	if err := c.blobsCache.Add(hash, data); err != nil {
		return err
	}
	return nil
}

// Get implements the BlobStore interface for filereader.File
func (c *cache) PutRemote(ctx context.Context, hash string, data []byte) error {
	if !c.fs.useRemote {
		return c.Put(ctx, hash, data)
	}
	logger.Printf("[remote] uploading %s (%d bytes) to remote", hash, len(data))

	c.mu.Lock()
	defer c.mu.Unlock()
	var err error

	if _, ok := c.remoteRefs[hash]; ok {
		return nil
	}

	if err := c.blobsCache.Add(hash, data); err != nil {
		return err
	}

	// Encrypt
	data, err = s3util.Seal(c.fs.key, &blob.Blob{Hash: hash, Data: data})
	if err != nil {
		return err
	}
	// Re-compute the hash
	ehash := hashutil.Compute(data)

	// Prepare the upload request
	params := &s3.PutObjectInput{
		Bucket:   aws.String(c.fs.profile.RemoteConfig.Bucket),
		Key:      aws.String("tmp/" + ehash),
		Body:     bytes.NewReader(data),
		Metadata: map[string]*string{},
	}

	// Actually upload the blob
	if _, err := c.fs.s3.PutObject(params); err != nil {
		return err
	}

	c.remoteRefs[hash] = "tmp/" + ehash

	return nil
}

// Get implements the BlobStore interface for filereader.File
func (c *cache) GetRemote(ctx context.Context, hash string) ([]byte, error) {
	if !c.fs.useRemote {
		return c.Get(ctx, hash)
	}

	c.mu.Lock()
	defer c.mu.Unlock()
	var err error
	cachedBlob, ok, err := c.blobsCache.Get(hash)
	if err != nil {
		return nil, fmt.Errorf("cache failed: %v", err)
	}
	logger.Printf("[remote] downloading %s from remote", hash)
	var data []byte
	if ok {
		data = cachedBlob
	} else {
		obj, err := s3util.NewBucket(c.fs.s3, c.fs.profile.RemoteConfig.Bucket).GetObject(hash)
		if err != nil {
			return nil, err
		}
		eblob := s3util.NewEncryptedBlob(obj, c.fs.key)
		data, err = eblob.PlainText()
		if err != nil {
			return nil, err
		}
		if err := c.blobsCache.Add(hash, data); err != nil {
			return nil, fmt.Errorf("failed to add to cache: %v", err)
		}
	}

	return data, nil
}

// Get implements the blobStore interface for filereader.File
func (c *cache) Get(ctx context.Context, hash string) ([]byte, error) {
	logger.Printf("Cache.Get(%q)\n", hash)
	if strings.HasPrefix(hash, "remote://") {
		return c.GetRemote(ctx, hash[9:])
	}

	var err error
	cachedBlob, ok, err := c.blobsCache.Get(hash)
	if err != nil {
		return nil, fmt.Errorf("cache failed: %v", err)
	}
	var data []byte
	if ok {
		data = cachedBlob
	} else {
		c.mu.Lock()
		defer c.mu.Unlock()

		data, err = c.bs.Get(ctx, hash)
		if err != nil {
			return nil, fmt.Errorf("failed to call blobstore: %v", err)
		}
		if err := c.blobsCache.Add(hash, data); err != nil {
			return nil, fmt.Errorf("failed to add to cache: %v", err)
		}
	}
	return data, nil
}
<commit_msg>filetree: remove dead code<commit_after>package main

import (
	"bytes"
	"fmt"
	"strings"
	"sync"

	"a4.io/blobstash/pkg/backend/s3/s3util"
	"a4.io/blobstash/pkg/blob"
	bcache "a4.io/blobstash/pkg/cache"
	"a4.io/blobstash/pkg/client/blobstore"
	"a4.io/blobstash/pkg/hashutil"
	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/service/s3"
	"golang.org/x/net/context"
)

// cache implements the blobStore interface with a local disk-backed LRU cache
type cache struct {
	fs         *FS
	bs         *blobstore.BlobStore
	mu         sync.Mutex
	remoteRefs map[string]string

	blobsCache *bcache.Cache
}

// newCache initializes a new cache instance
func newCache(fs *FS, bs *blobstore.BlobStore, path string) (*cache, error) {
	blobsCache, err := bcache.New(path, "blobs.cache", (5*1024)<<20) // 5GB on-disk LRU cache TODO(tsileo): make it configurable
	if err != nil {
		return nil, err
	}

	return &cache{
		fs:         fs,
		bs:         bs,
		blobsCache: blobsCache,
		remoteRefs: map[string]string{},
	}, nil
}

func (c *cache) RemoteRefs() map[string]string {
	c.mu.Lock()
	defer c.mu.Unlock()
	return c.remoteRefs
}

// Close implements the io.Closer interface
func (c *cache) Close() error {
	return c.blobsCache.Close()
}

// Stat implements the blobStore interface
func (c *cache) Stat(ctx context.Context, hash string) (bool, error) {
	c.mu.Lock()
	defer c.mu.Unlock()

	// Check if the blob has already been uploaded to the remote storage
	if _, ok := c.remoteRefs[hash]; ok {
		return true, nil
	}

	stat, err := c.bs.Stat(context.TODO(), hash)
	if err != nil {
		return false, err
	}

	return stat, nil
}

// Put implements the blobStore interface for filereader.File
func (c *cache) Put(ctx context.Context, hash string, data []byte) error {
	c.mu.Lock()
	defer c.mu.Unlock()

	if err := c.bs.Put(ctx, hash, data); err != nil {
		return err
	}

	if err := c.blobsCache.Add(hash, data); err != nil {
		return err
	}
	return nil
}

// Get implements the BlobStore interface for filereader.File
func (c *cache) PutRemote(ctx context.Context, hash string, data []byte) error {
	if !c.fs.useRemote {
		return c.Put(ctx, hash, data)
	}
	logger.Printf("[remote] uploading %s (%d bytes) to remote", hash, len(data))

	c.mu.Lock()
	defer c.mu.Unlock()
	var err error

	if _, ok := c.remoteRefs[hash]; ok {
		return nil
	}

	if err := c.blobsCache.Add(hash, data); err != nil {
		return err
	}

	// Encrypt
	data, err = s3util.Seal(c.fs.key, &blob.Blob{Hash: hash, Data: data})
	if err != nil {
		return err
	}
	// Re-compute the hash
	ehash := hashutil.Compute(data)

	// Prepare the upload request
	params := &s3.PutObjectInput{
		Bucket:   aws.String(c.fs.profile.RemoteConfig.Bucket),
		Key:      aws.String("tmp/" + ehash),
		Body:     bytes.NewReader(data),
		Metadata: map[string]*string{},
	}

	// Actually upload the blob
	if _, err := c.fs.s3.PutObject(params); err != nil {
		return err
	}

	c.remoteRefs[hash] = "tmp/" + ehash

	return nil
}

// Get implements the blobStore interface for filereader.File
func (c *cache) Get(ctx context.Context, hash string) ([]byte, error) {
	logger.Printf("Cache.Get(%q)\n", hash)
	if strings.HasPrefix(hash, "remote://") {
		return c.GetRemote(ctx, hash[9:])
	}

	var err error
	cachedBlob, ok, err := c.blobsCache.Get(hash)
	if err != nil {
		return nil, fmt.Errorf("cache failed: %v", err)
	}
	var data []byte
	if ok {
		data = cachedBlob
	} else {
		c.mu.Lock()
		defer c.mu.Unlock()

		data, err = c.bs.Get(ctx, hash)
		if err != nil {
			return nil, fmt.Errorf("failed to call blobstore: %v", err)
		}
		if err := c.blobsCache.Add(hash, data); err != nil {
			return nil, fmt.Errorf("failed to add to cache: %v", err)
		}
	}
	return data, nil
}
<|endoftext|>