<commit_before>package objects

import (
	"crypto/sha256"
	"fmt"
	"hash"
	"log"
	"sort"
	"strings"
)

type entry struct {
	Hash       HID
	TypeString string
}

type List map[string]entry

func (l List) Add(nameSegment string, hash HID, typeString string) List {
	l[nameSegment] = entry{hash, typeString}
	return l
}

func (l List) Remove(nameSegment string) List {
	delete(l, nameSegment)
	return l
}

func (l List) Hash_for_namesegment(namesegment string) (string, HID) {
	objectHash := l[namesegment].Hash
	typeString := l[namesegment].TypeString
	return typeString, objectHash
}

func (l List) String() string {
	var keys []string
	for key := range l {
		keys = append(keys, key)
	}
	sort.Strings(keys)
	s := ""
	for _, k := range keys {
		s = s + fmt.Sprintf("%s,%s,%s\n", l[k].Hash.Hex(), l[k].TypeString, k)
	}
	return s[:len(s)-1]
}

func (l List) Bytes() []byte {
	return []byte(l.String())
}

func (l List) Hash() HCID {
	var h hash.Hash = sha256.New()
	h.Write(l.Bytes())
	return h.Sum(nil)
}

func NewList(objectHash HID, typestring string, nameSegment string) List {
	l := make(List)
	l[nameSegment] = entry{objectHash, typestring}
	return l
}

func ListFromBytes(listbytes []byte) (newlist List, err error) {
	l := make(List)
	listEntries := strings.Split(string(listbytes), "\n")
	cols := []string{}
	for _, element := range listEntries {
		cols = strings.Split(element, ",")
		if len(cols) != 3 {
			return newlist, fmt.Errorf("Could not parse list")
		}
		entryTypeString := cols[1]
		var entryHID HID
		if entryTypeString == "blob" || entryTypeString == "list" {
			entryHID, err = HcidFromHex(cols[0])
			if err != nil {
				return nil, err
			}
		} else if entryTypeString == "commit" || entryTypeString == "tag" {
			entryHID, err = HkidFromHex(cols[0])
			if err != nil {
				return nil, err
			}
		} else {
			log.Fatalf("Unrecognised type: %s", entryTypeString)
		}
		entryNameSegment := cols[2]
		l[entryNameSegment] = entry{entryHID, entryTypeString}
	}
	return l, err
}
<commit_msg>replased a if else chain with a switch<commit_after>package objects

import (
	"crypto/sha256"
	"fmt"
	"hash"
	"log"
	"sort"
	"strings"
)

type entry struct {
	Hash       HID
	TypeString string
}

type List map[string]entry

func (l List) Add(nameSegment string, hash HID, typeString string) List {
	l[nameSegment] = entry{hash, typeString}
	return l
}

func (l List) Remove(nameSegment string) List {
	delete(l, nameSegment)
	return l
}

func (l List) Hash_for_namesegment(namesegment string) (string, HID) {
	objectHash := l[namesegment].Hash
	typeString := l[namesegment].TypeString
	return typeString, objectHash
}

func (l List) String() string {
	var keys []string
	for key := range l {
		keys = append(keys, key)
	}
	sort.Strings(keys)
	s := ""
	for _, k := range keys {
		s = s + fmt.Sprintf("%s,%s,%s\n", l[k].Hash.Hex(), l[k].TypeString, k)
	}
	return s[:len(s)-1]
}

func (l List) Bytes() []byte {
	return []byte(l.String())
}

func (l List) Hash() HCID {
	var h hash.Hash = sha256.New()
	h.Write(l.Bytes())
	return h.Sum(nil)
}

func NewList(objectHash HID, typestring string, nameSegment string) List {
	l := make(List)
	l[nameSegment] = entry{objectHash, typestring}
	return l
}

func ListFromBytes(listbytes []byte) (newlist List, err error) {
	l := make(List)
	listEntries := strings.Split(string(listbytes), "\n")
	cols := []string{}
	for _, element := range listEntries {
		cols = strings.Split(element, ",")
		if len(cols) != 3 {
			return newlist, fmt.Errorf("Could not parse list")
		}
		entryTypeString := cols[1]
		var entryHID HID
		switch entryTypeString {
		case "blob", "list":
			entryHID, err = HcidFromHex(cols[0])
			if err != nil {
				return nil, err
			}
		case "commit", "tag":
			entryHID, err = HkidFromHex(cols[0])
			if err != nil {
				return nil, err
			}
		default:
			log.Fatalf("Unrecognised type: %s", entryTypeString)
		}

		entryNameSegment := cols[2]
		l[entryNameSegment] = entry{entryHID, entryTypeString}
	}
	return l, err
}
<|endoftext|>


<commit_before>package networkdb

import (
	"fmt"
	"net"
	"time"

	"github.com/Sirupsen/logrus"
	"github.com/gogo/protobuf/proto"
)

type delegate struct {
	nDB *NetworkDB
}

func (d *delegate) NodeMeta(limit int) []byte {
	return []byte{}
}

func (nDB *NetworkDB) handleNetworkEvent(nEvent *NetworkEvent) bool {
	// Update our local clock if the received messages has newer
	// time.
	nDB.networkClock.Witness(nEvent.LTime)

	nDB.Lock()
	defer nDB.Unlock()

	nodeNetworks, ok := nDB.networks[nEvent.NodeName]
	if !ok {
		// We haven't heard about this node at all.  Ignore the leave
		if nEvent.Type == NetworkEventTypeLeave {
			return false
		}

		nodeNetworks = make(map[string]*network)
		nDB.networks[nEvent.NodeName] = nodeNetworks
	}

	if n, ok := nodeNetworks[nEvent.NetworkID]; ok {
		// We have the latest state. Ignore the event
		// since it is stale.
		if n.ltime >= nEvent.LTime {
			return false
		}

		n.ltime = nEvent.LTime
		n.leaving = nEvent.Type == NetworkEventTypeLeave
		if n.leaving {
			n.leaveTime = time.Now()
		}

		return true
	}

	if nEvent.Type == NetworkEventTypeLeave {
		return false
	}

	// This remote network join is being seen the first time.
	nodeNetworks[nEvent.NetworkID] = &network{
		id:    nEvent.NetworkID,
		ltime: nEvent.LTime,
	}

	nDB.networkNodes[nEvent.NetworkID] = append(nDB.networkNodes[nEvent.NetworkID], nEvent.NodeName)
	return true
}

func (nDB *NetworkDB) handleTableEvent(tEvent *TableEvent) bool {
	// Update our local clock if the received messages has newer
	// time.
	nDB.tableClock.Witness(tEvent.LTime)

	if entry, err := nDB.getEntry(tEvent.TableName, tEvent.NetworkID, tEvent.Key); err == nil {
		// We have the latest state. Ignore the event
		// since it is stale.
		if entry.ltime >= tEvent.LTime {
			return false
		}
	}

	entry := &entry{
		ltime:    tEvent.LTime,
		node:     tEvent.NodeName,
		value:    tEvent.Value,
		deleting: tEvent.Type == TableEventTypeDelete,
	}

	if entry.deleting {
		entry.deleteTime = time.Now()
	}

	nDB.Lock()
	nDB.indexes[byTable].Insert(fmt.Sprintf("/%s/%s/%s", tEvent.TableName, tEvent.NetworkID, tEvent.Key), entry)
	nDB.indexes[byNetwork].Insert(fmt.Sprintf("/%s/%s/%s", tEvent.NetworkID, tEvent.TableName, tEvent.Key), entry)
	nDB.Unlock()

	var op opType
	switch tEvent.Type {
	case TableEventTypeCreate:
		op = opCreate
	case TableEventTypeUpdate:
		op = opUpdate
	case TableEventTypeDelete:
		op = opDelete
	}

	nDB.broadcaster.Write(makeEvent(op, tEvent.TableName, tEvent.NetworkID, tEvent.Key, tEvent.Value))
	return true
}

func (nDB *NetworkDB) handleCompound(buf []byte, isBulkSync bool) {
	// Decode the parts
	parts, err := decodeCompoundMessage(buf)
	if err != nil {
		logrus.Errorf("Failed to decode compound request: %v", err)
		return
	}

	// Handle each message
	for _, part := range parts {
		nDB.handleMessage(part, isBulkSync)
	}
}

func (nDB *NetworkDB) handleTableMessage(buf []byte, isBulkSync bool) {
	var tEvent TableEvent
	if err := proto.Unmarshal(buf, &tEvent); err != nil {
		logrus.Errorf("Error decoding table event message: %v", err)
		return
	}

	// Ignore messages that this node generated.
	if tEvent.NodeName == nDB.config.NodeName {
		return
	}

	// Do not rebroadcast a bulk sync
	if rebroadcast := nDB.handleTableEvent(&tEvent); rebroadcast && !isBulkSync {
		var err error
		buf, err = encodeRawMessage(MessageTypeTableEvent, buf)
		if err != nil {
			logrus.Errorf("Error marshalling gossip message for network event rebroadcast: %v", err)
			return
		}

		nDB.RLock()
		n, ok := nDB.networks[nDB.config.NodeName][tEvent.NetworkID]
		nDB.RUnlock()

		if !ok {
			return
		}

		broadcastQ := n.tableBroadcasts

		if broadcastQ == nil {
			return
		}

		broadcastQ.QueueBroadcast(&tableEventMessage{
			msg:   buf,
			id:    tEvent.NetworkID,
			tname: tEvent.TableName,
			key:   tEvent.Key,
			node:  nDB.config.NodeName,
		})
	}
}

func (nDB *NetworkDB) handleNetworkMessage(buf []byte) {
	var nEvent NetworkEvent
	if err := proto.Unmarshal(buf, &nEvent); err != nil {
		logrus.Errorf("Error decoding network event message: %v", err)
		return
	}

	if rebroadcast := nDB.handleNetworkEvent(&nEvent); rebroadcast {
		var err error
		buf, err = encodeRawMessage(MessageTypeNetworkEvent, buf)
		if err != nil {
			logrus.Errorf("Error marshalling gossip message for network event rebroadcast: %v", err)
			return
		}

		nDB.networkBroadcasts.QueueBroadcast(&networkEventMessage{
			msg:  buf,
			id:   nEvent.NetworkID,
			node: nEvent.NodeName,
		})
	}
}

func (nDB *NetworkDB) handleBulkSync(buf []byte) {
	var bsm BulkSyncMessage
	if err := proto.Unmarshal(buf, &bsm); err != nil {
		logrus.Errorf("Error decoding bulk sync message: %v", err)
		return
	}

	if bsm.LTime > 0 {
		nDB.tableClock.Witness(bsm.LTime)
	}

	nDB.handleMessage(bsm.Payload, true)

	// Don't respond to a bulk sync which was not unsolicited
	if !bsm.Unsolicited {
		nDB.RLock()
		ch, ok := nDB.bulkSyncAckTbl[bsm.NodeName]
		nDB.RUnlock()
		if ok {
			close(ch)
		}

		return
	}

	var nodeAddr net.IP
	if node, ok := nDB.nodes[bsm.NodeName]; ok {
		nodeAddr = node.Addr
	}

	if err := nDB.bulkSyncNode(bsm.Networks, bsm.NodeName, false); err != nil {
		logrus.Errorf("Error in responding to bulk sync from node %s: %v", nodeAddr, err)
	}
}

func (nDB *NetworkDB) handleMessage(buf []byte, isBulkSync bool) {
	mType, data, err := decodeMessage(buf)
	if err != nil {
		logrus.Errorf("Error decoding gossip message to get message type: %v", err)
		return
	}

	switch mType {
	case MessageTypeNetworkEvent:
		nDB.handleNetworkMessage(data)
	case MessageTypeTableEvent:
		nDB.handleTableMessage(data, isBulkSync)
	case MessageTypeBulkSync:
		nDB.handleBulkSync(data)
	case MessageTypeCompound:
		nDB.handleCompound(data, isBulkSync)
	default:
		logrus.Errorf("%s: unknown message type %d", nDB.config.NodeName, mType)
	}
}

func (d *delegate) NotifyMsg(buf []byte) {
	if len(buf) == 0 {
		return
	}

	d.nDB.handleMessage(buf, false)
}

func (d *delegate) GetBroadcasts(overhead, limit int) [][]byte {
	return d.nDB.networkBroadcasts.GetBroadcasts(overhead, limit)
}

func (d *delegate) LocalState(join bool) []byte {
	d.nDB.RLock()
	defer d.nDB.RUnlock()

	pp := NetworkPushPull{
		LTime: d.nDB.networkClock.Time(),
	}

	for name, nn := range d.nDB.networks {
		for _, n := range nn {
			pp.Networks = append(pp.Networks, &NetworkEntry{
				LTime:     n.ltime,
				NetworkID: n.id,
				NodeName:  name,
				Leaving:   n.leaving,
			})
		}
	}

	buf, err := encodeMessage(MessageTypePushPull, &pp)
	if err != nil {
		logrus.Errorf("Failed to encode local network state: %v", err)
		return nil
	}

	return buf
}

func (d *delegate) MergeRemoteState(buf []byte, isJoin bool) {
	if len(buf) == 0 {
		logrus.Error("zero byte remote network state received")
		return
	}

	var gMsg GossipMessage
	err := proto.Unmarshal(buf, &gMsg)
	if err != nil {
		logrus.Errorf("Error unmarshalling push pull messsage: %v", err)
		return
	}

	if gMsg.Type != MessageTypePushPull {
		logrus.Errorf("Invalid message type %v received from remote", buf[0])
	}

	pp := NetworkPushPull{}
	if err := proto.Unmarshal(gMsg.Data, &pp); err != nil {
		logrus.Errorf("Failed to decode remote network state: %v", err)
		return
	}

	if pp.LTime > 0 {
		d.nDB.networkClock.Witness(pp.LTime)
	}

	for _, n := range pp.Networks {
		nEvent := &NetworkEvent{
			LTime:     n.LTime,
			NodeName:  n.NodeName,
			NetworkID: n.NetworkID,
			Type:      NetworkEventTypeJoin,
		}

		if n.Leaving {
			nEvent.Type = NetworkEventTypeLeave
		}

		d.nDB.handleNetworkEvent(nEvent)
	}

}
<commit_msg>Do not create network entry in networkdb for the local node based on table event from peer<commit_after>package networkdb

import (
	"fmt"
	"net"
	"time"

	"github.com/Sirupsen/logrus"
	"github.com/gogo/protobuf/proto"
)

type delegate struct {
	nDB *NetworkDB
}

func (d *delegate) NodeMeta(limit int) []byte {
	return []byte{}
}

func (nDB *NetworkDB) handleNetworkEvent(nEvent *NetworkEvent) bool {
	// Update our local clock if the received messages has newer
	// time.
	nDB.networkClock.Witness(nEvent.LTime)

	nDB.Lock()
	defer nDB.Unlock()

	if nEvent.NodeName == nDB.config.NodeName {
		return false
	}

	nodeNetworks, ok := nDB.networks[nEvent.NodeName]
	if !ok {
		// We haven't heard about this node at all.  Ignore the leave
		if nEvent.Type == NetworkEventTypeLeave {
			return false
		}

		nodeNetworks = make(map[string]*network)
		nDB.networks[nEvent.NodeName] = nodeNetworks
	}

	if n, ok := nodeNetworks[nEvent.NetworkID]; ok {
		// We have the latest state. Ignore the event
		// since it is stale.
		if n.ltime >= nEvent.LTime {
			return false
		}

		n.ltime = nEvent.LTime
		n.leaving = nEvent.Type == NetworkEventTypeLeave
		if n.leaving {
			n.leaveTime = time.Now()
		}

		return true
	}

	if nEvent.Type == NetworkEventTypeLeave {
		return false
	}

	// This remote network join is being seen the first time.
	nodeNetworks[nEvent.NetworkID] = &network{
		id:    nEvent.NetworkID,
		ltime: nEvent.LTime,
	}

	nDB.networkNodes[nEvent.NetworkID] = append(nDB.networkNodes[nEvent.NetworkID], nEvent.NodeName)
	return true
}

func (nDB *NetworkDB) handleTableEvent(tEvent *TableEvent) bool {
	// Update our local clock if the received messages has newer
	// time.
	nDB.tableClock.Witness(tEvent.LTime)

	if entry, err := nDB.getEntry(tEvent.TableName, tEvent.NetworkID, tEvent.Key); err == nil {
		// We have the latest state. Ignore the event
		// since it is stale.
		if entry.ltime >= tEvent.LTime {
			return false
		}
	}

	entry := &entry{
		ltime:    tEvent.LTime,
		node:     tEvent.NodeName,
		value:    tEvent.Value,
		deleting: tEvent.Type == TableEventTypeDelete,
	}

	if entry.deleting {
		entry.deleteTime = time.Now()
	}

	nDB.Lock()
	nDB.indexes[byTable].Insert(fmt.Sprintf("/%s/%s/%s", tEvent.TableName, tEvent.NetworkID, tEvent.Key), entry)
	nDB.indexes[byNetwork].Insert(fmt.Sprintf("/%s/%s/%s", tEvent.NetworkID, tEvent.TableName, tEvent.Key), entry)
	nDB.Unlock()

	var op opType
	switch tEvent.Type {
	case TableEventTypeCreate:
		op = opCreate
	case TableEventTypeUpdate:
		op = opUpdate
	case TableEventTypeDelete:
		op = opDelete
	}

	nDB.broadcaster.Write(makeEvent(op, tEvent.TableName, tEvent.NetworkID, tEvent.Key, tEvent.Value))
	return true
}

func (nDB *NetworkDB) handleCompound(buf []byte, isBulkSync bool) {
	// Decode the parts
	parts, err := decodeCompoundMessage(buf)
	if err != nil {
		logrus.Errorf("Failed to decode compound request: %v", err)
		return
	}

	// Handle each message
	for _, part := range parts {
		nDB.handleMessage(part, isBulkSync)
	}
}

func (nDB *NetworkDB) handleTableMessage(buf []byte, isBulkSync bool) {
	var tEvent TableEvent
	if err := proto.Unmarshal(buf, &tEvent); err != nil {
		logrus.Errorf("Error decoding table event message: %v", err)
		return
	}

	// Ignore messages that this node generated.
	if tEvent.NodeName == nDB.config.NodeName {
		return
	}

	// Do not rebroadcast a bulk sync
	if rebroadcast := nDB.handleTableEvent(&tEvent); rebroadcast && !isBulkSync {
		var err error
		buf, err = encodeRawMessage(MessageTypeTableEvent, buf)
		if err != nil {
			logrus.Errorf("Error marshalling gossip message for network event rebroadcast: %v", err)
			return
		}

		nDB.RLock()
		n, ok := nDB.networks[nDB.config.NodeName][tEvent.NetworkID]
		nDB.RUnlock()

		if !ok {
			return
		}

		broadcastQ := n.tableBroadcasts

		if broadcastQ == nil {
			return
		}

		broadcastQ.QueueBroadcast(&tableEventMessage{
			msg:   buf,
			id:    tEvent.NetworkID,
			tname: tEvent.TableName,
			key:   tEvent.Key,
			node:  nDB.config.NodeName,
		})
	}
}

func (nDB *NetworkDB) handleNetworkMessage(buf []byte) {
	var nEvent NetworkEvent
	if err := proto.Unmarshal(buf, &nEvent); err != nil {
		logrus.Errorf("Error decoding network event message: %v", err)
		return
	}

	if rebroadcast := nDB.handleNetworkEvent(&nEvent); rebroadcast {
		var err error
		buf, err = encodeRawMessage(MessageTypeNetworkEvent, buf)
		if err != nil {
			logrus.Errorf("Error marshalling gossip message for network event rebroadcast: %v", err)
			return
		}

		nDB.networkBroadcasts.QueueBroadcast(&networkEventMessage{
			msg:  buf,
			id:   nEvent.NetworkID,
			node: nEvent.NodeName,
		})
	}
}

func (nDB *NetworkDB) handleBulkSync(buf []byte) {
	var bsm BulkSyncMessage
	if err := proto.Unmarshal(buf, &bsm); err != nil {
		logrus.Errorf("Error decoding bulk sync message: %v", err)
		return
	}

	if bsm.LTime > 0 {
		nDB.tableClock.Witness(bsm.LTime)
	}

	nDB.handleMessage(bsm.Payload, true)

	// Don't respond to a bulk sync which was not unsolicited
	if !bsm.Unsolicited {
		nDB.RLock()
		ch, ok := nDB.bulkSyncAckTbl[bsm.NodeName]
		nDB.RUnlock()
		if ok {
			close(ch)
		}

		return
	}

	var nodeAddr net.IP
	if node, ok := nDB.nodes[bsm.NodeName]; ok {
		nodeAddr = node.Addr
	}

	if err := nDB.bulkSyncNode(bsm.Networks, bsm.NodeName, false); err != nil {
		logrus.Errorf("Error in responding to bulk sync from node %s: %v", nodeAddr, err)
	}
}

func (nDB *NetworkDB) handleMessage(buf []byte, isBulkSync bool) {
	mType, data, err := decodeMessage(buf)
	if err != nil {
		logrus.Errorf("Error decoding gossip message to get message type: %v", err)
		return
	}

	switch mType {
	case MessageTypeNetworkEvent:
		nDB.handleNetworkMessage(data)
	case MessageTypeTableEvent:
		nDB.handleTableMessage(data, isBulkSync)
	case MessageTypeBulkSync:
		nDB.handleBulkSync(data)
	case MessageTypeCompound:
		nDB.handleCompound(data, isBulkSync)
	default:
		logrus.Errorf("%s: unknown message type %d", nDB.config.NodeName, mType)
	}
}

func (d *delegate) NotifyMsg(buf []byte) {
	if len(buf) == 0 {
		return
	}

	d.nDB.handleMessage(buf, false)
}

func (d *delegate) GetBroadcasts(overhead, limit int) [][]byte {
	return d.nDB.networkBroadcasts.GetBroadcasts(overhead, limit)
}

func (d *delegate) LocalState(join bool) []byte {
	d.nDB.RLock()
	defer d.nDB.RUnlock()

	pp := NetworkPushPull{
		LTime: d.nDB.networkClock.Time(),
	}

	for name, nn := range d.nDB.networks {
		for _, n := range nn {
			pp.Networks = append(pp.Networks, &NetworkEntry{
				LTime:     n.ltime,
				NetworkID: n.id,
				NodeName:  name,
				Leaving:   n.leaving,
			})
		}
	}

	buf, err := encodeMessage(MessageTypePushPull, &pp)
	if err != nil {
		logrus.Errorf("Failed to encode local network state: %v", err)
		return nil
	}

	return buf
}

func (d *delegate) MergeRemoteState(buf []byte, isJoin bool) {
	if len(buf) == 0 {
		logrus.Error("zero byte remote network state received")
		return
	}

	var gMsg GossipMessage
	err := proto.Unmarshal(buf, &gMsg)
	if err != nil {
		logrus.Errorf("Error unmarshalling push pull messsage: %v", err)
		return
	}

	if gMsg.Type != MessageTypePushPull {
		logrus.Errorf("Invalid message type %v received from remote", buf[0])
	}

	pp := NetworkPushPull{}
	if err := proto.Unmarshal(gMsg.Data, &pp); err != nil {
		logrus.Errorf("Failed to decode remote network state: %v", err)
		return
	}

	if pp.LTime > 0 {
		d.nDB.networkClock.Witness(pp.LTime)
	}

	for _, n := range pp.Networks {
		nEvent := &NetworkEvent{
			LTime:     n.LTime,
			NodeName:  n.NodeName,
			NetworkID: n.NetworkID,
			Type:      NetworkEventTypeJoin,
		}

		if n.Leaving {
			nEvent.Type = NetworkEventTypeLeave
		}

		d.nDB.handleNetworkEvent(nEvent)
	}

}
<|endoftext|>


<commit_before>package octokit

import (
	"encoding/json"
	"net"

	"github.com/jingweno/go-sawyer/hypermedia"
)

// https://developer.github.com/v3/meta/
var (
	MetaURL = Hyperlink("/meta")
)

// Meta return an APIInfo with the current API meta information
func (c *Client) Meta(uri *Hyperlink) (info APIInfo, result *Result) {
	url, err := uri.Expand(nil)
	if err != nil {
		return info, &Result{Err: err}
	}
	var meta meta
	result = c.get(url, &meta)
	if !result.HasError() {
		info = meta.transform()
	}
	return
}

type ipNets []*net.IPNet

func (i *ipNets) UnmarshalJSON(raw []byte) error {
	*i = (*i)[:0]
	var ss []string
	if err := json.Unmarshal(raw, &ss); err != nil {
		return err
	}
	for _, s := range ss {
		_, ipNet, err := net.ParseCIDR(s)
		if err != nil {
			return err
		}
		*i = append(*i, ipNet)
	}
	return nil
}

type ips []net.IP

func (i *ips) UnmarshalJSON(raw []byte) error {
	*i = (*i)[:0]
	var ss []string
	if err := json.Unmarshal(raw, &ss); err != nil {
		return err
	}
	for _, s := range ss {
		*i = append(*i, net.ParseIP(s))
	}
	return nil
}

type meta struct {
	*hypermedia.HALResource

	VerifiablePasswordAuthentication bool   `json:"verifiable_password_authentication,omitempty"`
	GithubServicesSha                string `json:"github_services_sha,omitempty"`
	Hooks                            ipNets `json:"hooks,omitempty"`
	Git                              ipNets `json:"git,omitempty"`
	Pages                            ipNets `json:"pages,omitempty"`
	Importer                         ips    `json:"importer,omitempty"`
}

func (m meta) transform() (info APIInfo) {
	info.VerifiablePasswordAuthentication = m.VerifiablePasswordAuthentication
	info.GithubServicesSha = m.GithubServicesSha

	info.Hooks = ([]*net.IPNet)(m.Hooks)
	info.Git = ([]*net.IPNet)(m.Git)
	info.Pages = ([]*net.IPNet)(m.Pages)
	info.Importer = ([]net.IP)(m.Importer)

	return
}

// APIInfo contains the information described in https://developer.github.com/v3/meta/#body
type APIInfo struct {
	*hypermedia.HALResource

	VerifiablePasswordAuthentication bool         `json:"verifiable_password_authentication,omitempty"`
	GithubServicesSha                string       `json:"github_services_sha,omitempty"`
	Hooks                            []*net.IPNet `json:"hooks,omitempty"`
	Git                              []*net.IPNet `json:"git,omitempty"`
	Pages                            []*net.IPNet `json:"pages,omitempty"`
	Importer                         []net.IP     `json:"importer,omitempty"`
}
<commit_msg>Add Meta url<commit_after>package octokit

import (
	"encoding/json"
	"net"

	"github.com/jingweno/go-sawyer/hypermedia"
)

// https://developer.github.com/v3/meta/
var (
	MetaURL = Hyperlink("/meta")
)

// Meta return an APIInfo with the current API meta information
//
// https://developer.github.com/v3/meta/#meta
func (c *Client) Meta(uri *Hyperlink) (info APIInfo, result *Result) {
	url, err := uri.Expand(nil)
	if err != nil {
		return info, &Result{Err: err}
	}
	var meta meta
	result = c.get(url, &meta)
	if !result.HasError() {
		info = meta.transform()
	}
	return
}

type ipNets []*net.IPNet

func (i *ipNets) UnmarshalJSON(raw []byte) error {
	*i = (*i)[:0]
	var ss []string
	if err := json.Unmarshal(raw, &ss); err != nil {
		return err
	}
	for _, s := range ss {
		_, ipNet, err := net.ParseCIDR(s)
		if err != nil {
			return err
		}
		*i = append(*i, ipNet)
	}
	return nil
}

type ips []net.IP

func (i *ips) UnmarshalJSON(raw []byte) error {
	*i = (*i)[:0]
	var ss []string
	if err := json.Unmarshal(raw, &ss); err != nil {
		return err
	}
	for _, s := range ss {
		*i = append(*i, net.ParseIP(s))
	}
	return nil
}

type meta struct {
	*hypermedia.HALResource

	VerifiablePasswordAuthentication bool   `json:"verifiable_password_authentication,omitempty"`
	GithubServicesSha                string `json:"github_services_sha,omitempty"`
	Hooks                            ipNets `json:"hooks,omitempty"`
	Git                              ipNets `json:"git,omitempty"`
	Pages                            ipNets `json:"pages,omitempty"`
	Importer                         ips    `json:"importer,omitempty"`
}

func (m meta) transform() (info APIInfo) {
	info.VerifiablePasswordAuthentication = m.VerifiablePasswordAuthentication
	info.GithubServicesSha = m.GithubServicesSha

	info.Hooks = ([]*net.IPNet)(m.Hooks)
	info.Git = ([]*net.IPNet)(m.Git)
	info.Pages = ([]*net.IPNet)(m.Pages)
	info.Importer = ([]net.IP)(m.Importer)

	return
}

// APIInfo contains the information described in https://developer.github.com/v3/meta/#body
type APIInfo struct {
	*hypermedia.HALResource

	VerifiablePasswordAuthentication bool         `json:"verifiable_password_authentication,omitempty"`
	GithubServicesSha                string       `json:"github_services_sha,omitempty"`
	Hooks                            []*net.IPNet `json:"hooks,omitempty"`
	Git                              []*net.IPNet `json:"git,omitempty"`
	Pages                            []*net.IPNet `json:"pages,omitempty"`
	Importer                         []net.IP     `json:"importer,omitempty"`
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
	"strconv"
	"strings"
	"time"

	"github.com/pilu/traffic"
	"github.com/spf13/cast"
	"github.com/spf13/hugo/parser"
	"golang.org/x/net/context"
	"google.golang.org/appengine"
	"google.golang.org/appengine/log"
)

// For old urls still hosted in github pages.
func PseudowebHandler(w traffic.ResponseWriter, r *traffic.Request) {
	http.Redirect(w, r.Request, fmt.Sprintf("http://pseudoweb.net%s", r.Request.URL.Path), 301)
}

func LongformJsonHandler(w traffic.ResponseWriter, r *traffic.Request) {
	c := appengine.NewContext(r.Request)
	entries, err := LongformPosts(c)
	if err != nil {
		log.Errorf(c, "Error getting posts: %v", err.Error())
		http.Error(w, err.Error(), 500)
		return
	}

	data := map[string]int64{}
	for _, e := range *entries {
		data[e.Longform] = e.Id
	}
	w.Header().Set("Content-Type", "application/json; charset=utf-8")
	w.WriteJSON(data)
}

func LongformWorkHandler(w traffic.ResponseWriter, r *traffic.Request) {
	c := appengine.NewContext(r.Request)

	// Read drafts from disk
	dir := "./longform/drafts/"
	drafts, err := ioutil.ReadDir(dir)
	if err != nil {
		log.Errorf(c, "Error opening directory: %v", err.Error())
		http.Error(w, err.Error(), 500)
		return
	}

	// Iterate through
	for _, file := range drafts {
		if strings.HasPrefix(file.Name(), "20") && file.Mode().IsRegular() {
			draft := true
			err := createPostFromLongformFile(c, dir, file, draft)
			if err != nil {
				http.Error(w, err.Error(), 500)
				return
			}
		}
	}

	// Read posts from disk,
	dir = "./longform/posts/"
	posts, err := ioutil.ReadDir(dir)
	if err != nil {
		log.Errorf(c, "Error opening directory: %v", err.Error())
		http.Error(w, err.Error(), 500)
		return
	}

	// create entries for those that don't exist, update those that do
	for _, file := range posts {
		if strings.HasPrefix(file.Name(), "20") && file.Mode().IsRegular() {
			draft := false
			err := createPostFromLongformFile(c, dir, file, draft)
			if err != nil {
				http.Error(w, err.Error(), 500)
				return
			}
		}
	}

	all_posts, err := LongformPosts(c)
	for _, e := range *all_posts {
		dir := "./longform/posts/"
		if e.Draft {
			dir = "./longform/drafts/"
		}
		if _, err := os.Stat(dir + e.Longform); os.IsNotExist(err) {
			log.Infof(c, "Post no longer exists: %v", e.Longform)
			e.Delete()
		}
	}
}

func createPostFromLongformFile(c context.Context, dir string, file os.FileInfo, draft bool) error {
	f, err := os.Open(dir + file.Name())
	defer f.Close()
	if err != nil {
		log.Errorf(c, "Error opening file %v: %v", file.Name(), err.Error())
		return err
	}

	// get the page from file
	p, err := parser.ReadFrom(f)
	if err != nil {
		log.Errorf(c, "Error parsing file %v: %v", file.Name(), err.Error())
		return err
	}

	meta_uncast, err := p.Metadata()
	if err != nil {
		log.Errorf(c, "Error getting metadata from %v: %v", file.Name(), err.Error())
		return err
	}

	meta := map[string]string{}
	if meta_uncast != nil {
		meta, err = cast.ToStringMapStringE(meta_uncast)
		if err != nil {
			log.Errorf(c, "Error casting metadata for %v: %v. Metadata: %+v", file.Name(), err.Error(), meta_uncast)
			return err
		}
	}

	// create entries for those that don't exist, update those that do
	entry, err := GetLongform(c, file.Name())
	if err != nil {
		log.Warningf(c, "Error getting longform %v: %v", file.Name(), err.Error())
	}

	if entry == nil {
		entry = new(Entry)
		entry.Created = time.Now()
	}

	now := time.Now()

	year := now.Year()
	mnth := now.Month()
	day := now.Day()
	hour := now.Hour()
	min := now.Minute()

	// TODO: Don't throw away errors
	if meta["time"] != "" {
		split := strings.Split(meta["time"], ":")
		hour, _ = strconv.Atoi(split[0])
		min, _ = strconv.Atoi(split[1])
	}

	split := strings.Split(file.Name(), "-")
	year, _ = strconv.Atoi(split[0])
	m, _ := strconv.Atoi(split[1])
	mnth = time.Month(m)
	day, _ = strconv.Atoi(split[2])

	datetime := time.Date(year, mnth, day, hour, min, 0, 0, time.UTC)
	entry.Title = meta["title"]
	entry.Content = string(p.Content())
	entry.Datetime = datetime
	entry.Modified = time.Now()
	entry.Draft = draft
	entry.Longform = file.Name()

	err = entry.Save(c)
	if err != nil {
		log.Errorf(c, "Error saving entry: %v", err.Error())
		return err
	}

	return nil
}
<commit_msg>context<commit_after>package main

import (
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
	"strconv"
	"strings"
	"time"

	"github.com/pilu/traffic"
	"github.com/spf13/cast"
	"github.com/spf13/hugo/parser"
	"golang.org/x/net/context"
	"google.golang.org/appengine"
	"google.golang.org/appengine/log"
)

// For old urls still hosted in github pages.
func PseudowebHandler(w traffic.ResponseWriter, r *traffic.Request) {
	http.Redirect(w, r.Request, fmt.Sprintf("http://pseudoweb.net%s", r.Request.URL.Path), 301)
}

func LongformJsonHandler(w traffic.ResponseWriter, r *traffic.Request) {
	c := appengine.NewContext(r.Request)
	entries, err := LongformPosts(c)
	if err != nil {
		log.Errorf(c, "Error getting posts: %v", err.Error())
		http.Error(w, err.Error(), 500)
		return
	}

	data := map[string]int64{}
	for _, e := range *entries {
		data[e.Longform] = e.Id
	}
	w.Header().Set("Content-Type", "application/json; charset=utf-8")
	w.WriteJSON(data)
}

func LongformWorkHandler(w traffic.ResponseWriter, r *traffic.Request) {
	c := appengine.NewContext(r.Request)

	// Read drafts from disk
	dir := "./longform/drafts/"
	drafts, err := ioutil.ReadDir(dir)
	if err != nil {
		log.Errorf(c, "Error opening directory: %v", err.Error())
		http.Error(w, err.Error(), 500)
		return
	}

	// Iterate through
	for _, file := range drafts {
		if strings.HasPrefix(file.Name(), "20") && file.Mode().IsRegular() {
			draft := true
			err := createPostFromLongformFile(c, dir, file, draft)
			if err != nil {
				http.Error(w, err.Error(), 500)
				return
			}
		}
	}

	// Read posts from disk,
	dir = "./longform/posts/"
	posts, err := ioutil.ReadDir(dir)
	if err != nil {
		log.Errorf(c, "Error opening directory: %v", err.Error())
		http.Error(w, err.Error(), 500)
		return
	}

	// create entries for those that don't exist, update those that do
	for _, file := range posts {
		if strings.HasPrefix(file.Name(), "20") && file.Mode().IsRegular() {
			draft := false
			err := createPostFromLongformFile(c, dir, file, draft)
			if err != nil {
				http.Error(w, err.Error(), 500)
				return
			}
		}
	}

	all_posts, err := LongformPosts(c)
	for _, e := range *all_posts {
		dir := "./longform/posts/"
		if e.Draft {
			dir = "./longform/drafts/"
		}
		if _, err := os.Stat(dir + e.Longform); os.IsNotExist(err) {
			log.Infof(c, "Post no longer exists: %v", e.Longform)
			e.Delete(c)
		}
	}
}

func createPostFromLongformFile(c context.Context, dir string, file os.FileInfo, draft bool) error {
	f, err := os.Open(dir + file.Name())
	defer f.Close()
	if err != nil {
		log.Errorf(c, "Error opening file %v: %v", file.Name(), err.Error())
		return err
	}

	// get the page from file
	p, err := parser.ReadFrom(f)
	if err != nil {
		log.Errorf(c, "Error parsing file %v: %v", file.Name(), err.Error())
		return err
	}

	meta_uncast, err := p.Metadata()
	if err != nil {
		log.Errorf(c, "Error getting metadata from %v: %v", file.Name(), err.Error())
		return err
	}

	meta := map[string]string{}
	if meta_uncast != nil {
		meta, err = cast.ToStringMapStringE(meta_uncast)
		if err != nil {
			log.Errorf(c, "Error casting metadata for %v: %v. Metadata: %+v", file.Name(), err.Error(), meta_uncast)
			return err
		}
	}

	// create entries for those that don't exist, update those that do
	entry, err := GetLongform(c, file.Name())
	if err != nil {
		log.Warningf(c, "Error getting longform %v: %v", file.Name(), err.Error())
	}

	if entry == nil {
		entry = new(Entry)
		entry.Created = time.Now()
	}

	now := time.Now()

	year := now.Year()
	mnth := now.Month()
	day := now.Day()
	hour := now.Hour()
	min := now.Minute()

	// TODO: Don't throw away errors
	if meta["time"] != "" {
		split := strings.Split(meta["time"], ":")
		hour, _ = strconv.Atoi(split[0])
		min, _ = strconv.Atoi(split[1])
	}

	split := strings.Split(file.Name(), "-")
	year, _ = strconv.Atoi(split[0])
	m, _ := strconv.Atoi(split[1])
	mnth = time.Month(m)
	day, _ = strconv.Atoi(split[2])

	datetime := time.Date(year, mnth, day, hour, min, 0, 0, time.UTC)
	entry.Title = meta["title"]
	entry.Content = string(p.Content())
	entry.Datetime = datetime
	entry.Modified = time.Now()
	entry.Draft = draft
	entry.Longform = file.Name()

	err = entry.Save(c)
	if err != nil {
		log.Errorf(c, "Error saving entry: %v", err.Error())
		return err
	}

	return nil
}
<|endoftext|>


<commit_before>package oslice

import (
	"encoding/binary"
	"log"
	"sync"
	"testing"

	"github.com/stretchr/testify/assert"
)

func BenchmarkAppend(b *testing.B) {
	o := OSlice{}
	o.Init(false)

	word := []byte("中国")

	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		o.Append(word)
	}
}

func BenchmarkMapInsert(b *testing.B) {
	m := make(map[string]bool)

	word := "中国"
	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		m[word] = true
	}
}

func BenchmarkSearch(b *testing.B) {
	var o OSlice
	o.Init(false)

	for i := uint64(0); i < 588888; i++ {
		size := binary.Size(i)
		data := make([]byte, size)
		binary.LittleEndian.PutUint64(data, i)
		o.Append(data)
	}

	o.SortIfNot()

	size := binary.Size(uint64(10000))
	data := make([]byte, size)
	binary.LittleEndian.PutUint64(data, 23)

	b.ResetTimer()

	for i := 0; i < b.N; i++ {
		o.Search(data)
	}
}

func BenchmarkMapSearch(b *testing.B) {
	m := map[string]bool{"中国": true}

	word := "中国"
	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		_, _ = m[word]
	}
}

func TestAppend(t *testing.T) {
	var o OSlice
	o.Init(false)

	o.Append([]byte("a"))
	o.Append([]byte("c"))
	o.Append([]byte("b"))

	assert.Equal(t, o.BufLen(), 3)
	assert.Equal(t, o.buf, []byte{'a', 'c', 'b'})
	assert.Equal(t, o.regionList, []Region{{0, 1}, {1, 2}, {2, 3}})
	assert.Equal(t, o.idList, []RegionID{0, 1, 2})

	assert.Equal(t, o.SortIfNot(), true)
	log.Printf("len:%d, cap:%d", o.BufLen(), o.BufCap())
	assert.True(t, o.Shrink(0))
	assert.Equal(t, o.idList, []RegionID{0, 2, 1})
}

func TestAppend2(t *testing.T) {
	var o OSlice
	o.Init(true)

	o.Append([]byte("a"))
	o.Append([]byte("c"))
	o.Append([]byte("b"))

	assert.Equal(t, o.BufLen(), 3)
	assert.Equal(t, o.buf, []byte{'a', 'c', 'b'})
	assert.Equal(t, o.regionList, []Region{{0, 1}, {1, 2}, {2, 3}})
	assert.Equal(t, o.idList, []RegionID{0, 2, 1})

	assert.Equal(t, o.SortIfNot(), false)
	assert.Equal(t, o.idList, []RegionID{0, 2, 1})
}

func TestParrel(t *testing.T) {
	var o OSlice
	o.Init(false)

	strs := []string{"aa", "cc", "dd", "ee", "bb", "ff", "gg", "hh", "ii"}

	for i := 0; i < 3; i++ {
		o.Append([]byte(strs[i]))
	}

	var wg sync.WaitGroup

	assert.Equal(t, o.SortIfNot(), true)
	for i := 3; i < len(strs); i++ {
		wg.Add(1)
		go func(data []byte) {
			o.Append(data)
			wg.Done()
		}([]byte(strs[i]))
	}

	for i := 0; i < 3; i++ {
		wg.Add(1)
		go func(i int) {
			assert.Equal(t, o.ToByte(RegionID(i)), []byte(strs[i]))
			wg.Done()
		}(i)
	}

	wg.Wait()
	assert.False(t, o.SortIfNot())

	for i := 0; i < len(strs)-1; i++ {
		// log.Printf("%q - %q", o.Id(o.idList[i]), o.Id(o.idList[i+1]))
		assert.Equal(t, o.Less(i, i+1), true)
	}
}

func TestParralAppend(t *testing.T) {
	var o OSlice
	o.Init(false)

	strs := []string{"aa", "cc", "dd", "ee", "bb", "ff"}

	var wg sync.WaitGroup

	wg.Add(len(strs))
	for _, str := range strs {
		go func(data []byte) {
			o.Append(data)
			wg.Done()
		}([]byte(str))
	}

	wg.Wait()

	o.SortIfNot()

	assert.Equal(t, o.Len(), len(strs))
	assert.Equal(t, o.regionList, []Region{{0, 2}, {2, 4}, {4, 6}, {6, 8}, {8, 10}, {10, 12}})
	for i := 0; i < len(strs)-1; i++ {
		assert.Equal(t, o.Less(i, i+1), true)
	}
}

func TestParrelAppend2(t *testing.T) {
	var o OSlice
	o.Init(true)

	strs := []string{"aa", "cc", "dd", "ee", "bb", "ff"}

	var wg sync.WaitGroup

	wg.Add(len(strs))
	for _, str := range strs {
		go func(data []byte) {
			o.Append(data)
			wg.Done()
		}([]byte(str))
	}

	wg.Wait()
	// log.Printf("%q", o.buf)

	assert.Equal(t, o.SortIfNot(), false)
	assert.Equal(t, o.Len(), len(strs))
	assert.Equal(t, o.regionList, []Region{{0, 2}, {2, 4}, {4, 6}, {6, 8}, {8, 10}, {10, 12}})
	for i := 0; i < len(strs)-1; i++ {
		// log.Printf("%q - %q", o.Id(o.idList[i]), o.Id(o.idList[i+1]))
		assert.Equal(t, o.Less(i, i+1), true)
	}
}
<commit_msg>add test<commit_after>package oslice

import (
	"encoding/binary"
	"log"
	"sync"
	"testing"

	"github.com/stretchr/testify/assert"
)

<<<<<<< HEAD
func BenchmarkAppend(b *testing.B) {
	o := OSlice{}
	o.Init(false)

	word := []byte("中国")
=======
var o = New()
var isRead = false
var dictFile = "/Users/apple/.go/src/github.com/eruca/cedar-go/testdata/dict.txt"
>>>>>>> 6cb0c39c6e65899343e838e4cc3edca9fdf9fd75

	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		o.Append(word)
	}
}

<<<<<<< HEAD
func BenchmarkMapInsert(b *testing.B) {
	m := make(map[string]bool)

	word := "中国"
	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		m[word] = true
=======
func Test_OSlice(t *testing.T) {
	o := New()

	o.Append([]byte("a"))
	o.Append([]byte("b"))
	o.Append([]byte("e"))
	o.Append([]byte("c"))
	o.Append([]byte("k"))

	o.Sort()
	o.Shrink(0)

	expect(t, o.buf.Len(), 5)
	expect(t, o.regionList, []int{0, 1, 2, 3, 4})
	expect(t, o.idList, []RegionId{0, 1, 3, 2, 4})
	expect(t, cap(o.buf.Bytes()), 5)
	expect(t, o.Search([]byte("e")), true)
	expect(t, o.Search([]byte("d")), false)
	expect(t, o.Query(o.idList[0]), []byte("a"))
	expect(t, o.Query(o.idList[2]), []byte("c"))
}

func read() {
	f, err := os.Open(dictFile)
	if err != nil {
		panic(err)
>>>>>>> 6cb0c39c6e65899343e838e4cc3edca9fdf9fd75
	}
}

func BenchmarkSearch(b *testing.B) {
	var o OSlice
	o.Init(false)

	for i := uint64(0); i < 588888; i++ {
		size := binary.Size(i)
		data := make([]byte, size)
		binary.LittleEndian.PutUint64(data, i)
		o.Append(data)
	}

<<<<<<< HEAD
	o.SortIfNot()
=======
	o.Sort()
	o.Shrink(0)
>>>>>>> 6cb0c39c6e65899343e838e4cc3edca9fdf9fd75

	size := binary.Size(uint64(10000))
	data := make([]byte, size)
	binary.LittleEndian.PutUint64(data, 23)

	b.ResetTimer()

	for i := 0; i < b.N; i++ {
		o.Search(data)
	}
}

func BenchmarkMapSearch(b *testing.B) {
	m := map[string]bool{"中国": true}

	word := "中国"
	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		_, _ = m[word]
	}
}

func TestAppend(t *testing.T) {
	var o OSlice
	o.Init(false)

	o.Append([]byte("a"))
	o.Append([]byte("c"))
	o.Append([]byte("b"))

	assert.Equal(t, o.BufLen(), 3)
	assert.Equal(t, o.buf, []byte{'a', 'c', 'b'})
	assert.Equal(t, o.regionList, []Region{{0, 1}, {1, 2}, {2, 3}})
	assert.Equal(t, o.idList, []RegionID{0, 1, 2})

	assert.Equal(t, o.SortIfNot(), true)
	log.Printf("len:%d, cap:%d", o.BufLen(), o.BufCap())
	assert.True(t, o.Shrink(0))
	assert.Equal(t, o.idList, []RegionID{0, 2, 1})
}

func TestAppend2(t *testing.T) {
	var o OSlice
	o.Init(true)

	o.Append([]byte("a"))
	o.Append([]byte("c"))
	o.Append([]byte("b"))

	assert.Equal(t, o.BufLen(), 3)
	assert.Equal(t, o.buf, []byte{'a', 'c', 'b'})
	assert.Equal(t, o.regionList, []Region{{0, 1}, {1, 2}, {2, 3}})
	assert.Equal(t, o.idList, []RegionID{0, 2, 1})

	assert.Equal(t, o.SortIfNot(), false)
	assert.Equal(t, o.idList, []RegionID{0, 2, 1})
}

func TestParrel(t *testing.T) {
	var o OSlice
	o.Init(false)

	strs := []string{"aa", "cc", "dd", "ee", "bb", "ff", "gg", "hh", "ii"}

	for i := 0; i < 3; i++ {
		o.Append([]byte(strs[i]))
	}

	var wg sync.WaitGroup

	assert.Equal(t, o.SortIfNot(), true)
	for i := 3; i < len(strs); i++ {
		wg.Add(1)
		go func(data []byte) {
			o.Append(data)
			wg.Done()
		}([]byte(strs[i]))
	}

	for i := 0; i < 3; i++ {
		wg.Add(1)
		go func(i int) {
			assert.Equal(t, o.ToByte(RegionID(i)), []byte(strs[i]))
			wg.Done()
		}(i)
	}

	wg.Wait()
	assert.False(t, o.SortIfNot())

	for i := 0; i < len(strs)-1; i++ {
		// log.Printf("%q - %q", o.Id(o.idList[i]), o.Id(o.idList[i+1]))
		assert.Equal(t, o.Less(i, i+1), true)
	}
}

func TestParralAppend(t *testing.T) {
	var o OSlice
	o.Init(false)

	strs := []string{"aa", "cc", "dd", "ee", "bb", "ff"}

	var wg sync.WaitGroup

	wg.Add(len(strs))
	for _, str := range strs {
		go func(data []byte) {
			o.Append(data)
			wg.Done()
		}([]byte(str))
	}

	wg.Wait()

	o.SortIfNot()

	assert.Equal(t, o.Len(), len(strs))
	assert.Equal(t, o.regionList, []Region{{0, 2}, {2, 4}, {4, 6}, {6, 8}, {8, 10}, {10, 12}})
	for i := 0; i < len(strs)-1; i++ {
		assert.Equal(t, o.Less(i, i+1), true)
	}
}

func TestParrelAppend2(t *testing.T) {
	var o OSlice
	o.Init(true)

	strs := []string{"aa", "cc", "dd", "ee", "bb", "ff"}

	var wg sync.WaitGroup

	wg.Add(len(strs))
	for _, str := range strs {
		go func(data []byte) {
			o.Append(data)
			wg.Done()
		}([]byte(str))
	}

	wg.Wait()
	// log.Printf("%q", o.buf)

	assert.Equal(t, o.SortIfNot(), false)
	assert.Equal(t, o.Len(), len(strs))
	assert.Equal(t, o.regionList, []Region{{0, 2}, {2, 4}, {4, 6}, {6, 8}, {8, 10}, {10, 12}})
	for i := 0; i < len(strs)-1; i++ {
		// log.Printf("%q - %q", o.Id(o.idList[i]), o.Id(o.idList[i+1]))
		assert.Equal(t, o.Less(i, i+1), true)
	}
}
<|endoftext|>


<commit_before>/**
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *  http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */

package builder

import (
	"bufio"
	"bytes"
	"fmt"
	"io"
	"os"
	"path/filepath"
	"strings"

	log "github.com/Sirupsen/logrus"

	"mynewt.apache.org/newt/newt/interfaces"
	"mynewt.apache.org/newt/newt/project"
	"mynewt.apache.org/newt/newt/target"
	"mynewt.apache.org/newt/newt/toolchain"
	"mynewt.apache.org/newt/util"
)

const CMAKELISTS_FILENAME string = "CMakeLists.txt"

func CmakeListsPath() string {
	return project.GetProject().BasePath + "/" + CMAKELISTS_FILENAME
}

func EscapeName(name string) string {
	return strings.Replace(name, "/", "_", -1)
}

func trimProjectPath(path string) string {
	proj := interfaces.GetProject()
	path = strings.TrimPrefix(path, proj.Path()+"/")
	return path
}

func trimProjectPathSlice(elements []string) {
	for e := range elements {
		elements[e] = trimProjectPath(elements[e])
	}
}

func extractIncludes(flags *[]string, includes *[]string, other *[]string) {
	for _, f := range *flags {
		if strings.HasPrefix(f, "-I") {
			*includes = append(*includes, strings.TrimPrefix(f, "-I"))
		} else {
			*other = append(*other, f)
		}
	}
}

func CmakeSourceObjectWrite(w io.Writer, cj toolchain.CompilerJob, includeDirs *[]string) {
	c := cj.Compiler

	compileFlags := []string{}
	otherFlags := []string{}

	switch cj.CompilerType {
	case toolchain.COMPILER_TYPE_C:
		compileFlags = append(compileFlags, c.GetCompilerInfo().Cflags...)
		compileFlags = append(compileFlags, c.GetLocalCompilerInfo().Cflags...)
	case toolchain.COMPILER_TYPE_ASM:
		compileFlags = append(compileFlags, c.GetCompilerInfo().Cflags...)
		compileFlags = append(compileFlags, c.GetLocalCompilerInfo().Cflags...)
		compileFlags = append(compileFlags, c.GetCompilerInfo().Aflags...)
		compileFlags = append(compileFlags, c.GetLocalCompilerInfo().Aflags...)
	case toolchain.COMPILER_TYPE_CPP:
		compileFlags = append(compileFlags, c.GetCompilerInfo().Cflags...)
		compileFlags = append(compileFlags, c.GetLocalCompilerInfo().Cflags...)
	}

	extractIncludes(&compileFlags, includeDirs, &otherFlags)
	cj.Filename = trimProjectPath(cj.Filename)

	fmt.Fprintf(w, `set_property(SOURCE %s APPEND_STRING
														PROPERTY
														COMPILE_FLAGS
														"%s")`,
		cj.Filename,
		strings.Replace(strings.Join(otherFlags, " "), "\"", "\\\\\\\"", -1))
	fmt.Fprintln(w)
}

func (b *Builder) CMakeBuildPackageWrite(w io.Writer, bpkg *BuildPackage) (*BuildPackage, error) {
	entries, err := b.collectCompileEntriesBpkg(bpkg)
	if err != nil {
		return nil, err
	}

	if len(entries) <= 0 {
		return nil, nil
	}

	otherIncludes := []string{}
	files := []string{}

	for _, s := range entries {
		filename := filepath.ToSlash(s.Filename)
		if s.Compiler.ShouldIgnoreFile(filename) {
			log.Infof("Ignoring %s because package dictates it.\n", filename)
			continue
		}

		CmakeSourceObjectWrite(w, s, &otherIncludes)
		s.Filename = trimProjectPath(s.Filename)
		files = append(files, s.Filename)
	}

	if len(files) <= 0 {
		return nil, nil
	}

	pkgName := bpkg.rpkg.Lpkg.Name()

	util.StatusMessage(util.VERBOSITY_DEFAULT, "Generating CMakeLists.txt for %s\n", pkgName)
	fmt.Fprintf(w, "# Generating CMakeLists.txt for %s\n\n", pkgName)
	fmt.Fprintf(w, "add_library(%s %s)\n\n",
		EscapeName(pkgName),
		strings.Join(files, " "))

	archivePath := filepath.Dir(b.ArchivePath(bpkg))
	archivePath = trimProjectPath(archivePath)
	CmakeCompilerInfoWrite(w, archivePath, bpkg, entries[0], otherIncludes)

	return bpkg, nil
}

func (b *Builder) CMakeTargetWrite(w io.Writer, targetCompiler *toolchain.Compiler) error {
	bpkgs := b.sortedBuildPackages()

	c := targetCompiler

	builtPackages := []*BuildPackage{}
	for _, bpkg := range bpkgs {
		builtPackage, err := b.CMakeBuildPackageWrite(w, bpkg)
		if err != nil {
			return err
		}

		if builtPackage != nil {
			builtPackages = append(builtPackages, builtPackage)
		}
	}

	elfName := "cmake_" + filepath.Base(b.AppElfPath())
	fmt.Fprintf(w, "# Generating code for %s\n\n", elfName)

	var targetObjectsBuffer bytes.Buffer

	for _, bpkg := range builtPackages {
		targetObjectsBuffer.WriteString(fmt.Sprintf("%s ",
			EscapeName(bpkg.rpkg.Lpkg.Name())))
	}

	elfOutputDir := trimProjectPath(filepath.Dir(b.AppElfPath()))
	fmt.Fprintf(w, "file(WRITE %s \"\")\n", filepath.Join(elfOutputDir, "null.c"))
	fmt.Fprintf(w, "add_executable(%s %s)\n\n", elfName, filepath.Join(elfOutputDir, "null.c"))

	if c.GetLdResolveCircularDeps() {
		fmt.Fprintf(w, "target_link_libraries(%s -Wl,--start-group %s -Wl,--end-group)\n",
			elfName, targetObjectsBuffer.String())
	} else {
		fmt.Fprintf(w, "target_link_libraries(%s %s)\n",
			elfName, targetObjectsBuffer.String())
	}

	fmt.Fprintf(w, `set_property(TARGET %s APPEND_STRING
														PROPERTY
														COMPILE_FLAGS
														"%s")`,
		elfName,
		strings.Replace(strings.Join(append(c.GetCompilerInfo().Cflags,
			c.GetLocalCompilerInfo().Cflags...), " "), "\"", "\\\\\\\"", -1))
	fmt.Fprintln(w)

	lFlags := append(c.GetCompilerInfo().Lflags, c.GetLocalCompilerInfo().Lflags...)
	for _, ld := range c.LinkerScripts {
		lFlags = append(lFlags, "-T"+ld)
	}

	lFlags = append(lFlags, c.GetLocalCompilerInfo().Cflags...)
	fmt.Fprintf(w, `set_target_properties(%s
							PROPERTIES
							ARCHIVE_OUTPUT_DIRECTORY %s
							LIBRARY_OUTPUT_DIRECTORY %s
							RUNTIME_OUTPUT_DIRECTORY %s
							LINK_FLAGS "%s"
							LINKER_LANGUAGE C)`,
		elfName,
		elfOutputDir,
		elfOutputDir,
		elfOutputDir,
		strings.Replace(strings.Join(lFlags, " "), "\"", "\\\\\\\"", -1))

	fmt.Fprintln(w)

	libs := strings.Join(getLibsFromLinkerFlags(lFlags), " ")
	fmt.Fprintf(w, "# Workaround for gcc linker woes\n")
	fmt.Fprintf(w, "set(CMAKE_C_LINK_EXECUTABLE \"${CMAKE_C_LINK_EXECUTABLE} %s\")\n", libs)
	fmt.Fprintln(w)

	return nil
}

func getLibsFromLinkerFlags(lflags []string) []string {
	libs := []string{}

	for _, flag := range lflags {
		if strings.HasPrefix(flag, "-l") {
			libs = append(libs, flag)
		}
	}

	return libs
}

func CmakeCompilerInfoWrite(w io.Writer, archiveFile string, bpkg *BuildPackage,
	cj toolchain.CompilerJob, otherIncludes []string) {
	c := cj.Compiler

	var includes []string

	includes = append(includes, c.GetCompilerInfo().Includes...)
	includes = append(includes, c.GetLocalCompilerInfo().Includes...)
	includes = append(includes, otherIncludes...)

	// Sort and remove duplicate flags
	includes = util.SortFields(includes...)
	trimProjectPathSlice(includes)

	fmt.Fprintf(w, `set_target_properties(%s
							PROPERTIES
							ARCHIVE_OUTPUT_DIRECTORY %s
							LIBRARY_OUTPUT_DIRECTORY %s
							RUNTIME_OUTPUT_DIRECTORY %s)`,
		EscapeName(bpkg.rpkg.Lpkg.Name()),
		archiveFile,
		archiveFile,
		archiveFile,
	)
	fmt.Fprintln(w)
	fmt.Fprintf(w, "target_include_directories(%s PUBLIC %s)\n\n",
		EscapeName(bpkg.rpkg.Lpkg.Name()),
		strings.Join(includes, " "))
}

func (t *TargetBuilder) CMakeTargetBuilderWrite(w io.Writer, targetCompiler *toolchain.Compiler) error {
	if err := t.PrepBuild(); err != nil {
		return err
	}

	/* Build the Apps */
	project.ResetDeps(t.AppList)

	targetCompiler.LinkerScripts = t.bspPkg.LinkerScripts

	if err := t.bspPkg.Reload(t.AppBuilder.cfg.SettingValues()); err != nil {
		return err
	}

	if err := t.AppBuilder.CMakeTargetWrite(w, targetCompiler); err != nil {
		return err
	}

	return nil
}

func CmakeCompilerWrite(w io.Writer, c *toolchain.Compiler) {
	/* Since CMake 3 it is required to set a full path to the compiler */
	/* TODO: get rid of the prefix to /usr/bin */
	fmt.Fprintln(w, "set(CMAKE_SYSTEM_NAME Generic)")
	fmt.Fprintln(w, "set(CMAKE_TRY_COMPILE_TARGET_TYPE STATIC_LIBRARY)")
	fmt.Fprintf(w, "set(CMAKE_C_COMPILER %s)\n", c.GetCcPath())
	fmt.Fprintf(w, "set(CMAKE_CXX_COMPILER %s)\n", c.GetCppPath())
	fmt.Fprintf(w, "set(CMAKE_ASM_COMPILER %s)\n", c.GetAsPath())
	/* TODO: cmake returns error on link */
	//fmt.Fprintf(w, "set(CMAKE_AR %s)\n", c.GetArPath())
	fmt.Fprintln(w)
}

func CmakeHeaderWrite(w io.Writer, c *toolchain.Compiler, targetName string) {
	fmt.Fprintln(w, "cmake_minimum_required(VERSION 3.7)\n")
	CmakeCompilerWrite(w, c)
	fmt.Fprintf(w, "project(%s VERSION 0.0.0 LANGUAGES C ASM)\n\n", targetName)
	fmt.Fprintln(w, "SET(CMAKE_C_FLAGS_BACKUP  \"${CMAKE_C_FLAGS}\")")
	fmt.Fprintln(w, "SET(CMAKE_CXX_FLAGS_BACKUP  \"${CMAKE_CXX_FLAGS}\")")
	fmt.Fprintln(w, "SET(CMAKE_ASM_FLAGS_BACKUP  \"${CMAKE_ASM_FLAGS}\")")
	fmt.Fprintln(w)
}

func CMakeTargetGenerate(target *target.Target) error {
	CmakeFileHandle, err := os.Create(CmakeListsPath())
	if err != nil {
		return util.ChildNewtError(err)
	}

	var b = bytes.Buffer{}
	w := bufio.NewWriter(&b)
	defer CmakeFileHandle.Close()

	targetBuilder, err := NewTargetBuilder(target)
	if err != nil {
		return err
	}

	targetCompiler, err := targetBuilder.NewCompiler("")
	if err != nil {
		return err
	}

	CmakeHeaderWrite(w, targetCompiler, target.ShortName())

	if err := targetBuilder.CMakeTargetBuilderWrite(w, targetCompiler); err != nil {
		return err
	}

	w.Flush()

	CmakeFileHandle.Write(b.Bytes())
	return nil
}
<commit_msg>cmake: Sort and remove duplicate compile flags<commit_after>/**
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *  http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */

package builder

import (
	"bufio"
	"bytes"
	"fmt"
	"io"
	"os"
	"path/filepath"
	"strings"

	log "github.com/Sirupsen/logrus"

	"mynewt.apache.org/newt/newt/interfaces"
	"mynewt.apache.org/newt/newt/project"
	"mynewt.apache.org/newt/newt/target"
	"mynewt.apache.org/newt/newt/toolchain"
	"mynewt.apache.org/newt/util"
)

const CMAKELISTS_FILENAME string = "CMakeLists.txt"

func CmakeListsPath() string {
	return project.GetProject().BasePath + "/" + CMAKELISTS_FILENAME
}

func EscapeName(name string) string {
	return strings.Replace(name, "/", "_", -1)
}

func trimProjectPath(path string) string {
	proj := interfaces.GetProject()
	path = strings.TrimPrefix(path, proj.Path()+"/")
	return path
}

func trimProjectPathSlice(elements []string) {
	for e := range elements {
		elements[e] = trimProjectPath(elements[e])
	}
}

func extractIncludes(flags *[]string, includes *[]string, other *[]string) {
	for _, f := range *flags {
		if strings.HasPrefix(f, "-I") {
			*includes = append(*includes, strings.TrimPrefix(f, "-I"))
		} else {
			*other = append(*other, f)
		}
	}
}

func CmakeSourceObjectWrite(w io.Writer, cj toolchain.CompilerJob, includeDirs *[]string) {
	c := cj.Compiler

	compileFlags := []string{}
	otherFlags := []string{}

	switch cj.CompilerType {
	case toolchain.COMPILER_TYPE_C:
		compileFlags = append(compileFlags, c.GetCompilerInfo().Cflags...)
		compileFlags = append(compileFlags, c.GetLocalCompilerInfo().Cflags...)
	case toolchain.COMPILER_TYPE_ASM:
		compileFlags = append(compileFlags, c.GetCompilerInfo().Cflags...)
		compileFlags = append(compileFlags, c.GetLocalCompilerInfo().Cflags...)
		compileFlags = append(compileFlags, c.GetCompilerInfo().Aflags...)
		compileFlags = append(compileFlags, c.GetLocalCompilerInfo().Aflags...)
	case toolchain.COMPILER_TYPE_CPP:
		compileFlags = append(compileFlags, c.GetCompilerInfo().Cflags...)
		compileFlags = append(compileFlags, c.GetLocalCompilerInfo().Cflags...)
	}

	extractIncludes(&compileFlags, includeDirs, &otherFlags)
	cj.Filename = trimProjectPath(cj.Filename)

	// Sort and remove duplicate flags
	otherFlags = util.SortFields(otherFlags...)

	fmt.Fprintf(w, `set_property(SOURCE %s APPEND_STRING
														PROPERTY
														COMPILE_FLAGS
														"%s")`,
		cj.Filename,
		strings.Replace(strings.Join(otherFlags, " "), "\"", "\\\\\\\"", -1))
	fmt.Fprintln(w)
}

func (b *Builder) CMakeBuildPackageWrite(w io.Writer, bpkg *BuildPackage) (*BuildPackage, error) {
	entries, err := b.collectCompileEntriesBpkg(bpkg)
	if err != nil {
		return nil, err
	}

	if len(entries) <= 0 {
		return nil, nil
	}

	otherIncludes := []string{}
	files := []string{}

	for _, s := range entries {
		filename := filepath.ToSlash(s.Filename)
		if s.Compiler.ShouldIgnoreFile(filename) {
			log.Infof("Ignoring %s because package dictates it.\n", filename)
			continue
		}

		CmakeSourceObjectWrite(w, s, &otherIncludes)
		s.Filename = trimProjectPath(s.Filename)
		files = append(files, s.Filename)
	}

	if len(files) <= 0 {
		return nil, nil
	}

	pkgName := bpkg.rpkg.Lpkg.Name()

	util.StatusMessage(util.VERBOSITY_DEFAULT, "Generating CMakeLists.txt for %s\n", pkgName)
	fmt.Fprintf(w, "# Generating CMakeLists.txt for %s\n\n", pkgName)
	fmt.Fprintf(w, "add_library(%s %s)\n\n",
		EscapeName(pkgName),
		strings.Join(files, " "))

	archivePath := filepath.Dir(b.ArchivePath(bpkg))
	archivePath = trimProjectPath(archivePath)
	CmakeCompilerInfoWrite(w, archivePath, bpkg, entries[0], otherIncludes)

	return bpkg, nil
}

func (b *Builder) CMakeTargetWrite(w io.Writer, targetCompiler *toolchain.Compiler) error {
	bpkgs := b.sortedBuildPackages()

	c := targetCompiler

	builtPackages := []*BuildPackage{}
	for _, bpkg := range bpkgs {
		builtPackage, err := b.CMakeBuildPackageWrite(w, bpkg)
		if err != nil {
			return err
		}

		if builtPackage != nil {
			builtPackages = append(builtPackages, builtPackage)
		}
	}

	elfName := "cmake_" + filepath.Base(b.AppElfPath())
	fmt.Fprintf(w, "# Generating code for %s\n\n", elfName)

	var targetObjectsBuffer bytes.Buffer

	for _, bpkg := range builtPackages {
		targetObjectsBuffer.WriteString(fmt.Sprintf("%s ",
			EscapeName(bpkg.rpkg.Lpkg.Name())))
	}

	elfOutputDir := trimProjectPath(filepath.Dir(b.AppElfPath()))
	fmt.Fprintf(w, "file(WRITE %s \"\")\n", filepath.Join(elfOutputDir, "null.c"))
	fmt.Fprintf(w, "add_executable(%s %s)\n\n", elfName, filepath.Join(elfOutputDir, "null.c"))

	if c.GetLdResolveCircularDeps() {
		fmt.Fprintf(w, "target_link_libraries(%s -Wl,--start-group %s -Wl,--end-group)\n",
			elfName, targetObjectsBuffer.String())
	} else {
		fmt.Fprintf(w, "target_link_libraries(%s %s)\n",
			elfName, targetObjectsBuffer.String())
	}

	fmt.Fprintf(w, `set_property(TARGET %s APPEND_STRING
														PROPERTY
														COMPILE_FLAGS
														"%s")`,
		elfName,
		strings.Replace(strings.Join(append(c.GetCompilerInfo().Cflags,
			c.GetLocalCompilerInfo().Cflags...), " "), "\"", "\\\\\\\"", -1))
	fmt.Fprintln(w)

	lFlags := append(c.GetCompilerInfo().Lflags, c.GetLocalCompilerInfo().Lflags...)
	for _, ld := range c.LinkerScripts {
		lFlags = append(lFlags, "-T"+ld)
	}

	lFlags = append(lFlags, c.GetLocalCompilerInfo().Cflags...)
	fmt.Fprintf(w, `set_target_properties(%s
							PROPERTIES
							ARCHIVE_OUTPUT_DIRECTORY %s
							LIBRARY_OUTPUT_DIRECTORY %s
							RUNTIME_OUTPUT_DIRECTORY %s
							LINK_FLAGS "%s"
							LINKER_LANGUAGE C)`,
		elfName,
		elfOutputDir,
		elfOutputDir,
		elfOutputDir,
		strings.Replace(strings.Join(lFlags, " "), "\"", "\\\\\\\"", -1))

	fmt.Fprintln(w)

	libs := strings.Join(getLibsFromLinkerFlags(lFlags), " ")
	fmt.Fprintf(w, "# Workaround for gcc linker woes\n")
	fmt.Fprintf(w, "set(CMAKE_C_LINK_EXECUTABLE \"${CMAKE_C_LINK_EXECUTABLE} %s\")\n", libs)
	fmt.Fprintln(w)

	return nil
}

func getLibsFromLinkerFlags(lflags []string) []string {
	libs := []string{}

	for _, flag := range lflags {
		if strings.HasPrefix(flag, "-l") {
			libs = append(libs, flag)
		}
	}

	return libs
}

func CmakeCompilerInfoWrite(w io.Writer, archiveFile string, bpkg *BuildPackage,
	cj toolchain.CompilerJob, otherIncludes []string) {
	c := cj.Compiler

	var includes []string

	includes = append(includes, c.GetCompilerInfo().Includes...)
	includes = append(includes, c.GetLocalCompilerInfo().Includes...)
	includes = append(includes, otherIncludes...)

	// Sort and remove duplicate flags
	includes = util.SortFields(includes...)
	trimProjectPathSlice(includes)

	fmt.Fprintf(w, `set_target_properties(%s
							PROPERTIES
							ARCHIVE_OUTPUT_DIRECTORY %s
							LIBRARY_OUTPUT_DIRECTORY %s
							RUNTIME_OUTPUT_DIRECTORY %s)`,
		EscapeName(bpkg.rpkg.Lpkg.Name()),
		archiveFile,
		archiveFile,
		archiveFile,
	)
	fmt.Fprintln(w)
	fmt.Fprintf(w, "target_include_directories(%s PUBLIC %s)\n\n",
		EscapeName(bpkg.rpkg.Lpkg.Name()),
		strings.Join(includes, " "))
}

func (t *TargetBuilder) CMakeTargetBuilderWrite(w io.Writer, targetCompiler *toolchain.Compiler) error {
	if err := t.PrepBuild(); err != nil {
		return err
	}

	/* Build the Apps */
	project.ResetDeps(t.AppList)

	targetCompiler.LinkerScripts = t.bspPkg.LinkerScripts

	if err := t.bspPkg.Reload(t.AppBuilder.cfg.SettingValues()); err != nil {
		return err
	}

	if err := t.AppBuilder.CMakeTargetWrite(w, targetCompiler); err != nil {
		return err
	}

	return nil
}

func CmakeCompilerWrite(w io.Writer, c *toolchain.Compiler) {
	/* Since CMake 3 it is required to set a full path to the compiler */
	/* TODO: get rid of the prefix to /usr/bin */
	fmt.Fprintln(w, "set(CMAKE_SYSTEM_NAME Generic)")
	fmt.Fprintln(w, "set(CMAKE_TRY_COMPILE_TARGET_TYPE STATIC_LIBRARY)")
	fmt.Fprintf(w, "set(CMAKE_C_COMPILER %s)\n", c.GetCcPath())
	fmt.Fprintf(w, "set(CMAKE_CXX_COMPILER %s)\n", c.GetCppPath())
	fmt.Fprintf(w, "set(CMAKE_ASM_COMPILER %s)\n", c.GetAsPath())
	/* TODO: cmake returns error on link */
	//fmt.Fprintf(w, "set(CMAKE_AR %s)\n", c.GetArPath())
	fmt.Fprintln(w)
}

func CmakeHeaderWrite(w io.Writer, c *toolchain.Compiler, targetName string) {
	fmt.Fprintln(w, "cmake_minimum_required(VERSION 3.7)\n")
	CmakeCompilerWrite(w, c)
	fmt.Fprintf(w, "project(%s VERSION 0.0.0 LANGUAGES C ASM)\n\n", targetName)
	fmt.Fprintln(w, "SET(CMAKE_C_FLAGS_BACKUP  \"${CMAKE_C_FLAGS}\")")
	fmt.Fprintln(w, "SET(CMAKE_CXX_FLAGS_BACKUP  \"${CMAKE_CXX_FLAGS}\")")
	fmt.Fprintln(w, "SET(CMAKE_ASM_FLAGS_BACKUP  \"${CMAKE_ASM_FLAGS}\")")
	fmt.Fprintln(w)
}

func CMakeTargetGenerate(target *target.Target) error {
	CmakeFileHandle, err := os.Create(CmakeListsPath())
	if err != nil {
		return util.ChildNewtError(err)
	}

	var b = bytes.Buffer{}
	w := bufio.NewWriter(&b)
	defer CmakeFileHandle.Close()

	targetBuilder, err := NewTargetBuilder(target)
	if err != nil {
		return err
	}

	targetCompiler, err := targetBuilder.NewCompiler("")
	if err != nil {
		return err
	}

	CmakeHeaderWrite(w, targetCompiler, target.ShortName())

	if err := targetBuilder.CMakeTargetBuilderWrite(w, targetCompiler); err != nil {
		return err
	}

	w.Flush()

	CmakeFileHandle.Write(b.Bytes())
	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"crypto/x509"
	"encoding/base64"
	"encoding/hex"
	"encoding/pem"
	"fmt"
	"net"
	"net/http"

	"github.com/gorilla/mux"
	"golang.org/x/crypto/scrypt"

	"github.com/lxc/lxd/shared"
)

func certGenerateFingerprint(cert *x509.Certificate) string {
	return fmt.Sprintf("%x", cert.SerialNumber)
}

func (d *Daemon) hasPwd() bool {
	_, err := dbPasswordGet(d.db)
	return err == nil
}

func (d *Daemon) verifyAdminPwd(password string) bool {
	value, err := dbPasswordGet(d.db)

	if err != nil {
		shared.Debugf("verifyAdminPwd: %s", err)
		return false
	}

	buff, err := hex.DecodeString(value)
	if err != nil {
		shared.Debugf("hex decode failed")
		return false
	}

	salt := buff[0:PW_SALT_BYTES]
	hash, err := scrypt.Key([]byte(password), salt, 1<<14, 8, 1, PW_HASH_BYTES)
	if err != nil {
		shared.Debugf("failed to create hash to check")
		return false
	}
	if !bytes.Equal(hash, buff[PW_SALT_BYTES:]) {
		shared.Debugf("Bad password received")
		return false
	}
	shared.Debugf("Verified the admin password")
	return true
}

func certificatesGet(d *Daemon, r *http.Request) Response {
	recursion := d.isRecursionRequest(r)

	if recursion {
		certResponses := []shared.CertInfo{}

		baseCerts, err := dbCertsGet(d.db)
		if err != nil {
			return SmartError(err)
		}
		for _, baseCert := range baseCerts {
			resp := shared.CertInfo{}
			resp.Fingerprint = baseCert.Fingerprint
			resp.Certificate = baseCert.Certificate
			if baseCert.Type == 1 {
				resp.Type = "client"
			} else {
				resp.Type = "unknown"
			}
			certResponses = append(certResponses, resp)
		}
		return SyncResponse(true, certResponses)
	}

	body := []string{}
	for _, cert := range d.clientCerts {
		fingerprint := certGenerateFingerprint(&cert)
		body = append(body, fingerprint)
	}

	return SyncResponse(true, body)
}

type certificatesPostBody struct {
	Type        string `json:"type"`
	Certificate string `json:"certificate"`
	Name        string `json:"name"`
	Password    string `json:"password"`
}

func readSavedClientCAList(d *Daemon) {
	d.clientCerts = []x509.Certificate{}

	dbCerts, err := dbCertsGet(d.db)
	if err != nil {
		shared.Logf("Error reading certificates from database: %s\n", err)
		return
	}

	for _, dbCert := range dbCerts {
		certBlock, _ := pem.Decode([]byte(dbCert.Certificate))
		cert, err := x509.ParseCertificate(certBlock.Bytes)
		if err != nil {
			shared.Logf("Error reading certificate for %s: %s\n", dbCert.Name, err)
			continue
		}
		d.clientCerts = append(d.clientCerts, *cert)
	}
}

func saveCert(d *Daemon, host string, cert *x509.Certificate) error {

	baseCert := new(dbCertInfo)
	baseCert.Fingerprint = certGenerateFingerprint(cert)
	baseCert.Type = 1
	baseCert.Name = host
	baseCert.Certificate = string(
		pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: cert.Raw}),
	)

	return dbCertSave(d.db, baseCert)
}

func certificatesPost(d *Daemon, r *http.Request) Response {
	req := certificatesPostBody{}

	if err := shared.ReadToJSON(r.Body, &req); err != nil {
		return BadRequest(err)
	}

	if req.Type != "client" {
		return BadRequest(fmt.Errorf("Unknown request type %s", req.Type))
	}

	var cert *x509.Certificate
	var name string
	if req.Certificate != "" {

		data, err := base64.StdEncoding.DecodeString(req.Certificate)
		if err != nil {
			return BadRequest(err)
		}

		cert, err = x509.ParseCertificate(data)
		if err != nil {
			return BadRequest(err)
		}
		name = req.Name

	} else if r.TLS != nil {

		if len(r.TLS.PeerCertificates) < 1 {
			return BadRequest(fmt.Errorf("No client certificate provided"))
		}
		cert = r.TLS.PeerCertificates[len(r.TLS.PeerCertificates)-1]

		remoteHost, _, err := net.SplitHostPort(r.RemoteAddr)
		if err != nil {
			return InternalError(err)
		}

		name = remoteHost
	} else {
		return BadRequest(fmt.Errorf("Can't use TLS data on non-TLS link"))
	}

	serial := cert.SerialNumber
	for _, existingCert := range d.clientCerts {
		if serial == existingCert.SerialNumber {
			return EmptySyncResponse
		}
	}

	if !d.isTrustedClient(r) && !d.verifyAdminPwd(req.Password) {
		return Forbidden
	}

	err := saveCert(d, name, cert)
	if err != nil {
		return SmartError(err)
	}

	d.clientCerts = append(d.clientCerts, *cert)

	return EmptySyncResponse
}

var certificatesCmd = Command{
	"certificates",
	false,
	true,
	certificatesGet,
	nil,
	certificatesPost,
	nil,
}

func certificateFingerprintGet(d *Daemon, r *http.Request) Response {
	fingerprint := mux.Vars(r)["fingerprint"]

	cert, err := doCertificateGet(d, fingerprint)
	if err != nil {
		return SmartError(err)
	}

	return SyncResponse(true, cert)
}

func doCertificateGet(d *Daemon, fingerprint string) (shared.CertInfo, error) {
	resp := shared.CertInfo{}

	dbCertInfo, err := dbCertGet(d.db, fingerprint)
	if err != nil {
		return resp, err
	}

	resp.Fingerprint = dbCertInfo.Fingerprint
	resp.Certificate = dbCertInfo.Certificate
	if dbCertInfo.Type == 1 {
		resp.Type = "client"
	} else {
		resp.Type = "unknown"
	}

	return resp, nil
}

func certificateFingerprintDelete(d *Daemon, r *http.Request) Response {
	fingerprint := mux.Vars(r)["fingerprint"]

	certInfo, err := dbCertGet(d.db, fingerprint)
	if err != nil {
		return NotFound
	}

	err = dbCertDelete(d.db, certInfo.Fingerprint)
	if err != nil {
		return SmartError(err)
	}
	readSavedClientCAList(d)

	return EmptySyncResponse
}

var certificateFingerprintCmd = Command{
	"certificates/{fingerprint}",
	false,
	false,
	certificateFingerprintGet,
	nil,
	nil,
	certificateFingerprintDelete,
}
<commit_msg>Use SHA256 sums of the certs again.<commit_after>package main

import (
	"bytes"
	"crypto/sha256"
	"crypto/x509"
	"encoding/base64"
	"encoding/hex"
	"encoding/pem"
	"fmt"
	"net"
	"net/http"

	"github.com/gorilla/mux"
	"golang.org/x/crypto/scrypt"

	"github.com/lxc/lxd/shared"
)

func certGenerateFingerprint(cert *x509.Certificate) string {
	return fmt.Sprintf("%x", sha256.Sum256(cert.Raw))
}

func (d *Daemon) hasPwd() bool {
	_, err := dbPasswordGet(d.db)
	return err == nil
}

func (d *Daemon) verifyAdminPwd(password string) bool {
	value, err := dbPasswordGet(d.db)

	if err != nil {
		shared.Debugf("verifyAdminPwd: %s", err)
		return false
	}

	buff, err := hex.DecodeString(value)
	if err != nil {
		shared.Debugf("hex decode failed")
		return false
	}

	salt := buff[0:PW_SALT_BYTES]
	hash, err := scrypt.Key([]byte(password), salt, 1<<14, 8, 1, PW_HASH_BYTES)
	if err != nil {
		shared.Debugf("failed to create hash to check")
		return false
	}
	if !bytes.Equal(hash, buff[PW_SALT_BYTES:]) {
		shared.Debugf("Bad password received")
		return false
	}
	shared.Debugf("Verified the admin password")
	return true
}

func certificatesGet(d *Daemon, r *http.Request) Response {
	recursion := d.isRecursionRequest(r)

	if recursion {
		certResponses := []shared.CertInfo{}

		baseCerts, err := dbCertsGet(d.db)
		if err != nil {
			return SmartError(err)
		}
		for _, baseCert := range baseCerts {
			resp := shared.CertInfo{}
			resp.Fingerprint = baseCert.Fingerprint
			resp.Certificate = baseCert.Certificate
			if baseCert.Type == 1 {
				resp.Type = "client"
			} else {
				resp.Type = "unknown"
			}
			certResponses = append(certResponses, resp)
		}
		return SyncResponse(true, certResponses)
	}

	body := []string{}
	for _, cert := range d.clientCerts {
		fingerprint := certGenerateFingerprint(&cert)
		body = append(body, fingerprint)
	}

	return SyncResponse(true, body)
}

type certificatesPostBody struct {
	Type        string `json:"type"`
	Certificate string `json:"certificate"`
	Name        string `json:"name"`
	Password    string `json:"password"`
}

func readSavedClientCAList(d *Daemon) {
	d.clientCerts = []x509.Certificate{}

	dbCerts, err := dbCertsGet(d.db)
	if err != nil {
		shared.Logf("Error reading certificates from database: %s\n", err)
		return
	}

	for _, dbCert := range dbCerts {
		certBlock, _ := pem.Decode([]byte(dbCert.Certificate))
		cert, err := x509.ParseCertificate(certBlock.Bytes)
		if err != nil {
			shared.Logf("Error reading certificate for %s: %s\n", dbCert.Name, err)
			continue
		}
		d.clientCerts = append(d.clientCerts, *cert)
	}
}

func saveCert(d *Daemon, host string, cert *x509.Certificate) error {

	baseCert := new(dbCertInfo)
	baseCert.Fingerprint = certGenerateFingerprint(cert)
	baseCert.Type = 1
	baseCert.Name = host
	baseCert.Certificate = string(
		pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: cert.Raw}),
	)

	return dbCertSave(d.db, baseCert)
}

func certificatesPost(d *Daemon, r *http.Request) Response {
	req := certificatesPostBody{}

	if err := shared.ReadToJSON(r.Body, &req); err != nil {
		return BadRequest(err)
	}

	if req.Type != "client" {
		return BadRequest(fmt.Errorf("Unknown request type %s", req.Type))
	}

	var cert *x509.Certificate
	var name string
	if req.Certificate != "" {

		data, err := base64.StdEncoding.DecodeString(req.Certificate)
		if err != nil {
			return BadRequest(err)
		}

		cert, err = x509.ParseCertificate(data)
		if err != nil {
			return BadRequest(err)
		}
		name = req.Name

	} else if r.TLS != nil {

		if len(r.TLS.PeerCertificates) < 1 {
			return BadRequest(fmt.Errorf("No client certificate provided"))
		}
		cert = r.TLS.PeerCertificates[len(r.TLS.PeerCertificates)-1]

		remoteHost, _, err := net.SplitHostPort(r.RemoteAddr)
		if err != nil {
			return InternalError(err)
		}

		name = remoteHost
	} else {
		return BadRequest(fmt.Errorf("Can't use TLS data on non-TLS link"))
	}

	fingerprint := certGenerateFingerprint(cert)
	for _, existingCert := range d.clientCerts {
		if fingerprint == certGenerateFingerprint(&existingCert) {
			return EmptySyncResponse
		}
	}

	if !d.isTrustedClient(r) && !d.verifyAdminPwd(req.Password) {
		return Forbidden
	}

	err := saveCert(d, name, cert)
	if err != nil {
		return SmartError(err)
	}

	d.clientCerts = append(d.clientCerts, *cert)

	return EmptySyncResponse
}

var certificatesCmd = Command{
	"certificates",
	false,
	true,
	certificatesGet,
	nil,
	certificatesPost,
	nil,
}

func certificateFingerprintGet(d *Daemon, r *http.Request) Response {
	fingerprint := mux.Vars(r)["fingerprint"]

	cert, err := doCertificateGet(d, fingerprint)
	if err != nil {
		return SmartError(err)
	}

	return SyncResponse(true, cert)
}

func doCertificateGet(d *Daemon, fingerprint string) (shared.CertInfo, error) {
	resp := shared.CertInfo{}

	dbCertInfo, err := dbCertGet(d.db, fingerprint)
	if err != nil {
		return resp, err
	}

	resp.Fingerprint = dbCertInfo.Fingerprint
	resp.Certificate = dbCertInfo.Certificate
	if dbCertInfo.Type == 1 {
		resp.Type = "client"
	} else {
		resp.Type = "unknown"
	}

	return resp, nil
}

func certificateFingerprintDelete(d *Daemon, r *http.Request) Response {
	fingerprint := mux.Vars(r)["fingerprint"]

	certInfo, err := dbCertGet(d.db, fingerprint)
	if err != nil {
		return NotFound
	}

	err = dbCertDelete(d.db, certInfo.Fingerprint)
	if err != nil {
		return SmartError(err)
	}
	readSavedClientCAList(d)

	return EmptySyncResponse
}

var certificateFingerprintCmd = Command{
	"certificates/{fingerprint}",
	false,
	false,
	certificateFingerprintGet,
	nil,
	nil,
	certificateFingerprintDelete,
}
<|endoftext|>


<commit_before>// Package pacer makes pacing and retrying API calls easy
package pacer

import (
	"math/rand"
	"sync"
	"time"

	"github.com/ncw/rclone/fs"
)

// Pacer state
type Pacer struct {
	mu                 sync.Mutex    // Protecting read/writes
	minSleep           time.Duration // minimum sleep time
	maxSleep           time.Duration // maximum sleep time
	decayConstant      uint          // decay constant
	attackConstant     uint          // attack constant
	pacer              chan struct{} // To pace the operations
	sleepTime          time.Duration // Time to sleep for each transaction
	retries            int           // Max number of retries
	maxConnections     int           // Maximum number of concurrent connections
	connTokens         chan struct{} // Connection tokens
	calculatePace      func(bool)    // switchable pacing algorithm - call with mu held
	consecutiveRetries int           // number of consecutive retries
}

// Type is for selecting different pacing algorithms
type Type int

const (
	// DefaultPacer is a truncated exponential attack and decay.
	//
	// On retries the sleep time is doubled, on non errors then
	// sleeptime decays according to the decay constant as set
	// with SetDecayConstant.
	//
	// The sleep never goes below that set with SetMinSleep or
	// above that set with SetMaxSleep.
	DefaultPacer = Type(iota)

	// AmazonCloudDrivePacer is a specialised pacer for Amazon Cloud Drive
	//
	// It implements a truncated exponential backoff strategy with
	// randomization.  Normally operations are paced at the
	// interval set with SetMinSleep.  On errors the sleep timer
	// is set to 0..2**retries seconds.
	//
	// See https://developer.amazon.com/public/apis/experience/cloud-drive/content/restful-api-best-practices
	AmazonCloudDrivePacer
)

// Paced is a function which is called by the Call and CallNoRetry
// methods.  It should return a boolean, true if it would like to be
// retried, and an error.  This error may be returned or returned
// wrapped in a RetryError.
type Paced func() (bool, error)

// New returns a Pacer with sensible defaults
func New() *Pacer {
	p := &Pacer{
		minSleep:       10 * time.Millisecond,
		maxSleep:       2 * time.Second,
		decayConstant:  2,
		attackConstant: 1,
		retries:        fs.Config.LowLevelRetries,
		pacer:          make(chan struct{}, 1),
	}
	p.sleepTime = p.minSleep
	p.SetPacer(DefaultPacer)
	p.SetMaxConnections(fs.Config.Checkers + fs.Config.Transfers)

	// Put the first pacing token in
	p.pacer <- struct{}{}

	return p
}

// SetMinSleep sets the minimum sleep time for the pacer
func (p *Pacer) SetMinSleep(t time.Duration) *Pacer {
	p.mu.Lock()
	defer p.mu.Unlock()
	p.minSleep = t
	p.sleepTime = p.minSleep
	return p
}

// SetMaxSleep sets the maximum sleep time for the pacer
func (p *Pacer) SetMaxSleep(t time.Duration) *Pacer {
	p.mu.Lock()
	defer p.mu.Unlock()
	p.maxSleep = t
	p.sleepTime = p.minSleep
	return p
}

// SetMaxConnections sets the maximum number of concurrent connections.
// Setting the value to 0 will allow unlimited number of connections.
// Should not be changed once you have started calling the pacer.
// By default this will be set to fs.Config.Checkers.
func (p *Pacer) SetMaxConnections(n int) *Pacer {
	p.mu.Lock()
	defer p.mu.Unlock()
	p.maxConnections = n
	if n <= 0 {
		p.connTokens = nil
	} else {
		p.connTokens = make(chan struct{}, n)
		for i := 0; i < n; i++ {
			p.connTokens <- struct{}{}
		}
	}
	return p
}

// SetDecayConstant sets the decay constant for the pacer
//
// This is the speed the time falls back to the minimum after errors
// have occurred.
//
// bigger for slower decay, exponential. 1 is halve, 0 is go straight to minimum
func (p *Pacer) SetDecayConstant(decay uint) *Pacer {
	p.mu.Lock()
	defer p.mu.Unlock()
	p.decayConstant = decay
	return p
}

// SetAttackConstant sets the attack constant for the pacer
//
// This is the speed the time grows from the minimum after errors have
// occurred.
//
// bigger for slower attack, 1 is double, 0 is go straight to maximum
func (p *Pacer) SetAttackConstant(attack uint) *Pacer {
	p.mu.Lock()
	defer p.mu.Unlock()
	p.attackConstant = attack
	return p
}

// SetRetries sets the max number of tries for Call
func (p *Pacer) SetRetries(retries int) *Pacer {
	p.mu.Lock()
	defer p.mu.Unlock()
	p.retries = retries
	return p
}

// SetPacer sets the pacing algorithm
//
// It will choose the default algorithm if an incorrect value is
// passed in.
func (p *Pacer) SetPacer(t Type) *Pacer {
	p.mu.Lock()
	defer p.mu.Unlock()
	switch t {
	case AmazonCloudDrivePacer:
		p.calculatePace = p.acdPacer
	default:
		p.calculatePace = p.defaultPacer
	}
	return p
}

// Start a call to the API
//
// This must be called as a pair with endCall
//
// This waits for the pacer token
func (p *Pacer) beginCall() {
	// pacer starts with a token in and whenever we take one out
	// XXX ms later we put another in.  We could do this with a
	// Ticker more accurately, but then we'd have to work out how
	// not to run it when it wasn't needed
	<-p.pacer
	if p.maxConnections > 0 {
		<-p.connTokens
	}

	p.mu.Lock()
	// Restart the timer
	go func(t time.Duration) {
		// fs.Debug(f, "New sleep for %v at %v", t, time.Now())
		time.Sleep(t)
		p.pacer <- struct{}{}
	}(p.sleepTime)
	p.mu.Unlock()
}

// exponentialImplementation implements a exponentialImplementation up
// and down pacing algorithm
//
// See the description for DefaultPacer
//
// This should calculate a new sleepTime.  It takes a boolean as to
// whether the operation should be retried or not.
//
// Call with p.mu held
func (p *Pacer) defaultPacer(retry bool) {
	oldSleepTime := p.sleepTime
	if retry {
		if p.attackConstant == 0 {
			p.sleepTime = p.maxSleep
		} else {
			p.sleepTime = (p.sleepTime << p.attackConstant) / ((1 << p.attackConstant) - 1)
		}
		if p.sleepTime > p.maxSleep {
			p.sleepTime = p.maxSleep
		}
		if p.sleepTime != oldSleepTime {
			fs.Debug("pacer", "Rate limited, increasing sleep to %v", p.sleepTime)
		}
	} else {
		p.sleepTime = (p.sleepTime<<p.decayConstant - p.sleepTime) >> p.decayConstant
		if p.sleepTime < p.minSleep {
			p.sleepTime = p.minSleep
		}
		if p.sleepTime != oldSleepTime {
			fs.Debug("pacer", "Reducing sleep to %v", p.sleepTime)
		}
	}
}

// acdPacer implements a truncated exponential backoff
// strategy with randomization for Amazon Cloud Drive
//
// See the description for AmazonCloudDrivePacer
//
// This should calculate a new sleepTime.  It takes a boolean as to
// whether the operation should be retried or not.
//
// Call with p.mu held
func (p *Pacer) acdPacer(retry bool) {
	consecutiveRetries := p.consecutiveRetries
	if consecutiveRetries == 0 {
		if p.sleepTime != p.minSleep {
			p.sleepTime = p.minSleep
			fs.Debug("pacer", "Resetting sleep to minimum %v on success", p.sleepTime)
		}
	} else {
		if consecutiveRetries > 9 {
			consecutiveRetries = 9
		}
		// consecutiveRetries starts at 1 so
		// maxSleep is 2**(consecutiveRetries-1) seconds
		maxSleep := time.Second << uint(consecutiveRetries-1)
		// actual sleep is random from 0..maxSleep
		p.sleepTime = time.Duration(rand.Int63n(int64(maxSleep)))
		if p.sleepTime < p.minSleep {
			p.sleepTime = p.minSleep
		}
		fs.Debug("pacer", "Rate limited, sleeping for %v (%d consecutive low level retries)", p.sleepTime, consecutiveRetries)
	}
}

// endCall implements the pacing algorithm
//
// This should calculate a new sleepTime.  It takes a boolean as to
// whether the operation should be retried or not.
func (p *Pacer) endCall(retry bool) {
	if p.maxConnections > 0 {
		p.connTokens <- struct{}{}
	}
	p.mu.Lock()
	if retry {
		p.consecutiveRetries++
	} else {
		p.consecutiveRetries = 0
	}
	p.calculatePace(retry)
	p.mu.Unlock()
}

// call implements Call but with settable retries
func (p *Pacer) call(fn Paced, retries int) (err error) {
	var retry bool
	for i := 1; i <= retries; i++ {
		p.beginCall()
		retry, err = fn()
		p.endCall(retry)
		if !retry {
			break
		}
		fs.Debug("pacer", "low level retry %d/%d", i, retries)
	}
	if retry {
		err = fs.RetryError(err)
	}
	return err
}

// Call paces the remote operations to not exceed the limits and retry
// on rate limit exceeded
//
// This calls fn, expecting it to return a retry flag and an
// error. This error may be returned wrapped in a RetryError if the
// number of retries is exceeded.
func (p *Pacer) Call(fn Paced) (err error) {
	p.mu.Lock()
	retries := p.retries
	p.mu.Unlock()
	return p.call(fn, retries)
}

// CallNoRetry paces the remote operations to not exceed the limits
// and return a retry error on rate limit exceeded
//
// This calls fn and wraps the output in a RetryError if it would like
// it to be retried
func (p *Pacer) CallNoRetry(fn Paced) error {
	return p.call(fn, 1)
}
<commit_msg>Debug cause of low level retries<commit_after>// Package pacer makes pacing and retrying API calls easy
package pacer

import (
	"math/rand"
	"sync"
	"time"

	"github.com/ncw/rclone/fs"
)

// Pacer state
type Pacer struct {
	mu                 sync.Mutex    // Protecting read/writes
	minSleep           time.Duration // minimum sleep time
	maxSleep           time.Duration // maximum sleep time
	decayConstant      uint          // decay constant
	attackConstant     uint          // attack constant
	pacer              chan struct{} // To pace the operations
	sleepTime          time.Duration // Time to sleep for each transaction
	retries            int           // Max number of retries
	maxConnections     int           // Maximum number of concurrent connections
	connTokens         chan struct{} // Connection tokens
	calculatePace      func(bool)    // switchable pacing algorithm - call with mu held
	consecutiveRetries int           // number of consecutive retries
}

// Type is for selecting different pacing algorithms
type Type int

const (
	// DefaultPacer is a truncated exponential attack and decay.
	//
	// On retries the sleep time is doubled, on non errors then
	// sleeptime decays according to the decay constant as set
	// with SetDecayConstant.
	//
	// The sleep never goes below that set with SetMinSleep or
	// above that set with SetMaxSleep.
	DefaultPacer = Type(iota)

	// AmazonCloudDrivePacer is a specialised pacer for Amazon Cloud Drive
	//
	// It implements a truncated exponential backoff strategy with
	// randomization.  Normally operations are paced at the
	// interval set with SetMinSleep.  On errors the sleep timer
	// is set to 0..2**retries seconds.
	//
	// See https://developer.amazon.com/public/apis/experience/cloud-drive/content/restful-api-best-practices
	AmazonCloudDrivePacer
)

// Paced is a function which is called by the Call and CallNoRetry
// methods.  It should return a boolean, true if it would like to be
// retried, and an error.  This error may be returned or returned
// wrapped in a RetryError.
type Paced func() (bool, error)

// New returns a Pacer with sensible defaults
func New() *Pacer {
	p := &Pacer{
		minSleep:       10 * time.Millisecond,
		maxSleep:       2 * time.Second,
		decayConstant:  2,
		attackConstant: 1,
		retries:        fs.Config.LowLevelRetries,
		pacer:          make(chan struct{}, 1),
	}
	p.sleepTime = p.minSleep
	p.SetPacer(DefaultPacer)
	p.SetMaxConnections(fs.Config.Checkers + fs.Config.Transfers)

	// Put the first pacing token in
	p.pacer <- struct{}{}

	return p
}

// SetMinSleep sets the minimum sleep time for the pacer
func (p *Pacer) SetMinSleep(t time.Duration) *Pacer {
	p.mu.Lock()
	defer p.mu.Unlock()
	p.minSleep = t
	p.sleepTime = p.minSleep
	return p
}

// SetMaxSleep sets the maximum sleep time for the pacer
func (p *Pacer) SetMaxSleep(t time.Duration) *Pacer {
	p.mu.Lock()
	defer p.mu.Unlock()
	p.maxSleep = t
	p.sleepTime = p.minSleep
	return p
}

// SetMaxConnections sets the maximum number of concurrent connections.
// Setting the value to 0 will allow unlimited number of connections.
// Should not be changed once you have started calling the pacer.
// By default this will be set to fs.Config.Checkers.
func (p *Pacer) SetMaxConnections(n int) *Pacer {
	p.mu.Lock()
	defer p.mu.Unlock()
	p.maxConnections = n
	if n <= 0 {
		p.connTokens = nil
	} else {
		p.connTokens = make(chan struct{}, n)
		for i := 0; i < n; i++ {
			p.connTokens <- struct{}{}
		}
	}
	return p
}

// SetDecayConstant sets the decay constant for the pacer
//
// This is the speed the time falls back to the minimum after errors
// have occurred.
//
// bigger for slower decay, exponential. 1 is halve, 0 is go straight to minimum
func (p *Pacer) SetDecayConstant(decay uint) *Pacer {
	p.mu.Lock()
	defer p.mu.Unlock()
	p.decayConstant = decay
	return p
}

// SetAttackConstant sets the attack constant for the pacer
//
// This is the speed the time grows from the minimum after errors have
// occurred.
//
// bigger for slower attack, 1 is double, 0 is go straight to maximum
func (p *Pacer) SetAttackConstant(attack uint) *Pacer {
	p.mu.Lock()
	defer p.mu.Unlock()
	p.attackConstant = attack
	return p
}

// SetRetries sets the max number of tries for Call
func (p *Pacer) SetRetries(retries int) *Pacer {
	p.mu.Lock()
	defer p.mu.Unlock()
	p.retries = retries
	return p
}

// SetPacer sets the pacing algorithm
//
// It will choose the default algorithm if an incorrect value is
// passed in.
func (p *Pacer) SetPacer(t Type) *Pacer {
	p.mu.Lock()
	defer p.mu.Unlock()
	switch t {
	case AmazonCloudDrivePacer:
		p.calculatePace = p.acdPacer
	default:
		p.calculatePace = p.defaultPacer
	}
	return p
}

// Start a call to the API
//
// This must be called as a pair with endCall
//
// This waits for the pacer token
func (p *Pacer) beginCall() {
	// pacer starts with a token in and whenever we take one out
	// XXX ms later we put another in.  We could do this with a
	// Ticker more accurately, but then we'd have to work out how
	// not to run it when it wasn't needed
	<-p.pacer
	if p.maxConnections > 0 {
		<-p.connTokens
	}

	p.mu.Lock()
	// Restart the timer
	go func(t time.Duration) {
		// fs.Debug(f, "New sleep for %v at %v", t, time.Now())
		time.Sleep(t)
		p.pacer <- struct{}{}
	}(p.sleepTime)
	p.mu.Unlock()
}

// exponentialImplementation implements a exponentialImplementation up
// and down pacing algorithm
//
// See the description for DefaultPacer
//
// This should calculate a new sleepTime.  It takes a boolean as to
// whether the operation should be retried or not.
//
// Call with p.mu held
func (p *Pacer) defaultPacer(retry bool) {
	oldSleepTime := p.sleepTime
	if retry {
		if p.attackConstant == 0 {
			p.sleepTime = p.maxSleep
		} else {
			p.sleepTime = (p.sleepTime << p.attackConstant) / ((1 << p.attackConstant) - 1)
		}
		if p.sleepTime > p.maxSleep {
			p.sleepTime = p.maxSleep
		}
		if p.sleepTime != oldSleepTime {
			fs.Debug("pacer", "Rate limited, increasing sleep to %v", p.sleepTime)
		}
	} else {
		p.sleepTime = (p.sleepTime<<p.decayConstant - p.sleepTime) >> p.decayConstant
		if p.sleepTime < p.minSleep {
			p.sleepTime = p.minSleep
		}
		if p.sleepTime != oldSleepTime {
			fs.Debug("pacer", "Reducing sleep to %v", p.sleepTime)
		}
	}
}

// acdPacer implements a truncated exponential backoff
// strategy with randomization for Amazon Cloud Drive
//
// See the description for AmazonCloudDrivePacer
//
// This should calculate a new sleepTime.  It takes a boolean as to
// whether the operation should be retried or not.
//
// Call with p.mu held
func (p *Pacer) acdPacer(retry bool) {
	consecutiveRetries := p.consecutiveRetries
	if consecutiveRetries == 0 {
		if p.sleepTime != p.minSleep {
			p.sleepTime = p.minSleep
			fs.Debug("pacer", "Resetting sleep to minimum %v on success", p.sleepTime)
		}
	} else {
		if consecutiveRetries > 9 {
			consecutiveRetries = 9
		}
		// consecutiveRetries starts at 1 so
		// maxSleep is 2**(consecutiveRetries-1) seconds
		maxSleep := time.Second << uint(consecutiveRetries-1)
		// actual sleep is random from 0..maxSleep
		p.sleepTime = time.Duration(rand.Int63n(int64(maxSleep)))
		if p.sleepTime < p.minSleep {
			p.sleepTime = p.minSleep
		}
		fs.Debug("pacer", "Rate limited, sleeping for %v (%d consecutive low level retries)", p.sleepTime, consecutiveRetries)
	}
}

// endCall implements the pacing algorithm
//
// This should calculate a new sleepTime.  It takes a boolean as to
// whether the operation should be retried or not.
func (p *Pacer) endCall(retry bool) {
	if p.maxConnections > 0 {
		p.connTokens <- struct{}{}
	}
	p.mu.Lock()
	if retry {
		p.consecutiveRetries++
	} else {
		p.consecutiveRetries = 0
	}
	p.calculatePace(retry)
	p.mu.Unlock()
}

// call implements Call but with settable retries
func (p *Pacer) call(fn Paced, retries int) (err error) {
	var retry bool
	for i := 1; i <= retries; i++ {
		p.beginCall()
		retry, err = fn()
		p.endCall(retry)
		if !retry {
			break
		}
		fs.Debug("pacer", "low level retry %d/%d (error %v)", i, retries, err)
	}
	if retry {
		err = fs.RetryError(err)
	}
	return err
}

// Call paces the remote operations to not exceed the limits and retry
// on rate limit exceeded
//
// This calls fn, expecting it to return a retry flag and an
// error. This error may be returned wrapped in a RetryError if the
// number of retries is exceeded.
func (p *Pacer) Call(fn Paced) (err error) {
	p.mu.Lock()
	retries := p.retries
	p.mu.Unlock()
	return p.call(fn, retries)
}

// CallNoRetry paces the remote operations to not exceed the limits
// and return a retry error on rate limit exceeded
//
// This calls fn and wraps the output in a RetryError if it would like
// it to be retried
func (p *Pacer) CallNoRetry(fn Paced) error {
	return p.call(fn, 1)
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"net/http"

	"github.com/gorilla/mux"

	"github.com/lxc/lxd/lxd/db"
	deviceConfig "github.com/lxc/lxd/lxd/device/config"
	"github.com/lxc/lxd/lxd/instance"
	"github.com/lxc/lxd/lxd/operations"
	projecthelpers "github.com/lxc/lxd/lxd/project"
	"github.com/lxc/lxd/lxd/response"
	"github.com/lxc/lxd/lxd/state"
	"github.com/lxc/lxd/lxd/util"
	"github.com/lxc/lxd/shared"
	"github.com/lxc/lxd/shared/api"
	"github.com/lxc/lxd/shared/osarch"
)

/*
 * Update configuration, or, if 'restore:snapshot-name' is present, restore
 * the named snapshot
 */
func instancePut(d *Daemon, r *http.Request) response.Response {
	instanceType, err := urlInstanceTypeDetect(r)
	if err != nil {
		return response.SmartError(err)
	}

	project := projectParam(r)

	// Get the container
	name := mux.Vars(r)["name"]

	// Handle requests targeted to a container on a different node
	resp, err := forwardedResponseIfInstanceIsRemote(d, r, project, name, instanceType)
	if err != nil {
		return response.SmartError(err)
	}
	if resp != nil {
		return resp
	}

	c, err := instance.LoadByProjectAndName(d.State(), project, name)
	if err != nil {
		return response.NotFound(err)
	}

	// Validate the ETag
	etag := []interface{}{c.Architecture(), c.LocalConfig(), c.LocalDevices(), c.IsEphemeral(), c.Profiles()}
	err = util.EtagCheck(r, etag)
	if err != nil {
		return response.PreconditionFailed(err)
	}

	configRaw := api.InstancePut{}
	if err := json.NewDecoder(r.Body).Decode(&configRaw); err != nil {
		return response.BadRequest(err)
	}

	architecture, err := osarch.ArchitectureId(configRaw.Architecture)
	if err != nil {
		architecture = 0
	}

	// Check project limits.
	err = d.cluster.Transaction(func(tx *db.ClusterTx) error {
		return projecthelpers.AllowInstanceUpdate(tx, project, name, configRaw, c.LocalConfig())
	})
	if err != nil {
		return response.SmartError(err)
	}

	var do func(*operations.Operation) error
	var opType db.OperationType
	if configRaw.Restore == "" {
		// Update container configuration
		do = func(op *operations.Operation) error {
			args := db.InstanceArgs{
				Architecture: architecture,
				Config:       configRaw.Config,
				Description:  configRaw.Description,
				Devices:      deviceConfig.NewDevices(configRaw.Devices),
				Ephemeral:    configRaw.Ephemeral,
				Profiles:     configRaw.Profiles,
				Project:      project,
			}

			err = c.Update(args, true)
			if err != nil {
				return err
			}

			return nil
		}

		opType = db.OperationInstanceUpdate
	} else {
		// Snapshot Restore
		do = func(op *operations.Operation) error {
			return instanceSnapRestore(d.State(), project, name, configRaw.Restore, configRaw.Stateful)
		}

		opType = db.OperationSnapshotRestore
	}

	resources := map[string][]string{}
	resources["containers"] = []string{name}

	op, err := operations.OperationCreate(d.State(), project, operations.OperationClassTask, opType, resources, nil, do, nil, nil)
	if err != nil {
		return response.InternalError(err)
	}

	return operations.OperationResponse(op)
}

func instanceSnapRestore(s *state.State, project, name, snap string, stateful bool) error {
	// normalize snapshot name
	if !shared.IsSnapshot(snap) {
		snap = name + shared.SnapshotDelimiter + snap
	}

	inst, err := instance.LoadByProjectAndName(s, project, name)
	if err != nil {
		return err
	}

	source, err := instance.LoadByProjectAndName(s, project, snap)
	if err != nil {
		switch err {
		case db.ErrNoSuchObject:
			return fmt.Errorf("Snapshot %s does not exist", snap)
		default:
			return err
		}
	}

	err = inst.Restore(source, stateful)
	if err != nil {
		return err
	}

	return nil
}
<commit_msg>lxd/instance/put: Renames c to inst and makes containers resources conditional on instance type<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"net/http"

	"github.com/gorilla/mux"

	"github.com/lxc/lxd/lxd/db"
	deviceConfig "github.com/lxc/lxd/lxd/device/config"
	"github.com/lxc/lxd/lxd/instance"
	"github.com/lxc/lxd/lxd/instance/instancetype"
	"github.com/lxc/lxd/lxd/operations"
	projecthelpers "github.com/lxc/lxd/lxd/project"
	"github.com/lxc/lxd/lxd/response"
	"github.com/lxc/lxd/lxd/state"
	"github.com/lxc/lxd/lxd/util"
	"github.com/lxc/lxd/shared"
	"github.com/lxc/lxd/shared/api"
	"github.com/lxc/lxd/shared/osarch"
)

/*
 * Update configuration, or, if 'restore:snapshot-name' is present, restore
 * the named snapshot
 */
func instancePut(d *Daemon, r *http.Request) response.Response {
	instanceType, err := urlInstanceTypeDetect(r)
	if err != nil {
		return response.SmartError(err)
	}

	project := projectParam(r)

	// Get the container
	name := mux.Vars(r)["name"]

	// Handle requests targeted to a container on a different node
	resp, err := forwardedResponseIfInstanceIsRemote(d, r, project, name, instanceType)
	if err != nil {
		return response.SmartError(err)
	}
	if resp != nil {
		return resp
	}

	inst, err := instance.LoadByProjectAndName(d.State(), project, name)
	if err != nil {
		return response.NotFound(err)
	}

	// Validate the ETag
	etag := []interface{}{inst.Architecture(), inst.LocalConfig(), inst.LocalDevices(), inst.IsEphemeral(), inst.Profiles()}
	err = util.EtagCheck(r, etag)
	if err != nil {
		return response.PreconditionFailed(err)
	}

	configRaw := api.InstancePut{}
	if err := json.NewDecoder(r.Body).Decode(&configRaw); err != nil {
		return response.BadRequest(err)
	}

	architecture, err := osarch.ArchitectureId(configRaw.Architecture)
	if err != nil {
		architecture = 0
	}

	// Check project limits.
	err = d.cluster.Transaction(func(tx *db.ClusterTx) error {
		return projecthelpers.AllowInstanceUpdate(tx, project, name, configRaw, inst.LocalConfig())
	})
	if err != nil {
		return response.SmartError(err)
	}

	var do func(*operations.Operation) error
	var opType db.OperationType
	if configRaw.Restore == "" {
		// Update container configuration
		do = func(op *operations.Operation) error {
			args := db.InstanceArgs{
				Architecture: architecture,
				Config:       configRaw.Config,
				Description:  configRaw.Description,
				Devices:      deviceConfig.NewDevices(configRaw.Devices),
				Ephemeral:    configRaw.Ephemeral,
				Profiles:     configRaw.Profiles,
				Project:      project,
			}

			err = inst.Update(args, true)
			if err != nil {
				return err
			}

			return nil
		}

		opType = db.OperationInstanceUpdate
	} else {
		// Snapshot Restore
		do = func(op *operations.Operation) error {
			return instanceSnapRestore(d.State(), project, name, configRaw.Restore, configRaw.Stateful)
		}

		opType = db.OperationSnapshotRestore
	}

	resources := map[string][]string{}
	resources["instances"] = []string{name}

	if inst.Type() == instancetype.Container {
		resources["containers"] = resources["instances"]
	}

	op, err := operations.OperationCreate(d.State(), project, operations.OperationClassTask, opType, resources, nil, do, nil, nil)
	if err != nil {
		return response.InternalError(err)
	}

	return operations.OperationResponse(op)
}

func instanceSnapRestore(s *state.State, project, name, snap string, stateful bool) error {
	// normalize snapshot name
	if !shared.IsSnapshot(snap) {
		snap = name + shared.SnapshotDelimiter + snap
	}

	inst, err := instance.LoadByProjectAndName(s, project, name)
	if err != nil {
		return err
	}

	source, err := instance.LoadByProjectAndName(s, project, snap)
	if err != nil {
		switch err {
		case db.ErrNoSuchObject:
			return fmt.Errorf("Snapshot %s does not exist", snap)
		default:
			return err
		}
	}

	err = inst.Restore(source, stateful)
	if err != nil {
		return err
	}

	return nil
}
<|endoftext|>