<commit_before>/*
    DNS-over-HTTPS
    Copyright (C) 2017 Star Brilliant <m13253@hotmail.com>

    Permission is hereby granted, free of charge, to any person obtaining a
    copy of this software and associated documentation files (the "Software"),
    to deal in the Software without restriction, including without limitation
    the rights to use, copy, modify, merge, publish, distribute, sublicense,
    and/or sell copies of the Software, and to permit persons to whom the
    Software is furnished to do so, subject to the following conditions:

    The above copyright notice and this permission notice shall be included in
    all copies or substantial portions of the Software.

    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
    DEALINGS IN THE SOFTWARE.
*/

package main

import (
	"context"
	"encoding/json"
	"fmt"
	"math/rand"
	"io/ioutil"
	"log"
	"net"
	"net/http"
	"net/url"
	"strconv"
	"strings"
	"time"
	"github.com/miekg/dns"
	"../json-dns"
)

type Client struct {
	addr			string
	upstream		string
	bootstraps		[]string
	timeout			uint
	noECS			bool
	verbose			bool
	udpServer		*dns.Server
	tcpServer		*dns.Server
	httpClient		*http.Client
}

func NewClient(addr, upstream string, bootstraps []string, timeout uint, noECS, verbose bool) (c *Client, err error) {
	c = &Client {
		addr: addr,
		upstream: upstream,
		bootstraps: bootstraps,
		timeout: timeout,
		noECS: noECS,
		verbose: verbose,
	}
	c.udpServer = &dns.Server {
		Addr: addr,
		Net: "udp",
		Handler: dns.HandlerFunc(c.udpHandlerFunc),
		UDPSize: 4096,
	}
	c.tcpServer = &dns.Server {
		Addr: addr,
		Net: "tcp",
		Handler: dns.HandlerFunc(c.tcpHandlerFunc),
	}
	bootResolver := net.DefaultResolver
	if len(c.bootstraps) != 0 {
		for i, bootstrap := range c.bootstraps {
			bootstrapAddr, err := net.ResolveUDPAddr("udp", bootstrap)
			if err != nil {
				bootstrapAddr, err = net.ResolveUDPAddr("udp", "[" + bootstrap + "]:53")
			}
			if err != nil { return nil, err }
			c.bootstraps[i] = bootstrapAddr.String()
		}
		bootResolver = &net.Resolver {
			PreferGo: true,
			Dial: func(ctx context.Context, network, address string) (net.Conn, error) {
				var d net.Dialer
				num_servers := len(c.bootstraps)
				bootstrap := c.bootstraps[rand.Intn(num_servers)]
				conn, err := d.DialContext(ctx, network, bootstrap)
				return conn, err
			},
		}
	}
	httpTransport := *http.DefaultTransport.(*http.Transport)
	httpTransport.DialContext = (&net.Dialer {
		Timeout: time.Duration(c.timeout) * time.Second,
		KeepAlive: 30 * time.Second,
		DualStack: true,
		Resolver: bootResolver,
	}).DialContext
	c.httpClient = &http.Client {
		Transport: &httpTransport,
	}
	return c, nil
}

func (c *Client) Start() error {
	result := make(chan error)
	go func() {
		err := c.udpServer.ListenAndServe()
		if err != nil {
			log.Println(err)
		}
		result <- err
	} ()
	go func() {
		err := c.tcpServer.ListenAndServe()
		if err != nil {
			log.Println(err)
		}
		result <- err
	} ()
	err := <-result
	if err != nil {
		return err
	}
	err = <-result
	return err
}

func (c *Client) handlerFunc(w dns.ResponseWriter, r *dns.Msg, isTCP bool) {
	if r.Response == true {
		log.Println("Received a response packet")
		return
	}

	reply := jsonDNS.PrepareReply(r)

	if len(r.Question) != 1 {
		log.Println("Number of questions is not 1")
		reply.Rcode = dns.RcodeFormatError
		w.WriteMsg(reply)
		return
	}
	question := r.Question[0]
	questionName := strings.ToLower(question.Name)
	questionType := ""
	if qtype, ok := dns.TypeToString[question.Qtype]; ok {
		questionType = qtype
	} else {
		questionType = strconv.Itoa(int(question.Qtype))
	}

	if c.verbose{
		fmt.Printf("%s - - [%s] \"%s IN %s\"\n", w.RemoteAddr(), time.Now().Format("02/Jan/2006:15:04:05 -0700"), questionName, questionType)
	}

	requestURL := fmt.Sprintf("%s?name=%s&type=%s", c.upstream, url.QueryEscape(questionName), url.QueryEscape(questionType))

	if r.CheckingDisabled {
		requestURL += "&cd=1"
	}

	udpSize := uint16(512)
	if opt := r.IsEdns0(); opt != nil {
		udpSize = opt.UDPSize()
	}

	ednsClientAddress, ednsClientNetmask := c.findClientIP(w, r)
	if ednsClientAddress != nil {
		requestURL += fmt.Sprintf("&edns_client_subnet=%s/%d", ednsClientAddress.String(), ednsClientNetmask)
	}

	req, err := http.NewRequest("GET", requestURL, nil)
	if err != nil {
		log.Println(err)
		reply.Rcode = dns.RcodeServerFailure
		w.WriteMsg(reply)
		return
	}
	req.Header.Set("User-Agent", "DNS-over-HTTPS/1.0 (+https://github.com/m13253/dns-over-https)")
	resp, err := c.httpClient.Do(req)
	if err != nil {
		log.Println(err)
		reply.Rcode = dns.RcodeServerFailure
		w.WriteMsg(reply)
		return
	}
	if resp.StatusCode != 200 {
		log.Printf("Server returned error: %s\n", resp.Status)
		reply.Rcode = dns.RcodeServerFailure
		w.WriteMsg(reply)
		return
	}

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		log.Println(err)
		reply.Rcode = dns.RcodeServerFailure
		w.WriteMsg(reply)
		return
	}

	var respJson jsonDNS.Response
	err = json.Unmarshal(body, &respJson)
	if err != nil {
		log.Println(err)
		reply.Rcode = dns.RcodeServerFailure
		w.WriteMsg(reply)
		return
	}

	fullReply := jsonDNS.Unmarshal(reply, &respJson, udpSize, ednsClientNetmask)
	buf, err := fullReply.Pack()
	if err != nil {
		log.Println(err)
		reply.Rcode = dns.RcodeServerFailure
		w.WriteMsg(reply)
		return
	}
	if !isTCP && len(buf) > int(udpSize) {
		fullReply.Truncated = true
		buf, err = fullReply.Pack()
		if err != nil {
			log.Println(err)
			return
		}
		buf = buf[:udpSize]
	}
	w.Write(buf)
}

func (c *Client) udpHandlerFunc(w dns.ResponseWriter, r *dns.Msg) {
	c.handlerFunc(w, r, false)
}

func (c *Client) tcpHandlerFunc(w dns.ResponseWriter, r *dns.Msg) {
	c.handlerFunc(w, r, true)
}

var (
	ipv4Mask24	net.IPMask = net.IPMask { 255, 255, 255, 0 }
	ipv6Mask48	net.IPMask = net.CIDRMask(48, 128)
)

func (c *Client) findClientIP(w dns.ResponseWriter, r *dns.Msg) (ednsClientAddress net.IP, ednsClientNetmask uint8) {
	ednsClientNetmask = 255
	if c.noECS {
		return net.IPv4(0, 0, 0, 0), 0
	}
	if opt := r.IsEdns0(); opt != nil {
		for _, option := range opt.Option {
			if option.Option() == dns.EDNS0SUBNET {
				edns0Subnet := option.(*dns.EDNS0_SUBNET)
				ednsClientAddress = edns0Subnet.Address
				ednsClientNetmask = edns0Subnet.SourceNetmask
				return
			}
		}
	}
	remoteAddr, err := net.ResolveUDPAddr("udp", w.RemoteAddr().String())
	if err != nil {
		return
	}
	if ip := remoteAddr.IP; jsonDNS.IsGlobalIP(ip) {
		if ipv4 := ip.To4(); ipv4 != nil {
			ednsClientAddress = ipv4.Mask(ipv4Mask24)
			ednsClientNetmask = 24
		} else {
			ednsClientAddress = ip.Mask(ipv6Mask48)
			ednsClientNetmask = 48
		}
	}
	return
}
<commit_msg>Enable Cookie, which might have better CDN cache performance<commit_after>/*
    DNS-over-HTTPS
    Copyright (C) 2017 Star Brilliant <m13253@hotmail.com>

    Permission is hereby granted, free of charge, to any person obtaining a
    copy of this software and associated documentation files (the "Software"),
    to deal in the Software without restriction, including without limitation
    the rights to use, copy, modify, merge, publish, distribute, sublicense,
    and/or sell copies of the Software, and to permit persons to whom the
    Software is furnished to do so, subject to the following conditions:

    The above copyright notice and this permission notice shall be included in
    all copies or substantial portions of the Software.

    THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
    IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
    FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
    AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
    LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
    FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
    DEALINGS IN THE SOFTWARE.
*/

package main

import (
	"context"
	"encoding/json"
	"fmt"
	"math/rand"
	"io/ioutil"
	"log"
	"net"
	"net/http"
	"net/http/cookiejar"
	"net/url"
	"strconv"
	"strings"
	"time"
	"github.com/miekg/dns"
	"../json-dns"
)

type Client struct {
	addr			string
	upstream		string
	bootstraps		[]string
	timeout			uint
	noECS			bool
	verbose			bool
	udpServer		*dns.Server
	tcpServer		*dns.Server
	httpClient		*http.Client
}

func NewClient(addr, upstream string, bootstraps []string, timeout uint, noECS, verbose bool) (c *Client, err error) {
	c = &Client {
		addr: addr,
		upstream: upstream,
		bootstraps: bootstraps,
		timeout: timeout,
		noECS: noECS,
		verbose: verbose,
	}
	c.udpServer = &dns.Server {
		Addr: addr,
		Net: "udp",
		Handler: dns.HandlerFunc(c.udpHandlerFunc),
		UDPSize: 4096,
	}
	c.tcpServer = &dns.Server {
		Addr: addr,
		Net: "tcp",
		Handler: dns.HandlerFunc(c.tcpHandlerFunc),
	}
	bootResolver := net.DefaultResolver
	if len(c.bootstraps) != 0 {
		for i, bootstrap := range c.bootstraps {
			bootstrapAddr, err := net.ResolveUDPAddr("udp", bootstrap)
			if err != nil {
				bootstrapAddr, err = net.ResolveUDPAddr("udp", "[" + bootstrap + "]:53")
			}
			if err != nil { return nil, err }
			c.bootstraps[i] = bootstrapAddr.String()
		}
		bootResolver = &net.Resolver {
			PreferGo: true,
			Dial: func(ctx context.Context, network, address string) (net.Conn, error) {
				var d net.Dialer
				num_servers := len(c.bootstraps)
				bootstrap := c.bootstraps[rand.Intn(num_servers)]
				conn, err := d.DialContext(ctx, network, bootstrap)
				return conn, err
			},
		}
	}
	httpTransport := *http.DefaultTransport.(*http.Transport)
	httpTransport.DialContext = (&net.Dialer {
		Timeout: time.Duration(c.timeout) * time.Second,
		KeepAlive: 30 * time.Second,
		DualStack: true,
		Resolver: bootResolver,
	}).DialContext
	cookieJar, err := cookiejar.New(nil)
	if err != nil { return nil, err }
	c.httpClient = &http.Client {
		Transport: &httpTransport,
		Jar: cookieJar,
	}
	return c, nil
}

func (c *Client) Start() error {
	result := make(chan error)
	go func() {
		err := c.udpServer.ListenAndServe()
		if err != nil {
			log.Println(err)
		}
		result <- err
	} ()
	go func() {
		err := c.tcpServer.ListenAndServe()
		if err != nil {
			log.Println(err)
		}
		result <- err
	} ()
	err := <-result
	if err != nil {
		return err
	}
	err = <-result
	return err
}

func (c *Client) handlerFunc(w dns.ResponseWriter, r *dns.Msg, isTCP bool) {
	if r.Response == true {
		log.Println("Received a response packet")
		return
	}

	reply := jsonDNS.PrepareReply(r)

	if len(r.Question) != 1 {
		log.Println("Number of questions is not 1")
		reply.Rcode = dns.RcodeFormatError
		w.WriteMsg(reply)
		return
	}
	question := r.Question[0]
	questionName := strings.ToLower(question.Name)
	questionType := ""
	if qtype, ok := dns.TypeToString[question.Qtype]; ok {
		questionType = qtype
	} else {
		questionType = strconv.Itoa(int(question.Qtype))
	}

	if c.verbose{
		fmt.Printf("%s - - [%s] \"%s IN %s\"\n", w.RemoteAddr(), time.Now().Format("02/Jan/2006:15:04:05 -0700"), questionName, questionType)
	}

	requestURL := fmt.Sprintf("%s?name=%s&type=%s", c.upstream, url.QueryEscape(questionName), url.QueryEscape(questionType))

	if r.CheckingDisabled {
		requestURL += "&cd=1"
	}

	udpSize := uint16(512)
	if opt := r.IsEdns0(); opt != nil {
		udpSize = opt.UDPSize()
	}

	ednsClientAddress, ednsClientNetmask := c.findClientIP(w, r)
	if ednsClientAddress != nil {
		requestURL += fmt.Sprintf("&edns_client_subnet=%s/%d", ednsClientAddress.String(), ednsClientNetmask)
	}

	req, err := http.NewRequest("GET", requestURL, nil)
	if err != nil {
		log.Println(err)
		reply.Rcode = dns.RcodeServerFailure
		w.WriteMsg(reply)
		return
	}
	req.Header.Set("User-Agent", "DNS-over-HTTPS/1.0 (+https://github.com/m13253/dns-over-https)")
	resp, err := c.httpClient.Do(req)
	if err != nil {
		log.Println(err)
		reply.Rcode = dns.RcodeServerFailure
		w.WriteMsg(reply)
		return
	}
	if resp.StatusCode != 200 {
		log.Printf("Server returned error: %s\n", resp.Status)
		reply.Rcode = dns.RcodeServerFailure
		w.WriteMsg(reply)
		return
	}

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		log.Println(err)
		reply.Rcode = dns.RcodeServerFailure
		w.WriteMsg(reply)
		return
	}

	var respJson jsonDNS.Response
	err = json.Unmarshal(body, &respJson)
	if err != nil {
		log.Println(err)
		reply.Rcode = dns.RcodeServerFailure
		w.WriteMsg(reply)
		return
	}

	fullReply := jsonDNS.Unmarshal(reply, &respJson, udpSize, ednsClientNetmask)
	buf, err := fullReply.Pack()
	if err != nil {
		log.Println(err)
		reply.Rcode = dns.RcodeServerFailure
		w.WriteMsg(reply)
		return
	}
	if !isTCP && len(buf) > int(udpSize) {
		fullReply.Truncated = true
		buf, err = fullReply.Pack()
		if err != nil {
			log.Println(err)
			return
		}
		buf = buf[:udpSize]
	}
	w.Write(buf)
}

func (c *Client) udpHandlerFunc(w dns.ResponseWriter, r *dns.Msg) {
	c.handlerFunc(w, r, false)
}

func (c *Client) tcpHandlerFunc(w dns.ResponseWriter, r *dns.Msg) {
	c.handlerFunc(w, r, true)
}

var (
	ipv4Mask24	net.IPMask = net.IPMask { 255, 255, 255, 0 }
	ipv6Mask48	net.IPMask = net.CIDRMask(48, 128)
)

func (c *Client) findClientIP(w dns.ResponseWriter, r *dns.Msg) (ednsClientAddress net.IP, ednsClientNetmask uint8) {
	ednsClientNetmask = 255
	if c.noECS {
		return net.IPv4(0, 0, 0, 0), 0
	}
	if opt := r.IsEdns0(); opt != nil {
		for _, option := range opt.Option {
			if option.Option() == dns.EDNS0SUBNET {
				edns0Subnet := option.(*dns.EDNS0_SUBNET)
				ednsClientAddress = edns0Subnet.Address
				ednsClientNetmask = edns0Subnet.SourceNetmask
				return
			}
		}
	}
	remoteAddr, err := net.ResolveUDPAddr("udp", w.RemoteAddr().String())
	if err != nil {
		return
	}
	if ip := remoteAddr.IP; jsonDNS.IsGlobalIP(ip) {
		if ipv4 := ip.To4(); ipv4 != nil {
			ednsClientAddress = ipv4.Mask(ipv4Mask24)
			ednsClientNetmask = 24
		} else {
			ednsClientAddress = ip.Mask(ipv6Mask48)
			ednsClientNetmask = 48
		}
	}
	return
}
<|endoftext|>


<commit_before>package consensus

import (
	"github.com/NebulousLabs/Sia/build"
	"github.com/NebulousLabs/Sia/modules"

	siasync "github.com/NebulousLabs/Sia/sync"
	"github.com/coreos/bbolt"
)

// computeConsensusChange computes the consensus change from the change entry
// at index 'i' in the change log. If i is out of bounds, an error is returned.
func (cs *ConsensusSet) computeConsensusChange(tx *bolt.Tx, ce changeEntry) (modules.ConsensusChange, error) {
	cc := modules.ConsensusChange{
		ID: ce.ID(),
	}
	for _, revertedBlockID := range ce.RevertedBlocks {
		revertedBlock, err := getBlockMap(tx, revertedBlockID)
		if err != nil {
			cs.log.Critical("getBlockMap failed in computeConsensusChange:", err)
			return modules.ConsensusChange{}, err
		}

		// Because the direction is 'revert', the order of the diffs needs to
		// be flipped and the direction of the diffs also needs to be flipped.
		cc.RevertedBlocks = append(cc.RevertedBlocks, revertedBlock.Block)
		for i := len(revertedBlock.SiacoinOutputDiffs) - 1; i >= 0; i-- {
			scod := revertedBlock.SiacoinOutputDiffs[i]
			scod.Direction = !scod.Direction
			cc.SiacoinOutputDiffs = append(cc.SiacoinOutputDiffs, scod)
		}
		for i := len(revertedBlock.FileContractDiffs) - 1; i >= 0; i-- {
			fcd := revertedBlock.FileContractDiffs[i]
			fcd.Direction = !fcd.Direction
			cc.FileContractDiffs = append(cc.FileContractDiffs, fcd)
		}
		for i := len(revertedBlock.SiafundOutputDiffs) - 1; i >= 0; i-- {
			sfod := revertedBlock.SiafundOutputDiffs[i]
			sfod.Direction = !sfod.Direction
			cc.SiafundOutputDiffs = append(cc.SiafundOutputDiffs, sfod)
		}
		for i := len(revertedBlock.DelayedSiacoinOutputDiffs) - 1; i >= 0; i-- {
			dscod := revertedBlock.DelayedSiacoinOutputDiffs[i]
			dscod.Direction = !dscod.Direction
			cc.DelayedSiacoinOutputDiffs = append(cc.DelayedSiacoinOutputDiffs, dscod)
		}
		for i := len(revertedBlock.SiafundPoolDiffs) - 1; i >= 0; i-- {
			sfpd := revertedBlock.SiafundPoolDiffs[i]
			sfpd.Direction = modules.DiffRevert
			cc.SiafundPoolDiffs = append(cc.SiafundPoolDiffs, sfpd)
		}
	}
	for _, appliedBlockID := range ce.AppliedBlocks {
		appliedBlock, err := getBlockMap(tx, appliedBlockID)
		if err != nil {
			cs.log.Critical("getBlockMap failed in computeConsensusChange:", err)
			return modules.ConsensusChange{}, err
		}

		cc.AppliedBlocks = append(cc.AppliedBlocks, appliedBlock.Block)
		for _, scod := range appliedBlock.SiacoinOutputDiffs {
			cc.SiacoinOutputDiffs = append(cc.SiacoinOutputDiffs, scod)
		}
		for _, fcd := range appliedBlock.FileContractDiffs {
			cc.FileContractDiffs = append(cc.FileContractDiffs, fcd)
		}
		for _, sfod := range appliedBlock.SiafundOutputDiffs {
			cc.SiafundOutputDiffs = append(cc.SiafundOutputDiffs, sfod)
		}
		for _, dscod := range appliedBlock.DelayedSiacoinOutputDiffs {
			cc.DelayedSiacoinOutputDiffs = append(cc.DelayedSiacoinOutputDiffs, dscod)
		}
		for _, sfpd := range appliedBlock.SiafundPoolDiffs {
			cc.SiafundPoolDiffs = append(cc.SiafundPoolDiffs, sfpd)
		}
	}

	// Grab the child target and the minimum valid child timestamp.
	recentBlock := ce.AppliedBlocks[len(ce.AppliedBlocks)-1]
	pb, err := getBlockMap(tx, recentBlock)
	if err != nil {
		cs.log.Critical("could not find process block for known block")
	}
	cc.ChildTarget = pb.ChildTarget
	cc.MinimumValidChildTimestamp = cs.blockRuleHelper.minimumValidChildTimestamp(tx.Bucket(BlockMap), pb)

	currentBlock := currentBlockID(tx)
	if cs.synced && recentBlock == currentBlock {
		cc.Synced = true
	}

	// Add the unexported tryTransactionSet function.
	cc.TryTransactionSet = cs.tryTransactionSet

	return cc, nil
}

// updateSubscribers will inform all subscribers of a new update to the
// consensus set. updateSubscribers does not alter the changelog, the changelog
// must be updated beforehand.
func (cs *ConsensusSet) updateSubscribers(ce changeEntry) {
	if len(cs.subscribers) == 0 {
		return
	}
	// Get the consensus change and send it to all subscribers.
	var cc modules.ConsensusChange
	err := cs.db.View(func(tx *bolt.Tx) error {
		// Compute the consensus change so it can be sent to subscribers.
		var err error
		cc, err = cs.computeConsensusChange(tx, ce)
		return err
	})
	if err != nil {
		cs.log.Critical("computeConsensusChange failed:", err)
		return
	}
	for _, subscriber := range cs.subscribers {
		subscriber.ProcessConsensusChange(cc)
	}
}

// managedInitializeSubscribe will take a subscriber and feed them all of the
// consensus changes that have occurred since the change provided.
//
// As a special case, using an empty id as the start will have all the changes
// sent to the modules starting with the genesis block.
func (cs *ConsensusSet) managedInitializeSubscribe(subscriber modules.ConsensusSetSubscriber, start modules.ConsensusChangeID,
	cancel <-chan struct{}) (modules.ConsensusChangeID, error) {

	if start == modules.ConsensusChangeRecent {
		return start, nil
	}

	// 'exists' and 'entry' are going to be pointed to the first entry that
	// has not yet been seen by subscriber.
	var exists bool
	var entry changeEntry
	cs.mu.RLock()
	err := cs.db.View(func(tx *bolt.Tx) error {
		if start == modules.ConsensusChangeBeginning {
			// Special case: for modules.ConsensusChangeBeginning, create an
			// initial node pointing to the genesis block. The subscriber will
			// receive the diffs for all blocks in the consensus set, including
			// the genesis block.
			entry = cs.genesisEntry()
			exists = true
		} else {
			// The subscriber has provided an existing consensus change.
			// Because the subscriber already has this consensus change,
			// 'entry' and 'exists' need to be pointed at the next consensus
			// change.
			entry, exists = getEntry(tx, start)
			if !exists {
				// modules.ErrInvalidConsensusChangeID is a named error that
				// signals a break in synchronization between the consensus set
				// persistence and the subscriber persistence. Typically,
				// receiving this error means that the subscriber needs to
				// perform a rescan of the consensus set.
				return modules.ErrInvalidConsensusChangeID
			}
			entry, exists = entry.NextEntry(tx)
		}
		return nil
	})
	cs.mu.RUnlock()
	if err != nil {
		return modules.ConsensusChangeID{}, err
	}

	// Nothing to do if the changeEntry doesn't exist.
	if !exists {
		return start, nil
	}

	// Send all remaining consensus changes to the subscriber.
	var latestChangeID modules.ConsensusChangeID
	for exists {
		// Send changes in batches of 100 so that we don't hold the
		// lock for too long.
		cs.mu.RLock()
		err = cs.db.View(func(tx *bolt.Tx) error {
			for i := 0; i < 100 && exists; i++ {
				latestChangeID = entry.ID()
				select {
				case <-cancel:
					return siasync.ErrStopped
				default:
				}
				cc, err := cs.computeConsensusChange(tx, entry)
				if err != nil {
					return err
				}
				subscriber.ProcessConsensusChange(cc)
				entry, exists = entry.NextEntry(tx)
			}
			return nil
		})
		cs.mu.RUnlock()
		if err != nil {
			return modules.ConsensusChangeID{}, err
		}
	}
	return latestChangeID, nil
}

// recentConsensusChangeID gets the ConsensusChangeID of the most recent
// change.
func (cs *ConsensusSet) recentConsensusChangeID() (cid modules.ConsensusChangeID, err error) {
	err = cs.db.View(func(tx *bolt.Tx) error {
		cl := tx.Bucket(ChangeLog)
		copy(cid[:], cl.Get(ChangeLogTailID))
		return nil
	})
	return
}

// ConsensusSetSubscribe adds a subscriber to the list of subscribers, and
// gives them every consensus change that has occurred since the change with
// the provided id.
//
// As a special case, using an empty id as the start will have all the changes
// sent to the modules starting with the genesis block.
func (cs *ConsensusSet) ConsensusSetSubscribe(subscriber modules.ConsensusSetSubscriber, start modules.ConsensusChangeID,
	cancel <-chan struct{}) error {

	err := cs.tg.Add()
	if err != nil {
		return err
	}
	defer cs.tg.Done()

	// Call managedInitializeSubscribe until the new module is up-to-date.
	cs.mu.Lock()
	for {
		// Check if the start equals the most recent change id. If it does we
		// are done.
		recentID, err := cs.recentConsensusChangeID()
		if err != nil {
			cs.mu.Unlock()
			return err
		}
		if start == recentID {
			break
		}
		// If it doesn't, we need to call managedInitializeSubscribe again.
		cs.mu.Unlock()
		start, err = cs.managedInitializeSubscribe(subscriber, start, cancel)
		if err != nil {
			return err
		}
		// Check for shutdown.
		select {
		case <-cs.tg.StopChan():
			return siasync.ErrStopped
		default:
		}
		cs.mu.Lock()
	}

	// Add the module to the list of subscribers.
	// Sanity check - subscriber should not be already subscribed.
	for _, s := range cs.subscribers {
		if s == subscriber {
			build.Critical("refusing to double-subscribe subscriber")
		}
	}
	cs.subscribers = append(cs.subscribers, subscriber)
	cs.mu.Unlock()
	return nil
}

// Unsubscribe removes a subscriber from the list of subscribers, allowing for
// garbage collection and rescanning. If the subscriber is not found in the
// subscriber database, no action is taken.
func (cs *ConsensusSet) Unsubscribe(subscriber modules.ConsensusSetSubscriber) {
	if cs.tg.Add() != nil {
		return
	}
	defer cs.tg.Done()
	cs.mu.Lock()
	defer cs.mu.Unlock()

	// Search for the subscriber in the list of subscribers and remove it if
	// found.
	for i := range cs.subscribers {
		if cs.subscribers[i] == subscriber {
			// nil the subscriber entry (otherwise it will not be GC'd if it's
			// at the end of the subscribers slice).
			cs.subscribers[i] = nil
			// Delete the entry from the slice.
			cs.subscribers = append(cs.subscribers[0:i], cs.subscribers[i+1:]...)
			break
		}
	}
}
<commit_msg>add comment<commit_after>package consensus

import (
	"github.com/NebulousLabs/Sia/build"
	"github.com/NebulousLabs/Sia/modules"

	siasync "github.com/NebulousLabs/Sia/sync"
	"github.com/coreos/bbolt"
)

// computeConsensusChange computes the consensus change from the change entry
// at index 'i' in the change log. If i is out of bounds, an error is returned.
func (cs *ConsensusSet) computeConsensusChange(tx *bolt.Tx, ce changeEntry) (modules.ConsensusChange, error) {
	cc := modules.ConsensusChange{
		ID: ce.ID(),
	}
	for _, revertedBlockID := range ce.RevertedBlocks {
		revertedBlock, err := getBlockMap(tx, revertedBlockID)
		if err != nil {
			cs.log.Critical("getBlockMap failed in computeConsensusChange:", err)
			return modules.ConsensusChange{}, err
		}

		// Because the direction is 'revert', the order of the diffs needs to
		// be flipped and the direction of the diffs also needs to be flipped.
		cc.RevertedBlocks = append(cc.RevertedBlocks, revertedBlock.Block)
		for i := len(revertedBlock.SiacoinOutputDiffs) - 1; i >= 0; i-- {
			scod := revertedBlock.SiacoinOutputDiffs[i]
			scod.Direction = !scod.Direction
			cc.SiacoinOutputDiffs = append(cc.SiacoinOutputDiffs, scod)
		}
		for i := len(revertedBlock.FileContractDiffs) - 1; i >= 0; i-- {
			fcd := revertedBlock.FileContractDiffs[i]
			fcd.Direction = !fcd.Direction
			cc.FileContractDiffs = append(cc.FileContractDiffs, fcd)
		}
		for i := len(revertedBlock.SiafundOutputDiffs) - 1; i >= 0; i-- {
			sfod := revertedBlock.SiafundOutputDiffs[i]
			sfod.Direction = !sfod.Direction
			cc.SiafundOutputDiffs = append(cc.SiafundOutputDiffs, sfod)
		}
		for i := len(revertedBlock.DelayedSiacoinOutputDiffs) - 1; i >= 0; i-- {
			dscod := revertedBlock.DelayedSiacoinOutputDiffs[i]
			dscod.Direction = !dscod.Direction
			cc.DelayedSiacoinOutputDiffs = append(cc.DelayedSiacoinOutputDiffs, dscod)
		}
		for i := len(revertedBlock.SiafundPoolDiffs) - 1; i >= 0; i-- {
			sfpd := revertedBlock.SiafundPoolDiffs[i]
			sfpd.Direction = modules.DiffRevert
			cc.SiafundPoolDiffs = append(cc.SiafundPoolDiffs, sfpd)
		}
	}
	for _, appliedBlockID := range ce.AppliedBlocks {
		appliedBlock, err := getBlockMap(tx, appliedBlockID)
		if err != nil {
			cs.log.Critical("getBlockMap failed in computeConsensusChange:", err)
			return modules.ConsensusChange{}, err
		}

		cc.AppliedBlocks = append(cc.AppliedBlocks, appliedBlock.Block)
		for _, scod := range appliedBlock.SiacoinOutputDiffs {
			cc.SiacoinOutputDiffs = append(cc.SiacoinOutputDiffs, scod)
		}
		for _, fcd := range appliedBlock.FileContractDiffs {
			cc.FileContractDiffs = append(cc.FileContractDiffs, fcd)
		}
		for _, sfod := range appliedBlock.SiafundOutputDiffs {
			cc.SiafundOutputDiffs = append(cc.SiafundOutputDiffs, sfod)
		}
		for _, dscod := range appliedBlock.DelayedSiacoinOutputDiffs {
			cc.DelayedSiacoinOutputDiffs = append(cc.DelayedSiacoinOutputDiffs, dscod)
		}
		for _, sfpd := range appliedBlock.SiafundPoolDiffs {
			cc.SiafundPoolDiffs = append(cc.SiafundPoolDiffs, sfpd)
		}
	}

	// Grab the child target and the minimum valid child timestamp.
	recentBlock := ce.AppliedBlocks[len(ce.AppliedBlocks)-1]
	pb, err := getBlockMap(tx, recentBlock)
	if err != nil {
		cs.log.Critical("could not find process block for known block")
	}
	cc.ChildTarget = pb.ChildTarget
	cc.MinimumValidChildTimestamp = cs.blockRuleHelper.minimumValidChildTimestamp(tx.Bucket(BlockMap), pb)

	currentBlock := currentBlockID(tx)
	if cs.synced && recentBlock == currentBlock {
		cc.Synced = true
	}

	// Add the unexported tryTransactionSet function.
	cc.TryTransactionSet = cs.tryTransactionSet

	return cc, nil
}

// updateSubscribers will inform all subscribers of a new update to the
// consensus set. updateSubscribers does not alter the changelog, the changelog
// must be updated beforehand.
func (cs *ConsensusSet) updateSubscribers(ce changeEntry) {
	if len(cs.subscribers) == 0 {
		return
	}
	// Get the consensus change and send it to all subscribers.
	var cc modules.ConsensusChange
	err := cs.db.View(func(tx *bolt.Tx) error {
		// Compute the consensus change so it can be sent to subscribers.
		var err error
		cc, err = cs.computeConsensusChange(tx, ce)
		return err
	})
	if err != nil {
		cs.log.Critical("computeConsensusChange failed:", err)
		return
	}
	for _, subscriber := range cs.subscribers {
		subscriber.ProcessConsensusChange(cc)
	}
}

// managedInitializeSubscribe will take a subscriber and feed them all of the
// consensus changes that have occurred since the change provided.
//
// As a special case, using an empty id as the start will have all the changes
// sent to the modules starting with the genesis block.
func (cs *ConsensusSet) managedInitializeSubscribe(subscriber modules.ConsensusSetSubscriber, start modules.ConsensusChangeID,
	cancel <-chan struct{}) (modules.ConsensusChangeID, error) {

	if start == modules.ConsensusChangeRecent {
		return start, nil
	}

	// 'exists' and 'entry' are going to be pointed to the first entry that
	// has not yet been seen by subscriber.
	var exists bool
	var entry changeEntry
	cs.mu.RLock()
	err := cs.db.View(func(tx *bolt.Tx) error {
		if start == modules.ConsensusChangeBeginning {
			// Special case: for modules.ConsensusChangeBeginning, create an
			// initial node pointing to the genesis block. The subscriber will
			// receive the diffs for all blocks in the consensus set, including
			// the genesis block.
			entry = cs.genesisEntry()
			exists = true
		} else {
			// The subscriber has provided an existing consensus change.
			// Because the subscriber already has this consensus change,
			// 'entry' and 'exists' need to be pointed at the next consensus
			// change.
			entry, exists = getEntry(tx, start)
			if !exists {
				// modules.ErrInvalidConsensusChangeID is a named error that
				// signals a break in synchronization between the consensus set
				// persistence and the subscriber persistence. Typically,
				// receiving this error means that the subscriber needs to
				// perform a rescan of the consensus set.
				return modules.ErrInvalidConsensusChangeID
			}
			entry, exists = entry.NextEntry(tx)
		}
		return nil
	})
	cs.mu.RUnlock()
	if err != nil {
		return modules.ConsensusChangeID{}, err
	}

	// Nothing to do if the changeEntry doesn't exist.
	if !exists {
		return start, nil
	}

	// Send all remaining consensus changes to the subscriber.
	var latestChangeID modules.ConsensusChangeID
	for exists {
		// Send changes in batches of 100 so that we don't hold the
		// lock for too long.
		cs.mu.RLock()
		err = cs.db.View(func(tx *bolt.Tx) error {
			for i := 0; i < 100 && exists; i++ {
				latestChangeID = entry.ID()
				select {
				case <-cancel:
					return siasync.ErrStopped
				default:
				}
				cc, err := cs.computeConsensusChange(tx, entry)
				if err != nil {
					return err
				}
				subscriber.ProcessConsensusChange(cc)
				entry, exists = entry.NextEntry(tx)
			}
			return nil
		})
		cs.mu.RUnlock()
		if err != nil {
			return modules.ConsensusChangeID{}, err
		}
	}
	return latestChangeID, nil
}

// recentConsensusChangeID gets the ConsensusChangeID of the most recent
// change.
func (cs *ConsensusSet) recentConsensusChangeID() (cid modules.ConsensusChangeID, err error) {
	err = cs.db.View(func(tx *bolt.Tx) error {
		cl := tx.Bucket(ChangeLog)
		copy(cid[:], cl.Get(ChangeLogTailID))
		return nil
	})
	return
}

// ConsensusSetSubscribe adds a subscriber to the list of subscribers, and
// gives them every consensus change that has occurred since the change with
// the provided id.
//
// As a special case, using an empty id as the start will have all the changes
// sent to the modules starting with the genesis block.
func (cs *ConsensusSet) ConsensusSetSubscribe(subscriber modules.ConsensusSetSubscriber, start modules.ConsensusChangeID,
	cancel <-chan struct{}) error {

	err := cs.tg.Add()
	if err != nil {
		return err
	}
	defer cs.tg.Done()

	// Call managedInitializeSubscribe until the new module is up-to-date.
	for {
		start, err = cs.managedInitializeSubscribe(subscriber, start, cancel)
		if err != nil {
			return err
		}
		// Check if the start equals the most recent change id. If it does we
		// are done. If it doesn't, we need to call managedInitializeSubscribe
		// again.
		cs.mu.Lock()
		recentID, err := cs.recentConsensusChangeID()
		if err != nil {
			cs.mu.Unlock()
			return err
		}
		if start == recentID {
			// break out of the loop while still holding to lock to avoid
			// updating subscribers before the new module is appended to the
			// list of subscribers.
			break
		}
		cs.mu.Unlock()

		// Check for shutdown.
		select {
		case <-cs.tg.StopChan():
			return siasync.ErrStopped
		default:
		}
	}

	// Add the module to the list of subscribers.
	// Sanity check - subscriber should not be already subscribed.
	for _, s := range cs.subscribers {
		if s == subscriber {
			build.Critical("refusing to double-subscribe subscriber")
		}
	}
	cs.subscribers = append(cs.subscribers, subscriber)
	cs.mu.Unlock()
	return nil
}

// Unsubscribe removes a subscriber from the list of subscribers, allowing for
// garbage collection and rescanning. If the subscriber is not found in the
// subscriber database, no action is taken.
func (cs *ConsensusSet) Unsubscribe(subscriber modules.ConsensusSetSubscriber) {
	if cs.tg.Add() != nil {
		return
	}
	defer cs.tg.Done()
	cs.mu.Lock()
	defer cs.mu.Unlock()

	// Search for the subscriber in the list of subscribers and remove it if
	// found.
	for i := range cs.subscribers {
		if cs.subscribers[i] == subscriber {
			// nil the subscriber entry (otherwise it will not be GC'd if it's
			// at the end of the subscribers slice).
			cs.subscribers[i] = nil
			// Delete the entry from the slice.
			cs.subscribers = append(cs.subscribers[0:i], cs.subscribers[i+1:]...)
			break
		}
	}
}
<|endoftext|>


<commit_before><commit_msg>rename code chain<commit_after><|endoftext|>


<commit_before>package main

import (
	"bytes"
	"crypto/rand"
	"crypto/rsa"
	"encoding/json"
	"fmt"
	"net/http"
	"net/http/httptest"
	"testing"

	"github.com/nsheridan/cashier/lib"
	"github.com/nsheridan/cashier/testdata"

	"golang.org/x/crypto/ssh"
	"golang.org/x/crypto/ssh/agent"
)

func TestLoadCert(t *testing.T) {
	priv, _ := ssh.ParseRawPrivateKey(testdata.Priv)
	key := priv.(*rsa.PrivateKey)
	pub, _ := ssh.NewPublicKey(&key.PublicKey)
	c := &ssh.Certificate{
		Key:         pub,
		CertType:    ssh.UserCert,
		ValidBefore: ssh.CertTimeInfinity,
		ValidAfter:  0,
	}
	signer, err := ssh.NewSignerFromKey(key)
	if err != nil {
		t.Fatal(err)
	}
	c.SignCert(rand.Reader, signer)
	a := agent.NewKeyring()
	if err := installCert(a, c, key); err != nil {
		t.Fatal(err)
	}
	listedKeys, err := a.List()
	if err != nil {
		t.Fatalf("Error reading from agent: %v", err)
	}
	if len(listedKeys) != 1 {
		t.Fatalf("Expected 1 key, got %d", len(listedKeys))
	}
	if !bytes.Equal(listedKeys[0].Marshal(), c.Marshal()) {
		t.Fatal("Certs not equal")
	}
}

func TestSignGood(t *testing.T) {
	res := &lib.SignResponse{
		Status:   "ok",
		Response: string(testdata.Cert),
	}
	j, _ := json.Marshal(res)
	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		fmt.Fprintln(w, string(j))
	}))
	defer ts.Close()
	_, err := send([]byte(`{}`), "token", ts.URL)
	if err != nil {
		t.Fatal(err)
	}
	k, _, _, _, err := ssh.ParseAuthorizedKey(testdata.Pub)
	if err != nil {
		t.Fatal(err)
	}
	c := &config{
		CA:       ts.URL,
		Validity: "24h",
	}
	cert, err := sign(k, "token", c)
	if cert == nil && err != nil {
		t.Fatal(err)
	}
}

func TestSignBad(t *testing.T) {
	res := &lib.SignResponse{
		Status:   "error",
		Response: `{"response": "error"}`,
	}
	j, _ := json.Marshal(res)
	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		fmt.Fprintln(w, string(j))
	}))
	defer ts.Close()
	_, err := send([]byte(`{}`), "token", ts.URL)
	if err != nil {
		t.Fatal(err)
	}
	k, _, _, _, err := ssh.ParseAuthorizedKey(testdata.Pub)
	if err != nil {
		t.Fatal(err)
	}
	c := &config{
		CA:       ts.URL,
		Validity: "24h",
	}
	cert, err := sign(k, "token", c)
	if cert != nil && err == nil {
		t.Fatal(err)
	}
}
<commit_msg>fix client_test.go invocations of send method<commit_after>package main

import (
	"bytes"
	"crypto/rand"
	"crypto/rsa"
	"encoding/json"
	"fmt"
	"net/http"
	"net/http/httptest"
	"testing"

	"github.com/nsheridan/cashier/lib"
	"github.com/nsheridan/cashier/testdata"

	"golang.org/x/crypto/ssh"
	"golang.org/x/crypto/ssh/agent"
)

func TestLoadCert(t *testing.T) {
	priv, _ := ssh.ParseRawPrivateKey(testdata.Priv)
	key := priv.(*rsa.PrivateKey)
	pub, _ := ssh.NewPublicKey(&key.PublicKey)
	c := &ssh.Certificate{
		Key:         pub,
		CertType:    ssh.UserCert,
		ValidBefore: ssh.CertTimeInfinity,
		ValidAfter:  0,
	}
	signer, err := ssh.NewSignerFromKey(key)
	if err != nil {
		t.Fatal(err)
	}
	c.SignCert(rand.Reader, signer)
	a := agent.NewKeyring()
	if err := installCert(a, c, key); err != nil {
		t.Fatal(err)
	}
	listedKeys, err := a.List()
	if err != nil {
		t.Fatalf("Error reading from agent: %v", err)
	}
	if len(listedKeys) != 1 {
		t.Fatalf("Expected 1 key, got %d", len(listedKeys))
	}
	if !bytes.Equal(listedKeys[0].Marshal(), c.Marshal()) {
		t.Fatal("Certs not equal")
	}
}

func TestSignGood(t *testing.T) {
	res := &lib.SignResponse{
		Status:   "ok",
		Response: string(testdata.Cert),
	}
	j, _ := json.Marshal(res)
	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		fmt.Fprintln(w, string(j))
	}))
	defer ts.Close()
	_, err := send([]byte(`{}`), "token", ts.URL, true)
	if err != nil {
		t.Fatal(err)
	}
	k, _, _, _, err := ssh.ParseAuthorizedKey(testdata.Pub)
	if err != nil {
		t.Fatal(err)
	}
	c := &config{
		CA:       ts.URL,
		Validity: "24h",
	}
	cert, err := sign(k, "token", c)
	if cert == nil && err != nil {
		t.Fatal(err)
	}
}

func TestSignBad(t *testing.T) {
	res := &lib.SignResponse{
		Status:   "error",
		Response: `{"response": "error"}`,
	}
	j, _ := json.Marshal(res)
	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		fmt.Fprintln(w, string(j))
	}))
	defer ts.Close()
	_, err := send([]byte(`{}`), "token", ts.URL, true)
	if err != nil {
		t.Fatal(err)
	}
	k, _, _, _, err := ssh.ParseAuthorizedKey(testdata.Pub)
	if err != nil {
		t.Fatal(err)
	}
	c := &config{
		CA:       ts.URL,
		Validity: "24h",
	}
	cert, err := sign(k, "token", c)
	if cert != nil && err == nil {
		t.Fatal(err)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"path"
	"path/filepath"
	"strings"
)

func NewFileResource(dataRoot string, r *http.Request) *FileResource {
	return &FileResource{dataRoot: dataRoot, Request: r}
}

type FileResource struct {
	dataRoot string
	*http.Request
}

func (r *FileResource) LoadResource(p string) ([]byte, error) {
	return ioutil.ReadFile(r.dataRoot + p)
}

func (r *FileResource) root() string {
	if r.dataRoot == "" {
		panic("dataRoot must be set")
	}
	return r.dataRoot
}

func (r *FileResource) localPath() string {
	normalizedPath := r.root() + r.Request.URL.Path
	if strings.HasSuffix(normalizedPath, "/") {
		normalizedPath += "index"
	}
	return normalizedPath
}

func (r *FileResource) Store() error {
	return r.storeBody()
}

func (r *FileResource) Tags() (map[string]string, error) {
	files, e := filepath.Glob(r.localPath() + "/*")
	if e != nil {
		return nil, e
	}
	tags := map[string]string{}
	for _, f := range files {
		b, e := ioutil.ReadFile(f)
		if e != nil {
			continue
		}
		tags[path.Base(f)] = strings.Replace(string(b), `"`, "", -1)
	}
	return tags, nil
}

func (r *FileResource) storeBody() error {
	f, e := r.createFile(r.localPath())
	if e != nil {
		return e
	}
	defer f.Close()
	i, e := io.Copy(f, r.Body)
	if e != nil {
		return e
	}
	logger.Printf("stored %d byte", i)
	return nil
}

func (r *FileResource) createFile(p string) (io.WriteCloser, error) {
	if e := os.MkdirAll(path.Dir(p), 0755); e != nil {
		return nil, e
	}
	logger.Printf("creating file %s", p)
	return os.Create(p)
}

func (r *FileResource) Open() (io.Reader, int64, error) {
	f, e := os.Open(r.localPath())
	if e != nil {
		return nil, 0, e
	}
	defer f.Close()
	stat, e := f.Stat()
	if e != nil {
		return nil, 0, e
	}
	return f, stat.Size(), nil
}

func (r *FileResource) DockerSize() (int64, error) {
	stat, e := os.Stat(path.Dir(r.localPath()) + "/layer")
	return stat.Size(), e
}

func (r *FileResource) Exists() bool {
	return fileExists(r.localPath())
}
<commit_msg>do not close file resource too early<commit_after>package main

import (
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"path"
	"path/filepath"
	"strings"
)

func NewFileResource(dataRoot string, r *http.Request) *FileResource {
	return &FileResource{dataRoot: dataRoot, Request: r}
}

type FileResource struct {
	dataRoot string
	*http.Request
}

func (r *FileResource) LoadResource(p string) ([]byte, error) {
	return ioutil.ReadFile(r.dataRoot + p)
}

func (r *FileResource) root() string {
	if r.dataRoot == "" {
		panic("dataRoot must be set")
	}
	return r.dataRoot
}

func (r *FileResource) localPath() string {
	normalizedPath := r.root() + r.Request.URL.Path
	if strings.HasSuffix(normalizedPath, "/") {
		normalizedPath += "index"
	}
	return normalizedPath
}

func (r *FileResource) Store() error {
	return r.storeBody()
}

func (r *FileResource) Tags() (map[string]string, error) {
	files, e := filepath.Glob(r.localPath() + "/*")
	if e != nil {
		return nil, e
	}
	tags := map[string]string{}
	for _, f := range files {
		b, e := ioutil.ReadFile(f)
		if e != nil {
			continue
		}
		tags[path.Base(f)] = strings.Replace(string(b), `"`, "", -1)
	}
	return tags, nil
}

func (r *FileResource) storeBody() error {
	f, e := r.createFile(r.localPath())
	if e != nil {
		return e
	}
	defer f.Close()
	i, e := io.Copy(f, r.Body)
	if e != nil {
		return e
	}
	logger.Printf("stored %d byte", i)
	return nil
}

func (r *FileResource) createFile(p string) (io.WriteCloser, error) {
	if e := os.MkdirAll(path.Dir(p), 0755); e != nil {
		return nil, e
	}
	logger.Printf("creating file %s", p)
	return os.Create(p)
}

func (r *FileResource) Open() (io.Reader, int64, error) {
	f, e := os.Open(r.localPath())
	if e != nil {
		return nil, 0, e
	}
	stat, e := f.Stat()
	if e != nil {
		return nil, 0, e
	}
	return f, stat.Size(), nil
}

func (r *FileResource) DockerSize() (int64, error) {
	stat, e := os.Stat(path.Dir(r.localPath()) + "/layer")
	return stat.Size(), e
}

func (r *FileResource) Exists() bool {
	return fileExists(r.localPath())
}
<|endoftext|>


<commit_before>// Copyright 2015 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package draw

import (
	"fmt"
	"image"
	"image/png"
	"log"
	"math"
	"os"

	"golang.org/x/image/math/f64"
)

func ExampleDraw() {
	fSrc, err := os.Open("../testdata/blue-purple-pink.png")
	if err != nil {
		log.Fatal(err)
	}
	defer fSrc.Close()
	src, err := png.Decode(fSrc)
	if err != nil {
		log.Fatal(err)
	}

	sr := src.Bounds()
	dst := image.NewRGBA(image.Rect(0, 0, 400, 300))
	qs := []Interpolator{
		NearestNeighbor,
		ApproxBiLinear,
		CatmullRom,
	}
	c, s := math.Cos(math.Pi/3), math.Sin(math.Pi/3)
	t := &f64.Aff3{
		+2 * c, -2 * s, 100,
		+2 * s, +2 * c, 100,
	}

	Copy(dst, image.Point{20, 30}, src, sr, nil)
	for i, q := range qs {
		q.Scale(dst, image.Rect(200+10*i, 100*i, 600+10*i, 150+100*i), src, sr, nil)
	}
	// TODO: delete the "_ = t" and uncomment this when Transform is implemented.
	// NearestNeighbor.Transform(dst, t, src, sr, nil)
	_ = t

	// Change false to true to write the resultant image to disk.
	if false {
		fDst, err := os.Create("out.png")
		if err != nil {
			log.Fatal(err)
		}
		defer fDst.Close()
		err = png.Encode(fDst, dst)
		if err != nil {
			log.Fatal(err)
		}
	}

	fmt.Printf("dst has bounds %v.\n", dst.Bounds())
	// Output:
	// dst has bounds (0,0)-(400,300).
}
<commit_msg>draw: make example_test more example-like, by being outside "package draw".<commit_after>// Copyright 2015 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package draw_test

import (
	"fmt"
	"image"
	"image/png"
	"log"
	"math"
	"os"

	"golang.org/x/image/draw"
	"golang.org/x/image/math/f64"
)

func ExampleDraw() {
	fSrc, err := os.Open("../testdata/blue-purple-pink.png")
	if err != nil {
		log.Fatal(err)
	}
	defer fSrc.Close()
	src, err := png.Decode(fSrc)
	if err != nil {
		log.Fatal(err)
	}

	sr := src.Bounds()
	dst := image.NewRGBA(image.Rect(0, 0, 400, 300))
	qs := []draw.Interpolator{
		draw.NearestNeighbor,
		draw.ApproxBiLinear,
		draw.CatmullRom,
	}
	c, s := math.Cos(math.Pi/3), math.Sin(math.Pi/3)
	t := &f64.Aff3{
		+2 * c, -2 * s, 100,
		+2 * s, +2 * c, 100,
	}

	draw.Copy(dst, image.Point{20, 30}, src, sr, nil)
	for i, q := range qs {
		q.Scale(dst, image.Rect(200+10*i, 100*i, 600+10*i, 150+100*i), src, sr, nil)
	}
	// TODO: delete the "_ = t" and uncomment this when Transform is implemented.
	// draw.NearestNeighbor.Transform(dst, t, src, sr, nil)
	_ = t

	// Change false to true to write the resultant image to disk.
	if false {
		fDst, err := os.Create("out.png")
		if err != nil {
			log.Fatal(err)
		}
		defer fDst.Close()
		err = png.Encode(fDst, dst)
		if err != nil {
			log.Fatal(err)
		}
	}

	fmt.Printf("dst has bounds %v.\n", dst.Bounds())
	// Output:
	// dst has bounds (0,0)-(400,300).
}
<|endoftext|>


<commit_before>/*
 * (c) 2013, Caoimhe Chaos <caoimhechaos@protonmail.com>,
 *	     Starship Factory. All rights reserved.
 *
 * Redistribution and use in source  and binary forms, with or without
 * modification, are permitted  provided that the following conditions
 * are met:
 *
 * * Redistributions of  source code  must retain the  above copyright
 *   notice, this list of conditions and the following disclaimer.
 * * Redistributions in binary form must reproduce the above copyright
 *   notice, this  list of conditions and the  following disclaimer in
 *   the  documentation  and/or  other  materials  provided  with  the
 *   distribution.
 * * Neither  the name  of the Starship Factory  nor the  name  of its
 *   contributors may  be used to endorse or  promote products derived
 *   from this software without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 * "AS IS"  AND ANY EXPRESS  OR IMPLIED WARRANTIES  OF MERCHANTABILITY
 * AND FITNESS  FOR A PARTICULAR  PURPOSE ARE DISCLAIMED. IN  NO EVENT
 * SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
 * INDIRECT, INCIDENTAL, SPECIAL,  EXEMPLARY, OR CONSEQUENTIAL DAMAGES
 * (INCLUDING, BUT NOT LIMITED  TO, PROCUREMENT OF SUBSTITUTE GOODS OR
 * SERVICES; LOSS OF USE,  DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
 * STRICT  LIABILITY,  OR  TORT  (INCLUDING NEGLIGENCE  OR  OTHERWISE)
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
 * OF THE POSSIBILITY OF SUCH DAMAGE.
 */

package main

import (
	"flag"
	"html/template"
	"log"
	"net/http"
	"os"

	"ancient-solutions.com/doozer/exportedservice"
)

func main() {
	var help bool
	var bindto, template_dir string
	var lockserv, lockboot, servicename string
	var application_tmpl, print_tmpl *template.Template
	var exporter *exportedservice.ServiceExporter
	var err error

	flag.BoolVar(&help, "help", false, "Display help")
	flag.StringVar(&bindto, "bind", "0.0.0.0",
		"The address to bind the web server to")
	flag.StringVar(&lockserv, "lockserver-uri",
		os.Getenv("DOOZER_URI"),
		"URI of a Doozer cluster to connect to")
	flag.StringVar(&template_dir, "template-dir", "",
		"Path to the directory with the HTML templates")
	flag.StringVar(&lockboot, "lockserver-boot-uri",
		os.Getenv("DOOZER_BOOT_URI"),
		"Boot URI to resolve the Doozer cluster name (if required)")
	flag.StringVar(&servicename, "service-name",
		"", "Service name to publish as to the lock server")
	flag.Parse()

	if help {
		flag.Usage()
		os.Exit(1)
	}

	if len(template_dir) <= 0 {
		log.Fatal("The --template-dir flag must not be empty")
	}

	// Load and parse the HTML templates to be displayed.
	application_tmpl, err = template.ParseFiles(template_dir + "/form.html")
	if err != nil {
		log.Fatal("Unable to parse form template: ", err)
	}
	application_tmpl.Funcs(fmap)

	print_tmpl, err = template.ParseFiles(template_dir + "/printlayout.html")
	if err != nil {
		log.Fatal("Unable to parse print layout template: ", err)
	}
	print_tmpl.Funcs(fmap)

	// Register the URL handler to be invoked.
	http.Handle("/", &FormInputHandler{
		applicationTmpl: application_tmpl,
		printTmpl:       print_tmpl,
		passthrough:     http.FileServer(http.Dir(template_dir)),
	})

	// If a lock server was specified, attempt to use an anonymous port as
	// a Doozer exported HTTP service. Otherwise, just bind to the address
	// given in bindto, for debugging etc.
	if len(lockserv) > 0 {
		exporter, err = exportedservice.NewExporter(lockserv, lockboot)
		if err != nil {
			log.Fatal("doozer.DialUri ", lockserv, " (",
				lockboot, "): ", err)
		}

		err = exporter.ListenAndServeNamedHTTP("helloweb", bindto, nil)
		if err != nil {
			log.Fatal("ListenAndServe: ", err)
		}
	} else {
		err = http.ListenAndServe(bindto, nil)
		if err != nil {
			log.Fatal("ListenAndServe: ", err)
		}
	}
}
<commit_msg>Change the default --bind flag to localhost:8080 to make more sense. Even in the exported port case, we probably don't want 0.0.0.0.<commit_after>/*
 * (c) 2013, Caoimhe Chaos <caoimhechaos@protonmail.com>,
 *	     Starship Factory. All rights reserved.
 *
 * Redistribution and use in source  and binary forms, with or without
 * modification, are permitted  provided that the following conditions
 * are met:
 *
 * * Redistributions of  source code  must retain the  above copyright
 *   notice, this list of conditions and the following disclaimer.
 * * Redistributions in binary form must reproduce the above copyright
 *   notice, this  list of conditions and the  following disclaimer in
 *   the  documentation  and/or  other  materials  provided  with  the
 *   distribution.
 * * Neither  the name  of the Starship Factory  nor the  name  of its
 *   contributors may  be used to endorse or  promote products derived
 *   from this software without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
 * "AS IS"  AND ANY EXPRESS  OR IMPLIED WARRANTIES  OF MERCHANTABILITY
 * AND FITNESS  FOR A PARTICULAR  PURPOSE ARE DISCLAIMED. IN  NO EVENT
 * SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
 * INDIRECT, INCIDENTAL, SPECIAL,  EXEMPLARY, OR CONSEQUENTIAL DAMAGES
 * (INCLUDING, BUT NOT LIMITED  TO, PROCUREMENT OF SUBSTITUTE GOODS OR
 * SERVICES; LOSS OF USE,  DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
 * STRICT  LIABILITY,  OR  TORT  (INCLUDING NEGLIGENCE  OR  OTHERWISE)
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
 * OF THE POSSIBILITY OF SUCH DAMAGE.
 */

package main

import (
	"flag"
	"html/template"
	"log"
	"net/http"
	"os"

	"ancient-solutions.com/doozer/exportedservice"
)

func main() {
	var help bool
	var bindto, template_dir string
	var lockserv, lockboot, servicename string
	var application_tmpl, print_tmpl *template.Template
	var exporter *exportedservice.ServiceExporter
	var err error

	flag.BoolVar(&help, "help", false, "Display help")
	flag.StringVar(&bindto, "bind", "127.0.0.1:8080",
		"The address to bind the web server to")
	flag.StringVar(&lockserv, "lockserver-uri",
		os.Getenv("DOOZER_URI"),
		"URI of a Doozer cluster to connect to")
	flag.StringVar(&template_dir, "template-dir", "",
		"Path to the directory with the HTML templates")
	flag.StringVar(&lockboot, "lockserver-boot-uri",
		os.Getenv("DOOZER_BOOT_URI"),
		"Boot URI to resolve the Doozer cluster name (if required)")
	flag.StringVar(&servicename, "service-name",
		"", "Service name to publish as to the lock server")
	flag.Parse()

	if help {
		flag.Usage()
		os.Exit(1)
	}

	if len(template_dir) <= 0 {
		log.Fatal("The --template-dir flag must not be empty")
	}

	// Load and parse the HTML templates to be displayed.
	application_tmpl, err = template.ParseFiles(template_dir + "/form.html")
	if err != nil {
		log.Fatal("Unable to parse form template: ", err)
	}
	application_tmpl.Funcs(fmap)

	print_tmpl, err = template.ParseFiles(template_dir + "/printlayout.html")
	if err != nil {
		log.Fatal("Unable to parse print layout template: ", err)
	}
	print_tmpl.Funcs(fmap)

	// Register the URL handler to be invoked.
	http.Handle("/", &FormInputHandler{
		applicationTmpl: application_tmpl,
		printTmpl:       print_tmpl,
		passthrough:     http.FileServer(http.Dir(template_dir)),
	})

	// If a lock server was specified, attempt to use an anonymous port as
	// a Doozer exported HTTP service. Otherwise, just bind to the address
	// given in bindto, for debugging etc.
	if len(lockserv) > 0 {
		exporter, err = exportedservice.NewExporter(lockserv, lockboot)
		if err != nil {
			log.Fatal("doozer.DialUri ", lockserv, " (",
				lockboot, "): ", err)
		}

		err = exporter.ListenAndServeNamedHTTP("helloweb", bindto, nil)
		if err != nil {
			log.Fatal("ListenAndServe: ", err)
		}
	} else {
		err = http.ListenAndServe(bindto, nil)
		if err != nil {
			log.Fatal("ListenAndServe: ", err)
		}
	}
}
<|endoftext|>


<commit_before>package main

import "github.com/knakk/rdf"

type TripleAggregator struct {
	In  chan rdf.Triple
	Out chan *TripleAggregate
}

func NewTripleAggregator() *TripleAggregator {
	return &TripleAggregator{
		In:  make(chan rdf.Triple, BUFSIZE),
		Out: make(chan *TripleAggregate, BUFSIZE),
	}
}

func (p *TripleAggregator) Run() {
	defer close(p.Out)
	resourceIndex := make(map[rdf.Subject][]rdf.Triple)
	for triple := range p.In {
		resourceIndex[triple.Subj] = append(resourceIndex[triple.Subj], triple)
	}
	for subj, triples := range resourceIndex {
		tripleAggregate := NewTripleAggregate(subj, triples)
		p.Out <- tripleAggregate
	}
}
<commit_msg>Add comments to TripleAggregator<commit_after>package main

import "github.com/knakk/rdf"

// TripleAggregator aggregates triples by subject into a TripleAggregate object
// per subject, containing all the triples for that subject.
type TripleAggregator struct {
	In  chan rdf.Triple
	Out chan *TripleAggregate
}

// NewTripleAggregator returns an initialized TripleAggregator process.
func NewTripleAggregator() *TripleAggregator {
	return &TripleAggregator{
		In:  make(chan rdf.Triple, BUFSIZE),
		Out: make(chan *TripleAggregate, BUFSIZE),
	}
}

// Run runs the TripleAggregator process.
func (p *TripleAggregator) Run() {
	defer close(p.Out)
	resourceIndex := make(map[rdf.Subject][]rdf.Triple)
	for triple := range p.In {
		resourceIndex[triple.Subj] = append(resourceIndex[triple.Subj], triple)
	}
	for subj, triples := range resourceIndex {
		tripleAggregate := NewTripleAggregate(subj, triples)
		p.Out <- tripleAggregate
	}
}
<|endoftext|>


<commit_before>package unitconfig

import (
	"bytes"
	"encoding/json"
	"errors"
	"io/ioutil"
	"path/filepath"
	"strings"

	"github.com/BurntSushi/toml"
	"github.com/winchman/builder-core/filecheck"
	"gopkg.in/yaml.v2"
)

// Encoding is a constant that represents the config format of a file
// that is expected to contain an encoded unit-config
type Encoding int

const (
	// TOML is for files encoded with TOML (or ending in .toml)
	TOML Encoding = iota

	// JSON is for files encoded in JSON (or ending in .json)
	JSON

	// YAML is for files encoded in YAML (or ending in .yml or .yaml)
	YAML

	unknown
)

// ReadFromFile accepts an unsanitized path to a Bobfile and returns the
// decoded UnitConfig.  The value of encodings is a list of possible file
// encodings.  If none is provided, the encoding will be inferred from the file
// extension as described in the documentation for the UnitConfigEncoding
// constants.  If no encoding can be inferred, TOML will be used as the default.
func ReadFromFile(path string, encodings ...Encoding) (*UnitConfig, error) {
	// check file
	if path == "" {
		return nil, errors.New("no file path provided")
	}

	top := filepath.Dir(path)
	filename := filepath.Base(path)
	opts := filecheck.NewTrustedFilePathOptions{
		File: filename,
		Top:  top,
	}

	var file *filecheck.TrustedFilePath
	var err error
	if file, err = filecheck.NewTrustedFilePath(opts); err != nil {
		return nil, err
	}
	if file.Sanitize(); file.State != filecheck.OK {
		return nil, errors.New("provided file is unchecked or unsanitary")
	}

	// file is sanitary

	contents, err := ioutil.ReadFile(file.FullPath())
	if err != nil {
		return nil, err
	}

	// attempt to infer encoding by file extension
	if encodings == nil || len(encodings) == 0 {
		inferred := inferredEncoding(file)
		if inferred == unknown {
			inferred = TOML
		}
		encodings = []Encoding{inferred}
	}

	// return first validly-decoded UnitConfig
	for _, encoding := range encodings {
		switch encoding {
		case TOML:
			ret := &UnitConfig{}
			if _, err = toml.Decode(string(contents), &ret); err == nil {
				return ret, nil
			}
		case JSON:
			decoder := json.NewDecoder(bytes.NewReader(contents))
			ret := &UnitConfig{}
			if err := decoder.Decode(ret); err == nil {
				return ret, nil
			}
		case YAML:
			ret := &UnitConfig{}
			if err = yaml.Unmarshal(contents, &ret); err == nil {
				return ret, nil
			}
		}
	}

	if err == nil {
		err = errors.New("unable to decode file contents")
	}

	return nil, err
}

func inferredEncoding(file *filecheck.TrustedFilePath) Encoding {
	ext := strings.ToLower(filepath.Ext(file.File()))
	switch ext {
	case "json":
		return JSON
	case "toml":
		return TOML
	case "yaml", "yml":
		return YAML
	default:
		return unknown
	}
}
<commit_msg>If no unitconfig format specified, try all three instead of just TOML<commit_after>package unitconfig

import (
	"bytes"
	"encoding/json"
	"errors"
	"io/ioutil"
	"path/filepath"
	"strings"

	"github.com/BurntSushi/toml"
	"github.com/winchman/builder-core/filecheck"
	"gopkg.in/yaml.v2"
)

// Encoding is a constant that represents the config format of a file
// that is expected to contain an encoded unit-config
type Encoding int

const (
	// TOML is for files encoded with TOML (or ending in .toml)
	TOML Encoding = iota

	// JSON is for files encoded in JSON (or ending in .json)
	JSON

	// YAML is for files encoded in YAML (or ending in .yml or .yaml)
	YAML

	unknown
)

// ReadFromFile accepts an unsanitized path to a Bobfile and returns the
// decoded UnitConfig.  The value of encodings is a list of possible file
// encodings.  If none is provided, the encoding will be inferred from the file
// extension as described in the documentation for the UnitConfigEncoding
// constants.  If no encoding can be inferred, TOML will be used as the default.
func ReadFromFile(path string, encodings ...Encoding) (*UnitConfig, error) {
	if len(encodings) == 0 {
		encodings = []Encoding{TOML, JSON, YAML}
	}
	// check file
	if path == "" {
		return nil, errors.New("no file path provided")
	}

	top := filepath.Dir(path)
	filename := filepath.Base(path)
	opts := filecheck.NewTrustedFilePathOptions{
		File: filename,
		Top:  top,
	}

	var file *filecheck.TrustedFilePath
	var err error
	if file, err = filecheck.NewTrustedFilePath(opts); err != nil {
		return nil, err
	}
	if file.Sanitize(); file.State != filecheck.OK {
		return nil, errors.New("provided file is unchecked or unsanitary")
	}

	// file is sanitary

	contents, err := ioutil.ReadFile(file.FullPath())
	if err != nil {
		return nil, err
	}

	// attempt to infer encoding by file extension
	if encodings == nil || len(encodings) == 0 {
		inferred := inferredEncoding(file)
		if inferred == unknown {
			inferred = TOML
		}
		encodings = []Encoding{inferred}
	}

	// return first validly-decoded UnitConfig
	for _, encoding := range encodings {
		switch encoding {
		case TOML:
			ret := &UnitConfig{}
			if _, err = toml.Decode(string(contents), &ret); err == nil {
				return ret, nil
			}
		case JSON:
			decoder := json.NewDecoder(bytes.NewReader(contents))
			ret := &UnitConfig{}
			if err := decoder.Decode(ret); err == nil {
				return ret, nil
			}
		case YAML:
			ret := &UnitConfig{}
			if err = yaml.Unmarshal(contents, &ret); err == nil {
				return ret, nil
			}
		}
	}

	if err == nil {
		err = errors.New("unable to decode file contents")
	}

	return nil, err
}

func inferredEncoding(file *filecheck.TrustedFilePath) Encoding {
	ext := strings.ToLower(filepath.Ext(file.File()))
	switch ext {
	case "json":
		return JSON
	case "toml":
		return TOML
	case "yaml", "yml":
		return YAML
	default:
		return unknown
	}
}
<|endoftext|>


<commit_before>package testutil

import (
	"bytes"
	"github.com/pachyderm/pachyderm/src/client/pkg/require"
	"os"
	"os/exec"
	"strings"
	"testing"

	kube "k8s.io/client-go/kubernetes"
	"k8s.io/client-go/rest"
)

// GetKubeClient connects to the Kubernetes API server either from inside the
// cluster or from a test binary running on a machine with kubectl (it will
// connect to the same cluster as kubectl)
func GetKubeClient(t testing.TB) *kube.Clientset {
	var config *rest.Config
	host := os.Getenv("KUBERNETES_SERVICE_HOST")
	if host != "" {
		var err error
		config, err = rest.InClusterConfig()
		require.NoError(t, err)
	} else {
		// Use kubectl binary to parse .kube/config and get address of current
		// cluster. Hopefully, once we upgrade to k8s.io/client-go, we will be able
		// to do this in-process with a library
		// First, figure out if we're talking to minikube or localhost
		cmd := exec.Command("kubectl", "config", "current-context")
		cmd.Stderr = os.Stderr
		if context, err := cmd.Output(); err == nil {
			context = bytes.TrimSpace(context)
			// kubectl has a context -- not talking to localhost
			// Get cluster and user name from kubectl
			buf := &bytes.Buffer{}
			cmd := BashCmd(strings.Join([]string{
				`kubectl config get-contexts "{{.context}}" | tail -n+2 | awk '{print $3}'`,
				`kubectl config get-contexts "{{.context}}" | tail -n+2 | awk '{print $4}'`,
			}, "\n"),
				"context", string(context))
			cmd.Stdout = buf
			require.NoError(t, cmd.Run(), "couldn't get kubernetes context info")
			lines := strings.Split(buf.String(), "\n")
			clustername, username := lines[0], lines[1]

			// Get user info
			buf.Reset()
			cmd = BashCmd(strings.Join([]string{
				`cluster="$(kubectl config view -o json | jq -r '.users[] | select(.name == "{{.user}}") | .user' )"`,
				`echo "${cluster}" | jq -r '.["client-certificate"]'`,
				`echo "${cluster}" | jq -r '.["client-key"]'`,
			}, "\n"),
				"user", username)
			cmd.Stdout = buf
			require.NoError(t, cmd.Run(), "couldn't get kubernetes user info")
			lines = strings.Split(buf.String(), "\n")
			clientCert, clientKey := lines[0], lines[1]

			// Get cluster info
			buf.Reset()
			cmd = BashCmd(strings.Join([]string{
				`cluster="$(kubectl config view -o json | jq -r '.clusters[] | select(.name == "{{.cluster}}") | .cluster')"`,
				`echo "${cluster}" | jq -r .server`,
				`echo "${cluster}" | jq -r '.["certificate-authority"]'`,
			}, "\n"),
				"cluster", clustername)
			cmd.Stdout = buf
			require.NoError(t, cmd.Run(), "couldn't get kubernetes cluster info: %s", buf.String())
			lines = strings.Split(buf.String(), "\n")
			address, CAKey := lines[0], lines[1]

			// Generate config
			config = &rest.Config{
				Host: address,
				TLSClientConfig: rest.TLSClientConfig{
					CertFile: clientCert,
					KeyFile:  clientKey,
					CAFile:   CAKey,
				},
			}
		} else {
			// no context -- talking to localhost
			config = &rest.Config{
				Host: "http://0.0.0.0:8080",
				TLSClientConfig: rest.TLSClientConfig{
					Insecure: false,
				},
			}
		}
	}
	k, err := kube.NewForConfig(config)
	require.NoError(t, err)
	return k
}
<commit_msg>Simplify kube client creation in tests<commit_after>package testutil

import (
	"github.com/pachyderm/pachyderm/src/client/pkg/require"
	"os"
	"testing"

	kube "k8s.io/client-go/kubernetes"
	"k8s.io/client-go/rest"
	"k8s.io/client-go/tools/clientcmd"
)

// GetKubeClient connects to the Kubernetes API server either from inside the
// cluster or from a test binary running on a machine with kubectl (it will
// connect to the same cluster as kubectl)
func GetKubeClient(t testing.TB) *kube.Clientset {
	var config *rest.Config
	var err error
	host := os.Getenv("KUBERNETES_SERVICE_HOST")
	if host != "" {
		config, err = rest.InClusterConfig()
	} else {
		rules := clientcmd.NewDefaultClientConfigLoadingRules()
		kubeConfig := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(rules,
			&clientcmd.ConfigOverrides{})
		config, err = kubeConfig.ClientConfig()
	}
	require.NoError(t, err)
	k, err := kube.NewForConfig(config)
	require.NoError(t, err)
	return k
}
<|endoftext|>


<commit_before>/**
 * Copyright 2016 IBM Corp.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package main

import (
	"bytes"
	"encoding/json"
	"flag"
	"fmt"
	"go/format"
	"golang.org/x/tools/imports"
	"io/ioutil"
	"net/http"
	"os"
	"sort"
	"strings"
	"text/template"
)

type Type struct {
	Name       string              `json:"name"`
	Base       string              `json:"base"`
	TypeDoc    string              `json:"typeDoc"`
	Properties map[string]Property `json:"properties"`
	ServiceDoc string              `json:"serviceDoc"`
	Methods    map[string]Method   `json:"methods"`
	NoService  bool                `json:"noservice"`
}

type Property struct {
	Name      string `json:"name"`
	Type      string `json:"type"`
	TypeArray bool   `json:"typeArray"`
	Form      string `json:"form"`
	Doc       string `json:"doc"`
}

type Method struct {
	Name       string      `json:"name"`
	Type       string      `json:"type"`
	TypeArray  bool        `json:"typeArray"`
	Doc        string      `json:"doc"`
	Static     bool        `json:"static"`
	NoAuth     bool        `json:"noauth"`
	Limitable  bool        `json:"limitable"`
	Filterable bool        `json:"filterable"`
	Maskable   bool        `json:"maskable"`
	Parameters []Parameter `json:"parameters"`
}

type Parameter struct {
	Name         string      `json:"name"`
	Type         string      `json:"type"`
	TypeArray    bool        `json:"typeArray"`
	Doc          string      `json:"doc"`
	DefaultValue interface{} `json:"defaultValue"`
}

// Define custom template functions
var fMap = template.FuncMap{
	"convertType":       ConvertType,           // Converts SoftLayer types to Go types
	"prefixWithPackage": PrefixWithPackageName, // Prepend a type with the given package name
	"removePrefix":      RemovePrefix,          // Remove 'SoftLayer_' prefix. if it exists
	"removeReserved":    RemoveReservedWords,   // Substitute language-reserved identifiers
	"titleCase":         strings.Title,         // TitleCase the argument
	"desnake":           Desnake,               // Remove '_' from Snake_Case
}

const license = `/**
 * Copyright 2016 IBM Corp.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
 `

var datatype = fmt.Sprintf(`%s
package datatypes

{{range .}}type {{.Name|removePrefix}} struct {
	{{.Base|removePrefix}}

	{{range .Properties}}{{.Name|titleCase}} {{if .TypeArray}}[]{{else}}*{{end}}{{.Type|convertType|removePrefix}}`+
	"`json:\"{{.Name}},omitempty\"`"+`
	{{end}}
}

{{end}}
`, license)

var service = fmt.Sprintf(`%s
package service

{{range .}}{{$base := .Name|removePrefix}}
	type {{$base}} struct {
		Session *Session
		Options
	}

	func (r *Session) Get{{$base | desnake}}Service() {{$base}} {
		return {{$base}}{Session: r}
	}

	{{range .Methods}}func (r *{{$base}}) {{.Name|titleCase}}({{range .Parameters}}{{.Name|removeReserved}} {{if not .TypeArray}}*{{else}}[]{{end}}{{.Type|convertType|prefixWithPackage "datatypes"}}, {{end}}) ({{if .Type|ne "void"}}resp {{if .TypeArray}}[]{{end}}{{.Type|convertType|prefixWithPackage "datatypes"}}, {{end}}err error) {
		{{if .Type|eq "void"}}var resp datatypes.Void
		{{end}}{{if len .Parameters | lt 0}}params := []interface{}{
			{{range .Parameters}}{{.Name|removeReserved}},
			{{end}}
		}
		{{end}}err = invokeMethod({{if len .Parameters | lt 0}}params{{else}}nil{{end}}, r.Session, &r.Options, &resp)
	return
	}
	{{end}}

{{end}}
`, license)

func main() {
	var meta map[string]Type

	outputPath := flag.String("o", ".", "the root of the go project to be refreshed")
	flag.Parse()

	jsonResp, code, err := makeHttpRequest("https://api.softlayer.com/metadata/v3.1", "GET", new(bytes.Buffer))

	if err != nil {
		fmt.Printf("Error retrieving metadata API: %s", err)
		os.Exit(1)
	}

	if code != 200 {
		fmt.Printf("Unexpected HTTP status code received while retrieving metadata API: %d", code)
		os.Exit(1)
	}

	err = json.Unmarshal(jsonResp, &meta)
	if err != nil {
		fmt.Printf("Error unmarshaling json response: %s", err)
		os.Exit(1)
	}

	// Build an array of Types, sorted by name
	// This will ensure consistency in the order that code is later emitted
	keys := getSortedKeys(meta)

	sortedTypes := make([]Type, 0, len(keys))
	sortedServices := make([]Type, 0, len(keys))

	for _, name := range keys {
		t := meta[name]
		sortedTypes = append(sortedTypes, t)

		// Not every datatype is also a service
		if !t.NoService {
			createGetters(&t)
			sortedServices = append(sortedServices, t)
		}
	}

	err = writePackage(*outputPath, "datatypes", sortedTypes, datatype)
	if err != nil {
		fmt.Printf("Error writing to file: %s", err)
	}

	err = writePackage(*outputPath, "service", sortedServices, service)
	if err != nil {
		fmt.Printf("Error writing to file: %s", err)
	}
}

// Exported template functions

func RemovePrefix(args ...interface{}) string {
	s := args[0].(string)

	if strings.HasPrefix(s, "SoftLayer_") {
		return s[10:]
	}

	return s
}

func PrefixWithPackageName(args ...interface{}) string {
	p := args[0].(string)
	s := args[1].(string)

	if !strings.HasPrefix(s, "SoftLayer_") && s != "Time" {
		return s
	}

	return p + "." + RemovePrefix(s)
}

func ConvertType(args ...interface{}) string {
	t := args[0].(string)

	// Convert softlayer types to golang types
	switch t {
	case "unsignedLong", "unsignedInt":
		return "uint"
	case "boolean":
		return "bool"
	case "dateTime":
		return "Time"
	case "decimal", "float":
		return "float64"
	case "base64Binary":
		return "[]byte"
	case "json", "enum":
		return "string"
	}

	return t
}

func RemoveReservedWords(args ...interface{}) string {
	n := args[0].(string)

	// Replace language reserved identifiers with alternatives
	switch n {
	case "type":
		return "typ"
	}

	return n
}

// Remove '_' from Snake_Case values
func Desnake(args ...interface{}) string {
	s := args[0].(string)
	return strings.Replace(s, "_", "", -1)
}

// private

func createGetters(service *Type) {
	for _, p := range service.Properties {
		if p.Form == "relational" {
			m := Method{
				Name:       "get" + strings.Title(p.Name),
				Type:       p.Type,
				TypeArray:  p.TypeArray,
				Doc:        "Retrieve " + p.Doc, // TODO lowercase the first letter
				Parameters: []Parameter{},
			}

			service.Methods[m.Name] = m
		}
	}

}

func getSortedKeys(m map[string]Type) []string {
	keys := make([]string, 0, len(m))
	for key := range m {
		keys = append(keys, key)
	}
	sort.Strings(keys)

	return keys
}

func writePackage(base string, pkg string, meta []Type, ts string) error {
	var currPrefix string
	var start int

	for i, t := range meta {
		components := strings.Split(RemovePrefix(t.Name), "_")

		if i == 0 {
			currPrefix = components[0]
			continue
		}

		if components[0] != currPrefix {
			err := writeGoFile(base, pkg, currPrefix, meta[start:i], ts)
			if err != nil {
				return err
			}

			currPrefix = components[0]
			start = i
		}
	}

	writeGoFile(base, pkg, currPrefix, meta[start:], ts)

	return nil
}

// Executes a template against the metadata structure, and generates a go source file with the result
func writeGoFile(base string, pkg string, name string, meta []Type, ts string) error {
	filename := base + "/" + pkg + "/" + strings.ToLower(name) + ".go"

	// Generate the source
	var buf bytes.Buffer
	t := template.New(pkg).Funcs(fMap)
	template.Must(t.Parse(ts)).Execute(&buf, meta)

	/*if pkg == "service" && name == "Account"{
		fmt.Println(string(buf.String()))
		os.Exit(0)
	}*/

	// Add the imports
	src, err := imports.Process(filename, buf.Bytes(), &imports.Options{Comments: true})
	if err != nil {
		fmt.Printf("Error processing imports: %s", err)
	}

	// Format
	pretty, err := format.Source(src)
	if err != nil {
		return fmt.Errorf("Error while formatting source: %s", err)
	}

	f, err := os.Create(filename)
	if err != nil {
		return fmt.Errorf("Error creating file: %s", err)
	}
	defer f.Close()
	fmt.Fprintf(f, "%s", pretty)

	return nil
}

func makeHttpRequest(url string, requestType string, requestBody *bytes.Buffer) ([]byte, int, error) {
	client := http.DefaultClient

	req, err := http.NewRequest(requestType, url, requestBody)
	if err != nil {
		return nil, 0, err
	}

	resp, err := client.Do(req)
	if err != nil {
		return nil, 520, err
	}

	defer resp.Body.Close()

	responseBody, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, resp.StatusCode, err
	}

	return responseBody, resp.StatusCode, nil
}
<commit_msg>Copy base service methods to child services<commit_after>/**
 * Copyright 2016 IBM Corp.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package main

import (
	"bytes"
	"encoding/json"
	"flag"
	"fmt"
	"go/format"
	"golang.org/x/tools/imports"
	"io/ioutil"
	"net/http"
	"os"
	"sort"
	"strings"
	"text/template"
)

type Type struct {
	Name       string              `json:"name"`
	Base       string              `json:"base"`
	TypeDoc    string              `json:"typeDoc"`
	Properties map[string]Property `json:"properties"`
	ServiceDoc string              `json:"serviceDoc"`
	Methods    map[string]Method   `json:"methods"`
	NoService  bool                `json:"noservice"`
}

type Property struct {
	Name      string `json:"name"`
	Type      string `json:"type"`
	TypeArray bool   `json:"typeArray"`
	Form      string `json:"form"`
	Doc       string `json:"doc"`
}

type Method struct {
	Name       string      `json:"name"`
	Type       string      `json:"type"`
	TypeArray  bool        `json:"typeArray"`
	Doc        string      `json:"doc"`
	Static     bool        `json:"static"`
	NoAuth     bool        `json:"noauth"`
	Limitable  bool        `json:"limitable"`
	Filterable bool        `json:"filterable"`
	Maskable   bool        `json:"maskable"`
	Parameters []Parameter `json:"parameters"`
}

type Parameter struct {
	Name         string      `json:"name"`
	Type         string      `json:"type"`
	TypeArray    bool        `json:"typeArray"`
	Doc          string      `json:"doc"`
	DefaultValue interface{} `json:"defaultValue"`
}

// Define custom template functions
var fMap = template.FuncMap{
	"convertType":       ConvertType,           // Converts SoftLayer types to Go types
	"prefixWithPackage": PrefixWithPackageName, // Prepend a type with the given package name
	"removePrefix":      RemovePrefix,          // Remove 'SoftLayer_' prefix. if it exists
	"removeReserved":    RemoveReservedWords,   // Substitute language-reserved identifiers
	"titleCase":         strings.Title,         // TitleCase the argument
	"desnake":           Desnake,               // Remove '_' from Snake_Case
}

const license = `/**
 * Copyright 2016 IBM Corp.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
 `

var datatype = fmt.Sprintf(`%s
package datatypes

{{range .}}type {{.Name|removePrefix}} struct {
	{{.Base|removePrefix}}

	{{range .Properties}}{{.Name|titleCase}} {{if .TypeArray}}[]{{else}}*{{end}}{{.Type|convertType|removePrefix}}`+
	"`json:\"{{.Name}},omitempty\"`"+`
	{{end}}
}

{{end}}
`, license)

var service = fmt.Sprintf(`%s
package service

{{range .}}{{$base := .Name|removePrefix}}
	type {{$base}} struct {
		Session *Session
		Options
	}

	func (r *Session) Get{{$base | desnake}}Service() {{$base}} {
		return {{$base}}{Session: r}
	}

	{{range .Methods}}func (r *{{$base}}) {{.Name|titleCase}}({{range .Parameters}}{{.Name|removeReserved}} {{if not .TypeArray}}*{{else}}[]{{end}}{{.Type|convertType|prefixWithPackage "datatypes"}}, {{end}}) ({{if .Type|ne "void"}}resp {{if .TypeArray}}[]{{end}}{{.Type|convertType|prefixWithPackage "datatypes"}}, {{end}}err error) {
		{{if .Type|eq "void"}}var resp datatypes.Void
		{{end}}{{if len .Parameters | lt 0}}params := []interface{}{
			{{range .Parameters}}{{.Name|removeReserved}},
			{{end}}
		}
		{{end}}err = invokeMethod({{if len .Parameters | lt 0}}params{{else}}nil{{end}}, r.Session, &r.Options, &resp)
	return
	}
	{{end}}

{{end}}
`, license)

func main() {
	var meta map[string]Type

	outputPath := flag.String("o", ".", "the root of the go project to be refreshed")
	flag.Parse()

	jsonResp, code, err := makeHttpRequest("https://api.softlayer.com/metadata/v3.1", "GET", new(bytes.Buffer))

	if err != nil {
		fmt.Printf("Error retrieving metadata API: %s", err)
		os.Exit(1)
	}

	if code != 200 {
		fmt.Printf("Unexpected HTTP status code received while retrieving metadata API: %d", code)
		os.Exit(1)
	}

	err = json.Unmarshal(jsonResp, &meta)
	if err != nil {
		fmt.Printf("Error unmarshaling json response: %s", err)
		os.Exit(1)
	}

	// Build an array of Types, sorted by name
	// This will ensure consistency in the order that code is later emitted
	keys := getSortedKeys(meta)

	sortedTypes := make([]Type, 0, len(keys))
	sortedServices := make([]Type, 0, len(keys))

	for _, name := range keys {
		t := meta[name]
		sortedTypes = append(sortedTypes, t)

		// Not every datatype is also a service
		if !t.NoService {
			createGetters(&t)
			sortedServices = append(sortedServices, t)
		}
	}

	// Services can be subclasses of other services. Copy methods from each service's 'Base' entity to
	// the child service, only if a same-named method does not already exist (i.e., overridden by the
	// child service)
	for i, service := range sortedServices {
		sortedServices[i].Methods = getBaseMethods(service, meta)
	}

	err = writePackage(*outputPath, "datatypes", sortedTypes, datatype)
	if err != nil {
		fmt.Printf("Error writing to file: %s", err)
	}

	err = writePackage(*outputPath, "service", sortedServices, service)
	if err != nil {
		fmt.Printf("Error writing to file: %s", err)
	}
}

// Exported template functions

func RemovePrefix(args ...interface{}) string {
	s := args[0].(string)

	if strings.HasPrefix(s, "SoftLayer_") {
		return s[10:]
	}

	return s
}

func PrefixWithPackageName(args ...interface{}) string {
	p := args[0].(string)
	s := args[1].(string)

	if !strings.HasPrefix(s, "SoftLayer_") && s != "Time" {
		return s
	}

	return p + "." + RemovePrefix(s)
}

func ConvertType(args ...interface{}) string {
	t := args[0].(string)

	// Convert softlayer types to golang types
	switch t {
	case "unsignedLong", "unsignedInt":
		return "uint"
	case "boolean":
		return "bool"
	case "dateTime":
		return "Time"
	case "decimal", "float":
		return "float64"
	case "base64Binary":
		return "[]byte"
	case "json", "enum":
		return "string"
	}

	return t
}

func RemoveReservedWords(args ...interface{}) string {
	n := args[0].(string)

	// Replace language reserved identifiers with alternatives
	switch n {
	case "type":
		return "typ"
	}

	return n
}

// Remove '_' from Snake_Case values
func Desnake(args ...interface{}) string {
	s := args[0].(string)
	return strings.Replace(s, "_", "", -1)
}

// private

func createGetters(service *Type) {
	for _, p := range service.Properties {
		if p.Form == "relational" {
			m := Method{
				Name:       "get" + strings.Title(p.Name),
				Type:       p.Type,
				TypeArray:  p.TypeArray,
				Doc:        "Retrieve " + p.Doc, // TODO lowercase the first letter
				Parameters: []Parameter{},
			}

			service.Methods[m.Name] = m
		}
	}

}

func combineMethods(baseMethods map[string]Method, subclassMethods map[string]Method) map[string]Method {
	r := map[string]Method{}

	// Copy all subclass methods into the result set
	for k, v := range subclassMethods {
		r[k] = v
	}

	// Copy each method from the base class into the result set, but only if a like-named method
	// does not already exist (a method in the child should override a same-named method in the parent)
	for k, v := range baseMethods {
		if _, ok := r[k]; !ok {
			r[k] = v
		}
	}

	return r
}

func getBaseMethods(s Type, typeMap map[string]Type) map[string]Method {
	var methods, baseMethods map[string]Method

	methods = s.Methods

	if s.Base != "SoftLayer_Entity" {
		baseMethods = getBaseMethods(typeMap[s.Base], typeMap)

		// Add base methods to current service methods
		methods = combineMethods(baseMethods, methods)
	}

	// return my methods
	return methods
}

func getSortedKeys(m map[string]Type) []string {
	keys := make([]string, 0, len(m))
	for key := range m {
		keys = append(keys, key)
	}
	sort.Strings(keys)

	return keys
}

func writePackage(base string, pkg string, meta []Type, ts string) error {
	var currPrefix string
	var start int

	for i, t := range meta {
		components := strings.Split(RemovePrefix(t.Name), "_")

		if i == 0 {
			currPrefix = components[0]
			continue
		}

		if components[0] != currPrefix {
			err := writeGoFile(base, pkg, currPrefix, meta[start:i], ts)
			if err != nil {
				return err
			}

			currPrefix = components[0]
			start = i
		}
	}

	writeGoFile(base, pkg, currPrefix, meta[start:], ts)

	return nil
}

// Executes a template against the metadata structure, and generates a go source file with the result
func writeGoFile(base string, pkg string, name string, meta []Type, ts string) error {
	filename := base + "/" + pkg + "/" + strings.ToLower(name) + ".go"

	// Generate the source
	var buf bytes.Buffer
	t := template.New(pkg).Funcs(fMap)
	template.Must(t.Parse(ts)).Execute(&buf, meta)

	/*if pkg == "service" && name == "Account"{
		fmt.Println(string(buf.String()))
		os.Exit(0)
	}*/

	// Add the imports
	src, err := imports.Process(filename, buf.Bytes(), &imports.Options{Comments: true})
	if err != nil {
		fmt.Printf("Error processing imports: %s", err)
	}

	// Format
	pretty, err := format.Source(src)
	if err != nil {
		return fmt.Errorf("Error while formatting source: %s", err)
	}

	f, err := os.Create(filename)
	if err != nil {
		return fmt.Errorf("Error creating file: %s", err)
	}
	defer f.Close()
	fmt.Fprintf(f, "%s", pretty)

	return nil
}

func makeHttpRequest(url string, requestType string, requestBody *bytes.Buffer) ([]byte, int, error) {
	client := http.DefaultClient

	req, err := http.NewRequest(requestType, url, requestBody)
	if err != nil {
		return nil, 0, err
	}

	resp, err := client.Do(req)
	if err != nil {
		return nil, 520, err
	}

	defer resp.Body.Close()

	responseBody, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, resp.StatusCode, err
	}

	return responseBody, resp.StatusCode, nil
}
<|endoftext|>


<commit_before>//  Copyright (c) 2016 Couchbase, Inc.
//  Licensed under the Apache License, Version 2.0 (the "License");
//  you may not use this file except in compliance with the
//  License. You may obtain a copy of the License at
//    http://www.apache.org/licenses/LICENSE-2.0
//  Unless required by applicable law or agreed to in writing,
//  software distributed under the License is distributed on an "AS
//  IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
//  express or implied. See the License for the specific language
//  governing permissions and limitations under the License.

package moss

import (
	"github.com/couchbase/moss"

	"github.com/blevesearch/bleve/index/store"
)

type Reader struct {
	store *Store
	ss    moss.Snapshot
}

func (r *Reader) Get(k []byte) (v []byte, err error) {
	v, err = r.ss.Get(k, moss.ReadOptions{})
	if err != nil {
		return nil, err
	}
	if v != nil {
		return append([]byte(nil), v...), nil
	}
	return nil, nil
}

func (r *Reader) MultiGet(keys [][]byte) ([][]byte, error) {
	return store.MultiGet(r, keys)
}

func (r *Reader) PrefixIterator(k []byte) store.KVIterator {
	iter, err := r.ss.StartIterator(k, nil, moss.IteratorOptions{})
	if err != nil {
		return nil
	}

	rv := &Iterator{
		store:  r.store,
		ss:     r.ss,
		iter:   iter,
		prefix: k,
		start:  k,
		end:    nil,
	}

	rv.checkDone()

	return rv
}

func (r *Reader) RangeIterator(start, end []byte) store.KVIterator {
	iter, err := r.ss.StartIterator(start, end, moss.IteratorOptions{})
	if err != nil {
		return nil
	}

	rv := &Iterator{
		store:  r.store,
		ss:     r.ss,
		iter:   iter,
		prefix: nil,
		start:  start,
		end:    end,
	}

	rv.checkDone()

	return rv
}

func (r *Reader) Close() error {
	return r.ss.Close()
}
<commit_msg>MB-20101 - moss KV fix Get() of 0-length vals<commit_after>//  Copyright (c) 2016 Couchbase, Inc.
//  Licensed under the Apache License, Version 2.0 (the "License");
//  you may not use this file except in compliance with the
//  License. You may obtain a copy of the License at
//    http://www.apache.org/licenses/LICENSE-2.0
//  Unless required by applicable law or agreed to in writing,
//  software distributed under the License is distributed on an "AS
//  IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
//  express or implied. See the License for the specific language
//  governing permissions and limitations under the License.

package moss

import (
	"github.com/couchbase/moss"

	"github.com/blevesearch/bleve/index/store"
)

type Reader struct {
	store *Store
	ss    moss.Snapshot
}

func (r *Reader) Get(k []byte) (v []byte, err error) {
	v, err = r.ss.Get(k, moss.ReadOptions{})
	if err != nil {
		return nil, err
	}
	if v != nil {
		return append(make([]byte, 0, len(v)), v...), nil
	}
	return nil, nil
}

func (r *Reader) MultiGet(keys [][]byte) ([][]byte, error) {
	return store.MultiGet(r, keys)
}

func (r *Reader) PrefixIterator(k []byte) store.KVIterator {
	iter, err := r.ss.StartIterator(k, nil, moss.IteratorOptions{})
	if err != nil {
		return nil
	}

	rv := &Iterator{
		store:  r.store,
		ss:     r.ss,
		iter:   iter,
		prefix: k,
		start:  k,
		end:    nil,
	}

	rv.checkDone()

	return rv
}

func (r *Reader) RangeIterator(start, end []byte) store.KVIterator {
	iter, err := r.ss.StartIterator(start, end, moss.IteratorOptions{})
	if err != nil {
		return nil
	}

	rv := &Iterator{
		store:  r.store,
		ss:     r.ss,
		iter:   iter,
		prefix: nil,
		start:  start,
		end:    end,
	}

	rv.checkDone()

	return rv
}

func (r *Reader) Close() error {
	return r.ss.Close()
}
<|endoftext|>


<commit_before>// Copyright 2015 tsuru authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package docker

import (
	"fmt"

	dtesting "github.com/fsouza/go-dockerclient/testing"
	"github.com/tsuru/docker-cluster/cluster"
	"github.com/tsuru/tsuru/action"
	"github.com/tsuru/tsuru/app"
	"github.com/tsuru/tsuru/provision"
	"github.com/tsuru/tsuru/provision/provisiontest"
	"github.com/tsuru/tsuru/safe"
	"gopkg.in/check.v1"
)

func (s *S) TestRebalanceContainersManyAppsSegStress(c *check.C) {
	var nodes []cluster.Node
	var nodeHosts []string
	for i := 0; i < 6; i++ {
		newIp := fmt.Sprintf("127.0.0.%d", i+1)
		otherServer, err := dtesting.NewServer(newIp+":0", nil, nil)
		c.Assert(err, check.IsNil)
		defer otherServer.Stop()
		nodes = append(nodes, cluster.Node{Address: otherServer.URL(), Metadata: map[string]string{"pool": "pool1"}})
		nodeHosts = append(nodeHosts, urlToHost(otherServer.URL()))
	}
	var err error
	p := &dockerProvisioner{}
	err = p.Initialize()
	c.Assert(err, check.IsNil)
	p.storage, err = buildClusterStorage()
	c.Assert(err, check.IsNil)
	p.scheduler = &segregatedScheduler{provisioner: p}
	p.cluster, err = cluster.New(p.scheduler, p.storage, nodes...)
	c.Assert(err, check.IsNil)
	opts := provision.AddPoolOptions{Name: "pool1"}
	err = provision.AddPool(opts)
	c.Assert(err, check.IsNil)
	err = provision.AddTeamsToPool("pool1", []string{"team1"})
	c.Assert(err, check.IsNil)
	variation := []int{10, 20, 30, 40, 50, 100}
	maxContainers := 40
	for i := 0; i < maxContainers; i++ {
		appName := fmt.Sprintf("myapp-%d", i)
		err = s.newFakeImage(p, "tsuru/app-"+appName, nil)
		c.Assert(err, check.IsNil)
		appInstance := provisiontest.NewFakeApp(appName, "python", 0)
		defer p.Destroy(appInstance)
		p.Provision(appInstance)
		imageId, err := appCurrentImageName(appInstance.GetName())
		c.Assert(err, check.IsNil)
		var chosenNode string
		for j := range variation {
			if i < (maxContainers*variation[j])/100 {
				chosenNode = nodeHosts[j]
				break
			}
		}
		args := changeUnitsPipelineArgs{
			app:         appInstance,
			toAdd:       map[string]*containersToAdd{"web": {Quantity: 6}},
			imageId:     imageId,
			provisioner: p,
			toHost:      chosenNode,
		}
		pipeline := action.NewPipeline(
			&provisionAddUnitsToHost,
			&bindAndHealthcheck,
			&addNewRoutes,
			&updateAppImage,
		)
		err = pipeline.Execute(args)
		c.Assert(err, check.IsNil)
		appStruct := &app.App{
			Name:      appInstance.GetName(),
			TeamOwner: "team1",
			Pool:      "pool1",
		}
		err = s.storage.Apps().Insert(appStruct)
		c.Assert(err, check.IsNil)
	}
	buf := safe.NewBuffer(nil)
	cloneProv, err := p.rebalanceContainersByFilter(buf, []string{}, map[string]string{"pool": "pool1"}, false)
	c.Assert(err, check.IsNil)
	c.Assert(cloneProv.cluster.Healer, check.Equals, p.cluster.Healer)
	for i := range nodeHosts {
		conts, err := p.listContainersByHost(nodeHosts[i])
		c.Assert(err, check.IsNil)
		c.Logf("containers in %q: %d", nodeHosts[i], len(conts))
		c.Check(len(conts) >= 39 || len(conts) <= 41, check.Equals, true)
	}
}
<commit_msg>docker: fix tests on linux<commit_after>// Copyright 2015 tsuru authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package docker

import (
	"fmt"

	dtesting "github.com/fsouza/go-dockerclient/testing"
	"github.com/tsuru/docker-cluster/cluster"
	"github.com/tsuru/tsuru/action"
	"github.com/tsuru/tsuru/app"
	"github.com/tsuru/tsuru/net"
	"github.com/tsuru/tsuru/provision"
	"github.com/tsuru/tsuru/provision/provisiontest"
	"github.com/tsuru/tsuru/safe"
	"gopkg.in/check.v1"
)

func (s *S) TestRebalanceContainersManyAppsSegStress(c *check.C) {
	var nodes []cluster.Node
	var nodeHosts []string
	for i := 0; i < 6; i++ {
		newIp := fmt.Sprintf("127.0.0.%d", i+1)
		otherServer, err := dtesting.NewServer(newIp+":0", nil, nil)
		c.Assert(err, check.IsNil)
		defer otherServer.Stop()
		nodes = append(nodes, cluster.Node{Address: otherServer.URL(), Metadata: map[string]string{"pool": "pool1"}})
		nodeHosts = append(nodeHosts, net.URLToHost(otherServer.URL()))
	}
	var err error
	p := &dockerProvisioner{}
	err = p.Initialize()
	c.Assert(err, check.IsNil)
	p.storage, err = buildClusterStorage()
	c.Assert(err, check.IsNil)
	p.scheduler = &segregatedScheduler{provisioner: p}
	p.cluster, err = cluster.New(p.scheduler, p.storage, nodes...)
	c.Assert(err, check.IsNil)
	opts := provision.AddPoolOptions{Name: "pool1"}
	err = provision.AddPool(opts)
	c.Assert(err, check.IsNil)
	err = provision.AddTeamsToPool("pool1", []string{"team1"})
	c.Assert(err, check.IsNil)
	variation := []int{10, 20, 30, 40, 50, 100}
	maxContainers := 40
	for i := 0; i < maxContainers; i++ {
		appName := fmt.Sprintf("myapp-%d", i)
		err = s.newFakeImage(p, "tsuru/app-"+appName, nil)
		c.Assert(err, check.IsNil)
		appInstance := provisiontest.NewFakeApp(appName, "python", 0)
		defer p.Destroy(appInstance)
		p.Provision(appInstance)
		imageId, err := appCurrentImageName(appInstance.GetName())
		c.Assert(err, check.IsNil)
		var chosenNode string
		for j := range variation {
			if i < (maxContainers*variation[j])/100 {
				chosenNode = nodeHosts[j]
				break
			}
		}
		args := changeUnitsPipelineArgs{
			app:         appInstance,
			toAdd:       map[string]*containersToAdd{"web": {Quantity: 6}},
			imageId:     imageId,
			provisioner: p,
			toHost:      chosenNode,
		}
		pipeline := action.NewPipeline(
			&provisionAddUnitsToHost,
			&bindAndHealthcheck,
			&addNewRoutes,
			&updateAppImage,
		)
		err = pipeline.Execute(args)
		c.Assert(err, check.IsNil)
		appStruct := &app.App{
			Name:      appInstance.GetName(),
			TeamOwner: "team1",
			Pool:      "pool1",
		}
		err = s.storage.Apps().Insert(appStruct)
		c.Assert(err, check.IsNil)
	}
	buf := safe.NewBuffer(nil)
	cloneProv, err := p.rebalanceContainersByFilter(buf, []string{}, map[string]string{"pool": "pool1"}, false)
	c.Assert(err, check.IsNil)
	c.Assert(cloneProv.cluster.Healer, check.Equals, p.cluster.Healer)
	for i := range nodeHosts {
		conts, err := p.listContainersByHost(nodeHosts[i])
		c.Assert(err, check.IsNil)
		c.Logf("containers in %q: %d", nodeHosts[i], len(conts))
		c.Check(len(conts) >= 39 || len(conts) <= 41, check.Equals, true)
	}
}
<|endoftext|>