<commit_before>package godless

import (
	"bytes"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"

	ipfs "github.com/ipfs/go-ipfs-api"
	"github.com/pkg/errors"
)

type IPFSPath string

func (path IPFSPath) Path() string {
	return string(path)
}

func castIPFSPath(addr RemoteStoreAddress) IPFSPath {
	path, ok := addr.(IPFSPath)

	if !ok {
		panic("addr was not IPFSPath")
	}

	return path
}

type IPFSRecord struct {
	Namespace Namespace
}

func makeIpfsRecord(namespace Namespace) *IPFSRecord {
	return &IPFSRecord{
		Namespace: namespace,
	}
}

func (record *IPFSRecord) encode(w io.Writer) error {
	return EncodeNamespace(record.Namespace, w)
}

func (record *IPFSRecord) decode(r io.Reader) error {
	ns, err := DecodeNamespace(r)

	if err != nil {
		return err
	}

	record.Namespace = ns
	return nil
}

type encoder interface {
	encode(io.Writer) error
}

type decoder interface {
	decode(io.Reader) error
}

type IPFSIndex struct {
	Index RemoteNamespaceIndex
}

func makeIpfsIndex(index RemoteNamespaceIndex) *IPFSIndex {
	return &IPFSIndex{
		Index: index,
	}
}

func (index *IPFSIndex) encode(w io.Writer) error {
	return EncodeIndex(index.Index, w)
}

func (index *IPFSIndex) decode(r io.Reader) error {
	dx, err := DecodeIndex(r)

	if err != nil {
		return err
	}

	index.Index = dx
	return nil
}

// TODO Don't use Shell directly - invent an interface.  This would enable mocking.
type IPFSPeer struct {
	Url    string
	Client *http.Client
	Shell  *ipfs.Shell
	MyKey  RemoteStoreAddress
}

func MakeIPFSPeer(url string) RemoteStore {
	peer := &IPFSPeer{
		Url:    url,
		Client: defaultHttpClient(),
	}

	return peer
}

func (peer *IPFSPeer) Connect() error {
	peer.Shell = ipfs.NewShellWithClient(peer.Url, peer.Client)

	if !peer.Shell.IsUp() {
		return fmt.Errorf("IPFSPeer is not up at '%v'", peer.Url)
	}

	return nil
}

func (peer *IPFSPeer) Disconnect() error {
	// Nothing to do.
	return nil
}

func (peer *IPFSPeer) DereferenceIndex(name RemoteStoreAddress) (RemoteNamespaceIndex, error) {
	const failMsg = "IPFSPeer.DereferenceIndex failed"

	id := castIPFSPath(name)
	resolved, resolveErr := peer.Shell.Resolve(string(id))

	if resolveErr != nil {
		return EMPTY_INDEX, errors.Wrap(resolveErr, failMsg)
	}

	indexAddr := IPFSPath(resolved)
	index, catErr := peer.CatIndex(indexAddr)

	if catErr != nil {
		return EMPTY_INDEX, errors.Wrap(catErr, failMsg)
	}

	return index, nil
}

func (peer *IPFSPeer) UpdateIndex(index RemoteNamespaceIndex) (RemoteStoreAddress, error) {
	const failMsg = "IPFSPeer.AddIndex failed"

	chunk := makeIpfsIndex(index)

	path, err := peer.add(chunk)

	if err != nil {
		return nil, errors.Wrap(err, failMsg)
	}

	key, keyErr := peer.GetMyKey()

	if keyErr != nil {
		return nil, errors.Wrap(err, failMsg)
	}

	pubErr := peer.Shell.Publish(key.Path(), path.Path())

	if pubErr != nil {
		return nil, errors.Wrap(pubErr, failMsg)
	}

	loginfo("Published '%v' on key '%v'", path, key)

	return path, nil
}

func (peer *IPFSPeer) GetMyKey() (RemoteStoreAddress, error) {
	const failMsg = "IPFSPeer.GetMyKey failed"
	if peer.MyKey == nil {
		const noPeer = ""
		peerInfo, err := peer.Shell.FindPeer(noPeer)

		if err != nil {
			return nil, errors.Wrap(err, failMsg)
		}

		peer.MyKey = IPFSPath(peerInfo.ID)
	}

	return peer.MyKey, nil
}

func (peer *IPFSPeer) CatIndex(addr RemoteStoreAddress) (RemoteNamespaceIndex, error) {
	path := castIPFSPath(addr)

	chunk := &IPFSIndex{}
	caterr := peer.cat(path, chunk)

	if caterr != nil {
		return EMPTY_INDEX, errors.Wrap(caterr, "IPFSPeer.CatNamespace failed")
	}

	return chunk.Index, nil
}

func (peer *IPFSPeer) AddNamespace(record RemoteNamespaceRecord) (RemoteStoreAddress, error) {
	chunk := makeIpfsRecord(record.Namespace)

	path, err := peer.add(chunk)

	if err != nil {
		return nil, errors.Wrap(err, "IPFSPeer.AddNamespace failed")
	}

	return path, nil
}

func (peer *IPFSPeer) CatNamespace(addr RemoteStoreAddress) (RemoteNamespaceRecord, error) {
	path := castIPFSPath(addr)

	chunk := &IPFSRecord{}
	caterr := peer.cat(path, chunk)

	if caterr != nil {
		return EMPTY_RECORD, errors.Wrap(caterr, "IPFSPeer.CatNamespace failed")
	}

	record := RemoteNamespaceRecord{Namespace: chunk.Namespace}
	return record, nil
}

func (peer *IPFSPeer) add(chunk encoder) (IPFSPath, error) {
	const failMsg = "IPFSPeer.add failed"
	buff := &bytes.Buffer{}
	err := chunk.encode(buff)

	if err != nil {
		return "", errors.Wrap(err, failMsg)
	}

	path, sherr := peer.Shell.Add(buff)

	if sherr != nil {
		return "", errors.Wrap(err, failMsg)
	}

	return IPFSPath(path), nil
}

func (peer *IPFSPeer) cat(path IPFSPath, out decoder) error {
	const failMsg = "IPFSPeer.cat failed"
	reader, err := peer.Shell.Cat(string(path))

	if err != nil {
		return errors.Wrap(err, failMsg)
	}

	defer reader.Close()

	err = out.decode(reader)

	if err != nil {
		return errors.Wrap(err, failMsg)
	}

	// According to IPFS binding docs we must drain the reader.
	remainder, drainerr := ioutil.ReadAll(reader)

	if drainerr != nil {
		logwarn("error draining reader: %v", drainerr)
	}

	if len(remainder) != 0 {
		logwarn("remaining bits after gob: %v", remainder)
	}

	return nil
}
<commit_msg>Fix self key discovery<commit_after>package godless

import (
	"bytes"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"

	ipfs "github.com/ipfs/go-ipfs-api"
	"github.com/pkg/errors"
)

type IPFSPath string

func (path IPFSPath) Path() string {
	return string(path)
}

func castIPFSPath(addr RemoteStoreAddress) IPFSPath {
	path, ok := addr.(IPFSPath)

	if !ok {
		panic("addr was not IPFSPath")
	}

	return path
}

type IPFSRecord struct {
	Namespace Namespace
}

func makeIpfsRecord(namespace Namespace) *IPFSRecord {
	return &IPFSRecord{
		Namespace: namespace,
	}
}

func (record *IPFSRecord) encode(w io.Writer) error {
	return EncodeNamespace(record.Namespace, w)
}

func (record *IPFSRecord) decode(r io.Reader) error {
	ns, err := DecodeNamespace(r)

	if err != nil {
		return err
	}

	record.Namespace = ns
	return nil
}

type encoder interface {
	encode(io.Writer) error
}

type decoder interface {
	decode(io.Reader) error
}

type IPFSIndex struct {
	Index RemoteNamespaceIndex
}

func makeIpfsIndex(index RemoteNamespaceIndex) *IPFSIndex {
	return &IPFSIndex{
		Index: index,
	}
}

func (index *IPFSIndex) encode(w io.Writer) error {
	return EncodeIndex(index.Index, w)
}

func (index *IPFSIndex) decode(r io.Reader) error {
	dx, err := DecodeIndex(r)

	if err != nil {
		return err
	}

	index.Index = dx
	return nil
}

// TODO Don't use Shell directly - invent an interface.  This would enable mocking.
type IPFSPeer struct {
	Url    string
	Client *http.Client
	Shell  *ipfs.Shell
	MyKey  RemoteStoreAddress
}

func MakeIPFSPeer(url string) RemoteStore {
	peer := &IPFSPeer{
		Url:    url,
		Client: defaultHttpClient(),
	}

	return peer
}

func (peer *IPFSPeer) Connect() error {
	peer.Shell = ipfs.NewShellWithClient(peer.Url, peer.Client)

	if !peer.Shell.IsUp() {
		return fmt.Errorf("IPFSPeer is not up at '%v'", peer.Url)
	}

	return nil
}

func (peer *IPFSPeer) Disconnect() error {
	// Nothing to do.
	return nil
}

func (peer *IPFSPeer) DereferenceIndex(name RemoteStoreAddress) (RemoteNamespaceIndex, error) {
	const failMsg = "IPFSPeer.DereferenceIndex failed"

	id := castIPFSPath(name)
	resolved, resolveErr := peer.Shell.Resolve(string(id))

	if resolveErr != nil {
		return EMPTY_INDEX, errors.Wrap(resolveErr, failMsg)
	}

	indexAddr := IPFSPath(resolved)
	index, catErr := peer.CatIndex(indexAddr)

	if catErr != nil {
		return EMPTY_INDEX, errors.Wrap(catErr, failMsg)
	}

	return index, nil
}

func (peer *IPFSPeer) UpdateIndex(index RemoteNamespaceIndex) (RemoteStoreAddress, error) {
	const failMsg = "IPFSPeer.UpdateIndex failed"

	chunk := makeIpfsIndex(index)

	path, err := peer.add(chunk)

	if err != nil {
		return nil, errors.Wrap(err, failMsg)
	}

	key, keyErr := peer.GetMyKey()

	if keyErr != nil {
		return nil, errors.Wrap(keyErr, failMsg)
	}

	pubErr := peer.Shell.Publish(key.Path(), path.Path())

	if pubErr != nil {
		return nil, errors.Wrap(pubErr, failMsg)
	}

	loginfo("Published '%v' on key '%v'", path, key)

	return path, nil
}

func (peer *IPFSPeer) GetMyKey() (RemoteStoreAddress, error) {
	const failMsg = "IPFSPeer.GetMyKey failed"
	if peer.MyKey == nil {
		idInfo, err := peer.Shell.ID()

		if err != nil {
			return nil, errors.Wrap(err, failMsg)
		}

		peer.MyKey = IPFSPath(idInfo.ID)

		loginfo("Found self peer ID: %v", peer.MyKey)
	}

	return peer.MyKey, nil
}

func (peer *IPFSPeer) CatIndex(addr RemoteStoreAddress) (RemoteNamespaceIndex, error) {
	path := castIPFSPath(addr)

	chunk := &IPFSIndex{}
	caterr := peer.cat(path, chunk)

	if caterr != nil {
		return EMPTY_INDEX, errors.Wrap(caterr, "IPFSPeer.CatNamespace failed")
	}

	return chunk.Index, nil
}

func (peer *IPFSPeer) AddNamespace(record RemoteNamespaceRecord) (RemoteStoreAddress, error) {
	chunk := makeIpfsRecord(record.Namespace)

	path, err := peer.add(chunk)

	if err != nil {
		return nil, errors.Wrap(err, "IPFSPeer.AddNamespace failed")
	}

	return path, nil
}

func (peer *IPFSPeer) CatNamespace(addr RemoteStoreAddress) (RemoteNamespaceRecord, error) {
	path := castIPFSPath(addr)

	chunk := &IPFSRecord{}
	caterr := peer.cat(path, chunk)

	if caterr != nil {
		return EMPTY_RECORD, errors.Wrap(caterr, "IPFSPeer.CatNamespace failed")
	}

	record := RemoteNamespaceRecord{Namespace: chunk.Namespace}
	return record, nil
}

func (peer *IPFSPeer) add(chunk encoder) (IPFSPath, error) {
	const failMsg = "IPFSPeer.add failed"
	buff := &bytes.Buffer{}
	err := chunk.encode(buff)

	if err != nil {
		return "", errors.Wrap(err, failMsg)
	}

	path, sherr := peer.Shell.Add(buff)

	if sherr != nil {
		return "", errors.Wrap(err, failMsg)
	}

	return IPFSPath(path), nil
}

func (peer *IPFSPeer) cat(path IPFSPath, out decoder) error {
	const failMsg = "IPFSPeer.cat failed"
	reader, err := peer.Shell.Cat(string(path))

	if err != nil {
		return errors.Wrap(err, failMsg)
	}

	defer reader.Close()

	err = out.decode(reader)

	if err != nil {
		return errors.Wrap(err, failMsg)
	}

	// According to IPFS binding docs we must drain the reader.
	remainder, drainerr := ioutil.ReadAll(reader)

	if drainerr != nil {
		logwarn("error draining reader: %v", drainerr)
	}

	if len(remainder) != 0 {
		logwarn("remaining bits after gob: %v", remainder)
	}

	return nil
}
<|endoftext|>


<commit_before>package shell

import (
	"encoding/json"
)

// Publish updates a mutable name to point to a given value
func (s *Shell) Publish(node string, value string) error {
	args := []string{value}
	if node != "" {
		args = []string{node, value}
	}

	resp, err := s.newRequest("name/publish", args...).Send(s.httpcli)
	if err != nil {
		return err
	}
	defer resp.Close()

	return resp.Error
}

func (s *Shell) Resolve(id string) (string, error) {
	resp, err := s.newRequest("name/resolve", id).Send(s.httpcli)
	if err != nil {
		return "", err
	}
	defer resp.Close()

	if resp.Error != nil {
		return "", resp.Error
	}

	var out struct{ Path string }
	err = json.NewDecoder(resp.Output).Decode(&out)
	if err != nil {
		return "", err
	}

	return out.Path, nil
}
<commit_msg>Return *Error only if it's not nil in Publish<commit_after>package shell

import (
	"encoding/json"
)

// Publish updates a mutable name to point to a given value
func (s *Shell) Publish(node string, value string) error {
	args := []string{value}
	if node != "" {
		args = []string{node, value}
	}

	resp, err := s.newRequest("name/publish", args...).Send(s.httpcli)
	if err != nil {
		return err
	}
	defer resp.Close()

	if resp.Error != nil {
		return resp.Error
	}

	return nil
}

func (s *Shell) Resolve(id string) (string, error) {
	resp, err := s.newRequest("name/resolve", id).Send(s.httpcli)
	if err != nil {
		return "", err
	}
	defer resp.Close()

	if resp.Error != nil {
		return "", resp.Error
	}

	var out struct{ Path string }
	err = json.NewDecoder(resp.Output).Decode(&out)
	if err != nil {
		return "", err
	}

	return out.Path, nil
}
<|endoftext|>


<commit_before>// Copyright (c) Ilia Kravets, 2015. All rights reserved. PROVIDED "AS IS"
// WITHOUT ANY WARRANTY, EXPRESS OR IMPLIED. See LICENSE file for details.

package exch

import (
	"bytes"
	"encoding/binary"
	"fmt"
	"log"
	"net"
	"time"

	"github.com/lunixbochs/struc"

	"my/errs"

	"my/ev/exch/sbtcp"
)

func NewNasdaqExchangeSimulatorServer() ExchangeSimulator {
	return &exchangeNasdaq{
		glimpse: &glimpseServer{
			laddr:          ":16001",
			snapshotSeqNum: 5,
		},
		replay: &replayServer{
			laddr:        ":17001",
			sleepEnabled: true,
		},
		mcast: &mcastServer{
			laddr: "10.2.0.5:0",
			raddr: "233.54.12.1:18001",
			seq:   1000,
			pps:   1,
		},
	}
}

type exchangeNasdaq struct {
	glimpse *glimpseServer
	replay  *replayServer
	mcast   *mcastServer
}

func (e *exchangeNasdaq) Run() {
	go e.glimpse.run()
	go e.replay.run()
	go e.mcast.run()
	log.Println("started")
	select {}
}

type glimpseServer struct {
	laddr          string
	snapshotSeqNum int
}

func (s *glimpseServer) run() {
	l, err := net.Listen("tcp", s.laddr)
	errs.CheckE(err)
	defer l.Close()
	for {
		conn, err := l.Accept()
		errs.CheckE(err)
		log.Printf("accepted %s -> %s \n", conn.RemoteAddr(), conn.LocalAddr())
		go s.handleClient(conn)
	}
}
func (s *glimpseServer) handleClient(conn net.Conn) {
	defer conn.Close()
	m, err := sbtcp.ReadMessage(conn)
	errs.CheckE(err)
	log.Printf("got %#v\n", m)
	lr := m.(*sbtcp.MessageLoginRequest)
	errs.Check(lr != nil)

	la := sbtcp.MessageLoginAccepted{
		Session:        "00TestSess",
		SequenceNumber: 1,
	}
	errs.CheckE(sbtcp.WriteMessage(conn, &la))
	log.Printf("glimpse send: %v\n", la)

	for i := 0; i < s.snapshotSeqNum; i++ {
		s.sendSeqData(conn, generateIttoMessage(i))
	}
	snap := fmt.Sprintf("M%020d", s.snapshotSeqNum)
	s.sendSeqData(conn, []byte(snap))
}
func (s *glimpseServer) sendSeqData(conn net.Conn, data []byte) {
	sd := sbtcp.MessageSequencedData{}
	sd.SetPayload(data)
	errs.CheckE(sbtcp.WriteMessage(conn, &sd))
	log.Printf("glimpse send: %v\n", sd)
}

type replayServer struct {
	laddr        string
	sleepEnabled bool
}

func (s *replayServer) run() {
	type moldudp64request struct {
		Session        string
		SequenceNumber uint64
		MessageCount   uint16
	}

	laddr, err := net.ResolveUDPAddr("udp", s.laddr)
	errs.CheckE(err)
	conn, err := net.ListenUDP("udp", laddr)
	errs.CheckE(err)
	defer conn.Close()
	buf := make([]byte, 20, 65536)
	for {
		n, addr, err := conn.ReadFromUDP(buf)
		errs.CheckE(err)
		if n != 20 {
			log.Printf("ignore wrong request from %s: %v\n", addr, buf)
			continue
		}
		req := &moldudp64request{
			Session:        string(buf[0:10]),
			SequenceNumber: binary.BigEndian.Uint64(buf[10:18]),
			MessageCount:   binary.BigEndian.Uint16(buf[18:20]),
		}
		go func() {
			const MAX_MESSAGES = (1500 - 34 - 20) / 40
			log.Printf("got request: %v\n", req)
			//num := int(req.SequenceNumber) - 10
			num := int(req.MessageCount)
			if num <= 0 {
				num = 1
			} else if num > MAX_MESSAGES {
				num = MAX_MESSAGES
			}
			resp := createMoldPacket(int(req.SequenceNumber), num)
			errs.Check(len(resp) < 1500-34)

			if s.sleepEnabled {
				sleep := time.Duration(250+2500/num) * time.Millisecond
				log.Printf("sleeping for %s\n", sleep)
				time.Sleep(sleep)
			}
			log.Printf("send response: %v\n", resp)
			n, err = conn.WriteToUDP(resp, addr)
			errs.CheckE(err)
			errs.Check(n == len(resp), n, len(resp))
		}()
	}
}

type mcastServer struct {
	laddr string
	raddr string
	seq   int
	pps   int
}

func (s *mcastServer) run() {
	errs.Check(s.pps != 0)
	laddr, err := net.ResolveUDPAddr("udp", s.laddr)
	errs.CheckE(err)
	raddr, err := net.ResolveUDPAddr("udp", s.raddr)
	errs.CheckE(err)
	conn, err := net.DialUDP("udp", laddr, raddr)
	for {
		p := createMoldPacket(s.seq, 1)
		log.Printf("send mcast: %v\n", p)
		n, err := conn.Write(p)
		errs.CheckE(err)
		errs.Check(n == len(p), n, len(p))

		delay := time.Duration(1000/s.pps) * time.Millisecond
		time.Sleep(delay)
		s.seq++
	}
	defer conn.Close()
}

func createMoldPacket(startSeqNum, count int) []byte {
	type moldUDP64 struct {
		Session        string `struc:"[10]byte"`
		SequenceNumber uint64
		MessageCount   uint16
	}
	type moldUDP64MessageBlock struct {
		MessageLength int16 `struc:"sizeof=Payload"`
		Payload       []uint8
	}

	errs.Check(startSeqNum >= 0)
	errs.Check(count >= 0 && count < 1000)
	mh := moldUDP64{
		Session:        "00TestSess",
		SequenceNumber: uint64(startSeqNum),
		MessageCount:   uint16(count),
	}
	var bb bytes.Buffer
	errs.CheckE(struc.Pack(&bb, &mh))
	for i := 0; i < count; i++ {
		mb := moldUDP64MessageBlock{
			Payload: generateIttoMessage(startSeqNum + i),
		}
		errs.CheckE(struc.Pack(&bb, &mb))
	}
	return bb.Bytes()
}

func generateIttoMessage(seqNum int) []byte {
	type ittoMessageOptionsTrade struct {
		Type      byte
		Timestamp uint32
		Side      byte
		OId       uint32
		Cross     uint32
		Match     uint32
		Price     uint32
		Size      uint32
	}
	type ittoMessageOptionDirectory struct {
		Type             byte
		Timestamp        uint32
		OId              uint32
		Symbol           [6]byte
		ExpirationYear   byte
		ExpirationMonth  byte
		ExpirationDay    byte
		StrikePrice      uint32
		OType            byte
		Source           uint8
		UnderlyingSymbol [13]byte
		ClosingType      byte
		Tradable         byte
		MPV              byte
	}
	type ittoMessageAddOrder struct {
		Type      byte
		Timestamp uint32
		RefNumD   uint32
		Side      byte
		OId       uint32
		Price     uint32
		Size      uint32
	}

	errs.Check(seqNum >= 0)
	var bb bytes.Buffer
	switch seqNum % 3 {
	case 0:
		m := ittoMessageOptionDirectory{
			Type: 'R',
			OId:  uint32(seqNum / 3),
		}
		errs.CheckE(struc.Pack(&bb, &m))
	case 1:
		m := ittoMessageOptionsTrade{
			Type: 'P',
			OId:  uint32(seqNum / 3),
		}
		errs.CheckE(struc.Pack(&bb, &m))
	case 2:
		m := ittoMessageAddOrder{
			Type:    'A',
			RefNumD: uint32(seqNum),
			Side:    'B',
			OId:     uint32(seqNum/9) + 0x10000,
			Price:   uint32(seqNum % 10 * 1000),
			Size:    uint32(seqNum % 10),
		}
		errs.CheckE(struc.Pack(&bb, &m))
	}
	return bb.Bytes()
}
<commit_msg>exch:nasdaq: return error from createMoldPacket()<commit_after>// Copyright (c) Ilia Kravets, 2015. All rights reserved. PROVIDED "AS IS"
// WITHOUT ANY WARRANTY, EXPRESS OR IMPLIED. See LICENSE file for details.

package exch

import (
	"bytes"
	"encoding/binary"
	"fmt"
	"log"
	"net"
	"time"

	"github.com/lunixbochs/struc"

	"my/errs"

	"my/ev/exch/sbtcp"
)

func NewNasdaqExchangeSimulatorServer() ExchangeSimulator {
	return &exchangeNasdaq{
		glimpse: &glimpseServer{
			laddr:          ":16001",
			snapshotSeqNum: 5,
		},
		replay: &replayServer{
			laddr:        ":17001",
			sleepEnabled: true,
		},
		mcast: &mcastServer{
			laddr: "10.2.0.5:0",
			raddr: "233.54.12.1:18001",
			seq:   1000,
			pps:   1,
		},
	}
}

type exchangeNasdaq struct {
	glimpse *glimpseServer
	replay  *replayServer
	mcast   *mcastServer
}

func (e *exchangeNasdaq) Run() {
	go e.glimpse.run()
	go e.replay.run()
	go e.mcast.run()
	log.Println("started")
	select {}
}

type glimpseServer struct {
	laddr          string
	snapshotSeqNum int
}

func (s *glimpseServer) run() {
	l, err := net.Listen("tcp", s.laddr)
	errs.CheckE(err)
	defer l.Close()
	for {
		conn, err := l.Accept()
		errs.CheckE(err)
		log.Printf("accepted %s -> %s \n", conn.RemoteAddr(), conn.LocalAddr())
		go s.handleClient(conn)
	}
}
func (s *glimpseServer) handleClient(conn net.Conn) {
	defer conn.Close()
	m, err := sbtcp.ReadMessage(conn)
	errs.CheckE(err)
	log.Printf("got %#v\n", m)
	lr := m.(*sbtcp.MessageLoginRequest)
	errs.Check(lr != nil)

	la := sbtcp.MessageLoginAccepted{
		Session:        "00TestSess",
		SequenceNumber: 1,
	}
	errs.CheckE(sbtcp.WriteMessage(conn, &la))
	log.Printf("glimpse send: %v\n", la)

	for i := 0; i < s.snapshotSeqNum; i++ {
		s.sendSeqData(conn, generateIttoMessage(i))
	}
	snap := fmt.Sprintf("M%020d", s.snapshotSeqNum)
	s.sendSeqData(conn, []byte(snap))
}
func (s *glimpseServer) sendSeqData(conn net.Conn, data []byte) {
	sd := sbtcp.MessageSequencedData{}
	sd.SetPayload(data)
	errs.CheckE(sbtcp.WriteMessage(conn, &sd))
	log.Printf("glimpse send: %v\n", sd)
}

type replayServer struct {
	laddr        string
	sleepEnabled bool
}

func (s *replayServer) run() {
	type moldudp64request struct {
		Session        string
		SequenceNumber uint64
		MessageCount   uint16
	}

	laddr, err := net.ResolveUDPAddr("udp", s.laddr)
	errs.CheckE(err)
	conn, err := net.ListenUDP("udp", laddr)
	errs.CheckE(err)
	defer conn.Close()
	buf := make([]byte, 20, 65536)
	for {
		n, addr, err := conn.ReadFromUDP(buf)
		errs.CheckE(err)
		if n != 20 {
			log.Printf("ignore wrong request from %s: %v\n", addr, buf)
			continue
		}
		req := &moldudp64request{
			Session:        string(buf[0:10]),
			SequenceNumber: binary.BigEndian.Uint64(buf[10:18]),
			MessageCount:   binary.BigEndian.Uint16(buf[18:20]),
		}
		go func() {
			const MAX_MESSAGES = (1500 - 34 - 20) / 40
			log.Printf("got request: %v\n", req)
			//num := int(req.SequenceNumber) - 10
			num := int(req.MessageCount)
			if num <= 0 {
				num = 1
			} else if num > MAX_MESSAGES {
				num = MAX_MESSAGES
			}
			resp, err := createMoldPacket(int(req.SequenceNumber), num)
			errs.CheckE(err)
			errs.Check(len(resp) < 1500-34)

			if s.sleepEnabled {
				sleep := time.Duration(250+2500/num) * time.Millisecond
				log.Printf("sleeping for %s\n", sleep)
				time.Sleep(sleep)
			}
			log.Printf("send response: %v\n", resp)
			n, err = conn.WriteToUDP(resp, addr)
			errs.CheckE(err)
			errs.Check(n == len(resp), n, len(resp))
		}()
	}
}

type mcastServer struct {
	laddr string
	raddr string
	seq   int
	pps   int
}

func (s *mcastServer) run() {
	errs.Check(s.pps != 0)
	laddr, err := net.ResolveUDPAddr("udp", s.laddr)
	errs.CheckE(err)
	raddr, err := net.ResolveUDPAddr("udp", s.raddr)
	errs.CheckE(err)
	conn, err := net.DialUDP("udp", laddr, raddr)
	errs.CheckE(err)
	for {
		p, err := createMoldPacket(s.seq, 1)
		errs.CheckE(err)
		log.Printf("send mcast: %v\n", p)
		n, err := conn.Write(p)
		errs.CheckE(err)
		errs.Check(n == len(p), n, len(p))

		delay := time.Duration(1000/s.pps) * time.Millisecond
		time.Sleep(delay)
		s.seq++
	}
	defer conn.Close()
}

func createMoldPacket(startSeqNum, count int) (bs []byte, err error) {
	defer errs.PassE(&err)
	type moldUDP64 struct {
		Session        string `struc:"[10]byte"`
		SequenceNumber uint64
		MessageCount   uint16
	}
	type moldUDP64MessageBlock struct {
		MessageLength int16 `struc:"sizeof=Payload"`
		Payload       []uint8
	}

	errs.Check(startSeqNum >= 0)
	errs.Check(count >= 0 && count < 1000)
	mh := moldUDP64{
		Session:        "00TestSess",
		SequenceNumber: uint64(startSeqNum),
		MessageCount:   uint16(count),
	}
	var bb bytes.Buffer
	errs.CheckE(struc.Pack(&bb, &mh))
	for i := 0; i < count; i++ {
		mb := moldUDP64MessageBlock{
			Payload: generateIttoMessage(startSeqNum + i),
		}
		errs.CheckE(struc.Pack(&bb, &mb))
	}
	bs = bb.Bytes()
	return
}

func generateIttoMessage(seqNum int) []byte {
	type ittoMessageOptionsTrade struct {
		Type      byte
		Timestamp uint32
		Side      byte
		OId       uint32
		Cross     uint32
		Match     uint32
		Price     uint32
		Size      uint32
	}
	type ittoMessageOptionDirectory struct {
		Type             byte
		Timestamp        uint32
		OId              uint32
		Symbol           [6]byte
		ExpirationYear   byte
		ExpirationMonth  byte
		ExpirationDay    byte
		StrikePrice      uint32
		OType            byte
		Source           uint8
		UnderlyingSymbol [13]byte
		ClosingType      byte
		Tradable         byte
		MPV              byte
	}
	type ittoMessageAddOrder struct {
		Type      byte
		Timestamp uint32
		RefNumD   uint32
		Side      byte
		OId       uint32
		Price     uint32
		Size      uint32
	}

	errs.Check(seqNum >= 0)
	var bb bytes.Buffer
	switch seqNum % 3 {
	case 0:
		m := ittoMessageOptionDirectory{
			Type: 'R',
			OId:  uint32(seqNum / 3),
		}
		errs.CheckE(struc.Pack(&bb, &m))
	case 1:
		m := ittoMessageOptionsTrade{
			Type: 'P',
			OId:  uint32(seqNum / 3),
		}
		errs.CheckE(struc.Pack(&bb, &m))
	case 2:
		m := ittoMessageAddOrder{
			Type:    'A',
			RefNumD: uint32(seqNum),
			Side:    'B',
			OId:     uint32(seqNum/9) + 0x10000,
			Price:   uint32(seqNum % 10 * 1000),
			Size:    uint32(seqNum % 10),
		}
		errs.CheckE(struc.Pack(&bb, &m))
	}
	return bb.Bytes()
}
<|endoftext|>


<commit_before><commit_msg>Have docker always callback For now, assume it worked<commit_after><|endoftext|>


<commit_before>package internal

import (
	"math"
	"testing"
	"time"

	"github.com/AutoRoute/node/types"
)

// Make sure that bandwidthEstimator works in the most basic capacity.
func TestBasic(t *testing.T) {
	// Channel that we'll send information about outgoing packets down.
	outgoing := make(chan routingDecision)
	estimator := newBandwidthEstimator(outgoing)

	// Fake node addresses.
	node1 := types.NodeAddress("A")
	node2 := types.NodeAddress("B")
	src := types.NodeAddress("src")

	// Fake packet that we'll say we sent.
	packet1 := types.Packet{node1, 1, ""}
	packet2 := types.Packet{node2, 1, ""}

	decision1 := newRoutingDecision(packet1, src, node1, 10)
	decision2 := newRoutingDecision(packet2, src, node2, 10)
	// Send a few packets in quick succession from various sources.
	for i := 0; i < 10; i++ {
		outgoing <- decision1
		time.Sleep(10 * time.Millisecond)
		outgoing <- decision2
		time.Sleep(10 * time.Millisecond)
	}

	// Since everything took the same amount of time, the weights should be equal.
	nodes := make([]types.NodeAddress, 2)
	nodes[0] = node1
	nodes[1] = node2
	weight_total := float64(0)
	for _, weight := range estimator.GetWeights(nodes) {
		if math.Abs(weight-0.5) > 0.004 {
			t.Fatalf("Expected weight of 0.5, got weight of %f\n", weight)
		}
		weight_total += weight
	}

	// The weights should also approximately add to 1.
	if math.Abs(weight_total-1.0) > 0.004 {
		t.Fatalf("Expected total weight of 1.0, got %f\n", weight_total)
	}
}

// A slightly more complicated test with varying bandwidths.
func TestDifferingBandwidths(t *testing.T) {
	// Channel that we'll send information about outgoing packets down.
	outgoing := make(chan routingDecision)
	estimator := newBandwidthEstimator(outgoing)

	// Fake node addresses.
	node1 := types.NodeAddress("A")
	node2 := types.NodeAddress("B")
	src := types.NodeAddress("src")

	// Fake packet that we'll say we sent.
	packet1 := types.Packet{node1, 1, ""}
	packet2 := types.Packet{node2, 1, ""}

	decision1 := newRoutingDecision(packet1, src, node1, 10)
	decision2 := newRoutingDecision(packet2, src, node2, 10)
	for i := 0; i < 10; i++ {
		outgoing <- decision1
		time.Sleep(10 * time.Millisecond)

		outgoing <- decision2
		time.Sleep(20 * time.Millisecond)
	}

	// The weights should be split about 2/3 and 1/3.
	nodes := make([]types.NodeAddress, 2)
	nodes[0] = node1
	nodes[1] = node2
	weight_total := float64(0)
	for _, weight := range estimator.GetWeights(nodes) {
		ok := false
		if math.Abs(weight-0.666) <= 0.004 {
			ok = true
		} else if math.Abs(weight-0.333) <= 0.004 {
			ok = true
		}
		if !ok {
			t.Fatalf("Got weight of %f\n", weight)
		}
		weight_total += weight
	}

	// The weights should also approximately add to 1.
	if math.Abs(weight_total-1.0) > 0.004 {
		t.Fatalf("Expected total weight of 1.0, got %f\n", weight_total)
	}
}

// A test to make sure it can handle weights with only partial information.
func TestPartialData(t *testing.T) {
	outgoing := make(chan routingDecision)
	estimator := newBandwidthEstimator(outgoing)

	// Try to get weights for some nodes that don't exist.
	node1 := types.NodeAddress("A")
	node2 := types.NodeAddress("B")
	node3 := types.NodeAddress("C")
	src := types.NodeAddress("src")
	nodes := make([]types.NodeAddress, 3)
	nodes[0] = node1
	nodes[1] = node2
	nodes[2] = node3

	// Fake packets that we'll say we sent.
	packet1 := types.Packet{node1, 1, ""}
	packet2 := types.Packet{node2, 1, ""}

	decision1 := newRoutingDecision(packet1, src, node1, 10)
	decision2 := newRoutingDecision(packet2, src, node2, 10)

	// We should see equally distributed weights, since we have no data on
	// anything.
	weight_total := float64(0)
	for _, weight := range estimator.GetWeights(nodes) {
		if math.Abs(weight-0.333) > 0.004 {
			t.Fatalf("Expected weight of 0.333, got weight of %f\n", weight)
		}
		weight_total += weight
	}

	// The weights should also approximately add to 1.
	if weight_total != 1.0 {
		t.Fatalf("Expected total weight of 1.0, got %f\n", weight_total)
	}

	// Add sufficient data for two of the nodes.
	for i := 0; i < 10; i++ {
		outgoing <- decision1
		time.Sleep(10 * time.Millisecond)

		outgoing <- decision2
		time.Sleep(20 * time.Millisecond)
	}

	// The way this works is that for any node for which it lacks sufficient data,
	// it will just take 1 / number of nodes as the weight, and will calculate
	// everything else within 1 - that number.
	weight_total = 0
	for _, weight := range estimator.GetWeights(nodes) {
		ok := false
		if math.Abs(weight-0.222) <= 0.004 {
			ok = true
		} else if math.Abs(weight-0.444) <= 0.004 {
			ok = true
		} else if math.Abs(weight-0.333) <= 0.004 {
			ok = true
		}
		if !ok {
			t.Fatalf("Got weight of %f\n", weight)
		}

		weight_total += weight
	}

	// The weights should also approximately add to 1.
	if math.Abs(weight_total-1.0) > 0.004 {
		t.Fatalf("Expected total weight of 1.0, got %f\n", weight_total)
	}
}
<commit_msg>Fix overly tight bounds on the bandwidth test.<commit_after>package internal

import (
	"math"
	"testing"
	"time"

	"github.com/AutoRoute/node/types"
)

// Make sure that bandwidthEstimator works in the most basic capacity.
func TestBasic(t *testing.T) {
	// Channel that we'll send information about outgoing packets down.
	outgoing := make(chan routingDecision)
	estimator := newBandwidthEstimator(outgoing)

	// Fake node addresses.
	node1 := types.NodeAddress("A")
	node2 := types.NodeAddress("B")
	src := types.NodeAddress("src")

	// Fake packet that we'll say we sent.
	packet1 := types.Packet{node1, 1, ""}
	packet2 := types.Packet{node2, 1, ""}

	decision1 := newRoutingDecision(packet1, src, node1, 10)
	decision2 := newRoutingDecision(packet2, src, node2, 10)
	// Send a few packets in quick succession from various sources.
	for i := 0; i < 10; i++ {
		outgoing <- decision1
		time.Sleep(10 * time.Millisecond)
		outgoing <- decision2
		time.Sleep(10 * time.Millisecond)
	}

	// Since everything took the same amount of time, the weights should be equal.
	nodes := make([]types.NodeAddress, 2)
	nodes[0] = node1
	nodes[1] = node2
	weight_total := float64(0)
	for _, weight := range estimator.GetWeights(nodes) {
		if math.Abs(weight-0.5) > 0.1 {
			t.Fatalf("Expected weight of 0.5, got weight of %f\n", weight)
		}
		weight_total += weight
	}

	// The weights should also approximately add to 1.
	if math.Abs(weight_total-1.0) > 0.1 {
		t.Fatalf("Expected total weight of 1.0, got %f\n", weight_total)
	}
}

// A slightly more complicated test with varying bandwidths.
func TestDifferingBandwidths(t *testing.T) {
	// Channel that we'll send information about outgoing packets down.
	outgoing := make(chan routingDecision)
	estimator := newBandwidthEstimator(outgoing)

	// Fake node addresses.
	node1 := types.NodeAddress("A")
	node2 := types.NodeAddress("B")
	src := types.NodeAddress("src")

	// Fake packet that we'll say we sent.
	packet1 := types.Packet{node1, 1, ""}
	packet2 := types.Packet{node2, 1, ""}

	decision1 := newRoutingDecision(packet1, src, node1, 10)
	decision2 := newRoutingDecision(packet2, src, node2, 10)
	for i := 0; i < 10; i++ {
		outgoing <- decision1
		time.Sleep(10 * time.Millisecond)

		outgoing <- decision2
		time.Sleep(20 * time.Millisecond)
	}

	// The weights should be split about 2/3 and 1/3.
	nodes := make([]types.NodeAddress, 2)
	nodes[0] = node1
	nodes[1] = node2
	weight_total := float64(0)
	for _, weight := range estimator.GetWeights(nodes) {
		ok := false
		if math.Abs(weight-0.666) <= 0.1 {
			ok = true
		} else if math.Abs(weight-0.333) <= 0.1 {
			ok = true
		}
		if !ok {
			t.Fatalf("Got weight of %f\n", weight)
		}
		weight_total += weight
	}

	// The weights should also approximately add to 1.
	if math.Abs(weight_total-1.0) > 0.1 {
		t.Fatalf("Expected total weight of 1.0, got %f\n", weight_total)
	}
}

// A test to make sure it can handle weights with only partial information.
func TestPartialData(t *testing.T) {
	outgoing := make(chan routingDecision)
	estimator := newBandwidthEstimator(outgoing)

	// Try to get weights for some nodes that don't exist.
	node1 := types.NodeAddress("A")
	node2 := types.NodeAddress("B")
	node3 := types.NodeAddress("C")
	src := types.NodeAddress("src")
	nodes := make([]types.NodeAddress, 3)
	nodes[0] = node1
	nodes[1] = node2
	nodes[2] = node3

	// Fake packets that we'll say we sent.
	packet1 := types.Packet{node1, 1, ""}
	packet2 := types.Packet{node2, 1, ""}

	decision1 := newRoutingDecision(packet1, src, node1, 10)
	decision2 := newRoutingDecision(packet2, src, node2, 10)

	// We should see equally distributed weights, since we have no data on
	// anything.
	weight_total := float64(0)
	for _, weight := range estimator.GetWeights(nodes) {
		if math.Abs(weight-0.333) > 0.1 {
			t.Fatalf("Expected weight of 0.333, got weight of %f\n", weight)
		}
		weight_total += weight
	}

	// The weights should also approximately add to 1.
	if weight_total != 1.0 {
		t.Fatalf("Expected total weight of 1.0, got %f\n", weight_total)
	}

	// Add sufficient data for two of the nodes.
	for i := 0; i < 10; i++ {
		outgoing <- decision1
		time.Sleep(10 * time.Millisecond)

		outgoing <- decision2
		time.Sleep(20 * time.Millisecond)
	}

	// The way this works is that for any node for which it lacks sufficient data,
	// it will just take 1 / number of nodes as the weight, and will calculate
	// everything else within 1 - that number.
	weight_total = 0
	for _, weight := range estimator.GetWeights(nodes) {
		ok := false
		if math.Abs(weight-0.222) <= 0.1 {
			ok = true
		} else if math.Abs(weight-0.444) <= 0.1 {
			ok = true
		} else if math.Abs(weight-0.333) <= 0.1 {
			ok = true
		}
		if !ok {
			t.Fatalf("Got weight of %f\n", weight)
		}

		weight_total += weight
	}

	// The weights should also approximately add to 1.
	if math.Abs(weight_total-1.0) > 0.1 {
		t.Fatalf("Expected total weight of 1.0, got %f\n", weight_total)
	}
}
<|endoftext|>


<commit_before>package cache

import (
	"fmt"
	"go/build"
	goimporter "go/importer"
	"go/token"
	"go/types"
	"os"
	"path/filepath"
	"strings"
	"sync"
	"time"

	"golang.org/x/tools/go/gcexportdata"
)

// We need to mangle go/build.Default to make gcimporter work as
// intended, so use a lock to protect against concurrent accesses.
var buildDefaultLock sync.Mutex

// Mu must be held while using the cache importer.
var Mu sync.Mutex

var importCache = importerCache{
	fset:    token.NewFileSet(),
	imports: make(map[string]importCacheEntry),
}

func NewImporter(ctx *PackedContext, filename string, fallbackToSource bool) types.ImporterFrom {
	importCache.clean()

	imp := &importer{
		ctx:              ctx,
		importerCache:    &importCache,
		fallbackToSource: fallbackToSource,
	}

	slashed := filepath.ToSlash(filename)
	i := strings.LastIndex(slashed, "/vendor/src/")
	if i < 0 {
		i = strings.LastIndex(slashed, "/src/")
	}
	if i > 0 {
		paths := filepath.SplitList(imp.ctx.GOPATH)

		gbroot := filepath.FromSlash(slashed[:i])
		gbvendor := filepath.Join(gbroot, "vendor")
		if SamePath(gbroot, imp.ctx.GOROOT) {
			goto Found
		}
		for _, path := range paths {
			if SamePath(path, gbroot) || SamePath(path, gbvendor) {
				goto Found
			}
		}

		imp.gbroot = gbroot
		imp.gbvendor = gbvendor
	Found:
	}

	return imp
}

type importer struct {
	*importerCache
	gbroot, gbvendor string
	ctx              *PackedContext
	fallbackToSource bool
}

type importerCache struct {
	fset    *token.FileSet
	imports map[string]importCacheEntry
}

type importCacheEntry struct {
	pkg   *types.Package
	mtime time.Time
}

func (i *importer) Import(importPath string) (*types.Package, error) {
	return i.ImportFrom(importPath, "", 0)
}

func (i *importer) ImportFrom(importPath, srcDir string, mode types.ImportMode) (*types.Package, error) {
	buildDefaultLock.Lock()
	defer buildDefaultLock.Unlock()

	origDef := build.Default
	defer func() { build.Default = origDef }()

	def := &build.Default
	// The gb root of a project can be used as a $GOPATH because it contains pkg/.
	def.GOPATH = i.ctx.GOPATH
	if i.gbroot != "" {
		def.GOPATH = i.gbroot
	}
	def.GOARCH = i.ctx.GOARCH
	def.GOOS = i.ctx.GOOS
	def.GOROOT = i.ctx.GOROOT
	def.CgoEnabled = i.ctx.CgoEnabled
	def.UseAllFiles = i.ctx.UseAllFiles
	def.Compiler = i.ctx.Compiler
	def.BuildTags = i.ctx.BuildTags
	def.ReleaseTags = i.ctx.ReleaseTags
	def.InstallSuffix = i.ctx.InstallSuffix
	def.SplitPathList = i.splitPathList
	def.JoinPath = i.joinPath

	filename, path := gcexportdata.Find(importPath, srcDir)
	entry, ok := i.imports[path]
	if filename == "" {
		// If there is no export data, check the cache.
		// TODO(rstambler): Develop a better heuristic for entry eviction.
		if ok && time.Since(entry.mtime) <= time.Minute*20 {
			return entry.pkg, nil
		}
		// If there is no cache entry and the user has configured the correct
		// setting, import and cache using the source importer.
		if i.fallbackToSource {
			pkg, err := goimporter.For("source", nil).Import(path)
			if pkg == nil {
				return nil, err
			}
			entry = importCacheEntry{pkg, time.Now()}
			i.imports[path] = entry
			return entry.pkg, nil
		}
		return nil, fmt.Errorf("no export data for %s", path)
	}

	// If there is export data for the package.
	fi, err := os.Stat(filename)
	if err != nil {
		return nil, err
	}
	if entry.mtime != fi.ModTime() {
		f, err := os.Open(filename)
		if err != nil {
			return nil, err
		}
		in, err := gcexportdata.NewReader(f)
		if err != nil {
			return nil, err
		}
		pkg, err := gcexportdata.Read(in, i.fset, make(map[string]*types.Package), path)
		if err != nil {
			return nil, err
		}
		entry = importCacheEntry{pkg, fi.ModTime()}
		i.imports[path] = entry
	}

	return entry.pkg, nil
}

// Delete random files to keep the cache at most 100 entries.
// Only call while holding the importer's mutex.
func (i *importerCache) clean() {
	for k := range i.imports {
		if len(i.imports) <= 100 {
			break
		}
		delete(i.imports, k)
	}
}

func (i *importer) splitPathList(list string) []string {
	res := filepath.SplitList(list)
	if i.gbroot != "" {
		res = append(res, i.gbroot, i.gbvendor)
	}
	return res
}

func (i *importer) joinPath(elem ...string) string {
	res := filepath.Join(elem...)

	if i.gbroot != "" {
		// Want to rewrite "$GBROOT/(vendor/)?pkg/$GOOS_$GOARCH(_)?"
		// into "$GBROOT/pkg/$GOOS-$GOARCH(-)?".
		// Note: gb doesn't use vendor/pkg.
		if gbrel, err := filepath.Rel(i.gbroot, res); err == nil {
			gbrel = filepath.ToSlash(gbrel)
			gbrel, _ = match(gbrel, "vendor/")
			if gbrel, ok := match(gbrel, fmt.Sprintf("pkg/%s_%s", i.ctx.GOOS, i.ctx.GOARCH)); ok {
				gbrel, hasSuffix := match(gbrel, "_")

				// Reassemble into result.
				if hasSuffix {
					gbrel = "-" + gbrel
				}
				gbrel = fmt.Sprintf("pkg/%s-%s/", i.ctx.GOOS, i.ctx.GOARCH) + gbrel
				gbrel = filepath.FromSlash(gbrel)
				res = filepath.Join(i.gbroot, gbrel)
			}
		}
	}
	return res
}

func match(s, prefix string) (string, bool) {
	rest := strings.TrimPrefix(s, prefix)
	return rest, len(rest) < len(s)
}
<commit_msg>internal/cache: fallback to default importer if export data not found<commit_after>package cache

import (
	"fmt"
	"go/build"
	goimporter "go/importer"
	"go/token"
	"go/types"
	"os"
	"path/filepath"
	"strings"
	"sync"
	"time"

	"golang.org/x/tools/go/gcexportdata"
)

// We need to mangle go/build.Default to make gcimporter work as
// intended, so use a lock to protect against concurrent accesses.
var buildDefaultLock sync.Mutex

// Mu must be held while using the cache importer.
var Mu sync.Mutex

var importCache = importerCache{
	fset:    token.NewFileSet(),
	imports: make(map[string]importCacheEntry),
}

func NewImporter(ctx *PackedContext, filename string, fallbackToSource bool) types.ImporterFrom {
	importCache.clean()

	imp := &importer{
		ctx:              ctx,
		importerCache:    &importCache,
		fallbackToSource: fallbackToSource,
	}

	slashed := filepath.ToSlash(filename)
	i := strings.LastIndex(slashed, "/vendor/src/")
	if i < 0 {
		i = strings.LastIndex(slashed, "/src/")
	}
	if i > 0 {
		paths := filepath.SplitList(imp.ctx.GOPATH)

		gbroot := filepath.FromSlash(slashed[:i])
		gbvendor := filepath.Join(gbroot, "vendor")
		if SamePath(gbroot, imp.ctx.GOROOT) {
			goto Found
		}
		for _, path := range paths {
			if SamePath(path, gbroot) || SamePath(path, gbvendor) {
				goto Found
			}
		}

		imp.gbroot = gbroot
		imp.gbvendor = gbvendor
	Found:
	}

	return imp
}

type importer struct {
	*importerCache
	gbroot, gbvendor string
	ctx              *PackedContext
	fallbackToSource bool
}

type importerCache struct {
	fset    *token.FileSet
	imports map[string]importCacheEntry
}

type importCacheEntry struct {
	pkg   *types.Package
	mtime time.Time
}

func (i *importer) Import(importPath string) (*types.Package, error) {
	return i.ImportFrom(importPath, "", 0)
}

func (i *importer) ImportFrom(importPath, srcDir string, mode types.ImportMode) (*types.Package, error) {
	buildDefaultLock.Lock()
	defer buildDefaultLock.Unlock()

	origDef := build.Default
	defer func() { build.Default = origDef }()

	def := &build.Default
	// The gb root of a project can be used as a $GOPATH because it contains pkg/.
	def.GOPATH = i.ctx.GOPATH
	if i.gbroot != "" {
		def.GOPATH = i.gbroot
	}
	def.GOARCH = i.ctx.GOARCH
	def.GOOS = i.ctx.GOOS
	def.GOROOT = i.ctx.GOROOT
	def.CgoEnabled = i.ctx.CgoEnabled
	def.UseAllFiles = i.ctx.UseAllFiles
	def.Compiler = i.ctx.Compiler
	def.BuildTags = i.ctx.BuildTags
	def.ReleaseTags = i.ctx.ReleaseTags
	def.InstallSuffix = i.ctx.InstallSuffix
	def.SplitPathList = i.splitPathList
	def.JoinPath = i.joinPath

	filename, path := gcexportdata.Find(importPath, srcDir)
	entry, ok := i.imports[path]
	if filename == "" {
		// If there is no export data, check the cache.
		// TODO(rstambler): Develop a better heuristic for entry eviction.
		if ok && time.Since(entry.mtime) <= time.Minute*20 {
			return entry.pkg, nil
		}
		// If there is no cache entry and the user has configured the correct
		// setting, import and cache using the source importer.
		var pkg *types.Package
		var err error
		if i.fallbackToSource {
			pkg, err = goimporter.For("source", nil).Import(path)
		} else {
			pkg, err = goimporter.Default().Import(path)
		}
		if pkg == nil {
			return nil, err
		}
		entry = importCacheEntry{pkg, time.Now()}
		i.imports[path] = entry
		return entry.pkg, nil
	}

	// If there is export data for the package.
	fi, err := os.Stat(filename)
	if err != nil {
		return nil, err
	}
	if entry.mtime != fi.ModTime() {
		f, err := os.Open(filename)
		if err != nil {
			return nil, err
		}
		in, err := gcexportdata.NewReader(f)
		if err != nil {
			return nil, err
		}
		pkg, err := gcexportdata.Read(in, i.fset, make(map[string]*types.Package), path)
		if err != nil {
			return nil, err
		}
		entry = importCacheEntry{pkg, fi.ModTime()}
		i.imports[path] = entry
	}

	return entry.pkg, nil
}

// Delete random files to keep the cache at most 100 entries.
// Only call while holding the importer's mutex.
func (i *importerCache) clean() {
	for k := range i.imports {
		if len(i.imports) <= 100 {
			break
		}
		delete(i.imports, k)
	}
}

func (i *importer) splitPathList(list string) []string {
	res := filepath.SplitList(list)
	if i.gbroot != "" {
		res = append(res, i.gbroot, i.gbvendor)
	}
	return res
}

func (i *importer) joinPath(elem ...string) string {
	res := filepath.Join(elem...)

	if i.gbroot != "" {
		// Want to rewrite "$GBROOT/(vendor/)?pkg/$GOOS_$GOARCH(_)?"
		// into "$GBROOT/pkg/$GOOS-$GOARCH(-)?".
		// Note: gb doesn't use vendor/pkg.
		if gbrel, err := filepath.Rel(i.gbroot, res); err == nil {
			gbrel = filepath.ToSlash(gbrel)
			gbrel, _ = match(gbrel, "vendor/")
			if gbrel, ok := match(gbrel, fmt.Sprintf("pkg/%s_%s", i.ctx.GOOS, i.ctx.GOARCH)); ok {
				gbrel, hasSuffix := match(gbrel, "_")

				// Reassemble into result.
				if hasSuffix {
					gbrel = "-" + gbrel
				}
				gbrel = fmt.Sprintf("pkg/%s-%s/", i.ctx.GOOS, i.ctx.GOARCH) + gbrel
				gbrel = filepath.FromSlash(gbrel)
				res = filepath.Join(i.gbroot, gbrel)
			}
		}
	}
	return res
}

func match(s, prefix string) (string, bool) {
	rest := strings.TrimPrefix(s, prefix)
	return rest, len(rest) < len(s)
}
<|endoftext|>


<commit_before>package controllers

import (
	"bytes"
	"context"
	"encoding/json"
	"fmt"
	"io"
	"net/http"
	"reflect"
	"sort"
	"strconv"
	"strings"
	"time"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/s3"
	"github.com/go-openapi/swag"
	"github.com/pottava/aws-s3-proxy/internal/common"
	"github.com/pottava/aws-s3-proxy/internal/config"
)

// AwsS3 handles requests for Amazon S3
func AwsS3(w http.ResponseWriter, r *http.Request) {
	c := config.Config

	// Strip the prefix, if it's present.
	path := r.URL.Path
	if len(c.StripPath) > 0 {
		path = strings.TrimPrefix(path, c.StripPath)
	}

	// If there is a health check path defined, and if this path matches it,
	// then return 200 OK and return.
	// Note: we want to apply the health check *after* the prefix is stripped.
	if len(c.HealthCheckPath) > 0 && path == c.HealthCheckPath {
		w.WriteHeader(http.StatusOK)
		return
	}
	// Range header
	var rangeHeader *string
	if candidate := r.Header.Get("Range"); !swag.IsZero(candidate) {
		rangeHeader = aws.String(candidate)
	}

	sess := common.AwsSession(aws.String(config.Config.AwsRegion))
	ctx := r.Context()

	// Replace path with symlink.json
	idx := strings.Index(path, "symlink.json")
	if idx > -1 {
		replaced, err := replacePathWithSymlink(ctx, sess, c.S3Bucket, c.S3KeyPrefix+path[:idx+12])
		if err != nil {
			code, message := toHTTPError(err)
			http.Error(w, message, code)
			return
		}
		path = aws.StringValue(replaced) + path[idx+12:]
	}
	// Ends with / -> listing or index.html
	if strings.HasSuffix(path, "/") {
		if c.DirectoryListing {
			s3listFiles(ctx, sess, w, r, c.S3Bucket, c.S3KeyPrefix+path)
			return
		}
		path += c.IndexDocument
	}
	// Get a S3 object
	obj, err := common.S3get(ctx, sess, c.S3Bucket, c.S3KeyPrefix+path, rangeHeader)
	if err != nil {
		code, message := toHTTPError(err)
		http.Error(w, message, code)
		return
	}
	setHeadersFromAwsResponse(w, obj, c.HTTPCacheControl, c.HTTPExpires)

	io.Copy(w, obj.Body) // nolint
}

func replacePathWithSymlink(ctx context.Context, sess *session.Session, bucket, symlinkPath string,
) (*string, error) {
	obj, err := common.S3get(ctx, sess, bucket, symlinkPath, nil)
	if err != nil {
		return nil, err
	}
	link := struct {
		URL string
	}{}
	buf := new(bytes.Buffer)
	if _, err = buf.ReadFrom(obj.Body); err != nil {
		return nil, err
	}
	if err = json.Unmarshal(buf.Bytes(), &link); err != nil {
		return nil, err
	}
	return aws.String(link.URL), nil
}

func setHeadersFromAwsResponse(w http.ResponseWriter, obj *s3.GetObjectOutput, httpCacheControl, httpExpires string) {

	// Cache-Control
	if len(httpCacheControl) > 0 {
		setStrHeader(w, "Cache-Control", &httpCacheControl)
	} else {
		setStrHeader(w, "Cache-Control", obj.CacheControl)
	}
	// Expires
	if len(httpExpires) > 0 {
		setStrHeader(w, "Expires", &httpExpires)
	} else {
		setStrHeader(w, "Expires", obj.Expires)
	}
	setStrHeader(w, "Content-Disposition", obj.ContentDisposition)
	setStrHeader(w, "Content-Encoding", obj.ContentEncoding)
	setStrHeader(w, "Content-Language", obj.ContentLanguage)
	setIntHeader(w, "Content-Length", obj.ContentLength)
	setStrHeader(w, "Content-Range", obj.ContentRange)
	setStrHeader(w, "Content-Type", obj.ContentType)
	setStrHeader(w, "ETag", obj.ETag)
	setTimeHeader(w, "Last-Modified", obj.LastModified)

	w.WriteHeader(determineHTTPStatus(obj))
}

func setStrHeader(w http.ResponseWriter, key string, value *string) {
	if value != nil && len(*value) > 0 {
		w.Header().Add(key, *value)
	}
}

func setIntHeader(w http.ResponseWriter, key string, value *int64) {
	if value != nil && *value > 0 {
		w.Header().Add(key, strconv.FormatInt(*value, 10))
	}
}

func setTimeHeader(w http.ResponseWriter, key string, value *time.Time) {
	if value != nil && !reflect.DeepEqual(*value, time.Time{}) {
		w.Header().Add(key, value.UTC().Format(http.TimeFormat))
	}
}

func s3listFiles(ctx context.Context, sess *session.Session, w http.ResponseWriter, r *http.Request, bucket, prefix string) {
	prefix = strings.TrimPrefix(prefix, "/")

	result, err := common.S3listObjects(ctx, sess, bucket, prefix)
	if err != nil {
		code, message := toHTTPError(err)
		http.Error(w, message, code)
		return
	}
	files, updatedAt := convertToMaps(result, prefix)

	// Output as a HTML
	if strings.EqualFold(config.Config.DirListingFormat, "html") {
		w.Header().Set("Content-Type", "text/html; charset=utf-8")
		fmt.Fprintln(w, toHTML(files, updatedAt))
		return
	}
	// Output as a JSON
	bytes, merr := json.Marshal(files)
	if merr != nil {
		http.Error(w, merr.Error(), http.StatusInternalServerError)
		return
	}
	w.Header().Set("Content-Type", "application/json; charset=utf-8")
	fmt.Fprintln(w, string(bytes))
}

func convertToMaps(s3output *s3.ListObjectsOutput, prefix string) ([]string, map[string]time.Time) {
	candidates := map[string]bool{}
	updatedAt := map[string]time.Time{}

	// Prefixes
	for _, obj := range s3output.CommonPrefixes {
		candidate := strings.TrimPrefix(aws.StringValue(obj.Prefix), prefix)
		if len(candidate) == 0 {
			continue
		}
		candidates[candidate] = true
	}
	// Contents
	for _, obj := range s3output.Contents {
		candidate := strings.TrimPrefix(aws.StringValue(obj.Key), prefix)
		if len(candidate) == 0 {
			continue
		}
		candidates[candidate] = true
		updatedAt[candidate] = *obj.LastModified
	}
	// Sort file names
	files := []string{}
	for file := range candidates {
		files = append(files, file)
	}
	sort.Sort(s3objects(files))

	return files, updatedAt
}

func toHTML(files []string, updatedAt map[string]time.Time) string {
	html := "<!DOCTYPE html>
"
	for _, file := range files {
		html += "
" + file + ""
		if timestamp, ok := updatedAt[file]; ok {
			html += " " + timestamp.Format(time.RFC3339)
		}
		html += "
"
	}
	return html + "
"
}
<commit_msg>refs 	#20 Returns encoded content-length<commit_after>package controllers

import (
	"bytes"
	"context"
	"encoding/json"
	"fmt"
	"io"
	"net/http"
	"reflect"
	"sort"
	"strconv"
	"strings"
	"time"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/s3"
	"github.com/go-openapi/swag"
	"github.com/pottava/aws-s3-proxy/internal/common"
	"github.com/pottava/aws-s3-proxy/internal/config"
)

// AwsS3 handles requests for Amazon S3
func AwsS3(w http.ResponseWriter, r *http.Request) {
	c := config.Config

	// Strip the prefix, if it's present.
	path := r.URL.Path
	if len(c.StripPath) > 0 {
		path = strings.TrimPrefix(path, c.StripPath)
	}

	// If there is a health check path defined, and if this path matches it,
	// then return 200 OK and return.
	// Note: we want to apply the health check *after* the prefix is stripped.
	if len(c.HealthCheckPath) > 0 && path == c.HealthCheckPath {
		w.WriteHeader(http.StatusOK)
		return
	}
	// Range header
	var rangeHeader *string
	if candidate := r.Header.Get("Range"); !swag.IsZero(candidate) {
		rangeHeader = aws.String(candidate)
	}

	sess := common.AwsSession(aws.String(config.Config.AwsRegion))
	ctx := r.Context()

	// Replace path with symlink.json
	idx := strings.Index(path, "symlink.json")
	if idx > -1 {
		replaced, err := replacePathWithSymlink(ctx, sess, c.S3Bucket, c.S3KeyPrefix+path[:idx+12])
		if err != nil {
			code, message := toHTTPError(err)
			http.Error(w, message, code)
			return
		}
		path = aws.StringValue(replaced) + path[idx+12:]
	}
	// Ends with / -> listing or index.html
	if strings.HasSuffix(path, "/") {
		if c.DirectoryListing {
			s3listFiles(ctx, sess, w, r, c.S3Bucket, c.S3KeyPrefix+path)
			return
		}
		path += c.IndexDocument
	}
	// Get a S3 object
	obj, err := common.S3get(ctx, sess, c.S3Bucket, c.S3KeyPrefix+path, rangeHeader)
	if err != nil {
		code, message := toHTTPError(err)
		http.Error(w, message, code)
		return
	}
	setHeadersFromAwsResponse(w, obj, c.HTTPCacheControl, c.HTTPExpires)

	io.Copy(w, obj.Body) // nolint
}

func replacePathWithSymlink(ctx context.Context, sess *session.Session, bucket, symlinkPath string,
) (*string, error) {
	obj, err := common.S3get(ctx, sess, bucket, symlinkPath, nil)
	if err != nil {
		return nil, err
	}
	link := struct {
		URL string
	}{}
	buf := new(bytes.Buffer)
	if _, err = buf.ReadFrom(obj.Body); err != nil {
		return nil, err
	}
	if err = json.Unmarshal(buf.Bytes(), &link); err != nil {
		return nil, err
	}
	return aws.String(link.URL), nil
}

func setHeadersFromAwsResponse(w http.ResponseWriter, obj *s3.GetObjectOutput, httpCacheControl, httpExpires string) {

	// Cache-Control
	if len(httpCacheControl) > 0 {
		setStrHeader(w, "Cache-Control", &httpCacheControl)
	} else {
		setStrHeader(w, "Cache-Control", obj.CacheControl)
	}
	// Expires
	if len(httpExpires) > 0 {
		setStrHeader(w, "Expires", &httpExpires)
	} else {
		setStrHeader(w, "Expires", obj.Expires)
	}
	setStrHeader(w, "Content-Disposition", obj.ContentDisposition)
	setStrHeader(w, "Content-Encoding", obj.ContentEncoding)
	setStrHeader(w, "Content-Language", obj.ContentLanguage)

	// Fix https://github.com/pottava/aws-s3-proxy/issues/20
	if len(w.Header().Get("Content-Encoding")) == 0 {
		setIntHeader(w, "Content-Length", obj.ContentLength)
	}
	setStrHeader(w, "Content-Range", obj.ContentRange)
	setStrHeader(w, "Content-Type", obj.ContentType)
	setStrHeader(w, "ETag", obj.ETag)
	setTimeHeader(w, "Last-Modified", obj.LastModified)

	w.WriteHeader(determineHTTPStatus(obj))
}

func setStrHeader(w http.ResponseWriter, key string, value *string) {
	if value != nil && len(*value) > 0 {
		w.Header().Add(key, *value)
	}
}

func setIntHeader(w http.ResponseWriter, key string, value *int64) {
	if value != nil && *value > 0 {
		w.Header().Add(key, strconv.FormatInt(*value, 10))
	}
}

func setTimeHeader(w http.ResponseWriter, key string, value *time.Time) {
	if value != nil && !reflect.DeepEqual(*value, time.Time{}) {
		w.Header().Add(key, value.UTC().Format(http.TimeFormat))
	}
}

func s3listFiles(ctx context.Context, sess *session.Session, w http.ResponseWriter, r *http.Request, bucket, prefix string) {
	prefix = strings.TrimPrefix(prefix, "/")

	result, err := common.S3listObjects(ctx, sess, bucket, prefix)
	if err != nil {
		code, message := toHTTPError(err)
		http.Error(w, message, code)
		return
	}
	files, updatedAt := convertToMaps(result, prefix)

	// Output as a HTML
	if strings.EqualFold(config.Config.DirListingFormat, "html") {
		w.Header().Set("Content-Type", "text/html; charset=utf-8")
		fmt.Fprintln(w, toHTML(files, updatedAt))
		return
	}
	// Output as a JSON
	bytes, merr := json.Marshal(files)
	if merr != nil {
		http.Error(w, merr.Error(), http.StatusInternalServerError)
		return
	}
	w.Header().Set("Content-Type", "application/json; charset=utf-8")
	fmt.Fprintln(w, string(bytes))
}

func convertToMaps(s3output *s3.ListObjectsOutput, prefix string) ([]string, map[string]time.Time) {
	candidates := map[string]bool{}
	updatedAt := map[string]time.Time{}

	// Prefixes
	for _, obj := range s3output.CommonPrefixes {
		candidate := strings.TrimPrefix(aws.StringValue(obj.Prefix), prefix)
		if len(candidate) == 0 {
			continue
		}
		candidates[candidate] = true
	}
	// Contents
	for _, obj := range s3output.Contents {
		candidate := strings.TrimPrefix(aws.StringValue(obj.Key), prefix)
		if len(candidate) == 0 {
			continue
		}
		candidates[candidate] = true
		updatedAt[candidate] = *obj.LastModified
	}
	// Sort file names
	files := []string{}
	for file := range candidates {
		files = append(files, file)
	}
	sort.Sort(s3objects(files))

	return files, updatedAt
}

func toHTML(files []string, updatedAt map[string]time.Time) string {
	html := "<!DOCTYPE html>
"
	for _, file := range files {
		html += "
" + file + ""
		if timestamp, ok := updatedAt[file]; ok {
			html += " " + timestamp.Format(time.RFC3339)
		}
		html += "
"
	}
	return html + "
"
}
<|endoftext|>


<commit_before>// Copyright 2015 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package util

import (
	"encoding/hex"
	"fmt"
	"hash"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
	"os"
	"path/filepath"
	"strconv"

	cutil "github.com/coreos/ignition/v2/config/util"
	"github.com/coreos/ignition/v2/config/v3_4_experimental/types"
	"github.com/coreos/ignition/v2/internal/log"
	"github.com/coreos/ignition/v2/internal/resource"
	"github.com/coreos/ignition/v2/internal/util"

	"golang.org/x/sys/unix"
)

const (
	DefaultDirectoryPermissions os.FileMode = 0755
	DefaultFilePermissions      os.FileMode = 0644
)

type FetchOp struct {
	Hash         hash.Hash
	Url          url.URL
	FetchOptions resource.FetchOptions
	Append       bool
	Node         types.Node
}

func newFetchOp(l *log.Logger, node types.Node, contents types.Resource) (FetchOp, error) {
	var expectedSum []byte

	uri, err := url.Parse(*contents.Source)
	if err != nil {
		return FetchOp{}, err
	}

	hasher, err := util.GetHasher(contents.Verification)
	if err != nil {
		l.Crit("Error verifying file %q: %v", node.Path, err)
		return FetchOp{}, err
	}

	if hasher != nil {
		// explicitly ignoring the error here because the config should already
		// be validated by this point
		_, expectedSumString, _ := util.HashParts(contents.Verification)
		expectedSum, err = hex.DecodeString(expectedSumString)
		if err != nil {
			l.Crit("Error parsing verification string %q: %v", expectedSumString, err)
			return FetchOp{}, err
		}
	}
	compression := ""
	if contents.Compression != nil {
		compression = *contents.Compression
	}

	var headers http.Header
	if contents.HTTPHeaders != nil && len(contents.HTTPHeaders) > 0 {
		headers, err = contents.HTTPHeaders.Parse()
		if err != nil {
			return FetchOp{}, err
		}
	}

	return FetchOp{
		Hash: hasher,
		Node: node,
		Url:  *uri,
		FetchOptions: resource.FetchOptions{
			Hash:        hasher,
			Compression: compression,
			ExpectedSum: expectedSum,
			Headers:     headers,
		},
	}, nil
}

// PrepareFetches converts a given logger, http client, and types.File into a
// FetchOp. This includes operations such as parsing the source URL, generating
// a hasher, and performing user/group name lookups. If an error is encountered,
// the issue will be logged and nil will be returned.
func (u Util) PrepareFetches(l *log.Logger, f types.File) ([]FetchOp, error) {
	ops := []FetchOp{}

	if f.Contents.Source != nil {
		if base, err := newFetchOp(l, f.Node, f.Contents); err != nil {
			return nil, err
		} else {
			ops = append(ops, base)
		}
	}

	for _, appendee := range f.Append {
		if op, err := newFetchOp(l, f.Node, appendee); err != nil {
			return nil, err
		} else {
			op.Append = true
			ops = append(ops, op)
		}
	}

	return ops, nil
}

func (u Util) WriteLink(s types.Link) error {
	path := s.Path

	if err := MkdirForFile(path); err != nil {
		return fmt.Errorf("Could not create leading directories: %v", err)
	}

	if cutil.IsTrue(s.Hard) {
		targetPath, err := u.JoinPath(*s.Target)
		if err != nil {
			return err
		}
		return os.Link(targetPath, path)
	}

	if err := os.Symlink(*s.Target, path); err != nil {
		return fmt.Errorf("Could not create symlink: %v", err)
	}

	if err := u.SetPermissions(nil, s.Node); err != nil {
		return fmt.Errorf("error setting permissions of %s: %v", s.Path, err)
	}
	return nil
}

func (u Util) SetPermissions(mode *int, node types.Node) error {
	if mode != nil {
		mode := os.FileMode(*mode)
		if err := os.Chmod(node.Path, mode); err != nil {
			return fmt.Errorf("failed to change mode of %s: %v", node.Path, err)
		}
	}

	defaultUid, defaultGid, _ := getFileOwnerAndMode(node.Path)
	uid, gid, err := u.ResolveNodeUidAndGid(node, defaultUid, defaultGid)
	if err != nil {
		return fmt.Errorf("failed to determine correct uid and gid for %s: %v", node.Path, err)
	}
	if err := os.Lchown(node.Path, uid, gid); err != nil {
		return fmt.Errorf("failed to change ownership of %s: %v", node.Path, err)
	}
	return nil
}

// PerformFetch performs a fetch operation generated by PrepareFetch, retrieving
// the file and writing it to disk. Any encountered errors are returned.
func (u Util) PerformFetch(f FetchOp) error {
	path := f.Node.Path

	if err := MkdirForFile(path); err != nil {
		return err
	}

	// Create a temporary file in the same directory to ensure it's on the same filesystem
	tmp, err := ioutil.TempFile(filepath.Dir(path), "tmp")
	if err != nil {
		return err
	}
	defer tmp.Close()

	// ioutil.TempFile defaults to 0600
	if err := tmp.Chmod(DefaultFilePermissions); err != nil {
		return err
	}

	// sometimes the following line will fail (the file might be renamed),
	// but that's ok (we wanted to keep the file in that case).
	defer os.Remove(tmp.Name())

	err = u.Fetcher.Fetch(f.Url, tmp, f.FetchOptions)
	if err != nil {
		u.Crit("Error fetching file %q: %v", path, err)
		return err
	}

	if f.Append {
		// Make sure that we're appending to a file
		finfo, err := os.Lstat(path)
		switch {
		case os.IsNotExist(err):
			// No problem, we'll create it.
			break
		case err != nil:
			return err
		default:
			if !finfo.Mode().IsRegular() {
				return fmt.Errorf("can only append to files: %q", path)
			}
		}

		// Open with the default permissions, we'll chown/chmod it later
		targetFile, err := os.OpenFile(path, os.O_WRONLY|os.O_APPEND|os.O_CREATE, DefaultFilePermissions)
		if err != nil {
			return err
		}
		defer targetFile.Close()

		if _, err = tmp.Seek(0, io.SeekStart); err != nil {
			return err
		}
		if _, err = io.Copy(targetFile, tmp); err != nil {
			return err
		}
	} else {
		if err = os.Rename(tmp.Name(), path); err != nil {
			return err
		}
	}

	return nil
}

// MkdirForFile helper creates the directory components of path.
func MkdirForFile(path string) error {
	return os.MkdirAll(filepath.Dir(path), DefaultDirectoryPermissions)
}

// FindFirstMissingPathComponent returns the path up to the first component
// which was found to be missing, or the whole path if it already exists.
func FindFirstMissingPathComponent(path string) (string, error) {
	entry := path
	dir := filepath.Dir(path)
	for {
		exists := true
		if _, err := os.Stat(dir); err != nil && os.IsNotExist(err) {
			exists = false
		} else if err != nil {
			return "", err
		}

		// also sanity check we didn't somehow get all the way up to /sysroot
		if dir == "/" {
			return "", fmt.Errorf("/ doesn't seem to exist")
		}
		if exists {
			return entry, nil
		}
		entry = dir
		dir = filepath.Dir(dir)
	}
}

// FilesystemIsEmpty checks the mountpoint of a filesystem to see whether
// the filesystem is empty.
// Adapted from https://stackoverflow.com/a/30708914
func FilesystemIsEmpty(dirpath string) (bool, error) {
	dfd, err := os.Open(dirpath)
	if err != nil {
		return false, err
	}
	defer dfd.Close()

	for {
		names, err := dfd.Readdirnames(1)
		if err == io.EOF {
			return true, nil
		} else if err != nil {
			return false, err
		}

		switch names[0] {
		case "lost+found":
			// valid in a fresh filesystem; keep reading
		default:
			return false, nil
		}
	}
}

// getFileOwner will return the uid and gid for the file at a given path. If the
// file doesn't exist, or some other error is encountered when running stat on
// the path, 0, 0, and 0 will be returned.
func getFileOwnerAndMode(path string) (int, int, os.FileMode) {
	info := unix.Stat_t{}
	if err := unix.Stat(path, &info); err != nil {
		return 0, 0, 0
	}
	return int(info.Uid), int(info.Gid), os.FileMode(info.Mode)
}

// ResolveNodeUidAndGid attempts to convert a types.Node into a concrete uid and
// gid. If the node has the User.ID field set, that's used for the uid. If the
// node has the User.Name field set, a username -> uid lookup is performed. If
// neither are set, it returns the passed in defaultUid. The logic is identical
// for gids with equivalent fields.
func (u Util) ResolveNodeUidAndGid(n types.Node, defaultUid, defaultGid int) (int, int, error) {
	var err error
	uid, gid := defaultUid, defaultGid

	if n.User.ID != nil {
		uid = *n.User.ID
	} else if cutil.NotEmpty(n.User.Name) {
		uid, err = u.getUserID(*n.User.Name)
		if err != nil {
			return 0, 0, err
		}
	}

	if n.Group.ID != nil {
		gid = *n.Group.ID
	} else if cutil.NotEmpty(n.Group.Name) {
		gid, err = u.getGroupID(*n.Group.Name)
		if err != nil {
			return 0, 0, err
		}
	}
	return uid, gid, nil
}

func (u Util) getUserID(name string) (int, error) {
	usr, err := u.userLookup(name)
	if err != nil {
		return 0, fmt.Errorf("No such user %q: %v", name, err)
	}
	uid, err := strconv.ParseInt(usr.Uid, 0, 0)
	if err != nil {
		return 0, fmt.Errorf("Couldn't parse uid %q: %v", usr.Uid, err)
	}
	return int(uid), nil
}

func (u Util) getGroupID(name string) (int, error) {
	g, err := u.groupLookup(name)
	if err != nil {
		return 0, fmt.Errorf("No such group %q: %v", name, err)
	}
	gid, err := strconv.ParseInt(g.Gid, 0, 0)
	if err != nil {
		return 0, fmt.Errorf("Couldn't parse gid %q: %v", g.Gid, err)
	}
	return int(gid), nil
}
<commit_msg>internal/exec/*: preserve special mode bits for a given file/dir<commit_after>// Copyright 2015 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package util

import (
	"encoding/hex"
	"fmt"
	"hash"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
	"os"
	"path/filepath"
	"strconv"
	"syscall"

	cutil "github.com/coreos/ignition/v2/config/util"
	"github.com/coreos/ignition/v2/config/v3_4_experimental/types"
	"github.com/coreos/ignition/v2/internal/log"
	"github.com/coreos/ignition/v2/internal/resource"
	"github.com/coreos/ignition/v2/internal/util"

	"golang.org/x/sys/unix"
)

const (
	DefaultDirectoryPermissions os.FileMode = 0755
	DefaultFilePermissions      os.FileMode = 0644
)

type FetchOp struct {
	Hash         hash.Hash
	Url          url.URL
	FetchOptions resource.FetchOptions
	Append       bool
	Node         types.Node
}

func newFetchOp(l *log.Logger, node types.Node, contents types.Resource) (FetchOp, error) {
	var expectedSum []byte

	uri, err := url.Parse(*contents.Source)
	if err != nil {
		return FetchOp{}, err
	}

	hasher, err := util.GetHasher(contents.Verification)
	if err != nil {
		l.Crit("Error verifying file %q: %v", node.Path, err)
		return FetchOp{}, err
	}

	if hasher != nil {
		// explicitly ignoring the error here because the config should already
		// be validated by this point
		_, expectedSumString, _ := util.HashParts(contents.Verification)
		expectedSum, err = hex.DecodeString(expectedSumString)
		if err != nil {
			l.Crit("Error parsing verification string %q: %v", expectedSumString, err)
			return FetchOp{}, err
		}
	}
	compression := ""
	if contents.Compression != nil {
		compression = *contents.Compression
	}

	var headers http.Header
	if contents.HTTPHeaders != nil && len(contents.HTTPHeaders) > 0 {
		headers, err = contents.HTTPHeaders.Parse()
		if err != nil {
			return FetchOp{}, err
		}
	}

	return FetchOp{
		Hash: hasher,
		Node: node,
		Url:  *uri,
		FetchOptions: resource.FetchOptions{
			Hash:        hasher,
			Compression: compression,
			ExpectedSum: expectedSum,
			Headers:     headers,
		},
	}, nil
}

// PrepareFetches converts a given logger, http client, and types.File into a
// FetchOp. This includes operations such as parsing the source URL, generating
// a hasher, and performing user/group name lookups. If an error is encountered,
// the issue will be logged and nil will be returned.
func (u Util) PrepareFetches(l *log.Logger, f types.File) ([]FetchOp, error) {
	ops := []FetchOp{}

	if f.Contents.Source != nil {
		if base, err := newFetchOp(l, f.Node, f.Contents); err != nil {
			return nil, err
		} else {
			ops = append(ops, base)
		}
	}

	for _, appendee := range f.Append {
		if op, err := newFetchOp(l, f.Node, appendee); err != nil {
			return nil, err
		} else {
			op.Append = true
			ops = append(ops, op)
		}
	}

	return ops, nil
}

func (u Util) WriteLink(s types.Link) error {
	path := s.Path

	if err := MkdirForFile(path); err != nil {
		return fmt.Errorf("could not create leading directories: %v", err)
	}

	if cutil.IsTrue(s.Hard) {
		targetPath, err := u.JoinPath(*s.Target)
		if err != nil {
			return err
		}
		return os.Link(targetPath, path)
	}

	if err := os.Symlink(*s.Target, path); err != nil {
		return fmt.Errorf("could not create symlink: %v", err)
	}

	if err := u.SetPermissions(nil, s.Node); err != nil {
		return fmt.Errorf("error setting permissions of %s: %v", s.Path, err)
	}
	return nil
}

func (u Util) SetPermissions(mode *int, node types.Node) error {
	if mode != nil {
		if err := os.Chmod(node.Path, toFileMode(*mode)); err != nil {
			return fmt.Errorf("failed to change mode of %s: %v", node.Path, err)
		}
	}

	defaultUid, defaultGid, _ := getFileOwnerAndMode(node.Path)
	uid, gid, err := u.ResolveNodeUidAndGid(node, defaultUid, defaultGid)
	if err != nil {
		return fmt.Errorf("failed to determine correct uid and gid for %s: %v", node.Path, err)
	}
	if err := os.Lchown(node.Path, uid, gid); err != nil {
		return fmt.Errorf("failed to change ownership of %s: %v", node.Path, err)
	}
	return nil
}

// toFileMode converts Go permission bits to POSIX permission bits.
func toFileMode(m int) os.FileMode {
	mode := uint32(m)
	res := os.FileMode(mode & 0777)

	if mode&syscall.S_ISGID != 0 {
		res |= os.ModeSetgid
	}
	if mode&syscall.S_ISUID != 0 {
		res |= os.ModeSetuid
	}
	if mode&syscall.S_ISVTX != 0 {
		res |= os.ModeSticky
	}
	return res
}

// PerformFetch performs a fetch operation generated by PrepareFetch, retrieving
// the file and writing it to disk. Any encountered errors are returned.
func (u Util) PerformFetch(f FetchOp) error {
	path := f.Node.Path

	if err := MkdirForFile(path); err != nil {
		return err
	}

	// Create a temporary file in the same directory to ensure it's on the same filesystem
	tmp, err := ioutil.TempFile(filepath.Dir(path), "tmp")
	if err != nil {
		return err
	}
	defer tmp.Close()

	// ioutil.TempFile defaults to 0600
	if err := tmp.Chmod(DefaultFilePermissions); err != nil {
		return err
	}

	// sometimes the following line will fail (the file might be renamed),
	// but that's ok (we wanted to keep the file in that case).
	defer os.Remove(tmp.Name())

	err = u.Fetcher.Fetch(f.Url, tmp, f.FetchOptions)
	if err != nil {
		u.Crit("Error fetching file %q: %v", path, err)
		return err
	}

	if f.Append {
		// Make sure that we're appending to a file
		finfo, err := os.Lstat(path)
		switch {
		case os.IsNotExist(err):
			// No problem, we'll create it.
			break
		case err != nil:
			return err
		default:
			if !finfo.Mode().IsRegular() {
				return fmt.Errorf("can only append to files: %q", path)
			}
		}

		// Open with the default permissions, we'll chown/chmod it later
		targetFile, err := os.OpenFile(path, os.O_WRONLY|os.O_APPEND|os.O_CREATE, DefaultFilePermissions)
		if err != nil {
			return err
		}
		defer targetFile.Close()

		if _, err = tmp.Seek(0, io.SeekStart); err != nil {
			return err
		}
		if _, err = io.Copy(targetFile, tmp); err != nil {
			return err
		}
	} else {
		if err = os.Rename(tmp.Name(), path); err != nil {
			return err
		}
	}

	return nil
}

// MkdirForFile helper creates the directory components of path.
func MkdirForFile(path string) error {
	return os.MkdirAll(filepath.Dir(path), DefaultDirectoryPermissions)
}

// FindFirstMissingPathComponent returns the path up to the first component
// which was found to be missing, or the whole path if it already exists.
func FindFirstMissingPathComponent(path string) (string, error) {
	entry := path
	dir := filepath.Dir(path)
	for {
		exists := true
		if _, err := os.Stat(dir); err != nil && os.IsNotExist(err) {
			exists = false
		} else if err != nil {
			return "", err
		}

		// also sanity check we didn't somehow get all the way up to /sysroot
		if dir == "/" {
			return "", fmt.Errorf("/ doesn't seem to exist")
		}
		if exists {
			return entry, nil
		}
		entry = dir
		dir = filepath.Dir(dir)
	}
}

// FilesystemIsEmpty checks the mountpoint of a filesystem to see whether
// the filesystem is empty.
// Adapted from https://stackoverflow.com/a/30708914
func FilesystemIsEmpty(dirpath string) (bool, error) {
	dfd, err := os.Open(dirpath)
	if err != nil {
		return false, err
	}
	defer dfd.Close()

	for {
		names, err := dfd.Readdirnames(1)
		if err == io.EOF {
			return true, nil
		} else if err != nil {
			return false, err
		}

		switch names[0] {
		case "lost+found":
			// valid in a fresh filesystem; keep reading
		default:
			return false, nil
		}
	}
}

// getFileOwner will return the uid and gid for the file at a given path. If the
// file doesn't exist, or some other error is encountered when running stat on
// the path, 0, 0, and 0 will be returned.
func getFileOwnerAndMode(path string) (int, int, os.FileMode) {
	info := unix.Stat_t{}
	if err := unix.Stat(path, &info); err != nil {
		return 0, 0, 0
	}
	return int(info.Uid), int(info.Gid), os.FileMode(info.Mode)
}

// ResolveNodeUidAndGid attempts to convert a types.Node into a concrete uid and
// gid. If the node has the User.ID field set, that's used for the uid. If the
// node has the User.Name field set, a username -> uid lookup is performed. If
// neither are set, it returns the passed in defaultUid. The logic is identical
// for gids with equivalent fields.
func (u Util) ResolveNodeUidAndGid(n types.Node, defaultUid, defaultGid int) (int, int, error) {
	var err error
	uid, gid := defaultUid, defaultGid

	if n.User.ID != nil {
		uid = *n.User.ID
	} else if cutil.NotEmpty(n.User.Name) {
		uid, err = u.getUserID(*n.User.Name)
		if err != nil {
			return 0, 0, err
		}
	}

	if n.Group.ID != nil {
		gid = *n.Group.ID
	} else if cutil.NotEmpty(n.Group.Name) {
		gid, err = u.getGroupID(*n.Group.Name)
		if err != nil {
			return 0, 0, err
		}
	}
	return uid, gid, nil
}

func (u Util) getUserID(name string) (int, error) {
	usr, err := u.userLookup(name)
	if err != nil {
		return 0, fmt.Errorf("no such user %q: %v", name, err)
	}
	uid, err := strconv.ParseInt(usr.Uid, 0, 0)
	if err != nil {
		return 0, fmt.Errorf("couldn't parse uid %q: %v", usr.Uid, err)
	}
	return int(uid), nil
}

func (u Util) getGroupID(name string) (int, error) {
	g, err := u.groupLookup(name)
	if err != nil {
		return 0, fmt.Errorf("no such group %q: %v", name, err)
	}
	gid, err := strconv.ParseInt(g.Gid, 0, 0)
	if err != nil {
		return 0, fmt.Errorf("couldn't parse gid %q: %v", g.Gid, err)
	}
	return int(gid), nil
}
<|endoftext|>


<commit_before>// Copyright 2017 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package gps

import (
	"fmt"
	"math/rand"
	"strconv"
)

// ProjectRoot is the topmost import path in a tree of other import paths - the
// root of the tree. In gps' current design, ProjectRoots have to correspond to
// a repository root (mostly), but their real purpose is to identify the root
// import path of a "project", logically encompassing all child packages.
//
// Projects are a crucial unit of operation in gps. Constraints are declared by
// a project's manifest, and apply to all packages in a ProjectRoot's tree.
// Solving itself mostly proceeds on a project-by-project basis.
//
// Aliasing string types is usually a bit of an anti-pattern. gps does it here
// as a means of clarifying API intent. This is important because Go's package
// management domain has lots of different path-ish strings floating around:
//
//  actual directories:
//	/home/sdboyer/go/src/github.com/sdboyer/gps/example
//  URLs:
//	https://github.com/sdboyer/gps
//  import paths:
//	github.com/sdboyer/gps/example
//  portions of import paths that refer to a package:
//	example
//  portions that could not possibly refer to anything sane:
//	github.com/sdboyer
//  portions that correspond to a repository root:
//	github.com/sdboyer/gps
//
// While not a panacea, having ProjectRoot allows gps to clearly indicate via
// the type system when a path-ish string must have particular semantics.
type ProjectRoot string

// A ProjectIdentifier provides the name and source location of a dependency. It
// is related to, but differs in two keys ways from, an plain import path.
//
// First, ProjectIdentifiers do not identify a single package. Rather, they
// encompasses the whole tree of packages, including tree's root - the
// ProjectRoot. In gps' current design, this ProjectRoot almost always
// corresponds to the root of a repository.
//
// Second, ProjectIdentifiers can optionally carry a Source, which
// identifies where the underlying source code can be located on the network.
// These can be either a full URL, including protocol, or plain import paths.
// So, these are all valid data for Source:
//
//  github.com/sdboyer/gps
//  github.com/fork/gps
//  git@github.com:sdboyer/gps
//  https://github.com/sdboyer/gps
//
// With plain import paths, network addresses are derived purely through an
// algorithm. By having an explicit network name, it becomes possible to, for
// example, transparently substitute a fork for the original upstream source
// repository.
//
// Note that gps makes no guarantees about the actual import paths contained in
// a repository aligning with ImportRoot. If tools, or their users, specify an
// alternate Source that contains a repository with incompatible internal
// import paths, gps' solving operations will error. (gps does no import
// rewriting.)
//
// Also note that if different projects' manifests report a different
// Source for a given ImportRoot, it is a solve failure. Everyone has to
// agree on where a given import path should be sourced from.
//
// If Source is not explicitly set, gps will derive the network address from
// the ImportRoot using a similar algorithm to that utilized by `go get`.
type ProjectIdentifier struct {
	ProjectRoot ProjectRoot
	Source      string
}

func (i ProjectIdentifier) less(j ProjectIdentifier) bool {
	if i.ProjectRoot < j.ProjectRoot {
		return true
	}
	if j.ProjectRoot < i.ProjectRoot {
		return false
	}

	return i.normalizedSource() < j.normalizedSource()
}

func (i ProjectIdentifier) eq(j ProjectIdentifier) bool {
	if i.ProjectRoot != j.ProjectRoot {
		return false
	}
	if i.Source == j.Source {
		return true
	}

	if (i.Source == "" && j.Source == string(j.ProjectRoot)) ||
		(j.Source == "" && i.Source == string(i.ProjectRoot)) {
		return true
	}

	return false
}

// equiv will check if the two identifiers are "equivalent," under special
// rules.
//
// Given that the ProjectRoots are equal (==), equivalency occurs if:
//
// 1. The Sources are equal (==), OR
// 2. The LEFT (the receiver) Source is non-empty, and the right
// Source is empty.
//
// *This is asymmetry in this binary relation is intentional.* It facilitates
// the case where we allow for a ProjectIdentifier with an explicit Source
// to match one without.
func (i ProjectIdentifier) equiv(j ProjectIdentifier) bool {
	if i.ProjectRoot != j.ProjectRoot {
		return false
	}
	if i.Source == j.Source {
		return true
	}

	if i.Source != "" && j.Source == "" {
		return true
	}

	return false
}

func (i ProjectIdentifier) normalizedSource() string {
	if i.Source == "" {
		return string(i.ProjectRoot)
	}
	return i.Source
}

func (i ProjectIdentifier) errString() string {
	if i.Source == "" || i.Source == string(i.ProjectRoot) {
		return string(i.ProjectRoot)
	}
	return fmt.Sprintf("%s (from %s)", i.ProjectRoot, i.Source)
}

func (i ProjectIdentifier) normalize() ProjectIdentifier {
	if i.Source == "" {
		i.Source = string(i.ProjectRoot)
	}

	return i
}

// ProjectProperties comprise the properties that can be attached to a
// ProjectRoot.
//
// In general, these are declared in the context of a map of ProjectRoot to its
// ProjectProperties; they make little sense without their corresponding
// ProjectRoot.
type ProjectProperties struct {
	Source     string
	Constraint Constraint
}

// bimodalIdentifiers are used to track work to be done in the unselected queue.
type bimodalIdentifier struct {
	id ProjectIdentifier
	// List of packages required within/under the ProjectIdentifier
	pl []string
	// prefv is used to indicate a 'preferred' version. This is expected to be
	// derived from a dep's lock data, or else is empty.
	prefv Version
	// Indicates that the bmi came from the root project originally
	fromRoot bool
}

type atom struct {
	id ProjectIdentifier
	v  Version
}

// With a random revision and no name, collisions are...unlikely
var nilpa = atom{
	v: Revision(strconv.FormatInt(rand.Int63(), 36)),
}

type atomWithPackages struct {
	a  atom
	pl []string
}

// bmi converts an atomWithPackages into a bimodalIdentifier.
//
// This is mostly intended for (read-only) trace use, so the package list slice
// is not copied. It is the callers responsibility to not modify the pl slice,
// lest that backpropagate and cause inconsistencies.
func (awp atomWithPackages) bmi() bimodalIdentifier {
	return bimodalIdentifier{
		id: awp.a.id,
		pl: awp.pl,
	}
}

// completeDep (name hopefully to change) provides the whole picture of a
// dependency - the root (repo and project, since currently we assume the two
// are the same) name, a constraint, and the actual packages needed that are
// under that root.
type completeDep struct {
	// The base workingConstraint
	workingConstraint
	// The specific packages required from the ProjectDep
	pl []string
}

type dependency struct {
	depender atom
	dep      completeDep
}
<commit_msg>Cleanup: fix typos<commit_after>// Copyright 2017 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package gps

import (
	"fmt"
	"math/rand"
	"strconv"
)

// ProjectRoot is the topmost import path in a tree of other import paths - the
// root of the tree. In gps' current design, ProjectRoots have to correspond to
// a repository root (mostly), but their real purpose is to identify the root
// import path of a "project", logically encompassing all child packages.
//
// Projects are a crucial unit of operation in gps. Constraints are declared by
// a project's manifest, and apply to all packages in a ProjectRoot's tree.
// Solving itself mostly proceeds on a project-by-project basis.
//
// Aliasing string types is usually a bit of an anti-pattern. gps does it here
// as a means of clarifying API intent. This is important because Go's package
// management domain has lots of different path-ish strings floating around:
//
//  actual directories:
//	/home/sdboyer/go/src/github.com/sdboyer/gps/example
//  URLs:
//	https://github.com/sdboyer/gps
//  import paths:
//	github.com/sdboyer/gps/example
//  portions of import paths that refer to a package:
//	example
//  portions that could not possibly refer to anything sane:
//	github.com/sdboyer
//  portions that correspond to a repository root:
//	github.com/sdboyer/gps
//
// While not a panacea, having ProjectRoot allows gps to clearly indicate via
// the type system when a path-ish string must have particular semantics.
type ProjectRoot string

// A ProjectIdentifier provides the name and source location of a dependency. It
// is related to, but differs in two key ways from, a plain import path.
//
// First, ProjectIdentifiers do not identify a single package. Rather, they
// encompass the whole tree of packages, including tree's root - the
// ProjectRoot. In gps' current design, this ProjectRoot almost always
// corresponds to the root of a repository.
//
// Second, ProjectIdentifiers can optionally carry a Source, which
// identifies where the underlying source code can be located on the network.
// These can be either a full URL, including protocol, or plain import paths.
// So, these are all valid data for Source:
//
//  github.com/sdboyer/gps
//  github.com/fork/gps
//  git@github.com:sdboyer/gps
//  https://github.com/sdboyer/gps
//
// With plain import paths, network addresses are derived purely through an
// algorithm. By having an explicit network name, it becomes possible to, for
// example, transparently substitute a fork for the original upstream source
// repository.
//
// Note that gps makes no guarantees about the actual import paths contained in
// a repository aligning with ImportRoot. If tools, or their users, specify an
// alternate Source that contains a repository with incompatible internal
// import paths, gps' solving operations will error. (gps does no import
// rewriting.)
//
// Also note that if different projects' manifests report a different
// Source for a given ImportRoot, it is a solve failure. Everyone has to
// agree on where a given import path should be sourced from.
//
// If Source is not explicitly set, gps will derive the network address from
// the ImportRoot using a similar algorithm to that utilized by `go get`.
type ProjectIdentifier struct {
	ProjectRoot ProjectRoot
	Source      string
}

func (i ProjectIdentifier) less(j ProjectIdentifier) bool {
	if i.ProjectRoot < j.ProjectRoot {
		return true
	}
	if j.ProjectRoot < i.ProjectRoot {
		return false
	}

	return i.normalizedSource() < j.normalizedSource()
}

func (i ProjectIdentifier) eq(j ProjectIdentifier) bool {
	if i.ProjectRoot != j.ProjectRoot {
		return false
	}
	if i.Source == j.Source {
		return true
	}

	if (i.Source == "" && j.Source == string(j.ProjectRoot)) ||
		(j.Source == "" && i.Source == string(i.ProjectRoot)) {
		return true
	}

	return false
}

// equiv will check if the two identifiers are "equivalent," under special
// rules.
//
// Given that the ProjectRoots are equal (==), equivalency occurs if:
//
// 1. The Sources are equal (==), OR
// 2. The LEFT (the receiver) Source is non-empty, and the right
// Source is empty.
//
// *This is asymmetry in this binary relation is intentional.* It facilitates
// the case where we allow for a ProjectIdentifier with an explicit Source
// to match one without.
func (i ProjectIdentifier) equiv(j ProjectIdentifier) bool {
	if i.ProjectRoot != j.ProjectRoot {
		return false
	}
	if i.Source == j.Source {
		return true
	}

	if i.Source != "" && j.Source == "" {
		return true
	}

	return false
}

func (i ProjectIdentifier) normalizedSource() string {
	if i.Source == "" {
		return string(i.ProjectRoot)
	}
	return i.Source
}

func (i ProjectIdentifier) errString() string {
	if i.Source == "" || i.Source == string(i.ProjectRoot) {
		return string(i.ProjectRoot)
	}
	return fmt.Sprintf("%s (from %s)", i.ProjectRoot, i.Source)
}

func (i ProjectIdentifier) normalize() ProjectIdentifier {
	if i.Source == "" {
		i.Source = string(i.ProjectRoot)
	}

	return i
}

// ProjectProperties comprise the properties that can be attached to a
// ProjectRoot.
//
// In general, these are declared in the context of a map of ProjectRoot to its
// ProjectProperties; they make little sense without their corresponding
// ProjectRoot.
type ProjectProperties struct {
	Source     string
	Constraint Constraint
}

// bimodalIdentifiers are used to track work to be done in the unselected queue.
type bimodalIdentifier struct {
	id ProjectIdentifier
	// List of packages required within/under the ProjectIdentifier
	pl []string
	// prefv is used to indicate a 'preferred' version. This is expected to be
	// derived from a dep's lock data, or else is empty.
	prefv Version
	// Indicates that the bmi came from the root project originally
	fromRoot bool
}

type atom struct {
	id ProjectIdentifier
	v  Version
}

// With a random revision and no name, collisions are...unlikely
var nilpa = atom{
	v: Revision(strconv.FormatInt(rand.Int63(), 36)),
}

type atomWithPackages struct {
	a  atom
	pl []string
}

// bmi converts an atomWithPackages into a bimodalIdentifier.
//
// This is mostly intended for (read-only) trace use, so the package list slice
// is not copied. It is the callers responsibility to not modify the pl slice,
// lest that backpropagate and cause inconsistencies.
func (awp atomWithPackages) bmi() bimodalIdentifier {
	return bimodalIdentifier{
		id: awp.a.id,
		pl: awp.pl,
	}
}

// completeDep (name hopefully to change) provides the whole picture of a
// dependency - the root (repo and project, since currently we assume the two
// are the same) name, a constraint, and the actual packages needed that are
// under that root.
type completeDep struct {
	// The base workingConstraint
	workingConstraint
	// The specific packages required from the ProjectDep
	pl []string
}

type dependency struct {
	depender atom
	dep      completeDep
}
<|endoftext|>


<commit_before>package leveldb

import (
	"sync/atomic"
	"unsafe"

	"github.com/kezhuw/leveldb/internal/manifest"
	"github.com/kezhuw/leveldb/internal/memtable"
)

type bundle struct {
	mem     *memtable.MemTable
	imm     *memtable.MemTable
	version *manifest.Version
}

func (db *DB) loadBundle() *bundle {
	return (*bundle)(atomic.LoadPointer((*unsafe.Pointer)(unsafe.Pointer(&db.bundle))))
}

func (db *DB) swapBundle(old, new *bundle) bool {
	return atomic.CompareAndSwapPointer((*unsafe.Pointer)(unsafe.Pointer(&db.bundle)), unsafe.Pointer(old), unsafe.Pointer(new))
}

func (db *DB) storeBundle(new *bundle) {
	atomic.StorePointer((*unsafe.Pointer)(unsafe.Pointer(&db.bundle)), unsafe.Pointer(new))
}

func (db *DB) switchMemTable(mem *memtable.MemTable) *memtable.MemTable {
	imm := mem
	mem = memtable.New(db.options.Comparator)
	old := db.loadBundle()
	new := &bundle{
		mem:     mem,
		imm:     imm,
		version: old.version,
	}
	for !db.swapBundle(old, new) {
		old = db.loadBundle()
		// Use version from compaction goroutine.
		new.version = old.version
	}
	db.compactionMemtable <- imm
	return mem
}

func (db *DB) switchVersion(level int, version *manifest.Version) {
	db.manifest.Append(version)
	db.wakeupWrite(level)
	old := db.loadBundle()
	new := &bundle{
		mem:     old.mem,
		imm:     old.imm,
		version: version,
	}
	if level == -1 {
		new.imm = nil
	}
	for !db.swapBundle(old, new) {
		old = db.loadBundle()
		// Use memtables from write goroutine.
		new.mem = old.mem
		new.imm = old.imm
	}
}
<commit_msg>Wakeup write goroutine after switch version<commit_after>package leveldb

import (
	"sync/atomic"
	"unsafe"

	"github.com/kezhuw/leveldb/internal/manifest"
	"github.com/kezhuw/leveldb/internal/memtable"
)

type bundle struct {
	mem     *memtable.MemTable
	imm     *memtable.MemTable
	version *manifest.Version
}

func (db *DB) loadBundle() *bundle {
	return (*bundle)(atomic.LoadPointer((*unsafe.Pointer)(unsafe.Pointer(&db.bundle))))
}

func (db *DB) swapBundle(old, new *bundle) bool {
	return atomic.CompareAndSwapPointer((*unsafe.Pointer)(unsafe.Pointer(&db.bundle)), unsafe.Pointer(old), unsafe.Pointer(new))
}

func (db *DB) storeBundle(new *bundle) {
	atomic.StorePointer((*unsafe.Pointer)(unsafe.Pointer(&db.bundle)), unsafe.Pointer(new))
}

func (db *DB) switchMemTable(mem *memtable.MemTable) *memtable.MemTable {
	imm := mem
	mem = memtable.New(db.options.Comparator)
	old := db.loadBundle()
	new := &bundle{
		mem:     mem,
		imm:     imm,
		version: old.version,
	}
	for !db.swapBundle(old, new) {
		old = db.loadBundle()
		// Use version from compaction goroutine.
		new.version = old.version
	}
	db.compactionMemtable <- imm
	return mem
}

func (db *DB) switchVersion(level int, version *manifest.Version) {
	defer db.wakeupWrite(level)
	db.manifest.Append(version)
	old := db.loadBundle()
	new := &bundle{
		mem:     old.mem,
		imm:     old.imm,
		version: version,
	}
	if level == -1 {
		new.imm = nil
	}
	for !db.swapBundle(old, new) {
		old = db.loadBundle()
		// Use memtables from write goroutine.
		new.mem = old.mem
		new.imm = old.imm
	}
}
<|endoftext|>