<commit_before>package upstream

import (
	"fmt"
	"github.com/ngmoco/falcore"
	"net"
	"net/http"
	"time"
)

type Upstream struct {
	// The upstream host to connect to
	Host string
	// The port on the upstream host
	Port int
	// Default 60 seconds
	Timeout time.Duration
	// Will ignore https on the incoming request and always upstream http
	ForceHttp bool
	// Ping URL Path-only for checking upness
	PingPath string

	transport *http.Transport
	host      string
	tcpaddr   *net.TCPAddr
	tcpconn   *net.TCPConn
}

func NewUpstream(host string, port int, forceHttp bool) *Upstream {
	u := new(Upstream)
	u.Host = host
	u.Port = port
	u.ForceHttp = forceHttp
	ips, err := net.LookupIP(host)
	var ip net.IP = nil
	for i := range ips {
		ip = ips[i].To4()
		if ip != nil {
			break
		}
	}
	if err == nil && ip != nil {
		u.tcpaddr = new(net.TCPAddr)
		u.tcpaddr.Port = port
		u.tcpaddr.IP = ip
	} else {
		falcore.Warn("Can't get IP addr for %v: %v", host, err)
	}
	u.Timeout = 60e9
	u.host = fmt.Sprintf("%v:%v", u.Host, u.Port)

	u.transport = new(http.Transport)

	u.transport.Dial = func(n, addr string) (c net.Conn, err error) {
		falcore.Fine("Dialing connection to %v", u.tcpaddr)
		var ctcp *net.TCPConn
		ctcp, err = net.DialTCP("tcp4", nil, u.tcpaddr)
		if ctcp != nil {
			u.tcpconn = ctcp
			u.tcpconn.SetDeadline(time.Now().Add(u.Timeout))
		}
		if err != nil {
			falcore.Error("Dial Failed: %v", err)
		}
		return ctcp, err
	}
	u.transport.MaxIdleConnsPerHost = 15
	return u
}

// Alter the number of connections to multiplex with
func (u *Upstream) SetPoolSize(size int) {
	u.transport.MaxIdleConnsPerHost = size
}

func (u *Upstream) FilterRequest(request *falcore.Request) (res *http.Response) {
	var err error
	req := request.HttpRequest

	// Force the upstream to use http
	if u.ForceHttp || req.URL.Scheme == "" {
		req.URL.Scheme = "http"
		req.URL.Host = req.Host
	}
	before := time.Now()
	req.Header.Set("Connection", "Keep-Alive")
	if u.tcpconn != nil {
		u.tcpconn.SetDeadline(time.Now().Add(u.Timeout))
	}
	res, err = u.transport.RoundTrip(req)
	diff := falcore.TimeDiff(before, time.Now())
	if err != nil {
		if nerr, ok := err.(net.Error); ok && nerr.Timeout() {
			falcore.Error("%s Upstream Timeout error: %v", request.ID, err)
			res = falcore.SimpleResponse(req, 504, nil, "Gateway Timeout\n")
			request.CurrentStage.Status = 2 // Fail
		} else {
			falcore.Error("%s Upstream error: %v", request.ID, err)
			res = falcore.SimpleResponse(req, 502, nil, "Bad Gateway\n")
			request.CurrentStage.Status = 2 // Fail
		}
	}
	falcore.Debug("%s [%s] [%s%s] s=%d Time=%.4f", request.ID, req.Method, u.host, req.URL, res.StatusCode, diff)
	return
}

func (u *Upstream) ping() (up bool, ok bool) {
	if u.PingPath != "" {
		// the url must be syntactically valid for this to work but the host will be ignored because we
		// are overriding the connection always
		request, err := http.NewRequest("GET", "http://localhost"+u.PingPath, nil)
		request.Header.Set("Connection", "Keep-Alive") // not sure if this should be here for a ping
		if err != nil {
			falcore.Error("Bad Ping request: %v", err)
			return false, true
		}
		if u.tcpconn != nil {
			u.tcpconn.SetDeadline(time.Now().Add(u.Timeout))
		}
		res, err := u.transport.RoundTrip(request)

		if err != nil {
			falcore.Error("Failed Ping to %v:%v: %v", u.Host, u.Port, err)
			return false, true
		} else {
			res.Body.Close()
		}
		if res.StatusCode == 200 {
			return true, true
		}
		falcore.Error("Failed Ping to %v:%v: %v", u.Host, u.Port, res.Status)
		// bad status
		return false, true
	}
	return false, false
}
<commit_msg>cleaned up debug log output<commit_after>package upstream

import (
	"fmt"
	"github.com/ngmoco/falcore"
	"net"
	"net/http"
	"time"
)

type Upstream struct {
	// The upstream host to connect to
	Host string
	// The port on the upstream host
	Port int
	// Default 60 seconds
	Timeout time.Duration
	// Will ignore https on the incoming request and always upstream http
	ForceHttp bool
	// Ping URL Path-only for checking upness
	PingPath string

	transport *http.Transport
	host      string
	tcpaddr   *net.TCPAddr
	tcpconn   *net.TCPConn
}

func NewUpstream(host string, port int, forceHttp bool) *Upstream {
	u := new(Upstream)
	u.Host = host
	u.Port = port
	u.ForceHttp = forceHttp
	ips, err := net.LookupIP(host)
	var ip net.IP = nil
	for i := range ips {
		ip = ips[i].To4()
		if ip != nil {
			break
		}
	}
	if err == nil && ip != nil {
		u.tcpaddr = new(net.TCPAddr)
		u.tcpaddr.Port = port
		u.tcpaddr.IP = ip
	} else {
		falcore.Warn("Can't get IP addr for %v: %v", host, err)
	}
	u.Timeout = 60e9
	u.host = fmt.Sprintf("%v:%v", u.Host, u.Port)

	u.transport = new(http.Transport)

	u.transport.Dial = func(n, addr string) (c net.Conn, err error) {
		falcore.Fine("Dialing connection to %v", u.tcpaddr)
		var ctcp *net.TCPConn
		ctcp, err = net.DialTCP("tcp4", nil, u.tcpaddr)
		if ctcp != nil {
			u.tcpconn = ctcp
			u.tcpconn.SetDeadline(time.Now().Add(u.Timeout))
		}
		if err != nil {
			falcore.Error("Dial Failed: %v", err)
		}
		return ctcp, err
	}
	u.transport.MaxIdleConnsPerHost = 15
	return u
}

// Alter the number of connections to multiplex with
func (u *Upstream) SetPoolSize(size int) {
	u.transport.MaxIdleConnsPerHost = size
}

func (u *Upstream) FilterRequest(request *falcore.Request) (res *http.Response) {
	var err error
	req := request.HttpRequest

	// Force the upstream to use http
	if u.ForceHttp || req.URL.Scheme == "" {
		req.URL.Scheme = "http"
		req.URL.Host = req.Host
	}
	before := time.Now()
	req.Header.Set("Connection", "Keep-Alive")
	if u.tcpconn != nil {
		u.tcpconn.SetDeadline(time.Now().Add(u.Timeout))
	}
	res, err = u.transport.RoundTrip(req)
	diff := falcore.TimeDiff(before, time.Now())
	if err != nil {
		if nerr, ok := err.(net.Error); ok && nerr.Timeout() {
			falcore.Error("%s Upstream Timeout error: %v", request.ID, err)
			res = falcore.SimpleResponse(req, 504, nil, "Gateway Timeout\n")
			request.CurrentStage.Status = 2 // Fail
		} else {
			falcore.Error("%s Upstream error: %v", request.ID, err)
			res = falcore.SimpleResponse(req, 502, nil, "Bad Gateway\n")
			request.CurrentStage.Status = 2 // Fail
		}
	}
	falcore.Debug("%s [%s] [%s] %s s=%d Time=%.4f", request.ID, req.Method, u.host, req.URL, res.StatusCode, diff)
	return
}

func (u *Upstream) ping() (up bool, ok bool) {
	if u.PingPath != "" {
		// the url must be syntactically valid for this to work but the host will be ignored because we
		// are overriding the connection always
		request, err := http.NewRequest("GET", "http://localhost"+u.PingPath, nil)
		request.Header.Set("Connection", "Keep-Alive") // not sure if this should be here for a ping
		if err != nil {
			falcore.Error("Bad Ping request: %v", err)
			return false, true
		}
		if u.tcpconn != nil {
			u.tcpconn.SetDeadline(time.Now().Add(u.Timeout))
		}
		res, err := u.transport.RoundTrip(request)

		if err != nil {
			falcore.Error("Failed Ping to %v:%v: %v", u.Host, u.Port, err)
			return false, true
		} else {
			res.Body.Close()
		}
		if res.StatusCode == 200 {
			return true, true
		}
		falcore.Error("Failed Ping to %v:%v: %v", u.Host, u.Port, res.Status)
		// bad status
		return false, true
	}
	return false, false
}
<|endoftext|>


<commit_before>// Copyright (C) 2013-2015 by Maxim Bublis <b@codemonkey.ru>
//
// Permission is hereby granted, free of charge, to any person obtaining
// a copy of this software and associated documentation files (the
// "Software"), to deal in the Software without restriction, including
// without limitation the rights to use, copy, modify, merge, publish,
// distribute, sublicense, and/or sell copies of the Software, and to
// permit persons to whom the Software is furnished to do so, subject to
// the following conditions:
//
// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
// LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
// OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
// WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

// Package uuid provides implementation of Universally Unique Identifier (UUID).
// Supported versions are 1, 3, 4 and 5 (as specified in RFC 4122) and
// version 2 (as specified in DCE 1.1).
package uuid

import (
	"bytes"
	"crypto/md5"
	"crypto/rand"
	"crypto/sha1"
	"database/sql/driver"
	"encoding/binary"
	"encoding/hex"
	"encoding/json"
	"fmt"
	"hash"
	"net"
	"os"
	"sync"
	"time"
)

// UUID layout variants.
const (
	VariantNCS = iota
	VariantRFC4122
	VariantMicrosoft
	VariantFuture
)

// UUID DCE domains.
const (
	DomainPerson = iota
	DomainGroup
	DomainOrg
)

// Difference in 100-nanosecond intervals between
// UUID epoch (October 15, 1582) and Unix epoch (January 1, 1970).
const epochStart = 122192928000000000

// UUID v1/v2 storage.
var (
	storageMutex  sync.Mutex
	clockSequence uint16
	lastTime      uint64
	hardwareAddr  [6]byte
	posixUID      = uint32(os.Getuid())
	posixGID      = uint32(os.Getgid())
)

// String parse helpers.
var (
	urnPrefix  = []byte("urn:uuid:")
	byteGroups = []int{8, 4, 4, 4, 12}
)

// Epoch calculation function
var epochFunc func() uint64

// Initialize storage
func init() {
	buf := make([]byte, 2)
	rand.Read(buf)
	clockSequence = binary.BigEndian.Uint16(buf)

	// Initialize hardwareAddr randomly in case
	// of real network interfaces absence
	rand.Read(hardwareAddr[:])

	// Set multicast bit as recommended in RFC 4122
	hardwareAddr[0] |= 0x01

	interfaces, err := net.Interfaces()
	if err == nil {
		for _, iface := range interfaces {
			if len(iface.HardwareAddr) >= 6 {
				copy(hardwareAddr[:], iface.HardwareAddr)
				break
			}
		}
	}
	epochFunc = unixTimeFunc
}

// Returns difference in 100-nanosecond intervals between
// UUID epoch (October 15, 1582) and current time.
// This is default epoch calculation function.
func unixTimeFunc() uint64 {
	return epochStart + uint64(time.Now().UnixNano()/100)
}

// UUID representation compliant with specification
// described in RFC 4122.
type UUID [16]byte

// Predefined namespace UUIDs.
var (
	NamespaceDNS, _  = FromString("6ba7b810-9dad-11d1-80b4-00c04fd430c8")
	NamespaceURL, _  = FromString("6ba7b811-9dad-11d1-80b4-00c04fd430c8")
	NamespaceOID, _  = FromString("6ba7b812-9dad-11d1-80b4-00c04fd430c8")
	NamespaceX500, _ = FromString("6ba7b814-9dad-11d1-80b4-00c04fd430c8")
)

// And returns result of binary AND of two UUIDs.
func And(u1 UUID, u2 UUID) UUID {
	u := UUID{}
	for i := 0; i < 16; i++ {
		u[i] = u1[i] & u2[i]
	}
	return u
}

// Or returns result of binary OR of two UUIDs.
func Or(u1 UUID, u2 UUID) UUID {
	u := UUID{}
	for i := 0; i < 16; i++ {
		u[i] = u1[i] | u2[i]
	}
	return u
}

// Equal returns true if u1 and u2 equals, otherwise returns false.
func Equal(u1 UUID, u2 UUID) bool {
	return bytes.Equal(u1[:], u2[:])
}

// Version returns algorithm version used to generate UUID.
func (u UUID) Version() uint {
	return uint(u[6] >> 4)
}

// Variant returns UUID layout variant.
func (u UUID) Variant() uint {
	switch {
	case (u[8] & 0x80) == 0x00:
		return VariantNCS
	case (u[8]&0xc0)|0x80 == 0x80:
		return VariantRFC4122
	case (u[8]&0xe0)|0xc0 == 0xc0:
		return VariantMicrosoft
	}
	return VariantFuture
}

// Bytes returns bytes slice representation of UUID.
func (u UUID) Bytes() []byte {
	return u[:]
}

// Returns canonical string representation of UUID:
// xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx.
func (u UUID) String() string {
	return fmt.Sprintf("%x-%x-%x-%x-%x",
		u[:4], u[4:6], u[6:8], u[8:10], u[10:])
}

// SetVersion sets version bits.
func (u *UUID) SetVersion(v byte) {
	u[6] = (u[6] & 0x0f) | (v << 4)
}

// SetVariant sets variant bits as described in RFC 4122.
func (u *UUID) SetVariant() {
	u[8] = (u[8] & 0xbf) | 0x80
}

// MarshalText implements the encoding.TextMarshaler interface.
// The encoding is the same as returned by String.
func (u UUID) MarshalText() (text []byte, err error) {
	text = []byte(u.String())
	return
}

func (u UUID) MarshalJSON() ([]byte, error) {
	if u == (UUID{}) {
		return []byte("null"), nil
	} else {
		return []byte(`"` + u.String() + `"`), nil
	}
}

func (u *UUID) UnmarshalJSON(input []byte) error {
	var val interface{}
	if err := json.Unmarshal(input, &val); err != nil {
		return err
	}

	if val == nil {
		*u = UUID{}
		return nil
	}

	if v, ok := val.(string); ok {
		if n, err := FromString(v); err != nil {
			return err
		} else {
			*u = n
			return nil
		}
	}

	return fmt.Errorf("invalid value for UnmarshalJson()")
}

func (u UUID) Value() (driver.Value, error) {
	if u == (UUID{}) {
		return nil, nil
	} else {
		return u.String(), nil
	}
}

// UnmarshalText implements the encoding.TextUnmarshaler interface.
// Following formats are supported:
// "6ba7b810-9dad-11d1-80b4-00c04fd430c8",
// "{6ba7b810-9dad-11d1-80b4-00c04fd430c8}",
// "urn:uuid:6ba7b810-9dad-11d1-80b4-00c04fd430c8"
func (u *UUID) UnmarshalText(text []byte) (err error) {
	if len(text) < 32 {
		err = fmt.Errorf("uuid: invalid UUID string: %s", text)
		return
	}

	if bytes.Equal(text[:9], urnPrefix) {
		text = text[9:]
	} else if text[0] == '{' {
		text = text[1:]
	}

	b := u[:]

	for _, byteGroup := range byteGroups {
		if text[0] == '-' {
			text = text[1:]
		}

		_, err = hex.Decode(b[:byteGroup/2], text[:byteGroup])

		if err != nil {
			return
		}

		text = text[byteGroup:]
		b = b[byteGroup/2:]
	}

	return
}

// MarshalBinary implements the encoding.BinaryMarshaler interface.
func (u UUID) MarshalBinary() (data []byte, err error) {
	data = u.Bytes()
	return
}

// UnmarshalBinary implements the encoding.BinaryUnmarshaler interface.
// It will return error if the slice isn't 16 bytes long.
func (u *UUID) UnmarshalBinary(data []byte) (err error) {
	if len(data) != 16 {
		err = fmt.Errorf("uuid: UUID must be exactly 16 bytes long, got %d bytes", len(data))
		return
	}
	copy(u[:], data)

	return
}

// Scan implements the sql.Scanner interface.
// A 16-byte slice is handled by UnmarshalBinary, while
// a longer byte slice or a string is handled by UnmarshalText.
func (u *UUID) Scan(src interface{}) error {
	switch src := src.(type) {
	case []byte:
		if len(src) == 16 {
			return u.UnmarshalBinary(src)
		}
		return u.UnmarshalText(src)

	case string:
		return u.UnmarshalText([]byte(src))
	}

	return fmt.Errorf("uuid: cannot convert %T to UUID", src)
}

// FromBytes returns UUID converted from raw byte slice input.
// It will return error if the slice isn't 16 bytes long.
func FromBytes(input []byte) (u UUID, err error) {
	err = u.UnmarshalBinary(input)
	return
}

// FromString returns UUID parsed from string input.
// Input is expected in a form accepted by UnmarshalText.
func FromString(input string) (u UUID, err error) {
	err = u.UnmarshalText([]byte(input))
	return
}

// Returns UUID v1/v2 storage state.
// Returns epoch timestamp and clock sequence.
func getStorage() (uint64, uint16) {
	storageMutex.Lock()
	defer storageMutex.Unlock()

	timeNow := epochFunc()
	// Clock changed backwards since last UUID generation.
	// Should increase clock sequence.
	if timeNow <= lastTime {
		clockSequence++
	}
	lastTime = timeNow

	return timeNow, clockSequence
}

// NewV1 returns UUID based on current timestamp and MAC address.
func NewV1() UUID {
	u := UUID{}

	timeNow, clockSeq := getStorage()

	binary.BigEndian.PutUint32(u[0:], uint32(timeNow))
	binary.BigEndian.PutUint16(u[4:], uint16(timeNow>>32))
	binary.BigEndian.PutUint16(u[6:], uint16(timeNow>>48))
	binary.BigEndian.PutUint16(u[8:], clockSeq)

	copy(u[10:], hardwareAddr[:])

	u.SetVersion(1)
	u.SetVariant()

	return u
}

// NewV2 returns DCE Security UUID based on POSIX UID/GID.
func NewV2(domain byte) UUID {
	u := UUID{}

	switch domain {
	case DomainPerson:
		binary.BigEndian.PutUint32(u[0:], posixUID)
	case DomainGroup:
		binary.BigEndian.PutUint32(u[0:], posixGID)
	}

	timeNow, clockSeq := getStorage()

	binary.BigEndian.PutUint16(u[4:], uint16(timeNow>>32))
	binary.BigEndian.PutUint16(u[6:], uint16(timeNow>>48))
	binary.BigEndian.PutUint16(u[8:], clockSeq)
	u[9] = domain

	copy(u[10:], hardwareAddr[:])

	u.SetVersion(2)
	u.SetVariant()

	return u
}

// NewV3 returns UUID based on MD5 hash of namespace UUID and name.
func NewV3(ns UUID, name string) UUID {
	u := newFromHash(md5.New(), ns, name)
	u.SetVersion(3)
	u.SetVariant()

	return u
}

// NewV4 returns random generated UUID.
func NewV4() UUID {
	u := UUID{}
	rand.Read(u[:])
	u.SetVersion(4)
	u.SetVariant()

	return u
}

// NewV5 returns UUID based on SHA-1 hash of namespace UUID and name.
func NewV5(ns UUID, name string) UUID {
	u := newFromHash(sha1.New(), ns, name)
	u.SetVersion(5)
	u.SetVariant()

	return u
}

// Returns UUID based on hashing of namespace UUID and name.
func newFromHash(h hash.Hash, ns UUID, name string) UUID {
	u := UUID{}
	h.Write(ns[:])
	h.Write([]byte(name))
	copy(u[:], h.Sum(nil))

	return u
}
<commit_msg>support scanning a uuid into a uuid<commit_after>// Copyright (C) 2013-2015 by Maxim Bublis <b@codemonkey.ru>
//
// Permission is hereby granted, free of charge, to any person obtaining
// a copy of this software and associated documentation files (the
// "Software"), to deal in the Software without restriction, including
// without limitation the rights to use, copy, modify, merge, publish,
// distribute, sublicense, and/or sell copies of the Software, and to
// permit persons to whom the Software is furnished to do so, subject to
// the following conditions:
//
// The above copyright notice and this permission notice shall be
// included in all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
// EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
// MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
// NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
// LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
// OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
// WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

// Package uuid provides implementation of Universally Unique Identifier (UUID).
// Supported versions are 1, 3, 4 and 5 (as specified in RFC 4122) and
// version 2 (as specified in DCE 1.1).
package uuid

import (
	"bytes"
	"crypto/md5"
	"crypto/rand"
	"crypto/sha1"
	"database/sql/driver"
	"encoding/binary"
	"encoding/hex"
	"encoding/json"
	"fmt"
	"hash"
	"net"
	"os"
	"sync"
	"time"
)

// UUID layout variants.
const (
	VariantNCS = iota
	VariantRFC4122
	VariantMicrosoft
	VariantFuture
)

// UUID DCE domains.
const (
	DomainPerson = iota
	DomainGroup
	DomainOrg
)

// Difference in 100-nanosecond intervals between
// UUID epoch (October 15, 1582) and Unix epoch (January 1, 1970).
const epochStart = 122192928000000000

// UUID v1/v2 storage.
var (
	storageMutex  sync.Mutex
	clockSequence uint16
	lastTime      uint64
	hardwareAddr  [6]byte
	posixUID      = uint32(os.Getuid())
	posixGID      = uint32(os.Getgid())
)

// String parse helpers.
var (
	urnPrefix  = []byte("urn:uuid:")
	byteGroups = []int{8, 4, 4, 4, 12}
)

// Epoch calculation function
var epochFunc func() uint64

// Initialize storage
func init() {
	buf := make([]byte, 2)
	rand.Read(buf)
	clockSequence = binary.BigEndian.Uint16(buf)

	// Initialize hardwareAddr randomly in case
	// of real network interfaces absence
	rand.Read(hardwareAddr[:])

	// Set multicast bit as recommended in RFC 4122
	hardwareAddr[0] |= 0x01

	interfaces, err := net.Interfaces()
	if err == nil {
		for _, iface := range interfaces {
			if len(iface.HardwareAddr) >= 6 {
				copy(hardwareAddr[:], iface.HardwareAddr)
				break
			}
		}
	}
	epochFunc = unixTimeFunc
}

// Returns difference in 100-nanosecond intervals between
// UUID epoch (October 15, 1582) and current time.
// This is default epoch calculation function.
func unixTimeFunc() uint64 {
	return epochStart + uint64(time.Now().UnixNano()/100)
}

// UUID representation compliant with specification
// described in RFC 4122.
type UUID [16]byte

// Predefined namespace UUIDs.
var (
	NamespaceDNS, _  = FromString("6ba7b810-9dad-11d1-80b4-00c04fd430c8")
	NamespaceURL, _  = FromString("6ba7b811-9dad-11d1-80b4-00c04fd430c8")
	NamespaceOID, _  = FromString("6ba7b812-9dad-11d1-80b4-00c04fd430c8")
	NamespaceX500, _ = FromString("6ba7b814-9dad-11d1-80b4-00c04fd430c8")
)

// And returns result of binary AND of two UUIDs.
func And(u1 UUID, u2 UUID) UUID {
	u := UUID{}
	for i := 0; i < 16; i++ {
		u[i] = u1[i] & u2[i]
	}
	return u
}

// Or returns result of binary OR of two UUIDs.
func Or(u1 UUID, u2 UUID) UUID {
	u := UUID{}
	for i := 0; i < 16; i++ {
		u[i] = u1[i] | u2[i]
	}
	return u
}

// Equal returns true if u1 and u2 equals, otherwise returns false.
func Equal(u1 UUID, u2 UUID) bool {
	return bytes.Equal(u1[:], u2[:])
}

// Version returns algorithm version used to generate UUID.
func (u UUID) Version() uint {
	return uint(u[6] >> 4)
}

// Variant returns UUID layout variant.
func (u UUID) Variant() uint {
	switch {
	case (u[8] & 0x80) == 0x00:
		return VariantNCS
	case (u[8]&0xc0)|0x80 == 0x80:
		return VariantRFC4122
	case (u[8]&0xe0)|0xc0 == 0xc0:
		return VariantMicrosoft
	}
	return VariantFuture
}

// Bytes returns bytes slice representation of UUID.
func (u UUID) Bytes() []byte {
	return u[:]
}

// Returns canonical string representation of UUID:
// xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx.
func (u UUID) String() string {
	return fmt.Sprintf("%x-%x-%x-%x-%x",
		u[:4], u[4:6], u[6:8], u[8:10], u[10:])
}

// SetVersion sets version bits.
func (u *UUID) SetVersion(v byte) {
	u[6] = (u[6] & 0x0f) | (v << 4)
}

// SetVariant sets variant bits as described in RFC 4122.
func (u *UUID) SetVariant() {
	u[8] = (u[8] & 0xbf) | 0x80
}

// MarshalText implements the encoding.TextMarshaler interface.
// The encoding is the same as returned by String.
func (u UUID) MarshalText() (text []byte, err error) {
	text = []byte(u.String())
	return
}

func (u UUID) MarshalJSON() ([]byte, error) {
	if u == (UUID{}) {
		return []byte("null"), nil
	} else {
		return []byte(`"` + u.String() + `"`), nil
	}
}

func (u *UUID) UnmarshalJSON(input []byte) error {
	var val interface{}
	if err := json.Unmarshal(input, &val); err != nil {
		return err
	}

	if val == nil {
		*u = UUID{}
		return nil
	}

	if v, ok := val.(string); ok {
		if n, err := FromString(v); err != nil {
			return err
		} else {
			*u = n
			return nil
		}
	}

	return fmt.Errorf("invalid value for UnmarshalJson()")
}

func (u UUID) Value() (driver.Value, error) {
	if u == (UUID{}) {
		return nil, nil
	} else {
		return u.String(), nil
	}
}

// UnmarshalText implements the encoding.TextUnmarshaler interface.
// Following formats are supported:
// "6ba7b810-9dad-11d1-80b4-00c04fd430c8",
// "{6ba7b810-9dad-11d1-80b4-00c04fd430c8}",
// "urn:uuid:6ba7b810-9dad-11d1-80b4-00c04fd430c8"
func (u *UUID) UnmarshalText(text []byte) (err error) {
	if len(text) < 32 {
		err = fmt.Errorf("uuid: invalid UUID string: %s", text)
		return
	}

	if bytes.Equal(text[:9], urnPrefix) {
		text = text[9:]
	} else if text[0] == '{' {
		text = text[1:]
	}

	b := u[:]

	for _, byteGroup := range byteGroups {
		if text[0] == '-' {
			text = text[1:]
		}

		_, err = hex.Decode(b[:byteGroup/2], text[:byteGroup])

		if err != nil {
			return
		}

		text = text[byteGroup:]
		b = b[byteGroup/2:]
	}

	return
}

// MarshalBinary implements the encoding.BinaryMarshaler interface.
func (u UUID) MarshalBinary() (data []byte, err error) {
	data = u.Bytes()
	return
}

// UnmarshalBinary implements the encoding.BinaryUnmarshaler interface.
// It will return error if the slice isn't 16 bytes long.
func (u *UUID) UnmarshalBinary(data []byte) (err error) {
	if len(data) != 16 {
		err = fmt.Errorf("uuid: UUID must be exactly 16 bytes long, got %d bytes", len(data))
		return
	}
	copy(u[:], data)

	return
}

// Scan implements the sql.Scanner interface.
// A 16-byte slice is handled by UnmarshalBinary, while
// a longer byte slice or a string is handled by UnmarshalText.
func (u *UUID) Scan(src interface{}) error {
	switch src := src.(type) {
	case UUID:
		*u = src
		return nil
	case []byte:
		if len(src) == 16 {
			return u.UnmarshalBinary(src)
		}
		return u.UnmarshalText(src)

	case string:
		return u.UnmarshalText([]byte(src))
	}

	return fmt.Errorf("uuid: cannot convert %T to UUID", src)
}

// FromBytes returns UUID converted from raw byte slice input.
// It will return error if the slice isn't 16 bytes long.
func FromBytes(input []byte) (u UUID, err error) {
	err = u.UnmarshalBinary(input)
	return
}

// FromString returns UUID parsed from string input.
// Input is expected in a form accepted by UnmarshalText.
func FromString(input string) (u UUID, err error) {
	err = u.UnmarshalText([]byte(input))
	return
}

// Returns UUID v1/v2 storage state.
// Returns epoch timestamp and clock sequence.
func getStorage() (uint64, uint16) {
	storageMutex.Lock()
	defer storageMutex.Unlock()

	timeNow := epochFunc()
	// Clock changed backwards since last UUID generation.
	// Should increase clock sequence.
	if timeNow <= lastTime {
		clockSequence++
	}
	lastTime = timeNow

	return timeNow, clockSequence
}

// NewV1 returns UUID based on current timestamp and MAC address.
func NewV1() UUID {
	u := UUID{}

	timeNow, clockSeq := getStorage()

	binary.BigEndian.PutUint32(u[0:], uint32(timeNow))
	binary.BigEndian.PutUint16(u[4:], uint16(timeNow>>32))
	binary.BigEndian.PutUint16(u[6:], uint16(timeNow>>48))
	binary.BigEndian.PutUint16(u[8:], clockSeq)

	copy(u[10:], hardwareAddr[:])

	u.SetVersion(1)
	u.SetVariant()

	return u
}

// NewV2 returns DCE Security UUID based on POSIX UID/GID.
func NewV2(domain byte) UUID {
	u := UUID{}

	switch domain {
	case DomainPerson:
		binary.BigEndian.PutUint32(u[0:], posixUID)
	case DomainGroup:
		binary.BigEndian.PutUint32(u[0:], posixGID)
	}

	timeNow, clockSeq := getStorage()

	binary.BigEndian.PutUint16(u[4:], uint16(timeNow>>32))
	binary.BigEndian.PutUint16(u[6:], uint16(timeNow>>48))
	binary.BigEndian.PutUint16(u[8:], clockSeq)
	u[9] = domain

	copy(u[10:], hardwareAddr[:])

	u.SetVersion(2)
	u.SetVariant()

	return u
}

// NewV3 returns UUID based on MD5 hash of namespace UUID and name.
func NewV3(ns UUID, name string) UUID {
	u := newFromHash(md5.New(), ns, name)
	u.SetVersion(3)
	u.SetVariant()

	return u
}

// NewV4 returns random generated UUID.
func NewV4() UUID {
	u := UUID{}
	rand.Read(u[:])
	u.SetVersion(4)
	u.SetVariant()

	return u
}

// NewV5 returns UUID based on SHA-1 hash of namespace UUID and name.
func NewV5(ns UUID, name string) UUID {
	u := newFromHash(sha1.New(), ns, name)
	u.SetVersion(5)
	u.SetVariant()

	return u
}

// Returns UUID based on hashing of namespace UUID and name.
func newFromHash(h hash.Hash, ns UUID, name string) UUID {
	u := UUID{}
	h.Write(ns[:])
	h.Write([]byte(name))
	copy(u[:], h.Sum(nil))

	return u
}
<|endoftext|>


<commit_before>package uuid

import (
	"crypto/rand"
	"encoding/hex"
	"fmt"
)

// GenerateRandomBytes is used to generate random bytes of given size.
func GenerateRandomBytes(size int) ([]byte, error) {
	buf := make([]byte, size)
	if _, err := rand.Read(buf); err != nil {
		return nil, fmt.Errorf("failed to read random bytes: %v", err)
	}
	return buf, nil
}

// GenerateUUID is used to generate a random UUID
func GenerateUUID() (string, error) {
	buf, err := GenerateRandomBytes(16)
	if err != nil {
		return "", err
	}
	return FormatUUID(buf)
}

func FormatUUID(buf []byte) (string, error) {
	if len(buf) != 16 {
		return "", fmt.Errorf("wrong length byte slice (%d)", len(buf))
	}

	return fmt.Sprintf("%08x-%04x-%04x-%04x-%12x",
		buf[0:4],
		buf[4:6],
		buf[6:8],
		buf[8:10],
		buf[10:16]), nil
}

func ParseUUID(uuid string) ([]byte, error) {
	if len(uuid) != 36 {
		return nil, fmt.Errorf("uuid string is wrong length")
	}

	hyph := []byte("-")

	if uuid[8] != hyph[0] ||
		uuid[13] != hyph[0] ||
		uuid[18] != hyph[0] ||
		uuid[23] != hyph[0] {
		return nil, fmt.Errorf("uuid is improperly formatted")
	}

	hexStr := uuid[0:8] + uuid[9:13] + uuid[14:18] + uuid[19:23] + uuid[24:36]

	ret, err := hex.DecodeString(hexStr)
	if err != nil {
		return nil, err
	}
	if len(ret) != 16 {
		return nil, fmt.Errorf("decoded hex is the wrong length")
	}

	return ret, nil
}
<commit_msg>Refactor uuid.go (#9)<commit_after>package uuid

import (
	"crypto/rand"
	"encoding/hex"
	"fmt"
)

// GenerateRandomBytes is used to generate random bytes of given size.
func GenerateRandomBytes(size int) ([]byte, error) {
	buf := make([]byte, size)
	if _, err := rand.Read(buf); err != nil {
		return nil, fmt.Errorf("failed to read random bytes: %v", err)
	}
	return buf, nil
}

const uuidLen = 16

// GenerateUUID is used to generate a random UUID
func GenerateUUID() (string, error) {
	buf, err := GenerateRandomBytes(uuidLen)
	if err != nil {
		return "", err
	}
	return FormatUUID(buf)
}

func FormatUUID(buf []byte) (string, error) {
	if buflen := len(buf); buflen != uuidLen {
		return "", fmt.Errorf("wrong length byte slice (%d)", buflen)
	}

	return fmt.Sprintf("%x-%x-%x-%x-%x",
		buf[0:4],
		buf[4:6],
		buf[6:8],
		buf[8:10],
		buf[10:16]), nil
}

func ParseUUID(uuid string) ([]byte, error) {
	if len(uuid) != 2 * uuidLen + 4 {
		return nil, fmt.Errorf("uuid string is wrong length")
	}

	if uuid[8] != '-' ||
		uuid[13] != '-' ||
		uuid[18] != '-' ||
		uuid[23] != '-' {
		return nil, fmt.Errorf("uuid is improperly formatted")
	}

	hexStr := uuid[0:8] + uuid[9:13] + uuid[14:18] + uuid[19:23] + uuid[24:36]

	ret, err := hex.DecodeString(hexStr)
	if err != nil {
		return nil, err
	}
	if len(ret) != uuidLen {
		return nil, fmt.Errorf("decoded hex is the wrong length")
	}

	return ret, nil
}
<|endoftext|>


<commit_before>package driver

import (
	"github.com/WeisswurstSystems/WWM-BB/database"
	"github.com/WeisswurstSystems/WWM-BB/user"
	"gopkg.in/mgo.v2"
	"gopkg.in/mgo.v2/bson"
)

type mongoStore struct {
	session *mgo.Session
	users   *mgo.Collection
}

func NewMongoStore() user.Store {
	databaseName := database.GetEnv("DB_NAME", "wwmbb-dev")
	var store mongoStore

	store.session = database.NewMongoSession()
	store.session.SetMode(mgo.Monotonic, true)
	store.users = store.session.DB(databaseName).C("users")
	return &store
}

func (s *mongoStore) FindByMail(email string) (user.User, error) {
	var findByUserMail user.User
	err := s.users.Find(bson.M{"login.mail": email}).One(&findByUserMail)
	if err == mgo.ErrNotFound {
		return user.User{}, user.ErrNotFound
	}
	return findByUserMail, err
}

func (s *mongoStore) FindByRegistrationID(id string) (user.User, error) {
	var findByRegId user.User
	err := s.users.Find(bson.M{"registrationid": id}).One(&findByRegId)
	if err == mgo.ErrNotFound {
		return user.User{}, user.ErrNotFound
	}
	return findByRegId, err
}

// Returns all _active_ users. This func does not return users which have not finished registering
func (s *mongoStore) FindAll() ([]user.User, error) {
	var results []user.User
	err := s.users.Find(bson.M{"registrationid": ""}).All(&results)
	return results, err
}

// Returns all inactive users --> Users which have not completed the registration process.
func (s *mongoStore) FindAllUnregistered() ([]user.User, error) {
	var results []user.User
	err := s.users.Find(bson.M{"registrationid": bson.M{"$ne": ""}}).All(&results)
	return results, err
}

func (s *mongoStore) Save(user user.User) error {
	err := s.users.Insert(&user)
	return err
}
<commit_msg>change<commit_after>package driver

import (
	"github.com/WeisswurstSystems/WWM-BB/database"
	"github.com/WeisswurstSystems/WWM-BB/user"
	"gopkg.in/mgo.v2"
	"gopkg.in/mgo.v2/bson"
)

type mongoStore struct {
	session *mgo.Session
	users   *mgo.Collection
}

func NewMongoStore() user.Store {
	databaseName := database.GetEnv("DB_NAME", "wwmbb-dev")
	var store mongoStore

	store.session = database.NewMongoSession()
	store.session.SetMode(mgo.Monotonic, true)
	store.users = store.session.DB(databaseName).C("users")
	return &store
}

func (s *mongoStore) FindByMail(email string) (user.User, error) {
	var findByUserMail user.User
	err := s.users.Find(bson.M{"login.mail": email}).One(&findByUserMail)
	if err == mgo.ErrNotFound {
		return user.User{}, user.ErrNotFound
	}
	return findByUserMail, err
}

func (s *mongoStore) FindByRegistrationID(id string) (user.User, error) {
	var findByRegId user.User
	err := s.users.Find(bson.M{"registrationid": id}).One(&findByRegId)
	if err == mgo.ErrNotFound {
		return user.User{}, user.ErrNotFound
	}
	return findByRegId, err
}

// Returns all _active_ users. This func does not return users which have not finished registering
func (s *mongoStore) FindAll() ([]user.User, error) {
	var results []user.User
	err := s.users.Find(bson.M{"registrationid": ""}).All(&results)
	return results, err
}

// Returns all inactive users --> Users which have not completed the registration process.
func (s *mongoStore) FindAllUnregistered() ([]user.User, error) {
	var results []user.User
	err := s.users.Find(bson.M{"registrationid": bson.M{"$ne": ""}}).All(&results)
	return results, err
}

func (s *mongoStore) Save(user user.User) error {
	err := s.users.Insert(&user)
	return err
}<|endoftext|>


<commit_before>// Copyright 2020 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package moddeps_test

import (
	"encoding/json"
	"fmt"
	"internal/testenv"
	"io/fs"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"runtime"
	"strings"
	"sync"
	"testing"

	"golang.org/x/mod/module"
)

type gorootModule struct {
	Path      string
	Dir       string
	hasVendor bool
}

// findGorootModules returns the list of modules found in the GOROOT source tree.
func findGorootModules(t *testing.T) []gorootModule {
	t.Helper()
	goBin := testenv.GoToolPath(t)

	goroot.once.Do(func() {
		goroot.err = filepath.WalkDir(runtime.GOROOT(), func(path string, info fs.DirEntry, err error) error {
			if err != nil {
				return err
			}
			if info.IsDir() && (info.Name() == "vendor" || info.Name() == "testdata") {
				return filepath.SkipDir
			}
			if path == filepath.Join(runtime.GOROOT(), "pkg") {
				// GOROOT/pkg contains generated artifacts, not source code.
				//
				// In https://golang.org/issue/37929 it was observed to somehow contain
				// a module cache, so it is important to skip. (That helps with the
				// running time of this test anyway.)
				return filepath.SkipDir
			}
			if info.IsDir() || info.Name() != "go.mod" {
				return nil
			}
			dir := filepath.Dir(path)

			// Use 'go list' to describe the module contained in this directory (but
			// not its dependencies).
			cmd := exec.Command(goBin, "list", "-json", "-m")
			cmd.Env = append(os.Environ(), "GO111MODULE=on")
			cmd.Dir = dir
			cmd.Stderr = new(strings.Builder)
			out, err := cmd.Output()
			if err != nil {
				return fmt.Errorf("'go list -json -m' in %s: %w\n%s", dir, err, cmd.Stderr)
			}

			var m gorootModule
			if err := json.Unmarshal(out, &m); err != nil {
				return fmt.Errorf("decoding 'go list -json -m' in %s: %w", dir, err)
			}
			if m.Path == "" || m.Dir == "" {
				return fmt.Errorf("'go list -json -m' in %s failed to populate Path and/or Dir", dir)
			}
			if _, err := os.Stat(filepath.Join(dir, "vendor")); err == nil {
				m.hasVendor = true
			}
			goroot.modules = append(goroot.modules, m)
			return nil
		})
	})

	if goroot.err != nil {
		t.Fatal(goroot.err)
	}
	return goroot.modules
}

// goroot caches the list of modules found in the GOROOT source tree.
var goroot struct {
	once    sync.Once
	modules []gorootModule
	err     error
}

// TestAllDependenciesVendored ensures that all packages imported within GOROOT
// are vendored in the corresponding GOROOT module.
//
// This property allows offline development within the Go project, and ensures
// that all dependency changes are presented in the usual code review process.
//
// This test does NOT ensure that the vendored contents match the unmodified
// contents of the corresponding dependency versions. Such as test would require
// network access, and would currently either need to copy the entire GOROOT module
// or explicitly invoke version control to check for changes.
// (See golang.org/issue/36852 and golang.org/issue/27348.)
func TestAllDependenciesVendored(t *testing.T) {
	goBin := testenv.GoToolPath(t)

	for _, m := range findGorootModules(t) {
		t.Run(m.Path, func(t *testing.T) {
			if m.hasVendor {
				// Load all of the packages in the module to ensure that their
				// dependencies are vendored. If any imported package is missing,
				// 'go list -deps' will fail when attempting to load it.
				cmd := exec.Command(goBin, "list", "-mod=vendor", "-deps", "./...")
				cmd.Env = append(os.Environ(), "GO111MODULE=on")
				cmd.Dir = m.Dir
				cmd.Stderr = new(strings.Builder)
				_, err := cmd.Output()
				if err != nil {
					t.Errorf("%s: %v\n%s", strings.Join(cmd.Args, " "), err, cmd.Stderr)
					t.Logf("(Run 'go mod vendor' in %s to ensure that dependecies have been vendored.)", m.Dir)
				}
				return
			}

			// There is no vendor directory, so the module must have no dependencies.
			// Check that the list of active modules contains only the main module.
			cmd := exec.Command(goBin, "list", "-mod=mod", "-m", "all")
			cmd.Env = append(os.Environ(), "GO111MODULE=on")
			cmd.Dir = m.Dir
			cmd.Stderr = new(strings.Builder)
			out, err := cmd.Output()
			if err != nil {
				t.Fatalf("%s: %v\n%s", strings.Join(cmd.Args, " "), err, cmd.Stderr)
			}
			if strings.TrimSpace(string(out)) != m.Path {
				t.Errorf("'%s' reported active modules other than %s:\n%s", strings.Join(cmd.Args, " "), m.Path, out)
				t.Logf("(Run 'go mod tidy' in %s to ensure that no extraneous dependencies were added, or 'go mod vendor' to copy in imported packages.)", m.Dir)
			}
		})
	}
}

// TestDependencyVersionsConsistent verifies that each module in GOROOT that
// requires a given external dependency requires the same version of that
// dependency.
//
// This property allows us to maintain a single release branch of each such
// dependency, minimizing the number of backports needed to pull in critical
// fixes. It also ensures that any bug detected and fixed in one GOROOT module
// (such as "std") is fixed in all other modules (such as "cmd") as well.
func TestDependencyVersionsConsistent(t *testing.T) {
	// Collect the dependencies of all modules in GOROOT, indexed by module path.
	type requirement struct {
		Required    module.Version
		Replacement module.Version
	}
	seen := map[string]map[requirement][]gorootModule{} // module path → requirement → set of modules with that requirement
	for _, m := range findGorootModules(t) {
		if !m.hasVendor {
			// TestAllDependenciesVendored will ensure that the module has no
			// dependencies.
			continue
		}

		// We want this test to be able to run offline and with an empty module
		// cache, so we verify consistency only for the module versions listed in
		// vendor/modules.txt. That includes all direct dependencies and all modules
		// that provide any imported packages.
		//
		// It's ok if there are undetected differences in modules that do not
		// provide imported packages: we will not have to pull in any backports of
		// fixes to those modules anyway.
		vendor, err := ioutil.ReadFile(filepath.Join(m.Dir, "vendor", "modules.txt"))
		if err != nil {
			t.Error(err)
			continue
		}

		for _, line := range strings.Split(strings.TrimSpace(string(vendor)), "\n") {
			parts := strings.Fields(line)
			if len(parts) < 3 || parts[0] != "#" {
				continue
			}

			// This line is of the form "# module version [=> replacement [version]]".
			var r requirement
			r.Required.Path = parts[1]
			r.Required.Version = parts[2]
			if len(parts) >= 5 && parts[3] == "=>" {
				r.Replacement.Path = parts[4]
				if module.CheckPath(r.Replacement.Path) != nil {
					// If the replacement is a filesystem path (rather than a module path),
					// we don't know whether the filesystem contents have changed since
					// the module was last vendored.
					//
					// Fortunately, we do not currently use filesystem-local replacements
					// in GOROOT modules.
					t.Errorf("cannot check consistency for filesystem-local replacement in module %s (%s):\n%s", m.Path, m.Dir, line)
				}

				if len(parts) >= 6 {
					r.Replacement.Version = parts[5]
				}
			}

			if seen[r.Required.Path] == nil {
				seen[r.Required.Path] = make(map[requirement][]gorootModule)
			}
			seen[r.Required.Path][r] = append(seen[r.Required.Path][r], m)
		}
	}

	// Now verify that we saw only one distinct version for each module.
	for path, versions := range seen {
		if len(versions) > 1 {
			t.Errorf("Modules within GOROOT require different versions of %s.", path)
			for r, mods := range versions {
				desc := new(strings.Builder)
				desc.WriteString(r.Required.Version)
				if r.Replacement.Path != "" {
					fmt.Fprintf(desc, " => %s", r.Replacement.Path)
					if r.Replacement.Version != "" {
						fmt.Fprintf(desc, " %s", r.Replacement.Version)
					}
				}

				for _, m := range mods {
					t.Logf("%s\trequires %v", m.Path, desc)
				}
			}
		}
	}
}
<commit_msg>cmd/internal/moddeps: check content of all modules in GOROOT<commit_after>// Copyright 2020 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package moddeps_test

import (
	"encoding/json"
	"fmt"
	"internal/testenv"
	"io"
	"io/fs"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"runtime"
	"strings"
	"sync"
	"testing"

	"golang.org/x/mod/module"
)

// TestAllDependencies ensures dependencies of all
// modules in GOROOT are in a consistent state.
//
// In short mode, it does a limited quick check and stops there.
// In long mode, it also makes a copy of the entire GOROOT tree
// and requires network access to perform more thorough checks.
// Keep this distinction in mind when adding new checks.
//
// See issues 36852, 41409, and 43687.
// (Also see golang.org/issue/27348.)
func TestAllDependencies(t *testing.T) {
	goBin := testenv.GoToolPath(t)

	// Ensure that all packages imported within GOROOT
	// are vendored in the corresponding GOROOT module.
	//
	// This property allows offline development within the Go project, and ensures
	// that all dependency changes are presented in the usual code review process.
	//
	// As a quick first-order check, avoid network access and the need to copy the
	// entire GOROOT tree or explicitly invoke version control to check for changes.
	// Just check that packages are vendored. (In non-short mode, we go on to also
	// copy the GOROOT tree and perform more rigorous consistency checks. Jump below
	// for more details.)
	for _, m := range findGorootModules(t) {
		// This short test does NOT ensure that the vendored contents match
		// the unmodified contents of the corresponding dependency versions.
		t.Run(m.Path+"(quick)", func(t *testing.T) {
			if m.hasVendor {
				// Load all of the packages in the module to ensure that their
				// dependencies are vendored. If any imported package is missing,
				// 'go list -deps' will fail when attempting to load it.
				cmd := exec.Command(goBin, "list", "-mod=vendor", "-deps", "./...")
				cmd.Env = append(os.Environ(), "GO111MODULE=on")
				cmd.Dir = m.Dir
				cmd.Stderr = new(strings.Builder)
				_, err := cmd.Output()
				if err != nil {
					t.Errorf("%s: %v\n%s", strings.Join(cmd.Args, " "), err, cmd.Stderr)
					t.Logf("(Run 'go mod vendor' in %s to ensure that dependecies have been vendored.)", m.Dir)
				}
				return
			}

			// There is no vendor directory, so the module must have no dependencies.
			// Check that the list of active modules contains only the main module.
			cmd := exec.Command(goBin, "list", "-mod=mod", "-m", "all")
			cmd.Env = append(os.Environ(), "GO111MODULE=on")
			cmd.Dir = m.Dir
			cmd.Stderr = new(strings.Builder)
			out, err := cmd.Output()
			if err != nil {
				t.Fatalf("%s: %v\n%s", strings.Join(cmd.Args, " "), err, cmd.Stderr)
			}
			if strings.TrimSpace(string(out)) != m.Path {
				t.Errorf("'%s' reported active modules other than %s:\n%s", strings.Join(cmd.Args, " "), m.Path, out)
				t.Logf("(Run 'go mod tidy' in %s to ensure that no extraneous dependencies were added, or 'go mod vendor' to copy in imported packages.)", m.Dir)
			}
		})
	}

	// We now get to the slow, but more thorough part of the test.
	// Only run it in long test mode.
	if testing.Short() {
		return
	}

	// Ensure that all modules within GOROOT are tidy, vendored, and bundled.
	// Ensure that the vendored contents match the unmodified contents of the
	// corresponding dependency versions.
	//
	// The non-short section of this test requires network access and the diff
	// command.
	//
	// It makes a temporary copy of the entire GOROOT tree (where it can safely
	// perform operations that may mutate the tree), executes the same module
	// maintenance commands that we expect Go developers to run, and then
	// diffs the potentially modified module copy with the real one in GOROOT.
	// (We could try to rely on Git to do things differently, but that's not the
	// path we've chosen at this time. This allows the test to run when the tree
	// is not checked into Git.)

	testenv.MustHaveExternalNetwork(t)
	if haveDiff := func() bool {
		diff, err := exec.Command("diff", "--recursive", "--unified", ".", ".").CombinedOutput()
		if err != nil || len(diff) != 0 {
			return false
		}
		diff, err = exec.Command("diff", "--recursive", "--unified", ".", "..").CombinedOutput()
		if err == nil || len(diff) == 0 {
			return false
		}
		return true
	}(); !haveDiff {
		// For now, the diff command is a mandatory dependency of this test.
		// This test will primarily run on longtest builders, since few people
		// would test the cmd/internal/moddeps package directly, and all.bash
		// runs tests in short mode. It's fine to skip if diff is unavailable.
		t.Skip("skipping because a diff command with support for --recursive and --unified flags is unavailable")
	}

	// Build the bundle binary at the golang.org/x/tools
	// module version specified in GOROOT/src/cmd/go.mod.
	bundleDir := t.TempDir()
	r := runner{Dir: filepath.Join(runtime.GOROOT(), "src/cmd")}
	r.run(t, goBin, "build", "-mod=readonly", "-o", bundleDir, "golang.org/x/tools/cmd/bundle")

	var gorootCopyDir string
	for _, m := range findGorootModules(t) {
		// Create a test-wide GOROOT copy. It can be created once
		// and reused between subtests whenever they don't fail.
		//
		// This is a relatively expensive operation, but it's a pre-requisite to
		// be able to safely run commands like "go mod tidy", "go mod vendor", and
		// "go generate" on the GOROOT tree content. Those commands may modify the
		// tree, and we don't want to happen to the real tree as part of executing
		// a test.
		if gorootCopyDir == "" {
			gorootCopyDir = makeGOROOTCopy(t)
		}

		t.Run(m.Path+"(thorough)", func(t *testing.T) {
			defer func() {
				if t.Failed() {
					// The test failed, which means it's possible the GOROOT copy
					// may have been modified. No choice but to reset it for next
					// module test case. (This is slow, but it happens only during
					// test failures.)
					gorootCopyDir = ""
				}
			}()

			rel, err := filepath.Rel(runtime.GOROOT(), m.Dir)
			if err != nil {
				t.Fatalf("filepath.Rel(%q, %q): %v", runtime.GOROOT(), m.Dir, err)
			}
			r := runner{
				Dir: filepath.Join(gorootCopyDir, rel),
				Env: append(os.Environ(),
					// Set GOROOT.
					"GOROOT="+gorootCopyDir,
					// Explicitly clear PWD and GOROOT_FINAL so that GOROOT=gorootCopyDir is definitely used.
					"PWD=",
					"GOROOT_FINAL=",
					// Add GOROOTcopy/bin and bundleDir to front of PATH.
					"PATH="+filepath.Join(gorootCopyDir, "bin")+string(filepath.ListSeparator)+
						bundleDir+string(filepath.ListSeparator)+os.Getenv("PATH"),
				),
			}
			goBinCopy := filepath.Join(gorootCopyDir, "bin", "go")
			r.run(t, goBinCopy, "mod", "tidy")   // See issue 43687.
			r.run(t, goBinCopy, "mod", "verify") // Verify should be a no-op, but test it just in case.
			r.run(t, goBinCopy, "mod", "vendor") // See issue 36852.
			pkgs := packagePattern(m.Path)
			r.run(t, goBinCopy, "generate", `-run=^//go:generate bundle `, pkgs) // See issue 41409.
			advice := "$ cd " + m.Dir + "\n" +
				"$ go mod tidy                               # to remove extraneous dependencies\n" +
				"$ go mod vendor                             # to vendor dependecies\n" +
				"$ go generate -run=bundle " + pkgs + "               # to regenerate bundled packages\n"
			if m.Path == "std" {
				r.run(t, goBinCopy, "generate", "syscall", "internal/syscall/...") // See issue 43440.
				advice += "$ go generate syscall internal/syscall/...  # to regenerate syscall packages\n"
			}
			// TODO(golang.org/issue/43440): Check anything else influenced by dependency versions.

			diff, err := exec.Command("diff", "--recursive", "--unified", r.Dir, m.Dir).CombinedOutput()
			if err != nil || len(diff) != 0 {
				t.Errorf(`Module %s in %s is not tidy (-want +got):

%s
To fix it, run:

%s
(If module %[1]s is definitely tidy, this could mean
there's a problem in the go or bundle command.)`, m.Path, m.Dir, diff, advice)
			}
		})
	}
}

// packagePattern returns a package pattern that matches all packages
// in the module modulePath, and ideally as few others as possible.
func packagePattern(modulePath string) string {
	if modulePath == "std" {
		return "std"
	}
	return modulePath + "/..."
}

// makeGOROOTCopy makes a temporary copy of the current GOROOT tree.
// The goal is to allow the calling test t to safely mutate a GOROOT
// copy without also modifying the original GOROOT.
//
// It copies the entire tree as is, with the exception of the GOROOT/.git
// directory, which is skipped, and the GOROOT/{bin,pkg} directories,
// which are symlinked. This is done for speed, since a GOROOT tree is
// functional without being in a Git repository, and bin and pkg are
// deemed safe to share for the purpose of the TestAllDependencies test.
func makeGOROOTCopy(t *testing.T) string {
	t.Helper()
	gorootCopyDir := t.TempDir()
	err := filepath.Walk(runtime.GOROOT(), func(src string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}
		if src == filepath.Join(runtime.GOROOT(), ".git") {
			return filepath.SkipDir
		}

		rel, err := filepath.Rel(runtime.GOROOT(), src)
		if err != nil {
			return fmt.Errorf("filepath.Rel(%q, %q): %v", runtime.GOROOT(), src, err)
		}
		dst := filepath.Join(gorootCopyDir, rel)

		switch src {
		case filepath.Join(runtime.GOROOT(), "bin"),
			filepath.Join(runtime.GOROOT(), "pkg"):
			// If the OS supports symlinks, use them instead
			// of copying the bin and pkg directories.
			if err := os.Symlink(src, dst); err == nil {
				return filepath.SkipDir
			}
		}

		perm := info.Mode() & os.ModePerm
		if info.Mode()&os.ModeSymlink != 0 {
			info, err = os.Stat(src)
			if err != nil {
				return err
			}
			perm = info.Mode() & os.ModePerm
		}

		// If it's a directory, make a corresponding directory.
		if info.IsDir() {
			return os.MkdirAll(dst, perm|0200)
		}

		// Copy the file bytes.
		// We can't create a symlink because the file may get modified;
		// we need to ensure that only the temporary copy is affected.
		s, err := os.Open(src)
		if err != nil {
			return err
		}
		defer s.Close()
		d, err := os.OpenFile(dst, os.O_WRONLY|os.O_CREATE|os.O_EXCL, perm)
		if err != nil {
			return err
		}
		_, err = io.Copy(d, s)
		if err != nil {
			d.Close()
			return err
		}
		return d.Close()
	})
	if err != nil {
		t.Fatal(err)
	}
	return gorootCopyDir
}

type runner struct {
	Dir string
	Env []string
}

// run runs the command and requires that it succeeds.
func (r runner) run(t *testing.T, args ...string) {
	t.Helper()
	cmd := exec.Command(args[0], args[1:]...)
	cmd.Dir = r.Dir
	cmd.Env = r.Env
	out, err := cmd.CombinedOutput()
	if err != nil {
		t.Logf("> %s\n", strings.Join(args, " "))
		t.Fatalf("command failed: %s\n%s", err, out)
	}
}

// TestDependencyVersionsConsistent verifies that each module in GOROOT that
// requires a given external dependency requires the same version of that
// dependency.
//
// This property allows us to maintain a single release branch of each such
// dependency, minimizing the number of backports needed to pull in critical
// fixes. It also ensures that any bug detected and fixed in one GOROOT module
// (such as "std") is fixed in all other modules (such as "cmd") as well.
func TestDependencyVersionsConsistent(t *testing.T) {
	// Collect the dependencies of all modules in GOROOT, indexed by module path.
	type requirement struct {
		Required    module.Version
		Replacement module.Version
	}
	seen := map[string]map[requirement][]gorootModule{} // module path → requirement → set of modules with that requirement
	for _, m := range findGorootModules(t) {
		if !m.hasVendor {
			// TestAllDependencies will ensure that the module has no dependencies.
			continue
		}

		// We want this test to be able to run offline and with an empty module
		// cache, so we verify consistency only for the module versions listed in
		// vendor/modules.txt. That includes all direct dependencies and all modules
		// that provide any imported packages.
		//
		// It's ok if there are undetected differences in modules that do not
		// provide imported packages: we will not have to pull in any backports of
		// fixes to those modules anyway.
		vendor, err := ioutil.ReadFile(filepath.Join(m.Dir, "vendor", "modules.txt"))
		if err != nil {
			t.Error(err)
			continue
		}

		for _, line := range strings.Split(strings.TrimSpace(string(vendor)), "\n") {
			parts := strings.Fields(line)
			if len(parts) < 3 || parts[0] != "#" {
				continue
			}

			// This line is of the form "# module version [=> replacement [version]]".
			var r requirement
			r.Required.Path = parts[1]
			r.Required.Version = parts[2]
			if len(parts) >= 5 && parts[3] == "=>" {
				r.Replacement.Path = parts[4]
				if module.CheckPath(r.Replacement.Path) != nil {
					// If the replacement is a filesystem path (rather than a module path),
					// we don't know whether the filesystem contents have changed since
					// the module was last vendored.
					//
					// Fortunately, we do not currently use filesystem-local replacements
					// in GOROOT modules.
					t.Errorf("cannot check consistency for filesystem-local replacement in module %s (%s):\n%s", m.Path, m.Dir, line)
				}

				if len(parts) >= 6 {
					r.Replacement.Version = parts[5]
				}
			}

			if seen[r.Required.Path] == nil {
				seen[r.Required.Path] = make(map[requirement][]gorootModule)
			}
			seen[r.Required.Path][r] = append(seen[r.Required.Path][r], m)
		}
	}

	// Now verify that we saw only one distinct version for each module.
	for path, versions := range seen {
		if len(versions) > 1 {
			t.Errorf("Modules within GOROOT require different versions of %s.", path)
			for r, mods := range versions {
				desc := new(strings.Builder)
				desc.WriteString(r.Required.Version)
				if r.Replacement.Path != "" {
					fmt.Fprintf(desc, " => %s", r.Replacement.Path)
					if r.Replacement.Version != "" {
						fmt.Fprintf(desc, " %s", r.Replacement.Version)
					}
				}

				for _, m := range mods {
					t.Logf("%s\trequires %v", m.Path, desc)
				}
			}
		}
	}
}

type gorootModule struct {
	Path      string
	Dir       string
	hasVendor bool
}

// findGorootModules returns the list of modules found in the GOROOT source tree.
func findGorootModules(t *testing.T) []gorootModule {
	t.Helper()
	goBin := testenv.GoToolPath(t)

	goroot.once.Do(func() {
		goroot.err = filepath.WalkDir(runtime.GOROOT(), func(path string, info fs.DirEntry, err error) error {
			if err != nil {
				return err
			}
			if info.IsDir() && (info.Name() == "vendor" || info.Name() == "testdata") {
				return filepath.SkipDir
			}
			if path == filepath.Join(runtime.GOROOT(), "pkg") {
				// GOROOT/pkg contains generated artifacts, not source code.
				//
				// In https://golang.org/issue/37929 it was observed to somehow contain
				// a module cache, so it is important to skip. (That helps with the
				// running time of this test anyway.)
				return filepath.SkipDir
			}
			if info.IsDir() || info.Name() != "go.mod" {
				return nil
			}
			dir := filepath.Dir(path)

			// Use 'go list' to describe the module contained in this directory (but
			// not its dependencies).
			cmd := exec.Command(goBin, "list", "-json", "-m")
			cmd.Env = append(os.Environ(), "GO111MODULE=on")
			cmd.Dir = dir
			cmd.Stderr = new(strings.Builder)
			out, err := cmd.Output()
			if err != nil {
				return fmt.Errorf("'go list -json -m' in %s: %w\n%s", dir, err, cmd.Stderr)
			}

			var m gorootModule
			if err := json.Unmarshal(out, &m); err != nil {
				return fmt.Errorf("decoding 'go list -json -m' in %s: %w", dir, err)
			}
			if m.Path == "" || m.Dir == "" {
				return fmt.Errorf("'go list -json -m' in %s failed to populate Path and/or Dir", dir)
			}
			if _, err := os.Stat(filepath.Join(dir, "vendor")); err == nil {
				m.hasVendor = true
			}
			goroot.modules = append(goroot.modules, m)
			return nil
		})
	})

	if goroot.err != nil {
		t.Fatal(goroot.err)
	}
	return goroot.modules
}

// goroot caches the list of modules found in the GOROOT source tree.
var goroot struct {
	once    sync.Once
	modules []gorootModule
	err     error
}
<|endoftext|>


<commit_before>package vebt

import (
	"fmt"
	"math"
)

type VEB struct {
	u, min, max int    //universe size, min-, max value
	summary     *VEB   //pointer to summary
	cluster     []*VEB // array of pointers to each child cluster
}

func CreateTree(u int) *VEB {
	if u < 2 {
		fmt.Println("Tree universe size u needs to be bigger than 2")
		return nil
	}

	pow := math.Log2(float64(u))
	if math.Trunc(pow) != pow {
		fmt.Println("Tree universe size u not power of 2 (u != 2^x)")
		return nil
	}

	V := new(VEB)
	V.min, V.max = -1, -1
	V.u = u

	if u == 2 {
		return V
	}

	// create clusters
	clusterCount := HigherSqrt(u)
	clusterUnivSize := LowerSqrt(u)
	for i := 0; i < clusterCount; i++ {
		V.cluster = append(V.cluster, CreateTree(clusterUnivSize))
	}

	// create summary
	summaryUnivSize := HigherSqrt(u)
	V.summary = CreateTree(summaryUnivSize)

	return V
}

func (V VEB) Max() int { return V.max }

func (V VEB) Min() int { return V.min }

func (V VEB) High(x int) int {
	return int(math.Floor(float64(x) / float64(LowerSqrt(V.u))))
}

func (V VEB) Low(x int) int {
	return x % LowerSqrt(V.u)
}

func (V VEB) Index(x, y int) int {
	return x*LowerSqrt(V.u) + y
}

func (V VEB) IsMember(x int) bool {
	if x == V.min || x == V.max {
		return true
	} else if V.u == 2 {
		return false
	} else {
		return V.cluster[V.High(x)].IsMember(V.Low(x))
	}
}

func (V *VEB) Insert(x int) {
	if V.min == -1 {
		V.min, V.max = x, x
	} else {
		if x < V.min {
			temp := V.min
			V.min = x
			x = temp
		}
		if V.u > 2 {
			if V.cluster[V.High(x)].Min() == -1 {
				V.summary.Insert(V.High(x))
				V.cluster[V.High(x)].min, V.cluster[V.High(x)].max = V.Low(x), V.Low(x)
			} else {
				V.cluster[V.High(x)].Insert(V.Low(x))
			}
		}
		if x > V.max {
			V.max = x
		}
	}
}

func (V *VEB) Delete(x int) {
	if V.summary == nil || V.summary.Min() == -1 {
		// No nonempty cluster
		if x == V.min && x == V.max {
			// only element of V
			V.min, V.max = -1, -1
		} else if x == V.min {
			//2 elements in V: x is min element
			V.min = V.max
		} else {
			//2 elements in V: x is max element
			V.max = V.min
		}

	} else {
		// some nonempty cluster
		if x == V.min {
			// get smallest element in cluster
			y := V.Index(V.summary.min, V.cluster[V.summary.min].min)
			V.min = y
			// delete element from cluster
			V.cluster[V.High(y)].Delete(V.Low(y))
			if V.cluster[V.High(y)].min == -1 {
				// it was the only element in the cluster, update summary for this cluster
				V.summary.Delete(V.High(y))
			}
		} else if x == V.max {
			y := V.Index(V.summary.max, V.cluster[V.summary.max].max)
			// delete element from cluster
			V.cluster[V.High(y)].Delete(V.Low(y))
			if V.cluster[V.High(y)].min == -1 {
				// it was the only element in the cluster, update summary for this cluster
				V.summary.Delete(V.High(y))
			}

			if V.summary == nil || V.summary.min == -1 {
				// no summary anymore
				if V.min == y {
					V.min, V.max = -1, -1
				} else {
					V.max = V.min
				}
			} else {
				V.max = V.Index(V.summary.max, V.cluster[V.summary.max].max)
			}

		} else {
			// neither min nor max
			V.cluster[V.High(x)].Delete(V.Low(x))
			if V.cluster[V.High(x)].min == -1 {
				// cluster became empty, update summary
				V.summary.Delete(V.High(x))
			}
		}

	}
}

func (V VEB) Successor(x int) int {
	if V.u == 2 {
		if x == 0 && V.max == 1 {
			return 1
		} else {
			return -1
		}
	} else if V.min != -1 && x < V.min {
		return V.min
	} else {
		maxLow := V.cluster[V.High(x)].Max()
		if maxLow != -1 && V.Low(x) < maxLow {
			offset := V.cluster[V.High(x)].Successor(V.Low(x))
			return V.Index(V.High(x), offset)
		} else {
			succCluster := V.summary.Successor(V.High(x))
			if succCluster == -1 {
				return -1
			} else {
				offset := V.cluster[succCluster].Min()
				return V.Index(succCluster, offset)
			}
		}
	}
}

func (V VEB) Predecessor(x int) int {
	if V.u == 2 {
		if x == 1 && V.min == 0 {
			return 0
		} else {
			return -1
		}
	} else if V.max != -1 && x > V.max {
		return V.max
	} else {
		minLow := V.cluster[V.High(x)].Min()
		if minLow != -1 && V.Low(x) > minLow {
			offset := V.cluster[V.High(x)].Predecessor(V.Low(x))
			return V.Index(V.High(x), offset)
		} else {
			predCluster := V.summary.Predecessor(V.High(x))
			if predCluster == -1 {
				if V.min != -1 && x > V.min {
					return V.min
				} else {
					return -1
				}
			} else {
				offset := V.cluster[predCluster].Max()
				return V.Index(predCluster, offset)
			}
		}
	}
}

// Counts all struct objects of the tree (for testing purposes)
func (V VEB) Count() int {
	if V.u == 2 {
		return 1
	}

	sum := 1 // including structure itself
	for i := 0; i < len(V.cluster); i++ {
		sum += V.cluster[i].Count()
	}
	sum += V.summary.Count()
	return sum
}

// Prints the tree to std out (for debugging purposes only)
func (V VEB) Print() {
	//function just acts as wrapper, since default parameters are not possible in go
	V.PrintFunc(0, 0)
}

func (V VEB) PrintFunc(level, clusterNo int) {
	spacer := ""
	for i := 0; i < level; i++ {
		spacer += "\t"
	}

	if level == 0 {
		fmt.Printf("%vR: {u: %v, min: %v, max: %v, clusters: %v}\n", spacer, V.u, V.min, V.max, len(V.cluster))
	} else {
		fmt.Printf("%vC[%v]: {u: %v, min: %v, max: %v, clusters: %v}\n", spacer, clusterNo, V.u, V.min, V.max, len(V.cluster))
	}

	if len(V.cluster) > 0 {
		fmt.Printf("%v\tS:    {u: %v, min: %v, max: %v, clusters: %v}\n", spacer, V.summary.u, V.summary.min, V.summary.max, len(V.summary.cluster))
		for i := 0; i < len(V.summary.cluster); i++ {
			V.summary.cluster[i].PrintFunc(level+2, i)
		}
		for i := 0; i < len(V.cluster); i++ {
			V.cluster[i].PrintFunc(level+1, i)
		}
	}
}

// Counts all struct objects of the tree (for testing purposes)
func (V *VEB) Clear() {
	V.min, V.max = -1, -1

	if V.u == 2 {
		return
	}

	// clear clusters
	for i := 0; i < len(V.cluster); i++ {
		V.cluster[i].Clear()
	}
	// clear summary
	V.summary.Clear()
}

// Fills tree with all keys
func (V *VEB) Fill() {
	for i := 0; i < V.u; i++ {
		V.Insert(i)
	}
}

// Get array with members of tree
func (V VEB) Members() []int{
	members := []int{}
	for i := 0; i < V.u; i++ {
		if V.IsMember(i) {
			members = append(members, i)
		}
	}
	return members
}

// Calculate lower square root (helper function)
func LowerSqrt(u int) int {
	return int(math.Pow(2.0, math.Floor(math.Log2(float64(u))/2)))
}

// Calculate higher square root (helper function)
func HigherSqrt(u int) int {
	return int(math.Pow(2.0, math.Ceil(math.Log2(float64(u))/2)))
}
<commit_msg>createTree(u) now accepts u which is not exact power of too, comments<commit_after>package vebt

import (
	"fmt"
	"math"
)

type VEB struct {
	u, min, max int    //universe size, min-, max value
	summary     *VEB   //pointer to summary
	cluster     []*VEB // array of pointers to each child cluster
}

// Creates an tree with a universe size (u) with u=2^x, where x is chosen,
// so that 2^(x-1) < size <= 2^x. Returns the pointer on the root element
// or nil on failure
func CreateTree(size int) *VEB {
	if size < 0 {
		return nil
	}

	// choose x so that 2^(x-1) < u <= 2^x
	x := math.Ceil(math.Log2(float64(size)))
	u := int(math.Pow(2, x))

	V := new(VEB)
	V.min, V.max = -1, -1
	V.u = u

	if u == 2 {
		return V
	}

	// create clusters
	clusterCount := HigherSqrt(u)
	clusterUnivSize := LowerSqrt(u)
	for i := 0; i < clusterCount; i++ {
		V.cluster = append(V.cluster, CreateTree(clusterUnivSize))
	}

	// create summary
	summaryUnivSize := HigherSqrt(u)
	V.summary = CreateTree(summaryUnivSize)

	return V
}

// Returns the maximum of the tree
func (V VEB) Max() int { return V.max }

// Returns the minimum of the tree
func (V VEB) Min() int { return V.min }

// Calculates and returns the cluster number where x is stored in the tree
func (V VEB) High(x int) int {
	return int(math.Floor(float64(x) / float64(LowerSqrt(V.u))))
}

// Calculates and returns the position in which x appears in its cluster
func (V VEB) Low(x int) int {
	return x % LowerSqrt(V.u)
}

// Calculates and returns the index for x and y
func (V VEB) Index(x, y int) int {
	return x*LowerSqrt(V.u) + y
}

// Checks if x is stored in the tree. Returns true if so, false if not
func (V VEB) IsMember(x int) bool {
	if x == V.min || x == V.max {
		return true
	} else if V.u == 2 {
		return false
	} else {
		return V.cluster[V.High(x)].IsMember(V.Low(x))
	}
}

// Inserts x into the tree
func (V *VEB) Insert(x int) {
	if V.min == -1 {
		V.min, V.max = x, x
	} else {
		if x < V.min {
			temp := V.min
			V.min = x
			x = temp
		}
		if V.u > 2 {
			if V.cluster[V.High(x)].Min() == -1 {
				V.summary.Insert(V.High(x))
				V.cluster[V.High(x)].min, V.cluster[V.High(x)].max = V.Low(x), V.Low(x)
			} else {
				V.cluster[V.High(x)].Insert(V.Low(x))
			}
		}
		if x > V.max {
			V.max = x
		}
	}
}

// Removes x from the tree
func (V *VEB) Delete(x int) {
	if V.summary == nil || V.summary.Min() == -1 {
		// No nonempty cluster
		if x == V.min && x == V.max {
			// only element of V
			V.min, V.max = -1, -1
		} else if x == V.min {
			//2 elements in V: x is min element
			V.min = V.max
		} else {
			//2 elements in V: x is max element
			V.max = V.min
		}

	} else {
		// some nonempty cluster
		if x == V.min {
			// get smallest element in cluster
			y := V.Index(V.summary.min, V.cluster[V.summary.min].min)
			V.min = y
			// delete element from cluster
			V.cluster[V.High(y)].Delete(V.Low(y))
			if V.cluster[V.High(y)].min == -1 {
				// it was the only element in the cluster, update summary for this cluster
				V.summary.Delete(V.High(y))
			}
		} else if x == V.max {
			y := V.Index(V.summary.max, V.cluster[V.summary.max].max)
			// delete element from cluster
			V.cluster[V.High(y)].Delete(V.Low(y))
			if V.cluster[V.High(y)].min == -1 {
				// it was the only element in the cluster, update summary for this cluster
				V.summary.Delete(V.High(y))
			}

			if V.summary == nil || V.summary.min == -1 {
				// no summary anymore
				if V.min == y {
					V.min, V.max = -1, -1
				} else {
					V.max = V.min
				}
			} else {
				V.max = V.Index(V.summary.max, V.cluster[V.summary.max].max)
			}

		} else {
			// neither min nor max
			V.cluster[V.High(x)].Delete(V.Low(x))
			if V.cluster[V.High(x)].min == -1 {
				// cluster became empty, update summary
				V.summary.Delete(V.High(x))
			}
		}

	}
}

// Finds the successor element of x in the tree
func (V VEB) Successor(x int) int {
	if V.u == 2 {
		if x == 0 && V.max == 1 {
			return 1
		} else {
			return -1
		}
	} else if V.min != -1 && x < V.min {
		return V.min
	} else {
		maxLow := V.cluster[V.High(x)].Max()
		if maxLow != -1 && V.Low(x) < maxLow {
			offset := V.cluster[V.High(x)].Successor(V.Low(x))
			return V.Index(V.High(x), offset)
		} else {
			succCluster := V.summary.Successor(V.High(x))
			if succCluster == -1 {
				return -1
			} else {
				offset := V.cluster[succCluster].Min()
				return V.Index(succCluster, offset)
			}
		}
	}
}

// Finds the predecessor element of x in the tree
func (V VEB) Predecessor(x int) int {
	if V.u == 2 {
		if x == 1 && V.min == 0 {
			return 0
		} else {
			return -1
		}
	} else if V.max != -1 && x > V.max {
		return V.max
	} else {
		minLow := V.cluster[V.High(x)].Min()
		if minLow != -1 && V.Low(x) > minLow {
			offset := V.cluster[V.High(x)].Predecessor(V.Low(x))
			return V.Index(V.High(x), offset)
		} else {
			predCluster := V.summary.Predecessor(V.High(x))
			if predCluster == -1 {
				if V.min != -1 && x > V.min {
					return V.min
				} else {
					return -1
				}
			} else {
				offset := V.cluster[predCluster].Max()
				return V.Index(predCluster, offset)
			}
		}
	}
}

// Counts and returns all objects (structs) of the tree
func (V VEB) Count() int {
	if V.u == 2 {
		return 1
	}

	sum := 1 // including structure itself
	for i := 0; i < len(V.cluster); i++ {
		sum += V.cluster[i].Count()
	}
	sum += V.summary.Count()
	return sum
}

// Prints the tree to std out
func (V VEB) Print() {
	//function just acts as wrapper, since default parameters are not possible in go
	V.PrintFunc(0, 0)
}

// Prints the tree to std out, where level is used to keep track of idention and clusterNo to
// track the number of the cluster
func (V VEB) PrintFunc(level, clusterNo int) {
	spacer := ""
	for i := 0; i < level; i++ {
		spacer += "\t"
	}

	if level == 0 {
		fmt.Printf("%vR: {u: %v, min: %v, max: %v, clusters: %v}\n", spacer, V.u, V.min, V.max, len(V.cluster))
	} else {
		fmt.Printf("%vC[%v]: {u: %v, min: %v, max: %v, clusters: %v}\n", spacer, clusterNo, V.u, V.min, V.max, len(V.cluster))
	}

	if len(V.cluster) > 0 {
		fmt.Printf("%v\tS:    {u: %v, min: %v, max: %v, clusters: %v}\n", spacer, V.summary.u, V.summary.min, V.summary.max, len(V.summary.cluster))
		for i := 0; i < len(V.summary.cluster); i++ {
			V.summary.cluster[i].PrintFunc(level+2, i)
		}
		for i := 0; i < len(V.cluster); i++ {
			V.cluster[i].PrintFunc(level+1, i)
		}
	}
}

// Clears the tree by setting min and max to -1 for every node of the tree
func (V *VEB) Clear() {
	V.min, V.max = -1, -1

	if V.u == 2 {
		return
	}

	// clear clusters
	for i := 0; i < len(V.cluster); i++ {
		V.cluster[i].Clear()
	}
	// clear summary
	V.summary.Clear()
}

// Fills tree with all keys (inserts all keys from 0 to u)
func (V *VEB) Fill() {
	for i := 0; i < V.u; i++ {
		V.Insert(i)
	}
}

// Get array of tree members
func (V VEB) Members() []int{
	members := []int{}
	for i := 0; i < V.u; i++ {
		if V.IsMember(i) {
			members = append(members, i)
		}
	}
	return members
}

// Calculate lower square root (helper function)
func LowerSqrt(u int) int {
	return int(math.Pow(2.0, math.Floor(math.Log2(float64(u))/2)))
}

// Calculate higher square root (helper function)
func HigherSqrt(u int) int {
	return int(math.Pow(2.0, math.Ceil(math.Log2(float64(u))/2)))
}
<|endoftext|>


<commit_before>package controllers

import (
	"bytes"
	"chant/app/models"
	"encoding/json"
	"encoding/xml"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"net/url"
	"regexp"

	"chant/app/chatroom"

	"github.com/revel/revel"
)

// APIv1 ...
type APIv1 struct {
	*revel.Controller
}

// RoomStamps とりあえず
func (c APIv1) RoomStamps(id, token string) revel.Result {
	if !chatroom.Exists(id) {
		return c.RenderJson(map[string]interface{}{
			"stamps": []interface{}{},
		})
	}
	room := chatroom.GetRoom(id, token)
	if room == nil {
		return c.RenderJson(map[string]interface{}{
			"stamps": []interface{}{},
		})
	}
	stamps := room.Repo.GetAllStamps()
	return c.RenderJson(map[string]interface{}{
		"stamps": stamps,
	})
}

// RoomMessages とりあえず
func (c APIv1) RoomMessages(id, token string) revel.Result {
	if !chatroom.Exists(id) {
		return c.RenderJson(map[string]interface{}{
			"messages": []interface{}{},
		})
	}
	room := chatroom.GetRoom(id, token)
	if room == nil {
		return c.RenderJson(map[string]interface{}{
			"stamps": []interface{}{},
		})
	}
	// メッセージアーカイブを、最新の、最大10件を取得する
	messages := room.Repo.GetMessages(10, -1)
	return c.RenderJson(map[string]interface{}{
		"messages": messages,
	})
}

// RoomSay とりあえず
func (c APIv1) RoomSay(id, token string) revel.Result {
	c.Request.Format = "json"
	// FIXME: めんどいからここで
	type params struct {
		Type  string       `json:"type"`
		Value string       `json:"value"`
		User  *models.User `json:"user"`
	}
	p := new(params)
	if err := json.NewDecoder(c.Request.Body).Decode(p); err != nil {
		c.Response.Status = http.StatusBadRequest
		return c.RenderError(err)
	}

	raw, err := json.Marshal(struct {
		Type string `json:"type"`
		Raw  string `json:"raw"`
	}{p.Type, p.Value})

	if err != nil {
		c.Response.Status = http.StatusBadRequest
		return revel.ErrorResult{
			Error: err,
		}
	}

	room := chatroom.GetRoom(id, chatroom.PrivilegeAPIToken)

	event, err := room.Say(p.User, string(raw))
	if err != nil {
		c.Response.Status = http.StatusBadRequest
		return revel.ErrorResult{
			Error: err,
		}
	}

	return c.RenderJson(map[string]interface{}{
		"params":  p,
		"created": event,
	})
}

// WebPreview ...
func (c APIv1) WebPreview(u string) revel.Result {
	res, err := http.Get(u)
	if err != nil {
		return c.RenderJson(map[string]interface{}{})
	}
	if regexp.MustCompile("^ima?ge?/.*").MatchString(res.Header.Get("Content-Type")) {
		return c.RenderJson(map[string]interface{}{
			"content": "image",
			"url":     u,
		})
	}

	// clean response body
	b, _ := ioutil.ReadAll(res.Body)
	b = regexp.MustCompile("\\<script[\\S\\s]+?\\</script\\>").ReplaceAll(b, []byte{})
	buf := bytes.NewBuffer(b)

	page := new(HTMLPage)
	err = decoder(buf).Decode(page)
	if err != nil {
		log.Println("[WebPreview]", err)
	}

	return c.RenderJson(map[string]interface{}{
		"html":    page,
		"summary": page.Summarize(u),
	})
}

func decoder(reader io.Reader) *xml.Decoder {
	dec := xml.NewDecoder(reader)
	dec.Strict = false
	dec.AutoClose = xml.HTMLAutoClose
	dec.Entity = xml.HTMLEntity
	return dec
}

// Summary ...
type Summary struct {
	Title       string `json:"title"`
	Image       string `json:"image"`
	Description string `json:"description"`
	URL         string `json:"url"`
}

func (summary *Summary) setTitle(c string) {
	if len(summary.Title) != 0 || len(c) == 0 {
		return
	}
	summary.Title = c
}
func (summary *Summary) setImage(c string) {
	if len(summary.Image) != 0 || len(c) == 0 {
		return
	}
	summary.Image = c
}
func (summary *Summary) setDescription(c string) {
	if len(summary.Description) != 0 || len(c) == 0 {
		return
	}
	summary.Description = c
}

// HTMLPage ...
type HTMLPage struct {
	Head struct {
		Title string `json:"title" xml:"title"`
		Metas []Meta `json:"metas" xml:"meta"`
		Links []Link `json:"links" xml:"link"`
	} `json:"head" xml:"head"`
} // `xml:"html"`

// Meta ...
type Meta struct {
	Property string `json:"property" xml:"property,attr"`
	Name     string `json:"name"     xml:"name,attr"`
	Content  string `json:"content"  xml:"content,attr"`
}

// Link ...
type Link struct {
	Rel  string `json:"rel"  xml:"rel,attr"`
	Href string `json:"href" xml:"href,attr"`
}

// Summarize ...
func (hp *HTMLPage) Summarize(u string) *Summary {
	summary := new(Summary)
	summary.URL = u
	img := regexp.MustCompile("image")
	desc := regexp.MustCompile("description")
	title := regexp.MustCompile("title")
	icon := regexp.MustCompile("icon")
	for _, meta := range hp.Head.Metas {
		switch {
		case img.MatchString(meta.Property):
			summary.setImage(meta.Content)
		case desc.MatchString(meta.Property):
			summary.setDescription(meta.Content)
		case title.MatchString(meta.Property):
			summary.setTitle(meta.Content)
		}
	}
	summary.setTitle(hp.Head.Title)
	summary.setDescription(hp.Head.Title)
	if len(summary.Image) == 0 {
		for _, link := range hp.Head.Links {
			if icon.MatchString(link.Rel) {
				summary.setImage(link.Href)
				break
			}
		}
	}
	if len(summary.Image) == 0 {
		v, _ := url.Parse(u)
		summary.setImage(v.Scheme + "://" + v.Host + "/favicon.ico")
	}
	return summary
}
<commit_msg>Fix #165<commit_after>package controllers

import (
	"bytes"
	"chant/app/models"
	"encoding/json"
	"encoding/xml"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"net/url"
	"regexp"

	"chant/app/chatroom"

	"github.com/revel/revel"
)

// APIv1 ...
type APIv1 struct {
	*revel.Controller
}

// RoomStamps とりあえず
func (c APIv1) RoomStamps(id, token string) revel.Result {
	if !chatroom.Exists(id) {
		return c.RenderJson(map[string]interface{}{
			"stamps": []interface{}{},
		})
	}
	room := chatroom.GetRoom(id, token)
	if room == nil {
		return c.RenderJson(map[string]interface{}{
			"stamps": []interface{}{},
		})
	}
	stamps := room.Repo.GetAllStamps()
	return c.RenderJson(map[string]interface{}{
		"stamps": stamps,
	})
}

// RoomMessages とりあえず
func (c APIv1) RoomMessages(id, token string) revel.Result {
	if !chatroom.Exists(id) {
		return c.RenderJson(map[string]interface{}{
			"messages": []interface{}{},
		})
	}
	room := chatroom.GetRoom(id, token)
	if room == nil {
		return c.RenderJson(map[string]interface{}{
			"stamps": []interface{}{},
		})
	}
	// メッセージアーカイブを、最新の、最大10件を取得する
	messages := room.Repo.GetMessages(10, -1)
	return c.RenderJson(map[string]interface{}{
		"messages": messages,
	})
}

// RoomSay とりあえず
func (c APIv1) RoomSay(id, token string) revel.Result {
	c.Request.Format = "json"
	// FIXME: めんどいからここで
	type params struct {
		Type  string       `json:"type"`
		Value string       `json:"value"`
		User  *models.User `json:"user"`
	}
	p := new(params)
	if err := json.NewDecoder(c.Request.Body).Decode(p); err != nil {
		c.Response.Status = http.StatusBadRequest
		return c.RenderError(err)
	}

	raw, err := json.Marshal(struct {
		Type string `json:"type"`
		Raw  string `json:"raw"`
	}{p.Type, p.Value})

	if err != nil {
		c.Response.Status = http.StatusBadRequest
		return revel.ErrorResult{
			Error: err,
		}
	}

	room := chatroom.GetRoom(id, chatroom.PrivilegeAPIToken)

	event, err := room.Say(p.User, string(raw))
	if err != nil {
		c.Response.Status = http.StatusBadRequest
		return revel.ErrorResult{
			Error: err,
		}
	}

	return c.RenderJson(map[string]interface{}{
		"params":  p,
		"created": event,
	})
}

// WebPreview ...
func (c APIv1) WebPreview(u string) revel.Result {
	res, err := http.Get(u)
	if err != nil {
		return c.RenderJson(map[string]interface{}{})
	}
	if regexp.MustCompile("^ima?ge?/.*").MatchString(res.Header.Get("Content-Type")) {
		return c.RenderJson(map[string]interface{}{
			"content": "image",
			"url":     u,
		})
	}

	// clean response body
	b, _ := ioutil.ReadAll(res.Body)
	b = regexp.MustCompile("\\<script[\\S\\s]+?\\</script\\>").ReplaceAll(b, []byte{})
	buf := bytes.NewBuffer(b)

	page := new(HTMLPage)
	err = decoder(buf).Decode(page)
	if err != nil {
		log.Println("[WebPreview]", err)
	}

	return c.RenderJson(map[string]interface{}{
		"html":    page,
		"summary": page.Summarize(u),
	})
}

func decoder(reader io.Reader) *xml.Decoder {
	dec := xml.NewDecoder(reader)
	dec.Strict = false
	dec.AutoClose = xml.HTMLAutoClose
	dec.Entity = xml.HTMLEntity
	return dec
}

// Summary ...
type Summary struct {
	Title       string `json:"title"`
	Image       string `json:"image"`
	Description string `json:"description"`
	URL         string `json:"url"`
}

func (summary *Summary) setTitle(c string) {
	if len(summary.Title) != 0 || len(c) == 0 {
		return
	}
	summary.Title = c
}
func (summary *Summary) setImage(c string) {
	if len(summary.Image) != 0 || len(c) == 0 {
		return
	}
	summary.Image = c
}
func (summary *Summary) setDescription(c string) {
	if len(summary.Description) != 0 || len(c) == 0 {
		return
	}
	summary.Description = c
}

// HTMLPage ...
type HTMLPage struct {
	Head struct {
		Title string `json:"title" xml:"title"`
		Metas []Meta `json:"metas" xml:"meta"`
		Links []Link `json:"links" xml:"link"`
	} `json:"head" xml:"head"`
} // `xml:"html"`

// Meta ...
type Meta struct {
	Property string `json:"property" xml:"property,attr"`
	Name     string `json:"name"     xml:"name,attr"`
	Content  string `json:"content"  xml:"content,attr"`
}

// Link ...
type Link struct {
	Rel  string `json:"rel"  xml:"rel,attr"`
	Href string `json:"href" xml:"href,attr"`
}

// Summarize ...
func (hp *HTMLPage) Summarize(u string) *Summary {
	summary := new(Summary)
	summary.URL = u
	img := regexp.MustCompile("image")
	desc := regexp.MustCompile("description")
	title := regexp.MustCompile("title")
	icon := regexp.MustCompile("icon")
	for _, meta := range hp.Head.Metas {
		switch {
		case img.MatchString(meta.Property):
			summary.setImage(meta.Content)
		case desc.MatchString(meta.Property), desc.MatchString(meta.Name):
			summary.setDescription(meta.Content)
		case title.MatchString(meta.Property):
			summary.setTitle(meta.Content)
		}
	}
	summary.setTitle(hp.Head.Title)
	summary.setDescription(hp.Head.Title)
	if len(summary.Image) == 0 {
		for _, link := range hp.Head.Links {
			if icon.MatchString(link.Rel) {
				summary.setImage(abspath(u, link.Href))
				break
			}
		}
	}
	if len(summary.Image) == 0 {
		summary.setImage(abspath(u, "/favicon.ico"))
	}
	return summary
}

// abspath
func abspath(original, relative string) string {
	p, err := url.Parse(relative)
	if err == nil && p.IsAbs() {
		return p.String()
	}
	v, _ := url.Parse(original)
	v.Path = relative
	return v.String()
}
<|endoftext|>


<commit_before>package main

import (
	"database/sql"
	_ "github.com/lib/pq"
	"log"
	"net"
	"net/http"
)

func UpVoteMe(w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Access-Control-Allow-Origin", "*")
	w.Header().Set("Content-Type", "application/json")

	word := r.URL.Query().Get(":word")
	updateRating(word, 1, r.RemoteAddr)

	log.Print("Up vote for: ", word)
}

func DownVoteMe(w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Access-Control-Allow-Origin", "*")
	w.Header().Set("Content-Type", "application/json")

	word := r.URL.Query().Get(":word")
	updateRating(word, -1, r.RemoteAddr)

	log.Print("Down vote for: ", word)
}

func updateRating(word string, vote int, address string) {
	host, _, err := net.SplitHostPort(address)
	if err != nil {
		panic(err)
	}

	db, err := sql.Open("postgres", ConnectionString)
	if err != nil {
		panic(err)
	}
	defer db.Close()

	_, err = db.Exec("insert into votes (word, vote, ipaddress) values ($1, $2, $3)", word, vote, host)
	if err != nil {
		panic(err)
	}
}
<commit_msg>Update vote to properly grab the users IP Address.<commit_after>package main

import (
	"database/sql"
	_ "github.com/lib/pq"
	"log"
	"net"
	"net/http"
	"strings"
)

func UpVoteMe(w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Access-Control-Allow-Origin", "*")
	w.Header().Set("Content-Type", "application/json")

	word := r.URL.Query().Get(":word")
	updateRating(word, 1, r)

	log.Print("Up vote for: ", word)
}

func DownVoteMe(w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Access-Control-Allow-Origin", "*")
	w.Header().Set("Content-Type", "application/json")

	word := r.URL.Query().Get(":word")
	updateRating(word, -1, r)

	log.Print("Down vote for: ", word)
}

func updateRating(word string, vote int, r *http.Request) {
	host, _, err := net.SplitHostPort(r.RemoteAddr)
	if err != nil {
		panic(err)
	}

	realIp := r.Header.Get("X-Real-Ip")
	forwardedFor := r.Header.Get("X-Forwarded-For")

  if realIp != "" {
		host = realIp
	}

	if forwardedFor != "" {
    addresses := strings.Split(forwardedFor, ",")
		for _, ele := range addresses {
			if ele != "127.0.0.1" {
				host = strings.TrimSpace(ele)
        break
			}
		}
	}

	db, err := sql.Open("postgres", ConnectionString)
	if err != nil {
		panic(err)
	}
	defer db.Close()

	_, err = db.Exec("insert into votes (word, vote, ipaddress) values ($1, $2, $3)", word, vote, host)
	if err != nil {
		panic(err)
	}
}
<|endoftext|>


<commit_before>package filestore

import (
	"errors"
	"io"
	"os"
	"path/filepath"

	ds "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/ipfs/go-datastore"
	"github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/ipfs/go-datastore/query"
	k "github.com/ipfs/go-ipfs/blocks/key"
	//mh "gx/ipfs/QmYf7ng2hG5XBtJA3tN34DQ2GUN5HNksEw1rLDkmr6vGku/go-multihash"
	u "gx/ipfs/QmZNVWh8LLjAavuQ2JXuFmuYH3C11xo988vSgp7UQrTRj1/go-ipfs-util"
)

type Datastore struct {
	ds           ds.Datastore
	alwaysVerify bool
}

func New(d ds.Datastore, fileStorePath string) (*Datastore, error) {
	return &Datastore{d, true}, nil
}

func (d *Datastore) Put(key ds.Key, value interface{}) (err error) {
	val, ok := value.(*DataWOpts)
	if !ok {
		panic(ds.ErrInvalidType)
	}

	addType, ok := val.AddOpts.(int)
	if !ok {
		panic(ds.ErrInvalidType)
	}
	if addType != AddNoCopy {
		return errors.New("Only \"no-copy\" mode supported for now.")
	}

	dataObj, ok := val.DataObj.(*DataObj)
	if !ok {
		panic(ds.ErrInvalidType)
	}

	// Make sure the filename is an absolute path
	if !filepath.IsAbs(dataObj.FilePath) {
		return errors.New("datastore put: non-absolute filename: " + dataObj.FilePath)
	}

	// Make sure we can read the file as a sanity check
	if file, err := os.Open(dataObj.FilePath); err != nil {
		return err
	} else {
		file.Close()
	}

	data, err := dataObj.Marshal()
	if err != nil {
		return err
	}
	return d.ds.Put(key, data)
}

func (d *Datastore) Get(key ds.Key) (value interface{}, err error) {
	dataObj, err := d.ds.Get(key)
	if err != nil {
		return nil, err
	}
	val, err := d.decode(dataObj)
	if err != nil {
		return nil, err
	}
	return d.GetData(key, val, d.alwaysVerify)
}

// Get the key as a DataObj
func (d *Datastore) GetDirect(key ds.Key) (*DataObj, error) {
	dataObj, err := d.ds.Get(key)
	if err != nil {
		return nil, err
	}
	return d.decode(dataObj)
}

func (d *Datastore) decode(dataObj interface{}) (*DataObj, error) {
	data := dataObj.([]byte)
	val := new(DataObj)
	err := val.Unmarshal(data)
	if err != nil {
		return nil, err
	}
	return val, nil
}

type InvalidBlock struct{}

func (e InvalidBlock) Error() string {
	return "Datastore: Block Verification Failed"
}

// Get the orignal data out of the DataObj
func (d *Datastore) GetData(key ds.Key, val *DataObj, verify bool) ([]byte, error) {
	if val == nil {
		return nil, errors.New("Nil DataObj")
	} else if val.NoBlockData {
		file, err := os.Open(val.FilePath)
		if err != nil {
			return nil, err
		}
		_, err = file.Seek(int64(val.Offset), 0)
		if err != nil {
			return nil, err
		}
		buf := make([]byte, val.Size)
		_, err = io.ReadFull(file, buf)
		if err != nil {
			return nil, err
		}
		data, err := reconstruct(val.Data, buf)
		if err != nil {
			return nil, err
		}
		if verify {
			newKey := k.Key(u.Hash(data)).DsKey()
			if newKey != key {
				return nil, InvalidBlock{}
			}
		}
		return data, nil
	} else {
		return val.Data, nil
	}
}

func (d *Datastore) Has(key ds.Key) (exists bool, err error) {
	return d.ds.Has(key)
}

func (d *Datastore) Delete(key ds.Key) error {
	return ds.ErrNotFound
}

func (d *Datastore) DeleteDirect(key ds.Key) error {
	return d.ds.Delete(key)
}

func (d *Datastore) Query(q query.Query) (query.Results, error) {
	res, err := d.ds.Query(q)
	if err != nil {
		return nil, err
	}
	if q.KeysOnly {
		return res, nil
	}
	return nil, errors.New("filestore currently only supports keyonly queries")
	// return &queryResult{res, func(r query.Result) query.Result {
	// 	val, err := d.decode(r.Value)
	// 	if err != nil {
	// 		return query.Result{query.Entry{r.Key, nil}, err}
	// 	}
	// 	// Note: It should not be necessary to reclean the key
	// 	// here (by calling ds.NewKey) just to convert the
	// 	// string back to a ds.Key
	// 	data, err := d.GetData(ds.NewKey(r.Key), val, d.alwaysVerify)
	// 	if err != nil {
	// 		return query.Result{query.Entry{r.Key, nil}, err}
	// 	}
	// 	return query.Result{query.Entry{r.Key, data}, r.Error}
	// }}, nil
}

func (d *Datastore) QueryDirect(q query.Query) (query.Results, error) {
	res, err := d.ds.Query(q)
	if err != nil {
		return nil, err
	}
	if q.KeysOnly {
		return res, nil
	}
	return nil, errors.New("filestore currently only supports keyonly queries")
	// return &queryResult{res, func(r query.Result) query.Result {
	// 	val, err := d.decode(r.Value)
	// 	if err != nil {
	// 		return query.Result{query.Entry{r.Key, nil}, err}
	// 	}
	// 	return query.Result{query.Entry{r.Key, val}, r.Error}
	// }}, nil
}

// type queryResult struct {
// 	query.Results
// 	adjResult func(query.Result) query.Result
// }

// func (q *queryResult) Next() <-chan query.Result {
// 	in := q.Results.Next()
// 	out := make(chan query.Result)
// 	go func() {
// 		res := <-in
// 		if res.Error == nil {
// 			out <- res
// 		}
// 		out <- q.adjResult(res)
// 	}()
// 	return out
// }

// func (q *queryResult) Rest() ([]query.Entry, error) {
// 	res, err := q.Results.Rest()
// 	if err != nil {
// 		return nil, err
// 	}
// 	for _, entry := range res {
// 		newRes := q.adjResult(query.Result{entry, nil})
// 		if newRes.Error != nil {
// 			return nil, newRes.Error
// 		}
// 		entry.Value = newRes.Value
// 	}
// 	return res, nil
// }

func (d *Datastore) Close() error {
	c, ok := d.ds.(io.Closer)
	if ok {
		return c.Close()
	} else {
		return nil
	}
}

func (d *Datastore) Batch() (ds.Batch, error) {
	return ds.NewBasicBatch(d), nil
}
<commit_msg>Check if the WholeFile flag should be set for leaf nodes in filestore.Put.<commit_after>package filestore

import (
	"errors"
	"io"
	"os"
	"path/filepath"

	ds "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/ipfs/go-datastore"
	"github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/ipfs/go-datastore/query"
	k "github.com/ipfs/go-ipfs/blocks/key"
	//mh "gx/ipfs/QmYf7ng2hG5XBtJA3tN34DQ2GUN5HNksEw1rLDkmr6vGku/go-multihash"
	u "gx/ipfs/QmZNVWh8LLjAavuQ2JXuFmuYH3C11xo988vSgp7UQrTRj1/go-ipfs-util"
)

type Datastore struct {
	ds           ds.Datastore
	alwaysVerify bool
}

func New(d ds.Datastore, fileStorePath string) (*Datastore, error) {
	return &Datastore{d, true}, nil
}

func (d *Datastore) Put(key ds.Key, value interface{}) (err error) {
	val, ok := value.(*DataWOpts)
	if !ok {
		panic(ds.ErrInvalidType)
	}

	addType, ok := val.AddOpts.(int)
	if !ok {
		panic(ds.ErrInvalidType)
	}
	if addType != AddNoCopy {
		return errors.New("Only \"no-copy\" mode supported for now.")
	}

	dataObj, ok := val.DataObj.(*DataObj)
	if !ok {
		panic(ds.ErrInvalidType)
	}

	// Make sure the filename is an absolute path
	if !filepath.IsAbs(dataObj.FilePath) {
		return errors.New("datastore put: non-absolute filename: " + dataObj.FilePath)
	}

	// Make sure we can read the file as a sanity check
	file, err := os.Open(dataObj.FilePath)
	if err != nil {
		return err
	}

	// See if we have the whole file in the block
	if dataObj.Offset == 0 && !dataObj.WholeFile {
		// Get the file size
		info, err := file.Stat()
		if err != nil {
			return err
		}
		if dataObj.Size == uint64(info.Size()) {
			dataObj.WholeFile = true
		}
	}

	file.Close()

	data, err := dataObj.Marshal()
	if err != nil {
		return err
	}
	return d.ds.Put(key, data)
}

func (d *Datastore) Get(key ds.Key) (value interface{}, err error) {
	dataObj, err := d.ds.Get(key)
	if err != nil {
		return nil, err
	}
	val, err := d.decode(dataObj)
	if err != nil {
		return nil, err
	}
	return d.GetData(key, val, d.alwaysVerify)
}

// Get the key as a DataObj
func (d *Datastore) GetDirect(key ds.Key) (*DataObj, error) {
	dataObj, err := d.ds.Get(key)
	if err != nil {
		return nil, err
	}
	return d.decode(dataObj)
}

func (d *Datastore) decode(dataObj interface{}) (*DataObj, error) {
	data := dataObj.([]byte)
	val := new(DataObj)
	err := val.Unmarshal(data)
	if err != nil {
		return nil, err
	}
	return val, nil
}

type InvalidBlock struct{}

func (e InvalidBlock) Error() string {
	return "Datastore: Block Verification Failed"
}

// Get the orignal data out of the DataObj
func (d *Datastore) GetData(key ds.Key, val *DataObj, verify bool) ([]byte, error) {
	if val == nil {
		return nil, errors.New("Nil DataObj")
	} else if val.NoBlockData {
		file, err := os.Open(val.FilePath)
		if err != nil {
			return nil, err
		}
		_, err = file.Seek(int64(val.Offset), 0)
		if err != nil {
			return nil, err
		}
		buf := make([]byte, val.Size)
		_, err = io.ReadFull(file, buf)
		if err != nil {
			return nil, err
		}
		data, err := reconstruct(val.Data, buf)
		if err != nil {
			return nil, err
		}
		if verify {
			newKey := k.Key(u.Hash(data)).DsKey()
			if newKey != key {
				return nil, InvalidBlock{}
			}
		}
		return data, nil
	} else {
		return val.Data, nil
	}
}

func (d *Datastore) Has(key ds.Key) (exists bool, err error) {
	return d.ds.Has(key)
}

func (d *Datastore) Delete(key ds.Key) error {
	return ds.ErrNotFound
}

func (d *Datastore) DeleteDirect(key ds.Key) error {
	return d.ds.Delete(key)
}

func (d *Datastore) Query(q query.Query) (query.Results, error) {
	res, err := d.ds.Query(q)
	if err != nil {
		return nil, err
	}
	if q.KeysOnly {
		return res, nil
	}
	return nil, errors.New("filestore currently only supports keyonly queries")
	// return &queryResult{res, func(r query.Result) query.Result {
	// 	val, err := d.decode(r.Value)
	// 	if err != nil {
	// 		return query.Result{query.Entry{r.Key, nil}, err}
	// 	}
	// 	// Note: It should not be necessary to reclean the key
	// 	// here (by calling ds.NewKey) just to convert the
	// 	// string back to a ds.Key
	// 	data, err := d.GetData(ds.NewKey(r.Key), val, d.alwaysVerify)
	// 	if err != nil {
	// 		return query.Result{query.Entry{r.Key, nil}, err}
	// 	}
	// 	return query.Result{query.Entry{r.Key, data}, r.Error}
	// }}, nil
}

func (d *Datastore) QueryDirect(q query.Query) (query.Results, error) {
	res, err := d.ds.Query(q)
	if err != nil {
		return nil, err
	}
	if q.KeysOnly {
		return res, nil
	}
	return nil, errors.New("filestore currently only supports keyonly queries")
	// return &queryResult{res, func(r query.Result) query.Result {
	// 	val, err := d.decode(r.Value)
	// 	if err != nil {
	// 		return query.Result{query.Entry{r.Key, nil}, err}
	// 	}
	// 	return query.Result{query.Entry{r.Key, val}, r.Error}
	// }}, nil
}

// type queryResult struct {
// 	query.Results
// 	adjResult func(query.Result) query.Result
// }

// func (q *queryResult) Next() <-chan query.Result {
// 	in := q.Results.Next()
// 	out := make(chan query.Result)
// 	go func() {
// 		res := <-in
// 		if res.Error == nil {
// 			out <- res
// 		}
// 		out <- q.adjResult(res)
// 	}()
// 	return out
// }

// func (q *queryResult) Rest() ([]query.Entry, error) {
// 	res, err := q.Results.Rest()
// 	if err != nil {
// 		return nil, err
// 	}
// 	for _, entry := range res {
// 		newRes := q.adjResult(query.Result{entry, nil})
// 		if newRes.Error != nil {
// 			return nil, newRes.Error
// 		}
// 		entry.Value = newRes.Value
// 	}
// 	return res, nil
// }

func (d *Datastore) Close() error {
	c, ok := d.ds.(io.Closer)
	if ok {
		return c.Close()
	} else {
		return nil
	}
}

func (d *Datastore) Batch() (ds.Batch, error) {
	return ds.NewBasicBatch(d), nil
}
<|endoftext|>