<commit_before>// Copyright 2013 The lime Authors.
// Use of this source code is governed by a 2-clause
// BSD-style license that can be found in the LICENSE file.
package render

import (
	"github.com/quarnster/util/text"
	"image/color"
	"sort"
)

type (
	// Colour represented by a underlying color.RGBA structure
	Colour color.RGBA

	// The Flavour struct contains the specific settings
	// used to style a particular Region.
	Flavour struct {
		Background Colour
		Foreground Colour
		Font       Font
		Flags      ViewRegionFlags
	}

	// The Recipe type groups text.RegionSets by their Flavour.
	// The idea is to allow large groups of text be rendered as
	// a single batch without any state changes inbetween the
	// batches.
	Recipe map[Flavour]text.RegionSet

	// A RenderUnit is just a Flavour and an associated Region.
	RenderUnit struct {
		Flavour Flavour
		Region  text.Region
	}

	// A TranscribedRecipe is a linear (in text.Regions) representation of a Recipe
	TranscribedRecipe []RenderUnit

	ColourScheme interface {
		// Takes a ViewRegions pointer as input and uses the data contained in it
		// to determine the Flavour it should be rendered with.
		Spice(*ViewRegions) Flavour
	}

	Renderer interface {
		// Renders the given Recipe
		Render(Recipe)
	}
)

// Transform takes a ColourScheme, a ViewRegionMap and a viewport as input.
//
// The viewport would be the text.Region of the current buffer that is visible to the user
// and any ViewRegions outside of this area are not forwarded for further processing.
//
// The remaining ViewRegions are then passed on to the ColourScheme for determining the exact Flavour
// for which that RegionSet should be styled, adding Regions of the same Flavour to the same RegionSet.
//
// Typically there are more ViewRegions available in a text buffer than there are unique Flavours in
// a ColourScheme, so this operation can be viewed as reducing the number of state changes required to
// display the text to the user.
//
// The final output, the Recipe, contains a mapping of all unique Flavours and that Flavour's
// associated RegionSet.
func Transform(scheme ColourScheme, data ViewRegionMap, viewport text.Region) Recipe {
	// TODO:
	// 	caret_blink := true
	// if b, ok := v.Settings().Get("caret_blink", true).(bool); ok {
	// 	caret_blink = b
	// }
	//
	// highlight_line := false
	// if b, ok := v.Settings().Get("highlight_line", highlight_line).(bool); ok {
	// 	highlight_line = b
	// }
	//	if b, ok := v.Settings().Get("inverse_caret_state", false).(bool); !b && ok {
	// 	if caret_style == termbox.AttrReverse {
	// 		caret_style = termbox.AttrUnderline
	// 	} else {
	// 		caret_style = termbox.AttrReverse
	// 	}
	// }
	// caret_style := termbox.AttrUnderline
	// if b, ok := v.Settings().Get("caret_style", "underline").(string); ok {
	// 	if b == "block" {
	// 		caret_style = termbox.AttrReverse
	// 	}
	// }

	data.Cull(viewport)
	recipe := make(Recipe)
	for _, v := range data {
		k := scheme.Spice(&v)
		rs := recipe[k]
		rs.AddAll(v.Regions.Regions())
	}
	return recipe
}

// Transcribing the Recipe creates a linear step-by-step
// representation of it, which might or might not
// make it easier for Renderers to work with.
func (r Recipe) Transcribe() (ret TranscribedRecipe) {
	for flav, set := range r {
		for _, r := range set.Regions() {
			ret = append(ret, RenderUnit{Flavour: flav, Region: r})
		}
	}
	sort.Sort(&ret)
	return
}

// Just used to satisfy the sort.Interface interface, typically not used otherwise.
func (r *TranscribedRecipe) Len() int {
	return len(*r)
}

// Just used to satisfy the sort.Interface interface, typically not used otherwise.
func (r *TranscribedRecipe) Less(i, j int) bool {
	a, b := (*r)[i].Region, (*r)[j].Region
	if a.Begin() == b.Begin() {
		return a.End() > b.End()
	}
	return a.Begin() < b.Begin()
}

// Just used to satisfy the sort.Interface interface, typically not used otherwise.
func (r *TranscribedRecipe) Swap(i, j int) {
	(*r)[i], (*r)[j] = (*r)[j], (*r)[i]
}
<commit_msg>Fix overzealous "cleanup" ;)<commit_after>// Copyright 2013 The lime Authors.
// Use of this source code is governed by a 2-clause
// BSD-style license that can be found in the LICENSE file.
package render

import (
	"github.com/quarnster/util/text"
	"image/color"
	"sort"
)

type (
	// Colour represented by a underlying color.RGBA structure
	Colour color.RGBA

	// The Flavour struct contains the specific settings
	// used to style a particular Region.
	Flavour struct {
		Background Colour
		Foreground Colour
		Font       Font
		Flags      ViewRegionFlags
	}

	// The Recipe type groups text.RegionSets by their Flavour.
	// The idea is to allow large groups of text be rendered as
	// a single batch without any state changes inbetween the
	// batches.
	Recipe map[Flavour]text.RegionSet

	// A RenderUnit is just a Flavour and an associated Region.
	RenderUnit struct {
		Flavour Flavour
		Region  text.Region
	}

	// A TranscribedRecipe is a linear (in text.Regions) representation of a Recipe
	TranscribedRecipe []RenderUnit

	ColourScheme interface {
		// Takes a ViewRegions pointer as input and uses the data contained in it
		// to determine the Flavour it should be rendered with.
		Spice(*ViewRegions) Flavour
	}

	Renderer interface {
		// Renders the given Recipe
		Render(Recipe)
	}
)

// Transform takes a ColourScheme, a ViewRegionMap and a viewport as input.
//
// The viewport would be the text.Region of the current buffer that is visible to the user
// and any ViewRegions outside of this area are not forwarded for further processing.
//
// The remaining ViewRegions are then passed on to the ColourScheme for determining the exact Flavour
// for which that RegionSet should be styled, adding Regions of the same Flavour to the same RegionSet.
//
// Typically there are more ViewRegions available in a text buffer than there are unique Flavours in
// a ColourScheme, so this operation can be viewed as reducing the number of state changes required to
// display the text to the user.
//
// The final output, the Recipe, contains a mapping of all unique Flavours and that Flavour's
// associated RegionSet.
func Transform(scheme ColourScheme, data ViewRegionMap, viewport text.Region) Recipe {
	// TODO:
	// 	caret_blink := true
	// if b, ok := v.Settings().Get("caret_blink", true).(bool); ok {
	// 	caret_blink = b
	// }
	//
	// highlight_line := false
	// if b, ok := v.Settings().Get("highlight_line", highlight_line).(bool); ok {
	// 	highlight_line = b
	// }
	//	if b, ok := v.Settings().Get("inverse_caret_state", false).(bool); !b && ok {
	// 	if caret_style == termbox.AttrReverse {
	// 		caret_style = termbox.AttrUnderline
	// 	} else {
	// 		caret_style = termbox.AttrReverse
	// 	}
	// }
	// caret_style := termbox.AttrUnderline
	// if b, ok := v.Settings().Get("caret_style", "underline").(string); ok {
	// 	if b == "block" {
	// 		caret_style = termbox.AttrReverse
	// 	}
	// }

	data.Cull(viewport)
	recipe := make(Recipe)
	for _, v := range data {
		k := scheme.Spice(&v)
		rs := recipe[k]
		rs.AddAll(v.Regions.Regions())
		recipe[k] = rs
	}
	return recipe
}

// Transcribing the Recipe creates a linear step-by-step
// representation of it, which might or might not
// make it easier for Renderers to work with.
func (r Recipe) Transcribe() (ret TranscribedRecipe) {
	for flav, set := range r {
		for _, r := range set.Regions() {
			ret = append(ret, RenderUnit{Flavour: flav, Region: r})
		}
	}
	sort.Sort(&ret)
	return
}

// Just used to satisfy the sort.Interface interface, typically not used otherwise.
func (r *TranscribedRecipe) Len() int {
	return len(*r)
}

// Just used to satisfy the sort.Interface interface, typically not used otherwise.
func (r *TranscribedRecipe) Less(i, j int) bool {
	a, b := (*r)[i].Region, (*r)[j].Region
	if a.Begin() == b.Begin() {
		return a.End() > b.End()
	}
	return a.Begin() < b.Begin()
}

// Just used to satisfy the sort.Interface interface, typically not used otherwise.
func (r *TranscribedRecipe) Swap(i, j int) {
	(*r)[i], (*r)[j] = (*r)[j], (*r)[i]
}
<|endoftext|>


<commit_before>package hello

import (
	"fmt"
	"net/http"
	"io/ioutil"
)

func init() {
	http.HandleFunc("/", handler)
}

func handler(w http.ResponseWriter, r *http.Request) {
	var myval string
	myval = funkyConnection()
	fmt.Fprint(w, myval)
}

func funkyConnection() string {
	resp, _ := http.Get("http://google.com")
	bodyBytes, _ := ioutil.ReadAll(resp.Body)
	return string(bodyBytes)
}

func findMood()
<commit_msg>added sql dependency<commit_after>package hello

import (
	"fmt"
	"net/http"
	"io/ioutil"
	_ "database/sql"
	_ "github.com/go-sql-driver/mysql"
)

func init() {
	http.HandleFunc("/", handler)
}

func handler(w http.ResponseWriter, r *http.Request) {
	var myval string
	myval = funkyConnection()
	fmt.Fprint(w, myval)
}

func funkyConnection() string {
	resp, _ := http.Get("http://google.com")
	bodyBytes, _ := ioutil.ReadAll(resp.Body)
	return string(bodyBytes)
}

func findMood() {

}
<|endoftext|>


<commit_before>package shadowsocks

import (
	"encoding/binary"
	"fmt"
	"net"
	"strconv"
	"time"
	"syscall"
	"sync"
)

const (
	idType    = 0 // address type index
	idIP0     = 1 // ip addres start index
	idDmLen   = 1 // domain address length index
	idDm0     = 2 // domain address start index

	typeIPv4  = 1 // type is ipv4 address
	typeDm    = 3 // type is domain address
	typeIPv6  = 4 // type is ipv6 address

	lenIPv4   = 1 + net.IPv4len + 2 // 1addrType + ipv4 + 2port
	lenIPv6   = 1 + net.IPv6len + 2 // 1addrType + ipv6 + 2port
	lenDmBase = 1 + 1 + 2           // 1addrType + 1addrLen + 2port, plus addrLen
)

var (
	reqList   = ReqList{List:map[string]*ReqNode{}}
	nl        = NATlist{Conns: map[string]*CachedUDPConn{}}
	udpBuf    = NewLeakyBuf(nBuf, bufSize)
)

type UDP interface {
	ReadFromUDP(b []byte) (n int, src *net.UDPAddr, err error)
	Read(b []byte) (n int, err error)
	WriteToUDP(b []byte, src *net.UDPAddr) (n int, err error)
	Write(b []byte) (n int, err error)
	Close() error
	SetWriteDeadline(t time.Time) error
	SetReadDeadline(t time.Time) error
	LocalAddr() net.Addr
	RemoteAddr() net.Addr
	ReadFrom(b []byte) (int, net.Addr, error)
}

type UDPConn struct {
	UDP
	*Cipher
}

func NewUDPConn(cn UDP, cipher *Cipher) *UDPConn {
	return &UDPConn{cn, cipher}
}

type CachedUDPConn struct {
	timer *time.Timer
	UDP
	i string // scraddr
}

func NewCachedUDPConn(cn UDP) *CachedUDPConn {
	return &CachedUDPConn{nil, cn, ""}
}

func (c *CachedUDPConn) Check() {
	go nl.Delete(c.i)
}

func (c *CachedUDPConn) Close() error{
	c.timer.Stop()
	return c.UDP.Close()
}

func (c *CachedUDPConn) SetTimer(index string) {
	c.i = index
	c.timer = time.AfterFunc(120*time.Second, c.Check)
}

func (c *CachedUDPConn) Refresh() bool {
	return c.timer.Reset(120*time.Second)
}

type NATlist struct {
	sync.Mutex
	Conns map[string]*CachedUDPConn
	AliveConns int
}

func (nl *NATlist) Delete(srcaddr string) {
	nl.Lock()
	c , ok := nl.Conns[srcaddr]
	if ok {
		c.Close()
		delete(nl.Conns, srcaddr)
		nl.AliveConns -= 1
	}
	reqList.Refresh()
	defer nl.Unlock()
}

func (nl *NATlist) Get(srcaddr *net.UDPAddr, ss *UDPConn) (c *CachedUDPConn, ok bool, err error){
	nl.Lock()
	defer nl.Unlock()
	index := srcaddr.String()
	_ , ok = nl.Conns[index]
	if !ok {
		//NAT not exists or expired
		nl.AliveConns += 1
		delete(nl.Conns, index)
		ok = false
		//full cone
		conn, err := net.ListenUDP("udp", &net.UDPAddr{
			IP:   net.IPv6zero,
			Port: 0,
		})
		if err != nil {
			return nil, false, err
		}
		nl.Conns[index] = NewCachedUDPConn(conn)
		c , _ = nl.Conns[index]
		c.SetTimer(index)
		go Pipeloop(ss, srcaddr, c)
	} else {
		//NAT exists
		c , _ = nl.Conns[index]
		c.Refresh()
	}
	err = nil
	return
}

type ReqNode struct {
	Req []byte
	ReqLen int
}

type ReqList struct {
	List map[string]*ReqNode
	sync.Mutex
}

func (r *ReqList) Refresh() {
	r.Lock()
	defer r.Unlock()
	for k, _  := range r.List {
		delete(r.List, k)
	}
}

func (r *ReqList) Get(dstaddr string) (req *ReqNode, ok bool) {
	r.Lock()
	defer r.Unlock()
	req, ok = r.List[dstaddr]
	return
}

func (r *ReqList) Put(dstaddr string, req *ReqNode) {
	r.Lock()
	defer r.Unlock()
	r.List[dstaddr] = req
	return
}

func ParseHeader(addr net.Addr) ([]byte, int) {
//what if the request address type is domain???
	ip, port, err := net.SplitHostPort(addr.String())
	if err != nil {
		return nil, 0
	}
	buf := make([]byte, 20)
	IP := net.ParseIP(ip)
	b1 := IP.To4()
	iplen := 0
	if b1 == nil { //ipv6
		b1 = IP.To16()
		buf[0] = typeIPv6
		iplen = net.IPv6len
	} else { //ipv4
		buf[0] = typeIPv4
		iplen = net.IPv4len
	}
	copy(buf[1:], b1)
	port_i, _ := strconv.Atoi(port)
	binary.BigEndian.PutUint16(buf[1+iplen:], uint16(port_i))
	return buf[:1+iplen+2], 1+iplen+2
}

func Pipeloop(ss *UDPConn, srcaddr *net.UDPAddr, remote UDP) {
	buf := udpBuf.Get()
	defer udpBuf.Put(buf)
	defer nl.Delete(srcaddr.String())
	for{
		remote.SetReadDeadline(time.Now().Add(readTimeout))
		n, raddr, err := remote.ReadFrom(buf)
		if err != nil {
			if ne, ok := err.(*net.OpError); ok && (ne.Err == syscall.EMFILE || ne.Err == syscall.ENFILE) {
				// log too many open file error
				// EMFILE is process reaches open file limits, ENFILE is system limit
				fmt.Println("[udp]read error:", err)
			} else if ne.Err.Error() == "use of closed network connection" {
				fmt.Println("[udp]Connection Closing:", remote.LocalAddr())
			} else {
				fmt.Println("[udp]error reading from:", remote.LocalAddr(), err)
			}
			return
		}
		// need improvement here
		if N, ok := reqList.Get(raddr.String()); ok {
			go ss.WriteToUDP(append(N.Req[:N.ReqLen], buf[:n]...), srcaddr)
		}	else {
			header, hlen := ParseHeader(raddr)
			go ss.WriteToUDP(append(header[:hlen], buf[:n]...), srcaddr)
		}
	}
}

func (c *UDPConn) handleUDPConnection(n int, src *net.UDPAddr, receive []byte) {
	var dstIP net.IP
	var reqLen int
	defer udpBuf.Put(receive)

	switch receive[idType] {
	case typeIPv4:
		reqLen = lenIPv4
		dstIP = net.IP(receive[idIP0 : idIP0+net.IPv4len])
	case typeIPv6:
		reqLen = lenIPv6
		dstIP = net.IP(receive[idIP0 : idIP0+net.IPv6len])
	case typeDm:
		reqLen = int(receive[idDmLen]) + lenDmBase
		dIP, err := net.ResolveIPAddr("ip" ,string(receive[idDm0 : idDm0+receive[idDmLen]]))
		if err != nil{
			fmt.Sprintf("[udp]failed to resolve domain name: %s\n", string(receive[idDm0 : idDm0+receive[idDmLen]]))
			return
		}
		dstIP = dIP.IP
	default:
		fmt.Sprintf("[udp]addr type %d not supported", receive[idType])
		return
	}
	dst := &net.UDPAddr{
		IP:   dstIP,
		Port: int(binary.BigEndian.Uint16(receive[reqLen-2 : reqLen])),
	}
	if _, ok := reqList.Get(dst.String()); !ok {
		req := make([]byte, reqLen)
		for i:=0;i<reqLen;i++ {
			req[i] = receive[i]
		}
		reqList.Put(dst.String(), &ReqNode{req, reqLen})
	}

	remote, _, err := nl.Get(src, c)
	if err != nil {
		return
	}
	remote.SetWriteDeadline(time.Now().Add(readTimeout))
	_, err = remote.WriteToUDP(receive[reqLen:n], dst)
	if err != nil {
		if ne, ok := err.(*net.OpError); ok && (ne.Err == syscall.EMFILE || ne.Err == syscall.ENFILE) {
			// log too many open file error
			// EMFILE is process reaches open file limits, ENFILE is system limit
			fmt.Println("[udp]write error:", err)
		} else {
			fmt.Println("[udp]error connecting to:", dst, err)
		}
		return
	}
	// Pipeloop
	return
}

func (c *UDPConn) ReadAndHandleUDPReq()  {
	buf := udpBuf.Get()
	n, src, err := c.ReadFromUDP(buf[0:])
	if err != nil {
		return
	}
	go c.handleUDPConnection(n, src, buf)
}

//n is the size of the payload
func (c *UDPConn) ReadFromUDP(b []byte) (n int, src *net.UDPAddr, err error) {
	buf := udpBuf.Get()
	n, src, err = c.UDP.ReadFromUDP(buf[0:])
	if err != nil {
		return
	}
	defer udpBuf.Put(buf)

	iv := buf[:c.info.ivLen]
	if err = c.initDecrypt(iv); err != nil {
		return
	}
	if n < c.info.ivLen {
		return 0, nil, fmt.Errorf("[udp]write error: cannot decrypt")
	}
	c.decrypt(b[0:n - c.info.ivLen], buf[c.info.ivLen : n])
	n = n - c.info.ivLen
	return
}

func (c *UDPConn) Read(b []byte) (n int, err error) {
	buf := udpBuf.Get()
	n, err = c.UDP.Read(buf[0:])
	if err != nil {
		return
	}
	defer udpBuf.Put(buf)

	iv := buf[:c.info.ivLen]
	if err = c.initDecrypt(iv); err != nil {
		return
	}
	c.decrypt(b[0:n - c.info.ivLen], buf[c.info.ivLen : n])
	n = n - c.info.ivLen
	return
}

//n = iv + payload
func (c *UDPConn) WriteToUDP(b []byte, src *net.UDPAddr) (n int, err error) {
	var cipherData []byte
	dataStart := 0

	var iv []byte
	iv, err = c.initEncrypt()
	if err != nil {
		return
	}
	// Put initialization vector in buffer, do a single write to send both
	// iv and data.
	cipherData = make([]byte, len(b)+len(iv))
	copy(cipherData, iv)
	dataStart = len(iv)

	c.encrypt(cipherData[dataStart:], b)
	n, err = c.UDP.WriteToUDP(cipherData, src)
	return
}

func (c *UDPConn) Write(b []byte) (n int, err error) {
	var cipherData []byte
	dataStart := 0

	var iv []byte
	iv, err = c.initEncrypt()
	if err != nil {
		return
	}
	// Put initialization vector in buffer, do a single write to send both
	// iv and data.
	cipherData = make([]byte, len(b)+len(iv))
	copy(cipherData, iv)
	dataStart = len(iv)

	c.encrypt(cipherData[dataStart:], b)
	n, err = c.UDP.Write(cipherData)
	return
}
<commit_msg>Now compatible with 1.1.4 code. Passed preliminary test with salsa20 & chacha20.<commit_after>package shadowsocks

import (
	"encoding/binary"
	"fmt"
	"net"
	"strconv"
	"time"
	"syscall"
	"sync"
)

const (
	idType    = 0 // address type index
	idIP0     = 1 // ip addres start index
	idDmLen   = 1 // domain address length index
	idDm0     = 2 // domain address start index

	typeIPv4  = 1 // type is ipv4 address
	typeDm    = 3 // type is domain address
	typeIPv6  = 4 // type is ipv6 address

	lenIPv4   = 1 + net.IPv4len + 2 // 1addrType + ipv4 + 2port
	lenIPv6   = 1 + net.IPv6len + 2 // 1addrType + ipv6 + 2port
	lenDmBase = 1 + 1 + 2           // 1addrType + 1addrLen + 2port, plus addrLen
)

var (
	reqList   = ReqList{List:map[string]*ReqNode{}}
	nl        = NATlist{Conns: map[string]*CachedUDPConn{}}
)

type UDP interface {
	ReadFromUDP(b []byte) (n int, src *net.UDPAddr, err error)
	Read(b []byte) (n int, err error)
	WriteToUDP(b []byte, src *net.UDPAddr) (n int, err error)
	Write(b []byte) (n int, err error)
	Close() error
	SetWriteDeadline(t time.Time) error
	SetReadDeadline(t time.Time) error
	LocalAddr() net.Addr
	RemoteAddr() net.Addr
	ReadFrom(b []byte) (int, net.Addr, error)
}

type UDPConn struct {
	UDP
	*Cipher
}

func NewUDPConn(cn UDP, cipher *Cipher) *UDPConn {
	return &UDPConn{cn, cipher}
}

type CachedUDPConn struct {
	timer *time.Timer
	UDP
	i string // scraddr
}

func NewCachedUDPConn(cn UDP) *CachedUDPConn {
	return &CachedUDPConn{nil, cn, ""}
}

func (c *CachedUDPConn) Check() {
	go nl.Delete(c.i)
}

func (c *CachedUDPConn) Close() error{
	c.timer.Stop()
	return c.UDP.Close()
}

func (c *CachedUDPConn) SetTimer(index string) {
	c.i = index
	c.timer = time.AfterFunc(120*time.Second, c.Check)
}

func (c *CachedUDPConn) Refresh() bool {
	return c.timer.Reset(120*time.Second)
}

type NATlist struct {
	sync.Mutex
	Conns map[string]*CachedUDPConn
	AliveConns int
}

func (nl *NATlist) Delete(srcaddr string) {
	nl.Lock()
	c , ok := nl.Conns[srcaddr]
	if ok {
		c.Close()
		delete(nl.Conns, srcaddr)
		nl.AliveConns -= 1
	}
	reqList.Refresh()
	defer nl.Unlock()
}

func (nl *NATlist) Get(srcaddr *net.UDPAddr, ss *UDPConn) (c *CachedUDPConn, ok bool, err error){
	nl.Lock()
	defer nl.Unlock()
	index := srcaddr.String()
	_ , ok = nl.Conns[index]
	if !ok {
		//NAT not exists or expired
		nl.AliveConns += 1
		delete(nl.Conns, index)
		ok = false
		//full cone
		conn, err := net.ListenUDP("udp", &net.UDPAddr{
			IP:   net.IPv6zero,
			Port: 0,
		})
		if err != nil {
			return nil, false, err
		}
		nl.Conns[index] = NewCachedUDPConn(conn)
		c , _ = nl.Conns[index]
		c.SetTimer(index)
		go Pipeloop(ss, srcaddr, c)
	} else {
		//NAT exists
		c , _ = nl.Conns[index]
		c.Refresh()
	}
	err = nil
	return
}

type ReqNode struct {
	Req []byte
	ReqLen int
}

type ReqList struct {
	List map[string]*ReqNode
	sync.Mutex
}

func (r *ReqList) Refresh() {
	r.Lock()
	defer r.Unlock()
	for k, _  := range r.List {
		delete(r.List, k)
	}
}

func (r *ReqList) Get(dstaddr string) (req *ReqNode, ok bool) {
	r.Lock()
	defer r.Unlock()
	req, ok = r.List[dstaddr]
	return
}

func (r *ReqList) Put(dstaddr string, req *ReqNode) {
	r.Lock()
	defer r.Unlock()
	r.List[dstaddr] = req
	return
}

func ParseHeader(addr net.Addr) ([]byte, int) {
//what if the request address type is domain???
	ip, port, err := net.SplitHostPort(addr.String())
	if err != nil {
		return nil, 0
	}
	buf := make([]byte, 20)
	IP := net.ParseIP(ip)
	b1 := IP.To4()
	iplen := 0
	if b1 == nil { //ipv6
		b1 = IP.To16()
		buf[0] = typeIPv6
		iplen = net.IPv6len
	} else { //ipv4
		buf[0] = typeIPv4
		iplen = net.IPv4len
	}
	copy(buf[1:], b1)
	port_i, _ := strconv.Atoi(port)
	binary.BigEndian.PutUint16(buf[1+iplen:], uint16(port_i))
	return buf[:1+iplen+2], 1+iplen+2
}

func Pipeloop(ss *UDPConn, srcaddr *net.UDPAddr, remote UDP) {
	buf := leakyBuf.Get()
	defer leakyBuf.Put(buf)
	defer nl.Delete(srcaddr.String())
	for{
		remote.SetReadDeadline(time.Now().Add(readTimeout))
		n, raddr, err := remote.ReadFrom(buf)
		if err != nil {
			if ne, ok := err.(*net.OpError); ok && (ne.Err == syscall.EMFILE || ne.Err == syscall.ENFILE) {
				// log too many open file error
				// EMFILE is process reaches open file limits, ENFILE is system limit
				fmt.Println("[udp]read error:", err)
			} else if ne.Err.Error() == "use of closed network connection" {
				fmt.Println("[udp]Connection Closing:", remote.LocalAddr())
			} else {
				fmt.Println("[udp]error reading from:", remote.LocalAddr(), err)
			}
			return
		}
		// need improvement here
		if N, ok := reqList.Get(raddr.String()); ok {
			go ss.WriteToUDP(append(N.Req[:N.ReqLen], buf[:n]...), srcaddr)
		}	else {
			header, hlen := ParseHeader(raddr)
			go ss.WriteToUDP(append(header[:hlen], buf[:n]...), srcaddr)
		}
	}
}

func (c *UDPConn) handleUDPConnection(n int, src *net.UDPAddr, receive []byte) {
	var dstIP net.IP
	var reqLen int
	defer leakyBuf.Put(receive)

	switch receive[idType] {
	case typeIPv4:
		reqLen = lenIPv4
		dstIP = net.IP(receive[idIP0 : idIP0+net.IPv4len])
	case typeIPv6:
		reqLen = lenIPv6
		dstIP = net.IP(receive[idIP0 : idIP0+net.IPv6len])
	case typeDm:
		reqLen = int(receive[idDmLen]) + lenDmBase
		dIP, err := net.ResolveIPAddr("ip" ,string(receive[idDm0 : idDm0+receive[idDmLen]]))
		if err != nil{
			fmt.Sprintf("[udp]failed to resolve domain name: %s\n", string(receive[idDm0 : idDm0+receive[idDmLen]]))
			return
		}
		dstIP = dIP.IP
	default:
		fmt.Sprintf("[udp]addr type %d not supported", receive[idType])
		return
	}
	dst := &net.UDPAddr{
		IP:   dstIP,
		Port: int(binary.BigEndian.Uint16(receive[reqLen-2 : reqLen])),
	}
	if _, ok := reqList.Get(dst.String()); !ok {
		req := make([]byte, reqLen)
		for i:=0;i<reqLen;i++ {
			req[i] = receive[i]
		}
		reqList.Put(dst.String(), &ReqNode{req, reqLen})
	}

	remote, _, err := nl.Get(src, c)
	if err != nil {
		return
	}
	remote.SetWriteDeadline(time.Now().Add(readTimeout))
	_, err = remote.WriteToUDP(receive[reqLen:n], dst)
	if err != nil {
		if ne, ok := err.(*net.OpError); ok && (ne.Err == syscall.EMFILE || ne.Err == syscall.ENFILE) {
			// log too many open file error
			// EMFILE is process reaches open file limits, ENFILE is system limit
			fmt.Println("[udp]write error:", err)
		} else {
			fmt.Println("[udp]error connecting to:", dst, err)
		}
		return
	}
	// Pipeloop
	return
}

func (c *UDPConn) ReadAndHandleUDPReq()  {
	buf := leakyBuf.Get()
	n, src, err := c.ReadFromUDP(buf[0:])
	if err != nil {
		return
	}
	go c.handleUDPConnection(n, src, buf)
}

//n is the size of the payload
func (c *UDPConn) ReadFromUDP(b []byte) (n int, src *net.UDPAddr, err error) {
	buf := leakyBuf.Get()
	n, src, err = c.UDP.ReadFromUDP(buf[0:])
	if err != nil {
		return
	}
	defer leakyBuf.Put(buf)

	iv := buf[:c.info.ivLen]
	if err = c.initDecrypt(iv); err != nil {
		return
	}
	if n < c.info.ivLen {
		return 0, nil, fmt.Errorf("[udp]write error: cannot decrypt")
	}
	c.decrypt(b[0:n - c.info.ivLen], buf[c.info.ivLen : n])
	n = n - c.info.ivLen
	return
}

func (c *UDPConn) Read(b []byte) (n int, err error) {
	buf := leakyBuf.Get()
	n, err = c.UDP.Read(buf[0:])
	if err != nil {
		return
	}
	defer leakyBuf.Put(buf)

	iv := buf[:c.info.ivLen]
	if err = c.initDecrypt(iv); err != nil {
		return
	}
	c.decrypt(b[0:n - c.info.ivLen], buf[c.info.ivLen : n])
	n = n - c.info.ivLen
	return
}

//n = iv + payload
func (c *UDPConn) WriteToUDP(b []byte, src *net.UDPAddr) (n int, err error) {
	var cipherData []byte
	dataStart := 0

	var iv []byte
	iv, err = c.initEncrypt()
	if err != nil {
		return
	}
	// Put initialization vector in buffer, do a single write to send both
	// iv and data.
	cipherData = make([]byte, len(b)+len(iv))
	copy(cipherData, iv)
	dataStart = len(iv)

	c.encrypt(cipherData[dataStart:], b)
	n, err = c.UDP.WriteToUDP(cipherData, src)
	return
}

func (c *UDPConn) Write(b []byte) (n int, err error) {
	var cipherData []byte
	dataStart := 0

	var iv []byte
	iv, err = c.initEncrypt()
	if err != nil {
		return
	}
	// Put initialization vector in buffer, do a single write to send both
	// iv and data.
	cipherData = make([]byte, len(b)+len(iv))
	copy(cipherData, iv)
	dataStart = len(iv)

	c.encrypt(cipherData[dataStart:], b)
	n, err = c.UDP.Write(cipherData)
	return
}
<|endoftext|>


<commit_before>package shared

import (
	"fmt"
	"regexp"
	"strconv"
	"strings"
	"time"

	"github.com/pkg/errors"
	"gopkg.in/robfig/cron.v2"

	"github.com/lxc/lxd/shared/units"
	"github.com/lxc/lxd/shared/validate"
)

type InstanceAction string

const (
	Stop     InstanceAction = "stop"
	Start    InstanceAction = "start"
	Restart  InstanceAction = "restart"
	Freeze   InstanceAction = "freeze"
	Unfreeze InstanceAction = "unfreeze"
)

// IsRootDiskDevice returns true if the given device representation is configured as root disk for
// a container. It typically get passed a specific entry of api.Instance.Devices.
func IsRootDiskDevice(device map[string]string) bool {
	// Root disk devices also need a non-empty "pool" property, but we can't check that here
	// because this function is used with clients talking to older servers where there was no
	// concept of a storage pool, and also it is used for migrating from old to new servers.
	// The validation of the non-empty "pool" property is done inside the disk device itself.
	if device["type"] == "disk" && device["path"] == "/" && device["source"] == "" {
		return true
	}

	return false
}

// GetRootDiskDevice returns the container device that is configured as root disk
func GetRootDiskDevice(devices map[string]map[string]string) (string, map[string]string, error) {
	var devName string
	var dev map[string]string

	for n, d := range devices {
		if IsRootDiskDevice(d) {
			if devName != "" {
				return "", nil, fmt.Errorf("More than one root device found")
			}

			devName = n
			dev = d
		}
	}

	if devName != "" {
		return devName, dev, nil
	}

	return "", nil, fmt.Errorf("No root device could be found")
}

// HugePageSizeKeys is a list of known hugepage size configuration keys.
var HugePageSizeKeys = [...]string{"limits.hugepages.64KB", "limits.hugepages.1MB", "limits.hugepages.2MB", "limits.hugepages.1GB"}

// HugePageSizeSuffix contains the list of known hugepage size suffixes.
var HugePageSizeSuffix = [...]string{"64KB", "1MB", "2MB", "1GB"}

// KnownInstanceConfigKeys maps all fully defined, well-known config keys
// to an appropriate checker function, which validates whether or not a
// given value is syntactically legal.
var KnownInstanceConfigKeys = map[string]func(value string) error{
	"boot.autostart":             validate.IsBool,
	"boot.autostart.delay":       validate.IsInt64,
	"boot.autostart.priority":    validate.IsInt64,
	"boot.stop.priority":         validate.IsInt64,
	"boot.host_shutdown_timeout": validate.IsInt64,

	"limits.cpu": func(value string) error {
		if value == "" {
			return nil
		}

		// Validate the character set
		match, _ := regexp.MatchString("^[-,0-9]*$", value)
		if !match {
			return fmt.Errorf("Invalid CPU limit syntax")
		}

		// Validate first character
		if strings.HasPrefix(value, "-") || strings.HasPrefix(value, ",") {
			return fmt.Errorf("CPU limit can't start with a separator")
		}

		// Validate last character
		if strings.HasSuffix(value, "-") || strings.HasSuffix(value, ",") {
			return fmt.Errorf("CPU limit can't end with a separator")
		}

		return nil
	},
	"limits.cpu.allowance": func(value string) error {
		if value == "" {
			return nil
		}

		if strings.HasSuffix(value, "%") {
			// Percentage based allocation
			_, err := strconv.Atoi(strings.TrimSuffix(value, "%"))
			if err != nil {
				return err
			}

			return nil
		}

		// Time based allocation
		fields := strings.SplitN(value, "/", 2)
		if len(fields) != 2 {
			return fmt.Errorf("Invalid allowance: %s", value)
		}

		_, err := strconv.Atoi(strings.TrimSuffix(fields[0], "ms"))
		if err != nil {
			return err
		}

		_, err = strconv.Atoi(strings.TrimSuffix(fields[1], "ms"))
		if err != nil {
			return err
		}

		return nil
	},
	"limits.cpu.priority": validate.IsPriority,

	"limits.disk.priority": validate.IsPriority,

	"limits.hugepages.64KB": validate.IsSize,
	"limits.hugepages.1MB":  validate.IsSize,
	"limits.hugepages.2MB":  validate.IsSize,
	"limits.hugepages.1GB":  validate.IsSize,

	"limits.memory": func(value string) error {
		if value == "" {
			return nil
		}

		if strings.HasSuffix(value, "%") {
			_, err := strconv.ParseInt(strings.TrimSuffix(value, "%"), 10, 64)
			if err != nil {
				return err
			}

			return nil
		}

		_, err := units.ParseByteSizeString(value)
		if err != nil {
			return err
		}

		return nil
	},
	"limits.memory.enforce": func(value string) error {
		return validate.IsOneOf(value, []string{"soft", "hard"})
	},
	"limits.memory.swap":          validate.IsBool,
	"limits.memory.swap.priority": validate.IsPriority,
	"limits.memory.hugepages":     validate.IsBool,

	"limits.network.priority": validate.IsPriority,

	"limits.processes": validate.IsInt64,

	"linux.kernel_modules": validate.IsAny,

	"migration.incremental.memory":            validate.IsBool,
	"migration.incremental.memory.iterations": validate.IsUint32,
	"migration.incremental.memory.goal":       validate.IsUint32,

	"nvidia.runtime":             validate.IsBool,
	"nvidia.driver.capabilities": validate.IsAny,
	"nvidia.require.cuda":        validate.IsAny,
	"nvidia.require.driver":      validate.IsAny,

	"security.nesting":       validate.IsBool,
	"security.privileged":    validate.IsBool,
	"security.devlxd":        validate.IsBool,
	"security.devlxd.images": validate.IsBool,

	"security.protection.delete": validate.IsBool,
	"security.protection.shift":  validate.IsBool,

	"security.idmap.base":     validate.IsUint32,
	"security.idmap.isolated": validate.IsBool,
	"security.idmap.size":     validate.IsUint32,

	"security.secureboot": validate.IsBool,

	"security.syscalls.allow":                   validate.IsAny,
	"security.syscalls.blacklist_default":       validate.IsBool,
	"security.syscalls.blacklist_compat":        validate.IsBool,
	"security.syscalls.blacklist":               validate.IsAny,
	"security.syscalls.deny_default":            validate.IsBool,
	"security.syscalls.deny_compat":             validate.IsBool,
	"security.syscalls.deny":                    validate.IsAny,
	"security.syscalls.intercept.mknod":         validate.IsBool,
	"security.syscalls.intercept.mount":         validate.IsBool,
	"security.syscalls.intercept.mount.allowed": validate.IsAny,
	"security.syscalls.intercept.mount.fuse":    validate.IsAny,
	"security.syscalls.intercept.mount.shift":   validate.IsBool,
	"security.syscalls.intercept.setxattr":      validate.IsBool,
	"security.syscalls.whitelist":               validate.IsAny,

	"snapshots.schedule": func(value string) error {
		if value == "" {
			return nil
		}

		if len(strings.Split(value, " ")) != 5 {
			return fmt.Errorf("Schedule must be of the form: <minute> <hour> <day-of-month> <month> <day-of-week>")
		}

		_, err := cron.Parse(fmt.Sprintf("* %s", value))
		if err != nil {
			return errors.Wrap(err, "Error parsing schedule")
		}

		return nil
	},
	"snapshots.schedule.stopped": validate.IsBool,
	"snapshots.pattern":          validate.IsAny,
	"snapshots.expiry": func(value string) error {
		// Validate expression
		_, err := GetSnapshotExpiry(time.Time{}, value)
		return err
	},

	// Caller is responsible for full validation of any raw.* value
	"raw.apparmor": validate.IsAny,
	"raw.idmap":    validate.IsAny,
	"raw.lxc":      validate.IsAny,
	"raw.qemu":     validate.IsAny,
	"raw.seccomp":  validate.IsAny,

	"volatile.apply_template":   validate.IsAny,
	"volatile.base_image":       validate.IsAny,
	"volatile.last_state.idmap": validate.IsAny,
	"volatile.last_state.power": validate.IsAny,
	"volatile.idmap.base":       validate.IsAny,
	"volatile.idmap.current":    validate.IsAny,
	"volatile.idmap.next":       validate.IsAny,
	"volatile.apply_quota":      validate.IsAny,
}

// ConfigKeyChecker returns a function that will check whether or not
// a provide value is valid for the associate config key.  Returns an
// error if the key is not known.  The checker function only performs
// syntactic checking of the value, semantic and usage checking must
// be done by the caller.  User defined keys are always considered to
// be valid, e.g. user.* and environment.* keys.
func ConfigKeyChecker(key string) (func(value string) error, error) {
	if f, ok := KnownInstanceConfigKeys[key]; ok {
		return f, nil
	}

	if strings.HasPrefix(key, "volatile.") {
		if strings.HasSuffix(key, ".hwaddr") {
			return validate.IsAny, nil
		}

		if strings.HasSuffix(key, ".name") {
			return validate.IsAny, nil
		}

		if strings.HasSuffix(key, ".host_name") {
			return validate.IsAny, nil
		}

		if strings.HasSuffix(key, ".mtu") {
			return validate.IsAny, nil
		}

		if strings.HasSuffix(key, ".created") {
			return validate.IsAny, nil
		}

		if strings.HasSuffix(key, ".id") {
			return validate.IsAny, nil
		}

		if strings.HasSuffix(key, ".vlan") {
			return validate.IsAny, nil
		}

		if strings.HasSuffix(key, ".spoofcheck") {
			return validate.IsAny, nil
		}

		if strings.HasSuffix(key, ".apply_quota") {
			return validate.IsAny, nil
		}

		if strings.HasSuffix(key, "vm.uuid") {
			return validate.IsAny, nil
		}

		if strings.HasSuffix(key, ".ceph_rbd") {
			return validate.IsAny, nil
		}

		if strings.HasSuffix(key, ".driver") {
			return validate.IsAny, nil
		}
	}

	if strings.HasPrefix(key, "environment.") {
		return validate.IsAny, nil
	}

	if strings.HasPrefix(key, "user.") {
		return validate.IsAny, nil
	}

	if strings.HasPrefix(key, "image.") {
		return validate.IsAny, nil
	}

	if strings.HasPrefix(key, "limits.kernel.") &&
		(len(key) > len("limits.kernel.")) {
		return validate.IsAny, nil
	}

	return nil, fmt.Errorf("Unknown configuration key: %s", key)
}

// InstanceGetParentAndSnapshotName returns the parent instance name, snapshot name,
// and whether it actually was a snapshot name.
func InstanceGetParentAndSnapshotName(name string) (string, string, bool) {
	fields := strings.SplitN(name, SnapshotDelimiter, 2)
	if len(fields) == 1 {
		return name, "", false
	}

	return fields[0], fields[1], true
}
<commit_msg>shared/instance: Adds validate.Optional() wrapper for validate.IsInt64 usage<commit_after>package shared

import (
	"fmt"
	"regexp"
	"strconv"
	"strings"
	"time"

	"github.com/pkg/errors"
	"gopkg.in/robfig/cron.v2"

	"github.com/lxc/lxd/shared/units"
	"github.com/lxc/lxd/shared/validate"
)

type InstanceAction string

const (
	Stop     InstanceAction = "stop"
	Start    InstanceAction = "start"
	Restart  InstanceAction = "restart"
	Freeze   InstanceAction = "freeze"
	Unfreeze InstanceAction = "unfreeze"
)

// IsRootDiskDevice returns true if the given device representation is configured as root disk for
// a container. It typically get passed a specific entry of api.Instance.Devices.
func IsRootDiskDevice(device map[string]string) bool {
	// Root disk devices also need a non-empty "pool" property, but we can't check that here
	// because this function is used with clients talking to older servers where there was no
	// concept of a storage pool, and also it is used for migrating from old to new servers.
	// The validation of the non-empty "pool" property is done inside the disk device itself.
	if device["type"] == "disk" && device["path"] == "/" && device["source"] == "" {
		return true
	}

	return false
}

// GetRootDiskDevice returns the container device that is configured as root disk
func GetRootDiskDevice(devices map[string]map[string]string) (string, map[string]string, error) {
	var devName string
	var dev map[string]string

	for n, d := range devices {
		if IsRootDiskDevice(d) {
			if devName != "" {
				return "", nil, fmt.Errorf("More than one root device found")
			}

			devName = n
			dev = d
		}
	}

	if devName != "" {
		return devName, dev, nil
	}

	return "", nil, fmt.Errorf("No root device could be found")
}

// HugePageSizeKeys is a list of known hugepage size configuration keys.
var HugePageSizeKeys = [...]string{"limits.hugepages.64KB", "limits.hugepages.1MB", "limits.hugepages.2MB", "limits.hugepages.1GB"}

// HugePageSizeSuffix contains the list of known hugepage size suffixes.
var HugePageSizeSuffix = [...]string{"64KB", "1MB", "2MB", "1GB"}

// KnownInstanceConfigKeys maps all fully defined, well-known config keys
// to an appropriate checker function, which validates whether or not a
// given value is syntactically legal.
var KnownInstanceConfigKeys = map[string]func(value string) error{
	"boot.autostart":             validate.IsBool,
	"boot.autostart.delay":       validate.Optional(validate.IsInt64),
	"boot.autostart.priority":    validate.Optional(validate.IsInt64),
	"boot.stop.priority":         validate.Optional(validate.IsInt64),
	"boot.host_shutdown_timeout": validate.Optional(validate.IsInt64),

	"limits.cpu": func(value string) error {
		if value == "" {
			return nil
		}

		// Validate the character set
		match, _ := regexp.MatchString("^[-,0-9]*$", value)
		if !match {
			return fmt.Errorf("Invalid CPU limit syntax")
		}

		// Validate first character
		if strings.HasPrefix(value, "-") || strings.HasPrefix(value, ",") {
			return fmt.Errorf("CPU limit can't start with a separator")
		}

		// Validate last character
		if strings.HasSuffix(value, "-") || strings.HasSuffix(value, ",") {
			return fmt.Errorf("CPU limit can't end with a separator")
		}

		return nil
	},
	"limits.cpu.allowance": func(value string) error {
		if value == "" {
			return nil
		}

		if strings.HasSuffix(value, "%") {
			// Percentage based allocation
			_, err := strconv.Atoi(strings.TrimSuffix(value, "%"))
			if err != nil {
				return err
			}

			return nil
		}

		// Time based allocation
		fields := strings.SplitN(value, "/", 2)
		if len(fields) != 2 {
			return fmt.Errorf("Invalid allowance: %s", value)
		}

		_, err := strconv.Atoi(strings.TrimSuffix(fields[0], "ms"))
		if err != nil {
			return err
		}

		_, err = strconv.Atoi(strings.TrimSuffix(fields[1], "ms"))
		if err != nil {
			return err
		}

		return nil
	},
	"limits.cpu.priority": validate.IsPriority,

	"limits.disk.priority": validate.IsPriority,

	"limits.hugepages.64KB": validate.IsSize,
	"limits.hugepages.1MB":  validate.IsSize,
	"limits.hugepages.2MB":  validate.IsSize,
	"limits.hugepages.1GB":  validate.IsSize,

	"limits.memory": func(value string) error {
		if value == "" {
			return nil
		}

		if strings.HasSuffix(value, "%") {
			_, err := strconv.ParseInt(strings.TrimSuffix(value, "%"), 10, 64)
			if err != nil {
				return err
			}

			return nil
		}

		_, err := units.ParseByteSizeString(value)
		if err != nil {
			return err
		}

		return nil
	},
	"limits.memory.enforce": func(value string) error {
		return validate.IsOneOf(value, []string{"soft", "hard"})
	},
	"limits.memory.swap":          validate.IsBool,
	"limits.memory.swap.priority": validate.IsPriority,
	"limits.memory.hugepages":     validate.IsBool,

	"limits.network.priority": validate.IsPriority,

	"limits.processes": validate.Optional(validate.IsInt64),

	"linux.kernel_modules": validate.IsAny,

	"migration.incremental.memory":            validate.IsBool,
	"migration.incremental.memory.iterations": validate.IsUint32,
	"migration.incremental.memory.goal":       validate.IsUint32,

	"nvidia.runtime":             validate.IsBool,
	"nvidia.driver.capabilities": validate.IsAny,
	"nvidia.require.cuda":        validate.IsAny,
	"nvidia.require.driver":      validate.IsAny,

	"security.nesting":       validate.IsBool,
	"security.privileged":    validate.IsBool,
	"security.devlxd":        validate.IsBool,
	"security.devlxd.images": validate.IsBool,

	"security.protection.delete": validate.IsBool,
	"security.protection.shift":  validate.IsBool,

	"security.idmap.base":     validate.IsUint32,
	"security.idmap.isolated": validate.IsBool,
	"security.idmap.size":     validate.IsUint32,

	"security.secureboot": validate.IsBool,

	"security.syscalls.allow":                   validate.IsAny,
	"security.syscalls.blacklist_default":       validate.IsBool,
	"security.syscalls.blacklist_compat":        validate.IsBool,
	"security.syscalls.blacklist":               validate.IsAny,
	"security.syscalls.deny_default":            validate.IsBool,
	"security.syscalls.deny_compat":             validate.IsBool,
	"security.syscalls.deny":                    validate.IsAny,
	"security.syscalls.intercept.mknod":         validate.IsBool,
	"security.syscalls.intercept.mount":         validate.IsBool,
	"security.syscalls.intercept.mount.allowed": validate.IsAny,
	"security.syscalls.intercept.mount.fuse":    validate.IsAny,
	"security.syscalls.intercept.mount.shift":   validate.IsBool,
	"security.syscalls.intercept.setxattr":      validate.IsBool,
	"security.syscalls.whitelist":               validate.IsAny,

	"snapshots.schedule": func(value string) error {
		if value == "" {
			return nil
		}

		if len(strings.Split(value, " ")) != 5 {
			return fmt.Errorf("Schedule must be of the form: <minute> <hour> <day-of-month> <month> <day-of-week>")
		}

		_, err := cron.Parse(fmt.Sprintf("* %s", value))
		if err != nil {
			return errors.Wrap(err, "Error parsing schedule")
		}

		return nil
	},
	"snapshots.schedule.stopped": validate.IsBool,
	"snapshots.pattern":          validate.IsAny,
	"snapshots.expiry": func(value string) error {
		// Validate expression
		_, err := GetSnapshotExpiry(time.Time{}, value)
		return err
	},

	// Caller is responsible for full validation of any raw.* value
	"raw.apparmor": validate.IsAny,
	"raw.idmap":    validate.IsAny,
	"raw.lxc":      validate.IsAny,
	"raw.qemu":     validate.IsAny,
	"raw.seccomp":  validate.IsAny,

	"volatile.apply_template":   validate.IsAny,
	"volatile.base_image":       validate.IsAny,
	"volatile.last_state.idmap": validate.IsAny,
	"volatile.last_state.power": validate.IsAny,
	"volatile.idmap.base":       validate.IsAny,
	"volatile.idmap.current":    validate.IsAny,
	"volatile.idmap.next":       validate.IsAny,
	"volatile.apply_quota":      validate.IsAny,
}

// ConfigKeyChecker returns a function that will check whether or not
// a provide value is valid for the associate config key.  Returns an
// error if the key is not known.  The checker function only performs
// syntactic checking of the value, semantic and usage checking must
// be done by the caller.  User defined keys are always considered to
// be valid, e.g. user.* and environment.* keys.
func ConfigKeyChecker(key string) (func(value string) error, error) {
	if f, ok := KnownInstanceConfigKeys[key]; ok {
		return f, nil
	}

	if strings.HasPrefix(key, "volatile.") {
		if strings.HasSuffix(key, ".hwaddr") {
			return validate.IsAny, nil
		}

		if strings.HasSuffix(key, ".name") {
			return validate.IsAny, nil
		}

		if strings.HasSuffix(key, ".host_name") {
			return validate.IsAny, nil
		}

		if strings.HasSuffix(key, ".mtu") {
			return validate.IsAny, nil
		}

		if strings.HasSuffix(key, ".created") {
			return validate.IsAny, nil
		}

		if strings.HasSuffix(key, ".id") {
			return validate.IsAny, nil
		}

		if strings.HasSuffix(key, ".vlan") {
			return validate.IsAny, nil
		}

		if strings.HasSuffix(key, ".spoofcheck") {
			return validate.IsAny, nil
		}

		if strings.HasSuffix(key, ".apply_quota") {
			return validate.IsAny, nil
		}

		if strings.HasSuffix(key, "vm.uuid") {
			return validate.IsAny, nil
		}

		if strings.HasSuffix(key, ".ceph_rbd") {
			return validate.IsAny, nil
		}

		if strings.HasSuffix(key, ".driver") {
			return validate.IsAny, nil
		}
	}

	if strings.HasPrefix(key, "environment.") {
		return validate.IsAny, nil
	}

	if strings.HasPrefix(key, "user.") {
		return validate.IsAny, nil
	}

	if strings.HasPrefix(key, "image.") {
		return validate.IsAny, nil
	}

	if strings.HasPrefix(key, "limits.kernel.") &&
		(len(key) > len("limits.kernel.")) {
		return validate.IsAny, nil
	}

	return nil, fmt.Errorf("Unknown configuration key: %s", key)
}

// InstanceGetParentAndSnapshotName returns the parent instance name, snapshot name,
// and whether it actually was a snapshot name.
func InstanceGetParentAndSnapshotName(name string) (string, string, bool) {
	fields := strings.SplitN(name, SnapshotDelimiter, 2)
	if len(fields) == 1 {
		return name, "", false
	}

	return fields[0], fields[1], true
}
<|endoftext|>


<commit_before>package hcl2template

import (
	"fmt"
	"os"
	"path/filepath"

	"github.com/hashicorp/hcl/v2"
	"github.com/hashicorp/hcl/v2/ext/dynblock"
	"github.com/hashicorp/hcl/v2/hclparse"
	"github.com/hashicorp/packer/packer"
)

const (
	sourceLabel       = "source"
	variablesLabel    = "variables"
	variableLabel     = "variable"
	localsLabel       = "locals"
	buildLabel        = "build"
	communicatorLabel = "communicator"
)

var configSchema = &hcl.BodySchema{
	Blocks: []hcl.BlockHeaderSchema{
		{Type: sourceLabel, LabelNames: []string{"type", "name"}},
		{Type: variablesLabel},
		{Type: variableLabel, LabelNames: []string{"name"}},
		{Type: localsLabel},
		{Type: buildLabel},
		{Type: communicatorLabel, LabelNames: []string{"type", "name"}},
	},
}

// Parser helps you parse HCL folders. It will parse an hcl file or directory
// and start builders, provisioners and post-processors to configure them with
// the parsed HCL and then return a []packer.Build. Packer will use that list
// of Builds to run everything in order.
type Parser struct {
	*hclparse.Parser

	BuilderSchemas packer.BuilderStore

	ProvisionersSchemas packer.ProvisionerStore

	PostProcessorsSchemas packer.PostProcessorStore
}

const (
	hcl2FileExt        = ".pkr.hcl"
	hcl2JsonFileExt    = ".pkr.json"
	hcl2VarFileExt     = ".auto.pkrvars.hcl"
	hcl2VarJsonFileExt = ".auto.pkrvars.json"
)

// Parse will Parse all HCL files in filename. Path can be a folder or a file.
//
// Parse will first Parse variables and then the rest; so that interpolation
// can happen.
//
// Parse returns a PackerConfig that contains configuration layout of a packer
// build; sources(builders)/provisioners/posts-processors will not be started
// and their contents wont be verified; Most syntax errors will cause an error.
func (p *Parser) Parse(filename string, varFiles []string, argVars map[string]string) (*PackerConfig, hcl.Diagnostics) {
	var files []*hcl.File
	var diags hcl.Diagnostics

	// parse config files
	if filename != "" {
		hclFiles, jsonFiles, moreDiags := GetHCL2Files(filename, hcl2FileExt, hcl2JsonFileExt)
		diags = append(diags, moreDiags...)
		if len(hclFiles)+len(jsonFiles) == 0 {
			diags = append(diags, &hcl.Diagnostic{
				Severity: hcl.DiagError,
				Summary:  "Could not find any config file in " + filename,
				Detail: "A config file must be suffixed with `.pkr.hcl` or " +
					"`.pkr.json`. A folder can be referenced.",
			})
		}
		for _, filename := range hclFiles {
			f, moreDiags := p.ParseHCLFile(filename)
			diags = append(diags, moreDiags...)
			files = append(files, f)
		}
		for _, filename := range jsonFiles {
			f, moreDiags := p.ParseJSONFile(filename)
			diags = append(diags, moreDiags...)
			files = append(files, f)
		}
		if diags.HasErrors() {
			return nil, diags
		}
	}

	basedir := filename
	if isDir, err := isDir(basedir); err == nil && !isDir {
		basedir = filepath.Dir(basedir)
	}
	wd, err := os.Getwd()
	if err != nil {
		diags = append(diags, &hcl.Diagnostic{
			Severity: hcl.DiagError,
			Summary:  "Could not find current working directory",
			Detail:   err.Error(),
		})
	}
	cfg := &PackerConfig{
		Basedir:               basedir,
		Cwd:                   wd,
		builderSchemas:        p.BuilderSchemas,
		provisionersSchemas:   p.ProvisionersSchemas,
		postProcessorsSchemas: p.PostProcessorsSchemas,
		parser:                p,
		files:                 files,
	}

	// Decode variable blocks so that they are available later on. Here locals
	// can use input variables so we decode them firsthand.
	{
		for _, file := range files {
			diags = append(diags, cfg.decodeInputVariables(file)...)
		}

		for _, file := range files {
			moreLocals, morediags := cfg.parseLocalVariables(file)
			diags = append(diags, morediags...)
			cfg.LocalBlocks = append(cfg.LocalBlocks, moreLocals...)
		}
	}

	// parse var files
	{
		hclVarFiles, jsonVarFiles, moreDiags := GetHCL2Files(filename, hcl2VarFileExt, hcl2VarJsonFileExt)
		diags = append(diags, moreDiags...)
		for _, file := range varFiles {
			switch filepath.Ext(file) {
			case ".hcl":
				hclVarFiles = append(hclVarFiles, file)
			case ".json":
				jsonVarFiles = append(jsonVarFiles, file)
			default:
				diags = append(moreDiags, &hcl.Diagnostic{
					Severity: hcl.DiagError,
					Summary:  "Could not guess format of " + file,
					Detail:   "A var file must be suffixed with `.hcl` or `.json`.",
				})
			}
		}
		var varFiles []*hcl.File
		for _, filename := range hclVarFiles {
			f, moreDiags := p.ParseHCLFile(filename)
			diags = append(diags, moreDiags...)
			if moreDiags.HasErrors() {
				continue
			}
			varFiles = append(varFiles, f)
		}
		for _, filename := range jsonVarFiles {
			f, moreDiags := p.ParseJSONFile(filename)
			diags = append(diags, moreDiags...)
			if moreDiags.HasErrors() {
				continue
			}
			varFiles = append(varFiles, f)
		}

		diags = append(diags, cfg.collectInputVariableValues(os.Environ(), varFiles, argVars)...)
	}
	return cfg, diags
}

func (cfg *PackerConfig) Initialize() hcl.Diagnostics {
	var diags hcl.Diagnostics

	_, moreDiags := cfg.InputVariables.Values()
	diags = append(diags, moreDiags...)
	_, moreDiags = cfg.LocalVariables.Values()
	diags = append(diags, moreDiags...)
	diags = append(diags, cfg.evaluateLocalVariables(cfg.LocalBlocks)...)

	// decode the actual content
	for _, file := range cfg.files {
		diags = append(diags, cfg.parser.decodeConfig(file, cfg)...)
	}

	return diags
}

// decodeConfig looks in the found blocks for everything that is not a variable
// block. It should be called after parsing input variables and locals so that
// they can be referenced.
func (p *Parser) decodeConfig(f *hcl.File, cfg *PackerConfig) hcl.Diagnostics {
	var diags hcl.Diagnostics

	body := dynblock.Expand(f.Body, cfg.EvalContext(nil))
	content, moreDiags := body.Content(configSchema)
	diags = append(diags, moreDiags...)

	for _, block := range content.Blocks {
		switch block.Type {
		case sourceLabel:
			source, moreDiags := p.decodeSource(block)
			diags = append(diags, moreDiags...)
			if moreDiags.HasErrors() {
				continue
			}

			ref := source.Ref()
			if existing, found := cfg.Sources[ref]; found {
				diags = append(diags, &hcl.Diagnostic{
					Severity: hcl.DiagError,
					Summary:  "Duplicate " + sourceLabel + " block",
					Detail: fmt.Sprintf("This "+sourceLabel+" block has the "+
						"same builder type and name as a previous block declared "+
						"at %s. Each "+sourceLabel+" must have a unique name per builder type.",
						existing.block.DefRange.Ptr()),
					Subject: source.block.DefRange.Ptr(),
				})
				continue
			}

			if cfg.Sources == nil {
				cfg.Sources = map[SourceRef]SourceBlock{}
			}
			cfg.Sources[ref] = source

		case buildLabel:
			build, moreDiags := p.decodeBuildConfig(block, cfg)
			diags = append(diags, moreDiags...)
			if moreDiags.HasErrors() {
				continue
			}
			cfg.Builds = append(cfg.Builds, build)

		}
	}

	return diags
}
<commit_msg>HCL2: pass sensitive variables to packer.LogSecretFilter<commit_after>package hcl2template

import (
	"fmt"
	"os"
	"path/filepath"

	"github.com/hashicorp/hcl/v2"
	"github.com/hashicorp/hcl/v2/ext/dynblock"
	"github.com/hashicorp/hcl/v2/hclparse"
	"github.com/hashicorp/packer/packer"
	"github.com/zclconf/go-cty/cty"
)

const (
	sourceLabel       = "source"
	variablesLabel    = "variables"
	variableLabel     = "variable"
	localsLabel       = "locals"
	buildLabel        = "build"
	communicatorLabel = "communicator"
)

var configSchema = &hcl.BodySchema{
	Blocks: []hcl.BlockHeaderSchema{
		{Type: sourceLabel, LabelNames: []string{"type", "name"}},
		{Type: variablesLabel},
		{Type: variableLabel, LabelNames: []string{"name"}},
		{Type: localsLabel},
		{Type: buildLabel},
		{Type: communicatorLabel, LabelNames: []string{"type", "name"}},
	},
}

// Parser helps you parse HCL folders. It will parse an hcl file or directory
// and start builders, provisioners and post-processors to configure them with
// the parsed HCL and then return a []packer.Build. Packer will use that list
// of Builds to run everything in order.
type Parser struct {
	*hclparse.Parser

	BuilderSchemas packer.BuilderStore

	ProvisionersSchemas packer.ProvisionerStore

	PostProcessorsSchemas packer.PostProcessorStore
}

const (
	hcl2FileExt        = ".pkr.hcl"
	hcl2JsonFileExt    = ".pkr.json"
	hcl2VarFileExt     = ".auto.pkrvars.hcl"
	hcl2VarJsonFileExt = ".auto.pkrvars.json"
)

// Parse will Parse all HCL files in filename. Path can be a folder or a file.
//
// Parse will first Parse variables and then the rest; so that interpolation
// can happen.
//
// Parse returns a PackerConfig that contains configuration layout of a packer
// build; sources(builders)/provisioners/posts-processors will not be started
// and their contents wont be verified; Most syntax errors will cause an error.
func (p *Parser) Parse(filename string, varFiles []string, argVars map[string]string) (*PackerConfig, hcl.Diagnostics) {
	var files []*hcl.File
	var diags hcl.Diagnostics

	// parse config files
	if filename != "" {
		hclFiles, jsonFiles, moreDiags := GetHCL2Files(filename, hcl2FileExt, hcl2JsonFileExt)
		diags = append(diags, moreDiags...)
		if len(hclFiles)+len(jsonFiles) == 0 {
			diags = append(diags, &hcl.Diagnostic{
				Severity: hcl.DiagError,
				Summary:  "Could not find any config file in " + filename,
				Detail: "A config file must be suffixed with `.pkr.hcl` or " +
					"`.pkr.json`. A folder can be referenced.",
			})
		}
		for _, filename := range hclFiles {
			f, moreDiags := p.ParseHCLFile(filename)
			diags = append(diags, moreDiags...)
			files = append(files, f)
		}
		for _, filename := range jsonFiles {
			f, moreDiags := p.ParseJSONFile(filename)
			diags = append(diags, moreDiags...)
			files = append(files, f)
		}
		if diags.HasErrors() {
			return nil, diags
		}
	}

	basedir := filename
	if isDir, err := isDir(basedir); err == nil && !isDir {
		basedir = filepath.Dir(basedir)
	}
	wd, err := os.Getwd()
	if err != nil {
		diags = append(diags, &hcl.Diagnostic{
			Severity: hcl.DiagError,
			Summary:  "Could not find current working directory",
			Detail:   err.Error(),
		})
	}
	cfg := &PackerConfig{
		Basedir:               basedir,
		Cwd:                   wd,
		builderSchemas:        p.BuilderSchemas,
		provisionersSchemas:   p.ProvisionersSchemas,
		postProcessorsSchemas: p.PostProcessorsSchemas,
		parser:                p,
		files:                 files,
	}

	// Decode variable blocks so that they are available later on. Here locals
	// can use input variables so we decode them firsthand.
	{
		for _, file := range files {
			diags = append(diags, cfg.decodeInputVariables(file)...)
		}

		for _, file := range files {
			moreLocals, morediags := cfg.parseLocalVariables(file)
			diags = append(diags, morediags...)
			cfg.LocalBlocks = append(cfg.LocalBlocks, moreLocals...)
		}
	}

	// parse var files
	{
		hclVarFiles, jsonVarFiles, moreDiags := GetHCL2Files(filename, hcl2VarFileExt, hcl2VarJsonFileExt)
		diags = append(diags, moreDiags...)
		for _, file := range varFiles {
			switch filepath.Ext(file) {
			case ".hcl":
				hclVarFiles = append(hclVarFiles, file)
			case ".json":
				jsonVarFiles = append(jsonVarFiles, file)
			default:
				diags = append(moreDiags, &hcl.Diagnostic{
					Severity: hcl.DiagError,
					Summary:  "Could not guess format of " + file,
					Detail:   "A var file must be suffixed with `.hcl` or `.json`.",
				})
			}
		}
		var varFiles []*hcl.File
		for _, filename := range hclVarFiles {
			f, moreDiags := p.ParseHCLFile(filename)
			diags = append(diags, moreDiags...)
			if moreDiags.HasErrors() {
				continue
			}
			varFiles = append(varFiles, f)
		}
		for _, filename := range jsonVarFiles {
			f, moreDiags := p.ParseJSONFile(filename)
			diags = append(diags, moreDiags...)
			if moreDiags.HasErrors() {
				continue
			}
			varFiles = append(varFiles, f)
		}

		diags = append(diags, cfg.collectInputVariableValues(os.Environ(), varFiles, argVars)...)
	}
	return cfg, diags
}

func (cfg *PackerConfig) Initialize() hcl.Diagnostics {
	var diags hcl.Diagnostics

	_, moreDiags := cfg.InputVariables.Values()
	diags = append(diags, moreDiags...)
	_, moreDiags = cfg.LocalVariables.Values()
	diags = append(diags, moreDiags...)
	diags = append(diags, cfg.evaluateLocalVariables(cfg.LocalBlocks)...)

	for _, variable := range cfg.InputVariables {
		if !variable.Sensitive {
			continue
		}
		value, _ := variable.Value()
		if !value.IsWhollyKnown() && value.IsNull() && !value.Type().Equals(cty.String) {
			continue
		}
		cty.Walk(value, func(_ cty.Path, nested cty.Value) (bool, error) {
			if nested.IsWhollyKnown() && !nested.IsNull() && nested.Type().Equals(cty.String) {
				packer.LogSecretFilter.Set(nested.AsString())
			}
			return true, nil
		})
	}

	// decode the actual content
	for _, file := range cfg.files {
		diags = append(diags, cfg.parser.decodeConfig(file, cfg)...)
	}

	return diags
}

// decodeConfig looks in the found blocks for everything that is not a variable
// block. It should be called after parsing input variables and locals so that
// they can be referenced.
func (p *Parser) decodeConfig(f *hcl.File, cfg *PackerConfig) hcl.Diagnostics {
	var diags hcl.Diagnostics

	body := dynblock.Expand(f.Body, cfg.EvalContext(nil))
	content, moreDiags := body.Content(configSchema)
	diags = append(diags, moreDiags...)

	for _, block := range content.Blocks {
		switch block.Type {
		case sourceLabel:
			source, moreDiags := p.decodeSource(block)
			diags = append(diags, moreDiags...)
			if moreDiags.HasErrors() {
				continue
			}

			ref := source.Ref()
			if existing, found := cfg.Sources[ref]; found {
				diags = append(diags, &hcl.Diagnostic{
					Severity: hcl.DiagError,
					Summary:  "Duplicate " + sourceLabel + " block",
					Detail: fmt.Sprintf("This "+sourceLabel+" block has the "+
						"same builder type and name as a previous block declared "+
						"at %s. Each "+sourceLabel+" must have a unique name per builder type.",
						existing.block.DefRange.Ptr()),
					Subject: source.block.DefRange.Ptr(),
				})
				continue
			}

			if cfg.Sources == nil {
				cfg.Sources = map[SourceRef]SourceBlock{}
			}
			cfg.Sources[ref] = source

		case buildLabel:
			build, moreDiags := p.decodeBuildConfig(block, cfg)
			diags = append(diags, moreDiags...)
			if moreDiags.HasErrors() {
				continue
			}
			cfg.Builds = append(cfg.Builds, build)

		}
	}

	return diags
}
<|endoftext|>


<commit_before>package kvfs

import (
	"io"
	"net/url"
	"os"
	"path/filepath"

	"go.polydawn.net/rio/fs"
	. "go.polydawn.net/rio/lib/errcat"
	"go.polydawn.net/rio/lib/guid"
	"go.polydawn.net/rio/warehouse"
	"go.polydawn.net/rio/warehouse/util"
	"go.polydawn.net/timeless-api"
	"go.polydawn.net/timeless-api/rio"
)

var (
	_ warehouse.BlobstoreController      = Controller{}
	_ warehouse.BlobstoreWriteController = &WriteController{}
)

type Controller struct {
	addr     api.WarehouseAddr // user's string retained for messages
	basePath fs.AbsolutePath
	ctntAddr bool
}

/*
	Initialize a new warehouse controller that operates on a local filesystem.

	May return errors of category:

	  - `rio.ErrUsage` -- for unsupported addressses
	  - `rio.ErrWarehouseUnavailable` -- if the warehouse doesn't exist
*/
func NewController(addr api.WarehouseAddr) (warehouse.BlobstoreController, error) {
	// Stamp out a warehouse handle.
	//  More values will be accumulated in shortly.
	whCtrl := Controller{
		addr: addr,
	}

	// Verify that the addr is sensible up front, and extract features.
	//  - We parse things mostly like URLs.
	//  - We extract whether or not it's content-addressible mode here;
	//  - and extract the filesystem path, and normalize it to its absolute form.
	u, err := url.Parse(string(addr))
	if err != nil {
		return whCtrl, Errorf(rio.ErrUsage, "failed to parse URI: %s", err)
	}
	switch u.Scheme {
	case "file":
	case "file+ca":
		whCtrl.ctntAddr = true
	default:
		return whCtrl, Errorf(rio.ErrUsage, "unsupported scheme in warehouse addr: %q (valid options are 'file' or 'file+ca')", u.Scheme)
	}
	absPth, err := filepath.Abs(filepath.Join(u.Host, u.Path))
	if err != nil {
		panic(err)
	}
	whCtrl.basePath = fs.MustAbsolutePath(absPth)

	// Check that the warehouse exists.
	//  If it does, we're good: return happily.
	checkPath := whCtrl.basePath
	if !whCtrl.ctntAddr {
		// In non-CA mode, the check for warehouse existence is a little strange;
		//  for reading, we could declare 404 if the path doesn't exist... but we don't
		//  know whether this is going to be used for reading or writing yet!
		//  So we have to look at the path segment above it to see if a write might be valid.
		checkPath = checkPath.Dir()
	}
	stat, err := os.Stat(checkPath.String())
	switch {
	case os.IsNotExist(err):
		return whCtrl, Errorf(rio.ErrWarehouseUnavailable, "warehouse does not exist (%s)", err)
	case err != nil: // normally we'd style this as the default cause, but, we must check it before the IsDir option
		return whCtrl, Errorf(rio.ErrWarehouseUnavailable, "warehouse unavailable (%s)", err)
	case !stat.IsDir():
		return whCtrl, Errorf(rio.ErrWarehouseUnavailable, "warehouse does not exist (%s is not a dir)", checkPath)
	default: // only thing left is err == nil
		return whCtrl, nil
	}
}

func (whCtrl Controller) OpenReader(wareID api.WareID) (io.ReadCloser, error) {
	finalPath := whCtrl.basePath
	if whCtrl.ctntAddr {
		chunkA, chunkB, _ := util.ChunkifyHash(wareID)
		finalPath = finalPath.
			Join(fs.MustRelPath(chunkA)).
			Join(fs.MustRelPath(chunkB)).
			Join(fs.MustRelPath(wareID.Hash))
	}
	file, err := os.OpenFile(finalPath.String(), os.O_RDONLY, 0)
	switch {
	case err == nil:
		return file, nil
	case os.IsNotExist(err):
		return file, Errorf(rio.ErrWareNotFound, "ware %s not found in warehouse %s", wareID, whCtrl.addr)
	default:
		return file, Errorf(rio.ErrWarehouseUnavailable, "ware %s could not be retrieved from warehouse %s: %s", wareID, whCtrl.addr, err)
	}
}

func (whCtrl Controller) OpenWriter() (warehouse.BlobstoreWriteController, error) {
	wc := &WriteController{whCtrl: whCtrl}
	// Pick a random upload path.
	if whCtrl.ctntAddr {
		tmpName := fs.MustRelPath(".tmp.upload." + guid.New())
		wc.stagePath = whCtrl.basePath.Join(tmpName)
	} else {
		// In non-CA mode, "base" path isn't really "base"; it's the final destination.
		tmpName := fs.MustRelPath(".tmp.upload." + whCtrl.basePath.Last() + "." + guid.New())
		wc.stagePath = whCtrl.basePath.Dir().Join(tmpName)
	}
	// Open file the file for write.
	file, err := os.OpenFile(wc.stagePath.String(), os.O_CREATE|os.O_WRONLY|os.O_EXCL, 0644)
	if err != nil {
		return wc, Errorf(rio.ErrWarehouseUnwritable, "failed to reserve temp space in warehouse: %s", err)
	}
	wc.stream = file
	// Return the controller -- which has methods to either commit+close, or cancel+close.
	return wc, nil
}

type WriteController struct {
	stream    io.WriteCloser  // Write to this.
	whCtrl    Controller      // Needed for the final move-into-place.
	stagePath fs.AbsolutePath // Needed for the final move-into-place.
}

func (wc *WriteController) Write(bs []byte) (int, error) {
	return wc.stream.Write(bs)
}

/*
	Cancel the current write.  Close the stream, and remove any temporary files.
*/
func (wc *WriteController) Close() error {
	wc.stream.Close()
	return os.Remove(wc.stagePath.String())
}

/*
	Commit the current data as the given hash.
	Caller must be an adult and specify the hash truthfully.
	Closes the writer and invalidates any future use.
*/
func (wc *WriteController) Commit(wareID api.WareID) error {
	// Close the file.
	if err := wc.stream.Close(); err != nil {
		return Errorf(rio.ErrWarehouseUnwritable, "failed to commit to file: %s", err)
	}
	// Compute final path.
	// Make parent dirs if necessary in content-addr mode.
	finalPath := wc.whCtrl.basePath
	if wc.whCtrl.ctntAddr {
		chunkA, chunkB, _ := util.ChunkifyHash(wareID)
		finalPath = finalPath.Join(fs.MustRelPath(chunkA))
		if err := os.Mkdir(finalPath.String(), 0755); err != nil {
			return Errorf(rio.ErrWarehouseUnwritable, "failed to commit to file: %s", err)
		}
		finalPath = finalPath.Join(fs.MustRelPath(chunkB))
		if err := os.Mkdir(finalPath.String(), 0755); err != nil {
			return Errorf(rio.ErrWarehouseUnwritable, "failed to commit to file: %s", err)
		}
		finalPath = finalPath.Join(fs.MustRelPath(wareID.Hash))
	}
	// Move into place.
	if err := os.Rename(wc.stagePath.String(), finalPath.String()); err != nil {
		return Errorf(rio.ErrWarehouseUnwritable, "failed to commit to file: %s", err)
	}
	return nil
}
<commit_msg>kvfs warehouse: fix chunk dirs already existing being treated as error.<commit_after>package kvfs

import (
	"io"
	"net/url"
	"os"
	"path/filepath"

	"go.polydawn.net/rio/fs"
	. "go.polydawn.net/rio/lib/errcat"
	"go.polydawn.net/rio/lib/guid"
	"go.polydawn.net/rio/warehouse"
	"go.polydawn.net/rio/warehouse/util"
	"go.polydawn.net/timeless-api"
	"go.polydawn.net/timeless-api/rio"
)

var (
	_ warehouse.BlobstoreController      = Controller{}
	_ warehouse.BlobstoreWriteController = &WriteController{}
)

type Controller struct {
	addr     api.WarehouseAddr // user's string retained for messages
	basePath fs.AbsolutePath
	ctntAddr bool
}

/*
	Initialize a new warehouse controller that operates on a local filesystem.

	May return errors of category:

	  - `rio.ErrUsage` -- for unsupported addressses
	  - `rio.ErrWarehouseUnavailable` -- if the warehouse doesn't exist
*/
func NewController(addr api.WarehouseAddr) (warehouse.BlobstoreController, error) {
	// Stamp out a warehouse handle.
	//  More values will be accumulated in shortly.
	whCtrl := Controller{
		addr: addr,
	}

	// Verify that the addr is sensible up front, and extract features.
	//  - We parse things mostly like URLs.
	//  - We extract whether or not it's content-addressible mode here;
	//  - and extract the filesystem path, and normalize it to its absolute form.
	u, err := url.Parse(string(addr))
	if err != nil {
		return whCtrl, Errorf(rio.ErrUsage, "failed to parse URI: %s", err)
	}
	switch u.Scheme {
	case "file":
	case "file+ca":
		whCtrl.ctntAddr = true
	default:
		return whCtrl, Errorf(rio.ErrUsage, "unsupported scheme in warehouse addr: %q (valid options are 'file' or 'file+ca')", u.Scheme)
	}
	absPth, err := filepath.Abs(filepath.Join(u.Host, u.Path))
	if err != nil {
		panic(err)
	}
	whCtrl.basePath = fs.MustAbsolutePath(absPth)

	// Check that the warehouse exists.
	//  If it does, we're good: return happily.
	checkPath := whCtrl.basePath
	if !whCtrl.ctntAddr {
		// In non-CA mode, the check for warehouse existence is a little strange;
		//  for reading, we could declare 404 if the path doesn't exist... but we don't
		//  know whether this is going to be used for reading or writing yet!
		//  So we have to look at the path segment above it to see if a write might be valid.
		checkPath = checkPath.Dir()
	}
	stat, err := os.Stat(checkPath.String())
	switch {
	case os.IsNotExist(err):
		return whCtrl, Errorf(rio.ErrWarehouseUnavailable, "warehouse does not exist (%s)", err)
	case err != nil: // normally we'd style this as the default cause, but, we must check it before the IsDir option
		return whCtrl, Errorf(rio.ErrWarehouseUnavailable, "warehouse unavailable (%s)", err)
	case !stat.IsDir():
		return whCtrl, Errorf(rio.ErrWarehouseUnavailable, "warehouse does not exist (%s is not a dir)", checkPath)
	default: // only thing left is err == nil
		return whCtrl, nil
	}
}

func (whCtrl Controller) OpenReader(wareID api.WareID) (io.ReadCloser, error) {
	finalPath := whCtrl.basePath
	if whCtrl.ctntAddr {
		chunkA, chunkB, _ := util.ChunkifyHash(wareID)
		finalPath = finalPath.
			Join(fs.MustRelPath(chunkA)).
			Join(fs.MustRelPath(chunkB)).
			Join(fs.MustRelPath(wareID.Hash))
	}
	file, err := os.OpenFile(finalPath.String(), os.O_RDONLY, 0)
	switch {
	case err == nil:
		return file, nil
	case os.IsNotExist(err):
		return file, Errorf(rio.ErrWareNotFound, "ware %s not found in warehouse %s", wareID, whCtrl.addr)
	default:
		return file, Errorf(rio.ErrWarehouseUnavailable, "ware %s could not be retrieved from warehouse %s: %s", wareID, whCtrl.addr, err)
	}
}

func (whCtrl Controller) OpenWriter() (warehouse.BlobstoreWriteController, error) {
	wc := &WriteController{whCtrl: whCtrl}
	// Pick a random upload path.
	if whCtrl.ctntAddr {
		tmpName := fs.MustRelPath(".tmp.upload." + guid.New())
		wc.stagePath = whCtrl.basePath.Join(tmpName)
	} else {
		// In non-CA mode, "base" path isn't really "base"; it's the final destination.
		tmpName := fs.MustRelPath(".tmp.upload." + whCtrl.basePath.Last() + "." + guid.New())
		wc.stagePath = whCtrl.basePath.Dir().Join(tmpName)
	}
	// Open file the file for write.
	file, err := os.OpenFile(wc.stagePath.String(), os.O_CREATE|os.O_WRONLY|os.O_EXCL, 0644)
	if err != nil {
		return wc, Errorf(rio.ErrWarehouseUnwritable, "failed to reserve temp space in warehouse: %s", err)
	}
	wc.stream = file
	// Return the controller -- which has methods to either commit+close, or cancel+close.
	return wc, nil
}

type WriteController struct {
	stream    io.WriteCloser  // Write to this.
	whCtrl    Controller      // Needed for the final move-into-place.
	stagePath fs.AbsolutePath // Needed for the final move-into-place.
}

func (wc *WriteController) Write(bs []byte) (int, error) {
	return wc.stream.Write(bs)
}

/*
	Cancel the current write.  Close the stream, and remove any temporary files.
*/
func (wc *WriteController) Close() error {
	wc.stream.Close()
	return os.Remove(wc.stagePath.String())
}

/*
	Commit the current data as the given hash.
	Caller must be an adult and specify the hash truthfully.
	Closes the writer and invalidates any future use.
*/
func (wc *WriteController) Commit(wareID api.WareID) error {
	// Close the file.
	if err := wc.stream.Close(); err != nil {
		return Errorf(rio.ErrWarehouseUnwritable, "failed to commit to file: %s", err)
	}
	// Compute final path.
	// Make parent dirs if necessary in content-addr mode.
	finalPath := wc.whCtrl.basePath
	if wc.whCtrl.ctntAddr {
		chunkA, chunkB, _ := util.ChunkifyHash(wareID)
		finalPath = finalPath.Join(fs.MustRelPath(chunkA))
		if err := os.Mkdir(finalPath.String(), 0755); err != nil && !os.IsExist(err) {
			return Errorf(rio.ErrWarehouseUnwritable, "failed to commit to file: %s", err)
		}
		finalPath = finalPath.Join(fs.MustRelPath(chunkB))
		if err := os.Mkdir(finalPath.String(), 0755); err != nil && !os.IsExist(err) {
			return Errorf(rio.ErrWarehouseUnwritable, "failed to commit to file: %s", err)
		}
		finalPath = finalPath.Join(fs.MustRelPath(wareID.Hash))
	}
	// Move into place.
	if err := os.Rename(wc.stagePath.String(), finalPath.String()); err != nil {
		return Errorf(rio.ErrWarehouseUnwritable, "failed to commit to file: %s", err)
	}
	return nil
}
<|endoftext|>


<commit_before>package messages

import (
	"database/sql"
	"errors"
	"fmt"
	"log"
	"strconv"
	"strings"

	"github.com/oipwg/media-protocol/utility"
)

type MediaMultipartSingle struct {
	Part      int
	Max       int
	Reference string
	Address   string
	Signature string
	Data      string
	Txid      string
	Block     int
}

func CheckMediaMultipartComplete(reference string, dbtx *sql.Tx) ([]byte, error) {
	// using the reference tx, check how many different txs we have and determine if we have all transactions
	// if we have a valid media-multipart complete instance, let's return the byte array it consists of
	var ret []byte

	stmtstr := `select part, max, data from media_multipart where active = 1 and complete = 0 and reference = "` + reference + `" order by part asc`

	stmt, err := dbtx.Prepare(stmtstr)
	if err != nil {
		fmt.Println("exit 120")
		log.Fatal(err)
	}

	rows, stmterr := stmt.Query()
	if err != nil {
		fmt.Println("exit 121")
		log.Fatal(stmterr)
	}

	var rowsCount int = 0
	var pmax int
	var fullData string

	for rows.Next() {
		var part int
		var max int
		var data string
		rows.Scan(&part, &max, &data)

		// TODO: require signature verification for multipart messages
		if rowsCount > max {
			return ret, errors.New("too many rows in multipart message - check for reorg/bogus multipart data")
		}
		rowsCount++

		pmax = max
		fullData += data
	}

	if rowsCount != pmax+1 {
		return ret, errors.New("only found " + strconv.Itoa(rowsCount) + "/" + strconv.Itoa(pmax+1) + " multipart messages")
	}

	stmt.Close()
	rows.Close()

	// set complete to 1
	updatestr := `update media_multipart set complete = 1 where reference = "` + reference + `"`
	updatestmt, updateerr := dbtx.Prepare(updatestr)
	if updateerr != nil {
		fmt.Println("exit 122")
		log.Fatal(updateerr)
	}

	_, updatestmterr := updatestmt.Exec()
	if updatestmterr != nil {
		fmt.Println("exit 123")
		log.Fatal(updatestmterr)
	}
	updatestmt.Close()

	return []byte(fullData), nil
}

func StoreMediaMultipartSingle(mms MediaMultipartSingle, dbtx *sql.Tx) {
	// store in database
	stmtstr := `insert into media_multipart (part, max, address, reference, signature, data, txid, block, complete, success, active) values (` + strconv.Itoa(mms.Part) + `, ` + strconv.Itoa(mms.Max) + `, ?, ?, ?, ?, "` + mms.Txid + `", ` + strconv.Itoa(mms.Block) + `, 0, 0, 1)`

	stmt, err := dbtx.Prepare(stmtstr)
	if err != nil {
		fmt.Println("exit 160")
		log.Fatal(err)
	}

	_, stmterr := stmt.Exec(mms.Address, mms.Reference, mms.Signature, mms.Data)
	if stmterr != nil {
		fmt.Println("exit 161")
		log.Fatal(stmterr)
	}

	stmt.Close()

}

func UpdateMediaMultipartSuccess(reference string, dbtx *sql.Tx) {

	stmtstr := `update media_multipart set success = 1 where reference = "` + reference + `"`

	stmt, err := dbtx.Prepare(stmtstr)
	if err != nil {
		fmt.Println("exit 140")
		log.Fatal(err)
	}

	_, stmterr := stmt.Exec()
	if err != nil {
		fmt.Println("exit 141")
		log.Fatal(stmterr)
	}

}

func VerifyMediaMultipartSingle(s string, txid string, block int) (MediaMultipartSingle, error) {
	var ret MediaMultipartSingle
	prefix := "alexandria-media-multipart("
	prefix2 := "oip-mp("

	// check prefix
	checkPrefix := strings.HasPrefix(s, prefix)
	checkPrefix = checkPrefix || strings.HasPrefix(s, prefix2)
	if !checkPrefix {
		return ret, ErrWrongPrefix
	}

	// trim prefix off
	s = strings.TrimPrefix(s, prefix)
	s = strings.TrimPrefix(s, prefix2)

	comChunks := strings.Split(s, "):")
	if len(comChunks) < 2 {
		return ret, errors.New("Malformed multi-part")
	}

	metaString := comChunks[0]
	dataString := strings.Join(comChunks[1:], "):")

	meta := strings.Split(metaString, ",")
	lm := len(meta)
	// 4 if omitting reference, 5 with all fields, 6 if erroneous fluffy-enigma trailing comma
	if lm != 4 && lm != 5 && lm != 6 {
		return ret, errors.New("Malformed multi-part meta")
	}

	// check part and max
	partS := meta[0]
	part, err := strconv.Atoi(partS)
	if err != nil {
		fmt.Println("cannot convert part to int")
		return ret, errors.New("cannot convert part to int")
	}
	maxS := meta[1]
	max, err2 := strconv.Atoi(maxS)
	if err2 != nil {
		fmt.Println("cannot convert max to int")
		return ret, errors.New("cannot convert max to int")
	}

	// get and check address
	address := meta[2]
	if !utility.CheckAddress(address) {
		// fmt.Println("address doesn't check out: \"" + address + "\"")
		return ret, ErrInvalidAddress
	}

	reference := meta[3]
	signature := meta[lm-1]
	if signature == "" {
		// fluffy-enigma for a while appended an erroneous trailing comma
		signature = meta[lm-2]
	}

	// signature pre-image is <part>-<max>-<address>-<txid>-<data>
	// in the case of multipart[0], txid is 64 zeros
	// in the case of multipart[n], where n != 0, txid is the reference txid (from multipart[0])
	preimage := partS + "-" + maxS + "-" + address + "-" + reference + "-" + dataString
	// fmt.Printf("preimage: %v\n", preimage)

	val, _ := utility.CheckSignature(address, signature, preimage)
	if !val {
		// fmt.Println("signature didn't pass checksignature test")
		return ret, ErrBadSignature
	}

	//fmt.Printf("DEBUG: part = %v, reference = %v\n", partS, reference)

	// if part == 0, reference is ignored and may be omitted
	// the local DB will store reference = txid for this transaction after it's submitted
	// in case of a reorg, the publisher must re-publish this multipart message (sorry)
	if part == 0 {
		reference = txid

		// hard-fork 2017-08-25 for shorter txid reference ids
		if block >= 2280000 {
			reference = txid[:10]
		}
	}

	// all checks passed, verified!

	//fmt.Printf("data: %v\n", data)
	// fmt.Printf("=== VERIFIED ===\n")
	//fmt.Printf("part: %v\nmax: %v\nreference: %v\naddress: %v\nsignature: %v\ntxid: %v\nblock: %v\n", part, max, reference, address, signature, txid, block)

	ret = MediaMultipartSingle{
		Part:      part,
		Max:       max,
		Reference: reference,
		Address:   address,
		Signature: signature,
		Data:      dataString,
		Txid:      txid,
		Block:     block,
	}

	return ret, nil

}
<commit_msg>Store full txid rather than short form<commit_after>package messages

import (
	"database/sql"
	"errors"
	"fmt"
	"log"
	"strconv"
	"strings"

	"github.com/oipwg/media-protocol/utility"
)

type MediaMultipartSingle struct {
	Part      int
	Max       int
	Reference string
	Address   string
	Signature string
	Data      string
	Txid      string
	Block     int
}

func CheckMediaMultipartComplete(reference string, dbtx *sql.Tx) ([]byte, string, error) {
	// using the reference tx, check how many different txs we have and determine if we have all transactions
	// if we have a valid media-multipart complete instance, let's return the byte array it consists of
	var ret []byte
	var ref string

	if len(reference) < 10 {
		return ret, ref, errors.New("multipart reference must be at least 10 characters")
	}

	stmtstr := `select part, max, data, txid from media_multipart where active = 1 and complete = 0 and reference like "` + reference + `%" order by part asc`

	stmt, err := dbtx.Prepare(stmtstr)
	if err != nil {
		fmt.Println("exit 120")
		log.Fatal(err)
	}

	rows, stmterr := stmt.Query()
	if err != nil {
		fmt.Println("exit 121")
		log.Fatal(stmterr)
	}

	var rowsCount int = 0
	var pmax int
	var fullData string

	for rows.Next() {
		var part int
		var max int
		var data string
		var txid string
		rows.Scan(&part, &max, &data, &txid)

		if part == 0 {
			ref = txid
		}

		// TODO: require signature verification for multipart messages
		if rowsCount > max {
			return ret, ref, errors.New("too many rows in multipart message - check for reorg/bogus multipart data")
		}
		rowsCount++

		pmax = max
		fullData += data
	}

	if rowsCount != pmax+1 {
		return ret, ref, errors.New("only found " + strconv.Itoa(rowsCount) + "/" + strconv.Itoa(pmax+1) + " multipart messages")
	}

	stmt.Close()
	rows.Close()

	// set complete to 1
	updatestr := `update media_multipart set complete = 1 where reference = "` + reference + `"`
	updatestmt, updateerr := dbtx.Prepare(updatestr)
	if updateerr != nil {
		fmt.Println("exit 122")
		log.Fatal(updateerr)
	}

	_, updatestmterr := updatestmt.Exec()
	if updatestmterr != nil {
		fmt.Println("exit 123")
		log.Fatal(updatestmterr)
	}
	updatestmt.Close()

	return []byte(fullData), ref, nil
}

func StoreMediaMultipartSingle(mms MediaMultipartSingle, dbtx *sql.Tx) {
	// store in database
	stmtstr := `insert into media_multipart (part, max, address, reference, signature, data, txid, block, complete, success, active) values (` + strconv.Itoa(mms.Part) + `, ` + strconv.Itoa(mms.Max) + `, ?, ?, ?, ?, "` + mms.Txid + `", ` + strconv.Itoa(mms.Block) + `, 0, 0, 1)`

	stmt, err := dbtx.Prepare(stmtstr)
	if err != nil {
		fmt.Println("exit 160")
		log.Fatal(err)
	}

	_, stmterr := stmt.Exec(mms.Address, mms.Reference, mms.Signature, mms.Data)
	if stmterr != nil {
		fmt.Println("exit 161")
		log.Fatal(stmterr)
	}

	stmt.Close()

}

func UpdateMediaMultipartSuccess(reference string, dbtx *sql.Tx) {

	stmtstr := `update media_multipart set success = 1 where reference = "` + reference + `"`

	stmt, err := dbtx.Prepare(stmtstr)
	if err != nil {
		fmt.Println("exit 140")
		log.Fatal(err)
	}

	_, stmterr := stmt.Exec()
	if err != nil {
		fmt.Println("exit 141")
		log.Fatal(stmterr)
	}

}

func VerifyMediaMultipartSingle(s string, txid string, block int) (MediaMultipartSingle, error) {
	var ret MediaMultipartSingle
	prefix := "alexandria-media-multipart("
	prefix2 := "oip-mp("

	// check prefix
	checkPrefix := strings.HasPrefix(s, prefix)
	checkPrefix = checkPrefix || strings.HasPrefix(s, prefix2)
	if !checkPrefix {
		return ret, ErrWrongPrefix
	}

	// trim prefix off
	s = strings.TrimPrefix(s, prefix)
	s = strings.TrimPrefix(s, prefix2)

	comChunks := strings.Split(s, "):")
	if len(comChunks) < 2 {
		return ret, errors.New("Malformed multi-part")
	}

	metaString := comChunks[0]
	dataString := strings.Join(comChunks[1:], "):")

	meta := strings.Split(metaString, ",")
	lm := len(meta)
	// 4 if omitting reference, 5 with all fields, 6 if erroneous fluffy-enigma trailing comma
	if lm != 4 && lm != 5 && lm != 6 {
		return ret, errors.New("Malformed multi-part meta")
	}

	// check part and max
	partS := meta[0]
	part, err := strconv.Atoi(partS)
	if err != nil {
		fmt.Println("cannot convert part to int")
		return ret, errors.New("cannot convert part to int")
	}
	maxS := meta[1]
	max, err2 := strconv.Atoi(maxS)
	if err2 != nil {
		fmt.Println("cannot convert max to int")
		return ret, errors.New("cannot convert max to int")
	}

	// get and check address
	address := meta[2]
	if !utility.CheckAddress(address) {
		// fmt.Println("address doesn't check out: \"" + address + "\"")
		return ret, ErrInvalidAddress
	}

	reference := meta[3]
	signature := meta[lm-1]
	if signature == "" {
		// fluffy-enigma for a while appended an erroneous trailing comma
		signature = meta[lm-2]
	}

	// signature pre-image is <part>-<max>-<address>-<txid>-<data>
	// in the case of multipart[0], txid is 64 zeros
	// in the case of multipart[n], where n != 0, txid is the reference txid (from multipart[0])
	preimage := partS + "-" + maxS + "-" + address + "-" + reference + "-" + dataString
	// fmt.Printf("preimage: %v\n", preimage)

	val, _ := utility.CheckSignature(address, signature, preimage)
	if !val {
		// fmt.Println("signature didn't pass checksignature test")
		return ret, ErrBadSignature
	}

	//fmt.Printf("DEBUG: part = %v, reference = %v\n", partS, reference)

	// if part == 0, reference is ignored and may be omitted
	// the local DB will store reference = txid for this transaction after it's submitted
	// in case of a reorg, the publisher must re-publish this multipart message (sorry)
	if part == 0 {
		reference = txid

		//// hard-fork 2017-08-25 for shorter txid reference ids
		//if block >= 2280000 {
		//	reference = txid[:10]
		//}
	}

	// all checks passed, verified!

	//fmt.Printf("data: %v\n", data)
	// fmt.Printf("=== VERIFIED ===\n")
	//fmt.Printf("part: %v\nmax: %v\nreference: %v\naddress: %v\nsignature: %v\ntxid: %v\nblock: %v\n", part, max, reference, address, signature, txid, block)

	ret = MediaMultipartSingle{
		Part:      part,
		Max:       max,
		Reference: reference,
		Address:   address,
		Signature: signature,
		Data:      dataString,
		Txid:      txid,
		Block:     block,
	}

	return ret, nil

}
<|endoftext|>