<commit_before>// Copyright 2013 tsuru authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package app

import (
	"fmt"
	"strings"
	"sync"
	"sync/atomic"
	"time"

	"github.com/pkg/errors"
	"github.com/prometheus/client_golang/prometheus"
	"github.com/tsuru/tsuru/api/shutdown"
	"github.com/tsuru/tsuru/db"
	"github.com/tsuru/tsuru/log"
	"gopkg.in/mgo.v2"
	"gopkg.in/mgo.v2/bson"
)

var (
	bulkMaxWaitMongoTime = 1 * time.Second
	bulkMaxNumberMsgs    = 1000
	bulkQueueMaxSize     = 10000

	buckets = append([]float64{0.1, 0.5}, prometheus.ExponentialBuckets(1, 1.6, 15)...)

	logsInQueue = prometheus.NewGauge(prometheus.GaugeOpts{
		Name: "tsuru_logs_queue_current",
		Help: "The current number of log entries in dispatcher queue.",
	})

	logsInAppQueues = prometheus.NewGauge(prometheus.GaugeOpts{
		Name: "tsuru_logs_app_queues_current",
		Help: "The current number of log entries in app queues.",
	})

	logsQueueBlockedTotal = prometheus.NewCounter(prometheus.CounterOpts{
		Name: "tsuru_logs_queue_blocked_seconds_total",
		Help: "The total time spent blocked trying to add log to queue.",
	})

	logsQueueSize = prometheus.NewGauge(prometheus.GaugeOpts{
		Name: "tsuru_logs_dispatcher_queue_size",
		Help: "The max number of log entries in a dispatcher queue.",
	})

	logsEnqueued = prometheus.NewCounter(prometheus.CounterOpts{
		Name: "tsuru_logs_enqueued_total",
		Help: "The number of log entries enqueued for processing.",
	})

	logsWritten = prometheus.NewCounter(prometheus.CounterOpts{
		Name: "tsuru_logs_write_total",
		Help: "The number of log entries written to mongo.",
	})

	logsDropped = prometheus.NewCounter(prometheus.CounterOpts{
		Name: "tsuru_logs_dropped_total",
		Help: "The number of log entries dropped due to full buffers.",
	})

	logsMongoFullLatency = prometheus.NewHistogram(prometheus.HistogramOpts{
		Name:    "tsuru_logs_mongo_full_duration_seconds",
		Help:    "The latency distributions for log messages to be stored in database.",
		Buckets: buckets,
	})

	logsMongoLatency = prometheus.NewHistogram(prometheus.HistogramOpts{
		Name:    "tsuru_logs_mongo_duration_seconds",
		Help:    "The latency distributions for log messages to be stored in database.",
		Buckets: buckets,
	})
)

func init() {
	prometheus.MustRegister(logsInQueue)
	prometheus.MustRegister(logsInAppQueues)
	prometheus.MustRegister(logsQueueSize)
	prometheus.MustRegister(logsEnqueued)
	prometheus.MustRegister(logsWritten)
	prometheus.MustRegister(logsDropped)
	prometheus.MustRegister(logsQueueBlockedTotal)
	prometheus.MustRegister(logsMongoFullLatency)
	prometheus.MustRegister(logsMongoLatency)
}

type LogListener struct {
	c       <-chan Applog
	logConn *db.LogStorage
	quit    chan struct{}
}

func isCappedPositionLost(err error) bool {
	if err == nil {
		return false
	}
	return strings.Contains(err.Error(), "CappedPositionLost")
}

func isSessionClosed(r interface{}) bool {
	return fmt.Sprintf("%v", r) == "Session already closed"
}

func NewLogListener(a *App, filterLog Applog) (*LogListener, error) {
	conn, err := db.LogConn()
	if err != nil {
		return nil, err
	}
	c := make(chan Applog, 10)
	quit := make(chan struct{})
	coll := conn.Logs(a.Name)
	var lastLog Applog
	err = coll.Find(nil).Sort("-_id").Limit(1).One(&lastLog)
	if err == mgo.ErrNotFound {
		// Tail cursors do not work correctly if the collection is empty (the
		// Next() call wouldn't block). So if the collection is empty we insert
		// the very first log line in it. This is quite rare in the real world
		// though so the impact of this extra log message is really small.
		err = a.Log("Logs initialization", "tsuru", "")
		if err != nil {
			return nil, err
		}
		err = coll.Find(nil).Sort("-_id").Limit(1).One(&lastLog)
	}
	if err != nil {
		return nil, err
	}
	lastId := lastLog.MongoID
	mkQuery := func() bson.M {
		m := bson.M{
			"_id": bson.M{"$gt": lastId},
		}
		if filterLog.Source != "" {
			m["source"] = filterLog.Source
		}
		if filterLog.Unit != "" {
			m["unit"] = filterLog.Unit
		}
		return m
	}
	query := coll.Find(mkQuery())
	tailTimeout := 10 * time.Second
	iter := query.Sort("$natural").Tail(tailTimeout)
	go func() {
		defer close(c)
		defer func() {
			if r := recover(); r != nil {
				if isSessionClosed(r) {
					return
				}
				panic(err)
			}
		}()
		for {
			var applog Applog
			for iter.Next(&applog) {
				lastId = applog.MongoID
				select {
				case c <- applog:
				case <-quit:
					iter.Close()
					return
				}
			}
			if iter.Timeout() {
				continue
			}
			if err := iter.Err(); err != nil {
				if !isCappedPositionLost(err) {
					log.Errorf("error tailing logs: %v", err)
					iter.Close()
					return
				}
			}
			iter.Close()
			query = coll.Find(mkQuery())
			iter = query.Sort("$natural").Tail(tailTimeout)
		}
	}()
	l := LogListener{c: c, logConn: conn, quit: quit}
	return &l, nil
}

func (l *LogListener) ListenChan() <-chan Applog {
	return l.c
}

func (l *LogListener) Close() {
	l.logConn.Close()
	if l.quit != nil {
		close(l.quit)
		l.quit = nil
	}
}

type LogDispatcher struct {
	mu             sync.RWMutex
	dispatchers    map[string]*appLogDispatcher
	msgCh          chan *msgWithTS
	shuttingDown   int32
	doneProcessing chan struct{}
}

type msgWithTS struct {
	msg        *Applog
	arriveTime time.Time
}

func NewlogDispatcher(chanSize int) *LogDispatcher {
	d := &LogDispatcher{
		dispatchers:    make(map[string]*appLogDispatcher),
		msgCh:          make(chan *msgWithTS, chanSize),
		doneProcessing: make(chan struct{}),
	}
	go d.runWriter()
	shutdown.Register(d)
	logsQueueSize.Set(float64(chanSize))
	return d
}

func (d *LogDispatcher) getMessageDispatcher(msg *Applog) *appLogDispatcher {
	appName := msg.AppName
	d.mu.RLock()
	appD, ok := d.dispatchers[appName]
	if !ok {
		d.mu.RUnlock()
		d.mu.Lock()
		appD, ok = d.dispatchers[appName]
		if !ok {
			appD = newAppLogDispatcher(appName)
			d.dispatchers[appName] = appD
		}
		d.mu.Unlock()
	} else {
		d.mu.RUnlock()
	}
	return appD
}

func (d *LogDispatcher) runWriter() {
	defer close(d.doneProcessing)
	for msgExtra := range d.msgCh {
		if msgExtra == nil {
			break
		}
		logsInQueue.Dec()
		appD := d.getMessageDispatcher(msgExtra.msg)
		appD.send(msgExtra)
	}
}

func (d *LogDispatcher) Send(msg *Applog) error {
	if atomic.LoadInt32(&d.shuttingDown) == 1 {
		return errors.New("log dispatcher is shutting down")
	}
	logsInQueue.Inc()
	logsEnqueued.Inc()
	msgExtra := &msgWithTS{msg: msg, arriveTime: time.Now()}
	select {
	case d.msgCh <- msgExtra:
	default:
		t0 := time.Now()
		d.msgCh <- msgExtra
		logsQueueBlockedTotal.Add(time.Since(t0).Seconds())
	}
	return nil
}

func (a *LogDispatcher) String() string {
	return "log dispatcher"
}

func (d *LogDispatcher) Shutdown() {
	atomic.StoreInt32(&d.shuttingDown, 1)
	d.msgCh <- nil
	<-d.doneProcessing
	logsInQueue.Set(0)
	for _, appD := range d.dispatchers {
		appD.stopWait()
	}
}

type appLogDispatcher struct {
	appName string
	*bulkProcessor
}

func newAppLogDispatcher(appName string) *appLogDispatcher {
	d := &appLogDispatcher{
		bulkProcessor: initBulkProcessor(bulkMaxWaitMongoTime, bulkMaxNumberMsgs),
		appName:       appName,
	}
	d.flushable = d
	go d.run()
	return d
}

func (d *appLogDispatcher) flush(msgs []interface{}, lastMessage *msgWithTS) bool {
	conn, err := db.LogConn()
	if err != nil {
		log.Errorf("[log flusher] unable to connect to mongodb: %s", err)
		return false
	}
	coll := conn.Logs(d.appName)
	err = coll.Insert(msgs...)
	coll.Close()
	if err != nil {
		log.Errorf("[log flusher] unable to insert logs: %s", err)
		return false
	}
	if lastMessage != nil {
		logsMongoLatency.Observe(time.Since(lastMessage.arriveTime).Seconds())
		logsMongoFullLatency.Observe(time.Since(lastMessage.msg.Date).Seconds())
	}
	logsWritten.Add(float64(len(msgs)))
	return true
}

type bulkProcessor struct {
	maxWaitTime time.Duration
	bulkSize    int
	finished    chan struct{}
	ch          chan *msgWithTS
	nextNotify  *time.Timer
	flushable   interface {
		flush([]interface{}, *msgWithTS) bool
	}
}

func initBulkProcessor(maxWait time.Duration, bulkSize int) *bulkProcessor {
	return &bulkProcessor{
		maxWaitTime: maxWait,
		bulkSize:    bulkSize,
		finished:    make(chan struct{}),
		ch:          make(chan *msgWithTS, bulkQueueMaxSize),
		nextNotify:  time.NewTimer(0),
	}
}

func (p *bulkProcessor) send(msg *msgWithTS) {
	select {
	case p.ch <- msg:
		logsInAppQueues.Set(float64(len(p.ch)))
	default:
		logsDropped.Inc()
		select {
		case <-p.nextNotify.C:
			log.Errorf("dropping log messages to mongodb due to full channel buffer. app: %q, len: %d", msg.msg.AppName, len(p.ch))
			p.nextNotify.Reset(time.Minute)
		default:
		}
	}
}

func (p *bulkProcessor) stopWait() {
	p.ch <- nil
	<-p.finished
}

func (p *bulkProcessor) run() {
	defer close(p.finished)
	t := time.NewTimer(p.maxWaitTime)
	pos := 0
	bulkBuffer := make([]interface{}, p.bulkSize)
	shouldReturn := false
	var lastMessage *msgWithTS
	for {
		var flush bool
		select {
		case msgExtra := <-p.ch:
			logsInAppQueues.Set(float64(len(p.ch)))
			if msgExtra == nil {
				flush = true
				shouldReturn = true
				break
			}
			if pos == p.bulkSize {
				flush = true
				break
			}
			lastMessage = msgExtra
			bulkBuffer[pos] = msgExtra.msg
			pos++
			flush = p.bulkSize == pos
		case <-t.C:
			flush = true
			t.Reset(p.maxWaitTime)
		}
		if flush && pos > 0 {
			if p.flushable.flush(bulkBuffer[:pos], lastMessage) {
				lastMessage = nil
				pos = 0
			}
		}
		if shouldReturn {
			return
		}
	}
}
<commit_msg>app: reuse connection to log database<commit_after>// Copyright 2013 tsuru authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package app

import (
	"fmt"
	"strings"
	"sync"
	"sync/atomic"
	"time"

	"github.com/pkg/errors"
	"github.com/prometheus/client_golang/prometheus"
	"github.com/tsuru/tsuru/api/shutdown"
	"github.com/tsuru/tsuru/db"
	"github.com/tsuru/tsuru/db/storage"
	"github.com/tsuru/tsuru/log"
	"gopkg.in/mgo.v2"
	"gopkg.in/mgo.v2/bson"
)

var (
	bulkMaxWaitMongoTime = 1 * time.Second
	bulkMaxNumberMsgs    = 1000
	bulkQueueMaxSize     = 10000

	buckets = append([]float64{0.1, 0.5}, prometheus.ExponentialBuckets(1, 1.6, 15)...)

	logsInQueue = prometheus.NewGauge(prometheus.GaugeOpts{
		Name: "tsuru_logs_queue_current",
		Help: "The current number of log entries in dispatcher queue.",
	})

	logsInAppQueues = prometheus.NewGauge(prometheus.GaugeOpts{
		Name: "tsuru_logs_app_queues_current",
		Help: "The current number of log entries in app queues.",
	})

	logsQueueBlockedTotal = prometheus.NewCounter(prometheus.CounterOpts{
		Name: "tsuru_logs_queue_blocked_seconds_total",
		Help: "The total time spent blocked trying to add log to queue.",
	})

	logsQueueSize = prometheus.NewGauge(prometheus.GaugeOpts{
		Name: "tsuru_logs_dispatcher_queue_size",
		Help: "The max number of log entries in a dispatcher queue.",
	})

	logsEnqueued = prometheus.NewCounter(prometheus.CounterOpts{
		Name: "tsuru_logs_enqueued_total",
		Help: "The number of log entries enqueued for processing.",
	})

	logsWritten = prometheus.NewCounter(prometheus.CounterOpts{
		Name: "tsuru_logs_write_total",
		Help: "The number of log entries written to mongo.",
	})

	logsDropped = prometheus.NewCounter(prometheus.CounterOpts{
		Name: "tsuru_logs_dropped_total",
		Help: "The number of log entries dropped due to full buffers.",
	})

	logsMongoFullLatency = prometheus.NewHistogram(prometheus.HistogramOpts{
		Name:    "tsuru_logs_mongo_full_duration_seconds",
		Help:    "The latency distributions for log messages to be stored in database.",
		Buckets: buckets,
	})

	logsMongoLatency = prometheus.NewHistogram(prometheus.HistogramOpts{
		Name:    "tsuru_logs_mongo_duration_seconds",
		Help:    "The latency distributions for log messages to be stored in database.",
		Buckets: buckets,
	})
)

func init() {
	prometheus.MustRegister(logsInQueue)
	prometheus.MustRegister(logsInAppQueues)
	prometheus.MustRegister(logsQueueSize)
	prometheus.MustRegister(logsEnqueued)
	prometheus.MustRegister(logsWritten)
	prometheus.MustRegister(logsDropped)
	prometheus.MustRegister(logsQueueBlockedTotal)
	prometheus.MustRegister(logsMongoFullLatency)
	prometheus.MustRegister(logsMongoLatency)
}

type LogListener struct {
	c       <-chan Applog
	logConn *db.LogStorage
	quit    chan struct{}
}

func isCappedPositionLost(err error) bool {
	if err == nil {
		return false
	}
	return strings.Contains(err.Error(), "CappedPositionLost")
}

func isSessionClosed(r interface{}) bool {
	return fmt.Sprintf("%v", r) == "Session already closed"
}

func NewLogListener(a *App, filterLog Applog) (*LogListener, error) {
	conn, err := db.LogConn()
	if err != nil {
		return nil, err
	}
	c := make(chan Applog, 10)
	quit := make(chan struct{})
	coll := conn.Logs(a.Name)
	var lastLog Applog
	err = coll.Find(nil).Sort("-_id").Limit(1).One(&lastLog)
	if err == mgo.ErrNotFound {
		// Tail cursors do not work correctly if the collection is empty (the
		// Next() call wouldn't block). So if the collection is empty we insert
		// the very first log line in it. This is quite rare in the real world
		// though so the impact of this extra log message is really small.
		err = a.Log("Logs initialization", "tsuru", "")
		if err != nil {
			return nil, err
		}
		err = coll.Find(nil).Sort("-_id").Limit(1).One(&lastLog)
	}
	if err != nil {
		return nil, err
	}
	lastId := lastLog.MongoID
	mkQuery := func() bson.M {
		m := bson.M{
			"_id": bson.M{"$gt": lastId},
		}
		if filterLog.Source != "" {
			m["source"] = filterLog.Source
		}
		if filterLog.Unit != "" {
			m["unit"] = filterLog.Unit
		}
		return m
	}
	query := coll.Find(mkQuery())
	tailTimeout := 10 * time.Second
	iter := query.Sort("$natural").Tail(tailTimeout)
	go func() {
		defer close(c)
		defer func() {
			if r := recover(); r != nil {
				if isSessionClosed(r) {
					return
				}
				panic(err)
			}
		}()
		for {
			var applog Applog
			for iter.Next(&applog) {
				lastId = applog.MongoID
				select {
				case c <- applog:
				case <-quit:
					iter.Close()
					return
				}
			}
			if iter.Timeout() {
				continue
			}
			if err := iter.Err(); err != nil {
				if !isCappedPositionLost(err) {
					log.Errorf("error tailing logs: %v", err)
					iter.Close()
					return
				}
			}
			iter.Close()
			query = coll.Find(mkQuery())
			iter = query.Sort("$natural").Tail(tailTimeout)
		}
	}()
	l := LogListener{c: c, logConn: conn, quit: quit}
	return &l, nil
}

func (l *LogListener) ListenChan() <-chan Applog {
	return l.c
}

func (l *LogListener) Close() {
	l.logConn.Close()
	if l.quit != nil {
		close(l.quit)
		l.quit = nil
	}
}

type LogDispatcher struct {
	mu             sync.RWMutex
	dispatchers    map[string]*appLogDispatcher
	msgCh          chan *msgWithTS
	shuttingDown   int32
	doneProcessing chan struct{}
}

type msgWithTS struct {
	msg        *Applog
	arriveTime time.Time
}

func NewlogDispatcher(chanSize int) *LogDispatcher {
	d := &LogDispatcher{
		dispatchers:    make(map[string]*appLogDispatcher),
		msgCh:          make(chan *msgWithTS, chanSize),
		doneProcessing: make(chan struct{}),
	}
	go d.runWriter()
	shutdown.Register(d)
	logsQueueSize.Set(float64(chanSize))
	return d
}

func (d *LogDispatcher) getMessageDispatcher(msg *Applog) *appLogDispatcher {
	appName := msg.AppName
	d.mu.RLock()
	appD, ok := d.dispatchers[appName]
	if !ok {
		d.mu.RUnlock()
		d.mu.Lock()
		appD, ok = d.dispatchers[appName]
		if !ok {
			appD = newAppLogDispatcher(appName)
			d.dispatchers[appName] = appD
		}
		d.mu.Unlock()
	} else {
		d.mu.RUnlock()
	}
	return appD
}

func (d *LogDispatcher) runWriter() {
	defer close(d.doneProcessing)
	for msgExtra := range d.msgCh {
		if msgExtra == nil {
			break
		}
		logsInQueue.Dec()
		appD := d.getMessageDispatcher(msgExtra.msg)
		appD.send(msgExtra)
	}
}

func (d *LogDispatcher) Send(msg *Applog) error {
	if atomic.LoadInt32(&d.shuttingDown) == 1 {
		return errors.New("log dispatcher is shutting down")
	}
	logsInQueue.Inc()
	logsEnqueued.Inc()
	msgExtra := &msgWithTS{msg: msg, arriveTime: time.Now()}
	select {
	case d.msgCh <- msgExtra:
	default:
		t0 := time.Now()
		d.msgCh <- msgExtra
		logsQueueBlockedTotal.Add(time.Since(t0).Seconds())
	}
	return nil
}

func (a *LogDispatcher) String() string {
	return "log dispatcher"
}

func (d *LogDispatcher) Shutdown() {
	atomic.StoreInt32(&d.shuttingDown, 1)
	d.msgCh <- nil
	<-d.doneProcessing
	logsInQueue.Set(0)
	for _, appD := range d.dispatchers {
		appD.stopWait()
	}
}

type appLogDispatcher struct {
	appName string
	coll    *storage.Collection
	*bulkProcessor
}

func newAppLogDispatcher(appName string) *appLogDispatcher {
	d := &appLogDispatcher{
		bulkProcessor: initBulkProcessor(bulkMaxWaitMongoTime, bulkMaxNumberMsgs),
		appName:       appName,
	}
	d.flushable = d
	go d.run()
	return d
}

func (d *appLogDispatcher) getColl() (*storage.Collection, error) {
	if d.coll != nil {
		return d.coll, nil
	}
	conn, err := db.LogConn()
	if err != nil {
		return nil, err
	}
	d.coll = conn.Logs(d.appName)
	return d.coll, nil
}

func (d *appLogDispatcher) flush(msgs []interface{}, lastMessage *msgWithTS) bool {
	coll, err := d.getColl()
	if err != nil {
		log.Errorf("[log flusher] unable to connect to mongodb: %s", err)
		return false
	}
	err = coll.Insert(msgs...)
	if err != nil {
		coll.Close()
		d.coll = nil
		log.Errorf("[log flusher] unable to insert logs: %s", err)
		return false
	}
	if lastMessage != nil {
		logsMongoLatency.Observe(time.Since(lastMessage.arriveTime).Seconds())
		logsMongoFullLatency.Observe(time.Since(lastMessage.msg.Date).Seconds())
	}
	logsWritten.Add(float64(len(msgs)))
	return true
}

type bulkProcessor struct {
	maxWaitTime time.Duration
	bulkSize    int
	finished    chan struct{}
	ch          chan *msgWithTS
	nextNotify  *time.Timer
	flushable   interface {
		flush([]interface{}, *msgWithTS) bool
	}
}

func initBulkProcessor(maxWait time.Duration, bulkSize int) *bulkProcessor {
	return &bulkProcessor{
		maxWaitTime: maxWait,
		bulkSize:    bulkSize,
		finished:    make(chan struct{}),
		ch:          make(chan *msgWithTS, bulkQueueMaxSize),
		nextNotify:  time.NewTimer(0),
	}
}

func (p *bulkProcessor) send(msg *msgWithTS) {
	select {
	case p.ch <- msg:
		logsInAppQueues.Set(float64(len(p.ch)))
	default:
		logsDropped.Inc()
		select {
		case <-p.nextNotify.C:
			log.Errorf("dropping log messages to mongodb due to full channel buffer. app: %q, len: %d", msg.msg.AppName, len(p.ch))
			p.nextNotify.Reset(time.Minute)
		default:
		}
	}
}

func (p *bulkProcessor) stopWait() {
	p.ch <- nil
	<-p.finished
}

func (p *bulkProcessor) run() {
	defer close(p.finished)
	t := time.NewTimer(p.maxWaitTime)
	pos := 0
	bulkBuffer := make([]interface{}, p.bulkSize)
	shouldReturn := false
	var lastMessage *msgWithTS
	for {
		var flush bool
		select {
		case msgExtra := <-p.ch:
			logsInAppQueues.Set(float64(len(p.ch)))
			if msgExtra == nil {
				flush = true
				shouldReturn = true
				break
			}
			if pos == p.bulkSize {
				flush = true
				break
			}
			lastMessage = msgExtra
			bulkBuffer[pos] = msgExtra.msg
			pos++
			flush = p.bulkSize == pos
		case <-t.C:
			flush = true
			t.Reset(p.maxWaitTime)
		}
		if flush && pos > 0 {
			if p.flushable.flush(bulkBuffer[:pos], lastMessage) {
				lastMessage = nil
				pos = 0
			}
		}
		if shouldReturn {
			return
		}
	}
}
<|endoftext|>


<commit_before>package wini

import (
	"fmt"
	"path/filepath"
	"runtime"
	"testing"

	"github.com/bmizerany/assert"
)

func Test1(t *testing.T) {

	filename := filepath.Join(testDataDir(), "ini_parser_testfile.ini")
	ini := New()
	err := ini.ParseFile(filename)
	assert.Equal(t, nil, err)

	v, ok := ini.Get("", "mid")
	assert.Equal(t, v, "ac9219aa5232c4e519ae5fcb4d77ae5b")
	assert.Equal(t, ok, true)

	v, ok = ini.Get("", "product")
	assert.Equal(t, v, "ppp")
	assert.Equal(t, ok, true)

	v, ok = ini.Get("", "combo")
	assert.Equal(t, v, "ccc")
	assert.Equal(t, ok, true)

	v, ok = ini.Get("", "aa")
	assert.Equal(t, v, "bb")
	assert.Equal(t, ok, true)

	v, ok = ini.Get("", "axxxa")
	assert.Equal(t, v, "")
	assert.Equal(t, ok, false)

	m, ok := ini.GetKvmap("")
	assert.Equal(t, len(m), 6)
	assert.Equal(t, ok, true)

	n, ok := ini.GetKvmap("n")
	assert.Equal(t, len(n), 0)
	assert.Equal(t, ok, false)

	sss, ok := ini.GetKvmap("sss")
	assert.Equal(t, len(sss), 2)
	assert.Equal(t, ok, true)
	v, ok = ini.Get("sss", "aa")
	assert.Equal(t, v, "bb")
	assert.Equal(t, ok, true)
	v, ok = ini.Get("sss", "appext")
	assert.Equal(t, v, "ab=cd")
	assert.Equal(t, ok, true)
}

func TestUft8(t *testing.T) {
	/*
		title=百度搜索_ipad2
		url=http://www.baidu.com/s?bs=ipad&f=8&rsv_bp=1&wd=ipad2&inputT=397
		url_md5=5844a75423cd3372e1997360bd110a25
		refer=http://www.google.com
		anchor_text= google
		ret_form = json
		 ret_start = 0
		 ret_limit =    50
		page_info   =  0,0,50,1,0,20
		local=0
		mid=c4ca4238a0b923820dcc509a6f75849b
		product=test
		combo=test
		version=1.0.0.1
		debug=1
		encoding=1

	*/

	filename := filepath.Join(testDataDir(), "utf8.ini")
	ini := New()
	err := ini.ParseFile(filename)
	assert.Equal(t, nil, err)

	v, ok := ini.Get("", "title")
	assert.Equal(t, v, "百度搜索_ipad2")
	assert.Equal(t, ok, true)

	v, ok = ini.Get("", "url_md5")
	assert.Equal(t, v, "5844a75423cd3372e1997360bd110a25")
	assert.Equal(t, ok, true)

	v, ok = ini.Get("", "ret_form")
	assert.Equal(t, v, "json")
	assert.Equal(t, ok, true)

	v, ok = ini.Get("", "ret_start")
	assert.Equal(t, v, "0")
	assert.Equal(t, ok, true)

	v, ok = ini.Get("", "ret_limit")
	assert.Equal(t, v, "50")
	assert.Equal(t, ok, true)

	v, ok = ini.Get("", "axxxa")
	assert.Equal(t, v, "")
	assert.Equal(t, ok, false)

	m, ok := ini.GetKvmap("")
	assert.Equal(t, len(m), 16)
	assert.Equal(t, ok, true)

	n, ok := ini.GetKvmap("n")
	assert.Equal(t, len(n), 0)
	assert.Equal(t, ok, false)
}

func TestErrorFormat(t *testing.T) {
	filename := filepath.Join(testDataDir(), "error.ini")
	ini := New()
	err := ini.ParseFile(filename)
	assert.NotEqual(t, nil, err)
}

func TestMemoryData1(t *testing.T) {
	raw := []byte("a:av||b:bv||c:cv||||d:dv||||||")
	ini := New()
	err := ini.Parse(raw, "||", ":")
	assert.Equal(t, nil, err)

	v, ok := ini.Get("", "a")
	assert.Equal(t, v, "av")
	assert.Equal(t, ok, true)

	v, ok = ini.Get("", "b")
	assert.Equal(t, v, "bv")
	assert.Equal(t, ok, true)

	v, ok = ini.Get("", "c")
	assert.Equal(t, v, "cv")
	assert.Equal(t, ok, true)

	v, ok = ini.Get("", "d")
	assert.Equal(t, v, "dv")
	assert.Equal(t, ok, true)

	m, ok := ini.GetKvmap("")
	assert.Equal(t, len(m), 4)
	assert.Equal(t, ok, true)

	n, ok := ini.GetKvmap("n")
	assert.Equal(t, len(n), 0)
	assert.Equal(t, ok, false)
}

func testDataDir() string {
	var file string
	var ok bool
	if _, file, _, ok = runtime.Caller(0); ok {
		fmt.Printf("file=%v\n", file)
	}

	curdir := filepath.Dir(file)
	return filepath.Join(curdir, "test/data")
}
<commit_msg>Refactor : rename testDataDir() --> getTestDataDir()<commit_after>package wini

import (
	"fmt"
	"path/filepath"
	"runtime"
	"testing"

	"github.com/bmizerany/assert"
)

func Test1(t *testing.T) {

	filename := filepath.Join(getTestDataDir(), "ini_parser_testfile.ini")
	ini := New()
	err := ini.ParseFile(filename)
	assert.Equal(t, nil, err)

	v, ok := ini.Get("", "mid")
	assert.Equal(t, v, "ac9219aa5232c4e519ae5fcb4d77ae5b")
	assert.Equal(t, ok, true)

	v, ok = ini.Get("", "product")
	assert.Equal(t, v, "ppp")
	assert.Equal(t, ok, true)

	v, ok = ini.Get("", "combo")
	assert.Equal(t, v, "ccc")
	assert.Equal(t, ok, true)

	v, ok = ini.Get("", "aa")
	assert.Equal(t, v, "bb")
	assert.Equal(t, ok, true)

	v, ok = ini.Get("", "axxxa")
	assert.Equal(t, v, "")
	assert.Equal(t, ok, false)

	m, ok := ini.GetKvmap("")
	assert.Equal(t, len(m), 6)
	assert.Equal(t, ok, true)

	n, ok := ini.GetKvmap("n")
	assert.Equal(t, len(n), 0)
	assert.Equal(t, ok, false)

	sss, ok := ini.GetKvmap("sss")
	assert.Equal(t, len(sss), 2)
	assert.Equal(t, ok, true)
	v, ok = ini.Get("sss", "aa")
	assert.Equal(t, v, "bb")
	assert.Equal(t, ok, true)
	v, ok = ini.Get("sss", "appext")
	assert.Equal(t, v, "ab=cd")
	assert.Equal(t, ok, true)
}

func TestUft8(t *testing.T) {
	/*
		title=百度搜索_ipad2
		url=http://www.baidu.com/s?bs=ipad&f=8&rsv_bp=1&wd=ipad2&inputT=397
		url_md5=5844a75423cd3372e1997360bd110a25
		refer=http://www.google.com
		anchor_text= google
		ret_form = json
		 ret_start = 0
		 ret_limit =    50
		page_info   =  0,0,50,1,0,20
		local=0
		mid=c4ca4238a0b923820dcc509a6f75849b
		product=test
		combo=test
		version=1.0.0.1
		debug=1
		encoding=1

	*/

	filename := filepath.Join(getTestDataDir(), "utf8.ini")
	ini := New()
	err := ini.ParseFile(filename)
	assert.Equal(t, nil, err)

	v, ok := ini.Get("", "title")
	assert.Equal(t, v, "百度搜索_ipad2")
	assert.Equal(t, ok, true)

	v, ok = ini.Get("", "url_md5")
	assert.Equal(t, v, "5844a75423cd3372e1997360bd110a25")
	assert.Equal(t, ok, true)

	v, ok = ini.Get("", "ret_form")
	assert.Equal(t, v, "json")
	assert.Equal(t, ok, true)

	v, ok = ini.Get("", "ret_start")
	assert.Equal(t, v, "0")
	assert.Equal(t, ok, true)

	v, ok = ini.Get("", "ret_limit")
	assert.Equal(t, v, "50")
	assert.Equal(t, ok, true)

	v, ok = ini.Get("", "axxxa")
	assert.Equal(t, v, "")
	assert.Equal(t, ok, false)

	m, ok := ini.GetKvmap("")
	assert.Equal(t, len(m), 16)
	assert.Equal(t, ok, true)

	n, ok := ini.GetKvmap("n")
	assert.Equal(t, len(n), 0)
	assert.Equal(t, ok, false)
}

func TestErrorFormat(t *testing.T) {
	filename := filepath.Join(getTestDataDir(), "error.ini")
	ini := New()
	err := ini.ParseFile(filename)
	assert.NotEqual(t, nil, err)
}

func TestMemoryData1(t *testing.T) {
	raw := []byte("a:av||b:bv||c:cv||||d:dv||||||")
	ini := New()
	err := ini.Parse(raw, "||", ":")
	assert.Equal(t, nil, err)

	v, ok := ini.Get("", "a")
	assert.Equal(t, v, "av")
	assert.Equal(t, ok, true)

	v, ok = ini.Get("", "b")
	assert.Equal(t, v, "bv")
	assert.Equal(t, ok, true)

	v, ok = ini.Get("", "c")
	assert.Equal(t, v, "cv")
	assert.Equal(t, ok, true)

	v, ok = ini.Get("", "d")
	assert.Equal(t, v, "dv")
	assert.Equal(t, ok, true)

	m, ok := ini.GetKvmap("")
	assert.Equal(t, len(m), 4)
	assert.Equal(t, ok, true)

	n, ok := ini.GetKvmap("n")
	assert.Equal(t, len(n), 0)
	assert.Equal(t, ok, false)
}

func getTestDataDir() string {
	var file string
	var ok bool
	if _, file, _, ok = runtime.Caller(0); ok {
		fmt.Printf("file=%v\n", file)
	}

	curdir := filepath.Dir(file)
	return filepath.Join(curdir, "test/data")
}
<|endoftext|>


<commit_before>package mtp

import (
	"bytes"
	"fmt"
	"io"
	"log"
	"math/rand"
	"time"
)

var _  = log.Println

func init() {
	rand.Seed(time.Now().UnixNano())
}

// OpenSession opens a session, which is necesary for any command that
// queries or modifies storage. It is an error to open a session
// twice.
func (d *Device) OpenSession() error {
	if d.session != nil {
		return fmt.Errorf("session already open")
	}
	var req, rep Container
	req.Code = OC_OpenSession

	// avoid 0xFFFFFFFF and 0x00000000 for session IDs.
	sid := uint32(rand.Int31()) | 1
	req.Param = []uint32{sid} // session
	err := d.RunTransaction(&req, &rep, nil, nil, 0)
	if err != nil {
		return err
	}

	d.session = &sessionData{
		tid: 1,
		sid: sid,
	}
	return nil
}

// Closes a sessions. This is done automatically if the device is closed.
func (d *Device) CloseSession() error {
	var req, rep Container
	req.Code = OC_CloseSession
	err := d.RunTransaction(&req, &rep, nil, nil, 0)
	d.session = nil
	return err
}

func (d *Device) GetDeviceInfo(info *DeviceInfo) error {
	var req, rep Container

	req.Code = OC_GetDeviceInfo
	var buf bytes.Buffer
	err := d.RunTransaction(&req, &rep, &buf, nil, 0)
	if err != nil {
		return err
	}
	err = Decode(&buf, info)
	if err != nil {
		return err
	}
	return err
}

func (d *Device) GetStorageIDs(info *Uint32Array) error {
	var req, rep Container
	req.Code = OC_GetStorageIDs
	var buf bytes.Buffer
	err := d.RunTransaction(&req, &rep, &buf, nil, 0)
	if err != nil {
		return err
	}
	err = Decode(&buf, info)
	return err
}

func (d *Device) GetObjectPropDesc(objPropCode, objFormatCode uint16, info *ObjectPropDesc) error {
	var req, rep Container
	req.Code = OC_MTP_GetObjectPropDesc
	req.Param = []uint32{uint32(objPropCode), uint32(objFormatCode)}
	var buf bytes.Buffer
	err := d.RunTransaction(&req, &rep, &buf, nil, 0)
	if err != nil {
		return err
	}

	err = Decode(&buf, info)
	return err
}

func (d *Device) GetObjectPropValue(objHandle uint32, objPropCode uint16, value interface{}) error {
	var req, rep Container
	var buf bytes.Buffer

	req.Code = OC_MTP_GetObjectPropValue
	req.Param = []uint32{objHandle, uint32(objPropCode)}
	err := d.RunTransaction(&req, &rep, &buf, nil, 0)
	if err != nil {
		return err
	}
	err = Decode(&buf, value)
	return err
}

func (d *Device) SetObjectPropValue(objHandle uint32, objPropCode uint16, value interface{}) error {
	var req, rep Container
	req.Code = OC_MTP_SetObjectPropValue
	req.Param = []uint32{objHandle, uint32(objPropCode)}
	var buf bytes.Buffer
	err := Encode(&buf, value)
	if err != nil {
		return err
	}

	return d.RunTransaction(&req, &rep, nil, &buf, int64(buf.Len()))
}

func (d *Device) GetObjectPropsSupported(objFormatCode uint16, props *Uint16Array) error {
	var req, rep Container
	var buf bytes.Buffer

	req.Code = OC_MTP_GetObjectPropsSupported
	req.Param = []uint32{uint32(objFormatCode)}
	err := d.RunTransaction(&req, &rep, &buf, nil, 0)
	if err != nil {
		return err
	}
	err = Decode(&buf, props)
	return err
}

func (d *Device) GetDevicePropDesc(propCode uint16, info *DevicePropDesc) error {
	var req, rep Container
	req.Code = OC_GetDevicePropDesc
	req.Param = append(req.Param, uint32(propCode))

	var buf bytes.Buffer
	err := d.RunTransaction(&req, &rep, &buf, nil, 0)
	if err != nil {
		return err
	}

	err = info.Decode(&buf)
	return err
}

func (d *Device) SetDevicePropValue(propCode uint32, src interface{}) error {
	var req, rep Container
	req.Code = OC_SetDevicePropValue
	req.Param = []uint32{propCode}

	var buf bytes.Buffer
	err := Encode(&buf, src)
	if err != nil {
		return err
	}
	return d.RunTransaction(&req, &rep, nil, &buf, int64(buf.Len()))
}

func (d *Device) GetDevicePropValue(propCode uint32, dest interface{}) error {
	var req, rep Container
	req.Code = OC_GetDevicePropValue
	req.Param = []uint32{propCode}

	var buf bytes.Buffer
	err := d.RunTransaction(&req, &rep, &buf, nil, 0)
	if err != nil {
		return err
	}
	err = Decode(&buf, dest)
	return err
}

func (d *Device) ResetDevicePropValue(propCode uint32) error {
	var req, rep Container
	req.Code = OC_ResetDevicePropValue
	req.Param = []uint32{propCode}
	return d.RunTransaction(&req, &rep, nil, nil, 0)
}

func (d *Device) GetStorageInfo(ID uint32, info *StorageInfo) error {
	var req, rep Container
	req.Code = OC_GetStorageInfo
	req.Param = []uint32{ID}
	var buf bytes.Buffer
	err := d.RunTransaction(&req, &rep, &buf, nil, 0)
	if err != nil {
		return err
	}
	err = Decode(&buf, info)
	if err != nil {
		return err
	}
	return err
}

func (d *Device) GetObjectHandles(storageID, objFormatCode, parent uint32, info *Uint32Array) error {
	var req, rep Container
	req.Code = OC_GetObjectHandles
	req.Param = []uint32{storageID, objFormatCode, parent}
	var buf bytes.Buffer
	err := d.RunTransaction(&req, &rep, &buf, nil, 0)
	if err != nil {
		return err
	}
	err = Decode(&buf, info)
	return err
}

func (d *Device) GetObjectInfo(handle uint32, info *ObjectInfo) error {
	var req, rep Container
	req.Code = OC_GetObjectInfo
	req.Param = []uint32{handle}
	var buf bytes.Buffer
	err := d.RunTransaction(&req, &rep, &buf, nil, 0)
	if err != nil {
		return err
	}
	err = Decode(&buf, info)
	if err != nil {
		return err
	}
	return err
}

func (d *Device) GetNumObjects(storageId uint32, formatCode uint16, parent uint32) (uint32, error) {
	var req, rep Container
	req.Code = OC_GetNumObjects
	req.Param = []uint32{storageId, uint32(formatCode), parent}
	err := d.RunTransaction(&req, &rep, nil, nil, 0)
	if err != nil {
		return 0, err
	}
	return rep.Param[0], nil
}

func (d *Device) DeleteObject(handle uint32) error {
	var req, rep Container
	req.Code = OC_DeleteObject
	req.Param = []uint32{handle, 0x0}

	return d.RunTransaction(&req, &rep, nil, nil, 0)
}

func (d *Device) SendObjectInfo(wantStorageID, wantParent uint32, info *ObjectInfo) (storageID, parent, handle uint32, err error) {
	var req, rep Container
	req.Code = OC_SendObjectInfo
	req.Param = []uint32{wantStorageID, wantParent}

	buf := &bytes.Buffer{}
	err = Encode(buf, info)
	if err != nil {
		return
	}

	err = d.RunTransaction(&req, &rep, nil, buf, int64(buf.Len()))
	if err != nil {
		return
	}

	return rep.Param[0], rep.Param[1], rep.Param[2], nil
}

func (d *Device) SendObject(r io.Reader, size int64) error {
	var req, rep Container
	req.Code = OC_SendObject
	return d.RunTransaction(&req, &rep, nil, r, size)
}

func (d *Device) GetObject(handle uint32, w io.Writer) error {
	var req, rep Container
	req.Code = OC_GetObject
	req.Param = []uint32{handle}

	return d.RunTransaction(&req, &rep, w, nil, 0)
}

func (d *Device) GetPartialObject(handle uint32, w io.Writer, offset uint32, size uint32) error {
	var req, rep Container
	req.Code = OC_ANDROID_GET_PARTIAL_OBJECT64
	req.Param = []uint32{handle, offset, size}
	return d.RunTransaction(&req, &rep, w, nil, 0)
}
<commit_msg>mtp: fold error return.<commit_after>package mtp

import (
	"bytes"
	"fmt"
	"io"
	"log"
	"math/rand"
	"time"
)

var _  = log.Println

func init() {
	rand.Seed(time.Now().UnixNano())
}

// OpenSession opens a session, which is necesary for any command that
// queries or modifies storage. It is an error to open a session
// twice.
func (d *Device) OpenSession() error {
	if d.session != nil {
		return fmt.Errorf("session already open")
	}
	var req, rep Container
	req.Code = OC_OpenSession

	// avoid 0xFFFFFFFF and 0x00000000 for session IDs.
	sid := uint32(rand.Int31()) | 1
	req.Param = []uint32{sid} // session
	err := d.RunTransaction(&req, &rep, nil, nil, 0)
	if err != nil {
		return err
	}

	d.session = &sessionData{
		tid: 1,
		sid: sid,
	}
	return nil
}

// Closes a sessions. This is done automatically if the device is closed.
func (d *Device) CloseSession() error {
	var req, rep Container
	req.Code = OC_CloseSession
	err := d.RunTransaction(&req, &rep, nil, nil, 0)
	d.session = nil
	return err
}

func (d *Device) GetDeviceInfo(info *DeviceInfo) error {
	var req, rep Container

	req.Code = OC_GetDeviceInfo
	var buf bytes.Buffer
	err := d.RunTransaction(&req, &rep, &buf, nil, 0)
	if err != nil {
		return err
	}
	return Decode(&buf, info)
}

func (d *Device) GetStorageIDs(info *Uint32Array) error {
	var req, rep Container
	req.Code = OC_GetStorageIDs
	var buf bytes.Buffer
	err := d.RunTransaction(&req, &rep, &buf, nil, 0)
	if err != nil {
		return err
	}
	return Decode(&buf, info)
}

func (d *Device) GetObjectPropDesc(objPropCode, objFormatCode uint16, info *ObjectPropDesc) error {
	var req, rep Container
	req.Code = OC_MTP_GetObjectPropDesc
	req.Param = []uint32{uint32(objPropCode), uint32(objFormatCode)}
	var buf bytes.Buffer
	err := d.RunTransaction(&req, &rep, &buf, nil, 0)
	if err != nil {
		return err
	}

	return Decode(&buf, info)
}

func (d *Device) GetObjectPropValue(objHandle uint32, objPropCode uint16, value interface{}) error {
	var req, rep Container
	var buf bytes.Buffer

	req.Code = OC_MTP_GetObjectPropValue
	req.Param = []uint32{objHandle, uint32(objPropCode)}
	err := d.RunTransaction(&req, &rep, &buf, nil, 0)
	if err != nil {
		return err
	}
	return Decode(&buf, value)
}

func (d *Device) SetObjectPropValue(objHandle uint32, objPropCode uint16, value interface{}) error {
	var req, rep Container
	req.Code = OC_MTP_SetObjectPropValue
	req.Param = []uint32{objHandle, uint32(objPropCode)}
	var buf bytes.Buffer
	err := Encode(&buf, value)
	if err != nil {
		return err
	}

	return d.RunTransaction(&req, &rep, nil, &buf, int64(buf.Len()))
}

func (d *Device) GetObjectPropsSupported(objFormatCode uint16, props *Uint16Array) error {
	var req, rep Container
	var buf bytes.Buffer

	req.Code = OC_MTP_GetObjectPropsSupported
	req.Param = []uint32{uint32(objFormatCode)}
	err := d.RunTransaction(&req, &rep, &buf, nil, 0)
	if err != nil {
		return err
	}
	return Decode(&buf, props)
}

func (d *Device) GetDevicePropDesc(propCode uint16, info *DevicePropDesc) error {
	var req, rep Container
	req.Code = OC_GetDevicePropDesc
	req.Param = append(req.Param, uint32(propCode))

	var buf bytes.Buffer
	err := d.RunTransaction(&req, &rep, &buf, nil, 0)
	if err != nil {
		return err
	}

	return info.Decode(&buf)
}

func (d *Device) SetDevicePropValue(propCode uint32, src interface{}) error {
	var req, rep Container
	req.Code = OC_SetDevicePropValue
	req.Param = []uint32{propCode}

	var buf bytes.Buffer
	err := Encode(&buf, src)
	if err != nil {
		return err
	}
	return d.RunTransaction(&req, &rep, nil, &buf, int64(buf.Len()))
}

func (d *Device) GetDevicePropValue(propCode uint32, dest interface{}) error {
	var req, rep Container
	req.Code = OC_GetDevicePropValue
	req.Param = []uint32{propCode}

	var buf bytes.Buffer
	err := d.RunTransaction(&req, &rep, &buf, nil, 0)
	if err != nil {
		return err
	}
	return Decode(&buf, dest)
}

func (d *Device) ResetDevicePropValue(propCode uint32) error {
	var req, rep Container
	req.Code = OC_ResetDevicePropValue
	req.Param = []uint32{propCode}
	return d.RunTransaction(&req, &rep, nil, nil, 0)
}

func (d *Device) GetStorageInfo(ID uint32, info *StorageInfo) error {
	var req, rep Container
	req.Code = OC_GetStorageInfo
	req.Param = []uint32{ID}
	var buf bytes.Buffer
	err := d.RunTransaction(&req, &rep, &buf, nil, 0)
	if err != nil {
		return err
	}
	return Decode(&buf, info)
}

func (d *Device) GetObjectHandles(storageID, objFormatCode, parent uint32, info *Uint32Array) error {
	var req, rep Container
	req.Code = OC_GetObjectHandles
	req.Param = []uint32{storageID, objFormatCode, parent}
	var buf bytes.Buffer
	err := d.RunTransaction(&req, &rep, &buf, nil, 0)
	if err != nil {
		return err
	}
	return Decode(&buf, info)
}

func (d *Device) GetObjectInfo(handle uint32, info *ObjectInfo) error {
	var req, rep Container
	req.Code = OC_GetObjectInfo
	req.Param = []uint32{handle}
	var buf bytes.Buffer
	err := d.RunTransaction(&req, &rep, &buf, nil, 0)
	if err != nil {
		return err
	}
	return Decode(&buf, info)
}

func (d *Device) GetNumObjects(storageId uint32, formatCode uint16, parent uint32) (uint32, error) {
	var req, rep Container
	req.Code = OC_GetNumObjects
	req.Param = []uint32{storageId, uint32(formatCode), parent}
	err := d.RunTransaction(&req, &rep, nil, nil, 0)
	if err != nil {
		return 0, err
	}
	return rep.Param[0], nil
}

func (d *Device) DeleteObject(handle uint32) error {
	var req, rep Container
	req.Code = OC_DeleteObject
	req.Param = []uint32{handle, 0x0}

	return d.RunTransaction(&req, &rep, nil, nil, 0)
}

func (d *Device) SendObjectInfo(wantStorageID, wantParent uint32, info *ObjectInfo) (storageID, parent, handle uint32, err error) {
	var req, rep Container
	req.Code = OC_SendObjectInfo
	req.Param = []uint32{wantStorageID, wantParent}

	buf := &bytes.Buffer{}
	err = Encode(buf, info)
	if err != nil {
		return
	}

	err = d.RunTransaction(&req, &rep, nil, buf, int64(buf.Len()))
	if err != nil {
		return
	}

	return rep.Param[0], rep.Param[1], rep.Param[2], nil
}

func (d *Device) SendObject(r io.Reader, size int64) error {
	var req, rep Container
	req.Code = OC_SendObject
	return d.RunTransaction(&req, &rep, nil, r, size)
}

func (d *Device) GetObject(handle uint32, w io.Writer) error {
	var req, rep Container
	req.Code = OC_GetObject
	req.Param = []uint32{handle}

	return d.RunTransaction(&req, &rep, w, nil, 0)
}

func (d *Device) GetPartialObject(handle uint32, w io.Writer, offset uint32, size uint32) error {
	var req, rep Container
	req.Code = OC_ANDROID_GET_PARTIAL_OBJECT64
	req.Param = []uint32{handle, offset, size}
	return d.RunTransaction(&req, &rep, w, nil, 0)
}
<|endoftext|>


<commit_before>// Copyright (c) 2013, Suresh Sundriyal. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be found
// in the LICENSE file.
// This is a progressive processing version of the MurmurHash3 family of hash
// functions by Austin Appleby and is a translation of the public domain code
// that can be found here:
// https://code.google.com/p/smhasher/source/browse/trunk/MurmurHash3.cpp?r=150

package murmur3

import "hash"

// Extend hash.Hash to accomodate for setting the salt in Reset()
type HashM3 interface {
	hash.Hash
	//ResetAndSetSeed resets the hash and sets the seed.
	ResetAndSetSeed(seed uint32)
}

// Hash32 interface for 32-bit hash functions.
type Hash32 interface {
	HashM3
	Sum32() uint32
}

// Hash128 interface for 128-bit hash functions.
type Hash128 interface {
	HashM3
	Sum128() (uint64, uint64)
}
<commit_msg>[Documentation] Fix the package documentation<commit_after>// Copyright (c) 2013, Suresh Sundriyal. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be found
// in the LICENSE file.
// This is a progressive processing version of the MurmurHash3 family of hash
// functions by Austin Appleby and is a translation of the public domain code
// that can be found here:
// https://code.google.com/p/smhasher/source/browse/trunk/MurmurHash3.cpp?r=150

// Package murmur3 provides a progressive processing version of Austin Appleby's
// Murmur3 Hash functions.
package murmur3

import "hash"

// HashM3 extends hash.Hash and provides an additional function ResetAndSetSeed
// to reset the structures and set the seed.
type HashM3 interface {
	hash.Hash
	//ResetAndSetSeed resets the hash and sets the seed.
	ResetAndSetSeed(seed uint32)
}

// Hash32 interface for 32-bit hash functions.
type Hash32 interface {
	HashM3
	Sum32() uint32
}

// Hash128 interface for 128-bit hash functions.
type Hash128 interface {
	HashM3
	Sum128() (uint64, uint64)
}
<|endoftext|>


<commit_before>// Package mux is a wrapper on naoina/denco router.
// It uses request.Form for params instead of a separate Params
// argument. So, it requires a bit more memory and it is a bit slower.
// However, the downsides are an acceptable trade off for compatability
// with the standard library.
package mux

import (
	"net/http"

	"github.com/naoina/denco"
)

// Type represents a multiplexer for HTTP request.
type Type struct {
	routers map[string]*denco.Router
}

// Handler represents a handler of HTTP request.
type Handler struct {
	Method  string           // Method is an HTTP methods, e.g. "GET" or "POST".
	Pattern string           // Pattern is a routing path for handler.
	Func    http.HandlerFunc // Func is a function of handler of HTTP request.
}

// New allocates and returns a new multiplexer.
func New() *Type {
	return &Type{}
}

// ServeHTTP is used to implement http.Handler interface.
// It dispatches the request to the handler whose pattern
// most closely matches the request URL.
func (t *Type) ServeHTTP(w http.ResponseWriter, r *http.Request) {
}

// Handle registers the handler for the given pattern.
// If a handler already exists for pattern, it is being overridden.
func (t *Type) Handle(pattern string, handler Handler) {
}

// HandleFunc registers the handler function for the given pattern.
func (t *Type) HandleFunc(pattern string, handler http.HandlerFunc) {
}
<commit_msg>Update mux type name<commit_after>// Package mux is a wrapper on naoina/denco router.
// It uses request.Form for params instead of a separate Params
// argument. So, it requires a bit more memory and it is a bit slower.
// However, the downsides are an acceptable trade off for compatability
// with the standard library.
package mux

import (
	"net/http"

	"github.com/naoina/denco"
)

// Router represents a multiplexer for HTTP request.
type Router struct {
	routers map[string]*denco.Router
}

// Handler represents a handler of HTTP request.
type Handler struct {
	Method  string           // Method is an HTTP methods, e.g. "GET" or "POST".
	Pattern string           // Pattern is a routing path for handler.
	Func    http.HandlerFunc // Func is a function of handler of HTTP request.
}

// New allocates and returns a new multiplexer.
func New() *Router {
	return &Router{}
}

// ServeHTTP is used to implement http.Handler interface.
// It dispatches the request to the handler whose pattern
// most closely matches the request URL.
func (t *Router) ServeHTTP(w http.ResponseWriter, r *http.Request) {
}

// Handle registers the handler for the given pattern.
// If a handler already exists for pattern, it is being overridden.
func (t *Router) Handle(pattern string, handler Handler) {
}

// HandleFunc registers the handler function for the given pattern.
func (t *Router) HandleFunc(pattern string, handler http.HandlerFunc) {
}
<|endoftext|>


<commit_before>package ar

import (
	"bytes"
	"io"
	"io/ioutil"
	"os"
	"reflect"
	"strconv"
	"strings"
	"testing"
	"time"
)

// test whether fileInfo implements os.FileInfo
var _ os.FileInfo = new(fileInfo)

var testCommon = "!<arch>\n" +
	"debian-binary   1385068169  0     0     644     4         `\n" +
	"2.0\n" +
	"control.tar.gz  1385068169  0     0     644     0         `\n"

var testCommonFileHeaders = []struct {
	in   string
	want *fileInfo
	err  string
}{
	{
		in: "debian-binary   1385068169  0     0     100644  4         `\n",
		want: &fileInfo{
			name:  "debian-binary",
			mtime: time.Unix(1385068169, 0),
			mode:  os.FileMode(0100644) & os.ModePerm,
			size:  4,
		},
	},
	{
		in: "debian-binary   1385068169  0     0     644     4         `\n",
		want: &fileInfo{
			name:  "debian-binary",
			mtime: time.Unix(1385068169, 0),
			mode:  os.FileMode(0644),
			size:  4,
		},
	},
	{
		in:  "debian-binary   1385068169  0     0     120644  4         `\n",
		err: "feature not implemented: non-regular files",
	},
	{
		in:  "debian-binary   1385068169  0     0     220644  4         `\n",
		err: "corrupt archive: invalid file mode",
	},
}

func TestReadFileHeader(t *testing.T) {
	for i, test := range testCommonFileHeaders {
		got, err := parseFileHeader([]byte(test.in))
		switch {
		case err == nil && test.err != "":
			t.Errorf("%d: got no err, expected err %v", i, test.err)
			continue
		case err != nil && test.err != err.Error():
			t.Errorf("%d: got err %q, expected err %q", i, err, test.err)
			continue
		case err == nil && test.err == "":
			// no error as expected
		case err != nil && test.err == err.Error():
			t.Logf("%d: got expected error %q", i, err)
			continue
		}

		if !reflect.DeepEqual(got, test.want) {
			t.Errorf("%d: got %#v, expected %+v", i, got, test.want)
		} else {
			t.Logf("%d: got %#v", i, got)
		}
	}
}

func BenchmarkReadFileHeader(b *testing.B) {
	fh := []byte("debian-binary   1385068169  0     0     100644  4         `\n")
	b.ReportAllocs()
	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		_, _ = parseFileHeader(fh)
		b.SetBytes(60)
	}

}

var testMagic = []struct {
	in   string
	want error
}{
	{
		in: magic,
	},
	{
		in:   strings.Repeat("a", len(magic)),
		want: CorruptArchiveError("global archive header not found"),
	},
	{
		in:   "!",
		want: io.ErrUnexpectedEOF,
	},
	{
		in:   "",
		want: io.EOF,
	},
}

func TestReadMagic(t *testing.T) {
	for i, test := range testMagic {
		got := checkMagic(strings.NewReader(test.in))
		if !reflect.DeepEqual(got, test.want) {
			t.Errorf("%d: got %#v, expected %+v", i, got, test.want)
		} else {
			t.Logf("%d: got %#v", i, got)
		}
	}
}

func TestFileInfo(t *testing.T) {
	test := &fileInfo{
		name:  "debian-binary",
		mtime: time.Unix(1385068169, 0),
		mode:  os.FileMode(0644),
		size:  4,
	}

	if test.IsDir() != false {
		t.Error("IsDir")
	}
	if test.Mode() != os.FileMode(0644) {
		t.Error("Mode")
	}
	if test.ModTime() != time.Unix(1385068169, 0) {
		t.Error("ModTime")
	}
	if test.Name() != "debian-binary" {
		t.Error("Name")
	}
	if test.Size() != 4 {
		t.Error("Size")
	}
	if test.Sys() != nil {
		t.Error("Sys")
	}
}

func TestReaderBasics(t *testing.T) {
	test := strings.NewReader(testCommon)
	r := NewReader(test)
	fi, err := r.Next()
	if err != nil {
		t.Fatal(err)
		return
	}
	if fi.Mode() != os.FileMode(0644) {
		t.Error("Mode")
	}
	if fi.Name() != "debian-binary" {
		t.Error("Name")
	}
	if fi.Size() != 4 {
		t.Error("Size")
	}
	if fi.ModTime() != time.Unix(1385068169, 0) {
		t.Error("ModTime")
	}

	if content, err := ioutil.ReadAll(r); err != nil {
		t.Error(err)
	} else if string(content) != "2.0\n" {
		t.Error("Content")
	}
	fi, err = r.Next()
	if err != nil {
		t.Fatal(err)
		return
	}
	if fi.Mode() != os.FileMode(0644) {
		t.Error("Mode2")
	}
	if fi.Name() != "control.tar.gz" {
		t.Error("Name2")
	}
	if fi.Size() != 0 {
		t.Error("Size2")
	}
	if fi.ModTime() != time.Unix(1385068169, 0) {
		t.Error("ModTime2")
	}

	if content, err := ioutil.ReadAll(r); err != nil {
		t.Error(err)
	} else if string(content) != "" {
		t.Error("Content2")
	}

	fi, err = r.Next()
	if err != io.EOF {
		t.Errorf("expected EOF, got %v", err)
	}
}

func BenchmarkReaderBigFiles(b *testing.B) {
	benchmarkReader(b, 8, 8*1024*1024)
}

func BenchmarkReaderManySmallFiles(b *testing.B) {
	benchmarkReader(b, 1024, 8)
}

func genArchiveFile(meta os.FileInfo) []byte {
	ret := make([]byte, 60+meta.Size(), 60+meta.Size())
	for i := 0; i < 60; i++ {
		ret[i] = ' '
	}
	copy(ret[0:], meta.Name())
	copy(ret[16:], strconv.FormatInt(meta.ModTime().Unix(), 10))
	copy(ret[28:], "0")
	copy(ret[34:], "0")
	copy(ret[40:], strconv.FormatUint(uint64(meta.Mode()), 8))
	copy(ret[48:], strconv.FormatInt(meta.Size(), 10))
	copy(ret[58:], filemagic)
	return ret
}

func benchmarkReader(b *testing.B, numFiles int, sizeFiles int64) {
	buf := bytes.NewBufferString(magic)
	buf.Grow(numFiles * (int(sizeFiles) + 60))
	for i := 0; i < numFiles; i++ {
		buf.Write(genArchiveFile(&fileInfo{
			name:  strconv.Itoa(1000 + i),
			mtime: time.Unix(int64(i), 0),
			size:  sizeFiles,
			mode:  os.FileMode(0640),
		}))

	}
	test := bytes.NewReader(buf.Bytes())
	r := NewReader(test)

	var err error
	var read int64

	b.ReportAllocs()
	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		for j := 0; j < numFiles; j++ {
			_, err = r.Next()
			if err != nil {
				b.Fatal(err)
			}
			read += 60
			n, err := io.Copy(ioutil.Discard, r)
			if err != nil {
				b.Fatal(err)
			}
			read += n
		}
		b.SetBytes(read)
		read = 0
		test.Seek(0, 0)
		r.Reset(test)
	}
}

func TestWriterBasics(t *testing.T) {
	b := new(bytes.Buffer)
	b.Grow(len(testCommon))
	w := NewWriter(b)
	debian := &fileInfo{
		name:  "debian-binary",
		mtime: time.Unix(1385068169, 0),
		mode:  os.FileMode(0100644) & os.ModePerm,
		size:  4,
	}
	if _, err := w.WriteFile(debian, strings.NewReader("2.0\n")); err != nil {
		t.Error(err)
		return
	}

	control := &fileInfo{
		name:  "control.tar.gz",
		mtime: time.Unix(1385068169, 0),
		mode:  os.FileMode(0100644) & os.ModePerm,
		size:  0,
	}
	if _, err := w.WriteFile(control, strings.NewReader("")); err != nil {
		t.Error(err)
		return
	}

	if archive := b.String(); archive != testCommon {
		t.Errorf("got\n%q\nwant\n%q", archive, testCommon)
	}
}
<commit_msg>Add basic benchmark support for Writer<commit_after>package ar

import (
	"bytes"
	"io"
	"io/ioutil"
	"os"
	"reflect"
	"strconv"
	"strings"
	"testing"
	"time"
)

// test whether fileInfo implements os.FileInfo
var _ os.FileInfo = new(fileInfo)

var testCommon = "!<arch>\n" +
	"debian-binary   1385068169  0     0     644     4         `\n" +
	"2.0\n" +
	"control.tar.gz  1385068169  0     0     644     0         `\n"

var testCommonFileHeaders = []struct {
	in   string
	want *fileInfo
	err  string
}{
	{
		in: "debian-binary   1385068169  0     0     100644  4         `\n",
		want: &fileInfo{
			name:  "debian-binary",
			mtime: time.Unix(1385068169, 0),
			mode:  os.FileMode(0100644) & os.ModePerm,
			size:  4,
		},
	},
	{
		in: "debian-binary   1385068169  0     0     644     4         `\n",
		want: &fileInfo{
			name:  "debian-binary",
			mtime: time.Unix(1385068169, 0),
			mode:  os.FileMode(0644),
			size:  4,
		},
	},
	{
		in:  "debian-binary   1385068169  0     0     120644  4         `\n",
		err: "feature not implemented: non-regular files",
	},
	{
		in:  "debian-binary   1385068169  0     0     220644  4         `\n",
		err: "corrupt archive: invalid file mode",
	},
}

func TestReadFileHeader(t *testing.T) {
	for i, test := range testCommonFileHeaders {
		got, err := parseFileHeader([]byte(test.in))
		switch {
		case err == nil && test.err != "":
			t.Errorf("%d: got no err, expected err %v", i, test.err)
			continue
		case err != nil && test.err != err.Error():
			t.Errorf("%d: got err %q, expected err %q", i, err, test.err)
			continue
		case err == nil && test.err == "":
			// no error as expected
		case err != nil && test.err == err.Error():
			t.Logf("%d: got expected error %q", i, err)
			continue
		}

		if !reflect.DeepEqual(got, test.want) {
			t.Errorf("%d: got %#v, expected %+v", i, got, test.want)
		} else {
			t.Logf("%d: got %#v", i, got)
		}
	}
}

func BenchmarkReadFileHeader(b *testing.B) {
	fh := []byte("debian-binary   1385068169  0     0     100644  4         `\n")
	b.ReportAllocs()
	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		_, _ = parseFileHeader(fh)
		b.SetBytes(60)
	}

}

var testMagic = []struct {
	in   string
	want error
}{
	{
		in: magic,
	},
	{
		in:   strings.Repeat("a", len(magic)),
		want: CorruptArchiveError("global archive header not found"),
	},
	{
		in:   "!",
		want: io.ErrUnexpectedEOF,
	},
	{
		in:   "",
		want: io.EOF,
	},
}

func TestReadMagic(t *testing.T) {
	for i, test := range testMagic {
		got := checkMagic(strings.NewReader(test.in))
		if !reflect.DeepEqual(got, test.want) {
			t.Errorf("%d: got %#v, expected %+v", i, got, test.want)
		} else {
			t.Logf("%d: got %#v", i, got)
		}
	}
}

func TestFileInfo(t *testing.T) {
	test := &fileInfo{
		name:  "debian-binary",
		mtime: time.Unix(1385068169, 0),
		mode:  os.FileMode(0644),
		size:  4,
	}

	if test.IsDir() != false {
		t.Error("IsDir")
	}
	if test.Mode() != os.FileMode(0644) {
		t.Error("Mode")
	}
	if test.ModTime() != time.Unix(1385068169, 0) {
		t.Error("ModTime")
	}
	if test.Name() != "debian-binary" {
		t.Error("Name")
	}
	if test.Size() != 4 {
		t.Error("Size")
	}
	if test.Sys() != nil {
		t.Error("Sys")
	}
}

func TestReaderBasics(t *testing.T) {
	test := strings.NewReader(testCommon)
	r := NewReader(test)
	fi, err := r.Next()
	if err != nil {
		t.Fatal(err)
		return
	}
	if fi.Mode() != os.FileMode(0644) {
		t.Error("Mode")
	}
	if fi.Name() != "debian-binary" {
		t.Error("Name")
	}
	if fi.Size() != 4 {
		t.Error("Size")
	}
	if fi.ModTime() != time.Unix(1385068169, 0) {
		t.Error("ModTime")
	}

	if content, err := ioutil.ReadAll(r); err != nil {
		t.Error(err)
	} else if string(content) != "2.0\n" {
		t.Error("Content")
	}
	fi, err = r.Next()
	if err != nil {
		t.Fatal(err)
		return
	}
	if fi.Mode() != os.FileMode(0644) {
		t.Error("Mode2")
	}
	if fi.Name() != "control.tar.gz" {
		t.Error("Name2")
	}
	if fi.Size() != 0 {
		t.Error("Size2")
	}
	if fi.ModTime() != time.Unix(1385068169, 0) {
		t.Error("ModTime2")
	}

	if content, err := ioutil.ReadAll(r); err != nil {
		t.Error(err)
	} else if string(content) != "" {
		t.Error("Content2")
	}

	fi, err = r.Next()
	if err != io.EOF {
		t.Errorf("expected EOF, got %v", err)
	}
}

func BenchmarkReaderBigFiles(b *testing.B) {
	benchmarkReader(b, 8, 8*1024*1024)
}

func BenchmarkReaderManySmallFiles(b *testing.B) {
	benchmarkReader(b, 1024, 8)
}

func genArchiveFile(meta os.FileInfo) []byte {
	ret := make([]byte, 60+meta.Size(), 60+meta.Size())
	for i := 0; i < 60; i++ {
		ret[i] = ' '
	}
	copy(ret[0:], meta.Name())
	copy(ret[16:], strconv.FormatInt(meta.ModTime().Unix(), 10))
	copy(ret[28:], "0")
	copy(ret[34:], "0")
	copy(ret[40:], strconv.FormatUint(uint64(meta.Mode()), 8))
	copy(ret[48:], strconv.FormatInt(meta.Size(), 10))
	copy(ret[58:], filemagic)
	return ret
}

func benchmarkReader(b *testing.B, numFiles int, sizeFiles int64) {
	buf := bytes.NewBufferString(magic)
	buf.Grow(numFiles * (int(sizeFiles) + 60))
	for i := 0; i < numFiles; i++ {
		buf.Write(genArchiveFile(&fileInfo{
			name:  strconv.Itoa(1000 + i),
			mtime: time.Unix(int64(i), 0),
			size:  sizeFiles,
			mode:  os.FileMode(0640),
		}))

	}
	test := bytes.NewReader(buf.Bytes())
	r := NewReader(test)

	var err error
	var read int64

	b.ReportAllocs()
	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		for j := 0; j < numFiles; j++ {
			_, err = r.Next()
			if err != nil {
				b.Fatal(err)
			}
			read += 60
			n, err := io.Copy(ioutil.Discard, r)
			if err != nil {
				b.Fatal(err)
			}
			read += n
		}
		b.SetBytes(read)
		read = 0
		test.Seek(0, 0)
		r.Reset(test)
	}
}

func TestWriterBasics(t *testing.T) {
	b := new(bytes.Buffer)
	b.Grow(len(testCommon))
	w := NewWriter(b)
	debian := &fileInfo{
		name:  "debian-binary",
		mtime: time.Unix(1385068169, 0),
		mode:  os.FileMode(0100644) & os.ModePerm,
		size:  4,
	}
	if _, err := w.WriteFile(debian, strings.NewReader("2.0\n")); err != nil {
		t.Error(err)
		return
	}

	control := &fileInfo{
		name:  "control.tar.gz",
		mtime: time.Unix(1385068169, 0),
		mode:  os.FileMode(0100644) & os.ModePerm,
		size:  0,
	}
	if _, err := w.WriteFile(control, strings.NewReader("")); err != nil {
		t.Error(err)
		return
	}

	if archive := b.String(); archive != testCommon {
		t.Errorf("got\n%q\nwant\n%q", archive, testCommon)
	}
}

type fileEntry struct {
	meta  *fileInfo
	input *bytes.Reader
}

type fileSet []*fileEntry

func benchmarkWriter(b *testing.B, numFiles int, sizeFiles int64) {
	zero := make([]byte, len(magic)+numFiles*(len(filemagic)+int(sizeFiles)))
	fs := make(fileSet, 0, numFiles)
	for i := 0; i < numFiles; i++ {
		fs = append(fs, &fileEntry{
			meta: &fileInfo{
				name:  strconv.Itoa(1000 + i),
				mtime: time.Unix(int64(i), 0),
				size:  sizeFiles,
				mode:  os.FileMode(0640),
			},
			input: bytes.NewReader(zero),
		})

	}

	dest := new(bytes.Buffer)
	dest.Grow(len(magic) + numFiles*(len(filemagic)+int(sizeFiles)))
	w := NewWriter(dest)

	var written int64

	b.ReportAllocs()
	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		for _, fe := range fs {
			meta, input := fe.meta, fe.input
			input.Seek(0, 0)
			n, err := w.WriteFile(meta, input)
			if err != nil {
				b.Fatal(err)
				return
			}
			written += n
		}
		b.SetBytes(written)
		written = 0
		dest.Reset()
		w.Reset(dest)
	}
}

func BenchmarkWriterBigFiles(b *testing.B) {
	benchmarkWriter(b, 8, 8*1024*1024)
}

func BenchmarkWriterManySmallFiles(b *testing.B) {
	benchmarkWriter(b, 1024, 8)
}
<|endoftext|>


<commit_before>package command

import (
	"bytes"
	"encoding/json"
	"fmt"
	"reflect"
	"strings"

	"github.com/domonda/errors"
	reflection "github.com/ungerik/go-reflection"
)

var WithoutArgs ArgsDef

// ArgsDef implements Args
type ArgsDef struct {
	outerStructType reflect.Type
	argStructFields []reflection.StructFieldName
	argInfos        []Arg
	initialized     bool
}

func (def *ArgsDef) NumArgs() int {
	return len(def.argInfos)
}

func (def *ArgsDef) Args() []Arg {
	return def.argInfos
}

func (def *ArgsDef) ArgTag(index int, tag string) string {
	return def.argStructFields[index].Field.Tag.Get(tag)
}

func (def *ArgsDef) String() string {
	if !def.initialized {
		return "ArgsDef not initialized"
	}
	var b strings.Builder
	for _, arg := range def.argInfos {
		if b.Len() > 0 {
			b.WriteByte(' ')
		}
		fmt.Fprintf(&b, "<%s:%s>", arg.Name, reflection.DerefType(arg.Type))
	}
	return b.String()
}

// Init initializes ArgsDef with the reflection data from
// outerStructPtr wich has to be the address of the struct
// variable that embedds ArgsDef.
func (def *ArgsDef) Init(outerStructPtr interface{}) error {
	if def.initialized {
		return nil
	}

	if _, ok := outerStructPtr.(Args); !ok {
		return errors.Errorf("outerStructPtr of type %T does not implement interface Args", outerStructPtr)
	}

	def.outerStructType = reflection.DerefType(reflect.TypeOf(outerStructPtr))
	if def.outerStructType.Kind() != reflect.Struct {
		return errors.Errorf("ArgsDef must be contained in a struct, but outer type is %s", def.outerStructType)
	}

	def.argStructFields = reflection.FlatExportedStructFieldNames(def.outerStructType, ArgNameTag)

	def.argInfos = make([]Arg, len(def.argStructFields))
	for i := range def.argInfos {
		def.argInfos[i].Name = def.argStructFields[i].Name
		def.argInfos[i].Description = def.ArgTag(i, ArgDescriptionTag)
		def.argInfos[i].Type = def.argStructFields[i].Field.Type
	}

	def.initialized = true
	return nil
}

func (def *ArgsDef) argValsFromStringArgs(callerArgs []string) ([]reflect.Value, error) {
	// Allocate a new args struct because we need addressable
	// variables of struct field types to hold arg values.
	// Instead of new individual variable use fields of args struct.
	argsStruct := reflect.New(def.outerStructType).Elem()
	argVals := make([]reflect.Value, def.NumArgs())
	numStringArgs := len(callerArgs)
	for i := range argVals {
		argVals[i] = argsStruct.FieldByIndex(def.argStructFields[i].Field.Index)
		if i >= numStringArgs {
			continue
		}
		err := assignString(argVals[i], callerArgs[i])
		if err != nil {
			return nil, err
		}
	}
	return argVals, nil
}

func (def *ArgsDef) argValsFromStringMapArgs(callerArgs map[string]string) ([]reflect.Value, error) {
	// Allocate a new args struct because we need addressable
	// variables of struct field types to hold arg values.
	// Instead of new individual variable use fields of args struct.
	argsStruct := reflect.New(def.outerStructType).Elem()
	argVals := make([]reflect.Value, def.NumArgs())
	for i := range argVals {
		argVals[i] = argsStruct.FieldByIndex(def.argStructFields[i].Field.Index)
		argName := def.argStructFields[i].Name
		stringArg, hasArg := callerArgs[argName]
		if !hasArg {
			continue
		}
		err := assignString(argVals[i], stringArg)
		if err != nil {
			return nil, err
		}
	}
	return argVals, nil
}

func (def *ArgsDef) argValsFromMapArgs(callerArgs map[string]interface{}) ([]reflect.Value, error) {
	// Allocate a new args struct because we need addressable
	// variables of struct field types to hold arg values.
	// Instead of new individual variable use fields of args struct.
	argsStruct := reflect.New(def.outerStructType).Elem()
	argVals := make([]reflect.Value, def.NumArgs())
	for i := range argVals {
		argVals[i] = argsStruct.FieldByIndex(def.argStructFields[i].Field.Index)
		argName := def.argStructFields[i].Name
		varArg, hasArg := callerArgs[argName]
		if !hasArg {
			continue
		}
		err := assignAny(argVals[i], varArg)
		if err != nil {
			return nil, err
		}
	}
	return argVals, nil
}

func (def *ArgsDef) argValsFromJSON(callerArgs []byte) ([]reflect.Value, error) {
	callerArgs = bytes.TrimSpace(callerArgs)
	if len(callerArgs) < 2 {
		return nil, errors.Errorf("Invalid JSON: '%s'", string(callerArgs))
	}

	// Handle JSON array
	if callerArgs[0] == '[' {
		var callerArray []interface{}
		err := json.Unmarshal(callerArgs, &callerArray)
		if err != nil {
			return nil, err
		}
		argsStruct := reflect.New(def.outerStructType).Elem()
		argVals := make([]reflect.Value, def.NumArgs())
		for i := range argVals {
			argVals[i] = argsStruct.FieldByIndex(def.argStructFields[i].Field.Index)
			if i < len(callerArray) {
				err := assignAny(argVals[i], callerArray[i])
				if err != nil {
					return nil, err
				}
			}
		}
		return argVals, nil
	}

	// Unmarshal callerArgs JSON to new args struct
	argsStructPtr := reflect.New(def.outerStructType)
	err := json.Unmarshal(callerArgs, argsStructPtr.Interface())
	if err != nil {
		return nil, err
	}

	argsStruct := argsStructPtr.Elem()
	argVals := make([]reflect.Value, def.NumArgs())
	for i := range argVals {
		argVals[i] = argsStruct.FieldByIndex(def.argStructFields[i].Field.Index)
	}
	return argVals, nil
}

func (def *ArgsDef) StringArgsFunc(commandFunc interface{}, resultsHandlers []ResultsHandler) (StringArgsFunc, error) {
	dispatcher, err := newFuncDispatcher(def, commandFunc)
	if err != nil {
		return nil, err
	}

	return func(callerArgs ...string) error {
		argVals, err := def.argValsFromStringArgs(callerArgs)
		if err != nil {
			return err
		}
		return dispatcher.callWithResultsHandlers(argVals, resultsHandlers)
	}, nil
}

func (def *ArgsDef) StringMapArgsFunc(commandFunc interface{}, resultsHandlers []ResultsHandler) (StringMapArgsFunc, error) {
	dispatcher, err := newFuncDispatcher(def, commandFunc)
	if err != nil {
		return nil, err
	}

	return func(callerArgs map[string]string) (err error) {
		argVals, err := def.argValsFromStringMapArgs(callerArgs)
		if err != nil {
			return err
		}
		return dispatcher.callWithResultsHandlers(argVals, resultsHandlers)
	}, nil
}

func (def *ArgsDef) MapArgsFunc(commandFunc interface{}, resultsHandlers []ResultsHandler) (MapArgsFunc, error) {
	dispatcher, err := newFuncDispatcher(def, commandFunc)
	if err != nil {
		return nil, err
	}

	return func(callerArgs map[string]interface{}) (err error) {
		argVals, err := def.argValsFromMapArgs(callerArgs)
		if err != nil {
			return err
		}
		return dispatcher.callWithResultsHandlers(argVals, resultsHandlers)
	}, nil
}

func (def *ArgsDef) JSONArgsFunc(commandFunc interface{}, resultsHandlers []ResultsHandler) (JSONArgsFunc, error) {
	dispatcher, err := newFuncDispatcher(def, commandFunc)
	if err != nil {
		return nil, err
	}

	return func(callerArgs []byte) (err error) {
		argVals, err := def.argValsFromJSON(callerArgs)
		if err != nil {
			return err
		}
		return dispatcher.callWithResultsHandlers(argVals, resultsHandlers)
	}, nil
}

func (def *ArgsDef) StringArgsResultValuesFunc(commandFunc interface{}) (StringArgsResultValuesFunc, error) {
	dispatcher, err := newFuncDispatcher(def, commandFunc)
	if err != nil {
		return nil, err
	}

	return func(args []string) ([]reflect.Value, error) {
		argVals, err := def.argValsFromStringArgs(args)
		if err != nil {
			return nil, err
		}
		return dispatcher.callAndReturnResults(argVals)
	}, nil
}

func (def *ArgsDef) StringMapArgsResultValuesFunc(commandFunc interface{}) (StringMapArgsResultValuesFunc, error) {
	dispatcher, err := newFuncDispatcher(def, commandFunc)
	if err != nil {
		return nil, err
	}

	return func(args map[string]string) ([]reflect.Value, error) {
		argVals, err := def.argValsFromStringMapArgs(args)
		if err != nil {
			return nil, err
		}
		return dispatcher.callAndReturnResults(argVals)
	}, nil
}

func (def *ArgsDef) MapArgsResultValuesFunc(commandFunc interface{}) (MapArgsResultValuesFunc, error) {
	dispatcher, err := newFuncDispatcher(def, commandFunc)
	if err != nil {
		return nil, err
	}

	return func(args map[string]interface{}) ([]reflect.Value, error) {
		argVals, err := def.argValsFromMapArgs(args)
		if err != nil {
			return nil, err
		}
		return dispatcher.callAndReturnResults(argVals)
	}, nil
}

func (def *ArgsDef) JSONArgsResultValuesFunc(commandFunc interface{}) (JSONArgsResultValuesFunc, error) {
	dispatcher, err := newFuncDispatcher(def, commandFunc)
	if err != nil {
		return nil, err
	}

	return func(args []byte) ([]reflect.Value, error) {
		argVals, err := def.argValsFromJSON(args)
		if err != nil {
			return nil, err
		}
		return dispatcher.callAndReturnResults(argVals)
	}, nil
}
<commit_msg>comments<commit_after>package command

import (
	"bytes"
	"encoding/json"
	"fmt"
	"reflect"
	"strings"

	"github.com/domonda/errors"
	reflection "github.com/ungerik/go-reflection"
)

var WithoutArgs ArgsDef

// ArgsDef implements Args
type ArgsDef struct {
	outerStructType reflect.Type
	argStructFields []reflection.StructFieldName
	argInfos        []Arg
	initialized     bool
}

func (def *ArgsDef) NumArgs() int {
	return len(def.argInfos)
}

func (def *ArgsDef) Args() []Arg {
	return def.argInfos
}

func (def *ArgsDef) ArgTag(index int, tag string) string {
	return def.argStructFields[index].Field.Tag.Get(tag)
}

// String implements the fmt.Stringer interface.
func (def *ArgsDef) String() string {
	if !def.initialized {
		return "ArgsDef not initialized"
	}
	var b strings.Builder
	for _, arg := range def.argInfos {
		if b.Len() > 0 {
			b.WriteByte(' ')
		}
		fmt.Fprintf(&b, "<%s:%s>", arg.Name, reflection.DerefType(arg.Type))
	}
	return b.String()
}

// Init initializes ArgsDef with the reflection data from
// outerStructPtr wich has to be the address of the struct
// variable that embedds ArgsDef.
func (def *ArgsDef) Init(outerStructPtr interface{}) error {
	if def.initialized {
		return nil
	}

	if _, ok := outerStructPtr.(Args); !ok {
		return errors.Errorf("outerStructPtr of type %T does not implement interface Args", outerStructPtr)
	}

	def.outerStructType = reflection.DerefType(reflect.TypeOf(outerStructPtr))
	if def.outerStructType.Kind() != reflect.Struct {
		return errors.Errorf("ArgsDef must be contained in a struct, but outer type is %s", def.outerStructType)
	}

	def.argStructFields = reflection.FlatExportedStructFieldNames(def.outerStructType, ArgNameTag)

	def.argInfos = make([]Arg, len(def.argStructFields))
	for i := range def.argInfos {
		def.argInfos[i].Name = def.argStructFields[i].Name
		def.argInfos[i].Description = def.ArgTag(i, ArgDescriptionTag)
		def.argInfos[i].Type = def.argStructFields[i].Field.Type
	}

	def.initialized = true
	return nil
}

func (def *ArgsDef) argValsFromStringArgs(callerArgs []string) ([]reflect.Value, error) {
	// Allocate a new args struct because we need addressable
	// variables of struct field types to hold arg values.
	// Instead of new individual variable use fields of args struct.
	argsStruct := reflect.New(def.outerStructType).Elem()
	argVals := make([]reflect.Value, def.NumArgs())
	numStringArgs := len(callerArgs)
	for i := range argVals {
		argVals[i] = argsStruct.FieldByIndex(def.argStructFields[i].Field.Index)
		if i >= numStringArgs {
			continue
		}
		err := assignString(argVals[i], callerArgs[i])
		if err != nil {
			return nil, err
		}
	}
	return argVals, nil
}

func (def *ArgsDef) argValsFromStringMapArgs(callerArgs map[string]string) ([]reflect.Value, error) {
	// Allocate a new args struct because we need addressable
	// variables of struct field types to hold arg values.
	// Instead of new individual variable use fields of args struct.
	argsStruct := reflect.New(def.outerStructType).Elem()
	argVals := make([]reflect.Value, def.NumArgs())
	for i := range argVals {
		argVals[i] = argsStruct.FieldByIndex(def.argStructFields[i].Field.Index)
		argName := def.argStructFields[i].Name
		stringArg, hasArg := callerArgs[argName]
		if !hasArg {
			continue
		}
		err := assignString(argVals[i], stringArg)
		if err != nil {
			return nil, err
		}
	}
	return argVals, nil
}

func (def *ArgsDef) argValsFromMapArgs(callerArgs map[string]interface{}) ([]reflect.Value, error) {
	// Allocate a new args struct because we need addressable
	// variables of struct field types to hold arg values.
	// Instead of new individual variable use fields of args struct.
	argsStruct := reflect.New(def.outerStructType).Elem()
	argVals := make([]reflect.Value, def.NumArgs())
	for i := range argVals {
		argVals[i] = argsStruct.FieldByIndex(def.argStructFields[i].Field.Index)
		argName := def.argStructFields[i].Name
		varArg, hasArg := callerArgs[argName]
		if !hasArg {
			continue
		}
		err := assignAny(argVals[i], varArg)
		if err != nil {
			return nil, err
		}
	}
	return argVals, nil
}

func (def *ArgsDef) argValsFromJSON(callerArgs []byte) ([]reflect.Value, error) {
	callerArgs = bytes.TrimSpace(callerArgs)
	if len(callerArgs) < 2 {
		return nil, errors.Errorf("Invalid JSON: '%s'", string(callerArgs))
	}

	// Handle JSON array
	if callerArgs[0] == '[' {
		var callerArray []interface{}
		err := json.Unmarshal(callerArgs, &callerArray)
		if err != nil {
			return nil, err
		}
		argsStruct := reflect.New(def.outerStructType).Elem()
		argVals := make([]reflect.Value, def.NumArgs())
		for i := range argVals {
			argVals[i] = argsStruct.FieldByIndex(def.argStructFields[i].Field.Index)
			if i < len(callerArray) {
				err := assignAny(argVals[i], callerArray[i])
				if err != nil {
					return nil, err
				}
			}
		}
		return argVals, nil
	}

	// Unmarshal callerArgs JSON to new args struct
	argsStructPtr := reflect.New(def.outerStructType)
	err := json.Unmarshal(callerArgs, argsStructPtr.Interface())
	if err != nil {
		return nil, err
	}

	argsStruct := argsStructPtr.Elem()
	argVals := make([]reflect.Value, def.NumArgs())
	for i := range argVals {
		argVals[i] = argsStruct.FieldByIndex(def.argStructFields[i].Field.Index)
	}
	return argVals, nil
}

func (def *ArgsDef) StringArgsFunc(commandFunc interface{}, resultsHandlers []ResultsHandler) (StringArgsFunc, error) {
	dispatcher, err := newFuncDispatcher(def, commandFunc)
	if err != nil {
		return nil, err
	}

	return func(callerArgs ...string) error {
		argVals, err := def.argValsFromStringArgs(callerArgs)
		if err != nil {
			return err
		}
		return dispatcher.callWithResultsHandlers(argVals, resultsHandlers)
	}, nil
}

func (def *ArgsDef) StringMapArgsFunc(commandFunc interface{}, resultsHandlers []ResultsHandler) (StringMapArgsFunc, error) {
	dispatcher, err := newFuncDispatcher(def, commandFunc)
	if err != nil {
		return nil, err
	}

	return func(callerArgs map[string]string) (err error) {
		argVals, err := def.argValsFromStringMapArgs(callerArgs)
		if err != nil {
			return err
		}
		return dispatcher.callWithResultsHandlers(argVals, resultsHandlers)
	}, nil
}

func (def *ArgsDef) MapArgsFunc(commandFunc interface{}, resultsHandlers []ResultsHandler) (MapArgsFunc, error) {
	dispatcher, err := newFuncDispatcher(def, commandFunc)
	if err != nil {
		return nil, err
	}

	return func(callerArgs map[string]interface{}) (err error) {
		argVals, err := def.argValsFromMapArgs(callerArgs)
		if err != nil {
			return err
		}
		return dispatcher.callWithResultsHandlers(argVals, resultsHandlers)
	}, nil
}

func (def *ArgsDef) JSONArgsFunc(commandFunc interface{}, resultsHandlers []ResultsHandler) (JSONArgsFunc, error) {
	dispatcher, err := newFuncDispatcher(def, commandFunc)
	if err != nil {
		return nil, err
	}

	return func(callerArgs []byte) (err error) {
		argVals, err := def.argValsFromJSON(callerArgs)
		if err != nil {
			return err
		}
		return dispatcher.callWithResultsHandlers(argVals, resultsHandlers)
	}, nil
}

func (def *ArgsDef) StringArgsResultValuesFunc(commandFunc interface{}) (StringArgsResultValuesFunc, error) {
	dispatcher, err := newFuncDispatcher(def, commandFunc)
	if err != nil {
		return nil, err
	}

	return func(args []string) ([]reflect.Value, error) {
		argVals, err := def.argValsFromStringArgs(args)
		if err != nil {
			return nil, err
		}
		return dispatcher.callAndReturnResults(argVals)
	}, nil
}

func (def *ArgsDef) StringMapArgsResultValuesFunc(commandFunc interface{}) (StringMapArgsResultValuesFunc, error) {
	dispatcher, err := newFuncDispatcher(def, commandFunc)
	if err != nil {
		return nil, err
	}

	return func(args map[string]string) ([]reflect.Value, error) {
		argVals, err := def.argValsFromStringMapArgs(args)
		if err != nil {
			return nil, err
		}
		return dispatcher.callAndReturnResults(argVals)
	}, nil
}

func (def *ArgsDef) MapArgsResultValuesFunc(commandFunc interface{}) (MapArgsResultValuesFunc, error) {
	dispatcher, err := newFuncDispatcher(def, commandFunc)
	if err != nil {
		return nil, err
	}

	return func(args map[string]interface{}) ([]reflect.Value, error) {
		argVals, err := def.argValsFromMapArgs(args)
		if err != nil {
			return nil, err
		}
		return dispatcher.callAndReturnResults(argVals)
	}, nil
}

func (def *ArgsDef) JSONArgsResultValuesFunc(commandFunc interface{}) (JSONArgsResultValuesFunc, error) {
	dispatcher, err := newFuncDispatcher(def, commandFunc)
	if err != nil {
		return nil, err
	}

	return func(args []byte) ([]reflect.Value, error) {
		argVals, err := def.argValsFromJSON(args)
		if err != nil {
			return nil, err
		}
		return dispatcher.callAndReturnResults(argVals)
	}, nil
}
<|endoftext|>


<commit_before>package configserver

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"github.com/concourse/concourse/atc/exec"
	"io"
	"io/ioutil"
	"mime"
	"mime/multipart"
	"net/http"
	"strings"

	boshtemplate "github.com/cloudfoundry/bosh-cli/director/template"

	"code.cloudfoundry.org/lager"
	"github.com/concourse/concourse/atc"
	"github.com/concourse/concourse/atc/creds"
	"github.com/concourse/concourse/atc/db"
	"github.com/hashicorp/go-multierror"
	"github.com/mitchellh/mapstructure"
	"github.com/tedsuo/rata"
	"gopkg.in/yaml.v2"
)

var (
	ErrStatusUnsupportedMediaType = errors.New("content-type is not supported")
	ErrCannotParseContentType     = errors.New("content-type header could not be parsed")
	ErrMalformedRequestPayload    = errors.New("data in body could not be decoded")
	ErrFailedToConstructDecoder   = errors.New("decoder could not be constructed")
	ErrCouldNotDecode             = errors.New("data could not be decoded into config structure")
	ErrInvalidPausedValue         = errors.New("invalid paused value")
)

type ExtraKeysError struct {
	extraKeys []string
}

func (eke ExtraKeysError) Error() string {
	msg := &bytes.Buffer{}

	fmt.Fprintln(msg, "unknown/extra keys:")
	for _, unusedKey := range eke.extraKeys {
		fmt.Fprintf(msg, "  - %s\n", unusedKey)
	}

	return msg.String()
}

type SaveConfigResponse struct {
	Errors   []string      `json:"errors,omitempty"`
	Warnings []atc.Warning `json:"warnings,omitempty"`
}

func (s *Server) SaveConfig(w http.ResponseWriter, r *http.Request) {
	session := s.logger.Session("set-config")

	query := r.URL.Query()

	checkCredentials := false
	if _, exists := query[atc.SaveConfigCheckCreds]; exists {
		checkCredentials = true
	}

	var version db.ConfigVersion
	if configVersionStr := r.Header.Get(atc.ConfigVersionHeader); len(configVersionStr) != 0 {
		_, err := fmt.Sscanf(configVersionStr, "%d", &version)
		if err != nil {
			session.Error("malformed-config-version", err)
			s.handleBadRequest(w, []string{fmt.Sprintf("config version is malformed: %s", err)}, session)
			return
		}
	}

	config, pausedState, err := saveConfigRequestUnmarshaler(r)
	switch err {
	case ErrStatusUnsupportedMediaType:
		w.WriteHeader(http.StatusUnsupportedMediaType)
		return
	case ErrMalformedRequestPayload:
		session.Error("malformed-request-payload", err, lager.Data{
			"content-type": r.Header.Get("Content-Type"),
		})

		s.handleBadRequest(w, []string{"malformed config"}, session)
		return
	case ErrFailedToConstructDecoder:
		session.Error("failed-to-construct-decoder", err)
		w.WriteHeader(http.StatusInternalServerError)
		return
	case ErrCouldNotDecode:
		session.Error("could-not-decode", err)
		s.handleBadRequest(w, []string{"failed to decode config"}, session)
		return
	case ErrInvalidPausedValue:
		session.Error("invalid-paused-value", err)
		s.handleBadRequest(w, []string{"invalid paused value"}, session)
		return
	default:
		if err != nil {
			if eke, ok := err.(ExtraKeysError); ok {
				s.handleBadRequest(w, []string{eke.Error()}, session)
			} else {
				session.Error("unexpected-error", err)
				w.WriteHeader(http.StatusInternalServerError)
			}

			return
		}
	}

	warnings, errorMessages := config.Validate()
	if len(errorMessages) > 0 {
		session.Error("ignoring-invalid-config", err)
		s.handleBadRequest(w, errorMessages, session)
		return
	}

	pipelineName := rata.Param(r, "pipeline_name")
	teamName := rata.Param(r, "team_name")

	if checkCredentials {
		variables := s.variablesFactory.NewVariables(teamName, pipelineName)

		errs := validateCredParams(variables, config, session)
		if errs != nil {
			s.handleBadRequest(w, []string{errs.Error()}, session)
			return
		}
	}

	session.Info("saving")

	team, found, err := s.teamFactory.FindTeam(teamName)
	if err != nil {
		session.Error("failed-to-find-team", err)
		w.WriteHeader(http.StatusInternalServerError)
		return
	}

	if !found {
		session.Debug("team-not-found")
		w.WriteHeader(http.StatusInternalServerError)
		return
	}

	_, created, err := team.SavePipeline(pipelineName, config, version, pausedState)
	if err != nil {
		session.Error("failed-to-save-config", err)
		w.WriteHeader(http.StatusInternalServerError)
		fmt.Fprintf(w, "failed to save config: %s", err)
		return
	}

	session.Info("saved")

	w.Header().Set("Content-Type", "application/json")

	if created {
		w.WriteHeader(http.StatusCreated)
	} else {
		w.WriteHeader(http.StatusOK)
	}

	s.writeSaveConfigResponse(w, SaveConfigResponse{Warnings: warnings}, session)
}

// Simply validate that the credentials exist; don't do anything with the actual secrets
func validateCredParams(credMgrVars creds.Variables, config atc.Config, session lager.Logger) error {
	var errs error

	for _, resourceType := range config.ResourceTypes {
		_, err := creds.NewSource(credMgrVars, resourceType.Source).Evaluate()
		if err != nil {
			errs = multierror.Append(errs, err)
		}
	}

	for _, resource := range config.Resources {
		_, err := creds.NewSource(credMgrVars, resource.Source).Evaluate()
		if err != nil {
			errs = multierror.Append(errs, err)
		}

		_, err = creds.NewString(credMgrVars, resource.WebhookToken).Evaluate()
		if err != nil {
			errs = multierror.Append(errs, err)
		}
	}

	for _, job := range config.Jobs {
		for _, plan := range job.Plan {
			_, err := creds.NewParams(credMgrVars, plan.Params).Evaluate()
			if err != nil {
				errs = multierror.Append(errs, err)
			}

			if plan.TaskConfigPath != "" {
				// external task - we can't really validate much right now, because task yaml will be
				// retrieved in runtime during job execution. but we can validate vars and params which will be
				// passed to this task
				err = creds.NewTaskParamsValidator(credMgrVars, plan.TaskConfig.Params).Validate()
				if err != nil {
					errs = multierror.Append(errs, err)
				}

				err = creds.NewTaskVarsValidator(credMgrVars, plan.TaskVars).Validate()
				if err != nil {
					errs = multierror.Append(errs, err)
				}

			} else if plan.TaskConfig != nil {
				// embedded task - we can fully validate it, interpolating with cred mgr variables
				var taskConfigSource exec.TaskConfigSource
				embeddedTaskVars := []boshtemplate.Variables{credMgrVars}
				embeddedTaskParams := atc.Params{}
				taskConfigSource = exec.StaticConfigSource{Config: plan.TaskConfig, Vars: embeddedTaskVars}
				taskConfigSource = exec.OverrideParamsConfigSource{ConfigSource: taskConfigSource, Params: embeddedTaskParams}
				taskConfigSource = exec.ValidatingConfigSource{ConfigSource: taskConfigSource}
				_, err = taskConfigSource.FetchConfig(session, nil)
				if err != nil {
					errs = multierror.Append(errs, err)
				}
			}

		}
	}

	if errs != nil {
		session.Info("config-has-invalid-creds", lager.Data{"errors": errs.Error()})
	}

	return errs
}

func (s *Server) handleBadRequest(w http.ResponseWriter, errorMessages []string, session lager.Logger) {
	w.Header().Set("Content-Type", "application/json")
	w.WriteHeader(http.StatusBadRequest)
	s.writeSaveConfigResponse(w, SaveConfigResponse{
		Errors: errorMessages,
	}, session)
}

func (s *Server) writeSaveConfigResponse(w http.ResponseWriter, saveConfigResponse SaveConfigResponse, session lager.Logger) {
	responseJSON, err := json.Marshal(saveConfigResponse)
	if err != nil {
		session.Error("failed-to-marshal-validation-response", err)
		w.WriteHeader(http.StatusInternalServerError)
		fmt.Fprintf(w, "failed to generate error response: %s", err)
		return
	}

	_, err = w.Write(responseJSON)
	if err != nil {
		session.Error("failed-to-write-validation-response", err)
		w.WriteHeader(http.StatusInternalServerError)
		return
	}
}

func requestToConfig(contentType string, requestBody io.ReadCloser, configStructure interface{}) (db.PipelinePausedState, error) {
	pausedState := db.PipelineNoChange

	mediaType, params, err := mime.ParseMediaType(contentType)
	if err != nil {
		return db.PipelineNoChange, ErrCannotParseContentType
	}

	switch mediaType {
	case "application/json":
		err := json.NewDecoder(requestBody).Decode(configStructure)
		if err != nil {
			return db.PipelineNoChange, ErrMalformedRequestPayload
		}

	case "application/x-yaml":
		body, err := ioutil.ReadAll(requestBody)
		if err == nil {
			err = yaml.Unmarshal(body, configStructure)
		}

		if err != nil {
			return db.PipelineNoChange, ErrMalformedRequestPayload
		}

	case "multipart/form-data":
		multipartReader := multipart.NewReader(requestBody, params["boundary"])

		for {
			part, err := multipartReader.NextPart()

			if err == io.EOF {
				break
			}

			if err != nil {
				return db.PipelineNoChange, err
			}

			if part.FormName() == "paused" {
				pausedValue, err := ioutil.ReadAll(part)
				if err != nil {
					return db.PipelineNoChange, err
				}

				if string(pausedValue) == "true" {
					pausedState = db.PipelinePaused
				} else if string(pausedValue) == "false" {
					pausedState = db.PipelineUnpaused
				} else {
					return db.PipelineNoChange, ErrInvalidPausedValue
				}
			} else {
				partContentType := part.Header.Get("Content-type")
				_, err := requestToConfig(partContentType, part, configStructure)
				if err != nil {
					return db.PipelineNoChange, ErrMalformedRequestPayload
				}
			}
		}
	default:
		return db.PipelineNoChange, ErrStatusUnsupportedMediaType
	}

	return pausedState, nil
}

func saveConfigRequestUnmarshaler(r *http.Request) (atc.Config, db.PipelinePausedState, error) {
	var configStructure interface{}
	pausedState, err := requestToConfig(r.Header.Get("Content-Type"), r.Body, &configStructure)
	if err != nil {
		return atc.Config{}, db.PipelineNoChange, err
	}

	var config atc.Config
	var md mapstructure.Metadata
	msConfig := &mapstructure.DecoderConfig{
		Metadata:         &md,
		Result:           &config,
		WeaklyTypedInput: true,
		DecodeHook: mapstructure.ComposeDecodeHookFunc(
			atc.SanitizeDecodeHook,
			atc.VersionConfigDecodeHook,
			atc.InputsConfigDecodeHook,
			atc.ContainerLimitsDecodeHook,
		),
	}

	decoder, err := mapstructure.NewDecoder(msConfig)
	if err != nil {
		return atc.Config{}, db.PipelineNoChange, ErrFailedToConstructDecoder
	}

	if err := decoder.Decode(configStructure); err != nil {
		return atc.Config{}, db.PipelineNoChange, ErrCouldNotDecode
	}

	nestedUnused := []string{}
	for _, unused := range md.Unused {
		if strings.Contains(unused, ".") {
			nestedUnused = append(nestedUnused, unused)
		}
	}

	if len(nestedUnused) != 0 {
		return atc.Config{}, db.PipelineNoChange, ExtraKeysError{extraKeys: nestedUnused}
	}

	return config, pausedState, nil
}
<commit_msg>no need to override params since they are always empty here<commit_after>package configserver

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"github.com/concourse/concourse/atc/exec"
	"io"
	"io/ioutil"
	"mime"
	"mime/multipart"
	"net/http"
	"strings"

	boshtemplate "github.com/cloudfoundry/bosh-cli/director/template"

	"code.cloudfoundry.org/lager"
	"github.com/concourse/concourse/atc"
	"github.com/concourse/concourse/atc/creds"
	"github.com/concourse/concourse/atc/db"
	"github.com/hashicorp/go-multierror"
	"github.com/mitchellh/mapstructure"
	"github.com/tedsuo/rata"
	"gopkg.in/yaml.v2"
)

var (
	ErrStatusUnsupportedMediaType = errors.New("content-type is not supported")
	ErrCannotParseContentType     = errors.New("content-type header could not be parsed")
	ErrMalformedRequestPayload    = errors.New("data in body could not be decoded")
	ErrFailedToConstructDecoder   = errors.New("decoder could not be constructed")
	ErrCouldNotDecode             = errors.New("data could not be decoded into config structure")
	ErrInvalidPausedValue         = errors.New("invalid paused value")
)

type ExtraKeysError struct {
	extraKeys []string
}

func (eke ExtraKeysError) Error() string {
	msg := &bytes.Buffer{}

	fmt.Fprintln(msg, "unknown/extra keys:")
	for _, unusedKey := range eke.extraKeys {
		fmt.Fprintf(msg, "  - %s\n", unusedKey)
	}

	return msg.String()
}

type SaveConfigResponse struct {
	Errors   []string      `json:"errors,omitempty"`
	Warnings []atc.Warning `json:"warnings,omitempty"`
}

func (s *Server) SaveConfig(w http.ResponseWriter, r *http.Request) {
	session := s.logger.Session("set-config")

	query := r.URL.Query()

	checkCredentials := false
	if _, exists := query[atc.SaveConfigCheckCreds]; exists {
		checkCredentials = true
	}

	var version db.ConfigVersion
	if configVersionStr := r.Header.Get(atc.ConfigVersionHeader); len(configVersionStr) != 0 {
		_, err := fmt.Sscanf(configVersionStr, "%d", &version)
		if err != nil {
			session.Error("malformed-config-version", err)
			s.handleBadRequest(w, []string{fmt.Sprintf("config version is malformed: %s", err)}, session)
			return
		}
	}

	config, pausedState, err := saveConfigRequestUnmarshaler(r)
	switch err {
	case ErrStatusUnsupportedMediaType:
		w.WriteHeader(http.StatusUnsupportedMediaType)
		return
	case ErrMalformedRequestPayload:
		session.Error("malformed-request-payload", err, lager.Data{
			"content-type": r.Header.Get("Content-Type"),
		})

		s.handleBadRequest(w, []string{"malformed config"}, session)
		return
	case ErrFailedToConstructDecoder:
		session.Error("failed-to-construct-decoder", err)
		w.WriteHeader(http.StatusInternalServerError)
		return
	case ErrCouldNotDecode:
		session.Error("could-not-decode", err)
		s.handleBadRequest(w, []string{"failed to decode config"}, session)
		return
	case ErrInvalidPausedValue:
		session.Error("invalid-paused-value", err)
		s.handleBadRequest(w, []string{"invalid paused value"}, session)
		return
	default:
		if err != nil {
			if eke, ok := err.(ExtraKeysError); ok {
				s.handleBadRequest(w, []string{eke.Error()}, session)
			} else {
				session.Error("unexpected-error", err)
				w.WriteHeader(http.StatusInternalServerError)
			}

			return
		}
	}

	warnings, errorMessages := config.Validate()
	if len(errorMessages) > 0 {
		session.Error("ignoring-invalid-config", err)
		s.handleBadRequest(w, errorMessages, session)
		return
	}

	pipelineName := rata.Param(r, "pipeline_name")
	teamName := rata.Param(r, "team_name")

	if checkCredentials {
		variables := s.variablesFactory.NewVariables(teamName, pipelineName)

		errs := validateCredParams(variables, config, session)
		if errs != nil {
			s.handleBadRequest(w, []string{errs.Error()}, session)
			return
		}
	}

	session.Info("saving")

	team, found, err := s.teamFactory.FindTeam(teamName)
	if err != nil {
		session.Error("failed-to-find-team", err)
		w.WriteHeader(http.StatusInternalServerError)
		return
	}

	if !found {
		session.Debug("team-not-found")
		w.WriteHeader(http.StatusInternalServerError)
		return
	}

	_, created, err := team.SavePipeline(pipelineName, config, version, pausedState)
	if err != nil {
		session.Error("failed-to-save-config", err)
		w.WriteHeader(http.StatusInternalServerError)
		fmt.Fprintf(w, "failed to save config: %s", err)
		return
	}

	session.Info("saved")

	w.Header().Set("Content-Type", "application/json")

	if created {
		w.WriteHeader(http.StatusCreated)
	} else {
		w.WriteHeader(http.StatusOK)
	}

	s.writeSaveConfigResponse(w, SaveConfigResponse{Warnings: warnings}, session)
}

// Simply validate that the credentials exist; don't do anything with the actual secrets
func validateCredParams(credMgrVars creds.Variables, config atc.Config, session lager.Logger) error {
	var errs error

	for _, resourceType := range config.ResourceTypes {
		_, err := creds.NewSource(credMgrVars, resourceType.Source).Evaluate()
		if err != nil {
			errs = multierror.Append(errs, err)
		}
	}

	for _, resource := range config.Resources {
		_, err := creds.NewSource(credMgrVars, resource.Source).Evaluate()
		if err != nil {
			errs = multierror.Append(errs, err)
		}

		_, err = creds.NewString(credMgrVars, resource.WebhookToken).Evaluate()
		if err != nil {
			errs = multierror.Append(errs, err)
		}
	}

	for _, job := range config.Jobs {
		for _, plan := range job.Plan {
			_, err := creds.NewParams(credMgrVars, plan.Params).Evaluate()
			if err != nil {
				errs = multierror.Append(errs, err)
			}

			if plan.TaskConfigPath != "" {
				// external task - we can't really validate much right now, because task yaml will be
				// retrieved in runtime during job execution. but we can validate vars and params which will be
				// passed to this task
				err = creds.NewTaskParamsValidator(credMgrVars, plan.TaskConfig.Params).Validate()
				if err != nil {
					errs = multierror.Append(errs, err)
				}

				err = creds.NewTaskVarsValidator(credMgrVars, plan.TaskVars).Validate()
				if err != nil {
					errs = multierror.Append(errs, err)
				}

			} else if plan.TaskConfig != nil {
				// embedded task - we can fully validate it, interpolating with cred mgr variables
				var taskConfigSource exec.TaskConfigSource
				embeddedTaskVars := []boshtemplate.Variables{credMgrVars}
				taskConfigSource = exec.StaticConfigSource{Config: plan.TaskConfig, Vars: embeddedTaskVars}
				taskConfigSource = exec.ValidatingConfigSource{ConfigSource: taskConfigSource}
				_, err = taskConfigSource.FetchConfig(session, nil)
				if err != nil {
					errs = multierror.Append(errs, err)
				}
			}

		}
	}

	if errs != nil {
		session.Info("config-has-invalid-creds", lager.Data{"errors": errs.Error()})
	}

	return errs
}

func (s *Server) handleBadRequest(w http.ResponseWriter, errorMessages []string, session lager.Logger) {
	w.Header().Set("Content-Type", "application/json")
	w.WriteHeader(http.StatusBadRequest)
	s.writeSaveConfigResponse(w, SaveConfigResponse{
		Errors: errorMessages,
	}, session)
}

func (s *Server) writeSaveConfigResponse(w http.ResponseWriter, saveConfigResponse SaveConfigResponse, session lager.Logger) {
	responseJSON, err := json.Marshal(saveConfigResponse)
	if err != nil {
		session.Error("failed-to-marshal-validation-response", err)
		w.WriteHeader(http.StatusInternalServerError)
		fmt.Fprintf(w, "failed to generate error response: %s", err)
		return
	}

	_, err = w.Write(responseJSON)
	if err != nil {
		session.Error("failed-to-write-validation-response", err)
		w.WriteHeader(http.StatusInternalServerError)
		return
	}
}

func requestToConfig(contentType string, requestBody io.ReadCloser, configStructure interface{}) (db.PipelinePausedState, error) {
	pausedState := db.PipelineNoChange

	mediaType, params, err := mime.ParseMediaType(contentType)
	if err != nil {
		return db.PipelineNoChange, ErrCannotParseContentType
	}

	switch mediaType {
	case "application/json":
		err := json.NewDecoder(requestBody).Decode(configStructure)
		if err != nil {
			return db.PipelineNoChange, ErrMalformedRequestPayload
		}

	case "application/x-yaml":
		body, err := ioutil.ReadAll(requestBody)
		if err == nil {
			err = yaml.Unmarshal(body, configStructure)
		}

		if err != nil {
			return db.PipelineNoChange, ErrMalformedRequestPayload
		}

	case "multipart/form-data":
		multipartReader := multipart.NewReader(requestBody, params["boundary"])

		for {
			part, err := multipartReader.NextPart()

			if err == io.EOF {
				break
			}

			if err != nil {
				return db.PipelineNoChange, err
			}

			if part.FormName() == "paused" {
				pausedValue, err := ioutil.ReadAll(part)
				if err != nil {
					return db.PipelineNoChange, err
				}

				if string(pausedValue) == "true" {
					pausedState = db.PipelinePaused
				} else if string(pausedValue) == "false" {
					pausedState = db.PipelineUnpaused
				} else {
					return db.PipelineNoChange, ErrInvalidPausedValue
				}
			} else {
				partContentType := part.Header.Get("Content-type")
				_, err := requestToConfig(partContentType, part, configStructure)
				if err != nil {
					return db.PipelineNoChange, ErrMalformedRequestPayload
				}
			}
		}
	default:
		return db.PipelineNoChange, ErrStatusUnsupportedMediaType
	}

	return pausedState, nil
}

func saveConfigRequestUnmarshaler(r *http.Request) (atc.Config, db.PipelinePausedState, error) {
	var configStructure interface{}
	pausedState, err := requestToConfig(r.Header.Get("Content-Type"), r.Body, &configStructure)
	if err != nil {
		return atc.Config{}, db.PipelineNoChange, err
	}

	var config atc.Config
	var md mapstructure.Metadata
	msConfig := &mapstructure.DecoderConfig{
		Metadata:         &md,
		Result:           &config,
		WeaklyTypedInput: true,
		DecodeHook: mapstructure.ComposeDecodeHookFunc(
			atc.SanitizeDecodeHook,
			atc.VersionConfigDecodeHook,
			atc.InputsConfigDecodeHook,
			atc.ContainerLimitsDecodeHook,
		),
	}

	decoder, err := mapstructure.NewDecoder(msConfig)
	if err != nil {
		return atc.Config{}, db.PipelineNoChange, ErrFailedToConstructDecoder
	}

	if err := decoder.Decode(configStructure); err != nil {
		return atc.Config{}, db.PipelineNoChange, ErrCouldNotDecode
	}

	nestedUnused := []string{}
	for _, unused := range md.Unused {
		if strings.Contains(unused, ".") {
			nestedUnused = append(nestedUnused, unused)
		}
	}

	if len(nestedUnused) != 0 {
		return atc.Config{}, db.PipelineNoChange, ExtraKeysError{extraKeys: nestedUnused}
	}

	return config, pausedState, nil
}
<|endoftext|>


<commit_before>package resource

import (
	"context"

	"github.com/concourse/concourse/atc/runtime"
)

func (resource *resource) Get(
	ctx context.Context,
	spec runtime.ProcessSpec,
	runnable runtime.Runner,
) (runtime.VersionResult, error) {
	var vr runtime.VersionResult

	inputArgs, err := resource.Signature()
	if err != nil {
		return vr, err
	}

	err = runnable.RunScript(
		ctx,
		spec.Path,
		spec.Args,
		inputArgs,
		&vr,
		spec.StderrWriter,
		true,
	)
	if err != nil {
		return vr, err
	}

	return vr, nil
}
<commit_msg>simplify resource_get<commit_after>package resource

import (
	"context"

	"github.com/concourse/concourse/atc/runtime"
)

func (resource *resource) Get(
	ctx context.Context,
	spec runtime.ProcessSpec,
	runnable runtime.Runner,
) (runtime.VersionResult, error) {
	var vr runtime.VersionResult

	inputArgs, err := resource.Signature()
	if err != nil {
		return vr, err
	}

	err = runnable.RunScript(
		ctx,
		spec.Path,
		spec.Args,
		inputArgs,
		&vr,
		spec.StderrWriter,
		true,
	)
	return vr, err
}
<|endoftext|>