<commit_before>// Copyright 2016 tsuru authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package eventtest

import (
	"fmt"
	"strings"
	"time"

	"github.com/tsuru/tsuru/db"
	"github.com/tsuru/tsuru/event"
	"gopkg.in/check.v1"
	"gopkg.in/mgo.v2/bson"
)

type EventDesc struct {
	Target          event.Target
	Kind            string
	Owner           string
	StartCustomData interface{}
	EndCustomData   interface{}
	OtherCustomData interface{}
	LogMatches      string
	ErrorMatches    string
	IsEmpty         bool
}

type hasEventChecker struct{}

func (hasEventChecker) Info() *check.CheckerInfo {
	return &check.CheckerInfo{Name: "HasEvent", Params: []string{"event desc"}}
}

func queryPartCustom(query map[string]interface{}, name string, value interface{}) {
	if value == nil {
		return
	}
	switch data := value.(type) {
	case map[string]interface{}:
		for k, v := range data {
			query[name+"."+k] = v
		}
	case []map[string]interface{}:
		queryPart := []bson.M{}
		for _, el := range data {
			queryPart = append(queryPart, bson.M{
				name: bson.M{"$elemMatch": el},
			})
		}
		query["$and"] = queryPart
	}
}

func (hasEventChecker) Check(params []interface{}, names []string) (bool, string) {
	var evt EventDesc
	switch params[0].(type) {
	case EventDesc:
		evt = params[0].(EventDesc)
	case *EventDesc:
		evt = *params[0].(*EventDesc)
	default:
		return false, "First parameter must be of type EventDesc or *EventDesc"
	}
	conn, err := db.Conn()
	if err != nil {
		return false, err.Error()
	}
	defer conn.Close()
	if evt.IsEmpty {
		var n int
		n, err = conn.Events().Find(nil).Count()
		if err != nil {
			return false, err.Error()
		}
		if n != 0 {
			return false, fmt.Sprintf("expected 0 events, got %d", n)
		}
		return true, ""
	}
	query := map[string]interface{}{
		"target":     evt.Target,
		"kind.name":  evt.Kind,
		"owner.name": evt.Owner,
		"running":    false,
	}
	queryPartCustom(query, "startcustomdata", evt.StartCustomData)
	queryPartCustom(query, "endcustomdata", evt.EndCustomData)
	queryPartCustom(query, "othercustomdata", evt.OtherCustomData)
	if evt.LogMatches != "" {
		query["log"] = bson.M{"$regex": evt.LogMatches}
	}
	if evt.ErrorMatches != "" {
		query["error"] = bson.M{"$regex": evt.ErrorMatches}
	} else {
		query["error"] = ""
	}
	n, err := conn.Events().Find(query).Count()
	if err != nil {
		return false, err.Error()
	}
	if n == 0 {
		all, _ := event.All()
		msg := fmt.Sprintf("Event not found. Existing events in DB: %s", debugEvts(all))
		return false, msg
	}
	if n > 1 {
		return false, "Multiple events match query"
	}
	return true, ""
}

func debugEvts(evts []event.Event) string {
	var msgs []string
	for i := 0; i < len(evts); i++ {
		evt := evts[i]
		var sData, oData, eData interface{}
		evt.StartData(&sData)
		evt.OtherData(&oData)
		evt.EndData(&eData)
		evt.StartCustomData = bson.Raw{}
		evt.OtherCustomData = bson.Raw{}
		evt.EndCustomData = bson.Raw{}
		msgs = append(msgs, fmt.Sprintf("%#v\nstartData: %#v\notherData: %#v\nendData: %#v", evt, sData, oData, eData))
	}
	return strings.Join(msgs, "\n****\n")
}

var HasEvent check.Checker = hasEventChecker{}

type evtEqualsChecker struct {
	check.CheckerInfo
}

func (evtEqualsChecker) Check(params []interface{}, names []string) (bool, string) {
	evts := make([][]*event.Event, len(params))
	for i := range evts {
		switch e := params[i].(type) {
		case event.Event:
			evts[i] = []*event.Event{&e}
		case *event.Event:
			evts[i] = []*event.Event{e}
		case []event.Event:
			for j := range e {
				evts[i] = append(evts[i], &e[j])
			}
		case []*event.Event:
			evts[i] = e
		default:
			evts[i] = nil
		}
		for j := range evts[i] {
			e := evts[i][j]
			e.StartTime = time.Time{}
			e.EndTime = time.Time{}
			e.LockUpdateTime = time.Time{}
			e.RemoveDate = time.Time{}
		}
	}
	return check.DeepEquals.Check([]interface{}{evts[0], evts[1]}, names)
}

var EvtEquals check.Checker = &evtEqualsChecker{
	check.CheckerInfo{Name: "EvtEquals", Params: []string{"obtained", "expected"}},
}
<commit_msg>event/eventtest: fix go vet issues on tip<commit_after>// Copyright 2016 tsuru authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package eventtest

import (
	"fmt"
	"strings"
	"time"

	"github.com/tsuru/tsuru/db"
	"github.com/tsuru/tsuru/event"
	"gopkg.in/check.v1"
	"gopkg.in/mgo.v2/bson"
)

type EventDesc struct {
	Target          event.Target
	Kind            string
	Owner           string
	StartCustomData interface{}
	EndCustomData   interface{}
	OtherCustomData interface{}
	LogMatches      string
	ErrorMatches    string
	IsEmpty         bool
}

type hasEventChecker struct{}

func (hasEventChecker) Info() *check.CheckerInfo {
	return &check.CheckerInfo{Name: "HasEvent", Params: []string{"event desc"}}
}

func queryPartCustom(query map[string]interface{}, name string, value interface{}) {
	if value == nil {
		return
	}
	switch data := value.(type) {
	case map[string]interface{}:
		for k, v := range data {
			query[name+"."+k] = v
		}
	case []map[string]interface{}:
		queryPart := []bson.M{}
		for _, el := range data {
			queryPart = append(queryPart, bson.M{
				name: bson.M{"$elemMatch": el},
			})
		}
		query["$and"] = queryPart
	}
}

func (hasEventChecker) Check(params []interface{}, names []string) (bool, string) {
	var evt EventDesc
	switch params[0].(type) {
	case EventDesc:
		evt = params[0].(EventDesc)
	case *EventDesc:
		evt = *params[0].(*EventDesc)
	default:
		return false, "First parameter must be of type EventDesc or *EventDesc"
	}
	conn, err := db.Conn()
	if err != nil {
		return false, err.Error()
	}
	defer conn.Close()
	if evt.IsEmpty {
		var n int
		n, err = conn.Events().Find(nil).Count()
		if err != nil {
			return false, err.Error()
		}
		if n != 0 {
			return false, fmt.Sprintf("expected 0 events, got %d", n)
		}
		return true, ""
	}
	query := map[string]interface{}{
		"target":     evt.Target,
		"kind.name":  evt.Kind,
		"owner.name": evt.Owner,
		"running":    false,
	}
	queryPartCustom(query, "startcustomdata", evt.StartCustomData)
	queryPartCustom(query, "endcustomdata", evt.EndCustomData)
	queryPartCustom(query, "othercustomdata", evt.OtherCustomData)
	if evt.LogMatches != "" {
		query["log"] = bson.M{"$regex": evt.LogMatches}
	}
	if evt.ErrorMatches != "" {
		query["error"] = bson.M{"$regex": evt.ErrorMatches}
	} else {
		query["error"] = ""
	}
	n, err := conn.Events().Find(query).Count()
	if err != nil {
		return false, err.Error()
	}
	if n == 0 {
		all, _ := event.All()
		msg := fmt.Sprintf("Event not found. Existing events in DB: %s", debugEvts(all))
		return false, msg
	}
	if n > 1 {
		return false, "Multiple events match query"
	}
	return true, ""
}

func debugEvts(evts []event.Event) string {
	var msgs []string
	for i := range evts {
		evt := &evts[i]
		var sData, oData, eData interface{}
		evt.StartData(&sData)
		evt.OtherData(&oData)
		evt.EndData(&eData)
		evt.StartCustomData = bson.Raw{}
		evt.OtherCustomData = bson.Raw{}
		evt.EndCustomData = bson.Raw{}
		msgs = append(msgs, fmt.Sprintf("%#v\nstartData: %#v\notherData: %#v\nendData: %#v", evt, sData, oData, eData))
	}
	return strings.Join(msgs, "\n****\n")
}

var HasEvent check.Checker = hasEventChecker{}

type evtEqualsChecker struct {
	check.CheckerInfo
}

func (evtEqualsChecker) Check(params []interface{}, names []string) (bool, string) {
	evts := make([][]*event.Event, len(params))
	for i := range evts {
		switch e := params[i].(type) {
		case event.Event:
			evts[i] = []*event.Event{&e}
		case *event.Event:
			evts[i] = []*event.Event{e}
		case []event.Event:
			for j := range e {
				evts[i] = append(evts[i], &e[j])
			}
		case []*event.Event:
			evts[i] = e
		default:
			evts[i] = nil
		}
		for j := range evts[i] {
			e := evts[i][j]
			e.StartTime = time.Time{}
			e.EndTime = time.Time{}
			e.LockUpdateTime = time.Time{}
			e.RemoveDate = time.Time{}
		}
	}
	return check.DeepEquals.Check([]interface{}{evts[0], evts[1]}, names)
}

var EvtEquals check.Checker = &evtEqualsChecker{
	check.CheckerInfo{Name: "EvtEquals", Params: []string{"obtained", "expected"}},
}
<|endoftext|>


<commit_before>/*
 * Minio Client, (C) 2015 Minio, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

// Session V2 - Version 2 stores session header and session data in
// two separate files. Session data contains fully prepared URL list.
package main

import (
	"encoding/json"
	"fmt"
	"io"
	"os"
	"strings"
	"sync"
	"time"

	"github.com/minio/mc/pkg/console"
	"github.com/minio/minio/pkg/probe"
	"github.com/minio/minio/pkg/quick"
)

// migrateSessionV1ToV2 migrates all session files from v1 to v2.
// This function should be called from the main early on.
func migrateSessionV1ToV2() {
	for _, sid := range getSessionIDsV1() {
		sessionFileV1, err := getSessionFileV1(sid)
		fatalIf(err.Trace(sid), "Unable to determine session file for ‘"+sid+"’.")

		e := os.Remove(sessionFileV1)
		fatalIf(probe.NewError(e), "Migration failed. Unable to remove old session file ‘"+sessionFileV1+"’.")
	}
}

// sessionV2Header
type sessionV2Header struct {
	Version      string    `json:"version"`
	When         time.Time `json:"time"`
	RootPath     string    `json:"working-folder"`
	CommandType  string    `json:"command-type"`
	CommandArgs  []string  `json:"cmd-args"`
	LastCopied   string    `json:"last-copied"`
	TotalBytes   int64     `json:"total-bytes"`
	TotalObjects int       `json:"total-objects"`
}

// SessionMessage container for session messages
type SessionMessage struct {
	SessionID   string    `json:"sessionid"`
	Time        time.Time `json:"time"`
	CommandType string    `json:"command-type"`
	CommandArgs []string  `json:"command-args"`
}

// sessionV2
type sessionV2 struct {
	Header    *sessionV2Header
	SessionID string
	mutex     *sync.Mutex
	DataFP    *os.File
	sigCh     bool
}

func (s sessionV2) String() string {
	if !globalJSONFlag {
		message := console.Colorize("SessionID", fmt.Sprintf("%s -> ", s.SessionID))
		message = message + console.Colorize("SessionTime", fmt.Sprintf("[%s]", s.Header.When.Local().Format(printDate)))
		message = message + console.Colorize("Command", fmt.Sprintf(" %s %s", s.Header.CommandType, strings.Join(s.Header.CommandArgs, " ")))
		return message
	}
	sessionMesage := SessionMessage{
		SessionID:   s.SessionID,
		Time:        s.Header.When.Local(),
		CommandType: s.Header.CommandType,
		CommandArgs: s.Header.CommandArgs,
	}
	sessionBytes, e := json.Marshal(sessionMesage)
	fatalIf(probe.NewError(e), "Unable to marshal into JSON.")

	return string(sessionBytes)
}

// newSessionV2 provides a new session
func newSessionV2() *sessionV2 {
	s := &sessionV2{}
	s.Header = &sessionV2Header{}
	s.Header.Version = "1.1.0"
	// map of command and files copied
	s.Header.CommandArgs = nil
	s.Header.When = time.Now().UTC()
	s.mutex = new(sync.Mutex)
	s.SessionID = newSID(8)

	sessionDataFile, perr := getSessionDataFile(s.SessionID)
	fatalIf(perr.Trace(s.SessionID), "Unable to create session data file \""+sessionDataFile+"\".")

	var err error
	s.DataFP, err = os.Create(sessionDataFile)
	fatalIf(probe.NewError(err), "Unable to create session data file \""+sessionDataFile+"\".")

	return s
}

// String printer for SessionV2
func (s sessionV2) Info() {
	console.Infoln("Session terminated. To resume session ‘mc session resume " + s.SessionID + "’")
}

// HasData provides true if this is a session resume, false otherwise.
func (s sessionV2) HasData() bool {
	if s.Header.LastCopied == "" {
		return false
	}
	return true
}

// NewDataReader provides reader interface to session data file.
func (s *sessionV2) NewDataReader() io.Reader {
	// DataFP is always intitialized, either via new or load functions.
	s.DataFP.Seek(0, os.SEEK_SET)
	return io.Reader(s.DataFP)
}

// NewDataReader provides writer interface to session data file.
func (s *sessionV2) NewDataWriter() io.Writer {
	// DataFP is always intitialized, either via new or load functions.
	s.DataFP.Seek(0, os.SEEK_SET)
	return io.Writer(s.DataFP)
}

// Save this session
func (s *sessionV2) Save() *probe.Error {
	s.mutex.Lock()
	defer s.mutex.Unlock()

	if err := s.DataFP.Sync(); err != nil {
		return probe.NewError(err)
	}

	qs, err := quick.New(s.Header)
	if err != nil {
		return err.Trace(s.SessionID)
	}

	sessionFile, err := getSessionFile(s.SessionID)
	if err != nil {
		return err.Trace(s.SessionID)
	}
	return qs.Save(sessionFile).Trace()
}

// Close ends this session and removes all associated session files.
func (s *sessionV2) Close() *probe.Error {
	s.mutex.Lock()
	defer s.mutex.Unlock()

	if err := s.DataFP.Close(); err != nil {
		return probe.NewError(err)
	}

	qs, err := quick.New(s.Header)
	if err != nil {
		return err.Trace()
	}

	sessionFile, err := getSessionFile(s.SessionID)
	if err != nil {
		return err.Trace(s.SessionID)
	}
	return qs.Save(sessionFile).Trace()
}

// Delete removes all the session files.
func (s *sessionV2) Delete() *probe.Error {
	s.mutex.Lock()
	defer s.mutex.Unlock()

	if s.DataFP != nil {
		err := os.Remove(s.DataFP.Name())
		if err != nil {
			return probe.NewError(err)
		}
	}

	sessionFile, err := getSessionFile(s.SessionID)
	if err != nil {
		return err.Trace(s.SessionID)
	}

	if err := os.Remove(sessionFile); err != nil {
		return probe.NewError(err)
	}

	return nil
}

// loadSession - reads session file if exists and re-initiates internal variables
func loadSessionV2(sid string) (*sessionV2, *probe.Error) {
	if !isSessionDirExists() {
		return nil, errInvalidArgument().Trace()
	}
	sessionFile, err := getSessionFile(sid)
	if err != nil {
		return nil, err.Trace(sid)
	}

	if _, err := os.Stat(sessionFile); err != nil {
		return nil, probe.NewError(err)
	}

	s := &sessionV2{}
	s.Header = &sessionV2Header{}
	s.SessionID = sid
	s.Header.Version = "1.1.0"
	qs, err := quick.New(s.Header)
	if err != nil {
		return nil, err.Trace(sid, s.Header.Version)
	}
	err = qs.Load(sessionFile)
	if err != nil {
		return nil, err.Trace(sid, s.Header.Version)
	}

	s.mutex = new(sync.Mutex)
	s.Header = qs.Data().(*sessionV2Header)

	sessionDataFile, err := getSessionDataFile(s.SessionID)
	if err != nil {
		return nil, err.Trace(sid, s.Header.Version)
	}

	var e error
	s.DataFP, e = os.Open(sessionDataFile)
	fatalIf(probe.NewError(e), "Unable to open session data file ‘"+sessionDataFile+"’.")

	return s, nil
}

// Create a factory function to simplify checking if an
// object has been copied or not.
// isCopied(URL) -> true or false
func isCopiedFactory(lastCopied string) func(string) bool {
	copied := true // closure
	return func(sourceURL string) bool {
		if sourceURL == "" {
			fatalIf(errInvalidArgument().Trace(), "Empty source argument passed.")
		}
		if lastCopied == "" {
			return false
		}

		if copied {
			if lastCopied == sourceURL {
				copied = false // from next call onwards we say false.
			}
			return true
		}
		return false
	}
}
<commit_msg>Close session pro-actively during clear all session<commit_after>/*
 * Minio Client, (C) 2015 Minio, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

// Session V2 - Version 2 stores session header and session data in
// two separate files. Session data contains fully prepared URL list.
package main

import (
	"encoding/json"
	"fmt"
	"io"
	"os"
	"strings"
	"sync"
	"time"

	"github.com/minio/mc/pkg/console"
	"github.com/minio/minio/pkg/probe"
	"github.com/minio/minio/pkg/quick"
)

// migrateSessionV1ToV2 migrates all session files from v1 to v2.
// This function should be called from the main early on.
func migrateSessionV1ToV2() {
	for _, sid := range getSessionIDsV1() {
		sessionFileV1, err := getSessionFileV1(sid)
		fatalIf(err.Trace(sid), "Unable to determine session file for ‘"+sid+"’.")

		e := os.Remove(sessionFileV1)
		fatalIf(probe.NewError(e), "Migration failed. Unable to remove old session file ‘"+sessionFileV1+"’.")
	}
}

// sessionV2Header
type sessionV2Header struct {
	Version      string    `json:"version"`
	When         time.Time `json:"time"`
	RootPath     string    `json:"working-folder"`
	CommandType  string    `json:"command-type"`
	CommandArgs  []string  `json:"cmd-args"`
	LastCopied   string    `json:"last-copied"`
	TotalBytes   int64     `json:"total-bytes"`
	TotalObjects int       `json:"total-objects"`
}

// SessionMessage container for session messages
type SessionMessage struct {
	SessionID   string    `json:"sessionid"`
	Time        time.Time `json:"time"`
	CommandType string    `json:"command-type"`
	CommandArgs []string  `json:"command-args"`
}

// sessionV2
type sessionV2 struct {
	Header    *sessionV2Header
	SessionID string
	mutex     *sync.Mutex
	DataFP    *os.File
	sigCh     bool
}

func (s sessionV2) String() string {
	if !globalJSONFlag {
		message := console.Colorize("SessionID", fmt.Sprintf("%s -> ", s.SessionID))
		message = message + console.Colorize("SessionTime", fmt.Sprintf("[%s]", s.Header.When.Local().Format(printDate)))
		message = message + console.Colorize("Command", fmt.Sprintf(" %s %s", s.Header.CommandType, strings.Join(s.Header.CommandArgs, " ")))
		return message
	}
	sessionMesage := SessionMessage{
		SessionID:   s.SessionID,
		Time:        s.Header.When.Local(),
		CommandType: s.Header.CommandType,
		CommandArgs: s.Header.CommandArgs,
	}
	sessionBytes, e := json.Marshal(sessionMesage)
	fatalIf(probe.NewError(e), "Unable to marshal into JSON.")

	return string(sessionBytes)
}

// newSessionV2 provides a new session
func newSessionV2() *sessionV2 {
	s := &sessionV2{}
	s.Header = &sessionV2Header{}
	s.Header.Version = "1.1.0"
	// map of command and files copied
	s.Header.CommandArgs = nil
	s.Header.When = time.Now().UTC()
	s.mutex = new(sync.Mutex)
	s.SessionID = newSID(8)

	sessionDataFile, perr := getSessionDataFile(s.SessionID)
	fatalIf(perr.Trace(s.SessionID), "Unable to create session data file \""+sessionDataFile+"\".")

	var err error
	s.DataFP, err = os.Create(sessionDataFile)
	fatalIf(probe.NewError(err), "Unable to create session data file \""+sessionDataFile+"\".")

	return s
}

// String printer for SessionV2
func (s sessionV2) Info() {
	console.Infoln("Session terminated. To resume session ‘mc session resume " + s.SessionID + "’")
}

// HasData provides true if this is a session resume, false otherwise.
func (s sessionV2) HasData() bool {
	if s.Header.LastCopied == "" {
		return false
	}
	return true
}

// NewDataReader provides reader interface to session data file.
func (s *sessionV2) NewDataReader() io.Reader {
	// DataFP is always intitialized, either via new or load functions.
	s.DataFP.Seek(0, os.SEEK_SET)
	return io.Reader(s.DataFP)
}

// NewDataReader provides writer interface to session data file.
func (s *sessionV2) NewDataWriter() io.Writer {
	// DataFP is always intitialized, either via new or load functions.
	s.DataFP.Seek(0, os.SEEK_SET)
	return io.Writer(s.DataFP)
}

// Save this session
func (s *sessionV2) Save() *probe.Error {
	s.mutex.Lock()
	defer s.mutex.Unlock()

	if err := s.DataFP.Sync(); err != nil {
		return probe.NewError(err)
	}

	qs, err := quick.New(s.Header)
	if err != nil {
		return err.Trace(s.SessionID)
	}

	sessionFile, err := getSessionFile(s.SessionID)
	if err != nil {
		return err.Trace(s.SessionID)
	}
	return qs.Save(sessionFile).Trace()
}

// Close ends this session and removes all associated session files.
func (s *sessionV2) Close() *probe.Error {
	s.mutex.Lock()
	defer s.mutex.Unlock()

	if err := s.DataFP.Close(); err != nil {
		return probe.NewError(err)
	}

	qs, err := quick.New(s.Header)
	if err != nil {
		return err.Trace()
	}

	sessionFile, err := getSessionFile(s.SessionID)
	if err != nil {
		return err.Trace(s.SessionID)
	}
	return qs.Save(sessionFile).Trace()
}

// Delete removes all the session files.
func (s *sessionV2) Delete() *probe.Error {
	s.mutex.Lock()
	defer s.mutex.Unlock()

	if s.DataFP != nil {
		name := s.DataFP.Name()
		// close file pro-actively before deleting
		// ignore any error, it could be possibly that
		// the file is closed already
		s.DataFP.Close()

		err := os.Remove(name)
		if err != nil {
			return probe.NewError(err)
		}
	}

	sessionFile, err := getSessionFile(s.SessionID)
	if err != nil {
		return err.Trace(s.SessionID)
	}

	if err := os.Remove(sessionFile); err != nil {
		return probe.NewError(err)
	}

	return nil
}

// loadSession - reads session file if exists and re-initiates internal variables
func loadSessionV2(sid string) (*sessionV2, *probe.Error) {
	if !isSessionDirExists() {
		return nil, errInvalidArgument().Trace()
	}
	sessionFile, err := getSessionFile(sid)
	if err != nil {
		return nil, err.Trace(sid)
	}

	if _, err := os.Stat(sessionFile); err != nil {
		return nil, probe.NewError(err)
	}

	s := &sessionV2{}
	s.Header = &sessionV2Header{}
	s.SessionID = sid
	s.Header.Version = "1.1.0"
	qs, err := quick.New(s.Header)
	if err != nil {
		return nil, err.Trace(sid, s.Header.Version)
	}
	err = qs.Load(sessionFile)
	if err != nil {
		return nil, err.Trace(sid, s.Header.Version)
	}

	s.mutex = new(sync.Mutex)
	s.Header = qs.Data().(*sessionV2Header)

	sessionDataFile, err := getSessionDataFile(s.SessionID)
	if err != nil {
		return nil, err.Trace(sid, s.Header.Version)
	}

	var e error
	s.DataFP, e = os.Open(sessionDataFile)
	fatalIf(probe.NewError(e), "Unable to open session data file ‘"+sessionDataFile+"’.")

	return s, nil
}

// Create a factory function to simplify checking if an
// object has been copied or not.
// isCopied(URL) -> true or false
func isCopiedFactory(lastCopied string) func(string) bool {
	copied := true // closure
	return func(sourceURL string) bool {
		if sourceURL == "" {
			fatalIf(errInvalidArgument().Trace(), "Empty source argument passed.")
		}
		if lastCopied == "" {
			return false
		}

		if copied {
			if lastCopied == sourceURL {
				copied = false // from next call onwards we say false.
			}
			return true
		}
		return false
	}
}
<|endoftext|>


<commit_before><commit_msg>fix panic when a blockstring has only 1 line and blank<commit_after><|endoftext|>


<commit_before>package neurgo

import (
	"testing"
	"github.com/couchbaselabs/go.assert"
	"sync"
	"time"
)

func TestConnectBidirectional(t *testing.T) {

	// create nodes
	neuron := &Node{processor: &Neuron{}}
	sensor := &Node{processor: &Sensor{}}

	// give names
	neuron.Name = "neuron"
	sensor.Name = "sensor"

	// make connection
	weights := []float64{20,20,20,20,20}
	sensor.ConnectBidirectionalWeighted(neuron, weights)

	// make sure the reverse connection points back to correct node type
	sensorTypeCheck := neuron.inbound[0].other.processor.(*Sensor)
	assert.Equals(t, sensor, sensorTypeCheck)

	// assert that it worked
	assert.Equals(t, len(sensor.outbound), 1)
	assert.Equals(t, len(neuron.inbound), 1)
	assert.True(t, neuron.inbound[0].channel != nil)
	assert.True(t, sensor.outbound[0].channel != nil)
	assert.Equals(t, len(neuron.inbound[0].weights), len(weights))
	assert.Equals(t, neuron.inbound[0].weights[0], weights[0])

	// make a new node and connect it
	actuator := &Node{processor: &Actuator{}}
	neuron.ConnectBidirectional(actuator)

	// make sure the reverse connection points back to correct node type
	neuronTypeCheck := actuator.inbound[0].other.processor.(*Neuron)
	assert.Equals(t, neuron, neuronTypeCheck)

	// assert that it worked
	assert.Equals(t, len(neuron.outbound), 1)
	assert.Equals(t, len(actuator.inbound), 1)
	assert.Equals(t, len(actuator.inbound[0].weights), 0)

}

func TestRemoveConnection(t *testing.T) {

	// create network nodes
	neuronProcessor1 := &Neuron{Bias: 10, ActivationFunction: identity_activation}
	neuronProcessor2 := &Neuron{Bias: 10, ActivationFunction: identity_activation}
	neuron1 := &Node{Name: "neuron1", processor: neuronProcessor1}
	neuron2 := &Node{Name: "neuron2", processor: neuronProcessor2}
	sensor := &Node{Name: "sensor", processor: &Sensor{}}

	// connect nodes together
	weights := []float64{20,20,20,20,20}
	sensor.ConnectBidirectionalWeighted(neuron1, weights)
	sensor.ConnectBidirectionalWeighted(neuron2, weights)

	// remove connections
	neuron1.inbound = removeConnection(neuron1.inbound, 0)
	sensor.outbound = removeConnection(sensor.outbound, 0)

	// assert that it worked
	assert.Equals(t, len(neuron1.inbound), 0)
	assert.Equals(t, len(neuron2.inbound), 1)
	assert.Equals(t, len(sensor.outbound), 1)
	assert.Equals(t, sensor.outbound[0].channel, neuron2.inbound[0].channel)

}

func TestRemoveConnectionFromRunningNeuron(t *testing.T) {

	// create nodes
	sensor1 := &Node{Name: "sensor1", processor: &Sensor{}}
	sensor2 := &Node{Name: "sensor2", processor: &Sensor{}}
	neuronProcessor := &Neuron{Bias: 10, ActivationFunction: identity_activation}
	neuron := &Node{Name: "neuron", processor: neuronProcessor}

	// connect nodes together
	weights := []float64{20}
	sensor1.ConnectBidirectionalWeighted(neuron, weights)
	sensor2.ConnectBidirectionalWeighted(neuron, weights)

	// basic sanity check, send two inputs to neuron inbound channels
	// and verify that weightedInputs() returns both inputs
	go func() {
		sensor1.outbound[0].channel <- []float64{0}
	}()
	go func() {
		sensor2.outbound[0].channel <- []float64{0}
	}()
	weightedInputs := neuronProcessor.weightedInputs(neuron)
	assert.Equals(t, len(weightedInputs), 2)

	// close one channel while a neuron is reading from
	// both inbound connections, make sure it returns one value
	var wg sync.WaitGroup
	wg.Add(1)
	wg.Add(1)

	go func() {
		weightedInputs := neuronProcessor.weightedInputs(neuron)
		assert.Equals(t, len(weightedInputs), 1)
		wg.Done()
	}()

	go func() {

		// need to sleep so that we can be sure that the other go-routine
		// is blocked on the channel read of its inbound channels
		time.Sleep(0.1 * 1e9)

		sensor1.DisconnectBidirectional(neuron)
		sensor2.outbound[0].channel <- []float64{0}
		wg.Done()
	}()

	wg.Wait()

}

func TestRemoveConnectionFromRunningActuator(t *testing.T) {

	// create nodes
	neuronProcessor1 := &Neuron{Bias: 10, ActivationFunction: identity_activation}
	neuronProcessor2 := &Neuron{Bias: 10, ActivationFunction: identity_activation}
	neuron1 := &Node{Name: "neuron1", processor: neuronProcessor1}
	neuron2 := &Node{Name: "neuron2", processor: neuronProcessor2}
	actuatorProcessor := &Actuator{}
	actuator := &Node{Name: "actuator", processor: actuatorProcessor}

	// connect nodes together
	neuron1.ConnectBidirectional(actuator)
	neuron2.ConnectBidirectional(actuator)

	var wg sync.WaitGroup
	wg.Add(1)
	wg.Add(1)

	go func() {
		inputs := actuatorProcessor.gatherInputs(actuator)
		assert.Equals(t, len(inputs), 1)
		wg.Done()
	}()

	go func() {

		// need to sleep so that we can be sure that the other go-routine
		// is blocked on the channel read of its inbound channels
		time.Sleep(0.1 * 1e9)

		neuron1.DisconnectBidirectional(actuator)
		neuron2.outbound[0].channel <- []float64{0}
		wg.Done()
	}()

	wg.Wait()

}

func identity_activation(x float64) float64 {
	return x
}
<commit_msg>connect_test now passes, for reals<commit_after>package neurgo

import (
	"testing"
	"github.com/couchbaselabs/go.assert"
	"sync"
	"time"
)

func TestConnectBidirectional(t *testing.T) {

	// create nodes
	neuron := &Node{Name: "neuron", processor: &Neuron{}}
	sensor := &Node{Name: "sensor", processor: &Sensor{}}

	// make connection
	weights := []float64{20,20,20,20,20}
	sensor.ConnectBidirectionalWeighted(neuron, weights)

	// make sure the reverse connection points back to correct node type
	_, isSensor := neuron.inbound[0].other.processor.(*Sensor)
	assert.True(t, isSensor)

	// assert that it worked
	assert.Equals(t, len(sensor.outbound), 1)
	assert.Equals(t, len(neuron.inbound), 1)
	assert.True(t, neuron.inbound[0].channel != nil)
	assert.True(t, sensor.outbound[0].channel != nil)
	assert.Equals(t, len(neuron.inbound[0].weights), len(weights))
	assert.Equals(t, neuron.inbound[0].weights[0], weights[0])

	// make a new node and connect it
	actuator := &Node{processor: &Actuator{}}
	neuron.ConnectBidirectional(actuator)

	// make sure the reverse connection points back to correct node type
	_, isNeuron := actuator.inbound[0].other.processor.(*Neuron)
	assert.True(t, isNeuron)

	// assert that it worked
	assert.Equals(t, len(neuron.outbound), 1)
	assert.Equals(t, len(actuator.inbound), 1)
	assert.Equals(t, len(actuator.inbound[0].weights), 0)

}

func TestRemoveConnection(t *testing.T) {

	// create network nodes
	neuronProcessor1 := &Neuron{Bias: 10, ActivationFunction: identity_activation}
	neuronProcessor2 := &Neuron{Bias: 10, ActivationFunction: identity_activation}
	neuron1 := &Node{Name: "neuron1", processor: neuronProcessor1}
	neuron2 := &Node{Name: "neuron2", processor: neuronProcessor2}
	sensor := &Node{Name: "sensor", processor: &Sensor{}}

	// connect nodes together
	weights := []float64{20,20,20,20,20}
	sensor.ConnectBidirectionalWeighted(neuron1, weights)
	sensor.ConnectBidirectionalWeighted(neuron2, weights)

	// remove connections
	neuron1.inbound = removeConnection(neuron1.inbound, 0)
	sensor.outbound = removeConnection(sensor.outbound, 0)

	// assert that it worked
	assert.Equals(t, len(neuron1.inbound), 0)
	assert.Equals(t, len(neuron2.inbound), 1)
	assert.Equals(t, len(sensor.outbound), 1)
	assert.Equals(t, sensor.outbound[0].channel, neuron2.inbound[0].channel)

}

func TestRemoveConnectionFromRunningNeuron(t *testing.T) {

	// create nodes
	sensor1 := &Node{Name: "sensor1", processor: &Sensor{}}
	sensor2 := &Node{Name: "sensor2", processor: &Sensor{}}
	neuronProcessor := &Neuron{Bias: 10, ActivationFunction: identity_activation}
	neuron := &Node{Name: "neuron", processor: neuronProcessor}

	// connect nodes together
	weights := []float64{20}
	sensor1.ConnectBidirectionalWeighted(neuron, weights)
	sensor2.ConnectBidirectionalWeighted(neuron, weights)

	// basic sanity check, send two inputs to neuron inbound channels
	// and verify that weightedInputs() returns both inputs
	go func() {
		sensor1.outbound[0].channel <- []float64{0}
	}()
	go func() {
		sensor2.outbound[0].channel <- []float64{0}
	}()
	weightedInputs := neuronProcessor.weightedInputs(neuron)
	assert.Equals(t, len(weightedInputs), 2)

	// close one channel while a neuron is reading from
	// both inbound connections, make sure it returns one value
	var wg sync.WaitGroup
	wg.Add(1)
	wg.Add(1)

	go func() {
		weightedInputs := neuronProcessor.weightedInputs(neuron)
		assert.Equals(t, len(weightedInputs), 1)
		wg.Done()
	}()

	go func() {

		// need to sleep so that we can be sure that the other go-routine
		// is blocked on the channel read of its inbound channels
		time.Sleep(0.1 * 1e9)

		sensor1.DisconnectBidirectional(neuron)
		sensor2.outbound[0].channel <- []float64{0}
		wg.Done()
	}()

	wg.Wait()

}

func TestRemoveConnectionFromRunningActuator(t *testing.T) {

	// create nodes
	neuronProcessor1 := &Neuron{Bias: 10, ActivationFunction: identity_activation}
	neuronProcessor2 := &Neuron{Bias: 10, ActivationFunction: identity_activation}
	neuron1 := &Node{Name: "neuron1", processor: neuronProcessor1}
	neuron2 := &Node{Name: "neuron2", processor: neuronProcessor2}
	actuatorProcessor := &Actuator{}
	actuator := &Node{Name: "actuator", processor: actuatorProcessor}

	// connect nodes together
	neuron1.ConnectBidirectional(actuator)
	neuron2.ConnectBidirectional(actuator)

	var wg sync.WaitGroup
	wg.Add(1)
	wg.Add(1)

	go func() {
		inputs := actuatorProcessor.gatherInputs(actuator)
		assert.Equals(t, len(inputs), 1)
		wg.Done()
	}()

	go func() {

		// need to sleep so that we can be sure that the other go-routine
		// is blocked on the channel read of its inbound channels
		time.Sleep(0.1 * 1e9)

		neuron1.DisconnectBidirectional(actuator)
		neuron2.outbound[0].channel <- []float64{0}
		wg.Done()
	}()

	wg.Wait()

}

func identity_activation(x float64) float64 {
	return x
}
<|endoftext|>


<commit_before>package arel

import (
	"bytes"
	"log"
	"strings"
)

type ToSqlVisitor struct {
	conn *Connection
}

const (
	WHERE    = " WHERE "
	SPACE    = " "
	COMMA    = ", "
	GROUP_BY = " GROUP BY "
	ORDER_BY = " ORDER BY "
	WINDOW   = " WINDOW "
	AND      = " AND "
	DISTINCT = "DISTINCT"
)

func NewToSqlVisitor(c *Connection) ToSqlVisitor {
	return ToSqlVisitor{conn: c}
}

func (v ToSqlVisitor) Accept(a AstNode) string {
	return v.Visit(a)
}

func (v ToSqlVisitor) Visit(a AstNode) string {
	ret := ""
	switch val := a.(type) {
	case SelectStatementNode:
		ret = v.VisitSelectStatementNode(val)
	case AndNode:
		ret = v.VisitAndNode(val)
	case InNode:
		ret = v.VisitInNode(val)
	case SqlLiteralNode:
		ret = v.VisitSqlLiteralNode(val)
	case JoinSource:
		ret = v.VisitJoinSourceNode(val)
	default:
		panic("ToSqlVisitor#Visit AstNode not handled")
	}
	// log.Printf("ToSqlVisitor#Visit; type of node: %T, return: %v", a, ret)
	return ret
}

func (v ToSqlVisitor) VisitTopNode(a TopNode) string {
	return "TopNode"
}

func (v ToSqlVisitor) VisitLimitNode(a LimitNode) string {
	return "LimitNode"
}

func (v ToSqlVisitor) VisitLockNode(a LockNode) string {
	return "LockNode"
}

func (v ToSqlVisitor) VisitOffsetNode(a OffsetNode) string {
	return "OFFSET " + v.Visit(a.expr)
}

func (v ToSqlVisitor) VisitDistinctOnNode(a DistinctOnNode) string {
	return "DistinctOnNode"
}

func (v ToSqlVisitor) VisitAndNode(a AndNode) string {
	return "AndNode"
}

func (v ToSqlVisitor) VisitInNode(a InNode) string {
	return "InNode"
}

func (v ToSqlVisitor) VisitOrderingNode(a OrderingNode) string {
	return "OrderingNode"
}

func (v ToSqlVisitor) VisitTable(t *Table) string {
	var buf bytes.Buffer
	if len(t.TableAlias) > 0 {
		buf.WriteString(v.QuoteTableName(t.Name))
		buf.WriteString(SPACE)
		buf.WriteString(v.QuoteTableName(t.TableAlias))
	} else {
		buf.WriteString(v.QuoteTableName(t.Name))
	}
	return buf.String()
}

func (v ToSqlVisitor) QuoteTableName(name string) string {
	return v.conn.QuoteTableName(name)
}

func (v ToSqlVisitor) VisitJoinSourceNode(a JoinSource) string {
	var buf bytes.Buffer
	if a.Left != nil {
		log.Printf("VisitJoinSourceNode: %v", a.Left.Name)
		buf.WriteString(v.VisitTable(a.Left))
	}
	return buf.String()
}

func (v ToSqlVisitor) VisitSqlLiteralNode(a SqlLiteralNode) string {
	if len(a.Raw) > 0 {
		return a.Raw
	} else {
		return ""
	}
}

func (v ToSqlVisitor) VisitSelectCoreNode(s SelectCoreNode) string {
	var buf bytes.Buffer

	buf.WriteString("SELECT")

	// Add TOP statement to the buffer
	if s.Top != nil {
		buf.WriteString(SPACE)
		buf.WriteString(v.VisitTopNode(*s.Top))
	}

	if s.SetQuanifier != nil {
		buf.WriteString(SPACE)
		buf.WriteString(v.Visit(*s.SetQuanifier))
	}

	// add select projections
	if s.Projections != nil {
		claused := false
		for i, projection := range *s.Projections {

			if projection != nil {
				if !claused {
					buf.WriteString(SPACE)
					claused = true
				}
				buf.WriteString(v.Visit(projection))
				if (len(*s.Projections) - 1) != i {
					buf.WriteString(COMMA)
				}
			}
		}
	}

	// add FROM statement to the buffer
	if s.Source != nil &&
		s.Source.Left != nil &&
		len(s.Source.Left.Name) > 0 {
		buf.WriteString(" FROM ")
		buf.WriteString(v.Visit(*s.Source))
	}

	// add WHERE statement to the buffer
	if s.Wheres != nil {
		claused := false
		for i, where := range *s.Wheres {
			// add WHERE clause if it hasn't already been added
			if !claused {
				buf.WriteString(WHERE)
				claused = true
			}
			buf.WriteString(v.Visit(where))
			if (len(*s.Wheres) - 1) != i {
				buf.WriteString(COMMA)
			}
		}
	}

	// add GROUP BY statement to the buffer
	if s.Groups != nil {
		claused := false
		for i, group := range *s.Groups {
			// add GROUP BY clause if it hasn't already been added
			if !claused {
				buf.WriteString(GROUP_BY)
				claused = true
			}
			buf.WriteString(v.Visit(group))
			if (len(*s.Groups) - 1) != i {
				buf.WriteString(COMMA)
			}
		}
	}

	// add HAVING statement to the buffer
	if s.Having != nil {
		buf.WriteString(SPACE)
		buf.WriteString(v.Visit(s.Having))
	}

	// add WINDOW statements to the buffer
	if s.Windows != nil {
		claused := false
		for i, window := range *s.Windows {
			// add WINDOW clause if is hasn't already been added
			if !claused {
				buf.WriteString(WINDOW)
				claused = true
			}
			buf.WriteString(v.Visit(window))
			if (len(*s.Windows) - 1) != i {
				buf.WriteString(COMMA)
			}
		}
	}

	return buf.String()
}

func (v ToSqlVisitor) VisitSelectStatementNode(s SelectStatementNode) string {
	var buf bytes.Buffer

	// add WITH statement to the buffer
	if s.With != nil {
		buf.WriteString(v.Visit(s.With))
	}

	// add SELECT core to the buffer
	if s.Cores != nil {
		for _, core := range s.Cores {
			if core != nil {
				buf.WriteString(v.VisitSelectCoreNode(*core))
			}
		}
	}

	// add ORDER BY clauses to the buffer
	if s.Orders != nil {
		buf.WriteString(SPACE)
		buf.WriteString(ORDER_BY)
		for i, order := range *s.Orders {
			buf.WriteString(v.VisitOrderingNode(order))
			if (len(*s.Orders) - 1) != i {
				buf.WriteString(COMMA)
			}
		}
	}

	// add LIMIT clause to the buffer
	if s.Limit != nil {
		buf.WriteString(SPACE)
		buf.WriteString(v.VisitLimitNode(*s.Limit))
	}

	// add OFFSET clause to the buffer
	if s.Offset != nil {
		buf.WriteString(SPACE)
		buf.WriteString(v.VisitOffsetNode(*s.Offset))
	}

	// add LOCK clause to the buffer
	if s.Lock != nil {
		buf.WriteString(SPACE)
		buf.WriteString(v.VisitLockNode(*s.Lock))
	}

	return strings.TrimSpace(buf.String())
}
<commit_msg>Log fatal for unhandled nodes<commit_after>package arel

import (
	"bytes"
	"log"
	"strings"
)

type ToSqlVisitor struct {
	conn *Connection
}

const (
	WHERE    = " WHERE "
	SPACE    = " "
	COMMA    = ", "
	GROUP_BY = " GROUP BY "
	ORDER_BY = " ORDER BY "
	WINDOW   = " WINDOW "
	AND      = " AND "
	DISTINCT = "DISTINCT"
)

func NewToSqlVisitor(c *Connection) ToSqlVisitor {
	return ToSqlVisitor{conn: c}
}

func (v ToSqlVisitor) Accept(a AstNode) string {
	return v.Visit(a)
}

func (v ToSqlVisitor) Visit(a AstNode) string {
	ret := ""
	switch val := a.(type) {
	case SelectStatementNode:
		ret = v.VisitSelectStatementNode(val)
	case AndNode:
		ret = v.VisitAndNode(val)
	case InNode:
		ret = v.VisitInNode(val)
	case SqlLiteralNode:
		ret = v.VisitSqlLiteralNode(val)
	case JoinSource:
		ret = v.VisitJoinSourceNode(val)
	default:
		log.Fatalf("ToSqlVisitor#Visit %T not handled", a)
	}
	return ret
}

func (v ToSqlVisitor) VisitTopNode(a TopNode) string {
	return "TopNode"
}

func (v ToSqlVisitor) VisitLimitNode(a LimitNode) string {
	return "LimitNode"
}

func (v ToSqlVisitor) VisitLockNode(a LockNode) string {
	return "LockNode"
}

func (v ToSqlVisitor) VisitOffsetNode(a OffsetNode) string {
	return "OFFSET " + v.Visit(a.expr)
}

func (v ToSqlVisitor) VisitDistinctOnNode(a DistinctOnNode) string {
	return "DistinctOnNode"
}

func (v ToSqlVisitor) VisitAndNode(a AndNode) string {
	return "AndNode"
}

func (v ToSqlVisitor) VisitInNode(a InNode) string {
	return "InNode"
}

func (v ToSqlVisitor) VisitOrderingNode(a OrderingNode) string {
	return "OrderingNode"
}

func (v ToSqlVisitor) VisitTable(t *Table) string {
	var buf bytes.Buffer
	if len(t.TableAlias) > 0 {
		buf.WriteString(v.QuoteTableName(t.Name))
		buf.WriteString(SPACE)
		buf.WriteString(v.QuoteTableName(t.TableAlias))
	} else {
		buf.WriteString(v.QuoteTableName(t.Name))
	}
	return buf.String()
}

func (v ToSqlVisitor) QuoteTableName(name string) string {
	return v.conn.QuoteTableName(name)
}

func (v ToSqlVisitor) VisitJoinSourceNode(a JoinSource) string {
	var buf bytes.Buffer
	if a.Left != nil {
		log.Printf("VisitJoinSourceNode: %v", a.Left.Name)
		buf.WriteString(v.VisitTable(a.Left))
	}
	return buf.String()
}

func (v ToSqlVisitor) VisitSqlLiteralNode(a SqlLiteralNode) string {
	if len(a.Raw) > 0 {
		return a.Raw
	} else {
		return ""
	}
}

func (v ToSqlVisitor) VisitSelectCoreNode(s SelectCoreNode) string {
	var buf bytes.Buffer

	buf.WriteString("SELECT")

	// Add TOP statement to the buffer
	if s.Top != nil {
		buf.WriteString(SPACE)
		buf.WriteString(v.VisitTopNode(*s.Top))
	}

	if s.SetQuanifier != nil {
		buf.WriteString(SPACE)
		buf.WriteString(v.Visit(*s.SetQuanifier))
	}

	// add select projections
	if s.Projections != nil {
		claused := false
		for i, projection := range *s.Projections {

			if projection != nil {
				if !claused {
					buf.WriteString(SPACE)
					claused = true
				}
				buf.WriteString(v.Visit(projection))
				if (len(*s.Projections) - 1) != i {
					buf.WriteString(COMMA)
				}
			}
		}
	}

	// add FROM statement to the buffer
	if s.Source != nil &&
		s.Source.Left != nil &&
		len(s.Source.Left.Name) > 0 {
		buf.WriteString(" FROM ")
		buf.WriteString(v.Visit(*s.Source))
	}

	// add WHERE statement to the buffer
	if s.Wheres != nil {
		claused := false
		for i, where := range *s.Wheres {
			// add WHERE clause if it hasn't already been added
			if !claused {
				buf.WriteString(WHERE)
				claused = true
			}
			buf.WriteString(v.Visit(where))
			if (len(*s.Wheres) - 1) != i {
				buf.WriteString(COMMA)
			}
		}
	}

	// add GROUP BY statement to the buffer
	if s.Groups != nil {
		claused := false
		for i, group := range *s.Groups {
			// add GROUP BY clause if it hasn't already been added
			if !claused {
				buf.WriteString(GROUP_BY)
				claused = true
			}
			buf.WriteString(v.Visit(group))
			if (len(*s.Groups) - 1) != i {
				buf.WriteString(COMMA)
			}
		}
	}

	// add HAVING statement to the buffer
	if s.Having != nil {
		buf.WriteString(SPACE)
		buf.WriteString(v.Visit(s.Having))
	}

	// add WINDOW statements to the buffer
	if s.Windows != nil {
		claused := false
		for i, window := range *s.Windows {
			// add WINDOW clause if is hasn't already been added
			if !claused {
				buf.WriteString(WINDOW)
				claused = true
			}
			buf.WriteString(v.Visit(window))
			if (len(*s.Windows) - 1) != i {
				buf.WriteString(COMMA)
			}
		}
	}

	return buf.String()
}

func (v ToSqlVisitor) VisitSelectStatementNode(s SelectStatementNode) string {
	var buf bytes.Buffer

	// add WITH statement to the buffer
	if s.With != nil {
		buf.WriteString(v.Visit(s.With))
	}

	// add SELECT core to the buffer
	if s.Cores != nil {
		for _, core := range s.Cores {
			if core != nil {
				buf.WriteString(v.VisitSelectCoreNode(*core))
			}
		}
	}

	// add ORDER BY clauses to the buffer
	if s.Orders != nil {
		buf.WriteString(SPACE)
		buf.WriteString(ORDER_BY)
		for i, order := range *s.Orders {
			buf.WriteString(v.VisitOrderingNode(order))
			if (len(*s.Orders) - 1) != i {
				buf.WriteString(COMMA)
			}
		}
	}

	// add LIMIT clause to the buffer
	if s.Limit != nil {
		buf.WriteString(SPACE)
		buf.WriteString(v.VisitLimitNode(*s.Limit))
	}

	// add OFFSET clause to the buffer
	if s.Offset != nil {
		buf.WriteString(SPACE)
		buf.WriteString(v.VisitOffsetNode(*s.Offset))
	}

	// add LOCK clause to the buffer
	if s.Lock != nil {
		buf.WriteString(SPACE)
		buf.WriteString(v.VisitLockNode(*s.Lock))
	}

	return strings.TrimSpace(buf.String())
}
<|endoftext|>


<commit_before>package main

import (
	"context"
	"encoding/xml"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"sort"

	"cloud.google.com/go/logging"
	"github.com/google/osv/vulnfeeds/cves"
	"github.com/google/osv/vulnfeeds/utility"
	"golang.org/x/exp/slices"
)

type CPEDict struct {
	XMLName  xml.Name  `xml:"cpe-list"`
	CPEItems []CPEItem `xml:"cpe-item"`
}

type CPEItem struct {
	XMLName    xml.Name    `xml:"cpe-item" json:"-"`
	Name       string      `xml:"name,attr" json:"name"`
	Title      string      `xml:"title" json:"title"`
	References []Reference `xml:"references>reference" json:"references"`
	CPE23      CPE23Item   `xml:"cpe23-item" json:"cpe23-item"`
}

type Reference struct {
	URL         string `xml:"href,attr" json:"URL"`
	Description string `xml:",chardata" json:"description"`
}

type CPE23Item struct {
	Name string `xml:"name,attr"`
}

const (
	CPEDictionaryDefault = "cve_jsons/nvdcpematch-1.0.json"
	projectId            = "oss-vdb"
)

var (
	Logger utility.LoggerWrapper
	// These repos should never be considered authoritative for a product.
	InvalidRepos = []string{
		"https://github.com/CVEProject/cvelist", // Heavily in Advisory URLs, sometimes shows up elsewhere
		"https://github.com/github/cvelist",     // Fork of the above
	}
	CPEDictionaryFile = flag.String("cpe_dictionary", CPEDictionaryDefault, "CPE Dictionary file to parse")
)

func LoadCPEDictionary(f string) (CPEDict, error) {
	xmlFile, err := os.Open(f)
	if err != nil {
		Logger.Fatalf("Failed to open %s: %v", f, err)
	}

	defer xmlFile.Close()

	byteValue, _ := ioutil.ReadAll(xmlFile)

	var c CPEDict
	xml.Unmarshal(byteValue, &c)

	return c, nil
}

func main() {
	client, err := logging.NewClient(context.Background(), projectId)
	if err != nil {
		log.Fatalf("Failed to create client: %v", err)
	}
	defer client.Close()
	Logger.GCloudLogger = client.Logger("cperepos")
	flag.Parse()

	CPEDictionary, err := LoadCPEDictionary(*CPEDictionaryFile)
	if err != nil {
		Logger.Fatalf("Failed to load %s: %v", *CPEDictionaryFile, err)
	}
	ProductToRepo := make(map[string]*string)
	DescriptionFrequency := make(map[string]int)
	for _, c := range CPEDictionary.CPEItems {
		CPE, err := cves.ParseCPE(c.CPE23.Name)
		if err != nil {
			Logger.Infof("Failed to parse %q", c.CPE23.Name)
			continue
		}
		if CPE.Part != "a" {
			// Not interested in hardware or operating systems.
			continue
		}
		if _, exists := ProductToRepo[CPE.Product]; !exists {
			// This way every seen product will exist in the map,
			// and ones we never determine a repo for will have a
			// nil value.
			ProductToRepo[CPE.Product] = nil
		} else {
			if ProductToRepo[CPE.Product] != nil {
				Logger.Infof("Already have %q for %q, skipping relookup", *ProductToRepo[CPE.Product], CPE.Product)
				continue
			}
		}
		for _, r := range c.References {
			DescriptionFrequency[r.Description] += 1
			repo, err := cves.Repo(r.URL)
			if err != nil {
				Logger.Infof("Disregarding %q for %q (%s) because %v", r.URL, CPE.Product, r.Description, err)
				continue
			}
			// Disregard the repos we know we don't like.
			if slices.Contains(InvalidRepos, repo) {
				Logger.Infof("Disliking %q for %q (%s)", repo, CPE.Product, r.Description)
				continue
			}
			Logger.Infof("Liking %q for %q (%s)", repo, CPE.Product, r.Description)
			ProductToRepo[CPE.Product] = &repo
		}
	}
	productsWithoutRepos := 0
	products := make([]string, 0, len(ProductToRepo))
	for p := range ProductToRepo {
		products = append(products, p)
	}
	sort.Strings(products)
	for _, product := range products {
		if ProductToRepo[product] == nil {
			productsWithoutRepos++
			continue
		}
		fmt.Printf("%s -> %s\n", product, *ProductToRepo[product])
	}
	Logger.Infof("Loaded information about %d application products, %d do not have repos", len(ProductToRepo), productsWithoutRepos)
	fmt.Printf("Loaded information about %d products, %d do not have repos\n", len(ProductToRepo), productsWithoutRepos)
	descriptions := make([]string, 0, len(DescriptionFrequency))
	for description := range DescriptionFrequency {
		descriptions = append(descriptions, description)
	}
	sort.SliceStable(descriptions, func(i, j int) bool {
		return DescriptionFrequency[descriptions[i]] > DescriptionFrequency[descriptions[j]]
	})
	for _, d := range descriptions {
		fmt.Printf("%s,%d\n", d, DescriptionFrequency[d])
	}
	Logger.Infof("Seen %d descriptions", len(DescriptionFrequency))
	fmt.Printf("Seen %d descriptions \n", len(DescriptionFrequency))
}
<commit_msg>Produce reusable output from cperepo tool (#853)<commit_after>/*
cperepos analyzes the NVD CPE Dictionary for Open Source repository information.
It reads the NVD CPE Dictionary XML file and outputs a JSON map of CPE products to discovered repository URLs.

It can also output on stdout additional data about colliding CPE package names.

Usage:

	go run cmd/cperepos/main.go [flags]

The flags are:

	  --cpe_dictionary
		The path to the uncompressed NVD CPE Dictionary XML file, see https://nvd.nist.gov/products/cpe

	  --output_dir
	        The directory to output cpe_product_to_repo.json and cpe_reference_description_frequency.csv in

	  --gcp_logging_project
		The GCP project ID to utilise for Cloud Logging. Set to the empty string to log to stdout

	  --verbose
		Output additional telemetry to stdout
*/
package main

import (
	"context"
	"encoding/csv"
	"encoding/json"
	"encoding/xml"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"sort"
	"strconv"
	"strings"

	"cloud.google.com/go/logging"
	"github.com/google/osv/vulnfeeds/cves"
	"github.com/google/osv/vulnfeeds/utility"
	"golang.org/x/exp/slices"
)

type CPEDict struct {
	XMLName  xml.Name  `xml:"cpe-list"`
	CPEItems []CPEItem `xml:"cpe-item"`
}

type CPEItem struct {
	XMLName    xml.Name    `xml:"cpe-item" json:"-"`
	Name       string      `xml:"name,attr" json:"name"`
	Title      string      `xml:"title" json:"title"`
	References []Reference `xml:"references>reference" json:"references"`
	CPE23      CPE23Item   `xml:"cpe23-item" json:"cpe23-item"`
}

type Reference struct {
	URL         string `xml:"href,attr" json:"URL"`
	Description string `xml:",chardata" json:"description"`
}

type CPE23Item struct {
	Name string `xml:"name,attr"`
}

const (
	CPEDictionaryDefault = "cve_jsons/nvdcpematch-1.0.json"
	OutputDirDefault     = "."
	projectId            = "oss-vdb"
)

var (
	Logger utility.LoggerWrapper
	// These repos should never be considered authoritative for a product.
	InvalidRepos = []string{
		"https://github.com/CVEProject/cvelist", // Heavily in Advisory URLs, sometimes shows up elsewhere
		"https://github.com/github/cvelist",     // Fork of the above
	}
	CPEDictionaryFile = flag.String("cpe_dictionary", CPEDictionaryDefault, "CPE Dictionary file to parse")
	OutputDir         = flag.String("output_dir", OutputDirDefault, "Directory to output cpe_product_to_repo.json and cpe_reference_description_frequency.csv in")
	GCPLoggingProject = flag.String("gcp_logging_project", projectId, "GCP project ID to use for logging, set to an empty string to log locally only")
	Verbose           = flag.Bool("verbose", false, "Output some telemetry to stdout during execution")
)

func LoadCPEDictionary(f string) (CPEDict, error) {
	xmlFile, err := os.Open(f)
	if err != nil {
		Logger.Fatalf("Failed to open %s: %v", f, err)
	}

	defer xmlFile.Close()

	byteValue, _ := ioutil.ReadAll(xmlFile)

	var c CPEDict
	xml.Unmarshal(byteValue, &c)

	return c, nil
}

// Outputs a JSON file of the product-to-repo map.
func outputProductToRepoMap(prm map[string]*string, f io.Writer) error {
	productsWithoutRepos := 0
	for p := range prm {
		if prm[p] == nil {
			productsWithoutRepos++
			delete(prm, p) // we don't want the repo-less products in our JSON output
			continue
		}
	}

	e := json.NewEncoder(f)

	if err := e.Encode(&prm); err != nil {
		return err
	}

	if *Verbose {
		fmt.Printf("Loaded information about %d products, %d do not have repos\n", len(prm), productsWithoutRepos)
	}
	Logger.Infof("Loaded information about %d application products, %d do not have repos", len(prm), productsWithoutRepos)
	return nil
}

// Outputs a CSV file of the description frequency map, sorted in descending order.
func outputDescriptionFrequency(df map[string]int, f io.Writer) error {
	descriptions := make([]string, 0, len(df))
	for description := range df {
		descriptions = append(descriptions, description)
	}
	sort.SliceStable(descriptions, func(i, j int) bool {
		return df[descriptions[i]] > df[descriptions[j]]
	})

	w := csv.NewWriter(f)

	if err := w.Write([]string{"Description", "Frequency"}); err != nil {
		return err
	}
	for _, d := range descriptions {
		if err := w.Write([]string{d, strconv.Itoa(df[d])}); err != nil {
			return err
		}
	}

	w.Flush()

	if err := w.Error(); err != nil {
		return err
	}

	if *Verbose {
		fmt.Printf("Seen %d descriptions \n", len(df))
	}
	Logger.Infof("Seen %d descriptions", len(df))
	return nil
}

// Analyze CPE Dictionary
func analyzeCPEDictionary(d CPEDict) (ProductToRepo map[string]*string, DescriptionFrequency map[string]int, ProductToVendor map[string][]string) {
	ProductToRepo = make(map[string]*string)
	DescriptionFrequency = make(map[string]int)
	ProductToVendor = make(map[string][]string)
	for _, c := range d.CPEItems {
		CPE, err := cves.ParseCPE(c.CPE23.Name)
		if err != nil {
			Logger.Infof("Failed to parse %q", c.CPE23.Name)
			continue
		}
		if CPE.Part != "a" {
			// Not interested in hardware or operating systems.
			continue
		}
		// Gather the data to answer the question: "are there product name collisions?"
		if *Verbose {
			if _, exists := ProductToVendor[CPE.Product]; exists {
				if !slices.Contains(ProductToVendor[CPE.Product], CPE.Vendor) {
					ProductToVendor[CPE.Product] = append(ProductToVendor[CPE.Product], CPE.Vendor)
				}
			} else {
				ProductToVendor[CPE.Product] = []string{CPE.Vendor}
			}
		}
		if _, exists := ProductToRepo[CPE.Product]; !exists {
			// This way every seen product will exist in the map,
			// and ones we never determine a repo for will have a
			// nil value.
			ProductToRepo[CPE.Product] = nil
		} else {
			if ProductToRepo[CPE.Product] != nil {
				Logger.Infof("Already have %q for %q, skipping relookup", *ProductToRepo[CPE.Product], CPE.Product)
				continue
			}
		}
		for _, r := range c.References {
			DescriptionFrequency[r.Description] += 1
			repo, err := cves.Repo(r.URL)
			if err != nil {
				Logger.Infof("Disregarding %q for %q (%s) because %v", r.URL, CPE.Product, r.Description, err)
				continue
			}
			// Disregard the repos we know we don't like.
			if slices.Contains(InvalidRepos, repo) {
				Logger.Infof("Disliking %q for %q (%s)", repo, CPE.Product, r.Description)
				continue
			}
			Logger.Infof("Liking %q for %q (%s)", repo, CPE.Product, r.Description)
			ProductToRepo[CPE.Product] = &repo
		}
	}
	return ProductToRepo, DescriptionFrequency, ProductToVendor
}

func main() {
	flag.Usage = func() {
		fmt.Fprint(flag.CommandLine.Output(), "Utility to analyze NVD CPE Dictionary\n\n")
		fmt.Fprintf(flag.CommandLine.Output(), "Usage of %s:\n", os.Args[0])
		flag.PrintDefaults()
	}
	flag.Parse()

	if *GCPLoggingProject != "" {
		client, err := logging.NewClient(context.Background(), *GCPLoggingProject)
		if err != nil {
			log.Fatalf("Failed to create client: %v", err)
		}
		defer client.Close()
		Logger.GCloudLogger = client.Logger("cperepos")
	}

	CPEDictionary, err := LoadCPEDictionary(*CPEDictionaryFile)
	if err != nil {
		Logger.Fatalf("Failed to load %s: %v", *CPEDictionaryFile, err)
	}

	productToRepo, descriptionFrequency, productToVendor := analyzeCPEDictionary(CPEDictionary)

	mappingFile, err := os.Create(filepath.Join(*OutputDir, "cpe_product_to_repo.json"))
	if err != nil {
		Logger.Fatalf("%v", err)
	}
	defer mappingFile.Close()
	err = outputProductToRepoMap(productToRepo, mappingFile)
	if err != nil {
		Logger.Fatalf("%v", err)
	}
	// Answer the question: "are there product name collisions?"
	if *Verbose {
		for product, vendors := range productToVendor {
			if len(vendors) == 1 {
				continue
			}
			Logger.Infof("Product %s has >1 vendors: [%s]", product, strings.Join(vendors, ", "))
			fmt.Printf("Product %s has >1 vendors: [%s]\n", product, strings.Join(vendors, ", "))
		}
	}
	frequencyFile, err := os.Create(filepath.Join(*OutputDir, "cpe_reference_description_frequency.csv"))
	if err != nil {
		Logger.Fatalf("%v", err)
	}
	defer frequencyFile.Close()
	err = outputDescriptionFrequency(descriptionFrequency, frequencyFile)
	if err != nil {
		Logger.Fatalf("%v", err)
	}
}
<|endoftext|>


<commit_before>/*
rqlite -- replicating SQLite via the Raft consensus protocol..

rqlite is a distributed system that provides a replicated relational database,
using SQLite as the storage engine.

rqlite is written in Go and uses Raft to achieve consensus across all the
instances of the SQLite databases. rqlite ensures that every change made to
the database is made to a majority of underlying SQLite files, or none-at-all.
*/

package main

import (
	"flag"
	"fmt"
	"log"
	"net"
	"os"
	"os/signal"
	"path/filepath"
	"runtime"
	"runtime/pprof"
	"time"

	"github.com/rqlite/rqlite/auth"
	"github.com/rqlite/rqlite/cluster"
	"github.com/rqlite/rqlite/disco"
	httpd "github.com/rqlite/rqlite/http"
	"github.com/rqlite/rqlite/store"
	"github.com/rqlite/rqlite/tcp"
)

const sqliteDSN = "db.sqlite"

const logo = `
            _ _ _           _
           | (_) |         | |
  _ __ __ _| |_| |_ ___  __| |
 | '__/ _  | | | __/ _ \/ _  |  The lightweight, distributed
 | | | (_| | | | ||  __/ (_| |  relational database.
 |_|  \__, |_|_|\__\___|\__,_|
         | |
         |_|
`

// These variables are populated via the Go linker.
var (
	version   = "3"
	commit    = "unknown"
	branch    = "unknown"
	buildtime = "unknown"
)

const (
	muxRaftHeader = 1 // Raft consensus communications
	muxMetaHeader = 2 // Cluster meta communications
)

const (
	publishPeerDelay   = 1 * time.Second
	publishPeerTimeout = 30 * time.Second
)

var httpAddr string
var httpAdv string
var authFile string
var x509Cert string
var x509Key string
var raftAddr string
var raftAdv string
var joinAddr string
var noVerify bool
var discoURL string
var discoID string
var expvar bool
var pprofEnabled bool
var dsn string
var onDisk bool
var raftSnapThreshold uint64
var raftHeartbeatTimeout string
var raftApplyTimeout string
var raftOpenTimeout string
var showVersion bool
var cpuProfile string
var memProfile string

const name = `rqlited`
const desc = `rqlite is a distributed system that provides a replicated relational database.`

func init() {
	flag.StringVar(&httpAddr, "http", "localhost:4001", "HTTP server bind address. For HTTPS, set X.509 cert and key")
	flag.StringVar(&httpAdv, "httpadv", "", "Advertised HTTP address. If not set, same as HTTP server")
	flag.StringVar(&x509Cert, "x509cert", "", "Path to X.509 certificate")
	flag.StringVar(&x509Key, "x509key", "", "Path to X.509 private key for certificate")
	flag.StringVar(&authFile, "auth", "", "Path to authentication and authorization file. If not set, not enabled")
	flag.StringVar(&raftAddr, "raft", "localhost:4002", "Raft communication bind address")
	flag.StringVar(&raftAdv, "raftadv", "", "Advertised Raft communication address. If not set, same as Raft bind")
	flag.StringVar(&joinAddr, "join", "", "Join a cluster via node at protocol://host:port")
	flag.BoolVar(&noVerify, "noverify", false, "Skip verification of remote HTTPS cert when joining cluster")
	flag.StringVar(&discoURL, "disco", "http://discovery.rqlite.com", "Set Discovery Service URL")
	flag.StringVar(&discoID, "discoid", "", "Set Discovery ID. If not set, Discovery Service not used")
	flag.BoolVar(&expvar, "expvar", true, "Serve expvar data on HTTP server")
	flag.BoolVar(&pprofEnabled, "pprof", true, "Serve pprof data on HTTP server")
	flag.StringVar(&dsn, "dsn", "", `SQLite DSN parameters. E.g. "cache=shared&mode=memory"`)
	flag.BoolVar(&onDisk, "ondisk", false, "Use an on-disk SQLite database")
	flag.BoolVar(&showVersion, "version", false, "Show version information and exit")
	flag.StringVar(&raftHeartbeatTimeout, "rafttimeout", "1s", "Raft heartbeat timeout")
	flag.StringVar(&raftApplyTimeout, "raftapplytimeout", "10s", "Raft apply timeout")
	flag.StringVar(&raftOpenTimeout, "raftopentimeout", "120s", "Time for initial Raft logs to be applied. Use 0s duration to skip wait")
	flag.Uint64Var(&raftSnapThreshold, "raftsnap", 8192, "Number of outstanding log entries that trigger snapshot")
	flag.StringVar(&cpuProfile, "cpuprofile", "", "Path to file for CPU profiling information")
	flag.StringVar(&memProfile, "memprofile", "", "Path to file for memory profiling information")
	flag.Usage = func() {
		fmt.Fprintf(os.Stderr, "\n%s\n\n", desc)
		fmt.Fprintf(os.Stderr, "Usage: %s [arguments] <data directory>\n", name)
		flag.PrintDefaults()
	}
}

func main() {
	flag.Parse()

	if showVersion {
		fmt.Printf("%s %s %s %s (commit %s, branch %s)\n",
			name, version, runtime.GOOS, runtime.GOARCH, commit, branch)
		os.Exit(0)
	}

	// Ensure the data path is set.
	if flag.NArg() == 0 {
		flag.Usage()
		os.Exit(1)
	}

	dataPath := flag.Arg(0)

	// Display logo.
	fmt.Println(logo)

	// Configure logging and pump out initial message.
	log.SetFlags(log.LstdFlags)
	log.SetOutput(os.Stderr)
	log.SetPrefix(fmt.Sprintf("[%s] ", name))
	log.Printf("%s starting, version %s, commit %s, branch %s", name, version, commit, branch)
	log.Printf("target architecture is %s, operating system target is %s", runtime.GOARCH, runtime.GOOS)

	// Start requested profiling.
	startProfile(cpuProfile, memProfile)

	// Set up TCP communication between nodes.
	ln, err := net.Listen("tcp", raftAddr)
	if err != nil {
		log.Fatalf("failed to listen on %s: %s", raftAddr, err.Error())
	}
	var adv net.Addr
	if raftAdv != "" {
		adv, err = net.ResolveTCPAddr("tcp", raftAdv)
		if err != nil {
			log.Fatalf("failed to resolve advertise address %s: %s", raftAdv, err.Error())
		}
	}
	mux := tcp.NewMux(ln, adv)
	go mux.Serve()

	// Start up mux and get transports for cluster.
	raftTn := mux.Listen(muxRaftHeader)

	// Create and open the store.
	dataPath, err = filepath.Abs(dataPath)
	if err != nil {
		log.Fatalf("failed to determine absolute data path: %s", err.Error())
	}
	dbConf := store.NewDBConfig(dsn, !onDisk)

	str := store.New(&store.StoreConfig{
		DBConf: dbConf,
		Dir:    dataPath,
		Tn:     raftTn,
	})

	// Set optional parameters on store.
	str.SnapshotThreshold = raftSnapThreshold
	str.HeartbeatTimeout, err = time.ParseDuration(raftHeartbeatTimeout)
	if err != nil {
		log.Fatalf("failed to parse Raft heartbeat timeout %s: %s", raftHeartbeatTimeout, err.Error())
	}
	str.ApplyTimeout, err = time.ParseDuration(raftApplyTimeout)
	if err != nil {
		log.Fatalf("failed to parse Raft apply timeout %s: %s", raftApplyTimeout, err.Error())
	}
	str.OpenTimeout, err = time.ParseDuration(raftOpenTimeout)
	if err != nil {
		log.Fatalf("failed to parse Raft open timeout %s: %s", raftOpenTimeout, err.Error())
	}

	// Determine join addresses, if necessary.
	ja, err := store.JoinAllowed(dataPath)
	if err != nil {
		log.Fatalf("unable to determine if join permitted: %s", err.Error())
	}

	var joins []string
	if ja {
		joins, err = determineJoinAddresses()
		if err != nil {
			log.Fatalf("unable to determine join addresses: %s", err.Error())
		}
	} else {
		log.Println("node is already member of cluster, skip determining join addresses")
	}

	// Now, open it.
	if err := str.Open(len(joins) == 0); err != nil {
		log.Fatalf("failed to open store: %s", err.Error())
	}

	// Create and configure cluster service.
	tn := mux.Listen(muxMetaHeader)
	cs := cluster.NewService(tn, str)
	if err := cs.Open(); err != nil {
		log.Fatalf("failed to open cluster service: %s", err.Error())
	}

	// Execute any requested join operation.
	if len(joins) > 0 {
		log.Println("join addresses are:", joins)
		advAddr := raftAddr
		if raftAdv != "" {
			advAddr = raftAdv
		}
		if j, err := cluster.Join(joins, advAddr, noVerify); err != nil {
			log.Fatalf("failed to join cluster at %s: %s", joins, err.Error())
		} else {
			log.Println("successfully joined cluster at", j)
		}

	} else {
		log.Println("no join addresses set")
	}

	// Publish to the cluster the mapping between this Raft address and API address.
	// The Raft layer broadcasts the resolved address, so use that as the key. But
	// only set different HTTP advertise address if set.
	apiAdv := httpAddr
	if httpAdv != "" {
		apiAdv = httpAdv
	}

	if err := publishAPIAddr(cs, raftTn.Addr().String(), apiAdv, publishPeerTimeout); err != nil {
		log.Fatalf("failed to set peer for %s to %s: %s", raftAddr, httpAddr, err.Error())
	}
	log.Printf("set peer for %s to %s", raftTn.Addr().String(), apiAdv)

	// Create HTTP server and load authentication information, if supplied.
	var s *httpd.Service
	if authFile != "" {
		f, err := os.Open(authFile)
		if err != nil {
			log.Fatalf("failed to open authentication file %s: %s", authFile, err.Error())
		}
		credentialStore := auth.NewCredentialsStore()
		if err := credentialStore.Load(f); err != nil {
			log.Fatalf("failed to load authentication file: %s", err.Error())
		}
		s = httpd.New(httpAddr, str, credentialStore)
	} else {
		s = httpd.New(httpAddr, str, nil)
	}

	s.CertFile = x509Cert
	s.KeyFile = x509Key
	s.Expvar = expvar
	s.Pprof = pprofEnabled
	s.BuildInfo = map[string]interface{}{
		"commit":     commit,
		"branch":     branch,
		"version":    version,
		"build_time": buildtime,
	}
	if err := s.Start(); err != nil {
		log.Fatalf("failed to start HTTP server: %s", err.Error())
	}

	terminate := make(chan os.Signal, 1)
	signal.Notify(terminate, os.Interrupt)
	<-terminate
	if err := str.Close(true); err != nil {
		log.Printf("failed to close store: %s", err.Error())
	}
	stopProfile()
	log.Println("rqlite server stopped")
}

func determineJoinAddresses() ([]string, error) {
	apiAdv := httpAddr
	if httpAdv != "" {
		apiAdv = httpAdv
	}

	var addrs []string
	if joinAddr != "" {
		// An explicit join address is first priority.
		addrs = append(addrs, joinAddr)
	}

	if discoID != "" {
		log.Printf("registering with Discovery Service at %s with ID %s", discoURL, discoID)
		c := disco.New(discoURL)
		r, err := c.Register(discoID, apiAdv)
		if err != nil {
			return nil, err
		}
		log.Println("Discovery Service responded with nodes:", r.Nodes)
		for _, a := range r.Nodes {
			if a != apiAdv {
				// Only other nodes can be joined.
				addrs = append(addrs, a)
			}
		}
	}

	return addrs, nil
}

func publishAPIAddr(c *cluster.Service, raftAddr, apiAddr string, t time.Duration) error {
	tck := time.NewTicker(publishPeerDelay)
	defer tck.Stop()
	tmr := time.NewTimer(t)
	defer tmr.Stop()

	for {
		select {
		case <-tck.C:
			if err := c.SetPeer(raftAddr, apiAddr); err != nil {
				log.Printf("failed to set peer for %s to %s: %s (retrying)",
					raftAddr, apiAddr, err.Error())
				continue
			}
			return nil
		case <-tmr.C:
			return fmt.Errorf("set peer timeout expired")
		}
	}
}

// prof stores the file locations of active profiles.
var prof struct {
	cpu *os.File
	mem *os.File
}

// startProfile initializes the CPU and memory profile, if specified.
func startProfile(cpuprofile, memprofile string) {
	if cpuprofile != "" {
		f, err := os.Create(cpuprofile)
		if err != nil {
			log.Fatalf("failed to create CPU profile file at %s: %s", cpuprofile, err.Error())
		}
		log.Printf("writing CPU profile to: %s\n", cpuprofile)
		prof.cpu = f
		pprof.StartCPUProfile(prof.cpu)
	}

	if memprofile != "" {
		f, err := os.Create(memprofile)
		if err != nil {
			log.Fatalf("failed to create memory profile file at %s: %s", cpuprofile, err.Error())
		}
		log.Printf("writing memory profile to: %s\n", memprofile)
		prof.mem = f
		runtime.MemProfileRate = 4096
	}
}

// stopProfile closes the CPU and memory profiles if they are running.
func stopProfile() {
	if prof.cpu != nil {
		pprof.StopCPUProfile()
		prof.cpu.Close()
		log.Println("CPU profiling stopped")
	}
	if prof.mem != nil {
		pprof.Lookup("heap").WriteTo(prof.mem, 0)
		prof.mem.Close()
		log.Println("memory profiling stopped")
	}
}
<commit_msg>Tweak rqlite description<commit_after>/*
rqlite -- replicating SQLite via the Raft consensus protocol..

rqlite is a distributed system that provides a replicated relational database,
using SQLite as the storage engine.

rqlite is written in Go and uses Raft to achieve consensus across all the
instances of the SQLite databases. rqlite ensures that every change made to
the database is made to a majority of underlying SQLite files, or none-at-all.
*/

package main

import (
	"flag"
	"fmt"
	"log"
	"net"
	"os"
	"os/signal"
	"path/filepath"
	"runtime"
	"runtime/pprof"
	"time"

	"github.com/rqlite/rqlite/auth"
	"github.com/rqlite/rqlite/cluster"
	"github.com/rqlite/rqlite/disco"
	httpd "github.com/rqlite/rqlite/http"
	"github.com/rqlite/rqlite/store"
	"github.com/rqlite/rqlite/tcp"
)

const sqliteDSN = "db.sqlite"

const logo = `
            _ _ _           _
           | (_) |         | |
  _ __ __ _| |_| |_ ___  __| |
 | '__/ _  | | | __/ _ \/ _  |  The lightweight, distributed
 | | | (_| | | | ||  __/ (_| |  relational database.
 |_|  \__, |_|_|\__\___|\__,_|
         | |
         |_|
`

// These variables are populated via the Go linker.
var (
	version   = "3"
	commit    = "unknown"
	branch    = "unknown"
	buildtime = "unknown"
)

const (
	muxRaftHeader = 1 // Raft consensus communications
	muxMetaHeader = 2 // Cluster meta communications
)

const (
	publishPeerDelay   = 1 * time.Second
	publishPeerTimeout = 30 * time.Second
)

var httpAddr string
var httpAdv string
var authFile string
var x509Cert string
var x509Key string
var raftAddr string
var raftAdv string
var joinAddr string
var noVerify bool
var discoURL string
var discoID string
var expvar bool
var pprofEnabled bool
var dsn string
var onDisk bool
var raftSnapThreshold uint64
var raftHeartbeatTimeout string
var raftApplyTimeout string
var raftOpenTimeout string
var showVersion bool
var cpuProfile string
var memProfile string

const name = `rqlited`
const desc = `rqlite is a lightweight, distributed relational database, which uses SQLite as its
storage engine. It provides an easy-to-use, fault-tolerant store for relational data.`

func init() {
	flag.StringVar(&httpAddr, "http", "localhost:4001", "HTTP server bind address. For HTTPS, set X.509 cert and key")
	flag.StringVar(&httpAdv, "httpadv", "", "Advertised HTTP address. If not set, same as HTTP server")
	flag.StringVar(&x509Cert, "x509cert", "", "Path to X.509 certificate")
	flag.StringVar(&x509Key, "x509key", "", "Path to X.509 private key for certificate")
	flag.StringVar(&authFile, "auth", "", "Path to authentication and authorization file. If not set, not enabled")
	flag.StringVar(&raftAddr, "raft", "localhost:4002", "Raft communication bind address")
	flag.StringVar(&raftAdv, "raftadv", "", "Advertised Raft communication address. If not set, same as Raft bind")
	flag.StringVar(&joinAddr, "join", "", "Join a cluster via node at protocol://host:port")
	flag.BoolVar(&noVerify, "noverify", false, "Skip verification of remote HTTPS cert when joining cluster")
	flag.StringVar(&discoURL, "disco", "http://discovery.rqlite.com", "Set Discovery Service URL")
	flag.StringVar(&discoID, "discoid", "", "Set Discovery ID. If not set, Discovery Service not used")
	flag.BoolVar(&expvar, "expvar", true, "Serve expvar data on HTTP server")
	flag.BoolVar(&pprofEnabled, "pprof", true, "Serve pprof data on HTTP server")
	flag.StringVar(&dsn, "dsn", "", `SQLite DSN parameters. E.g. "cache=shared&mode=memory"`)
	flag.BoolVar(&onDisk, "ondisk", false, "Use an on-disk SQLite database")
	flag.BoolVar(&showVersion, "version", false, "Show version information and exit")
	flag.StringVar(&raftHeartbeatTimeout, "rafttimeout", "1s", "Raft heartbeat timeout")
	flag.StringVar(&raftApplyTimeout, "raftapplytimeout", "10s", "Raft apply timeout")
	flag.StringVar(&raftOpenTimeout, "raftopentimeout", "120s", "Time for initial Raft logs to be applied. Use 0s duration to skip wait")
	flag.Uint64Var(&raftSnapThreshold, "raftsnap", 8192, "Number of outstanding log entries that trigger snapshot")
	flag.StringVar(&cpuProfile, "cpuprofile", "", "Path to file for CPU profiling information")
	flag.StringVar(&memProfile, "memprofile", "", "Path to file for memory profiling information")
	flag.Usage = func() {
		fmt.Fprintf(os.Stderr, "\n%s\n\n", desc)
		fmt.Fprintf(os.Stderr, "Usage: %s [arguments] <data directory>\n", name)
		flag.PrintDefaults()
	}
}

func main() {
	flag.Parse()

	if showVersion {
		fmt.Printf("%s %s %s %s (commit %s, branch %s)\n",
			name, version, runtime.GOOS, runtime.GOARCH, commit, branch)
		os.Exit(0)
	}

	// Ensure the data path is set.
	if flag.NArg() == 0 {
		flag.Usage()
		os.Exit(1)
	}

	dataPath := flag.Arg(0)

	// Display logo.
	fmt.Println(logo)

	// Configure logging and pump out initial message.
	log.SetFlags(log.LstdFlags)
	log.SetOutput(os.Stderr)
	log.SetPrefix(fmt.Sprintf("[%s] ", name))
	log.Printf("%s starting, version %s, commit %s, branch %s", name, version, commit, branch)
	log.Printf("target architecture is %s, operating system target is %s", runtime.GOARCH, runtime.GOOS)

	// Start requested profiling.
	startProfile(cpuProfile, memProfile)

	// Set up TCP communication between nodes.
	ln, err := net.Listen("tcp", raftAddr)
	if err != nil {
		log.Fatalf("failed to listen on %s: %s", raftAddr, err.Error())
	}
	var adv net.Addr
	if raftAdv != "" {
		adv, err = net.ResolveTCPAddr("tcp", raftAdv)
		if err != nil {
			log.Fatalf("failed to resolve advertise address %s: %s", raftAdv, err.Error())
		}
	}
	mux := tcp.NewMux(ln, adv)
	go mux.Serve()

	// Start up mux and get transports for cluster.
	raftTn := mux.Listen(muxRaftHeader)

	// Create and open the store.
	dataPath, err = filepath.Abs(dataPath)
	if err != nil {
		log.Fatalf("failed to determine absolute data path: %s", err.Error())
	}
	dbConf := store.NewDBConfig(dsn, !onDisk)

	str := store.New(&store.StoreConfig{
		DBConf: dbConf,
		Dir:    dataPath,
		Tn:     raftTn,
	})

	// Set optional parameters on store.
	str.SnapshotThreshold = raftSnapThreshold
	str.HeartbeatTimeout, err = time.ParseDuration(raftHeartbeatTimeout)
	if err != nil {
		log.Fatalf("failed to parse Raft heartbeat timeout %s: %s", raftHeartbeatTimeout, err.Error())
	}
	str.ApplyTimeout, err = time.ParseDuration(raftApplyTimeout)
	if err != nil {
		log.Fatalf("failed to parse Raft apply timeout %s: %s", raftApplyTimeout, err.Error())
	}
	str.OpenTimeout, err = time.ParseDuration(raftOpenTimeout)
	if err != nil {
		log.Fatalf("failed to parse Raft open timeout %s: %s", raftOpenTimeout, err.Error())
	}

	// Determine join addresses, if necessary.
	ja, err := store.JoinAllowed(dataPath)
	if err != nil {
		log.Fatalf("unable to determine if join permitted: %s", err.Error())
	}

	var joins []string
	if ja {
		joins, err = determineJoinAddresses()
		if err != nil {
			log.Fatalf("unable to determine join addresses: %s", err.Error())
		}
	} else {
		log.Println("node is already member of cluster, skip determining join addresses")
	}

	// Now, open it.
	if err := str.Open(len(joins) == 0); err != nil {
		log.Fatalf("failed to open store: %s", err.Error())
	}

	// Create and configure cluster service.
	tn := mux.Listen(muxMetaHeader)
	cs := cluster.NewService(tn, str)
	if err := cs.Open(); err != nil {
		log.Fatalf("failed to open cluster service: %s", err.Error())
	}

	// Execute any requested join operation.
	if len(joins) > 0 {
		log.Println("join addresses are:", joins)
		advAddr := raftAddr
		if raftAdv != "" {
			advAddr = raftAdv
		}
		if j, err := cluster.Join(joins, advAddr, noVerify); err != nil {
			log.Fatalf("failed to join cluster at %s: %s", joins, err.Error())
		} else {
			log.Println("successfully joined cluster at", j)
		}

	} else {
		log.Println("no join addresses set")
	}

	// Publish to the cluster the mapping between this Raft address and API address.
	// The Raft layer broadcasts the resolved address, so use that as the key. But
	// only set different HTTP advertise address if set.
	apiAdv := httpAddr
	if httpAdv != "" {
		apiAdv = httpAdv
	}

	if err := publishAPIAddr(cs, raftTn.Addr().String(), apiAdv, publishPeerTimeout); err != nil {
		log.Fatalf("failed to set peer for %s to %s: %s", raftAddr, httpAddr, err.Error())
	}
	log.Printf("set peer for %s to %s", raftTn.Addr().String(), apiAdv)

	// Create HTTP server and load authentication information, if supplied.
	var s *httpd.Service
	if authFile != "" {
		f, err := os.Open(authFile)
		if err != nil {
			log.Fatalf("failed to open authentication file %s: %s", authFile, err.Error())
		}
		credentialStore := auth.NewCredentialsStore()
		if err := credentialStore.Load(f); err != nil {
			log.Fatalf("failed to load authentication file: %s", err.Error())
		}
		s = httpd.New(httpAddr, str, credentialStore)
	} else {
		s = httpd.New(httpAddr, str, nil)
	}

	s.CertFile = x509Cert
	s.KeyFile = x509Key
	s.Expvar = expvar
	s.Pprof = pprofEnabled
	s.BuildInfo = map[string]interface{}{
		"commit":     commit,
		"branch":     branch,
		"version":    version,
		"build_time": buildtime,
	}
	if err := s.Start(); err != nil {
		log.Fatalf("failed to start HTTP server: %s", err.Error())
	}

	terminate := make(chan os.Signal, 1)
	signal.Notify(terminate, os.Interrupt)
	<-terminate
	if err := str.Close(true); err != nil {
		log.Printf("failed to close store: %s", err.Error())
	}
	stopProfile()
	log.Println("rqlite server stopped")
}

func determineJoinAddresses() ([]string, error) {
	apiAdv := httpAddr
	if httpAdv != "" {
		apiAdv = httpAdv
	}

	var addrs []string
	if joinAddr != "" {
		// An explicit join address is first priority.
		addrs = append(addrs, joinAddr)
	}

	if discoID != "" {
		log.Printf("registering with Discovery Service at %s with ID %s", discoURL, discoID)
		c := disco.New(discoURL)
		r, err := c.Register(discoID, apiAdv)
		if err != nil {
			return nil, err
		}
		log.Println("Discovery Service responded with nodes:", r.Nodes)
		for _, a := range r.Nodes {
			if a != apiAdv {
				// Only other nodes can be joined.
				addrs = append(addrs, a)
			}
		}
	}

	return addrs, nil
}

func publishAPIAddr(c *cluster.Service, raftAddr, apiAddr string, t time.Duration) error {
	tck := time.NewTicker(publishPeerDelay)
	defer tck.Stop()
	tmr := time.NewTimer(t)
	defer tmr.Stop()

	for {
		select {
		case <-tck.C:
			if err := c.SetPeer(raftAddr, apiAddr); err != nil {
				log.Printf("failed to set peer for %s to %s: %s (retrying)",
					raftAddr, apiAddr, err.Error())
				continue
			}
			return nil
		case <-tmr.C:
			return fmt.Errorf("set peer timeout expired")
		}
	}
}

// prof stores the file locations of active profiles.
var prof struct {
	cpu *os.File
	mem *os.File
}

// startProfile initializes the CPU and memory profile, if specified.
func startProfile(cpuprofile, memprofile string) {
	if cpuprofile != "" {
		f, err := os.Create(cpuprofile)
		if err != nil {
			log.Fatalf("failed to create CPU profile file at %s: %s", cpuprofile, err.Error())
		}
		log.Printf("writing CPU profile to: %s\n", cpuprofile)
		prof.cpu = f
		pprof.StartCPUProfile(prof.cpu)
	}

	if memprofile != "" {
		f, err := os.Create(memprofile)
		if err != nil {
			log.Fatalf("failed to create memory profile file at %s: %s", cpuprofile, err.Error())
		}
		log.Printf("writing memory profile to: %s\n", memprofile)
		prof.mem = f
		runtime.MemProfileRate = 4096
	}
}

// stopProfile closes the CPU and memory profiles if they are running.
func stopProfile() {
	if prof.cpu != nil {
		pprof.StopCPUProfile()
		prof.cpu.Close()
		log.Println("CPU profiling stopped")
	}
	if prof.mem != nil {
		pprof.Lookup("heap").WriteTo(prof.mem, 0)
		prof.mem.Close()
		log.Println("memory profiling stopped")
	}
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"os"

	"fmt"
	"io/ioutil"
	"strings"

	"github.com/j-delaney/typewriter"
)

func main() {
	padding := flag.Int("padding", 5, "The padding between the two columns")
	separator := flag.String("separator", "", "Character to separate the two columns")
	markDifference := flag.Bool("diff", false, "Mark the first difference found")
	lineNumbers := flag.Bool("linenums", false, "Show line numbers")
	showHeader := flag.Bool("header", false, "Use the filenames as headers")

	flag.Parse()

	if flag.NArg() != 2 {
		flag.Usage()
		os.Exit(0)
	}

	filePath1 := flag.Arg(0)
	filePath2 := flag.Arg(1)

	bytes1, err := ioutil.ReadFile(filePath1)
	if err != nil {
		fmt.Fprint(os.Stderr, "Could not read %v: %v", filePath1, err)
		os.Exit(1)
	}

	bytes2, err := ioutil.ReadFile(filePath2)
	if err != nil {
		fmt.Fprint(os.Stderr, "Could not read %v: %v", filePath2, err)
		os.Exit(1)
	}

	lines1 := strings.Split(string(bytes1), "\n")
	lines2 := strings.Split(string(bytes2), "\n")

	config := typewriter.Config{
		Padding:   *padding,
		Separator: *separator,

		MarkFirstDifference: *markDifference,
		ShowLineNumbers:     *lineNumbers,
	}

	if *showHeader {
		config.LeftHeader = filePath1
		config.RightHeader = filePath2
	}

	s := typewriter.Run(lines1, lines2, config)

	fmt.Print(s)
}
<commit_msg>Better cli<commit_after>package main

import (
	"flag"
	"os"

	"fmt"
	"io/ioutil"
	"strings"

	"github.com/j-delaney/typewriter"
)

func main() {
	flag.Usage = func() {
		fmt.Printf("Usage: side_by_side [options] leftFile rightFile\nOptions:\n")
		flag.PrintDefaults()
	}

	padding := flag.Int("padding", 5, "The padding between the two columns")
	separator := flag.String("separator", "", "Character to separate the two columns")
	markDifference := flag.Bool("diff", false, "Mark the first difference found")
	lineNumbers := flag.Bool("linenums", false, "Show line numbers")
	showHeader := flag.Bool("header", false, "Use the filenames as headers")

	flag.Parse()

	if flag.NArg() != 2 {
		flag.Usage()
		os.Exit(2)
	}

	filePath1 := flag.Arg(0)
	filePath2 := flag.Arg(1)

	bytes1, err := ioutil.ReadFile(filePath1)
	if err != nil {
		fmt.Fprint(os.Stderr, "Could not read %v: %v", filePath1, err)
		os.Exit(1)
	}

	bytes2, err := ioutil.ReadFile(filePath2)
	if err != nil {
		fmt.Fprint(os.Stderr, "Could not read %v: %v", filePath2, err)
		os.Exit(1)
	}

	lines1 := strings.Split(string(bytes1), "\n")
	lines2 := strings.Split(string(bytes2), "\n")

	config := typewriter.Config{
		Padding:   *padding,
		Separator: *separator,

		MarkFirstDifference: *markDifference,
		ShowLineNumbers:     *lineNumbers,
	}

	if *showHeader {
		config.LeftHeader = filePath1
		config.RightHeader = filePath2
	}

	s := typewriter.Run(lines1, lines2, config)

	fmt.Print(s)
}
<|endoftext|>


<commit_before>/*
 * Minio Cloud Storage, (C) 2018 Minio, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package cmd

import (
	"net/http"

	"github.com/gorilla/mux"
	"github.com/minio/minio/cmd/logger"
	"github.com/minio/minio/pkg/auth"
	"github.com/minio/minio/pkg/iam/validator"
)

const (
	// STS API version.
	stsAPIVersion = "2011-06-15"
)

// stsAPIHandlers implements and provides http handlers for AWS STS API.
type stsAPIHandlers struct{}

// registerSTSRouter - registers AWS STS compatible APIs.
func registerSTSRouter(router *mux.Router) {
	// Initialize STS.
	sts := &stsAPIHandlers{}

	// STS Router
	stsRouter := router.NewRoute().PathPrefix("/").Subrouter()

	// AssumeRoleWithClientGrants
	stsRouter.Methods("POST").HandlerFunc(httpTraceAll(sts.AssumeRoleWithClientGrants)).
		Queries("Action", "AssumeRoleWithClientGrants").
		Queries("Version", stsAPIVersion).
		Queries("Token", "{Token:.*}")

	// AssumeRoleWithWebIdentity
	stsRouter.Methods("POST").HandlerFunc(httpTraceAll(sts.AssumeRoleWithWebIdentity)).
		Queries("Action", "AssumeRoleWithWebIdentity").
		Queries("Version", stsAPIVersion).
		Queries("WebIdentityToken", "{Token:.*}")

}

// AssumeRoleWithWebIdentity - implementation of AWS STS API supporting OAuth2.0
// users from web identity provider such as Facebook, Google, or any OpenID
// Connect-compatible identity provider.
//
// Eg:-
//    $ curl https://minio:9000/?Action=AssumeRoleWithWebIdentity&WebIdentityToken=<jwt>
func (sts *stsAPIHandlers) AssumeRoleWithWebIdentity(w http.ResponseWriter, r *http.Request) {
	ctx := newContext(r, w, "AssumeRoleWithWebIdentity")

	defer logger.AuditLog(w, r, "AssumeRoleWithWebIdentity", nil)

	if globalIAMValidators == nil {
		writeSTSErrorResponse(w, ErrSTSNotInitialized)
		return
	}

	// NOTE: this API only accepts JWT tokens.
	v, err := globalIAMValidators.Get("jwt")
	if err != nil {
		writeSTSErrorResponse(w, ErrSTSInvalidParameterValue)
		return
	}

	vars := mux.Vars(r)
	m, err := v.Validate(vars["Token"], r.URL.Query().Get("DurationSeconds"))
	if err != nil {
		switch err {
		case validator.ErrTokenExpired:
			writeSTSErrorResponse(w, ErrSTSWebIdentityExpiredToken)
		case validator.ErrInvalidDuration:
			writeSTSErrorResponse(w, ErrSTSInvalidParameterValue)
		default:
			logger.LogIf(ctx, err)
			writeSTSErrorResponse(w, ErrSTSInvalidParameterValue)
		}
		return
	}

	secret := globalServerConfig.GetCredential().SecretKey
	cred, err := auth.GetNewCredentialsWithMetadata(m, secret)
	if err != nil {
		logger.LogIf(ctx, err)
		writeSTSErrorResponse(w, ErrSTSInternalError)
		return
	}

	// JWT has requested a custom claim with policy value set.
	// This is a Minio STS API specific value, this value should
	// be set and configured on your identity provider as part of
	// JWT custom claims.
	var policyName string
	if v, ok := m["policy"]; ok {
		policyName, _ = v.(string)
	}

	var subFromToken string
	if v, ok := m["sub"]; ok {
		subFromToken, _ = v.(string)
	}

	// Set the newly generated credentials.
	if err = globalIAMSys.SetTempUser(cred.AccessKey, cred, policyName); err != nil {
		logger.LogIf(ctx, err)
		writeSTSErrorResponse(w, ErrSTSInternalError)
		return
	}

	// Notify all other Minio peers to reload temp users
	for _, nerr := range globalNotificationSys.LoadUsers() {
		if nerr.Err != nil {
			logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String())
			logger.LogIf(ctx, nerr.Err)
		}
	}

	encodedSuccessResponse := encodeResponse(&AssumeRoleWithWebIdentityResponse{
		Result: WebIdentityResult{
			Credentials:                 cred,
			SubjectFromWebIdentityToken: subFromToken,
		},
	})

	writeSuccessResponseXML(w, encodedSuccessResponse)
}

// AssumeRoleWithClientGrants - implementation of AWS STS extension API supporting
// OAuth2.0 client credential grants.
//
// Eg:-
//    $ curl https://minio:9000/?Action=AssumeRoleWithClientGrants&Token=<jwt>
func (sts *stsAPIHandlers) AssumeRoleWithClientGrants(w http.ResponseWriter, r *http.Request) {
	ctx := newContext(r, w, "AssumeRoleWithClientGrants")

	defer logger.AuditLog(w, r, "AssumeRoleWithClientGrants", nil)

	if globalIAMValidators == nil {
		writeSTSErrorResponse(w, ErrSTSNotInitialized)
		return
	}

	// NOTE: this API only accepts JWT tokens.
	v, err := globalIAMValidators.Get("jwt")
	if err != nil {
		writeSTSErrorResponse(w, ErrSTSInvalidParameterValue)
		return
	}

	vars := mux.Vars(r)
	m, err := v.Validate(vars["Token"], r.URL.Query().Get("DurationSeconds"))
	if err != nil {
		switch err {
		case validator.ErrTokenExpired:
			writeSTSErrorResponse(w, ErrSTSClientGrantsExpiredToken)
		case validator.ErrInvalidDuration:
			writeSTSErrorResponse(w, ErrSTSInvalidParameterValue)
		default:
			logger.LogIf(ctx, err)
			writeSTSErrorResponse(w, ErrSTSInvalidParameterValue)
		}
		return
	}

	secret := globalServerConfig.GetCredential().SecretKey
	cred, err := auth.GetNewCredentialsWithMetadata(m, secret)
	if err != nil {
		logger.LogIf(ctx, err)
		writeSTSErrorResponse(w, ErrSTSInternalError)
		return
	}

	// JWT has requested a custom claim with policy value set.
	// This is a Minio STS API specific value, this value should
	// be set and configured on your identity provider as part of
	// JWT custom claims.
	var policyName string
	if v, ok := m["policy"]; ok {
		policyName, _ = v.(string)
	}

	var subFromToken string
	if v, ok := m["sub"]; ok {
		subFromToken, _ = v.(string)
	}

	// Set the newly generated credentials.
	if err = globalIAMSys.SetTempUser(cred.AccessKey, cred, policyName); err != nil {
		logger.LogIf(ctx, err)
		writeSTSErrorResponse(w, ErrSTSInternalError)
		return
	}

	// Notify all other Minio peers to reload temp users
	for _, nerr := range globalNotificationSys.LoadUsers() {
		if nerr.Err != nil {
			logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String())
			logger.LogIf(ctx, nerr.Err)
		}
	}

	encodedSuccessResponse := encodeResponse(&AssumeRoleWithClientGrantsResponse{
		Result: ClientGrantsResult{
			Credentials:      cred,
			SubjectFromToken: subFromToken,
		},
	})

	writeSuccessResponseXML(w, encodedSuccessResponse)
}
<commit_msg>Added support for reading body in STS API (#7188)<commit_after>/*
 * Minio Cloud Storage, (C) 2018 Minio, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package cmd

import (
	"fmt"
	"net/http"

	"github.com/gorilla/mux"
	"github.com/minio/minio/cmd/logger"
	"github.com/minio/minio/pkg/auth"
	"github.com/minio/minio/pkg/iam/validator"
)

const (
	// STS API version.
	stsAPIVersion = "2011-06-15"

	// STS API action constants
	clientGrants = "AssumeRoleWithClientGrants"
	webIdentity  = "AssumeRoleWithWebIdentity"
)

// stsAPIHandlers implements and provides http handlers for AWS STS API.
type stsAPIHandlers struct{}

// registerSTSRouter - registers AWS STS compatible APIs.
func registerSTSRouter(router *mux.Router) {
	// Initialize STS.
	sts := &stsAPIHandlers{}

	// STS Router
	stsRouter := router.NewRoute().PathPrefix("/").Subrouter()

	// Assume roles with JWT handler, handles both ClientGrants and WebIdentity.
	stsRouter.Methods("POST").HeadersRegexp("Content-Type", "application/x-www-form-urlencoded*").
		HandlerFunc(httpTraceAll(sts.AssumeRoleWithJWT))

	// AssumeRoleWithClientGrants
	stsRouter.Methods("POST").HandlerFunc(httpTraceAll(sts.AssumeRoleWithClientGrants)).
		Queries("Action", clientGrants).
		Queries("Version", stsAPIVersion).
		Queries("Token", "{Token:.*}")

	// AssumeRoleWithWebIdentity
	stsRouter.Methods("POST").HandlerFunc(httpTraceAll(sts.AssumeRoleWithWebIdentity)).
		Queries("Action", webIdentity).
		Queries("Version", stsAPIVersion).
		Queries("WebIdentityToken", "{Token:.*}")

}

func (sts *stsAPIHandlers) AssumeRoleWithJWT(w http.ResponseWriter, r *http.Request) {
	ctx := newContext(r, w, "AssumeRoleInternalFunction")

	// Parse the incoming form data.
	if err := r.ParseForm(); err != nil {
		logger.LogIf(ctx, err)
		writeSTSErrorResponse(w, ErrSTSInvalidParameterValue)
		return
	}

	if r.Form.Get("Version") != stsAPIVersion {
		logger.LogIf(ctx, fmt.Errorf("Invalid STS API version %s, expecting %s", r.Form.Get("Version"), stsAPIVersion))
		writeSTSErrorResponse(w, ErrSTSMissingParameter)
		return
	}

	action := r.Form.Get("Action")
	switch action {
	case clientGrants, webIdentity:
	default:
		logger.LogIf(ctx, fmt.Errorf("Unsupported action %s", action))
		writeSTSErrorResponse(w, ErrSTSInvalidParameterValue)
		return
	}

	ctx = newContext(r, w, action)
	defer logger.AuditLog(w, r, action, nil)

	if globalIAMValidators == nil {
		writeSTSErrorResponse(w, ErrSTSNotInitialized)
		return
	}

	v, err := globalIAMValidators.Get("jwt")
	if err != nil {
		logger.LogIf(ctx, err)
		writeSTSErrorResponse(w, ErrSTSInvalidParameterValue)
		return
	}

	token := r.Form.Get("Token")
	if token == "" {
		token = r.Form.Get("WebIdentityToken")
	}

	m, err := v.Validate(token, r.Form.Get("DurationSeconds"))
	if err != nil {
		switch err {
		case validator.ErrTokenExpired:
			switch action {
			case clientGrants:
				writeSTSErrorResponse(w, ErrSTSClientGrantsExpiredToken)
			case webIdentity:
				writeSTSErrorResponse(w, ErrSTSWebIdentityExpiredToken)
			}
			return
		case validator.ErrInvalidDuration:
			writeSTSErrorResponse(w, ErrSTSInvalidParameterValue)
			return
		}
		logger.LogIf(ctx, err)
		writeSTSErrorResponse(w, ErrSTSInvalidParameterValue)
		return
	}

	secret := globalServerConfig.GetCredential().SecretKey
	cred, err := auth.GetNewCredentialsWithMetadata(m, secret)
	if err != nil {
		logger.LogIf(ctx, err)
		writeSTSErrorResponse(w, ErrSTSInternalError)
		return
	}

	// JWT has requested a custom claim with policy value set.
	// This is a Minio STS API specific value, this value should
	// be set and configured on your identity provider as part of
	// JWT custom claims.
	var policyName string
	if v, ok := m["policy"]; ok {
		policyName, _ = v.(string)
	}

	var subFromToken string
	if v, ok := m["sub"]; ok {
		subFromToken, _ = v.(string)
	}

	// Set the newly generated credentials.
	if err = globalIAMSys.SetTempUser(cred.AccessKey, cred, policyName); err != nil {
		logger.LogIf(ctx, err)
		writeSTSErrorResponse(w, ErrSTSInternalError)
		return
	}

	// Notify all other Minio peers to reload temp users
	for _, nerr := range globalNotificationSys.LoadUsers() {
		if nerr.Err != nil {
			logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String())
			logger.LogIf(ctx, nerr.Err)
		}
	}

	var encodedSuccessResponse []byte
	switch action {
	case clientGrants:
		encodedSuccessResponse = encodeResponse(&AssumeRoleWithClientGrantsResponse{
			Result: ClientGrantsResult{
				Credentials:      cred,
				SubjectFromToken: subFromToken,
			},
		})
	case webIdentity:
		encodedSuccessResponse = encodeResponse(&AssumeRoleWithWebIdentityResponse{
			Result: WebIdentityResult{
				Credentials:                 cred,
				SubjectFromWebIdentityToken: subFromToken,
			},
		})
	}

	writeSuccessResponseXML(w, encodedSuccessResponse)
}

// AssumeRoleWithWebIdentity - implementation of AWS STS API supporting OAuth2.0
// users from web identity provider such as Facebook, Google, or any OpenID
// Connect-compatible identity provider.
//
// Eg:-
//    $ curl https://minio:9000/?Action=AssumeRoleWithWebIdentity&WebIdentityToken=<jwt>
func (sts *stsAPIHandlers) AssumeRoleWithWebIdentity(w http.ResponseWriter, r *http.Request) {
	sts.AssumeRoleWithJWT(w, r)
}

// AssumeRoleWithClientGrants - implementation of AWS STS extension API supporting
// OAuth2.0 client credential grants.
//
// Eg:-
//    $ curl https://minio:9000/?Action=AssumeRoleWithClientGrants&Token=<jwt>
func (sts *stsAPIHandlers) AssumeRoleWithClientGrants(w http.ResponseWriter, r *http.Request) {
	sts.AssumeRoleWithJWT(w, r)
}
<|endoftext|>