<commit_before>// Copyright 2018 Adam Tauber
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package colly

import (
	"strings"

	"github.com/PuerkitoBio/goquery"
	"golang.org/x/net/html"
)

// HTMLElement is the representation of a HTML tag.
type HTMLElement struct {
	// Name is the name of the tag
	Name       string
	Text       string
	attributes []html.Attribute
	// Request is the request object of the element's HTML document
	Request *Request
	// Response is the Response object of the element's HTML document
	Response *Response
	// DOM is the goquery parsed DOM object of the page. DOM is relative
	// to the current HTMLElement
	DOM *goquery.Selection
}

// NewHTMLElementFromSelectionNode creates a HTMLElement from a goquery.Selection Node.
func NewHTMLElementFromSelectionNode(resp *Response, s *goquery.Selection, n *html.Node) *HTMLElement {
	return &HTMLElement{
		Name:       n.Data,
		Request:    resp.Request,
		Response:   resp,
		Text:       goquery.NewDocumentFromNode(n).Text(),
		DOM:        s,
		attributes: n.Attr,
	}
}

// Attr returns the selected attribute of a HTMLElement or empty string
// if no attribute found
func (h *HTMLElement) Attr(k string) string {
	for _, a := range h.attributes {
		if a.Key == k {
			return a.Val
		}
	}
	return ""
}

// ChildText returns the concatenated and stripped text content of the matching
// elements.
func (h *HTMLElement) ChildText(goquerySelector string) string {
	return strings.TrimSpace(h.DOM.Find(goquerySelector).Text())
}

// ChildAttr returns the stripped text content of the first matching
// element's attribute.
func (h *HTMLElement) ChildAttr(goquerySelector, attrName string) string {
	if attr, ok := h.DOM.Find(goquerySelector).Attr(attrName); ok {
		return strings.TrimSpace(attr)
	}
	return ""
}

// ChildAttrs returns the stripped text content of all the matching
// element's attributes.
func (h *HTMLElement) ChildAttrs(goquerySelector, attrName string) []string {
	var res []string
	h.DOM.Find(goquerySelector).Each(func(_ int, s *goquery.Selection) {
		if attr, ok := s.Attr(attrName); ok {
			res = append(res, strings.TrimSpace(attr))
		}
	})
	return res
}

// ForEach iterates over the elements matched by the first argument
// and calls the callback function on every HTMLElement match.
func (h *HTMLElement) ForEach(goquerySelector string, callback func(int, *HTMLElement)) {
	i := 0
	h.DOM.Find(goquerySelector).Each(func(_ int, s *goquery.Selection) {
		for _, n := range s.Nodes {
			callback(i, NewHTMLElementFromSelectionNode(h.Response, s, n))
			i++
		}
	})
}

// ForEach iterates over the elements matched by the first argument
// and calls the callback function on every HTMLElement match.
// It is identical to ForEach except that it is possible to break
// out of the loop by returning false in the callback function. It returns the
// current Selection object.
func (h *HTMLElement) ForEachWithBreak(goquerySelector string, callback func(int, *HTMLElement) bool) {
	i := 0
	h.DOM.Find(goquerySelector).EachWithBreak(func(_ int, s *goquery.Selection) bool {
		for _, n := range s.Nodes {
			if callback(i, NewHTMLElementFromSelectionNode(h.Response, s, n)) {
				return true
			}
			i++
		}
		return false
	})
}
<commit_msg>fix golint comment format<commit_after>// Copyright 2018 Adam Tauber
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package colly

import (
	"strings"

	"github.com/PuerkitoBio/goquery"
	"golang.org/x/net/html"
)

// HTMLElement is the representation of a HTML tag.
type HTMLElement struct {
	// Name is the name of the tag
	Name       string
	Text       string
	attributes []html.Attribute
	// Request is the request object of the element's HTML document
	Request *Request
	// Response is the Response object of the element's HTML document
	Response *Response
	// DOM is the goquery parsed DOM object of the page. DOM is relative
	// to the current HTMLElement
	DOM *goquery.Selection
}

// NewHTMLElementFromSelectionNode creates a HTMLElement from a goquery.Selection Node.
func NewHTMLElementFromSelectionNode(resp *Response, s *goquery.Selection, n *html.Node) *HTMLElement {
	return &HTMLElement{
		Name:       n.Data,
		Request:    resp.Request,
		Response:   resp,
		Text:       goquery.NewDocumentFromNode(n).Text(),
		DOM:        s,
		attributes: n.Attr,
	}
}

// Attr returns the selected attribute of a HTMLElement or empty string
// if no attribute found
func (h *HTMLElement) Attr(k string) string {
	for _, a := range h.attributes {
		if a.Key == k {
			return a.Val
		}
	}
	return ""
}

// ChildText returns the concatenated and stripped text content of the matching
// elements.
func (h *HTMLElement) ChildText(goquerySelector string) string {
	return strings.TrimSpace(h.DOM.Find(goquerySelector).Text())
}

// ChildAttr returns the stripped text content of the first matching
// element's attribute.
func (h *HTMLElement) ChildAttr(goquerySelector, attrName string) string {
	if attr, ok := h.DOM.Find(goquerySelector).Attr(attrName); ok {
		return strings.TrimSpace(attr)
	}
	return ""
}

// ChildAttrs returns the stripped text content of all the matching
// element's attributes.
func (h *HTMLElement) ChildAttrs(goquerySelector, attrName string) []string {
	var res []string
	h.DOM.Find(goquerySelector).Each(func(_ int, s *goquery.Selection) {
		if attr, ok := s.Attr(attrName); ok {
			res = append(res, strings.TrimSpace(attr))
		}
	})
	return res
}

// ForEach iterates over the elements matched by the first argument
// and calls the callback function on every HTMLElement match.
func (h *HTMLElement) ForEach(goquerySelector string, callback func(int, *HTMLElement)) {
	i := 0
	h.DOM.Find(goquerySelector).Each(func(_ int, s *goquery.Selection) {
		for _, n := range s.Nodes {
			callback(i, NewHTMLElementFromSelectionNode(h.Response, s, n))
			i++
		}
	})
}

// ForEachWithBreak iterates over the elements matched by the first argument
// and calls the callback function on every HTMLElement match.
// It is identical to ForEach except that it is possible to break
// out of the loop by returning false in the callback function. It returns the
// current Selection object.
func (h *HTMLElement) ForEachWithBreak(goquerySelector string, callback func(int, *HTMLElement) bool) {
	i := 0
	h.DOM.Find(goquerySelector).EachWithBreak(func(_ int, s *goquery.Selection) bool {
		for _, n := range s.Nodes {
			if callback(i, NewHTMLElementFromSelectionNode(h.Response, s, n)) {
				return true
			}
			i++
		}
		return false
	})
}
<|endoftext|>


<commit_before>package models

import (
	"socialapi/workers/common/runner"
	"testing"

	"github.com/koding/bongo"
	. "github.com/smartystreets/goconvey/convey"
)

func TestAccountNewAccount(t *testing.T) {
	Convey("while testing new account", t, func() {
		Convey("Function call should return account", func() {
			So(NewAccount(), ShouldNotBeNil)
		})
	})
}

func TestAccountGetId(t *testing.T) {
	Convey("while testing get id", t, func() {
		Convey("Initialized struct ", func() {
			Convey("should return given id", func() {
				a := Account{Id: 42}
				So(a.GetId(), ShouldEqual, 42)
			})

			Convey("Uninitialized struct ", func() {
				Convey("should return 0", func() {
					So(NewAccount().GetId(), ShouldEqual, 0)
				})
				So(NewAccount(), ShouldNotBeNil)
			})
		})
	})
}

func TestAccountFetchOrCreate(t *testing.T) {
	r := runner.New("test")
	if err := r.Init(); err != nil {
		t.Fatalf("couldnt start bongo %s", err.Error())
	}
	defer r.Close()

	Convey("while fetching or creating account", t, func() {
		Convey("it should have old id", func() {
			a := NewAccount()
			err := a.FetchOrCreate()
			So(err, ShouldNotBeNil)
			So(err, ShouldEqual, ErrOldIdIsNotSet)
		})

		Convey("it should have error if nick contains guest-", func() {
			a := NewAccount()
			a.OldId = "oldestId"
			a.Nick = "guest-test"
			err := a.FetchOrCreate()
			So(err, ShouldNotBeNil)
			So(err, ShouldEqual, ErrGuestsAreNotAllowed)
		})

		Convey("it should not have error if required fields are exist", func() {
			a := NewAccount()
			a.OldId = "oldIdOfAccount"
			a.Nick = "WhatANick"
			err := a.FetchOrCreate()
			So(err, ShouldBeNil)
		})

	})
}

func TestAccountFetchAccountById(t *testing.T) {
	r := runner.New("test")
	if err := r.Init(); err != nil {
		t.Fatalf("couldnt start bongo %s", err.Error())
	}
	defer r.Close()

	Convey("while fetching an account by id", t, func() {
		Convey("it should not have error while fething", func() {
			// create account
			acc := createAccountWithTest()
			So(acc.Create(), ShouldBeNil)

			// fetch the account by id of account
			fa, err := FetchAccountById(acc.Id)
			// error should be nil
			// means that fetching is done successfully
			So(err, ShouldBeNil)
			// account in the db should equal to fetched account
			So(fa.Id, ShouldEqual, acc.Id)
			So(fa.OldId, ShouldEqual, acc.OldId)
			So(fa.Nick, ShouldEqual, acc.Nick)
		})

		Convey("it should have error if record is not found", func() {
			// init account
			a := NewAccount()
			a.Id = 12345
			a.OldId = "oldIdOfAccount"
			a.Nick = "WhatANick"

			// this account id is not exist
			_, err := FetchAccountById(a.Id)
			So(err, ShouldNotBeNil)
			So(err, ShouldEqual, bongo.RecordNotFound)

		})

	})
}

func TestAccountFetchOldIdsByAccountIds(t *testing.T) {
	r := runner.New("test")
	if err := r.Init(); err != nil {
		t.Fatalf("couldnt start bongo %s", err.Error())
	}
	defer r.Close()

	Convey("while fetching Old Ids by account ids", t, func() {
		Convey("it should have account id (ids of length) more than zero", func() {
			acc := []int64{}
			foi, err := FetchOldIdsByAccountIds(acc)
			So(err, ShouldBeNil)
			So(foi, ShouldBeEmpty)
		})

		Convey("it should not have error if account is exist in db", func() {
			// create account
			acc := createAccountWithTest()
			So(acc.Create(), ShouldBeNil)
			// we created slice to send to FetchOldIdsByAccountIds as argument
			idd := []int64{acc.Id}

			// FetchOldIdsByAccountIds returns as slice
			foi, err := FetchOldIdsByAccountIds(idd)
			So(err, ShouldBeNil)
			// used shouldcontain because foi is a slice
			So(foi, ShouldContain, acc.OldId)

		})

		Convey("it should append successfully", func() {
			acc1 := NewAccount()
			acc1.Id = 1
			acc1.OldId = "11"
			acc1.Nick = "acc1"
			So(acc1.Create(), ShouldBeNil)

			acc2 := NewAccount()
			acc2.Id = 2
			acc2.OldId = "22"
			acc2.Nick = "acc2"
			So(acc2.Create(), ShouldBeNil)

			idd := []int64{acc1.Id, acc2.Id}
			old := []string{acc1.OldId, acc2.OldId}

			foi, err := FetchOldIdsByAccountIds(idd)
			So(err, ShouldBeNil)
			So(foi[1], ShouldEqual, old[1])

		})

	})
}

func TestAccountCreateFollowingFeedChannel(t *testing.T) {
	r := runner.New("test")
	if err := r.Init(); err != nil {
		t.Fatalf("couldnt start bongo %s", err.Error())
	}
	defer r.Close()

	Convey("while creating feed channel", t, func() {
		Convey("it should have account id", func() {
			// create account
			acc := NewAccount()

			_, err := acc.CreateFollowingFeedChannel()
			So(err, ShouldNotBeNil)
			So(err, ShouldEqual, ErrAccountIdIsNotSet)
		})

		Convey("it should have creator id as account id", func() {
			// create account
			acc := NewAccount()
			acc.Id = 1111

			cff, err := acc.CreateFollowingFeedChannel()
			So(err, ShouldBeNil)
			So(cff.CreatorId, ShouldEqual, acc.Id)
		})

		Convey("it should have channel name as required", func() {
			// create account
			acc := NewAccount()
			acc.Id = 1111

			cff, err := acc.CreateFollowingFeedChannel()
			So(err, ShouldBeNil)
			So(cff.Name, ShouldEqual, "1111-FollowingFeedChannel")
		})

		Convey("it should have group name as Channel_KODING_NAME", func() {
			// create account
			acc := NewAccount()
			acc.Id = 1111

			cff, err := acc.CreateFollowingFeedChannel()
			So(err, ShouldBeNil)
			So(cff.GroupName, ShouldEqual, Channel_KODING_NAME)
		})

		Convey("it should have purpose as Following Feed for me", func() {
			// create account
			acc := NewAccount()
			acc.Id = 1111

			cff, err := acc.CreateFollowingFeedChannel()
			So(err, ShouldBeNil)
			So(cff.Purpose, ShouldEqual, "Following Feed for Me")
		})

		Convey("it should have type constant as Channel_TYPE_FOLLOWERS", func() {
			// create account
			acc := NewAccount()
			acc.Id = 1111

			cff, err := acc.CreateFollowingFeedChannel()
			So(err, ShouldBeNil)
			So(cff.TypeConstant, ShouldEqual, Channel_TYPE_FOLLOWERS)
		})

	})
}
<commit_msg>social/tests: test errors are fixed<commit_after>package models

import (
	"socialapi/workers/common/runner"
	"testing"

	"github.com/koding/bongo"
	. "github.com/smartystreets/goconvey/convey"
)

func TestAccountNewAccount(t *testing.T) {
	Convey("while testing new account", t, func() {
		Convey("Function call should return account", func() {
			So(NewAccount(), ShouldNotBeNil)
		})
	})
}

func TestAccountGetId(t *testing.T) {
	Convey("while testing get id", t, func() {
		Convey("Initialized struct ", func() {
			Convey("should return given id", func() {
				a := Account{Id: 42}
				So(a.GetId(), ShouldEqual, 42)
			})

			Convey("Uninitialized struct ", func() {
				Convey("should return 0", func() {
					So(NewAccount().GetId(), ShouldEqual, 0)
				})
				So(NewAccount(), ShouldNotBeNil)
			})
		})
	})
}

func TestAccountFetchOrCreate(t *testing.T) {
	r := runner.New("test")
	if err := r.Init(); err != nil {
		t.Fatalf("couldnt start bongo %s", err.Error())
	}
	defer r.Close()

	Convey("while fetching or creating account", t, func() {
		Convey("it should have old id", func() {
			a := NewAccount()
			err := a.FetchOrCreate()
			So(err, ShouldNotBeNil)
			So(err, ShouldEqual, ErrOldIdIsNotSet)
		})

		Convey("it should have error if nick contains guest-", func() {
			a := NewAccount()
			a.OldId = "oldestId"
			a.Nick = "guest-test"
			err := a.FetchOrCreate()
			So(err, ShouldNotBeNil)
			So(err, ShouldEqual, ErrGuestsAreNotAllowed)
		})

		Convey("it should not have error if required fields are exist", func() {
			a := NewAccount()
			a.OldId = "oldIdOfAccount"
			a.Nick = "WhatANick"
			err := a.FetchOrCreate()
			So(err, ShouldBeNil)
		})

	})
}

func TestAccountFetchAccountById(t *testing.T) {
	r := runner.New("test")
	if err := r.Init(); err != nil {
		t.Fatalf("couldnt start bongo %s", err.Error())
	}
	defer r.Close()

	Convey("while fetching an account by id", t, func() {
		Convey("it should not have error while fething", func() {
			// create account
			acc := createAccountWithTest()
			So(acc.Create(), ShouldBeNil)

			// fetch the account by id of account
			fa, err := FetchAccountById(acc.Id)
			// error should be nil
			// means that fetching is done successfully
			So(err, ShouldBeNil)
			// account in the db should equal to fetched account
			So(fa.Id, ShouldEqual, acc.Id)
			So(fa.OldId, ShouldEqual, acc.OldId)
			So(fa.Nick, ShouldEqual, acc.Nick)
		})

		Convey("it should have error if record is not found", func() {
			// init account
			a := NewAccount()
			a.Id = 12345
			a.OldId = "oldIdOfAccount"
			a.Nick = "WhatANick"

			// this account id is not exist
			_, err := FetchAccountById(a.Id)
			So(err, ShouldNotBeNil)
			So(err, ShouldEqual, bongo.RecordNotFound)

		})

	})
}

func TestAccountFetchOldIdsByAccountIds(t *testing.T) {
	r := runner.New("test")
	if err := r.Init(); err != nil {
		t.Fatalf("couldnt start bongo %s", err.Error())
	}
	defer r.Close()

	Convey("while fetching Old Ids by account ids", t, func() {
		Convey("it should have account id (ids of length) more than zero", func() {
			acc := []int64{}
			foi, err := FetchOldIdsByAccountIds(acc)
			So(err, ShouldBeNil)
			So(foi, ShouldBeEmpty)
		})

		Convey("it should not have error if account is exist in db", func() {
			// create account
			acc := createAccountWithTest()
			So(acc.Create(), ShouldBeNil)
			// we created slice to send to FetchOldIdsByAccountIds as argument
			idd := []int64{acc.Id}

			// FetchOldIdsByAccountIds returns as slice
			foi, err := FetchOldIdsByAccountIds(idd)
			So(err, ShouldBeNil)
			// used shouldcontain because foi is a slice
			So(foi, ShouldContain, acc.OldId)

		})

		Convey("it should append successfully", func() {
			acc1 := NewAccount()
			acc1.Id = 1
			acc1.OldId = "11"
			acc1.Nick = "acc1"
			So(acc1.Create(), ShouldBeNil)

			acc2 := NewAccount()
			acc2.Id = 2
			acc2.OldId = "22"
			acc2.Nick = "acc2"
			So(acc2.Create(), ShouldBeNil)

			idd := []int64{acc1.Id, acc2.Id}
			old := []string{acc1.OldId, acc2.OldId}

			foi, err := FetchOldIdsByAccountIds(idd)
			So(err, ShouldBeNil)
			So(foi[1], ShouldEqual, old[1])

		})

	})
}

func TestAccountCreateFollowingFeedChannel(t *testing.T) {
	r := runner.New("test")
	if err := r.Init(); err != nil {
		t.Fatalf("couldnt start bongo %s", err.Error())
	}
	defer r.Close()

	Convey("while creating feed channel", t, func() {
		Convey("it should have account id", func() {
			// create account
			acc := NewAccount()

			_, err := acc.CreateFollowingFeedChannel()
			So(err, ShouldNotBeNil)
			So(err, ShouldEqual, ErrAccountIdIsNotSet)
		})

		Convey("it should have creator id as account id", func() {
			// create account
			acc := createAccountWithTest()
			So(acc.Create(), ShouldBeNil)

			cff, err := acc.CreateFollowingFeedChannel()
			So(err, ShouldBeNil)
			So(cff.CreatorId, ShouldEqual, acc.Id)
		})

		Convey("it should have channel name as required", func() {
			// create account
			acc := NewAccount()
			acc.Id = 1
			acc.OldId = "11"
			acc.Nick = "acc1"
			So(acc.Create(), ShouldBeNil)

			cff, err := acc.CreateFollowingFeedChannel()
			So(err, ShouldBeNil)
			So(cff.Name, ShouldEqual, "1-FollowingFeedChannel")
		})

		Convey("it should have group name as Channel_KODING_NAME", func() {
			// create account
			acc := createAccountWithTest()
			So(acc.Create(), ShouldBeNil)

			cff, err := acc.CreateFollowingFeedChannel()
			So(err, ShouldBeNil)
			So(cff.GroupName, ShouldEqual, Channel_KODING_NAME)
		})

		Convey("it should have purpose as Following Feed for me", func() {
			// create account
			acc := createAccountWithTest()
			So(acc.Create(), ShouldBeNil)

			cff, err := acc.CreateFollowingFeedChannel()
			So(err, ShouldBeNil)
			So(cff.Purpose, ShouldEqual, "Following Feed for Me")
		})

		Convey("it should have type constant as Channel_TYPE_FOLLOWERS", func() {
			// create account
			acc := createAccountWithTest()
			So(acc.Create(), ShouldBeNil)

			cff, err := acc.CreateFollowingFeedChannel()
			So(err, ShouldBeNil)
			So(cff.TypeConstant, ShouldEqual, Channel_TYPE_FOLLOWERS)
		})

	})
}
<|endoftext|>


<commit_before>package commands

import (
	"fmt"
	"io"

	core "github.com/ipfs/go-ipfs/core"
	cmdenv "github.com/ipfs/go-ipfs/core/commands/cmdenv"
	filestore "github.com/ipfs/go-ipfs/filestore"

	cid "gx/ipfs/QmR8BauakNcBa3RbE4nbQu76PDiJgoQgz8AJdhJuiU4TAw/go-cid"
	cmds "gx/ipfs/Qma6uuSyjkecGhMFFLfzyJDPyoDtNJSHJNweDccZhaWkgU/go-ipfs-cmds"
	"gx/ipfs/Qmde5VP1qUkyQXKCfmEUA7bP64V2HAptbJ7phuPp7jXWwg/go-ipfs-cmdkit"
)

var FileStoreCmd = &cmds.Command{
	Helptext: cmdkit.HelpText{
		Tagline: "Interact with filestore objects.",
	},
	Subcommands: map[string]*cmds.Command{
		"ls":     lsFileStore,
		"verify": verifyFileStore,
		"dups":   dupsFileStore,
	},
}

const (
	fileOrderOptionName = "file-order"
)

var lsFileStore = &cmds.Command{
	Helptext: cmdkit.HelpText{
		Tagline: "List objects in filestore.",
		LongDescription: `
List objects in the filestore.

If one or more <obj> is specified only list those specific objects,
otherwise list all objects.

The output is:

<hash> <size> <path> <offset>
`,
	},
	Arguments: []cmdkit.Argument{
		cmdkit.StringArg("obj", false, true, "Cid of objects to list."),
	},
	Options: []cmdkit.Option{
		cmdkit.BoolOption(fileOrderOptionName, "sort the results based on the path of the backing file"),
	},
	Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error {
		_, fs, err := getFilestore(env)
		if err != nil {
			return err
		}
		args := req.Arguments
		if len(args) > 0 {
			return listByArgs(res, fs, args)
		}

		fileOrder, _ := req.Options[fileOrderOptionName].(bool)
		next, err := filestore.ListAll(fs, fileOrder)
		if err != nil {
			return err
		}

		for {
			r := next()
			if r == nil {
				break
			}
			if err := res.Emit(r); err != nil {
				return err
			}
		}

		return nil
	},
	PostRun: cmds.PostRunMap{
		cmds.CLI: streamResult(func(v interface{}, out io.Writer) nonFatalError {
			r := v.(*filestore.ListRes)
			if r.ErrorMsg != "" {
				return nonFatalError(r.ErrorMsg)
			}
			fmt.Fprintf(out, "%s\n", r.FormatLong())
			return ""
		}),
	},
	Type: filestore.ListRes{},
}

var verifyFileStore = &cmds.Command{
	Helptext: cmdkit.HelpText{
		Tagline: "Verify objects in filestore.",
		LongDescription: `
Verify objects in the filestore.

If one or more <obj> is specified only verify those specific objects,
otherwise verify all objects.

The output is:

<status> <hash> <size> <path> <offset>

Where <status> is one of:
ok:       the block can be reconstructed
changed:  the contents of the backing file have changed
no-file:  the backing file could not be found
error:    there was some other problem reading the file
missing:  <obj> could not be found in the filestore
ERROR:    internal error, most likely due to a corrupt database

For ERROR entries the error will also be printed to stderr.
`,
	},
	Arguments: []cmdkit.Argument{
		cmdkit.StringArg("obj", false, true, "Cid of objects to verify."),
	},
	Options: []cmdkit.Option{
		cmdkit.BoolOption(fileOrderOptionName, "verify the objects based on the order of the backing file"),
	},
	Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error {
		_, fs, err := getFilestore(env)
		if err != nil {
			return err
		}
		args := req.Arguments
		if len(args) > 0 {
			return listByArgs(res, fs, args)
		}

		fileOrder, _ := req.Options[fileOrderOptionName].(bool)
		next, err := filestore.VerifyAll(fs, fileOrder)
		if err != nil {
			return err
		}

		for {
			r := next()
			if r == nil {
				break
			}
			if err := res.Emit(r); err != nil {
				return err
			}
		}

		return nil
	},
	Encoders: cmds.EncoderMap{
		cmds.Text: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, out *filestore.ListRes) error {
			if out.Status == filestore.StatusOtherError {
				return fmt.Errorf(out.ErrorMsg)
			}
			fmt.Fprintf(w, "%s %s\n", out.Status.Format(), out.FormatLong())
			return nil
		}),
	},
	Type: filestore.ListRes{},
}

var dupsFileStore = &cmds.Command{
	Helptext: cmdkit.HelpText{
		Tagline: "List blocks that are both in the filestore and standard block storage.",
	},
	Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error {
		_, fs, err := getFilestore(env)
		if err != nil {
			return err
		}
		ch, err := fs.FileManager().AllKeysChan(req.Context)
		if err != nil {
			return err
		}

		for cid := range ch {
			have, err := fs.MainBlockstore().Has(cid)
			if err != nil {
				return res.Emit(&RefWrapper{Err: err.Error()})
			}
			if have {
				if err := res.Emit(&RefWrapper{Ref: cid.String()}); err != nil {
					return err
				}
			}
		}

		return nil
	},
	Encoders: cmds.EncoderMap{
		cmds.Text: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, out *RefWrapper) error {
			if out.Err != "" {
				return fmt.Errorf(out.Err)
			}

			fmt.Fprintln(w, out.Ref)

			return nil
		}),
	},
	Type: RefWrapper{},
}

func getFilestore(env cmds.Environment) (*core.IpfsNode, *filestore.Filestore, error) {
	n, err := cmdenv.GetNode(env)
	if err != nil {
		return nil, nil, err
	}
	fs := n.Filestore
	if fs == nil {
		return n, nil, filestore.ErrFilestoreNotEnabled
	}
	return n, fs, err
}

func listByArgs(res cmds.ResponseEmitter, fs *filestore.Filestore, args []string) error {
	for _, arg := range args {
		c, err := cid.Decode(arg)
		if err != nil {
			ret := &filestore.ListRes{
				Status:   filestore.StatusOtherError,
				ErrorMsg: fmt.Sprintf("%s: %v", arg, err),
			}
			if err := res.Emit(ret); err != nil {
				return err
			}
			continue
		}
		r := filestore.Verify(fs, c)
		if err := res.Emit(r); err != nil {
			return err
		}
	}

	return nil
}
<commit_msg>commands/filestore: print StatusOtherError error<commit_after>package commands

import (
	"fmt"
	"io"
	"os"

	core "github.com/ipfs/go-ipfs/core"
	cmdenv "github.com/ipfs/go-ipfs/core/commands/cmdenv"
	filestore "github.com/ipfs/go-ipfs/filestore"

	cid "gx/ipfs/QmR8BauakNcBa3RbE4nbQu76PDiJgoQgz8AJdhJuiU4TAw/go-cid"
	cmds "gx/ipfs/Qma6uuSyjkecGhMFFLfzyJDPyoDtNJSHJNweDccZhaWkgU/go-ipfs-cmds"
	"gx/ipfs/Qmde5VP1qUkyQXKCfmEUA7bP64V2HAptbJ7phuPp7jXWwg/go-ipfs-cmdkit"
)

var FileStoreCmd = &cmds.Command{
	Helptext: cmdkit.HelpText{
		Tagline: "Interact with filestore objects.",
	},
	Subcommands: map[string]*cmds.Command{
		"ls":     lsFileStore,
		"verify": verifyFileStore,
		"dups":   dupsFileStore,
	},
}

const (
	fileOrderOptionName = "file-order"
)

var lsFileStore = &cmds.Command{
	Helptext: cmdkit.HelpText{
		Tagline: "List objects in filestore.",
		LongDescription: `
List objects in the filestore.

If one or more <obj> is specified only list those specific objects,
otherwise list all objects.

The output is:

<hash> <size> <path> <offset>
`,
	},
	Arguments: []cmdkit.Argument{
		cmdkit.StringArg("obj", false, true, "Cid of objects to list."),
	},
	Options: []cmdkit.Option{
		cmdkit.BoolOption(fileOrderOptionName, "sort the results based on the path of the backing file"),
	},
	Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error {
		_, fs, err := getFilestore(env)
		if err != nil {
			return err
		}
		args := req.Arguments
		if len(args) > 0 {
			return listByArgs(res, fs, args)
		}

		fileOrder, _ := req.Options[fileOrderOptionName].(bool)
		next, err := filestore.ListAll(fs, fileOrder)
		if err != nil {
			return err
		}

		for {
			r := next()
			if r == nil {
				break
			}
			if err := res.Emit(r); err != nil {
				return err
			}
		}

		return nil
	},
	PostRun: cmds.PostRunMap{
		cmds.CLI: streamResult(func(v interface{}, out io.Writer) nonFatalError {
			r := v.(*filestore.ListRes)
			if r.ErrorMsg != "" {
				return nonFatalError(r.ErrorMsg)
			}
			fmt.Fprintf(out, "%s\n", r.FormatLong())
			return ""
		}),
	},
	Type: filestore.ListRes{},
}

var verifyFileStore = &cmds.Command{
	Helptext: cmdkit.HelpText{
		Tagline: "Verify objects in filestore.",
		LongDescription: `
Verify objects in the filestore.

If one or more <obj> is specified only verify those specific objects,
otherwise verify all objects.

The output is:

<status> <hash> <size> <path> <offset>

Where <status> is one of:
ok:       the block can be reconstructed
changed:  the contents of the backing file have changed
no-file:  the backing file could not be found
error:    there was some other problem reading the file
missing:  <obj> could not be found in the filestore
ERROR:    internal error, most likely due to a corrupt database

For ERROR entries the error will also be printed to stderr.
`,
	},
	Arguments: []cmdkit.Argument{
		cmdkit.StringArg("obj", false, true, "Cid of objects to verify."),
	},
	Options: []cmdkit.Option{
		cmdkit.BoolOption(fileOrderOptionName, "verify the objects based on the order of the backing file"),
	},
	Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error {
		_, fs, err := getFilestore(env)
		if err != nil {
			return err
		}
		args := req.Arguments
		if len(args) > 0 {
			return listByArgs(res, fs, args)
		}

		fileOrder, _ := req.Options[fileOrderOptionName].(bool)
		next, err := filestore.VerifyAll(fs, fileOrder)
		if err != nil {
			return err
		}

		for {
			r := next()
			if r == nil {
				break
			}
			if err := res.Emit(r); err != nil {
				return err
			}
		}

		return nil
	},
	Encoders: cmds.EncoderMap{
		cmds.Text: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, out *filestore.ListRes) error {
			if out.Status == filestore.StatusOtherError {
				fmt.Fprintf(os.Stderr, "%s\n", out.ErrorMsg)
			}
			fmt.Fprintf(w, "%s %s\n", out.Status.Format(), out.FormatLong())
			return nil
		}),
	},
	Type: filestore.ListRes{},
}

var dupsFileStore = &cmds.Command{
	Helptext: cmdkit.HelpText{
		Tagline: "List blocks that are both in the filestore and standard block storage.",
	},
	Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error {
		_, fs, err := getFilestore(env)
		if err != nil {
			return err
		}
		ch, err := fs.FileManager().AllKeysChan(req.Context)
		if err != nil {
			return err
		}

		for cid := range ch {
			have, err := fs.MainBlockstore().Has(cid)
			if err != nil {
				return res.Emit(&RefWrapper{Err: err.Error()})
			}
			if have {
				if err := res.Emit(&RefWrapper{Ref: cid.String()}); err != nil {
					return err
				}
			}
		}

		return nil
	},
	Encoders: cmds.EncoderMap{
		cmds.Text: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, out *RefWrapper) error {
			if out.Err != "" {
				return fmt.Errorf(out.Err)
			}

			fmt.Fprintln(w, out.Ref)

			return nil
		}),
	},
	Type: RefWrapper{},
}

func getFilestore(env cmds.Environment) (*core.IpfsNode, *filestore.Filestore, error) {
	n, err := cmdenv.GetNode(env)
	if err != nil {
		return nil, nil, err
	}
	fs := n.Filestore
	if fs == nil {
		return n, nil, filestore.ErrFilestoreNotEnabled
	}
	return n, fs, err
}

func listByArgs(res cmds.ResponseEmitter, fs *filestore.Filestore, args []string) error {
	for _, arg := range args {
		c, err := cid.Decode(arg)
		if err != nil {
			ret := &filestore.ListRes{
				Status:   filestore.StatusOtherError,
				ErrorMsg: fmt.Sprintf("%s: %v", arg, err),
			}
			if err := res.Emit(ret); err != nil {
				return err
			}
			continue
		}
		r := filestore.Verify(fs, c)
		if err := res.Emit(r); err != nil {
			return err
		}
	}

	return nil
}
<|endoftext|>


<commit_before>package commands

import (
	"errors"
	"fmt"
	"io"

	//ds "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/ipfs/go-datastore"
	bs "github.com/ipfs/go-ipfs/blocks/blockstore"
	k "github.com/ipfs/go-ipfs/blocks/key"
	cmds "github.com/ipfs/go-ipfs/commands"
	"github.com/ipfs/go-ipfs/core"
	"github.com/ipfs/go-ipfs/filestore"
	"github.com/ipfs/go-ipfs/repo/fsrepo"
	context "gx/ipfs/QmZy2y8t9zQH2a1b8q2ZSLKp17ATuJoCNxxyMFG5qFExpt/go-net/context"
)

type chanWriter struct {
	ch     <-chan *filestore.ListRes
	buf    string
	offset int
}

func (w *chanWriter) Read(p []byte) (int, error) {
	if w.offset >= len(w.buf) {
		w.offset = 0
		res, more := <-w.ch
		if !more {
			return 0, io.EOF
		}
		w.buf = res.Format()
	}
	sz := copy(p, w.buf[w.offset:])
	w.offset += sz
	return sz, nil
}

var FileStoreCmd = &cmds.Command{
	Helptext: cmds.HelpText{
		Tagline: "Interact with filestore objects",
	},
	Subcommands: map[string]*cmds.Command{
		"ls":                 lsFileStore,
		"verify":             verifyFileStore,
		"rm":                 rmFilestoreObjs,
		"find-dangling-pins": findDanglingPins,
	},
}

var lsFileStore = &cmds.Command{
	Helptext: cmds.HelpText{
		Tagline: "List objects on filestore",
	},

	Run: func(req cmds.Request, res cmds.Response) {
		_, fs, err := extractFilestore(req)
		if err != nil {
			res.SetError(err, cmds.ErrNormal)
			return
		}
		ch := make(chan *filestore.ListRes)
		go func() {
			defer close(ch)
			filestore.List(fs, ch)
		}()
		res.SetOutput(&chanWriter{ch, "", 0})
	},
	Marshalers: cmds.MarshalerMap{
		cmds.Text: func(res cmds.Response) (io.Reader, error) {
			return res.(io.Reader), nil
		},
	},
}

var verifyFileStore = &cmds.Command{
	Helptext: cmds.HelpText{
		Tagline: "Verify objects in filestore",
	},

	Run: func(req cmds.Request, res cmds.Response) {
		_, fs, err := extractFilestore(req)
		if err != nil {
			res.SetError(err, cmds.ErrNormal)
			return
		}
		ch := make(chan *filestore.ListRes)
		go func() {
			defer close(ch)
			filestore.Verify(fs, ch)
		}()
		res.SetOutput(&chanWriter{ch, "", 0})
	},
	Marshalers: cmds.MarshalerMap{
		cmds.Text: func(res cmds.Response) (io.Reader, error) {
			return res.(io.Reader), nil
		},
	},
}

var rmFilestoreObjs = &cmds.Command{
	Helptext: cmds.HelpText{
		Tagline: "Remove objects from the Filestore",
	},

	Arguments: []cmds.Argument{
		cmds.StringArg("hash", true, true, "Multi-hashes to remove.").EnableStdin(),
	},
	Run: func(req cmds.Request, res cmds.Response) {
		node, fs, err := extractFilestore(req)
		_ = fs
		if err != nil {
			res.SetError(err, cmds.ErrNormal)
			return
		}
		hashes := req.Arguments()
		serr := res.Stderr()
		numErrors := 0
		for _, mhash := range hashes {
			err = delFilestoreObj(req, node, fs, mhash)
			if err != nil {
				fmt.Fprintf(serr, "Error deleting %s: %s\n", mhash, err.Error())
				numErrors += 1
			}
		}
		if numErrors > 0 {
			res.SetError(errors.New("Could not delete some keys"), cmds.ErrNormal)
			return
		}
		return
	},
}

func delFilestoreObj(req cmds.Request, node *core.IpfsNode, fs *filestore.Datastore, mhash string) error {
	key := k.B58KeyDecode(mhash)
	err := fs.DeleteDirect(key.DsKey())
	if err != nil {
		return err
	}
	stillExists, err := node.Blockstore.Has(key)
	if err != nil {
		return err
	}
	if stillExists {
		return nil
	}
	_, pinned1, err := node.Pinning.IsPinnedWithType(key, "recursive")
	if err != nil {
		return err
	}
	_, pinned2, err := node.Pinning.IsPinnedWithType(key, "direct")
	if err != nil {
		return err
	}
	if pinned1 || pinned2 {
		println("unpinning")
		ctx, cancel := context.WithCancel(req.Context())
		defer cancel()
		err = node.Pinning.Unpin(ctx, key, true)
		if err != nil {
			return err
		}
		err := node.Pinning.Flush()
		if err != nil {
			return err
		}
	}
	return nil
}

func extractFilestore(req cmds.Request) (node *core.IpfsNode, fs *filestore.Datastore, err error) {
	node, err = req.InvocContext().GetNode()
	if err != nil {
		return
	}
	repo, ok := node.Repo.Self().(*fsrepo.FSRepo)
	if !ok {
		err = errors.New("Not a FSRepo")
		return
	}
	fs = repo.Filestore()
	return
}

var findDanglingPins = &cmds.Command{
	Helptext: cmds.HelpText{
		Tagline: "List pinned objects that no longer exists",
	},
	Run: func(req cmds.Request, res cmds.Response) {
		n, err := req.InvocContext().GetNode()
		if err != nil {
			return
		}
		r, w := io.Pipe()
		go func() {
			defer w.Close()
			err := listDanglingPins(n.Pinning.DirectKeys(), w, n.Blockstore)
			if err != nil {
				res.SetError(err, cmds.ErrNormal)
				return
			}
			err = listDanglingPins(n.Pinning.RecursiveKeys(), w, n.Blockstore)
			if err != nil {
				res.SetError(err, cmds.ErrNormal)
				return
			}
		}()
		res.SetOutput(r)
	},
	Marshalers: cmds.MarshalerMap{
		cmds.Text: func(res cmds.Response) (io.Reader, error) {
			return res.(io.Reader), nil
		},
	},
}

func listDanglingPins(keys []k.Key, out io.Writer, d bs.Blockstore) error {
	for _, k := range keys {
		exists, err := d.Has(k)
		if err != nil {
			return err
		}
		if !exists {
			fmt.Fprintln(out, k.B58String())
		}
	}
	return nil
}
<commit_msg>"filestore ls": add "--quiet" option<commit_after>package commands

import (
	"errors"
	"fmt"
	"io"

	//ds "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/ipfs/go-datastore"
	bs "github.com/ipfs/go-ipfs/blocks/blockstore"
	k "github.com/ipfs/go-ipfs/blocks/key"
	cmds "github.com/ipfs/go-ipfs/commands"
	"github.com/ipfs/go-ipfs/core"
	"github.com/ipfs/go-ipfs/filestore"
	"github.com/ipfs/go-ipfs/repo/fsrepo"
	b58 "gx/ipfs/QmT8rehPR3F6bmwL6zjUN8XpiDBFFpMP2myPdC6ApsWfJf/go-base58"
	context "gx/ipfs/QmZy2y8t9zQH2a1b8q2ZSLKp17ATuJoCNxxyMFG5qFExpt/go-net/context"
)

type chanWriter struct {
	ch       <-chan *filestore.ListRes
	buf      string
	offset   int
	hashOnly bool
}

func (w *chanWriter) Read(p []byte) (int, error) {
	if w.offset >= len(w.buf) {
		w.offset = 0
		res, more := <-w.ch
		if !more {
			return 0, io.EOF
		}
		if w.hashOnly {
			w.buf = b58.Encode(res.Key) + "\n"
		} else {
			w.buf = res.Format()
		}
	}
	sz := copy(p, w.buf[w.offset:])
	w.offset += sz
	return sz, nil
}

var FileStoreCmd = &cmds.Command{
	Helptext: cmds.HelpText{
		Tagline: "Interact with filestore objects",
	},
	Subcommands: map[string]*cmds.Command{
		"ls":                 lsFileStore,
		"verify":             verifyFileStore,
		"rm":                 rmFilestoreObjs,
		"find-dangling-pins": findDanglingPins,
	},
}

var lsFileStore = &cmds.Command{
	Helptext: cmds.HelpText{
		Tagline: "List objects in filestore",
	},
	Options: []cmds.Option{
		cmds.BoolOption("quiet", "q", "Write just hashes of objects."),
	},
	Run: func(req cmds.Request, res cmds.Response) {
		_, fs, err := extractFilestore(req)
		if err != nil {
			res.SetError(err, cmds.ErrNormal)
			return
		}
		quiet, _, err := res.Request().Option("quiet").Bool()
		if err != nil {
			res.SetError(err, cmds.ErrNormal)
			return
		}
		ch := make(chan *filestore.ListRes)
		go func() {
			defer close(ch)
			filestore.List(fs, ch)
		}()
		res.SetOutput(&chanWriter{ch, "", 0, quiet})
	},
	Marshalers: cmds.MarshalerMap{
		cmds.Text: func(res cmds.Response) (io.Reader, error) {
			return res.(io.Reader), nil
		},
	},
}

var verifyFileStore = &cmds.Command{
	Helptext: cmds.HelpText{
		Tagline: "Verify objects in filestore",
	},

	Run: func(req cmds.Request, res cmds.Response) {
		_, fs, err := extractFilestore(req)
		if err != nil {
			res.SetError(err, cmds.ErrNormal)
			return
		}
		ch := make(chan *filestore.ListRes)
		go func() {
			defer close(ch)
			filestore.Verify(fs, ch)
		}()
		res.SetOutput(&chanWriter{ch, "", 0, false})
	},
	Marshalers: cmds.MarshalerMap{
		cmds.Text: func(res cmds.Response) (io.Reader, error) {
			return res.(io.Reader), nil
		},
	},
}

var rmFilestoreObjs = &cmds.Command{
	Helptext: cmds.HelpText{
		Tagline: "Remove objects from the Filestore",
	},

	Arguments: []cmds.Argument{
		cmds.StringArg("hash", true, true, "Multi-hashes to remove.").EnableStdin(),
	},
	Run: func(req cmds.Request, res cmds.Response) {
		node, fs, err := extractFilestore(req)
		_ = fs
		if err != nil {
			res.SetError(err, cmds.ErrNormal)
			return
		}
		hashes := req.Arguments()
		serr := res.Stderr()
		numErrors := 0
		for _, mhash := range hashes {
			err = delFilestoreObj(req, node, fs, mhash)
			if err != nil {
				fmt.Fprintf(serr, "Error deleting %s: %s\n", mhash, err.Error())
				numErrors += 1
			}
		}
		if numErrors > 0 {
			res.SetError(errors.New("Could not delete some keys"), cmds.ErrNormal)
			return
		}
		return
	},
}

func delFilestoreObj(req cmds.Request, node *core.IpfsNode, fs *filestore.Datastore, mhash string) error {
	key := k.B58KeyDecode(mhash)
	err := fs.DeleteDirect(key.DsKey())
	if err != nil {
		return err
	}
	stillExists, err := node.Blockstore.Has(key)
	if err != nil {
		return err
	}
	if stillExists {
		return nil
	}
	_, pinned1, err := node.Pinning.IsPinnedWithType(key, "recursive")
	if err != nil {
		return err
	}
	_, pinned2, err := node.Pinning.IsPinnedWithType(key, "direct")
	if err != nil {
		return err
	}
	if pinned1 || pinned2 {
		println("unpinning")
		ctx, cancel := context.WithCancel(req.Context())
		defer cancel()
		err = node.Pinning.Unpin(ctx, key, true)
		if err != nil {
			return err
		}
		err := node.Pinning.Flush()
		if err != nil {
			return err
		}
	}
	return nil
}

func extractFilestore(req cmds.Request) (node *core.IpfsNode, fs *filestore.Datastore, err error) {
	node, err = req.InvocContext().GetNode()
	if err != nil {
		return
	}
	repo, ok := node.Repo.Self().(*fsrepo.FSRepo)
	if !ok {
		err = errors.New("Not a FSRepo")
		return
	}
	fs = repo.Filestore()
	return
}

var findDanglingPins = &cmds.Command{
	Helptext: cmds.HelpText{
		Tagline: "List pinned objects that no longer exists",
	},
	Run: func(req cmds.Request, res cmds.Response) {
		n, err := req.InvocContext().GetNode()
		if err != nil {
			return
		}
		r, w := io.Pipe()
		go func() {
			defer w.Close()
			err := listDanglingPins(n.Pinning.DirectKeys(), w, n.Blockstore)
			if err != nil {
				res.SetError(err, cmds.ErrNormal)
				return
			}
			err = listDanglingPins(n.Pinning.RecursiveKeys(), w, n.Blockstore)
			if err != nil {
				res.SetError(err, cmds.ErrNormal)
				return
			}
		}()
		res.SetOutput(r)
	},
	Marshalers: cmds.MarshalerMap{
		cmds.Text: func(res cmds.Response) (io.Reader, error) {
			return res.(io.Reader), nil
		},
	},
}

func listDanglingPins(keys []k.Key, out io.Writer, d bs.Blockstore) error {
	for _, k := range keys {
		exists, err := d.Has(k)
		if err != nil {
			return err
		}
		if !exists {
			fmt.Fprintln(out, k.B58String())
		}
	}
	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"crypto/tls"
	"flag"
	"io/ioutil"
	"math/rand"
	"net/http"
	"strconv"
	"sync"
	"time"

	"bytes"
	"encoding/json"
	"fmt"
	"github.com/Sirupsen/logrus"
	"net/url"
)

var Log = logrus.New()

var baseAddr string
var zuul bool

func main() {

	usersPtr := flag.Int("users", 10, "Number of users")
	delayPtr := flag.Int("delay", 1000, "Delay between calls per user")
	zuulPtr := flag.Bool("zuul", true, "Route traffic through zuul")
	baseAddrPtr := flag.String("baseAddr", "192.168.99.100", "Base address of your Swarm cluster")

	flag.Parse()

	baseAddr = *baseAddrPtr
	zuul = *zuulPtr
	users := *usersPtr
	var _ int = *delayPtr

	for i := 0; i < users; i++ {
		// go securedTest()
		go standardTest()
	}

	// Block...
	wg := sync.WaitGroup{} // Use a WaitGroup to block main() exit
	wg.Add(1)
	wg.Wait()

}

func getToken() string {

	data := url.Values{}
	data.Set("grant_type", "password")
	data.Add("client_id", "acme")
	data.Add("scope", "webshop")
	data.Add("username", "user")
	data.Add("password", "password")
	req, err := http.NewRequest("POST", "https://"+baseAddr+":9999/uaa/oauth/token", bytes.NewBufferString(data.Encode()))
	if err != nil {
		panic(err.Error())
	}
	var DefaultTransport http.RoundTripper = &http.Transport{
		TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
	}

	headers := make(map[string][]string)
	headers["Authorization"] = []string{"Basic YWNtZTphY21lc2VjcmV0"}
	headers["Content-Type"] = []string{"application/x-www-form-urlencoded"}

	req.Header = headers

	resp, err := DefaultTransport.RoundTrip(req)
	if err != nil {
		panic(err.Error())
	}
	if resp.StatusCode > 299 {
		panic("Call to get auth token returned status " + resp.Status)
	}
	respdata, _ := ioutil.ReadAll(resp.Body)
	m := make(map[string]interface{})
	err = json.Unmarshal(respdata, &m)
	if err != nil {
		panic(err.Error())
	}

	fmt.Println("Got TOKEN: " + string(m["access_token"].(string)))
	return string(m["access_token"].(string))
}

var defaultTransport http.RoundTripper = &http.Transport{
	TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
}

func securedTest() {

	var token = getToken()
	for {
		accountId := rand.Intn(99) + 10000
		url := "https://" + baseAddr + ":8765/api/secured/account/" + strconv.Itoa(accountId)

		req, _ := http.NewRequest("GET", url, nil)
		req.Header.Add("Authorization", "Bearer "+token)
		req.Header.Add("Connection", "keep-alive")
		req.Header.Add("Keep-Alive", "timeout=10, max=5")
		resp, err := defaultTransport.RoundTrip(req)
		if resp.StatusCode != 200 {
			fmt.Println("Status: " + resp.Status)
		}
		if err != nil {
			fmt.Println("Error: " + err.Error())
			panic(err)
		}
		body, err := ioutil.ReadAll(resp.Body)
		if err != nil {
			fmt.Println("Error: " + err.Error())
			fmt.Println("Body: " + string(body))
			panic(err)
		}
		m := make(map[string]string)
		json.Unmarshal(body, &m)
		Log.Println("Account: " + m["accountServedBy"] + "\tComposite: " + m["imageServedBy"] + "\tQuote: " + m["quoteServedBy"])
		time.Sleep(time.Second * 1)
	}
}

func standardTest() {
	var url string
	if zuul {
		url = "https://" + baseAddr + ":8765/api/accounts/"
	} else {
		url = "http://" + baseAddr + ":6767/accounts/"
	}
	for {
		accountId := rand.Intn(99) + 10000
		serviceUrl := url + strconv.Itoa(accountId)

		var DefaultTransport http.RoundTripper = &http.Transport{
			TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
		}
		req, _ := http.NewRequest("GET", serviceUrl, nil)
		resp, err := DefaultTransport.RoundTrip(req)

		if err != nil {
			panic(err)
		}
		body, _ := ioutil.ReadAll(resp.Body)
		Log.Println(string(body))
		time.Sleep(time.Second * 1)
	}

}
<commit_msg>Some logging in load-test<commit_after>package main

import (
	"crypto/tls"
	"flag"
	"io/ioutil"
	"math/rand"
	"net/http"
	"strconv"
	"sync"
	"time"

	"bytes"
	"encoding/json"
	"fmt"
	"github.com/Sirupsen/logrus"
	"net/url"
)

var Log = logrus.New()

var baseAddr string
var zuul bool

func main() {

	usersPtr := flag.Int("users", 10, "Number of users")
	delayPtr := flag.Int("delay", 1000, "Delay between calls per user")
	zuulPtr := flag.Bool("zuul", true, "Route traffic through zuul")
	baseAddrPtr := flag.String("baseAddr", "192.168.99.100", "Base address of your Swarm cluster")

	flag.Parse()

	baseAddr = *baseAddrPtr
	zuul = *zuulPtr
	users := *usersPtr
	var _ int = *delayPtr

	for i := 0; i < users; i++ {
		// go securedTest()
		go standardTest()
	}

	// Block...
	wg := sync.WaitGroup{} // Use a WaitGroup to block main() exit
	wg.Add(1)
	wg.Wait()

}

func getToken() string {

	data := url.Values{}
	data.Set("grant_type", "password")
	data.Add("client_id", "acme")
	data.Add("scope", "webshop")
	data.Add("username", "user")
	data.Add("password", "password")
	req, err := http.NewRequest("POST", "https://"+baseAddr+":9999/uaa/oauth/token", bytes.NewBufferString(data.Encode()))
	if err != nil {
		panic(err.Error())
	}
	var DefaultTransport http.RoundTripper = &http.Transport{
		TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
	}

	headers := make(map[string][]string)
	headers["Authorization"] = []string{"Basic YWNtZTphY21lc2VjcmV0"}
	headers["Content-Type"] = []string{"application/x-www-form-urlencoded"}

	req.Header = headers

	resp, err := DefaultTransport.RoundTrip(req)
	if err != nil {
		panic(err.Error())
	}
	if resp.StatusCode > 299 {
		panic("Call to get auth token returned status " + resp.Status)
	}
	respdata, _ := ioutil.ReadAll(resp.Body)
	m := make(map[string]interface{})
	err = json.Unmarshal(respdata, &m)
	if err != nil {
		panic(err.Error())
	}

	fmt.Println("Got TOKEN: " + string(m["access_token"].(string)))
	return string(m["access_token"].(string))
}

var defaultTransport http.RoundTripper = &http.Transport{
	TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
}

func securedTest() {

	var token = getToken()
	for {
		accountId := rand.Intn(99) + 10000
		url := "https://" + baseAddr + ":8765/api/secured/account/" + strconv.Itoa(accountId)

		req, _ := http.NewRequest("GET", url, nil)
		req.Header.Add("Authorization", "Bearer "+token)
		req.Header.Add("Connection", "keep-alive")
		req.Header.Add("Keep-Alive", "timeout=10, max=5")
		resp, err := defaultTransport.RoundTrip(req)
		if resp.StatusCode != 200 {
			fmt.Println("Status: " + resp.Status)
		}
		if err != nil {
			fmt.Println("Error: " + err.Error())
			panic(err)
		}
		body, err := ioutil.ReadAll(resp.Body)
		if err != nil {
			fmt.Println("Error: " + err.Error())
			fmt.Println("Body: " + string(body))
			panic(err)
		}
		m := make(map[string]string)
		json.Unmarshal(body, &m)
		Log.Println("Account: " + m["accountServedBy"] + "\tComposite: " + m["imageServedBy"] + "\tQuote: " + m["quoteServedBy"])
		time.Sleep(time.Second * 1)
	}
}

func standardTest() {
	var url string
	if zuul {
		Log.Println("Using HTTPS through ZUUL")
		url = "https://" + baseAddr + ":8765/api/accounts/"
	} else {
		url = "http://" + baseAddr + ":6767/accounts/"
	}
	for {
		accountId := rand.Intn(99) + 10000
		serviceUrl := url + strconv.Itoa(accountId)

		var DefaultTransport http.RoundTripper = &http.Transport{
			TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
			DisableKeepAlives: true,
		}
		req, _ := http.NewRequest("GET", serviceUrl, nil)
		resp, err := DefaultTransport.RoundTrip(req)

		if err != nil {
			panic(err)
		}
		body, _ := ioutil.ReadAll(resp.Body)
		Log.Println(string(body))
		time.Sleep(time.Second * 1)
	}

}
<|endoftext|>


<commit_before>// Copyright 2016 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"context"
	"time"

	"github.com/golang/protobuf/ptypes"
	"go.chromium.org/luci/common/clock"
	"go.chromium.org/luci/common/data/rand/mathrand"
	"go.chromium.org/luci/common/errors"
	"go.chromium.org/luci/common/gcloud/gs"
	"go.chromium.org/luci/common/logging"
	log "go.chromium.org/luci/common/logging"
	"go.chromium.org/luci/common/tsmon/distribution"
	"go.chromium.org/luci/common/tsmon/field"
	"go.chromium.org/luci/common/tsmon/metric"
	montypes "go.chromium.org/luci/common/tsmon/types"
	"go.chromium.org/luci/logdog/api/config/svcconfig"
	logdog "go.chromium.org/luci/logdog/api/endpoints/coordinator/services/v1"
	"go.chromium.org/luci/logdog/appengine/coordinator"
	"go.chromium.org/luci/logdog/server/archivist"
	"go.chromium.org/luci/logdog/server/bundleServicesClient"
	"go.chromium.org/luci/logdog/server/service"

	"go.chromium.org/luci/hardcoded/chromeinfra"
)

var (
	errInvalidConfig = errors.New("invalid configuration")

	errNoWorkToDo = errors.New("no work to do")

	// batchSize is the number of jobs to lease from taskqueue per cycle.
	// TaskQueue has a limit of 10qps for leasing tasks, so the batch size must be set to:
	// batchSize * 10 > (max expected stream creation QPS)
	// In 2019, max stream QPS is approximately 400 QPS
	// TODO(hinoka): Make these configurable in settings.
	batchSize = int64(50)

	// leaseTime is the amount of time to to lease the batch of tasks for.
	// We need:
	// (time to process avg stream) * batchSize < leaseTime
	// As of 2019, 90th percentile process time per stream is ~4s.
	leaseTime = 15 * time.Minute

	// maxSleepTime is the max amount of time to sleep in-between errors, in seconds.
	maxSleepTime = 32

	// tsTaskProcessingTime measures the amount of time spent processing a single
	// task.
	//
	// The "consumed" field is true if the underlying task was consumed and
	// false if it was not.
	tsTaskProcessingTime = metric.NewCumulativeDistribution("logdog/archivist/task_processing_time_ms_ng",
		"The amount of time (in milliseconds) that a single task takes to process in the new pipeline.",
		&montypes.MetricMetadata{Units: montypes.Milliseconds},
		distribution.DefaultBucketer,
		field.Bool("consumed"))

	tsLoopCycleTime = metric.NewCumulativeDistribution("logdog/archivist/loop_cycle_time_ms",
		"The amount of time a single batch of leases takes to process.",
		&montypes.MetricMetadata{Units: montypes.Milliseconds},
		distribution.DefaultBucketer)

	tsLeaseCount = metric.NewCounter("logdog/archivist/tasks_leased",
		"Number of tasks leased.",
		nil)

	tsNackCount = metric.NewCounter("logdog/archivist/tasks_not_acked",
		"Number of tasks leased but failed.",
		nil)

	tsAckCount = metric.NewCounter("logdog/archivist/tasks_acked",
		"Number of tasks successfully completed and acked.",
		nil)
)

// application is the Archivist application state.
type application struct {
	service.Service
}

// runForever runs the archivist loop forever.
func runForever(c context.Context, ar archivist.Archivist) error {
	sleepTime := 1
	client := ar.Service
	// Forever loop.
	for {
		// Only exit out if the context is cancelled (I.E. Ctrl + c).
		if c.Err() != nil {
			return c.Err()
		}
		if err := func() error {
			cycleStartTime := clock.Now(c)
			leaseTimeProto := ptypes.DurationProto(leaseTime)
			nc, cancel := context.WithTimeout(c, leaseTime)
			defer cancel()

			tasks, err := client.LeaseArchiveTasks(c, &logdog.LeaseRequest{
				MaxTasks:  batchSize,
				LeaseTime: leaseTimeProto,
			})
			if err != nil {
				return err
			}
			logging.Fields{
				"cycleStart": true,
				"numLeased":  len(tasks.Tasks),
				"noWork":     len(tasks.Tasks) == 0,
			}.Infof(c, "Start of cycled, leased %d tasks", len(tasks.Tasks))
			tsLeaseCount.Add(c, int64(len(tasks.Tasks)))
			if len(tasks.Tasks) == 0 {
				return errNoWorkToDo
			}
			merr := errors.NewLazyMultiError(len(tasks.Tasks))
			ackTasks := make([]*logdog.ArchiveTask, 0, len(tasks.Tasks))
			for i, task := range tasks.Tasks {
				deleteTask := false
				startTime := clock.Now(c)
				if err := ar.ArchiveTask(nc, task); err != nil {
					merr.Assign(i, err)
				} else { // err == nil
					ackTasks = append(ackTasks, task)
					deleteTask = true
				}
				duration := clock.Now(c).Sub(startTime)
				tsTaskProcessingTime.Add(c, float64(duration.Nanoseconds())/1000000, deleteTask)
			}
			tsNackCount.Add(c, int64(len(tasks.Tasks)-len(ackTasks)))
			// Ack the successful tasks.  We log the error here since there's nothing we can do.
			if len(ackTasks) > 0 {
				tsAckCount.Add(c, int64(len(ackTasks)))
				c := log.SetFields(c, log.Fields{"Tasks": ackTasks})
				if _, err := client.DeleteArchiveTasks(c, &logdog.DeleteRequest{Tasks: ackTasks}); err != nil {
					log.WithError(err).Errorf(c, "error while acking tasks (%s)", ackTasks)
				}
			}

			duration := clock.Now(c).Sub(cycleStartTime)
			logging.Fields{
				"cycleEnd":     true,
				"numLeased":    len(tasks.Tasks),
				"numCompleted": len(ackTasks),
				"timeSpentSec": duration.Seconds(),
			}.Infof(c, "Done archiving %d items (%d successful) took %.2fs", len(tasks.Tasks), len(ackTasks), duration.Seconds())
			tsLoopCycleTime.Add(c, float64(duration.Nanoseconds()/1000000))
			return merr.Get()
		}(); err != nil {
			// Back off on errors.
			sleepTime *= 2
			if sleepTime > maxSleepTime {
				sleepTime = maxSleepTime
			}
			switch err {
			case errNoWorkToDo:
				logging.Infof(c, "no work to do, sleeping for %d seconds", sleepTime)
			default:
				logging.WithError(err).Errorf(c, "got error in loop, sleeping for %d seconds", sleepTime)
			}
			clock.Sleep(c, time.Duration(sleepTime)*time.Second)
		} else {
			sleepTime = 1
		}
	}
}

// settingsUpdater updates the settings from datastore every once in a while.
func settingsUpdater(c context.Context) {
	for {
		set := coordinator.GetSettings(c)
		logging.Debugf(c, "updating settings: %v", set)
		if batchSize != set.ArchivistBatchSize {
			batchSize = set.ArchivistBatchSize
		}
		if leaseTime != set.ArchivistLeaseTime {
			leaseTime = set.ArchivistLeaseTime
		}
		clock.Sleep(c, 5*time.Minute)
	}
}

// run is the main execution function.
func (a *application) runArchivist(c context.Context) error {
	cfg := a.ServiceConfig()

	coordCfg, acfg := cfg.GetCoordinator(), cfg.GetArchivist()
	switch {
	case coordCfg == nil:
		fallthrough

	case acfg == nil:
		return errors.New("missing required config: archivist")
	case acfg.GsStagingBucket == "":
		return errors.New("missing required config: archivist.gs_staging_bucket")
	}

	// Initialize our Storage.
	//
	// NOTE: We're requesting read/write access even though we only need read-only
	// access because GKE doesn't understand the read-only scope:
	// https://www.googleapis.com/auth/bigtable.readonly
	st, err := a.IntermediateStorage(c, true)
	if err != nil {
		log.WithError(err).Errorf(c, "Failed to get storage instance.")
		return err
	}
	defer st.Close()

	// Defines our Google Storage client project scoped factory.
	gsClientFactory := func(ctx context.Context, project string) (gs.Client, error) {
		gsClient, err := a.GSClient(ctx, project)
		if err != nil {
			log.WithError(err).Errorf(c, "Failed to get Google Storage client.")
			return nil, err
		}
		return gsClient, nil
	}

	// Initialize a Coordinator client that bundles requests together.
	coordClient := &bundleServicesClient.Client{
		ServicesClient:       a.Coordinator(),
		DelayThreshold:       time.Second,
		BundleCountThreshold: 100,
	}
	defer coordClient.Flush()

	ar := archivist.Archivist{
		Service:         coordClient,
		SettingsLoader:  a.GetSettingsLoader(acfg),
		Storage:         st,
		GSClientFactory: gsClientFactory,
	}

	// Application shutdown will now operate by stopping the Iterator.
	c, cancelFunc := context.WithCancel(c)
	defer cancelFunc()

	// Application shutdown will now operate by cancelling the Archivist's
	// shutdown Context.
	a.SetShutdownFunc(cancelFunc)

	// Update our settings periodicall.
	go settingsUpdater(c)

	return runForever(c, ar)
}

// GetSettingsLoader is an archivist.SettingsLoader implementation that merges
// global and project-specific settings.
//
// The resulting settings object will be verified by the Archivist.
func (a *application) GetSettingsLoader(acfg *svcconfig.Archivist) archivist.SettingsLoader {
	serviceID := a.ServiceID()

	return func(c context.Context, project string) (*archivist.Settings, error) {
		// Fold in our project-specific configuration, if valid.
		pcfg, err := a.ProjectConfig(c, project)
		if err != nil {
			log.Fields{
				log.ErrorKey: err,
				"project":    project,
			}.Errorf(c, "Failed to fetch project configuration.")
			return nil, err
		}

		indexParam := func(get func(ic *svcconfig.ArchiveIndexConfig) int32) int {
			if ic := pcfg.ArchiveIndexConfig; ic != nil {
				if v := get(ic); v > 0 {
					return int(v)
				}
			}

			if ic := acfg.ArchiveIndexConfig; ic != nil {
				if v := get(ic); v > 0 {
					return int(v)
				}
			}

			return 0
		}

		// Load our base settings.
		//
		// Archival bases are:
		// Staging: gs://<services:gs_staging_bucket>/<project-id>/...
		// Archive: gs://<project:archive_gs_bucket>/<project-id>/...
		st := archivist.Settings{
			GSBase:        gs.MakePath(pcfg.ArchiveGsBucket, "").Concat(serviceID),
			GSStagingBase: gs.MakePath(acfg.GsStagingBucket, "").Concat(serviceID),

			IndexStreamRange: indexParam(func(ic *svcconfig.ArchiveIndexConfig) int32 { return ic.StreamRange }),
			IndexPrefixRange: indexParam(func(ic *svcconfig.ArchiveIndexConfig) int32 { return ic.PrefixRange }),
			IndexByteRange:   indexParam(func(ic *svcconfig.ArchiveIndexConfig) int32 { return ic.ByteRange }),
		}

		// Fold project settings into loaded ones.
		return &st, nil
	}
}

// Entry point.
func main() {
	mathrand.SeedRandomly()
	a := application{
		Service: service.Service{
			Name:               "archivist",
			DefaultAuthOptions: chromeinfra.DefaultAuthOptions(),
		},
	}
	a.Run(context.Background(), a.runArchivist)
}
<commit_msg>[archivist] Don't start the loop with wrong parameters.<commit_after>// Copyright 2016 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"context"
	"sync"
	"time"

	"github.com/golang/protobuf/ptypes"
	"go.chromium.org/luci/common/clock"
	"go.chromium.org/luci/common/data/rand/mathrand"
	"go.chromium.org/luci/common/errors"
	"go.chromium.org/luci/common/gcloud/gs"
	"go.chromium.org/luci/common/logging"
	log "go.chromium.org/luci/common/logging"
	"go.chromium.org/luci/common/tsmon/distribution"
	"go.chromium.org/luci/common/tsmon/field"
	"go.chromium.org/luci/common/tsmon/metric"
	montypes "go.chromium.org/luci/common/tsmon/types"
	"go.chromium.org/luci/logdog/api/config/svcconfig"
	logdog "go.chromium.org/luci/logdog/api/endpoints/coordinator/services/v1"
	"go.chromium.org/luci/logdog/appengine/coordinator"
	"go.chromium.org/luci/logdog/server/archivist"
	"go.chromium.org/luci/logdog/server/bundleServicesClient"
	"go.chromium.org/luci/logdog/server/service"

	"go.chromium.org/luci/hardcoded/chromeinfra"
)

// loopParams control the outer archivist loop.
type loopParams struct {
	batchSize int64
	leaseTime time.Duration
}

var (
	errInvalidConfig = errors.New("invalid configuration")
	errNoWorkToDo    = errors.New("no work to do")

	// Note: these initial values are mostly bogus, they are replaced by
	// fetchLoopParams before the loop starts.
	loop = loopParams{
		// batchSize is the number of jobs to lease from taskqueue per cycle.
		//
		// TaskQueue has a limit of 10qps for leasing tasks, so the batch size must
		// be set to:
		// batchSize * 10 > (max expected stream creation QPS)
		//
		// In 2020, max stream QPS is approximately 1000 QPS
		batchSize: 500,

		// leaseTime is the amount of time to to lease the batch of tasks for.
		//
		// We need:
		// (time to process avg stream) * batchSize < leaseTime
		//
		// As of 2020, 90th percentile process time per stream is ~5s, 95th
		// percentile of the loop time is 25m.
		leaseTime: 40 * time.Minute,
	}

	// loopM protects 'loop'.
	loopM = sync.Mutex{}

	// maxSleepTime is the max amount of time to sleep in-between errors, in seconds.
	maxSleepTime = 32

	// tsTaskProcessingTime measures the amount of time spent processing a single
	// task.
	//
	// The "consumed" field is true if the underlying task was consumed and
	// false if it was not.
	tsTaskProcessingTime = metric.NewCumulativeDistribution("logdog/archivist/task_processing_time_ms_ng",
		"The amount of time (in milliseconds) that a single task takes to process in the new pipeline.",
		&montypes.MetricMetadata{Units: montypes.Milliseconds},
		distribution.DefaultBucketer,
		field.Bool("consumed"))

	tsLoopCycleTime = metric.NewCumulativeDistribution("logdog/archivist/loop_cycle_time_ms",
		"The amount of time a single batch of leases takes to process.",
		&montypes.MetricMetadata{Units: montypes.Milliseconds},
		distribution.DefaultBucketer)

	tsLeaseCount = metric.NewCounter("logdog/archivist/tasks_leased",
		"Number of tasks leased.",
		nil)

	tsNackCount = metric.NewCounter("logdog/archivist/tasks_not_acked",
		"Number of tasks leased but failed.",
		nil)

	tsAckCount = metric.NewCounter("logdog/archivist/tasks_acked",
		"Number of tasks successfully completed and acked.",
		nil)
)

// application is the Archivist application state.
type application struct {
	service.Service
}

// runForever runs the archivist loop forever.
func runForever(c context.Context, ar archivist.Archivist) error {
	sleepTime := 1
	client := ar.Service
	// Forever loop.
	for {
		// Only exit out if the context is cancelled (I.E. Ctrl + c).
		if c.Err() != nil {
			return c.Err()
		}
		if err := func() error {
			loop := grabLoopParams()

			// Sizes of archival tasks are pretty uniform. It means the total time it
			// takes to process equally sized batches of tasks is approximately the
			// same across all archivist instances. Once a bunch of them start at the
			// same time, they end up hitting LeaseArchiveTasks in waves at
			// approximately the same time. Randomize the batch size +-25% to remove
			// this synchronization.
			factor := int64(mathrand.Intn(c, 500) + 750) // [750, 1250)
			loop.batchSize = loop.batchSize * factor / 1000
			loop.leaseTime = loop.leaseTime * time.Duration(factor) / 1000

			cycleStartTime := clock.Now(c)
			leaseTimeProto := ptypes.DurationProto(loop.leaseTime)
			nc, cancel := context.WithTimeout(c, loop.leaseTime)
			defer cancel()

			logging.Infof(c, "Leasing max %d tasks for %s", loop.batchSize, loop.leaseTime)
			tasks, err := client.LeaseArchiveTasks(c, &logdog.LeaseRequest{
				MaxTasks:  loop.batchSize,
				LeaseTime: leaseTimeProto,
			})
			if err != nil {
				return err
			}
			logging.Fields{
				"cycleStart": true,
				"numLeased":  len(tasks.Tasks),
				"noWork":     len(tasks.Tasks) == 0,
			}.Infof(c, "Start of cycled, leased %d tasks", len(tasks.Tasks))
			tsLeaseCount.Add(c, int64(len(tasks.Tasks)))
			if len(tasks.Tasks) == 0 {
				return errNoWorkToDo
			}
			merr := errors.NewLazyMultiError(len(tasks.Tasks))
			ackTasks := make([]*logdog.ArchiveTask, 0, len(tasks.Tasks))
			for i, task := range tasks.Tasks {
				deleteTask := false
				startTime := clock.Now(c)
				if err := ar.ArchiveTask(nc, task); err != nil {
					merr.Assign(i, err)
				} else { // err == nil
					ackTasks = append(ackTasks, task)
					deleteTask = true
				}
				duration := clock.Now(c).Sub(startTime)
				tsTaskProcessingTime.Add(c, float64(duration.Nanoseconds())/1000000, deleteTask)
			}
			tsNackCount.Add(c, int64(len(tasks.Tasks)-len(ackTasks)))
			// Ack the successful tasks.  We log the error here since there's nothing we can do.
			if len(ackTasks) > 0 {
				tsAckCount.Add(c, int64(len(ackTasks)))
				c := log.SetFields(c, log.Fields{"Tasks": ackTasks})
				if _, err := client.DeleteArchiveTasks(c, &logdog.DeleteRequest{Tasks: ackTasks}); err != nil {
					log.WithError(err).Errorf(c, "error while acking tasks (%s)", ackTasks)
				}
			}

			duration := clock.Now(c).Sub(cycleStartTime)
			logging.Fields{
				"cycleEnd":     true,
				"numLeased":    len(tasks.Tasks),
				"numCompleted": len(ackTasks),
				"timeSpentSec": duration.Seconds(),
			}.Infof(c, "Done archiving %d items (%d successful) took %.2fs", len(tasks.Tasks), len(ackTasks), duration.Seconds())
			tsLoopCycleTime.Add(c, float64(duration.Nanoseconds()/1000000))
			return merr.Get()
		}(); err != nil {
			// Back off on errors.
			sleepTime *= 2
			if sleepTime > maxSleepTime {
				sleepTime = maxSleepTime
			}
			switch err {
			case errNoWorkToDo:
				logging.Infof(c, "no work to do, sleeping for %d seconds", sleepTime)
			default:
				logging.WithError(err).Errorf(c, "got error in loop, sleeping for %d seconds", sleepTime)
			}
			clock.Sleep(c, time.Duration(sleepTime)*time.Second)
		} else {
			sleepTime = 1
		}
	}
}

// grabLoopParams returns a copy of most recent value of `loop`.
func grabLoopParams() loopParams {
	loopM.Lock()
	defer loopM.Unlock()
	return loop
}

// fetchLoopParams updates `loop` based on settings in datastore (or panics).
func fetchLoopParams(c context.Context) {
	// Note: these are eventually controlled through /admin/portal/archivist.
	set := coordinator.GetSettings(c)
	loopM.Lock()
	defer loopM.Unlock()
	if set.ArchivistBatchSize != 0 {
		loop.batchSize = set.ArchivistBatchSize
	}
	if set.ArchivistLeaseTime != 0 {
		loop.leaseTime = set.ArchivistLeaseTime
	}
	logging.Debugf(c, "loop settings: batchSize:%d, leastTime:%s", loop.batchSize, loop.leaseTime)
}

// loopParamsUpdater updates loopParams based on setting in datastore every once
// in a while.
func loopParamsUpdater(c context.Context) {
	for {
		clock.Sleep(c, 5*time.Minute)
		fetchLoopParams(c)
	}
}

// run is the main execution function.
func (a *application) runArchivist(c context.Context) error {
	cfg := a.ServiceConfig()

	coordCfg, acfg := cfg.GetCoordinator(), cfg.GetArchivist()
	switch {
	case coordCfg == nil:
		fallthrough

	case acfg == nil:
		return errors.New("missing required config: archivist")
	case acfg.GsStagingBucket == "":
		return errors.New("missing required config: archivist.gs_staging_bucket")
	}

	// Initialize our Storage.
	//
	// NOTE: We're requesting read/write access even though we only need read-only
	// access because GKE doesn't understand the read-only scope:
	// https://www.googleapis.com/auth/bigtable.readonly
	st, err := a.IntermediateStorage(c, true)
	if err != nil {
		log.WithError(err).Errorf(c, "Failed to get storage instance.")
		return err
	}
	defer st.Close()

	// Defines our Google Storage client project scoped factory.
	gsClientFactory := func(ctx context.Context, project string) (gs.Client, error) {
		gsClient, err := a.GSClient(ctx, project)
		if err != nil {
			log.WithError(err).Errorf(c, "Failed to get Google Storage client.")
			return nil, err
		}
		return gsClient, nil
	}

	// Initialize a Coordinator client that bundles requests together.
	coordClient := &bundleServicesClient.Client{
		ServicesClient:       a.Coordinator(),
		DelayThreshold:       time.Second,
		BundleCountThreshold: 100,
	}
	defer coordClient.Flush()

	ar := archivist.Archivist{
		Service:         coordClient,
		SettingsLoader:  a.GetSettingsLoader(acfg),
		Storage:         st,
		GSClientFactory: gsClientFactory,
	}

	// Application shutdown will now operate by stopping the Iterator.
	c, cancelFunc := context.WithCancel(c)
	defer cancelFunc()

	// Application shutdown will now operate by cancelling the Archivist's
	// shutdown Context.
	a.SetShutdownFunc(cancelFunc)

	// Load our settings and update them periodically.
	fetchLoopParams(c)
	go loopParamsUpdater(c)

	return runForever(c, ar)
}

// GetSettingsLoader is an archivist.SettingsLoader implementation that merges
// global and project-specific settings.
//
// The resulting settings object will be verified by the Archivist.
func (a *application) GetSettingsLoader(acfg *svcconfig.Archivist) archivist.SettingsLoader {
	serviceID := a.ServiceID()

	return func(c context.Context, project string) (*archivist.Settings, error) {
		// Fold in our project-specific configuration, if valid.
		pcfg, err := a.ProjectConfig(c, project)
		if err != nil {
			log.Fields{
				log.ErrorKey: err,
				"project":    project,
			}.Errorf(c, "Failed to fetch project configuration.")
			return nil, err
		}

		indexParam := func(get func(ic *svcconfig.ArchiveIndexConfig) int32) int {
			if ic := pcfg.ArchiveIndexConfig; ic != nil {
				if v := get(ic); v > 0 {
					return int(v)
				}
			}

			if ic := acfg.ArchiveIndexConfig; ic != nil {
				if v := get(ic); v > 0 {
					return int(v)
				}
			}

			return 0
		}

		// Load our base settings.
		//
		// Archival bases are:
		// Staging: gs://<services:gs_staging_bucket>/<project-id>/...
		// Archive: gs://<project:archive_gs_bucket>/<project-id>/...
		st := archivist.Settings{
			GSBase:        gs.MakePath(pcfg.ArchiveGsBucket, "").Concat(serviceID),
			GSStagingBase: gs.MakePath(acfg.GsStagingBucket, "").Concat(serviceID),

			IndexStreamRange: indexParam(func(ic *svcconfig.ArchiveIndexConfig) int32 { return ic.StreamRange }),
			IndexPrefixRange: indexParam(func(ic *svcconfig.ArchiveIndexConfig) int32 { return ic.PrefixRange }),
			IndexByteRange:   indexParam(func(ic *svcconfig.ArchiveIndexConfig) int32 { return ic.ByteRange }),
		}

		// Fold project settings into loaded ones.
		return &st, nil
	}
}

// Entry point.
func main() {
	mathrand.SeedRandomly()
	a := application{
		Service: service.Service{
			Name:               "archivist",
			DefaultAuthOptions: chromeinfra.DefaultAuthOptions(),
		},
	}
	a.Run(context.Background(), a.runArchivist)
}
<|endoftext|>


<commit_before>package materials

import (
	"bufio"
	"errors"
	"fmt"
	"os"
	"os/user"
	"path/filepath"
	"strings"
)

// Project describes the information we track about a users
// projects. Here we keep the name of the project and the
// directory path. The name of the project is the top level
// directory of the project. The path is the full path to
// the project excluding the name.
type Project struct {
	Name   string `json:"name" xml:"name"`
	Path   string `json:"path" xml:"path"`
	Status string `json:"status" xml:"status"`
}

// MaterialsProjects contains a list of user projects and information that
// is needed by the methods to load the projects file.
type MaterialsProjects struct {
	dir      string
	projects []Project
}

// CurrentUserProjects retrieves the projects for the currently logged in user.
func CurrentUserProjects() (*MaterialsProjects, error) {
	u, err := user.Current()
	if err != nil {
		return nil, err
	}
	return loadProjectsFrom(u.HomeDir)
}

// ProjectsForUser retrieves the projects for the named user.
func ProjectsForUser(username string) (*MaterialsProjects, error) {
	u, err := user.Lookup(username)
	if err != nil {
		return nil, err
	}
	return loadProjectsFrom(u.HomeDir)
}

// ProjectsFromHome retrieves the projects located in the directory
// pointed at by the HOME environment variable.
func ProjectsFromHome() (*MaterialsProjects, error) {
	home := os.Getenv("HOME")
	if home == "" {
		return nil, errors.New("HOME Environment variable is undefined")
	}
	return loadProjectsFrom(home)
}

// ProjectsFrom retrieves the projects from the specified directory. The
// path given cannot contain the .materials subdirectory. A .materials
// subdirectory must exist in the path.
func ProjectsFrom(dir string) (*MaterialsProjects, error) {
	return loadProjectsFrom(dir)
}

// Load projects using the specified directory as a base.
// The projects file is located in {dir}/.materials/projects.
func loadProjectsFrom(dir string) (*MaterialsProjects, error) {
	userProjects := MaterialsProjects{dir: dir}
	err := userProjects.loadProjects()
	if err != nil {
		return nil, err
	}
	return &userProjects, err
}

// Reload re-reads and loads the projects file.
func (p *MaterialsProjects) Reload() error {
	return p.loadProjects()
}

// Reads the projects file, parses its contents and loads it into
// the Projects struct.
func (p *MaterialsProjects) loadProjects() error {
	projectsFile, err := os.Open(p.projectsFile())
	if err != nil {
		return err
	}
	defer projectsFile.Close()

	projects := []Project{}
	scanner := bufio.NewScanner(projectsFile)
	for scanner.Scan() {
		splitLine := strings.Split(scanner.Text(), "|")
		if len(splitLine) == 3 {
			projects = append(projects, Project{
				Name:   strings.TrimSpace(splitLine[0]),
				Path:   strings.TrimSpace(splitLine[1]),
				Status: strings.TrimSpace(splitLine[2]),
			})
		}
	}
	p.projects = projects
	return nil
}

// Creates the path to the projects file:
// {dir}/.materias/projects
func (p *MaterialsProjects) projectsFile() string {
	return filepath.Join(p.dir, ".materials", "projects")
}

// Projects returns the list of loaded projects.
func (p *MaterialsProjects) Projects() []Project {
	return p.projects
}

// Add adds a new project and updates the projects file.
func (p *MaterialsProjects) Add(proj Project) error {
	if p.Exists(proj.Name) {
		return errors.New(fmt.Sprintf("Project already exists: %s", proj.Name))
	}
	p.projects = append(p.projects, proj)
	return p.writeToProjectsFile(p.projects)
}

// Remove removes a project and updates the projects file.
func (p *MaterialsProjects) Remove(projectName string) error {
	projects, projectsUpdated := p.projectsExceptFor(projectName)

	// We found the entry to remove. Thus we need to
	// update the projects file.
	if projectsUpdated {
		err := p.writeToProjectsFile(projects)
		if err == nil {
			p.projects = projects
		}
		return err
	} else {
		// project wasn't found so we don't update anything and we return no error
		return nil
	}
}

func (p *MaterialsProjects) Update(proj Project) error {
	_, index := p.find(proj.Name)
	if index != -1 {
		projects := p.Projects()
		projects[index].Path = proj.Path
		projects[index].Status = proj.Status
		return p.writeToProjectsFile(p.Projects())
	}

	return errors.New(fmt.Sprintf("Project not found: %s", proj.Name))
}

// projectsExceptFor returns a new list of projects except for the project
// matching projectName. It returns true if it found a project matching
// projectName.
func (p *MaterialsProjects) projectsExceptFor(projectName string) ([]Project, bool) {
	projects := []Project{}
	found := false
	for _, project := range p.projects {
		if project.Name != projectName {
			projects = append(projects, project)
		} else {
			found = true
		}
	}
	return projects, found
}

// writeToProjectsFile overwrites the projects file with the list
// of projects.
func (p *MaterialsProjects) writeToProjectsFile(projects []Project) error {
	file, err := os.Create(p.projectsFile())
	if err != nil {
		return err
	}
	defer file.Close()
	for _, project := range projects {
		projectLine := fmt.Sprintf("%s|%s|%s\n",
			strings.TrimSpace(project.Name),
			strings.TrimSpace(project.Path),
			strings.TrimSpace(project.Status))
		file.WriteString(projectLine)
	}
	return nil
}

// Exists returns true if there is a project matching
// the given Name.
func (p *MaterialsProjects) Exists(projectName string) bool {
	_, found := p.Find(projectName)
	return found
}

// Find returns (Project, true) if the project is found otherwise
// it returns (Project{}, false)
func (p *MaterialsProjects) Find(projectName string) (Project, bool) {
	project, index := p.find(projectName)
	return project, index != -1
}

// find returns (Project, index) where index is -1 if
// the project wasn't found, otherwise it is the index
// in the Projects array.
func (p *MaterialsProjects) find(projectName string) (Project, int) {
	for index, project := range p.projects {
		if project.Name == projectName {
			return project, index
		}
	}

	return Project{}, -1
}
<commit_msg>Create projects file if it doesn't exist.<commit_after>package materials

import (
	"bufio"
	"errors"
	"fmt"
	"os"
	"os/user"
	"path/filepath"
	"strings"
)

// Project describes the information we track about a users
// projects. Here we keep the name of the project and the
// directory path. The name of the project is the top level
// directory of the project. The path is the full path to
// the project excluding the name.
type Project struct {
	Name   string `json:"name" xml:"name"`
	Path   string `json:"path" xml:"path"`
	Status string `json:"status" xml:"status"`
}

// MaterialsProjects contains a list of user projects and information that
// is needed by the methods to load the projects file.
type MaterialsProjects struct {
	dir      string
	projects []Project
}

// CurrentUserProjects retrieves the projects for the currently logged in user.
func CurrentUserProjects() (*MaterialsProjects, error) {
	u, err := user.Current()
	if err != nil {
		return nil, err
	}
	return loadProjectsFrom(u.HomeDir)
}

// ProjectsForUser retrieves the projects for the named user.
func ProjectsForUser(username string) (*MaterialsProjects, error) {
	u, err := user.Lookup(username)
	if err != nil {
		return nil, err
	}
	return loadProjectsFrom(u.HomeDir)
}

// ProjectsFromHome retrieves the projects located in the directory
// pointed at by the HOME environment variable.
func ProjectsFromHome() (*MaterialsProjects, error) {
	home := os.Getenv("HOME")
	if home == "" {
		return nil, errors.New("HOME Environment variable is undefined")
	}
	return loadProjectsFrom(home)
}

// ProjectsFrom retrieves the projects from the specified directory. The
// path given cannot contain the .materials subdirectory. A .materials
// subdirectory must exist in the path.
func ProjectsFrom(dir string) (*MaterialsProjects, error) {
	return loadProjectsFrom(dir)
}

// Load projects using the specified directory as a base.
// The projects file is located in {dir}/.materials/projects.
func loadProjectsFrom(dir string) (*MaterialsProjects, error) {
	userProjects := MaterialsProjects{dir: dir}
	err := userProjects.loadProjects()
	if err != nil {
		return nil, err
	}
	return &userProjects, err
}

// Reload re-reads and loads the projects file.
func (p *MaterialsProjects) Reload() error {
	return p.loadProjects()
}

// Reads the projects file, parses its contents and loads it into
// the Projects struct.
func (p *MaterialsProjects) loadProjects() error {
	projectsFile, err := os.Open(p.projectsFile())
	if err != nil {
		return p.createEmptyProjectsFile()
	}
	defer projectsFile.Close()

	projects := []Project{}
	scanner := bufio.NewScanner(projectsFile)
	for scanner.Scan() {
		splitLine := strings.Split(scanner.Text(), "|")
		if len(splitLine) == 3 {
			projects = append(projects, Project{
				Name:   strings.TrimSpace(splitLine[0]),
				Path:   strings.TrimSpace(splitLine[1]),
				Status: strings.TrimSpace(splitLine[2]),
			})
		}
	}
	p.projects = projects
	return nil
}

// Creates the path to the projects file:
// {dir}/.materias/projects
func (p *MaterialsProjects) projectsFile() string {
	return filepath.Join(p.dir, ".materials", "projects")
}

// Attempts to create an empty projects file.
func (p *MaterialsProjects) createEmptyProjectsFile() error {
	file, err := os.Create(p.projectsFile())
	if err != nil {
		return err
	}
	defer file.Close()
	return nil
}

// Projects returns the list of loaded projects.
func (p *MaterialsProjects) Projects() []Project {
	return p.projects
}

// Add adds a new project and updates the projects file.
func (p *MaterialsProjects) Add(proj Project) error {
	if p.Exists(proj.Name) {
		return errors.New(fmt.Sprintf("Project already exists: %s", proj.Name))
	}
	p.projects = append(p.projects, proj)
	return p.writeToProjectsFile(p.projects)
}

// Remove removes a project and updates the projects file.
func (p *MaterialsProjects) Remove(projectName string) error {
	projects, projectsUpdated := p.projectsExceptFor(projectName)

	// We found the entry to remove. Thus we need to
	// update the projects file.
	if projectsUpdated {
		err := p.writeToProjectsFile(projects)
		if err == nil {
			p.projects = projects
		}
		return err
	} else {
		// project wasn't found so we don't update anything and we return no error
		return nil
	}
}

func (p *MaterialsProjects) Update(proj Project) error {
	_, index := p.find(proj.Name)
	if index != -1 {
		projects := p.Projects()
		projects[index].Path = proj.Path
		projects[index].Status = proj.Status
		return p.writeToProjectsFile(p.Projects())
	}

	return errors.New(fmt.Sprintf("Project not found: %s", proj.Name))
}

// projectsExceptFor returns a new list of projects except for the project
// matching projectName. It returns true if it found a project matching
// projectName.
func (p *MaterialsProjects) projectsExceptFor(projectName string) ([]Project, bool) {
	projects := []Project{}
	found := false
	for _, project := range p.projects {
		if project.Name != projectName {
			projects = append(projects, project)
		} else {
			found = true
		}
	}
	return projects, found
}

// writeToProjectsFile overwrites the projects file with the list
// of projects.
func (p *MaterialsProjects) writeToProjectsFile(projects []Project) error {
	file, err := os.Create(p.projectsFile())
	if err != nil {
		return err
	}
	defer file.Close()
	for _, project := range projects {
		projectLine := fmt.Sprintf("%s|%s|%s\n",
			strings.TrimSpace(project.Name),
			strings.TrimSpace(project.Path),
			strings.TrimSpace(project.Status))
		file.WriteString(projectLine)
	}
	return nil
}

// Exists returns true if there is a project matching
// the given Name.
func (p *MaterialsProjects) Exists(projectName string) bool {
	_, found := p.Find(projectName)
	return found
}

// Find returns (Project, true) if the project is found otherwise
// it returns (Project{}, false)
func (p *MaterialsProjects) Find(projectName string) (Project, bool) {
	project, index := p.find(projectName)
	return project, index != -1
}

// find returns (Project, index) where index is -1 if
// the project wasn't found, otherwise it is the index
// in the Projects array.
func (p *MaterialsProjects) find(projectName string) (Project, int) {
	for index, project := range p.projects {
		if project.Name == projectName {
			return project, index
		}
	}

	return Project{}, -1
}
<|endoftext|>


<commit_before>package logging

import (
	"strings"
)

type kafkaConfigurationMessage struct {
	CallerInfo
	hook    *kafkaSink
	brokers []string
	topic   string
}

func reportKafkaConfig(hook *kafkaSink, cfg Config, ls LogSink) {
	msg := kafkaConfigurationMessage{
		CallerInfo: GetCallerInfo(),
		hook:       hook,
		brokers:    cfg.getBrokers(),
		topic:      cfg.Kafka.Topic,
	}
	msg.ExcludeMe()

	if hook == nil {
		Deliver(ls,
			SousKafkaConfigV1,
			WarningLevel,
			GetCallerInfo(NotHere()),
			ConsoleAndMessage("Not connecting to Kafka."),
			KV(SousSuccessfulConnection, false),
		)
		return
	}

	Deliver(ls,
		SousKafkaConfigV1,
		InformationLevel,
		GetCallerInfo(NotHere()),
		MessageField("Connecting to Kafka"),
		KV(SousSuccessfulConnection, true),
		KV(KafkaLoggingTopic, cfg.Kafka.Topic),
		KV(KafkaBrokers, strings.Join(cfg.getBrokers(), ",")),
		KV(KafkaLoggerId, hook.ID()),
		KV(KafkaLoggingLevels, hook.level.String()),
	)
}
<commit_msg>Console output about connecting to kafka<commit_after>package logging

import (
	"strings"
)

type kafkaConfigurationMessage struct {
	CallerInfo
	hook    *kafkaSink
	brokers []string
	topic   string
}

func reportKafkaConfig(hook *kafkaSink, cfg Config, ls LogSink) {
	msg := kafkaConfigurationMessage{
		CallerInfo: GetCallerInfo(),
		hook:       hook,
		brokers:    cfg.getBrokers(),
		topic:      cfg.Kafka.Topic,
	}
	msg.ExcludeMe()

	if hook == nil {
		Deliver(ls,
			SousKafkaConfigV1,
			WarningLevel,
			GetCallerInfo(NotHere()),
			ConsoleAndMessage("Not connecting to Kafka."),
			KV(SousSuccessfulConnection, false),
		)
		return
	}

	Deliver(ls,
		SousKafkaConfigV1,
		InformationLevel,
		GetCallerInfo(NotHere()),
		ConsoleAndMessage("Connecting to Kafka"),
		KV(SousSuccessfulConnection, true),
		KV(KafkaLoggingTopic, cfg.Kafka.Topic),
		KV(KafkaBrokers, strings.Join(cfg.getBrokers(), ",")),
		KV(KafkaLoggerId, hook.ID()),
		KV(KafkaLoggingLevels, hook.level.String()),
	)
}
<|endoftext|>


<commit_before>/*

   Copyright 2013 Niklas Voss

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

      http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.

*/

package golem

import (
	"encoding/json"
	"errors"
	"strings"
)

var (
	initialProtocol Protocol = Protocol(&DefaultJSONProtocol{})
)

// Protocol-interface provides the required methods necessary for any
// protocol, that should be used with golem, to implement.
// The evented system of golem needs several steps to process incoming data:
//  1. Unpack
//  2. Unmarshal
// For emitting data the process is reversed:
//  1. Marshal
//  2. Pack
type Protocol interface {
	// Unpack splits/extracts event name from incoming data.
	Unpack([]byte) (string, []byte, error)
	// Unmarshals leftover data into associated type of callback.
	Unmarshal([]byte, interface{}) error
	// Marshal data into byte array
	Marshal(interface{}) ([]byte, error)
	// Pack event name into byte array.
	Pack(string, []byte) ([]byte, error)
}

// SetInitialProtocol sets the initial protocol for router creation. Every router
// created after changing the initial protocol will use the new protocol by default.
func SetInitialProtocol(protocol Protocol) {
	initialProtocol = protocol
}

const (
	protocolSeperator = " "
)

// DefaultJSONProtocol is the initial protocol used by golem. It implements the
// Protocol-Interface.
// (Note: there is an article about this simple protocol in golem's wiki)
type DefaultJSONProtocol struct{}

// Unpack splits the event name from the incoming message.
func (_ *DefaultJSONProtocol) Unpack(data []byte) (string, []byte, error) {
	result := strings.SplitN(string(data), protocolSeperator, 2)
	if len(result) != 2 {
		return "", nil, errors.New("Unable to extract event name from data.")
	}
	return result[0], []byte(result[1]), nil
}

// Unmarshals data into requested structure. If not successful the function return an error.
func (_ *DefaultJSONProtocol) Unmarshal(data []byte, structPtr interface{}) error {
	return json.Unmarshal(data, structPtr)
}

// Marshals structure into JSON. If not successful second return value is an error.
func (_ *DefaultJSONProtocol) Marshal(structPtr interface{}) ([]byte, error) {
	return json.Marshal(structPtr)
}

// Adds the event name to the message.
func (_ *DefaultJSONProtocol) Pack(name string, data []byte) ([]byte, error) {
	result := []byte(name + protocolSeperator)
	result = append(result, data...)
	return result, nil
}
<commit_msg>Documentation improved.<commit_after>/*

   Copyright 2013 Niklas Voss

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

      http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.

*/

package golem

import (
	"encoding/json"
	"errors"
	"strings"
)

const (
	protocolSeperator = " "
)

var (
	initialProtocol Protocol = Protocol(&DefaultJSONProtocol{})
)

// Protocol-interface provides the required methods necessary for any
// protocol, that should be used with golem, to implement.
// The evented system of golem needs several steps to process incoming data:
//  1. Unpack
//  2. Unmarshal
// For emitting data the process is reversed:
//  1. Marshal
//  2. Pack
type Protocol interface {
	// Unpack splits/extracts event name from incoming data.
	Unpack([]byte) (string, []byte, error)
	// Unmarshals leftover data into associated type of callback.
	Unmarshal([]byte, interface{}) error
	// Marshal data into byte array
	Marshal(interface{}) ([]byte, error)
	// Pack event name into byte array.
	Pack(string, []byte) ([]byte, error)
}

// SetInitialProtocol sets the initial protocol for router creation. Every router
// created after changing the initial protocol will use the new protocol by default.
func SetInitialProtocol(protocol Protocol) {
	initialProtocol = protocol
}

// DefaultJSONProtocol is the initial protocol used by golem. It implements the
// Protocol-Interface.
// (Note: there is an article about this simple protocol in golem's wiki)
type DefaultJSONProtocol struct{}

// Unpack splits the event name from the incoming message.
func (_ *DefaultJSONProtocol) Unpack(data []byte) (string, []byte, error) {
	result := strings.SplitN(string(data), protocolSeperator, 2)
	if len(result) != 2 {
		return "", nil, errors.New("Unable to extract event name from data.")
	}
	return result[0], []byte(result[1]), nil
}

// Unmarshals data into requested structure. If not successful the function return an error.
func (_ *DefaultJSONProtocol) Unmarshal(data []byte, structPtr interface{}) error {
	return json.Unmarshal(data, structPtr)
}

// Marshals structure into JSON. If not successful second return value is an error.
func (_ *DefaultJSONProtocol) Marshal(structPtr interface{}) ([]byte, error) {
	return json.Marshal(structPtr)
}

// Adds the event name to the message and returns it.
func (_ *DefaultJSONProtocol) Pack(name string, data []byte) ([]byte, error) {
	result := []byte(name + protocolSeperator)
	result = append(result, data...)
	return result, nil
}
<|endoftext|>


<commit_before>package fabric

import (
	"context"
)

// Protocol deals with both sides, server and client
type Protocol interface {
	Handle(HandlerFunc) HandlerFunc
	Negotiate(NegotiatorFunc) NegotiatorFunc
	Name() string
}

// HandlerFunc for Handle
type HandlerFunc func(ctx context.Context, c Conn) error

// NegotiatorFunc for Negotiate
type NegotiatorFunc func(ctx context.Context, c Conn) error

// HandlerChain -
func HandlerChain(fns ...Protocol) HandlerFunc {
	if len(fns) == 0 {
		return nil
	}
	return fns[0].Handle(HandlerChain(fns[1:cap(fns)]...))
}

// NegotiatorChain -
func NegotiatorChain(fns ...Protocol) NegotiatorFunc {
	if len(fns) == 0 {
		return nil
	}
	return fns[0].Negotiate(NegotiatorChain(fns[1:cap(fns)]...))
}
<commit_msg>Fix protocol bug<commit_after>package fabric

import (
	"context"
)

// Protocol deals with both sides, server and client
type Protocol interface {
	Handle(HandlerFunc) HandlerFunc
	Negotiate(NegotiatorFunc) NegotiatorFunc
	Name() string
}

// HandlerFunc for Handle
type HandlerFunc func(ctx context.Context, c Conn) error

// NegotiatorFunc for Negotiate
type NegotiatorFunc func(ctx context.Context, c Conn) error

// HandlerChain -
func HandlerChain(fns ...Protocol) HandlerFunc {
	if len(fns) == 0 {
		return nil
	}
	for i, fn := range fns {
		if fn != nil {
			return fn.Handle(HandlerChain(fns[i+1 : cap(fns)]...))
		}
	}
	return nil
}

// NegotiatorChain -
func NegotiatorChain(fns ...Protocol) NegotiatorFunc {
	if len(fns) == 0 {
		return nil
	}
	for i, fn := range fns {
		if fn != nil {
			return fn.Negotiate(NegotiatorChain(fns[i+1 : cap(fns)]...))
		}
	}
	return nil
}
<|endoftext|>