<commit_before>// Middleware for keeping track of users, login states and permissions.
package permissions

import (
	"net/http"
	"strings"
)

// The structure that keeps track of the permissions for various path prefixes
type Permissions struct {
	state              *UserState
	adminPathPrefixes  []string
	userPathPrefixes   []string
	publicPathPrefixes []string
	rootIsPublic       bool
	denied             http.HandlerFunc
}

const (
	// Version number. Stable API within major version numbers.
	Version = 2.0
)

// Initialize a Permissions struct with all the default settings.
// This will also connect to the redis host at localhost:6379.
func New() *Permissions {
	return NewPermissions(NewUserStateSimple())
}

// Initialize a Permissions struct with Redis DB index and host:port
func NewWithRedisConf(dbindex int, hostPort string) *Permissions {
	return NewPermissions(NewUserState(dbindex, true, hostPort))
}

// Initialize a Permissions struct with the given UserState and
// a few default paths for admin/user/public path prefixes.
func NewPermissions(state *UserState) *Permissions {
	// default permissions
	return &Permissions{state,
		[]string{"/admin"},         // admin path prefixes
		[]string{"/repo", "/data"}, // user path prefixes
		[]string{"/", "/login", "/register", "/favicon.ico", "/style", "/img", "/js",
			"/favicon.ico", "/robots.txt", "/sitemap_index.xml"}, // public
		true,
		PermissionDenied}
}

// Specify the http.HandlerFunc for when the permissions are denied
func (perm *Permissions) SetDenyFunction(f http.HandlerFunc) {
	perm.denied = f
}

// Get the current http.HandlerFunc for when permissions are denied
func (perm *Permissions) DenyFunction() http.HandlerFunc {
	return perm.denied
}

// Retrieve the UserState struct
func (perm *Permissions) UserState() *UserState {
	return perm.state
}

// Set everything to public
func (perm *Permissions) Clear() {
	perm.adminPathPrefixes = []string{}
	perm.userPathPrefixes = []string{}
}

// Add an url path prefix that is a page for the logged in administrators
func (perm *Permissions) AddAdminPath(prefix string) {
	perm.adminPathPrefixes = append(perm.adminPathPrefixes, prefix)
}

// Add an url path prefix that is a page for the logged in users
func (perm *Permissions) AddUserPath(prefix string) {
	perm.userPathPrefixes = append(perm.userPathPrefixes, prefix)
}

// Add an url path prefix that is a public page
func (perm *Permissions) AddPublicPath(prefix string) {
	perm.publicPathPrefixes = append(perm.publicPathPrefixes, prefix)
}

// Set all url path prefixes that are for the logged in administrator pages
func (perm *Permissions) SetAdminPath(pathPrefixes []string) {
	perm.adminPathPrefixes = pathPrefixes
}

// Set all url path prefixes that are for the logged in user pages
func (perm *Permissions) SetUserPath(pathPrefixes []string) {
	perm.userPathPrefixes = pathPrefixes
}

// Set all url path prefixes that are for the public pages
func (perm *Permissions) SetPublicPath(pathPrefixes []string) {
	perm.publicPathPrefixes = pathPrefixes
}

// The default "permission denied" http handler.
func PermissionDenied(w http.ResponseWriter, req *http.Request) {
	http.Error(w, "Permission denied.", http.StatusForbidden)
}

// Check if a given request should be rejected.
func (perm *Permissions) Rejected(w http.ResponseWriter, req *http.Request) bool {
	reject := false
	path := req.URL.Path // the path of the url that the user wish to visit

	// If it's not "/" and set to be public regardless of permissions
	if !(perm.rootIsPublic && path == "/") {

		// Reject if it is an admin page and user does not have admin permissions
		for _, prefix := range perm.adminPathPrefixes {
			if strings.HasPrefix(path, prefix) {
				if !perm.state.AdminRights(req) {
					reject = true
					break
				}
			}
		}

		if !reject {
			// Reject if it's a user page and the user does not have user rights
			for _, prefix := range perm.userPathPrefixes {
				if strings.HasPrefix(path, prefix) {
					if !perm.state.UserRights(req) {
						reject = true
						break
					}
				}
			}
		}

		if !reject {
			// Reject if it's not a public page
			found := false
			for _, prefix := range perm.publicPathPrefixes {
				if strings.HasPrefix(path, prefix) {
					found = true
					break
				}
			}
			if !found {
				reject = true
			}
		}

	}

	return reject
}

// Middleware handler (compatible with Negroni)
func (perm *Permissions) ServeHTTP(w http.ResponseWriter, req *http.Request, next http.HandlerFunc) {
	// Check if the user has the right admin/user rights
	if perm.Rejected(w, req) {
		// Get and call the Permission Denied function
		perm.DenyFunction()(w, req)
		// Reject the request by not calling the next handler below
		return
	}

	// Call the next middleware handler
	next(w, req)
}
<commit_msg>Updated a comment, as suggested by golint<commit_after>// Package permissions provides a way to keep track of users, login states and permissions.
package permissions

import (
	"net/http"
	"strings"
)

// The structure that keeps track of the permissions for various path prefixes
type Permissions struct {
	state              *UserState
	adminPathPrefixes  []string
	userPathPrefixes   []string
	publicPathPrefixes []string
	rootIsPublic       bool
	denied             http.HandlerFunc
}

const (
	// Version number. Stable API within major version numbers.
	Version = 2.0
)

// Initialize a Permissions struct with all the default settings.
// This will also connect to the redis host at localhost:6379.
func New() *Permissions {
	return NewPermissions(NewUserStateSimple())
}

// Initialize a Permissions struct with Redis DB index and host:port
func NewWithRedisConf(dbindex int, hostPort string) *Permissions {
	return NewPermissions(NewUserState(dbindex, true, hostPort))
}

// Initialize a Permissions struct with the given UserState and
// a few default paths for admin/user/public path prefixes.
func NewPermissions(state *UserState) *Permissions {
	// default permissions
	return &Permissions{state,
		[]string{"/admin"},         // admin path prefixes
		[]string{"/repo", "/data"}, // user path prefixes
		[]string{"/", "/login", "/register", "/favicon.ico", "/style", "/img", "/js",
			"/favicon.ico", "/robots.txt", "/sitemap_index.xml"}, // public
		true,
		PermissionDenied}
}

// Specify the http.HandlerFunc for when the permissions are denied
func (perm *Permissions) SetDenyFunction(f http.HandlerFunc) {
	perm.denied = f
}

// Get the current http.HandlerFunc for when permissions are denied
func (perm *Permissions) DenyFunction() http.HandlerFunc {
	return perm.denied
}

// Retrieve the UserState struct
func (perm *Permissions) UserState() *UserState {
	return perm.state
}

// Set everything to public
func (perm *Permissions) Clear() {
	perm.adminPathPrefixes = []string{}
	perm.userPathPrefixes = []string{}
}

// Add an url path prefix that is a page for the logged in administrators
func (perm *Permissions) AddAdminPath(prefix string) {
	perm.adminPathPrefixes = append(perm.adminPathPrefixes, prefix)
}

// Add an url path prefix that is a page for the logged in users
func (perm *Permissions) AddUserPath(prefix string) {
	perm.userPathPrefixes = append(perm.userPathPrefixes, prefix)
}

// Add an url path prefix that is a public page
func (perm *Permissions) AddPublicPath(prefix string) {
	perm.publicPathPrefixes = append(perm.publicPathPrefixes, prefix)
}

// Set all url path prefixes that are for the logged in administrator pages
func (perm *Permissions) SetAdminPath(pathPrefixes []string) {
	perm.adminPathPrefixes = pathPrefixes
}

// Set all url path prefixes that are for the logged in user pages
func (perm *Permissions) SetUserPath(pathPrefixes []string) {
	perm.userPathPrefixes = pathPrefixes
}

// Set all url path prefixes that are for the public pages
func (perm *Permissions) SetPublicPath(pathPrefixes []string) {
	perm.publicPathPrefixes = pathPrefixes
}

// The default "permission denied" http handler.
func PermissionDenied(w http.ResponseWriter, req *http.Request) {
	http.Error(w, "Permission denied.", http.StatusForbidden)
}

// Check if a given request should be rejected.
func (perm *Permissions) Rejected(w http.ResponseWriter, req *http.Request) bool {
	reject := false
	path := req.URL.Path // the path of the url that the user wish to visit

	// If it's not "/" and set to be public regardless of permissions
	if !(perm.rootIsPublic && path == "/") {

		// Reject if it is an admin page and user does not have admin permissions
		for _, prefix := range perm.adminPathPrefixes {
			if strings.HasPrefix(path, prefix) {
				if !perm.state.AdminRights(req) {
					reject = true
					break
				}
			}
		}

		if !reject {
			// Reject if it's a user page and the user does not have user rights
			for _, prefix := range perm.userPathPrefixes {
				if strings.HasPrefix(path, prefix) {
					if !perm.state.UserRights(req) {
						reject = true
						break
					}
				}
			}
		}

		if !reject {
			// Reject if it's not a public page
			found := false
			for _, prefix := range perm.publicPathPrefixes {
				if strings.HasPrefix(path, prefix) {
					found = true
					break
				}
			}
			if !found {
				reject = true
			}
		}

	}

	return reject
}

// Middleware handler (compatible with Negroni)
func (perm *Permissions) ServeHTTP(w http.ResponseWriter, req *http.Request, next http.HandlerFunc) {
	// Check if the user has the right admin/user rights
	if perm.Rejected(w, req) {
		// Get and call the Permission Denied function
		perm.DenyFunction()(w, req)
		// Reject the request by not calling the next handler below
		return
	}

	// Call the next middleware handler
	next(w, req)
}
<|endoftext|>


<commit_before>package nstcc

import (
	"reflect"
	"testing"
)

func TestParser(t *testing.T) {
	// Map identifier strings like "main" to (negative) placeholder token
	// values.
	t2s := map[token]string{}
	s2t := map[string]token{}
	ident := func(s string) token {
		x, ok := s2t[s]
		if !ok {
			x = token(^len(s2t))
			s2t[s] = x
			t2s[x] = s
		}
		return x
	}

	testCases := []struct {
		desc string
		src  string
		want []token
	}{{
		"hello world",
		// TODO: s/XXX/42/
		"int main(int argc, char** argv) { return XXX; }",
		[]token{
			tokInt,
			ident("main"),
			'(',
			tokInt,
			ident("argc"),
			',',
			tokChar,
			'*',
			'*',
			ident("argv"),
			')',
			'{',
			tokReturn,
			ident("XXX"),
			';',
			'}',
		},
	}, {
		"slash star",
		"int /*foo*/ x;",
		[]token{
			tokInt,
			ident("x"),
			';',
		},
	}, {
		"slash slash",
		"int x;\n// int y;\nint z;\n",
		[]token{
			tokInt,
			ident("x"),
			';',
			tokInt,
			ident("z"),
			';',
		},
	}}

	for _, tc := range testCases {
		c := newCompiler(nil, nil, []byte(tc.src))
		var got []token
		for {
			if err := c.next(); err != nil {
				t.Fatal(err)
			}
			if c.tok == tokEOF {
				break
			}
			got = append(got, c.tok)
		}

		// Replace the placeholder identifier tokens with their real values.
		for i, x := range tc.want {
			if x >= 0 {
				continue
			}
			ts, err := c.idents.byStr([]byte(t2s[x]))
			if err != nil {
				t.Fatal(err)
			}
			tc.want[i] = ts.tok
		}

		if !reflect.DeepEqual(got, tc.want) {
			t.Fatalf("parsing %q:\ngot  %v\nwant %v", tc.desc, got, tc.want)
		}
	}
}
<commit_msg>Make TestParser allow arbitrary 'want' token types.<commit_after>package nstcc

import (
	"reflect"
	"testing"
)

func TestParser(t *testing.T) {
	testCases := []struct {
		desc string
		src  string
		want []interface{}
	}{{
		"hello world",
		// TODO: s/XXX/42/
		"int main(int argc, char** argv) { return XXX; }",
		[]interface{}{
			tokInt,
			"main",
			'(',
			tokInt,
			"argc",
			',',
			tokChar,
			'*',
			'*',
			"argv",
			')',
			'{',
			tokReturn,
			"XXX",
			';',
			'}',
		},
	}, {
		"slash star",
		"int /*foo*/ x;",
		[]interface{}{
			tokInt,
			"x",
			';',
		},
	}, {
		"slash slash",
		"int x;\n// int y;\nint z;\n",
		[]interface{}{
			tokInt,
			"x",
			';',
			tokInt,
			"z",
			';',
		},
	}}

	for _, tc := range testCases {
		c := newCompiler(nil, nil, []byte(tc.src))
		var got []token
		for {
			if err := c.next(); err != nil {
				t.Fatal(err)
			}
			if c.tok == tokEOF {
				break
			}
			got = append(got, c.tok)
		}

		// Replace the placeholder tokens with their real values.
		want := make([]token, len(tc.want))
		for i, x := range tc.want {
			switch x := x.(type) {
			case rune:
				want[i] = token(x)
			case string:
				ts, err := c.idents.byStr([]byte(x))
				if err != nil {
					t.Fatal(err)
				}
				want[i] = ts.tok
			case token:
				want[i] = x
			default:
				t.Fatalf("invalid type %T", x)
			}
		}

		if !reflect.DeepEqual(got, want) {
			t.Errorf("parsing %q:\ngot  %v\nwant %v", tc.desc, got, want)
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"io/ioutil"
	"os"
	"strings"

	"github.com/mattn/go-isatty"
	"github.com/mozillazg/go-pinyin"
)

func main() {
	heteronym := flag.Bool("e", false, "启用多音字模式")
	style := flag.String("s", "Tone", "指定拼音风格。可选值：Normal, Tone, Tone2, Tone3, Initials, FirstLetter, Finals, FinalsTone, FinalsTone2, FinalsTone3")
	flag.Parse()
	hans := flag.Args()
	stdin := []byte{}
	if !isatty.IsTerminal(os.Stdin.Fd()) {
		stdin, _ = ioutil.ReadAll(os.Stdin)
	}
	if len(stdin) > 0 {
		hans = append(hans, string(stdin))
	}

	if len(hans) == 0 {
		fmt.Println("请至少输入一个汉字: pinyin [-e] [-s STYLE] HANS [HANS ...]")
		os.Exit(1)
	}

	args := pinyin.NewArgs()
	if *heteronym {
		args.Heteronym = true
	}
	switch *style {
	case "Normal":
		args.Style = pinyin.Normal
	case "Tone2":
		args.Style = pinyin.Tone2
	case "Tone3":
		args.Style = pinyin.Tone3
	case "Initials":
		args.Style = pinyin.Initials
	case "FirstLetter":
		args.Style = pinyin.FirstLetter
	case "Finals":
		args.Style = pinyin.Finals
	case "FinalsTone":
		args.Style = pinyin.FinalsTone
	case "FinalsTone2":
		args.Style = pinyin.FinalsTone2
	case "FinalsTone3":
		args.Style = pinyin.FinalsTone3
	default:
		args.Style = pinyin.Tone
	}

	pys := pinyin.Pinyin(strings.Join(hans, ""), args)
	for _, s := range pys {
		fmt.Print(strings.Join(s, ","), " ")
	}
	if len(pys) > 0 {
		fmt.Println()
	}
}
<commit_msg>Use example-based command-line option for pinyin style.<commit_after>package main

import (
	"flag"
	"fmt"
	"io/ioutil"
	"os"
	"strings"

	"github.com/mattn/go-isatty"
	"github.com/mozillazg/go-pinyin"
)

func main() {
	heteronym := flag.Bool("e", false, "启用多音字模式")
	style := flag.String("s", "zh4ao", "指定拼音风格。可选值：zhao, zh4ao, zha4o, zhao4, zh, z, ao, 4ao, a4o, ao4")
	flag.Parse()
	hans := flag.Args()
	stdin := []byte{}
	if !isatty.IsTerminal(os.Stdin.Fd()) {
		stdin, _ = ioutil.ReadAll(os.Stdin)
	}
	if len(stdin) > 0 {
		hans = append(hans, string(stdin))
	}

	if len(hans) == 0 {
		fmt.Println("请至少输入一个汉字: pinyin [-e] [-s STYLE] HANS [HANS ...]")
		os.Exit(1)
	}

	args := pinyin.NewArgs()
	if *heteronym {
		args.Heteronym = true
	}
	switch *style {
	case "zhao":
		args.Style = pinyin.Normal
	case "zha4o":
		args.Style = pinyin.Tone2
	case "zhao4":
		args.Style = pinyin.Tone3
	case "zh":
		args.Style = pinyin.Initials
	case "z":
		args.Style = pinyin.FirstLetter
	case "ao":
		args.Style = pinyin.Finals
	case "4ao":
		args.Style = pinyin.FinalsTone
	case "a4o":
		args.Style = pinyin.FinalsTone2
	case "ao4":
		args.Style = pinyin.FinalsTone3
	default:
		args.Style = pinyin.Tone
	}

	pys := pinyin.Pinyin(strings.Join(hans, ""), args)
	for _, s := range pys {
		fmt.Print(strings.Join(s, ","), " ")
	}
	if len(pys) > 0 {
		fmt.Println()
	}
}
<|endoftext|>


<commit_before>package qt_recipe

import (
	"encoding/csv"
	rice "github.com/GeertJohan/go.rice"
	"github.com/pkg/profile"
	"github.com/tidwall/gjson"
	"github.com/watermint/toolbox/infra/api/api_auth"
	"github.com/watermint/toolbox/infra/api/api_auth_impl"
	"github.com/watermint/toolbox/infra/control/app_control"
	"github.com/watermint/toolbox/infra/control/app_control_impl"
	"github.com/watermint/toolbox/infra/control/app_root"
	"github.com/watermint/toolbox/infra/control/app_run_impl"
	"github.com/watermint/toolbox/infra/network/nw_ratelimit"
	"github.com/watermint/toolbox/infra/recpie/app_conn"
	"github.com/watermint/toolbox/infra/recpie/app_conn_impl"
	"github.com/watermint/toolbox/infra/recpie/app_recipe"
	"github.com/watermint/toolbox/infra/recpie/app_vo"
	"github.com/watermint/toolbox/infra/recpie/app_vo_impl"
	"github.com/watermint/toolbox/infra/ui/app_msg_container"
	"github.com/watermint/toolbox/infra/ui/app_ui"
	"github.com/watermint/toolbox/infra/util/ut_memory"
	"github.com/watermint/toolbox/quality/infra/qt_control_impl"
	"go.uber.org/zap"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"
	"runtime"
	"testing"
)

const (
	EndToEndPeer       = "end_to_end_test"
	TestTeamFolderName = "watermint-toolbox-test"
)

// Returns conn if v & end to end peer exist. found = true when v is an interface of app_conn.ConnDropboxAPI
func ApplyConn(v interface{}, c app_control.Control) (conn app_conn.ConnDropboxApi, found bool) {
	l := c.Log()
	a := api_auth_impl.NewCached(c, api_auth_impl.PeerName(EndToEndPeer))

	if _, ok := v.(app_conn.ConnBusinessInfo); ok {
		if _, err := a.Auth(api_auth.DropboxTokenBusinessInfo); err != nil {
			l.Info("BusinessInfo: Skip end to end test")
			return nil, true
		}
		return &app_conn_impl.ConnBusinessInfo{
			PeerName: EndToEndPeer,
		}, true
	}

	if _, ok := v.(app_conn.ConnBusinessFile); ok {
		if _, err := a.Auth(api_auth.DropboxTokenBusinessFile); err != nil {
			l.Info("BusinessFile: Skip end to end test")
			return nil, true
		}
		return &app_conn_impl.ConnBusinessFile{
			PeerName: EndToEndPeer,
		}, true
	}
	if _, ok := v.(app_conn.ConnBusinessAudit); ok {
		if _, err := a.Auth(api_auth.DropboxTokenBusinessAudit); err != nil {
			l.Info("BusinessAudit: Skip end to end test")
			return nil, true
		}
		return &app_conn_impl.ConnBusinessAudit{
			PeerName: EndToEndPeer,
		}, true
	}
	if _, ok := v.(app_conn.ConnBusinessMgmt); ok {
		if _, err := a.Auth(api_auth.DropboxTokenBusinessManagement); err != nil {
			l.Info("BusinessManagement: Skip end to end test")
			return nil, true
		}
		return &app_conn_impl.ConnBusinessMgmt{
			PeerName: EndToEndPeer,
		}, true
	}
	if _, ok := v.(app_conn.ConnUserFile); ok {
		if _, err := a.Auth(api_auth.DropboxTokenFull); err != nil {
			l.Info("UserFull: Skip end to end test")
			return nil, true
		}
		return &app_conn_impl.ConnUserFile{
			PeerName: EndToEndPeer,
		}, true
	}

	return nil, false
}

func ApplyTestPeers(ctl app_control.Control, vo app_vo.ValueObject) bool {
	l := ctl.Log()
	l.Debug("Prepare for applying test peers")

	vc := app_vo_impl.NewValueContainer(vo)
	for k, v := range vc.Values {
		if conn, found := ApplyConn(v, ctl); found {
			l.Debug("Conn found for key", zap.String("k", k), zap.Bool("connApplied", conn != nil))
			if conn == nil {
				return false
			} else {
				vc.Values[k] = conn
			}
		}
	}

	l.Debug("Applying for debug")
	vc.Apply(vo)

	return true
}

func Resources(t *testing.T) (bx, web *rice.Box, mc app_msg_container.Container, ui app_ui.UI) {
	bx = rice.MustFindBox("../../../resources")
	web = rice.MustFindBox("../../../web")

	mc = app_run_impl.NewContainer(bx)
	ui = app_ui.NewConsole(mc, qt_control_impl.NewMessageTest(t), true)
	return
}

func findTestResource() (resource gjson.Result, found bool) {
	l := app_root.Log()
	p, found := os.LookupEnv("TOOLBOX_TESTRESOURCE")
	if !found {
		return gjson.Parse("{}"), false
	}
	l = l.With(zap.String("path", p))
	b, err := ioutil.ReadFile(p)
	if err != nil {
		l.Debug("unable to read file", zap.Error(err))
		return gjson.Parse("{}"), false
	}
	if !gjson.ValidBytes(b) {
		l.Debug("invalid file content", zap.ByteString("resource", b))
		return gjson.Parse("{}"), false
	}
	return gjson.ParseBytes(b), true
}

func TestWithControl(t *testing.T, twc func(ctl app_control.Control)) {
	nw_ratelimit.SetTestMode(true)
	bx, web, mc, ui := Resources(t)

	ctl := app_control_impl.NewSingle(ui, bx, web, mc, false, make([]app_recipe.Recipe, 0))
	cs := ctl.(*app_control_impl.Single)
	if res, found := findTestResource(); found {
		var err error
		ctl, err = cs.NewTestControl(res)
		if err != nil {
			t.Error("Unable to create new test control", err)
			return
		}
	}
	err := ctl.Up(app_control.Test(), app_control.Concurrency(runtime.NumCPU()))
	if err != nil {
		os.Exit(app_control.FatalStartup)
	}
	defer ctl.Down()

	twc(ctl)
}

func TestRecipe(t *testing.T, re app_recipe.Recipe) {
	nw_ratelimit.SetTestMode(true)
	TestWithControl(t, func(ctl app_control.Control) {
		l := ctl.Log()
		l.Debug("Start testing")
		pr := profile.Start(
			profile.ProfilePath(ctl.Workspace().Log()),
			profile.MemProfile,
		)

		err := re.Test(ctl)

		pr.Stop()
		ut_memory.DumpStats(l)

		if err == nil {
			return
		}

		switch err.(type) {
		case *ErrorNoTestRequired:
			l.Info("Skip: No test required for this recipe")

		case *ErrorHumanInteractionRequired:
			l.Info("Skip: Human interaction required for this test")

		case *ErrorNotEnoughResource:
			l.Info("Skip: Not enough resource")

		case *ErrorScenarioTest:
			l.Info("Skip: Implemented as scenario test")

		case *ErrorImplementMe:
			l.Warn("Test is not implemented for this recipe")

		default:
			t.Error(err)
		}
	})
}

type RowTester func(cols map[string]string) error

func TestRows(ctl app_control.Control, reportName string, tester RowTester) error {
	l := ctl.Log().With(zap.String("reportName", reportName))
	csvFile := filepath.Join(ctl.Workspace().Report(), reportName+".csv")

	l.Debug("Start loading report", zap.String("csvFile", csvFile))

	cf, err := os.Open(csvFile)
	if err != nil {
		l.Warn("Unable to open report CSV", zap.Error(err))
		return err
	}
	defer cf.Close()
	csf := csv.NewReader(cf)
	var header []string
	isFirstLine := true

	for {
		cols, err := csf.Read()
		if err == io.EOF {
			break
		}
		if err != nil {
			l.Warn("An error occurred during read report file", zap.Error(err))
			return err
		}
		if isFirstLine {
			header = cols
			isFirstLine = false
		} else {
			colMap := make(map[string]string)
			for i, h := range header {
				colMap[h] = cols[i]
			}
			if err := tester(colMap); err != nil {
				l.Warn("Tester returned an error", zap.Error(err), zap.Any("cols", colMap))
				return err
			}
		}
	}

	return nil
}
<commit_msg>fix #222 : skip end to end test when the env variable set<commit_after>package qt_recipe

import (
	"encoding/csv"
	rice "github.com/GeertJohan/go.rice"
	"github.com/pkg/profile"
	"github.com/tidwall/gjson"
	"github.com/watermint/toolbox/infra/api/api_auth"
	"github.com/watermint/toolbox/infra/api/api_auth_impl"
	"github.com/watermint/toolbox/infra/control/app_control"
	"github.com/watermint/toolbox/infra/control/app_control_impl"
	"github.com/watermint/toolbox/infra/control/app_root"
	"github.com/watermint/toolbox/infra/control/app_run_impl"
	"github.com/watermint/toolbox/infra/network/nw_ratelimit"
	"github.com/watermint/toolbox/infra/recpie/app_conn"
	"github.com/watermint/toolbox/infra/recpie/app_conn_impl"
	"github.com/watermint/toolbox/infra/recpie/app_recipe"
	"github.com/watermint/toolbox/infra/recpie/app_vo"
	"github.com/watermint/toolbox/infra/recpie/app_vo_impl"
	"github.com/watermint/toolbox/infra/ui/app_msg_container"
	"github.com/watermint/toolbox/infra/ui/app_ui"
	"github.com/watermint/toolbox/infra/util/ut_memory"
	"github.com/watermint/toolbox/quality/infra/qt_control_impl"
	"go.uber.org/zap"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"
	"runtime"
	"strconv"
	"testing"
)

const (
	EndToEndPeer       = "end_to_end_test"
	TestTeamFolderName = "watermint-toolbox-test"
)

// Returns conn if v & end to end peer exist. found = true when v is an interface of app_conn.ConnDropboxAPI
func ApplyConn(v interface{}, c app_control.Control) (conn app_conn.ConnDropboxApi, found bool) {
	l := c.Log()
	a := api_auth_impl.NewCached(c, api_auth_impl.PeerName(EndToEndPeer))
	if p, found := os.LookupEnv("TOOLBOX_SKIPENDTOENDTEST"); found {
		if b, _ := strconv.ParseBool(p); b {
			l.Info("Skip end to end test")
			return nil, true
		}
	}

	if _, ok := v.(app_conn.ConnBusinessInfo); ok {
		if _, err := a.Auth(api_auth.DropboxTokenBusinessInfo); err != nil {
			l.Info("BusinessInfo: Skip end to end test")
			return nil, true
		}
		return &app_conn_impl.ConnBusinessInfo{
			PeerName: EndToEndPeer,
		}, true
	}

	if _, ok := v.(app_conn.ConnBusinessFile); ok {
		if _, err := a.Auth(api_auth.DropboxTokenBusinessFile); err != nil {
			l.Info("BusinessFile: Skip end to end test")
			return nil, true
		}
		return &app_conn_impl.ConnBusinessFile{
			PeerName: EndToEndPeer,
		}, true
	}
	if _, ok := v.(app_conn.ConnBusinessAudit); ok {
		if _, err := a.Auth(api_auth.DropboxTokenBusinessAudit); err != nil {
			l.Info("BusinessAudit: Skip end to end test")
			return nil, true
		}
		return &app_conn_impl.ConnBusinessAudit{
			PeerName: EndToEndPeer,
		}, true
	}
	if _, ok := v.(app_conn.ConnBusinessMgmt); ok {
		if _, err := a.Auth(api_auth.DropboxTokenBusinessManagement); err != nil {
			l.Info("BusinessManagement: Skip end to end test")
			return nil, true
		}
		return &app_conn_impl.ConnBusinessMgmt{
			PeerName: EndToEndPeer,
		}, true
	}
	if _, ok := v.(app_conn.ConnUserFile); ok {
		if _, err := a.Auth(api_auth.DropboxTokenFull); err != nil {
			l.Info("UserFull: Skip end to end test")
			return nil, true
		}
		return &app_conn_impl.ConnUserFile{
			PeerName: EndToEndPeer,
		}, true
	}

	return nil, false
}

func ApplyTestPeers(ctl app_control.Control, vo app_vo.ValueObject) bool {
	l := ctl.Log()
	l.Debug("Prepare for applying test peers")

	vc := app_vo_impl.NewValueContainer(vo)
	for k, v := range vc.Values {
		if conn, found := ApplyConn(v, ctl); found {
			l.Debug("Conn found for key", zap.String("k", k), zap.Bool("connApplied", conn != nil))
			if conn == nil {
				return false
			} else {
				vc.Values[k] = conn
			}
		}
	}

	l.Debug("Applying for debug")
	vc.Apply(vo)

	return true
}

func Resources(t *testing.T) (bx, web *rice.Box, mc app_msg_container.Container, ui app_ui.UI) {
	bx = rice.MustFindBox("../../../resources")
	web = rice.MustFindBox("../../../web")

	mc = app_run_impl.NewContainer(bx)
	ui = app_ui.NewConsole(mc, qt_control_impl.NewMessageTest(t), true)
	return
}

func findTestResource() (resource gjson.Result, found bool) {
	l := app_root.Log()
	p, found := os.LookupEnv("TOOLBOX_TESTRESOURCE")
	if !found {
		return gjson.Parse("{}"), false
	}
	l = l.With(zap.String("path", p))
	b, err := ioutil.ReadFile(p)
	if err != nil {
		l.Debug("unable to read file", zap.Error(err))
		return gjson.Parse("{}"), false
	}
	if !gjson.ValidBytes(b) {
		l.Debug("invalid file content", zap.ByteString("resource", b))
		return gjson.Parse("{}"), false
	}
	return gjson.ParseBytes(b), true
}

func TestWithControl(t *testing.T, twc func(ctl app_control.Control)) {
	nw_ratelimit.SetTestMode(true)
	bx, web, mc, ui := Resources(t)

	ctl := app_control_impl.NewSingle(ui, bx, web, mc, false, make([]app_recipe.Recipe, 0))
	cs := ctl.(*app_control_impl.Single)
	if res, found := findTestResource(); found {
		var err error
		ctl, err = cs.NewTestControl(res)
		if err != nil {
			t.Error("Unable to create new test control", err)
			return
		}
	}
	err := ctl.Up(app_control.Test(), app_control.Concurrency(runtime.NumCPU()))
	if err != nil {
		os.Exit(app_control.FatalStartup)
	}
	defer ctl.Down()

	twc(ctl)
}

func TestRecipe(t *testing.T, re app_recipe.Recipe) {
	nw_ratelimit.SetTestMode(true)
	TestWithControl(t, func(ctl app_control.Control) {
		l := ctl.Log()
		l.Debug("Start testing")
		pr := profile.Start(
			profile.ProfilePath(ctl.Workspace().Log()),
			profile.MemProfile,
		)

		err := re.Test(ctl)

		pr.Stop()
		ut_memory.DumpStats(l)

		if err == nil {
			return
		}

		switch err.(type) {
		case *ErrorNoTestRequired:
			l.Info("Skip: No test required for this recipe")

		case *ErrorHumanInteractionRequired:
			l.Info("Skip: Human interaction required for this test")

		case *ErrorNotEnoughResource:
			l.Info("Skip: Not enough resource")

		case *ErrorScenarioTest:
			l.Info("Skip: Implemented as scenario test")

		case *ErrorImplementMe:
			l.Warn("Test is not implemented for this recipe")

		default:
			t.Error(err)
		}
	})
}

type RowTester func(cols map[string]string) error

func TestRows(ctl app_control.Control, reportName string, tester RowTester) error {
	l := ctl.Log().With(zap.String("reportName", reportName))
	csvFile := filepath.Join(ctl.Workspace().Report(), reportName+".csv")

	l.Debug("Start loading report", zap.String("csvFile", csvFile))

	cf, err := os.Open(csvFile)
	if err != nil {
		l.Warn("Unable to open report CSV", zap.Error(err))
		return err
	}
	defer cf.Close()
	csf := csv.NewReader(cf)
	var header []string
	isFirstLine := true

	for {
		cols, err := csf.Read()
		if err == io.EOF {
			break
		}
		if err != nil {
			l.Warn("An error occurred during read report file", zap.Error(err))
			return err
		}
		if isFirstLine {
			header = cols
			isFirstLine = false
		} else {
			colMap := make(map[string]string)
			for i, h := range header {
				colMap[h] = cols[i]
			}
			if err := tester(colMap); err != nil {
				l.Warn("Tester returned an error", zap.Error(err), zap.Any("cols", colMap))
				return err
			}
		}
	}

	return nil
}
<|endoftext|>


<commit_before>// Copyright 2015 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package cmd

import (
	"time"

	"github.com/juju/testing"
	jc "github.com/juju/testing/checkers"
	gc "gopkg.in/check.v1"
	charmresource "gopkg.in/juju/charm.v6-unstable/resource"

	"github.com/juju/juju/resource"
)

var _ = gc.Suite(&CharmTabularSuite{})

type CharmTabularSuite struct {
	testing.IsolationSuite
}

func (s *CharmTabularSuite) TestFormatCharmTabularOkay(c *gc.C) {
	res := charmRes(c, "spam", ".tgz", "...", "")
	formatted := []FormattedCharmResource{FormatCharmResource(res)}

	data, err := FormatCharmTabular(formatted)
	c.Assert(err, jc.ErrorIsNil)

	c.Check(string(data), gc.Equals, `
RESOURCE REVISION
spam     1
`[1:])
}

func (s *CharmTabularSuite) TestFormatCharmTabularMinimal(c *gc.C) {
	res := charmRes(c, "spam", "", "", "")
	formatted := []FormattedCharmResource{FormatCharmResource(res)}

	data, err := FormatCharmTabular(formatted)
	c.Assert(err, jc.ErrorIsNil)

	c.Check(string(data), gc.Equals, `
RESOURCE REVISION
spam     1
`[1:])
}

func (s *CharmTabularSuite) TestFormatCharmTabularUpload(c *gc.C) {
	res := charmRes(c, "spam", "", "", "")
	res.Origin = charmresource.OriginUpload
	formatted := []FormattedCharmResource{FormatCharmResource(res)}

	data, err := FormatCharmTabular(formatted)
	c.Assert(err, jc.ErrorIsNil)

	c.Check(string(data), gc.Equals, `
RESOURCE REVISION
spam     1
`[1:])
}

func (s *CharmTabularSuite) TestFormatCharmTabularMulti(c *gc.C) {
	formatted := []FormattedCharmResource{
		FormatCharmResource(charmRes(c, "spam", ".tgz", "spamspamspamspam", "")),
		FormatCharmResource(charmRes(c, "eggs", "", "...", "")),
		FormatCharmResource(charmRes(c, "somethingbig", ".zip", "", "")),
		FormatCharmResource(charmRes(c, "song", ".mp3", "your favorite", "")),
		FormatCharmResource(charmRes(c, "avatar", ".png", "your picture", "")),
	}
	formatted[1].Revision = 2

	data, err := FormatCharmTabular(formatted)
	c.Assert(err, jc.ErrorIsNil)

	c.Check(string(data), gc.Equals, `
RESOURCE     REVISION
spam         1
eggs         2
somethingbig 1
song         1
avatar       1
`[1:])
}

func (s *CharmTabularSuite) TestFormatCharmTabularBadValue(c *gc.C) {
	bogus := "should have been something else"
	_, err := FormatCharmTabular(bogus)

	c.Check(err, gc.ErrorMatches, `expected value of type .*`)
}

var _ = gc.Suite(&SvcTabularSuite{})

type SvcTabularSuite struct {
	testing.IsolationSuite
}

func (s *SvcTabularSuite) TestFormatServiceOkay(c *gc.C) {
	res := resource.Resource{

		Resource: charmresource.Resource{
			Meta: charmresource.Meta{
				Name:        "openjdk",
				Description: "the java runtime",
			},
			Origin:   charmresource.OriginStore,
			Revision: 7,
		},
		Timestamp: time.Now(),
	}

	formatted := []FormattedSvcResource{FormatSvcResource(res)}

	data, err := FormatSvcTabular(formatted)
	c.Assert(err, jc.ErrorIsNil)

	c.Check(string(data), gc.Equals, `
[Service]
RESOURCE SUPPLIED BY REVISION
openjdk  charmstore  7
`[1:])
}

func (s *SvcTabularSuite) TestFormatUnitOkay(c *gc.C) {
	res := resource.Resource{

		Resource: charmresource.Resource{
			Meta: charmresource.Meta{
				Name:        "openjdk",
				Description: "the java runtime",
			},
			Origin:   charmresource.OriginStore,
			Revision: 7,
		},
		Timestamp: time.Now(),
	}

	formatted := []FormattedUnitResource{
		FormattedUnitResource{FormatSvcResource(res)},
	}

	data, err := FormatSvcTabular(formatted)
	c.Assert(err, jc.ErrorIsNil)

	c.Check(string(data), gc.Equals, `
[Unit]
RESOURCE REVISION
openjdk  7
`[1:])
}

func (s *SvcTabularSuite) TestFormatCharmTabularMulti(c *gc.C) {
	res := []resource.Resource{
		{
			Resource: charmresource.Resource{
				Meta: charmresource.Meta{
					Name:        "openjdk",
					Description: "the java runtime",
				},
				Origin:   charmresource.OriginStore,
				Revision: 7,
			},
		},
		{
			Resource: charmresource.Resource{
				Meta: charmresource.Meta{
					Name:        "website",
					Description: "your website data",
				},
				Origin: charmresource.OriginUpload,
			},
		},
		{
			Resource: charmresource.Resource{
				Meta: charmresource.Meta{
					Name:        "openjdk2",
					Description: "another java runtime",
				},
				Origin:   charmresource.OriginStore,
				Revision: 8,
			},
			Timestamp: time.Date(2012, 12, 12, 12, 12, 12, 0, time.UTC),
		},
		{
			Resource: charmresource.Resource{
				Meta: charmresource.Meta{
					Name:        "website2",
					Description: "your website data",
				},
				Origin: charmresource.OriginUpload,
			},
			Username:  "Bill User",
			Timestamp: time.Date(2012, 12, 12, 12, 12, 12, 0, time.UTC),
		},
	}

	formatted := make([]FormattedSvcResource, len(res))
	for i := range res {
		formatted[i] = FormatSvcResource(res[i])
	}

	data, err := FormatSvcTabular(formatted)
	c.Assert(err, jc.ErrorIsNil)

	// Notes: sorted by name, then by revision, newest first.
	c.Check(string(data), gc.Equals, `
[Service]
RESOURCE SUPPLIED BY REVISION
openjdk  charmstore  7
website  upload      -
openjdk2 charmstore  8
website2 Bill User   2012-12-12T12:12
`[1:])
}
<commit_msg>put back test that shouldn't have been deleted<commit_after>// Copyright 2015 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package cmd

import (
	"time"

	"github.com/juju/testing"
	jc "github.com/juju/testing/checkers"
	gc "gopkg.in/check.v1"
	charmresource "gopkg.in/juju/charm.v6-unstable/resource"

	"github.com/juju/juju/resource"
)

var _ = gc.Suite(&CharmTabularSuite{})

type CharmTabularSuite struct {
	testing.IsolationSuite
}

func (s *CharmTabularSuite) TestFormatCharmTabularOkay(c *gc.C) {
	res := charmRes(c, "spam", ".tgz", "...", "")
	formatted := []FormattedCharmResource{FormatCharmResource(res)}

	data, err := FormatCharmTabular(formatted)
	c.Assert(err, jc.ErrorIsNil)

	c.Check(string(data), gc.Equals, `
RESOURCE REVISION
spam     1
`[1:])
}

func (s *CharmTabularSuite) TestFormatCharmTabularMinimal(c *gc.C) {
	res := charmRes(c, "spam", "", "", "")
	formatted := []FormattedCharmResource{FormatCharmResource(res)}

	data, err := FormatCharmTabular(formatted)
	c.Assert(err, jc.ErrorIsNil)

	c.Check(string(data), gc.Equals, `
RESOURCE REVISION
spam     1
`[1:])
}

func (s *CharmTabularSuite) TestFormatCharmTabularUpload(c *gc.C) {
	res := charmRes(c, "spam", "", "", "")
	res.Origin = charmresource.OriginUpload
	formatted := []FormattedCharmResource{FormatCharmResource(res)}

	data, err := FormatCharmTabular(formatted)
	c.Assert(err, jc.ErrorIsNil)

	c.Check(string(data), gc.Equals, `
RESOURCE REVISION
spam     1
`[1:])
}

func (s *CharmTabularSuite) TestFormatCharmTabularMulti(c *gc.C) {
	formatted := []FormattedCharmResource{
		FormatCharmResource(charmRes(c, "spam", ".tgz", "spamspamspamspam", "")),
		FormatCharmResource(charmRes(c, "eggs", "", "...", "")),
		FormatCharmResource(charmRes(c, "somethingbig", ".zip", "", "")),
		FormatCharmResource(charmRes(c, "song", ".mp3", "your favorite", "")),
		FormatCharmResource(charmRes(c, "avatar", ".png", "your picture", "")),
	}
	formatted[1].Revision = 2

	data, err := FormatCharmTabular(formatted)
	c.Assert(err, jc.ErrorIsNil)

	c.Check(string(data), gc.Equals, `
RESOURCE     REVISION
spam         1
eggs         2
somethingbig 1
song         1
avatar       1
`[1:])
}

func (s *CharmTabularSuite) TestFormatCharmTabularBadValue(c *gc.C) {
	bogus := "should have been something else"
	_, err := FormatCharmTabular(bogus)

	c.Check(err, gc.ErrorMatches, `expected value of type .*`)
}

var _ = gc.Suite(&SvcTabularSuite{})

type SvcTabularSuite struct {
	testing.IsolationSuite
}

func (s *SvcTabularSuite) TestFormatServiceOkay(c *gc.C) {
	res := resource.Resource{

		Resource: charmresource.Resource{
			Meta: charmresource.Meta{
				Name:        "openjdk",
				Description: "the java runtime",
			},
			Origin:   charmresource.OriginStore,
			Revision: 7,
		},
		Timestamp: time.Now(),
	}

	formatted := []FormattedSvcResource{FormatSvcResource(res)}

	data, err := FormatSvcTabular(formatted)
	c.Assert(err, jc.ErrorIsNil)

	c.Check(string(data), gc.Equals, `
[Service]
RESOURCE SUPPLIED BY REVISION
openjdk  charmstore  7
`[1:])
}

func (s *SvcTabularSuite) TestFormatUnitOkay(c *gc.C) {
	res := resource.Resource{

		Resource: charmresource.Resource{
			Meta: charmresource.Meta{
				Name:        "openjdk",
				Description: "the java runtime",
			},
			Origin:   charmresource.OriginStore,
			Revision: 7,
		},
		Timestamp: time.Now(),
	}

	formatted := []FormattedUnitResource{
		FormattedUnitResource{FormatSvcResource(res)},
	}

	data, err := FormatSvcTabular(formatted)
	c.Assert(err, jc.ErrorIsNil)

	c.Check(string(data), gc.Equals, `
[Unit]
RESOURCE REVISION
openjdk  7
`[1:])
}

func (s *SvcTabularSuite) TestFormatCharmTabularMulti(c *gc.C) {
	res := []resource.Resource{
		{
			Resource: charmresource.Resource{
				Meta: charmresource.Meta{
					Name:        "openjdk",
					Description: "the java runtime",
				},
				Origin:   charmresource.OriginStore,
				Revision: 7,
			},
		},
		{
			Resource: charmresource.Resource{
				Meta: charmresource.Meta{
					Name:        "website",
					Description: "your website data",
				},
				Origin: charmresource.OriginUpload,
			},
		},
		{
			Resource: charmresource.Resource{
				Meta: charmresource.Meta{
					Name:        "openjdk2",
					Description: "another java runtime",
				},
				Origin:   charmresource.OriginStore,
				Revision: 8,
			},
			Timestamp: time.Date(2012, 12, 12, 12, 12, 12, 0, time.UTC),
		},
		{
			Resource: charmresource.Resource{
				Meta: charmresource.Meta{
					Name:        "website2",
					Description: "your website data",
				},
				Origin: charmresource.OriginUpload,
			},
			Username:  "Bill User",
			Timestamp: time.Date(2012, 12, 12, 12, 12, 12, 0, time.UTC),
		},
	}

	formatted := make([]FormattedSvcResource, len(res))
	for i := range res {
		formatted[i] = FormatSvcResource(res[i])
	}

	data, err := FormatSvcTabular(formatted)
	c.Assert(err, jc.ErrorIsNil)

	// Notes: sorted by name, then by revision, newest first.
	c.Check(string(data), gc.Equals, `
[Service]
RESOURCE SUPPLIED BY REVISION
openjdk  charmstore  7
website  upload      -
openjdk2 charmstore  8
website2 Bill User   2012-12-12T12:12
`[1:])
}

func (s *SvcTabularSuite) TestFormatCharmTabularBadValue(c *gc.C) {
	bogus := "should have been something else"
	_, err := FormatSvcTabular(bogus)
	c.Check(err, gc.ErrorMatches, `unexpected type for data: string`)
}
<|endoftext|>


<commit_before>// Copyright 2017 syzkaller project authors. All rights reserved.
// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.

// Package git provides helper functions for working with git repositories.
package git

import (
	"bufio"
	"bytes"
	"fmt"
	"io"
	"net/mail"
	"os"
	"os/exec"
	"regexp"
	"strings"
	"time"

	"github.com/google/syzkaller/pkg/osutil"
)

const (
	DateFormat = "Mon Jan 2 15:04:05 2006 -0700"
	timeout    = time.Hour // timeout for all git invocations
)

// Poll checkouts the specified repository/branch in dir.
// This involves fetching/resetting/cloning as necessary to recover from all possible problems.
// Returns hash of the HEAD commit in the specified branch.
func Poll(dir, repo, branch string) (*Commit, error) {
	runSandboxed(dir, "git", "reset", "--hard")
	origin, err := runSandboxed(dir, "git", "remote", "get-url", "origin")
	if err != nil || strings.TrimSpace(string(origin)) != repo {
		// The repo is here, but it has wrong origin (e.g. repo in config has changed), re-clone.
		if err := clone(dir, repo, branch); err != nil {
			return nil, err
		}
	}
	// Use origin/branch for the case the branch was force-pushed,
	// in such case branch is not the same is origin/branch and we will
	// stuck with the local version forever (git checkout won't fail).
	if _, err := runSandboxed(dir, "git", "checkout", "origin/"+branch); err != nil {
		// No such branch (e.g. branch in config has changed), re-clone.
		if err := clone(dir, repo, branch); err != nil {
			return nil, err
		}
	}
	if _, err := runSandboxed(dir, "git", "fetch", "--no-tags"); err != nil {
		// Something else is wrong, re-clone.
		if err := clone(dir, repo, branch); err != nil {
			return nil, err
		}
	}
	if _, err := runSandboxed(dir, "git", "checkout", "origin/"+branch); err != nil {
		return nil, err
	}
	return HeadCommit(dir)
}

// CheckoutBranch checkouts the specified repository/branch in dir.
func CheckoutBranch(dir, repo, branch string) (*Commit, error) {
	if _, err := runSandboxed(dir, "git", "reset", "--hard"); err != nil {
		if err := initRepo(dir); err != nil {
			return nil, err
		}
	}
	_, err := runSandboxed(dir, "git", "fetch", repo, branch)
	if err != nil {
		return nil, err
	}
	if _, err := runSandboxed(dir, "git", "checkout", "FETCH_HEAD"); err != nil {
		return nil, err
	}
	return HeadCommit(dir)
}

// CheckoutCommit checkouts the specified repository on the specified commit in dir.
func CheckoutCommit(dir, repo, commit string) (*Commit, error) {
	if _, err := runSandboxed(dir, "git", "reset", "--hard"); err != nil {
		if err := initRepo(dir); err != nil {
			return nil, err
		}
	}
	_, err := runSandboxed(dir, "git", "fetch", repo)
	if err != nil {
		return nil, err
	}
	if _, err := runSandboxed(dir, "git", "checkout", commit); err != nil {
		return nil, err
	}
	return HeadCommit(dir)
}

func clone(dir, repo, branch string) error {
	if err := initRepo(dir); err != nil {
		return err
	}
	if _, err := runSandboxed(dir, "git", "remote", "add", "origin", repo); err != nil {
		return err
	}
	if _, err := runSandboxed(dir, "git", "fetch", "origin", branch); err != nil {
		return err
	}
	return nil
}

func initRepo(dir string) error {
	if err := os.RemoveAll(dir); err != nil {
		return fmt.Errorf("failed to remove repo dir: %v", err)
	}
	if err := osutil.MkdirAll(dir); err != nil {
		return fmt.Errorf("failed to create repo dir: %v", err)
	}
	if err := osutil.SandboxChown(dir); err != nil {
		return err
	}
	if _, err := runSandboxed(dir, "git", "init"); err != nil {
		return err
	}
	return nil
}

type Commit struct {
	Hash  string
	Title string
	Date  time.Time
}

// HeadCommit returns info about the HEAD commit of the current branch of git repository in dir.
func HeadCommit(dir string) (*Commit, error) {
	output, err := runSandboxed(dir, "git", "log", "--pretty=format:%H%n%s%n%ad", "-n", "1")
	if err != nil {
		return nil, err
	}
	lines := bytes.Split(output, []byte{'\n'})
	if len(lines) != 3 || len(lines[0]) != 40 {
		return nil, fmt.Errorf("unexpected git log output: %q", output)
	}
	date, err := time.Parse(DateFormat, string(lines[2]))
	if err != nil {
		return nil, fmt.Errorf("failed to parse date in git log output: %v\n%q", err, output)
	}
	com := &Commit{
		Hash:  string(lines[0]),
		Title: string(lines[1]),
		Date:  date,
	}
	return com, nil
}

// ListRecentCommits returns list of recent commit titles starting from baseCommit.
func ListRecentCommits(dir, baseCommit string) ([]string, error) {
	// On upstream kernel this produces ~11MB of output.
	// Somewhat inefficient to collect whole output in a slice
	// and then convert to string, but should be bearable.
	output, err := runSandboxed(dir, "git", "log",
		"--pretty=format:%s", "--no-merges", "-n", "200000", baseCommit)
	if err != nil {
		return nil, err
	}
	return strings.Split(string(output), "\n"), nil
}

type FixCommit struct {
	Tag   string
	Title string
}

// ExtractFixTagsFromCommits extracts fixing tags for bugs from git log.
// Given email = "user@domain.com", it searches for tags of the form "user+tag@domain.com"
// and return pairs {tag, commit title}.
func ExtractFixTagsFromCommits(dir, baseCommit, email string) ([]FixCommit, error) {
	since := time.Now().Add(-time.Hour * 24 * 365).Format("01-02-2006")
	cmd := exec.Command("git", "log", "--no-merges", "--since", since, baseCommit)
	cmd.Dir = dir
	stdout, err := cmd.StdoutPipe()
	if err != nil {
		return nil, err
	}
	if err := cmd.Start(); err != nil {
		return nil, err
	}
	defer cmd.Wait()
	defer cmd.Process.Kill()
	return extractFixTags(stdout, email)
}

func extractFixTags(r io.Reader, email string) ([]FixCommit, error) {
	user, domain, err := splitEmail(email)
	if err != nil {
		return nil, fmt.Errorf("failed to parse email %q: %v", email, err)
	}
	var (
		s           = bufio.NewScanner(r)
		commits     []FixCommit
		commitTitle = ""
		commitStart = []byte("commit ")
		bodyPrefix  = []byte("    ")
		userBytes   = []byte(user + "+")
		domainBytes = []byte(domain)
	)
	for s.Scan() {
		ln := s.Bytes()
		if bytes.HasPrefix(ln, commitStart) {
			commitTitle = ""
			continue
		}
		if !bytes.HasPrefix(ln, bodyPrefix) {
			continue
		}
		ln = ln[len(bodyPrefix):]
		if len(ln) == 0 {
			continue
		}
		if commitTitle == "" {
			commitTitle = string(ln)
			continue
		}
		userPos := bytes.Index(ln, userBytes)
		if userPos == -1 {
			continue
		}
		domainPos := bytes.Index(ln[userPos+len(userBytes)+1:], domainBytes)
		if domainPos == -1 {
			continue
		}
		startPos := userPos + len(userBytes)
		endPos := userPos + len(userBytes) + domainPos + 1
		tag := string(ln[startPos:endPos])
		commits = append(commits, FixCommit{tag, commitTitle})
	}
	return commits, s.Err()
}

func splitEmail(email string) (user, domain string, err error) {
	addr, err := mail.ParseAddress(email)
	if err != nil {
		return "", "", err
	}
	at := strings.IndexByte(addr.Address, '@')
	if at == -1 {
		return "", "", fmt.Errorf("no @ in email address")
	}
	user = addr.Address[:at]
	domain = addr.Address[at:]
	if plus := strings.IndexByte(user, '+'); plus != -1 {
		user = user[:plus]
	}
	return
}

// CanonicalizeCommit returns commit title that can be used when checking
// if a particular commit is present in a git tree.
// Some trees add prefixes to commit titles during backporting,
// so we want e.g. commit "foo bar" match "BACKPORT: foo bar".
func CanonicalizeCommit(title string) string {
	for _, prefix := range commitPrefixes {
		if strings.HasPrefix(title, prefix) {
			title = title[len(prefix):]
			break
		}
	}
	return strings.TrimSpace(title)
}

var commitPrefixes = []string{
	"UPSTREAM:",
	"CHROMIUM:",
	"FROMLIST:",
	"BACKPORT:",
	"FROMGIT:",
	"net-backports:",
}

func Patch(dir string, patch []byte) error {
	// Do --dry-run first to not mess with partially consistent state.
	cmd := osutil.Command("patch", "-p1", "--force", "--ignore-whitespace", "--dry-run")
	if err := osutil.Sandbox(cmd, true, true); err != nil {
		return err
	}
	cmd.Stdin = bytes.NewReader(patch)
	cmd.Dir = dir
	if output, err := cmd.CombinedOutput(); err != nil {
		// If it reverses clean, then it's already applied
		// (seems to be the easiest way to detect it).
		cmd = osutil.Command("patch", "-p1", "--force", "--ignore-whitespace", "--reverse", "--dry-run")
		if err := osutil.Sandbox(cmd, true, true); err != nil {
			return err
		}
		cmd.Stdin = bytes.NewReader(patch)
		cmd.Dir = dir
		if _, err := cmd.CombinedOutput(); err == nil {
			return fmt.Errorf("patch is already applied")
		}
		return fmt.Errorf("failed to apply patch:\n%s", output)
	}
	// Now apply for real.
	cmd = osutil.Command("patch", "-p1", "--force", "--ignore-whitespace")
	if err := osutil.Sandbox(cmd, true, true); err != nil {
		return err
	}
	cmd.Stdin = bytes.NewReader(patch)
	cmd.Dir = dir
	if output, err := cmd.CombinedOutput(); err != nil {
		return fmt.Errorf("failed to apply patch after dry run:\n%s", output)
	}
	return nil
}

func runSandboxed(dir, command string, args ...string) ([]byte, error) {
	cmd := osutil.Command(command, args...)
	cmd.Dir = dir
	if err := osutil.Sandbox(cmd, true, false); err != nil {
		return nil, err
	}
	return osutil.Run(timeout, cmd)
}

// CheckRepoAddress does a best-effort approximate check of a git repo address.
func CheckRepoAddress(repo string) bool {
	return gitRepoRe.MatchString(repo)
}

// CheckBranch does a best-effort approximate check of a git branch name.
func CheckBranch(branch string) bool {
	return gitBranchRe.MatchString(branch)
}

func CheckCommitHash(hash string) bool {
	if !gitHashRe.MatchString(hash) {
		return false
	}
	ln := len(hash)
	return ln == 8 || ln == 10 || ln == 12 || ln == 16 || ln == 20 || ln == 40
}

var (
	// nolint: lll
	gitRepoRe   = regexp.MustCompile(`^(git|ssh|http|https|ftp|ftps)://[a-zA-Z0-9-_]+(\.[a-zA-Z0-9-_]+)+(:[0-9]+)?/[a-zA-Z0-9-_./]+\.git(/)?$`)
	gitBranchRe = regexp.MustCompile("^[a-zA-Z0-9-_/.]{2,200}$")
	gitHashRe   = regexp.MustCompile("^[a-f0-9]+$")
)
<commit_msg>pkg/git: add SwitchCommit and GetCommit<commit_after>// Copyright 2017 syzkaller project authors. All rights reserved.
// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.

// Package git provides helper functions for working with git repositories.
package git

import (
	"bufio"
	"bytes"
	"fmt"
	"io"
	"net/mail"
	"os"
	"os/exec"
	"regexp"
	"strings"
	"time"

	"github.com/google/syzkaller/pkg/osutil"
)

const (
	DateFormat = "Mon Jan 2 15:04:05 2006 -0700"
	timeout    = time.Hour // timeout for all git invocations
)

// Poll checkouts the specified repository/branch in dir.
// This involves fetching/resetting/cloning as necessary to recover from all possible problems.
// Returns hash of the HEAD commit in the specified branch.
func Poll(dir, repo, branch string) (*Commit, error) {
	runSandboxed(dir, "git", "reset", "--hard")
	origin, err := runSandboxed(dir, "git", "remote", "get-url", "origin")
	if err != nil || strings.TrimSpace(string(origin)) != repo {
		// The repo is here, but it has wrong origin (e.g. repo in config has changed), re-clone.
		if err := clone(dir, repo, branch); err != nil {
			return nil, err
		}
	}
	// Use origin/branch for the case the branch was force-pushed,
	// in such case branch is not the same is origin/branch and we will
	// stuck with the local version forever (git checkout won't fail).
	if _, err := runSandboxed(dir, "git", "checkout", "origin/"+branch); err != nil {
		// No such branch (e.g. branch in config has changed), re-clone.
		if err := clone(dir, repo, branch); err != nil {
			return nil, err
		}
	}
	if _, err := runSandboxed(dir, "git", "fetch", "--no-tags"); err != nil {
		// Something else is wrong, re-clone.
		if err := clone(dir, repo, branch); err != nil {
			return nil, err
		}
	}
	if _, err := runSandboxed(dir, "git", "checkout", "origin/"+branch); err != nil {
		return nil, err
	}
	return HeadCommit(dir)
}

// CheckoutBranch checkouts the specified repository/branch in dir.
func CheckoutBranch(dir, repo, branch string) (*Commit, error) {
	if _, err := runSandboxed(dir, "git", "reset", "--hard"); err != nil {
		if err := initRepo(dir); err != nil {
			return nil, err
		}
	}
	_, err := runSandboxed(dir, "git", "fetch", repo, branch)
	if err != nil {
		return nil, err
	}
	if _, err := runSandboxed(dir, "git", "checkout", "FETCH_HEAD"); err != nil {
		return nil, err
	}
	return HeadCommit(dir)
}

// CheckoutCommit checkouts the specified repository on the specified commit in dir.
func CheckoutCommit(dir, repo, commit string) (*Commit, error) {
	if _, err := runSandboxed(dir, "git", "reset", "--hard"); err != nil {
		if err := initRepo(dir); err != nil {
			return nil, err
		}
	}
	_, err := runSandboxed(dir, "git", "fetch", repo)
	if err != nil {
		return nil, err
	}
	return SwitchCommit(dir, commit)
}

// SwitchCommit checkouts the specified commit without fetching.
func SwitchCommit(dir, commit string) (*Commit, error) {
	if _, err := runSandboxed(dir, "git", "checkout", commit); err != nil {
		return nil, err
	}
	return HeadCommit(dir)
}

func clone(dir, repo, branch string) error {
	if err := initRepo(dir); err != nil {
		return err
	}
	if _, err := runSandboxed(dir, "git", "remote", "add", "origin", repo); err != nil {
		return err
	}
	if _, err := runSandboxed(dir, "git", "fetch", "origin", branch); err != nil {
		return err
	}
	return nil
}

func initRepo(dir string) error {
	if err := os.RemoveAll(dir); err != nil {
		return fmt.Errorf("failed to remove repo dir: %v", err)
	}
	if err := osutil.MkdirAll(dir); err != nil {
		return fmt.Errorf("failed to create repo dir: %v", err)
	}
	if err := osutil.SandboxChown(dir); err != nil {
		return err
	}
	if _, err := runSandboxed(dir, "git", "init"); err != nil {
		return err
	}
	return nil
}

type Commit struct {
	Hash  string
	Title string
	Date  time.Time
}

// HeadCommit returns info about the HEAD commit of the current branch of git repository in dir.
func HeadCommit(dir string) (*Commit, error) {
	return GetCommit(dir, "HEAD")
}

func GetCommit(dir, commit string) (*Commit, error) {
	output, err := runSandboxed(dir, "git", "log", "--pretty=format:%H%n%s%n%ad", "-n", "1", commit)
	if err != nil {
		return nil, err
	}
	lines := bytes.Split(output, []byte{'\n'})
	if len(lines) != 3 || len(lines[0]) != 40 {
		return nil, fmt.Errorf("unexpected git log output: %q", output)
	}
	date, err := time.Parse(DateFormat, string(lines[2]))
	if err != nil {
		return nil, fmt.Errorf("failed to parse date in git log output: %v\n%q", err, output)
	}
	com := &Commit{
		Hash:  string(lines[0]),
		Title: string(lines[1]),
		Date:  date,
	}
	return com, nil
}

// ListRecentCommits returns list of recent commit titles starting from baseCommit.
func ListRecentCommits(dir, baseCommit string) ([]string, error) {
	// On upstream kernel this produces ~11MB of output.
	// Somewhat inefficient to collect whole output in a slice
	// and then convert to string, but should be bearable.
	output, err := runSandboxed(dir, "git", "log",
		"--pretty=format:%s", "--no-merges", "-n", "200000", baseCommit)
	if err != nil {
		return nil, err
	}
	return strings.Split(string(output), "\n"), nil
}

type FixCommit struct {
	Tag   string
	Title string
}

// ExtractFixTagsFromCommits extracts fixing tags for bugs from git log.
// Given email = "user@domain.com", it searches for tags of the form "user+tag@domain.com"
// and return pairs {tag, commit title}.
func ExtractFixTagsFromCommits(dir, baseCommit, email string) ([]FixCommit, error) {
	since := time.Now().Add(-time.Hour * 24 * 365).Format("01-02-2006")
	cmd := exec.Command("git", "log", "--no-merges", "--since", since, baseCommit)
	cmd.Dir = dir
	stdout, err := cmd.StdoutPipe()
	if err != nil {
		return nil, err
	}
	if err := cmd.Start(); err != nil {
		return nil, err
	}
	defer cmd.Wait()
	defer cmd.Process.Kill()
	return extractFixTags(stdout, email)
}

func extractFixTags(r io.Reader, email string) ([]FixCommit, error) {
	user, domain, err := splitEmail(email)
	if err != nil {
		return nil, fmt.Errorf("failed to parse email %q: %v", email, err)
	}
	var (
		s           = bufio.NewScanner(r)
		commits     []FixCommit
		commitTitle = ""
		commitStart = []byte("commit ")
		bodyPrefix  = []byte("    ")
		userBytes   = []byte(user + "+")
		domainBytes = []byte(domain)
	)
	for s.Scan() {
		ln := s.Bytes()
		if bytes.HasPrefix(ln, commitStart) {
			commitTitle = ""
			continue
		}
		if !bytes.HasPrefix(ln, bodyPrefix) {
			continue
		}
		ln = ln[len(bodyPrefix):]
		if len(ln) == 0 {
			continue
		}
		if commitTitle == "" {
			commitTitle = string(ln)
			continue
		}
		userPos := bytes.Index(ln, userBytes)
		if userPos == -1 {
			continue
		}
		domainPos := bytes.Index(ln[userPos+len(userBytes)+1:], domainBytes)
		if domainPos == -1 {
			continue
		}
		startPos := userPos + len(userBytes)
		endPos := userPos + len(userBytes) + domainPos + 1
		tag := string(ln[startPos:endPos])
		commits = append(commits, FixCommit{tag, commitTitle})
	}
	return commits, s.Err()
}

func splitEmail(email string) (user, domain string, err error) {
	addr, err := mail.ParseAddress(email)
	if err != nil {
		return "", "", err
	}
	at := strings.IndexByte(addr.Address, '@')
	if at == -1 {
		return "", "", fmt.Errorf("no @ in email address")
	}
	user = addr.Address[:at]
	domain = addr.Address[at:]
	if plus := strings.IndexByte(user, '+'); plus != -1 {
		user = user[:plus]
	}
	return
}

// CanonicalizeCommit returns commit title that can be used when checking
// if a particular commit is present in a git tree.
// Some trees add prefixes to commit titles during backporting,
// so we want e.g. commit "foo bar" match "BACKPORT: foo bar".
func CanonicalizeCommit(title string) string {
	for _, prefix := range commitPrefixes {
		if strings.HasPrefix(title, prefix) {
			title = title[len(prefix):]
			break
		}
	}
	return strings.TrimSpace(title)
}

var commitPrefixes = []string{
	"UPSTREAM:",
	"CHROMIUM:",
	"FROMLIST:",
	"BACKPORT:",
	"FROMGIT:",
	"net-backports:",
}

func Patch(dir string, patch []byte) error {
	// Do --dry-run first to not mess with partially consistent state.
	cmd := osutil.Command("patch", "-p1", "--force", "--ignore-whitespace", "--dry-run")
	if err := osutil.Sandbox(cmd, true, true); err != nil {
		return err
	}
	cmd.Stdin = bytes.NewReader(patch)
	cmd.Dir = dir
	if output, err := cmd.CombinedOutput(); err != nil {
		// If it reverses clean, then it's already applied
		// (seems to be the easiest way to detect it).
		cmd = osutil.Command("patch", "-p1", "--force", "--ignore-whitespace", "--reverse", "--dry-run")
		if err := osutil.Sandbox(cmd, true, true); err != nil {
			return err
		}
		cmd.Stdin = bytes.NewReader(patch)
		cmd.Dir = dir
		if _, err := cmd.CombinedOutput(); err == nil {
			return fmt.Errorf("patch is already applied")
		}
		return fmt.Errorf("failed to apply patch:\n%s", output)
	}
	// Now apply for real.
	cmd = osutil.Command("patch", "-p1", "--force", "--ignore-whitespace")
	if err := osutil.Sandbox(cmd, true, true); err != nil {
		return err
	}
	cmd.Stdin = bytes.NewReader(patch)
	cmd.Dir = dir
	if output, err := cmd.CombinedOutput(); err != nil {
		return fmt.Errorf("failed to apply patch after dry run:\n%s", output)
	}
	return nil
}

func runSandboxed(dir, command string, args ...string) ([]byte, error) {
	cmd := osutil.Command(command, args...)
	cmd.Dir = dir
	if err := osutil.Sandbox(cmd, true, false); err != nil {
		return nil, err
	}
	return osutil.Run(timeout, cmd)
}

// CheckRepoAddress does a best-effort approximate check of a git repo address.
func CheckRepoAddress(repo string) bool {
	return gitRepoRe.MatchString(repo)
}

// CheckBranch does a best-effort approximate check of a git branch name.
func CheckBranch(branch string) bool {
	return gitBranchRe.MatchString(branch)
}

func CheckCommitHash(hash string) bool {
	if !gitHashRe.MatchString(hash) {
		return false
	}
	ln := len(hash)
	return ln == 8 || ln == 10 || ln == 12 || ln == 16 || ln == 20 || ln == 40
}

var (
	// nolint: lll
	gitRepoRe   = regexp.MustCompile(`^(git|ssh|http|https|ftp|ftps)://[a-zA-Z0-9-_]+(\.[a-zA-Z0-9-_]+)+(:[0-9]+)?/[a-zA-Z0-9-_./]+\.git(/)?$`)
	gitBranchRe = regexp.MustCompile("^[a-zA-Z0-9-_/.]{2,200}$")
	gitHashRe   = regexp.MustCompile("^[a-f0-9]+$")
)
<|endoftext|>


<commit_before>package mux

import (
	"errors"
	"fmt"
	"log"
	"sync"
	"time"

	"github.com/bitfinexcom/bitfinex-api-go/pkg/models/event"
	"github.com/bitfinexcom/bitfinex-api-go/pkg/mux/client"
	"github.com/bitfinexcom/bitfinex-api-go/pkg/mux/msg"
)

// Mux will manage all connections and subscriptions. Will check if subscriptions
// limit is reached and spawn new connection when that happens. It will also listen
// to all incomming client messages and reconnect client with all its subscriptions
// in case of a failure
type Mux struct {
	cid           int
	dms           int
	publicChan    chan msg.Msg
	publicClients map[int]*client.Client
	privateChan   chan msg.Msg
	closeChan     chan bool
	privateClient *client.Client
	mtx           *sync.RWMutex
	Err           error
	transform     bool
	apikey        string
	apisec        string
	subInfo       map[int64]event.Info
	authenticated bool
	publicURL     string
	authURL       string
	online        bool
	subsRateLimit int
}

// api rate limit is 20 calls per minute. 1x3s, 20x1min
const (
	rateLimitDuration  = 3 * time.Second
	rateLimitQueueSize = 20
)

// New returns pointer to instance of mux
func New() *Mux {
	return &Mux{
		publicChan:    make(chan msg.Msg),
		privateChan:   make(chan msg.Msg),
		closeChan:     make(chan bool),
		publicClients: make(map[int]*client.Client),
		mtx:           &sync.RWMutex{},
		subInfo:       map[int64]event.Info{},
		publicURL:     "wss://api-pub.bitfinex.com/ws/2",
		authURL:       "wss://api.bitfinex.com/ws/2",
	}
}

// TransformRaw enables data transformation and mapping to appropriate
// models before sending it to consumer
func (m *Mux) TransformRaw() *Mux {
	m.transform = true
	return m
}

// WithAPIKEY accepts and persists api key
func (m *Mux) WithAPIKEY(key string) *Mux {
	m.apikey = key
	return m
}

// WithDeadManSwitch - when socket is closed, cancel all account orders
func (m *Mux) WithDeadManSwitch() *Mux {
	m.dms = 4
	return m
}

// WithAPISEC accepts and persists api secret
func (m *Mux) WithAPISEC(sec string) *Mux {
	m.apisec = sec
	return m
}

// WithPublicURL accepts and persists public api url
func (m *Mux) WithPublicURL(url string) *Mux {
	m.publicURL = url
	return m
}

// WithAuthURL accepts and persists auth api url
func (m *Mux) WithAuthURL(url string) *Mux {
	m.authURL = url
	return m
}

func (m *Mux) IsConnected() bool {
	return m.online
}

func (m *Mux) Close() bool {
	m.closeChan <- true
	return true
}

// Subscribe - given the details in form of event.Subscribe, subscribes client to public
// channels. If rate limit is reached, calls itself recursively after 1s with same params
func (m *Mux) Subscribe(sub event.Subscribe) *Mux {
	if m.Err != nil {
		return m
	}

	// if limit is reached, wait 1 second and recuresively
	// call Subscribe again with same subscription details
	if m.subsRateLimit == rateLimitQueueSize {
		time.Sleep(1 * time.Second)
		return m.Subscribe(sub)
	}

	m.mtx.RLock()
	defer m.mtx.RUnlock()
	if m.publicClients[m.cid].SubAdded(sub) {
		return m
	}

	if m.Err = m.publicClients[m.cid].Subscribe(sub); m.Err != nil {
		return m
	}

	if limitReached := m.publicClients[m.cid].SubsLimitReached(); limitReached {
		log.Printf("subs limit is reached on cid: %d, spawning new conn\n", m.cid)
		m.addPublicClient()
	}

	m.subsRateLimit++
	return m
}

// Start creates initial clients for accepting connections
func (m *Mux) Start() *Mux {
	if m.hasAPIKeys() && m.privateClient == nil {
		m.addPrivateClient()
	}

	m.watchRateLimit()
	m.addPublicClient()
	return m
}

// Listen accepts a callback func that will get called each time mux
// receives a message from any of its clients/subscriptions. It
// should be called last, after all setup calls are made
func (m *Mux) Listen(cb func(interface{}, error)) error {
	if m.Err != nil {
		return m.Err
	}

	m.online = true
	for {
		select {
		case ms, ok := <-m.publicChan:
			if !ok {
				return errors.New("public channel has closed unexpectedly")
			}
			if ms.Err != nil {
				cb(nil, fmt.Errorf("conn:%d has failed | err:%s | reconnecting", ms.CID, ms.Err))
				m.resetPublicClient(ms.CID)
				continue
			}
			// return raw payload data if transform is off
			if !m.transform {
				cb(ms.Data, nil)
				continue
			}
			// handle event type message
			if ms.IsEvent() {
				cb(m.recordEvent(ms.ProcessEvent()))
				continue
			}
			// handle data type message
			if ms.IsRaw() {
				raw, pld, chID, _, err := ms.PreprocessRaw()
				if err != nil {
					cb(nil, err)
					continue
				}

				inf, ok := m.subInfo[chID]
				if !ok {
					cb(nil, fmt.Errorf("unrecognized chanId:%d", chID))
					continue
				}
				cb(ms.ProcessPublic(raw, pld, chID, inf))
				continue
			}
			cb(nil, fmt.Errorf("unrecognized msg signature: %s", ms.Data))
		case ms, ok := <-m.privateChan:
			if !ok {
				return errors.New("private channel has closed unexpectedly")
			}
			if ms.Err != nil {
				cb(nil, fmt.Errorf("err: %s | reconnecting", ms.Err))
				m.resetPrivateClient()
				continue
			}
			// return raw payload data if transform is off
			if !m.transform {
				cb(ms.Data, nil)
				continue
			}
			// handle event type message
			if ms.IsEvent() {
				cb(m.recordEvent(ms.ProcessEvent()))
				continue
			}
			// handle data type message
			if ms.IsRaw() {
				raw, pld, chID, msgType, err := ms.PreprocessRaw()
				if err != nil {
					cb(nil, err)
					continue
				}
				cb(ms.ProcessPrivate(raw, pld, chID, msgType))
				continue
			}
			cb(nil, fmt.Errorf("unrecognized msg signature: %s", ms.Data))
		case <-m.closeChan:
			m.mtx.Lock()
			defer m.mtx.Unlock()

			for _, v := range m.publicClients {
				if v == nil {
					continue
				}
				if err := v.Close(); err != nil {
					log.Printf("failed closing public client: %s\n", err)
				}
			}

			if m.privateClient != nil {
				if err := m.privateClient.Close(); err != nil {
					log.Printf("failed closing private client: %s\n", err)
				}
			}

			m.online = false
			return nil
		}
	}
}

// Send meant for authenticated input, takes payload in form of interface
// and calls client with it
func (m *Mux) Send(pld interface{}) error {
	if !m.authenticated || m.privateClient == nil {
		return errors.New("not authorized")
	}
	return m.privateClient.Send(pld)
}

func (m *Mux) hasAPIKeys() bool {
	return len(m.apikey) != 0 && len(m.apisec) != 0
}

func (m *Mux) recordEvent(i event.Info, err error) (event.Info, error) {
	switch i.Event {
	case "subscribed":
		m.subInfo[i.ChanID] = i
	case "auth":
		if i.Status == "OK" {
			m.subInfo[i.ChanID] = i
			m.authenticated = true
		}
	}
	// add more cases if/when needed
	return i, err
}

func (m *Mux) resetPublicClient(cid int) {
	// pull old client subscriptions
	subs := m.publicClients[cid].GetAllSubs()
	// add fresh client
	m.addPublicClient()
	// resubscribe old events
	for _, sub := range subs {
		log.Printf("resubscribing: %+v\n", sub)
		m.Subscribe(sub)
	}
	// remove old, closed channel from the list
	delete(m.publicClients, cid)
}

func (m *Mux) resetPrivateClient() {
	m.authenticated = false
	m.privateClient = nil
	m.addPrivateClient()
}

func (m *Mux) addPublicClient() *Mux {
	// adding new client so making sure we increment cid
	m.cid++
	// create new public client and pass error to mux if any
	c, err := client.
		New().
		WithID(m.cid).
		WithSubsLimit(30).
		Public(m.publicURL)
	if err != nil {
		m.Err = err
		return m
	}
	// add new client to list for later reference
	m.publicClients[m.cid] = c
	// start listening for incoming client messages
	go c.Read(m.publicChan)
	return m
}

func (m *Mux) addPrivateClient() *Mux {
	// create new private client and pass error to mux if any
	c, err := client.New().Private(m.apikey, m.apisec, m.authURL, m.dms)
	if err != nil {
		m.Err = err
		return m
	}

	m.privateClient = c
	go c.Read(m.privateChan)
	return m
}

// watchRateLimit will run once every rateLimitDuration
// and free up the queue
func (m *Mux) watchRateLimit() {
	go func() {
		for {
			if m.subsRateLimit > 0 {
				m.subsRateLimit--
			}

			time.Sleep(rateLimitDuration)
		}
	}()
}
<commit_msg>meaningful naming<commit_after>package mux

import (
	"errors"
	"fmt"
	"log"
	"sync"
	"time"

	"github.com/bitfinexcom/bitfinex-api-go/pkg/models/event"
	"github.com/bitfinexcom/bitfinex-api-go/pkg/mux/client"
	"github.com/bitfinexcom/bitfinex-api-go/pkg/mux/msg"
)

// Mux will manage all connections and subscriptions. Will check if subscriptions
// limit is reached and spawn new connection when that happens. It will also listen
// to all incomming client messages and reconnect client with all its subscriptions
// in case of a failure
type Mux struct {
	cid                int
	dms                int
	publicChan         chan msg.Msg
	publicClients      map[int]*client.Client
	privateChan        chan msg.Msg
	closeChan          chan bool
	privateClient      *client.Client
	mtx                *sync.RWMutex
	Err                error
	transform          bool
	apikey             string
	apisec             string
	subInfo            map[int64]event.Info
	authenticated      bool
	publicURL          string
	authURL            string
	online             bool
	rateLimitQueueSize int
}

// api rate limit is 20 calls per minute. 1x3s, 20x1min
const (
	rateLimitDuration     = 3 * time.Second
	maxRateLimitQueueSize = 20
)

// New returns pointer to instance of mux
func New() *Mux {
	return &Mux{
		publicChan:    make(chan msg.Msg),
		privateChan:   make(chan msg.Msg),
		closeChan:     make(chan bool),
		publicClients: make(map[int]*client.Client),
		mtx:           &sync.RWMutex{},
		subInfo:       map[int64]event.Info{},
		publicURL:     "wss://api-pub.bitfinex.com/ws/2",
		authURL:       "wss://api.bitfinex.com/ws/2",
	}
}

// TransformRaw enables data transformation and mapping to appropriate
// models before sending it to consumer
func (m *Mux) TransformRaw() *Mux {
	m.transform = true
	return m
}

// WithAPIKEY accepts and persists api key
func (m *Mux) WithAPIKEY(key string) *Mux {
	m.apikey = key
	return m
}

// WithDeadManSwitch - when socket is closed, cancel all account orders
func (m *Mux) WithDeadManSwitch() *Mux {
	m.dms = 4
	return m
}

// WithAPISEC accepts and persists api secret
func (m *Mux) WithAPISEC(sec string) *Mux {
	m.apisec = sec
	return m
}

// WithPublicURL accepts and persists public api url
func (m *Mux) WithPublicURL(url string) *Mux {
	m.publicURL = url
	return m
}

// WithAuthURL accepts and persists auth api url
func (m *Mux) WithAuthURL(url string) *Mux {
	m.authURL = url
	return m
}

func (m *Mux) IsConnected() bool {
	return m.online
}

func (m *Mux) Close() bool {
	m.closeChan <- true
	return true
}

// Subscribe - given the details in form of event.Subscribe, subscribes client to public
// channels. If rate limit is reached, calls itself recursively after 1s with same params
func (m *Mux) Subscribe(sub event.Subscribe) *Mux {
	if m.Err != nil {
		return m
	}

	// if limit is reached, wait 1 second and recuresively
	// call Subscribe again with same subscription details
	if m.rateLimitQueueSize == maxRateLimitQueueSize {
		time.Sleep(1 * time.Second)
		return m.Subscribe(sub)
	}

	m.mtx.RLock()
	defer m.mtx.RUnlock()
	if m.publicClients[m.cid].SubAdded(sub) {
		return m
	}

	if m.Err = m.publicClients[m.cid].Subscribe(sub); m.Err != nil {
		return m
	}

	if limitReached := m.publicClients[m.cid].SubsLimitReached(); limitReached {
		log.Printf("subs limit is reached on cid: %d, spawning new conn\n", m.cid)
		m.addPublicClient()
	}

	m.rateLimitQueueSize++
	return m
}

// Start creates initial clients for accepting connections
func (m *Mux) Start() *Mux {
	if m.hasAPIKeys() && m.privateClient == nil {
		m.addPrivateClient()
	}

	m.watchRateLimit()
	m.addPublicClient()
	return m
}

// Listen accepts a callback func that will get called each time mux
// receives a message from any of its clients/subscriptions. It
// should be called last, after all setup calls are made
func (m *Mux) Listen(cb func(interface{}, error)) error {
	if m.Err != nil {
		return m.Err
	}

	m.online = true
	for {
		select {
		case ms, ok := <-m.publicChan:
			if !ok {
				return errors.New("public channel has closed unexpectedly")
			}
			if ms.Err != nil {
				cb(nil, fmt.Errorf("conn:%d has failed | err:%s | reconnecting", ms.CID, ms.Err))
				m.resetPublicClient(ms.CID)
				continue
			}
			// return raw payload data if transform is off
			if !m.transform {
				cb(ms.Data, nil)
				continue
			}
			// handle event type message
			if ms.IsEvent() {
				cb(m.recordEvent(ms.ProcessEvent()))
				continue
			}
			// handle data type message
			if ms.IsRaw() {
				raw, pld, chID, _, err := ms.PreprocessRaw()
				if err != nil {
					cb(nil, err)
					continue
				}

				inf, ok := m.subInfo[chID]
				if !ok {
					cb(nil, fmt.Errorf("unrecognized chanId:%d", chID))
					continue
				}
				cb(ms.ProcessPublic(raw, pld, chID, inf))
				continue
			}
			cb(nil, fmt.Errorf("unrecognized msg signature: %s", ms.Data))
		case ms, ok := <-m.privateChan:
			if !ok {
				return errors.New("private channel has closed unexpectedly")
			}
			if ms.Err != nil {
				cb(nil, fmt.Errorf("err: %s | reconnecting", ms.Err))
				m.resetPrivateClient()
				continue
			}
			// return raw payload data if transform is off
			if !m.transform {
				cb(ms.Data, nil)
				continue
			}
			// handle event type message
			if ms.IsEvent() {
				cb(m.recordEvent(ms.ProcessEvent()))
				continue
			}
			// handle data type message
			if ms.IsRaw() {
				raw, pld, chID, msgType, err := ms.PreprocessRaw()
				if err != nil {
					cb(nil, err)
					continue
				}
				cb(ms.ProcessPrivate(raw, pld, chID, msgType))
				continue
			}
			cb(nil, fmt.Errorf("unrecognized msg signature: %s", ms.Data))
		case <-m.closeChan:
			m.mtx.Lock()
			defer m.mtx.Unlock()

			for _, v := range m.publicClients {
				if v == nil {
					continue
				}
				if err := v.Close(); err != nil {
					log.Printf("failed closing public client: %s\n", err)
				}
			}

			if m.privateClient != nil {
				if err := m.privateClient.Close(); err != nil {
					log.Printf("failed closing private client: %s\n", err)
				}
			}

			m.online = false
			return nil
		}
	}
}

// Send meant for authenticated input, takes payload in form of interface
// and calls client with it
func (m *Mux) Send(pld interface{}) error {
	if !m.authenticated || m.privateClient == nil {
		return errors.New("not authorized")
	}
	return m.privateClient.Send(pld)
}

func (m *Mux) hasAPIKeys() bool {
	return len(m.apikey) != 0 && len(m.apisec) != 0
}

func (m *Mux) recordEvent(i event.Info, err error) (event.Info, error) {
	switch i.Event {
	case "subscribed":
		m.subInfo[i.ChanID] = i
	case "auth":
		if i.Status == "OK" {
			m.subInfo[i.ChanID] = i
			m.authenticated = true
		}
	}
	// add more cases if/when needed
	return i, err
}

func (m *Mux) resetPublicClient(cid int) {
	// pull old client subscriptions
	subs := m.publicClients[cid].GetAllSubs()
	// add fresh client
	m.addPublicClient()
	// resubscribe old events
	for _, sub := range subs {
		log.Printf("resubscribing: %+v\n", sub)
		m.Subscribe(sub)
	}
	// remove old, closed channel from the list
	delete(m.publicClients, cid)
}

func (m *Mux) resetPrivateClient() {
	m.authenticated = false
	m.privateClient = nil
	m.addPrivateClient()
}

func (m *Mux) addPublicClient() *Mux {
	// adding new client so making sure we increment cid
	m.cid++
	// create new public client and pass error to mux if any
	c, err := client.
		New().
		WithID(m.cid).
		WithSubsLimit(30).
		Public(m.publicURL)
	if err != nil {
		m.Err = err
		return m
	}
	// add new client to list for later reference
	m.publicClients[m.cid] = c
	// start listening for incoming client messages
	go c.Read(m.publicChan)
	return m
}

func (m *Mux) addPrivateClient() *Mux {
	// create new private client and pass error to mux if any
	c, err := client.New().Private(m.apikey, m.apisec, m.authURL, m.dms)
	if err != nil {
		m.Err = err
		return m
	}

	m.privateClient = c
	go c.Read(m.privateChan)
	return m
}

// watchRateLimit will run once every rateLimitDuration
// and free up the queue
func (m *Mux) watchRateLimit() {
	go func() {
		for {
			if m.rateLimitQueueSize > 0 {
				m.rateLimitQueueSize--
			}

			time.Sleep(rateLimitDuration)
		}
	}()
}
<|endoftext|>


<commit_before>package web

import (
	"http"
	"io"
	"junta/store"
	"junta/util"
	"json"
	"log"
	"net"
	"strings"
	"template"
	"websocket"
)

var Store *store.Store
var ClusterName, evPrefix string
var mainTpl  = template.MustParse(main_html, nil)

type info struct {
	Path string
}

func Serve(listener net.Listener) {
	prefix := "/j/" + ClusterName
	evPrefix = "/events" + prefix

	http.Handle("/", http.RedirectHandler("/view/j/"+ClusterName+"/", 307))
	http.HandleFunc("/view/", viewHtml)
	http.HandleFunc("/main.js", mainJs)
	http.HandleFunc("/main.css", mainCss)
	http.HandleFunc(evPrefix+"/", evServer)

	http.Serve(listener, nil)
}

func send(ws *websocket.Conn, path string, evs chan store.Event, logger *log.Logger) {
	defer close(evs)
	l := len(path) - 1
	for ev := range evs {
		ev.Getter = nil // don't marshal the entire snapshot
		ev.Path = ev.Path[l:]
		logger.Log("sending", ev)
		b, err := json.Marshal(ev)
		if err != nil {
			logger.Log(err)
			return
		}
		_, err = ws.Write(b)
		if err != nil {
			logger.Log(err)
			return
		}
	}
}

func evServer(c *http.Conn, r *http.Request) {
	evs, wevs := make(chan store.Event), make(chan store.Event)
	logger := util.NewLogger(c.RemoteAddr)
	path := r.URL.Path[len(evPrefix):]
	logger.Log("new", path)

	Store.Watch(path+"**", evs)

	// TODO convert store.Snapshot to json and use that
	go func() {
		walk(path, Store, wevs)
		close(wevs)
	}()

	websocket.Handler(func(ws *websocket.Conn) {
		send(ws, path, wevs, logger)
		send(ws, path, evs, logger)
		ws.Close()
	}).ServeHTTP(c, r)
}

func viewHtml(c *http.Conn, r *http.Request) {
	if !strings.HasSuffix(r.URL.Path, "/") {
		c.WriteHeader(404)
		return
	}
	var x info
	x.Path = r.URL.Path[len("/view"):]
	c.SetHeader("content-type", "text/html")
	mainTpl.Execute(x, c)
}

func mainJs(c *http.Conn, r *http.Request) {
	c.SetHeader("content-type", "application/javascript")
	io.WriteString(c, main_js)
}

func mainCss(c *http.Conn, r *http.Request) {
	c.SetHeader("content-type", "text/css")
	io.WriteString(c, main_css)
}

func walk(path string, st *store.Store, ch chan store.Event) {
	for path != "/" && strings.HasSuffix(path, "/") {
		// TODO generalize and factor this into pkg store.
		path = path[0:len(path)-1]
	}
	v, cas := st.Get(path)
	if cas != store.Dir {
		ch <- store.Event{0, path, v[0], cas, "", nil, nil}
		return
	}
	if path == "/" {
		path = ""
	}
	for _, ent := range v {
		walk(path+"/"+ent, st, ch)
	}
}
<commit_msg>refactor<commit_after>package web

import (
	"http"
	"io"
	"junta/store"
	"junta/util"
	"json"
	"log"
	"net"
	"strings"
	"template"
	"websocket"
)

var Store *store.Store
var ClusterName, evPrefix string
var mainTpl  = template.MustParse(main_html, nil)

type info struct {
	Path string
}

type stringHandler struct {
	contentType string
	body string
}

func (sh stringHandler) ServeHTTP(c *http.Conn, r *http.Request) {
	c.SetHeader("content-type", sh.contentType)
	io.WriteString(c, sh.body)
}

func Serve(listener net.Listener) {
	prefix := "/j/" + ClusterName
	evPrefix = "/events" + prefix

	http.Handle("/", http.RedirectHandler("/view/j/"+ClusterName+"/", 307))
	http.HandleFunc("/view/", viewHtml)
	http.Handle("/main.js", stringHandler{"application/javascript", main_js})
	http.Handle("/main.css", stringHandler{"text/css", main_css})
	http.HandleFunc(evPrefix+"/", evServer)

	http.Serve(listener, nil)
}

func send(ws *websocket.Conn, path string, evs chan store.Event, logger *log.Logger) {
	defer close(evs)
	l := len(path) - 1
	for ev := range evs {
		ev.Getter = nil // don't marshal the entire snapshot
		ev.Path = ev.Path[l:]
		logger.Log("sending", ev)
		b, err := json.Marshal(ev)
		if err != nil {
			logger.Log(err)
			return
		}
		_, err = ws.Write(b)
		if err != nil {
			logger.Log(err)
			return
		}
	}
}

func evServer(c *http.Conn, r *http.Request) {
	evs, wevs := make(chan store.Event), make(chan store.Event)
	logger := util.NewLogger(c.RemoteAddr)
	path := r.URL.Path[len(evPrefix):]
	logger.Log("new", path)

	Store.Watch(path+"**", evs)

	// TODO convert store.Snapshot to json and use that
	go func() {
		walk(path, Store, wevs)
		close(wevs)
	}()

	websocket.Handler(func(ws *websocket.Conn) {
		send(ws, path, wevs, logger)
		send(ws, path, evs, logger)
		ws.Close()
	}).ServeHTTP(c, r)
}

func viewHtml(c *http.Conn, r *http.Request) {
	if !strings.HasSuffix(r.URL.Path, "/") {
		c.WriteHeader(404)
		return
	}
	var x info
	x.Path = r.URL.Path[len("/view"):]
	c.SetHeader("content-type", "text/html")
	mainTpl.Execute(x, c)
}

func walk(path string, st *store.Store, ch chan store.Event) {
	for path != "/" && strings.HasSuffix(path, "/") {
		// TODO generalize and factor this into pkg store.
		path = path[0:len(path)-1]
	}
	v, cas := st.Get(path)
	if cas != store.Dir {
		ch <- store.Event{0, path, v[0], cas, "", nil, nil}
		return
	}
	if path == "/" {
		path = ""
	}
	for _, ent := range v {
		walk(path+"/"+ent, st, ch)
	}
}
<|endoftext|>


<commit_before>// Copyright 2017 phcurtis fn Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package fn_test

import (
	"fmt"
	"io"
	"log"
	"os"
	"testing"

	"github.com/phcurtis/fn"
)

func Test_pkgcfggroup(t *testing.T) {
	fnpass := "Test_pkgcfggroup - test passed "

	// subTESTS:
	// #1 (PkgCfgDef)            verify PkgCfgDef returns expected values
	// #2 (PkgCfg)               verify individual setter funcs alter what makes up a PkgCfg
	// #3 (SetPkgCfgDef/PkgCfg)) verify rewriting defaults matches what should be in the pkgCfgDef
	// #4 (PkgCfg)               verify setter funcs mods match a corresponding fetched pkgCfg
	// #5 (SetPkgCfg/PkgCfg))    verify setting a pkgcfg matches returned values from PkgCfg
	// on completion of these test restore pkgCfgDef

	// prelims
	var got, want *fn.PkgCfgStruct
	var giowr, wiowr io.Writer
	var gotstr, wantstr, pkgdefwantstr string
	defiowr := fn.LogGetOutputDef()
	note := fmt.Sprintf("\n    Note:os.Stdout:%v \n    Note:os.Stderr:%v", os.Stdout, os.Stderr)
	pkgDefWant := &fn.PkgCfgStruct{
		LogFlags:      fn.LflagsDef,
		LogPrefix:     fn.LogPrefixDef,
		LogTraceFlags: fn.TrFlagsDef,
		LogAlignFile:  fn.LogAlignFileDef,
		LogAlignFunc:  fn.LogAlignFuncDef,
	}
	pkgdefwantstr = fmt.Sprintf("%+v", pkgDefWant)

	// #1 (PkgCfgDef)    verify PkgCfgDef returns expected values
	want = pkgDefWant
	got, _ = fn.PkgCfgDef()
	wantstr = pkgdefwantstr
	gotstr = fmt.Sprintf("%+v", got)
	if gotstr != wantstr {
		t.Errorf("#1: PkgCfgDef() incorrect:\n got:%s \nwant:%s\n", gotstr, wantstr)
	}
	if testing.Verbose() {
		log.Println(fnpass + "#1 (PkgCfgDef)")
	}

	// #2 (PkgCfg)       verify individual setter funcs alter what makes up a PkgCfg
	fn.LogSetAlignFile(5)
	fn.LogSetAlignFunc(6)
	fn.LogSetFlags(0xffffffff)
	fn.LogSetPrefix("pRe")
	fn.LogSetTraceFlags(0xdeadbeef) //3735928559
	wiowr = os.Stderr
	fn.LogSetOutput(wiowr)
	got, giowr = fn.PkgCfg()
	gotstr = fmt.Sprintf("%+v", got)
	if gotstr == wantstr {
		t.Errorf("#2: PkgCfg() incorrect:\n got:%s \nwant:%s \n", gotstr, wantstr)
	}
	if giowr != wiowr {
		t.Errorf("#2a: PkgCfg() incorrect:\n gotiowr:%x \nwantowr:%x %s\n",
			giowr, wiowr, note)
	}
	if testing.Verbose() {
		log.Println(fnpass + "#2 (PkgCfg)")
	}

	// #3 (SetPkgCfgDef/PkgCfg) verify rewriting the defaults matches what should be in the pkg defaults
	fn.SetPkgCfgDef(true)
	wiowr = defiowr
	got, giowr = fn.PkgCfg()
	gotstr = fmt.Sprintf("%+v", got)
	if gotstr != wantstr {
		t.Errorf("#3: SetPkgCfgDef() incorrect:\n got:%s \nwant:%s %s\n", gotstr, wantstr, note)
	}
	if giowr != wiowr {
		t.Errorf("#3a: PkgCfgDef() incorrect:\n gotiowr:%x \nwantiowr:%x %s\n", giowr, wiowr, note)
	}
	if testing.Verbose() {
		log.Println(fnpass + "#3 (SetPkgCfgDef/PkgCfg)")
	}

	// #4 (PkgCfg)       verify setter funcs mods  match a corresponding fetched pkgCfg
	fn.SetPkgCfgDef(true)
	wiowr = os.Stderr
	fn.LogSetOutput(wiowr)
	fn.LogSetFlags(0xffff)
	fn.LogSetPrefix("ZyxAbcd")
	fn.LogSetTraceFlags(0xbeef) //dec=48879
	fn.LogSetAlignFile(11)
	fn.LogSetAlignFunc(12)
	got, _ = fn.PkgCfg()
	want = &fn.PkgCfgStruct{
		LogFlags:      0xffff,
		LogPrefix:     "ZyxAbcd",
		LogTraceFlags: 0xbeef, //dec=48879
		LogAlignFile:  11,
		LogAlignFunc:  12,
	}
	got, giowr = fn.PkgCfg()
	gotstr = fmt.Sprintf("%+v", got)
	wantstr = fmt.Sprintf("%+v", want)
	if gotstr != wantstr {
		t.Errorf(" #4: PkgCfgDef() incorrect:\n got:%s \nwant:%s \n", gotstr, wantstr)
	}
	if giowr != wiowr {
		t.Errorf("#4a: PkgCfgDef() incorrect:\n gotiowr:%x \nwantiowr:%x %s\n", giowr, wiowr, note)
	}
	if testing.Verbose() {
		log.Println(fnpass + "#4 (PkgCfg)")
	}

	// #5 (SetPkgCfg/PkgCfg)    verify setting a pkgcfg matches returned values from  PkgCfg
	want.LogAlignFile = 8
	wiowr = os.Stdout
	fn.SetPkgCfg(want, wiowr)
	got, giowr = fn.PkgCfg()
	gotstr = fmt.Sprintf("%+v", got)
	wantstr = fmt.Sprintf("%+v", want)
	if gotstr != wantstr {
		t.Fatalf(" #5: SetPkgCfg() incorrect:\n got:%s \nwant:%s ", gotstr, wantstr)
	}
	if giowr != wiowr {
		t.Fatalf("#5a: SetPkgCfg() incorrect:\n gotiowr:%x \nwantiowr:%x ", giowr, wiowr)
	}
	if testing.Verbose() {
		log.Println(fnpass + "#5 (SetPkgCfg/PkgCfg)")
	}

	// reset settings to package defaults
	fn.SetPkgCfgDef(true)
	if testing.Verbose() {
		log.Println("restored pkgCfgDef")
	}
}
<commit_msg>refactored code<commit_after>// Copyright 2017 phcurtis fn Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package fn_test

import (
	"fmt"
	"io"
	"os"
	"testing"

	"github.com/phcurtis/fn"
)

var pkgCfgDefWant = &fn.PkgCfgStruct{
	LogFlags:      fn.LflagsDef,
	LogPrefix:     fn.LogPrefixDef,
	LogTraceFlags: fn.TrFlagsDef,
	LogAlignFile:  fn.LogAlignFileDef,
	LogAlignFunc:  fn.LogAlignFuncDef,
}
var pkgCfgDefWantstr = fmt.Sprintf("%+v", pkgCfgDefWant)
var noteStdio = fmt.Sprintf("\n    Note:os.Stdout:%v \n    Note:os.Stderr:%v", os.Stdout, os.Stderr)

func TestPkgCfgDef(t *testing.T) {
	// verify PkgCfgDef returns expected values

	fn.SetPkgCfgDef(true) // set pkg config to (what should be) a known default state
	got, giowr := fn.PkgCfgDef()
	gotstr := fmt.Sprintf("%+v", got)
	wantstr := pkgCfgDefWantstr
	if gotstr != wantstr {
		t.Errorf("PkgCfgDef() incorrect:\n got:%s \nwant:%s\n", gotstr, wantstr)
	}
	wiowr := fn.LogGetOutputDef()
	if giowr != wiowr {
		t.Errorf("PkgCfgDef() incorrect:\n got-iowr:%s \nwant-iowr:%s\n", giowr, wiowr)
	}

	fn.SetPkgCfgDef(true) // set pkg config to (what should be) a known default state
}

func TestPkgCfg(t *testing.T) {
	// verify individual setter funcs alter what makes up a PkgCfg

	fn.SetPkgCfgDef(true) // set pkg config to (what should be) a known default state
	fn.LogSetAlignFile(5)
	fn.LogSetAlignFunc(6)
	fn.LogSetFlags(0xffffffff)
	fn.LogSetPrefix("pRe")
	fn.LogSetTraceFlags(0xdeadbeef) //3735928559
	var wiowr io.Writer = os.Stderr
	fn.LogSetOutput(wiowr)
	got, giowr := fn.PkgCfg()
	gotstr := fmt.Sprintf("%+v", got)
	wantstr := pkgCfgDefWantstr
	if gotstr == wantstr {
		t.Errorf("PkgCfg() incorrect:\n got:%s \nwant:%s \n", gotstr, wantstr)
	}
	if giowr != wiowr {
		t.Errorf("PkgCfg() incorrect:\n gotiowr:%x \nwantowr:%x %s\n",
			giowr, wiowr, noteStdio)
	}
	fn.SetPkgCfgDef(true) // set pkg config to (what should be) a known default state
}

func TestSetPkgCfgDef(t *testing.T) {
	// verify rewriting the defaults matches what should be in the pkg defaults

	fn.SetPkgCfgDef(true) // set pkg config to (what should be) a known default state
	wiowr := fn.LogGetOutputDef()
	got, giowr := fn.PkgCfg()
	gotstr := fmt.Sprintf("%+v", got)
	wantstr := pkgCfgDefWantstr
	if gotstr != wantstr {
		t.Errorf("SetPkgCfgDef() incorrect:\n got:%s \nwant:%s %s\n", gotstr, wantstr, noteStdio)
	}
	if giowr != wiowr {
		t.Errorf("PkgCfgDef() incorrect:\n gotiowr:%x \nwantiowr:%x %s\n", giowr, wiowr, noteStdio)
	}
	fn.SetPkgCfgDef(true) // set pkg config to (what should be) a known default state
}

func Test_fetchedpkgconfig(t *testing.T) {
	//  verify setter funcs mods  match a corresponding fetched pkgCfg
	fn.SetPkgCfgDef(true) // set pkg config to (what should be) a known default state
	wiowr := io.Writer(os.Stderr)
	fn.LogSetOutput(wiowr)
	fn.LogSetFlags(0xffff)
	fn.LogSetPrefix("ZyxAbcd")
	fn.LogSetTraceFlags(0xbeef) //dec=48879
	fn.LogSetAlignFile(11)
	fn.LogSetAlignFunc(12)
	got, _ := fn.PkgCfg()
	want := &fn.PkgCfgStruct{
		LogFlags:      0xffff,
		LogPrefix:     "ZyxAbcd",
		LogTraceFlags: 0xbeef, //dec=48879
		LogAlignFile:  11,
		LogAlignFunc:  12,
	}
	var giowr io.Writer
	got, giowr = fn.PkgCfg()
	gotstr := fmt.Sprintf("%+v", got)
	wantstr := fmt.Sprintf("%+v", want)
	if gotstr != wantstr {
		t.Errorf("PkgCfgDef() incorrect:\n got:%s \nwant:%s \n", gotstr, wantstr)
	}
	if giowr != wiowr {
		t.Errorf("PkgCfgDef() incorrect:\n gotiowr:%x \nwantiowr:%x %s\n", giowr, wiowr, noteStdio)
	}

	fn.SetPkgCfgDef(true) // set pkg config to (what should be) a known default state
}

func Test_pkgcfg_matches_setpkgcfg(t *testing.T) {
	// verify setting a pkgcfg matches returned values from  PkgCfg

	fn.SetPkgCfgDef(true) // set pkg config to (what should be) a known default state
	want, _ := fn.PkgCfgDef()
	wiowr := fn.LogGetOutputDef()
	want.LogAlignFile = 8
	fn.SetPkgCfg(want, wiowr)
	got, giowr := fn.PkgCfg()
	gotstr := fmt.Sprintf("%+v", got)
	wantstr := fmt.Sprintf("%+v", want)
	if gotstr != wantstr {
		t.Fatalf("SetPkgCfg() incorrect:\n got:%s \nwant:%s ", gotstr, wantstr)
	}
	if giowr != wiowr {
		t.Fatalf("SetPkgCfg() incorrect:\n gotiowr:%x \nwantiowr:%x ", giowr, wiowr)
	}

	fn.SetPkgCfgDef(true) // set pkg config to (what should be) a known default state
}
<|endoftext|>


<commit_before>package cli

import (
	"fmt"
	"os"

	"github.com/containerum/chkit/pkg/cli/configmap"
	"github.com/containerum/chkit/pkg/cli/containerumapi"
	"github.com/containerum/chkit/pkg/cli/deployment"
	"github.com/containerum/chkit/pkg/cli/ingress"
	"github.com/containerum/chkit/pkg/cli/namespace"
	"github.com/containerum/chkit/pkg/cli/pod"
	"github.com/containerum/chkit/pkg/cli/postrun"
	"github.com/containerum/chkit/pkg/cli/prerun"
	"github.com/containerum/chkit/pkg/cli/service"
	"github.com/containerum/chkit/pkg/cli/solution"
	"github.com/containerum/chkit/pkg/cli/user"
	"github.com/containerum/chkit/pkg/cli/volume"
	"github.com/containerum/chkit/pkg/configuration"
	"github.com/containerum/chkit/pkg/context"
	"github.com/spf13/cobra"
)

func Get(ctx *context.Context) *cobra.Command {
	command := &cobra.Command{
		Use:   "get",
		Short: "Get resource data",
		Run: func(command *cobra.Command, args []string) {
			command.Help()
		},
		PersistentPostRun: postrun.PostRunFunc(ctx),
	}
	command.AddCommand(
		prerun.WithInit(ctx, clideployment.Get),      //
		prerun.WithInit(ctx, clinamespace.Get),       //
		prerun.WithInit(ctx, clinamespace.GetAccess), //
		prerun.WithInit(ctx, cliserv.Get),            //
		prerun.WithInit(ctx, clipod.Get),             //
		prerun.WithInit(ctx, clingress.Get),          //
		prerun.WithInit(ctx, cliuser.Get),            //
		prerun.WithInit(ctx, clisolution.Get),        //
		containerumapi.Get(ctx),                      //
		prerun.WithInit(ctx, cliconfigmap.Get),       //
		prerun.WithInit(ctx, volume.Get),             //
		prerun.WithInit(ctx, clideployment.GetVersions),
		&cobra.Command{
			Use:     "default-namespace",
			Short:   "print default",
			Aliases: []string{"default-ns", "def-ns"},
			PreRun: func(cmd *cobra.Command, args []string) {
				if err := configuration.SyncConfig(ctx); err != nil {
					fmt.Printf("Unable to setup config:\n%v\n", err)
					os.Exit(1)
				}
			},
			Run: func(cmd *cobra.Command, args []string) {
				fmt.Printf("%s\n", ctx.Namespace)
			},
		},
	)
	command.PersistentFlags().
		StringP("namespace", "n", ctx.Namespace.ID, "")
	return command
}
<commit_msg>switch off volume get<commit_after>package cli

import (
	"fmt"
	"os"

	"github.com/containerum/chkit/pkg/cli/configmap"
	"github.com/containerum/chkit/pkg/cli/containerumapi"
	"github.com/containerum/chkit/pkg/cli/deployment"
	"github.com/containerum/chkit/pkg/cli/ingress"
	"github.com/containerum/chkit/pkg/cli/namespace"
	"github.com/containerum/chkit/pkg/cli/pod"
	"github.com/containerum/chkit/pkg/cli/postrun"
	"github.com/containerum/chkit/pkg/cli/prerun"
	"github.com/containerum/chkit/pkg/cli/service"
	"github.com/containerum/chkit/pkg/cli/solution"
	"github.com/containerum/chkit/pkg/cli/user"
	"github.com/containerum/chkit/pkg/configuration"
	"github.com/containerum/chkit/pkg/context"
	"github.com/spf13/cobra"
)

func Get(ctx *context.Context) *cobra.Command {
	command := &cobra.Command{
		Use:   "get",
		Short: "Get resource data",
		Run: func(command *cobra.Command, args []string) {
			command.Help()
		},
		PersistentPostRun: postrun.PostRunFunc(ctx),
	}
	command.AddCommand(
		prerun.WithInit(ctx, clideployment.Get),      //
		prerun.WithInit(ctx, clinamespace.Get),       //
		prerun.WithInit(ctx, clinamespace.GetAccess), //
		prerun.WithInit(ctx, cliserv.Get),            //
		prerun.WithInit(ctx, clipod.Get),             //
		prerun.WithInit(ctx, clingress.Get),          //
		prerun.WithInit(ctx, cliuser.Get),            //
		prerun.WithInit(ctx, clisolution.Get),        //
		containerumapi.Get(ctx),                      //
		prerun.WithInit(ctx, cliconfigmap.Get),       //
		//prerun.WithInit(ctx, volume.Get),             //
		prerun.WithInit(ctx, clideployment.GetVersions),
		&cobra.Command{
			Use:     "default-namespace",
			Short:   "print default",
			Aliases: []string{"default-ns", "def-ns"},
			PreRun: func(cmd *cobra.Command, args []string) {
				if err := configuration.SyncConfig(ctx); err != nil {
					fmt.Printf("Unable to setup config:\n%v\n", err)
					os.Exit(1)
				}
			},
			Run: func(cmd *cobra.Command, args []string) {
				fmt.Printf("%s\n", ctx.Namespace)
			},
		},
	)
	command.PersistentFlags().
		StringP("namespace", "n", ctx.Namespace.ID, "")
	return command
}
<|endoftext|>


<commit_before>/*
Copyright 2020 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package gcp

import (
	"github.com/pkg/errors"

	"k8s.io/release/pkg/command"
)

const (
	GCloudExecutable = "gcloud"
	gSUtilExecutable = "gsutil"
)

// PreCheck checks if all requirements are fulfilled to run this package and
// all sub-packages
func PreCheck() error {
	for _, e := range []string{
		GCloudExecutable,
		gSUtilExecutable,
	} {
		if !command.Available(e) {
			return errors.Errorf(
				"%s executable is not available in $PATH", e,
			)
		}
	}

	return nil
}

// GSUtil can be used to run a gsutil command
func GSUtil(args ...string) error {
	return command.New(gSUtilExecutable, args...).RunSilentSuccess()
}

// GSUtilOutput can be used to run a gsutil command while capturing its output
func GSUtilOutput(args ...string) (string, error) {
	stream, err := command.New(gSUtilExecutable, args...).RunSilentSuccessOutput()
	if err != nil {
		return "", errors.Wrapf(err, "executing %s", gSUtilExecutable)
	}
	return stream.OutputTrimNL(), nil
}
<commit_msg>Rename gSUtilExecutable -> gsutilExecutable<commit_after>/*
Copyright 2020 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package gcp

import (
	"github.com/pkg/errors"

	"k8s.io/release/pkg/command"
)

const (
	GCloudExecutable = "gcloud"
	gsutilExecutable = "gsutil"
)

// PreCheck checks if all requirements are fulfilled to run this package and
// all sub-packages
func PreCheck() error {
	for _, e := range []string{
		GCloudExecutable,
		gsutilExecutable,
	} {
		if !command.Available(e) {
			return errors.Errorf(
				"%s executable is not available in $PATH", e,
			)
		}
	}

	return nil
}

// GSUtil can be used to run a gsutil command
func GSUtil(args ...string) error {
	return command.New(gsutilExecutable, args...).RunSilentSuccess()
}

// GSUtilOutput can be used to run a gsutil command while capturing its output
func GSUtilOutput(args ...string) (string, error) {
	stream, err := command.New(gsutilExecutable, args...).RunSilentSuccessOutput()
	if err != nil {
		return "", errors.Wrapf(err, "executing %s", gsutilExecutable)
	}
	return stream.OutputTrimNL(), nil
}
<|endoftext|>