<commit_before>package genapp

import (
	"fmt"
	"os"
	"path/filepath"
	"regexp"
	"strconv"
	"text/template"

	"github.com/goadesign/goa/design"
	"github.com/goadesign/goa/goagen/codegen"
)

func makeTestDir(g *Generator, apiName string) (outDir string, err error) {
	outDir = filepath.Join(g.outDir, "test")
	if err = os.RemoveAll(outDir); err != nil {
		return
	}
	if err = os.MkdirAll(outDir, 0755); err != nil {
		return
	}
	g.genfiles = append(g.genfiles, outDir)
	return
}

// TestMethod structure
type TestMethod struct {
	Name           string
	Comment        string
	ResourceName   string
	ActionName     string
	ControllerName string
	ContextVarName string
	ContextType    string
	RouteVerb      string
	FullPath       string
	Status         int
	ReturnType     *ObjectType
	Params         []ObjectType
	Payload        *ObjectType
}

// ObjectType structure
type ObjectType struct {
	Label       string
	Name        string
	Type        string
	Pointer     string
	Validatable bool
}

func (g *Generator) generateResourceTest(api *design.APIDefinition) error {
	if len(api.Resources) == 0 {
		return nil
	}
	testTmpl := template.Must(template.New("resources").Parse(testTmpl))
	outDir, err := makeTestDir(g, api.Name)
	if err != nil {
		return err
	}
	appPkg, err := codegen.PackagePath(g.outDir)
	if err != nil {
		return err
	}
	imports := []*codegen.ImportSpec{
		codegen.SimpleImport("bytes"),
		codegen.SimpleImport("fmt"),
		codegen.SimpleImport("net/http"),
		codegen.SimpleImport("net/http/httptest"),
		codegen.SimpleImport("net/url"),
		codegen.SimpleImport("testing"),
		codegen.SimpleImport(appPkg),
		codegen.SimpleImport("github.com/goadesign/goa"),
		codegen.SimpleImport("github.com/goadesign/goa/goatest"),
		codegen.SimpleImport("golang.org/x/net/context"),
	}

	return api.IterateResources(func(res *design.ResourceDefinition) error {
		filename := filepath.Join(outDir, codegen.SnakeCase(res.Name)+".go")
		file, err := codegen.SourceFileFor(filename)
		if err != nil {
			return err
		}
		if err := file.WriteHeader("", "test", imports); err != nil {
			return err
		}

		var methods = []TestMethod{}

		if err := res.IterateActions(func(action *design.ActionDefinition) error {
			if err := action.IterateResponses(func(response *design.ResponseDefinition) error {
				if response.Status == 101 { // SwitchingProtocols, Don't currently handle WebSocket endpoints
					return nil
				}
				for routeIndex, route := range action.Routes {
					mediaType := design.Design.MediaTypeWithIdentifier(response.MediaType)
					if mediaType == nil {
						methods = append(methods, g.createTestMethod(res, action, response, route, routeIndex, nil, nil))
					} else {
						if err := mediaType.IterateViews(func(view *design.ViewDefinition) error {
							methods = append(methods, g.createTestMethod(res, action, response, route, routeIndex, mediaType, view))
							return nil
						}); err != nil {
							return err
						}
					}
				}
				return nil
			}); err != nil {
				return err
			}
			return nil
		}); err != nil {
			return err
		}
		g.genfiles = append(g.genfiles, filename)
		err = testTmpl.Execute(file, methods)
		if err != nil {
			panic(err)
		}
		return file.FormatCode()
	})
}

func (g *Generator) createTestMethod(resource *design.ResourceDefinition, action *design.ActionDefinition, response *design.ResponseDefinition, route *design.RouteDefinition, routeIndex int, mediaType *design.MediaTypeDefinition, view *design.ViewDefinition) TestMethod {
	routeNameQualifier := suffixRoute(action.Routes, routeIndex)
	viewNameQualifier := func() string {
		if view != nil && view.Name != "default" {
			return view.Name
		}
		return ""
	}()
	method := TestMethod{}
	method.Name = fmt.Sprintf("%s%s%s%s%s", codegen.Goify(action.Name, true), codegen.Goify(resource.Name, true), codegen.Goify(response.Name, true), routeNameQualifier, codegen.Goify(viewNameQualifier, true))
	method.ActionName = codegen.Goify(action.Name, true)
	method.ResourceName = codegen.Goify(resource.Name, true)
	method.Comment = fmt.Sprintf("test setup")
	method.ControllerName = fmt.Sprintf("%s.%sController", g.target, codegen.Goify(resource.Name, true))
	method.ContextVarName = fmt.Sprintf("%sCtx", codegen.Goify(action.Name, false))
	method.ContextType = fmt.Sprintf("%s.New%s%sContext", g.target, codegen.Goify(action.Name, true), codegen.Goify(resource.Name, true))
	method.RouteVerb = route.Verb
	method.Status = response.Status
	method.FullPath = goPathFormat(route.FullPath())

	if view != nil && mediaType != nil {
		p, _, err := mediaType.Project(view.Name)
		if err != nil {
			panic(err)
		}
		tmp := codegen.GoTypeName(p, nil, 0, false)
		if !p.IsBuiltIn() {
			tmp = fmt.Sprintf("%s.%s", g.target, tmp)
		}
		validate := codegen.RecursiveChecker(p.AttributeDefinition, false, false, false, "payload", "raw", 1, true)

		returnType := ObjectType{}
		returnType.Type = tmp
		returnType.Pointer = "*"
		returnType.Validatable = validate != ""

		method.ReturnType = &returnType
	}

	if len(route.Params()) > 0 {
		var params = []ObjectType{}
		for _, paramName := range route.Params() {
			for name, att := range action.Params.Type.ToObject() {
				if name == paramName {
					param := ObjectType{}
					param.Label = name
					param.Name = codegen.Goify(name, false)
					param.Type = codegen.GoTypeRef(att.Type, nil, 0, false)
					if att.Type.IsPrimitive() && action.Params.IsPrimitivePointer(name) {
						param.Pointer = "*"
					}
					params = append(params, param)
				}
			}
		}
		method.Params = params
	}

	if action.Payload != nil {
		payload := ObjectType{}
		payload.Name = "payload"
		payload.Type = fmt.Sprintf("%s.%s", g.target, codegen.Goify(action.Payload.TypeName, true))
		if !action.Payload.IsPrimitive() && !action.Payload.IsArray() && !action.Payload.IsHash() {
			payload.Pointer = "*"
		}

		validate := codegen.RecursiveChecker(action.Payload.AttributeDefinition, false, false, false, "payload", "raw", 1, false)
		if validate != "" {
			payload.Validatable = true
		}
		method.Payload = &payload
	}
	return method
}

func goPathFormat(path string) string {
	re := regexp.MustCompile(":[a-zA-Z]+")
	return re.ReplaceAllString(path, "%v")
}

func suffixRoute(routes []*design.RouteDefinition, currIndex int) string {
	if len(routes) > 1 && currIndex > 0 {
		return strconv.Itoa(currIndex)
	}
	return ""
}

var testTmpl = `
{{ range $test := . }}
// {{ $test.Name }} {{ $test.Comment }}
func {{ $test.Name }}(t *testing.T, ctrl {{ $test.ControllerName}}{{/*
*/}}{{ range $param := $test.Params }}, {{ $param.Name }} {{ $param.Pointer }}{{ $param.Type }}{{ end }}{{/*
*/}}{{ if $test.Payload }}, {{ $test.Payload.Name }} {{ $test.Payload.Pointer }}{{ $test.Payload.Type }}{{ end }}){{/*
*/}}{{ if $test.ReturnType }} {{ $test.ReturnType.Pointer }}{{ $test.ReturnType.Type }}{{ end }} {
	{{ if $test.ReturnType }}return {{ end }}{{ $test.Name }}Ctx(t, context.Background(), ctrl{{/*
*/}}{{ range $param := $test.Params }}, {{ $param.Name }}{{ end }}{{ if $test.Payload }}, {{ $test.Payload.Name }}{{ end }})
}

// {{ $test.Name }}Ctx {{ $test.Comment }}
func {{ $test.Name }}Ctx(t *testing.T, ctx context.Context, ctrl {{ $test.ControllerName}}{{/*
*/}}{{ range $param := $test.Params }}, {{ $param.Name }} {{ $param.Pointer }}{{ $param.Type }}{{ end }}{{/*
*/}}{{ if $test.Payload }}, {{ $test.Payload.Name }} {{ $test.Payload.Pointer }}{{ $test.Payload.Type }}{{ end }}){{/*
*/}}{{ if $test.ReturnType }} {{ $test.ReturnType.Pointer }}{{ $test.ReturnType.Type }}{{ end }} { {{/*
*/}}{{ if $test.Payload }}{{ if $test.Payload.Validatable }}
	err := {{ $test.Payload.Name }}.Validate()
	if err != nil {
		panic(err)
	}{{ end }}{{ end }}
	var logBuf bytes.Buffer
	var resp interface{}
	respSetter := func(r interface{}) { resp = r }
	service := goatest.Service(&logBuf, respSetter)
	rw := httptest.NewRecorder()
	req, err := http.NewRequest("{{ $test.RouteVerb }}", fmt.Sprintf("{{ $test.FullPath }}"{{ range $param := $test.Params }}, {{ $param.Name }}{{ end }}), nil)
	if err != nil {
		panic("invalid test " + err.Error()) // bug
	}
	prms := url.Values{}
	{{ range $param := $test.Params }}prms["{{ $param.Label }}"] = []string{fmt.Sprintf("%v",{{ $param.Name}})}
	{{ end }}
	goaCtx := goa.NewContext(goa.WithAction(ctx, "{{ $test.ResourceName }}Test"), rw, req, prms)
	{{ $test.ContextVarName }}, err := {{ $test.ContextType }}(goaCtx, service){{ if $test.Payload }}
	{{ $test.ContextVarName }}.Payload = {{ $test.Payload.Name }}
	{{ end }}
	if err != nil {
		panic("invalid test data " + err.Error()) // bug
	}
	err = ctrl.{{ $test.ActionName}}({{ $test.ContextVarName }})
	if err != nil {
		t.Fatalf("controller returned %s, logs:\n%s", err, logBuf.String())
	}
	{{if $test.ReturnType }}
	a, ok := resp.({{ $test.ReturnType.Pointer }}{{ $test.ReturnType.Type }})
	if !ok {
		t.Errorf("invalid response media: got %+v, expected instance of {{ $test.ReturnType.Type }}", resp)
	}
	{{ end }}
	if rw.Code != {{ $test.Status }} {
		t.Errorf("invalid response status code: got %+v, expected {{ $test.Status }}", rw.Code)
	}
	{{ if $test.ReturnType }}{{ if $test.ReturnType.Validatable }}
	err = a.Validate()
	if err != nil {
		t.Errorf("invalid response payload: got %v", err)
	}
	{{ end }}return a
	{{ end }}
}
{{ end }}`
<commit_msg>Fix payload validations in app tests (#497)<commit_after>package genapp

import (
	"fmt"
	"os"
	"path/filepath"
	"regexp"
	"strconv"
	"text/template"

	"github.com/goadesign/goa/design"
	"github.com/goadesign/goa/goagen/codegen"
)

func makeTestDir(g *Generator, apiName string) (outDir string, err error) {
	outDir = filepath.Join(g.outDir, "test")
	if err = os.RemoveAll(outDir); err != nil {
		return
	}
	if err = os.MkdirAll(outDir, 0755); err != nil {
		return
	}
	g.genfiles = append(g.genfiles, outDir)
	return
}

// TestMethod structure
type TestMethod struct {
	Name           string
	Comment        string
	ResourceName   string
	ActionName     string
	ControllerName string
	ContextVarName string
	ContextType    string
	RouteVerb      string
	FullPath       string
	Status         int
	ReturnType     *ObjectType
	Params         []ObjectType
	Payload        *ObjectType
}

// ObjectType structure
type ObjectType struct {
	Label       string
	Name        string
	Type        string
	Pointer     string
	Validatable bool
}

func (g *Generator) generateResourceTest(api *design.APIDefinition) error {
	if len(api.Resources) == 0 {
		return nil
	}
	testTmpl := template.Must(template.New("resources").Parse(testTmpl))
	outDir, err := makeTestDir(g, api.Name)
	if err != nil {
		return err
	}
	appPkg, err := codegen.PackagePath(g.outDir)
	if err != nil {
		return err
	}
	imports := []*codegen.ImportSpec{
		codegen.SimpleImport("bytes"),
		codegen.SimpleImport("fmt"),
		codegen.SimpleImport("net/http"),
		codegen.SimpleImport("net/http/httptest"),
		codegen.SimpleImport("net/url"),
		codegen.SimpleImport("testing"),
		codegen.SimpleImport(appPkg),
		codegen.SimpleImport("github.com/goadesign/goa"),
		codegen.SimpleImport("github.com/goadesign/goa/goatest"),
		codegen.SimpleImport("golang.org/x/net/context"),
	}

	return api.IterateResources(func(res *design.ResourceDefinition) error {
		filename := filepath.Join(outDir, codegen.SnakeCase(res.Name)+".go")
		file, err := codegen.SourceFileFor(filename)
		if err != nil {
			return err
		}
		if err := file.WriteHeader("", "test", imports); err != nil {
			return err
		}

		var methods = []TestMethod{}

		if err := res.IterateActions(func(action *design.ActionDefinition) error {
			if err := action.IterateResponses(func(response *design.ResponseDefinition) error {
				if response.Status == 101 { // SwitchingProtocols, Don't currently handle WebSocket endpoints
					return nil
				}
				for routeIndex, route := range action.Routes {
					mediaType := design.Design.MediaTypeWithIdentifier(response.MediaType)
					if mediaType == nil {
						methods = append(methods, g.createTestMethod(res, action, response, route, routeIndex, nil, nil))
					} else {
						if err := mediaType.IterateViews(func(view *design.ViewDefinition) error {
							methods = append(methods, g.createTestMethod(res, action, response, route, routeIndex, mediaType, view))
							return nil
						}); err != nil {
							return err
						}
					}
				}
				return nil
			}); err != nil {
				return err
			}
			return nil
		}); err != nil {
			return err
		}
		g.genfiles = append(g.genfiles, filename)
		err = testTmpl.Execute(file, methods)
		if err != nil {
			panic(err)
		}
		return file.FormatCode()
	})
}

func (g *Generator) createTestMethod(resource *design.ResourceDefinition, action *design.ActionDefinition, response *design.ResponseDefinition, route *design.RouteDefinition, routeIndex int, mediaType *design.MediaTypeDefinition, view *design.ViewDefinition) TestMethod {
	routeNameQualifier := suffixRoute(action.Routes, routeIndex)
	viewNameQualifier := func() string {
		if view != nil && view.Name != "default" {
			return view.Name
		}
		return ""
	}()
	method := TestMethod{}
	method.Name = fmt.Sprintf("%s%s%s%s%s", codegen.Goify(action.Name, true), codegen.Goify(resource.Name, true), codegen.Goify(response.Name, true), routeNameQualifier, codegen.Goify(viewNameQualifier, true))
	method.ActionName = codegen.Goify(action.Name, true)
	method.ResourceName = codegen.Goify(resource.Name, true)
	method.Comment = fmt.Sprintf("test setup")
	method.ControllerName = fmt.Sprintf("%s.%sController", g.target, codegen.Goify(resource.Name, true))
	method.ContextVarName = fmt.Sprintf("%sCtx", codegen.Goify(action.Name, false))
	method.ContextType = fmt.Sprintf("%s.New%s%sContext", g.target, codegen.Goify(action.Name, true), codegen.Goify(resource.Name, true))
	method.RouteVerb = route.Verb
	method.Status = response.Status
	method.FullPath = goPathFormat(route.FullPath())

	if view != nil && mediaType != nil {
		p, _, err := mediaType.Project(view.Name)
		if err != nil {
			panic(err)
		}
		tmp := codegen.GoTypeName(p, nil, 0, false)
		if !p.IsBuiltIn() {
			tmp = fmt.Sprintf("%s.%s", g.target, tmp)
		}
		validate := codegen.RecursiveChecker(p.AttributeDefinition, false, false, false, "payload", "raw", 1, true)

		returnType := ObjectType{}
		returnType.Type = tmp
		returnType.Pointer = "*"
		returnType.Validatable = validate != ""

		method.ReturnType = &returnType
	}

	if len(route.Params()) > 0 {
		var params = []ObjectType{}
		for _, paramName := range route.Params() {
			for name, att := range action.Params.Type.ToObject() {
				if name == paramName {
					param := ObjectType{}
					param.Label = name
					param.Name = codegen.Goify(name, false)
					param.Type = codegen.GoTypeRef(att.Type, nil, 0, false)
					if att.Type.IsPrimitive() && action.Params.IsPrimitivePointer(name) {
						param.Pointer = "*"
					}
					params = append(params, param)
				}
			}
		}
		method.Params = params
	}

	if action.Payload != nil {
		payload := ObjectType{}
		payload.Name = "payload"
		payload.Type = fmt.Sprintf("%s.%s", g.target, codegen.Goify(action.Payload.TypeName, true))
		if !action.Payload.IsPrimitive() && !action.Payload.IsArray() && !action.Payload.IsHash() {
			payload.Pointer = "*"
		}

		validate := codegen.RecursiveChecker(action.Payload.AttributeDefinition, false, false, false, "payload", "raw", 1, false)
		if validate != "" {
			payload.Validatable = true
		}
		method.Payload = &payload
	}
	return method
}

func goPathFormat(path string) string {
	re := regexp.MustCompile(":[a-zA-Z]+")
	return re.ReplaceAllString(path, "%v")
}

func suffixRoute(routes []*design.RouteDefinition, currIndex int) string {
	if len(routes) > 1 && currIndex > 0 {
		return strconv.Itoa(currIndex)
	}
	return ""
}

var testTmpl = `
{{ range $test := . }}
// {{ $test.Name }} {{ $test.Comment }}
func {{ $test.Name }}(t *testing.T, ctrl {{ $test.ControllerName}}{{/*
*/}}{{ range $param := $test.Params }}, {{ $param.Name }} {{ $param.Pointer }}{{ $param.Type }}{{ end }}{{/*
*/}}{{ if $test.Payload }}, {{ $test.Payload.Name }} {{ $test.Payload.Pointer }}{{ $test.Payload.Type }}{{ end }}){{/*
*/}}{{ if $test.ReturnType }} {{ $test.ReturnType.Pointer }}{{ $test.ReturnType.Type }}{{ end }} {
	{{ if $test.ReturnType }}return {{ end }}{{ $test.Name }}Ctx(t, context.Background(), ctrl{{/*
*/}}{{ range $param := $test.Params }}, {{ $param.Name }}{{ end }}{{ if $test.Payload }}, {{ $test.Payload.Name }}{{ end }})
}

// {{ $test.Name }}Ctx {{ $test.Comment }}
func {{ $test.Name }}Ctx(t *testing.T, ctx context.Context, ctrl {{ $test.ControllerName}}{{/*
*/}}{{ range $param := $test.Params }}, {{ $param.Name }} {{ $param.Pointer }}{{ $param.Type }}{{ end }}{{/*
*/}}{{ if $test.Payload }}, {{ $test.Payload.Name }} {{ $test.Payload.Pointer }}{{ $test.Payload.Type }}{{ end }}){{/*
*/}}{{ if $test.ReturnType }} {{ $test.ReturnType.Pointer }}{{ $test.ReturnType.Type }}{{ end }} { {{/*
*/}}{{ if $test.Payload }}{{ if $test.Payload.Validatable }}
	err := {{ $test.Payload.Name }}.Validate()
	if err != nil {
		e, ok := err.(*goa.Error)
		if !ok {
			panic(err) //bug
		}
		if e.Status != {{ $test.Status }} {
			t.Errorf("unexpected payload validation error: %+v", e)
		}
		{{ if $test.ReturnType }}{{ if $test.ReturnType.Pointer }}return nil{{/*
                                     */}}{{ else }}return {{ $test.ReturnType.Type }}{}{{ end }}{{/*
            */}}{{ else }}return{{ end }}
	}{{ end }}{{ end }}
	var logBuf bytes.Buffer
	var resp interface{}
	respSetter := func(r interface{}) { resp = r }
	service := goatest.Service(&logBuf, respSetter)
	rw := httptest.NewRecorder()
	req, err := http.NewRequest("{{ $test.RouteVerb }}", fmt.Sprintf("{{ $test.FullPath }}"{{ range $param := $test.Params }}, {{ $param.Name }}{{ end }}), nil)
	if err != nil {
		panic("invalid test " + err.Error()) // bug
	}
	prms := url.Values{}
	{{ range $param := $test.Params }}prms["{{ $param.Label }}"] = []string{fmt.Sprintf("%v",{{ $param.Name}})}
	{{ end }}
	goaCtx := goa.NewContext(goa.WithAction(ctx, "{{ $test.ResourceName }}Test"), rw, req, prms)
	{{ $test.ContextVarName }}, err := {{ $test.ContextType }}(goaCtx, service)
	if err != nil {
		panic("invalid test data " + err.Error()) // bug
	}
	{{ if $test.Payload }}{{ $test.ContextVarName }}.Payload = {{ $test.Payload.Name }}{{ end }}

	err = ctrl.{{ $test.ActionName}}({{ $test.ContextVarName }})
	if err != nil {
		t.Fatalf("controller returned %s, logs:\n%s", err, logBuf.String())
	}
	{{if $test.ReturnType }}
	a, ok := resp.({{ $test.ReturnType.Pointer }}{{ $test.ReturnType.Type }})
	if !ok {
		t.Errorf("invalid response media: got %+v, expected instance of {{ $test.ReturnType.Type }}", resp)
	}
	{{ end }}
	if rw.Code != {{ $test.Status }} {
		t.Errorf("invalid response status code: got %+v, expected {{ $test.Status }}", rw.Code)
	}
	{{ if $test.ReturnType }}{{ if $test.ReturnType.Validatable }}
	err = a.Validate()
	if err != nil {
		t.Errorf("invalid response payload: got %v", err)
	}
	{{ end }}return a
	{{ end }}
}
{{ end }}`
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"os"
	"strings"

	"github.com/krasoffski/goplts/gopl/ch04/wordcount"
	"golang.org/x/net/html"
)

func main() {
	noLinks := flag.Bool("nolinks", false, "Do not print links.")
	noCount := flag.Bool("nocount", false, "Do not print element count.")
	noWords := flag.Bool("nowords", false, "Do not print number of words.")
	noText := flag.Bool("notext", false, "Do not print text of nodes.")
	noICS := flag.Bool("noics", false, "Do not print imgs, css, scripts of nodes.")
	flag.Parse()

	if *noICS && *noText && *noLinks && *noCount && *noWords {
		fmt.Fprintln(os.Stderr, "nothing to do, exiting...")
		os.Exit(1)
	}

	doc, err := html.Parse(os.Stdin)

	if err != nil {
		fmt.Fprintf(os.Stderr, "findlinks: %v\n", err)
		os.Exit(1)
	}
	if !*noLinks {
		for _, link := range collectLinks(nil, doc) {
			fmt.Println(link)
		}
	}
	if !*noICS {
		for _, link := range collectICS(nil, doc) {
			fmt.Println(link)
		}
	}
	if !*noCount {
		m := make(map[string]int)
		countElements(m, doc)
		for k, v := range m {
			fmt.Printf("%-10s: % 4d\n", k, v)
		}
	}
	if !*noText {
		for _, text := range collectText(nil, doc) {
			fmt.Println(text)
		}
	}
	if !*noWords {
		m := make(map[string]int)
		total := 0
		countWords(m, doc)
		for k, v := range m {
			total += v
			fmt.Printf("%-10s: % 4d\n", k, v)
		}
		fmt.Printf("Total words: %d\n", total)
	}
}

func getAttrVal(n *html.Node, attr string) (string, bool) {
	for _, a := range n.Attr {
		if a.Key == attr {
			return a.Val, true
		}
	}
	return "", false
}

func countWords(m wordcount.Dict, n *html.Node) {
	if n.Type == html.TextNode {
		if n.Parent.Data != "script" && n.Parent.Data != "style" {
			wordcount.Update(strings.NewReader(n.Data), m)
		}
	}

	for c := n.FirstChild; c != nil; c = c.NextSibling {
		countWords(m, c)
	}
}

func collectICS(links []string, n *html.Node) []string {
	if n == nil {
		return links
	}

	if n.Type == html.ElementNode {
		switch n.Data {
		case "link":
			// NOTE: not efficient approach due to repeated traversing of attrs.
			if val, ok := getAttrVal(n, "type"); ok && val == "text/css" {
				if val, ok := getAttrVal(n, "href"); ok {
					links = append(links, val)
				}
			}
		case "script":
			// NOTE: not efficient approach due to repeated traversing of attrs.
			if val, ok := getAttrVal(n, "type"); ok && val == "text/javascript" {
				if val, ok := getAttrVal(n, "src"); ok {
					links = append(links, val)
				}
			}
		case "img":
			if val, ok := getAttrVal(n, "src"); ok {
				links = append(links, val)
			}
		}
	}
	links = collectICS(links, n.FirstChild)
	return collectICS(links, n.NextSibling)
}

func collectLinks(links []string, n *html.Node) []string {
	if n == nil {
		return links
	}

	if n.Type == html.ElementNode && n.Data == "a" {
		if val, ok := getAttrVal(n, "href"); ok {
			links = append(links, val)
		}
	}
	links = collectLinks(links, n.FirstChild)
	return collectLinks(links, n.NextSibling)
}

func countElements(m map[string]int, n *html.Node) {
	if n.Type == html.ElementNode {
		m[n.Data]++
	}
	for c := n.FirstChild; c != nil; c = c.NextSibling {
		countElements(m, c)
	}
}

func collectText(txt []string, n *html.Node) []string {
	if n == nil {
		return txt
	}

	if n.Type == html.TextNode {
		if n.Parent.Data != "script" && n.Parent.Data != "style" {
			for _, line := range strings.Split(n.Data, "\n") {
				line = strings.TrimLeft(line, " \t")
				if len(line) == 0 {
					continue
				}
				txt = append(txt, line)
			}
		}
	}

	for c := n.FirstChild; c != nil; c = c.NextSibling {
		txt = collectText(txt, c)
	}
	return txt
}
<commit_msg>Fixed cyclomatic complexity for gocyclo.<commit_after>package main

import (
	"flag"
	"fmt"
	"os"
	"strings"

	"github.com/krasoffski/goplts/gopl/ch04/wordcount"
	"golang.org/x/net/html"
)

func main() {
	action := flag.String("action", "links",
		"Action to perform: links|count|words|text|ics")
	flag.Parse()

	if *action == "" {
		fmt.Fprintln(os.Stderr, "choose from: links|count|words|text|ics")
		os.Exit(1)
	}

	doc, err := html.Parse(os.Stdin)

	if err != nil {
		fmt.Fprintf(os.Stderr, "findlinks: %v\n", err)
		os.Exit(1)
	}

	switch *action {
	case "links":
		for _, link := range collectLinks(nil, doc) {
			fmt.Println(link)
		}
	case "count":
		m := make(map[string]int)
		countElements(m, doc)
		for k, v := range m {
			fmt.Printf("%-10s: % 4d\n", k, v)
		}
	case "words":
		m := make(map[string]int)
		total := 0
		countWords(m, doc)
		for k, v := range m {
			total += v
			fmt.Printf("%-10s: % 4d\n", k, v)
		}
		fmt.Printf("Total words: %d\n", total)
	case "text":
		for _, text := range collectText(nil, doc) {
			fmt.Println(text)
		}
	case "ics":
		for _, link := range collectICS(nil, doc) {
			fmt.Println(link)
		}
	default:
		fmt.Fprintln(os.Stderr, "choose from: links|count|words|text|ics")
		os.Exit(1)
	}
}

func getAttrVal(n *html.Node, attr string) (string, bool) {
	for _, a := range n.Attr {
		if a.Key == attr {
			return a.Val, true
		}
	}
	return "", false
}

func countWords(m wordcount.Dict, n *html.Node) {
	if n.Type == html.TextNode {
		if n.Parent.Data != "script" && n.Parent.Data != "style" {
			wordcount.Update(strings.NewReader(n.Data), m)
		}
	}

	for c := n.FirstChild; c != nil; c = c.NextSibling {
		countWords(m, c)
	}
}

func collectICS(links []string, n *html.Node) []string {
	if n == nil {
		return links
	}

	if n.Type == html.ElementNode {
		switch n.Data {
		case "link":
			// NOTE: not efficient approach due to repeated traversing of attrs.
			if val, ok := getAttrVal(n, "type"); ok && val == "text/css" {
				if val, ok := getAttrVal(n, "href"); ok {
					links = append(links, val)
				}
			}
		case "script":
			// NOTE: not efficient approach due to repeated traversing of attrs.
			if val, ok := getAttrVal(n, "type"); ok && val == "text/javascript" {
				if val, ok := getAttrVal(n, "src"); ok {
					links = append(links, val)
				}
			}
		case "img":
			if val, ok := getAttrVal(n, "src"); ok {
				links = append(links, val)
			}
		}
	}
	links = collectICS(links, n.FirstChild)
	return collectICS(links, n.NextSibling)
}

func collectLinks(links []string, n *html.Node) []string {
	if n == nil {
		return links
	}

	if n.Type == html.ElementNode && n.Data == "a" {
		if val, ok := getAttrVal(n, "href"); ok {
			links = append(links, val)
		}
	}
	links = collectLinks(links, n.FirstChild)
	return collectLinks(links, n.NextSibling)
}

func countElements(m map[string]int, n *html.Node) {
	if n.Type == html.ElementNode {
		m[n.Data]++
	}
	for c := n.FirstChild; c != nil; c = c.NextSibling {
		countElements(m, c)
	}
}

func collectText(txt []string, n *html.Node) []string {
	if n == nil {
		return txt
	}

	if n.Type == html.TextNode {
		if n.Parent.Data != "script" && n.Parent.Data != "style" {
			for _, line := range strings.Split(n.Data, "\n") {
				line = strings.TrimLeft(line, " \t")
				if len(line) == 0 {
					continue
				}
				txt = append(txt, line)
			}
		}
	}

	for c := n.FirstChild; c != nil; c = c.NextSibling {
		txt = collectText(txt, c)
	}
	return txt
}
<|endoftext|>


<commit_before>package dfpp

import (
	"bufio"
	"fmt"
	"gopkg.in/op/go-logging.v1"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"runtime"
	"strings"
)

var log = logging.MustGetLogger("dfpp")

type Dfpp struct {
	Processors map[string]func(string, []string) bool
	Output     io.Writer
}

func NewDfpp() *Dfpp {
	pp := &Dfpp{
		Output: os.Stdout,
	}
	pp.Processors = map[string]func(string, []string) bool{
		"INCLUDE": pp.ProcessInclude,
	}
	return pp
}

func (pp *Dfpp) ProcessDockerfile(input io.Reader) {
	for line := range InstructionScanner(input) {
		parts := strings.Fields(line)
		if len(parts) > 0 {
			instruction := parts[0]
			if fn, ok := pp.Processors[instruction]; ok {
				if fn(line, parts) {
					continue
				}
			}
		}
		fmt.Fprintf(pp.Output, "%s\n", line)
	}
}

func InstructionScanner(input io.Reader) chan string {
	ch := make(chan string)
	go func() {
		scanner := bufio.NewScanner(input)
		for scanner.Scan() {
			line := scanner.Text()
			for len(line) > 0 && line[len(line)-1] == '\\' {
				scanner.Scan()
				line += "\n" + scanner.Text()
			}
			ch <- line
		}
		close(ch)
	}()
	return ch
}

func (pp *Dfpp) ProcessInclude(line string, fields []string) bool {
	merge := false
	exclude := make(map[string]bool)
	include := make(map[string]bool)

	uris := make([]string, 0, len(fields)-1)
	for _, field := range fields {
		if _, err := os.Stat(field); err == nil {
			uris = append(uris, field)
			continue
		}
		clude := include
		if field[0] == '-' {
			clude = exclude
			field = field[1:]
		}

		switch field {
		case "\\":
			continue
		case "INCLUDE":
			continue
		case "MERGE":
			merge = true
		case "ADD":
			fallthrough
		case "CMD":
			fallthrough
		case "COPY":
			fallthrough
		case "ENTRYPOINT":
			fallthrough
		case "ENV":
			fallthrough
		case "EXPOSE":
			fallthrough
		case "FROM":
			fallthrough
		case "LABEL":
			fallthrough
		case "MAINTAINER":
			fallthrough
		case "ONBUILD":
			fallthrough
		case "RUN":
			fallthrough
		case "USER":
			fallthrough
		case "VOLUME":
			fallthrough
		case "WORKDIR":
			clude[field] = true
		default:
			uris = append(uris, field)
		}
	}

	docs := make([]string, 0, len(uris))
	for _, uri := range uris {
		if _, err := os.Stat(uri); err == nil {
			content, err := ioutil.ReadFile(uri)
			if err != nil {
				log.Errorf("Failed to read %s: %s", uri, err)
				os.Exit(1)
			}
			docs = append(docs, string(content))
		} else {
			req, _ := http.NewRequest("GET", uri, nil)
			ua := &http.Client{}
			if resp, err := ua.Do(req); err != nil {
				log.Errorf("Failed to %s %s: %s", req.Method, req.URL.String(), err)
				os.Exit(1)
			} else {
				if resp.StatusCode < 200 || resp.StatusCode >= 300 && resp.StatusCode != 401 {
					log.Errorf("response status: %s", resp.Status)
				}
				runtime.SetFinalizer(resp, func(r *http.Response) {
					r.Body.Close()
				})
				if buf, err := ioutil.ReadAll(resp.Body); err == nil {
					docs = append(docs, string(buf))
				}
			}
		}
	}
	pp.Merge(merge, docs, include, exclude)
	return true
}

func (pp *Dfpp) Merge(merge bool, docs []string, include, exclude map[string]bool) {
	result := make([]*string, 0)
	ops := make(map[string]*string)
	for _, doc := range docs {
		for line := range InstructionScanner(strings.NewReader(doc)) {
			fields := strings.Fields(line)
			if len(fields) > 0 {
				op := fields[0]
				if _, ok := exclude[op]; ok {
					continue
				}
				if _, ok := include[op]; len(include) > 0 && !ok {
					continue
				}
				details := strings.TrimPrefix(line, fields[0]+" ")

				if sref, ok := ops[op]; merge && ok {
					if op == "ENV" {
						if !strings.Contains(details, "=") {
							fields := strings.Fields(details)
							details = strings.Join(fields, "=")
						}
						*sref += " \\\n" + strings.Repeat(" ", len(op)+1) + details
					} else if op == "LABEL" {
						*sref += " \\\n" + strings.Repeat(" ", len(op)+1) + details
					} else if op == "RUN" {
						*sref += " && \\\n    " + details

						// squash redundent apt-get updates
						squash := "apt-get update"
						if ix := strings.Index(*sref, squash); ix >= 0 {
							rest := strings.Replace((*sref)[ix+len(squash):], squash, "echo skipping redundent apt-get-update", -1)
							*sref = (*sref)[0:ix+len(squash)] + rest
						}

					} else {
						dup := string(line)
						result = append(result, &dup)
					}
				} else {
					if op == "ENV" && !strings.Contains(line, "=") {
						line = strings.TrimPrefix(line, "ENV")
						line = strings.TrimSpace(line)
						fields := strings.Fields(line)
						line = fmt.Sprintf("ENV %s", strings.Join(fields, "="))
					}
					dup := string(line)
					result = append(result, &dup)
					ops[op] = result[len(result)-1]
				}
			}
		}
	}
	for _, line := range result {
		fmt.Fprintf(pp.Output, "%s\n", *line)
	}
}
<commit_msg>add recursive handling for INCLUDE statements<commit_after>package dfpp

import (
	"bufio"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"runtime"
	"strings"

	"gopkg.in/op/go-logging.v1"
)

var log = logging.MustGetLogger("dfpp")

type Dfpp struct {
	Processors map[string]func(string, []string) bool
	Output     io.Writer
}

func NewDfpp() *Dfpp {
	pp := &Dfpp{
		Output: os.Stdout,
	}
	pp.Processors = map[string]func(string, []string) bool{
		"INCLUDE": pp.ProcessInclude,
	}
	return pp
}

func (pp *Dfpp) ProcessDockerfile(input io.Reader) {
	for line := range InstructionScanner(input) {
		parts := strings.Fields(line)
		if len(parts) > 0 {
			instruction := parts[0]
			if fn, ok := pp.Processors[instruction]; ok {
				if fn(line, parts) {
					continue
				}
			}
		}
		fmt.Fprintf(pp.Output, "%s\n", line)
	}
}

func InstructionScanner(input io.Reader) chan string {
	ch := make(chan string)
	go func() {
		scanner := bufio.NewScanner(input)
		for scanner.Scan() {
			line := scanner.Text()
			for len(line) > 0 && line[len(line)-1] == '\\' {
				scanner.Scan()
				line += "\n" + scanner.Text()
			}
			ch <- line
		}
		close(ch)
	}()
	return ch
}

func (pp *Dfpp) ProcessInclude(line string, fields []string) bool {
	merge := false
	exclude := make(map[string]bool)
	include := make(map[string]bool)

	uris := make([]string, 0, len(fields)-1)
	for _, field := range fields {
		if _, err := os.Stat(field); err == nil {
			uris = append(uris, field)
			continue
		}
		clude := include
		if field[0] == '-' {
			clude = exclude
			field = field[1:]
		}

		switch field {
		case "\\":
			continue
		case "INCLUDE":
			continue
		case "MERGE":
			merge = true
		case "ADD":
			fallthrough
		case "ARG":
			fallthrough
		case "CMD":
			fallthrough
		case "COPY":
			fallthrough
		case "ENTRYPOINT":
			fallthrough
		case "ENV":
			fallthrough
		case "EXPOSE":
			fallthrough
		case "FROM":
			fallthrough
		case "LABEL":
			fallthrough
		case "MAINTAINER":
			fallthrough
		case "ONBUILD":
			fallthrough
		case "RUN":
			fallthrough
		case "USER":
			fallthrough
		case "VOLUME":
			fallthrough
		case "WORKDIR":
			clude[field] = true
		default:
			uris = append(uris, field)
		}
	}

	docs := make([]string, 0, len(uris))
	for _, uri := range uris {
		if _, err := os.Stat(uri); err == nil {
			content, err := ioutil.ReadFile(uri)
			if err != nil {
				log.Errorf("Failed to read %s: %s", uri, err)
				os.Exit(1)
			}
			docs = append(docs, string(content))
		} else {
			req, _ := http.NewRequest("GET", uri, nil)
			ua := &http.Client{}
			if resp, err := ua.Do(req); err != nil {
				log.Errorf("Failed to %s %s: %s", req.Method, req.URL.String(), err)
				os.Exit(1)
			} else {
				if resp.StatusCode < 200 || resp.StatusCode >= 300 && resp.StatusCode != 401 {
					log.Errorf("response status: %s", resp.Status)
				}
				runtime.SetFinalizer(resp, func(r *http.Response) {
					r.Body.Close()
				})
				if buf, err := ioutil.ReadAll(resp.Body); err == nil {
					docs = append(docs, string(buf))
				}
			}
		}
	}
	pp.Merge(merge, docs, include, exclude)
	return true
}

type stringListReader struct {
	lines     []*string
	lineIndex int
	offset    int
}

func (s *stringListReader) Read(p []byte) (n int, err error) {
	want := cap(p)
	readBytes := 0
	for want > 0 {
		if s.lineIndex >= len(s.lines) {
			return readBytes, io.EOF
		}
		line := s.lines[s.lineIndex]
		remainder := len(*line) - s.offset
		if remainder > want {
			copy(p[readBytes:], (*line)[s.offset:want])
			s.offset += want
			readBytes += want - s.offset
			want = 0
		} else {
			copy(p[readBytes:], (*line)[s.offset:])
			readBytes += remainder
			s.offset = 0
			s.lineIndex++
			want -= remainder
			// memory buffer was split on new lines, so make sure to add
			// the newlines back on the output merged document.
			copy(p[readBytes:], "\n")
			readBytes++
		}
	}
	return readBytes, nil
}

func (pp *Dfpp) Merge(merge bool, docs []string, include, exclude map[string]bool) {
	result := make([]*string, 0)
	ops := make(map[string]*string)
	for _, doc := range docs {
		for line := range InstructionScanner(strings.NewReader(doc)) {
			fields := strings.Fields(line)
			if len(fields) > 0 {
				op := fields[0]
				if _, ok := exclude[op]; ok {
					continue
				}
				if _, ok := include[op]; len(include) > 0 && !ok {
					continue
				}
				details := strings.TrimPrefix(line, fields[0]+" ")

				if sref, ok := ops[op]; merge && ok {
					if op == "ENV" {
						if !strings.Contains(details, "=") {
							fields := strings.Fields(details)
							details = strings.Join(fields, "=")
						}
						*sref += " \\\n" + strings.Repeat(" ", len(op)+1) + details
					} else if op == "LABEL" {
						*sref += " \\\n" + strings.Repeat(" ", len(op)+1) + details
					} else if op == "RUN" {
						*sref += " && \\\n    " + details

						// squash redundent apt-get updates
						squash := "apt-get update"
						if ix := strings.Index(*sref, squash); ix >= 0 {
							rest := strings.Replace((*sref)[ix+len(squash):], squash, "echo skipping redundent apt-get-update", -1)
							*sref = (*sref)[0:ix+len(squash)] + rest
						}

					} else {
						dup := string(line)
						result = append(result, &dup)
					}
				} else {
					if op == "ENV" && !strings.Contains(line, "=") {
						line = strings.TrimPrefix(line, "ENV")
						line = strings.TrimSpace(line)
						fields := strings.Fields(line)
						line = fmt.Sprintf("ENV %s", strings.Join(fields, "="))
					}
					dup := string(line)
					result = append(result, &dup)
					ops[op] = result[len(result)-1]
				}
			}
		}
	}

	pp.ProcessDockerfile(&stringListReader{lines: result})
}
<|endoftext|>


<commit_before>package amqputil

import (
	"time"

	"github.com/streadway/amqp"
)

// Conn is the amqp connection
type Conn struct {
	conn *amqp.Connection

	// closure info of connection
	dialFn   func() error
	attempts uint8
}

// New do what the fucking name says...
func New() *Conn {
	return &Conn{}
}

// Dial to rabbitmq
func (conn *Conn) Dial(uri string) error {
	// closure the uri for handle reconnects
	conn.dialFn = func() error {
		var err error

		conn.conn, err = amqp.Dial(uri)

		if err != nil {
			return err
		}

		return nil
	}

	return conn.dialFn()
}

// Close the connection
func (conn *Conn) Close() error {
	return conn.conn.Close()
}

// AutoRedial manages the automatic redial of connection when unexpected closed.
// outChan is an unbuffered channel required to receive the errors that results from
// attempts of reconnect. On successfully reconnected, the function onSuccess
// is invoked.
//
// The outChan parameter can receive *amqp.Error for AMQP connection errors
// or errors.Error for any other net/tcp internal error.
//
// Redial strategy:
// If the connection is closed in an unexpected way (opposite of conn.Close()), then
// AutoRedial will try to automatically reconnect waiting for N seconds before each
// attempt, where N is the number of attempts of reconnecting. If the number of
// attempts reach 60, it will be zero'ed.
func (conn *Conn) AutoRedial(outChan chan error, onSuccess func()) {
	errChan := conn.conn.NotifyClose(make(chan *amqp.Error))

	go func() {
		var err error

		select {
		case amqpErr := <-errChan:
			err = amqpErr

			if amqpErr == nil {
				// Gracefull connection close
				return
			}
		attempts:
			outChan <- err

			if conn.attempts > 60 {
				conn.attempts = 0
			}

			// Wait n Seconds where n == conn.attempts...
			time.Sleep(time.Duration(conn.attempts) * time.Second)

			err = conn.dialFn()

			if err != nil {
				conn.attempts++
				goto attempts
			}

			conn.attempts = 0

			// enabled AutoRedial on the new connection
			conn.AutoRedial(outChan, onSuccess)
			onSuccess()
			return
		}
	}()
}
<commit_msg>[CHANGE] change from embedding to anonymous embedding in Conn type<commit_after>package amqputil

import (
	"time"

	"github.com/streadway/amqp"
)

// Conn is the amqp connection
type Conn struct {
	*amqp.Connection

	// closure info of connection
	dialFn   func() error
	attempts uint8
}

// New do what the fucking name says...
func New() *Conn {
	return &Conn{}
}

// Dial to rabbitmq
func (conn *Conn) Dial(uri string) error {
	// closure the uri for handle reconnects
	conn.dialFn = func() error {
		var err error

		conn.Connection, err = amqp.Dial(uri)

		if err != nil {
			return err
		}

		return nil
	}

	return conn.dialFn()
}

// AutoRedial manages the automatic redial of connection when unexpected closed.
// outChan is an unbuffered channel required to receive the errors that results from
// attempts of reconnect. On successfully reconnected, the function onSuccess
// is invoked.
//
// The outChan parameter can receive *amqp.Error for AMQP connection errors
// or errors.Error for any other net/tcp internal error.
//
// Redial strategy:
// If the connection is closed in an unexpected way (opposite of conn.Close()), then
// AutoRedial will try to automatically reconnect waiting for N seconds before each
// attempt, where N is the number of attempts of reconnecting. If the number of
// attempts reach 60, it will be zero'ed.
func (conn *Conn) AutoRedial(outChan chan error, onSuccess func()) {
	errChan := conn.NotifyClose(make(chan *amqp.Error))

	go func() {
		var err error

		select {
		case amqpErr := <-errChan:
			err = amqpErr

			if amqpErr == nil {
				// Gracefull connection close
				return
			}
		attempts:
			outChan <- err

			if conn.attempts > 60 {
				conn.attempts = 0
			}

			// Wait n Seconds where n == conn.attempts...
			time.Sleep(time.Duration(conn.attempts) * time.Second)

			err = conn.dialFn()

			if err != nil {
				conn.attempts++
				goto attempts
			}

			conn.attempts = 0

			// enabled AutoRedial on the new connection
			conn.AutoRedial(outChan, onSuccess)
			onSuccess()
			return
		}
	}()
}
<|endoftext|>


<commit_before>// Package dice utilises the standard "nds" notation where n = number of die and s = number of sides; i.e 1d6, 3d10, 8d8 etc
// in order to create individual sets of a single type of die or bags of mixed collections that can then be manipulated or rolled
package dice

import (
	"fmt"
	"math/rand"
	"strings"
	"time"
)

var rng = rand.New(rand.NewSource(time.Now().UnixNano()))

// Dice represents a set of 1 type of dice, i.e: 3d20 OR 2d4 OR 1d6
type Dice struct {
	number, sides int
}

// NewDice takes the common notation "nds" where n is the number of dice and s is the number of sides;
// i.e 1d6 and returns a new Dice set. Returns error if s is not a valid dice string
func NewDice(s string) (*Dice, error) {
	number, sides, err := strToVal(s)
	if err != nil {
		return new(Dice), err
	}

	return &Dice{number, sides}, nil
}

// Add adds n die to a single set
func (d *Dice) Add(n int) {
	d.number += n
}

// Remove removes n die from a single set to a minimum of 1
func (d *Dice) Remove(n int) {
	if d.number-n < 1 {
		d.number = 1
	} else {
		d.number -= n
	}
}

// Min returns the minimume possible roll
func (d *Dice) Min() int {
	return d.number
}

// Max returns the maximum possible roll
func (d *Dice) Max() int {
	return d.number * d.sides
}

// Roll all dice in set and return the aggregate result and an array of individual results
func (d *Dice) Roll() (int, []int) {
	t, a := 0, []int{}

	for i := 0; i < d.number; i++ {
		n := rng.Intn(d.sides) + 1
		t += n
		a = append(a, n)
	}

	return t, a
}

// String satisfies the Stringer interface for Dice
func (d *Dice) String() string {
	return fmt.Sprintf("%dd%d", d.number, d.sides)
}

// Bag is a collection of different types of Dice; i.e [3d20, 2d4, 1d6]
type Bag struct {
	dice []*Dice
}

// NewBag returns a new Bag object. A bag can be created with a collection of
// dice specified in string form for convenience. I.e b := NewBag("2d20", "1d6", "8d8").
// Returns error if any item in dice is not a valid dice string
func NewBag(dice ...string) (*Bag, error) {
	b := new(Bag)

	for _, a := range dice {
		if err := b.Add(a); err != nil {
			return b, err
		}
	}

	return b, nil
}

// Add puts more dice in the bag, adding to existing sets where possible.
// Returns error if s is not a valid dice string
func (b *Bag) Add(s string) error {
	d, err := NewDice(s)
	if err != nil {
		return err
	}

	// increment existing set if it exists
	for _, set := range b.dice {
		if set.sides == d.sides {
			set.number += d.number
			return nil
		}
	}

	// Otherwise add a new set
	b.dice = append(b.dice, d)
	return nil
}

// Remove reduces the number of dice by the specified s string if s exists in the bag.
// Returns error if s is not a valid dice string
func (b *Bag) Remove(s string) error {
	number, sides, err := strToVal(s)
	if err != nil {
		return err
	}

	// Remove specified dice from set
	for _, set := range b.dice {
		if set.sides == sides {
			// ensure no < 0 values
			if set.number-number < 0 {
				set.number = 0
			} else {
				set.number -= number
			}
			break
		}
	}

	return nil
}

// Min returns the minimum possible roll
func (b *Bag) Min() int {
	t := 0
	for _, d := range b.dice {
		t += d.Min()
	}
	return t
}

// Max returns the maximum possible roll
func (b *Bag) Max() int {
	t := 0
	for _, d := range b.dice {
		t += d.Max()
	}
	return t
}

// Roll returns aggregate rolls of all Dice in the bag and a map set of results
func (b *Bag) Roll() (int, map[string][]int) {
	t, a := 0, make(map[string][]int)

	for _, d := range b.dice {
		n, s := d.Roll()
		t += n

		a[d.String()] = s
	}

	return t, a
}

// String satisfies the Stringer interface for Bags
func (b *Bag) String() string {
	v := make([]string, len(b.dice))

	for i, d := range b.dice {
		v[i] = fmt.Sprint(d)
	}

	return strings.Join(v, ", ")
}

// returns int values for numbers, sides
func strToVal(a string) (number, sides int, err error) {
	_, err = fmt.Sscanf(a, "%dd%d", &number, &sides)
	if err != nil {
		return number, sides, fmt.Errorf("%s is not a valid dice string", a)
	}

	if number > 0 && sides > 0 {
		return number, sides, nil
	}

	return number, sides, fmt.Errorf("%s is not a valid dice string", a)
}
<commit_msg>make NewBag more robust<commit_after>// Package dice utilises the standard "nds" notation where n = number of die and s = number of sides; i.e 1d6, 3d10, 8d8 etc
// in order to create individual sets of a single type of die or bags of mixed collections that can then be manipulated or rolled
package dice

import (
	"fmt"
	"math/rand"
	"strings"
	"time"
)

var rng = rand.New(rand.NewSource(time.Now().UnixNano()))

// Dice represents a set of 1 type of dice, i.e: 3d20 OR 2d4 OR 1d6
type Dice struct {
	number, sides int
}

// NewDice takes the common notation "nds" where n is the number of dice and s is the number of sides;
// i.e 1d6 and returns a new Dice set. Returns error if s is not a valid dice string
func NewDice(s string) (*Dice, error) {
	number, sides, err := strToVal(s)
	if err != nil {
		return new(Dice), err
	}

	return &Dice{number, sides}, nil
}

// Add adds n die to a single set
func (d *Dice) Add(n int) {
	d.number += n
}

// Remove removes n die from a single set to a minimum of 1
func (d *Dice) Remove(n int) {
	if d.number-n < 1 {
		d.number = 1
	} else {
		d.number -= n
	}
}

// Min returns the minimume possible roll
func (d *Dice) Min() int {
	return d.number
}

// Max returns the maximum possible roll
func (d *Dice) Max() int {
	return d.number * d.sides
}

// Roll all dice in set and return the aggregate result and an array of individual results
func (d *Dice) Roll() (int, []int) {
	t, a := 0, []int{}

	for i := 0; i < d.number; i++ {
		n := rng.Intn(d.sides) + 1
		t += n
		a = append(a, n)
	}

	return t, a
}

// String satisfies the Stringer interface for Dice
func (d *Dice) String() string {
	return fmt.Sprintf("%dd%d", d.number, d.sides)
}

// Bag is a collection of different types of Dice; i.e [3d20, 2d4, 1d6]
type Bag struct {
	dice []*Dice
}

// NewBag returns a new Bag object. A bag can be created with a collection of
// dice specified in string form for convenience. I.e b := NewBag("2d20", "1d6", "8d8").
// Returns error if any item in dice is not a valid dice string
func NewBag(dice ...string) (*Bag, error) {
	b := new(Bag)

	// Account for different inputs
	for _, a := range dice {
		items := strings.Split(a, ",")
		for _, i := range items {
			if err := b.Add(i); err != nil {
				return b, err
			}
		}
	}

	return b, nil
}

// Add puts more dice in the bag, adding to existing sets where possible.
// Returns error if s is not a valid dice string
func (b *Bag) Add(s string) error {
	d, err := NewDice(s)
	if err != nil {
		return err
	}

	// increment existing set if it exists
	for _, set := range b.dice {
		if set.sides == d.sides {
			set.number += d.number
			return nil
		}
	}

	// Otherwise add a new set
	b.dice = append(b.dice, d)
	return nil
}

// Remove reduces the number of dice by the specified s string if s exists in the bag.
// Returns error if s is not a valid dice string
func (b *Bag) Remove(s string) error {
	number, sides, err := strToVal(s)
	if err != nil {
		return err
	}

	// Remove specified dice from set
	for _, set := range b.dice {
		if set.sides == sides {
			// ensure no < 0 values
			if set.number-number < 0 {
				set.number = 0
			} else {
				set.number -= number
			}
			break
		}
	}

	return nil
}

// Min returns the minimum possible roll
func (b *Bag) Min() int {
	t := 0
	for _, d := range b.dice {
		t += d.Min()
	}
	return t
}

// Max returns the maximum possible roll
func (b *Bag) Max() int {
	t := 0
	for _, d := range b.dice {
		t += d.Max()
	}
	return t
}

// Roll returns aggregate rolls of all Dice in the bag and a map set of results
func (b *Bag) Roll() (int, map[string][]int) {
	t, a := 0, make(map[string][]int)

	for _, d := range b.dice {
		n, s := d.Roll()
		t += n

		a[d.String()] = s
	}

	return t, a
}

// String satisfies the Stringer interface for Bags
func (b *Bag) String() string {
	v := make([]string, len(b.dice))

	for i, d := range b.dice {
		v[i] = fmt.Sprint(d)
	}

	return strings.Join(v, ", ")
}

// returns int values for numbers, sides
func strToVal(a string) (number, sides int, err error) {
	_, err = fmt.Sscanf(a, "%dd%d", &number, &sides)
	if err != nil {
		return number, sides, fmt.Errorf("%s is not a valid dice string", a)
	}

	if number > 0 && sides > 0 {
		return number, sides, nil
	}

	return number, sides, fmt.Errorf("%s is not a valid dice string", a)
}
<|endoftext|>


<commit_before>// Copyright 2016 The etcd Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package command

import (
	"fmt"
	"os"

	"github.com/spf13/cobra"
	"go.etcd.io/etcd/etcdutl/v3/etcdutl"
	"go.etcd.io/etcd/pkg/v3/cobrautl"
)

var (
	defragDataDir string
)

// NewDefragCommand returns the cobra command for "Defrag".
func NewDefragCommand() *cobra.Command {
	cmd := &cobra.Command{
		Use:   "defrag",
		Short: "Defragments the storage of the etcd members with given endpoints",
		Run:   defragCommandFunc,
	}
	cmd.PersistentFlags().BoolVar(&epClusterEndpoints, "cluster", false, "use all endpoints from the cluster member list")
	cmd.Flags().StringVar(&defragDataDir, "data-dir", "", "Optional. If present, defragments a data directory not in use by etcd.")
	cmd.MarkFlagDirname("data-dir")
	return cmd
}

func defragCommandFunc(cmd *cobra.Command, args []string) {
	if len(defragDataDir) > 0 {
		fmt.Fprintf(os.Stderr, "Use `etcdutl defrag` instead. The --data-dir is going to be decomissioned in v3.6.\n\n")
		err := etcdutl.DefragData(defragDataDir)
		if err != nil {
			cobrautl.ExitWithError(cobrautl.ExitError, err)
		}
	}

	failures := 0
	c := mustClientFromCmd(cmd)
	for _, ep := range endpointsFromCluster(cmd) {
		ctx, cancel := commandCtx(cmd)
		_, err := c.Defragment(ctx, ep)
		cancel()
		if err != nil {
			fmt.Fprintf(os.Stderr, "Failed to defragment etcd member[%s] (%v)\n", ep, err)
			failures++
		} else {
			fmt.Printf("Finished defragmenting etcd member[%s]\n", ep)
		}
	}

	if failures != 0 {
		os.Exit(cobrautl.ExitError)
	}
}
<commit_msg>etcdctl/ctlv3/command: add use time for defrag command<commit_after>// Copyright 2016 The etcd Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package command

import (
	"fmt"
	"os"
	"time"

	"github.com/spf13/cobra"
	"go.etcd.io/etcd/etcdutl/v3/etcdutl"
	"go.etcd.io/etcd/pkg/v3/cobrautl"
)

var (
	defragDataDir string
)

// NewDefragCommand returns the cobra command for "Defrag".
func NewDefragCommand() *cobra.Command {
	cmd := &cobra.Command{
		Use:   "defrag",
		Short: "Defragments the storage of the etcd members with given endpoints",
		Run:   defragCommandFunc,
	}
	cmd.PersistentFlags().BoolVar(&epClusterEndpoints, "cluster", false, "use all endpoints from the cluster member list")
	cmd.Flags().StringVar(&defragDataDir, "data-dir", "", "Optional. If present, defragments a data directory not in use by etcd.")
	cmd.MarkFlagDirname("data-dir")
	return cmd
}

func defragCommandFunc(cmd *cobra.Command, args []string) {
	if len(defragDataDir) > 0 {
		fmt.Fprintf(os.Stderr, "Use `etcdutl defrag` instead. The --data-dir is going to be decomissioned in v3.6.\n\n")
		err := etcdutl.DefragData(defragDataDir)
		if err != nil {
			cobrautl.ExitWithError(cobrautl.ExitError, err)
		}
	}

	failures := 0
	c := mustClientFromCmd(cmd)
	for _, ep := range endpointsFromCluster(cmd) {
		ctx, cancel := commandCtx(cmd)
		start := time.Now()
		_, err := c.Defragment(ctx, ep)
		d := time.Now().Sub(start)
		cancel()
		if err != nil {
			fmt.Fprintf(os.Stderr, "Failed to defragment etcd member[%s]. took %s. (%v)\n", ep, d.String(), err)
			failures++
		} else {
			fmt.Printf("Finished defragmenting etcd member[%s]. took %s\n", ep, d.String())
		}
	}

	if failures != 0 {
		os.Exit(cobrautl.ExitError)
	}
}
<|endoftext|>


<commit_before>/*
 * Copyright 2014 Jason Woods.
 *
 * This file is a modification of code from Logstash Forwarder.
 * Copyright 2012-2013 Jordan Sissel and contributors.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package main

import (
  "flag"
  "fmt"
  "github.com/op/go-logging"
  stdlog "log"
  "os"
  "runtime/pprof"
  "sync"
  "time"
)

const Log_Courier_Version string = "0.11"

var log *logging.Logger

func init() {
  log = logging.MustGetLogger("")
}

func main() {
  logcourier := NewLogCourier()
  logcourier.Run()
}

type LogCourierMasterControl struct {
  signal chan interface{}
  sinks  map[*LogCourierControl]chan *Config
  group  sync.WaitGroup
}

func NewLogCourierMasterControl() *LogCourierMasterControl {
  return &LogCourierMasterControl{
    signal: make(chan interface{}),
    sinks:  make(map[*LogCourierControl]chan *Config),
  }
}

func (lcs *LogCourierMasterControl) Shutdown() {
  close(lcs.signal)
}

func (lcs *LogCourierMasterControl) SendConfig(config *Config) {
  for _, sink := range lcs.sinks {
    sink <- config
  }
}

func (lcs *LogCourierMasterControl) Register() *LogCourierControl {
  return lcs.register()
}

func (lcs *LogCourierMasterControl) RegisterWithRecvConfig() *LogCourierControl {
  ret := lcs.register()

  config_chan := make(chan *Config)
  lcs.sinks[ret] = config_chan
  ret.sink = config_chan

  return ret
}

func (lcs *LogCourierMasterControl) register() *LogCourierControl {
  lcs.group.Add(1)

  return &LogCourierControl{
    signal: lcs.signal,
    group:  &lcs.group,
  }
}

func (lcs *LogCourierMasterControl) Wait() {
  lcs.group.Wait()
}

type LogCourierControl struct {
  signal <-chan interface{}
  sink   <-chan *Config
  group  *sync.WaitGroup
}

func (lcs *LogCourierControl) ShutdownSignal() <-chan interface{} {
  return lcs.signal
}

func (lcs *LogCourierControl) RecvConfig() <-chan *Config {
  if lcs.sink == nil {
    panic("RecvConfig invalid: LogCourierControl was not registered with RegisterWithRecvConfig")
  }
  return lcs.sink
}

func (lcs *LogCourierControl) Done() {
  lcs.group.Done()
}

type LogCourier struct {
  control        *LogCourierMasterControl
  config         *Config
  shutdown_chan  chan os.Signal
  reload_chan    chan os.Signal
  config_file    string
  from_beginning bool
  log_file       *os.File
}

func NewLogCourier() *LogCourier {
  ret := &LogCourier{
    control: NewLogCourierMasterControl(),
  }
  return ret
}

func (lc *LogCourier) Run() {
  lc.startUp()

  event_chan := make(chan *FileEvent, 16)
  publisher_chan := make(chan []*FileEvent, 1)

  log.Info("Starting pipeline")

  registrar := NewRegistrar(lc.config.General.PersistDir, lc.control)

  publisher, err := NewPublisher(&lc.config.Network, registrar, lc.control)
  if err != nil {
    log.Fatalf("Failed to initialise: %s", err)
  }

  spooler := NewSpooler(&lc.config.General, lc.control)

  prospector, err := NewProspector(lc.config, lc.from_beginning, registrar, lc.control)
  if err != nil {
    log.Fatalf("Failed to initialise: %s", err)
  }

  // Start the pipeline
  go prospector.Prospect(event_chan)

  go spooler.Spool(event_chan, publisher_chan)

  go publisher.Publish(publisher_chan)

  go registrar.Register()

  log.Notice("Pipeline ready")

  lc.shutdown_chan = make(chan os.Signal, 1)
  lc.reload_chan = make(chan os.Signal, 1)
  lc.registerSignals()

SignalLoop:
  for {
    select {
    case <-lc.shutdown_chan:
      lc.cleanShutdown()
      break SignalLoop
    case <-lc.reload_chan:
      lc.reloadConfig()
    }
  }

  log.Notice("Exiting")

  if lc.log_file != nil {
    lc.log_file.Close()
  }
}

func (lc *LogCourier) startUp() {
  var version bool
  var config_test bool
  var list_supported bool
  var cpu_profile string

  flag.BoolVar(&version, "version", false, "show version information")
  flag.BoolVar(&config_test, "config-test", false, "Test the configuration specified by -config and exit")
  flag.BoolVar(&list_supported, "list-supported", false, "List supported transports and codecs")
  flag.StringVar(&cpu_profile, "cpuprofile", "", "write cpu profile to file")

  flag.StringVar(&lc.config_file, "config", "", "The config file to load")
  flag.BoolVar(&lc.from_beginning, "from-beginning", false, "On first run, read new files from the beginning instead of the end")

  flag.Parse()

  if version {
    fmt.Printf("Log Courier version %s\n", Log_Courier_Version)
    os.Exit(0)
  }

  if list_supported {
    fmt.Printf("Available transports:\n")
    for _, transport := range AvailableTransports() {
      fmt.Printf("  %s\n", transport)
    }

    fmt.Printf("Available codecs:\n")
    for _, codec := range AvailableCodecs() {
      fmt.Printf("  %s\n", codec)
    }
    os.Exit(0)
  }

  if lc.config_file == "" {
    fmt.Fprintf(os.Stderr, "Please specify a configuration file with -config.\n\n")
    flag.PrintDefaults()
    os.Exit(1)
  }

  err := lc.loadConfig()

  if config_test {
    if err == nil {
      fmt.Printf("Configuration OK\n")
      os.Exit(0)
    }
    fmt.Printf("Configuration test failed: %s\n", err)
    os.Exit(1)
  }

  if err != nil {
    fmt.Printf("Configuration error: %s\n", err)
    os.Exit(1)
  }

  if err = lc.configureLogging(); err != nil {
    fmt.Printf("Failed to initialise logging: %s", err)
    os.Exit(1)
  }

  if cpu_profile != "" {
    log.Notice("Starting CPU profiler")
    f, err := os.Create(cpu_profile)
    if err != nil {
      log.Fatal(err)
    }
    pprof.StartCPUProfile(f)
    go func() {
      time.Sleep(60 * time.Second)
      pprof.StopCPUProfile()
      log.Panic("CPU profile completed")
    }()
  }
}

func (lc *LogCourier) configureLogging() (err error) {
  backends := make([]logging.Backend, 0, 1)

  // First, the stdout backend
  if lc.config.General.LogStdout {
    backends = append(backends, logging.NewLogBackend(os.Stdout, "", stdlog.LstdFlags|stdlog.Lmicroseconds))
  }

  // Log file?
  if lc.config.General.LogFile != "" {
    lc.log_file, err = os.OpenFile(lc.config.General.LogFile, os.O_CREATE|os.O_RDWR|os.O_APPEND, 0640)
    if err != nil {
      return
    }

    backends = append(backends, logging.NewLogBackend(lc.log_file, "", stdlog.LstdFlags|stdlog.Lmicroseconds))
  }

  if err = lc.configureLoggingPlatform(&backends); err != nil {
    return
  }

  // Set backends BEFORE log level (or we reset log level)
  logging.SetBackend(backends...)

  // Set the logging level
  logging.SetLevel(lc.config.General.LogLevel, "")

  return nil
}

func (lc *LogCourier) loadConfig() error {
  lc.config = NewConfig()
  if err := lc.config.Load(lc.config_file); err != nil {
    return err
  }

  if len(lc.config.Files) == 0 {
    return fmt.Errorf("No file groups were found in the configuration.")
  }

  return nil
}

func (lc *LogCourier) reloadConfig() {
  if err := lc.loadConfig(); err != nil {
    log.Warning("Configuration error, reload unsuccessful: %s", err)
    return
  }

  log.Notice("Configuration reload successful")

  // Update the log level
  logging.SetLevel(lc.config.General.LogLevel, "")

  // Pass the new config to the pipeline workers
  lc.control.SendConfig(lc.config)
}

func (lc *LogCourier) cleanShutdown() {
  log.Notice("Initiating shutdown")
  lc.control.Shutdown()
  lc.control.Wait()
}
<commit_msg>Update log-courier.go version number<commit_after>/*
 * Copyright 2014 Jason Woods.
 *
 * This file is a modification of code from Logstash Forwarder.
 * Copyright 2012-2013 Jordan Sissel and contributors.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package main

import (
  "flag"
  "fmt"
  "github.com/op/go-logging"
  stdlog "log"
  "os"
  "runtime/pprof"
  "sync"
  "time"
)

const Log_Courier_Version string = "0.12"

var log *logging.Logger

func init() {
  log = logging.MustGetLogger("")
}

func main() {
  logcourier := NewLogCourier()
  logcourier.Run()
}

type LogCourierMasterControl struct {
  signal chan interface{}
  sinks  map[*LogCourierControl]chan *Config
  group  sync.WaitGroup
}

func NewLogCourierMasterControl() *LogCourierMasterControl {
  return &LogCourierMasterControl{
    signal: make(chan interface{}),
    sinks:  make(map[*LogCourierControl]chan *Config),
  }
}

func (lcs *LogCourierMasterControl) Shutdown() {
  close(lcs.signal)
}

func (lcs *LogCourierMasterControl) SendConfig(config *Config) {
  for _, sink := range lcs.sinks {
    sink <- config
  }
}

func (lcs *LogCourierMasterControl) Register() *LogCourierControl {
  return lcs.register()
}

func (lcs *LogCourierMasterControl) RegisterWithRecvConfig() *LogCourierControl {
  ret := lcs.register()

  config_chan := make(chan *Config)
  lcs.sinks[ret] = config_chan
  ret.sink = config_chan

  return ret
}

func (lcs *LogCourierMasterControl) register() *LogCourierControl {
  lcs.group.Add(1)

  return &LogCourierControl{
    signal: lcs.signal,
    group:  &lcs.group,
  }
}

func (lcs *LogCourierMasterControl) Wait() {
  lcs.group.Wait()
}

type LogCourierControl struct {
  signal <-chan interface{}
  sink   <-chan *Config
  group  *sync.WaitGroup
}

func (lcs *LogCourierControl) ShutdownSignal() <-chan interface{} {
  return lcs.signal
}

func (lcs *LogCourierControl) RecvConfig() <-chan *Config {
  if lcs.sink == nil {
    panic("RecvConfig invalid: LogCourierControl was not registered with RegisterWithRecvConfig")
  }
  return lcs.sink
}

func (lcs *LogCourierControl) Done() {
  lcs.group.Done()
}

type LogCourier struct {
  control        *LogCourierMasterControl
  config         *Config
  shutdown_chan  chan os.Signal
  reload_chan    chan os.Signal
  config_file    string
  from_beginning bool
  log_file       *os.File
}

func NewLogCourier() *LogCourier {
  ret := &LogCourier{
    control: NewLogCourierMasterControl(),
  }
  return ret
}

func (lc *LogCourier) Run() {
  lc.startUp()

  event_chan := make(chan *FileEvent, 16)
  publisher_chan := make(chan []*FileEvent, 1)

  log.Info("Starting pipeline")

  registrar := NewRegistrar(lc.config.General.PersistDir, lc.control)

  publisher, err := NewPublisher(&lc.config.Network, registrar, lc.control)
  if err != nil {
    log.Fatalf("Failed to initialise: %s", err)
  }

  spooler := NewSpooler(&lc.config.General, lc.control)

  prospector, err := NewProspector(lc.config, lc.from_beginning, registrar, lc.control)
  if err != nil {
    log.Fatalf("Failed to initialise: %s", err)
  }

  // Start the pipeline
  go prospector.Prospect(event_chan)

  go spooler.Spool(event_chan, publisher_chan)

  go publisher.Publish(publisher_chan)

  go registrar.Register()

  log.Notice("Pipeline ready")

  lc.shutdown_chan = make(chan os.Signal, 1)
  lc.reload_chan = make(chan os.Signal, 1)
  lc.registerSignals()

SignalLoop:
  for {
    select {
    case <-lc.shutdown_chan:
      lc.cleanShutdown()
      break SignalLoop
    case <-lc.reload_chan:
      lc.reloadConfig()
    }
  }

  log.Notice("Exiting")

  if lc.log_file != nil {
    lc.log_file.Close()
  }
}

func (lc *LogCourier) startUp() {
  var version bool
  var config_test bool
  var list_supported bool
  var cpu_profile string

  flag.BoolVar(&version, "version", false, "show version information")
  flag.BoolVar(&config_test, "config-test", false, "Test the configuration specified by -config and exit")
  flag.BoolVar(&list_supported, "list-supported", false, "List supported transports and codecs")
  flag.StringVar(&cpu_profile, "cpuprofile", "", "write cpu profile to file")

  flag.StringVar(&lc.config_file, "config", "", "The config file to load")
  flag.BoolVar(&lc.from_beginning, "from-beginning", false, "On first run, read new files from the beginning instead of the end")

  flag.Parse()

  if version {
    fmt.Printf("Log Courier version %s\n", Log_Courier_Version)
    os.Exit(0)
  }

  if list_supported {
    fmt.Printf("Available transports:\n")
    for _, transport := range AvailableTransports() {
      fmt.Printf("  %s\n", transport)
    }

    fmt.Printf("Available codecs:\n")
    for _, codec := range AvailableCodecs() {
      fmt.Printf("  %s\n", codec)
    }
    os.Exit(0)
  }

  if lc.config_file == "" {
    fmt.Fprintf(os.Stderr, "Please specify a configuration file with -config.\n\n")
    flag.PrintDefaults()
    os.Exit(1)
  }

  err := lc.loadConfig()

  if config_test {
    if err == nil {
      fmt.Printf("Configuration OK\n")
      os.Exit(0)
    }
    fmt.Printf("Configuration test failed: %s\n", err)
    os.Exit(1)
  }

  if err != nil {
    fmt.Printf("Configuration error: %s\n", err)
    os.Exit(1)
  }

  if err = lc.configureLogging(); err != nil {
    fmt.Printf("Failed to initialise logging: %s", err)
    os.Exit(1)
  }

  if cpu_profile != "" {
    log.Notice("Starting CPU profiler")
    f, err := os.Create(cpu_profile)
    if err != nil {
      log.Fatal(err)
    }
    pprof.StartCPUProfile(f)
    go func() {
      time.Sleep(60 * time.Second)
      pprof.StopCPUProfile()
      log.Panic("CPU profile completed")
    }()
  }
}

func (lc *LogCourier) configureLogging() (err error) {
  backends := make([]logging.Backend, 0, 1)

  // First, the stdout backend
  if lc.config.General.LogStdout {
    backends = append(backends, logging.NewLogBackend(os.Stdout, "", stdlog.LstdFlags|stdlog.Lmicroseconds))
  }

  // Log file?
  if lc.config.General.LogFile != "" {
    lc.log_file, err = os.OpenFile(lc.config.General.LogFile, os.O_CREATE|os.O_RDWR|os.O_APPEND, 0640)
    if err != nil {
      return
    }

    backends = append(backends, logging.NewLogBackend(lc.log_file, "", stdlog.LstdFlags|stdlog.Lmicroseconds))
  }

  if err = lc.configureLoggingPlatform(&backends); err != nil {
    return
  }

  // Set backends BEFORE log level (or we reset log level)
  logging.SetBackend(backends...)

  // Set the logging level
  logging.SetLevel(lc.config.General.LogLevel, "")

  return nil
}

func (lc *LogCourier) loadConfig() error {
  lc.config = NewConfig()
  if err := lc.config.Load(lc.config_file); err != nil {
    return err
  }

  if len(lc.config.Files) == 0 {
    return fmt.Errorf("No file groups were found in the configuration.")
  }

  return nil
}

func (lc *LogCourier) reloadConfig() {
  if err := lc.loadConfig(); err != nil {
    log.Warning("Configuration error, reload unsuccessful: %s", err)
    return
  }

  log.Notice("Configuration reload successful")

  // Update the log level
  logging.SetLevel(lc.config.General.LogLevel, "")

  // Pass the new config to the pipeline workers
  lc.control.SendConfig(lc.config)
}

func (lc *LogCourier) cleanShutdown() {
  log.Notice("Initiating shutdown")
  lc.control.Shutdown()
  lc.control.Wait()
}
<|endoftext|>


<commit_before>package nativeregion

import (
	"testing"
	"functorama.com/demo/base"
	"functorama.com/demo/nativebase"
)

func TestRegionSplitPos(t *testing.T) {
	helper := NativeRegionSplitHelper{
		left:   1.0,
		right:  3.0,
		top:    3.0,
		bottom: 1.0,
		midR:   2.0,
		midI:   2.0,
	}

	testRegionSplit(helper, t)
}

func TestRegionSplitNeg(t *testing.T) {
	helper := NativeRegionSplitHelper{
		left:   -100.0,
		right:  -24.0,
		top:    -10.0,
		bottom: -340.0,
		midR:   -62.0,
		midI:   -175.0,
	}

	testRegionSplit(helper, t)
}

func TestRegionSplitNegPos(t *testing.T) {
	helper := NativeRegionSplitHelper{
		left:   -100.0,
		right:  24.0,
		top:    10.0,
		bottom: -340.0,
		midR:   -38.0,
		midI:   -165.0,
	}

	testRegionSplit(helper, t)
}

func TestChildrenPopulated(t *testing.T) {
	const inputChildCount = 4
	numerics := NativeRegionNumerics{
		subregion: nativeSubregion{
			populated: true,
			// We are not inspecting the children here
			children: make([]nativeRegion, inputChildCount),
		},
	}
	children := numerics.Children()

	expectedChildren := 4
	actualChildren := len(children)
	if actualChildren != expectedChildren {
		t.Error("Expected", expectedChildren, "but received", actualChildren)
	}
}

func TestChildrenEmpty(t *testing.T) {
	numerics := NativeRegionNumerics{}

	recovered := false
	triggerPanic := func() {
		defer func() {
			r := recover()
			recovered = r != nil
		}()
		numerics.Children()
	}

	triggerPanic()

	if !recovered {
		t.Error("Expected panic e.g. \"Error when raising error\"")
	}
}

func TestMandelbrotPoints(t *testing.T) {
	numerics := NativeRegionNumerics{}

	points := numerics.MandelbrotPoints()

	expectedPoints := 5
	actualPoints := len(points)

	if expectedPoints != actualPoints {
		t.Error("Expected to receive", expectedPoints, "but received", actualPoints)
	}
}

func TestEvaluateAllPoints(t *testing.T) {
	numerics := NativeRegionNumerics{}
	numerics.EvaluateAllPoints(1)
	region := numerics.Region

	okay := true
	for _, thunk := range numerics.thunks() {
		if !thunk.evaluated {
			okay = false
		}
		break
	}

	if !okay {
		t.Error("Expected all points to be evaluated, but region was:", region)
	}
}

func TestRect(t *testing.T) {
	const picSide = 2
	const planeSide = 2.0

	const left = -1
	const bottom = -1
	const right = left + planeSide
	const top = bottom + planeSide

	min := complex(left, bottom)
	max := complex(right, top)
	numerics := &NativeRegionNumerics{
		NativeBaseNumerics: nativebase.NativeBaseNumerics{
			BaseNumerics: base.BaseNumerics{
				PicXMin: 0,
				PicXMax: picSide,
				PicYMin: 0,
				PicYMax: picSide,
			},
			RealMin: left,
			RealMax: right,
			ImagMin: bottom,
			ImagMax: top,
		},
		Region:  createNativeRegion(min, max),
	}

	uq := nativebase.UnitQuery{picSide, picSide, planeSide, planeSide}
	numerics.Runit, numerics.Iunit = uq.PixelUnits()

	expectMinX := 0
	expectMaxX := 2
	expectMinY := 0
	expectMaxY := 2

	r := numerics.Rect()

	okay := expectMinX == r.Min.X
	okay = okay && expectMaxX == r.Max.X
	okay = okay && expectMinY == r.Min.Y
	okay = okay && expectMaxY == r.Max.Y
	if !okay {
		t.Error("Rectangle had unexpected bounds", r)
	}

}

func testRegionSplit(helper NativeRegionSplitHelper, t *testing.T) {
	initMin := complex(helper.left, helper.bottom)
	initMax := complex(helper.right, helper.top)

	topLeftMin := complex(helper.left, helper.midI)
	topLeftMax := complex(helper.midR, helper.top)

	topRightMin := complex(helper.midR, helper.midI)
	topRightMax := complex(helper.right, helper.top)

	bottomLeftMin := complex(helper.left, helper.bottom)
	bottomLeftMax := complex(helper.midR, helper.midI)

	bottomRightMin := complex(helper.midR, helper.bottom)
	bottomRightMax := complex(helper.right, helper.midI)

	subjectRegion := createNativeRegion(initMin, initMax)

	expected := []nativeRegion{
		createNativeRegion(topLeftMin, topLeftMax),
		createNativeRegion(topRightMin, topRightMax),
		createNativeRegion(bottomLeftMin, bottomLeftMax),
		createNativeRegion(bottomRightMin, bottomRightMax),
	}

	numerics := NativeRegionNumerics{
		Region: subjectRegion,
	}
	numerics.SqrtDivergeLimit = 2.0
	numerics.Split()
	actualChildren := numerics.subregion.children

	for i, ex := range expected {
		actual := actualChildren[i]
		exPoints := thunks(ex)
		acPoints := thunks(actual)
		fail := false
		for j, expectThunk := range exPoints {
			actThunk := acPoints[j]
			if expectThunk != actThunk {
				fail = true
				t.Log("Region", i, "error at thunk", j,
					"expected", expectThunk,
					"but received", actThunk)
			}
		}
		if fail {
			t.Fail()
		}
	}
}

func thunks(region nativeRegion) []nativeMandelbrotThunk {
	return []nativeMandelbrotThunk{
		region.topLeft,
		region.topRight,
		region.bottomLeft,
		region.bottomRight,
		region.midPoint,
	}
}

type NativeRegionSplitHelper struct {
	left   float64
	right  float64
	bottom float64
	top    float64
	midR   float64
	midI   float64
}<commit_msg>Fixed failing test caused by last commit<commit_after>package nativeregion

import (
	"testing"
	"functorama.com/demo/base"
	"functorama.com/demo/nativebase"
)

func TestRegionSplitPos(t *testing.T) {
	helper := NativeRegionSplitHelper{
		left:   1.0,
		right:  3.0,
		top:    3.0,
		bottom: 1.0,
		midR:   2.0,
		midI:   2.0,
	}

	testRegionSplit(helper, t)
}

func TestRegionSplitNeg(t *testing.T) {
	helper := NativeRegionSplitHelper{
		left:   -100.0,
		right:  -24.0,
		top:    -10.0,
		bottom: -340.0,
		midR:   -62.0,
		midI:   -175.0,
	}

	testRegionSplit(helper, t)
}

func TestRegionSplitNegPos(t *testing.T) {
	helper := NativeRegionSplitHelper{
		left:   -100.0,
		right:  24.0,
		top:    10.0,
		bottom: -340.0,
		midR:   -38.0,
		midI:   -165.0,
	}

	testRegionSplit(helper, t)
}

func TestChildrenPopulated(t *testing.T) {
	const inputChildCount = 4
	numerics := NativeRegionNumerics{
		subregion: nativeSubregion{
			populated: true,
			// We are not inspecting the children here
			children: make([]nativeRegion, inputChildCount),
		},
	}
	children := numerics.Children()

	expectedChildren := 4
	actualChildren := len(children)
	if actualChildren != expectedChildren {
		t.Error("Expected", expectedChildren, "but received", actualChildren)
	}
}

func TestChildrenEmpty(t *testing.T) {
	numerics := NativeRegionNumerics{}

	recovered := false
	triggerPanic := func() {
		defer func() {
			r := recover()
			recovered = r != nil
		}()
		numerics.Children()
	}

	triggerPanic()

	if !recovered {
		t.Error("Expected panic e.g. \"Error when raising error\"")
	}
}

func TestMandelbrotPoints(t *testing.T) {
	numerics := NativeRegionNumerics{}

	points := numerics.MandelbrotPoints()

	expectedPoints := 5
	actualPoints := len(points)

	if expectedPoints != actualPoints {
		t.Error("Expected to receive", expectedPoints, "but received", actualPoints)
	}
}

func TestEvaluateAllPoints(t *testing.T) {
	numerics := NativeRegionNumerics{}
	numerics.EvaluateAllPoints(1)
	region := numerics.Region

	okay := true
	for _, thunk := range numerics.thunks() {
		if !thunk.evaluated {
			okay = false
		}
		break
	}

	if !okay {
		t.Error("Expected all points to be evaluated, but region was:", region)
	}
}

func TestRect(t *testing.T) {
	const picSide = 2
	const planeSide = 2.0

	const left = -1
	const bottom = -1
	const right = left + planeSide
	const top = bottom + planeSide

	min := complex(left, bottom)
	max := complex(right, top)
	numerics := &NativeRegionNumerics{
		NativeBaseNumerics: nativebase.NativeBaseNumerics{
			BaseNumerics: base.BaseNumerics{
				PicXMin: 0,
				PicXMax: picSide,
				PicYMin: 0,
				PicYMax: picSide,
			},
			RealMin: left,
			RealMax: right,
			ImagMin: bottom,
			ImagMax: top,
		},
		Region:  createNativeRegion(min, max),
	}

	uq := nativebase.UnitQuery{picSide, picSide, planeSide, planeSide}
	numerics.Runit, numerics.Iunit = uq.PixelUnits()

	expectMinX := 0
	expectMaxX := 2
	expectMinY := 0
	expectMaxY := 2

	r := numerics.Rect()

	okay := expectMinX == r.Min.X
	okay = okay && expectMaxX == r.Max.X
	okay = okay && expectMinY == r.Min.Y
	okay = okay && expectMaxY == r.Max.Y
	if !okay {
		t.Error("Rectangle had unexpected bounds", r)
	}

}

func testRegionSplit(helper NativeRegionSplitHelper, t *testing.T) {
	initMin := complex(helper.left, helper.bottom)
	initMax := complex(helper.right, helper.top)

	topLeftMin := complex(helper.left, helper.midI)
	topLeftMax := complex(helper.midR, helper.top)

	topRightMin := complex(helper.midR, helper.midI)
	topRightMax := complex(helper.right, helper.top)

	bottomLeftMin := complex(helper.left, helper.bottom)
	bottomLeftMax := complex(helper.midR, helper.midI)

	bottomRightMin := complex(helper.midR, helper.bottom)
	bottomRightMax := complex(helper.right, helper.midI)

	subjectRegion := createNativeRegion(initMin, initMax)

	expected := []nativeRegion{
		createNativeRegion(topLeftMin, topLeftMax),
		createNativeRegion(topRightMin, topRightMax),
		createNativeRegion(bottomLeftMin, bottomLeftMax),
		createNativeRegion(bottomRightMin, bottomRightMax),
	}

	numerics := NativeRegionNumerics{
		Region: subjectRegion,
	}
	numerics.SqrtDivergeLimit = 2.0
	numerics.Split()
	actualChildren := numerics.subregion.children

	for i, ex := range expected {
		actual := actualChildren[i]
		exPoints := ex.thunks()
		acPoints := actual.thunks()
		fail := false
		for j, expectThunk := range exPoints {
			actThunk := acPoints[j]
			if *expectThunk != *actThunk {
				fail = true
				t.Log("Region", i, "error at thunk", j,
					"expected", expectThunk,
					"but received", actThunk)
			}
		}
		if fail {
			t.Fail()
		}
	}
}

type NativeRegionSplitHelper struct {
	left   float64
	right  float64
	bottom float64
	top    float64
	midR   float64
	midI   float64
}<|endoftext|>


<commit_before>// Package packaged provided access to configuration embedded directly in Lantern installation
// packages. On OSX, that means data embedded in the Lantern.app app bundle in
// Lantern.app/Contents/Resources/.lantern.yaml, while on Windows that means data embedded
// in AppData/Roaming/Lantern/.lantern.yaml. This allows customization embedded in the
// installer outside of the auto-updated binary that should only be used under special
// circumstances.
package packaged

import (
	"errors"
	"io/ioutil"
	"os"
	"path/filepath"
	"runtime"
	"strings"

	"github.com/getlantern/appdir"
	"github.com/getlantern/golog"
	"github.com/getlantern/yaml"
)

var (
	log  = golog.LoggerFor("flashlight.packaged")
	name = ".packaged-lantern.yaml"

	// This is the local copy of our embedded configuration file. This is necessary
	// to ensure we remember the embedded configuration across auto-updated
	// binaries. We write to the local file system instead of to the package
	// itself (app bundle on OSX, install directory on Windows) because
	// we're not always sure we can write to that directory.
	local = appdir.General("Lantern") + "/" + name
)

// PackagedSettings provided access to configuration embedded in the package.
type PackagedSettings struct {
	StartupUrl string
}

// ReadSettings reads packaged settings from pre-determined paths
// on the various OSes.
func ReadSettings() (string, *PackagedSettings, error) {
	yamlPath, err := packagedSettingsPath()
	if err != nil {
		return "", &PackagedSettings{}, err
	}

	path, ps, er := readSettingsFromFile(yamlPath)
	if er != nil {
		return readSettingsFromFile(local)
	}
	return path, ps, nil
}

// ReadSettingsFromFile reads PackagedSettings from the yaml file at the specified
// path.
func readSettingsFromFile(yamlPath string) (string, *PackagedSettings, error) {
	log.Debugf("Opening file at: %v", yamlPath)
	data, err := ioutil.ReadFile(yamlPath)
	if err != nil {
		// This will happen whenever there's no packaged settings, which is often
		log.Debugf("Error reading file %v", err)
		return "", &PackagedSettings{}, err
	}

	trimmed := strings.TrimSpace(string(data))

	log.Debugf("Read bytes: %v", trimmed)

	if trimmed == "" {
		log.Debugf("Ignoring empty string")
		return "", &PackagedSettings{}, errors.New("Empty string")
	}
	var s PackagedSettings
	err = yaml.Unmarshal([]byte(trimmed), &s)

	if err != nil {
		log.Errorf("Could not read yaml: %v", err)
		return "", &PackagedSettings{}, err
	}
	return yamlPath, &s, nil
}

func packagedSettingsPath() (string, error) {
	dir, err := filepath.Abs(filepath.Dir(os.Args[0]))
	if err != nil {
		log.Errorf("Could not get current directory %v", err)
		return "", err
	}
	log.Debugf("Opening externalUrl from: %v", dir)
	var yamldir string
	if runtime.GOOS == "windows" {
		yamldir = dir
	} else if runtime.GOOS == "darwin" {
		// Code signing doesn't like this file in the current directory
		// for whatever reason, so we grab it from the Resources
		// directory in the app bundle.
		yamldir = dir + "/../Resources"
	} else if runtime.GOOS == "linux" {
		yamldir = dir
	}
	yamlPath := yamldir + "/" + name
	return yamlPath, nil
}

func writeToDisk(ps *PackagedSettings) (string, error) {
	data, err := yaml.Marshal(ps)
	if err != nil {
		log.Errorf("Could not write to disk: %v", err)
		return "", err
	}
	err = ioutil.WriteFile(local, data, 0644)
	if err != nil {
		log.Errorf("Could not write to disk: %v", err)
		return "", err
	}
	return local, nil
}
<commit_msg>Check en.lproj directory instead<commit_after>// Package packaged provided access to configuration embedded directly in Lantern installation
// packages. On OSX, that means data embedded in the Lantern.app app bundle in
// Lantern.app/Contents/Resources/.lantern.yaml, while on Windows that means data embedded
// in AppData/Roaming/Lantern/.lantern.yaml. This allows customization embedded in the
// installer outside of the auto-updated binary that should only be used under special
// circumstances.
package packaged

import (
	"errors"
	"io/ioutil"
	"os"
	"path/filepath"
	"runtime"
	"strings"

	"github.com/getlantern/appdir"
	"github.com/getlantern/golog"
	"github.com/getlantern/yaml"
)

var (
	log  = golog.LoggerFor("flashlight.packaged")
	name = ".packaged-lantern.yaml"

	// This is the local copy of our embedded configuration file. This is necessary
	// to ensure we remember the embedded configuration across auto-updated
	// binaries. We write to the local file system instead of to the package
	// itself (app bundle on OSX, install directory on Windows) because
	// we're not always sure we can write to that directory.
	local = appdir.General("Lantern") + "/" + name
)

// PackagedSettings provided access to configuration embedded in the package.
type PackagedSettings struct {
	StartupUrl string
}

// ReadSettings reads packaged settings from pre-determined paths
// on the various OSes.
func ReadSettings() (string, *PackagedSettings, error) {
	yamlPath, err := packagedSettingsPath()
	if err != nil {
		return "", &PackagedSettings{}, err
	}

	path, ps, er := readSettingsFromFile(yamlPath)
	if er != nil {
		return readSettingsFromFile(local)
	}
	return path, ps, nil
}

// ReadSettingsFromFile reads PackagedSettings from the yaml file at the specified
// path.
func readSettingsFromFile(yamlPath string) (string, *PackagedSettings, error) {
	log.Debugf("Opening file at: %v", yamlPath)
	data, err := ioutil.ReadFile(yamlPath)
	if err != nil {
		// This will happen whenever there's no packaged settings, which is often
		log.Debugf("Error reading file %v", err)
		return "", &PackagedSettings{}, err
	}

	trimmed := strings.TrimSpace(string(data))

	log.Debugf("Read bytes: %v", trimmed)

	if trimmed == "" {
		log.Debugf("Ignoring empty string")
		return "", &PackagedSettings{}, errors.New("Empty string")
	}
	var s PackagedSettings
	err = yaml.Unmarshal([]byte(trimmed), &s)

	if err != nil {
		log.Errorf("Could not read yaml: %v", err)
		return "", &PackagedSettings{}, err
	}
	return yamlPath, &s, nil
}

func packagedSettingsPath() (string, error) {
	dir, err := filepath.Abs(filepath.Dir(os.Args[0]))
	if err != nil {
		log.Errorf("Could not get current directory %v", err)
		return "", err
	}
	log.Debugf("Opening externalUrl from: %v", dir)
	var yamldir string
	if runtime.GOOS == "windows" {
		yamldir = dir
	} else if runtime.GOOS == "darwin" {
		// Code signing doesn't like this file in the current directory
		// for whatever reason, so we grab it from the Resources
		// directory in the app bundle.
		yamldir = dir + "/../Resources/en.lproj"
	} else if runtime.GOOS == "linux" {
		yamldir = dir
	}
	yamlPath := yamldir + "/" + name
	return yamlPath, nil
}

func writeToDisk(ps *PackagedSettings) (string, error) {
	data, err := yaml.Marshal(ps)
	if err != nil {
		log.Errorf("Could not write to disk: %v", err)
		return "", err
	}
	err = ioutil.WriteFile(local, data, 0644)
	if err != nil {
		log.Errorf("Could not write to disk: %v", err)
		return "", err
	}
	return local, nil
}
<|endoftext|>


<commit_before>package hdfs

import (
	"errors"
	//"fmt"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"strconv"
	"sync"
)

func (h *WebHdfs) GetToLocal(path string, destination string, permission string) error {
	d, err := h.Get(path)
	if err != nil {
		return err
	}
	if permission == "" {
		permission = "755"
	}
	iperm, _ := strconv.Atoi(permission)
	err = ioutil.WriteFile(destination, d, os.FileMode(iperm))
	if err != nil {
		return err
	}
	return nil
}

func (h *WebHdfs) Get(path string) ([]byte, error) {
	r, err := h.call("GET", path, OP_OPEN, nil)
	if err != nil {
		return nil, err
	}
	if r.StatusCode != 307 {
		return nil, errors.New("Invalid Response Header on OP_OPEN: " + r.Status)
	}

	location := r.Header["Location"][0]
	r, err = h.call("GET", location, OP_OPEN, nil)
	if err != nil {
		return nil, err
	}
	if r.StatusCode != 200 {
		return nil, errors.New(r.Status)
	}
	d, err := ioutil.ReadAll(r.Body)
	if err != nil {
		return nil, err
	}
	defer r.Body.Close()
	return d, nil
}

func mergeMapString(source map[string]string, adds map[string]string) map[string]string {
	if source == nil {
		source = make(map[string]string)
	}
	if adds != nil {
		for k, v := range adds {
			source[k] = v
		}
	}
	return source
}

func (h *WebHdfs) Put(localfile string, destination string, permission string, parms map[string]string) error {
	if permission == "" {
		permission = "755"
	}
	parms = mergeMapString(parms, map[string]string{"permission": permission})
	r, err := h.call("PUT", destination, OP_CREATE, parms)
	if err != nil {
		return err
	}
	if r.StatusCode != 307 {
		return errors.New("Invalid Response Header on OP_CREATE: " + r.Status)
	}

	location := r.Header["Location"][0]

	r, err = h.callPayload("PUT", location, OP_CREATE, localfile, nil)
	if err != nil {
		return err
	}
	if r.StatusCode != 201 {
		return errors.New(r.Status)
	}
	return nil
}

func (h *WebHdfs) Puts(paths []string, destinationFolder string, permission string, parms map[string]string) map[string]error {
	var es map[string]error
	if permission == "" {
		permission = "755"
	}

	fileCount := len(paths)

	//parms = mergeMapString(parms, map[string]string{"permission": strconv.Itoa(permission)})
	ipool := 0
	iprocessing := 0
	iread := 0
	files := []string{}
	for _, path := range paths {
		ipool = ipool + 1
		iread = iread + 1
		files = append(files, path)
		if ipool == h.Config.PoolSize || iread == fileCount {
			wg := sync.WaitGroup{}
			wg.Add(ipool)

			for _, f := range files {
				go func(path string, swg *sync.WaitGroup) {
					defer swg.Done()
					iprocessing = iprocessing + 1
					_, filename := filepath.Split(path)
					newfilename := filepath.Join(destinationFolder, filename)
					e := h.Put(path, newfilename, permission, parms)
					//var e error
					if e != nil {
						if es == nil {
							es = make(map[string]error)
							es[path] = e
						}
						//fmt.Println(path, "=> ", newfilename, " ... FAIL => ", e.Error(), " | Processing ", iprocessing, " of ", fileCount)
					} else {
						//fmt.Println(path, "=> ", newfilename, " ... SUCCESS | Processing ", iprocessing, " of ", fileCount)
					}
				}(f, &wg)
			}

			wg.Wait()
			ipool = 0
			files = []string{}
		}
	}

	return es
}

func (h *WebHdfs) Append(localfile string, destination string) error {
	r, err := h.call("POST", destination, OP_APPEND, nil)
	if err != nil {
		return err
	}
	if r.StatusCode != 307 {
		return errors.New("Invalid Response Header on OP_APPEND: " + r.Status)
	}

	location := r.Header["Location"][0]

	r, err = h.callPayload("POST", location, OP_APPEND, localfile, nil)
	if err != nil {
		return err
	}
	if r.StatusCode != 201 {
		return errors.New(r.Status)
	}
	return nil
}

func (h *WebHdfs) SetOwner(path string, owner string, group string) error {
	ownerInfo := map[string]string{}
	if owner != "" {
		ownerInfo["owner"] = owner
	}
	if group != "" {
		ownerInfo["group"] = group
	}
	r, e := h.call("PUT", path, OP_SETOWNER, ownerInfo)
	if e != nil {
		return e
	}
	if r.StatusCode != 200 {
		return errors.New("Invalid Response Header on OP_SETOWNER: " + r.Status)
	}
	return nil
}

func (h *WebHdfs) SetPermission(path string, permission string) error {
	if permission == "" {
		permission = "755"
	}

	parms := map[string]string{}
	parms["permission"] = permission

	r, e := h.call("PUT", path, OP_SETPERMISSION, parms)
	if e != nil {
		return e
	}
	if r.StatusCode != 200 {
		return errors.New("Invalid Response Header on OP_SETPERMISSION: " + r.Status)
	}
	return nil
}

/*
func (h *WebHdfs) CreateNewFile(path, filename, permission string) error {
	if permission == "" {
		permission = "755"
	}

	parms := map[string]string{}
	parms["permission"] = permission

	var fullpath string

	if string(path[len(path)-1]) == "/" {
		fullpath = path + filename
	} else {
		fullpath = path + "/" + filename
	}

	log.Println(fullpath)

	r, e := h.call("PUT", fullpath, OP_CREATE, parms)
	if e != nil {
		return e
	}
	if r.StatusCode != 200 {
		return errors.New("Invalid Response Header on OP_CREATE: " + r.Status)
	}
	return nil
}
*/
<commit_msg>remove log<commit_after>package hdfs

import (
	"errors"
	//"fmt"
	"io/ioutil"
	//"log"
	"os"
	"path/filepath"
	"strconv"
	"sync"
)

func (h *WebHdfs) GetToLocal(path string, destination string, permission string) error {
	d, err := h.Get(path)
	if err != nil {
		return err
	}
	if permission == "" {
		permission = "755"
	}
	iperm, _ := strconv.Atoi(permission)
	err = ioutil.WriteFile(destination, d, os.FileMode(iperm))
	if err != nil {
		return err
	}
	return nil
}

func (h *WebHdfs) Get(path string) ([]byte, error) {
	r, err := h.call("GET", path, OP_OPEN, nil)
	if err != nil {
		return nil, err
	}
	if r.StatusCode != 307 {
		return nil, errors.New("Invalid Response Header on OP_OPEN: " + r.Status)
	}

	location := r.Header["Location"][0]
	r, err = h.call("GET", location, OP_OPEN, nil)
	if err != nil {
		return nil, err
	}
	if r.StatusCode != 200 {
		return nil, errors.New(r.Status)
	}
	d, err := ioutil.ReadAll(r.Body)
	if err != nil {
		return nil, err
	}
	defer r.Body.Close()
	return d, nil
}

func mergeMapString(source map[string]string, adds map[string]string) map[string]string {
	if source == nil {
		source = make(map[string]string)
	}
	if adds != nil {
		for k, v := range adds {
			source[k] = v
		}
	}
	return source
}

func (h *WebHdfs) Put(localfile string, destination string, permission string, parms map[string]string) error {
	if permission == "" {
		permission = "755"
	}
	parms = mergeMapString(parms, map[string]string{"permission": permission})
	r, err := h.call("PUT", destination, OP_CREATE, parms)
	if err != nil {
		return err
	}
	if r.StatusCode != 307 {
		return errors.New("Invalid Response Header on OP_CREATE: " + r.Status)
	}

	location := r.Header["Location"][0]

	r, err = h.callPayload("PUT", location, OP_CREATE, localfile, nil)
	if err != nil {
		return err
	}
	if r.StatusCode != 201 {
		return errors.New(r.Status)
	}
	return nil
}

func (h *WebHdfs) Puts(paths []string, destinationFolder string, permission string, parms map[string]string) map[string]error {
	var es map[string]error
	if permission == "" {
		permission = "755"
	}

	fileCount := len(paths)

	//parms = mergeMapString(parms, map[string]string{"permission": strconv.Itoa(permission)})
	ipool := 0
	iprocessing := 0
	iread := 0
	files := []string{}
	for _, path := range paths {
		ipool = ipool + 1
		iread = iread + 1
		files = append(files, path)
		if ipool == h.Config.PoolSize || iread == fileCount {
			wg := sync.WaitGroup{}
			wg.Add(ipool)

			for _, f := range files {
				go func(path string, swg *sync.WaitGroup) {
					defer swg.Done()
					iprocessing = iprocessing + 1
					_, filename := filepath.Split(path)
					newfilename := filepath.Join(destinationFolder, filename)
					e := h.Put(path, newfilename, permission, parms)
					//var e error
					if e != nil {
						if es == nil {
							es = make(map[string]error)
							es[path] = e
						}
						//fmt.Println(path, "=> ", newfilename, " ... FAIL => ", e.Error(), " | Processing ", iprocessing, " of ", fileCount)
					} else {
						//fmt.Println(path, "=> ", newfilename, " ... SUCCESS | Processing ", iprocessing, " of ", fileCount)
					}
				}(f, &wg)
			}

			wg.Wait()
			ipool = 0
			files = []string{}
		}
	}

	return es
}

func (h *WebHdfs) Append(localfile string, destination string) error {
	r, err := h.call("POST", destination, OP_APPEND, nil)
	if err != nil {
		return err
	}
	if r.StatusCode != 307 {
		return errors.New("Invalid Response Header on OP_APPEND: " + r.Status)
	}

	location := r.Header["Location"][0]

	r, err = h.callPayload("POST", location, OP_APPEND, localfile, nil)
	if err != nil {
		return err
	}
	if r.StatusCode != 201 {
		return errors.New(r.Status)
	}
	return nil
}

func (h *WebHdfs) SetOwner(path string, owner string, group string) error {
	ownerInfo := map[string]string{}
	if owner != "" {
		ownerInfo["owner"] = owner
	}
	if group != "" {
		ownerInfo["group"] = group
	}
	r, e := h.call("PUT", path, OP_SETOWNER, ownerInfo)
	if e != nil {
		return e
	}
	if r.StatusCode != 200 {
		return errors.New("Invalid Response Header on OP_SETOWNER: " + r.Status)
	}
	return nil
}

func (h *WebHdfs) SetPermission(path string, permission string) error {
	if permission == "" {
		permission = "755"
	}

	parms := map[string]string{}
	parms["permission"] = permission

	r, e := h.call("PUT", path, OP_SETPERMISSION, parms)
	if e != nil {
		return e
	}
	if r.StatusCode != 200 {
		return errors.New("Invalid Response Header on OP_SETPERMISSION: " + r.Status)
	}
	return nil
}

/*
func (h *WebHdfs) CreateNewFile(path, filename, permission string) error {
	if permission == "" {
		permission = "755"
	}

	parms := map[string]string{}
	parms["permission"] = permission

	var fullpath string

	if string(path[len(path)-1]) == "/" {
		fullpath = path + filename
	} else {
		fullpath = path + "/" + filename
	}

	log.Println(fullpath)

	r, e := h.call("PUT", fullpath, OP_CREATE, parms)
	if e != nil {
		return e
	}
	if r.StatusCode != 200 {
		return errors.New("Invalid Response Header on OP_CREATE: " + r.Status)
	}
	return nil
}
*/
<|endoftext|>