<commit_before>// Package requestor makes calling RESTful API's easy.
package requestor

import (
	"bytes"
	"crypto/tls"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"
	"strings"
)

// Options contains parameters to be defined before sending the request to the server. Certain values
// can be omitted based on the request method (i.e. GET typically won't need to send a Body).
type Options struct {
	Method      string
	Query       map[string]string
	ContentType string
	Body        string
	Auth        []string
	Headers     map[string]string
}

// HTTPData contains the information returned from our request.
type HTTPData struct {
	Status  string
	Code    int
	Headers http.Header
	Body    []byte
	Error   error
}

// Send issues an HTTP request with the values specified in Options.
func Send(uri string, options *Options) *HTTPData {
	var req *http.Request
	var data HTTPData
	client := &http.Client{
		Transport: &http.Transport{
			TLSClientConfig: &tls.Config{
				InsecureSkipVerify: true,
			},
		},
	}
	u, err := url.Parse(uri)
	if err != nil {
		data.Error = err

		return &data
	}

	query := u.Query()
	for k := range options.Query {
		query.Add(k, options.Query[k])
	}

	u.RawQuery = query.Encode()
	body := bytes.NewReader([]byte(options.Body))
	req, _ = http.NewRequest(strings.ToUpper(options.Method), u.String(), body)

	if len(options.Auth) > 0 {
		req.SetBasicAuth(options.Auth[0], options.Auth[1])
	}

	if len(options.ContentType) > 0 {
		req.Header.Set("Content-Type", options.ContentType)
	}

	if len(options.Headers) > 0 {
		for k := range options.Headers {
			req.Header.Add(k, options.Headers[k])
		}
	}

	res, err := client.Do(req)
	if err != nil {
		data.Error = err

		return &data
	}

	defer res.Body.Close()

	payload, _ := ioutil.ReadAll(res.Body)
	data.Body = payload
	data.Code = res.StatusCode
	data.Status = res.Status
	data.Headers = res.Header

	if res.StatusCode >= 400 {
		data.Error = fmt.Errorf("HTTP %d: %s", res.StatusCode, string(payload))
	}

	return &data
}
<commit_msg>Added ability to call a simple GET request<commit_after>// Package requestor makes calling RESTful API's easy.
package requestor

import (
	"bytes"
	"crypto/tls"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"
	"strings"
)

// Options contains parameters to be defined before sending the request to the server. Certain values
// can be omitted based on the request method (i.e. GET typically won't need to send a Body).
type Options struct {
	Method      string
	Query       map[string]string
	ContentType string
	Body        string
	Auth        []string
	Headers     map[string]string
}

// HTTPData contains the information returned from our request.
type HTTPData struct {
	Status  string
	Code    int
	Headers http.Header
	Body    []byte
	Error   error
}

// Send issues an HTTP request with the values specified in Options.
func Send(uri string, options *Options) *HTTPData {
	var req *http.Request
	var data HTTPData

	if options == nil {
		res, err := http.Get(uri)
		if err != nil {
			data.Error = err

			return &data
		}

		defer res.Body.Close()

		payload, _ := ioutil.ReadAll(res.Body)
		data.Body = payload
		data.Code = res.StatusCode
		data.Status = res.Status
		data.Headers = res.Header

		if res.StatusCode >= 400 {
			data.Error = fmt.Errorf("HTTP %d: %s", res.StatusCode, string(payload))
		}

		return &data
	}

	client := &http.Client{
		Transport: &http.Transport{
			TLSClientConfig: &tls.Config{
				InsecureSkipVerify: true,
			},
		},
	}
	u, err := url.Parse(uri)
	if err != nil {
		data.Error = err

		return &data
	}

	query := u.Query()
	for k := range options.Query {
		query.Add(k, options.Query[k])
	}

	u.RawQuery = query.Encode()
	body := bytes.NewReader([]byte(options.Body))
	req, _ = http.NewRequest(strings.ToUpper(options.Method), u.String(), body)

	if len(options.Auth) > 0 {
		req.SetBasicAuth(options.Auth[0], options.Auth[1])
	}

	if len(options.ContentType) > 0 {
		req.Header.Set("Content-Type", options.ContentType)
	}

	if len(options.Headers) > 0 {
		for k := range options.Headers {
			req.Header.Add(k, options.Headers[k])
		}
	}

	res, err := client.Do(req)
	if err != nil {
		data.Error = err

		return &data
	}

	defer res.Body.Close()

	payload, _ := ioutil.ReadAll(res.Body)
	data.Body = payload
	data.Code = res.StatusCode
	data.Status = res.Status
	data.Headers = res.Header

	if res.StatusCode >= 400 {
		data.Error = fmt.Errorf("HTTP %d: %s", res.StatusCode, string(payload))
	}

	return &data
}
<|endoftext|>


<commit_before>package util

import (
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"runtime"
	"time"

	"github.com/mndrix/tap-go"
	"github.com/mrunalp/fileutils"
	"github.com/opencontainers/runtime-tools/generate"
	"github.com/satori/go.uuid"
)

var (
	// RuntimeCommand is the default runtime command.
	RuntimeCommand = "runc"
)

// PreFunc initializes the test environment after preparing the bundle
// but before creating the container.
type PreFunc func(string) error

func init() {
	runtimeInEnv := os.Getenv("RUNTIME")
	if runtimeInEnv != "" {
		RuntimeCommand = runtimeInEnv
	}
}

// Fatal prints a warning to stderr and exits.
func Fatal(err error) {
	fmt.Fprintf(os.Stderr, "%+v\n", err)
	os.Exit(1)
}

// Skip skips a full TAP suite.
func Skip(message string, diagnostic string) {
	t := tap.New()
	t.Header(1)
	t.Skip(1, message)
	if diagnostic != "" {
		t.Diagnostic(diagnostic)
	}
}

// PrepareBundle creates a test bundle in a temporary directory.
func PrepareBundle() (string, error) {
	bundleDir, err := ioutil.TempDir("", "ocitest")
	if err != nil {
		return "", err
	}

	// Untar the root fs
	untarCmd := exec.Command("tar", "-xf", fmt.Sprintf("rootfs-%s.tar.gz", runtime.GOARCH), "-C", bundleDir)
	output, err := untarCmd.CombinedOutput()
	if err != nil {
		os.Stderr.Write(output)
		os.RemoveAll(bundleDir)
		return "", err
	}

	return bundleDir, nil
}

// GetDefaultGenerator creates a default configuration generator.
func GetDefaultGenerator() *generate.Generator {
	g := generate.New()
	g.SetRootPath(".")
	g.SetProcessArgs([]string{"/runtimetest", "--path=/"})
	return &g
}

// RuntimeInsideValidate runs runtimetest inside a container.
func RuntimeInsideValidate(g *generate.Generator, f PreFunc) (err error) {
	bundleDir, err := PrepareBundle()
	if err != nil {
		return err
	}

	if f != nil {
		if err := f(bundleDir); err != nil {
			return err
		}
	}

	r, err := NewRuntime(RuntimeCommand, bundleDir)
	if err != nil {
		os.RemoveAll(bundleDir)
		return err
	}
	defer r.Clean(true, true)
	err = r.SetConfig(g)
	if err != nil {
		return err
	}
	err = fileutils.CopyFile("runtimetest", filepath.Join(r.BundleDir, "runtimetest"))
	if err != nil {
		return err
	}

	r.SetID(uuid.NewV4().String())
	stderr, err := r.Create()
	if err != nil {
		os.Stderr.WriteString("failed to create the container\n")
		os.Stderr.Write(stderr)
		return err
	}

	// FIXME: why do we need this?  Without a sleep here, I get:
	//   failed to start the container
	//   container "..." does not exist
	time.Sleep(1 * time.Second)

	stderr, err = r.Start()
	if err != nil {
		os.Stderr.WriteString("failed to start the container\n")
		os.Stderr.Write(stderr)
		return err
	}

	// FIXME: wait until the container exits and collect its exit code.
	time.Sleep(1 * time.Second)

	stdout, stderr, err := r.ReadStandardStreams()
	if err != nil {
		if len(stderr) == 0 {
			stderr = stdout
		}
		os.Stderr.WriteString("failed to read standard streams\n")
		os.Stderr.Write(stderr)
		return err
	}

	os.Stdout.Write(stdout)
	return nil
}
<commit_msg>add runtimeOutsideValidate<commit_after>package util

import (
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"runtime"
	"time"

	"github.com/mndrix/tap-go"
	"github.com/mrunalp/fileutils"
	"github.com/opencontainers/runtime-tools/generate"
	"github.com/satori/go.uuid"
)

var (
	// RuntimeCommand is the default runtime command.
	RuntimeCommand = "runc"
)

// PreFunc initializes the test environment after preparing the bundle
// but before creating the container.
type PreFunc func(string) error

func init() {
	runtimeInEnv := os.Getenv("RUNTIME")
	if runtimeInEnv != "" {
		RuntimeCommand = runtimeInEnv
	}
}

// Fatal prints a warning to stderr and exits.
func Fatal(err error) {
	fmt.Fprintf(os.Stderr, "%+v\n", err)
	os.Exit(1)
}

// Skip skips a full TAP suite.
func Skip(message string, diagnostic string) {
	t := tap.New()
	t.Header(1)
	t.Skip(1, message)
	if diagnostic != "" {
		t.Diagnostic(diagnostic)
	}
}

// PrepareBundle creates a test bundle in a temporary directory.
func PrepareBundle() (string, error) {
	bundleDir, err := ioutil.TempDir("", "ocitest")
	if err != nil {
		return "", err
	}

	// Untar the root fs
	untarCmd := exec.Command("tar", "-xf", fmt.Sprintf("rootfs-%s.tar.gz", runtime.GOARCH), "-C", bundleDir)
	output, err := untarCmd.CombinedOutput()
	if err != nil {
		os.Stderr.Write(output)
		os.RemoveAll(bundleDir)
		return "", err
	}

	return bundleDir, nil
}

// GetDefaultGenerator creates a default configuration generator.
func GetDefaultGenerator() *generate.Generator {
	g := generate.New()
	g.SetRootPath(".")
	g.SetProcessArgs([]string{"/runtimetest", "--path=/"})
	return &g
}

// RuntimeInsideValidate runs runtimetest inside a container.
func RuntimeInsideValidate(g *generate.Generator, f PreFunc) (err error) {
	bundleDir, err := PrepareBundle()
	if err != nil {
		return err
	}

	if f != nil {
		if err := f(bundleDir); err != nil {
			return err
		}
	}

	r, err := NewRuntime(RuntimeCommand, bundleDir)
	if err != nil {
		os.RemoveAll(bundleDir)
		return err
	}
	defer r.Clean(true, true)
	err = r.SetConfig(g)
	if err != nil {
		return err
	}
	err = fileutils.CopyFile("runtimetest", filepath.Join(r.BundleDir, "runtimetest"))
	if err != nil {
		return err
	}

	r.SetID(uuid.NewV4().String())
	stderr, err := r.Create()
	if err != nil {
		os.Stderr.WriteString("failed to create the container\n")
		os.Stderr.Write(stderr)
		return err
	}

	// FIXME: why do we need this?  Without a sleep here, I get:
	//   failed to start the container
	//   container "..." does not exist
	time.Sleep(1 * time.Second)

	stderr, err = r.Start()
	if err != nil {
		os.Stderr.WriteString("failed to start the container\n")
		os.Stderr.Write(stderr)
		return err
	}

	// FIXME: wait until the container exits and collect its exit code.
	time.Sleep(1 * time.Second)

	stdout, stderr, err := r.ReadStandardStreams()
	if err != nil {
		if len(stderr) == 0 {
			stderr = stdout
		}
		os.Stderr.WriteString("failed to read standard streams\n")
		os.Stderr.Write(stderr)
		return err
	}

	os.Stdout.Write(stdout)
	return nil
}

// RuntimeOutsideValidate validate runtime outside a container.
func RuntimeOutsideValidate(g *generate.Generator, f PreFunc) error {
	bundleDir, err := PrepareBundle()
	if err != nil {
		return err
	}

	r, err := NewRuntime(RuntimeCommand, bundleDir)
	if err != nil {
		os.RemoveAll(bundleDir)
		return err
	}
	defer r.Clean(true, true)
	err = r.SetConfig(g)
	if err != nil {
		return err
	}
	err = fileutils.CopyFile("runtimetest", filepath.Join(r.BundleDir, "runtimetest"))
	if err != nil {
		return err
	}

	r.SetID(uuid.NewV4().String())
	stderr, err := r.Create()
	if err != nil {
		os.Stderr.WriteString("failed to start the container\n")
		os.Stderr.Write(stderr)
		return err
	}

	if f != nil {
		if err := f("test"); err != nil {
			return err
		}
	}
	return nil
}
<|endoftext|>


<commit_before>// Copyright 2017 The Neugram Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package format

import (
	"bytes"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"os/exec"
	"reflect"
	"strings"
)

type debugPrinter struct {
	buf     *bytes.Buffer
	ptrseen map[interface{}]int // ptr -> count seen
	ptrdone map[interface{}]bool
	indent  int
}

func (p *debugPrinter) collectPtrs(v reflect.Value) {
	switch v.Kind() {
	case reflect.Ptr:
		if !v.CanInterface() {
			return
		}
		ptr := v.Interface()
		p.ptrseen[ptr]++
		if p.ptrseen[ptr] == 1 {
			p.collectPtrs(v.Elem())
		}
	case reflect.Interface:
		if repack := reflect.ValueOf(v.Interface()); repack.Kind() == reflect.Ptr {
			p.collectPtrs(repack)
		} else {
			p.collectPtrs(v.Elem())
		}
	case reflect.Map:
		for _, key := range v.MapKeys() {
			p.collectPtrs(key)
			p.collectPtrs(v.MapIndex(key))
		}
	case reflect.Array:
	case reflect.Slice:
		for i := 0; i < v.Len(); i++ {
			p.collectPtrs(v.Index(i))
		}
	case reflect.Struct:
		for i := 0; i < v.NumField(); i++ {
			p.collectPtrs(v.Field(i))
		}
	}
}

func (p *debugPrinter) printf(format string, args ...interface{}) {
	fmt.Fprintf(p.buf, format, args...)
}

func (p *debugPrinter) newline() {
	p.buf.WriteByte('\n')
	for i := 0; i < p.indent; i++ {
		p.buf.WriteByte('\t')
	}
}

func isZero(v reflect.Value) bool {
	switch v.Kind() {
	case reflect.Chan, reflect.Func, reflect.Interface, reflect.Map, reflect.Ptr, reflect.Slice:
		return v.IsNil()
	case reflect.Struct:
		for i := 0; i < v.NumField(); i++ {
			if !isZero(v.Field(i)) {
				return false
			}
		}
		return true
	case reflect.Array:
		for i := 0; i < v.Len(); i++ {
			if !isZero(v.Index(i)) {
				return false
			}
		}
		return true
	case reflect.Bool:
		return !v.Bool()
	case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
		return v.Int() == 0
	case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr:
		return v.Uint() == 0
	case reflect.Float32, reflect.Float64:
		return v.Float() == 0
	case reflect.Complex64, reflect.Complex128:
		return v.Complex() == 0
	case reflect.String:
		return v.String() == ""
	}
	return false
}

func (p *debugPrinter) printv(v reflect.Value) {
	switch v.Kind() {
	case reflect.Chan, reflect.Func, reflect.Interface, reflect.Map, reflect.Ptr, reflect.Slice:
		if v.IsNil() {
			p.buf.WriteString("nil")
			return
		}
	}

	switch v.Kind() {
	case reflect.Ptr:
		if !v.CanInterface() {
			p.printf("unexported")
			return
		}
		p.printf("&")
		ptr := v.Interface()
		if p.ptrdone[ptr] {
			p.printf("%p", ptr)
		} else if p.ptrseen[ptr] > 1 {
			// TODO: p.printv(v.Elem())
			p.printf(" (TODO type %T)", ptr)
			p.ptrdone[ptr] = true
			p.printf(" (ptr %p)", ptr)
		} else {
			p.printv(v.Elem())
		}
	case reflect.Interface:
		if repack := reflect.ValueOf(v.Interface()); repack.Kind() == reflect.Ptr {
			p.printv(repack)
			return
		}
		p.printv(v.Elem())
	case reflect.Map:
		p.printf("%s{", v.Type())
		if v.Len() == 1 {
			key := v.MapKeys()[0]
			p.printf("%s: ", key)
			p.printv(v.MapIndex(key))
		} else if v.Len() > 0 {
			p.indent++
			for _, key := range v.MapKeys() {
				p.newline()
				p.printf("%s: ", key)
				p.printv(v.MapIndex(key))
				p.buf.WriteByte(',')
			}
			p.indent--
			p.newline()
		}
		p.buf.WriteByte('}')
	case reflect.Slice:
		if v.Type().Elem().Kind() == reflect.Int8 {
			s := v.Bytes()
			p.printf("%#q", s)
			return
		}
		fallthrough
	case reflect.Array:
		p.printf("%s{", v.Type())
		if v.Len() > 0 {
			p.indent++
			for i := 0; i < v.Len(); i++ {
				p.newline()
				p.printv(v.Index(i))
				p.buf.WriteByte(',')
			}
			p.indent--
			p.newline()
		}
		p.buf.WriteByte('}')
	case reflect.Struct:
		t := v.Type()
		p.printf("%s{", t)
		if v.NumField() > 0 {
			p.indent++
			for i := 0; i < v.NumField(); i++ {
				if isZero(v.Field(i)) {
					continue
				}
				p.newline()
				p.printf("%s: ", t.Field(i).Name)
				p.printv(v.Field(i))
				p.buf.WriteByte(',')
			}
			p.indent--
			p.newline()
		}
		p.buf.WriteByte('}')
	case reflect.Bool,
		reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64,
		reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr,
		reflect.Float32, reflect.Float64, reflect.Complex64, reflect.Complex128:
		p.printf("%s(%v)", v.Type().Name(), v)
	default:
		if !v.IsValid() {
			p.printf("?")
		} else if v.Kind() == reflect.String {
			p.printf("%q", v.String())
		} else if v.CanInterface() {
			p.printf("%#v", v.Interface())
		} else {
			p.printf("?")
		}
	}
}

func printToFile(x interface{}) (path string, err error) {
	f, err := ioutil.TempFile("", "neugram-diff-")
	if err != nil {
		return "", err
	}
	defer func() {
		err2 := f.Close()
		if err == nil {
			err = err2
		}
		if err != nil {
			os.Remove(f.Name())
		}
	}()

	str := Debug(x)
	if _, err := io.WriteString(f, str); err != nil {
		return "", err
	}
	return f.Name(), nil
}

func diffVal(x, y interface{}) (string, error) {
	fx, err := printToFile(x)
	if err != nil {
		return "", fmt.Errorf("diff print lhs error: %v", err)
	}
	defer os.Remove(fx)
	fy, err := printToFile(y)
	if err != nil {
		return "", fmt.Errorf("diff print rhs error: %v", err)
	}
	defer os.Remove(fy)

	b, _ := ioutil.ReadFile(fx)
	fmt.Printf("fx: %s\n", b)

	data, err := exec.Command("diff", "-U100", "-u", fx, fy).CombinedOutput()
	if err != nil && len(data) == 0 {
		// diff exits with a non-zero status when the files don't match.
		return "", fmt.Errorf("diff error: %v", err)
	}
	res := string(data)
	res = strings.Replace(res, fx, "/x", 1)
	res = strings.Replace(res, fy, "/y", 1)
	return res, nil
}

func WriteDebug(buf *bytes.Buffer, e interface{}) {
	p := debugPrinter{
		buf:     buf,
		ptrseen: make(map[interface{}]int),
		ptrdone: make(map[interface{}]bool),
	}
	v := reflect.ValueOf(e)
	p.collectPtrs(v)
	p.printv(v)
}

func Debug(e interface{}) string {
	buf := new(bytes.Buffer)
	WriteDebug(buf, e)
	return buf.String()
}

func Diff(x, y interface{}) string {
	s, err := diffVal(x, y)
	if err != nil {
		return fmt.Sprintf("format.Diff: %v", err)
	}
	return s
}
<commit_msg>format: print compilable named basic types<commit_after>// Copyright 2017 The Neugram Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package format

import (
	"bytes"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"os/exec"
	"reflect"
	"strings"
)

type debugPrinter struct {
	buf     *bytes.Buffer
	ptrseen map[interface{}]int // ptr -> count seen
	ptrdone map[interface{}]bool
	indent  int
}

func (p *debugPrinter) collectPtrs(v reflect.Value) {
	switch v.Kind() {
	case reflect.Ptr:
		if !v.CanInterface() {
			return
		}
		ptr := v.Interface()
		p.ptrseen[ptr]++
		if p.ptrseen[ptr] == 1 {
			p.collectPtrs(v.Elem())
		}
	case reflect.Interface:
		if repack := reflect.ValueOf(v.Interface()); repack.Kind() == reflect.Ptr {
			p.collectPtrs(repack)
		} else {
			p.collectPtrs(v.Elem())
		}
	case reflect.Map:
		for _, key := range v.MapKeys() {
			p.collectPtrs(key)
			p.collectPtrs(v.MapIndex(key))
		}
	case reflect.Array:
	case reflect.Slice:
		for i := 0; i < v.Len(); i++ {
			p.collectPtrs(v.Index(i))
		}
	case reflect.Struct:
		for i := 0; i < v.NumField(); i++ {
			p.collectPtrs(v.Field(i))
		}
	}
}

func (p *debugPrinter) printf(format string, args ...interface{}) {
	fmt.Fprintf(p.buf, format, args...)
}

func (p *debugPrinter) newline() {
	p.buf.WriteByte('\n')
	for i := 0; i < p.indent; i++ {
		p.buf.WriteByte('\t')
	}
}

func isZero(v reflect.Value) bool {
	switch v.Kind() {
	case reflect.Chan, reflect.Func, reflect.Interface, reflect.Map, reflect.Ptr, reflect.Slice:
		return v.IsNil()
	case reflect.Struct:
		for i := 0; i < v.NumField(); i++ {
			if !isZero(v.Field(i)) {
				return false
			}
		}
		return true
	case reflect.Array:
		for i := 0; i < v.Len(); i++ {
			if !isZero(v.Index(i)) {
				return false
			}
		}
		return true
	case reflect.Bool:
		return !v.Bool()
	case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
		return v.Int() == 0
	case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr:
		return v.Uint() == 0
	case reflect.Float32, reflect.Float64:
		return v.Float() == 0
	case reflect.Complex64, reflect.Complex128:
		return v.Complex() == 0
	case reflect.String:
		return v.String() == ""
	}
	return false
}

func (p *debugPrinter) printv(v reflect.Value) {
	switch v.Kind() {
	case reflect.Chan, reflect.Func, reflect.Interface, reflect.Map, reflect.Ptr, reflect.Slice:
		if v.IsNil() {
			p.buf.WriteString("nil")
			return
		}
	}

	switch v.Kind() {
	case reflect.Ptr:
		if !v.CanInterface() {
			p.printf("unexported")
			return
		}
		p.printf("&")
		ptr := v.Interface()
		if p.ptrdone[ptr] {
			p.printf("%p", ptr)
		} else if p.ptrseen[ptr] > 1 {
			// TODO: p.printv(v.Elem())
			p.printf(" (TODO type %T)", ptr)
			p.ptrdone[ptr] = true
			p.printf(" (ptr %p)", ptr)
		} else {
			p.printv(v.Elem())
		}
	case reflect.Interface:
		if repack := reflect.ValueOf(v.Interface()); repack.Kind() == reflect.Ptr {
			p.printv(repack)
			return
		}
		p.printv(v.Elem())
	case reflect.Map:
		p.printf("%s{", v.Type())
		if v.Len() == 1 {
			key := v.MapKeys()[0]
			p.printf("%s: ", key)
			p.printv(v.MapIndex(key))
		} else if v.Len() > 0 {
			p.indent++
			for _, key := range v.MapKeys() {
				p.newline()
				p.printv(key)
				p.printf(": ")
				p.printv(v.MapIndex(key))
				p.buf.WriteByte(',')
			}
			p.indent--
			p.newline()
		}
		p.buf.WriteByte('}')
	case reflect.Slice:
		if v.Type().Elem().Kind() == reflect.Int8 {
			s := v.Bytes()
			p.printf("%#q", s)
			return
		}
		fallthrough
	case reflect.Array:
		p.printf("%s{", v.Type())
		if v.Len() > 0 {
			p.indent++
			for i := 0; i < v.Len(); i++ {
				p.newline()
				p.printv(v.Index(i))
				p.buf.WriteByte(',')
			}
			p.indent--
			p.newline()
		}
		p.buf.WriteByte('}')
	case reflect.Struct:
		t := v.Type()
		p.printf("%s{", t)
		if v.NumField() > 0 {
			p.indent++
			for i := 0; i < v.NumField(); i++ {
				if isZero(v.Field(i)) {
					continue
				}
				p.newline()
				p.printf("%s: ", t.Field(i).Name)
				p.printv(v.Field(i))
				p.buf.WriteByte(',')
			}
			p.indent--
			p.newline()
		}
		p.buf.WriteByte('}')
	case reflect.Bool,
		reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64,
		reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr,
		reflect.Float32, reflect.Float64, reflect.Complex64, reflect.Complex128:
		str := fmt.Sprintf("%#v", v)
		p.printf("%s(%s", v.Type(), str)
		if v.CanInterface() {
			if stringer, is := v.Interface().(fmt.Stringer); is {
				str2 := stringer.String()
				if str2 != str {
					p.printf(" /* %s */", str2)
				}
			}
		}
		p.printf(")")
	default:
		if !v.IsValid() {
			p.printf("?")
		} else if v.Kind() == reflect.String {
			p.printf("%q", v.String())
		} else if v.CanInterface() {
			p.printf("%#v", v.Interface())
		} else {
			p.printf("?")
		}
	}
}

func printToFile(x interface{}) (path string, err error) {
	f, err := ioutil.TempFile("", "neugram-diff-")
	if err != nil {
		return "", err
	}
	defer func() {
		err2 := f.Close()
		if err == nil {
			err = err2
		}
		if err != nil {
			os.Remove(f.Name())
		}
	}()

	str := Debug(x)
	if _, err := io.WriteString(f, str); err != nil {
		return "", err
	}
	return f.Name(), nil
}

func diffVal(x, y interface{}) (string, error) {
	fx, err := printToFile(x)
	if err != nil {
		return "", fmt.Errorf("diff print lhs error: %v", err)
	}
	defer os.Remove(fx)
	fy, err := printToFile(y)
	if err != nil {
		return "", fmt.Errorf("diff print rhs error: %v", err)
	}
	defer os.Remove(fy)

	b, _ := ioutil.ReadFile(fx)
	fmt.Printf("fx: %s\n", b)

	data, err := exec.Command("diff", "-U100", "-u", fx, fy).CombinedOutput()
	if err != nil && len(data) == 0 {
		// diff exits with a non-zero status when the files don't match.
		return "", fmt.Errorf("diff error: %v", err)
	}
	res := string(data)
	res = strings.Replace(res, fx, "/x", 1)
	res = strings.Replace(res, fy, "/y", 1)
	return res, nil
}

func WriteDebug(buf *bytes.Buffer, e interface{}) {
	p := debugPrinter{
		buf:     buf,
		ptrseen: make(map[interface{}]int),
		ptrdone: make(map[interface{}]bool),
	}
	v := reflect.ValueOf(e)
	p.collectPtrs(v)
	p.printv(v)
}

func Debug(e interface{}) string {
	buf := new(bytes.Buffer)
	WriteDebug(buf, e)
	return buf.String()
}

func Diff(x, y interface{}) string {
	s, err := diffVal(x, y)
	if err != nil {
		return fmt.Sprintf("format.Diff: %v", err)
	}
	return s
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"github.com/thoj/go-ircevent"
	"io"
	"io/ioutil"
	"os"
	"strings"
	"time"
)

const (
	TIME_FORMAT   = "Jan 2 2006 15:04:05"
)

type IRCConfig struct {
	Nick        string
	Username    string
	Server      string
	Channels    []string
	LogFileDir  string
	LogFiles    map[string]*os.File
}

func main() {
	config, err := getConfig()
	if err != nil {
		fmt.Println(err)
		return
	}

	connection := irc.IRC(config.Nick, config.Username)
	connection.UseTLS = true

	config = setupLogFiles(config)
	defer closeLogFiles(config)
	addCallbacks(connection, config)

	//Connect to the server
	err = connection.Connect(config.Server)
	if err != nil {
		fmt.Println("Failed to connect.")
		return
	}

	// Connect and wait
	connection.Loop()
}

func addCallbacks(connection *irc.Connection, config IRCConfig) {

	// Join the channels
	connection.AddCallback("001", func(e *irc.Event) {
		for _, channel := range config.Channels {
			logMessage(config.LogFiles[channel], "Joining %s", channel)
			connection.Join(channel)
		}
	})

	connection.AddCallback("JOIN", func(e *irc.Event) {
		channel := e.Message()
		nick := e.Nick
		var message string
		if nick == config.Nick {
			message = fmt.Sprintf("Hello, I am %s - yet another logbot written in Go.", nick)
		} else {
			message = fmt.Sprintf("Hello %s, Welcome to %s!", nick, channel)
		}
		connection.Privmsg(channel, message)
		logMessage(config.LogFiles[channel], "%s entered %s", nick, channel)
	})

	connection.AddCallback("PRIVMSG", func(e *irc.Event) {
		channel := e.Arguments[0]
		nick := e.Nick
		switch channel {
		case config.Nick:
			connection.Privmsg(nick, "Sorry, I don't accept direct messages!")
		default:
			logMessage(config.LogFiles[channel], "%s: %s", nick, e.Message())
		}
	})

	connection.AddCallback("CTCP_ACTION", func(e *irc.Event) {
		channel := e.Arguments[0]
		nick := e.Nick
		logMessage(config.LogFiles[channel], "***%s %s", nick, e.Message())
	})

	connection.AddCallback("PART", func(e *irc.Event) {
		channel := e.Arguments[0]
		nick := e.Nick
		logMessage(config.LogFiles[channel], "%s left %s", nick, channel)
	})

	connection.AddCallback("QUIT", func(e *irc.Event) {
		channel := e.Arguments[0]
		nick := e.Nick
		logMessage(config.LogFiles[channel], "%s closed IRC", nick)
	})

}

func getConfig() (config IRCConfig, err error) {
	data, err := ioutil.ReadFile("goyal-config.json")
	if err != nil {
		return IRCConfig{}, fmt.Errorf("Could not read config file.\n")
	}

	decoder := json.NewDecoder(strings.NewReader(string(data)))
	if err := decoder.Decode(&config); err == io.EOF {
		// Return config ...
	} else if err != nil {
		return IRCConfig{}, fmt.Errorf("Could not read config file.\n")
	}

	return config, nil
}

func setupLogFiles(config IRCConfig) (IRCConfig) {
	config.LogFiles = make(map[string]*os.File)
	for _, channel := range config.Channels {
		/// FIXME: fix path manipulation
		logFileName := config.LogFileDir + "/" + channel
		logFile, err := os.OpenFile(logFileName, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0600)
		if err != nil {
			fmt.Println(err)
			fmt.Println("Log file opening failed.")
		}
		config.LogFiles[channel] = logFile
	}
	return config
}

func closeLogFiles(config IRCConfig) {
	for _, logFile := range config.LogFiles {
		logFile.Close()
	}
}

func logMessage(logFile *os.File, format string, args ...interface{}) {
	if logFile == nil {
		panic(fmt.Sprintf("Writing is failing %+v\n", err))
	}
	now := time.Now().UTC().Format(TIME_FORMAT)
	message := fmt.Sprintf(fmt.Sprintf("<%s> %s\n", now, format), args...)
	_, err := logFile.WriteString(message)
	if err != nil {
		panic(fmt.Sprintf("Writing is failing %+v\n", err))
	}
}
<commit_msg>Fix broken error message.<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"github.com/thoj/go-ircevent"
	"io"
	"io/ioutil"
	"os"
	"strings"
	"time"
)

const (
	TIME_FORMAT   = "Jan 2 2006 15:04:05"
)

type IRCConfig struct {
	Nick        string
	Username    string
	Server      string
	Channels    []string
	LogFileDir  string
	LogFiles    map[string]*os.File
}

func main() {
	config, err := getConfig()
	if err != nil {
		fmt.Println(err)
		return
	}

	connection := irc.IRC(config.Nick, config.Username)
	connection.UseTLS = true

	config = setupLogFiles(config)
	defer closeLogFiles(config)
	addCallbacks(connection, config)

	//Connect to the server
	err = connection.Connect(config.Server)
	if err != nil {
		fmt.Println("Failed to connect.")
		return
	}

	// Connect and wait
	connection.Loop()
}

func addCallbacks(connection *irc.Connection, config IRCConfig) {

	// Join the channels
	connection.AddCallback("001", func(e *irc.Event) {
		for _, channel := range config.Channels {
			logMessage(config.LogFiles[channel], "Joining %s", channel)
			connection.Join(channel)
		}
	})

	connection.AddCallback("JOIN", func(e *irc.Event) {
		channel := e.Message()
		nick := e.Nick
		var message string
		if nick == config.Nick {
			message = fmt.Sprintf("Hello, I am %s - yet another logbot written in Go.", nick)
		} else {
			message = fmt.Sprintf("Hello %s, Welcome to %s!", nick, channel)
		}
		connection.Privmsg(channel, message)
		logMessage(config.LogFiles[channel], "%s entered %s", nick, channel)
	})

	connection.AddCallback("PRIVMSG", func(e *irc.Event) {
		channel := e.Arguments[0]
		nick := e.Nick
		switch channel {
		case config.Nick:
			connection.Privmsg(nick, "Sorry, I don't accept direct messages!")
		default:
			logMessage(config.LogFiles[channel], "%s: %s", nick, e.Message())
		}
	})

	connection.AddCallback("CTCP_ACTION", func(e *irc.Event) {
		channel := e.Arguments[0]
		nick := e.Nick
		logMessage(config.LogFiles[channel], "***%s %s", nick, e.Message())
	})

	connection.AddCallback("PART", func(e *irc.Event) {
		channel := e.Arguments[0]
		nick := e.Nick
		logMessage(config.LogFiles[channel], "%s left %s", nick, channel)
	})

	connection.AddCallback("QUIT", func(e *irc.Event) {
		channel := e.Arguments[0]
		nick := e.Nick
		logMessage(config.LogFiles[channel], "%s closed IRC", nick)
	})

}

func getConfig() (config IRCConfig, err error) {
	data, err := ioutil.ReadFile("goyal-config.json")
	if err != nil {
		return IRCConfig{}, fmt.Errorf("Could not read config file.\n")
	}

	decoder := json.NewDecoder(strings.NewReader(string(data)))
	if err := decoder.Decode(&config); err == io.EOF {
		// Return config ...
	} else if err != nil {
		return IRCConfig{}, fmt.Errorf("Could not read config file.\n")
	}

	return config, nil
}

func setupLogFiles(config IRCConfig) (IRCConfig) {
	config.LogFiles = make(map[string]*os.File)
	for _, channel := range config.Channels {
		/// FIXME: fix path manipulation
		logFileName := config.LogFileDir + "/" + channel
		logFile, err := os.OpenFile(logFileName, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0600)
		if err != nil {
			fmt.Println(err)
			fmt.Println("Log file opening failed.")
		}
		config.LogFiles[channel] = logFile
	}
	return config
}

func closeLogFiles(config IRCConfig) {
	for _, logFile := range config.LogFiles {
		logFile.Close()
	}
}

func logMessage(logFile *os.File, format string, args ...interface{}) {
	if logFile == nil {
		panic(fmt.Sprintf("No such log file provided."))
	}
	now := time.Now().UTC().Format(TIME_FORMAT)
	message := fmt.Sprintf(fmt.Sprintf("<%s> %s\n", now, format), args...)
	_, err := logFile.WriteString(message)
	if err != nil {
		panic(fmt.Sprintf("Writing is failing %+v\n", err))
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"github.com/c-14/grue/config"
	"github.com/mmcdole/gofeed"
	"math"
	"time"
)

type FeedParser struct {
	parser   *gofeed.Parser
	messages chan *Email
	sem      chan int
	finished chan int
}

type RSSFeed struct {
	config      config.AccountConfig
	LastFetched int64               `json:",omitempty"`
	LastQueried int64               `json:",omitempty"`
	NextQuery   int64               `json:",omitempty"`
	Tries       int                 `json:",omitempty"`
	GUIDList    map[string]struct{} `json:",omitempty"`
}

type DateType int

const (
	NoDate DateType = iota
	DateNewer
	DateOlder
)

func hasNewerDate(item *gofeed.Item, lastFetched int64) (time.Time, DateType) {
	if item.PublishedParsed != nil {
		if item.PublishedParsed.Unix() > lastFetched {
			return *item.PublishedParsed, DateNewer
		} else {
			return *item.PublishedParsed, DateOlder
		}
	} else if item.UpdatedParsed != nil {
		if item.UpdatedParsed.Unix() > lastFetched {
			return *item.UpdatedParsed, DateNewer
		} else {
			return *item.UpdatedParsed, DateOlder
		}
	} else if date, exists := item.Extensions["dc"]["date"]; exists {
		dateParsed, err := time.Parse(time.RFC3339, date[0].Value)
		if err != nil {
			fmt.Printf("Can't parse (%v) as dc:date for (%v)\n", date, item.Link)
			return time.Now(), NoDate
		}
		if dateParsed.Unix() > lastFetched {
			return dateParsed, DateNewer
		} else {
			return dateParsed, DateOlder
		}
	}
	return time.Now(), NoDate
}

func fetchFeed(fp FeedParser, feedName string, account *RSSFeed, config *config.GrueConfig) {
	// if account.UserAgent != nil {
	// 	feed.SetUserAgent(*account.UserAgent)
	// }
	now := time.Now()
	if account.NextQuery > now.Unix() {
		<-fp.sem
		fp.finished <- 1
		return
	}
	feed, err := fp.parser.ParseURL(account.config.URI)
	account.LastQueried = now.Unix()
	if err != nil {
		if account.Tries > 0 {
			account.NextQuery = now.Add(time.Duration(math.Exp2(float64(account.Tries+4))) * time.Minute).Unix()
		}
		account.Tries++
		if account.Tries > 1 {
			fmt.Printf("Caught error (#%d) when parsing %s: %s\n", account.Tries, account.config.URI, err)
		}
		<-fp.sem
		fp.finished <- 1
		return
	}
	account.NextQuery = 0
	account.Tries = 0
	guids := account.GUIDList
	if float64(len(guids)) > 1.2*float64(len(feed.Items)) {
		account.GUIDList = make(map[string]struct{})
	}
	for _, item := range feed.Items {
		date, newer := hasNewerDate(item, account.LastFetched)
		if newer == DateNewer {
			e := createEmail(feedName, feed.Title, item, date, account.config, config)
			err = e.Send(fp.messages)
			if err != nil {
				break
			}
		} else if newer == NoDate {
			_, exists := guids[item.GUID]
			if !exists {
				e := createEmail(feedName, feed.Title, item, date, account.config, config)
				err = e.Send(fp.messages)
			}
			if err == nil {
				account.GUIDList[item.GUID] = struct{}{}
			} else {
				break
			}
		}
	}
	if err == nil {
		account.LastFetched = account.LastQueried
	}

	<-fp.sem
	fp.finished <- 1
}

func fetchFeeds(ret chan error, conf *config.GrueConfig, init bool) {
	var ch chan *Email = make(chan *Email)
	var dial chan int = make(chan int)
	defer close(ret)
	hist, err := ReadHistory()
	if err != nil {
		ret <- err
		return
	}
	if !init {
		s, err := setupDialer(conf)
		if err != nil {
			ret <- err
			return
		}
		go startDialing(s, ch, dial, ret)
	} else {
		go func() {
			for m := range ch {
				m.ret <- nil
			}
		}()
		close(dial)
	}

	fp := FeedParser{parser: gofeed.NewParser(), messages: ch, sem: make(chan int, 10), finished: make(chan int)}
	go func() {
		for name, accountConfig := range conf.Accounts {
			fp.sem <- 1
			account, exist := hist.Feeds[name]
			if !exist {
				account = new(RSSFeed)
				account.GUIDList = make(map[string]struct{})
				hist.Feeds[name] = account
			} else if len(account.GUIDList) == 0 {
				account.GUIDList = make(map[string]struct{})
			}
			account.config = accountConfig
			go fetchFeed(fp, name, account, conf)
		}
	}()
	for range conf.Accounts {
		<-fp.finished
	}
	ret <- hist.Write()
	close(ch)
	<-dial
}
<commit_msg>Prefer GUIDs to Timestamps.<commit_after>package main

import (
	"fmt"
	"github.com/c-14/grue/config"
	"github.com/mmcdole/gofeed"
	"math"
	"time"
)

type FeedParser struct {
	parser   *gofeed.Parser
	messages chan *Email
	sem      chan int
	finished chan int
}

type RSSFeed struct {
	config      config.AccountConfig
	LastFetched int64               `json:",omitempty"`
	LastQueried int64               `json:",omitempty"`
	NextQuery   int64               `json:",omitempty"`
	Tries       int                 `json:",omitempty"`
	GUIDList    map[string]struct{} `json:",omitempty"`
}

type DateType int

const (
	NoDate DateType = iota
	DateNewer
	DateOlder
)

func hasNewerDate(item *gofeed.Item, lastFetched int64) (time.Time, DateType) {
	if item.PublishedParsed != nil {
		if item.PublishedParsed.Unix() > lastFetched {
			return *item.PublishedParsed, DateNewer
		} else {
			return *item.PublishedParsed, DateOlder
		}
	} else if item.UpdatedParsed != nil {
		if item.UpdatedParsed.Unix() > lastFetched {
			return *item.UpdatedParsed, DateNewer
		} else {
			return *item.UpdatedParsed, DateOlder
		}
	} else if date, exists := item.Extensions["dc"]["date"]; exists {
		dateParsed, err := time.Parse(time.RFC3339, date[0].Value)
		if err != nil {
			fmt.Printf("Can't parse (%v) as dc:date for (%v)\n", date, item.Link)
			return time.Now(), NoDate
		}
		if dateParsed.Unix() > lastFetched {
			return dateParsed, DateNewer
		} else {
			return dateParsed, DateOlder
		}
	}
	return time.Now(), NoDate
}

func fetchFeed(fp FeedParser, feedName string, account *RSSFeed, config *config.GrueConfig) {
	// if account.UserAgent != nil {
	// 	feed.SetUserAgent(*account.UserAgent)
	// }
	now := time.Now()
	if account.NextQuery > now.Unix() {
		<-fp.sem
		fp.finished <- 1
		return
	}
	feed, err := fp.parser.ParseURL(account.config.URI)
	account.LastQueried = now.Unix()
	if err != nil {
		if account.Tries > 0 {
			account.NextQuery = now.Add(time.Duration(math.Exp2(float64(account.Tries+4))) * time.Minute).Unix()
		}
		account.Tries++
		if account.Tries > 1 {
			fmt.Printf("Caught error (#%d) when parsing %s: %s\n", account.Tries, account.config.URI, err)
		}
		<-fp.sem
		fp.finished <- 1
		return
	}
	account.NextQuery = 0
	account.Tries = 0
	guids := account.GUIDList
	if float64(len(guids)) > 1.2*float64(len(feed.Items)) {
		account.GUIDList = make(map[string]struct{})
	}
	for _, item := range feed.Items {
		_, exists := guids[item.GUID]
		date, newer := hasNewerDate(item, account.LastFetched)
		if !exists || (item.GUID == "" && newer == DateNewer) {
			e := createEmail(feedName, feed.Title, item, date, account.config, config)
			err = e.Send(fp.messages)
		}
		if err == nil {
			account.GUIDList[item.GUID] = struct{}{}
		} else {
			break
		}
	}
	if err == nil {
		account.LastFetched = time.Now().Unix()
	}

	<-fp.sem
	fp.finished <- 1
}

func fetchFeeds(ret chan error, conf *config.GrueConfig, init bool) {
	var ch chan *Email = make(chan *Email)
	var dial chan int = make(chan int)
	defer close(ret)
	hist, err := ReadHistory()
	if err != nil {
		ret <- err
		return
	}
	if !init {
		s, err := setupDialer(conf)
		if err != nil {
			ret <- err
			return
		}
		go startDialing(s, ch, dial, ret)
	} else {
		go func() {
			for m := range ch {
				m.ret <- nil
			}
		}()
		close(dial)
	}

	fp := FeedParser{parser: gofeed.NewParser(), messages: ch, sem: make(chan int, 10), finished: make(chan int)}
	go func() {
		for name, accountConfig := range conf.Accounts {
			fp.sem <- 1
			account, exist := hist.Feeds[name]
			if !exist {
				account = new(RSSFeed)
				account.GUIDList = make(map[string]struct{})
				hist.Feeds[name] = account
			} else if len(account.GUIDList) == 0 {
				account.GUIDList = make(map[string]struct{})
			}
			account.config = accountConfig
			go fetchFeed(fp, name, account, conf)
		}
	}()
	for range conf.Accounts {
		<-fp.finished
	}
	ret <- hist.Write()
	close(ch)
	<-dial
}
<|endoftext|>


<commit_before>package module

//
// Xunlei 迅雷离线下载
//
import (
	"encoding/json"
	"errors"
	"io/ioutil"
	"lib"
	"strconv"
	"strings"
)

// Xunlei 迅雷离线下载
type Xunlei struct {
	YunBase
}

// LoadData 加载数据
// @param data {account,id}
// @return 返回 {account:account,id:id,list:[{id,title,size,url}]}
func (xl *Xunlei) LoadData(sender *Sender, data interface{}) {
	data2, ok := data.(map[string]interface{})
	if !ok {
		sender.Err = "error data"
		return
	}
	accountName, _ := data2["account"].(string)
	id, _ := data2["id"].(string)
	cc := xl.getCookieContainer(accountName)
	if cc == nil {
		sender.Err = "No account name: " + accountName
		return
	}
	if id == "" {
		// 获取主页面
		resultList, err := xl.getMainList(cc)
		if err != nil {
			sender.Err = err.Error() + " | xunlei:39"
			return
		}
		sender.Data = map[string]interface{}{"account": accountName, "id": id, "list": resultList}
	} else {
		// 获取bt
		resultList, err := xl.getBt(cc, id)
		if err != nil {
			sender.Err = err.Error() + " | xunlei:47"
			return
		}
		sender.Data = map[string]interface{}{"account": accountName, "id": id, "list": resultList}
	}
}

// Download 下载
// @param data {account:xxx,list:[{title,url},xxx]}
// @return 如果成功返回ok
func (xl *Xunlei) Download(sender *Sender, data interface{}) {
	data2, ok := data.(map[string]interface{})
	if !ok {
		sender.Err = "error data"
		return
	}
	accountName, _ := data2["account"].(string)
	list, ok := data2["list"].([]interface{})
	if !ok {
		sender.Err = "convert list fail"
		return
	}
	cc := xl.getCookieContainer(accountName)
	if cc == nil {
		sender.Err = "No accountName name: " + accountName
		return
	}
	// 要加上头
	header := "Cookie: gdriveid=" + cc.GetValueByName("gdriveid")
	success := true
	for i := 0; i < len(list); i++ {
		obj, ok := list[i].(map[string]interface{})
		if !ok {
			sender.Err = "convert list item fail"
			return
		}
		title, _ := obj["title"].(string)
		urlStr, _ := obj["url"].(string)
		_, err := C.Aria2.AddDownload(urlStr, title, header)
		if err != nil {
			success = false
			sender.Err = err.Error() + " | xunlei:88"
		}
	}
	if success {
		// 添加成功
		sender.Data = "ok"
	}
}

// getMainList 获取主页面列表信息
// @return 返回 [{id,title,size,url,isdir}]
func (xl *Xunlei) getMainList(cc *lib.CookieContainer) (resultList []interface{}, err error) {
	callback := "jsonp1465702254496"
	urlStr := "http://dynamic.cloud.vip.xunlei.com/interface/showtask_unfresh?callback=" + callback + "&type_id=4&page=1&tasknum=300&p=1&interfrom=task"
	res, err := lib.GetHTML(urlStr, cc)
	if err != nil {
		return
	}
	defer res.Body.Close()
	b, err := ioutil.ReadAll(res.Body)
	if err != nil {
		return
	}
	str := string(b)
	// 获取jsonp中的内容
	str = str[len(callback)+1 : len(str)-1]
	result := map[string]interface{}{}
	err = json.Unmarshal([]byte(str), &result)
	if err != nil {
		return
	}
	// 检测返回代码
	rtcode, ok := result["rtcode"].(float64)
	if !ok {
		err = errors.New("bad response data")
		return
	}
	if rtcode != 0 {
		err = errors.New("wrong rtcode " + strconv.Itoa(int(rtcode)))
		return
	}
	info, ok := result["info"].(map[string]interface{})
	if !ok {
		err = errors.New("bad response data['info']")
		return
	}
	tasks, ok := info["tasks"].([]interface{})
	if !ok {
		err = errors.New("bad response data['info']['tasks']")
		return
	}
	resultList = []interface{}{}
	for i := 0; i < len(tasks); i++ {
		task, ok := tasks[i].(map[string]interface{})
		if !ok {
			continue
		}
		id := task["id"]
		title := task["taskname"]
		isdir := false
		urlStr, _ := task["lixian_url"].(string)
		if strings.HasPrefix(urlStr, "bt:") {
			// bt文件夹
			urlStr = ""
			isdir = true
		}
		status, _ := task["download_status"].(string)
		if status != "2" && !isdir {
			// 没有下载完成且非文件夹的，不计算
			continue
		}
		size, _ := task["file_size"].(string)
		size = lib.GetReadableSize(size) + "B"
		resultList = append(resultList, map[string]interface{}{"id": id, "title": title, "size": size, "url": urlStr, "isdir": isdir})
	}
	return
}

// getBt 获取bt列表
// @return 返回 [{id,title,size,url,isdir}]，url为空则是bt文件夹
func (xl *Xunlei) getBt(cc *lib.CookieContainer, taskID string) (resultList []interface{}, err error) {
	// 请求页面数据
	userid := cc.GetValueByName("userid")
	callback := "fill_bt_list"
	urlStr := "http://dynamic.cloud.vip.xunlei.com/interface/fill_bt_list?tid=" + taskID + "&g_net=1&p=1&uid=" + userid + "&callback=" + callback
	res, err := lib.GetHTML(urlStr, cc)
	if err != nil {
		return
	}
	defer res.Body.Close()
	b, err := ioutil.ReadAll(res.Body)
	if err != nil {
		return
	}
	str := string(b)
	// 获取jsonp中的内容
	str = str[len(callback)+1 : len(str)-1]
	result := map[string]interface{}{}
	err = json.Unmarshal([]byte(str), &result)
	if err != nil {
		return
	}
	result, ok := result["Result"].(map[string]interface{})
	if !ok {
		err = errors.New("bad response data['Result']")
		return
	}
	list, ok := result["Record"].([]interface{})
	if !ok {
		err = errors.New("bad response data['Result']['Record']")
		return
	}
	resultList = []interface{}{}
	for i := 0; i < len(list); i++ {
		obj, ok := list[i].(map[string]interface{})
		if !ok {
			err = errors.New("bad response data['Result']['Record'][...]")
			return
		}
		// 查看下载状态是否已完成
		status, _ := obj["download_status"].(string)
		if status != "2" {
			continue
		}
		id1, _ := obj["id"].(int)
		id := strconv.Itoa(id1)
		taskid, _ := obj["taskid"].(string)
		id = taskid + id
		title := obj["title"]
		size, _ := obj["filesize"].(string)
		size = lib.GetReadableSize(size) + "B"
		urlStr := obj["downurl"]
		isdir := false
		if urlStr == "" {
			isdir = true
		}
		resultList = append(resultList, map[string]interface{}{"id": id, "title": title, "size": size, "url": urlStr, "isdir": isdir})
	}
	return
}

// NewXunlei 新建
func NewXunlei() (xunlei *Xunlei) {
	xunlei = &Xunlei{YunBase{accountType: "xunlei", accountList: []lib.Account{}}}
	xunlei.initAccountList()
	return
}
<commit_msg>modify xunlei callback<commit_after>package module

//
// Xunlei 迅雷离线下载
//
import (
	"encoding/json"
	"errors"
	"io/ioutil"
	"lib"
	"strconv"
	"strings"
	"time"
)

// Xunlei 迅雷离线下载
type Xunlei struct {
	YunBase
}

// LoadData 加载数据
// @param data {account,id}
// @return 返回 {account:account,id:id,list:[{id,title,size,url}]}
func (xl *Xunlei) LoadData(sender *Sender, data interface{}) {
	data2, ok := data.(map[string]interface{})
	if !ok {
		sender.Err = "error data"
		return
	}
	accountName, _ := data2["account"].(string)
	id, _ := data2["id"].(string)
	cc := xl.getCookieContainer(accountName)
	if cc == nil {
		sender.Err = "No account name: " + accountName
		return
	}
	if id == "" {
		// 获取主页面
		resultList, err := xl.getMainList(cc)
		if err != nil {
			sender.Err = err.Error() + " | xunlei:39"
			return
		}
		sender.Data = map[string]interface{}{"account": accountName, "id": id, "list": resultList}
	} else {
		// 获取bt
		resultList, err := xl.getBt(cc, id)
		if err != nil {
			sender.Err = err.Error() + " | xunlei:47"
			return
		}
		sender.Data = map[string]interface{}{"account": accountName, "id": id, "list": resultList}
	}
}

// Download 下载
// @param data {account:xxx,list:[{title,url},xxx]}
// @return 如果成功返回ok
func (xl *Xunlei) Download(sender *Sender, data interface{}) {
	data2, ok := data.(map[string]interface{})
	if !ok {
		sender.Err = "error data"
		return
	}
	accountName, _ := data2["account"].(string)
	list, ok := data2["list"].([]interface{})
	if !ok {
		sender.Err = "convert list fail"
		return
	}
	cc := xl.getCookieContainer(accountName)
	if cc == nil {
		sender.Err = "No accountName name: " + accountName
		return
	}
	// 要加上头
	header := "Cookie: gdriveid=" + cc.GetValueByName("gdriveid")
	success := true
	for i := 0; i < len(list); i++ {
		obj, ok := list[i].(map[string]interface{})
		if !ok {
			sender.Err = "convert list item fail"
			return
		}
		title, _ := obj["title"].(string)
		urlStr, _ := obj["url"].(string)
		_, err := C.Aria2.AddDownload(urlStr, title, header)
		if err != nil {
			success = false
			sender.Err = err.Error() + " | xunlei:88"
		}
	}
	if success {
		// 添加成功
		sender.Data = "ok"
	}
}

// getMainList 获取主页面列表信息
// @return 返回 [{id,title,size,url,isdir}]
func (xl *Xunlei) getMainList(cc *lib.CookieContainer) (resultList []interface{}, err error) {
	// 需要随机
	ran := strconv.FormatInt(time.Now().UnixNano(), 10)
	callback := "jsonp" + ran
	urlStr := "http://dynamic.cloud.vip.xunlei.com/interface/showtask_unfresh?callback=" + callback + "&type_id=4&page=1&tasknum=300&p=1&interfrom=task"
	res, err := lib.GetHTML(urlStr, cc)
	if err != nil {
		return
	}
	defer res.Body.Close()
	b, err := ioutil.ReadAll(res.Body)
	if err != nil {
		return
	}
	str := string(b)
	// 获取jsonp中的内容
	str = str[len(callback)+1 : len(str)-1]
	result := map[string]interface{}{}
	err = json.Unmarshal([]byte(str), &result)
	if err != nil {
		return
	}
	// 检测返回代码
	rtcode, ok := result["rtcode"].(float64)
	if !ok {
		err = errors.New("bad response data")
		return
	}
	if rtcode != 0 {
		err = errors.New("wrong rtcode " + strconv.Itoa(int(rtcode)))
		return
	}
	info, ok := result["info"].(map[string]interface{})
	if !ok {
		err = errors.New("bad response data['info']")
		return
	}
	tasks, ok := info["tasks"].([]interface{})
	if !ok {
		err = errors.New("bad response data['info']['tasks']")
		return
	}
	resultList = []interface{}{}
	for i := 0; i < len(tasks); i++ {
		task, ok := tasks[i].(map[string]interface{})
		if !ok {
			continue
		}
		id := task["id"]
		title := task["taskname"]
		isdir := false
		urlStr, _ := task["lixian_url"].(string)
		if strings.HasPrefix(urlStr, "bt:") {
			// bt文件夹
			urlStr = ""
			isdir = true
		}
		status, _ := task["download_status"].(string)
		if status != "2" && !isdir {
			// 没有下载完成且非文件夹的，不计算
			continue
		}
		size, _ := task["file_size"].(string)
		size = lib.GetReadableSize(size) + "B"
		resultList = append(resultList, map[string]interface{}{"id": id, "title": title, "size": size, "url": urlStr, "isdir": isdir})
	}
	return
}

// getBt 获取bt列表
// @return 返回 [{id,title,size,url,isdir}]，url为空则是bt文件夹
func (xl *Xunlei) getBt(cc *lib.CookieContainer, taskID string) (resultList []interface{}, err error) {
	// 请求页面数据
	userid := cc.GetValueByName("userid")
	callback := "fill_bt_list"
	urlStr := "http://dynamic.cloud.vip.xunlei.com/interface/fill_bt_list?tid=" + taskID + "&g_net=1&p=1&uid=" + userid + "&callback=" + callback
	res, err := lib.GetHTML(urlStr, cc)
	if err != nil {
		return
	}
	defer res.Body.Close()
	b, err := ioutil.ReadAll(res.Body)
	if err != nil {
		return
	}
	str := string(b)
	// 获取jsonp中的内容
	str = str[len(callback)+1 : len(str)-1]
	result := map[string]interface{}{}
	err = json.Unmarshal([]byte(str), &result)
	if err != nil {
		return
	}
	result, ok := result["Result"].(map[string]interface{})
	if !ok {
		err = errors.New("bad response data['Result']")
		return
	}
	list, ok := result["Record"].([]interface{})
	if !ok {
		err = errors.New("bad response data['Result']['Record']")
		return
	}
	resultList = []interface{}{}
	for i := 0; i < len(list); i++ {
		obj, ok := list[i].(map[string]interface{})
		if !ok {
			err = errors.New("bad response data['Result']['Record'][...]")
			return
		}
		// 查看下载状态是否已完成
		status, _ := obj["download_status"].(string)
		if status != "2" {
			continue
		}
		id1, _ := obj["id"].(int)
		id := strconv.Itoa(id1)
		taskid, _ := obj["taskid"].(string)
		id = taskid + id
		title := obj["title"]
		size, _ := obj["filesize"].(string)
		size = lib.GetReadableSize(size) + "B"
		urlStr := obj["downurl"]
		isdir := false
		if urlStr == "" {
			isdir = true
		}
		resultList = append(resultList, map[string]interface{}{"id": id, "title": title, "size": size, "url": urlStr, "isdir": isdir})
	}
	return
}

// NewXunlei 新建
func NewXunlei() (xunlei *Xunlei) {
	xunlei = &Xunlei{YunBase{accountType: "xunlei", accountList: []lib.Account{}}}
	xunlei.initAccountList()
	return
}
<|endoftext|>


<commit_before>package executor

import (
	"fmt"
	"os"
	"os/user"
	"path/filepath"
	"strconv"
	"strings"
	"syscall"
	"time"

	"github.com/hashicorp/go-multierror"
	"github.com/mitchellh/go-ps"
	"github.com/opencontainers/runc/libcontainer/cgroups"
	cgroupFs "github.com/opencontainers/runc/libcontainer/cgroups/fs"
	cgroupConfig "github.com/opencontainers/runc/libcontainer/configs"
	"github.com/opencontainers/runc/libcontainer/system"

	"github.com/hashicorp/nomad/client/allocdir"
	"github.com/hashicorp/nomad/client/stats"
	cstructs "github.com/hashicorp/nomad/client/structs"
	"github.com/hashicorp/nomad/nomad/structs"
)

var (
	// A mapping of directories on the host OS to attempt to embed inside each
	// task's chroot.
	chrootEnv = map[string]string{
		"/bin":            "/bin",
		"/etc":            "/etc",
		"/lib":            "/lib",
		"/lib32":          "/lib32",
		"/lib64":          "/lib64",
		"/run/resolvconf": "/run/resolvconf",
		"/sbin":           "/sbin",
		"/usr":            "/usr",
	}

	// clockTicks is the clocks per second of the machine
	clockTicks = uint64(system.GetClockTicks())

	// The statistics the executor exposes when using cgroups
	ExecutorCgroupMeasuredMemStats = []string{"RSS", "Cache", "Swap", "Max Usage", "Kernel Usage", "Kernel Max Usage"}
	ExecutorCgroupMeasuredCpuStats = []string{"System Mode", "User Mode", "Throttled Periods", "Throttled Time", "Percent"}
)

// configureIsolation configures chroot and creates cgroups
func (e *UniversalExecutor) configureIsolation() error {
	if e.command.FSIsolation {
		if err := e.configureChroot(); err != nil {
			return err
		}
	}

	if e.command.ResourceLimits {
		if err := e.configureCgroups(e.ctx.Task.Resources); err != nil {
			return fmt.Errorf("error creating cgroups: %v", err)
		}
	}
	return nil
}

// applyLimits puts a process in a pre-configured cgroup
func (e *UniversalExecutor) applyLimits(pid int) error {
	if !e.command.ResourceLimits {
		return nil
	}

	// Entering the process in the cgroup
	manager := getCgroupManager(e.groups, nil)
	if err := manager.Apply(pid); err != nil {
		e.logger.Printf("[ERR] executor: error applying pid to cgroup: %v", err)
		if er := e.removeChrootMounts(); er != nil {
			e.logger.Printf("[ERR] executor: error removing chroot: %v", er)
		}
		return err
	}
	e.cgPaths = manager.GetPaths()
	cgConfig := cgroupConfig.Config{Cgroups: e.groups}
	if err := manager.Set(&cgConfig); err != nil {
		e.logger.Printf("[ERR] executor: error setting cgroup config: %v", err)
		if er := DestroyCgroup(e.groups, e.cgPaths, os.Getpid()); er != nil {
			e.logger.Printf("[ERR] executor: error destroying cgroup: %v", er)
		}
		if er := e.removeChrootMounts(); er != nil {
			e.logger.Printf("[ERR] executor: error removing chroot: %v", er)
		}
		return err
	}
	return nil
}

// configureCgroups converts a Nomad Resources specification into the equivalent
// cgroup configuration. It returns an error if the resources are invalid.
func (e *UniversalExecutor) configureCgroups(resources *structs.Resources) error {
	e.groups = &cgroupConfig.Cgroup{}
	e.groups.Resources = &cgroupConfig.Resources{}
	cgroupName := structs.GenerateUUID()
	e.groups.Path = filepath.Join("/nomad", cgroupName)

	// TODO: verify this is needed for things like network access
	e.groups.Resources.AllowAllDevices = true

	if resources.MemoryMB > 0 {
		// Total amount of memory allowed to consume
		e.groups.Resources.Memory = int64(resources.MemoryMB * 1024 * 1024)
		// Disable swap to avoid issues on the machine
		e.groups.Resources.MemorySwap = int64(-1)
	}

	if resources.CPU < 2 {
		return fmt.Errorf("resources.CPU must be equal to or greater than 2: %v", resources.CPU)
	}

	// Set the relative CPU shares for this cgroup.
	e.groups.Resources.CpuShares = int64(resources.CPU)

	if resources.IOPS != 0 {
		// Validate it is in an acceptable range.
		if resources.IOPS < 10 || resources.IOPS > 1000 {
			return fmt.Errorf("resources.IOPS must be between 10 and 1000: %d", resources.IOPS)
		}

		e.groups.Resources.BlkioWeight = uint16(resources.IOPS)
	}

	return nil
}

// Stats reports the resource utilization of the cgroup. If there is no resource
// isolation we aggregate the resource utilization of all the pids launched by
// the executor.
func (e *UniversalExecutor) Stats() (*cstructs.TaskResourceUsage, error) {
	if !e.command.ResourceLimits {
		pidStats, err := e.pidStats()
		if err != nil {
			return nil, err
		}
		return e.aggregatedResourceUsage(pidStats), nil
	}
	ts := time.Now()
	manager := getCgroupManager(e.groups, e.cgPaths)
	stats, err := manager.GetStats()
	if err != nil {
		return nil, err
	}

	// Memory Related Stats
	swap := stats.MemoryStats.SwapUsage
	maxUsage := stats.MemoryStats.Usage.MaxUsage
	rss := stats.MemoryStats.Stats["rss"]
	cache := stats.MemoryStats.Stats["cache"]
	ms := &cstructs.MemoryStats{
		RSS:            rss,
		Cache:          cache,
		Swap:           swap.Usage,
		MaxUsage:       maxUsage,
		KernelUsage:    stats.MemoryStats.KernelUsage.Usage,
		KernelMaxUsage: stats.MemoryStats.KernelUsage.MaxUsage,
		Measured:       ExecutorCgroupMeasuredMemStats,
	}

	// CPU Related Stats
	totalProcessCPUUsage := float64(stats.CpuStats.CpuUsage.TotalUsage)
	userModeTime := float64(stats.CpuStats.CpuUsage.UsageInUsermode)
	kernelModeTime := float64(stats.CpuStats.CpuUsage.UsageInKernelmode)

	totalPercent := e.totalCpuStats.Percent(totalProcessCPUUsage)
	cs := &cstructs.CpuStats{
		SystemMode:       e.systemCpuStats.Percent(kernelModeTime),
		UserMode:         e.userCpuStats.Percent(userModeTime),
		Percent:          totalPercent,
		ThrottledPeriods: stats.CpuStats.ThrottlingData.ThrottledPeriods,
		ThrottledTime:    stats.CpuStats.ThrottlingData.ThrottledTime,
		TotalTicks:       e.systemCpuStats.TicksConsumed(totalPercent),
		Measured:         ExecutorCgroupMeasuredCpuStats,
	}
	taskResUsage := cstructs.TaskResourceUsage{
		ResourceUsage: &cstructs.ResourceUsage{
			MemoryStats: ms,
			CpuStats:    cs,
		},
		Timestamp: ts.UTC().UnixNano(),
	}
	if pidStats, err := e.pidStats(); err == nil {
		taskResUsage.Pids = pidStats
	}
	return &taskResUsage, nil
}

// runAs takes a user id as a string and looks up the user, and sets the command
// to execute as that user.
func (e *UniversalExecutor) runAs(userid string) error {
	u, err := user.Lookup(userid)
	if err != nil {
		return fmt.Errorf("Failed to identify user %v: %v", userid, err)
	}

	// Convert the uid and gid
	uid, err := strconv.ParseUint(u.Uid, 10, 32)
	if err != nil {
		return fmt.Errorf("Unable to convert userid to uint32: %s", err)
	}
	gid, err := strconv.ParseUint(u.Gid, 10, 32)
	if err != nil {
		return fmt.Errorf("Unable to convert groupid to uint32: %s", err)
	}

	// Set the command to run as that user and group.
	if e.cmd.SysProcAttr == nil {
		e.cmd.SysProcAttr = &syscall.SysProcAttr{}
	}
	if e.cmd.SysProcAttr.Credential == nil {
		e.cmd.SysProcAttr.Credential = &syscall.Credential{}
	}
	e.cmd.SysProcAttr.Credential.Uid = uint32(uid)
	e.cmd.SysProcAttr.Credential.Gid = uint32(gid)

	return nil
}

// configureChroot configures a chroot
func (e *UniversalExecutor) configureChroot() error {
	allocDir := e.ctx.AllocDir
	if err := allocDir.MountSharedDir(e.ctx.Task.Name); err != nil {
		return err
	}

	if err := allocDir.Embed(e.ctx.Task.Name, chrootEnv); err != nil {
		return err
	}

	// Set the tasks AllocDir environment variable.
	e.ctx.TaskEnv.
		SetAllocDir(filepath.Join("/", allocdir.SharedAllocName)).
		SetTaskLocalDir(filepath.Join("/", allocdir.TaskLocal)).
		Build()

	if e.cmd.SysProcAttr == nil {
		e.cmd.SysProcAttr = &syscall.SysProcAttr{}
	}
	e.cmd.SysProcAttr.Chroot = e.taskDir
	e.cmd.Dir = "/"

	if err := allocDir.MountSpecialDirs(e.taskDir); err != nil {
		return err
	}

	e.fsIsolationEnforced = true
	return nil
}

// cleanTaskDir is an idempotent operation to clean the task directory and
// should be called when tearing down the task.
func (e *UniversalExecutor) removeChrootMounts() error {
	// Prevent a race between Wait/ForceStop
	e.cgLock.Lock()
	defer e.cgLock.Unlock()
	return e.ctx.AllocDir.UnmountAll()
}

// getAllPids returns the pids of all the processes spun up by the executor. We
// use the libcontainer apis to get the pids when the user is using cgroup
// isolation and we scan the entire process table if the user is not using any
// isolation
func (e *UniversalExecutor) getAllPids() (map[int]*nomadPid, error) {
	if e.command.ResourceLimits {
		manager := getCgroupManager(e.groups, e.cgPaths)
		pids, err := manager.GetAllPids()
		if err != nil {
			return nil, err
		}
		np := make(map[int]*nomadPid, len(pids))
		for _, pid := range pids {
			np[pid] = &nomadPid{
				pid:           pid,
				cpuStatsTotal: stats.NewCpuStats(),
				cpuStatsSys:   stats.NewCpuStats(),
				cpuStatsUser:  stats.NewCpuStats(),
			}
		}
		return np, nil
	}
	allProcesses, err := ps.Processes()
	if err != nil {
		return nil, err
	}
	return e.scanPids(os.Getpid(), allProcesses)
}

// destroyCgroup kills all processes in the cgroup and removes the cgroup
// configuration from the host. This function is idempotent.
func DestroyCgroup(groups *cgroupConfig.Cgroup, cgPaths map[string]string, executorPid int) error {
	mErrs := new(multierror.Error)
	if groups == nil {
		return fmt.Errorf("Can't destroy: cgroup configuration empty")
	}

	// Move the executor into the global cgroup so that the task specific
	// cgroup can be destroyed.
	nilGroup := &cgroupConfig.Cgroup{}
	nilGroup.Path = "/"
	nilGroup.Resources = groups.Resources
	nilManager := getCgroupManager(nilGroup, nil)
	err := nilManager.Apply(executorPid)
	if err != nil && !strings.Contains(err.Error(), "no such process") {
		return fmt.Errorf("failed to remove executor pid %d: %v", executorPid, err)
	}

	// Freeze the Cgroup so that it can not continue to fork/exec.
	manager := getCgroupManager(groups, cgPaths)
	err = manager.Freeze(cgroupConfig.Frozen)
	if err != nil && !strings.Contains(err.Error(), "no such file or directory") {
		return fmt.Errorf("failed to freeze cgroup: %v", err)
	}

	var procs []*os.Process
	pids, err := manager.GetAllPids()
	if err != nil {
		multierror.Append(mErrs, fmt.Errorf("error getting pids: %v", err))

		// Unfreeze the cgroup.
		err = manager.Freeze(cgroupConfig.Thawed)
		if err != nil && !strings.Contains(err.Error(), "no such file or directory") {
			multierror.Append(mErrs, fmt.Errorf("failed to unfreeze cgroup: %v", err))
		}
		return mErrs.ErrorOrNil()
	}

	// Kill the processes in the cgroup
	for _, pid := range pids {
		proc, err := os.FindProcess(pid)
		if err != nil {
			multierror.Append(mErrs, fmt.Errorf("error finding process %v: %v", pid, err))
			continue
		}

		procs = append(procs, proc)
		if e := proc.Kill(); e != nil {
			multierror.Append(mErrs, fmt.Errorf("error killing process %v: %v", pid, e))
		}
	}

	// Unfreeze the cgroug so we can wait.
	err = manager.Freeze(cgroupConfig.Thawed)
	if err != nil && !strings.Contains(err.Error(), "no such file or directory") {
		multierror.Append(mErrs, fmt.Errorf("failed to unfreeze cgroup: %v", err))
	}

	// Wait on the killed processes to ensure they are cleaned up.
	for _, proc := range procs {
		// Don't capture the error because we expect this to fail for
		// processes we didn't fork.
		proc.Wait()
	}

	// Remove the cgroup.
	if err := manager.Destroy(); err != nil {
		multierror.Append(mErrs, fmt.Errorf("failed to delete the cgroup directories: %v", err))
	}
	return mErrs.ErrorOrNil()
}

// getCgroupManager returns the correct libcontainer cgroup manager.
func getCgroupManager(groups *cgroupConfig.Cgroup, paths map[string]string) cgroups.Manager {
	return &cgroupFs.Manager{Cgroups: groups, Paths: paths}
}
<commit_msg>Push down the Linux-specific bits into resourceContainer<commit_after>package executor

import (
	"fmt"
	"os"
	"os/user"
	"path/filepath"
	"strconv"
	"strings"
	"syscall"
	"time"

	"github.com/hashicorp/go-multierror"
	"github.com/mitchellh/go-ps"
	"github.com/opencontainers/runc/libcontainer/cgroups"
	cgroupFs "github.com/opencontainers/runc/libcontainer/cgroups/fs"
	cgroupConfig "github.com/opencontainers/runc/libcontainer/configs"
	"github.com/opencontainers/runc/libcontainer/system"

	"github.com/hashicorp/nomad/client/allocdir"
	"github.com/hashicorp/nomad/client/stats"
	cstructs "github.com/hashicorp/nomad/client/structs"
	"github.com/hashicorp/nomad/nomad/structs"
)

var (
	// A mapping of directories on the host OS to attempt to embed inside each
	// task's chroot.
	chrootEnv = map[string]string{
		"/bin":            "/bin",
		"/etc":            "/etc",
		"/lib":            "/lib",
		"/lib32":          "/lib32",
		"/lib64":          "/lib64",
		"/run/resolvconf": "/run/resolvconf",
		"/sbin":           "/sbin",
		"/usr":            "/usr",
	}

	// clockTicks is the clocks per second of the machine
	clockTicks = uint64(system.GetClockTicks())

	// The statistics the executor exposes when using cgroups
	ExecutorCgroupMeasuredMemStats = []string{"RSS", "Cache", "Swap", "Max Usage", "Kernel Usage", "Kernel Max Usage"}
	ExecutorCgroupMeasuredCpuStats = []string{"System Mode", "User Mode", "Throttled Periods", "Throttled Time", "Percent"}
)

// configureIsolation configures chroot and creates cgroups
func (e *UniversalExecutor) configureIsolation() error {
	if e.command.FSIsolation {
		if err := e.configureChroot(); err != nil {
			return err
		}
	}

	if e.command.ResourceLimits {
		if err := e.configureCgroups(e.ctx.Task.Resources); err != nil {
			return fmt.Errorf("error creating cgroups: %v", err)
		}
	}
	return nil
}

// applyLimits puts a process in a pre-configured cgroup
func (e *UniversalExecutor) applyLimits(pid int) error {
	if !e.command.ResourceLimits {
		return nil
	}

	// Entering the process in the cgroup
	manager := getCgroupManager(e.resCon.groups, nil)
	if err := manager.Apply(pid); err != nil {
		e.logger.Printf("[ERR] executor: error applying pid to cgroup: %v", err)
		if er := e.removeChrootMounts(); er != nil {
			e.logger.Printf("[ERR] executor: error removing chroot: %v", er)
		}
		return err
	}
	e.resCon.cgPaths = manager.GetPaths()
	cgConfig := cgroupConfig.Config{Cgroups: e.resCon.groups}
	if err := manager.Set(&cgConfig); err != nil {
		e.logger.Printf("[ERR] executor: error setting cgroup config: %v", err)
		if er := DestroyCgroup(e.resCon.groups, e.resCon.cgPaths, os.Getpid()); er != nil {
			e.logger.Printf("[ERR] executor: error destroying cgroup: %v", er)
		}
		if er := e.removeChrootMounts(); er != nil {
			e.logger.Printf("[ERR] executor: error removing chroot: %v", er)
		}
		return err
	}
	return nil
}

// configureCgroups converts a Nomad Resources specification into the equivalent
// cgroup configuration. It returns an error if the resources are invalid.
func (e *UniversalExecutor) configureCgroups(resources *structs.Resources) error {
	e.resCon.groups = &cgroupConfig.Cgroup{}
	e.resCon.groups.Resources = &cgroupConfig.Resources{}
	cgroupName := structs.GenerateUUID()
	e.resCon.groups.Path = filepath.Join("/nomad", cgroupName)

	// TODO: verify this is needed for things like network access
	e.resCon.groups.Resources.AllowAllDevices = true

	if resources.MemoryMB > 0 {
		// Total amount of memory allowed to consume
		e.resCon.groups.Resources.Memory = int64(resources.MemoryMB * 1024 * 1024)
		// Disable swap to avoid issues on the machine
		e.resCon.groups.Resources.MemorySwap = int64(-1)
	}

	if resources.CPU < 2 {
		return fmt.Errorf("resources.CPU must be equal to or greater than 2: %v", resources.CPU)
	}

	// Set the relative CPU shares for this cgroup.
	e.resCon.groups.Resources.CpuShares = int64(resources.CPU)

	if resources.IOPS != 0 {
		// Validate it is in an acceptable range.
		if resources.IOPS < 10 || resources.IOPS > 1000 {
			return fmt.Errorf("resources.IOPS must be between 10 and 1000: %d", resources.IOPS)
		}

		e.resCon.groups.Resources.BlkioWeight = uint16(resources.IOPS)
	}

	return nil
}

// Stats reports the resource utilization of the cgroup. If there is no resource
// isolation we aggregate the resource utilization of all the pids launched by
// the executor.
func (e *UniversalExecutor) Stats() (*cstructs.TaskResourceUsage, error) {
	if !e.command.ResourceLimits {
		pidStats, err := e.pidStats()
		if err != nil {
			return nil, err
		}
		return e.aggregatedResourceUsage(pidStats), nil
	}
	ts := time.Now()
	manager := getCgroupManager(e.resCon.groups, e.resCon.cgPaths)
	stats, err := manager.GetStats()
	if err != nil {
		return nil, err
	}

	// Memory Related Stats
	swap := stats.MemoryStats.SwapUsage
	maxUsage := stats.MemoryStats.Usage.MaxUsage
	rss := stats.MemoryStats.Stats["rss"]
	cache := stats.MemoryStats.Stats["cache"]
	ms := &cstructs.MemoryStats{
		RSS:            rss,
		Cache:          cache,
		Swap:           swap.Usage,
		MaxUsage:       maxUsage,
		KernelUsage:    stats.MemoryStats.KernelUsage.Usage,
		KernelMaxUsage: stats.MemoryStats.KernelUsage.MaxUsage,
		Measured:       ExecutorCgroupMeasuredMemStats,
	}

	// CPU Related Stats
	totalProcessCPUUsage := float64(stats.CpuStats.CpuUsage.TotalUsage)
	userModeTime := float64(stats.CpuStats.CpuUsage.UsageInUsermode)
	kernelModeTime := float64(stats.CpuStats.CpuUsage.UsageInKernelmode)

	totalPercent := e.totalCpuStats.Percent(totalProcessCPUUsage)
	cs := &cstructs.CpuStats{
		SystemMode:       e.systemCpuStats.Percent(kernelModeTime),
		UserMode:         e.userCpuStats.Percent(userModeTime),
		Percent:          totalPercent,
		ThrottledPeriods: stats.CpuStats.ThrottlingData.ThrottledPeriods,
		ThrottledTime:    stats.CpuStats.ThrottlingData.ThrottledTime,
		TotalTicks:       e.systemCpuStats.TicksConsumed(totalPercent),
		Measured:         ExecutorCgroupMeasuredCpuStats,
	}
	taskResUsage := cstructs.TaskResourceUsage{
		ResourceUsage: &cstructs.ResourceUsage{
			MemoryStats: ms,
			CpuStats:    cs,
		},
		Timestamp: ts.UTC().UnixNano(),
	}
	if pidStats, err := e.pidStats(); err == nil {
		taskResUsage.Pids = pidStats
	}
	return &taskResUsage, nil
}

// runAs takes a user id as a string and looks up the user, and sets the command
// to execute as that user.
func (e *UniversalExecutor) runAs(userid string) error {
	u, err := user.Lookup(userid)
	if err != nil {
		return fmt.Errorf("Failed to identify user %v: %v", userid, err)
	}

	// Convert the uid and gid
	uid, err := strconv.ParseUint(u.Uid, 10, 32)
	if err != nil {
		return fmt.Errorf("Unable to convert userid to uint32: %s", err)
	}
	gid, err := strconv.ParseUint(u.Gid, 10, 32)
	if err != nil {
		return fmt.Errorf("Unable to convert groupid to uint32: %s", err)
	}

	// Set the command to run as that user and group.
	if e.cmd.SysProcAttr == nil {
		e.cmd.SysProcAttr = &syscall.SysProcAttr{}
	}
	if e.cmd.SysProcAttr.Credential == nil {
		e.cmd.SysProcAttr.Credential = &syscall.Credential{}
	}
	e.cmd.SysProcAttr.Credential.Uid = uint32(uid)
	e.cmd.SysProcAttr.Credential.Gid = uint32(gid)

	return nil
}

// configureChroot configures a chroot
func (e *UniversalExecutor) configureChroot() error {
	allocDir := e.ctx.AllocDir
	if err := allocDir.MountSharedDir(e.ctx.Task.Name); err != nil {
		return err
	}

	if err := allocDir.Embed(e.ctx.Task.Name, chrootEnv); err != nil {
		return err
	}

	// Set the tasks AllocDir environment variable.
	e.ctx.TaskEnv.
		SetAllocDir(filepath.Join("/", allocdir.SharedAllocName)).
		SetTaskLocalDir(filepath.Join("/", allocdir.TaskLocal)).
		Build()

	if e.cmd.SysProcAttr == nil {
		e.cmd.SysProcAttr = &syscall.SysProcAttr{}
	}
	e.cmd.SysProcAttr.Chroot = e.taskDir
	e.cmd.Dir = "/"

	if err := allocDir.MountSpecialDirs(e.taskDir); err != nil {
		return err
	}

	e.fsIsolationEnforced = true
	return nil
}

// cleanTaskDir is an idempotent operation to clean the task directory and
// should be called when tearing down the task.
func (e *UniversalExecutor) removeChrootMounts() error {
	// Prevent a race between Wait/ForceStop
	e.cgLock.Lock()
	defer e.cgLock.Unlock()
	return e.ctx.AllocDir.UnmountAll()
}

// getAllPids returns the pids of all the processes spun up by the executor. We
// use the libcontainer apis to get the pids when the user is using cgroup
// isolation and we scan the entire process table if the user is not using any
// isolation
func (e *UniversalExecutor) getAllPids() (map[int]*nomadPid, error) {
	if e.command.ResourceLimits {
		manager := getCgroupManager(e.resCon.groups, e.resCon.cgPaths)
		pids, err := manager.GetAllPids()
		if err != nil {
			return nil, err
		}
		np := make(map[int]*nomadPid, len(pids))
		for _, pid := range pids {
			np[pid] = &nomadPid{
				pid:           pid,
				cpuStatsTotal: stats.NewCpuStats(),
				cpuStatsSys:   stats.NewCpuStats(),
				cpuStatsUser:  stats.NewCpuStats(),
			}
		}
		return np, nil
	}
	allProcesses, err := ps.Processes()
	if err != nil {
		return nil, err
	}
	return e.scanPids(os.Getpid(), allProcesses)
}

// destroyCgroup kills all processes in the cgroup and removes the cgroup
// configuration from the host. This function is idempotent.
func DestroyCgroup(groups *cgroupConfig.Cgroup, cgPaths map[string]string, executorPid int) error {
	mErrs := new(multierror.Error)
	if groups == nil {
		return fmt.Errorf("Can't destroy: cgroup configuration empty")
	}

	// Move the executor into the global cgroup so that the task specific
	// cgroup can be destroyed.
	nilGroup := &cgroupConfig.Cgroup{}
	nilGroup.Path = "/"
	nilGroup.Resources = groups.Resources
	nilManager := getCgroupManager(nilGroup, nil)
	err := nilManager.Apply(executorPid)
	if err != nil && !strings.Contains(err.Error(), "no such process") {
		return fmt.Errorf("failed to remove executor pid %d: %v", executorPid, err)
	}

	// Freeze the Cgroup so that it can not continue to fork/exec.
	manager := getCgroupManager(groups, cgPaths)
	err = manager.Freeze(cgroupConfig.Frozen)
	if err != nil && !strings.Contains(err.Error(), "no such file or directory") {
		return fmt.Errorf("failed to freeze cgroup: %v", err)
	}

	var procs []*os.Process
	pids, err := manager.GetAllPids()
	if err != nil {
		multierror.Append(mErrs, fmt.Errorf("error getting pids: %v", err))

		// Unfreeze the cgroup.
		err = manager.Freeze(cgroupConfig.Thawed)
		if err != nil && !strings.Contains(err.Error(), "no such file or directory") {
			multierror.Append(mErrs, fmt.Errorf("failed to unfreeze cgroup: %v", err))
		}
		return mErrs.ErrorOrNil()
	}

	// Kill the processes in the cgroup
	for _, pid := range pids {
		proc, err := os.FindProcess(pid)
		if err != nil {
			multierror.Append(mErrs, fmt.Errorf("error finding process %v: %v", pid, err))
			continue
		}

		procs = append(procs, proc)
		if e := proc.Kill(); e != nil {
			multierror.Append(mErrs, fmt.Errorf("error killing process %v: %v", pid, e))
		}
	}

	// Unfreeze the cgroug so we can wait.
	err = manager.Freeze(cgroupConfig.Thawed)
	if err != nil && !strings.Contains(err.Error(), "no such file or directory") {
		multierror.Append(mErrs, fmt.Errorf("failed to unfreeze cgroup: %v", err))
	}

	// Wait on the killed processes to ensure they are cleaned up.
	for _, proc := range procs {
		// Don't capture the error because we expect this to fail for
		// processes we didn't fork.
		proc.Wait()
	}

	// Remove the cgroup.
	if err := manager.Destroy(); err != nil {
		multierror.Append(mErrs, fmt.Errorf("failed to delete the cgroup directories: %v", err))
	}
	return mErrs.ErrorOrNil()
}

// getCgroupManager returns the correct libcontainer cgroup manager.
func getCgroupManager(groups *cgroupConfig.Cgroup, paths map[string]string) cgroups.Manager {
	return &cgroupFs.Manager{Cgroups: groups, Paths: paths}
}
<|endoftext|>


<commit_before>/*
Copyright IBM Corp. All Rights Reserved.

SPDX-License-Identifier: Apache-2.0
*/

package common

import (
	"context"
	"crypto/tls"
	"fmt"
	"io/ioutil"
	"os"
	"runtime"
	"strings"
	"time"

	"github.com/hyperledger/fabric/bccsp/factory"
	"github.com/hyperledger/fabric/common/channelconfig"
	"github.com/hyperledger/fabric/common/flogging"
	"github.com/hyperledger/fabric/common/viperutil"
	"github.com/hyperledger/fabric/core/comm"
	"github.com/hyperledger/fabric/core/config"
	"github.com/hyperledger/fabric/core/scc/cscc"
	"github.com/hyperledger/fabric/msp"
	mspmgmt "github.com/hyperledger/fabric/msp/mgmt"
	"github.com/hyperledger/fabric/peer/common/api"
	pcommon "github.com/hyperledger/fabric/protos/common"
	pb "github.com/hyperledger/fabric/protos/peer"
	putils "github.com/hyperledger/fabric/protos/utils"
	"github.com/pkg/errors"
	"github.com/spf13/cobra"
	"github.com/spf13/viper"
)

// UndefinedParamValue defines what undefined parameters in the command line will initialise to
const UndefinedParamValue = ""
const CmdRoot = "core"

var mainLogger = flogging.MustGetLogger("main")
var logOutput = os.Stderr

var (
	defaultConnTimeout = 3 * time.Second
	// These function variables (xyzFnc) can be used to invoke corresponding xyz function
	// this will allow the invoking packages to mock these functions in their unit test cases

	// GetEndorserClientFnc is a function that returns a new endorser client connection
	// to the provided peer address using the TLS root cert file,
	// by default it is set to GetEndorserClient function
	GetEndorserClientFnc func(address, tlsRootCertFile string) (pb.EndorserClient, error)

	// GetPeerDeliverClientFnc is a function that returns a new deliver client connection
	// to the provided peer address using the TLS root cert file,
	// by default it is set to GetDeliverClient function
	GetPeerDeliverClientFnc func(address, tlsRootCertFile string) (api.PeerDeliverClient, error)

	// GetDeliverClientFnc is a function that returns a new deliver client connection
	// to the provided peer address using the TLS root cert file,
	// by default it is set to GetDeliverClient function
	GetDeliverClientFnc func(address, tlsRootCertFile string) (pb.Deliver_DeliverClient, error)

	// GetDefaultSignerFnc is a function that returns a default Signer(Default/PERR)
	// by default it is set to GetDefaultSigner function
	GetDefaultSignerFnc func() (msp.SigningIdentity, error)

	// GetBroadcastClientFnc returns an instance of the BroadcastClient interface
	// by default it is set to GetBroadcastClient function
	GetBroadcastClientFnc func() (BroadcastClient, error)

	// GetOrdererEndpointOfChainFnc returns orderer endpoints of given chain
	// by default it is set to GetOrdererEndpointOfChain function
	GetOrdererEndpointOfChainFnc func(chainID string, signer msp.SigningIdentity,
		endorserClient pb.EndorserClient) ([]string, error)

	// GetCertificateFnc is a function that returns the client TLS certificate
	GetCertificateFnc func() (tls.Certificate, error)
)

type commonClient struct {
	*comm.GRPCClient
	address string
	sn      string
}

func init() {
	GetEndorserClientFnc = GetEndorserClient
	GetDefaultSignerFnc = GetDefaultSigner
	GetBroadcastClientFnc = GetBroadcastClient
	GetOrdererEndpointOfChainFnc = GetOrdererEndpointOfChain
	GetDeliverClientFnc = GetDeliverClient
	GetPeerDeliverClientFnc = GetPeerDeliverClient
	GetCertificateFnc = GetCertificate
}

// InitConfig initializes viper config
func InitConfig(cmdRoot string) error {

	err := config.InitViper(nil, cmdRoot)
	if err != nil {
		return err
	}

	err = viper.ReadInConfig() // Find and read the config file
	if err != nil {            // Handle errors reading the config file
		// The version of Viper we use claims the config type isn't supported when in fact the file hasn't been found
		// Display a more helpful message to avoid confusing the user.
		if strings.Contains(fmt.Sprint(err), "Unsupported Config Type") {
			return errors.New(fmt.Sprintf("Could not find config file. "+
				"Please make sure that FABRIC_CFG_PATH or --configPath is set to a path "+
				"which contains %s.yaml", cmdRoot))
		} else {
			return errors.WithMessage(err, fmt.Sprintf("error when reading %s config file", cmdRoot))
		}
	}

	return nil
}

// InitCrypto initializes crypto for this peer
func InitCrypto(mspMgrConfigDir, localMSPID, localMSPType string) error {
	var err error
	// Check whether msp folder exists
	fi, err := os.Stat(mspMgrConfigDir)
	if os.IsNotExist(err) || !fi.IsDir() {
		// No need to try to load MSP from folder which is not available
		return errors.Errorf("cannot init crypto, folder \"%s\" does not exist", mspMgrConfigDir)
	}
	// Check whether localMSPID exists
	if localMSPID == "" {
		return errors.New("the local MSP must have an ID")
	}

	// Init the BCCSP
	SetBCCSPKeystorePath()
	var bccspConfig *factory.FactoryOpts
	err = viperutil.EnhancedExactUnmarshalKey("peer.BCCSP", &bccspConfig)
	if err != nil {
		return errors.WithMessage(err, "could not parse YAML config")
	}

	err = mspmgmt.LoadLocalMspWithType(mspMgrConfigDir, bccspConfig, localMSPID, localMSPType)
	if err != nil {
		return errors.WithMessage(err, fmt.Sprintf("error when setting up MSP of type %s from directory %s", localMSPType, mspMgrConfigDir))
	}

	return nil
}

// SetBCCSPKeystorePath sets the file keystore path for the SW BCCSP provider
// to an absolute path relative to the config file
func SetBCCSPKeystorePath() {
	viper.Set("peer.BCCSP.SW.FileKeyStore.KeyStore",
		config.GetPath("peer.BCCSP.SW.FileKeyStore.KeyStore"))
}

// GetDefaultSigner return a default Signer(Default/PERR) for cli
func GetDefaultSigner() (msp.SigningIdentity, error) {
	signer, err := mspmgmt.GetLocalMSP().GetDefaultSigningIdentity()
	if err != nil {
		return nil, errors.WithMessage(err, "error obtaining the default signing identity")
	}

	return signer, err
}

// GetOrdererEndpointOfChain returns orderer endpoints of given chain
func GetOrdererEndpointOfChain(chainID string, signer msp.SigningIdentity, endorserClient pb.EndorserClient) ([]string, error) {
	// query cscc for chain config block
	invocation := &pb.ChaincodeInvocationSpec{
		ChaincodeSpec: &pb.ChaincodeSpec{
			Type:        pb.ChaincodeSpec_Type(pb.ChaincodeSpec_Type_value["GOLANG"]),
			ChaincodeId: &pb.ChaincodeID{Name: "cscc"},
			Input:       &pb.ChaincodeInput{Args: [][]byte{[]byte(cscc.GetConfigBlock), []byte(chainID)}},
		},
	}

	creator, err := signer.Serialize()
	if err != nil {
		return nil, errors.WithMessage(err, fmt.Sprintf("error serializing identity for %s", signer.GetIdentifier()))
	}

	prop, _, err := putils.CreateProposalFromCIS(pcommon.HeaderType_CONFIG, "", invocation, creator)
	if err != nil {
		return nil, errors.WithMessage(err, "error creating GetConfigBlock proposal")
	}

	signedProp, err := putils.GetSignedProposal(prop, signer)
	if err != nil {
		return nil, errors.WithMessage(err, "error creating signed GetConfigBlock proposal")
	}

	proposalResp, err := endorserClient.ProcessProposal(context.Background(), signedProp)
	if err != nil {
		return nil, errors.WithMessage(err, "error endorsing GetConfigBlock")
	}

	if proposalResp == nil {
		return nil, errors.WithMessage(err, "error nil proposal response")
	}

	if proposalResp.Response.Status != 0 && proposalResp.Response.Status != 200 {
		return nil, errors.Errorf("error bad proposal response %d: %s", proposalResp.Response.Status, proposalResp.Response.Message)
	}

	// parse config block
	block, err := putils.GetBlockFromBlockBytes(proposalResp.Response.Payload)
	if err != nil {
		return nil, errors.WithMessage(err, "error unmarshaling config block")
	}

	envelopeConfig, err := putils.ExtractEnvelope(block, 0)
	if err != nil {
		return nil, errors.WithMessage(err, "error extracting config block envelope")
	}
	bundle, err := channelconfig.NewBundleFromEnvelope(envelopeConfig)
	if err != nil {
		return nil, errors.WithMessage(err, "error loading config block")
	}

	return bundle.ChannelConfig().OrdererAddresses(), nil
}

// SetLogLevelFromViper sets the log level for 'module' logger to the value in
// core.yaml
func SetLogLevelFromViper(module string) error {
	var err error
	if module == "" {
		return errors.New("log level not set, no module name provided")
	}
	logLevelFromViper := viper.GetString("logging." + module)
	err = CheckLogLevel(logLevelFromViper)
	if err != nil {
		return err
	}
	// replace period in module name with forward slash to allow override
	// of logging submodules
	module = strings.Replace(module, ".", "/", -1)
	// only set logging modules that begin with the supplied module name here
	err = flogging.SetModuleLevels("^"+module, logLevelFromViper)
	return err
}

// CheckLogLevel checks that a given log level string is valid
func CheckLogLevel(level string) error {
	if !flogging.IsValidLevel(level) {
		return errors.Errorf("invalid log level provided - %s", level)
	}
	return nil
}

func configFromEnv(prefix string) (address, override string, clientConfig comm.ClientConfig, err error) {
	address = viper.GetString(prefix + ".address")
	override = viper.GetString(prefix + ".tls.serverhostoverride")
	clientConfig = comm.ClientConfig{}
	connTimeout := viper.GetDuration(prefix + ".client.connTimeout")
	if connTimeout == time.Duration(0) {
		connTimeout = defaultConnTimeout
	}
	clientConfig.Timeout = connTimeout
	secOpts := &comm.SecureOptions{
		UseTLS:            viper.GetBool(prefix + ".tls.enabled"),
		RequireClientCert: viper.GetBool(prefix + ".tls.clientAuthRequired")}
	if secOpts.UseTLS {
		caPEM, res := ioutil.ReadFile(config.GetPath(prefix + ".tls.rootcert.file"))
		if res != nil {
			err = errors.WithMessage(res,
				fmt.Sprintf("unable to load %s.tls.rootcert.file", prefix))
			return
		}
		secOpts.ServerRootCAs = [][]byte{caPEM}
	}
	if secOpts.RequireClientCert {
		keyPEM, res := ioutil.ReadFile(config.GetPath(prefix + ".tls.clientKey.file"))
		if res != nil {
			err = errors.WithMessage(res,
				fmt.Sprintf("unable to load %s.tls.clientKey.file", prefix))
			return
		}
		secOpts.Key = keyPEM
		certPEM, res := ioutil.ReadFile(config.GetPath(prefix + ".tls.clientCert.file"))
		if res != nil {
			err = errors.WithMessage(res,
				fmt.Sprintf("unable to load %s.tls.clientCert.file", prefix))
			return
		}
		secOpts.Certificate = certPEM
	}
	clientConfig.SecOpts = secOpts
	return
}

func InitCmd(cmd *cobra.Command, args []string) {
	err := InitConfig(CmdRoot)
	if err != nil { // Handle errors reading the config file
		mainLogger.Errorf("Fatal error when initializing %s config : %s", CmdRoot, err)
		os.Exit(1)
	}

	// check for --logging-level pflag first, which should override all other
	// log settings. if --logging-level is not set, use CORE_LOGGING_LEVEL
	// (environment variable takes priority; otherwise, the value set in
	// core.yaml)
	var loggingSpec string
	if viper.GetString("logging_level") != "" {
		loggingSpec = viper.GetString("logging_level")
	} else {
		loggingSpec = viper.GetString("logging.level")
	}
	flogging.Init(flogging.Config{
		Format:  viper.GetString("logging.format"),
		Writer:  logOutput,
		LogSpec: loggingSpec,
	})

	// Init the MSP
	var mspMgrConfigDir = config.GetPath("peer.mspConfigPath")
	var mspID = viper.GetString("peer.localMspId")
	var mspType = viper.GetString("peer.localMspType")
	if mspType == "" {
		mspType = msp.ProviderTypeToString(msp.FABRIC)
	}
	err = InitCrypto(mspMgrConfigDir, mspID, mspType)
	if err != nil { // Handle errors reading the config file
		mainLogger.Errorf("Cannot run peer because %s", err.Error())
		os.Exit(1)
	}

	runtime.GOMAXPROCS(viper.GetInt("peer.gomaxprocs"))
}
<commit_msg>[FAB-12419] Fix error mesage in peer command<commit_after>/*
Copyright IBM Corp. All Rights Reserved.

SPDX-License-Identifier: Apache-2.0
*/

package common

import (
	"context"
	"crypto/tls"
	"fmt"
	"io/ioutil"
	"os"
	"runtime"
	"strings"
	"time"

	"github.com/hyperledger/fabric/bccsp/factory"
	"github.com/hyperledger/fabric/common/channelconfig"
	"github.com/hyperledger/fabric/common/flogging"
	"github.com/hyperledger/fabric/common/viperutil"
	"github.com/hyperledger/fabric/core/comm"
	"github.com/hyperledger/fabric/core/config"
	"github.com/hyperledger/fabric/core/scc/cscc"
	"github.com/hyperledger/fabric/msp"
	mspmgmt "github.com/hyperledger/fabric/msp/mgmt"
	"github.com/hyperledger/fabric/peer/common/api"
	pcommon "github.com/hyperledger/fabric/protos/common"
	pb "github.com/hyperledger/fabric/protos/peer"
	putils "github.com/hyperledger/fabric/protos/utils"
	"github.com/pkg/errors"
	"github.com/spf13/cobra"
	"github.com/spf13/viper"
)

// UndefinedParamValue defines what undefined parameters in the command line will initialise to
const UndefinedParamValue = ""
const CmdRoot = "core"

var mainLogger = flogging.MustGetLogger("main")
var logOutput = os.Stderr

var (
	defaultConnTimeout = 3 * time.Second
	// These function variables (xyzFnc) can be used to invoke corresponding xyz function
	// this will allow the invoking packages to mock these functions in their unit test cases

	// GetEndorserClientFnc is a function that returns a new endorser client connection
	// to the provided peer address using the TLS root cert file,
	// by default it is set to GetEndorserClient function
	GetEndorserClientFnc func(address, tlsRootCertFile string) (pb.EndorserClient, error)

	// GetPeerDeliverClientFnc is a function that returns a new deliver client connection
	// to the provided peer address using the TLS root cert file,
	// by default it is set to GetDeliverClient function
	GetPeerDeliverClientFnc func(address, tlsRootCertFile string) (api.PeerDeliverClient, error)

	// GetDeliverClientFnc is a function that returns a new deliver client connection
	// to the provided peer address using the TLS root cert file,
	// by default it is set to GetDeliverClient function
	GetDeliverClientFnc func(address, tlsRootCertFile string) (pb.Deliver_DeliverClient, error)

	// GetDefaultSignerFnc is a function that returns a default Signer(Default/PERR)
	// by default it is set to GetDefaultSigner function
	GetDefaultSignerFnc func() (msp.SigningIdentity, error)

	// GetBroadcastClientFnc returns an instance of the BroadcastClient interface
	// by default it is set to GetBroadcastClient function
	GetBroadcastClientFnc func() (BroadcastClient, error)

	// GetOrdererEndpointOfChainFnc returns orderer endpoints of given chain
	// by default it is set to GetOrdererEndpointOfChain function
	GetOrdererEndpointOfChainFnc func(chainID string, signer msp.SigningIdentity,
		endorserClient pb.EndorserClient) ([]string, error)

	// GetCertificateFnc is a function that returns the client TLS certificate
	GetCertificateFnc func() (tls.Certificate, error)
)

type commonClient struct {
	*comm.GRPCClient
	address string
	sn      string
}

func init() {
	GetEndorserClientFnc = GetEndorserClient
	GetDefaultSignerFnc = GetDefaultSigner
	GetBroadcastClientFnc = GetBroadcastClient
	GetOrdererEndpointOfChainFnc = GetOrdererEndpointOfChain
	GetDeliverClientFnc = GetDeliverClient
	GetPeerDeliverClientFnc = GetPeerDeliverClient
	GetCertificateFnc = GetCertificate
}

// InitConfig initializes viper config
func InitConfig(cmdRoot string) error {

	err := config.InitViper(nil, cmdRoot)
	if err != nil {
		return err
	}

	err = viper.ReadInConfig() // Find and read the config file
	if err != nil {            // Handle errors reading the config file
		// The version of Viper we use claims the config type isn't supported when in fact the file hasn't been found
		// Display a more helpful message to avoid confusing the user.
		if strings.Contains(fmt.Sprint(err), "Unsupported Config Type") {
			return errors.New(fmt.Sprintf("Could not find config file. "+
				"Please make sure that FABRIC_CFG_PATH is set to a path "+
				"which contains %s.yaml", cmdRoot))
		} else {
			return errors.WithMessage(err, fmt.Sprintf("error when reading %s config file", cmdRoot))
		}
	}

	return nil
}

// InitCrypto initializes crypto for this peer
func InitCrypto(mspMgrConfigDir, localMSPID, localMSPType string) error {
	var err error
	// Check whether msp folder exists
	fi, err := os.Stat(mspMgrConfigDir)
	if os.IsNotExist(err) || !fi.IsDir() {
		// No need to try to load MSP from folder which is not available
		return errors.Errorf("cannot init crypto, folder \"%s\" does not exist", mspMgrConfigDir)
	}
	// Check whether localMSPID exists
	if localMSPID == "" {
		return errors.New("the local MSP must have an ID")
	}

	// Init the BCCSP
	SetBCCSPKeystorePath()
	var bccspConfig *factory.FactoryOpts
	err = viperutil.EnhancedExactUnmarshalKey("peer.BCCSP", &bccspConfig)
	if err != nil {
		return errors.WithMessage(err, "could not parse YAML config")
	}

	err = mspmgmt.LoadLocalMspWithType(mspMgrConfigDir, bccspConfig, localMSPID, localMSPType)
	if err != nil {
		return errors.WithMessage(err, fmt.Sprintf("error when setting up MSP of type %s from directory %s", localMSPType, mspMgrConfigDir))
	}

	return nil
}

// SetBCCSPKeystorePath sets the file keystore path for the SW BCCSP provider
// to an absolute path relative to the config file
func SetBCCSPKeystorePath() {
	viper.Set("peer.BCCSP.SW.FileKeyStore.KeyStore",
		config.GetPath("peer.BCCSP.SW.FileKeyStore.KeyStore"))
}

// GetDefaultSigner return a default Signer(Default/PERR) for cli
func GetDefaultSigner() (msp.SigningIdentity, error) {
	signer, err := mspmgmt.GetLocalMSP().GetDefaultSigningIdentity()
	if err != nil {
		return nil, errors.WithMessage(err, "error obtaining the default signing identity")
	}

	return signer, err
}

// GetOrdererEndpointOfChain returns orderer endpoints of given chain
func GetOrdererEndpointOfChain(chainID string, signer msp.SigningIdentity, endorserClient pb.EndorserClient) ([]string, error) {
	// query cscc for chain config block
	invocation := &pb.ChaincodeInvocationSpec{
		ChaincodeSpec: &pb.ChaincodeSpec{
			Type:        pb.ChaincodeSpec_Type(pb.ChaincodeSpec_Type_value["GOLANG"]),
			ChaincodeId: &pb.ChaincodeID{Name: "cscc"},
			Input:       &pb.ChaincodeInput{Args: [][]byte{[]byte(cscc.GetConfigBlock), []byte(chainID)}},
		},
	}

	creator, err := signer.Serialize()
	if err != nil {
		return nil, errors.WithMessage(err, fmt.Sprintf("error serializing identity for %s", signer.GetIdentifier()))
	}

	prop, _, err := putils.CreateProposalFromCIS(pcommon.HeaderType_CONFIG, "", invocation, creator)
	if err != nil {
		return nil, errors.WithMessage(err, "error creating GetConfigBlock proposal")
	}

	signedProp, err := putils.GetSignedProposal(prop, signer)
	if err != nil {
		return nil, errors.WithMessage(err, "error creating signed GetConfigBlock proposal")
	}

	proposalResp, err := endorserClient.ProcessProposal(context.Background(), signedProp)
	if err != nil {
		return nil, errors.WithMessage(err, "error endorsing GetConfigBlock")
	}

	if proposalResp == nil {
		return nil, errors.WithMessage(err, "error nil proposal response")
	}

	if proposalResp.Response.Status != 0 && proposalResp.Response.Status != 200 {
		return nil, errors.Errorf("error bad proposal response %d: %s", proposalResp.Response.Status, proposalResp.Response.Message)
	}

	// parse config block
	block, err := putils.GetBlockFromBlockBytes(proposalResp.Response.Payload)
	if err != nil {
		return nil, errors.WithMessage(err, "error unmarshaling config block")
	}

	envelopeConfig, err := putils.ExtractEnvelope(block, 0)
	if err != nil {
		return nil, errors.WithMessage(err, "error extracting config block envelope")
	}
	bundle, err := channelconfig.NewBundleFromEnvelope(envelopeConfig)
	if err != nil {
		return nil, errors.WithMessage(err, "error loading config block")
	}

	return bundle.ChannelConfig().OrdererAddresses(), nil
}

// SetLogLevelFromViper sets the log level for 'module' logger to the value in
// core.yaml
func SetLogLevelFromViper(module string) error {
	var err error
	if module == "" {
		return errors.New("log level not set, no module name provided")
	}
	logLevelFromViper := viper.GetString("logging." + module)
	err = CheckLogLevel(logLevelFromViper)
	if err != nil {
		return err
	}
	// replace period in module name with forward slash to allow override
	// of logging submodules
	module = strings.Replace(module, ".", "/", -1)
	// only set logging modules that begin with the supplied module name here
	err = flogging.SetModuleLevels("^"+module, logLevelFromViper)
	return err
}

// CheckLogLevel checks that a given log level string is valid
func CheckLogLevel(level string) error {
	if !flogging.IsValidLevel(level) {
		return errors.Errorf("invalid log level provided - %s", level)
	}
	return nil
}

func configFromEnv(prefix string) (address, override string, clientConfig comm.ClientConfig, err error) {
	address = viper.GetString(prefix + ".address")
	override = viper.GetString(prefix + ".tls.serverhostoverride")
	clientConfig = comm.ClientConfig{}
	connTimeout := viper.GetDuration(prefix + ".client.connTimeout")
	if connTimeout == time.Duration(0) {
		connTimeout = defaultConnTimeout
	}
	clientConfig.Timeout = connTimeout
	secOpts := &comm.SecureOptions{
		UseTLS:            viper.GetBool(prefix + ".tls.enabled"),
		RequireClientCert: viper.GetBool(prefix + ".tls.clientAuthRequired")}
	if secOpts.UseTLS {
		caPEM, res := ioutil.ReadFile(config.GetPath(prefix + ".tls.rootcert.file"))
		if res != nil {
			err = errors.WithMessage(res,
				fmt.Sprintf("unable to load %s.tls.rootcert.file", prefix))
			return
		}
		secOpts.ServerRootCAs = [][]byte{caPEM}
	}
	if secOpts.RequireClientCert {
		keyPEM, res := ioutil.ReadFile(config.GetPath(prefix + ".tls.clientKey.file"))
		if res != nil {
			err = errors.WithMessage(res,
				fmt.Sprintf("unable to load %s.tls.clientKey.file", prefix))
			return
		}
		secOpts.Key = keyPEM
		certPEM, res := ioutil.ReadFile(config.GetPath(prefix + ".tls.clientCert.file"))
		if res != nil {
			err = errors.WithMessage(res,
				fmt.Sprintf("unable to load %s.tls.clientCert.file", prefix))
			return
		}
		secOpts.Certificate = certPEM
	}
	clientConfig.SecOpts = secOpts
	return
}

func InitCmd(cmd *cobra.Command, args []string) {
	err := InitConfig(CmdRoot)
	if err != nil { // Handle errors reading the config file
		mainLogger.Errorf("Fatal error when initializing %s config : %s", CmdRoot, err)
		os.Exit(1)
	}

	// check for --logging-level pflag first, which should override all other
	// log settings. if --logging-level is not set, use CORE_LOGGING_LEVEL
	// (environment variable takes priority; otherwise, the value set in
	// core.yaml)
	var loggingSpec string
	if viper.GetString("logging_level") != "" {
		loggingSpec = viper.GetString("logging_level")
	} else {
		loggingSpec = viper.GetString("logging.level")
	}
	flogging.Init(flogging.Config{
		Format:  viper.GetString("logging.format"),
		Writer:  logOutput,
		LogSpec: loggingSpec,
	})

	// Init the MSP
	var mspMgrConfigDir = config.GetPath("peer.mspConfigPath")
	var mspID = viper.GetString("peer.localMspId")
	var mspType = viper.GetString("peer.localMspType")
	if mspType == "" {
		mspType = msp.ProviderTypeToString(msp.FABRIC)
	}
	err = InitCrypto(mspMgrConfigDir, mspID, mspType)
	if err != nil { // Handle errors reading the config file
		mainLogger.Errorf("Cannot run peer because %s", err.Error())
		os.Exit(1)
	}

	runtime.GOMAXPROCS(viper.GetInt("peer.gomaxprocs"))
}
<|endoftext|>