<commit_before>package http2

import (
	"reflect"

	"golang.org/x/net/http2"

	"github.com/summerwind/h2spec/config"
	"github.com/summerwind/h2spec/spec"
)

func Ping() *spec.TestGroup {
	tg := NewTestGroup("6.7", "PING")

	// Receivers of a PING frame that does not include an ACK flag MUST
	// send a PING frame with the ACK flag set in response, with an
	// identical payload.
	tg.AddTestCase(&spec.TestCase{
		Desc:        "Sends a PING frame",
		Requirement: "The endpoint MUST sends a PING frame with ACK, with an identical payload.",
		Run: func(c *config.Config, conn *spec.Conn) error {
			var actual spec.Event

			err := conn.Handshake()
			if err != nil {
				return err
			}

			data := [8]byte{'h', '2', 's', 'p', 'e', 'c'}
			conn.WritePing(false, data)

			passed := false
			for !conn.Closed {
				ev := conn.WaitEvent()

				switch event := ev.(type) {
				case spec.EventPingFrame:
					actual = event
					if event.IsAck() && reflect.DeepEqual(event.Data, data) {
						passed = true
					}
				case spec.EventTimeout:
					if actual == nil {
						actual = event
					}
				default:
					actual = ev
				}

				if passed {
					break
				}
			}

			if !passed {
				expected := []string{
					"PING Frame (length:8, flags:0x01, stream_id:0)",
				}

				return &spec.TestError{
					Expected: expected,
					Actual:   actual.String(),
				}
			}

			return nil
		},
	})

	// ACK (0x1):
	// When set, bit 0 indicates that this PING frame is a PING
	// response. An endpoint MUST set this flag in PING responses.
	// An endpoint MUST NOT respond to PING frames containing this
	// flag.
	tg.AddTestCase(&spec.TestCase{
		Desc:        "Sends a PING frame with ACK",
		Requirement: "The endpoint MUST NOT respond to PING frames with ACK.",
		Run: func(c *config.Config, conn *spec.Conn) error {
			var actual spec.Event

			err := conn.Handshake()
			if err != nil {
				return err
			}

			unexpectedData := [8]byte{'i', 'n', 'v', 'a', 'l', 'i', 'd'}
			expectedData := [8]byte{'h', '2', 's', 'p', 'e', 'c'}
			conn.WritePing(true, unexpectedData)
			conn.WritePing(false, expectedData)

			passed := false
			invalid := false
			for !conn.Closed {
				ev := conn.WaitEvent()

				switch event := ev.(type) {
				case spec.EventPingFrame:
					actual = event
					if reflect.DeepEqual(event.Data, unexpectedData) {
						invalid = true
					} else if event.IsAck() && reflect.DeepEqual(event.Data, expectedData) {
						passed = true
					}
				case spec.EventTimeout:
					if actual == nil {
						actual = event
					}
				default:
					actual = ev
				}

				if passed || invalid {
					break
				}
			}

			if !passed {
				expected := []string{
					"PING Frame (length:8, flags:0x01, stream_id:0)",
				}

				return &spec.TestError{
					Expected: expected,
					Actual:   actual.String(),
				}
			}

			return nil
		},
	})

	// If a PING frame is received with a stream identifier field value
	// other than 0x0, the recipient MUST respond with a connection
	// error (Section 5.4.1) of type PROTOCOL_ERROR.
	tg.AddTestCase(&spec.TestCase{
		Desc:        "Sends a PING frame with a stream identifier field value other than 0x0",
		Requirement: "The endpoint MUST respond with a connection error of type PROTOCOL_ERROR.",
		Run: func(c *config.Config, conn *spec.Conn) error {
			err := conn.Handshake()
			if err != nil {
				return err
			}

			// PING frame:
			// length: 8, flags: 0x0, stream_id: 1
			conn.Send([]byte("\x00\x00\x08\x06\x00\x00\x00\x00\x01"))
			conn.Send([]byte("\x00\x00\x00\x00\x00\x00\x00\x00"))

			return spec.VerifyConnectionError(conn, http2.ErrCodeProtocol)
		},
	})

	// Receipt of a PING frame with a length field value other than 8
	// MUST be treated as a connection error (Section 5.4.1) of type
	// FRAME_SIZE_ERROR.
	tg.AddTestCase(&spec.TestCase{
		Desc:        "Sends a PING frame with a length field value other than 8",
		Requirement: "The endpoint MUST treated as a connection error of type FRAME_SIZE_ERROR.",
		Run: func(c *config.Config, conn *spec.Conn) error {
			err := conn.Handshake()
			if err != nil {
				return err
			}

			// PING frame:
			// length: 8, flags: 0x0, stream_id: 1
			conn.Send([]byte("\x00\x00\x06\x06\x00\x00\x00\x00\x01"))
			conn.Send([]byte("\x00\x00\x00\x00\x00\x00"))

			return spec.VerifyConnectionError(conn, http2.ErrCodeProtocol)
		},
	})

	return tg
}
<commit_msg>Fix expected error code<commit_after>package http2

import (
	"reflect"

	"golang.org/x/net/http2"

	"github.com/summerwind/h2spec/config"
	"github.com/summerwind/h2spec/spec"
)

func Ping() *spec.TestGroup {
	tg := NewTestGroup("6.7", "PING")

	// Receivers of a PING frame that does not include an ACK flag MUST
	// send a PING frame with the ACK flag set in response, with an
	// identical payload.
	tg.AddTestCase(&spec.TestCase{
		Desc:        "Sends a PING frame",
		Requirement: "The endpoint MUST sends a PING frame with ACK, with an identical payload.",
		Run: func(c *config.Config, conn *spec.Conn) error {
			var actual spec.Event

			err := conn.Handshake()
			if err != nil {
				return err
			}

			data := [8]byte{'h', '2', 's', 'p', 'e', 'c'}
			conn.WritePing(false, data)

			passed := false
			for !conn.Closed {
				ev := conn.WaitEvent()

				switch event := ev.(type) {
				case spec.EventPingFrame:
					actual = event
					if event.IsAck() && reflect.DeepEqual(event.Data, data) {
						passed = true
					}
				case spec.EventTimeout:
					if actual == nil {
						actual = event
					}
				default:
					actual = ev
				}

				if passed {
					break
				}
			}

			if !passed {
				expected := []string{
					"PING Frame (length:8, flags:0x01, stream_id:0)",
				}

				return &spec.TestError{
					Expected: expected,
					Actual:   actual.String(),
				}
			}

			return nil
		},
	})

	// ACK (0x1):
	// When set, bit 0 indicates that this PING frame is a PING
	// response. An endpoint MUST set this flag in PING responses.
	// An endpoint MUST NOT respond to PING frames containing this
	// flag.
	tg.AddTestCase(&spec.TestCase{
		Desc:        "Sends a PING frame with ACK",
		Requirement: "The endpoint MUST NOT respond to PING frames with ACK.",
		Run: func(c *config.Config, conn *spec.Conn) error {
			var actual spec.Event

			err := conn.Handshake()
			if err != nil {
				return err
			}

			unexpectedData := [8]byte{'i', 'n', 'v', 'a', 'l', 'i', 'd'}
			expectedData := [8]byte{'h', '2', 's', 'p', 'e', 'c'}
			conn.WritePing(true, unexpectedData)
			conn.WritePing(false, expectedData)

			passed := false
			invalid := false
			for !conn.Closed {
				ev := conn.WaitEvent()

				switch event := ev.(type) {
				case spec.EventPingFrame:
					actual = event
					if reflect.DeepEqual(event.Data, unexpectedData) {
						invalid = true
					} else if event.IsAck() && reflect.DeepEqual(event.Data, expectedData) {
						passed = true
					}
				case spec.EventTimeout:
					if actual == nil {
						actual = event
					}
				default:
					actual = ev
				}

				if passed || invalid {
					break
				}
			}

			if !passed {
				expected := []string{
					"PING Frame (length:8, flags:0x01, stream_id:0)",
				}

				return &spec.TestError{
					Expected: expected,
					Actual:   actual.String(),
				}
			}

			return nil
		},
	})

	// If a PING frame is received with a stream identifier field value
	// other than 0x0, the recipient MUST respond with a connection
	// error (Section 5.4.1) of type PROTOCOL_ERROR.
	tg.AddTestCase(&spec.TestCase{
		Desc:        "Sends a PING frame with a stream identifier field value other than 0x0",
		Requirement: "The endpoint MUST respond with a connection error of type PROTOCOL_ERROR.",
		Run: func(c *config.Config, conn *spec.Conn) error {
			err := conn.Handshake()
			if err != nil {
				return err
			}

			// PING frame:
			// length: 8, flags: 0x0, stream_id: 1
			conn.Send([]byte("\x00\x00\x08\x06\x00\x00\x00\x00\x01"))
			conn.Send([]byte("\x00\x00\x00\x00\x00\x00\x00\x00"))

			return spec.VerifyConnectionError(conn, http2.ErrCodeProtocol)
		},
	})

	// Receipt of a PING frame with a length field value other than 8
	// MUST be treated as a connection error (Section 5.4.1) of type
	// FRAME_SIZE_ERROR.
	tg.AddTestCase(&spec.TestCase{
		Desc:        "Sends a PING frame with a length field value other than 8",
		Requirement: "The endpoint MUST treated as a connection error of type FRAME_SIZE_ERROR.",
		Run: func(c *config.Config, conn *spec.Conn) error {
			err := conn.Handshake()
			if err != nil {
				return err
			}

			// PING frame:
			// length: 8, flags: 0x0, stream_id: 1
			conn.Send([]byte("\x00\x00\x06\x06\x00\x00\x00\x00\x01"))
			conn.Send([]byte("\x00\x00\x00\x00\x00\x00"))

			return spec.VerifyConnectionError(conn, http2.ErrCodeFrameSize)
		},
	})

	return tg
}
<|endoftext|>


<commit_before>package air

import (
	"encoding/json"
	"encoding/xml"
	"errors"
	"mime"
	"reflect"
	"strconv"
)

// binder is a binder that binds request based on the MIME types.
type binder struct{}

// binderSingleton is the singleton of the `binder`.
var binderSingleton = &binder{}

// bind binds the r into the v.
func (b *binder) bind(v interface{}, r *Request) error {
	if r.Method == "GET" {
		err := b.bindParams(v, r.QueryParams)
		if err != nil {
			err = &Error{
				Code:    400,
				Message: err.Error(),
			}
		}
		return err
	} else if r.Body == nil {
		return &Error{
			Code:    400,
			Message: "request body can't be empty",
		}
	}

	mt, _, err := mime.ParseMediaType(r.Headers["Content-Type"])
	if err != nil {
		return &Error{
			Code:    400,
			Message: err.Error(),
		}
	}

	switch mt {
	case "application/json":
		err = json.NewDecoder(r.Body).Decode(v)
	case "application/xml":
		err = xml.NewDecoder(r.Body).Decode(v)
	case "application/x-www-form-urlencoded", "multipart/form-data":
		err = b.bindParams(v, r.FormParams)
	default:
		return &Error{
			Code:    415,
			Message: "Unsupported Media Type",
		}
	}

	return &Error{
		Code:    400,
		Message: err.Error(),
	}
}

// bindParams binds the params into the v.
func (b *binder) bindParams(v interface{}, params map[string]string) error {
	typ := reflect.TypeOf(v).Elem()
	if typ.Kind() != reflect.Struct {
		return errors.New("binding element must be a struct")
	}

	val := reflect.ValueOf(v).Elem()
	for i := 0; i < typ.NumField(); i++ {
		vf := val.Field(i)
		if !vf.CanSet() {
			continue
		}

		vfk := vf.Kind()
		if vfk == reflect.Struct {
			err := b.bindParams(vf.Addr().Interface(), params)
			if err != nil {
				return err
			}
			continue
		}

		tf := typ.Field(i)

		p, ok := params[tf.Name]
		if !ok {
			continue
		}

		switch tf.Type.Kind() {
		case reflect.Int,
			reflect.Int8,
			reflect.Int16,
			reflect.Int32,
			reflect.Int64:
			if p == "" {
				p = "0"
			}
			v, err := strconv.ParseInt(p, 10, 64)
			if err != nil {
				return err
			}
			vf.SetInt(v)
		case reflect.Uint,
			reflect.Uint8,
			reflect.Uint16,
			reflect.Uint32,
			reflect.Uint64:
			if p == "" {
				p = "0"
			}
			v, err := strconv.ParseUint(p, 10, 64)
			if err != nil {
				return err
			}
			vf.SetUint(v)
		case reflect.Bool:
			if p == "" {
				p = "false"
			}
			v, err := strconv.ParseBool(p)
			if err != nil {
				return err
			}
			vf.SetBool(v)
		case reflect.Float32, reflect.Float64:
			if p == "" {
				p = "0.0"
			}
			v, err := strconv.ParseFloat(p, 64)
			if err != nil {
				return err
			}
			vf.SetFloat(v)
		case reflect.String:
			vf.SetString(p)
		default:
			return errors.New("unknown type")
		}
	}

	return nil
}
<commit_msg>fix: `binder#bind()`<commit_after>package air

import (
	"encoding/json"
	"encoding/xml"
	"errors"
	"mime"
	"reflect"
	"strconv"
)

// binder is a binder that binds request based on the MIME types.
type binder struct{}

// binderSingleton is the singleton of the `binder`.
var binderSingleton = &binder{}

// bind binds the r into the v.
func (b *binder) bind(v interface{}, r *Request) (err error) {
	defer func() {
		if _, ok := err.(*Error); !ok && err != nil {
			err = &Error{
				Code:    400,
				Message: err.Error(),
			}
		}
	}()

	if r.Method == "GET" {
		return b.bindParams(v, r.QueryParams)
	} else if r.Body == nil {
		return errors.New("request body can't be empty")
	}

	mt, _, err := mime.ParseMediaType(r.Headers["Content-Type"])
	if err != nil {
		return err
	}

	switch mt {
	case "application/json":
		err = json.NewDecoder(r.Body).Decode(v)
	case "application/xml":
		err = xml.NewDecoder(r.Body).Decode(v)
	case "application/x-www-form-urlencoded", "multipart/form-data":
		err = b.bindParams(v, r.FormParams)
	default:
		err = &Error{
			Code:    415,
			Message: "Unsupported Media Type",
		}
	}

	return err
}

// bindParams binds the params into the v.
func (b *binder) bindParams(v interface{}, params map[string]string) error {
	typ := reflect.TypeOf(v).Elem()
	if typ.Kind() != reflect.Struct {
		return errors.New("binding element must be a struct")
	}

	val := reflect.ValueOf(v).Elem()
	for i := 0; i < typ.NumField(); i++ {
		vf := val.Field(i)
		if !vf.CanSet() {
			continue
		}

		vfk := vf.Kind()
		if vfk == reflect.Struct {
			err := b.bindParams(vf.Addr().Interface(), params)
			if err != nil {
				return err
			}
			continue
		}

		tf := typ.Field(i)

		p, ok := params[tf.Name]
		if !ok {
			continue
		}

		switch tf.Type.Kind() {
		case reflect.Int,
			reflect.Int8,
			reflect.Int16,
			reflect.Int32,
			reflect.Int64:
			if p == "" {
				p = "0"
			}
			v, err := strconv.ParseInt(p, 10, 64)
			if err != nil {
				return err
			}
			vf.SetInt(v)
		case reflect.Uint,
			reflect.Uint8,
			reflect.Uint16,
			reflect.Uint32,
			reflect.Uint64:
			if p == "" {
				p = "0"
			}
			v, err := strconv.ParseUint(p, 10, 64)
			if err != nil {
				return err
			}
			vf.SetUint(v)
		case reflect.Bool:
			if p == "" {
				p = "false"
			}
			v, err := strconv.ParseBool(p)
			if err != nil {
				return err
			}
			vf.SetBool(v)
		case reflect.Float32, reflect.Float64:
			if p == "" {
				p = "0.0"
			}
			v, err := strconv.ParseFloat(p, 64)
			if err != nil {
				return err
			}
			vf.SetFloat(v)
		case reflect.String:
			vf.SetString(p)
		default:
			return errors.New("unknown type")
		}
	}

	return nil
}
<|endoftext|>


<commit_before>// Copyright 2012 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package runtime_test

import (
	"fmt"
	"internal/testenv"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"runtime"
	"strings"
	"sync"
	"testing"
)

var toRemove []string

func TestMain(m *testing.M) {
	status := m.Run()
	for _, file := range toRemove {
		os.RemoveAll(file)
	}
	os.Exit(status)
}

func testEnv(cmd *exec.Cmd) *exec.Cmd {
	if cmd.Env != nil {
		panic("environment already set")
	}
	for _, env := range os.Environ() {
		// Exclude GODEBUG from the environment to prevent its output
		// from breaking tests that are trying to parse other command output.
		if strings.HasPrefix(env, "GODEBUG=") {
			continue
		}
		// Exclude GOTRACEBACK for the same reason.
		if strings.HasPrefix(env, "GOTRACEBACK=") {
			continue
		}
		cmd.Env = append(cmd.Env, env)
	}
	return cmd
}

var testprog struct {
	sync.Mutex
	dir    string
	target map[string]buildexe
}

type buildexe struct {
	exe string
	err error
}

func runTestProg(t *testing.T, binary, name string) string {
	testenv.MustHaveGoBuild(t)

	exe, err := buildTestProg(t, binary)
	if err != nil {
		t.Fatal(err)
	}
	got, _ := testEnv(exec.Command(exe, name)).CombinedOutput()
	return string(got)
}

func buildTestProg(t *testing.T, binary string) (string, error) {
	checkStaleRuntime(t)

	testprog.Lock()
	defer testprog.Unlock()
	if testprog.dir == "" {
		dir, err := ioutil.TempDir("", "go-build")
		if err != nil {
			t.Fatalf("failed to create temp directory: %v", err)
		}
		testprog.dir = dir
		toRemove = append(toRemove, dir)
	}

	if testprog.target == nil {
		testprog.target = make(map[string]buildexe)
	}
	target, ok := testprog.target[binary]
	if ok {
		return target.exe, target.err
	}

	exe := filepath.Join(testprog.dir, binary+".exe")
	cmd := exec.Command("go", "build", "-o", exe)
	cmd.Dir = "testdata/" + binary
	out, err := testEnv(cmd).CombinedOutput()
	if err != nil {
		exe = ""
		target.err = fmt.Errorf("building %s: %v\n%s", binary, err, out)
		testprog.target[binary] = target
		return "", err
	}
	target.exe = exe
	testprog.target[binary] = target
	return exe, nil
}

var (
	staleRuntimeOnce sync.Once // guards init of staleRuntimeErr
	staleRuntimeErr  error
)

func checkStaleRuntime(t *testing.T) {
	staleRuntimeOnce.Do(func() {
		// 'go run' uses the installed copy of runtime.a, which may be out of date.
		out, err := testEnv(exec.Command("go", "list", "-f", "{{.Stale}}", "runtime")).CombinedOutput()
		if err != nil {
			staleRuntimeErr = fmt.Errorf("failed to execute 'go list': %v\n%v", err, string(out))
			return
		}
		if string(out) != "false\n" {
			staleRuntimeErr = fmt.Errorf("Stale runtime.a. Run 'go install runtime'.")
		}
	})
	if staleRuntimeErr != nil {
		t.Fatal(staleRuntimeErr)
	}
}

func testCrashHandler(t *testing.T, cgo bool) {
	type crashTest struct {
		Cgo bool
	}
	var output string
	if cgo {
		output = runTestProg(t, "testprogcgo", "Crash")
	} else {
		output = runTestProg(t, "testprog", "Crash")
	}
	want := "main: recovered done\nnew-thread: recovered done\nsecond-new-thread: recovered done\nmain-again: recovered done\n"
	if output != want {
		t.Fatalf("output:\n%s\n\nwanted:\n%s", output, want)
	}
}

func TestCrashHandler(t *testing.T) {
	testCrashHandler(t, false)
}

func testDeadlock(t *testing.T, name string) {
	output := runTestProg(t, "testprog", name)
	want := "fatal error: all goroutines are asleep - deadlock!\n"
	if !strings.HasPrefix(output, want) {
		t.Fatalf("output does not start with %q:\n%s", want, output)
	}
}

func TestSimpleDeadlock(t *testing.T) {
	testDeadlock(t, "SimpleDeadlock")
}

func TestInitDeadlock(t *testing.T) {
	testDeadlock(t, "InitDeadlock")
}

func TestLockedDeadlock(t *testing.T) {
	testDeadlock(t, "LockedDeadlock")
}

func TestLockedDeadlock2(t *testing.T) {
	testDeadlock(t, "LockedDeadlock2")
}

func TestGoexitDeadlock(t *testing.T) {
	output := runTestProg(t, "testprog", "GoexitDeadlock")
	want := "no goroutines (main called runtime.Goexit) - deadlock!"
	if !strings.Contains(output, want) {
		t.Fatalf("output:\n%s\n\nwant output containing: %s", output, want)
	}
}

func TestStackOverflow(t *testing.T) {
	output := runTestProg(t, "testprog", "StackOverflow")
	want := "runtime: goroutine stack exceeds 1474560-byte limit\nfatal error: stack overflow"
	if !strings.HasPrefix(output, want) {
		t.Fatalf("output does not start with %q:\n%s", want, output)
	}
}

func TestThreadExhaustion(t *testing.T) {
	output := runTestProg(t, "testprog", "ThreadExhaustion")
	want := "runtime: program exceeds 10-thread limit\nfatal error: thread exhaustion"
	if !strings.HasPrefix(output, want) {
		t.Fatalf("output does not start with %q:\n%s", want, output)
	}
}

func TestRecursivePanic(t *testing.T) {
	output := runTestProg(t, "testprog", "RecursivePanic")
	want := `wrap: bad
panic: again

`
	if !strings.HasPrefix(output, want) {
		t.Fatalf("output does not start with %q:\n%s", want, output)
	}

}

func TestGoexitCrash(t *testing.T) {
	output := runTestProg(t, "testprog", "GoexitExit")
	want := "no goroutines (main called runtime.Goexit) - deadlock!"
	if !strings.Contains(output, want) {
		t.Fatalf("output:\n%s\n\nwant output containing: %s", output, want)
	}
}

func TestGoexitDefer(t *testing.T) {
	c := make(chan struct{})
	go func() {
		defer func() {
			r := recover()
			if r != nil {
				t.Errorf("non-nil recover during Goexit")
			}
			c <- struct{}{}
		}()
		runtime.Goexit()
	}()
	// Note: if the defer fails to run, we will get a deadlock here
	<-c
}

func TestGoNil(t *testing.T) {
	output := runTestProg(t, "testprog", "GoNil")
	want := "go of nil func value"
	if !strings.Contains(output, want) {
		t.Fatalf("output:\n%s\n\nwant output containing: %s", output, want)
	}
}

func TestMainGoroutineID(t *testing.T) {
	output := runTestProg(t, "testprog", "MainGoroutineID")
	want := "panic: test\n\ngoroutine 1 [running]:\n"
	if !strings.HasPrefix(output, want) {
		t.Fatalf("output does not start with %q:\n%s", want, output)
	}
}

func TestNoHelperGoroutines(t *testing.T) {
	output := runTestProg(t, "testprog", "NoHelperGoroutines")
	matches := regexp.MustCompile(`goroutine [0-9]+ \[`).FindAllStringSubmatch(output, -1)
	if len(matches) != 1 || matches[0][0] != "goroutine 1 [" {
		t.Fatalf("want to see only goroutine 1, see:\n%s", output)
	}
}

func TestBreakpoint(t *testing.T) {
	output := runTestProg(t, "testprog", "Breakpoint")
	want := "runtime.Breakpoint()"
	if !strings.Contains(output, want) {
		t.Fatalf("output:\n%s\n\nwant output containing: %s", output, want)
	}
}

const crashSource = `
package main

import (
	"fmt"
	"runtime"
)

{{if .Cgo}}
import "C"
{{end}}
`

func TestGoexitInPanic(t *testing.T) {
	// see issue 8774: this code used to trigger an infinite recursion
	output := runTestProg(t, "testprog", "GoexitInPanic")
	want := "fatal error: no goroutines (main called runtime.Goexit) - deadlock!"
	if !strings.HasPrefix(output, want) {
		t.Fatalf("output does not start with %q:\n%s", want, output)
	}
}

func TestPanicAfterGoexit(t *testing.T) {
	// an uncaught panic should still work after goexit
	output := runTestProg(t, "testprog", "PanicAfterGoexit")
	want := "panic: hello"
	if !strings.HasPrefix(output, want) {
		t.Fatalf("output does not start with %q:\n%s", want, output)
	}
}

func TestRecoveredPanicAfterGoexit(t *testing.T) {
	output := runTestProg(t, "testprog", "RecoveredPanicAfterGoexit")
	want := "fatal error: no goroutines (main called runtime.Goexit) - deadlock!"
	if !strings.HasPrefix(output, want) {
		t.Fatalf("output does not start with %q:\n%s", want, output)
	}
}

func TestRecoverBeforePanicAfterGoexit(t *testing.T) {
	// 1. defer a function that recovers
	// 2. defer a function that panics
	// 3. call goexit
	// Goexit should run the #2 defer.  Its panic
	// should be caught by the #1 defer, and execution
	// should resume in the caller.  Like the Goexit
	// never happened!
	defer func() {
		r := recover()
		if r == nil {
			panic("bad recover")
		}
	}()
	defer func() {
		panic("hello")
	}()
	runtime.Goexit()
}

func TestNetpollDeadlock(t *testing.T) {
	output := runTestProg(t, "testprognet", "NetpollDeadlock")
	want := "done\n"
	if !strings.HasSuffix(output, want) {
		t.Fatalf("output does not start with %q:\n%s", want, output)
	}
}

const netpollDeadlockSource = `
package main
import (
	"fmt"
	"net"
)
func init() {
	fmt.Println("dialing")
	c, err := net.Dial("tcp", "localhost:14356")
	if err == nil {
		c.Close()
	} else {
		fmt.Println("error: ", err)
	}
}
func main() {
	fmt.Println("done")
}
`
<commit_msg>runtime: remove now-unused test string constants<commit_after>// Copyright 2012 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package runtime_test

import (
	"fmt"
	"internal/testenv"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"runtime"
	"strings"
	"sync"
	"testing"
)

var toRemove []string

func TestMain(m *testing.M) {
	status := m.Run()
	for _, file := range toRemove {
		os.RemoveAll(file)
	}
	os.Exit(status)
}

func testEnv(cmd *exec.Cmd) *exec.Cmd {
	if cmd.Env != nil {
		panic("environment already set")
	}
	for _, env := range os.Environ() {
		// Exclude GODEBUG from the environment to prevent its output
		// from breaking tests that are trying to parse other command output.
		if strings.HasPrefix(env, "GODEBUG=") {
			continue
		}
		// Exclude GOTRACEBACK for the same reason.
		if strings.HasPrefix(env, "GOTRACEBACK=") {
			continue
		}
		cmd.Env = append(cmd.Env, env)
	}
	return cmd
}

var testprog struct {
	sync.Mutex
	dir    string
	target map[string]buildexe
}

type buildexe struct {
	exe string
	err error
}

func runTestProg(t *testing.T, binary, name string) string {
	testenv.MustHaveGoBuild(t)

	exe, err := buildTestProg(t, binary)
	if err != nil {
		t.Fatal(err)
	}
	got, _ := testEnv(exec.Command(exe, name)).CombinedOutput()
	return string(got)
}

func buildTestProg(t *testing.T, binary string) (string, error) {
	checkStaleRuntime(t)

	testprog.Lock()
	defer testprog.Unlock()
	if testprog.dir == "" {
		dir, err := ioutil.TempDir("", "go-build")
		if err != nil {
			t.Fatalf("failed to create temp directory: %v", err)
		}
		testprog.dir = dir
		toRemove = append(toRemove, dir)
	}

	if testprog.target == nil {
		testprog.target = make(map[string]buildexe)
	}
	target, ok := testprog.target[binary]
	if ok {
		return target.exe, target.err
	}

	exe := filepath.Join(testprog.dir, binary+".exe")
	cmd := exec.Command("go", "build", "-o", exe)
	cmd.Dir = "testdata/" + binary
	out, err := testEnv(cmd).CombinedOutput()
	if err != nil {
		exe = ""
		target.err = fmt.Errorf("building %s: %v\n%s", binary, err, out)
		testprog.target[binary] = target
		return "", err
	}
	target.exe = exe
	testprog.target[binary] = target
	return exe, nil
}

var (
	staleRuntimeOnce sync.Once // guards init of staleRuntimeErr
	staleRuntimeErr  error
)

func checkStaleRuntime(t *testing.T) {
	staleRuntimeOnce.Do(func() {
		// 'go run' uses the installed copy of runtime.a, which may be out of date.
		out, err := testEnv(exec.Command("go", "list", "-f", "{{.Stale}}", "runtime")).CombinedOutput()
		if err != nil {
			staleRuntimeErr = fmt.Errorf("failed to execute 'go list': %v\n%v", err, string(out))
			return
		}
		if string(out) != "false\n" {
			staleRuntimeErr = fmt.Errorf("Stale runtime.a. Run 'go install runtime'.")
		}
	})
	if staleRuntimeErr != nil {
		t.Fatal(staleRuntimeErr)
	}
}

func testCrashHandler(t *testing.T, cgo bool) {
	type crashTest struct {
		Cgo bool
	}
	var output string
	if cgo {
		output = runTestProg(t, "testprogcgo", "Crash")
	} else {
		output = runTestProg(t, "testprog", "Crash")
	}
	want := "main: recovered done\nnew-thread: recovered done\nsecond-new-thread: recovered done\nmain-again: recovered done\n"
	if output != want {
		t.Fatalf("output:\n%s\n\nwanted:\n%s", output, want)
	}
}

func TestCrashHandler(t *testing.T) {
	testCrashHandler(t, false)
}

func testDeadlock(t *testing.T, name string) {
	output := runTestProg(t, "testprog", name)
	want := "fatal error: all goroutines are asleep - deadlock!\n"
	if !strings.HasPrefix(output, want) {
		t.Fatalf("output does not start with %q:\n%s", want, output)
	}
}

func TestSimpleDeadlock(t *testing.T) {
	testDeadlock(t, "SimpleDeadlock")
}

func TestInitDeadlock(t *testing.T) {
	testDeadlock(t, "InitDeadlock")
}

func TestLockedDeadlock(t *testing.T) {
	testDeadlock(t, "LockedDeadlock")
}

func TestLockedDeadlock2(t *testing.T) {
	testDeadlock(t, "LockedDeadlock2")
}

func TestGoexitDeadlock(t *testing.T) {
	output := runTestProg(t, "testprog", "GoexitDeadlock")
	want := "no goroutines (main called runtime.Goexit) - deadlock!"
	if !strings.Contains(output, want) {
		t.Fatalf("output:\n%s\n\nwant output containing: %s", output, want)
	}
}

func TestStackOverflow(t *testing.T) {
	output := runTestProg(t, "testprog", "StackOverflow")
	want := "runtime: goroutine stack exceeds 1474560-byte limit\nfatal error: stack overflow"
	if !strings.HasPrefix(output, want) {
		t.Fatalf("output does not start with %q:\n%s", want, output)
	}
}

func TestThreadExhaustion(t *testing.T) {
	output := runTestProg(t, "testprog", "ThreadExhaustion")
	want := "runtime: program exceeds 10-thread limit\nfatal error: thread exhaustion"
	if !strings.HasPrefix(output, want) {
		t.Fatalf("output does not start with %q:\n%s", want, output)
	}
}

func TestRecursivePanic(t *testing.T) {
	output := runTestProg(t, "testprog", "RecursivePanic")
	want := `wrap: bad
panic: again

`
	if !strings.HasPrefix(output, want) {
		t.Fatalf("output does not start with %q:\n%s", want, output)
	}

}

func TestGoexitCrash(t *testing.T) {
	output := runTestProg(t, "testprog", "GoexitExit")
	want := "no goroutines (main called runtime.Goexit) - deadlock!"
	if !strings.Contains(output, want) {
		t.Fatalf("output:\n%s\n\nwant output containing: %s", output, want)
	}
}

func TestGoexitDefer(t *testing.T) {
	c := make(chan struct{})
	go func() {
		defer func() {
			r := recover()
			if r != nil {
				t.Errorf("non-nil recover during Goexit")
			}
			c <- struct{}{}
		}()
		runtime.Goexit()
	}()
	// Note: if the defer fails to run, we will get a deadlock here
	<-c
}

func TestGoNil(t *testing.T) {
	output := runTestProg(t, "testprog", "GoNil")
	want := "go of nil func value"
	if !strings.Contains(output, want) {
		t.Fatalf("output:\n%s\n\nwant output containing: %s", output, want)
	}
}

func TestMainGoroutineID(t *testing.T) {
	output := runTestProg(t, "testprog", "MainGoroutineID")
	want := "panic: test\n\ngoroutine 1 [running]:\n"
	if !strings.HasPrefix(output, want) {
		t.Fatalf("output does not start with %q:\n%s", want, output)
	}
}

func TestNoHelperGoroutines(t *testing.T) {
	output := runTestProg(t, "testprog", "NoHelperGoroutines")
	matches := regexp.MustCompile(`goroutine [0-9]+ \[`).FindAllStringSubmatch(output, -1)
	if len(matches) != 1 || matches[0][0] != "goroutine 1 [" {
		t.Fatalf("want to see only goroutine 1, see:\n%s", output)
	}
}

func TestBreakpoint(t *testing.T) {
	output := runTestProg(t, "testprog", "Breakpoint")
	want := "runtime.Breakpoint()"
	if !strings.Contains(output, want) {
		t.Fatalf("output:\n%s\n\nwant output containing: %s", output, want)
	}
}

func TestGoexitInPanic(t *testing.T) {
	// see issue 8774: this code used to trigger an infinite recursion
	output := runTestProg(t, "testprog", "GoexitInPanic")
	want := "fatal error: no goroutines (main called runtime.Goexit) - deadlock!"
	if !strings.HasPrefix(output, want) {
		t.Fatalf("output does not start with %q:\n%s", want, output)
	}
}

func TestPanicAfterGoexit(t *testing.T) {
	// an uncaught panic should still work after goexit
	output := runTestProg(t, "testprog", "PanicAfterGoexit")
	want := "panic: hello"
	if !strings.HasPrefix(output, want) {
		t.Fatalf("output does not start with %q:\n%s", want, output)
	}
}

func TestRecoveredPanicAfterGoexit(t *testing.T) {
	output := runTestProg(t, "testprog", "RecoveredPanicAfterGoexit")
	want := "fatal error: no goroutines (main called runtime.Goexit) - deadlock!"
	if !strings.HasPrefix(output, want) {
		t.Fatalf("output does not start with %q:\n%s", want, output)
	}
}

func TestRecoverBeforePanicAfterGoexit(t *testing.T) {
	// 1. defer a function that recovers
	// 2. defer a function that panics
	// 3. call goexit
	// Goexit should run the #2 defer.  Its panic
	// should be caught by the #1 defer, and execution
	// should resume in the caller.  Like the Goexit
	// never happened!
	defer func() {
		r := recover()
		if r == nil {
			panic("bad recover")
		}
	}()
	defer func() {
		panic("hello")
	}()
	runtime.Goexit()
}

func TestNetpollDeadlock(t *testing.T) {
	output := runTestProg(t, "testprognet", "NetpollDeadlock")
	want := "done\n"
	if !strings.HasSuffix(output, want) {
		t.Fatalf("output does not start with %q:\n%s", want, output)
	}
}
<|endoftext|>


<commit_before>//   Copyright 2019 MSolution.IO
//
//   Licensed under the Apache License, Version 2.0 (the "License");
//   you may not use this file except in compliance with the License.
//   You may obtain a copy of the License at
//
//       http://www.apache.org/licenses/LICENSE-2.0
//
//   Unless required by applicable law or agreed to in writing, software
//   distributed under the License is distributed on an "AS IS" BASIS,
//   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//   See the License for the specific language governing permissions and
//   limitations under the License.

package ec2

import (
	"regexp"
	"strconv"
	"strings"
)

type InstanceSize struct {
	factor float64
	size   string
	types  []string
}

var (
	instancesList = []InstanceSize{
		{1, "nano", []string{"t2", "t3", "t3a"}},
		{2, "micro", []string{"t1", "t2", "t3", "t3a"}},
		{4, "small", []string{"t2", "t3", "t3a", "m1"}},
		{8, "medium", []string{"t2", "t3", "t3a", "m1", "m3", "c1", "a1"}},
		{16, "large", []string{"t2", "t3", "t3a", "c5", "c4", "r4", "i3", "m1", "m3", "m4", "m5", "m5d", "m5a", "m5ad", "c3", "r3", "a1"}},
		{32, "xlarge", []string{"t2", "t3", "t3a", "c1", "c3", "c4", "c5", "p2", "x1e", "r2", "i3", "r4", "i3", "d2", "m1", "m2", "m3", "m4", "m5", "m5d", "m5a", "m5ad", "a1"}},
		{64, "2xlarge", []string{"t2", "t3", "t3a", "c3", "c4", "c5", "p3", "x1e", "i3", "h1", "d2", "m2", "m3", "m4", "m5", "m5d", "m5a", "m5ad", "g2", "r3", "r4", "i2", "a1"}},
		{128, "4xlarge", []string{"m2", "m4", "m5", "m5d", "m5a", "m5ad", "c3", "c4", "c5", "g3", "x1e", "r3", "r4", "i3", "h1", "d2", "i2", "a1"}},
		{256, "8xlarge", []string{"c4", "p2", "p3", "g3", "x1e", "r3", "r4", "i2", "i3", "h1", "d2", "cc2", "c3", "cr1", "g2", "hs1", "m5", "m5d", "m5a"}},
		{288, "9xlarge", []string{"c5"}},
		{320, "10xlarge", []string{"m4"}},
		{384, "12xlarge", []string{"m5", "m5d", "m5a", "m5ad"}},
		{512, "16xlarge", []string{"m4",  "m5", "m5d", "m5a", "p2", "p3", "g3", "x1", "x1e", "r4", "i3", "h1"}},
		{576, "18xlarge", []string{"c5"}},
		{768, "24xlarge", []string{"m5", "m5d", "m5a", "m5ad"}},
		{1024, "32xlarge", []string{"x1", "x1e"}},
	}

	rgx = regexp.MustCompile(`([a-zA-Z]+)([\\d])+`)
)

func getEC2RecommendationTypeReason(instance Instance) Recommendation {
	size, family := getInstanceSizeFamily(instance.Type)
	cpuDelta := instance.Stats.Cpu.Average / 0.80
	targetNormFactor := cpuDelta * getNormFactorFromSize(size)
	if instance.Stats.Cpu.Average <= 0 || targetNormFactor == 0 {
		return Recommendation{"", "", getNewGeneration(size, family)}
	}
	recommendedInstance := ""
	finalSize := ""
	var recommendedTemp string
	metaFamily := getSizesForType(family)
	for _, instanceSize := range metaFamily {
		if targetNormFactor <= instanceSize.factor {
			recommendedInstance = family + "." + instanceSize.size
			finalSize = instanceSize.size
			break
		}
		recommendedTemp = instanceSize.size
	}
	if recommendedInstance == instance.Type {
		return Recommendation{"", "", getNewGeneration(size, family)}
	} else if recommendedInstance == "" {
		if recommendedTemp == "" {
			return Recommendation{"", "", getNewGeneration(size, family)}
		}
		return Recommendation{
			InstanceType:  family + "." + recommendedTemp,
			Reason:        getEC2RecommendationReason(getNormFactorFromSize(size), getNormFactorFromSize(recommendedTemp)),
			NewGeneration: getNewGeneration(size, family)}
	}
	reason := getEC2RecommendationReason(getNormFactorFromSize(size), getNormFactorFromSize(finalSize))
	return Recommendation{
		InstanceType:  recommendedInstance,
		Reason:        reason,
		NewGeneration: getNewGeneration(size, family)}
}

func containEc2Type(idx int, family string) bool {
	for _, familyMeta := range instancesList[idx].types {
		if familyMeta == family {
			return true
		}
	}
	return false
}

func getInstanceSizeFamily(instanceType string) (size, family string) {
	sizeFamily := strings.Split(instanceType, ".")
	if len(sizeFamily) <= 0 {
		return
	}
	family = sizeFamily[0]
	if len(sizeFamily) > 1 {
		size = sizeFamily[1]
	}
	return
}

func getSizesForType(currentType string) []InstanceSize {
	size := make([]InstanceSize, 0)
	for idx, value := range instancesList {
		if containEc2Type(idx, currentType) {
			size = append(size, value)
		}
	}
	return size
}

func getNormFactorFromSize(size string) float64 {
	for _, instanceSize := range instancesList {
		if size == instanceSize.size {
			return instanceSize.factor
		}
	}
	return 0
}

func getEC2RecommendationReason(oldSize, newSize float64) string {
	if oldSize < newSize {
		return "High CPU usage"
	} else if oldSize > newSize {
		return "Low CPU usage"
	}
	return ""
}

func getNewGeneration(size, family string) string {
	for _, instanceSize := range instancesList {
		if instanceSize.size == size {
			if newgeneration, available := checkNewGenerationAvailable(size, family, instanceSize); available {
				return strings.Join(newgeneration, ",")
			}
			return ""
		}
	}
	return ""
}

func checkNewGenerationAvailable(size, family string, instanceSize InstanceSize) (recommendedType []string, available bool) {
	available = false
	actualType := rgx.FindStringSubmatch(family)
	if len(actualType) < 3 {
		return
	}
	actualGen, _ := strconv.Atoi(actualType[2])
	for _, instanceType := range instanceSize.types {
		newGenType := rgx.FindStringSubmatch(instanceType)
		newGen, _ := strconv.Atoi(newGenType[2])
		if len(newGenType) >= 3 && newGenType[1] == actualType[1] && actualGen <= newGen && actualType[0] != newGenType[0] {
			recommendedType = append(recommendedType, instanceType+"."+size)
			available = true
		}
	}
	return
}
<commit_msg>fix algorithm<commit_after>//   Copyright 2019 MSolution.IO
//
//   Licensed under the Apache License, Version 2.0 (the "License");
//   you may not use this file except in compliance with the License.
//   You may obtain a copy of the License at
//
//       http://www.apache.org/licenses/LICENSE-2.0
//
//   Unless required by applicable law or agreed to in writing, software
//   distributed under the License is distributed on an "AS IS" BASIS,
//   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//   See the License for the specific language governing permissions and
//   limitations under the License.

package ec2

import (
	"regexp"
	"strconv"
	"strings"
)

type InstanceSize struct {
	factor float64
	size   string
	types  []string
}

var (
	instancesList = []InstanceSize{
		{1, "nano", []string{"t2", "t3", "t3a"}},
		{2, "micro", []string{"t1", "t2", "t3", "t3a"}},
		{4, "small", []string{"t2", "t3", "t3a", "m1"}},
		{8, "medium", []string{"t2", "t3", "t3a", "m1", "m3", "c1", "a1"}},
		{16, "large", []string{"t2", "t3", "t3a", "c5", "c4", "r4", "i3", "m1", "m3", "m4", "m5", "m5d", "m5a", "m5ad", "c3", "r3", "a1"}},
		{32, "xlarge", []string{"t2", "t3", "t3a", "c1", "c3", "c4", "c5", "p2", "x1e", "r2", "i3", "r4", "i3", "d2", "m1", "m2", "m3", "m4", "m5", "m5d", "m5a", "m5ad", "a1"}},
		{64, "2xlarge", []string{"t2", "t3", "t3a", "c3", "c4", "c5", "p3", "x1e", "i3", "h1", "d2", "m2", "m3", "m4", "m5", "m5d", "m5a", "m5ad", "g2", "r3", "r4", "i2", "a1"}},
		{128, "4xlarge", []string{"m2", "m4", "m5", "m5d", "m5a", "m5ad", "c3", "c4", "c5", "g3", "x1e", "r3", "r4", "i3", "h1", "d2", "i2", "a1"}},
		{256, "8xlarge", []string{"c4", "p2", "p3", "g3", "x1e", "r3", "r4", "i2", "i3", "h1", "d2", "cc2", "c3", "cr1", "g2", "hs1", "m5", "m5d", "m5a"}},
		{288, "9xlarge", []string{"c5"}},
		{320, "10xlarge", []string{"m4"}},
		{384, "12xlarge", []string{"m5", "m5d", "m5a", "m5ad"}},
		{512, "16xlarge", []string{"m4",  "m5", "m5d", "m5a", "p2", "p3", "g3", "x1", "x1e", "r4", "i3", "h1"}},
		{576, "18xlarge", []string{"c5"}},
		{768, "24xlarge", []string{"m5", "m5d", "m5a", "m5ad"}},
		{1024, "32xlarge", []string{"x1", "x1e"}},
	}

	rgx = regexp.MustCompile(`([a-zA-Z]+)([0-9])+`)
)

func getEC2RecommendationTypeReason(instance Instance) Recommendation {
	size, family := getInstanceSizeFamily(instance.Type)
	cpuDelta := instance.Stats.Cpu.Average / 100 / 0.80
	targetNormFactor := cpuDelta * getNormFactorFromSize(size)
	if instance.Stats.Cpu.Average <= 0 || targetNormFactor == 0 {
		return Recommendation{"", "", getNewGeneration(size, family)}
	}
	recommendedInstance := ""
	finalSize := ""
	var recommendedTemp string
	metaFamily := getSizesForType(family)
	for _, instanceSize := range metaFamily {
		if targetNormFactor <= instanceSize.factor {
			recommendedInstance = family + "." + instanceSize.size
			finalSize = instanceSize.size
			break
		}
		recommendedTemp = instanceSize.size
	}
	if recommendedInstance == instance.Type {
		return Recommendation{"", "", getNewGeneration(size, family)}
	} else if recommendedInstance == "" {
		if recommendedTemp == "" {
			return Recommendation{"", "", getNewGeneration(size, family)}
		}
		return Recommendation{
			InstanceType:  family + "." + recommendedTemp,
			Reason:        getEC2RecommendationReason(getNormFactorFromSize(size), getNormFactorFromSize(recommendedTemp)),
			NewGeneration: getNewGeneration(size, family)}
	}
	reason := getEC2RecommendationReason(getNormFactorFromSize(size), getNormFactorFromSize(finalSize))
	return Recommendation{
		InstanceType:  recommendedInstance,
		Reason:        reason,
		NewGeneration: getNewGeneration(size, family)}
}

func containEc2Type(idx int, family string) bool {
	for _, familyMeta := range instancesList[idx].types {
		if familyMeta == family {
			return true
		}
	}
	return false
}

func getInstanceSizeFamily(instanceType string) (size, family string) {
	sizeFamily := strings.Split(instanceType, ".")
	if len(sizeFamily) <= 0 {
		return
	}
	family = sizeFamily[0]
	if len(sizeFamily) > 1 {
		size = sizeFamily[1]
	}
	return
}

func getSizesForType(currentType string) []InstanceSize {
	size := make([]InstanceSize, 0)
	for idx, value := range instancesList {
		if containEc2Type(idx, currentType) {
			size = append(size, value)
		}
	}
	return size
}

func getNormFactorFromSize(size string) float64 {
	for _, instanceSize := range instancesList {
		if size == instanceSize.size {
			return instanceSize.factor
		}
	}
	return 0
}

func getEC2RecommendationReason(oldSize, newSize float64) string {
	if oldSize < newSize {
		return "High CPU usage"
	} else if oldSize > newSize {
		return "Low CPU usage"
	}
	return ""
}

func getNewGeneration(size, family string) string {
	for _, instanceSize := range instancesList {
		if instanceSize.size == size {
			if newgeneration, available := checkNewGenerationAvailable(size, family, instanceSize); available {
				return strings.Join(newgeneration, ",")
			}
			return ""
		}
	}
	return ""
}

func checkNewGenerationAvailable(size, family string, instanceSize InstanceSize) (recommendedType []string, available bool) {
	available = false
	actualType := rgx.FindStringSubmatch(family)
	if len(actualType) < 3 {
		return
	}
	actualGen, _ := strconv.Atoi(actualType[2])
	for _, instanceType := range instanceSize.types {
		newGenType := rgx.FindStringSubmatch(instanceType)
		newGen, _ := strconv.Atoi(newGenType[2])
		if len(newGenType) >= 3 && newGenType[1] == actualType[1] && actualGen <= newGen && actualType[0] != newGenType[0] {
			recommendedType = append(recommendedType, instanceType+"."+size)
			available = true
		}
	}
	return
}
<|endoftext|>


<commit_before>package commands

import cmds "github.com/ipfs/go-ipfs/commands"

type IpnsEntry struct {
	Name  string
	Value string
}

var NameCmd = &cmds.Command{
	Helptext: cmds.HelpText{
		Tagline: "IPFS namespace (IPNS) tool",
		Synopsis: `
ipfs name publish [<name>] <ipfs-path> - Publish an object to IPNS
ipfs name resolve [<name>]             - Gets the value currently published at an IPNS name
`,
		ShortDescription: `
IPNS is a PKI namespace, where names are the hashes of public keys, and
the private key enables publishing new (signed) values. In both publish
and resolve, the default value of <name> is your own identity public key.
`,
		LongDescription: `
IPNS is a PKI namespace, where names are the hashes of public keys, and
the private key enables publishing new (signed) values. In both publish
and resolve, the default value of <name> is your own identity public key.

Examples:

Publish an <ipfs-path> to your identity name:

  > ipfs name publish /ipfs/QmatmE9msSfkKxoffpHwNLNKgwZG8eT9Bud6YoPab52vpy
  Published to QmbCMUZw6JFeZ7Wp9jkzbye3Fzp2GGcPgC3nmeUjfVF87n: /ipfs/QmatmE9msSfkKxoffpHwNLNKgwZG8eT9Bud6YoPab52vpy

Publish an <ipfs-path> to another public key:

  > ipfs name publish /ipfs/QmatmE9msSfkKxoffpHwNLNKgwZG8eT9Bud6YoPab52vpy QmbCMUZw6JFeZ7Wp9jkzbye3Fzp2GGcPgC3nmeUjfVF87n
  Published to QmbCMUZw6JFeZ7Wp9jkzbye3Fzp2GGcPgC3nmeUjfVF87n: /ipfs/QmatmE9msSfkKxoffpHwNLNKgwZG8eT9Bud6YoPab52vpy

Resolve the value of your identity:

  > ipfs name resolve
  /ipns/QmatmE9msSfkKxoffpHwNLNKgwZG8eT9Bud6YoPab52vpy

Resolve the value of another name:

  > ipfs name resolve QmbCMUZw6JFeZ7Wp9jkzbye3Fzp2GGcPgC3nmeUjfVF87n
  /ipns/QmatmE9msSfkKxoffpHwNLNKgwZG8eT9Bud6YoPab52vpy

`,
	},

	Subcommands: map[string]*cmds.Command{
		"publish": publishCmd,
		"resolve": ipnsCmd,
	},
}
<commit_msg>Text under `ipfs name --help` incorrect<commit_after>package commands

import cmds "github.com/ipfs/go-ipfs/commands"

type IpnsEntry struct {
	Name  string
	Value string
}

var NameCmd = &cmds.Command{
	Helptext: cmds.HelpText{
		Tagline: "IPFS namespace (IPNS) tool",
		Synopsis: `
ipfs name publish [<name>] <ipfs-path> - Publish an object to IPNS
ipfs name resolve [<name>]             - Gets the value currently published at an IPNS name
`,
		ShortDescription: `
IPNS is a PKI namespace, where names are the hashes of public keys, and
the private key enables publishing new (signed) values. In both publish
and resolve, the default value of <name> is your own identity public key.
`,
		LongDescription: `
IPNS is a PKI namespace, where names are the hashes of public keys, and
the private key enables publishing new (signed) values. In both publish
and resolve, the default value of <name> is your own identity public key.

Examples:

Publish an <ipfs-path> to your identity name:

  > ipfs name publish /ipfs/QmatmE9msSfkKxoffpHwNLNKgwZG8eT9Bud6YoPab52vpy
  Published to QmbCMUZw6JFeZ7Wp9jkzbye3Fzp2GGcPgC3nmeUjfVF87n: /ipfs/QmatmE9msSfkKxoffpHwNLNKgwZG8eT9Bud6YoPab52vpy

Publish an <ipfs-path> to another public key:

  > ipfs name publish /ipfs/QmatmE9msSfkKxoffpHwNLNKgwZG8eT9Bud6YoPab52vpy QmbCMUZw6JFeZ7Wp9jkzbye3Fzp2GGcPgC3nmeUjfVF87n
  Published to QmbCMUZw6JFeZ7Wp9jkzbye3Fzp2GGcPgC3nmeUjfVF87n: /ipfs/QmatmE9msSfkKxoffpHwNLNKgwZG8eT9Bud6YoPab52vpy

Resolve the value of your identity:

  > ipfs name resolve
  /ipfs/QmatmE9msSfkKxoffpHwNLNKgwZG8eT9Bud6YoPab52vpy

Resolve the value of another name:

  > ipfs name resolve QmbCMUZw6JFeZ7Wp9jkzbye3Fzp2GGcPgC3nmeUjfVF87n
  /ipfs/QmatmE9msSfkKxoffpHwNLNKgwZG8eT9Bud6YoPab52vpy

`,
	},

	Subcommands: map[string]*cmds.Command{
		"publish": publishCmd,
		"resolve": ipnsCmd,
	},
}
<|endoftext|>


<commit_before>package qshell

import (
	"bufio"
	"bytes"
	"errors"
	"fmt"
	"github.com/qiniu/api/auth/digest"
	"github.com/qiniu/api/rs"
	"github.com/qiniu/log"
	"io/ioutil"
	"net/http"
	"net/url"
	"strings"
)

type BucketDomain []string

func M3u8FileList(mac *digest.Mac, bucket string, m3u8Key string, isPrivate bool) (slicesToDelete []rs.EntryPath, err error) {
	client := rs.New(mac)
	//check m3u8 file exists
	_, sErr := client.Stat(nil, bucket, m3u8Key)
	if sErr != nil {
		err = errors.New(fmt.Sprintf("stat m3u8 file error, %s", sErr.Error()))
		return
	}
	//get domain list of bucket
	bucketDomainUrl := "http://api.qiniu.com/v6/domain/list"
	bucketDomainData := map[string][]string{
		"tbl": []string{bucket},
	}
	bucketDomains := BucketDomain{}
	bErr := client.Conn.CallWithForm(nil, &bucketDomains, bucketDomainUrl, bucketDomainData)
	if bErr != nil {
		err = errors.New(fmt.Sprintf("get domain of bucket failed due to, %s", bErr.Error()))
		return
	}
	if len(bucketDomains) == 0 {
		err = errors.New("no domain found for the bucket")
		return
	}
	var domain string
	for _, d := range bucketDomains {
		if strings.HasSuffix(d, "qiniudn.com") ||
			strings.HasSuffix(d, "clouddn.com") {
			domain = d
			break
		}
	}
	if domain == "" {
		err = errors.New("no valid domain found for the bucket")
		return
	}
	//create downoad link
	dnLink := fmt.Sprintf("http://%s/%s", domain, m3u8Key)
	if isPrivate {
		dnLink = PrivateUrl(mac, dnLink, 3600)
	}
	//get m3u8 file content
	m3u8Resp, m3u8Err := http.Get(dnLink)
	if m3u8Err != nil {
		err = errors.New(fmt.Sprintf("open url %s error due to, %s", dnLink, m3u8Err))
		return
	}
	defer m3u8Resp.Body.Close()
	if m3u8Resp.StatusCode != 200 {
		err = errors.New(fmt.Sprintf("download file error due to, %s", m3u8Resp.Status))
		return
	}
	m3u8Bytes, readErr := ioutil.ReadAll(m3u8Resp.Body)
	if readErr != nil {
		err = errors.New(fmt.Sprintf("read m3u8 file content error due to, %s", readErr.Error()))
		return
	}
	//check content
	if !strings.HasPrefix(string(m3u8Bytes), "#EXTM3U") {
		err = errors.New("invalid m3u8 file")
		return
	}
	slicesToDelete = make([]rs.EntryPath, 0)
	bReader := bufio.NewScanner(bytes.NewReader(m3u8Bytes))
	bReader.Split(bufio.ScanLines)
	for bReader.Scan() {
		line := strings.TrimSpace(bReader.Text())
		if !strings.HasPrefix(line, "#") {
			var sliceKey string
			if strings.HasPrefix(line, "http://") ||
				strings.HasPrefix(line, "https://") {
				uri, pErr := url.Parse(line)
				if pErr != nil {
					log.Error(fmt.Sprintf("invalid url, %s", line))
					continue
				}
				sliceKey = strings.TrimPrefix(uri.Path, "/")
			} else {
				sliceKey = strings.TrimPrefix(line, "/")
			}
			//append to delete list
			slicesToDelete = append(slicesToDelete, rs.EntryPath{bucket, sliceKey})
		}
	}
	slicesToDelete = append(slicesToDelete, rs.EntryPath{bucket, m3u8Key})
	return
}
<commit_msg>Fix the bug in m3u8delete, set error deadline for private download link.<commit_after>package qshell

import (
	"bufio"
	"bytes"
	"errors"
	"fmt"
	"github.com/qiniu/api/auth/digest"
	"github.com/qiniu/api/rs"
	"github.com/qiniu/log"
	"io/ioutil"
	"net/http"
	"net/url"
	"strings"
	"time"
)

type BucketDomain []string

func M3u8FileList(mac *digest.Mac, bucket string, m3u8Key string, isPrivate bool) (slicesToDelete []rs.EntryPath, err error) {
	client := rs.New(mac)
	//check m3u8 file exists
	_, sErr := client.Stat(nil, bucket, m3u8Key)
	if sErr != nil {
		err = errors.New(fmt.Sprintf("stat m3u8 file error, %s", sErr.Error()))
		return
	}
	//get domain list of bucket
	bucketDomainUrl := "http://api.qiniu.com/v6/domain/list"
	bucketDomainData := map[string][]string{
		"tbl": []string{bucket},
	}
	bucketDomains := BucketDomain{}
	bErr := client.Conn.CallWithForm(nil, &bucketDomains, bucketDomainUrl, bucketDomainData)
	if bErr != nil {
		err = errors.New(fmt.Sprintf("get domain of bucket failed due to, %s", bErr.Error()))
		return
	}
	if len(bucketDomains) == 0 {
		err = errors.New("no domain found for the bucket")
		return
	}
	var domain string
	for _, d := range bucketDomains {
		if strings.HasSuffix(d, "qiniudn.com") ||
			strings.HasSuffix(d, "clouddn.com") {
			domain = d
			break
		}
	}
	if domain == "" {
		err = errors.New("no valid domain found for the bucket")
		return
	}
	//create downoad link
	dnLink := fmt.Sprintf("http://%s/%s", domain, m3u8Key)
	if isPrivate {
		dnLink = PrivateUrl(mac, dnLink, time.Now().Add(time.Second*3600).Unix())
	}
	//get m3u8 file content
	m3u8Resp, m3u8Err := http.Get(dnLink)
	if m3u8Err != nil {
		err = errors.New(fmt.Sprintf("open url %s error due to, %s", dnLink, m3u8Err))
		return
	}
	defer m3u8Resp.Body.Close()
	if m3u8Resp.StatusCode != 200 {
		err = errors.New(fmt.Sprintf("download file error due to, %s", m3u8Resp.Status))
		return
	}
	m3u8Bytes, readErr := ioutil.ReadAll(m3u8Resp.Body)
	if readErr != nil {
		err = errors.New(fmt.Sprintf("read m3u8 file content error due to, %s", readErr.Error()))
		return
	}
	//check content
	if !strings.HasPrefix(string(m3u8Bytes), "#EXTM3U") {
		err = errors.New("invalid m3u8 file")
		return
	}
	slicesToDelete = make([]rs.EntryPath, 0)
	bReader := bufio.NewScanner(bytes.NewReader(m3u8Bytes))
	bReader.Split(bufio.ScanLines)
	for bReader.Scan() {
		line := strings.TrimSpace(bReader.Text())
		if !strings.HasPrefix(line, "#") {
			var sliceKey string
			if strings.HasPrefix(line, "http://") ||
				strings.HasPrefix(line, "https://") {
				uri, pErr := url.Parse(line)
				if pErr != nil {
					log.Error(fmt.Sprintf("invalid url, %s", line))
					continue
				}
				sliceKey = strings.TrimPrefix(uri.Path, "/")
			} else {
				sliceKey = strings.TrimPrefix(line, "/")
			}
			//append to delete list
			slicesToDelete = append(slicesToDelete, rs.EntryPath{bucket, sliceKey})
		}
	}
	slicesToDelete = append(slicesToDelete, rs.EntryPath{bucket, m3u8Key})
	return
}
<|endoftext|>


<commit_before>package qshell

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"github.com/astaxie/beego/logs"
	"io"
	"net/http"
	"os"
	"path/filepath"
	"qiniu/api.v6/auth/digest"
	rio "qiniu/api.v6/resumable/io"
	"qiniu/api.v6/rs"
	"qiniu/rpc"
	"strconv"
	"strings"
	"time"
)

//range get and chunk upload

const (
	RETRY_MAX_TIMES = 5
	RETRY_INTERVAL  = time.Second * 1
	HTTP_TIMEOUT    = time.Second * 10
)

type PutRet struct {
	Key      string `json:"key"`
	Hash     string `json:"hash"`
	MimeType string `json:"mimeType"`
	Fsize    int64  `json:"fsize"`
}

type SyncProgress struct {
	BlkCtxs   []rio.BlkputRet `json:"blk_ctxs"`
	Offset    int64           `json:"offset"`
	TotalSize int64           `json:"total_size"`
}

func Sync(mac *digest.Mac, srcResUrl, bucket, key, upHostIp string) (putRet PutRet, err error) {
	if exists, cErr := checkExists(mac, bucket, key); cErr != nil {
		err = cErr
		return
	} else if exists {
		err = errors.New("File with same key` already exists in bucket")
		return
	}

	syncProgress := SyncProgress{}
	//create sync id
	syncId := Md5Hex(fmt.Sprintf("%s:%s:%s", srcResUrl, bucket, key))

	//local storage path
	storePath := filepath.Join(QShellRootPath, ".qshell", "sync")
	if mkdirErr := os.MkdirAll(storePath, 0775); mkdirErr != nil {
		logs.Error("Failed to mkdir `%s` due to `%s`", storePath, mkdirErr)
		return
	}

	progressFile := filepath.Join(storePath, fmt.Sprintf("%s.progress", syncId))
	if statInfo, statErr := os.Stat(progressFile); statErr == nil {
		//check file last modified time, if older than one week, ignore
		if statInfo.ModTime().Add(time.Hour * 24 * 5).After(time.Now()) {
			//try read old progress
			progressFh, openErr := os.Open(progressFile)
			if openErr == nil {
				decoder := json.NewDecoder(progressFh)
				decoder.Decode(&syncProgress)
				progressFh.Close()
			}
		}
	}

	//check offset valid or not
	if syncProgress.Offset%BLOCK_SIZE != 0 {
		logs.Info("Invalid offset from progress file,", syncProgress.Offset)
		syncProgress.Offset = 0
		syncProgress.TotalSize = 0
		syncProgress.BlkCtxs = make([]rio.BlkputRet, 0)
	}

	//check offset and blk ctxs
	if syncProgress.Offset != 0 && syncProgress.BlkCtxs != nil {
		if int(syncProgress.Offset/BLOCK_SIZE) != len(syncProgress.BlkCtxs) {
			logs.Info("Invalid offset and block contexts")
			syncProgress.Offset = 0
			syncProgress.TotalSize = 0
			syncProgress.BlkCtxs = make([]rio.BlkputRet, 0)
		}
	}

	//check blk ctxs, when no progress found
	if syncProgress.Offset == 0 || syncProgress.BlkCtxs == nil {
		syncProgress.Offset = 0
		syncProgress.TotalSize = 0
		syncProgress.BlkCtxs = make([]rio.BlkputRet, 0)
	}

	//get total size
	totalSize, hErr := getRemoteFileLength(srcResUrl)
	if hErr != nil {
		err = hErr
		return
	}

	if totalSize != syncProgress.TotalSize {
		if syncProgress.TotalSize != 0 {
			logs.Warning("Remote file length changed, progress file out of date")
		}
		syncProgress.Offset = 0
		syncProgress.TotalSize = totalSize
		syncProgress.BlkCtxs = make([]rio.BlkputRet, 0)
	}

	//get total block count
	totalBlkCnt := 0
	if totalSize%BLOCK_SIZE == 0 {
		totalBlkCnt = int(totalSize / BLOCK_SIZE)
	} else {
		totalBlkCnt = int(totalSize/BLOCK_SIZE) + 1
	}

	//init the range offset
	rangeStartOffset := syncProgress.Offset
	fromBlkIndex := int(rangeStartOffset / BLOCK_SIZE)

	lastBlock := false

	//create upload token
	policy := rs.PutPolicy{Scope: bucket}
	//token is valid for one year
	policy.Expires = 3600 * 24 * 365
	policy.ReturnBody = `{"key":"$(key)","hash":"$(etag)","fsize":$(fsize),"mimeType":"$(mimeType)"}`
	uptoken := policy.Token(mac)
	putClient := rio.NewClient(uptoken, upHostIp)

	//range get and mkblk upload
	for blkIndex := fromBlkIndex; blkIndex < totalBlkCnt; blkIndex++ {
		if blkIndex == totalBlkCnt-1 {
			lastBlock = true
		}

		syncPercent := fmt.Sprintf("%.2f", float64(blkIndex+1)*100.0/float64(totalBlkCnt))
		logs.Info("Syncing block %d [%s] ...", blkIndex, syncPercent)
		blkCtx, pErr := rangeMkblkPipe(srcResUrl, rangeStartOffset, BLOCK_SIZE, lastBlock, putClient)
		if pErr != nil {
			logs.Error(pErr.Error())
			time.Sleep(RETRY_INTERVAL)

			for retryTimes := 1; retryTimes <= RETRY_MAX_TIMES; retryTimes++ {
				logs.Info("Retrying %d time range & mkblk block [%d]", retryTimes, blkIndex)
				blkCtx, pErr = rangeMkblkPipe(srcResUrl, rangeStartOffset, BLOCK_SIZE, lastBlock, putClient)
				if pErr != nil {
					logs.Error(pErr)
					//wait a interval and retry
					time.Sleep(RETRY_INTERVAL)
					continue
				} else {
					break
				}
			}
		}

		if pErr != nil {
			err = errors.New("Max retry reached and range & mkblk still failed, check your network")
			return
		}

		//advance range offset
		rangeStartOffset += BLOCK_SIZE

		syncProgress.BlkCtxs = append(syncProgress.BlkCtxs, blkCtx)
		syncProgress.Offset = rangeStartOffset

		rErr := recordProgress(progressFile, syncProgress)
		if rErr != nil {
			logs.Info(rErr.Error())
		}
	}

	//make file
	putExtra := rio.PutExtra{
		Progresses: syncProgress.BlkCtxs,
	}
	mkErr := rio.Mkfile(putClient, nil, &putRet, key, true, totalSize, &putExtra)
	if mkErr != nil {
		err = fmt.Errorf("Mkfile error, %s", mkErr.Error())
		return
	}

	//delete progress file
	os.Remove(progressFile)

	return
}

func rangeMkblkPipe(srcResUrl string, rangeStartOffset int64, rangeBlockSize int64, lastBlock bool,
	putClient rpc.Client) (putRet rio.BlkputRet, err error) {
	//range get
	dReq, dReqErr := http.NewRequest("GET", srcResUrl, nil)
	if dReqErr != nil {
		err = fmt.Errorf("New request error, %s", dReqErr.Error())
		return
	}

	//set range header
	rangeEndOffset := rangeStartOffset + rangeBlockSize - 1
	dReq.Header.Set("Range", fmt.Sprintf("bytes=%d-%d", rangeStartOffset, rangeEndOffset))

	//get resp
	client := http.DefaultClient
	client.Timeout = time.Duration(HTTP_TIMEOUT)
	dResp, dRespErr := client.Do(dReq)
	if dRespErr != nil {
		err = fmt.Errorf("Get response error, %s", dRespErr.Error())
		return
	}
	defer dResp.Body.Close()

	//status error
	if dResp.StatusCode/100 != 2 {
		err = fmt.Errorf("Get resource error, %s", dResp.Status)
		return
	}

	//if not support range, go back and err
	if dResp.Header.Get("Accept-Ranges") == "" {
		err = errors.New("Remote server not support range")
		return
	}

	//parse content-range
	contentRange := dResp.Header.Get("Content-Range")
	rangeSize, _ := parseContentRange(contentRange)

	//check ranged block size
	if !lastBlock && rangeSize != rangeBlockSize {
		err = errors.New("Block read error, only the last range block can has bytes less than <RangeBlockSize>")
		return
	}

	//read content
	buffer := bytes.NewBuffer(nil)
	cpCnt, cpErr := io.Copy(buffer, dResp.Body)
	if cpErr != nil || cpCnt != rangeSize {
		err = errors.New("Read range block response error, not fully read")
		return
	}

	//mkblk
	blkPutRet := rio.BlkputRet{}
	blockSize := int(rangeSize)
	blockDataReader := bytes.NewReader(buffer.Bytes())
	blockDataSize := buffer.Len()

	mkErr := rio.Mkblock(putClient, nil, &blkPutRet, blockSize, blockDataReader, blockDataSize)
	if mkErr != nil {
		err = fmt.Errorf("Mkblk error, %s", mkErr.Error())
		return
	}

	putRet = blkPutRet

	return
}

func recordProgress(progressFile string, syncProgress SyncProgress) (err error) {
	fh, openErr := os.Create(progressFile)
	if openErr != nil {
		err = fmt.Errorf("Open progress file %s error, %s", progressFile, openErr.Error())
		return
	}
	defer fh.Close()

	jsonBytes, mErr := json.Marshal(&syncProgress)
	if mErr != nil {
		err = fmt.Errorf("Marshal sync progress error, %s", mErr.Error())
		return
	}

	_, wErr := fh.Write(jsonBytes)
	if wErr != nil {
		err = fmt.Errorf("Write sync progress error, %s", wErr.Error())
	}

	return
}

func getRemoteFileLength(srcResUrl string) (totalSize int64, err error) {
	resp, respErr := http.Head(srcResUrl)
	if respErr != nil {
		err = fmt.Errorf("New head request failed, %s", respErr.Error())
		return
	}
	defer resp.Body.Close()

	contentLength := resp.Header.Get("Content-Length")
	if contentLength == "" {
		err = errors.New("Head request with no Content-Length found error")
		return
	}

	totalSize, _ = strconv.ParseInt(contentLength, 10, 64)

	return
}

func checkExists(mac *digest.Mac, bucket, key string) (exists bool, err error) {
	client := rs.NewMac(mac)
	entry, sErr := client.Stat(nil, bucket, key)
	if sErr != nil {
		if v, ok := sErr.(*rpc.ErrorInfo); !ok {
			err = fmt.Errorf("Check file exists error, %s", sErr.Error())
			return
		} else {
			if v.Code != 612 {
				err = fmt.Errorf("Check file exists error, %s", v.Err)
				return
			} else {
				exists = false
				return
			}
		}
	}

	if entry.Hash != "" {
		exists = true
	}

	return
}

//Content-Range: bytes 25538640-25538647/25538648
func parseContentRange(contentRange string) (rangeSize, totalSize int64) {
	contentRangeItems := strings.Split(contentRange, " ")
	sizeItems := strings.Split(contentRangeItems[1], "/")

	rangePartItems := strings.Split(sizeItems[0], "-")
	totalSize, _ = strconv.ParseInt(sizeItems[1], 10, 64)

	fromOffset, _ := strconv.ParseInt(rangePartItems[0], 10, 64)
	toOffset, _ := strconv.ParseInt(rangePartItems[1], 10, 64)

	rangeSize = toOffset - fromOffset + 1

	return
}
<commit_msg>[ISSUE-99] fix the bug of sync command not support url with 302 redirect<commit_after>package qshell

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"github.com/astaxie/beego/logs"
	"io"
	"net/http"
	"os"
	"path/filepath"
	"qiniu/api.v6/auth/digest"
	rio "qiniu/api.v6/resumable/io"
	"qiniu/api.v6/rs"
	"qiniu/rpc"
	"strconv"
	"strings"
	"time"
)

//range get and chunk upload

const (
	RETRY_MAX_TIMES = 5
	RETRY_INTERVAL  = time.Second * 1
	HTTP_TIMEOUT    = time.Second * 10
)

type PutRet struct {
	Key      string `json:"key"`
	Hash     string `json:"hash"`
	MimeType string `json:"mimeType"`
	Fsize    int64  `json:"fsize"`
}

type SyncProgress struct {
	BlkCtxs   []rio.BlkputRet `json:"blk_ctxs"`
	Offset    int64           `json:"offset"`
	TotalSize int64           `json:"total_size"`
}

func Sync(mac *digest.Mac, srcResUrl, bucket, key, upHostIp string) (putRet PutRet, err error) {
	if exists, cErr := checkExists(mac, bucket, key); cErr != nil {
		err = cErr
		return
	} else if exists {
		err = errors.New("File with same key` already exists in bucket")
		return
	}

	syncProgress := SyncProgress{}
	//create sync id
	syncId := Md5Hex(fmt.Sprintf("%s:%s:%s", srcResUrl, bucket, key))

	//local storage path
	storePath := filepath.Join(QShellRootPath, ".qshell", "sync")
	if mkdirErr := os.MkdirAll(storePath, 0775); mkdirErr != nil {
		logs.Error("Failed to mkdir `%s` due to `%s`", storePath, mkdirErr)
		return
	}

	progressFile := filepath.Join(storePath, fmt.Sprintf("%s.progress", syncId))
	if statInfo, statErr := os.Stat(progressFile); statErr == nil {
		//check file last modified time, if older than one week, ignore
		if statInfo.ModTime().Add(time.Hour * 24 * 5).After(time.Now()) {
			//try read old progress
			progressFh, openErr := os.Open(progressFile)
			if openErr == nil {
				decoder := json.NewDecoder(progressFh)
				decoder.Decode(&syncProgress)
				progressFh.Close()
			}
		}
	}

	//check offset valid or not
	if syncProgress.Offset%BLOCK_SIZE != 0 {
		logs.Info("Invalid offset from progress file,", syncProgress.Offset)
		syncProgress.Offset = 0
		syncProgress.TotalSize = 0
		syncProgress.BlkCtxs = make([]rio.BlkputRet, 0)
	}

	//check offset and blk ctxs
	if syncProgress.Offset != 0 && syncProgress.BlkCtxs != nil {
		if int(syncProgress.Offset/BLOCK_SIZE) != len(syncProgress.BlkCtxs) {
			logs.Info("Invalid offset and block contexts")
			syncProgress.Offset = 0
			syncProgress.TotalSize = 0
			syncProgress.BlkCtxs = make([]rio.BlkputRet, 0)
		}
	}

	//check blk ctxs, when no progress found
	if syncProgress.Offset == 0 || syncProgress.BlkCtxs == nil {
		syncProgress.Offset = 0
		syncProgress.TotalSize = 0
		syncProgress.BlkCtxs = make([]rio.BlkputRet, 0)
	}

	//get total size
	totalSize, hErr := getRemoteFileLength(srcResUrl)
	if hErr != nil {
		err = hErr
		return
	}

	if totalSize != syncProgress.TotalSize {
		if syncProgress.TotalSize != 0 {
			logs.Warning("Remote file length changed, progress file out of date")
		}
		syncProgress.Offset = 0
		syncProgress.TotalSize = totalSize
		syncProgress.BlkCtxs = make([]rio.BlkputRet, 0)
	}

	//get total block count
	totalBlkCnt := 0
	if totalSize%BLOCK_SIZE == 0 {
		totalBlkCnt = int(totalSize / BLOCK_SIZE)
	} else {
		totalBlkCnt = int(totalSize/BLOCK_SIZE) + 1
	}

	//init the range offset
	rangeStartOffset := syncProgress.Offset
	fromBlkIndex := int(rangeStartOffset / BLOCK_SIZE)

	lastBlock := false

	//create upload token
	policy := rs.PutPolicy{Scope: bucket}
	//token is valid for one year
	policy.Expires = 3600 * 24 * 365
	policy.ReturnBody = `{"key":"$(key)","hash":"$(etag)","fsize":$(fsize),"mimeType":"$(mimeType)"}`
	uptoken := policy.Token(mac)
	putClient := rio.NewClient(uptoken, upHostIp)

	//range get and mkblk upload
	for blkIndex := fromBlkIndex; blkIndex < totalBlkCnt; blkIndex++ {
		if blkIndex == totalBlkCnt-1 {
			lastBlock = true
		}

		syncPercent := fmt.Sprintf("%.2f", float64(blkIndex+1)*100.0/float64(totalBlkCnt))
		logs.Info("Syncing block %d [%s] ...", blkIndex, syncPercent)
		blkCtx, pErr := rangeMkblkPipe(srcResUrl, rangeStartOffset, BLOCK_SIZE, lastBlock, putClient)
		if pErr != nil {
			logs.Error(pErr.Error())
			time.Sleep(RETRY_INTERVAL)

			for retryTimes := 1; retryTimes <= RETRY_MAX_TIMES; retryTimes++ {
				logs.Info("Retrying %d time range & mkblk block [%d]", retryTimes, blkIndex)
				blkCtx, pErr = rangeMkblkPipe(srcResUrl, rangeStartOffset, BLOCK_SIZE, lastBlock, putClient)
				if pErr != nil {
					logs.Error(pErr)
					//wait a interval and retry
					time.Sleep(RETRY_INTERVAL)
					continue
				} else {
					break
				}
			}
		}

		if pErr != nil {
			err = errors.New("Max retry reached and range & mkblk still failed, check your network")
			return
		}

		//advance range offset
		rangeStartOffset += BLOCK_SIZE

		syncProgress.BlkCtxs = append(syncProgress.BlkCtxs, blkCtx)
		syncProgress.Offset = rangeStartOffset

		rErr := recordProgress(progressFile, syncProgress)
		if rErr != nil {
			logs.Info(rErr.Error())
		}
	}

	//make file
	putExtra := rio.PutExtra{
		Progresses: syncProgress.BlkCtxs,
	}
	mkErr := rio.Mkfile(putClient, nil, &putRet, key, true, totalSize, &putExtra)
	if mkErr != nil {
		err = fmt.Errorf("Mkfile error, %s", mkErr.Error())
		return
	}

	//delete progress file
	os.Remove(progressFile)

	return
}

func rangeMkblkPipe(srcResUrl string, rangeStartOffset int64, rangeBlockSize int64, lastBlock bool,
	putClient rpc.Client) (putRet rio.BlkputRet, err error) {
	//range get
	dReq, dReqErr := http.NewRequest("GET", srcResUrl, nil)
	if dReqErr != nil {
		err = fmt.Errorf("New request error, %s", dReqErr.Error())
		return
	}

	//proxyURL, _ := url.Parse("http://localhost:8888")

	//set range header
	rangeEndOffset := rangeStartOffset + rangeBlockSize - 1
	dReq.Header.Add("Range", fmt.Sprintf("bytes=%d-%d", rangeStartOffset, rangeEndOffset))

	//set client properties
	client := http.DefaultClient
	client.Timeout = time.Duration(HTTP_TIMEOUT)
	//client.Transport = &http.Transport{
	//	Proxy: http.ProxyURL(proxyURL),
	//}

	client.CheckRedirect = func(rReq *http.Request, rVias []*http.Request) (err error) {
		rReq.Header.Add("Range", dReq.Header.Get("Range"))
		return nil
	}

	//get response
	dResp, dRespErr := client.Do(dReq)
	if dRespErr != nil {
		err = fmt.Errorf("Get response error, %s", dRespErr.Error())
		return
	}
	defer dResp.Body.Close()

	//fmt.Println("-------------------")
	//fmt.Println(dResp.StatusCode)
	//for k, v := range dResp.Header {
	//	fmt.Println(k, ":", strings.Join(v, ","))
	//}

	//status error
	if dResp.StatusCode/100 != 2 {
		err = fmt.Errorf("Get resource error, %s", dResp.Status)
		return
	}

	//if not support range, go back and err
	if dResp.Header.Get("Accept-Ranges") == "" {
		err = errors.New("Remote server not support range")
		return
	}

	//parse content-range
	contentRange := dResp.Header.Get("Content-Range")
	rangeSize, _ := parseContentRange(contentRange)

	//check ranged block size
	if !lastBlock && rangeSize != rangeBlockSize {
		err = errors.New("Block read error, only the last range block can has bytes less than <RangeBlockSize>")
		return
	}

	//read content
	buffer := bytes.NewBuffer(nil)
	cpCnt, cpErr := io.Copy(buffer, dResp.Body)
	if cpErr != nil || cpCnt != rangeSize {
		err = errors.New("Read range block response error, not fully read")
		return
	}

	//mkblk
	blkPutRet := rio.BlkputRet{}
	blockSize := int(rangeSize)
	blockDataReader := bytes.NewReader(buffer.Bytes())
	blockDataSize := buffer.Len()

	mkErr := rio.Mkblock(putClient, nil, &blkPutRet, blockSize, blockDataReader, blockDataSize)
	if mkErr != nil {
		err = fmt.Errorf("Mkblk error, %s", mkErr.Error())
		return
	}

	putRet = blkPutRet

	return
}

func recordProgress(progressFile string, syncProgress SyncProgress) (err error) {
	fh, openErr := os.Create(progressFile)
	if openErr != nil {
		err = fmt.Errorf("Open progress file %s error, %s", progressFile, openErr.Error())
		return
	}
	defer fh.Close()

	jsonBytes, mErr := json.Marshal(&syncProgress)
	if mErr != nil {
		err = fmt.Errorf("Marshal sync progress error, %s", mErr.Error())
		return
	}

	_, wErr := fh.Write(jsonBytes)
	if wErr != nil {
		err = fmt.Errorf("Write sync progress error, %s", wErr.Error())
	}

	return
}

func getRemoteFileLength(srcResUrl string) (totalSize int64, err error) {
	resp, respErr := http.Head(srcResUrl)
	if respErr != nil {
		err = fmt.Errorf("New head request failed, %s", respErr.Error())
		return
	}
	defer resp.Body.Close()

	contentLength := resp.Header.Get("Content-Length")
	if contentLength == "" {
		err = errors.New("Head request with no Content-Length found error")
		return
	}

	totalSize, _ = strconv.ParseInt(contentLength, 10, 64)

	return
}

func checkExists(mac *digest.Mac, bucket, key string) (exists bool, err error) {
	client := rs.NewMac(mac)
	entry, sErr := client.Stat(nil, bucket, key)
	if sErr != nil {
		if v, ok := sErr.(*rpc.ErrorInfo); !ok {
			err = fmt.Errorf("Check file exists error, %s", sErr.Error())
			return
		} else {
			if v.Code != 612 {
				err = fmt.Errorf("Check file exists error, %s", v.Err)
				return
			} else {
				exists = false
				return
			}
		}
	}

	if entry.Hash != "" {
		exists = true
	}

	return
}

//Content-Range: bytes 25538640-25538647/25538648
func parseContentRange(contentRange string) (rangeSize, totalSize int64) {
	contentRangeItems := strings.Split(contentRange, " ")
	sizeItems := strings.Split(contentRangeItems[1], "/")

	rangePartItems := strings.Split(sizeItems[0], "-")
	totalSize, _ = strconv.ParseInt(sizeItems[1], 10, 64)

	fromOffset, _ := strconv.ParseInt(rangePartItems[0], 10, 64)
	toOffset, _ := strconv.ParseInt(rangePartItems[1], 10, 64)

	rangeSize = toOffset - fromOffset + 1

	return
}
<|endoftext|>


<commit_before>package array

func sortArrayByParity(arr []int) []int {
	return arr
}
<commit_msg>solve 905 use one pass<commit_after>package array

func sortArrayByParity(arr []int) []int {
	return useOnePass(arr)
}

// useOnePass time complexity O(N), space complexity O(1)
func useOnePass(arr []int) []int {
	n := len(arr)
	l, r := 0, n-1
	for l < r {
		if arr[l]%2 == 0 {
			l++
		} else {
			arr[l], arr[r] = arr[r], arr[l]
		}
		if arr[r]%2 != 0 {
			r--
		} else {
			arr[r], arr[l] = arr[l], arr[r]
		}
	}
	return arr
}
<|endoftext|>


<commit_before>package sync

import (
	"fmt"
	"runtime"
	"sync"
	"time"
)

// RWMutex provides locking functions, and an ability to detect and remove
// deadlocks.
type RWMutex struct {
	openLocks        map[int]struct{}
	openLocksCounter int
	openLocksMutex   sync.Mutex

	callDepth   int
	maxLockTime time.Duration

	mu sync.RWMutex
}

// New takes a maxLockTime and returns a lock. The lock will never stay locked
// for more than maxLockTime, instead printing an error and unlocking after
// maxLockTime has passed.
func New(maxLockTime time.Duration, callDepth int) RWMutex {
	return RWMutex{
		openLocks:   make(map[int]struct{}),
		maxLockTime: maxLockTime,
		callDepth:   callDepth,
	}
}

// safeLock is the generic function for doing safe locking. If the read flag is
// set, then a readlock will be used, otherwise a lock will be used.
func (rwm *RWMutex) safeLock(read bool) int {
	// Get the call stack.
	callingFiles := make([]string, rwm.callDepth+1)
	callingLines := make([]int, rwm.callDepth+1)
	for i := 0; i <= rwm.callDepth; i++ {
		_, callingFiles[i], callingLines[i], _ = runtime.Caller(2 + i)
	}

	// Safely register that a lock has been triggered.
	rwm.openLocksMutex.Lock()
	counter := rwm.openLocksCounter
	rwm.openLocks[counter] = struct{}{}
	rwm.openLocksCounter++
	rwm.openLocksMutex.Unlock()

	// Lock the mutex.
	if read {
		rwm.mu.RLock()
	} else {
		rwm.mu.Lock()
	}

	// Create the function that will wait for 'maxLockTime' and then check that
	// the lock has been disabled.

	go func() {
		time.Sleep(rwm.maxLockTime)

		rwm.openLocksMutex.Lock()
		defer rwm.openLocksMutex.Unlock()

		// Check that the lock has been removed and if it hasn't, remove it.
		_, exists := rwm.openLocks[counter]
		if exists {
			delete(rwm.openLocks, counter)
			if read {
				rwm.mu.RUnlock()
			} else {
				rwm.mu.Unlock()
			}

			var lockType string
			if read {
				lockType = "read lock"
			} else {
				lockType = "lock"
			}
			fmt.Printf("A %v was held for too long, id '%v'. Call stack:\n", lockType, counter)
			for i := 0; i <= rwm.callDepth; i++ {
				fmt.Printf("\tFile '%v', Line '%v'\n", callingFiles[i], callingLines[i])
			}
		}
	}()

	return counter
}

// safeUnlock is the generic function for doing safe unlocking. If the lock had
// to be removed because a deadlock was detected, an error is printed.
func (rwm *RWMutex) safeUnlock(read bool, counter int) {
	// Get the call stack.
	callingFiles := make([]string, rwm.callDepth+1)
	callingLines := make([]int, rwm.callDepth+1)
	for i := 0; i <= rwm.callDepth; i++ {
		_, callingFiles[i], callingLines[i], _ = runtime.Caller(2 + i)
	}

	rwm.openLocksMutex.Lock()
	defer rwm.openLocksMutex.Unlock()

	// Check if a deadlock has been detected and fixed manually.
	_, exists := rwm.openLocks[counter]
	if !exists {
		var lockType string
		if read {
			lockType = "read "
		} else {
			lockType = ""
		}
		fmt.Printf("A %v lock was held until deadlock, subsequent call to %v unlock failed. id '%v'. Call stack:\n", lockType, lockType, counter)
		for i := 0; i <= rwm.callDepth; i++ {
			fmt.Printf("\tFile '%v', Line '%v'\n", callingFiles[i], callingLines[i])
		}
		return
	}

	// Remove the lock.
	delete(rwm.openLocks, counter)
	if read {
		rwm.mu.RUnlock()
	} else {
		rwm.mu.Unlock()
	}
}

// RLock will read lock the RWMutex. The return value must be used as input
// when calling RUnlock.
func (rwm *RWMutex) RLock() int {
	return rwm.safeLock(true)
}

// RUnlock will read unlock the RWMutex. The return value of calling RLock must
// be used as input.
func (rwm *RWMutex) RUnlock(counter int) {
	rwm.safeUnlock(true, counter)
}

// Lock will lock the RWMutex. The return value must be used as input when
// calling RUnlock.
func (rwm *RWMutex) Lock() int {
	return rwm.safeLock(false)
}

// Unlock will unlock the RWMutex. The return value of calling Lock must be
// used as input.
func (rwm *RWMutex) Unlock(counter int) {
	rwm.safeUnlock(false, counter)
}
<commit_msg>fix spacing error<commit_after>package sync

import (
	"fmt"
	"runtime"
	"sync"
	"time"
)

// RWMutex provides locking functions, and an ability to detect and remove
// deadlocks.
type RWMutex struct {
	openLocks        map[int]struct{}
	openLocksCounter int
	openLocksMutex   sync.Mutex

	callDepth   int
	maxLockTime time.Duration

	mu sync.RWMutex
}

// New takes a maxLockTime and returns a lock. The lock will never stay locked
// for more than maxLockTime, instead printing an error and unlocking after
// maxLockTime has passed.
func New(maxLockTime time.Duration, callDepth int) RWMutex {
	return RWMutex{
		openLocks:   make(map[int]struct{}),
		maxLockTime: maxLockTime,
		callDepth:   callDepth,
	}
}

// safeLock is the generic function for doing safe locking. If the read flag is
// set, then a readlock will be used, otherwise a lock will be used.
func (rwm *RWMutex) safeLock(read bool) int {
	// Get the call stack.
	callingFiles := make([]string, rwm.callDepth+1)
	callingLines := make([]int, rwm.callDepth+1)
	for i := 0; i <= rwm.callDepth; i++ {
		_, callingFiles[i], callingLines[i], _ = runtime.Caller(2 + i)
	}

	// Safely register that a lock has been triggered.
	rwm.openLocksMutex.Lock()
	counter := rwm.openLocksCounter
	rwm.openLocks[counter] = struct{}{}
	rwm.openLocksCounter++
	rwm.openLocksMutex.Unlock()

	// Lock the mutex.
	if read {
		rwm.mu.RLock()
	} else {
		rwm.mu.Lock()
	}

	// Create the function that will wait for 'maxLockTime' and then check that
	// the lock has been disabled.

	go func() {
		time.Sleep(rwm.maxLockTime)

		rwm.openLocksMutex.Lock()
		defer rwm.openLocksMutex.Unlock()

		// Check that the lock has been removed and if it hasn't, remove it.
		_, exists := rwm.openLocks[counter]
		if exists {
			delete(rwm.openLocks, counter)
			if read {
				rwm.mu.RUnlock()
			} else {
				rwm.mu.Unlock()
			}

			var lockType string
			if read {
				lockType = "read lock"
			} else {
				lockType = "lock"
			}
			fmt.Printf("A %v was held for too long, id '%v'. Call stack:\n", lockType, counter)
			for i := 0; i <= rwm.callDepth; i++ {
				fmt.Printf("\tFile '%v', Line '%v'\n", callingFiles[i], callingLines[i])
			}
		}
	}()

	return counter
}

// safeUnlock is the generic function for doing safe unlocking. If the lock had
// to be removed because a deadlock was detected, an error is printed.
func (rwm *RWMutex) safeUnlock(read bool, counter int) {
	// Get the call stack.
	callingFiles := make([]string, rwm.callDepth+1)
	callingLines := make([]int, rwm.callDepth+1)
	for i := 0; i <= rwm.callDepth; i++ {
		_, callingFiles[i], callingLines[i], _ = runtime.Caller(2 + i)
	}

	rwm.openLocksMutex.Lock()
	defer rwm.openLocksMutex.Unlock()

	// Check if a deadlock has been detected and fixed manually.
	_, exists := rwm.openLocks[counter]
	if !exists {
		var lockType string
		if read {
			lockType = "read "
		} else {
			lockType = ""
		}
		fmt.Printf("A%v lock was held until deadlock, subsequent call to%v unlock failed. id '%v'. Call stack:\n", lockType, lockType, counter)
		for i := 0; i <= rwm.callDepth; i++ {
			fmt.Printf("\tFile '%v', Line '%v'\n", callingFiles[i], callingLines[i])
		}
		return
	}

	// Remove the lock.
	delete(rwm.openLocks, counter)
	if read {
		rwm.mu.RUnlock()
	} else {
		rwm.mu.Unlock()
	}
}

// RLock will read lock the RWMutex. The return value must be used as input
// when calling RUnlock.
func (rwm *RWMutex) RLock() int {
	return rwm.safeLock(true)
}

// RUnlock will read unlock the RWMutex. The return value of calling RLock must
// be used as input.
func (rwm *RWMutex) RUnlock(counter int) {
	rwm.safeUnlock(true, counter)
}

// Lock will lock the RWMutex. The return value must be used as input when
// calling RUnlock.
func (rwm *RWMutex) Lock() int {
	return rwm.safeLock(false)
}

// Unlock will unlock the RWMutex. The return value of calling Lock must be
// used as input.
func (rwm *RWMutex) Unlock(counter int) {
	rwm.safeUnlock(false, counter)
}
<|endoftext|>


<commit_before>package main

import (
	"io"
	"os"
	"strconv"
	"strings"
	"testing"
	"time"
)

func TestRecursive(t *testing.T) {
	// Recursive is depth first
	input := [][]int{{1, 0, 6}, {0, 4, 5}, {2, 3}}
	ch := make(chan File)
	go func() {
		recursive(File{}, ch, fakeIndex(input))
		close(ch)
	}()

	expN := 1
LOOP:
	for {
		select {
		case f, ok := <-ch:
			if !ok {
				break LOOP
			}
			n, _ := strconv.Atoi(f.Path)
			if n != expN {
				t.Errorf("%d should have been %d", n, expN)
			}
			expN++
		case <-time.After(100 * time.Millisecond):
			t.Error("timeout expected:", expN)
			break LOOP
		}
	}
	expected := flattenInts(input)
	if expN < len(expected) {
		t.Error("Missing:", expected[expN:])
	}
}

func fakeIndex(nums [][]int) IndexFn {
	i := 0
	return func(f File, ch chan File) {
		for _, n := range nums[i] {
			nf := f
			nf.Path = strconv.Itoa(n)
			if n == 0 {
				i++
			} else {
				nf = nf.SetLeaf()
			}
			ch <- nf
		}
	}
}

func flattenInts(ints [][]int) []int {
	res := []int{}
	for _, g := range ints {
		for _, n := range g {
			if n == 0 {
				continue
			}
			res = append(res, n)
		}
	}
	return res
}

func TestLocalNew(t *testing.T) {
	testLocal(t, func(f File, r io.ReadCloser) File {
		return f
	}, true)
}

func TestLocalOverwriteOlder(t *testing.T) {
	testLocal(t, func(f File, r io.ReadCloser) File {
		err := Local(f, r)
		if err != nil {
			t.Error(err)
		}
		f.Mtime = f.Mtime.Add(time.Second)
		return f
	}, true)
}

func TestLocalNotOverwriteNewer(t *testing.T) {
	testLocal(t, func(f File, r io.ReadCloser) File {
		err := Local(f, r)
		if err != nil {
			t.Error(err)
		}
		f.Mtime = f.Mtime.Add(-time.Second)
		return f
	}, false)
}

func TestLocalCreateDirs(t *testing.T) {
	testLocal(t, func(f File, r io.ReadCloser) File {
		f.Path += "a/dir/oh/uh/hi/ho"
		return f
	}, true)
}

// Trying to overwrite a directory fails
func TestLocalOverwriteDir(t *testing.T) {
	tmp, rm := TempDir()
	defer rm()
	f, r := someTestFile(tmp)
	err := os.Mkdir(f.Path, 777)
	if err != nil {
		t.Fatal(err)
	}
	err = os.Chtimes(f.Path, time.Now(), time.Now().Add(-time.Hour))
	if err != nil {
		t.Fatal(err)
	}
	if Local(f, r) == nil {
		t.Error("should have failed")
	}
}

func testLocal(t *testing.T, init func(File, io.ReadCloser) File, overwrite bool) {
	tmp, rm := TempDir()
	of, r := someTestFile(tmp)
	f := init(of, r)
	err := Local(f, r)
	if err != nil {
		t.Error(err)
	}
	if overwrite {
		of = f
	}
	checkFile(t, of)
	rm()
}

func someTestFile(tmp string) (File, io.ReadCloser) {
	f := File{Mtime: time.Now(), Path: tmp + "/a"}
	return f, newFakeCloser("test")
}

func checkFile(t *testing.T, f File) {
	st, err := os.Stat(f.Path)

	f.Mtime = removeSubSecond(f.Mtime)

	if err != nil && os.IsNotExist(err) {
		t.Error("File does not exist:", f.Path)
	} else {
		if !(st.ModTime().Equal(f.Mtime)) {
			t.Errorf("Not overwritten")
		}
	}
}

// OSX does not store time resolutions below seconds
func removeSubSecond(in time.Time) time.Time {
	return time.Date(
		in.Year(),
		in.Month(),
		in.Day(),
		in.Hour(),
		in.Minute(),
		in.Second(),
		0, in.Location())
}
<commit_msg>fix sync tests<commit_after>package main

import (
	"io"
	"os"
	"strconv"
	"testing"
	"time"
)

func TestRecursive(t *testing.T) {
	// Recursive is depth first
	input := [][]int{{1, 0, 6}, {0, 4, 5}, {2, 3}}
	ch := make(chan File)
	go func() {
		recursive(File{}, ch, fakeIndex(input))
		close(ch)
	}()

	expN := 1
LOOP:
	for {
		select {
		case f, ok := <-ch:
			if !ok {
				break LOOP
			}
			n, _ := strconv.Atoi(f.Path)
			if n != expN {
				t.Errorf("%d should have been %d", n, expN)
			}
			expN++
		case <-time.After(100 * time.Millisecond):
			t.Error("timeout expected:", expN)
			break LOOP
		}
	}
	expected := flattenInts(input)
	if expN < len(expected) {
		t.Error("Missing:", expected[expN:])
	}
}

func fakeIndex(nums [][]int) IndexFn {
	i := 0
	return func(f File, ch chan File) {
		for _, n := range nums[i] {
			nf := f
			nf.Path = strconv.Itoa(n)
			if n == 0 {
				i++
			} else {
				nf = nf.SetLeaf()
			}
			ch <- nf
		}
	}
}

func flattenInts(ints [][]int) []int {
	res := []int{}
	for _, g := range ints {
		for _, n := range g {
			if n == 0 {
				continue
			}
			res = append(res, n)
		}
	}
	return res
}

func TestLocalNew(t *testing.T) {
	testLocal(t, func(f File, r io.ReadCloser) File {
		return f
	}, true)
}

func TestLocalOverwriteOlder(t *testing.T) {
	testLocal(t, func(f File, r io.ReadCloser) File {
		err := local(f, r)
		if err != nil {
			t.Error(err)
		}
		f.Mtime = f.Mtime.Add(time.Second)
		return f
	}, true)
}

func TestLocalNotOverwriteNewer(t *testing.T) {
	testLocal(t, func(f File, r io.ReadCloser) File {
		err := local(f, r)
		if err != nil {
			t.Error(err)
		}
		f.Mtime = f.Mtime.Add(-time.Second)
		return f
	}, false)
}

func TestLocalCreateDirs(t *testing.T) {
	testLocal(t, func(f File, r io.ReadCloser) File {
		f.Path += "a/dir/oh/uh/hi/ho"
		return f
	}, true)
}

// Trying to overwrite a directory fails
func TestLocalOverwriteDir(t *testing.T) {
	tmp, rm := TempDir()
	defer rm()
	f, r := someTestFile(tmp)
	err := os.Mkdir(f.Path, 777)
	if err != nil {
		t.Fatal(err)
	}
	err = os.Chtimes(f.Path, time.Now(), time.Now().Add(-time.Hour))
	if err != nil {
		t.Fatal(err)
	}
	if local(f, r) == nil {
		t.Error("should have failed")
	}
}

func testLocal(t *testing.T, init func(File, io.ReadCloser) File, overwrite bool) {
	tmp, rm := TempDir()
	of, r := someTestFile(tmp)
	f := init(of, r)
	err := local(f, r)
	if err != nil {
		t.Error(err)
	}
	if overwrite {
		of = f
	}
	checkFile(t, of)
	rm()
}

func someTestFile(tmp string) (File, io.ReadCloser) {
	f := File{Mtime: time.Now(), Path: tmp + "/a"}
	return f, newFakeCloser("test")
}

func checkFile(t *testing.T, f File) {
	st, err := os.Stat(f.Path)

	f.Mtime = removeSubSecond(f.Mtime)

	if err != nil && os.IsNotExist(err) {
		t.Error("File does not exist:", f.Path)
	} else {
		if !(st.ModTime().Equal(f.Mtime)) {
			t.Errorf("Not overwritten")
		}
	}
}

// OSX does not store time resolutions below seconds
func removeSubSecond(in time.Time) time.Time {
	return time.Date(
		in.Year(),
		in.Month(),
		in.Day(),
		in.Hour(),
		in.Minute(),
		in.Second(),
		0, in.Location())
}
<|endoftext|>


<commit_before>package multiverse

import (
	"strings"
	"unicode"

	"github.com/CasualSuperman/Diorite/trie"
	"github.com/CasualSuperman/levenshtein"
	"github.com/dotCypress/phonetics"
)

func generatePhoneticsMaps(cards []*Card) trie.Trie {
	metaphoneMap := trie.Alt()

	for i, c := range cards {
		name := preventUnicode(c.Name)
		for _, word := range strings.Split(name, " ") {
			if len(word) < 4 {
				continue
			}
			mtp := phonetics.EncodeMetaphone(word)

			others, ok := metaphoneMap.Get(mtp)
			if ok {
				slice := others.([]int)
				slice = append(slice, i)
				metaphoneMap.Remove(mtp)
				metaphoneMap.Add(mtp, slice)
			} else {
				metaphoneMap.Add(mtp, []int{i})
			}
		}
	}

	return metaphoneMap
}

var phoneticsCache = make(map[string]string)

func getMetaphone(s string) string {
	if cached, ok := phoneticsCache[s]; ok {
		return cached
	}

	m := phonetics.EncodeMetaphone(s)
	phoneticsCache[s] = m
	return m
}

var unicodeCache = make(map[string]string)

func preventUnicode(name string) string {
	name = strings.ToLower(name)
	if cached, ok := unicodeCache[name]; ok {
		return cached
	}

	clean := ""
	for _, r := range name {
		if r > 128 {
			switch r {
			case 'á', 'à', 'â':
				clean += "a"
			case 'é':
				clean += "e"
			case 'í':
				clean += "i"
			case 'ö':
				clean += "o"
			case 'û', 'ú':
				clean += "u"

			case 'Æ', 'æ':
				clean += "ae"

			case '®':
				// We know this is an option but we're explicitly ignoring it.

			default:
			}
		} else {
			if r == ' ' || unicode.IsLetter(r) {
				clean += string(r)
			}
		}
	}

	unicodeCache[name] = clean

	return clean
}

type fuzzySearchList []struct {
	index      int
	similarity float32
}

// FuzzyNameSearch searches for a card with a similar name to the searchPhrase, and returns count or less of the most likely results.
func (m Multiverse) FuzzyNameSearch(searchPhrase string, count int) []*Card {
	var aggregator = make(fuzzySearchList, 0, count)
	searchPhrase = preventUnicode(searchPhrase)
	searchGrams2 := newNGram(searchPhrase, 2)
	searchGrams3 := newNGram(searchPhrase, 3)

	for _, searchTerm := range strings.Split(searchPhrase, " ") {
		for _, candidate := range m.Pronunciations.Search(getMetaphone(searchTerm)) {
			cardIndices, _ := m.Pronunciations.Get(candidate)
		cardLoop:
			for _, cardIndex := range cardIndices.([]int) {
				for _, i := range aggregator {
					if i.index == cardIndex {
						continue cardLoop
					}
				}
				name := preventUnicode(m.Cards.List[cardIndex].Name)

				bestMatch := 0
				for _, word := range strings.Split(name, " ") {
					match := phonetics.DifferenceSoundex(word, searchTerm)
					if match > bestMatch {
						bestMatch = match
					}
				}

				similarity := searchGrams2.Similarity(name)
				similarity *= searchGrams3.Similarity(name)
				similarity *= float32(bestMatch)
				similarity /= float32(levenshtein.Distance(searchPhrase, name))
				similarity *= float32(len(name))

				if strings.Contains(name, searchPhrase) {
					similarity *= 10
				}

				var app = struct {
					index      int
					similarity float32
				}{
					cardIndex,
					similarity,
				}

				if len(aggregator) < cap(aggregator) {
					i := len(aggregator) + 1
					aggregator = aggregator[:i]
					aggregator[i-1] = app
				} else {
					for i := count - 1; i >= 0; i-- {
						if aggregator[i].similarity < app.similarity {
							if i < count-1 {
								aggregator[i+1] = aggregator[i]
							}
							aggregator[i] = app
						} else {
							i = 0
						}
					}
				}
			}
		}
	levenshteinLoop:
		for cardIndex, card := range m.Cards.List {
			for _, word := range strings.Split(preventUnicode(card.Name), " ") {
				if levenshtein.Distance(word, searchTerm) <= len(searchTerm)/3 {

					name := preventUnicode(card.Name)
					similarity := searchGrams2.Similarity(name)
					similarity *= searchGrams3.Similarity(name)
					similarity *= float32(phonetics.DifferenceSoundex(word, searchTerm)) / 10.0
					similarity /= float32(levenshtein.Distance(searchPhrase, name))
					similarity *= float32(len(name))
					var app = struct {
						index      int
						similarity float32
					}{
						cardIndex,
						similarity,
					}

					for i, ci := range aggregator {
						if cardIndex == ci.index {
							if ci.similarity < similarity {
								aggregator[i] = app
							}
							continue levenshteinLoop
						}
					}

					if len(aggregator) < cap(aggregator) {
						i := len(aggregator) + 1
						aggregator = aggregator[:i]
						aggregator[i-1] = app
					} else {
						for i := count - 1; i >= 0; i-- {
							if aggregator[i].similarity < app.similarity {
								if i < count-1 {
									aggregator[i+1] = aggregator[i]
								}
								aggregator[i] = app
							} else {
								i = 0
							}
						}
					}
				}
			}
		}
	}

	if len(aggregator) < count {
		count = len(aggregator)
	}

	results := make([]*Card, count)

	for i, card := range aggregator {
		results[i] = m.Cards.List[card.index]
	}

	return results
}
<commit_msg>Removed extra math for resizing slice.<commit_after>package multiverse

import (
	"strings"
	"unicode"

	"github.com/CasualSuperman/Diorite/trie"
	"github.com/CasualSuperman/levenshtein"
	"github.com/dotCypress/phonetics"
)

func generatePhoneticsMaps(cards []*Card) trie.Trie {
	metaphoneMap := trie.Alt()

	for i, c := range cards {
		name := preventUnicode(c.Name)
		for _, word := range strings.Split(name, " ") {
			if len(word) < 4 {
				continue
			}
			mtp := phonetics.EncodeMetaphone(word)

			others, ok := metaphoneMap.Get(mtp)
			if ok {
				slice := others.([]int)
				slice = append(slice, i)
				metaphoneMap.Remove(mtp)
				metaphoneMap.Add(mtp, slice)
			} else {
				metaphoneMap.Add(mtp, []int{i})
			}
		}
	}

	return metaphoneMap
}

var phoneticsCache = make(map[string]string)

func getMetaphone(s string) string {
	if cached, ok := phoneticsCache[s]; ok {
		return cached
	}

	m := phonetics.EncodeMetaphone(s)
	phoneticsCache[s] = m
	return m
}

var unicodeCache = make(map[string]string)

func preventUnicode(name string) string {
	name = strings.ToLower(name)
	if cached, ok := unicodeCache[name]; ok {
		return cached
	}

	clean := ""
	for _, r := range name {
		if r > 128 {
			switch r {
			case 'á', 'à', 'â':
				clean += "a"
			case 'é':
				clean += "e"
			case 'í':
				clean += "i"
			case 'ö':
				clean += "o"
			case 'û', 'ú':
				clean += "u"

			case 'Æ', 'æ':
				clean += "ae"

			case '®':
				// We know this is an option but we're explicitly ignoring it.

			default:
			}
		} else {
			if r == ' ' || unicode.IsLetter(r) {
				clean += string(r)
			}
		}
	}

	unicodeCache[name] = clean

	return clean
}

type fuzzySearchList []struct {
	index      int
	similarity float32
}

// FuzzyNameSearch searches for a card with a similar name to the searchPhrase, and returns count or less of the most likely results.
func (m Multiverse) FuzzyNameSearch(searchPhrase string, count int) []*Card {
	var aggregator = make(fuzzySearchList, 0, count)
	searchPhrase = preventUnicode(searchPhrase)
	searchGrams2 := newNGram(searchPhrase, 2)
	searchGrams3 := newNGram(searchPhrase, 3)

	for _, searchTerm := range strings.Split(searchPhrase, " ") {
		for _, candidate := range m.Pronunciations.Search(getMetaphone(searchTerm)) {
			cardIndices, _ := m.Pronunciations.Get(candidate)
		cardLoop:
			for _, cardIndex := range cardIndices.([]int) {
				for _, i := range aggregator {
					if i.index == cardIndex {
						continue cardLoop
					}
				}
				name := preventUnicode(m.Cards.List[cardIndex].Name)

				bestMatch := 0
				for _, word := range strings.Split(name, " ") {
					match := phonetics.DifferenceSoundex(word, searchTerm)
					if match > bestMatch {
						bestMatch = match
					}
				}

				similarity := searchGrams2.Similarity(name)
				similarity *= searchGrams3.Similarity(name)
				similarity *= float32(bestMatch)
				similarity /= float32(levenshtein.Distance(searchPhrase, name))
				similarity *= float32(len(name))

				if strings.Contains(name, searchPhrase) {
					similarity *= 10
				}

				var app = struct {
					index      int
					similarity float32
				}{
					cardIndex,
					similarity,
				}

				if len(aggregator) < cap(aggregator) {
					i := len(aggregator)
					aggregator = aggregator[:i+1]
					aggregator[i] = app
				} else {
					for i := count - 1; i >= 0; i-- {
						if aggregator[i].similarity < app.similarity {
							if i < count-1 {
								aggregator[i+1] = aggregator[i]
							}
							aggregator[i] = app
						} else {
							i = 0
						}
					}
				}
			}
		}
	levenshteinLoop:
		for cardIndex, card := range m.Cards.List {
			for _, word := range strings.Split(preventUnicode(card.Name), " ") {
				if levenshtein.Distance(word, searchTerm) <= len(searchTerm)/3 {

					name := preventUnicode(card.Name)
					similarity := searchGrams2.Similarity(name)
					similarity *= searchGrams3.Similarity(name)
					similarity *= float32(phonetics.DifferenceSoundex(word, searchTerm)) / 10.0
					similarity /= float32(levenshtein.Distance(searchPhrase, name))
					similarity *= float32(len(name))
					var app = struct {
						index      int
						similarity float32
					}{
						cardIndex,
						similarity,
					}

					for i, ci := range aggregator {
						if cardIndex == ci.index {
							if ci.similarity < similarity {
								aggregator[i] = app
							}
							continue levenshteinLoop
						}
					}

					if len(aggregator) < cap(aggregator) {
						i := len(aggregator)
						aggregator = aggregator[:i+1]
						aggregator[i] = app
					} else {
						for i := count - 1; i >= 0; i-- {
							if aggregator[i].similarity < app.similarity {
								if i < count-1 {
									aggregator[i+1] = aggregator[i]
								}
								aggregator[i] = app
							} else {
								i = 0
							}
						}
					}
				}
			}
		}
	}

	if len(aggregator) < count {
		count = len(aggregator)
	}

	results := make([]*Card, count)

	for i, card := range aggregator {
		results[i] = m.Cards.List[card.index]
	}

	return results
}
<|endoftext|>