<commit_before>// Copyright (c) 2015 Uber Technologies, Inc.

// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

// vbumper helps bump version numbers in the repository and in the CHANGELOG.
package main

import (
	"bytes"
	"errors"
	"flag"
	"html/template"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"strings"
	"time"
)

var (
	_changelogFile = flag.String("changelog-file", "CHANGELOG.md", "Filename of the changelog file")
	_versionFile   = flag.String("version-file", "version.go", "Filename of where the version information is stored")
	_version       = flag.String("version", "", "Version to mention in changelog and version.go")
	_versionDate   = flag.String("version-date", "", "Date to use in the changelog, by default the current date")
	_skipChangelog = flag.Bool("skip-changelog", false, "Skip updating the changelog")
)

func main() {
	*_versionDate = time.Now().Format("2006-01-02")
	flag.Parse()

	if *_version == "" {
		log.Fatal("Please specify the version to release using --version")
	}
	*_version = strings.TrimPrefix(*_version, "v")

	prevVersion, err := updateChangelog()
	if err != nil {
		log.Fatal("failed to update changelog", err)
	}

	if err := updateVersion(prevVersion); err != nil {
		log.Fatal("failed to update version", err)
	}
}

func updateVersion(prevVersion string) error {
	versionBytes, err := ioutil.ReadFile(*_versionFile)
	if err != nil {
		return err
	}

	newContents := insertNewVersion(string(versionBytes), prevVersion, *_version)
	return ioutil.WriteFile(*_versionFile, []byte(newContents), 0666)
}

func insertNewVersion(contents, prevVersion, newVersion string) string {
	// Find the version string in the file
	versionStart := strings.Index(contents, prevVersion)
	versionLine := contents[versionStart:]
	versionEnd := strings.Index(versionLine, `"`) + versionStart
	return contents[:versionStart] + newVersion + contents[versionEnd:]
}

func updateChangelog() (oldVersion string, _ error) {
	changelogBytes, err := ioutil.ReadFile(*_changelogFile)
	if err != nil {
		return "", err
	}

	newLog, oldVersion, err := insertNewChangelog(string(changelogBytes))
	if err != nil {
		return "", err
	}

	if *_skipChangelog {
		return oldVersion, nil
	}

	return oldVersion, ioutil.WriteFile(*_changelogFile, []byte(newLog), 0666)
}

func insertNewChangelog(contents string) (string, string, error) {
	prevVersionHeader := strings.Index(contents, "\n# ")
	if prevVersionHeader < 0 {
		return "", "", errors.New("failed to find version header in changelog")
	}

	// Skip the newline
	prevVersionHeader++
	versionLine := contents[prevVersionHeader:]
	prevVersionEnd := strings.Index(versionLine, "(")
	prevVersionTag := strings.TrimSpace(versionLine[1 : prevVersionEnd-1])

	newChanges, err := getNewChangelog(prevVersionTag)
	if err != nil {
		return "", "", err
	}

	// Strip the 'v' prefix.
	prevVersion := prevVersionTag[1:]
	newContents := contents[:prevVersionHeader] + newChanges + contents[prevVersionHeader:]
	return newContents, prevVersion, nil
}

func getNewChangelog(prevVersion string) (string, error) {
	changes, err := getChanges(prevVersion)
	if err != nil {
		return "", err
	}

	if len(changes) == 0 {
		changes = []string{"No changes yet"}
	}

	buf := &bytes.Buffer{}
	_changeTmpl.Execute(buf, struct {
		Version string
		Date    string
		Changes []string
	}{
		Version: *_version,
		Date:    *_versionDate,
		Changes: changes,
	})
	return buf.String(), nil
}

var _changeTmpl = template.Must(template.New("changelog").Parse(
	`# v{{ .Version }} ({{ .Date }})
{{ range .Changes }}
 * {{ . -}}
{{ end }}

`))

func getChanges(prevVersion string) ([]string, error) {
	cmd := exec.Command("git", "log", "--format=%s", "--no-merges", prevVersion+"..HEAD")
	cmd.Stderr = os.Stderr
	out, err := cmd.Output()
	if err != nil {
		return nil, err
	}

	lines := strings.Split(string(out), "\n")
	newLines := make([]string, 0, len(lines))
	for _, line := range lines {
		line = strings.TrimSpace(line)
		if line == "" {
			continue
		}
		newLines = append(newLines, line)
	}
	return newLines, nil
}
<commit_msg>Update version bumper script for new format<commit_after>// Copyright (c) 2015 Uber Technologies, Inc.

// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

// vbumper helps bump version numbers in the repository and in the CHANGELOG.
package main

import (
	"bytes"
	"errors"
	"flag"
	"fmt"
	"html/template"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"strings"
	"time"
)

var (
	_changelogFile = flag.String("changelog-file", "CHANGELOG.md", "Filename of the changelog file")
	_versionFile   = flag.String("version-file", "version.go", "Filename of where the version information is stored")
	_version       = flag.String("version", "", "Version to mention in changelog and version.go")
	_versionDate   = flag.String("version-date", "", "Date to use in the changelog, by default the current date")
	_skipChangelog = flag.Bool("skip-changelog", false, "Skip updating the changelog")
)

func main() {
	*_versionDate = time.Now().Format("2006-01-02")
	flag.Parse()

	if *_version == "" {
		log.Fatal("Please specify the version to release using --version")
	}
	*_version = strings.TrimPrefix(*_version, "v")

	prevVersion, err := updateChangelog()
	if err != nil {
		log.Fatal("failed to update changelog", err)
	}

	if err := updateVersion(prevVersion); err != nil {
		log.Fatal("failed to update version", err)
	}
}

func updateVersion(prevVersion string) error {
	versionBytes, err := ioutil.ReadFile(*_versionFile)
	if err != nil {
		return err
	}

	newContents := insertNewVersion(string(versionBytes), prevVersion, *_version)
	return ioutil.WriteFile(*_versionFile, []byte(newContents), 0666)
}

func insertNewVersion(contents, prevVersion, newVersion string) string {
	// Find the version string in the file
	versionStart := strings.Index(contents, prevVersion)
	versionLine := contents[versionStart:]
	versionEnd := strings.Index(versionLine, `"`) + versionStart
	return contents[:versionStart] + newVersion + contents[versionEnd:]
}

func updateChangelog() (oldVersion string, _ error) {
	changelogBytes, err := ioutil.ReadFile(*_changelogFile)
	if err != nil {
		return "", err
	}

	newLog, oldVersion, err := insertNewChangelog(string(changelogBytes))
	if err != nil {
		return "", err
	}

	newLog, err = insertChangesLink(newLog, oldVersion, *_version)
	if err != nil {
		return "", err
	}

	if *_skipChangelog {
		return oldVersion, nil
	}

	return oldVersion, ioutil.WriteFile(*_changelogFile, []byte(newLog), 0666)
}

func insertNewChangelog(contents string) (string, string, error) {
	prevVersionHeader := strings.Index(contents, "\n## [")
	if prevVersionHeader < 0 {
		return "", "", errors.New("failed to find version header in changelog")
	}

	// Skip the newline
	prevVersionHeader++
	versionLine := contents[prevVersionHeader:]
	prevVersionEnd := strings.Index(versionLine, "]")
	prevVersion := strings.TrimSpace(versionLine[4:prevVersionEnd])

	// The version tag has a "v" prefix.
	newChanges, err := getNewChangelog("v" + prevVersion)
	if err != nil {
		return "", "", err
	}

	newContents := contents[:prevVersionHeader] + newChanges + contents[prevVersionHeader:]
	return newContents, prevVersion, nil
}

func getNewChangelog(prevVersion string) (string, error) {
	changes, err := getChanges(prevVersion)
	if err != nil {
		return "", err
	}

	if len(changes) == 0 {
		changes = []string{"No changes yet"}
	}

	buf := &bytes.Buffer{}
	_changeTmpl.Execute(buf, struct {
		Version string
		Date    string
		Changes []string
	}{
		Version: *_version,
		Date:    *_versionDate,
		Changes: changes,
	})
	return buf.String(), nil
}

var _changeTmpl = template.Must(template.New("changelog").Parse(
	`## [{{ .Version }}] - {{ .Date }}
### Changed
{{ range .Changes }}
 * {{ . -}}
{{ end }}

`))

func getChanges(prevVersion string) ([]string, error) {
	cmd := exec.Command("git", "log", "--format=%s", "--no-merges", prevVersion+"..HEAD")
	cmd.Stderr = os.Stderr
	out, err := cmd.Output()
	if err != nil {
		return nil, err
	}

	lines := strings.Split(string(out), "\n")
	newLines := make([]string, 0, len(lines))
	for _, line := range lines {
		line = strings.TrimSpace(line)
		if line == "" {
			continue
		}
		newLines = append(newLines, line)
	}
	return newLines, nil
}

func insertChangesLink(contents, prevVersion, version string) (string, error) {
	linksMarker := strings.Index(contents, "(Version Links)")
	if linksMarker == -1 {
		return "", errors.New("failed to find marker for version links section")
	}

	newLine := strings.IndexByte(contents[linksMarker:], '\n')
	if newLine < 0 {
		return "", errors.New("failed to find newline after version links section")
	}

	insertAt := linksMarker + newLine + 1

	linkBlock := fmt.Sprintf("[%v]: %v\n", version, getChangesLink(prevVersion, version))
	newContents := contents[:insertAt] + linkBlock + contents[insertAt:]
	return newContents, nil
}

func getChangesLink(prevVersion, curVersion string) string {
	// Example link:
	// https://github.com/uber/tchannel-go/compare/v1.8.0...v1.8.1
	return fmt.Sprintf("https://github.com/uber/tchannel-go/compare/v%v...v%v", prevVersion, curVersion)
}
<|endoftext|>


<commit_before>package signify

import (
	"bufio"
	"bytes"
	"crypto/rand"
	"crypto/sha512"
	"crypto/subtle"
	"encoding/base64"
	"encoding/binary"
	"errors"
	"fmt"
	"io"
	"strings"

	"github.com/agl/ed25519"
	"github.com/ebfe/bcrypt_pbkdf"
)

const (
	commentHdr       = "untrusted comment: "
	defaultKDFRounds = 42
)

var (
	algoEd     = []byte{'E', 'd'}
	algoBcrypt = []byte{'B', 'K'}
)

type PrivateKey struct {
	Bytes       [ed25519.PrivateKeySize]byte
	Fingerprint [8]byte
}

type PublicKey struct {
	Bytes       [ed25519.PublicKeySize]byte
	Fingerprint [8]byte
}

type Signature struct {
	Bytes       [ed25519.SignatureSize]byte
	Fingerprint [8]byte
}

type rawEncryptedKey struct {
	PKAlgo       [2]byte
	KDFAlgo      [2]byte
	KDFRounds    uint32
	Salt         [16]byte
	Checksum     [8]byte
	Fingerprint  [8]byte
	EncryptedKey [ed25519.PrivateKeySize]byte
}

type rawPublicKey struct {
	PKAlgo      [2]byte
	Fingerprint [8]byte
	PublicKey   [ed25519.PublicKeySize]byte
}

type rawSignature struct {
	PKAlgo      [2]byte
	Fingerprint [8]byte
	Signature   [ed25519.SignatureSize]byte
}

func ReadFile(r io.Reader) (comment string, content []byte, err error) {
	sc := bufio.NewScanner(r)

	if !sc.Scan() {
		return "", nil, fmt.Errorf("signify: read error %s", sc.Err())
	}
	comment = sc.Text()
	if !strings.HasPrefix(comment, commentHdr) {
		return "", nil, errors.New("signify: missing header")
	}
	comment = comment[len(commentHdr):]

	if !sc.Scan() {
		return "", nil, fmt.Errorf("signify: read error %s", sc.Err())
	}
	content, err = base64.StdEncoding.DecodeString(sc.Text())

	return
}

func WriteFile(w io.Writer, comment string, content []byte) error {
	b64 := base64.StdEncoding.EncodeToString(content)
	_, err := fmt.Fprintf(w, "%s%s\n%s\n", commentHdr, comment, b64)
	return err
}

func parseRawEncryptedKey(data []byte) (*rawEncryptedKey, error) {
	var ek rawEncryptedKey
	if err := binary.Read(bytes.NewReader(data), binary.BigEndian, &ek); err != nil {
		return nil, err
	}
	return &ek, nil
}

func marshalRawEncryptedKey(rek *rawEncryptedKey) []byte {
	var wbuf bytes.Buffer
	binary.Write(&wbuf, binary.BigEndian, rek)
	return wbuf.Bytes()
}

func parseRawPublicKey(data []byte) (*rawPublicKey, error) {
	var pub rawPublicKey
	if err := binary.Read(bytes.NewReader(data), binary.BigEndian, &pub); err != nil {
		return nil, err
	}
	return &pub, nil
}

func marshalRawPublicKey(rpub *rawPublicKey) []byte {
	var wbuf bytes.Buffer
	binary.Write(&wbuf, binary.BigEndian, rpub)
	return wbuf.Bytes()
}

func parseRawSignature(data []byte) (*rawSignature, error) {
	var sig rawSignature
	if err := binary.Read(bytes.NewReader(data), binary.BigEndian, &sig); err != nil {
		return nil, err
	}
	return &sig, nil
}

func marshalRawSignature(rsig *rawSignature) []byte {
	var wbuf bytes.Buffer
	binary.Write(&wbuf, binary.BigEndian, rsig)
	return wbuf.Bytes()
}

func decryptPrivateKey(rek *rawEncryptedKey, passphrase []byte) (*PrivateKey, error) {
	var priv PrivateKey
	var xorkey []byte

	if rek.KDFRounds > 0 {
		xorkey = bcrypt_pbkdf.Key(passphrase, rek.Salt[:], int(rek.KDFRounds), ed25519.PrivateKeySize)
	} else {
		xorkey = make([]byte, ed25519.PrivateKeySize)
	}

	for i := range priv.Bytes {
		priv.Bytes[i] = rek.EncryptedKey[i] ^ xorkey[i]
	}

	privcs := checksum(priv.Bytes[:])
	if subtle.ConstantTimeCompare(privcs[:], rek.Checksum[:]) != 1 {
		return nil, errors.New("signify: invalid passphrase")
	}

	priv.Fingerprint = rek.Fingerprint

	return &priv, nil
}

func encryptPrivateKey(priv *PrivateKey, passphrase []byte) (*rawEncryptedKey, error) {
	var rke rawEncryptedKey

	copy(rke.PKAlgo[:], algoEd)
	copy(rke.KDFAlgo[:], algoBcrypt)
	rke.KDFRounds = defaultKDFRounds
	if _, err := io.ReadFull(rand.Reader, rke.Salt[:]); err != nil {
		return nil, err
	}
	rke.Checksum = checksum(priv.Bytes[:])
	if _, err := io.ReadFull(rand.Reader, rke.Fingerprint[:]); err != nil {
		return nil, err
	}

	xorkey := bcrypt_pbkdf.Key(passphrase, rke.Salt[:], int(rke.KDFRounds), ed25519.PrivateKeySize)
	for i := range rke.EncryptedKey {
		rke.EncryptedKey[i] = priv.Bytes[i] ^ xorkey[i]
	}

	return &rke, nil
}

func ParsePrivateKey(data, passphrase []byte) (*PrivateKey, error) {
	if !bytes.Equal(algoEd, data[:2]) {
		return nil, errors.New("signify: unknown public key algorithm")
	}
	if !bytes.Equal(algoBcrypt, data[2:4]) {
		return nil, errors.New("signify: unknown kdf algorithm")
	}

	rek, err := parseRawEncryptedKey(data)
	if err != nil {
		return nil, err
	}

	return decryptPrivateKey(rek, passphrase)
}

func MarshalPrivateKey(priv *PrivateKey, passphrase []byte) ([]byte, error) {
	rek, err := encryptPrivateKey(priv, passphrase)
	if err != nil {
		return nil, err
	}
	return marshalRawEncryptedKey(rek), nil
}

func ParsePublicKey(data []byte) (*PublicKey, error) {
	if !bytes.Equal(algoEd, data[:2]) {
		return nil, errors.New("signify: unknown public key algorithm")
	}

	rpk, err := parseRawPublicKey(data)
	if err != nil {
		return nil, err
	}

	pk := PublicKey{
		Bytes:       rpk.PublicKey,
		Fingerprint: rpk.Fingerprint,
	}
	return &pk, nil
}

func MarshalPublicKey(pub *PublicKey) []byte {
	var rpub rawPublicKey
	copy(rpub.PKAlgo[:], algoEd)
	rpub.PublicKey = pub.Bytes
	rpub.Fingerprint = pub.Fingerprint
	return marshalRawPublicKey(&rpub)
}

func ParseSignature(data []byte) (*Signature, error) {
	if !bytes.Equal(algoEd, data[:2]) {
		return nil, errors.New("signify: unknown public key algorithm")
	}

	rs, err := parseRawSignature(data)
	if err != nil {
		return nil, err
	}

	sig := Signature{
		Bytes:       rs.Signature,
		Fingerprint: rs.Fingerprint,
	}
	return &sig, nil
}

func MarshalSignature(sig *Signature) []byte {
	var rsig rawSignature
	copy(rsig.PKAlgo[:], algoEd)
	rsig.Signature = sig.Bytes
	rsig.Fingerprint = sig.Fingerprint
	return marshalRawSignature(&rsig)
}

func Sign(priv *PrivateKey, msg []byte) *Signature {
	return &Signature{
		Bytes:       *ed25519.Sign(&priv.Bytes, msg),
		Fingerprint: priv.Fingerprint,
	}
}

func Verify(pub *PublicKey, msg []byte, sig *Signature) bool {
	return ed25519.Verify(&pub.Bytes, msg, &sig.Bytes)
}

func checksum(d []byte) [8]byte {
	var chk [8]byte
	sha := sha512.New()
	sha.Write(d)
	copy(chk[:], sha.Sum(nil))
	return chk
}
<commit_msg>use array for algo*<commit_after>package signify

import (
	"bufio"
	"bytes"
	"crypto/rand"
	"crypto/sha512"
	"crypto/subtle"
	"encoding/base64"
	"encoding/binary"
	"errors"
	"fmt"
	"io"
	"strings"

	"github.com/agl/ed25519"
	"github.com/ebfe/bcrypt_pbkdf"
)

const (
	commentHdr       = "untrusted comment: "
	defaultKDFRounds = 42
)

var (
	algoEd     = [2]byte{'E', 'd'}
	algoBcrypt = [2]byte{'B', 'K'}
)

type PrivateKey struct {
	Bytes       [ed25519.PrivateKeySize]byte
	Fingerprint [8]byte
}

type PublicKey struct {
	Bytes       [ed25519.PublicKeySize]byte
	Fingerprint [8]byte
}

type Signature struct {
	Bytes       [ed25519.SignatureSize]byte
	Fingerprint [8]byte
}

type rawEncryptedKey struct {
	PKAlgo       [2]byte
	KDFAlgo      [2]byte
	KDFRounds    uint32
	Salt         [16]byte
	Checksum     [8]byte
	Fingerprint  [8]byte
	EncryptedKey [ed25519.PrivateKeySize]byte
}

type rawPublicKey struct {
	PKAlgo      [2]byte
	Fingerprint [8]byte
	PublicKey   [ed25519.PublicKeySize]byte
}

type rawSignature struct {
	PKAlgo      [2]byte
	Fingerprint [8]byte
	Signature   [ed25519.SignatureSize]byte
}

func ReadFile(r io.Reader) (comment string, content []byte, err error) {
	sc := bufio.NewScanner(r)

	if !sc.Scan() {
		return "", nil, fmt.Errorf("signify: read error %s", sc.Err())
	}
	comment = sc.Text()
	if !strings.HasPrefix(comment, commentHdr) {
		return "", nil, errors.New("signify: missing header")
	}
	comment = comment[len(commentHdr):]

	if !sc.Scan() {
		return "", nil, fmt.Errorf("signify: read error %s", sc.Err())
	}
	content, err = base64.StdEncoding.DecodeString(sc.Text())

	return
}

func WriteFile(w io.Writer, comment string, content []byte) error {
	b64 := base64.StdEncoding.EncodeToString(content)
	_, err := fmt.Fprintf(w, "%s%s\n%s\n", commentHdr, comment, b64)
	return err
}

func parseRawEncryptedKey(data []byte) (*rawEncryptedKey, error) {
	var ek rawEncryptedKey
	if err := binary.Read(bytes.NewReader(data), binary.BigEndian, &ek); err != nil {
		return nil, err
	}
	return &ek, nil
}

func marshalRawEncryptedKey(rek *rawEncryptedKey) []byte {
	var wbuf bytes.Buffer
	binary.Write(&wbuf, binary.BigEndian, rek)
	return wbuf.Bytes()
}

func parseRawPublicKey(data []byte) (*rawPublicKey, error) {
	var pub rawPublicKey
	if err := binary.Read(bytes.NewReader(data), binary.BigEndian, &pub); err != nil {
		return nil, err
	}
	return &pub, nil
}

func marshalRawPublicKey(rpub *rawPublicKey) []byte {
	var wbuf bytes.Buffer
	binary.Write(&wbuf, binary.BigEndian, rpub)
	return wbuf.Bytes()
}

func parseRawSignature(data []byte) (*rawSignature, error) {
	var sig rawSignature
	if err := binary.Read(bytes.NewReader(data), binary.BigEndian, &sig); err != nil {
		return nil, err
	}
	return &sig, nil
}

func marshalRawSignature(rsig *rawSignature) []byte {
	var wbuf bytes.Buffer
	binary.Write(&wbuf, binary.BigEndian, rsig)
	return wbuf.Bytes()
}

func decryptPrivateKey(rek *rawEncryptedKey, passphrase []byte) (*PrivateKey, error) {
	var priv PrivateKey
	var xorkey []byte

	if rek.KDFRounds > 0 {
		xorkey = bcrypt_pbkdf.Key(passphrase, rek.Salt[:], int(rek.KDFRounds), ed25519.PrivateKeySize)
	} else {
		xorkey = make([]byte, ed25519.PrivateKeySize)
	}

	for i := range priv.Bytes {
		priv.Bytes[i] = rek.EncryptedKey[i] ^ xorkey[i]
	}

	privcs := checksum(priv.Bytes[:])
	if subtle.ConstantTimeCompare(privcs[:], rek.Checksum[:]) != 1 {
		return nil, errors.New("signify: invalid passphrase")
	}

	priv.Fingerprint = rek.Fingerprint

	return &priv, nil
}

func encryptPrivateKey(priv *PrivateKey, passphrase []byte) (*rawEncryptedKey, error) {
	var rke rawEncryptedKey
	// FIXME: rand, kdfrounds param
	rke.PKAlgo = algoEd
	rke.KDFAlgo = algoBcrypt
	rke.KDFRounds = defaultKDFRounds
	if _, err := io.ReadFull(rand.Reader, rke.Salt[:]); err != nil {
		return nil, err
	}
	rke.Checksum = checksum(priv.Bytes[:])
	if _, err := io.ReadFull(rand.Reader, rke.Fingerprint[:]); err != nil {
		return nil, err
	}

	xorkey := bcrypt_pbkdf.Key(passphrase, rke.Salt[:], int(rke.KDFRounds), ed25519.PrivateKeySize)
	for i := range rke.EncryptedKey {
		rke.EncryptedKey[i] = priv.Bytes[i] ^ xorkey[i]
	}

	return &rke, nil
}

func ParsePrivateKey(data, passphrase []byte) (*PrivateKey, error) {
	if !bytes.Equal(algoEd[:], data[:2]) {
		return nil, errors.New("signify: unknown public key algorithm")
	}
	if !bytes.Equal(algoBcrypt[:], data[2:4]) {
		return nil, errors.New("signify: unknown kdf algorithm")
	}

	rek, err := parseRawEncryptedKey(data)
	if err != nil {
		return nil, err
	}

	return decryptPrivateKey(rek, passphrase)
}

func MarshalPrivateKey(priv *PrivateKey, passphrase []byte) ([]byte, error) {
	rek, err := encryptPrivateKey(priv, passphrase)
	if err != nil {
		return nil, err
	}
	return marshalRawEncryptedKey(rek), nil
}

func ParsePublicKey(data []byte) (*PublicKey, error) {
	if !bytes.Equal(algoEd[:], data[:2]) {
		return nil, errors.New("signify: unknown public key algorithm")
	}

	rpk, err := parseRawPublicKey(data)
	if err != nil {
		return nil, err
	}

	pk := PublicKey{
		Bytes:       rpk.PublicKey,
		Fingerprint: rpk.Fingerprint,
	}
	return &pk, nil
}

func MarshalPublicKey(pub *PublicKey) []byte {
	return marshalRawPublicKey(&rawPublicKey{
		PKAlgo: algoEd,
		PublicKey: pub.Bytes,
		Fingerprint: pub.Fingerprint,
	})
}

func ParseSignature(data []byte) (*Signature, error) {
	if !bytes.Equal(algoEd[:], data[:2]) {
		return nil, errors.New("signify: unknown public key algorithm")
	}

	rs, err := parseRawSignature(data)
	if err != nil {
		return nil, err
	}

	sig := Signature{
		Bytes:       rs.Signature,
		Fingerprint: rs.Fingerprint,
	}
	return &sig, nil
}

func MarshalSignature(sig *Signature) []byte {
	return marshalRawSignature(&rawSignature{
		PKAlgo: algoEd,
		Signature: sig.Bytes,
		Fingerprint: sig.Fingerprint,
	})
}

func Sign(priv *PrivateKey, msg []byte) *Signature {
	return &Signature{
		Bytes:       *ed25519.Sign(&priv.Bytes, msg),
		Fingerprint: priv.Fingerprint,
	}
}

func Verify(pub *PublicKey, msg []byte, sig *Signature) bool {
	return ed25519.Verify(&pub.Bytes, msg, &sig.Bytes)
}

func checksum(d []byte) [8]byte {
	var chk [8]byte
	sha := sha512.New()
	sha.Write(d)
	copy(chk[:], sha.Sum(nil))
	return chk
}
<|endoftext|>


<commit_before>package search

import (
	"errors"
	"math"
	"sort"

	gr "github.com/gonum/graph"
)

// Finds all shortest paths between start and goal
type AllPathFunc func(start, goal gr.Node) (path [][]gr.Node, cost float64, err error)

// Finds one path between start and goal, which it finds is arbitrary
type SinglePathFunc func(start, goal gr.Node) (path []gr.Node, cost float64, err error)

<<<<<<< HEAD
<<<<<<< HEAD
// This function returns two functions: one that will generate all shortest paths between two nodes with ids i and j, and one that will generate just one path.
=======
// This function returns two functions that will generate all shortest paths between two nodes with ids i and j.
>>>>>>> Basic skeleton implementation of FW, tests to follow
=======
// This function returns two functions: one that will generate all shortest paths between two nodes with ids i and j, and one that will generate just one path.
>>>>>>> Basic skeleton implementation of FW, tests to follow
//
// This algorithm requires the CrunchGraph interface which means it only works on nodes with dense ids since it uses an adjacency matrix.
//
// This algorithm isn't blazingly fast, but is relatively fast for the domain. It runs at O((number of vertices)^3), and successfully computes
<<<<<<< HEAD
<<<<<<< HEAD
=======
>>>>>>> Basic skeleton implementation of FW, tests to follow
// the cost between all pairs of vertices.
//
// Generating a single path should be pretty cheap after FW is done running. The AllPathFunc is likely to be considerably more expensive,
// simply because it has to effectively generate all combinations of known valid paths at each recursive step of the algorithm.
<<<<<<< HEAD
=======
// the cost between all pairs of vertices. Using just a little extra memory, we can remember all shortest paths
>>>>>>> Basic skeleton implementation of FW, tests to follow
=======
>>>>>>> Basic skeleton implementation of FW, tests to follow
func FloydWarshall(graph gr.CrunchGraph, cost func(gr.Node, gr.Node) float64) (AllPathFunc, SinglePathFunc) {
	graph.Crunch()
	_, _, _, _, _, _, cost, _ = setupFuncs(graph, cost, nil)

	nodes := denseNodeSorter(graph.NodeList())
	sort.Sort(nodes)
	numNodes := len(nodes)

	dist := make([]float64, numNodes*numNodes)
	next := make([][]int, numNodes*numNodes)
	for i := 0; i < numNodes; i++ {
		for j := 0; j < numNodes; j++ {
			if j != i {
				dist[i+j*numNodes] = math.Inf(1)
			}
		}
	}

	edges := graph.EdgeList()
	for _, edge := range edges {
		u := edge.Head().ID()
		v := edge.Tail().ID()

		dist[u+v*numNodes] = cost(edge.Head(), edge.Tail())
	}

	for k := 0; k < numNodes; k++ {
		for i := 0; i < numNodes; i++ {
			for j := 0; j < numNodes; j++ {
				if dist[i+j*numNodes] > dist[i+k*numNodes]+dist[k+j*numNodes] {
					dist[i+j*numNodes] = dist[i+k*numNodes] + dist[k+j*numNodes]

					// Avoid generating too much garbage by reusing the memory in the list if we've allocated one already
					if next[i+j*numNodes] == nil {
						next[i+j*numNodes] = []int{k}
					} else {
						next[i+j*numNodes] = next[i+j*numNodes][:1]
						next[i+j*numNodes][0] = k
					}
					// If the cost between the nodes happens to be the same cost as what we know, add the approriate
					// intermediary to the list
<<<<<<< HEAD
					//
					// NOTE: This may be a straight else, awaiting tests.
=======
>>>>>>> Basic skeleton implementation of FW, tests to follow
				} else if math.Abs(dist[i+k*numNodes]+dist[k+j*numNodes]-dist[i+j*numNodes]) < 0.00001 {
					next[i+j*numNodes] = append(next[i+j*numNodes], k)
				}
			}
		}
	}

	return genAllPathsFunc(dist, next, nodes), genSinglePathFunc(dist, next, nodes)
}

func genAllPathsFunc(dist []float64, next [][]int, nodes []gr.Node) func(start, goal gr.Node) ([][]gr.Node, float64, error) {
	numNodes := len(nodes)

	// A recursive function to reconstruct all possible paths.
	// It's not fast, but it's about as fast as can be reasonably expected
	var allPathFinder func(i, j int) ([][]gr.Node, error)
	allPathFinder = func(i, j int) ([][]gr.Node, error) {
		if dist[i+j*numNodes] == math.Inf(1) {
			return nil, errors.New("No path")
		}
		intermediates := next[i+j*numNodes]
		if intermediates == nil {
			return [][]gr.Node{}, nil
		}

		toReturn := make([][]gr.Node, 0, len(intermediates))

		// This step is a tad convoluted: we have some list of intermediates.
		// We can think of each intermediate as a path junction
		//
		// At this junction, we can find all the shortest paths back to i,
		// and all the shortest paths down to j. Since this is a junction,
		// any predecessor path that runs through this intermediate may
		// freely choose any successor path to get to j. They'll all be
		// of equivalent length.
		//
		// Thus, for each intermediate, we run through and join each predecessor
		// path with each successor path via its junction.
		for _, intermediate := range intermediates {

			// Find predecessors
			preds, err := allPathFinder(i, intermediate)
			if err != nil {
				return nil, err
			}

			// Join each predecessor with its junction
			for a := range preds {
				preds[a] = append(preds[a], nodes[intermediate])
			}

			// Find successors
			succs, err := allPathFinder(intermediate, j)
			if err != nil {
				return nil, err
			}

			// Join each successor with its predecessor at the junction.
			// (the copying stuff is because slices are reference types)
			for a := range succs {
				for b := range preds {
					path := make([]gr.Node, len(succs[a]), len(succs[a])+len(preds[b]))
					copy(path, succs[a])
					path = append(path, preds[b]...)
					toReturn = append(toReturn, path)
				}
			}

		}

		return toReturn, nil
	}

	return func(start, goal gr.Node) ([][]gr.Node, float64, error) {
		paths, err := allPathFinder(start.ID(), goal.ID())
		if err != nil {
			return nil, math.Inf(1), nil
		}

		for i := range paths {
			// Prepend start and postpend goal. pathFinder only does the intermediate steps
			paths[i] = append(paths[i], nil)
			copy(paths[i][1:], paths[i][:len(paths[i])-1])
			paths[i][0] = start
			paths[i] = append(paths[i], goal)
		}

		return paths, dist[start.ID()+goal.ID()*numNodes], nil
	}
}

func genSinglePathFunc(dist []float64, next [][]int, nodes []gr.Node) func(start, goal gr.Node) ([]gr.Node, float64, error) {
	numNodes := len(nodes)

	var singlePathFinder func(i, j int) ([]gr.Node, error)
	singlePathFinder = func(i, j int) ([]gr.Node, error) {
		if dist[i+j*numNodes] == math.Inf(1) {
			return nil, errors.New("No path")
		}

		intermediates := next[i+j*numNodes]
		if intermediates == nil {
			return []gr.Node{}, nil
		}

		intermediate := intermediates[0]
		path, err := singlePathFinder(i, intermediate)
		if err != nil {
			return nil, err
		}

		path = append(path, nodes[intermediate])
		p, err := singlePathFinder(intermediate, j)
		if err != nil {
			return nil, err
		}
		path = append(path, p...)

		return path, nil
	}

	return func(start, goal gr.Node) ([]gr.Node, float64, error) {
		path, err := singlePathFinder(start.ID(), goal.ID())
		if err != nil {
			return nil, math.Inf(1), nil
		}

		path = append(path, nil)
		copy(path[1:], path[:len(path)-1])
		path[0] = start
		path = append(path, goal)

		return path, dist[start.ID()+goal.ID()*numNodes], nil
	}
}
<commit_msg>Added a note<commit_after>package search

import (
	"errors"
	"math"
	"sort"

	gr "github.com/gonum/graph"
)

// Finds all shortest paths between start and goal
type AllPathFunc func(start, goal gr.Node) (path [][]gr.Node, cost float64, err error)

// Finds one path between start and goal, which it finds is arbitrary
type SinglePathFunc func(start, goal gr.Node) (path []gr.Node, cost float64, err error)

<<<<<<< HEAD
<<<<<<< HEAD
// This function returns two functions: one that will generate all shortest paths between two nodes with ids i and j, and one that will generate just one path.
=======
// This function returns two functions that will generate all shortest paths between two nodes with ids i and j.
>>>>>>> Basic skeleton implementation of FW, tests to follow
=======
// This function returns two functions: one that will generate all shortest paths between two nodes with ids i and j, and one that will generate just one path.
>>>>>>> Basic skeleton implementation of FW, tests to follow
//
// This algorithm requires the CrunchGraph interface which means it only works on nodes with dense ids since it uses an adjacency matrix.
//
// This algorithm isn't blazingly fast, but is relatively fast for the domain. It runs at O((number of vertices)^3), and successfully computes
<<<<<<< HEAD
<<<<<<< HEAD
=======
>>>>>>> Basic skeleton implementation of FW, tests to follow
// the cost between all pairs of vertices.
//
// Generating a single path should be pretty cheap after FW is done running. The AllPathFunc is likely to be considerably more expensive,
// simply because it has to effectively generate all combinations of known valid paths at each recursive step of the algorithm.
<<<<<<< HEAD
=======
// the cost between all pairs of vertices. Using just a little extra memory, we can remember all shortest paths
>>>>>>> Basic skeleton implementation of FW, tests to follow
=======
>>>>>>> Basic skeleton implementation of FW, tests to follow
func FloydWarshall(graph gr.CrunchGraph, cost func(gr.Node, gr.Node) float64) (AllPathFunc, SinglePathFunc) {
	graph.Crunch()
	_, _, _, _, _, _, cost, _ = setupFuncs(graph, cost, nil)

	nodes := denseNodeSorter(graph.NodeList())
	sort.Sort(nodes)
	numNodes := len(nodes)

	dist := make([]float64, numNodes*numNodes)
	next := make([][]int, numNodes*numNodes)
	for i := 0; i < numNodes; i++ {
		for j := 0; j < numNodes; j++ {
			if j != i {
				dist[i+j*numNodes] = math.Inf(1)
			}
		}
	}

	edges := graph.EdgeList()
	for _, edge := range edges {
		u := edge.Head().ID()
		v := edge.Tail().ID()

		dist[u+v*numNodes] = cost(edge.Head(), edge.Tail())
	}

	for k := 0; k < numNodes; k++ {
		for i := 0; i < numNodes; i++ {
			for j := 0; j < numNodes; j++ {
				if dist[i+j*numNodes] > dist[i+k*numNodes]+dist[k+j*numNodes] {
					dist[i+j*numNodes] = dist[i+k*numNodes] + dist[k+j*numNodes]

					// Avoid generating too much garbage by reusing the memory in the list if we've allocated one already
					if next[i+j*numNodes] == nil {
						next[i+j*numNodes] = []int{k}
					} else {
						next[i+j*numNodes] = next[i+j*numNodes][:1]
						next[i+j*numNodes][0] = k
					}
					// If the cost between the nodes happens to be the same cost as what we know, add the approriate
					// intermediary to the list
<<<<<<< HEAD
<<<<<<< HEAD
					//
					// NOTE: This may be a straight else, awaiting tests.
=======
>>>>>>> Basic skeleton implementation of FW, tests to follow
=======
					//
					// NOTE: This may be a straight else, awaiting tests.
>>>>>>> Added a note
				} else if math.Abs(dist[i+k*numNodes]+dist[k+j*numNodes]-dist[i+j*numNodes]) < 0.00001 {
					next[i+j*numNodes] = append(next[i+j*numNodes], k)
				}
			}
		}
	}

	return genAllPathsFunc(dist, next, nodes), genSinglePathFunc(dist, next, nodes)
}

func genAllPathsFunc(dist []float64, next [][]int, nodes []gr.Node) func(start, goal gr.Node) ([][]gr.Node, float64, error) {
	numNodes := len(nodes)

	// A recursive function to reconstruct all possible paths.
	// It's not fast, but it's about as fast as can be reasonably expected
	var allPathFinder func(i, j int) ([][]gr.Node, error)
	allPathFinder = func(i, j int) ([][]gr.Node, error) {
		if dist[i+j*numNodes] == math.Inf(1) {
			return nil, errors.New("No path")
		}
		intermediates := next[i+j*numNodes]
		if intermediates == nil {
			return [][]gr.Node{}, nil
		}

		toReturn := make([][]gr.Node, 0, len(intermediates))

		// This step is a tad convoluted: we have some list of intermediates.
		// We can think of each intermediate as a path junction
		//
		// At this junction, we can find all the shortest paths back to i,
		// and all the shortest paths down to j. Since this is a junction,
		// any predecessor path that runs through this intermediate may
		// freely choose any successor path to get to j. They'll all be
		// of equivalent length.
		//
		// Thus, for each intermediate, we run through and join each predecessor
		// path with each successor path via its junction.
		for _, intermediate := range intermediates {

			// Find predecessors
			preds, err := allPathFinder(i, intermediate)
			if err != nil {
				return nil, err
			}

			// Join each predecessor with its junction
			for a := range preds {
				preds[a] = append(preds[a], nodes[intermediate])
			}

			// Find successors
			succs, err := allPathFinder(intermediate, j)
			if err != nil {
				return nil, err
			}

			// Join each successor with its predecessor at the junction.
			// (the copying stuff is because slices are reference types)
			for a := range succs {
				for b := range preds {
					path := make([]gr.Node, len(succs[a]), len(succs[a])+len(preds[b]))
					copy(path, succs[a])
					path = append(path, preds[b]...)
					toReturn = append(toReturn, path)
				}
			}

		}

		return toReturn, nil
	}

	return func(start, goal gr.Node) ([][]gr.Node, float64, error) {
		paths, err := allPathFinder(start.ID(), goal.ID())
		if err != nil {
			return nil, math.Inf(1), nil
		}

		for i := range paths {
			// Prepend start and postpend goal. pathFinder only does the intermediate steps
			paths[i] = append(paths[i], nil)
			copy(paths[i][1:], paths[i][:len(paths[i])-1])
			paths[i][0] = start
			paths[i] = append(paths[i], goal)
		}

		return paths, dist[start.ID()+goal.ID()*numNodes], nil
	}
}

func genSinglePathFunc(dist []float64, next [][]int, nodes []gr.Node) func(start, goal gr.Node) ([]gr.Node, float64, error) {
	numNodes := len(nodes)

	var singlePathFinder func(i, j int) ([]gr.Node, error)
	singlePathFinder = func(i, j int) ([]gr.Node, error) {
		if dist[i+j*numNodes] == math.Inf(1) {
			return nil, errors.New("No path")
		}

		intermediates := next[i+j*numNodes]
		if intermediates == nil {
			return []gr.Node{}, nil
		}

		intermediate := intermediates[0]
		path, err := singlePathFinder(i, intermediate)
		if err != nil {
			return nil, err
		}

		path = append(path, nodes[intermediate])
		p, err := singlePathFinder(intermediate, j)
		if err != nil {
			return nil, err
		}
		path = append(path, p...)

		return path, nil
	}

	return func(start, goal gr.Node) ([]gr.Node, float64, error) {
		path, err := singlePathFinder(start.ID(), goal.ID())
		if err != nil {
			return nil, math.Inf(1), nil
		}

		path = append(path, nil)
		copy(path[1:], path[:len(path)-1])
		path[0] = start
		path = append(path, goal)

		return path, dist[start.ID()+goal.ID()*numNodes], nil
	}
}
<|endoftext|>


<commit_before>package option

type Option struct {
	NoColor          bool     `long:"nocolor" description:"Don't print color codes in results (Disabled by default)"`
	NoGroup          bool     `long:"nogroup" description:"Don't print file name at header (Disabled by default)"`
	FilesWithMatches bool     `short:"l" long:"files-with-matches" description:"Only print filenames that don't contain matches"`
	VcsIgnore        []string `long:"vcs-ignore" description:"VCS ignore files (Default: .gitignore, .hgignore)"`
	Ignore           []string `long:"ignore" description:"Ignore files/directories matching pattern"`
	IgnoreCase       bool     `short:"i" long:"ignore-case" description:"Match case insensitively"`
	SmartCase        bool     `short:"S" long:"smart-case" description:"Match case insensitively unless PATTERN contains uppercase characters"`
	FileSearchRegexp string   `short:"G" long:"file-search-regexp" description:"PATTERN Limit search to filenames matching PATTERN"`
	Depth            int      `long:"depth" default:"25" default-mask:"-" description:"Search up to NUM derectories deep (Default: 25)"`
	Proc             int      // Number of goroutine. Not user option.
	Version          bool     `long:"version" description:"Show version"`
}

func (self *Option) VcsIgnores() []string {
	if len(self.VcsIgnore) == 0 {
		self.VcsIgnore = []string{".gitignore", ".hgignore"}
	}
	return self.VcsIgnore
}
<commit_msg>Added a follow option.<commit_after>package option

type Option struct {
	NoColor          bool     `long:"nocolor" description:"Don't print color codes in results (Disabled by default)"`
	NoGroup          bool     `long:"nogroup" description:"Don't print file name at header (Disabled by default)"`
	FilesWithMatches bool     `short:"l" long:"files-with-matches" description:"Only print filenames that don't contain matches"`
	VcsIgnore        []string `long:"vcs-ignore" description:"VCS ignore files (Default: .gitignore, .hgignore)"`
	Ignore           []string `long:"ignore" description:"Ignore files/directories matching pattern"`
	IgnoreCase       bool     `short:"i" long:"ignore-case" description:"Match case insensitively"`
	SmartCase        bool     `short:"S" long:"smart-case" description:"Match case insensitively unless PATTERN contains uppercase characters"`
	FileSearchRegexp string   `short:"G" long:"file-search-regexp" description:"PATTERN Limit search to filenames matching PATTERN"`
	Depth            int      `long:"depth" default:"25" default-mask:"-" description:"Search up to NUM derectories deep (Default: 25)"`
	Follow           bool     `short:"f" long:"follow" description:"Follow symlinks"`
	Proc             int      // Number of goroutine. Not user option.
	Version          bool     `long:"version" description:"Show version"`
}

func (self *Option) VcsIgnores() []string {
	if len(self.VcsIgnore) == 0 {
		self.VcsIgnore = []string{".gitignore", ".hgignore"}
	}
	return self.VcsIgnore
}
<|endoftext|>


<commit_before>package semtech

import (
	"errors"
	"testing"
	"time"

	. "github.com/smartystreets/goconvey/convey"
)

func TestDatR(t *testing.T) {
	Convey("Given an empty DatR", t, func() {
		var d DatR

		Convey("Then MarshalJSON returns '0'", func() {
			b, err := d.MarshalJSON()
			So(err, ShouldBeNil)
			So(string(b), ShouldEqual, "0")
		})

		Convey("Given LoRa=SF7BW125", func() {
			d.LoRa = "SF7BW125"
			Convey("Then MarshalJSON returns '\"SF7BW125\"'", func() {
				b, err := d.MarshalJSON()
				So(err, ShouldBeNil)
				So(string(b), ShouldEqual, `"SF7BW125"`)
			})
		})

		Convey("Given FSK=1234", func() {
			d.FSK = 1234
			Convey("Then MarshalJSON returns '1234'", func() {
				b, err := d.MarshalJSON()
				So(err, ShouldBeNil)
				So(string(b), ShouldEqual, "1234")
			})
		})

		Convey("Given the string '1234'", func() {
			s := "1234"
			Convey("Then UnmarshalJSON returns FSK=1234", func() {
				err := d.UnmarshalJSON([]byte(s))
				So(err, ShouldBeNil)
				So(d.FSK, ShouldEqual, 1234)
			})
		})

		Convey("Given the string '\"SF7BW125\"'", func() {
			s := `"SF7BW125"`
			Convey("Then UnmarshalJSON returns LoRa=SF7BW125", func() {
				err := d.UnmarshalJSON([]byte(s))
				So(err, ShouldBeNil)
				So(d.LoRa, ShouldEqual, "SF7BW125")
			})
		})
	})
}

func TestCompactTime(t *testing.T) {
	Convey("Given the date 'Mon Jan 2 15:04:05 -0700 MST 2006'", t, func() {
		tStr := "Mon Jan 2 15:04:05 -0700 MST 2006"
		ts, err := time.Parse(tStr, tStr)
		So(err, ShouldBeNil)

		Convey("MarshalJSON returns '\"2006-01-02T22:04:05Z\"'", func() {

			b, err := CompactTime(ts).MarshalJSON()
			So(err, ShouldBeNil)
			So(string(b), ShouldEqual, `"2006-01-02T22:04:05Z"`)
		})

		Convey("Given the JSON value of the date (\"2006-01-02T22:04:05Z\")", func() {
			s := `"2006-01-02T22:04:05Z"`
			Convey("UnmarshalJSON returns the correct date", func() {
				var ct CompactTime
				err := ct.UnmarshalJSON([]byte(s))
				So(err, ShouldBeNil)
				So(time.Time(ct).Equal(ts), ShouldBeTrue)
			})
		})
	})
}

func TestGetPacketType(t *testing.T) {
	Convey("Given an empty slice []byte{}", t, func() {
		var b []byte

		Convey("Then GetPacketType returns an error (length)", func() {
			_, err := GetPacketType(b)
			So(err, ShouldResemble, errors.New("lorawan/semtech: at least 4 bytes of data are expected"))
		})

		Convey("Given the slice []byte{2, 1, 3, 4}", func() {
			b = []byte{2, 1, 3, 4}
			Convey("Then GetPacketType returns an error (protocol version)", func() {
				_, err := GetPacketType(b)
				So(err, ShouldResemble, errors.New("lorawan/semtech: unknown protocol version"))
			})
		})

		Convey("Given the slice []byte{1, 1, 3, 4}", func() {
			b = []byte{1, 1, 3, 4}
			Convey("Then GetPacketType returns PullACK", func() {
				t, err := GetPacketType(b)
				So(err, ShouldBeNil)
				So(t, ShouldEqual, PullACK)
			})
		})
	})
}

func TestPushDataPacket(t *testing.T) {
	Convey("Given an empty PushDataPacket", t, func() {
		var p PushDataPacket
		Convey("Then MarshalBinary returns []byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 123, 125}", func() {
			b, err := p.MarshalBinary()
			So(err, ShouldBeNil)
			So(b, ShouldResemble, []byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 123, 125})
		})

		Convey("Given ProtocolVersion=1, RandomToken=123, GatewayMAC=[]{1, 2, 3, 4, 5, 6, 7, 8}", func() {
			p = PushDataPacket{
				ProtocolVersion: ProtocolVersion1,
				RandomToken:     123,
				GatewayMAC:      [8]byte{1, 2, 3, 4, 5, 6, 7, 8},
			}
			Convey("Then MarshalBinary returns []byte{1, 123, 0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 123, 125}", func() {
				b, err := p.MarshalBinary()
				So(err, ShouldBeNil)
				So(b, ShouldResemble, []byte{1, 123, 0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 123, 125})
			})
		})

		Convey("Given the slice []byte{1, 123, 0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 123, 125}", func() {
			b := []byte{1, 123, 0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 123, 125}
			Convey("Then UnmarshalBinary returns ProtocolVersion=1, RandomToken=123, GatewayMAC=[]{1, 2, 3, 4, 5, 6, 7, 8}", func() {
				err := p.UnmarshalBinary(b)
				So(err, ShouldBeNil)
				So(p, ShouldResemble, PushDataPacket{
					ProtocolVersion: ProtocolVersion1,
					RandomToken:     123,
					GatewayMAC:      [8]byte{1, 2, 3, 4, 5, 6, 7, 8},
				})
			})
		})
	})
}

func TestPushACKPacket(t *testing.T) {
	Convey("Given an empty PushACKPacket", t, func() {
		var p PushACKPacket
		Convey("Then MarshalBinary returns []byte{0, 0, 0, 1}", func() {
			b, err := p.MarshalBinary()
			So(err, ShouldBeNil)
			So(b, ShouldResemble, []byte{0, 0, 0, 1})
		})

		Convey("Given ProtocolVersion=1, RandomToken=123", func() {
			p = PushACKPacket{
				ProtocolVersion: ProtocolVersion1,
				RandomToken:     123,
			}
			Convey("Then MarshalBinary returns []byte{1, 123, 0, 1}", func() {
				b, err := p.MarshalBinary()
				So(err, ShouldBeNil)
				So(b, ShouldResemble, []byte{1, 123, 0, 1})
			})
		})

		Convey("Given the slice []byte{1, 123, 0, 1}", func() {
			Convey("Then UnmarshalBinary returns ProtocolVersion=1, RandomToken=123", func() {
				b := []byte{1, 123, 0, 1}
				err := p.UnmarshalBinary(b)
				So(err, ShouldBeNil)
				So(p, ShouldResemble, PushACKPacket{
					ProtocolVersion: ProtocolVersion1,
					RandomToken:     123,
				})
			})
		})
	})
}

func TestPullDataPacket(t *testing.T) {
	Convey("Given an empty PullDataPacket", t, func() {
		var p PullDataPacket
		Convey("Then MarshalBinary returns []byte{0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0}", func() {
			b, err := p.MarshalBinary()
			So(err, ShouldBeNil)
			So(b, ShouldResemble, []byte{0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0})
		})

		Convey("Given ProtocolVersion=1, RandomToken=123, GatewayMAC=[]byte{1, 2, 3, 4, 5, 6, 8, 8}", func() {
			p = PullDataPacket{
				ProtocolVersion: 1,
				RandomToken:     123,
				GatewayMAC:      [8]byte{1, 2, 3, 4, 5, 6, 7, 8},
			}
			Convey("Then MarshalBinary returns []byte{1, 123, 0, 2, 1, 2, 3, 4, 5, 6, 7, 8}", func() {
				b, err := p.MarshalBinary()
				So(err, ShouldBeNil)
				So(b, ShouldResemble, []byte{1, 123, 0, 2, 1, 2, 3, 4, 5, 6, 7, 8})
			})
		})
	})
}

func TestPullACKPacket(t *testing.T) {
	Convey("Given an empty PullACKPacket", t, func() {
		var p PullACKPacket
		Convey("Then MarshalBinary returns []byte{0, 0, 0, 4}", func() {
			b, err := p.MarshalBinary()
			So(err, ShouldBeNil)
			So(b, ShouldResemble, []byte{0, 0, 0, 4})
		})

		Convey("Given ProtocolVersion=1, RandomToken=123}", func() {
			p = PullACKPacket{
				ProtocolVersion: 1,
				RandomToken:     123,
			}
			Convey("Then MarshalBinary returns []byte{1, 123, 0, 4}", func() {
				b, err := p.MarshalBinary()
				So(err, ShouldBeNil)
				So(b, ShouldResemble, []byte{1, 123, 0, 4})
			})
		})
	})
}

func TestPullRespPacket(t *testing.T) {
	Convey("Given an empty PullRespPacket", t, func() {
		var p PullRespPacket
		Convey("Then MarshalBinary returns []byte{0, 0, 0, 3} as first 4 bytes", func() {
			b, err := p.MarshalBinary()
			So(err, ShouldBeNil)
			So(b[0:4], ShouldResemble, []byte{0, 0, 0, 3})
		})

		Convey("Given ProtocolVersion=1, RandomToken=123", func() {
			p = PullRespPacket{
				ProtocolVersion: 1,
				RandomToken:     123,
			}
			Convey("Then MarshalBinary returns []byte{1, 123, 0, 3} as first 4 bytes", func() {
				b, err := p.MarshalBinary()
				So(err, ShouldBeNil)
				So(b[0:4], ShouldResemble, []byte{1, 123, 0, 3})
			})
		})

		Convey("Given the slice []byte{1, 123, 0, 3, 123, 125}", func() {
			b := []byte{1, 123, 0, 3, 123, 125}
			Convey("Then UnmarshalBinary returns ProtocolVersion=1, RandomToken=123", func() {
				err := p.UnmarshalBinary(b)
				So(err, ShouldBeNil)
				So(p, ShouldResemble, PullRespPacket{
					ProtocolVersion: 1,
					RandomToken:     123,
				})
			})
		})
	})
}
<commit_msg>Add missing tests.<commit_after>package semtech

import (
	"errors"
	"testing"
	"time"

	. "github.com/smartystreets/goconvey/convey"
)

func TestDatR(t *testing.T) {
	Convey("Given an empty DatR", t, func() {
		var d DatR

		Convey("Then MarshalJSON returns '0'", func() {
			b, err := d.MarshalJSON()
			So(err, ShouldBeNil)
			So(string(b), ShouldEqual, "0")
		})

		Convey("Given LoRa=SF7BW125", func() {
			d.LoRa = "SF7BW125"
			Convey("Then MarshalJSON returns '\"SF7BW125\"'", func() {
				b, err := d.MarshalJSON()
				So(err, ShouldBeNil)
				So(string(b), ShouldEqual, `"SF7BW125"`)
			})
		})

		Convey("Given FSK=1234", func() {
			d.FSK = 1234
			Convey("Then MarshalJSON returns '1234'", func() {
				b, err := d.MarshalJSON()
				So(err, ShouldBeNil)
				So(string(b), ShouldEqual, "1234")
			})
		})

		Convey("Given the string '1234'", func() {
			s := "1234"
			Convey("Then UnmarshalJSON returns FSK=1234", func() {
				err := d.UnmarshalJSON([]byte(s))
				So(err, ShouldBeNil)
				So(d.FSK, ShouldEqual, 1234)
			})
		})

		Convey("Given the string '\"SF7BW125\"'", func() {
			s := `"SF7BW125"`
			Convey("Then UnmarshalJSON returns LoRa=SF7BW125", func() {
				err := d.UnmarshalJSON([]byte(s))
				So(err, ShouldBeNil)
				So(d.LoRa, ShouldEqual, "SF7BW125")
			})
		})
	})
}

func TestCompactTime(t *testing.T) {
	Convey("Given the date 'Mon Jan 2 15:04:05 -0700 MST 2006'", t, func() {
		tStr := "Mon Jan 2 15:04:05 -0700 MST 2006"
		ts, err := time.Parse(tStr, tStr)
		So(err, ShouldBeNil)

		Convey("MarshalJSON returns '\"2006-01-02T22:04:05Z\"'", func() {

			b, err := CompactTime(ts).MarshalJSON()
			So(err, ShouldBeNil)
			So(string(b), ShouldEqual, `"2006-01-02T22:04:05Z"`)
		})

		Convey("Given the JSON value of the date (\"2006-01-02T22:04:05Z\")", func() {
			s := `"2006-01-02T22:04:05Z"`
			Convey("UnmarshalJSON returns the correct date", func() {
				var ct CompactTime
				err := ct.UnmarshalJSON([]byte(s))
				So(err, ShouldBeNil)
				So(time.Time(ct).Equal(ts), ShouldBeTrue)
			})
		})
	})
}

func TestGetPacketType(t *testing.T) {
	Convey("Given an empty slice []byte{}", t, func() {
		var b []byte

		Convey("Then GetPacketType returns an error (length)", func() {
			_, err := GetPacketType(b)
			So(err, ShouldResemble, errors.New("lorawan/semtech: at least 4 bytes of data are expected"))
		})

		Convey("Given the slice []byte{2, 1, 3, 4}", func() {
			b = []byte{2, 1, 3, 4}
			Convey("Then GetPacketType returns an error (protocol version)", func() {
				_, err := GetPacketType(b)
				So(err, ShouldResemble, errors.New("lorawan/semtech: unknown protocol version"))
			})
		})

		Convey("Given the slice []byte{1, 1, 3, 4}", func() {
			b = []byte{1, 1, 3, 4}
			Convey("Then GetPacketType returns PullACK", func() {
				t, err := GetPacketType(b)
				So(err, ShouldBeNil)
				So(t, ShouldEqual, PullACK)
			})
		})
	})
}

func TestPushDataPacket(t *testing.T) {
	Convey("Given an empty PushDataPacket", t, func() {
		var p PushDataPacket
		Convey("Then MarshalBinary returns []byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 123, 125}", func() {
			b, err := p.MarshalBinary()
			So(err, ShouldBeNil)
			So(b, ShouldResemble, []byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 123, 125})
		})

		Convey("Given ProtocolVersion=1, RandomToken=123, GatewayMAC=[]{1, 2, 3, 4, 5, 6, 7, 8}", func() {
			p = PushDataPacket{
				ProtocolVersion: ProtocolVersion1,
				RandomToken:     123,
				GatewayMAC:      [8]byte{1, 2, 3, 4, 5, 6, 7, 8},
			}
			Convey("Then MarshalBinary returns []byte{1, 123, 0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 123, 125}", func() {
				b, err := p.MarshalBinary()
				So(err, ShouldBeNil)
				So(b, ShouldResemble, []byte{1, 123, 0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 123, 125})
			})
		})

		Convey("Given the slice []byte{1, 123, 0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 123, 125}", func() {
			b := []byte{1, 123, 0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 123, 125}
			Convey("Then UnmarshalBinary returns ProtocolVersion=1, RandomToken=123, GatewayMAC=[]{1, 2, 3, 4, 5, 6, 7, 8}", func() {
				err := p.UnmarshalBinary(b)
				So(err, ShouldBeNil)
				So(p, ShouldResemble, PushDataPacket{
					ProtocolVersion: ProtocolVersion1,
					RandomToken:     123,
					GatewayMAC:      [8]byte{1, 2, 3, 4, 5, 6, 7, 8},
				})
			})
		})
	})
}

func TestPushACKPacket(t *testing.T) {
	Convey("Given an empty PushACKPacket", t, func() {
		var p PushACKPacket
		Convey("Then MarshalBinary returns []byte{0, 0, 0, 1}", func() {
			b, err := p.MarshalBinary()
			So(err, ShouldBeNil)
			So(b, ShouldResemble, []byte{0, 0, 0, 1})
		})

		Convey("Given ProtocolVersion=1, RandomToken=123", func() {
			p = PushACKPacket{
				ProtocolVersion: ProtocolVersion1,
				RandomToken:     123,
			}
			Convey("Then MarshalBinary returns []byte{1, 123, 0, 1}", func() {
				b, err := p.MarshalBinary()
				So(err, ShouldBeNil)
				So(b, ShouldResemble, []byte{1, 123, 0, 1})
			})
		})

		Convey("Given the slice []byte{1, 123, 0, 1}", func() {
			Convey("Then UnmarshalBinary returns ProtocolVersion=1, RandomToken=123", func() {
				b := []byte{1, 123, 0, 1}
				err := p.UnmarshalBinary(b)
				So(err, ShouldBeNil)
				So(p, ShouldResemble, PushACKPacket{
					ProtocolVersion: ProtocolVersion1,
					RandomToken:     123,
				})
			})
		})
	})
}

func TestPullDataPacket(t *testing.T) {
	Convey("Given an empty PullDataPacket", t, func() {
		var p PullDataPacket
		Convey("Then MarshalBinary returns []byte{0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0}", func() {
			b, err := p.MarshalBinary()
			So(err, ShouldBeNil)
			So(b, ShouldResemble, []byte{0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 0})
		})

		Convey("Given ProtocolVersion=1, RandomToken=123, GatewayMAC=[]byte{1, 2, 3, 4, 5, 6, 8, 8}", func() {
			p = PullDataPacket{
				ProtocolVersion: 1,
				RandomToken:     123,
				GatewayMAC:      [8]byte{1, 2, 3, 4, 5, 6, 7, 8},
			}
			Convey("Then MarshalBinary returns []byte{1, 123, 0, 2, 1, 2, 3, 4, 5, 6, 7, 8}", func() {
				b, err := p.MarshalBinary()
				So(err, ShouldBeNil)
				So(b, ShouldResemble, []byte{1, 123, 0, 2, 1, 2, 3, 4, 5, 6, 7, 8})
			})
		})

		Convey("Given the slice []byte{1, 123, 0, 2, 1, 2, 3, 4, 5, 6, 7, 8}", func() {
			b := []byte{1, 123, 0, 2, 1, 2, 3, 4, 5, 6, 7, 8}
			Convey("Then UnmarshalBinary returns ProtocolVersion=1, RandomToken=123, GatewayMAC=[]byte{1, 2, 3, 4, 5, 6, 8, 8}", func() {
				err := p.UnmarshalBinary(b)
				So(err, ShouldBeNil)
				So(p, ShouldResemble, PullDataPacket{
					ProtocolVersion: 1,
					RandomToken:     123,
					GatewayMAC:      [8]byte{1, 2, 3, 4, 5, 6, 7, 8},
				})
			})
		})
	})
}

func TestPullACKPacket(t *testing.T) {
	Convey("Given an empty PullACKPacket", t, func() {
		var p PullACKPacket
		Convey("Then MarshalBinary returns []byte{0, 0, 0, 4}", func() {
			b, err := p.MarshalBinary()
			So(err, ShouldBeNil)
			So(b, ShouldResemble, []byte{0, 0, 0, 4})
		})

		Convey("Given ProtocolVersion=1, RandomToken=123}", func() {
			p = PullACKPacket{
				ProtocolVersion: 1,
				RandomToken:     123,
			}
			Convey("Then MarshalBinary returns []byte{1, 123, 0, 4}", func() {
				b, err := p.MarshalBinary()
				So(err, ShouldBeNil)
				So(b, ShouldResemble, []byte{1, 123, 0, 4})
			})
		})

		Convey("Given the slice []byte{1, 123, 0, 4}", func() {
			b := []byte{1, 123, 0, 4}
			Convey("Then UnmarshalBinary returns ProtocolVersion=1, RandomToken=123", func() {
				err := p.UnmarshalBinary(b)
				So(err, ShouldBeNil)
				So(p, ShouldResemble, PullACKPacket{
					ProtocolVersion: 1,
					RandomToken:     123,
				})
			})
		})
	})
}

func TestPullRespPacket(t *testing.T) {
	Convey("Given an empty PullRespPacket", t, func() {
		var p PullRespPacket
		Convey("Then MarshalBinary returns []byte{0, 0, 0, 3} as first 4 bytes", func() {
			b, err := p.MarshalBinary()
			So(err, ShouldBeNil)
			So(b[0:4], ShouldResemble, []byte{0, 0, 0, 3})
		})

		Convey("Given ProtocolVersion=1, RandomToken=123", func() {
			p = PullRespPacket{
				ProtocolVersion: 1,
				RandomToken:     123,
			}
			Convey("Then MarshalBinary returns []byte{1, 123, 0, 3} as first 4 bytes", func() {
				b, err := p.MarshalBinary()
				So(err, ShouldBeNil)
				So(b[0:4], ShouldResemble, []byte{1, 123, 0, 3})
			})
		})

		Convey("Given the slice []byte{1, 123, 0, 3, 123, 125}", func() {
			b := []byte{1, 123, 0, 3, 123, 125}
			Convey("Then UnmarshalBinary returns ProtocolVersion=1, RandomToken=123", func() {
				err := p.UnmarshalBinary(b)
				So(err, ShouldBeNil)
				So(p, ShouldResemble, PullRespPacket{
					ProtocolVersion: 1,
					RandomToken:     123,
				})
			})
		})
	})
}
<|endoftext|>


<commit_before>// Copyright 2019 gf Author(https://github.com/gogf/gf). All Rights Reserved.
//
// This Source Code Form is subject to the terms of the MIT License.
// If a copy of the MIT was not distributed with this file,
// You can obtain one at https://github.com/gogf/gf.

package gspath_test

import (
	"testing"

	"github.com/gogf/gf/os/gfile"
	"github.com/gogf/gf/os/gspath"
	"github.com/gogf/gf/test/gtest"
)

func TestSPath_Api(t *testing.T) {
	gtest.Case(t, func() {
		pwd := gfile.Pwd()
		root := pwd + gfile.Separator
		gfile.Create(root + "gf_tmp" + gfile.Separator + "gf.txt")
		defer gfile.Remove(root + "gf_tmp")
		fp, isDir := gspath.Search(root, "gf_tmp")
		gtest.Assert(fp, root+"gf_tmp")
		gtest.Assert(isDir, true)
		fp, isDir = gspath.Search(root, "gf_tmp", "gf.txt")
		gtest.Assert(fp, root+"gf_tmp"+gfile.Separator+"gf.txt")
		gtest.Assert(isDir, false)

		fp, isDir = gspath.SearchWithCache(root, "gf_tmp")
		gtest.Assert(fp, root+"gf_tmp")
		gtest.Assert(isDir, true)
		fp, isDir = gspath.SearchWithCache(root, "gf_tmp", "gf.txt")
		gtest.Assert(fp, root+"gf_tmp"+gfile.Separator+"gf.txt")
		gtest.Assert(isDir, false)
	})
}

func TestSPath_Basic(t *testing.T) {
	gtest.Case(t, func() {
		pwd := gfile.Pwd()
		root := pwd + gfile.Separator
		gfile.Create(root + "gf_tmp" + gfile.Separator + "gf.txt")
		defer gfile.Remove(root + "gf_tmp")
		gsp := gspath.New(root, false)
		realPath, err := gsp.Add(root + "gf_tmp")
		gtest.Assert(err, nil)
		gtest.Assert(realPath, root+"gf_tmp")
		realPath, err = gsp.Add("gf_tmp1")
		gtest.Assert(err != nil, true)
		gtest.Assert(realPath, "")
		realPath, err = gsp.Add(root + "gf_tmp" + gfile.Separator + "gf.txt")
		gtest.Assert(err != nil, true)
		gtest.Assert(realPath, "")
		gsp.Remove("gf_tmp1")
		gtest.Assert(gsp.Size(), 2)
		gtest.Assert(len(gsp.Paths()), 2)
		gtest.Assert(len(gsp.AllPaths()), 0)
		realPath, err = gsp.Set(root + "gf_tmp1")
		gtest.Assert(err != nil, true)
		gtest.Assert(realPath, "")
		realPath, err = gsp.Set(root + "gf_tmp" + gfile.Separator + "gf.txt")
		gtest.Assert(err != nil, true)
		gtest.Assert(realPath, "")
		gsp.Set(root)
		fp, isDir := gsp.Search("gf_tmp")
		gtest.Assert(fp, root+"gf_tmp")
		gtest.Assert(isDir, true)
		fp, isDir = gsp.Search("gf_tmp", "gf.txt")
		gtest.Assert(fp, root+"gf_tmp"+gfile.Separator+"gf.txt")
		gtest.Assert(isDir, false)
		fp, isDir = gsp.Search("/", "gf.txt")
		gtest.Assert(fp, root+gfile.Separator)
		gtest.Assert(isDir, true)

		gsp = gspath.New(root, true)
		realPath, err = gsp.Add(root + "gf_tmp")
		gtest.Assert(err, nil)
		gtest.Assert(realPath, root+"gf_tmp")

		gfile.Mkdir(root + "gf_tmp1")
		gfile.Rename(root+"gf_tmp1", root+"gf_tmp2")
		gfile.Rename(root+"gf_tmp2", root+"gf_tmp1")
		defer gfile.Remove(root + "gf_tmp1")
		realPath, err = gsp.Add("gf_tmp1")
		gtest.Assert(err != nil, false)
		gtest.Assert(realPath, root+"gf_tmp1")
		realPath, err = gsp.Add("gf_tmp3")
		gtest.Assert(err != nil, true)
		gtest.Assert(realPath, "")
		gsp.Remove(root + "gf_tmp")
		gsp.Remove(root + "gf_tmp1")
		gsp.Remove(root + "gf_tmp3")
		gtest.Assert(gsp.Size(), 3)
		gtest.Assert(len(gsp.Paths()), 3)
		gsp.AllPaths()
		gsp.Set(root)
		fp, isDir = gsp.Search("gf_tmp")
		gtest.Assert(fp, root+"gf_tmp")
		gtest.Assert(isDir, true)
		fp, isDir = gsp.Search("gf_tmp", "gf.txt")
		gtest.Assert(fp, root+"gf_tmp"+gfile.Separator+"gf.txt")
		gtest.Assert(isDir, false)
		fp, isDir = gsp.Search("/", "gf.txt")
		gtest.Assert(fp, pwd)
		gtest.Assert(isDir, true)
	})
}
<commit_msg>improve unit test cases for gspath<commit_after>// Copyright 2019 gf Author(https://github.com/gogf/gf). All Rights Reserved.
//
// This Source Code Form is subject to the terms of the MIT License.
// If a copy of the MIT was not distributed with this file,
// You can obtain one at https://github.com/gogf/gf.

package gspath_test

import (
	"testing"

	"github.com/gogf/gf/os/gfile"
	"github.com/gogf/gf/os/gspath"
	"github.com/gogf/gf/test/gtest"
)

func TestSPath_Api(t *testing.T) {
	gtest.Case(t, func() {
		pwd := gfile.Pwd()
		root := pwd
		gfile.Create(gfile.Join(root, "gf_tmp", "gf.txt"))
		defer gfile.Remove(gfile.Join(root, "gf_tmp"))
		fp, isDir := gspath.Search(root, "gf_tmp")
		gtest.Assert(fp, gfile.Join(root, "gf_tmp"))
		gtest.Assert(isDir, true)
		fp, isDir = gspath.Search(root, "gf_tmp", "gf.txt")
		gtest.Assert(fp, gfile.Join(root, "gf_tmp", "gf.txt"))
		gtest.Assert(isDir, false)

		fp, isDir = gspath.SearchWithCache(root, "gf_tmp")
		gtest.Assert(fp, gfile.Join(root, "gf_tmp"))
		gtest.Assert(isDir, true)
		fp, isDir = gspath.SearchWithCache(root, "gf_tmp", "gf.txt")
		gtest.Assert(fp, gfile.Join(root, "gf_tmp", "gf.txt"))
		gtest.Assert(isDir, false)
	})
}

func TestSPath_Basic(t *testing.T) {
	gtest.Case(t, func() {
		pwd := gfile.Pwd()
		root := pwd

		gfile.Create(gfile.Join(root, "gf_tmp", "gf.txt"))
		defer gfile.Remove(gfile.Join(root, "gf_tmp"))
		gsp := gspath.New(root, false)
		realPath, err := gsp.Add(gfile.Join(root, "gf_tmp"))
		gtest.Assert(err, nil)
		gtest.Assert(realPath, gfile.Join(root, "gf_tmp"))
		realPath, err = gsp.Add("gf_tmp1")
		gtest.Assert(err != nil, true)
		gtest.Assert(realPath, "")
		realPath, err = gsp.Add(gfile.Join(root, "gf_tmp", "gf.txt"))
		gtest.Assert(err != nil, true)
		gtest.Assert(realPath, "")
		gsp.Remove("gf_tmp1")
		gtest.Assert(gsp.Size(), 2)
		gtest.Assert(len(gsp.Paths()), 2)
		gtest.Assert(len(gsp.AllPaths()), 0)
		realPath, err = gsp.Set(gfile.Join(root, "gf_tmp1"))
		gtest.Assert(err != nil, true)
		gtest.Assert(realPath, "")
		realPath, err = gsp.Set(gfile.Join(root, "gf_tmp", "gf.txt"))
		gtest.AssertNE(err, nil)
		gtest.Assert(realPath, "")

		realPath, err = gsp.Set(root)
		gtest.Assert(err, nil)
		gtest.Assert(realPath, root)

		fp, isDir := gsp.Search("gf_tmp")
		gtest.Assert(fp, gfile.Join(root, "gf_tmp"))
		gtest.Assert(isDir, true)
		fp, isDir = gsp.Search("gf_tmp", "gf.txt")
		gtest.Assert(fp, gfile.Join(root, "gf_tmp", "gf.txt"))
		gtest.Assert(isDir, false)
		fp, isDir = gsp.Search("/", "gf.txt")
		gtest.Assert(fp, root)
		gtest.Assert(isDir, true)

		gsp = gspath.New(root, true)
		realPath, err = gsp.Add(gfile.Join(root, "gf_tmp"))
		gtest.Assert(err, nil)
		gtest.Assert(realPath, gfile.Join(root, "gf_tmp"))

		gfile.Mkdir(gfile.Join(root, "gf_tmp1"))
		gfile.Rename(gfile.Join(root, "gf_tmp1"), gfile.Join(root, "gf_tmp2"))
		gfile.Rename(gfile.Join(root, "gf_tmp2"), gfile.Join(root, "gf_tmp1"))
		defer gfile.Remove(gfile.Join(root, "gf_tmp1"))
		realPath, err = gsp.Add("gf_tmp1")
		gtest.Assert(err != nil, false)
		gtest.Assert(realPath, gfile.Join(root, "gf_tmp1"))
		realPath, err = gsp.Add("gf_tmp3")
		gtest.Assert(err != nil, true)
		gtest.Assert(realPath, "")
		gsp.Remove(gfile.Join(root, "gf_tmp"))
		gsp.Remove(gfile.Join(root, "gf_tmp1"))
		gsp.Remove(gfile.Join(root, "gf_tmp3"))
		gtest.Assert(gsp.Size(), 3)
		gtest.Assert(len(gsp.Paths()), 3)
		gsp.AllPaths()
		gsp.Set(root)
		fp, isDir = gsp.Search("gf_tmp")
		gtest.Assert(fp, gfile.Join(root, "gf_tmp"))
		gtest.Assert(isDir, true)
		fp, isDir = gsp.Search("gf_tmp", "gf.txt")
		gtest.Assert(fp, gfile.Join(root, "gf_tmp", "gf.txt"))
		gtest.Assert(isDir, false)
		fp, isDir = gsp.Search("/", "gf.txt")
		gtest.Assert(fp, pwd)
		gtest.Assert(isDir, true)
	})
}
<|endoftext|>


<commit_before>package stream

import (
	"sync"
)

func newThreshold(cutoff int) *threshold {
	t := &threshold{
		threshold: cutoff,
	}
	t.cond.L = &t.mu
	return t
}

type threshold struct {
	mu   sync.Mutex
	cond sync.Cond

	count     int
	threshold int
}

// Acquire increments the counter. It will not block.
func (t *threshold) Acquire() {
	t.mu.Lock()
	t.count++
	t.mu.Unlock()
}

// Release decrements the counter.
func (t *threshold) Release() {
	t.mu.Lock()
	if t.count == 0 {
		panic("negative count")
	}
	if t.threshold == t.count {
		t.cond.Broadcast()
	}
	t.count--
	t.mu.Unlock()
}

// Wait waits for the counter to drop below the threshold
func (t *threshold) Wait() {
	t.mu.Lock()
	for t.count > t.threshold {
		t.cond.Wait()
	}
	t.mu.Unlock()
}
<commit_msg>fix off-by-one error in the threshold<commit_after>package stream

import (
	"sync"
)

func newThreshold(cutoff int) *threshold {
	t := &threshold{
		threshold: cutoff,
	}
	t.cond.L = &t.mu
	return t
}

type threshold struct {
	mu   sync.Mutex
	cond sync.Cond

	count     int
	threshold int
}

// Acquire increments the counter. It will not block.
func (t *threshold) Acquire() {
	t.mu.Lock()
	t.count++
	t.mu.Unlock()
}

// Release decrements the counter.
func (t *threshold) Release() {
	t.mu.Lock()
	if t.count == 0 {
		panic("negative count")
	}
	if t.threshold == t.count {
		t.cond.Broadcast()
	}
	t.count--
	t.mu.Unlock()
}

// Wait waits for the counter to drop below the threshold
func (t *threshold) Wait() {
	t.mu.Lock()
	for t.count >= t.threshold {
		t.cond.Wait()
	}
	t.mu.Unlock()
}
<|endoftext|>


<commit_before>package page_writer

import (
	"fmt"
	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/util"
	"github.com/chrislusf/seaweedfs/weed/util/mem"
	"sync"
	"sync/atomic"
	"time"
)

type LogicChunkIndex int

type UploadPipeline struct {
	filepath             util.FullPath
	ChunkSize            int64
	writers              *util.LimitedConcurrentExecutor
	writableChunks       map[LogicChunkIndex]*MemChunk
	writableChunksLock   sync.Mutex
	sealedChunks         map[LogicChunkIndex]*SealedChunk
	sealedChunksLock     sync.Mutex
	activeWriterCond     *sync.Cond
	activeWriterCount    int32
	activeReadChunks     map[LogicChunkIndex]int
	activeReadChunksLock sync.Mutex
	saveToStorageFn      SaveToStorageFunc
}

type SealedChunk struct {
	chunk            *MemChunk
	referenceCounter int // track uploading or reading processes
}

func (sc *SealedChunk) FreeReference(messageOnFree string) {
	sc.referenceCounter--
	if sc.referenceCounter == 0 {
		glog.V(4).Infof("Free sealed chunk: %s", messageOnFree)
		mem.Free(sc.chunk.buf)
	}
}

func NewUploadPipeline(filepath util.FullPath, writers *util.LimitedConcurrentExecutor, chunkSize int64, saveToStorageFn SaveToStorageFunc) *UploadPipeline {
	return &UploadPipeline{
		ChunkSize:        chunkSize,
		writableChunks:   make(map[LogicChunkIndex]*MemChunk),
		sealedChunks:     make(map[LogicChunkIndex]*SealedChunk),
		writers:          writers,
		activeWriterCond: sync.NewCond(&sync.Mutex{}),
		saveToStorageFn:  saveToStorageFn,
		filepath:         filepath,
		activeReadChunks: make(map[LogicChunkIndex]int),
	}
}

func (cw *UploadPipeline) SaveDataAt(p []byte, off int64) (n int) {
	cw.writableChunksLock.Lock()
	defer cw.writableChunksLock.Unlock()

	logicChunkIndex := LogicChunkIndex(off / cw.ChunkSize)
	offsetRemainder := off % cw.ChunkSize

	memChunk, found := cw.writableChunks[logicChunkIndex]
	if !found {
		memChunk = &MemChunk{
			buf:   mem.Allocate(int(cw.ChunkSize)),
			usage: newChunkWrittenIntervalList(),
		}
		cw.writableChunks[logicChunkIndex] = memChunk
	}
	n = copy(memChunk.buf[offsetRemainder:], p)
	memChunk.usage.MarkWritten(offsetRemainder, offsetRemainder+int64(n))
	cw.maybeMoveToSealed(memChunk, logicChunkIndex)

	return
}

func (cw *UploadPipeline) MaybeReadDataAt(p []byte, off int64) (maxStop int64) {
	logicChunkIndex := LogicChunkIndex(off / cw.ChunkSize)

	// read from sealed chunks first
	cw.sealedChunksLock.Lock()
	sealedChunk, found := cw.sealedChunks[logicChunkIndex]
	if found {
		sealedChunk.referenceCounter++
	}
	cw.sealedChunksLock.Unlock()
	if found {
		maxStop = readMemChunk(sealedChunk.chunk, p, off, logicChunkIndex, cw.ChunkSize)
		glog.V(4).Infof("%s read sealed memchunk [%d,%d)", cw.filepath, off, maxStop)
		sealedChunk.FreeReference(fmt.Sprintf("%s finish reading chunk %d", cw.filepath, logicChunkIndex))
	}

	// read from writable chunks last
	cw.writableChunksLock.Lock()
	defer cw.writableChunksLock.Unlock()
	writableChunk, found := cw.writableChunks[logicChunkIndex]
	if !found {
		return
	}
	writableMaxStop := readMemChunk(writableChunk, p, off, logicChunkIndex, cw.ChunkSize)
	glog.V(4).Infof("%s read writable memchunk [%d,%d)", cw.filepath, off, writableMaxStop)
	maxStop = max(maxStop, writableMaxStop)

	return
}

func (cw *UploadPipeline) FlushAll() {
	cw.writableChunksLock.Lock()
	defer cw.writableChunksLock.Unlock()

	for logicChunkIndex, memChunk := range cw.writableChunks {
		cw.moveToSealed(memChunk, logicChunkIndex)
	}

	cw.waitForCurrentWritersToComplete()
}

func (cw *UploadPipeline) LockForRead(startOffset, stopOffset int64) {
	startLogicChunkIndex := LogicChunkIndex(startOffset / cw.ChunkSize)
	stopLogicChunkIndex := LogicChunkIndex(stopOffset / cw.ChunkSize)
	if stopOffset%cw.ChunkSize > 0 {
		stopLogicChunkIndex += 1
	}
	cw.activeReadChunksLock.Lock()
	defer cw.activeReadChunksLock.Unlock()
	for i := startLogicChunkIndex; i < stopLogicChunkIndex; i++ {
		if count, found := cw.activeReadChunks[i]; found {
			cw.activeReadChunks[i] = count + 1
		} else {
			cw.activeReadChunks[i] = 1
		}
	}
}

func (cw *UploadPipeline) UnlockForRead(startOffset, stopOffset int64) {
	startLogicChunkIndex := LogicChunkIndex(startOffset / cw.ChunkSize)
	stopLogicChunkIndex := LogicChunkIndex(stopOffset / cw.ChunkSize)
	if stopOffset%cw.ChunkSize > 0 {
		stopLogicChunkIndex += 1
	}
	cw.activeReadChunksLock.Lock()
	defer cw.activeReadChunksLock.Unlock()
	for i := startLogicChunkIndex; i < stopLogicChunkIndex; i++ {
		if count, found := cw.activeReadChunks[i]; found {
			if count == 1 {
				delete(cw.activeReadChunks, i)
			} else {
				cw.activeReadChunks[i] = count - 1
			}
		}
	}
}

func (cw *UploadPipeline) IsLocked(logicChunkIndex LogicChunkIndex) bool {
	cw.activeReadChunksLock.Lock()
	defer cw.activeReadChunksLock.Unlock()
	if count, found := cw.activeReadChunks[logicChunkIndex]; found {
		return count > 0
	}
	return false
}

func (cw *UploadPipeline) waitForCurrentWritersToComplete() {
	cw.activeWriterCond.L.Lock()
	t := int32(100)
	for {
		t = atomic.LoadInt32(&cw.activeWriterCount)
		if t <= 0 {
			break
		}
		glog.V(4).Infof("activeWriterCond is %d", t)
		cw.activeWriterCond.Wait()
	}
	cw.activeWriterCond.L.Unlock()
}

func (cw *UploadPipeline) maybeMoveToSealed(memChunk *MemChunk, logicChunkIndex LogicChunkIndex) {
	if memChunk.usage.IsComplete(cw.ChunkSize) {
		cw.moveToSealed(memChunk, logicChunkIndex)
	}
}

func (cw *UploadPipeline) moveToSealed(memChunk *MemChunk, logicChunkIndex LogicChunkIndex) {
	atomic.AddInt32(&cw.activeWriterCount, 1)
	glog.V(4).Infof("%s activeWriterCount %d ++> %d", cw.filepath, cw.activeWriterCount-1, cw.activeWriterCount)

	cw.sealedChunksLock.Lock()

	if oldMemChunk, found := cw.sealedChunks[logicChunkIndex]; found {
		oldMemChunk.FreeReference(fmt.Sprintf("%s replace chunk %d", cw.filepath, logicChunkIndex))
	}
	sealedChunk := &SealedChunk{
		chunk:            memChunk,
		referenceCounter: 1, // default 1 is for uploading process
	}
	cw.sealedChunks[logicChunkIndex] = sealedChunk
	delete(cw.writableChunks, logicChunkIndex)

	cw.sealedChunksLock.Unlock()

	cw.writers.Execute(func() {
		// first add to the file chunks
		cw.saveOneChunk(sealedChunk.chunk, logicChunkIndex)

		// notify waiting process
		atomic.AddInt32(&cw.activeWriterCount, -1)
		glog.V(4).Infof("%s activeWriterCount %d --> %d", cw.filepath, cw.activeWriterCount+1, cw.activeWriterCount)
		// Lock and Unlock are not required,
		// but it may signal multiple times during one wakeup,
		// and the waiting goroutine may miss some of them!
		cw.activeWriterCond.L.Lock()
		cw.activeWriterCond.Broadcast()
		cw.activeWriterCond.L.Unlock()

		// wait for readers
		for cw.IsLocked(logicChunkIndex) {
			time.Sleep(59 * time.Millisecond)
		}

		// then remove from sealed chunks
		cw.sealedChunksLock.Lock()
		defer cw.sealedChunksLock.Unlock()
		delete(cw.sealedChunks, logicChunkIndex)
		sealedChunk.FreeReference(fmt.Sprintf("%s finished uploading chunk %d", cw.filepath, logicChunkIndex))

	})
}

func (cw *UploadPipeline) saveOneChunk(memChunk *MemChunk, logicChunkIndex LogicChunkIndex) {
	if cw.saveToStorageFn == nil {
		return
	}
	for t := memChunk.usage.head.next; t != memChunk.usage.tail; t = t.next {
		reader := util.NewBytesReader(memChunk.buf[t.StartOffset:t.stopOffset])
		cw.saveToStorageFn(reader, int64(logicChunkIndex)*cw.ChunkSize+t.StartOffset, t.Size(), func() {
		})
	}
}

func readMemChunk(memChunk *MemChunk, p []byte, off int64, logicChunkIndex LogicChunkIndex, chunkSize int64) (maxStop int64) {
	memChunkBaseOffset := int64(logicChunkIndex) * chunkSize
	for t := memChunk.usage.head.next; t != memChunk.usage.tail; t = t.next {
		logicStart := max(off, int64(logicChunkIndex)*chunkSize+t.StartOffset)
		logicStop := min(off+int64(len(p)), memChunkBaseOffset+t.stopOffset)
		if logicStart < logicStop {
			copy(p[logicStart-off:logicStop-off], memChunk.buf[logicStart-memChunkBaseOffset:logicStop-memChunkBaseOffset])
			maxStop = max(maxStop, logicStop)
		}
	}
	return
}

func (p2 *UploadPipeline) Shutdown() {

}
<commit_msg>re-order<commit_after>package page_writer

import (
	"fmt"
	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/util"
	"github.com/chrislusf/seaweedfs/weed/util/mem"
	"sync"
	"sync/atomic"
	"time"
)

type LogicChunkIndex int

type UploadPipeline struct {
	filepath             util.FullPath
	ChunkSize            int64
	writableChunks       map[LogicChunkIndex]*MemChunk
	writableChunksLock   sync.Mutex
	sealedChunks         map[LogicChunkIndex]*SealedChunk
	sealedChunksLock     sync.Mutex
	writers              *util.LimitedConcurrentExecutor
	activeWriterCond     *sync.Cond
	activeWriterCount    int32
	saveToStorageFn      SaveToStorageFunc
	activeReadChunks     map[LogicChunkIndex]int
	activeReadChunksLock sync.Mutex
}

type SealedChunk struct {
	chunk            *MemChunk
	referenceCounter int // track uploading or reading processes
}

func (sc *SealedChunk) FreeReference(messageOnFree string) {
	sc.referenceCounter--
	if sc.referenceCounter == 0 {
		glog.V(4).Infof("Free sealed chunk: %s", messageOnFree)
		mem.Free(sc.chunk.buf)
	}
}

func NewUploadPipeline(filepath util.FullPath, writers *util.LimitedConcurrentExecutor, chunkSize int64, saveToStorageFn SaveToStorageFunc) *UploadPipeline {
	return &UploadPipeline{
		ChunkSize:        chunkSize,
		writableChunks:   make(map[LogicChunkIndex]*MemChunk),
		sealedChunks:     make(map[LogicChunkIndex]*SealedChunk),
		writers:          writers,
		activeWriterCond: sync.NewCond(&sync.Mutex{}),
		saveToStorageFn:  saveToStorageFn,
		filepath:         filepath,
		activeReadChunks: make(map[LogicChunkIndex]int),
	}
}

func (cw *UploadPipeline) SaveDataAt(p []byte, off int64) (n int) {
	cw.writableChunksLock.Lock()
	defer cw.writableChunksLock.Unlock()

	logicChunkIndex := LogicChunkIndex(off / cw.ChunkSize)
	offsetRemainder := off % cw.ChunkSize

	memChunk, found := cw.writableChunks[logicChunkIndex]
	if !found {
		memChunk = &MemChunk{
			buf:   mem.Allocate(int(cw.ChunkSize)),
			usage: newChunkWrittenIntervalList(),
		}
		cw.writableChunks[logicChunkIndex] = memChunk
	}
	n = copy(memChunk.buf[offsetRemainder:], p)
	memChunk.usage.MarkWritten(offsetRemainder, offsetRemainder+int64(n))
	cw.maybeMoveToSealed(memChunk, logicChunkIndex)

	return
}

func (cw *UploadPipeline) MaybeReadDataAt(p []byte, off int64) (maxStop int64) {
	logicChunkIndex := LogicChunkIndex(off / cw.ChunkSize)

	// read from sealed chunks first
	cw.sealedChunksLock.Lock()
	sealedChunk, found := cw.sealedChunks[logicChunkIndex]
	if found {
		sealedChunk.referenceCounter++
	}
	cw.sealedChunksLock.Unlock()
	if found {
		maxStop = readMemChunk(sealedChunk.chunk, p, off, logicChunkIndex, cw.ChunkSize)
		glog.V(4).Infof("%s read sealed memchunk [%d,%d)", cw.filepath, off, maxStop)
		sealedChunk.FreeReference(fmt.Sprintf("%s finish reading chunk %d", cw.filepath, logicChunkIndex))
	}

	// read from writable chunks last
	cw.writableChunksLock.Lock()
	defer cw.writableChunksLock.Unlock()
	writableChunk, found := cw.writableChunks[logicChunkIndex]
	if !found {
		return
	}
	writableMaxStop := readMemChunk(writableChunk, p, off, logicChunkIndex, cw.ChunkSize)
	glog.V(4).Infof("%s read writable memchunk [%d,%d)", cw.filepath, off, writableMaxStop)
	maxStop = max(maxStop, writableMaxStop)

	return
}

func (cw *UploadPipeline) FlushAll() {
	cw.writableChunksLock.Lock()
	defer cw.writableChunksLock.Unlock()

	for logicChunkIndex, memChunk := range cw.writableChunks {
		cw.moveToSealed(memChunk, logicChunkIndex)
	}

	cw.waitForCurrentWritersToComplete()
}

func (cw *UploadPipeline) LockForRead(startOffset, stopOffset int64) {
	startLogicChunkIndex := LogicChunkIndex(startOffset / cw.ChunkSize)
	stopLogicChunkIndex := LogicChunkIndex(stopOffset / cw.ChunkSize)
	if stopOffset%cw.ChunkSize > 0 {
		stopLogicChunkIndex += 1
	}
	cw.activeReadChunksLock.Lock()
	defer cw.activeReadChunksLock.Unlock()
	for i := startLogicChunkIndex; i < stopLogicChunkIndex; i++ {
		if count, found := cw.activeReadChunks[i]; found {
			cw.activeReadChunks[i] = count + 1
		} else {
			cw.activeReadChunks[i] = 1
		}
	}
}

func (cw *UploadPipeline) UnlockForRead(startOffset, stopOffset int64) {
	startLogicChunkIndex := LogicChunkIndex(startOffset / cw.ChunkSize)
	stopLogicChunkIndex := LogicChunkIndex(stopOffset / cw.ChunkSize)
	if stopOffset%cw.ChunkSize > 0 {
		stopLogicChunkIndex += 1
	}
	cw.activeReadChunksLock.Lock()
	defer cw.activeReadChunksLock.Unlock()
	for i := startLogicChunkIndex; i < stopLogicChunkIndex; i++ {
		if count, found := cw.activeReadChunks[i]; found {
			if count == 1 {
				delete(cw.activeReadChunks, i)
			} else {
				cw.activeReadChunks[i] = count - 1
			}
		}
	}
}

func (cw *UploadPipeline) IsLocked(logicChunkIndex LogicChunkIndex) bool {
	cw.activeReadChunksLock.Lock()
	defer cw.activeReadChunksLock.Unlock()
	if count, found := cw.activeReadChunks[logicChunkIndex]; found {
		return count > 0
	}
	return false
}

func (cw *UploadPipeline) waitForCurrentWritersToComplete() {
	cw.activeWriterCond.L.Lock()
	t := int32(100)
	for {
		t = atomic.LoadInt32(&cw.activeWriterCount)
		if t <= 0 {
			break
		}
		glog.V(4).Infof("activeWriterCond is %d", t)
		cw.activeWriterCond.Wait()
	}
	cw.activeWriterCond.L.Unlock()
}

func (cw *UploadPipeline) maybeMoveToSealed(memChunk *MemChunk, logicChunkIndex LogicChunkIndex) {
	if memChunk.usage.IsComplete(cw.ChunkSize) {
		cw.moveToSealed(memChunk, logicChunkIndex)
	}
}

func (cw *UploadPipeline) moveToSealed(memChunk *MemChunk, logicChunkIndex LogicChunkIndex) {
	atomic.AddInt32(&cw.activeWriterCount, 1)
	glog.V(4).Infof("%s activeWriterCount %d ++> %d", cw.filepath, cw.activeWriterCount-1, cw.activeWriterCount)

	cw.sealedChunksLock.Lock()

	if oldMemChunk, found := cw.sealedChunks[logicChunkIndex]; found {
		oldMemChunk.FreeReference(fmt.Sprintf("%s replace chunk %d", cw.filepath, logicChunkIndex))
	}
	sealedChunk := &SealedChunk{
		chunk:            memChunk,
		referenceCounter: 1, // default 1 is for uploading process
	}
	cw.sealedChunks[logicChunkIndex] = sealedChunk
	delete(cw.writableChunks, logicChunkIndex)

	cw.sealedChunksLock.Unlock()

	cw.writers.Execute(func() {
		// first add to the file chunks
		cw.saveOneChunk(sealedChunk.chunk, logicChunkIndex)

		// notify waiting process
		atomic.AddInt32(&cw.activeWriterCount, -1)
		glog.V(4).Infof("%s activeWriterCount %d --> %d", cw.filepath, cw.activeWriterCount+1, cw.activeWriterCount)
		// Lock and Unlock are not required,
		// but it may signal multiple times during one wakeup,
		// and the waiting goroutine may miss some of them!
		cw.activeWriterCond.L.Lock()
		cw.activeWriterCond.Broadcast()
		cw.activeWriterCond.L.Unlock()

		// wait for readers
		for cw.IsLocked(logicChunkIndex) {
			time.Sleep(59 * time.Millisecond)
		}

		// then remove from sealed chunks
		cw.sealedChunksLock.Lock()
		defer cw.sealedChunksLock.Unlock()
		delete(cw.sealedChunks, logicChunkIndex)
		sealedChunk.FreeReference(fmt.Sprintf("%s finished uploading chunk %d", cw.filepath, logicChunkIndex))

	})
}

func (cw *UploadPipeline) saveOneChunk(memChunk *MemChunk, logicChunkIndex LogicChunkIndex) {
	if cw.saveToStorageFn == nil {
		return
	}
	for t := memChunk.usage.head.next; t != memChunk.usage.tail; t = t.next {
		reader := util.NewBytesReader(memChunk.buf[t.StartOffset:t.stopOffset])
		cw.saveToStorageFn(reader, int64(logicChunkIndex)*cw.ChunkSize+t.StartOffset, t.Size(), func() {
		})
	}
}

func readMemChunk(memChunk *MemChunk, p []byte, off int64, logicChunkIndex LogicChunkIndex, chunkSize int64) (maxStop int64) {
	memChunkBaseOffset := int64(logicChunkIndex) * chunkSize
	for t := memChunk.usage.head.next; t != memChunk.usage.tail; t = t.next {
		logicStart := max(off, int64(logicChunkIndex)*chunkSize+t.StartOffset)
		logicStop := min(off+int64(len(p)), memChunkBaseOffset+t.stopOffset)
		if logicStart < logicStop {
			copy(p[logicStart-off:logicStop-off], memChunk.buf[logicStart-memChunkBaseOffset:logicStop-memChunkBaseOffset])
			maxStop = max(maxStop, logicStop)
		}
	}
	return
}

func (p2 *UploadPipeline) Shutdown() {

}
<|endoftext|>


<commit_before>package weed_server

import (
	"errors"
	"fmt"
	"net/http"
	"strconv"
	"strings"
	"time"

	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/operation"
	"github.com/chrislusf/seaweedfs/weed/stats"
	"github.com/chrislusf/seaweedfs/weed/storage/needle"
	"github.com/chrislusf/seaweedfs/weed/topology"
	"github.com/chrislusf/seaweedfs/weed/util"
)

func (vs *VolumeServer) PostHandler(w http.ResponseWriter, r *http.Request) {

	stats.VolumeServerRequestCounter.WithLabelValues("post").Inc()
	start := time.Now()
	defer func() {
		stats.VolumeServerRequestHistogram.WithLabelValues("post").Observe(time.Since(start).Seconds())
	}()

	if e := r.ParseForm(); e != nil {
		glog.V(0).Infoln("form parse error:", e)
		writeJsonError(w, r, http.StatusBadRequest, e)
		return
	}

	vid, fid, _, _, _ := parseURLPath(r.URL.Path)
	volumeId, ve := needle.NewVolumeId(vid)
	if ve != nil {
		glog.V(0).Infoln("NewVolumeId error:", ve)
		writeJsonError(w, r, http.StatusBadRequest, ve)
		return
	}

	if !vs.maybeCheckJwtAuthorization(r, vid, fid, true) {
		writeJsonError(w, r, http.StatusUnauthorized, errors.New("wrong jwt"))
		return
	}

	reqNeedle, originalSize, contentMd5, ne := needle.CreateNeedleFromRequest(r, vs.FixJpgOrientation, vs.fileSizeLimitBytes)
	if ne != nil {
		writeJsonError(w, r, http.StatusBadRequest, ne)
		return
	}

	ret := operation.UploadResult{}
	isUnchanged, writeError := topology.ReplicatedWrite(vs.GetMaster, vs.store, volumeId, reqNeedle, r)

	// http 204 status code does not allow body
	if writeError == nil && isUnchanged {
		setEtag(w, reqNeedle.Etag())
		w.WriteHeader(http.StatusNoContent)
		return
	}

	httpStatus := http.StatusCreated
	if writeError != nil {
		httpStatus = http.StatusInternalServerError
		ret.Error = writeError.Error()
	}
	if reqNeedle.HasName() {
		ret.Name = string(reqNeedle.Name)
	}
	ret.Size = uint32(originalSize)
	ret.ETag = fmt.Sprintf("%x", util.Base64Md5ToBytes(contentMd5))
	ret.Mime = string(reqNeedle.Mime)
	setEtag(w, ret.ETag)
	w.Header().Set("Content-MD5", contentMd5)
	writeJsonQuiet(w, r, httpStatus, ret)
}

func (vs *VolumeServer) DeleteHandler(w http.ResponseWriter, r *http.Request) {

	stats.VolumeServerRequestCounter.WithLabelValues("delete").Inc()
	start := time.Now()
	defer func() {
		stats.VolumeServerRequestHistogram.WithLabelValues("delete").Observe(time.Since(start).Seconds())
	}()

	n := new(needle.Needle)
	vid, fid, _, _, _ := parseURLPath(r.URL.Path)
	volumeId, _ := needle.NewVolumeId(vid)
	n.ParsePath(fid)

	if !vs.maybeCheckJwtAuthorization(r, vid, fid, true) {
		writeJsonError(w, r, http.StatusUnauthorized, errors.New("wrong jwt"))
		return
	}

	// glog.V(2).Infof("volume %s deleting %s", vid, n)

	cookie := n.Cookie

	ecVolume, hasEcVolume := vs.store.FindEcVolume(volumeId)

	if hasEcVolume {
		count, err := vs.store.DeleteEcShardNeedle(ecVolume, n, cookie)
		writeDeleteResult(err, count, w, r)
		return
	}

	_, ok := vs.store.ReadVolumeNeedle(volumeId, n, nil)
	if ok != nil {
		m := make(map[string]uint32)
		m["size"] = 0
		writeJsonQuiet(w, r, http.StatusNotFound, m)
		return
	}

	if n.Cookie != cookie {
		glog.V(0).Infoln("delete", r.URL.Path, "with unmaching cookie from ", r.RemoteAddr, "agent", r.UserAgent())
		writeJsonError(w, r, http.StatusBadRequest, errors.New("File Random Cookie does not match."))
		return
	}

	count := int64(n.Size)

	if n.IsChunkedManifest() {
		chunkManifest, e := operation.LoadChunkManifest(n.Data, n.IsCompressed())
		if e != nil {
			writeJsonError(w, r, http.StatusInternalServerError, fmt.Errorf("Load chunks manifest error: %v", e))
			return
		}
		// make sure all chunks had deleted before delete manifest
		if e := chunkManifest.DeleteChunks(vs.GetMaster, false, vs.grpcDialOption); e != nil {
			writeJsonError(w, r, http.StatusInternalServerError, fmt.Errorf("Delete chunks error: %v", e))
			return
		}
		count = chunkManifest.Size
	}

	n.LastModified = uint64(time.Now().Unix())
	if len(r.FormValue("ts")) > 0 {
		modifiedTime, err := strconv.ParseInt(r.FormValue("ts"), 10, 64)
		if err == nil {
			n.LastModified = uint64(modifiedTime)
		}
	}

	_, err := topology.ReplicatedDelete(vs.GetMaster, vs.store, volumeId, n, r)

	writeDeleteResult(err, count, w, r)

}

func writeDeleteResult(err error, count int64, w http.ResponseWriter, r *http.Request) {
	if err == nil {
		m := make(map[string]int64)
		m["size"] = count
		writeJsonQuiet(w, r, http.StatusAccepted, m)
	} else {
		writeJsonError(w, r, http.StatusInternalServerError, fmt.Errorf("Deletion Failed: %v", err))
	}
}

func setEtag(w http.ResponseWriter, etag string) {
	if etag != "" {
		if strings.HasPrefix(etag, "\"") {
			w.Header().Set("ETag", etag)
		} else {
			w.Header().Set("ETag", "\""+etag+"\"")
		}
	}
}

func getEtag(resp *http.Response) (etag string) {
	etag = resp.Header.Get("ETag")
	if strings.HasPrefix(etag, "\"") && strings.HasSuffix(etag, "\"") {
		return etag[1 : len(etag)-1]
	}
	return
}
<commit_msg>revert volume etag<commit_after>package weed_server

import (
	"errors"
	"fmt"
	"net/http"
	"strconv"
	"strings"
	"time"

	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/operation"
	"github.com/chrislusf/seaweedfs/weed/stats"
	"github.com/chrislusf/seaweedfs/weed/storage/needle"
	"github.com/chrislusf/seaweedfs/weed/topology"
)

func (vs *VolumeServer) PostHandler(w http.ResponseWriter, r *http.Request) {

	stats.VolumeServerRequestCounter.WithLabelValues("post").Inc()
	start := time.Now()
	defer func() {
		stats.VolumeServerRequestHistogram.WithLabelValues("post").Observe(time.Since(start).Seconds())
	}()

	if e := r.ParseForm(); e != nil {
		glog.V(0).Infoln("form parse error:", e)
		writeJsonError(w, r, http.StatusBadRequest, e)
		return
	}

	vid, fid, _, _, _ := parseURLPath(r.URL.Path)
	volumeId, ve := needle.NewVolumeId(vid)
	if ve != nil {
		glog.V(0).Infoln("NewVolumeId error:", ve)
		writeJsonError(w, r, http.StatusBadRequest, ve)
		return
	}

	if !vs.maybeCheckJwtAuthorization(r, vid, fid, true) {
		writeJsonError(w, r, http.StatusUnauthorized, errors.New("wrong jwt"))
		return
	}

	reqNeedle, originalSize, contentMd5, ne := needle.CreateNeedleFromRequest(r, vs.FixJpgOrientation, vs.fileSizeLimitBytes)
	if ne != nil {
		writeJsonError(w, r, http.StatusBadRequest, ne)
		return
	}

	ret := operation.UploadResult{}
	isUnchanged, writeError := topology.ReplicatedWrite(vs.GetMaster, vs.store, volumeId, reqNeedle, r)

	// http 204 status code does not allow body
	if writeError == nil && isUnchanged {
		setEtag(w, reqNeedle.Etag())
		w.WriteHeader(http.StatusNoContent)
		return
	}

	httpStatus := http.StatusCreated
	if writeError != nil {
		httpStatus = http.StatusInternalServerError
		ret.Error = writeError.Error()
	}
	if reqNeedle.HasName() {
		ret.Name = string(reqNeedle.Name)
	}
	ret.Size = uint32(originalSize)
	ret.ETag = reqNeedle.Etag()
	ret.Mime = string(reqNeedle.Mime)
	setEtag(w, ret.ETag)
	w.Header().Set("Content-MD5", contentMd5)
	writeJsonQuiet(w, r, httpStatus, ret)
}

func (vs *VolumeServer) DeleteHandler(w http.ResponseWriter, r *http.Request) {

	stats.VolumeServerRequestCounter.WithLabelValues("delete").Inc()
	start := time.Now()
	defer func() {
		stats.VolumeServerRequestHistogram.WithLabelValues("delete").Observe(time.Since(start).Seconds())
	}()

	n := new(needle.Needle)
	vid, fid, _, _, _ := parseURLPath(r.URL.Path)
	volumeId, _ := needle.NewVolumeId(vid)
	n.ParsePath(fid)

	if !vs.maybeCheckJwtAuthorization(r, vid, fid, true) {
		writeJsonError(w, r, http.StatusUnauthorized, errors.New("wrong jwt"))
		return
	}

	// glog.V(2).Infof("volume %s deleting %s", vid, n)

	cookie := n.Cookie

	ecVolume, hasEcVolume := vs.store.FindEcVolume(volumeId)

	if hasEcVolume {
		count, err := vs.store.DeleteEcShardNeedle(ecVolume, n, cookie)
		writeDeleteResult(err, count, w, r)
		return
	}

	_, ok := vs.store.ReadVolumeNeedle(volumeId, n, nil)
	if ok != nil {
		m := make(map[string]uint32)
		m["size"] = 0
		writeJsonQuiet(w, r, http.StatusNotFound, m)
		return
	}

	if n.Cookie != cookie {
		glog.V(0).Infoln("delete", r.URL.Path, "with unmaching cookie from ", r.RemoteAddr, "agent", r.UserAgent())
		writeJsonError(w, r, http.StatusBadRequest, errors.New("File Random Cookie does not match."))
		return
	}

	count := int64(n.Size)

	if n.IsChunkedManifest() {
		chunkManifest, e := operation.LoadChunkManifest(n.Data, n.IsCompressed())
		if e != nil {
			writeJsonError(w, r, http.StatusInternalServerError, fmt.Errorf("Load chunks manifest error: %v", e))
			return
		}
		// make sure all chunks had deleted before delete manifest
		if e := chunkManifest.DeleteChunks(vs.GetMaster, false, vs.grpcDialOption); e != nil {
			writeJsonError(w, r, http.StatusInternalServerError, fmt.Errorf("Delete chunks error: %v", e))
			return
		}
		count = chunkManifest.Size
	}

	n.LastModified = uint64(time.Now().Unix())
	if len(r.FormValue("ts")) > 0 {
		modifiedTime, err := strconv.ParseInt(r.FormValue("ts"), 10, 64)
		if err == nil {
			n.LastModified = uint64(modifiedTime)
		}
	}

	_, err := topology.ReplicatedDelete(vs.GetMaster, vs.store, volumeId, n, r)

	writeDeleteResult(err, count, w, r)

}

func writeDeleteResult(err error, count int64, w http.ResponseWriter, r *http.Request) {
	if err == nil {
		m := make(map[string]int64)
		m["size"] = count
		writeJsonQuiet(w, r, http.StatusAccepted, m)
	} else {
		writeJsonError(w, r, http.StatusInternalServerError, fmt.Errorf("Deletion Failed: %v", err))
	}
}

func setEtag(w http.ResponseWriter, etag string) {
	if etag != "" {
		if strings.HasPrefix(etag, "\"") {
			w.Header().Set("ETag", etag)
		} else {
			w.Header().Set("ETag", "\""+etag+"\"")
		}
	}
}

func getEtag(resp *http.Response) (etag string) {
	etag = resp.Header.Get("ETag")
	if strings.HasPrefix(etag, "\"") && strings.HasSuffix(etag, "\"") {
		return etag[1 : len(etag)-1]
	}
	return
}
<|endoftext|>