<commit_before>// The version package implements version parsing.
// It also acts as guardian of the current client Juju version number.
package version

import (
	"fmt"
	"io/ioutil"
	"labix.org/v2/mgo/bson"
	"os"
	"path/filepath"
	"regexp"
	"runtime"
	"strconv"
	"strings"
)

// The presence and format of this constant is very important.
// The debian/rules build recipe uses this value for the version
// number of the release package.
const version = "1.9.2"

// Current gives the current version of the system.  If the file
// "FORCE-VERSION" is present in the same directory as the running
// binary, it will override this.
var Current = Binary{
	Number: MustParse(version),
	Series: readSeries("/etc/lsb-release"), // current Ubuntu release name.
	Arch:   ubuntuArch(runtime.GOARCH),
}

func init() {
	toolsDir := filepath.Dir(os.Args[0])
	v, err := ioutil.ReadFile(filepath.Join(toolsDir, "FORCE-VERSION"))
	if err != nil {
		if os.IsNotExist(err) {
			return
		}
		panic(fmt.Errorf("version: cannot read forced version: %v", err))
	}
	Current = MustParseBinary(strings.TrimSpace(string(v)))
}

// Number represents a juju version.  When bugs are fixed the patch
// number is incremented; when new features are added the minor number
// is incremented and patch is reset; and when compatibility is broken
// the major version is incremented and minor and patch are reset.  The
// build number is automatically assigned and has no well defined
// sequence.  If the build number is greater than zero or any of the
// other numbers are odd, it indicates that the release is still in
// development.
type Number struct {
	Major int
	Minor int
	Patch int
	Build int
}

// Binary specifies a binary version of juju.
type Binary struct {
	Number
	Series string
	Arch   string
}

func (v Binary) String() string {
	return fmt.Sprintf("%v-%s-%s", v.Number, v.Series, v.Arch)
}

// GetBSON turns v into a bson.Getter so it can be saved directly
// on a MongoDB database with mgo.
func (v Binary) GetBSON() (interface{}, error) {
	return v.String(), nil
}

// SetBSON turns v into a bson.Setter so it can be loaded directly
// from a MongoDB database with mgo.
func (vp *Binary) SetBSON(raw bson.Raw) error {
	var s string
	err := raw.Unmarshal(&s)
	if err != nil {
		return err
	}
	v, err := ParseBinary(s)
	if err != nil {
		return err
	}
	*vp = v
	return nil
}

var (
	binaryPat = regexp.MustCompile(`^(\d{1,9})\.(\d{1,9})\.(\d{1,9})(\.\d{1,9})?-([^-]+)-([^-]+)$`)
	numberPat = regexp.MustCompile(`^(\d{1,9})\.(\d{1,9})\.(\d{1,9})(\.\d{1,9})?$`)
)

// MustParse parses a version and panics if it does
// not parse correctly.
func MustParse(s string) Number {
	v, err := Parse(s)
	if err != nil {
		panic(err)
	}
	return v
}

// MustParseBinary parses a binary version and panics if it does
// not parse correctly.
func MustParseBinary(s string) Binary {
	v, err := ParseBinary(s)
	if err != nil {
		panic(err)
	}
	return v
}

// ParseBinary parses a binary version of the form "1.2.3-series-arch".
func ParseBinary(s string) (Binary, error) {
	m := binaryPat.FindStringSubmatch(s)
	if m == nil {
		return Binary{}, fmt.Errorf("invalid binary version %q", s)
	}
	var v Binary
	v.Major = atoi(m[1])
	v.Minor = atoi(m[2])
	v.Patch = atoi(m[3])
	if m[4] != "" {
		v.Build = atoi(m[4][1:])
	}
	v.Series = m[5]
	v.Arch = m[6]
	return v, nil
}

// Parse parses the version, which is of the form 1.2.3
// giving the major, minor and release versions
// respectively.
func Parse(s string) (Number, error) {
	m := numberPat.FindStringSubmatch(s)
	if m == nil {
		return Number{}, fmt.Errorf("invalid version %q", s)
	}
	var v Number
	v.Major = atoi(m[1])
	v.Minor = atoi(m[2])
	v.Patch = atoi(m[3])
	if m[4] != "" {
		v.Build = atoi(m[4][1:])
	}
	return v, nil
}

// atoi is the same as strconv.Atoi but assumes that
// the string has been verified to be a valid integer.
func atoi(s string) int {
	n, err := strconv.Atoi(s)
	if err != nil {
		panic(err)
	}
	return n
}

func (v Number) String() string {
	s := fmt.Sprintf("%d.%d.%d", v.Major, v.Minor, v.Patch)
	if v.Build > 0 {
		s += fmt.Sprintf(".%d", v.Build)
	}
	return s
}

// Less returns whether v is semantically earlier in the
// version sequence than w.
func (v Number) Less(w Number) bool {
	switch {
	case v.Major != w.Major:
		return v.Major < w.Major
	case v.Minor != w.Minor:
		return v.Minor < w.Minor
	case v.Patch != w.Patch:
		return v.Patch < w.Patch
	case v.Build != w.Build:
		return v.Build < w.Build
	}
	return false
}

// GetBSON turns v into a bson.Getter so it can be saved directly
// on a MongoDB database with mgo.
func (v Number) GetBSON() (interface{}, error) {
	return v.String(), nil
}

// SetBSON turns v into a bson.Setter so it can be loaded directly
// from a MongoDB database with mgo.
func (vp *Number) SetBSON(raw bson.Raw) error {
	var s string
	err := raw.Unmarshal(&s)
	if err != nil {
		return err
	}
	v, err := Parse(s)
	if err != nil {
		return err
	}
	*vp = v
	return nil
}

func isOdd(x int) bool {
	return x%2 != 0
}

// IsDev returns whether the version represents a development
// version. A version with an odd-numbered major, minor
// or patch version is considered to be a development version.
func (v Number) IsDev() bool {
	return isOdd(v.Major) || isOdd(v.Minor) || isOdd(v.Patch) || v.Build > 0
}

func readSeries(releaseFile string) string {
	data, err := ioutil.ReadFile(releaseFile)
	if err != nil {
		return "unknown"
	}
	for _, line := range strings.Split(string(data), "\n") {
		const p = "DISTRIB_CODENAME="
		if strings.HasPrefix(line, p) {
			return strings.Trim(line[len(p):], "\t '\"")
		}
	}
	return "unknown"
}

func ubuntuArch(arch string) string {
	if arch == "386" {
		arch = "i386"
	}
	return arch
}
<commit_msg>set dev version to 1.9.3<commit_after>// The version package implements version parsing.
// It also acts as guardian of the current client Juju version number.
package version

import (
	"fmt"
	"io/ioutil"
	"labix.org/v2/mgo/bson"
	"os"
	"path/filepath"
	"regexp"
	"runtime"
	"strconv"
	"strings"
)

// The presence and format of this constant is very important.
// The debian/rules build recipe uses this value for the version
// number of the release package.
const version = "1.9.3"

// Current gives the current version of the system.  If the file
// "FORCE-VERSION" is present in the same directory as the running
// binary, it will override this.
var Current = Binary{
	Number: MustParse(version),
	Series: readSeries("/etc/lsb-release"), // current Ubuntu release name.
	Arch:   ubuntuArch(runtime.GOARCH),
}

func init() {
	toolsDir := filepath.Dir(os.Args[0])
	v, err := ioutil.ReadFile(filepath.Join(toolsDir, "FORCE-VERSION"))
	if err != nil {
		if os.IsNotExist(err) {
			return
		}
		panic(fmt.Errorf("version: cannot read forced version: %v", err))
	}
	Current = MustParseBinary(strings.TrimSpace(string(v)))
}

// Number represents a juju version.  When bugs are fixed the patch
// number is incremented; when new features are added the minor number
// is incremented and patch is reset; and when compatibility is broken
// the major version is incremented and minor and patch are reset.  The
// build number is automatically assigned and has no well defined
// sequence.  If the build number is greater than zero or any of the
// other numbers are odd, it indicates that the release is still in
// development.
type Number struct {
	Major int
	Minor int
	Patch int
	Build int
}

// Binary specifies a binary version of juju.
type Binary struct {
	Number
	Series string
	Arch   string
}

func (v Binary) String() string {
	return fmt.Sprintf("%v-%s-%s", v.Number, v.Series, v.Arch)
}

// GetBSON turns v into a bson.Getter so it can be saved directly
// on a MongoDB database with mgo.
func (v Binary) GetBSON() (interface{}, error) {
	return v.String(), nil
}

// SetBSON turns v into a bson.Setter so it can be loaded directly
// from a MongoDB database with mgo.
func (vp *Binary) SetBSON(raw bson.Raw) error {
	var s string
	err := raw.Unmarshal(&s)
	if err != nil {
		return err
	}
	v, err := ParseBinary(s)
	if err != nil {
		return err
	}
	*vp = v
	return nil
}

var (
	binaryPat = regexp.MustCompile(`^(\d{1,9})\.(\d{1,9})\.(\d{1,9})(\.\d{1,9})?-([^-]+)-([^-]+)$`)
	numberPat = regexp.MustCompile(`^(\d{1,9})\.(\d{1,9})\.(\d{1,9})(\.\d{1,9})?$`)
)

// MustParse parses a version and panics if it does
// not parse correctly.
func MustParse(s string) Number {
	v, err := Parse(s)
	if err != nil {
		panic(err)
	}
	return v
}

// MustParseBinary parses a binary version and panics if it does
// not parse correctly.
func MustParseBinary(s string) Binary {
	v, err := ParseBinary(s)
	if err != nil {
		panic(err)
	}
	return v
}

// ParseBinary parses a binary version of the form "1.2.3-series-arch".
func ParseBinary(s string) (Binary, error) {
	m := binaryPat.FindStringSubmatch(s)
	if m == nil {
		return Binary{}, fmt.Errorf("invalid binary version %q", s)
	}
	var v Binary
	v.Major = atoi(m[1])
	v.Minor = atoi(m[2])
	v.Patch = atoi(m[3])
	if m[4] != "" {
		v.Build = atoi(m[4][1:])
	}
	v.Series = m[5]
	v.Arch = m[6]
	return v, nil
}

// Parse parses the version, which is of the form 1.2.3
// giving the major, minor and release versions
// respectively.
func Parse(s string) (Number, error) {
	m := numberPat.FindStringSubmatch(s)
	if m == nil {
		return Number{}, fmt.Errorf("invalid version %q", s)
	}
	var v Number
	v.Major = atoi(m[1])
	v.Minor = atoi(m[2])
	v.Patch = atoi(m[3])
	if m[4] != "" {
		v.Build = atoi(m[4][1:])
	}
	return v, nil
}

// atoi is the same as strconv.Atoi but assumes that
// the string has been verified to be a valid integer.
func atoi(s string) int {
	n, err := strconv.Atoi(s)
	if err != nil {
		panic(err)
	}
	return n
}

func (v Number) String() string {
	s := fmt.Sprintf("%d.%d.%d", v.Major, v.Minor, v.Patch)
	if v.Build > 0 {
		s += fmt.Sprintf(".%d", v.Build)
	}
	return s
}

// Less returns whether v is semantically earlier in the
// version sequence than w.
func (v Number) Less(w Number) bool {
	switch {
	case v.Major != w.Major:
		return v.Major < w.Major
	case v.Minor != w.Minor:
		return v.Minor < w.Minor
	case v.Patch != w.Patch:
		return v.Patch < w.Patch
	case v.Build != w.Build:
		return v.Build < w.Build
	}
	return false
}

// GetBSON turns v into a bson.Getter so it can be saved directly
// on a MongoDB database with mgo.
func (v Number) GetBSON() (interface{}, error) {
	return v.String(), nil
}

// SetBSON turns v into a bson.Setter so it can be loaded directly
// from a MongoDB database with mgo.
func (vp *Number) SetBSON(raw bson.Raw) error {
	var s string
	err := raw.Unmarshal(&s)
	if err != nil {
		return err
	}
	v, err := Parse(s)
	if err != nil {
		return err
	}
	*vp = v
	return nil
}

func isOdd(x int) bool {
	return x%2 != 0
}

// IsDev returns whether the version represents a development
// version. A version with an odd-numbered major, minor
// or patch version is considered to be a development version.
func (v Number) IsDev() bool {
	return isOdd(v.Major) || isOdd(v.Minor) || isOdd(v.Patch) || v.Build > 0
}

func readSeries(releaseFile string) string {
	data, err := ioutil.ReadFile(releaseFile)
	if err != nil {
		return "unknown"
	}
	for _, line := range strings.Split(string(data), "\n") {
		const p = "DISTRIB_CODENAME="
		if strings.HasPrefix(line, p) {
			return strings.Trim(line[len(p):], "\t '\"")
		}
	}
	return "unknown"
}

func ubuntuArch(arch string) string {
	if arch == "386" {
		arch = "i386"
	}
	return arch
}
<|endoftext|>


<commit_before>// Copyright 2012, 2013 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

// The version package implements version parsing.
// It also acts as guardian of the current client Juju version number.
package version

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"regexp"
	"runtime"
	"strconv"
	"strings"

	"labix.org/v2/mgo/bson"
)

// The presence and format of this constant is very important.
// The debian/rules build recipe uses this value for the version
// number of the release package.
const version = "1.15.0"

// Current gives the current version of the system.  If the file
// "FORCE-VERSION" is present in the same directory as the running
// binary, it will override this.
var Current = Binary{
	Number: MustParse(version),
	Series: readSeries("/etc/lsb-release"),
	Arch:   ubuntuArch(runtime.GOARCH),
}

func init() {
	toolsDir := filepath.Dir(os.Args[0])
	v, err := ioutil.ReadFile(filepath.Join(toolsDir, "FORCE-VERSION"))
	if err != nil {
		if !os.IsNotExist(err) {
			fmt.Fprintf(os.Stderr, "WARNING: cannot read forced version: %v\n", err)
		}
		return
	}
	Current.Number = MustParse(strings.TrimSpace(string(v)))
}

// Number represents a juju version.  When bugs are fixed the patch
// number is incremented; when new features are added the minor number
// is incremented and patch is reset; and when compatibility is broken
// the major version is incremented and minor and patch are reset.  The
// build number is automatically assigned and has no well defined
// sequence.  If the build number is greater than zero or the minor
// version is odd, it indicates that the release is still in
// development.
type Number struct {
	Major int
	Minor int
	Patch int
	Build int
}

// Zero is occasionally convenient and readable.
// Please don't change its value.
var Zero = Number{}

// Binary specifies a binary version of juju.
type Binary struct {
	Number
	Series string
	Arch   string
}

func (v Binary) String() string {
	return fmt.Sprintf("%v-%s-%s", v.Number, v.Series, v.Arch)
}

// GetBSON turns v into a bson.Getter so it can be saved directly
// on a MongoDB database with mgo.
func (v Binary) GetBSON() (interface{}, error) {
	return v.String(), nil
}

// SetBSON turns v into a bson.Setter so it can be loaded directly
// from a MongoDB database with mgo.
func (vp *Binary) SetBSON(raw bson.Raw) error {
	var s string
	err := raw.Unmarshal(&s)
	if err != nil {
		return err
	}
	v, err := ParseBinary(s)
	if err != nil {
		return err
	}
	*vp = v
	return nil
}

func (v Binary) MarshalJSON() ([]byte, error) {
	return json.Marshal(v.String())
}

func (vp *Binary) UnmarshalJSON(data []byte) error {
	var s string
	if err := json.Unmarshal(data, &s); err != nil {
		return err
	}
	v, err := ParseBinary(s)
	if err != nil {
		return err
	}
	*vp = v
	return nil
}

var (
	binaryPat = regexp.MustCompile(`^(\d{1,9})\.(\d{1,9})\.(\d{1,9})(\.\d{1,9})?-([^-]+)-([^-]+)$`)
	numberPat = regexp.MustCompile(`^(\d{1,9})\.(\d{1,9})\.(\d{1,9})(\.\d{1,9})?$`)
)

// MustParse parses a version and panics if it does
// not parse correctly.
func MustParse(s string) Number {
	v, err := Parse(s)
	if err != nil {
		panic(err)
	}
	return v
}

// MustParseBinary parses a binary version and panics if it does
// not parse correctly.
func MustParseBinary(s string) Binary {
	v, err := ParseBinary(s)
	if err != nil {
		panic(err)
	}
	return v
}

// ParseBinary parses a binary version of the form "1.2.3-series-arch".
func ParseBinary(s string) (Binary, error) {
	m := binaryPat.FindStringSubmatch(s)
	if m == nil {
		return Binary{}, fmt.Errorf("invalid binary version %q", s)
	}
	var v Binary
	v.Major = atoi(m[1])
	v.Minor = atoi(m[2])
	v.Patch = atoi(m[3])
	if m[4] != "" {
		v.Build = atoi(m[4][1:])
	}
	v.Series = m[5]
	v.Arch = m[6]
	return v, nil
}

// Parse parses the version, which is of the form 1.2.3
// giving the major, minor and release versions
// respectively.
func Parse(s string) (Number, error) {
	m := numberPat.FindStringSubmatch(s)
	if m == nil {
		return Number{}, fmt.Errorf("invalid version %q", s)
	}
	var v Number
	v.Major = atoi(m[1])
	v.Minor = atoi(m[2])
	v.Patch = atoi(m[3])
	if m[4] != "" {
		v.Build = atoi(m[4][1:])
	}
	return v, nil
}

// atoi is the same as strconv.Atoi but assumes that
// the string has been verified to be a valid integer.
func atoi(s string) int {
	n, err := strconv.Atoi(s)
	if err != nil {
		panic(err)
	}
	return n
}

func (v Number) String() string {
	s := fmt.Sprintf("%d.%d.%d", v.Major, v.Minor, v.Patch)
	if v.Build > 0 {
		s += fmt.Sprintf(".%d", v.Build)
	}
	return s
}

// Less returns whether v is semantically earlier in the
// version sequence than w.
func (v Number) Less(w Number) bool {
	switch {
	case v.Major != w.Major:
		return v.Major < w.Major
	case v.Minor != w.Minor:
		return v.Minor < w.Minor
	case v.Patch != w.Patch:
		return v.Patch < w.Patch
	case v.Build != w.Build:
		return v.Build < w.Build
	}
	return false
}

// GetBSON turns v into a bson.Getter so it can be saved directly
// on a MongoDB database with mgo.
func (v Number) GetBSON() (interface{}, error) {
	return v.String(), nil
}

// SetBSON turns v into a bson.Setter so it can be loaded directly
// from a MongoDB database with mgo.
func (vp *Number) SetBSON(raw bson.Raw) error {
	var s string
	err := raw.Unmarshal(&s)
	if err != nil {
		return err
	}
	v, err := Parse(s)
	if err != nil {
		return err
	}
	*vp = v
	return nil
}

func (v Number) MarshalJSON() ([]byte, error) {
	return json.Marshal(v.String())
}

func (vp *Number) UnmarshalJSON(data []byte) error {
	var s string
	if err := json.Unmarshal(data, &s); err != nil {
		return err
	}
	v, err := Parse(s)
	if err != nil {
		return err
	}
	*vp = v
	return nil
}

func isOdd(x int) bool {
	return x%2 != 0
}

// IsDev returns whether the version represents a development
// version. A version with an odd-numbered minor component or
// a nonzero build component is considered to be a development
// version.
func (v Number) IsDev() bool {
	return isOdd(v.Minor) || v.Build > 0
}

func readSeries(releaseFile string) string {
	data, err := ioutil.ReadFile(releaseFile)
	if err != nil {
		return "unknown"
	}
	for _, line := range strings.Split(string(data), "\n") {
		const p = "DISTRIB_CODENAME="
		if strings.HasPrefix(line, p) {
			return strings.Trim(line[len(p):], "\t '\"")
		}
	}
	return "unknown"
}

func ubuntuArch(arch string) string {
	if arch == "386" {
		arch = "i386"
	}
	return arch
}

// ParseMajorMinor takes an argument of the form "major.minor" and returns ints major and minor.
func ParseMajorMinor(vers string) (int, int, error) {
	parts := strings.Split(vers, ".")
	major, err := strconv.Atoi(parts[0])
	minor := -1
	if err != nil {
		return -1, -1, fmt.Errorf("invalid major version number %s: %v", parts[0], err)
	}
	if len(parts) == 2 {
		minor, err = strconv.Atoi(parts[1])
		if err != nil {
			return -1, -1, fmt.Errorf("invalid minor version number %s: %v", parts[1], err)
		}
	} else if len(parts) > 2 {
		return -1, -1, fmt.Errorf("invalid major.minor version number %s", vers)
	}
	return major, minor, nil
}
<commit_msg>Increment version to 1.15.1 because 1.15.0 is inthe wild.<commit_after>// Copyright 2012, 2013 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

// The version package implements version parsing.
// It also acts as guardian of the current client Juju version number.
package version

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"regexp"
	"runtime"
	"strconv"
	"strings"

	"labix.org/v2/mgo/bson"
)

// The presence and format of this constant is very important.
// The debian/rules build recipe uses this value for the version
// number of the release package.
const version = "1.15.1"

// Current gives the current version of the system.  If the file
// "FORCE-VERSION" is present in the same directory as the running
// binary, it will override this.
var Current = Binary{
	Number: MustParse(version),
	Series: readSeries("/etc/lsb-release"),
	Arch:   ubuntuArch(runtime.GOARCH),
}

func init() {
	toolsDir := filepath.Dir(os.Args[0])
	v, err := ioutil.ReadFile(filepath.Join(toolsDir, "FORCE-VERSION"))
	if err != nil {
		if !os.IsNotExist(err) {
			fmt.Fprintf(os.Stderr, "WARNING: cannot read forced version: %v\n", err)
		}
		return
	}
	Current.Number = MustParse(strings.TrimSpace(string(v)))
}

// Number represents a juju version.  When bugs are fixed the patch
// number is incremented; when new features are added the minor number
// is incremented and patch is reset; and when compatibility is broken
// the major version is incremented and minor and patch are reset.  The
// build number is automatically assigned and has no well defined
// sequence.  If the build number is greater than zero or the minor
// version is odd, it indicates that the release is still in
// development.
type Number struct {
	Major int
	Minor int
	Patch int
	Build int
}

// Zero is occasionally convenient and readable.
// Please don't change its value.
var Zero = Number{}

// Binary specifies a binary version of juju.
type Binary struct {
	Number
	Series string
	Arch   string
}

func (v Binary) String() string {
	return fmt.Sprintf("%v-%s-%s", v.Number, v.Series, v.Arch)
}

// GetBSON turns v into a bson.Getter so it can be saved directly
// on a MongoDB database with mgo.
func (v Binary) GetBSON() (interface{}, error) {
	return v.String(), nil
}

// SetBSON turns v into a bson.Setter so it can be loaded directly
// from a MongoDB database with mgo.
func (vp *Binary) SetBSON(raw bson.Raw) error {
	var s string
	err := raw.Unmarshal(&s)
	if err != nil {
		return err
	}
	v, err := ParseBinary(s)
	if err != nil {
		return err
	}
	*vp = v
	return nil
}

func (v Binary) MarshalJSON() ([]byte, error) {
	return json.Marshal(v.String())
}

func (vp *Binary) UnmarshalJSON(data []byte) error {
	var s string
	if err := json.Unmarshal(data, &s); err != nil {
		return err
	}
	v, err := ParseBinary(s)
	if err != nil {
		return err
	}
	*vp = v
	return nil
}

var (
	binaryPat = regexp.MustCompile(`^(\d{1,9})\.(\d{1,9})\.(\d{1,9})(\.\d{1,9})?-([^-]+)-([^-]+)$`)
	numberPat = regexp.MustCompile(`^(\d{1,9})\.(\d{1,9})\.(\d{1,9})(\.\d{1,9})?$`)
)

// MustParse parses a version and panics if it does
// not parse correctly.
func MustParse(s string) Number {
	v, err := Parse(s)
	if err != nil {
		panic(err)
	}
	return v
}

// MustParseBinary parses a binary version and panics if it does
// not parse correctly.
func MustParseBinary(s string) Binary {
	v, err := ParseBinary(s)
	if err != nil {
		panic(err)
	}
	return v
}

// ParseBinary parses a binary version of the form "1.2.3-series-arch".
func ParseBinary(s string) (Binary, error) {
	m := binaryPat.FindStringSubmatch(s)
	if m == nil {
		return Binary{}, fmt.Errorf("invalid binary version %q", s)
	}
	var v Binary
	v.Major = atoi(m[1])
	v.Minor = atoi(m[2])
	v.Patch = atoi(m[3])
	if m[4] != "" {
		v.Build = atoi(m[4][1:])
	}
	v.Series = m[5]
	v.Arch = m[6]
	return v, nil
}

// Parse parses the version, which is of the form 1.2.3
// giving the major, minor and release versions
// respectively.
func Parse(s string) (Number, error) {
	m := numberPat.FindStringSubmatch(s)
	if m == nil {
		return Number{}, fmt.Errorf("invalid version %q", s)
	}
	var v Number
	v.Major = atoi(m[1])
	v.Minor = atoi(m[2])
	v.Patch = atoi(m[3])
	if m[4] != "" {
		v.Build = atoi(m[4][1:])
	}
	return v, nil
}

// atoi is the same as strconv.Atoi but assumes that
// the string has been verified to be a valid integer.
func atoi(s string) int {
	n, err := strconv.Atoi(s)
	if err != nil {
		panic(err)
	}
	return n
}

func (v Number) String() string {
	s := fmt.Sprintf("%d.%d.%d", v.Major, v.Minor, v.Patch)
	if v.Build > 0 {
		s += fmt.Sprintf(".%d", v.Build)
	}
	return s
}

// Less returns whether v is semantically earlier in the
// version sequence than w.
func (v Number) Less(w Number) bool {
	switch {
	case v.Major != w.Major:
		return v.Major < w.Major
	case v.Minor != w.Minor:
		return v.Minor < w.Minor
	case v.Patch != w.Patch:
		return v.Patch < w.Patch
	case v.Build != w.Build:
		return v.Build < w.Build
	}
	return false
}

// GetBSON turns v into a bson.Getter so it can be saved directly
// on a MongoDB database with mgo.
func (v Number) GetBSON() (interface{}, error) {
	return v.String(), nil
}

// SetBSON turns v into a bson.Setter so it can be loaded directly
// from a MongoDB database with mgo.
func (vp *Number) SetBSON(raw bson.Raw) error {
	var s string
	err := raw.Unmarshal(&s)
	if err != nil {
		return err
	}
	v, err := Parse(s)
	if err != nil {
		return err
	}
	*vp = v
	return nil
}

func (v Number) MarshalJSON() ([]byte, error) {
	return json.Marshal(v.String())
}

func (vp *Number) UnmarshalJSON(data []byte) error {
	var s string
	if err := json.Unmarshal(data, &s); err != nil {
		return err
	}
	v, err := Parse(s)
	if err != nil {
		return err
	}
	*vp = v
	return nil
}

func isOdd(x int) bool {
	return x%2 != 0
}

// IsDev returns whether the version represents a development
// version. A version with an odd-numbered minor component or
// a nonzero build component is considered to be a development
// version.
func (v Number) IsDev() bool {
	return isOdd(v.Minor) || v.Build > 0
}

func readSeries(releaseFile string) string {
	data, err := ioutil.ReadFile(releaseFile)
	if err != nil {
		return "unknown"
	}
	for _, line := range strings.Split(string(data), "\n") {
		const p = "DISTRIB_CODENAME="
		if strings.HasPrefix(line, p) {
			return strings.Trim(line[len(p):], "\t '\"")
		}
	}
	return "unknown"
}

func ubuntuArch(arch string) string {
	if arch == "386" {
		arch = "i386"
	}
	return arch
}

// ParseMajorMinor takes an argument of the form "major.minor" and returns ints major and minor.
func ParseMajorMinor(vers string) (int, int, error) {
	parts := strings.Split(vers, ".")
	major, err := strconv.Atoi(parts[0])
	minor := -1
	if err != nil {
		return -1, -1, fmt.Errorf("invalid major version number %s: %v", parts[0], err)
	}
	if len(parts) == 2 {
		minor, err = strconv.Atoi(parts[1])
		if err != nil {
			return -1, -1, fmt.Errorf("invalid minor version number %s: %v", parts[1], err)
		}
	} else if len(parts) > 2 {
		return -1, -1, fmt.Errorf("invalid major.minor version number %s", vers)
	}
	return major, minor, nil
}
<|endoftext|>


<commit_before><commit_msg>Rearranged messages ordering<commit_after><|endoftext|>


<commit_before>package mongodb

import (
	"reflect"
	"testing"
	"time"

	"github.com/honeycombio/honeytail/event"
)

const (
	CONTROL             = "2010-01-02T12:34:56.000Z I CONTROL [conn123456789] git version fooooooo"
	UBUNTU_3_2_9_INSERT = `2016-09-14T23:39:23.450+0000 I COMMAND  [conn68] command protecteddb.comedy command: insert { insert: "comedy", documents: [ { _id: ObjectId('57d9dfab9fc9998ce4e0c072'), name: "Bill & Ted's Excellent Adventure", year: 1989.0, extrakey: "benisawesome" } ], ordered: true } ninserted:1 keyUpdates:0 writeConflicts:0 numYields:0 reslen:25 locks:{ Global: { acquireCount: { r: 1, w: 1 } }, Database: { acquireCount: { w: 1 } }, Collection: { acquireCount: { w: 1 } } } protocol:op_command 0ms`
	UBUNTU_3_2_9_FIND   = `2016-09-15T00:01:55.387+0000 I COMMAND [conn93] command protecteddb.comedy command: find { find: "comedy", filter: { $where: "this.year > 2000" } } planSummary: COLLSCAN keysExamined:0 docsExamined:5 cursorExhausted:1 keyUpdates:0 writeConflicts:0 numYields:0 nreturned:2 reslen:245 locks:{ Global: { acquireCount: { r: 7, w: 1 } }, Database: { acquireCount: { r: 3, w: 1 } }, Collection: { acquireCount: { r: 3, w: 1 } }, Metadata: { acquireCount: { W: 1 } } } protocol:op_command 29ms`
	UBUNTU_3_2_9_UPDATE = `2016-09-14T23:36:36.793+0000 I WRITE [conn61] update protecteddb.comedy query: { name: "Hulk" } update: { $unset: { cast: 1.0 } } keysExamined:0 docsExamined:4 nMatched:0 nModified:0 keyUpdates:0 writeConflicts:0 numYields:0 locks:{ Global: { acquireCount: { r: 1, w: 1 } }, Database: { acquireCount: { w: 1 } }, Collection: { acquireCount: { w: 1 } } } 0ms`
	UBUNTU_2_4_FIND     = `Tue Sep 13 21:10:33.961 [TTLMonitor] query btest.system.indexes query: { expireAfterSeconds: { $exists: true } } ntoreturn:0 ntoskip:0 nscanned:1 keyUpdates:0 locks(micros) r:60 nreturned:0 reslen:20 0ms`
	OSX_3_2_9_AGGREGATE = `2016-09-14T14:46:13.879-0700 I COMMAND [conn1] command testtest.zips command: aggregate { aggregate: "zips", pipeline: [ { $group: { _id: "$state", totalPop: { $sum: "$pop" } } }, { $match: { totalPop: { $gte: 10000000.0 } } } ], cursor: {} } keyUpdates:0 writeConflicts:0 numYields:229 reslen:342 locks:{ Global: { acquireCount: { r: 466 } }, Database: { acquireCount: { r: 233 } }, Collection: { acquireCount: { r: 233 } } } protocol:op_command 34ms`

	T1_STRING = "2010-01-02T12:34:56.000Z"
)

var (
	T1                          = time.Date(2010, time.January, 2, 12, 34, 56, 0, time.UTC)
	UBUNTU_3_2_9_INSERT_TIME, _ = time.Parse(iso8601LocalTimeFormat, "2016-09-14T23:39:23.450+0000")
	UBUNTU_3_2_9_FIND_TIME, _   = time.Parse(iso8601LocalTimeFormat, "2016-09-15T00:01:55.387+0000")
	UBUNTU_3_2_9_UPDATE_TIME, _ = time.Parse(iso8601LocalTimeFormat, "2016-09-14T23:36:36.793+0000")
	UBUNTU_2_4_FIND_TIME, _     = time.Parse(ctimeTimeFormat, "Tue Sep 13 21:10:33.961")
	OSX_3_2_9_AGGREGATE_TIME, _ = time.Parse(iso8601LocalTimeFormat, "2016-09-14T14:46:13.879-0700 ")
)

type processed struct {
	time        time.Time
	includeData map[string]interface{}
	excludeKeys []string
}

func TestProcessLines(t *testing.T) {
	locks_string1 := "locks:{ Global: { acquireCount: { r: 2, w: 2 } }, Database: { acquireCount: { w: 2 } }, Collection: { acquireCount: { w: 1 } }, oplog: { acquireCount: { w: 1 } } }"
	tlm := []struct {
		line     string
		expected processed
	}{
		{
			line: CONTROL,
			expected: processed{
				time: T1,
				includeData: map[string]interface{}{
					"severity":  "informational",
					"component": "CONTROL",
					"context":   "conn123456789",
					"message":   "git version fooooooo",
				},
			},
		},

		{
			line: T1_STRING + " I QUERY    [context12345] query database.collection:Stuff query: {} " + locks_string1 + " 0ms",
			expected: processed{
				time: T1,
				includeData: map[string]interface{}{
					"severity":              "informational",
					"component":             "QUERY",
					"context":               "context12345",
					"operation":             "query",
					"namespace":             "database.collection:Stuff",
					"database":              "database",         // decomposed from namespace
					"collection":            "collection:Stuff", // decomposed from namespace
					"query":                 map[string]interface{}{},
					"normalized_query":      "{  }",
					"global_read_lock":      float64(2),
					"global_write_lock":     float64(2),
					"database_write_lock":   float64(2),
					"collection_write_lock": float64(1),
					"oplog_write_lock":      float64(1),
					"duration_ms":           float64(0),
				},
			},
		},

		{
			line: T1_STRING + " I WRITE    [context12345] insert database.collection:Stuff query: {} " + locks_string1 + " 0ms",
			expected: processed{
				time: T1,
				includeData: map[string]interface{}{
					"severity":              "informational",
					"component":             "WRITE",
					"context":               "context12345",
					"operation":             "insert",
					"namespace":             "database.collection:Stuff",
					"database":              "database",         // decomposed from namespace
					"collection":            "collection:Stuff", // decomposed from namespace
					"query":                 map[string]interface{}{},
					"normalized_query":      "{  }",
					"global_read_lock":      float64(2),
					"global_write_lock":     float64(2),
					"database_write_lock":   float64(2),
					"collection_write_lock": float64(1),
					"oplog_write_lock":      float64(1),
					"duration_ms":           float64(0),
				},
				excludeKeys: []string{
					"database_read_lock",
				},
			},
		},

		{
			line: T1_STRING + " I COMMAND    [context12345] command database.$cmd command: insert { } " + locks_string1 + " 0ms",
			expected: processed{
				time: T1,
				includeData: map[string]interface{}{
					"severity":              "informational",
					"component":             "COMMAND",
					"context":               "context12345",
					"operation":             "command",
					"namespace":             "database.$cmd",
					"database":              "database", // decomposed from namespace
					"collection":            "$cmd",     // decomposed from namespace
					"command_type":          "insert",
					"command":               map[string]interface{}{},
					"global_read_lock":      float64(2),
					"global_write_lock":     float64(2),
					"database_write_lock":   float64(2),
					"collection_write_lock": float64(1),
					"oplog_write_lock":      float64(1),
					"duration_ms":           float64(0),
				},
				excludeKeys: []string{
					"database_read_lock",
				},
			},
		},

		{
			line: UBUNTU_3_2_9_INSERT,
			expected: processed{
				time: UBUNTU_3_2_9_INSERT_TIME,
				includeData: map[string]interface{}{
					"severity":              "informational",
					"operation":             "command",
					"command_type":          "insert",
					"namespace":             "protecteddb.comedy",
					"keyUpdates":            0.0,
					"reslen":                25.0,
					"global_read_lock":      1.0,
					"global_write_lock":     1.0,
					"database_write_lock":   1.0,
					"collection_write_lock": 1.0,
				},
				excludeKeys: []string{
					"oplog_write_lock",
					"collection_read_lock",
				},
			},
		},

		{
			line: UBUNTU_3_2_9_FIND,
			expected: processed{
				time: UBUNTU_3_2_9_FIND_TIME,
				includeData: map[string]interface{}{
					"severity":              "informational",
					"operation":             "command",
					"command_type":          "find",
					"namespace":             "protecteddb.comedy",
					"keyUpdates":            0.0,
					"reslen":                245.0,
					"global_read_lock":      7.0,
					"global_write_lock":     1.0,
					"database_read_lock":    3.0,
					"database_write_lock":   1.0,
					"collection_read_lock":  3.0,
					"collection_write_lock": 1.0,
					"docsExamined":          5.0,
					"duration_ms":           29.0,
				},
				excludeKeys: []string{
					"oplog_write_lock",
				},
			},
		},

		{
			line: UBUNTU_3_2_9_UPDATE,
			expected: processed{
				time: UBUNTU_3_2_9_UPDATE_TIME,
				includeData: map[string]interface{}{
					"severity":              "informational",
					"operation":             "update",
					"namespace":             "protecteddb.comedy",
					"keyUpdates":            0.0,
					"global_read_lock":      1.0,
					"global_write_lock":     1.0,
					"database_write_lock":   1.0,
					"collection_write_lock": 1.0,
					"docsExamined":          4.0,
				},
				excludeKeys: []string{
					"database_read_lock",
					"oplog_write_lock",
				},
			},
		},

		{
			line: UBUNTU_2_4_FIND,
			expected: processed{
				time: UBUNTU_2_4_FIND_TIME,
				includeData: map[string]interface{}{
					"command_type": "query",
				},
			},
		},

		{
			line: OSX_3_2_9_AGGREGATE,
			expected: processed{
				time: OSX_3_2_9_AGGREGATE_TIME,
				includeData: map[string]interface{}{
					"duration_ms": 34.0,
				},
				excludeKeys: []string{},
			},
		},
	}
	m := &Parser{
		conf:       Options{},
		nower:      &FakeNower{},
		lineParser: &MongoLineParser{},
	}
	lines := make(chan string)
	send := make(chan event.Event)
	// prep the incoming channel with test lines for the processor
	go func() {
		for _, pair := range tlm {
			lines <- pair.line
		}
		close(lines)
	}()
	// spin up the processor to process our test lines
	go m.ProcessLines(lines, send)
	for _, pair := range tlm {
		ev := <-send

		if ev.Timestamp.UnixNano() != pair.expected.time.UnixNano() {
			t.Errorf("Parsed timestamp didn't match up for %s.\n  Expected: %+v\n  Actual: %+v",
				pair.line, pair.expected.time, ev.Timestamp)
		}

		var missing []string
		for k := range pair.expected.includeData {
			if _, ok := ev.Data[k]; !ok {
				missing = append(missing, k)
			} else if !reflect.DeepEqual(ev.Data[k], pair.expected.includeData[k]) {
				t.Errorf("  Parsed data value %s didn't match up for %s.\n  Expected: %+v\n  Actual: %+v",
					k, pair.line, pair.expected.includeData[k], ev.Data[k])
			}
		}
		if missing != nil {
			t.Errorf("  Parsed data was missing keys for line: %s\n  Missing: %+v\n  Parsed data: %+v",
				pair.line, missing, ev.Data)
		}
		for _, k := range pair.expected.excludeKeys {
			if _, ok := ev.Data[k]; ok {
				t.Errorf("  Parsed data included unexpected key %s for line: %s", k, pair.line)
			}
		}
	}
}

type FakeNower struct{}

func (f *FakeNower) Now() time.Time {
	fakeTime, _ := time.Parse(iso8601UTCTimeFormat, "2010-10-02T12:34:56.000Z")
	return fakeTime
}
<commit_msg>dates missing a year get the year from our nower.  and non-COMMAND log lines don't use command_type.<commit_after>package mongodb

import (
	"reflect"
	"testing"
	"time"

	"github.com/honeycombio/honeytail/event"
)

const (
	CONTROL             = "2010-01-02T12:34:56.000Z I CONTROL [conn123456789] git version fooooooo"
	UBUNTU_3_2_9_INSERT = `2016-09-14T23:39:23.450+0000 I COMMAND  [conn68] command protecteddb.comedy command: insert { insert: "comedy", documents: [ { _id: ObjectId('57d9dfab9fc9998ce4e0c072'), name: "Bill & Ted's Excellent Adventure", year: 1989.0, extrakey: "benisawesome" } ], ordered: true } ninserted:1 keyUpdates:0 writeConflicts:0 numYields:0 reslen:25 locks:{ Global: { acquireCount: { r: 1, w: 1 } }, Database: { acquireCount: { w: 1 } }, Collection: { acquireCount: { w: 1 } } } protocol:op_command 0ms`
	UBUNTU_3_2_9_FIND   = `2016-09-15T00:01:55.387+0000 I COMMAND [conn93] command protecteddb.comedy command: find { find: "comedy", filter: { $where: "this.year > 2000" } } planSummary: COLLSCAN keysExamined:0 docsExamined:5 cursorExhausted:1 keyUpdates:0 writeConflicts:0 numYields:0 nreturned:2 reslen:245 locks:{ Global: { acquireCount: { r: 7, w: 1 } }, Database: { acquireCount: { r: 3, w: 1 } }, Collection: { acquireCount: { r: 3, w: 1 } }, Metadata: { acquireCount: { W: 1 } } } protocol:op_command 29ms`
	UBUNTU_3_2_9_UPDATE = `2016-09-14T23:36:36.793+0000 I WRITE [conn61] update protecteddb.comedy query: { name: "Hulk" } update: { $unset: { cast: 1.0 } } keysExamined:0 docsExamined:4 nMatched:0 nModified:0 keyUpdates:0 writeConflicts:0 numYields:0 locks:{ Global: { acquireCount: { r: 1, w: 1 } }, Database: { acquireCount: { w: 1 } }, Collection: { acquireCount: { w: 1 } } } 0ms`
	UBUNTU_2_4_FIND     = `Tue Sep 13 21:10:33.961 [TTLMonitor] query btest.system.indexes query: { expireAfterSeconds: { $exists: true } } ntoreturn:0 ntoskip:0 nscanned:1 keyUpdates:0 locks(micros) r:60 nreturned:0 reslen:20 0ms`
	OSX_3_2_9_AGGREGATE = `2016-09-14T14:46:13.879-0700 I COMMAND [conn1] command testtest.zips command: aggregate { aggregate: "zips", pipeline: [ { $group: { _id: "$state", totalPop: { $sum: "$pop" } } }, { $match: { totalPop: { $gte: 10000000.0 } } } ], cursor: {} } keyUpdates:0 writeConflicts:0 numYields:229 reslen:342 locks:{ Global: { acquireCount: { r: 466 } }, Database: { acquireCount: { r: 233 } }, Collection: { acquireCount: { r: 233 } } } protocol:op_command 34ms`

	T1_STRING = "2010-01-02T12:34:56.000Z"
)

var (
	T1                          = time.Date(2010, time.January, 2, 12, 34, 56, 0, time.UTC)
	UBUNTU_3_2_9_INSERT_TIME, _ = time.Parse(iso8601LocalTimeFormat, "2016-09-14T23:39:23.450+0000")
	UBUNTU_3_2_9_FIND_TIME, _   = time.Parse(iso8601LocalTimeFormat, "2016-09-15T00:01:55.387+0000")
	UBUNTU_3_2_9_UPDATE_TIME, _ = time.Parse(iso8601LocalTimeFormat, "2016-09-14T23:36:36.793+0000")
	UBUNTU_2_4_FIND_TIME, _     = time.Parse(ctimeTimeFormat, "Tue Sep 13 21:10:33.961")
	OSX_3_2_9_AGGREGATE_TIME, _ = time.Parse(iso8601LocalTimeFormat, "2016-09-14T14:46:13.879-0700")
)

type processed struct {
	time        time.Time
	includeData map[string]interface{}
	excludeKeys []string
}

func TestProcessLines(t *testing.T) {
	nower := &FakeNower{}
	locks_string1 := "locks:{ Global: { acquireCount: { r: 2, w: 2 } }, Database: { acquireCount: { w: 2 } }, Collection: { acquireCount: { w: 1 } }, oplog: { acquireCount: { w: 1 } } }"
	tlm := []struct {
		line     string
		expected processed
	}{
		{
			line: CONTROL,
			expected: processed{
				time: T1,
				includeData: map[string]interface{}{
					"severity":  "informational",
					"component": "CONTROL",
					"context":   "conn123456789",
					"message":   "git version fooooooo",
				},
			},
		},

		{
			line: T1_STRING + " I QUERY    [context12345] query database.collection:Stuff query: {} " + locks_string1 + " 0ms",
			expected: processed{
				time: T1,
				includeData: map[string]interface{}{
					"severity":              "informational",
					"component":             "QUERY",
					"context":               "context12345",
					"operation":             "query",
					"namespace":             "database.collection:Stuff",
					"database":              "database",         // decomposed from namespace
					"collection":            "collection:Stuff", // decomposed from namespace
					"query":                 map[string]interface{}{},
					"normalized_query":      "{  }",
					"global_read_lock":      float64(2),
					"global_write_lock":     float64(2),
					"database_write_lock":   float64(2),
					"collection_write_lock": float64(1),
					"oplog_write_lock":      float64(1),
					"duration_ms":           float64(0),
				},
			},
		},

		{
			line: T1_STRING + " I WRITE    [context12345] insert database.collection:Stuff query: {} " + locks_string1 + " 0ms",
			expected: processed{
				time: T1,
				includeData: map[string]interface{}{
					"severity":              "informational",
					"component":             "WRITE",
					"context":               "context12345",
					"operation":             "insert",
					"namespace":             "database.collection:Stuff",
					"database":              "database",         // decomposed from namespace
					"collection":            "collection:Stuff", // decomposed from namespace
					"query":                 map[string]interface{}{},
					"normalized_query":      "{  }",
					"global_read_lock":      float64(2),
					"global_write_lock":     float64(2),
					"database_write_lock":   float64(2),
					"collection_write_lock": float64(1),
					"oplog_write_lock":      float64(1),
					"duration_ms":           float64(0),
				},
				excludeKeys: []string{
					"database_read_lock",
				},
			},
		},

		{
			line: T1_STRING + " I COMMAND    [context12345] command database.$cmd command: insert { } " + locks_string1 + " 0ms",
			expected: processed{
				time: T1,
				includeData: map[string]interface{}{
					"severity":              "informational",
					"component":             "COMMAND",
					"context":               "context12345",
					"operation":             "command",
					"namespace":             "database.$cmd",
					"database":              "database", // decomposed from namespace
					"collection":            "$cmd",     // decomposed from namespace
					"command_type":          "insert",
					"command":               map[string]interface{}{},
					"global_read_lock":      float64(2),
					"global_write_lock":     float64(2),
					"database_write_lock":   float64(2),
					"collection_write_lock": float64(1),
					"oplog_write_lock":      float64(1),
					"duration_ms":           float64(0),
				},
				excludeKeys: []string{
					"database_read_lock",
				},
			},
		},

		{
			line: UBUNTU_3_2_9_INSERT,
			expected: processed{
				time: UBUNTU_3_2_9_INSERT_TIME,
				includeData: map[string]interface{}{
					"severity":              "informational",
					"operation":             "command",
					"command_type":          "insert",
					"namespace":             "protecteddb.comedy",
					"keyUpdates":            0.0,
					"reslen":                25.0,
					"global_read_lock":      1.0,
					"global_write_lock":     1.0,
					"database_write_lock":   1.0,
					"collection_write_lock": 1.0,
				},
				excludeKeys: []string{
					"oplog_write_lock",
					"collection_read_lock",
				},
			},
		},

		{
			line: UBUNTU_3_2_9_FIND,
			expected: processed{
				time: UBUNTU_3_2_9_FIND_TIME,
				includeData: map[string]interface{}{
					"severity":              "informational",
					"operation":             "command",
					"command_type":          "find",
					"namespace":             "protecteddb.comedy",
					"keyUpdates":            0.0,
					"reslen":                245.0,
					"global_read_lock":      7.0,
					"global_write_lock":     1.0,
					"database_read_lock":    3.0,
					"database_write_lock":   1.0,
					"collection_read_lock":  3.0,
					"collection_write_lock": 1.0,
					"docsExamined":          5.0,
					"duration_ms":           29.0,
				},
				excludeKeys: []string{
					"oplog_write_lock",
				},
			},
		},

		{
			line: UBUNTU_3_2_9_UPDATE,
			expected: processed{
				time: UBUNTU_3_2_9_UPDATE_TIME,
				includeData: map[string]interface{}{
					"severity":              "informational",
					"operation":             "update",
					"namespace":             "protecteddb.comedy",
					"keyUpdates":            0.0,
					"global_read_lock":      1.0,
					"global_write_lock":     1.0,
					"database_write_lock":   1.0,
					"collection_write_lock": 1.0,
					"docsExamined":          4.0,
				},
				excludeKeys: []string{
					"database_read_lock",
					"oplog_write_lock",
				},
			},
		},

		{
			line: UBUNTU_2_4_FIND,
			expected: processed{
				time: UBUNTU_2_4_FIND_TIME.AddDate(nower.Now().Year(), 0, 0),
				includeData: map[string]interface{}{
					"operation": "query",
				},
			},
		},

		{
			line: OSX_3_2_9_AGGREGATE,
			expected: processed{
				time: OSX_3_2_9_AGGREGATE_TIME,
				includeData: map[string]interface{}{
					"duration_ms": 34.0,
				},
				excludeKeys: []string{},
			},
		},
	}
	m := &Parser{
		conf:       Options{},
		nower:      nower,
		lineParser: &MongoLineParser{},
	}
	lines := make(chan string)
	send := make(chan event.Event)
	// prep the incoming channel with test lines for the processor
	go func() {
		for _, pair := range tlm {
			lines <- pair.line
		}
		close(lines)
	}()
	// spin up the processor to process our test lines
	go m.ProcessLines(lines, send)
	for _, pair := range tlm {
		ev := <-send

		if ev.Timestamp.UnixNano() != pair.expected.time.UnixNano() {
			t.Errorf("Parsed timestamp didn't match up for %s.\n  Expected: %+v\n  Actual: %+v",
				pair.line, pair.expected.time, ev.Timestamp)
		}

		var missing []string
		for k := range pair.expected.includeData {
			if _, ok := ev.Data[k]; !ok {
				missing = append(missing, k)
			} else if !reflect.DeepEqual(ev.Data[k], pair.expected.includeData[k]) {
				t.Errorf("  Parsed data value %s didn't match up for %s.\n  Expected: %+v\n  Actual: %+v",
					k, pair.line, pair.expected.includeData[k], ev.Data[k])
			}
		}
		if missing != nil {
			t.Errorf("  Parsed data was missing keys for line: %s\n  Missing: %+v\n  Parsed data: %+v",
				pair.line, missing, ev.Data)
		}
		for _, k := range pair.expected.excludeKeys {
			if _, ok := ev.Data[k]; ok {
				t.Errorf("  Parsed data included unexpected key %s for line: %s", k, pair.line)
			}
		}
	}
}

type FakeNower struct{}

func (f *FakeNower) Now() time.Time {
	fakeTime, _ := time.Parse(iso8601UTCTimeFormat, "2010-10-02T12:34:56.000Z")
	return fakeTime
}
<|endoftext|>


<commit_before>package database

import (
	"database/sql"
)

type Driver interface {
	Queryer
	QueryRower
	Execer
	Close() error
	Ping() error
	Prepare(string) (*sql.Stmt, error)
	SetMaxIdleConns(int)
	SetMaxOpenConns(int)
}

type Transaction interface {
	Queryer
	QueryRower
	Execer
	Rollback() error
	Commit() error
	Prepare(string) (*sql.Stmt, error)
	Stmt(*sql.Stmt) *sql.Stmt
}

type Model interface {
}

type Slice interface {
}

type Queryer interface {
	Query(string, ...interface{}) (Rower, error)
}

type QueryRower interface {
	QueryRow(string, ...interface{}) (Rower, error)
}

type Execer interface {
	Exec(string, ...interface{}) (Result, error)
}

type Scanner interface {
	Scan(...interface{}) error
}

type Rower interface {
	Scanner
	Next() bool
	Columns() ([]string, error)
	Close() error
}

type Result interface {
	sql.Result
	// InsertIds() ([]interface{}, error)
}

type Tabler interface {
	Driver
	Name() string
	Columns(string) string
}

type Table struct {
	name    string
	schema  string
	columns map[string]string
}

func (t *Table) SetName(name string) {
	if t.name == "" {
		t.name = name
	}
}

func (t *Table) SetSchema(schema string) {
	if t.schema == "" {
		t.schema = schema
	}
}

func (t *Table) SetColumns(columns map[string]string) {
	if t.columns == nil || len(t.columns) == 0 {
		t.columns = columns
	}
}

func (t *Table) Name() string {
	return t.name
}

func (t *Table) Column(a string) string {
	return t.columns[a]
}

func (t *Table) Columns(a ...string) []string {
	var b []string

	for _, c := range a {
		b = append(b, t.columns[c])
	}

	return b
}

type DB struct {
	*sql.DB
	name string
}

type Tx struct {
	*sql.Tx
}

func New() *DB {
	return &DB{}
}

func (db *DB) SetName(name string) {
	db.name = name
}

func (db *DB) Query(query string, args ...interface{}) (Rower, error) {
	return db.DB.Query(query, args...)
}

func (db *DB) Begin() (Transaction, error) {
	xtx, err := db.DB.Begin()
	if err != nil {
		return nil, err
	}
	return &Tx{xtx}, err
}

func (db *DB) QueryRow(query string, args ...interface{}) (Rower, error) {
	return db.DB.Query(query, args...)
}

func (db *DB) Exec(query string, args ...interface{}) (Result, error) {
	return db.DB.Exec(query, args...)
}

func (tx *Tx) Query(query string, args ...interface{}) (Rower, error) {
	return tx.Tx.Query(query, args...)
}

func (tx *Tx) QueryRow(query string, args ...interface{}) (Rower, error) {
	return tx.Tx.Query(query, args...)
}

func (tx *Tx) Exec(query string, args ...interface{}) (Result, error) {
	return tx.Tx.Exec(query, args...)
}
<commit_msg>Changes to database interfaces<commit_after>package database

import (
	"database/sql"

	"github.com/wlMalk/gorator/database/query"
)

type Runner interface {
	Queryer
	QueryRower
	Execer
}

type Driver interface {
	Runner
	Close() error
	Ping() error
	Prepare(string) (*sql.Stmt, error)
	SetMaxIdleConns(int)
	SetMaxOpenConns(int)
}

type Transaction interface {
	Runner
	Rollback() error
	Commit() error
	Prepare(string) (*sql.Stmt, error)
	Stmt(*sql.Stmt) *sql.Stmt
}

type Model interface {
}

type Slice interface {
}

type Queryer interface {
	QueryQ(query.Query) (Rower, error)
	Query(string, ...interface{}) (Rower, error)
}

type QueryRower interface {
	QueryRowQ(query.Query) (Rower, error)
	QueryRow(string, ...interface{}) (Rower, error)
}

type Execer interface {
	ExecQ(query.Query) (Result, error)
	Exec(string, ...interface{}) (Result, error)
}

type Scanner interface {
	Scan(...interface{}) error
}

type Rower interface {
	Scanner
	Next() bool
	Columns() ([]string, error)
	Close() error
}

type Result interface {
	sql.Result
	// InsertIds() ([]interface{}, error)
}

type Tabler interface {
	Driver
	Name() string
	Columns(string) string
}

type Table struct {
	name    string
	schema  string
	columns map[string]string
}

func (t *Table) SetName(name string) {
	if t.name == "" {
		t.name = name
	}
}

func (t *Table) SetSchema(schema string) {
	if t.schema == "" {
		t.schema = schema
	}
}

func (t *Table) SetColumns(columns map[string]string) {
	if t.columns == nil || len(t.columns) == 0 {
		t.columns = columns
	}
}

func (t *Table) Name() string {
	return t.name
}

func (t *Table) Column(a string) string {
	return t.columns[a]
}

func (t *Table) Columns(a ...string) []string {
	var b []string

	for _, c := range a {
		b = append(b, t.columns[c])
	}

	return b
}

type DB struct {
	*sql.DB
	name string
}

type Tx struct {
	*sql.Tx
}

func New() *DB {
	return &DB{}
}

func (db *DB) SetName(name string) {
	db.name = name
}

func (db *DB) Begin() (Transaction, error) {
	xtx, err := db.DB.Begin()
	if err != nil {
		return nil, err
	}
	return &Tx{xtx}, err
}

func (db *DB) Query(query string, args ...interface{}) (Rower, error) {
	return db.DB.Query(query, args...)
}

func (db *DB) QueryRow(query string, args ...interface{}) (Rower, error) {
	return db.DB.Query(query, args...)
}

func (db *DB) Exec(query string, args ...interface{}) (Result, error) {
	return db.DB.Exec(query, args...)
}

func (tx *Tx) Query(query string, args ...interface{}) (Rower, error) {
	return tx.Tx.Query(query, args...)
}

func (tx *Tx) QueryRow(query string, args ...interface{}) (Rower, error) {
	return tx.Tx.Query(query, args...)
}

func (tx *Tx) Exec(query string, args ...interface{}) (Result, error) {
	return tx.Tx.Exec(query, args...)
}

func (db *DB) QueryQ(q query.Query) (Rower, error) {
	str, args, err := q.ToSql()
	if err != nil {
		return nil, err
	}
	return db.DB.Query(str, args...)
}

func (db *DB) QueryRowQ(q query.Query) (Rower, error) {
	str, args, err := q.ToSql()
	if err != nil {
		return nil, err
	}
	return db.DB.Query(str, args...)
}

func (db *DB) ExecQ(q query.Query) (Result, error) {
	str, args, err := q.ToSql()
	if err != nil {
		return nil, err
	}
	return db.DB.Exec(str, args...)
}

func (tx *Tx) QueryQ(q query.Query) (Rower, error) {
	str, args, err := q.ToSql()
	if err != nil {
		return nil, err
	}
	return tx.Tx.Query(str, args...)
}

func (tx *Tx) QueryRowQ(q query.Query) (Rower, error) {
	str, args, err := q.ToSql()
	if err != nil {
		return nil, err
	}
	return tx.Tx.Query(str, args...)
}

func (tx *Tx) ExecQ(q query.Query) (Result, error) {
	str, args, err := q.ToSql()
	if err != nil {
		return nil, err
	}
	return tx.Tx.Exec(str, args...)
}
<|endoftext|>


<commit_before>package data

import (
	"io"

	"github.com/leeola/errors"
	ct "github.com/leeola/kala/contenttype"
	"github.com/leeola/kala/contenttype/ctutil"
	"github.com/leeola/kala/index"
	"github.com/leeola/kala/store"
	"github.com/leeola/kala/store/roller/camli"
)

type Config struct {
	Store store.Store
	Index index.Indexer
}

type Data struct {
	store store.Store
	index index.Indexer
}

func New(c Config) (*Data, error) {
	if c.Store == nil {
		return nil, errors.New("missing required Config field: Store")
	}
	if c.Index == nil {
		return nil, errors.New("missing required Config field: Index")
	}

	return &Data{
		store: c.Store,
		index: c.Index,
	}, nil
}

// TODO(leeola): centralize the common tasks in this method into helpers.
// A lot of this (writing content roller and multipart, etc) is going to be
// duplicated on every ContentType handler.
func (f *Data) StoreContent(rc io.ReadCloser, v store.Version, c ct.Changes) ([]string, error) {
	if rc == nil {
		return nil, errors.New("missing ReadCloser")
	}
	defer rc.Close()

	roller, err := camli.New(rc)
	if err != nil {
		return nil, errors.Stack(err)
	}

	// write the actual content
	hashes, err := ctutil.WritePartRoller(f.store, f.index, roller)
	if err != nil {
		return nil, errors.Stack(err)
	}

	// write the multipart
	h, err := store.WriteMultiPart(f.store, store.MultiPart{
		Parts: hashes,
	})
	if err != nil {
		return nil, errors.Stack(err)
	}
	hashes = append(hashes, h)
	c.SetMultiHash(h)

	// Write the entries, not including the final metadata hash
	// The last hash is metadata, and we'll add that manually.
	for _, h := range hashes {
		if err := f.index.Entry(h); err != nil {
			return nil, errors.Stack(err)
		}
	}

	metaHashes, err := f.StoreMeta(v, c)
	if err != nil {
		return nil, errors.Stack(err)
	}

	return append(hashes, metaHashes...), nil
}

func (f *Data) StoreMeta(v store.Version, c ct.Changes) ([]string, error) {
	var meta ct.Meta

	if v.Meta != "" {
		if err := store.ReadAndUnmarshal(f.store, v.Meta, &meta); err != nil {
			return nil, errors.Stack(err)
		}
	}

	mH, vH, err := ct.WriteMetaAndVersion(f.store, f.index, v, meta)
	if err != nil {
		return nil, errors.Stack(err)
	}

	return []string{mH, vH}, nil
}

func (d *Data) MetaToChanges([]byte) (ct.Changes, error) {
	return nil, errors.New("not implemented")
}
<commit_msg>fix(contenttype/data): apply changes to meta<commit_after>package data

import (
	"io"

	"github.com/leeola/errors"
	ct "github.com/leeola/kala/contenttype"
	"github.com/leeola/kala/contenttype/ctutil"
	"github.com/leeola/kala/index"
	"github.com/leeola/kala/store"
	"github.com/leeola/kala/store/roller/camli"
)

type Config struct {
	Store store.Store
	Index index.Indexer
}

type Data struct {
	store store.Store
	index index.Indexer
}

func New(c Config) (*Data, error) {
	if c.Store == nil {
		return nil, errors.New("missing required Config field: Store")
	}
	if c.Index == nil {
		return nil, errors.New("missing required Config field: Index")
	}

	return &Data{
		store: c.Store,
		index: c.Index,
	}, nil
}

// TODO(leeola): centralize the common tasks in this method into helpers.
// A lot of this (writing content roller and multipart, etc) is going to be
// duplicated on every ContentType handler.
func (f *Data) StoreContent(rc io.ReadCloser, v store.Version, c ct.Changes) ([]string, error) {
	if rc == nil {
		return nil, errors.New("missing ReadCloser")
	}
	defer rc.Close()

	roller, err := camli.New(rc)
	if err != nil {
		return nil, errors.Stack(err)
	}

	// write the actual content
	hashes, err := ctutil.WritePartRoller(f.store, f.index, roller)
	if err != nil {
		return nil, errors.Stack(err)
	}

	// write the multipart
	h, err := store.WriteMultiPart(f.store, store.MultiPart{
		Parts: hashes,
	})
	if err != nil {
		return nil, errors.Stack(err)
	}
	hashes = append(hashes, h)
	c.SetMultiHash(h)

	// Write the entries, not including the final metadata hash
	// The last hash is metadata, and we'll add that manually.
	for _, h := range hashes {
		if err := f.index.Entry(h); err != nil {
			return nil, errors.Stack(err)
		}
	}

	metaHashes, err := f.StoreMeta(v, c)
	if err != nil {
		return nil, errors.Stack(err)
	}

	return append(hashes, metaHashes...), nil
}

func (f *Data) StoreMeta(v store.Version, c ct.Changes) ([]string, error) {
	var meta ct.Meta

	if v.Meta != "" {
		if err := store.ReadAndUnmarshal(f.store, v.Meta, &meta); err != nil {
			return nil, errors.Stack(err)
		}
	}

	// update the meta with any changes matching the meta
	meta.FromChanges(c)

	mH, vH, err := ct.WriteMetaAndVersion(f.store, f.index, v, meta)
	if err != nil {
		return nil, errors.Stack(err)
	}

	return []string{mH, vH}, nil
}

func (d *Data) MetaToChanges([]byte) (ct.Changes, error) {
	return nil, errors.New("not implemented")
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"log"
	"os"

	"github.com/koron/go-ssdp"
)

func main() {
	t := flag.String("t", ssdp.All, "search type")
	w := flag.Int("w", 1, "wait time")
	l := flag.String("l", "", "local address to listen")
	v := flag.Bool("v", false, "verbose mode")
	h := flag.Bool("h", false, "show help")
	flag.Parse()
	if *h {
		flag.Usage()
		return
	}
	if *v {
		ssdp.Logger = log.New(os.Stderr, "[SSDP] ", log.LstdFlags)
	}
	list, err := ssdp.Search(*t, *w, *l)
	if err != nil {
		log.Fatal(err)
	}
	for i, srv := range list {
		//fmt.Printf("%d: %#v\n", i, srv)
		fmt.Printf("%d: %s %s\n", i, srv.Type, srv.Location)
	}
}
<commit_msg>Update search example<commit_after>package main

import (
	"flag"
	"fmt"
	"log"
	"os"

	ssdp "github.com/pulento/go-ssdp"
)

func main() {
	t := flag.String("t", ssdp.All, "search type")
	w := flag.Int("w", 1, "wait time")
	l := flag.String("l", "", "local address to listen")
	m := flag.String("m", "", "multicast address to send")
	v := flag.Bool("v", false, "verbose mode")
	h := flag.Bool("h", false, "show help")
	flag.Parse()
	if *h {
		flag.Usage()
		return
	}
	if *v {
		ssdp.Logger = log.New(os.Stderr, "[SSDP] ", log.LstdFlags)
	}
	list, err := ssdp.Search(*t, *w, *l, *m)
	if err != nil {
		log.Fatal(err)
	}
	for i, srv := range list {
		//fmt.Printf("%d: %#v\n", i, srv)
		fmt.Printf("%d: %s %s\n", i, srv.Type, srv.Location)
	}
}
<|endoftext|>


<commit_before>package datecalc

var days []string = []string{
    "Sunday", "Monday", "Tuesday", "Wednesday", "Thursday", "Friday", "Saturday",
}
var calTypes []string = []string{
    "GREGORIAN", "CE", "JULIAN", "ENGLISH", "ROMAN",
}
var monthNames []string = []string{
    "January", "February", "March", "April", "May", "June", "July", "August", "September", "October", "November", "December",
}

func inRange(x, min, max int8) bool {
    for i := min; i < max; i ++ {
        if i == x {
            return true
        }
    }
    return false
}

func ceLeapYear(y int) bool {
    if y % 100 == 0 {
        if y % 400 == 0 {
            return true
        }
    } else if y % 4 == 0 {
        return true
    }
    return false
}

func julLeapYear(y int) bool {
    if y < 0 {
        y = (((y + 1) % 700) + 700) % 700
    }
    if y % 4 == 0 {
        return true
    }
    return false
}

func isLeapYear(year int, calType string) bool {
    if calType == "GREGORIAN" {
        newYear := year
        if year < 0 {
            newYear = (((year + 1) % 400) + 400) % 400
        }
        return ceLeapYear(newYear)
    } else if calType == "CE" {
        return ceLeapYear(year)
    } else if calType == "JULIAN" {
        return julLeapYear(year)
    } else if calType == "ENGLISH" {
        if year >= 1800 {
            return ceLeapYear(year)
        } else {
            return julLeapYear(year)
        }
    } else if calType == "ROMAN" {
        if year >= 1700 {
            return ceLeapYear(year)
        } else {
            return julLeapYear(year)
        }
    }
    return false
}

func isRealDate(year int, month, date int8, calType string) int {
    month30 := []int8{4, 6, 9, 11}
    if inRange(month, 1, 13) == false {
        return 0
    }
    if date < 1 || date > 31 {
        return 0
    }
    inTypes := false
    for _, x := range calTypes {
        if calType == x {
            inTypes = true
            break
        }
    }
    if !inTypes {
        return 0
    }
    if year == 0 && calType != "CE" {
        return 0
    } else {
        inMonths := false
        for _, x := range month30 {
            if month == x {
                inMonths = true
                break
            }
        }
        if inMonths && date > 30 {
            return 0
        } else if month == 1 && !(date < 29 || (isLeapYear(year, calType) && date == 29)) {
            return 0
        }
    }
    if calType == "ENGLISH" {
        if month == 9 && date > 2 && date < 15 && year == 1752 {
            return 0
        }
    } else if calType == "ROMAN" {
        if month == 10 && date > 4 && date < 16 && year == 1582 {
            return 0
        }
    }
    return 1
}

func addxxYY(year int, calType string) int8 {
    newYear := ((year % 100) + 100) % 100
    if calType != "CE" && year < 0 {
        newYear = (((year + 1) % 100) + 100) % 100
    }
    return  int8((((newYear / 12 + (((newYear % 12) + 12) % 12) + ((((newYear % 12) / 12) % 12) / 4)) % 7) + 7) % 7)
}

func ceAddYYxx(y int) int8 {
    YYxx := []int8{2, 0, 5, 3}
    return YYxx[(((y / 100) % 4) + 4) % 4]
}

func julAddYYxx(y int) int8 {
    return int8((((7 - y / 100) % 7) + 7) % 7)
}

func addYYxx(year int, month, date int8, calType string) int8{
    if calType == "GREGORIAN" {
        newYear := year
        if year < 0 {
            newYear = (((year + 1) % 400) + 400) % 400
        newYear = ((newYear % 400) + 400) % 400
        return ceAddYYxx(newYear)
        }
    } else if calType == "CE" {
        newYear := ((year % 400) + 400) % 400
        return ceAddYYxx(newYear)
    } else if calType == "JULIAN" {
        newYear := year
        if year < 0 {
            newYear = (((year + 1) % 700) + 700) % 700
        }
        newYear = ((newYear % 700) + 700) % 700
        return julAddYYxx(newYear)
    } else if calType == "ENGLISH" {
        if year >= 1752 {
            if year == 1752 {
                if inRange(month, 9, 13) {
                    if month == 9 {
                        if date >= 14 {
                            return 0
                        } else {
                            return 4
                        }
                    } else {
                        return 0
                    }
                } else {
                    return 4
                }
            } else {
                newYear := ((year % 400) + 400) % 400
                return ceAddYYxx(newYear)
            }
        } else {
            newYear := year
            if year < 0 {
                newYear = (((year + 1) % 700) + 700) % 700
            newYear = ((newYear % 700) + 700) % 700
            return julAddYYxx(newYear)
            }
        }
    } else if calType == "ROMAN" {
        if year >= 1582 {
            if year == 1582 {
                if inRange(month, 10, 13) {
                    if month == 10 {
                        if date >= 15 {
                            return 3
                        } else {
                            return 6
                        }
                    } else {
                        return 3
                    }
                } else {
                    return 6
                }
            } else {
                newYear := ((year % 400) + 400) % 400
                return ceAddYYxx(newYear)
            }
        } else {
            newYear := ((year % 400) + 400) % 400
            if year < 0 {
                newYear = (((year + 1) % 700) + 700) % 700
            newYear = ((newYear % 700) + 700) % 700
            return julAddYYxx(newYear)
            }
        }
    }
    return 0
}

func addYear(year int, month, date int8, calType string) int8{
    return addYYxx(year, month, date, calType) + addxxYY(year, calType)
}

func addMonth(year int, month int8, calType string) int8 {
    monthOffsetKey := []int8{0, 0, 0, 3, 5, 1, 3, 6, 2, 4, 0, 2}
    monthOffset := make([][]int8, 12)
    for i, value := range monthOffsetKey {
        if i == 0 {
            monthOffset[i] = []int8{4, 3}
        } else if i == 1 {
            monthOffset[i] = []int8{0, 6}
        } else {
            monthOffset[i] = []int8{value}
        }
    }
    if isLeapYear(year, calType) && len(monthOffset[month - 1]) > 1 {
        return monthOffset[month - 1][1]
    }
    return monthOffset[month - 1][0]
}

func Date(year int, month, date int8, calType string) string {
    check := isRealDate(year, month, date, calType)
    if check == 1 {
        total := (((addYear(year, month, date, calType) + addMonth(year, month, calType) + date) % 7) + 7) % 7
        return days[total]
    }
    return ""
}
<commit_msg>datecalc.go deleted online with Bitbucket<commit_after><|endoftext|>


<commit_before>package azure

import (
	"context"
	"io"
	"net/http"
	"os"
	"path"
	"strings"

	"github.com/Azure/azure-sdk-for-go/storage"
	"github.com/restic/restic/internal/backend"
	"github.com/restic/restic/internal/debug"
	"github.com/restic/restic/internal/errors"
	"github.com/restic/restic/internal/restic"
)

// Backend stores data on an azure endpoint.
type Backend struct {
	accountName  string
	container    *storage.Container
	sem          *backend.Semaphore
	prefix       string
	listMaxItems int
	backend.Layout
}

const defaultListMaxItems = 5000

// make sure that *Backend implements backend.Backend
var _ restic.Backend = &Backend{}

func open(cfg Config, rt http.RoundTripper) (*Backend, error) {
	debug.Log("open, config %#v", cfg)

	client, err := storage.NewBasicClient(cfg.AccountName, cfg.AccountKey)
	if err != nil {
		return nil, errors.Wrap(err, "NewBasicClient")
	}

	client.HTTPClient = &http.Client{Transport: rt}

	service := client.GetBlobService()

	sem, err := backend.NewSemaphore(cfg.Connections)
	if err != nil {
		return nil, err
	}

	be := &Backend{
		container:   service.GetContainerReference(cfg.Container),
		accountName: cfg.AccountName,
		sem:         sem,
		prefix:      cfg.Prefix,
		Layout: &backend.DefaultLayout{
			Path: cfg.Prefix,
			Join: path.Join,
		},
		listMaxItems: defaultListMaxItems,
	}

	return be, nil
}

// Open opens the Azure backend at specified container.
func Open(cfg Config, rt http.RoundTripper) (restic.Backend, error) {
	return open(cfg, rt)
}

// Create opens the Azure backend at specified container and creates the container if
// it does not exist yet.
func Create(cfg Config, rt http.RoundTripper) (restic.Backend, error) {
	be, err := open(cfg, rt)

	if err != nil {
		return nil, errors.Wrap(err, "open")
	}

	options := storage.CreateContainerOptions{
		Access: storage.ContainerAccessTypePrivate,
	}

	_, err = be.container.CreateIfNotExists(&options)
	if err != nil {
		return nil, errors.Wrap(err, "container.CreateIfNotExists")
	}

	return be, nil
}

// SetListMaxItems sets the number of list items to load per request.
func (be *Backend) SetListMaxItems(i int) {
	be.listMaxItems = i
}

// IsNotExist returns true if the error is caused by a not existing file.
func (be *Backend) IsNotExist(err error) bool {
	debug.Log("IsNotExist(%T, %#v)", err, err)
	return os.IsNotExist(err)
}

// Join combines path components with slashes.
func (be *Backend) Join(p ...string) string {
	return path.Join(p...)
}

// Location returns this backend's location (the container name).
func (be *Backend) Location() string {
	return be.Join(be.container.Name, be.prefix)
}

// Path returns the path in the bucket that is used for this backend.
func (be *Backend) Path() string {
	return be.prefix
}

// preventCloser wraps an io.Reader to run a function instead of the original Close() function.
type preventCloser struct {
	io.Reader
	f func()
}

func (wr preventCloser) Close() error {
	wr.f()
	return nil
}

// Save stores data in the backend at the handle.
func (be *Backend) Save(ctx context.Context, h restic.Handle, rd io.Reader) (err error) {
	if err := h.Valid(); err != nil {
		return err
	}

	objName := be.Filename(h)

	debug.Log("Save %v at %v", h, objName)

	// Check key does not already exist
	found, err := be.container.GetBlobReference(objName).Exists()
	if err != nil {
		return errors.Wrap(err, "GetBlobReference().Exists()")
	}
	if found {
		debug.Log("%v already exists", h)
		return errors.New("key already exists")
	}

	be.sem.GetToken()

	// wrap the reader so that net/http client cannot close the reader, return
	// the token instead.
	rd = preventCloser{
		Reader: rd,
		f: func() {
			debug.Log("Close()")
		},
	}

	debug.Log("InsertObject(%v, %v)", be.container.Name, objName)

	err = be.container.GetBlobReference(objName).CreateBlockBlobFromReader(rd, nil)

	be.sem.ReleaseToken()
	debug.Log("%v, err %#v", objName, err)

	return errors.Wrap(err, "CreateBlockBlobFromReader")
}

// wrapReader wraps an io.ReadCloser to run an additional function on Close.
type wrapReader struct {
	io.ReadCloser
	f func()
}

func (wr wrapReader) Close() error {
	err := wr.ReadCloser.Close()
	wr.f()
	return err
}

// Load returns a reader that yields the contents of the file at h at the
// given offset. If length is nonzero, only a portion of the file is
// returned. rd must be closed after use.
func (be *Backend) Load(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) {
	debug.Log("Load %v, length %v, offset %v from %v", h, length, offset, be.Filename(h))
	if err := h.Valid(); err != nil {
		return nil, err
	}

	if offset < 0 {
		return nil, errors.New("offset is negative")
	}

	if length < 0 {
		return nil, errors.Errorf("invalid length %d", length)
	}

	objName := be.Filename(h)
	blob := be.container.GetBlobReference(objName)

	start := uint64(offset)
	var end uint64

	if length > 0 {
		end = uint64(offset + int64(length) - 1)
	} else {
		end = 0
	}

	be.sem.GetToken()

	rd, err := blob.GetRange(&storage.GetBlobRangeOptions{Range: &storage.BlobRange{Start: start, End: end}})
	if err != nil {
		be.sem.ReleaseToken()
		return nil, err
	}

	closeRd := wrapReader{
		ReadCloser: rd,
		f: func() {
			debug.Log("Close()")
			be.sem.ReleaseToken()
		},
	}

	return closeRd, err
}

// Stat returns information about a blob.
func (be *Backend) Stat(ctx context.Context, h restic.Handle) (bi restic.FileInfo, err error) {
	debug.Log("%v", h)

	objName := be.Filename(h)
	blob := be.container.GetBlobReference(objName)

	if err := blob.GetProperties(nil); err != nil {
		debug.Log("blob.GetProperties err %v", err)
		return restic.FileInfo{}, errors.Wrap(err, "blob.GetProperties")
	}

	return restic.FileInfo{Size: int64(blob.Properties.ContentLength)}, nil
}

// Test returns true if a blob of the given type and name exists in the backend.
func (be *Backend) Test(ctx context.Context, h restic.Handle) (bool, error) {
	objName := be.Filename(h)
	found, err := be.container.GetBlobReference(objName).Exists()
	if err != nil {
		return false, err
	}
	return found, nil
}

// Remove removes the blob with the given name and type.
func (be *Backend) Remove(ctx context.Context, h restic.Handle) error {
	objName := be.Filename(h)
	_, err := be.container.GetBlobReference(objName).DeleteIfExists(nil)
	debug.Log("Remove(%v) at %v -> err %v", h, objName, err)
	return errors.Wrap(err, "client.RemoveObject")
}

// List returns a channel that yields all names of blobs of type t. A
// goroutine is started for this. If the channel done is closed, sending
// stops.
func (be *Backend) List(ctx context.Context, t restic.FileType) <-chan string {
	debug.Log("listing %v", t)
	ch := make(chan string)

	prefix := be.Dirname(restic.Handle{Type: t})

	// make sure prefix ends with a slash
	if prefix[len(prefix)-1] != '/' {
		prefix += "/"
	}

	params := storage.ListBlobsParameters{
		MaxResults: uint(be.listMaxItems),
		Prefix:     prefix,
	}

	go func() {
		defer close(ch)

		for {
			obj, err := be.container.ListBlobs(params)
			if err != nil {
				return
			}

			debug.Log("got %v objects", len(obj.Blobs))

			for _, item := range obj.Blobs {
				m := strings.TrimPrefix(item.Name, prefix)
				if m == "" {
					continue
				}

				select {
				case ch <- path.Base(m):
				case <-ctx.Done():
					return
				}
			}

			if obj.NextMarker == "" {
				break
			}
			params.Marker = obj.NextMarker
		}
	}()

	return ch
}

// Remove keys for a specified backend type.
func (be *Backend) removeKeys(ctx context.Context, t restic.FileType) error {
	for key := range be.List(ctx, restic.DataFile) {
		err := be.Remove(ctx, restic.Handle{Type: restic.DataFile, Name: key})
		if err != nil {
			return err
		}
	}

	return nil
}

// Delete removes all restic keys in the bucket. It will not remove the bucket itself.
func (be *Backend) Delete(ctx context.Context) error {
	alltypes := []restic.FileType{
		restic.DataFile,
		restic.KeyFile,
		restic.LockFile,
		restic.SnapshotFile,
		restic.IndexFile}

	for _, t := range alltypes {
		err := be.removeKeys(ctx, t)
		if err != nil {
			return nil
		}
	}

	return be.Remove(ctx, restic.Handle{Type: restic.ConfigFile})
}

// Close does nothing
func (be *Backend) Close() error { return nil }
<commit_msg>Azure backend: limit http concurrency in Stat(), Test(), Remove()<commit_after>package azure

import (
	"context"
	"io"
	"net/http"
	"os"
	"path"
	"strings"

	"github.com/Azure/azure-sdk-for-go/storage"
	"github.com/restic/restic/internal/backend"
	"github.com/restic/restic/internal/debug"
	"github.com/restic/restic/internal/errors"
	"github.com/restic/restic/internal/restic"
)

// Backend stores data on an azure endpoint.
type Backend struct {
	accountName  string
	container    *storage.Container
	sem          *backend.Semaphore
	prefix       string
	listMaxItems int
	backend.Layout
}

const defaultListMaxItems = 5000

// make sure that *Backend implements backend.Backend
var _ restic.Backend = &Backend{}

func open(cfg Config, rt http.RoundTripper) (*Backend, error) {
	debug.Log("open, config %#v", cfg)

	client, err := storage.NewBasicClient(cfg.AccountName, cfg.AccountKey)
	if err != nil {
		return nil, errors.Wrap(err, "NewBasicClient")
	}

	client.HTTPClient = &http.Client{Transport: rt}

	service := client.GetBlobService()

	sem, err := backend.NewSemaphore(cfg.Connections)
	if err != nil {
		return nil, err
	}

	be := &Backend{
		container:   service.GetContainerReference(cfg.Container),
		accountName: cfg.AccountName,
		sem:         sem,
		prefix:      cfg.Prefix,
		Layout: &backend.DefaultLayout{
			Path: cfg.Prefix,
			Join: path.Join,
		},
		listMaxItems: defaultListMaxItems,
	}

	return be, nil
}

// Open opens the Azure backend at specified container.
func Open(cfg Config, rt http.RoundTripper) (restic.Backend, error) {
	return open(cfg, rt)
}

// Create opens the Azure backend at specified container and creates the container if
// it does not exist yet.
func Create(cfg Config, rt http.RoundTripper) (restic.Backend, error) {
	be, err := open(cfg, rt)

	if err != nil {
		return nil, errors.Wrap(err, "open")
	}

	options := storage.CreateContainerOptions{
		Access: storage.ContainerAccessTypePrivate,
	}

	_, err = be.container.CreateIfNotExists(&options)
	if err != nil {
		return nil, errors.Wrap(err, "container.CreateIfNotExists")
	}

	return be, nil
}

// SetListMaxItems sets the number of list items to load per request.
func (be *Backend) SetListMaxItems(i int) {
	be.listMaxItems = i
}

// IsNotExist returns true if the error is caused by a not existing file.
func (be *Backend) IsNotExist(err error) bool {
	debug.Log("IsNotExist(%T, %#v)", err, err)
	return os.IsNotExist(err)
}

// Join combines path components with slashes.
func (be *Backend) Join(p ...string) string {
	return path.Join(p...)
}

// Location returns this backend's location (the container name).
func (be *Backend) Location() string {
	return be.Join(be.container.Name, be.prefix)
}

// Path returns the path in the bucket that is used for this backend.
func (be *Backend) Path() string {
	return be.prefix
}

// preventCloser wraps an io.Reader to run a function instead of the original Close() function.
type preventCloser struct {
	io.Reader
	f func()
}

func (wr preventCloser) Close() error {
	wr.f()
	return nil
}

// Save stores data in the backend at the handle.
func (be *Backend) Save(ctx context.Context, h restic.Handle, rd io.Reader) (err error) {
	if err := h.Valid(); err != nil {
		return err
	}

	objName := be.Filename(h)

	debug.Log("Save %v at %v", h, objName)

	// Check key does not already exist
	found, err := be.container.GetBlobReference(objName).Exists()
	if err != nil {
		return errors.Wrap(err, "GetBlobReference().Exists()")
	}
	if found {
		debug.Log("%v already exists", h)
		return errors.New("key already exists")
	}

	be.sem.GetToken()

	// wrap the reader so that net/http client cannot close the reader, return
	// the token instead.
	rd = preventCloser{
		Reader: rd,
		f: func() {
			debug.Log("Close()")
		},
	}

	debug.Log("InsertObject(%v, %v)", be.container.Name, objName)

	err = be.container.GetBlobReference(objName).CreateBlockBlobFromReader(rd, nil)

	be.sem.ReleaseToken()
	debug.Log("%v, err %#v", objName, err)

	return errors.Wrap(err, "CreateBlockBlobFromReader")
}

// wrapReader wraps an io.ReadCloser to run an additional function on Close.
type wrapReader struct {
	io.ReadCloser
	f func()
}

func (wr wrapReader) Close() error {
	err := wr.ReadCloser.Close()
	wr.f()
	return err
}

// Load returns a reader that yields the contents of the file at h at the
// given offset. If length is nonzero, only a portion of the file is
// returned. rd must be closed after use.
func (be *Backend) Load(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) {
	debug.Log("Load %v, length %v, offset %v from %v", h, length, offset, be.Filename(h))
	if err := h.Valid(); err != nil {
		return nil, err
	}

	if offset < 0 {
		return nil, errors.New("offset is negative")
	}

	if length < 0 {
		return nil, errors.Errorf("invalid length %d", length)
	}

	objName := be.Filename(h)
	blob := be.container.GetBlobReference(objName)

	start := uint64(offset)
	var end uint64

	if length > 0 {
		end = uint64(offset + int64(length) - 1)
	} else {
		end = 0
	}

	be.sem.GetToken()

	rd, err := blob.GetRange(&storage.GetBlobRangeOptions{Range: &storage.BlobRange{Start: start, End: end}})
	if err != nil {
		be.sem.ReleaseToken()
		return nil, err
	}

	closeRd := wrapReader{
		ReadCloser: rd,
		f: func() {
			debug.Log("Close()")
			be.sem.ReleaseToken()
		},
	}

	return closeRd, err
}

// Stat returns information about a blob.
func (be *Backend) Stat(ctx context.Context, h restic.Handle) (restic.FileInfo, error) {
	debug.Log("%v", h)

	objName := be.Filename(h)
	blob := be.container.GetBlobReference(objName)

	be.sem.GetToken()
	err := blob.GetProperties(nil)
	be.sem.ReleaseToken()

	if err != nil {
		debug.Log("blob.GetProperties err %v", err)
		return restic.FileInfo{}, errors.Wrap(err, "blob.GetProperties")
	}

	return restic.FileInfo{Size: int64(blob.Properties.ContentLength)}, nil
}

// Test returns true if a blob of the given type and name exists in the backend.
func (be *Backend) Test(ctx context.Context, h restic.Handle) (bool, error) {
	objName := be.Filename(h)

	be.sem.GetToken()
	found, err := be.container.GetBlobReference(objName).Exists()
	be.sem.ReleaseToken()

	if err != nil {
		return false, err
	}
	return found, nil
}

// Remove removes the blob with the given name and type.
func (be *Backend) Remove(ctx context.Context, h restic.Handle) error {
	objName := be.Filename(h)

	be.sem.GetToken()
	_, err := be.container.GetBlobReference(objName).DeleteIfExists(nil)
	be.sem.ReleaseToken()

	debug.Log("Remove(%v) at %v -> err %v", h, objName, err)
	return errors.Wrap(err, "client.RemoveObject")
}

// List returns a channel that yields all names of blobs of type t. A
// goroutine is started for this. If the channel done is closed, sending
// stops.
func (be *Backend) List(ctx context.Context, t restic.FileType) <-chan string {
	debug.Log("listing %v", t)
	ch := make(chan string)

	prefix := be.Dirname(restic.Handle{Type: t})

	// make sure prefix ends with a slash
	if prefix[len(prefix)-1] != '/' {
		prefix += "/"
	}

	params := storage.ListBlobsParameters{
		MaxResults: uint(be.listMaxItems),
		Prefix:     prefix,
	}

	go func() {
		defer close(ch)

		for {
			be.sem.GetToken()
			obj, err := be.container.ListBlobs(params)
			be.sem.ReleaseToken()

			if err != nil {
				return
			}

			debug.Log("got %v objects", len(obj.Blobs))

			for _, item := range obj.Blobs {
				m := strings.TrimPrefix(item.Name, prefix)
				if m == "" {
					continue
				}

				select {
				case ch <- path.Base(m):
				case <-ctx.Done():
					return
				}
			}

			if obj.NextMarker == "" {
				break
			}
			params.Marker = obj.NextMarker
		}
	}()

	return ch
}

// Remove keys for a specified backend type.
func (be *Backend) removeKeys(ctx context.Context, t restic.FileType) error {
	for key := range be.List(ctx, restic.DataFile) {
		err := be.Remove(ctx, restic.Handle{Type: restic.DataFile, Name: key})
		if err != nil {
			return err
		}
	}

	return nil
}

// Delete removes all restic keys in the bucket. It will not remove the bucket itself.
func (be *Backend) Delete(ctx context.Context) error {
	alltypes := []restic.FileType{
		restic.DataFile,
		restic.KeyFile,
		restic.LockFile,
		restic.SnapshotFile,
		restic.IndexFile}

	for _, t := range alltypes {
		err := be.removeKeys(ctx, t)
		if err != nil {
			return nil
		}
	}

	return be.Remove(ctx, restic.Handle{Type: restic.ConfigFile})
}

// Close does nothing
func (be *Backend) Close() error { return nil }
<|endoftext|>


<commit_before>package local

import (
	"context"
	"io"
	"os"
	"path/filepath"
	"syscall"

	"github.com/restic/restic/internal/errors"
	"github.com/restic/restic/internal/restic"

	"github.com/restic/restic/internal/backend"
	"github.com/restic/restic/internal/debug"
	"github.com/restic/restic/internal/fs"
)

// Local is a backend in a local directory.
type Local struct {
	Config
	backend.Layout
}

// ensure statically that *Local implements restic.Backend.
var _ restic.Backend = &Local{}

const defaultLayout = "default"

// Open opens the local backend as specified by config.
func Open(cfg Config) (*Local, error) {
	debug.Log("open local backend at %v (layout %q)", cfg.Path, cfg.Layout)
	l, err := backend.ParseLayout(&backend.LocalFilesystem{}, cfg.Layout, defaultLayout, cfg.Path)
	if err != nil {
		return nil, err
	}

	return &Local{Config: cfg, Layout: l}, nil
}

// Create creates all the necessary files and directories for a new local
// backend at dir. Afterwards a new config blob should be created.
func Create(cfg Config) (*Local, error) {
	debug.Log("create local backend at %v (layout %q)", cfg.Path, cfg.Layout)

	l, err := backend.ParseLayout(&backend.LocalFilesystem{}, cfg.Layout, defaultLayout, cfg.Path)
	if err != nil {
		return nil, err
	}

	be := &Local{
		Config: cfg,
		Layout: l,
	}

	// test if config file already exists
	_, err = fs.Lstat(be.Filename(restic.Handle{Type: restic.ConfigFile}))
	if err == nil {
		return nil, errors.New("config file already exists")
	}

	// create paths for data and refs
	for _, d := range be.Paths() {
		err := fs.MkdirAll(d, backend.Modes.Dir)
		if err != nil {
			return nil, errors.Wrap(err, "MkdirAll")
		}
	}

	return be, nil
}

// Location returns this backend's location (the directory name).
func (b *Local) Location() string {
	return b.Path
}

// IsNotExist returns true if the error is caused by a non existing file.
func (b *Local) IsNotExist(err error) bool {
	return os.IsNotExist(errors.Cause(err))
}

// Save stores data in the backend at the handle.
func (b *Local) Save(ctx context.Context, h restic.Handle, rd restic.RewindReader) error {
	debug.Log("Save %v", h)
	if err := h.Valid(); err != nil {
		return err
	}

	filename := b.Filename(h)

	// create new file
	f, err := fs.OpenFile(filename, os.O_CREATE|os.O_EXCL|os.O_WRONLY, backend.Modes.File)

	if b.IsNotExist(err) {
		debug.Log("error %v: creating dir", err)

		// error is caused by a missing directory, try to create it
		mkdirErr := os.MkdirAll(filepath.Dir(filename), backend.Modes.Dir)
		if mkdirErr != nil {
			debug.Log("error creating dir %v: %v", filepath.Dir(filename), mkdirErr)
		} else {
			// try again
			f, err = fs.OpenFile(filename, os.O_CREATE|os.O_EXCL|os.O_WRONLY, backend.Modes.File)
		}
	}

	if err != nil {
		return errors.Wrap(err, "OpenFile")
	}

	// save data, then sync
	_, err = io.Copy(f, rd)
	if err != nil {
		_ = f.Close()
		return errors.Wrap(err, "Write")
	}

	if err = f.Sync(); err != nil {
		pathErr, ok := err.(*os.PathError)
		isNotSupported := ok && pathErr.Op == "sync" && pathErr.Err == syscall.ENOTSUP
		// ignore error if filesystem does not support the sync operation
		if !isNotSupported {
			_ = f.Close()
			return errors.Wrap(err, "Sync")
		}
	}

	err = f.Close()
	if err != nil {
		return errors.Wrap(err, "Close")
	}

	return setNewFileMode(filename, backend.Modes.File)
}

// Load runs fn with a reader that yields the contents of the file at h at the
// given offset.
func (b *Local) Load(ctx context.Context, h restic.Handle, length int, offset int64, fn func(rd io.Reader) error) error {
	return backend.DefaultLoad(ctx, h, length, offset, b.openReader, fn)
}

func (b *Local) openReader(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) {
	debug.Log("Load %v, length %v, offset %v", h, length, offset)
	if err := h.Valid(); err != nil {
		return nil, err
	}

	if offset < 0 {
		return nil, errors.New("offset is negative")
	}

	f, err := fs.Open(b.Filename(h))
	if err != nil {
		return nil, err
	}

	if offset > 0 {
		_, err = f.Seek(offset, 0)
		if err != nil {
			_ = f.Close()
			return nil, err
		}
	}

	if length > 0 {
		return backend.LimitReadCloser(f, int64(length)), nil
	}

	return f, nil
}

// Stat returns information about a blob.
func (b *Local) Stat(ctx context.Context, h restic.Handle) (restic.FileInfo, error) {
	debug.Log("Stat %v", h)
	if err := h.Valid(); err != nil {
		return restic.FileInfo{}, err
	}

	fi, err := fs.Stat(b.Filename(h))
	if err != nil {
		return restic.FileInfo{}, errors.Wrap(err, "Stat")
	}

	return restic.FileInfo{Size: fi.Size(), Name: h.Name}, nil
}

// Test returns true if a blob of the given type and name exists in the backend.
func (b *Local) Test(ctx context.Context, h restic.Handle) (bool, error) {
	debug.Log("Test %v", h)
	_, err := fs.Stat(b.Filename(h))
	if err != nil {
		if os.IsNotExist(errors.Cause(err)) {
			return false, nil
		}
		return false, errors.Wrap(err, "Stat")
	}

	return true, nil
}

// Remove removes the blob with the given name and type.
func (b *Local) Remove(ctx context.Context, h restic.Handle) error {
	debug.Log("Remove %v", h)
	fn := b.Filename(h)

	// reset read-only flag
	err := fs.Chmod(fn, 0666)
	if err != nil {
		return errors.Wrap(err, "Chmod")
	}

	return fs.Remove(fn)
}

func isFile(fi os.FileInfo) bool {
	return fi.Mode()&(os.ModeType|os.ModeCharDevice) == 0
}

// List runs fn for each file in the backend which has the type t. When an
// error occurs (or fn returns an error), List stops and returns it.
func (b *Local) List(ctx context.Context, t restic.FileType, fn func(restic.FileInfo) error) error {
	debug.Log("List %v", t)

	basedir, subdirs := b.Basedir(t)
	err := fs.Walk(basedir, func(path string, fi os.FileInfo, err error) error {
		debug.Log("walk on %v\n", path)
		if err != nil {
			return err
		}

		if path == basedir {
			return nil
		}

		if !isFile(fi) {
			return nil
		}

		if fi.IsDir() && !subdirs {
			return filepath.SkipDir
		}

		debug.Log("send %v\n", filepath.Base(path))

		rfi := restic.FileInfo{
			Name: filepath.Base(path),
			Size: fi.Size(),
		}

		if ctx.Err() != nil {
			return ctx.Err()
		}

		err = fn(rfi)
		if err != nil {
			return err
		}

		return ctx.Err()
	})

	if b.IsNotExist(err) {
		debug.Log("ignoring non-existing directory")
		return nil
	}

	return err
}

// Delete removes the repository and all files.
func (b *Local) Delete(ctx context.Context) error {
	debug.Log("Delete()")
	return fs.RemoveAll(b.Path)
}

// Close closes all open files.
func (b *Local) Close() error {
	debug.Log("Close()")
	// this does not need to do anything, all open files are closed within the
	// same function.
	return nil
}
<commit_msg>local: Ignore permission errors on chmod call in Save/Remove operation<commit_after>package local

import (
	"context"
	"io"
	"os"
	"path/filepath"
	"syscall"

	"github.com/restic/restic/internal/errors"
	"github.com/restic/restic/internal/restic"

	"github.com/restic/restic/internal/backend"
	"github.com/restic/restic/internal/debug"
	"github.com/restic/restic/internal/fs"
)

// Local is a backend in a local directory.
type Local struct {
	Config
	backend.Layout
}

// ensure statically that *Local implements restic.Backend.
var _ restic.Backend = &Local{}

const defaultLayout = "default"

// Open opens the local backend as specified by config.
func Open(cfg Config) (*Local, error) {
	debug.Log("open local backend at %v (layout %q)", cfg.Path, cfg.Layout)
	l, err := backend.ParseLayout(&backend.LocalFilesystem{}, cfg.Layout, defaultLayout, cfg.Path)
	if err != nil {
		return nil, err
	}

	return &Local{Config: cfg, Layout: l}, nil
}

// Create creates all the necessary files and directories for a new local
// backend at dir. Afterwards a new config blob should be created.
func Create(cfg Config) (*Local, error) {
	debug.Log("create local backend at %v (layout %q)", cfg.Path, cfg.Layout)

	l, err := backend.ParseLayout(&backend.LocalFilesystem{}, cfg.Layout, defaultLayout, cfg.Path)
	if err != nil {
		return nil, err
	}

	be := &Local{
		Config: cfg,
		Layout: l,
	}

	// test if config file already exists
	_, err = fs.Lstat(be.Filename(restic.Handle{Type: restic.ConfigFile}))
	if err == nil {
		return nil, errors.New("config file already exists")
	}

	// create paths for data and refs
	for _, d := range be.Paths() {
		err := fs.MkdirAll(d, backend.Modes.Dir)
		if err != nil {
			return nil, errors.Wrap(err, "MkdirAll")
		}
	}

	return be, nil
}

// Location returns this backend's location (the directory name).
func (b *Local) Location() string {
	return b.Path
}

// IsNotExist returns true if the error is caused by a non existing file.
func (b *Local) IsNotExist(err error) bool {
	return os.IsNotExist(errors.Cause(err))
}

// Save stores data in the backend at the handle.
func (b *Local) Save(ctx context.Context, h restic.Handle, rd restic.RewindReader) error {
	debug.Log("Save %v", h)
	if err := h.Valid(); err != nil {
		return err
	}

	filename := b.Filename(h)

	// create new file
	f, err := fs.OpenFile(filename, os.O_CREATE|os.O_EXCL|os.O_WRONLY, backend.Modes.File)

	if b.IsNotExist(err) {
		debug.Log("error %v: creating dir", err)

		// error is caused by a missing directory, try to create it
		mkdirErr := os.MkdirAll(filepath.Dir(filename), backend.Modes.Dir)
		if mkdirErr != nil {
			debug.Log("error creating dir %v: %v", filepath.Dir(filename), mkdirErr)
		} else {
			// try again
			f, err = fs.OpenFile(filename, os.O_CREATE|os.O_EXCL|os.O_WRONLY, backend.Modes.File)
		}
	}

	if err != nil {
		return errors.Wrap(err, "OpenFile")
	}

	// save data, then sync
	_, err = io.Copy(f, rd)
	if err != nil {
		_ = f.Close()
		return errors.Wrap(err, "Write")
	}

	if err = f.Sync(); err != nil {
		pathErr, ok := err.(*os.PathError)
		isNotSupported := ok && pathErr.Op == "sync" && pathErr.Err == syscall.ENOTSUP
		// ignore error if filesystem does not support the sync operation
		if !isNotSupported {
			_ = f.Close()
			return errors.Wrap(err, "Sync")
		}
	}

	err = f.Close()
	if err != nil {
		return errors.Wrap(err, "Close")
	}

	// ignore if the operation fails as some filesystems don't allow the chmod call
	// e.g. exfat and network file systems with certain mount options
	err = setNewFileMode(filename, backend.Modes.File)
	if err != nil && !os.IsPermission(err) {
		return errors.Wrap(err, "Chmod")
	}

	return nil
}

// Load runs fn with a reader that yields the contents of the file at h at the
// given offset.
func (b *Local) Load(ctx context.Context, h restic.Handle, length int, offset int64, fn func(rd io.Reader) error) error {
	return backend.DefaultLoad(ctx, h, length, offset, b.openReader, fn)
}

func (b *Local) openReader(ctx context.Context, h restic.Handle, length int, offset int64) (io.ReadCloser, error) {
	debug.Log("Load %v, length %v, offset %v", h, length, offset)
	if err := h.Valid(); err != nil {
		return nil, err
	}

	if offset < 0 {
		return nil, errors.New("offset is negative")
	}

	f, err := fs.Open(b.Filename(h))
	if err != nil {
		return nil, err
	}

	if offset > 0 {
		_, err = f.Seek(offset, 0)
		if err != nil {
			_ = f.Close()
			return nil, err
		}
	}

	if length > 0 {
		return backend.LimitReadCloser(f, int64(length)), nil
	}

	return f, nil
}

// Stat returns information about a blob.
func (b *Local) Stat(ctx context.Context, h restic.Handle) (restic.FileInfo, error) {
	debug.Log("Stat %v", h)
	if err := h.Valid(); err != nil {
		return restic.FileInfo{}, err
	}

	fi, err := fs.Stat(b.Filename(h))
	if err != nil {
		return restic.FileInfo{}, errors.Wrap(err, "Stat")
	}

	return restic.FileInfo{Size: fi.Size(), Name: h.Name}, nil
}

// Test returns true if a blob of the given type and name exists in the backend.
func (b *Local) Test(ctx context.Context, h restic.Handle) (bool, error) {
	debug.Log("Test %v", h)
	_, err := fs.Stat(b.Filename(h))
	if err != nil {
		if os.IsNotExist(errors.Cause(err)) {
			return false, nil
		}
		return false, errors.Wrap(err, "Stat")
	}

	return true, nil
}

// Remove removes the blob with the given name and type.
func (b *Local) Remove(ctx context.Context, h restic.Handle) error {
	debug.Log("Remove %v", h)
	fn := b.Filename(h)

	// reset read-only flag
	err := fs.Chmod(fn, 0666)
	if err != nil && !os.IsPermission(err) {
		return errors.Wrap(err, "Chmod")
	}

	return fs.Remove(fn)
}

func isFile(fi os.FileInfo) bool {
	return fi.Mode()&(os.ModeType|os.ModeCharDevice) == 0
}

// List runs fn for each file in the backend which has the type t. When an
// error occurs (or fn returns an error), List stops and returns it.
func (b *Local) List(ctx context.Context, t restic.FileType, fn func(restic.FileInfo) error) error {
	debug.Log("List %v", t)

	basedir, subdirs := b.Basedir(t)
	err := fs.Walk(basedir, func(path string, fi os.FileInfo, err error) error {
		debug.Log("walk on %v\n", path)
		if err != nil {
			return err
		}

		if path == basedir {
			return nil
		}

		if !isFile(fi) {
			return nil
		}

		if fi.IsDir() && !subdirs {
			return filepath.SkipDir
		}

		debug.Log("send %v\n", filepath.Base(path))

		rfi := restic.FileInfo{
			Name: filepath.Base(path),
			Size: fi.Size(),
		}

		if ctx.Err() != nil {
			return ctx.Err()
		}

		err = fn(rfi)
		if err != nil {
			return err
		}

		return ctx.Err()
	})

	if b.IsNotExist(err) {
		debug.Log("ignoring non-existing directory")
		return nil
	}

	return err
}

// Delete removes the repository and all files.
func (b *Local) Delete(ctx context.Context) error {
	debug.Log("Delete()")
	return fs.RemoveAll(b.Path)
}

// Close closes all open files.
func (b *Local) Close() error {
	debug.Log("Close()")
	// this does not need to do anything, all open files are closed within the
	// same function.
	return nil
}
<|endoftext|>