<commit_before>package promhttputil

import (
	"fmt"
	"reflect"

	"github.com/prometheus/common/model"
)

func ValueAddLabelSet(a model.Value, l model.LabelSet) error {
	switch aTyped := a.(type) {
	case model.Vector:
		for _, item := range aTyped {
			for k, v := range l {
				item.Metric[k] = v
			}
		}

	case model.Matrix:
		for _, item := range aTyped {
			for k, v := range l {
				item.Metric[k] = v
			}
		}
	}

	return nil

}

// TODO: always make copies? Now we sometimes return one, or make a copy, or do nothing
// Merge 2 values and
func MergeValues(a, b model.Value) (model.Value, error) {
	if a.Type() != b.Type() {
		return nil, fmt.Errorf("Error!")
	}

	switch aTyped := a.(type) {
	// TODO: more logic? for now we assume both are correct if they exist
	// In the case where it is a single datapoint, we're going to assume that
	// either is valid, we just need one
	case *model.Scalar:
		bTyped := b.(*model.Scalar)

		if aTyped.Value != 0 && aTyped.Timestamp != 0 {
			return aTyped, nil
		} else {
			return bTyped, nil
		}

	// In the case where it is a single datapoint, we're going to assume that
	// either is valid, we just need one
	case *model.String:
		bTyped := b.(*model.String)

		if aTyped.Value != "" && aTyped.Timestamp != 0 {
			return aTyped, nil
		} else {
			return bTyped, nil
		}

	// List of *model.Sample -- only 1 value (guaranteed same timestamp)
	case model.Vector:
		bTyped := b.(model.Vector)

		newValue := make(model.Vector, 0, len(aTyped)+len(bTyped))
		fingerPrintMap := make(map[model.Fingerprint]int)

		addItem := func(item *model.Sample) {
			finger := item.Metric.Fingerprint()

			// If we've seen this fingerPrint before, lets make sure that a value exists
			if index, ok := fingerPrintMap[finger]; ok {
				// TODO: better? For now we only replace if we have no value (which seems reasonable)
				if newValue[index].Value == model.SampleValue(0) {
					newValue[index].Value = item.Value
				}
			} else {
				newValue = append(newValue, item)
				fingerPrintMap[finger] = len(newValue) - 1
			}
		}

		for _, item := range aTyped {
			addItem(item)
		}

		for _, item := range bTyped {
			addItem(item)
		}
		return newValue, nil

	case model.Matrix:
		bTyped := b.(model.Matrix)

		newValue := make(model.Matrix, 0, len(aTyped)+len(bTyped))
		fingerPrintMap := make(map[model.Fingerprint]int)

		addStream := func(stream *model.SampleStream) {
			finger := stream.Metric.Fingerprint()

			// If we've seen this fingerPrint before, lets make sure that a value exists
			if index, ok := fingerPrintMap[finger]; ok {
				// TODO: check this error? For now the only one is sig collision, which we check
				newValue[index], _ = MergeSampleStream(newValue[index], stream)
			} else {
				newValue = append(newValue, stream)
				fingerPrintMap[finger] = len(newValue) - 1
			}
		}

		for _, item := range aTyped {
			addStream(item)
		}

		for _, item := range bTyped {
			addStream(item)
		}
		return newValue, nil
	}

	return nil, fmt.Errorf("Unknown type! %v", reflect.TypeOf(a))
}

func MergeSampleStream(a, b *model.SampleStream) (*model.SampleStream, error) {
	if a.Metric.Fingerprint() != b.Metric.Fingerprint() {
		return nil, fmt.Errorf("Cannot merge mismatch fingerprints")
	}

	// TODO: really there should be a library method for this in prometheus IMO
	// At this point we have 2 sorted lists of datapoints which we need to merge
	seenTimes := make(map[model.Time]struct{})
	newValues := make([]model.SamplePair, 0, len(a.Values)+len(b.Values))

	ai := 0 // Offset in a
	bi := 0 // Offset in b

	// When combining series from 2 different prometheus hosts we can run into some problems
	// with clock skew (from a variety of sources). The primary one I've run into is issues
	// with the time that prometheus stores. Since the time associated with the datapoint is
	// the *start* time of the scrape, there can be quite a lot of time (which can vary
	// dramatically between hosts) for the exporter to return. In an attempt to mitigate
	// this problem we're going to *not* merge any datapoint within 5s of another point
	// we have. This means we can tolerate 2.5s on either side (which can be used by either
	// clock skew or from this scrape skew).

	// TODO: config
	antiAffinityBuffer := model.TimeFromUnix(5) // 5s
	var lastTime model.Time

	for {
		if ai >= len(a.Values) && bi >= len(b.Values) {
			break
		}

		var item model.SamplePair

		if ai < len(a.Values) { // If a exists
			if bi < len(b.Values) {
				// both items
				if a.Values[ai].Timestamp < b.Values[bi].Timestamp {
					item = a.Values[ai]
					ai++
				} else {
					item = b.Values[bi]
					bi++
				}
			} else {
				// Only A
				item = a.Values[ai]
				ai++
			}
		} else {
			if bi < len(b.Values) {
				// Only B
				item = b.Values[bi]
				bi++
			}
		}
		// If we've already seen this timestamp, skip
		if _, ok := seenTimes[item.Timestamp]; ok {
			continue
		}

		if lastTime == 0 {
			lastTime = item.Timestamp
		}

		if item.Timestamp-lastTime < antiAffinityBuffer {
			continue
		}
		lastTime = item.Timestamp

		// Otherwise, lets add it
		newValues = append(newValues, item)
		seenTimes[item.Timestamp] = struct{}{}
	}

	return &model.SampleStream{
		Metric: a.Metric,
		Values: newValues,
	}, nil
}
<commit_msg>Up anti-affinity to 10s<commit_after>package promhttputil

import (
	"fmt"
	"reflect"

	"github.com/prometheus/common/model"
)

func ValueAddLabelSet(a model.Value, l model.LabelSet) error {
	switch aTyped := a.(type) {
	case model.Vector:
		for _, item := range aTyped {
			for k, v := range l {
				item.Metric[k] = v
			}
		}

	case model.Matrix:
		for _, item := range aTyped {
			for k, v := range l {
				item.Metric[k] = v
			}
		}
	}

	return nil

}

// TODO: always make copies? Now we sometimes return one, or make a copy, or do nothing
// Merge 2 values and
func MergeValues(a, b model.Value) (model.Value, error) {
	if a.Type() != b.Type() {
		return nil, fmt.Errorf("Error!")
	}

	switch aTyped := a.(type) {
	// TODO: more logic? for now we assume both are correct if they exist
	// In the case where it is a single datapoint, we're going to assume that
	// either is valid, we just need one
	case *model.Scalar:
		bTyped := b.(*model.Scalar)

		if aTyped.Value != 0 && aTyped.Timestamp != 0 {
			return aTyped, nil
		} else {
			return bTyped, nil
		}

	// In the case where it is a single datapoint, we're going to assume that
	// either is valid, we just need one
	case *model.String:
		bTyped := b.(*model.String)

		if aTyped.Value != "" && aTyped.Timestamp != 0 {
			return aTyped, nil
		} else {
			return bTyped, nil
		}

	// List of *model.Sample -- only 1 value (guaranteed same timestamp)
	case model.Vector:
		bTyped := b.(model.Vector)

		newValue := make(model.Vector, 0, len(aTyped)+len(bTyped))
		fingerPrintMap := make(map[model.Fingerprint]int)

		addItem := func(item *model.Sample) {
			finger := item.Metric.Fingerprint()

			// If we've seen this fingerPrint before, lets make sure that a value exists
			if index, ok := fingerPrintMap[finger]; ok {
				// TODO: better? For now we only replace if we have no value (which seems reasonable)
				if newValue[index].Value == model.SampleValue(0) {
					newValue[index].Value = item.Value
				}
			} else {
				newValue = append(newValue, item)
				fingerPrintMap[finger] = len(newValue) - 1
			}
		}

		for _, item := range aTyped {
			addItem(item)
		}

		for _, item := range bTyped {
			addItem(item)
		}
		return newValue, nil

	case model.Matrix:
		bTyped := b.(model.Matrix)

		newValue := make(model.Matrix, 0, len(aTyped)+len(bTyped))
		fingerPrintMap := make(map[model.Fingerprint]int)

		addStream := func(stream *model.SampleStream) {
			finger := stream.Metric.Fingerprint()

			// If we've seen this fingerPrint before, lets make sure that a value exists
			if index, ok := fingerPrintMap[finger]; ok {
				// TODO: check this error? For now the only one is sig collision, which we check
				newValue[index], _ = MergeSampleStream(newValue[index], stream)
			} else {
				newValue = append(newValue, stream)
				fingerPrintMap[finger] = len(newValue) - 1
			}
		}

		for _, item := range aTyped {
			addStream(item)
		}

		for _, item := range bTyped {
			addStream(item)
		}
		return newValue, nil
	}

	return nil, fmt.Errorf("Unknown type! %v", reflect.TypeOf(a))
}

func MergeSampleStream(a, b *model.SampleStream) (*model.SampleStream, error) {
	if a.Metric.Fingerprint() != b.Metric.Fingerprint() {
		return nil, fmt.Errorf("Cannot merge mismatch fingerprints")
	}

	// TODO: really there should be a library method for this in prometheus IMO
	// At this point we have 2 sorted lists of datapoints which we need to merge
	seenTimes := make(map[model.Time]struct{})
	newValues := make([]model.SamplePair, 0, len(a.Values)+len(b.Values))

	ai := 0 // Offset in a
	bi := 0 // Offset in b

	// When combining series from 2 different prometheus hosts we can run into some problems
	// with clock skew (from a variety of sources). The primary one I've run into is issues
	// with the time that prometheus stores. Since the time associated with the datapoint is
	// the *start* time of the scrape, there can be quite a lot of time (which can vary
	// dramatically between hosts) for the exporter to return. In an attempt to mitigate
	// this problem we're going to *not* merge any datapoint within 10s of another point
	// we have. This means we can tolerate 5s on either side (which can be used by either
	// clock skew or from this scrape skew).

	// TODO: config
	antiAffinityBuffer := model.TimeFromUnix(10) // 10s
	var lastTime model.Time

	for {
		if ai >= len(a.Values) && bi >= len(b.Values) {
			break
		}

		var item model.SamplePair

		if ai < len(a.Values) { // If a exists
			if bi < len(b.Values) {
				// both items
				if a.Values[ai].Timestamp < b.Values[bi].Timestamp {
					item = a.Values[ai]
					ai++
				} else {
					item = b.Values[bi]
					bi++
				}
			} else {
				// Only A
				item = a.Values[ai]
				ai++
			}
		} else {
			if bi < len(b.Values) {
				// Only B
				item = b.Values[bi]
				bi++
			}
		}
		// If we've already seen this timestamp, skip
		if _, ok := seenTimes[item.Timestamp]; ok {
			continue
		}

		if lastTime == 0 {
			lastTime = item.Timestamp
		}

		if item.Timestamp-lastTime < antiAffinityBuffer {
			continue
		}
		lastTime = item.Timestamp

		// Otherwise, lets add it
		newValues = append(newValues, item)
		seenTimes[item.Timestamp] = struct{}{}
	}

	return &model.SampleStream{
		Metric: a.Metric,
		Values: newValues,
	}, nil
}
<|endoftext|>


<commit_before>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package http

import (
	"bytes"
	"fmt"
	"io"
	"os"
	"sort"
	"strconv"
	"strings"
	"time"
)

// This implementation is done according to IETF draft-ietf-httpstate-cookie-23, found at
//
//    http://tools.ietf.org/html/draft-ietf-httpstate-cookie-23

// A Cookie represents an HTTP cookie as sent in the Set-Cookie header of an
// HTTP response or the Cookie header of an HTTP request.
type Cookie struct {
	Name       string
	Value      string
	Path       string
	Domain     string
	Expires    time.Time
	RawExpires string

	// MaxAge=0 means no 'Max-Age' attribute specified.
	// MaxAge<0 means delete cookie now, equivalently 'Max-Age: 0'
	// MaxAge>0 means Max-Age attribute present and given in seconds
	MaxAge   int
	Secure   bool
	HttpOnly bool
	Raw      string
	Unparsed []string // Raw text of unparsed attribute-value pairs
}

// readSetCookies parses all "Set-Cookie" values from
// the header h, removes the successfully parsed values from the
// "Set-Cookie" key in h and returns the parsed Cookies.
func readSetCookies(h Header) []*Cookie {
	cookies := []*Cookie{}
	var unparsedLines []string
	for _, line := range h["Set-Cookie"] {
		parts := strings.Split(strings.TrimSpace(line), ";", -1)
		if len(parts) == 1 && parts[0] == "" {
			continue
		}
		parts[0] = strings.TrimSpace(parts[0])
		j := strings.Index(parts[0], "=")
		if j < 0 {
			unparsedLines = append(unparsedLines, line)
			continue
		}
		name, value := parts[0][:j], parts[0][j+1:]
		if !isCookieNameValid(name) {
			unparsedLines = append(unparsedLines, line)
			continue
		}
		value, success := parseCookieValue(value)
		if !success {
			unparsedLines = append(unparsedLines, line)
			continue
		}
		c := &Cookie{
			Name:  name,
			Value: value,
			Raw:   line,
		}
		for i := 1; i < len(parts); i++ {
			parts[i] = strings.TrimSpace(parts[i])
			if len(parts[i]) == 0 {
				continue
			}

			attr, val := parts[i], ""
			if j := strings.Index(attr, "="); j >= 0 {
				attr, val = attr[:j], attr[j+1:]
			}
			val, success = parseCookieValue(val)
			if !success {
				c.Unparsed = append(c.Unparsed, parts[i])
				continue
			}
			switch strings.ToLower(attr) {
			case "secure":
				c.Secure = true
				continue
			case "httponly":
				c.HttpOnly = true
				continue
			case "domain":
				c.Domain = val
				// TODO: Add domain parsing
				continue
			case "max-age":
				secs, err := strconv.Atoi(val)
				if err != nil || secs < 0 || secs != 0 && val[0] == '0' {
					break
				}
				if secs <= 0 {
					c.MaxAge = -1
				} else {
					c.MaxAge = secs
				}
				continue
			case "expires":
				c.RawExpires = val
				exptime, err := time.Parse(time.RFC1123, val)
				if err != nil {
					c.Expires = time.Time{}
					break
				}
				c.Expires = *exptime
				continue
			case "path":
				c.Path = val
				// TODO: Add path parsing
				continue
			}
			c.Unparsed = append(c.Unparsed, parts[i])
		}
		cookies = append(cookies, c)
	}
	h["Set-Cookie"] = unparsedLines, unparsedLines != nil
	return cookies
}

// writeSetCookies writes the wire representation of the set-cookies
// to w. Each cookie is written on a separate "Set-Cookie: " line.
// This choice is made because HTTP parsers tend to have a limit on
// line-length, so it seems safer to place cookies on separate lines.
func writeSetCookies(w io.Writer, kk []*Cookie) os.Error {
	if kk == nil {
		return nil
	}
	lines := make([]string, 0, len(kk))
	var b bytes.Buffer
	for _, c := range kk {
		b.Reset()
		fmt.Fprintf(&b, "%s=%s", sanitizeName(c.Name), sanitizeValue(c.Value))
		if len(c.Path) > 0 {
			fmt.Fprintf(&b, "; Path=%s", sanitizeValue(c.Path))
		}
		if len(c.Domain) > 0 {
			fmt.Fprintf(&b, "; Domain=%s", sanitizeValue(c.Domain))
		}
		if len(c.Expires.Zone) > 0 {
			fmt.Fprintf(&b, "; Expires=%s", c.Expires.Format(time.RFC1123))
		}
		if c.MaxAge > 0 {
			fmt.Fprintf(&b, "; Max-Age=%d", c.MaxAge)
		} else if c.MaxAge < 0 {
			fmt.Fprintf(&b, "; Max-Age=0")
		}
		if c.HttpOnly {
			fmt.Fprintf(&b, "; HttpOnly")
		}
		if c.Secure {
			fmt.Fprintf(&b, "; Secure")
		}
		lines = append(lines, "Set-Cookie: "+b.String()+"\r\n")
	}
	sort.SortStrings(lines)
	for _, l := range lines {
		if _, err := io.WriteString(w, l); err != nil {
			return err
		}
	}
	return nil
}

// readCookies parses all "Cookie" values from
// the header h, removes the successfully parsed values from the
// "Cookie" key in h and returns the parsed Cookies.
func readCookies(h Header) []*Cookie {
	cookies := []*Cookie{}
	lines, ok := h["Cookie"]
	if !ok {
		return cookies
	}
	unparsedLines := []string{}
	for _, line := range lines {
		parts := strings.Split(strings.TrimSpace(line), ";", -1)
		if len(parts) == 1 && parts[0] == "" {
			continue
		}
		// Per-line attributes
		parsedPairs := 0
		for i := 0; i < len(parts); i++ {
			parts[i] = strings.TrimSpace(parts[i])
			if len(parts[i]) == 0 {
				continue
			}
			attr, val := parts[i], ""
			if j := strings.Index(attr, "="); j >= 0 {
				attr, val = attr[:j], attr[j+1:]
			}
			if !isCookieNameValid(attr) {
				continue
			}
			val, success := parseCookieValue(val)
			if !success {
				continue
			}
			cookies = append(cookies, &Cookie{Name: attr, Value: val})
			parsedPairs++
		}
		if parsedPairs == 0 {
			unparsedLines = append(unparsedLines, line)
		}
	}
	h["Cookie"] = unparsedLines, len(unparsedLines) > 0
	return cookies
}

// writeCookies writes the wire representation of the cookies
// to w. Each cookie is written on a separate "Cookie: " line.
// This choice is made because HTTP parsers tend to have a limit on
// line-length, so it seems safer to place cookies on separate lines.
func writeCookies(w io.Writer, kk []*Cookie) os.Error {
	lines := make([]string, 0, len(kk))
	for _, c := range kk {
		lines = append(lines, fmt.Sprintf("Cookie: %s=%s\r\n", sanitizeName(c.Name), sanitizeValue(c.Value)))
	}
	sort.SortStrings(lines)
	for _, l := range lines {
		if _, err := io.WriteString(w, l); err != nil {
			return err
		}
	}
	return nil
}

func sanitizeName(n string) string {
	n = strings.Replace(n, "\n", "-", -1)
	n = strings.Replace(n, "\r", "-", -1)
	return n
}

func sanitizeValue(v string) string {
	v = strings.Replace(v, "\n", " ", -1)
	v = strings.Replace(v, "\r", " ", -1)
	v = strings.Replace(v, ";", " ", -1)
	return v
}

func unquoteCookieValue(v string) string {
	if len(v) > 1 && v[0] == '"' && v[len(v)-1] == '"' {
		return v[1 : len(v)-1]
	}
	return v
}

func isCookieByte(c byte) bool {
	switch true {
	case c == 0x21, 0x23 <= c && c <= 0x2b, 0x2d <= c && c <= 0x3a,
		0x3c <= c && c <= 0x5b, 0x5d <= c && c <= 0x7e:
		return true
	}
	return false
}

func parseCookieValue(raw string) (string, bool) {
	raw = unquoteCookieValue(raw)
	for i := 0; i < len(raw); i++ {
		if !isCookieByte(raw[i]) {
			return "", false
		}
	}
	return raw, true
}

func isCookieNameValid(raw string) bool {
	for _, c := range raw {
		if !isToken(byte(c)) {
			return false
		}
	}
	return true
}
<commit_msg>http: update cookie doc to reference new RFC 6265<commit_after>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package http

import (
	"bytes"
	"fmt"
	"io"
	"os"
	"sort"
	"strconv"
	"strings"
	"time"
)

// This implementation is done according to RFC 6265:
//
//    http://tools.ietf.org/html/rfc6265

// A Cookie represents an HTTP cookie as sent in the Set-Cookie header of an
// HTTP response or the Cookie header of an HTTP request.
type Cookie struct {
	Name       string
	Value      string
	Path       string
	Domain     string
	Expires    time.Time
	RawExpires string

	// MaxAge=0 means no 'Max-Age' attribute specified.
	// MaxAge<0 means delete cookie now, equivalently 'Max-Age: 0'
	// MaxAge>0 means Max-Age attribute present and given in seconds
	MaxAge   int
	Secure   bool
	HttpOnly bool
	Raw      string
	Unparsed []string // Raw text of unparsed attribute-value pairs
}

// readSetCookies parses all "Set-Cookie" values from
// the header h, removes the successfully parsed values from the
// "Set-Cookie" key in h and returns the parsed Cookies.
func readSetCookies(h Header) []*Cookie {
	cookies := []*Cookie{}
	var unparsedLines []string
	for _, line := range h["Set-Cookie"] {
		parts := strings.Split(strings.TrimSpace(line), ";", -1)
		if len(parts) == 1 && parts[0] == "" {
			continue
		}
		parts[0] = strings.TrimSpace(parts[0])
		j := strings.Index(parts[0], "=")
		if j < 0 {
			unparsedLines = append(unparsedLines, line)
			continue
		}
		name, value := parts[0][:j], parts[0][j+1:]
		if !isCookieNameValid(name) {
			unparsedLines = append(unparsedLines, line)
			continue
		}
		value, success := parseCookieValue(value)
		if !success {
			unparsedLines = append(unparsedLines, line)
			continue
		}
		c := &Cookie{
			Name:  name,
			Value: value,
			Raw:   line,
		}
		for i := 1; i < len(parts); i++ {
			parts[i] = strings.TrimSpace(parts[i])
			if len(parts[i]) == 0 {
				continue
			}

			attr, val := parts[i], ""
			if j := strings.Index(attr, "="); j >= 0 {
				attr, val = attr[:j], attr[j+1:]
			}
			val, success = parseCookieValue(val)
			if !success {
				c.Unparsed = append(c.Unparsed, parts[i])
				continue
			}
			switch strings.ToLower(attr) {
			case "secure":
				c.Secure = true
				continue
			case "httponly":
				c.HttpOnly = true
				continue
			case "domain":
				c.Domain = val
				// TODO: Add domain parsing
				continue
			case "max-age":
				secs, err := strconv.Atoi(val)
				if err != nil || secs < 0 || secs != 0 && val[0] == '0' {
					break
				}
				if secs <= 0 {
					c.MaxAge = -1
				} else {
					c.MaxAge = secs
				}
				continue
			case "expires":
				c.RawExpires = val
				exptime, err := time.Parse(time.RFC1123, val)
				if err != nil {
					c.Expires = time.Time{}
					break
				}
				c.Expires = *exptime
				continue
			case "path":
				c.Path = val
				// TODO: Add path parsing
				continue
			}
			c.Unparsed = append(c.Unparsed, parts[i])
		}
		cookies = append(cookies, c)
	}
	h["Set-Cookie"] = unparsedLines, unparsedLines != nil
	return cookies
}

// writeSetCookies writes the wire representation of the set-cookies
// to w. Each cookie is written on a separate "Set-Cookie: " line.
// This choice is made because HTTP parsers tend to have a limit on
// line-length, so it seems safer to place cookies on separate lines.
func writeSetCookies(w io.Writer, kk []*Cookie) os.Error {
	if kk == nil {
		return nil
	}
	lines := make([]string, 0, len(kk))
	var b bytes.Buffer
	for _, c := range kk {
		b.Reset()
		fmt.Fprintf(&b, "%s=%s", sanitizeName(c.Name), sanitizeValue(c.Value))
		if len(c.Path) > 0 {
			fmt.Fprintf(&b, "; Path=%s", sanitizeValue(c.Path))
		}
		if len(c.Domain) > 0 {
			fmt.Fprintf(&b, "; Domain=%s", sanitizeValue(c.Domain))
		}
		if len(c.Expires.Zone) > 0 {
			fmt.Fprintf(&b, "; Expires=%s", c.Expires.Format(time.RFC1123))
		}
		if c.MaxAge > 0 {
			fmt.Fprintf(&b, "; Max-Age=%d", c.MaxAge)
		} else if c.MaxAge < 0 {
			fmt.Fprintf(&b, "; Max-Age=0")
		}
		if c.HttpOnly {
			fmt.Fprintf(&b, "; HttpOnly")
		}
		if c.Secure {
			fmt.Fprintf(&b, "; Secure")
		}
		lines = append(lines, "Set-Cookie: "+b.String()+"\r\n")
	}
	sort.SortStrings(lines)
	for _, l := range lines {
		if _, err := io.WriteString(w, l); err != nil {
			return err
		}
	}
	return nil
}

// readCookies parses all "Cookie" values from
// the header h, removes the successfully parsed values from the
// "Cookie" key in h and returns the parsed Cookies.
func readCookies(h Header) []*Cookie {
	cookies := []*Cookie{}
	lines, ok := h["Cookie"]
	if !ok {
		return cookies
	}
	unparsedLines := []string{}
	for _, line := range lines {
		parts := strings.Split(strings.TrimSpace(line), ";", -1)
		if len(parts) == 1 && parts[0] == "" {
			continue
		}
		// Per-line attributes
		parsedPairs := 0
		for i := 0; i < len(parts); i++ {
			parts[i] = strings.TrimSpace(parts[i])
			if len(parts[i]) == 0 {
				continue
			}
			attr, val := parts[i], ""
			if j := strings.Index(attr, "="); j >= 0 {
				attr, val = attr[:j], attr[j+1:]
			}
			if !isCookieNameValid(attr) {
				continue
			}
			val, success := parseCookieValue(val)
			if !success {
				continue
			}
			cookies = append(cookies, &Cookie{Name: attr, Value: val})
			parsedPairs++
		}
		if parsedPairs == 0 {
			unparsedLines = append(unparsedLines, line)
		}
	}
	h["Cookie"] = unparsedLines, len(unparsedLines) > 0
	return cookies
}

// writeCookies writes the wire representation of the cookies
// to w. Each cookie is written on a separate "Cookie: " line.
// This choice is made because HTTP parsers tend to have a limit on
// line-length, so it seems safer to place cookies on separate lines.
func writeCookies(w io.Writer, kk []*Cookie) os.Error {
	lines := make([]string, 0, len(kk))
	for _, c := range kk {
		lines = append(lines, fmt.Sprintf("Cookie: %s=%s\r\n", sanitizeName(c.Name), sanitizeValue(c.Value)))
	}
	sort.SortStrings(lines)
	for _, l := range lines {
		if _, err := io.WriteString(w, l); err != nil {
			return err
		}
	}
	return nil
}

func sanitizeName(n string) string {
	n = strings.Replace(n, "\n", "-", -1)
	n = strings.Replace(n, "\r", "-", -1)
	return n
}

func sanitizeValue(v string) string {
	v = strings.Replace(v, "\n", " ", -1)
	v = strings.Replace(v, "\r", " ", -1)
	v = strings.Replace(v, ";", " ", -1)
	return v
}

func unquoteCookieValue(v string) string {
	if len(v) > 1 && v[0] == '"' && v[len(v)-1] == '"' {
		return v[1 : len(v)-1]
	}
	return v
}

func isCookieByte(c byte) bool {
	switch true {
	case c == 0x21, 0x23 <= c && c <= 0x2b, 0x2d <= c && c <= 0x3a,
		0x3c <= c && c <= 0x5b, 0x5d <= c && c <= 0x7e:
		return true
	}
	return false
}

func parseCookieValue(raw string) (string, bool) {
	raw = unquoteCookieValue(raw)
	for i := 0; i < len(raw); i++ {
		if !isCookieByte(raw[i]) {
			return "", false
		}
	}
	return raw, true
}

func isCookieNameValid(raw string) bool {
	for _, c := range raw {
		if !isToken(byte(c)) {
			return false
		}
	}
	return true
}
<|endoftext|>


<commit_before>package bot

import (
	"bufio"
	"fmt"
	"path/filepath"
	"strconv"
	"strings"
)

/*

Job builtins are special:
- They're available in every channel
- Permissions are checked against the job being operated on, not job builtin

*/

const histPageSize = 2048 // how much history to display at a time

const builtInHistoryConfig = `
AllChannels: true
AllowDirect: false
Help:
- Keywords: [ "history", "job" ]
  Helptext: [ "(bot), history <job> - list the histories available for a job" ]
- Keywords: [ "quit" ]
  Helptext: [ "(bot), history (job) <run#> - start paging the history text for a job run" ]
CommandMatchers:
- Command: history
  Regex: '(?i:history(?: ([A-Za-z][\w-]*))?)'
- Command: showhistory
  Regex: '(?i:history (?:([A-Za-z][\w-]*) )?(\d+))'
MessageMatchers:
- Command: history
  Regex: '(?i:^history(?: ([A-Za-z][\w-]*))?$)'
  Contexts: [ "task" ]
- Command: showhistory
  Regex: '(?i:^history (?:([A-Za-z][\w-]*) )?(\d+)$)'
  Contexts: [ "task" ]
ReplyMatchers:
- Label: paging
  Regex: '(?i:(c|n|q))'
`

func init() {
	RegisterPlugin("builtInhistory", PluginHandler{DefaultConfig: builtInHistoryConfig, Handler: jobhistory})
}

func jobhistory(r *Robot, command string, args ...string) (retval TaskRetVal) {
	if command == "init" {
		return
	}

	taskName := args[0]
	// boilerplate availability and security checking for job commands
	c := r.getContext()
	t := c.jobAvailable(taskName, true)
	if t == nil {
		return
	}
	if !c.jobSecurityCheck(t) {
		return
	}

	switch command {
	case "history":
		var th taskHistory
		key := histPrefix + taskName
		_, _, ret := checkoutDatum(key, &th, false)
		if ret != Ok || len(th.Histories) == 0 {
			r.Say(fmt.Sprintf("No history found for '%s'", taskName))
			return
		}
		hl := make([]string, len(th.Histories)+1)
		hl = append(hl, fmt.Sprintf("History of job runs for '%s':", taskName))
		for _, he := range th.Histories {
			hl = append(hl, fmt.Sprintf("Run #%d - %s", he.LogIndex, he.CreateTime))
		}
		r.Say(strings.Join(hl, "\n"))
	case "showhistory":
		index, _ := strconv.Atoi(args[1])
		f, err := robot.history.GetHistory(taskName, index)
		if err != nil {
			Log(Error, fmt.Sprintf("Error getting history #%d for task '%s': %v", index, taskName, err))
			r.Say(fmt.Sprintf("History #%d for '%s' not available", index, taskName))
			return
		}
		var line string
		scanner := bufio.NewScanner(f)
		finished := false
	PageLoop:
		for {
			size := 0
			lines := make([]string, 0, 40)
			if len(line) > 0 {
				lines = append(lines, line)
				size += len(line) + 1
				line = ""
			}
			for size < histPageSize {
				if scanner.Scan() {
					line = scanner.Text()
					size += len(line) + 1
					if size < histPageSize {
						lines = append(lines, line)
						line = ""
					}
				} else {
					finished = true
					break
				}
			}
			r.Fixed().Say(strings.Join(lines, "\n"))
			if finished {
				break
			}
			rep, ret := r.PromptForReply("paging", "'c' to continue, 'q' to quit, or 'n' to skip to the next section")
			if ret != Ok && ret != TimeoutExpired {
				r.Say("(quitting)")
				break PageLoop
			} else {
			ContinueSwitch:
				switch rep {
				case "q", "Q":
					break PageLoop
				case "n", "N":
					for scanner.Scan() {
						line = scanner.Text()
						if strings.HasPrefix(line, "***") {
							break ContinueSwitch
						}
					}
				}
			}
		}
	}
	return
}

// jobSecurityCheck performs all security checks - RequireAdmin, Authorization
// and Elevation - and returns true if passed. It will message the user and
// return false if a check fails.
func (c *botContext) jobSecurityCheck(t interface{}) bool {
	if c.automaticTask {
		return true
	}
	task, _, _ := getTask(t)
	if task.RequireAdmin {
		r := c.makeRobot()
		if !r.CheckAdmin() {
			r.Say("Sorry, that command is only available to bot administrators")
			return false
		}
	}
	if c.checkAuthorization(t, "run") != Success {
		return false
	}
	if !c.elevated {
		eret, required := c.checkElevation(t, "")
		if eret != Success {
			return false
		}
		if required {
			c.elevated = true
		}
	}
	return true
}

// jobAvailable does the work of looking up a job and checking whether it's
// available, and messaging the user if it's not. Only called for interactive
// job commands like history, run job, etc.
func (c *botContext) jobAvailable(taskName string, pluginOk bool) interface{} {
	r := c.makeRobot()
	t := c.tasks.getTaskByName(taskName)
	if t == nil {
		r.Say(fmt.Sprintf("Sorry, I don't have a task named '%s' configured", taskName))
		return nil
	}
	task, plugin, job := getTask(t)
	isPlugin := plugin != nil
	isJob := job != nil
	if !isJob && (!isPlugin && pluginOk) {
		r.Say(fmt.Sprintf("Sorry, '%s' isn't a job", taskName))
		return nil
	}
	if isPlugin {
		ok := c.pluginAvailable(task, false, false)
		if ok {
			return t
		}
		r.Say(fmt.Sprintf("Sorry, '%s' isn't available", taskName))
		return nil
	}
	if r.Channel != task.Channel {
		c.debugTask(task, fmt.Sprintf("not available in channel '%s'", task.Channel), false)
		r.Say(fmt.Sprintf("Sorry, job '%s' isn't available in this channel, try '%s'", taskName, task.Channel))
		return nil
	}
	if len(task.Users) > 0 {
		userOk := false
		for _, allowedUser := range task.Users {
			match, err := filepath.Match(allowedUser, r.User)
			if match && err == nil {
				userOk = true
			}
		}
		if !userOk {
			r.Say("Sorry, you're not on the list of allowed users for that job")
			c.debugTask(task, "user is not on the list of allowed users", false)
			return nil
		}
	}
	return t
}
<commit_msg>Remove '#'<commit_after>package bot

import (
	"bufio"
	"fmt"
	"path/filepath"
	"strconv"
	"strings"
)

/*

Job builtins are special:
- They're available in every channel
- Permissions are checked against the job being operated on, not job builtin

*/

const histPageSize = 2048 // how much history to display at a time

const builtInHistoryConfig = `
AllChannels: true
AllowDirect: false
Help:
- Keywords: [ "history", "job" ]
  Helptext: [ "(bot), history <job> - list the histories available for a job" ]
- Keywords: [ "quit" ]
  Helptext: [ "(bot), history (job) <run#> - start paging the history text for a job run" ]
CommandMatchers:
- Command: history
  Regex: '(?i:history(?: ([A-Za-z][\w-]*))?)'
- Command: showhistory
  Regex: '(?i:history (?:([A-Za-z][\w-]*) )?(\d+))'
MessageMatchers:
- Command: history
  Regex: '(?i:^history(?: ([A-Za-z][\w-]*))?$)'
  Contexts: [ "task" ]
- Command: showhistory
  Regex: '(?i:^history (?:([A-Za-z][\w-]*) )?(\d+)$)'
  Contexts: [ "task" ]
ReplyMatchers:
- Label: paging
  Regex: '(?i:(c|n|q))'
`

func init() {
	RegisterPlugin("builtInhistory", PluginHandler{DefaultConfig: builtInHistoryConfig, Handler: jobhistory})
}

func jobhistory(r *Robot, command string, args ...string) (retval TaskRetVal) {
	if command == "init" {
		return
	}

	taskName := args[0]
	// boilerplate availability and security checking for job commands
	c := r.getContext()
	t := c.jobAvailable(taskName, true)
	if t == nil {
		return
	}
	if !c.jobSecurityCheck(t) {
		return
	}

	switch command {
	case "history":
		var th taskHistory
		key := histPrefix + taskName
		_, _, ret := checkoutDatum(key, &th, false)
		if ret != Ok || len(th.Histories) == 0 {
			r.Say(fmt.Sprintf("No history found for '%s'", taskName))
			return
		}
		hl := make([]string, len(th.Histories)+1)
		hl = append(hl, fmt.Sprintf("History of job runs for '%s':", taskName))
		for _, he := range th.Histories {
			hl = append(hl, fmt.Sprintf("Run %d - %s", he.LogIndex, he.CreateTime))
		}
		r.Say(strings.Join(hl, "\n"))
	case "showhistory":
		index, _ := strconv.Atoi(args[1])
		f, err := robot.history.GetHistory(taskName, index)
		if err != nil {
			Log(Error, fmt.Sprintf("Error getting history %d for task '%s': %v", index, taskName, err))
			r.Say(fmt.Sprintf("History %d for '%s' not available", index, taskName))
			return
		}
		var line string
		scanner := bufio.NewScanner(f)
		finished := false
	PageLoop:
		for {
			size := 0
			lines := make([]string, 0, 40)
			if len(line) > 0 {
				lines = append(lines, line)
				size += len(line) + 1
				line = ""
			}
			for size < histPageSize {
				if scanner.Scan() {
					line = scanner.Text()
					size += len(line) + 1
					if size < histPageSize {
						lines = append(lines, line)
						line = ""
					}
				} else {
					finished = true
					break
				}
			}
			r.Fixed().Say(strings.Join(lines, "\n"))
			if finished {
				break
			}
			rep, ret := r.PromptForReply("paging", "'c' to continue, 'q' to quit, or 'n' to skip to the next section")
			if ret != Ok && ret != TimeoutExpired {
				r.Say("(quitting)")
				break PageLoop
			} else {
			ContinueSwitch:
				switch rep {
				case "q", "Q":
					break PageLoop
				case "n", "N":
					for scanner.Scan() {
						line = scanner.Text()
						if strings.HasPrefix(line, "***") {
							break ContinueSwitch
						}
					}
				}
			}
		}
	}
	return
}

// jobSecurityCheck performs all security checks - RequireAdmin, Authorization
// and Elevation - and returns true if passed. It will message the user and
// return false if a check fails.
func (c *botContext) jobSecurityCheck(t interface{}) bool {
	if c.automaticTask {
		return true
	}
	task, _, _ := getTask(t)
	if task.RequireAdmin {
		r := c.makeRobot()
		if !r.CheckAdmin() {
			r.Say("Sorry, that command is only available to bot administrators")
			return false
		}
	}
	if c.checkAuthorization(t, "run") != Success {
		return false
	}
	if !c.elevated {
		eret, required := c.checkElevation(t, "")
		if eret != Success {
			return false
		}
		if required {
			c.elevated = true
		}
	}
	return true
}

// jobAvailable does the work of looking up a job and checking whether it's
// available, and messaging the user if it's not. Only called for interactive
// job commands like history, run job, etc.
func (c *botContext) jobAvailable(taskName string, pluginOk bool) interface{} {
	r := c.makeRobot()
	t := c.tasks.getTaskByName(taskName)
	if t == nil {
		r.Say(fmt.Sprintf("Sorry, I don't have a task named '%s' configured", taskName))
		return nil
	}
	task, plugin, job := getTask(t)
	isPlugin := plugin != nil
	isJob := job != nil
	if !isJob && (!isPlugin && pluginOk) {
		r.Say(fmt.Sprintf("Sorry, '%s' isn't a job", taskName))
		return nil
	}
	if isPlugin {
		ok := c.pluginAvailable(task, false, false)
		if ok {
			return t
		}
		r.Say(fmt.Sprintf("Sorry, '%s' isn't available", taskName))
		return nil
	}
	if r.Channel != task.Channel {
		c.debugTask(task, fmt.Sprintf("not available in channel '%s'", task.Channel), false)
		r.Say(fmt.Sprintf("Sorry, job '%s' isn't available in this channel, try '%s'", taskName, task.Channel))
		return nil
	}
	if len(task.Users) > 0 {
		userOk := false
		for _, allowedUser := range task.Users {
			match, err := filepath.Match(allowedUser, r.User)
			if match && err == nil {
				userOk = true
			}
		}
		if !userOk {
			r.Say("Sorry, you're not on the list of allowed users for that job")
			c.debugTask(task, "user is not on the list of allowed users", false)
			return nil
		}
	}
	return t
}
<|endoftext|>


<commit_before>package healthcheck

import (
	"fmt"
	"log"
	"net"
	"testing"
)

func TestHealthcheckTcpNoPort(t *testing.T) {
	c := make(map[string]string)
	h := Healthcheck{
		Type:        "tcp",
		Destination: "127.0.0.1",
		Config:      c,
	}
	h.Default()
	h.Validate("foo")
	err := h.Setup()
	if err == nil {
		t.Fail()
	} else {
		if err.Error() != "'port' not defined in tcp healthcheck config to 127.0.0.1" {
			t.Log(err.Error())
			t.Fail()
		}
	}
}

func TestHealthcheckTcp(t *testing.T) {
	ln, err := net.Listen("tcp", "127.0.0.1:0")
	if err != nil {
		t.Fatal(err)
	}
	port := ln.Addr().(*net.TCPAddr).Port
	defer ln.Close()
	go func() {
		for {
			conn, err := ln.Accept()
			if err != nil {
				t.Fatal(fmt.Printf("Error accepting: ", err.Error()))
			}
			go func(conn net.Conn) {
				buf := make([]byte, 1024)
				n, err := conn.Read(buf)
				if err != nil {
					t.Log(fmt.Printf("Error reading: %s", err.Error()))
					t.Fail()
				}
				if string(buf[:n]) != "HEAD / HTTP/1.0\r\n\r\n" {
					t.Log(string(buf[:n]))
					t.Fail()
				}
				conn.Write([]byte("200 OK"))
				conn.Close()
			}(conn)
		}
	}()
	c := make(map[string]string)
	c["port"] = fmt.Sprintf("%d", port)
	h := Healthcheck{
		Type:        "tcp",
		Destination: "127.0.0.1",
		Config:      c,
	}
	h.Default()
	err = h.Validate("foo")
	if err != nil {
		t.Log(err)
		t.Fail()
	}
	err = h.Setup()
	if err != nil {
		t.Log("Setup failed: %s", err.Error())
		t.Fail()
	} else {
		log.Printf("%+v", h)
		res := h.healthchecker.Healthcheck()
		if !res {
			t.Log("h.healthchecker.Healthcheck() returned false")
			t.Fail()
		}
	}
}

/*
func TestHealthcheckTcpFail(t *testing.T) {
	h := Healthcheck{
		Type:        "tcp",
		Destination: "169.254.255.45", // Hopefully you can't talk to this :)
	}
	h.Default()
	err := h.Validate("foo")
	if err != nil {
		t.Log(err)
		t.Fail()
	}
	h.Setup()
	res := h.healthchecker.Healthcheck()
	if res {
		t.Fail()
	}
}
*/
<commit_msg>Move close explicitly to the bottom<commit_after>package healthcheck

import (
	"fmt"
	"log"
	"net"
	"testing"
)

func TestHealthcheckTcpNoPort(t *testing.T) {
	c := make(map[string]string)
	h := Healthcheck{
		Type:        "tcp",
		Destination: "127.0.0.1",
		Config:      c,
	}
	h.Default()
	h.Validate("foo")
	err := h.Setup()
	if err == nil {
		t.Fail()
	} else {
		if err.Error() != "'port' not defined in tcp healthcheck config to 127.0.0.1" {
			t.Log(err.Error())
			t.Fail()
		}
	}
}

func TestHealthcheckTcp(t *testing.T) {
	ln, err := net.Listen("tcp", "127.0.0.1:0")
	if err != nil {
		t.Fatal(err)
	}
	port := ln.Addr().(*net.TCPAddr).Port
	go func() {
		for {
			conn, err := ln.Accept()
			if err != nil {
				t.Fatal(fmt.Printf("Error accepting: ", err.Error()))
			}
			go func(conn net.Conn) {
				buf := make([]byte, 1024)
				n, err := conn.Read(buf)
				if err != nil {
					t.Log(fmt.Printf("Error reading: %s", err.Error()))
					t.Fail()
				}
				if string(buf[:n]) != "HEAD / HTTP/1.0\r\n\r\n" {
					t.Log(string(buf[:n]))
					t.Fail()
				}
				conn.Write([]byte("200 OK"))
				conn.Close()
			}(conn)
		}
	}()
	c := make(map[string]string)
	c["port"] = fmt.Sprintf("%d", port)
	h := Healthcheck{
		Type:        "tcp",
		Destination: "127.0.0.1",
		Config:      c,
	}
	h.Default()
	err = h.Validate("foo")
	if err != nil {
		t.Log(err)
		t.Fail()
	}
	err = h.Setup()
	if err != nil {
		t.Log("Setup failed: %s", err.Error())
		t.Fail()
	} else {
		log.Printf("%+v", h)
		res := h.healthchecker.Healthcheck()
		if !res {
			t.Log("h.healthchecker.Healthcheck() returned false")
			t.Fail()
		}
	}
	ln.Close()
}

/*
func TestHealthcheckTcpFail(t *testing.T) {
	h := Healthcheck{
		Type:        "tcp",
		Destination: "169.254.255.45", // Hopefully you can't talk to this :)
	}
	h.Default()
	err := h.Validate("foo")
	if err != nil {
		t.Log(err)
		t.Fail()
	}
	h.Setup()
	res := h.healthchecker.Healthcheck()
	if res {
		t.Fail()
	}
}
*/
<|endoftext|>


<commit_before>package sudoku

import (
	"testing"
)

func TestPairwiseBlocks(t *testing.T) {
	grid := NewGrid()
	result := pairwiseBlocks(grid)

	//TODO: should this be encoded as DIM x DIM?
	if len(result) != 18 {
		t.Error("Pairwise blocks had wrong length")
	}

	for i, pair := range result {
		if len(pair) != 2 {
			t.Error(i, "th pair was wrong size: ", pair)
		}
		if pair[0] == pair[1] {
			t.Error(i, "th pair was not two unique cells", pair)
		}
	}
}

func TestBlockBlockInteraction(t *testing.T) {
	//TODO: test description
	options := solveTechniqueTestHelperOptions{
		targetCells:  []cellRef{{6, 3}, {6, 4}, {7, 3}, {7, 4}, {8, 3}, {8, 4}},
		pointerCells: []cellRef{{2, 3}, {2, 4}, {3, 4}, {5, 3}},
		targetSame:   GROUP_BLOCK,
		targetGroup:  7,
		targetNums:   IntSlice([]int{3}),
	}
	humanSolveTechniqueTestHelper(t, "blockblocktest.sdk", "Block Block Interactions", options)

	options = solveTechniqueTestHelperOptions{
		targetCells:  []cellRef{{3, 3}, {3, 4}, {3, 5}, {5, 3}, {5, 4}, {5, 5}},
		pointerCells: []cellRef{{3, 0}, {3, 1}, {3, 6}, {3, 7}, {5, 0}, {5, 1}, {5, 6}, {5, 7}},
		targetSame:   GROUP_BLOCK,
		targetGroup:  4,
		targetNums:   IntSlice([]int{2}),
	}
	humanSolveTechniqueTestHelper(t, "blockblocktest1.sdk", "Block Block Interactions", options)

}
<commit_msg>TESTS FAIL. Broke them into two methods so they will both fail if they should.<commit_after>package sudoku

import (
	"testing"
)

func TestPairwiseBlocks(t *testing.T) {
	grid := NewGrid()
	result := pairwiseBlocks(grid)

	//TODO: should this be encoded as DIM x DIM?
	if len(result) != 18 {
		t.Error("Pairwise blocks had wrong length")
	}

	for i, pair := range result {
		if len(pair) != 2 {
			t.Error(i, "th pair was wrong size: ", pair)
		}
		if pair[0] == pair[1] {
			t.Error(i, "th pair was not two unique cells", pair)
		}
	}
}

func TestBlockBlockInteraction(t *testing.T) {
	//TODO: test description

	options := solveTechniqueTestHelperOptions{
		targetCells:  []cellRef{{6, 3}, {6, 4}, {7, 3}, {7, 4}, {8, 3}, {8, 4}},
		pointerCells: []cellRef{{2, 3}, {2, 4}, {3, 4}, {5, 3}},
		targetSame:   GROUP_BLOCK,
		targetGroup:  7,
		targetNums:   IntSlice([]int{3}),
	}
	humanSolveTechniqueTestHelper(t, "blockblocktest.sdk", "Block Block Interactions", options)

}

func TestBlockBlockInteractionAgain(t *testing.T) {
	options := solveTechniqueTestHelperOptions{
		targetCells:  []cellRef{{3, 3}, {3, 4}, {3, 5}, {5, 3}, {5, 4}, {5, 5}},
		pointerCells: []cellRef{{3, 0}, {3, 1}, {3, 6}, {3, 7}, {5, 0}, {5, 1}, {5, 6}, {5, 7}},
		targetSame:   GROUP_BLOCK,
		targetGroup:  4,
		targetNums:   IntSlice([]int{2}),
	}
	humanSolveTechniqueTestHelper(t, "blockblocktest1.sdk", "Block Block Interactions", options)
}
<|endoftext|>


<commit_before>// Copyright (c) 2016, Daniel Martí <mvdan@mvdan.cc>
// See LICENSE for licensing information

package syntax_test

import (
	"os"
	"strings"

	"github.com/mvdan/sh/syntax"
)

func Example() {
	in := strings.NewReader("{ foo;bar; }")
	f, err := syntax.Parse(in, "", 0)
	if err != nil {
		return
	}
	syntax.Fprint(os.Stdout, f)
	// Output:
	// {
	//	foo
	//	bar
	// }
}
<commit_msg>syntax: make example input more readable<commit_after>// Copyright (c) 2016, Daniel Martí <mvdan@mvdan.cc>
// See LICENSE for licensing information

package syntax_test

import (
	"os"
	"strings"

	"github.com/mvdan/sh/syntax"
)

func Example() {
	in := strings.NewReader("{ foo; bar; }")
	f, err := syntax.Parse(in, "", 0)
	if err != nil {
		return
	}
	syntax.Fprint(os.Stdout, f)
	// Output:
	// {
	//	foo
	//	bar
	// }
}
<|endoftext|>


<commit_before>package kasper

import (
	"encoding/json"
	"fmt"
	"errors"
	"strings"

	"golang.org/x/net/context"
	elastic "gopkg.in/olivere/elastic.v5"
)

// ElasticsearchOpts allow you to set custom mappings and settings depending on index name
type ElasticsearchOpts struct {
	// Returns index mappings JSON string
	GetIndexSettings func(indexName string) string
	// Returns index settings JSON string
	GetIndexMappings func(indexName string) string
}

// DefaultElasticsearchOpts allow to create new indexes with
// unindexed mappings, so ESStore can be ised only as basic key-value store.
var DefaultElasticsearchOpts = ElasticsearchOpts{
	GetIndexSettings: func(indexName string) string {
		return `{
			"index.translog.durability": "request"
		}`
	},
	GetIndexMappings: func(indexName string) string {
		return `{
			"_all" : {
				"enabled" : false
			},
			"dynamic_templates": [{
				"no_index": {
					"mapping": {
						"index": "no"
					},
					"match": "*"
				}
			}]
		}`
	},
}

type indexAndType struct {
	indexName string
	indexType string
}

// ElasticsearchKeyValueStore is a key-value storage that uses ElasticSearch.
// In this key-value store, all keys must have the format "<index>/<type>/<_id>".
type ElasticsearchKeyValueStore struct {
	elasticSearchOpts *ElasticsearchOpts
	witness           *structPtrWitness
	client            *elastic.Client
	context           context.Context
	existingIndexes   []indexAndType
	metricsProvider   MetricsProvider
	getCounter        Counter
	getAllSummary     Summary
	putCounter        Counter
	putAllSummary     Summary
	deleteCounter     Counter
	flushCounter      Counter
}

// NewElasticsearchKeyValueStore creates new ElasticsearchKeyValueStore instance.
// Host must of the format hostname:port.
// StructPtr should be a pointer to struct type that is used.
// for serialization and deserialization of store values.
func NewElasticsearchKeyValueStore(url string, structPtr interface{}) *ElasticsearchKeyValueStore {
	return NewElasticsearchKeyValueStoreWithOpts(url, structPtr, &DefaultElasticsearchOpts, &NoopMetricsProvider{})
}

// NewElasticsearchKeyValueStoreWithOpts creates new ElasticsearchKeyValueStore instance.
// Host must of the format hostname:port.
// StructPtr should be a pointer to struct type that is used.
// for serialization and deserialization of store values.
// Opts allow you to set custom settings and mappings on index creatiion.
// metricsProvider is used for reporting store requests metrics.
func NewElasticsearchKeyValueStoreWithOpts(url string, structPtr interface{}, opts *ElasticsearchOpts, metricsProvider MetricsProvider) *ElasticsearchKeyValueStore {
	client, err := elastic.NewClient(
		elastic.SetURL(url),
		elastic.SetSniff(false), // FIXME: workaround for issues with ES in docker
	)
	logger.Info("Connected to Elasticsearch at ", url)
	if err != nil {
		logger.Panic(fmt.Sprintf("Cannot create ElasticSearch Client to '%s': %s", url, err))
	}
	s := &ElasticsearchKeyValueStore{
		elasticSearchOpts: opts,
		witness:           newStructPtrWitness(structPtr),
		client:            client,
		context:           context.Background(),
		existingIndexes:   nil,
		metricsProvider:   metricsProvider,
	}
	s.createMetrics()
	return s
}

func (s *ElasticsearchKeyValueStore) createMetrics() {
	s.getCounter = s.metricsProvider.NewCounter("ElasticsearchKeyValueStore_Get", "Number of Get() calls", "type")
	s.getAllSummary = s.metricsProvider.NewSummary("ElasticsearchKeyValueStore_GetAll", "Summary of GetAll() calls", "type")
	s.putCounter = s.metricsProvider.NewCounter("ElasticsearchKeyValueStore_Put", "Number of Put() calls", "type")
	s.putAllSummary = s.metricsProvider.NewSummary("ElasticsearchKeyValueStore_PutAll", "Summary of PutAll() calls", "type")
	s.deleteCounter = s.metricsProvider.NewCounter("ElasticsearchKeyValueStore_Delete", "Number of Delete() calls", "type")
	s.flushCounter = s.metricsProvider.NewCounter("ElasticsearchKeyValueStore_Flush", "Summary of Flush() calls", "type")
}

func (s *ElasticsearchKeyValueStore) checkOrCreateIndex(indexName string, indexType string) {
	for _, existing := range s.existingIndexes {
		if existing.indexName == indexName && existing.indexType == indexType {
			return
		}
	}
	exists, err := s.client.IndexExists(indexName).Do(s.context)
	if err != nil {
		logger.Panic(fmt.Sprintf("Failed to check if index exists: %s", err))
	}
	if !exists {
		logger.Infof("Creating index %s", indexName)
		_, err = s.client.
			CreateIndex(indexName).
			BodyString(s.elasticSearchOpts.GetIndexSettings(indexName)).
			Do(s.context)
		if err != nil {
			logger.Panic(fmt.Sprintf("Failed to create index: %s", err))
		}
		s.putMapping(indexName, indexType)
	}

	s.existingIndexes = append(s.existingIndexes, indexAndType{indexName, indexType})
}

func (s *ElasticsearchKeyValueStore) putMapping(indexName string, indexType string) {
	resp, err := s.client.
		PutMapping().
		Index(indexName).
		Type(indexType).
		BodyString(s.elasticSearchOpts.GetIndexMappings(indexName)).
		Do(s.context)
	if err != nil {
		logger.Panic(fmt.Sprintf("Failed to put mapping for index: %s/%s: %s", indexName, indexType, err))
	}
	if resp == nil {
		logger.Panic(fmt.Sprintf("Expected put mapping response; got: %v", resp))
	}
	if !resp.Acknowledged {
		logger.Panic(fmt.Sprintf("Expected put mapping ack; got: %v", resp.Acknowledged))
	}
}

// Get gets value by key from store
func (s *ElasticsearchKeyValueStore) Get(key string) (interface{}, error) {
	logger.Debug("Elasticsearch Get: ", key)
	s.getCounter.Inc(s.witness.name)
	keyParts := strings.Split(key, "/")
	if len(keyParts) != 3 {
		logger.Panicf("invalid key: '%s'", key)
	}
	indexName := keyParts[0]
	indexType := keyParts[1]
	valueID := keyParts[2]

	s.checkOrCreateIndex(indexName, indexType)

	rawValue, err := s.client.Get().
		Index(indexName).
		Type(indexType).
		Id(valueID).
		Do(s.context)

	if fmt.Sprintf("%s", err) == "elastic: Error 404 (Not Found)" {
		return s.witness.nil(), nil
	}

	if err != nil {
		return s.witness.nil(), err
	}

	if !rawValue.Found {
		return s.witness.nil(), nil
	}

	structPtr := s.witness.allocate()
	err = json.Unmarshal(*rawValue.Source, structPtr)
	if err != nil {
		return s.witness.nil(), err
	}
	return structPtr, nil
}

// GetAll gets multiple keys from store using MultiGet.
func (s *ElasticsearchKeyValueStore) GetAll(keys []string) ([]*KeyValue, error) {
	logger.Debug("Elasticsearch GetAll: ", keys)
	s.getAllSummary.Observe(float64(len(keys)), s.witness.name)
	multiGet := s.client.MultiGet()
	for _, key := range keys {
		keyParts := strings.Split(key, "/")
		if len(keyParts) != 3 {
			logger.Panicf("invalid key: '%s'", key)
		}
		indexName := keyParts[0]
		indexType := keyParts[1]
		valueID := keyParts[2]

		s.checkOrCreateIndex(indexName, indexType)

		item := elastic.NewMultiGetItem().
			Index(indexName).
			Type(indexType).
			Id(valueID)

		multiGet.Add(item)
	}
	response, err := multiGet.Do(s.context)
	if err != nil {
		return nil, err
	}
	kvs := make([]*KeyValue, len(keys))
	for i, doc := range response.Docs {
		var structPtr interface{}
		if !doc.Found {
			structPtr = s.witness.nil()
		} else {
			structPtr = s.witness.allocate()
			err = json.Unmarshal(*doc.Source, structPtr)
			if err != nil {
				return nil, err
			}
		}
		kvs[i] = &KeyValue{keys[i], structPtr}
	}
	return kvs, nil
}

// Put updates key in store with serialized value
func (s *ElasticsearchKeyValueStore) Put(key string, structPtr interface{}) error {
	logger.Debugf("Elasticsearch Put: %s %#v", key, structPtr)
	s.witness.assert(structPtr)
	s.putCounter.Inc(s.witness.name)
	keyParts := strings.Split(key, "/")
	if len(keyParts) != 3 {
		logger.Panicf("invalid key: '%s'", key)
	}
	indexName := keyParts[0]
	indexType := keyParts[1]
	valueID := keyParts[2]

	s.checkOrCreateIndex(indexName, indexType)

	_, err := s.client.Index().
		Index(indexName).
		Type(indexType).
		Id(valueID).
		BodyJson(structPtr).
		Do(s.context)

	return err
}

// PutAll bulk executes Put operation for several kvs
func (s *ElasticsearchKeyValueStore) PutAll(kvs []*KeyValue) error {
	logger.Debugf("Elasticsearch PutAll of %d keys", len(kvs))
	s.putAllSummary.Observe(float64(len(kvs)), s.witness.name)
	if len(kvs) == 0 {
		return nil
	}
	bulk := s.client.Bulk()
	for _, kv := range kvs {
		keyParts := strings.Split(kv.Key, "/")
		if len(keyParts) != 3 {
			logger.Panicf("invalid key: '%s'", kv.Key)
		}
		indexName := keyParts[0]
		indexType := keyParts[1]
		valueID := keyParts[2]

		s.witness.assert(kv.Value)
		s.checkOrCreateIndex(indexName, indexType)

		bulk.Add(elastic.NewBulkIndexRequest().
			Index(indexName).
			Type(indexType).
			Id(valueID).
			Doc(kv.Value),
		)
	}
	response, err := bulk.Do(s.context)
	if err != nil {
		return err
	}
	if response.Errors {
		reasons := []string{}
		failed := response.Failed()
		for i, item := range failed {
			if item.Error != nil {
				reason := fmt.Sprintf("id = %s, error = %s\n", item.Id, item.Error.Reason)
				reasons = append(reasons, reason)
			}
			if i == 4 {
				reason := fmt.Sprintf("(ommited %d more errors)", len(failed) - 5)
				reasons = append(reasons, reason)
				break
			}
		}
		return errors.New(fmt.Sprintf("PutAll failed for some requests:\n%s", strings.Join(reasons, "")))
	}
	return nil
}

// Delete removes key from store
func (s *ElasticsearchKeyValueStore) Delete(key string) error {
	logger.Debug("Elasticsearch Delete: ", key)
	s.deleteCounter.Inc(s.witness.name)
	keyParts := strings.Split(key, "/")
	if len(keyParts) != 3 {
		logger.Panicf("invalid key: '%s'", key)
	}
	indexName := keyParts[0]
	indexType := keyParts[1]
	valueID := keyParts[2]

	s.checkOrCreateIndex(indexName, indexType)

	_, err := s.client.Delete().
		Index(indexName).
		Type(indexType).
		Id(valueID).
		Do(s.context)

	if err != nil && err.(*elastic.Error).Status == 404 {
		return nil
	}

	return err
}

// Flush the Elasticsearch translog to disk
func (s *ElasticsearchKeyValueStore) Flush() error {
	s.flushCounter.Inc(s.witness.name)
	logger.Info("Elasticsearch Flush...")
	_, err := s.client.Flush("_all").
		WaitIfOngoing(true).
		Do(s.context)
	logger.Info("Elasticsearch Flush complete")
	return err
}
<commit_msg>Extract constant<commit_after>package kasper

import (
	"encoding/json"
	"fmt"
	"errors"
	"strings"

	"golang.org/x/net/context"
	elastic "gopkg.in/olivere/elastic.v5"
)

const maxBulkErrorReasons = 5

// ElasticsearchOpts allow you to set custom mappings and settings depending on index name
type ElasticsearchOpts struct {
	// Returns index mappings JSON string
	GetIndexSettings func(indexName string) string
	// Returns index settings JSON string
	GetIndexMappings func(indexName string) string
}

// DefaultElasticsearchOpts allow to create new indexes with
// unindexed mappings, so ESStore can be ised only as basic key-value store.
var DefaultElasticsearchOpts = ElasticsearchOpts{
	GetIndexSettings: func(indexName string) string {
		return `{
			"index.translog.durability": "request"
		}`
	},
	GetIndexMappings: func(indexName string) string {
		return `{
			"_all" : {
				"enabled" : false
			},
			"dynamic_templates": [{
				"no_index": {
					"mapping": {
						"index": "no"
					},
					"match": "*"
				}
			}]
		}`
	},
}

type indexAndType struct {
	indexName string
	indexType string
}

// ElasticsearchKeyValueStore is a key-value storage that uses ElasticSearch.
// In this key-value store, all keys must have the format "<index>/<type>/<_id>".
type ElasticsearchKeyValueStore struct {
	elasticSearchOpts *ElasticsearchOpts
	witness           *structPtrWitness
	client            *elastic.Client
	context           context.Context
	existingIndexes   []indexAndType
	metricsProvider   MetricsProvider
	getCounter        Counter
	getAllSummary     Summary
	putCounter        Counter
	putAllSummary     Summary
	deleteCounter     Counter
	flushCounter      Counter
}

// NewElasticsearchKeyValueStore creates new ElasticsearchKeyValueStore instance.
// Host must of the format hostname:port.
// StructPtr should be a pointer to struct type that is used.
// for serialization and deserialization of store values.
func NewElasticsearchKeyValueStore(url string, structPtr interface{}) *ElasticsearchKeyValueStore {
	return NewElasticsearchKeyValueStoreWithOpts(url, structPtr, &DefaultElasticsearchOpts, &NoopMetricsProvider{})
}

// NewElasticsearchKeyValueStoreWithOpts creates new ElasticsearchKeyValueStore instance.
// Host must of the format hostname:port.
// StructPtr should be a pointer to struct type that is used.
// for serialization and deserialization of store values.
// Opts allow you to set custom settings and mappings on index creatiion.
// metricsProvider is used for reporting store requests metrics.
func NewElasticsearchKeyValueStoreWithOpts(url string, structPtr interface{}, opts *ElasticsearchOpts, metricsProvider MetricsProvider) *ElasticsearchKeyValueStore {
	client, err := elastic.NewClient(
		elastic.SetURL(url),
		elastic.SetSniff(false), // FIXME: workaround for issues with ES in docker
	)
	logger.Info("Connected to Elasticsearch at ", url)
	if err != nil {
		logger.Panic(fmt.Sprintf("Cannot create ElasticSearch Client to '%s': %s", url, err))
	}
	s := &ElasticsearchKeyValueStore{
		elasticSearchOpts: opts,
		witness:           newStructPtrWitness(structPtr),
		client:            client,
		context:           context.Background(),
		existingIndexes:   nil,
		metricsProvider:   metricsProvider,
	}
	s.createMetrics()
	return s
}

func (s *ElasticsearchKeyValueStore) createMetrics() {
	s.getCounter = s.metricsProvider.NewCounter("ElasticsearchKeyValueStore_Get", "Number of Get() calls", "type")
	s.getAllSummary = s.metricsProvider.NewSummary("ElasticsearchKeyValueStore_GetAll", "Summary of GetAll() calls", "type")
	s.putCounter = s.metricsProvider.NewCounter("ElasticsearchKeyValueStore_Put", "Number of Put() calls", "type")
	s.putAllSummary = s.metricsProvider.NewSummary("ElasticsearchKeyValueStore_PutAll", "Summary of PutAll() calls", "type")
	s.deleteCounter = s.metricsProvider.NewCounter("ElasticsearchKeyValueStore_Delete", "Number of Delete() calls", "type")
	s.flushCounter = s.metricsProvider.NewCounter("ElasticsearchKeyValueStore_Flush", "Summary of Flush() calls", "type")
}

func (s *ElasticsearchKeyValueStore) checkOrCreateIndex(indexName string, indexType string) {
	for _, existing := range s.existingIndexes {
		if existing.indexName == indexName && existing.indexType == indexType {
			return
		}
	}
	exists, err := s.client.IndexExists(indexName).Do(s.context)
	if err != nil {
		logger.Panic(fmt.Sprintf("Failed to check if index exists: %s", err))
	}
	if !exists {
		logger.Infof("Creating index %s", indexName)
		_, err = s.client.
			CreateIndex(indexName).
			BodyString(s.elasticSearchOpts.GetIndexSettings(indexName)).
			Do(s.context)
		if err != nil {
			logger.Panic(fmt.Sprintf("Failed to create index: %s", err))
		}
		s.putMapping(indexName, indexType)
	}

	s.existingIndexes = append(s.existingIndexes, indexAndType{indexName, indexType})
}

func (s *ElasticsearchKeyValueStore) putMapping(indexName string, indexType string) {
	resp, err := s.client.
		PutMapping().
		Index(indexName).
		Type(indexType).
		BodyString(s.elasticSearchOpts.GetIndexMappings(indexName)).
		Do(s.context)
	if err != nil {
		logger.Panic(fmt.Sprintf("Failed to put mapping for index: %s/%s: %s", indexName, indexType, err))
	}
	if resp == nil {
		logger.Panic(fmt.Sprintf("Expected put mapping response; got: %v", resp))
	}
	if !resp.Acknowledged {
		logger.Panic(fmt.Sprintf("Expected put mapping ack; got: %v", resp.Acknowledged))
	}
}

// Get gets value by key from store
func (s *ElasticsearchKeyValueStore) Get(key string) (interface{}, error) {
	logger.Debug("Elasticsearch Get: ", key)
	s.getCounter.Inc(s.witness.name)
	keyParts := strings.Split(key, "/")
	if len(keyParts) != 3 {
		logger.Panicf("invalid key: '%s'", key)
	}
	indexName := keyParts[0]
	indexType := keyParts[1]
	valueID := keyParts[2]

	s.checkOrCreateIndex(indexName, indexType)

	rawValue, err := s.client.Get().
		Index(indexName).
		Type(indexType).
		Id(valueID).
		Do(s.context)

	if fmt.Sprintf("%s", err) == "elastic: Error 404 (Not Found)" {
		return s.witness.nil(), nil
	}

	if err != nil {
		return s.witness.nil(), err
	}

	if !rawValue.Found {
		return s.witness.nil(), nil
	}

	structPtr := s.witness.allocate()
	err = json.Unmarshal(*rawValue.Source, structPtr)
	if err != nil {
		return s.witness.nil(), err
	}
	return structPtr, nil
}

// GetAll gets multiple keys from store using MultiGet.
func (s *ElasticsearchKeyValueStore) GetAll(keys []string) ([]*KeyValue, error) {
	logger.Debug("Elasticsearch GetAll: ", keys)
	s.getAllSummary.Observe(float64(len(keys)), s.witness.name)
	multiGet := s.client.MultiGet()
	for _, key := range keys {
		keyParts := strings.Split(key, "/")
		if len(keyParts) != 3 {
			logger.Panicf("invalid key: '%s'", key)
		}
		indexName := keyParts[0]
		indexType := keyParts[1]
		valueID := keyParts[2]

		s.checkOrCreateIndex(indexName, indexType)

		item := elastic.NewMultiGetItem().
			Index(indexName).
			Type(indexType).
			Id(valueID)

		multiGet.Add(item)
	}
	response, err := multiGet.Do(s.context)
	if err != nil {
		return nil, err
	}
	kvs := make([]*KeyValue, len(keys))
	for i, doc := range response.Docs {
		var structPtr interface{}
		if !doc.Found {
			structPtr = s.witness.nil()
		} else {
			structPtr = s.witness.allocate()
			err = json.Unmarshal(*doc.Source, structPtr)
			if err != nil {
				return nil, err
			}
		}
		kvs[i] = &KeyValue{keys[i], structPtr}
	}
	return kvs, nil
}

// Put updates key in store with serialized value
func (s *ElasticsearchKeyValueStore) Put(key string, structPtr interface{}) error {
	logger.Debugf("Elasticsearch Put: %s %#v", key, structPtr)
	s.witness.assert(structPtr)
	s.putCounter.Inc(s.witness.name)
	keyParts := strings.Split(key, "/")
	if len(keyParts) != 3 {
		logger.Panicf("invalid key: '%s'", key)
	}
	indexName := keyParts[0]
	indexType := keyParts[1]
	valueID := keyParts[2]

	s.checkOrCreateIndex(indexName, indexType)

	_, err := s.client.Index().
		Index(indexName).
		Type(indexType).
		Id(valueID).
		BodyJson(structPtr).
		Do(s.context)

	return err
}

// PutAll bulk executes Put operation for several kvs
func (s *ElasticsearchKeyValueStore) PutAll(kvs []*KeyValue) error {
	logger.Debugf("Elasticsearch PutAll of %d keys", len(kvs))
	s.putAllSummary.Observe(float64(len(kvs)), s.witness.name)
	if len(kvs) == 0 {
		return nil
	}
	bulk := s.client.Bulk()
	for _, kv := range kvs {
		keyParts := strings.Split(kv.Key, "/")
		if len(keyParts) != 3 {
			logger.Panicf("invalid key: '%s'", kv.Key)
		}
		indexName := keyParts[0]
		indexType := keyParts[1]
		valueID := keyParts[2]

		s.witness.assert(kv.Value)
		s.checkOrCreateIndex(indexName, indexType)

		bulk.Add(elastic.NewBulkIndexRequest().
			Index(indexName).
			Type(indexType).
			Id(valueID).
			Doc(kv.Value),
		)
	}
	response, err := bulk.Do(s.context)
	if err != nil {
		return err
	}
	if response.Errors {
		reasons := []string{}
		failed := response.Failed()
		for i, item := range failed {
			if item.Error != nil {
				reason := fmt.Sprintf("id = %s, error = %s\n", item.Id, item.Error.Reason)
				reasons = append(reasons, reason)
			}
			if i == maxBulkErrorReasons-1 {
				reason := fmt.Sprintf("(ommited %d more errors)", len(failed)-maxBulkErrorReasons)
				reasons = append(reasons, reason)
				break
			}
		}
		return errors.New(fmt.Sprintf("PutAll failed for some requests:\n%s", strings.Join(reasons, "")))
	}
	return nil
}

// Delete removes key from store
func (s *ElasticsearchKeyValueStore) Delete(key string) error {
	logger.Debug("Elasticsearch Delete: ", key)
	s.deleteCounter.Inc(s.witness.name)
	keyParts := strings.Split(key, "/")
	if len(keyParts) != 3 {
		logger.Panicf("invalid key: '%s'", key)
	}
	indexName := keyParts[0]
	indexType := keyParts[1]
	valueID := keyParts[2]

	s.checkOrCreateIndex(indexName, indexType)

	_, err := s.client.Delete().
		Index(indexName).
		Type(indexType).
		Id(valueID).
		Do(s.context)

	if err != nil && err.(*elastic.Error).Status == 404 {
		return nil
	}

	return err
}

// Flush the Elasticsearch translog to disk
func (s *ElasticsearchKeyValueStore) Flush() error {
	s.flushCounter.Inc(s.witness.name)
	logger.Info("Elasticsearch Flush...")
	_, err := s.client.Flush("_all").
		WaitIfOngoing(true).
		Do(s.context)
	logger.Info("Elasticsearch Flush complete")
	return err
}
<|endoftext|>


<commit_before>package cmd

import (
	"context"
	"fmt"
	"log"
	"os"
	"os/exec"

	"github.com/gobuffalo/buffalo/generators/assets/webpack"
	rg "github.com/gobuffalo/buffalo/generators/refresh"
	"github.com/markbates/refresh/refresh"
	"github.com/spf13/cobra"
)

// devCmd represents the dev command
var devCmd = &cobra.Command{
	Use:   "dev",
	Short: "Runs your Buffalo app in 'development' mode",
	Long: `Runs your Buffalo app in 'development' mode.
This includes rebuilding your application when files change.
This behavior can be changed in your .buffalo.dev.yml file.`,
	Run: func(c *cobra.Command, args []string) {
		defer func() {
			msg := "There was a problem starting the dev server, Please review the troubleshooting docs: %s\n"
			cause := "Unknown"
			if r := recover(); r != nil {
				if err, ok := r.(error); ok {
					cause = err.Error()
				}
			}
			fmt.Printf(msg, cause)
		}()
		os.Setenv("GO_ENV", "development")
		ctx := context.Background()
		ctx, cancelFunc := context.WithCancel(ctx)
		go func() {
			err := startDevServer(ctx)
			if err != nil {
				cancelFunc()
				log.Fatal(err)
			}
		}()
		go func() {
			err := startWebpack(ctx)
			if err != nil {
				cancelFunc()
				log.Fatal(err)
			}
		}()
		// wait for the ctx to finish
		<-ctx.Done()
	},
}

func startWebpack(ctx context.Context) error {
	cfgFile := "./webpack.config.js"
	_, err := os.Stat(cfgFile)
	if err != nil {
		// there's no webpack, so don't do anything
		return nil
	}
	cmd := exec.Command(webpack.BinPath, "--watch")
	cmd.Stdin = os.Stdin
	cmd.Stderr = os.Stderr
	cmd.Stdout = os.Stdout
	return cmd.Run()
}

func startDevServer(ctx context.Context) error {
	cfgFile := "./.buffalo.dev.yml"
	_, err := os.Stat(cfgFile)
	if err != nil {
		g, err := rg.New()
		if err != nil {
			return err
		}
		err = g.Run("./", map[string]interface{}{
			"name": "buffalo",
		})
	}
	c := &refresh.Configuration{}
	err = c.Load(cfgFile)
	if err != nil {
		return err
	}
	r := refresh.NewWithContext(c, ctx)
	return r.Start()
}

func init() {
	RootCmd.AddCommand(devCmd)
}
<commit_msg>turn off colored output on windows in dev mode. fixes #348<commit_after>package cmd

import (
	"context"
	"fmt"
	"log"
	"os"
	"os/exec"
	"runtime"

	"github.com/fatih/color"
	"github.com/gobuffalo/buffalo/generators/assets/webpack"
	rg "github.com/gobuffalo/buffalo/generators/refresh"
	"github.com/markbates/refresh/refresh"
	"github.com/spf13/cobra"
)

// devCmd represents the dev command
var devCmd = &cobra.Command{
	Use:   "dev",
	Short: "Runs your Buffalo app in 'development' mode",
	Long: `Runs your Buffalo app in 'development' mode.
This includes rebuilding your application when files change.
This behavior can be changed in your .buffalo.dev.yml file.`,
	Run: func(c *cobra.Command, args []string) {
		if runtime.GOOS == "windows" {
			color.NoColor = true
		}
		defer func() {
			msg := "There was a problem starting the dev server, Please review the troubleshooting docs: %s\n"
			cause := "Unknown"
			if r := recover(); r != nil {
				if err, ok := r.(error); ok {
					cause = err.Error()
				}
			}
			fmt.Printf(msg, cause)
		}()
		os.Setenv("GO_ENV", "development")
		ctx := context.Background()
		ctx, cancelFunc := context.WithCancel(ctx)
		go func() {
			err := startDevServer(ctx)
			if err != nil {
				cancelFunc()
				log.Fatal(err)
			}
		}()
		go func() {
			err := startWebpack(ctx)
			if err != nil {
				cancelFunc()
				log.Fatal(err)
			}
		}()
		// wait for the ctx to finish
		<-ctx.Done()
	},
}

func startWebpack(ctx context.Context) error {
	cfgFile := "./webpack.config.js"
	_, err := os.Stat(cfgFile)
	if err != nil {
		// there's no webpack, so don't do anything
		return nil
	}
	cmd := exec.Command(webpack.BinPath, "--watch")
	cmd.Stdin = os.Stdin
	cmd.Stderr = os.Stderr
	cmd.Stdout = os.Stdout
	return cmd.Run()
}

func startDevServer(ctx context.Context) error {
	cfgFile := "./.buffalo.dev.yml"
	_, err := os.Stat(cfgFile)
	if err != nil {
		g, err := rg.New()
		if err != nil {
			return err
		}
		err = g.Run("./", map[string]interface{}{
			"name": "buffalo",
		})
	}
	c := &refresh.Configuration{}
	err = c.Load(cfgFile)
	if err != nil {
		return err
	}
	r := refresh.NewWithContext(c, ctx)
	return r.Start()
}

func init() {
	RootCmd.AddCommand(devCmd)
}
<|endoftext|>


<commit_before>package sync

import (
	"encoding/hex"
	"fmt"
	"io/ioutil"
	"os"
	"path"
	"strings"

	"gopkg.in/yaml.v2"

	"github.com/deckarep/golang-set"
	"github.com/dreampuf/evernote-sdk-golang/client"
	"github.com/dreampuf/evernote-sdk-golang/notestore"
	"github.com/dreampuf/evernote-sdk-golang/types"
	"github.com/dreampuf/evernote-sdk-golang/userstore"
	"github.com/pkg/errors"
)

const minimumFetchIntervalSeconds = 15 * 60
const cacheExtension = ".yml"

// Sync local cache from evernote server
func Sync(cacheRoot string, noteCacheDirName string, resourceCacheDirName string, clientKey string, clientSecret string, developerToken string, isSandbox bool, notebookName string) error {
	cacheRoot = path.Clean(cacheRoot)
	noteCacheDir := path.Join(cacheRoot, noteCacheDirName)
	resourceCacheDir := path.Join(cacheRoot, resourceCacheDirName)

	if err := os.MkdirAll(noteCacheDir, os.ModePerm); err != nil {
		return errors.Wrapf(err, "couldn't create note cache path %v", noteCacheDir)
	}

	if err := os.MkdirAll(resourceCacheDir, os.ModePerm); err != nil {
		return errors.Wrapf(err, "couldn't create resource cache path %v", resourceCacheDir)
	}

	cli := client.NewClient(clientKey, clientSecret, getEnvironment(isSandbox))

	us, err := getUserStore(cli, isSandbox)
	if err != nil {
		return err
	}

	ns, err := getNoteStore(cli, us, &developerToken)
	if err != nil {
		return err
	}

	notUpdated, err := checkUpdate(ns, &cacheRoot, &developerToken)
	if err != nil {
		return err
	}
	if notUpdated {
		return errors.Errorf("note is not updated")
	}

	bookGUID, err := findNotebookGUID(&notebookName, cli, ns, &developerToken)
	if err != nil {
		return err
	}

	metadatas, err := findMetadatas(ns, bookGUID, &developerToken)
	if err != nil {
		return err
	}

	cachedIds, err := createCachedNoteIDMap(&noteCacheDir)
	if err != nil {
		return err
	}

	for _, note := range metadatas.GetNotes() {
		fmt.Printf("processing %v\n", note.GUID)

		noteCachePath := path.Join(noteCacheDir, string(note.GUID)+cacheExtension)
		cachedNote, err := readCachedNote(&noteCachePath, &note.GUID)
		if err != nil {
			return errors.Wrapf(err, "can't read cached note")
		}

		if cachedNote == nil || cachedNote.UpdateSequenceNum == nil || note.UpdateSequenceNum == nil || *cachedNote.UpdateSequenceNum != *note.UpdateSequenceNum {
			note, err := ns.GetNote(developerToken, note.GUID, true, false, false, false)
			if err != nil {
				return errors.Wrapf(err, "can't get note %v", *note.GUID)
			}

			tags, err := ns.GetNoteTagNames(developerToken, *note.GUID)
			if err != nil {
				return errors.Wrapf(err, "can't get note tags %v", *note.GUID)
			}
			note.TagNames = tags

			fmt.Printf("downloaded %v[%v]\n", *note.Title, *note.GUID)

			writeCachedNoteToFile(&noteCachePath, note)
			saveResources(&resourceCacheDir, cachedNote, note, ns, &developerToken)
		}

		cachedIds.Remove(note.GUID)
	}

	for _, id := range cachedIds.ToSlice() {
		guid := id.(string)
		os.Remove(noteCacheDir + guid + cacheExtension)
	}

	return nil
}

func getEnvironment(isSandbox bool) client.EnvironmentType {
	if isSandbox {
		return client.SANDBOX
	}
	return client.PRODUCTION
}

func getUserStore(cli *client.EvernoteClient, isSandbox bool) (*userstore.UserStoreClient, error) {
	us, err := cli.GetUserStore()
	if err != nil {
		return nil, errors.Wrapf(err, "can't get user store (environment: %v)", getEnvironment(isSandbox))
	}

	versionOk, err := us.CheckVersion("chienote", userstore.EDAM_VERSION_MAJOR, userstore.EDAM_VERSION_MINOR)
	if !versionOk {
		return nil, errors.New("user store isn't correct version")
	}
	if err != nil {
		return nil, errors.Wrap(err, "error occured on checking user store version")
	}

	return us, nil
}

func getNoteStore(cli *client.EvernoteClient, us *userstore.UserStoreClient, developerToken *string) (*notestore.NoteStoreClient, error) {
	url, err := us.GetNoteStoreUrl(*developerToken)
	if err != nil {
		return nil, errors.Wrap(err, "can't get note store url")
	}
	if url == "" {
		return nil, errors.New("empty notestore url received")
	}

	ns, err := cli.GetNoteStoreWithURL(url)
	if err != nil {
		return nil, errors.Wrap(err, "can't get note store")
	}

	return ns, nil
}

func findNotebookGUID(notebookName *string, cli *client.EvernoteClient, ns *notestore.NoteStoreClient, developerToken *string) (notebookGUID *types.GUID, err error) {
	notebooks, err := ns.ListNotebooks(*developerToken)
	if err != nil {
		return nil, errors.Wrap(err, "can't get notebook list")
	}

	for _, book := range notebooks {
		if *book.Name == *notebookName {
			notebookGUID = book.GUID
			break
		}
	}

	if notebookGUID == nil || *notebookGUID == "" {
		return nil, errors.Errorf("can't get notebook %v", *notebookName)
	}

	return notebookGUID, nil
}

func findMetadatas(ns *notestore.NoteStoreClient, bookGUID *types.GUID, developerToken *string) (metadatas *notestore.NotesMetadataList, err error) {
	ascending := false
	filter := &notestore.NoteFilter{NotebookGuid: bookGUID, Ascending: &ascending}

	var resultSpec notestore.NotesMetadataResultSpec
	metadatas, err = ns.FindNotesMetadata(*developerToken, filter, 0, 1000, &resultSpec)
	if err != nil {
		return nil, errors.Wrap(err, "couldn't find notes")
	}
	return metadatas, err
}

func createCachedNoteIDMap(noteCachePath *string) (mapset.Set, error) {
	cachedIDs := mapset.NewSet()
	cacheFileInfos, err := ioutil.ReadDir(*noteCachePath)
	if err != nil {
		return nil, errors.Wrapf(err, "can't read cache directory %v", *noteCachePath)
	}

	for _, cacheFileInfo := range cacheFileInfos {
		if strings.HasSuffix(cacheFileInfo.Name(), cacheExtension) {
			cachedIDs.Add(strings.Replace(cacheFileInfo.Name(), cacheExtension, "", 1))
		}
	}

	return cachedIDs, nil
}

func readCachedNoteFile(noteCachePath *string) (*types.Note, error) {
	cacheReader, err := os.OpenFile(*noteCachePath, os.O_RDONLY, os.ModePerm)
	if err != nil {
		return nil, errors.Wrapf(err, "can't open cached note file %v", *noteCachePath)
	}
	defer cacheReader.Close()

	cachedYAML, err := ioutil.ReadAll(cacheReader)
	if err != nil {
		return nil, errors.Wrapf(err, "can't read cached note file %v", *noteCachePath)
	}

	cachedNote := &types.Note{}
	if err := yaml.Unmarshal(cachedYAML, cachedNote); err != nil {
		return nil, errors.Wrapf(err, "can't parse cached yaml file %v", *noteCachePath)
	}

	return cachedNote, nil
}

func readCachedNote(noteCachePath *string, noteGUID *types.GUID) (*types.Note, error) {
	var cachedNote *types.Note
	if _, err := os.Stat(*noteCachePath); err == nil {
		cachedNote, err = readCachedNoteFile(noteCachePath)
		if err != nil {
			return nil, errors.Wrapf(err, "error occurs when reading cached note file %v", *noteCachePath)
		}
	}

	return cachedNote, nil
}

func writeCachedNoteToFile(cachePath *string, note *types.Note) error {
	yamlBytes, err := yaml.Marshal(note)
	if err != nil {
		return errors.Wrapf(err, "can't marshal note as YAML %v", *cachePath)
	}

	if err := ioutil.WriteFile(*cachePath, yamlBytes, os.ModePerm); err != nil {
		return errors.Wrapf(err, "can't write marshaled note to %v", *cachePath)
	}

	return nil
}

func saveResources(resourceCacheDir *string, cachedNote *types.Note, receivedNote *types.Note, ns *notestore.NoteStoreClient, developerToken *string) error {
	localResourceMap := map[types.GUID]int32{}
	if cachedNote != nil {
		for _, cachedResource := range cachedNote.Resources {
			localResourceMap[*cachedResource.GUID] = *cachedResource.UpdateSequenceNum
		}
	}

	for _, resource := range receivedNote.Resources {
		fetchingNeeded := false
		cachedUpdateNum, exists := localResourceMap[*resource.GUID]
		if exists {

			if cachedUpdateNum != *resource.UpdateSequenceNum {
				fetchingNeeded = true
			}
			delete(localResourceMap, *resource.GUID)
		} else {
			fetchingNeeded = true
		}

		if fetchingNeeded {
			resourceWithBytes, err := ns.GetResource(*developerToken, *resource.GUID, true, false, true, false)
			if err != nil {
				return errors.Wrapf(err, "can't get resource %v", *resource.Attributes.FileName)
			}

			var resourceFileName string
			if resourceWithBytes.Attributes.FileName == nil {
				// https://dev.evernote.com/doc/articles/resources.php#downloading
				var extension string
				switch *resourceWithBytes.Mime {
				case "image/gif":
					extension = ".gif"
				case "image/jpeg":
					extension = ".jpg"
				case "image/png":
					extension = ".png"
				case "audio/wav":
					extension = ".wav"
				case "audio/mpeg":
					extension = ".mp3"
				case "audio/amr":
					extension = ".amr"
				case "audio/pdf":
					extension = ".pdf"
				}
				resourceFileName = hex.EncodeToString(resourceWithBytes.Data.BodyHash) + extension
			} else {
				resourceFileName = hex.EncodeToString(resourceWithBytes.Data.BodyHash) + "-" + *resourceWithBytes.Attributes.FileName
			}

			p := path.Join(*resourceCacheDir, resourceFileName)
			ioutil.WriteFile(p, resourceWithBytes.Data.Body, os.ModePerm)
			fmt.Println("write resource to " + p)
		} else {
			fmt.Printf("not updated %v\n", *resource.Attributes.FileName)
		}
	}

	return nil
}

func checkUpdate(ns *notestore.NoteStoreClient, cacheDir *string, developerToken *string) (notUpdated bool, err error) {
	syncState, err := ns.GetSyncState(*developerToken)
	if err != nil {
		return false, errors.Wrap(err, "can't get sync state")
	}

	syncStatePath := path.Join(*cacheDir, "sync_state.yml")

	prevStateBytes, err := ioutil.ReadFile(syncStatePath)
	if err == nil {
		prevState := &notestore.SyncState{Uploaded: new(int64)}
		if err := yaml.Unmarshal(prevStateBytes, prevState); err == nil {
			if prevState.UpdateCount == syncState.UpdateCount {
				return true, nil
			}

			if syncState.CurrentTime-prevState.CurrentTime < minimumFetchIntervalSeconds*1000 {
				return false, errors.Errorf("you should wait at least %v seconds from last update", minimumFetchIntervalSeconds)
			}
		}
	}

	stateBytes, err := yaml.Marshal(syncState)
	if err != nil {
		return false, errors.Wrapf(err, "can't marshal sync state")
	}

	if err := ioutil.WriteFile(syncStatePath, stateBytes, os.ModePerm); err != nil {
		return false, errors.Wrapf(err, "can't write sync state %v", syncStatePath)
	}

	return false, nil
}
<commit_msg>Fix panic on showing message with filename<commit_after>package sync

import (
	"encoding/hex"
	"fmt"
	"io/ioutil"
	"os"
	"path"
	"strings"

	"gopkg.in/yaml.v2"

	"github.com/deckarep/golang-set"
	"github.com/dreampuf/evernote-sdk-golang/client"
	"github.com/dreampuf/evernote-sdk-golang/notestore"
	"github.com/dreampuf/evernote-sdk-golang/types"
	"github.com/dreampuf/evernote-sdk-golang/userstore"
	"github.com/pkg/errors"
)

const minimumFetchIntervalSeconds = 15 * 60
const cacheExtension = ".yml"

// Sync local cache from evernote server
func Sync(cacheRoot string, noteCacheDirName string, resourceCacheDirName string, clientKey string, clientSecret string, developerToken string, isSandbox bool, notebookName string) error {
	cacheRoot = path.Clean(cacheRoot)
	noteCacheDir := path.Join(cacheRoot, noteCacheDirName)
	resourceCacheDir := path.Join(cacheRoot, resourceCacheDirName)

	if err := os.MkdirAll(noteCacheDir, os.ModePerm); err != nil {
		return errors.Wrapf(err, "couldn't create note cache path %v", noteCacheDir)
	}

	if err := os.MkdirAll(resourceCacheDir, os.ModePerm); err != nil {
		return errors.Wrapf(err, "couldn't create resource cache path %v", resourceCacheDir)
	}

	cli := client.NewClient(clientKey, clientSecret, getEnvironment(isSandbox))

	us, err := getUserStore(cli, isSandbox)
	if err != nil {
		return err
	}

	ns, err := getNoteStore(cli, us, &developerToken)
	if err != nil {
		return err
	}

	notUpdated, err := checkUpdate(ns, &cacheRoot, &developerToken)
	if err != nil {
		return err
	}
	if notUpdated {
		return errors.Errorf("note is not updated")
	}

	bookGUID, err := findNotebookGUID(&notebookName, cli, ns, &developerToken)
	if err != nil {
		return err
	}

	metadatas, err := findMetadatas(ns, bookGUID, &developerToken)
	if err != nil {
		return err
	}

	cachedIds, err := createCachedNoteIDMap(&noteCacheDir)
	if err != nil {
		return err
	}

	for _, note := range metadatas.GetNotes() {
		fmt.Printf("processing %v\n", note.GUID)

		noteCachePath := path.Join(noteCacheDir, string(note.GUID)+cacheExtension)
		cachedNote, err := readCachedNote(&noteCachePath, &note.GUID)
		if err != nil {
			return errors.Wrapf(err, "can't read cached note")
		}

		if cachedNote == nil || cachedNote.UpdateSequenceNum == nil || note.UpdateSequenceNum == nil || *cachedNote.UpdateSequenceNum != *note.UpdateSequenceNum {
			note, err := ns.GetNote(developerToken, note.GUID, true, false, false, false)
			if err != nil {
				return errors.Wrapf(err, "can't get note %v", *note.GUID)
			}

			tags, err := ns.GetNoteTagNames(developerToken, *note.GUID)
			if err != nil {
				return errors.Wrapf(err, "can't get note tags %v", *note.GUID)
			}
			note.TagNames = tags

			fmt.Printf("downloaded %v[%v]\n", *note.Title, *note.GUID)

			writeCachedNoteToFile(&noteCachePath, note)
			saveResources(&resourceCacheDir, cachedNote, note, ns, &developerToken)
		}

		cachedIds.Remove(note.GUID)
	}

	for _, id := range cachedIds.ToSlice() {
		guid := id.(string)
		os.Remove(noteCacheDir + guid + cacheExtension)
	}

	return nil
}

func getEnvironment(isSandbox bool) client.EnvironmentType {
	if isSandbox {
		return client.SANDBOX
	}
	return client.PRODUCTION
}

func getUserStore(cli *client.EvernoteClient, isSandbox bool) (*userstore.UserStoreClient, error) {
	us, err := cli.GetUserStore()
	if err != nil {
		return nil, errors.Wrapf(err, "can't get user store (environment: %v)", getEnvironment(isSandbox))
	}

	versionOk, err := us.CheckVersion("chienote", userstore.EDAM_VERSION_MAJOR, userstore.EDAM_VERSION_MINOR)
	if !versionOk {
		return nil, errors.New("user store isn't correct version")
	}
	if err != nil {
		return nil, errors.Wrap(err, "error occured on checking user store version")
	}

	return us, nil
}

func getNoteStore(cli *client.EvernoteClient, us *userstore.UserStoreClient, developerToken *string) (*notestore.NoteStoreClient, error) {
	url, err := us.GetNoteStoreUrl(*developerToken)
	if err != nil {
		return nil, errors.Wrap(err, "can't get note store url")
	}
	if url == "" {
		return nil, errors.New("empty notestore url received")
	}

	ns, err := cli.GetNoteStoreWithURL(url)
	if err != nil {
		return nil, errors.Wrap(err, "can't get note store")
	}

	return ns, nil
}

func findNotebookGUID(notebookName *string, cli *client.EvernoteClient, ns *notestore.NoteStoreClient, developerToken *string) (notebookGUID *types.GUID, err error) {
	notebooks, err := ns.ListNotebooks(*developerToken)
	if err != nil {
		return nil, errors.Wrap(err, "can't get notebook list")
	}

	for _, book := range notebooks {
		if *book.Name == *notebookName {
			notebookGUID = book.GUID
			break
		}
	}

	if notebookGUID == nil || *notebookGUID == "" {
		return nil, errors.Errorf("can't get notebook %v", *notebookName)
	}

	return notebookGUID, nil
}

func findMetadatas(ns *notestore.NoteStoreClient, bookGUID *types.GUID, developerToken *string) (metadatas *notestore.NotesMetadataList, err error) {
	ascending := false
	filter := &notestore.NoteFilter{NotebookGuid: bookGUID, Ascending: &ascending}

	var resultSpec notestore.NotesMetadataResultSpec
	metadatas, err = ns.FindNotesMetadata(*developerToken, filter, 0, 1000, &resultSpec)
	if err != nil {
		return nil, errors.Wrap(err, "couldn't find notes")
	}
	return metadatas, err
}

func createCachedNoteIDMap(noteCachePath *string) (mapset.Set, error) {
	cachedIDs := mapset.NewSet()
	cacheFileInfos, err := ioutil.ReadDir(*noteCachePath)
	if err != nil {
		return nil, errors.Wrapf(err, "can't read cache directory %v", *noteCachePath)
	}

	for _, cacheFileInfo := range cacheFileInfos {
		if strings.HasSuffix(cacheFileInfo.Name(), cacheExtension) {
			cachedIDs.Add(strings.Replace(cacheFileInfo.Name(), cacheExtension, "", 1))
		}
	}

	return cachedIDs, nil
}

func readCachedNoteFile(noteCachePath *string) (*types.Note, error) {
	cacheReader, err := os.OpenFile(*noteCachePath, os.O_RDONLY, os.ModePerm)
	if err != nil {
		return nil, errors.Wrapf(err, "can't open cached note file %v", *noteCachePath)
	}
	defer cacheReader.Close()

	cachedYAML, err := ioutil.ReadAll(cacheReader)
	if err != nil {
		return nil, errors.Wrapf(err, "can't read cached note file %v", *noteCachePath)
	}

	cachedNote := &types.Note{}
	if err := yaml.Unmarshal(cachedYAML, cachedNote); err != nil {
		return nil, errors.Wrapf(err, "can't parse cached yaml file %v", *noteCachePath)
	}

	return cachedNote, nil
}

func readCachedNote(noteCachePath *string, noteGUID *types.GUID) (*types.Note, error) {
	var cachedNote *types.Note
	if _, err := os.Stat(*noteCachePath); err == nil {
		cachedNote, err = readCachedNoteFile(noteCachePath)
		if err != nil {
			return nil, errors.Wrapf(err, "error occurs when reading cached note file %v", *noteCachePath)
		}
	}

	return cachedNote, nil
}

func writeCachedNoteToFile(cachePath *string, note *types.Note) error {
	yamlBytes, err := yaml.Marshal(note)
	if err != nil {
		return errors.Wrapf(err, "can't marshal note as YAML %v", *cachePath)
	}

	if err := ioutil.WriteFile(*cachePath, yamlBytes, os.ModePerm); err != nil {
		return errors.Wrapf(err, "can't write marshaled note to %v", *cachePath)
	}

	return nil
}

func saveResources(resourceCacheDir *string, cachedNote *types.Note, receivedNote *types.Note, ns *notestore.NoteStoreClient, developerToken *string) error {
	localResourceMap := map[types.GUID]int32{}
	if cachedNote != nil {
		for _, cachedResource := range cachedNote.Resources {
			localResourceMap[*cachedResource.GUID] = *cachedResource.UpdateSequenceNum
		}
	}

	for _, resource := range receivedNote.Resources {
		fetchingNeeded := false
		cachedUpdateNum, exists := localResourceMap[*resource.GUID]
		if exists {

			if cachedUpdateNum != *resource.UpdateSequenceNum {
				fetchingNeeded = true
			}
			delete(localResourceMap, *resource.GUID)
		} else {
			fetchingNeeded = true
		}

		if fetchingNeeded {
			resourceWithBytes, err := ns.GetResource(*developerToken, *resource.GUID, true, false, true, false)
			if err != nil {
				return errors.Wrapf(err, "can't get resource %v", *resource.GUID)
			}

			var resourceFileName string
			if resourceWithBytes.Attributes.FileName == nil {
				// https://dev.evernote.com/doc/articles/resources.php#downloading
				var extension string
				switch *resourceWithBytes.Mime {
				case "image/gif":
					extension = ".gif"
				case "image/jpeg":
					extension = ".jpg"
				case "image/png":
					extension = ".png"
				case "audio/wav":
					extension = ".wav"
				case "audio/mpeg":
					extension = ".mp3"
				case "audio/amr":
					extension = ".amr"
				case "audio/pdf":
					extension = ".pdf"
				}
				resourceFileName = hex.EncodeToString(resourceWithBytes.Data.BodyHash) + extension
			} else {
				resourceFileName = hex.EncodeToString(resourceWithBytes.Data.BodyHash) + "-" + *resourceWithBytes.Attributes.FileName
			}

			p := path.Join(*resourceCacheDir, resourceFileName)
			ioutil.WriteFile(p, resourceWithBytes.Data.Body, os.ModePerm)
			fmt.Println("write resource to " + p)
		} else {
			fmt.Printf("not updated %v\n", *resource.GUID)
		}
	}

	return nil
}

func checkUpdate(ns *notestore.NoteStoreClient, cacheDir *string, developerToken *string) (notUpdated bool, err error) {
	syncState, err := ns.GetSyncState(*developerToken)
	if err != nil {
		return false, errors.Wrap(err, "can't get sync state")
	}

	syncStatePath := path.Join(*cacheDir, "sync_state.yml")

	prevStateBytes, err := ioutil.ReadFile(syncStatePath)
	if err == nil {
		prevState := &notestore.SyncState{Uploaded: new(int64)}
		if err := yaml.Unmarshal(prevStateBytes, prevState); err == nil {
			if prevState.UpdateCount == syncState.UpdateCount {
				return true, nil
			}

			if syncState.CurrentTime-prevState.CurrentTime < minimumFetchIntervalSeconds*1000 {
				return false, errors.Errorf("you should wait at least %v seconds from last update", minimumFetchIntervalSeconds)
			}
		}
	}

	stateBytes, err := yaml.Marshal(syncState)
	if err != nil {
		return false, errors.Wrapf(err, "can't marshal sync state")
	}

	if err := ioutil.WriteFile(syncStatePath, stateBytes, os.ModePerm); err != nil {
		return false, errors.Wrapf(err, "can't write sync state %v", syncStatePath)
	}

	return false, nil
}
<|endoftext|>