<commit_before>package slack

import (
	"bytes"
	"context"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"mime/multipart"
	"net/http"
	"net/http/httputil"
	"net/url"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"time"
)

// HTTPRequester defines the minimal interface needed for an http.Client to be implemented.
//
// Use it in conjunction with the SetHTTPClient function to allow for other capabilities
// like a tracing http.Client
type HTTPRequester interface {
	Do(*http.Request) (*http.Response, error)
}

var customHTTPClient HTTPRequester

// Default duration for rate limiting if Retry-After header does not exist/parse
const defaultRetrySeconds int64 = 60

// HTTPClient sets a custom http.Client
// deprecated: in favor of SetHTTPClient()
var HTTPClient = &http.Client{}

type WebResponse struct {
	Ok    bool      `json:"ok"`
	Error *WebError `json:"error"`
}

type WebError string

func (s WebError) Error() string {
	return string(s)
}

type RateLimitedError struct {
	RetryAfter time.Duration
}

func (e *RateLimitedError) Error() string {
	return fmt.Sprintf("Slack rate limit exceeded, retry after %s", e.RetryAfter)
}

func fileUploadReq(ctx context.Context, path, fieldname, filename string, values url.Values, r io.Reader) (*http.Request, error) {
	body := &bytes.Buffer{}
	wr := multipart.NewWriter(body)

	ioWriter, err := wr.CreateFormFile(fieldname, filename)
	if err != nil {
		wr.Close()
		return nil, err
	}
	_, err = io.Copy(ioWriter, r)
	if err != nil {
		wr.Close()
		return nil, err
	}
	// Close the multipart writer or the footer won't be written
	wr.Close()
	req, err := http.NewRequest("POST", path, body)
	req = req.WithContext(ctx)
	if err != nil {
		return nil, err
	}
	req.Header.Add("Content-Type", wr.FormDataContentType())
	req.URL.RawQuery = (values).Encode()
	return req, nil
}

func parseResponseBody(body io.ReadCloser, intf *interface{}, debug bool) error {
	response, err := ioutil.ReadAll(body)
	if err != nil {
		return err
	}

	// FIXME: will be api.Debugf
	if debug {
		logger.Printf("parseResponseBody: %s\n", string(response))
	}

	return json.Unmarshal(response, &intf)
}

func postLocalWithMultipartResponse(ctx context.Context, path, fpath, fieldname string, values url.Values, intf interface{}, debug bool) error {
	fullpath, err := filepath.Abs(fpath)
	if err != nil {
		return err
	}
	file, err := os.Open(fullpath)
	if err != nil {
		return err
	}
	defer file.Close()
	return postWithMultipartResponse(ctx, path, filepath.Base(fpath), fieldname, values, file, intf, debug)
}

func postWithMultipartResponse(ctx context.Context, path, name, fieldname string, values url.Values, r io.Reader, intf interface{}, debug bool) error {
	req, err := fileUploadReq(ctx, SLACK_API+path, fieldname, name, values, r)
	if err != nil {
		return err
	}
	req = req.WithContext(ctx)
	resp, err := getHTTPClient().Do(req)
	if err != nil {
		return err
	}
	defer resp.Body.Close()

	if resp.StatusCode == http.StatusTooManyRequests {
		retry, err := strconv.ParseInt(resp.Header.Get("Retry-After"), 10, 64)
		if err != nil {
			retry = defaultRetrySeconds
		}
		return &RateLimitedError{time.Duration(retry) * time.Second}
	}

	// Slack seems to send an HTML body along with 5xx error codes. Don't parse it.
	if resp.StatusCode != 200 {
		logResponse(resp, debug)
		return fmt.Errorf("Slack server error: %s.", resp.Status)
	}

	return parseResponseBody(resp.Body, &intf, debug)
}

func postForm(ctx context.Context, endpoint string, values url.Values, intf interface{}, debug bool) error {
	reqBody := strings.NewReader(values.Encode())
	req, err := http.NewRequest("POST", endpoint, reqBody)
	if err != nil {
		return err
	}
	req.Header.Set("Content-Type", "application/x-www-form-urlencoded")

	req = req.WithContext(ctx)
	resp, err := getHTTPClient().Do(req)
	if err != nil {
		return err
	}
	defer resp.Body.Close()

	if resp.StatusCode == http.StatusTooManyRequests {
		retry, err := strconv.ParseInt(resp.Header.Get("Retry-After"), 10, 64)
		if err != nil {
			retry = defaultRetrySeconds
		}
		return &RateLimitedError{time.Duration(retry) * time.Second}
	}

	// Slack seems to send an HTML body along with 5xx error codes. Don't parse it.
	if resp.StatusCode != 200 {
		logResponse(resp, debug)
		return fmt.Errorf("Slack server error: %s.", resp.Status)
	}

	return parseResponseBody(resp.Body, &intf, debug)
}

func post(ctx context.Context, path string, values url.Values, intf interface{}, debug bool) error {
	return postForm(ctx, SLACK_API+path, values, intf, debug)
}

func parseAdminResponse(ctx context.Context, method string, teamName string, values url.Values, intf interface{}, debug bool) error {
	endpoint := fmt.Sprintf(SLACK_WEB_API_FORMAT, teamName, method, time.Now().Unix())
	return postForm(ctx, endpoint, values, intf, debug)
}

func logResponse(resp *http.Response, debug bool) error {
	if debug {
		text, err := httputil.DumpResponse(resp, true)
		if err != nil {
			return err
		}

		logger.Print(string(text))
	}

	return nil
}

func getHTTPClient() HTTPRequester {
	if customHTTPClient != nil {
		return customHTTPClient
	}

	return HTTPClient
}

// SetHTTPClient allows you to specify a custom http.Client
// Use this instead of the package level HTTPClient variable if you want to use a custom client like the
// Stackdriver Trace HTTPClient https://godoc.org/cloud.google.com/go/trace#HTTPClient
func SetHTTPClient(client HTTPRequester) {
	customHTTPClient = client
}
<commit_msg>Remove default retry value and return error.<commit_after>package slack

import (
	"bytes"
	"context"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"mime/multipart"
	"net/http"
	"net/http/httputil"
	"net/url"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"time"
)

// HTTPRequester defines the minimal interface needed for an http.Client to be implemented.
//
// Use it in conjunction with the SetHTTPClient function to allow for other capabilities
// like a tracing http.Client
type HTTPRequester interface {
	Do(*http.Request) (*http.Response, error)
}

var customHTTPClient HTTPRequester

// HTTPClient sets a custom http.Client
// deprecated: in favor of SetHTTPClient()
var HTTPClient = &http.Client{}

type WebResponse struct {
	Ok    bool      `json:"ok"`
	Error *WebError `json:"error"`
}

type WebError string

func (s WebError) Error() string {
	return string(s)
}

type RateLimitedError struct {
	RetryAfter time.Duration
}

func (e *RateLimitedError) Error() string {
	return fmt.Sprintf("Slack rate limit exceeded, retry after %s", e.RetryAfter)
}

func fileUploadReq(ctx context.Context, path, fieldname, filename string, values url.Values, r io.Reader) (*http.Request, error) {
	body := &bytes.Buffer{}
	wr := multipart.NewWriter(body)

	ioWriter, err := wr.CreateFormFile(fieldname, filename)
	if err != nil {
		wr.Close()
		return nil, err
	}
	_, err = io.Copy(ioWriter, r)
	if err != nil {
		wr.Close()
		return nil, err
	}
	// Close the multipart writer or the footer won't be written
	wr.Close()
	req, err := http.NewRequest("POST", path, body)
	req = req.WithContext(ctx)
	if err != nil {
		return nil, err
	}
	req.Header.Add("Content-Type", wr.FormDataContentType())
	req.URL.RawQuery = (values).Encode()
	return req, nil
}

func parseResponseBody(body io.ReadCloser, intf *interface{}, debug bool) error {
	response, err := ioutil.ReadAll(body)
	if err != nil {
		return err
	}

	// FIXME: will be api.Debugf
	if debug {
		logger.Printf("parseResponseBody: %s\n", string(response))
	}

	return json.Unmarshal(response, &intf)
}

func postLocalWithMultipartResponse(ctx context.Context, path, fpath, fieldname string, values url.Values, intf interface{}, debug bool) error {
	fullpath, err := filepath.Abs(fpath)
	if err != nil {
		return err
	}
	file, err := os.Open(fullpath)
	if err != nil {
		return err
	}
	defer file.Close()
	return postWithMultipartResponse(ctx, path, filepath.Base(fpath), fieldname, values, file, intf, debug)
}

func postWithMultipartResponse(ctx context.Context, path, name, fieldname string, values url.Values, r io.Reader, intf interface{}, debug bool) error {
	req, err := fileUploadReq(ctx, SLACK_API+path, fieldname, name, values, r)
	if err != nil {
		return err
	}
	req = req.WithContext(ctx)
	resp, err := getHTTPClient().Do(req)
	if err != nil {
		return err
	}
	defer resp.Body.Close()

	if resp.StatusCode == http.StatusTooManyRequests {
		retry, err := strconv.ParseInt(resp.Header.Get("Retry-After"), 10, 64)
		if err != nil {
			return err
		}
		return &RateLimitedError{time.Duration(retry) * time.Second}
	}

	// Slack seems to send an HTML body along with 5xx error codes. Don't parse it.
	if resp.StatusCode != 200 {
		logResponse(resp, debug)
		return fmt.Errorf("Slack server error: %s.", resp.Status)
	}

	return parseResponseBody(resp.Body, &intf, debug)
}

func postForm(ctx context.Context, endpoint string, values url.Values, intf interface{}, debug bool) error {
	reqBody := strings.NewReader(values.Encode())
	req, err := http.NewRequest("POST", endpoint, reqBody)
	if err != nil {
		return err
	}
	req.Header.Set("Content-Type", "application/x-www-form-urlencoded")

	req = req.WithContext(ctx)
	resp, err := getHTTPClient().Do(req)
	if err != nil {
		return err
	}
	defer resp.Body.Close()

	if resp.StatusCode == http.StatusTooManyRequests {
		retry, err := strconv.ParseInt(resp.Header.Get("Retry-After"), 10, 64)
		if err != nil {
			return err
		}
		return &RateLimitedError{time.Duration(retry) * time.Second}
	}

	// Slack seems to send an HTML body along with 5xx error codes. Don't parse it.
	if resp.StatusCode != 200 {
		logResponse(resp, debug)
		return fmt.Errorf("Slack server error: %s.", resp.Status)
	}

	return parseResponseBody(resp.Body, &intf, debug)
}

func post(ctx context.Context, path string, values url.Values, intf interface{}, debug bool) error {
	return postForm(ctx, SLACK_API+path, values, intf, debug)
}

func parseAdminResponse(ctx context.Context, method string, teamName string, values url.Values, intf interface{}, debug bool) error {
	endpoint := fmt.Sprintf(SLACK_WEB_API_FORMAT, teamName, method, time.Now().Unix())
	return postForm(ctx, endpoint, values, intf, debug)
}

func logResponse(resp *http.Response, debug bool) error {
	if debug {
		text, err := httputil.DumpResponse(resp, true)
		if err != nil {
			return err
		}

		logger.Print(string(text))
	}

	return nil
}

func getHTTPClient() HTTPRequester {
	if customHTTPClient != nil {
		return customHTTPClient
	}

	return HTTPClient
}

// SetHTTPClient allows you to specify a custom http.Client
// Use this instead of the package level HTTPClient variable if you want to use a custom client like the
// Stackdriver Trace HTTPClient https://godoc.org/cloud.google.com/go/trace#HTTPClient
func SetHTTPClient(client HTTPRequester) {
	customHTTPClient = client
}
<|endoftext|>


<commit_before>package dht

import (
	"fmt"
	"hash/fnv"
	"net"

	"github.com/anacrolix/missinggo/v2"
	"github.com/lukechampine/stm/stmutil"

	"github.com/anacrolix/dht/v2/krpc"
)

func mustListen(addr string) net.PacketConn {
	ret, err := net.ListenPacket("udp", addr)
	if err != nil {
		panic(err)
	}
	return ret
}

func addrResolver(addr string) func() ([]Addr, error) {
	return func() ([]Addr, error) {
		ua, err := net.ResolveUDPAddr("udp", addr)
		return []Addr{NewAddr(ua)}, err
	}
}

type addrMaybeId struct {
	Addr krpc.NodeAddr
	Id   *int160
}

func (me addrMaybeId) String() string {
	if me.Id == nil {
		return fmt.Sprintf("unknown id at %s", me.Addr)
	} else {
		return fmt.Sprintf("%x at %v", *me.Id, me.Addr)
	}
}

func nodesByDistance(target int160) stmutil.Settish {
	return stmutil.NewSortedSet(func(_l, _r interface{}) bool {
		var ml missinggo.MultiLess
		l := _l.(addrMaybeId)
		r := _r.(addrMaybeId)
		ml.NextBool(r.Id == nil, l.Id == nil)
		if l.Id != nil && r.Id != nil {
			d := distance(*l.Id, target).Cmp(distance(*r.Id, target))
			ml.StrictNext(d == 0, d < 0)
		}
		hashString := func(s string) uint64 {
			h := fnv.New64a()
			h.Write([]byte(s))
			return h.Sum64()
		}
		lh := hashString(l.Addr.String())
		rh := hashString(r.Addr.String())
		ml.StrictNext(lh == rh, lh < rh)
		//ml.StrictNext(l.Addr.String() == r.Addr.String(), l.Addr.String() < r.Addr.String())
		return ml.Less()
	})
}
<commit_msg>Add TODO about using bytes/hash<commit_after>package dht

import (
	"fmt"
	"hash/fnv"
	"net"

	"github.com/anacrolix/missinggo/v2"
	"github.com/lukechampine/stm/stmutil"

	"github.com/anacrolix/dht/v2/krpc"
)

func mustListen(addr string) net.PacketConn {
	ret, err := net.ListenPacket("udp", addr)
	if err != nil {
		panic(err)
	}
	return ret
}

func addrResolver(addr string) func() ([]Addr, error) {
	return func() ([]Addr, error) {
		ua, err := net.ResolveUDPAddr("udp", addr)
		return []Addr{NewAddr(ua)}, err
	}
}

type addrMaybeId struct {
	Addr krpc.NodeAddr
	Id   *int160
}

func (me addrMaybeId) String() string {
	if me.Id == nil {
		return fmt.Sprintf("unknown id at %s", me.Addr)
	} else {
		return fmt.Sprintf("%x at %v", *me.Id, me.Addr)
	}
}

func nodesByDistance(target int160) stmutil.Settish {
	return stmutil.NewSortedSet(func(_l, _r interface{}) bool {
		var ml missinggo.MultiLess
		l := _l.(addrMaybeId)
		r := _r.(addrMaybeId)
		ml.NextBool(r.Id == nil, l.Id == nil)
		if l.Id != nil && r.Id != nil {
			d := distance(*l.Id, target).Cmp(distance(*r.Id, target))
			ml.StrictNext(d == 0, d < 0)
		}
		// TODO: Use bytes/hash when it's available (go1.14?), and have a unique seed for each
		// instance.
		hashString := func(s string) uint64 {
			h := fnv.New64a()
			h.Write([]byte(s))
			return h.Sum64()
		}
		lh := hashString(l.Addr.String())
		rh := hashString(r.Addr.String())
		ml.StrictNext(lh == rh, lh < rh)
		//ml.StrictNext(l.Addr.String() == r.Addr.String(), l.Addr.String() < r.Addr.String())
		return ml.Less()
	})
}
<|endoftext|>


<commit_before>// Copyright 2013, Christian Vozar
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package hipchat

import (
	"encoding/json"
	"errors"
	"fmt"
	"github.com/mozilla-services/heka/message"
	. "github.com/mozilla-services/heka/pipeline"
	"io/ioutil"
	"net/http"
	"net/url"
)

type HipchatOutput struct {
	conf   *HipchatOutputConfig
	url    string
	format string
}

// Hipchat Output config struct
type HipchatOutputConfig struct {
	// Outputs the payload attribute in the HipChat message vs a full JSON message dump
	PayloadOnly bool `toml:"payload_only"`
	// Url for HttpInput to GET.
	AuthToken string `toml:"auth_token"`
	// Required. ID or name of the room.
	RoomId string `toml:"room_id"`
	// Required. Name the message will appear be sent. Must be less than 15
	// characters long. May contain letters, numbers, -, _, and spaces.
	From string
	// Whether or not this message should trigger a notification for people
	// in the room (change the tab color, play a sound, etc).
	// Each recipient's notification preferences are taken into account.
	// Default is false
	Notify bool
	// Background color for message.
	// One of "yellow", "red", "green", "purple", "gray", or "random".
	// Default is gray
	Color string
}

func (ho *HipchatOutput) ConfigStruct() interface{} {
	return &HipchatOutputConfig{
		PayloadOnly: true,
		From:        "Heka",
		Notify:      false,
		Color:       "gray",
	}
}

func (ho *HipchatOutput) sendMessage(mc string) error {
	messageUri := fmt.Sprintf("%s/rooms/message?auth_token=%s", ho.url, url.QueryEscape(ho.conf.AuthToken))

	messagePayload := url.Values{
		"room_id":        {ho.conf.RoomId},
		"from":           {ho.conf.From},
		"message":        {mc},
		"color":          {ho.conf.Color},
		"message_format": {ho.format},
	}

	if ho.conf.Notify == true {
		messagePayload.Add("notify", "1")
	}

	resp, err := http.PostForm(messageUri, messagePayload)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return err
	}

	messageResponse := &struct{ Status string }{}
	if err := json.Unmarshal(body, messageResponse); err != nil {
		return err
	}
	if messageResponse.Status != "sent" {
		return errors.New("HipchatOutput: Status response was not sent.")
	}

	return nil
}

func (ho *HipchatOutput) Init(config interface{}) (err error) {
	ho.conf = config.(*HipchatOutputConfig)

	if ho.conf.RoomId == "" {
		return fmt.Errorf("room_id must contain a HipChat room ID or name")
	}

	ho.url = "https://api.hipchat.com/v1"
	ho.format = "text"
	return
}

func (ho *HipchatOutput) Run(or OutputRunner, h PluginHelper) (err error) {
	inChan := or.InChan()

	var (
		pack     *PipelinePack
		msg      *message.Message
		contents []byte
	)

	for pack = range inChan {
		msg = pack.Message
		if ho.conf.PayloadOnly {
			err = ho.sendMessage(msg.GetPayload())
		} else {
			if contents, err = json.Marshal(msg); err == nil {
				err = ho.sendMessage(string(contents))
			} else {
				or.LogError(err)
			}
		}
		if err != nil {
			or.LogError(err)
		}
		pack.Recycle()
	}
	return
}

func init() {
	RegisterPlugin("HipchatOutput", func() interface{} {
		return new(HipchatOutput)
	})
}
<commit_msg>Clarification<commit_after>// Copyright 2013, Christian Vozar
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package hipchat

import (
	"encoding/json"
	"errors"
	"fmt"
	"github.com/mozilla-services/heka/message"
	. "github.com/mozilla-services/heka/pipeline"
	"io/ioutil"
	"net/http"
	"net/url"
)

type HipchatOutput struct {
	conf   *HipchatOutputConfig
	url    string
	format string
}

// Hipchat Output config struct
type HipchatOutputConfig struct {
	// Outputs the payload attribute in the HipChat message vs a full JSON message dump
	PayloadOnly bool `toml:"payload_only"`
	// HipChat Authorization token. Notification token is appropriate.
	AuthToken string `toml:"auth_token"`
	// Required. ID or name of the room.
	RoomId string `toml:"room_id"`
	// Required. Name the message will appear be sent. Must be less than 15
	// characters long. May contain letters, numbers, -, _, and spaces.
	From string
	// Whether or not this message should trigger a notification for people
	// in the room (change the tab color, play a sound, etc).
	// Each recipient's notification preferences are taken into account.
	// Default is false
	Notify bool
	// Background color for message.
	// One of "yellow", "red", "green", "purple", "gray", or "random".
	// Default is gray
	Color string
}

func (ho *HipchatOutput) ConfigStruct() interface{} {
	return &HipchatOutputConfig{
		PayloadOnly: true,
		From:        "Heka",
		Notify:      false,
		Color:       "gray",
	}
}

func (ho *HipchatOutput) sendMessage(mc string) error {
	messageUri := fmt.Sprintf("%s/rooms/message?auth_token=%s", ho.url, url.QueryEscape(ho.conf.AuthToken))

	messagePayload := url.Values{
		"room_id":        {ho.conf.RoomId},
		"from":           {ho.conf.From},
		"message":        {mc},
		"color":          {ho.conf.Color},
		"message_format": {ho.format},
	}

	if ho.conf.Notify == true {
		messagePayload.Add("notify", "1")
	}

	resp, err := http.PostForm(messageUri, messagePayload)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return err
	}

	messageResponse := &struct{ Status string }{}
	if err := json.Unmarshal(body, messageResponse); err != nil {
		return err
	}
	if messageResponse.Status != "sent" {
		return errors.New("HipchatOutput: Status response was not sent.")
	}

	return nil
}

func (ho *HipchatOutput) Init(config interface{}) (err error) {
	ho.conf = config.(*HipchatOutputConfig)

	if ho.conf.RoomId == "" {
		return fmt.Errorf("room_id must contain a HipChat room ID or name")
	}

	ho.url = "https://api.hipchat.com/v1"
	ho.format = "text"
	return
}

func (ho *HipchatOutput) Run(or OutputRunner, h PluginHelper) (err error) {
	inChan := or.InChan()

	var (
		pack     *PipelinePack
		msg      *message.Message
		contents []byte
	)

	for pack = range inChan {
		msg = pack.Message
		if ho.conf.PayloadOnly {
			err = ho.sendMessage(msg.GetPayload())
		} else {
			if contents, err = json.Marshal(msg); err == nil {
				err = ho.sendMessage(string(contents))
			} else {
				or.LogError(err)
			}
		}
		if err != nil {
			or.LogError(err)
		}
		pack.Recycle()
	}
	return
}

func init() {
	RegisterPlugin("HipchatOutput", func() interface{} {
		return new(HipchatOutput)
	})
}
<|endoftext|>


<commit_before>package opentracing

import "sync"

// A TraceContextID is the smallest amount of state needed to describe a span's
// identity within a larger [potentially distributed] trace. The TraceContextID
// is not intended to encode the span's operation name, timing, or log data,
// but merely any unique identifiers (etc) needed to contextualize it within a
// larger trace tree.
//
// TraceContextIDs are sufficient to propagate the, well, *context* of a particular
// trace from process to process.
type TraceContextID interface {
	// Create a child context for this TraceContextID, and return both that child's
	// own TraceContextID as well as any Tags that should be added to the child's
	// Span.
	//
	// The returned TraceContextID type must be the same as the type of the
	// TraceContextID implementation itself.
	NewChild() (childCtx TraceContextID, childSpanTags Tags)

	// Serializes the TraceContextID as a valid unicode string.
	SerializeASCII() string

	// Serializes the TraceContextID as arbitrary binary data.
	SerializeBinary() []byte
}

// A long-lived interface that knows how to create a root TraceContextID and
// serialize/deserialize any other.
type TraceContextIDSource interface {
	// Create a TraceContextID which has no parent (and thus begins its own trace).
	// A TraceContextIDSource must always return the same type in successive calls
	// to NewRootTraceContextID().
	NewRootTraceContextID() TraceContextID

	// Converts the encoded binary data (see `TraceContextID.Serialize()`) into a
	// TraceContextID of the same type as returned by NewRootTraceContextID().
	DeserializeBinaryTraceContextID(encoded []byte) (TraceContextID, error)
	DeserializeASCIITraceContextID(encoded string) (TraceContextID, error)
}

// A `TraceContext` builds off of an implementation-provided TraceContextID and
// adds a simple string map of "trace tags". The trace tags are special in that
// they are propagated *in-band*, presumably alongside application data and the
// `TraceContextID` proper. See the documentation for `SetTraceTag()` for more
// details and some important caveats.
//
// Note that the `TraceContext` is managed internally by the opentracer system;
// opentracer implementations only need to concern themselves with the
// `TraceContextID` (which does not know about trace tags).
type TraceContext struct {
	Id TraceContextID

	tagLock   sync.RWMutex
	traceTags map[string]string
}

// `tags` may be nil.
func newTraceContext(id TraceContextID, tags map[string]string) *TraceContext {
	if tags == nil {
		tags = map[string]string{}
	}
	return &TraceContext{
		Id:        id,
		traceTags: tags,
	}
}

// Set a tag on this TraceContext that also propagates to future TraceContext
// children per `NewChild()`.
//
// `SetTraceTag()` enables powerful functionality given a full-stack
// opentracing integration (e.g., arbitrary application data from a mobile app
// can make it, transparently, all the way into the depths of a storage
// system), and with it some powerful costs: use this feature with care.
//
// IMPORTANT NOTE #1: `SetTraceTag()` will only propagate trace tags to
// *future* children of the `TraceContext` (see `NewChild()`) and/or the `Span`
// that references it.
//
// IMPORTANT NOTE #2: Use this thoughtfully and with care. Every key and value
// is copied into every local *and remote* child of this `TraceContext`, and
// that can add up to a lot of network and cpu overhead for large strings.
//
// Returns a reference to this TraceContext for chaining, etc.
func (t *TraceContext) SetTraceTag(key, value string) *TraceContext {
	t.tagLock.Lock()
	defer t.tagLock.Unlock()

	t.traceTags[key] = value
	return t
}

// Gets the value for a trace tag given its key. Returns the empty string if
// the value isn't found in this TraceContext.
func (t *TraceContext) TraceTag(key string) string {
	t.tagLock.RLock()
	defer t.tagLock.RUnlock()

	return t.traceTags[key]
}

func (t *TraceContext) NewChild() (childCtx *TraceContext, childSpanTags Tags) {
	t.tagLock.RLock()
	newTags := make(map[string]string, len(t.traceTags))
	for k, v := range t.traceTags {
		newTags[k] = v
	}
	t.tagLock.RUnlock()

	ctxId, childSpanTags := t.Id.NewChild()
	return &TraceContext{
		Id:        ctxId,
		traceTags: newTags,
	}, childSpanTags
}

func (t *TraceContext) SerializeASCII() string {
	// XXX: implement correctly if we like this API
	return t.Id.SerializeASCII()
}

func (t *TraceContext) SerializeBinary() []byte {
	// XXX: implement correctly if we like this API
	return t.Id.SerializeBinary()
}

func NewRootTraceContext(source TraceContextIDSource) *TraceContext {
	return &TraceContext{
		Id:        source.NewRootTraceContextID(),
		traceTags: make(map[string]string),
	}
}

func DeserializeBinaryTraceContext(
	source TraceContextIDSource,
	encoded []byte,
) (*TraceContext, error) {
	// XXX: implement correctly if we like this API
	tcid, err := source.DeserializeBinaryTraceContextID(encoded)
	if err != nil {
		return nil, err
	}
	return newTraceContext(tcid, nil), nil
}

func DeserializeASCIITraceContext(
	source TraceContextIDSource,
	encoded string,
) (*TraceContext, error) {
	// XXX: implement correctly if we like this API
	tcid, err := source.DeserializeASCIITraceContextID(encoded)
	if err != nil {
		return nil, err
	}
	return newTraceContext(tcid, nil), nil
}
<commit_msg>call it ASCII<commit_after>package opentracing

import "sync"

// A TraceContextID is the smallest amount of state needed to describe a span's
// identity within a larger [potentially distributed] trace. The TraceContextID
// is not intended to encode the span's operation name, timing, or log data,
// but merely any unique identifiers (etc) needed to contextualize it within a
// larger trace tree.
//
// TraceContextIDs are sufficient to propagate the, well, *context* of a particular
// trace from process to process.
type TraceContextID interface {
	// Create a child context for this TraceContextID, and return both that child's
	// own TraceContextID as well as any Tags that should be added to the child's
	// Span.
	//
	// The returned TraceContextID type must be the same as the type of the
	// TraceContextID implementation itself.
	NewChild() (childCtx TraceContextID, childSpanTags Tags)

	// Serializes the TraceContextID as a printable ASCII string (e.g.,
	// base64).
	SerializeASCII() string

	// Serializes the TraceContextID as arbitrary binary data.
	SerializeBinary() []byte
}

// A long-lived interface that knows how to create a root TraceContextID and
// serialize/deserialize any other.
type TraceContextIDSource interface {
	// Create a TraceContextID which has no parent (and thus begins its own trace).
	// A TraceContextIDSource must always return the same type in successive calls
	// to NewRootTraceContextID().
	NewRootTraceContextID() TraceContextID

	// Converts the encoded binary data (see
	// `TraceContextID.SerializeBinary()`) into a TraceContextID of the same
	// type as returned by NewRootTraceContextID().
	DeserializeBinaryTraceContextID(encoded []byte) (TraceContextID, error)

	// Converts the encoded ASCII data (see `TraceContextID.SerializeASCII()`)
	// into a TraceContextID of the same type as returned by
	// NewRootTraceContextID().
	DeserializeASCIITraceContextID(encoded string) (TraceContextID, error)
}

// A `TraceContext` builds off of an implementation-provided TraceContextID and
// adds a simple string map of "trace tags". The trace tags are special in that
// they are propagated *in-band*, presumably alongside application data and the
// `TraceContextID` proper. See the documentation for `SetTraceTag()` for more
// details and some important caveats.
//
// Note that the `TraceContext` is managed internally by the opentracer system;
// opentracer implementations only need to concern themselves with the
// `TraceContextID` (which does not know about trace tags).
type TraceContext struct {
	Id TraceContextID

	tagLock   sync.RWMutex
	traceTags map[string]string
}

// `tags` may be nil.
func newTraceContext(id TraceContextID, tags map[string]string) *TraceContext {
	if tags == nil {
		tags = map[string]string{}
	}
	return &TraceContext{
		Id:        id,
		traceTags: tags,
	}
}

// Set a tag on this TraceContext that also propagates to future TraceContext
// children per `NewChild()`.
//
// `SetTraceTag()` enables powerful functionality given a full-stack
// opentracing integration (e.g., arbitrary application data from a mobile app
// can make it, transparently, all the way into the depths of a storage
// system), and with it some powerful costs: use this feature with care.
//
// IMPORTANT NOTE #1: `SetTraceTag()` will only propagate trace tags to
// *future* children of the `TraceContext` (see `NewChild()`) and/or the `Span`
// that references it.
//
// IMPORTANT NOTE #2: Use this thoughtfully and with care. Every key and value
// is copied into every local *and remote* child of this `TraceContext`, and
// that can add up to a lot of network and cpu overhead for large strings.
//
// Returns a reference to this TraceContext for chaining, etc.
func (t *TraceContext) SetTraceTag(key, value string) *TraceContext {
	t.tagLock.Lock()
	defer t.tagLock.Unlock()

	t.traceTags[key] = value
	return t
}

// Gets the value for a trace tag given its key. Returns the empty string if
// the value isn't found in this TraceContext.
func (t *TraceContext) TraceTag(key string) string {
	t.tagLock.RLock()
	defer t.tagLock.RUnlock()

	return t.traceTags[key]
}

func (t *TraceContext) NewChild() (childCtx *TraceContext, childSpanTags Tags) {
	t.tagLock.RLock()
	newTags := make(map[string]string, len(t.traceTags))
	for k, v := range t.traceTags {
		newTags[k] = v
	}
	t.tagLock.RUnlock()

	ctxId, childSpanTags := t.Id.NewChild()
	return &TraceContext{
		Id:        ctxId,
		traceTags: newTags,
	}, childSpanTags
}

func (t *TraceContext) SerializeASCII() string {
	// XXX: implement correctly if we like this API
	return t.Id.SerializeASCII()
}

func (t *TraceContext) SerializeBinary() []byte {
	// XXX: implement correctly if we like this API
	return t.Id.SerializeBinary()
}

func NewRootTraceContext(source TraceContextIDSource) *TraceContext {
	return &TraceContext{
		Id:        source.NewRootTraceContextID(),
		traceTags: make(map[string]string),
	}
}

func DeserializeBinaryTraceContext(
	source TraceContextIDSource,
	encoded []byte,
) (*TraceContext, error) {
	// XXX: implement correctly if we like this API
	tcid, err := source.DeserializeBinaryTraceContextID(encoded)
	if err != nil {
		return nil, err
	}
	return newTraceContext(tcid, nil), nil
}

func DeserializeASCIITraceContext(
	source TraceContextIDSource,
	encoded string,
) (*TraceContext, error) {
	// XXX: implement correctly if we like this API
	tcid, err := source.DeserializeASCIITraceContextID(encoded)
	if err != nil {
		return nil, err
	}
	return newTraceContext(tcid, nil), nil
}
<|endoftext|>


<commit_before><commit_msg>Add example of using a registry in a CollectableArray<commit_after><|endoftext|>


<commit_before>package integration

import (
	"os"

	influxdb "github.com/influxdb/influxdb/client"
	. "github.com/influxdb/influxdb/integration/helpers"
	. "launchpad.net/gocheck"
)

type RemoveNodeSuite struct {
	serverProcesses []*Server
}

var _ = Suite(&RemoveNodeSuite{})

func (self *RemoveNodeSuite) SetUpSuite(c *C) {
}

func (self *RemoveNodeSuite) TearDownSuite(c *C) {
	for _, s := range self.serverProcesses {
		s.Stop()
	}
}

func (self *RemoveNodeSuite) TestRemovingNode(c *C) {
	err := os.RemoveAll("/tmp/influxdb/test")
	c.Assert(err, IsNil)
	s1 := NewServer("integration/test_rf_1.toml", c)
	defer s1.Stop()
	s1.WaitForServerToStart()

	s2 := NewServer("integration/test_rf_2.toml", c)
	s2.WaitForServerToStart()

	client := s1.GetClient("", c)
	servers, err := client.Servers()
	c.Assert(err, IsNil)
	c.Assert(servers, HasLen, 2)

	s2.Stop()

	c.Assert(client.RemoveServer(2), IsNil)

	servers, err = client.Servers()
	c.Assert(err, IsNil)
	c.Assert(servers, HasLen, 1)

	c.Assert(client.CreateDatabase("test"), IsNil)
	series := CreatePoints("test_replication_factor_1", 1, 1)
	client = s1.GetClient("test", c)
	c.Assert(client.WriteSeries(series), IsNil)

	s1.WaitForServerToSync()
}

func (self *RemoveNodeSuite) TestRemovingNodeAndShards(c *C) {
	err := os.RemoveAll("/tmp/influxdb/test")
	c.Assert(err, IsNil)
	s1 := NewServer("integration/test_replication_1.toml", c)
	defer s1.Stop()
	s1.WaitForServerToStart()

	s2 := NewServer("integration/test_replication_2.toml", c)
	s2.WaitForServerToStart()

	client := s1.GetClient("", c)
	servers, err := client.Servers()
	c.Assert(err, IsNil)
	c.Assert(servers, HasLen, 2)

	c.Assert(client.CreateDatabase("test"), IsNil)
	space := &influxdb.ShardSpace{Name: "test_space", RetentionPolicy: "1h", Database: "test", Regex: "/test_removing_node_and_shards/", ReplicationFactor: 2}
	c.Assert(client.CreateShardSpace(space), IsNil)

	series := CreatePoints("test_removing_node_and_shards", 5, 10)
	client = s1.GetClient("test", c)
	c.Assert(client.WriteSeries(series), IsNil)

	s1.WaitForServerToSync()

	shards, err := client.GetShards()
	c.Assert(err, IsNil)
	c.Assert(shards.All, HasLen, 1)
	c.Assert(shards.All[0].ServerIds, HasLen, 2)
	c.Assert(shards.All[0].ServerIds[0], Equals, uint32(1))
	c.Assert(shards.All[0].ServerIds[1], Equals, uint32(2))

	s2.Stop()

	c.Assert(client.RemoveServer(2), IsNil)

	servers, err = client.Servers()
	c.Assert(err, IsNil)
	c.Assert(servers, HasLen, 1)

	shards, err = client.GetShards()
	c.Assert(err, IsNil)
	c.Assert(shards.All, HasLen, 1)
	c.Assert(shards.All[0].ServerIds, HasLen, 1)
	c.Assert(shards.All[0].ServerIds[0], Equals, uint32(1))
}
<commit_msg>Fix the name of the config files in the integration test suite<commit_after>package integration

import (
	"os"

	influxdb "github.com/influxdb/influxdb/client"
	. "github.com/influxdb/influxdb/integration/helpers"
	. "launchpad.net/gocheck"
)

type RemoveNodeSuite struct {
	serverProcesses []*Server
}

var _ = Suite(&RemoveNodeSuite{})

func (self *RemoveNodeSuite) SetUpSuite(c *C) {
}

func (self *RemoveNodeSuite) TearDownSuite(c *C) {
	for _, s := range self.serverProcesses {
		s.Stop()
	}
}

func (self *RemoveNodeSuite) TestRemovingNode(c *C) {
	err := os.RemoveAll("/tmp/influxdb/test")
	c.Assert(err, IsNil)
	s1 := NewServer("integration/test_rf_1.toml", c)
	defer s1.Stop()
	s1.WaitForServerToStart()

	s2 := NewServer("integration/test_rf_2.toml", c)
	s2.WaitForServerToStart()

	client := s1.GetClient("", c)
	servers, err := client.Servers()
	c.Assert(err, IsNil)
	c.Assert(servers, HasLen, 2)

	s2.Stop()

	c.Assert(client.RemoveServer(2), IsNil)

	servers, err = client.Servers()
	c.Assert(err, IsNil)
	c.Assert(servers, HasLen, 1)

	c.Assert(client.CreateDatabase("test"), IsNil)
	series := CreatePoints("test_replication_factor_1", 1, 1)
	client = s1.GetClient("test", c)
	c.Assert(client.WriteSeries(series), IsNil)

	s1.WaitForServerToSync()
}

func (self *RemoveNodeSuite) TestRemovingNodeAndShards(c *C) {
	err := os.RemoveAll("/tmp/influxdb/test")
	c.Assert(err, IsNil)
	s1 := NewServer("integration/test_rf_1.toml", c)
	defer s1.Stop()
	s1.WaitForServerToStart()

	s2 := NewServer("integration/test_rf_2.toml", c)
	s2.WaitForServerToStart()

	client := s1.GetClient("", c)
	servers, err := client.Servers()
	c.Assert(err, IsNil)
	c.Assert(servers, HasLen, 2)

	c.Assert(client.CreateDatabase("test"), IsNil)
	space := &influxdb.ShardSpace{Name: "test_space", RetentionPolicy: "1h", Database: "test", Regex: "/test_removing_node_and_shards/", ReplicationFactor: 2}
	c.Assert(client.CreateShardSpace(space), IsNil)

	series := CreatePoints("test_removing_node_and_shards", 5, 10)
	client = s1.GetClient("test", c)
	c.Assert(client.WriteSeries(series), IsNil)

	s1.WaitForServerToSync()

	shards, err := client.GetShards()
	c.Assert(err, IsNil)
	c.Assert(shards.All, HasLen, 1)
	c.Assert(shards.All[0].ServerIds, HasLen, 2)
	c.Assert(shards.All[0].ServerIds[0], Equals, uint32(1))
	c.Assert(shards.All[0].ServerIds[1], Equals, uint32(2))

	s2.Stop()

	c.Assert(client.RemoveServer(2), IsNil)

	servers, err = client.Servers()
	c.Assert(err, IsNil)
	c.Assert(servers, HasLen, 1)

	shards, err = client.GetShards()
	c.Assert(err, IsNil)
	c.Assert(shards.All, HasLen, 1)
	c.Assert(shards.All[0].ServerIds, HasLen, 1)
	c.Assert(shards.All[0].ServerIds[0], Equals, uint32(1))
}
<|endoftext|>


<commit_before>// Copyright ©2016 The gonum Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package optimize_test

import (
	"fmt"
	"log"

	"github.com/gonum/optimize"
	"github.com/gonum/optimize/functions"
)

func ExampleLocal_BFGS() {
	p := optimize.Problem{
		Func: functions.ExtendedRosenbrock{}.Func,
		Grad: functions.ExtendedRosenbrock{}.Grad,
	}

	x := []float64{1.3, 0.7, 0.8, 1.9, 1.2}
	settings := optimize.DefaultSettings()
	settings.Recorder = nil
	settings.GradientThreshold = 1e-12
	settings.FunctionConverge = nil

	result, err := optimize.Local(p, x, settings, &optimize.BFGS{})
	if err != nil {
		log.Fatal(err)
	}
	if err = result.Status.Err(); err != nil {
		log.Fatal(err)
	}
	fmt.Printf("result.Status: %v\n", result.Status)
	fmt.Printf("result.X: %v\n", result.X)
	fmt.Printf("result.F: %v\n", result.F)
	fmt.Printf("result.Stats.FuncEvaluations: %d\n", result.Stats.FuncEvaluations)
	// Output:
	// result.Status: GradientThreshold
	// result.X: [1 1 1 1 1]
	// result.F: 0
	// result.Stats.FuncEvaluations: 35
}
<commit_msg>optimize: fix example name for godoc<commit_after>// Copyright ©2016 The gonum Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package optimize_test

import (
	"fmt"
	"log"

	"github.com/gonum/optimize"
	"github.com/gonum/optimize/functions"
)

func ExampleLocal() {
	p := optimize.Problem{
		Func: functions.ExtendedRosenbrock{}.Func,
		Grad: functions.ExtendedRosenbrock{}.Grad,
	}

	x := []float64{1.3, 0.7, 0.8, 1.9, 1.2}
	settings := optimize.DefaultSettings()
	settings.Recorder = nil
	settings.GradientThreshold = 1e-12
	settings.FunctionConverge = nil

	result, err := optimize.Local(p, x, settings, &optimize.BFGS{})
	if err != nil {
		log.Fatal(err)
	}
	if err = result.Status.Err(); err != nil {
		log.Fatal(err)
	}
	fmt.Printf("result.Status: %v\n", result.Status)
	fmt.Printf("result.X: %v\n", result.X)
	fmt.Printf("result.F: %v\n", result.F)
	fmt.Printf("result.Stats.FuncEvaluations: %d\n", result.Stats.FuncEvaluations)
	// Output:
	// result.Status: GradientThreshold
	// result.X: [1 1 1 1 1]
	// result.F: 0
	// result.Stats.FuncEvaluations: 35
}
<|endoftext|>


<commit_before>package format

import (
	"reflect"
	"testing"
	"unsafe"
)

var testCases = []struct {
	pat  string
	want *NumberFormat
}{{
	"#",
	&NumberFormat{
		FormatWidth: 1,
		// TODO: Should MinIntegerDigits be 1?
	},
}, {
	"0",
	&NumberFormat{
		FormatWidth:      1,
		MinIntegerDigits: 1,
	},
}, {
	"0000",
	&NumberFormat{
		FormatWidth:      4,
		MinIntegerDigits: 4,
	},
}, {
	".#",
	&NumberFormat{
		FormatWidth:       2,
		MaxFractionDigits: 1,
	},
}, {
	"#0.###",
	&NumberFormat{
		FormatWidth:       6,
		MinIntegerDigits:  1,
		MaxFractionDigits: 3,
	},
}, {
	"#0.######",
	&NumberFormat{
		FormatWidth:       9,
		MinIntegerDigits:  1,
		MaxFractionDigits: 6,
	},
}, {
	"#,##0.###",
	&NumberFormat{
		FormatWidth:       9,
		GroupingSize:      [2]uint8{3, 0},
		MinIntegerDigits:  1,
		MaxFractionDigits: 3,
	},
}, {
	"#,##,##0.###",
	&NumberFormat{
		FormatWidth:       12,
		GroupingSize:      [2]uint8{3, 2},
		MinIntegerDigits:  1,
		MaxFractionDigits: 3,
	},
}, {
	// Ignore additional separators.
	"#,####,##,##0.###",
	&NumberFormat{
		FormatWidth:       17,
		GroupingSize:      [2]uint8{3, 2},
		MinIntegerDigits:  1,
		MaxFractionDigits: 3,
	},
}, {
	"#E0",
	&NumberFormat{
		FormatWidth:       3,
		MaxIntegerDigits:  1,
		MinExponentDigits: 1,
	},
}, {
	"0E0",
	&NumberFormat{
		FormatWidth:       3,
		MinIntegerDigits:  1,
		MinExponentDigits: 1,
	},
}, {
	"##00.0#E0",
	&NumberFormat{
		FormatWidth:       9,
		MinIntegerDigits:  2,
		MaxIntegerDigits:  4,
		MinFractionDigits: 1,
		MaxFractionDigits: 2,
		MinExponentDigits: 1,
	},
}, {
	"#00.0E+0",
	&NumberFormat{
		FormatWidth:       8,
		Flags:             AlwaysExpSign,
		MinIntegerDigits:  2,
		MaxIntegerDigits:  3,
		MinFractionDigits: 1,
		MaxFractionDigits: 1,
		MinExponentDigits: 1,
	},
}, {
	"0.0E++0",
	nil,
}, {
	"#0E+",
	nil,
}, {
	// significant digits
	"@",
	&NumberFormat{
		FormatWidth:          1,
		MinSignificantDigits: 1,
		MaxSignificantDigits: 1,
	},
}, {
	// significant digits
	"@@@@",
	&NumberFormat{
		FormatWidth:          4,
		MinSignificantDigits: 4,
		MaxSignificantDigits: 4,
	},
}, {
	"@###",
	&NumberFormat{
		FormatWidth:          4,
		MinSignificantDigits: 1,
		MaxSignificantDigits: 4,
	},
}, {
	// Exponents in significant digits mode gets normalized.
	"@@E0",
	&NumberFormat{
		FormatWidth:       4,
		MinIntegerDigits:  1,
		MaxIntegerDigits:  1,
		MinFractionDigits: 1,
		MaxFractionDigits: 1,
		MinExponentDigits: 1,
	},
}, {
	"@###E00",
	&NumberFormat{
		FormatWidth:       7,
		MinIntegerDigits:  1,
		MaxIntegerDigits:  1,
		MinFractionDigits: 0,
		MaxFractionDigits: 3,
		MinExponentDigits: 2,
	},
}, {
	// The significant digits mode does not allow fractions.
	"@###.#E0",
	nil,
}, {
	//alternative negative pattern
	"#0.###;(#0.###)",
	&NumberFormat{
		Affix:             "\x00\x00\x01(\x01)",
		NegOffset:         2,
		FormatWidth:       6,
		MinIntegerDigits:  1,
		MaxFractionDigits: 3,
	},
}, {
	// Rounding increments
	"1.05",
	&NumberFormat{
		RoundIncrement:    105,
		FormatWidth:       4,
		MinIntegerDigits:  1,
		MinFractionDigits: 2,
		MaxFractionDigits: 2,
	},
}, {
	"0.0%",
	&NumberFormat{
		Affix:             "\x00\x01%",
		Multiplier:        100,
		FormatWidth:       4,
		MinIntegerDigits:  1,
		MinFractionDigits: 1,
		MaxFractionDigits: 1,
	},
}, {
	"0.0‰",
	&NumberFormat{
		Affix:             "\x00\x03‰",
		Multiplier:        1000,
		FormatWidth:       4,
		MinIntegerDigits:  1,
		MinFractionDigits: 1,
		MaxFractionDigits: 1,
	},
}, {
	"#,##0.00¤",
	&NumberFormat{
		Affix:             "\x00\x02¤",
		FormatWidth:       9,
		GroupingSize:      [2]uint8{3, 0},
		MinIntegerDigits:  1,
		MinFractionDigits: 2,
		MaxFractionDigits: 2,
	},
}, {
	"#,##0.00 ¤;(#,##0.00 ¤)",
	&NumberFormat{Affix: "\x00\x04\u00a0¤\x01(\x05\u00a0¤)",
		NegOffset:         6,
		Multiplier:        0,
		FormatWidth:       10,
		GroupingSize:      [2]uint8{3, 0},
		MinIntegerDigits:  1,
		MinFractionDigits: 2,
		MaxFractionDigits: 2,
	},
}, {
	// padding
	"*x#",
	&NumberFormat{
		PadRune:     'x',
		FormatWidth: 1,
	},
}, {
	// padding
	"#*x",
	&NumberFormat{
		PadRune:     'x',
		FormatWidth: 1,
		Flags:       PadBeforeSuffix,
	},
}, {
	"*xpre#suf",
	&NumberFormat{
		Affix:       "\x03pre\x03suf",
		PadRune:     'x',
		FormatWidth: 7,
	},
}, {
	"pre*x#suf",
	&NumberFormat{
		Affix:       "\x03pre\x03suf",
		PadRune:     'x',
		FormatWidth: 7,
		Flags:       PadAfterPrefix,
	},
}, {
	"pre#*xsuf",
	&NumberFormat{
		Affix:       "\x03pre\x03suf",
		PadRune:     'x',
		FormatWidth: 7,
		Flags:       PadBeforeSuffix,
	},
}, {
	"pre#suf*x",
	&NumberFormat{
		Affix:       "\x03pre\x03suf",
		PadRune:     'x',
		FormatWidth: 7,
		Flags:       PadAfterSuffix,
	},
}, {
	// no duplicate padding
	"*xpre#suf*x", nil,
}, {
	// no duplicate padding
	"*xpre#suf*x", nil,
}}

func TestParseNumberPattern(t *testing.T) {
	for i, tc := range testCases {
		f, err := ParseNumberPattern(tc.pat)
		if !reflect.DeepEqual(f, tc.want) {
			t.Errorf("%d:%s:\ngot %#v;\nwant %#v", i, tc.pat, f, tc.want)
		}
		if got, want := err != nil, tc.want == nil; got != want {
			t.Errorf("%d:%s:error: got %v; want %v", i, tc.pat, err, want)
		}
	}
}

func TestPatternSize(t *testing.T) {
	if sz := unsafe.Sizeof(NumberFormat{}); sz != 48 {
		t.Errorf("got %d; want 48", sz)
	}

}
<commit_msg>internal/format: fixed test for 32-bit architectures<commit_after>package format

import (
	"reflect"
	"testing"
	"unsafe"
)

var testCases = []struct {
	pat  string
	want *NumberFormat
}{{
	"#",
	&NumberFormat{
		FormatWidth: 1,
		// TODO: Should MinIntegerDigits be 1?
	},
}, {
	"0",
	&NumberFormat{
		FormatWidth:      1,
		MinIntegerDigits: 1,
	},
}, {
	"0000",
	&NumberFormat{
		FormatWidth:      4,
		MinIntegerDigits: 4,
	},
}, {
	".#",
	&NumberFormat{
		FormatWidth:       2,
		MaxFractionDigits: 1,
	},
}, {
	"#0.###",
	&NumberFormat{
		FormatWidth:       6,
		MinIntegerDigits:  1,
		MaxFractionDigits: 3,
	},
}, {
	"#0.######",
	&NumberFormat{
		FormatWidth:       9,
		MinIntegerDigits:  1,
		MaxFractionDigits: 6,
	},
}, {
	"#,##0.###",
	&NumberFormat{
		FormatWidth:       9,
		GroupingSize:      [2]uint8{3, 0},
		MinIntegerDigits:  1,
		MaxFractionDigits: 3,
	},
}, {
	"#,##,##0.###",
	&NumberFormat{
		FormatWidth:       12,
		GroupingSize:      [2]uint8{3, 2},
		MinIntegerDigits:  1,
		MaxFractionDigits: 3,
	},
}, {
	// Ignore additional separators.
	"#,####,##,##0.###",
	&NumberFormat{
		FormatWidth:       17,
		GroupingSize:      [2]uint8{3, 2},
		MinIntegerDigits:  1,
		MaxFractionDigits: 3,
	},
}, {
	"#E0",
	&NumberFormat{
		FormatWidth:       3,
		MaxIntegerDigits:  1,
		MinExponentDigits: 1,
	},
}, {
	"0E0",
	&NumberFormat{
		FormatWidth:       3,
		MinIntegerDigits:  1,
		MinExponentDigits: 1,
	},
}, {
	"##00.0#E0",
	&NumberFormat{
		FormatWidth:       9,
		MinIntegerDigits:  2,
		MaxIntegerDigits:  4,
		MinFractionDigits: 1,
		MaxFractionDigits: 2,
		MinExponentDigits: 1,
	},
}, {
	"#00.0E+0",
	&NumberFormat{
		FormatWidth:       8,
		Flags:             AlwaysExpSign,
		MinIntegerDigits:  2,
		MaxIntegerDigits:  3,
		MinFractionDigits: 1,
		MaxFractionDigits: 1,
		MinExponentDigits: 1,
	},
}, {
	"0.0E++0",
	nil,
}, {
	"#0E+",
	nil,
}, {
	// significant digits
	"@",
	&NumberFormat{
		FormatWidth:          1,
		MinSignificantDigits: 1,
		MaxSignificantDigits: 1,
	},
}, {
	// significant digits
	"@@@@",
	&NumberFormat{
		FormatWidth:          4,
		MinSignificantDigits: 4,
		MaxSignificantDigits: 4,
	},
}, {
	"@###",
	&NumberFormat{
		FormatWidth:          4,
		MinSignificantDigits: 1,
		MaxSignificantDigits: 4,
	},
}, {
	// Exponents in significant digits mode gets normalized.
	"@@E0",
	&NumberFormat{
		FormatWidth:       4,
		MinIntegerDigits:  1,
		MaxIntegerDigits:  1,
		MinFractionDigits: 1,
		MaxFractionDigits: 1,
		MinExponentDigits: 1,
	},
}, {
	"@###E00",
	&NumberFormat{
		FormatWidth:       7,
		MinIntegerDigits:  1,
		MaxIntegerDigits:  1,
		MinFractionDigits: 0,
		MaxFractionDigits: 3,
		MinExponentDigits: 2,
	},
}, {
	// The significant digits mode does not allow fractions.
	"@###.#E0",
	nil,
}, {
	//alternative negative pattern
	"#0.###;(#0.###)",
	&NumberFormat{
		Affix:             "\x00\x00\x01(\x01)",
		NegOffset:         2,
		FormatWidth:       6,
		MinIntegerDigits:  1,
		MaxFractionDigits: 3,
	},
}, {
	// Rounding increments
	"1.05",
	&NumberFormat{
		RoundIncrement:    105,
		FormatWidth:       4,
		MinIntegerDigits:  1,
		MinFractionDigits: 2,
		MaxFractionDigits: 2,
	},
}, {
	"0.0%",
	&NumberFormat{
		Affix:             "\x00\x01%",
		Multiplier:        100,
		FormatWidth:       4,
		MinIntegerDigits:  1,
		MinFractionDigits: 1,
		MaxFractionDigits: 1,
	},
}, {
	"0.0‰",
	&NumberFormat{
		Affix:             "\x00\x03‰",
		Multiplier:        1000,
		FormatWidth:       4,
		MinIntegerDigits:  1,
		MinFractionDigits: 1,
		MaxFractionDigits: 1,
	},
}, {
	"#,##0.00¤",
	&NumberFormat{
		Affix:             "\x00\x02¤",
		FormatWidth:       9,
		GroupingSize:      [2]uint8{3, 0},
		MinIntegerDigits:  1,
		MinFractionDigits: 2,
		MaxFractionDigits: 2,
	},
}, {
	"#,##0.00 ¤;(#,##0.00 ¤)",
	&NumberFormat{Affix: "\x00\x04\u00a0¤\x01(\x05\u00a0¤)",
		NegOffset:         6,
		Multiplier:        0,
		FormatWidth:       10,
		GroupingSize:      [2]uint8{3, 0},
		MinIntegerDigits:  1,
		MinFractionDigits: 2,
		MaxFractionDigits: 2,
	},
}, {
	// padding
	"*x#",
	&NumberFormat{
		PadRune:     'x',
		FormatWidth: 1,
	},
}, {
	// padding
	"#*x",
	&NumberFormat{
		PadRune:     'x',
		FormatWidth: 1,
		Flags:       PadBeforeSuffix,
	},
}, {
	"*xpre#suf",
	&NumberFormat{
		Affix:       "\x03pre\x03suf",
		PadRune:     'x',
		FormatWidth: 7,
	},
}, {
	"pre*x#suf",
	&NumberFormat{
		Affix:       "\x03pre\x03suf",
		PadRune:     'x',
		FormatWidth: 7,
		Flags:       PadAfterPrefix,
	},
}, {
	"pre#*xsuf",
	&NumberFormat{
		Affix:       "\x03pre\x03suf",
		PadRune:     'x',
		FormatWidth: 7,
		Flags:       PadBeforeSuffix,
	},
}, {
	"pre#suf*x",
	&NumberFormat{
		Affix:       "\x03pre\x03suf",
		PadRune:     'x',
		FormatWidth: 7,
		Flags:       PadAfterSuffix,
	},
}, {
	// no duplicate padding
	"*xpre#suf*x", nil,
}, {
	// no duplicate padding
	"*xpre#suf*x", nil,
}}

func TestParseNumberPattern(t *testing.T) {
	for i, tc := range testCases {
		f, err := ParseNumberPattern(tc.pat)
		if !reflect.DeepEqual(f, tc.want) {
			t.Errorf("%d:%s:\ngot %#v;\nwant %#v", i, tc.pat, f, tc.want)
		}
		if got, want := err != nil, tc.want == nil; got != want {
			t.Errorf("%d:%s:error: got %v; want %v", i, tc.pat, err, want)
		}
	}
}

func TestPatternSize(t *testing.T) {
	if sz := unsafe.Sizeof(NumberFormat{}); sz > 48 {
		t.Errorf("got %d; want 48", sz)
	}

}
<|endoftext|>


<commit_before>package store

import (
	"context"

	"github.com/gempir/gempbot/internal/dto"
	"gorm.io/gorm"
)

type EmoteAdd struct {
	gorm.Model
	ID              uint                `gorm:"primarykey,autoIncrement"`
	ChannelTwitchID string              `gorm:"index"`
	Type            dto.RewardType      `gorm:"index"`
	ChangeType      dto.EmoteChangeType `gorm:"index"`
	Blocked         bool                `gorm:"index"`
	EmoteID         string
}

func (db *Database) GetEmoteAdd(channelTwitchID string, emoteID string) *EmoteAdd {
	var emoteAdd EmoteAdd
	db.Client.Where("channel_twitch_id = ? AND emote_id = ?", channelTwitchID, emoteID).First(&emoteAdd)
	return &emoteAdd
}

func (db *Database) BlockEmoteAdd(channelTwitchID string, emoteID string) {
	db.Client.Model(&EmoteAdd{}).Where("channel_twitch_id = ? AND emote_id = ? AND change_type = ?", channelTwitchID, emoteID, dto.EMOTE_ADD_ADD).Update("blocked", true)
}

func (db *Database) RemoveEmoteAdd(channelTwitchID string, emoteID string) {
	db.Client.Where("channel_twitch_id = ? AND emote_id = ? AND change_type = ? ", channelTwitchID, emoteID, dto.EMOTE_ADD_ADD).Delete(&EmoteAdd{})
}

func (db *Database) CreateEmoteAdd(channelTwitchID string, addType dto.RewardType, emoteID string, emoteChangeType dto.EmoteChangeType) {
	add := EmoteAdd{ChannelTwitchID: channelTwitchID, Type: addType, EmoteID: emoteID, ChangeType: emoteChangeType}
	db.Client.Create(&add)
}

func (db *Database) GetEmoteAdded(channelTwitchID string, addType dto.RewardType, limit int) []EmoteAdd {
	var emotes []EmoteAdd

	db.Client.Where("channel_twitch_id = ? AND type = ? AND change_type = ?", channelTwitchID, addType, dto.EMOTE_ADD_ADD).Order("updated_at desc").Limit(limit).Find(&emotes)

	return emotes
}

func (db *Database) GetEmoteHistory(ctx context.Context, ownerTwitchID string, page int, pageSize int, added bool) []EmoteAdd {
	var emoteHistory []EmoteAdd

	query := db.Client.WithContext(ctx)
	if added {
		query = query.Where("channel_twitch_id = ? AND change_type = ?", ownerTwitchID, dto.EMOTE_ADD_ADD)
	} else {
		query = query.Where("channel_twitch_id = ? AND change_type != ?", ownerTwitchID, dto.EMOTE_ADD_ADD)
	}

	query.Offset((page * pageSize) - pageSize).Limit(pageSize).Order("updated_at desc").Find(&emoteHistory)

	emoteHistory = []EmoteAdd{}

	return emoteHistory
}
<commit_msg>remove accidental dev change<commit_after>package store

import (
	"context"

	"github.com/gempir/gempbot/internal/dto"
	"gorm.io/gorm"
)

type EmoteAdd struct {
	gorm.Model
	ID              uint                `gorm:"primarykey,autoIncrement"`
	ChannelTwitchID string              `gorm:"index"`
	Type            dto.RewardType      `gorm:"index"`
	ChangeType      dto.EmoteChangeType `gorm:"index"`
	Blocked         bool                `gorm:"index"`
	EmoteID         string
}

func (db *Database) GetEmoteAdd(channelTwitchID string, emoteID string) *EmoteAdd {
	var emoteAdd EmoteAdd
	db.Client.Where("channel_twitch_id = ? AND emote_id = ?", channelTwitchID, emoteID).First(&emoteAdd)
	return &emoteAdd
}

func (db *Database) BlockEmoteAdd(channelTwitchID string, emoteID string) {
	db.Client.Model(&EmoteAdd{}).Where("channel_twitch_id = ? AND emote_id = ? AND change_type = ?", channelTwitchID, emoteID, dto.EMOTE_ADD_ADD).Update("blocked", true)
}

func (db *Database) RemoveEmoteAdd(channelTwitchID string, emoteID string) {
	db.Client.Where("channel_twitch_id = ? AND emote_id = ? AND change_type = ? ", channelTwitchID, emoteID, dto.EMOTE_ADD_ADD).Delete(&EmoteAdd{})
}

func (db *Database) CreateEmoteAdd(channelTwitchID string, addType dto.RewardType, emoteID string, emoteChangeType dto.EmoteChangeType) {
	add := EmoteAdd{ChannelTwitchID: channelTwitchID, Type: addType, EmoteID: emoteID, ChangeType: emoteChangeType}
	db.Client.Create(&add)
}

func (db *Database) GetEmoteAdded(channelTwitchID string, addType dto.RewardType, limit int) []EmoteAdd {
	var emotes []EmoteAdd

	db.Client.Where("channel_twitch_id = ? AND type = ? AND change_type = ?", channelTwitchID, addType, dto.EMOTE_ADD_ADD).Order("updated_at desc").Limit(limit).Find(&emotes)

	return emotes
}

func (db *Database) GetEmoteHistory(ctx context.Context, ownerTwitchID string, page int, pageSize int, added bool) []EmoteAdd {
	var emoteHistory []EmoteAdd

	query := db.Client.WithContext(ctx)
	if added {
		query = query.Where("channel_twitch_id = ? AND change_type = ?", ownerTwitchID, dto.EMOTE_ADD_ADD)
	} else {
		query = query.Where("channel_twitch_id = ? AND change_type != ?", ownerTwitchID, dto.EMOTE_ADD_ADD)
	}

	query.Offset((page * pageSize) - pageSize).Limit(pageSize).Order("updated_at desc").Find(&emoteHistory)

	return emoteHistory
}
<|endoftext|>


<commit_before>// Copyright 2015 Google Inc. All Rights Reserved.
// This file is available under the Apache license.

package watcher

import (
	"context"
	"expvar"
	"fmt"
	"os"
	"path"
	"path/filepath"
	"sync"
	"time"

	"github.com/fsnotify/fsnotify"
	"github.com/golang/glog"
	"github.com/pkg/errors"
)

var (
	errorCount = expvar.NewInt("log_watcher_error_count")
)

type watch struct {
	ps []Processor
	fi os.FileInfo
}

// LogWatcher implements a Watcher for watching real filesystems.
type LogWatcher struct {
	watcher    *fsnotify.Watcher
	pollTicker *time.Ticker

	watchedMu sync.RWMutex // protects `watched'
	watched   map[string]*watch

	stopTicks chan struct{} // Channel to notify ticker to stop.

	ticksDone  chan struct{} // Channel to notify when the ticks handler is done.
	eventsDone chan struct{} // Channel to notify when the events handler is done.

	closeOnce sync.Once
}

// NewLogWatcher returns a new LogWatcher, or returns an error.
func NewLogWatcher(pollInterval time.Duration, enableFsnotify bool) (*LogWatcher, error) {
	var f *fsnotify.Watcher
	if enableFsnotify {
		var err error
		f, err = fsnotify.NewWatcher()
		if err != nil {
			glog.Warning(err)
		}
	}
	if f == nil && pollInterval == 0 {
		glog.Infof("fsnotify disabled and no poll interval specified; defaulting to 250ms poll")
		pollInterval = time.Millisecond * 250
	}
	w := &LogWatcher{
		watcher: f,
		watched: make(map[string]*watch),
	}
	if pollInterval > 0 {
		w.pollTicker = time.NewTicker(pollInterval)
		w.stopTicks = make(chan struct{})
		w.ticksDone = make(chan struct{})
		go w.runTicks()
	}
	if f != nil {
		w.eventsDone = make(chan struct{})
		go w.runEvents()
	}
	return w, nil
}

func (w *LogWatcher) sendEvent(e Event) {
	w.watchedMu.RLock()
	watch, ok := w.watched[e.Pathname]
	w.watchedMu.RUnlock()
	if !ok {
		d := filepath.Dir(e.Pathname)
		w.watchedMu.RLock()
		watch, ok = w.watched[d]
		w.watchedMu.RUnlock()
		if !ok {
			glog.V(2).Infof("No watch for path %q", e.Pathname)
			return
		}
	}
	w.sendWatchedEvent(watch, e)
}

// Send an event to a watch; all locks assumed to be held.
func (w *LogWatcher) sendWatchedEvent(watch *watch, e Event) {
	for _, p := range watch.ps {
		p.ProcessFileEvent(context.TODO(), e)
	}
}

func (w *LogWatcher) runTicks() {
	defer close(w.ticksDone)

	if w.pollTicker == nil {
		return
	}

	for {
		select {
		case <-w.pollTicker.C:
			w.watchedMu.Lock()
			for n, watched := range w.watched {
				w.pollWatchedPathLocked(n, watched)
			}
			w.watchedMu.Unlock()
		case <-w.stopTicks:
			w.pollTicker.Stop()
			return
		}
	}
}

// pollWatchedPathLocked polls an already-watched path for updates.  w.watchedMu must be locked when called.
func (w *LogWatcher) pollWatchedPathLocked(pathname string, watched *watch) {
	glog.V(2).Info("stat")
	fi, err := os.Stat(pathname)
	if err != nil {
		glog.V(1).Info(err)
		return
	}

	// fsnotify does not send update events for the directory itself.
	if fi.IsDir() {
		w.pollDirectoryLocked(watched, pathname)
	} else if watched.fi == nil || fi.ModTime().Sub(watched.fi.ModTime()) > 0 {
		glog.V(2).Infof("sending update for %s", pathname)
		w.sendWatchedEvent(watched, Event{Update, pathname})
	}

	glog.V(2).Info("Update fi")
	watched.fi = fi
}

func (w *LogWatcher) pollDirectoryLocked(pwatch *watch, pathname string) {
	matches, err := filepath.Glob(path.Join(pathname, "*"))
	if err != nil {
		glog.V(1).Info(err)
		return
	}
	// TODO(jaq): how do we avoid duplicate notifies for things that are already in the watch list?
	for _, match := range matches {
		fi, err := os.Stat(match)
		if err != nil {
			glog.V(1).Info(err)
			continue
		}

		watched, ok := w.watched[match]
		switch {
		case !ok:
			glog.V(2).Infof("sending create for %s", match)
			w.sendWatchedEvent(pwatch, Event{Create, match})
			w.watched[match] = &watch{ps: pwatch.ps, fi: fi}
		case watched.fi != nil && fi.ModTime().Sub(watched.fi.ModTime()) > 0:
			glog.V(2).Infof("sending update for %s", match)
			w.sendWatchedEvent(watched, Event{Update, match})
			w.watched[match].fi = fi
		default:
			glog.V(2).Infof("No modtime change for %s, no send", match)
		}
		if fi.IsDir() {
			w.pollDirectoryLocked(pwatch, match)
		}
	}
}

// runEvents assumes that w.watcher is not nil
func (w *LogWatcher) runEvents() {
	defer close(w.eventsDone)

	// Suck out errors and dump them to the error log.
	go func() {
		for err := range w.watcher.Errors {
			errorCount.Add(1)
			glog.Errorf("fsnotify error: %s\n", err)
		}
	}()

	for e := range w.watcher.Events {
		glog.V(2).Infof("watcher event %v", e)
		switch {
		case e.Op&fsnotify.Create == fsnotify.Create:
			w.sendEvent(Event{Create, e.Name})
		case e.Op&fsnotify.Write == fsnotify.Write,
			e.Op&fsnotify.Chmod == fsnotify.Chmod:
			w.sendEvent(Event{Update, e.Name})
		case e.Op&fsnotify.Remove == fsnotify.Remove:
			w.sendEvent(Event{Delete, e.Name})
		case e.Op&fsnotify.Rename == fsnotify.Rename:
			// Rename is only issued on the original file path; the new name receives a Create event
			w.sendEvent(Event{Delete, e.Name})
		default:
			panic(fmt.Sprintf("unknown op type %v", e.Op))
		}
	}
	glog.Infof("Shutting down log watcher.")
}

// Close shuts down the LogWatcher.  It is safe to call this from multiple clients.
func (w *LogWatcher) Close() (err error) {
	w.closeOnce.Do(func() {
		if w.watcher != nil {
			err = w.watcher.Close()
			<-w.eventsDone
		}
		if w.pollTicker != nil {
			close(w.stopTicks)
			<-w.ticksDone
		}
		glog.Info("Closing events channels")
	})
	return nil
}

// Observe adds a path to the list of watched items.
// If this path has a new event, then the processor being registered will be sent the event.
func (w *LogWatcher) Observe(path string, processor Processor) error {
	absPath, err := w.addWatch(path)
	if err != nil {
		return err
	}
	w.watchedMu.Lock()
	defer w.watchedMu.Unlock()
	watched, ok := w.watched[absPath]
	if !ok {
		w.watched[absPath] = &watch{ps: []Processor{processor}}
		glog.Infof("No abspath in watched list, added new one for %s", absPath)
		return nil
	}
	for _, p := range watched.ps {
		if p == processor {
			glog.Infof("Found this processor in watched list")
			return nil
		}
	}
	watched.ps = append(watched.ps, processor)
	glog.Infof("appended this processor")
	return nil

}

func (w *LogWatcher) addWatch(path string) (string, error) {
	absPath, err := filepath.Abs(path)
	if err != nil {
		return "", errors.Wrapf(err, "Failed to lookup absolutepath of %q", path)
	}
	glog.V(2).Infof("Adding a watch on resolved path %q", absPath)
	if w.watcher != nil {
		err = w.watcher.Add(absPath)
		if err != nil {
			if os.IsPermission(err) {
				glog.V(2).Infof("Skipping permission denied error on adding a watch.")
			} else {
				return "", errors.Wrapf(err, "Failed to create a new watch on %q", absPath)
			}
		}
	}
	return absPath, nil
}

// IsWatching indicates if the path is being watched. It includes both
// filenames and directories.
func (w *LogWatcher) IsWatching(path string) bool {
	absPath, err := filepath.Abs(path)
	if err != nil {
		glog.V(2).Infof("Couldn't resolve path %q: %s", absPath, err)
		return false
	}
	glog.V(2).Infof("Resolved path for lookup %q", absPath)
	w.watchedMu.RLock()
	_, ok := w.watched[absPath]
	w.watchedMu.RUnlock()
	return ok
}

func (w *LogWatcher) Unobserve(path string, processor Processor) error {
	w.watchedMu.Lock()
	defer w.watchedMu.Unlock()
	_, ok := w.watched[path]
	if !ok {
		return nil
	}

	for i, p := range w.watched[path].ps {
		if p == processor {
			w.watched[path].ps = append(w.watched[path].ps[0:i], w.watched[path].ps[i+1:]...)
			break
		}
	}
	if len(w.watched[path].ps) == 0 {
		delete(w.watched, path)
	}
	if w.watcher != nil {
		return w.watcher.Remove(path)
	}
	return nil
}
<commit_msg>Fix race condition in log watcher when recursing on the watched item map.<commit_after>// Copyright 2015 Google Inc. All Rights Reserved.
// This file is available under the Apache license.

package watcher

import (
	"context"
	"expvar"
	"fmt"
	"os"
	"path"
	"path/filepath"
	"sync"
	"time"

	"github.com/fsnotify/fsnotify"
	"github.com/golang/glog"
	"github.com/pkg/errors"
)

var (
	errorCount = expvar.NewInt("log_watcher_error_count")
)

type watch struct {
	ps []Processor
	fi os.FileInfo
}

// LogWatcher implements a Watcher for watching real filesystems.
type LogWatcher struct {
	watcher    *fsnotify.Watcher
	pollTicker *time.Ticker

	watchedMu sync.RWMutex // protects `watched'
	watched   map[string]*watch

	stopTicks chan struct{} // Channel to notify ticker to stop.

	ticksDone  chan struct{} // Channel to notify when the ticks handler is done.
	eventsDone chan struct{} // Channel to notify when the events handler is done.

	closeOnce sync.Once
}

// NewLogWatcher returns a new LogWatcher, or returns an error.
func NewLogWatcher(pollInterval time.Duration, enableFsnotify bool) (*LogWatcher, error) {
	var f *fsnotify.Watcher
	if enableFsnotify {
		var err error
		f, err = fsnotify.NewWatcher()
		if err != nil {
			glog.Warning(err)
		}
	}
	if f == nil && pollInterval == 0 {
		glog.Infof("fsnotify disabled and no poll interval specified; defaulting to 250ms poll")
		pollInterval = time.Millisecond * 250
	}
	w := &LogWatcher{
		watcher: f,
		watched: make(map[string]*watch),
	}
	if pollInterval > 0 {
		w.pollTicker = time.NewTicker(pollInterval)
		w.stopTicks = make(chan struct{})
		w.ticksDone = make(chan struct{})
		go w.runTicks()
	}
	if f != nil {
		w.eventsDone = make(chan struct{})
		go w.runEvents()
	}
	return w, nil
}

func (w *LogWatcher) sendEvent(e Event) {
	w.watchedMu.RLock()
	watch, ok := w.watched[e.Pathname]
	w.watchedMu.RUnlock()
	if !ok {
		d := filepath.Dir(e.Pathname)
		w.watchedMu.RLock()
		watch, ok = w.watched[d]
		w.watchedMu.RUnlock()
		if !ok {
			glog.V(2).Infof("No watch for path %q", e.Pathname)
			return
		}
	}
	w.sendWatchedEvent(watch, e)
}

// Send an event to a watch; all locks assumed to be held.
func (w *LogWatcher) sendWatchedEvent(watch *watch, e Event) {
	for _, p := range watch.ps {
		p.ProcessFileEvent(context.TODO(), e)
	}
}

func (w *LogWatcher) runTicks() {
	defer close(w.ticksDone)

	if w.pollTicker == nil {
		return
	}

	for {
		select {
		case <-w.pollTicker.C:
			w.watchedMu.RLock()
			for n, watch := range w.watched {
				w.watchedMu.RUnlock()
				w.pollWatchedPath(n, watch)
				w.watchedMu.RLock()
			}
			w.watchedMu.RUnlock()
		case <-w.stopTicks:
			w.pollTicker.Stop()
			return
		}
	}
}

// pollWatchedPathLocked polls an already-watched path for updates.
func (w *LogWatcher) pollWatchedPath(pathname string, watched *watch) {
	glog.V(2).Info("stat")
	fi, err := os.Stat(pathname)
	if err != nil {
		glog.V(1).Info(err)
		return
	}

	// fsnotify does not send update events for the directory itself.
	if fi.IsDir() {
		w.pollDirectory(watched, pathname)
	} else if watched.fi == nil || fi.ModTime().Sub(watched.fi.ModTime()) > 0 {
		glog.V(2).Infof("sending update for %s", pathname)
		w.sendWatchedEvent(watched, Event{Update, pathname})
	}

	glog.V(2).Info("Update fi")
	w.watchedMu.Lock()
	if _, ok := w.watched[pathname]; ok {
		w.watched[pathname].fi = fi
	}
	w.watchedMu.Unlock()
}

func (w *LogWatcher) pollDirectory(parentWatch *watch, pathname string) {
	matches, err := filepath.Glob(path.Join(pathname, "*"))
	if err != nil {
		glog.V(1).Info(err)
		return
	}
	// TODO(jaq): how do we avoid duplicate notifies for things that are already in the watch list?
	for _, match := range matches {
		fi, err := os.Stat(match)
		if err != nil {
			glog.V(1).Info(err)
			continue
		}

		w.watchedMu.RLock()
		watched, ok := w.watched[match]
		w.watchedMu.RUnlock()
		switch {
		case !ok:
			glog.V(2).Infof("sending create for %s", match)
			w.sendWatchedEvent(parentWatch, Event{Create, match})
			w.watchedMu.Lock()
			w.watched[match] = &watch{ps: parentWatch.ps, fi: fi}
			w.watchedMu.Unlock()
		case watched.fi != nil && fi.ModTime().Sub(watched.fi.ModTime()) > 0:
			glog.V(2).Infof("sending update for %s", match)
			w.sendWatchedEvent(watched, Event{Update, match})
			w.watchedMu.Lock()
			w.watched[match].fi = fi
			w.watchedMu.Unlock()
		default:
			glog.V(2).Infof("No modtime change for %s, no send", match)
		}
		if fi.IsDir() {
			w.pollDirectory(parentWatch, match)
		}
	}
}

// runEvents assumes that w.watcher is not nil
func (w *LogWatcher) runEvents() {
	defer close(w.eventsDone)

	// Suck out errors and dump them to the error log.
	go func() {
		for err := range w.watcher.Errors {
			errorCount.Add(1)
			glog.Errorf("fsnotify error: %s\n", err)
		}
	}()

	for e := range w.watcher.Events {
		glog.V(2).Infof("watcher event %v", e)
		switch {
		case e.Op&fsnotify.Create == fsnotify.Create:
			w.sendEvent(Event{Create, e.Name})
		case e.Op&fsnotify.Write == fsnotify.Write,
			e.Op&fsnotify.Chmod == fsnotify.Chmod:
			w.sendEvent(Event{Update, e.Name})
		case e.Op&fsnotify.Remove == fsnotify.Remove:
			w.sendEvent(Event{Delete, e.Name})
		case e.Op&fsnotify.Rename == fsnotify.Rename:
			// Rename is only issued on the original file path; the new name receives a Create event
			w.sendEvent(Event{Delete, e.Name})
		default:
			panic(fmt.Sprintf("unknown op type %v", e.Op))
		}
	}
	glog.Infof("Shutting down log watcher.")
}

// Close shuts down the LogWatcher.  It is safe to call this from multiple clients.
func (w *LogWatcher) Close() (err error) {
	w.closeOnce.Do(func() {
		if w.watcher != nil {
			err = w.watcher.Close()
			<-w.eventsDone
		}
		if w.pollTicker != nil {
			close(w.stopTicks)
			<-w.ticksDone
		}
		glog.Info("Closing events channels")
	})
	return nil
}

// Observe adds a path to the list of watched items.
// If this path has a new event, then the processor being registered will be sent the event.
func (w *LogWatcher) Observe(path string, processor Processor) error {
	absPath, err := w.addWatch(path)
	if err != nil {
		return err
	}
	w.watchedMu.Lock()
	defer w.watchedMu.Unlock()
	watched, ok := w.watched[absPath]
	if !ok {
		w.watched[absPath] = &watch{ps: []Processor{processor}}
		glog.Infof("No abspath in watched list, added new one for %s", absPath)
		return nil
	}
	for _, p := range watched.ps {
		if p == processor {
			glog.Infof("Found this processor in watched list")
			return nil
		}
	}
	watched.ps = append(watched.ps, processor)
	glog.Infof("appended this processor")
	return nil

}

func (w *LogWatcher) addWatch(path string) (string, error) {
	absPath, err := filepath.Abs(path)
	if err != nil {
		return "", errors.Wrapf(err, "Failed to lookup absolutepath of %q", path)
	}
	glog.V(2).Infof("Adding a watch on resolved path %q", absPath)
	if w.watcher != nil {
		err = w.watcher.Add(absPath)
		if err != nil {
			if os.IsPermission(err) {
				glog.V(2).Infof("Skipping permission denied error on adding a watch.")
			} else {
				return "", errors.Wrapf(err, "Failed to create a new watch on %q", absPath)
			}
		}
	}
	return absPath, nil
}

// IsWatching indicates if the path is being watched. It includes both
// filenames and directories.
func (w *LogWatcher) IsWatching(path string) bool {
	absPath, err := filepath.Abs(path)
	if err != nil {
		glog.V(2).Infof("Couldn't resolve path %q: %s", absPath, err)
		return false
	}
	glog.V(2).Infof("Resolved path for lookup %q", absPath)
	w.watchedMu.RLock()
	_, ok := w.watched[absPath]
	w.watchedMu.RUnlock()
	return ok
}

func (w *LogWatcher) Unobserve(path string, processor Processor) error {
	w.watchedMu.Lock()
	defer w.watchedMu.Unlock()
	_, ok := w.watched[path]
	if !ok {
		return nil
	}

	for i, p := range w.watched[path].ps {
		if p == processor {
			w.watched[path].ps = append(w.watched[path].ps[0:i], w.watched[path].ps[i+1:]...)
			break
		}
	}
	if len(w.watched[path].ps) == 0 {
		delete(w.watched, path)
	}
	if w.watcher != nil {
		return w.watcher.Remove(path)
	}
	return nil
}
<|endoftext|>


<commit_before>package main

import(
	"crypto/md5"
	"database/sql"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"strings"
	_ "github.com/mattn/go-sqlite3"
)

var(
	verp = flag.Bool("version", false, "Show version info")
	facetp = flag.Bool("facets", false, "List facets")
	scanp = flag.Bool("scan", false, "Check for new/modified files and sweep db for orphan records")
	cleanp = flag.Bool("cleandb", false, "Just clean db (no file scan)")
	tagp = flag.Bool("tag", false, "Tag [dir] with [facet]")
	getp = flag.Bool("get", false, "Get filenames for tracks tagged with [facet]")
	mdflag = flag.String("musicdir", "", "Set location of your mpd music directory")
	musicdir = ""
	seen = 0
)

func init() {
	flag.Parse()
	config := os.Getenv("HOME") + "/.mpcf"
	if *mdflag != "" {
		err := ioutil.WriteFile(config, []byte(*mdflag), 0644)
		if err != nil {
			log.Fatal(err)
		}
		musicdir = *mdflag
	} else {
		mdbytes, err := ioutil.ReadFile(config)
		if err != nil {
			log.Fatal("Please run 'mpcf -musicdir /path/to/music' to set your musicdir path.")
		}
		musicdir = string(mdbytes)
	}
}

func main() {
	db, err := sql.Open("sqlite3", musicdir + "/.mpcf.db")
	if err != nil {
		log.Fatal(err)
	}
	defer db.Close()

	if *verp {
		fmt.Println("This is mpcf v0.5.2")
		os.Exit(0)
	}
	if *scanp {
		db.Exec("PRAGMA synchronous=0")
		scandir("", db)
		cleandb(db)
		os.Exit(0)
	}
	if *cleanp {
		cleandb(db)
		os.Exit(0)
	}
	if *tagp {
		tagdir(flag.Args(), db)
		os.Exit(0)
	}
	if *getp {
		getfacettracks(flag.Args(), db)
		os.Exit(0)
	}
	if *facetp {
		lsfacets(db)
		os.Exit(0)
	}

	// create db if needed
	var tracks int
	res := db.QueryRow("select count(id) from tracks")
	err = res.Scan(&tracks)
	if err != nil {
		log.Println("Creating db")
		createdb(db)
		log.Println("Updating track list")
		db.Exec("PRAGMA synchronous=0")
		scandir("", db)
	} else {
		var tags, facets int
		db.QueryRow("select count(tid) from t2f").Scan(&tags)
		db.QueryRow("select count(id) from facets").Scan(&facets)
		fmt.Printf("%v tracks; %v tagged, with %v facets\n", tracks, tags, facets)
	}
}

func lsfacets(db *sql.DB) {
	rows, err := db.Query("SELECT facet FROM facets ORDER BY facet")
	if err != nil {
		log.Fatal(err)
	}
	defer rows.Close()
	var f string
	for rows.Next() {
		if err := rows.Scan(&f); err != nil {
			log.Fatal(err)
		}
		fmt.Println(f)
	}
	if err := rows.Err(); err != nil {
		log.Fatal(err)
	}
}

func getfacettracks(args []string, db *sql.DB) {
	if len(args) != 1 {
		log.Fatal("Too many/few arguments to -get; need a facet name")
	}
	var fid int
	db.QueryRow("select id from facets where facet = ?", args[0]).Scan(&fid)
	if fid == 0 {
		return
	}
	rows, err := db.Query("SELECT filename FROM tracks WHERE id IN (SELECT DISTINCT tid FROM t2f WHERE fid = ?)", fid)
	if err != nil {
		log.Fatal(err)
	}
	defer rows.Close()
	var name string
	for rows.Next() {
		if err := rows.Scan(&name); err != nil {
			log.Fatal(err)
		}
		fmt.Println(name)
	}
	if err := rows.Err(); err != nil {
		log.Fatal(err)
	}
}

func tagdir(args []string, db *sql.DB) {
	if len(args) != 2 {
		log.Fatal("Too many/few arguments to -tag; need a directory and a facet")
	}
	// create the tag if it doesn't exist
	var fid int
	db.QueryRow("select id from facets where facet = ?", args[1]).Scan(&fid)
	if fid == 0 {
		db.Exec("insert into facets (facet) values (?)", args[1])
		db.QueryRow("select id from facets where facet = ?", args[1]).Scan(&fid)
	}
	// now actually tag tracks under this dir
	args[0] = strings.TrimRight(args[0], "/")
	args[0] = strings.TrimLeft(args[0], "./")
	args[0] = strings.TrimLeft(args[0], musicdir)
	tagdir2(args[0], fid, db)
}

func tagdir2(dir string, fid int, db *sql.DB) {
	err := os.Chdir(musicdir + "/" + dir)
	if err != nil {
		log.Fatalf("Can't chdir to %v", dir)
	}
	ls, err := ioutil.ReadDir(".")
	for _, direntry := range ls {
		name := dir + "/" + direntry.Name()
		if direntry.IsDir() {
			tagdir2(name, fid, db)
		} else {
			var tid, fcnt int
			db.QueryRow("select id from tracks where filename = ?", name).Scan(&tid)
			db.QueryRow("select count(tid) from t2f where tid = ? and fid = ?", tid, fid).Scan(&fcnt)
			if fcnt > 0 {
				continue
			}
			db.Exec("insert into t2f (tid, fid) values (?, ?)", tid, fid)
		}
	}
}

func createdb(db *sql.DB) {
	var err error
	var stmts = []string{
		"create table tracks (id integer primary key, filename text unique, hash text unique)",
		"create table facets (id integer primary key, facet text)",
		"create table t2f (tid integer, fid integer)",
		"create index fididx on t2f(fid)",
		"create table config (key text, value text)",
		"insert into config (key, value) values('mpdconf', '/etc/mpd.conf')",
	}
	for _, stmt := range stmts {
		if err != nil {
			break
		}
		_, err = db.Exec(stmt)
	}
	if err != nil {
		log.Fatal(err)
	}
}

func scandir(dir string, db *sql.DB) {
	os.Chdir(musicdir + "/" + dir)
	ls, err := ioutil.ReadDir(".")
	if err != nil {
		log.Fatal(err, dir)
	}
	for _, direntry := range ls {
		if direntry.IsDir() {
			if dir == "" {
				scandir(direntry.Name(), db)
			} else {
				scandir(dir + "/" + direntry.Name(), db)
			}
		} else {
			seen ++
			if seen % 100 == 0 {
				log.Printf("Processed %v tracks\n", seen)
			}
			name := dir + "/" + direntry.Name()
			md5 := fmt.Sprintf("%x", calcMD5(direntry.Name()))
			// _, err := db.Exec("INSERT OR REPLACE INTO tracks (filename, hash) VALUES(COALESCE((SELECT filename FROM tracks WHERE filename = ?),?), COALESCE((SELECT hash FROM tracks WHERE hash = ?), ?))", name, name, md5, md5)
			_, err := db.Exec("INSERT OR IGNORE INTO tracks (filename, hash) VALUES(?, ?)", name, md5)
			if err != nil {
				log.Fatal(err)
			}
			_, err = db.Exec("UPDATE tracks SET filename = ?, hash = ? WHERE filename = ?", name, md5, name)
			if err != nil {
				log.Fatal(err)
			}
		}
	}
}

func cleandb(db *sql.DB) {
	log.Printf("Scanning db for orphaned records")
	rows, err := db.Query("SELECT id, filename FROM tracks")
	if err != nil {
		log.Fatal(err)
	}
	defer rows.Close()
	var id int64
	var name string
	for rows.Next() {
		if err := rows.Scan(&id, &name); err != nil {
			log.Fatal(err)
		}
		_, err = os.Stat(musicdir + "/" + name)
		if err == nil {
			continue
		}
		// remove track entry
		_, err = db.Exec("delete from tracks where id = ?", id)
		if err != nil {
			log.Fatal(err)
		}
		// remove tag links
		_, err = db.Exec("delete from t2f where tid = ?", id)
		if err != nil {
			log.Fatal(err)
		}
	}
	if err := rows.Err(); err != nil {
		log.Fatal(err)
	}
	_, err = db.Exec("vacuum")
}

func calcMD5(filename string) []byte {
	file, err := os.Open(filename)
	if err != nil {
		log.Fatal(err)
	}
	defer file.Close()

	hash := md5.New()
	if _, err := io.CopyN(hash, file, 262144); err != nil && err != io.EOF {
		log.Fatal(err)
	}
	return hash.Sum(nil)
}
<commit_msg>tweaks and better messaging<commit_after>package main

import(
	"crypto/md5"
	"database/sql"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"strings"
	_ "github.com/mattn/go-sqlite3"
)

var(
	verp = flag.Bool("version", false, "Show version info")
	facetp = flag.Bool("facets", false, "List facets")
	scanp = flag.Bool("scan", false, "Check for new/modified files and sweep db for orphan records")
	cleanp = flag.Bool("cleandb", false, "Just clean db (no file scan)")
	tagp = flag.Bool("tag", false, "Tag [dir] with [facet]")
	getp = flag.Bool("get", false, "Get filenames for tracks tagged with [facet]")
	mdflag = flag.String("musicdir", "", "Set location of your mpd music directory")
	musicdir = ""
	seen      int64
	touched   int64
)

func init() {
	flag.Parse()
	config := os.Getenv("HOME") + "/.mpcf"
	if *mdflag != "" {
		err := ioutil.WriteFile(config, []byte(*mdflag), 0644)
		if err != nil {
			log.Fatal(err)
		}
		musicdir = *mdflag
	} else {
		mdbytes, err := ioutil.ReadFile(config)
		if err != nil {
			log.Fatal("Please run 'mpcf -musicdir /path/to/music' to set your musicdir path.")
		}
		musicdir = string(mdbytes)
	}
}

func main() {
	db, err := sql.Open("sqlite3", musicdir + "/.mpcf.db")
	if err != nil {
		log.Fatal(err)
	}
	defer db.Close()
	// create db if needed
	var tracks int
	res := db.QueryRow("select count(id) from tracks")
	err = res.Scan(&tracks)
	if err != nil {
		db.Exec("PRAGMA synchronous=0")
		log.Println("Creating db")
		createdb(db)
		log.Println("Updating track list")
		scandir("", db)
	}

	if *verp {
		fmt.Println("This is mpcf v0.5.3")
		os.Exit(0)
	}
	if *scanp {
		db.Exec("PRAGMA synchronous=0")
		scandir("", db)
		cleandb(db)
		os.Exit(0)
	}
	if *cleanp {
		cleandb(db)
		os.Exit(0)
	}
	if *tagp {
		tagdir(flag.Args(), db)
		os.Exit(0)
	}
	if *getp {
		getfacettracks(flag.Args(), db)
		os.Exit(0)
	}
	if *facetp {
		lsfacets(db)
		os.Exit(0)
	}

	var tags, facets int
	db.QueryRow("select count(tid) from t2f").Scan(&tags)
	db.QueryRow("select count(id) from facets").Scan(&facets)
	fmt.Printf("%v tracks; %v tagged, with %v facets\n", tracks, tags, facets)
}

func lsfacets(db *sql.DB) {
	rows, err := db.Query("SELECT facet FROM facets ORDER BY facet")
	if err != nil {
		log.Fatal(err)
	}
	defer rows.Close()
	var f string
	for rows.Next() {
		if err := rows.Scan(&f); err != nil {
			log.Fatal(err)
		}
		fmt.Println(f)
	}
	if err := rows.Err(); err != nil {
		log.Fatal(err)
	}
}

func getfacettracks(args []string, db *sql.DB) {
	if len(args) != 1 {
		log.Fatal("Too many/few arguments to -get; need a facet name")
	}
	var fid int
	db.QueryRow("select id from facets where facet = ?", args[0]).Scan(&fid)
	if fid == 0 {
		return
	}
	rows, err := db.Query("SELECT filename FROM tracks WHERE id IN (SELECT DISTINCT tid FROM t2f WHERE fid = ?)", fid)
	if err != nil {
		log.Fatal(err)
	}
	defer rows.Close()
	var name string
	for rows.Next() {
		if err := rows.Scan(&name); err != nil {
			log.Fatal(err)
		}
		fmt.Println(name)
	}
	if err := rows.Err(); err != nil {
		log.Fatal(err)
	}
}

func tagdir(args []string, db *sql.DB) {
	if len(args) != 2 {
		log.Fatal("Too many/few arguments to -tag; need a directory and a facet")
	}
	// create the tag if it doesn't exist
	var fid int
	db.QueryRow("select id from facets where facet = ?", args[1]).Scan(&fid)
	if fid == 0 {
		db.Exec("insert into facets (facet) values (?)", args[1])
		db.QueryRow("select id from facets where facet = ?", args[1]).Scan(&fid)
	}
	// now actually tag tracks under this dir
	args[0] = strings.TrimRight(args[0], "/")
	args[0] = strings.TrimLeft(args[0], "./")
	args[0] = strings.TrimLeft(args[0], musicdir)
	tagdir2(args[0], fid, db)
}

func tagdir2(dir string, fid int, db *sql.DB) {
	err := os.Chdir(musicdir + "/" + dir)
	if err != nil {
		log.Fatalf("Can't chdir to %v", dir)
	}
	ls, err := ioutil.ReadDir(".")
	for _, direntry := range ls {
		name := dir + "/" + direntry.Name()
		if direntry.IsDir() {
			tagdir2(name, fid, db)
		} else {
			var tid, fcnt int
			db.QueryRow("select id from tracks where filename = ?", name).Scan(&tid)
			db.QueryRow("select count(tid) from t2f where tid = ? and fid = ?", tid, fid).Scan(&fcnt)
			if fcnt > 0 {
				continue
			}
			db.Exec("insert into t2f (tid, fid) values (?, ?)", tid, fid)
		}
	}
}

func createdb(db *sql.DB) {
	var err error
	var stmts = []string{
		"create table tracks (id integer primary key, filename text unique, hash text unique)",
		"create table facets (id integer primary key, facet text)",
		"create table t2f (tid integer, fid integer)",
		"create index fididx on t2f(fid)",
		"create table config (key text, value text)",
		"insert into config (key, value) values('mpdconf', '/etc/mpd.conf')",
	}
	for _, stmt := range stmts {
		if err != nil {
			break
		}
		_, err = db.Exec(stmt)
	}
	if err != nil {
		log.Fatal(err)
	}
}

func scandir(dir string, db *sql.DB) {
	os.Chdir(musicdir + "/" + dir)
	ls, err := ioutil.ReadDir(".")
	if err != nil {
		log.Fatal(err, dir)
	}
	for _, direntry := range ls {
		if direntry.IsDir() {
			if dir == "" {
				scandir(direntry.Name(), db)
			} else {
				scandir(dir + "/" + direntry.Name(), db)
			}
		} else {
			seen ++
			if seen % 100 == 0 {
				log.Printf("Processed %v tracks; updated %v\n", seen, touched)
			}
			name := dir + "/" + direntry.Name()
			md5 := fmt.Sprintf("%x", calcMD5(direntry.Name()))
			// _, err := db.Exec("INSERT OR REPLACE INTO tracks (filename, hash) VALUES(COALESCE((SELECT filename FROM tracks WHERE filename = ?),?), COALESCE((SELECT hash FROM tracks WHERE hash = ?), ?))", name, name, md5, md5)
			r, err := db.Exec("INSERT OR IGNORE INTO tracks (filename, hash) VALUES(?, ?)", name, md5)
			if err != nil {
				log.Fatal(err)
			}
			touch, _ := r.RowsAffected()
			touched += touch
			//r, err = db.Exec("UPDATE tracks SET filename = ?, hash = ? WHERE filename = ?", name, md5, name)
			//if err != nil {
			//	log.Fatal(err)
			//}
		}
	}
}

func cleandb(db *sql.DB) {
	log.Printf("Scanning db for orphaned records")
	rows, err := db.Query("SELECT id, filename FROM tracks")
	if err != nil {
		log.Fatal(err)
	}
	defer rows.Close()
	var id int64
	var name string
	for rows.Next() {
		if err := rows.Scan(&id, &name); err != nil {
			log.Fatal(err)
		}
		_, err = os.Stat(musicdir + "/" + name)
		if err == nil {
			continue
		}
		// remove track entry
		_, err = db.Exec("delete from tracks where id = ?", id)
		if err != nil {
			log.Fatal(err)
		}
		// remove tag links
		_, err = db.Exec("delete from t2f where tid = ?", id)
		if err != nil {
			log.Fatal(err)
		}
		log.Printf("Removed orphan record for %v\n", name)
	}
	if err := rows.Err(); err != nil {
		log.Fatal(err)
	}
	_, err = db.Exec("vacuum")
}

func calcMD5(filename string) []byte {
	file, err := os.Open(filename)
	if err != nil {
		log.Fatal(err)
	}
	defer file.Close()

	hash := md5.New()
	if _, err := io.CopyN(hash, file, 524288); err != nil && err != io.EOF {
		log.Fatal(err)
	}
	return hash.Sum(nil)
}
<|endoftext|>


<commit_before>package init

import (
	"os"
	"os/exec"
	"syscall"

	log "github.com/Sirupsen/logrus"
	"github.com/rancherio/os/config"
	"github.com/rancherio/os/docker"
	"github.com/rancherio/os/util"
	"github.com/rancherio/rancher-compose/project"
)

func autoformat(cfg *config.Config) error {
	if len(cfg.State.Autoformat) == 0 || util.ResolveDevice(cfg.State.Dev) != "" {
		return nil
	}

	var format string

outer:
	for _, dev := range cfg.State.Autoformat {
		log.Infof("Checking %s to auto-format", dev)
		if _, err := os.Stat(dev); os.IsNotExist(err) {
			continue
		}

		f, err := os.Open(dev)
		if err != nil {
			return err
		}
		defer f.Close()

		buffer := make([]byte, 1048576, 1048576)
		c, err := f.Read(buffer)
		if err != nil {
			return err
		}

		if c != 1048576 {
			log.Infof("%s not right size", dev)
			continue
		}

		for _, b := range buffer {
			if b != 0 {
				log.Infof("%s not empty", dev)
				continue outer
			}
		}

		format = dev
		break
	}

	if format != "" {
		log.Infof("Auto formatting : %s", format)
		return docker.RunServices("autoformat", cfg, map[string]*project.ServiceConfig{
			"autoformat": {
				Net:        "none",
				Privileged: true,
				Image:      "autoformat",
				Command:    format,
			},
			"udev": cfg.BootstrapContainers["udev"],
		})
	}

	return nil
}

func runBootstrapContainers(cfg *config.Config) error {
	return docker.RunServices("bootstrap", cfg, cfg.BootstrapContainers)
}

func startDocker(cfg *config.Config) (chan interface{}, error) {
	for _, d := range []string{config.DOCKER_SYSTEM_HOST, "/var/run"} {
		err := os.MkdirAll(d, 0700)
		if err != nil {
			return nil, err
		}
	}

	cmd := exec.Command(cfg.BootstrapDocker.Args[0], cfg.BootstrapDocker.Args[1:]...)
	if cfg.Debug {
		cmd.Stdout = os.Stdout
		cmd.Stderr = os.Stderr
	}
	err := cmd.Start()
	if err != nil {
		return nil, err
	}

	c := make(chan interface{})
	go func() {
		<-c
		cmd.Process.Signal(syscall.SIGTERM)
		cmd.Wait()
		c <- struct{}{}
	}()

	return c, nil
}

func stopDocker(c chan interface{}) error {
	c <- struct{}{}
	<-c

	return os.RemoveAll(config.DOCKER_SYSTEM_HOME)
}

func bootstrap(cfg *config.Config) error {
	log.Info("Starting bootstrap")
	c, err := startDocker(cfg)
	if err != nil {
		return err
	}

	initFuncs := []config.InitFunc{
		loadImages,
		runBootstrapContainers,
		autoformat,
	}

	defer stopDocker(c)

	return config.RunInitFuncs(cfg, initFuncs)
}
<commit_msg>Fix autoformat<commit_after>package init

import (
	"os"
	"os/exec"
	"syscall"

	log "github.com/Sirupsen/logrus"
	"github.com/rancherio/os/config"
	"github.com/rancherio/os/docker"
	"github.com/rancherio/os/util"
	"github.com/rancherio/rancher-compose/project"
)

func autoformat(cfg *config.Config) error {
	if len(cfg.State.Autoformat) == 0 || util.ResolveDevice(cfg.State.Dev) != "" {
		return nil
	}

	var format string

outer:
	for _, dev := range cfg.State.Autoformat {
		log.Infof("Checking %s to auto-format", dev)
		if _, err := os.Stat(dev); os.IsNotExist(err) {
			continue
		}

		f, err := os.Open(dev)
		if err != nil {
			return err
		}
		defer f.Close()

		buffer := make([]byte, 1048576, 1048576)
		c, err := f.Read(buffer)
		if err != nil {
			return err
		}

		if c != 1048576 {
			log.Infof("%s not right size", dev)
			continue
		}

		for _, b := range buffer {
			if b != 0 {
				log.Infof("%s not empty", dev)
				continue outer
			}
		}

		format = dev
		break
	}

	if format != "" {
		log.Infof("Auto formatting : %s", format)

		// copy
		udev := *cfg.BootstrapContainers["udev"]
		udev.Links = append(udev.Links, "autoformat")
		udev.LogDriver = "json-file"

		err := docker.RunServices("autoformat", cfg, map[string]*project.ServiceConfig{
			"autoformat": {
				Net:        "none",
				Privileged: true,
				Image:      "autoformat",
				Command:    format,
				Labels: []string{
					config.DETACH + "=false",
					config.SCOPE + "=" + config.SYSTEM,
				},
				LogDriver: "json-file",
			},
			"udev": &udev,
		})

		return err
	}

	return nil
}

func runBootstrapContainers(cfg *config.Config) error {
	return docker.RunServices("bootstrap", cfg, cfg.BootstrapContainers)
}

func startDocker(cfg *config.Config) (chan interface{}, error) {
	for _, d := range []string{config.DOCKER_SYSTEM_HOST, "/var/run"} {
		err := os.MkdirAll(d, 0700)
		if err != nil {
			return nil, err
		}
	}

	cmd := exec.Command(cfg.BootstrapDocker.Args[0], cfg.BootstrapDocker.Args[1:]...)
	if cfg.Debug {
		cmd.Stdout = os.Stdout
		cmd.Stderr = os.Stderr
	}
	err := cmd.Start()
	if err != nil {
		return nil, err
	}

	c := make(chan interface{})
	go func() {
		<-c
		cmd.Process.Signal(syscall.SIGTERM)
		cmd.Wait()
		c <- struct{}{}
	}()

	return c, nil
}

func stopDocker(c chan interface{}) error {
	c <- struct{}{}
	<-c

	return os.RemoveAll(config.DOCKER_SYSTEM_HOME)
}

func bootstrap(cfg *config.Config) error {
	log.Info("Starting bootstrap")
	c, err := startDocker(cfg)
	if err != nil {
		return err
	}

	initFuncs := []config.InitFunc{
		loadImages,
		runBootstrapContainers,
		autoformat,
	}

	defer stopDocker(c)

	return config.RunInitFuncs(cfg, initFuncs)
}
<|endoftext|>


<commit_before>package plugin

import (
	"fmt"
	"github.com/mattn/go-shellwords"
	"os"
	"os/exec"
	"syscall"
)

const NOPIPE = 0
const STDIN = 1
const STDOUT = 2

type ExecOptions struct {
	Stdout   *os.File
	Stdin    *os.File
	Stderr   *os.File
	Cmd      string
	ExpectRC []int
}

func ExecWithOptions(opts ExecOptions) error {
	cmdArgs, err := shellwords.Parse(opts.Cmd)
	if err != nil {
		return ExecFailure{Err: fmt.Sprintf("Could not parse '%s' into exec-able command: %s", opts.Cmd, err.Error())}
	}
	DEBUG("Executing '%s' with arguments %v", cmdArgs[0], cmdArgs[1:])

	cmd := exec.Command(cmdArgs[0], cmdArgs[1:]...)
	if opts.Stdout != nil {
		cmd.Stdout = opts.Stdout
	}
	if opts.Stderr != nil {
		cmd.Stderr = opts.Stderr
	}
	if opts.Stdin != nil {
		cmd.Stdin = opts.Stdin
	}

	if len(opts.ExpectRC) == 0 {
		opts.ExpectRC = []int{0}
	}

	err = cmd.Run()
	if err != nil {
		if exitErr, ok := err.(*exec.ExitError); ok {
			sys := exitErr.ProcessState.Sys()
			if rc, ok := sys.(syscall.WaitStatus); ok {
				code := rc.ExitStatus()
				for _, expect := range opts.ExpectRC {
					if code == expect {
						return nil
					}
				}
			}
		}
		return ExecFailure{Err: fmt.Sprintf("Unable to exec '%s': %s", cmdArgs[0], err.Error())}
	}
	return nil
}

func Exec(cmdString string, flags int) error {
	opts := ExecOptions{
		Cmd:    cmdString,
		Stderr: os.Stderr,
	}

	if flags&STDOUT == STDOUT {
		opts.Stdout = os.Stdout
	}
	if flags&STDIN == STDIN {
		opts.Stdin = os.Stdin
	}

	return ExecWithOptions(opts)
}
<commit_msg>Commented rc validation, force errors on traps/signals/stops<commit_after>package plugin

import (
	"fmt"
	"github.com/mattn/go-shellwords"
	"os"
	"os/exec"
	"syscall"
)

const NOPIPE = 0
const STDIN = 1
const STDOUT = 2

type ExecOptions struct {
	Stdout   *os.File
	Stdin    *os.File
	Stderr   *os.File
	Cmd      string
	ExpectRC []int
}

func ExecWithOptions(opts ExecOptions) error {
	cmdArgs, err := shellwords.Parse(opts.Cmd)
	if err != nil {
		return ExecFailure{Err: fmt.Sprintf("Could not parse '%s' into exec-able command: %s", opts.Cmd, err.Error())}
	}
	DEBUG("Executing '%s' with arguments %v", cmdArgs[0], cmdArgs[1:])

	cmd := exec.Command(cmdArgs[0], cmdArgs[1:]...)
	if opts.Stdout != nil {
		cmd.Stdout = opts.Stdout
	}
	if opts.Stderr != nil {
		cmd.Stderr = opts.Stderr
	}
	if opts.Stdin != nil {
		cmd.Stdin = opts.Stdin
	}

	if len(opts.ExpectRC) == 0 {
		opts.ExpectRC = []int{0}
	}

	err = cmd.Run()
	if err != nil {
		// make sure we got an Exit error
		if exitErr, ok := err.(*exec.ExitError); ok {
			sys := exitErr.ProcessState.Sys()
			// os.ProcessState.Sys() may not return syscall.WaitStatus on non-UNIX machines,
			// so currently this feature only works on UNIX, but shouldn't crash on other OSes
			if rc, ok := sys.(syscall.WaitStatus); ok {
				code := rc.ExitStatus()
				// -1 indicates signals, stops, or traps, so force an error
				if code >= 0 {
					for _, expect := range opts.ExpectRC {
						if code == expect {
							return nil
						}
					}
				}
			}
		}
		return ExecFailure{Err: fmt.Sprintf("Unable to exec '%s': %s", cmdArgs[0], err.Error())}
	}
	return nil
}

func Exec(cmdString string, flags int) error {
	opts := ExecOptions{
		Cmd:    cmdString,
		Stderr: os.Stderr,
	}

	if flags&STDOUT == STDOUT {
		opts.Stdout = os.Stdout
	}
	if flags&STDIN == STDIN {
		opts.Stdin = os.Stdin
	}

	return ExecWithOptions(opts)
}
<|endoftext|>


<commit_before>package fsrateio

import (
	"fmt"
	"github.com/Symantec/Dominator/lib/format"
	"github.com/Symantec/Dominator/lib/rateio"
	"syscall"
)

type ReadMeasurer struct {
	blocksAtLastMeasurement uint64
}

func newReadMeasurer() *ReadMeasurer {
	var measurer ReadMeasurer
	return &measurer
}

func (measurer *ReadMeasurer) MeasureReadIO(bytesSinceLastMeasurement uint64) (
	uint64, error) {
	var rusage syscall.Rusage
	err := syscall.Getrusage(syscall.RUSAGE_SELF, &rusage)
	if err != nil {
		return 0, err
	}
	blocks := uint64(rusage.Inblock)
	blocksSinceLastMeasurement := blocks - measurer.blocksAtLastMeasurement
	measurer.blocksAtLastMeasurement = blocks
	return blocksSinceLastMeasurement, nil
}

func (measurer *ReadMeasurer) Reset() {
	var rusage syscall.Rusage
	syscall.Getrusage(syscall.RUSAGE_SELF, &rusage)
	measurer.blocksAtLastMeasurement = uint64(rusage.Inblock)
}

func newReaderContext(maxBytesPerSecond uint64, maxBlocksPerSecond uint64,
	speedPercent uint64) *ReaderContext {
	var ctx ReaderContext
	ctx.maxBytesPerSecond = maxBytesPerSecond
	ctx.maxBlocksPerSecond = maxBlocksPerSecond
	if maxBlocksPerSecond > 0 {
		ctx.ctx = rateio.NewReaderContext(maxBytesPerSecond, speedPercent,
			newReadMeasurer())
	} else {
		ctx.ctx = rateio.NewReaderContext(maxBytesPerSecond, speedPercent,
			&rateio.ReadMeasurer{})
	}
	return &ctx
}

func (ctx *ReaderContext) format() string {
	var blocksString string
	if ctx.maxBlocksPerSecond > 0 {
		blocksString = fmt.Sprintf("(%d blocks/s)", ctx.maxBlocksPerSecond)
	} else {
		blocksString = ""
	}
	speedPercent := uint64(ctx.GetContext().SpeedPercent())
	return fmt.Sprintf("max speed=%s/s%s limit=%d%% %s/s(%d blocks/s)",
		format.FormatBytes(ctx.maxBytesPerSecond), blocksString,
		speedPercent,
		format.FormatBytes(ctx.maxBytesPerSecond*speedPercent/100),
		ctx.maxBlocksPerSecond*speedPercent/100)
}
<commit_msg>Fix bug in recent fsrateio refactor.<commit_after>package fsrateio

import (
	"fmt"
	"github.com/Symantec/Dominator/lib/format"
	"github.com/Symantec/Dominator/lib/rateio"
	"syscall"
)

type ReadMeasurer struct {
	blocksAtLastMeasurement uint64
}

func newReadMeasurer() *ReadMeasurer {
	var measurer ReadMeasurer
	return &measurer
}

func (measurer *ReadMeasurer) MeasureReadIO(bytesSinceLastMeasurement uint64) (
	uint64, error) {
	var rusage syscall.Rusage
	err := syscall.Getrusage(syscall.RUSAGE_SELF, &rusage)
	if err != nil {
		return 0, err
	}
	blocks := uint64(rusage.Inblock)
	blocksSinceLastMeasurement := blocks - measurer.blocksAtLastMeasurement
	measurer.blocksAtLastMeasurement = blocks
	return blocksSinceLastMeasurement, nil
}

func (measurer *ReadMeasurer) Reset() {
	var rusage syscall.Rusage
	syscall.Getrusage(syscall.RUSAGE_SELF, &rusage)
	measurer.blocksAtLastMeasurement = uint64(rusage.Inblock)
}

func newReaderContext(maxBytesPerSecond uint64, maxBlocksPerSecond uint64,
	speedPercent uint64) *ReaderContext {
	var ctx ReaderContext
	ctx.maxBytesPerSecond = maxBytesPerSecond
	ctx.maxBlocksPerSecond = maxBlocksPerSecond
	if maxBlocksPerSecond > 0 {
		ctx.ctx = rateio.NewReaderContext(maxBlocksPerSecond, speedPercent,
			newReadMeasurer())
	} else {
		ctx.ctx = rateio.NewReaderContext(maxBytesPerSecond, speedPercent,
			&rateio.ReadMeasurer{})
	}
	return &ctx
}

func (ctx *ReaderContext) format() string {
	var blocksString string
	if ctx.maxBlocksPerSecond > 0 {
		blocksString = fmt.Sprintf("(%d blocks/s)", ctx.maxBlocksPerSecond)
	} else {
		blocksString = ""
	}
	speedPercent := uint64(ctx.GetContext().SpeedPercent())
	return fmt.Sprintf("max speed=%s/s%s limit=%d%% %s/s(%d blocks/s)",
		format.FormatBytes(ctx.maxBytesPerSecond), blocksString,
		speedPercent,
		format.FormatBytes(ctx.maxBytesPerSecond*speedPercent/100),
		ctx.maxBlocksPerSecond*speedPercent/100)
}
<|endoftext|>