<commit_before>package libvirt

import (
	"fmt"
	"log"
	"time"

	"github.com/hashicorp/terraform/helper/resource"
	"github.com/libvirt/libvirt-go"
)

const (
	volExistsID    = "EXISTS"
	volNotExistsID = "NOT-EXISTS"
)

// volumeExists returns "EXISTS" or "NOT-EXISTS" depending on the current volume existence
func volumeExists(virConn *libvirt.Connect, key string) resource.StateRefreshFunc {
	return func() (interface{}, string, error) {
		vol, err := virConn.LookupStorageVolByKey(key)
		if err != nil {
			if err.(libvirt.Error).Code == libvirt.ERR_NO_STORAGE_VOL {
				log.Printf("Volume %s does not exist", key)
				return virConn, "NOT-EXISTS", nil
			}
			log.Printf("Volume %s: error: %s", key, err.(libvirt.Error).Message)
		}
		defer vol.Free()
		return virConn, volExistsID, err
	}
}

// volumeWaitForExists waits for a storage volume to be up and timeout after 5 minutes.
func volumeWaitForExists(virConn *libvirt.Connect, key string) error {
	log.Printf("Waiting for volume %s to be active...", key)
	stateConf := &resource.StateChangeConf{
		Pending:    []string{volNotExistsID},
		Target:     []string{volExistsID},
		Refresh:    volumeExists(virConn, key),
		Timeout:    1 * time.Minute,
		MinTimeout: 3 * time.Second,
	}

	if _, err := stateConf.WaitForState(); err != nil {
		return fmt.Errorf("Error waiting for volume to reach EXISTS state: %s", err)
	}
	return nil
}

// volumeWaitDeleted waits for a storage volume to be removed
func volumeWaitDeleted(virConn *libvirt.Connect, key string) error {
	log.Printf("Waiting for volume %s to be deleted...", key)
	stateConf := &resource.StateChangeConf{
		Pending:    []string{volExistsID},
		Target:     []string{volNotExistsID},
		Refresh:    volumeExists(virConn, key),
		Timeout:    1 * time.Minute,
		MinTimeout: 3 * time.Second,
	}

	if _, err := stateConf.WaitForState(); err != nil {
		return fmt.Errorf("Error waiting for volume to reach NOT-EXISTS state: %s", err)
	}
	return nil
}

// volumeDelete removes the volume identified by `key` from libvirt
func volumeDelete(client *Client, key string) error {
	volume, err := client.libvirt.LookupStorageVolByKey(key)
	if err != nil {
		return fmt.Errorf("Can't retrieve volume %s: %v", key, err)
	}
	defer volume.Free()

	// Refresh the pool of the volume so that libvirt knows it is
	// not longer in use.
	volPool, err := volume.LookupPoolByVolume()
	if err != nil {
		return fmt.Errorf("Error retrieving pool for volume: %s", err)
	}
	defer volPool.Free()

	poolName, err := volPool.GetName()
	if err != nil {
		return fmt.Errorf("Error retrieving name of volume: %s", err)
	}

	client.poolMutexKV.Lock(poolName)
	defer client.poolMutexKV.Unlock(poolName)

	waitForSuccess("Error refreshing pool for volume", func() error {
		return volPool.Refresh(0)
	})

	// Workaround for redhat#1293804
	// https://bugzilla.redhat.com/show_bug.cgi?id=1293804#c12
	// Does not solve the problem but it makes it happen less often.
	_, err = volume.GetXMLDesc(0)
	if err != nil {
		return fmt.Errorf("Can't retrieve volume %s XML desc: %s", key, err)
	}

	err = volume.Delete(0)
	if err != nil {
		return fmt.Errorf("Can't delete volume %s: %s", key, err)
	}

	return volumeWaitDeleted(client.libvirt, key)
}

// tries really hard to find volume with `key`
// it will try to start the pool if it does not find it
//
// You have to call volume.Free() on the returned volume
func volumeLookupReallyHard(client *Client, volPoolName string, key string) (*libvirt.StorageVol, error) {
	virConn := client.libvirt
	if virConn == nil {
		return nil, fmt.Errorf(LibVirtConIsNil)
	}

	volume, err := virConn.LookupStorageVolByKey(key)
	if err != nil {
		virErr := err.(libvirt.Error)
		if virErr.Code != libvirt.ERR_NO_STORAGE_VOL {
			return nil, fmt.Errorf("Can't retrieve volume %s", key)
		}
		log.Printf("[INFO] Volume %s not found, attempting to start its pool", key)

		volPool, err := virConn.LookupStoragePoolByName(volPoolName)
		if err != nil {
			return nil, fmt.Errorf("Error retrieving pool %s for volume %s: %s", volPoolName, key, err)
		}
		defer volPool.Free()

		active, err := volPool.IsActive()
		if err != nil {
			return nil, fmt.Errorf("error retrieving status of pool %s for volume %s: %s", volPoolName, key, err)
		}
		if active {
			log.Printf("Can't retrieve volume %s (and pool is active)", key)
			return nil, nil
		}

		err = volPool.Create(0)
		if err != nil {
			return nil, fmt.Errorf("error starting pool %s: %s", volPoolName, err)
		}

		// attempt a new lookup
		volume, err = virConn.LookupStorageVolByKey(key)
		if err != nil {
			virErr := err.(libvirt.Error)
			if virErr.Code != libvirt.ERR_NO_STORAGE_VOL {
				return nil, fmt.Errorf("Can't retrieve volume %s", key)
			}
			// does not exist, but no error
			return nil, nil
		}
	}
	return volume, nil
}
<commit_msg>Make volume deletion more robust in case the volume was already deleted (#683)<commit_after>package libvirt

import (
	"fmt"
	"log"
	"time"

	"github.com/hashicorp/terraform/helper/resource"
	"github.com/libvirt/libvirt-go"
)

const (
	volExistsID    = "EXISTS"
	volNotExistsID = "NOT-EXISTS"
)

// volumeExists returns "EXISTS" or "NOT-EXISTS" depending on the current volume existence
func volumeExists(virConn *libvirt.Connect, key string) resource.StateRefreshFunc {
	return func() (interface{}, string, error) {
		vol, err := virConn.LookupStorageVolByKey(key)
		if err != nil {
			if err.(libvirt.Error).Code == libvirt.ERR_NO_STORAGE_VOL {
				log.Printf("Volume %s does not exist", key)
				return virConn, "NOT-EXISTS", nil
			}
			log.Printf("Volume %s: error: %s", key, err.(libvirt.Error).Message)
		}
		defer vol.Free()
		return virConn, volExistsID, err
	}
}

// volumeWaitForExists waits for a storage volume to be up and timeout after 5 minutes.
func volumeWaitForExists(virConn *libvirt.Connect, key string) error {
	log.Printf("Waiting for volume %s to be active...", key)
	stateConf := &resource.StateChangeConf{
		Pending:    []string{volNotExistsID},
		Target:     []string{volExistsID},
		Refresh:    volumeExists(virConn, key),
		Timeout:    1 * time.Minute,
		MinTimeout: 3 * time.Second,
	}

	if _, err := stateConf.WaitForState(); err != nil {
		return fmt.Errorf("Error waiting for volume to reach EXISTS state: %s", err)
	}
	return nil
}

// volumeWaitDeleted waits for a storage volume to be removed
func volumeWaitDeleted(virConn *libvirt.Connect, key string) error {
	log.Printf("Waiting for volume %s to be deleted...", key)
	stateConf := &resource.StateChangeConf{
		Pending:    []string{volExistsID},
		Target:     []string{volNotExistsID},
		Refresh:    volumeExists(virConn, key),
		Timeout:    1 * time.Minute,
		MinTimeout: 3 * time.Second,
	}

	if _, err := stateConf.WaitForState(); err != nil {
		return fmt.Errorf("Error waiting for volume to reach NOT-EXISTS state: %s", err)
	}
	return nil
}

// volumeDelete removes the volume identified by `key` from libvirt
func volumeDelete(client *Client, key string) error {
	volume, err := client.libvirt.LookupStorageVolByKey(key)
	if err != nil {
		return fmt.Errorf("Can't retrieve volume %s: %v", key, err)
	}
	defer volume.Free()

	// Refresh the pool of the volume so that libvirt knows it is
	// not longer in use.
	volPool, err := volume.LookupPoolByVolume()
	if err != nil {
		return fmt.Errorf("Error retrieving pool for volume: %s", err)
	}
	defer volPool.Free()

	poolName, err := volPool.GetName()
	if err != nil {
		return fmt.Errorf("Error retrieving name of volume: %s", err)
	}

	client.poolMutexKV.Lock(poolName)
	defer client.poolMutexKV.Unlock(poolName)

	waitForSuccess("Error refreshing pool for volume", func() error {
		return volPool.Refresh(0)
	})

	// Workaround for redhat#1293804
	// https://bugzilla.redhat.com/show_bug.cgi?id=1293804#c12
	// Does not solve the problem but it makes it happen less often.
	_, err = volume.GetXMLDesc(0)
	if err != nil {
		virErr := err.(libvirt.Error)
		if virErr.Code != libvirt.ERR_NO_STORAGE_VOL {
			return fmt.Errorf("Can't retrieve volume %s XML desc: %s", key, err)
		}
		// Volume is probably gone already, getting its XML description is pointless
	}

	err = volume.Delete(0)
	if err != nil {
		virErr := err.(libvirt.Error)
		if virErr.Code != libvirt.ERR_NO_STORAGE_VOL {
			return fmt.Errorf("Can't delete volume %s: %s", key, err)
		}
		// Volume is gone already
		return nil
	}

	return volumeWaitDeleted(client.libvirt, key)
}

// tries really hard to find volume with `key`
// it will try to start the pool if it does not find it
//
// You have to call volume.Free() on the returned volume
func volumeLookupReallyHard(client *Client, volPoolName string, key string) (*libvirt.StorageVol, error) {
	virConn := client.libvirt
	if virConn == nil {
		return nil, fmt.Errorf(LibVirtConIsNil)
	}

	volume, err := virConn.LookupStorageVolByKey(key)
	if err != nil {
		virErr := err.(libvirt.Error)
		if virErr.Code != libvirt.ERR_NO_STORAGE_VOL {
			return nil, fmt.Errorf("Can't retrieve volume %s", key)
		}
		log.Printf("[INFO] Volume %s not found, attempting to start its pool", key)

		volPool, err := virConn.LookupStoragePoolByName(volPoolName)
		if err != nil {
			return nil, fmt.Errorf("Error retrieving pool %s for volume %s: %s", volPoolName, key, err)
		}
		defer volPool.Free()

		active, err := volPool.IsActive()
		if err != nil {
			return nil, fmt.Errorf("error retrieving status of pool %s for volume %s: %s", volPoolName, key, err)
		}
		if active {
			log.Printf("Can't retrieve volume %s (and pool is active)", key)
			return nil, nil
		}

		err = volPool.Create(0)
		if err != nil {
			return nil, fmt.Errorf("error starting pool %s: %s", volPoolName, err)
		}

		// attempt a new lookup
		volume, err = virConn.LookupStorageVolByKey(key)
		if err != nil {
			virErr := err.(libvirt.Error)
			if virErr.Code != libvirt.ERR_NO_STORAGE_VOL {
				return nil, fmt.Errorf("Can't retrieve volume %s", key)
			}
			// does not exist, but no error
			return nil, nil
		}
	}
	return volume, nil
}
<|endoftext|>


<commit_before>/*
Copyright 2017 Google Inc.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// Package queue implements an update queue for use in testing a telemetry
// stream.
package queue

import (
	"fmt"
	"math/rand"
	"sync"
	"time"

	"github.com/golang/protobuf/proto"

	gpb "github.com/openconfig/gnmi/proto/gnmi"
	fpb "github.com/openconfig/gnmi/testing/fake/proto"
)

// Queue is a generic interface for getting the next element from either a
// FixedQeueue or UpdateQueue.
type Queue interface {
	Next() (interface{}, error)
}

// UpdateQueue is a structure that orders a set of device updates by their
// timestamps and repeatedly generates new pseudo-random updates based on
// a set of device path configurations.
type UpdateQueue struct {
	mu       sync.Mutex
	delay    bool
	duration time.Duration
	latest   int64
	q        [][]*value
	r        *rand.Rand
}

// New creates a new UpdateQueue. If delay is true, a call to Next()
// will invoke a sleep based on the duration between timestamps of the last
// returned update and the update to be returned by the current call. The
// values are the configuration for the updates stored in the queue.
func New(delay bool, seed int64, values []*fpb.Value) *UpdateQueue {
	if seed == 0 {
		seed = time.Now().UnixNano()
	}
	u := &UpdateQueue{delay: delay, r: rand.New(rand.NewSource(seed))}
	for _, v := range values {
		u.addValue(newValue(v, u.r))
	}
	return u
}

// Add inserts v into the queue in correct timestamp order.
func (u *UpdateQueue) Add(v *fpb.Value) {
	u.mu.Lock()
	defer u.mu.Unlock()
	u.addValue(newValue(v, u.r))
}

// Latest returns the maximum timestamp in the queue.
func (u *UpdateQueue) Latest() int64 {
	return u.latest
}

// Next returns the next update in the queue or an error.  If the queue is
// exhausted, a nil is returned for the update. The return will always be a
// *fpb.Value for proper type assertion.
func (u *UpdateQueue) Next() (interface{}, error) {
	u.mu.Lock()
	defer u.mu.Unlock()
	if len(u.q) == 0 {
		return nil, nil
	}
	// Incur a real delay if requested.
	if u.duration > 0 {
		time.Sleep(u.duration)
		u.duration = 0
	}
	v := u.q[0][0]
	// Make a copy of the current proto message to return.
	val := v.v
	// Update the value.
	if err := v.nextValue(); err != nil {
		return nil, err
	}
	var checkDelay bool
	// If this is the last update in the queue move to the next update set.
	// else move to the next update in the same update set.
	if len(u.q[0]) == 1 {
		u.q = u.q[1:]
		checkDelay = true
	} else {
		u.q[0] = u.q[0][1:]
	}
	// Add the updated value to the queue if the repeats are not exhausted.
	if v.v != nil {
		u.addValue(v)
	}
	// Set up a delay for the next retrieval if realtime delays are requested.
	if u.delay && checkDelay && len(u.q) > 0 {
		u.duration = time.Duration(u.q[0][0].v.Timestamp.Timestamp-val.Timestamp.Timestamp) * time.Nanosecond
	}
	return val, nil
}

func (u *UpdateQueue) addValue(v *value) {
	// If Timestamp was not provided create a default.
	if v.v.Timestamp == nil {
		v.v.Timestamp = &fpb.Timestamp{}
	}
	t := v.v.Timestamp.Timestamp
	l, r := 0, len(u.q)
	if t > u.latest {
		u.latest = t
	}
	// Binary search for sorted placement in queue.
	for {
		if l == r {
			// Insert a new list of updates for a new timestamp.
			u.q = append(u.q[:r], append([][]*value{{v}}, u.q[r:]...)...)
			return
		}
		i := (r-l)/2 + l
		t2 := u.q[i][0].v.Timestamp.Timestamp
		switch {
		case t == t2:
			// Append update to a list for an existing timestamp.
			u.q[i] = append(u.q[i], v)
			return
		case t < t2:
			r = i
		case t > t2:
			l = i + 1
		}
	}
}

type value struct {
	v *fpb.Value // The configuration for a stream of updates for a single value.
	r *rand.Rand // The PRNG used to generate subsequent updates for this value.
}

func (v value) String() string {
	return v.v.String()
}

func newValue(v *fpb.Value, r *rand.Rand) *value {
	if v.Seed == 0 {
		return &value{v: v, r: r}
	}
	return &value{v: v, r: rand.New(rand.NewSource(v.Seed))}
}

func (v *value) updateTimestamp() error {
	if v.v.Timestamp == nil {
		return fmt.Errorf("timestamp not set for %q", v.v)
	}
	t := v.v.Timestamp.Timestamp
	if t < 0 {
		return fmt.Errorf("timestamp must be positive for %q", v.v)
	}
	min, max := v.v.Timestamp.DeltaMin, v.v.Timestamp.DeltaMax
	if min > max || min < 0 {
		return fmt.Errorf("invalid delta_min/delta_max on timestamp for %q", v.v)
	}
	v.v.Timestamp.Timestamp = t + v.r.Int63n(max-min+1) + min
	return nil
}

func (v *value) updateIntValue() error {
	val := v.v.GetIntValue()
	if val == nil {
		return fmt.Errorf("invalid IntValue for %q", v.v)
	}
	var newval int64
	switch val.Distribution.(type) {
	case *fpb.IntValue_Range:
		rng := val.GetRange()
		if rng.Minimum > rng.Maximum {
			return fmt.Errorf("invalid minimum/maximum in IntRange for %q", v.v)
		}
		if val.Value < rng.Minimum || val.Value > rng.Maximum {
			return fmt.Errorf("value not in [minimum, maximum] in IntRange for %q", v.v)
		}

		left, right := rng.GetMinimum(), rng.GetMaximum()
		if rng.DeltaMin != 0 || rng.DeltaMax != 0 {
			if rng.DeltaMin > rng.DeltaMax {
				return fmt.Errorf("invalid delta_min/delta_max in IntRange for %q", v.v)
			}
			left, right = rng.GetDeltaMin(), rng.GetDeltaMax()
			newval = val.Value
		}

		newval += v.r.Int63n(right-left+1) + left
		if newval > rng.Maximum {
			newval = rng.Maximum
		}
		if newval < rng.Minimum {
			newval = rng.Minimum
		}
	case *fpb.IntValue_List:
		list := val.GetList()
		options := list.GetOptions()
		if len(options) == 0 {
			return fmt.Errorf("missing options on IntValue_List for %q", v.v)
		}
		if list.GetRandom() {
			newval = options[v.r.Intn(len(options))]
		} else {
			newval = options[0]
			list.Options = append(options[1:], options[0])
		}
	default:
		newval = val.Value
	}
	val.Value = newval
	return nil
}

func (v *value) updateDoubleValue() error {
	val := v.v.GetDoubleValue()
	if val == nil {
		return fmt.Errorf("invalid DoubleValue for %q", v.v)
	}
	var newval float64
	switch val.Distribution.(type) {
	case *fpb.DoubleValue_Range:
		rng := val.GetRange()
		if rng.Minimum > rng.Maximum {
			return fmt.Errorf("invalid minimum/maximum on DoubleValue_Range for %q", v.v)
		}
		if val.Value < rng.Minimum || val.Value > rng.Maximum {
			return fmt.Errorf("value not in [minimum, maximum] on DoubleValue_Range for %q", v.v)
		}

		left, right := rng.GetMinimum(), rng.GetMaximum()
		if rng.DeltaMin != 0 || rng.DeltaMax != 0 {
			if rng.DeltaMin > rng.DeltaMax {
				return fmt.Errorf("invalid delta_min/delta_max on DoubleValue_Range for %q", v.v)
			}
			left, right = rng.GetDeltaMin(), rng.GetDeltaMax()
			newval = val.Value
		}

		newval += v.r.Float64()*(right-left) + left
		if newval > rng.Maximum {
			newval = rng.Maximum
		}
		if newval < rng.Minimum {
			newval = rng.Minimum
		}
	case *fpb.DoubleValue_List:
		list := val.GetList()
		options := list.GetOptions()
		if len(options) == 0 {
			return fmt.Errorf("missing options on DoubleValue_List for %q", v.v)
		}
		if list.GetRandom() {
			newval = options[v.r.Intn(len(options))]
		} else {
			newval = options[0]
			list.Options = append(options[1:], options[0])
		}
	default:
		newval = val.Value
	}
	val.Value = newval
	return nil
}

func (v *value) updateStringValue() error {
	val := v.v.GetStringValue()
	if val == nil {
		return fmt.Errorf("invalid StringValue for %q", v.v)
	}
	var newval string
	switch val.Distribution.(type) {
	case *fpb.StringValue_List:
		list := val.GetList()
		options := list.Options
		if len(options) == 0 {
			return fmt.Errorf("missing options on StringValue_List for %q", v.v)
		}
		if list.Random {
			newval = options[v.r.Intn(len(options))]
		} else {
			newval = options[0]
			list.Options = append(options[1:], options[0])
		}
	default:
		newval = val.Value
	}
	val.Value = newval
	return nil
}

func (v *value) updateBoolValue() error {
	val := v.v.GetBoolValue()
	if val == nil {
		return fmt.Errorf("invalid BoolValue for %q", v.v)
	}
	var newval bool
	switch val.Distribution.(type) {
	case *fpb.BoolValue_List:
		list := val.GetList()
		options := list.Options
		if len(options) == 0 {
			return fmt.Errorf("missing options on BoolValue_List for %q", v.v)
		}
		if list.Random {
			newval = options[v.r.Intn(len(options))]
		} else {
			newval = options[0]
			list.Options = append(options[1:], options[0])
		}
	default:
		newval = val.Value
	}
	val.Value = newval
	return nil
}

func (v *value) nextValue() error {
	if v.v.Repeat == 1 {
		// This value has exhausted all of its updates, drop it.
		v.v = nil
		return nil
	}
	// Make a new proto message for the next value.
	v.v = proto.Clone(v.v).(*fpb.Value)
	if v.v.Repeat > 1 {
		v.v.Repeat--
	}
	if err := v.updateTimestamp(); err != nil {
		return err
	}
	switch v.v.GetValue().(type) {
	case *fpb.Value_IntValue:
		return v.updateIntValue()
	case *fpb.Value_DoubleValue:
		return v.updateDoubleValue()
	case *fpb.Value_StringValue:
		return v.updateStringValue()
	case *fpb.Value_BoolValue:
		return v.updateBoolValue()
	case *fpb.Value_Sync:
		return nil
	case *fpb.Value_Delete:
		return nil
	default:
		return fmt.Errorf("value type not found in %q", v.v)
	}
}

// ValueOf returns the concrete value of v.
func ValueOf(v *fpb.Value) interface{} {
	switch val := v.GetValue().(type) {
	case *fpb.Value_IntValue:
		return val.IntValue.Value
	case *fpb.Value_DoubleValue:
		return val.DoubleValue.Value
	case *fpb.Value_StringValue:
		return val.StringValue.Value
	case *fpb.Value_BoolValue:
		return val.BoolValue.Value
	case *fpb.Value_Sync:
		return val.Sync
	case *fpb.Value_Delete:
		return val.Delete
	default:
		return nil
	}
}

// TypedValueOf returns the gNMI TypedValue of v. If v is a Sync or Delete,
// TypedValueOf returns nil.
func TypedValueOf(v *fpb.Value) *gpb.TypedValue {
	tv := &gpb.TypedValue{}
	switch val := v.GetValue().(type) {
	case *fpb.Value_IntValue:
		tv.Value = &gpb.TypedValue_IntVal{val.IntValue.Value}
	case *fpb.Value_DoubleValue:
		tv.Value = &gpb.TypedValue_FloatVal{float32(val.DoubleValue.Value)}
	case *fpb.Value_StringValue:
		tv.Value = &gpb.TypedValue_StringVal{val.StringValue.Value}
	case *fpb.Value_BoolValue:
		tv.Value = &gpb.TypedValue_BoolVal{val.BoolValue.Value}
	default:
		return nil
	}
	return tv
}
<commit_msg>Fix typo.<commit_after>/*
Copyright 2017 Google Inc.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// Package queue implements an update queue for use in testing a telemetry
// stream.
package queue

import (
	"fmt"
	"math/rand"
	"sync"
	"time"

	"github.com/golang/protobuf/proto"

	gpb "github.com/openconfig/gnmi/proto/gnmi"
	fpb "github.com/openconfig/gnmi/testing/fake/proto"
)

// Queue is a generic interface for getting the next element from either a
// FixedQueue or UpdateQueue.
type Queue interface {
	Next() (interface{}, error)
}

// UpdateQueue is a structure that orders a set of device updates by their
// timestamps and repeatedly generates new pseudo-random updates based on
// a set of device path configurations.
type UpdateQueue struct {
	mu       sync.Mutex
	delay    bool
	duration time.Duration
	latest   int64
	q        [][]*value
	r        *rand.Rand
}

// New creates a new UpdateQueue. If delay is true, a call to Next()
// will invoke a sleep based on the duration between timestamps of the last
// returned update and the update to be returned by the current call. The
// values are the configuration for the updates stored in the queue.
func New(delay bool, seed int64, values []*fpb.Value) *UpdateQueue {
	if seed == 0 {
		seed = time.Now().UnixNano()
	}
	u := &UpdateQueue{delay: delay, r: rand.New(rand.NewSource(seed))}
	for _, v := range values {
		u.addValue(newValue(v, u.r))
	}
	return u
}

// Add inserts v into the queue in correct timestamp order.
func (u *UpdateQueue) Add(v *fpb.Value) {
	u.mu.Lock()
	defer u.mu.Unlock()
	u.addValue(newValue(v, u.r))
}

// Latest returns the maximum timestamp in the queue.
func (u *UpdateQueue) Latest() int64 {
	return u.latest
}

// Next returns the next update in the queue or an error.  If the queue is
// exhausted, a nil is returned for the update. The return will always be a
// *fpb.Value for proper type assertion.
func (u *UpdateQueue) Next() (interface{}, error) {
	u.mu.Lock()
	defer u.mu.Unlock()
	if len(u.q) == 0 {
		return nil, nil
	}
	// Incur a real delay if requested.
	if u.duration > 0 {
		time.Sleep(u.duration)
		u.duration = 0
	}
	v := u.q[0][0]
	// Make a copy of the current proto message to return.
	val := v.v
	// Update the value.
	if err := v.nextValue(); err != nil {
		return nil, err
	}
	var checkDelay bool
	// If this is the last update in the queue move to the next update set.
	// else move to the next update in the same update set.
	if len(u.q[0]) == 1 {
		u.q = u.q[1:]
		checkDelay = true
	} else {
		u.q[0] = u.q[0][1:]
	}
	// Add the updated value to the queue if the repeats are not exhausted.
	if v.v != nil {
		u.addValue(v)
	}
	// Set up a delay for the next retrieval if realtime delays are requested.
	if u.delay && checkDelay && len(u.q) > 0 {
		u.duration = time.Duration(u.q[0][0].v.Timestamp.Timestamp-val.Timestamp.Timestamp) * time.Nanosecond
	}
	return val, nil
}

func (u *UpdateQueue) addValue(v *value) {
	// If Timestamp was not provided create a default.
	if v.v.Timestamp == nil {
		v.v.Timestamp = &fpb.Timestamp{}
	}
	t := v.v.Timestamp.Timestamp
	l, r := 0, len(u.q)
	if t > u.latest {
		u.latest = t
	}
	// Binary search for sorted placement in queue.
	for {
		if l == r {
			// Insert a new list of updates for a new timestamp.
			u.q = append(u.q[:r], append([][]*value{{v}}, u.q[r:]...)...)
			return
		}
		i := (r-l)/2 + l
		t2 := u.q[i][0].v.Timestamp.Timestamp
		switch {
		case t == t2:
			// Append update to a list for an existing timestamp.
			u.q[i] = append(u.q[i], v)
			return
		case t < t2:
			r = i
		case t > t2:
			l = i + 1
		}
	}
}

type value struct {
	v *fpb.Value // The configuration for a stream of updates for a single value.
	r *rand.Rand // The PRNG used to generate subsequent updates for this value.
}

func (v value) String() string {
	return v.v.String()
}

func newValue(v *fpb.Value, r *rand.Rand) *value {
	if v.Seed == 0 {
		return &value{v: v, r: r}
	}
	return &value{v: v, r: rand.New(rand.NewSource(v.Seed))}
}

func (v *value) updateTimestamp() error {
	if v.v.Timestamp == nil {
		return fmt.Errorf("timestamp not set for %q", v.v)
	}
	t := v.v.Timestamp.Timestamp
	if t < 0 {
		return fmt.Errorf("timestamp must be positive for %q", v.v)
	}
	min, max := v.v.Timestamp.DeltaMin, v.v.Timestamp.DeltaMax
	if min > max || min < 0 {
		return fmt.Errorf("invalid delta_min/delta_max on timestamp for %q", v.v)
	}
	v.v.Timestamp.Timestamp = t + v.r.Int63n(max-min+1) + min
	return nil
}

func (v *value) updateIntValue() error {
	val := v.v.GetIntValue()
	if val == nil {
		return fmt.Errorf("invalid IntValue for %q", v.v)
	}
	var newval int64
	switch val.Distribution.(type) {
	case *fpb.IntValue_Range:
		rng := val.GetRange()
		if rng.Minimum > rng.Maximum {
			return fmt.Errorf("invalid minimum/maximum in IntRange for %q", v.v)
		}
		if val.Value < rng.Minimum || val.Value > rng.Maximum {
			return fmt.Errorf("value not in [minimum, maximum] in IntRange for %q", v.v)
		}

		left, right := rng.GetMinimum(), rng.GetMaximum()
		if rng.DeltaMin != 0 || rng.DeltaMax != 0 {
			if rng.DeltaMin > rng.DeltaMax {
				return fmt.Errorf("invalid delta_min/delta_max in IntRange for %q", v.v)
			}
			left, right = rng.GetDeltaMin(), rng.GetDeltaMax()
			newval = val.Value
		}

		newval += v.r.Int63n(right-left+1) + left
		if newval > rng.Maximum {
			newval = rng.Maximum
		}
		if newval < rng.Minimum {
			newval = rng.Minimum
		}
	case *fpb.IntValue_List:
		list := val.GetList()
		options := list.GetOptions()
		if len(options) == 0 {
			return fmt.Errorf("missing options on IntValue_List for %q", v.v)
		}
		if list.GetRandom() {
			newval = options[v.r.Intn(len(options))]
		} else {
			newval = options[0]
			list.Options = append(options[1:], options[0])
		}
	default:
		newval = val.Value
	}
	val.Value = newval
	return nil
}

func (v *value) updateDoubleValue() error {
	val := v.v.GetDoubleValue()
	if val == nil {
		return fmt.Errorf("invalid DoubleValue for %q", v.v)
	}
	var newval float64
	switch val.Distribution.(type) {
	case *fpb.DoubleValue_Range:
		rng := val.GetRange()
		if rng.Minimum > rng.Maximum {
			return fmt.Errorf("invalid minimum/maximum on DoubleValue_Range for %q", v.v)
		}
		if val.Value < rng.Minimum || val.Value > rng.Maximum {
			return fmt.Errorf("value not in [minimum, maximum] on DoubleValue_Range for %q", v.v)
		}

		left, right := rng.GetMinimum(), rng.GetMaximum()
		if rng.DeltaMin != 0 || rng.DeltaMax != 0 {
			if rng.DeltaMin > rng.DeltaMax {
				return fmt.Errorf("invalid delta_min/delta_max on DoubleValue_Range for %q", v.v)
			}
			left, right = rng.GetDeltaMin(), rng.GetDeltaMax()
			newval = val.Value
		}

		newval += v.r.Float64()*(right-left) + left
		if newval > rng.Maximum {
			newval = rng.Maximum
		}
		if newval < rng.Minimum {
			newval = rng.Minimum
		}
	case *fpb.DoubleValue_List:
		list := val.GetList()
		options := list.GetOptions()
		if len(options) == 0 {
			return fmt.Errorf("missing options on DoubleValue_List for %q", v.v)
		}
		if list.GetRandom() {
			newval = options[v.r.Intn(len(options))]
		} else {
			newval = options[0]
			list.Options = append(options[1:], options[0])
		}
	default:
		newval = val.Value
	}
	val.Value = newval
	return nil
}

func (v *value) updateStringValue() error {
	val := v.v.GetStringValue()
	if val == nil {
		return fmt.Errorf("invalid StringValue for %q", v.v)
	}
	var newval string
	switch val.Distribution.(type) {
	case *fpb.StringValue_List:
		list := val.GetList()
		options := list.Options
		if len(options) == 0 {
			return fmt.Errorf("missing options on StringValue_List for %q", v.v)
		}
		if list.Random {
			newval = options[v.r.Intn(len(options))]
		} else {
			newval = options[0]
			list.Options = append(options[1:], options[0])
		}
	default:
		newval = val.Value
	}
	val.Value = newval
	return nil
}

func (v *value) updateBoolValue() error {
	val := v.v.GetBoolValue()
	if val == nil {
		return fmt.Errorf("invalid BoolValue for %q", v.v)
	}
	var newval bool
	switch val.Distribution.(type) {
	case *fpb.BoolValue_List:
		list := val.GetList()
		options := list.Options
		if len(options) == 0 {
			return fmt.Errorf("missing options on BoolValue_List for %q", v.v)
		}
		if list.Random {
			newval = options[v.r.Intn(len(options))]
		} else {
			newval = options[0]
			list.Options = append(options[1:], options[0])
		}
	default:
		newval = val.Value
	}
	val.Value = newval
	return nil
}

func (v *value) nextValue() error {
	if v.v.Repeat == 1 {
		// This value has exhausted all of its updates, drop it.
		v.v = nil
		return nil
	}
	// Make a new proto message for the next value.
	v.v = proto.Clone(v.v).(*fpb.Value)
	if v.v.Repeat > 1 {
		v.v.Repeat--
	}
	if err := v.updateTimestamp(); err != nil {
		return err
	}
	switch v.v.GetValue().(type) {
	case *fpb.Value_IntValue:
		return v.updateIntValue()
	case *fpb.Value_DoubleValue:
		return v.updateDoubleValue()
	case *fpb.Value_StringValue:
		return v.updateStringValue()
	case *fpb.Value_BoolValue:
		return v.updateBoolValue()
	case *fpb.Value_Sync:
		return nil
	case *fpb.Value_Delete:
		return nil
	default:
		return fmt.Errorf("value type not found in %q", v.v)
	}
}

// ValueOf returns the concrete value of v.
func ValueOf(v *fpb.Value) interface{} {
	switch val := v.GetValue().(type) {
	case *fpb.Value_IntValue:
		return val.IntValue.Value
	case *fpb.Value_DoubleValue:
		return val.DoubleValue.Value
	case *fpb.Value_StringValue:
		return val.StringValue.Value
	case *fpb.Value_BoolValue:
		return val.BoolValue.Value
	case *fpb.Value_Sync:
		return val.Sync
	case *fpb.Value_Delete:
		return val.Delete
	default:
		return nil
	}
}

// TypedValueOf returns the gNMI TypedValue of v. If v is a Sync or Delete,
// TypedValueOf returns nil.
func TypedValueOf(v *fpb.Value) *gpb.TypedValue {
	tv := &gpb.TypedValue{}
	switch val := v.GetValue().(type) {
	case *fpb.Value_IntValue:
		tv.Value = &gpb.TypedValue_IntVal{val.IntValue.Value}
	case *fpb.Value_DoubleValue:
		tv.Value = &gpb.TypedValue_FloatVal{float32(val.DoubleValue.Value)}
	case *fpb.Value_StringValue:
		tv.Value = &gpb.TypedValue_StringVal{val.StringValue.Value}
	case *fpb.Value_BoolValue:
		tv.Value = &gpb.TypedValue_BoolVal{val.BoolValue.Value}
	default:
		return nil
	}
	return tv
}
<|endoftext|>


<commit_before>package lint

import (
	"regexp"
	"strings"
)

// RuleInfo stores information of a rule.
type RuleInfo struct {
	Name        RuleName   // rule name in the set.
	Description string     // a short description of this rule.
	URI         string     // a link to a document for more details.
	FileTypes   []FileType // types of files that this rule targets to.
	Category    Category   // category of problems this rule produces.
	Status      Status     // status of this rule, either enabled, disabled, or deprecated.

	noPositional struct{} // Prevent positional composite literal instantiation
}

// FileType defines a file type that a rule is targeting to.
type FileType string

const (
	// ProtoFile indicates that the targeted file is a protobuf-definition file.
	ProtoFile FileType = "proto-file"
)

// Category defines the category of the findings produced by a rule.
type Category string

const (
	// Error indicates that in the API, something will cause errors.
	Error Category = "error"
	// Warning indicates that in the API, something can be do better.
	Warning Category = "warning"
	// DefaultCategory denotes the default value of Category
	DefaultCategory Category = Warning
)

// Status defines whether a rule is enabled, disabled or deprecated.
type Status string

const (
	// Enabled indicates that a rule should be enabled.
	Enabled Status = "enabled"
	// Disabled indicates that a rule should be disabled.
	Disabled Status = "disabled"
	// Deprecated indicates that a rule should be deprecated.
	Deprecated Status = "Deprecated"
	// DefaultStatus denotes the default value of Status.
	DefaultStatus Status = Disabled
)

// RuleName is an identifier for a rule. Allowed characters include a-z, A-Z, 0-9, -, _. The
// namespace separator :: is allowed between RuleName segments (for example, my_namespace::my_rule).
type RuleName string

const nameSeparator string = "::"

var ruleNameValidator = regexp.MustCompile("^([a-zA-Z0-9-_]+(::[a-zA-Z0-9-_]+)?)+$")

// NewRuleName creates a RuleName from segments. It will join the segments with the "::" separator.
func NewRuleName(segments ...string) RuleName {
	return RuleName(strings.Join(segments, nameSeparator))
}

// IsValid checks if a RuleName is syntactically valid
func (r RuleName) IsValid() bool {
	return r != "" && ruleNameValidator.Match([]byte(r))
}

// WithPrefix prepends a prefix to the RuleName, separating it with ::
func (r RuleName) WithPrefix(prefix ...string) RuleName {
	fullPrefix := strings.Join(prefix, nameSeparator)

	return RuleName(fullPrefix + nameSeparator + string(r))
}

// HasPrefix returns true if r contains prefix as a namespace. prefix parameters can be "::" delimited
// or specified as independent parameters.
// For example:
//
// r := NewRuleName("foo", "bar", "baz")   // string(r) == "foo::bar::baz"
//
// r.HasPrefix("foo::bar")          == true
// r.HasPrefix("foo", "bar")        == true
// r.HasPrefix("foo", "bar", "baz") == true   // matches the entire string
// r.HasPrefix("foo", "ba")         == false  // prefix must end on a delimiter
func (r RuleName) HasPrefix(prefix ...string) bool {
	prefixSegments := make([]string, 0, len(prefix))

	for _, prefixSegment := range prefix {
		prefixSegments = append(prefixSegments, strings.Split(prefixSegment, "::")...)
	}

	prefixStr := strings.Join(prefixSegments, nameSeparator)

	return string(r) == prefixStr || strings.HasPrefix(string(r), prefixStr+nameSeparator)
}
<commit_msg>Reusing NewRuleName<commit_after>package lint

import (
	"regexp"
	"strings"
)

// RuleInfo stores information of a rule.
type RuleInfo struct {
	Name        RuleName   // rule name in the set.
	Description string     // a short description of this rule.
	URI         string     // a link to a document for more details.
	FileTypes   []FileType // types of files that this rule targets to.
	Category    Category   // category of problems this rule produces.
	Status      Status     // status of this rule, either enabled, disabled, or deprecated.

	noPositional struct{} // Prevent positional composite literal instantiation
}

// FileType defines a file type that a rule is targeting to.
type FileType string

const (
	// ProtoFile indicates that the targeted file is a protobuf-definition file.
	ProtoFile FileType = "proto-file"
)

// Category defines the category of the findings produced by a rule.
type Category string

const (
	// Error indicates that in the API, something will cause errors.
	Error Category = "error"
	// Warning indicates that in the API, something can be do better.
	Warning Category = "warning"
	// DefaultCategory denotes the default value of Category
	DefaultCategory Category = Warning
)

// Status defines whether a rule is enabled, disabled or deprecated.
type Status string

const (
	// Enabled indicates that a rule should be enabled.
	Enabled Status = "enabled"
	// Disabled indicates that a rule should be disabled.
	Disabled Status = "disabled"
	// Deprecated indicates that a rule should be deprecated.
	Deprecated Status = "Deprecated"
	// DefaultStatus denotes the default value of Status.
	DefaultStatus Status = Disabled
)

// RuleName is an identifier for a rule. Allowed characters include a-z, A-Z, 0-9, -, _. The
// namespace separator :: is allowed between RuleName segments (for example, my_namespace::my_rule).
type RuleName string

const nameSeparator string = "::"

var ruleNameValidator = regexp.MustCompile("^([a-zA-Z0-9-_]+(::[a-zA-Z0-9-_]+)?)+$")

// NewRuleName creates a RuleName from segments. It will join the segments with the "::" separator.
func NewRuleName(segments ...string) RuleName {
	return RuleName(strings.Join(segments, nameSeparator))
}

// IsValid checks if a RuleName is syntactically valid
func (r RuleName) IsValid() bool {
	return r != "" && ruleNameValidator.Match([]byte(r))
}

// WithPrefix prepends a prefix to the RuleName, separating it with ::
func (r RuleName) WithPrefix(prefix ...string) RuleName {
	fullPrefix := NewRuleName(prefix...)

	return RuleName(string(fullPrefix) + nameSeparator + string(r))
}

// HasPrefix returns true if r contains prefix as a namespace. prefix parameters can be "::" delimited
// or specified as independent parameters.
// For example:
//
// r := NewRuleName("foo", "bar", "baz")   // string(r) == "foo::bar::baz"
//
// r.HasPrefix("foo::bar")          == true
// r.HasPrefix("foo", "bar")        == true
// r.HasPrefix("foo", "bar", "baz") == true   // matches the entire string
// r.HasPrefix("foo", "ba")         == false  // prefix must end on a delimiter
func (r RuleName) HasPrefix(prefix ...string) bool {
	prefixSegments := make([]string, 0, len(prefix))

	for _, prefixSegment := range prefix {
		prefixSegments = append(prefixSegments, strings.Split(prefixSegment, "::")...)
	}

	prefixStr := strings.Join(prefixSegments, nameSeparator)

	return string(r) == prefixStr || strings.HasPrefix(string(r), prefixStr+nameSeparator)
}
<|endoftext|>


<commit_before>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// godoc: Go Documentation Server

// Web server tree:
//
//	http://godoc/		main landing page
//	http://godoc/doc/	serve from $GOROOT/doc - spec, mem, tutorial, etc.
//	http://godoc/src/	serve files from $GOROOT/src; .go gets pretty-printed
//	http://godoc/cmd/	serve documentation about commands
//	http://godoc/pkg/	serve documentation about packages
//				(idea is if you say import "compress/zlib", you go to
//				http://godoc/pkg/compress/zlib)
//
// Command-line interface:
//
//	godoc packagepath [name ...]
//
//	godoc compress/zlib
//		- prints doc for package compress/zlib
//	godoc crypto/block Cipher NewCMAC
//		- prints doc for Cipher and NewCMAC in package crypto/block

package main

import (
	"bytes"
	_ "expvar" // to serve /debug/vars
	"flag"
	"fmt"
	"go/ast"
	"http"
	_ "http/pprof" // to serve /debug/pprof/*
	"io"
	"log"
	"os"
	pathutil "path"
	"regexp"
	"runtime"
	"strings"
	"time"
)

const defaultAddr = ":6060" // default webserver address

var (
	// periodic sync
	syncCmd   = flag.String("sync", "", "sync command; disabled if empty")
	syncMin   = flag.Int("sync_minutes", 0, "sync interval in minutes; disabled if <= 0")
	syncDelay delayTime // actual sync delay in minutes; usually syncDelay == syncMin, but delay may back off exponentially

	// network
	httpAddr   = flag.String("http", "", "HTTP service address (e.g., '"+defaultAddr+"')")
	serverAddr = flag.String("server", "", "webserver address for command line searches")

	// layout control
	html    = flag.Bool("html", false, "print HTML in command-line mode")
	srcMode = flag.Bool("src", false, "print (exported) source in command-line mode")

	// command-line searches
	query = flag.Bool("q", false, "arguments are considered search queries")
)

func serveError(c *http.Conn, r *http.Request, relpath string, err os.Error) {
	contents := applyTemplate(errorHTML, "errorHTML", err) // err may contain an absolute path!
	servePage(c, "File "+relpath, "", contents)
}

func exec(c *http.Conn, args []string) (status int) {
	r, w, err := os.Pipe()
	if err != nil {
		log.Stderrf("os.Pipe(): %v\n", err)
		return 2
	}

	bin := args[0]
	fds := []*os.File{nil, w, w}
	if *verbose {
		log.Stderrf("executing %v", args)
	}
	pid, err := os.ForkExec(bin, args, os.Environ(), *goroot, fds)
	defer r.Close()
	w.Close()
	if err != nil {
		log.Stderrf("os.ForkExec(%q): %v\n", bin, err)
		return 2
	}

	var buf bytes.Buffer
	io.Copy(&buf, r)
	wait, err := os.Wait(pid, 0)
	if err != nil {
		os.Stderr.Write(buf.Bytes())
		log.Stderrf("os.Wait(%d, 0): %v\n", pid, err)
		return 2
	}
	status = wait.ExitStatus()
	if !wait.Exited() || status > 1 {
		os.Stderr.Write(buf.Bytes())
		log.Stderrf("executing %v failed (exit status = %d)", args, status)
		return
	}

	if *verbose {
		os.Stderr.Write(buf.Bytes())
	}
	if c != nil {
		c.SetHeader("content-type", "text/plain; charset=utf-8")
		c.Write(buf.Bytes())
	}

	return
}

// Maximum directory depth, adjust as needed.
const maxDirDepth = 24

func dosync(c *http.Conn, r *http.Request) {
	args := []string{"/bin/sh", "-c", *syncCmd}
	switch exec(c, args) {
	case 0:
		// sync succeeded and some files have changed;
		// update package tree.
		// TODO(gri): The directory tree may be temporarily out-of-sync.
		//            Consider keeping separate time stamps so the web-
		//            page can indicate this discrepancy.
		fsTree.set(newDirectory(*goroot, maxDirDepth))
		fallthrough
	case 1:
		// sync failed because no files changed;
		// don't change the package tree
		syncDelay.set(*syncMin) //  revert to regular sync schedule
	default:
		// sync failed because of an error - back off exponentially, but try at least once a day
		syncDelay.backoff(24 * 60)
	}
}

func usage() {
	fmt.Fprintf(os.Stderr,
		"usage: godoc package [name ...]\n"+
			"	godoc -http="+defaultAddr+"\n")
	flag.PrintDefaults()
	os.Exit(2)
}

func loggingHandler(h http.Handler) http.Handler {
	return http.HandlerFunc(func(c *http.Conn, req *http.Request) {
		log.Stderrf("%s\t%s", c.RemoteAddr, req.URL)
		h.ServeHTTP(c, req)
	})
}

func remoteSearch(query string) (res *http.Response, err os.Error) {
	search := "/search?f=text&q=" + http.URLEscape(query)

	// list of addresses to try
	var addrs []string
	if *serverAddr != "" {
		// explicit server address - only try this one
		addrs = []string{*serverAddr}
	} else {
		addrs = []string{
			defaultAddr,
			"golang.org",
		}
	}

	// remote search
	for _, addr := range addrs {
		url := "http://" + addr + search
		res, _, err = http.Get(url)
		if err == nil && res.StatusCode == http.StatusOK {
			break
		}
	}

	if err == nil && res.StatusCode != http.StatusOK {
		err = os.NewError(res.Status)
	}

	return
}

// Does s look like a regular expression?
func isRegexp(s string) bool {
	return strings.IndexAny(s, ".(|)*+?^$[]") >= 0
}

// Make a regular expression of the form
// names[0]|names[1]|...names[len(names)-1].
// Returns nil if the regular expression is illegal.
func makeRx(names []string) (rx *regexp.Regexp) {
	if len(names) > 0 {
		s := ""
		for i, name := range names {
			if i > 0 {
				s += "|"
			}
			if isRegexp(name) {
				s += name
			} else {
				s += "^" + name + "$" // must match exactly
			}
		}
		rx, _ = regexp.Compile(s) // rx is nil if there's a compilation error
	}
	return
}

func main() {
	flag.Usage = usage
	flag.Parse()

	// Check usage: either server and no args, or command line and args
	if (*httpAddr != "") != (flag.NArg() == 0) {
		usage()
	}

	if *tabwidth < 0 {
		log.Exitf("negative tabwidth %d", *tabwidth)
	}

	initHandlers()
	readTemplates()

	if *httpAddr != "" {
		// HTTP server mode.
		var handler http.Handler = http.DefaultServeMux
		if *verbose {
			log.Stderrf("Go Documentation Server\n")
			log.Stderrf("version = %s\n", runtime.Version())
			log.Stderrf("address = %s\n", *httpAddr)
			log.Stderrf("goroot = %s\n", *goroot)
			log.Stderrf("tabwidth = %d\n", *tabwidth)
			if !fsMap.IsEmpty() {
				log.Stderr("user-defined mapping:")
				fsMap.Fprint(os.Stderr)
			}
			handler = loggingHandler(handler)
		}

		registerPublicHandlers(http.DefaultServeMux)
		if *syncCmd != "" {
			http.Handle("/debug/sync", http.HandlerFunc(dosync))
		}

		// Initialize directory tree with corresponding timestamp.
		// Do it in two steps:
		// 1) set timestamp right away so that the indexer is kicked on
		fsTree.set(nil)
		// 2) compute initial directory tree in a goroutine so that launch is quick
		go func() { fsTree.set(newDirectory(*goroot, maxDirDepth)) }()

		// Start sync goroutine, if enabled.
		if *syncCmd != "" && *syncMin > 0 {
			syncDelay.set(*syncMin) // initial sync delay
			go func() {
				for {
					dosync(nil, nil)
					delay, _ := syncDelay.get()
					if *verbose {
						log.Stderrf("next sync in %dmin", delay.(int))
					}
					time.Sleep(int64(delay.(int)) * 60e9)
				}
			}()
		}

		// Start indexing goroutine.
		go indexer()

		// Start http server.
		if err := http.ListenAndServe(*httpAddr, handler); err != nil {
			log.Exitf("ListenAndServe %s: %v", *httpAddr, err)
		}

		return
	}

	// Command line mode.
	if *html {
		packageText = packageHTML
		searchText = packageHTML
	}

	if *query {
		// Command-line queries.
		for i := 0; i < flag.NArg(); i++ {
			res, err := remoteSearch(flag.Arg(i))
			if err != nil {
				log.Exitf("remoteSearch: %s", err)
			}
			io.Copy(os.Stdout, res.Body)
		}
		return
	}

	// determine paths
	path := flag.Arg(0)
	if len(path) > 0 && path[0] == '.' {
		// assume cwd; don't assume -goroot
		cwd, _ := os.Getwd() // ignore errors
		path = pathutil.Join(cwd, path)
	}
	relpath := path
	abspath := path
	if len(path) > 0 && path[0] != '/' {
		abspath = absolutePath(path, pkgHandler.fsRoot)
	} else {
		relpath = relativePath(path)
	}

	var mode PageInfoMode
	if *srcMode {
		// only filter exports if we don't have explicit command-line filter arguments
		if flag.NArg() == 1 {
			mode |= exportsOnly
		}
	} else {
		mode = exportsOnly | genDoc
	}
	// TODO(gri): Provide a mechanism (flag?) to select a package
	//            if there are multiple packages in a directory.
	info := pkgHandler.getPageInfo(abspath, relpath, "", mode|tryMode)

	if info.PAst == nil && info.PDoc == nil && info.Dirs == nil {
		// try again, this time assume it's a command
		if len(path) > 0 && path[0] != '/' {
			abspath = absolutePath(path, cmdHandler.fsRoot)
		}
		info = cmdHandler.getPageInfo(abspath, relpath, "", mode)
	}

	// If we have more than one argument, use the remaining arguments for filtering
	if flag.NArg() > 1 {
		args := flag.Args()[1:]
		rx := makeRx(args)
		if rx == nil {
			log.Exitf("illegal regular expression from %v", args)
		}

		filter := func(s string) bool { return rx.MatchString(s) }
		switch {
		case info.PAst != nil:
			ast.FilterFile(info.PAst, filter)
			info.PAst.Doc = nil // don't show package comment in this case
		case info.PDoc != nil:
			info.PDoc.Filter(filter)
		}
	}

	if err := packageText.Execute(info, os.Stdout); err != nil {
		log.Stderrf("packageText.Execute: %s", err)
	}
}
<commit_msg>godoc: don't print package clause in -src command-line mode with filtering<commit_after>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// godoc: Go Documentation Server

// Web server tree:
//
//	http://godoc/		main landing page
//	http://godoc/doc/	serve from $GOROOT/doc - spec, mem, tutorial, etc.
//	http://godoc/src/	serve files from $GOROOT/src; .go gets pretty-printed
//	http://godoc/cmd/	serve documentation about commands
//	http://godoc/pkg/	serve documentation about packages
//				(idea is if you say import "compress/zlib", you go to
//				http://godoc/pkg/compress/zlib)
//
// Command-line interface:
//
//	godoc packagepath [name ...]
//
//	godoc compress/zlib
//		- prints doc for package compress/zlib
//	godoc crypto/block Cipher NewCMAC
//		- prints doc for Cipher and NewCMAC in package crypto/block

package main

import (
	"bytes"
	_ "expvar" // to serve /debug/vars
	"flag"
	"fmt"
	"go/ast"
	"http"
	_ "http/pprof" // to serve /debug/pprof/*
	"io"
	"log"
	"os"
	pathutil "path"
	"regexp"
	"runtime"
	"strings"
	"time"
)

const defaultAddr = ":6060" // default webserver address

var (
	// periodic sync
	syncCmd   = flag.String("sync", "", "sync command; disabled if empty")
	syncMin   = flag.Int("sync_minutes", 0, "sync interval in minutes; disabled if <= 0")
	syncDelay delayTime // actual sync delay in minutes; usually syncDelay == syncMin, but delay may back off exponentially

	// network
	httpAddr   = flag.String("http", "", "HTTP service address (e.g., '"+defaultAddr+"')")
	serverAddr = flag.String("server", "", "webserver address for command line searches")

	// layout control
	html    = flag.Bool("html", false, "print HTML in command-line mode")
	srcMode = flag.Bool("src", false, "print (exported) source in command-line mode")

	// command-line searches
	query = flag.Bool("q", false, "arguments are considered search queries")
)

func serveError(c *http.Conn, r *http.Request, relpath string, err os.Error) {
	contents := applyTemplate(errorHTML, "errorHTML", err) // err may contain an absolute path!
	servePage(c, "File "+relpath, "", contents)
}

func exec(c *http.Conn, args []string) (status int) {
	r, w, err := os.Pipe()
	if err != nil {
		log.Stderrf("os.Pipe(): %v\n", err)
		return 2
	}

	bin := args[0]
	fds := []*os.File{nil, w, w}
	if *verbose {
		log.Stderrf("executing %v", args)
	}
	pid, err := os.ForkExec(bin, args, os.Environ(), *goroot, fds)
	defer r.Close()
	w.Close()
	if err != nil {
		log.Stderrf("os.ForkExec(%q): %v\n", bin, err)
		return 2
	}

	var buf bytes.Buffer
	io.Copy(&buf, r)
	wait, err := os.Wait(pid, 0)
	if err != nil {
		os.Stderr.Write(buf.Bytes())
		log.Stderrf("os.Wait(%d, 0): %v\n", pid, err)
		return 2
	}
	status = wait.ExitStatus()
	if !wait.Exited() || status > 1 {
		os.Stderr.Write(buf.Bytes())
		log.Stderrf("executing %v failed (exit status = %d)", args, status)
		return
	}

	if *verbose {
		os.Stderr.Write(buf.Bytes())
	}
	if c != nil {
		c.SetHeader("content-type", "text/plain; charset=utf-8")
		c.Write(buf.Bytes())
	}

	return
}

// Maximum directory depth, adjust as needed.
const maxDirDepth = 24

func dosync(c *http.Conn, r *http.Request) {
	args := []string{"/bin/sh", "-c", *syncCmd}
	switch exec(c, args) {
	case 0:
		// sync succeeded and some files have changed;
		// update package tree.
		// TODO(gri): The directory tree may be temporarily out-of-sync.
		//            Consider keeping separate time stamps so the web-
		//            page can indicate this discrepancy.
		fsTree.set(newDirectory(*goroot, maxDirDepth))
		fallthrough
	case 1:
		// sync failed because no files changed;
		// don't change the package tree
		syncDelay.set(*syncMin) //  revert to regular sync schedule
	default:
		// sync failed because of an error - back off exponentially, but try at least once a day
		syncDelay.backoff(24 * 60)
	}
}

func usage() {
	fmt.Fprintf(os.Stderr,
		"usage: godoc package [name ...]\n"+
			"	godoc -http="+defaultAddr+"\n")
	flag.PrintDefaults()
	os.Exit(2)
}

func loggingHandler(h http.Handler) http.Handler {
	return http.HandlerFunc(func(c *http.Conn, req *http.Request) {
		log.Stderrf("%s\t%s", c.RemoteAddr, req.URL)
		h.ServeHTTP(c, req)
	})
}

func remoteSearch(query string) (res *http.Response, err os.Error) {
	search := "/search?f=text&q=" + http.URLEscape(query)

	// list of addresses to try
	var addrs []string
	if *serverAddr != "" {
		// explicit server address - only try this one
		addrs = []string{*serverAddr}
	} else {
		addrs = []string{
			defaultAddr,
			"golang.org",
		}
	}

	// remote search
	for _, addr := range addrs {
		url := "http://" + addr + search
		res, _, err = http.Get(url)
		if err == nil && res.StatusCode == http.StatusOK {
			break
		}
	}

	if err == nil && res.StatusCode != http.StatusOK {
		err = os.NewError(res.Status)
	}

	return
}

// Does s look like a regular expression?
func isRegexp(s string) bool {
	return strings.IndexAny(s, ".(|)*+?^$[]") >= 0
}

// Make a regular expression of the form
// names[0]|names[1]|...names[len(names)-1].
// Returns nil if the regular expression is illegal.
func makeRx(names []string) (rx *regexp.Regexp) {
	if len(names) > 0 {
		s := ""
		for i, name := range names {
			if i > 0 {
				s += "|"
			}
			if isRegexp(name) {
				s += name
			} else {
				s += "^" + name + "$" // must match exactly
			}
		}
		rx, _ = regexp.Compile(s) // rx is nil if there's a compilation error
	}
	return
}

func main() {
	flag.Usage = usage
	flag.Parse()

	// Check usage: either server and no args, or command line and args
	if (*httpAddr != "") != (flag.NArg() == 0) {
		usage()
	}

	if *tabwidth < 0 {
		log.Exitf("negative tabwidth %d", *tabwidth)
	}

	initHandlers()
	readTemplates()

	if *httpAddr != "" {
		// HTTP server mode.
		var handler http.Handler = http.DefaultServeMux
		if *verbose {
			log.Stderrf("Go Documentation Server\n")
			log.Stderrf("version = %s\n", runtime.Version())
			log.Stderrf("address = %s\n", *httpAddr)
			log.Stderrf("goroot = %s\n", *goroot)
			log.Stderrf("tabwidth = %d\n", *tabwidth)
			if !fsMap.IsEmpty() {
				log.Stderr("user-defined mapping:")
				fsMap.Fprint(os.Stderr)
			}
			handler = loggingHandler(handler)
		}

		registerPublicHandlers(http.DefaultServeMux)
		if *syncCmd != "" {
			http.Handle("/debug/sync", http.HandlerFunc(dosync))
		}

		// Initialize directory tree with corresponding timestamp.
		// Do it in two steps:
		// 1) set timestamp right away so that the indexer is kicked on
		fsTree.set(nil)
		// 2) compute initial directory tree in a goroutine so that launch is quick
		go func() { fsTree.set(newDirectory(*goroot, maxDirDepth)) }()

		// Start sync goroutine, if enabled.
		if *syncCmd != "" && *syncMin > 0 {
			syncDelay.set(*syncMin) // initial sync delay
			go func() {
				for {
					dosync(nil, nil)
					delay, _ := syncDelay.get()
					if *verbose {
						log.Stderrf("next sync in %dmin", delay.(int))
					}
					time.Sleep(int64(delay.(int)) * 60e9)
				}
			}()
		}

		// Start indexing goroutine.
		go indexer()

		// Start http server.
		if err := http.ListenAndServe(*httpAddr, handler); err != nil {
			log.Exitf("ListenAndServe %s: %v", *httpAddr, err)
		}

		return
	}

	// Command line mode.
	if *html {
		packageText = packageHTML
		searchText = packageHTML
	}

	if *query {
		// Command-line queries.
		for i := 0; i < flag.NArg(); i++ {
			res, err := remoteSearch(flag.Arg(i))
			if err != nil {
				log.Exitf("remoteSearch: %s", err)
			}
			io.Copy(os.Stdout, res.Body)
		}
		return
	}

	// determine paths
	path := flag.Arg(0)
	if len(path) > 0 && path[0] == '.' {
		// assume cwd; don't assume -goroot
		cwd, _ := os.Getwd() // ignore errors
		path = pathutil.Join(cwd, path)
	}
	relpath := path
	abspath := path
	if len(path) > 0 && path[0] != '/' {
		abspath = absolutePath(path, pkgHandler.fsRoot)
	} else {
		relpath = relativePath(path)
	}

	var mode PageInfoMode
	if *srcMode {
		// only filter exports if we don't have explicit command-line filter arguments
		if flag.NArg() == 1 {
			mode |= exportsOnly
		}
	} else {
		mode = exportsOnly | genDoc
	}
	// TODO(gri): Provide a mechanism (flag?) to select a package
	//            if there are multiple packages in a directory.
	info := pkgHandler.getPageInfo(abspath, relpath, "", mode|tryMode)

	if info.PAst == nil && info.PDoc == nil && info.Dirs == nil {
		// try again, this time assume it's a command
		if len(path) > 0 && path[0] != '/' {
			abspath = absolutePath(path, cmdHandler.fsRoot)
		}
		info = cmdHandler.getPageInfo(abspath, relpath, "", mode)
	}

	// If we have more than one argument, use the remaining arguments for filtering
	if flag.NArg() > 1 {
		args := flag.Args()[1:]
		rx := makeRx(args)
		if rx == nil {
			log.Exitf("illegal regular expression from %v", args)
		}

		filter := func(s string) bool { return rx.MatchString(s) }
		switch {
		case info.PAst != nil:
			ast.FilterFile(info.PAst, filter)
			// Special case: Don't use templates for printing
			// so we only get the filtered declarations without
			// package clause or extra whitespace.
			for i, d := range info.PAst.Decls {
				if i > 0 {
					fmt.Println()
				}
				writeAny(os.Stdout, d, *html)
				fmt.Println()
			}
			return

		case info.PDoc != nil:
			info.PDoc.Filter(filter)
		}
	}

	if err := packageText.Execute(info, os.Stdout); err != nil {
		log.Stderrf("packageText.Execute: %s", err)
	}
}
<|endoftext|>


<commit_before>// Copyright 2016, RadiantBlue Technologies, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package logger

import (
	"encoding/json"
	"fmt"
	"log"
	"net/http"
	"sync"
	"time"

	"github.com/venicegeo/pz-gocommon/elasticsearch"
	"github.com/venicegeo/pz-gocommon/gocommon"
	pzsyslog "github.com/venicegeo/pz-gocommon/syslog"
)

type Service struct {
	sync.Mutex

	stats  Stats
	origin string
	async  bool

	logWriter   pzsyslog.Writer
	auditWriter pzsyslog.Writer

	esIndex elasticsearch.IIndex
}

func (service *Service) Init(sys *piazza.SystemConfig, logWriter pzsyslog.Writer, auditWriter pzsyslog.Writer, esi elasticsearch.IIndex, asyncLogging bool) error {
	service.stats.CreatedOn = time.Now()
	service.stats.NumMessagesByApplication = map[string]int{}

	service.logWriter = logWriter
	service.auditWriter = auditWriter

	service.esIndex = esi

	service.origin = string(sys.Name)

	service.async = asyncLogging

	return nil
}

func (service *Service) newInternalErrorResponse(err error) *piazza.JsonResponse {
	return &piazza.JsonResponse{
		StatusCode: http.StatusInternalServerError,
		Message:    err.Error(),
		Origin:     service.origin,
	}
}

func (service *Service) newBadRequestResponse(err error) *piazza.JsonResponse {
	return &piazza.JsonResponse{
		StatusCode: http.StatusBadRequest,
		Message:    err.Error(),
		Origin:     service.origin,
	}
}

func (service *Service) incrementStats(application string) {
	service.Lock()
	service.stats.NumMessages++
	service.stats.NumMessagesByApplication[application]++
	service.Unlock()
}

func (service *Service) GetRoot() *piazza.JsonResponse {
	resp := &piazza.JsonResponse{
		StatusCode: 200,
		Data:       "Hi. I'm pz-logger.",
	}

	err := resp.SetType()
	if err != nil {
		return service.newInternalErrorResponse(err)
	}

	return resp
}

func (service *Service) GetStats() *piazza.JsonResponse {
	service.Lock()
	t := service.stats
	service.Unlock()

	resp := &piazza.JsonResponse{
		StatusCode: http.StatusOK,
		Data:       t,
	}

	err := resp.SetType()
	if err != nil {
		return service.newInternalErrorResponse(err)
	}

	return resp
}

func createQueryDslAsString(
	pagination *piazza.JsonPagination,
	params *piazza.HttpQueryParams) (string, error) {

	must := []map[string]interface{}{}

	service, err := params.GetAsString("service", "")
	if err != nil {
		return "", err
	}

	contains, err := params.GetAsString("contains", "")
	if err != nil {
		return "", err
	}

	before, err := params.GetBefore(time.Time{})
	if err != nil {
		return "", err
	}

	after, err := params.GetAfter(time.Time{})
	if err != nil {
		return "", err
	}

	if service != "" {
		must = append(must, map[string]interface{}{
			"match": map[string]interface{}{
				"application": service,
			},
		})
	}
	if contains != "" {
		must = append(must, map[string]interface{}{
			"wildcard": map[string]interface{}{
				"message":  "*" + contains + "*",
			},
		})
	}

	if !after.IsZero() || !before.IsZero() {
		rangeParams := map[string]time.Time{}

		if !after.IsZero() {
			rangeParams["gte"] = after
		}

		if !before.IsZero() {
			rangeParams["lte"] = before
		}

		must = append(must, map[string]interface{}{
			"range": map[string]interface{}{
				"timeStamp": rangeParams,
			},
		})
	}

	if len(must) == 0 {
		return "", nil
	}

	dsl := map[string]interface{}{
		"query": map[string]interface{}{
			"filtered": map[string]interface{}{
				"query": map[string]interface{}{
					"bool": map[string]interface{}{
						"must": must,
					},
				},
			},
		},
		"size": pagination.PerPage,
		"from": pagination.PerPage * pagination.Page,
	}

	dsl["sort"] = map[string]string{
		pagination.SortBy: string(pagination.Order),
	}

	output, err := json.Marshal(dsl)
	if err != nil {
		return "", err
	}

	return string(output), nil
}

func (service *Service) PostSyslog(mNew *pzsyslog.Message) *piazza.JsonResponse {
	err := mNew.Validate()
	if err != nil {
		return service.newBadRequestResponse(err)
	}

	err = service.postSyslog(mNew)
	if err != nil {
		return service.newInternalErrorResponse(err)
	}

	service.incrementStats(mNew.Application)

	resp := &piazza.JsonResponse{
		StatusCode: http.StatusOK,
	}
	return resp
}

func (service *Service) postSyslog(mssg *pzsyslog.Message) error {
	var err error
	isAudit := mssg.AuditData != nil

	if service.logWriter != nil {
		err = service.logWriter.Write(mssg, service.async)
		if err != nil {
			return fmt.Errorf("syslog.Service.postSyslog: %s", err.Error())
		}
	}

	if isAudit && service.auditWriter != nil {
		err = service.auditWriter.Write(mssg, service.async)
		if err != nil {
			return fmt.Errorf("syslog.Service.postSyslog (audit): %s", err.Error())
		}
	}

	return nil
}

func (service *Service) getMessageCommon(params *piazza.HttpQueryParams) (*elasticsearch.SearchResult, *piazza.JsonPagination, *piazza.JsonResponse) {
	pagination, err := piazza.NewJsonPagination(params)
	if err != nil {
		return nil, nil, service.newBadRequestResponse(err)
	}

	paginationCreatedOnToTimeStamp(pagination)

	dsl, err := createQueryDslAsString(pagination, params)
	if err != nil {
		return nil, pagination, service.newBadRequestResponse(err)
	}

	var searchResult *elasticsearch.SearchResult

	if dsl == "" {
		searchResult, err = service.esIndex.FilterByMatchAll(logSchema, pagination)
	} else {
		searchResult, err = service.esIndex.SearchByJSON(logSchema, dsl)
	}
	if err != nil {
		return nil, pagination, service.newInternalErrorResponse(err)
	}

	return searchResult, pagination, nil
}

func (service *Service) GetSyslog(params *piazza.HttpQueryParams) *piazza.JsonResponse {
	var err error

	format, err := params.GetAsString("format", "json")
	if err != nil {
		return service.newInternalErrorResponse(err)
	}

	searchResult, pagination, jErr := service.getMessageCommon(params)
	if jErr != nil {
		return jErr
	}

	lines, err := extractFromSearchResult(searchResult)
	if err != nil {
		return service.newInternalErrorResponse(err)
	}

	var data interface{} = lines

	if format == "string" {
		t := make([]string, len(lines))
		for i := 0; i < len(lines); i++ {
			t[i] = lines[i].String()
		}
		data = t
	}

	pagination.Count = int(searchResult.TotalHits())
	resp := &piazza.JsonResponse{
		StatusCode: http.StatusOK,
		Data:       data,
		Pagination: pagination,
	}

	err = resp.SetType()
	if err != nil {
		return service.newInternalErrorResponse(err)
	}

	return resp
}

func extractFromSearchResult(searchResult *elasticsearch.SearchResult) ([]pzsyslog.Message, error) {

	var lines = make([]pzsyslog.Message, 0, len(*searchResult.GetHits()))

	if searchResult == nil || searchResult.GetHits() == nil {
		return lines, nil
	}

	for _, hit := range *searchResult.GetHits() {
		if hit.Source == nil {
			log.Printf("null source hit")
			continue
		}

		msg := pzsyslog.Message{}
		err := json.Unmarshal(*hit.Source, &msg)
		if err != nil {
			log.Printf("UNABLE TO PARSE: %s", string(*hit.Source))
			continue
		}

		// just in case
		/*err = msg.Validate()
		if err != nil {
			log.Printf("UNABLE TO VALIDATE: %s", string(*hit.Source))
			continue
		}*/

		lines = append(lines, msg)
	}

	return lines, nil
}

func (service *Service) PostQuery(params *piazza.HttpQueryParams, jsnQuery string) *piazza.JsonResponse {

	pagination, err := piazza.NewJsonPagination(params)
	if err != nil {
		return service.newBadRequestResponse(err)
	}

	searchResult, err := service.esIndex.SearchByJSON(logSchema, jsnQuery)
	if err != nil {
		return service.newInternalErrorResponse(err)
	}

	lines, err := extractFromSearchResult(searchResult)
	if err != nil {
		return service.newInternalErrorResponse(err)
	}

	pagination.Count = int(searchResult.TotalHits())
	resp := &piazza.JsonResponse{
		StatusCode: http.StatusOK,
		Data:       lines,
		Pagination: pagination,
	}

	err = resp.SetType()
	if err != nil {
		return service.newInternalErrorResponse(err)
	}

	return resp
}
<commit_msg>Change query to include more fields<commit_after>// Copyright 2016, RadiantBlue Technologies, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package logger

import (
	"encoding/json"
	"fmt"
	"log"
	"net/http"
	"sync"
	"time"

	"github.com/venicegeo/pz-gocommon/elasticsearch"
	"github.com/venicegeo/pz-gocommon/gocommon"
	pzsyslog "github.com/venicegeo/pz-gocommon/syslog"
)

type Service struct {
	sync.Mutex

	stats  Stats
	origin string
	async  bool

	logWriter   pzsyslog.Writer
	auditWriter pzsyslog.Writer

	esIndex elasticsearch.IIndex
}

func (service *Service) Init(sys *piazza.SystemConfig, logWriter pzsyslog.Writer, auditWriter pzsyslog.Writer, esi elasticsearch.IIndex, asyncLogging bool) error {
	service.stats.CreatedOn = time.Now()
	service.stats.NumMessagesByApplication = map[string]int{}

	service.logWriter = logWriter
	service.auditWriter = auditWriter

	service.esIndex = esi

	service.origin = string(sys.Name)

	service.async = asyncLogging

	return nil
}

func (service *Service) newInternalErrorResponse(err error) *piazza.JsonResponse {
	return &piazza.JsonResponse{
		StatusCode: http.StatusInternalServerError,
		Message:    err.Error(),
		Origin:     service.origin,
	}
}

func (service *Service) newBadRequestResponse(err error) *piazza.JsonResponse {
	return &piazza.JsonResponse{
		StatusCode: http.StatusBadRequest,
		Message:    err.Error(),
		Origin:     service.origin,
	}
}

func (service *Service) incrementStats(application string) {
	service.Lock()
	service.stats.NumMessages++
	service.stats.NumMessagesByApplication[application]++
	service.Unlock()
}

func (service *Service) GetRoot() *piazza.JsonResponse {
	resp := &piazza.JsonResponse{
		StatusCode: 200,
		Data:       "Hi. I'm pz-logger.",
	}

	err := resp.SetType()
	if err != nil {
		return service.newInternalErrorResponse(err)
	}

	return resp
}

func (service *Service) GetStats() *piazza.JsonResponse {
	service.Lock()
	t := service.stats
	service.Unlock()

	resp := &piazza.JsonResponse{
		StatusCode: http.StatusOK,
		Data:       t,
	}

	err := resp.SetType()
	if err != nil {
		return service.newInternalErrorResponse(err)
	}

	return resp
}

func createQueryDslAsString(
	pagination *piazza.JsonPagination,
	params *piazza.HttpQueryParams) (string, error) {

	must := []map[string]interface{}{}

	service, err := params.GetAsString("service", "")
	if err != nil {
		return "", err
	}

	contains, err := params.GetAsString("contains", "")
	if err != nil {
		return "", err
	}

	before, err := params.GetBefore(time.Time{})
	if err != nil {
		return "", err
	}

	after, err := params.GetAfter(time.Time{})
	if err != nil {
		return "", err
	}

	if service != "" {
		must = append(must, map[string]interface{}{
			"match": map[string]interface{}{
				"application": service,
			},
		})
	}
	if contains != "" {
		should := []map[string]interface{}{}
		should = append(should, map[string]interface{}{
			"query": map[string]interface{}{
				"wildcard": map[string]interface{}{
					"hostName": map[string]interface{}{
						"value": "*" + contains + "*",
					},
				},
			},
		})
		should = append(should, map[string]interface{}{
			"query": map[string]interface{}{
				"wildcard": map[string]interface{}{
					"application": map[string]interface{}{
						"value": "*" + contains + "*",
					},
				},
			},
		})
		should = append(should, map[string]interface{}{
			"query": map[string]interface{}{
				"wildcard": map[string]interface{}{
					"process": map[string]interface{}{
						"value": "*" + contains + "*",
					},
				},
			},
		})
		should = append(should, map[string]interface{}{
			"query": map[string]interface{}{
				"wildcard": map[string]interface{}{
					"messageId": map[string]interface{}{
						"value": "*" + contains + "*",
					},
				},
			},
		})
		should = append(should, map[string]interface{}{
			"query": map[string]interface{}{
				"wildcard": map[string]interface{}{
					"message": map[string]interface{}{
						"value": "*" + contains + "*",
					},
				},
			},
		})
		queries := make([]interface{}, len(should))
		for i, s := range should {
			queries[i] = s
		}
		must = append(must, map[string]interface{}{
			"filtered": map[string]interface{}{
				"query":  map[string]interface{}{
					"match_all": map[string]interface{}{},
				},
				"filter": map[string]interface{}{
					"bool": map[string]interface{}{
						"should": queries,
					},
				},
			},
		})
	}

	if !after.IsZero() || !before.IsZero() {
		rangeParams := map[string]time.Time{}

		if !after.IsZero() {
			rangeParams["gte"] = after
		}

		if !before.IsZero() {
			rangeParams["lte"] = before
		}

		must = append(must, map[string]interface{}{
			"range": map[string]interface{}{
				"timeStamp": rangeParams,
			},
		})
	}

	if len(must) == 0 {
		return "", nil
	}

	dsl := map[string]interface{}{
		"query": map[string]interface{}{
			"filtered": map[string]interface{}{
				"query": map[string]interface{}{
					"bool": map[string]interface{}{
						"must": must,
					},
				},
			},
		},
		"size": pagination.PerPage,
		"from": pagination.PerPage * pagination.Page,
	}

	dsl["sort"] = map[string]string{
		pagination.SortBy: string(pagination.Order),
	}

	output, err := json.Marshal(dsl)
	if err != nil {
		return "", err
	}

	return string(output), nil
}

func (service *Service) PostSyslog(mNew *pzsyslog.Message) *piazza.JsonResponse {
	err := mNew.Validate()
	if err != nil {
		return service.newBadRequestResponse(err)
	}

	err = service.postSyslog(mNew)
	if err != nil {
		return service.newInternalErrorResponse(err)
	}

	service.incrementStats(mNew.Application)

	resp := &piazza.JsonResponse{
		StatusCode: http.StatusOK,
	}
	return resp
}

func (service *Service) postSyslog(mssg *pzsyslog.Message) error {
	var err error
	isAudit := mssg.AuditData != nil

	if service.logWriter != nil {
		err = service.logWriter.Write(mssg, service.async)
		if err != nil {
			return fmt.Errorf("syslog.Service.postSyslog: %s", err.Error())
		}
	}

	if isAudit && service.auditWriter != nil {
		err = service.auditWriter.Write(mssg, service.async)
		if err != nil {
			return fmt.Errorf("syslog.Service.postSyslog (audit): %s", err.Error())
		}
	}

	return nil
}

func (service *Service) getMessageCommon(params *piazza.HttpQueryParams) (*elasticsearch.SearchResult, *piazza.JsonPagination, *piazza.JsonResponse) {
	pagination, err := piazza.NewJsonPagination(params)
	if err != nil {
		return nil, nil, service.newBadRequestResponse(err)
	}

	paginationCreatedOnToTimeStamp(pagination)

	dsl, err := createQueryDslAsString(pagination, params)
	if err != nil {
		return nil, pagination, service.newBadRequestResponse(err)
	}

	var searchResult *elasticsearch.SearchResult

	if dsl == "" {
		searchResult, err = service.esIndex.FilterByMatchAll(logSchema, pagination)
	} else {
		searchResult, err = service.esIndex.SearchByJSON(logSchema, dsl)
	}
	if err != nil {
		return nil, pagination, service.newInternalErrorResponse(err)
	}

	return searchResult, pagination, nil
}

func (service *Service) GetSyslog(params *piazza.HttpQueryParams) *piazza.JsonResponse {
	var err error

	format, err := params.GetAsString("format", "json")
	if err != nil {
		return service.newInternalErrorResponse(err)
	}

	searchResult, pagination, jErr := service.getMessageCommon(params)
	if jErr != nil {
		return jErr
	}

	lines, err := extractFromSearchResult(searchResult)
	if err != nil {
		return service.newInternalErrorResponse(err)
	}

	var data interface{} = lines

	if format == "string" {
		t := make([]string, len(lines))
		for i := 0; i < len(lines); i++ {
			t[i] = lines[i].String()
		}
		data = t
	}

	pagination.Count = int(searchResult.TotalHits())
	resp := &piazza.JsonResponse{
		StatusCode: http.StatusOK,
		Data:       data,
		Pagination: pagination,
	}

	err = resp.SetType()
	if err != nil {
		return service.newInternalErrorResponse(err)
	}

	return resp
}

func extractFromSearchResult(searchResult *elasticsearch.SearchResult) ([]pzsyslog.Message, error) {

	var lines = make([]pzsyslog.Message, 0, len(*searchResult.GetHits()))

	if searchResult == nil || searchResult.GetHits() == nil {
		return lines, nil
	}

	for _, hit := range *searchResult.GetHits() {
		if hit.Source == nil {
			log.Printf("null source hit")
			continue
		}

		msg := pzsyslog.Message{}
		err := json.Unmarshal(*hit.Source, &msg)
		if err != nil {
			log.Printf("UNABLE TO PARSE: %s", string(*hit.Source))
			continue
		}

		// just in case
		/*err = msg.Validate()
		if err != nil {
			log.Printf("UNABLE TO VALIDATE: %s", string(*hit.Source))
			continue
		}*/

		lines = append(lines, msg)
	}

	return lines, nil
}

func (service *Service) PostQuery(params *piazza.HttpQueryParams, jsnQuery string) *piazza.JsonResponse {

	pagination, err := piazza.NewJsonPagination(params)
	if err != nil {
		return service.newBadRequestResponse(err)
	}

	searchResult, err := service.esIndex.SearchByJSON(logSchema, jsnQuery)
	if err != nil {
		return service.newInternalErrorResponse(err)
	}

	lines, err := extractFromSearchResult(searchResult)
	if err != nil {
		return service.newInternalErrorResponse(err)
	}

	pagination.Count = int(searchResult.TotalHits())
	resp := &piazza.JsonResponse{
		StatusCode: http.StatusOK,
		Data:       lines,
		Pagination: pagination,
	}

	err = resp.SetType()
	if err != nil {
		return service.newInternalErrorResponse(err)
	}

	return resp
}
<|endoftext|>


<commit_before>// Copyright 2013, Cong Ding. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// author: Cong Ding <dinggnu@gmail.com>
//
package logging

import (
	"os"
	"path"
	"runtime"
	"sync/atomic"
	"time"
)

// The struct for each log record
type record struct {
	level    Level
	seqid    uint64
	pathname string
	filename string
	module   string
	lineno   int
	funcname string
	thread   int
	process  int
	message  string
	time     time.Time
}

// This variable maps fields in format to relavent function signatures
var fields = map[string]func(*Logger, *record) interface{}{
	"name":            (*Logger).lname,           // name of the logger
	"seqid":           (*Logger).nextSeqid,       // sequence number
	"levelno":         (*Logger).levelno,         // level number
	"levelname":       (*Logger).levelname,       // level name
	"filename":        (*Logger).filename,        // source filename of the caller
	"pathname":        (*Logger).pathname,        // filename with path
	"module":          (*Logger).module,          // executable filename
	"lineno":          (*Logger).lineno,          // line number in source code
	"funcname":        (*Logger).funcname,        // function name of the caller
	"created":         (*Logger).created,         // starting time of the logger
	"time":            (*Logger).time,            // record created time
	"msecs":           (*Logger).msecs,           // nanosecond of the starting time
	"relativeCreated": (*Logger).relativeCreated, // nanoseconds since started
	"thread":          (*Logger).thread,          // thread id
	"process":         (*Logger).process,         // process id
	"message":         (*Logger).message,         // logger message
	"timestamp":       (*Logger).timestamp,       // timestamp of record
}

// If it fails to get some fields with string type, these fields are set to
// errString value.
const errString = "???"

// GetGoID returns the id of goroutine, which is defined in ./get_go_id.c
func GetGoID() int32

// genRuntime generates the runtime information, including pathname, function
// name, filename, line number.
func genRuntime(r *record) {
	calldepth := 5
	pc, file, line, ok := runtime.Caller(calldepth)
	if ok {
		// Generate short function name
		fname := runtime.FuncForPC(pc).Name()
		fshort := fname
		for i := len(fname) - 1; i > 0; i-- {
			if fname[i] == '.' {
				fshort = fname[i+1:]
				break
			}
		}

		r.pathname = file
		r.funcname = fshort
		r.filename = path.Base(file)
		r.lineno = line
	} else {
		r.pathname = errString
		r.funcname = errString
		r.filename = errString
		// Here we uses -1 rather than 0, because the default value in
		// golang is 0 and we should know the value is uninitialized
		// or failed to get
		r.lineno = -1
	}
}

// Logger name
func (logger *Logger) lname(r *record) interface{} {
	return logger.name
}

// Next sequence number
func (logger *Logger) nextSeqid(r *record) interface{} {
	if r.seqid == 0 {
		r.seqid = atomic.AddUint64(&(logger.seqid), 1)
	}
	return r.seqid
}

// Log level number
func (logger *Logger) levelno(r *record) interface{} {
	return int(r.level)
}

// Log level name
func (logger *Logger) levelname(r *record) interface{} {
	return levelNames[r.level]
}

// File name of calling logger, with whole path
func (logger *Logger) pathname(r *record) interface{} {
	if r.pathname == "" {
		genRuntime(r)
	}
	return r.pathname
}

// File name of calling logger
func (logger *Logger) filename(r *record) interface{} {
	if r.filename == "" {
		genRuntime(r)
	}
	return r.filename
}

// TODO: module name
func (logger *Logger) module(r *record) interface{} {
	return ""
}

// Line number
func (logger *Logger) lineno(r *record) interface{} {
	if r.lineno == 0 {
		genRuntime(r)
	}
	return r.lineno
}

// Function name
func (logger *Logger) funcname(r *record) interface{} {
	if r.funcname == "" {
		genRuntime(r)
	}
	return r.funcname
}

// Timestamp of starting time
func (logger *Logger) created(r *record) interface{} {
	return logger.startTime.UnixNano()
}

// RFC3339Nano time
func (logger *Logger) time(r *record) interface{} {
	if r.time.IsZero() {
		r.time = time.Now()
	}
	return r.time.Format(logger.timeFormat)
}

// Nanosecond of starting time
func (logger *Logger) msecs(r *record) interface{} {
	return logger.startTime.Nanosecond()
}

// Nanosecond timestamp
func (logger *Logger) timestamp(r *record) interface{} {
	if r.time.IsZero() {
		r.time = time.Now()
	}
	return r.time.UnixNano()
}

// Nanoseconds since logger created
func (logger *Logger) relativeCreated(r *record) interface{} {
	if r.time.IsZero() {
		r.time = time.Now()
	}
	return r.time.Sub(logger.startTime).Nanoseconds()
}

// Thread ID
func (logger *Logger) thread(r *record) interface{} {
	if r.thread == 0 {
		r.thread = int(GetGoID())
	}
	return r.thread
}

// Process ID
func (logger *Logger) process(r *record) interface{} {
	if r.process == 0 {
		r.process = os.Getpid()
	}
	return r.process
}

// The log message
func (logger *Logger) message(r *record) interface{} {
	return r.message
}
<commit_msg>update fields<commit_after>// Copyright 2013, Cong Ding. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// author: Cong Ding <dinggnu@gmail.com>
//
package logging

import (
	"os"
	"path"
	"runtime"
	"sync/atomic"
	"time"
)

// The struct for each log record
type record struct {
	level    Level
	seqid    uint64
	pathname string
	filename string
	module   string
	lineno   int
	funcname string
	thread   int
	process  int
	message  string
	time     time.Time
}

// This variable maps fields in format to relavent function signatures
var fields = map[string]func(*Logger, *record) interface{}{
	"name":      (*Logger).lname,     // name of the logger
	"seqid":     (*Logger).nextSeqid, // sequence number
	"levelno":   (*Logger).levelno,   // level number
	"levelname": (*Logger).levelname, // level name
	"created":   (*Logger).created,   // starting time of the logger
	"nsecs":     (*Logger).nsecs,     // nanosecond of the starting time
	"time":      (*Logger).time,      // record created time
	"timestamp": (*Logger).timestamp, // timestamp of record
	"rtime":     (*Logger).rtime,     // relative time since started
	"filename":  (*Logger).filename,  // source filename of the caller
	"pathname":  (*Logger).pathname,  // filename with path
	"module":    (*Logger).module,    // executable filename
	"lineno":    (*Logger).lineno,    // line number in source code
	"funcname":  (*Logger).funcname,  // function name of the caller
	"thread":    (*Logger).thread,    // thread id
	"process":   (*Logger).process,   // process id
	"message":   (*Logger).message,   // logger message
}

// If it fails to get some fields with string type, these fields are set to
// errString value.
const errString = "???"

// GetGoID returns the id of goroutine, which is defined in ./get_go_id.c
func GetGoID() int32

// genRuntime generates the runtime information, including pathname, function
// name, filename, line number.
func genRuntime(r *record) {
	calldepth := 5
	pc, file, line, ok := runtime.Caller(calldepth)
	if ok {
		// Generate short function name
		fname := runtime.FuncForPC(pc).Name()
		fshort := fname
		for i := len(fname) - 1; i > 0; i-- {
			if fname[i] == '.' {
				fshort = fname[i+1:]
				break
			}
		}

		r.pathname = file
		r.funcname = fshort
		r.filename = path.Base(file)
		r.lineno = line
	} else {
		r.pathname = errString
		r.funcname = errString
		r.filename = errString
		// Here we uses -1 rather than 0, because the default value in
		// golang is 0 and we should know the value is uninitialized
		// or failed to get
		r.lineno = -1
	}
}

// Logger name
func (logger *Logger) lname(r *record) interface{} {
	return logger.name
}

// Next sequence number
func (logger *Logger) nextSeqid(r *record) interface{} {
	if r.seqid == 0 {
		r.seqid = atomic.AddUint64(&(logger.seqid), 1)
	}
	return r.seqid
}

// Log level number
func (logger *Logger) levelno(r *record) interface{} {
	return int(r.level)
}

// Log level name
func (logger *Logger) levelname(r *record) interface{} {
	return levelNames[r.level]
}

// File name of calling logger, with whole path
func (logger *Logger) pathname(r *record) interface{} {
	if r.pathname == "" {
		genRuntime(r)
	}
	return r.pathname
}

// File name of calling logger
func (logger *Logger) filename(r *record) interface{} {
	if r.filename == "" {
		genRuntime(r)
	}
	return r.filename
}

// TODO: module name
func (logger *Logger) module(r *record) interface{} {
	return ""
}

// Line number
func (logger *Logger) lineno(r *record) interface{} {
	if r.lineno == 0 {
		genRuntime(r)
	}
	return r.lineno
}

// Function name
func (logger *Logger) funcname(r *record) interface{} {
	if r.funcname == "" {
		genRuntime(r)
	}
	return r.funcname
}

// Timestamp of starting time
func (logger *Logger) created(r *record) interface{} {
	return logger.startTime.UnixNano()
}

// RFC3339Nano time
func (logger *Logger) time(r *record) interface{} {
	if r.time.IsZero() {
		r.time = time.Now()
	}
	return r.time.Format(logger.timeFormat)
}

// Nanosecond of starting time
func (logger *Logger) nsecs(r *record) interface{} {
	return logger.startTime.Nanosecond()
}

// Nanosecond timestamp
func (logger *Logger) timestamp(r *record) interface{} {
	if r.time.IsZero() {
		r.time = time.Now()
	}
	return r.time.UnixNano()
}

// Nanoseconds since logger created
func (logger *Logger) rtime(r *record) interface{} {
	if r.time.IsZero() {
		r.time = time.Now()
	}
	return r.time.Sub(logger.startTime).Nanoseconds()
}

// Thread ID
func (logger *Logger) thread(r *record) interface{} {
	if r.thread == 0 {
		r.thread = int(GetGoID())
	}
	return r.thread
}

// Process ID
func (logger *Logger) process(r *record) interface{} {
	if r.process == 0 {
		r.process = os.Getpid()
	}
	return r.process
}

// The log message
func (logger *Logger) message(r *record) interface{} {
	return r.message
}
<|endoftext|>


<commit_before>package logging

import (
	"io"
	"os"
	"strings"

	"github.com/go-kit/kit/log"
	"github.com/go-kit/kit/log/level"
	"gopkg.in/natefinch/lumberjack.v2"
)

var (
	defaultLogger = New(&Options{Level: "DEBUG"})

	callerKey    interface{} = "caller"
	messageKey   interface{} = "msg"
	errorKey     interface{} = "error"
	timestampKey interface{} = "ts"
)

// CallerKey returns the logging key to be used for the stack location of the logging call
func CallerKey() interface{} {
	return callerKey
}

// MessageKey returns the logging key to be used for the textual message of the log entry
func MessageKey() interface{} {
	return messageKey
}

// ErrorKey returns the logging key to be used for error instances
func ErrorKey() interface{} {
	return errorKey
}

// TimestampKey returns the logging key to be used for the timestamp
func TimestampKey() interface{} {
	return timestampKey
}

// DefaultLogger returns a global singleton default that logs to os.Stdout.
// This returned instance is safe for concurrent access.
func DefaultLogger() log.Logger {
	return defaultLogger
}

// Options stores the configuration of a Logger.  Lumberjack is used for rolling files.
type Options struct {
	// File is the lumberjack Logger file information.  If nil, output is sent to the console.
	File *lumberjack.Logger `json:"file"`

	// JSON is a flag indicating whether JSON logging output is used.  The default is false,
	// meaning that logfmt output is used.
	JSON bool `json:"json"`

	// Level is the error level to output: ERROR, INFO, WARN, or DEBUG.  Any unrecognized string,
	// including the empty string, is equivalent to passing ERROR.
	Level string `json:"level"`

	// LoggerFactory overrides the JSON field if specified.  This function is used to produce
	// a go-kit Logger from an io.Writer.
	LoggerFactory func(io.Writer) log.Logger
}

func (o *Options) file() *lumberjack.Logger {
	if o != nil {
		return o.File
	}

	return nil
}

func (o *Options) json() bool {
	if o != nil {
		return o.JSON
	}

	return false
}

func (o *Options) output() io.Writer {
	if o != nil && o.File != nil {
		return o.File
	}

	return log.NewSyncWriter(os.Stdout)
}

func (o *Options) loggerFactory() func(io.Writer) log.Logger {
	if o != nil {
		if o.LoggerFactory != nil {
			return o.LoggerFactory
		} else if o.JSON {
			return log.NewJSONLogger
		}
	}

	return log.NewLogfmtLogger
}

func (o *Options) level() string {
	if o != nil {
		return o.Level
	}

	return ""
}

// New creates a go-kit Logger from a set of options.  The options object can be nil,
// in which case a default logger that logs to os.Stdout is returned.  The returned logger
// includes the timestamp in UTC format and will filter according to the Level field.
//
// In order to allow arbitrary decoration, this function does not insert the caller information.
// Use DefaultCaller or Caller in this package to do that.
func New(o *Options) log.Logger {
	return NewFilter(
		log.With(
			o.loggerFactory()(o.output()),
			TimestampKey(), log.DefaultTimestampUTC,
		),
		o,
	)
}

// NewFilter applies the Options filtering rules in the package to an arbitrary go-kit Logger.
func NewFilter(next log.Logger, o *Options) log.Logger {
	switch strings.ToUpper(o.level()) {
	case "DEBUG":
		return level.NewFilter(next, level.AllowDebug())

	case "INFO":
		return level.NewFilter(next, level.AllowInfo())

	case "WARN":
		return level.NewFilter(next, level.AllowWarn())

	default:
		return level.NewFilter(next, level.AllowError())
	}
}

// DefaultCaller produces a contextual logger as with log.With, but automatically prepends the
// caller under the CallerKey.
//
// The logger returned by this function should not be further decorated.  This will cause the
// callstack to include the decorators, which is pointless.  Instead, decorate the next parameter
// prior to passing it to this function.
func DefaultCaller(next log.Logger, keyvals ...interface{}) log.Logger {
	return log.WithPrefix(
		next,
		append([]interface{}{CallerKey(), log.DefaultCaller}, keyvals...)...,
	)
}

func Error(next log.Logger, keyvals ...interface{}) log.Logger {
	return log.WithPrefix(
		next,
		append([]interface{}{CallerKey(), log.DefaultCaller, level.Key(), level.ErrorValue()}, keyvals...)...,
	)
}

func Info(next log.Logger, keyvals ...interface{}) log.Logger {
	return log.WithPrefix(
		next,
		append([]interface{}{CallerKey(), log.DefaultCaller, level.Key(), level.InfoValue()}, keyvals...)...,
	)
}

func Warn(next log.Logger, keyvals ...interface{}) log.Logger {
	return log.WithPrefix(
		next,
		append([]interface{}{CallerKey(), log.DefaultCaller, level.Key(), level.WarnValue()}, keyvals...)...,
	)
}

func Debug(next log.Logger, keyvals ...interface{}) log.Logger {
	return log.WithPrefix(
		next,
		append([]interface{}{CallerKey(), log.DefaultCaller, level.Key(), level.DebugValue()}, keyvals...)...,
	)
}
<commit_msg>Moved to a flat structure to accomodate viper<commit_after>package logging

import (
	"io"
	"os"
	"strings"

	lumberjack "gopkg.in/natefinch/lumberjack.v2"

	"github.com/go-kit/kit/log"
	"github.com/go-kit/kit/log/level"
)

const (
	StdoutFile = "stdout"
)

var (
	defaultLogger = New(&Options{Level: "DEBUG"})

	callerKey    interface{} = "caller"
	messageKey   interface{} = "msg"
	errorKey     interface{} = "error"
	timestampKey interface{} = "ts"
)

// CallerKey returns the logging key to be used for the stack location of the logging call
func CallerKey() interface{} {
	return callerKey
}

// MessageKey returns the logging key to be used for the textual message of the log entry
func MessageKey() interface{} {
	return messageKey
}

// ErrorKey returns the logging key to be used for error instances
func ErrorKey() interface{} {
	return errorKey
}

// TimestampKey returns the logging key to be used for the timestamp
func TimestampKey() interface{} {
	return timestampKey
}

// DefaultLogger returns a global singleton default that logs to os.Stdout.
// This returned instance is safe for concurrent access.
func DefaultLogger() log.Logger {
	return defaultLogger
}

// Options stores the configuration of a Logger.  Lumberjack is used for rolling files.
type Options struct {
	// File is the system file path for the log file.  If set to "stdout", this will log to os.Stdout.
	// Otherwise, a lumberjack.Logger is created
	File string `json:"file"`

	// MaxSize is the lumberjack MaxSize
	MaxSize int `json:"maxsize"`

	// MaxAge is the lumberjack MaxAge
	MaxAge int `json:"maxage"`

	// MaxBackups is the lumberjack MaxBackups
	MaxBackups int `json:"maxbackups"`

	// JSON is a flag indicating whether JSON logging output is used.  The default is false,
	// meaning that logfmt output is used.
	JSON bool `json:"json"`

	// Level is the error level to output: ERROR, INFO, WARN, or DEBUG.  Any unrecognized string,
	// including the empty string, is equivalent to passing ERROR.
	Level string `json:"level"`

	// LoggerFactory overrides the JSON field if specified.  This function is used to produce
	// a go-kit Logger from an io.Writer.
	LoggerFactory func(io.Writer) log.Logger
}

func (o *Options) json() bool {
	if o != nil {
		return o.JSON
	}

	return false
}

func (o *Options) output() io.Writer {
	if o != nil && o.File != StdoutFile {
		return &lumberjack.Logger{
			Filename:   o.File,
			MaxSize:    o.MaxSize,
			MaxAge:     o.MaxAge,
			MaxBackups: o.MaxBackups,
		}
	}

	return log.NewSyncWriter(os.Stdout)
}

func (o *Options) loggerFactory() func(io.Writer) log.Logger {
	if o != nil {
		if o.LoggerFactory != nil {
			return o.LoggerFactory
		} else if o.JSON {
			return log.NewJSONLogger
		}
	}

	return log.NewLogfmtLogger
}

func (o *Options) level() string {
	if o != nil {
		return o.Level
	}

	return ""
}

// New creates a go-kit Logger from a set of options.  The options object can be nil,
// in which case a default logger that logs to os.Stdout is returned.  The returned logger
// includes the timestamp in UTC format and will filter according to the Level field.
//
// In order to allow arbitrary decoration, this function does not insert the caller information.
// Use DefaultCaller or Caller in this package to do that.
func New(o *Options) log.Logger {
	return NewFilter(
		log.With(
			o.loggerFactory()(o.output()),
			TimestampKey(), log.DefaultTimestampUTC,
		),
		o,
	)
}

// NewFilter applies the Options filtering rules in the package to an arbitrary go-kit Logger.
func NewFilter(next log.Logger, o *Options) log.Logger {
	switch strings.ToUpper(o.level()) {
	case "DEBUG":
		return level.NewFilter(next, level.AllowDebug())

	case "INFO":
		return level.NewFilter(next, level.AllowInfo())

	case "WARN":
		return level.NewFilter(next, level.AllowWarn())

	default:
		return level.NewFilter(next, level.AllowError())
	}
}

// DefaultCaller produces a contextual logger as with log.With, but automatically prepends the
// caller under the CallerKey.
//
// The logger returned by this function should not be further decorated.  This will cause the
// callstack to include the decorators, which is pointless.  Instead, decorate the next parameter
// prior to passing it to this function.
func DefaultCaller(next log.Logger, keyvals ...interface{}) log.Logger {
	return log.WithPrefix(
		next,
		append([]interface{}{CallerKey(), log.DefaultCaller}, keyvals...)...,
	)
}

func Error(next log.Logger, keyvals ...interface{}) log.Logger {
	return log.WithPrefix(
		next,
		append([]interface{}{CallerKey(), log.DefaultCaller, level.Key(), level.ErrorValue()}, keyvals...)...,
	)
}

func Info(next log.Logger, keyvals ...interface{}) log.Logger {
	return log.WithPrefix(
		next,
		append([]interface{}{CallerKey(), log.DefaultCaller, level.Key(), level.InfoValue()}, keyvals...)...,
	)
}

func Warn(next log.Logger, keyvals ...interface{}) log.Logger {
	return log.WithPrefix(
		next,
		append([]interface{}{CallerKey(), log.DefaultCaller, level.Key(), level.WarnValue()}, keyvals...)...,
	)
}

func Debug(next log.Logger, keyvals ...interface{}) log.Logger {
	return log.WithPrefix(
		next,
		append([]interface{}{CallerKey(), log.DefaultCaller, level.Key(), level.DebugValue()}, keyvals...)...,
	)
}
<|endoftext|>