<commit_before>package docker_log

import (
	"bufio"
	"context"
	"crypto/tls"
	"io"
	"strings"
	"sync"
	"time"
	"unicode"

	"github.com/docker/docker/api/types"
	"github.com/docker/docker/api/types/filters"
	"github.com/docker/docker/pkg/stdcopy"
	"github.com/influxdata/telegraf"
	"github.com/influxdata/telegraf/filter"
	"github.com/influxdata/telegraf/internal"
	"github.com/influxdata/telegraf/internal/docker"
	tlsint "github.com/influxdata/telegraf/internal/tls"
	"github.com/influxdata/telegraf/plugins/inputs"
)

var sampleConfig = `
  ## Docker Endpoint
  ##   To use TCP, set endpoint = "tcp://[ip]:[port]"
  ##   To use environment variables (ie, docker-machine), set endpoint = "ENV"
  # endpoint = "unix:///var/run/docker.sock"

  ## When true, container logs are read from the beginning; otherwise
  ## reading begins at the end of the log.
  # from_beginning = false

  ## Timeout for Docker API calls.
  # timeout = "5s"

  ## Containers to include and exclude. Globs accepted.
  ## Note that an empty array for both will include all containers
  # container_name_include = []
  # container_name_exclude = []

  ## Container states to include and exclude. Globs accepted.
  ## When empty only containers in the "running" state will be captured.
  # container_state_include = []
  # container_state_exclude = []

  ## docker labels to include and exclude as tags.  Globs accepted.
  ## Note that an empty array for both will include all labels as tags
  # docker_label_include = []
  # docker_label_exclude = []

  ## Set the source tag for the metrics to the container ID hostname, eg first 12 chars
  source_tag = false

  ## Optional TLS Config
  # tls_ca = "/etc/telegraf/ca.pem"
  # tls_cert = "/etc/telegraf/cert.pem"
  # tls_key = "/etc/telegraf/key.pem"
  ## Use TLS but skip chain & host verification
  # insecure_skip_verify = false
`

const (
	defaultEndpoint = "unix:///var/run/docker.sock"

	// Maximum bytes of a log line before it will be split, size is mirroring
	// docker code:
	// https://github.com/moby/moby/blob/master/daemon/logger/copier.go#L21
	maxLineBytes = 16 * 1024
)

var (
	containerStates = []string{"created", "restarting", "running", "removing", "paused", "exited", "dead"}
	// ensure *DockerLogs implements telegaf.ServiceInput
	_ telegraf.ServiceInput = (*DockerLogs)(nil)
)

type DockerLogs struct {
	Endpoint              string            `toml:"endpoint"`
	FromBeginning         bool              `toml:"from_beginning"`
	Timeout               internal.Duration `toml:"timeout"`
	LabelInclude          []string          `toml:"docker_label_include"`
	LabelExclude          []string          `toml:"docker_label_exclude"`
	ContainerInclude      []string          `toml:"container_name_include"`
	ContainerExclude      []string          `toml:"container_name_exclude"`
	ContainerStateInclude []string          `toml:"container_state_include"`
	ContainerStateExclude []string          `toml:"container_state_exclude"`
	IncludeSourceTag      bool              `toml:"source_tag"`

	tlsint.ClientConfig

	newEnvClient func() (Client, error)
	newClient    func(string, *tls.Config) (Client, error)

	client          Client
	labelFilter     filter.Filter
	containerFilter filter.Filter
	stateFilter     filter.Filter
	opts            types.ContainerListOptions
	wg              sync.WaitGroup
	mu              sync.Mutex
	containerList   map[string]context.CancelFunc
}

func (d *DockerLogs) Description() string {
	return "Read logging output from the Docker engine"
}

func (d *DockerLogs) SampleConfig() string {
	return sampleConfig
}

func (d *DockerLogs) Init() error {
	var err error
	if d.Endpoint == "ENV" {
		d.client, err = d.newEnvClient()
		if err != nil {
			return err
		}
	} else {
		tlsConfig, err := d.ClientConfig.TLSConfig()
		if err != nil {
			return err
		}
		d.client, err = d.newClient(d.Endpoint, tlsConfig)
		if err != nil {
			return err
		}
	}

	// Create filters
	err = d.createLabelFilters()
	if err != nil {
		return err
	}
	err = d.createContainerFilters()
	if err != nil {
		return err
	}
	err = d.createContainerStateFilters()
	if err != nil {
		return err
	}

	filterArgs := filters.NewArgs()
	for _, state := range containerStates {
		if d.stateFilter.Match(state) {
			filterArgs.Add("status", state)
		}
	}

	if filterArgs.Len() != 0 {
		d.opts = types.ContainerListOptions{
			Filters: filterArgs,
		}
	}

	return nil
}

func (d *DockerLogs) addToContainerList(containerID string, cancel context.CancelFunc) error {
	d.mu.Lock()
	defer d.mu.Unlock()
	d.containerList[containerID] = cancel
	return nil
}

func (d *DockerLogs) removeFromContainerList(containerID string) error {
	d.mu.Lock()
	defer d.mu.Unlock()
	delete(d.containerList, containerID)
	return nil
}

func (d *DockerLogs) containerInContainerList(containerID string) bool {
	d.mu.Lock()
	defer d.mu.Unlock()
	_, ok := d.containerList[containerID]
	return ok
}

func (d *DockerLogs) cancelTails() error {
	d.mu.Lock()
	defer d.mu.Unlock()
	for _, cancel := range d.containerList {
		cancel()
	}
	return nil
}

func (d *DockerLogs) matchedContainerName(names []string) string {
	// Check if all container names are filtered; in practice I believe
	// this array is always of length 1.
	for _, name := range names {
		trimmedName := strings.TrimPrefix(name, "/")
		match := d.containerFilter.Match(trimmedName)
		if match {
			return trimmedName
		}
	}
	return ""
}

func (d *DockerLogs) Gather(acc telegraf.Accumulator) error {
	ctx := context.Background()

	ctx, cancel := context.WithTimeout(ctx, d.Timeout.Duration)
	defer cancel()
	containers, err := d.client.ContainerList(ctx, d.opts)
	if err != nil {
		return err
	}

	for _, container := range containers {
		if d.containerInContainerList(container.ID) {
			continue
		}

		containerName := d.matchedContainerName(container.Names)
		if containerName == "" {
			continue
		}

		ctx, cancel := context.WithCancel(context.Background())
		d.addToContainerList(container.ID, cancel)

		// Start a new goroutine for every new container that has logs to collect
		d.wg.Add(1)
		go func(container types.Container) {
			defer d.wg.Done()
			defer d.removeFromContainerList(container.ID)

			err = d.tailContainerLogs(ctx, acc, container, containerName)
			if err != nil && err != context.Canceled {
				acc.AddError(err)
			}
		}(container)
	}
	return nil
}

func (d *DockerLogs) hasTTY(ctx context.Context, container types.Container) (bool, error) {
	ctx, cancel := context.WithTimeout(ctx, d.Timeout.Duration)
	defer cancel()
	c, err := d.client.ContainerInspect(ctx, container.ID)
	if err != nil {
		return false, err
	}
	return c.Config.Tty, nil
}

func (d *DockerLogs) tailContainerLogs(
	ctx context.Context,
	acc telegraf.Accumulator,
	container types.Container,
	containerName string,
) error {
	imageName, imageVersion := docker.ParseImage(container.Image)
	tags := map[string]string{
		"container_name":    containerName,
		"container_image":   imageName,
		"container_version": imageVersion,
	}

	if d.IncludeSourceTag {
		tags["source"] = hostnameFromID(container.ID)
	}

	// Add matching container labels as tags
	for k, label := range container.Labels {
		if d.labelFilter.Match(k) {
			tags[k] = label
		}
	}

	hasTTY, err := d.hasTTY(ctx, container)
	if err != nil {
		return err
	}

	tail := "0"
	if d.FromBeginning {
		tail = "all"
	}

	logOptions := types.ContainerLogsOptions{
		ShowStdout: true,
		ShowStderr: true,
		Timestamps: false,
		Details:    false,
		Follow:     true,
		Tail:       tail,
	}

	logReader, err := d.client.ContainerLogs(ctx, container.ID, logOptions)
	if err != nil {
		return err
	}

	// If the container is using a TTY, there is only a single stream
	// (stdout), and data is copied directly from the container output stream,
	// no extra multiplexing or headers.
	//
	// If the container is *not* using a TTY, streams for stdout and stderr are
	// multiplexed.
	if hasTTY {
		return tailStream(acc, tags, container.ID, logReader, "tty")
	} else {
		return tailMultiplexed(acc, tags, container.ID, logReader)
	}
}

func tailStream(
	acc telegraf.Accumulator,
	baseTags map[string]string,
	containerID string,
	reader io.ReadCloser,
	stream string,
) error {
	defer reader.Close()

	tags := make(map[string]string, len(baseTags)+1)
	for k, v := range baseTags {
		tags[k] = v
	}
	tags["stream"] = stream

	r := bufio.NewReaderSize(reader, 64*1024)

	var err error
	var message string
	for {
		message, err = r.ReadString('\n')

		// Keep any leading space, but remove whitespace from end of line.
		// This preserves space in, for example, stacktraces, while removing
		// annoying end of line characters and is similar to how other logging
		// plugins such as syslog behave.
		message = strings.TrimRightFunc(message, unicode.IsSpace)

		if len(message) != 0 {
			acc.AddFields("docker_log", map[string]interface{}{
				"container_id": containerID,
				"message":      message,
			}, tags)
		}

		if err != nil {
			if err == io.EOF {
				return nil
			}
			return err
		}
	}
}

func tailMultiplexed(
	acc telegraf.Accumulator,
	tags map[string]string,
	containerID string,
	src io.ReadCloser,
) error {
	outReader, outWriter := io.Pipe()
	errReader, errWriter := io.Pipe()

	var wg sync.WaitGroup
	wg.Add(1)
	go func() {
		defer wg.Done()
		err := tailStream(acc, tags, containerID, outReader, "stdout")
		if err != nil {
			acc.AddError(err)
		}
	}()

	wg.Add(1)
	go func() {
		defer wg.Done()
		err := tailStream(acc, tags, containerID, errReader, "stderr")
		if err != nil {
			acc.AddError(err)
		}
	}()

	_, err := stdcopy.StdCopy(outWriter, errWriter, src)
	outWriter.Close()
	errWriter.Close()
	src.Close()
	wg.Wait()
	return err
}

// Start is a noop which is required for a *DockerLogs to implement
// the telegraf.ServiceInput interface
func (d *DockerLogs) Start(telegraf.Accumulator) error {
	return nil
}

func (d *DockerLogs) Stop() {
	d.cancelTails()
	d.wg.Wait()
}

// Following few functions have been inherited from telegraf docker input plugin
func (d *DockerLogs) createContainerFilters() error {
	filter, err := filter.NewIncludeExcludeFilter(d.ContainerInclude, d.ContainerExclude)
	if err != nil {
		return err
	}
	d.containerFilter = filter
	return nil
}

func (d *DockerLogs) createLabelFilters() error {
	filter, err := filter.NewIncludeExcludeFilter(d.LabelInclude, d.LabelExclude)
	if err != nil {
		return err
	}
	d.labelFilter = filter
	return nil
}

func (d *DockerLogs) createContainerStateFilters() error {
	if len(d.ContainerStateInclude) == 0 && len(d.ContainerStateExclude) == 0 {
		d.ContainerStateInclude = []string{"running"}
	}
	filter, err := filter.NewIncludeExcludeFilter(d.ContainerStateInclude, d.ContainerStateExclude)
	if err != nil {
		return err
	}
	d.stateFilter = filter
	return nil
}

func init() {
	inputs.Add("docker_log", func() telegraf.Input {
		return &DockerLogs{
			Timeout:       internal.Duration{Duration: time.Second * 5},
			Endpoint:      defaultEndpoint,
			newEnvClient:  NewEnvClient,
			newClient:     NewClient,
			containerList: make(map[string]context.CancelFunc),
		}
	})
}

func hostnameFromID(id string) string {
	if len(id) > 12 {
		return id[0:12]
	}
	return id
}
<commit_msg>Use nanosecond precision in docker_log input (#6663)<commit_after>package docker_log

import (
	"bufio"
	"context"
	"crypto/tls"
	"io"
	"strings"
	"sync"
	"time"
	"unicode"

	"github.com/docker/docker/api/types"
	"github.com/docker/docker/api/types/filters"
	"github.com/docker/docker/pkg/stdcopy"
	"github.com/influxdata/telegraf"
	"github.com/influxdata/telegraf/filter"
	"github.com/influxdata/telegraf/internal"
	"github.com/influxdata/telegraf/internal/docker"
	tlsint "github.com/influxdata/telegraf/internal/tls"
	"github.com/influxdata/telegraf/plugins/inputs"
)

var sampleConfig = `
  ## Docker Endpoint
  ##   To use TCP, set endpoint = "tcp://[ip]:[port]"
  ##   To use environment variables (ie, docker-machine), set endpoint = "ENV"
  # endpoint = "unix:///var/run/docker.sock"

  ## When true, container logs are read from the beginning; otherwise
  ## reading begins at the end of the log.
  # from_beginning = false

  ## Timeout for Docker API calls.
  # timeout = "5s"

  ## Containers to include and exclude. Globs accepted.
  ## Note that an empty array for both will include all containers
  # container_name_include = []
  # container_name_exclude = []

  ## Container states to include and exclude. Globs accepted.
  ## When empty only containers in the "running" state will be captured.
  # container_state_include = []
  # container_state_exclude = []

  ## docker labels to include and exclude as tags.  Globs accepted.
  ## Note that an empty array for both will include all labels as tags
  # docker_label_include = []
  # docker_label_exclude = []

  ## Set the source tag for the metrics to the container ID hostname, eg first 12 chars
  source_tag = false

  ## Optional TLS Config
  # tls_ca = "/etc/telegraf/ca.pem"
  # tls_cert = "/etc/telegraf/cert.pem"
  # tls_key = "/etc/telegraf/key.pem"
  ## Use TLS but skip chain & host verification
  # insecure_skip_verify = false
`

const (
	defaultEndpoint = "unix:///var/run/docker.sock"

	// Maximum bytes of a log line before it will be split, size is mirroring
	// docker code:
	// https://github.com/moby/moby/blob/master/daemon/logger/copier.go#L21
	maxLineBytes = 16 * 1024
)

var (
	containerStates = []string{"created", "restarting", "running", "removing", "paused", "exited", "dead"}
	// ensure *DockerLogs implements telegaf.ServiceInput
	_ telegraf.ServiceInput = (*DockerLogs)(nil)
)

type DockerLogs struct {
	Endpoint              string            `toml:"endpoint"`
	FromBeginning         bool              `toml:"from_beginning"`
	Timeout               internal.Duration `toml:"timeout"`
	LabelInclude          []string          `toml:"docker_label_include"`
	LabelExclude          []string          `toml:"docker_label_exclude"`
	ContainerInclude      []string          `toml:"container_name_include"`
	ContainerExclude      []string          `toml:"container_name_exclude"`
	ContainerStateInclude []string          `toml:"container_state_include"`
	ContainerStateExclude []string          `toml:"container_state_exclude"`
	IncludeSourceTag      bool              `toml:"source_tag"`

	tlsint.ClientConfig

	newEnvClient func() (Client, error)
	newClient    func(string, *tls.Config) (Client, error)

	client          Client
	labelFilter     filter.Filter
	containerFilter filter.Filter
	stateFilter     filter.Filter
	opts            types.ContainerListOptions
	wg              sync.WaitGroup
	mu              sync.Mutex
	containerList   map[string]context.CancelFunc
}

func (d *DockerLogs) Description() string {
	return "Read logging output from the Docker engine"
}

func (d *DockerLogs) SampleConfig() string {
	return sampleConfig
}

func (d *DockerLogs) Init() error {
	var err error
	if d.Endpoint == "ENV" {
		d.client, err = d.newEnvClient()
		if err != nil {
			return err
		}
	} else {
		tlsConfig, err := d.ClientConfig.TLSConfig()
		if err != nil {
			return err
		}
		d.client, err = d.newClient(d.Endpoint, tlsConfig)
		if err != nil {
			return err
		}
	}

	// Create filters
	err = d.createLabelFilters()
	if err != nil {
		return err
	}
	err = d.createContainerFilters()
	if err != nil {
		return err
	}
	err = d.createContainerStateFilters()
	if err != nil {
		return err
	}

	filterArgs := filters.NewArgs()
	for _, state := range containerStates {
		if d.stateFilter.Match(state) {
			filterArgs.Add("status", state)
		}
	}

	if filterArgs.Len() != 0 {
		d.opts = types.ContainerListOptions{
			Filters: filterArgs,
		}
	}

	return nil
}

func (d *DockerLogs) addToContainerList(containerID string, cancel context.CancelFunc) error {
	d.mu.Lock()
	defer d.mu.Unlock()
	d.containerList[containerID] = cancel
	return nil
}

func (d *DockerLogs) removeFromContainerList(containerID string) error {
	d.mu.Lock()
	defer d.mu.Unlock()
	delete(d.containerList, containerID)
	return nil
}

func (d *DockerLogs) containerInContainerList(containerID string) bool {
	d.mu.Lock()
	defer d.mu.Unlock()
	_, ok := d.containerList[containerID]
	return ok
}

func (d *DockerLogs) cancelTails() error {
	d.mu.Lock()
	defer d.mu.Unlock()
	for _, cancel := range d.containerList {
		cancel()
	}
	return nil
}

func (d *DockerLogs) matchedContainerName(names []string) string {
	// Check if all container names are filtered; in practice I believe
	// this array is always of length 1.
	for _, name := range names {
		trimmedName := strings.TrimPrefix(name, "/")
		match := d.containerFilter.Match(trimmedName)
		if match {
			return trimmedName
		}
	}
	return ""
}

func (d *DockerLogs) Gather(acc telegraf.Accumulator) error {
	ctx := context.Background()
	acc.SetPrecision(time.Nanosecond)

	ctx, cancel := context.WithTimeout(ctx, d.Timeout.Duration)
	defer cancel()
	containers, err := d.client.ContainerList(ctx, d.opts)
	if err != nil {
		return err
	}

	for _, container := range containers {
		if d.containerInContainerList(container.ID) {
			continue
		}

		containerName := d.matchedContainerName(container.Names)
		if containerName == "" {
			continue
		}

		ctx, cancel := context.WithCancel(context.Background())
		d.addToContainerList(container.ID, cancel)

		// Start a new goroutine for every new container that has logs to collect
		d.wg.Add(1)
		go func(container types.Container) {
			defer d.wg.Done()
			defer d.removeFromContainerList(container.ID)

			err = d.tailContainerLogs(ctx, acc, container, containerName)
			if err != nil && err != context.Canceled {
				acc.AddError(err)
			}
		}(container)
	}
	return nil
}

func (d *DockerLogs) hasTTY(ctx context.Context, container types.Container) (bool, error) {
	ctx, cancel := context.WithTimeout(ctx, d.Timeout.Duration)
	defer cancel()
	c, err := d.client.ContainerInspect(ctx, container.ID)
	if err != nil {
		return false, err
	}
	return c.Config.Tty, nil
}

func (d *DockerLogs) tailContainerLogs(
	ctx context.Context,
	acc telegraf.Accumulator,
	container types.Container,
	containerName string,
) error {
	imageName, imageVersion := docker.ParseImage(container.Image)
	tags := map[string]string{
		"container_name":    containerName,
		"container_image":   imageName,
		"container_version": imageVersion,
	}

	if d.IncludeSourceTag {
		tags["source"] = hostnameFromID(container.ID)
	}

	// Add matching container labels as tags
	for k, label := range container.Labels {
		if d.labelFilter.Match(k) {
			tags[k] = label
		}
	}

	hasTTY, err := d.hasTTY(ctx, container)
	if err != nil {
		return err
	}

	tail := "0"
	if d.FromBeginning {
		tail = "all"
	}

	logOptions := types.ContainerLogsOptions{
		ShowStdout: true,
		ShowStderr: true,
		Timestamps: false,
		Details:    false,
		Follow:     true,
		Tail:       tail,
	}

	logReader, err := d.client.ContainerLogs(ctx, container.ID, logOptions)
	if err != nil {
		return err
	}

	// If the container is using a TTY, there is only a single stream
	// (stdout), and data is copied directly from the container output stream,
	// no extra multiplexing or headers.
	//
	// If the container is *not* using a TTY, streams for stdout and stderr are
	// multiplexed.
	if hasTTY {
		return tailStream(acc, tags, container.ID, logReader, "tty")
	} else {
		return tailMultiplexed(acc, tags, container.ID, logReader)
	}
}

func tailStream(
	acc telegraf.Accumulator,
	baseTags map[string]string,
	containerID string,
	reader io.ReadCloser,
	stream string,
) error {
	defer reader.Close()

	tags := make(map[string]string, len(baseTags)+1)
	for k, v := range baseTags {
		tags[k] = v
	}
	tags["stream"] = stream

	r := bufio.NewReaderSize(reader, 64*1024)

	var err error
	var message string
	for {
		message, err = r.ReadString('\n')

		// Keep any leading space, but remove whitespace from end of line.
		// This preserves space in, for example, stacktraces, while removing
		// annoying end of line characters and is similar to how other logging
		// plugins such as syslog behave.
		message = strings.TrimRightFunc(message, unicode.IsSpace)

		if len(message) != 0 {
			acc.AddFields("docker_log", map[string]interface{}{
				"container_id": containerID,
				"message":      message,
			}, tags)
		}

		if err != nil {
			if err == io.EOF {
				return nil
			}
			return err
		}
	}
}

func tailMultiplexed(
	acc telegraf.Accumulator,
	tags map[string]string,
	containerID string,
	src io.ReadCloser,
) error {
	outReader, outWriter := io.Pipe()
	errReader, errWriter := io.Pipe()

	var wg sync.WaitGroup
	wg.Add(1)
	go func() {
		defer wg.Done()
		err := tailStream(acc, tags, containerID, outReader, "stdout")
		if err != nil {
			acc.AddError(err)
		}
	}()

	wg.Add(1)
	go func() {
		defer wg.Done()
		err := tailStream(acc, tags, containerID, errReader, "stderr")
		if err != nil {
			acc.AddError(err)
		}
	}()

	_, err := stdcopy.StdCopy(outWriter, errWriter, src)
	outWriter.Close()
	errWriter.Close()
	src.Close()
	wg.Wait()
	return err
}

// Start is a noop which is required for a *DockerLogs to implement
// the telegraf.ServiceInput interface
func (d *DockerLogs) Start(telegraf.Accumulator) error {
	return nil
}

func (d *DockerLogs) Stop() {
	d.cancelTails()
	d.wg.Wait()
}

// Following few functions have been inherited from telegraf docker input plugin
func (d *DockerLogs) createContainerFilters() error {
	filter, err := filter.NewIncludeExcludeFilter(d.ContainerInclude, d.ContainerExclude)
	if err != nil {
		return err
	}
	d.containerFilter = filter
	return nil
}

func (d *DockerLogs) createLabelFilters() error {
	filter, err := filter.NewIncludeExcludeFilter(d.LabelInclude, d.LabelExclude)
	if err != nil {
		return err
	}
	d.labelFilter = filter
	return nil
}

func (d *DockerLogs) createContainerStateFilters() error {
	if len(d.ContainerStateInclude) == 0 && len(d.ContainerStateExclude) == 0 {
		d.ContainerStateInclude = []string{"running"}
	}
	filter, err := filter.NewIncludeExcludeFilter(d.ContainerStateInclude, d.ContainerStateExclude)
	if err != nil {
		return err
	}
	d.stateFilter = filter
	return nil
}

func init() {
	inputs.Add("docker_log", func() telegraf.Input {
		return &DockerLogs{
			Timeout:       internal.Duration{Duration: time.Second * 5},
			Endpoint:      defaultEndpoint,
			newEnvClient:  NewEnvClient,
			newClient:     NewClient,
			containerList: make(map[string]context.CancelFunc),
		}
	})
}

func hostnameFromID(id string) string {
	if len(id) > 12 {
		return id[0:12]
	}
	return id
}
<|endoftext|>


<commit_before>// Copyright 2015-2017 Kuzzle
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// 		http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package types

import "encoding/json"

type KuzzleRequest struct {
	RequestId  string        `json:"requestId,omitempty"`
	Controller string        `json:"controller,omitempty"`
	Action     string        `json:"action,omitempty"`
	Index      string        `json:"index,omitempty"`
	Collection string        `json:"collection,omitempty"`
	Body       interface{}   `json:"body"`
	Id         string        `json:"_id,omitempty"`
	From       int           `json:"from"`
	Size       int           `json:"size"`
	Scroll     string        `json:"scroll,omitempty"`
	ScrollId   string        `json:"scrollId,omitempty"`
	Strategy   string        `json:"strategy,omitempty"`
	ExpiresIn  int           `json:"expiresIn"`
	Volatile   VolatileData  `json:"volatile"`
	Scope      string        `json:"scope"`
	State      string        `json:"state"`
	Users      string        `json:"users"`
	Start      int           `json:"start,omitempty"`
	Stop       int           `json:"stop,omitempty"`
	End        int           `json:"end,omitempty"`
	Bit        int           `json:"bit,omitempty"`
	Member     string        `json:"member,omitempty"`
	Member1    string        `json:"member1,omitempty"`
	Member2    string        `json:"member2,omitempty"`
	Members    []string      `json:"members,omitempty"`
	Lon        float64       `json:"lon,omitempty"`
	Lat        float64       `json:"lat,omitempty"`
	Distance   float64       `json:"distance,omitempty"`
	Unit       string        `json:"unit,omitempty"`
	Options    []interface{} `json:"options,omitempty"`
	Keys       []string      `json:"keys,omitempty"`
	Cursor     int           `json:"cursor,omitempty"`
	Offset     int           `json:"offset,omitempty"`
	Field      string        `json:"field,omitempty"`
	Fields     []string      `json:"fields,omitempty"`
	Subcommand string        `json:"subcommand,omitempty"`
	Pattern    string        `json:"pattern,omitempty"`
	Idx        int           `json:"idx, omitempty"`
	Min        string        `json:"min,omitempty"`
	Max        string        `json:"max,omitempty"`
	Limit      string        `json:"limit,omitempty"`
	Count      int           `json:"count,omitempty"`
	Match      string        `json:"match,omitempty"`
	Reset      bool          `json:"reset,omitempty"`
}

type SubscribeQuery struct {
	Scope string      `json:"scope"`
	State string      `json:"state"`
	User  string      `json:"user"`
	Body  interface{} `json:"body"`
}

type VolatileData = json.RawMessage

type UserData struct {
	ProfileIds []string               `json:"profileIds"`
	Content    map[string]interface{} `json:"content"`
}

type PolicyRestriction struct {
	Index       string   `json:"index"`
	Collections []string `json:"collections,omitempty"`
}

type Policy struct {
	RoleId       string               `json:"roleId"`
	RestrictedTo []*PolicyRestriction `json:"restrictedTo,omitempty"`
}

type Policies struct {
	Policies []*Policy `json:"policies"`
}

type GeoPoint struct {
	Lon  float64 `json:"lon"`
	Lat  float64 `json:"lat"`
	Name string  `json:"name"`
}

type MsHashField struct {
	Field string `json:"field"`
	Value string `json:"value"`
}

type MSKeyValue struct {
	Key   string `json:"key"`
	Value string `json:"value"`
}

type MSSortedSet struct {
	Score  float64 `json:"score"`
	Member string  `json:"member"`
}

type SearchFilters struct {
	Query        json.RawMessage `json:"query,omitempty"`
	Sort         json.RawMessage `json:"sort,omitempty"`
	Aggregations json.RawMessage `json:"aggregations,omitempty"`
	SearchAfter  json.RawMessage `json:"search_after,omitempty"`
}
<commit_msg>omit expiresIn if 0<commit_after>// Copyright 2015-2017 Kuzzle
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// 		http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package types

import "encoding/json"

type KuzzleRequest struct {
	RequestId  string        `json:"requestId,omitempty"`
	Controller string        `json:"controller,omitempty"`
	Action     string        `json:"action,omitempty"`
	Index      string        `json:"index,omitempty"`
	Collection string        `json:"collection,omitempty"`
	Body       interface{}   `json:"body"`
	Id         string        `json:"_id,omitempty"`
	From       int           `json:"from"`
	Size       int           `json:"size"`
	Scroll     string        `json:"scroll,omitempty"`
	ScrollId   string        `json:"scrollId,omitempty"`
	Strategy   string        `json:"strategy,omitempty"`
	ExpiresIn  int           `json:"expiresIn,omitempty"`
	Volatile   VolatileData  `json:"volatile"`
	Scope      string        `json:"scope"`
	State      string        `json:"state"`
	Users      string        `json:"users"`
	Start      int           `json:"start,omitempty"`
	Stop       int           `json:"stop,omitempty"`
	End        int           `json:"end,omitempty"`
	Bit        int           `json:"bit,omitempty"`
	Member     string        `json:"member,omitempty"`
	Member1    string        `json:"member1,omitempty"`
	Member2    string        `json:"member2,omitempty"`
	Members    []string      `json:"members,omitempty"`
	Lon        float64       `json:"lon,omitempty"`
	Lat        float64       `json:"lat,omitempty"`
	Distance   float64       `json:"distance,omitempty"`
	Unit       string        `json:"unit,omitempty"`
	Options    []interface{} `json:"options,omitempty"`
	Keys       []string      `json:"keys,omitempty"`
	Cursor     int           `json:"cursor,omitempty"`
	Offset     int           `json:"offset,omitempty"`
	Field      string        `json:"field,omitempty"`
	Fields     []string      `json:"fields,omitempty"`
	Subcommand string        `json:"subcommand,omitempty"`
	Pattern    string        `json:"pattern,omitempty"`
	Idx        int           `json:"idx, omitempty"`
	Min        string        `json:"min,omitempty"`
	Max        string        `json:"max,omitempty"`
	Limit      string        `json:"limit,omitempty"`
	Count      int           `json:"count,omitempty"`
	Match      string        `json:"match,omitempty"`
	Reset      bool          `json:"reset,omitempty"`
}

type SubscribeQuery struct {
	Scope string      `json:"scope"`
	State string      `json:"state"`
	User  string      `json:"user"`
	Body  interface{} `json:"body"`
}

type VolatileData = json.RawMessage

type UserData struct {
	ProfileIds []string               `json:"profileIds"`
	Content    map[string]interface{} `json:"content"`
}

type PolicyRestriction struct {
	Index       string   `json:"index"`
	Collections []string `json:"collections,omitempty"`
}

type Policy struct {
	RoleId       string               `json:"roleId"`
	RestrictedTo []*PolicyRestriction `json:"restrictedTo,omitempty"`
}

type Policies struct {
	Policies []*Policy `json:"policies"`
}

type GeoPoint struct {
	Lon  float64 `json:"lon"`
	Lat  float64 `json:"lat"`
	Name string  `json:"name"`
}

type MsHashField struct {
	Field string `json:"field"`
	Value string `json:"value"`
}

type MSKeyValue struct {
	Key   string `json:"key"`
	Value string `json:"value"`
}

type MSSortedSet struct {
	Score  float64 `json:"score"`
	Member string  `json:"member"`
}

type SearchFilters struct {
	Query        json.RawMessage `json:"query,omitempty"`
	Sort         json.RawMessage `json:"sort,omitempty"`
	Aggregations json.RawMessage `json:"aggregations,omitempty"`
	SearchAfter  json.RawMessage `json:"search_after,omitempty"`
}
<|endoftext|>


<commit_before><commit_msg>Fixed dumb mistake.<commit_after><|endoftext|>


<commit_before><commit_msg>fix test case<commit_after><|endoftext|>


<commit_before><commit_msg>Improve logging, properly trigger stop when shutting down<commit_after><|endoftext|>


<commit_before>package distributor

import (
	"testing"

	components "github.com/LoRaWanSoFa/LoRaWanSoFa/Components"
	"github.com/LoRaWanSoFa/LoRaWanSoFa/DBC/DatabaseConnector"
)

var dist = New()
var devEuiS = "00000000ABCDEF12"

func TestConvertMessage(t *testing.T) {
	DatabaseConnector.Connect()
	gpsSensor := components.NewSensor(3, 0, 0, 0, 2, 4, 1, 2, "", "0")
	inputMessage := components.NewMessageUplink(123, devEuiS)
	inputMessage.AddPayload([]byte{0x42, 0x22, 0xEC, 0x25}, gpsSensor)
	inputMessage.AddPayload([]byte{0xC2, 0x93, 0xDE, 0xD8}, gpsSensor)
	expectedMessage := components.NewMessageUplink(123, devEuiS)
	expectedMessage.AddPayloadString("40.730610", gpsSensor)
	expectedMessage.AddPayloadString("-73.935242", gpsSensor)
	mp, _ := dist.InputUplink(inputMessage)
	payloads := mp.GetPayloads()
	for i := range payloads {
		inputPayload := payloads[i]
		expectedPayload := expectedMessage.GetPayloads()[i]
		if !inputPayload.Equals(expectedPayload) {
			t.Errorf("The payload of the message should be %s, but was %s.",
				expectedPayload.GetPayload(), inputPayload.GetPayload())
		}
	}
	DatabaseConnector.Close()
}
<commit_msg>updated test for soft_deleted field<commit_after>package distributor

import (
	"testing"

	components "github.com/LoRaWanSoFa/LoRaWanSoFa/Components"
	"github.com/LoRaWanSoFa/LoRaWanSoFa/DBC/DatabaseConnector"
)

var dist = New()
var devEuiS = "00000000ABCDEF12"

func TestConvertMessage(t *testing.T) {
	DatabaseConnector.Connect()
	gpsSensor := components.NewSensor(3, 0, 0, 0, 2, 4, 1, 2, "", "0", false)
	inputMessage := components.NewMessageUplink(123, devEuiS)
	inputMessage.AddPayload([]byte{0x42, 0x22, 0xEC, 0x25}, gpsSensor)
	inputMessage.AddPayload([]byte{0xC2, 0x93, 0xDE, 0xD8}, gpsSensor)
	expectedMessage := components.NewMessageUplink(123, devEuiS)
	expectedMessage.AddPayloadString("40.730610", gpsSensor)
	expectedMessage.AddPayloadString("-73.935242", gpsSensor)
	mp, _ := dist.InputUplink(inputMessage)
	payloads := mp.GetPayloads()
	for i := range payloads {
		inputPayload := payloads[i]
		expectedPayload := expectedMessage.GetPayloads()[i]
		if !inputPayload.Equals(expectedPayload) {
			t.Errorf("The payload of the message should be %s, but was %s.",
				expectedPayload.GetPayload(), inputPayload.GetPayload())
		}
	}
	DatabaseConnector.Close()
}
<|endoftext|>


<commit_before>package main

import (
  "bytes"
  "compress/zlib"
  "encoding/binary"
  "encoding/json"
  "errors"
  "fmt"
  "io"
  "log"
  "math/rand"
  "os"
  "time"
)

const (
  default_publisher_hostname string = "localhost.localdomain"
  keepalive_timeout          time.Duration = 900 * time.Second
  max_pending_payloads       int = 100
)

type PendingPayload struct {
  next       *PendingPayload
  nonce      string
  events     []*FileEvent
  num_events int
  ack_events int
  payload_start int
  payload    []byte
}

type Publisher struct {
  config           *NetworkConfig
  transport        Transport
  hostname         string
  can_send         <-chan int
  pending_ping     bool
  pending_payloads map[string]*PendingPayload
  first_payload    *PendingPayload
  last_payload     *PendingPayload
  num_payloads     int
}

func (p *Publisher) Init() error {
  var err error

  p.hostname, err = os.Hostname()
  if err != nil {
    log.Printf("Failed to determine the FQDN; using localhost.localdomain.\n")
    p.hostname = default_publisher_hostname
  }

  // Set up the selected transport (currently only TLS)
  if p.transport, err = p.config.transport.NewTransport(p.config); err != nil {
    return err
  }

  p.pending_payloads = make(map[string]*PendingPayload)

  return nil
}

func (p *Publisher) Publish(input <-chan []*FileEvent, registrar_chan chan<- []RegistrarEvent) {
  var input_toggle <-chan []*FileEvent
  var retry_payload *PendingPayload
  var err error

  // TODO(driskell): Make the idle timeout configurable like the network timeout is?
  timer := time.NewTimer(keepalive_timeout)

  // TODO: We should still obey network timeout if we've sent events and not yet received response
  //       as its the quickest way to detect a connection problem after idle

  for {
    if err = p.transport.Connect(); err != nil {
      log.Printf("Connect attempt failed: %s\n", err)
      // TODO: implement shutdown select
      time.Sleep(p.config.Reconnect)
      continue
    }
    p.can_send = p.transport.CanSend()
    input_toggle = nil

  SelectLoop:
    for {
      // TODO: implement shutdown select
      select {
      case <-p.can_send:
        if retry_payload != nil {
          var buffer bytes.Buffer

          // Do we need to regenerate the payload? Remember to account for ACK we have but not yet sent to registrar due to out-of-order receive
          if retry_payload.payload == nil {
            if err = p.bufferJdatData(&buffer, retry_payload.events[retry_payload.ack_events:], retry_payload.nonce); err != nil {
              break SelectLoop
            }

            retry_payload.payload = buffer.Bytes()
            retry_payload.payload_start = retry_payload.ack_events
          }

          // Send the payload again
          if err = p.transport.Write("JDAT", retry_payload.payload); err != nil {
            break SelectLoop
          }

          retry_payload = retry_payload.next
          break
        }

        // No pending payloads, enable event wait
        input_toggle = input

        // Continue loop so we don't reset the ping timer - we've not performed any activity just yet
        continue
      case events := <-input_toggle:
        // Send JDAT
        if err = p.sendJdat(events); err != nil {
          break SelectLoop
        }

        // Wait for send signal again
        input_toggle = nil

        if p.num_payloads >= max_pending_payloads {
          // Too many pending payloads, disable send temporarily
          p.can_send = nil
        }
      case data := <-p.transport.Read():
        var signature, message []byte

        // Error? Or data?
        switch data.(type) {
          case error:
            err = data.(error)
            break SelectLoop
          default:
            signature = data.([][]byte)[0]
            message = data.([][]byte)[1]
        }

        switch {
        case bytes.Compare(signature, []byte("PONG")) == 0:
          if err = p.processPong(message); err != nil {
            break SelectLoop
          }
        case bytes.Compare(signature, []byte("ACKN")) == 0:
          if err = p.processAck(message, registrar_chan); err != nil {
            break SelectLoop
          }
        default:
          err = errors.New(fmt.Sprintf("Unknown message received: % X", signature))
          break SelectLoop
        }
      case <-timer.C:
        log.Printf("<-timer.C")
        // If we haven't received a PONG yet this is a timeout
        if p.pending_ping {
          err = errors.New("Server did not respond to PING")
          break SelectLoop
        }

        // If the send buffer is full, we should have been receiving ACK by now...
        if input_toggle == nil {
          err = errors.New("Server stopped responding")
          break SelectLoop
        }

        // Send a ping and expect a pong back (eventually)
        // If we receive an ACK first, that's fine we'll reset timer
        // But after those ACKs we should get a PONG
        if err = p.transport.Write("PING", nil); err != nil {
          break SelectLoop
        }

        // We may have just filled the send buffer
        input_toggle = nil
      } /* select */

      // Reset the timer
      timer.Reset(keepalive_timeout)
    } /* loop forever, break to reconnect */

    // Disconnect and retry payloads
    log.Printf("Transport error, will reconnect: %s\n", err)
    p.transport.Disconnect()
    retry_payload = p.first_payload

    time.Sleep(p.config.Reconnect)
  } /* Publish: for loop, break to shutdown */
} // Publish

func (p *Publisher) generateNonce() string {
  // This could maybe be made a bit more efficient
  nonce := make([]byte, 16)
  for i := 0; i < 16; i++ {
    nonce[i] = byte(rand.Intn(255))
  }
  return string(nonce)
}

func (p *Publisher) sendJdat(events []*FileEvent) (err error) {
  var buffer bytes.Buffer

  // Calculate a nonce
  nonce := p.generateNonce()
  for {
    if _, found := p.pending_payloads[nonce]; !found {
      break
    }
    // Collision - generate again - should be extremely rare
    nonce = p.generateNonce()
  }

  // Generate the data first
  if err = p.bufferJdatData(&buffer, events, nonce); err != nil {
    return
  }

  // Save pending payload until we receive ack, and discard buffer
  payload := &PendingPayload{events: events, nonce: nonce, num_events: len(events), payload_start: 0, payload: buffer.Bytes()}
  p.pending_payloads[nonce] = payload
  if p.first_payload == nil {
    p.first_payload = payload
  } else {
    p.last_payload.next = payload
  }
  p.last_payload = payload
  p.num_payloads++

  return p.transport.Write("JDAT", payload.payload)
}

func (p *Publisher) bufferJdatData(output io.Writer, events []*FileEvent, nonce string) (err error) {
  // Begin with the nonce
  if _, err = output.Write([]byte(nonce)); err != nil {
    return
  }

  var compressor *zlib.Writer
  if compressor, err = zlib.NewWriterLevel(output, 3); err != nil {
    return
  }

  // Append all the events
  for _, event := range events {
    // Add host field
    event.Event["host"] = p.hostname
    if err = p.bufferJdatDataEvent(compressor, event); err != nil {
      return
    }
  }

  compressor.Close()

  return nil
}

func (p *Publisher) bufferJdatDataEvent(output io.Writer, event *FileEvent) (err error) {
  var value []byte
  value, err = json.Marshal(event.Event)
  if err != nil {
    log.Printf("JSON event encoding error: %s\n", err)

    if err = binary.Write(output, binary.BigEndian, 2); err != nil {
      return
    }
    if _, err = output.Write([]byte("{}")); err != nil {
      return
    }

    return
  }

  if err = binary.Write(output, binary.BigEndian, uint32(len(value))); err != nil {
    return
  }
  if _, err = output.Write(value); err != nil {
    return
  }

  return nil
}

func (p *Publisher) processPong(message []byte) error {
  if len(message) != 0 {
    return errors.New(fmt.Sprintf("PONG message overflow (%d)", len(message)))
  }

  // Were we pending a ping?
  if !p.pending_ping {
    return errors.New("Unexpected PONG received")
  }

  p.pending_ping = false
  return nil
}

func (p *Publisher) processAck(message []byte, registrar_chan chan<- []RegistrarEvent) (err error) {
  if len(message) != 20 {
    err = errors.New(fmt.Sprintf("ACKN message corruption (%d)", len(message)))
    return
  }

  // Read the nonce and sequence number acked
  nonce, sequence := string(message[:16]), binary.BigEndian.Uint32(message[16:20])

  // Grab the payload the ACK corresponds to by using nonce
  payload, found := p.pending_payloads[nonce]
  if !found {
    // Don't fail here in case we had temporary issues and resend a payload, only for us to receive duplicate ACKN
    return
  }

  // Full ACK?
  // TODO: Protocol error if sequence is too large?
  if int(sequence) >= payload.num_events - payload.payload_start {
    // No more events left for this payload, free the payload memory
    payload.ack_events = len(payload.events)
    payload.payload = nil
    delete(p.pending_payloads, nonce)
  } else {
    // Only process the ACK if something was actually processed
    if int(sequence) > payload.num_events - payload.ack_events {
      payload.ack_events = int(sequence) + payload.payload_start
      // If we need to resend, we'll need to regenerate payload, so free that memory early
      payload.payload = nil
    }
  }

  // We potentially receive out-of-order ACKs due to payloads distributed across servers
  // This is where we enforce ordering again to ensure registrar receives ACK in order
  if payload == p.first_payload {
    for payload.ack_events != 0 {
      if payload.ack_events == len(payload.events) {
        registrar_chan <- []RegistrarEvent{&EventsEvent{Events: payload.events}}
        payload = payload.next
        p.first_payload = payload
        p.num_payloads--

        // Resume sending if we stopped due to excessive pending payload count
        if p.can_send == nil {
          p.can_send = p.transport.CanSend()
        }
      } else {
        registrar_chan <- []RegistrarEvent{&EventsEvent{Events: payload.events[:payload.ack_events]}}
        payload.events = payload.events[payload.ack_events:]
        payload.num_events = len(payload.events)
        payload.ack_events = 0
        payload.payload_start = 0
      }

      if payload == nil {
        break
      }
    }
  }

  return
}
<commit_msg>Spring cleaning and finish payload retry mechanisms for zmq recovery<commit_after>package main

import (
  "bytes"
  "compress/zlib"
  "encoding/binary"
  "encoding/json"
  "errors"
  "fmt"
  "io"
  "log"
  "math/rand"
  "os"
  "time"
)

const (
  default_publisher_hostname string = "localhost.localdomain"
  keepalive_timeout          time.Duration = 900 * time.Second
  max_pending_payloads       int = 100
)

type PendingPayload struct {
  next       *PendingPayload
  nonce      string
  events     []*FileEvent
  num_events int
  ack_events int
  payload_start int
  payload    []byte
  timeout    *time.Time
}

type Publisher struct {
  config           *NetworkConfig
  transport        Transport
  hostname         string
  can_send         <-chan int
  pending_ping     bool
  pending_payloads map[string]*PendingPayload
  first_payload    *PendingPayload
  last_payload     *PendingPayload
  num_payloads     int
  out_of_sync      int
}

func NewPendingPayload(events []*FileEvent, nonce string, hostname string) (*PendingPayload, error) {
  payload := &PendingPayload{
    events: events,
    nonce: nonce,
    num_events: len(events),
  }

  if err := payload.Generate(hostname); err != nil {
    return nil, err
  }

  return payload, nil
}

func (pp *PendingPayload) Generate(hostname string) (err error) {
  var buffer bytes.Buffer

  // Begin with the nonce
  if _, err = buffer.Write([]byte(pp.nonce)); err != nil {
    return
  }

  var compressor *zlib.Writer
  if compressor, err = zlib.NewWriterLevel(&buffer, 3); err != nil {
    return
  }

  // Append all the events
  for _, event := range pp.events[pp.ack_events:] {
    // Add host field
    event.Event["host"] = hostname
    if err = pp.bufferJdatDataEvent(compressor, event); err != nil {
      return
    }
  }

  compressor.Close()

  pp.payload = buffer.Bytes()
  pp.payload_start = pp.ack_events

  return
}

func (pp *PendingPayload) bufferJdatDataEvent(output io.Writer, event *FileEvent) (err error) {
  var value []byte
  value, err = json.Marshal(event.Event)
  if err != nil {
    log.Printf("JSON event encoding error: %s\n", err)

    if err = binary.Write(output, binary.BigEndian, 2); err != nil {
      return
    }
    if _, err = output.Write([]byte("{}")); err != nil {
      return
    }

    return
  }

  if err = binary.Write(output, binary.BigEndian, uint32(len(value))); err != nil {
    return
  }
  if _, err = output.Write(value); err != nil {
    return
  }

  return nil
}

func (p *Publisher) Init() error {
  var err error

  p.hostname, err = os.Hostname()
  if err != nil {
    log.Printf("Failed to determine the FQDN; using localhost.localdomain.\n")
    p.hostname = default_publisher_hostname
  }

  // Set up the selected transport (currently only TLS)
  if p.transport, err = p.config.transport.NewTransport(p.config); err != nil {
    return err
  }

  p.pending_payloads = make(map[string]*PendingPayload)

  return nil
}

func (p *Publisher) Publish(input <-chan []*FileEvent, registrar_chan chan<- []RegistrarEvent) {
  var input_toggle <-chan []*FileEvent
  var retry_payload *PendingPayload
  var err error

  // TODO(driskell): Make the idle timeout configurable like the network timeout is?
  timer := time.NewTimer(keepalive_timeout)

  // TODO: We should still obey network timeout if we've sent events and not yet received response
  //       as its the quickest way to detect a connection problem after idle

  for {
    if err = p.transport.Connect(); err != nil {
      log.Printf("Connect attempt failed: %s\n", err)
      // TODO: implement shutdown select
      time.Sleep(p.config.Reconnect)
      continue
    }
    p.can_send = p.transport.CanSend()
    input_toggle = nil

  SelectLoop:
    for {
      // TODO: implement shutdown select
      select {
      case <-p.can_send:
        // Resend payloads from full retry first
        if retry_payload != nil {
          // Do we need to regenerate the payload?
          if retry_payload.payload == nil {
            if err = retry_payload.Generate(p.hostname); err != nil {
              break SelectLoop
            }
          }

          // Reset timeout
          retry_payload.timeout = nil

          // Send the payload again
          if err = p.transport.Write("JDAT", retry_payload.payload); err != nil {
            break SelectLoop
          }

          retry_payload = retry_payload.next

          // Expect an ACK within network timeout
          if p.first_payload.timeout != nil {
            timer.Reset(p.first_payload.timeout.Sub(time.Now()))
          } else {
            timer.Reset(p.config.Timeout)
          }
          break
        } else if p.out_of_sync != 0 {
          var resent bool
          if resent, err = p.checkResend(); err != nil {
            break SelectLoop
          } else if resent {
            // Expect an ACK within network timeout
            timer.Reset(p.config.Timeout)
            break
          }
        }

        // No pending payloads, enable event wait
        input_toggle = input
      case events := <-input_toggle:
        // Send
        if err = p.sendNewPayload(events); err != nil {
          break SelectLoop
        }

        // Wait for send signal again
        input_toggle = nil

        if p.num_payloads >= max_pending_payloads {
          // Too many pending payloads, disable send temporarily
          p.can_send = nil
        }

        // Expect an ACK within network timeout
        if p.first_payload.timeout != nil {
          timer.Reset(p.first_payload.timeout.Sub(time.Now()))
        } else {
          timer.Reset(p.config.Timeout)
        }
      case data := <-p.transport.Read():
        var signature, message []byte

        // Error? Or data?
        switch data.(type) {
          case error:
            err = data.(error)
            break SelectLoop
          default:
            signature = data.([][]byte)[0]
            message = data.([][]byte)[1]
        }

        switch {
        case bytes.Compare(signature, []byte("PONG")) == 0:
          if err = p.processPong(message); err != nil {
            break SelectLoop
          }
        case bytes.Compare(signature, []byte("ACKN")) == 0:
          if err = p.processAck(message, registrar_chan); err != nil {
            break SelectLoop
          }
        default:
          err = errors.New(fmt.Sprintf("Unknown message received: % X", signature))
          break SelectLoop
        }

        // If no more pending payloads, set keepalive, otherwise reset to network timeout
        if p.num_payloads == 0 {
          timer.Reset(keepalive_timeout)
        } else if p.first_payload.timeout != nil {
          timer.Reset(p.first_payload.timeout.Sub(time.Now()))
        } else {
          timer.Reset(p.config.Timeout)
        }
      case <-timer.C:
        // Do we need to resend first payload?
        if p.out_of_sync != 0 {
          var resent bool
          if resent, err = p.checkResend(); err != nil {
            break SelectLoop
          } else if resent {
            // Expect an ACK within network timeout
            timer.Reset(p.config.Timeout)
            break
          }
        }

        // If we have pending payloads, we should've received something by now
        if p.num_payloads != 0 || input_toggle == nil {
          err = errors.New("Server did not respond within network timeout")
          break SelectLoop
        }

        // If we haven't received a PONG yet this is a timeout
        if p.pending_ping {
          err = errors.New("Server did not respond to PING")
          break SelectLoop
        }

        // Send a ping and expect a pong back (eventually)
        // If we receive an ACK first, that's fine we'll reset timer
        // But after those ACKs we should get a PONG
        if err = p.transport.Write("PING", nil); err != nil {
          break SelectLoop
        }

        // We may have just filled the send buffer
        input_toggle = nil

        // Allow network timeout to receive something
        timer.Reset(p.config.Timeout)
      }
    }

    // Disconnect and retry payloads
    log.Printf("Transport error, will reconnect: %s\n", err)
    p.transport.Disconnect()

    retry_payload = p.first_payload
    p.out_of_sync = 0

    time.Sleep(p.config.Reconnect)
  }
}

func (p *Publisher) checkResend() (bool, error) {
  // We're out of sync (received ACKs for later payloads but not earlier ones)
  // Check timeouts of earlier payloads and resend if necessary
  if payload := p.first_payload; payload.timeout.Before(time.Now()) {
    // Do we need to regenerate the payload?
    if payload.payload == nil {
      if err := payload.Generate(p.hostname); err != nil {
        return false, err
      }
    }

    // Update timeout
    timeout := time.Now().Add(p.config.Timeout)
    payload.timeout = &timeout

    // Send the payload again
    if err := p.transport.Write("JDAT", payload.payload); err != nil {
      return false, err
    }

    return true, nil
  }

  return false, nil
}

func (p *Publisher) generateNonce() string {
  // This could maybe be made a bit more efficient
  nonce := make([]byte, 16)
  for i := 0; i < 16; i++ {
    nonce[i] = byte(rand.Intn(255))
  }
  return string(nonce)
}

func (p *Publisher) sendNewPayload(events []*FileEvent) (err error) {
  // Calculate a nonce
  nonce := p.generateNonce()
  for {
    if _, found := p.pending_payloads[nonce]; !found {
      break
    }
    // Collision - generate again - should be extremely rare
    nonce = p.generateNonce()
  }

  var payload *PendingPayload
  if payload, err = NewPendingPayload(events, nonce, p.hostname); err != nil {
    return
  }

  // Save pending payload until we receive ack, and discard buffer
  p.pending_payloads[nonce] = payload
  if p.first_payload == nil {
    p.first_payload = payload
  } else {
    p.last_payload.next = payload
  }
  p.last_payload = payload
  p.num_payloads++

  return p.transport.Write("JDAT", payload.payload)
}

func (p *Publisher) processPong(message []byte) error {
  if len(message) != 0 {
    return errors.New(fmt.Sprintf("PONG message overflow (%d)", len(message)))
  }

  // Were we pending a ping?
  if !p.pending_ping {
    return errors.New("Unexpected PONG received")
  }

  p.pending_ping = false
  return nil
}

func (p *Publisher) processAck(message []byte, registrar_chan chan<- []RegistrarEvent) (err error) {
  if len(message) != 20 {
    err = errors.New(fmt.Sprintf("ACKN message corruption (%d)", len(message)))
    return
  }

  // Read the nonce and sequence number acked
  nonce, sequence := string(message[:16]), binary.BigEndian.Uint32(message[16:20])

  // Grab the payload the ACK corresponds to by using nonce
  payload, found := p.pending_payloads[nonce]
  if !found {
    // Don't fail here in case we had temporary issues and resend a payload, only for us to receive duplicate ACKN
    return
  }

  ack_events := payload.ack_events

  // Full ACK?
  // TODO: Protocol error if sequence is too large?
  if int(sequence) >= payload.num_events - payload.payload_start {
    // No more events left for this payload, free the payload memory
    payload.ack_events = len(payload.events)
    payload.payload = nil
    delete(p.pending_payloads, nonce)
  } else {
    // Only process the ACK if something was actually processed
    if int(sequence) > payload.num_events - payload.ack_events {
      payload.ack_events = int(sequence) + payload.payload_start
      // If we need to resend, we'll need to regenerate payload, so free that memory early
      payload.payload = nil
    }
  }

  // We potentially receive out-of-order ACKs due to payloads distributed across servers
  // This is where we enforce ordering again to ensure registrar receives ACK in order
  if payload == p.first_payload {
    out_of_sync := p.out_of_sync + 1
    for payload.ack_events != 0 {
      if payload.ack_events != len(payload.events) {
        registrar_chan <- []RegistrarEvent{&EventsEvent{Events: payload.events[:payload.ack_events]}}
        payload.events = payload.events[payload.ack_events:]
        payload.num_events = len(payload.events)
        payload.ack_events = 0
        payload.payload_start = 0
        break
      }

      registrar_chan <- []RegistrarEvent{&EventsEvent{Events: payload.events}}
      payload = payload.next
      p.first_payload = payload
      p.num_payloads--
      out_of_sync--
      p.out_of_sync = out_of_sync

      // Resume sending if we stopped due to excessive pending payload count
      if p.can_send == nil {
        p.can_send = p.transport.CanSend()
      }

      if payload == nil {
        break
      }
    }
  } else if ack_events == 0 {
    // Mark out of sync so we resend earlier packets in case they were lost
    p.out_of_sync++
  }

  // Set a timeout of the first payload if out of sync as we should be expecting it any time
  if p.out_of_sync != 0 && p.first_payload.timeout == nil {
    timeout := time.Now().Add(p.config.Timeout)
    p.first_payload.timeout = &timeout
  }

  return
}
<|endoftext|>


<commit_before><commit_msg>update code style and godoc<commit_after><|endoftext|>


<commit_before>package putio

import "context"

// EventsService is the service to gather information about user's events.
type EventsService struct {
	client *Client
}

// FIXME: events list returns inconsistent data structures.

// List gets list of dashboard events. It includes downloads and share events.
func (e *EventsService) List(ctx context.Context) ([]Event, error) {
	req, err := e.client.NewRequest(ctx, "GET", "/v2/events/list", nil)
	if err != nil {
		return nil, err
	}

	var r struct {
		Events []Event
	}
	_, err = e.client.Do(req, &r)
	if err != nil {
		return nil, err
	}
	return r.Events, nil

}

// Delete Clears all all dashboard events.
func (e *EventsService) Delete(ctx context.Context) error {
	req, err := e.client.NewRequest(ctx, "POST", "/v2/events/delete", nil)
	if err != nil {
		return err
	}
	req.Header.Set("Content-Type", "application/x-www-form-urlencoded")

	_, err = e.client.Do(req, &struct{}{})
	if err != nil {
		return err
	}

	return nil
}
<commit_msg>events: fix method documentation.<commit_after>package putio

import "context"

// EventsService is the service to gather information about user's events.
type EventsService struct {
	client *Client
}

// FIXME: events list returns inconsistent data structures.

// List gets list of dashboard events. It includes downloads and share events.
func (e *EventsService) List(ctx context.Context) ([]Event, error) {
	req, err := e.client.NewRequest(ctx, "GET", "/v2/events/list", nil)
	if err != nil {
		return nil, err
	}

	var r struct {
		Events []Event
	}
	_, err = e.client.Do(req, &r)
	if err != nil {
		return nil, err
	}
	return r.Events, nil

}

// Delete clears all dashboard events.
func (e *EventsService) Delete(ctx context.Context) error {
	req, err := e.client.NewRequest(ctx, "POST", "/v2/events/delete", nil)
	if err != nil {
		return err
	}
	req.Header.Set("Content-Type", "application/x-www-form-urlencoded")

	_, err = e.client.Do(req, &struct{}{})
	if err != nil {
		return err
	}

	return nil
}
<|endoftext|>


<commit_before>package router

import (
	"fmt"

	"github.com/ellcrys/util"
	"github.com/hashicorp/consul/api"
	logging "github.com/op/go-logging"
)

// RouterDomain is the router's domain
var RouterDomain = util.Env("ROUTER_DOMAIN", "whatbay.co")

// Helper defines a structure for hooking up to the
// reverse proxy tool within the cluster. Current implementation
// is designed to add frontend and backend entries in consul which is a
// traefik backend
type Helper struct {
	client         *api.Client
	l              *logging.Logger
	httpServerAddr string
}

// NewHelper creates a new router helper object. Returns error
// if unable to connector to consul
func NewHelper(l *logging.Logger, httpServerAddr string) (*Helper, error) {
	cfg := api.DefaultConfig()
	cfg.Address = util.Env("CONSUL_ADDR", cfg.Address)
	client, err := api.NewClient(cfg)
	if err != nil {
		return nil, fmt.Errorf("failed to create client: %s", err)
	}
	_, err = client.Status().Leader()
	if err != nil {
		return nil, fmt.Errorf("failed to create client: %s", err)
	}
	return &Helper{
		client: client,
		l:      l,
	}, nil
}

// AddFrontend adds a frontend to receive traffic from public internet
func (h *Helper) AddFrontend(name string) error {
	var frontend = fmt.Sprintf("traefik/frontends/%s", name)
	var backendName = fmt.Sprintf("%s", name)
	var keys = map[string]string{
		frontend + "/backend":          backendName,
		frontend + "/entrypoints/0":    "http",
		frontend + "/routes/main/rule": fmt.Sprintf("Host:%s.%s", name, RouterDomain),
	}

	kv := h.client.KV()
	var ops api.KVTxnOps
	for key, value := range keys {
		ops = append(ops, &api.KVTxnOp{
			Verb:  api.KVSet,
			Key:   key,
			Value: []byte(value),
		})
	}

	ok, _, _, err := kv.Txn(ops, nil)
	if err != nil {
		return fmt.Errorf("failed to add frontend: %s", err)
	}
	if ok {
		return nil
	}
	return fmt.Errorf("failed to add frontend")
}

// AddBackend adds the connector's http server as a backend server.
func (h *Helper) AddBackend(name string) error {
	var backend = fmt.Sprintf("traefik/backends/%s", name)
	var backendServer = fmt.Sprintf("%s/servers/%s_server", backend, name)
	var keys = map[string]string{
		backend + "/loadbalancer/method": "drr",
		backendServer + "/url":           h.httpServerAddr,
		backendServer + "/weight":        "10",
	}

	kv := h.client.KV()
	var ops api.KVTxnOps
	for key, value := range keys {
		ops = append(ops, &api.KVTxnOp{
			Verb:  api.KVSet,
			Key:   key,
			Value: []byte(value),
		})
	}

	ok, _, _, err := kv.Txn(ops, nil)
	if err != nil {
		return fmt.Errorf("failed to add backend: %s", err)
	}

	if ok {
		return nil
	}
	return fmt.Errorf("failed to add backend")
}
<commit_msg>Set http server address in new instance<commit_after>package router

import (
	"fmt"

	"github.com/ellcrys/util"
	"github.com/hashicorp/consul/api"
	logging "github.com/op/go-logging"
)

// RouterDomain is the router's domain
var RouterDomain = util.Env("ROUTER_DOMAIN", "whatbay.co")

// Helper defines a structure for hooking up to the
// reverse proxy tool within the cluster. Current implementation
// is designed to add frontend and backend entries in consul which is a
// traefik backend
type Helper struct {
	client         *api.Client
	l              *logging.Logger
	httpServerAddr string
}

// NewHelper creates a new router helper object. Returns error
// if unable to connector to consul
func NewHelper(l *logging.Logger, httpServerAddr string) (*Helper, error) {
	cfg := api.DefaultConfig()
	cfg.Address = util.Env("CONSUL_ADDR", cfg.Address)
	client, err := api.NewClient(cfg)
	if err != nil {
		return nil, fmt.Errorf("failed to create client: %s", err)
	}
	_, err = client.Status().Leader()
	if err != nil {
		return nil, fmt.Errorf("failed to create client: %s", err)
	}
	return &Helper{
		client:         client,
		l:              l,
		httpServerAddr: "http://" + httpServerAddr,
	}, nil
}

// AddFrontend adds a frontend to receive traffic from public internet
func (h *Helper) AddFrontend(name string) error {
	var frontend = fmt.Sprintf("traefik/frontends/%s", name)
	var backendName = fmt.Sprintf("%s", name)
	var keys = map[string]string{
		frontend + "/backend":          backendName,
		frontend + "/entrypoints/0":    "http",
		frontend + "/routes/main/rule": fmt.Sprintf("Host:%s.%s", name, RouterDomain),
	}

	kv := h.client.KV()
	var ops api.KVTxnOps
	for key, value := range keys {
		ops = append(ops, &api.KVTxnOp{
			Verb:  api.KVSet,
			Key:   key,
			Value: []byte(value),
		})
	}

	ok, _, _, err := kv.Txn(ops, nil)
	if err != nil {
		return fmt.Errorf("failed to add frontend: %s", err)
	}
	if ok {
		return nil
	}
	return fmt.Errorf("failed to add frontend")
}

// AddBackend adds the connector's http server as a backend server.
func (h *Helper) AddBackend(name string) error {
	var backend = fmt.Sprintf("traefik/backends/%s", name)
	var backendServer = fmt.Sprintf("%s/servers/%s_server", backend, name)
	var keys = map[string]string{
		backend + "/loadbalancer/method": "drr",
		backendServer + "/url":           h.httpServerAddr,
		backendServer + "/weight":        "10",
	}

	kv := h.client.KV()
	var ops api.KVTxnOps
	for key, value := range keys {
		ops = append(ops, &api.KVTxnOp{
			Verb:  api.KVSet,
			Key:   key,
			Value: []byte(value),
		})
	}

	ok, _, _, err := kv.Txn(ops, nil)
	if err != nil {
		return fmt.Errorf("failed to add backend: %s", err)
	}

	if ok {
		return nil
	}
	return fmt.Errorf("failed to add backend")
}
<|endoftext|>


<commit_before>package crawl

import (
	"errors"
	"fmt"
	"log"
	"os"
	"path/filepath"
	"time"

	"github.com/nzai/stockrecorder/config"
	"github.com/nzai/stockrecorder/io"
)

const (
	gccount           = 16
	retryCount        = 5
	retryDelay        = 10 * time.Minute
	configKeyTimeZone = "timezone"
	//	雅虎财经的历史分时数据没有超过60天的
	lastestDays = 60
)

//	抓取雅虎今日数据
func marketAll(market string) error {
	//	股票编码
	codes := config.GetArray(market, "codes")
	if len(codes) == 0 {
		return errors.New(fmt.Sprintf("市场[%s]的codes配置有误", market))
	}

	chanSend := make(chan int, gccount)
	chanReceive := make(chan int)

	for _, code := range codes {
		//	并发抓取
		go func(c string) {
			err := stockAll(market, c)
			if err != nil {
				log.Fatal(err)
			}

			<-chanSend
			chanReceive <- 1
		}(code)

		chanSend <- 1
	}

	//	阻塞，直到抓取所有
	for _, _ = range codes {
		<-chanReceive
	}

	close(chanSend)
	close(chanReceive)

	return nil
}

//	抓取股票所有数据
func stockAll(market, code string) error {

	location := time.Local
	timezone := config.GetString(market, configKeyTimeZone, "")
	if timezone != "" {
		loc, err := time.LoadLocation(timezone)
		if err != nil {
			return err
		}
		location = loc
	}
	now := time.Now().In(location)
	today := time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, location)

	//	定时器今天
	go func() {
		ticker := time.NewTicker(time.Hour * 24)
		log.Printf("已启动%s的定时抓取任务", code)
		for _ = range ticker.C {
			now = time.Now().In(location)
			today = time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, location)

			err := stockToday(code, today)
			if err != nil {
				log.Fatalf("抓取%s在%s的数据出错:%v", code, today.Format("20060102"), err)
			}
		}
	}()

	//	历史
	return stockHistory(code, today)
}

//	抓取今天
func stockToday(code string, today time.Time) error {
	log.Printf("%s在%s分时数据抓取任务-开始", code, today.Format("20060102"))
	//	保存原始数据
	day := today.Add(-time.Hour * 24)
	for try := 0; try < retryCount; try++ {
		//	抓取数据
		err := thatDay(code, day)
		if err != nil {
			log.Fatalf("[%d]抓取%s在%s的数据出错:%v", try, code, today.Format("20060102"), err)
			time.Sleep(retryDelay)
		}

		log.Printf("%s在%s分时数据抓取任务-结束", code, day.Format("20060102"))
		return nil
	}

	return errors.New(fmt.Sprintf("%s在%s分时数据抓取任务失败", code, today.Format("20060102")))
}

//	抓取历史
func stockHistory(code string, today time.Time) error {
	log.Printf("%s历史分时数据抓取任务-开始", code)
	//	保存原始数据
	day := today.Add(-time.Hour * 24)
	for index := 0; index < lastestDays; index++ {

		for try := 0; try < retryCount; try++ {
			//	抓取数据
			err := thatDay(code, day)
			if err == nil {
				//	往前一天递推
				day = day.Add(-time.Hour * 24)
				break
			} else {
				log.Fatalf("[%d]抓取%s在%s的数据出错:%v", try, code, day.Format("20060102"), err)
				time.Sleep(retryDelay)
			}
		}
	}
	log.Printf("%s历史分时数据抓取任务-结束", code)
	return nil
}

//	抓取某一天
func thatDay(code string, day time.Time) error {

	//	文件保存路径
	dataDir := config.GetDataDir()
	fileName := fmt.Sprintf("%s_raw.txt", day.Format("20060102"))
	filePath := filepath.Join(dataDir, code, fileName)

	//	如果文件已存在就忽略
	_, err := os.Stat(filePath)
	if os.IsNotExist(err) {
		//	如果不存在就抓取并保存
		start := time.Date(day.Year(), day.Month(), day.Day(), 0, 0, 0, 0, day.Location())
		end := start.Add(time.Hour * 24)
		html, err := peroid(code, start, end)
		if err != nil {
			return err
		}

		//	写入文件
		return io.WriteString(filePath, html)
	}

	return nil
}

//	抓取一段时间
func peroid(code string, start, end time.Time) (string, error) {

	pattern := "https://finance-yql.media.yahoo.com/v7/finance/chart/%s?period2=%d&period1=%d&interval=1m&indicators=quote&includeTimestamps=true&includePrePost=true&events=div%7Csplit%7Cearn&corsDomain=finance.yahoo.com"
	url := fmt.Sprintf(pattern, code, end.Unix(), start.Unix())

	//	抓取数据
	return io.GetString(url)
}
<commit_msg>完善了定时任务的逻辑，调整了日志提示<commit_after>package crawl

import (
	"errors"
	"fmt"
	"log"
	"os"
	"path/filepath"
	"time"

	"github.com/nzai/stockrecorder/config"
	"github.com/nzai/stockrecorder/io"
)

const (
	gccount           = 16
	retryCount        = 5
	retryDelay        = 10 * time.Minute
	configKeyTimeZone = "timezone"
	//	雅虎财经的历史分时数据没有超过60天的
	lastestDays = 60
)

//	抓取雅虎今日数据
func marketAll(market string) error {
	//	股票编码
	codes := config.GetArray(market, "codes")
	if len(codes) == 0 {
		return errors.New(fmt.Sprintf("市场[%s]的codes配置有误", market))
	}

	chanSend := make(chan int, gccount)
	chanReceive := make(chan int)

	for _, code := range codes {
		//	并发抓取
		go func(c string) {
			err := stockAll(market, c)
			if err != nil {
				log.Fatal(err)
			}

			<-chanSend
			chanReceive <- 1
		}(code)

		chanSend <- 1
	}

	//	阻塞，直到抓取所有
	for _, _ = range codes {
		<-chanReceive
	}

	close(chanSend)
	close(chanReceive)

	return nil
}

//	抓取股票所有数据
func stockAll(market, code string) error {

	location := time.Local
	timezone := config.GetString(market, configKeyTimeZone, "")
	if timezone != "" {
		loc, err := time.LoadLocation(timezone)
		if err != nil {
			return err
		}
		location = loc
	}
	now := time.Now().In(location)
	today := time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, location)

	//	定时器今天
	du := today.Add(time.Hour * 24).Sub(now)
	log.Printf("[%s]\t%s后启动首次定时任务", code, du.String())
	time.AfterFunc(du, func() {
		ticker := time.NewTicker(time.Hour * 24)
		log.Printf("[%s]\t已启动定时抓取任务", code)

		//	立刻运行一次
		go func() {
			now = time.Now().In(location)
			today = time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, location)

			err := stockToday(code, today)
			if err != nil {
				log.Fatalf("[%s]\t抓取%s的数据出错:%v", code, today.Format("20060102"), err)
			}
		}()

		//	每天运行一次
		for _ = range ticker.C {
			now = time.Now().In(location)
			today = time.Date(now.Year(), now.Month(), now.Day(), 0, 0, 0, 0, location)

			err := stockToday(code, today)
			if err != nil {
				log.Fatalf("[%s]\t抓取%s的数据出错:%v", code, today.Format("20060102"), err)
			}
		}
	})

	//	历史
	return stockHistory(code, today)
}

//	抓取今天
func stockToday(code string, today time.Time) error {
	log.Printf("[%s]\t%s分时数据抓取任务-开始", code, today.Format("20060102"))
	//	保存原始数据
	day := today.Add(-time.Hour * 24)
	for try := 0; try < retryCount; try++ {
		//	抓取数据
		err := thatDay(code, day)
		if err != nil {
			log.Fatalf("[%s]\t抓取%s的数据出错(还有%d次):%v", try, code, today.Format("20060102"), err)
			time.Sleep(retryDelay)
		}

		log.Printf("[%s]\t%s分时数据抓取任务-结束", code, day.Format("20060102"))
		return nil
	}

	return errors.New(fmt.Sprintf("[%s]\t%s分时数据抓取任务失败", code, today.Format("20060102")))
}

//	抓取历史
func stockHistory(code string, today time.Time) error {
	log.Printf("[%s]\t历史分时数据抓取任务-开始", code)
	//	保存原始数据
	day := today.Add(-time.Hour * 24)
	for index := 0; index < lastestDays; index++ {

		for try := 0; try < retryCount; try++ {
			//	抓取数据
			err := thatDay(code, day)
			if err == nil {
				//	往前一天递推
				day = day.Add(-time.Hour * 24)
				break
			} else {
				log.Fatalf("[%s]\t抓取%s的数据出错(还有%d次):%v", code, day.Format("20060102"), retryCount-try-1, err)
				time.Sleep(retryDelay)
			}
		}
	}
	log.Printf("[%s]\t历史分时数据抓取任务-结束", code)
	return nil
}

//	抓取某一天
func thatDay(code string, day time.Time) error {

	//	文件保存路径
	dataDir := config.GetDataDir()
	fileName := fmt.Sprintf("%s_raw.txt", day.Format("20060102"))
	filePath := filepath.Join(dataDir, code, fileName)

	//	如果文件已存在就忽略
	_, err := os.Stat(filePath)
	if os.IsNotExist(err) {
		//	如果不存在就抓取并保存
		start := time.Date(day.Year(), day.Month(), day.Day(), 0, 0, 0, 0, day.Location())
		end := start.Add(time.Hour * 24)
		html, err := peroid(code, start, end)
		if err != nil {
			return err
		}

		//	写入文件
		return io.WriteString(filePath, html)
	}

	return nil
}

//	抓取一段时间
func peroid(code string, start, end time.Time) (string, error) {

	pattern := "https://finance-yql.media.yahoo.com/v7/finance/chart/%s?period2=%d&period1=%d&interval=1m&indicators=quote&includeTimestamps=true&includePrePost=true&events=div%7Csplit%7Cearn&corsDomain=finance.yahoo.com"
	url := fmt.Sprintf(pattern, code, end.Unix(), start.Unix())

	//	抓取数据
	return io.GetString(url)
}
<|endoftext|>


<commit_before>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// IP address manipulations
//
// IPv4 addresses are 4 bytes; IPv6 addresses are 16 bytes.
// An IPv4 address can be converted to an IPv6 address by
// adding a canonical prefix (10 zeros, 2 0xFFs).
// This library accepts either size of byte array but always
// returns 16-byte addresses.

package net

// IP address lengths (bytes).
const (
	IPv4len = 4
	IPv6len = 16
)

// An IP is a single IP address, an array of bytes.
// Functions in this package accept either 4-byte (IP v4)
// or 16-byte (IP v6) arrays as input.  Unless otherwise
// specified, functions in this package always return
// IP addresses in 16-byte form using the canonical
// embedding.
//
// Note that in this documentation, referring to an
// IP address as an IPv4 address or an IPv6 address
// is a semantic property of the address, not just the
// length of the byte array: a 16-byte array can still
// be an IPv4 address.
type IP []byte

// An IP mask is an IP address.
type IPMask []byte

// IPv4 returns the IP address (in 16-byte form) of the
// IPv4 address a.b.c.d.
func IPv4(a, b, c, d byte) IP {
	p := make(IP, IPv6len)
	for i := 0; i < 10; i++ {
		p[i] = 0
	}
	p[10] = 0xff
	p[11] = 0xff
	p[12] = a
	p[13] = b
	p[14] = c
	p[15] = d
	return p
}

// Well-known IPv4 addresses
var (
	IPv4bcast     = IPv4(255, 255, 255, 255) // broadcast
	IPv4allsys    = IPv4(224, 0, 0, 1)       // all systems
	IPv4allrouter = IPv4(224, 0, 0, 2)       // all routers
	IPv4zero      = IPv4(0, 0, 0, 0)         // all zeros
)

// Well-known IPv6 addresses
var (
	IPzero = make(IP, IPv6len) // all zeros
)

// Is p all zeros?
func isZeros(p IP) bool {
	for i := 0; i < len(p); i++ {
		if p[i] != 0 {
			return false
		}
	}
	return true
}

// To4 converts the IPv4 address ip to a 4-byte representation.
// If ip is not an IPv4 address, To4 returns nil.
func (ip IP) To4() IP {
	if len(ip) == IPv4len {
		return ip
	}
	if len(ip) == IPv6len &&
		isZeros(ip[0:10]) &&
		ip[10] == 0xff &&
		ip[11] == 0xff {
		return ip[12:16]
	}
	return nil
}

// To16 converts the IP address ip to a 16-byte representation.
// If ip is not an IP address (it is the wrong length), To16 returns nil.
func (ip IP) To16() IP {
	if len(ip) == IPv4len {
		return IPv4(ip[0], ip[1], ip[2], ip[3])
	}
	if len(ip) == IPv6len {
		return ip
	}
	return nil
}

// Default route masks for IPv4.
var (
	classAMask = IPMask(IPv4(0xff, 0, 0, 0))
	classBMask = IPMask(IPv4(0xff, 0xff, 0, 0))
	classCMask = IPMask(IPv4(0xff, 0xff, 0xff, 0))
)

// DefaultMask returns the default IP mask for the IP address ip.
// Only IPv4 addresses have default masks; DefaultMask returns
// nil if ip is not a valid IPv4 address.
func (ip IP) DefaultMask() IPMask {
	if ip = ip.To4(); ip == nil {
		return nil
	}
	switch true {
	case ip[0] < 0x80:
		return classAMask
	case ip[0] < 0xC0:
		return classBMask
	default:
		return classCMask
	}
	return nil // not reached
}

// Mask returns the result of masking the IP address ip with mask.
func (ip IP) Mask(mask IPMask) IP {
	n := len(ip)
	if n != len(mask) {
		return nil
	}
	out := make(IP, n)
	for i := 0; i < n; i++ {
		out[i] = ip[i] & mask[i]
	}
	return out
}

// Convert i to decimal string.
func itod(i uint) string {
	if i == 0 {
		return "0"
	}

	// Assemble decimal in reverse order.
	var b [32]byte
	bp := len(b)
	for ; i > 0; i /= 10 {
		bp--
		b[bp] = byte(i%10) + '0'
	}

	return string(b[bp:])
}

// Convert i to hexadecimal string.
func itox(i uint) string {
	if i == 0 {
		return "0"
	}

	// Assemble hexadecimal in reverse order.
	var b [32]byte
	bp := len(b)
	for ; i > 0; i /= 16 {
		bp--
		b[bp] = "0123456789abcdef"[byte(i%16)]
	}

	return string(b[bp:])
}

// String returns the string form of the IP address ip.
// If the address is an IPv4 address, the string representation
// is dotted decimal ("74.125.19.99").  Otherwise the representation
// is IPv6 ("2001:4860:0:2001::68").
func (ip IP) String() string {
	p := ip

	if len(ip) == 0 {
		return ""
	}

	// If IPv4, use dotted notation.
	if p4 := p.To4(); len(p4) == 4 {
		return itod(uint(p4[0])) + "." +
			itod(uint(p4[1])) + "." +
			itod(uint(p4[2])) + "." +
			itod(uint(p4[3]))
	}
	if len(p) != IPv6len {
		return "?"
	}

	// Find longest run of zeros.
	e0 := -1
	e1 := -1
	for i := 0; i < 16; i += 2 {
		j := i
		for j < 16 && p[j] == 0 && p[j+1] == 0 {
			j += 2
		}
		if j > i && j-i > e1-e0 {
			e0 = i
			e1 = j
		}
	}

	// Print with possible :: in place of run of zeros
	var s string
	for i := 0; i < 16; i += 2 {
		if i == e0 {
			s += "::"
			i = e1
			if i >= 16 {
				break
			}
		} else if i > 0 {
			s += ":"
		}
		s += itox((uint(p[i]) << 8) | uint(p[i+1]))
	}
	return s
}

// If mask is a sequence of 1 bits followed by 0 bits,
// return the number of 1 bits.
func simpleMaskLength(mask IPMask) int {
	var i int
	for i = 0; i < len(mask); i++ {
		if mask[i] != 0xFF {
			break
		}
	}
	n := 8 * i
	v := mask[i]
	for v&0x80 != 0 {
		n++
		v <<= 1
	}
	if v != 0 {
		return -1
	}
	for i++; i < len(mask); i++ {
		if mask[i] != 0 {
			return -1
		}
	}
	return n
}

// String returns the string representation of mask.
// If the mask is in the canonical form--ones followed by zeros--the
// string representation is just the decimal number of ones.
// If the mask is in a non-canonical form, it is formatted
// as an IP address.
func (mask IPMask) String() string {
	switch len(mask) {
	case 4:
		n := simpleMaskLength(mask)
		if n >= 0 {
			return itod(uint(n + (IPv6len-IPv4len)*8))
		}
	case 16:
		n := simpleMaskLength(mask)
		if n >= 0 {
			return itod(uint(n))
		}
	}
	return IP(mask).String()
}

// Parse IPv4 address (d.d.d.d).
func parseIPv4(s string) IP {
	var p [IPv4len]byte
	i := 0
	for j := 0; j < IPv4len; j++ {
		if i >= len(s) {
			// Missing octets.
			return nil
		}
		if j > 0 {
			if s[i] != '.' {
				return nil
			}
			i++
		}
		var (
			n  int
			ok bool
		)
		n, i, ok = dtoi(s, i)
		if !ok || n > 0xFF {
			return nil
		}
		p[j] = byte(n)
	}
	if i != len(s) {
		return nil
	}
	return IPv4(p[0], p[1], p[2], p[3])
}

// Parse IPv6 address.  Many forms.
// The basic form is a sequence of eight colon-separated
// 16-bit hex numbers separated by colons,
// as in 0123:4567:89ab:cdef:0123:4567:89ab:cdef.
// Two exceptions:
//	* A run of zeros can be replaced with "::".
//	* The last 32 bits can be in IPv4 form.
// Thus, ::ffff:1.2.3.4 is the IPv4 address 1.2.3.4.
func parseIPv6(s string) IP {
	p := make(IP, 16)
	ellipsis := -1 // position of ellipsis in p
	i := 0         // index in string s

	// Might have leading ellipsis
	if len(s) >= 2 && s[0] == ':' && s[1] == ':' {
		ellipsis = 0
		i = 2
		// Might be only ellipsis
		if i == len(s) {
			return p
		}
	}

	// Loop, parsing hex numbers followed by colon.
	j := 0
L: for j < IPv6len {
		// Hex number.
		n, i1, ok := xtoi(s, i)
		if !ok || n > 0xFFFF {
			return nil
		}

		// If followed by dot, might be in trailing IPv4.
		if i1 < len(s) && s[i1] == '.' {
			if ellipsis < 0 && j != IPv6len-IPv4len {
				// Not the right place.
				return nil
			}
			if j+IPv4len > IPv6len {
				// Not enough room.
				return nil
			}
			p4 := parseIPv4(s[i:])
			if p4 == nil {
				return nil
			}
			p[j] = p4[12]
			p[j+1] = p4[13]
			p[j+2] = p4[14]
			p[j+3] = p4[15]
			i = len(s)
			j += 4
			break
		}

		// Save this 16-bit chunk.
		p[j] = byte(n >> 8)
		p[j+1] = byte(n)
		j += 2

		// Stop at end of string.
		i = i1
		if i == len(s) {
			break
		}

		// Otherwise must be followed by colon and more.
		if s[i] != ':' && i+1 == len(s) {
			return nil
		}
		i++

		// Look for ellipsis.
		if s[i] == ':' {
			if ellipsis >= 0 { // already have one
				return nil
			}
			ellipsis = j
			if i++; i == len(s) { // can be at end
				break
			}
		}
	}

	// Must have used entire string.
	if i != len(s) {
		return nil
	}

	// If didn't parse enough, expand ellipsis.
	if j < IPv6len {
		if ellipsis < 0 {
			return nil
		}
		n := IPv6len - j
		for k := j - 1; k >= ellipsis; k-- {
			p[k+n] = p[k]
		}
		for k := ellipsis + n - 1; k >= ellipsis; k-- {
			p[k] = 0
		}
	}
	return p
}

// ParseIP parses s as an IP address, returning the result.
// The string s can be in dotted decimal ("74.125.19.99")
// or IPv6 ("2001:4860:0:2001::68") form.
// If s is not a valid textual representation of an IP address,
// ParseIP returns nil.
func ParseIP(s string) IP {
	p := parseIPv4(s)
	if p != nil {
		return p
	}
	return parseIPv6(s)
}
<commit_msg>net: fix IPMask.String not to crash on all-0xff mask<commit_after>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// IP address manipulations
//
// IPv4 addresses are 4 bytes; IPv6 addresses are 16 bytes.
// An IPv4 address can be converted to an IPv6 address by
// adding a canonical prefix (10 zeros, 2 0xFFs).
// This library accepts either size of byte array but always
// returns 16-byte addresses.

package net

// IP address lengths (bytes).
const (
	IPv4len = 4
	IPv6len = 16
)

// An IP is a single IP address, an array of bytes.
// Functions in this package accept either 4-byte (IP v4)
// or 16-byte (IP v6) arrays as input.  Unless otherwise
// specified, functions in this package always return
// IP addresses in 16-byte form using the canonical
// embedding.
//
// Note that in this documentation, referring to an
// IP address as an IPv4 address or an IPv6 address
// is a semantic property of the address, not just the
// length of the byte array: a 16-byte array can still
// be an IPv4 address.
type IP []byte

// An IP mask is an IP address.
type IPMask []byte

// IPv4 returns the IP address (in 16-byte form) of the
// IPv4 address a.b.c.d.
func IPv4(a, b, c, d byte) IP {
	p := make(IP, IPv6len)
	for i := 0; i < 10; i++ {
		p[i] = 0
	}
	p[10] = 0xff
	p[11] = 0xff
	p[12] = a
	p[13] = b
	p[14] = c
	p[15] = d
	return p
}

// IPv4Mask returns the IP mask (in 16-byte form) of the
// IPv4 mask a.b.c.d.
func IPv4Mask(a, b, c, d byte) IPMask {
	p := make(IPMask, IPv6len)
	for i := 0; i < 12; i++ {
		p[i] = 0xff
	}
	p[12] = a
	p[13] = b
	p[14] = c
	p[15] = d
	return p
}

// Well-known IPv4 addresses
var (
	IPv4bcast     = IPv4(255, 255, 255, 255) // broadcast
	IPv4allsys    = IPv4(224, 0, 0, 1)       // all systems
	IPv4allrouter = IPv4(224, 0, 0, 2)       // all routers
	IPv4zero      = IPv4(0, 0, 0, 0)         // all zeros
)

// Well-known IPv6 addresses
var (
	IPzero = make(IP, IPv6len) // all zeros
)

// Is p all zeros?
func isZeros(p IP) bool {
	for i := 0; i < len(p); i++ {
		if p[i] != 0 {
			return false
		}
	}
	return true
}

// To4 converts the IPv4 address ip to a 4-byte representation.
// If ip is not an IPv4 address, To4 returns nil.
func (ip IP) To4() IP {
	if len(ip) == IPv4len {
		return ip
	}
	if len(ip) == IPv6len &&
		isZeros(ip[0:10]) &&
		ip[10] == 0xff &&
		ip[11] == 0xff {
		return ip[12:16]
	}
	return nil
}

// To16 converts the IP address ip to a 16-byte representation.
// If ip is not an IP address (it is the wrong length), To16 returns nil.
func (ip IP) To16() IP {
	if len(ip) == IPv4len {
		return IPv4(ip[0], ip[1], ip[2], ip[3])
	}
	if len(ip) == IPv6len {
		return ip
	}
	return nil
}

// Default route masks for IPv4.
var (
	classAMask = IPv4Mask(0xff, 0, 0, 0)
	classBMask = IPv4Mask(0xff, 0xff, 0, 0)
	classCMask = IPv4Mask(0xff, 0xff, 0xff, 0)
)

// DefaultMask returns the default IP mask for the IP address ip.
// Only IPv4 addresses have default masks; DefaultMask returns
// nil if ip is not a valid IPv4 address.
func (ip IP) DefaultMask() IPMask {
	if ip = ip.To4(); ip == nil {
		return nil
	}
	switch true {
	case ip[0] < 0x80:
		return classAMask
	case ip[0] < 0xC0:
		return classBMask
	default:
		return classCMask
	}
	return nil // not reached
}

// Mask returns the result of masking the IP address ip with mask.
func (ip IP) Mask(mask IPMask) IP {
	n := len(ip)
	if n != len(mask) {
		return nil
	}
	out := make(IP, n)
	for i := 0; i < n; i++ {
		out[i] = ip[i] & mask[i]
	}
	return out
}

// Convert i to decimal string.
func itod(i uint) string {
	if i == 0 {
		return "0"
	}

	// Assemble decimal in reverse order.
	var b [32]byte
	bp := len(b)
	for ; i > 0; i /= 10 {
		bp--
		b[bp] = byte(i%10) + '0'
	}

	return string(b[bp:])
}

// Convert i to hexadecimal string.
func itox(i uint) string {
	if i == 0 {
		return "0"
	}

	// Assemble hexadecimal in reverse order.
	var b [32]byte
	bp := len(b)
	for ; i > 0; i /= 16 {
		bp--
		b[bp] = "0123456789abcdef"[byte(i%16)]
	}

	return string(b[bp:])
}

// String returns the string form of the IP address ip.
// If the address is an IPv4 address, the string representation
// is dotted decimal ("74.125.19.99").  Otherwise the representation
// is IPv6 ("2001:4860:0:2001::68").
func (ip IP) String() string {
	p := ip

	if len(ip) == 0 {
		return ""
	}

	// If IPv4, use dotted notation.
	if p4 := p.To4(); len(p4) == 4 {
		return itod(uint(p4[0])) + "." +
			itod(uint(p4[1])) + "." +
			itod(uint(p4[2])) + "." +
			itod(uint(p4[3]))
	}
	if len(p) != IPv6len {
		return "?"
	}

	// Find longest run of zeros.
	e0 := -1
	e1 := -1
	for i := 0; i < 16; i += 2 {
		j := i
		for j < 16 && p[j] == 0 && p[j+1] == 0 {
			j += 2
		}
		if j > i && j-i > e1-e0 {
			e0 = i
			e1 = j
		}
	}

	// Print with possible :: in place of run of zeros
	var s string
	for i := 0; i < 16; i += 2 {
		if i == e0 {
			s += "::"
			i = e1
			if i >= 16 {
				break
			}
		} else if i > 0 {
			s += ":"
		}
		s += itox((uint(p[i]) << 8) | uint(p[i+1]))
	}
	return s
}

// If mask is a sequence of 1 bits followed by 0 bits,
// return the number of 1 bits.
func simpleMaskLength(mask IPMask) int {
	var n int
	for i, v := range mask {
		if v == 0xff {
			n += 8
			continue
		}
		// found non-ff byte
		// count 1 bits
		for v&0x80 != 0 {
			n++
			v <<= 1
		}
		// rest must be 0 bits
		if v != 0 {
			return -1
		}
		for i++; i < len(mask); i++ {
			if mask[i] != 0 {
				return -1
			}
		}
		break
	}
	return n
}

// String returns the string representation of mask.
// If the mask is in the canonical form--ones followed by zeros--the
// string representation is just the decimal number of ones.
// If the mask is in a non-canonical form, it is formatted
// as an IP address.
func (mask IPMask) String() string {
	switch len(mask) {
	case 4:
		n := simpleMaskLength(mask)
		if n >= 0 {
			return itod(uint(n + (IPv6len-IPv4len)*8))
		}
	case 16:
		n := simpleMaskLength(mask)
		if n >= 12*8 {
			return itod(uint(n - 12*8))
		}
	}
	return IP(mask).String()
}

// Parse IPv4 address (d.d.d.d).
func parseIPv4(s string) IP {
	var p [IPv4len]byte
	i := 0
	for j := 0; j < IPv4len; j++ {
		if i >= len(s) {
			// Missing octets.
			return nil
		}
		if j > 0 {
			if s[i] != '.' {
				return nil
			}
			i++
		}
		var (
			n  int
			ok bool
		)
		n, i, ok = dtoi(s, i)
		if !ok || n > 0xFF {
			return nil
		}
		p[j] = byte(n)
	}
	if i != len(s) {
		return nil
	}
	return IPv4(p[0], p[1], p[2], p[3])
}

// Parse IPv6 address.  Many forms.
// The basic form is a sequence of eight colon-separated
// 16-bit hex numbers separated by colons,
// as in 0123:4567:89ab:cdef:0123:4567:89ab:cdef.
// Two exceptions:
//	* A run of zeros can be replaced with "::".
//	* The last 32 bits can be in IPv4 form.
// Thus, ::ffff:1.2.3.4 is the IPv4 address 1.2.3.4.
func parseIPv6(s string) IP {
	p := make(IP, 16)
	ellipsis := -1 // position of ellipsis in p
	i := 0         // index in string s

	// Might have leading ellipsis
	if len(s) >= 2 && s[0] == ':' && s[1] == ':' {
		ellipsis = 0
		i = 2
		// Might be only ellipsis
		if i == len(s) {
			return p
		}
	}

	// Loop, parsing hex numbers followed by colon.
	j := 0
L: for j < IPv6len {
		// Hex number.
		n, i1, ok := xtoi(s, i)
		if !ok || n > 0xFFFF {
			return nil
		}

		// If followed by dot, might be in trailing IPv4.
		if i1 < len(s) && s[i1] == '.' {
			if ellipsis < 0 && j != IPv6len-IPv4len {
				// Not the right place.
				return nil
			}
			if j+IPv4len > IPv6len {
				// Not enough room.
				return nil
			}
			p4 := parseIPv4(s[i:])
			if p4 == nil {
				return nil
			}
			p[j] = p4[12]
			p[j+1] = p4[13]
			p[j+2] = p4[14]
			p[j+3] = p4[15]
			i = len(s)
			j += 4
			break
		}

		// Save this 16-bit chunk.
		p[j] = byte(n >> 8)
		p[j+1] = byte(n)
		j += 2

		// Stop at end of string.
		i = i1
		if i == len(s) {
			break
		}

		// Otherwise must be followed by colon and more.
		if s[i] != ':' && i+1 == len(s) {
			return nil
		}
		i++

		// Look for ellipsis.
		if s[i] == ':' {
			if ellipsis >= 0 { // already have one
				return nil
			}
			ellipsis = j
			if i++; i == len(s) { // can be at end
				break
			}
		}
	}

	// Must have used entire string.
	if i != len(s) {
		return nil
	}

	// If didn't parse enough, expand ellipsis.
	if j < IPv6len {
		if ellipsis < 0 {
			return nil
		}
		n := IPv6len - j
		for k := j - 1; k >= ellipsis; k-- {
			p[k+n] = p[k]
		}
		for k := ellipsis + n - 1; k >= ellipsis; k-- {
			p[k] = 0
		}
	}
	return p
}

// ParseIP parses s as an IP address, returning the result.
// The string s can be in dotted decimal ("74.125.19.99")
// or IPv6 ("2001:4860:0:2001::68") form.
// If s is not a valid textual representation of an IP address,
// ParseIP returns nil.
func ParseIP(s string) IP {
	p := parseIPv4(s)
	if p != nil {
		return p
	}
	return parseIPv6(s)
}
<|endoftext|>


<commit_before>package gcache

import (
	"errors"
	"sync"
	"time"
)

const (
	TYPE_SIMPLE = "simple"
	TYPE_LRU    = "lru"
	TYPE_LFU    = "lfu"
	TYPE_ARC    = "arc"
)

var KeyNotFoundError = errors.New("Key not found.")

type Cache interface {
	Set(interface{}, interface{}) error
	SetWithExpire(interface{}, interface{}, time.Duration) error
	Get(interface{}) (interface{}, error)
	GetIFPresent(interface{}) (interface{}, error)
	GetALL() map[interface{}]interface{}
	get(interface{}, bool) (interface{}, error)
	Remove(interface{}) bool
	Purge()
	Keys() []interface{}
	Len() int

	statsAccessor
}

type baseCache struct {
	clock            Clock
	size             int
	loaderExpireFunc LoaderExpireFunc
	evictedFunc      EvictedFunc
	addedFunc        AddedFunc
	deserializeFunc  DeserializeFunc
	serializeFunc    SerializeFunc
	expiration       *time.Duration
	mu               sync.RWMutex
	loadGroup        Group
	*stats
}

type (
	LoaderFunc       func(interface{}) (interface{}, error)
	LoaderExpireFunc func(interface{}) (interface{}, *time.Duration, error)
	EvictedFunc      func(interface{}, interface{})
	AddedFunc        func(interface{}, interface{})
	DeserializeFunc  func(interface{}, interface{}) (interface{}, error)
	SerializeFunc    func(interface{}, interface{}) (interface{}, error)
)

type CacheBuilder struct {
	clock            Clock
	tp               string
	size             int
	loaderExpireFunc LoaderExpireFunc
	evictedFunc      EvictedFunc
	addedFunc        AddedFunc
	expiration       *time.Duration
	deserializeFunc  DeserializeFunc
	serializeFunc    SerializeFunc
}

func New(size int) *CacheBuilder {
	if size <= 0 {
		panic("gcache: size <= 0")
	}
	return &CacheBuilder{
		clock: NewRealClock(),
		tp:    TYPE_SIMPLE,
		size:  size,
	}
}

func (cb *CacheBuilder) Clock(clock Clock) *CacheBuilder {
	cb.clock = clock
	return cb
}

// Set a loader function.
// loaderFunc: create a new value with this function if cached value is expired.
func (cb *CacheBuilder) LoaderFunc(loaderFunc LoaderFunc) *CacheBuilder {
	cb.loaderExpireFunc = func(k interface{}) (interface{}, *time.Duration, error) {
		v, err := loaderFunc(k)
		return v, nil, err
	}
	return cb
}

// Set a loader function with expiration.
// loaderExpireFunc: create a new value with this function if cached value is expired.
// If nil returned instead of time.Duration from loaderExpireFunc than value will never expire.
func (cb *CacheBuilder) LoaderExpireFunc(loaderExpireFunc LoaderExpireFunc) *CacheBuilder {
	cb.loaderExpireFunc = loaderExpireFunc
	return cb
}

func (cb *CacheBuilder) EvictType(tp string) *CacheBuilder {
	cb.tp = tp
	return cb
}

func (cb *CacheBuilder) Simple() *CacheBuilder {
	return cb.EvictType(TYPE_SIMPLE)
}

func (cb *CacheBuilder) LRU() *CacheBuilder {
	return cb.EvictType(TYPE_LRU)
}

func (cb *CacheBuilder) LFU() *CacheBuilder {
	return cb.EvictType(TYPE_LFU)
}

func (cb *CacheBuilder) ARC() *CacheBuilder {
	return cb.EvictType(TYPE_ARC)
}

func (cb *CacheBuilder) EvictedFunc(evictedFunc EvictedFunc) *CacheBuilder {
	cb.evictedFunc = evictedFunc
	return cb
}

func (cb *CacheBuilder) AddedFunc(addedFunc AddedFunc) *CacheBuilder {
	cb.addedFunc = addedFunc
	return cb
}

func (cb *CacheBuilder) DeserializeFunc(deserializeFunc DeserializeFunc) *CacheBuilder {
	cb.deserializeFunc = deserializeFunc
	return cb
}

func (cb *CacheBuilder) SerializeFunc(serializeFunc SerializeFunc) *CacheBuilder {
	cb.serializeFunc = serializeFunc
	return cb
}

func (cb *CacheBuilder) Expiration(expiration time.Duration) *CacheBuilder {
	cb.expiration = &expiration
	return cb
}

func (cb *CacheBuilder) Build() Cache {
	return cb.build()
}

func (cb *CacheBuilder) build() Cache {
	switch cb.tp {
	case TYPE_SIMPLE:
		return newSimpleCache(cb)
	case TYPE_LRU:
		return newLRUCache(cb)
	case TYPE_LFU:
		return newLFUCache(cb)
	case TYPE_ARC:
		return newARC(cb)
	default:
		panic("gcache: Unknown type " + cb.tp)
	}
}

func buildCache(c *baseCache, cb *CacheBuilder) {
	c.clock = cb.clock
	c.size = cb.size
	c.loaderExpireFunc = cb.loaderExpireFunc
	c.expiration = cb.expiration
	c.addedFunc = cb.addedFunc
	c.deserializeFunc = cb.deserializeFunc
	c.serializeFunc = cb.serializeFunc
	c.evictedFunc = cb.evictedFunc
	c.stats = &stats{}
}

// load a new value using by specified key.
func (c *baseCache) load(key interface{}, cb func(interface{}, *time.Duration, error) (interface{}, error), isWait bool) (interface{}, bool, error) {
	v, called, err := c.loadGroup.Do(key, func() (interface{}, error) {
		return cb(c.loaderExpireFunc(key))
	}, isWait)
	if err != nil {
		return nil, called, err
	}
	return v, called, nil
}
<commit_msg>add recover function that handles panic (#41)<commit_after>package gcache

import (
	"errors"
	"fmt"
	"sync"
	"time"
)

const (
	TYPE_SIMPLE = "simple"
	TYPE_LRU    = "lru"
	TYPE_LFU    = "lfu"
	TYPE_ARC    = "arc"
)

var KeyNotFoundError = errors.New("Key not found.")

type Cache interface {
	Set(interface{}, interface{}) error
	SetWithExpire(interface{}, interface{}, time.Duration) error
	Get(interface{}) (interface{}, error)
	GetIFPresent(interface{}) (interface{}, error)
	GetALL() map[interface{}]interface{}
	get(interface{}, bool) (interface{}, error)
	Remove(interface{}) bool
	Purge()
	Keys() []interface{}
	Len() int

	statsAccessor
}

type baseCache struct {
	clock            Clock
	size             int
	loaderExpireFunc LoaderExpireFunc
	evictedFunc      EvictedFunc
	addedFunc        AddedFunc
	deserializeFunc  DeserializeFunc
	serializeFunc    SerializeFunc
	expiration       *time.Duration
	mu               sync.RWMutex
	loadGroup        Group
	*stats
}

type (
	LoaderFunc       func(interface{}) (interface{}, error)
	LoaderExpireFunc func(interface{}) (interface{}, *time.Duration, error)
	EvictedFunc      func(interface{}, interface{})
	AddedFunc        func(interface{}, interface{})
	DeserializeFunc  func(interface{}, interface{}) (interface{}, error)
	SerializeFunc    func(interface{}, interface{}) (interface{}, error)
)

type CacheBuilder struct {
	clock            Clock
	tp               string
	size             int
	loaderExpireFunc LoaderExpireFunc
	evictedFunc      EvictedFunc
	addedFunc        AddedFunc
	expiration       *time.Duration
	deserializeFunc  DeserializeFunc
	serializeFunc    SerializeFunc
}

func New(size int) *CacheBuilder {
	if size <= 0 {
		panic("gcache: size <= 0")
	}
	return &CacheBuilder{
		clock: NewRealClock(),
		tp:    TYPE_SIMPLE,
		size:  size,
	}
}

func (cb *CacheBuilder) Clock(clock Clock) *CacheBuilder {
	cb.clock = clock
	return cb
}

// Set a loader function.
// loaderFunc: create a new value with this function if cached value is expired.
func (cb *CacheBuilder) LoaderFunc(loaderFunc LoaderFunc) *CacheBuilder {
	cb.loaderExpireFunc = func(k interface{}) (interface{}, *time.Duration, error) {
		v, err := loaderFunc(k)
		return v, nil, err
	}
	return cb
}

// Set a loader function with expiration.
// loaderExpireFunc: create a new value with this function if cached value is expired.
// If nil returned instead of time.Duration from loaderExpireFunc than value will never expire.
func (cb *CacheBuilder) LoaderExpireFunc(loaderExpireFunc LoaderExpireFunc) *CacheBuilder {
	cb.loaderExpireFunc = loaderExpireFunc
	return cb
}

func (cb *CacheBuilder) EvictType(tp string) *CacheBuilder {
	cb.tp = tp
	return cb
}

func (cb *CacheBuilder) Simple() *CacheBuilder {
	return cb.EvictType(TYPE_SIMPLE)
}

func (cb *CacheBuilder) LRU() *CacheBuilder {
	return cb.EvictType(TYPE_LRU)
}

func (cb *CacheBuilder) LFU() *CacheBuilder {
	return cb.EvictType(TYPE_LFU)
}

func (cb *CacheBuilder) ARC() *CacheBuilder {
	return cb.EvictType(TYPE_ARC)
}

func (cb *CacheBuilder) EvictedFunc(evictedFunc EvictedFunc) *CacheBuilder {
	cb.evictedFunc = evictedFunc
	return cb
}

func (cb *CacheBuilder) AddedFunc(addedFunc AddedFunc) *CacheBuilder {
	cb.addedFunc = addedFunc
	return cb
}

func (cb *CacheBuilder) DeserializeFunc(deserializeFunc DeserializeFunc) *CacheBuilder {
	cb.deserializeFunc = deserializeFunc
	return cb
}

func (cb *CacheBuilder) SerializeFunc(serializeFunc SerializeFunc) *CacheBuilder {
	cb.serializeFunc = serializeFunc
	return cb
}

func (cb *CacheBuilder) Expiration(expiration time.Duration) *CacheBuilder {
	cb.expiration = &expiration
	return cb
}

func (cb *CacheBuilder) Build() Cache {
	return cb.build()
}

func (cb *CacheBuilder) build() Cache {
	switch cb.tp {
	case TYPE_SIMPLE:
		return newSimpleCache(cb)
	case TYPE_LRU:
		return newLRUCache(cb)
	case TYPE_LFU:
		return newLFUCache(cb)
	case TYPE_ARC:
		return newARC(cb)
	default:
		panic("gcache: Unknown type " + cb.tp)
	}
}

func buildCache(c *baseCache, cb *CacheBuilder) {
	c.clock = cb.clock
	c.size = cb.size
	c.loaderExpireFunc = cb.loaderExpireFunc
	c.expiration = cb.expiration
	c.addedFunc = cb.addedFunc
	c.deserializeFunc = cb.deserializeFunc
	c.serializeFunc = cb.serializeFunc
	c.evictedFunc = cb.evictedFunc
	c.stats = &stats{}
}

// load a new value using by specified key.
func (c *baseCache) load(key interface{}, cb func(interface{}, *time.Duration, error) (interface{}, error), isWait bool) (interface{}, bool, error) {
	v, called, err := c.loadGroup.Do(key, func() (v interface{}, e error) {
		defer func() {
			if r := recover(); r != nil {
				e = fmt.Errorf("Loader panics: %v", r)
			}
		}()
		return cb(c.loaderExpireFunc(key))
	}, isWait)
	if err != nil {
		return nil, called, err
	}
	return v, called, nil
}
<|endoftext|>