<commit_before>// Package docker provides a layer on top of Docker's API via Kite handlers.
package docker

import (
	"errors"

	"github.com/koding/kite"
)

// Docker defines the main configuration. One instance is running as one Docker
// client, so multiple instances of a Docker struct can connect to multiple
// Docker Servers. After creating a new Docker struct, it'll be able to manage
// Docker containers. A usual working lifecyle is:
// 1. Build a Docker image
// 2. Create a new Docker container from that image
// 3. Start this container
// 4. Connect and open a terminal instance to it (optional)
// 5. Stop the container
// 6. Remove the container
// 7. Destroy the image
type Docker struct{}

// TODO:
// 1. Docker Deamon needs to be run in TCP Mode. If UNIX sockets are used we
// need to setup the client so it has access to the `docker` group which
// maintains the the socket. So initiallty TCP is a good start.
// 2. TCP mode is enabled by adding "-H tcp://bind-ip:port" to
// /etc/default/docker or DOCKER_OPTS
// 3. But only authenticated Client should access it over TCP, so we need to
// generate TLS cert keys and let both the Deamon and Client use it.
// 4. The boot2docker guys are using this package:
// https://github.com/SvenDowideit/generate_cert for it which we can also use.
// 5. Here is also some information on how to start the Docker Deamon securely
// https://docs.docker.com/articles/https/
func New() *Docker {
	return &Docker{}
}

// Build builds a new container image from a public Docker path or froma a
// given Dockerfile
func (d *Docker) Build(r *kite.Request) (interface{}, error) {
	return nil, errors.New("not implemented yet.")
}

// Create creates a new container
func (d *Docker) Create(r *kite.Request) (interface{}, error) {
	return nil, errors.New("not implemented yet.")
}

// Connects connects to an existing Container by spawning a new process and
// attaching to it.
func (d *Docker) Connect(r *kite.Request) (interface{}, error) {
	return nil, errors.New("not implemented yet.")
}

// Stop stops a running container
func (d *Docker) Stop(r *kite.Request) (interface{}, error) {
	return nil, errors.New("not implemented yet.")
}

// Start starts a stopped container
func (d *Docker) Start(r *kite.Request) (interface{}, error) {
	return nil, errors.New("not implemented yet.")
}

// Kill kills and delete a container
func (d *Docker) Kill(r *kite.Request) (interface{}, error) {
	return nil, errors.New("not implemented yet.")
}

// Destroy destroys and removes an image.
func (d *Docker) Destroy(r *kite.Request) (interface{}, error) {
	return nil, errors.New("not implemented yet.")
}
<commit_msg>docker: add consructor and include handlers into Klient<commit_after>// Package docker provides a layer on top of Docker's API via Kite handlers.
package docker

import (
	"errors"

	"github.com/koding/kite"
	"github.com/samalba/dockerclient"
)

// Docker defines the main configuration. One instance is running as one Docker
// client, so multiple instances of a Docker struct can connect to multiple
// Docker Servers. After creating a new Docker struct, it'll be able to manage
// Docker containers. A usual working lifecyle is:
// 1. Build a Docker image
// 2. Create a new Docker container from that image
// 3. Start this container
// 4. Connect and open a terminal instance to it (optional)
// 5. Stop the container
// 6. Remove the container
// 7. Destroy the image
type Docker struct {
	client *dockerclient.DockerClient
}

// New connects to a Docker Deamon specified with the given URL. It can be a
// TCP address or a UNIX socket.
func New(url string) *Docker {
	// the error is returned only when the passed URL is not parsable via
	// url.Parse, so we can safely neglect it
	client, _ := dockerclient.NewDockerClient(url, nil)
	return &Docker{
		client: client,
	}
}

// Build builds a new container image from a public Docker path or froma a
// given Dockerfile
func (d *Docker) Build(r *kite.Request) (interface{}, error) {
	return nil, errors.New("not implemented yet.")
}

// Create creates a new container
func (d *Docker) Create(r *kite.Request) (interface{}, error) {
	return nil, errors.New("not implemented yet.")
}

// Connects connects to an existing Container by spawning a new process and
// attaching to it.
func (d *Docker) Connect(r *kite.Request) (interface{}, error) {
	return nil, errors.New("not implemented yet.")
}

// Stop stops a running container
func (d *Docker) Stop(r *kite.Request) (interface{}, error) {
	return nil, errors.New("not implemented yet.")
}

// Start starts a stopped container
func (d *Docker) Start(r *kite.Request) (interface{}, error) {
	return nil, errors.New("not implemented yet.")
}

// Kill kills and delete a container
func (d *Docker) Kill(r *kite.Request) (interface{}, error) {
	return nil, errors.New("not implemented yet.")
}

// Destroy destroys and removes an image.
func (d *Docker) Destroy(r *kite.Request) (interface{}, error) {
	return nil, errors.New("not implemented yet.")
}

// List lists all available containers
func (d *Docker) List(r *kite.Request) (interface{}, error) {
	return nil, errors.New("not implemented yet.")
}
<|endoftext|>


<commit_before>package docker

import (
	"encoding/json"
	"io"
    "log"
    "os"
    "time"
    "github.com/docker/engine-api/client"
    "github.com/docker/engine-api/types"
    "github.com/docker/engine-api/types/filters"
    eventtypes "github.com/docker/engine-api/types/events"
    "golang.org/x/net/context"
    api "github.com/docking-tools/register/api"
    "github.com/docking-tools/register/config"

)

type DockerRegistry struct {
    docker      *client.Client
    events      <-chan *io.ReadCloser
    config		*config.ConfigFile
}

type DockerServicePort struct {
    api.ServicePort
    ContainerHostname string
	ContainerID       string
	ContainerName     string
	container         *types.ContainerJSON
}

func assert(err error) {
	if err != nil {
		log.Fatal(err)
	}
}

func New(config *config.ConfigFile) (*DockerRegistry, error) {

    // Init docker

   dockerHost:= config.DockerUrl
   if dockerHost == "" {
   	dockerHost= os.Getenv("DOCKER_HOST")
   }
	log.Printf("Start docker client %s", dockerHost)

	// Init client docker
	defaultHeaders := map[string]string{"User-Agent": "engine-api-cli-1.0"}
	docker, err := client.NewClient(dockerHost, "v1.22", nil, defaultHeaders)

   if err != nil {
        panic(err)
    }

    return &DockerRegistry{
        docker:   docker,
        events: nil,
        config: config,
    }, nil
}

func (doc * DockerRegistry) Start(ep api.EventProcessor) {

   // check if an error occur during watch event
   closeChan := make(chan error)

	monitorContainerEvents := func(started chan<- struct{}, c chan<-eventtypes.Message) {
		f := filters.NewArgs()
		f.Add("type", "container")
		options := types.EventsOptions{
			Filters: f,
		}
		resBody, err := doc.docker.Events(context.Background(), options)
		// Whether we successfully subscribed to events or not, we can now
		// unblock the main goroutine.
		close(started)
		log.Printf("Start listenig docker event")
		if err != nil {
			closeChan <- err
			return
		}
		defer resBody.Close()

		// Decode event
		dec := json.NewDecoder(resBody)
		for {
			var event eventtypes.Message
			err := dec.Decode(&event)
			if err != nil && err == io.EOF {
				break
			}
			c <- event

		}
	}

	parseService := func(id string, status string) {
		container, err := doc.docker.ContainerInspect(context.Background(), id)
		if err != nil {
			closeChan <- err
		}
		if (status=="") {
			status =container.State.Status
		}

		services :=  make([]*api.Service,0)
		services, err = createService(doc.config, &container)
		if err != nil {
			closeChan <- err
		}

		for _,service := range services {
			go ep(status, service , closeChan)
		}

	}

	// getContainerList simulates creation event for all previously existing
	// containers.
	getContainerList := func() {
		options := types.ContainerListOptions{
			Quiet: false,
		}
		cs, err := doc.docker.ContainerList(context.Background(), options)
		if err != nil {
			closeChan <- err
		}
		for _, container := range cs {
			parseService(container.ID, container.State)
		}
	}

	eh := eventHandler{handlers: make(map[string]func(eventtypes.Message))}
		eh.Handle("*", func(e eventtypes.Message) {
			log.Printf("new event %v id: %v",e.Status,e.ID)
			parseService(e.ID, e.Status)
		})

	// start listening event
	started := make(chan struct{})
	eventChan := make(chan eventtypes.Message)
	go eh.Watch(eventChan)
	go monitorContainerEvents(started, eventChan)
	defer close(eventChan)
	<-started

	getContainerList ()

   for range time.Tick(500 * time.Millisecond) {
		select {
		case err, ok := <-closeChan:
			if ok {
				if err != nil {
					// this is suppressing "unexpected EOF" in the cli when the
					// daemon restarts so it shutdowns cleanly
					if err != io.ErrUnexpectedEOF {
						closeChan <- err
						log.Printf("Error on run",err)
						close(closeChan)
					}
				}
			}
		default:
			// just skip
		}
   }
   log.Fatal("Docker event loop closed")
}

var Hostname string

func init() {
	// It's ok for Hostname to ultimately be an empty string
	// An empty string will fall back to trying to make a best guess
	Hostname, _ = os.Hostname()
}<commit_msg>bug fix: correct dead container on start before inspect<commit_after>package docker

import (
	"encoding/json"
	"io"
    "log"
    "os"
    "time"
    "github.com/docker/engine-api/client"
    "github.com/docker/engine-api/types"
    "github.com/docker/engine-api/types/filters"
    eventtypes "github.com/docker/engine-api/types/events"
    "golang.org/x/net/context"
    api "github.com/docking-tools/register/api"
    "github.com/docking-tools/register/config"

)

type DockerRegistry struct {
    docker      *client.Client
    events      <-chan *io.ReadCloser
    config		*config.ConfigFile
}

type DockerServicePort struct {
    api.ServicePort
    ContainerHostname string
	ContainerID       string
	ContainerName     string
	container         *types.ContainerJSON
}

func assert(err error) {
	if err != nil {
		log.Fatal(err)
	}
}

func New(config *config.ConfigFile) (*DockerRegistry, error) {

    // Init docker

   dockerHost:= config.DockerUrl
   if dockerHost == "" {
   	dockerHost= os.Getenv("DOCKER_HOST")
   }
	log.Printf("Start docker client %s", dockerHost)

	// Init client docker
	defaultHeaders := map[string]string{"User-Agent": "engine-api-cli-1.0"}
	docker, err := client.NewClient(dockerHost, "v1.22", nil, defaultHeaders)

   if err != nil {
        panic(err)
    }

    return &DockerRegistry{
        docker:   docker,
        events: nil,
        config: config,
    }, nil
}

func (doc * DockerRegistry) Start(ep api.EventProcessor) {

   // check if an error occur during watch event
   closeChan := make(chan error)

	monitorContainerEvents := func(started chan<- struct{}, c chan<-eventtypes.Message) {
		f := filters.NewArgs()
		f.Add("type", "container")
		options := types.EventsOptions{
			Filters: f,
		}
		resBody, err := doc.docker.Events(context.Background(), options)
		// Whether we successfully subscribed to events or not, we can now
		// unblock the main goroutine.
		close(started)
		log.Printf("Start listenig docker event")
		if err != nil {
			closeChan <- err
			return
		}
		defer resBody.Close()

		// Decode event
		dec := json.NewDecoder(resBody)
		for {
			var event eventtypes.Message
			err := dec.Decode(&event)
			if err != nil && err == io.EOF {
				break
			}
			c <- event

		}
	}

	parseService := func(id string, status string) {
		container, err := doc.docker.ContainerInspect(context.Background(), id)
		if err != nil {
			closeChan <- err
		} else {
			if (status=="") {
				status =container.State.Status
			}

			services :=  make([]*api.Service,0)
			services, err = createService(doc.config, &container)
			if err != nil {
				closeChan <- err
			}

			for _,service := range services {
				go ep(status, service , closeChan)
			}
		}
	}

	// getContainerList simulates creation event for all previously existing
	// containers.
	getContainerList := func() {
		options := types.ContainerListOptions{
			Quiet: false,
		}
		cs, err := doc.docker.ContainerList(context.Background(), options)
		if err != nil {
			closeChan <- err
		}
		for _, container := range cs {
			parseService(container.ID, container.State)
		}
	}

	eh := eventHandler{handlers: make(map[string]func(eventtypes.Message))}
		eh.Handle("*", func(e eventtypes.Message) {
			log.Printf("new event %v id: %v",e.Status,e.ID)
			parseService(e.ID, e.Status)
		})

	// start listening event
	started := make(chan struct{})
	eventChan := make(chan eventtypes.Message)
	go eh.Watch(eventChan)
	go monitorContainerEvents(started, eventChan)
	defer close(eventChan)
	<-started

	getContainerList ()

   for range time.Tick(500 * time.Millisecond) {
		select {
		case err, ok := <-closeChan:
			if ok {
				if err != nil {
					// this is suppressing "unexpected EOF" in the cli when the
					// daemon restarts so it shutdowns cleanly
					if err != io.ErrUnexpectedEOF {
						closeChan <- err
						log.Printf("Error on run",err)
						close(closeChan)
					}
				}
			}
		default:
			// just skip
		}
   }
   log.Fatal("Docker event loop closed")
}

var Hostname string

func init() {
	// It's ok for Hostname to ultimately be an empty string
	// An empty string will fall back to trying to make a best guess
	Hostname, _ = os.Hostname()
}<|endoftext|>


<commit_before>package logging

import (
	"bufio"
	"bytes"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"sort"
	"strconv"
	"strings"
	"sync"
	"time"

	hclog "github.com/hashicorp/go-hclog"
)

const (
	// logBufferSize is the size of the buffer.
	logBufferSize = 64 * 1024

	// bufferFlushDuration is the duration at which we flush the buffer.
	bufferFlushDuration = 100 * time.Millisecond

	// lineScanLimit is the number of bytes we will attempt to scan for new
	// lines when approaching the end of the file to avoid a log line being
	// split between two files. Any single line that is greater than this limit
	// may be split.
	lineScanLimit = 32 * 1024

	// newLineDelimiter is the delimiter used for new lines.
	newLineDelimiter = '\n'
)

// FileRotator writes bytes to a rotated set of files
type FileRotator struct {
	MaxFiles int   // MaxFiles is the maximum number of rotated files allowed in a path
	FileSize int64 // FileSize is the size a rotated file is allowed to grow

	path             string // path is the path on the file system where the rotated set of files are opened
	baseFileName     string // baseFileName is the base file name of the rotated files
	logFileIdx       int    // logFileIdx is the current index of the rotated files
	oldestLogFileIdx int    // oldestLogFileIdx is the index of the oldest log file in a path

	currentFile *os.File // currentFile is the file that is currently getting written
	currentWr   int64    // currentWr is the number of bytes written to the current file
	bufw        *bufio.Writer
	bufLock     sync.Mutex

	flushTicker *time.Ticker
	logger      hclog.Logger
	purgeCh     chan struct{}
	doneCh      chan struct{}

	closed     bool
	closedLock sync.Mutex
}

// NewFileRotator returns a new file rotator
func NewFileRotator(path string, baseFile string, maxFiles int,
	fileSize int64, logger hclog.Logger) (*FileRotator, error) {
	logger = logger.Named("rotator")
	rotator := &FileRotator{
		MaxFiles: maxFiles,
		FileSize: fileSize,

		path:         path,
		baseFileName: baseFile,

		flushTicker: time.NewTicker(bufferFlushDuration),
		logger:      logger,
		purgeCh:     make(chan struct{}, 1),
		doneCh:      make(chan struct{}, 1),
	}

	if err := rotator.lastFile(); err != nil {
		return nil, err
	}
	go rotator.purgeOldFiles()
	go rotator.flushPeriodically()
	return rotator, nil
}

// Write writes a byte array to a file and rotates the file if it's size becomes
// equal to the maximum size the user has defined.
func (f *FileRotator) Write(p []byte) (n int, err error) {
	n = 0
	var forceRotate bool

	for n < len(p) {
		// Check if we still have space in the current file, otherwise close and
		// open the next file
		if forceRotate || f.currentWr >= f.FileSize {
			forceRotate = false
			f.flushBuffer()
			f.currentFile.Close()
			if err := f.nextFile(); err != nil {
				f.logger.Error("error creating next file", "err", err)
				return 0, err
			}
		}
		// Calculate the remaining size on this file and how much we have left
		// to write
		remainingSpace := f.FileSize - f.currentWr
		remainingToWrite := int64(len(p[n:]))

		// Check if we are near the end of the file. If we are we attempt to
		// avoid a log line being split between two files.
		var nw int
		if (remainingSpace - lineScanLimit) < remainingToWrite {
			// Scan for new line and if the data up to new line fits in current
			// file, write to buffer
			idx := bytes.IndexByte(p[n:], newLineDelimiter)
			if idx >= 0 && (remainingSpace-int64(idx)-1) >= 0 {
				// We have space so write it to buffer
				nw, err = f.writeToBuffer(p[n : n+idx+1])
			} else if idx >= 0 {
				// We found a new line but don't have space so just force rotate
				forceRotate = true
			} else if remainingToWrite > f.FileSize || f.FileSize-lineScanLimit < 0 {
				// There is no new line remaining but there is no point in
				// rotating since the remaining data will not even fit in the
				// next file either so just fill this one up.
				li := int64(n) + remainingSpace
				if remainingSpace > remainingToWrite {
					li = int64(n) + remainingToWrite
				}
				nw, err = f.writeToBuffer(p[n:li])
			} else {
				// There is no new line in the data remaining for us to write
				// and it will fit in the next file so rotate.
				forceRotate = true
			}
		} else {
			// Write all the bytes in the current file
			nw, err = f.writeToBuffer(p[n:])
		}

		// Increment the number of bytes written so far in this method
		// invocation
		n += nw

		// Increment the total number of bytes in the file
		f.currentWr += int64(n)
		if err != nil {
			f.logger.Error("error writing to file", "err", err)

			// As bufio writer does not automatically recover in case of any
			// io error, we need to recover from it manually resetting the
			// writter.
			f.createOrResetBuffer()

			return
		}
	}
	return
}

// nextFile opens the next file and purges older files if the number of rotated
// files is larger than the maximum files configured by the user
func (f *FileRotator) nextFile() error {
	nextFileIdx := f.logFileIdx
	for {
		nextFileIdx += 1
		logFileName := filepath.Join(f.path, fmt.Sprintf("%s.%d", f.baseFileName, nextFileIdx))
		if fi, err := os.Stat(logFileName); err == nil {
			if fi.IsDir() || fi.Size() >= f.FileSize {
				continue
			}
		}
		f.logFileIdx = nextFileIdx
		if err := f.createFile(); err != nil {
			return err
		}
		break
	}
	// Purge old files if we have more files than MaxFiles
	f.closedLock.Lock()
	defer f.closedLock.Unlock()
	if f.logFileIdx-f.oldestLogFileIdx >= f.MaxFiles && !f.closed {
		select {
		case f.purgeCh <- struct{}{}:
		default:
		}
	}
	return nil
}

// lastFile finds out the rotated file with the largest index in a path.
func (f *FileRotator) lastFile() error {
	finfos, err := ioutil.ReadDir(f.path)
	if err != nil {
		return err
	}

	prefix := fmt.Sprintf("%s.", f.baseFileName)
	for _, fi := range finfos {
		if fi.IsDir() {
			continue
		}
		if strings.HasPrefix(fi.Name(), prefix) {
			fileIdx := strings.TrimPrefix(fi.Name(), prefix)
			n, err := strconv.Atoi(fileIdx)
			if err != nil {
				continue
			}
			if n > f.logFileIdx {
				f.logFileIdx = n
			}
		}
	}
	if err := f.createFile(); err != nil {
		return err
	}
	return nil
}

// createFile opens a new or existing file for writing
func (f *FileRotator) createFile() error {
	logFileName := filepath.Join(f.path, fmt.Sprintf("%s.%d", f.baseFileName, f.logFileIdx))
	cFile, err := os.OpenFile(logFileName, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0644)
	if err != nil {
		return err
	}

	f.currentFile = cFile
	fi, err := f.currentFile.Stat()
	if err != nil {
		return err
	}
	f.currentWr = fi.Size()
	f.createOrResetBuffer()
	return nil
}

// flushPeriodically flushes the buffered writer every 100ms to the underlying
// file
func (f *FileRotator) flushPeriodically() {
	for range f.flushTicker.C {
		f.flushBuffer()
	}
}

// Close flushes and closes the rotator. It never returns an error.
func (f *FileRotator) Close() error {
	f.closedLock.Lock()
	defer f.closedLock.Unlock()

	// Stop the ticker and flush for one last time
	f.flushTicker.Stop()
	f.flushBuffer()

	// Stop the purge go routine
	if !f.closed {
		f.doneCh <- struct{}{}
		close(f.purgeCh)
		f.closed = true
		f.currentFile.Close()
	}

	return nil
}

// purgeOldFiles removes older files and keeps only the last N files rotated for
// a file
func (f *FileRotator) purgeOldFiles() {
	for {
		select {
		case <-f.purgeCh:
			var fIndexes []int
			files, err := ioutil.ReadDir(f.path)
			if err != nil {
				f.logger.Error("error getting directory listing", "err", err)
				return
			}
			// Inserting all the rotated files in a slice
			for _, fi := range files {
				if strings.HasPrefix(fi.Name(), f.baseFileName) {
					fileIdx := strings.TrimPrefix(fi.Name(), fmt.Sprintf("%s.", f.baseFileName))
					n, err := strconv.Atoi(fileIdx)
					if err != nil {
						f.logger.Error("error extracting file index", "err", err)
						continue
					}
					fIndexes = append(fIndexes, n)
				}
			}

			// Not continuing to delete files if the number of files is not more
			// than MaxFiles
			if len(fIndexes) <= f.MaxFiles {
				continue
			}

			// Sorting the file indexes so that we can purge the older files and keep
			// only the number of files as configured by the user
			sort.Ints(fIndexes)
			toDelete := fIndexes[0 : len(fIndexes)-f.MaxFiles]
			for _, fIndex := range toDelete {
				fname := filepath.Join(f.path, fmt.Sprintf("%s.%d", f.baseFileName, fIndex))
				err := os.RemoveAll(fname)
				if err != nil {
					f.logger.Error("error removing file", "filename", fname, "err", err)
				}
			}
			f.oldestLogFileIdx = fIndexes[0]
		case <-f.doneCh:
			return
		}
	}
}

// flushBuffer flushes the buffer
func (f *FileRotator) flushBuffer() error {
	f.bufLock.Lock()
	defer f.bufLock.Unlock()
	if f.bufw != nil {
		return f.bufw.Flush()
	}
	return nil
}

// writeToBuffer writes the byte array to buffer
func (f *FileRotator) writeToBuffer(p []byte) (int, error) {
	f.bufLock.Lock()
	defer f.bufLock.Unlock()
	return f.bufw.Write(p)
}

// createOrResetBuffer creates a new buffer if we don't have one otherwise
// resets the buffer
func (f *FileRotator) createOrResetBuffer() {
	f.bufLock.Lock()
	defer f.bufLock.Unlock()
	if f.bufw == nil {
		f.bufw = bufio.NewWriterSize(f.currentFile, logBufferSize)
	} else {
		f.bufw.Reset(f.currentFile)
	}
}
<commit_msg>logmon: Fix a memory leak on task restart<commit_after>package logging

import (
	"bufio"
	"bytes"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"sort"
	"strconv"
	"strings"
	"sync"
	"time"

	hclog "github.com/hashicorp/go-hclog"
)

const (
	// logBufferSize is the size of the buffer.
	logBufferSize = 64 * 1024

	// bufferFlushDuration is the duration at which we flush the buffer.
	bufferFlushDuration = 100 * time.Millisecond

	// lineScanLimit is the number of bytes we will attempt to scan for new
	// lines when approaching the end of the file to avoid a log line being
	// split between two files. Any single line that is greater than this limit
	// may be split.
	lineScanLimit = 32 * 1024

	// newLineDelimiter is the delimiter used for new lines.
	newLineDelimiter = '\n'
)

// FileRotator writes bytes to a rotated set of files
type FileRotator struct {
	MaxFiles int   // MaxFiles is the maximum number of rotated files allowed in a path
	FileSize int64 // FileSize is the size a rotated file is allowed to grow

	path             string // path is the path on the file system where the rotated set of files are opened
	baseFileName     string // baseFileName is the base file name of the rotated files
	logFileIdx       int    // logFileIdx is the current index of the rotated files
	oldestLogFileIdx int    // oldestLogFileIdx is the index of the oldest log file in a path

	currentFile *os.File // currentFile is the file that is currently getting written
	currentWr   int64    // currentWr is the number of bytes written to the current file
	bufw        *bufio.Writer
	bufLock     sync.Mutex

	flushTicker *time.Ticker
	logger      hclog.Logger
	purgeCh     chan struct{}
	doneCh      chan struct{}

	closed     bool
	closedLock sync.Mutex
}

// NewFileRotator returns a new file rotator
func NewFileRotator(path string, baseFile string, maxFiles int,
	fileSize int64, logger hclog.Logger) (*FileRotator, error) {
	logger = logger.Named("rotator")
	rotator := &FileRotator{
		MaxFiles: maxFiles,
		FileSize: fileSize,

		path:         path,
		baseFileName: baseFile,

		flushTicker: time.NewTicker(bufferFlushDuration),
		logger:      logger,
		purgeCh:     make(chan struct{}, 1),
		doneCh:      make(chan struct{}),
	}

	if err := rotator.lastFile(); err != nil {
		return nil, err
	}
	go rotator.purgeOldFiles()
	go rotator.flushPeriodically()
	return rotator, nil
}

// Write writes a byte array to a file and rotates the file if it's size becomes
// equal to the maximum size the user has defined.
func (f *FileRotator) Write(p []byte) (n int, err error) {
	n = 0
	var forceRotate bool

	for n < len(p) {
		// Check if we still have space in the current file, otherwise close and
		// open the next file
		if forceRotate || f.currentWr >= f.FileSize {
			forceRotate = false
			f.flushBuffer()
			f.currentFile.Close()
			if err := f.nextFile(); err != nil {
				f.logger.Error("error creating next file", "err", err)
				return 0, err
			}
		}
		// Calculate the remaining size on this file and how much we have left
		// to write
		remainingSpace := f.FileSize - f.currentWr
		remainingToWrite := int64(len(p[n:]))

		// Check if we are near the end of the file. If we are we attempt to
		// avoid a log line being split between two files.
		var nw int
		if (remainingSpace - lineScanLimit) < remainingToWrite {
			// Scan for new line and if the data up to new line fits in current
			// file, write to buffer
			idx := bytes.IndexByte(p[n:], newLineDelimiter)
			if idx >= 0 && (remainingSpace-int64(idx)-1) >= 0 {
				// We have space so write it to buffer
				nw, err = f.writeToBuffer(p[n : n+idx+1])
			} else if idx >= 0 {
				// We found a new line but don't have space so just force rotate
				forceRotate = true
			} else if remainingToWrite > f.FileSize || f.FileSize-lineScanLimit < 0 {
				// There is no new line remaining but there is no point in
				// rotating since the remaining data will not even fit in the
				// next file either so just fill this one up.
				li := int64(n) + remainingSpace
				if remainingSpace > remainingToWrite {
					li = int64(n) + remainingToWrite
				}
				nw, err = f.writeToBuffer(p[n:li])
			} else {
				// There is no new line in the data remaining for us to write
				// and it will fit in the next file so rotate.
				forceRotate = true
			}
		} else {
			// Write all the bytes in the current file
			nw, err = f.writeToBuffer(p[n:])
		}

		// Increment the number of bytes written so far in this method
		// invocation
		n += nw

		// Increment the total number of bytes in the file
		f.currentWr += int64(n)
		if err != nil {
			f.logger.Error("error writing to file", "err", err)

			// As bufio writer does not automatically recover in case of any
			// io error, we need to recover from it manually resetting the
			// writter.
			f.createOrResetBuffer()

			return
		}
	}
	return
}

// nextFile opens the next file and purges older files if the number of rotated
// files is larger than the maximum files configured by the user
func (f *FileRotator) nextFile() error {
	nextFileIdx := f.logFileIdx
	for {
		nextFileIdx += 1
		logFileName := filepath.Join(f.path, fmt.Sprintf("%s.%d", f.baseFileName, nextFileIdx))
		if fi, err := os.Stat(logFileName); err == nil {
			if fi.IsDir() || fi.Size() >= f.FileSize {
				continue
			}
		}
		f.logFileIdx = nextFileIdx
		if err := f.createFile(); err != nil {
			return err
		}
		break
	}
	// Purge old files if we have more files than MaxFiles
	f.closedLock.Lock()
	defer f.closedLock.Unlock()
	if f.logFileIdx-f.oldestLogFileIdx >= f.MaxFiles && !f.closed {
		select {
		case f.purgeCh <- struct{}{}:
		default:
		}
	}
	return nil
}

// lastFile finds out the rotated file with the largest index in a path.
func (f *FileRotator) lastFile() error {
	finfos, err := ioutil.ReadDir(f.path)
	if err != nil {
		return err
	}

	prefix := fmt.Sprintf("%s.", f.baseFileName)
	for _, fi := range finfos {
		if fi.IsDir() {
			continue
		}
		if strings.HasPrefix(fi.Name(), prefix) {
			fileIdx := strings.TrimPrefix(fi.Name(), prefix)
			n, err := strconv.Atoi(fileIdx)
			if err != nil {
				continue
			}
			if n > f.logFileIdx {
				f.logFileIdx = n
			}
		}
	}
	if err := f.createFile(); err != nil {
		return err
	}
	return nil
}

// createFile opens a new or existing file for writing
func (f *FileRotator) createFile() error {
	logFileName := filepath.Join(f.path, fmt.Sprintf("%s.%d", f.baseFileName, f.logFileIdx))
	cFile, err := os.OpenFile(logFileName, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0644)
	if err != nil {
		return err
	}

	f.currentFile = cFile
	fi, err := f.currentFile.Stat()
	if err != nil {
		return err
	}
	f.currentWr = fi.Size()
	f.createOrResetBuffer()
	return nil
}

// flushPeriodically flushes the buffered writer every 100ms to the underlying
// file
func (f *FileRotator) flushPeriodically() {
	for {
		select {
		case <-f.flushTicker.C:
			f.flushBuffer()
		case <-f.doneCh:
			return
		}

	}
}

// Close flushes and closes the rotator. It never returns an error.
func (f *FileRotator) Close() error {
	f.closedLock.Lock()
	defer f.closedLock.Unlock()

	// Stop the ticker and flush for one last time
	f.flushTicker.Stop()
	f.flushBuffer()

	// Stop the go routines
	if !f.closed {
		close(f.doneCh)
		close(f.purgeCh)
		f.closed = true
		f.currentFile.Close()
	}

	return nil
}

// purgeOldFiles removes older files and keeps only the last N files rotated for
// a file
func (f *FileRotator) purgeOldFiles() {
	for {
		select {
		case <-f.purgeCh:
			var fIndexes []int
			files, err := ioutil.ReadDir(f.path)
			if err != nil {
				f.logger.Error("error getting directory listing", "err", err)
				return
			}
			// Inserting all the rotated files in a slice
			for _, fi := range files {
				if strings.HasPrefix(fi.Name(), f.baseFileName) {
					fileIdx := strings.TrimPrefix(fi.Name(), fmt.Sprintf("%s.", f.baseFileName))
					n, err := strconv.Atoi(fileIdx)
					if err != nil {
						f.logger.Error("error extracting file index", "err", err)
						continue
					}
					fIndexes = append(fIndexes, n)
				}
			}

			// Not continuing to delete files if the number of files is not more
			// than MaxFiles
			if len(fIndexes) <= f.MaxFiles {
				continue
			}

			// Sorting the file indexes so that we can purge the older files and keep
			// only the number of files as configured by the user
			sort.Ints(fIndexes)
			toDelete := fIndexes[0 : len(fIndexes)-f.MaxFiles]
			for _, fIndex := range toDelete {
				fname := filepath.Join(f.path, fmt.Sprintf("%s.%d", f.baseFileName, fIndex))
				err := os.RemoveAll(fname)
				if err != nil {
					f.logger.Error("error removing file", "filename", fname, "err", err)
				}
			}
			f.oldestLogFileIdx = fIndexes[0]
		case <-f.doneCh:
			return
		}
	}
}

// flushBuffer flushes the buffer
func (f *FileRotator) flushBuffer() error {
	f.bufLock.Lock()
	defer f.bufLock.Unlock()
	if f.bufw != nil {
		return f.bufw.Flush()
	}
	return nil
}

// writeToBuffer writes the byte array to buffer
func (f *FileRotator) writeToBuffer(p []byte) (int, error) {
	f.bufLock.Lock()
	defer f.bufLock.Unlock()
	return f.bufw.Write(p)
}

// createOrResetBuffer creates a new buffer if we don't have one otherwise
// resets the buffer
func (f *FileRotator) createOrResetBuffer() {
	f.bufLock.Lock()
	defer f.bufLock.Unlock()
	if f.bufw == nil {
		f.bufw = bufio.NewWriterSize(f.currentFile, logBufferSize)
	} else {
		f.bufw.Reset(f.currentFile)
	}
}
<|endoftext|>


<commit_before>package repl

import (
	"bufio"
	"fmt"
	"io"

	"github.com/skatsuta/monkey-interpreter/lexer"
	"github.com/skatsuta/monkey-interpreter/token"
)

const prompt = ">> "

// Start starts Monkey REPL.
func Start(in io.Reader, out io.Writer) {
	scanner := bufio.NewScanner(in)

	for {
		fmt.Printf(prompt)
		if !scanner.Scan() {
			return
		}

		line := scanner.Text()
		l := lexer.New(line)

		for tok := l.NextToken(); tok.Type != token.EOF; tok = l.NextToken() {
			fmt.Printf("%+v\n", tok)
		}
	}
}
<commit_msg>repl: make REPL print an AST of input<commit_after>package repl

import (
	"bufio"
	"fmt"
	"io"

	"github.com/skatsuta/monkey-interpreter/lexer"
	"github.com/skatsuta/monkey-interpreter/parser"
)

const prompt = ">> "

// Start starts Monkey REPL.
func Start(in io.Reader, out io.Writer) {
	scanner := bufio.NewScanner(in)

	for {
		fmt.Printf(prompt)
		if !scanner.Scan() {
			return
		}

		line := scanner.Text()
		l := lexer.New(line)
		p := parser.New(l)

		program := p.ParseProgram()
		if len(p.Errors()) != 0 {
			printParserErrors(out, p.Errors())
			continue
		}

		io.WriteString(out, program.String())
		io.WriteString(out, "\n")
	}
}

func printParserErrors(out io.Writer, errors []string) {
	for _, msg := range errors {
		io.WriteString(out, "\t"+msg+"\n")
	}
}
<|endoftext|>


<commit_before>package repo

import (
	"os"
)

func checkAndMake(path string) {
	if _, err := os.Stat("/path/to/whatever"); os.IsNotExist(err) {
		os.MkdirAll(path, os.ModePerm) // TODO: better mode
	}
}
<commit_msg>[repo] fix utils<commit_after>package repo

import (
	"os"
)

func checkAndMake(path string) {
	if _, err := os.Stat(path); os.IsNotExist(err) {
		os.MkdirAll(path, os.ModePerm) // TODO: better mode
	}
}
<|endoftext|>


<commit_before>package executor

import (
	"errors"
	"time"

	"code.cloudfoundry.org/bbs/models"
)

const ContainerOwnerProperty = "executor:owner"

type State string

const (
	StateInvalid      State = ""
	StateReserved     State = "reserved"
	StateInitializing State = "initializing"
	StateCreated      State = "created"
	StateRunning      State = "running"
	StateCompleted    State = "completed"
)

const (
	HealthcheckTag      = "executor-healthcheck"
	HealthcheckTagValue = "executor-healthcheck"
)

type ProxyPortMapping struct {
	AppPort   uint16 `json:"app_port"`
	ProxyPort uint16 `json:"proxy_port"`
}

type Container struct {
	Guid string `json:"guid"`
	Resource
	RunInfo
	Tags                                  Tags
	State                                 State              `json:"state"`
	AllocatedAt                           int64              `json:"allocated_at"`
	ExternalIP                            string             `json:"external_ip"`
	InternalIP                            string             `json:"internal_ip"`
	RunResult                             ContainerRunResult `json:"run_result"`
	MemoryLimit                           uint64             `json:"memory_limit"`
	DiskLimit                             uint64             `json:"disk_limit"`
	AdvertisePreferenceForInstanceAddress bool               `json:"advertise_preference_for_instance_address"`
}

func NewContainerFromResource(guid string, resource *Resource, tags Tags) Container {
	return Container{
		Guid:     guid,
		Resource: *resource,
		Tags:     tags,
	}
}

func (c *Container) ValidateTransitionTo(newState State) bool {
	if newState == StateCompleted {
		return true
	}
	switch c.State {
	case StateReserved:
		return newState == StateInitializing
	case StateInitializing:
		return newState == StateCreated
	case StateCreated:
		return newState == StateRunning
	default:
		return false
	}
}

func (c *Container) TransistionToInitialize(req *RunRequest) error {
	if !c.ValidateTransitionTo(StateInitializing) {
		return ErrInvalidTransition
	}
	c.State = StateInitializing
	c.RunInfo = req.RunInfo
	c.Tags.Add(req.Tags)
	return nil
}

func (c *Container) TransitionToCreate() error {
	if !c.ValidateTransitionTo(StateCreated) {
		return ErrInvalidTransition
	}

	c.State = StateCreated
	return nil
}

func (c *Container) TransitionToComplete(failed bool, failureReason string, retryable bool) {
	c.RunResult.Failed = failed
	c.RunResult.FailureReason = failureReason
	c.RunResult.Retryable = retryable
	c.State = StateCompleted
}

func (newContainer Container) Copy() Container {
	newContainer.Tags = newContainer.Tags.Copy()
	return newContainer
}

func (c *Container) IsCreated() bool {
	return c.State != StateReserved && c.State != StateInitializing && c.State != StateCompleted
}

func (c *Container) HasTags(tags Tags) bool {
	if c.Tags == nil {
		return tags == nil
	}

	if tags == nil {
		return false
	}

	for key, val := range tags {
		v, ok := c.Tags[key]
		if !ok || val != v {
			return false
		}
	}

	return true
}

func NewReservedContainerFromAllocationRequest(req *AllocationRequest, allocatedAt int64) Container {
	c := NewContainerFromResource(req.Guid, &req.Resource, req.Tags)
	c.State = StateReserved
	c.AllocatedAt = allocatedAt
	return c
}

type Resource struct {
	MemoryMB int `json:"memory_mb"`
	DiskMB   int `json:"disk_mb"`
	MaxPids  int `json:"max_pids"`
}

func NewResource(memoryMB, diskMB, maxPids int) Resource {
	return Resource{
		MemoryMB: memoryMB,
		DiskMB:   diskMB,
		MaxPids:  maxPids,
	}
}

type CachedDependency struct {
	Name              string `json:"name"`
	From              string `json:"from"`
	To                string `json:"to"`
	CacheKey          string `json:"cache_key"`
	LogSource         string `json:"log_source"`
	ChecksumValue     string `json:"checksum_value"`
	ChecksumAlgorithm string `json:"checksum_algorithm"`
}

type CertificateProperties struct {
	OrganizationalUnit []string `json:"organizational_unit"`
}

type RunInfo struct {
	RootFSPath                    string                      `json:"rootfs"`
	CPUWeight                     uint                        `json:"cpu_weight"`
	Ports                         []PortMapping               `json:"ports"`
	LogConfig                     LogConfig                   `json:"log_config"`
	MetricsConfig                 MetricsConfig               `json:"metrics_config"`
	StartTimeoutMs                uint                        `json:"start_timeout_ms"`
	Privileged                    bool                        `json:"privileged"`
	CachedDependencies            []CachedDependency          `json:"cached_dependencies"`
	Setup                         *models.Action              `json:"setup"`
	Action                        *models.Action              `json:"run"`
	Monitor                       *models.Action              `json:"monitor"`
	CheckDefinition               *models.CheckDefinition     `json:"check_definition"`
	EgressRules                   []*models.SecurityGroupRule `json:"egress_rules,omitempty"`
	Env                           []EnvironmentVariable       `json:"env,omitempty"`
	TrustedSystemCertificatesPath string                      `json:"trusted_system_certificates_path,omitempty"`
	VolumeMounts                  []VolumeMount               `json:"volume_mounts"`
	Network                       *Network                    `json:"network,omitempty"`
	CertificateProperties         CertificateProperties       `json:"certificate_properties"`
	ImageUsername                 string                      `json:"image_username"`
	ImagePassword                 string                      `json:"image_password"`
	EnableContainerProxy          bool                        `json:"enable_container_proxy"`
}

type BindMountMode uint8

const (
	BindMountModeRO BindMountMode = 0
	BindMountModeRW BindMountMode = 1
)

type VolumeMount struct {
	Driver        string                 `json:"driver"`
	VolumeId      string                 `json:"volume_id"`
	Config        map[string]interface{} `json:"config"`
	ContainerPath string                 `json:"container_path"`
	Mode          BindMountMode          `json:"mode"`
}

type Network struct {
	Properties map[string]string `json:"properties,omitempty"`
}

type InnerContainer Container

type EnvironmentVariable struct {
	Name  string `json:"name"`
	Value string `json:"value"`
}

type ContainerMetrics struct {
	MemoryUsageInBytes                  uint64        `json:"memory_usage_in_bytes"`
	DiskUsageInBytes                    uint64        `json:"disk_usage_in_bytes"`
	MemoryLimitInBytes                  uint64        `json:"memory_limit_in_bytes"`
	DiskLimitInBytes                    uint64        `json:"disk_limit_in_bytes"`
	TimeSpentInCPU                      time.Duration `json:"time_spent_in_cpu"`
	AbsoluteCPUEntitlementInNanoseconds uint64        `json:"absolute_cpu_entitlement_in_ns"`
	ContainerAgeInNanoseconds           uint64        `json:"container_age_in_ns"`
}

type MetricsConfig struct {
	Guid  string            `json:"guid"`
	Index int               `json:"index"`
	Tags  map[string]string `json:"tags"`
}

type Metrics struct {
	MetricsConfig
	ContainerMetrics
}

type LogConfig struct {
	Guid       string            `json:"guid"`
	Index      int               `json:"index"`
	SourceName string            `json:"source_name"`
	Tags       map[string]string `json:"tags"`
}

type PortMapping struct {
	ContainerPort         uint16 `json:"container_port"`
	HostPort              uint16 `json:"host_port,omitempty"`
	ContainerTLSProxyPort uint16 `json:"container_tls_proxy_port,omitempty"`
	HostTLSProxyPort      uint16 `json:"host_tls_proxy_port,omitempty"`
}

type ContainerRunResult struct {
	Failed        bool   `json:"failed"`
	FailureReason string `json:"failure_reason"`
	Retryable     bool

	Stopped bool `json:"stopped"`
}

type ExecutorResources struct {
	MemoryMB   int `json:"memory_mb"`
	DiskMB     int `json:"disk_mb"`
	Containers int `json:"containers"`
}

func NewExecutorResources(memoryMB, diskMB, containers int) ExecutorResources {
	return ExecutorResources{
		MemoryMB:   memoryMB,
		DiskMB:     diskMB,
		Containers: containers,
	}
}

func (e ExecutorResources) Copy() ExecutorResources {
	return e
}

func (r *ExecutorResources) canSubtract(res *Resource) bool {
	return r.MemoryMB >= res.MemoryMB && r.DiskMB >= res.DiskMB && r.Containers > 0
}

func (r *ExecutorResources) Subtract(res *Resource) bool {
	if !r.canSubtract(res) {
		return false
	}
	r.MemoryMB -= res.MemoryMB
	r.DiskMB -= res.DiskMB
	r.Containers -= 1
	return true
}

func (r *ExecutorResources) Add(res *Resource) {
	r.MemoryMB += res.MemoryMB
	r.DiskMB += res.DiskMB
	r.Containers += 1
}

type Tags map[string]string

func (t Tags) Copy() Tags {
	if t == nil {
		return nil
	}
	newTags := make(Tags, len(t))
	newTags.Add(t)
	return newTags
}

func (t Tags) Add(other Tags) {
	for key := range other {
		t[key] = other[key]
	}
}

type Event interface {
	EventType() EventType
}

type EventType string

var ErrUnknownEventType = errors.New("unknown event type")

const (
	EventTypeInvalid EventType = ""

	EventTypeContainerComplete EventType = "container_complete"
	EventTypeContainerRunning  EventType = "container_running"
	EventTypeContainerReserved EventType = "container_reserved"
)

type LifecycleEvent interface {
	Container() Container
	lifecycleEvent()
}

type ContainerCompleteEvent struct {
	RawContainer Container `json:"container"`
}

func NewContainerCompleteEvent(container Container) ContainerCompleteEvent {
	return ContainerCompleteEvent{
		RawContainer: container,
	}
}

func (ContainerCompleteEvent) EventType() EventType   { return EventTypeContainerComplete }
func (e ContainerCompleteEvent) Container() Container { return e.RawContainer }
func (ContainerCompleteEvent) lifecycleEvent()        {}

type ContainerRunningEvent struct {
	RawContainer Container `json:"container"`
}

func NewContainerRunningEvent(container Container) ContainerRunningEvent {
	return ContainerRunningEvent{
		RawContainer: container,
	}
}

func (ContainerRunningEvent) EventType() EventType   { return EventTypeContainerRunning }
func (e ContainerRunningEvent) Container() Container { return e.RawContainer }
func (ContainerRunningEvent) lifecycleEvent()        {}

type ContainerReservedEvent struct {
	RawContainer Container `json:"container"`
}

func NewContainerReservedEvent(container Container) ContainerReservedEvent {
	return ContainerReservedEvent{
		RawContainer: container,
	}
}

func (ContainerReservedEvent) EventType() EventType   { return EventTypeContainerReserved }
func (e ContainerReservedEvent) Container() Container { return e.RawContainer }
func (ContainerReservedEvent) lifecycleEvent()        {}
<commit_msg>add Sidecars to RunInfo<commit_after>package executor

import (
	"errors"
	"time"

	"code.cloudfoundry.org/bbs/models"
)

const ContainerOwnerProperty = "executor:owner"

type State string

const (
	StateInvalid      State = ""
	StateReserved     State = "reserved"
	StateInitializing State = "initializing"
	StateCreated      State = "created"
	StateRunning      State = "running"
	StateCompleted    State = "completed"
)

const (
	HealthcheckTag      = "executor-healthcheck"
	HealthcheckTagValue = "executor-healthcheck"
)

type ProxyPortMapping struct {
	AppPort   uint16 `json:"app_port"`
	ProxyPort uint16 `json:"proxy_port"`
}

type Container struct {
	Guid string `json:"guid"`
	Resource
	RunInfo
	Tags                                  Tags
	State                                 State              `json:"state"`
	AllocatedAt                           int64              `json:"allocated_at"`
	ExternalIP                            string             `json:"external_ip"`
	InternalIP                            string             `json:"internal_ip"`
	RunResult                             ContainerRunResult `json:"run_result"`
	MemoryLimit                           uint64             `json:"memory_limit"`
	DiskLimit                             uint64             `json:"disk_limit"`
	AdvertisePreferenceForInstanceAddress bool               `json:"advertise_preference_for_instance_address"`
}

func NewContainerFromResource(guid string, resource *Resource, tags Tags) Container {
	return Container{
		Guid:     guid,
		Resource: *resource,
		Tags:     tags,
	}
}

func (c *Container) ValidateTransitionTo(newState State) bool {
	if newState == StateCompleted {
		return true
	}
	switch c.State {
	case StateReserved:
		return newState == StateInitializing
	case StateInitializing:
		return newState == StateCreated
	case StateCreated:
		return newState == StateRunning
	default:
		return false
	}
}

func (c *Container) TransistionToInitialize(req *RunRequest) error {
	if !c.ValidateTransitionTo(StateInitializing) {
		return ErrInvalidTransition
	}
	c.State = StateInitializing
	c.RunInfo = req.RunInfo
	c.Tags.Add(req.Tags)
	return nil
}

func (c *Container) TransitionToCreate() error {
	if !c.ValidateTransitionTo(StateCreated) {
		return ErrInvalidTransition
	}

	c.State = StateCreated
	return nil
}

func (c *Container) TransitionToComplete(failed bool, failureReason string, retryable bool) {
	c.RunResult.Failed = failed
	c.RunResult.FailureReason = failureReason
	c.RunResult.Retryable = retryable
	c.State = StateCompleted
}

func (newContainer Container) Copy() Container {
	newContainer.Tags = newContainer.Tags.Copy()
	return newContainer
}

func (c *Container) IsCreated() bool {
	return c.State != StateReserved && c.State != StateInitializing && c.State != StateCompleted
}

func (c *Container) HasTags(tags Tags) bool {
	if c.Tags == nil {
		return tags == nil
	}

	if tags == nil {
		return false
	}

	for key, val := range tags {
		v, ok := c.Tags[key]
		if !ok || val != v {
			return false
		}
	}

	return true
}

func NewReservedContainerFromAllocationRequest(req *AllocationRequest, allocatedAt int64) Container {
	c := NewContainerFromResource(req.Guid, &req.Resource, req.Tags)
	c.State = StateReserved
	c.AllocatedAt = allocatedAt
	return c
}

type Resource struct {
	MemoryMB int `json:"memory_mb"`
	DiskMB   int `json:"disk_mb"`
	MaxPids  int `json:"max_pids"`
}

func NewResource(memoryMB, diskMB, maxPids int) Resource {
	return Resource{
		MemoryMB: memoryMB,
		DiskMB:   diskMB,
		MaxPids:  maxPids,
	}
}

type CachedDependency struct {
	Name              string `json:"name"`
	From              string `json:"from"`
	To                string `json:"to"`
	CacheKey          string `json:"cache_key"`
	LogSource         string `json:"log_source"`
	ChecksumValue     string `json:"checksum_value"`
	ChecksumAlgorithm string `json:"checksum_algorithm"`
}

type CertificateProperties struct {
	OrganizationalUnit []string `json:"organizational_unit"`
}

type Sidecar struct {
	Action   *models.Action `json:"run"`
	DiskMB   int32          `json:"disk_mb"`
	MemoryMB int32          `json:"memory_mb"`
}

type RunInfo struct {
	RootFSPath                    string                      `json:"rootfs"`
	CPUWeight                     uint                        `json:"cpu_weight"`
	Ports                         []PortMapping               `json:"ports"`
	LogConfig                     LogConfig                   `json:"log_config"`
	MetricsConfig                 MetricsConfig               `json:"metrics_config"`
	StartTimeoutMs                uint                        `json:"start_timeout_ms"`
	Privileged                    bool                        `json:"privileged"`
	CachedDependencies            []CachedDependency          `json:"cached_dependencies"`
	Setup                         *models.Action              `json:"setup"`
	Action                        *models.Action              `json:"run"`
	Monitor                       *models.Action              `json:"monitor"`
	CheckDefinition               *models.CheckDefinition     `json:"check_definition"`
	EgressRules                   []*models.SecurityGroupRule `json:"egress_rules,omitempty"`
	Env                           []EnvironmentVariable       `json:"env,omitempty"`
	TrustedSystemCertificatesPath string                      `json:"trusted_system_certificates_path,omitempty"`
	VolumeMounts                  []VolumeMount               `json:"volume_mounts"`
	Network                       *Network                    `json:"network,omitempty"`
	CertificateProperties         CertificateProperties       `json:"certificate_properties"`
	ImageUsername                 string                      `json:"image_username"`
	ImagePassword                 string                      `json:"image_password"`
	EnableContainerProxy          bool                        `json:"enable_container_proxy"`
	Sidecars                      []Sidecar                   `json:"sidecars"`
}

type BindMountMode uint8

const (
	BindMountModeRO BindMountMode = 0
	BindMountModeRW BindMountMode = 1
)

type VolumeMount struct {
	Driver        string                 `json:"driver"`
	VolumeId      string                 `json:"volume_id"`
	Config        map[string]interface{} `json:"config"`
	ContainerPath string                 `json:"container_path"`
	Mode          BindMountMode          `json:"mode"`
}

type Network struct {
	Properties map[string]string `json:"properties,omitempty"`
}

type InnerContainer Container

type EnvironmentVariable struct {
	Name  string `json:"name"`
	Value string `json:"value"`
}

type ContainerMetrics struct {
	MemoryUsageInBytes                  uint64        `json:"memory_usage_in_bytes"`
	DiskUsageInBytes                    uint64        `json:"disk_usage_in_bytes"`
	MemoryLimitInBytes                  uint64        `json:"memory_limit_in_bytes"`
	DiskLimitInBytes                    uint64        `json:"disk_limit_in_bytes"`
	TimeSpentInCPU                      time.Duration `json:"time_spent_in_cpu"`
	AbsoluteCPUEntitlementInNanoseconds uint64        `json:"absolute_cpu_entitlement_in_ns"`
	ContainerAgeInNanoseconds           uint64        `json:"container_age_in_ns"`
}

type MetricsConfig struct {
	Guid  string            `json:"guid"`
	Index int               `json:"index"`
	Tags  map[string]string `json:"tags"`
}

type Metrics struct {
	MetricsConfig
	ContainerMetrics
}

type LogConfig struct {
	Guid       string            `json:"guid"`
	Index      int               `json:"index"`
	SourceName string            `json:"source_name"`
	Tags       map[string]string `json:"tags"`
}

type PortMapping struct {
	ContainerPort         uint16 `json:"container_port"`
	HostPort              uint16 `json:"host_port,omitempty"`
	ContainerTLSProxyPort uint16 `json:"container_tls_proxy_port,omitempty"`
	HostTLSProxyPort      uint16 `json:"host_tls_proxy_port,omitempty"`
}

type ContainerRunResult struct {
	Failed        bool   `json:"failed"`
	FailureReason string `json:"failure_reason"`
	Retryable     bool

	Stopped bool `json:"stopped"`
}

type ExecutorResources struct {
	MemoryMB   int `json:"memory_mb"`
	DiskMB     int `json:"disk_mb"`
	Containers int `json:"containers"`
}

func NewExecutorResources(memoryMB, diskMB, containers int) ExecutorResources {
	return ExecutorResources{
		MemoryMB:   memoryMB,
		DiskMB:     diskMB,
		Containers: containers,
	}
}

func (e ExecutorResources) Copy() ExecutorResources {
	return e
}

func (r *ExecutorResources) canSubtract(res *Resource) bool {
	return r.MemoryMB >= res.MemoryMB && r.DiskMB >= res.DiskMB && r.Containers > 0
}

func (r *ExecutorResources) Subtract(res *Resource) bool {
	if !r.canSubtract(res) {
		return false
	}
	r.MemoryMB -= res.MemoryMB
	r.DiskMB -= res.DiskMB
	r.Containers -= 1
	return true
}

func (r *ExecutorResources) Add(res *Resource) {
	r.MemoryMB += res.MemoryMB
	r.DiskMB += res.DiskMB
	r.Containers += 1
}

type Tags map[string]string

func (t Tags) Copy() Tags {
	if t == nil {
		return nil
	}
	newTags := make(Tags, len(t))
	newTags.Add(t)
	return newTags
}

func (t Tags) Add(other Tags) {
	for key := range other {
		t[key] = other[key]
	}
}

type Event interface {
	EventType() EventType
}

type EventType string

var ErrUnknownEventType = errors.New("unknown event type")

const (
	EventTypeInvalid EventType = ""

	EventTypeContainerComplete EventType = "container_complete"
	EventTypeContainerRunning  EventType = "container_running"
	EventTypeContainerReserved EventType = "container_reserved"
)

type LifecycleEvent interface {
	Container() Container
	lifecycleEvent()
}

type ContainerCompleteEvent struct {
	RawContainer Container `json:"container"`
}

func NewContainerCompleteEvent(container Container) ContainerCompleteEvent {
	return ContainerCompleteEvent{
		RawContainer: container,
	}
}

func (ContainerCompleteEvent) EventType() EventType   { return EventTypeContainerComplete }
func (e ContainerCompleteEvent) Container() Container { return e.RawContainer }
func (ContainerCompleteEvent) lifecycleEvent()        {}

type ContainerRunningEvent struct {
	RawContainer Container `json:"container"`
}

func NewContainerRunningEvent(container Container) ContainerRunningEvent {
	return ContainerRunningEvent{
		RawContainer: container,
	}
}

func (ContainerRunningEvent) EventType() EventType   { return EventTypeContainerRunning }
func (e ContainerRunningEvent) Container() Container { return e.RawContainer }
func (ContainerRunningEvent) lifecycleEvent()        {}

type ContainerReservedEvent struct {
	RawContainer Container `json:"container"`
}

func NewContainerReservedEvent(container Container) ContainerReservedEvent {
	return ContainerReservedEvent{
		RawContainer: container,
	}
}

func (ContainerReservedEvent) EventType() EventType   { return EventTypeContainerReserved }
func (e ContainerReservedEvent) Container() Container { return e.RawContainer }
func (ContainerReservedEvent) lifecycleEvent()        {}
<|endoftext|>


<commit_before>package rest

import (
	"fmt"
	"log"
	"reflect"
)

// TODO:
//  - Consider using/writing a more robust data validation library.
//    E.g. https://github.com/Assembli/beautiful-validity
//    This would allow for semantic validation and custom validation logic.
//    For now, we are only providing type validation.
//
//  - Make type coercion pluggable (i.e. conversion/validation of custom types).

// Rules is a slice of Rule pointers.
type Rules []*Rule

// Rule provides schema validation and type coercion for request input and fine-grained
// control over response output. If a ResourceHandler provides input Rules which
// specify types, input fields will attempt to be coerced to those types. If coercion
// fails, an error will be returned in the response. If a ResourceHandler provides
// output Rules, only the fields corresponding to those Rules will be sent back. This
// prevents new fields from leaking into old API versions.
type Rule struct {
	// Name of the resource field. This is the name as it appears in the struct
	// definition.
	Field string

	// Name of the input/output field. Use Name() to retrieve the field alias while
	// falling back to the field name if it's not specified.
	FieldAlias string

	// Type to coerce field value to. If the value cannot be coerced, an error will be
	// returned in the response. Defaults to Unspecified, which is the equivalent of
	// an interface{} value.
	Type Type

	// Indicates if the field must have a value. Defaults to false.
	Required bool

	// Versions is a list of the API versions this Rule applies to. If empty, it will
	// be applied to all versions.
	Versions []string

	// Indicates if the Rule should only be applied to requests.
	InputOnly bool

	// Indicates if the Rule should only be applied to responses.
	OutputOnly bool

	// Function which produces the field value to receive.
	InputHandler func(interface{}) interface{}

	// Function which produces the field value to send.
	OutputHandler func(interface{}) interface{}

	// resourceType is the reflect.Type of the resource this Rule applies to. Set
	// by the framework.
	resourceType reflect.Type
}

// Name returns the name of the input/output field alias. It defaults to the field
// name if the alias was not specified.
func (r Rule) Name() string {
	alias := r.FieldAlias
	if alias == "" {
		alias = r.Field
	}
	return alias
}

// Applies returns whether or not the Rule applies to the given version.
func (r Rule) Applies(version string) bool {
	if r.Versions == nil {
		return true
	}

	for _, v := range r.Versions {
		if v == version {
			return true
		}
	}

	return false
}

// validType returns whether or not the Rule is valid for the given reflect.Type.
func (r Rule) validType(fieldType reflect.Type) bool {
	if r.Type == Unspecified {
		return true
	}

	kind := typeToKind[r.Type]
	return fieldType.Kind() == kind
}

// validate verifies that the Rules are valid for the given reflect.Type, meaning
// they specify fields that exist and correct types. If a Rule is invalid, an
// error is returned. If the Rules are valid, nil is returned.
func (r Rules) validate() error {
	for _, rule := range r {
		resourceType := rule.resourceType
		if resourceType.Kind() != reflect.Struct && resourceType.Kind() != reflect.Map {
			return fmt.Errorf(fmt.Sprintf(
				"Invalid resource type: must be struct or map, got %s", resourceType))
		}

		field, found := resourceType.FieldByName(rule.Field)
		if !found {
			return fmt.Errorf(fmt.Sprintf(
				"Invalid Rule for %s: field '%s' does not exist",
				resourceType, rule.Field))
		}

		if !rule.validType(field.Type) {
			return fmt.Errorf(fmt.Sprintf(
				"Invalid Rule for %s: field '%s' is type %s, not %s",
				resourceType, rule.Field, field.Type, typeToName[rule.Type]))
		}
	}

	return nil
}

// applyInboundRules applies Rules which are not specified as output only to the
// provided Payload. If the Payload is nil, an empty Payload will be returned. If no
// Rules are provided, this acts as an identity function. If Rules are provided, any
// incoming fields which are not specified will be discarded. If Rules specify types,
// incoming values will attempted to be coerced. If coercion fails, an error will be
// returned.
func applyInboundRules(payload Payload, rules Rules) (Payload, error) {
	if payload == nil {
		return Payload{}, nil
	}

	// Apply only inbound Rules.
	rules = filterRules(rules, true)

	if len(rules) == 0 {
		return payload, nil
	}

	newPayload := Payload{}

fieldLoop:
	for field, value := range payload {
		for _, rule := range rules {
			if rule.FieldAlias == field {
				if rule.Type != Unspecified {
					// Coerce to specified type.
					coerced, err := coerceType(value, rule.Type)
					if err != nil {
						return nil, err
					}
					value = coerced
				}
				if rule.InputHandler != nil {
					value = rule.InputHandler(value)
				}
				newPayload[field] = value
				continue fieldLoop
			}
		}

		log.Printf("Discarding field '%s'", field)
	}

	// Ensure no required fields are missing.
	if err := enforceRequiredFields(rules, newPayload); err != nil {
		log.Println(err)
		return nil, err
	}

	return newPayload, nil
}

// applyOutboundRules applies Rules which are not specified as input only to the
// provided Resource. If the Resource is nil, not a struct or
// map[string]interface{}, or no Rules are provided, this acts as an identity
// function. If Rules are provided, only the fields specified by them will be
// included in the returned Resource. This is to prevent new fields from leaking
// into old API versions.
func applyOutboundRules(resource Resource, rules Rules) Resource {
	// Apply only outbound Rules.
	rules = filterRules(rules, false)

	if resource == nil || len(rules) == 0 {
		// Return resource as-is if no Rules are provided.
		return resource
	}

	// Get the underlying value by dereferencing the pointer if there is one.
	resourceValue := reflect.Indirect(reflect.ValueOf(resource))
	resource = resourceValue.Interface()
	resourceType := reflect.TypeOf(resource)
	var payload Resource

	if resourceType.Kind() == reflect.Map {
		if resourceMap, ok := resource.(map[string]interface{}); ok {
			payload = applyOutboundRulesForMap(resourceMap, rules)
		} else {
			// Nothing we can do if the keys aren't strings.
			payload = resource
		}
	} else if resourceType == rules[0].resourceType {
		payload = applyOutboundRulesForStruct(resourceValue, rules)
	} else {
		// Only apply Rules to resource structs and maps.
		payload = resource
	}

	return payload
}

func applyOutboundRulesForMap(resource map[string]interface{}, rules Rules) Payload {
	payload := Payload{}
	for _, rule := range rules {
		fieldValue, ok := resource[rule.Field]
		if !ok {
			log.Printf("Map resource missing field '%s'", rule.Field)
			continue
		}
		if rule.OutputHandler != nil {
			fieldValue = rule.OutputHandler(fieldValue)
		}
		payload[rule.Name()] = fieldValue
	}

	return payload
}

func applyOutboundRulesForStruct(resourceValue reflect.Value, rules Rules) Payload {
	payload := Payload{}
	for _, rule := range rules {
		// Rule validation occurs at server start. No need to check for field existence.
		field := resourceValue.FieldByName(rule.Field)
		fieldValue := field.Interface()
		if rule.OutputHandler != nil {
			fieldValue = rule.OutputHandler(fieldValue)
		}
		payload[rule.Name()] = fieldValue
	}

	return payload
}

// filterRules filters the slice of Rules based on the specified bool. True means to
// filter out outbound Rules such that the returned slice contains only inbound Rules.
// False means to filter out inbound Rules such that the returned slice contains only
// outbound Rules.
func filterRules(rules Rules, inbound bool) Rules {
	filtered := make(Rules, 0, len(rules))
	for _, rule := range rules {
		if inbound && rule.OutputOnly {
			// Filter out outbound Rules.
			continue
		} else if !inbound && rule.InputOnly {
			// Filter out inbound Rules.
			continue
		}
		filtered = append(filtered, rule)
	}

	return filtered
}

// enforceRequiredFields verifies that the provided Payload has values for any Rules
// with the Required flag set to true. If any required fields are missing, an error
// will be returned. Otherwise nil is returned.
func enforceRequiredFields(rules Rules, payload Payload) error {
ruleLoop:
	for _, rule := range rules {
		if !rule.Required {
			continue
		}

		for field := range payload {
			if rule.Name() == field {
				continue ruleLoop
			}
		}

		return fmt.Errorf("Missing required field '%s'", rule.Name())
	}

	return nil
}
<commit_msg>Remove unnecessary Sprintf<commit_after>package rest

import (
	"fmt"
	"log"
	"reflect"
)

// TODO:
//  - Consider using/writing a more robust data validation library.
//    E.g. https://github.com/Assembli/beautiful-validity
//    This would allow for semantic validation and custom validation logic.
//    For now, we are only providing type validation.
//
//  - Make type coercion pluggable (i.e. conversion/validation of custom types).

// Rules is a slice of Rule pointers.
type Rules []*Rule

// Rule provides schema validation and type coercion for request input and fine-grained
// control over response output. If a ResourceHandler provides input Rules which
// specify types, input fields will attempt to be coerced to those types. If coercion
// fails, an error will be returned in the response. If a ResourceHandler provides
// output Rules, only the fields corresponding to those Rules will be sent back. This
// prevents new fields from leaking into old API versions.
type Rule struct {
	// Name of the resource field. This is the name as it appears in the struct
	// definition.
	Field string

	// Name of the input/output field. Use Name() to retrieve the field alias while
	// falling back to the field name if it's not specified.
	FieldAlias string

	// Type to coerce field value to. If the value cannot be coerced, an error will be
	// returned in the response. Defaults to Unspecified, which is the equivalent of
	// an interface{} value.
	Type Type

	// Indicates if the field must have a value. Defaults to false.
	Required bool

	// Versions is a list of the API versions this Rule applies to. If empty, it will
	// be applied to all versions.
	Versions []string

	// Indicates if the Rule should only be applied to requests.
	InputOnly bool

	// Indicates if the Rule should only be applied to responses.
	OutputOnly bool

	// Function which produces the field value to receive.
	InputHandler func(interface{}) interface{}

	// Function which produces the field value to send.
	OutputHandler func(interface{}) interface{}

	// resourceType is the reflect.Type of the resource this Rule applies to. Set
	// by the framework.
	resourceType reflect.Type
}

// Name returns the name of the input/output field alias. It defaults to the field
// name if the alias was not specified.
func (r Rule) Name() string {
	alias := r.FieldAlias
	if alias == "" {
		alias = r.Field
	}
	return alias
}

// Applies returns whether or not the Rule applies to the given version.
func (r Rule) Applies(version string) bool {
	if r.Versions == nil {
		return true
	}

	for _, v := range r.Versions {
		if v == version {
			return true
		}
	}

	return false
}

// validType returns whether or not the Rule is valid for the given reflect.Type.
func (r Rule) validType(fieldType reflect.Type) bool {
	if r.Type == Unspecified {
		return true
	}

	kind := typeToKind[r.Type]
	return fieldType.Kind() == kind
}

// validate verifies that the Rules are valid for the given reflect.Type, meaning
// they specify fields that exist and correct types. If a Rule is invalid, an
// error is returned. If the Rules are valid, nil is returned.
func (r Rules) validate() error {
	for _, rule := range r {
		resourceType := rule.resourceType
		if resourceType.Kind() != reflect.Struct && resourceType.Kind() != reflect.Map {
			return fmt.Errorf(
				"Invalid resource type: must be struct or map, got %s",
				resourceType)
		}

		field, found := resourceType.FieldByName(rule.Field)
		if !found {
			return fmt.Errorf(
				"Invalid Rule for %s: field '%s' does not exist",
				resourceType, rule.Field)
		}

		if !rule.validType(field.Type) {
			return fmt.Errorf(
				"Invalid Rule for %s: field '%s' is type %s, not %s",
				resourceType, rule.Field, field.Type, typeToName[rule.Type])
		}
	}

	return nil
}

// applyInboundRules applies Rules which are not specified as output only to the
// provided Payload. If the Payload is nil, an empty Payload will be returned. If no
// Rules are provided, this acts as an identity function. If Rules are provided, any
// incoming fields which are not specified will be discarded. If Rules specify types,
// incoming values will attempted to be coerced. If coercion fails, an error will be
// returned.
func applyInboundRules(payload Payload, rules Rules) (Payload, error) {
	if payload == nil {
		return Payload{}, nil
	}

	// Apply only inbound Rules.
	rules = filterRules(rules, true)

	if len(rules) == 0 {
		return payload, nil
	}

	newPayload := Payload{}

fieldLoop:
	for field, value := range payload {
		for _, rule := range rules {
			if rule.FieldAlias == field {
				if rule.Type != Unspecified {
					// Coerce to specified type.
					coerced, err := coerceType(value, rule.Type)
					if err != nil {
						return nil, err
					}
					value = coerced
				}
				if rule.InputHandler != nil {
					value = rule.InputHandler(value)
				}
				newPayload[field] = value
				continue fieldLoop
			}
		}

		log.Printf("Discarding field '%s'", field)
	}

	// Ensure no required fields are missing.
	if err := enforceRequiredFields(rules, newPayload); err != nil {
		log.Println(err)
		return nil, err
	}

	return newPayload, nil
}

// applyOutboundRules applies Rules which are not specified as input only to the
// provided Resource. If the Resource is nil, not a struct or
// map[string]interface{}, or no Rules are provided, this acts as an identity
// function. If Rules are provided, only the fields specified by them will be
// included in the returned Resource. This is to prevent new fields from leaking
// into old API versions.
func applyOutboundRules(resource Resource, rules Rules) Resource {
	// Apply only outbound Rules.
	rules = filterRules(rules, false)

	if resource == nil || len(rules) == 0 {
		// Return resource as-is if no Rules are provided.
		return resource
	}

	// Get the underlying value by dereferencing the pointer if there is one.
	resourceValue := reflect.Indirect(reflect.ValueOf(resource))
	resource = resourceValue.Interface()
	resourceType := reflect.TypeOf(resource)
	var payload Resource

	if resourceType.Kind() == reflect.Map {
		if resourceMap, ok := resource.(map[string]interface{}); ok {
			payload = applyOutboundRulesForMap(resourceMap, rules)
		} else {
			// Nothing we can do if the keys aren't strings.
			payload = resource
		}
	} else if resourceType == rules[0].resourceType {
		payload = applyOutboundRulesForStruct(resourceValue, rules)
	} else {
		// Only apply Rules to resource structs and maps.
		payload = resource
	}

	return payload
}

func applyOutboundRulesForMap(resource map[string]interface{}, rules Rules) Payload {
	payload := Payload{}
	for _, rule := range rules {
		fieldValue, ok := resource[rule.Field]
		if !ok {
			log.Printf("Map resource missing field '%s'", rule.Field)
			continue
		}
		if rule.OutputHandler != nil {
			fieldValue = rule.OutputHandler(fieldValue)
		}
		payload[rule.Name()] = fieldValue
	}

	return payload
}

func applyOutboundRulesForStruct(resourceValue reflect.Value, rules Rules) Payload {
	payload := Payload{}
	for _, rule := range rules {
		// Rule validation occurs at server start. No need to check for field existence.
		field := resourceValue.FieldByName(rule.Field)
		fieldValue := field.Interface()
		if rule.OutputHandler != nil {
			fieldValue = rule.OutputHandler(fieldValue)
		}
		payload[rule.Name()] = fieldValue
	}

	return payload
}

// filterRules filters the slice of Rules based on the specified bool. True means to
// filter out outbound Rules such that the returned slice contains only inbound Rules.
// False means to filter out inbound Rules such that the returned slice contains only
// outbound Rules.
func filterRules(rules Rules, inbound bool) Rules {
	filtered := make(Rules, 0, len(rules))
	for _, rule := range rules {
		if inbound && rule.OutputOnly {
			// Filter out outbound Rules.
			continue
		} else if !inbound && rule.InputOnly {
			// Filter out inbound Rules.
			continue
		}
		filtered = append(filtered, rule)
	}

	return filtered
}

// enforceRequiredFields verifies that the provided Payload has values for any Rules
// with the Required flag set to true. If any required fields are missing, an error
// will be returned. Otherwise nil is returned.
func enforceRequiredFields(rules Rules, payload Payload) error {
ruleLoop:
	for _, rule := range rules {
		if !rule.Required {
			continue
		}

		for field := range payload {
			if rule.Name() == field {
				continue ruleLoop
			}
		}

		return fmt.Errorf("Missing required field '%s'", rule.Name())
	}

	return nil
}
<|endoftext|>


<commit_before>package spider

import (
	"github.com/henrylee2cn/pholcus/common/pinyin"
)

// 蜘蛛种类列表
type SpiderSpecies struct {
	list   []*Spider
	sorted bool
}

// 全局蜘蛛种类实例
var Species = &SpiderSpecies{
	list: []*Spider{},
}

// 向蜘蛛种类清单添加新种类
func (self *SpiderSpecies) Add(sp *Spider) *Spider {
	self.list = append(self.list, sp)
	return sp
}

// 获取全部蜘蛛种类
func (self *SpiderSpecies) Get() []*Spider {
	if !self.sorted {
		l := len(self.list)
		initials := make([]string, l)
		newlist := map[string]*Spider{}
		for i := 0; i < l; i++ {
			initials[i] = self.list[i].GetName()
			newlist[initials[i]] = self.list[i]
		}
		pinyin.SortInitials(initials)
		for i := 0; i < l; i++ {
			self.list[i] = newlist[initials[i]]
		}
		self.sorted = true
	}
	return self.list
}

func (self *SpiderSpecies) GetByName(n string) *Spider {
	for _, sp := range self.list {
		if sp.GetName() == n {
			return sp
		}
	}
	return nil
}
<commit_msg>同名规则自动加"(2)"形式的后缀<commit_after>package spider

import (
	"fmt"

	"github.com/henrylee2cn/pholcus/common/pinyin"
)

// 蜘蛛种类列表
type SpiderSpecies struct {
	list   []*Spider
	hash   map[string]*Spider
	sorted bool
}

// 全局蜘蛛种类实例
var Species = &SpiderSpecies{
	list: []*Spider{},
	hash: map[string]*Spider{},
}

// 向蜘蛛种类清单添加新种类
func (self *SpiderSpecies) Add(sp *Spider) *Spider {
	for i, name := 2, sp.Name; true; i++ {
		if _, ok := self.hash[name]; !ok {
			sp.Name = name
			self.hash[sp.Name] = sp
			break
		}
		name = fmt.Sprintf("%s(%d)", sp.Name, i)
	}
	self.list = append(self.list, sp)
	return sp
}

// 获取全部蜘蛛种类
func (self *SpiderSpecies) Get() []*Spider {
	if !self.sorted {
		l := len(self.list)
		initials := make([]string, l)
		newlist := map[string]*Spider{}
		for i := 0; i < l; i++ {
			initials[i] = self.list[i].GetName()
			newlist[initials[i]] = self.list[i]
		}
		pinyin.SortInitials(initials)
		for i := 0; i < l; i++ {
			self.list[i] = newlist[initials[i]]
		}
		self.sorted = true
	}
	return self.list
}

func (self *SpiderSpecies) GetByName(name string) *Spider {
	return self.hash[name]
}
<|endoftext|>


<commit_before>package parser

import (
	"fmt"
	"path/filepath"
	"strings"
	"testing"

	"github.com/wellington/sass/ast"
	"github.com/wellington/sass/token"
)

func TestParse_files(t *testing.T) {
	if testing.Short() {
		t.Skip("Skip robust testing to better indicate errors")
	}

	inputs, err := filepath.Glob("../sass-spec/spec/basic/*/input.scss")
	if err != nil {
		t.Fatal(err)
	}

	mode := DeclarationErrors
	mode = 0 //Trace | ParseComments
	var name string
	defer func() { fmt.Println("exit parsing", name) }()
	for _, name = range inputs {

		if !strings.Contains(name, "23_") {
			continue
		}
		if strings.Contains(name, "06_") {
			continue
		}
		// These are fucked things in Sass like lists
		if strings.Contains(name, "15_") {
			continue
		}
		if strings.Contains(name, "16_") {
			continue
		}
		// namespaces are wtf
		if strings.Contains(name, "24_") {
			continue
		}
		fmt.Println("Parsing", name)
		_, err := ParseFile(token.NewFileSet(), name, nil, mode)
		if err != nil {
			t.Fatalf("ParseFile(%s): %v", name, err)
		}
		fmt.Println("Parsed", name)
	}
}

func testString(t *testing.T, in string, mode Mode) (*ast.File, *token.FileSet) {
	fset := token.NewFileSet()
	f, err := ParseFile(fset, "testfile", in, mode)
	if err != nil {
		t.Fatal(err)
	}
	return f, fset

}

func TestSelMath(t *testing.T) {
	// Selectors act like boolean math
	in := `
div ~ span { }`
	f, fset := testString(t, in, 0)
	_ = fset
	sel, ok := f.Decls[0].(*ast.SelDecl)
	if !ok {
		t.Fatal("SelDecl expected")
	}

	bexpr, ok := sel.Sel.(*ast.BinaryExpr)
	if !ok {
		t.Fatal("BinaryExpr expected")
	}

	lit, ok := bexpr.X.(*ast.BasicLit)
	if !ok {
		t.Fatal("BasicLit expected")
	}

	if e := "div"; lit.Value != e {
		t.Errorf("got: %s wanted: %s", lit.Value, e)
	}

	if e := token.TIL; bexpr.Op != e {
		t.Errorf("got: %s wanted: %s", bexpr.Op, e)
	}

	lit, ok = bexpr.Y.(*ast.BasicLit)
	if !ok {
		t.Fatal("BasicLit expected")
	}

	if e := "span"; lit.Value != e {
		t.Errorf("got: %s wanted: %s", lit.Value, e)
	}
}

func TestBackRef(t *testing.T) {
	// Selectors act like boolean math
	in := `div { & { color: red; } }`
	f, fset := testString(t, in, 0)
	_, _ = f, fset

	decl, ok := f.Decls[0].(*ast.SelDecl)
	if !ok {
		t.Fatal("SelDecl expected")
	}
	sel := decl.SelStmt
	lit, ok := sel.Sel.(*ast.BasicLit)
	if !ok {
		t.Fatal("BasicLit expected")
	}
	if e := "div"; lit.Value != e {
		t.Errorf("got: %s wanted: %s", lit.Value, e)
	}

	nested, ok := sel.Body.List[0].(*ast.SelStmt)
	if !ok {
		t.Fatal("expected SelStmt")
	}

	if e := "&"; nested.Name.String() != e {
		t.Fatalf("got: %s wanted: %s", nested.Name.String(), e)
	}

	if e := "div"; e != nested.Resolved.Value {
		t.Errorf("got: %s wanted: %s", nested.Resolved.Value, e)
	}
}

func TestExprMath(t *testing.T) {
	// Selectors act like boolean math
	in := `
div {
  value: 1*(2+3);
}`
	f, fset := testString(t, in, 0)
	ast.Print(fset, f.Decls[0].(*ast.SelDecl).Body.List[0].(*ast.DeclStmt).Decl.(*ast.GenDecl).Specs[0].(*ast.RuleSpec).Values[0])
}
<commit_msg>remove bad tests<commit_after>package parser

import (
	"fmt"
	"path/filepath"
	"strings"
	"testing"

	"github.com/wellington/sass/ast"
	"github.com/wellington/sass/token"
)

func TestParse_files(t *testing.T) {
	if testing.Short() {
		t.Skip("Skip robust testing to better indicate errors")
	}

	inputs, err := filepath.Glob("../sass-spec/spec/basic/*/input.scss")
	if err != nil {
		t.Fatal(err)
	}

	mode := DeclarationErrors
	mode = 0 //Trace | ParseComments
	var name string
	defer func() { fmt.Println("exit parsing", name) }()
	for _, name = range inputs {

		if !strings.Contains(name, "23_") {
			continue
		}
		if strings.Contains(name, "06_") {
			continue
		}
		// These are fucked things in Sass like lists
		if strings.Contains(name, "15_") {
			continue
		}
		if strings.Contains(name, "16_") {
			continue
		}
		// namespaces are wtf
		if strings.Contains(name, "24_") {
			continue
		}
		fmt.Println("Parsing", name)
		_, err := ParseFile(token.NewFileSet(), name, nil, mode)
		if err != nil {
			t.Fatalf("ParseFile(%s): %v", name, err)
		}
		fmt.Println("Parsed", name)
	}
}

func testString(t *testing.T, in string, mode Mode) (*ast.File, *token.FileSet) {
	fset := token.NewFileSet()
	f, err := ParseFile(fset, "testfile", in, mode)
	if err != nil {
		t.Fatal(err)
	}
	return f, fset

}

func TestSelMath(t *testing.T) {
	// Selectors act like boolean math
	in := `
div ~ span { }`
	f, fset := testString(t, in, 0)
	_ = fset
	sel, ok := f.Decls[0].(*ast.SelDecl)
	if !ok {
		t.Fatal("SelDecl expected")
	}

	bexpr, ok := sel.Sel.(*ast.BinaryExpr)
	if !ok {
		t.Fatal("BinaryExpr expected")
	}

	lit, ok := bexpr.X.(*ast.BasicLit)
	if !ok {
		t.Fatal("BasicLit expected")
	}

	if e := "div"; lit.Value != e {
		t.Errorf("got: %s wanted: %s", lit.Value, e)
	}

	if e := token.TIL; bexpr.Op != e {
		t.Errorf("got: %s wanted: %s", bexpr.Op, e)
	}

	lit, ok = bexpr.Y.(*ast.BasicLit)
	if !ok {
		t.Fatal("BasicLit expected")
	}

	if e := "span"; lit.Value != e {
		t.Errorf("got: %s wanted: %s", lit.Value, e)
	}
}

func TestBackRef(t *testing.T) {
	// Selectors act like boolean math
	in := `div { & { color: red; } }`
	f, fset := testString(t, in, 0)
	_, _ = f, fset

	decl, ok := f.Decls[0].(*ast.SelDecl)
	if !ok {
		t.Fatal("SelDecl expected")
	}
	sel := decl.SelStmt
	lit, ok := sel.Sel.(*ast.BasicLit)
	if !ok {
		t.Fatal("BasicLit expected")
	}
	if e := "div"; lit.Value != e {
		t.Errorf("got: %s wanted: %s", lit.Value, e)
	}

	nested, ok := sel.Body.List[0].(*ast.SelStmt)
	if !ok {
		t.Fatal("expected SelStmt")
	}

	if e := "&"; nested.Name.String() != e {
		t.Fatalf("got: %s wanted: %s", nested.Name.String(), e)
	}

	if e := "div"; e != nested.Resolved.Value {
		t.Errorf("got: %s wanted: %s", nested.Resolved.Value, e)
	}
}
<|endoftext|>


<commit_before>package metrics

import (
	"bytes"
	"fmt"
	"math"
	"net/url"
	"strings"
	"sync"
	"time"
)

var spaceReplacer = strings.NewReplacer(" ", "_")

// InmemSink provides a MetricSink that does in-memory aggregation
// without sending metrics over a network. It can be embedded within
// an application to provide profiling information.
type InmemSink struct {
	// How long is each aggregation interval
	interval time.Duration

	// Retain controls how many metrics interval we keep
	retain time.Duration

	// maxIntervals is the maximum length of intervals.
	// It is retain / interval.
	maxIntervals int

	// intervals is a slice of the retained intervals
	intervals    []*IntervalMetrics
	intervalLock sync.RWMutex

	rateDenom float64
}

// IntervalMetrics stores the aggregated metrics
// for a specific interval
type IntervalMetrics struct {
	sync.RWMutex

	// The start time of the interval
	Interval time.Time

	// Gauges maps the key to the last set value
	Gauges map[string]GaugeValue

	// Points maps the string to the list of emitted values
	// from EmitKey
	Points map[string][]float32

	// Counters maps the string key to a sum of the counter
	// values
	Counters map[string]SampledValue

	// Samples maps the key to an AggregateSample,
	// which has the rolled up view of a sample
	Samples map[string]SampledValue
}

// NewIntervalMetrics creates a new IntervalMetrics for a given interval
func NewIntervalMetrics(intv time.Time) *IntervalMetrics {
	return &IntervalMetrics{
		Interval: intv,
		Gauges:   make(map[string]GaugeValue),
		Points:   make(map[string][]float32),
		Counters: make(map[string]SampledValue),
		Samples:  make(map[string]SampledValue),
	}
}

// AggregateSample is used to hold aggregate metrics
// about a sample
type AggregateSample struct {
	Count       int       // The count of emitted pairs
	Rate        float64   // The values rate per time unit (usually 1 second)
	Sum         float64   // The sum of values
	SumSq       float64   `json:"-"` // The sum of squared values
	Min         float64   // Minimum value
	Max         float64   // Maximum value
	LastUpdated time.Time `json:"-"` // When value was last updated
}

// Computes a Stddev of the values
func (a *AggregateSample) Stddev() float64 {
	num := (float64(a.Count) * a.SumSq) - math.Pow(a.Sum, 2)
	div := float64(a.Count * (a.Count - 1))
	if div == 0 {
		return 0
	}
	return math.Sqrt(num / div)
}

// Computes a mean of the values
func (a *AggregateSample) Mean() float64 {
	if a.Count == 0 {
		return 0
	}
	return a.Sum / float64(a.Count)
}

// Ingest is used to update a sample
func (a *AggregateSample) Ingest(v float64, rateDenom float64) {
	a.Count++
	a.Sum += v
	a.SumSq += (v * v)
	if v < a.Min || a.Count == 1 {
		a.Min = v
	}
	if v > a.Max || a.Count == 1 {
		a.Max = v
	}
	a.Rate = float64(a.Sum) / rateDenom
	a.LastUpdated = time.Now()
}

func (a *AggregateSample) String() string {
	if a.Count == 0 {
		return "Count: 0"
	} else if a.Stddev() == 0 {
		return fmt.Sprintf("Count: %d Sum: %0.3f LastUpdated: %s", a.Count, a.Sum, a.LastUpdated)
	} else {
		return fmt.Sprintf("Count: %d Min: %0.3f Mean: %0.3f Max: %0.3f Stddev: %0.3f Sum: %0.3f LastUpdated: %s",
			a.Count, a.Min, a.Mean(), a.Max, a.Stddev(), a.Sum, a.LastUpdated)
	}
}

// NewInmemSinkFromURL creates an InmemSink from a URL. It is used
// (and tested) from NewMetricSinkFromURL.
func NewInmemSinkFromURL(u *url.URL) (MetricSink, error) {
	params := u.Query()

	interval, err := time.ParseDuration(params.Get("interval"))
	if err != nil {
		return nil, fmt.Errorf("Bad 'interval' param: %s", err)
	}

	retain, err := time.ParseDuration(params.Get("retain"))
	if err != nil {
		return nil, fmt.Errorf("Bad 'retain' param: %s", err)
	}

	return NewInmemSink(interval, retain), nil
}

// NewInmemSink is used to construct a new in-memory sink.
// Uses an aggregation interval and maximum retention period.
func NewInmemSink(interval, retain time.Duration) *InmemSink {
	rateTimeUnit := time.Second
	i := &InmemSink{
		interval:     interval,
		retain:       retain,
		maxIntervals: int(retain / interval),
		rateDenom:    float64(interval.Nanoseconds()) / float64(rateTimeUnit.Nanoseconds()),
	}
	i.intervals = make([]*IntervalMetrics, 0, i.maxIntervals)
	return i
}

func (i *InmemSink) SetGauge(key []string, val float32) {
	i.SetGaugeWithLabels(key, val, nil)
}

func (i *InmemSink) SetGaugeWithLabels(key []string, val float32, labels []Label) {
	k, name := i.flattenKeyLabels(key, labels)
	intv := i.getInterval()

	intv.Lock()
	defer intv.Unlock()
	intv.Gauges[k] = GaugeValue{Name: name, Value: val, Labels: labels}
}

func (i *InmemSink) EmitKey(key []string, val float32) {
	k := i.flattenKey(key)
	intv := i.getInterval()

	intv.Lock()
	defer intv.Unlock()
	vals := intv.Points[k]
	intv.Points[k] = append(vals, val)
}

func (i *InmemSink) IncrCounter(key []string, val float32) {
	i.IncrCounterWithLabels(key, val, nil)
}

func (i *InmemSink) IncrCounterWithLabels(key []string, val float32, labels []Label) {
	k, name := i.flattenKeyLabels(key, labels)
	intv := i.getInterval()

	intv.Lock()
	defer intv.Unlock()

	agg, ok := intv.Counters[k]
	if !ok {
		agg = SampledValue{
			Name:            name,
			AggregateSample: &AggregateSample{},
			Labels:          labels,
		}
		intv.Counters[k] = agg
	}
	agg.Ingest(float64(val), i.rateDenom)
}

func (i *InmemSink) AddSample(key []string, val float32) {
	i.AddSampleWithLabels(key, val, nil)
}

func (i *InmemSink) AddSampleWithLabels(key []string, val float32, labels []Label) {
	k, name := i.flattenKeyLabels(key, labels)
	intv := i.getInterval()

	intv.Lock()
	defer intv.Unlock()

	agg, ok := intv.Samples[k]
	if !ok {
		agg = SampledValue{
			Name:            name,
			AggregateSample: &AggregateSample{},
			Labels:          labels,
		}
		intv.Samples[k] = agg
	}
	agg.Ingest(float64(val), i.rateDenom)
}

// Data is used to retrieve all the aggregated metrics
// Intervals may be in use, and a read lock should be acquired
func (i *InmemSink) Data() []*IntervalMetrics {
	// Get the current interval, forces creation
	i.getInterval()

	i.intervalLock.RLock()
	defer i.intervalLock.RUnlock()

	n := len(i.intervals)
	intervals := make([]*IntervalMetrics, n)

	copy(intervals[:n-1], i.intervals[:n-1])
	current := i.intervals[n-1]

	// make its own copy for current interval
	intervals[n-1] = &IntervalMetrics{}
	copyCurrent := intervals[n-1]
	current.RLock()
	*copyCurrent = *current

	copyCurrent.Gauges = make(map[string]GaugeValue, len(current.Gauges))
	for k, v := range current.Gauges {
		copyCurrent.Gauges[k] = v
	}
	// saved values will be not change, just copy its link
	copyCurrent.Points = make(map[string][]float32, len(current.Points))
	for k, v := range current.Points {
		copyCurrent.Points[k] = v
	}
	copyCurrent.Counters = make(map[string]SampledValue, len(current.Counters))
	for k, v := range current.Counters {
		copyCurrent.Counters[k] = v.deepCopy()
	}
	copyCurrent.Samples = make(map[string]SampledValue, len(current.Samples))
	for k, v := range current.Samples {
		copyCurrent.Samples[k] = v.deepCopy()
	}
	current.RUnlock()

	return intervals
}

func (i *InmemSink) getExistingInterval(intv time.Time) *IntervalMetrics {
	i.intervalLock.RLock()
	defer i.intervalLock.RUnlock()

	n := len(i.intervals)
	if n > 0 && i.intervals[n-1].Interval == intv {
		return i.intervals[n-1]
	}
	return nil
}

func (i *InmemSink) createInterval(intv time.Time) *IntervalMetrics {
	i.intervalLock.Lock()
	defer i.intervalLock.Unlock()

	// Check for an existing interval
	n := len(i.intervals)
	if n > 0 && i.intervals[n-1].Interval == intv {
		return i.intervals[n-1]
	}

	// Add the current interval
	current := NewIntervalMetrics(intv)
	i.intervals = append(i.intervals, current)
	n++

	// Truncate the intervals if they are too long
	if n >= i.maxIntervals {
		copy(i.intervals[0:], i.intervals[n-i.maxIntervals:])
		i.intervals = i.intervals[:i.maxIntervals]
	}
	return current
}

// getInterval returns the current interval to write to
func (i *InmemSink) getInterval() *IntervalMetrics {
	intv := time.Now().Truncate(i.interval)
	if m := i.getExistingInterval(intv); m != nil {
		return m
	}
	return i.createInterval(intv)
}

// Flattens the key for formatting, removes spaces
func (i *InmemSink) flattenKey(parts []string) string {
	buf := &bytes.Buffer{}

	joined := strings.Join(parts, ".")

	spaceReplacer.WriteString(buf, joined)

	return buf.String()
}

// Flattens the key for formatting along with its labels, removes spaces
func (i *InmemSink) flattenKeyLabels(parts []string, labels []Label) (string, string) {
	key := i.flattenKey(parts)
	buf := bytes.NewBufferString(key)

	for _, label := range labels {
		spaceReplacer.WriteString(buf, fmt.Sprintf(";%s=%s", label.Name, label.Value))
	}

	return buf.String(), key
}
<commit_msg>inmem: create a new RWMutex, because they are not safe to copy<commit_after>package metrics

import (
	"bytes"
	"fmt"
	"math"
	"net/url"
	"strings"
	"sync"
	"time"
)

var spaceReplacer = strings.NewReplacer(" ", "_")

// InmemSink provides a MetricSink that does in-memory aggregation
// without sending metrics over a network. It can be embedded within
// an application to provide profiling information.
type InmemSink struct {
	// How long is each aggregation interval
	interval time.Duration

	// Retain controls how many metrics interval we keep
	retain time.Duration

	// maxIntervals is the maximum length of intervals.
	// It is retain / interval.
	maxIntervals int

	// intervals is a slice of the retained intervals
	intervals    []*IntervalMetrics
	intervalLock sync.RWMutex

	rateDenom float64
}

// IntervalMetrics stores the aggregated metrics
// for a specific interval
type IntervalMetrics struct {
	sync.RWMutex

	// The start time of the interval
	Interval time.Time

	// Gauges maps the key to the last set value
	Gauges map[string]GaugeValue

	// Points maps the string to the list of emitted values
	// from EmitKey
	Points map[string][]float32

	// Counters maps the string key to a sum of the counter
	// values
	Counters map[string]SampledValue

	// Samples maps the key to an AggregateSample,
	// which has the rolled up view of a sample
	Samples map[string]SampledValue
}

// NewIntervalMetrics creates a new IntervalMetrics for a given interval
func NewIntervalMetrics(intv time.Time) *IntervalMetrics {
	return &IntervalMetrics{
		Interval: intv,
		Gauges:   make(map[string]GaugeValue),
		Points:   make(map[string][]float32),
		Counters: make(map[string]SampledValue),
		Samples:  make(map[string]SampledValue),
	}
}

// AggregateSample is used to hold aggregate metrics
// about a sample
type AggregateSample struct {
	Count       int       // The count of emitted pairs
	Rate        float64   // The values rate per time unit (usually 1 second)
	Sum         float64   // The sum of values
	SumSq       float64   `json:"-"` // The sum of squared values
	Min         float64   // Minimum value
	Max         float64   // Maximum value
	LastUpdated time.Time `json:"-"` // When value was last updated
}

// Computes a Stddev of the values
func (a *AggregateSample) Stddev() float64 {
	num := (float64(a.Count) * a.SumSq) - math.Pow(a.Sum, 2)
	div := float64(a.Count * (a.Count - 1))
	if div == 0 {
		return 0
	}
	return math.Sqrt(num / div)
}

// Computes a mean of the values
func (a *AggregateSample) Mean() float64 {
	if a.Count == 0 {
		return 0
	}
	return a.Sum / float64(a.Count)
}

// Ingest is used to update a sample
func (a *AggregateSample) Ingest(v float64, rateDenom float64) {
	a.Count++
	a.Sum += v
	a.SumSq += (v * v)
	if v < a.Min || a.Count == 1 {
		a.Min = v
	}
	if v > a.Max || a.Count == 1 {
		a.Max = v
	}
	a.Rate = float64(a.Sum) / rateDenom
	a.LastUpdated = time.Now()
}

func (a *AggregateSample) String() string {
	if a.Count == 0 {
		return "Count: 0"
	} else if a.Stddev() == 0 {
		return fmt.Sprintf("Count: %d Sum: %0.3f LastUpdated: %s", a.Count, a.Sum, a.LastUpdated)
	} else {
		return fmt.Sprintf("Count: %d Min: %0.3f Mean: %0.3f Max: %0.3f Stddev: %0.3f Sum: %0.3f LastUpdated: %s",
			a.Count, a.Min, a.Mean(), a.Max, a.Stddev(), a.Sum, a.LastUpdated)
	}
}

// NewInmemSinkFromURL creates an InmemSink from a URL. It is used
// (and tested) from NewMetricSinkFromURL.
func NewInmemSinkFromURL(u *url.URL) (MetricSink, error) {
	params := u.Query()

	interval, err := time.ParseDuration(params.Get("interval"))
	if err != nil {
		return nil, fmt.Errorf("Bad 'interval' param: %s", err)
	}

	retain, err := time.ParseDuration(params.Get("retain"))
	if err != nil {
		return nil, fmt.Errorf("Bad 'retain' param: %s", err)
	}

	return NewInmemSink(interval, retain), nil
}

// NewInmemSink is used to construct a new in-memory sink.
// Uses an aggregation interval and maximum retention period.
func NewInmemSink(interval, retain time.Duration) *InmemSink {
	rateTimeUnit := time.Second
	i := &InmemSink{
		interval:     interval,
		retain:       retain,
		maxIntervals: int(retain / interval),
		rateDenom:    float64(interval.Nanoseconds()) / float64(rateTimeUnit.Nanoseconds()),
	}
	i.intervals = make([]*IntervalMetrics, 0, i.maxIntervals)
	return i
}

func (i *InmemSink) SetGauge(key []string, val float32) {
	i.SetGaugeWithLabels(key, val, nil)
}

func (i *InmemSink) SetGaugeWithLabels(key []string, val float32, labels []Label) {
	k, name := i.flattenKeyLabels(key, labels)
	intv := i.getInterval()

	intv.Lock()
	defer intv.Unlock()
	intv.Gauges[k] = GaugeValue{Name: name, Value: val, Labels: labels}
}

func (i *InmemSink) EmitKey(key []string, val float32) {
	k := i.flattenKey(key)
	intv := i.getInterval()

	intv.Lock()
	defer intv.Unlock()
	vals := intv.Points[k]
	intv.Points[k] = append(vals, val)
}

func (i *InmemSink) IncrCounter(key []string, val float32) {
	i.IncrCounterWithLabels(key, val, nil)
}

func (i *InmemSink) IncrCounterWithLabels(key []string, val float32, labels []Label) {
	k, name := i.flattenKeyLabels(key, labels)
	intv := i.getInterval()

	intv.Lock()
	defer intv.Unlock()

	agg, ok := intv.Counters[k]
	if !ok {
		agg = SampledValue{
			Name:            name,
			AggregateSample: &AggregateSample{},
			Labels:          labels,
		}
		intv.Counters[k] = agg
	}
	agg.Ingest(float64(val), i.rateDenom)
}

func (i *InmemSink) AddSample(key []string, val float32) {
	i.AddSampleWithLabels(key, val, nil)
}

func (i *InmemSink) AddSampleWithLabels(key []string, val float32, labels []Label) {
	k, name := i.flattenKeyLabels(key, labels)
	intv := i.getInterval()

	intv.Lock()
	defer intv.Unlock()

	agg, ok := intv.Samples[k]
	if !ok {
		agg = SampledValue{
			Name:            name,
			AggregateSample: &AggregateSample{},
			Labels:          labels,
		}
		intv.Samples[k] = agg
	}
	agg.Ingest(float64(val), i.rateDenom)
}

// Data is used to retrieve all the aggregated metrics
// Intervals may be in use, and a read lock should be acquired
func (i *InmemSink) Data() []*IntervalMetrics {
	// Get the current interval, forces creation
	i.getInterval()

	i.intervalLock.RLock()
	defer i.intervalLock.RUnlock()

	n := len(i.intervals)
	intervals := make([]*IntervalMetrics, n)

	copy(intervals[:n-1], i.intervals[:n-1])
	current := i.intervals[n-1]

	// make its own copy for current interval
	intervals[n-1] = &IntervalMetrics{}
	copyCurrent := intervals[n-1]
	current.RLock()
	*copyCurrent = *current
	// RWMutex is not safe to copy, so create a new instance on the copy
	copyCurrent.RWMutex = sync.RWMutex{}

	copyCurrent.Gauges = make(map[string]GaugeValue, len(current.Gauges))
	for k, v := range current.Gauges {
		copyCurrent.Gauges[k] = v
	}
	// saved values will be not change, just copy its link
	copyCurrent.Points = make(map[string][]float32, len(current.Points))
	for k, v := range current.Points {
		copyCurrent.Points[k] = v
	}
	copyCurrent.Counters = make(map[string]SampledValue, len(current.Counters))
	for k, v := range current.Counters {
		copyCurrent.Counters[k] = v.deepCopy()
	}
	copyCurrent.Samples = make(map[string]SampledValue, len(current.Samples))
	for k, v := range current.Samples {
		copyCurrent.Samples[k] = v.deepCopy()
	}
	current.RUnlock()

	return intervals
}

func (i *InmemSink) getExistingInterval(intv time.Time) *IntervalMetrics {
	i.intervalLock.RLock()
	defer i.intervalLock.RUnlock()

	n := len(i.intervals)
	if n > 0 && i.intervals[n-1].Interval == intv {
		return i.intervals[n-1]
	}
	return nil
}

func (i *InmemSink) createInterval(intv time.Time) *IntervalMetrics {
	i.intervalLock.Lock()
	defer i.intervalLock.Unlock()

	// Check for an existing interval
	n := len(i.intervals)
	if n > 0 && i.intervals[n-1].Interval == intv {
		return i.intervals[n-1]
	}

	// Add the current interval
	current := NewIntervalMetrics(intv)
	i.intervals = append(i.intervals, current)
	n++

	// Truncate the intervals if they are too long
	if n >= i.maxIntervals {
		copy(i.intervals[0:], i.intervals[n-i.maxIntervals:])
		i.intervals = i.intervals[:i.maxIntervals]
	}
	return current
}

// getInterval returns the current interval to write to
func (i *InmemSink) getInterval() *IntervalMetrics {
	intv := time.Now().Truncate(i.interval)
	if m := i.getExistingInterval(intv); m != nil {
		return m
	}
	return i.createInterval(intv)
}

// Flattens the key for formatting, removes spaces
func (i *InmemSink) flattenKey(parts []string) string {
	buf := &bytes.Buffer{}

	joined := strings.Join(parts, ".")

	spaceReplacer.WriteString(buf, joined)

	return buf.String()
}

// Flattens the key for formatting along with its labels, removes spaces
func (i *InmemSink) flattenKeyLabels(parts []string, labels []Label) (string, string) {
	key := i.flattenKey(parts)
	buf := bytes.NewBufferString(key)

	for _, label := range labels {
		spaceReplacer.WriteString(buf, fmt.Sprintf(";%s=%s", label.Name, label.Value))
	}

	return buf.String(), key
}
<|endoftext|>