<commit_before>package containerdexecutor

import (
	"context"
	"os"
	"path/filepath"
	"strings"
	"sync"
	"syscall"
	"time"

	"github.com/containerd/containerd"
	"github.com/containerd/containerd/cio"
	containerdoci "github.com/containerd/containerd/oci"
	"github.com/containerd/continuity/fs"
	"github.com/docker/docker/pkg/idtools"
	"github.com/moby/buildkit/cache"
	"github.com/moby/buildkit/executor"
	"github.com/moby/buildkit/executor/oci"
	"github.com/moby/buildkit/identity"
	"github.com/moby/buildkit/snapshot"
	"github.com/moby/buildkit/solver/pb"
	"github.com/moby/buildkit/util/network"
	"github.com/opencontainers/runtime-spec/specs-go"
	"github.com/pkg/errors"
	"github.com/sirupsen/logrus"
)

type containerdExecutor struct {
	client           *containerd.Client
	root             string
	networkProviders map[pb.NetMode]network.Provider
	cgroupParent     string
	dnsConfig        *oci.DNSConfig
	running          map[string]chan error
	mu               sync.Mutex
}

// New creates a new executor backed by connection to containerd API
func New(client *containerd.Client, root, cgroup string, networkProviders map[pb.NetMode]network.Provider, dnsConfig *oci.DNSConfig) executor.Executor {
	// clean up old hosts/resolv.conf file. ignore errors
	os.RemoveAll(filepath.Join(root, "hosts"))
	os.RemoveAll(filepath.Join(root, "resolv.conf"))

	return &containerdExecutor{
		client:           client,
		root:             root,
		networkProviders: networkProviders,
		cgroupParent:     cgroup,
		dnsConfig:        dnsConfig,
		running:          make(map[string]chan error),
	}
}

func (w *containerdExecutor) Run(ctx context.Context, id string, root cache.Mountable, mounts []executor.Mount, process executor.ProcessInfo, started chan<- struct{}) (err error) {
	if id == "" {
		id = identity.NewID()
	}

	startedOnce := sync.Once{}
	done := make(chan error, 1)
	w.mu.Lock()
	w.running[id] = done
	w.mu.Unlock()
	defer func() {
		w.mu.Lock()
		delete(w.running, id)
		w.mu.Unlock()
		done <- err
		close(done)
		if started != nil {
			startedOnce.Do(func() {
				close(started)
			})
		}
	}()

	meta := process.Meta

	resolvConf, err := oci.GetResolvConf(ctx, w.root, nil, w.dnsConfig)
	if err != nil {
		return err
	}

	hostsFile, clean, err := oci.GetHostsFile(ctx, w.root, meta.ExtraHosts, nil)
	if err != nil {
		return err
	}
	if clean != nil {
		defer clean()
	}

	mountable, err := root.Mount(ctx, false)
	if err != nil {
		return err
	}

	rootMounts, release, err := mountable.Mount()
	if err != nil {
		return err
	}
	if release != nil {
		defer release()
	}

	var sgids []uint32
	uid, gid, err := oci.ParseUIDGID(meta.User)
	if err != nil {
		lm := snapshot.LocalMounterWithMounts(rootMounts)
		rootfsPath, err := lm.Mount()
		if err != nil {
			return err
		}
		uid, gid, sgids, err = oci.GetUser(rootfsPath, meta.User)
		if err != nil {
			lm.Unmount()
			return err
		}

		identity := idtools.Identity{
			UID: int(uid),
			GID: int(gid),
		}

		newp, err := fs.RootPath(rootfsPath, meta.Cwd)
		if err != nil {
			lm.Unmount()
			return errors.Wrapf(err, "working dir %s points to invalid target", newp)
		}
		if _, err := os.Stat(newp); err != nil {
			if err := idtools.MkdirAllAndChown(newp, 0755, identity); err != nil {
				lm.Unmount()
				return errors.Wrapf(err, "failed to create working directory %s", newp)
			}
		}

		lm.Unmount()
	}

	provider, ok := w.networkProviders[meta.NetMode]
	if !ok {
		return errors.Errorf("unknown network mode %s", meta.NetMode)
	}
	namespace, err := provider.New()
	if err != nil {
		return err
	}
	defer namespace.Close()

	if meta.NetMode == pb.NetMode_HOST {
		logrus.Info("enabling HostNetworking")
	}

	opts := []containerdoci.SpecOpts{oci.WithUIDGID(uid, gid, sgids)}
	if meta.ReadonlyRootFS {
		opts = append(opts, containerdoci.WithRootFSReadonly())
	}

	if w.cgroupParent != "" {
		var cgroupsPath string
		lastSeparator := w.cgroupParent[len(w.cgroupParent)-1:]
		if strings.Contains(w.cgroupParent, ".slice") && lastSeparator == ":" {
			cgroupsPath = w.cgroupParent + id
		} else {
			cgroupsPath = filepath.Join("/", w.cgroupParent, "buildkit", id)
		}
		opts = append(opts, containerdoci.WithCgroup(cgroupsPath))
	}
	processMode := oci.ProcessSandbox // FIXME(AkihiroSuda)
	spec, cleanup, err := oci.GenerateSpec(ctx, meta, mounts, id, resolvConf, hostsFile, namespace, processMode, nil, opts...)
	if err != nil {
		return err
	}
	defer cleanup()
	spec.Process.Terminal = meta.Tty

	container, err := w.client.NewContainer(ctx, id,
		containerd.WithSpec(spec),
	)
	if err != nil {
		return err
	}

	defer func() {
		if err1 := container.Delete(context.TODO()); err == nil && err1 != nil {
			err = errors.Wrapf(err1, "failed to delete container %s", id)
		}
	}()

	cioOpts := []cio.Opt{cio.WithStreams(process.Stdin, process.Stdout, process.Stderr)}
	if meta.Tty {
		cioOpts = append(cioOpts, cio.WithTerminal)
	}

	task, err := container.NewTask(ctx, cio.NewCreator(cioOpts...), containerd.WithRootFS(rootMounts))
	if err != nil {
		return err
	}

	defer func() {
		if _, err1 := task.Delete(context.TODO()); err == nil && err1 != nil {
			err = errors.Wrapf(err1, "failed to delete task %s", id)
		}
	}()

	err = w.runProcess(ctx, task, process.Resize, func() {
		startedOnce.Do(func() {
			if started != nil {
				close(started)
			}
		})
	})
	return err
}

func (w *containerdExecutor) Exec(ctx context.Context, id string, process executor.ProcessInfo) (err error) {
	meta := process.Meta

	// first verify the container is running, if we get an error assume the container
	// is in the process of being created and check again every 100ms or until
	// context is canceled.

	var container containerd.Container
	var task containerd.Task
	for {
		w.mu.Lock()
		done, ok := w.running[id]
		w.mu.Unlock()

		if !ok {
			return errors.Errorf("container %s not found", id)
		}

		if container == nil {
			container, _ = w.client.LoadContainer(ctx, id)
		}
		if container != nil && task == nil {
			task, _ = container.Task(ctx, nil)
		}
		if task != nil {
			status, _ := task.Status(ctx)
			if status.Status == containerd.Running {
				break
			}
		}
		select {
		case <-ctx.Done():
			return ctx.Err()
		case err, ok := <-done:
			if !ok || err == nil {
				return errors.Errorf("container %s has stopped", id)
			}
			return errors.Wrapf(err, "container %s has exited with error", id)
		case <-time.After(100 * time.Millisecond):
			continue
		}
	}

	spec, err := container.Spec(ctx)
	if err != nil {
		return errors.WithStack(err)
	}

	proc := spec.Process

	// TODO how do we get rootfsPath for oci.GetUser in case user passed in username rather than uid:gid?
	// For now only support uid:gid
	if meta.User != "" {
		uid, gid, err := oci.ParseUIDGID(meta.User)
		if err != nil {
			return errors.WithStack(err)
		}
		proc.User = specs.User{
			UID:            uid,
			GID:            gid,
			AdditionalGids: []uint32{},
		}
	}

	proc.Terminal = meta.Tty
	proc.Args = meta.Args
	if meta.Cwd != "" {
		spec.Process.Cwd = meta.Cwd
	}
	if len(process.Meta.Env) > 0 {
		spec.Process.Env = process.Meta.Env
	}

	cioOpts := []cio.Opt{cio.WithStreams(process.Stdin, process.Stdout, process.Stderr)}
	if meta.Tty {
		cioOpts = append(cioOpts, cio.WithTerminal)
	}

	taskProcess, err := task.Exec(ctx, identity.NewID(), proc, cio.NewCreator(cioOpts...))
	if err != nil {
		return errors.WithStack(err)
	}

	err = w.runProcess(ctx, taskProcess, process.Resize, nil)
	return err
}

func (w *containerdExecutor) runProcess(ctx context.Context, p containerd.Process, resize <-chan executor.WinSize, started func()) error {
	statusCh, err := p.Wait(context.Background())
	if err != nil {
		return err
	}

	err = p.Start(ctx)
	if err != nil {
		return err
	}

	if started != nil {
		started()
	}

	p.CloseIO(ctx, containerd.WithStdinCloser)

	var cancel func()
	ctxDone := ctx.Done()
	for {
		select {
		case <-ctxDone:
			ctxDone = nil
			var killCtx context.Context
			killCtx, cancel = context.WithTimeout(context.Background(), 10*time.Second)
			p.Kill(killCtx, syscall.SIGKILL)
		case size := <-resize:
			err := p.Resize(ctx, size.Cols, size.Rows)
			if err != nil {
				cancel()
				return err
			}
		case status := <-statusCh:
			if cancel != nil {
				cancel()
			}
			if status.ExitCode() != 0 {
				exitErr := &executor.ExitError{
					ExitCode: status.ExitCode(),
					Err:      status.Error(),
				}
				if status.ExitCode() == containerd.UnknownExitStatus && status.Error() != nil {
					exitErr.Err = errors.Wrap(status.Error(), "failure waiting for process")
				}
				select {
				case <-ctx.Done():
					exitErr.Err = errors.Wrap(ctx.Err(), exitErr.Error())
				default:
				}
				return exitErr
			}
			return nil
		}
	}
}
<commit_msg>only warn on resize errors prevent resize from blocking exit fix edgecase where kill signal never reaches process<commit_after>package containerdexecutor

import (
	"context"
	"fmt"
	"os"
	"path/filepath"
	"strings"
	"sync"
	"syscall"
	"time"

	"github.com/containerd/containerd"
	"github.com/containerd/containerd/cio"
	containerdoci "github.com/containerd/containerd/oci"
	"github.com/containerd/continuity/fs"
	"github.com/docker/docker/pkg/idtools"
	"github.com/moby/buildkit/cache"
	"github.com/moby/buildkit/executor"
	"github.com/moby/buildkit/executor/oci"
	"github.com/moby/buildkit/identity"
	"github.com/moby/buildkit/snapshot"
	"github.com/moby/buildkit/solver/pb"
	"github.com/moby/buildkit/util/network"
	"github.com/opencontainers/runtime-spec/specs-go"
	"github.com/pkg/errors"
	"github.com/sirupsen/logrus"
)

type containerdExecutor struct {
	client           *containerd.Client
	root             string
	networkProviders map[pb.NetMode]network.Provider
	cgroupParent     string
	dnsConfig        *oci.DNSConfig
	running          map[string]chan error
	mu               sync.Mutex
}

// New creates a new executor backed by connection to containerd API
func New(client *containerd.Client, root, cgroup string, networkProviders map[pb.NetMode]network.Provider, dnsConfig *oci.DNSConfig) executor.Executor {
	// clean up old hosts/resolv.conf file. ignore errors
	os.RemoveAll(filepath.Join(root, "hosts"))
	os.RemoveAll(filepath.Join(root, "resolv.conf"))

	return &containerdExecutor{
		client:           client,
		root:             root,
		networkProviders: networkProviders,
		cgroupParent:     cgroup,
		dnsConfig:        dnsConfig,
		running:          make(map[string]chan error),
	}
}

func (w *containerdExecutor) Run(ctx context.Context, id string, root cache.Mountable, mounts []executor.Mount, process executor.ProcessInfo, started chan<- struct{}) (err error) {
	if id == "" {
		id = identity.NewID()
	}

	startedOnce := sync.Once{}
	done := make(chan error, 1)
	w.mu.Lock()
	w.running[id] = done
	w.mu.Unlock()
	defer func() {
		w.mu.Lock()
		delete(w.running, id)
		w.mu.Unlock()
		done <- err
		close(done)
		if started != nil {
			startedOnce.Do(func() {
				close(started)
			})
		}
	}()

	meta := process.Meta

	resolvConf, err := oci.GetResolvConf(ctx, w.root, nil, w.dnsConfig)
	if err != nil {
		return err
	}

	hostsFile, clean, err := oci.GetHostsFile(ctx, w.root, meta.ExtraHosts, nil)
	if err != nil {
		return err
	}
	if clean != nil {
		defer clean()
	}

	mountable, err := root.Mount(ctx, false)
	if err != nil {
		return err
	}

	rootMounts, release, err := mountable.Mount()
	if err != nil {
		return err
	}
	if release != nil {
		defer release()
	}

	var sgids []uint32
	uid, gid, err := oci.ParseUIDGID(meta.User)
	if err != nil {
		lm := snapshot.LocalMounterWithMounts(rootMounts)
		rootfsPath, err := lm.Mount()
		if err != nil {
			return err
		}
		uid, gid, sgids, err = oci.GetUser(rootfsPath, meta.User)
		if err != nil {
			lm.Unmount()
			return err
		}

		identity := idtools.Identity{
			UID: int(uid),
			GID: int(gid),
		}

		newp, err := fs.RootPath(rootfsPath, meta.Cwd)
		if err != nil {
			lm.Unmount()
			return errors.Wrapf(err, "working dir %s points to invalid target", newp)
		}
		if _, err := os.Stat(newp); err != nil {
			if err := idtools.MkdirAllAndChown(newp, 0755, identity); err != nil {
				lm.Unmount()
				return errors.Wrapf(err, "failed to create working directory %s", newp)
			}
		}

		lm.Unmount()
	}

	provider, ok := w.networkProviders[meta.NetMode]
	if !ok {
		return errors.Errorf("unknown network mode %s", meta.NetMode)
	}
	namespace, err := provider.New()
	if err != nil {
		return err
	}
	defer namespace.Close()

	if meta.NetMode == pb.NetMode_HOST {
		logrus.Info("enabling HostNetworking")
	}

	opts := []containerdoci.SpecOpts{oci.WithUIDGID(uid, gid, sgids)}
	if meta.ReadonlyRootFS {
		opts = append(opts, containerdoci.WithRootFSReadonly())
	}

	if w.cgroupParent != "" {
		var cgroupsPath string
		lastSeparator := w.cgroupParent[len(w.cgroupParent)-1:]
		if strings.Contains(w.cgroupParent, ".slice") && lastSeparator == ":" {
			cgroupsPath = w.cgroupParent + id
		} else {
			cgroupsPath = filepath.Join("/", w.cgroupParent, "buildkit", id)
		}
		opts = append(opts, containerdoci.WithCgroup(cgroupsPath))
	}
	processMode := oci.ProcessSandbox // FIXME(AkihiroSuda)
	spec, cleanup, err := oci.GenerateSpec(ctx, meta, mounts, id, resolvConf, hostsFile, namespace, processMode, nil, opts...)
	if err != nil {
		return err
	}
	defer cleanup()
	spec.Process.Terminal = meta.Tty

	container, err := w.client.NewContainer(ctx, id,
		containerd.WithSpec(spec),
	)
	if err != nil {
		return err
	}

	defer func() {
		if err1 := container.Delete(context.TODO()); err == nil && err1 != nil {
			err = errors.Wrapf(err1, "failed to delete container %s", id)
		}
	}()

	cioOpts := []cio.Opt{cio.WithStreams(process.Stdin, process.Stdout, process.Stderr)}
	if meta.Tty {
		cioOpts = append(cioOpts, cio.WithTerminal)
	}

	task, err := container.NewTask(ctx, cio.NewCreator(cioOpts...), containerd.WithRootFS(rootMounts))
	if err != nil {
		return err
	}

	defer func() {
		if _, err1 := task.Delete(context.TODO()); err == nil && err1 != nil {
			err = errors.Wrapf(err1, "failed to delete task %s", id)
		}
	}()

	err = w.runProcess(ctx, task, process.Resize, func() {
		startedOnce.Do(func() {
			if started != nil {
				close(started)
			}
		})
	})
	return err
}

func (w *containerdExecutor) Exec(ctx context.Context, id string, process executor.ProcessInfo) (err error) {
	meta := process.Meta

	// first verify the container is running, if we get an error assume the container
	// is in the process of being created and check again every 100ms or until
	// context is canceled.

	var container containerd.Container
	var task containerd.Task
	for {
		w.mu.Lock()
		done, ok := w.running[id]
		w.mu.Unlock()

		if !ok {
			return errors.Errorf("container %s not found", id)
		}

		if container == nil {
			container, _ = w.client.LoadContainer(ctx, id)
		}
		if container != nil && task == nil {
			task, _ = container.Task(ctx, nil)
		}
		if task != nil {
			status, _ := task.Status(ctx)
			if status.Status == containerd.Running {
				break
			}
		}
		select {
		case <-ctx.Done():
			return ctx.Err()
		case err, ok := <-done:
			if !ok || err == nil {
				return errors.Errorf("container %s has stopped", id)
			}
			return errors.Wrapf(err, "container %s has exited with error", id)
		case <-time.After(100 * time.Millisecond):
			continue
		}
	}

	spec, err := container.Spec(ctx)
	if err != nil {
		return errors.WithStack(err)
	}

	proc := spec.Process

	// TODO how do we get rootfsPath for oci.GetUser in case user passed in username rather than uid:gid?
	// For now only support uid:gid
	if meta.User != "" {
		uid, gid, err := oci.ParseUIDGID(meta.User)
		if err != nil {
			return errors.WithStack(err)
		}
		proc.User = specs.User{
			UID:            uid,
			GID:            gid,
			AdditionalGids: []uint32{},
		}
	}

	proc.Terminal = meta.Tty
	proc.Args = meta.Args
	if meta.Cwd != "" {
		spec.Process.Cwd = meta.Cwd
	}
	if len(process.Meta.Env) > 0 {
		spec.Process.Env = process.Meta.Env
	}

	cioOpts := []cio.Opt{cio.WithStreams(process.Stdin, process.Stdout, process.Stderr)}
	if meta.Tty {
		cioOpts = append(cioOpts, cio.WithTerminal)
	}

	taskProcess, err := task.Exec(ctx, identity.NewID(), proc, cio.NewCreator(cioOpts...))
	if err != nil {
		return errors.WithStack(err)
	}

	err = w.runProcess(ctx, taskProcess, process.Resize, nil)
	return err
}

func (w *containerdExecutor) runProcess(ctx context.Context, p containerd.Process, resize <-chan executor.WinSize, started func()) error {
	statusCh, err := p.Wait(context.Background())
	if err != nil {
		return err
	}

	err = p.Start(ctx)
	if err != nil {
		return err
	}

	if started != nil {
		started()
	}

	p.CloseIO(ctx, containerd.WithStdinCloser)

	var cancel func()
	var killCtxDone <-chan struct{}
	ctxDone := ctx.Done()
	for {
		select {
		case <-ctxDone:
			ctxDone = nil
			var killCtx context.Context
			killCtx, cancel = context.WithTimeout(context.Background(), 10*time.Second)
			killCtxDone = killCtx.Done()
			p.Kill(killCtx, syscall.SIGKILL)
		case status := <-statusCh:
			if cancel != nil {
				cancel()
			}
			if status.ExitCode() != 0 {
				exitErr := &executor.ExitError{
					ExitCode: status.ExitCode(),
					Err:      status.Error(),
				}
				if status.ExitCode() == containerd.UnknownExitStatus && status.Error() != nil {
					exitErr.Err = errors.Wrap(status.Error(), "failure waiting for process")
				}
				select {
				case <-ctx.Done():
					exitErr.Err = errors.Wrap(ctx.Err(), exitErr.Error())
				default:
				}
				return exitErr
			}
			return nil
		case <-killCtxDone:
			if cancel != nil {
				cancel()
			}
			return fmt.Errorf("failed to kill process on cancel")
		case size := <-resize:
			ctxTimeout, cancelTimeout := context.WithTimeout(ctx, time.Second)
			go func() {
				defer cancelTimeout()
				err = p.Resize(ctxTimeout, size.Cols, size.Rows)
				if err != nil {
					logrus.Warnf("Failed to resize %s: %s", p.ID(), err)
				}
			}()
		}
	}
}
<|endoftext|>


<commit_before>// Copyright 2013 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package taglib provides utilities for parsing audio tags in
// various formats.
package taglib

import (
	"bytes"
	"errors"
	"io"
	"time"

	"github.com/hjfreyer/taglib-go/taglib/id3"
)

var (
	ErrUnrecognizedFormat = errors.New("taglib: format not recognized")
)

// GenericTag is implemented by all the tag types in this project. It
// gives an incomplete view of the information in each tag type, but
// is good enough for most purposes.
type GenericTag interface {
	Title() string
	Artist() string
	Album() string
	Comment() string
	Genre() string
	Year() time.Time
	Track() uint32
	Disc() uint32

	// CustomFrames returns non-standard, user-defined frames as a map from
	// descriptions (e.g. "PERFORMER", "MusicBrainz Album Id", etc.) to
	// values.
	CustomFrames() map[string]string

	// TagSize returns the total size of the tag's header and frames,
	// i.e. the position at which audio data starts.
	TagSize() uint32
}

// Decode reads r and determines which tag format the data is in, if
// any, and calls the decoding function for that format. size
// indicates the total number of bytes accessible through r.
func Decode(r io.ReaderAt, size int64) (GenericTag, error) {
	magic := make([]byte, 4)
	if _, err := r.ReadAt(magic, 0); err != nil {
		return nil, err
	}

	if !bytes.Equal(magic[:3], []byte("ID3")) {
		return nil, ErrUnrecognizedFormat
	}

	switch magic[3] {
	case 3:
		return id3.Decode23(r)
	case 4:
		return id3.Decode24(r)
	default:
		return nil, ErrUnrecognizedFormat
	}
}
<commit_msg>Use more meaning full parse errors<commit_after>// Copyright 2013 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package taglib provides utilities for parsing audio tags in
// various formats.
package taglib

import (
	"bytes"
	"errors"
	"fmt"
	"io"
	"time"

	"github.com/hjfreyer/taglib-go/taglib/id3"
)

// GenericTag is implemented by all the tag types in this project. It
// gives an incomplete view of the information in each tag type, but
// is good enough for most purposes.
type GenericTag interface {
	Title() string
	Artist() string
	Album() string
	Comment() string
	Genre() string
	Year() time.Time
	Track() uint32
	Disc() uint32

	// CustomFrames returns non-standard, user-defined frames as a map from
	// descriptions (e.g. "PERFORMER", "MusicBrainz Album Id", etc.) to
	// values.
	CustomFrames() map[string]string

	// TagSize returns the total size of the tag's header and frames,
	// i.e. the position at which audio data starts.
	TagSize() uint32
}

// Decode reads r and determines which tag format the data is in, if
// any, and calls the decoding function for that format. size
// indicates the total number of bytes accessible through r.
func Decode(r io.ReaderAt, size int64) (GenericTag, error) {
	magic := make([]byte, 4)
	if _, err := r.ReadAt(magic, 0); err != nil {
		return nil, err
	}

	if !bytes.Equal(magic[:3], []byte("ID3")) {
		return nil, errors.New("taglib: format not recognised (not ID3)")
	}

	switch magic[3] {
	case 2:
		return nil, errors.New("taglib: format not supported (ID3 v2.2)")
	case 3:
		return id3.Decode23(r)
	case 4:
		return id3.Decode24(r)
	default:
		return nil, fmt.Errorf("taglib: format not supported (ID3 %d)", magic[3])
	}
}
<|endoftext|>


<commit_before>/*
Copyright 2019 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package distros

import (
	"k8s.io/klog"
	"k8s.io/kops/upup/pkg/fi/nodeup/tags"
)

type Distribution string

var (
	DistributionJessie      Distribution = "jessie"
	DistributionDebian9     Distribution = "debian9"
	DistributionDebian10    Distribution = "buster"
	DistributionXenial      Distribution = "xenial"
	DistributionBionic      Distribution = "bionic"
	DistributionRhel7       Distribution = "rhel7"
	DistributionCentos7     Distribution = "centos7"
	DistributionRhel8       Distribution = "rhel8"
	DistributionCentos8     Distribution = "centos8"
	DistributionCoreOS      Distribution = "coreos"
	DistributionFlatcar     Distribution = "flatcar"
	DistributionContainerOS Distribution = "containeros"
)

func (d Distribution) BuildTags() []string {
	var t []string

	switch d {
	case DistributionJessie:
		t = []string{"_jessie"}
	case DistributionDebian9, DistributionDebian10:
		t = []string{} // trying to move away from tags
	case DistributionXenial:
		t = []string{"_xenial"}
	case DistributionBionic:
		t = []string{"_bionic"}
	case DistributionCentos7:
		t = []string{"_centos7"}
	case DistributionRhel7:
		t = []string{"_rhel7"}
	case DistributionCentos8:
		t = []string{"_centos8"}
	case DistributionRhel8:
		t = []string{"_rhel8"}
	case DistributionCoreOS:
		t = []string{"_coreos"}
	case DistributionFlatcar:
		t = []string{"_flatcar"}
	case DistributionContainerOS:
		t = []string{"_containeros"}
	default:
		klog.Fatalf("unknown distribution: %s", d)
		return nil
	}

	if d.IsDebianFamily() {
		t = append(t, tags.TagOSFamilyDebian)
	}
	if d.IsRHELFamily() {
		t = append(t, tags.TagOSFamilyRHEL)
	}
	if d.IsSystemd() {
		t = append(t, tags.TagSystemd)
	}

	return t
}

func (d Distribution) IsDebianFamily() bool {
	switch d {
	case DistributionJessie, DistributionDebian9, DistributionDebian10:
		return true
	case DistributionXenial, DistributionBionic:
		return true
	case DistributionCentos7, DistributionRhel7:
		return false
	case DistributionCoreOS, DistributionContainerOS:
		return false
	default:
		klog.Fatalf("unknown distribution: %s", d)
		return false
	}
}

func (d Distribution) IsUbuntu() bool {
	switch d {
	case DistributionJessie, DistributionDebian9, DistributionDebian10:
		return false
	case DistributionXenial, DistributionBionic:
		return true
	case DistributionCentos7, DistributionRhel7, DistributionCentos8, DistributionRhel8:
		return false
	case DistributionCoreOS, DistributionFlatcar, DistributionContainerOS:
		return false
	default:
		klog.Fatalf("unknown distribution: %s", d)
		return false
	}
}

func (d Distribution) IsRHELFamily() bool {
	switch d {
	case DistributionCentos7, DistributionRhel7, DistributionCentos8, DistributionRhel8:
		return true
	case DistributionJessie, DistributionXenial, DistributionBionic, DistributionDebian9, DistributionDebian10:
		return false
	case DistributionCoreOS, DistributionFlatcar, DistributionContainerOS:
		return false
	default:
		klog.Fatalf("unknown distribution: %s", d)
		return false
	}
}

func (d Distribution) IsSystemd() bool {
	switch d {
	case DistributionJessie, DistributionXenial, DistributionBionic, DistributionDebian9, DistributionDebian10:
		return true
	case DistributionCentos7, DistributionRhel7, DistributionCentos8, DistributionRhel8:
		return true
	case DistributionCoreOS, DistributionFlatcar:
		return true
	case DistributionContainerOS:
		return true
	default:
		klog.Fatalf("unknown distribution: %s", d)
		return false
	}
}
<commit_msg>Add tag during isDebian check family<commit_after>/*
Copyright 2019 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package distros

import (
	"k8s.io/klog"
	"k8s.io/kops/upup/pkg/fi/nodeup/tags"
)

type Distribution string

var (
	DistributionJessie      Distribution = "jessie"
	DistributionDebian9     Distribution = "debian9"
	DistributionDebian10    Distribution = "buster"
	DistributionXenial      Distribution = "xenial"
	DistributionBionic      Distribution = "bionic"
	DistributionRhel7       Distribution = "rhel7"
	DistributionCentos7     Distribution = "centos7"
	DistributionRhel8       Distribution = "rhel8"
	DistributionCentos8     Distribution = "centos8"
	DistributionCoreOS      Distribution = "coreos"
	DistributionFlatcar     Distribution = "flatcar"
	DistributionContainerOS Distribution = "containeros"
)

func (d Distribution) BuildTags() []string {
	var t []string

	switch d {
	case DistributionJessie:
		t = []string{"_jessie"}
	case DistributionDebian9, DistributionDebian10:
		t = []string{} // trying to move away from tags
	case DistributionXenial:
		t = []string{"_xenial"}
	case DistributionBionic:
		t = []string{"_bionic"}
	case DistributionCentos7:
		t = []string{"_centos7"}
	case DistributionRhel7:
		t = []string{"_rhel7"}
	case DistributionCentos8:
		t = []string{"_centos8"}
	case DistributionRhel8:
		t = []string{"_rhel8"}
	case DistributionCoreOS:
		t = []string{"_coreos"}
	case DistributionFlatcar:
		t = []string{"_flatcar"}
	case DistributionContainerOS:
		t = []string{"_containeros"}
	default:
		klog.Fatalf("unknown distribution: %s", d)
		return nil
	}

	if d.IsDebianFamily() {
		t = append(t, tags.TagOSFamilyDebian)
	}
	if d.IsRHELFamily() {
		t = append(t, tags.TagOSFamilyRHEL)
	}
	if d.IsSystemd() {
		t = append(t, tags.TagSystemd)
	}

	return t
}

func (d Distribution) IsDebianFamily() bool {
	switch d {
	case DistributionJessie, DistributionDebian9, DistributionDebian10:
		return true
	case DistributionXenial, DistributionBionic:
		return true
	case DistributionCentos7, DistributionRhel7:
		return false
	case DistributionCoreOS, DistributionFlatcar, DistributionContainerOS:
		return false
	default:
		klog.Fatalf("unknown distribution: %s", d)
		return false
	}
}

func (d Distribution) IsUbuntu() bool {
	switch d {
	case DistributionJessie, DistributionDebian9, DistributionDebian10:
		return false
	case DistributionXenial, DistributionBionic:
		return true
	case DistributionCentos7, DistributionRhel7, DistributionCentos8, DistributionRhel8:
		return false
	case DistributionCoreOS, DistributionFlatcar, DistributionContainerOS:
		return false
	default:
		klog.Fatalf("unknown distribution: %s", d)
		return false
	}
}

func (d Distribution) IsRHELFamily() bool {
	switch d {
	case DistributionCentos7, DistributionRhel7, DistributionCentos8, DistributionRhel8:
		return true
	case DistributionJessie, DistributionXenial, DistributionBionic, DistributionDebian9, DistributionDebian10:
		return false
	case DistributionCoreOS, DistributionFlatcar, DistributionContainerOS:
		return false
	default:
		klog.Fatalf("unknown distribution: %s", d)
		return false
	}
}

func (d Distribution) IsSystemd() bool {
	switch d {
	case DistributionJessie, DistributionXenial, DistributionBionic, DistributionDebian9, DistributionDebian10:
		return true
	case DistributionCentos7, DistributionRhel7, DistributionCentos8, DistributionRhel8:
		return true
	case DistributionCoreOS, DistributionFlatcar:
		return true
	case DistributionContainerOS:
		return true
	default:
		klog.Fatalf("unknown distribution: %s", d)
		return false
	}
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"io/ioutil"
	"net/http"
	"math"
	"math/rand"
	"os"
	"os/exec"
	"runtime"
	"strconv"
	"strings"
	"sync"
	"time"
	)

func main() {
	// Constants
	const MaxInt = int(^uint(0) >> 1)

	// Flags
	curlFlag := flag.Bool("curl", false, "switch to `curl` for requests (from the Go `net/http` package)")
	workersFlag := flag.Int("workers", 8, "number of concurrent worker processes")
	lengthFlag := flag.Int("length", 10000, "number of requests per target per worker")
	flag.Parse()

	// Using the --curl flag switches to `curl` for requests (from the Go `net/http` package)
	curl := *curlFlag

	// Workers are the number of concurrent processes used to request target URLs
	workers := workersFlag

	// Length is the number of requests per target per worker
	var length int64
	if *lengthFlag == 0 {
		length = int64(MaxInt)
	} else {
		length = int64(*lengthFlag)
	}

	// Targets are the target URLs to cURL
	var targetsArg string
	var targets []string
	if len(flag.Args()) == 0 {
		flag.Usage()
		os.Exit(1)
	} else {
		targetsArg = flag.Args()[0]
		st := strings.Split(targetsArg, ",")
		for _, v := range st {
			targets = append(targets, strings.TrimSpace(v))
		}
	}

	// Review
	fmt.Println("stormbringer=start workers=" + strconv.Itoa(*workers), "length=" + strconv.FormatInt(length, 10), "targets=\"" + targetsArg + "\"")

	// Spawn `workers` goroutines
	// http://www.goinggo.net/2014/01/concurrency-goroutines-and-gomaxprocs.html
	runtime.GOMAXPROCS(*workers)
  var wg sync.WaitGroup
  for worker := 1; worker <= *workers; worker ++ {
    wg.Add(1)
    go func(worker int) {
			loadGen(curl, worker, length, targets)
      wg.Done()
    }(worker)
  }
	wg.Wait()

	fmt.Println("stormbringer=end")
	time.Sleep(time.Hour * 24 * 365)

}

// Iterate through `targets` for `length` cycles
func loadGen(curl bool, worker int, length int64, targets []string) {

	var iteration int64
	for iteration = 1; iteration <= length; iteration ++ {

		rand.Seed(time.Now().UnixNano())
    shuffle(targets)

		for _, value := range targets {

			if curl == true {

				// curl version
				cmd := exec.Command(
					"curl",
					"-sSLw",
					"worker=" + strconv.Itoa(worker) + " iteration=" + strconv.FormatInt(iteration, 10) + " target=\"%{url_effective}\" status=%{http_code} total_time=%{time_total} time_connect=%{time_connect} time_start=%{time_starttransfer}\n",
					value,
					"-o",
					"/dev/null",
					)
				out, err := cmd.Output()

				if err != nil {
					fmt.Println(err.Error())
				return
				}

				fmt.Print(string(out))

			} else {

				// Go `net/http` version
				start_time := time.Now()

				resp, err := http.Get(value)

		    if err != nil {
					fmt.Println(err.Error())
	        os.Exit(1)
		    } else {
					_, err := ioutil.ReadAll(resp.Body)
	        if err != nil {
						fmt.Println(err.Error())
	          os.Exit(1)
	        }

					end_time := time.Now()

	        // fmt.Println(string(body))
					fmt.Println(
						"worker=" + strconv.Itoa(worker),
						"iteration=" + strconv.FormatInt(iteration, 10),
						"target=\"" + value + "\"",
						"status=" + strconv.Itoa(resp.StatusCode),
						"total_time=" + strconv.FormatFloat(timer(start_time, end_time), 'f', 3, 64),
					)

				}
				resp.Body.Close()
	    }

		}

	}

}

// Shuffle slice elements
// http://marcelom.github.io/2013/06/07/goshuffle.html
func shuffle(a []string) {
    for i := range a {
        j := rand.Intn(i + 1)
        a[i], a[j] = a[j], a[i]
    }
}

// Timer
func timer(start time.Time, end time.Time) float64 {
	// time.Since(start)
	elapsed := end.Sub(start)
	return toFixed(elapsed.Seconds(), 3)
}

// `float64` truncation
// http://stackoverflow.com/a/29786394
func round(num float64) int {
  return int(num + math.Copysign(0.5, num))
}

func toFixed(num float64, precision int) float64 {
  output := math.Pow(10, float64(precision))
  return float64(round(num * output)) / output
}
<commit_msg>cleaned up formatting<commit_after>package main

import (
	"flag"
	"fmt"
	"io/ioutil"
	"math"
	"math/rand"
	"net/http"
	"os"
	"os/exec"
	"runtime"
	"strconv"
	"strings"
	"sync"
	"time"
)

func main() {
	// Constants
	const MaxInt = int(^uint(0) >> 1)

	// Flags
	curlFlag := flag.Bool("curl", false, "switch to `curl` for requests (from the Go `net/http` package)")
	workersFlag := flag.Int("workers", 8, "number of concurrent worker processes")
	lengthFlag := flag.Int("length", 10000, "number of requests per target per worker")
	flag.Parse()

	// Using the --curl flag switches to `curl` for requests (from the Go `net/http` package)
	curl := *curlFlag

	// Workers are the number of concurrent processes used to request target URLs
	workers := workersFlag

	// Length is the number of requests per target per worker
	var length int64
	if *lengthFlag == 0 {
		length = int64(MaxInt)
	} else {
		length = int64(*lengthFlag)
	}

	// Targets are the target URLs to cURL
	var targetsArg string
	var targets []string
	if len(flag.Args()) == 0 {
		flag.Usage()
		os.Exit(1)
	} else {
		targetsArg = flag.Args()[0]
		st := strings.Split(targetsArg, ",")
		for _, v := range st {
			targets = append(targets, strings.TrimSpace(v))
		}
	}

	// Review
	fmt.Println("stormbringer=start workers="+strconv.Itoa(*workers), "length="+strconv.FormatInt(length, 10), "targets=\""+targetsArg+"\"")

	// Spawn `workers` goroutines
	// http://www.goinggo.net/2014/01/concurrency-goroutines-and-gomaxprocs.html
	runtime.GOMAXPROCS(*workers)
	var wg sync.WaitGroup
	for worker := 1; worker <= *workers; worker++ {
		wg.Add(1)
		go func(worker int) {
			loadGen(curl, worker, length, targets)
			wg.Done()
		}(worker)
	}
	wg.Wait()

	fmt.Println("stormbringer=end")
	time.Sleep(time.Hour * 24 * 365)

}

// Iterate through `targets` for `length` cycles
func loadGen(curl bool, worker int, length int64, targets []string) {

	var iteration int64
	for iteration = 1; iteration <= length; iteration++ {

		rand.Seed(time.Now().UnixNano())
		shuffle(targets)

		for _, value := range targets {

			if curl == true {

				// curl version
				cmd := exec.Command(
					"curl",
					"-sSLw",
					"worker="+strconv.Itoa(worker)+" iteration="+strconv.FormatInt(iteration, 10)+" target=\"%{url_effective}\" status=%{http_code} total_time=%{time_total} time_connect=%{time_connect} time_start=%{time_starttransfer}\n",
					value,
					"-o",
					"/dev/null",
				)
				out, err := cmd.Output()

				if err != nil {
					fmt.Println(err.Error())
					return
				}

				fmt.Print(string(out))

			} else {

				// Go `net/http` version
				start_time := time.Now()

				resp, err := http.Get(value)

				if err != nil {
					fmt.Println(err.Error())
					os.Exit(1)
				} else {
					_, err := ioutil.ReadAll(resp.Body)
					if err != nil {
						fmt.Println(err.Error())
						os.Exit(1)
					}

					end_time := time.Now()

					// fmt.Println(string(body))
					fmt.Println(
						"worker="+strconv.Itoa(worker),
						"iteration="+strconv.FormatInt(iteration, 10),
						"target=\""+value+"\"",
						"status="+strconv.Itoa(resp.StatusCode),
						"total_time="+strconv.FormatFloat(timer(start_time, end_time), 'f', 3, 64),
					)

				}
				resp.Body.Close()
			}

		}

	}

}

// Shuffle slice elements
// http://marcelom.github.io/2013/06/07/goshuffle.html
func shuffle(a []string) {
	for i := range a {
		j := rand.Intn(i + 1)
		a[i], a[j] = a[j], a[i]
	}
}

// Timer
func timer(start time.Time, end time.Time) float64 {
	// time.Since(start)
	elapsed := end.Sub(start)
	return toFixed(elapsed.Seconds(), 3)
}

// `float64` truncation
// http://stackoverflow.com/a/29786394
func round(num float64) int {
	return int(num + math.Copysign(0.5, num))
}

func toFixed(num float64, precision int) float64 {
	output := math.Pow(10, float64(precision))
	return float64(round(num*output)) / output
}
<|endoftext|>


<commit_before>// +build !windows

package daemon // import "github.com/docker/docker/daemon"

import (
	"context"
	"fmt"
	"os/exec"
	"path/filepath"
	"strings"

	"github.com/docker/docker/api/types"
	containertypes "github.com/docker/docker/api/types/container"
	"github.com/docker/docker/dockerversion"
	"github.com/docker/docker/pkg/sysinfo"
	"github.com/pkg/errors"
	"github.com/sirupsen/logrus"
)

// fillPlatformInfo fills the platform related info.
func (daemon *Daemon) fillPlatformInfo(v *types.Info, sysInfo *sysinfo.SysInfo) {
	v.CgroupDriver = daemon.getCgroupDriver()
	v.CgroupVersion = "1"
	if sysInfo.CgroupUnified {
		v.CgroupVersion = "2"
	}

	v.MemoryLimit = sysInfo.MemoryLimit
	v.SwapLimit = sysInfo.SwapLimit
	v.KernelMemory = sysInfo.KernelMemory
	v.KernelMemoryTCP = sysInfo.KernelMemoryTCP
	v.OomKillDisable = sysInfo.OomKillDisable
	v.CPUCfsPeriod = sysInfo.CPUCfs
	v.CPUCfsQuota = sysInfo.CPUCfs
	v.CPUShares = sysInfo.CPUShares
	v.CPUSet = sysInfo.Cpuset
	v.PidsLimit = sysInfo.PidsLimit
	v.Runtimes = daemon.configStore.GetAllRuntimes()
	v.DefaultRuntime = daemon.configStore.GetDefaultRuntimeName()
	v.InitBinary = daemon.configStore.GetInitPath()

	defaultRuntimeBinary := daemon.configStore.GetRuntime(v.DefaultRuntime).Path
	if rv, err := exec.Command(defaultRuntimeBinary, "--version").Output(); err == nil {
		if _, _, commit, err := parseRuntimeVersion(string(rv)); err != nil {
			logrus.Warnf("failed to parse %s version: %v", defaultRuntimeBinary, err)
			v.RuncCommit.ID = "N/A"
		} else {
			v.RuncCommit.ID = commit
		}
	} else {
		logrus.Warnf("failed to retrieve %s version: %v", defaultRuntimeBinary, err)
		v.RuncCommit.ID = "N/A"
	}

	// runc is now shipped as a separate package. Set "expected" to same value
	// as "ID" to prevent clients from reporting a version-mismatch
	v.RuncCommit.Expected = v.RuncCommit.ID

	if rv, err := daemon.containerd.Version(context.Background()); err == nil {
		v.ContainerdCommit.ID = rv.Revision
	} else {
		logrus.Warnf("failed to retrieve containerd version: %v", err)
		v.ContainerdCommit.ID = "N/A"
	}

	// containerd is now shipped as a separate package. Set "expected" to same
	// value as "ID" to prevent clients from reporting a version-mismatch
	v.ContainerdCommit.Expected = v.ContainerdCommit.ID

	// TODO is there still a need to check the expected version for tini?
	// if not, we can change this, and just set "Expected" to v.InitCommit.ID
	v.InitCommit.Expected = dockerversion.InitCommitID

	defaultInitBinary := daemon.configStore.GetInitPath()
	if rv, err := exec.Command(defaultInitBinary, "--version").Output(); err == nil {
		if _, commit, err := parseInitVersion(string(rv)); err != nil {
			logrus.Warnf("failed to parse %s version: %s", defaultInitBinary, err)
			v.InitCommit.ID = "N/A"
		} else {
			v.InitCommit.ID = commit
			if len(dockerversion.InitCommitID) > len(commit) {
				v.InitCommit.Expected = dockerversion.InitCommitID[0:len(commit)]
			}
		}
	} else {
		logrus.Warnf("failed to retrieve %s version: %s", defaultInitBinary, err)
		v.InitCommit.ID = "N/A"
	}

	if v.CgroupDriver == cgroupNoneDriver {
		if v.CgroupVersion == "2" {
			v.Warnings = append(v.Warnings, "WARNING: Running in rootless-mode without cgroups. Systemd is required to enable cgroups in rootless-mode.")
		} else {
			v.Warnings = append(v.Warnings, "WARNING: Running in rootless-mode without cgroups. To enable cgroups in rootless-mode, you need to boot the system in cgroup v2 mode.")
		}
	} else {
		if !v.MemoryLimit {
			v.Warnings = append(v.Warnings, "WARNING: No memory limit support")
		}
		if !v.SwapLimit {
			v.Warnings = append(v.Warnings, "WARNING: No swap limit support")
		}
		if !v.KernelMemoryTCP && v.CgroupVersion == "1" {
			// kernel memory is not available for cgroup v2.
			// Warning is not printed on cgroup v2, because there is no action user can take.
			v.Warnings = append(v.Warnings, "WARNING: No kernel memory TCP limit support")
		}
		if !v.OomKillDisable && v.CgroupVersion == "1" {
			// oom kill disable is not available for cgroup v2.
			// Warning is not printed on cgroup v2, because there is no action user can take.
			v.Warnings = append(v.Warnings, "WARNING: No oom kill disable support")
		}
		if !v.CPUCfsQuota {
			v.Warnings = append(v.Warnings, "WARNING: No cpu cfs quota support")
		}
		if !v.CPUCfsPeriod {
			v.Warnings = append(v.Warnings, "WARNING: No cpu cfs period support")
		}
		if !v.CPUShares {
			v.Warnings = append(v.Warnings, "WARNING: No cpu shares support")
		}
		if !v.CPUSet {
			v.Warnings = append(v.Warnings, "WARNING: No cpuset support")
		}
		if v.CgroupVersion == "2" {
			v.Warnings = append(v.Warnings, "WARNING: Support for cgroup v2 is experimental")
		}
		// TODO add fields for these options in types.Info
		if !sysInfo.BlkioWeight && v.CgroupVersion == "2" {
			// blkio weight is not available on cgroup v1 since kernel 5.0.
			// Warning is not printed on cgroup v1, because there is no action user can take.
			// On cgroup v2, blkio weight is implemented using io.weight
			v.Warnings = append(v.Warnings, "WARNING: No blkio weight support")
		}
		if !sysInfo.BlkioWeightDevice && v.CgroupVersion == "2" {
			v.Warnings = append(v.Warnings, "WARNING: No blkio weight_device support")
		}
		if !sysInfo.BlkioReadBpsDevice {
			v.Warnings = append(v.Warnings, "WARNING: No blkio throttle.read_bps_device support")
		}
		if !sysInfo.BlkioWriteBpsDevice {
			v.Warnings = append(v.Warnings, "WARNING: No blkio throttle.write_bps_device support")
		}
		if !sysInfo.BlkioReadIOpsDevice {
			v.Warnings = append(v.Warnings, "WARNING: No blkio throttle.read_iops_device support")
		}
		if !sysInfo.BlkioWriteIOpsDevice {
			v.Warnings = append(v.Warnings, "WARNING: No blkio throttle.write_iops_device support")
		}
	}
	if !v.IPv4Forwarding {
		v.Warnings = append(v.Warnings, "WARNING: IPv4 forwarding is disabled")
	}
	if !v.BridgeNfIptables {
		v.Warnings = append(v.Warnings, "WARNING: bridge-nf-call-iptables is disabled")
	}
	if !v.BridgeNfIP6tables {
		v.Warnings = append(v.Warnings, "WARNING: bridge-nf-call-ip6tables is disabled")
	}
}

func (daemon *Daemon) fillPlatformVersion(v *types.Version) {
	if rv, err := daemon.containerd.Version(context.Background()); err == nil {
		v.Components = append(v.Components, types.ComponentVersion{
			Name:    "containerd",
			Version: rv.Version,
			Details: map[string]string{
				"GitCommit": rv.Revision,
			},
		})
	}

	defaultRuntime := daemon.configStore.GetDefaultRuntimeName()
	defaultRuntimeBinary := daemon.configStore.GetRuntime(defaultRuntime).Path
	if rv, err := exec.Command(defaultRuntimeBinary, "--version").Output(); err == nil {
		if _, ver, commit, err := parseRuntimeVersion(string(rv)); err != nil {
			logrus.Warnf("failed to parse %s version: %v", defaultRuntimeBinary, err)
		} else {
			v.Components = append(v.Components, types.ComponentVersion{
				Name:    defaultRuntime,
				Version: ver,
				Details: map[string]string{
					"GitCommit": commit,
				},
			})
		}
	} else {
		logrus.Warnf("failed to retrieve %s version: %v", defaultRuntimeBinary, err)
	}

	defaultInitBinary := daemon.configStore.GetInitPath()
	if rv, err := exec.Command(defaultInitBinary, "--version").Output(); err == nil {
		if ver, commit, err := parseInitVersion(string(rv)); err != nil {
			logrus.Warnf("failed to parse %s version: %s", defaultInitBinary, err)
		} else {
			v.Components = append(v.Components, types.ComponentVersion{
				Name:    filepath.Base(defaultInitBinary),
				Version: ver,
				Details: map[string]string{
					"GitCommit": commit,
				},
			})
		}
	} else {
		logrus.Warnf("failed to retrieve %s version: %s", defaultInitBinary, err)
	}
}

func fillDriverWarnings(v *types.Info) {
	for _, pair := range v.DriverStatus {
		if pair[0] == "Data loop file" {
			msg := fmt.Sprintf("WARNING: %s: usage of loopback devices is "+
				"strongly discouraged for production use.\n         "+
				"Use `--storage-opt dm.thinpooldev` to specify a custom block storage device.", v.Driver)

			v.Warnings = append(v.Warnings, msg)
			continue
		}
		if pair[0] == "Supports d_type" && pair[1] == "false" {
			backingFs := getBackingFs(v)

			msg := fmt.Sprintf("WARNING: %s: the backing %s filesystem is formatted without d_type support, which leads to incorrect behavior.\n", v.Driver, backingFs)
			if backingFs == "xfs" {
				msg += "         Reformat the filesystem with ftype=1 to enable d_type support.\n"
			}
			msg += "         Running without d_type support will not be supported in future releases."

			v.Warnings = append(v.Warnings, msg)
			continue
		}
	}
}

func getBackingFs(v *types.Info) string {
	for _, pair := range v.DriverStatus {
		if pair[0] == "Backing Filesystem" {
			return pair[1]
		}
	}
	return ""
}

// parseInitVersion parses a Tini version string, and extracts the "version"
// and "git commit" from the output.
//
// Output example from `docker-init --version`:
//
//     tini version 0.18.0 - git.fec3683
func parseInitVersion(v string) (version string, commit string, err error) {
	parts := strings.Split(v, " - ")

	if len(parts) >= 2 {
		gitParts := strings.Split(strings.TrimSpace(parts[1]), ".")
		if len(gitParts) == 2 && gitParts[0] == "git" {
			commit = gitParts[1]
		}
	}
	parts[0] = strings.TrimSpace(parts[0])
	if strings.HasPrefix(parts[0], "tini version ") {
		version = strings.TrimPrefix(parts[0], "tini version ")
	}
	if version == "" && commit == "" {
		err = errors.Errorf("unknown output format: %s", v)
	}
	return version, commit, err
}

// parseRuntimeVersion parses the output of `[runtime] --version` and extracts the
// "name", "version" and "git commit" from the output.
//
// Output example from `runc --version`:
//
//   runc version 1.0.0-rc5+dev
//   commit: 69663f0bd4b60df09991c08812a60108003fa340
//   spec: 1.0.0
func parseRuntimeVersion(v string) (runtime string, version string, commit string, err error) {
	lines := strings.Split(strings.TrimSpace(v), "\n")
	for _, line := range lines {
		if strings.Contains(line, "version") {
			s := strings.Split(line, "version")
			runtime = strings.TrimSpace(s[0])
			version = strings.TrimSpace(s[len(s)-1])
			continue
		}
		if strings.HasPrefix(line, "commit:") {
			commit = strings.TrimSpace(strings.TrimPrefix(line, "commit:"))
			continue
		}
	}
	if version == "" && commit == "" {
		err = errors.Errorf("unknown output format: %s", v)
	}
	return runtime, version, commit, err
}

func (daemon *Daemon) cgroupNamespacesEnabled(sysInfo *sysinfo.SysInfo) bool {
	return sysInfo.CgroupNamespaces && containertypes.CgroupnsMode(daemon.configStore.CgroupNamespaceMode).IsPrivate()
}

// Rootless returns true if daemon is running in rootless mode
func (daemon *Daemon) Rootless() bool {
	return daemon.configStore.Rootless
}
<commit_msg>docker info: adjust warning strings for cgroup v2<commit_after>// +build !windows

package daemon // import "github.com/docker/docker/daemon"

import (
	"context"
	"fmt"
	"os/exec"
	"path/filepath"
	"strings"

	"github.com/docker/docker/api/types"
	containertypes "github.com/docker/docker/api/types/container"
	"github.com/docker/docker/dockerversion"
	"github.com/docker/docker/pkg/sysinfo"
	"github.com/pkg/errors"
	"github.com/sirupsen/logrus"
)

// fillPlatformInfo fills the platform related info.
func (daemon *Daemon) fillPlatformInfo(v *types.Info, sysInfo *sysinfo.SysInfo) {
	v.CgroupDriver = daemon.getCgroupDriver()
	v.CgroupVersion = "1"
	if sysInfo.CgroupUnified {
		v.CgroupVersion = "2"
	}

	v.MemoryLimit = sysInfo.MemoryLimit
	v.SwapLimit = sysInfo.SwapLimit
	v.KernelMemory = sysInfo.KernelMemory
	v.KernelMemoryTCP = sysInfo.KernelMemoryTCP
	v.OomKillDisable = sysInfo.OomKillDisable
	v.CPUCfsPeriod = sysInfo.CPUCfs
	v.CPUCfsQuota = sysInfo.CPUCfs
	v.CPUShares = sysInfo.CPUShares
	v.CPUSet = sysInfo.Cpuset
	v.PidsLimit = sysInfo.PidsLimit
	v.Runtimes = daemon.configStore.GetAllRuntimes()
	v.DefaultRuntime = daemon.configStore.GetDefaultRuntimeName()
	v.InitBinary = daemon.configStore.GetInitPath()

	defaultRuntimeBinary := daemon.configStore.GetRuntime(v.DefaultRuntime).Path
	if rv, err := exec.Command(defaultRuntimeBinary, "--version").Output(); err == nil {
		if _, _, commit, err := parseRuntimeVersion(string(rv)); err != nil {
			logrus.Warnf("failed to parse %s version: %v", defaultRuntimeBinary, err)
			v.RuncCommit.ID = "N/A"
		} else {
			v.RuncCommit.ID = commit
		}
	} else {
		logrus.Warnf("failed to retrieve %s version: %v", defaultRuntimeBinary, err)
		v.RuncCommit.ID = "N/A"
	}

	// runc is now shipped as a separate package. Set "expected" to same value
	// as "ID" to prevent clients from reporting a version-mismatch
	v.RuncCommit.Expected = v.RuncCommit.ID

	if rv, err := daemon.containerd.Version(context.Background()); err == nil {
		v.ContainerdCommit.ID = rv.Revision
	} else {
		logrus.Warnf("failed to retrieve containerd version: %v", err)
		v.ContainerdCommit.ID = "N/A"
	}

	// containerd is now shipped as a separate package. Set "expected" to same
	// value as "ID" to prevent clients from reporting a version-mismatch
	v.ContainerdCommit.Expected = v.ContainerdCommit.ID

	// TODO is there still a need to check the expected version for tini?
	// if not, we can change this, and just set "Expected" to v.InitCommit.ID
	v.InitCommit.Expected = dockerversion.InitCommitID

	defaultInitBinary := daemon.configStore.GetInitPath()
	if rv, err := exec.Command(defaultInitBinary, "--version").Output(); err == nil {
		if _, commit, err := parseInitVersion(string(rv)); err != nil {
			logrus.Warnf("failed to parse %s version: %s", defaultInitBinary, err)
			v.InitCommit.ID = "N/A"
		} else {
			v.InitCommit.ID = commit
			if len(dockerversion.InitCommitID) > len(commit) {
				v.InitCommit.Expected = dockerversion.InitCommitID[0:len(commit)]
			}
		}
	} else {
		logrus.Warnf("failed to retrieve %s version: %s", defaultInitBinary, err)
		v.InitCommit.ID = "N/A"
	}

	if v.CgroupDriver == cgroupNoneDriver {
		if v.CgroupVersion == "2" {
			v.Warnings = append(v.Warnings, "WARNING: Running in rootless-mode without cgroups. Systemd is required to enable cgroups in rootless-mode.")
		} else {
			v.Warnings = append(v.Warnings, "WARNING: Running in rootless-mode without cgroups. To enable cgroups in rootless-mode, you need to boot the system in cgroup v2 mode.")
		}
	} else {
		if !v.MemoryLimit {
			v.Warnings = append(v.Warnings, "WARNING: No memory limit support")
		}
		if !v.SwapLimit {
			v.Warnings = append(v.Warnings, "WARNING: No swap limit support")
		}
		if !v.KernelMemoryTCP && v.CgroupVersion == "1" {
			// kernel memory is not available for cgroup v2.
			// Warning is not printed on cgroup v2, because there is no action user can take.
			v.Warnings = append(v.Warnings, "WARNING: No kernel memory TCP limit support")
		}
		if !v.OomKillDisable && v.CgroupVersion == "1" {
			// oom kill disable is not available for cgroup v2.
			// Warning is not printed on cgroup v2, because there is no action user can take.
			v.Warnings = append(v.Warnings, "WARNING: No oom kill disable support")
		}
		if !v.CPUCfsQuota {
			v.Warnings = append(v.Warnings, "WARNING: No cpu cfs quota support")
		}
		if !v.CPUCfsPeriod {
			v.Warnings = append(v.Warnings, "WARNING: No cpu cfs period support")
		}
		if !v.CPUShares {
			v.Warnings = append(v.Warnings, "WARNING: No cpu shares support")
		}
		if !v.CPUSet {
			v.Warnings = append(v.Warnings, "WARNING: No cpuset support")
		}
		if v.CgroupVersion == "2" {
			v.Warnings = append(v.Warnings, "WARNING: Support for cgroup v2 is experimental")
		}
		// TODO add fields for these options in types.Info
		if !sysInfo.BlkioWeight && v.CgroupVersion == "2" {
			// blkio weight is not available on cgroup v1 since kernel 5.0.
			// Warning is not printed on cgroup v1, because there is no action user can take.
			// On cgroup v2, blkio weight is implemented using io.weight
			v.Warnings = append(v.Warnings, "WARNING: No io.weight support")
		}
		if !sysInfo.BlkioWeightDevice && v.CgroupVersion == "2" {
			v.Warnings = append(v.Warnings, "WARNING: No io.weight (per device) support")
		}
		if !sysInfo.BlkioReadBpsDevice {
			if v.CgroupVersion == "2" {
				v.Warnings = append(v.Warnings, "WARNING: No io.max (rbps) support")
			} else {
				v.Warnings = append(v.Warnings, "WARNING: No blkio throttle.read_bps_device support")
			}
		}
		if !sysInfo.BlkioWriteBpsDevice {
			if v.CgroupVersion == "2" {
				v.Warnings = append(v.Warnings, "WARNING: No io.max (wbps) support")
			} else {
				v.Warnings = append(v.Warnings, "WARNING: No blkio throttle.write_bps_device support")
			}
		}
		if !sysInfo.BlkioReadIOpsDevice {
			if v.CgroupVersion == "2" {
				v.Warnings = append(v.Warnings, "WARNING: No io.max (riops) support")
			} else {
				v.Warnings = append(v.Warnings, "WARNING: No blkio throttle.read_iops_device support")
			}
		}
		if !sysInfo.BlkioWriteIOpsDevice {
			if v.CgroupVersion == "2" {
				v.Warnings = append(v.Warnings, "WARNING: No io.max (wiops) support")
			} else {
				v.Warnings = append(v.Warnings, "WARNING: No blkio throttle.write_iops_device support")
			}
		}
	}
	if !v.IPv4Forwarding {
		v.Warnings = append(v.Warnings, "WARNING: IPv4 forwarding is disabled")
	}
	if !v.BridgeNfIptables {
		v.Warnings = append(v.Warnings, "WARNING: bridge-nf-call-iptables is disabled")
	}
	if !v.BridgeNfIP6tables {
		v.Warnings = append(v.Warnings, "WARNING: bridge-nf-call-ip6tables is disabled")
	}
}

func (daemon *Daemon) fillPlatformVersion(v *types.Version) {
	if rv, err := daemon.containerd.Version(context.Background()); err == nil {
		v.Components = append(v.Components, types.ComponentVersion{
			Name:    "containerd",
			Version: rv.Version,
			Details: map[string]string{
				"GitCommit": rv.Revision,
			},
		})
	}

	defaultRuntime := daemon.configStore.GetDefaultRuntimeName()
	defaultRuntimeBinary := daemon.configStore.GetRuntime(defaultRuntime).Path
	if rv, err := exec.Command(defaultRuntimeBinary, "--version").Output(); err == nil {
		if _, ver, commit, err := parseRuntimeVersion(string(rv)); err != nil {
			logrus.Warnf("failed to parse %s version: %v", defaultRuntimeBinary, err)
		} else {
			v.Components = append(v.Components, types.ComponentVersion{
				Name:    defaultRuntime,
				Version: ver,
				Details: map[string]string{
					"GitCommit": commit,
				},
			})
		}
	} else {
		logrus.Warnf("failed to retrieve %s version: %v", defaultRuntimeBinary, err)
	}

	defaultInitBinary := daemon.configStore.GetInitPath()
	if rv, err := exec.Command(defaultInitBinary, "--version").Output(); err == nil {
		if ver, commit, err := parseInitVersion(string(rv)); err != nil {
			logrus.Warnf("failed to parse %s version: %s", defaultInitBinary, err)
		} else {
			v.Components = append(v.Components, types.ComponentVersion{
				Name:    filepath.Base(defaultInitBinary),
				Version: ver,
				Details: map[string]string{
					"GitCommit": commit,
				},
			})
		}
	} else {
		logrus.Warnf("failed to retrieve %s version: %s", defaultInitBinary, err)
	}
}

func fillDriverWarnings(v *types.Info) {
	for _, pair := range v.DriverStatus {
		if pair[0] == "Data loop file" {
			msg := fmt.Sprintf("WARNING: %s: usage of loopback devices is "+
				"strongly discouraged for production use.\n         "+
				"Use `--storage-opt dm.thinpooldev` to specify a custom block storage device.", v.Driver)

			v.Warnings = append(v.Warnings, msg)
			continue
		}
		if pair[0] == "Supports d_type" && pair[1] == "false" {
			backingFs := getBackingFs(v)

			msg := fmt.Sprintf("WARNING: %s: the backing %s filesystem is formatted without d_type support, which leads to incorrect behavior.\n", v.Driver, backingFs)
			if backingFs == "xfs" {
				msg += "         Reformat the filesystem with ftype=1 to enable d_type support.\n"
			}
			msg += "         Running without d_type support will not be supported in future releases."

			v.Warnings = append(v.Warnings, msg)
			continue
		}
	}
}

func getBackingFs(v *types.Info) string {
	for _, pair := range v.DriverStatus {
		if pair[0] == "Backing Filesystem" {
			return pair[1]
		}
	}
	return ""
}

// parseInitVersion parses a Tini version string, and extracts the "version"
// and "git commit" from the output.
//
// Output example from `docker-init --version`:
//
//     tini version 0.18.0 - git.fec3683
func parseInitVersion(v string) (version string, commit string, err error) {
	parts := strings.Split(v, " - ")

	if len(parts) >= 2 {
		gitParts := strings.Split(strings.TrimSpace(parts[1]), ".")
		if len(gitParts) == 2 && gitParts[0] == "git" {
			commit = gitParts[1]
		}
	}
	parts[0] = strings.TrimSpace(parts[0])
	if strings.HasPrefix(parts[0], "tini version ") {
		version = strings.TrimPrefix(parts[0], "tini version ")
	}
	if version == "" && commit == "" {
		err = errors.Errorf("unknown output format: %s", v)
	}
	return version, commit, err
}

// parseRuntimeVersion parses the output of `[runtime] --version` and extracts the
// "name", "version" and "git commit" from the output.
//
// Output example from `runc --version`:
//
//   runc version 1.0.0-rc5+dev
//   commit: 69663f0bd4b60df09991c08812a60108003fa340
//   spec: 1.0.0
func parseRuntimeVersion(v string) (runtime string, version string, commit string, err error) {
	lines := strings.Split(strings.TrimSpace(v), "\n")
	for _, line := range lines {
		if strings.Contains(line, "version") {
			s := strings.Split(line, "version")
			runtime = strings.TrimSpace(s[0])
			version = strings.TrimSpace(s[len(s)-1])
			continue
		}
		if strings.HasPrefix(line, "commit:") {
			commit = strings.TrimSpace(strings.TrimPrefix(line, "commit:"))
			continue
		}
	}
	if version == "" && commit == "" {
		err = errors.Errorf("unknown output format: %s", v)
	}
	return runtime, version, commit, err
}

func (daemon *Daemon) cgroupNamespacesEnabled(sysInfo *sysinfo.SysInfo) bool {
	return sysInfo.CgroupNamespaces && containertypes.CgroupnsMode(daemon.configStore.CgroupNamespaceMode).IsPrivate()
}

// Rootless returns true if daemon is running in rootless mode
func (daemon *Daemon) Rootless() bool {
	return daemon.configStore.Rootless
}
<|endoftext|>


<commit_before>package spec

import (
	"bytes"
	"context"
	"encoding/binary"
	"encoding/hex"
	"fmt"
	"io/ioutil"
	"strings"
	"time"

	"github.com/Songmu/retry"
)

var uuidFiles = [2]string{
	"/sys/hypervisor/uuid",
	"/sys/devices/virtual/dmi/id/product_uuid",
}

// If the OS is Linux, check /sys/hypervisor/uuid and /sys/devices/virtual/dmi/id/product_uuid files first. If UUID seems to be EC2-ish, call the metadata API (up to 3 times).
// ref. https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/identify_ec2_instances.html
func isEC2(ctx context.Context) bool {
	looksLikeEC2 := false
	for _, u := range uuidFiles {
		data, err := ioutil.ReadFile(u)
		if err != nil {
			continue
		}
		if isEC2UUID(string(data)) {
			looksLikeEC2 = true
			break
		}
	}
	if !looksLikeEC2 {
		return false
	}

	// give up if ctx already closed
	if ctx.Err() != nil {
		return false
	}

	res := false
	cl := httpCli()
	err := retry.Retry(3, 2*time.Second, func() error {
		// '/ami-id` is probably an AWS specific URL
		req, err := http.NewRequest("GET", ec2BaseURL.String()+"/ami-id", nil)
		if err != nil {
			return nil // something wrong. give up
		}
		resp, err := cl.Do(req.WithContext(ctx))
		if err != nil {
			return err
		}
		defer resp.Body.Close()

		res = resp.StatusCode == 200
		return nil
	})

	if err == nil {
		return res
	}

	return false
}

func isEC2UUID(uuid string) bool {
	conds := func(uuid string) bool {
		if strings.HasPrefix(uuid, "ec2") || strings.HasPrefix(uuid, "EC2") {
			return true
		}
		return false
	}

	if conds(uuid) {
		return true
	}

	// Check as littele endian.
	// see. https://docs.aws.amazon.com/ja_jp/AWSEC2/latest/UserGuide/identify_ec2_instances.html
	fields := strings.Split(uuid, "-")
	decoded, _ := hex.DecodeString(fields[0]) // fields[0]: UUID time_low(uint32)
	r := bytes.NewReader(decoded)
	var data uint32
	binary.Read(r, binary.LittleEndian, &data)

	return conds(fmt.Sprintf("%x", data))
}
<commit_msg>import<commit_after>package spec

import (
	"bytes"
	"context"
	"encoding/binary"
	"encoding/hex"
	"fmt"
	"io/ioutil"
	"net/http"
	"strings"
	"time"

	"github.com/Songmu/retry"
)

var uuidFiles = [2]string{
	"/sys/hypervisor/uuid",
	"/sys/devices/virtual/dmi/id/product_uuid",
}

// If the OS is Linux, check /sys/hypervisor/uuid and /sys/devices/virtual/dmi/id/product_uuid files first. If UUID seems to be EC2-ish, call the metadata API (up to 3 times).
// ref. https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/identify_ec2_instances.html
func isEC2(ctx context.Context) bool {
	looksLikeEC2 := false
	for _, u := range uuidFiles {
		data, err := ioutil.ReadFile(u)
		if err != nil {
			continue
		}
		if isEC2UUID(string(data)) {
			looksLikeEC2 = true
			break
		}
	}
	if !looksLikeEC2 {
		return false
	}

	// give up if ctx already closed
	if ctx.Err() != nil {
		return false
	}

	res := false
	cl := httpCli()
	err := retry.Retry(3, 2*time.Second, func() error {
		// '/ami-id` is probably an AWS specific URL
		req, err := http.NewRequest("GET", ec2BaseURL.String()+"/ami-id", nil)
		if err != nil {
			return nil // something wrong. give up
		}
		resp, err := cl.Do(req.WithContext(ctx))
		if err != nil {
			return err
		}
		defer resp.Body.Close()

		res = resp.StatusCode == 200
		return nil
	})

	if err == nil {
		return res
	}

	return false
}

func isEC2UUID(uuid string) bool {
	conds := func(uuid string) bool {
		if strings.HasPrefix(uuid, "ec2") || strings.HasPrefix(uuid, "EC2") {
			return true
		}
		return false
	}

	if conds(uuid) {
		return true
	}

	// Check as littele endian.
	// see. https://docs.aws.amazon.com/ja_jp/AWSEC2/latest/UserGuide/identify_ec2_instances.html
	fields := strings.Split(uuid, "-")
	decoded, _ := hex.DecodeString(fields[0]) // fields[0]: UUID time_low(uint32)
	r := bytes.NewReader(decoded)
	var data uint32
	binary.Read(r, binary.LittleEndian, &data)

	return conds(fmt.Sprintf("%x", data))
}
<|endoftext|>


<commit_before>package statsd

import (
	"net"
	"math/rand"
	"fmt"
	"errors"
)

type StatsdClient struct {
	// connection of type interface net.Conn
	conn net.Conn
	// prefix for statsd name
	prefix string
}

// Increments a statsd count type
// stat is a string name for the metric
// value is the integer value
// rate is the sample rate (0.0 to 1.0)
func (s *StatsdClient) Inc(stat string, value int64, rate float32) error {
	dap := fmt.Sprintf("%d|c", value)
	return s.submit(stat, dap, rate)
}

// Decrements a statsd count type
// stat is a string name for the metric
// value is the integer value
// rate is the sample rate (0.0 to 1.0)
func (s *StatsdClient) Dec(stat string, value int64, rate float32) error {
	return s.Inc(stat, -value, rate)
}

// Submits/Updates a statsd guage type
// stat is a string name for the metric
// value is the integer value
// rate is the sample rate (0.0 to 1.0)
func (s *StatsdClient) Guage(stat string, value int64, rate float32) error {
	dap := fmt.Sprintf("%d|g", value)
	return s.submit(stat, dap, rate)
}

// Submits a statsd timing type
// stat is a string name for the metric
// value is the integer value
// rate is the sample rate (0.0 to 1.0)
func (s *StatsdClient) Timing(stat string, delta int64, rate float32) error {
	dap := fmt.Sprintf("%d|ms", delta)
	return s.submit(stat, dap, rate)
}

// Sets/Updates the statsd client prefix
func (s *StatsdClient) SetPrefix(prefix string) {
	s.prefix = prefix
}

// submit formats the statsd event data, handles sampling, and prepares it,
// and sends it to the server.
func (s *StatsdClient) submit(stat string, value string, rate float32) error {
	if rate < 1 {
		if rand.Float32() < rate {
			value = fmt.Sprintf("%s|@%f", value, rate)
		} else {
			return nil
		}
	}

	if s.prefix != "" {
		stat = fmt.Sprintf("%s.%s", s.prefix, stat)
	}

	data := fmt.Sprintf("%s:%s", stat, value)

	i, err := s.send([]byte(data))
	if err != nil {
		return err
	}
	if i == 0 {
		return errors.New("Wrote no bytes")
	}
	return nil
}

// sends the data to the server endpoint over the net.Conn
func (s *StatsdClient) send(data []byte) (int, error) {
	i, err := s.conn.Write([]byte(data))
	return i, err
}

// Returns a pointer to a new StatsdClient
// addr is a string of the format "hostname:port", and must be parsable by
// net.ResolveUDPAddr.
// prefix is the statsd client prefix. Can be "" if no prefix is desired.
func New(addr string, prefix string) (*StatsdClient, error) {
	udpaddr, err := net.ResolveUDPAddr("udp", addr)
	if err != nil {
		return nil, err
	}

	conn, err := net.DialUDP(udpaddr.Network(), nil, udpaddr)
	if err != nil {
		return nil, err
	}

	client := &StatsdClient{
		conn: conn,
		prefix: prefix}

	return client, nil
}
<commit_msg>comment formatting<commit_after>package statsd

import (
	"net"
	"math/rand"
	"fmt"
	"errors"
)

type StatsdClient struct {
	// connection of type interface net.Conn
	conn net.Conn
	// prefix for statsd name
	prefix string
}

// Increments a statsd count type.
// stat is a string name for the metric.
// value is the integer value
// rate is the sample rate (0.0 to 1.0)
func (s *StatsdClient) Inc(stat string, value int64, rate float32) error {
	dap := fmt.Sprintf("%d|c", value)
	return s.submit(stat, dap, rate)
}

// Decrements a statsd count type.
// stat is a string name for the metric.
// value is the integer value.
// rate is the sample rate (0.0 to 1.0).
func (s *StatsdClient) Dec(stat string, value int64, rate float32) error {
	return s.Inc(stat, -value, rate)
}

// Submits/Updates a statsd guage type.
// stat is a string name for the metric.
// value is the integer value.
// rate is the sample rate (0.0 to 1.0).
func (s *StatsdClient) Guage(stat string, value int64, rate float32) error {
	dap := fmt.Sprintf("%d|g", value)
	return s.submit(stat, dap, rate)
}

// Submits a statsd timing type.
// stat is a string name for the metric.
// value is the integer value.
// rate is the sample rate (0.0 to 1.0).
func (s *StatsdClient) Timing(stat string, delta int64, rate float32) error {
	dap := fmt.Sprintf("%d|ms", delta)
	return s.submit(stat, dap, rate)
}

// Sets/Updates the statsd client prefix
func (s *StatsdClient) SetPrefix(prefix string) {
	s.prefix = prefix
}

// submit formats the statsd event data, handles sampling, and prepares it,
// and sends it to the server.
func (s *StatsdClient) submit(stat string, value string, rate float32) error {
	if rate < 1 {
		if rand.Float32() < rate {
			value = fmt.Sprintf("%s|@%f", value, rate)
		} else {
			return nil
		}
	}

	if s.prefix != "" {
		stat = fmt.Sprintf("%s.%s", s.prefix, stat)
	}

	data := fmt.Sprintf("%s:%s", stat, value)

	i, err := s.send([]byte(data))
	if err != nil {
		return err
	}
	if i == 0 {
		return errors.New("Wrote no bytes")
	}
	return nil
}

// sends the data to the server endpoint over the net.Conn
func (s *StatsdClient) send(data []byte) (int, error) {
	i, err := s.conn.Write([]byte(data))
	return i, err
}

// Returns a pointer to a new StatsdClient.
// addr is a string of the format "hostname:port", and must be parsable by
// net.ResolveUDPAddr.
// prefix is the statsd client prefix. Can be "" if no prefix is desired.
func New(addr string, prefix string) (*StatsdClient, error) {
	udpaddr, err := net.ResolveUDPAddr("udp", addr)
	if err != nil {
		return nil, err
	}

	conn, err := net.DialUDP(udpaddr.Network(), nil, udpaddr)
	if err != nil {
		return nil, err
	}

	client := &StatsdClient{
		conn: conn,
		prefix: prefix}

	return client, nil
}
<|endoftext|>


<commit_before>// Copyright 2016, Joe Tsai. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE.md file.

package flate

import (
	"bytes"
	"io"
	"testing"

	// TODO(dsnet): We should not be relying on the standard library for the
	// round-trip test.
	"compress/flate"

	"github.com/dsnet/compress/internal/testutil"
)

var testdata = []struct {
	name string
	data []byte
}{
	{"Nil", nil},
	{"Binary", testutil.MustLoadFile("../testdata/binary.bin")},
	{"Digits", testutil.MustLoadFile("../testdata/digits.txt")},
	{"Huffman", testutil.MustLoadFile("../testdata/huffman.txt")},
	{"Random", testutil.MustLoadFile("../testdata/random.bin")},
	{"Repeats", testutil.MustLoadFile("../testdata/repeats.bin")},
	{"Twain", testutil.MustLoadFile("../testdata/twain.txt")},
	{"Zeros", testutil.MustLoadFile("../testdata/zeros.bin")},
}

var levels = []struct {
	name  string
	level int
}{
	{"Huffman", flate.HuffmanOnly},
	{"Speed", flate.BestSpeed},
	{"Default", flate.DefaultCompression},
	{"Compression", flate.BestCompression},
}

var sizes = []struct {
	name string
	size int
}{
	{"1e4", 1e4},
	{"1e5", 1e5},
	{"1e6", 1e6},
}

func TestRoundTrip(t *testing.T) {
	for i, v := range testdata {
		var buf1, buf2 bytes.Buffer

		// Compress the input.
		wr, err := flate.NewWriter(&buf1, flate.DefaultCompression)
		if err != nil {
			t.Errorf("test %d, NewWriter() = (_, %v), want (_, nil)", i, err)
		}
		n, err := io.Copy(wr, bytes.NewReader(v.data))
		if n != int64(len(v.data)) || err != nil {
			t.Errorf("test %d, Copy() = (%d, %v), want (%d, nil)", i, n, err, len(v.data))
		}
		if err := wr.Close(); err != nil {
			t.Errorf("test %d, Close() = %v, want nil", i, err)
		}

		// Write a canary byte to ensure this does not get read.
		buf1.WriteByte(0x7a)

		// Decompress the output.
		rd, err := NewReader(&buf1, nil)
		if err != nil {
			t.Errorf("test %d, NewReader() = (_, %v), want (_, nil)", i, err)
		}
		n, err = io.Copy(&buf2, rd)
		if n != int64(len(v.data)) || err != nil {
			t.Errorf("test %d, Copy() = (%d, %v), want (%d, nil)", i, n, err, len(v.data))
		}
		if err := rd.Close(); err != nil {
			t.Errorf("test %d, Close() = %v, want nil", i, err)
		}
		if !bytes.Equal(buf2.Bytes(), v.data) {
			t.Errorf("test %d, output data mismatch", i)
		}

		// Read back the canary byte.
		if v, _ := buf1.ReadByte(); v != 0x7a {
			t.Errorf("Read consumed more data than necessary")
		}
	}
}

// TestSync tests that the Reader can read all data compressed thus far by the
// Writer once Flush is called.
func TestSync(t *testing.T) {
	const prime = 13
	var flushSizes []int
	for i := 1; i < 1000; i++ {
		flushSizes = append(flushSizes, i)
	}
	for i := 1; i <= 1<<16; i *= 2 {
		flushSizes = append(flushSizes, i)
		flushSizes = append(flushSizes, i+prime)
	}
	for i := 1; i <= 10000; i *= 10 {
		flushSizes = append(flushSizes, i)
		flushSizes = append(flushSizes, i+prime)
	}

	// Load test data of sufficient size.
	var maxSize, totalSize int
	for _, n := range flushSizes {
		totalSize += n
		if maxSize < n {
			maxSize = n
		}
	}
	rdBuf := make([]byte, maxSize)
	data := testutil.MustLoadFile("../testdata/twain.txt")
	data = testutil.ResizeData(data, totalSize)

	var buf bytes.Buffer
	wr, _ := flate.NewWriter(&buf, flate.DefaultCompression)
	rd, err := NewReader(&buf, nil)
	if err != nil {
		t.Errorf("unexpected NewReader error: %v", err)
	}
	for i, n := range flushSizes {
		// Write and flush some portion of the test data.
		want := data[:n]
		data = data[n:]
		if _, err := wr.Write(want); err != nil {
			t.Errorf("test %d, flushSize: %d, unexpected Write error: %v", i, n, err)
		}
		if err := wr.Flush(); err != nil {
			t.Errorf("test %d, flushSize: %d, unexpected Flush error: %v", i, n, err)
		}

		// Verify that we can read all data flushed so far.
		m, err := io.ReadAtLeast(rd, rdBuf, n)
		if err != nil {
			t.Errorf("test %d, flushSize: %d, unexpected ReadAtLeast error: %v", i, n, err)
		}
		got := rdBuf[:m]
		if !bytes.Equal(got, want) {
			t.Errorf("test %d, flushSize: %d, output mismatch:\ngot  %q\nwant %q", i, n, got, want)
		}
		if buf.Len() != 0 {
			t.Errorf("test %d, flushSize: %d, unconsumed buffer data: %d bytes", i, n, buf.Len())
		}
	}
}

func runBenchmarks(b *testing.B, f func(b *testing.B, buf []byte, lvl int)) {
	for _, td := range testdata {
		if len(td.data) == 0 {
			continue
		}
		if testing.Short() && !(td.name == "Twain" || td.name == "Digits") {
			continue
		}
		for _, tl := range levels {
			for _, ts := range sizes {
				buf := testutil.ResizeData(td.data, ts.size)
				b.Run(td.name+"/"+tl.name+"/"+ts.name, func(b *testing.B) {
					f(b, buf, tl.level)
				})
			}
		}
	}
}
<commit_msg>flate: add test to ensure Reader never peeks more than necessary<commit_after>// Copyright 2016, Joe Tsai. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE.md file.

package flate

import (
	"bufio"
	"bytes"
	"io"
	"testing"

	// TODO(dsnet): We should not be relying on the standard library for the
	// round-trip test.
	"compress/flate"

	"github.com/dsnet/compress/internal/testutil"
)

var testdata = []struct {
	name string
	data []byte
}{
	{"Nil", nil},
	{"Binary", testutil.MustLoadFile("../testdata/binary.bin")},
	{"Digits", testutil.MustLoadFile("../testdata/digits.txt")},
	{"Huffman", testutil.MustLoadFile("../testdata/huffman.txt")},
	{"Random", testutil.MustLoadFile("../testdata/random.bin")},
	{"Repeats", testutil.MustLoadFile("../testdata/repeats.bin")},
	{"Twain", testutil.MustLoadFile("../testdata/twain.txt")},
	{"Zeros", testutil.MustLoadFile("../testdata/zeros.bin")},
}

var levels = []struct {
	name  string
	level int
}{
	{"Huffman", flate.HuffmanOnly},
	{"Speed", flate.BestSpeed},
	{"Default", flate.DefaultCompression},
	{"Compression", flate.BestCompression},
}

var sizes = []struct {
	name string
	size int
}{
	{"1e4", 1e4},
	{"1e5", 1e5},
	{"1e6", 1e6},
}

func TestRoundTrip(t *testing.T) {
	for i, v := range testdata {
		var buf1, buf2 bytes.Buffer

		// Compress the input.
		wr, err := flate.NewWriter(&buf1, flate.DefaultCompression)
		if err != nil {
			t.Errorf("test %d, NewWriter() = (_, %v), want (_, nil)", i, err)
		}
		n, err := io.Copy(wr, bytes.NewReader(v.data))
		if n != int64(len(v.data)) || err != nil {
			t.Errorf("test %d, Copy() = (%d, %v), want (%d, nil)", i, n, err, len(v.data))
		}
		if err := wr.Close(); err != nil {
			t.Errorf("test %d, Close() = %v, want nil", i, err)
		}

		// Write a canary byte to ensure this does not get read.
		buf1.WriteByte(0x7a)

		// Decompress the output.
		rd, err := NewReader(&buf1, nil)
		if err != nil {
			t.Errorf("test %d, NewReader() = (_, %v), want (_, nil)", i, err)
		}
		n, err = io.Copy(&buf2, rd)
		if n != int64(len(v.data)) || err != nil {
			t.Errorf("test %d, Copy() = (%d, %v), want (%d, nil)", i, n, err, len(v.data))
		}
		if err := rd.Close(); err != nil {
			t.Errorf("test %d, Close() = %v, want nil", i, err)
		}
		if !bytes.Equal(buf2.Bytes(), v.data) {
			t.Errorf("test %d, output data mismatch", i)
		}

		// Read back the canary byte.
		if v, _ := buf1.ReadByte(); v != 0x7a {
			t.Errorf("Read consumed more data than necessary")
		}
	}
}

// syncBuffer is a special reader that records whether the Reader ever tried to
// read past the io.EOF. Since the flate Writer and Reader should be in sync,
// the reader should never attempt to read past the sync marker, otherwise the
// reader could potentially end up blocking on a network read when it had enough
// data to report back to the user.
type syncBuffer struct {
	bytes.Buffer
	blocked bool // blocked reports where a Read would have blocked
}

func (sb *syncBuffer) Read(buf []byte) (int, error) {
	n, err := sb.Buffer.Read(buf)
	if n == 0 && len(buf) > 0 {
		sb.blocked = true
	}
	return n, err
}

func (sb *syncBuffer) ReadByte() (byte, error) {
	b, err := sb.Buffer.ReadByte()
	if err == io.EOF {
		sb.blocked = true
	}
	return b, err
}

// TestSync tests that the Reader can read all data compressed thus far by the
// Writer once Flush is called.
func TestSync(t *testing.T) {
	const prime = 13
	var flushSizes []int
	for i := 1; i < 100; i += 3 {
		flushSizes = append(flushSizes, i)
	}
	for i := 1; i <= 1<<16; i *= 4 {
		flushSizes = append(flushSizes, i)
		flushSizes = append(flushSizes, i+prime)
	}
	for i := 1; i <= 10000; i *= 10 {
		flushSizes = append(flushSizes, i)
		flushSizes = append(flushSizes, i+prime)
	}

	// Load test data of sufficient size.
	var maxSize, totalSize int
	for _, n := range flushSizes {
		totalSize += n
		if maxSize < n {
			maxSize = n
		}
	}
	maxBuf := make([]byte, maxSize)
	data := testutil.MustLoadFile("../testdata/twain.txt")
	data = testutil.ResizeData(data, totalSize)

	for _, name := range []string{"Reader", "ByteReader", "BufferedReader"} {
		t.Run(name, func(t *testing.T) {
			data := data // Closure to ensure fresh data per iteration

			// Test each type of reader.
			var rdBuf io.Reader
			buf := new(syncBuffer)
			switch name {
			case "Reader":
				rdBuf = struct{ io.Reader }{buf}
			case "ByteReader":
				rdBuf = buf // syncBuffer already has a ReadByte method
			case "BufferedReader":
				rdBuf = bufio.NewReader(buf)
			default:
				t.Errorf("unknown reader type: %s", name)
				return
			}

			wr, _ := flate.NewWriter(buf, flate.DefaultCompression)
			rd, err := NewReader(rdBuf, nil)
			if err != nil {
				t.Errorf("unexpected NewReader error: %v", err)
			}
			for _, n := range flushSizes {
				// Write and flush some portion of the test data.
				want := data[:n]
				data = data[n:]
				if _, err := wr.Write(want); err != nil {
					t.Errorf("flushSize: %d, unexpected Write error: %v", n, err)
				}
				if err := wr.Flush(); err != nil {
					t.Errorf("flushSize: %d, unexpected Flush error: %v", n, err)
				}

				// Verify that we can read all data flushed so far.
				m, err := io.ReadAtLeast(rd, maxBuf, n)
				if err != nil {
					t.Errorf("flushSize: %d, unexpected ReadAtLeast error: %v", n, err)
				}
				got := maxBuf[:m]
				if !bytes.Equal(got, want) {
					t.Errorf("flushSize: %d, output mismatch:\ngot  %q\nwant %q", n, got, want)
				}
				if buf.Len() > 0 {
					t.Errorf("flushSize: %d, unconsumed buffer data: %d bytes", n, buf.Len())
				}
				if buf.blocked {
					t.Errorf("flushSize: %d, attempted over-consumption of buffer", n)
				}
				buf.blocked = false
			}
		})
	}
}

func runBenchmarks(b *testing.B, f func(b *testing.B, buf []byte, lvl int)) {
	for _, td := range testdata {
		if len(td.data) == 0 {
			continue
		}
		if testing.Short() && !(td.name == "Twain" || td.name == "Digits") {
			continue
		}
		for _, tl := range levels {
			for _, ts := range sizes {
				buf := testutil.ResizeData(td.data, ts.size)
				b.Run(td.name+"/"+tl.name+"/"+ts.name, func(b *testing.B) {
					f(b, buf, tl.level)
				})
			}
		}
	}
}
<|endoftext|>


<commit_before>// Copyright (c) Ilia Kravets, 2015. All rights reserved. PROVIDED "AS IS"
// WITHOUT ANY WARRANTY, EXPRESS OR IMPLIED. See LICENSE file for details.

package rec

import (
	"bytes"
	"encoding/binary"
	"fmt"
	"io"

	"my/errs"

	"my/ev/packet"
	"my/ev/packet/bats"
	"my/ev/packet/nasdaq"
	"my/ev/sim"
)

type SimLogger struct {
	w              io.Writer
	tobOld, tobNew []sim.PriceLevel
	efhLogger      EfhLogger
}

const SimLoggerSupernodeLevels = 256

func NewSimLogger(w io.Writer) *SimLogger {
	s := &SimLogger{w: w}
	s.efhLogger = *NewEfhLogger(s)
	return s
}
func (s *SimLogger) SetOutputMode(mode EfhLoggerOutputMode) {
	s.efhLogger.SetOutputMode(mode)
}

func (s *SimLogger) printf(format string, vs ...interface{}) {
	_, err := fmt.Fprintf(s.w, format, vs...)
	errs.CheckE(err)
}
func (s *SimLogger) printfln(format string, vs ...interface{}) {
	f := format + "\n"
	s.printf(f, vs...)
}
func (s *SimLogger) MessageArrived(idm *sim.SimMessage) {
	outItto := func(name string, typ nasdaq.IttoMessageType, f string, vs ...interface{}) {
		s.printf("NORM %s %c ", name, typ)
		s.printfln(f, vs...)
	}
	outBats := func(f string, vs ...interface{}) {
		s.printf("NORM ORDER %02x ", idm.Pam.Layer().(bats.PitchMessage).Base().Type.ToInt())
		s.printfln(f, vs...)
	}
	sideChar := func(s packet.MarketSide) byte {
		if s == packet.MarketSideAsk {
			return 'S'
		}
		return byte(s)
	}
	switch im := idm.Pam.Layer().(type) {
	case *nasdaq.IttoMessageAddOrder:
		outItto("ORDER", im.Type, "%c %08x %08x %08x %08x", sideChar(im.Side), im.OId, im.RefNumD.ToUint32(), im.Size, im.Price)
	case *nasdaq.IttoMessageAddQuote:
		outItto("QBID", im.Type, "%08x %08x %08x %08x", im.OId, im.Bid.RefNumD.ToUint32(), im.Bid.Size, im.Bid.Price)
		outItto("QASK", im.Type, "%08x %08x %08x %08x", im.OId, im.Ask.RefNumD.ToUint32(), im.Ask.Size, im.Ask.Price)
	case *nasdaq.IttoMessageSingleSideExecuted:
		outItto("ORDER", im.Type, "%08x %08x", im.OrigRefNumD.ToUint32(), im.Size)
	case *nasdaq.IttoMessageSingleSideExecutedWithPrice:
		outItto("ORDER", im.Type, "%08x %08x", im.OrigRefNumD.ToUint32(), im.Size)
	case *nasdaq.IttoMessageOrderCancel:
		outItto("ORDER", im.Type, "%08x %08x", im.OrigRefNumD.ToUint32(), im.Size)
	case *nasdaq.IttoMessageSingleSideReplace:
		outItto("ORDER", im.Type, "%08x %08x %08x %08x", im.RefNumD.ToUint32(), im.OrigRefNumD.ToUint32(), im.Size, im.Price)
	case *nasdaq.IttoMessageSingleSideDelete:
		outItto("ORDER", im.Type, "%08x", im.OrigRefNumD.ToUint32())
	case *nasdaq.IttoMessageSingleSideUpdate:
		outItto("ORDER", im.Type, "%08x %08x %08x", im.RefNumD.ToUint32(), im.Size, im.Price)
	case *nasdaq.IttoMessageQuoteReplace:
		outItto("QBID", im.Type, "%08x %08x %08x %08x", im.Bid.RefNumD.ToUint32(), im.Bid.OrigRefNumD.ToUint32(), im.Bid.Size, im.Bid.Price)
		outItto("QASK", im.Type, "%08x %08x %08x %08x", im.Ask.RefNumD.ToUint32(), im.Ask.OrigRefNumD.ToUint32(), im.Ask.Size, im.Ask.Price)
	case *nasdaq.IttoMessageQuoteDelete:
		outItto("QBID", im.Type, "%08x", im.BidOrigRefNumD.ToUint32())
		outItto("QASK", im.Type, "%08x", im.AskOrigRefNumD.ToUint32())
	case *nasdaq.IttoMessageBlockSingleSideDelete:
		for _, r := range im.RefNumDs {
			outItto("ORDER", im.Type, "%08x", r.ToUint32())
		}

	case *bats.PitchMessageAddOrder:
		outBats("%c %012x %016x %08x %08x", sideChar(im.Side), im.Symbol.ToUint64(), im.OrderId.ToUint64(), im.Size, packet.PriceTo4Dec(im.Price))
	case *bats.PitchMessageDeleteOrder:
		outBats("%016x", im.OrderId.ToUint64())
	case *bats.PitchMessageOrderExecuted:
		outBats("%016x %08x", im.OrderId.ToUint64(), im.Size)
	case *bats.PitchMessageOrderExecutedAtPriceSize:
		outBats("%016x %08x", im.OrderId.ToUint64(), im.Size)
	case *bats.PitchMessageReduceSize:
		outBats("%016x %08x", im.OrderId.ToUint64(), im.Size)
	case *bats.PitchMessageModifyOrder:
		outBats("%016x %08x %08x", im.OrderId.ToUint64(), im.Size, packet.PriceTo4Dec(im.Price))
	}
	s.efhLogger.MessageArrived(idm)
}
func (s *SimLogger) OperationAppliedToOrders(operation sim.SimOperation) {
	type ordrespLogInfo struct {
		notFound, addOp   int
		orderId           packet.OrderId
		optionId          packet.OptionId
		side, price, size int
		ordlSuffix        string
	}
	type orduLogInfo struct {
		orderId           packet.OrderId
		optionId          packet.OptionId
		side, price, size int
	}

	var or ordrespLogInfo
	var ou orduLogInfo
	if op, ok := operation.(*sim.OperationAdd); ok {
		var oid packet.OptionId
		if op.Independent() {
			oid = op.GetOptionId()
		}
		or = ordrespLogInfo{
			addOp:      1,
			orderId:    op.OrderId,
			optionId:   oid,
			ordlSuffix: fmt.Sprintf(" %012x", oid.ToUint64()),
		}
		ou = orduLogInfo{
			orderId:  or.orderId,
			optionId: op.GetOptionId(),
			price:    op.GetPrice(),
			size:     op.GetNewSize(),
		}
		if op.GetSide() == packet.MarketSideAsk {
			ou.side = 1
		}
	} else {
		if operation.GetOptionId().Invalid() {
			or = ordrespLogInfo{notFound: 1}
		} else {
			or = ordrespLogInfo{
				optionId: operation.GetOptionId(),
				price:    operation.GetPrice(),
				size:     operation.GetNewSize() - operation.GetSizeDelta(),
			}
			if operation.GetSide() == packet.MarketSideAsk {
				or.side = 1
			}
			if operation.GetNewSize() != 0 {
				ou = orduLogInfo{
					optionId: or.optionId,
					side:     or.side,
					price:    or.price,
					size:     operation.GetNewSize(),
				}
			}
		}
		or.orderId = operation.GetOrigOrderId()
		ou.orderId = or.orderId
	}
	s.printfln("ORDL %d %016x%s", or.addOp, or.orderId.ToUint64(), or.ordlSuffix)
	s.printfln("ORDRESP %d %d %d %08x %08x %012x %016x", or.notFound, or.addOp, or.side, or.size, or.price, or.optionId.ToUint64(), or.orderId.ToUint64())
	if operation.GetOptionId().Valid() {
		s.printfln("ORDU %016x %012x %d %08x %08x", ou.orderId.ToUint64(), ou.optionId.ToUint64(), ou.side, ou.price, ou.size)
	}
}
func (s *SimLogger) BeforeBookUpdate(book sim.Book, operation sim.SimOperation) {
	s.tobOld = book.GetTop(operation.GetOptionId(), operation.GetSide(), SimLoggerSupernodeLevels)
	s.efhLogger.BeforeBookUpdate(book, operation)
}
func (s *SimLogger) AfterBookUpdate(book sim.Book, operation sim.SimOperation) {
	if operation.GetOptionId().Valid() {
		s.tobNew = book.GetTop(operation.GetOptionId(), operation.GetSide(), SimLoggerSupernodeLevels)
		empty := sim.PriceLevel{}
		if operation.GetSide() == packet.MarketSideAsk {
			empty.Price = -1
		}
		for i := 0; i < SimLoggerSupernodeLevels; i++ {
			plo, pln := empty, empty
			if i < len(s.tobOld) {
				plo = s.tobOld[i]
			}
			if i < len(s.tobNew) {
				pln = s.tobNew[i]
			}
			s.printfln("SN_OLD_NEW %02d %08x %08x  %08x %08x", i,
				plo.Size, uint32(plo.Price),
				pln.Size, uint32(pln.Price),
			)
		}
	}
	s.efhLogger.AfterBookUpdate(book, operation)
}

func (s *SimLogger) PrintOrder(m efhm_order) error {
	return s.genAppUpdate(m)
}
func (s *SimLogger) PrintQuote(m efhm_quote) error {
	return s.genAppUpdate(m)
}
func (s *SimLogger) PrintTrade(m efhm_trade) error {
	return s.genAppUpdate(m)
}
func (s *SimLogger) PrintDefinitionNom(m efhm_definition_nom) error {
	return s.genAppUpdate(m)
}
func (s *SimLogger) PrintDefinitionBats(m efhm_definition_bats) error {
	return s.genAppUpdate(m)
}

func (s *SimLogger) genAppUpdate(appMessage interface{}) (err error) {
	defer errs.PassE(&err)
	var bb bytes.Buffer
	errs.CheckE(binary.Write(&bb, binary.LittleEndian, appMessage))
	if r := bb.Len() % 8; r > 0 {
		// pad to  multiple of 8 bytes
		z := make([]byte, 8)
		_, err = bb.Write(z[0 : 8-r])
		errs.CheckE(err)
	}

	for {
		var qw uint64
		if err = binary.Read(&bb, binary.LittleEndian, &qw); err != nil {
			if err == io.EOF {
				break
			}
			errs.CheckE(err)
		} else {
			s.printfln("DMATOHOST_DATA %016x", qw)
		}
	}
	s.printfln("DMATOHOST_TRAILER 00656e696c616b45")
	return
}
<commit_msg>rec:SimLogger: fix mistakenly returned EOF error<commit_after>// Copyright (c) Ilia Kravets, 2015. All rights reserved. PROVIDED "AS IS"
// WITHOUT ANY WARRANTY, EXPRESS OR IMPLIED. See LICENSE file for details.

package rec

import (
	"bytes"
	"encoding/binary"
	"fmt"
	"io"

	"my/errs"

	"my/ev/packet"
	"my/ev/packet/bats"
	"my/ev/packet/nasdaq"
	"my/ev/sim"
)

type SimLogger struct {
	w              io.Writer
	tobOld, tobNew []sim.PriceLevel
	efhLogger      EfhLogger
}

const SimLoggerSupernodeLevels = 256

func NewSimLogger(w io.Writer) *SimLogger {
	s := &SimLogger{w: w}
	s.efhLogger = *NewEfhLogger(s)
	return s
}
func (s *SimLogger) SetOutputMode(mode EfhLoggerOutputMode) {
	s.efhLogger.SetOutputMode(mode)
}

func (s *SimLogger) printf(format string, vs ...interface{}) {
	_, err := fmt.Fprintf(s.w, format, vs...)
	errs.CheckE(err)
}
func (s *SimLogger) printfln(format string, vs ...interface{}) {
	f := format + "\n"
	s.printf(f, vs...)
}
func (s *SimLogger) MessageArrived(idm *sim.SimMessage) {
	outItto := func(name string, typ nasdaq.IttoMessageType, f string, vs ...interface{}) {
		s.printf("NORM %s %c ", name, typ)
		s.printfln(f, vs...)
	}
	outBats := func(f string, vs ...interface{}) {
		s.printf("NORM ORDER %02x ", idm.Pam.Layer().(bats.PitchMessage).Base().Type.ToInt())
		s.printfln(f, vs...)
	}
	sideChar := func(s packet.MarketSide) byte {
		if s == packet.MarketSideAsk {
			return 'S'
		}
		return byte(s)
	}
	switch im := idm.Pam.Layer().(type) {
	case *nasdaq.IttoMessageAddOrder:
		outItto("ORDER", im.Type, "%c %08x %08x %08x %08x", sideChar(im.Side), im.OId, im.RefNumD.ToUint32(), im.Size, im.Price)
	case *nasdaq.IttoMessageAddQuote:
		outItto("QBID", im.Type, "%08x %08x %08x %08x", im.OId, im.Bid.RefNumD.ToUint32(), im.Bid.Size, im.Bid.Price)
		outItto("QASK", im.Type, "%08x %08x %08x %08x", im.OId, im.Ask.RefNumD.ToUint32(), im.Ask.Size, im.Ask.Price)
	case *nasdaq.IttoMessageSingleSideExecuted:
		outItto("ORDER", im.Type, "%08x %08x", im.OrigRefNumD.ToUint32(), im.Size)
	case *nasdaq.IttoMessageSingleSideExecutedWithPrice:
		outItto("ORDER", im.Type, "%08x %08x", im.OrigRefNumD.ToUint32(), im.Size)
	case *nasdaq.IttoMessageOrderCancel:
		outItto("ORDER", im.Type, "%08x %08x", im.OrigRefNumD.ToUint32(), im.Size)
	case *nasdaq.IttoMessageSingleSideReplace:
		outItto("ORDER", im.Type, "%08x %08x %08x %08x", im.RefNumD.ToUint32(), im.OrigRefNumD.ToUint32(), im.Size, im.Price)
	case *nasdaq.IttoMessageSingleSideDelete:
		outItto("ORDER", im.Type, "%08x", im.OrigRefNumD.ToUint32())
	case *nasdaq.IttoMessageSingleSideUpdate:
		outItto("ORDER", im.Type, "%08x %08x %08x", im.RefNumD.ToUint32(), im.Size, im.Price)
	case *nasdaq.IttoMessageQuoteReplace:
		outItto("QBID", im.Type, "%08x %08x %08x %08x", im.Bid.RefNumD.ToUint32(), im.Bid.OrigRefNumD.ToUint32(), im.Bid.Size, im.Bid.Price)
		outItto("QASK", im.Type, "%08x %08x %08x %08x", im.Ask.RefNumD.ToUint32(), im.Ask.OrigRefNumD.ToUint32(), im.Ask.Size, im.Ask.Price)
	case *nasdaq.IttoMessageQuoteDelete:
		outItto("QBID", im.Type, "%08x", im.BidOrigRefNumD.ToUint32())
		outItto("QASK", im.Type, "%08x", im.AskOrigRefNumD.ToUint32())
	case *nasdaq.IttoMessageBlockSingleSideDelete:
		for _, r := range im.RefNumDs {
			outItto("ORDER", im.Type, "%08x", r.ToUint32())
		}

	case *bats.PitchMessageAddOrder:
		outBats("%c %012x %016x %08x %08x", sideChar(im.Side), im.Symbol.ToUint64(), im.OrderId.ToUint64(), im.Size, packet.PriceTo4Dec(im.Price))
	case *bats.PitchMessageDeleteOrder:
		outBats("%016x", im.OrderId.ToUint64())
	case *bats.PitchMessageOrderExecuted:
		outBats("%016x %08x", im.OrderId.ToUint64(), im.Size)
	case *bats.PitchMessageOrderExecutedAtPriceSize:
		outBats("%016x %08x", im.OrderId.ToUint64(), im.Size)
	case *bats.PitchMessageReduceSize:
		outBats("%016x %08x", im.OrderId.ToUint64(), im.Size)
	case *bats.PitchMessageModifyOrder:
		outBats("%016x %08x %08x", im.OrderId.ToUint64(), im.Size, packet.PriceTo4Dec(im.Price))
	}
	s.efhLogger.MessageArrived(idm)
}
func (s *SimLogger) OperationAppliedToOrders(operation sim.SimOperation) {
	type ordrespLogInfo struct {
		notFound, addOp   int
		orderId           packet.OrderId
		optionId          packet.OptionId
		side, price, size int
		ordlSuffix        string
	}
	type orduLogInfo struct {
		orderId           packet.OrderId
		optionId          packet.OptionId
		side, price, size int
	}

	var or ordrespLogInfo
	var ou orduLogInfo
	if op, ok := operation.(*sim.OperationAdd); ok {
		var oid packet.OptionId
		if op.Independent() {
			oid = op.GetOptionId()
		}
		or = ordrespLogInfo{
			addOp:      1,
			orderId:    op.OrderId,
			optionId:   oid,
			ordlSuffix: fmt.Sprintf(" %012x", oid.ToUint64()),
		}
		ou = orduLogInfo{
			orderId:  or.orderId,
			optionId: op.GetOptionId(),
			price:    op.GetPrice(),
			size:     op.GetNewSize(),
		}
		if op.GetSide() == packet.MarketSideAsk {
			ou.side = 1
		}
	} else {
		if operation.GetOptionId().Invalid() {
			or = ordrespLogInfo{notFound: 1}
		} else {
			or = ordrespLogInfo{
				optionId: operation.GetOptionId(),
				price:    operation.GetPrice(),
				size:     operation.GetNewSize() - operation.GetSizeDelta(),
			}
			if operation.GetSide() == packet.MarketSideAsk {
				or.side = 1
			}
			if operation.GetNewSize() != 0 {
				ou = orduLogInfo{
					optionId: or.optionId,
					side:     or.side,
					price:    or.price,
					size:     operation.GetNewSize(),
				}
			}
		}
		or.orderId = operation.GetOrigOrderId()
		ou.orderId = or.orderId
	}
	s.printfln("ORDL %d %016x%s", or.addOp, or.orderId.ToUint64(), or.ordlSuffix)
	s.printfln("ORDRESP %d %d %d %08x %08x %012x %016x", or.notFound, or.addOp, or.side, or.size, or.price, or.optionId.ToUint64(), or.orderId.ToUint64())
	if operation.GetOptionId().Valid() {
		s.printfln("ORDU %016x %012x %d %08x %08x", ou.orderId.ToUint64(), ou.optionId.ToUint64(), ou.side, ou.price, ou.size)
	}
}
func (s *SimLogger) BeforeBookUpdate(book sim.Book, operation sim.SimOperation) {
	s.tobOld = book.GetTop(operation.GetOptionId(), operation.GetSide(), SimLoggerSupernodeLevels)
	s.efhLogger.BeforeBookUpdate(book, operation)
}
func (s *SimLogger) AfterBookUpdate(book sim.Book, operation sim.SimOperation) {
	if operation.GetOptionId().Valid() {
		s.tobNew = book.GetTop(operation.GetOptionId(), operation.GetSide(), SimLoggerSupernodeLevels)
		empty := sim.PriceLevel{}
		if operation.GetSide() == packet.MarketSideAsk {
			empty.Price = -1
		}
		for i := 0; i < SimLoggerSupernodeLevels; i++ {
			plo, pln := empty, empty
			if i < len(s.tobOld) {
				plo = s.tobOld[i]
			}
			if i < len(s.tobNew) {
				pln = s.tobNew[i]
			}
			s.printfln("SN_OLD_NEW %02d %08x %08x  %08x %08x", i,
				plo.Size, uint32(plo.Price),
				pln.Size, uint32(pln.Price),
			)
		}
	}
	s.efhLogger.AfterBookUpdate(book, operation)
}

func (s *SimLogger) PrintOrder(m efhm_order) error {
	return s.genAppUpdate(m)
}
func (s *SimLogger) PrintQuote(m efhm_quote) error {
	return s.genAppUpdate(m)
}
func (s *SimLogger) PrintTrade(m efhm_trade) error {
	return s.genAppUpdate(m)
}
func (s *SimLogger) PrintDefinitionNom(m efhm_definition_nom) error {
	return s.genAppUpdate(m)
}
func (s *SimLogger) PrintDefinitionBats(m efhm_definition_bats) error {
	return s.genAppUpdate(m)
}

func (s *SimLogger) genAppUpdate(appMessage interface{}) (err error) {
	defer errs.PassE(&err)
	var bb bytes.Buffer
	errs.CheckE(binary.Write(&bb, binary.LittleEndian, appMessage))
	if r := bb.Len() % 8; r > 0 {
		// pad to  multiple of 8 bytes
		z := make([]byte, 8)
		_, err = bb.Write(z[0 : 8-r])
		errs.CheckE(err)
	}

	for {
		var qw uint64
		if err := binary.Read(&bb, binary.LittleEndian, &qw); err != nil {
			if err == io.EOF {
				break
			}
			errs.CheckE(err)
		} else {
			s.printfln("DMATOHOST_DATA %016x", qw)
		}
	}
	s.printfln("DMATOHOST_TRAILER 00656e696c616b45")
	return
}
<|endoftext|>