<commit_before>/*
Copyright 2014 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package empty_dir

import (
	"fmt"
	"os"

	"github.com/GoogleCloudPlatform/kubernetes/pkg/api"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/types"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/util"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/util/mount"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/volume"
	"github.com/golang/glog"
)

// This is the primary entrypoint for volume plugins.
func ProbeVolumePlugins() []volume.VolumePlugin {
	return []volume.VolumePlugin{
		&emptyDirPlugin{nil, false},
		&emptyDirPlugin{nil, true},
	}
}

type emptyDirPlugin struct {
	host       volume.VolumeHost
	legacyMode bool // if set, plugin answers to the legacy name
}

var _ volume.VolumePlugin = &emptyDirPlugin{}

const (
	emptyDirPluginName       = "kubernetes.io/empty-dir"
	emptyDirPluginLegacyName = "empty"
)

func (plugin *emptyDirPlugin) Init(host volume.VolumeHost) {
	plugin.host = host
}

func (plugin *emptyDirPlugin) Name() string {
	if plugin.legacyMode {
		return emptyDirPluginLegacyName
	}
	return emptyDirPluginName
}

func (plugin *emptyDirPlugin) CanSupport(spec *volume.Spec) bool {
	if plugin.legacyMode {
		// Legacy mode instances can be cleaned up but not created anew.
		return false
	}

	if spec.VolumeSource.EmptyDir != nil {
		return true
	}
	return false
}

func (plugin *emptyDirPlugin) NewBuilder(spec *volume.Spec, pod *api.Pod, opts volume.VolumeOptions, mounter mount.Interface) (volume.Builder, error) {
	return plugin.newBuilderInternal(spec, pod, mounter, &realMountDetector{mounter}, opts)
}

func (plugin *emptyDirPlugin) newBuilderInternal(spec *volume.Spec, pod *api.Pod, mounter mount.Interface, mountDetector mountDetector, opts volume.VolumeOptions) (volume.Builder, error) {
	if plugin.legacyMode {
		// Legacy mode instances can be cleaned up but not created anew.
		return nil, fmt.Errorf("legacy mode: can not create new instances")
	}
	medium := api.StorageMediumDefault
	if spec.VolumeSource.EmptyDir != nil { // Support a non-specified source as EmptyDir.
		medium = spec.VolumeSource.EmptyDir.Medium
	}
	return &emptyDir{
		podUID:        pod.UID,
		volName:       spec.Name,
		medium:        medium,
		mounter:       mounter,
		mountDetector: mountDetector,
		plugin:        plugin,
		legacyMode:    false,
		rootContext:   opts.RootContext,
	}, nil
}

func (plugin *emptyDirPlugin) NewCleaner(volName string, podUID types.UID, mounter mount.Interface) (volume.Cleaner, error) {
	// Inject real implementations here, test through the internal function.
	return plugin.newCleanerInternal(volName, podUID, mounter, &realMountDetector{mounter})
}

func (plugin *emptyDirPlugin) newCleanerInternal(volName string, podUID types.UID, mounter mount.Interface, mountDetector mountDetector) (volume.Cleaner, error) {
	legacy := false
	if plugin.legacyMode {
		legacy = true
	}
	ed := &emptyDir{
		podUID:        podUID,
		volName:       volName,
		medium:        api.StorageMediumDefault, // might be changed later
		mounter:       mounter,
		mountDetector: mountDetector,
		plugin:        plugin,
		legacyMode:    legacy,
	}
	return ed, nil
}

// mountDetector abstracts how to find what kind of mount a path is backed by.
type mountDetector interface {
	// GetMountMedium determines what type of medium a given path is backed
	// by and whether that path is a mount point.  For example, if this
	// returns (mediumMemory, false, nil), the caller knows that the path is
	// on a memory FS (tmpfs on Linux) but is not the root mountpoint of
	// that tmpfs.
	GetMountMedium(path string) (storageMedium, bool, error)
}

type storageMedium int

const (
	mediumUnknown storageMedium = 0 // assume anything we don't explicitly handle is this
	mediumMemory  storageMedium = 1 // memory (e.g. tmpfs on linux)
)

// EmptyDir volumes are temporary directories exposed to the pod.
// These do not persist beyond the lifetime of a pod.
type emptyDir struct {
	podUID        types.UID
	volName       string
	medium        api.StorageMedium
	mounter       mount.Interface
	mountDetector mountDetector
	plugin        *emptyDirPlugin
	legacyMode    bool
	rootContext   string
}

// SetUp creates new directory.
func (ed *emptyDir) SetUp() error {
	return ed.SetUpAt(ed.GetPath())
}

// SetUpAt creates new directory.
func (ed *emptyDir) SetUpAt(dir string) error {
	if ed.legacyMode {
		return fmt.Errorf("legacy mode: can not create new instances")
	}
	switch ed.medium {
	case api.StorageMediumDefault:
		return ed.setupDefault(dir)
	case api.StorageMediumMemory:
		return ed.setupTmpfs(dir)
	default:
		return fmt.Errorf("unknown storage medium %q", ed.medium)
	}
}

func (ed *emptyDir) setupDefault(dir string) error {
	return os.MkdirAll(dir, 0750)
}

func (ed *emptyDir) setupTmpfs(dir string) error {
	if ed.mounter == nil {
		return fmt.Errorf("memory storage requested, but mounter is nil")
	}
	if err := os.MkdirAll(dir, 0750); err != nil {
		return err
	}
	// Make SetUp idempotent.
	medium, isMnt, err := ed.mountDetector.GetMountMedium(dir)
	if err != nil {
		return err
	}
	// If the directory is a mountpoint with medium memory, there is no
	// work to do since we are already in the desired state.
	if isMnt && medium == mediumMemory {
		return nil
	}

	// By default a tmpfs mount will receive a different SELinux context
	// from that of the Kubelet root directory which is not readable from
	// the SELinux context of a docker container.
	//
	// getTmpfsMountOptions gets the mount option to set the context of
	// the tmpfs mount so that it can be read from the SELinux context of
	// the container.
	opts := ed.getTmpfsMountOptions()
	glog.V(3).Infof("pod %v: mounting tmpfs for volume %v with opts %v", ed.podUID, ed.volName, opts)
	return ed.mounter.Mount("tmpfs", dir, "tmpfs", opts)
}

func (ed *emptyDir) getTmpfsMountOptions() []string {
	if ed.rootContext == "" {
		return []string{""}
	}

	return []string{fmt.Sprintf("rootcontext=\"%v\"", ed.rootContext)}
}

func (ed *emptyDir) GetPath() string {
	name := emptyDirPluginName
	if ed.legacyMode {
		name = emptyDirPluginLegacyName
	}
	return ed.plugin.host.GetPodVolumeDir(ed.podUID, util.EscapeQualifiedNameForDisk(name), ed.volName)
}

// TearDown simply discards everything in the directory.
func (ed *emptyDir) TearDown() error {
	return ed.TearDownAt(ed.GetPath())
}

// TearDownAt simply discards everything in the directory.
func (ed *emptyDir) TearDownAt(dir string) error {
	// Figure out the medium.
	medium, isMnt, err := ed.mountDetector.GetMountMedium(dir)
	if err != nil {
		return err
	}
	if isMnt && medium == mediumMemory {
		ed.medium = api.StorageMediumMemory
		return ed.teardownTmpfs(dir)
	}
	// assume StorageMediumDefault
	return ed.teardownDefault(dir)
}

func (ed *emptyDir) teardownDefault(dir string) error {
	tmpDir, err := volume.RenameDirectory(dir, ed.volName+".deleting~")
	if err != nil {
		return err
	}
	err = os.RemoveAll(tmpDir)
	if err != nil {
		return err
	}
	return nil
}

func (ed *emptyDir) teardownTmpfs(dir string) error {
	if ed.mounter == nil {
		return fmt.Errorf("memory storage requested, but mounter is nil")
	}
	if err := ed.mounter.Unmount(dir); err != nil {
		return err
	}
	if err := os.RemoveAll(dir); err != nil {
		return err
	}
	return nil
}
<commit_msg>UPSTREAM: 9844: Support emptyDir volumes for containers running as uid != 0<commit_after>/*
Copyright 2014 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package empty_dir

import (
	"fmt"
	"os"

	"github.com/GoogleCloudPlatform/kubernetes/pkg/api"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/types"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/util"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/util/mount"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/volume"
	"github.com/golang/glog"
)

// TODO: in the near future, this will be changed to be more restrictive
// and the group will be set to allow containers to use emptyDir volumes
// from the group attribute.
const perm os.FileMode = 0777

// This is the primary entrypoint for volume plugins.
func ProbeVolumePlugins() []volume.VolumePlugin {
	return []volume.VolumePlugin{
		&emptyDirPlugin{nil, false},
		&emptyDirPlugin{nil, true},
	}
}

type emptyDirPlugin struct {
	host       volume.VolumeHost
	legacyMode bool // if set, plugin answers to the legacy name
}

var _ volume.VolumePlugin = &emptyDirPlugin{}

const (
	emptyDirPluginName       = "kubernetes.io/empty-dir"
	emptyDirPluginLegacyName = "empty"
)

func (plugin *emptyDirPlugin) Init(host volume.VolumeHost) {
	plugin.host = host
}

func (plugin *emptyDirPlugin) Name() string {
	if plugin.legacyMode {
		return emptyDirPluginLegacyName
	}
	return emptyDirPluginName
}

func (plugin *emptyDirPlugin) CanSupport(spec *volume.Spec) bool {
	if plugin.legacyMode {
		// Legacy mode instances can be cleaned up but not created anew.
		return false
	}

	if spec.VolumeSource.EmptyDir != nil {
		return true
	}
	return false
}

func (plugin *emptyDirPlugin) NewBuilder(spec *volume.Spec, pod *api.Pod, opts volume.VolumeOptions, mounter mount.Interface) (volume.Builder, error) {
	return plugin.newBuilderInternal(spec, pod, mounter, &realMountDetector{mounter}, opts)
}

func (plugin *emptyDirPlugin) newBuilderInternal(spec *volume.Spec, pod *api.Pod, mounter mount.Interface, mountDetector mountDetector, opts volume.VolumeOptions) (volume.Builder, error) {
	if plugin.legacyMode {
		// Legacy mode instances can be cleaned up but not created anew.
		return nil, fmt.Errorf("legacy mode: can not create new instances")
	}
	medium := api.StorageMediumDefault
	if spec.VolumeSource.EmptyDir != nil { // Support a non-specified source as EmptyDir.
		medium = spec.VolumeSource.EmptyDir.Medium
	}
	return &emptyDir{
		podUID:        pod.UID,
		volName:       spec.Name,
		medium:        medium,
		mounter:       mounter,
		mountDetector: mountDetector,
		plugin:        plugin,
		legacyMode:    false,
		rootContext:   opts.RootContext,
	}, nil
}

func (plugin *emptyDirPlugin) NewCleaner(volName string, podUID types.UID, mounter mount.Interface) (volume.Cleaner, error) {
	// Inject real implementations here, test through the internal function.
	return plugin.newCleanerInternal(volName, podUID, mounter, &realMountDetector{mounter})
}

func (plugin *emptyDirPlugin) newCleanerInternal(volName string, podUID types.UID, mounter mount.Interface, mountDetector mountDetector) (volume.Cleaner, error) {
	legacy := false
	if plugin.legacyMode {
		legacy = true
	}
	ed := &emptyDir{
		podUID:        podUID,
		volName:       volName,
		medium:        api.StorageMediumDefault, // might be changed later
		mounter:       mounter,
		mountDetector: mountDetector,
		plugin:        plugin,
		legacyMode:    legacy,
	}
	return ed, nil
}

// mountDetector abstracts how to find what kind of mount a path is backed by.
type mountDetector interface {
	// GetMountMedium determines what type of medium a given path is backed
	// by and whether that path is a mount point.  For example, if this
	// returns (mediumMemory, false, nil), the caller knows that the path is
	// on a memory FS (tmpfs on Linux) but is not the root mountpoint of
	// that tmpfs.
	GetMountMedium(path string) (storageMedium, bool, error)
}

type storageMedium int

const (
	mediumUnknown storageMedium = 0 // assume anything we don't explicitly handle is this
	mediumMemory  storageMedium = 1 // memory (e.g. tmpfs on linux)
)

// EmptyDir volumes are temporary directories exposed to the pod.
// These do not persist beyond the lifetime of a pod.
type emptyDir struct {
	podUID        types.UID
	volName       string
	medium        api.StorageMedium
	mounter       mount.Interface
	mountDetector mountDetector
	plugin        *emptyDirPlugin
	legacyMode    bool
	rootContext   string
}

// SetUp creates new directory.
func (ed *emptyDir) SetUp() error {
	return ed.SetUpAt(ed.GetPath())
}

// SetUpAt creates new directory.
func (ed *emptyDir) SetUpAt(dir string) error {
	if ed.legacyMode {
		return fmt.Errorf("legacy mode: can not create new instances")
	}
	switch ed.medium {
	case api.StorageMediumDefault:
		return ed.setupDefault(dir)
	case api.StorageMediumMemory:
		return ed.setupTmpfs(dir)
	default:
		return fmt.Errorf("unknown storage medium %q", ed.medium)
	}
}

func (ed *emptyDir) setupDefault(dir string) error {
	return os.MkdirAll(dir, perm)
}

func (ed *emptyDir) setupTmpfs(dir string) error {
	if ed.mounter == nil {
		return fmt.Errorf("memory storage requested, but mounter is nil")
	}
	if err := os.MkdirAll(dir, perm); err != nil {
		return err
	}
	// Make SetUp idempotent.
	medium, isMnt, err := ed.mountDetector.GetMountMedium(dir)
	if err != nil {
		return err
	}
	// If the directory is a mountpoint with medium memory, there is no
	// work to do since we are already in the desired state.
	if isMnt && medium == mediumMemory {
		return nil
	}

	// By default a tmpfs mount will receive a different SELinux context
	// from that of the Kubelet root directory which is not readable from
	// the SELinux context of a docker container.
	//
	// getTmpfsMountOptions gets the mount option to set the context of
	// the tmpfs mount so that it can be read from the SELinux context of
	// the container.
	opts := ed.getTmpfsMountOptions()
	glog.V(3).Infof("pod %v: mounting tmpfs for volume %v with opts %v", ed.podUID, ed.volName, opts)
	return ed.mounter.Mount("tmpfs", dir, "tmpfs", opts)
}

func (ed *emptyDir) getTmpfsMountOptions() []string {
	if ed.rootContext == "" {
		return []string{""}
	}

	return []string{fmt.Sprintf("rootcontext=\"%v\"", ed.rootContext)}
}

func (ed *emptyDir) GetPath() string {
	name := emptyDirPluginName
	if ed.legacyMode {
		name = emptyDirPluginLegacyName
	}
	return ed.plugin.host.GetPodVolumeDir(ed.podUID, util.EscapeQualifiedNameForDisk(name), ed.volName)
}

// TearDown simply discards everything in the directory.
func (ed *emptyDir) TearDown() error {
	return ed.TearDownAt(ed.GetPath())
}

// TearDownAt simply discards everything in the directory.
func (ed *emptyDir) TearDownAt(dir string) error {
	// Figure out the medium.
	medium, isMnt, err := ed.mountDetector.GetMountMedium(dir)
	if err != nil {
		return err
	}
	if isMnt && medium == mediumMemory {
		ed.medium = api.StorageMediumMemory
		return ed.teardownTmpfs(dir)
	}
	// assume StorageMediumDefault
	return ed.teardownDefault(dir)
}

func (ed *emptyDir) teardownDefault(dir string) error {
	tmpDir, err := volume.RenameDirectory(dir, ed.volName+".deleting~")
	if err != nil {
		return err
	}
	err = os.RemoveAll(tmpDir)
	if err != nil {
		return err
	}
	return nil
}

func (ed *emptyDir) teardownTmpfs(dir string) error {
	if ed.mounter == nil {
		return fmt.Errorf("memory storage requested, but mounter is nil")
	}
	if err := ed.mounter.Unmount(dir); err != nil {
		return err
	}
	if err := os.RemoveAll(dir); err != nil {
		return err
	}
	return nil
}
<|endoftext|>


<commit_before>// Package restful provides RESTful API features to CellAdvisor
package restful

import (
	"errors"
	"expvar"
	"fmt"
	"log"
	"sync"
	"time"

	"net/http"
	"net/url"

	"github.com/gorilla/mux"
	"github.com/tomahawk28/cell"
)

var (
	sendSuccessCount    = expvar.NewInt("sendSuccessCount")
	receiveSucessCount  = expvar.NewInt("receiveSucessCount")
	sendPendingCount    = expvar.NewInt("sendPendingCount")
	receivePendingCount = expvar.NewInt("receivePendingCount")
)

var (
	OKSIGN = []byte("OK")
)

type pollRequest struct {
	command string
	args    url.Values
	result  chan pollResult
}

type pollResult struct {
	resultByte  []byte
	requestType string
	requestErr  error
}

type pollScreenCache struct {
	last  time.Time
	cache []byte
	mu    sync.RWMutex
}

// cellServer implements http API method to cellAdvisor,
type cellServer struct {
	//ScreenCache implements web cache, for any cases clients order new screen image,
	//API fetch screen only if its 1 minutes later after last capture
	screenCache    pollScreenCache
	requestChannel chan *pollRequest
	pollPeriod     time.Duration
}

// NewCellHTTPServer retuning CellAdviosr http server object
func createCellAdvisorHTTPServer(threadNumber int, cellAddr string, pollPeriod time.Duration) cellServer {
	screenCache := pollScreenCache{time.Now(), []byte{}, sync.RWMutex{}}

	rc := make(chan *pollRequest, threadNumber)

	server := cellServer{screenCache, rc, pollPeriod}

	for i := 0; i < threadNumber; i++ {
		element := cell.NewCellAdvisor(cellAddr)
		go server.poller(&element, i)
	}

	return server
}

// BuildCellAdvisorRestfulAPI returning automatic RESTful API server set
// user could access directly api/screen/*, and api/scpi/*
// after deploy retuning object to sever
func BuildCellAdvisorRestfulAPI(prefix string, threadNumber int, cellAddr string, pollPeriod time.Duration) *mux.Router {

	s := createCellAdvisorHTTPServer(threadNumber, cellAddr, pollPeriod)
	rtr := mux.NewRouter()
	rtr.Handle("/"+prefix+"api/{command}.json", s)
	rtr.Handle("/"+prefix+"api/screen/{command}", s)
	rtr.Handle("/"+prefix+"api/scpi/{command}", s).Methods("POST")

	return rtr
}

func (server cellServer) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	var values url.Values
	params := mux.Vars(r)
	command := params["command"]
	if r.Method == "POST" {
		err := r.ParseForm()
		if err != nil {
			w.WriteHeader(http.StatusBadRequest)
			w.Write([]byte(err.Error()))
			return
		}
		values = r.Form
	}
	request := createRequest(command, values)
	server.requestChannel <- request
	result := receiveResult(request.result)
	if result == nil {
		w.Header().Set("Content-Type", "text/plain")
		w.WriteHeader(http.StatusServiceUnavailable)
		w.Write([]byte("Poller thread respond timeout"))
		return
	}
	w.Header().Set("Content-Type", result.requestType)
	if result.requestErr != nil {
		w.WriteHeader(http.StatusBadRequest)
		w.Write([]byte(result.requestErr.Error()))
		return
	}

	w.Write(result.resultByte)
}

func (server *cellServer) poller(cell *cell.CellAdvisor, threadNumber int) {
	done := make(chan struct{})
	defer close(done)
	var err error
	var msg []byte
	for {
		log.Println("waiting..", threadNumber)
		select {
		case request := <-server.requestChannel:
			log.Println("Thread ", threadNumber, ":", request.command)
			switch request.command {
			case "keyp":
				if value := request.args.Get("value"); value == "" {
					sendResult(done, request.result, createResult(nil, "", errors.New("keyp value missing")))
				} else {
					scpicmd := fmt.Sprintf("KEYP:%s", request.args.Get("value"))
					_, err = cell.SendSCPI(scpicmd)
					sendResult(done, request.result, createResult([]byte("OK"), "", nil))
				}
			case "touch":
				if x, y := request.args.Get("x"), request.args.Get("y"); x == "" || y == "" {
					sendResult(done, request.result, createResult(nil, "", errors.New("x,y value missing")))
				} else {
					scpicmd := fmt.Sprintf("KEYP %s %s", request.args.Get("x"), request.args.Get("y"))
					_, err = cell.SendSCPI(scpicmd)
					sendResult(done, request.result, createResult([]byte("OK"), "", nil))
				}
			case "screen":
				server.screenCache.mu.RLock()
				sendResult(done, request.result, createResult(server.screenCache.cache, "application/jpeg", nil))
				server.screenCache.mu.RUnlock()
			case "refresh_screen":
				func() {
					if len(server.screenCache.cache) == 0 || time.Now().Sub(server.screenCache.last).Seconds() > 1 {
						server.screenCache.mu.Lock()
						defer server.screenCache.mu.Unlock()
						if len(server.screenCache.cache) == 0 || time.Now().Sub(server.screenCache.last).Seconds() > 1 {
							server.screenCache.last = time.Now()
							server.screenCache.cache, err = cell.GetScreen()
							if err != nil {
								log.Println(err.Error())
							}
						}
					}
				}()
				sendResult(done, request.result, createResult(OKSIGN, "", nil))
			case "interference_power":
				js, err := cell.GetInterferencePower()
				if err != nil {
					sendResult(done, request.result, createResult(nil, "", err))
				} else {
					sendResult(done, request.result, createResult(js, "application/json", nil))
				}
			case "heartbeat":
				msg, err = cell.GetStatusMessage()
				sendResult(done, request.result, createResult(msg, "", nil))
			default:
				sendResult(done, request.result, createResult(nil, "", errors.New("unknown command")))
			}
		case <-time.After(server.pollPeriod):
			r := createRequest("heartbeat", nil)
			server.requestChannel <- r
			<-r.result
		}
		//Check Error Status == EOF
		if err != nil {
			switch err.Error() {
			case "EOF":
				log.Println("Connection loses on ", threadNumber, ", Poller exited")
				return
			default:
				log.Println("Thread ", threadNumber, " got error: ", err.Error())
			}
		}
	}
}

func createRequest(command string, args url.Values) *pollRequest {
	return &pollRequest{command, args, make(chan pollResult)}
}

func createResult(result []byte, resultType string, err error) pollResult {
	if resultType == "" {
		resultType = "text/plain"
	}
	return pollResult{result, resultType, err}
}

func sendResult(done <-chan struct{}, pipe chan<- pollResult, result pollResult) {
	select {
	case pipe <- result:
		sendSuccessCount.Add(1)
	case <-time.After(time.Second * 3):
		log.Println("Sending Timeout")
		sendPendingCount.Add(1)
	case <-done:
		return
	}
}
func receiveResult(pipe <-chan pollResult) *pollResult {
	select {
	case result := <-pipe:
		receiveSucessCount.Add(1)
		return &result
	case <-time.After(time.Second * 5):
		log.Println("Receive Timeout")
		receivePendingCount.Add(1)
	}
	return nil
}
<commit_msg>Hide unexported variable<commit_after>// Package restful provides RESTful API features to CellAdvisor
package restful

import (
	"errors"
	"expvar"
	"fmt"
	"log"
	"sync"
	"time"

	"net/http"
	"net/url"

	"github.com/gorilla/mux"
	"github.com/tomahawk28/cell"
)

var (
	sendSuccessCount    = expvar.NewInt("sendSuccessCount")
	receiveSucessCount  = expvar.NewInt("receiveSucessCount")
	sendPendingCount    = expvar.NewInt("sendPendingCount")
	receivePendingCount = expvar.NewInt("receivePendingCount")
)

var (
	okb = []byte("OK")
)

type pollRequest struct {
	command string
	args    url.Values
	result  chan pollResult
}

type pollResult struct {
	resultByte  []byte
	requestType string
	requestErr  error
}

type pollScreenCache struct {
	last  time.Time
	cache []byte
	mu    sync.RWMutex
}

// cellServer implements http API method to cellAdvisor,
type cellServer struct {
	//ScreenCache implements web cache, for any cases clients order new screen image,
	//API fetch screen only if its 1 minutes later after last capture
	screenCache    pollScreenCache
	requestChannel chan *pollRequest
	pollPeriod     time.Duration
}

// NewCellHTTPServer retuning CellAdviosr http server object
func createCellAdvisorHTTPServer(threadNumber int, cellAddr string, pollPeriod time.Duration) cellServer {
	screenCache := pollScreenCache{time.Now(), []byte{}, sync.RWMutex{}}

	rc := make(chan *pollRequest, threadNumber)

	server := cellServer{screenCache, rc, pollPeriod}

	for i := 0; i < threadNumber; i++ {
		element := cell.NewCellAdvisor(cellAddr)
		go server.poller(&element, i)
	}

	return server
}

// BuildCellAdvisorRestfulAPI returning automatic RESTful API server set
// user could access directly api/screen/*, and api/scpi/*
// after deploy retuning object to sever
func BuildCellAdvisorRestfulAPI(prefix string, threadNumber int, cellAddr string, pollPeriod time.Duration) *mux.Router {

	s := createCellAdvisorHTTPServer(threadNumber, cellAddr, pollPeriod)
	rtr := mux.NewRouter()
	rtr.Handle("/"+prefix+"api/{command}.json", s)
	rtr.Handle("/"+prefix+"api/screen/{command}", s)
	rtr.Handle("/"+prefix+"api/scpi/{command}", s).Methods("POST")

	return rtr
}

func (server cellServer) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	var values url.Values
	params := mux.Vars(r)
	command := params["command"]
	if r.Method == "POST" {
		err := r.ParseForm()
		if err != nil {
			w.WriteHeader(http.StatusBadRequest)
			w.Write([]byte(err.Error()))
			return
		}
		values = r.Form
	}
	request := createRequest(command, values)
	server.requestChannel <- request
	result := receiveResult(request.result)
	if result == nil {
		w.Header().Set("Content-Type", "text/plain")
		w.WriteHeader(http.StatusServiceUnavailable)
		w.Write([]byte("Poller thread respond timeout"))
		return
	}
	w.Header().Set("Content-Type", result.requestType)
	if result.requestErr != nil {
		w.WriteHeader(http.StatusBadRequest)
		w.Write([]byte(result.requestErr.Error()))
		return
	}

	w.Write(result.resultByte)
}

func (server *cellServer) poller(cell *cell.CellAdvisor, threadNumber int) {
	done := make(chan struct{})
	defer close(done)
	var err error
	var msg []byte
	for {
		log.Println("waiting..", threadNumber)
		select {
		case request := <-server.requestChannel:
			log.Println("Thread ", threadNumber, ":", request.command)
			switch request.command {
			case "keyp":
				if value := request.args.Get("value"); value == "" {
					sendResult(done, request.result, createResult(nil, "", errors.New("keyp value missing")))
				} else {
					scpicmd := fmt.Sprintf("KEYP:%s", request.args.Get("value"))
					_, err = cell.SendSCPI(scpicmd)
					sendResult(done, request.result, createResult(okb, "", nil))
				}
			case "touch":
				if x, y := request.args.Get("x"), request.args.Get("y"); x == "" || y == "" {
					sendResult(done, request.result, createResult(nil, "", errors.New("x,y value missing")))
				} else {
					scpicmd := fmt.Sprintf("KEYP %s %s", request.args.Get("x"), request.args.Get("y"))
					_, err = cell.SendSCPI(scpicmd)
					sendResult(done, request.result, createResult(okb, "", nil))
				}
			case "screen":
				server.screenCache.mu.RLock()
				sendResult(done, request.result, createResult(server.screenCache.cache, "application/jpeg", nil))
				server.screenCache.mu.RUnlock()
			case "refresh_screen":
				func() {
					if len(server.screenCache.cache) == 0 || time.Now().Sub(server.screenCache.last).Seconds() > 1 {
						server.screenCache.mu.Lock()
						defer server.screenCache.mu.Unlock()
						if len(server.screenCache.cache) == 0 || time.Now().Sub(server.screenCache.last).Seconds() > 1 {
							server.screenCache.last = time.Now()
							server.screenCache.cache, err = cell.GetScreen()
							if err != nil {
								log.Println(err.Error())
							}
						}
					}
				}()
				sendResult(done, request.result, createResult(okb, "", nil))
			case "interference_power":
				js, err := cell.GetInterferencePower()
				if err != nil {
					sendResult(done, request.result, createResult(nil, "", err))
				} else {
					sendResult(done, request.result, createResult(js, "application/json", nil))
				}
			case "heartbeat":
				msg, err = cell.GetStatusMessage()
				sendResult(done, request.result, createResult(msg, "", nil))
			default:
				sendResult(done, request.result, createResult(nil, "", errors.New("unknown command")))
			}
		case <-time.After(server.pollPeriod):
			r := createRequest("heartbeat", nil)
			server.requestChannel <- r
			<-r.result
		}
		//Check Error Status == EOF
		if err != nil {
			switch err.Error() {
			case "EOF":
				log.Println("Connection loses on ", threadNumber, ", Poller exited")
				return
			default:
				log.Println("Thread ", threadNumber, " got error: ", err.Error())
			}
		}
	}
}

func createRequest(command string, args url.Values) *pollRequest {
	return &pollRequest{command, args, make(chan pollResult)}
}

func createResult(result []byte, resultType string, err error) pollResult {
	if resultType == "" {
		resultType = "text/plain"
	}
	return pollResult{result, resultType, err}
}

func sendResult(done <-chan struct{}, pipe chan<- pollResult, result pollResult) {
	select {
	case pipe <- result:
		sendSuccessCount.Add(1)
	case <-time.After(time.Second * 3):
		log.Println("Sending Timeout")
		sendPendingCount.Add(1)
	case <-done:
		return
	}
}
func receiveResult(pipe <-chan pollResult) *pollResult {
	select {
	case result := <-pipe:
		receiveSucessCount.Add(1)
		return &result
	case <-time.After(time.Second * 5):
		log.Println("Receive Timeout")
		receivePendingCount.Add(1)
	}
	return nil
}
<|endoftext|>


<commit_before>/*
   Copyright 2020 Docker Compose CLI authors

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
*/

package cmd

import (
	"context"
	"fmt"
	"io"
	"os"
	"strings"

	"github.com/pkg/errors"
	"github.com/spf13/cobra"

	"github.com/docker/compose-cli/api/client"
	"github.com/docker/compose-cli/api/containers"
	formatter2 "github.com/docker/compose-cli/formatter"
	"github.com/docker/compose-cli/utils/formatter"
)

type psOpts struct {
	all    bool
	quiet  bool
	json   bool
	format string
}

// PsCommand lists containers
func PsCommand() *cobra.Command {
	var opts psOpts
	cmd := &cobra.Command{
		Use:   "ps",
		Short: "List containers",
		RunE: func(cmd *cobra.Command, args []string) error {
			return runPs(cmd.Context(), opts)
		},
	}

	cmd.Flags().BoolVarP(&opts.quiet, "quiet", "q", false, "Only display IDs")
	cmd.Flags().BoolVarP(&opts.all, "all", "a", false, "Show all containers (default shows just running)")
	cmd.Flags().BoolVar(&opts.json, "json", false, "Format output as JSON")
	cmd.Flags().StringVar(&opts.format, "format", "", "Format the output. Values: [pretty | json]. (Default: pretty)")
	_ = cmd.Flags().MarkHidden("json")

	return cmd
}

func (o psOpts) validate() error {
	if o.quiet && o.json {
		return errors.New(`cannot combine "quiet" and "json" options`)
	}
	return nil
}

func runPs(ctx context.Context, opts psOpts) error {
	err := opts.validate()
	if err != nil {
		return err
	}
	c, err := client.New(ctx)
	if err != nil {
		return errors.Wrap(err, "cannot connect to backend")
	}

	containerList, err := c.ContainerService().List(ctx, opts.all)
	if err != nil {
		return errors.Wrap(err, "fetch containers")
	}

	if opts.quiet {
		for _, c := range containerList {
			fmt.Println(c.ID)
		}
		return nil
	}

	if opts.json {
		opts.format = formatter2.JSON
	}

	return formatter2.Print(containerList, opts.format, os.Stdout, func(w io.Writer) {
		for _, c := range containerList {
			_, _ = fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s\n", c.ID, c.Image, c.Command, c.Status,
				strings.Join(formatter.PortsToStrings(c.Ports, fqdn(c)), ", "))
		}
	}, "CONTAINER ID", "IMAGE", "COMMAND", "STATUS", "PORTS")
}

func fqdn(container containers.Container) string {
	fqdn := ""
	if container.Config != nil {
		fqdn = container.Config.FQDN
	}
	return fqdn
}
<commit_msg>Add comment on legacy --json option<commit_after>/*
   Copyright 2020 Docker Compose CLI authors

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
*/

package cmd

import (
	"context"
	"fmt"
	"io"
	"os"
	"strings"

	"github.com/pkg/errors"
	"github.com/spf13/cobra"

	"github.com/docker/compose-cli/api/client"
	"github.com/docker/compose-cli/api/containers"
	formatter2 "github.com/docker/compose-cli/formatter"
	"github.com/docker/compose-cli/utils/formatter"
)

type psOpts struct {
	all    bool
	quiet  bool
	json   bool
	format string
}

// PsCommand lists containers
func PsCommand() *cobra.Command {
	var opts psOpts
	cmd := &cobra.Command{
		Use:   "ps",
		Short: "List containers",
		RunE: func(cmd *cobra.Command, args []string) error {
			return runPs(cmd.Context(), opts)
		},
	}

	cmd.Flags().BoolVarP(&opts.quiet, "quiet", "q", false, "Only display IDs")
	cmd.Flags().BoolVarP(&opts.all, "all", "a", false, "Show all containers (default shows just running)")
	cmd.Flags().BoolVar(&opts.json, "json", false, "Format output as JSON")
	cmd.Flags().StringVar(&opts.format, "format", "", "Format the output. Values: [pretty | json]. (Default: pretty)")
	_ = cmd.Flags().MarkHidden("json") // Legacy. This is used by VSCode Docker extension

	return cmd
}

func (o psOpts) validate() error {
	if o.quiet && o.json {
		return errors.New(`cannot combine "quiet" and "json" options`)
	}
	return nil
}

func runPs(ctx context.Context, opts psOpts) error {
	err := opts.validate()
	if err != nil {
		return err
	}
	c, err := client.New(ctx)
	if err != nil {
		return errors.Wrap(err, "cannot connect to backend")
	}

	containerList, err := c.ContainerService().List(ctx, opts.all)
	if err != nil {
		return errors.Wrap(err, "fetch containers")
	}

	if opts.quiet {
		for _, c := range containerList {
			fmt.Println(c.ID)
		}
		return nil
	}

	if opts.json {
		opts.format = formatter2.JSON
	}

	return formatter2.Print(containerList, opts.format, os.Stdout, func(w io.Writer) {
		for _, c := range containerList {
			_, _ = fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s\n", c.ID, c.Image, c.Command, c.Status,
				strings.Join(formatter.PortsToStrings(c.Ports, fqdn(c)), ", "))
		}
	}, "CONTAINER ID", "IMAGE", "COMMAND", "STATUS", "PORTS")
}

func fqdn(container containers.Container) string {
	fqdn := ""
	if container.Config != nil {
		fqdn = container.Config.FQDN
	}
	return fqdn
}
<|endoftext|>


<commit_before>package cli

import (
	"fmt"
	"os"
	"sort"
	"strings"
)

// The basic datastructure used in cli. Actions are added for different paths to a router via the "Register" and
// "RegisterFunc" methods. These actions can be executed using the "Run" and "RunWithArgs" methods.
type Router struct {
	root *routingTreeNode

	initFailed bool
}

// Create a new router that will be used to register and run the actions of the application.
func NewRouter() *Router {
	r := &Router{}
	r.root = &routingTreeNode{children: map[string]*routingTreeNode{}}
	return r
}

// Run the given arguments against the registered actions, i.e. try to find a matching route and run the according
// action.
func (r *Router) Run(args ...string) (e error) {
	if r.initFailed {
		logger.Fatal("errors found during initialization")
	}
	// Find action and parse args.
	node, args := r.findNode(args, true)
	if node != nil && node.action != nil {
		if e := node.action.parseArgs(args); e != nil {
			node.showHelp()
			return e
		}
	} else { // Failed to find node.
		node.showHelp()
		return ErrorNoRoute
	}

	return node.action.runner.Run()
}

// Run the arguments from the commandline (aka os.Args) against the registered actions, i.e. try to find a matching
// route and run the according action.
func (r *Router) RunWithArgs() (e error) {
	return r.Run(os.Args[1:]...)
}

type annonymousAction struct {
	runner      func() error
	description string
}

func (aA *annonymousAction) Run() error {
	return aA.runner()
}

// Register the given function as handler for the given route. This is a shortcut for actions that don't need options or
// arguments. A description can be provided as an optional argument.
func (r *Router) RegisterFunc(path string, f func() error, desc string) {
	aA := &annonymousAction{runner: f}
	r.Register(path, aA, desc)
}

// Register the given action (some struct implementing the Runner interface) for the given route.
func (r *Router) Register(path string, runner Runner, desc string) {
	a, e := newAction(path, runner, desc)
	if e != nil {
		logger.Printf("%s", e)
		r.initFailed = true
		return
	}

	pathSegments := strings.Split(a.path, "/")
	node, pathSegments := r.findNode(pathSegments, false)
	if node != nil {
		if node.action != nil {
			logger.Printf("failed to register action for path %q: action for path %q already registered", a.path, node.action.path)
			r.initFailed = true
			return
		} else if len(pathSegments) == 0 && len(node.children) > 0 {
			logger.Printf("failed to register action for path %q: longer paths with this prefix exist", a.path)
			r.initFailed = true
			return
		}
	} else {
		node = r.root
	}

	for _, p := range pathSegments {
		newNode := &routingTreeNode{children: map[string]*routingTreeNode{}}
		node.children[p] = newNode
		node = newNode
	}

	node.action = a
}

func (r *Router) showHelp() {
}

// A tree used for easy access to the matching action. An action can only be set if there are no children, i.e. only
// leaf nodes can have actions.
type routingTreeNode struct {
	children map[string]*routingTreeNode
	action   *action
}

func (rt *routingTreeNode) showHelp() {
	if rt.action != nil {
		rt.action.showHelp()
	} else {
		t := &table{}
		rt.showTabularHelp(t)
		fmt.Println(t)
	}
}

func (rt *routingTreeNode) showTabularHelp(t *table) {
	if rt.action != nil {
		rt.action.showTabularHelp(t)
	} else {
		pathSegments := make([]string, 0, len(rt.children))
		for k, _ := range rt.children {
			pathSegments = append(pathSegments, k)
		}
		sort.Strings(pathSegments)

		for _, ps := range pathSegments {
			rt.children[ps].showTabularHelp(t)
		}
	}
}

// Find the node matching most segments of the given path. Will return the according tree node and the remaining (non
// matched) path segments.
func (r *Router) findNode(pathSegments []string, fuzzy bool) (*routingTreeNode, []string) {
	node := r.root
	for i, p := range pathSegments {
		if c, found := node.children[p]; found {
			node = c
		} else {
			if fuzzy { // try fuzzy search
				candidates := []string{}
				for key, _ := range node.children {
					if strings.HasPrefix(key, p) {
						candidates = append(candidates, key)
					}
				}
				if len(candidates) == 1 {
					node = node.children[candidates[0]]
					continue
				}
			}
			return node, pathSegments[i:]
		}
	}
	return node, nil
}
<commit_msg>add simple initializer for router<commit_after>package cli

import (
	"fmt"
	"os"
	"sort"
	"strings"
)

// The basic datastructure used in cli. Actions are added for different paths to a router via the "Register" and
// "RegisterFunc" methods. These actions can be executed using the "Run" and "RunWithArgs" methods.
type Router struct {
	root *routingTreeNode

	initFailed bool
}

func New(routes ...func(*Router) error) (*Router, error) {
	router := NewRouter()
	for _, route := range routes {
		if err := route(router); err != nil {
			return nil, err
		}
	}
	return router, nil
}

// Create a new router that will be used to register and run the actions of the application.
func NewRouter() *Router {
	r := &Router{}
	r.root = &routingTreeNode{children: map[string]*routingTreeNode{}}
	return r
}

// Run the given arguments against the registered actions, i.e. try to find a matching route and run the according
// action.
func (r *Router) Run(args ...string) (e error) {
	if r.initFailed {
		logger.Fatal("errors found during initialization")
	}
	// Find action and parse args.
	node, args := r.findNode(args, true)
	if node != nil && node.action != nil {
		if e := node.action.parseArgs(args); e != nil {
			node.showHelp()
			return e
		}
	} else { // Failed to find node.
		node.showHelp()
		return ErrorNoRoute
	}

	return node.action.runner.Run()
}

// Run the arguments from the commandline (aka os.Args) against the registered actions, i.e. try to find a matching
// route and run the according action.
func (r *Router) RunWithArgs() (e error) {
	return r.Run(os.Args[1:]...)
}

type annonymousAction struct {
	runner      func() error
	description string
}

func (aA *annonymousAction) Run() error {
	return aA.runner()
}

// Register the given function as handler for the given route. This is a shortcut for actions that don't need options or
// arguments. A description can be provided as an optional argument.
func (r *Router) RegisterFunc(path string, f func() error, desc string) {
	aA := &annonymousAction{runner: f}
	r.Register(path, aA, desc)
}

// Register the given action (some struct implementing the Runner interface) for the given route.
func (r *Router) Register(path string, runner Runner, desc string) {
	a, e := newAction(path, runner, desc)
	if e != nil {
		logger.Printf("%s", e)
		r.initFailed = true
		return
	}

	pathSegments := strings.Split(a.path, "/")
	node, pathSegments := r.findNode(pathSegments, false)
	if node != nil {
		if node.action != nil {
			logger.Printf("failed to register action for path %q: action for path %q already registered", a.path, node.action.path)
			r.initFailed = true
			return
		} else if len(pathSegments) == 0 && len(node.children) > 0 {
			logger.Printf("failed to register action for path %q: longer paths with this prefix exist", a.path)
			r.initFailed = true
			return
		}
	} else {
		node = r.root
	}

	for _, p := range pathSegments {
		newNode := &routingTreeNode{children: map[string]*routingTreeNode{}}
		node.children[p] = newNode
		node = newNode
	}

	node.action = a
}

func (r *Router) showHelp() {
}

// A tree used for easy access to the matching action. An action can only be set if there are no children, i.e. only
// leaf nodes can have actions.
type routingTreeNode struct {
	children map[string]*routingTreeNode
	action   *action
}

func (rt *routingTreeNode) showHelp() {
	if rt.action != nil {
		rt.action.showHelp()
	} else {
		t := &table{}
		rt.showTabularHelp(t)
		fmt.Println(t)
	}
}

func (rt *routingTreeNode) showTabularHelp(t *table) {
	if rt.action != nil {
		rt.action.showTabularHelp(t)
	} else {
		pathSegments := make([]string, 0, len(rt.children))
		for k, _ := range rt.children {
			pathSegments = append(pathSegments, k)
		}
		sort.Strings(pathSegments)

		for _, ps := range pathSegments {
			rt.children[ps].showTabularHelp(t)
		}
	}
}

// Find the node matching most segments of the given path. Will return the according tree node and the remaining (non
// matched) path segments.
func (r *Router) findNode(pathSegments []string, fuzzy bool) (*routingTreeNode, []string) {
	node := r.root
	for i, p := range pathSegments {
		if c, found := node.children[p]; found {
			node = c
		} else {
			if fuzzy { // try fuzzy search
				candidates := []string{}
				for key, _ := range node.children {
					if strings.HasPrefix(key, p) {
						candidates = append(candidates, key)
					}
				}
				if len(candidates) == 1 {
					node = node.children[candidates[0]]
					continue
				}
			}
			return node, pathSegments[i:]
		}
	}
	return node, nil
}
<|endoftext|>


<commit_before>package nsf

import (
	"errors"
	"io"
	"io/ioutil"
	"time"

	"github.com/mjibson/mog/codec/nsf/cpu6502"
)

const (
	// 1.79 MHz
	cpuClock = 236250000 / 11 / 12
)

var (
	// DefaultSampleRate is the default sample rate of a track after calling
	// Init().
	DefaultSampleRate = 44100
	ErrUnrecognized   = errors.New("nsf: unrecognized format")
)

const (
	NSF_HEADER_LEN = 0x80
	NSF_VERSION    = 0x5
	NSF_SONGS      = 0x6
	NSF_START      = 0x7
	NSF_LOAD       = 0x8
	NSF_INIT       = 0xa
	NSF_PLAY       = 0xc
	NSF_SONG       = 0xe
	NSF_ARTIST     = 0x2e
	NSF_COPYRIGHT  = 0x4e
	NSF_SPEED_NTSC = 0x6e
	NSF_BANKSWITCH = 0x70
	NSF_SPEED_PAL  = 0x78
	NSF_PAL_NTSC   = 0x7a
	NSF_EXTRA      = 0x7b
	NSF_ZERO       = 0x7c
)

func ReadNSF(r io.Reader) (n *NSF, err error) {
	n = &NSF{}
	n.b, err = ioutil.ReadAll(r)
	if err != nil {
		return
	}
	if len(n.b) < NSF_HEADER_LEN ||
		string(n.b[0:NSF_VERSION]) != "NESM\u001a" {
		return nil, ErrUnrecognized
	}
	n.Version = n.b[NSF_VERSION]
	n.Songs = n.b[NSF_SONGS]
	n.Start = n.b[NSF_START]
	n.LoadAddr = bLEtoUint16(n.b[NSF_LOAD:])
	n.InitAddr = bLEtoUint16(n.b[NSF_INIT:])
	n.PlayAddr = bLEtoUint16(n.b[NSF_PLAY:])
	n.Song = bToString(n.b[NSF_SONG:])
	n.Artist = bToString(n.b[NSF_ARTIST:])
	n.Copyright = bToString(n.b[NSF_COPYRIGHT:])
	n.SpeedNTSC = bLEtoUint16(n.b[NSF_SPEED_NTSC:])
	copy(n.Bankswitch[:], n.b[NSF_BANKSWITCH:NSF_SPEED_PAL])
	n.SpeedPAL = bLEtoUint16(n.b[NSF_SPEED_PAL:])
	n.PALNTSC = n.b[NSF_PAL_NTSC]
	n.Extra = n.b[NSF_EXTRA]
	n.Data = n.b[NSF_HEADER_LEN:]
	return
}

type NSF struct {
	*Ram
	*cpu6502.Cpu

	b []byte // raw NSF data

	Version byte
	Songs   byte
	Start   byte

	LoadAddr uint16
	InitAddr uint16
	PlayAddr uint16

	Song      string
	Artist    string
	Copyright string

	SpeedNTSC  uint16
	Bankswitch [8]byte
	SpeedPAL   uint16
	PALNTSC    byte
	Extra      byte
	Data       []byte

	// SampleRate is the sample rate at which samples will be generated. If not
	// set before Init(), it is set to DefaultSampleRate.
	SampleRate  int64
	totalTicks  int64
	frameTicks  int64
	sampleTicks int64
	playTicks   int64
	samples     []float32
	prevs       [4]float32
	pi          int // prevs index
}

func (n *NSF) Tick() {
	n.Ram.A.Step()
	n.totalTicks++
	n.frameTicks++
	if n.frameTicks == cpuClock/240 {
		n.frameTicks = 0
		n.Ram.A.FrameStep()
	}
	n.sampleTicks++
	if n.sampleTicks >= cpuClock/n.SampleRate {
		n.sampleTicks = 0
		n.append(n.Ram.A.Volume())
	}
	n.playTicks++
}

func (n *NSF) append(v float32) {
	n.prevs[n.pi] = v
	n.pi++
	if n.pi >= len(n.prevs) {
		n.pi = 0
	}
	var sum float32
	for _, s := range n.prevs {
		sum += s
	}
	sum /= float32(len(n.prevs))
	n.samples = append(n.samples, sum)
}

func (n *NSF) Init(song byte) {
	if n.SampleRate == 0 {
		n.SampleRate = int64(DefaultSampleRate)
	}
	n.Ram = new(Ram)
	n.Cpu = cpu6502.New(n.Ram)
	copy(n.Ram.M[n.LoadAddr:], n.Data)
	n.Ram.A.Init()
	n.Cpu.A = song - 1
	n.Cpu.PC = n.InitAddr
	n.Cpu.T = nil
	n.Cpu.Run()
	n.Cpu.T = n
}

func (n *NSF) Play(samples int) []float32 {
	playDur := time.Duration(n.SpeedNTSC) * time.Nanosecond * 1000
	ticksPerPlay := int64(playDur / (time.Second / cpuClock))
	n.samples = make([]float32, 0, samples)
	for len(n.samples) < samples {
		n.playTicks = 0
		n.Cpu.PC = n.PlayAddr
		n.Cpu.Halt = false
		for !n.Cpu.Halt && len(n.samples) < samples {
			n.Cpu.Step()
			if !n.Cpu.I() {
				panic("INTERRUPT")
			}
		}
		for i := ticksPerPlay - n.playTicks; i > 0 && len(n.samples) < samples; i-- {
			n.Tick()
		}
	}
	return n.samples
}

// little-endian [2]byte to uint16 conversion
func bLEtoUint16(b []byte) uint16 {
	return uint16(b[1])<<8 + uint16(b[0])
}

// null-terminated bytes to string
func bToString(b []byte) string {
	i := 0
	for i = range b {
		if b[i] == 0 {
			break
		}
	}
	return string(b[:i])
}

type Ram struct {
	M [0xffff + 1]byte
	A Apu
}

func (r *Ram) Read(v uint16) byte {
	if v == 0x4015 {
		return r.A.Read(v)
	} else {
		return r.M[v]
	}
}

func (r *Ram) Write(v uint16, b byte) {
	r.M[v] = b
	if v&0xf000 == 0x4000 {
		r.A.Write(v, b)
	}
}
<commit_msg>Use a switch here<commit_after>package nsf

import (
	"errors"
	"io"
	"io/ioutil"
	"time"

	"github.com/mjibson/mog/codec/nsf/cpu6502"
)

const (
	// 1.79 MHz
	cpuClock = 236250000 / 11 / 12
)

var (
	// DefaultSampleRate is the default sample rate of a track after calling
	// Init().
	DefaultSampleRate = 44100
	ErrUnrecognized   = errors.New("nsf: unrecognized format")
)

const (
	NSF_HEADER_LEN = 0x80
	NSF_VERSION    = 0x5
	NSF_SONGS      = 0x6
	NSF_START      = 0x7
	NSF_LOAD       = 0x8
	NSF_INIT       = 0xa
	NSF_PLAY       = 0xc
	NSF_SONG       = 0xe
	NSF_ARTIST     = 0x2e
	NSF_COPYRIGHT  = 0x4e
	NSF_SPEED_NTSC = 0x6e
	NSF_BANKSWITCH = 0x70
	NSF_SPEED_PAL  = 0x78
	NSF_PAL_NTSC   = 0x7a
	NSF_EXTRA      = 0x7b
	NSF_ZERO       = 0x7c
)

func ReadNSF(r io.Reader) (n *NSF, err error) {
	n = &NSF{}
	n.b, err = ioutil.ReadAll(r)
	if err != nil {
		return
	}
	if len(n.b) < NSF_HEADER_LEN ||
		string(n.b[0:NSF_VERSION]) != "NESM\u001a" {
		return nil, ErrUnrecognized
	}
	n.Version = n.b[NSF_VERSION]
	n.Songs = n.b[NSF_SONGS]
	n.Start = n.b[NSF_START]
	n.LoadAddr = bLEtoUint16(n.b[NSF_LOAD:])
	n.InitAddr = bLEtoUint16(n.b[NSF_INIT:])
	n.PlayAddr = bLEtoUint16(n.b[NSF_PLAY:])
	n.Song = bToString(n.b[NSF_SONG:])
	n.Artist = bToString(n.b[NSF_ARTIST:])
	n.Copyright = bToString(n.b[NSF_COPYRIGHT:])
	n.SpeedNTSC = bLEtoUint16(n.b[NSF_SPEED_NTSC:])
	copy(n.Bankswitch[:], n.b[NSF_BANKSWITCH:NSF_SPEED_PAL])
	n.SpeedPAL = bLEtoUint16(n.b[NSF_SPEED_PAL:])
	n.PALNTSC = n.b[NSF_PAL_NTSC]
	n.Extra = n.b[NSF_EXTRA]
	n.Data = n.b[NSF_HEADER_LEN:]
	return
}

type NSF struct {
	*Ram
	*cpu6502.Cpu

	b []byte // raw NSF data

	Version byte
	Songs   byte
	Start   byte

	LoadAddr uint16
	InitAddr uint16
	PlayAddr uint16

	Song      string
	Artist    string
	Copyright string

	SpeedNTSC  uint16
	Bankswitch [8]byte
	SpeedPAL   uint16
	PALNTSC    byte
	Extra      byte
	Data       []byte

	// SampleRate is the sample rate at which samples will be generated. If not
	// set before Init(), it is set to DefaultSampleRate.
	SampleRate  int64
	totalTicks  int64
	frameTicks  int64
	sampleTicks int64
	playTicks   int64
	samples     []float32
	prevs       [4]float32
	pi          int // prevs index
}

func (n *NSF) Tick() {
	n.Ram.A.Step()
	n.totalTicks++
	n.frameTicks++
	if n.frameTicks == cpuClock/240 {
		n.frameTicks = 0
		n.Ram.A.FrameStep()
	}
	n.sampleTicks++
	if n.sampleTicks >= cpuClock/n.SampleRate {
		n.sampleTicks = 0
		n.append(n.Ram.A.Volume())
	}
	n.playTicks++
}

func (n *NSF) append(v float32) {
	n.prevs[n.pi] = v
	n.pi++
	if n.pi >= len(n.prevs) {
		n.pi = 0
	}
	var sum float32
	for _, s := range n.prevs {
		sum += s
	}
	sum /= float32(len(n.prevs))
	n.samples = append(n.samples, sum)
}

func (n *NSF) Init(song byte) {
	if n.SampleRate == 0 {
		n.SampleRate = int64(DefaultSampleRate)
	}
	n.Ram = new(Ram)
	n.Cpu = cpu6502.New(n.Ram)
	copy(n.Ram.M[n.LoadAddr:], n.Data)
	n.Ram.A.Init()
	n.Cpu.A = song - 1
	n.Cpu.PC = n.InitAddr
	n.Cpu.T = nil
	n.Cpu.Run()
	n.Cpu.T = n
}

func (n *NSF) Play(samples int) []float32 {
	playDur := time.Duration(n.SpeedNTSC) * time.Nanosecond * 1000
	ticksPerPlay := int64(playDur / (time.Second / cpuClock))
	n.samples = make([]float32, 0, samples)
	for len(n.samples) < samples {
		n.playTicks = 0
		n.Cpu.PC = n.PlayAddr
		n.Cpu.Halt = false
		for !n.Cpu.Halt && len(n.samples) < samples {
			n.Cpu.Step()
			if !n.Cpu.I() {
				panic("INTERRUPT")
			}
		}
		for i := ticksPerPlay - n.playTicks; i > 0 && len(n.samples) < samples; i-- {
			n.Tick()
		}
	}
	return n.samples
}

// little-endian [2]byte to uint16 conversion
func bLEtoUint16(b []byte) uint16 {
	return uint16(b[1])<<8 + uint16(b[0])
}

// null-terminated bytes to string
func bToString(b []byte) string {
	i := 0
	for i = range b {
		if b[i] == 0 {
			break
		}
	}
	return string(b[:i])
}

type Ram struct {
	M [0xffff + 1]byte
	A Apu
}

func (r *Ram) Read(v uint16) byte {
	switch v {
	//case 0x2002:
	//return 0x80
	case 0x4015:
		return r.A.Read(v)
	default:
		return r.M[v]
	}
}

func (r *Ram) Write(v uint16, b byte) {
	r.M[v] = b
	if v&0xf000 == 0x4000 {
		r.A.Write(v, b)
	}
}
<|endoftext|>


<commit_before>package collect

import (
	"bytes"
	"compress/gzip"
	"encoding/json"
	"io/ioutil"
	"net/http"
	"time"

	"bosun.org/opentsdb"
	"bosun.org/slog"
)

func queuer() {
	for dp := range tchan {
		qlock.Lock()
		for {
			if len(queue) > MaxQueueLen {
				slock.Lock()
				dropped++
				slock.Unlock()
				break
			}
			m, err := json.Marshal(dp)
			if err != nil {
				slog.Error(err)
			} else {
				queue = append(queue, m)
			}
			select {
			case dp = <-tchan:
				continue
			default:
			}
			break
		}
		qlock.Unlock()
	}
}

func send() {
	for {
		qlock.Lock()
		if i := len(queue); i > 0 {
			if i > BatchSize {
				i = BatchSize
			}
			sending := queue[:i]
			queue = queue[i:]
			if Debug {
				slog.Infof("sending: %d, remaining: %d", i, len(queue))
			}
			qlock.Unlock()
			sendBatch(sending)
		} else {
			qlock.Unlock()
			time.Sleep(time.Second)
		}
	}
}

func sendBatch(batch []json.RawMessage) {
	if Print {
		for _, d := range batch {
			slog.Info(string(d))
		}
		recordSent(len(batch))
		return
	}
	var buf bytes.Buffer
	g := gzip.NewWriter(&buf)
	if err := json.NewEncoder(g).Encode(batch); err != nil {
		slog.Error(err)
		return
	}
	if err := g.Close(); err != nil {
		slog.Error(err)
		return
	}
	req, err := http.NewRequest("POST", tsdbURLs[currentTsdbURL], &buf)
	if err != nil {
		slog.Error(err)
		return
	}
	req.Header.Set("Content-Type", "application/json")
	req.Header.Set("Content-Encoding", "gzip")
	now := time.Now()
	resp, err := client.Do(req)
	d := time.Since(now).Nanoseconds() / 1e6
	if err == nil {
		defer resp.Body.Close()
	}
	Add("collect.post.total_duration", Tags, d)
	Add("collect.post.count", Tags, 1)
	// Some problem with connecting to the server; retry later.
	if err != nil || (resp.StatusCode != http.StatusNoContent && resp.StatusCode != http.Status) {
		if err != nil {
			Add("collect.post.error", Tags, 1)
			slog.Error(err)
			// Switch endpoint if possible
			currentTsdbURL = (currentTsdbURL + 1) % len(tsdbURLs)
		} else if resp.StatusCode != http.StatusNoContent && resp.StatusCode != http.StatusOK {
			Add("collect.post.bad_status", Tags, 1)
			slog.Errorln(resp.Status)
			body, err := ioutil.ReadAll(resp.Body)
			if err != nil {
				slog.Error(err)
			}
			if len(body) > 0 {
				slog.Error(string(body))
			}
			// Switch endpoint if possible
			currentTsdbURL = (currentTsdbURL + 1) % len(tsdbURLs)
		}
		restored := 0
		for _, msg := range batch {
			var dp opentsdb.DataPoint
			if err := json.Unmarshal(msg, &dp); err != nil {
				slog.Error(err)
				continue
			}
			restored++
			tchan <- &dp
		}
		d := time.Second * 5
		Add("collect.post.restore", Tags, int64(restored))
		slog.Infof("restored %d, sleeping %s", restored, d)
		time.Sleep(d)
		return
	}
	recordSent(len(batch))
}

func recordSent(num int) {
	if Debug {
		slog.Infoln("sent", num)
	}
	slock.Lock()
	sent += int64(num)
	slock.Unlock()
}
<commit_msg>Typo.<commit_after>package collect

import (
	"bytes"
	"compress/gzip"
	"encoding/json"
	"io/ioutil"
	"net/http"
	"time"

	"bosun.org/opentsdb"
	"bosun.org/slog"
)

func queuer() {
	for dp := range tchan {
		qlock.Lock()
		for {
			if len(queue) > MaxQueueLen {
				slock.Lock()
				dropped++
				slock.Unlock()
				break
			}
			m, err := json.Marshal(dp)
			if err != nil {
				slog.Error(err)
			} else {
				queue = append(queue, m)
			}
			select {
			case dp = <-tchan:
				continue
			default:
			}
			break
		}
		qlock.Unlock()
	}
}

func send() {
	for {
		qlock.Lock()
		if i := len(queue); i > 0 {
			if i > BatchSize {
				i = BatchSize
			}
			sending := queue[:i]
			queue = queue[i:]
			if Debug {
				slog.Infof("sending: %d, remaining: %d", i, len(queue))
			}
			qlock.Unlock()
			sendBatch(sending)
		} else {
			qlock.Unlock()
			time.Sleep(time.Second)
		}
	}
}

func sendBatch(batch []json.RawMessage) {
	if Print {
		for _, d := range batch {
			slog.Info(string(d))
		}
		recordSent(len(batch))
		return
	}
	var buf bytes.Buffer
	g := gzip.NewWriter(&buf)
	if err := json.NewEncoder(g).Encode(batch); err != nil {
		slog.Error(err)
		return
	}
	if err := g.Close(); err != nil {
		slog.Error(err)
		return
	}
	req, err := http.NewRequest("POST", tsdbURLs[currentTsdbURL], &buf)
	if err != nil {
		slog.Error(err)
		return
	}
	req.Header.Set("Content-Type", "application/json")
	req.Header.Set("Content-Encoding", "gzip")
	now := time.Now()
	resp, err := client.Do(req)
	d := time.Since(now).Nanoseconds() / 1e6
	if err == nil {
		defer resp.Body.Close()
	}
	Add("collect.post.total_duration", Tags, d)
	Add("collect.post.count", Tags, 1)
	// Some problem with connecting to the server; retry later.
	if err != nil || (resp.StatusCode != http.StatusNoContent && resp.StatusCode != http.StatusOK) {
		if err != nil {
			Add("collect.post.error", Tags, 1)
			slog.Error(err)
			// Switch endpoint if possible
			currentTsdbURL = (currentTsdbURL + 1) % len(tsdbURLs)
		} else if resp.StatusCode != http.StatusNoContent && resp.StatusCode != http.StatusOK {
			Add("collect.post.bad_status", Tags, 1)
			slog.Errorln(resp.Status)
			body, err := ioutil.ReadAll(resp.Body)
			if err != nil {
				slog.Error(err)
			}
			if len(body) > 0 {
				slog.Error(string(body))
			}
			// Switch endpoint if possible
			currentTsdbURL = (currentTsdbURL + 1) % len(tsdbURLs)
		}
		restored := 0
		for _, msg := range batch {
			var dp opentsdb.DataPoint
			if err := json.Unmarshal(msg, &dp); err != nil {
				slog.Error(err)
				continue
			}
			restored++
			tchan <- &dp
		}
		d := time.Second * 5
		Add("collect.post.restore", Tags, int64(restored))
		slog.Infof("restored %d, sleeping %s", restored, d)
		time.Sleep(d)
		return
	}
	recordSent(len(batch))
}

func recordSent(num int) {
	if Debug {
		slog.Infoln("sent", num)
	}
	slock.Lock()
	sent += int64(num)
	slock.Unlock()
}
<|endoftext|>


<commit_before>package rockredis

import (
	"encoding/json"
	"errors"
	"github.com/absolute8511/ZanRedisDB/common"
	"github.com/absolute8511/gorocksdb"
	"log"
	"os"
	"path"
	"sync"
	"time"
)

type RockConfig struct {
	DataDir          string
	DefaultReadOpts  *gorocksdb.ReadOptions
	DefaultWriteOpts *gorocksdb.WriteOptions
}

func NewRockConfig() *RockConfig {
	c := &RockConfig{
		DefaultReadOpts:  gorocksdb.NewDefaultReadOptions(),
		DefaultWriteOpts: gorocksdb.NewDefaultWriteOptions(),
	}
	c.DefaultReadOpts.SetVerifyChecksums(false)
	return c
}

type RockDB struct {
	cfg              *RockConfig
	eng              *gorocksdb.DB
	dbOpts           *gorocksdb.Options
	defaultWriteOpts *gorocksdb.WriteOptions
	defaultReadOpts  *gorocksdb.ReadOptions
	wb               *gorocksdb.WriteBatch
	quit             chan struct{}
	wg               sync.WaitGroup
	backupC          chan *BackupInfo
}

func OpenRockDB(cfg *RockConfig) (*RockDB, error) {
	if len(cfg.DataDir) == 0 {
		return nil, errors.New("config error")
	}

	os.MkdirAll(cfg.DataDir, common.DIR_PERM)
	// options need be adjust due to using hdd or sdd, please reference
	// https://github.com/facebook/rocksdb/wiki/RocksDB-Tuning-Guide
	bbto := gorocksdb.NewDefaultBlockBasedTableOptions()
	// use large block to reduce index block size for hdd
	// if using ssd, should use the default value
	bbto.SetBlockSize(1024 * 16)
	// should about 20% less than host RAM
	// http://smalldatum.blogspot.com/2016/09/tuning-rocksdb-block-cache.html
	bbto.SetBlockCache(gorocksdb.NewLRUCache(1024 * 1024 * 1024))
	// for hdd , we nee cache index and filter blocks
	bbto.SetCacheIndexAndFilterBlocks(true)
	filter := gorocksdb.NewBloomFilter(10)
	bbto.SetFilterPolicy(filter)
	opts := gorocksdb.NewDefaultOptions()
	opts.SetBlockBasedTableFactory(bbto)
	opts.SetCreateIfMissing(true)
	opts.SetMaxOpenFiles(-1)
	// keep level0_file_num_compaction_trigger * write_buffer_size = max_bytes_for_level_base to minimize write amplification
	opts.SetWriteBufferSize(1024 * 1024 * 128)
	opts.SetMaxWriteBufferNumber(8)
	opts.SetLevel0FileNumCompactionTrigger(4)
	opts.SetMaxBytesForLevelBase(1024 * 1024 * 1024)
	opts.SetMinWriteBufferNumberToMerge(2)
	opts.SetTargetFileSizeBase(1024 * 1024 * 64)
	opts.SetMaxBackgroundFlushes(2)
	opts.SetMaxBackgroundCompactions(4)
	opts.SetMinLevelToCompress(2)
	// we use table, so we use prefix seek feature
	opts.SetPrefixExtractor(gorocksdb.NewFixedPrefixTransform(3))
	//opts.SetMemtablePrefixBloomSizeRatio(0.1)
	opts.EnableStatistics()
	// https://github.com/facebook/mysql-5.6/wiki/my.cnf-tuning
	// rate limiter need to reduce the compaction io

	db := &RockDB{
		cfg:              cfg,
		dbOpts:           opts,
		defaultReadOpts:  cfg.DefaultReadOpts,
		defaultWriteOpts: cfg.DefaultWriteOpts,
		wb:               gorocksdb.NewWriteBatch(),
		backupC:          make(chan *BackupInfo),
		quit:             make(chan struct{}),
	}
	eng, err := gorocksdb.OpenDb(opts, db.GetDataDir())
	if err != nil {
		return nil, err
	}
	db.eng = eng

	db.wg.Add(1)
	go func() {
		defer db.wg.Done()
		db.backupLoop()
	}()
	return db, nil
}

func GetBackupDir(base string) string {
	return path.Join(base, "rocksdb_backup")
}

func (r *RockDB) GetBackupBase() string {
	return r.cfg.DataDir
}

func (r *RockDB) GetBackupDir() string {
	return GetBackupDir(r.cfg.DataDir)
}

func (r *RockDB) GetDataDir() string {
	return path.Join(r.cfg.DataDir, "rocksdb")
}

func (r *RockDB) reOpen() error {
	var err error
	r.eng, err = gorocksdb.OpenDb(r.dbOpts, r.GetDataDir())
	return err
}

func (r *RockDB) CompactRange() {
	var rg gorocksdb.Range
	r.eng.CompactRange(rg)
}

func (r *RockDB) Close() {
	close(r.quit)
	r.wg.Wait()
	if r.defaultReadOpts != nil {
		r.defaultReadOpts.Destroy()
	}
	if r.defaultWriteOpts != nil {
		r.defaultWriteOpts.Destroy()
	}
	if r.eng != nil {
		r.eng.Close()
	}
}

func (r *RockDB) GetStatistics() string {
	return r.dbOpts.GetStatistics()
}

func (r *RockDB) GetInternalStatus() map[string]interface{} {
	status := make(map[string]interface{})
	bbt := r.dbOpts.GetBlockBasedTableFactory()
	if bbt != nil {
		bc := bbt.GetBlockCache()
		if bc != nil {
			status["block-cache-usage"] = bc.GetUsage()
			status["block-cache-pinned-usage"] = bc.GetPinnedUsage()
		}
	}

	memStr := r.eng.GetProperty("rocksdb.estimate-table-readers-mem")
	status["estimate-table-readers-mem"] = memStr
	memStr = r.eng.GetProperty("rocksdb.cur-size-all-mem-tables")
	status["cur-size-all-mem-tables"] = memStr
	memStr = r.eng.GetProperty("rocksdb.cur-size-active-mem-table")
	status["cur-size-active-mem-tables"] = memStr
	return status
}

func (r *RockDB) GetInternalPropertyStatus(p string) string {
	return r.eng.GetProperty(p)
}

func (r *RockDB) ReadRange(sKey, eKey []byte, maxNum int) chan common.KVRecord {
	retChan := make(chan common.KVRecord, 32)
	go func() {
		it := NewDBRangeLimitIterator(r.eng, sKey, eKey, RangeClose, 0, maxNum, false)
		defer it.Close()
		for it = it; it.Valid(); it.Next() {
			key := it.Key()
			value := it.Value()
			retChan <- common.KVRecord{Key: key, Value: value}
		}
		close(retChan)
	}()
	return retChan
}

type BackupInfo struct {
	backupDir string
	started   chan struct{}
	done      chan struct{}
	rsp       []byte
	err       error
}

func newBackupInfo(dir string) *BackupInfo {
	return &BackupInfo{
		backupDir: dir,
		started:   make(chan struct{}),
		done:      make(chan struct{}),
	}
}

func (self *BackupInfo) WaitReady() {
	select {
	case <-self.started:
	case <-self.done:
	}
}

func (self *BackupInfo) GetResult() ([]byte, error) {
	select {
	case <-self.done:
	}
	return self.rsp, self.err
}

func (r *RockDB) backupLoop() {
	for {
		select {
		case rsp, ok := <-r.backupC:
			if !ok {
				return
			}

			func() {
				defer close(rsp.done)
				opts := gorocksdb.NewDefaultOptions()
				log.Printf("begin backup \n")
				start := time.Now()
				be, err := gorocksdb.OpenBackupEngine(opts, rsp.backupDir)
				if err != nil {
					log.Printf("backup engine failed: %v", err)
					rsp.err = err
					return
				}
				time.AfterFunc(time.Second*2, func() {
					close(rsp.started)
				})
				err = be.CreateNewBackup(r.eng)
				if err != nil {
					log.Printf("backup failed: %v", err)
					rsp.err = err
					return
				}
				cost := time.Now().Sub(start)
				beInfo := be.GetInfo()

				log.Printf("backup done (cost %v), total backup : %v\n", cost.String(), beInfo.GetCount())

				lastID := beInfo.GetBackupId(beInfo.GetCount() - 1)
				for i := 0; i < beInfo.GetCount(); i++ {
					id := beInfo.GetBackupId(i)
					log.Printf("backup data :%v, timestamp: %v, files: %v, size: %v", id, beInfo.GetTimestamp(i), beInfo.GetNumFiles(i),
						beInfo.GetSize(i))
				}
				beInfo.Destroy()
				be.PurgeOldBackups(r.eng, 3)
				be.Close()
				d, _ := json.Marshal(lastID)
				rsp.rsp = d
			}()
		case <-r.quit:
			return
		}
	}
}

func (r *RockDB) Backup() *BackupInfo {
	bi := newBackupInfo(r.GetBackupDir())
	select {
	case r.backupC <- bi:
	default:
		return nil
	}
	return bi
}

func (r *RockDB) IsLocalBackupOK(metaData []byte) (bool, error) {
	backupDir := r.GetBackupDir()
	var backupID int64
	err := json.Unmarshal(metaData, &backupID)
	if err != nil {
		return false, err
	}
	opts := gorocksdb.NewDefaultOptions()
	be, err := gorocksdb.OpenBackupEngine(opts, backupDir)
	if err != nil {
		log.Printf("backup engine open failed: %v", err)
		return false, err
	}
	beInfo := be.GetInfo()
	lastID := int64(0)
	if beInfo.GetCount() > 0 {
		lastID = beInfo.GetBackupId(beInfo.GetCount() - 1)
	}
	log.Printf("local total backup : %v, last: %v\n", beInfo.GetCount(), lastID)
	hasBackup := false
	for i := 0; i < beInfo.GetCount(); i++ {
		id := beInfo.GetBackupId(i)
		log.Printf("backup data :%v, timestamp: %v, files: %v, size: %v", id, beInfo.GetTimestamp(i), beInfo.GetNumFiles(i),
			beInfo.GetSize(i))
		if id == backupID {
			hasBackup = true
			break
		}
	}
	return hasBackup, nil
}

func (r *RockDB) Restore(metaData []byte) error {
	backupDir := r.GetBackupDir()
	hasBackup, _ := r.IsLocalBackupOK(metaData)
	if !hasBackup {
		return errors.New("no backup for restore")
	}
	var backupID int64
	err := json.Unmarshal(metaData, &backupID)
	if err != nil {
		return err
	}

	opts := gorocksdb.NewDefaultOptions()
	be, err := gorocksdb.OpenBackupEngine(opts, backupDir)
	if err != nil {
		log.Printf("backup engine open failed: %v", err)
		return err
	}
	beInfo := be.GetInfo()
	lastID := int64(0)
	if beInfo.GetCount() > 0 {
		lastID = beInfo.GetBackupId(beInfo.GetCount() - 1)
	}
	log.Printf("after sync total backup : %v, last: %v\n", beInfo.GetCount(), lastID)
	if lastID < backupID {
		return errors.New("no backup for restore")
	}
	if lastID > backupID {
		for i := 0; i < beInfo.GetCount(); i++ {
			id := beInfo.GetBackupId(i)
			log.Printf("backup data :%v, timestamp: %v, files: %v, size: %v", id, beInfo.GetTimestamp(i), beInfo.GetNumFiles(i),
				beInfo.GetSize(i))
			if int64(i) > backupID {
				// TODO: delete the backup with new id
			}
		}
	}

	start := time.Now()
	log.Printf("begin restore\n")
	r.eng.Close()
	restoreOpts := gorocksdb.NewRestoreOptions()
	err = be.RestoreDBFromLatestBackup(r.GetDataDir(), r.GetDataDir(), restoreOpts)
	if err != nil {
		log.Printf("restore failed: %v\n", err)
		return err
	}
	log.Printf("restore done, cost: %v\n", time.Now().Sub(start))
	be.Close()
	return r.reOpen()
}

func (r *RockDB) ClearBackup(metaData []byte) error {
	backupDir := r.GetBackupDir()
	var backupID int64
	err := json.Unmarshal(metaData, &backupID)
	if err != nil {
		return err
	}
	opts := gorocksdb.NewDefaultOptions()
	be, err := gorocksdb.OpenBackupEngine(opts, backupDir)
	if err != nil {
		log.Printf("backup engine open failed: %v", err)
		return err
	}
	beInfo := be.GetInfo()
	lastID := int64(0)
	if beInfo.GetCount() > 0 {
		lastID = beInfo.GetBackupId(beInfo.GetCount() - 1)
	}
	if lastID >= backupID {
		be.PurgeOldBackups(r.eng, 0)
	}
	beInfo.Destroy()
	be.Close()
	return nil
}
<commit_msg>use db options while backup<commit_after>package rockredis

import (
	"encoding/json"
	"errors"
	"github.com/absolute8511/ZanRedisDB/common"
	"github.com/absolute8511/gorocksdb"
	"log"
	"os"
	"path"
	"sync"
	"time"
)

type RockConfig struct {
	DataDir          string
	DefaultReadOpts  *gorocksdb.ReadOptions
	DefaultWriteOpts *gorocksdb.WriteOptions
}

func NewRockConfig() *RockConfig {
	c := &RockConfig{
		DefaultReadOpts:  gorocksdb.NewDefaultReadOptions(),
		DefaultWriteOpts: gorocksdb.NewDefaultWriteOptions(),
	}
	c.DefaultReadOpts.SetVerifyChecksums(false)
	return c
}

type RockDB struct {
	cfg              *RockConfig
	eng              *gorocksdb.DB
	dbOpts           *gorocksdb.Options
	defaultWriteOpts *gorocksdb.WriteOptions
	defaultReadOpts  *gorocksdb.ReadOptions
	wb               *gorocksdb.WriteBatch
	quit             chan struct{}
	wg               sync.WaitGroup
	backupC          chan *BackupInfo
}

func OpenRockDB(cfg *RockConfig) (*RockDB, error) {
	if len(cfg.DataDir) == 0 {
		return nil, errors.New("config error")
	}

	os.MkdirAll(cfg.DataDir, common.DIR_PERM)
	// options need be adjust due to using hdd or sdd, please reference
	// https://github.com/facebook/rocksdb/wiki/RocksDB-Tuning-Guide
	bbto := gorocksdb.NewDefaultBlockBasedTableOptions()
	// use large block to reduce index block size for hdd
	// if using ssd, should use the default value
	bbto.SetBlockSize(1024 * 16)
	// should about 20% less than host RAM
	// http://smalldatum.blogspot.com/2016/09/tuning-rocksdb-block-cache.html
	bbto.SetBlockCache(gorocksdb.NewLRUCache(1024 * 1024 * 1024))
	// for hdd , we nee cache index and filter blocks
	bbto.SetCacheIndexAndFilterBlocks(true)
	filter := gorocksdb.NewBloomFilter(10)
	bbto.SetFilterPolicy(filter)
	opts := gorocksdb.NewDefaultOptions()
	opts.SetBlockBasedTableFactory(bbto)
	opts.SetCreateIfMissing(true)
	opts.SetMaxOpenFiles(-1)
	// keep level0_file_num_compaction_trigger * write_buffer_size = max_bytes_for_level_base to minimize write amplification
	opts.SetWriteBufferSize(1024 * 1024 * 128)
	opts.SetMaxWriteBufferNumber(8)
	opts.SetLevel0FileNumCompactionTrigger(4)
	opts.SetMaxBytesForLevelBase(1024 * 1024 * 1024)
	opts.SetMinWriteBufferNumberToMerge(2)
	opts.SetTargetFileSizeBase(1024 * 1024 * 64)
	opts.SetMaxBackgroundFlushes(2)
	opts.SetMaxBackgroundCompactions(4)
	opts.SetMinLevelToCompress(2)
	// we use table, so we use prefix seek feature
	opts.SetPrefixExtractor(gorocksdb.NewFixedPrefixTransform(3))
	//opts.SetMemtablePrefixBloomSizeRatio(0.1)
	opts.EnableStatistics()
	// https://github.com/facebook/mysql-5.6/wiki/my.cnf-tuning
	// rate limiter need to reduce the compaction io

	db := &RockDB{
		cfg:              cfg,
		dbOpts:           opts,
		defaultReadOpts:  cfg.DefaultReadOpts,
		defaultWriteOpts: cfg.DefaultWriteOpts,
		wb:               gorocksdb.NewWriteBatch(),
		backupC:          make(chan *BackupInfo),
		quit:             make(chan struct{}),
	}
	eng, err := gorocksdb.OpenDb(opts, db.GetDataDir())
	if err != nil {
		return nil, err
	}
	db.eng = eng

	db.wg.Add(1)
	go func() {
		defer db.wg.Done()
		db.backupLoop()
	}()
	return db, nil
}

func GetBackupDir(base string) string {
	return path.Join(base, "rocksdb_backup")
}

func (r *RockDB) GetBackupBase() string {
	return r.cfg.DataDir
}

func (r *RockDB) GetBackupDir() string {
	return GetBackupDir(r.cfg.DataDir)
}

func (r *RockDB) GetDataDir() string {
	return path.Join(r.cfg.DataDir, "rocksdb")
}

func (r *RockDB) reOpen() error {
	var err error
	r.eng, err = gorocksdb.OpenDb(r.dbOpts, r.GetDataDir())
	return err
}

func (r *RockDB) CompactRange() {
	var rg gorocksdb.Range
	r.eng.CompactRange(rg)
}

func (r *RockDB) Close() {
	close(r.quit)
	r.wg.Wait()
	if r.defaultReadOpts != nil {
		r.defaultReadOpts.Destroy()
	}
	if r.defaultWriteOpts != nil {
		r.defaultWriteOpts.Destroy()
	}
	if r.eng != nil {
		r.eng.Close()
	}
}

func (r *RockDB) GetStatistics() string {
	return r.dbOpts.GetStatistics()
}

func (r *RockDB) GetInternalStatus() map[string]interface{} {
	status := make(map[string]interface{})
	bbt := r.dbOpts.GetBlockBasedTableFactory()
	if bbt != nil {
		bc := bbt.GetBlockCache()
		if bc != nil {
			status["block-cache-usage"] = bc.GetUsage()
			status["block-cache-pinned-usage"] = bc.GetPinnedUsage()
		}
	}

	memStr := r.eng.GetProperty("rocksdb.estimate-table-readers-mem")
	status["estimate-table-readers-mem"] = memStr
	memStr = r.eng.GetProperty("rocksdb.cur-size-all-mem-tables")
	status["cur-size-all-mem-tables"] = memStr
	memStr = r.eng.GetProperty("rocksdb.cur-size-active-mem-table")
	status["cur-size-active-mem-tables"] = memStr
	return status
}

func (r *RockDB) GetInternalPropertyStatus(p string) string {
	return r.eng.GetProperty(p)
}

func (r *RockDB) ReadRange(sKey, eKey []byte, maxNum int) chan common.KVRecord {
	retChan := make(chan common.KVRecord, 32)
	go func() {
		it := NewDBRangeLimitIterator(r.eng, sKey, eKey, RangeClose, 0, maxNum, false)
		defer it.Close()
		for it = it; it.Valid(); it.Next() {
			key := it.Key()
			value := it.Value()
			retChan <- common.KVRecord{Key: key, Value: value}
		}
		close(retChan)
	}()
	return retChan
}

type BackupInfo struct {
	backupDir string
	started   chan struct{}
	done      chan struct{}
	rsp       []byte
	err       error
}

func newBackupInfo(dir string) *BackupInfo {
	return &BackupInfo{
		backupDir: dir,
		started:   make(chan struct{}),
		done:      make(chan struct{}),
	}
}

func (self *BackupInfo) WaitReady() {
	select {
	case <-self.started:
	case <-self.done:
	}
}

func (self *BackupInfo) GetResult() ([]byte, error) {
	select {
	case <-self.done:
	}
	return self.rsp, self.err
}

func (r *RockDB) backupLoop() {
	for {
		select {
		case rsp, ok := <-r.backupC:
			if !ok {
				return
			}

			func() {
				defer close(rsp.done)
				log.Printf("begin backup \n")
				start := time.Now()
				be, err := gorocksdb.OpenBackupEngine(r.dbOpts, rsp.backupDir)
				if err != nil {
					log.Printf("backup engine failed: %v", err)
					rsp.err = err
					return
				}
				time.AfterFunc(time.Second*2, func() {
					close(rsp.started)
				})
				err = be.CreateNewBackup(r.eng)
				if err != nil {
					log.Printf("backup failed: %v", err)
					rsp.err = err
					return
				}
				cost := time.Now().Sub(start)
				beInfo := be.GetInfo()

				log.Printf("backup done (cost %v), total backup : %v\n", cost.String(), beInfo.GetCount())

				lastID := beInfo.GetBackupId(beInfo.GetCount() - 1)
				for i := 0; i < beInfo.GetCount(); i++ {
					id := beInfo.GetBackupId(i)
					log.Printf("backup data :%v, timestamp: %v, files: %v, size: %v", id, beInfo.GetTimestamp(i), beInfo.GetNumFiles(i),
						beInfo.GetSize(i))
				}
				beInfo.Destroy()
				be.PurgeOldBackups(r.eng, 3)
				be.Close()
				d, _ := json.Marshal(lastID)
				rsp.rsp = d
			}()
		case <-r.quit:
			return
		}
	}
}

func (r *RockDB) Backup() *BackupInfo {
	bi := newBackupInfo(r.GetBackupDir())
	select {
	case r.backupC <- bi:
	default:
		return nil
	}
	return bi
}

func (r *RockDB) IsLocalBackupOK(metaData []byte) (bool, error) {
	backupDir := r.GetBackupDir()
	var backupID int64
	err := json.Unmarshal(metaData, &backupID)
	if err != nil {
		return false, err
	}
	be, err := gorocksdb.OpenBackupEngine(r.dbOpts, backupDir)
	if err != nil {
		log.Printf("backup engine open failed: %v", err)
		return false, err
	}
	beInfo := be.GetInfo()
	lastID := int64(0)
	if beInfo.GetCount() > 0 {
		lastID = beInfo.GetBackupId(beInfo.GetCount() - 1)
	}
	log.Printf("local total backup : %v, last: %v\n", beInfo.GetCount(), lastID)
	hasBackup := false
	for i := 0; i < beInfo.GetCount(); i++ {
		id := beInfo.GetBackupId(i)
		log.Printf("backup data :%v, timestamp: %v, files: %v, size: %v", id, beInfo.GetTimestamp(i), beInfo.GetNumFiles(i),
			beInfo.GetSize(i))
		if id == backupID {
			hasBackup = true
			break
		}
	}
	return hasBackup, nil
}

func (r *RockDB) Restore(metaData []byte) error {
	backupDir := r.GetBackupDir()
	hasBackup, _ := r.IsLocalBackupOK(metaData)
	if !hasBackup {
		return errors.New("no backup for restore")
	}
	var backupID int64
	err := json.Unmarshal(metaData, &backupID)
	if err != nil {
		return err
	}

	opts := gorocksdb.NewDefaultOptions()
	be, err := gorocksdb.OpenBackupEngine(opts, backupDir)
	if err != nil {
		log.Printf("backup engine open failed: %v", err)
		return err
	}
	beInfo := be.GetInfo()
	lastID := int64(0)
	if beInfo.GetCount() > 0 {
		lastID = beInfo.GetBackupId(beInfo.GetCount() - 1)
	}
	log.Printf("after sync total backup : %v, last: %v\n", beInfo.GetCount(), lastID)
	if lastID < backupID {
		return errors.New("no backup for restore")
	}
	if lastID > backupID {
		for i := 0; i < beInfo.GetCount(); i++ {
			id := beInfo.GetBackupId(i)
			log.Printf("backup data :%v, timestamp: %v, files: %v, size: %v", id, beInfo.GetTimestamp(i), beInfo.GetNumFiles(i),
				beInfo.GetSize(i))
			if int64(i) > backupID {
				// TODO: delete the backup with new id
			}
		}
	}

	start := time.Now()
	log.Printf("begin restore\n")
	r.eng.Close()
	restoreOpts := gorocksdb.NewRestoreOptions()
	err = be.RestoreDBFromLatestBackup(r.GetDataDir(), r.GetDataDir(), restoreOpts)
	if err != nil {
		log.Printf("restore failed: %v\n", err)
		return err
	}
	log.Printf("restore done, cost: %v\n", time.Now().Sub(start))
	be.Close()
	return r.reOpen()
}

func (r *RockDB) ClearBackup(metaData []byte) error {
	backupDir := r.GetBackupDir()
	var backupID int64
	err := json.Unmarshal(metaData, &backupID)
	if err != nil {
		return err
	}
	opts := gorocksdb.NewDefaultOptions()
	be, err := gorocksdb.OpenBackupEngine(opts, backupDir)
	if err != nil {
		log.Printf("backup engine open failed: %v", err)
		return err
	}
	beInfo := be.GetInfo()
	lastID := int64(0)
	if beInfo.GetCount() > 0 {
		lastID = beInfo.GetBackupId(beInfo.GetCount() - 1)
	}
	if lastID >= backupID {
		be.PurgeOldBackups(r.eng, 0)
	}
	beInfo.Destroy()
	be.Close()
	return nil
}
<|endoftext|>


<commit_before>package auth

import (
    "crypto/rand"
    "encoding/base64"
	"log"
)

type SessionInfo struct {
	UserId int
    Username string
}

// GenerateRandomBytes returns securely generated random bytes.
// It will return an error if the system's secure random
// number generator fails to function correctly, in which
// case the caller should not continue.
func GenerateRandomBytes(n int) ([]byte, error) {
    b := make([]byte, n)
    _, err := rand.Read(b)
    // Note that err == nil only if we read len(b) bytes.
    if err != nil {
        return nil, err
    }

    return b, nil
}

// GenerateRandomString returns a URL-safe, base64 encoded
// securely generated random string.
func GenerateRandomString(s int) (string, error) {
    b, err := GenerateRandomBytes(s)
    return base64.URLEncoding.EncodeToString(b), err
}

// GenerateUID creates a random session ID, for use with the session store.
func GenerateUID() (string, error) {
	return GenerateRandomString(26)
}

// SetSession stores the given value under the given name in the database.
// If the given name is already used, its value is overwritten.
func SetSession(name string, value SessionInfo) {
	stmt, err := DB.Prepare("INSERT INTO sessions(id, userId, username) VALUES(?, ?, ?)")
	if err != nil {
		log.Println("Error while setting session: ")
		log.Println(err)
		return
	}
	_, err = stmt.Exec(name, value.UserId, value.Username)
	if err != nil {
		// must be because it already exists
		// so, try to UPDATE instead
		stmt, err := DB.Prepare("UPDATE sessions SET userId=?, username=? WHERE id=?")
		if err != nil {
			log.Println("Error while setting session: ")
			log.Println(err)
			return
		}
		_, err = stmt.Exec(value.UserId, value.Username, name)
		if err != nil {
			log.Println("Error while setting session: ")
			log.Println(err)
			return
		}
	}
}

// GetSession retrieves the session information for the given name
func GetSession(name string) (SessionInfo) {
	rows, err := DB.Query("SELECT userId, username from sessions where id = ?", name)
	if err != nil {
		log.Println("Error while getting session: ")
		log.Println(err)
		return SessionInfo{-1, ""}
	}
	defer rows.Close()
	rows.Next()
	retval := SessionInfo{-1, ""}
	err = rows.Scan(&retval.UserId, &retval.Username)

	return retval
}
<commit_msg>move sessions to use redis<commit_after>package auth

import (
    "crypto/rand"
    "encoding/base64"
	"log"
	"strconv"
)

type SessionInfo struct {
	UserId int
    Username string
}

// GenerateRandomBytes returns securely generated random bytes.
// It will return an error if the system's secure random
// number generator fails to function correctly, in which
// case the caller should not continue.
func GenerateRandomBytes(n int) ([]byte, error) {
    b := make([]byte, n)
    _, err := rand.Read(b)
    // Note that err == nil only if we read len(b) bytes.
    if err != nil {
        return nil, err
    }

    return b, nil
}

// GenerateRandomString returns a URL-safe, base64 encoded
// securely generated random string.
func GenerateRandomString(s int) (string, error) {
    b, err := GenerateRandomBytes(s)
    return base64.URLEncoding.EncodeToString(b), err
}

// GenerateUID creates a random session ID, for use with the session store.
func GenerateUID() (string, error) {
	return GenerateRandomString(26)
}

// SetSession stores the given value under the given name in the database.
// If the given name is already used, its value is overwritten.
func SetSession(name string, value SessionInfo) {
	result := RedisClient.HMSet("session:" + name, map[string]string{
		"userId": strconv.Itoa(value.UserId),
		"username": value.Username,
	})
	if result.Err() != nil {
		log.Println("Error while setting session: ")
		log.Println(result.Err())
		return
	}
}

// GetSession retrieves the session information for the given name
func GetSession(name string) (SessionInfo) {
	result := RedisClient.HGetAll("session:" + name)
	if result.Err() != nil {
		log.Println("Error while getting session: ")
		log.Println(result.Err())
		return SessionInfo{-1, ""}
	}

	resultMap, err := result.Result()
	if err != nil {
		log.Println("Error while getting session: ")
		log.Println(err)
		return SessionInfo{-1, ""}
	}

	retval := SessionInfo{-1, ""}

	retval.UserId, err = strconv.Atoi(resultMap["userId"])
	retval.Username = resultMap["username"]

	return retval
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"io"
	"net/http"
	"os"
	"regexp"
	"strconv"
	"time"

	proc "github.com/abulimov/cadvisor-companion/process"
)

var version = "0.0.5"

// set up cli vars
var argIP = flag.String("listen_ip", "", "IP to listen on, defaults to all IPs")
var argPort = flag.Int("port", 8801, "port to listen")
var versionFlag = flag.Bool("version", false, "print cAdvisor-companion version and exit")

var history proc.HistoryDB

// apiHandler handles http requests
func apiHandler(res http.ResponseWriter, req *http.Request) {
	var validPath = regexp.MustCompile("^/api/v1.0/([a-zA-Z0-9/]+)/processes$")

	// validate requested URL
	m := validPath.FindStringSubmatch(req.URL.Path)
	if m == nil {
		http.NotFound(res, req)
		return
	}

	// process get parameters

	// sortStr is used to get top sorted procs
	sortStr := req.URL.Query().Get("sort")

	// limit is used to limit top sorted procs
	limitStr := req.URL.Query().Get("limit")
	limit, err := strconv.Atoi(limitStr)
	if err != nil || limit < 1 {
		limit = 5
	}

	// interval is the interval we use to calculate CPU usage
	// and to iterate back to the past
	intervalStr := req.URL.Query().Get("interval")
	interval, err := strconv.Atoi(intervalStr)
	if err != nil || interval < 1 {
		interval = 1
	}

	// count is the count of resulting points in time
	countStr := req.URL.Query().Get("count")
	count, err := strconv.Atoi(countStr)
	if err != nil || count < 1 {
		count = 1
	}

	// containerID is the name of our container/cgroup
	containerID := "/" + m[1]

	// set response headers
	res.Header().Set(
		"Content-Type",
		"text/json",
	)
	var result []proc.Snapshot
	var ps *proc.Snapshot
	fail := func(err error) {
		message := map[string]string{"error": err.Error()}
		messageJSON, _ := json.Marshal(message)
		fmt.Printf("Error: %s\n", err.Error())
		res.WriteHeader(500) // HTTP 500
		io.WriteString(res, string(messageJSON))
		return
	}
	// case our possible sort parameters
	switch sortStr {
	case "cpu":
		for i := count - 1; i >= 0; i-- {
			ps, err = history.GetTopCPU(containerID, limit, interval, i*interval+1)
			if err != nil {
				fail(err)
				return
			}
			result = append(result, *ps)
		}
	case "mem":
		for i := count - 1; i >= 0; i-- {
			ps, err = history.GetTopMem(containerID, limit, interval, i*interval+1)
			if err != nil {
				fail(err)
				return
			}
			result = append(result, *ps)
		}
	case "":
		for i := count - 1; i >= 0; i-- {
			ps, err = history.GetLastData(containerID, interval, i*interval+1)
			if err != nil {
				fail(err)
				return
			}
			result = append(result, *ps)
		}
	}
	jsonResult, _ := json.Marshal(result)
	io.WriteString(res, string(jsonResult))
}

// collectData scrapes procs data for all containers every second
// and keeps it in global history var
func collectData(rootPath string) {
	for {
		timeStamp := time.Now()
		// get all processes without cgroup grouping
		allProcs, _ := proc.GetProcesses(rootPath)
		// group all processes by their cgroups
		cgroupsProcs := allProcs.GetCgroupsMap()

		entry := make(proc.HistoryEntry)
		for e, p := range cgroupsProcs {
			entry[e] = proc.Snapshot{timeStamp, p}
		}
		history.Push(entry)

		time.Sleep(time.Second)
	}
}

func getRootPath() string {
	dir, err := os.Stat("/rootfs")
	if err != nil {
		return "/"
	}
	// check if the /rootfs is indeed a directory or not
	if !dir.IsDir() {
		return "/"
	}
	return "/rootfs"
}

func main() {
	flag.Parse()

	if *versionFlag {
		fmt.Printf("cAdvisor-companion version %s\n", version)
		os.Exit(0)
	}

	// rootPath is where our /proc is mounted.
	// When we are inside the container, host's /proc should be mounted
	// at /rootfs/proc, so rootPath is /rootfs
	rootPath := getRootPath()

	// start collecting data
	go collectData(rootPath)

	addr := fmt.Sprintf("%s:%d", *argIP, *argPort)
	fmt.Printf("Starting cAdvisor-companion version: %q on port %d\n", version, *argPort)
	http.HandleFunc("/api/", apiHandler)
	err := http.ListenAndServe(addr, nil)
	if err != nil {
		fmt.Println(err)
		os.Exit(1)
	}
}
<commit_msg>Simplified apiHandler logic<commit_after>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"io"
	"net/http"
	"os"
	"regexp"
	"strconv"
	"time"

	proc "github.com/abulimov/cadvisor-companion/process"
)

var version = "0.0.5"

// set up cli vars
var argIP = flag.String("listen_ip", "", "IP to listen on, defaults to all IPs")
var argPort = flag.Int("port", 8801, "port to listen")
var versionFlag = flag.Bool("version", false, "print cAdvisor-companion version and exit")

var history proc.HistoryDB

// apiHandler handles http requests
func apiHandler(res http.ResponseWriter, req *http.Request) {
	var validPath = regexp.MustCompile("^/api/v1.0/([a-zA-Z0-9/]+)/processes$")

	// validate requested URL
	m := validPath.FindStringSubmatch(req.URL.Path)
	if m == nil {
		http.NotFound(res, req)
		return
	}

	// process get parameters

	// sortStr is used to get top sorted procs
	sortStr := req.URL.Query().Get("sort")

	// limit is used to limit top sorted procs
	limitStr := req.URL.Query().Get("limit")
	limit, err := strconv.Atoi(limitStr)
	if err != nil || limit < 1 {
		limit = 5
	}

	// interval is the interval we use to calculate CPU usage
	// and to iterate back to the past
	intervalStr := req.URL.Query().Get("interval")
	interval, err := strconv.Atoi(intervalStr)
	if err != nil || interval < 1 {
		interval = 1
	}

	// count is the count of resulting points in time
	countStr := req.URL.Query().Get("count")
	count, err := strconv.Atoi(countStr)
	if err != nil || count < 1 {
		count = 1
	}

	// containerID is the name of our container/cgroup
	containerID := "/" + m[1]

	// set response headers
	res.Header().Set(
		"Content-Type",
		"text/json",
	)
	var result []proc.Snapshot
	var ps *proc.Snapshot
	fail := func(err error) {
		message := map[string]string{"error": err.Error()}
		messageJSON, _ := json.Marshal(message)
		fmt.Printf("Error: %s\n", err.Error())
		res.WriteHeader(500) // HTTP 500
		io.WriteString(res, string(messageJSON))
		return
	}

	// get data for all `count` HistoryEntries
	for i := count - 1; i >= 0; i-- {
		// case our possible sort parameters
		switch sortStr {
		case "cpu":
			ps, err = history.GetTopCPU(containerID, limit, interval, i*interval+1)
		case "mem":
			ps, err = history.GetTopMem(containerID, limit, interval, i*interval+1)
		case "":
			ps, err = history.GetLastData(containerID, interval, i*interval+1)
		}
		if err != nil {
			fail(err)
			return
		}
		result = append(result, *ps)
	}
	jsonResult, _ := json.Marshal(result)
	io.WriteString(res, string(jsonResult))
}

// collectData scrapes procs data for all containers every second
// and keeps it in global history var
func collectData(rootPath string) {
	for {
		timeStamp := time.Now()
		// get all processes without cgroup grouping
		allProcs, _ := proc.GetProcesses(rootPath)
		// group all processes by their cgroups
		cgroupsProcs := allProcs.GetCgroupsMap()

		entry := make(proc.HistoryEntry)
		for e, p := range cgroupsProcs {
			entry[e] = proc.Snapshot{Timestamp: timeStamp, Processes: p}
		}
		history.Push(entry)

		time.Sleep(time.Second)
	}
}

func getRootPath() string {
	dir, err := os.Stat("/rootfs")
	if err != nil {
		return "/"
	}
	// check if the /rootfs is indeed a directory or not
	if !dir.IsDir() {
		return "/"
	}
	return "/rootfs"
}

func main() {
	flag.Parse()

	if *versionFlag {
		fmt.Printf("cAdvisor-companion version %s\n", version)
		os.Exit(0)
	}

	// rootPath is where our /proc is mounted.
	// When we are inside the container, host's /proc should be mounted
	// at /rootfs/proc, so rootPath is /rootfs
	rootPath := getRootPath()

	// start collecting data
	go collectData(rootPath)

	addr := fmt.Sprintf("%s:%d", *argIP, *argPort)
	fmt.Printf("Starting cAdvisor-companion version: %q on port %d\n", version, *argPort)
	http.HandleFunc("/api/", apiHandler)
	err := http.ListenAndServe(addr, nil)
	if err != nil {
		fmt.Println(err)
		os.Exit(1)
	}
}
<|endoftext|>


<commit_before>package storage

import (
	"io"
	"testing"

	"github.com/docker/distribution"
	"github.com/docker/distribution/context"
	"github.com/docker/distribution/registry/storage/cache/memory"
	"github.com/docker/distribution/registry/storage/driver"
	"github.com/docker/distribution/registry/storage/driver/inmemory"
)

type setupEnv struct {
	ctx      context.Context
	driver   driver.StorageDriver
	expected []string
	registry distribution.Namespace
}

func setupFS(t *testing.T) *setupEnv {
	d := inmemory.New()
	c := []byte("")
	ctx := context.Background()
	registry, err := NewRegistry(ctx, d, BlobDescriptorCacheProvider(memory.NewInMemoryBlobDescriptorCacheProvider()), EnableRedirect)
	if err != nil {
		t.Fatalf("error creating registry: %v", err)
	}
	rootpath, _ := pathFor(repositoriesRootPathSpec{})

	repos := []string{
		"/foo/a/_layers/1",
		"/foo/b/_layers/2",
		"/bar/c/_layers/3",
		"/bar/d/_layers/4",
		"/foo/d/in/_layers/5",
		"/an/invalid/repo",
		"/bar/d/_layers/ignored/dir/6",
	}

	for _, repo := range repos {
		if err := d.PutContent(ctx, rootpath+repo, c); err != nil {
			t.Fatalf("Unable to put to inmemory fs")
		}
	}

	expected := []string{
		"bar/c",
		"bar/d",
		"foo/a",
		"foo/b",
		"foo/d/in",
	}

	return &setupEnv{
		ctx:      ctx,
		driver:   d,
		expected: expected,
		registry: registry,
	}
}

func TestCatalog(t *testing.T) {
	env := setupFS(t)

	p := make([]string, 50)

	numFilled, err := env.registry.Repositories(env.ctx, p, "")

	if !testEq(p, env.expected, numFilled) {
		t.Errorf("Expected catalog repos err")
	}

	if err != io.EOF {
		t.Errorf("Catalog has more values which we aren't expecting")
	}
}

func TestCatalogInParts(t *testing.T) {
	env := setupFS(t)

	chunkLen := 2
	p := make([]string, chunkLen)

	numFilled, err := env.registry.Repositories(env.ctx, p, "")
	if err == io.EOF || numFilled != len(p) {
		t.Errorf("Expected more values in catalog")
	}

	if !testEq(p, env.expected[0:chunkLen], numFilled) {
		t.Errorf("Expected catalog first chunk err")
	}

	lastRepo := p[len(p)-1]
	numFilled, err = env.registry.Repositories(env.ctx, p, lastRepo)

	if err == io.EOF || numFilled != len(p) {
		t.Errorf("Expected more values in catalog")
	}

	if !testEq(p, env.expected[chunkLen:chunkLen*2], numFilled) {
		t.Errorf("Expected catalog second chunk err")
	}

	lastRepo = p[len(p)-1]
	numFilled, err = env.registry.Repositories(env.ctx, p, lastRepo)

	if err != io.EOF {
		t.Errorf("Catalog has more values which we aren't expecting")
	}

	if !testEq(p, env.expected[chunkLen*2:chunkLen*3-1], numFilled) {
		t.Errorf("Expected catalog third chunk err")
	}

}

func testEq(a, b []string, size int) bool {
	for cnt := 0; cnt < size-1; cnt++ {
		if a[cnt] != b[cnt] {
			return false
		}
	}
	return true
}
<commit_msg>Use distribution APIs to generate catalog test environment<commit_after>package storage

import (
	"io"
	"testing"

	"github.com/docker/distribution"
	"github.com/docker/distribution/context"
	"github.com/docker/distribution/digest"
	"github.com/docker/distribution/reference"
	"github.com/docker/distribution/registry/storage/cache/memory"
	"github.com/docker/distribution/registry/storage/driver"
	"github.com/docker/distribution/registry/storage/driver/inmemory"
	"github.com/docker/distribution/testutil"
)

type setupEnv struct {
	ctx      context.Context
	driver   driver.StorageDriver
	expected []string
	registry distribution.Namespace
}

func setupFS(t *testing.T) *setupEnv {
	d := inmemory.New()
	ctx := context.Background()
	registry, err := NewRegistry(ctx, d, BlobDescriptorCacheProvider(memory.NewInMemoryBlobDescriptorCacheProvider()), EnableRedirect)
	if err != nil {
		t.Fatalf("error creating registry: %v", err)
	}

	repos := []string{
		"foo/a",
		"foo/b",
		"bar/c",
		"bar/d",
		"foo/d/in",
	}

	for _, repo := range repos {
		makeRepo(t, ctx, repo, registry)
	}

	expected := []string{
		"bar/c",
		"bar/d",
		"foo/a",
		"foo/b",
		"foo/d/in",
	}

	return &setupEnv{
		ctx:      ctx,
		driver:   d,
		expected: expected,
		registry: registry,
	}
}

func makeRepo(t *testing.T, ctx context.Context, name string, reg distribution.Namespace) {
	named, err := reference.ParseNamed(name)
	if err != nil {
		t.Fatal(err)
	}

	repo, _ := reg.Repository(ctx, named)
	manifests, _ := repo.Manifests(ctx)

	layers, err := testutil.CreateRandomLayers(1)
	if err != nil {
		t.Fatal(err)
	}

	err = testutil.UploadBlobs(repo, layers)
	if err != nil {
		t.Fatalf("failed to upload layers: %v", err)
	}

	getKeys := func(digests map[digest.Digest]io.ReadSeeker) (ds []digest.Digest) {
		for d := range digests {
			ds = append(ds, d)
		}
		return
	}

	manifest, err := testutil.MakeSchema1Manifest(getKeys(layers))
	if err != nil {
		t.Fatal(err)
	}

	_, err = manifests.Put(ctx, manifest)
	if err != nil {
		t.Fatalf("manifest upload failed: %v", err)
	}

}

func TestCatalog(t *testing.T) {
	env := setupFS(t)

	p := make([]string, 50)

	numFilled, err := env.registry.Repositories(env.ctx, p, "")
	if numFilled != len(env.expected) {
		t.Errorf("missing items in catalog")
	}

	if !testEq(p, env.expected, len(env.expected)) {
		t.Errorf("Expected catalog repos err")
	}

	if err != io.EOF {
		t.Errorf("Catalog has more values which we aren't expecting")
	}
}

func TestCatalogInParts(t *testing.T) {
	env := setupFS(t)

	chunkLen := 2
	p := make([]string, chunkLen)

	numFilled, err := env.registry.Repositories(env.ctx, p, "")
	if err == io.EOF || numFilled != len(p) {
		t.Errorf("Expected more values in catalog")
	}

	if !testEq(p, env.expected[0:chunkLen], numFilled) {
		t.Errorf("Expected catalog first chunk err")
	}

	lastRepo := p[len(p)-1]
	numFilled, err = env.registry.Repositories(env.ctx, p, lastRepo)

	if err == io.EOF || numFilled != len(p) {
		t.Errorf("Expected more values in catalog")
	}

	if !testEq(p, env.expected[chunkLen:chunkLen*2], numFilled) {
		t.Errorf("Expected catalog second chunk err")
	}

	lastRepo = p[len(p)-1]
	numFilled, err = env.registry.Repositories(env.ctx, p, lastRepo)

	if err != io.EOF {
		t.Errorf("Catalog has more values which we aren't expecting")
	}

	if !testEq(p, env.expected[chunkLen*2:chunkLen*3-1], numFilled) {
		t.Errorf("Expected catalog third chunk err")
	}

}

func testEq(a, b []string, size int) bool {
	for cnt := 0; cnt < size-1; cnt++ {
		if a[cnt] != b[cnt] {
			return false
		}
	}
	return true
}
<|endoftext|>