<commit_before>// +build linux darwin

package shell

import (
	"bytes"
	"strings"
	"sync"
	"testing"

	"github.com/stretchr/testify/require"

	"github.com/gruntwork-io/terragrunt/options"
	"github.com/stretchr/testify/assert"
)

func TestCommandOutputOrder(t *testing.T) {
	t.Parallel()

	terragruntOptions, err := options.NewTerragruntOptionsForTest("")
	require.NoError(t, err, "Unexpected error creating NewTerragruntOptionsForTest: %v", err)

	// Specify a single (locking) buffer for both as a way to check that the output is being written in the correct
	// order
	var allOutputBuffer BufferWithLocking
	terragruntOptions.Writer = &allOutputBuffer
	terragruntOptions.ErrWriter = &allOutputBuffer

	terragruntOptions.TerraformCliArgs = append(terragruntOptions.TerraformCliArgs, "same")
	out, err := RunShellCommandWithOutput(terragruntOptions, "", false, "../testdata/test_outputs.sh", "same")

	require.NotNil(t, out, "Should get output")
	assert.Nil(t, err, "Should have no error")

	allOutputs := strings.Split(strings.TrimSpace(allOutputBuffer.String()), "\n")

	require.True(t, len(allOutputs) == 5, "Expected 5 entries, but got %d: %v", len(allOutputs), allOutputs)
	assert.Equal(t, "stdout1", allOutputs[0], "First one from stdout")
	assert.Equal(t, "stderr1", allOutputs[1], "First one from stderr")
	assert.Equal(t, "stdout2", allOutputs[2], "Second one from stdout")
	assert.Equal(t, "stderr2", allOutputs[3], "Second one from stderr")
	assert.Equal(t, "stderr3", allOutputs[4], "Third one from stderr")

	stdOutputs := strings.Split(strings.TrimSpace(out.Stdout), "\n")

	require.True(t, len(stdOutputs) == 2, "Expected 2 entries, but got %d: %v", len(stdOutputs), stdOutputs)
	assert.Equal(t, "stdout1", stdOutputs[0], "First one from stdout")
	assert.Equal(t, "stdout2", stdOutputs[1], "Second one from stdout")

	stdErrs := strings.Split(strings.TrimSpace(out.Stderr), "\n")

	require.True(t, len(stdErrs) == 3, "Expected 3 entries, but got %d: %v", len(stdErrs), stdErrs)
	assert.Equal(t, "stderr1", stdErrs[0], "First one from stderr")
	assert.Equal(t, "stderr2", stdErrs[1], "Second one from stderr")
	assert.Equal(t, "stderr3", stdErrs[2], "Second one from stderr")
}

// A goroutine-safe bytes.Buffer
type BufferWithLocking struct {
	buffer bytes.Buffer
	mutex  sync.Mutex
}

// Write appends the contents of p to the buffer, growing the buffer as needed. It returns
// the number of bytes written.
func (s *BufferWithLocking) Write(p []byte) (n int, err error) {
	s.mutex.Lock()
	defer s.mutex.Unlock()
	return s.buffer.Write(p)
}

// String returns the contents of the unread portion of the buffer
// as a string.  If the Buffer is a nil pointer, it returns "<nil>".
func (s *BufferWithLocking) String() string {
	s.mutex.Lock()
	defer s.mutex.Unlock()
	return s.buffer.String()
}
<commit_msg>Fix test<commit_after>// +build linux darwin

package shell

import (
	"bytes"
	"strings"
	"sync"
	"testing"

	"github.com/stretchr/testify/require"

	"github.com/gruntwork-io/terragrunt/options"
	"github.com/stretchr/testify/assert"
)

func TestCommandOutputOrder(t *testing.T) {
	t.Parallel()

	t.Run("withPtty", func(t *testing.T) {
		t.Parallel()
		testCommandOutputOrder(t, true)
	})
	t.Run("withoutPtty", func(t *testing.T) {
		t.Parallel()
		testCommandOutputOrder(t, false)
	})
}

func testCommandOutputOrder(t *testing.T, withPtty bool) {
	terragruntOptions, err := options.NewTerragruntOptionsForTest("")
	require.NoError(t, err, "Unexpected error creating NewTerragruntOptionsForTest: %v", err)

	// Specify a single (locking) buffer for both as a way to check that the output is being written in the correct
	// order
	var allOutputBuffer BufferWithLocking
	terragruntOptions.Writer = &allOutputBuffer
	terragruntOptions.ErrWriter = &allOutputBuffer

	terragruntOptions.TerraformCliArgs = append(terragruntOptions.TerraformCliArgs, "same")

	out, err := RunShellCommandWithOutput(terragruntOptions, "", false, false, "../testdata/test_outputs.sh", "same")

	require.NotNil(t, out, "Should get output")
	assert.Nil(t, err, "Should have no error")

	allOutputs := strings.Split(strings.TrimSpace(allOutputBuffer.String()), "\n")

	require.True(t, len(allOutputs) == 5, "Expected 5 entries, but got %d: %v", len(allOutputs), allOutputs)
	assert.Equal(t, "stdout1", allOutputs[0], "First one from stdout")
	assert.Equal(t, "stderr1", allOutputs[1], "First one from stderr")
	assert.Equal(t, "stdout2", allOutputs[2], "Second one from stdout")
	assert.Equal(t, "stderr2", allOutputs[3], "Second one from stderr")
	assert.Equal(t, "stderr3", allOutputs[4], "Third one from stderr")

	stdOutputs := strings.Split(strings.TrimSpace(out.Stdout), "\n")

	require.True(t, len(stdOutputs) == 2, "Expected 2 entries, but got %d: %v", len(stdOutputs), stdOutputs)
	assert.Equal(t, "stdout1", stdOutputs[0], "First one from stdout")
	assert.Equal(t, "stdout2", stdOutputs[1], "Second one from stdout")

	stdErrs := strings.Split(strings.TrimSpace(out.Stderr), "\n")

	require.True(t, len(stdErrs) == 3, "Expected 3 entries, but got %d: %v", len(stdErrs), stdErrs)
	assert.Equal(t, "stderr1", stdErrs[0], "First one from stderr")
	assert.Equal(t, "stderr2", stdErrs[1], "Second one from stderr")
	assert.Equal(t, "stderr3", stdErrs[2], "Second one from stderr")
}

// A goroutine-safe bytes.Buffer
type BufferWithLocking struct {
	buffer bytes.Buffer
	mutex  sync.Mutex
}

// Write appends the contents of p to the buffer, growing the buffer as needed. It returns
// the number of bytes written.
func (s *BufferWithLocking) Write(p []byte) (n int, err error) {
	s.mutex.Lock()
	defer s.mutex.Unlock()
	return s.buffer.Write(p)
}

// String returns the contents of the unread portion of the buffer
// as a string.  If the Buffer is a nil pointer, it returns "<nil>".
func (s *BufferWithLocking) String() string {
	s.mutex.Lock()
	defer s.mutex.Unlock()
	return s.buffer.String()
}
<|endoftext|>


<commit_before>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package time

// Sleep pauses the current goroutine for the duration d.
func Sleep(d Duration)

func nano() int64 {
	sec, nsec := now()
	return sec*1e9 + int64(nsec)
}

// Interface to timers implemented in package runtime.
// Must be in sync with ../runtime/runtime.h:/^struct.Timer$
type runtimeTimer struct {
	i      int32
	when   int64
	period int64
	f      func(int64, interface{}) // NOTE: must not be closure
	arg    interface{}
}

// when is a helper function for setting the 'when' field of a runtimeTimer.
// It returns what the time will be, in nanoseconds, Duration d in the future.
// If d is negative, it is ignored.  If the returned value would be less than
// zero because of an overflow, MaxInt64 is returned.
func when(d Duration) int64 {
	if d <= 0 {
		return nano()
	}
	t := nano() + int64(d)
	if t < 0 {
		t = 1<<63 - 1 // math.MaxInt64
	}
	return t
}

func startTimer(*runtimeTimer)
func stopTimer(*runtimeTimer) bool

// The Timer type represents a single event.
// When the Timer expires, the current time will be sent on C,
// unless the Timer was created by AfterFunc.
type Timer struct {
	C <-chan Time
	r runtimeTimer
}

// Stop prevents the Timer from firing.
// It returns true if the call stops the timer, false if the timer has already
// expired or been stopped.
// Stop does not close the channel, to prevent a read from the channel succeeding
// incorrectly.
func (t *Timer) Stop() bool {
	return stopTimer(&t.r)
}

// NewTimer creates a new Timer that will send
// the current time on its channel after at least duration d.
func NewTimer(d Duration) *Timer {
	c := make(chan Time, 1)
	t := &Timer{
		C: c,
		r: runtimeTimer{
			when: when(d),
			f:    sendTime,
			arg:  c,
		},
	}
	startTimer(&t.r)
	return t
}

// Reset changes the timer to expire after duration d.
// It returns true if the timer had been active, false if the timer had
// expired or been stopped.
func (t *Timer) Reset(d Duration) bool {
	w := when(d)
	active := stopTimer(&t.r)
	t.r.when = w
	startTimer(&t.r)
	return active
}

func sendTime(now int64, c interface{}) {
	// Non-blocking send of time on c.
	// Used in NewTimer, it cannot block anyway (buffer).
	// Used in NewTicker, dropping sends on the floor is
	// the desired behavior when the reader gets behind,
	// because the sends are periodic.
	select {
	case c.(chan Time) <- Unix(0, now):
	default:
	}
}

// After waits for the duration to elapse and then sends the current time
// on the returned channel.
// It is equivalent to NewTimer(d).C.
func After(d Duration) <-chan Time {
	return NewTimer(d).C
}

// AfterFunc waits for the duration to elapse and then calls f
// in its own goroutine. It returns a Timer that can
// be used to cancel the call using its Stop method.
func AfterFunc(d Duration, f func()) *Timer {
	t := &Timer{
		r: runtimeTimer{
			when: when(d),
			f:    goFunc,
			arg:  f,
		},
	}
	startTimer(&t.r)
	return t
}

func goFunc(now int64, arg interface{}) {
	go arg.(func())()
}
<commit_msg>time: add more docs on Sleep<commit_after>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package time

// Sleep pauses the current goroutine for at least the duration d.
// A negative or zero duration causes Sleep to return immediately.
func Sleep(d Duration)

func nano() int64 {
	sec, nsec := now()
	return sec*1e9 + int64(nsec)
}

// Interface to timers implemented in package runtime.
// Must be in sync with ../runtime/runtime.h:/^struct.Timer$
type runtimeTimer struct {
	i      int32
	when   int64
	period int64
	f      func(int64, interface{}) // NOTE: must not be closure
	arg    interface{}
}

// when is a helper function for setting the 'when' field of a runtimeTimer.
// It returns what the time will be, in nanoseconds, Duration d in the future.
// If d is negative, it is ignored.  If the returned value would be less than
// zero because of an overflow, MaxInt64 is returned.
func when(d Duration) int64 {
	if d <= 0 {
		return nano()
	}
	t := nano() + int64(d)
	if t < 0 {
		t = 1<<63 - 1 // math.MaxInt64
	}
	return t
}

func startTimer(*runtimeTimer)
func stopTimer(*runtimeTimer) bool

// The Timer type represents a single event.
// When the Timer expires, the current time will be sent on C,
// unless the Timer was created by AfterFunc.
type Timer struct {
	C <-chan Time
	r runtimeTimer
}

// Stop prevents the Timer from firing.
// It returns true if the call stops the timer, false if the timer has already
// expired or been stopped.
// Stop does not close the channel, to prevent a read from the channel succeeding
// incorrectly.
func (t *Timer) Stop() bool {
	return stopTimer(&t.r)
}

// NewTimer creates a new Timer that will send
// the current time on its channel after at least duration d.
func NewTimer(d Duration) *Timer {
	c := make(chan Time, 1)
	t := &Timer{
		C: c,
		r: runtimeTimer{
			when: when(d),
			f:    sendTime,
			arg:  c,
		},
	}
	startTimer(&t.r)
	return t
}

// Reset changes the timer to expire after duration d.
// It returns true if the timer had been active, false if the timer had
// expired or been stopped.
func (t *Timer) Reset(d Duration) bool {
	w := when(d)
	active := stopTimer(&t.r)
	t.r.when = w
	startTimer(&t.r)
	return active
}

func sendTime(now int64, c interface{}) {
	// Non-blocking send of time on c.
	// Used in NewTimer, it cannot block anyway (buffer).
	// Used in NewTicker, dropping sends on the floor is
	// the desired behavior when the reader gets behind,
	// because the sends are periodic.
	select {
	case c.(chan Time) <- Unix(0, now):
	default:
	}
}

// After waits for the duration to elapse and then sends the current time
// on the returned channel.
// It is equivalent to NewTimer(d).C.
func After(d Duration) <-chan Time {
	return NewTimer(d).C
}

// AfterFunc waits for the duration to elapse and then calls f
// in its own goroutine. It returns a Timer that can
// be used to cancel the call using its Stop method.
func AfterFunc(d Duration, f func()) *Timer {
	t := &Timer{
		r: runtimeTimer{
			when: when(d),
			f:    goFunc,
			arg:  f,
		},
	}
	startTimer(&t.r)
	return t
}

func goFunc(now int64, arg interface{}) {
	go arg.(func())()
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"log"
	"net/url"
)

import (
	"github.com/qiniu/api/conf"
	"github.com/qiniu/api/rs"
)

type pfopRet struct {
	PersistentId string `json:"persistentId"`
	Code         int    `json:"code"`
	Error        string `json:"error"`
}

func main() {
	bucket := flag.String("b", "", "bucket")
	key := flag.String("k", "", "file key")
	accessKey := flag.String("ak", "", "access key")
	secretKey := flag.String("sk", "", "secret key")
	pfop := flag.String("pfop", "", "pfop option")
	notify := flag.String("n", "", "notify url")
	flag.Parse()
	if *bucket == "" || *key == "" || *accessKey == "" || *accessKey == "" || *pfop == "" {
		flag.PrintDefaults()
		log.Fatalln("invalid args")
		return
	}
	conf.ACCESS_KEY = *accessKey
	conf.SECRET_KEY = *secretKey

	client := rs.New(nil)
	param := url.Values{}
	param.Set("bucket", *bucket)
	param.Set("key", url.QueryEscape(*key))
	param.Set("fops", url.QueryEscape(*pfop))
	param.Set("notifyURL", url.QueryEscape(*notify))
	var ret pfopRet
	err := client.Conn.CallWithForm(nil, &ret, "http://api.qiniu.com/pfop/", param)
	if err != nil {
		log.Fatalln("error", err, ret)
	}
	log.Println("id", ret.PersistentId)
}
<commit_msg>fixed dup url encode<commit_after>package main

import (
	"flag"
	"log"
	"net/url"
)

import (
	"github.com/qiniu/api/conf"
	"github.com/qiniu/api/rs"
)

type pfopRet struct {
	PersistentId string `json:"persistentId"`
	Code         int    `json:"code"`
	Error        string `json:"error"`
}

func main() {
	bucket := flag.String("b", "", "bucket")
	key := flag.String("k", "", "file key")
	accessKey := flag.String("ak", "", "access key")
	secretKey := flag.String("sk", "", "secret key")
	pfop := flag.String("pfop", "", "pfop option")
	notify := flag.String("n", "", "notify url")
	flag.Parse()
	if *bucket == "" || *key == "" || *accessKey == "" || *accessKey == "" || *pfop == "" {
		flag.PrintDefaults()
		log.Fatalln("invalid args")
		return
	}
	conf.ACCESS_KEY = *accessKey
	conf.SECRET_KEY = *secretKey

	client := rs.New(nil)
	param := url.Values{}
	param.Set("bucket", *bucket)
	param.Set("key", *key)
	param.Set("fops", *pfop)
	param.Set("notifyURL", *notify)
	var ret pfopRet
	err := client.Conn.CallWithForm(nil, &ret, "http://api.qiniu.com/pfop/", param)
	if err != nil {
		log.Fatalln("error", err, ret)
	}
	log.Println("id", ret.PersistentId)
}
<|endoftext|>


<commit_before>package sandbox

import (
	"path/filepath"
	"os"
	"io/ioutil"
)

const (
	cgCPUPathPrefix    = "/sys/fs/cgroup/cpu/"
	cgPidPathPrefix    = "/sys/fs/cgroup/pids/"
	cgMemoryPathPrefix = "/sys/fs/cgroup/memory/"
)

func InitCGroup(pid, containerID, memory string) error {
	os.Stderr.WriteString("InitCGroup starting...\n")

	dirs := []string{
		filepath.Join(cgCPUPathPrefix, containerID),
		filepath.Join(cgPidPathPrefix, containerID),
		filepath.Join(cgMemoryPathPrefix, containerID),
	}

	for _, dir := range dirs {
		os.Stderr.WriteString("os.MkdirAll " + dir + "\n")
		if err := os.MkdirAll(dir, os.ModePerm); err != nil {
			os.Stderr.WriteString(err.Error() + "\n")
			os.Stderr.WriteString("os.MkdirAll " + dir + " failed \n")
			return err
		}
	}

	if err := cpuCGroup(pid, containerID); err != nil {
		os.Stderr.WriteString(err.Error() + "\n")
		os.Stderr.WriteString("cpuCGroup failed...\n")
		return err
	}

	if err := pidCGroup(pid, containerID); err != nil {
		os.Stderr.WriteString(err.Error() + "\n")
		os.Stderr.WriteString("pidCGroup failed...\n")
		return err
	}

	if err := memoryCGroup(pid, containerID, memory); err != nil {
		os.Stderr.WriteString(err.Error() + "\n")
		os.Stderr.WriteString("memoryCGroup failed...\n")
		return err
	}

	os.Stderr.WriteString("InitCGroup done...\n")
	return nil
}

// https://www.kernel.org/doc/Documentation/scheduler/sched-bwc.txt
func cpuCGroup(pid, containerID string) error {
	cgCPUPath := filepath.Join(cgCPUPathPrefix, containerID)

	keys := []string{"tasks", "cpu.cfs_quota_us"}
	values := []string{pid, "10000"}
	for k, v := range keys {
		path := filepath.Join(cgCPUPath, v)
		os.Stderr.WriteString("writing [" + values[k] + "] to file: " + path + "\n")
		if err := ioutil.WriteFile(path, []byte(values[k]), 0644); err != nil {
			os.Stderr.WriteString("write failed \n")
			return err
		}
		c, _ := ioutil.ReadFile(path)
		os.Stderr.WriteString("content of " + path + ": " + string(c))
	}

	os.Stderr.WriteString("cpuCGroup done\n")
	return nil
}

// https://www.kernel.org/doc/Documentation/cgroup-v1/pids.txt
func pidCGroup(pid, containerID string) error {
	cgPidPath := filepath.Join(cgPidPathPrefix, containerID)

	keys := []string{"cgroup.procs", "pids.max"}
	values := []string{pid, "64"}
	for k, v := range keys {
		path := filepath.Join(cgPidPath, v)
		os.Stderr.WriteString("writing [" + values[k] + "] to file: " + path + "\n")
		if err := ioutil.WriteFile(path, []byte(values[k]), 0644); err != nil {
			os.Stderr.WriteString("write failed \n")
			return err
		}
		c, _ := ioutil.ReadFile(path)
		os.Stderr.WriteString("content of " + path + ": " + string(c))
	}

	os.Stderr.WriteString("pidCGroup done \n")
	return nil
}

// https://www.kernel.org/doc/Documentation/cgroup-v1/memory.txt
func memoryCGroup(pid, containerID, memory string) error {
	cgMemoryPath := filepath.Join(cgMemoryPathPrefix, containerID)

	keys := []string{"memory.kmem.limit_in_bytes", "tasks", "memory.limit_in_bytes", "memory.memsw.limit_in_bytes"}
	values := []string{"256m", pid, memory + "m", memory + "m"}
	for k, v := range keys {
		path := filepath.Join(cgMemoryPath, v)
		os.Stderr.WriteString("writing [" + values[k] + "] to file: " + path + "\n")
		if err := ioutil.WriteFile(path, []byte(values[k]), 0644); err != nil {
			os.Stderr.WriteString("write failed \n")
			return err
		}
		c, _ := ioutil.ReadFile(path)
		os.Stderr.WriteString("content of " + path + ": " + string(c))
	}

	os.Stderr.WriteString("memoryCGroup done \n")
	return nil
}
<commit_msg>remove "memory.memsw.limit_in_bytes" cgroup for ubuntu server<commit_after>package sandbox

import (
	"path/filepath"
	"os"
	"io/ioutil"
)

const (
	cgCPUPathPrefix    = "/sys/fs/cgroup/cpu/"
	cgPidPathPrefix    = "/sys/fs/cgroup/pids/"
	cgMemoryPathPrefix = "/sys/fs/cgroup/memory/"
)

func InitCGroup(pid, containerID, memory string) error {
	os.Stderr.WriteString("InitCGroup starting...\n")

	dirs := []string{
		filepath.Join(cgCPUPathPrefix, containerID),
		filepath.Join(cgPidPathPrefix, containerID),
		filepath.Join(cgMemoryPathPrefix, containerID),
	}

	for _, dir := range dirs {
		os.Stderr.WriteString("os.MkdirAll " + dir + "\n")
		if err := os.MkdirAll(dir, os.ModePerm); err != nil {
			os.Stderr.WriteString(err.Error() + "\n")
			os.Stderr.WriteString("os.MkdirAll " + dir + " failed \n")
			return err
		}
	}

	if err := cpuCGroup(pid, containerID); err != nil {
		os.Stderr.WriteString(err.Error() + "\n")
		os.Stderr.WriteString("cpuCGroup failed...\n")
		return err
	}

	if err := pidCGroup(pid, containerID); err != nil {
		os.Stderr.WriteString(err.Error() + "\n")
		os.Stderr.WriteString("pidCGroup failed...\n")
		return err
	}

	if err := memoryCGroup(pid, containerID, memory); err != nil {
		os.Stderr.WriteString(err.Error() + "\n")
		os.Stderr.WriteString("memoryCGroup failed...\n")
		return err
	}

	os.Stderr.WriteString("InitCGroup done...\n")
	return nil
}

// https://www.kernel.org/doc/Documentation/scheduler/sched-bwc.txt
func cpuCGroup(pid, containerID string) error {
	cgCPUPath := filepath.Join(cgCPUPathPrefix, containerID)

	keys := []string{"tasks", "cpu.cfs_quota_us"}
	values := []string{pid, "10000"}
	for k, v := range keys {
		path := filepath.Join(cgCPUPath, v)
		os.Stderr.WriteString("writing [" + values[k] + "] to file: " + path + "\n")
		if err := ioutil.WriteFile(path, []byte(values[k]), 0644); err != nil {
			os.Stderr.WriteString("write failed \n")
			return err
		}
		c, _ := ioutil.ReadFile(path)
		os.Stderr.WriteString("content of " + path + ": " + string(c))
	}

	os.Stderr.WriteString("cpuCGroup done\n")
	return nil
}

// https://www.kernel.org/doc/Documentation/cgroup-v1/pids.txt
func pidCGroup(pid, containerID string) error {
	cgPidPath := filepath.Join(cgPidPathPrefix, containerID)

	keys := []string{"cgroup.procs", "pids.max"}
	values := []string{pid, "64"}
	for k, v := range keys {
		path := filepath.Join(cgPidPath, v)
		os.Stderr.WriteString("writing [" + values[k] + "] to file: " + path + "\n")
		if err := ioutil.WriteFile(path, []byte(values[k]), 0644); err != nil {
			os.Stderr.WriteString("write failed \n")
			return err
		}
		c, _ := ioutil.ReadFile(path)
		os.Stderr.WriteString("content of " + path + ": " + string(c))
	}

	os.Stderr.WriteString("pidCGroup done \n")
	return nil
}

// https://www.kernel.org/doc/Documentation/cgroup-v1/memory.txt
func memoryCGroup(pid, containerID, memory string) error {
	cgMemoryPath := filepath.Join(cgMemoryPathPrefix, containerID)

	keys := []string{"memory.kmem.limit_in_bytes", "tasks", "memory.limit_in_bytes"}
	values := []string{"64m", pid, memory + "m"}
	for k, v := range keys {
		path := filepath.Join(cgMemoryPath, v)
		os.Stderr.WriteString("writing [" + values[k] + "] to file: " + path + "\n")
		if err := ioutil.WriteFile(path, []byte(values[k]), 0644); err != nil {
			os.Stderr.WriteString("write failed \n")
			return err
		}
		c, _ := ioutil.ReadFile(path)
		os.Stderr.WriteString("content of " + path + ": " + string(c))
	}

	os.Stderr.WriteString("memoryCGroup done \n")
	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"bytes"
	"io"
	"log"
	"os/exec"
	"strings"
)

type Runner struct {
	printer     PrintResetter
	current     *exec.Cmd
	template    string
	placeholder string
	buf         *bytes.Buffer
}

func (r *Runner) runWithInput(input []byte) {
	cmd := r.cmdWithInput(string(input))

	stdout, err := cmd.StdoutPipe()
	if err != nil {
		log.Fatal(err)
	}
	ch := r.readCmdStdout(stdout)

	err = cmd.Start()
	if err != nil {
		log.Fatal(err)
	}

	r.buf = new(bytes.Buffer)
	r.current = cmd

	for str := range ch {
		r.printer.Print(str)
		r.buf.WriteString(str)
	}
	cmd.Wait()
	r.printer.Reset()
}

func (r *Runner) cmdWithInput(input string) *exec.Cmd {
	line := strings.Replace(r.template, r.placeholder, input, -1)
	splitted := strings.Split(line, " ")

	return exec.Command(splitted[0], splitted[1:len(splitted)]...)
}

func (r *Runner) readCmdStdout(stdout io.ReadCloser) <-chan string {
	ch := make(chan string)
	cmdreader := bufio.NewReader(stdout)

	go func() {
		for {
			line, err := cmdreader.ReadBytes('\n')
			if err != nil || err == io.EOF {
				break
			}
			ch <- string(line)
		}
		close(ch)
	}()

	return ch
}

func (r *Runner) writeCmdStdout(out io.Writer) (n int64, err error) {
	return io.Copy(out, r.buf)
}

func (r *Runner) killCurrent() {
	if r.current != nil {
		r.current.Process.Kill()
		r.current.Wait()

		r.current = nil
	}

	r.printer.Reset()
}
<commit_msg>Handle errors in Runner<commit_after>package main

import (
	"bufio"
	"bytes"
	"io"
	"log"
	"os/exec"
	"strings"
)

type Runner struct {
	printer     PrintResetter
	current     *exec.Cmd
	template    string
	placeholder string
	buf         *bytes.Buffer
}

func (r *Runner) runWithInput(input []byte) {
	cmd := r.cmdWithInput(string(input))

	stdout, err := cmd.StdoutPipe()
	if err != nil {
		log.Fatal(err)
	}

	stderr, err := cmd.StderrPipe()
	if err != nil {
		log.Fatal(err)
	}

	outch := r.streamOutput(stdout)
	errch := r.streamOutput(stderr)

	err = cmd.Start()
	if err != nil {
		log.Fatal(err)
	}

	r.buf = new(bytes.Buffer)
	r.current = cmd

	for str := range outch {
		r.printer.Print(str)
		r.buf.WriteString(str)
	}

	err = cmd.Wait()
	if err != nil {
		for str := range errch {
			r.printer.Print(str)
		}
		r.printer.Print(err.Error())
		return
	}

	r.printer.Reset()
}

func (r *Runner) cmdWithInput(input string) *exec.Cmd {
	line := strings.Replace(r.template, r.placeholder, input, -1)
	splitted := strings.Split(line, " ")

	return exec.Command(splitted[0], splitted[1:len(splitted)]...)
}

func (r *Runner) streamOutput(stdout io.ReadCloser) <-chan string {
	ch := make(chan string)
	cmdreader := bufio.NewReader(stdout)

	go func() {
		for {
			line, err := cmdreader.ReadBytes('\n')
			if err != nil || err == io.EOF {
				break
			}
			ch <- string(line)
		}
		close(ch)
	}()

	return ch
}

func (r *Runner) writeCmdStdout(out io.Writer) (n int64, err error) {
	return io.Copy(out, r.buf)
}

func (r *Runner) killCurrent() {
	if r.current != nil {
		r.current.Process.Kill()
		r.current.Wait()

		r.current = nil
	}

	r.printer.Reset()
}
<|endoftext|>


<commit_before>/* For license and copyright information please see LEGAL file in repository */

package log

import (
	"fmt"
	"os"
	"path"
	"strconv"
	"time"

	etime "../earth-time"
)

const (
	// DevMode use to show more log when enabled and disabled||enabled some rules!
	DevMode = false
	// DebugMode use to show more log when enabled!
	DebugMode = false
)

const (
	timeFormat = "2006-01-02 15:04:05"
)

// TODO::: fix problem with multi CPU core parallelism (data race condition)

// hold logs until app running.
var logFile *os.File

// Init will initialize log to do some interval saving
func Init(name, repoLocation string, interval int64) (err error) {
	var logFolder = path.Join(repoLocation, "log")
	os.Mkdir(logFolder, 0700)
	logFile, err = os.OpenFile(path.Join(logFolder, name+strconv.FormatInt(etime.RoundSeconds(etime.Now(), interval), 10)+".log"), os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0700)
	if err != nil {
		return
	}
	go intervalSaving(name, repoLocation, interval)
	return
}

// Debug show log in standard console & append log to buffer to save them later.
func Debug(a ...interface{}) {
	var log = fmt.Sprintln("[Debug]", time.Now().Format(timeFormat), a)
	if DevMode {
		os.Stderr.WriteString(log)
	}
	logFile.WriteString(log)
}

// Info show log in standard console & append log to buffer to save them later.
func Info(a ...interface{}) {
	var log = fmt.Sprintln("[Info]", time.Now().Format(timeFormat), a)
	if DevMode {
		os.Stderr.WriteString(log)
	}
	logFile.WriteString(log)
}

// Warn show log in standard console & append log to buffer to save them later.
func Warn(a ...interface{}) {
	var log = fmt.Sprintln("[Warn]", time.Now().Format(timeFormat), a)
	if DevMode {
		os.Stderr.WriteString(log)
	}
	logFile.WriteString(log)
}

// Fatal show log in standard console & append log to buffer to save them later and exit app.
func Fatal(a ...interface{}) {
	var log = fmt.Sprintln("[Fatal]", time.Now().Format(timeFormat), a)
	if DevMode {
		os.Stderr.WriteString(log)
	}
	logFile.WriteString(log)
	panic("Due to important log, panic situation occur")
}

// SaveToStorage use to make||flush file!
func SaveToStorage() {
	logFile.Close()
}

func intervalSaving(name, location string, interval int64) {
	var timer = time.NewTimer(time.Duration(etime.UntilRoundSeconds(etime.Now(), interval)) * time.Second)
	for {
		select {
		// case shutdownFeedback := <-pcs.shutdownSignal:
		// 	timer.Stop()
		// 	shutdownFeedback <- struct{}{}
		// 	return
		case <-timer.C:
			logFile.Close()

			logFile, _ = os.OpenFile(path.Join(location, name+strconv.FormatInt(etime.RoundSeconds(etime.Now(), interval), 10)+".log"), os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0700)
			timer.Reset(time.Duration(interval) * time.Second)
		}
	}
}
<commit_msg>[Package] Log >> - Add two new compile time const to change how logging - Add language const to change language of logging - some minor fixes due to other package APIs changed!<commit_after>/* For license and copyright information please see LEGAL file in repository */

package log

import (
	"fmt"
	"os"
	"path"
	"strconv"
	"time"

	etime "../earth-time"
	lang "../language"
)

const (
	// ScreenMode use to show the logs on screen when enabled!
	ScreenMode = false
	// DevMode use to show more log when enabled and disabled||enabled some rules!
	DevMode = false
	// DebugMode use to show more log when enabled!
	DebugMode = false
	// DeepDebugMode use to show most details log when enabled like RAW req&&res in any protocol like HTTP, sRPC, ...!
	DeepDebugMode = false
	// Language indicate which language use to print error logs!
	Language = lang.LanguageEnglish
)

const (
	timeFormat = "2006-01-02 15:04:05"
)

// TODO::: fix problem with multi CPU core parallelism (data race condition)

// hold logs until app running.
var logFile *os.File

// Init will initialize log to do some interval saving
func Init(name, repoLocation string, interval etime.Duration) (err error) {
	var logFolder = path.Join(repoLocation, "log")
	os.Mkdir(logFolder, 0700)
	logFile, err = os.OpenFile(path.Join(logFolder, name+strconv.FormatInt(etime.Now().RoundTo(interval), 10)+".log"), os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0700)
	if err != nil {
		return
	}
	go intervalSaving(name, logFolder, interval)
	return
}

// Debug show log in standard console if requested by ScreenMode const & append log to buffer to save them later.
func Debug(a ...interface{}) {
	var log = fmt.Sprintln("[Debug]", time.Now().Format(timeFormat), a)
	if ScreenMode {
		os.Stderr.WriteString(log)
	}
	logFile.WriteString(log)
}

// DeepDebug show deep debug log in standard console if requested by ScreenMode const & append log to buffer to save them later.
func DeepDebug(a ...interface{}) {
	var log = fmt.Sprintln("[Deep]", time.Now().Format(timeFormat), a)
	if ScreenMode {
		os.Stderr.WriteString(log)
	}
	logFile.WriteString(log)
}

// Info show log in standard console if requested by ScreenMode const & append log to buffer to save them later.
func Info(a ...interface{}) {
	var log = fmt.Sprintln("[Info]", time.Now().Format(timeFormat), a)
	if ScreenMode {
		os.Stderr.WriteString(log)
	}
	logFile.WriteString(log)
}

// Warn show log in standard console if requested by ScreenMode const & append log to buffer to save them later.
func Warn(a ...interface{}) {
	var log = fmt.Sprintln("[Warn]", time.Now().Format(timeFormat), a)
	if ScreenMode {
		os.Stderr.WriteString(log)
	}
	logFile.WriteString(log)
}

// Fatal show log in standard console if requested by ScreenMode const & append log to buffer to save them later and exit app.
func Fatal(a ...interface{}) {
	var log = fmt.Sprintln("[Fatal]", time.Now().Format(timeFormat), a)
	if ScreenMode {
		os.Stderr.WriteString(log)
	}
	logFile.WriteString(log)
	panic("Due to important log, panic situation occur")
}

// SaveToStorage use to make||flush file!
func SaveToStorage() {
	logFile.Close()
}

func intervalSaving(name, location string, interval etime.Duration) {
	var timer = time.NewTimer(time.Duration(etime.Now().UntilRoundTo(interval)) * time.Second)
	for {
		select {
		// case shutdownFeedback := <-pcs.shutdownSignal:
		// 	timer.Stop()
		// 	shutdownFeedback <- struct{}{}
		// 	return
		case <-timer.C:
			logFile.Close()

			logFile, _ = os.OpenFile(path.Join(location, name+strconv.FormatInt(etime.Now().RoundTo(interval), 10)+".log"), os.O_WRONLY|os.O_CREATE|os.O_APPEND, 0700)
			timer.Reset(time.Duration(interval) * time.Second)
		}
	}
}
<|endoftext|>


<commit_before>package log

import "fmt"

func I(msg string) {
	fmt.Printf("INFO: %s\n", msg)
}

func E(msg string) {
	fmt.Printf("ERROR: %s\n", msg)
}
<commit_msg>add stuff<commit_after>package log

import "fmt"

// print to log with level INFO
func I(msg string) {
	fmt.Printf("INFO: %s\n", msg)
}

//print to log with level ERROR
func E(msg string) {
	fmt.Printf("ERROR: %s\n", msg)
}
<|endoftext|>


<commit_before>package collectors

import "errors"
import "fmt"
import "os/exec"
import "strconv"
import "strings"
import "syscall"
import "github.com/c9s/goprocinfo/linux"
import "github.com/josegonzalez/metricsd/mappings"
import "github.com/josegonzalez/metricsd/structs"
import "github.com/Sirupsen/logrus"
import "github.com/vaughan0/go-ini"

type DiskspaceCollector struct{
	enabled bool
}

var filesystems = map[string]bool{
	"ext2":      true,
	"ext3":      true,
	"ext4":      true,
	"xfs":       true,
	"glusterfs": true,
	"rootfs":    true,
	"nfs":       true,
	"ntfs":      true,
	"hfs":       true,
	"fat32":     true,
	"fat16":     true,
	"btrfs":     true,
}

var excludeFilters = []string{}

func (c *DiskspaceCollector) Enabled() (bool) {
	return c.enabled
}

func (c *DiskspaceCollector) State(state bool) {
	c.enabled = state
}

func (c *DiskspaceCollector) Setup(conf ini.File) {
	c.State(true)

	fs, ok := conf.Get("DiskspaceCollector", "filesystems")
	if ok {
		for filesystem, _ := range filesystems {
			filesystems[filesystem] = false
		}

		enabledFilesystems := strings.Split(fs, ",")
		for _, enabledFilesystem := range enabledFilesystems {
			filesystems[strings.TrimSpace(enabledFilesystem)] = true
		}
	}

	ef, ok := conf.Get("DiskspaceCollector", "exclude_filters")
	if ok {
		excludeFilters := strings.Split(ef, ",")
		for _, excludeFilter := range excludeFilters {
			excludeFilters = append(excludeFilters, strings.TrimSpace(excludeFilter))
		}
	}
}

func (c *DiskspaceCollector) Collect() (map[string]mappings.MetricMap, error) {
	stat, err := linux.ReadMounts("/proc/mounts")
	if err != nil {
		logrus.Fatal("stat read fail")
		return nil, err
	}

	var statfs_t syscall.Statfs_t
	diskspaceMapping := map[string]mappings.MetricMap{}

	for _, mount := range stat.Mounts {
		if !filesystems[mount.FSType] {
			continue
		}
		syscall.Statfs(mount.MountPoint, &statfs_t)
		byte_avail := statfs_t.Bavail * uint64(statfs_t.Bsize)
		byte_free := statfs_t.Bfree * uint64(statfs_t.Bsize)

		diskspaceMapping[mount.MountPoint] = mappings.MetricMap{
			"byte_avail":     byte_avail,
			"byte_free":      byte_free,
			"byte_used":      byte_avail - byte_free,
			"gigabyte_avail": byte_avail / 1073741824,
			"gigabyte_free":  byte_free / 1073741824,
			"gigabyte_used":  (byte_avail - byte_free) / 1073741824,
		}
	}

	dfMapping, e := collectDf()
	if e != nil {
		return diskspaceMapping, e
	}

	for mountpoint, metricMap := range dfMapping {
		if _, ok := diskspaceMapping[mountpoint]; !ok {
			continue
		}

		for key, value := range metricMap {
			diskspaceMapping[mountpoint][key] = value
		}
	}

	return diskspaceMapping, nil
}

func (c *DiskspaceCollector) Report() (structs.MetricSlice, error) {
	var report structs.MetricSlice
	data, _ := c.Collect()

	if data != nil {
		units := map[string]string{
			"gigabyte": "GB",
			"byte":     "B",
			"inodes":   "Ino",
		}

		for mountpoint, values := range data {
			// TODO: Add exclude_filters support
			mountpoint = parseMountpoint(mountpoint)
			for k, v := range values {
				s := strings.Split(k, "_")
				unit, mtype := s[0], s[1]

				metric := structs.BuildMetric("diskspace", "gauge", mtype, v, structs.FieldsMap{
					"mountpoint": mountpoint,
					"unit":       units[unit],
					"raw_key":    k,
					"raw_value":  v,
				})
				metric.Path = fmt.Sprintf("diskspace.%s", mountpoint)
				report = append(report, metric)
			}
		}
	}

	return report, nil
}

func parseMountpoint(device string) string {
	mountpoint := strings.Replace(device, "/", "_", -1)
	mountpoint = strings.Replace(mountpoint, ".", "_", -1)
	if mountpoint == "_" {
		mountpoint = "root"
	}

	return mountpoint
}

func collectDf() (map[string]mappings.MetricMap, error) {
	data := map[string]mappings.MetricMap{}
	lines, e := readDf("-i")
	if e != nil {
		return data, e
	}

	for _, line := range lines[1:] {
		if !strings.HasPrefix(line, "/") {
			continue
		}
		chunks := strings.Fields(line)
		if len(chunks) >= 6 {
			mountpoint := chunks[5]
			if _, ok := data[mountpoint]; !ok {
				data[mountpoint] = mappings.MetricMap{}
			}

			if v, e := strconv.ParseInt(chunks[1], 10, 64); e == nil {
				data[mountpoint]["inodes_total"] = v
			}
			if v, e := strconv.ParseInt(chunks[2], 10, 64); e == nil {
				data[mountpoint]["inodes_used"] = v
			}
			if v, e := strconv.ParseInt(chunks[3], 10, 64); e == nil {
				data[mountpoint]["inodes_avail"] = v
			}
			if v, e := strconv.ParseInt(strings.Replace(chunks[4], "%", "", 1), 10, 64); e == nil {
				data[mountpoint]["inodes_use"] = v
			}
		}
	}
	return data, nil
}

func readDf(flag string) ([]string, error) {
	lines := []string{}
	raw, e := exec.Command("df", flag).Output()
	if e != nil {
		return lines, e
	}
	if len(raw) == 0 {
		return lines, errors.New("Reading df returned an empty string")
	}

	lines = strings.Split(strings.TrimSpace(string(raw)), "\n")
	return lines, nil
}
<commit_msg>Move excludeFilters to a property<commit_after>package collectors

import "errors"
import "fmt"
import "os/exec"
import "strconv"
import "strings"
import "syscall"
import "github.com/c9s/goprocinfo/linux"
import "github.com/josegonzalez/metricsd/mappings"
import "github.com/josegonzalez/metricsd/structs"
import "github.com/Sirupsen/logrus"
import "github.com/vaughan0/go-ini"

type DiskspaceCollector struct{
	enabled bool
	excludeFilters []string{}
}

var filesystems = map[string]bool{
	"ext2":      true,
	"ext3":      true,
	"ext4":      true,
	"xfs":       true,
	"glusterfs": true,
	"rootfs":    true,
	"nfs":       true,
	"ntfs":      true,
	"hfs":       true,
	"fat32":     true,
	"fat16":     true,
	"btrfs":     true,
}

func (c *DiskspaceCollector) Enabled() (bool) {
	return c.enabled
}

func (c *DiskspaceCollector) State(state bool) {
	c.enabled = state
}

func (c *DiskspaceCollector) Setup(conf ini.File) {
	c.State(true)

	fs, ok := conf.Get("DiskspaceCollector", "filesystems")
	if ok {
		for filesystem, _ := range filesystems {
			filesystems[filesystem] = false
		}

		enabledFilesystems := strings.Split(fs, ",")
		for _, enabledFilesystem := range enabledFilesystems {
			filesystems[strings.TrimSpace(enabledFilesystem)] = true
		}
	}

	ef, ok := conf.Get("DiskspaceCollector", "exclude_filters")
	if ok {
		excludeFilters := strings.Split(ef, ",")
		for _, excludeFilter := range excludeFilters {
			c.excludeFilters = append(c.excludeFilters, strings.TrimSpace(excludeFilter))
		}
	}
}

func (c *DiskspaceCollector) Collect() (map[string]mappings.MetricMap, error) {
	stat, err := linux.ReadMounts("/proc/mounts")
	if err != nil {
		logrus.Fatal("stat read fail")
		return nil, err
	}

	var statfs_t syscall.Statfs_t
	diskspaceMapping := map[string]mappings.MetricMap{}

	for _, mount := range stat.Mounts {
		if !filesystems[mount.FSType] {
			continue
		}
		syscall.Statfs(mount.MountPoint, &statfs_t)
		byte_avail := statfs_t.Bavail * uint64(statfs_t.Bsize)
		byte_free := statfs_t.Bfree * uint64(statfs_t.Bsize)

		diskspaceMapping[mount.MountPoint] = mappings.MetricMap{
			"byte_avail":     byte_avail,
			"byte_free":      byte_free,
			"byte_used":      byte_avail - byte_free,
			"gigabyte_avail": byte_avail / 1073741824,
			"gigabyte_free":  byte_free / 1073741824,
			"gigabyte_used":  (byte_avail - byte_free) / 1073741824,
		}
	}

	dfMapping, e := collectDf()
	if e != nil {
		return diskspaceMapping, e
	}

	for mountpoint, metricMap := range dfMapping {
		if _, ok := diskspaceMapping[mountpoint]; !ok {
			continue
		}

		for key, value := range metricMap {
			diskspaceMapping[mountpoint][key] = value
		}
	}

	return diskspaceMapping, nil
}

func (c *DiskspaceCollector) Report() (structs.MetricSlice, error) {
	var report structs.MetricSlice
	data, _ := c.Collect()

	if data != nil {
		units := map[string]string{
			"gigabyte": "GB",
			"byte":     "B",
			"inodes":   "Ino",
		}

		for mountpoint, values := range data {
			// TODO: Add exclude_filters support
			mountpoint = parseMountpoint(mountpoint)
			for k, v := range values {
				s := strings.Split(k, "_")
				unit, mtype := s[0], s[1]

				metric := structs.BuildMetric("diskspace", "gauge", mtype, v, structs.FieldsMap{
					"mountpoint": mountpoint,
					"unit":       units[unit],
					"raw_key":    k,
					"raw_value":  v,
				})
				metric.Path = fmt.Sprintf("diskspace.%s", mountpoint)
				report = append(report, metric)
			}
		}
	}

	return report, nil
}

func parseMountpoint(device string) string {
	mountpoint := strings.Replace(device, "/", "_", -1)
	mountpoint = strings.Replace(mountpoint, ".", "_", -1)
	if mountpoint == "_" {
		mountpoint = "root"
	}

	return mountpoint
}

func collectDf() (map[string]mappings.MetricMap, error) {
	data := map[string]mappings.MetricMap{}
	lines, e := readDf("-i")
	if e != nil {
		return data, e
	}

	for _, line := range lines[1:] {
		if !strings.HasPrefix(line, "/") {
			continue
		}
		chunks := strings.Fields(line)
		if len(chunks) >= 6 {
			mountpoint := chunks[5]
			if _, ok := data[mountpoint]; !ok {
				data[mountpoint] = mappings.MetricMap{}
			}

			if v, e := strconv.ParseInt(chunks[1], 10, 64); e == nil {
				data[mountpoint]["inodes_total"] = v
			}
			if v, e := strconv.ParseInt(chunks[2], 10, 64); e == nil {
				data[mountpoint]["inodes_used"] = v
			}
			if v, e := strconv.ParseInt(chunks[3], 10, 64); e == nil {
				data[mountpoint]["inodes_avail"] = v
			}
			if v, e := strconv.ParseInt(strings.Replace(chunks[4], "%", "", 1), 10, 64); e == nil {
				data[mountpoint]["inodes_use"] = v
			}
		}
	}
	return data, nil
}

func readDf(flag string) ([]string, error) {
	lines := []string{}
	raw, e := exec.Command("df", flag).Output()
	if e != nil {
		return lines, e
	}
	if len(raw) == 0 {
		return lines, errors.New("Reading df returned an empty string")
	}

	lines = strings.Split(strings.TrimSpace(string(raw)), "\n")
	return lines, nil
}
<|endoftext|>


<commit_before>package sbloom

import "hash"

const (
	elemSize = 3
	idxShift = elemSize
	idxMask  = 1<<elemSize - 1
)

//set turns the nth bit in the slice of bytes to one.
func set(m []uint8, n uint64) {
	idx, mask := n>>idxShift, n&idxMask
	m[idx] |= 1 << mask
}

//get returns true if the nth bit in the slice of bytes is one.
func get(m []uint8, n uint64) bool {
	idx, mask := n>>idxShift, n&idxMask
	return m[idx]&(1<<mask) != 0
}

type Filter struct {
	bh hash.Hash64
	fs []*filter
}

func (f Filter) Add(p []byte) {
	last := f.fs[len(f.fs)-1]
	last.Add(p)
	if last.left == 0 {
		//this one filled, add a new one
	}
}

func (f Filter) Lookup(p []byte) bool {
	for _, subfil := range f.fs {
		if subfil.Lookup(p) {
			return true
		}
	}
	return false
}

type filter struct {
	size uint   // N == 1 << size
	mask uint64 // x % N == x & mask == x & (1 << size) - 1

	bins [][]uint8 // [k][1<<size]uint8 bins
	hs   []sHash   // [k]sHash hashes
	left uint64    // number of additions left until new filter
}

func (f *filter) Add(p []byte) {
	for i, h := range f.hs {
		val := h.Hash(p)
		set(f.bins[i], val&f.mask)
	}
	f.left--
}

func (f *filter) Lookup(p []byte) bool {
	for i, h := range f.hs {
		val := h.Hash(p)
		if !get(f.bins[i], val&f.mask) {
			return false
		}
	}
	return true
}
<commit_msg>some implementation work<commit_after>package sbloom

import (
	"hash"
	"math/rand"
)

const (
	elemSize = 3
	idxShift = elemSize
	idxMask  = 1<<elemSize - 1
)

//set turns the nth bit in the slice of bytes to one.
func set(m []uint8, n uint64) {
	idx, mask := n>>idxShift, n&idxMask
	m[idx] |= 1 << mask
}

//get returns true if the nth bit in the slice of bytes is one.
func get(m []uint8, n uint64) bool {
	idx, mask := n>>idxShift, n&idxMask
	return m[idx]&(1<<mask) != 0
}

type Filter struct {
	bh hash.Hash64
	fs []*filter
	hs []sHash
}

func (f *Filter) Add(p []byte) {
	last := f.fs[len(f.fs)-1]
	last.Add(p, f.hs[:last.k])
	if last.left == 0 {
		newSize := last.size << 1
		newMask := last.mask<<1 + 1
		newK := last.k + 1
		f.addNewFilter(newSize, newMask, newK)
	}
}

func (f *Filter) Lookup(p []byte) bool {
	for _, subfil := range f.fs {
		if subfil.Lookup(p, f.hs[:subfil.k]) {
			return true
		}
	}
	return false
}

func (f *Filter) addNewFilter(size, mask uint64, k int) {
	bins := make([][]uint8, k+1) //allocate one more bin
	binSize := size >> elemSize  //size of each bin to have size bits
	for i := range bins {
		bins[i] = make([]uint8, binSize)
	}

	//make sure we have up to k hashes
	for len(f.hs) < k {
		f.hs = append(f.hs, sHash{
			ha:   f.bh,
			seed: randSeed(),
		})
	}

	//add the new bloom filter
	f.fs = append(f.fs, &filter{
		size: size,
		mask: mask,
		bins: bins,
		k:    k,
		left: size / 5 * 7,
	})
}

func randSeed() (p []byte) {
	for i := 0; i < 4; i++ {
		p = append(p, byte(rand.Intn(256)))
	}
	return
}

type filter struct {
	size uint64 // N == 1 << size
	mask uint64 // x % N == x & mask == x & (1 << size) - 1

	k    int
	bins [][]uint8 // [k][1<<size]uint8 bins
	left uint64    // number of additions left until new filter
}

func (f *filter) Add(p []byte, hs []sHash) {
	for i, h := range hs {
		val := h.Hash(p)
		set(f.bins[i], val&f.mask)
	}
	f.left--
}

func (f *filter) Lookup(p []byte, hs []sHash) bool {
	for i, h := range hs {
		val := h.Hash(p)
		if !get(f.bins[i], val&f.mask) {
			return false
		}
	}
	return true
}
<|endoftext|>


<commit_before>// +build !windows

// Copyright 2013, Örjan Persson. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package logging

import (
	"bytes"
	"fmt"
	"io"
	"log"
)

type color int

const (
	colorBlack = iota + 30
	colorRed
	colorGreen
	colorYellow
	colorBlue
	colorMagenta
	colorCyan
	colorWhite
)

var (
	colors = []string{
		CRITICAL: colorSeq(colorMagenta),
		ERROR:    colorSeq(colorRed),
		WARNING:  colorSeq(colorYellow),
		NOTICE:   colorSeq(colorGreen),
		DEBUG:    colorSeq(colorCyan),
	}
	boldcolors = []string{
		CRITICAL: colorSeqBold(colorMagenta),
		ERROR:    colorSeqBold(colorRed),
		WARNING:  colorSeqBold(colorYellow),
		NOTICE:   colorSeqBold(colorGreen),
		DEBUG:    colorSeqBold(colorCyan),
	}
)

// LogBackend utilizes the standard log module.
type LogBackend struct {
	Logger *log.Logger
	Color  bool
}

// NewLogBackend creates a new LogBackend.
func NewLogBackend(out io.Writer, prefix string, flag int) *LogBackend {
	return &LogBackend{Logger: log.New(out, prefix, flag)}
}

// Log implements the Backend interface.
func (b *LogBackend) Log(level Level, calldepth int, rec *Record) error {
	if b.Color {
		buf := &bytes.Buffer{}
		buf.Write([]byte(colors[level]))
		buf.Write([]byte(rec.Formatted(calldepth + 1)))
		buf.Write([]byte("\033[0m"))
		// For some reason, the Go logger arbitrarily decided "2" was the correct
		// call depth...
		return b.Logger.Output(calldepth+2, buf.String())
	}

	return b.Logger.Output(calldepth+2, rec.Formatted(calldepth+1))
}

func colorSeq(color color) string {
	return fmt.Sprintf("\033[%dm", int(color))
}

func colorSeqBold(color color) string {
	return fmt.Sprintf("\033[%d;1m", int(color))
}

func doFmtVerbLevelColor(layout string, level Level, output io.Writer) {
	if layout == "bold" {
		output.Write([]byte(boldcolors[level]))
	} else if layout == "reset" {
		output.Write([]byte("\033[0m"))
	} else {
		output.Write([]byte(colors[level]))
	}
}
<commit_msg>Support customisation of default colours in unix<commit_after>// +build !windows

// Copyright 2013, Örjan Persson. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package logging

import (
	"bytes"
	"fmt"
	"io"
	"log"
)

type color int

const (
	ColorBlack = iota + 30
	ColorRed
	ColorGreen
	ColorYellow
	ColorBlue
	ColorMagenta
	ColorCyan
	ColorWhite
)

var (
	colors = []string{
		CRITICAL: ColorSeq(ColorMagenta),
		ERROR:    ColorSeq(ColorRed),
		WARNING:  ColorSeq(ColorYellow),
		NOTICE:   ColorSeq(ColorGreen),
		DEBUG:    ColorSeq(ColorCyan),
	}
	boldcolors = []string{
		CRITICAL: ColorSeqBold(ColorMagenta),
		ERROR:    ColorSeqBold(ColorRed),
		WARNING:  ColorSeqBold(ColorYellow),
		NOTICE:   ColorSeqBold(ColorGreen),
		DEBUG:    ColorSeqBold(ColorCyan),
	}
)

// LogBackend utilizes the standard log module.
type LogBackend struct {
	Logger      *log.Logger
	Color       bool
	ColorConfig []string
}

// NewLogBackend creates a new LogBackend.
func NewLogBackend(out io.Writer, prefix string, flag int) *LogBackend {
	return &LogBackend{Logger: log.New(out, prefix, flag)}
}

// Log implements the Backend interface.
func (b *LogBackend) Log(level Level, calldepth int, rec *Record) error {
	if b.Color {
		col := colors[level]
		if len(b.ColorConfig) > int(level) && b.ColorConfig[level] != "" {
			col = b.ColorConfig[level]
		}

		buf := &bytes.Buffer{}
		buf.Write([]byte(col))
		buf.Write([]byte(rec.Formatted(calldepth + 1)))
		buf.Write([]byte("\033[0m"))
		// For some reason, the Go logger arbitrarily decided "2" was the correct
		// call depth...
		return b.Logger.Output(calldepth+2, buf.String())
	}

	return b.Logger.Output(calldepth+2, rec.Formatted(calldepth+1))
}

// ConvertColors takes a list of ints representing colors for log levels and
// converts them into strings for ANSI color formatting
func ConvertColors(colors []int, bold bool) []string {
	converted := []string{}
	for _, i := range colors {
		if bold {
			converted = append(converted, ColorSeqBold(color(i)))
		} else {
			converted = append(converted, ColorSeq(color(i)))
		}
	}

	return converted
}

func ColorSeq(color color) string {
	return fmt.Sprintf("\033[%dm", int(color))
}

func ColorSeqBold(color color) string {
	return fmt.Sprintf("\033[%d;1m", int(color))
}

func doFmtVerbLevelColor(layout string, level Level, output io.Writer) {
	if layout == "bold" {
		output.Write([]byte(boldcolors[level]))
	} else if layout == "reset" {
		output.Write([]byte("\033[0m"))
	} else {
		output.Write([]byte(colors[level]))
	}
}
<|endoftext|>


<commit_before>///bin/true; exec /usr/bin/env go run "$0" "$@"

package main

import (
	"flag"
	"fmt"
	"github.com/omakoto/mlib"
	"log"
	"os/exec"
	"regexp"
	"strconv"
	"strings"
	"time"
)

var (
	// Patterns to extract PIDs from logs.
	pidPatterns = []*regexp.Regexp{
		// brief
		regexp.MustCompile(`^[A-Z]/.+?\(\s*(\d+)`),
		// process or thread
		regexp.MustCompile(`^[A-Z]\(\s*(\d+)`),
		// threadtime
		regexp.MustCompile(`^\d{2}-\d{2}\s+[\d\:\.]+\s+(\d+)`),
		// time
		regexp.MustCompile(`^\d{2}-\d{2} [\d\:\.]+ [A-Z]\/.*?\(\s*(\d+)`),
		// regexp.MustCompile(`^.*?\(\s*(\d+)`),
		// long
		regexp.MustCompile(`^\[\s*\d{2}-\d{2}\s+[\d\:\.]+\s+(\d+)`),
	}

	// Patterns to extract PIDs for process deaths logs.
	diePatterns = []*regexp.Regexp{
		regexp.MustCompile(`ActivityManager.*?Process .*?\(pid (\d+)\) has died`),
		regexp.MustCompile(`ActivityManager.*?Killing (\d+)`),
		// Any more?
	}

	whiteSpaces = regexp.MustCompile(`\s+`)

	// Flags
	width = flag.Int("w", 40, "formatting width")
	// autoflush = flag.Bool("f", false, "autoflush") // Stdout seems like always flushing

	// Output line format
	outFormat string

	// Process info cache.
	procecces = make(map[int]processInfo)

	cacheExpiration = time.Minute * 10
)

const (
	PRE_INITIALIZED                       = "<pre-initialized>"
	MAX_PRE_INITIALIZED_RETRY             = 5
	MAX_PRE_INITIALIZED_RETRY_INTERVAL_MS = 200
)

type processInfo struct {
	name       string
	expiration time.Time
}

func getProcessNameFromAdbRaw(pid int) string {
	cmd := exec.Command("adb", "shell", fmt.Sprintf("cat /proc/%d/cmdline 2>/dev/null", pid))
	out, err := cmd.Output()
	mlib.Check(err)

	procname := string(out)
	procname = strings.TrimRight(procname, "\000")
	procname = strings.Replace(procname, "\000", " ", -1)
	return procname
}

// Same as getProcessNameFromAdbRaw, but retries when getting PRE_INITIALIZED
func getProcessNameFromAdb(pid int) string {
	var pname string = "(unknown)"

	mlib.Debug("Getting process name for %d\n", pid)

	for i := 0; i <= MAX_PRE_INITIALIZED_RETRY; i++ {
		rawName := getProcessNameFromAdbRaw(pid)
		if rawName == PRE_INITIALIZED {
			mlib.Debug("%s detected\n", PRE_INITIALIZED)
			time.Sleep(MAX_PRE_INITIALIZED_RETRY_INTERVAL_MS * time.Millisecond)
			continue
		}
		if rawName != "" {
			pname = rawName
		}
		break
	}
	return fmt.Sprintf("%s %d", pname, pid)
}

func getProcessNameWithCache(pid int) string {
	now := time.Now()
	pinfo, ok := procecces[pid]
	if !ok || now.After(pinfo.expiration) {
		name := getProcessNameFromAdb(pid)
		procecces[pid] = processInfo{name: name, expiration: now.Add(cacheExpiration)}
	}
	return pinfo.name
}

// Run for each line
func processLine(line string) {
	var pid = 0
	var processName = ""

	// Find the pid from the line and get the process name
	for _, re := range pidPatterns {
		s := re.FindStringSubmatch(line)
		if s != nil {
			pid, _ = strconv.Atoi(s[1])
			processName = getProcessNameWithCache(pid)
			break
		}
	}

	// Any process died?
	for _, re := range diePatterns {
		s := re.FindStringSubmatch(line)
		if s != nil {
			diedPid, _ := strconv.Atoi(s[1])
			log.Printf("Process %d died\n", diedPid)
			delete(procecces, diedPid)
			break
		}
	}

	fmt.Printf(outFormat, processName, line)
}

func main() {
	log.SetPrefix("logcatp: ")
	log.SetFlags(0)
	flag.Parse()

	outFormat = fmt.Sprintf("[%%-%ds] %%s", *width)

	for line := range mlib.ReadFilesFromArgs() {
		processLine(line)
	}
}
<commit_msg>Fix cache bug<commit_after>///bin/true; exec /usr/bin/env go run "$0" "$@"

package main

import (
	"flag"
	"fmt"
	"github.com/omakoto/mlib"
	"log"
	"os/exec"
	"regexp"
	"strconv"
	"strings"
	"time"
)

var (
	// Patterns to extract PIDs from logs.
	pidPatterns = []*regexp.Regexp{
		// brief
		regexp.MustCompile(`^[A-Z]/.+?\(\s*(\d+)`),
		// process or thread
		regexp.MustCompile(`^[A-Z]\(\s*(\d+)`),
		// threadtime
		regexp.MustCompile(`^\d{2}-\d{2}\s+[\d\:\.]+\s+(\d+)`),
		// time
		regexp.MustCompile(`^\d{2}-\d{2} [\d\:\.]+ [A-Z]\/.*?\(\s*(\d+)`),
		// regexp.MustCompile(`^.*?\(\s*(\d+)`),
		// long
		regexp.MustCompile(`^\[\s*\d{2}-\d{2}\s+[\d\:\.]+\s+(\d+)`),
	}

	// Patterns to extract PIDs for process deaths logs.
	diePatterns = []*regexp.Regexp{
		regexp.MustCompile(`ActivityManager.*?Process .*?\(pid (\d+)\) has died`),
		regexp.MustCompile(`ActivityManager.*?Killing (\d+)`),
		// Any more?
	}

	whiteSpaces = regexp.MustCompile(`\s+`)

	// Flags
	width = flag.Int("w", 40, "formatting width")
	// autoflush = flag.Bool("f", false, "autoflush") // Stdout seems like always flushing

	// Output line format
	outFormat string

	// Process info cache.
	procecces = make(map[int]processInfo)

	cacheExpiration = time.Minute * 10
)

const (
	PRE_INITIALIZED                       = "<pre-initialized>"
	MAX_PRE_INITIALIZED_RETRY             = 5
	MAX_PRE_INITIALIZED_RETRY_INTERVAL_MS = 200
)

type processInfo struct {
	name       string
	expiration time.Time
}

func getProcessNameFromAdbRaw(pid int) string {
	cmd := exec.Command("adb", "shell", fmt.Sprintf("cat /proc/%d/cmdline 2>/dev/null", pid))
	out, err := cmd.Output()
	mlib.Check(err)

	procname := string(out)
	procname = strings.TrimRight(procname, "\000")
	procname = strings.Replace(procname, "\000", " ", -1)
	return procname
}

// Same as getProcessNameFromAdbRaw, but retries when getting PRE_INITIALIZED
func getProcessNameFromAdb(pid int) string {
	var pname string = "(unknown)"

	mlib.Debug("Getting process name for %d\n", pid)

	for i := 0; i <= MAX_PRE_INITIALIZED_RETRY; i++ {
		rawName := getProcessNameFromAdbRaw(pid)
		if rawName == PRE_INITIALIZED {
			mlib.Debug("%s detected\n", PRE_INITIALIZED)
			time.Sleep(MAX_PRE_INITIALIZED_RETRY_INTERVAL_MS * time.Millisecond)
			continue
		}
		if rawName != "" {
			pname = rawName
		}
		break
	}
	return fmt.Sprintf("%s %d", pname, pid)
}

func getProcessNameWithCache(pid int) string {
	now := time.Now()
	pinfo, ok := procecces[pid]
	if !ok || now.After(pinfo.expiration) {
		name := getProcessNameFromAdb(pid)
		pinfo = processInfo{name: name, expiration: now.Add(cacheExpiration)}
		procecces[pid] = pinfo
	}
	return pinfo.name
}

// Run for each line
func processLine(line string) {
	var pid = 0
	var processName = "(pid not found)"

	// Find the pid from the line and get the process name
	for _, re := range pidPatterns {
		s := re.FindStringSubmatch(line)
		if s != nil {
			pid, _ = strconv.Atoi(s[1])
			processName = getProcessNameWithCache(pid)
			break
		}
	}

	// Any process died?
	for _, re := range diePatterns {
		s := re.FindStringSubmatch(line)
		if s != nil {
			diedPid, _ := strconv.Atoi(s[1])
			log.Printf("Process %d died\n", diedPid)
			delete(procecces, diedPid)
			break
		}
	}

	fmt.Printf(outFormat, processName, line)
}

func main() {
	log.SetPrefix("logcatp: ")
	log.SetFlags(0)
	flag.Parse()

	outFormat = fmt.Sprintf("[%%-%ds] %%s", *width)

	for line := range mlib.ReadFilesFromArgs() {
		processLine(line)
	}
}
<|endoftext|>


<commit_before>// Copyright 2016 Circonus, Inc. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package circonusgometrics

import (
	"bytes"
	"context"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"net"
	"net/http"
	"strconv"
	"time"

	"github.com/circonus-labs/go-apiclient"
	"github.com/hashicorp/go-retryablehttp"
	"github.com/pkg/errors"
)

func (m *CirconusMetrics) submit(output Metrics, newMetrics map[string]*apiclient.CheckBundleMetric) {

	// if there is nowhere to send metrics to, just return.
	if !m.check.IsReady() {
		m.Log.Printf("check not ready, skipping metric submission")
		return
	}

	// update check if there are any new metrics or, if metric tags have been added since last submit
	m.check.UpdateCheck(newMetrics)

	str, err := json.Marshal(output)
	if err != nil {
		m.Log.Printf("error preparing metrics %s", err)
		return
	}

	numStats, err := m.trapCall(str)
	if err != nil {
		m.Log.Printf("error sending metrics - %s\n", err)
		return
	}

	// OK response from circonus-agent does not
	// indicate how many metrics were received
	if numStats == -1 {
		numStats = len(output)
	}

	if m.Debug {
		m.Log.Printf("%d stats sent\n", numStats)
	}
}

func (m *CirconusMetrics) trapCall(payload []byte) (int, error) {
	trap, err := m.check.GetSubmissionURL()
	if err != nil {
		return 0, errors.Wrap(err, "trap call")
	}

	dataReader := bytes.NewReader(payload)

	req, err := retryablehttp.NewRequest("PUT", trap.URL.String(), dataReader)
	if err != nil {
		return 0, err
	}
	req.Header.Add("Content-Type", "application/json")
	req.Header.Add("Accept", "application/json")

	// keep last HTTP error in the event of retry failure
	var lastHTTPError error
	retryPolicy := func(ctx context.Context, resp *http.Response, err error) (bool, error) {
		if ctxErr := ctx.Err(); ctxErr != nil {
			return false, ctxErr
		}

		if err != nil {
			lastHTTPError = err
			return true, errors.Wrap(err, "retry policy")
		}
		// Check the response code. We retry on 500-range responses to allow
		// the server time to recover, as 500's are typically not permanent
		// errors and may relate to outages on the server side. This will catch
		// invalid response codes as well, like 0 and 999.
		if resp.StatusCode == 0 || resp.StatusCode >= 500 {
			body, readErr := ioutil.ReadAll(resp.Body)
			if readErr != nil {
				lastHTTPError = fmt.Errorf("- last HTTP error: %d %+v", resp.StatusCode, readErr)
			} else {
				lastHTTPError = fmt.Errorf("- last HTTP error: %d %s", resp.StatusCode, string(body))
			}
			return true, nil
		}
		return false, nil
	}

	client := retryablehttp.NewClient()
	switch {
	case trap.URL.Scheme == "https":
		client.HTTPClient.Transport = &http.Transport{
			Proxy: http.ProxyFromEnvironment,
			Dial: (&net.Dialer{
				Timeout:   30 * time.Second,
				KeepAlive: 30 * time.Second,
			}).Dial,
			TLSHandshakeTimeout: 10 * time.Second,
			TLSClientConfig:     trap.TLS,
			DisableKeepAlives:   true,
			MaxIdleConnsPerHost: -1,
			DisableCompression:  false,
		}
	case trap.URL.Scheme == "http":
		client.HTTPClient.Transport = &http.Transport{
			Proxy: http.ProxyFromEnvironment,
			Dial: (&net.Dialer{
				Timeout:   30 * time.Second,
				KeepAlive: 30 * time.Second,
			}).Dial,
			DisableKeepAlives:   true,
			MaxIdleConnsPerHost: -1,
			DisableCompression:  false,
		}
	case trap.IsSocket:
		m.Log.Printf("using socket transport\n")
		client.HTTPClient.Transport = trap.SockTransport
	default:
		return 0, errors.Errorf("unknown scheme (%s), skipping submission", trap.URL.Scheme)
	}
	client.RetryWaitMin = 1 * time.Second
	client.RetryWaitMax = 5 * time.Second
	client.RetryMax = 3
	// retryablehttp only groks log or no log
	// but, outputs everything as [DEBUG] messages
	if m.Debug {
		client.Logger = m.Log.(*log.Logger)
	} else {
		client.Logger = log.New(ioutil.Discard, "", log.LstdFlags)
	}
	client.CheckRetry = retryPolicy

	attempts := -1
	client.RequestLogHook = func(logger retryablehttp.Logger, req *http.Request, retryNumber int) {
		attempts = retryNumber
	}

	resp, err := client.Do(req)
	if err != nil {
		if lastHTTPError != nil {
			return 0, fmt.Errorf("submitting: %+v %+v", err, lastHTTPError)
		}
		if attempts == client.RetryMax {
			if err := m.check.RefreshTrap(); err != nil {
				return 0, errors.Wrap(err, "refreshing trap")
			}
		}
		return 0, errors.Wrap(err, "trap call")
	}

	defer resp.Body.Close()

	// no content - expected result from
	// circonus-agent when metrics accepted
	if resp.StatusCode == http.StatusNoContent {
		return -1, nil
	}

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		m.Log.Printf("error reading body, proceeding - %s\n", err)
	}

	var response map[string]interface{}
	if err := json.Unmarshal(body, &response); err != nil {
		m.Log.Printf("error parsing body, proceeding - %s (%s)\n", err, body)
	}

	if resp.StatusCode != http.StatusOK {
		return 0, errors.New("bad response code: " + strconv.Itoa(resp.StatusCode))
	}
	switch v := response["stats"].(type) {
	case float64:
		return int(v), nil
	case int:
		return v, nil
	default:
	}
	return 0, errors.New("error, bad response data type (not numeric)")
}
<commit_msg>fix: internal log interface satisfies rh log interface no need to cast to log.Logger any longer<commit_after>// Copyright 2016 Circonus, Inc. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package circonusgometrics

import (
	"bytes"
	"context"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"net"
	"net/http"
	"strconv"
	"time"

	"github.com/circonus-labs/go-apiclient"
	"github.com/hashicorp/go-retryablehttp"
	"github.com/pkg/errors"
)

func (m *CirconusMetrics) submit(output Metrics, newMetrics map[string]*apiclient.CheckBundleMetric) {

	// if there is nowhere to send metrics to, just return.
	if !m.check.IsReady() {
		m.Log.Printf("check not ready, skipping metric submission")
		return
	}

	// update check if there are any new metrics or, if metric tags have been added since last submit
	m.check.UpdateCheck(newMetrics)

	str, err := json.Marshal(output)
	if err != nil {
		m.Log.Printf("error preparing metrics %s", err)
		return
	}

	numStats, err := m.trapCall(str)
	if err != nil {
		m.Log.Printf("error sending metrics - %s\n", err)
		return
	}

	// OK response from circonus-agent does not
	// indicate how many metrics were received
	if numStats == -1 {
		numStats = len(output)
	}

	if m.Debug {
		m.Log.Printf("%d stats sent\n", numStats)
	}
}

func (m *CirconusMetrics) trapCall(payload []byte) (int, error) {
	trap, err := m.check.GetSubmissionURL()
	if err != nil {
		return 0, errors.Wrap(err, "trap call")
	}

	dataReader := bytes.NewReader(payload)

	req, err := retryablehttp.NewRequest("PUT", trap.URL.String(), dataReader)
	if err != nil {
		return 0, err
	}
	req.Header.Add("Content-Type", "application/json")
	req.Header.Add("Accept", "application/json")

	// keep last HTTP error in the event of retry failure
	var lastHTTPError error
	retryPolicy := func(ctx context.Context, resp *http.Response, err error) (bool, error) {
		if ctxErr := ctx.Err(); ctxErr != nil {
			return false, ctxErr
		}

		if err != nil {
			lastHTTPError = err
			return true, errors.Wrap(err, "retry policy")
		}
		// Check the response code. We retry on 500-range responses to allow
		// the server time to recover, as 500's are typically not permanent
		// errors and may relate to outages on the server side. This will catch
		// invalid response codes as well, like 0 and 999.
		if resp.StatusCode == 0 || resp.StatusCode >= 500 {
			body, readErr := ioutil.ReadAll(resp.Body)
			if readErr != nil {
				lastHTTPError = fmt.Errorf("- last HTTP error: %d %+v", resp.StatusCode, readErr)
			} else {
				lastHTTPError = fmt.Errorf("- last HTTP error: %d %s", resp.StatusCode, string(body))
			}
			return true, nil
		}
		return false, nil
	}

	client := retryablehttp.NewClient()
	switch {
	case trap.URL.Scheme == "https":
		client.HTTPClient.Transport = &http.Transport{
			Proxy: http.ProxyFromEnvironment,
			Dial: (&net.Dialer{
				Timeout:   30 * time.Second,
				KeepAlive: 30 * time.Second,
			}).Dial,
			TLSHandshakeTimeout: 10 * time.Second,
			TLSClientConfig:     trap.TLS,
			DisableKeepAlives:   true,
			MaxIdleConnsPerHost: -1,
			DisableCompression:  false,
		}
	case trap.URL.Scheme == "http":
		client.HTTPClient.Transport = &http.Transport{
			Proxy: http.ProxyFromEnvironment,
			Dial: (&net.Dialer{
				Timeout:   30 * time.Second,
				KeepAlive: 30 * time.Second,
			}).Dial,
			DisableKeepAlives:   true,
			MaxIdleConnsPerHost: -1,
			DisableCompression:  false,
		}
	case trap.IsSocket:
		m.Log.Printf("using socket transport\n")
		client.HTTPClient.Transport = trap.SockTransport
	default:
		return 0, errors.Errorf("unknown scheme (%s), skipping submission", trap.URL.Scheme)
	}
	client.RetryWaitMin = 1 * time.Second
	client.RetryWaitMax = 5 * time.Second
	client.RetryMax = 3
	// retryablehttp only groks log or no log
	// but, outputs everything as [DEBUG] messages
	if m.Debug {
		client.Logger = m.Log
	} else {
		client.Logger = log.New(ioutil.Discard, "", log.LstdFlags)
	}
	client.CheckRetry = retryPolicy

	attempts := -1
	client.RequestLogHook = func(logger retryablehttp.Logger, req *http.Request, retryNumber int) {
		attempts = retryNumber
	}

	resp, err := client.Do(req)
	if err != nil {
		if lastHTTPError != nil {
			return 0, fmt.Errorf("submitting: %+v %+v", err, lastHTTPError)
		}
		if attempts == client.RetryMax {
			if err := m.check.RefreshTrap(); err != nil {
				return 0, errors.Wrap(err, "refreshing trap")
			}
		}
		return 0, errors.Wrap(err, "trap call")
	}

	defer resp.Body.Close()

	// no content - expected result from
	// circonus-agent when metrics accepted
	if resp.StatusCode == http.StatusNoContent {
		return -1, nil
	}

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		m.Log.Printf("error reading body, proceeding - %s\n", err)
	}

	var response map[string]interface{}
	if err := json.Unmarshal(body, &response); err != nil {
		m.Log.Printf("error parsing body, proceeding - %s (%s)\n", err, body)
	}

	if resp.StatusCode != http.StatusOK {
		return 0, errors.New("bad response code: " + strconv.Itoa(resp.StatusCode))
	}
	switch v := response["stats"].(type) {
	case float64:
		return int(v), nil
	case int:
		return v, nil
	default:
	}
	return 0, errors.New("error, bad response data type (not numeric)")
}
<|endoftext|>


<commit_before>package payload

import (
	."github.com/DNAProject/DNA/core/code"
	"github.com/DNAProject/DNA/common/serialization"
	"io"
)

type DeployCode struct {
	Code 		FunctionCode
	Name 		string
	CodeVersion 	string
	Author		string
	Email		string
	Description 	string
}

func (dc *DeployCode) Data() []byte {
	// TODO: Data()

	return []byte{0}
}

func (dc *DeployCode) Serialize(w io.Writer) error {

	err := dc.Code.Serialize(w)
	if err != nil {
		return err
	}

	err = serialization.WriteVarString(w,dc.Name)
	if err != nil {
		return err
	}

	err = serialization.WriteVarString(w,dc.CodeVersion)
	if err != nil {
		return err
	}

	err = serialization.WriteVarString(w,dc.Author)
	if err != nil {
		return err
	}

	err = serialization.WriteVarString(w,dc.Email)
	if err != nil {
		return err
	}

	err = serialization.WriteVarString(w,dc.Description)
	if err != nil {
		return err
	}

	return nil
}

func (dc *DeployCode) Deserialize(r io.Reader) error {
	err := dc.Code.Deserialize(r)
	if err != nil {
		return err
	}

	dc.Name,err = serialization.ReadVarString(r)
	if err != nil {
		return err
	}

	dc.CodeVersion,err = serialization.ReadVarString(r)
	if err != nil {
		return err
	}

	dc.Author,err = serialization.ReadVarString(r)
	if err != nil {
		return err
	}

	dc.Email,err = serialization.ReadVarString(r)
	if err != nil {
		return err
	}

	dc.Description,err = serialization.ReadVarString(r)
	if err != nil {
		return err
	}

	return nil
}

<commit_msg>Update DeployCode.go<commit_after>package payload

import (
	."github.com/DNAProject/DNA/core/code"
	"github.com/DNAProject/DNA/common/serialization"
	"io"
)

type DeployCode struct {
	Code 		*FunctionCode
	Name 		string
	CodeVersion 	string
	Author		string
	Email		string
	Description 	string
}

func (dc *DeployCode) Data() []byte {
	// TODO: Data()

	return []byte{0}
}

func (dc *DeployCode) Serialize(w io.Writer) error {

	err := dc.Code.Serialize(w)
	if err != nil {
		return err
	}

	err = serialization.WriteVarString(w,dc.Name)
	if err != nil {
		return err
	}

	err = serialization.WriteVarString(w,dc.CodeVersion)
	if err != nil {
		return err
	}

	err = serialization.WriteVarString(w,dc.Author)
	if err != nil {
		return err
	}

	err = serialization.WriteVarString(w,dc.Email)
	if err != nil {
		return err
	}

	err = serialization.WriteVarString(w,dc.Description)
	if err != nil {
		return err
	}

	return nil
}

func (dc *DeployCode) Deserialize(r io.Reader) error {
	err := dc.Code.Deserialize(r)
	if err != nil {
		return err
	}

	dc.Name,err = serialization.ReadVarString(r)
	if err != nil {
		return err
	}

	dc.CodeVersion,err = serialization.ReadVarString(r)
	if err != nil {
		return err
	}

	dc.Author,err = serialization.ReadVarString(r)
	if err != nil {
		return err
	}

	dc.Email,err = serialization.ReadVarString(r)
	if err != nil {
		return err
	}

	dc.Description,err = serialization.ReadVarString(r)
	if err != nil {
		return err
	}

	return nil
}

<|endoftext|>


<commit_before>package gosubway

import (
	"fmt"
	"io/ioutil"
	"net/http"
	"sort"
	"strings"
	"time"

	"code.google.com/p/goprotobuf/proto"

	_ "github.com/jprobinson/gtfs/nyct_subway_proto"
	"github.com/jprobinson/gtfs/transit_realtime"
)

type FeedMessage struct {
	transit_realtime.FeedMessage
}

// GetFeed takes an API key generated from http://datamine.mta.info/user/register
// and a boolean specifying which feed (1,2,3,4,5,6,S trains OR L train) and
// it will return a transit_realtime.FeedMessage with NYCT extensions.
func GetFeed(key string, lTrain bool) (*FeedMessage, error) {
	url := fmt.Sprint("http://datamine.mta.info/mta_esi.php?key=", key)
	if lTrain {
		url = fmt.Sprint(url, "&feed_id=2")
	}
	resp, err := http.Get(url)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}

	transit := &transit_realtime.FeedMessage{}
	err = proto.Unmarshal(body, transit)
	if err != nil {
		return nil, err
	}
	return &FeedMessage{*transit}, nil
}

type StopTimeUpdate struct {
	transit_realtime.TripUpdate_StopTimeUpdate
}

// Trains will accept a stopId (found here: http://web.mta.info/developers/data/nyct/subway/google_transit.zip)
// and returns a list of updates from northbound and southbound trains
func (f *FeedMessage) Trains(stopId string) (northbound, southbound []*StopTimeUpdate) {

	for _, ent := range f.Entity {
		if ent.TripUpdate != nil {
			for _, upd := range ent.TripUpdate.StopTimeUpdate {
				if strings.HasPrefix(*upd.StopId, stopId) {
					if strings.HasSuffix(*upd.StopId, "N") {
						northbound = append(northbound, &StopTimeUpdate{*upd})
					} else {
						southbound = append(southbound, &StopTimeUpdate{*upd})
					}
				}
			}
		}

	}

	return
}

// NextTrainTimes will return an ordered slice of upcoming train departure times
// in either direction.
func (f *FeedMessage) NextTrainTimes(stopId string) (northbound, southbound []time.Time) {
	north, south := f.Trains(stopId)
	northbound = NextTrainTimes(north)
	southbound = NextTrainTimes(south)
	return
}

// NextTrainTimes will extract the departure times from the given
// update slice, order and return them.
func NextTrainTimes(updates []*StopTimeUpdate) []time.Time {
	var times []time.Time

	for _, upd := range updates {
		if upd.Departure == nil {
			continue
		}
		unix := *upd.Departure.Time
		if upd.Departure.Delay != nil {
			unix += int64(*upd.Departure.Delay * 1000)
		}
		dept := time.Unix(unix, 0)
		if dept.After(time.Now()) {
			times = append(times, dept)
		}
	}
	sort.Sort(timeSlice(times))
	if len(times) > 5 {
		times = times[:5]
	}
	return times
}

type timeSlice []time.Time

func (t timeSlice) Len() int {
	return len(t)
}

func (t timeSlice) Less(i, j int) bool {
	return t[i].Before(t[j])
}

func (t timeSlice) Swap(i, j int) {
	t[i], t[j] = t[j], t[i]
}
<commit_msg>added second feed<commit_after>package gosubway

import (
	"fmt"
	"io/ioutil"
	"net/http"
	"sort"
	"strings"
	"time"

	"code.google.com/p/goprotobuf/proto"

	_ "github.com/jprobinson/gtfs/nyct_subway_proto"
	"github.com/jprobinson/gtfs/transit_realtime"
)

type FeedMessage struct {
	transit_realtime.FeedMessage
}

// GetFeed takes an API key generated from http://datamine.mta.info/user/register
// and a boolean specifying which feed (1,2,3,4,5,6,S trains OR L train) and
// it will return a transit_realtime.FeedMessage with NYCT extensions.
func GetFeed(key string, lTrain bool) (*FeedMessage, error) {
	url := fmt.Sprint("http://datamine.mta.info/mta_esi.php?key=", key)
	if lTrain {
		url = fmt.Sprint(url, "&feed_id=2")
	} else {
		url = fmt.Sprint(url, "&feed_id=1")
	}
	resp, err := http.Get(url)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}

	transit := &transit_realtime.FeedMessage{}
	err = proto.Unmarshal(body, transit)
	if err != nil {
		return nil, err
	}
	return &FeedMessage{*transit}, nil
}

type StopTimeUpdate struct {
	transit_realtime.TripUpdate_StopTimeUpdate
}

// Trains will accept a stopId (found here: http://web.mta.info/developers/data/nyct/subway/google_transit.zip)
// and returns a list of updates from northbound and southbound trains
func (f *FeedMessage) Trains(stopId string) (northbound, southbound []*StopTimeUpdate) {

	for _, ent := range f.Entity {
		if ent.TripUpdate != nil {
			for _, upd := range ent.TripUpdate.StopTimeUpdate {
				if strings.HasPrefix(*upd.StopId, stopId) {
					if strings.HasSuffix(*upd.StopId, "N") {
						northbound = append(northbound, &StopTimeUpdate{*upd})
					} else if strings.HasSuffix(*upd.StopId, "S") {
						southbound = append(southbound, &StopTimeUpdate{*upd})
					}
				}
			}
		}

	}

	return
}

// NextTrainTimes will return an ordered slice of upcoming train departure times
// in either direction.
func (f *FeedMessage) NextTrainTimes(stopId string) (northbound, southbound []time.Time) {
	north, south := f.Trains(stopId)
	northbound = NextTrainTimes(north)
	southbound = NextTrainTimes(south)
	return
}

// NextTrainTimes will extract the departure times from the given
// update slice, order and return them.
func NextTrainTimes(updates []*StopTimeUpdate) []time.Time {
	var times []time.Time

	for _, upd := range updates {
		if upd.Departure == nil {
			continue
		}
		unix := *upd.Departure.Time
		if upd.Departure.Delay != nil {
			unix += int64(*upd.Departure.Delay * 1000)
		}
		dept := time.Unix(unix, 0)
		if dept.After(time.Now()) {
			times = append(times, dept)
		}
	}
	sort.Sort(timeSlice(times))
	if len(times) > 5 {
		times = times[:5]
	}
	return times
}

type timeSlice []time.Time

func (t timeSlice) Len() int {
	return len(t)
}

func (t timeSlice) Less(i, j int) bool {
	return t[i].Before(t[j])
}

func (t timeSlice) Swap(i, j int) {
	t[i], t[j] = t[j], t[i]
}
<|endoftext|>


<commit_before>package runtime

// Version is the current version of the buffalo binary
var Version = "v0.16.26"
<commit_msg>bumping version<commit_after>package runtime

// Version is the current version of the buffalo binary
var Version = "v0.16.27"
<|endoftext|>


<commit_before>// 29 december 2012
package main

import (
	"os"
	"encoding/xml"
	"log"
)

const nodump = "nodump"	// for ROM.Status

// CHDs only have Name, SHA1, and Status
type ROM struct {
	Name	string		`xml:"name,attr"`
	Size		uint32		`xml:"size,attr"`		// uint32 because that's what archive/zip.FIleHeader.UncompressedSize is
	CRC32	string		`xml:"crc,attr"`
	SHA1	string		`xml:"sha1,attr"`
	Status	string		`xml:"status,attr"`
}

type Game struct {
	Name	string	`xml:"name,attr"`
	CloneOf	string	`xml:"cloneof,attr"`
	ROMOf	string	`xml:"romof,attr"`
	// TODO do I need sampleof?
	ROMs	[]ROM	`xml:"rom"`
	CHDs	[]ROM	`xml:"disk"`
}

var games = map[string]*Game{}

func getGames(filename string) {
	f, err := os.Open(filename)
	if err != nil {
		log.Fatalf("could not open MAME XML file %s: %v", filename, err)
	}
	defer f.Close()

	var g struct {
		Games	[]Game	`xml:"game"`
	}

	mamexml := xml.NewDecoder(f)
	err = mamexml.Decode(&g)
	if err != nil {
		log.Fatalf("could not read MAME XML file %s: %v", filename, err)
	}

	for i := range g.Games {
		games[g.Games[i].Name] = &(g.Games[i])
	}
}
<commit_msg>Refactor out repeated CloneOf/ROMOf by using a slice and a loop containing the two values; the slice is computed by getGames(). (1/2)<commit_after>// 29 december 2012
package main

import (
	"os"
	"encoding/xml"
	"log"
)

const nodump = "nodump"	// for ROM.Status

// CHDs only have Name, SHA1, and Status
type ROM struct {
	Name	string		`xml:"name,attr"`
	Size		uint32		`xml:"size,attr"`		// uint32 because that's what archive/zip.FIleHeader.UncompressedSize is
	CRC32	string		`xml:"crc,attr"`
	SHA1	string		`xml:"sha1,attr"`
	Status	string		`xml:"status,attr"`
}

type Game struct {
	Name	string	`xml:"name,attr"`
	CloneOf	string	`xml:"cloneof,attr"`
	ROMOf	string	`xml:"romof,attr"`
	// TODO do I need sampleof?
	ROMs	[]ROM	`xml:"rom"`
	CHDs	[]ROM	`xml:"disk"`

	// prepared by getGames()
	Parents	[]string	// [CloneOf, ROMOf] but only if either is specified and no repeats; avoids code duplication in check.go
}

var games = map[string]*Game{}

func getGames(filename string) {
	f, err := os.Open(filename)
	if err != nil {
		log.Fatalf("could not open MAME XML file %s: %v", filename, err)
	}
	defer f.Close()

	var g struct {
		Games	[]Game	`xml:"game"`
	}

	mamexml := xml.NewDecoder(f)
	err = mamexml.Decode(&g)
	if err != nil {
		log.Fatalf("could not read MAME XML file %s: %v", filename, err)
	}

	for i := range g.Games {
		this := &(g.Games[i])
		games[this.Name] = this
		if this.CloneOf != "" {
			this.Parents = append(this.Parents, this.CloneOf)
		}
		if this.ROMOf != "" && this.ROMOf != this.CloneOf {
			this.Parents = append(this.Parents, this.ROMOf)
		}
	}
}
<|endoftext|>


<commit_before>package parser

import (
	"encoding/json"
	"errors"
	"fmt"
	"strconv"
	"strings"
	"unicode"
)

func ParseArguments(args []string) (res map[string]interface{}, err error) {
	res = make(map[string]interface{}, 0)
	if len(args) == 0 {
		return res, nil
	}

	var key string
	isSettingKey := false
	for i := 0; i < len(args); i++ {
		if args[i] == "--" {
			return nil, fmt.Errorf("-- is an invalid argument.")
		}
		if strings.HasPrefix(args[i], "--") {
			key = normalizePropertyName(args[i])
			if _, ok := res[key]; ok {
				return nil, fmt.Errorf("Option '%s' is specified twice.", key)
			}
			if i+1 == len(args) || strings.HasPrefix(args[i+1], "--") {
				res[key] = nil
			} else {
				isSettingKey = true
			}
			continue
		}
		if isSettingKey {
			if _, ok := res[key]; !ok {
				res[key] = args[i]
			} else if _, ok := res[key].(string); ok {
				array := make([]interface{}, 0)
				array = append(array, res[key], args[i])
				res[key] = array
			} else {
				res[key] = append(res[key].([]interface{}), args[i])
			}
			continue
		} else {
			jsonArg := map[string]interface{}{}
			err = json.Unmarshal([]byte(args[i]), &jsonArg)
			if err != nil {
				return nil, fmt.Errorf("Invalid JSON: %s.", args[i])
			}
			NormalizeKeys(jsonArg)
			for k, v := range jsonArg {
				if _, ok := res[k]; ok {
					return nil, fmt.Errorf("Option '%s' is specified twice.", k)
				}
				res[k] = v
			}
			continue
		}
	}
	return res, nil
}

func NormalizeKeys(arg interface{}) {
	if argObj, isObj := arg.(map[string]interface{}); isObj {
		for k, v := range argObj {
			n := normalizePropertyName(k)
			delete(argObj, k)
			(argObj)[n] = v
			NormalizeKeys(v)
		}
	} else if argArray, isArray := arg.([]interface{}); isArray {
		for _, v := range argArray {
			NormalizeKeys(v)
		}
	}
}

func normalizePropertyName(prName string) string {
	prName = strings.TrimLeft(prName, "--")
	array := strings.Split(prName, "-")
	res := make([]rune, 0)
	for _, item := range array {
		chars := []rune(item)
		chars[0] = unicode.ToUpper(chars[0])
		res = append(res, chars...)
	}
	return string(res)
}

func normalizeValue(value string) interface{} {
	var obj interface{} = value
	if val, err := strconv.ParseFloat(value, 64); err == nil {
		obj = val
	} else if val, err := strconv.ParseBool(value); err == nil {
		obj = val
	}
	return obj
}

type state func(r rune) error

var curState state
var curQuote rune
var curItem []rune
var items []string

func ParseObject(obj string) (map[string]interface{}, error) {
	curState = startParseKey
	curQuote = '\000'
	curItem = []rune{}
	items = []string{}
	for _, c := range obj {
		err := curState(c)
		if err != nil {
			return nil, err
		}
	}
	curState('\000')
	if len(items) <= 1 {
		return nil, errors.New("Object is not in a=b,c=d,.. notation.")
	}
	res := make(map[string]interface{}, 0)
	for i := 0; i < len(items); i += 2 {
		key := normalizePropertyName(items[i])
		if i == len(items)-1 {
			res[key] = nil
		} else {
			res[key] = normalizeValue(items[i+1])
		}
	}
	return res, nil
}

func saveCurItem() {
	items = append(items, string(curItem))
	curItem = []rune{}
}

func startParseKey(r rune) error {
	switch r {
	case '\'', '"':
		curQuote = r
		curState = parseQuotedKey
	default:
		curItem = append(curItem, r)
		curState = parseSimpleKey
	}
	return nil
}

func parseSimpleKey(r rune) error {
	switch r {
	case '=':
		curState = startParseValue
		saveCurItem()
	case '\000':
		saveCurItem()
	default:
		curItem = append(curItem, r)
	}
	return nil
}

func parseQuotedKey(r rune) error {
	switch r {
	case curQuote:
		curState = keyParsed
		saveCurItem()
	case '\000':
		return errors.New("")
	default:
		curItem = append(curItem, r)
	}
	return nil
}

func keyParsed(r rune) error {
	switch r {
	case '=':
		curState = startParseValue
		return nil
	case '\000':
		return nil
	default:
		return errors.New("")

	}
}

func startParseValue(r rune) error {
	switch r {
	case '\'', '"':
		curQuote = r
		curState = parseQuotedValue
	default:
		curItem = append(curItem, r)
		curState = parseSimpleValue
	}
	return nil
}

func parseSimpleValue(r rune) error {
	switch r {
	case ',':
		curState = startParseKey
		saveCurItem()
	case '\000':
		saveCurItem()
	default:
		curItem = append(curItem, r)
	}
	return nil
}

func parseQuotedValue(r rune) error {
	switch r {
	case curQuote:
		curState = valueParsed
		saveCurItem()
	case '\000':
		return errors.New("")
	default:
		curItem = append(curItem, r)
	}
	return nil
}

func valueParsed(r rune) error {
	switch r {
	case ',':
		curState = startParseKey
		return nil
	case '\000':
		return nil
	default:
		return errors.New("")

	}
}
<commit_msg>remove the obsolete normalizeValue function<commit_after>package parser

import (
	"encoding/json"
	"errors"
	"fmt"
	"strings"
	"unicode"
)

func ParseArguments(args []string) (res map[string]interface{}, err error) {
	res = make(map[string]interface{}, 0)
	if len(args) == 0 {
		return res, nil
	}

	var key string
	isSettingKey := false
	for i := 0; i < len(args); i++ {
		if args[i] == "--" {
			return nil, fmt.Errorf("-- is an invalid argument.")
		}
		if strings.HasPrefix(args[i], "--") {
			key = normalizePropertyName(args[i])
			if _, ok := res[key]; ok {
				return nil, fmt.Errorf("Option '%s' is specified twice.", key)
			}
			if i+1 == len(args) || strings.HasPrefix(args[i+1], "--") {
				res[key] = nil
			} else {
				isSettingKey = true
			}
			continue
		}
		if isSettingKey {
			if _, ok := res[key]; !ok {
				res[key] = args[i]
			} else if _, ok := res[key].(string); ok {
				array := make([]interface{}, 0)
				array = append(array, res[key], args[i])
				res[key] = array
			} else {
				res[key] = append(res[key].([]interface{}), args[i])
			}
			continue
		} else {
			jsonArg := map[string]interface{}{}
			err = json.Unmarshal([]byte(args[i]), &jsonArg)
			if err != nil {
				return nil, fmt.Errorf("Invalid JSON: %s.", args[i])
			}
			NormalizeKeys(jsonArg)
			for k, v := range jsonArg {
				if _, ok := res[k]; ok {
					return nil, fmt.Errorf("Option '%s' is specified twice.", k)
				}
				res[k] = v
			}
			continue
		}
	}
	return res, nil
}

func NormalizeKeys(arg interface{}) {
	if argObj, isObj := arg.(map[string]interface{}); isObj {
		for k, v := range argObj {
			n := normalizePropertyName(k)
			delete(argObj, k)
			(argObj)[n] = v
			NormalizeKeys(v)
		}
	} else if argArray, isArray := arg.([]interface{}); isArray {
		for _, v := range argArray {
			NormalizeKeys(v)
		}
	}
}

func normalizePropertyName(prName string) string {
	prName = strings.TrimLeft(prName, "--")
	array := strings.Split(prName, "-")
	res := make([]rune, 0)
	for _, item := range array {
		chars := []rune(item)
		chars[0] = unicode.ToUpper(chars[0])
		res = append(res, chars...)
	}
	return string(res)
}

type state func(r rune) error

var curState state
var curQuote rune
var curItem []rune
var items []string

func ParseObject(obj string) (map[string]interface{}, error) {
	curState = startParseKey
	curQuote = '\000'
	curItem = []rune{}
	items = []string{}
	for _, c := range obj {
		err := curState(c)
		if err != nil {
			return nil, err
		}
	}
	curState('\000')
	if len(items) <= 1 {
		return nil, errors.New("Object is not in a=b,c=d,.. notation.")
	}
	res := make(map[string]interface{}, 0)
	for i := 0; i < len(items); i += 2 {
		key := normalizePropertyName(items[i])
		if i == len(items)-1 {
			res[key] = nil
		} else {
			res[key] = items[i+1]
		}
	}
	return res, nil
}

func saveCurItem() {
	items = append(items, string(curItem))
	curItem = []rune{}
}

func startParseKey(r rune) error {
	switch r {
	case '\'', '"':
		curQuote = r
		curState = parseQuotedKey
	default:
		curItem = append(curItem, r)
		curState = parseSimpleKey
	}
	return nil
}

func parseSimpleKey(r rune) error {
	switch r {
	case '=':
		curState = startParseValue
		saveCurItem()
	case '\000':
		saveCurItem()
	default:
		curItem = append(curItem, r)
	}
	return nil
}

func parseQuotedKey(r rune) error {
	switch r {
	case curQuote:
		curState = keyParsed
		saveCurItem()
	case '\000':
		return errors.New("")
	default:
		curItem = append(curItem, r)
	}
	return nil
}

func keyParsed(r rune) error {
	switch r {
	case '=':
		curState = startParseValue
		return nil
	case '\000':
		return nil
	default:
		return errors.New("")

	}
}

func startParseValue(r rune) error {
	switch r {
	case '\'', '"':
		curQuote = r
		curState = parseQuotedValue
	default:
		curItem = append(curItem, r)
		curState = parseSimpleValue
	}
	return nil
}

func parseSimpleValue(r rune) error {
	switch r {
	case ',':
		curState = startParseKey
		saveCurItem()
	case '\000':
		saveCurItem()
	default:
		curItem = append(curItem, r)
	}
	return nil
}

func parseQuotedValue(r rune) error {
	switch r {
	case curQuote:
		curState = valueParsed
		saveCurItem()
	case '\000':
		return errors.New("")
	default:
		curItem = append(curItem, r)
	}
	return nil
}

func valueParsed(r rune) error {
	switch r {
	case ',':
		curState = startParseKey
		return nil
	case '\000':
		return nil
	default:
		return errors.New("")

	}
}
<|endoftext|>