<commit_before>/*
 *
 * k6 - a next-generation load testing tool
 * Copyright (C) 2019 Load Impact
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as
 * published by the Free Software Foundation, either version 3 of the
 * License, or (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Affero General Public License for more details.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 *
 */

package executor

import (
	"context"
	"fmt"
	"sync"
	"sync/atomic"
	"testing"
	"time"

	"github.com/sirupsen/logrus"
	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
	"gopkg.in/guregu/null.v3"

	"github.com/loadimpact/k6/lib"
	"github.com/loadimpact/k6/lib/types"
	"github.com/loadimpact/k6/stats"
)

func newExecutionSegmentFromString(str string) *lib.ExecutionSegment {
	r, err := lib.NewExecutionSegmentFromString(str)
	if err != nil {
		panic(err)
	}
	return r
}

func newExecutionSegmentSequenceFromString(str string) *lib.ExecutionSegmentSequence {
	r, err := lib.NewExecutionSegmentSequenceFromString(str)
	if err != nil {
		panic(err)
	}
	return &r
}

func getTestConstantArrivalRateConfig() ConstantArrivalRateConfig {
	return ConstantArrivalRateConfig{
		BaseConfig:      BaseConfig{GracefulStop: types.NullDurationFrom(1 * time.Second)},
		TimeUnit:        types.NullDurationFrom(time.Second),
		Rate:            null.IntFrom(50),
		Duration:        types.NullDurationFrom(5 * time.Second),
		PreAllocatedVUs: null.IntFrom(10),
		MaxVUs:          null.IntFrom(20),
	}
}

func TestConstantArrivalRateRunNotEnoughAllocatedVUsWarn(t *testing.T) {
	t.Parallel()
	et, err := lib.NewExecutionTuple(nil, nil)
	require.NoError(t, err)
	es := lib.NewExecutionState(lib.Options{}, et, 10, 50)
	ctx, cancel, executor, logHook := setupExecutor(
		t, getTestConstantArrivalRateConfig(), es,
		simpleRunner(func(ctx context.Context) error {
			time.Sleep(time.Second)
			return nil
		}),
	)
	defer cancel()
	engineOut := make(chan stats.SampleContainer, 1000)
	err = executor.Run(ctx, engineOut)
	require.NoError(t, err)
	entries := logHook.Drain()
	require.NotEmpty(t, entries)
	for _, entry := range entries {
		require.Equal(t,
			"Insufficient VUs, reached 20 active VUs and cannot allocate more",
			entry.Message)
		require.Equal(t, logrus.WarnLevel, entry.Level)
	}
}

func TestConstantArrivalRateRunCorrectRate(t *testing.T) {
	t.Parallel()
	var count int64
	et, err := lib.NewExecutionTuple(nil, nil)
	require.NoError(t, err)
	es := lib.NewExecutionState(lib.Options{}, et, 10, 50)
	ctx, cancel, executor, logHook := setupExecutor(
		t, getTestConstantArrivalRateConfig(), es,
		simpleRunner(func(ctx context.Context) error {
			atomic.AddInt64(&count, 1)
			return nil
		}),
	)
	defer cancel()
	var wg sync.WaitGroup
	wg.Add(1)
	go func() {
		defer wg.Done()
		// check that we got around the amount of VU iterations as we would expect
		var currentCount int64

		for i := 0; i < 5; i++ {
			time.Sleep(time.Second)
			currentCount = atomic.SwapInt64(&count, 0)
			require.InDelta(t, 50, currentCount, 1)
		}
	}()
	engineOut := make(chan stats.SampleContainer, 1000)
	err = executor.Run(ctx, engineOut)
	wg.Wait()
	require.NoError(t, err)
	require.Empty(t, logHook.Drain())
}

func TestConstantArrivalRateRunCorrectTiming(t *testing.T) {
	tests := []struct {
		segment  *lib.ExecutionSegment
		sequence *lib.ExecutionSegmentSequence
		start    time.Duration
		steps    []int64
	}{
		{
			segment: newExecutionSegmentFromString("0:1/3"),
			start:   time.Millisecond * 20,
			steps:   []int64{40, 60, 60, 60, 60, 60, 60},
		},
		{
			segment: newExecutionSegmentFromString("1/3:2/3"),
			start:   time.Millisecond * 20,
			steps:   []int64{60, 60, 60, 60, 60, 60, 40},
		},
		{
			segment: newExecutionSegmentFromString("2/3:1"),
			start:   time.Millisecond * 20,
			steps:   []int64{40, 60, 60, 60, 60, 60, 60},
		},
		{
			segment: newExecutionSegmentFromString("1/6:3/6"),
			start:   time.Millisecond * 20,
			steps:   []int64{40, 80, 40, 80, 40, 80, 40},
		},
		{
			segment:  newExecutionSegmentFromString("1/6:3/6"),
			sequence: newExecutionSegmentSequenceFromString("1/6,3/6"),
			start:    time.Millisecond * 20,
			steps:    []int64{40, 80, 40, 80, 40, 80, 40},
		},
		// sequences
		{
			segment:  newExecutionSegmentFromString("0:1/3"),
			sequence: newExecutionSegmentSequenceFromString("0,1/3,2/3,1"),
			start:    time.Millisecond * 00,
			steps:    []int64{60, 60, 60, 60, 60, 60, 40},
		},
		{
			segment:  newExecutionSegmentFromString("1/3:2/3"),
			sequence: newExecutionSegmentSequenceFromString("0,1/3,2/3,1"),
			start:    time.Millisecond * 20,
			steps:    []int64{60, 60, 60, 60, 60, 60, 40},
		},
		{
			segment:  newExecutionSegmentFromString("2/3:1"),
			sequence: newExecutionSegmentSequenceFromString("0,1/3,2/3,1"),
			start:    time.Millisecond * 40,
			steps:    []int64{60, 60, 60, 60, 60, 100},
		},
	}
	for _, test := range tests {
		test := test

		t.Run(fmt.Sprintf("segment %s sequence %s", test.segment, test.sequence), func(t *testing.T) {
			t.Parallel()
			et, err := lib.NewExecutionTuple(test.segment, test.sequence)
			require.NoError(t, err)
			es := lib.NewExecutionState(lib.Options{
				ExecutionSegment:         test.segment,
				ExecutionSegmentSequence: test.sequence,
			}, et, 10, 50)
			var count int64
			config := getTestConstantArrivalRateConfig()
			newET, err := es.ExecutionTuple.GetNewExecutionTupleFromValue(config.MaxVUs.Int64)
			require.NoError(t, err)
			rateScaled := newET.ScaleInt64(config.Rate.Int64)
			startTime := time.Now()
			expectedTimeInt64 := int64(test.start)
			ctx, cancel, executor, logHook := setupExecutor(
				t, config, es,
				simpleRunner(func(ctx context.Context) error {
					current := atomic.AddInt64(&count, 1)

					expectedTime := test.start
					if current != 1 {
						expectedTime = time.Duration(atomic.AddInt64(&expectedTimeInt64,
							int64(time.Millisecond)*test.steps[(current-2)%int64(len(test.steps))]))
					}
					assert.WithinDuration(t,
						startTime.Add(expectedTime),
						time.Now(),
						time.Millisecond*10,
						"%d expectedTime %s", current, expectedTime,
					)

					return nil
				}),
			)

			defer cancel()
			var wg sync.WaitGroup
			wg.Add(1)
			go func() {
				defer wg.Done()
				// check that we got around the amount of VU iterations as we would expect
				var currentCount int64

				for i := 0; i < 5; i++ {
					time.Sleep(time.Second)
					currentCount = atomic.LoadInt64(&count)
					assert.InDelta(t, int64(i+1)*rateScaled, currentCount, 3)
				}
			}()
			startTime = time.Now()
			engineOut := make(chan stats.SampleContainer, 1000)
			err = executor.Run(ctx, engineOut)
			wg.Wait()
			require.NoError(t, err)
			require.Empty(t, logHook.Drain())
		})
	}
}

func TestArrivalRateCancel(t *testing.T) {
	t.Parallel()

	testCases := map[string]lib.ExecutorConfig{
		"constant": getTestConstantArrivalRateConfig(),
		"variable": getTestVariableArrivalRateConfig(),
	}
	for name, config := range testCases {
		config := config
		t.Run(name, func(t *testing.T) {
			t.Parallel()
			ch := make(chan struct{})
			errCh := make(chan error, 1)
			weAreDoneCh := make(chan struct{})
			et, err := lib.NewExecutionTuple(nil, nil)
			require.NoError(t, err)
			es := lib.NewExecutionState(lib.Options{}, et, 10, 50)
			ctx, cancel, executor, logHook := setupExecutor(
				t, config, es, simpleRunner(func(ctx context.Context) error {
					select {
					case <-ch:
						<-ch
					default:
					}
					return nil
				}))
			defer cancel()
			var wg sync.WaitGroup
			wg.Add(1)
			go func() {
				defer wg.Done()

				engineOut := make(chan stats.SampleContainer, 1000)
				errCh <- executor.Run(ctx, engineOut)
				close(weAreDoneCh)
			}()

			time.Sleep(time.Second)
			ch <- struct{}{}
			cancel()
			time.Sleep(time.Second)
			select {
			case <-weAreDoneCh:
				t.Fatal("Run returned before all VU iterations were finished")
			default:
			}
			close(ch)
			<-weAreDoneCh
			wg.Wait()
			require.NoError(t, <-errCh)
			require.Empty(t, logHook.Drain())
		})
	}
}
<commit_msg>Make TestConstantArrivalRateRunCorrectTiming faster and stabler<commit_after>/*
 *
 * k6 - a next-generation load testing tool
 * Copyright (C) 2019 Load Impact
 *
 * This program is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Affero General Public License as
 * published by the Free Software Foundation, either version 3 of the
 * License, or (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Affero General Public License for more details.
 *
 * You should have received a copy of the GNU Affero General Public License
 * along with this program.  If not, see <http://www.gnu.org/licenses/>.
 *
 */

package executor

import (
	"context"
	"fmt"
	"sync"
	"sync/atomic"
	"testing"
	"time"

	"github.com/sirupsen/logrus"
	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
	"gopkg.in/guregu/null.v3"

	"github.com/loadimpact/k6/lib"
	"github.com/loadimpact/k6/lib/types"
	"github.com/loadimpact/k6/stats"
)

func newExecutionSegmentFromString(str string) *lib.ExecutionSegment {
	r, err := lib.NewExecutionSegmentFromString(str)
	if err != nil {
		panic(err)
	}
	return r
}

func newExecutionSegmentSequenceFromString(str string) *lib.ExecutionSegmentSequence {
	r, err := lib.NewExecutionSegmentSequenceFromString(str)
	if err != nil {
		panic(err)
	}
	return &r
}

func getTestConstantArrivalRateConfig() ConstantArrivalRateConfig {
	return ConstantArrivalRateConfig{
		BaseConfig:      BaseConfig{GracefulStop: types.NullDurationFrom(1 * time.Second)},
		TimeUnit:        types.NullDurationFrom(time.Second),
		Rate:            null.IntFrom(50),
		Duration:        types.NullDurationFrom(5 * time.Second),
		PreAllocatedVUs: null.IntFrom(10),
		MaxVUs:          null.IntFrom(20),
	}
}

func TestConstantArrivalRateRunNotEnoughAllocatedVUsWarn(t *testing.T) {
	t.Parallel()
	et, err := lib.NewExecutionTuple(nil, nil)
	require.NoError(t, err)
	es := lib.NewExecutionState(lib.Options{}, et, 10, 50)
	ctx, cancel, executor, logHook := setupExecutor(
		t, getTestConstantArrivalRateConfig(), es,
		simpleRunner(func(ctx context.Context) error {
			time.Sleep(time.Second)
			return nil
		}),
	)
	defer cancel()
	engineOut := make(chan stats.SampleContainer, 1000)
	err = executor.Run(ctx, engineOut)
	require.NoError(t, err)
	entries := logHook.Drain()
	require.NotEmpty(t, entries)
	for _, entry := range entries {
		require.Equal(t,
			"Insufficient VUs, reached 20 active VUs and cannot allocate more",
			entry.Message)
		require.Equal(t, logrus.WarnLevel, entry.Level)
	}
}

func TestConstantArrivalRateRunCorrectRate(t *testing.T) {
	t.Parallel()
	var count int64
	et, err := lib.NewExecutionTuple(nil, nil)
	require.NoError(t, err)
	es := lib.NewExecutionState(lib.Options{}, et, 10, 50)
	ctx, cancel, executor, logHook := setupExecutor(
		t, getTestConstantArrivalRateConfig(), es,
		simpleRunner(func(ctx context.Context) error {
			atomic.AddInt64(&count, 1)
			return nil
		}),
	)
	defer cancel()
	var wg sync.WaitGroup
	wg.Add(1)
	go func() {
		defer wg.Done()
		// check that we got around the amount of VU iterations as we would expect
		var currentCount int64

		for i := 0; i < 5; i++ {
			time.Sleep(time.Second)
			currentCount = atomic.SwapInt64(&count, 0)
			require.InDelta(t, 50, currentCount, 1)
		}
	}()
	engineOut := make(chan stats.SampleContainer, 1000)
	err = executor.Run(ctx, engineOut)
	wg.Wait()
	require.NoError(t, err)
	require.Empty(t, logHook.Drain())
}

func TestConstantArrivalRateRunCorrectTiming(t *testing.T) {
	tests := []struct {
		segment  *lib.ExecutionSegment
		sequence *lib.ExecutionSegmentSequence
		start    time.Duration
		steps    []int64
	}{
		{
			segment: newExecutionSegmentFromString("0:1/3"),
			start:   time.Millisecond * 20,
			steps:   []int64{40, 60, 60, 60, 60, 60, 60},
		},
		{
			segment: newExecutionSegmentFromString("1/3:2/3"),
			start:   time.Millisecond * 20,
			steps:   []int64{60, 60, 60, 60, 60, 60, 40},
		},
		{
			segment: newExecutionSegmentFromString("2/3:1"),
			start:   time.Millisecond * 20,
			steps:   []int64{40, 60, 60, 60, 60, 60, 60},
		},
		{
			segment: newExecutionSegmentFromString("1/6:3/6"),
			start:   time.Millisecond * 20,
			steps:   []int64{40, 80, 40, 80, 40, 80, 40},
		},
		{
			segment:  newExecutionSegmentFromString("1/6:3/6"),
			sequence: newExecutionSegmentSequenceFromString("1/6,3/6"),
			start:    time.Millisecond * 20,
			steps:    []int64{40, 80, 40, 80, 40, 80, 40},
		},
		// sequences
		{
			segment:  newExecutionSegmentFromString("0:1/3"),
			sequence: newExecutionSegmentSequenceFromString("0,1/3,2/3,1"),
			start:    time.Millisecond * 00,
			steps:    []int64{60, 60, 60, 60, 60, 60, 40},
		},
		{
			segment:  newExecutionSegmentFromString("1/3:2/3"),
			sequence: newExecutionSegmentSequenceFromString("0,1/3,2/3,1"),
			start:    time.Millisecond * 20,
			steps:    []int64{60, 60, 60, 60, 60, 60, 40},
		},
		{
			segment:  newExecutionSegmentFromString("2/3:1"),
			sequence: newExecutionSegmentSequenceFromString("0,1/3,2/3,1"),
			start:    time.Millisecond * 40,
			steps:    []int64{60, 60, 60, 60, 60, 100},
		},
	}
	for _, test := range tests {
		test := test

		t.Run(fmt.Sprintf("segment %s sequence %s", test.segment, test.sequence), func(t *testing.T) {
			t.Parallel()
			et, err := lib.NewExecutionTuple(test.segment, test.sequence)
			require.NoError(t, err)
			es := lib.NewExecutionState(lib.Options{
				ExecutionSegment:         test.segment,
				ExecutionSegmentSequence: test.sequence,
			}, et, 10, 50)
			var count int64
			config := getTestConstantArrivalRateConfig()
			config.Duration.Duration = types.Duration(time.Second * 3)
			newET, err := es.ExecutionTuple.GetNewExecutionTupleFromValue(config.MaxVUs.Int64)
			require.NoError(t, err)
			rateScaled := newET.ScaleInt64(config.Rate.Int64)
			startTime := time.Now()
			expectedTimeInt64 := int64(test.start)
			ctx, cancel, executor, logHook := setupExecutor(
				t, config, es,
				simpleRunner(func(ctx context.Context) error {
					current := atomic.AddInt64(&count, 1)

					expectedTime := test.start
					if current != 1 {
						expectedTime = time.Duration(atomic.AddInt64(&expectedTimeInt64,
							int64(time.Millisecond)*test.steps[(current-2)%int64(len(test.steps))]))
					}
					assert.WithinDuration(t,
						startTime.Add(expectedTime),
						time.Now(),
						time.Millisecond*10,
						"%d expectedTime %s", current, expectedTime,
					)

					return nil
				}),
			)

			defer cancel()
			var wg sync.WaitGroup
			wg.Add(1)
			go func() {
				defer wg.Done()
				// check that we got around the amount of VU iterations as we would expect
				var currentCount int64

				for i := 0; i < 3; i++ {
					time.Sleep(time.Second)
					currentCount = atomic.LoadInt64(&count)
					assert.InDelta(t, int64(i+1)*rateScaled, currentCount, 3)
				}
			}()
			startTime = time.Now()
			engineOut := make(chan stats.SampleContainer, 1000)
			err = executor.Run(ctx, engineOut)
			wg.Wait()
			require.NoError(t, err)
			require.Empty(t, logHook.Drain())
		})
	}
}

func TestArrivalRateCancel(t *testing.T) {
	t.Parallel()

	testCases := map[string]lib.ExecutorConfig{
		"constant": getTestConstantArrivalRateConfig(),
		"variable": getTestVariableArrivalRateConfig(),
	}
	for name, config := range testCases {
		config := config
		t.Run(name, func(t *testing.T) {
			t.Parallel()
			ch := make(chan struct{})
			errCh := make(chan error, 1)
			weAreDoneCh := make(chan struct{})
			et, err := lib.NewExecutionTuple(nil, nil)
			require.NoError(t, err)
			es := lib.NewExecutionState(lib.Options{}, et, 10, 50)
			ctx, cancel, executor, logHook := setupExecutor(
				t, config, es, simpleRunner(func(ctx context.Context) error {
					select {
					case <-ch:
						<-ch
					default:
					}
					return nil
				}))
			defer cancel()
			var wg sync.WaitGroup
			wg.Add(1)
			go func() {
				defer wg.Done()

				engineOut := make(chan stats.SampleContainer, 1000)
				errCh <- executor.Run(ctx, engineOut)
				close(weAreDoneCh)
			}()

			time.Sleep(time.Second)
			ch <- struct{}{}
			cancel()
			time.Sleep(time.Second)
			select {
			case <-weAreDoneCh:
				t.Fatal("Run returned before all VU iterations were finished")
			default:
			}
			close(ch)
			<-weAreDoneCh
			wg.Wait()
			require.NoError(t, <-errCh)
			require.Empty(t, logHook.Drain())
		})
	}
}
<|endoftext|>


<commit_before>package fscommon

import (
	"os"
	"strings"
	"sync"

	securejoin "github.com/cyphar/filepath-securejoin"
	"github.com/pkg/errors"
	"github.com/sirupsen/logrus"
	"golang.org/x/sys/unix"
)

const (
	cgroupfsDir    = "/sys/fs/cgroup"
	cgroupfsPrefix = cgroupfsDir + "/"
)

var (
	// Set to true by fs unit tests
	TestMode bool

	cgroupFd     int = -1
	prepOnce     sync.Once
	prepErr      error
	resolveFlags uint64
)

func prepareOpenat2() error {
	prepOnce.Do(func() {
		fd, err := unix.Openat2(-1, cgroupfsDir, &unix.OpenHow{
			Flags: unix.O_DIRECTORY | unix.O_PATH})
		if err != nil {
			prepErr = &os.PathError{Op: "openat2", Path: cgroupfsDir, Err: err}
			if err != unix.ENOSYS {
				logrus.Warnf("falling back to securejoin: %s", prepErr)
			} else {
				logrus.Debug("openat2 not available, falling back to securejoin")
			}
			return
		}
		var st unix.Statfs_t
		if err = unix.Fstatfs(fd, &st); err != nil {
			prepErr = &os.PathError{Op: "statfs", Path: cgroupfsDir, Err: err}
			logrus.Warnf("falling back to securejoin: %s", prepErr)
			return
		}

		cgroupFd = fd

		resolveFlags = unix.RESOLVE_BENEATH | unix.RESOLVE_NO_MAGICLINKS
		if st.Type == unix.CGROUP2_SUPER_MAGIC {
			// cgroupv2 has a single mountpoint and no "cpu,cpuacct" symlinks
			resolveFlags |= unix.RESOLVE_NO_XDEV | unix.RESOLVE_NO_SYMLINKS
		}

	})

	return prepErr
}

// OpenFile opens a cgroup file in a given dir with given flags.
// It is supposed to be used for cgroup files only.
func OpenFile(dir, file string, flags int) (*os.File, error) {
	if dir == "" {
		return nil, errors.Errorf("no directory specified for %s", file)
	}
	mode := os.FileMode(0)
	if TestMode && flags&os.O_WRONLY != 0 {
		// "emulate" cgroup fs for unit tests
		flags |= os.O_TRUNC | os.O_CREATE
		mode = 0o600
	}
	reldir := strings.TrimPrefix(dir, cgroupfsPrefix)
	if len(reldir) == len(dir) { // non-standard path, old system?
		return openWithSecureJoin(dir, file, flags, mode)
	}
	if prepareOpenat2() != nil {
		return openWithSecureJoin(dir, file, flags, mode)
	}

	relname := reldir + "/" + file
	fd, err := unix.Openat2(cgroupFd, relname,
		&unix.OpenHow{
			Resolve: resolveFlags,
			Flags:   uint64(flags) | unix.O_CLOEXEC,
			Mode:    uint64(mode),
		})
	if err != nil {
		return nil, &os.PathError{Op: "openat2", Path: dir + "/" + file, Err: err}
	}

	return os.NewFile(uintptr(fd), cgroupfsPrefix+relname), nil
}

func openWithSecureJoin(dir, file string, flags int, mode os.FileMode) (*os.File, error) {
	path, err := securejoin.SecureJoin(dir, file)
	if err != nil {
		return nil, err
	}

	return os.OpenFile(path, flags, mode)
}
<commit_msg>libct/cg/fscommon.OpenFile: reverse checks order<commit_after>package fscommon

import (
	"os"
	"strings"
	"sync"

	securejoin "github.com/cyphar/filepath-securejoin"
	"github.com/pkg/errors"
	"github.com/sirupsen/logrus"
	"golang.org/x/sys/unix"
)

const (
	cgroupfsDir    = "/sys/fs/cgroup"
	cgroupfsPrefix = cgroupfsDir + "/"
)

var (
	// Set to true by fs unit tests
	TestMode bool

	cgroupFd     int = -1
	prepOnce     sync.Once
	prepErr      error
	resolveFlags uint64
)

func prepareOpenat2() error {
	prepOnce.Do(func() {
		fd, err := unix.Openat2(-1, cgroupfsDir, &unix.OpenHow{
			Flags: unix.O_DIRECTORY | unix.O_PATH})
		if err != nil {
			prepErr = &os.PathError{Op: "openat2", Path: cgroupfsDir, Err: err}
			if err != unix.ENOSYS {
				logrus.Warnf("falling back to securejoin: %s", prepErr)
			} else {
				logrus.Debug("openat2 not available, falling back to securejoin")
			}
			return
		}
		var st unix.Statfs_t
		if err = unix.Fstatfs(fd, &st); err != nil {
			prepErr = &os.PathError{Op: "statfs", Path: cgroupfsDir, Err: err}
			logrus.Warnf("falling back to securejoin: %s", prepErr)
			return
		}

		cgroupFd = fd

		resolveFlags = unix.RESOLVE_BENEATH | unix.RESOLVE_NO_MAGICLINKS
		if st.Type == unix.CGROUP2_SUPER_MAGIC {
			// cgroupv2 has a single mountpoint and no "cpu,cpuacct" symlinks
			resolveFlags |= unix.RESOLVE_NO_XDEV | unix.RESOLVE_NO_SYMLINKS
		}

	})

	return prepErr
}

// OpenFile opens a cgroup file in a given dir with given flags.
// It is supposed to be used for cgroup files only.
func OpenFile(dir, file string, flags int) (*os.File, error) {
	if dir == "" {
		return nil, errors.Errorf("no directory specified for %s", file)
	}
	mode := os.FileMode(0)
	if TestMode && flags&os.O_WRONLY != 0 {
		// "emulate" cgroup fs for unit tests
		flags |= os.O_TRUNC | os.O_CREATE
		mode = 0o600
	}
	if prepareOpenat2() != nil {
		return openWithSecureJoin(dir, file, flags, mode)
	}
	reldir := strings.TrimPrefix(dir, cgroupfsPrefix)
	if len(reldir) == len(dir) { // non-standard path, old system?
		return openWithSecureJoin(dir, file, flags, mode)
	}

	relname := reldir + "/" + file
	fd, err := unix.Openat2(cgroupFd, relname,
		&unix.OpenHow{
			Resolve: resolveFlags,
			Flags:   uint64(flags) | unix.O_CLOEXEC,
			Mode:    uint64(mode),
		})
	if err != nil {
		return nil, &os.PathError{Op: "openat2", Path: dir + "/" + file, Err: err}
	}

	return os.NewFile(uintptr(fd), cgroupfsPrefix+relname), nil
}

func openWithSecureJoin(dir, file string, flags int, mode os.FileMode) (*os.File, error) {
	path, err := securejoin.SecureJoin(dir, file)
	if err != nil {
		return nil, err
	}

	return os.OpenFile(path, flags, mode)
}
<|endoftext|>


<commit_before>package router

import (
	"math"
	"time"
)

const (
	EthernetOverhead    = 14
	UDPOverhead         = 28 // 20 bytes for IPv4, 8 bytes for UDP
	Port                = 6783
	HTTPPort            = Port + 1
	DefaultPMTU         = 65535
	MaxUDPPacketSize    = 65536
	ChannelSize         = 16
	FragTestSize        = 60001
	PMTUDiscoverySize   = 60000
	TCPHeartbeat        = 30 * time.Second
	FastHeartbeat       = 500 * time.Millisecond
	SlowHeartbeat       = 10 * time.Second
	FragTestInterval    = 5 * time.Minute
	GossipInterval      = 30 * time.Second
	PMTUVerifyAttempts  = 8
	PMTUVerifyTimeout   = 10 * time.Millisecond // gets doubled with every attempt
	MaxDuration         = time.Duration(math.MaxInt64)
	MaxMissedHeartbeats = 6
	HeaderTimeout       = 10 * time.Second
	HeartbeatTimeout    = MaxMissedHeartbeats * SlowHeartbeat
)

var (
	FragTest      = make([]byte, FragTestSize)
	PMTUDiscovery = make([]byte, PMTUDiscoverySize)
)
<commit_msg>Correct MaxUDPPacketSize constant<commit_after>package router

import (
	"math"
	"time"
)

const (
	EthernetOverhead    = 14
	UDPOverhead         = 28 // 20 bytes for IPv4, 8 bytes for UDP
	Port                = 6783
	HTTPPort            = Port + 1
	DefaultPMTU         = 65535
	MaxUDPPacketSize    = 65535
	ChannelSize         = 16
	FragTestSize        = 60001
	PMTUDiscoverySize   = 60000
	TCPHeartbeat        = 30 * time.Second
	FastHeartbeat       = 500 * time.Millisecond
	SlowHeartbeat       = 10 * time.Second
	FragTestInterval    = 5 * time.Minute
	GossipInterval      = 30 * time.Second
	PMTUVerifyAttempts  = 8
	PMTUVerifyTimeout   = 10 * time.Millisecond // gets doubled with every attempt
	MaxDuration         = time.Duration(math.MaxInt64)
	MaxMissedHeartbeats = 6
	HeaderTimeout       = 10 * time.Second
	HeartbeatTimeout    = MaxMissedHeartbeats * SlowHeartbeat
)

var (
	FragTest      = make([]byte, FragTestSize)
	PMTUDiscovery = make([]byte, PMTUDiscoverySize)
)
<|endoftext|>


<commit_before>package main

import (
	"html/template"
	"net/http"
	"os"
	"path"
	"strconv"
	"strings"
	"time"

	"github.com/Zamiell/isaac-racing-server/src/log"
	"github.com/Zamiell/isaac-racing-server/src/models"
	"github.com/didip/tollbooth"
	"github.com/didip/tollbooth_gin"
	"github.com/gin-contrib/sessions"
	"github.com/gin-gonic/gin"
)

const (
	sessionName = "isaac.sid"
)

var (
	sessionStore sessions.CookieStore
	GATrackingID string
	myHTTPClient = &http.Client{ // We don't want to use the default http.Client structure because it has no default timeout set
		Timeout: 10 * time.Second,
	}
)

/*
	Data structures
*/

type TemplateData struct {
	Title string

	// Races stuff
	RaceResults          []models.RaceHistory
	ResultsRaces         []models.RaceHistory
	TotalRaceCount       int
	TotalPages           int
	PreviousPage         int
	NextPage             int
	RaceResultsRanked    []models.RaceHistory
	RaceResultsAll       []models.RaceHistory

	// Profiles/profile stuff
	ResultsProfiles   []models.ProfilesRow
	ResultsProfile    models.ProfileData
	TotalProfileCount int
	UsersPerPage      int

	// Leaderboard stuff
	LeaderboardUnseeded  []models.LeaderboardRowUnseeded
	LeaderboardDiversity []models.LeaderboardRowDiversity
	//LeaderboardSeeded []models.LeaderboardRowSeeded
}

/*
	Initialization function
*/

func httpInit() {
	// Create a new Gin HTTP router
	gin.SetMode(gin.ReleaseMode) // Comment this out to debug HTTP stuff
	httpRouter := gin.Default()

	// Read some HTTP server configuration values from environment variables
	// (they were loaded from the .env file in main.go)
	sessionSecret := os.Getenv("SESSION_SECRET")
	if len(sessionSecret) == 0 {
		log.Info("The \"SESSION_SECRET\" environment variable is blank; aborting HTTP initalization.")
		return
	}
	domain := os.Getenv("DOMAIN")
	if len(domain) == 0 {
		log.Info("The \"DOMAIN\" environment variable is blank; aborting HTTP initalization.")
		return
	}
	tlsCertFile := os.Getenv("TLS_CERT_FILE")
	tlsKeyFile := os.Getenv("TLS_KEY_FILE")
	useTLS := true
	if len(tlsCertFile) == 0 || len(tlsKeyFile) == 0 {
		useTLS = false
	}

	// Create a session store
	sessionStore = sessions.NewCookieStore([]byte(sessionSecret))
	options := sessions.Options{
		Path:   "/",
		Domain: domain,
		MaxAge: 5, // 5 seconds
		// After getting a cookie via "/login", the client will immediately
		// establish a WebSocket connection via "/ws", so the cookie only needs
		// to exist for that time frame
		Secure: true,
		// Only send the cookie over HTTPS:
		// https://www.owasp.org/index.php/Testing_for_cookies_attributes_(OTG-SESS-002)
		HttpOnly: true,
		// Mitigate XSS attacks:
		// https://www.owasp.org/index.php/HttpOnly
	}
	if !useTLS {
		options.Secure = false
	}
	sessionStore.Options(options)
	httpRouter.Use(sessions.Sessions(sessionName, sessionStore))

	// Use the Tollbooth Gin middleware for Google Analytics tracking
	limiter := tollbooth.NewLimiter(1, time.Second, nil) // Limit each user to 1 request per second
	// When a user requests "/", they will also request the CSS and images;
	// this middleware is smart enough to know that it is considered part of the first request
	// However, it is still not possible to spam download CSS or image files
	limiterMiddleware := tollbooth_gin.LimitHandler(limiter)
	httpRouter.Use(limiterMiddleware)

	/*
		This was used as an alterate to the Tollbooth middleware when it wasn't working

		// Use the gin-limiter middleware for rate-limiting
		// We only allow 60 request per minute, an average of 1 per second
		// This is because when a user requests "/", they will also request the CSS and images
		// Based on: https://github.com/julianshen/gin-limiter/blob/master/example/web.go
		limiterMiddleware := limiter.NewRateLimiter(time.Second*60, 60, func(c *gin.Context) (string, error) {
			// Local variables
			r := c.Request
			ip, _, _ := net.SplitHostPort(r.RemoteAddr)

			// Just use the IP address as the key
			return ip, nil
		}).Middleware()
		httpRouter.Use(limiterMiddleware)
	*/

	// Use a custom middleware for Google Analytics tracking
	GATrackingID = os.Getenv("GA_TRACKING_ID")
	if len(GATrackingID) != 0 {
		httpRouter.Use(httpMwGoogleAnalytics)
	}

	// Path handlers (for the WebSocket server)
	httpRouter.POST("/login", httpLogin)
	httpRouter.POST("/register", httpRegister)
	httpRouter.GET("/ws", httpWS)

	// Path handlers (for the website)
	httpRouter.GET("/", httpHome)

	// Path handlers for single profile
	httpRouter.GET("/profile", httpProfile)
	httpRouter.GET("/profile/:player", httpProfile) // Handles profile username

	// Path handlers for all profiles
	httpRouter.GET("/profiles", httpProfiles)
	httpRouter.GET("/profiles/:page", httpProfiles) // Handles extra pages for profiles

	// Path handlers for race page
	httpRouter.GET("/race", httpRace)
	httpRouter.GET("/race/:raceid", httpRace)

	// Path handlers for races page
	httpRouter.GET("/races", httpRaces)
	httpRouter.GET("/races/:page", httpRaces)

	httpRouter.GET("/leaderboards", httpLeaderboards)
	httpRouter.GET("/info", httpInfo)
	httpRouter.GET("/download", httpDownload)
	httpRouter.Static("/public", "../public")

	// Figure out the port that we are using for the HTTP server
	var port int
	if useTLS {
		// We want all HTTP requests to be redirected to HTTPS
		// (but make an exception for Let's Encrypt)
		// The Gin router is using the default serve mux, so we need to create a
		// new fresh one for the HTTP handler
		HTTPServeMux := http.NewServeMux()
		HTTPServeMux.Handle("/.well-known/acme-challenge/", http.FileServer(http.FileSystem(http.Dir("../letsencrypt"))))
		HTTPServeMux.Handle("/", http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
			http.Redirect(w, req, "https://"+req.Host+req.URL.String(), http.StatusMovedPermanently)
		}))

		// ListenAndServe is blocking, so start listening on a new goroutine
		go func() {
			http.ListenAndServe(":80", HTTPServeMux) // Nothing before the colon implies 0.0.0.0
			log.Fatal("http.ListenAndServe ended for port 80.", nil)
		}()

		// 443 is the default port for HTTPS
		port = 443
	} else {
		// 80 is the defeault port for HTTP
		port = 80
	}

	// Start listening and serving requests (which is blocking)
	log.Info("Listening on port " + strconv.Itoa(port) + ".")
	if useTLS {
		if err := http.ListenAndServeTLS(
			":"+strconv.Itoa(port), // Nothing before the colon implies 0.0.0.0
			tlsCertFile,
			tlsKeyFile,
			httpRouter,
		); err != nil {
			log.Fatal("http.ListenAndServeTLS failed:", err)
		}
		log.Fatal("http.ListenAndServeTLS ended prematurely.", nil)
	} else {
		// Listen and serve (HTTP)
		if err := http.ListenAndServe(
			":"+strconv.Itoa(port), // Nothing before the colon implies 0.0.0.0
			httpRouter,
		); err != nil {
			log.Fatal("http.ListenAndServe failed:", err)
		}
		log.Fatal("http.ListenAndServe ended prematurely.", nil)
	}
}

/*
	HTTP miscellaneous subroutines
*/

func httpServeTemplate(w http.ResponseWriter, templateName string, data interface{}) {
	lp := path.Join("views", "layout.tmpl")
	fp := path.Join("views", templateName+".tmpl")

	// Return a 404 if the template doesn't exist
	info, err := os.Stat(fp)
	if err != nil {
		if os.IsNotExist(err) {
			http.Error(w, http.StatusText(http.StatusNotFound), http.StatusNotFound)
			return
		}
	}

	// Return a 404 if the request is for a directory
	if info.IsDir() {
		http.Error(w, http.StatusText(http.StatusNotFound), http.StatusNotFound)
		return
	}

	// Create the template
	tmpl, err := template.ParseFiles(lp, fp)
	if err != nil {
		log.Error("Failed to create the template: " + err.Error())
		http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
		return
	}

	// Execute the template and send it to the user
	if err := tmpl.ExecuteTemplate(w, "layout", data); err != nil {
		if strings.HasSuffix(err.Error(), ": write: broken pipe") ||
			strings.HasSuffix(err.Error(), ": client disconnected") ||
			strings.HasSuffix(err.Error(), ": http2: stream closed") ||
			strings.HasSuffix(err.Error(), ": write: connection timed out") {

			// Broken pipe errors can occur when the user presses the "Stop" button while the template is executing
			// We don't want to reporting these errors to Sentry
			// https://stackoverflow.com/questions/26853200/filter-out-broken-pipe-errors-from-template-execution
			// I don't know exactly what the other errors mean
			log.Info("Ordinary error when executing the template: " + err.Error())
		} else {
			log.Error("Failed to execute the template: " + err.Error())
		}
		http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
	}
}
<commit_msg>fixing tollbooth<commit_after>package main

import (
	"html/template"
	"net/http"
	"os"
	"path"
	"strconv"
	"strings"
	"time"

	"github.com/Zamiell/isaac-racing-server/src/log"
	"github.com/Zamiell/isaac-racing-server/src/models"
	"github.com/didip/tollbooth"
	"github.com/didip/tollbooth_gin"
	"github.com/gin-contrib/sessions"
	"github.com/gin-gonic/gin"
)

const (
	sessionName = "isaac.sid"
)

var (
	sessionStore sessions.CookieStore
	GATrackingID string
	myHTTPClient = &http.Client{ // We don't want to use the default http.Client structure because it has no default timeout set
		Timeout: 10 * time.Second,
	}
)

/*
	Data structures
*/

type TemplateData struct {
	Title string

	// Races stuff
	RaceResults          []models.RaceHistory
	ResultsRaces         []models.RaceHistory
	TotalRaceCount       int
	TotalPages           int
	PreviousPage         int
	NextPage             int
	RaceResultsRanked    []models.RaceHistory
	RaceResultsAll       []models.RaceHistory

	// Profiles/profile stuff
	ResultsProfiles   []models.ProfilesRow
	ResultsProfile    models.ProfileData
	TotalProfileCount int
	UsersPerPage      int

	// Leaderboard stuff
	LeaderboardUnseeded  []models.LeaderboardRowUnseeded
	LeaderboardDiversity []models.LeaderboardRowDiversity
	//LeaderboardSeeded []models.LeaderboardRowSeeded
}

/*
	Initialization function
*/

func httpInit() {
	// Create a new Gin HTTP router
	gin.SetMode(gin.ReleaseMode) // Comment this out to debug HTTP stuff
	httpRouter := gin.Default()

	// Read some HTTP server configuration values from environment variables
	// (they were loaded from the .env file in main.go)
	sessionSecret := os.Getenv("SESSION_SECRET")
	if len(sessionSecret) == 0 {
		log.Info("The \"SESSION_SECRET\" environment variable is blank; aborting HTTP initalization.")
		return
	}
	domain := os.Getenv("DOMAIN")
	if len(domain) == 0 {
		log.Info("The \"DOMAIN\" environment variable is blank; aborting HTTP initalization.")
		return
	}
	tlsCertFile := os.Getenv("TLS_CERT_FILE")
	tlsKeyFile := os.Getenv("TLS_KEY_FILE")
	useTLS := true
	if len(tlsCertFile) == 0 || len(tlsKeyFile) == 0 {
		useTLS = false
	}

	// Create a session store
	sessionStore = sessions.NewCookieStore([]byte(sessionSecret))
	options := sessions.Options{
		Path:   "/",
		Domain: domain,
		MaxAge: 5, // 5 seconds
		// After getting a cookie via "/login", the client will immediately
		// establish a WebSocket connection via "/ws", so the cookie only needs
		// to exist for that time frame
		Secure: true,
		// Only send the cookie over HTTPS:
		// https://www.owasp.org/index.php/Testing_for_cookies_attributes_(OTG-SESS-002)
		HttpOnly: true,
		// Mitigate XSS attacks:
		// https://www.owasp.org/index.php/HttpOnly
	}
	if !useTLS {
		options.Secure = false
	}
	sessionStore.Options(options)
	httpRouter.Use(sessions.Sessions(sessionName, sessionStore))

	// Use the Tollbooth Gin middleware for Google Analytics tracking
	limiter := tollbooth.NewLimiter(1, nil) // Limit each user to 1 request per second
	// When a user requests "/", they will also request the CSS and images;
	// this middleware is smart enough to know that it is considered part of the first request
	// However, it is still not possible to spam download CSS or image files
	limiterMiddleware := tollbooth_gin.LimitHandler(limiter)
	httpRouter.Use(limiterMiddleware)

	/*
		This was used as an alterate to the Tollbooth middleware when it wasn't working

		// Use the gin-limiter middleware for rate-limiting
		// We only allow 60 request per minute, an average of 1 per second
		// This is because when a user requests "/", they will also request the CSS and images
		// Based on: https://github.com/julianshen/gin-limiter/blob/master/example/web.go
		limiterMiddleware := limiter.NewRateLimiter(time.Second*60, 60, func(c *gin.Context) (string, error) {
			// Local variables
			r := c.Request
			ip, _, _ := net.SplitHostPort(r.RemoteAddr)

			// Just use the IP address as the key
			return ip, nil
		}).Middleware()
		httpRouter.Use(limiterMiddleware)
	*/

	// Use a custom middleware for Google Analytics tracking
	GATrackingID = os.Getenv("GA_TRACKING_ID")
	if len(GATrackingID) != 0 {
		httpRouter.Use(httpMwGoogleAnalytics)
	}

	// Path handlers (for the WebSocket server)
	httpRouter.POST("/login", httpLogin)
	httpRouter.POST("/register", httpRegister)
	httpRouter.GET("/ws", httpWS)

	// Path handlers (for the website)
	httpRouter.GET("/", httpHome)

	// Path handlers for single profile
	httpRouter.GET("/profile", httpProfile)
	httpRouter.GET("/profile/:player", httpProfile) // Handles profile username

	// Path handlers for all profiles
	httpRouter.GET("/profiles", httpProfiles)
	httpRouter.GET("/profiles/:page", httpProfiles) // Handles extra pages for profiles

	// Path handlers for race page
	httpRouter.GET("/race", httpRace)
	httpRouter.GET("/race/:raceid", httpRace)

	// Path handlers for races page
	httpRouter.GET("/races", httpRaces)
	httpRouter.GET("/races/:page", httpRaces)

	httpRouter.GET("/leaderboards", httpLeaderboards)
	httpRouter.GET("/info", httpInfo)
	httpRouter.GET("/download", httpDownload)
	httpRouter.Static("/public", "../public")

	// Figure out the port that we are using for the HTTP server
	var port int
	if useTLS {
		// We want all HTTP requests to be redirected to HTTPS
		// (but make an exception for Let's Encrypt)
		// The Gin router is using the default serve mux, so we need to create a
		// new fresh one for the HTTP handler
		HTTPServeMux := http.NewServeMux()
		HTTPServeMux.Handle("/.well-known/acme-challenge/", http.FileServer(http.FileSystem(http.Dir("../letsencrypt"))))
		HTTPServeMux.Handle("/", http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
			http.Redirect(w, req, "https://"+req.Host+req.URL.String(), http.StatusMovedPermanently)
		}))

		// ListenAndServe is blocking, so start listening on a new goroutine
		go func() {
			http.ListenAndServe(":80", HTTPServeMux) // Nothing before the colon implies 0.0.0.0
			log.Fatal("http.ListenAndServe ended for port 80.", nil)
		}()

		// 443 is the default port for HTTPS
		port = 443
	} else {
		// 80 is the defeault port for HTTP
		port = 80
	}

	// Start listening and serving requests (which is blocking)
	log.Info("Listening on port " + strconv.Itoa(port) + ".")
	if useTLS {
		if err := http.ListenAndServeTLS(
			":"+strconv.Itoa(port), // Nothing before the colon implies 0.0.0.0
			tlsCertFile,
			tlsKeyFile,
			httpRouter,
		); err != nil {
			log.Fatal("http.ListenAndServeTLS failed:", err)
		}
		log.Fatal("http.ListenAndServeTLS ended prematurely.", nil)
	} else {
		// Listen and serve (HTTP)
		if err := http.ListenAndServe(
			":"+strconv.Itoa(port), // Nothing before the colon implies 0.0.0.0
			httpRouter,
		); err != nil {
			log.Fatal("http.ListenAndServe failed:", err)
		}
		log.Fatal("http.ListenAndServe ended prematurely.", nil)
	}
}

/*
	HTTP miscellaneous subroutines
*/

func httpServeTemplate(w http.ResponseWriter, templateName string, data interface{}) {
	lp := path.Join("views", "layout.tmpl")
	fp := path.Join("views", templateName+".tmpl")

	// Return a 404 if the template doesn't exist
	info, err := os.Stat(fp)
	if err != nil {
		if os.IsNotExist(err) {
			http.Error(w, http.StatusText(http.StatusNotFound), http.StatusNotFound)
			return
		}
	}

	// Return a 404 if the request is for a directory
	if info.IsDir() {
		http.Error(w, http.StatusText(http.StatusNotFound), http.StatusNotFound)
		return
	}

	// Create the template
	tmpl, err := template.ParseFiles(lp, fp)
	if err != nil {
		log.Error("Failed to create the template: " + err.Error())
		http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
		return
	}

	// Execute the template and send it to the user
	if err := tmpl.ExecuteTemplate(w, "layout", data); err != nil {
		if strings.HasSuffix(err.Error(), ": write: broken pipe") ||
			strings.HasSuffix(err.Error(), ": client disconnected") ||
			strings.HasSuffix(err.Error(), ": http2: stream closed") ||
			strings.HasSuffix(err.Error(), ": write: connection timed out") {

			// Broken pipe errors can occur when the user presses the "Stop" button while the template is executing
			// We don't want to reporting these errors to Sentry
			// https://stackoverflow.com/questions/26853200/filter-out-broken-pipe-errors-from-template-execution
			// I don't know exactly what the other errors mean
			log.Info("Ordinary error when executing the template: " + err.Error())
		} else {
			log.Error("Failed to execute the template: " + err.Error())
		}
		http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
	}
}
<|endoftext|>


<commit_before>package streamer

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"strings"
)

func FavoriteDota2Streams() []string {
	favorites := favoriteStreams()
	concatenated := strings.Replace(favorites, "\n", ",", -1)
	requestURL := "https://api.twitch.tv/kraken/streams?game=Dota+2&limit=10&channel=" + concatenated
	res, err := http.Get(requestURL)
	if err != nil {
		log.Fatal(err)
	}
	streams, err := ioutil.ReadAll(res.Body)
	res.Body.Close()
	if err != nil {
		log.Fatal(err)
	}

	var dat JSONResult
	if err := json.Unmarshal(streams, &dat); err != nil {
		panic(err)
	}

	sslice := make([]string, 0)
	for _, g := range dat.Streams {
		s := fmt.Sprintf("%s (%d) - %s - %s", g.Channel.Name, g.Viewers, g.Channel.Status, g.Channel.URL)
		sslice = append(sslice, s)
	}

	return sslice
}

func TopDota2Streams() {
	requestURL := "https://api.twitch.tv/kraken/streams?game=Dota+2&limit=10"
	res, err := http.Get(requestURL)
	if err != nil {
		log.Fatal(err)
	}
	streams, err := ioutil.ReadAll(res.Body)
	res.Body.Close()
	if err != nil {
		log.Fatal(err)
	}

	var dat JSONResult
	if err := json.Unmarshal(streams, &dat); err != nil {
		panic(err)
	}

	for _, g := range dat.Streams {
		if !isBlacklisted(g.Channel.Name) {
			fmt.Println("Stream: " + g.Channel.Name + " - " + g.Channel.Status + " - " + g.Channel.URL)
		}
	}
}

func clientID() string {
	file, e := ioutil.ReadFile("./client.id")
	if e != nil {
		panic(e)
	}
	return string(file)
}

func favoriteStreams() string {
	file, e := ioutil.ReadFile("./favorites.txt")
	if e != nil {
		panic(e)
	}
	return string(file)
}

func blacklistStreams() []string {
	file, e := ioutil.ReadFile("./blacklist.txt")
	if e != nil {
		panic(e)
	}
	return strings.Split(string(file), "\n")
}

func isBlacklisted(stream string) bool {
	blacklist := blacklistStreams()
	for _, b := range blacklist {
		if b == stream {
			return true
		}
	}
	return false
}

// JSON structs
type JSONResult struct {
	Streams []JSONStreams `json:"streams"`
}

type JSONStreams struct {
	Channel JSONChannel `json:"channel"`
	Viewers int         `json:"viewers"`
}

type JSONChannel struct {
	Name   string `json:"display_name"`
	URL    string `json:"url"`
	Status string `json:"status"`
}
<commit_msg>improved top streams<commit_after>package streamer

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"strings"
)

func FavoriteDota2Streams() []string {
	favorites := favoriteStreams()
	concatenated := strings.Replace(favorites, "\n", ",", -1)
	requestURL := "https://api.twitch.tv/kraken/streams?game=Dota+2&limit=10&channel=" + concatenated
	res, err := http.Get(requestURL)
	if err != nil {
		log.Fatal(err)
	}
	streams, err := ioutil.ReadAll(res.Body)
	res.Body.Close()
	if err != nil {
		log.Fatal(err)
	}

	var dat JSONResult
	if err := json.Unmarshal(streams, &dat); err != nil {
		panic(err)
	}

	sslice := make([]string, 0)
	for _, g := range dat.Streams {
		s := fmt.Sprintf("%s (%d) - %s - %s", g.Channel.Name, g.Viewers, g.Channel.Status, g.Channel.URL)
		sslice = append(sslice, s)
	}

	return sslice
}

func TopDota2Streams() []string {
	requestURL := "https://api.twitch.tv/kraken/streams?game=Dota+2&limit=10"
	res, err := http.Get(requestURL)
	if err != nil {
		log.Fatal(err)
	}
	streams, err := ioutil.ReadAll(res.Body)
	res.Body.Close()
	if err != nil {
		log.Fatal(err)
	}

	var dat JSONResult
	if err := json.Unmarshal(streams, &dat); err != nil {
		panic(err)
	}

	sslice := make([]string, 0)
	for _, g := range dat.Streams {
		if !isBlacklisted(g.Channel.Name) {
			s := fmt.Sprintf("%s (%d) - %s - %s", g.Channel.Name, g.Viewers, g.Channel.Status, g.Channel.URL)
			sslice = append(sslice, s)
		}
	}

	return sslice
}

func clientID() string {
	file, e := ioutil.ReadFile("./client.id")
	if e != nil {
		panic(e)
	}
	return string(file)
}

func favoriteStreams() string {
	file, e := ioutil.ReadFile("./favorites.txt")
	if e != nil {
		panic(e)
	}
	return string(file)
}

func blacklistStreams() []string {
	file, e := ioutil.ReadFile("./blacklist.txt")
	if e != nil {
		panic(e)
	}
	return strings.Split(string(file), "\n")
}

func isBlacklisted(stream string) bool {
	blacklist := blacklistStreams()
	for _, b := range blacklist {
		if b == stream {
			return true
		}
	}
	return false
}

// JSON structs
type JSONResult struct {
	Streams []JSONStreams `json:"streams"`
}

type JSONStreams struct {
	Channel JSONChannel `json:"channel"`
	Viewers int         `json:"viewers"`
}

type JSONChannel struct {
	Name   string `json:"display_name"`
	URL    string `json:"url"`
	Status string `json:"status"`
}
<|endoftext|>


<commit_before>package main

import (
	"crypto/md5"
	"encoding/binary"
	"encoding/json"
	"fmt"
	"os"
	"os/exec"
	"os/user"
	"strings"
	"time"

	"github.com/gtfierro/hod/config"
	hod "github.com/gtfierro/hod/db"
	"github.com/gtfierro/hod/goraptor"
	"github.com/gtfierro/hod/query"
	"github.com/gtfierro/hod/server"

	"github.com/chzyer/readline"
	"github.com/fatih/color"
	"github.com/pkg/errors"
	"github.com/urfave/cli"
)

func benchLoad(c *cli.Context) error {
	if c.NArg() == 0 {
		log.Fatal("Need to specify a turtle file to load")
	}
	filename := c.Args().Get(0)
	p := turtle.GetParser()
	ds, duration := p.Parse(filename)
	rate := float64((float64(ds.NumTriples()) / float64(duration.Nanoseconds())) * 1e9)
	fmt.Printf("Loaded %d triples, %d namespaces in %s (%.0f/sec)\n", ds.NumTriples(), ds.NumNamespaces(), duration, rate)
	return nil
}

func load(c *cli.Context) error {
	if c.NArg() == 0 {
		log.Fatal("Need to specify a turtle file to load")
	}
	filename := c.Args().Get(0)
	cfg, err := config.ReadConfig(c.String("config"))
	if err != nil {
		return err
	}
	p := turtle.GetParser()
	ds, duration := p.Parse(filename)
	rate := float64((float64(ds.NumTriples()) / float64(duration.Nanoseconds())) * 1e9)
	log.Infof("Loaded %d triples, %d namespaces in %s (%.0f/sec)", ds.NumTriples(), ds.NumNamespaces(), duration, rate)

	cfg.ReloadBrick = true

	db, err := hod.NewDB(cfg)
	if err != nil {
		return err
	}
	err = db.LoadDataset(ds)
	if err != nil {
		return err
	}

	return nil
}

func loadLinks(c *cli.Context) error {
	if c.NArg() == 0 {
		return errors.New("Need to specify a JSON file to load")
	}
	filename := c.Args().Get(0)
	cfg, err := config.ReadConfig(c.String("config"))
	if err != nil {
		return err
	}
	cfg.ReloadBrick = false
	db, err := hod.NewDB(cfg)
	if err != nil {
		return err
	}
	file, err := os.Open(filename)
	if err != nil {
		return err
	}
	var updates = new(hod.LinkUpdates)
	decoder := json.NewDecoder(file)
	if err := decoder.Decode(updates); err != nil {
		return err
	}
	log.Noticef("Adding %d links, Removing %d links", len(updates.Adding), len(updates.Removing))
	return db.UpdateLinks(updates)
}

func startCLI(c *cli.Context) error {
	cfg, err := config.ReadConfig(c.String("config"))
	if err != nil {
		return err
	}
	cfg.ReloadBrick = false
	db, err := hod.NewDB(cfg)
	if err != nil {
		return err
	}
	return runInteractiveQuery(db)
}

func startHTTP(c *cli.Context) error {
	cfg, err := config.ReadConfig(c.String("config"))
	if err != nil {
		return err
	}
	cfg.ReloadBrick = false
	db, err := hod.NewDB(cfg)
	if err != nil {
		return err
	}
	server.StartHodServer(db, cfg)
	return nil
}

func dump(c *cli.Context) error {
	if c.NArg() == 0 {
		return errors.New("Need to specify a turtle file to load")
	}
	filename := c.Args().Get(0)
	p := turtle.GetParser()
	ds, _ := p.Parse(filename)
	for _, triple := range ds.Triples {
		var s = triple.Subject.Value
		var p = triple.Predicate.Value
		var o = triple.Object.Value
		for pfx, full := range ds.Namespaces {
			if triple.Subject.Namespace == full {
				s = pfx + ":" + s
			}
			if triple.Predicate.Namespace == full {
				p = pfx + ":" + p
			}
			if triple.Object.Namespace == full {
				o = pfx + ":" + o
			}
		}
		fmt.Printf("%s\t%s\t%s\n", s, p, o)
	}
	return nil
}

func classGraph(c *cli.Context) error {
	if c.NArg() == 0 {
		return errors.New("Need to specify a turtle file to load")
	}
	filename := c.Args().Get(0)
	p := turtle.GetParser()
	ds, _ := p.Parse(filename)

	name := gethash() + ".gv"
	f, err := os.Create(name)
	if err != nil {
		return err
	}

	nodes := make(map[string]struct{})
	edges := make(map[string]struct{})
	for _, triple := range ds.Triples {
		if triple.Predicate.String() == "http://www.w3.org/1999/02/22-rdf-syntax-ns#type" && triple.Object.String() == "http://www.w3.org/2002/07/owl#Class" {
			x := fmt.Sprintf("%s;\n", triple.Subject.Value)
			nodes[x] = struct{}{}
		} else if triple.Predicate.String() == "http://www.w3.org/2000/01/rdf-schema#subClassOf" {
			if strings.HasPrefix(triple.Object.Value, "genid") || strings.HasPrefix(triple.Subject.Value, "genid") {
				continue
			}
			x := fmt.Sprintf("%s -> %s [label=\"%s\"];\n", triple.Object.Value, triple.Subject.Value, "hasSubclass")
			edges[x] = struct{}{}
		}
	}

	fmt.Fprintln(f, "digraph G {")
	fmt.Fprintln(f, "ratio=\"auto\"")
	fmt.Fprintln(f, "rankdir=\"LR\"")
	fmt.Fprintln(f, "size=\"7.5,10\"")
	for node := range nodes {
		fmt.Fprintf(f, node)
	}
	for edge := range edges {
		fmt.Fprintf(f, edge)
	}
	fmt.Fprintln(f, "}")
	cmd := exec.Command("dot", "-Tpdf", name)
	pdf, err := cmd.Output()
	if err != nil {
		return err
	}
	f2, err := os.Create(filename + ".pdf")
	if err != nil {
		return err
	}
	_, err = f2.Write(pdf)
	if err != nil {
		return err
	}

	// remove DOT file
	os.Remove(name)
	return nil
}

func dumpGraph(c *cli.Context) error {
	if c.NArg() == 0 {
		return errors.New("Need to specify a turtle file to load")
	}
	filename := c.Args().Get(0)
	p := turtle.GetParser()
	ds, _ := p.Parse(filename)

	name := gethash() + ".gv"
	f, err := os.Create(name)
	if err != nil {
		return err
	}

	nodes := make(map[string]struct{})
	edges := make(map[string]struct{})
	for _, triple := range ds.Triples {
		x := fmt.Sprintf("%s;\n", triple.Subject.Value)
		nodes[x] = struct{}{}
		x = fmt.Sprintf("%s -> %s [label=\"%s\"];\n", triple.Subject.Value, triple.Object.Value, triple.Predicate.Value)
		edges[x] = struct{}{}
	}

	fmt.Fprintln(f, "digraph G {")
	fmt.Fprintln(f, "ratio=\"auto\"")
	fmt.Fprintln(f, "rankdir=\"LR\"")
	fmt.Fprintln(f, "size=\"7.5,10\"")
	for node := range nodes {
		fmt.Fprintf(f, node)
	}
	for edge := range edges {
		fmt.Fprintf(f, edge)
	}
	fmt.Fprintln(f, "}")
	cmd := exec.Command("sfdp", "-Tpdf", name)
	pdf, err := cmd.Output()
	if err != nil {
		// try graphviz dot then
		cmd = exec.Command("dot", "-Tpdf", name)
		pdf, err = cmd.Output()
		if err != nil {
			return err
		}
	}
	f2, err := os.Create(filename + ".pdf")
	if err != nil {
		return err
	}
	_, err = f2.Write(pdf)
	if err != nil {
		return err
	}

	// remove DOT file
	os.Remove(name)
	return nil
}

func gethash() string {
	h := md5.New()
	seed := make([]byte, 16)
	binary.PutVarint(seed, time.Now().UnixNano())
	h.Write(seed)
	return fmt.Sprintf("%x", h.Sum(nil))
}

func runInteractiveQuery(db *hod.DB) error {
	currentUser, err := user.Current()
	if err != nil {
		return err
	}
	fmt.Println("Successfully loaded dataset!")
	bufQuery := ""

	//setup color for prompt
	c := color.New(color.FgCyan)
	c.Add(color.Bold)
	cyan := c.SprintFunc()

	rl, err := readline.NewEx(&readline.Config{
		Prompt:                 cyan("(hod)> "),
		HistoryFile:            currentUser.HomeDir + "/.hod-query-history",
		DisableAutoSaveHistory: true,
	})
	for {
		line, err := rl.Readline()
		if err != nil {
			break
		}
		if len(line) == 0 {
			continue
		}
		bufQuery += line + " "
		if !strings.HasSuffix(strings.TrimSpace(line), ";") {
			rl.SetPrompt(">>> ...")
			continue
		}
		rl.SetPrompt(cyan("(hod)> "))
		rl.SaveHistory(bufQuery)
		q, err := query.Parse(strings.NewReader(bufQuery))
		if err != nil {
			log.Error(err)
		} else {
			res := db.RunQuery(q)
			res.Dump()
		}
		bufQuery = ""
	}
	return nil
}
<commit_msg>make sure to close database in all actions<commit_after>package main

import (
	"crypto/md5"
	"encoding/binary"
	"encoding/json"
	"fmt"
	"os"
	"os/exec"
	"os/user"
	"strings"
	"time"

	"github.com/gtfierro/hod/config"
	hod "github.com/gtfierro/hod/db"
	"github.com/gtfierro/hod/goraptor"
	"github.com/gtfierro/hod/query"
	"github.com/gtfierro/hod/server"

	"github.com/chzyer/readline"
	"github.com/fatih/color"
	"github.com/pkg/errors"
	"github.com/urfave/cli"
)

func benchLoad(c *cli.Context) error {
	if c.NArg() == 0 {
		log.Fatal("Need to specify a turtle file to load")
	}
	filename := c.Args().Get(0)
	p := turtle.GetParser()
	ds, duration := p.Parse(filename)
	rate := float64((float64(ds.NumTriples()) / float64(duration.Nanoseconds())) * 1e9)
	fmt.Printf("Loaded %d triples, %d namespaces in %s (%.0f/sec)\n", ds.NumTriples(), ds.NumNamespaces(), duration, rate)
	return nil
}

func load(c *cli.Context) error {
	if c.NArg() == 0 {
		log.Fatal("Need to specify a turtle file to load")
	}
	filename := c.Args().Get(0)
	cfg, err := config.ReadConfig(c.String("config"))
	if err != nil {
		return err
	}
	p := turtle.GetParser()
	ds, duration := p.Parse(filename)
	rate := float64((float64(ds.NumTriples()) / float64(duration.Nanoseconds())) * 1e9)
	log.Infof("Loaded %d triples, %d namespaces in %s (%.0f/sec)", ds.NumTriples(), ds.NumNamespaces(), duration, rate)

	cfg.ReloadBrick = true

	db, err := hod.NewDB(cfg)
	if err != nil {
		return err
	}
	defer db.Close()
	err = db.LoadDataset(ds)
	if err != nil {
		return err
	}

	return nil
}

func loadLinks(c *cli.Context) error {
	if c.NArg() == 0 {
		return errors.New("Need to specify a JSON file to load")
	}
	filename := c.Args().Get(0)
	cfg, err := config.ReadConfig(c.String("config"))
	if err != nil {
		return err
	}
	cfg.ReloadBrick = false
	db, err := hod.NewDB(cfg)
	if err != nil {
		return err
	}
	defer db.Close()
	file, err := os.Open(filename)
	if err != nil {
		return err
	}
	var updates = new(hod.LinkUpdates)
	decoder := json.NewDecoder(file)
	if err := decoder.Decode(updates); err != nil {
		return err
	}
	log.Noticef("Adding %d links, Removing %d links", len(updates.Adding), len(updates.Removing))
	return db.UpdateLinks(updates)
}

func startCLI(c *cli.Context) error {
	cfg, err := config.ReadConfig(c.String("config"))
	if err != nil {
		return err
	}
	cfg.ReloadBrick = false
	db, err := hod.NewDB(cfg)
	if err != nil {
		return err
	}
	defer db.Close()
	return runInteractiveQuery(db)
}

func startHTTP(c *cli.Context) error {
	cfg, err := config.ReadConfig(c.String("config"))
	if err != nil {
		return err
	}
	cfg.ReloadBrick = false
	db, err := hod.NewDB(cfg)
	if err != nil {
		return err
	}
	server.StartHodServer(db, cfg)
	return nil
}

func dump(c *cli.Context) error {
	if c.NArg() == 0 {
		return errors.New("Need to specify a turtle file to load")
	}
	filename := c.Args().Get(0)
	p := turtle.GetParser()
	ds, _ := p.Parse(filename)
	for _, triple := range ds.Triples {
		var s = triple.Subject.Value
		var p = triple.Predicate.Value
		var o = triple.Object.Value
		for pfx, full := range ds.Namespaces {
			if triple.Subject.Namespace == full {
				s = pfx + ":" + s
			}
			if triple.Predicate.Namespace == full {
				p = pfx + ":" + p
			}
			if triple.Object.Namespace == full {
				o = pfx + ":" + o
			}
		}
		fmt.Printf("%s\t%s\t%s\n", s, p, o)
	}
	return nil
}

func classGraph(c *cli.Context) error {
	if c.NArg() == 0 {
		return errors.New("Need to specify a turtle file to load")
	}
	filename := c.Args().Get(0)
	p := turtle.GetParser()
	ds, _ := p.Parse(filename)

	name := gethash() + ".gv"
	f, err := os.Create(name)
	if err != nil {
		return err
	}

	nodes := make(map[string]struct{})
	edges := make(map[string]struct{})
	for _, triple := range ds.Triples {
		if triple.Predicate.String() == "http://www.w3.org/1999/02/22-rdf-syntax-ns#type" && triple.Object.String() == "http://www.w3.org/2002/07/owl#Class" {
			x := fmt.Sprintf("%s;\n", triple.Subject.Value)
			nodes[x] = struct{}{}
		} else if triple.Predicate.String() == "http://www.w3.org/2000/01/rdf-schema#subClassOf" {
			if strings.HasPrefix(triple.Object.Value, "genid") || strings.HasPrefix(triple.Subject.Value, "genid") {
				continue
			}
			x := fmt.Sprintf("%s -> %s [label=\"%s\"];\n", triple.Object.Value, triple.Subject.Value, "hasSubclass")
			edges[x] = struct{}{}
		}
	}

	fmt.Fprintln(f, "digraph G {")
	fmt.Fprintln(f, "ratio=\"auto\"")
	fmt.Fprintln(f, "rankdir=\"LR\"")
	fmt.Fprintln(f, "size=\"7.5,10\"")
	for node := range nodes {
		fmt.Fprintf(f, node)
	}
	for edge := range edges {
		fmt.Fprintf(f, edge)
	}
	fmt.Fprintln(f, "}")
	cmd := exec.Command("dot", "-Tpdf", name)
	pdf, err := cmd.Output()
	if err != nil {
		return err
	}
	f2, err := os.Create(filename + ".pdf")
	if err != nil {
		return err
	}
	_, err = f2.Write(pdf)
	if err != nil {
		return err
	}

	// remove DOT file
	os.Remove(name)
	return nil
}

func dumpGraph(c *cli.Context) error {
	if c.NArg() == 0 {
		return errors.New("Need to specify a turtle file to load")
	}
	filename := c.Args().Get(0)
	p := turtle.GetParser()
	ds, _ := p.Parse(filename)

	name := gethash() + ".gv"
	f, err := os.Create(name)
	if err != nil {
		return err
	}

	nodes := make(map[string]struct{})
	edges := make(map[string]struct{})
	for _, triple := range ds.Triples {
		x := fmt.Sprintf("%s;\n", triple.Subject.Value)
		nodes[x] = struct{}{}
		x = fmt.Sprintf("%s -> %s [label=\"%s\"];\n", triple.Subject.Value, triple.Object.Value, triple.Predicate.Value)
		edges[x] = struct{}{}
	}

	fmt.Fprintln(f, "digraph G {")
	fmt.Fprintln(f, "ratio=\"auto\"")
	fmt.Fprintln(f, "rankdir=\"LR\"")
	fmt.Fprintln(f, "size=\"7.5,10\"")
	for node := range nodes {
		fmt.Fprintf(f, node)
	}
	for edge := range edges {
		fmt.Fprintf(f, edge)
	}
	fmt.Fprintln(f, "}")
	cmd := exec.Command("sfdp", "-Tpdf", name)
	pdf, err := cmd.Output()
	if err != nil {
		// try graphviz dot then
		cmd = exec.Command("dot", "-Tpdf", name)
		pdf, err = cmd.Output()
		if err != nil {
			return err
		}
	}
	f2, err := os.Create(filename + ".pdf")
	if err != nil {
		return err
	}
	_, err = f2.Write(pdf)
	if err != nil {
		return err
	}

	// remove DOT file
	os.Remove(name)
	return nil
}

func gethash() string {
	h := md5.New()
	seed := make([]byte, 16)
	binary.PutVarint(seed, time.Now().UnixNano())
	h.Write(seed)
	return fmt.Sprintf("%x", h.Sum(nil))
}

func runInteractiveQuery(db *hod.DB) error {
	currentUser, err := user.Current()
	if err != nil {
		return err
	}
	fmt.Println("Successfully loaded dataset!")
	bufQuery := ""

	//setup color for prompt
	c := color.New(color.FgCyan)
	c.Add(color.Bold)
	cyan := c.SprintFunc()

	rl, err := readline.NewEx(&readline.Config{
		Prompt:                 cyan("(hod)> "),
		HistoryFile:            currentUser.HomeDir + "/.hod-query-history",
		DisableAutoSaveHistory: true,
	})
	for {
		line, err := rl.Readline()
		if err != nil {
			break
		}
		if len(line) == 0 {
			continue
		}
		bufQuery += line + " "
		if !strings.HasSuffix(strings.TrimSpace(line), ";") {
			rl.SetPrompt(">>> ...")
			continue
		}
		rl.SetPrompt(cyan("(hod)> "))
		rl.SaveHistory(bufQuery)
		q, err := query.Parse(strings.NewReader(bufQuery))
		if err != nil {
			log.Error(err)
		} else {
			res := db.RunQuery(q)
			res.Dump()
		}
		bufQuery = ""
	}
	return nil
}
<|endoftext|>


<commit_before>package server

import (
	"time"

	// Register PostgreSQL driver bits
	_ "github.com/lib/pq"

	"github.com/jmoiron/sqlx"
	"github.com/travis-ci/jupiter-brain"
)

type database interface {
	SaveInstance(*jupiterbrain.Instance) error
	DestroyInstance(string) error
	FetchInstances(*databaseQuery) ([]*jupiterbrain.Instance, error)
}

type databaseQuery struct {
	MinAge time.Duration
}

type pgDatabase struct {
	conn *sqlx.DB
}

func newPGDatabase(url string) (*pgDatabase, error) {
	conn, err := sqlx.Open("postgres", url)
	if err != nil {
		return nil, err
	}

	return &pgDatabase{
		conn: conn,
	}, nil
}

func (db *pgDatabase) SaveInstance(inst *jupiterbrain.Instance) error {
	_, err := db.conn.Exec(`INSERT INTO jupiter_brain.instances(id, created_at) VALUES ($1, $2)`, inst.ID, inst.CreatedAt)
	return err
}

func (db *pgDatabase) FetchInstances(q *databaseQuery) ([]*jupiterbrain.Instance, error) {
	instances := []*jupiterbrain.Instance{}
	rows, err := db.conn.Queryx(`SELECT * FROM jupiter_brain.instances WHERE destroyed_at IS NULL AND ((now() AT TIME ZONE 'UTC') - created_at) >= $1::interval`, q.MinAge.String())
	if err != nil {
		return instances, err
	}

	defer func() { _ = rows.Close() }()

	for rows.Next() {
		instance := &jupiterbrain.Instance{}
		err = rows.StructScan(instance)
		if err != nil {
			return instances, err
		}

		instances = append(instances, instance)
	}

	return instances, nil
}

func (db *pgDatabase) DestroyInstance(id string) error {
	_, err := db.conn.Queryx(`UPDATE jupiter_brain.instances SET destroyed_at = now() WHERE id = $1`, id)
	return err
}
<commit_msg>Switch to an Exec to mitigate conn leakage<commit_after>package server

import (
	"time"

	// Register PostgreSQL driver bits
	_ "github.com/lib/pq"

	"github.com/jmoiron/sqlx"
	"github.com/travis-ci/jupiter-brain"
)

type database interface {
	SaveInstance(*jupiterbrain.Instance) error
	DestroyInstance(string) error
	FetchInstances(*databaseQuery) ([]*jupiterbrain.Instance, error)
}

type databaseQuery struct {
	MinAge time.Duration
}

type pgDatabase struct {
	conn *sqlx.DB
}

func newPGDatabase(url string) (*pgDatabase, error) {
	conn, err := sqlx.Open("postgres", url)
	if err != nil {
		return nil, err
	}

	return &pgDatabase{
		conn: conn,
	}, nil
}

func (db *pgDatabase) SaveInstance(inst *jupiterbrain.Instance) error {
	_, err := db.conn.Exec(`INSERT INTO jupiter_brain.instances(id, created_at) VALUES ($1, $2)`, inst.ID, inst.CreatedAt)
	return err
}

func (db *pgDatabase) FetchInstances(q *databaseQuery) ([]*jupiterbrain.Instance, error) {
	instances := []*jupiterbrain.Instance{}
	rows, err := db.conn.Queryx(`SELECT * FROM jupiter_brain.instances WHERE destroyed_at IS NULL AND ((now() AT TIME ZONE 'UTC') - created_at) >= $1::interval`, q.MinAge.String())
	if err != nil {
		return instances, err
	}

	defer func() { _ = rows.Close() }()

	for rows.Next() {
		instance := &jupiterbrain.Instance{}
		err = rows.StructScan(instance)
		if err != nil {
			return instances, err
		}

		instances = append(instances, instance)
	}

	return instances, nil
}

func (db *pgDatabase) DestroyInstance(id string) error {
	_, err := db.conn.Exec(`UPDATE jupiter_brain.instances SET destroyed_at = now() WHERE id = $1`, id)
	return err
}
<|endoftext|>


<commit_before>package geddit

import (
	"encoding/json"
	"fmt"
	"net/url"
)

const (
	MaxLimit = 100
)

// Paginator represents a Listing endpoint
type Paginator struct {
	s     *Session
	url   string
	name  string // Fullname of reference Thing
	count int    // Current item offset *NOT USED*
	limit int    // Limit of items returned
}

// Next returns a new set of links directly following a previous request.
func (p *Paginator) Next() ([]Link, error) {
	resp, err := p.list(true)
	if err != nil {
		return nil, err
	}
	if len(resp) > 0 {
		p.name = resp[len(resp)-1].Name
	}
	return resp, err
}

// Previous returns a new set of links directly preceeding a previous request.
func (p *Paginator) Previous() ([]Link, error) {
	resp, err := p.list(false)
	if err != nil {
		return nil, err
	}
	if len(resp) > 0 {
		p.name = resp[0].Name
	}
	return resp, err
}

// SetLimit sets the max number of links returned from calls to Previous and Next
func (p *Paginator) SetLimit(l int) {
	if l < 0 {
		l = 0
	}
	if l > MaxLimit {
		l = MaxLimit
	}
	p.limit = l
}

func (p *Paginator) list(after bool) ([]Link, error) {
	v := p.values(after)
	resp, err := p.s.get(p.url, v)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()

	listing := Listing{}
	err = json.NewDecoder(resp.Body).Decode(&listing)
	if err != nil {
		return nil, err
	}

	if listing.Kind != TypeListing {
		// TODO.. handle error
	}
	var links []Link
	for _, c := range listing.Data.Children {
		link := Link{}
		err = json.Unmarshal(c.Data, &link)
		if err != nil {
			return nil, err
		}
		links = append(links, link)
	}

	return links, nil
}

func (p *Paginator) values(after bool) *url.Values {
	v := url.Values{}
	if p.name != "" {
		if after {
			v.Set("after", p.name)
		} else {
			v.Set("before", p.name)
		}
	}
	if p.count > 0 {
		v.Set("count", fmt.Sprintf("%d", p.count))
	}
	if p.limit > 0 {
		v.Set("limit", fmt.Sprintf("%d", p.limit))
	}

	return &v
}
<commit_msg>Removed unnecesary pointer reference<commit_after>package geddit

import (
	"encoding/json"
	"fmt"
	"net/url"
)

const (
	MaxLimit = 100
)

// Paginator represents a Listing endpoint
type Paginator struct {
	s     *Session
	url   string
	name  string // Fullname of reference Thing
	count int    // Current item offset *NOT USED*
	limit int    // Limit of items returned
}

// Next returns a new set of links directly following a previous request.
func (p *Paginator) Next() ([]Link, error) {
	resp, err := p.list(true)
	if err != nil {
		return nil, err
	}
	if len(resp) > 0 {
		p.name = resp[len(resp)-1].Name
	}
	return resp, err
}

// Previous returns a new set of links directly preceeding a previous request.
func (p *Paginator) Previous() ([]Link, error) {
	resp, err := p.list(false)
	if err != nil {
		return nil, err
	}
	if len(resp) > 0 {
		p.name = resp[0].Name
	}
	return resp, err
}

// SetLimit sets the max number of links returned from calls to Previous and Next
func (p *Paginator) SetLimit(l int) {
	if l < 0 {
		l = 0
	}
	if l > MaxLimit {
		l = MaxLimit
	}
	p.limit = l
}

func (p *Paginator) list(after bool) ([]Link, error) {
	v := p.values(after)
	resp, err := p.s.get(p.url, v)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()

	listing := Listing{}
	err = json.NewDecoder(resp.Body).Decode(&listing)
	if err != nil {
		return nil, err
	}

	if listing.Kind != TypeListing {
		// TODO.. handle error
	}
	var links []Link
	for _, c := range listing.Data.Children {
		link := Link{}
		err = json.Unmarshal(c.Data, &link)
		if err != nil {
			return nil, err
		}
		links = append(links, link)
	}

	return links, nil
}

func (p *Paginator) values(after bool) url.Values {
	v := url.Values{}
	if p.name != "" {
		if after {
			v.Set("after", p.name)
		} else {
			v.Set("before", p.name)
		}
	}
	if p.count > 0 {
		v.Set("count", fmt.Sprintf("%d", p.count))
	}
	if p.limit > 0 {
		v.Set("limit", fmt.Sprintf("%d", p.limit))
	}

	return v
}
<|endoftext|>


<commit_before>package acceptance_test

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"math/rand"
	"net/http"
	"os"
	"strings"
	"time"

	"github.com/cloudfoundry-incubator/cf-test-helpers/cf"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"github.com/onsi/gomega/gexec"
)

const Timeout_Push = 5 * time.Minute
const Timeout_Short = 10 * time.Second

var ports []int

func getSubnet(ip string) string {
	return strings.Split(ip, ".")[2]
}

func isSameCell(sourceIP, destIP string) bool {
	return getSubnet(sourceIP) == getSubnet(destIP)
}

var _ = Describe("connectivity between containers on the overlay network", func() {
	Describe("networking policy", func() {
		var (
			appProxy     string
			appsReflex   []string
			orgName      string
			spaceName    string
			appInstances int
			applications int
		)

		BeforeEach(func() {
			appInstances = testConfig.AppInstances
			applications = testConfig.Applications

			appProxy = fmt.Sprintf("proxy-%d", rand.Int31())
			for i := 0; i < applications; i++ {
				appsReflex = append(appsReflex, fmt.Sprintf("reflex-%d-%d", i, rand.Int31()))
			}

			ports = []int{8080}
			for i := 0; i < testConfig.Policies; i++ {
				ports = append(ports, 7000+i)
			}

			Auth(testConfig.TestUser, testConfig.TestUserPassword)

			orgName = "test-org"
			Expect(cf.Cf("create-org", orgName).Wait(Timeout_Push)).To(gexec.Exit(0))
			Expect(cf.Cf("target", "-o", orgName).Wait(Timeout_Push)).To(gexec.Exit(0))

			spaceName = "test-space"
			Expect(cf.Cf("create-space", spaceName).Wait(Timeout_Push)).To(gexec.Exit(0))
			Expect(cf.Cf("target", "-o", orgName, "-s", spaceName).Wait(Timeout_Push)).To(gexec.Exit(0))

			pushApp(appProxy)

			newManifest := modifyReflexManifest()
			pushAppsOfType(appsReflex, "reflex", newManifest)
			for _, app := range appsReflex {
				By("creating a new policy to allow the reflex app to talk to itself")
				session := cf.Cf("access-allow", app, app, "--protocol", "tcp", "--port", fmt.Sprintf("%d", ports[0])).Wait(2 * Timeout_Short)
				Expect(session.Wait(Timeout_Short)).To(gexec.Exit(0))
				scaleApp(app, appInstances)
			}
		})

		AfterEach(func() {
			appReport(appProxy, Timeout_Short)
			appsReport(appsReflex, Timeout_Short)

			// clean up everything
			Expect(cf.Cf("delete-org", orgName, "-f").Wait(Timeout_Push)).To(gexec.Exit(0))
		})

		It("allows the user to configure connections", func(done Done) {
			By("checking that the reflex app has discovered all its instances")
			checkPeers(appsReflex, 60*time.Second, 500*time.Millisecond, appInstances)

			By("checking that the connection fails")
			assertConnectionFails(appProxy, appsReflex, ports)

			By("creating a new policy")
			for _, app := range appsReflex {
				for _, port := range ports {
					session := cf.Cf("access-allow", appProxy, app, "--protocol", "tcp", "--port", fmt.Sprintf("%d", port)).Wait(2 * Timeout_Short)
					Expect(session.Wait(Timeout_Short)).To(gexec.Exit(0))
				}
			}

			By(fmt.Sprintf("checking that %s can reach %s", appProxy, appsReflex))
			assertConnectionSucceeds(appProxy, appsReflex, ports)

			dumpStats(appProxy, config.AppsDomain)

			By("deleting the policy")
			for _, app := range appsReflex {
				for _, port := range ports {
					session := cf.Cf("access-deny", appProxy, app, "--protocol", "tcp", "--port", fmt.Sprintf("%d", port)).Wait(2 * Timeout_Short)
					Expect(session.Wait(Timeout_Short)).To(gexec.Exit(0))
				}
			}

			By(fmt.Sprintf("checking that %s can NOT reach %s", appProxy, appsReflex))
			assertConnectionFails(appProxy, appsReflex, ports)

			By("checking that reflex no longer reports deleted instances")
			scaleApps(appsReflex, 1 /* instances */)
			checkPeers(appsReflex, 60*time.Second, 500*time.Millisecond, appInstances)

			close(done)
		}, 300 /* <-- overall spec timeout in seconds */)
	})
})

func dumpStats(host, domain string) {
	resp, err := http.Get(fmt.Sprintf("http://%s.%s/stats", host, domain))
	Expect(err).NotTo(HaveOccurred())
	respBytes, err := ioutil.ReadAll(resp.Body)
	Expect(err).NotTo(HaveOccurred())
	defer resp.Body.Close()

	fmt.Printf("STATS: %s\n", string(respBytes))
	netStatsFile := os.Getenv("NETWORK_STATS_FILE")
	if netStatsFile != "" {
		Expect(ioutil.WriteFile(netStatsFile, respBytes, 0600)).To(Succeed())
	}
}

func checkPeers(apps []string, timeout, pollingInterval time.Duration, instances int) {
	for _, app := range apps {
		getPeers := func() ([]string, error) {
			resp, err := http.Get(fmt.Sprintf("http://%s.%s/peers", app, config.AppsDomain))
			if err != nil {
				return nil, err
			}

			respBytes, err := ioutil.ReadAll(resp.Body)
			if err != nil {
				return nil, err
			}
			defer resp.Body.Close()

			var peersResponse struct {
				IPs []string
			}
			err = json.Unmarshal(respBytes, &peersResponse)
			if err != nil {
				return nil, err
			}
			return peersResponse.IPs, nil
		}

		Eventually(getPeers, timeout, pollingInterval).Should(HaveLen(instances))
	}
}

func assertConnectionSucceeds(sourceApp string, destApps []string, ports []int) {
	for _, app := range destApps {
		for _, port := range ports {
			assertAllConnectionStatus(sourceApp, app, port, true)
		}
	}
}

func assertConnectionFails(sourceApp string, destApps []string, ports []int) {
	for _, app := range destApps {
		for _, port := range ports {
			assertAllConnectionStatus(sourceApp, app, port, false)
		}
	}
}

func assertAllConnectionStatus(sourceApp, destApp string, port int, shouldSucceed bool) {
	resp, err := http.Get(fmt.Sprintf("http://%s.%s/peers", destApp, config.AppsDomain))
	Expect(err).NotTo(HaveOccurred())
	respBytes, err := ioutil.ReadAll(resp.Body)
	Expect(err).NotTo(HaveOccurred())
	defer resp.Body.Close()

	var addressListJson struct {
		IPs []string
	}
	Expect(json.Unmarshal(respBytes, &addressListJson)).To(Succeed())

	for _, destIP := range addressListJson.IPs {
		assertSingleConnection(sourceApp, destIP, port, shouldSucceed)
	}
}

func assertSingleConnection(sourceAppName string, destIP string, port int, shouldSucceed bool) {
	proxyTest := func() (string, error) {
		resp, err := http.Get(fmt.Sprintf("http://%s.%s/proxy/%s:%d/peers", sourceAppName, config.AppsDomain, destIP, port))
		if err != nil {
			return "", err
		}
		defer resp.Body.Close()
		respBytes, err := ioutil.ReadAll(resp.Body)
		if err != nil {
			return "", err
		}
		return string(respBytes), nil
	}
	if shouldSucceed {
		Eventually(proxyTest, 10*time.Second).ShouldNot(ContainSubstring("failed"))
	} else {
		Eventually(proxyTest, 10*time.Second).Should(ContainSubstring("request failed"))
	}
}
<commit_msg>Increase timeout on netman-cf-acceptance test spec<commit_after>package acceptance_test

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"math/rand"
	"net/http"
	"os"
	"strings"
	"time"

	"github.com/cloudfoundry-incubator/cf-test-helpers/cf"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"github.com/onsi/gomega/gexec"
)

const Timeout_Push = 5 * time.Minute
const Timeout_Short = 10 * time.Second

var ports []int

func getSubnet(ip string) string {
	return strings.Split(ip, ".")[2]
}

func isSameCell(sourceIP, destIP string) bool {
	return getSubnet(sourceIP) == getSubnet(destIP)
}

var _ = Describe("connectivity between containers on the overlay network", func() {
	Describe("networking policy", func() {
		var (
			appProxy     string
			appsReflex   []string
			orgName      string
			spaceName    string
			appInstances int
			applications int
		)

		BeforeEach(func() {
			appInstances = testConfig.AppInstances
			applications = testConfig.Applications

			appProxy = fmt.Sprintf("proxy-%d", rand.Int31())
			for i := 0; i < applications; i++ {
				appsReflex = append(appsReflex, fmt.Sprintf("reflex-%d-%d", i, rand.Int31()))
			}

			ports = []int{8080}
			for i := 0; i < testConfig.Policies; i++ {
				ports = append(ports, 7000+i)
			}

			Auth(testConfig.TestUser, testConfig.TestUserPassword)

			orgName = "test-org"
			Expect(cf.Cf("create-org", orgName).Wait(Timeout_Push)).To(gexec.Exit(0))
			Expect(cf.Cf("target", "-o", orgName).Wait(Timeout_Push)).To(gexec.Exit(0))

			spaceName = "test-space"
			Expect(cf.Cf("create-space", spaceName).Wait(Timeout_Push)).To(gexec.Exit(0))
			Expect(cf.Cf("target", "-o", orgName, "-s", spaceName).Wait(Timeout_Push)).To(gexec.Exit(0))

			pushApp(appProxy)

			newManifest := modifyReflexManifest()
			pushAppsOfType(appsReflex, "reflex", newManifest)
			for _, app := range appsReflex {
				By("creating a new policy to allow the reflex app to talk to itself")
				session := cf.Cf("access-allow", app, app, "--protocol", "tcp", "--port", fmt.Sprintf("%d", ports[0])).Wait(2 * Timeout_Short)
				Expect(session.Wait(Timeout_Short)).To(gexec.Exit(0))
				scaleApp(app, appInstances)
			}
		})

		AfterEach(func() {
			appReport(appProxy, Timeout_Short)
			appsReport(appsReflex, Timeout_Short)

			// clean up everything
			Expect(cf.Cf("delete-org", orgName, "-f").Wait(Timeout_Push)).To(gexec.Exit(0))
		})

		It("allows the user to configure connections", func(done Done) {
			By("checking that the reflex app has discovered all its instances")
			checkPeers(appsReflex, 60*time.Second, 500*time.Millisecond, appInstances)

			By("checking that the connection fails")
			assertConnectionFails(appProxy, appsReflex, ports)

			By("creating a new policy")
			for _, app := range appsReflex {
				for _, port := range ports {
					session := cf.Cf("access-allow", appProxy, app, "--protocol", "tcp", "--port", fmt.Sprintf("%d", port)).Wait(2 * Timeout_Short)
					Expect(session.Wait(Timeout_Short)).To(gexec.Exit(0))
				}
			}

			By(fmt.Sprintf("checking that %s can reach %s", appProxy, appsReflex))
			assertConnectionSucceeds(appProxy, appsReflex, ports)

			dumpStats(appProxy, config.AppsDomain)

			By("deleting the policy")
			for _, app := range appsReflex {
				for _, port := range ports {
					session := cf.Cf("access-deny", appProxy, app, "--protocol", "tcp", "--port", fmt.Sprintf("%d", port)).Wait(2 * Timeout_Short)
					Expect(session.Wait(Timeout_Short)).To(gexec.Exit(0))
				}
			}

			By(fmt.Sprintf("checking that %s can NOT reach %s", appProxy, appsReflex))
			assertConnectionFails(appProxy, appsReflex, ports)

			By("checking that reflex no longer reports deleted instances")
			scaleApps(appsReflex, 1 /* instances */)
			checkPeers(appsReflex, 60*time.Second, 500*time.Millisecond, appInstances)

			close(done)
		}, 900 /* <-- overall spec timeout in seconds */)
	})
})

func dumpStats(host, domain string) {
	resp, err := http.Get(fmt.Sprintf("http://%s.%s/stats", host, domain))
	Expect(err).NotTo(HaveOccurred())
	respBytes, err := ioutil.ReadAll(resp.Body)
	Expect(err).NotTo(HaveOccurred())
	defer resp.Body.Close()

	fmt.Printf("STATS: %s\n", string(respBytes))
	netStatsFile := os.Getenv("NETWORK_STATS_FILE")
	if netStatsFile != "" {
		Expect(ioutil.WriteFile(netStatsFile, respBytes, 0600)).To(Succeed())
	}
}

func checkPeers(apps []string, timeout, pollingInterval time.Duration, instances int) {
	for _, app := range apps {
		getPeers := func() ([]string, error) {
			resp, err := http.Get(fmt.Sprintf("http://%s.%s/peers", app, config.AppsDomain))
			if err != nil {
				return nil, err
			}

			respBytes, err := ioutil.ReadAll(resp.Body)
			if err != nil {
				return nil, err
			}
			defer resp.Body.Close()

			var peersResponse struct {
				IPs []string
			}
			err = json.Unmarshal(respBytes, &peersResponse)
			if err != nil {
				return nil, err
			}
			return peersResponse.IPs, nil
		}

		Eventually(getPeers, timeout, pollingInterval).Should(HaveLen(instances))
	}
}

func assertConnectionSucceeds(sourceApp string, destApps []string, ports []int) {
	for _, app := range destApps {
		for _, port := range ports {
			assertAllConnectionStatus(sourceApp, app, port, true)
		}
	}
}

func assertConnectionFails(sourceApp string, destApps []string, ports []int) {
	for _, app := range destApps {
		for _, port := range ports {
			assertAllConnectionStatus(sourceApp, app, port, false)
		}
	}
}

func assertAllConnectionStatus(sourceApp, destApp string, port int, shouldSucceed bool) {
	resp, err := http.Get(fmt.Sprintf("http://%s.%s/peers", destApp, config.AppsDomain))
	Expect(err).NotTo(HaveOccurred())
	respBytes, err := ioutil.ReadAll(resp.Body)
	Expect(err).NotTo(HaveOccurred())
	defer resp.Body.Close()

	var addressListJson struct {
		IPs []string
	}
	Expect(json.Unmarshal(respBytes, &addressListJson)).To(Succeed())

	for _, destIP := range addressListJson.IPs {
		assertSingleConnection(sourceApp, destIP, port, shouldSucceed)
	}
}

func assertSingleConnection(sourceAppName string, destIP string, port int, shouldSucceed bool) {
	proxyTest := func() (string, error) {
		resp, err := http.Get(fmt.Sprintf("http://%s.%s/proxy/%s:%d/peers", sourceAppName, config.AppsDomain, destIP, port))
		if err != nil {
			return "", err
		}
		defer resp.Body.Close()
		respBytes, err := ioutil.ReadAll(resp.Body)
		if err != nil {
			return "", err
		}
		return string(respBytes), nil
	}
	if shouldSucceed {
		Eventually(proxyTest, 10*time.Second).ShouldNot(ContainSubstring("failed"))
	} else {
		Eventually(proxyTest, 10*time.Second).Should(ContainSubstring("request failed"))
	}
}
<|endoftext|>


<commit_before>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Rudimentary logging package. Defines a type, Logger, with simple
// methods for formatting output to one or two destinations. Also has
// predefined Loggers accessible through helper functions Stdout[f],
// Stderr[f], Exit[f], and Crash[f], which are easier to use than creating
// a Logger manually.
// Exit exits when written to.
// Crash causes a crash when written to.
package log

import (
	"fmt";
	"io";
	"runtime";
	"os";
	"time";
)

// These flags define the properties of the Logger and the output they produce.
const (
	// Flags
	Lok = iota;
	Lexit;	// terminate execution when written
	Lcrash;	// crash (panic) when written
	// Bits or'ed together to control what's printed. There is no control over the
	// order they appear (the order listed here) or the format they present (as
	// described in the comments).  A colon appears after these items:
	//	2009/0123 01:23:23.123123 /a/b/c/d.go:23: message
	Ldate = 1 << iota;	// the date: 2009/0123
	Ltime;	// the time: 01:23:23
	Lmicroseconds;	// microsecond resolution: 01:23:23.123123.  assumes Ltime.
	Llongfile;	// full file name and line number: /a/b/c/d.go:23
	Lshortfile;	// final file name element and line number: d.go:23. overrides Llongfile
	lAllBits = Ldate | Ltime | Lmicroseconds | Llongfile | Lshortfile;
)

// Logger represents an active logging object.
type Logger struct {
	out0	io.Writer;	// first destination for output
	out1	io.Writer;	// second destination for output; may be nil
	prefix string;	// prefix to write at beginning of each line
	flag int;	// properties
}

// New creates a new Logger.   The out0 and out1 variables set the
// destinations to which log data will be written; out1 may be nil.
// The prefix appears at the beginning of each generated log line.
// The flag argument defines the logging properties.
func New(out0, out1 io.Writer, prefix string, flag int) *Logger {
	return &Logger{out0, out1, prefix, flag}
}

var (
	stdout = New(os.Stdout, nil, "", Lok|Ldate|Ltime);
	stderr = New(os.Stderr, nil, "", Lok|Ldate|Ltime);
	exit = New(os.Stderr, nil, "", Lexit|Ldate|Ltime);
	crash = New(os.Stderr, nil, "", Lcrash|Ldate|Ltime);
)

var shortnames = make(map[string] string)	// cache of short names to avoid allocation.

// Cheap integer to fixed-width decimal ASCII.  Use a negative width to avoid zero-padding
func itoa(i int, wid int) string {
	var u uint = uint(i);
	if u == 0 && wid <= 1 {
		return "0"
	}

	// Assemble decimal in reverse order.
	var b [32]byte;
	bp := len(b);
	for ; u > 0 || wid > 0; u /= 10 {
		bp--;
		wid--;
		b[bp] = byte(u%10) + '0';
	}

	return string(b[bp:len(b)])
}

func (l *Logger) formatHeader(ns int64, calldepth int) string {
	h := l.prefix;
	if l.flag & (Ldate | Ltime | Lmicroseconds) != 0 {
		t := time.SecondsToLocalTime(ns/1e9);
		if l.flag & (Ldate) != 0 {
			h += itoa(int(t.Year), 4) + "/" + itoa(t.Month, 2) + itoa(t.Day, 2) + " "
		}
		if l.flag & (Ltime | Lmicroseconds) != 0 {
			h += itoa(t.Hour, 2) + ":" + itoa(t.Minute, 2) + ":" + itoa(t.Second, 2);
			if l.flag & Lmicroseconds != 0 {
				h += "." + itoa(int(ns % 1e9)/1e3, 6);
			}
			h += " ";
		}
	}
	if l.flag & (Lshortfile | Llongfile) != 0 {
		pc, file, line, ok := runtime.Caller(calldepth);
		if ok {
			if l.flag & Lshortfile != 0 {
				short, ok := shortnames[file];
				if !ok {
					short = file;
					for i := len(file) - 1; i > 0; i-- {
						if file[i] == '/' {
							short = file[i+1:len(file)];
							break;
						}
					}
					shortnames[file] = short;
				}
				file = short;
			}
		} else {
			file = "???";
			line = 0;
		}
		h += file + ":" + itoa(line, -1) + ": ";
	}
	return h;
}

// Output writes the output for a logging event.  The string s contains the text to print after
// the time stamp;  calldepth is used to recover the PC.  It is provided for generality, although
// at the moment on all pre-defined paths it will be 2.
func (l *Logger) Output(calldepth int, s string) {
	now := time.Nanoseconds();	// get this early.
	newline := "\n";
	if len(s) > 0 && s[len(s)-1] == '\n' {
		newline = ""
	}
	s = l.formatHeader(now, calldepth+1) + s + newline;
	io.WriteString(l.out0, s);
	if l.out1 != nil {
		io.WriteString(l.out1, s);
	}
	switch l.flag & ^lAllBits {
	case Lcrash:
		panic("log: fatal error");
	case Lexit:
		os.Exit(1);
	}
}

// Logf is analogous to Printf() for a Logger.
func (l *Logger) Logf(format string, v ...) {
	l.Output(2, fmt.Sprintf(format, v))
}

// Log is analogouts to Print() for a Logger.
func (l *Logger) Log(v ...) {
	l.Output(2, fmt.Sprintln(v))
}

// Stdout is a helper function for easy logging to stdout. It is analogous to Print().
func Stdout(v ...) {
	stdout.Output(2, fmt.Sprint(v))
}

// Stderr is a helper function for easy logging to stderr. It is analogous to Fprint(os.Stderr).
func Stderr(v ...) {
	stderr.Output(2, fmt.Sprintln(v))
}

// Stdoutf is a helper functions for easy formatted logging to stdout. It is analogous to Printf().
func Stdoutf(format string, v ...) {
	stdout.Output(2, fmt.Sprintf(format, v))
}

// Stderrf is a helper function for easy formatted logging to stderr. It is analogous to Fprintf(os.Stderr).
func Stderrf(format string, v ...) {
	stderr.Output(2, fmt.Sprintf(format, v))
}

// Exit is equivalent to Stderr() followed by a call to os.Exit(1).
func Exit(v ...) {
	exit.Output(2, fmt.Sprintln(v))
}

// Exitf is equivalent to Stderrf() followed by a call to os.Exit(1).
func Exitf(format string, v ...) {
	exit.Output(2, fmt.Sprintf(format, v))
}

// Crash is equivalent to Stderrf() followed by a call to panic().
func Crash(v ...) {
	crash.Output(2, fmt.Sprintln(v))
}

// Crashf is equivalent to Stderrf() followed by a call to panic().
func Crashf(format string, v ...) {
	crash.Output(2, fmt.Sprintf(format, v))
}
<commit_msg>fix typo<commit_after>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Rudimentary logging package. Defines a type, Logger, with simple
// methods for formatting output to one or two destinations. Also has
// predefined Loggers accessible through helper functions Stdout[f],
// Stderr[f], Exit[f], and Crash[f], which are easier to use than creating
// a Logger manually.
// Exit exits when written to.
// Crash causes a crash when written to.
package log

import (
	"fmt";
	"io";
	"runtime";
	"os";
	"time";
)

// These flags define the properties of the Logger and the output they produce.
const (
	// Flags
	Lok = iota;
	Lexit;	// terminate execution when written
	Lcrash;	// crash (panic) when written
	// Bits or'ed together to control what's printed. There is no control over the
	// order they appear (the order listed here) or the format they present (as
	// described in the comments).  A colon appears after these items:
	//	2009/0123 01:23:23.123123 /a/b/c/d.go:23: message
	Ldate = 1 << iota;	// the date: 2009/0123
	Ltime;	// the time: 01:23:23
	Lmicroseconds;	// microsecond resolution: 01:23:23.123123.  assumes Ltime.
	Llongfile;	// full file name and line number: /a/b/c/d.go:23
	Lshortfile;	// final file name element and line number: d.go:23. overrides Llongfile
	lAllBits = Ldate | Ltime | Lmicroseconds | Llongfile | Lshortfile;
)

// Logger represents an active logging object.
type Logger struct {
	out0	io.Writer;	// first destination for output
	out1	io.Writer;	// second destination for output; may be nil
	prefix string;	// prefix to write at beginning of each line
	flag int;	// properties
}

// New creates a new Logger.   The out0 and out1 variables set the
// destinations to which log data will be written; out1 may be nil.
// The prefix appears at the beginning of each generated log line.
// The flag argument defines the logging properties.
func New(out0, out1 io.Writer, prefix string, flag int) *Logger {
	return &Logger{out0, out1, prefix, flag}
}

var (
	stdout = New(os.Stdout, nil, "", Lok|Ldate|Ltime);
	stderr = New(os.Stderr, nil, "", Lok|Ldate|Ltime);
	exit = New(os.Stderr, nil, "", Lexit|Ldate|Ltime);
	crash = New(os.Stderr, nil, "", Lcrash|Ldate|Ltime);
)

var shortnames = make(map[string] string)	// cache of short names to avoid allocation.

// Cheap integer to fixed-width decimal ASCII.  Use a negative width to avoid zero-padding
func itoa(i int, wid int) string {
	var u uint = uint(i);
	if u == 0 && wid <= 1 {
		return "0"
	}

	// Assemble decimal in reverse order.
	var b [32]byte;
	bp := len(b);
	for ; u > 0 || wid > 0; u /= 10 {
		bp--;
		wid--;
		b[bp] = byte(u%10) + '0';
	}

	return string(b[bp:len(b)])
}

func (l *Logger) formatHeader(ns int64, calldepth int) string {
	h := l.prefix;
	if l.flag & (Ldate | Ltime | Lmicroseconds) != 0 {
		t := time.SecondsToLocalTime(ns/1e9);
		if l.flag & (Ldate) != 0 {
			h += itoa(int(t.Year), 4) + "/" + itoa(t.Month, 2) + itoa(t.Day, 2) + " "
		}
		if l.flag & (Ltime | Lmicroseconds) != 0 {
			h += itoa(t.Hour, 2) + ":" + itoa(t.Minute, 2) + ":" + itoa(t.Second, 2);
			if l.flag & Lmicroseconds != 0 {
				h += "." + itoa(int(ns % 1e9)/1e3, 6);
			}
			h += " ";
		}
	}
	if l.flag & (Lshortfile | Llongfile) != 0 {
		pc, file, line, ok := runtime.Caller(calldepth);
		if ok {
			if l.flag & Lshortfile != 0 {
				short, ok := shortnames[file];
				if !ok {
					short = file;
					for i := len(file) - 1; i > 0; i-- {
						if file[i] == '/' {
							short = file[i+1:len(file)];
							break;
						}
					}
					shortnames[file] = short;
				}
				file = short;
			}
		} else {
			file = "???";
			line = 0;
		}
		h += file + ":" + itoa(line, -1) + ": ";
	}
	return h;
}

// Output writes the output for a logging event.  The string s contains the text to print after
// the time stamp;  calldepth is used to recover the PC.  It is provided for generality, although
// at the moment on all pre-defined paths it will be 2.
func (l *Logger) Output(calldepth int, s string) {
	now := time.Nanoseconds();	// get this early.
	newline := "\n";
	if len(s) > 0 && s[len(s)-1] == '\n' {
		newline = ""
	}
	s = l.formatHeader(now, calldepth+1) + s + newline;
	io.WriteString(l.out0, s);
	if l.out1 != nil {
		io.WriteString(l.out1, s);
	}
	switch l.flag & ^lAllBits {
	case Lcrash:
		panic("log: fatal error");
	case Lexit:
		os.Exit(1);
	}
}

// Logf is analogous to Printf() for a Logger.
func (l *Logger) Logf(format string, v ...) {
	l.Output(2, fmt.Sprintf(format, v))
}

// Log is analogouts to Print() for a Logger.
func (l *Logger) Log(v ...) {
	l.Output(2, fmt.Sprintln(v))
}

// Stdout is a helper function for easy logging to stdout. It is analogous to Print().
func Stdout(v ...) {
	stdout.Output(2, fmt.Sprint(v))
}

// Stderr is a helper function for easy logging to stderr. It is analogous to Fprint(os.Stderr).
func Stderr(v ...) {
	stderr.Output(2, fmt.Sprintln(v))
}

// Stdoutf is a helper functions for easy formatted logging to stdout. It is analogous to Printf().
func Stdoutf(format string, v ...) {
	stdout.Output(2, fmt.Sprintf(format, v))
}

// Stderrf is a helper function for easy formatted logging to stderr. It is analogous to Fprintf(os.Stderr).
func Stderrf(format string, v ...) {
	stderr.Output(2, fmt.Sprintf(format, v))
}

// Exit is equivalent to Stderr() followed by a call to os.Exit(1).
func Exit(v ...) {
	exit.Output(2, fmt.Sprintln(v))
}

// Exitf is equivalent to Stderrf() followed by a call to os.Exit(1).
func Exitf(format string, v ...) {
	exit.Output(2, fmt.Sprintf(format, v))
}

// Crash is equivalent to Stderr() followed by a call to panic().
func Crash(v ...) {
	crash.Output(2, fmt.Sprintln(v))
}

// Crashf is equivalent to Stderrf() followed by a call to panic().
func Crashf(format string, v ...) {
	crash.Output(2, fmt.Sprintf(format, v))
}
<|endoftext|>


<commit_before>/*
Copyright 2019 The Vitess Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package vindexes

import (
	"bytes"
	"crypto/md5"

	"vitess.io/vitess/go/sqltypes"
	"vitess.io/vitess/go/vt/key"
)

var (
	_ SingleColumn = (*BinaryMD5)(nil)
)

// BinaryMD5 is a vindex that hashes binary bits to a keyspace id.
type BinaryMD5 struct {
	name string
}

// NewBinaryMD5 creates a new BinaryMD5.
func NewBinaryMD5(name string, _ map[string]string) (Vindex, error) {
	return &BinaryMD5{name: name}, nil
}

// String returns the name of the vindex.
func (vind *BinaryMD5) String() string {
	return vind.name
}

// Cost returns the cost as 1.
func (vind *BinaryMD5) Cost() int {
	return 1
}

// IsUnique returns true since the Vindex is unique.
func (vind *BinaryMD5) IsUnique() bool {
	return true
}

// NeedsVCursor satisfies the Vindex interface.
func (vind *BinaryMD5) NeedsVCursor() bool {
	return false
}

// Verify returns true if ids maps to ksids.
func (vind *BinaryMD5) Verify(_ VCursor, ids []sqltypes.Value, ksids [][]byte) ([]bool, error) {
	out := make([]bool, len(ids))
	for i := range ids {
		idBytes, err := ids[i].ToBytes()
		if err != nil {
			return out, err
		}
		out[i] = bytes.Equal(vMD5Hash(idBytes), ksids[i])
	}
	return out, nil
}

// Map can map ids to key.Destination objects.
func (vind *BinaryMD5) Map(cursor VCursor, ids []sqltypes.Value) ([]key.Destination, error) {
	out := make([]key.Destination, len(ids))
	for i, id := range ids {
		idBytes, err := id.ToBytes()
		if err != nil {
			return out, err
		}
		out[i] = key.DestinationKeyspaceID(vMD5Hash(idBytes))
	}
	return out, nil
}

func vMD5Hash(source []byte) []byte {
	sum := md5.Sum(source)
	return sum[:]
}

func init() {
	Register("binary_md5", NewBinaryMD5)
}
<commit_msg>feat: binarymd5 vindex implemented hashing interface<commit_after>/*
Copyright 2019 The Vitess Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package vindexes

import (
	"bytes"
	"crypto/md5"

	"vitess.io/vitess/go/sqltypes"
	"vitess.io/vitess/go/vt/key"
)

var (
	_ SingleColumn = (*BinaryMD5)(nil)
	_ Hashing      = (*BinaryMD5)(nil)
)

// BinaryMD5 is a vindex that hashes binary bits to a keyspace id.
type BinaryMD5 struct {
	name string
}

// NewBinaryMD5 creates a new BinaryMD5.
func NewBinaryMD5(name string, _ map[string]string) (Vindex, error) {
	return &BinaryMD5{name: name}, nil
}

// String returns the name of the vindex.
func (vind *BinaryMD5) String() string {
	return vind.name
}

// Cost returns the cost as 1.
func (vind *BinaryMD5) Cost() int {
	return 1
}

// IsUnique returns true since the Vindex is unique.
func (vind *BinaryMD5) IsUnique() bool {
	return true
}

// NeedsVCursor satisfies the Vindex interface.
func (vind *BinaryMD5) NeedsVCursor() bool {
	return false
}

// Verify returns true if ids maps to ksids.
func (vind *BinaryMD5) Verify(_ VCursor, ids []sqltypes.Value, ksids [][]byte) ([]bool, error) {
	out := make([]bool, 0, len(ids))
	for i, id := range ids {
		ksid, err := vind.Hash(id)
		if err != nil {
			return nil, err
		}
		out = append(out, bytes.Equal(ksid, ksids[i]))
	}
	return out, nil
}

// Map can map ids to key.Destination objects.
func (vind *BinaryMD5) Map(_ VCursor, ids []sqltypes.Value) ([]key.Destination, error) {
	out := make([]key.Destination, 0, len(ids))
	for _, id := range ids {
		ksid, err := vind.Hash(id)
		if err != nil {
			return out, err
		}
		out = append(out, key.DestinationKeyspaceID(ksid))
	}
	return out, nil
}

func (vind *BinaryMD5) Hash(id sqltypes.Value) ([]byte, error) {
	idBytes, err := id.ToBytes()
	if err != nil {
		return nil, err
	}
	return vMD5Hash(idBytes), nil
}

func vMD5Hash(source []byte) []byte {
	sum := md5.Sum(source)
	return sum[:]
}

func init() {
	Register("binary_md5", NewBinaryMD5)
}
<|endoftext|>