<commit_before>package retention

import (
	"context"
	"io/ioutil"
	"path/filepath"
	"testing"
	"time"

	"github.com/cortexproject/cortex/pkg/chunk"
	"github.com/cortexproject/cortex/pkg/chunk/local"
	cortex_storage "github.com/cortexproject/cortex/pkg/chunk/storage"
	chunk_util "github.com/cortexproject/cortex/pkg/chunk/util"
	util_log "github.com/cortexproject/cortex/pkg/util/log"
	"github.com/prometheus/client_golang/prometheus"
	"github.com/prometheus/common/model"
	"github.com/prometheus/prometheus/pkg/labels"
	"github.com/stretchr/testify/require"
	ww "github.com/weaveworks/common/server"
	"github.com/weaveworks/common/user"
	"go.etcd.io/bbolt"

	"github.com/grafana/loki/pkg/logql"
	"github.com/grafana/loki/pkg/storage"
	"github.com/grafana/loki/pkg/storage/stores/shipper"
	shipper_util "github.com/grafana/loki/pkg/storage/stores/shipper/util"
	"github.com/grafana/loki/pkg/validation"
)

func dayFromTime(t model.Time) chunk.DayTime {
	parsed, err := time.Parse("2006-01-02", t.Time().Format("2006-01-02"))
	if err != nil {
		panic(err)
	}
	return chunk.DayTime{
		Time: model.TimeFromUnix(parsed.Unix()),
	}
}

var (
	start     = model.Now().Add(-30 * 24 * time.Hour)
	schemaCfg = storage.SchemaConfig{
		SchemaConfig: chunk.SchemaConfig{
			// we want to test over all supported schema.
			Configs: []chunk.PeriodConfig{
				{
					From:       dayFromTime(start),
					IndexType:  "boltdb",
					ObjectType: "filesystem",
					Schema:     "v9",
					IndexTables: chunk.PeriodicTableConfig{
						Prefix: "index_",
						Period: time.Hour * 24,
					},
					RowShards: 16,
				},
				{
					From:       dayFromTime(start.Add(25 * time.Hour)),
					IndexType:  "boltdb",
					ObjectType: "filesystem",
					Schema:     "v10",
					IndexTables: chunk.PeriodicTableConfig{
						Prefix: "index_",
						Period: time.Hour * 24,
					},
					RowShards: 16,
				},
				{
					From:       dayFromTime(start.Add(49 * time.Hour)),
					IndexType:  "boltdb",
					ObjectType: "filesystem",
					Schema:     "v11",
					IndexTables: chunk.PeriodicTableConfig{
						Prefix: "index_",
						Period: time.Hour * 24,
					},
					RowShards: 16,
				},
			},
		},
	}
	allSchemas = []struct {
		schema string
		from   model.Time
		config chunk.PeriodConfig
	}{
		{"v9", schemaCfg.Configs[0].From.Time, schemaCfg.Configs[0]},
		{"v10", schemaCfg.Configs[1].From.Time, schemaCfg.Configs[1]},
		{"v11", schemaCfg.Configs[2].From.Time, schemaCfg.Configs[2]},
	}

	sweepMetrics = newSweeperMetrics(prometheus.DefaultRegisterer)
)

func newChunkEntry(userID, labels string, from, through model.Time) ChunkEntry {
	lbs, err := logql.ParseLabels(labels)
	if err != nil {
		panic(err)
	}
	return ChunkEntry{
		ChunkRef: ChunkRef{
			UserID:   []byte(userID),
			SeriesID: labelsSeriesID(lbs),
			From:     from,
			Through:  through,
		},
		Labels: lbs,
	}
}

type testStore struct {
	storage.Store
	cfg                storage.Config
	objectClient       chunk.ObjectClient
	indexDir, chunkDir string
	schemaCfg          storage.SchemaConfig
	t                  testing.TB
	limits             cortex_storage.StoreLimits
}

// testObjectClient is a testing object client
type testObjectClient struct {
	chunk.ObjectClient
	path string
}

func newTestObjectClient(path string) chunk.ObjectClient {
	c, err := cortex_storage.NewObjectClient("filesystem", cortex_storage.Config{
		FSConfig: local.FSConfig{
			Directory: path,
		},
	})
	if err != nil {
		panic(err)
	}
	return &testObjectClient{
		ObjectClient: c,
		path:         path,
	}
}

type table struct {
	name string
	*bbolt.DB
}

func (t *testStore) indexTables() []table {
	t.t.Helper()
	res := []table{}
	indexFilesInfo, err := ioutil.ReadDir(t.indexDir)
	require.NoError(t.t, err)
	for _, indexFileInfo := range indexFilesInfo {
		db, err := shipper_util.SafeOpenBoltdbFile(filepath.Join(t.indexDir, indexFileInfo.Name()))
		require.NoError(t.t, err)
		res = append(res, table{name: indexFileInfo.Name(), DB: db})
	}
	return res
}

func (t *testStore) HasChunk(c chunk.Chunk) bool {
	t.t.Helper()
	var matchers []*labels.Matcher
	for _, l := range c.Metric {
		matchers = append(matchers, labels.MustNewMatcher(labels.MatchEqual, l.Name, l.Value))
	}
	chunks, err := t.Store.Get(user.InjectOrgID(context.Background(), c.UserID),
		c.UserID, c.From, c.Through, matchers...)
	require.NoError(t.t, err)
	return len(chunks) == 1 && c.ExternalKey() == chunks[0].ExternalKey()
}

func (t *testStore) GetChunks(userID string, from, through model.Time, metric labels.Labels) []chunk.Chunk {
	t.t.Helper()
	var matchers []*labels.Matcher
	for _, l := range metric {
		matchers = append(matchers, labels.MustNewMatcher(labels.MatchEqual, l.Name, l.Value))
	}
	chunks, err := t.Store.Get(user.InjectOrgID(context.Background(), userID),
		userID, from, through, matchers...)
	require.NoError(t.t, err)

	return chunks
}

func (t *testStore) open() {
	chunkStore, err := cortex_storage.NewStore(
		t.cfg.Config,
		chunk.StoreConfig{},
		schemaCfg.SchemaConfig,
		t.limits,
		nil,
		nil,
		util_log.Logger,
	)
	require.NoError(t.t, err)

	store, err := storage.NewStore(t.cfg, schemaCfg, chunkStore, nil)
	require.NoError(t.t, err)
	t.Store = store
}

func newTestStore(t testing.TB) *testStore {
	t.Helper()
	cfg := &ww.Config{}
	require.Nil(t, cfg.LogLevel.Set("debug"))
	util_log.InitLogger(cfg)
	workdir := t.TempDir()
	filepath.Join(workdir, "index")
	indexDir := filepath.Join(workdir, "index")
	err := chunk_util.EnsureDirectory(indexDir)
	require.Nil(t, err)

	chunkDir := filepath.Join(workdir, "chunk_test")
	err = chunk_util.EnsureDirectory(indexDir)
	require.Nil(t, err)
	require.Nil(t, err)

	defer func() {
	}()
	limits, err := validation.NewOverrides(validation.Limits{}, nil)
	require.NoError(t, err)

	require.NoError(t, schemaCfg.SchemaConfig.Validate())

	config := storage.Config{
		Config: cortex_storage.Config{
			BoltDBConfig: local.BoltDBConfig{
				Directory: indexDir,
			},
			FSConfig: local.FSConfig{
				Directory: chunkDir,
			},
		},
		BoltDBShipperConfig: shipper.Config{
			ActiveIndexDirectory: indexDir,
			SharedStoreType:      "filesystem",
			SharedStoreKeyPrefix: "index",
			ResyncInterval:       1 * time.Millisecond,
			IngesterName:         "foo",
			Mode:                 shipper.ModeReadWrite,
		},
	}
	chunkStore, err := cortex_storage.NewStore(
		config.Config,
		chunk.StoreConfig{},
		schemaCfg.SchemaConfig,
		limits,
		nil,
		nil,
		util_log.Logger,
	)
	require.NoError(t, err)

	store, err := storage.NewStore(config, schemaCfg, chunkStore, nil)
	require.NoError(t, err)
	return &testStore{
		indexDir:     indexDir,
		chunkDir:     chunkDir,
		t:            t,
		Store:        store,
		schemaCfg:    schemaCfg,
		objectClient: newTestObjectClient(workdir),
		cfg:          config,
		limits:       limits,
	}
}
<commit_msg>fix flaky retention tests (#3976)<commit_after>package retention

import (
	"context"
	"io/ioutil"
	"path/filepath"
	"testing"
	"time"

	"github.com/cortexproject/cortex/pkg/chunk"
	"github.com/cortexproject/cortex/pkg/chunk/local"
	cortex_storage "github.com/cortexproject/cortex/pkg/chunk/storage"
	chunk_util "github.com/cortexproject/cortex/pkg/chunk/util"
	util_log "github.com/cortexproject/cortex/pkg/util/log"
	"github.com/prometheus/client_golang/prometheus"
	"github.com/prometheus/common/model"
	"github.com/prometheus/prometheus/pkg/labels"
	"github.com/stretchr/testify/require"
	ww "github.com/weaveworks/common/server"
	"github.com/weaveworks/common/user"
	"go.etcd.io/bbolt"

	"github.com/grafana/loki/pkg/logql"
	"github.com/grafana/loki/pkg/storage"
	"github.com/grafana/loki/pkg/storage/stores/shipper"
	shipper_util "github.com/grafana/loki/pkg/storage/stores/shipper/util"
	"github.com/grafana/loki/pkg/validation"
)

func dayFromTime(t model.Time) chunk.DayTime {
	parsed, err := time.Parse("2006-01-02", t.Time().Format("2006-01-02"))
	if err != nil {
		panic(err)
	}
	return chunk.DayTime{
		Time: model.TimeFromUnix(parsed.Unix()),
	}
}

var (
	start     = model.Now().Add(-30 * 24 * time.Hour)
	schemaCfg = storage.SchemaConfig{
		SchemaConfig: chunk.SchemaConfig{
			// we want to test over all supported schema.
			Configs: []chunk.PeriodConfig{
				{
					From:       dayFromTime(start),
					IndexType:  "boltdb",
					ObjectType: "filesystem",
					Schema:     "v9",
					IndexTables: chunk.PeriodicTableConfig{
						Prefix: "index_",
						Period: time.Hour * 24,
					},
					RowShards: 16,
				},
				{
					From:       dayFromTime(start.Add(25 * time.Hour)),
					IndexType:  "boltdb",
					ObjectType: "filesystem",
					Schema:     "v10",
					IndexTables: chunk.PeriodicTableConfig{
						Prefix: "index_",
						Period: time.Hour * 24,
					},
					RowShards: 16,
				},
				{
					From:       dayFromTime(start.Add(73 * time.Hour)),
					IndexType:  "boltdb",
					ObjectType: "filesystem",
					Schema:     "v11",
					IndexTables: chunk.PeriodicTableConfig{
						Prefix: "index_",
						Period: time.Hour * 24,
					},
					RowShards: 16,
				},
			},
		},
	}
	allSchemas = []struct {
		schema string
		from   model.Time
		config chunk.PeriodConfig
	}{
		{"v9", schemaCfg.Configs[0].From.Time, schemaCfg.Configs[0]},
		{"v10", schemaCfg.Configs[1].From.Time, schemaCfg.Configs[1]},
		{"v11", schemaCfg.Configs[2].From.Time, schemaCfg.Configs[2]},
	}

	sweepMetrics = newSweeperMetrics(prometheus.DefaultRegisterer)
)

func newChunkEntry(userID, labels string, from, through model.Time) ChunkEntry {
	lbs, err := logql.ParseLabels(labels)
	if err != nil {
		panic(err)
	}
	return ChunkEntry{
		ChunkRef: ChunkRef{
			UserID:   []byte(userID),
			SeriesID: labelsSeriesID(lbs),
			From:     from,
			Through:  through,
		},
		Labels: lbs,
	}
}

type testStore struct {
	storage.Store
	cfg                storage.Config
	objectClient       chunk.ObjectClient
	indexDir, chunkDir string
	schemaCfg          storage.SchemaConfig
	t                  testing.TB
	limits             cortex_storage.StoreLimits
}

// testObjectClient is a testing object client
type testObjectClient struct {
	chunk.ObjectClient
	path string
}

func newTestObjectClient(path string) chunk.ObjectClient {
	c, err := cortex_storage.NewObjectClient("filesystem", cortex_storage.Config{
		FSConfig: local.FSConfig{
			Directory: path,
		},
	})
	if err != nil {
		panic(err)
	}
	return &testObjectClient{
		ObjectClient: c,
		path:         path,
	}
}

type table struct {
	name string
	*bbolt.DB
}

func (t *testStore) indexTables() []table {
	t.t.Helper()
	res := []table{}
	indexFilesInfo, err := ioutil.ReadDir(t.indexDir)
	require.NoError(t.t, err)
	for _, indexFileInfo := range indexFilesInfo {
		db, err := shipper_util.SafeOpenBoltdbFile(filepath.Join(t.indexDir, indexFileInfo.Name()))
		require.NoError(t.t, err)
		res = append(res, table{name: indexFileInfo.Name(), DB: db})
	}
	return res
}

func (t *testStore) HasChunk(c chunk.Chunk) bool {
	chunks := t.GetChunks(c.UserID, c.From, c.Through, c.Metric)

	chunkIDs := make(map[string]struct{})
	for _, chk := range chunks {
		chunkIDs[chk.ExternalKey()] = struct{}{}
	}
	return len(chunkIDs) == 1 && c.ExternalKey() == chunks[0].ExternalKey()
}

func (t *testStore) GetChunks(userID string, from, through model.Time, metric labels.Labels) []chunk.Chunk {
	t.t.Helper()
	var matchers []*labels.Matcher
	for _, l := range metric {
		matchers = append(matchers, labels.MustNewMatcher(labels.MatchEqual, l.Name, l.Value))
	}
	chunks, err := t.Store.Get(user.InjectOrgID(context.Background(), userID),
		userID, from, through, matchers...)
	require.NoError(t.t, err)

	return chunks
}

func (t *testStore) open() {
	chunkStore, err := cortex_storage.NewStore(
		t.cfg.Config,
		chunk.StoreConfig{},
		schemaCfg.SchemaConfig,
		t.limits,
		nil,
		nil,
		util_log.Logger,
	)
	require.NoError(t.t, err)

	store, err := storage.NewStore(t.cfg, schemaCfg, chunkStore, nil)
	require.NoError(t.t, err)
	t.Store = store
}

func newTestStore(t testing.TB) *testStore {
	t.Helper()
	cfg := &ww.Config{}
	require.Nil(t, cfg.LogLevel.Set("debug"))
	util_log.InitLogger(cfg)
	workdir := t.TempDir()
	filepath.Join(workdir, "index")
	indexDir := filepath.Join(workdir, "index")
	err := chunk_util.EnsureDirectory(indexDir)
	require.Nil(t, err)

	chunkDir := filepath.Join(workdir, "chunk_test")
	err = chunk_util.EnsureDirectory(indexDir)
	require.Nil(t, err)
	require.Nil(t, err)

	defer func() {
	}()
	limits, err := validation.NewOverrides(validation.Limits{}, nil)
	require.NoError(t, err)

	require.NoError(t, schemaCfg.SchemaConfig.Validate())

	config := storage.Config{
		Config: cortex_storage.Config{
			BoltDBConfig: local.BoltDBConfig{
				Directory: indexDir,
			},
			FSConfig: local.FSConfig{
				Directory: chunkDir,
			},
		},
		BoltDBShipperConfig: shipper.Config{
			ActiveIndexDirectory: indexDir,
			SharedStoreType:      "filesystem",
			SharedStoreKeyPrefix: "index",
			ResyncInterval:       1 * time.Millisecond,
			IngesterName:         "foo",
			Mode:                 shipper.ModeReadWrite,
		},
	}
	chunkStore, err := cortex_storage.NewStore(
		config.Config,
		chunk.StoreConfig{},
		schemaCfg.SchemaConfig,
		limits,
		nil,
		nil,
		util_log.Logger,
	)
	require.NoError(t, err)

	store, err := storage.NewStore(config, schemaCfg, chunkStore, nil)
	require.NoError(t, err)
	return &testStore{
		indexDir:     indexDir,
		chunkDir:     chunkDir,
		t:            t,
		Store:        store,
		schemaCfg:    schemaCfg,
		objectClient: newTestObjectClient(workdir),
		cfg:          config,
		limits:       limits,
	}
}
<|endoftext|>


<commit_before>package host

import (
	"net"
	"sync/atomic"
	"time"

	"github.com/NebulousLabs/Sia/build"
	"github.com/NebulousLabs/Sia/encoding"
	"github.com/NebulousLabs/Sia/modules"
	"github.com/NebulousLabs/Sia/types"
)

// initNetworking performs actions like port forwarding, and gets the host
// established on the network.
func (h *Host) initNetworking(address string) error {
	// Create listener and set address.
	var err error
	h.listener, err = net.Listen("tcp", address)
	if err != nil {
		return err
	}
	h.netAddress = modules.NetAddress(h.listener.Addr().String())

	// Non-blocking, perform port forwarding and hostname discovery.
	go func() {
		h.resourceLock.RLock()
		defer h.resourceLock.RUnlock()
		if h.closed {
			return
		}

		h.mu.RLock()
		port := h.netAddress.Port()
		h.mu.RUnlock()
		err := h.forwardPort(port)
		if err != nil {
			h.log.Println("ERROR: failed to forward port:", err)
		}
		h.learnHostname()
	}()

	// Launch the listener.
	go h.threadedListen()
	return nil
}

// threadedHandleConn handles an incoming connection to the host, typically an
// RPC.
func (h *Host) threadedHandleConn(conn net.Conn) {
	h.resourceLock.RLock()
	defer h.resourceLock.RUnlock()
	if h.closed {
		return
	}

	// Set an initial duration that is generous, but finite. RPCs can extend
	// this if desired.
	err := conn.SetDeadline(time.Now().Add(5 * time.Minute))
	if err != nil {
		h.log.Println("WARN: could not set deadline on connection:", err)
		return
	}
	defer conn.Close()

	// Read a specifier indicating which action is beeing called.
	var id types.Specifier
	if err := encoding.ReadObject(conn, &id, 16); err != nil {
		atomic.AddUint64(&h.atomicUnrecognizedCalls, 1)
		atomic.AddUint64(&h.atomicErroredCalls, 1)
		h.log.Printf("WARN: incoming conn %v was malformed", conn.RemoteAddr())
		return
	}

	switch id {
	case modules.RPCDownload:
		atomic.AddUint64(&h.atomicDownloadCalls, 1)
		err = h.managedRPCDownload(conn)
	case modules.RPCRenew:
		atomic.AddUint64(&h.atomicRenewCalls, 1)
		err = h.managedRPCRenew(conn)
	case modules.RPCRevise:
		atomic.AddUint64(&h.atomicReviseCalls, 1)
		err = h.managedRPCRevise(conn)
	case modules.RPCSettings:
		atomic.AddUint64(&h.atomicSettingsCalls, 1)
		err = h.managedRPCSettings(conn)
	case modules.RPCUpload:
		atomic.AddUint64(&h.atomicUploadCalls, 1)
		err = h.managedRPCUpload(conn)
	default:
		atomic.AddUint64(&h.atomicErroredCalls, 1)
		h.log.Printf("WARN: incoming conn %v requested unknown RPC \"%v\"", conn.RemoteAddr(), id)
		return
	}
	if err != nil {
		atomic.AddUint64(&h.atomicErroredCalls, 1)
		h.log.Printf("WARN: incoming RPC \"%v\" failed: %v", id, err)
	}
}

// listen listens for incoming RPCs and spawns an appropriate handler for each.
func (h *Host) threadedListen() {
	h.resourceLock.RLock()
	defer h.resourceLock.RUnlock()
	if build.DEBUG && h.closed {
		panic("threaded listen does not have access to host resources - is only called at startup")
	}

	// Receive connections until an error is returned by the listener. When an
	// error is returned, there will be no more calls to receive.
	for {
		// Block until there is a connection to handle.
		conn, err := h.listener.Accept()
		if err != nil {
			return
		}

		// Grab the resource lock before creating a goroutine.
		go h.threadedHandleConn(conn)
	}
}

// managedRPCSettings is an rpc that returns the host's settings.
func (h *Host) managedRPCSettings(conn net.Conn) error {
	h.mu.RLock()
	defer h.mu.RUnlock()
	return encoding.WriteObject(conn, h.Settings())
}
<commit_msg>declutter some of the logging<commit_after>package host

import (
	"net"
	"sync/atomic"
	"time"

	"github.com/NebulousLabs/Sia/build"
	"github.com/NebulousLabs/Sia/crypto"
	"github.com/NebulousLabs/Sia/encoding"
	"github.com/NebulousLabs/Sia/modules"
	"github.com/NebulousLabs/Sia/types"
)

// initNetworking performs actions like port forwarding, and gets the host
// established on the network.
func (h *Host) initNetworking(address string) error {
	// Create listener and set address.
	var err error
	h.listener, err = net.Listen("tcp", address)
	if err != nil {
		return err
	}
	h.netAddress = modules.NetAddress(h.listener.Addr().String())

	// Non-blocking, perform port forwarding and hostname discovery.
	go func() {
		h.resourceLock.RLock()
		defer h.resourceLock.RUnlock()
		if h.closed {
			return
		}

		h.mu.RLock()
		port := h.netAddress.Port()
		h.mu.RUnlock()
		err := h.forwardPort(port)
		if err != nil {
			h.log.Println("ERROR: failed to forward port:", err)
		}
		h.learnHostname()
	}()

	// Launch the listener.
	go h.threadedListen()
	return nil
}

// threadedHandleConn handles an incoming connection to the host, typically an
// RPC.
func (h *Host) threadedHandleConn(conn net.Conn) {
	h.resourceLock.RLock()
	defer h.resourceLock.RUnlock()
	if h.closed {
		return
	}

	// Set an initial duration that is generous, but finite. RPCs can extend
	// this if desired.
	err := conn.SetDeadline(time.Now().Add(5 * time.Minute))
	if err != nil {
		h.log.Println("WARN: could not set deadline on connection:", err)
		return
	}
	defer conn.Close()

	// Read a specifier indicating which action is beeing called.
	var id types.Specifier
	if err := encoding.ReadObject(conn, &id, 16); err != nil {
		atomic.AddUint64(&h.atomicUnrecognizedCalls, 1)
		atomic.AddUint64(&h.atomicErroredCalls, 1)

		// Don't clutter the logs with repeat messages - after 1000 messages
		// have been printed, only print 1-in-200.
		randInt, err := crypto.RandIntn(200)
		if err != nil {
			return
		}
		unrecognizedCalls := atomic.LoadUint64(&h.atomicUnrecognizedCalls)
		if unrecognizedCalls < 1e3 || (unrecognizedCalls > 1e3 && randInt == 0) {
			h.log.Printf("WARN: incoming conn %v was malformed", conn.RemoteAddr())
		}
		return
	}

	switch id {
	case modules.RPCDownload:
		atomic.AddUint64(&h.atomicDownloadCalls, 1)
		err = h.managedRPCDownload(conn)
	case modules.RPCRenew:
		atomic.AddUint64(&h.atomicRenewCalls, 1)
		err = h.managedRPCRenew(conn)
	case modules.RPCRevise:
		atomic.AddUint64(&h.atomicReviseCalls, 1)
		err = h.managedRPCRevise(conn)
	case modules.RPCSettings:
		atomic.AddUint64(&h.atomicSettingsCalls, 1)
		err = h.managedRPCSettings(conn)
	case modules.RPCUpload:
		atomic.AddUint64(&h.atomicUploadCalls, 1)
		err = h.managedRPCUpload(conn)
	default:
		atomic.AddUint64(&h.atomicErroredCalls, 1)

		// Don't clutter the logs with repeat messages - after 1000 messages
		// have been printed, only print 1-in-200.
		randInt, err := crypto.RandIntn(200)
		if err != nil {
			return
		}
		erroredCalls := atomic.LoadUint64(&h.atomicErroredCalls)
		if erroredCalls < 1e3 || (erroredCalls > 1e3 && randInt == 0) {
			h.log.Printf("WARN: incoming conn %v requested unknown RPC \"%v\"", conn.RemoteAddr(), id)
		}
		return
	}
	if err != nil {
		atomic.AddUint64(&h.atomicErroredCalls, 1)

		// Don't clutter the logs with repeat messages - after 1000 messages
		// have been printed, only print 1-in-200.
		randInt, err := crypto.RandIntn(200)
		if err != nil {
			return
		}
		erroredCalls := atomic.LoadUint64(&h.atomicErroredCalls)
		if erroredCalls < 1e3 || (erroredCalls > 1e3 && randInt == 0) {
			h.log.Printf("WARN: incoming RPC \"%v\" failed: %v", id, err)
		}
	}
}

// listen listens for incoming RPCs and spawns an appropriate handler for each.
func (h *Host) threadedListen() {
	h.resourceLock.RLock()
	defer h.resourceLock.RUnlock()
	if build.DEBUG && h.closed {
		panic("threaded listen does not have access to host resources - is only called at startup")
	}

	// Receive connections until an error is returned by the listener. When an
	// error is returned, there will be no more calls to receive.
	for {
		// Block until there is a connection to handle.
		conn, err := h.listener.Accept()
		if err != nil {
			return
		}

		// Grab the resource lock before creating a goroutine.
		go h.threadedHandleConn(conn)
	}
}

// managedRPCSettings is an rpc that returns the host's settings.
func (h *Host) managedRPCSettings(conn net.Conn) error {
	h.mu.RLock()
	defer h.mu.RUnlock()
	return encoding.WriteObject(conn, h.Settings())
}
<|endoftext|>


<commit_before>package miner

import (
	"github.com/NebulousLabs/Sia/build"
	"github.com/NebulousLabs/Sia/encoding"
	"github.com/NebulousLabs/Sia/modules"
	"github.com/NebulousLabs/Sia/types"
)

// ReceiveTransactionPoolUpdate listens to the transaction pool for changes in
// the transaction pool. These changes will be applied to the blocks being
// mined.
func (m *Miner) ReceiveTransactionPoolUpdate(cc modules.ConsensusChange, unconfirmedTransactions []types.Transaction, _ []modules.SiacoinOutputDiff) {
	lockID := m.mu.Lock()
	defer m.mu.Unlock(lockID)
	defer m.notifySubscribers()

	m.height -= types.BlockHeight(len(cc.RevertedBlocks))
	m.height += types.BlockHeight(len(cc.AppliedBlocks))

	// The total encoded size of the transactions cannot exceed the block size.
	m.transactions = nil
	remainingSize := int(types.BlockSizeLimit - 5e3)
	for {
		if len(unconfirmedTransactions) == 0 {
			break
		}
		remainingSize -= len(encoding.Marshal(unconfirmedTransactions[0]))
		if remainingSize < 0 {
			break
		}

		m.transactions = append(m.transactions, unconfirmedTransactions[0])
		unconfirmedTransactions = unconfirmedTransactions[1:]
	}

	// If no blocks have been applied, the block variables do not need to be
	// updated.
	if len(cc.AppliedBlocks) == 0 {
		if build.DEBUG {
			if len(cc.RevertedBlocks) != 0 {
				panic("blocks reverted without being added")
			}
		}
		return
	}

	// Update the parent, target, and earliest timestamp fields for the miner.
	m.parent = cc.AppliedBlocks[len(cc.AppliedBlocks)-1].ID()
	target, exists1 := m.cs.ChildTarget(m.parent)
	timestamp, exists2 := m.cs.EarliestChildTimestamp(m.parent)
	if build.DEBUG {
		if !exists1 {
			panic("could not get child target")
		}
		if !exists2 {
			panic("could not get child earliest timestamp")
		}
	}
	m.target = target
	m.earliestTimestamp = timestamp
}
<commit_msg>New block upon consensus update<commit_after>package miner

import (
	"github.com/NebulousLabs/Sia/build"
	"github.com/NebulousLabs/Sia/encoding"
	"github.com/NebulousLabs/Sia/modules"
	"github.com/NebulousLabs/Sia/types"
)

// ReceiveTransactionPoolUpdate listens to the transaction pool for changes in
// the transaction pool. These changes will be applied to the blocks being
// mined.
func (m *Miner) ReceiveTransactionPoolUpdate(cc modules.ConsensusChange, unconfirmedTransactions []types.Transaction, _ []modules.SiacoinOutputDiff) {
	lockID := m.mu.Lock()
	defer m.mu.Unlock(lockID)
	defer m.notifySubscribers()

	m.height -= types.BlockHeight(len(cc.RevertedBlocks))
	m.height += types.BlockHeight(len(cc.AppliedBlocks))

	// The total encoded size of the transactions cannot exceed the block size.
	m.transactions = nil
	remainingSize := int(types.BlockSizeLimit - 5e3)
	for {
		if len(unconfirmedTransactions) == 0 {
			break
		}
		remainingSize -= len(encoding.Marshal(unconfirmedTransactions[0]))
		if remainingSize < 0 {
			break
		}

		m.transactions = append(m.transactions, unconfirmedTransactions[0])
		unconfirmedTransactions = unconfirmedTransactions[1:]
	}

	// If no blocks have been applied, the block variables do not need to be
	// updated.
	if len(cc.AppliedBlocks) == 0 {
		if build.DEBUG {
			if len(cc.RevertedBlocks) != 0 {
				panic("blocks reverted without being added")
			}
		}
		return
	}

	// Update the parent, target, and earliest timestamp fields for the miner.
	m.parent = cc.AppliedBlocks[len(cc.AppliedBlocks)-1].ID()
	target, exists1 := m.cs.ChildTarget(m.parent)
	timestamp, exists2 := m.cs.EarliestChildTimestamp(m.parent)
	if build.DEBUG {
		if !exists1 {
			panic("could not get child target")
		}
		if !exists2 {
			panic("could not get child earliest timestamp")
		}
	}
	m.target = target
	m.earliestTimestamp = timestamp
	m.prepareNewBlock()
}
<|endoftext|>


<commit_before>package testutil

import (
	"fmt"
	"os"
	"os/exec"
	"runtime"
	"testing"

	"github.com/hashicorp/nomad/nomad/structs"
	"github.com/hashicorp/nomad/nomad/structs/config"
	vapi "github.com/hashicorp/vault/api"
)

// TestVault is a test helper. It uses a fork/exec model to create a test Vault
// server instance in the background and can be initialized with policies, roles
// and backends mounted. The test Vault instances can be used to run a unit test
// and offers and easy API to tear itself down on test end. The only
// prerequisite is that the Vault binary is on the $PATH.

const (
	// vaultStartPort is the starting port we use to bind Vault servers to
	vaultStartPort uint64 = 40000
)

// vaultPortOffset is used to atomically increment the port numbers.
var vaultPortOffset uint64

// TestVault wraps a test Vault server launched in dev mode, suitable for
// testing.
type TestVault struct {
	cmd *exec.Cmd
	t   *testing.T

	Addr      string
	HTTPAddr  string
	RootToken string
	Config    *config.VaultConfig
	Client    *vapi.Client
}

// NewTestVault returns a new TestVault instance that has yet to be started
func NewTestVault(t *testing.T) *TestVault {
	port := getPort()
	token := structs.GenerateUUID()
	bind := fmt.Sprintf("-dev-listen-address=127.0.0.1:%d", port)
	http := fmt.Sprintf("http://127.0.0.1:%d", port)
	root := fmt.Sprintf("-dev-root-token-id=%s", token)

	bin := "vault"
	if runtime.GOOS == "windows" {
		bin = "vault.exe"
	}
	cmd := exec.Command(bin, "server", "-dev", bind, root)
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr

	// Build the config
	conf := vapi.DefaultConfig()
	conf.Address = http

	// Make the client and set the token to the root token
	client, err := vapi.NewClient(conf)
	if err != nil {
		t.Fatalf("failed to build Vault API client: %v", err)
	}
	client.SetToken(token)

	enable := true
	tv := &TestVault{
		cmd:       cmd,
		t:         t,
		Addr:      bind,
		HTTPAddr:  http,
		RootToken: token,
		Client:    client,
		Config: &config.VaultConfig{
			Enabled: &enable,
			Token:   token,
			Addr:    http,
		},
	}

	return tv
}

// Start starts the test Vault server and waits for it to respond to its HTTP
// API
func (tv *TestVault) Start() *TestVault {
	if err := tv.cmd.Start(); err != nil {
		tv.t.Fatalf("failed to start vault: %v", err)
	}

	tv.waitForAPI()
	return tv
}

// Stop stops the test Vault server
func (tv *TestVault) Stop() {
	if tv.cmd.Process == nil {
		return
	}

	if err := tv.cmd.Process.Kill(); err != nil {
		tv.t.Errorf("err: %s", err)
	}
	tv.cmd.Wait()
}

// waitForAPI waits for the Vault HTTP endpoint to start
// responding. This is an indication that the agent has started.
func (tv *TestVault) waitForAPI() {
	WaitForResult(func() (bool, error) {
		inited, err := tv.Client.Sys().InitStatus()
		if err != nil {
			return false, err
		}
		return inited, nil
	}, func(err error) {
		defer tv.Stop()
		tv.t.Fatalf("err: %s", err)
	})
}

// getPort returns the next available port to bind Vault against
func getPort() uint64 {
	p := vaultStartPort + vaultPortOffset
	vaultPortOffset += 1
	return p
}

// VaultVersion returns the Vault version as a string or an error if it couldn't
// be determined
func VaultVersion() (string, error) {
	cmd := exec.Command("vault", "version")
	out, err := cmd.Output()
	return string(out), err
}
<commit_msg>Use a lock to increment the vault port<commit_after>package testutil

import (
	"fmt"
	"os"
	"os/exec"
	"runtime"
	"sync"
	"testing"

	"github.com/hashicorp/nomad/nomad/structs"
	"github.com/hashicorp/nomad/nomad/structs/config"
	vapi "github.com/hashicorp/vault/api"
)

// TestVault is a test helper. It uses a fork/exec model to create a test Vault
// server instance in the background and can be initialized with policies, roles
// and backends mounted. The test Vault instances can be used to run a unit test
// and offers and easy API to tear itself down on test end. The only
// prerequisite is that the Vault binary is on the $PATH.

const (
	// vaultStartPort is the starting port we use to bind Vault servers to
	vaultStartPort uint64 = 40000
)

var (
	// vaultPortOffset is used to atomically increment the port numbers.
	vaultPortOffset uint64
	vaultPortLock   sync.Mutex
)

// TestVault wraps a test Vault server launched in dev mode, suitable for
// testing.
type TestVault struct {
	cmd *exec.Cmd
	t   *testing.T

	Addr      string
	HTTPAddr  string
	RootToken string
	Config    *config.VaultConfig
	Client    *vapi.Client
}

// NewTestVault returns a new TestVault instance that has yet to be started
func NewTestVault(t *testing.T) *TestVault {
	port := getPort()
	token := structs.GenerateUUID()
	bind := fmt.Sprintf("-dev-listen-address=127.0.0.1:%d", port)
	http := fmt.Sprintf("http://127.0.0.1:%d", port)
	root := fmt.Sprintf("-dev-root-token-id=%s", token)

	bin := "vault"
	if runtime.GOOS == "windows" {
		bin = "vault.exe"
	}
	cmd := exec.Command(bin, "server", "-dev", bind, root)
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr

	// Build the config
	conf := vapi.DefaultConfig()
	conf.Address = http

	// Make the client and set the token to the root token
	client, err := vapi.NewClient(conf)
	if err != nil {
		t.Fatalf("failed to build Vault API client: %v", err)
	}
	client.SetToken(token)

	enable := true
	tv := &TestVault{
		cmd:       cmd,
		t:         t,
		Addr:      bind,
		HTTPAddr:  http,
		RootToken: token,
		Client:    client,
		Config: &config.VaultConfig{
			Enabled: &enable,
			Token:   token,
			Addr:    http,
		},
	}

	return tv
}

// Start starts the test Vault server and waits for it to respond to its HTTP
// API
func (tv *TestVault) Start() *TestVault {
	if err := tv.cmd.Start(); err != nil {
		tv.t.Fatalf("failed to start vault: %v", err)
	}

	tv.waitForAPI()
	return tv
}

// Stop stops the test Vault server
func (tv *TestVault) Stop() {
	if tv.cmd.Process == nil {
		return
	}

	if err := tv.cmd.Process.Kill(); err != nil {
		tv.t.Errorf("err: %s", err)
	}
	tv.cmd.Wait()
}

// waitForAPI waits for the Vault HTTP endpoint to start
// responding. This is an indication that the agent has started.
func (tv *TestVault) waitForAPI() {
	WaitForResult(func() (bool, error) {
		inited, err := tv.Client.Sys().InitStatus()
		if err != nil {
			return false, err
		}
		return inited, nil
	}, func(err error) {
		defer tv.Stop()
		tv.t.Fatalf("err: %s", err)
	})
}

// getPort returns the next available port to bind Vault against
func getPort() uint64 {
	vaultPortLock.Lock()
	defer vaultPortLock.Unlock()
	p := vaultStartPort + vaultPortOffset
	vaultPortOffset += 1
	return p
}

// VaultVersion returns the Vault version as a string or an error if it couldn't
// be determined
func VaultVersion() (string, error) {
	cmd := exec.Command("vault", "version")
	out, err := cmd.Output()
	return string(out), err
}
<|endoftext|>


<commit_before>package secret

import (
	"testing"
)

func TestXor(t *testing.T) {
	batch := []struct {
		clear, key, cipher []byte
	}{
		{[]byte{4}, []byte{0}, []byte{4}},
		{[]byte{2}, []byte{2}, []byte{0}},
		{
			[]byte{1, 2, 3, 5, 6, 7},
			[]byte{212, 16, 24, 32, 68, 44},
			[]byte{213, 18, 27, 37, 66, 43},
		},
	}
	x := NewXor()
	for _, b := range batch {
		c := x.Encrypt(b.key, b.clear)
		for k := range b.cipher {
			if c[k] != b.cipher[k] {
				t.Errorf("expect byte %v at pos %d, got %v", b.cipher[k], k, c[k])
			}
		}
	}
}
<commit_msg>UT for XOR decryption<commit_after>package secret

import (
	"testing"
)

func TestXor(t *testing.T) {
	batch := []struct {
		clear, key, cipher []byte
	}{
		{[]byte{4}, []byte{0}, []byte{4}},
		{[]byte{2}, []byte{2}, []byte{0}},
		{
			[]byte{1, 2, 3, 5, 6, 7},
			[]byte{212, 16, 24, 32, 68, 44},
			[]byte{213, 18, 27, 37, 66, 43},
		},
	}
	x := NewXor()
	for _, b := range batch {
		c := x.Encrypt(b.key, b.clear)
		for k := range b.cipher {
			if c[k] != b.cipher[k] {
				t.Errorf("encrypt: expect byte %v at pos %d, got %v", b.cipher[k], k, c[k])
			}
		}
		d := x.Decrypt(b.key, b.cipher)
		for k := range b.clear {
			if d[k] != b.clear[k] {
				t.Errorf("decrypt: expect byte %v at pos %d, got %v", b.clear[k], k, d[k])
			}
		}
	}
}
<|endoftext|>


<commit_before>package log551
import (
	"os"
	"fmt"
	"time"
	"runtime"
	"path"
)

type logLevel int

const (
	DEBUG logLevel = iota
	INFORMATION
	WARNING
	ERROR
	CRITICAL
)

func (ll logLevel) String() string {
	switch ll {
	case DEBUG:
		return "DEBUG"
	case INFORMATION:
		return "INFORMATION"
	case WARNING:
		return "WARNING"
	case ERROR:
		return "ERROR"
	case CRITICAL:
		return "CRITICAL"
	}
	return "NONE"
}

type Config struct {
	Debug       ConfigDetail `json:"debug"`
	Information ConfigDetail `json:"information"`
	Warning     ConfigDetail `json:"warning"`
	Error       ConfigDetail `json:"error"`
	Critical    ConfigDetail `json:"critical"`
	Path        string `json:"path"`
}

type ConfigDetail struct {
	EachTime bool `json:"each_time"`
	Cache    bool `json:"cache"`
}

type Log551 struct {
	config *Config
	file   *os.File
	caches []*cache
}

type cache struct {
	datetime time.Time
	file     string
	line     int
	level    logLevel
	format   string
	a        []interface{}
}

func New(config *Config) *Log551 {
	return &Log551{
		config:config,
		caches:make([]*cache, 0),
	}

}

func (l *Log551) Open() {
	if l.file != nil {
		return
	}

	file, err := os.OpenFile(l.config.Path, os.O_RDWR | os.O_CREATE | os.O_APPEND, 0666)
	if err != nil {
		panic(err)
	}

	l.file = file
}

func (l *Log551) Close() {
	l.outputCacheLog()

	err := l.file.Close()
	if err != nil {
		panic(err)
	}

	l.caches = nil
	l.file = nil
}

func (l *Log551) Debug(a interface{}) {
	_, file, line, _ := runtime.Caller(1)
	l.output(DEBUG, file, line, "%v", a)
	l.addCache(DEBUG, file, line, "%v", a)
}

func (l *Log551) Debugf(format string, a ...interface{}) {
	_, file, line, _ := runtime.Caller(1)
	l.output(DEBUG, file, line, format, a...)
	l.addCache(DEBUG, file, line, format, a...)
}

func (l *Log551) Debugln(a interface{}) {
	_, file, line, _ := runtime.Caller(1)
	l.output(DEBUG, file, line, "%v", a)
	l.addCache(DEBUG, file, line, "%v", a)
}

func (l *Log551) Information(a interface{}) {
	_, file, line, _ := runtime.Caller(1)
	l.output(INFORMATION, file, line, "%v", a)
	l.addCache(INFORMATION, file, line, "%v", a)
}

func (l *Log551) Informationf(format string, a ...interface{}) {
	_, file, line, _ := runtime.Caller(1)
	l.output(INFORMATION, file, line, format, a...)
	l.addCache(INFORMATION, file, line, format, a...)
}

func (l *Log551) Informationln(a interface{}) {
	_, file, line, _ := runtime.Caller(1)
	l.output(INFORMATION, file, line, "%v", a)
	l.addCache(INFORMATION, file, line, "%v", a)
}

func (l *Log551) Warning(a interface{}) {
	_, file, line, _ := runtime.Caller(1)
	l.output(WARNING, file, line, "%v", a)
	l.addCache(WARNING, file, line, "%v", a)
}

func (l *Log551) Warningf(format string, a ...interface{}) {
	_, file, line, _ := runtime.Caller(1)
	l.output(WARNING, file, line, format, a...)
	l.addCache(WARNING, file, line, format, a...)
}

func (l *Log551) Warningln(a interface{}) {
	_, file, line, _ := runtime.Caller(1)
	l.output(WARNING, file, line, "%v", a)
	l.addCache(WARNING, file, line, "%v", a)
}

func (l *Log551) Error(a interface{}) {
	_, file, line, _ := runtime.Caller(1)
	l.output(ERROR, file, line, "%v", a)
	l.addCache(ERROR, file, line, "%v", a)
}

func (l *Log551) Errorf(format string, a ...interface{}) {
	_, file, line, _ := runtime.Caller(1)
	l.output(ERROR, file, line, format, a...)
	l.addCache(ERROR, file, line, format, a...)
}

func (l *Log551) Errorln(a interface{}) {
	_, file, line, _ := runtime.Caller(1)
	l.output(ERROR, file, line, "%v", a)
	l.addCache(ERROR, file, line, "%v", a)
}

func (l *Log551) Critical(a interface{}) {
	_, file, line, _ := runtime.Caller(1)
	l.output(CRITICAL, file, line, "%v", a)
	l.addCache(CRITICAL, file, line, "%v", a)
}

func (l *Log551) Criticalf(format string, a ...interface{}) {
	_, file, line, _ := runtime.Caller(1)
	l.output(CRITICAL, file, line, format, a...)
	l.addCache(CRITICAL, file, line, format, a...)
}

func (l *Log551) Criticalln(a interface{}) {
	_, file, line, _ := runtime.Caller(1)
	l.output(CRITICAL, file, line, "%v", a)
	l.addCache(CRITICAL, file, line, "%v", a)
}

func (l *Log551) output(level logLevel, file string, line int, format string, a ...interface{}) {
	if ! l.isOutput(level) {
		return
	}

	param := []interface{}{
		time.Now().Format("2006/01/02 15:04:05.00000"),
		path.Base(file),
		line,
		level.String(),
	}
	param = append(param, a...)

	fmt.Fprintf(l.file, "%s %s %d [%s] " + format + "\n", param...)

}

func (l *Log551) isOutput(level logLevel) bool {
	if !l.config.Debug.EachTime && level == DEBUG {
		return false
	}
	if !l.config.Information.EachTime && level == INFORMATION {
		return false
	}
	if !l.config.Warning.EachTime && level == WARNING {
		return false
	}
	if !l.config.Error.EachTime && level == ERROR {
		return false
	}
	if !l.config.Critical.EachTime && level == CRITICAL {
		return false
	}

	return true
}

func (l *Log551) addCache(level logLevel, file string, line int, format string, a ...interface{}) {
	if ! l.isCache(level) {
		return
	}

	cache := &cache{
		datetime: time.Now(),
		file: file,
		line: line,
		level: level,
		format: format,
		a: a,
	}

	l.caches = append(l.caches, cache)

}

func (l *Log551) isCache(level logLevel) bool {
	if !l.config.Debug.Cache && level == DEBUG {
		return false
	}
	if !l.config.Information.Cache && level == INFORMATION {
		return false
	}
	if !l.config.Warning.Cache && level == WARNING {
		return false
	}
	if !l.config.Error.Cache && level == ERROR {
		return false
	}
	if !l.config.Critical.Cache && level == CRITICAL {
		return false
	}

	return true
}

func (l *Log551) outputCacheLog() {
	if len(l.caches) == 0 {
		return
	}

	for _, v := range l.caches {
		param := []interface{}{
			v.datetime.Format("2006/01/02 15:04:05.00000"),
			path.Base(v.file),
			v.line,
			v.level.String(),
		}
		param = append(param, v.a...)
		fmt.Fprintf(l.file, "%s %s %d [%s] " + v.format + "\n", param...)
	}
}<commit_msg>refs #13 出力内容にパッケージ名を出力する<commit_after>package log551
import (
	"os"
	"fmt"
	"time"
	"runtime"
	"path"
	"regexp"
)

type logLevel int

const (
	DEBUG logLevel = iota
	INFORMATION
	WARNING
	ERROR
	CRITICAL
)

func (ll logLevel) String() string {
	switch ll {
	case DEBUG:
		return "DEBUG"
	case INFORMATION:
		return "INFORMATION"
	case WARNING:
		return "WARNING"
	case ERROR:
		return "ERROR"
	case CRITICAL:
		return "CRITICAL"
	}
	return "NONE"
}

type Config struct {
	Debug       ConfigDetail `json:"debug"`
	Information ConfigDetail `json:"information"`
	Warning     ConfigDetail `json:"warning"`
	Error       ConfigDetail `json:"error"`
	Critical    ConfigDetail `json:"critical"`
	Path        string `json:"path"`
}

type ConfigDetail struct {
	EachTime bool `json:"each_time"`
	Cache    bool `json:"cache"`
}

type Log551 struct {
	config *Config
	file   *os.File
	caches []*cache
}

type cache struct {
	datetime    time.Time
	packageName string
	file        string
	line        int
	level       logLevel
	format      string
	a           []interface{}
}

func New(config *Config) *Log551 {
	return &Log551{
		config:config,
		caches:make([]*cache, 0),
	}

}

func (l *Log551) Open() {
	if l.file != nil {
		return
	}

	file, err := os.OpenFile(l.config.Path, os.O_RDWR | os.O_CREATE | os.O_APPEND, 0666)
	if err != nil {
		panic(err)
	}

	l.file = file
}

func (l *Log551) Close() {
	l.outputCacheLog()

	err := l.file.Close()
	if err != nil {
		panic(err)
	}

	l.caches = nil
	l.file = nil
}

func (l *Log551) Debug(a interface{}) {
	pc, file, line, _ := runtime.Caller(1)
	packageName := l.getPackageName(runtime.FuncForPC(pc).Name())
	l.output(DEBUG, packageName, file, line, "%v", a)
	l.addCache(DEBUG, packageName, file, line, "%v", a)
}

func (l *Log551) Debugf(format string, a ...interface{}) {
	pc, file, line, _ := runtime.Caller(1)
	packageName := l.getPackageName(runtime.FuncForPC(pc).Name())
	l.output(DEBUG, packageName, file, line, format, a...)
	l.addCache(DEBUG, packageName, file, line, format, a...)
}

func (l *Log551) Debugln(a interface{}) {
	pc, file, line, _ := runtime.Caller(1)
	packageName := l.getPackageName(runtime.FuncForPC(pc).Name())
	l.output(DEBUG, packageName, file, line, "%v", a)
	l.addCache(DEBUG, packageName, file, line, "%v", a)
}

func (l *Log551) Information(a interface{}) {
	pc, file, line, _ := runtime.Caller(1)
	packageName := l.getPackageName(runtime.FuncForPC(pc).Name())
	l.output(INFORMATION, packageName, file, line, "%v", a)
	l.addCache(INFORMATION, packageName, file, line, "%v", a)
}

func (l *Log551) Informationf(format string, a ...interface{}) {
	pc, file, line, _ := runtime.Caller(1)
	packageName := l.getPackageName(runtime.FuncForPC(pc).Name())
	l.output(INFORMATION, packageName, file, line, format, a...)
	l.addCache(INFORMATION, packageName, file, line, format, a...)
}

func (l *Log551) Informationln(a interface{}) {
	pc, file, line, _ := runtime.Caller(1)
	packageName := l.getPackageName(runtime.FuncForPC(pc).Name())
	l.output(INFORMATION, packageName, file, line, "%v", a)
	l.addCache(INFORMATION, packageName, file, line, "%v", a)
}

func (l *Log551) Warning(a interface{}) {
	pc, file, line, _ := runtime.Caller(1)
	packageName := l.getPackageName(runtime.FuncForPC(pc).Name())
	l.output(WARNING, packageName, file, line, "%v", a)
	l.addCache(WARNING, packageName, file, line, "%v", a)
}

func (l *Log551) Warningf(format string, a ...interface{}) {
	pc, file, line, _ := runtime.Caller(1)
	packageName := l.getPackageName(runtime.FuncForPC(pc).Name())
	l.output(WARNING, packageName, file, line, format, a...)
	l.addCache(WARNING, packageName, file, line, format, a...)
}

func (l *Log551) Warningln(a interface{}) {
	pc, file, line, _ := runtime.Caller(1)
	packageName := l.getPackageName(runtime.FuncForPC(pc).Name())
	l.output(WARNING, packageName, file, line, "%v", a)
	l.addCache(WARNING, packageName, file, line, "%v", a)
}

func (l *Log551) Error(a interface{}) {
	pc, file, line, _ := runtime.Caller(1)
	packageName := l.getPackageName(runtime.FuncForPC(pc).Name())
	l.output(ERROR, packageName, file, line, "%v", a)
	l.addCache(ERROR, packageName, file, line, "%v", a)
}

func (l *Log551) Errorf(format string, a ...interface{}) {
	pc, file, line, _ := runtime.Caller(1)
	packageName := l.getPackageName(runtime.FuncForPC(pc).Name())
	l.output(ERROR, packageName, file, line, format, a...)
	l.addCache(ERROR, packageName, file, line, format, a...)
}

func (l *Log551) Errorln(a interface{}) {
	pc, file, line, _ := runtime.Caller(1)
	packageName := l.getPackageName(runtime.FuncForPC(pc).Name())
	l.output(ERROR, packageName, file, line, "%v", a)
	l.addCache(ERROR, packageName, file, line, "%v", a)
}

func (l *Log551) Critical(a interface{}) {
	pc, file, line, _ := runtime.Caller(1)
	packageName := l.getPackageName(runtime.FuncForPC(pc).Name())
	l.output(CRITICAL, packageName, file, line, "%v", a)
	l.addCache(CRITICAL, packageName, file, line, "%v", a)
}

func (l *Log551) Criticalf(format string, a ...interface{}) {
	pc, file, line, _ := runtime.Caller(1)
	packageName := l.getPackageName(runtime.FuncForPC(pc).Name())
	l.output(CRITICAL, packageName, file, line, format, a...)
	l.addCache(CRITICAL, packageName, file, line, format, a...)
}

func (l *Log551) Criticalln(a interface{}) {
	pc, file, line, _ := runtime.Caller(1)
	packageName := l.getPackageName(runtime.FuncForPC(pc).Name())
	l.output(CRITICAL, packageName, file, line, "%v", a)
	l.addCache(CRITICAL, packageName, file, line, "%v", a)
}

func (l *Log551) output(level logLevel, packageName, file string, line int, format string, a ...interface{}) {
	if ! l.isOutput(level) {
		return
	}

	param := []interface{}{
		time.Now().Format("2006/01/02 15:04:05.00000"),
		packageName,
		path.Base(file),
		line,
		level.String(),
	}
	param = append(param, a...)

	fmt.Fprintf(l.file, "%s %s/%s %d [%s] " + format + "\n", param...)

}

func (l *Log551) isOutput(level logLevel) bool {
	if !l.config.Debug.EachTime && level == DEBUG {
		return false
	}
	if !l.config.Information.EachTime && level == INFORMATION {
		return false
	}
	if !l.config.Warning.EachTime && level == WARNING {
		return false
	}
	if !l.config.Error.EachTime && level == ERROR {
		return false
	}
	if !l.config.Critical.EachTime && level == CRITICAL {
		return false
	}

	return true
}

func (l *Log551) addCache(level logLevel, packageName, file string, line int, format string, a ...interface{}) {
	if ! l.isCache(level) {
		return
	}

	cache := &cache{
		datetime: time.Now(),
		packageName:packageName,
		file: file,
		line: line,
		level: level,
		format: format,
		a: a,
	}

	l.caches = append(l.caches, cache)

}

func (l *Log551) isCache(level logLevel) bool {
	if !l.config.Debug.Cache && level == DEBUG {
		return false
	}
	if !l.config.Information.Cache && level == INFORMATION {
		return false
	}
	if !l.config.Warning.Cache && level == WARNING {
		return false
	}
	if !l.config.Error.Cache && level == ERROR {
		return false
	}
	if !l.config.Critical.Cache && level == CRITICAL {
		return false
	}

	return true
}

func (l *Log551) outputCacheLog() {
	if len(l.caches) == 0 {
		return
	}

	for _, v := range l.caches {
		param := []interface{}{
			v.datetime.Format("2006/01/02 15:04:05.00000"),
			v.packageName,
			path.Base(v.file),
			v.line,
			v.level.String(),
		}
		param = append(param, v.a...)
		fmt.Fprintf(l.file, "%s %s/%s %d [%s] " + v.format + "\n", param...)
	}
}

var regexpControllerPattern *regexp.Regexp

func (r *Log551) getPackageName(path string) string {
	if regexpControllerPattern == nil {
		regexpControllerPattern = regexp.MustCompile(`\/\w+\.`)
	}

	packageName := regexpControllerPattern.FindString(path)

	return packageName[1 : len(packageName)-1]
}
<|endoftext|>


<commit_before>// Copyright 2014 Manu Martinez-Almeida.  All rights reserved.
// Use of this source code is governed by a MIT style
// license that can be found in the LICENSE file.

package gin

import (
	"log"
	"os"
	"time"
)

var (
	green   = string([]byte{27, 91, 57, 55, 59, 52, 50, 109})
	white   = string([]byte{27, 91, 57, 48, 59, 52, 55, 109})
	yellow  = string([]byte{27, 91, 57, 55, 59, 52, 51, 109})
	red     = string([]byte{27, 91, 57, 55, 59, 52, 49, 109})
	blue    = string([]byte{27, 91, 57, 55, 59, 52, 52, 109})
	magenta = string([]byte{27, 91, 57, 55, 59, 52, 53, 109})
	cyan    = string([]byte{27, 91, 57, 55, 59, 52, 54, 109})
	reset   = string([]byte{27, 91, 48, 109})
)

func ErrorLogger() HandlerFunc {
	return ErrorLoggerT(ErrorTypeAll)
}

func ErrorLoggerT(typ uint32) HandlerFunc {
	return func(c *Context) {
		c.Next()

		errs := c.Errors.ByType(typ)
		if len(errs) > 0 {
			// -1 status code = do not change current one
			c.JSON(-1, c.Errors)
		}
	}
}

func Logger() HandlerFunc {
	stdlogger := log.New(os.Stdout, "", 0)
	//errlogger := log.New(os.Stderr, "", 0)

	return func(c *Context) {
		// Start timer
		start := time.Now()

		// Process request
		c.Next()

		// Stop timer
		end := time.Now()
		latency := end.Sub(start)

		clientIP := c.ClientIP()
		method := c.Request.Method
		statusCode := c.Writer.Status()
		statusColor := colorForStatus(statusCode)
		methodColor := colorForMethod(method)

		stdlogger.Printf("[GIN] %v |%s %3d %s| %12v | %s |%s  %s %-7s %s\n%s",
			end.Format("2006/01/02 - 15:04:05"),
			statusColor, statusCode, reset,
			latency,
			clientIP,
			methodColor, reset, method,
			c.Request.URL.Path,
			c.Errors.String(),
		)
	}
}

func colorForStatus(code int) string {
	switch {
	case code >= 200 && code <= 299:
		return green
	case code >= 300 && code <= 399:
		return white
	case code >= 400 && code <= 499:
		return yellow
	default:
		return red
	}
}

func colorForMethod(method string) string {
	switch {
	case method == "GET":
		return blue
	case method == "POST":
		return cyan
	case method == "PUT":
		return yellow
	case method == "DELETE":
		return red
	case method == "PATCH":
		return green
	case method == "HEAD":
		return magenta
	case method == "OPTIONS":
		return white
	default:
		return reset
	}
}
<commit_msg>colorful logger on windows<commit_after>// Copyright 2014 Manu Martinez-Almeida.  All rights reserved.
// Use of this source code is governed by a MIT style
// license that can be found in the LICENSE file.

package gin

import (
	"log"
	"time"

	"github.com/mattn/go-colorable"
)

var (
	green   = string([]byte{27, 91, 57, 55, 59, 52, 50, 109})
	white   = string([]byte{27, 91, 57, 48, 59, 52, 55, 109})
	yellow  = string([]byte{27, 91, 57, 55, 59, 52, 51, 109})
	red     = string([]byte{27, 91, 57, 55, 59, 52, 49, 109})
	blue    = string([]byte{27, 91, 57, 55, 59, 52, 52, 109})
	magenta = string([]byte{27, 91, 57, 55, 59, 52, 53, 109})
	cyan    = string([]byte{27, 91, 57, 55, 59, 52, 54, 109})
	reset   = string([]byte{27, 91, 48, 109})
)

func ErrorLogger() HandlerFunc {
	return ErrorLoggerT(ErrorTypeAll)
}

func ErrorLoggerT(typ uint32) HandlerFunc {
	return func(c *Context) {
		c.Next()

		errs := c.Errors.ByType(typ)
		if len(errs) > 0 {
			// -1 status code = do not change current one
			c.JSON(-1, c.Errors)
		}
	}
}

func Logger() HandlerFunc {
	stdlogger := log.New(colorable.NewColorableStdout(), "", 0)
	//errlogger := log.New(os.Stderr, "", 0)

	return func(c *Context) {
		// Start timer
		start := time.Now()

		// Process request
		c.Next()

		// Stop timer
		end := time.Now()
		latency := end.Sub(start)

		clientIP := c.ClientIP()
		method := c.Request.Method
		statusCode := c.Writer.Status()
		statusColor := colorForStatus(statusCode)
		methodColor := colorForMethod(method)

		stdlogger.Printf("[GIN] %v |%s %3d %s| %12v | %s |%s  %s %-7s %s\n%s",
			end.Format("2006/01/02 - 15:04:05"),
			statusColor, statusCode, reset,
			latency,
			clientIP,
			methodColor, reset, method,
			c.Request.URL.Path,
			c.Errors.String(),
		)
	}
}

func colorForStatus(code int) string {
	switch {
	case code >= 200 && code <= 299:
		return green
	case code >= 300 && code <= 399:
		return white
	case code >= 400 && code <= 499:
		return yellow
	default:
		return red
	}
}

func colorForMethod(method string) string {
	switch {
	case method == "GET":
		return blue
	case method == "POST":
		return cyan
	case method == "PUT":
		return yellow
	case method == "DELETE":
		return red
	case method == "PATCH":
		return green
	case method == "HEAD":
		return magenta
	case method == "OPTIONS":
		return white
	default:
		return reset
	}
}
<|endoftext|>


<commit_before>package gorm

import (
	"database/sql/driver"
	"fmt"
	"log"
	"os"
	"reflect"
	"regexp"
	"strconv"
	"time"
	"unicode"
)

var (
	defaultLogger            = Logger{log.New(os.Stdout, "\r\n", 0)}
	sqlRegexp                = regexp.MustCompile(`\?`)
	numericPlaceHolderRegexp = regexp.MustCompile(`\$\d+`)
)

func isPrintable(s string) bool {
	for _, r := range s {
		if !unicode.IsPrint(r) {
			return false
		}
	}
	return true
}

var LogFormatter = func(values ...interface{}) (messages []interface{}) {
	if len(values) > 1 {
		var (
			sql             string
			formattedValues []string
			level           = values[0]
			currentTime     = "\n\033[33m[" + NowFunc().Format("2006-01-02 15:04:05") + "]\033[0m"
			source          = fmt.Sprintf("\033[35m(%v)\033[0m", values[1])
		)

		messages = []interface{}{source, currentTime}

		if level == "sql" {
			// duration
			messages = append(messages, fmt.Sprintf(" \033[36;1m[%.2fms]\033[0m ", float64(values[2].(time.Duration).Nanoseconds()/1e4)/100.0))
			// sql

			for _, value := range values[4].([]interface{}) {
				indirectValue := reflect.Indirect(reflect.ValueOf(value))
				if indirectValue.IsValid() {
					value = indirectValue.Interface()
					if t, ok := value.(time.Time); ok {
						formattedValues = append(formattedValues, fmt.Sprintf("'%v'", t.Format("2006-01-02 15:04:05")))
					} else if b, ok := value.([]byte); ok {
						if str := string(b); isPrintable(str) {
							formattedValues = append(formattedValues, fmt.Sprintf("'%v'", str))
						} else {
							formattedValues = append(formattedValues, "'<binary>'")
						}
					} else if r, ok := value.(driver.Valuer); ok {
						if value, err := r.Value(); err == nil && value != nil {
							formattedValues = append(formattedValues, fmt.Sprintf("'%v'", value))
						} else {
							formattedValues = append(formattedValues, "NULL")
						}
					} else {
						formattedValues = append(formattedValues, fmt.Sprintf("'%v'", value))
					}
				} else {
					formattedValues = append(formattedValues, "NULL")
				}
			}

			// differentiate between $n placeholders or else treat like ?
			if numericPlaceHolderRegexp.MatchString(values[3].(string)) {
				sql = values[3].(string)
				for index, value := range formattedValues {
					placeholder := fmt.Sprintf(`\$%d([^\d]|$)`, index+1)
					sql = regexp.MustCompile(placeholder).ReplaceAllString(sql, value+"$1")
				}
			} else {
				formattedValuesLength := len(formattedValues)
				for index, value := range sqlRegexp.Split(values[3].(string), -1) {
					sql += value
					if index < formattedValuesLength {
						sql += formattedValues[index]
					}
				}
			}

			messages = append(messages, sql)
			messages = append(messages, fmt.Sprintf(" \n\033[36;31m[%v]\033[0m ", strconv.FormatInt(values[5].(int64), 10)+" rows affected or returned "))
		} else {
			messages = append(messages, "\033[31;1m")
			messages = append(messages, values[2:]...)
			messages = append(messages, "\033[0m")
		}
	}

	return
}

type logger interface {
	Print(v ...interface{})
}

// LogWriter log writer interface
type LogWriter interface {
	Println(v ...interface{})
}

// Logger default logger
type Logger struct {
	LogWriter
}

// Print format & print log
func (logger Logger) Print(values ...interface{}) {
	logger.Println(LogFormatter(values...)...)
}
<commit_msg>Better log output int8, int, int16, int32, int64, float32, float64, bool. (#2258)<commit_after>package gorm

import (
	"database/sql/driver"
	"fmt"
	"log"
	"os"
	"reflect"
	"regexp"
	"strconv"
	"time"
	"unicode"
)

var (
	defaultLogger            = Logger{log.New(os.Stdout, "\r\n", 0)}
	sqlRegexp                = regexp.MustCompile(`\?`)
	numericPlaceHolderRegexp = regexp.MustCompile(`\$\d+`)
)

func isPrintable(s string) bool {
	for _, r := range s {
		if !unicode.IsPrint(r) {
			return false
		}
	}
	return true
}

var LogFormatter = func(values ...interface{}) (messages []interface{}) {
	if len(values) > 1 {
		var (
			sql             string
			formattedValues []string
			level           = values[0]
			currentTime     = "\n\033[33m[" + NowFunc().Format("2006-01-02 15:04:05") + "]\033[0m"
			source          = fmt.Sprintf("\033[35m(%v)\033[0m", values[1])
		)

		messages = []interface{}{source, currentTime}

		if level == "sql" {
			// duration
			messages = append(messages, fmt.Sprintf(" \033[36;1m[%.2fms]\033[0m ", float64(values[2].(time.Duration).Nanoseconds()/1e4)/100.0))
			// sql

			for _, value := range values[4].([]interface{}) {
				indirectValue := reflect.Indirect(reflect.ValueOf(value))
				if indirectValue.IsValid() {
					value = indirectValue.Interface()
					if t, ok := value.(time.Time); ok {
						formattedValues = append(formattedValues, fmt.Sprintf("'%v'", t.Format("2006-01-02 15:04:05")))
					} else if b, ok := value.([]byte); ok {
						if str := string(b); isPrintable(str) {
							formattedValues = append(formattedValues, fmt.Sprintf("'%v'", str))
						} else {
							formattedValues = append(formattedValues, "'<binary>'")
						}
					} else if r, ok := value.(driver.Valuer); ok {
						if value, err := r.Value(); err == nil && value != nil {
							formattedValues = append(formattedValues, fmt.Sprintf("'%v'", value))
						} else {
							formattedValues = append(formattedValues, "NULL")
						}
					} else {
						switch value.(type) {
						case int8, int, int16, int32, int64, float32, float64, bool:
							formattedValues = append(formattedValues, fmt.Sprintf("%v", value))
							break
						default:
							formattedValues = append(formattedValues, fmt.Sprintf("'%v'", value))
						}
					}
				} else {
					formattedValues = append(formattedValues, "NULL")
				}
			}

			// differentiate between $n placeholders or else treat like ?
			if numericPlaceHolderRegexp.MatchString(values[3].(string)) {
				sql = values[3].(string)
				for index, value := range formattedValues {
					placeholder := fmt.Sprintf(`\$%d([^\d]|$)`, index+1)
					sql = regexp.MustCompile(placeholder).ReplaceAllString(sql, value+"$1")
				}
			} else {
				formattedValuesLength := len(formattedValues)
				for index, value := range sqlRegexp.Split(values[3].(string), -1) {
					sql += value
					if index < formattedValuesLength {
						sql += formattedValues[index]
					}
				}
			}

			messages = append(messages, sql)
			messages = append(messages, fmt.Sprintf(" \n\033[36;31m[%v]\033[0m ", strconv.FormatInt(values[5].(int64), 10)+" rows affected or returned "))
		} else {
			messages = append(messages, "\033[31;1m")
			messages = append(messages, values[2:]...)
			messages = append(messages, "\033[0m")
		}
	}

	return
}

type logger interface {
	Print(v ...interface{})
}

// LogWriter log writer interface
type LogWriter interface {
	Println(v ...interface{})
}

// Logger default logger
type Logger struct {
	LogWriter
}

// Print format & print log
func (logger Logger) Print(values ...interface{}) {
	logger.Println(LogFormatter(values...)...)
}
<|endoftext|>


<commit_before>package fluent

import (
	"log"
	"net"
	"strconv"
	"time"
)

// Logger owns asynchronous logging to fluentd.
type Logger struct {
	config   Config
	postCh   chan message
	buffer   []byte
	conn     net.Conn
	ticker   *time.Ticker
	logError bool
}

// NewLogger() launches a goroutine to log and returns logger.
// Logger has a channel to interact with the goroutine.
func NewLogger(config Config) *Logger {
	config.applyDefaultValues()

	logger := &Logger{
		config:   config,
		postCh:   make(chan message, config.ChannelLength),
		ticker:   time.NewTicker(config.BufferingTimeout),
		logError: true,
	}
	go logger.loop()
	defer logger.sendMessage()

	return logger
}

// You can send message to logger's goroutine via channel.
func (l *Logger) Post(tag string, data interface{}) {
	if l.config.TagPrefix != "" {
		tag = l.config.TagPrefix + "." + tag
	}
	l.postCh <- message{tag: tag, time: time.Now(), data: data}
}

func (l *Logger) loop() {
	for {
		select {
		case msg := <-l.postCh:
			pack, err := msg.toMsgpack()
			if err != nil {
				log.Printf("message pack dump error: " + err.Error())
				continue
			}

			l.buffer = append(l.buffer, pack...)
			if len(l.buffer) > l.config.BufferLength {
				l.sendMessage()
			}
		case <-l.ticker.C:
			l.sendMessage()
		}
	}
}

func (l *Logger) sendMessage() {
	if len(l.buffer) == 0 {
		return
	}

	l.connect()
	if l.conn == nil {
		return
	}

	_, err := l.conn.Write(l.buffer)

	if err == nil {
		l.buffer = l.buffer[0:0]
	} else {
		log.Printf("failed to send message: " + err.Error())
		l.conn.Close()
		l.conn = nil
	}
}

func (l *Logger) connect() {
	if l.conn != nil {
		return
	}

	var err error
	for i := 0; i < l.config.MaxTrialForConnection; i++ {
		l.conn, err = net.DialTimeout(
			"tcp",
			l.config.FluentHost+":"+strconv.Itoa(l.config.FluentPort),
			l.config.ConnectionTimeout,
		)

		if err == nil {
			l.logError = true
			return
		}
	}

	if l.logError {
		log.Printf("failed to establish connection with fluentd: " + err.Error())
		l.logError = false
	}
}
<commit_msg>Implement API to log synchronously<commit_after>package fluent

import (
	"errors"
	"log"
	"net"
	"strconv"
	"time"
)

// Logger owns asynchronous logging to fluentd.
type Logger struct {
	config   Config
	postCh   chan message
	buffer   []byte
	conn     net.Conn
	ticker   *time.Ticker
	logError bool
}

// NewLogger() launches a goroutine to log and returns logger.
// Logger has a channel to interact with the goroutine.
func NewLogger(config Config) *Logger {
	config.applyDefaultValues()

	logger := &Logger{
		config:   config,
		postCh:   make(chan message, config.ChannelLength),
		ticker:   time.NewTicker(config.BufferingTimeout),
		logError: true,
	}
	go logger.loop()
	defer logger.sendMessage()

	return logger
}

// You can send message to logger's goroutine via channel.
// This logging is executed asynchronously.
func (l *Logger) Post(tag string, data interface{}) {
	tag = l.prependTagPrefix(tag)
	l.postCh <- message{tag: tag, time: time.Now(), data: data}
}

// You can send message immediately to fluentd.
func (l *Logger) Log(tag string, data interface{}) error {
	tag = l.prependTagPrefix(tag)
	msg := &message{tag: tag, time: time.Now(), data: data}
	pack, err := msg.toMsgpack()
	if err != nil {
		return err
	}

	l.buffer = append(l.buffer, pack...)
	return l.sendMessage()
}

func (l *Logger) loop() {
	for {
		select {
		case msg := <-l.postCh:
			pack, err := msg.toMsgpack()
			if err != nil {
				log.Printf("message pack dump error: " + err.Error())
				continue
			}

			l.buffer = append(l.buffer, pack...)
			if len(l.buffer) > l.config.BufferLength {
				l.sendMessage()
			}
		case <-l.ticker.C:
			l.sendMessage()
		}
	}
}

func (l *Logger) sendMessage() error {
	if len(l.buffer) == 0 {
		return errors.New("Buffer is empty")
	}

	l.connect()
	if l.conn == nil {
		return errors.New("Failed to establish connection with fluentd")
	}

	_, err := l.conn.Write(l.buffer)

	if err == nil {
		l.buffer = l.buffer[0:0]
	} else {
		log.Printf("failed to send message: " + err.Error())
		l.conn.Close()
		l.conn = nil
	}
	return err
}

func (l *Logger) connect() {
	if l.conn != nil {
		return
	}

	var err error
	for i := 0; i < l.config.MaxTrialForConnection; i++ {
		l.conn, err = net.DialTimeout(
			"tcp",
			l.config.FluentHost+":"+strconv.Itoa(l.config.FluentPort),
			l.config.ConnectionTimeout,
		)

		if err == nil {
			l.logError = true
			return
		}
	}

	if l.logError {
		log.Printf("failed to establish connection with fluentd: " + err.Error())
		l.logError = false
	}
}

func (l *Logger) prependTagPrefix(tag string) string {
	if l.config.TagPrefix != "" {
		tag = l.config.TagPrefix + "." + tag
	}
	return tag
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"io"
	"runtime"
	"strings"

	"github.com/sirupsen/logrus"
)

type logger struct {
	entry *logrus.Entry
}

// Logger is the interface for loggers used in the Prometheus components.
type Logger interface {
	Debug(...interface{})
	Debugln(...interface{})
	Debugf(string, ...interface{})

	Info(...interface{})
	Infoln(...interface{})
	Infof(string, ...interface{})

	Warn(...interface{})
	Warnln(...interface{})
	Warnf(string, ...interface{})

	Error(...interface{})
	Errorln(...interface{})
	Errorf(string, ...interface{})

	Fatal(...interface{})
	Fatalln(...interface{})
	Fatalf(string, ...interface{})

	With(key string, value interface{}) Logger

	SetLevel(string) error
}

func (l logger) With(key string, value interface{}) Logger {
	return logger{l.entry.WithField(key, value)}
}

// Debug logs a message at level Debug on the standard logger.
func (l logger) Debug(args ...interface{}) {
	l.sourced().Debug(args...)
}

// Debug logs a message at level Debug on the standard logger.
func (l logger) Debugln(args ...interface{}) {
	l.sourced().Debugln(args...)
}

// Debugf logs a message at level Debug on the standard logger.
func (l logger) Debugf(format string, args ...interface{}) {
	l.sourced().Debugf(format, args...)
}

// Info logs a message at level Info on the standard logger.
func (l logger) Info(args ...interface{}) {
	l.sourced().Info(args...)
}

// Info logs a message at level Info on the standard logger.
func (l logger) Infoln(args ...interface{}) {
	l.sourced().Infoln(args...)
}

// Infof logs a message at level Info on the standard logger.
func (l logger) Infof(format string, args ...interface{}) {
	l.sourced().Infof(format, args...)
}

// Warn logs a message at level Warn on the standard logger.
func (l logger) Warn(args ...interface{}) {
	l.sourced().Warn(args...)
}

// Warn logs a message at level Warn on the standard logger.
func (l logger) Warnln(args ...interface{}) {
	l.sourced().Warnln(args...)
}

// Warnf logs a message at level Warn on the standard logger.
func (l logger) Warnf(format string, args ...interface{}) {
	l.sourced().Warnf(format, args...)
}

// Error logs a message at level Error on the standard logger.
func (l logger) Error(args ...interface{}) {
	l.sourced().Error(args...)
}

// Error logs a message at level Error on the standard logger.
func (l logger) Errorln(args ...interface{}) {
	l.sourced().Errorln(args...)
}

// Errorf logs a message at level Error on the standard logger.
func (l logger) Errorf(format string, args ...interface{}) {
	l.sourced().Errorf(format, args...)
}

// Fatal logs a message at level Fatal on the standard logger.
func (l logger) Fatal(args ...interface{}) {
	l.sourced().Fatal(args...)
}

// Fatal logs a message at level Fatal on the standard logger.
func (l logger) Fatalln(args ...interface{}) {
	l.sourced().Fatalln(args...)
}

// Fatalf logs a message at level Fatal on the standard logger.
func (l logger) Fatalf(format string, args ...interface{}) {
	l.sourced().Fatalf(format, args...)
}

func (l logger) SetLevel(level string) error {
	lvl, err := logrus.ParseLevel(level)
	if err != nil {
		return err
	}

	l.entry.Logger.Level = lvl
	return nil
}

// sourced adds a source field to the logger that contains
// the file name and line where the logging happened.
func (l *logger) sourced() *logrus.Entry {
	_, file, line, ok := runtime.Caller(2)
	if !ok {
		file = "<???>"
		line = 1
	} else {
		slash := strings.LastIndex(file, "/")
		file = file[slash+1:]
	}
	return l.entry.WithField("source", fmt.Sprintf("%s:%d", file, line))
}

// NewLogger returns a new Logger logging to out.
func NewLogger(w io.Writer) Logger {
	l := logrus.New()
	l.Out = w
	return logger{entry: logrus.NewEntry(l)}
}
<commit_msg>logger: remove un-applicable comment<commit_after>package main

import (
	"fmt"
	"io"
	"runtime"
	"strings"

	"github.com/sirupsen/logrus"
)

type logger struct {
	entry *logrus.Entry
}

type Logger interface {
	Debug(...interface{})
	Debugln(...interface{})
	Debugf(string, ...interface{})

	Info(...interface{})
	Infoln(...interface{})
	Infof(string, ...interface{})

	Warn(...interface{})
	Warnln(...interface{})
	Warnf(string, ...interface{})

	Error(...interface{})
	Errorln(...interface{})
	Errorf(string, ...interface{})

	Fatal(...interface{})
	Fatalln(...interface{})
	Fatalf(string, ...interface{})

	With(key string, value interface{}) Logger

	SetLevel(string) error
}

func (l logger) With(key string, value interface{}) Logger {
	return logger{l.entry.WithField(key, value)}
}

// Debug logs a message at level Debug on the standard logger.
func (l logger) Debug(args ...interface{}) {
	l.sourced().Debug(args...)
}

// Debug logs a message at level Debug on the standard logger.
func (l logger) Debugln(args ...interface{}) {
	l.sourced().Debugln(args...)
}

// Debugf logs a message at level Debug on the standard logger.
func (l logger) Debugf(format string, args ...interface{}) {
	l.sourced().Debugf(format, args...)
}

// Info logs a message at level Info on the standard logger.
func (l logger) Info(args ...interface{}) {
	l.sourced().Info(args...)
}

// Info logs a message at level Info on the standard logger.
func (l logger) Infoln(args ...interface{}) {
	l.sourced().Infoln(args...)
}

// Infof logs a message at level Info on the standard logger.
func (l logger) Infof(format string, args ...interface{}) {
	l.sourced().Infof(format, args...)
}

// Warn logs a message at level Warn on the standard logger.
func (l logger) Warn(args ...interface{}) {
	l.sourced().Warn(args...)
}

// Warn logs a message at level Warn on the standard logger.
func (l logger) Warnln(args ...interface{}) {
	l.sourced().Warnln(args...)
}

// Warnf logs a message at level Warn on the standard logger.
func (l logger) Warnf(format string, args ...interface{}) {
	l.sourced().Warnf(format, args...)
}

// Error logs a message at level Error on the standard logger.
func (l logger) Error(args ...interface{}) {
	l.sourced().Error(args...)
}

// Error logs a message at level Error on the standard logger.
func (l logger) Errorln(args ...interface{}) {
	l.sourced().Errorln(args...)
}

// Errorf logs a message at level Error on the standard logger.
func (l logger) Errorf(format string, args ...interface{}) {
	l.sourced().Errorf(format, args...)
}

// Fatal logs a message at level Fatal on the standard logger.
func (l logger) Fatal(args ...interface{}) {
	l.sourced().Fatal(args...)
}

// Fatal logs a message at level Fatal on the standard logger.
func (l logger) Fatalln(args ...interface{}) {
	l.sourced().Fatalln(args...)
}

// Fatalf logs a message at level Fatal on the standard logger.
func (l logger) Fatalf(format string, args ...interface{}) {
	l.sourced().Fatalf(format, args...)
}

func (l logger) SetLevel(level string) error {
	lvl, err := logrus.ParseLevel(level)
	if err != nil {
		return err
	}

	l.entry.Logger.Level = lvl
	return nil
}

// sourced adds a source field to the logger that contains
// the file name and line where the logging happened.
func (l *logger) sourced() *logrus.Entry {
	_, file, line, ok := runtime.Caller(2)
	if !ok {
		file = "<???>"
		line = 1
	} else {
		slash := strings.LastIndex(file, "/")
		file = file[slash+1:]
	}
	return l.entry.WithField("source", fmt.Sprintf("%s:%d", file, line))
}

// NewLogger returns a new Logger logging to out.
func NewLogger(w io.Writer) Logger {
	l := logrus.New()
	l.Out = w
	return logger{entry: logrus.NewEntry(l)}
}
<|endoftext|>


<commit_before>package streamlog

import (
	"io"
	"log"
	"os"

	"github.com/joefitzgerald/standardlog"
)

// OutputStream represents standard output streams
type OutputStream int

// The standard output streams
const (
	Out OutputStream = iota
	Err
)

// StreamLogger allows logging to stderr and stdout streams
type StreamLogger interface {
	Flags() int
	Prefix() string
	Print(s OutputStream, v ...interface{})
	Printf(s OutputStream, format string, v ...interface{})
	Println(s OutputStream, v ...interface{})
	PrintErr(v ...interface{})
	PrintfErr(format string, v ...interface{})
	PrintlnErr(v ...interface{})
	PrintOut(v ...interface{})
	PrintfOut(format string, v ...interface{})
	PrintlnOut(v ...interface{})
	SetFlags(flag int)
	SetPrefix(prefix string)
}

// NewWithWriters creates a new StdOutErrStreamLogger. The out and err variables set the
// destination to which log data will be written.
// The prefix appears at the beginning of each generated log line.
// The flag argument defines the logging properties.
func NewWithWriters(out io.Writer, err io.Writer, prefix string, flag int) *StdOutErrStreamLogger {
	outLogger := log.New(os.Stdout, prefix, log.LstdFlags)
	errLogger := log.New(os.Stderr, prefix, log.LstdFlags)
	return &StdOutErrStreamLogger{out: outLogger, err: errLogger, prefix: prefix, flag: flag}
}

// StdOutErrStreamLogger is an OutErrLogger using stderr and stdout
type StdOutErrStreamLogger struct {
	out    standardlog.Logger // Logger for OutputStream.Out
	err    standardlog.Logger // Logger for OutputStream.Err
	prefix string             // prefix to write at beginning of each line
	flag   int                // properties
}

// New creates a new StdOutErrStreamLogger that writes to os.Stdout and
// os.Stderr.
func (l *StdOutErrStreamLogger) New() *StdOutErrStreamLogger {
	return NewWithWriters(os.Stdout, os.Stderr, "", log.LstdFlags)
}
<commit_msg>Add NewWithLoggers<commit_after>package streamlog

import (
	"io"
	"log"
	"os"

	"github.com/joefitzgerald/standardlog"
)

// OutputStream represents standard output streams
type OutputStream int

// The standard output streams
const (
	Out OutputStream = iota
	Err
)

// StreamLogger allows logging to stderr and stdout streams
type StreamLogger interface {
	Flags() int
	Prefix() string
	Print(s OutputStream, v ...interface{})
	Printf(s OutputStream, format string, v ...interface{})
	Println(s OutputStream, v ...interface{})
	PrintErr(v ...interface{})
	PrintfErr(format string, v ...interface{})
	PrintlnErr(v ...interface{})
	PrintOut(v ...interface{})
	PrintfOut(format string, v ...interface{})
	PrintlnOut(v ...interface{})
	SetFlags(flag int)
	SetPrefix(prefix string)
}

// NewWithWriters creates a new StdOutErrStreamLogger. The out and err variables set the
// destination to which log data will be written.
// The prefix appears at the beginning of each generated log line.
// The flag argument defines the logging properties.
func NewWithWriters(out io.Writer, err io.Writer, prefix string, flag int) *StdOutErrStreamLogger {
	outLogger := log.New(os.Stdout, prefix, log.LstdFlags)
	errLogger := log.New(os.Stderr, prefix, log.LstdFlags)
	return &StdOutErrStreamLogger{out: outLogger, err: errLogger, prefix: prefix, flag: flag}
}

// NewWithLoggers creates a new StdOutErrStreamLogger. The out and err variables
// set the destination to which log data will be written.
// The prefix appears at the beginning of each generated log line.
// The flag argument defines the logging properties.
func NewWithLoggers(out standardlog.Logger, err standardlog.Logger, prefix string, flag int) *StdOutErrStreamLogger {
	return &StdOutErrStreamLogger{out: out, err: err, prefix: prefix, flag: flag}
}

// StdOutErrStreamLogger is an OutErrLogger using stderr and stdout
type StdOutErrStreamLogger struct {
	out    standardlog.Logger // Logger for OutputStream.Out
	err    standardlog.Logger // Logger for OutputStream.Err
	prefix string             // prefix to write at beginning of each line
	flag   int                // properties
}

// New creates a new StdOutErrStreamLogger that writes to os.Stdout and
// os.Stderr.
func (l *StdOutErrStreamLogger) New() *StdOutErrStreamLogger {
	return NewWithWriters(os.Stdout, os.Stderr, "", log.LstdFlags)
}
<|endoftext|>


<commit_before>package main

import (
	"context"
	"fmt"
	"sort"
	"strings"
	"sync"
	"time"

	"github.com/dsnet/golib/strconv"
)

type packetLogger struct {
	logger logger
	last   time.Time // Last time we printed statistics
	c      chan packetLog
	m      map[packetLog]packetCount
}

type packetLog struct {
	direction  direction
	dropped    bool
	ipVersion  int
	ipProtocol int
	srcAddr    [4]byte
	dstAddr    [4]byte
	srcPort    uint16
	dstPort    uint16
	length     int
}

type packetCount struct {
	count uint64
	sizes uint64
}

func newPacketLogger(ctx context.Context, wg *sync.WaitGroup, logger logger) *packetLogger {
	pl := &packetLogger{
		logger: logger,
		last:   time.Now().Round(time.Second),
		c:      make(chan packetLog, 1024),
		m:      make(map[packetLog]packetCount),
	}
	wg.Add(1)
	go func() {
		defer wg.Done()
		pl.monitor(ctx)
	}()
	return pl
}

// Log logs the packet for periodic printing of aggregated statistics.
func (pl *packetLogger) Log(b []byte, d direction, dropped bool) {
	ip := ipPacket(b)
	p := packetLog{
		direction:  d,
		dropped:    dropped,
		ipVersion:  ip.Version(),
		length:     len(b),
		ipProtocol: ip.Protocol(),
	}
	if p.ipVersion == 4 {
		p.srcAddr, p.dstAddr = ip.AddressesV4()
	}
	if p.ipProtocol == udp || p.ipProtocol == tcp {
		p.srcPort, p.dstPort = transportPacket(ip.Body()).Ports()
	}
	pl.c <- p
}

func (pl *packetLogger) monitor(ctx context.Context) {
	t := time.NewTicker(30 * time.Second)
	defer t.Stop()

	for {
		select {
		case p := <-pl.c:
			size := p.length
			p.length = 0
			pc := pl.m[p]
			pc.count++
			pc.sizes += uint64(size)
			pl.m[p] = pc
		case <-t.C:
			var count, sizes uint64
			for _, v := range pl.m {
				count += v.count
				sizes += v.sizes
			}
			if count < (1<<12) && sizes < (1<<18) && time.Now().Sub(pl.last) < time.Hour {
				continue // Skip printing statistics if traffic is low.
			}
			pl.print()
		case <-ctx.Done():
			pl.print()
			return
		}
	}
}

// print prints an aggregated summary of all packets.
// To avoid races, this method is only called from monitor.
func (pl *packetLogger) print() {
	if len(pl.m) == 0 {
		return
	}
	var stats []string
	protoNames := map[int]string{icmp: "ICMP", udp: "UDP", tcp: "TCP"}
	for k, v := range pl.m {
		proto := protoNames[k.ipProtocol]
		if proto == "" {
			proto = fmt.Sprintf("Unknown%d", k.ipProtocol)
		}
		var drop string
		if k.dropped {
			drop = "dropped "
		}
		ipv4tos := func(ip [4]byte) string {
			return fmt.Sprintf("%d.%d.%d.%d", ip[0], ip[1], ip[2], ip[3])
		}
		link := fmt.Sprintf("%s:%d -> %s:%d", ipv4tos(k.srcAddr), k.srcPort, ipv4tos(k.dstAddr), k.dstPort)
		if k.ipProtocol != tcp && k.ipProtocol != udp {
			link = fmt.Sprintf("%s -> %s", ipv4tos(k.srcAddr), ipv4tos(k.dstAddr))
		}
		sizes := strconv.FormatPrefix(float64(v.sizes), strconv.IEC, 2)
		sizes = strings.TrimSuffix(strings.TrimRight(sizes, "0"), ".")
		stats = append(stats, fmt.Sprintf("\tIPv%d/%s %s - %cx %d %spackets (%sB)",
			k.ipVersion, proto, link, k.direction, v.count, drop, sizes))
	}
	sort.Strings(stats)
	period := time.Now().Round(time.Second).Sub(pl.last)
	pl.logger.Printf("Packet statistics (%v):\n%s", period, strings.Join(stats, "\n"))
	pl.m = map[packetLog]packetCount{}
	pl.last = time.Now().Round(time.Second)
}
<commit_msg>Add total statistics to logger<commit_after>package main

import (
	"context"
	"fmt"
	"sort"
	"strings"
	"sync"
	"time"

	"github.com/dsnet/golib/strconv"
)

type packetLogger struct {
	logger logger
	last   time.Time // Last time we printed statistics
	c      chan packetLog
	m      map[packetLog]packetCount
	rx, tx struct{ okay, drop packetCount }
}

type packetLog struct {
	direction  direction
	dropped    bool
	ipVersion  int
	ipProtocol int
	srcAddr    [4]byte
	dstAddr    [4]byte
	srcPort    uint16
	dstPort    uint16
	length     int
}

type packetCount struct {
	count uint64
	sizes uint64
}

func newPacketLogger(ctx context.Context, wg *sync.WaitGroup, logger logger) *packetLogger {
	pl := &packetLogger{
		logger: logger,
		last:   time.Now().Round(time.Second),
		c:      make(chan packetLog, 1024),
		m:      make(map[packetLog]packetCount),
	}
	wg.Add(1)
	go func() {
		defer wg.Done()
		pl.monitor(ctx)
	}()
	return pl
}

// Log logs the packet for periodic printing of aggregated statistics.
func (pl *packetLogger) Log(b []byte, d direction, dropped bool) {
	ip := ipPacket(b)
	p := packetLog{
		direction:  d,
		dropped:    dropped,
		ipVersion:  ip.Version(),
		length:     len(b),
		ipProtocol: ip.Protocol(),
	}
	if p.ipVersion == 4 {
		p.srcAddr, p.dstAddr = ip.AddressesV4()
	}
	if p.ipProtocol == udp || p.ipProtocol == tcp {
		p.srcPort, p.dstPort = transportPacket(ip.Body()).Ports()
	}
	pl.c <- p
}

func (pl *packetLogger) monitor(ctx context.Context) {
	t := time.NewTicker(30 * time.Second)
	defer t.Stop()

	for {
		select {
		case p := <-pl.c:
			size := p.length
			p.length = 0
			pc := pl.m[p]
			pc.count++
			pc.sizes += uint64(size)
			pl.m[p] = pc
		case <-t.C:
			var count, sizes uint64
			for _, v := range pl.m {
				count += v.count
				sizes += v.sizes
			}
			if count < (1<<12) && sizes < (1<<18) && time.Now().Sub(pl.last) < time.Hour {
				continue // Skip printing statistics if traffic is low.
			}
			pl.print()
		case <-ctx.Done():
			pl.print()
			return
		}
	}
}

// print prints an aggregated summary of all packets.
// To avoid races, this method is only called from monitor.
func (pl *packetLogger) print() {
	if len(pl.m) == 0 {
		return
	}
	stats := make([]string, 2) // First 2 lines for total stats
	protoNames := map[int]string{icmp: "ICMP", udp: "UDP", tcp: "TCP"}
	for k, v := range pl.m {
		var pc *packetCount
		switch {
		case !k.dropped && k.direction == outbound:
			pc = &pl.tx.okay
		case !k.dropped && k.direction == inbound:
			pc = &pl.rx.okay
		case k.dropped && k.direction == outbound:
			pc = &pl.tx.drop
		case k.dropped && k.direction == inbound:
			pc = &pl.rx.drop
		}
		pc.count += v.count
		pc.sizes += v.sizes

		proto := protoNames[k.ipProtocol]
		if proto == "" {
			proto = fmt.Sprintf("Unknown%d", k.ipProtocol)
		}
		var drop string
		if k.dropped {
			drop = "dropped "
		}
		ipv4tos := func(ip [4]byte) string {
			return fmt.Sprintf("%d.%d.%d.%d", ip[0], ip[1], ip[2], ip[3])
		}
		link := fmt.Sprintf("%s:%d -> %s:%d", ipv4tos(k.srcAddr), k.srcPort, ipv4tos(k.dstAddr), k.dstPort)
		if k.ipProtocol != tcp && k.ipProtocol != udp {
			link = fmt.Sprintf("%s -> %s", ipv4tos(k.srcAddr), ipv4tos(k.dstAddr))
		}
		sizes := strconv.FormatPrefix(float64(v.sizes), strconv.IEC, 2)
		sizes = strings.TrimSuffix(strings.TrimRight(sizes, "0"), ".")
		stats = append(stats, fmt.Sprintf("\tIPv%d/%s %s - %cx %d %spackets (%sB)",
			k.ipVersion, proto, link, k.direction, v.count, drop, sizes))
	}
	stats[0] = fmt.Sprintf("\tRx %d total packets (%dB), dropped %d total packets (%dB)",
		pl.rx.okay.count, pl.rx.okay.sizes, pl.rx.drop.count, pl.rx.drop.sizes)
	stats[1] = fmt.Sprintf("\tTx %d total packets (%dB), dropped %d total packets (%dB)",
		pl.tx.okay.count, pl.tx.okay.sizes, pl.tx.drop.count, pl.tx.drop.sizes)
	sort.Strings(stats[2:])
	period := time.Now().Round(time.Second).Sub(pl.last)
	pl.logger.Printf("Packet statistics (%v):\n%s", period, strings.Join(stats, "\n"))
	pl.m = map[packetLog]packetCount{}
	pl.last = time.Now().Round(time.Second)
}
<|endoftext|>


<commit_before>// Copyright 2014 Manu Martinez-Almeida.  All rights reserved.
// Use of this source code is governed by a MIT style
// license that can be found in the LICENSE file.

package zest

import (
	"log"
	"net/http"
	"os"
	"strings"
	"time"

	"github.com/codegangsta/negroni"
)

// Logger is a middleware handler that logs the request as it goes in and the response as it goes out.
type Logger struct {
	// Logger inherits from log.Logger used to log messages with the Logger middleware
	*log.Logger
}

// NewLogger returns a new Logger instance
func NewLogger() *Logger {
	return &Logger{log.New(os.Stdout, "\n[Zest] ", 0)}
}

func (l *Logger) ServeHTTP(rw http.ResponseWriter, r *http.Request, next http.HandlerFunc) {
	path := r.URL.Path
	// Start timer
	start := time.Now()

	// Process request
	next(rw, r)

	if strings.Contains(path, "explorer") || strings.Contains(path, "favicon") {
		return
	}

	// Stop timer
	end := time.Now()
	latency := end.Sub(start)

	clientIP := r.RemoteAddr
	method := r.Method
	res := rw.(negroni.ResponseWriter)
	statusCode := res.Status()
	statusColor := colorForStatus(statusCode)
	methodColor := colorForMethod(method)

	l.Printf("%v | %s %3d %s | %v | %s | %s %s %s %s",
		end.Format("2006/01/02 - 15:04:05"),
		statusColor, statusCode, reset,
		latency,
		clientIP,
		methodColor, method, reset,
		path,
	)
}

var (
	green   = string([]byte{27, 91, 57, 55, 59, 52, 50, 109})
	white   = string([]byte{27, 91, 57, 48, 59, 52, 55, 109})
	yellow  = string([]byte{27, 91, 57, 55, 59, 52, 51, 109})
	red     = string([]byte{27, 91, 57, 55, 59, 52, 49, 109})
	blue    = string([]byte{27, 91, 57, 55, 59, 52, 52, 109})
	magenta = string([]byte{27, 91, 57, 55, 59, 52, 53, 109})
	cyan    = string([]byte{27, 91, 57, 55, 59, 52, 54, 109})
	reset   = string([]byte{27, 91, 48, 109})
)

func colorForStatus(code int) string {
	switch {
	case code >= 200 && code < 300:
		return green
	case code >= 300 && code < 400:
		return white
	case code >= 400 && code < 500:
		return yellow
	default:
		return red
	}
}

func colorForMethod(method string) string {
	switch method {
	case "GET":
		return blue
	case "POST":
		return cyan
	case "PUT":
		return yellow
	case "DELETE":
		return red
	case "PATCH":
		return green
	case "HEAD":
		return magenta
	case "OPTIONS":
		return white
	default:
		return reset
	}
}
<commit_msg>Logger fixed.<commit_after>package zest

import (
	"log"
	"net/http"
	"os"
	"time"

	"github.com/codegangsta/negroni"
)

// Logger is a middleware handler that logs the request as it goes in and the response as it goes out.
type Logger struct {
	// Logger inherits from log.Logger used to log messages with the Logger middleware
	*log.Logger
}

// NewLogger returns a new Logger instance
func NewLogger() *Logger {
	return &Logger{log.New(os.Stdout, "\n[Zest] ", 0)}
}

func (l *Logger) ServeHTTP(rw http.ResponseWriter, r *http.Request, next http.HandlerFunc) {
	path := r.URL.Path
	// Start timer
	start := time.Now()

	// Process request
	next(rw, r)

	// Stop timer
	end := time.Now()
	latency := end.Sub(start)

	clientIP := r.RemoteAddr
	method := r.Method
	res := rw.(negroni.ResponseWriter)
	statusCode := res.Status()
	statusColor := colorForStatus(statusCode)
	methodColor := colorForMethod(method)

	l.Printf("%v | %s %3d %s | %v | %s | %s %s %s %s",
		end.Format("2006/01/02 - 15:04:05"),
		statusColor, statusCode, reset,
		latency,
		clientIP,
		methodColor, method, reset,
		path,
	)
}

var (
	green   = string([]byte{27, 91, 57, 55, 59, 52, 50, 109})
	white   = string([]byte{27, 91, 57, 48, 59, 52, 55, 109})
	yellow  = string([]byte{27, 91, 57, 55, 59, 52, 51, 109})
	red     = string([]byte{27, 91, 57, 55, 59, 52, 49, 109})
	blue    = string([]byte{27, 91, 57, 55, 59, 52, 52, 109})
	magenta = string([]byte{27, 91, 57, 55, 59, 52, 53, 109})
	cyan    = string([]byte{27, 91, 57, 55, 59, 52, 54, 109})
	reset   = string([]byte{27, 91, 48, 109})
)

func colorForStatus(code int) string {
	switch {
	case code >= 200 && code < 300:
		return green
	case code >= 300 && code < 400:
		return white
	case code >= 400 && code < 500:
		return yellow
	default:
		return red
	}
}

func colorForMethod(method string) string {
	switch method {
	case "GET":
		return blue
	case "POST":
		return cyan
	case "PUT":
		return yellow
	case "DELETE":
		return red
	case "PATCH":
		return green
	case "HEAD":
		return magenta
	case "OPTIONS":
		return white
	default:
		return reset
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"io"
	"math/rand"
	"time"

	"github.com/vbauerster/mpb/v4"
	"github.com/vbauerster/mpb/v4/decor"
)

func main() {
	p := mpb.New()

	total := 100
	msgCh := make(chan string)
	filler, nextCh := makeCustomFiller(15, msgCh)
	bar := p.Add(int64(total), filler,
		mpb.PrependDecorators(
			decor.Name("my bar:"),
		),
		mpb.AppendDecorators(
			newCustomPercentage(decor.Percentage(), nextCh),
		),
	)
	// simulating some work
	go func() {
		rng := rand.New(rand.NewSource(time.Now().UnixNano()))
		max := 100 * time.Millisecond
		for i := 0; i < total; i++ {
			time.Sleep(time.Duration(rng.Intn(10)+1) * max / 10)
			if i == 33 {
				msgCh <- "some error, retrying..."
			}
			bar.Increment()
		}
	}()

	p.Wait()
}

func makeCustomFiller(maxFlash int, ch <-chan string) (mpb.FillerFunc, <-chan struct{}) {
	type refiller interface {
		SetRefill(int64)
	}
	filler := mpb.NewBarFiller()
	nextCh := make(chan struct{}, 1)
	var msg string
	var count int
	return func(w io.Writer, width int, st *decor.Statistics) {
		if count != 0 {
			nextCh <- struct{}{}
			limitFmt := fmt.Sprintf("%%.%ds", width)
			fmt.Fprintf(w, limitFmt, msg)
			count--
			return
		}
		select {
		case msg = <-ch:
			nextCh <- struct{}{}
			if f, ok := filler.(refiller); ok {
				defer f.SetRefill(st.Current)
			}
			count = maxFlash
		default:
		}
		filler.Fill(w, width, st)
	}, nextCh
}

type myPercentageDecorator struct {
	decor.Decorator
	ch <-chan struct{}
}

func (d *myPercentageDecorator) Decor(st *decor.Statistics) string {
	select {
	case <-d.ch:
		return ""
	default:
		return d.Decorator.Decor(st)
	}
}

func newCustomPercentage(base decor.Decorator, ch <-chan struct{}) decor.Decorator {
	return &myPercentageDecorator{
		Decorator: base,
		ch:        ch,
	}
}
<commit_msg>suppress example: resume on event<commit_after>package main

import (
	"errors"
	"fmt"
	"io"
	"math/rand"
	"time"

	"github.com/vbauerster/mpb/v4"
	"github.com/vbauerster/mpb/v4/decor"
)

func main() {
	p := mpb.New()

	total := 100
	msgCh := make(chan string)
	resumeCh := make(chan struct{})
	filler, nextCh := makeCustomFiller(msgCh, resumeCh)
	bar := p.Add(int64(total), filler,
		mpb.PrependDecorators(
			decor.Name("my bar:"),
		),
		mpb.AppendDecorators(
			newCustomPercentage(decor.Percentage(), nextCh),
		),
	)
	// simulating some work
	go func() {
		rng := rand.New(rand.NewSource(time.Now().UnixNano()))
		max := 100 * time.Millisecond
		var err error
		for i := 0; i < total; i++ {
			if err != nil {
				time.Sleep(3 * time.Second)
				resumeCh <- struct{}{}
				err = nil
			} else {
				time.Sleep(time.Duration(rng.Intn(10)+1) * max / 10)
			}
			if i == 33 {
				err = errors.New("some error")
				msgCh <- fmt.Sprintf("%s retrying...", err.Error())
			}
			bar.Increment()
		}
	}()

	p.Wait()
}

func makeCustomFiller(ch <-chan string, resume <-chan struct{}) (mpb.FillerFunc, <-chan struct{}) {
	type refiller interface {
		SetRefill(int64)
	}
	filler := mpb.NewBarFiller()
	nextCh := make(chan struct{}, 1)
	var msg *string
	return func(w io.Writer, width int, st *decor.Statistics) {
		select {
		case m := <-ch:
			if f, ok := filler.(refiller); ok {
				defer f.SetRefill(st.Current)
			}
			defer func() {
				msg = &m
			}()
			nextCh <- struct{}{}
		case <-resume:
			msg = nil
		default:
		}
		if msg != nil {
			limitFmt := fmt.Sprintf("%%.%ds", width)
			fmt.Fprintf(w, limitFmt, *msg)
			nextCh <- struct{}{}
		} else {
			filler.Fill(w, width, st)
		}
	}, nextCh
}

type myPercentageDecorator struct {
	decor.Decorator
	ch <-chan struct{}
}

func (d *myPercentageDecorator) Decor(st *decor.Statistics) string {
	select {
	case <-d.ch:
		return ""
	default:
		return d.Decorator.Decor(st)
	}
}

func newCustomPercentage(base decor.Decorator, ch <-chan struct{}) decor.Decorator {
	return &myPercentageDecorator{
		Decorator: base,
		ch:        ch,
	}
}
<|endoftext|>


<commit_before>package main

import (
	"github.com/mattn/go-uwsgi"
	"path/filepath"
	"net"
	"net/http"
	"os"
	"strings"

)

func main() {
	s := "/tmp/uwsgi.sock"
	os.Remove(s)
	l, e := net.Listen("unix", s)
	os.Chmod(s, 0666)
	if e != nil {
		println(e.Error())
		os.Exit(1)
	}
	root, _ := filepath.Split(os.Args[0])
	root, _ = filepath.Abs(root)
    http.Serve(&uwsgi.Listener{l}, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		script_name := r.Header.Get("SCRIPT_NAME")
		path := r.URL.Path
		if strings.HasPrefix(path, script_name) {
			path = path[len(script_name):]
		}
		file := filepath.Join(root, filepath.FromSlash(path))
		f, e := os.Stat(file)
		if e == nil && f.IsDir() && len(path) > 0 && path[len(path)-1] != '/' {
			w.Header().Set("Location", r.URL.Path + "/")
			w.WriteHeader(http.StatusFound)
			return
		}
		http.ServeFile(w, r, file)
	}))
}
<commit_msg>test for CI.<commit_after>package main

import (
	"github.com/mattn/go-uwsgi"
	"path/filepath"
	"net"
	"net/http"
	"os"
	"strings"

)

func main() {
	s := "/tmp/uwsgi.sock"
	os.Remove(s)
	l, e := net.Listen("unix", s)
	os.Chmod(s, 0666)
	if e != nil {
		println(e.Error())
		os.Exit(1)
	}
	root, _ := filepath.Split(os.Args[0])
	root, _ = filepath.Abs(root)
    http.Serve(&uwsgi.Listener{l}, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		script_name := r.Header.Get("SCRIPT_NAME")
		path := r.URL.Path
		if strings.HasPrefix(path, script_name) {
			path = path[len(script_name):]
		}
		file := filepath.Join(root, filepath.FromSlash(path))
		f, e := os.Stat(file)
		if e == nil && f.IsDir() && len(path) > 0 && path[len(path)-1] != '/' {
			w.Header().Set("Location", r.URL.Path + "/")
			w.WriteHeader(http.StatusFound)
			return
		}

		http.ServeFile(w, r, file)
	}))
}
<|endoftext|>


<commit_before>package main

import "github.com/tkrajina/typescriptify-golang-structs/typescriptify"

type Address struct {
	// Used in html
	City    string  `json:"city"`
	Number  float64 `json:"number"`
	Country string  `json:"country,omitempty"`
}

type PersonalInfo struct {
	Hobbies []string `json:"hobby"`
	PetName string   `json:"pet_name"`
}

type Person struct {
	Name         string       `json:"name"`
	PersonalInfo PersonalInfo `json:"personal_info"`
	Nicknames    []string     `json:"nicknames"`
	Addresses    []Address    `json:"addresses"`
}

func main() {
	converter := typescriptify.New()
	converter.CreateFromMethod = true
	converter.Indent = "    "

	converter.Add(Person{})

	err := converter.ConvertToFile("browser_test/example_output.ts")
	if err != nil {
		panic(err.Error())
	}
}
<commit_msg>Package fix<commit_after>package example

import "github.com/tkrajina/typescriptify-golang-structs/typescriptify"

type Address struct {
	// Used in html
	City    string  `json:"city"`
	Number  float64 `json:"number"`
	Country string  `json:"country,omitempty"`
}

type PersonalInfo struct {
	Hobbies []string `json:"hobby"`
	PetName string   `json:"pet_name"`
}

type Person struct {
	Name         string       `json:"name"`
	PersonalInfo PersonalInfo `json:"personal_info"`
	Nicknames    []string     `json:"nicknames"`
	Addresses    []Address    `json:"addresses"`
}

func main() {
	converter := typescriptify.New()
	converter.CreateFromMethod = true
	converter.Indent = "    "

	converter.Add(Person{})

	err := converter.ConvertToFile("browser_test/example_output.ts")
	if err != nil {
		panic(err.Error())
	}
}
<|endoftext|>


<commit_before>package main

import "fmt"
import "github.com/nikkolasg/ip-reachable"

func TestHostCheckNet() {
	checker := ipreach.HostCheckNet{}

	valid_ip := "128.30.52.45:80" // w3.org
	if err := checker.CheckTCP(valid_ip); err != nil {
		fmt.Println("Should be valid")
	}

	invalid_ip := "128.30.52.45:678"
	if err := checker.CheckTCP(invalid_ip); err == nil {
		fmt.Println("Should be invalid")
	}
}

func TestWhatsMyIP() {
	checker := ipreach.WhatsMyIp{}
	if err := checker.CheckTCP("127.0.0.1:3100"); err == nil {
		fmt.Println("Should not work if port not open")
	}
	if err := checker.CheckTCP("127.0.0.1:3000"); err != nil {
		fmt.Println("Should not work if port not open", err)
	}
}

func main() {
	TestWhatsMyIP()
}
<commit_msg>printout<commit_after>package main

import "fmt"
import "github.com/nikkolasg/ip-reachable"

func TestHostCheckNet() {
	checker := ipreach.HostCheckNet{}

	valid_ip := "128.30.52.45:80" // w3.org
	if err := checker.CheckTCP(valid_ip); err != nil {
		fmt.Println("Should be valid", err)
	}

	invalid_ip := "128.30.52.45:678"
	if err := checker.CheckTCP(invalid_ip); err == nil {
		fmt.Println("Should be invalid", err)
	}
}

func TestWhatsMyIP() {
	checker := ipreach.WhatsMyIp{}
	if err := checker.CheckTCP("127.0.0.1:3100"); err == nil {
		fmt.Println("[-] Should not work if port not open")
	}
	fmt.Println("[+] 3100 port not open correctly detected!")
	if err := checker.CheckTCP("127.0.0.1:3000"); err != nil {
		fmt.Println("Should not work if port not open", err)
	}
	fmt.Println("[+] 3000 port OPEN correctly detected!")
}

func main() {
	TestWhatsMyIP()
}
<|endoftext|>