<commit_before>package consensus

import (
	"bytes"
	"crypto/rand"
	"io/ioutil"
	"os"
	"path/filepath"

	// "sync"
	"testing"
	"time"

	"github.com/tendermint/tendermint/consensus/types"
	"github.com/tendermint/tendermint/libs/autofile"
	"github.com/tendermint/tendermint/libs/log"
	tmtypes "github.com/tendermint/tendermint/types"
	tmtime "github.com/tendermint/tendermint/types/time"

	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
)

const (
	walTestFlushInterval = time.Duration(100) * time.Millisecond
)

func TestWALTruncate(t *testing.T) {
	walDir, err := ioutil.TempDir("", "wal")
	require.NoError(t, err)
	defer os.RemoveAll(walDir)

	walFile := filepath.Join(walDir, "wal")

	//this magic number 4K can truncate the content when RotateFile. defaultHeadSizeLimit(10M) is hard to simulate.
	//this magic number 1 * time.Millisecond make RotateFile check frequently. defaultGroupCheckDuration(5s) is hard to simulate.
	wal, err := NewWAL(walFile,
		autofile.GroupHeadSizeLimit(4096),
		autofile.GroupCheckDuration(1*time.Millisecond),
	)
	require.NoError(t, err)
	wal.SetLogger(log.TestingLogger())
	err = wal.Start()
	require.NoError(t, err)
	defer func() {
		wal.Stop()
		// wait for the wal to finish shutting down so we
		// can safely remove the directory
		wal.Wait()
	}()

	//60 block's size nearly 70K, greater than group's headBuf size(4096 * 10), when headBuf is full, truncate content will Flush to the file.
	//at this time, RotateFile is called, truncate content exist in each file.
	err = WALGenerateNBlocks(t, wal.Group(), 60)
	require.NoError(t, err)

	time.Sleep(1 * time.Millisecond) //wait groupCheckDuration, make sure RotateFile run

	wal.Group().Flush()

	h := int64(50)
	gr, found, err := wal.SearchForEndHeight(h, &WALSearchOptions{})
	assert.NoError(t, err, "expected not to err on height %d", h)
	assert.True(t, found, "expected to find end height for %d", h)
	assert.NotNil(t, gr)
	defer gr.Close()

	dec := NewWALDecoder(gr)
	msg, err := dec.Decode()
	assert.NoError(t, err, "expected to decode a message")
	rs, ok := msg.Msg.(tmtypes.EventDataRoundState)
	assert.True(t, ok, "expected message of type EventDataRoundState")
	assert.Equal(t, rs.Height, h+1, "wrong height")
}

func TestWALEncoderDecoder(t *testing.T) {
	now := tmtime.Now()
	msgs := []TimedWALMessage{
		{Time: now, Msg: EndHeightMessage{0}},
		{Time: now, Msg: timeoutInfo{Duration: time.Second, Height: 1, Round: 1, Step: types.RoundStepPropose}},
	}

	b := new(bytes.Buffer)

	for _, msg := range msgs {
		b.Reset()

		enc := NewWALEncoder(b)
		err := enc.Encode(&msg)
		require.NoError(t, err)

		dec := NewWALDecoder(b)
		decoded, err := dec.Decode()
		require.NoError(t, err)

		assert.Equal(t, msg.Time.UTC(), decoded.Time)
		assert.Equal(t, msg.Msg, decoded.Msg)
	}
}

func TestWALWritePanicsIfMsgIsTooBig(t *testing.T) {
	walDir, err := ioutil.TempDir("", "wal")
	require.NoError(t, err)
	defer os.RemoveAll(walDir)
	walFile := filepath.Join(walDir, "wal")

	wal, err := NewWAL(walFile)
	require.NoError(t, err)
	err = wal.Start()
	require.NoError(t, err)
	defer func() {
		wal.Stop()
		// wait for the wal to finish shutting down so we
		// can safely remove the directory
		wal.Wait()
	}()

	assert.Panics(t, func() { wal.Write(make([]byte, maxMsgSizeBytes+1)) })
}

func TestWALSearchForEndHeight(t *testing.T) {
	walBody, err := WALWithNBlocks(t, 6)
	if err != nil {
		t.Fatal(err)
	}
	walFile := tempWALWithData(walBody)

	wal, err := NewWAL(walFile)
	require.NoError(t, err)
	wal.SetLogger(log.TestingLogger())

	h := int64(3)
	gr, found, err := wal.SearchForEndHeight(h, &WALSearchOptions{})
	assert.NoError(t, err, "expected not to err on height %d", h)
	assert.True(t, found, "expected to find end height for %d", h)
	assert.NotNil(t, gr)
	defer gr.Close()

	dec := NewWALDecoder(gr)
	msg, err := dec.Decode()
	assert.NoError(t, err, "expected to decode a message")
	rs, ok := msg.Msg.(tmtypes.EventDataRoundState)
	assert.True(t, ok, "expected message of type EventDataRoundState")
	assert.Equal(t, rs.Height, h+1, "wrong height")
}

func TestWALPeriodicSync(t *testing.T) {
	walDir, err := ioutil.TempDir("", "wal")
	require.NoError(t, err)
	defer os.RemoveAll(walDir)

	walFile := filepath.Join(walDir, "wal")
	wal, err := NewWAL(walFile, autofile.GroupCheckDuration(1*time.Millisecond))
	require.NoError(t, err)

	wal.SetFlushInterval(walTestFlushInterval)
	wal.SetLogger(log.TestingLogger())

	require.NoError(t, wal.Start())
	defer func() {
		wal.Stop()
		wal.Wait()
	}()

	err = WALGenerateNBlocks(t, wal.Group(), 5)
	require.NoError(t, err)

	// We should have data in the buffer now
	assert.NotZero(t, wal.Group().Buffered())

	time.Sleep(walTestFlushInterval + (10 * time.Millisecond))

	// The data should have been flushed by the periodic sync
	assert.Zero(t, wal.Group().Buffered())

	h := int64(4)
	gr, found, err := wal.SearchForEndHeight(h, &WALSearchOptions{})
	assert.NoError(t, err, "expected not to err on height %d", h)
	assert.True(t, found, "expected to find end height for %d", h)
	assert.NotNil(t, gr)
	if gr != nil {
		gr.Close()
	}
}

/*
var initOnce sync.Once

func registerInterfacesOnce() {
	initOnce.Do(func() {
		var _ = wire.RegisterInterface(
			struct{ WALMessage }{},
			wire.ConcreteType{[]byte{}, 0x10},
		)
	})
}
*/

func nBytes(n int) []byte {
	buf := make([]byte, n)
	n, _ = rand.Read(buf)
	return buf[:n]
}

func benchmarkWalDecode(b *testing.B, n int) {
	// registerInterfacesOnce()

	buf := new(bytes.Buffer)
	enc := NewWALEncoder(buf)

	data := nBytes(n)
	enc.Encode(&TimedWALMessage{Msg: data, Time: time.Now().Round(time.Second).UTC()})

	encoded := buf.Bytes()

	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		buf.Reset()
		buf.Write(encoded)
		dec := NewWALDecoder(buf)
		if _, err := dec.Decode(); err != nil {
			b.Fatal(err)
		}
	}
	b.ReportAllocs()
}

func BenchmarkWalDecode512B(b *testing.B) {
	benchmarkWalDecode(b, 512)
}

func BenchmarkWalDecode10KB(b *testing.B) {
	benchmarkWalDecode(b, 10*1024)
}
func BenchmarkWalDecode100KB(b *testing.B) {
	benchmarkWalDecode(b, 100*1024)
}
func BenchmarkWalDecode1MB(b *testing.B) {
	benchmarkWalDecode(b, 1024*1024)
}
func BenchmarkWalDecode10MB(b *testing.B) {
	benchmarkWalDecode(b, 10*1024*1024)
}
func BenchmarkWalDecode100MB(b *testing.B) {
	benchmarkWalDecode(b, 100*1024*1024)
}
func BenchmarkWalDecode1GB(b *testing.B) {
	benchmarkWalDecode(b, 1024*1024*1024)
}
<commit_msg>fix TestWALPeriodicSync (#3342)<commit_after>package consensus

import (
	"bytes"
	"crypto/rand"
	"io/ioutil"
	"os"
	"path/filepath"

	// "sync"
	"testing"
	"time"

	"github.com/tendermint/tendermint/consensus/types"
	"github.com/tendermint/tendermint/libs/autofile"
	"github.com/tendermint/tendermint/libs/log"
	tmtypes "github.com/tendermint/tendermint/types"
	tmtime "github.com/tendermint/tendermint/types/time"

	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
)

const (
	walTestFlushInterval = time.Duration(100) * time.Millisecond
)

func TestWALTruncate(t *testing.T) {
	walDir, err := ioutil.TempDir("", "wal")
	require.NoError(t, err)
	defer os.RemoveAll(walDir)

	walFile := filepath.Join(walDir, "wal")

	//this magic number 4K can truncate the content when RotateFile. defaultHeadSizeLimit(10M) is hard to simulate.
	//this magic number 1 * time.Millisecond make RotateFile check frequently. defaultGroupCheckDuration(5s) is hard to simulate.
	wal, err := NewWAL(walFile,
		autofile.GroupHeadSizeLimit(4096),
		autofile.GroupCheckDuration(1*time.Millisecond),
	)
	require.NoError(t, err)
	wal.SetLogger(log.TestingLogger())
	err = wal.Start()
	require.NoError(t, err)
	defer func() {
		wal.Stop()
		// wait for the wal to finish shutting down so we
		// can safely remove the directory
		wal.Wait()
	}()

	//60 block's size nearly 70K, greater than group's headBuf size(4096 * 10), when headBuf is full, truncate content will Flush to the file.
	//at this time, RotateFile is called, truncate content exist in each file.
	err = WALGenerateNBlocks(t, wal.Group(), 60)
	require.NoError(t, err)

	time.Sleep(1 * time.Millisecond) //wait groupCheckDuration, make sure RotateFile run

	wal.Group().Flush()

	h := int64(50)
	gr, found, err := wal.SearchForEndHeight(h, &WALSearchOptions{})
	assert.NoError(t, err, "expected not to err on height %d", h)
	assert.True(t, found, "expected to find end height for %d", h)
	assert.NotNil(t, gr)
	defer gr.Close()

	dec := NewWALDecoder(gr)
	msg, err := dec.Decode()
	assert.NoError(t, err, "expected to decode a message")
	rs, ok := msg.Msg.(tmtypes.EventDataRoundState)
	assert.True(t, ok, "expected message of type EventDataRoundState")
	assert.Equal(t, rs.Height, h+1, "wrong height")
}

func TestWALEncoderDecoder(t *testing.T) {
	now := tmtime.Now()
	msgs := []TimedWALMessage{
		{Time: now, Msg: EndHeightMessage{0}},
		{Time: now, Msg: timeoutInfo{Duration: time.Second, Height: 1, Round: 1, Step: types.RoundStepPropose}},
	}

	b := new(bytes.Buffer)

	for _, msg := range msgs {
		b.Reset()

		enc := NewWALEncoder(b)
		err := enc.Encode(&msg)
		require.NoError(t, err)

		dec := NewWALDecoder(b)
		decoded, err := dec.Decode()
		require.NoError(t, err)

		assert.Equal(t, msg.Time.UTC(), decoded.Time)
		assert.Equal(t, msg.Msg, decoded.Msg)
	}
}

func TestWALWritePanicsIfMsgIsTooBig(t *testing.T) {
	walDir, err := ioutil.TempDir("", "wal")
	require.NoError(t, err)
	defer os.RemoveAll(walDir)
	walFile := filepath.Join(walDir, "wal")

	wal, err := NewWAL(walFile)
	require.NoError(t, err)
	err = wal.Start()
	require.NoError(t, err)
	defer func() {
		wal.Stop()
		// wait for the wal to finish shutting down so we
		// can safely remove the directory
		wal.Wait()
	}()

	assert.Panics(t, func() { wal.Write(make([]byte, maxMsgSizeBytes+1)) })
}

func TestWALSearchForEndHeight(t *testing.T) {
	walBody, err := WALWithNBlocks(t, 6)
	if err != nil {
		t.Fatal(err)
	}
	walFile := tempWALWithData(walBody)

	wal, err := NewWAL(walFile)
	require.NoError(t, err)
	wal.SetLogger(log.TestingLogger())

	h := int64(3)
	gr, found, err := wal.SearchForEndHeight(h, &WALSearchOptions{})
	assert.NoError(t, err, "expected not to err on height %d", h)
	assert.True(t, found, "expected to find end height for %d", h)
	assert.NotNil(t, gr)
	defer gr.Close()

	dec := NewWALDecoder(gr)
	msg, err := dec.Decode()
	assert.NoError(t, err, "expected to decode a message")
	rs, ok := msg.Msg.(tmtypes.EventDataRoundState)
	assert.True(t, ok, "expected message of type EventDataRoundState")
	assert.Equal(t, rs.Height, h+1, "wrong height")
}

func TestWALPeriodicSync(t *testing.T) {
	walDir, err := ioutil.TempDir("", "wal")
	require.NoError(t, err)
	defer os.RemoveAll(walDir)

	walFile := filepath.Join(walDir, "wal")
	wal, err := NewWAL(walFile, autofile.GroupCheckDuration(1*time.Millisecond))
	require.NoError(t, err)

	wal.SetFlushInterval(walTestFlushInterval)
	wal.SetLogger(log.TestingLogger())

	// Generate some data
	err = WALGenerateNBlocks(t, wal.Group(), 5)
	require.NoError(t, err)

	// We should have data in the buffer now
	assert.NotZero(t, wal.Group().Buffered())

	require.NoError(t, wal.Start())
	defer func() {
		wal.Stop()
		wal.Wait()
	}()

	time.Sleep(walTestFlushInterval + (10 * time.Millisecond))

	// The data should have been flushed by the periodic sync
	assert.Zero(t, wal.Group().Buffered())

	h := int64(4)
	gr, found, err := wal.SearchForEndHeight(h, &WALSearchOptions{})
	assert.NoError(t, err, "expected not to err on height %d", h)
	assert.True(t, found, "expected to find end height for %d", h)
	assert.NotNil(t, gr)
	if gr != nil {
		gr.Close()
	}
}

/*
var initOnce sync.Once

func registerInterfacesOnce() {
	initOnce.Do(func() {
		var _ = wire.RegisterInterface(
			struct{ WALMessage }{},
			wire.ConcreteType{[]byte{}, 0x10},
		)
	})
}
*/

func nBytes(n int) []byte {
	buf := make([]byte, n)
	n, _ = rand.Read(buf)
	return buf[:n]
}

func benchmarkWalDecode(b *testing.B, n int) {
	// registerInterfacesOnce()

	buf := new(bytes.Buffer)
	enc := NewWALEncoder(buf)

	data := nBytes(n)
	enc.Encode(&TimedWALMessage{Msg: data, Time: time.Now().Round(time.Second).UTC()})

	encoded := buf.Bytes()

	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		buf.Reset()
		buf.Write(encoded)
		dec := NewWALDecoder(buf)
		if _, err := dec.Decode(); err != nil {
			b.Fatal(err)
		}
	}
	b.ReportAllocs()
}

func BenchmarkWalDecode512B(b *testing.B) {
	benchmarkWalDecode(b, 512)
}

func BenchmarkWalDecode10KB(b *testing.B) {
	benchmarkWalDecode(b, 10*1024)
}
func BenchmarkWalDecode100KB(b *testing.B) {
	benchmarkWalDecode(b, 100*1024)
}
func BenchmarkWalDecode1MB(b *testing.B) {
	benchmarkWalDecode(b, 1024*1024)
}
func BenchmarkWalDecode10MB(b *testing.B) {
	benchmarkWalDecode(b, 10*1024*1024)
}
func BenchmarkWalDecode100MB(b *testing.B) {
	benchmarkWalDecode(b, 100*1024*1024)
}
func BenchmarkWalDecode1GB(b *testing.B) {
	benchmarkWalDecode(b, 1024*1024*1024)
}
<|endoftext|>


<commit_before>// +build autopilotrpc

package main

import (
	"context"

	"github.com/lightningnetwork/lnd/lnrpc/autopilotrpc"
	"github.com/urfave/cli"
)

func getAutopilotClient(ctx *cli.Context) (autopilotrpc.AutopilotClient, func()) {
	conn := getClientConn(ctx, false)

	cleanUp := func() {
		conn.Close()
	}

	return autopilotrpc.NewAutopilotClient(conn), cleanUp
}

var getStatusCommand = cli.Command{
	Name:        "status",
	Usage:       "Get the active status of autopilot.",
	Description: "",
	Action:      actionDecorator(getStatus),
}

func getStatus(ctx *cli.Context) error {
	ctxb := context.Background()
	client, cleanUp := getAutopilotClient(ctx)
	defer cleanUp()

	req := &autopilotrpc.StatusRequest{}

	resp, err := client.Status(ctxb, req)
	if err != nil {
		return err
	}

	printRespJSON(resp)
	return nil
}

var enableCommand = cli.Command{
	Name:        "enable",
	Usage:       "Enable the autopilot.",
	Description: "",
	Action:      actionDecorator(enable),
}

var disableCommand = cli.Command{
	Name:        "disable",
	Usage:       "Disable the active autopilot.",
	Description: "",
	Action:      actionDecorator(disable),
}

func enable(ctx *cli.Context) error {
	ctxb := context.Background()
	client, cleanUp := getAutopilotClient(ctx)
	defer cleanUp()

	// We will enable the autopilot.
	req := &autopilotrpc.ModifyStatusRequest{
		Enable: true,
	}

	resp, err := client.ModifyStatus(ctxb, req)
	if err != nil {
		return err
	}

	printRespJSON(resp)
	return nil
}

func disable(ctx *cli.Context) error {
	ctxb := context.Background()
	client, cleanUp := getAutopilotClient(ctx)
	defer cleanUp()

	// We will disable the autopilot.
	req := &autopilotrpc.ModifyStatusRequest{
		Enable: false,
	}

	resp, err := client.ModifyStatus(ctxb, req)
	if err != nil {
		return err
	}

	printRespJSON(resp)
	return nil
}

var queryScoresCommand = cli.Command{
	Name:        "query",
	Usage:       "Query the autopilot heuristcs for nodes' scores.",
	ArgsUsage:   "<pubkey> <pubkey> <pubkey> ...",
	Description: "",
	Action:      actionDecorator(queryScores),
}

func queryScores(ctx *cli.Context) error {
	ctxb := context.Background()
	client, cleanUp := getAutopilotClient(ctx)
	defer cleanUp()

	args := ctx.Args()
	var pubs []string

	// Keep reading pubkeys as long as there are arguments.
loop:
	for {
		switch {
		case args.Present():
			pubs = append(pubs, args.First())
			args = args.Tail()
		default:
			break loop
		}
	}

	req := &autopilotrpc.QueryScoresRequest{
		Pubkeys: pubs,
	}

	resp, err := client.QueryScores(ctxb, req)
	if err != nil {
		return err
	}

	printRespJSON(resp)
	return nil
}

// autopilotCommands will return the set of commands to enable for autopilotrpc
// builds.
func autopilotCommands() []cli.Command {
	return []cli.Command{
		{
			Name:        "autopilot",
			Category:    "Autopilot",
			Usage:       "Interact with a running autopilot.",
			Description: "",
			Subcommands: []cli.Command{
				getStatusCommand,
				enableCommand,
				disableCommand,
				queryScoresCommand,
			},
		},
	}
}
<commit_msg>lncli/autopilot: add -ignorelocal state flag to query<commit_after>// +build autopilotrpc

package main

import (
	"context"

	"github.com/lightningnetwork/lnd/lnrpc/autopilotrpc"
	"github.com/urfave/cli"
)

func getAutopilotClient(ctx *cli.Context) (autopilotrpc.AutopilotClient, func()) {
	conn := getClientConn(ctx, false)

	cleanUp := func() {
		conn.Close()
	}

	return autopilotrpc.NewAutopilotClient(conn), cleanUp
}

var getStatusCommand = cli.Command{
	Name:        "status",
	Usage:       "Get the active status of autopilot.",
	Description: "",
	Action:      actionDecorator(getStatus),
}

func getStatus(ctx *cli.Context) error {
	ctxb := context.Background()
	client, cleanUp := getAutopilotClient(ctx)
	defer cleanUp()

	req := &autopilotrpc.StatusRequest{}

	resp, err := client.Status(ctxb, req)
	if err != nil {
		return err
	}

	printRespJSON(resp)
	return nil
}

var enableCommand = cli.Command{
	Name:        "enable",
	Usage:       "Enable the autopilot.",
	Description: "",
	Action:      actionDecorator(enable),
}

var disableCommand = cli.Command{
	Name:        "disable",
	Usage:       "Disable the active autopilot.",
	Description: "",
	Action:      actionDecorator(disable),
}

func enable(ctx *cli.Context) error {
	ctxb := context.Background()
	client, cleanUp := getAutopilotClient(ctx)
	defer cleanUp()

	// We will enable the autopilot.
	req := &autopilotrpc.ModifyStatusRequest{
		Enable: true,
	}

	resp, err := client.ModifyStatus(ctxb, req)
	if err != nil {
		return err
	}

	printRespJSON(resp)
	return nil
}

func disable(ctx *cli.Context) error {
	ctxb := context.Background()
	client, cleanUp := getAutopilotClient(ctx)
	defer cleanUp()

	// We will disable the autopilot.
	req := &autopilotrpc.ModifyStatusRequest{
		Enable: false,
	}

	resp, err := client.ModifyStatus(ctxb, req)
	if err != nil {
		return err
	}

	printRespJSON(resp)
	return nil
}

var queryScoresCommand = cli.Command{
	Name:        "query",
	Usage:       "Query the autopilot heuristcs for nodes' scores.",
	ArgsUsage:   "[flags] <pubkey> <pubkey> <pubkey> ...",
	Description: "",
	Action:      actionDecorator(queryScores),
	Flags: []cli.Flag{
		cli.BoolFlag{
			Name: "ignorelocalstate, i",
			Usage: "Ignore local channel state when calculating " +
				"scores.",
		},
	},
}

func queryScores(ctx *cli.Context) error {
	ctxb := context.Background()
	client, cleanUp := getAutopilotClient(ctx)
	defer cleanUp()

	args := ctx.Args()
	var pubs []string

	// Keep reading pubkeys as long as there are arguments.
loop:
	for {
		switch {
		case args.Present():
			pubs = append(pubs, args.First())
			args = args.Tail()
		default:
			break loop
		}
	}

	req := &autopilotrpc.QueryScoresRequest{
		Pubkeys:          pubs,
		IgnoreLocalState: ctx.Bool("ignorelocalstate"),
	}

	resp, err := client.QueryScores(ctxb, req)
	if err != nil {
		return err
	}

	printRespJSON(resp)
	return nil
}

// autopilotCommands will return the set of commands to enable for autopilotrpc
// builds.
func autopilotCommands() []cli.Command {
	return []cli.Command{
		{
			Name:        "autopilot",
			Category:    "Autopilot",
			Usage:       "Interact with a running autopilot.",
			Description: "",
			Subcommands: []cli.Command{
				getStatusCommand,
				enableCommand,
				disableCommand,
				queryScoresCommand,
			},
		},
	}
}
<|endoftext|>


<commit_before>package gitmediaclient

import (
	"bytes"
	"encoding/base64"
	"encoding/json"
	"fmt"
	"io"
	"net/http"
	"net/url"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
)

func Put(filename string) error {
	oid := filepath.Base(filename)
	stat, err := os.Stat(filename)
	if err != nil {
		return err
	}

	file, err := os.Open(filename)
	if err != nil {
		return err
	}

	req, err := clientRequest("PUT", oid)
	if err != nil {
		return err
	}
	req.Body = file
	req.ContentLength = stat.Size()

	res, err := http.DefaultClient.Do(req)
	if err != nil {
		return err
	}

	defer res.Body.Close()

	if res.StatusCode > 299 {
		apierr := &Error{}
		dec := json.NewDecoder(res.Body)
		if err = dec.Decode(apierr); err != nil {
			return err
		}
		return apierr
	}

	fmt.Printf("Sending %s from %s: %d\n", oid, filename, res.StatusCode)
	return nil
}

func Get(filename string) (io.ReadCloser, error) {
	oid := filepath.Base(filename)
	if stat, err := os.Stat(filename); err != nil || stat == nil {
		req, err := clientRequest("GET", oid)
		if err != nil {
			return nil, err
		}

		req.Header.Set("Accept", "application/vnd.git-media")
		res, err := http.DefaultClient.Do(req)
		if err != nil {
			return nil, err
		}

		return res.Body, nil
	}

	return os.Open(filename)
}

func clientRequest(method, oid string) (*http.Request, error) {
	u := objectUrl(oid)
	req, err := http.NewRequest(method, u.String(), nil)
	if err == nil {
		creds, err := credentials(u)
		if err != nil {
			return req, err
		}

		token := fmt.Sprintf("%s:%s", creds["username"], creds["password"])
		auth := "Basic " + base64.URLEncoding.EncodeToString([]byte(token))
		req.Header.Set("Authorization", auth)
	}

	return req, err
}

func objectUrl(oid string) *url.URL {
	u, _ := url.Parse("http://localhost:8080")
	u.Path = "/objects/" + oid
	return u
}

func credentials(u *url.URL) (map[string]string, error) {
	credInput := fmt.Sprintf("protocol=%s\nhost=%s\n", u.Scheme, u.Host)
	cmd, err := execCreds(credInput, "fill")
	if err != nil {
		return nil, err
	}
	return cmd.Credentials(), nil
}

func execCreds(input, subCommand string) (*CredentialCmd, error) {
	cmd := NewCommand(input, subCommand)
	err := cmd.Start()
	if err != nil {
		return cmd, err
	}

	err = cmd.Wait()
	return cmd, err
}

type CredentialCmd struct {
	bufOut *bytes.Buffer
	bufErr *bytes.Buffer
	*exec.Cmd
}

func NewCommand(input, subCommand string) *CredentialCmd {
	buf1 := new(bytes.Buffer)
	buf2 := new(bytes.Buffer)
	cmd := exec.Command("git", "credential", subCommand)
	cmd.Stdin = bytes.NewBufferString(input)
	cmd.Stdout = buf1
	cmd.Stderr = buf2
	return &CredentialCmd{buf1, buf2, cmd}
}

func (c *CredentialCmd) StderrString() string {
	return c.bufErr.String()
}

func (c *CredentialCmd) StdoutString() string {
	return c.bufOut.String()
}

func (c *CredentialCmd) Credentials() map[string]string {
	creds := make(map[string]string)

	for _, line := range strings.Split(c.StdoutString(), "\n") {
		pieces := strings.SplitN(line, "=", 2)
		if len(pieces) < 2 {
			continue
		}
		creds[pieces[0]] = pieces[1]
	}

	return creds
}

type Error struct {
	Message   string `json:"message"`
	RequestId string `json:"request_id,omitempty"`
}

func (e *Error) Error() string {
	return e.Message
}
<commit_msg>ンンー ンンンン ンーンン<commit_after>package gitmediaclient

import (
	"bytes"
	"encoding/base64"
	"encoding/json"
	"fmt"
	"io"
	"net/http"
	"net/url"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
)

func Put(filename string) error {
	oid := filepath.Base(filename)
	stat, err := os.Stat(filename)
	if err != nil {
		return err
	}

	file, err := os.Open(filename)
	if err != nil {
		return err
	}

	req, _, err := clientRequest("PUT", oid)
	if err != nil {
		return err
	}
	req.Body = file
	req.ContentLength = stat.Size()

	res, err := http.DefaultClient.Do(req)
	if err != nil {
		return err
	}

	defer res.Body.Close()

	if res.StatusCode > 299 {
		apierr := &Error{}
		dec := json.NewDecoder(res.Body)
		if err = dec.Decode(apierr); err != nil {
			return err
		}
		return apierr
	}

	fmt.Printf("Sending %s from %s: %d\n", oid, filename, res.StatusCode)
	return nil
}

func Get(filename string) (io.ReadCloser, error) {
	oid := filepath.Base(filename)
	if stat, err := os.Stat(filename); err != nil || stat == nil {
		req, _, err := clientRequest("GET", oid)
		if err != nil {
			return nil, err
		}

		req.Header.Set("Accept", "application/vnd.git-media")
		res, err := http.DefaultClient.Do(req)
		if err != nil {
			return nil, err
		}

		return res.Body, nil
	}

	return os.Open(filename)
}

func clientRequest(method, oid string) (*http.Request, map[string]string, error) {
	u := objectUrl(oid)
	req, err := http.NewRequest(method, u.String(), nil)
	if err == nil {
		creds, err := credentials(u)
		if err != nil {
			return req, nil, err
		}

		token := fmt.Sprintf("%s:%s", creds["username"], creds["password"])
		auth := "Basic " + base64.URLEncoding.EncodeToString([]byte(token))
		req.Header.Set("Authorization", auth)
		return req, creds, nil
	}

	return req, nil, err
}

func objectUrl(oid string) *url.URL {
	u, _ := url.Parse("http://localhost:8080")
	u.Path = "/objects/" + oid
	return u
}

func credentials(u *url.URL) (map[string]string, error) {
	credInput := fmt.Sprintf("protocol=%s\nhost=%s\n", u.Scheme, u.Host)
	cmd, err := execCreds(credInput, "fill")
	if err != nil {
		return nil, err
	}
	return cmd.Credentials(), nil
}

func execCreds(input, subCommand string) (*CredentialCmd, error) {
	cmd := NewCommand(input, subCommand)
	err := cmd.Start()
	if err != nil {
		return cmd, err
	}

	err = cmd.Wait()
	return cmd, err
}

type CredentialCmd struct {
	bufOut *bytes.Buffer
	bufErr *bytes.Buffer
	*exec.Cmd
}

func NewCommand(input, subCommand string) *CredentialCmd {
	buf1 := new(bytes.Buffer)
	buf2 := new(bytes.Buffer)
	cmd := exec.Command("git", "credential", subCommand)
	cmd.Stdin = bytes.NewBufferString(input)
	cmd.Stdout = buf1
	cmd.Stderr = buf2
	return &CredentialCmd{buf1, buf2, cmd}
}

func (c *CredentialCmd) StderrString() string {
	return c.bufErr.String()
}

func (c *CredentialCmd) StdoutString() string {
	return c.bufOut.String()
}

func (c *CredentialCmd) Credentials() map[string]string {
	creds := make(map[string]string)

	for _, line := range strings.Split(c.StdoutString(), "\n") {
		pieces := strings.SplitN(line, "=", 2)
		if len(pieces) < 2 {
			continue
		}
		creds[pieces[0]] = pieces[1]
	}

	return creds
}

type Error struct {
	Message   string `json:"message"`
	RequestId string `json:"request_id,omitempty"`
}

func (e *Error) Error() string {
	return e.Message
}
<|endoftext|>


<commit_before>// Copyright (c) 2017 Intel Corporation
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"errors"
	"fmt"
	"io/ioutil"
	"os"
	"path"
	"regexp"
	"testing"
	"time"

	"github.com/Sirupsen/logrus"
	"github.com/stretchr/testify/assert"
)

type testData struct {
	path          string
	expectFailure bool
}

func init() {
	// Ensure all log levels are logged
	ccLog.Level = logrus.DebugLevel

	// Discard "normal" log output: this test only cares about the
	// (additional) global log output
	ccLog.Out = ioutil.Discard
}

func grep(pattern, file string) error {
	if file == "" {
		return errors.New("need file")
	}

	bytes, err := ioutil.ReadFile(file)
	if err != nil {
		return err
	}

	re := regexp.MustCompile(pattern)
	matches := re.FindAllStringSubmatch(string(bytes), -1)

	if matches == nil {
		return fmt.Errorf("pattern %q not found in file %q", pattern, file)
	}

	return nil
}

func TestNewGlobalLogHook(t *testing.T) {
	tmpdir, err := ioutil.TempDir("", "")
	if err != nil {
		t.Fatal(err)
	}
	defer os.RemoveAll(tmpdir)

	tmpfile := path.Join(tmpdir, "global.log")

	data := []testData{
		{"", true},
		{tmpfile, false},
	}

	for _, d := range data {
		hook, err := newGlobalLogHook(d.path)
		if d.expectFailure {
			if err == nil {
				t.Fatal(fmt.Sprintf("unexpected succes from newGlobalLogHook(path=%v)", d.path))
			}
		} else {
			if err != nil {
				t.Fatal(fmt.Sprintf("unexpected failure from newGlobalLogHook(path=%q): %v", d.path, err))
			}
			if hook.path != d.path {
				t.Fatal(fmt.Sprintf("expected hook to contain path %q, found %q", d.path, hook.path))
			}
		}
	}
}

func TestHandleGlobalLog(t *testing.T) {
	tmpdir, err := ioutil.TempDir("", "")
	if err != nil {
		t.Fatal(err)
	}
	defer os.RemoveAll(tmpdir)

	subDir := path.Join(tmpdir, "a/b/global.log")
	err = os.MkdirAll(subDir, testDirMode)
	assert.NoError(t, err)

	existingFile := path.Join(tmpdir, "c")
	err = createEmptyFile(existingFile)
	assert.NoError(t, err)

	tmpfile := path.Join(tmpdir, "global.log")

	data := []testData{
		{"", false},

		// path must be absolute, so these should fail
		{"foo/bar/global.log", true},
		{"../foo/bar/global.log", true},
		{"./foo/bar/global.log", true},
		{subDir, true},
		{path.Join(existingFile, "global.log"), true},

		{tmpfile, false},
	}

	for _, d := range data {
		err := handleGlobalLog(d.path)
		if d.expectFailure {
			if err == nil {
				t.Fatal(fmt.Sprintf("unexpected success from handleGlobalLog(path=%q)", d.path))
			}
			continue
		}

		if err != nil {
			t.Fatal(fmt.Sprintf("unexpected failure from handleGlobalLog(path=%q): %v", d.path, err))
		}

		// It's valid to pass a blank path to handleGlobalLog(),
		// but no point in checking for log entries in that
		// case!
		if d.path == "" {
			continue
		}

		// Add a log entry
		str := "hello. foo bar baz!"
		ccLog.Debug(str)

		// Check that the string was logged
		err = grep(fmt.Sprintf("debug:.*%s", str), d.path)
		if err != nil {
			t.Fatal(err)
		}

		// Check expected perms
		st, err := os.Stat(d.path)
		if err != nil {
			t.Fatal(err)
		}

		expectedPerms := "-rw-r-----"
		actualPerms := st.Mode().String()
		if expectedPerms != actualPerms {
			t.Fatal(fmt.Sprintf("logfile %v should have perms %v, but found %v",
				d.path, expectedPerms, actualPerms))
		}
	}
}

func TestHandleGlobalLogEnvVar(t *testing.T) {
	envvar := "CC_RUNTIME_GLOBAL_LOG"
	tmpdir, err := ioutil.TempDir("", "")
	if err != nil {
		t.Fatal(err)
	}
	defer os.RemoveAll(tmpdir)

	tmpfile := path.Join(tmpdir, "global.log")
	tmpfile2 := path.Join(tmpdir, "global-envvar.log")

	os.Setenv(envvar, tmpfile2)
	defer os.Unsetenv(envvar)

	err = handleGlobalLog(tmpfile)
	if err != nil {
		t.Fatal(err)
	}

	str := "foo or moo?"
	ccLog.Debug(str)
	tmpfileExists := fileExists(tmpfile)
	tmpfile2Exists := fileExists(tmpfile2)

	if tmpfileExists == true {
		t.Fatal(fmt.Sprintf("tmpfile %q exists unexpectedly", tmpfile))
	}

	if tmpfile2Exists == false {
		t.Fatal(fmt.Sprintf("tmpfile2 %q does not exist unexpectedly", tmpfile2))
	}

	// Check that the string was logged
	err = grep(fmt.Sprintf("debug:.*%s", str), tmpfile2)
	if err != nil {
		t.Fatal(err)
	}
}

func TestLoggerFire(t *testing.T) {
	tmpdir, err := ioutil.TempDir("", "")
	if err != nil {
		t.Fatal(err)
	}

	ccLog = logrus.New()

	logFile := path.Join(tmpdir, "a/b/global.log")
	err = handleGlobalLog(logFile)
	assert.NoError(t, err)

	//ccLog.Debug("foo")

	entry := &logrus.Entry{
		Logger:  ccLog,
		Time:    time.Now().UTC(),
		Level:   logrus.DebugLevel,
		Message: "foo",
	}

	err = ccLog.Hooks.Fire(logrus.DebugLevel, entry)
	assert.NoError(t, err)

	assert.Equal(t, len(ccLog.Hooks[logrus.DebugLevel]), 1)
	hook, ok := ccLog.Hooks[logrus.DebugLevel][0].(*GlobalLogHook)
	assert.True(t, ok)

	err = hook.file.Close()
	assert.NoError(t, err)

	err = os.RemoveAll(tmpdir)
	assert.NoError(t, err)

	err = ccLog.Hooks.Fire(logrus.DebugLevel, entry)
	assert.Error(t, err)
}
<commit_msg>tests: Remove comment.<commit_after>// Copyright (c) 2017 Intel Corporation
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"errors"
	"fmt"
	"io/ioutil"
	"os"
	"path"
	"regexp"
	"testing"
	"time"

	"github.com/Sirupsen/logrus"
	"github.com/stretchr/testify/assert"
)

type testData struct {
	path          string
	expectFailure bool
}

func init() {
	// Ensure all log levels are logged
	ccLog.Level = logrus.DebugLevel

	// Discard "normal" log output: this test only cares about the
	// (additional) global log output
	ccLog.Out = ioutil.Discard
}

func grep(pattern, file string) error {
	if file == "" {
		return errors.New("need file")
	}

	bytes, err := ioutil.ReadFile(file)
	if err != nil {
		return err
	}

	re := regexp.MustCompile(pattern)
	matches := re.FindAllStringSubmatch(string(bytes), -1)

	if matches == nil {
		return fmt.Errorf("pattern %q not found in file %q", pattern, file)
	}

	return nil
}

func TestNewGlobalLogHook(t *testing.T) {
	tmpdir, err := ioutil.TempDir("", "")
	if err != nil {
		t.Fatal(err)
	}
	defer os.RemoveAll(tmpdir)

	tmpfile := path.Join(tmpdir, "global.log")

	data := []testData{
		{"", true},
		{tmpfile, false},
	}

	for _, d := range data {
		hook, err := newGlobalLogHook(d.path)
		if d.expectFailure {
			if err == nil {
				t.Fatal(fmt.Sprintf("unexpected succes from newGlobalLogHook(path=%v)", d.path))
			}
		} else {
			if err != nil {
				t.Fatal(fmt.Sprintf("unexpected failure from newGlobalLogHook(path=%q): %v", d.path, err))
			}
			if hook.path != d.path {
				t.Fatal(fmt.Sprintf("expected hook to contain path %q, found %q", d.path, hook.path))
			}
		}
	}
}

func TestHandleGlobalLog(t *testing.T) {
	tmpdir, err := ioutil.TempDir("", "")
	if err != nil {
		t.Fatal(err)
	}
	defer os.RemoveAll(tmpdir)

	subDir := path.Join(tmpdir, "a/b/global.log")
	err = os.MkdirAll(subDir, testDirMode)
	assert.NoError(t, err)

	existingFile := path.Join(tmpdir, "c")
	err = createEmptyFile(existingFile)
	assert.NoError(t, err)

	tmpfile := path.Join(tmpdir, "global.log")

	data := []testData{
		{"", false},

		// path must be absolute, so these should fail
		{"foo/bar/global.log", true},
		{"../foo/bar/global.log", true},
		{"./foo/bar/global.log", true},
		{subDir, true},
		{path.Join(existingFile, "global.log"), true},

		{tmpfile, false},
	}

	for _, d := range data {
		err := handleGlobalLog(d.path)
		if d.expectFailure {
			if err == nil {
				t.Fatal(fmt.Sprintf("unexpected success from handleGlobalLog(path=%q)", d.path))
			}
			continue
		}

		if err != nil {
			t.Fatal(fmt.Sprintf("unexpected failure from handleGlobalLog(path=%q): %v", d.path, err))
		}

		// It's valid to pass a blank path to handleGlobalLog(),
		// but no point in checking for log entries in that
		// case!
		if d.path == "" {
			continue
		}

		// Add a log entry
		str := "hello. foo bar baz!"
		ccLog.Debug(str)

		// Check that the string was logged
		err = grep(fmt.Sprintf("debug:.*%s", str), d.path)
		if err != nil {
			t.Fatal(err)
		}

		// Check expected perms
		st, err := os.Stat(d.path)
		if err != nil {
			t.Fatal(err)
		}

		expectedPerms := "-rw-r-----"
		actualPerms := st.Mode().String()
		if expectedPerms != actualPerms {
			t.Fatal(fmt.Sprintf("logfile %v should have perms %v, but found %v",
				d.path, expectedPerms, actualPerms))
		}
	}
}

func TestHandleGlobalLogEnvVar(t *testing.T) {
	envvar := "CC_RUNTIME_GLOBAL_LOG"
	tmpdir, err := ioutil.TempDir("", "")
	if err != nil {
		t.Fatal(err)
	}
	defer os.RemoveAll(tmpdir)

	tmpfile := path.Join(tmpdir, "global.log")
	tmpfile2 := path.Join(tmpdir, "global-envvar.log")

	os.Setenv(envvar, tmpfile2)
	defer os.Unsetenv(envvar)

	err = handleGlobalLog(tmpfile)
	if err != nil {
		t.Fatal(err)
	}

	str := "foo or moo?"
	ccLog.Debug(str)
	tmpfileExists := fileExists(tmpfile)
	tmpfile2Exists := fileExists(tmpfile2)

	if tmpfileExists == true {
		t.Fatal(fmt.Sprintf("tmpfile %q exists unexpectedly", tmpfile))
	}

	if tmpfile2Exists == false {
		t.Fatal(fmt.Sprintf("tmpfile2 %q does not exist unexpectedly", tmpfile2))
	}

	// Check that the string was logged
	err = grep(fmt.Sprintf("debug:.*%s", str), tmpfile2)
	if err != nil {
		t.Fatal(err)
	}
}

func TestLoggerFire(t *testing.T) {
	tmpdir, err := ioutil.TempDir("", "")
	if err != nil {
		t.Fatal(err)
	}

	ccLog = logrus.New()

	logFile := path.Join(tmpdir, "a/b/global.log")
	err = handleGlobalLog(logFile)
	assert.NoError(t, err)

	entry := &logrus.Entry{
		Logger:  ccLog,
		Time:    time.Now().UTC(),
		Level:   logrus.DebugLevel,
		Message: "foo",
	}

	err = ccLog.Hooks.Fire(logrus.DebugLevel, entry)
	assert.NoError(t, err)

	assert.Equal(t, len(ccLog.Hooks[logrus.DebugLevel]), 1)
	hook, ok := ccLog.Hooks[logrus.DebugLevel][0].(*GlobalLogHook)
	assert.True(t, ok)

	err = hook.file.Close()
	assert.NoError(t, err)

	err = os.RemoveAll(tmpdir)
	assert.NoError(t, err)

	err = ccLog.Hooks.Fire(logrus.DebugLevel, entry)
	assert.Error(t, err)
}
<|endoftext|>


<commit_before>package admin

import (
	"fmt"
	"reflect"
	"strconv"
	"strings"
	"time"

	"github.com/jinzhu/gorm"
	"github.com/jinzhu/now"
	"github.com/qor/qor"
	"github.com/qor/qor/resource"
	"github.com/qor/qor/roles"
	"github.com/qor/qor/utils"
)

type Resource struct {
	resource.Resource
	admin         *Admin
	Config        *Config
	Metas         []*Meta
	actions       []*Action
	scopes        []*Scope
	filters       map[string]*Filter
	searchAttrs   []string
	indexAttrs    []string
	newAttrs      []string
	editAttrs     []string
	showAttrs     []string
	cachedMetas   *map[string][]*Meta
	SearchHandler func(keyword string, context *qor.Context) *gorm.DB
}

func (res *Resource) Meta(meta *Meta) {
	if res.GetMeta(meta.Name) != nil {
		utils.ExitWithMsg("Duplicated meta %v defined for resource %v", meta.Name, res.Name)
	}

	meta.base = res
	meta.updateMeta()
	res.Metas = append(res.Metas, meta)
}

func (res Resource) GetAdmin() *Admin {
	return res.admin
}

func (res Resource) ToParam() string {
	return utils.ToParamString(res.Name)
}

func (res Resource) UseTheme(theme string) []string {
	if res.Config != nil {
		res.Config.Themes = append(res.Config.Themes, theme)
		return res.Config.Themes
	}
	return []string{}
}

func (res *Resource) convertObjectToMap(context *Context, value interface{}, kind string) interface{} {
	reflectValue := reflect.Indirect(reflect.ValueOf(value))
	switch reflectValue.Kind() {
	case reflect.Slice:
		values := []interface{}{}
		for i := 0; i < reflectValue.Len(); i++ {
			values = append(values, res.convertObjectToMap(context, reflectValue.Index(i).Interface(), kind))
		}
		return values
	case reflect.Struct:
		var metas []*Meta
		if kind == "index" {
			metas = res.indexMetas()
		} else if kind == "show" {
			metas = res.showMetas()
		}

		values := map[string]interface{}{}
		for _, meta := range metas {
			if meta.HasPermission(roles.Read, context.Context) {
				value := meta.GetValuer()(value, context.Context)
				if meta.Resource != nil {
					value = meta.Resource.(*Resource).convertObjectToMap(context, value, kind)
				}
				values[meta.GetName()] = value
			}
		}
		return values
	default:
		panic(fmt.Sprintf("Can't convert %v (%v) to map", reflectValue, reflectValue.Kind()))
	}
}

func (res *Resource) Decode(context *qor.Context, value interface{}) (errs []error) {
	return resource.Decode(context, value, res)
}

func (res *Resource) IndexAttrs(columns ...string) []string {
	if len(columns) > 0 {
		res.indexAttrs = columns
	}
	return res.indexAttrs
}

func (res *Resource) NewAttrs(columns ...string) []string {
	if len(columns) > 0 {
		res.newAttrs = columns
	}
	return res.newAttrs
}

func (res *Resource) EditAttrs(columns ...string) []string {
	if len(columns) > 0 {
		res.editAttrs = columns
	}
	return res.editAttrs
}

func (res *Resource) ShowAttrs(columns ...string) []string {
	if len(columns) > 0 {
		res.showAttrs = columns
	}
	return res.showAttrs
}

func (res *Resource) SearchAttrs(columns ...string) []string {
	if len(columns) > 0 {
		res.searchAttrs = columns
		res.SearchHandler = func(keyword string, context *qor.Context) *gorm.DB {
			db := context.GetDB()
			var conditions []string
			var keywords []interface{}
			scope := db.NewScope(res.Value)

			for _, column := range columns {
				if field, ok := scope.FieldByName(column); ok {
					switch field.Field.Kind() {
					case reflect.String:
						conditions = append(conditions, fmt.Sprintf("upper(%v) like upper(?)", scope.Quote(field.DBName)))
						keywords = append(keywords, "%"+keyword+"%")
					case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
						if _, err := strconv.Atoi(keyword); err == nil {
							conditions = append(conditions, fmt.Sprintf("%v = ?", scope.Quote(field.DBName)))
							keywords = append(keywords, keyword)
						}
					case reflect.Float32, reflect.Float64:
						if _, err := strconv.ParseFloat(keyword, 64); err == nil {
							conditions = append(conditions, fmt.Sprintf("%v = ?", scope.Quote(field.DBName)))
							keywords = append(keywords, keyword)
						}
					case reflect.Struct:
						// time ?
						if _, ok := field.Field.Interface().(time.Time); ok {
							if parsedTime, err := now.Parse(keyword); err == nil {
								conditions = append(conditions, fmt.Sprintf("%v = ?", scope.Quote(field.DBName)))
								keywords = append(keywords, parsedTime)
							}
						}
					case reflect.Ptr:
						// time ?
						if _, ok := field.Field.Interface().(*time.Time); ok {
							if parsedTime, err := now.Parse(keyword); err == nil {
								conditions = append(conditions, fmt.Sprintf("%v = ?", scope.Quote(field.DBName)))
								keywords = append(keywords, parsedTime)
							}
						}
					default:
						conditions = append(conditions, fmt.Sprintf("%v = ?", scope.Quote(field.DBName)))
						keywords = append(keywords, keyword)
					}
				}
			}

			if len(conditions) > 0 {
				return context.GetDB().Where(strings.Join(conditions, " OR "), keywords...)
			} else {
				return context.GetDB()
			}
		}
	}
	return res.searchAttrs
}

func (res *Resource) getCachedMetas(cacheKey string, fc func() []resource.Metaor) []*Meta {
	if res.cachedMetas == nil {
		res.cachedMetas = &map[string][]*Meta{}
	}

	if values, ok := (*res.cachedMetas)[cacheKey]; ok {
		return values
	} else {
		values := fc()
		var metas []*Meta
		for _, value := range values {
			metas = append(metas, value.(*Meta))
		}
		(*res.cachedMetas)[cacheKey] = metas
		return metas
	}
}

func (res *Resource) GetMetas(_attrs ...[]string) []resource.Metaor {
	var attrs, ignoredAttrs []string
	for _, value := range _attrs {
		if value != nil {
			for _, v := range value {
				if strings.HasPrefix(v, "-") {
					ignoredAttrs = append(ignoredAttrs, strings.TrimLeft(v, "-"))
				} else {
					attrs = append(attrs, v)
				}
			}
			break
		}
	}

	if attrs == nil {
		scope := &gorm.Scope{Value: res.Value}
		structFields := scope.GetModelStruct().StructFields
		attrs = []string{}

	Fields:
		for _, field := range structFields {
			for _, attr := range ignoredAttrs {
				if attr == field.Name {
					continue Fields
				}
			}

			for _, meta := range res.Metas {
				if field.Name == meta.Alias {
					attrs = append(attrs, meta.Name)
					continue Fields
				}
			}

			if field.IsForeignKey {
				continue
			}

			for _, value := range []string{"CreatedAt", "UpdatedAt", "DeletedAt"} {
				if value == field.Name {
					continue Fields
				}
			}

			attrs = append(attrs, field.Name)
		}

	MetaIncluded:
		for _, meta := range res.Metas {
			for _, attr := range ignoredAttrs {
				if attr == meta.Name || attr == meta.Alias {
					continue MetaIncluded
				}
			}

			for _, attr := range attrs {
				if attr == meta.Alias || attr == meta.Name {
					continue MetaIncluded
				}
			}
			attrs = append(attrs, meta.Name)
		}
	}

	primaryKey := res.PrimaryFieldName()

	metas := []resource.Metaor{}
	for _, attr := range attrs {
		var meta *Meta
		for _, m := range res.Metas {
			if m.GetName() == attr {
				meta = m
				break
			}
		}

		if meta == nil {
			meta = &Meta{}
			meta.Name = attr
			meta.base = res
			if attr == primaryKey {
				meta.Type = "hidden"
			}
			meta.updateMeta()
		}

		metas = append(metas, meta)
	}

	return metas
}

func (res *Resource) GetMeta(name string) *Meta {
	for _, meta := range res.Metas {
		if meta.Name == name || meta.GetFieldName() == name {
			return meta
		}
	}
	return nil
}

func (res *Resource) indexMetas() []*Meta {
	return res.getCachedMetas("index_metas", func() []resource.Metaor {
		return res.GetMetas(res.indexAttrs, res.showAttrs)
	})
}

func (res *Resource) newMetas() []*Meta {
	return res.getCachedMetas("new_metas", func() []resource.Metaor {
		return res.GetMetas(res.newAttrs, res.editAttrs)
	})
}

func (res *Resource) editMetas() []*Meta {
	return res.getCachedMetas("edit_metas", func() []resource.Metaor {
		return res.GetMetas(res.editAttrs)
	})
}

func (res *Resource) showMetas() []*Meta {
	return res.getCachedMetas("show_metas", func() []resource.Metaor {
		return res.GetMetas(res.showAttrs, res.editAttrs)
	})
}

func (res *Resource) allMetas() []*Meta {
	return res.getCachedMetas("all_metas", func() []resource.Metaor {
		return res.GetMetas()
	})
}

func (res *Resource) allowedMetas(attrs []*Meta, context *Context, roles ...roles.PermissionMode) []*Meta {
	var metas = []*Meta{}
	for _, meta := range attrs {
		for _, role := range roles {
			if meta.HasPermission(role, context.Context) {
				metas = append(metas, meta)
				break
			}
		}
	}
	return metas
}

func (res *Resource) HasPermission(mode roles.PermissionMode, context *qor.Context) bool {
	if res.Config == nil || res.Config.Permission == nil {
		return true
	}
	return res.Config.Permission.HasPermission(mode, context.Roles...)
}
<commit_msg>Refactor get metas<commit_after>package admin

import (
	"fmt"
	"reflect"
	"strconv"
	"strings"
	"time"

	"github.com/jinzhu/gorm"
	"github.com/jinzhu/now"
	"github.com/qor/qor"
	"github.com/qor/qor/resource"
	"github.com/qor/qor/roles"
	"github.com/qor/qor/utils"
)

type Resource struct {
	resource.Resource
	admin         *Admin
	Config        *Config
	Metas         []*Meta
	actions       []*Action
	scopes        []*Scope
	filters       map[string]*Filter
	searchAttrs   []string
	indexAttrs    []string
	newAttrs      []string
	editAttrs     []string
	showAttrs     []string
	cachedMetas   *map[string][]*Meta
	SearchHandler func(keyword string, context *qor.Context) *gorm.DB
}

func (res *Resource) Meta(meta *Meta) {
	if res.GetMeta(meta.Name) != nil {
		utils.ExitWithMsg("Duplicated meta %v defined for resource %v", meta.Name, res.Name)
	}

	meta.base = res
	meta.updateMeta()
	res.Metas = append(res.Metas, meta)
}

func (res Resource) GetAdmin() *Admin {
	return res.admin
}

func (res Resource) ToParam() string {
	return utils.ToParamString(res.Name)
}

func (res Resource) UseTheme(theme string) []string {
	if res.Config != nil {
		res.Config.Themes = append(res.Config.Themes, theme)
		return res.Config.Themes
	}
	return []string{}
}

func (res *Resource) convertObjectToMap(context *Context, value interface{}, kind string) interface{} {
	reflectValue := reflect.Indirect(reflect.ValueOf(value))
	switch reflectValue.Kind() {
	case reflect.Slice:
		values := []interface{}{}
		for i := 0; i < reflectValue.Len(); i++ {
			values = append(values, res.convertObjectToMap(context, reflectValue.Index(i).Interface(), kind))
		}
		return values
	case reflect.Struct:
		var metas []*Meta
		if kind == "index" {
			metas = res.indexMetas()
		} else if kind == "show" {
			metas = res.showMetas()
		}

		values := map[string]interface{}{}
		for _, meta := range metas {
			if meta.HasPermission(roles.Read, context.Context) {
				value := meta.GetValuer()(value, context.Context)
				if meta.Resource != nil {
					value = meta.Resource.(*Resource).convertObjectToMap(context, value, kind)
				}
				values[meta.GetName()] = value
			}
		}
		return values
	default:
		panic(fmt.Sprintf("Can't convert %v (%v) to map", reflectValue, reflectValue.Kind()))
	}
}

func (res *Resource) Decode(context *qor.Context, value interface{}) (errs []error) {
	return resource.Decode(context, value, res)
}

func (res *Resource) IndexAttrs(columns ...string) []string {
	if len(columns) > 0 {
		res.indexAttrs = columns
	}
	return res.indexAttrs
}

func (res *Resource) NewAttrs(columns ...string) []string {
	if len(columns) > 0 {
		res.newAttrs = columns
	}
	return res.newAttrs
}

func (res *Resource) EditAttrs(columns ...string) []string {
	if len(columns) > 0 {
		res.editAttrs = columns
	}
	return res.editAttrs
}

func (res *Resource) ShowAttrs(columns ...string) []string {
	if len(columns) > 0 {
		res.showAttrs = columns
	}
	return res.showAttrs
}

func (res *Resource) SearchAttrs(columns ...string) []string {
	if len(columns) > 0 {
		res.searchAttrs = columns
		res.SearchHandler = func(keyword string, context *qor.Context) *gorm.DB {
			db := context.GetDB()
			var conditions []string
			var keywords []interface{}
			scope := db.NewScope(res.Value)

			for _, column := range columns {
				if field, ok := scope.FieldByName(column); ok {
					switch field.Field.Kind() {
					case reflect.String:
						conditions = append(conditions, fmt.Sprintf("upper(%v) like upper(?)", scope.Quote(field.DBName)))
						keywords = append(keywords, "%"+keyword+"%")
					case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
						if _, err := strconv.Atoi(keyword); err == nil {
							conditions = append(conditions, fmt.Sprintf("%v = ?", scope.Quote(field.DBName)))
							keywords = append(keywords, keyword)
						}
					case reflect.Float32, reflect.Float64:
						if _, err := strconv.ParseFloat(keyword, 64); err == nil {
							conditions = append(conditions, fmt.Sprintf("%v = ?", scope.Quote(field.DBName)))
							keywords = append(keywords, keyword)
						}
					case reflect.Struct:
						// time ?
						if _, ok := field.Field.Interface().(time.Time); ok {
							if parsedTime, err := now.Parse(keyword); err == nil {
								conditions = append(conditions, fmt.Sprintf("%v = ?", scope.Quote(field.DBName)))
								keywords = append(keywords, parsedTime)
							}
						}
					case reflect.Ptr:
						// time ?
						if _, ok := field.Field.Interface().(*time.Time); ok {
							if parsedTime, err := now.Parse(keyword); err == nil {
								conditions = append(conditions, fmt.Sprintf("%v = ?", scope.Quote(field.DBName)))
								keywords = append(keywords, parsedTime)
							}
						}
					default:
						conditions = append(conditions, fmt.Sprintf("%v = ?", scope.Quote(field.DBName)))
						keywords = append(keywords, keyword)
					}
				}
			}

			if len(conditions) > 0 {
				return context.GetDB().Where(strings.Join(conditions, " OR "), keywords...)
			} else {
				return context.GetDB()
			}
		}
	}
	return res.searchAttrs
}

func (res *Resource) getCachedMetas(cacheKey string, fc func() []resource.Metaor) []*Meta {
	if res.cachedMetas == nil {
		res.cachedMetas = &map[string][]*Meta{}
	}

	if values, ok := (*res.cachedMetas)[cacheKey]; ok {
		return values
	} else {
		values := fc()
		var metas []*Meta
		for _, value := range values {
			metas = append(metas, value.(*Meta))
		}
		(*res.cachedMetas)[cacheKey] = metas
		return metas
	}
}

func (res *Resource) GetMetas(_attrs ...[]string) []resource.Metaor {
	var attrs, ignoredAttrs []string
	for _, value := range _attrs {
		if len(value) != 0 {
			attrs, ignoredAttrs = []string{}, []string{}
			for _, v := range value {
				if strings.HasPrefix(v, "-") {
					ignoredAttrs = append(ignoredAttrs, strings.TrimLeft(v, "-"))
				} else {
					attrs = append(attrs, v)
				}
			}
			if len(attrs) > 0 {
				break
			}
		}
	}

	if len(attrs) == 0 {
		scope := &gorm.Scope{Value: res.Value}
		structFields := scope.GetModelStruct().StructFields

	Fields:
		for _, field := range structFields {
			for _, meta := range res.Metas {
				if field.Name == meta.Alias {
					attrs = append(attrs, meta.Name)
					continue Fields
				}
			}

			if field.IsForeignKey {
				continue
			}

			for _, value := range []string{"CreatedAt", "UpdatedAt", "DeletedAt"} {
				if value == field.Name {
					continue Fields
				}
			}

			attrs = append(attrs, field.Name)
		}

	MetaIncluded:
		for _, meta := range res.Metas {
			for _, attr := range attrs {
				if attr == meta.Alias || attr == meta.Name {
					continue MetaIncluded
				}
			}
			attrs = append(attrs, meta.Name)
		}
	}

	primaryKey := res.PrimaryFieldName()

	metas := []resource.Metaor{}
Attrs:
	for _, attr := range attrs {
		for _, a := range ignoredAttrs {
			if attr == a {
				continue Attrs
			}
		}

		var meta *Meta
		for _, m := range res.Metas {
			if m.GetName() == attr {
				meta = m
				break
			}
		}

		if meta == nil {
			meta = &Meta{}
			meta.Name = attr
			meta.base = res
			if attr == primaryKey {
				meta.Type = "hidden"
			}
			meta.updateMeta()
		}

		metas = append(metas, meta)
	}

	return metas
}

func (res *Resource) GetMeta(name string) *Meta {
	for _, meta := range res.Metas {
		if meta.Name == name || meta.GetFieldName() == name {
			return meta
		}
	}
	return nil
}

func (res *Resource) indexMetas() []*Meta {
	return res.getCachedMetas("index_metas", func() []resource.Metaor {
		return res.GetMetas(res.indexAttrs, res.showAttrs)
	})
}

func (res *Resource) newMetas() []*Meta {
	return res.getCachedMetas("new_metas", func() []resource.Metaor {
		return res.GetMetas(res.newAttrs, res.editAttrs)
	})
}

func (res *Resource) editMetas() []*Meta {
	return res.getCachedMetas("edit_metas", func() []resource.Metaor {
		return res.GetMetas(res.editAttrs)
	})
}

func (res *Resource) showMetas() []*Meta {
	return res.getCachedMetas("show_metas", func() []resource.Metaor {
		return res.GetMetas(res.showAttrs, res.editAttrs)
	})
}

func (res *Resource) allMetas() []*Meta {
	return res.getCachedMetas("all_metas", func() []resource.Metaor {
		return res.GetMetas()
	})
}

func (res *Resource) allowedMetas(attrs []*Meta, context *Context, roles ...roles.PermissionMode) []*Meta {
	var metas = []*Meta{}
	for _, meta := range attrs {
		for _, role := range roles {
			if meta.HasPermission(role, context.Context) {
				metas = append(metas, meta)
				break
			}
		}
	}
	return metas
}

func (res *Resource) HasPermission(mode roles.PermissionMode, context *qor.Context) bool {
	if res.Config == nil || res.Config.Permission == nil {
		return true
	}
	return res.Config.Permission.HasPermission(mode, context.Roles...)
}
<|endoftext|>


<commit_before>package core

import (
	"bytes"
	"fmt"
	"math/big"
	"sync"

	"github.com/ethereum/go-ethereum/core/types"
	"github.com/ethereum/go-ethereum/ethutil"
	"github.com/ethereum/go-ethereum/event"
	"github.com/ethereum/go-ethereum/logger"
	"github.com/ethereum/go-ethereum/rlp"
	"github.com/ethereum/go-ethereum/state"
)

var chainlogger = logger.NewLogger("CHAIN")

type StateQuery interface {
	GetAccount(addr []byte) *state.StateObject
}

func CalcDifficulty(block, parent *types.Block) *big.Int {
	diff := new(big.Int)

	bh, ph := block.Header(), parent.Header()
	adjust := new(big.Int).Rsh(ph.Difficulty, 10)
	if bh.Time >= ph.Time+13 {
		diff.Sub(ph.Difficulty, adjust)
	} else {
		diff.Add(ph.Difficulty, adjust)
	}

	return diff
}

func CalcGasLimit(parent, block *types.Block) *big.Int {
	if block.Number().Cmp(big.NewInt(0)) == 0 {
		return ethutil.BigPow(10, 6)
	}

	// ((1024-1) * parent.gasLimit + (gasUsed * 6 / 5)) / 1024

	previous := new(big.Int).Mul(big.NewInt(1024-1), parent.GasLimit())
	current := new(big.Rat).Mul(new(big.Rat).SetInt(parent.GasUsed()), big.NewRat(6, 5))
	curInt := new(big.Int).Div(current.Num(), current.Denom())

	result := new(big.Int).Add(previous, curInt)
	result.Div(result, big.NewInt(1024))

	min := big.NewInt(125000)

	return ethutil.BigMax(min, result)
}

type ChainManager struct {
	//eth          EthManager
	db           ethutil.Database
	processor    types.BlockProcessor
	eventMux     *event.TypeMux
	genesisBlock *types.Block
	// Last known total difficulty
	mu              sync.RWMutex
	td              *big.Int
	lastBlockNumber uint64
	currentBlock    *types.Block
	lastBlockHash   []byte

	transState *state.StateDB
}

func (self *ChainManager) Td() *big.Int {
	self.mu.RLock()
	defer self.mu.RUnlock()

	return self.td
}

func (self *ChainManager) LastBlockNumber() uint64 {
	self.mu.RLock()
	defer self.mu.RUnlock()

	return self.lastBlockNumber
}

func (self *ChainManager) LastBlockHash() []byte {
	self.mu.RLock()
	defer self.mu.RUnlock()

	return self.lastBlockHash
}

func (self *ChainManager) CurrentBlock() *types.Block {
	self.mu.RLock()
	defer self.mu.RUnlock()

	return self.currentBlock
}

func NewChainManager(db ethutil.Database, mux *event.TypeMux) *ChainManager {
	bc := &ChainManager{db: db, genesisBlock: GenesisBlock(db), eventMux: mux}
	bc.setLastBlock()
	bc.transState = bc.State().Copy()

	return bc
}

func (self *ChainManager) Status() (td *big.Int, currentBlock []byte, genesisBlock []byte) {
	self.mu.RLock()
	defer self.mu.RUnlock()

	return self.td, self.currentBlock.Hash(), self.Genesis().Hash()
}

func (self *ChainManager) SetProcessor(proc types.BlockProcessor) {
	self.processor = proc
}

func (self *ChainManager) State() *state.StateDB {
	return state.New(self.CurrentBlock().Root(), self.db)
}

func (self *ChainManager) TransState() *state.StateDB {
	return self.transState
}

func (bc *ChainManager) setLastBlock() {
	data, _ := bc.db.Get([]byte("LastBlock"))
	if len(data) != 0 {
		var block types.Block
		rlp.Decode(bytes.NewReader(data), &block)
		bc.currentBlock = &block
		bc.lastBlockHash = block.Hash()
		bc.lastBlockNumber = block.Header().Number.Uint64()

		// Set the last know difficulty (might be 0x0 as initial value, Genesis)
		bc.td = ethutil.BigD(bc.db.LastKnownTD())
	} else {
		bc.Reset()
	}

	chainlogger.Infof("Last block (#%d) %x\n", bc.lastBlockNumber, bc.currentBlock.Hash())
}

// Block creation & chain handling
func (bc *ChainManager) NewBlock(coinbase []byte) *types.Block {
	bc.mu.RLock()
	defer bc.mu.RUnlock()

	var root []byte
	parentHash := ZeroHash256

	if bc.CurrentBlock != nil {
		root = bc.currentBlock.Header().Root
		parentHash = bc.lastBlockHash
	}

	block := types.NewBlock(
		parentHash,
		coinbase,
		root,
		ethutil.BigPow(2, 32),
		nil,
		"")

	parent := bc.currentBlock
	if parent != nil {
		header := block.Header()
		header.Difficulty = CalcDifficulty(block, parent)
		header.Number = new(big.Int).Add(parent.Header().Number, ethutil.Big1)
		header.GasLimit = CalcGasLimit(parent, block)

	}

	return block
}

func (bc *ChainManager) Reset() {
	bc.mu.Lock()
	defer bc.mu.Unlock()

	for block := bc.currentBlock; block != nil; block = bc.GetBlock(block.Header().ParentHash) {
		bc.db.Delete(block.Hash())
	}

	// Prepare the genesis block
	bc.write(bc.genesisBlock)
	bc.insert(bc.genesisBlock)
	bc.currentBlock = bc.genesisBlock

	bc.setTotalDifficulty(ethutil.Big("0"))
}

func (self *ChainManager) Export() []byte {
	self.mu.RLock()
	defer self.mu.RUnlock()

	chainlogger.Infof("exporting %v blocks...\n", self.currentBlock.Header().Number)

	blocks := make([]*types.Block, int(self.currentBlock.NumberU64())+1)
	for block := self.currentBlock; block != nil; block = self.GetBlock(block.Header().ParentHash) {
		blocks[block.NumberU64()] = block
	}

	return ethutil.Encode(blocks)
}

func (bc *ChainManager) insert(block *types.Block) {
	encodedBlock := ethutil.Encode(block)
	bc.db.Put([]byte("LastBlock"), encodedBlock)
	bc.currentBlock = block
	bc.lastBlockHash = block.Hash()
}

func (bc *ChainManager) write(block *types.Block) {
	bc.writeBlockInfo(block)

	encodedBlock := ethutil.Encode(block)
	bc.db.Put(block.Hash(), encodedBlock)
}

// Accessors
func (bc *ChainManager) Genesis() *types.Block {
	return bc.genesisBlock
}

// Block fetching methods
func (bc *ChainManager) HasBlock(hash []byte) bool {
	data, _ := bc.db.Get(hash)
	return len(data) != 0
}

func (self *ChainManager) GetBlockHashesFromHash(hash []byte, max uint64) (chain [][]byte) {
	block := self.GetBlock(hash)
	if block == nil {
		return
	}

	// XXX Could be optimised by using a different database which only holds hashes (i.e., linked list)
	for i := uint64(0); i < max; i++ {
		chain = append(chain, block.Hash())

		if block.Header().Number.Cmp(ethutil.Big0) <= 0 {
			break
		}

		block = self.GetBlock(block.Header().ParentHash)
	}

	return
}

func (self *ChainManager) GetBlock(hash []byte) *types.Block {
	data, _ := self.db.Get(hash)
	if len(data) == 0 {
		return nil
	}
	var block types.Block
	if err := rlp.Decode(bytes.NewReader(data), &block); err != nil {
		fmt.Println(err)
		return nil
	}

	return &block
}

func (self *ChainManager) GetBlockByNumber(num uint64) *types.Block {
	self.mu.RLock()
	defer self.mu.RUnlock()

	var block *types.Block

	if num <= self.currentBlock.Number().Uint64() {
		block = self.currentBlock
		for ; block != nil; block = self.GetBlock(block.Header().ParentHash) {
			if block.Header().Number.Uint64() == num {
				break
			}
		}
	}

	return block
}

func (bc *ChainManager) setTotalDifficulty(td *big.Int) {
	bc.db.Put([]byte("LTD"), td.Bytes())
	bc.td = td
}

func (self *ChainManager) CalcTotalDiff(block *types.Block) (*big.Int, error) {
	parent := self.GetBlock(block.Header().ParentHash)
	if parent == nil {
		return nil, fmt.Errorf("Unable to calculate total diff without known parent %x", block.Header().ParentHash)
	}

	parentTd := parent.Td

	uncleDiff := new(big.Int)
	for _, uncle := range block.Uncles() {
		uncleDiff = uncleDiff.Add(uncleDiff, uncle.Difficulty)
	}

	td := new(big.Int)
	td = td.Add(parentTd, uncleDiff)
	td = td.Add(td, block.Header().Difficulty)

	return td, nil
}

// Unexported method for writing extra non-essential block info to the db
func (bc *ChainManager) writeBlockInfo(block *types.Block) {
	bc.lastBlockNumber++
}

func (bc *ChainManager) Stop() {
	if bc.CurrentBlock != nil {
		chainlogger.Infoln("Stopped")
	}
}

func (self *ChainManager) InsertChain(chain types.Blocks) error {
	for _, block := range chain {
		td, messages, err := self.processor.Process(block)
		if err != nil {
			if IsKnownBlockErr(err) {
				continue
			}

			h := block.Header()
			chainlogger.Infof("block #%v process failed (%x)\n", h.Number, h.Hash()[:4])
			chainlogger.Infoln(block)
			chainlogger.Infoln(err)
			return err
		}
		block.Td = td

		self.mu.Lock()
		{
			self.write(block)
			cblock := self.currentBlock
			if td.Cmp(self.td) > 0 {
				if block.Header().Number.Cmp(new(big.Int).Add(cblock.Header().Number, ethutil.Big1)) < 0 {
					chainlogger.Infof("Split detected. New head #%v (%x), was #%v (%x)\n", block.Header().Number, block.Hash()[:4], cblock.Header().Number, cblock.Hash()[:4])
				}

				self.setTotalDifficulty(td)
				self.insert(block)
				self.transState = state.New(cblock.Root(), self.db) //state.New(cblock.Trie().Copy())
			}

		}
		self.mu.Unlock()

		self.eventMux.Post(NewBlockEvent{block})
		self.eventMux.Post(messages)
	}

	return nil
}

// Satisfy state query interface
func (self *ChainManager) GetAccount(addr []byte) *state.StateObject {
	return self.State().GetAccount(addr)
}
<commit_msg>GetBlockHashesFromHash(hash, max) gives back max hashes starting from PARENT of hash<commit_after>package core

import (
	"bytes"
	"fmt"
	"math/big"
	"sync"

	"github.com/ethereum/go-ethereum/core/types"
	"github.com/ethereum/go-ethereum/ethutil"
	"github.com/ethereum/go-ethereum/event"
	"github.com/ethereum/go-ethereum/logger"
	"github.com/ethereum/go-ethereum/rlp"
	"github.com/ethereum/go-ethereum/state"
)

var chainlogger = logger.NewLogger("CHAIN")

type StateQuery interface {
	GetAccount(addr []byte) *state.StateObject
}

func CalcDifficulty(block, parent *types.Block) *big.Int {
	diff := new(big.Int)

	bh, ph := block.Header(), parent.Header()
	adjust := new(big.Int).Rsh(ph.Difficulty, 10)
	if bh.Time >= ph.Time+13 {
		diff.Sub(ph.Difficulty, adjust)
	} else {
		diff.Add(ph.Difficulty, adjust)
	}

	return diff
}

func CalcGasLimit(parent, block *types.Block) *big.Int {
	if block.Number().Cmp(big.NewInt(0)) == 0 {
		return ethutil.BigPow(10, 6)
	}

	// ((1024-1) * parent.gasLimit + (gasUsed * 6 / 5)) / 1024

	previous := new(big.Int).Mul(big.NewInt(1024-1), parent.GasLimit())
	current := new(big.Rat).Mul(new(big.Rat).SetInt(parent.GasUsed()), big.NewRat(6, 5))
	curInt := new(big.Int).Div(current.Num(), current.Denom())

	result := new(big.Int).Add(previous, curInt)
	result.Div(result, big.NewInt(1024))

	min := big.NewInt(125000)

	return ethutil.BigMax(min, result)
}

type ChainManager struct {
	//eth          EthManager
	db           ethutil.Database
	processor    types.BlockProcessor
	eventMux     *event.TypeMux
	genesisBlock *types.Block
	// Last known total difficulty
	mu              sync.RWMutex
	td              *big.Int
	lastBlockNumber uint64
	currentBlock    *types.Block
	lastBlockHash   []byte

	transState *state.StateDB
}

func (self *ChainManager) Td() *big.Int {
	self.mu.RLock()
	defer self.mu.RUnlock()

	return self.td
}

func (self *ChainManager) LastBlockNumber() uint64 {
	self.mu.RLock()
	defer self.mu.RUnlock()

	return self.lastBlockNumber
}

func (self *ChainManager) LastBlockHash() []byte {
	self.mu.RLock()
	defer self.mu.RUnlock()

	return self.lastBlockHash
}

func (self *ChainManager) CurrentBlock() *types.Block {
	self.mu.RLock()
	defer self.mu.RUnlock()

	return self.currentBlock
}

func NewChainManager(db ethutil.Database, mux *event.TypeMux) *ChainManager {
	bc := &ChainManager{db: db, genesisBlock: GenesisBlock(db), eventMux: mux}
	bc.setLastBlock()
	bc.transState = bc.State().Copy()

	return bc
}

func (self *ChainManager) Status() (td *big.Int, currentBlock []byte, genesisBlock []byte) {
	self.mu.RLock()
	defer self.mu.RUnlock()

	return self.td, self.currentBlock.Hash(), self.Genesis().Hash()
}

func (self *ChainManager) SetProcessor(proc types.BlockProcessor) {
	self.processor = proc
}

func (self *ChainManager) State() *state.StateDB {
	return state.New(self.CurrentBlock().Root(), self.db)
}

func (self *ChainManager) TransState() *state.StateDB {
	return self.transState
}

func (bc *ChainManager) setLastBlock() {
	data, _ := bc.db.Get([]byte("LastBlock"))
	if len(data) != 0 {
		var block types.Block
		rlp.Decode(bytes.NewReader(data), &block)
		bc.currentBlock = &block
		bc.lastBlockHash = block.Hash()
		bc.lastBlockNumber = block.Header().Number.Uint64()

		// Set the last know difficulty (might be 0x0 as initial value, Genesis)
		bc.td = ethutil.BigD(bc.db.LastKnownTD())
	} else {
		bc.Reset()
	}

	chainlogger.Infof("Last block (#%d) %x\n", bc.lastBlockNumber, bc.currentBlock.Hash())
}

// Block creation & chain handling
func (bc *ChainManager) NewBlock(coinbase []byte) *types.Block {
	bc.mu.RLock()
	defer bc.mu.RUnlock()

	var root []byte
	parentHash := ZeroHash256

	if bc.CurrentBlock != nil {
		root = bc.currentBlock.Header().Root
		parentHash = bc.lastBlockHash
	}

	block := types.NewBlock(
		parentHash,
		coinbase,
		root,
		ethutil.BigPow(2, 32),
		nil,
		"")

	parent := bc.currentBlock
	if parent != nil {
		header := block.Header()
		header.Difficulty = CalcDifficulty(block, parent)
		header.Number = new(big.Int).Add(parent.Header().Number, ethutil.Big1)
		header.GasLimit = CalcGasLimit(parent, block)

	}

	return block
}

func (bc *ChainManager) Reset() {
	bc.mu.Lock()
	defer bc.mu.Unlock()

	for block := bc.currentBlock; block != nil; block = bc.GetBlock(block.Header().ParentHash) {
		bc.db.Delete(block.Hash())
	}

	// Prepare the genesis block
	bc.write(bc.genesisBlock)
	bc.insert(bc.genesisBlock)
	bc.currentBlock = bc.genesisBlock

	bc.setTotalDifficulty(ethutil.Big("0"))
}

func (self *ChainManager) Export() []byte {
	self.mu.RLock()
	defer self.mu.RUnlock()

	chainlogger.Infof("exporting %v blocks...\n", self.currentBlock.Header().Number)

	blocks := make([]*types.Block, int(self.currentBlock.NumberU64())+1)
	for block := self.currentBlock; block != nil; block = self.GetBlock(block.Header().ParentHash) {
		blocks[block.NumberU64()] = block
	}

	return ethutil.Encode(blocks)
}

func (bc *ChainManager) insert(block *types.Block) {
	encodedBlock := ethutil.Encode(block)
	bc.db.Put([]byte("LastBlock"), encodedBlock)
	bc.currentBlock = block
	bc.lastBlockHash = block.Hash()
}

func (bc *ChainManager) write(block *types.Block) {
	bc.writeBlockInfo(block)

	encodedBlock := ethutil.Encode(block)
	bc.db.Put(block.Hash(), encodedBlock)
}

// Accessors
func (bc *ChainManager) Genesis() *types.Block {
	return bc.genesisBlock
}

// Block fetching methods
func (bc *ChainManager) HasBlock(hash []byte) bool {
	data, _ := bc.db.Get(hash)
	return len(data) != 0
}

func (self *ChainManager) GetBlockHashesFromHash(hash []byte, max uint64) (chain [][]byte) {
	block := self.GetBlock(hash)
	if block == nil {
		return
	}

	// XXX Could be optimised by using a different database which only holds hashes (i.e., linked list)
	for i := uint64(0); i < max; i++ {
		block = self.GetBlock(block.Header().ParentHash)
		chain = append(chain, block.Hash())
		if block.Header().Number.Cmp(ethutil.Big0) <= 0 {
			break
		}
	}

	return
}

func (self *ChainManager) GetBlock(hash []byte) *types.Block {
	data, _ := self.db.Get(hash)
	if len(data) == 0 {
		return nil
	}
	var block types.Block
	if err := rlp.Decode(bytes.NewReader(data), &block); err != nil {
		fmt.Println(err)
		return nil
	}

	return &block
}

func (self *ChainManager) GetBlockByNumber(num uint64) *types.Block {
	self.mu.RLock()
	defer self.mu.RUnlock()

	var block *types.Block

	if num <= self.currentBlock.Number().Uint64() {
		block = self.currentBlock
		for ; block != nil; block = self.GetBlock(block.Header().ParentHash) {
			if block.Header().Number.Uint64() == num {
				break
			}
		}
	}

	return block
}

func (bc *ChainManager) setTotalDifficulty(td *big.Int) {
	bc.db.Put([]byte("LTD"), td.Bytes())
	bc.td = td
}

func (self *ChainManager) CalcTotalDiff(block *types.Block) (*big.Int, error) {
	parent := self.GetBlock(block.Header().ParentHash)
	if parent == nil {
		return nil, fmt.Errorf("Unable to calculate total diff without known parent %x", block.Header().ParentHash)
	}

	parentTd := parent.Td

	uncleDiff := new(big.Int)
	for _, uncle := range block.Uncles() {
		uncleDiff = uncleDiff.Add(uncleDiff, uncle.Difficulty)
	}

	td := new(big.Int)
	td = td.Add(parentTd, uncleDiff)
	td = td.Add(td, block.Header().Difficulty)

	return td, nil
}

// Unexported method for writing extra non-essential block info to the db
func (bc *ChainManager) writeBlockInfo(block *types.Block) {
	bc.lastBlockNumber++
}

func (bc *ChainManager) Stop() {
	if bc.CurrentBlock != nil {
		chainlogger.Infoln("Stopped")
	}
}

func (self *ChainManager) InsertChain(chain types.Blocks) error {
	for _, block := range chain {
		td, messages, err := self.processor.Process(block)
		if err != nil {
			if IsKnownBlockErr(err) {
				continue
			}

			h := block.Header()
			chainlogger.Infof("block #%v process failed (%x)\n", h.Number, h.Hash()[:4])
			chainlogger.Infoln(block)
			chainlogger.Infoln(err)
			return err
		}
		block.Td = td

		self.mu.Lock()
		{
			self.write(block)
			cblock := self.currentBlock
			if td.Cmp(self.td) > 0 {
				if block.Header().Number.Cmp(new(big.Int).Add(cblock.Header().Number, ethutil.Big1)) < 0 {
					chainlogger.Infof("Split detected. New head #%v (%x), was #%v (%x)\n", block.Header().Number, block.Hash()[:4], cblock.Header().Number, cblock.Hash()[:4])
				}

				self.setTotalDifficulty(td)
				self.insert(block)
				self.transState = state.New(cblock.Root(), self.db) //state.New(cblock.Trie().Copy())
			}

		}
		self.mu.Unlock()

		self.eventMux.Post(NewBlockEvent{block})
		self.eventMux.Post(messages)
	}

	return nil
}

// Satisfy state query interface
func (self *ChainManager) GetAccount(addr []byte) *state.StateObject {
	return self.State().GetAccount(addr)
}
<|endoftext|>


<commit_before>// Copyright 2012 Google, Inc. All rights reserved.
//
// Use of this source code is governed by a BSD-style license
// that can be found in the LICENSE file in the root of the source
// tree.

// +build darwin dragonfly freebsd netbsd openbsd

package bsdbpf

import (
	"github.com/google/gopacket"
	"golang.org/x/sys/unix"

	"fmt"
	"syscall"
	"time"
	"unsafe"
)

const wordSize = int(unsafe.Sizeof(uintptr(0)))

func bpfWordAlign(x int) int {
	return (((x) + (wordSize - 1)) &^ (wordSize - 1))
}

// Options is used to configure various properties of the BPF sniffer.
// Default values are used when a nil Options pointer is passed to NewBPFSniffer.
type Options struct {
	// BPFDeviceName is name of the bpf device to use for sniffing
	// the network device. The default value of BPFDeviceName is empty string
	// which causes the first available BPF device file /dev/bpfX to be used.
	BPFDeviceName string
	// ReadBufLen specifies the size of the buffer used to read packets
	// off the wire such that multiple packets are buffered with each read syscall.
	// Note that an individual packet larger than the buffer size is necessarily truncated.
	// A larger buffer should increase performance because fewer read syscalls would be made.
	// If zero is used, the system's default buffer length will be used which depending on the
	// system may default to 4096 bytes which is not big enough to accomodate some link layers
	// such as WLAN (802.11).
	// ReadBufLen defaults to 32767... however typical BSD manual pages for BPF indicate that
	// if the requested buffer size cannot be accommodated, the closest allowable size will be
	// set and returned... hence our GetReadBufLen method.
	ReadBufLen int
	// Timeout is the length of time to wait before timing out on a read request.
	// Timeout defaults to nil which means no timeout is used.
	Timeout *syscall.Timeval
	// Promisc is set to true for promiscuous mode ethernet sniffing.
	// Promisc defaults to true.
	Promisc bool
	// Immediate is set to true to make our read requests return as soon as a packet becomes available.
	// Otherwise, a read will block until either the kernel buffer becomes full or a timeout occurs.
	// The default is true.
	Immediate bool
	// PreserveLinkAddr is set to false if the link level source address should be filled in automatically
	// by the interface output routine. Set to true if the link level source address will be written,
	// as provided, to the wire.
	// The default is true.
	PreserveLinkAddr bool
}

var defaultOptions = Options{
	BPFDeviceName:    "",
	ReadBufLen:       32767,
	Timeout:          nil,
	Promisc:          true,
	Immediate:        true,
	PreserveLinkAddr: true,
}

// BPFSniffer is a struct used to track state of a BSD BPF ethernet sniffer
// such that gopacket's PacketDataSource interface is implemented.
type BPFSniffer struct {
	options           *Options
	sniffDeviceName   string
	fd                int
	readBuffer        []byte
	lastReadLen       int
	readBytesConsumed int
}

// NewBPFSniffer is used to create BSD-only BPF ethernet sniffer
// iface is the network interface device name that you wish to sniff
// options can set to nil in order to utilize default values for everything.
// Each field of Options also have a default setting if left unspecified by
// the user's custome Options struct.
func NewBPFSniffer(iface string, options *Options) *BPFSniffer {
	sniffer := BPFSniffer{
		sniffDeviceName: iface,
	}
	if options == nil {
		sniffer.options = &defaultOptions
	} else {
		sniffer.options = options
	}
	return &sniffer
}

// Close is used to close the file-descriptor of the BPF device file.
func (b *BPFSniffer) Close() error {
	return syscall.Close(b.fd)
}

func (b *BPFSniffer) pickBpfDevice() {
	var err error
	for i := 0; i < 99; i++ {
		b.options.BPFDeviceName = fmt.Sprintf("/dev/bpf%d", i)
		b.fd, err = syscall.Open(b.options.BPFDeviceName, syscall.O_RDWR, 0)
		if err == nil {
			break
		}
	}
}

// Init is used to initialize a BPF device for promiscuous sniffing.
// It also starts a goroutine to continuously read frames.
func (b *BPFSniffer) Init() error {
	var err error
	enable := 1

	if b.options.BPFDeviceName == "" {
		b.pickBpfDevice()
	}

	// setup our read buffer
	if b.options.ReadBufLen == 0 {
		b.options.ReadBufLen, err = syscall.BpfBuflen(b.fd)
		if err != nil {
			panic(err)
		}
	} else {
		b.options.ReadBufLen, err = syscall.SetBpfBuflen(b.fd, b.options.ReadBufLen)
		if err != nil {
			panic(err)
		}
	}
	b.readBuffer = make([]byte, b.options.ReadBufLen)

	err = syscall.SetBpfInterface(b.fd, b.sniffDeviceName)
	if err != nil {
		return err
	}

	if b.options.Immediate {
		// turn immediate mode on. This makes the snffer non-blocking.
		err = syscall.SetBpfImmediate(b.fd, enable)
		if err != nil {
			return err
		}
	}

	// the above call to syscall.SetBpfImmediate needs to be made
	// before setting a timer otherwise the reads will block for the
	// entire timer duration even if there are packets to return.
	if b.options.Timeout != nil {
		err = syscall.SetBpfTimeout(b.fd, b.options.Timeout)
		if err != nil {
			panic(err)
		}
	}

	if b.options.PreserveLinkAddr {
		// preserves the link level source address...
		// higher level protocol analyzers will not need this
		err = syscall.SetBpfHeadercmpl(b.fd, enable)
		if err != nil {
			return err
		}
	}

	if b.options.Promisc {
		// forces the interface into promiscuous mode
		err = syscall.SetBpfPromisc(b.fd, enable)
		if err != nil {
			return err
		}
	}

	return nil
}

func (b *BPFSniffer) ReadPacketData() ([]byte, gopacket.CaptureInfo, error) {
	var err error
	if b.readBytesConsumed >= b.lastReadLen {
		b.readBytesConsumed = 0
		b.readBuffer = make([]byte, b.options.ReadBufLen)
		b.lastReadLen, err = syscall.Read(b.fd, b.readBuffer)
		if err != nil {
			b.lastReadLen = 0
			return nil, gopacket.CaptureInfo{}, err
		}
	}
	hdr := (*unix.BpfHdr)(unsafe.Pointer(&b.readBuffer[b.readBytesConsumed]))
	frameStart := b.readBytesConsumed + int(hdr.Hdrlen)
	b.readBytesConsumed += bpfWordAlign(int(hdr.Hdrlen) + int(hdr.Caplen))
	rawFrame := b.readBuffer[frameStart : frameStart+int(hdr.Caplen)]
	captureInfo := gopacket.CaptureInfo{
		Timestamp:     time.Unix(int64(hdr.Tstamp.Sec), int64(hdr.Tstamp.Usec)*1000),
		CaptureLength: len(rawFrame),
		Length:        len(rawFrame),
	}
	return rawFrame, captureInfo, nil
}

// GetReadBufLen returns the BPF read buffer length
func (b *BPFSniffer) GetReadBufLen() int {
	return b.options.ReadBufLen
}
<commit_msg>Move initialization logic into NewBPFSniffer<commit_after>// Copyright 2012 Google, Inc. All rights reserved.
//
// Use of this source code is governed by a BSD-style license
// that can be found in the LICENSE file in the root of the source
// tree.

// +build darwin dragonfly freebsd netbsd openbsd

package bsdbpf

import (
	"github.com/google/gopacket"
	"golang.org/x/sys/unix"

	"fmt"
	"syscall"
	"time"
	"unsafe"
)

const wordSize = int(unsafe.Sizeof(uintptr(0)))

func bpfWordAlign(x int) int {
	return (((x) + (wordSize - 1)) &^ (wordSize - 1))
}

// Options is used to configure various properties of the BPF sniffer.
// Default values are used when a nil Options pointer is passed to NewBPFSniffer.
type Options struct {
	// BPFDeviceName is name of the bpf device to use for sniffing
	// the network device. The default value of BPFDeviceName is empty string
	// which causes the first available BPF device file /dev/bpfX to be used.
	BPFDeviceName string
	// ReadBufLen specifies the size of the buffer used to read packets
	// off the wire such that multiple packets are buffered with each read syscall.
	// Note that an individual packet larger than the buffer size is necessarily truncated.
	// A larger buffer should increase performance because fewer read syscalls would be made.
	// If zero is used, the system's default buffer length will be used which depending on the
	// system may default to 4096 bytes which is not big enough to accomodate some link layers
	// such as WLAN (802.11).
	// ReadBufLen defaults to 32767... however typical BSD manual pages for BPF indicate that
	// if the requested buffer size cannot be accommodated, the closest allowable size will be
	// set and returned... hence our GetReadBufLen method.
	ReadBufLen int
	// Timeout is the length of time to wait before timing out on a read request.
	// Timeout defaults to nil which means no timeout is used.
	Timeout *syscall.Timeval
	// Promisc is set to true for promiscuous mode ethernet sniffing.
	// Promisc defaults to true.
	Promisc bool
	// Immediate is set to true to make our read requests return as soon as a packet becomes available.
	// Otherwise, a read will block until either the kernel buffer becomes full or a timeout occurs.
	// The default is true.
	Immediate bool
	// PreserveLinkAddr is set to false if the link level source address should be filled in automatically
	// by the interface output routine. Set to true if the link level source address will be written,
	// as provided, to the wire.
	// The default is true.
	PreserveLinkAddr bool
}

var defaultOptions = Options{
	BPFDeviceName:    "",
	ReadBufLen:       32767,
	Timeout:          nil,
	Promisc:          true,
	Immediate:        true,
	PreserveLinkAddr: true,
}

// BPFSniffer is a struct used to track state of a BSD BPF ethernet sniffer
// such that gopacket's PacketDataSource interface is implemented.
type BPFSniffer struct {
	options           *Options
	sniffDeviceName   string
	fd                int
	readBuffer        []byte
	lastReadLen       int
	readBytesConsumed int
}

// NewBPFSniffer is used to create BSD-only BPF ethernet sniffer
// iface is the network interface device name that you wish to sniff
// options can set to nil in order to utilize default values for everything.
// Each field of Options also have a default setting if left unspecified by
// the user's custome Options struct.
func NewBPFSniffer(iface string, options *Options) (*BPFSniffer, error) {
	var err error
	enable := 1
	sniffer := BPFSniffer{
		sniffDeviceName: iface,
	}
	if options == nil {
		sniffer.options = &defaultOptions
	} else {
		sniffer.options = options
	}

	if sniffer.options.BPFDeviceName == "" {
		sniffer.pickBpfDevice()
	}

	// setup our read buffer
	if sniffer.options.ReadBufLen == 0 {
		sniffer.options.ReadBufLen, err = syscall.BpfBuflen(sniffer.fd)
		if err != nil {
			return nil, err
		}
	} else {
		sniffer.options.ReadBufLen, err = syscall.SetBpfBuflen(sniffer.fd, sniffer.options.ReadBufLen)
		if err != nil {
			return nil, err
		}
	}
	sniffer.readBuffer = make([]byte, sniffer.options.ReadBufLen)

	err = syscall.SetBpfInterface(sniffer.fd, sniffer.sniffDeviceName)
	if err != nil {
		return nil, err
	}

	if sniffer.options.Immediate {
		// turn immediate mode on. This makes the snffer non-blocking.
		err = syscall.SetBpfImmediate(sniffer.fd, enable)
		if err != nil {
			return nil, err
		}
	}

	// the above call to syscall.SetBpfImmediate needs to be made
	// before setting a timer otherwise the reads will block for the
	// entire timer duration even if there are packets to return.
	if sniffer.options.Timeout != nil {
		err = syscall.SetBpfTimeout(sniffer.fd, sniffer.options.Timeout)
		if err != nil {
			return nil, err
		}
	}

	if sniffer.options.PreserveLinkAddr {
		// preserves the link level source address...
		// higher level protocol analyzers will not need this
		err = syscall.SetBpfHeadercmpl(sniffer.fd, enable)
		if err != nil {
			return nil, err
		}
	}

	if sniffer.options.Promisc {
		// forces the interface into promiscuous mode
		err = syscall.SetBpfPromisc(sniffer.fd, enable)
		if err != nil {
			return nil, err
		}
	}

	return &sniffer, nil
}

// Close is used to close the file-descriptor of the BPF device file.
func (b *BPFSniffer) Close() error {
	return syscall.Close(b.fd)
}

func (b *BPFSniffer) pickBpfDevice() {
	var err error
	for i := 0; i < 99; i++ {
		b.options.BPFDeviceName = fmt.Sprintf("/dev/bpf%d", i)
		b.fd, err = syscall.Open(b.options.BPFDeviceName, syscall.O_RDWR, 0)
		if err == nil {
			break
		}
	}
}

func (b *BPFSniffer) ReadPacketData() ([]byte, gopacket.CaptureInfo, error) {
	var err error
	if b.readBytesConsumed >= b.lastReadLen {
		b.readBytesConsumed = 0
		b.readBuffer = make([]byte, b.options.ReadBufLen)
		b.lastReadLen, err = syscall.Read(b.fd, b.readBuffer)
		if err != nil {
			b.lastReadLen = 0
			return nil, gopacket.CaptureInfo{}, err
		}
	}
	hdr := (*unix.BpfHdr)(unsafe.Pointer(&b.readBuffer[b.readBytesConsumed]))
	frameStart := b.readBytesConsumed + int(hdr.Hdrlen)
	b.readBytesConsumed += bpfWordAlign(int(hdr.Hdrlen) + int(hdr.Caplen))
	rawFrame := b.readBuffer[frameStart : frameStart+int(hdr.Caplen)]
	captureInfo := gopacket.CaptureInfo{
		Timestamp:     time.Unix(int64(hdr.Tstamp.Sec), int64(hdr.Tstamp.Usec)*1000),
		CaptureLength: len(rawFrame),
		Length:        len(rawFrame),
	}
	return rawFrame, captureInfo, nil
}

// GetReadBufLen returns the BPF read buffer length
func (b *BPFSniffer) GetReadBufLen() int {
	return b.options.ReadBufLen
}
<|endoftext|>