<commit_before>package empire

import (
	"archive/tar"
	"bytes"
	"errors"
	"fmt"
	"io"
	"path"

	"golang.org/x/net/context"

	"github.com/remind101/empire/pkg/dockerutil"
	"github.com/remind101/empire/pkg/image"
	"github.com/remind101/empire/procfile"

	"github.com/fsouza/go-dockerclient"
)

// Procfile is the name of the Procfile file that Empire will use to
// determine the process formation.
const Procfile = "Procfile"

// ProcfileExtractor represents something that can extract a Procfile from an image.
type ProcfileExtractor interface {
	Extract(context.Context, image.Image, io.Writer) ([]byte, error)
}

type ProcfileExtractorFunc func(context.Context, image.Image, io.Writer) ([]byte, error)

func (fn ProcfileExtractorFunc) Extract(ctx context.Context, image image.Image, w io.Writer) ([]byte, error) {
	return fn(ctx, image, w)
}

// cmdExtractor is an Extractor implementation that returns a Procfile based
// on the CMD directive in the Dockerfile. It makes the assumption that the cmd
// is a "web" process.
type cmdExtractor struct {
	// Client is the docker client to use to pull the container image.
	client *dockerutil.Client
}

func newCMDExtractor(c *dockerutil.Client) *cmdExtractor {
	return &cmdExtractor{client: c}
}

func (e *cmdExtractor) Extract(ctx context.Context, img image.Image, _ io.Writer) ([]byte, error) {
	i, err := e.client.InspectImage(img.String())
	if err != nil {
		return nil, err
	}

	return procfile.Marshal(procfile.ExtendedProcfile{
		"web": procfile.Process{
			Command: i.Config.Cmd,
		},
	})
}

// multiExtractor is an Extractor implementation that tries multiple Extractors
// in succession until one succeeds.
func multiExtractor(extractors ...ProcfileExtractor) ProcfileExtractor {
	return ProcfileExtractorFunc(func(ctx context.Context, image image.Image, w io.Writer) ([]byte, error) {
		for _, extractor := range extractors {
			p, err := extractor.Extract(ctx, image, w)

			// Yay!
			if err == nil {
				return p, nil
			}

			// Try the next one
			if _, ok := err.(*procfileError); ok {
				continue
			}

			// Bubble up the error
			return p, err
		}

		return nil, &procfileError{
			Err: errors.New("no suitable Procfile extractor found"),
		}
	})
}

// fileExtractor is an implementation of the Extractor interface that extracts
// the Procfile from the images WORKDIR.
type fileExtractor struct {
	// Client is the docker client to use to pull the container image.
	client *dockerutil.Client
}

func newFileExtractor(c *dockerutil.Client) *fileExtractor {
	return &fileExtractor{client: c}
}

// Extract implements Extractor Extract.
func (e *fileExtractor) Extract(ctx context.Context, img image.Image, w io.Writer) ([]byte, error) {
	c, err := e.createContainer(ctx, img)
	if err != nil {
		return nil, err
	}

	defer e.removeContainer(ctx, c.ID)

	pfile, err := e.procfile(ctx, c.ID)
	if err != nil {
		return nil, err
	}

	b, err := e.copyFile(ctx, c.ID, pfile)
	if err != nil {
		return nil, &procfileError{Err: err}
	}

	return b, nil
}

// procfile returns the path to the Procfile. If the container has a WORKDIR
// set, then this will return a path to the Procfile within that directory.
func (e *fileExtractor) procfile(ctx context.Context, id string) (string, error) {
	p := ""

	c, err := e.client.InspectContainer(id)
	if err != nil {
		return "", err
	}

	if c.Config != nil {
		p = c.Config.WorkingDir
	}

	return path.Join(p, Procfile), nil
}

// createContainer creates a new docker container for the given docker image.
func (e *fileExtractor) createContainer(ctx context.Context, img image.Image) (*docker.Container, error) {
	return e.client.CreateContainer(ctx, docker.CreateContainerOptions{
		Config: &docker.Config{
			Image: img.String(),
		},
	})
}

// removeContainer removes a container by its ID.
func (e *fileExtractor) removeContainer(ctx context.Context, containerID string) error {
	return e.client.RemoveContainer(ctx, docker.RemoveContainerOptions{
		ID: containerID,
	})
}

// copyFile copies a file from a container.
func (e *fileExtractor) copyFile(ctx context.Context, containerID, path string) ([]byte, error) {
	var buf bytes.Buffer
	if err := e.client.CopyFromContainer(ctx, docker.CopyFromContainerOptions{
		Container:    containerID,
		Resource:     path,
		OutputStream: &buf,
	}); err != nil {
		return nil, err
	}

	// Open the tar archive for reading.
	r := bytes.NewReader(buf.Bytes())

	return firstFile(tar.NewReader(r))
}

// Example instance: Procfile doesn't exist
type procfileError struct {
	Err error
}

func (e *procfileError) Error() string {
	return fmt.Sprintf("Procfile not found: %s", e.Err)
}

// firstFile extracts the first file from a tar archive.
func firstFile(tr *tar.Reader) ([]byte, error) {
	if _, err := tr.Next(); err != nil {
		return nil, err
	}

	var buf bytes.Buffer
	if _, err := io.Copy(&buf, tr); err != nil {
		return nil, err
	}

	return buf.Bytes(), nil
}

func formationFromProcfile(p procfile.Procfile) (Formation, error) {
	switch p := p.(type) {
	case procfile.StandardProcfile:
		return formationFromStandardProcfile(p)
	case procfile.ExtendedProcfile:
		return formationFromExtendedProcfile(p)
	default:
		return nil, &procfileError{
			Err: errors.New("unknown Procfile format"),
		}
	}
}

func formationFromStandardProcfile(p procfile.StandardProcfile) (Formation, error) {
	f := make(Formation)

	for name, command := range p {
		cmd, err := ParseCommand(command)
		if err != nil {
			return nil, err
		}

		f[name] = Process{
			Command: cmd,
		}
	}

	return f, nil
}

func formationFromExtendedProcfile(p procfile.ExtendedProcfile) (Formation, error) {
	f := make(Formation)

	for name, process := range p {
		var cmd Command
		var err error

		switch command := process.Command.(type) {
		case string:
			cmd, err = ParseCommand(command)
			if err != nil {
				return nil, err
			}
		case []interface{}:
			for _, v := range command {
				cmd = append(cmd, v.(string))
			}
		default:
			return nil, errors.New("unknown command format")
		}

		var ports []Port

		for _, port := range process.Ports {
			protocol := port.Protocol
			if protocol == "" {
				switch port.Host {
				case 80, 8080:
					protocol = "http"
				case 443:
					protocol = "https"
				default:
					protocol = "tcp"
				}
			}
			ports = append(ports, Port{
				Host:      port.Host,
				Container: port.Container,
				Protocol:  protocol,
			})
		}

		f[name] = Process{
			Command:   cmd,
			Cron:      process.Cron,
			NoService: process.NoService,
			Ports:     ports,
		}
	}

	return f, nil
}
<commit_msg>Move automatic protocol determination to named function.<commit_after>package empire

import (
	"archive/tar"
	"bytes"
	"errors"
	"fmt"
	"io"
	"path"

	"golang.org/x/net/context"

	"github.com/remind101/empire/pkg/dockerutil"
	"github.com/remind101/empire/pkg/image"
	"github.com/remind101/empire/procfile"

	"github.com/fsouza/go-dockerclient"
)

// Procfile is the name of the Procfile file that Empire will use to
// determine the process formation.
const Procfile = "Procfile"

// ProcfileExtractor represents something that can extract a Procfile from an image.
type ProcfileExtractor interface {
	Extract(context.Context, image.Image, io.Writer) ([]byte, error)
}

type ProcfileExtractorFunc func(context.Context, image.Image, io.Writer) ([]byte, error)

func (fn ProcfileExtractorFunc) Extract(ctx context.Context, image image.Image, w io.Writer) ([]byte, error) {
	return fn(ctx, image, w)
}

// cmdExtractor is an Extractor implementation that returns a Procfile based
// on the CMD directive in the Dockerfile. It makes the assumption that the cmd
// is a "web" process.
type cmdExtractor struct {
	// Client is the docker client to use to pull the container image.
	client *dockerutil.Client
}

func newCMDExtractor(c *dockerutil.Client) *cmdExtractor {
	return &cmdExtractor{client: c}
}

func (e *cmdExtractor) Extract(ctx context.Context, img image.Image, _ io.Writer) ([]byte, error) {
	i, err := e.client.InspectImage(img.String())
	if err != nil {
		return nil, err
	}

	return procfile.Marshal(procfile.ExtendedProcfile{
		"web": procfile.Process{
			Command: i.Config.Cmd,
		},
	})
}

// multiExtractor is an Extractor implementation that tries multiple Extractors
// in succession until one succeeds.
func multiExtractor(extractors ...ProcfileExtractor) ProcfileExtractor {
	return ProcfileExtractorFunc(func(ctx context.Context, image image.Image, w io.Writer) ([]byte, error) {
		for _, extractor := range extractors {
			p, err := extractor.Extract(ctx, image, w)

			// Yay!
			if err == nil {
				return p, nil
			}

			// Try the next one
			if _, ok := err.(*procfileError); ok {
				continue
			}

			// Bubble up the error
			return p, err
		}

		return nil, &procfileError{
			Err: errors.New("no suitable Procfile extractor found"),
		}
	})
}

// fileExtractor is an implementation of the Extractor interface that extracts
// the Procfile from the images WORKDIR.
type fileExtractor struct {
	// Client is the docker client to use to pull the container image.
	client *dockerutil.Client
}

func newFileExtractor(c *dockerutil.Client) *fileExtractor {
	return &fileExtractor{client: c}
}

// Extract implements Extractor Extract.
func (e *fileExtractor) Extract(ctx context.Context, img image.Image, w io.Writer) ([]byte, error) {
	c, err := e.createContainer(ctx, img)
	if err != nil {
		return nil, err
	}

	defer e.removeContainer(ctx, c.ID)

	pfile, err := e.procfile(ctx, c.ID)
	if err != nil {
		return nil, err
	}

	b, err := e.copyFile(ctx, c.ID, pfile)
	if err != nil {
		return nil, &procfileError{Err: err}
	}

	return b, nil
}

// procfile returns the path to the Procfile. If the container has a WORKDIR
// set, then this will return a path to the Procfile within that directory.
func (e *fileExtractor) procfile(ctx context.Context, id string) (string, error) {
	p := ""

	c, err := e.client.InspectContainer(id)
	if err != nil {
		return "", err
	}

	if c.Config != nil {
		p = c.Config.WorkingDir
	}

	return path.Join(p, Procfile), nil
}

// createContainer creates a new docker container for the given docker image.
func (e *fileExtractor) createContainer(ctx context.Context, img image.Image) (*docker.Container, error) {
	return e.client.CreateContainer(ctx, docker.CreateContainerOptions{
		Config: &docker.Config{
			Image: img.String(),
		},
	})
}

// removeContainer removes a container by its ID.
func (e *fileExtractor) removeContainer(ctx context.Context, containerID string) error {
	return e.client.RemoveContainer(ctx, docker.RemoveContainerOptions{
		ID: containerID,
	})
}

// copyFile copies a file from a container.
func (e *fileExtractor) copyFile(ctx context.Context, containerID, path string) ([]byte, error) {
	var buf bytes.Buffer
	if err := e.client.CopyFromContainer(ctx, docker.CopyFromContainerOptions{
		Container:    containerID,
		Resource:     path,
		OutputStream: &buf,
	}); err != nil {
		return nil, err
	}

	// Open the tar archive for reading.
	r := bytes.NewReader(buf.Bytes())

	return firstFile(tar.NewReader(r))
}

// Example instance: Procfile doesn't exist
type procfileError struct {
	Err error
}

func (e *procfileError) Error() string {
	return fmt.Sprintf("Procfile not found: %s", e.Err)
}

// firstFile extracts the first file from a tar archive.
func firstFile(tr *tar.Reader) ([]byte, error) {
	if _, err := tr.Next(); err != nil {
		return nil, err
	}

	var buf bytes.Buffer
	if _, err := io.Copy(&buf, tr); err != nil {
		return nil, err
	}

	return buf.Bytes(), nil
}

func formationFromProcfile(p procfile.Procfile) (Formation, error) {
	switch p := p.(type) {
	case procfile.StandardProcfile:
		return formationFromStandardProcfile(p)
	case procfile.ExtendedProcfile:
		return formationFromExtendedProcfile(p)
	default:
		return nil, &procfileError{
			Err: errors.New("unknown Procfile format"),
		}
	}
}

func formationFromStandardProcfile(p procfile.StandardProcfile) (Formation, error) {
	f := make(Formation)

	for name, command := range p {
		cmd, err := ParseCommand(command)
		if err != nil {
			return nil, err
		}

		f[name] = Process{
			Command: cmd,
		}
	}

	return f, nil
}

func formationFromExtendedProcfile(p procfile.ExtendedProcfile) (Formation, error) {
	f := make(Formation)

	for name, process := range p {
		var cmd Command
		var err error

		switch command := process.Command.(type) {
		case string:
			cmd, err = ParseCommand(command)
			if err != nil {
				return nil, err
			}
		case []interface{}:
			for _, v := range command {
				cmd = append(cmd, v.(string))
			}
		default:
			return nil, errors.New("unknown command format")
		}

		var ports []Port

		for _, port := range process.Ports {
			protocol := port.Protocol
			if protocol == "" {
				protocol = protocolFromPort(port.Host)
			}

			ports = append(ports, Port{
				Host:      port.Host,
				Container: port.Container,
				Protocol:  protocol,
			})
		}

		f[name] = Process{
			Command:   cmd,
			Cron:      process.Cron,
			NoService: process.NoService,
			Ports:     ports,
		}
	}

	return f, nil
}

// protocolFromPort attempts to automatically determine what protocol a port
// should use. For example, port 80 is well known to be http, so we can assume
// that http should be used. Defaults to "tcp" if unknown.
func protocolFromPort(port int) string {
	switch port {
	case 80, 8080:
		return "http"
	case 443:
		return "https"
	default:
		return "tcp"
	}
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"
	"os"
)

func factoidtx(args []string) error {
	os.Args = args

	var (
		serv = flag.String("s", "localhost:8088", "path to the factomclient")
		wallet = flag.String("w", "", "Factoid wallet address")
	)
	flag.Parse()
	args = flag.Args()
	if len(args) < 1 {
		return fmt.Errorf("the ammount of factoids to be transferd must be specified")
	}
	amt := args[1]
	server := "http://" + *serv + "/v1/factoidtx"
	data := url.Values{
		"to":      {*wallet},
		"ammount": {amt},
	}

	resp, err := http.PostForm(server, data)
	if err != nil {
		return err
	}
	p, err := ioutil.ReadAll(resp.Body)
	resp.Body.Close()
	fmt.Println(string(p))

	return nil
}
<commit_msg>fatoidtx.go testing flag parsing<commit_after>package main

import (
	"flag"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"
	"os"
)

func factoidtx(args []string) error {
	os.Args = args

	var (
		serv = flag.String("s", "localhost:8088", "path to the factomclient")
		wallet = flag.String("w", "", "Factoid wallet address")
	)
	flag.Parse()
	args = flag.Args()
	if len(args) < 1 {
		return fmt.Errorf("the ammount of factoids to be transferd must be specified")
	}
	amt := args[0]
	server := "http://" + *serv + "/v1/factoidtx"
	data := url.Values{
		"to":      {*wallet},
		"ammount": {amt},
	}

	resp, err := http.PostForm(server, data)
	if err != nil {
		return err
	}
	p, err := ioutil.ReadAll(resp.Body)
	resp.Body.Close()
	fmt.Println(string(p))

	return nil
}
<|endoftext|>


<commit_before>package fastimage

import (
	"bytes"
	"io"
	"net/http"
)

// DetectImageType is the main function used to detect the type and size
// of a remote image represented by the url.
//
// Only check ImageType and ImageSize if error is not nil.
func DetectImageType(uri string) (ImageType, *ImageSize, error) {
	buffer := bytes.Buffer{}

	logger.Printf("Opening HTTP stream")
	resp, err := http.Get(uri)
	defer closeHTTPStream(resp, &buffer)

	if err != nil {
		return Unknown, nil, err
	}

	logger.Printf("Starting operation")

	for {
		err := readToBuffer(resp.Body, &buffer)
		if buffer.Len() < 2 {
			continue
		}

		if err != nil {
			logger.Printf("Bailing out because of err %v", err)
			return Unknown, nil, err
		}

		for _, ImageTypeParser := range imageTypeParsers {
			if ImageTypeParser.Detect(buffer.Bytes()) {
				t := ImageTypeParser.Type()
				size, err := ImageTypeParser.GetSize(buffer.Bytes())

				if err == nil {
					logger.Printf("Found image type %v with size %v", t, size)
					return t, size, nil
				}
				break
			}
		}
	}
}

func readToBuffer(body io.Reader, buffer *bytes.Buffer) error {
	chunk := make([]byte, 8)
	count, err := body.Read(chunk)

	logger.Printf("Read %v bytes", count)
	buffer.Write(chunk[:count])

	return err
}

func closeHTTPStream(http *http.Response, buffer *bytes.Buffer) {
	logger.Printf("Closing HTTP Stream")
	http.Body.Close()
	logger.Printf("Closed after reading just %v bytes out of %v bytes", buffer.Len(), http.ContentLength)
}
<commit_msg>refactor(DetectImageType): Add possibility to tweak request before detection<commit_after>package fastimage

import (
	"bytes"
	"io"
	"net/http"
)

// DetectImageType is the main function used to detect the type and size
// of a remote image represented by the url.
//
// Only check ImageType and ImageSize if error is not nil.
func DetectImageType(uri string) (ImageType, *ImageSize, error) {

	logger.Printf("Opening HTTP stream")
	resp, err := http.Get(uri)

	defer resp.Body.Close()

	if err != nil {
		return Unknown, nil, err
	}

	return DetectImageTypeFromResponse(resp)
}

// DetectImageTypeFromResponse is a secondary function used to detect the type and size
// of a remote image represented by the resp.
// This way you can create your own request and then pass it here.
// Check examples from http://golang.org/pkg/net/http/
//
// Only check ImageType and ImageSize if error is not nil.
func DetectImageTypeFromResponse(resp *http.Response) (ImageType, *ImageSize, error) {
	logger.Printf("Starting operation")

	buffer := bytes.Buffer{}
	defer logger.Printf("Ended after reading %v bytes out of %v bytes", buffer.Len(), resp.ContentLength)

	for {
		err := readToBuffer(resp.Body, &buffer)
		if buffer.Len() < 2 {
			continue
		}

		if err != nil {
			logger.Printf("Bailing out because of err %v", err)
			return Unknown, nil, err
		}

		for _, ImageTypeParser := range imageTypeParsers {
			if ImageTypeParser.Detect(buffer.Bytes()) {
				t := ImageTypeParser.Type()
				size, err := ImageTypeParser.GetSize(buffer.Bytes())

				if err == nil {
					logger.Printf("Found image type %v with size %v", t, size)
					return t, size, nil
				}
				break
			}
		}
	}
}

func readToBuffer(body io.Reader, buffer *bytes.Buffer) error {
	chunk := make([]byte, 8)
	count, err := body.Read(chunk)

	logger.Printf("Read %v bytes", count)
	buffer.Write(chunk[:count])

	return err
}
<|endoftext|>


<commit_before>package persist

import (
	"flag"
	"fmt"
	"os"
	"testing"
	"time"

	"github.com/pachyderm/pachyderm/src/client/pfs"
	"github.com/pachyderm/pachyderm/src/client/pkg/require"
	"github.com/pachyderm/pachyderm/src/client/pkg/uuid"

	"github.com/dancannon/gorethink"
	"go.pedge.io/pb/go/google/protobuf"
)

const (
	RethinkAddress = "localhost:28015"
	RethinkTestDB  = "pachyderm_test"
)

func TestMain(m *testing.M) {
	flag.Parse()
	if err := InitDB(RethinkAddress, RethinkTestDB); err != nil {
		panic(err)
	}
	code := m.Run()
	if err := RemoveDB(RethinkAddress, RethinkTestDB); err != nil {
		panic(err)
	}
	os.Exit(code)
}

func timestampNow() *google_protobuf.Timestamp {
	return &google_protobuf.Timestamp{Seconds: time.Now().Unix()}
}

func persistCommitToPFSCommit(rawCommit *Commit) *pfs.Commit {
	return &pfs.Commit{
		Repo: &pfs.Repo{
			Name: rawCommit.Repo,
		},
		ID: rawCommit.ID,
	}
}

/*

CASES:

- start commit - no parent ID, branch name = master --> creates first commit on master
- do this and repeate start commit call pattern -> new commit should have first as parent
- start commit w parent ID

*/

func TestStartCommit(t *testing.T) {
	d, err := NewDriver("localhost:1523", RethinkAddress, RethinkTestDB)
	require.NoError(t, err)
	fmt.Printf("got a driver")

	dbClient, err := dbConnect(RethinkAddress)
	require.NoError(t, err)

	commitID := uuid.NewWithoutDashes()
	err = d.StartCommit(
		&pfs.Repo{},
		commitID,
		"",
		"master",
		timestampNow(),
		make([]*pfs.Commit, 0),
		make(map[uint64]bool),
	)
	require.NoError(t, err)

	cursor, err := gorethink.DB(RethinkTestDB).Table(commitTable).Get(commitID).Default(gorethink.Error("value not found")).Run(dbClient)
	defer func() {
		require.NoError(t, cursor.Close())
	}()

	rawCommit := &Commit{}
	cursor.Next(rawCommit)
	require.NoError(t, cursor.Err())

	fmt.Printf("Commit info: %v\n", rawCommit)

	require.Equal(t, 1, len(rawCommit.BranchClocks))
	require.Equal(t, rawCommit.BranchClocks[0], &Clock{Branch: "master", Clock: 0})

	commit := persistCommitToPFSCommit(rawCommit)
	err = d.FinishCommit(commit, timestampNow(), false, make(map[uint64]bool))
	require.NoError(t, err)
}
<commit_msg>Add failing test when making new commit by branch only<commit_after>package persist

import (
	"flag"
	"fmt"
	"os"
	"testing"
	"time"

	"github.com/pachyderm/pachyderm/src/client/pfs"
	"github.com/pachyderm/pachyderm/src/client/pkg/require"
	"github.com/pachyderm/pachyderm/src/client/pkg/uuid"

	"github.com/dancannon/gorethink"
	"go.pedge.io/pb/go/google/protobuf"
)

const (
	RethinkAddress = "localhost:28015"
	RethinkTestDB  = "pachyderm_test"
)

func TestMain(m *testing.M) {
	flag.Parse()
	if err := InitDB(RethinkAddress, RethinkTestDB); err != nil {
		panic(err)
	}
	code := m.Run()
	if err := RemoveDB(RethinkAddress, RethinkTestDB); err != nil {
		panic(err)
	}
	os.Exit(code)
}

func timestampNow() *google_protobuf.Timestamp {
	return &google_protobuf.Timestamp{Seconds: time.Now().Unix()}
}

func persistCommitToPFSCommit(rawCommit *Commit) *pfs.Commit {
	return &pfs.Commit{
		Repo: &pfs.Repo{
			Name: rawCommit.Repo,
		},
		ID: rawCommit.ID,
	}
}

/*

CASES:

- start commit - no parent ID, branch name = master --> creates first commit on master
- do this and repeate start commit call pattern -> new commit should have first as parent
- start commit w parent ID
- when branch and parent nil, make a new branch

- check uniqueness -- see if creating branch w same id results in rethink error
*/

func TestStartCommit(t *testing.T) {
	d, err := NewDriver("localhost:1523", RethinkAddress, RethinkTestDB)
	require.NoError(t, err)
	fmt.Printf("got a driver")

	dbClient, err := dbConnect(RethinkAddress)
	require.NoError(t, err)

	commitID := uuid.NewWithoutDashes()
	err = d.StartCommit(
		&pfs.Repo{},
		commitID,
		"",
		"master",
		timestampNow(),
		make([]*pfs.Commit, 0),
		make(map[uint64]bool),
	)
	require.NoError(t, err)

	cursor, err := gorethink.DB(RethinkTestDB).Table(commitTable).Get(commitID).Default(gorethink.Error("value not found")).Run(dbClient)
	defer func() {
		require.NoError(t, cursor.Close())
	}()

	rawCommit := &Commit{}
	cursor.Next(rawCommit)
	require.NoError(t, cursor.Err())

	fmt.Printf("Commit info: %v\n", rawCommit)

	require.Equal(t, 1, len(rawCommit.BranchClocks))
	require.Equal(t, rawCommit.BranchClocks[0], &Clock{Branch: "master", Clock: 0})

	commit := persistCommitToPFSCommit(rawCommit)
	err = d.FinishCommit(commit, timestampNow(), false, make(map[uint64]bool))
	require.NoError(t, err)
}

func TestStartCommitJustByBranch(t *testing.T) {
	d, err := NewDriver("localhost:1523", RethinkAddress, RethinkTestDB)
	require.NoError(t, err)
	fmt.Printf("got a driver")

	dbClient, err := dbConnect(RethinkAddress)
	require.NoError(t, err)

	commitID := uuid.NewWithoutDashes()
	err = d.StartCommit(
		&pfs.Repo{},
		commitID,
		"",
		"master",
		timestampNow(),
		make([]*pfs.Commit, 0),
		make(map[uint64]bool),
	)
	require.NoError(t, err)

	cursor, err := gorethink.DB(RethinkTestDB).Table(commitTable).Get(commitID).Default(gorethink.Error("value not found")).Run(dbClient)
	defer func() {
		require.NoError(t, cursor.Close())
	}()

	rawCommit := &Commit{}
	cursor.Next(rawCommit)
	require.NoError(t, cursor.Err())

	fmt.Printf("Commit info: %v\n", rawCommit)

	require.Equal(t, 1, len(rawCommit.BranchClocks))
	require.Equal(t, rawCommit.BranchClocks[0], &Clock{Branch: "master", Clock: 0})

	commit := persistCommitToPFSCommit(rawCommit)
	err = d.FinishCommit(commit, timestampNow(), false, make(map[uint64]bool))
	require.NoError(t, err)

	commit2ID := uuid.NewWithoutDashes()
	err = d.StartCommit(
		&pfs.Repo{},
		commit2ID,
		"",
		"master",
		timestampNow(),
		make([]*pfs.Commit, 0),
		make(map[uint64]bool),
	)
	require.NoError(t, err)

	cursor, err = gorethink.DB(RethinkTestDB).Table(commitTable).Get(commitID).Default(gorethink.Error("value not found")).Run(dbClient)

	rawCommit2 := &Commit{}
	cursor.Next(rawCommit2)
	require.NoError(t, cursor.Err())

	fmt.Printf("Commit info: %v\n", rawCommit2)

	require.Equal(t, 2, len(rawCommit2.BranchClocks))
	require.Equal(t, rawCommit2.BranchClocks[0], &Clock{Branch: "master", Clock: 0})
	require.Equal(t, rawCommit2.BranchClocks[1], &Clock{Branch: "master", Clock: 1})

	commit2 := persistCommitToPFSCommit(rawCommit2)
	err = d.FinishCommit(commit2, timestampNow(), false, make(map[uint64]bool))
	require.NoError(t, err)

}
<|endoftext|>


<commit_before>package main

// Version is the version of gateway
const Version = "1.0.0-rc11"
<commit_msg>bump gateway version (#1181)<commit_after>package main

// Version is the version of gateway
const Version = "1.0.0-rc12"
<|endoftext|>


<commit_before>package main

import (
	"context"
	"crypto/tls"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"net/http/pprof"
	"os"
	"time"

	"cloud.google.com/go/pubsub"
	"code.cloudfoundry.org/clock"
	"code.cloudfoundry.org/lager"
	"github.com/google/go-github/github"
	flags "github.com/jessevdk/go-flags"
	"github.com/tedsuo/ifrit"
	"github.com/tedsuo/ifrit/grouper"
	"github.com/tedsuo/ifrit/http_server"
	"github.com/tedsuo/ifrit/sigmon"
	"golang.org/x/net/trace"
	"golang.org/x/oauth2"
	"google.golang.org/grpc"

	"cred-alert/config"
	"cred-alert/crypto"
	"cred-alert/db"
	"cred-alert/db/migrations"
	"cred-alert/gitclient"
	"cred-alert/metrics"
	"cred-alert/notifications"
	"cred-alert/queue"
	"cred-alert/revok"
	"cred-alert/revok/stats"
	"cred-alert/revokpb"
	"cred-alert/search"
	"cred-alert/sniff"
	"red/redrunner"
)

func main() {
	var cfg *config.WorkerConfig
	var flagOpts config.WorkerOpts

	logger := lager.NewLogger("revok-worker")
	logger.RegisterSink(lager.NewWriterSink(os.Stdout, lager.DEBUG))

	logger.Info("starting")

	_, err := flags.Parse(&flagOpts)
	if err != nil {
		os.Exit(1)
	}

	if flagOpts.ConfigFile != "" {
		bs, err := ioutil.ReadFile(string(flagOpts.ConfigFile))
		if err != nil {
			logger.Error("failed-opening-config-file", err)
			os.Exit(1)
		}

		cfg, err = config.LoadWorkerConfig(bs)
		cfg.Merge(flagOpts.WorkerConfig)
	} else {
		cfg = flagOpts.WorkerConfig
	}

	errs := cfg.Validate()
	if errs != nil {
		for _, err := range errs {
			fmt.Println(err.Error())
		}
		os.Exit(1)
	}

	if cfg.Metrics.SentryDSN != "" {
		logger.RegisterSink(revok.NewSentrySink(cfg.Metrics.SentryDSN, cfg.Metrics.Environment))
	}

	workdir := cfg.WorkDir
	_, err = os.Lstat(workdir)
	if err != nil {
		log.Fatalf("workdir error: %s", err)
	}

	dbURI := db.NewDSN(cfg.MySQL.Username, cfg.MySQL.Password, cfg.MySQL.DBName, cfg.MySQL.Hostname, int(cfg.MySQL.Port))
	database, err := migrations.LockDBAndMigrate(logger, "mysql", dbURI)
	if err != nil {
		log.Fatalf("db error: %s", err)
	}

	database.LogMode(false)

	clock := clock.NewClock()

	cloneMsgCh := make(chan revok.CloneMsg)

	scanRepository := db.NewScanRepository(database, clock)
	repositoryRepository := db.NewRepositoryRepository(database)
	fetchRepository := db.NewFetchRepository(database)
	credentialRepository := db.NewCredentialRepository(database)
	emitter := metrics.BuildEmitter(cfg.Metrics.DatadogAPIKey, cfg.Metrics.Environment)
	gitClient := gitclient.New(string(cfg.GitHub.PrivateKeyPath), string(cfg.GitHub.PublicKeyPath))
	repoWhitelist := notifications.BuildWhitelist(cfg.Whitelist...)

	var notifier notifications.Notifier
	if cfg.Slack.WebhookURL != "" {
		formatter := notifications.NewSlackNotificationFormatter()
		notifier = notifications.NewSlackNotifier(cfg.Slack.WebhookURL, clock, repoWhitelist, formatter)
	} else {
		notifier = notifications.NewNullNotifier()
	}

	sniffer := sniff.NewDefaultSniffer()
	ancestryScanner := revok.NewScanner(
		gitClient,
		repositoryRepository,
		scanRepository,
		credentialRepository,
		sniffer,
		notifier,
	)

	changeFetcher := revok.NewChangeFetcher(
		logger,
		gitClient,
		ancestryScanner,
		repositoryRepository,
		fetchRepository,
		emitter,
	)

	changeScheduleRunner := revok.NewScheduleRunner(logger)

	changeScheduler := revok.NewChangeScheduler(
		logger,
		repositoryRepository,
		changeScheduleRunner,
		changeFetcher,
	)

	cloner := revok.NewCloner(
		logger,
		workdir,
		cloneMsgCh,
		gitClient,
		repositoryRepository,
		ancestryScanner,
		emitter,
		changeScheduler,
	)

	dirscanUpdater := revok.NewRescanner(
		logger,
		scanRepository,
		credentialRepository,
		ancestryScanner,
		notifier,
		emitter,
	)

	statsReporter := stats.NewReporter(
		logger,
		clock,
		60*time.Second,
		db.NewStatsRepository(database),
		emitter,
	)

	headCredentialCounter := revok.NewHeadCredentialCounter(
		logger,
		repositoryRepository,
		clock,
		cfg.CredentialCounterInterval,
		gitClient,
		sniffer,
	)

	members := []grouper.Member{
		{"cloner", cloner},
		{"dirscan-updater", dirscanUpdater},
		{"stats-reporter", statsReporter},
		{"head-credential-counter", headCredentialCounter},
		{"change-schedule-runner", changeScheduleRunner},
		{"debug", http_server.New("127.0.0.1:6060", debugHandler())},
	}

	if cfg.IsRPCConfigured() {
		certificate, err := config.LoadCertificate(
			string(cfg.RPC.CertificatePath),
			string(cfg.RPC.PrivateKeyPath),
			cfg.RPC.PrivateKeyPassphrase,
		)
		if err != nil {
			log.Fatalln(err)
		}

		clientCertPool, err := config.LoadCertificatePool(string(cfg.RPC.ClientCACertificatePath))
		if err != nil {
			log.Fatalln(err)
		}

		looper := gitclient.NewLooper()
		searcher := search.NewSearcher(repositoryRepository, looper)

		grpcServer := redrunner.NewGRPCServer(
			logger,
			fmt.Sprintf("%s:%d", cfg.RPC.BindIP, cfg.RPC.BindPort),
			&tls.Config{
				ClientAuth:   tls.RequireAndVerifyClientCert,
				Certificates: []tls.Certificate{certificate},
				ClientCAs:    clientCertPool,
			},
			func(server *grpc.Server) {
				revokpb.RegisterRevokServer(server, revok.NewServer(logger, repositoryRepository, searcher))
			},
		)

		members = append(members, grouper.Member{
			Name:   "grpc-server",
			Runner: grpcServer,
		})
	}

	if cfg.IsPubSubConfigured() {
		pubSubClient, err := pubsub.NewClient(context.Background(), cfg.PubSub.ProjectName)
		if err != nil {
			logger.Fatal("failed", err)
			os.Exit(1)
		}

		subscription := pubSubClient.Subscription(cfg.PubSub.FetchHint.Subscription)

		publicKey, err := crypto.ReadRSAPublicKey(string(cfg.PubSub.PublicKeyPath))
		if err != nil {
			logger.Fatal("failed", err)
			os.Exit(1)
		}
		pushEventProcessor := queue.NewPushEventProcessor(
			changeFetcher,
			crypto.NewRSAVerifier(publicKey),
			emitter,
		)

		members = append(members, grouper.Member{
			Name:   "github-hint-handler",
			Runner: queue.NewPubSubSubscriber(logger, subscription, pushEventProcessor, emitter),
		})
	}

	if cfg.GitHub.AccessToken != "" {
		githubHTTPClient := &http.Client{
			Timeout: 30 * time.Second,
			Transport: &oauth2.Transport{
				Source: oauth2.StaticTokenSource(
					&oauth2.Token{AccessToken: cfg.GitHub.AccessToken},
				),
				Base: &http.Transport{
					DisableKeepAlives: true,
				},
			},
		}

		ghClient := revok.NewGitHubClient(github.NewClient(githubHTTPClient))

		repoDiscoverer := revok.NewRepoDiscoverer(
			logger,
			workdir,
			cloneMsgCh,
			ghClient,
			clock,
			cfg.RepositoryDiscoveryInterval,
			repositoryRepository,
		)

		members = append(members, grouper.Member{
			Name:   "repo-discoverer",
			Runner: repoDiscoverer,
		})
	}

	startupTasks := []grouper.Member{
		{
			Name:   "schedule-fetches",
			Runner: changeScheduler,
		},
	}

	system := []grouper.Member{
		{
			Name:   "servers",
			Runner: grouper.NewParallel(os.Interrupt, members),
		},
		{
			Name:   "startup-tasks",
			Runner: grouper.NewParallel(os.Interrupt, startupTasks),
		},
	}

	runner := sigmon.New(grouper.NewOrdered(os.Interrupt, system))

	err = <-ifrit.Invoke(runner).Wait()
	if err != nil {
		log.Fatalf("failed-to-start: %s", err)
	}
}

func debugHandler() http.Handler {
	debugRouter := http.NewServeMux()
	debugRouter.Handle("/debug/pprof/", http.HandlerFunc(pprof.Index))
	debugRouter.Handle("/debug/pprof/cmdline", http.HandlerFunc(pprof.Cmdline))
	debugRouter.Handle("/debug/pprof/profile", http.HandlerFunc(pprof.Profile))
	debugRouter.Handle("/debug/pprof/symbol", http.HandlerFunc(pprof.Symbol))
	debugRouter.Handle("/debug/pprof/trace", http.HandlerFunc(pprof.Trace))

	debugRouter.HandleFunc("/debug/requests", func(w http.ResponseWriter, req *http.Request) {
		any, sensitive := trace.AuthRequest(req)
		if !any {
			http.Error(w, "not allowed", http.StatusUnauthorized)
			return
		}
		w.Header().Set("Content-Type", "text/html; charset=utf-8")
		trace.Render(w, req, sensitive)
	})

	debugRouter.HandleFunc("/debug/events", func(w http.ResponseWriter, req *http.Request) {
		any, sensitive := trace.AuthRequest(req)
		if !any {
			http.Error(w, "not allowed", http.StatusUnauthorized)
			return
		}
		w.Header().Set("Content-Type", "text/html; charset=utf-8")
		trace.RenderEvents(w, req, sensitive)
	})

	return debugRouter
}
<commit_msg>use the acceptor in revok<commit_after>package main

import (
	"context"
	"crypto/tls"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"net/http/pprof"
	"os"
	"time"

	"cloud.google.com/go/pubsub"
	"code.cloudfoundry.org/clock"
	"code.cloudfoundry.org/lager"
	"github.com/google/go-github/github"
	flags "github.com/jessevdk/go-flags"
	"github.com/tedsuo/ifrit"
	"github.com/tedsuo/ifrit/grouper"
	"github.com/tedsuo/ifrit/http_server"
	"github.com/tedsuo/ifrit/sigmon"
	"golang.org/x/net/trace"
	"golang.org/x/oauth2"
	"google.golang.org/grpc"

	"cred-alert/config"
	"cred-alert/crypto"
	"cred-alert/db"
	"cred-alert/db/migrations"
	"cred-alert/gitclient"
	"cred-alert/metrics"
	"cred-alert/notifications"
	"cred-alert/queue"
	"cred-alert/revok"
	"cred-alert/revok/stats"
	"cred-alert/revokpb"
	"cred-alert/search"
	"cred-alert/sniff"
	"red/redrunner"
)

func main() {
	var cfg *config.WorkerConfig
	var flagOpts config.WorkerOpts

	logger := lager.NewLogger("revok-worker")
	logger.RegisterSink(lager.NewWriterSink(os.Stdout, lager.DEBUG))

	logger.Info("starting")

	_, err := flags.Parse(&flagOpts)
	if err != nil {
		os.Exit(1)
	}

	if flagOpts.ConfigFile != "" {
		bs, err := ioutil.ReadFile(string(flagOpts.ConfigFile))
		if err != nil {
			logger.Error("failed-opening-config-file", err)
			os.Exit(1)
		}

		cfg, err = config.LoadWorkerConfig(bs)
		cfg.Merge(flagOpts.WorkerConfig)
	} else {
		cfg = flagOpts.WorkerConfig
	}

	errs := cfg.Validate()
	if errs != nil {
		for _, err := range errs {
			fmt.Println(err.Error())
		}
		os.Exit(1)
	}

	if cfg.Metrics.SentryDSN != "" {
		logger.RegisterSink(revok.NewSentrySink(cfg.Metrics.SentryDSN, cfg.Metrics.Environment))
	}

	workdir := cfg.WorkDir
	_, err = os.Lstat(workdir)
	if err != nil {
		log.Fatalf("workdir error: %s", err)
	}

	dbURI := db.NewDSN(cfg.MySQL.Username, cfg.MySQL.Password, cfg.MySQL.DBName, cfg.MySQL.Hostname, int(cfg.MySQL.Port))
	database, err := migrations.LockDBAndMigrate(logger, "mysql", dbURI)
	if err != nil {
		log.Fatalf("db error: %s", err)
	}

	database.LogMode(false)

	clock := clock.NewClock()

	cloneMsgCh := make(chan revok.CloneMsg)

	scanRepository := db.NewScanRepository(database, clock)
	repositoryRepository := db.NewRepositoryRepository(database)
	fetchRepository := db.NewFetchRepository(database)
	credentialRepository := db.NewCredentialRepository(database)
	emitter := metrics.BuildEmitter(cfg.Metrics.DatadogAPIKey, cfg.Metrics.Environment)
	gitClient := gitclient.New(string(cfg.GitHub.PrivateKeyPath), string(cfg.GitHub.PublicKeyPath))
	repoWhitelist := notifications.BuildWhitelist(cfg.Whitelist...)

	var notifier notifications.Notifier
	if cfg.Slack.WebhookURL != "" {
		formatter := notifications.NewSlackNotificationFormatter()
		notifier = notifications.NewSlackNotifier(cfg.Slack.WebhookURL, clock, repoWhitelist, formatter)
	} else {
		notifier = notifications.NewNullNotifier()
	}

	sniffer := sniff.NewDefaultSniffer()
	ancestryScanner := revok.NewScanner(
		gitClient,
		repositoryRepository,
		scanRepository,
		credentialRepository,
		sniffer,
		notifier,
	)

	changeFetcher := revok.NewChangeFetcher(
		logger,
		gitClient,
		ancestryScanner,
		repositoryRepository,
		fetchRepository,
		emitter,
	)

	changeScheduleRunner := revok.NewScheduleRunner(logger)

	changeScheduler := revok.NewChangeScheduler(
		logger,
		repositoryRepository,
		changeScheduleRunner,
		changeFetcher,
	)

	cloner := revok.NewCloner(
		logger,
		workdir,
		cloneMsgCh,
		gitClient,
		repositoryRepository,
		ancestryScanner,
		emitter,
		changeScheduler,
	)

	dirscanUpdater := revok.NewRescanner(
		logger,
		scanRepository,
		credentialRepository,
		ancestryScanner,
		notifier,
		emitter,
	)

	statsReporter := stats.NewReporter(
		logger,
		clock,
		60*time.Second,
		db.NewStatsRepository(database),
		emitter,
	)

	headCredentialCounter := revok.NewHeadCredentialCounter(
		logger,
		repositoryRepository,
		clock,
		cfg.CredentialCounterInterval,
		gitClient,
		sniffer,
	)

	members := []grouper.Member{
		{"cloner", cloner},
		{"dirscan-updater", dirscanUpdater},
		{"stats-reporter", statsReporter},
		{"head-credential-counter", headCredentialCounter},
		{"change-schedule-runner", changeScheduleRunner},
		{"debug", http_server.New("127.0.0.1:6060", debugHandler())},
	}

	if cfg.IsRPCConfigured() {
		certificate, err := config.LoadCertificate(
			string(cfg.RPC.CertificatePath),
			string(cfg.RPC.PrivateKeyPath),
			cfg.RPC.PrivateKeyPassphrase,
		)
		if err != nil {
			log.Fatalln(err)
		}

		clientCertPool, err := config.LoadCertificatePool(string(cfg.RPC.ClientCACertificatePath))
		if err != nil {
			log.Fatalln(err)
		}

		looper := gitclient.NewLooper()
		searcher := search.NewSearcher(repositoryRepository, looper)

		grpcServer := redrunner.NewGRPCServer(
			logger,
			fmt.Sprintf("%s:%d", cfg.RPC.BindIP, cfg.RPC.BindPort),
			&tls.Config{
				ClientAuth:   tls.RequireAndVerifyClientCert,
				Certificates: []tls.Certificate{certificate},
				ClientCAs:    clientCertPool,
			},
			func(server *grpc.Server) {
				revokpb.RegisterRevokServer(server, revok.NewServer(logger, repositoryRepository, searcher))
			},
		)

		members = append(members, grouper.Member{
			Name:   "grpc-server",
			Runner: grpcServer,
		})
	}

	if cfg.IsPubSubConfigured() {
		pubSubClient, err := pubsub.NewClient(context.Background(), cfg.PubSub.ProjectName)
		if err != nil {
			logger.Fatal("failed", err)
			os.Exit(1)
		}

		subscription := pubSubClient.Subscription(cfg.PubSub.FetchHint.Subscription)

		publicKey, err := crypto.ReadRSAPublicKey(string(cfg.PubSub.PublicKeyPath))
		if err != nil {
			logger.Fatal("failed", err)
			os.Exit(1)
		}
		pushEventProcessor := queue.NewPushEventProcessor(
			changeFetcher,
			crypto.NewRSAVerifier(publicKey),
			emitter,
		)

		members = append(members, grouper.Member{
			Name:   "github-hint-handler",
			Runner: queue.NewPubSubAcceptor(logger, subscription, pushEventProcessor, emitter),
		})
	}

	if cfg.GitHub.AccessToken != "" {
		githubHTTPClient := &http.Client{
			Timeout: 30 * time.Second,
			Transport: &oauth2.Transport{
				Source: oauth2.StaticTokenSource(
					&oauth2.Token{AccessToken: cfg.GitHub.AccessToken},
				),
				Base: &http.Transport{
					DisableKeepAlives: true,
				},
			},
		}

		ghClient := revok.NewGitHubClient(github.NewClient(githubHTTPClient))

		repoDiscoverer := revok.NewRepoDiscoverer(
			logger,
			workdir,
			cloneMsgCh,
			ghClient,
			clock,
			cfg.RepositoryDiscoveryInterval,
			repositoryRepository,
		)

		members = append(members, grouper.Member{
			Name:   "repo-discoverer",
			Runner: repoDiscoverer,
		})
	}

	startupTasks := []grouper.Member{
		{
			Name:   "schedule-fetches",
			Runner: changeScheduler,
		},
	}

	system := []grouper.Member{
		{
			Name:   "servers",
			Runner: grouper.NewParallel(os.Interrupt, members),
		},
		{
			Name:   "startup-tasks",
			Runner: grouper.NewParallel(os.Interrupt, startupTasks),
		},
	}

	runner := sigmon.New(grouper.NewOrdered(os.Interrupt, system))

	err = <-ifrit.Invoke(runner).Wait()
	if err != nil {
		log.Fatalf("failed-to-start: %s", err)
	}
}

func debugHandler() http.Handler {
	debugRouter := http.NewServeMux()
	debugRouter.Handle("/debug/pprof/", http.HandlerFunc(pprof.Index))
	debugRouter.Handle("/debug/pprof/cmdline", http.HandlerFunc(pprof.Cmdline))
	debugRouter.Handle("/debug/pprof/profile", http.HandlerFunc(pprof.Profile))
	debugRouter.Handle("/debug/pprof/symbol", http.HandlerFunc(pprof.Symbol))
	debugRouter.Handle("/debug/pprof/trace", http.HandlerFunc(pprof.Trace))

	debugRouter.HandleFunc("/debug/requests", func(w http.ResponseWriter, req *http.Request) {
		any, sensitive := trace.AuthRequest(req)
		if !any {
			http.Error(w, "not allowed", http.StatusUnauthorized)
			return
		}
		w.Header().Set("Content-Type", "text/html; charset=utf-8")
		trace.Render(w, req, sensitive)
	})

	debugRouter.HandleFunc("/debug/events", func(w http.ResponseWriter, req *http.Request) {
		any, sensitive := trace.AuthRequest(req)
		if !any {
			http.Error(w, "not allowed", http.StatusUnauthorized)
			return
		}
		w.Header().Set("Content-Type", "text/html; charset=utf-8")
		trace.RenderEvents(w, req, sensitive)
	})

	return debugRouter
}
<|endoftext|>


<commit_before>package main

import (
	"context"
	"crypto/tls"
	"crypto/x509"
	"fmt"
	"io/ioutil"
	"log"
	"net"
	"net/http"
	"os"
	"time"

	"cloud.google.com/go/pubsub"
	cloudtrace "cloud.google.com/go/trace"
	"code.cloudfoundry.org/clock"
	"code.cloudfoundry.org/lager"
	"github.com/google/go-github/github"
	flags "github.com/jessevdk/go-flags"
	"github.com/pivotal-cf/paraphernalia/operate/admin"
	"github.com/pivotal-cf/paraphernalia/secure/tlsconfig"
	"github.com/pivotal-cf/paraphernalia/serve/grpcrunner"
	"github.com/tedsuo/ifrit"
	"github.com/tedsuo/ifrit/grouper"
	"github.com/tedsuo/ifrit/sigmon"
	"golang.org/x/oauth2"
	"google.golang.org/grpc"
	"google.golang.org/grpc/credentials"

	"cred-alert/config"
	"cred-alert/crypto"
	"cred-alert/db"
	"cred-alert/db/migrations"
	"cred-alert/gitclient"
	"cred-alert/metrics"
	"cred-alert/notifications"
	"cred-alert/queue"
	"cred-alert/revok"
	"cred-alert/revok/stats"
	"cred-alert/revokpb"
	"cred-alert/search"
	"cred-alert/sniff"
	"rolodex/rolodexpb"
)

var info = admin.ServiceInfo{
	Name:        "revok",
	Description: "A service which fetches new Git commits and scans them for credentials.",
	Team:        "PCF Security Enablement",
}

func main() {
	var cfg *config.WorkerConfig
	var flagOpts config.WorkerOpts

	logger := lager.NewLogger("revok-worker")
	logger.RegisterSink(lager.NewWriterSink(os.Stdout, lager.DEBUG))

	logger.Info("starting")

	_, err := flags.Parse(&flagOpts)
	if err != nil {
		os.Exit(1)
	}

	bs, err := ioutil.ReadFile(string(flagOpts.ConfigFile))
	if err != nil {
		logger.Error("failed-to-open-config-file", err)
		os.Exit(1)
	}

	cfg, err = config.LoadWorkerConfig(bs)
	if err != nil {
		logger.Error("failed-to-load-config-file", err)
		os.Exit(1)
	}

	errs := cfg.Validate()
	if errs != nil {
		for _, err := range errs {
			fmt.Println(err.Error())
		}
		os.Exit(1)
	}

	if cfg.Metrics.SentryDSN != "" {
		logger.RegisterSink(revok.NewSentrySink(cfg.Metrics.SentryDSN, cfg.Metrics.Environment))
	}

	workdir := cfg.WorkDir
	_, err = os.Lstat(workdir)
	if err != nil {
		log.Fatalf("workdir error: %s", err)
	}

	dbCertificate, dbCaCertPool := loadCerts(
		cfg.MySQL.CertificatePath,
		cfg.MySQL.PrivateKeyPath,
		cfg.MySQL.PrivateKeyPassphrase,
		cfg.MySQL.CACertificatePath,
	)

	dbURI := db.NewDSN(
		cfg.MySQL.Username,
		cfg.MySQL.Password,
		cfg.MySQL.DBName,
		cfg.MySQL.Hostname,
		int(cfg.MySQL.Port),
		cfg.MySQL.ServerName,
		dbCertificate,
		dbCaCertPool,
	)

	database, err := migrations.LockDBAndMigrate(logger, "mysql", dbURI)
	if err != nil {
		log.Fatalf("db error: %s", err)
	}

	database.LogMode(false)

	clk := clock.NewClock()

	cloneMsgCh := make(chan revok.CloneMsg)

	scanRepository := db.NewScanRepository(database, clk)
	repositoryRepository := db.NewRepositoryRepository(database)
	fetchRepository := db.NewFetchRepository(database)
	credentialRepository := db.NewCredentialRepository(database)
	branchRepository := db.NewBranchRepository(database)

	emitter := metrics.BuildEmitter(cfg.Metrics.DatadogAPIKey, cfg.Metrics.Environment)
	gitClient := gitclient.New(cfg.GitHub.PrivateKeyPath, cfg.GitHub.PublicKeyPath)
	repoWhitelist := notifications.BuildWhitelist(cfg.Whitelist...)
	formatter := notifications.NewSlackNotificationFormatter()

	slackHTTPClient := &http.Client{
		Timeout: 3 * time.Second,
	}
	notifier := notifications.NewSlackNotifier(clk, slackHTTPClient, formatter)

	certificate, caCertPool := loadCerts(
		cfg.Identity.CertificatePath,
		cfg.Identity.PrivateKeyPath,
		cfg.Identity.PrivateKeyPassphrase,
		cfg.Identity.CACertificatePath,
	)

	rolodexServerAddr := fmt.Sprintf("%s:%d", cfg.Rolodex.ServerAddress, cfg.Rolodex.ServerPort)

	tlsConfig := tlsconfig.Build(
		tlsconfig.WithPivotalDefaults(),
		tlsconfig.WithIdentity(certificate),
	)

	transportCreds := credentials.NewTLS(tlsConfig.Client(tlsconfig.WithAuthority(caCertPool)))

	traceClient, err := cloudtrace.NewClient(context.Background(), cfg.Trace.ProjectName)
	if err != nil {
		logger.Error("failed-to-create-trace-client", err)
	}

	conn, err := grpc.Dial(
		rolodexServerAddr,
		grpc.WithDialer(keepAliveDial),
		grpc.WithTransportCredentials(transportCreds),
		grpc.WithUnaryInterceptor(cloudtrace.GRPCClientInterceptor()),
	)

	rolodexClient := rolodexpb.NewRolodexClient(conn)

	teamURLs := notifications.NewTeamURLs(
		cfg.Slack.DefaultURL,
		cfg.Slack.DefaultChannel,
		cfg.Slack.TeamURLs,
	)

	addressBook := notifications.NewRolodex(
		rolodexClient,
		teamURLs,
	)

	router := notifications.NewRouter(
		notifier,
		addressBook,
		repoWhitelist,
	)

	sniffer := sniff.NewDefaultSniffer()
	scanner := revok.NewScanner(
		gitClient,
		repositoryRepository,
		scanRepository,
		credentialRepository,
		sniffer,
	)

	notificationComposer := revok.NewNotificationComposer(
		repositoryRepository,
		router,
		scanner,
	)

	changeFetcher := revok.NewChangeFetcher(
		logger,
		gitClient,
		notificationComposer,
		repositoryRepository,
		fetchRepository,
		emitter,
	)

	changeScheduleRunner := revok.NewScheduleRunner(logger)

	changeScheduler := revok.NewChangeScheduler(
		logger,
		repositoryRepository,
		changeScheduleRunner,
		changeFetcher,
	)

	cloner := revok.NewCloner(
		logger,
		workdir,
		cloneMsgCh,
		gitClient,
		repositoryRepository,
		notificationComposer,
		emitter,
		changeScheduler,
	)

	dirscanUpdater := revok.NewRescanner(
		logger,
		scanRepository,
		credentialRepository,
		scanner,
		router,
		emitter,
	)

	statsReporter := stats.NewReporter(
		logger,
		clk,
		60*time.Second,
		db.NewStatsRepository(database),
		emitter,
	)

	headCredentialCounter := revok.NewHeadCredentialCounter(
		logger,
		branchRepository,
		repositoryRepository,
		clk,
		cfg.CredentialCounterInterval,
		gitClient,
		sniffer,
	)

	debug := admin.Runner(
		"6060",
		admin.WithInfo(info),
		admin.WithUptime(),
	)

	members := []grouper.Member{
		{"cloner", cloner},
		{"dirscan-updater", dirscanUpdater},
		{"stats-reporter", statsReporter},
		{"head-credential-counter", headCredentialCounter},
		{"change-schedule-runner", changeScheduleRunner},
		{"debug", debug},
	}

	looper := gitclient.NewLooper()
	searcher := search.NewSearcher(repositoryRepository, looper)
	handler := revok.NewServer(logger, searcher, repositoryRepository, branchRepository)

	serverTls := tlsConfig.Server(tlsconfig.WithClientAuthentication(caCertPool))

	grpcServer := grpcrunner.New(
		logger,
		fmt.Sprintf("%s:%d", cfg.API.BindIP, cfg.API.BindPort),
		func(server *grpc.Server) {
			revokpb.RegisterRevokServer(server, handler)
		},
		grpc.Creds(credentials.NewTLS(serverTls)),
	)

	members = append(members, grouper.Member{
		Name:   "grpc-server",
		Runner: grpcServer,
	})

	pubSubClient, err := pubsub.NewClient(context.Background(), cfg.PubSub.ProjectName)
	if err != nil {
		logger.Fatal("failed", err)
		os.Exit(1)
	}

	subscription := pubSubClient.Subscription(cfg.PubSub.FetchHint.Subscription)

	publicKey, err := crypto.ReadRSAPublicKey(cfg.PubSub.PublicKeyPath)
	if err != nil {
		logger.Fatal("failed", err)
		os.Exit(1)
	}
	pushEventProcessor := queue.NewPushEventProcessor(
		changeFetcher,
		crypto.NewRSAVerifier(publicKey),
		emitter,
		clk,
	)

	members = append(members, grouper.Member{
		Name:   "github-hint-handler",
		Runner: queue.NewPubSubSubscriber(logger, subscription, pushEventProcessor, emitter, traceClient),
	})

	if cfg.GitHub.AccessToken != "" {
		githubHTTPClient := &http.Client{
			Timeout: 30 * time.Second,
			Transport: &oauth2.Transport{
				Source: oauth2.StaticTokenSource(
					&oauth2.Token{AccessToken: cfg.GitHub.AccessToken},
				),
				Base: &http.Transport{
					DisableKeepAlives: true,
				},
			},
		}

		ghClient := revok.NewGitHubClient(github.NewClient(githubHTTPClient))

		repoDiscoverer := revok.NewRepoDiscoverer(
			logger,
			workdir,
			cloneMsgCh,
			ghClient,
			clk,
			cfg.RepositoryDiscovery.Interval,
			cfg.RepositoryDiscovery.Organizations,
			cfg.RepositoryDiscovery.Users,
			repositoryRepository,
		)

		members = append(members, grouper.Member{
			Name:   "repo-discoverer",
			Runner: repoDiscoverer,
		})
	}

	startupTasks := []grouper.Member{
		{
			Name:   "schedule-fetches",
			Runner: changeScheduler,
		},
	}

	system := []grouper.Member{
		{
			Name:   "servers",
			Runner: grouper.NewParallel(os.Interrupt, members),
		},
		{
			Name:   "startup-tasks",
			Runner: grouper.NewParallel(os.Interrupt, startupTasks),
		},
	}

	runner := sigmon.New(grouper.NewOrdered(os.Interrupt, system))

	err = <-ifrit.Invoke(runner).Wait()
	if err != nil {
		log.Fatalf("failed-to-start: %s", err)
	}
}

func loadCerts(certificatePath, privateKeyPath, privateKeyPassphrase, caCertificatePath string) (tls.Certificate, *x509.CertPool) {
	certificate, err := config.LoadCertificate(
		certificatePath,
		privateKeyPath,
		privateKeyPassphrase,
	)
	if err != nil {
		log.Fatalln(err)
	}

	caCertPool, err := config.LoadCertificatePool(caCertificatePath)
	if err != nil {
		log.Fatalln(err)
	}

	return certificate, caCertPool
}

func keepAliveDial(addr string, timeout time.Duration) (net.Conn, error) {
	d := net.Dialer{
		Timeout:   timeout,
		KeepAlive: 60 * time.Second,
	}
	return d.Dial("tcp", addr)
}
<commit_msg>use keyed struct fields<commit_after>package main

import (
	"context"
	"crypto/tls"
	"crypto/x509"
	"fmt"
	"io/ioutil"
	"log"
	"net"
	"net/http"
	"os"
	"time"

	"cloud.google.com/go/pubsub"
	cloudtrace "cloud.google.com/go/trace"
	"code.cloudfoundry.org/clock"
	"code.cloudfoundry.org/lager"
	"github.com/google/go-github/github"
	flags "github.com/jessevdk/go-flags"
	"github.com/pivotal-cf/paraphernalia/operate/admin"
	"github.com/pivotal-cf/paraphernalia/secure/tlsconfig"
	"github.com/pivotal-cf/paraphernalia/serve/grpcrunner"
	"github.com/tedsuo/ifrit"
	"github.com/tedsuo/ifrit/grouper"
	"github.com/tedsuo/ifrit/sigmon"
	"golang.org/x/oauth2"
	"google.golang.org/grpc"
	"google.golang.org/grpc/credentials"

	"cred-alert/config"
	"cred-alert/crypto"
	"cred-alert/db"
	"cred-alert/db/migrations"
	"cred-alert/gitclient"
	"cred-alert/metrics"
	"cred-alert/notifications"
	"cred-alert/queue"
	"cred-alert/revok"
	"cred-alert/revok/stats"
	"cred-alert/revokpb"
	"cred-alert/search"
	"cred-alert/sniff"
	"rolodex/rolodexpb"
)

var info = admin.ServiceInfo{
	Name:        "revok",
	Description: "A service which fetches new Git commits and scans them for credentials.",
	Team:        "PCF Security Enablement",
}

func main() {
	var cfg *config.WorkerConfig
	var flagOpts config.WorkerOpts

	logger := lager.NewLogger("revok-worker")
	logger.RegisterSink(lager.NewWriterSink(os.Stdout, lager.DEBUG))

	logger.Info("starting")

	_, err := flags.Parse(&flagOpts)
	if err != nil {
		os.Exit(1)
	}

	bs, err := ioutil.ReadFile(string(flagOpts.ConfigFile))
	if err != nil {
		logger.Error("failed-to-open-config-file", err)
		os.Exit(1)
	}

	cfg, err = config.LoadWorkerConfig(bs)
	if err != nil {
		logger.Error("failed-to-load-config-file", err)
		os.Exit(1)
	}

	errs := cfg.Validate()
	if errs != nil {
		for _, err := range errs {
			fmt.Println(err.Error())
		}
		os.Exit(1)
	}

	if cfg.Metrics.SentryDSN != "" {
		logger.RegisterSink(revok.NewSentrySink(cfg.Metrics.SentryDSN, cfg.Metrics.Environment))
	}

	workdir := cfg.WorkDir
	_, err = os.Lstat(workdir)
	if err != nil {
		log.Fatalf("workdir error: %s", err)
	}

	dbCertificate, dbCaCertPool := loadCerts(
		cfg.MySQL.CertificatePath,
		cfg.MySQL.PrivateKeyPath,
		cfg.MySQL.PrivateKeyPassphrase,
		cfg.MySQL.CACertificatePath,
	)

	dbURI := db.NewDSN(
		cfg.MySQL.Username,
		cfg.MySQL.Password,
		cfg.MySQL.DBName,
		cfg.MySQL.Hostname,
		int(cfg.MySQL.Port),
		cfg.MySQL.ServerName,
		dbCertificate,
		dbCaCertPool,
	)

	database, err := migrations.LockDBAndMigrate(logger, "mysql", dbURI)
	if err != nil {
		log.Fatalf("db error: %s", err)
	}

	database.LogMode(false)

	clk := clock.NewClock()

	cloneMsgCh := make(chan revok.CloneMsg)

	scanRepository := db.NewScanRepository(database, clk)
	repositoryRepository := db.NewRepositoryRepository(database)
	fetchRepository := db.NewFetchRepository(database)
	credentialRepository := db.NewCredentialRepository(database)
	branchRepository := db.NewBranchRepository(database)

	emitter := metrics.BuildEmitter(cfg.Metrics.DatadogAPIKey, cfg.Metrics.Environment)
	gitClient := gitclient.New(cfg.GitHub.PrivateKeyPath, cfg.GitHub.PublicKeyPath)
	repoWhitelist := notifications.BuildWhitelist(cfg.Whitelist...)
	formatter := notifications.NewSlackNotificationFormatter()

	slackHTTPClient := &http.Client{
		Timeout: 3 * time.Second,
	}
	notifier := notifications.NewSlackNotifier(clk, slackHTTPClient, formatter)

	certificate, caCertPool := loadCerts(
		cfg.Identity.CertificatePath,
		cfg.Identity.PrivateKeyPath,
		cfg.Identity.PrivateKeyPassphrase,
		cfg.Identity.CACertificatePath,
	)

	rolodexServerAddr := fmt.Sprintf("%s:%d", cfg.Rolodex.ServerAddress, cfg.Rolodex.ServerPort)

	tlsConfig := tlsconfig.Build(
		tlsconfig.WithPivotalDefaults(),
		tlsconfig.WithIdentity(certificate),
	)

	transportCreds := credentials.NewTLS(tlsConfig.Client(tlsconfig.WithAuthority(caCertPool)))

	traceClient, err := cloudtrace.NewClient(context.Background(), cfg.Trace.ProjectName)
	if err != nil {
		logger.Error("failed-to-create-trace-client", err)
	}

	conn, err := grpc.Dial(
		rolodexServerAddr,
		grpc.WithDialer(keepAliveDial),
		grpc.WithTransportCredentials(transportCreds),
		grpc.WithUnaryInterceptor(cloudtrace.GRPCClientInterceptor()),
	)

	rolodexClient := rolodexpb.NewRolodexClient(conn)

	teamURLs := notifications.NewTeamURLs(
		cfg.Slack.DefaultURL,
		cfg.Slack.DefaultChannel,
		cfg.Slack.TeamURLs,
	)

	addressBook := notifications.NewRolodex(
		rolodexClient,
		teamURLs,
	)

	router := notifications.NewRouter(
		notifier,
		addressBook,
		repoWhitelist,
	)

	sniffer := sniff.NewDefaultSniffer()
	scanner := revok.NewScanner(
		gitClient,
		repositoryRepository,
		scanRepository,
		credentialRepository,
		sniffer,
	)

	notificationComposer := revok.NewNotificationComposer(
		repositoryRepository,
		router,
		scanner,
	)

	changeFetcher := revok.NewChangeFetcher(
		logger,
		gitClient,
		notificationComposer,
		repositoryRepository,
		fetchRepository,
		emitter,
	)

	changeScheduleRunner := revok.NewScheduleRunner(logger)

	changeScheduler := revok.NewChangeScheduler(
		logger,
		repositoryRepository,
		changeScheduleRunner,
		changeFetcher,
	)

	cloner := revok.NewCloner(
		logger,
		workdir,
		cloneMsgCh,
		gitClient,
		repositoryRepository,
		notificationComposer,
		emitter,
		changeScheduler,
	)

	dirscanUpdater := revok.NewRescanner(
		logger,
		scanRepository,
		credentialRepository,
		scanner,
		router,
		emitter,
	)

	statsReporter := stats.NewReporter(
		logger,
		clk,
		60*time.Second,
		db.NewStatsRepository(database),
		emitter,
	)

	headCredentialCounter := revok.NewHeadCredentialCounter(
		logger,
		branchRepository,
		repositoryRepository,
		clk,
		cfg.CredentialCounterInterval,
		gitClient,
		sniffer,
	)

	debug := admin.Runner(
		"6060",
		admin.WithInfo(info),
		admin.WithUptime(),
	)

	members := []grouper.Member{
		{Name: "cloner", Runner: cloner},
		{Name: "dirscan-updater", Runner: dirscanUpdater},
		{Name: "stats-reporter", Runner: statsReporter},
		{Name: "head-credential-counter", Runner: headCredentialCounter},
		{Name: "change-schedule-runner", Runner: changeScheduleRunner},
		{Name: "debug", Runner: debug},
	}

	looper := gitclient.NewLooper()
	searcher := search.NewSearcher(repositoryRepository, looper)
	handler := revok.NewServer(logger, searcher, repositoryRepository, branchRepository)

	serverTls := tlsConfig.Server(tlsconfig.WithClientAuthentication(caCertPool))

	grpcServer := grpcrunner.New(
		logger,
		fmt.Sprintf("%s:%d", cfg.API.BindIP, cfg.API.BindPort),
		func(server *grpc.Server) {
			revokpb.RegisterRevokServer(server, handler)
		},
		grpc.Creds(credentials.NewTLS(serverTls)),
	)

	members = append(members, grouper.Member{
		Name:   "grpc-server",
		Runner: grpcServer,
	})

	pubSubClient, err := pubsub.NewClient(context.Background(), cfg.PubSub.ProjectName)
	if err != nil {
		logger.Fatal("failed", err)
		os.Exit(1)
	}

	subscription := pubSubClient.Subscription(cfg.PubSub.FetchHint.Subscription)

	publicKey, err := crypto.ReadRSAPublicKey(cfg.PubSub.PublicKeyPath)
	if err != nil {
		logger.Fatal("failed", err)
		os.Exit(1)
	}
	pushEventProcessor := queue.NewPushEventProcessor(
		changeFetcher,
		crypto.NewRSAVerifier(publicKey),
		emitter,
		clk,
	)

	members = append(members, grouper.Member{
		Name:   "github-hint-handler",
		Runner: queue.NewPubSubSubscriber(logger, subscription, pushEventProcessor, emitter, traceClient),
	})

	if cfg.GitHub.AccessToken != "" {
		githubHTTPClient := &http.Client{
			Timeout: 30 * time.Second,
			Transport: &oauth2.Transport{
				Source: oauth2.StaticTokenSource(
					&oauth2.Token{AccessToken: cfg.GitHub.AccessToken},
				),
				Base: &http.Transport{
					DisableKeepAlives: true,
				},
			},
		}

		ghClient := revok.NewGitHubClient(github.NewClient(githubHTTPClient))

		repoDiscoverer := revok.NewRepoDiscoverer(
			logger,
			workdir,
			cloneMsgCh,
			ghClient,
			clk,
			cfg.RepositoryDiscovery.Interval,
			cfg.RepositoryDiscovery.Organizations,
			cfg.RepositoryDiscovery.Users,
			repositoryRepository,
		)

		members = append(members, grouper.Member{
			Name:   "repo-discoverer",
			Runner: repoDiscoverer,
		})
	}

	startupTasks := []grouper.Member{
		{
			Name:   "schedule-fetches",
			Runner: changeScheduler,
		},
	}

	system := []grouper.Member{
		{
			Name:   "servers",
			Runner: grouper.NewParallel(os.Interrupt, members),
		},
		{
			Name:   "startup-tasks",
			Runner: grouper.NewParallel(os.Interrupt, startupTasks),
		},
	}

	runner := sigmon.New(grouper.NewOrdered(os.Interrupt, system))

	err = <-ifrit.Invoke(runner).Wait()
	if err != nil {
		log.Fatalf("failed-to-start: %s", err)
	}
}

func loadCerts(certificatePath, privateKeyPath, privateKeyPassphrase, caCertificatePath string) (tls.Certificate, *x509.CertPool) {
	certificate, err := config.LoadCertificate(
		certificatePath,
		privateKeyPath,
		privateKeyPassphrase,
	)
	if err != nil {
		log.Fatalln(err)
	}

	caCertPool, err := config.LoadCertificatePool(caCertificatePath)
	if err != nil {
		log.Fatalln(err)
	}

	return certificate, caCertPool
}

func keepAliveDial(addr string, timeout time.Duration) (net.Conn, error) {
	d := net.Dialer{
		Timeout:   timeout,
		KeepAlive: 60 * time.Second,
	}
	return d.Dial("tcp", addr)
}
<|endoftext|>


<commit_before>package ethchain

import (
	"math/big"
)

var TxFeeRat *big.Int = big.NewInt(100000000000000)

var TxFee *big.Int = big.NewInt(100)
var StepFee *big.Int = big.NewInt(1)
var StoreFee *big.Int = big.NewInt(0)
var DataFee *big.Int = big.NewInt(20)
var ExtroFee *big.Int = big.NewInt(40)
var CryptoFee *big.Int = big.NewInt(20)
var ContractFee *big.Int = big.NewInt(100)

var BlockReward *big.Int = big.NewInt(1.5e+18)
var UncleReward *big.Int = big.NewInt(1.125e+18)
var UncleInclusionReward *big.Int = big.NewInt(1.875e+17)

var Period1Reward *big.Int = new(big.Int)
var Period2Reward *big.Int = new(big.Int)
var Period3Reward *big.Int = new(big.Int)
var Period4Reward *big.Int = new(big.Int)

func InitFees() {
	StepFee.Mul(StepFee, TxFeeRat)
	StoreFee.Mul(StoreFee, TxFeeRat)
	DataFee.Mul(DataFee, TxFeeRat)
	ExtroFee.Mul(ExtroFee, TxFeeRat)
	CryptoFee.Mul(CryptoFee, TxFeeRat)
	ContractFee.Mul(ContractFee, TxFeeRat)
	/*
		// Base for 2**64
		b60 := new(big.Int)
		b60.Exp(big.NewInt(2), big.NewInt(64), big.NewInt(0))
		// Base for 2**80
		b80 := new(big.Int)
		b80.Exp(big.NewInt(2), big.NewInt(80), big.NewInt(0))

		StepFee.Exp(big.NewInt(10), big.NewInt(16), big.NewInt(0))
		//StepFee.Div(b60, big.NewInt(64))
		//fmt.Println("StepFee:", StepFee)

		TxFee.Exp(big.NewInt(2), big.NewInt(64), big.NewInt(0))
		//fmt.Println("TxFee:", TxFee)

		ContractFee.Exp(big.NewInt(2), big.NewInt(64), big.NewInt(0))
		//fmt.Println("ContractFee:", ContractFee)

		MemFee.Div(b60, big.NewInt(4))
		//fmt.Println("MemFee:", MemFee)

		DataFee.Div(b60, big.NewInt(16))
		//fmt.Println("DataFee:", DataFee)

		CryptoFee.Div(b60, big.NewInt(16))
		//fmt.Println("CrytoFee:", CryptoFee)

		ExtroFee.Div(b60, big.NewInt(16))
		//fmt.Println("ExtroFee:", ExtroFee)

		Period1Reward.Mul(b80, big.NewInt(1024))
		//fmt.Println("Period1Reward:", Period1Reward)

		Period2Reward.Mul(b80, big.NewInt(512))
		//fmt.Println("Period2Reward:", Period2Reward)

		Period3Reward.Mul(b80, big.NewInt(256))
		//fmt.Println("Period3Reward:", Period3Reward)

		Period4Reward.Mul(b80, big.NewInt(128))
		//fmt.Println("Period4Reward:", Period4Reward)
	*/
}
<commit_msg>Updated fees<commit_after>package ethchain

import (
	"math/big"
)

var TxFeeRat *big.Int = big.NewInt(100000000000000)

var TxFee *big.Int = big.NewInt(100)
var StepFee *big.Int = big.NewInt(1)
var StoreFee *big.Int = big.NewInt(5)
var DataFee *big.Int = big.NewInt(20)
var ExtroFee *big.Int = big.NewInt(40)
var CryptoFee *big.Int = big.NewInt(20)
var ContractFee *big.Int = big.NewInt(100)

var BlockReward *big.Int = big.NewInt(1.5e+18)
var UncleReward *big.Int = big.NewInt(1.125e+18)
var UncleInclusionReward *big.Int = big.NewInt(1.875e+17)

var Period1Reward *big.Int = new(big.Int)
var Period2Reward *big.Int = new(big.Int)
var Period3Reward *big.Int = new(big.Int)
var Period4Reward *big.Int = new(big.Int)

func InitFees() {
	StepFee.Mul(StepFee, TxFeeRat)
	StoreFee.Mul(StoreFee, TxFeeRat)
	DataFee.Mul(DataFee, TxFeeRat)
	ExtroFee.Mul(ExtroFee, TxFeeRat)
	CryptoFee.Mul(CryptoFee, TxFeeRat)
	ContractFee.Mul(ContractFee, TxFeeRat)
	/*
		// Base for 2**64
		b60 := new(big.Int)
		b60.Exp(big.NewInt(2), big.NewInt(64), big.NewInt(0))
		// Base for 2**80
		b80 := new(big.Int)
		b80.Exp(big.NewInt(2), big.NewInt(80), big.NewInt(0))

		StepFee.Exp(big.NewInt(10), big.NewInt(16), big.NewInt(0))
		//StepFee.Div(b60, big.NewInt(64))
		//fmt.Println("StepFee:", StepFee)

		TxFee.Exp(big.NewInt(2), big.NewInt(64), big.NewInt(0))
		//fmt.Println("TxFee:", TxFee)

		ContractFee.Exp(big.NewInt(2), big.NewInt(64), big.NewInt(0))
		//fmt.Println("ContractFee:", ContractFee)

		MemFee.Div(b60, big.NewInt(4))
		//fmt.Println("MemFee:", MemFee)

		DataFee.Div(b60, big.NewInt(16))
		//fmt.Println("DataFee:", DataFee)

		CryptoFee.Div(b60, big.NewInt(16))
		//fmt.Println("CrytoFee:", CryptoFee)

		ExtroFee.Div(b60, big.NewInt(16))
		//fmt.Println("ExtroFee:", ExtroFee)

		Period1Reward.Mul(b80, big.NewInt(1024))
		//fmt.Println("Period1Reward:", Period1Reward)

		Period2Reward.Mul(b80, big.NewInt(512))
		//fmt.Println("Period2Reward:", Period2Reward)

		Period3Reward.Mul(b80, big.NewInt(256))
		//fmt.Println("Period3Reward:", Period3Reward)

		Period4Reward.Mul(b80, big.NewInt(128))
		//fmt.Println("Period4Reward:", Period4Reward)
	*/
}
<|endoftext|>


<commit_before>package eval

//go:generate ./boilerplate.py

import (
	"fmt"
	"strconv"
	"strings"

	"github.com/elves/elvish/parse"
	"github.com/elves/elvish/util"
)

// compiler maintains the set of states needed when compiling a single source
// file.
type compiler struct {
	// Builtin namespace.
	builtin staticNs
	// Lexical namespaces.
	scopes []staticNs
	// Variables captured from outer scopes.
	capture staticNs
	// Position of what is being compiled.
	begin, end int
	// Information about the source.
	srcMeta *Source
}

func compile(b, g staticNs, n *parse.Chunk, src *Source) (op Op, err error) {
	cp := &compiler{b, []staticNs{g}, make(staticNs), 0, 0, src}
	defer util.Catch(&err)
	return cp.chunkOp(n), nil
}

func (cp *compiler) compiling(n parse.Node) {
	cp.begin, cp.end = n.Begin(), n.End()
}

func (cp *compiler) errorpf(begin, end int, format string, args ...interface{}) {
	throw(&CompilationError{fmt.Sprintf(format, args...),
		*util.NewSourceRange(cp.srcMeta.describePath(), cp.srcMeta.code, begin, end, nil)})
}

func (cp *compiler) errorf(format string, args ...interface{}) {
	cp.errorpf(cp.begin, cp.end, format, args...)
}

func (cp *compiler) thisScope() staticNs {
	return cp.scopes[len(cp.scopes)-1]
}

func (cp *compiler) pushScope() staticNs {
	sc := make(staticNs)
	cp.scopes = append(cp.scopes, sc)
	return sc
}

func (cp *compiler) popScope() {
	cp.scopes[len(cp.scopes)-1] = make(staticNs)
	cp.scopes = cp.scopes[:len(cp.scopes)-1]
}

func (cp *compiler) registerVariableGetQname(qname string) bool {
	_, ns, name := ParseVariableRef(qname)
	return cp.registerVariableGet(ns, name)
}

func (cp *compiler) registerVariableGet(ns, name string) bool {
	switch ns {
	case "", "local", "up":
		// Handled below
	case "e", "E":
		return true
	default:
		return cp.registerModAccess(ns)
	}
	_, err := strconv.Atoi(name)
	isnum := err == nil
	// Find in local scope
	if ns == "" || ns == "local" {
		if cp.thisScope().has(name) || isnum {
			return true
		}
	}
	// Find in upper scopes
	if ns == "" || ns == "up" {
		for i := len(cp.scopes) - 2; i >= 0; i-- {
			if cp.scopes[i].has(name) || isnum {
				// Existing name: record capture and return.
				cp.capture.set(name)
				return true
			}
		}
	}
	// Find in builtin scope
	if ns == "" || ns == "builtin" {
		if cp.builtin.has(name) {
			return true
		}
	}
	return false
}

func (cp *compiler) registerVariableSetQname(qname string) bool {
	_, ns, name := ParseVariableRef(qname)
	return cp.registerVariableSet(ns, name)
}

func (cp *compiler) registerVariableSet(ns, name string) bool {
	switch ns {
	case "local":
		cp.thisScope().set(name)
		return true
	case "up":
		for i := len(cp.scopes) - 2; i >= 0; i-- {
			if cp.scopes[i].has(name) {
				// Existing name: record capture and return.
				cp.capture.set(name)
				return true
			}
		}
		return false
	case "builtin":
		cp.errorf("cannot set builtin variable")
		return false
	case "":
		if cp.thisScope().has(name) {
			// A name on current scope. Do nothing.
			return true
		}
		// Walk up the upper scopes
		for i := len(cp.scopes) - 2; i >= 0; i-- {
			if cp.scopes[i].has(name) {
				// Existing name. Do nothing
				cp.capture.set(name)
				return true
			}
		}
		// New name. Register on this scope!
		cp.thisScope().set(name)
		return true
	case "e", "E":
		// Special namespaces, do nothing
		return true
	default:
		return cp.registerModAccess(ns)
	}
}

func (cp *compiler) registerModAccess(name string) bool {
	if strings.ContainsRune(name, ':') {
		name = name[:strings.IndexByte(name, ':')]
	}
	return cp.registerVariableGet("", name+NsSuffix)
}
<commit_msg>Remove remains of number variables ($1, $2, ...)<commit_after>package eval

//go:generate ./boilerplate.py

import (
	"fmt"
	"strings"

	"github.com/elves/elvish/parse"
	"github.com/elves/elvish/util"
)

// compiler maintains the set of states needed when compiling a single source
// file.
type compiler struct {
	// Builtin namespace.
	builtin staticNs
	// Lexical namespaces.
	scopes []staticNs
	// Variables captured from outer scopes.
	capture staticNs
	// Position of what is being compiled.
	begin, end int
	// Information about the source.
	srcMeta *Source
}

func compile(b, g staticNs, n *parse.Chunk, src *Source) (op Op, err error) {
	cp := &compiler{b, []staticNs{g}, make(staticNs), 0, 0, src}
	defer util.Catch(&err)
	return cp.chunkOp(n), nil
}

func (cp *compiler) compiling(n parse.Node) {
	cp.begin, cp.end = n.Begin(), n.End()
}

func (cp *compiler) errorpf(begin, end int, format string, args ...interface{}) {
	throw(&CompilationError{fmt.Sprintf(format, args...),
		*util.NewSourceRange(cp.srcMeta.describePath(), cp.srcMeta.code, begin, end, nil)})
}

func (cp *compiler) errorf(format string, args ...interface{}) {
	cp.errorpf(cp.begin, cp.end, format, args...)
}

func (cp *compiler) thisScope() staticNs {
	return cp.scopes[len(cp.scopes)-1]
}

func (cp *compiler) pushScope() staticNs {
	sc := make(staticNs)
	cp.scopes = append(cp.scopes, sc)
	return sc
}

func (cp *compiler) popScope() {
	cp.scopes[len(cp.scopes)-1] = make(staticNs)
	cp.scopes = cp.scopes[:len(cp.scopes)-1]
}

func (cp *compiler) registerVariableGetQname(qname string) bool {
	_, ns, name := ParseVariableRef(qname)
	return cp.registerVariableGet(ns, name)
}

func (cp *compiler) registerVariableGet(ns, name string) bool {
	switch ns {
	case "", "local", "up":
		// Handled below
	case "e", "E":
		return true
	default:
		return cp.registerModAccess(ns)
	}
	// Find in local scope
	if ns == "" || ns == "local" {
		if cp.thisScope().has(name) {
			return true
		}
	}
	// Find in upper scopes
	if ns == "" || ns == "up" {
		for i := len(cp.scopes) - 2; i >= 0; i-- {
			if cp.scopes[i].has(name) {
				// Existing name: record capture and return.
				cp.capture.set(name)
				return true
			}
		}
	}
	// Find in builtin scope
	if ns == "" || ns == "builtin" {
		if cp.builtin.has(name) {
			return true
		}
	}
	return false
}

func (cp *compiler) registerVariableSetQname(qname string) bool {
	_, ns, name := ParseVariableRef(qname)
	return cp.registerVariableSet(ns, name)
}

func (cp *compiler) registerVariableSet(ns, name string) bool {
	switch ns {
	case "local":
		cp.thisScope().set(name)
		return true
	case "up":
		for i := len(cp.scopes) - 2; i >= 0; i-- {
			if cp.scopes[i].has(name) {
				// Existing name: record capture and return.
				cp.capture.set(name)
				return true
			}
		}
		return false
	case "builtin":
		cp.errorf("cannot set builtin variable")
		return false
	case "":
		if cp.thisScope().has(name) {
			// A name on current scope. Do nothing.
			return true
		}
		// Walk up the upper scopes
		for i := len(cp.scopes) - 2; i >= 0; i-- {
			if cp.scopes[i].has(name) {
				// Existing name. Do nothing
				cp.capture.set(name)
				return true
			}
		}
		// New name. Register on this scope!
		cp.thisScope().set(name)
		return true
	case "e", "E":
		// Special namespaces, do nothing
		return true
	default:
		return cp.registerModAccess(ns)
	}
}

func (cp *compiler) registerModAccess(name string) bool {
	if strings.ContainsRune(name, ':') {
		name = name[:strings.IndexByte(name, ':')]
	}
	return cp.registerVariableGet("", name+NsSuffix)
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"log"
	"os"
	"os/signal"
	"strings"
	"syscall"
	"time"

	"github.com/flynn/go-discoverd"
)

func main() {
	flag.Parse()
	name := flag.Arg(0)
	addr := flag.Arg(1)

	client, err := discoverd.NewClient()
	if err != nil {
		log.Fatal("Error making client:", err)
	}
	if err = client.Register(name, addr); err != nil {
		log.Fatal("Error registering:", err)
	}
	log.Printf("Registered %s at %s.\n", name, addr)

	exit := make(chan os.Signal, 1)
	signal.Notify(exit, os.Interrupt, syscall.SIGTERM)
	go func() {
		<-exit
		log.Println("Shutting down...")
		client.Unregister(name, addr)
		os.Exit(0)
	}()

	set, err := client.ServiceSet(name)
	if err != nil {
		log.Fatal("Error getting ServiceSet:", err)
	}
	for _ = range time.Tick(time.Second) {
		log.Println(strings.Join(set.Addrs(), ", "))
	}
}
<commit_msg>discoverd/client: Fix example<commit_after>package main

import (
	"flag"
	"log"
	"os"
	"os/signal"
	"strings"
	"syscall"
	"time"

	"github.com/flynn/go-discoverd"
)

func main() {
	flag.Parse()
	name := flag.Arg(0)
	addr := flag.Arg(1)

	client, err := discoverd.NewClient()
	if err != nil {
		log.Fatal("Error making client:", err)
	}
	if err = client.Register(name, addr); err != nil {
		log.Fatal("Error registering:", err)
	}
	log.Printf("Registered %s at %s.\n", name, addr)

	exit := make(chan os.Signal, 1)
	signal.Notify(exit, os.Interrupt, syscall.SIGTERM)
	go func() {
		<-exit
		log.Println("Shutting down...")
		client.Unregister(name, addr)
		os.Exit(0)
	}()

	set, err := client.NewServiceSet(name)
	if err != nil {
		log.Fatal("Error getting ServiceSet:", err)
	}
	for _ = range time.Tick(time.Second) {
		log.Println(strings.Join(set.Addrs(), ", "))
	}
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"fmt"
	"os"
	"os/user"
	"path/filepath"
	"regexp"
	"strconv"

	"github.com/codegangsta/cli"
	"github.com/shenwei356/cnote/cnotedb"
)

var (
	funcs  map[string]func(c *cli.Context)
	DBFILE string
	notedb *cnotedb.NoteDB
)

func init() {
	// DBFILE
	usr, err := user.Current()
	if err != nil {
		os.Exit(1)
	}
	DBFILE = filepath.Join(usr.HomeDir, ".cnote")

	funcs = make(map[string]func(c *cli.Context))
	funcs["new"] = funNew
	funcs["del"] = funDel
	funcs["use"] = funUse
	funcs["list"] = funLs

	funcs["add"] = funAdd
	funcs["rm"] = funRm

	funcs["tag"] = funTag
	funcs["search"] = funSearch

	funcs["dump"] = funDump
	funcs["wipe"] = funWipe
	funcs["restore"] = funRestore
	funcs["import"] = funImport

}

func getFunc(funcs map[string]func(c *cli.Context), name string) func(c *cli.Context) {
	if f, ok := funcs[name]; ok {
		return f
	} else {
		return func(c *cli.Context) {
			fmt.Printf("command %s not implemented\n", name)
		}
	}
}

func funLs(c *cli.Context) {
	if len(c.Args()) > 0 {
		fmt.Println("no arguments should be given.")
		return
	}

	for _, notename := range notedb.NotesList {

		// read note
		note, err := notedb.ReadNote(notename)
		if err != nil {
			fmt.Println(err)
			os.Exit(0)
		}

		fmt.Printf("note: %s\t(#. of items: %d, last update: %s).",
			notename, note.Sum, note.LastUpdate)
		if notedb.CurrentNote != nil &&
			notename == notedb.CurrentNote.NoteID {

			fmt.Printf(" (current note)")
		}
		fmt.Println()
	}
}

func funNew(c *cli.Context) {
	if len(c.Args()) == 0 {
		fmt.Println("note name needed.")
		return
	}
	if len(c.Args()) > 1 {
		fmt.Println("only one note name allowed.")
		return
	}

	notename := c.Args().First()

	err := notedb.NewNote(notename)
	if err != nil {
		fmt.Println(err)
		return
	}

	fmt.Printf("note \"%s\" created.\n", notename)
	fmt.Printf("current note: \"%s\".\n", notename)
}

func funDel(c *cli.Context) {
	if len(c.Args()) == 0 {
		fmt.Println("note name needed.")
		return
	}

	for _, notename := range c.Args() {
		err := notedb.DeleteNote(notename)
		if err != nil {
			fmt.Println(err)
			continue
		}

		// fmt.Printf("note \"%s\" deleted.\n", notename)
	}
}

func funUse(c *cli.Context) {
	if len(c.Args()) == 0 {
		fmt.Println("note name needed.")
		return
	}
	if len(c.Args()) > 1 {
		fmt.Println("only one note name allowed.")
		return
	}

	notename := c.Args().First()
	err := notedb.UseNote(notename)
	if err != nil {
		fmt.Println(err)
		return
	}

	fmt.Printf("current note: \"%s\" (last update: %s).\n",
		notename, notedb.CurrentNote.LastUpdate)
}

func funAdd(c *cli.Context) {
	if len(c.Args()) != 2 {
		fmt.Println("tag and content needed.")
		return
	}

	item, err := notedb.AddNoteItem(c.Args()[0], c.Args()[1])
	if err != nil {
		fmt.Println(err)
		return
	}

	fmt.Println(item)
}

func funRm(c *cli.Context) {
	if len(c.Args()) == 0 {
		fmt.Println("item ID needed.")
		return
	}

	for _, itemid := range c.Args() {

		itemid, err := strconv.Atoi(itemid)
		if err != nil {
			fmt.Println("item ID should be integer.")
			continue
		}

		err = notedb.RemoveNoteItem(notedb.CurrentNote, itemid)
		if err != nil {
			fmt.Println(err)
			continue
		}

		// fmt.Printf("note item \"%d\" deleted from note \"%s\".\n", itemid, notedb.Currentcnote.NoteID)
	}
}

func funTag(c *cli.Context) {
	// list all tags
	note, err := notedb.GetCurrentNote()
	if err != nil {
		fmt.Println(err)
		os.Exit(0)
	}

	if len(c.Args()) == 0 {
		for tag, taginfo := range note.Tags {
			fmt.Printf("tag: %s\t(#. of items: %d).\n", tag, len(taginfo))
		}
		return
	}

	items, err := notedb.ItemByTag(c.Args())
	if err != nil {
		fmt.Println(err)
		os.Exit(0)
	}

	for _, item := range items {
		fmt.Println(item)
	}
}

func funSearch(c *cli.Context) {
	if len(c.Args()) == 0 {
		fmt.Println("search keyword needed.")
		return
	}

	items, err := notedb.ItemByRegexp(c.Args())
	if err != nil {
		fmt.Println(err)
		os.Exit(0)
	}

	for _, item := range items {
		fmt.Println(item)
	}
}

func funDump(c *cli.Context) {
	if len(c.Args()) > 0 {
		fmt.Println("no arguments should be given.")
		return
	}

	err := notedb.Dump()
	if err != nil {
		fmt.Println(err)
		os.Exit(0)
	}
}

func request_reply(reply string) (bool, error) {
	fmt.Printf("Attention, it will clear all the data."+
		" type \"%s\" to continue:", reply)

	reader := bufio.NewReader(os.Stdin)
	str, err := reader.ReadString('\n')
	if err != nil {
		return false, err
	}

	str = regexp.MustCompile(`[\r\n]`).ReplaceAllString(str, "")
	str = regexp.MustCompile(`^\s+|\s+$`).ReplaceAllString(str, "")
	if str != "yes" {
		fmt.Println("\ngiven up.")
		return false, nil
	}
	return true, nil
}

func funWipe(c *cli.Context) {
	if len(c.Args()) > 0 {
		fmt.Println("no arguments should be given.")
		return
	}

	reply, err := request_reply("yes")
	if err != nil {
		fmt.Println(err)
		os.Exit(0)
	}

	if reply == false {
		return
	}

	err = notedb.Wipe()
	if err != nil {
		fmt.Println(err)
		os.Exit(0)
	}
}

func funRestore(c *cli.Context) {
	if len(c.Args()) != 1 {
		fmt.Println("dumpped filename needed.")
		return
	}

	reply, err := request_reply("yes")
	if err != nil {
		fmt.Println(err)
		os.Exit(0)
	}

	if reply == false {
		return
	}

	err = notedb.Restore(c.Args().First())
	if err != nil {
		fmt.Println(err)
		os.Exit(0)
	}
}

func funImport(c *cli.Context) {
	if len(c.Args()) != 3 {
		fmt.Println("three arguments needed: <notename in your cnote>" +
			" <notename in dumpped note> <dumpped filename>.")
		return
	}
	notename, othernotename, filename := c.Args()[0], c.Args()[1], c.Args()[2]
	n, err := notedb.Import(notename, othernotename, filename)
	if err != nil {
		fmt.Println(err)
		os.Exit(0)
	}
	fmt.Printf("%d items imported into note \"%s\".\n", n, notename)
}

func main() {
	notedb = cnotedb.NewNoteDB(DBFILE)

	app := cli.NewApp()
	app.Name = "cnote"
	app.Usage = "A command line note app. https://github.com/shenwei356/cnote"
	app.Version = "1.1 (2014-07-20)"
	app.Author = "Wei Shen"
	app.Email = "shenwei356@gmail.com"

	app.Commands = []cli.Command{
		{
			Name:   "new",
			Usage:  "Create a new note",
			Action: getFunc(funcs, "new"),
		},
		{
			Name:   "del",
			Usage:  "Delete a note",
			Action: getFunc(funcs, "del"),
		},
		{
			Name:   "use",
			Usage:  "Select a note",
			Action: getFunc(funcs, "use"),
		},
		{
			Name:      "list",
			ShortName: "ls",
			Usage:     "List all notes",
			Action:    getFunc(funcs, "list"),
		},
		{
			Name:   "add",
			Usage:  "add a note item",
			Action: getFunc(funcs, "add"),
		},
		{
			Name:   "rm",
			Usage:  "Remove a note item",
			Action: getFunc(funcs, "rm"),
		},
		{
			Name:      "tag",
			ShortName: "t",
			Usage:     "List items by tags",
			Action:    getFunc(funcs, "tag"),
		},
		{
			Name:      "search",
			ShortName: "s",
			Usage:     "Search for",
			Action:    getFunc(funcs, "search"),
		},
		{
			Name:   "dump",
			Usage:  "Dump whole database, for backup or transfer",
			Action: getFunc(funcs, "dump"),
		},
		{
			Name:   "wipe",
			Usage:  "Attention! Wipe whole database",
			Action: getFunc(funcs, "wipe"),
		},
		{
			Name:   "restore",
			Usage:  "Wipe whole database, and restore from dumpped file",
			Action: getFunc(funcs, "restore"),
		},
		{
			Name:   "import",
			Usage:  "Import note items from dumpped data",
			Action: getFunc(funcs, "import"),
		},
	}

	app.Run(os.Args)

	notedb.Close()
}
<commit_msg>add defer to db.Close()<commit_after>package main

import (
	"bufio"
	"fmt"
	"os"
	"os/user"
	"path/filepath"
	"regexp"
	"strconv"

	"github.com/codegangsta/cli"
	"github.com/shenwei356/cnote/cnotedb"
)

var (
	funcs  map[string]func(c *cli.Context)
	DBFILE string
	notedb *cnotedb.NoteDB
)

func init() {
	// DBFILE
	usr, err := user.Current()
	if err != nil {
		os.Exit(1)
	}
	DBFILE = filepath.Join(usr.HomeDir, ".cnote")

	funcs = make(map[string]func(c *cli.Context))
	funcs["new"] = funNew
	funcs["del"] = funDel
	funcs["use"] = funUse
	funcs["list"] = funLs

	funcs["add"] = funAdd
	funcs["rm"] = funRm

	funcs["tag"] = funTag
	funcs["search"] = funSearch

	funcs["dump"] = funDump
	funcs["wipe"] = funWipe
	funcs["restore"] = funRestore
	funcs["import"] = funImport

}

func getFunc(funcs map[string]func(c *cli.Context), name string) func(c *cli.Context) {
	if f, ok := funcs[name]; ok {
		return f
	} else {
		return func(c *cli.Context) {
			fmt.Printf("command %s not implemented\n", name)
		}
	}
}

func funLs(c *cli.Context) {
	if len(c.Args()) > 0 {
		fmt.Println("no arguments should be given.")
		return
	}

	for _, notename := range notedb.NotesList {

		// read note
		note, err := notedb.ReadNote(notename)
		if err != nil {
			fmt.Println(err)
			os.Exit(0)
		}

		fmt.Printf("note: %s\t(#. of items: %d, last update: %s).",
			notename, note.Sum, note.LastUpdate)
		if notedb.CurrentNote != nil &&
			notename == notedb.CurrentNote.NoteID {

			fmt.Printf(" (current note)")
		}
		fmt.Println()
	}
}

func funNew(c *cli.Context) {
	if len(c.Args()) == 0 {
		fmt.Println("note name needed.")
		return
	}
	if len(c.Args()) > 1 {
		fmt.Println("only one note name allowed.")
		return
	}

	notename := c.Args().First()

	err := notedb.NewNote(notename)
	if err != nil {
		fmt.Println(err)
		return
	}

	fmt.Printf("note \"%s\" created.\n", notename)
	fmt.Printf("current note: \"%s\".\n", notename)
}

func funDel(c *cli.Context) {
	if len(c.Args()) == 0 {
		fmt.Println("note name needed.")
		return
	}

	for _, notename := range c.Args() {
		err := notedb.DeleteNote(notename)
		if err != nil {
			fmt.Println(err)
			continue
		}

		// fmt.Printf("note \"%s\" deleted.\n", notename)
	}
}

func funUse(c *cli.Context) {
	if len(c.Args()) == 0 {
		fmt.Println("note name needed.")
		return
	}
	if len(c.Args()) > 1 {
		fmt.Println("only one note name allowed.")
		return
	}

	notename := c.Args().First()
	err := notedb.UseNote(notename)
	if err != nil {
		fmt.Println(err)
		return
	}

	fmt.Printf("current note: \"%s\" (last update: %s).\n",
		notename, notedb.CurrentNote.LastUpdate)
}

func funAdd(c *cli.Context) {
	if len(c.Args()) != 2 {
		fmt.Println("tag and content needed.")
		return
	}

	item, err := notedb.AddNoteItem(c.Args()[0], c.Args()[1])
	if err != nil {
		fmt.Println(err)
		return
	}

	fmt.Println(item)
}

func funRm(c *cli.Context) {
	if len(c.Args()) == 0 {
		fmt.Println("item ID needed.")
		return
	}

	for _, itemid := range c.Args() {

		itemid, err := strconv.Atoi(itemid)
		if err != nil {
			fmt.Println("item ID should be integer.")
			continue
		}

		err = notedb.RemoveNoteItem(notedb.CurrentNote, itemid)
		if err != nil {
			fmt.Println(err)
			continue
		}

		// fmt.Printf("note item \"%d\" deleted from note \"%s\".\n", itemid, notedb.Currentcnote.NoteID)
	}
}

func funTag(c *cli.Context) {
	// list all tags
	note, err := notedb.GetCurrentNote()
	if err != nil {
		fmt.Println(err)
		os.Exit(0)
	}

	if len(c.Args()) == 0 {
		for tag, taginfo := range note.Tags {
			fmt.Printf("tag: %s\t(#. of items: %d).\n", tag, len(taginfo))
		}
		return
	}

	items, err := notedb.ItemByTag(c.Args())
	if err != nil {
		fmt.Println(err)
		os.Exit(0)
	}

	for _, item := range items {
		fmt.Println(item)
	}
}

func funSearch(c *cli.Context) {
	if len(c.Args()) == 0 {
		fmt.Println("search keyword needed.")
		return
	}

	items, err := notedb.ItemByRegexp(c.Args())
	if err != nil {
		fmt.Println(err)
		os.Exit(0)
	}

	for _, item := range items {
		fmt.Println(item)
	}
}

func funDump(c *cli.Context) {
	if len(c.Args()) > 0 {
		fmt.Println("no arguments should be given.")
		return
	}

	err := notedb.Dump()
	if err != nil {
		fmt.Println(err)
		os.Exit(0)
	}
}

func request_reply(reply string) (bool, error) {
	fmt.Printf("Attention, it will clear all the data."+
		" type \"%s\" to continue:", reply)

	reader := bufio.NewReader(os.Stdin)
	str, err := reader.ReadString('\n')
	if err != nil {
		return false, err
	}

	str = regexp.MustCompile(`[\r\n]`).ReplaceAllString(str, "")
	str = regexp.MustCompile(`^\s+|\s+$`).ReplaceAllString(str, "")
	if str != "yes" {
		fmt.Println("\ngiven up.")
		return false, nil
	}
	return true, nil
}

func funWipe(c *cli.Context) {
	if len(c.Args()) > 0 {
		fmt.Println("no arguments should be given.")
		return
	}

	reply, err := request_reply("yes")
	if err != nil {
		fmt.Println(err)
		os.Exit(0)
	}

	if reply == false {
		return
	}

	err = notedb.Wipe()
	if err != nil {
		fmt.Println(err)
		os.Exit(0)
	}
}

func funRestore(c *cli.Context) {
	if len(c.Args()) != 1 {
		fmt.Println("dumpped filename needed.")
		return
	}

	reply, err := request_reply("yes")
	if err != nil {
		fmt.Println(err)
		os.Exit(0)
	}

	if reply == false {
		return
	}

	err = notedb.Restore(c.Args().First())
	if err != nil {
		fmt.Println(err)
		os.Exit(0)
	}
}

func funImport(c *cli.Context) {
	if len(c.Args()) != 3 {
		fmt.Println("three arguments needed: <notename in your cnote>" +
			" <notename in dumpped note> <dumpped filename>.")
		return
	}
	notename, othernotename, filename := c.Args()[0], c.Args()[1], c.Args()[2]
	n, err := notedb.Import(notename, othernotename, filename)
	if err != nil {
		fmt.Println(err)
		os.Exit(0)
	}
	fmt.Printf("%d items imported into note \"%s\".\n", n, notename)
}

func main() {
	notedb = cnotedb.NewNoteDB(DBFILE)
	defer notedb.Close()

	app := cli.NewApp()
	app.Name = "cnote"
	app.Usage = "A command line note app. https://github.com/shenwei356/cnote"
	app.Version = "1.1 (2014-07-20)"
	app.Author = "Wei Shen"
	app.Email = "shenwei356@gmail.com"

	app.Commands = []cli.Command{
		{
			Name:   "new",
			Usage:  "Create a new note",
			Action: getFunc(funcs, "new"),
		},
		{
			Name:   "del",
			Usage:  "Delete a note",
			Action: getFunc(funcs, "del"),
		},
		{
			Name:   "use",
			Usage:  "Select a note",
			Action: getFunc(funcs, "use"),
		},
		{
			Name:      "list",
			ShortName: "ls",
			Usage:     "List all notes",
			Action:    getFunc(funcs, "list"),
		},
		{
			Name:   "add",
			Usage:  "add a note item",
			Action: getFunc(funcs, "add"),
		},
		{
			Name:   "rm",
			Usage:  "Remove a note item",
			Action: getFunc(funcs, "rm"),
		},
		{
			Name:      "tag",
			ShortName: "t",
			Usage:     "List items by tags",
			Action:    getFunc(funcs, "tag"),
		},
		{
			Name:      "search",
			ShortName: "s",
			Usage:     "Search for",
			Action:    getFunc(funcs, "search"),
		},
		{
			Name:   "dump",
			Usage:  "Dump whole database, for backup or transfer",
			Action: getFunc(funcs, "dump"),
		},
		{
			Name:   "wipe",
			Usage:  "Attention! Wipe whole database",
			Action: getFunc(funcs, "wipe"),
		},
		{
			Name:   "restore",
			Usage:  "Wipe whole database, and restore from dumpped file",
			Action: getFunc(funcs, "restore"),
		},
		{
			Name:   "import",
			Usage:  "Import note items from dumpped data",
			Action: getFunc(funcs, "import"),
		},
	}

	app.Run(os.Args)
}
<|endoftext|>