<commit_before>package main

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"log"
	"net/url"
	"os"
	"os/exec"
	"path"
	"path/filepath"
	"strings"

	"golang.org/x/net/context"
)

type (
	CommandConfig struct {
		Template []string            `json:"-"`
		Options  map[string][]string `json:"options,omitempty"`
		Dryrun   bool                `json:"dryrun,omitempty"`
	}

	Job struct {
		config *CommandConfig
		// https://godoc.org/google.golang.org/genproto/googleapis/pubsub/v1#ReceivedMessage
		message      *JobMessage
		notification *ProgressNotification
		storage      Storage

		// These are set at at setupWorkspace
		workspace     string
		downloads_dir string
		uploads_dir   string

		// These are set at setupDownloadFiles
		downloadFileMap     map[string]string
		remoteDownloadFiles interface{}
		localDownloadFiles  interface{}
	}
)

func (job *Job) run(ctx context.Context) error {
	verr := job.message.Validate()
	if verr != nil {
		log.Printf("Invalid Message: MessageId: %v, Message: %v, error: %v\n", job.message.MessageId(), job.message.raw.Message, verr)
		err := job.withNotify(CANCELLING, job.message.Ack)()
		if err != nil {
			return err
		}
		return nil
	}

	go job.message.sendMADPeriodically()
	defer job.message.Done()

	job.notification.notify(PROCESSING, job.message.MessageId(), "info")

	err := job.setupWorkspace()
	if err != nil {
		return err
	}

	defer job.clearWorkspace()

	err = job.withNotify(PREPARING, job.setupDownloadFiles)()
	if err != nil {
		return err
	}

	err = job.withNotify(DOWNLOADING, job.downloadFiles)()
	if err != nil {
		return err
	}

	err = job.withNotify(EXECUTING, job.execute)()
	if err != nil {
		return err
	}

	err = job.withNotify(UPLOADING, job.uploadFiles)()
	if err != nil {
		return err
	}

	err = job.withNotify(ACKSENDING, job.message.Ack)()
	if err != nil {
		return err
	}

	job.notification.notify(CLEANUP, job.message.MessageId(), "info")
	return err
}

func (job *Job) withNotify(progress int, f func() error) func() error {
	msg_id := job.message.MessageId()
	return func() error {
		job.notification.notify(progress, msg_id, "info")
		err := f()
		if err != nil {
			job.notification.notify(progress+2, msg_id, "error")
			return err
		}
		job.notification.notify(progress+1, msg_id, "info")
		return nil
	}
}

func (job *Job) setupWorkspace() error {
	dir, err := ioutil.TempDir("", "workspace")
	if err != nil {
		log.Fatal(err)
		return err
	}

	subdirs := []string{
		filepath.Join(dir, "downloads"),
		filepath.Join(dir, "uploads"),
	}
	for _, subdir := range subdirs {
		err := os.MkdirAll(subdir, 0700)
		if err != nil {
			return err
		}
	}
	job.workspace = dir
	job.downloads_dir = subdirs[0]
	job.uploads_dir = subdirs[1]
	return nil
}

func (job *Job) clearWorkspace() error {
	return os.RemoveAll(job.workspace)
}

func (job *Job) setupDownloadFiles() error {
	job.downloadFileMap = map[string]string{}
	job.remoteDownloadFiles = job.message.DownloadFiles()
	objects := job.flatten(job.remoteDownloadFiles)
	remoteUrls := []string{}
	for _, obj := range objects {
		switch obj.(type) {
		case string:
			remoteUrls = append(remoteUrls, obj.(string))
		default:
			log.Printf("Invalid download file URL: %v [%T]", obj, obj)
		}
	}
	for _, remote_url := range remoteUrls {
		url, err := url.Parse(remote_url)
		if err != nil {
			log.Fatalf("Invalid URL: %v because of %v\n", remote_url, err)
			return err
		}
		urlstr := fmt.Sprintf("gs://%v%v", url.Host, url.Path)
		destPath := filepath.Join(job.downloads_dir, url.Host, url.Path)
		job.downloadFileMap[urlstr] = destPath
	}
	job.localDownloadFiles = job.copyWithFileMap(job.remoteDownloadFiles)
	return nil
}

func (job *Job) copyWithFileMap(obj interface{}) interface{} {
	switch obj.(type) {
	case map[string]interface{}:
		result := map[string]interface{}{}
		for k, v := range obj.(map[string]interface{}) {
			result[k] = job.copyWithFileMap(v)
		}
		return result
	case []interface{}:
		result := []interface{}{}
		for _, v := range obj.([]interface{}) {
			result = append(result, job.copyWithFileMap(v))
		}
		return result
	case string:
		return job.downloadFileMap[obj.(string)]
	default:
		return obj
	}
}

func (job *Job) buildVariable() *Variable {
	return &Variable{
		data: map[string]interface{}{
			"workspace":             job.workspace,
			"downloads_dir":         job.downloads_dir,
			"uploads_dir":           job.uploads_dir,
			"download_files":        job.localDownloadFiles,
			"local_download_files":  job.localDownloadFiles,
			"remote_download_files": job.remoteDownloadFiles,
			"attrs":                 job.message.raw.Message.Attributes,
			"attributes":            job.message.raw.Message.Attributes,
			"data":                  job.message.raw.Message.Data,
		},
	}
}

func (job *Job) build() (*exec.Cmd, error) {
	v := job.buildVariable()

	values, err := job.extract(v, job.config.Template)
	if err != nil {
		return nil, err
	}
	if len(job.config.Options) > 0 {
		key := strings.Join(values, " ")
		t := job.config.Options[key]
		if t == nil {
			t = job.config.Options["default"]
		}
		if t != nil {
			values, err = job.extract(v, t)
			if err != nil {
				return nil, err
			}
		}
	}
	cmd := exec.Command(values[0], values[1:]...)
	return cmd, nil
}

func (job *Job) extract(v *Variable, values []string) ([]string, error) {
	result := []string{}
	for _, src := range values {
		extracted, err := v.expand(src)
		if err != nil {
			return nil, err
		}
		vals := strings.Split(extracted, v.separator)
		for _, val := range vals {
			result = append(result, val)
		}
	}
	return result, nil
}

func (job *Job) downloadFiles() error {
	for remoteURL, destPath := range job.downloadFileMap {
		url, err := url.Parse(remoteURL)
		if err != nil {
			log.Fatalf("Invalid URL: %v because of %v\n", remoteURL, err)
			return err
		}

		dir := path.Dir(destPath)
		err = os.MkdirAll(dir, 0700)
		if err != nil {
			return err
		}

		err = job.storage.Download(url.Host, url.Path[1:], destPath)
		if err != nil {
			return err
		}
	}
	return nil
}

func (job *Job) execute() error {
	cmd, err := job.build()
	if err != nil {
		log.Fatalf("Command build Error template: %v msg: %v cause of %v\n", job.config.Template, job.message, err)
		return err
	}
	var out bytes.Buffer
	cmd.Stdout = &out
	cmd.Stderr = &out
	log.Printf("EXECUTE running: %v\n", cmd)
	err = cmd.Run()
	if err != nil {
		log.Printf("Command Error: cmd: %v cause of %v\n%v\n", cmd, err, out.String())
		return err
	}
	return nil
}

func (job *Job) uploadFiles() error {
	localPaths, err := job.listFiles(job.uploads_dir)
	if err != nil {
		return err
	}
	for _, localPath := range localPaths {
		relPath, err := filepath.Rel(job.uploads_dir, localPath)
		if err != nil {
			log.Fatalf("Error getting relative path of %v: %v\n", localPath, err)
			return err
		}
		sep := string([]rune{os.PathSeparator})
		parts := strings.Split(relPath, sep)
		bucket := parts[0]
		object := strings.Join(parts[1:], sep)
		err = job.storage.Upload(bucket, object, localPath)
		if err != nil {
			log.Fatalf("Error uploading %v to gs://%v/%v: %v\n", localPath, bucket, object, err)
			return err
		}
	}
	return nil
}

func (job *Job) listFiles(dir string) ([]string, error) {
	result := []string{}
	err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
		if !info.IsDir() {
			result = append(result, path)
		}
		return nil
	})
	if err != nil {
		log.Fatalf("Error listing upload files: %v\n", err)
		return nil, err
	}
	return result, nil
}

func (job *Job) flatten(obj interface{}) []interface{} {
	// Support only unmarshalled object from JSON
	// See https://golang.org/pkg/encoding/json/#Unmarshal also
	switch obj.(type) {
	case []interface{}:
		res := []interface{}{}
		for _, i := range obj.([]interface{}) {
			switch i.(type) {
			case bool, float64, string, nil:
				res = append(res, i)
			default:
				for _, j := range job.flatten(i) {
					res = append(res, j)
				}
			}
		}
		return res
	case map[string]interface{}:
		values := []interface{}{}
		for _, val := range obj.(map[string]interface{}) {
			values = append(values, val)
		}
		return job.flatten(values)
	default:
		return []interface{}{obj}
	}
}
<commit_msg>:recycle: Don't return *exec.Cmd from build<commit_after>package main

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"log"
	"net/url"
	"os"
	"os/exec"
	"path"
	"path/filepath"
	"strings"

	"golang.org/x/net/context"
)

type (
	CommandConfig struct {
		Template []string            `json:"-"`
		Options  map[string][]string `json:"options,omitempty"`
		Dryrun   bool                `json:"dryrun,omitempty"`
	}

	Job struct {
		config *CommandConfig
		// https://godoc.org/google.golang.org/genproto/googleapis/pubsub/v1#ReceivedMessage
		message      *JobMessage
		notification *ProgressNotification
		storage      Storage

		// These are set at at setupWorkspace
		workspace     string
		downloads_dir string
		uploads_dir   string

		// These are set at setupDownloadFiles
		downloadFileMap     map[string]string
		remoteDownloadFiles interface{}
		localDownloadFiles  interface{}

		cmd *exec.Cmd
	}
)

func (job *Job) run(ctx context.Context) error {
	verr := job.message.Validate()
	if verr != nil {
		log.Printf("Invalid Message: MessageId: %v, Message: %v, error: %v\n", job.message.MessageId(), job.message.raw.Message, verr)
		err := job.withNotify(CANCELLING, job.message.Ack)()
		if err != nil {
			return err
		}
		return nil
	}

	go job.message.sendMADPeriodically()
	defer job.message.Done()

	job.notification.notify(PROCESSING, job.message.MessageId(), "info")

	err := job.setupWorkspace()
	if err != nil {
		return err
	}

	defer job.clearWorkspace()

	err = job.withNotify(PREPARING, job.setupDownloadFiles)()
	if err != nil {
		return err
	}

	err = job.withNotify(DOWNLOADING, job.downloadFiles)()
	if err != nil {
		return err
	}

	err = job.withNotify(EXECUTING, job.execute)()
	if err != nil {
		return err
	}

	err = job.withNotify(UPLOADING, job.uploadFiles)()
	if err != nil {
		return err
	}

	err = job.withNotify(ACKSENDING, job.message.Ack)()
	if err != nil {
		return err
	}

	job.notification.notify(CLEANUP, job.message.MessageId(), "info")
	return err
}

func (job *Job) withNotify(progress int, f func() error) func() error {
	msg_id := job.message.MessageId()
	return func() error {
		job.notification.notify(progress, msg_id, "info")
		err := f()
		if err != nil {
			job.notification.notify(progress+2, msg_id, "error")
			return err
		}
		job.notification.notify(progress+1, msg_id, "info")
		return nil
	}
}

func (job *Job) setupWorkspace() error {
	dir, err := ioutil.TempDir("", "workspace")
	if err != nil {
		log.Fatal(err)
		return err
	}

	subdirs := []string{
		filepath.Join(dir, "downloads"),
		filepath.Join(dir, "uploads"),
	}
	for _, subdir := range subdirs {
		err := os.MkdirAll(subdir, 0700)
		if err != nil {
			return err
		}
	}
	job.workspace = dir
	job.downloads_dir = subdirs[0]
	job.uploads_dir = subdirs[1]
	return nil
}

func (job *Job) clearWorkspace() error {
	return os.RemoveAll(job.workspace)
}

func (job *Job) setupDownloadFiles() error {
	job.downloadFileMap = map[string]string{}
	job.remoteDownloadFiles = job.message.DownloadFiles()
	objects := job.flatten(job.remoteDownloadFiles)
	remoteUrls := []string{}
	for _, obj := range objects {
		switch obj.(type) {
		case string:
			remoteUrls = append(remoteUrls, obj.(string))
		default:
			log.Printf("Invalid download file URL: %v [%T]", obj, obj)
		}
	}
	for _, remote_url := range remoteUrls {
		url, err := url.Parse(remote_url)
		if err != nil {
			log.Fatalf("Invalid URL: %v because of %v\n", remote_url, err)
			return err
		}
		urlstr := fmt.Sprintf("gs://%v%v", url.Host, url.Path)
		destPath := filepath.Join(job.downloads_dir, url.Host, url.Path)
		job.downloadFileMap[urlstr] = destPath
	}
	job.localDownloadFiles = job.copyWithFileMap(job.remoteDownloadFiles)
	return nil
}

func (job *Job) copyWithFileMap(obj interface{}) interface{} {
	switch obj.(type) {
	case map[string]interface{}:
		result := map[string]interface{}{}
		for k, v := range obj.(map[string]interface{}) {
			result[k] = job.copyWithFileMap(v)
		}
		return result
	case []interface{}:
		result := []interface{}{}
		for _, v := range obj.([]interface{}) {
			result = append(result, job.copyWithFileMap(v))
		}
		return result
	case string:
		return job.downloadFileMap[obj.(string)]
	default:
		return obj
	}
}

func (job *Job) buildVariable() *Variable {
	return &Variable{
		data: map[string]interface{}{
			"workspace":             job.workspace,
			"downloads_dir":         job.downloads_dir,
			"uploads_dir":           job.uploads_dir,
			"download_files":        job.localDownloadFiles,
			"local_download_files":  job.localDownloadFiles,
			"remote_download_files": job.remoteDownloadFiles,
			"attrs":                 job.message.raw.Message.Attributes,
			"attributes":            job.message.raw.Message.Attributes,
			"data":                  job.message.raw.Message.Data,
		},
	}
}

func (job *Job) build() error {
	v := job.buildVariable()

	values, err := job.extract(v, job.config.Template)
	if err != nil {
		return err
	}
	if len(job.config.Options) > 0 {
		key := strings.Join(values, " ")
		t := job.config.Options[key]
		if t == nil {
			t = job.config.Options["default"]
		}
		if t != nil {
			values, err = job.extract(v, t)
			if err != nil {
				return err
			}
		}
	}
	job.cmd = exec.Command(values[0], values[1:]...)
	return nil
}

func (job *Job) extract(v *Variable, values []string) ([]string, error) {
	result := []string{}
	for _, src := range values {
		extracted, err := v.expand(src)
		if err != nil {
			return nil, err
		}
		vals := strings.Split(extracted, v.separator)
		for _, val := range vals {
			result = append(result, val)
		}
	}
	return result, nil
}

func (job *Job) downloadFiles() error {
	for remoteURL, destPath := range job.downloadFileMap {
		url, err := url.Parse(remoteURL)
		if err != nil {
			log.Fatalf("Invalid URL: %v because of %v\n", remoteURL, err)
			return err
		}

		dir := path.Dir(destPath)
		err = os.MkdirAll(dir, 0700)
		if err != nil {
			return err
		}

		err = job.storage.Download(url.Host, url.Path[1:], destPath)
		if err != nil {
			return err
		}
	}
	return nil
}

func (job *Job) execute() error {
	err := job.build()
	if err != nil {
		log.Fatalf("Command build Error template: %v msg: %v cause of %v\n", job.config.Template, job.message, err)
		return err
	}
	var out bytes.Buffer
	job.cmd.Stdout = &out
	job.cmd.Stderr = &out
	log.Printf("EXECUTE running: %v\n", job.cmd)
	err = job.cmd.Run()
	if err != nil {
		log.Printf("Command Error: cmd: %v cause of %v\n%v\n", job.cmd, err, out.String())
		return err
	}
	return nil
}

func (job *Job) uploadFiles() error {
	localPaths, err := job.listFiles(job.uploads_dir)
	if err != nil {
		return err
	}
	for _, localPath := range localPaths {
		relPath, err := filepath.Rel(job.uploads_dir, localPath)
		if err != nil {
			log.Fatalf("Error getting relative path of %v: %v\n", localPath, err)
			return err
		}
		sep := string([]rune{os.PathSeparator})
		parts := strings.Split(relPath, sep)
		bucket := parts[0]
		object := strings.Join(parts[1:], sep)
		err = job.storage.Upload(bucket, object, localPath)
		if err != nil {
			log.Fatalf("Error uploading %v to gs://%v/%v: %v\n", localPath, bucket, object, err)
			return err
		}
	}
	return nil
}

func (job *Job) listFiles(dir string) ([]string, error) {
	result := []string{}
	err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
		if !info.IsDir() {
			result = append(result, path)
		}
		return nil
	})
	if err != nil {
		log.Fatalf("Error listing upload files: %v\n", err)
		return nil, err
	}
	return result, nil
}

func (job *Job) flatten(obj interface{}) []interface{} {
	// Support only unmarshalled object from JSON
	// See https://golang.org/pkg/encoding/json/#Unmarshal also
	switch obj.(type) {
	case []interface{}:
		res := []interface{}{}
		for _, i := range obj.([]interface{}) {
			switch i.(type) {
			case bool, float64, string, nil:
				res = append(res, i)
			default:
				for _, j := range job.flatten(i) {
					res = append(res, j)
				}
			}
		}
		return res
	case map[string]interface{}:
		values := []interface{}{}
		for _, val := range obj.(map[string]interface{}) {
			values = append(values, val)
		}
		return job.flatten(values)
	default:
		return []interface{}{obj}
	}
}
<|endoftext|>


<commit_before>// Copyright 2015 Keybase, Inc. All rights reserved. Use of
// this source code is governed by the included BSD license.

package saltpack

import (
	"crypto/hmac"
)

// RawBoxKey is the raw byte-representation of what a box key should
// look like --- a static 32-byte buffer
type RawBoxKey [32]byte

// SymmetricKey is a template for a symmetric key, a 32-byte static
// buffer.  Used for both NaCl SecretBox.
type SymmetricKey [32]byte

type KIDExtractor interface {
	// ToKID outputs the "key ID" that corresponds to this BoxPublicKey.
	// You can do whatever you'd like here, but probably it makes sense just
	// to output the public key as is.
	ToKID() []byte
}

// BoxPublicKey is an generic interface to NaCl's public key Box function.
type BoxPublicKey interface {
	KIDExtractor

	// ToRawBoxKeyPointer returns this public key as a *[32]byte,
	// for use with nacl.box.Seal
	ToRawBoxKeyPointer() *RawBoxKey

	// CreateEmphemeralKey creates an ephemeral key of the same type,
	// but totally random.
	CreateEphemeralKey() (BoxSecretKey, error)

	// HideIdentity returns true if we should hide the identity of this
	// key in our output message format.
	HideIdentity() bool
}

// BoxPrecomputedSharedKey results from a Precomputation below.
type BoxPrecomputedSharedKey interface {
	Unbox(nonce *Nonce, msg []byte) ([]byte, error)
	Box(nonce *Nonce, msg []byte) ([]byte, error)
}

// BoxSecretKey is the secret key corresponding to a BoxPublicKey
type BoxSecretKey interface {

	// Box boxes up data, sent from this secret key, and to the receiver
	// specified.
	Box(receiver BoxPublicKey, nonce *Nonce, msg []byte) ([]byte, error)

	// Unobx opens up the box, using this secret key as the receiver key
	// abd the give public key as the sender key.
	Unbox(sender BoxPublicKey, nonce *Nonce, msg []byte) ([]byte, error)

	// GetPublicKey gets the public key associated with this secret key.
	GetPublicKey() BoxPublicKey

	// Precompute computes a DH with the given key
	Precompute(sender BoxPublicKey) BoxPrecomputedSharedKey
}

// SigningSecretKey is a secret NaCl key that can sign messages.
type SigningSecretKey interface {
	// Sign signs message with this secret key.
	Sign(message []byte) ([]byte, error)

	// PublicKey gets the public key associated with this secret key.
	PublicKey() SigningPublicKey
}

// SigningPublicKey is a public NaCl key that can verify
// signatures.
type SigningPublicKey interface {
	KIDExtractor

	// Verify verifies that signature is a valid signature of message for
	// this public key.
	Verify(message []byte, signature []byte) error
}

// Keyring is an interface used with decryption; it is called to
// recover public or private keys during the decryption process.
// Calls can block on network action.
type Keyring interface {
	// LookupBoxSecretKey looks in the Keyring for the secret key corresponding
	// to one of the given Key IDs.  Returns the index and the key on success,
	// or -1 and nil on failure.
	LookupBoxSecretKey(kids [][]byte) (int, BoxSecretKey)

	// LookupBoxPublicKey returns a public key given the specified key ID.
	// For most cases, the key ID will be the key itself.
	LookupBoxPublicKey(kid []byte) BoxPublicKey

	// GetAllSecretKeys returns all keys, needed if we want to support
	// "hidden" receivers via trial and error
	GetAllSecretKeys() []BoxSecretKey

	// ImportEphemeralKey imports the ephemeral key into
	// BoxPublicKey format. This key has never been seen before, so
	// will be ephemeral.
	ImportEphemeralKey(kid []byte) BoxPublicKey
}

// SigKeyring is an interface used during verification to find
// the public key for the signer of a message.
type SigKeyring interface {
	// LookupSigningPublicKey returns a public signing key for the specified key ID.
	LookupSigningPublicKey(kid []byte) SigningPublicKey
}

// SecretKeyEqual returns true if the two secret keys are equal.
func SecretKeyEqual(sk1, sk2 BoxSecretKey) bool {
	return PublicKeyEqual(sk1.GetPublicKey(), sk2.GetPublicKey())
}

// PublicKeyEqual returns true if the two public keys are equal.
func PublicKeyEqual(pk1, pk2 BoxPublicKey) bool {
	return KIDEqual(pk1, pk2)
}

// KIDEqual return true if the KIDs for two keys are equal.
func KIDEqual(k1, k2 KIDExtractor) bool {
	return hmac.Equal(k1.ToKID(), k2.ToKID())
}
<commit_msg>Add comments on KIDExtractor<commit_after>// Copyright 2015 Keybase, Inc. All rights reserved. Use of
// this source code is governed by the included BSD license.

package saltpack

import (
	"crypto/hmac"
)

// RawBoxKey is the raw byte-representation of what a box key should
// look like --- a static 32-byte buffer
type RawBoxKey [32]byte

// SymmetricKey is a template for a symmetric key, a 32-byte static
// buffer.  Used for both NaCl SecretBox.
type SymmetricKey [32]byte

// KIDExtractor key types can output a key ID corresponding to the
// key.
type KIDExtractor interface {
	// ToKID outputs the "key ID" that corresponds to this key.
	// You can do whatever you'd like here, but probably it makes sense just
	// to output the public key as is.
	ToKID() []byte
}

// BoxPublicKey is an generic interface to NaCl's public key Box function.
type BoxPublicKey interface {
	KIDExtractor

	// ToRawBoxKeyPointer returns this public key as a *[32]byte,
	// for use with nacl.box.Seal
	ToRawBoxKeyPointer() *RawBoxKey

	// CreateEmphemeralKey creates an ephemeral key of the same type,
	// but totally random.
	CreateEphemeralKey() (BoxSecretKey, error)

	// HideIdentity returns true if we should hide the identity of this
	// key in our output message format.
	HideIdentity() bool
}

// BoxPrecomputedSharedKey results from a Precomputation below.
type BoxPrecomputedSharedKey interface {
	Unbox(nonce *Nonce, msg []byte) ([]byte, error)
	Box(nonce *Nonce, msg []byte) ([]byte, error)
}

// BoxSecretKey is the secret key corresponding to a BoxPublicKey
type BoxSecretKey interface {

	// Box boxes up data, sent from this secret key, and to the receiver
	// specified.
	Box(receiver BoxPublicKey, nonce *Nonce, msg []byte) ([]byte, error)

	// Unobx opens up the box, using this secret key as the receiver key
	// abd the give public key as the sender key.
	Unbox(sender BoxPublicKey, nonce *Nonce, msg []byte) ([]byte, error)

	// GetPublicKey gets the public key associated with this secret key.
	GetPublicKey() BoxPublicKey

	// Precompute computes a DH with the given key
	Precompute(sender BoxPublicKey) BoxPrecomputedSharedKey
}

// SigningSecretKey is a secret NaCl key that can sign messages.
type SigningSecretKey interface {
	// Sign signs message with this secret key.
	Sign(message []byte) ([]byte, error)

	// PublicKey gets the public key associated with this secret key.
	PublicKey() SigningPublicKey
}

// SigningPublicKey is a public NaCl key that can verify
// signatures.
type SigningPublicKey interface {
	KIDExtractor

	// Verify verifies that signature is a valid signature of message for
	// this public key.
	Verify(message []byte, signature []byte) error
}

// Keyring is an interface used with decryption; it is called to
// recover public or private keys during the decryption process.
// Calls can block on network action.
type Keyring interface {
	// LookupBoxSecretKey looks in the Keyring for the secret key corresponding
	// to one of the given Key IDs.  Returns the index and the key on success,
	// or -1 and nil on failure.
	LookupBoxSecretKey(kids [][]byte) (int, BoxSecretKey)

	// LookupBoxPublicKey returns a public key given the specified key ID.
	// For most cases, the key ID will be the key itself.
	LookupBoxPublicKey(kid []byte) BoxPublicKey

	// GetAllSecretKeys returns all keys, needed if we want to support
	// "hidden" receivers via trial and error
	GetAllSecretKeys() []BoxSecretKey

	// ImportEphemeralKey imports the ephemeral key into
	// BoxPublicKey format. This key has never been seen before, so
	// will be ephemeral.
	ImportEphemeralKey(kid []byte) BoxPublicKey
}

// SigKeyring is an interface used during verification to find
// the public key for the signer of a message.
type SigKeyring interface {
	// LookupSigningPublicKey returns a public signing key for the specified key ID.
	LookupSigningPublicKey(kid []byte) SigningPublicKey
}

// SecretKeyEqual returns true if the two secret keys are equal.
func SecretKeyEqual(sk1, sk2 BoxSecretKey) bool {
	return PublicKeyEqual(sk1.GetPublicKey(), sk2.GetPublicKey())
}

// PublicKeyEqual returns true if the two public keys are equal.
func PublicKeyEqual(pk1, pk2 BoxPublicKey) bool {
	return KIDEqual(pk1, pk2)
}

// KIDEqual return true if the KIDs for two keys are equal.
func KIDEqual(k1, k2 KIDExtractor) bool {
	return hmac.Equal(k1.ToKID(), k2.ToKID())
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"encoding/xml"
	"flag"
	"fmt"
	"io"
	"os"
	"regexp"
	"strconv"
	"time"

	"github.com/hpcloud/tail"
)

const (
	timeFormat = "02/Jan/2006:03:04:05 -0700"
)

//Event (request(s) paired with response) to be output in either
//JSON or XML depending on user preference.
type Event struct {
	OppTime         string   `json:"time" xml:"DateTime"`
	Client          string   `json:"client" xml:"Client"`
	Server          string   `json:"server" xml:"Server"`
	Connection      int      `json:"connection" xml:"Connection"`
	SSL             bool     `json:"ssl" xml:"SSL"`
	SSLCipher       string   `json:"sslcipher,omitempty" xml:"SSLCipher,omitempty"`
	SSLStrength     string   `json:"sslstrength,omitempty" xml:"SSLStrength,omitempty"`
	Operation       int      `json:"operation" xml:"Operation"`
	AuthenticatedDN string   `json:"authenticateddn,omitempty" xml:"AuthenticatedDN,omitempty"`
	Action          string   `json:"action" xml:"Action"`
	Requests        []string `json:"requests" xml:"Requests>Request"`
	Responses       []string `json:"responses" xml:"Responses>Response"`
	Duration        int      `json:"duration,omitempty" xml:"Duration,omitempty"`
	ConnTime        string   `json:"-" xml:"-"`
}

type config struct {
	TailFile     *bool
	OutputFormat *string
	LogFiles     *[]string
	Output       io.Writer
}

//holds config
var c config

//regexes to extract relevent fields from log lines
var lineMatch = `^\[(?P<time>.*)\] conn=(?P<conn_num>\d+) (?P<event>.*)`
var connectionMatch = `(?P<ssl>SSL)? connection from (?P<client_ip>.*) to (?P<server_ip>.*)`
var operationMatch = `op=(?P<opnum>\-?\d+) (?P<operation>\w+)(?P<details>.+)?`
var bindDNMatch = `dn=\"(?P<dn>.+)\"`
var connectionClosedMatch = ` closed `
var sslCipherMatch = `SSL (?P<strength>.*)-bit (?P<cipher>.*)`

var lineRe *regexp.Regexp
var connectionOpenRe *regexp.Regexp
var operationRe *regexp.Regexp
var bindDNRe *regexp.Regexp
var connectionClosedRe *regexp.Regexp
var sslCipherRe *regexp.Regexp

func check(e error) {
	if e != nil {
		panic(e)
	}
}

func matchLine(re *regexp.Regexp, line string) (map[string]string, bool) {
	matches := re.FindStringSubmatch(line)
	if len(matches) < 1 {
		return map[string]string{}, false
	}

	kvmap := make(map[string]string)
	for n, k := range re.SubexpNames() {
		kvmap[k] = matches[n]
	}

	return kvmap, true
}

func timeDuration(start string, end string) (int, error) {
	startTime, err := time.Parse(timeFormat, start)
	if err != nil {
		return -1, err
	}
	endTime, err := time.Parse(timeFormat, end)
	if err != nil {
		return -1, err
	}

	duration := endTime.Sub(startTime)

	return int(duration / time.Second), nil
}

func init() {
	c.TailFile = flag.Bool("tail", false, "tail the log file to receive future events")
	c.OutputFormat = flag.String("format", "json", "format to output log events.  possible values are 'json' or 'xml'.")
	c.Output = os.Stdout //configurable to help with unit testing

	flag.Usage = func() {
		fmt.Fprintf(os.Stderr, "Usage of %s:\n", os.Args[0])
		flag.PrintDefaults()
		os.Exit(1)
	}
}

func (c config) printEvent(event Event) {
	var output []byte
	var err error
	if format := *c.OutputFormat; format == "xml" {
		output, err = xml.MarshalIndent(event, "", "    ")
	} else {
		output, err = json.Marshal(event)
	}

	if err == nil {
		fmt.Fprintln(c.Output, string(output))
	}
}

func (c config) parseFile(ac map[int]Event, f string) map[int]Event {
	tc := tail.Config{}
	if *c.TailFile {
		tc.Follow = true
		tc.ReOpen = true
	}

	//open file for parsing
	t, err := tail.TailFile(f, tc)
	check(err)

	//loop through file contents
	for line := range t.Lines {
		var lineMap map[string]string
		var ok bool
		if lineMap, ok = matchLine(lineRe, line.Text); !ok {
			continue
		}

		connum, err := strconv.Atoi(lineMap["conn_num"])
		if err != nil {
			fmt.Printf("failed to parse '%s' into int\n", lineMap["conn_num"])
		}

		if connectionMap, ok := matchLine(connectionOpenRe, lineMap["event"]); ok {
			//new connection made
			ssl := false
			if connectionMap["ssl"] == "SSL" {
				ssl = true
			}

			ac[connum] = Event{
				Client:     connectionMap["client_ip"],
				Server:     connectionMap["server_ip"],
				Connection: connum,
				Operation:  -2, //number that shouldn't exist in logs
				ConnTime:   lineMap["time"],
				SSL:        ssl,
			}

			continue
		}

		if _, exists := ac[connum]; !exists {
			//skip operation if no connection exists
			// this is caused by connections that were created before we started
			// parsing the log file.
			continue
		}

		conn := ac[connum]

		if sslMap, ok := matchLine(sslCipherRe, lineMap["event"]); ok {
			conn.SSLCipher = sslMap["cipher"]
			conn.SSLStrength = sslMap["strength"]
			ac[connum] = conn
		}

		if operationMap, ok := matchLine(operationRe, lineMap["event"]); ok {
			//new operation
			opnum, err := strconv.Atoi(operationMap["opnum"])
			if err != nil {
				fmt.Printf("failed to parse '%s' into int\n", operationMap["opnum"])
			}

			if opnum != conn.Operation {
				if conn.Operation != -2 {
					c.printEvent(conn)
				}
				if operationMap["operation"] == "BIND" {
					if bindDN, ok := matchLine(bindDNRe, lineMap["event"]); ok {
						conn.AuthenticatedDN = bindDN["dn"]
					} else {
						conn.AuthenticatedDN = "__anonymous__"
					}
				}

				conn.OppTime = lineMap["time"]
				conn.Operation = opnum
				conn.Action = operationMap["operation"]
				conn.Requests = make([]string, 0)
				conn.Responses = make([]string, 0)
				conn.Requests = append(conn.Requests, operationMap["operation"]+operationMap["details"])
			} else {
				if operationMap["operation"] == "SORT" || operationMap["operation"] == "VLV" {
					conn.Requests = append(conn.Requests, operationMap["operation"]+operationMap["details"])
				} else {
					conn.Responses = append(conn.Responses, operationMap["operation"]+operationMap["details"])

					c.printEvent(conn)
					conn.Operation = -2
				}
			}

			ac[connum] = conn
		}

		if connectionClosedRe.MatchString(lineMap["event"]) {
			delete(ac, connum)
		}
	}
	return ac
}

func compileRegexes() {
	lineRe = regexp.MustCompile(lineMatch)
	connectionOpenRe = regexp.MustCompile(connectionMatch)
	operationRe = regexp.MustCompile(operationMatch)
	bindDNRe = regexp.MustCompile(bindDNMatch)
	connectionClosedRe = regexp.MustCompile(connectionClosedMatch)
	sslCipherRe = regexp.MustCompile(sslCipherMatch)
}

func main() {
	//prepare regex's
	compileRegexes()
	activeConnections := map[int]Event{}

	flag.Parse()

	if len(flag.Args()) < 1 {
		fmt.Println("ERROR: You must specify at least one log file.")
		flag.Usage()
	}

	c.parseFile(activeConnections, flag.Args()[0])

}
<commit_msg>add version information<commit_after>package main

import (
	"encoding/json"
	"encoding/xml"
	"flag"
	"fmt"
	"io"
	"os"
	"regexp"
	"strconv"
	"time"

	"github.com/hpcloud/tail"
)

const (
	timeFormat = "02/Jan/2006:03:04:05 -0700"
	version = "v1.0"
)

//Event (request(s) paired with response) to be output in either
//JSON or XML depending on user preference.
type Event struct {
	OppTime         string   `json:"time" xml:"DateTime"`
	Client          string   `json:"client" xml:"Client"`
	Server          string   `json:"server" xml:"Server"`
	Connection      int      `json:"connection" xml:"Connection"`
	SSL             bool     `json:"ssl" xml:"SSL"`
	SSLCipher       string   `json:"sslcipher,omitempty" xml:"SSLCipher,omitempty"`
	SSLStrength     string   `json:"sslstrength,omitempty" xml:"SSLStrength,omitempty"`
	Operation       int      `json:"operation" xml:"Operation"`
	AuthenticatedDN string   `json:"authenticateddn,omitempty" xml:"AuthenticatedDN,omitempty"`
	Action          string   `json:"action" xml:"Action"`
	Requests        []string `json:"requests" xml:"Requests>Request"`
	Responses       []string `json:"responses" xml:"Responses>Response"`
	Duration        int      `json:"duration,omitempty" xml:"Duration,omitempty"`
	ConnTime        string   `json:"-" xml:"-"`
}

type config struct {
	Version      *bool
	TailFile     *bool
	OutputFormat *string
	LogFiles     *[]string
	Output       io.Writer
}

//holds config
var c config

//regexes to extract relevent fields from log lines
var lineMatch = `^\[(?P<time>.*)\] conn=(?P<conn_num>\d+) (?P<event>.*)`
var connectionMatch = `(?P<ssl>SSL)? connection from (?P<client_ip>.*) to (?P<server_ip>.*)`
var operationMatch = `op=(?P<opnum>\-?\d+) (?P<operation>\w+)(?P<details>.+)?`
var bindDNMatch = `dn=\"(?P<dn>.+)\"`
var connectionClosedMatch = ` closed `
var sslCipherMatch = `SSL (?P<strength>.*)-bit (?P<cipher>.*)`

var lineRe *regexp.Regexp
var connectionOpenRe *regexp.Regexp
var operationRe *regexp.Regexp
var bindDNRe *regexp.Regexp
var connectionClosedRe *regexp.Regexp
var sslCipherRe *regexp.Regexp

func check(e error) {
	if e != nil {
		panic(e)
	}
}

func matchLine(re *regexp.Regexp, line string) (map[string]string, bool) {
	matches := re.FindStringSubmatch(line)
	if len(matches) < 1 {
		return map[string]string{}, false
	}

	kvmap := make(map[string]string)
	for n, k := range re.SubexpNames() {
		kvmap[k] = matches[n]
	}

	return kvmap, true
}

func timeDuration(start string, end string) (int, error) {
	startTime, err := time.Parse(timeFormat, start)
	if err != nil {
		return -1, err
	}
	endTime, err := time.Parse(timeFormat, end)
	if err != nil {
		return -1, err
	}

	duration := endTime.Sub(startTime)

	return int(duration / time.Second), nil
}

func init() {
	c.Version = flag.Bool("V", false, "prints version information")
	c.TailFile = flag.Bool("tail", false, "tail the log file to receive future events")
	c.OutputFormat = flag.String("format", "json", "format to output log events.  possible values are 'json' or 'xml'.")
	c.Output = os.Stdout //configurable to help with unit testing

	flag.Usage = func() {
		fmt.Fprintf(os.Stderr, "Usage of %s:\n", os.Args[0])
		flag.PrintDefaults()
		os.Exit(1)
	}
}

func (c config) printEvent(event Event) {
	var output []byte
	var err error
	if format := *c.OutputFormat; format == "xml" {
		output, err = xml.MarshalIndent(event, "", "    ")
	} else {
		output, err = json.Marshal(event)
	}

	if err == nil {
		fmt.Fprintln(c.Output, string(output))
	}
}

func (c config) parseFile(ac map[int]Event, f string) map[int]Event {
	tc := tail.Config{}
	if *c.TailFile {
		tc.Follow = true
		tc.ReOpen = true
	}

	//open file for parsing
	t, err := tail.TailFile(f, tc)
	check(err)

	//loop through file contents
	for line := range t.Lines {
		var lineMap map[string]string
		var ok bool
		if lineMap, ok = matchLine(lineRe, line.Text); !ok {
			continue
		}

		connum, err := strconv.Atoi(lineMap["conn_num"])
		if err != nil {
			fmt.Printf("failed to parse '%s' into int\n", lineMap["conn_num"])
		}

		if connectionMap, ok := matchLine(connectionOpenRe, lineMap["event"]); ok {
			//new connection made
			ssl := false
			if connectionMap["ssl"] == "SSL" {
				ssl = true
			}

			ac[connum] = Event{
				Client:     connectionMap["client_ip"],
				Server:     connectionMap["server_ip"],
				Connection: connum,
				Operation:  -2, //number that shouldn't exist in logs
				ConnTime:   lineMap["time"],
				SSL:        ssl,
			}

			continue
		}

		if _, exists := ac[connum]; !exists {
			//skip operation if no connection exists
			// this is caused by connections that were created before we started
			// parsing the log file.
			continue
		}

		conn := ac[connum]

		if sslMap, ok := matchLine(sslCipherRe, lineMap["event"]); ok {
			conn.SSLCipher = sslMap["cipher"]
			conn.SSLStrength = sslMap["strength"]
			ac[connum] = conn
		}

		if operationMap, ok := matchLine(operationRe, lineMap["event"]); ok {
			//new operation
			opnum, err := strconv.Atoi(operationMap["opnum"])
			if err != nil {
				fmt.Printf("failed to parse '%s' into int\n", operationMap["opnum"])
			}

			if opnum != conn.Operation {
				if conn.Operation != -2 {
					c.printEvent(conn)
				}
				if operationMap["operation"] == "BIND" {
					if bindDN, ok := matchLine(bindDNRe, lineMap["event"]); ok {
						conn.AuthenticatedDN = bindDN["dn"]
					} else {
						conn.AuthenticatedDN = "__anonymous__"
					}
				}

				conn.OppTime = lineMap["time"]
				conn.Operation = opnum
				conn.Action = operationMap["operation"]
				conn.Requests = make([]string, 0)
				conn.Responses = make([]string, 0)
				conn.Requests = append(conn.Requests, operationMap["operation"]+operationMap["details"])
			} else {
				if operationMap["operation"] == "SORT" || operationMap["operation"] == "VLV" {
					conn.Requests = append(conn.Requests, operationMap["operation"]+operationMap["details"])
				} else {
					conn.Responses = append(conn.Responses, operationMap["operation"]+operationMap["details"])

					c.printEvent(conn)
					conn.Operation = -2
				}
			}

			ac[connum] = conn
		}

		if connectionClosedRe.MatchString(lineMap["event"]) {
			delete(ac, connum)
		}
	}
	return ac
}

func compileRegexes() {
	lineRe = regexp.MustCompile(lineMatch)
	connectionOpenRe = regexp.MustCompile(connectionMatch)
	operationRe = regexp.MustCompile(operationMatch)
	bindDNRe = regexp.MustCompile(bindDNMatch)
	connectionClosedRe = regexp.MustCompile(connectionClosedMatch)
	sslCipherRe = regexp.MustCompile(sslCipherMatch)
}

func main() {
	//prepare regex's
	compileRegexes()
	activeConnections := map[int]Event{}

	flag.Parse()

	if *c.Version {
		fmt.Printf("%s %s\n", os.Args[0], version)
		os.Exit(0)
	}

	if len(flag.Args()) < 1 {
		fmt.Println("ERROR: You must specify at least one log file.")
		flag.Usage()
	}

	c.parseFile(activeConnections, flag.Args()[0])

}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"strings"
	"unicode"
	"unicode/utf8"

	log "github.com/Sirupsen/logrus"
)

const (
	/* TODO: Maybe remove [] and () from this and handle them differently
	since it may break things that shouldn't be translated */
	slashCharacters = "0123456789[]{}()\\/<>abcdefghijklmnopqrstuvxzwyABCDEFGHIJKLMNOPQRSTUVXZWY!|$^."
)

// next returns the next rune in the input.
func (l *lexer) next() rune {
	if l.pos >= len(l.input) {
		l.width = 0
		return eof
	}

	r, w := utf8.DecodeRuneInString(l.input[l.pos:])

	l.mark = r
	l.width = w

	l.pos += l.width

	return r
}

func (l *lexer) nextItem() item {
	item := <-l.items

	l.lastPos = item.pos

	return item
}

func (l *lexer) backup(pos int) {
	for i := 0; i < pos; i++ {
		l.pos -= l.width
		if l.pos < 0 {
			l.pos = 0
		}

		r, w := utf8.DecodeRuneInString(l.input[l.pos:])
		l.mark = r
		l.width = w
	}
}

func (l *lexer) peek(locs int) rune {
	pos := saveLexerPosition(l)

	var r rune

	x := 0

	for x < locs {
		l.next()

		if x == locs-1 {
			r = l.mark
		}

		x++
	}

	pos.restore(l)

	return r
}

// emit passes an item back to the client.
func (l *lexer) emit(t itemType) {
	l.items <- item{t, l.start, l.input[l.start:l.pos]}

	l.start = l.pos
}

func (l *lexer) emitBefore(t itemType) {
	l.backup(1)

	if l.pos > l.start {
		l.emit(t)

		l.ignore()
	}
}

// ignore skips over the pending input before this point.
func (l *lexer) ignore() {
	l.start = l.pos
}

// accept consumes the next rune if it's from the valid set.
func (l *lexer) accept(valid string) bool {
	if strings.ContainsRune(valid, l.next()) {
		return true
	}

	l.backup(1)

	return false
}

// acceptRun consumes a run of runes from the valid set.
func (l *lexer) acceptRun(valid string) {
	count := 0

	for strings.ContainsRune(valid, l.next()) {
		count++
	}

	l.backup(1)
}

// errorf returns an error token and terminates the scan by passing
// back a nil pointer that will be the next state, terminating l.nextItem.
func (l *lexer) errorf(format string, args ...interface{}) stateFn {
	l.items <- item{itemError, l.start, fmt.Sprintf(format, args...)}

	return nil
}

// lex creates a new scanner for the input string.
func lex(input string) *lexer {
	l := &lexer{
		input: input,
		items: make(chan item),
	}

	go l.run()

	return l
}

// run runs the state machine for the lexer.
func (l *lexer) run() {
	for l.state = lexText; l.state != nil; {
		l.state = l.state(l)
	}

	close(l.items)
}

func lexText(l *lexer) stateFn {
	log.Debugf("lexText %q", l.input[l.pos:])

	l.width = 0

Loop:
	for {
		//switch l.next() {
		switch r := l.next(); {
		case r == eof:
			break Loop
		case r == '%' && l.peek(1) == 's':
			l.emitBefore(itemText)

			l.next()

			if l.accept("s") {
				l.emit(itemRawString)
			}
		case (r == 'i' && strings.HasPrefix(l.input[l.pos:], "f(")) ||
			(r == 'e' && strings.HasPrefix(l.input[l.pos:], "n(")):
			l.emitBefore(itemText)

			return lexScript
		case r == '\\':
			r = l.peek(1)

			if r == '\\' {
				l.emitBefore(itemText)

				return lexScript
			}

			// These shouldn't even appear but google has sent me these back in translations
			if r == 'u' {
				log.Debug("Found escaped rune")

				l.emitBefore(itemText)

				l.next()

				l.acceptRun("u0123456789")

				l.ignore()
			}

			if r == 'n' {
				log.Debug("Found escaped newline")

				l.emitBefore(itemText)

				l.next()

				l.accept("n")

				l.acceptRun("[0123456789]")

				l.emit(itemRawString)
			}

		case strings.ContainsRune("\u3000（）・！？。…【】「」『』\n()/\"[]", r) || unicode.IsSymbol(r):
			l.emitBefore(itemText)

			l.next()

			l.emit(itemRawString)

			return lexText
		}
	}

	if l.pos > l.start {
		l.emit(itemText)

		l.ignore()
	}

	l.emit(itemEOF)

	return nil

}

func lexScript(l *lexer) stateFn {
	log.Debugf("lexScript %q", l.input[l.pos:])

Loop:
	for {
		switch l.next() {
		case eof:
			log.Warn("Script not terminated properly %q", l.input[l.start:])
			break Loop
		case '(':
			l.emitBefore(itemScript)

			l.next()

			l.emit(itemLeftParen)

			l.parenDepth++

			return lexInsideAction
		case '[':
			l.backup(1)
			l.emit(itemScript)

			return lexLeftDelim
		case '\\':
			l.acceptRun(slashCharacters)

			break Loop
		case '\n':
			l.acceptRun("[0123456789]")

			break Loop

		default:
			log.Debug(string(l.mark))
		}
	}

	l.emit(itemScript)

	return lexText
}

func lexLeftDelim(l *lexer) stateFn {
	l.next()

	log.Debug("leftDelim: ", string(l.mark))

	l.emit(itemLeftDelim)

	return lexInsideAction
}

func lexRightDelim(l *lexer) stateFn {
	l.next()

	log.Debug("rightDelim: ", string(l.mark))

	l.emit(itemRightDelim)

	log.Debug("Paren depth: ", l.parenDepth)

	if l.parenDepth == 0 {
		return lexText
	}

	return lexInsideAction
}

// lexInsideAction scans the elements inside action delimiters.
func lexInsideAction(l *lexer) stateFn {
	log.Debugf("lexInsideAction %q", l.input[l.pos:])

	switch r := l.next(); {
	case r == eof:
		return l.errorf("unclosed action")
	case r == '(':
		l.emitBefore(itemParameter)

		l.next()

		l.emit(itemLeftParen)

		l.parenDepth++
	case r == ')':
		l.emitBefore(itemParameter)

		l.next()

		l.emit(itemRightParen)

		l.parenDepth--

		if l.parenDepth < 0 {
			return l.errorf("unexpected right paren %#U", r)
		}

		if l.parenDepth == 0 {
			return lexText
		}
	case r == '[':
		l.emitBefore(itemParameter)

		return lexLeftDelim
	case r == ']':
		l.emitBefore(itemParameter)

		return lexRightDelim
	case r == '%' && l.accept("s"):
		l.emit(itemParameter)
	case r == '"':
		l.emit(itemParameter)

		return lexText
	}

	return lexInsideAction
}
<commit_msg>Avoid translating ':'<commit_after>package main

import (
	"fmt"
	"strings"
	"unicode"
	"unicode/utf8"

	log "github.com/Sirupsen/logrus"
)

const (
	/* TODO: Maybe remove [] and () from this and handle them differently
	since it may break things that shouldn't be translated */
	slashCharacters = "0123456789[]{}()\\/<>abcdefghijklmnopqrstuvxzwyABCDEFGHIJKLMNOPQRSTUVXZWY!|$^."
	rawCharacters   = "\u3000（）・！？。…【】「」『』\n()/\"[]：:"
)

// next returns the next rune in the input.
func (l *lexer) next() rune {
	if l.pos >= len(l.input) {
		l.width = 0
		return eof
	}

	r, w := utf8.DecodeRuneInString(l.input[l.pos:])

	l.mark = r
	l.width = w

	l.pos += l.width

	return r
}

func (l *lexer) nextItem() item {
	item := <-l.items

	l.lastPos = item.pos

	return item
}

func (l *lexer) backup(pos int) {
	for i := 0; i < pos; i++ {
		l.pos -= l.width
		if l.pos < 0 {
			l.pos = 0
		}

		r, w := utf8.DecodeRuneInString(l.input[l.pos:])
		l.mark = r
		l.width = w
	}
}

func (l *lexer) peek(locs int) rune {
	pos := saveLexerPosition(l)

	var r rune

	x := 0

	for x < locs {
		l.next()

		if x == locs-1 {
			r = l.mark
		}

		x++
	}

	pos.restore(l)

	return r
}

// emit passes an item back to the client.
func (l *lexer) emit(t itemType) {
	l.items <- item{t, l.start, l.input[l.start:l.pos]}

	l.start = l.pos
}

func (l *lexer) emitBefore(t itemType) {
	l.backup(1)

	if l.pos > l.start {
		l.emit(t)

		l.ignore()
	}
}

// ignore skips over the pending input before this point.
func (l *lexer) ignore() {
	l.start = l.pos
}

// accept consumes the next rune if it's from the valid set.
func (l *lexer) accept(valid string) bool {
	if strings.ContainsRune(valid, l.next()) {
		return true
	}

	l.backup(1)

	return false
}

// acceptRun consumes a run of runes from the valid set.
func (l *lexer) acceptRun(valid string) {
	count := 0

	for strings.ContainsRune(valid, l.next()) {
		count++
	}

	l.backup(1)
}

// errorf returns an error token and terminates the scan by passing
// back a nil pointer that will be the next state, terminating l.nextItem.
func (l *lexer) errorf(format string, args ...interface{}) stateFn {
	l.items <- item{itemError, l.start, fmt.Sprintf(format, args...)}

	return nil
}

// lex creates a new scanner for the input string.
func lex(input string) *lexer {
	l := &lexer{
		input: input,
		items: make(chan item),
	}

	go l.run()

	return l
}

// run runs the state machine for the lexer.
func (l *lexer) run() {
	for l.state = lexText; l.state != nil; {
		l.state = l.state(l)
	}

	close(l.items)
}

func lexText(l *lexer) stateFn {
	log.Debugf("lexText %q", l.input[l.pos:])

	l.width = 0

Loop:
	for {
		//switch l.next() {
		switch r := l.next(); {
		case r == eof:
			break Loop
		case r == '%' && l.peek(1) == 's':
			l.emitBefore(itemText)

			l.next()

			if l.accept("s") {
				l.emit(itemRawString)
			}
		case (r == 'i' && strings.HasPrefix(l.input[l.pos:], "f(")) ||
			(r == 'e' && strings.HasPrefix(l.input[l.pos:], "n(")):
			l.emitBefore(itemText)

			return lexScript
		case r == '\\':
			r = l.peek(1)

			if r == '\\' {
				l.emitBefore(itemText)

				return lexScript
			}

			// These shouldn't even appear but google has sent me these back in translations
			if r == 'u' {
				log.Debug("Found escaped rune")

				l.emitBefore(itemText)

				l.next()

				l.acceptRun("u0123456789")

				l.ignore()
			}

			if r == 'n' {
				log.Debug("Found escaped newline")

				l.emitBefore(itemText)

				l.next()

				l.accept("n")

				l.acceptRun("[0123456789]")

				l.emit(itemRawString)
			}

		case strings.ContainsRune(rawCharacters, r) || unicode.IsSymbol(r):
			l.emitBefore(itemText)

			l.next()

			l.emit(itemRawString)

			return lexText
		}
	}

	if l.pos > l.start {
		l.emit(itemText)

		l.ignore()
	}

	l.emit(itemEOF)

	return nil

}

func lexScript(l *lexer) stateFn {
	log.Debugf("lexScript %q", l.input[l.pos:])

Loop:
	for {
		switch l.next() {
		case eof:
			log.Warn("Script not terminated properly %q", l.input[l.start:])
			break Loop
		case '(':
			l.emitBefore(itemScript)

			l.next()

			l.emit(itemLeftParen)

			l.parenDepth++

			return lexInsideAction
		case '[':
			l.backup(1)
			l.emit(itemScript)

			return lexLeftDelim
		case '\\':
			l.acceptRun(slashCharacters)

			break Loop
		case '\n':
			l.acceptRun("[0123456789]")

			break Loop

		default:
			log.Debug(string(l.mark))
		}
	}

	l.emit(itemScript)

	return lexText
}

func lexLeftDelim(l *lexer) stateFn {
	l.next()

	log.Debug("leftDelim: ", string(l.mark))

	l.emit(itemLeftDelim)

	return lexInsideAction
}

func lexRightDelim(l *lexer) stateFn {
	l.next()

	log.Debug("rightDelim: ", string(l.mark))

	l.emit(itemRightDelim)

	log.Debug("Paren depth: ", l.parenDepth)

	if l.parenDepth == 0 {
		return lexText
	}

	return lexInsideAction
}

// lexInsideAction scans the elements inside action delimiters.
func lexInsideAction(l *lexer) stateFn {
	log.Debugf("lexInsideAction %q", l.input[l.pos:])

	switch r := l.next(); {
	case r == eof:
		return l.errorf("unclosed action")
	case r == '(':
		l.emitBefore(itemParameter)

		l.next()

		l.emit(itemLeftParen)

		l.parenDepth++
	case r == ')':
		l.emitBefore(itemParameter)

		l.next()

		l.emit(itemRightParen)

		l.parenDepth--

		if l.parenDepth < 0 {
			return l.errorf("unexpected right paren %#U", r)
		}

		if l.parenDepth == 0 {
			return lexText
		}
	case r == '[':
		l.emitBefore(itemParameter)

		return lexLeftDelim
	case r == ']':
		l.emitBefore(itemParameter)

		return lexRightDelim
	case r == '%' && l.accept("s"):
		l.emit(itemParameter)
	case r == '"':
		l.emit(itemParameter)

		return lexText
	}

	return lexInsideAction
}
<|endoftext|>


<commit_before>/*
SPDX-License-Identifier: MIT

Copyright (c) 2017 Thanh Ha

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
*/

// lhc is a checker to find code files missing license headers.
package main

import (
	"bufio"
	"flag"
	"fmt"
	"os"
	"path/filepath"
	"strings"
	"unicode"

	"github.com/zxiiro/license-header-checker/license"
)

var VERSION = "0.1.0"

// Compare a license header with an approved list of license headers.
func accepted_license(check string, approved []string) bool {
	for _, i := range approved {
		if check == i {
			return true
		}
	}

	return false
}

// check and exit if error.
func check(e error) {
	if e != nil {
		fmt.Println(e)
		os.Exit(1)
	}
}

func findFiles(directory string, patterns []string) []string {
	var files []string
	filepath.Walk(directory, func(path string, info os.FileInfo, err error) error {
		if info.IsDir() {
			for _, p := range patterns {
				f, _ := filepath.Glob(filepath.Join(path, p))
				files = append(files, f...)
			}
		}
		return nil
	})
	return files
}

// fetchLicense from file and return license text.
func fetchLicense(filename string) string {
	comment, multilineComment := false, false
	licenseText := ""

	var scanner *bufio.Scanner
	if filename == "MIT" {
		scanner = bufio.NewScanner(strings.NewReader(license.MIT_LICENSE))
	} else if filename == "EPL-1.0" {
		scanner = bufio.NewScanner(strings.NewReader(license.EPL_10_LICENSE))
	} else {
		file, err := os.Open(filename)
		check(err)
		defer file.Close()

		// Read the first 2 bytes to decide if it is a comment string
		b := make([]byte, 2)
		_, err = file.Read(b)
		check(err)
		if isComment(string(b)) {
			comment = true
		}
		file.Seek(0, 0) // Reset so we can read the full file next

		scanner = bufio.NewScanner(file)
	}

	i := 0
	for scanner.Scan() {
		// Read only the first few lines to not read entire code file
		i++
		if i > 50 {
			break
		}

		s := scanner.Text()

		if ignoreComment(s) {
			continue
		}

		if comment == true {
			if strings.HasPrefix(s, "/*") {
				multilineComment = true
			} else if strings.Contains(s, "*/") {
				multilineComment = false
			}

			if !multilineComment && !isComment(s) ||
				// EPL headers can contain contributors list.
				strings.Contains(strings.ToUpper(s), " * CONTRIBUTORS:") {
				break
			}

			s = trimComment(s)
		}

		licenseText += s
	}

	return stripSpaces(licenseText)
}

// Check if a string is a comment line.
func isComment(str string) bool {
	if !strings.HasPrefix(str, "#") &&
		!strings.HasPrefix(str, "//") &&
		!strings.HasPrefix(str, "/*") {
		return false
	}

	return true
}

// Ignore certain lines containing key strings
func ignoreComment(str string) bool {
	s := strings.ToUpper(str)
	if strings.HasPrefix(s, "#!") ||
		strings.Contains(s, "COPYRIGHT") ||
		strings.Contains(s, "SPDX-LICENSE-IDENTIFIER") ||
		// License name in LICENSE file but not header
		strings.Contains(s, "MIT LICENSE") {
		return true
	}

	return false
}

// Strip whitespace from string.
func stripSpaces(str string) string {
	return strings.Map(func(r rune) rune {
		if unicode.IsSpace(r) {
			return -1
		}
		return r
	}, str)
}

// Trim the comment prefix from string.
func trimComment(str string) string {
	str = strings.TrimLeft(str, "#")
	str = strings.TrimLeft(str, "//")
	str = strings.TrimLeft(str, "/*")
	str = strings.TrimLeft(str, " *")
	str = strings.Split(str, "*/")[0]
	str = strings.TrimLeft(str, "*")
	return str
}

// Usage prints a statement to explain how to use this command.
func usage() {
	fmt.Printf("Usage: %s [OPTIONS] [FILE]...\n", os.Args[0])
	fmt.Printf("Compare FILE with an expected license header.\n")
	fmt.Printf("\nOptions:\n")
	flag.PrintDefaults()
}

func main() {
	directoryPtr := flag.String("directory", ".", "Directory to search for files.")
	licensePtr := flag.String("license", "license.txt", "Comma-separated list of license files to compare against.")
	versionPtr := flag.Bool("version", false, "Print version")

	flag.Usage = usage
	flag.Parse()

	if *versionPtr {
		fmt.Println("License Checker version", VERSION)
		os.Exit(0)
	}

	fmt.Println("Search Patterns:", flag.Args())

	var accepted_licenses []string
	for _, l := range strings.Split(*licensePtr, ",") {
		accepted_licenses = append(accepted_licenses, fetchLicense(l))
	}
	checkFiles := findFiles(*directoryPtr, flag.Args())

	for _, f := range checkFiles {
		headerText := fetchLicense(f)
		if accepted_license(headerText, accepted_licenses) {
			fmt.Println("✔", f)
		} else {
			fmt.Println("✘", f)
		}
	}
}
<commit_msg>Add ability to check which license was approved<commit_after>/*
SPDX-License-Identifier: MIT

Copyright (c) 2017 Thanh Ha

Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:

The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
*/

// lhc is a checker to find code files missing license headers.
package main

import (
	"bufio"
	"flag"
	"fmt"
	"os"
	"path/filepath"
	"strings"
	"unicode"

	"github.com/zxiiro/license-header-checker/license"
)

var VERSION = "0.1.0"

type License struct {
	Name string
	Text string
}

// Compare a license header with an approved list of license headers.
// Returns the name of the license that was approved. Else "".
func accepted_license(check string, approved []License) string {
	for _, i := range approved {
		if check == i.Text {
			return i.Name
		}
	}

	return ""
}

// check and exit if error.
func check(e error) {
	if e != nil {
		fmt.Println(e)
		os.Exit(1)
	}
}

func findFiles(directory string, patterns []string) []string {
	var files []string
	filepath.Walk(directory, func(path string, info os.FileInfo, err error) error {
		if info.IsDir() {
			for _, p := range patterns {
				f, _ := filepath.Glob(filepath.Join(path, p))
				files = append(files, f...)
			}
		}
		return nil
	})
	return files
}

// fetchLicense from file and return license text.
func fetchLicense(filename string) string {
	comment, multilineComment := false, false
	licenseText := ""

	var scanner *bufio.Scanner
	if filename == "MIT" {
		scanner = bufio.NewScanner(strings.NewReader(license.MIT_LICENSE))
	} else if filename == "EPL-1.0" {
		scanner = bufio.NewScanner(strings.NewReader(license.EPL_10_LICENSE))
	} else {
		file, err := os.Open(filename)
		check(err)
		defer file.Close()

		// Read the first 2 bytes to decide if it is a comment string
		b := make([]byte, 2)
		_, err = file.Read(b)
		check(err)
		if isComment(string(b)) {
			comment = true
		}
		file.Seek(0, 0) // Reset so we can read the full file next

		scanner = bufio.NewScanner(file)
	}

	i := 0
	for scanner.Scan() {
		// Read only the first few lines to not read entire code file
		i++
		if i > 50 {
			break
		}

		s := scanner.Text()

		if ignoreComment(s) {
			continue
		}

		if comment == true {
			if strings.HasPrefix(s, "/*") {
				multilineComment = true
			} else if strings.Contains(s, "*/") {
				multilineComment = false
			}

			if !multilineComment && !isComment(s) ||
				// EPL headers can contain contributors list.
				strings.Contains(strings.ToUpper(s), " * CONTRIBUTORS:") {
				break
			}

			s = trimComment(s)
		}

		licenseText += s
	}

	return stripSpaces(licenseText)
}

// Check if a string is a comment line.
func isComment(str string) bool {
	if !strings.HasPrefix(str, "#") &&
		!strings.HasPrefix(str, "//") &&
		!strings.HasPrefix(str, "/*") {
		return false
	}

	return true
}

// Ignore certain lines containing key strings
func ignoreComment(str string) bool {
	s := strings.ToUpper(str)
	if strings.HasPrefix(s, "#!") ||
		strings.Contains(s, "COPYRIGHT") ||
		strings.Contains(s, "SPDX-LICENSE-IDENTIFIER") ||
		// License name in LICENSE file but not header
		strings.Contains(s, "MIT LICENSE") {
		return true
	}

	return false
}

// Strip whitespace from string.
func stripSpaces(str string) string {
	return strings.Map(func(r rune) rune {
		if unicode.IsSpace(r) {
			return -1
		}
		return r
	}, str)
}

// Trim the comment prefix from string.
func trimComment(str string) string {
	str = strings.TrimLeft(str, "#")
	str = strings.TrimLeft(str, "//")
	str = strings.TrimLeft(str, "/*")
	str = strings.TrimLeft(str, " *")
	str = strings.Split(str, "*/")[0]
	str = strings.TrimLeft(str, "*")
	return str
}

// Usage prints a statement to explain how to use this command.
func usage() {
	fmt.Printf("Usage: %s [OPTIONS] [FILE]...\n", os.Args[0])
	fmt.Printf("Compare FILE with an expected license header.\n")
	fmt.Printf("\nOptions:\n")
	flag.PrintDefaults()
}

func main() {
	directoryPtr := flag.String("directory", ".", "Directory to search for files.")
	licensePtr := flag.String("license", "license.txt", "Comma-separated list of license files to compare against.")
	versionPtr := flag.Bool("version", false, "Print version")

	flag.Usage = usage
	flag.Parse()

	if *versionPtr {
		fmt.Println("License Checker version", VERSION)
		os.Exit(0)
	}

	fmt.Println("Search Patterns:", flag.Args())

	var accepted_licenses []License
	for _, l := range strings.Split(*licensePtr, ",") {
		license := License{l, fetchLicense(l)}
		accepted_licenses = append(accepted_licenses, license)
	}
	checkFiles := findFiles(*directoryPtr, flag.Args())

	for _, f := range checkFiles {
		headerText := fetchLicense(f)
		if accepted_license(headerText, accepted_licenses) != "" {
			fmt.Println("✔", f)
		} else {
			fmt.Println("✘", f)
		}
	}
}
<|endoftext|>


<commit_before>/* This implements an alternative logger to the one found in the standard
 * library with support for more logging levels, formatters and handlers.
 * The main goal is to provide easy and flexible way to handle new handlers and formats
 * Author: Robert Zaremba
 *
 * https://github.com/scale-it/go-log
 */
package log

import (
	"errors"
	"fmt"
	"io"
	"runtime"
	"strconv"
	"strings"
	"sync"
	"time"
)

// Flags from std log package
// These flags define which text to prefix to each log entry generated by the Logger.
const (
	// Bits or'ed together to control what's printed. There is no control over the
	// order they appear (the order listed here) or the format they present (as
	// described in the comments).  A colon appears after these items:
	//	2009/01/23 01:23:23.123123 /a/b/c/d.go:23: message
	Ldate         = 1 << iota     // the date: 2009/01/23
	Ltime                         // the time: 01:23:23
	Lmicroseconds                 // microsecond resolution: 01:23:23.123123.  assumes Ltime.
	Llongfile                     // full file name and line number: /a/b/c/d.go:23
	Lshortfile                    // final file name element and line number: d.go:23. overrides Llongfile
	LstdFlags     = Ldate | Ltime // initial values for the standard logger
)

// Represents how critical the logged
// message is.
type Level uint8

var Levels = struct {
	Trace   Level
	Debug   Level
	Info    Level
	Warning Level
	Error   Level
	Fatal   Level
}{0, 10, 20, 30, 40, 50}

// Verbose names of the levels
var levelStrings = map[Level]string{
	Levels.Trace:   "TRACE",
	Levels.Debug:   "DEBUG",
	Levels.Info:    "INFO ",
	Levels.Warning: "WARN ",
	Levels.Error:   "ERROR",
	Levels.Fatal:   "FATAL",
}

// Verbose and colored names of the levels
var levelCStrings = map[Level]string{
	Levels.Trace:   levelStrings[Levels.Trace],
	Levels.Debug:   levelStrings[Levels.Debug],
	Levels.Info:    AnsiEscape(MAGENTA, levelStrings[Levels.Info], OFF),
	Levels.Warning: AnsiEscape(YELLOW, levelStrings[Levels.Warning], OFF),
	Levels.Error:   AnsiEscape(RED, levelStrings[Levels.Error], OFF),
	Levels.Fatal:   AnsiEscape(RED, BOLD, levelStrings[Levels.Fatal], OFF),
}

// Returns an log Level which name match given string.
// If there is no such Level, then Levels.Debug is returned
func String2Level(level string) (Level, error) {
	if level == "" {
		return Levels.Debug, errors.New("level is empty")
	}
	for li, ls := range levelStrings {
		if ls == level {
			return li, nil
		}
	}
	return Levels.Debug, errors.New("Wrong log level " + level)
}

type Formatter interface {
	Format(Level, string) []byte
}

type handler struct {
	writer io.Writer
	level  Level
	fmt    Formatter
}

type Logger struct {
	mtx      sync.Mutex
	handlers []handler
}

var vv map[io.Writer]sync.Mutex

// Instantiate a new Logger
func New() *Logger {
	return &Logger{sync.Mutex{}, make([]handler, 0)}
}

// Convenience function to create logger with StdFormatter
func NewStd(w io.Writer, level Level, prefix string, flag int, colored bool) *Logger {
	l := Logger{sync.Mutex{}, make([]handler, 0)}
	l.AddHandler(w, level, StdFormatter{prefix, flag, colored})
	return &l
}

// Standard Formatter
type StdFormatter struct {
	Prefix  string // prefix to write at beginning of each line
	Flag    int    // format flags - based flags from std log package
	Colored bool   // use colored level names
}

func (this StdFormatter) Format(level Level, msg string) []byte {
	var slevel string
	var ok bool
	var out []string

	// adding time info
	if this.Flag&(Ldate|Ltime|Lmicroseconds) != 0 {
		now := time.Now()
		if this.Flag&Ldate != 0 {
			out = append(out, fmt.Sprintf("%v-%02d-%02d", now.Year(), now.Month(), now.Day()))
		}
		if this.Flag&(Lmicroseconds) != 0 {
			out = append(out, fmt.Sprintf("%02d:%02d:%02d.%06d", now.Hour(), now.Minute(), now.Second(), now.Nanosecond()/1000000))
		} else if this.Flag&(Ltime) != 0 {
			out = append(out, fmt.Sprintf("%02d:%02d:%02d", now.Hour(), now.Minute(), now.Second()))
		}
	}

	// adding level info
	if this.Colored {
		slevel, ok = levelCStrings[level]
	} else {
		slevel, ok = levelStrings[level]
	}
	if !ok {
		slevel = strconv.Itoa(int(level))
	}
	out = append(out, slevel)

	out = append(out, this.Prefix)

	// adding caller info. It's quiet exepnsive
	if this.Flag&(Lshortfile|Llongfile) != 0 {
		if _, file, line, ok := runtime.Caller(2); ok { // 2: calldepth
			if this.Flag&Lshortfile != 0 {
				file = file[strings.LastIndex(file, "/")+1:]
			}
			out = append(out, fmt.Sprintf("%s:%d", file, line))
		} else {
			out = append(out, "???")
		}
	}

	out = append(out, msg)
	return []byte(strings.Join(out, " "))
}

/* LOGGER
 * ------
 */

// Adds a handler, specifying the maximum log Level
// you want to be written to this output. For instance,
// if you pass Warning for level, all logs of type
// Warning, Error, and Fatal would be logged to this handler.
func (this *Logger) AddHandler(writer io.Writer, level Level, fm Formatter) {
	this.mtx.Lock()
	this.handlers = append(this.handlers, handler{writer, level, fm})
	this.mtx.Unlock()
}

// Logs a message for the given level. Most callers will likely
// prefer to use one of the provided convenience functions (Debug, Info...).
func (this *Logger) Log(level Level, msg string) {
	var out []byte
	for _, h := range this.handlers {
		if h.level <= level {
			out = h.fmt.Format(level, msg)
			this.mtx.Lock()
			h.writer.Write(out)
			this.mtx.Unlock()
		}
	}
}

// Logs a formatted message message for the given level.
// Wrapper around Log method
func (this *Logger) Logf(level Level, format string, v ...interface{}) {
	this.Log(level, fmt.Sprintf(format+"\n", v...))
}

// Convenience function
func (this *Logger) Trace(format string, v ...interface{}) {
	// TODO: split the string
	this.Log(Levels.Trace, fmt.Sprintf(format+"\n", v...))
}

// Convenience function
func (this *Logger) Debug(format string, v ...interface{}) {
	this.Log(Levels.Debug, fmt.Sprintf(format+"\n", v...))
}

// Convenience function
func (this *Logger) Info(format string, v ...interface{}) {
	this.Log(Levels.Info, fmt.Sprintf(format+"\n", v...))
}

// Convenience function
func (this *Logger) Warning(format string, v ...interface{}) {
	this.Log(Levels.Warning, fmt.Sprintf(format+"\n", v...))
}

// Convenience function
func (this *Logger) Error(format string, v ...interface{}) {
	this.Log(Levels.Error, fmt.Sprintf(format+"\n", v...))
}

// Convenience function, will not terminate the program
func (this *Logger) Fatal(format string, v ...interface{}) {
	this.Log(Levels.Fatal, fmt.Sprintf(format+"\n", v...))
}
<commit_msg>Removed prefix argument from NewStd function<commit_after>/* This implements an alternative logger to the one found in the standard
 * library with support for more logging levels, formatters and handlers.
 * The main goal is to provide easy and flexible way to handle new handlers and formats
 * Author: Robert Zaremba
 *
 * https://github.com/scale-it/go-log
 */
package log

import (
	"errors"
	"fmt"
	"io"
	"runtime"
	"strconv"
	"strings"
	"sync"
	"time"
)

// Flags from std log package
// These flags define which text to prefix to each log entry generated by the Logger.
const (
	// Bits or'ed together to control what's printed. There is no control over the
	// order they appear (the order listed here) or the format they present (as
	// described in the comments).  A colon appears after these items:
	//	2009/01/23 01:23:23.123123 /a/b/c/d.go:23: message
	Ldate         = 1 << iota     // the date: 2009/01/23
	Ltime                         // the time: 01:23:23
	Lmicroseconds                 // microsecond resolution: 01:23:23.123123.  assumes Ltime.
	Llongfile                     // full file name and line number: /a/b/c/d.go:23
	Lshortfile                    // final file name element and line number: d.go:23. overrides Llongfile
	LstdFlags     = Ldate | Ltime // initial values for the standard logger
)

// Represents how critical the logged
// message is.
type Level uint8

var Levels = struct {
	Trace   Level
	Debug   Level
	Info    Level
	Warning Level
	Error   Level
	Fatal   Level
}{0, 10, 20, 30, 40, 50}

// Verbose names of the levels
var levelStrings = map[Level]string{
	Levels.Trace:   "TRACE",
	Levels.Debug:   "DEBUG",
	Levels.Info:    "INFO ",
	Levels.Warning: "WARN ",
	Levels.Error:   "ERROR",
	Levels.Fatal:   "FATAL",
}

// Verbose and colored names of the levels
var levelCStrings = map[Level]string{
	Levels.Trace:   levelStrings[Levels.Trace],
	Levels.Debug:   levelStrings[Levels.Debug],
	Levels.Info:    AnsiEscape(MAGENTA, levelStrings[Levels.Info], OFF),
	Levels.Warning: AnsiEscape(YELLOW, levelStrings[Levels.Warning], OFF),
	Levels.Error:   AnsiEscape(RED, levelStrings[Levels.Error], OFF),
	Levels.Fatal:   AnsiEscape(RED, BOLD, levelStrings[Levels.Fatal], OFF),
}

// Returns an log Level which name match given string.
// If there is no such Level, then Levels.Debug is returned
func String2Level(level string) (Level, error) {
	if level == "" {
		return Levels.Debug, errors.New("level is empty")
	}
	for li, ls := range levelStrings {
		if ls == level {
			return li, nil
		}
	}
	return Levels.Debug, errors.New("Wrong log level " + level)
}

type Formatter interface {
	Format(Level, string) []byte
}

type handler struct {
	writer io.Writer
	level  Level
	fmt    Formatter
}

type Logger struct {
	mtx      sync.Mutex
	handlers []handler
}

var vv map[io.Writer]sync.Mutex

// Instantiate a new Logger
func New() *Logger {
	return &Logger{sync.Mutex{}, make([]handler, 0)}
}

// Convenience function to create logger with StdFormatter
func NewStd(w io.Writer, level Level, flag int, colored bool) *Logger {
	l := Logger{sync.Mutex{}, make([]handler, 0)}
	l.AddHandler(w, level, StdFormatter{"", flag, colored})
	return &l
}

// Standard Formatter
type StdFormatter struct {
	Prefix  string // prefix to write at beginning of each line
	Flag    int    // format flags - based flags from std log package
	Colored bool   // use colored level names
}

func (this StdFormatter) Format(level Level, msg string) []byte {
	var slevel string
	var ok bool
	var out []string

	// adding time info
	if this.Flag&(Ldate|Ltime|Lmicroseconds) != 0 {
		now := time.Now()
		if this.Flag&Ldate != 0 {
			out = append(out, fmt.Sprintf("%v-%02d-%02d", now.Year(), now.Month(), now.Day()))
		}
		if this.Flag&(Lmicroseconds) != 0 {
			out = append(out, fmt.Sprintf("%02d:%02d:%02d.%06d", now.Hour(), now.Minute(), now.Second(), now.Nanosecond()/1000000))
		} else if this.Flag&(Ltime) != 0 {
			out = append(out, fmt.Sprintf("%02d:%02d:%02d", now.Hour(), now.Minute(), now.Second()))
		}
	}

	// adding level info
	if this.Colored {
		slevel, ok = levelCStrings[level]
	} else {
		slevel, ok = levelStrings[level]
	}
	if !ok {
		slevel = strconv.Itoa(int(level))
	}
	out = append(out, slevel)

	out = append(out, this.Prefix)

	// adding caller info. It's quiet exepnsive
	if this.Flag&(Lshortfile|Llongfile) != 0 {
		if _, file, line, ok := runtime.Caller(2); ok { // 2: calldepth
			if this.Flag&Lshortfile != 0 {
				file = file[strings.LastIndex(file, "/")+1:]
			}
			out = append(out, fmt.Sprintf("%s:%d", file, line))
		} else {
			out = append(out, "???")
		}
	}

	out = append(out, msg)
	return []byte(strings.Join(out, " "))
}

/* LOGGER
 * ------
 */

// Adds a handler, specifying the maximum log Level
// you want to be written to this output. For instance,
// if you pass Warning for level, all logs of type
// Warning, Error, and Fatal would be logged to this handler.
func (this *Logger) AddHandler(writer io.Writer, level Level, fm Formatter) {
	this.mtx.Lock()
	this.handlers = append(this.handlers, handler{writer, level, fm})
	this.mtx.Unlock()
}

// Logs a message for the given level. Most callers will likely
// prefer to use one of the provided convenience functions (Debug, Info...).
func (this *Logger) Log(level Level, msg string) {
	var out []byte
	for _, h := range this.handlers {
		if h.level <= level {
			out = h.fmt.Format(level, msg)
			this.mtx.Lock()
			h.writer.Write(out)
			this.mtx.Unlock()
		}
	}
}

// Logs a formatted message message for the given level.
// Wrapper around Log method
func (this *Logger) Logf(level Level, format string, v ...interface{}) {
	this.Log(level, fmt.Sprintf(format+"\n", v...))
}

// Convenience function
func (this *Logger) Trace(format string, v ...interface{}) {
	// TODO: split the string
	this.Log(Levels.Trace, fmt.Sprintf(format+"\n", v...))
}

// Convenience function
func (this *Logger) Debug(format string, v ...interface{}) {
	this.Log(Levels.Debug, fmt.Sprintf(format+"\n", v...))
}

// Convenience function
func (this *Logger) Info(format string, v ...interface{}) {
	this.Log(Levels.Info, fmt.Sprintf(format+"\n", v...))
}

// Convenience function
func (this *Logger) Warning(format string, v ...interface{}) {
	this.Log(Levels.Warning, fmt.Sprintf(format+"\n", v...))
}

// Convenience function
func (this *Logger) Error(format string, v ...interface{}) {
	this.Log(Levels.Error, fmt.Sprintf(format+"\n", v...))
}

// Convenience function, will not terminate the program
func (this *Logger) Fatal(format string, v ...interface{}) {
	this.Log(Levels.Fatal, fmt.Sprintf(format+"\n", v...))
}
<|endoftext|>


<commit_before>// python like log package
// can use instead of standard log package
package log

import (
	"fmt"
	"log"
	"os"
)

const (
	LL_Debug = 1 << iota
	LL_Info
	LL_Warn
	LL_Error
	LL_Fatal
	LL_All = LL_Debug | LL_Info | LL_Warn | LL_Error | LL_Fatal
)

var LogLevel2str = map[int]string{
	LL_Debug: "Debug",
	LL_Info:  "Info",
	LL_Warn:  "Warn",
	LL_Error: "Error",
	LL_Fatal: "Fatal",
}

type Log struct {
	loglevel int
	l        map[int]*log.Logger
}

func New(prefix string, loglevel int, release bool) Log {
	l := Log{
		loglevel: loglevel,
		l:        make(map[int]*log.Logger),
	}
	flags := log.LstdFlags
	if !release {
		flags = log.Ltime | log.Lshortfile
	}
	for i, v := range LogLevel2str {
		l.l[i] = log.New(os.Stderr, fmt.Sprintf("%v %v:", prefix, v), flags)
	}
	return l
}

func (l Log) String() string {
	levelstr := ""
	for i, v := range LogLevel2str {
		if l.IsLevel(i) {
			levelstr += v + " "
		}
	}
	return fmt.Sprintf("log level is %v", levelstr)
}
func (l *Log) AddLevel(level int) {
	l.loglevel |= level
}
func (l *Log) SetLevel(level int) {
	l.loglevel = level
}
func (l *Log) DelLevel(level int) {
	l.loglevel &= ^level
}
func (l *Log) IsLevel(level int) bool {
	return l.loglevel&level != 0
}

func (l Log) printf(ll int, format string, v ...interface{}) {
	if !l.IsLevel(ll) {
		return
	}
	l.l[ll].Output(3, fmt.Sprintf(format, v...))
}

func (l Log) Info(format string, v ...interface{}) {
	l.printf(LL_Info, format, v...)
}
func (l Log) Warn(format string, v ...interface{}) {
	l.printf(LL_Warn, format, v...)
}
func (l Log) Debug(format string, v ...interface{}) {
	l.printf(LL_Debug, format, v...)
}
func (l Log) Error(format string, v ...interface{}) {
	l.printf(LL_Error, format, v...)
}
func (l Log) Fatal(format string, v ...interface{}) {
	l.printf(LL_Fatal, format, v...)
	os.Exit(1)
}

// ===

var logger = New("", LL_All, false)

func LevelString() string {
	return logger.String()
}

func SetReleaseLogger() {
	logger = New("", LL_All, true)
}
func Printf(format string, v ...interface{}) {
	logger.printf(LL_Info, format, v...)
}
func Fatalf(format string, v ...interface{}) {
	logger.printf(LL_Fatal, format, v...)
	os.Exit(1)
}
func Info(format string, v ...interface{}) {
	logger.printf(LL_Info, format, v...)
}
func Warn(format string, v ...interface{}) {
	logger.printf(LL_Warn, format, v...)
}
func Debug(format string, v ...interface{}) {
	logger.printf(LL_Debug, format, v...)
}
func Error(format string, v ...interface{}) {
	logger.printf(LL_Error, format, v...)
}
func Fatal(format string, v ...interface{}) {
	logger.printf(LL_Fatal, format, v...)
	os.Exit(1)
}
func AddLevel(level int) {
	logger.loglevel |= level
}
func SetLevel(level int) {
	logger.loglevel = level
}
func DelLevel(level int) {
	logger.loglevel &= ^level
}
func IsLevel(level int) bool {
	return logger.loglevel&level != 0
}
<commit_msg>use *Log, default prefix<commit_after>// python like log package
// can use instead of standard log package
package log

import (
	"fmt"
	"log"
	"os"
)

const (
	LL_Debug = 1 << iota
	LL_Info
	LL_Warn
	LL_Error
	LL_Fatal
	LL_All = LL_Debug | LL_Info | LL_Warn | LL_Error | LL_Fatal
)

var LogLevel2str = map[int]string{
	LL_Debug: "Debug",
	LL_Info:  "Info",
	LL_Warn:  "Warn",
	LL_Error: "Error",
	LL_Fatal: "Fatal",
}

type Log struct {
	loglevel int
	l        map[int]*log.Logger
}

func New(prefix string, loglevel int, release bool) *Log {
	l := Log{
		loglevel: loglevel,
		l:        make(map[int]*log.Logger),
	}
	flags := log.LstdFlags
	if !release {
		flags = log.Ltime | log.Lshortfile
	}
	for i, v := range LogLevel2str {
		l.l[i] = log.New(os.Stderr, fmt.Sprintf("%v %v:", prefix, v), flags)
	}
	return &l
}

func (l Log) String() string {
	levelstr := ""
	for i, v := range LogLevel2str {
		if l.IsLevel(i) {
			levelstr += v + " "
		}
	}
	return fmt.Sprintf("log level is %v", levelstr)
}
func (l *Log) AddLevel(level int) {
	l.loglevel |= level
}
func (l *Log) SetLevel(level int) {
	l.loglevel = level
}
func (l *Log) DelLevel(level int) {
	l.loglevel &= ^level
}
func (l *Log) IsLevel(level int) bool {
	return l.loglevel&level != 0
}

func (l Log) printf(ll int, format string, v ...interface{}) {
	if !l.IsLevel(ll) {
		return
	}
	l.l[ll].Output(3, fmt.Sprintf(format, v...))
}

func (l Log) Info(format string, v ...interface{}) {
	l.printf(LL_Info, format, v...)
}
func (l Log) Warn(format string, v ...interface{}) {
	l.printf(LL_Warn, format, v...)
}
func (l Log) Debug(format string, v ...interface{}) {
	l.printf(LL_Debug, format, v...)
}
func (l Log) Error(format string, v ...interface{}) {
	l.printf(LL_Error, format, v...)
}
func (l Log) Fatal(format string, v ...interface{}) {
	l.printf(LL_Fatal, format, v...)
	os.Exit(1)
}

// ===

var logger = New("Global", LL_All, false)

func LevelString() string {
	return logger.String()
}

func SetReleaseLogger() {
	logger = New("", LL_All, true)
}
func Printf(format string, v ...interface{}) {
	logger.printf(LL_Info, format, v...)
}
func Fatalf(format string, v ...interface{}) {
	logger.printf(LL_Fatal, format, v...)
	os.Exit(1)
}
func Info(format string, v ...interface{}) {
	logger.printf(LL_Info, format, v...)
}
func Warn(format string, v ...interface{}) {
	logger.printf(LL_Warn, format, v...)
}
func Debug(format string, v ...interface{}) {
	logger.printf(LL_Debug, format, v...)
}
func Error(format string, v ...interface{}) {
	logger.printf(LL_Error, format, v...)
}
func Fatal(format string, v ...interface{}) {
	logger.printf(LL_Fatal, format, v...)
	os.Exit(1)
}
func AddLevel(level int) {
	logger.loglevel |= level
}
func SetLevel(level int) {
	logger.loglevel = level
}
func DelLevel(level int) {
	logger.loglevel &= ^level
}
func IsLevel(level int) bool {
	return logger.loglevel&level != 0
}
<|endoftext|>


<commit_before>package main

////////////////////////////////////////////////////////////////////////////////
// Copyright (c) 2016, Gianluca Fiore
//
//    This program is free software: you can redistribute it and/or modify
//    it under the terms of the GNU General Public License as published by
//    the Free Software Foundation, either version 3 of the License, or
//    (at your option) any later version.
//
////////////////////////////////////////////////////////////////////////////////

import (
	"flag"
	"fmt"
	"os"
	"regexp"
)

var mgnt_type string
var watchdir string

// Parse flags
func init() {

	const (
		def_watchdir	= "/media/private/torrent/watch"
		def_type		= "others"
	)

	flag.StringVar(&mgnt_type, "type", def_type, "")
	flag.StringVar(&mgnt_type, "t", def_type, "")
	flag.StringVar(&watchdir, "watchdir", def_watchdir, "")
	flag.StringVar(&watchdir, "w", def_watchdir, "")

}

func main() {
	var torrentFilename string

	// initialize cli arguments
	flag.Parse()

	// at least the magnet link is needed, bail out if it was not given
	if flag.NArg() == 0 {
		fmt.Println("No magnet link given, exiting...")
		os.Exit(1)
	}

	// Compile a couple of regexp we need
	var validMagnet = regexp.MustCompile(`xt=urn:btih:([^&/]+)`)
	var displayName = regexp.MustCompile(`dn=([^&/]+)`)

	if validMagnet.MatchString(flag.Arg(0)) {
		if displayName.MatchString(flag.Arg(0)) {
			torrentFilename = displayName.FindString(flag.Arg(0))
			// split at '='
			fileName := regexp.MustCompile(`=`).Split(torrentFilename, -1)[1]
			if (len(fileName) == 0) {

			fmt.Println(torrentFilename)
			fmt.Println(fileName)
		}
	} else {
		// not a valid magnet URI given
		fmt.Println("The magnet URI is not correct or unparseable")
		os.Exit(1)
	}
}
<commit_msg>use the xt param for links without a dn<commit_after>package main

////////////////////////////////////////////////////////////////////////////////
// Copyright (c) 2016, Gianluca Fiore
//
//    This program is free software: you can redistribute it and/or modify
//    it under the terms of the GNU General Public License as published by
//    the Free Software Foundation, either version 3 of the License, or
//    (at your option) any later version.
//
////////////////////////////////////////////////////////////////////////////////

import (
	"flag"
	"fmt"
	"os"
	"regexp"
)

var mgnt_type string
var watchdir string

// Parse flags
func init() {

	const (
		def_watchdir	= "/media/private/torrent/watch"
		def_type		= "others"
	)

	flag.StringVar(&mgnt_type, "type", def_type, "")
	flag.StringVar(&mgnt_type, "t", def_type, "")
	flag.StringVar(&watchdir, "watchdir", def_watchdir, "")
	flag.StringVar(&watchdir, "w", def_watchdir, "")

}

func main() {
	var torrentFilename string

	// initialize cli arguments
	flag.Parse()

	// at least the magnet link is needed, bail out if it was not given
	if flag.NArg() == 0 {
		fmt.Println("No magnet link given, exiting...")
		os.Exit(1)
	}

	// Compile a couple of regexp we need
	var validMagnet = regexp.MustCompile(`xt=urn:btih:([^&/]+)`)
	var displayName = regexp.MustCompile(`dn=([^&/]+)`)

	if validMagnet.MatchString(flag.Arg(0)) {
		if displayName.MatchString(flag.Arg(0)) {
			torrentFilename = displayName.FindString(flag.Arg(0))
			// split at '='
			fileName := regexp.MustCompile(`=`).Split(torrentFilename, -1)[1]
			if (len(fileName) == 0) {
				xt := validMagnet.FindString(flag.Arg(0))
				fileName = regexp.MustCompile(`:`).Split(xt, -1)[2]
			}
			fmt.Println(torrentFilename)
			fmt.Println(fileName)
		}
	} else {
		// not a valid magnet URI given
		fmt.Println("The magnet URI is not correct or unparseable")
		os.Exit(1)
	}
}
<|endoftext|>


<commit_before>package mrT

import (
	"bytes"
	"encoding/binary"
	"io"
	"os"
	"path"
	"sync"

	"github.com/itsmontoya/async/file"
	"github.com/itsmontoya/seeker"
	"github.com/missionMeteora/journaler"
	"github.com/missionMeteora/toolkit/errors"
	"github.com/missionMeteora/uuid"
)

const (
	// NilLine represents a zero-value for line types
	NilLine byte = iota
	// TransactionLine is the line with the transaction tag for the data lines following it
	TransactionLine
	// CommentLine is a comment line, will be ignored when parsing data
	// Note: This line type will always ignore middleware
	CommentLine
	// PutLine is for setting data
	PutLine
	// DeleteLine is for removing data
	DeleteLine
)

const (
	// ErrInvalidLine is returned when an invalid line is encountered while parsing
	ErrInvalidLine = errors.Error("invalid line")
)

// New will return a new instance of MrT
func New(dir, name string) (mp *MrT, err error) {
	var mrT MrT
	// Make the dirs needed for file
	if err = os.MkdirAll(dir, 0755); err != nil {
		return
	}

	if mrT.f, err = file.OpenFile(path.Join(dir, name+".tdb"), os.O_RDWR|os.O_CREATE, 0644); err != nil {
		return
	}

	mrT.dir = dir
	mrT.name = name

	mrT.ug = uuid.NewGen()
	mrT.buf = bytes.NewBuffer(nil)
	mrT.s = seeker.New(mrT.f)

	if err = mrT.s.SeekToEnd(); err != nil {
		return
	}

	mp = &mrT
	return
}

// MrT is Mister Transaction, he manages file transactions
// He also pities a fool
type MrT struct {
	mux sync.RWMutex

	dir  string
	name string
	// Copy on read
	cor bool

	f  *file.File
	s  *seeker.Seeker
	ug *uuid.Gen

	buf  *bytes.Buffer
	nbuf [8]byte

	closed bool
}

func (m *MrT) writeData(key, value []byte) {
	m.writeLine(PutLine, key, value)
}

func (m *MrT) writeBytes(b []byte) {
	binary.LittleEndian.PutUint64(m.nbuf[:], uint64(len(b)))
	m.buf.Write(m.nbuf[:])
	m.buf.Write(b)
}

func (m *MrT) writeLine(lineType byte, key, value []byte) {
	m.buf.WriteByte(lineType)
	m.writeBytes(key)
	m.writeBytes(value)
	m.buf.WriteByte('\n')
}

func (m *MrT) flush() (err error) {
	// Ensure we are at the end before flushing
	if err = m.s.SeekToEnd(); err != nil {
		return
	}

	_, err = io.Copy(m.f, m.buf)
	m.buf.Reset()
	return
}

func (m *MrT) rollback() {
	m.buf.Reset()
}

func (m *MrT) newCommitID() (commitID []byte) {
	return []byte(m.ug.New().String())
}

// Txn will create a transaction
func (m *MrT) Txn(fn TxnFn) (err error) {
	var txn Txn
	m.mux.Lock()
	defer m.mux.Unlock()

	txn.writeLine = m.writeLine
	defer txn.clear()

	m.writeLine(TransactionLine, m.newCommitID(), nil)

	if err = fn(&txn); err != nil {
		m.rollback()
		return
	}

	return m.flush()
}

// Comment will write a comment line
func (m *MrT) Comment(b []byte) (err error) {
	m.mux.Lock()
	defer m.mux.Unlock()

	// Write the comment line
	m.writeLine(CommentLine, b, nil)

	return m.flush()
}

// ForEach will iterate through all the file lines
func (m *MrT) ForEach(fn ForEachFn) (err error) {
	var key, value []byte
	m.mux.Lock()
	defer m.mux.Unlock()

	if err = m.s.SeekToStart(); err != nil {
		return
	}
	defer m.s.SeekToEnd()

	return m.s.ReadLines(func(buf *bytes.Buffer) (end bool) {
		b := buf.Bytes()
		switch b[0] {
		case TransactionLine, CommentLine:
		case PutLine, DeleteLine:
			if m.cor {
				key, value = getKVSafe(b)
			} else {
				key, value = getKV(b)
			}

			return fn(b[0], key, value)

		default:
			err = ErrInvalidLine
			return true
		}

		return
	})
}

// Close will close MrT
func (m *MrT) Close() (err error) {
	m.mux.Lock()
	defer m.mux.Unlock()
	if m.closed {
		return errors.ErrIsClosed
	}

	m.closed = true
	m.buf = nil
	m.s = nil
	m.ug = nil
	return m.f.Close()
}

// TxnInfo is information about a transaction
type TxnInfo struct {
	// Transaction id
	ID string `json:"id"`
	// Timestamp of transaction
	TS int64 `json:"ts"`
	// List of actions
	Actions []*ActionInfo `json:"actions"`
}

func newActionInfo(put bool, key, value []byte) *ActionInfo {
	var a ActionInfo
	a.Put = put
	a.Key = string(key)
	a.Value = string(value)
	return &a
}

// ActionInfo is information about an action
type ActionInfo struct {
	Put   bool   `json:"put"`
	Key   string `json:"key"`
	Value string `json:"value"`
}

// ForEachTxn will iterate through all the file transactions
func (m *MrT) ForEachTxn(fn ForEachTxnFn) (err error) {
	var (
		ti    *TxnInfo
		txnID []byte
		key   []byte
		value []byte
	)

	m.mux.Lock()
	defer m.mux.Unlock()

	if err = m.s.SeekToStart(); err != nil {
		return
	}
	defer m.s.SeekToEnd()

	if err = m.s.ReadLines(func(buf *bytes.Buffer) (end bool) {
		b := buf.Bytes()
		// Switch on the first byte (line indicator)
		switch b[0] {
		case TransactionLine:
			if ti != nil {
				// A transaction item already exists, let's pass it to the func!
				fn(ti)
			}

			// Extract transaction id from the key
			txnID, _ = getKV(b)

			// Parse uuid from transaction id
			tu, err := uuid.ParseStr(string(txnID))
			if err != nil {
				// Something is definitely wrong here
				// TODO: Handle error logging
				journaler.Error("Error parsing transaction: %v", err)
				return
			}

			ti = &TxnInfo{
				ID: string(txnID),
				TS: tu.Time().Unix(),
			}

		case CommentLine:
		case PutLine, DeleteLine:
			if ti == nil {
				return
			}

			key, value = getKV(b)
			ti.Actions = append(ti.Actions, newActionInfo(b[0] == PutLine, key, value))

		default:
			err = ErrInvalidLine
			return true
		}

		return
	}); err != nil {
		return
	}

	if ti != nil {
		fn(ti)
	}

	return
}

// Archive will archive the current data
func (m *MrT) Archive(populate TxnFn) (err error) {
	var (
		af  *file.File
		txn Txn
	)

	m.mux.Lock()
	defer m.mux.Unlock()
	defer m.buf.Reset()

	txn.writeLine = m.writeLine
	defer txn.clear()

	if err = populate(&txn); err != nil {
		return
	}

	// Open our archive file as an appending file
	if af, err = file.OpenFile(path.Join(m.dir, m.name+".archive.tdb"), os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0644); err != nil {
		return
	}
	defer af.Close()

	if err = m.s.SeekToStart(); err != nil {
		return
	}

	// Get the first commit
	if err = m.s.ReadLines(getFirstCommit); err != nil {
		return
	}

	// Move back a line so we can include the first commit
	if err = m.s.PrevLine(); err != nil {
		return
	}

	if _, err = io.Copy(af, m.f); err != nil {
		return
	}

	if err = m.f.Close(); err != nil {
		return
	}

	if m.f, err = file.Create(path.Join(m.dir, m.name+".tdb")); err != nil {
		return
	}

	return m.flush()
}

// ForEachFn is used for iterating through entries
type ForEachFn func(lineType byte, key, value []byte) (end bool)

// ForEachTxnFn is used for iterating through transactions
type ForEachTxnFn func(ti *TxnInfo) (end bool)

// TxnFn is used for transactions
type TxnFn func(txn *Txn) error

func getFirstCommit(buf *bytes.Buffer) (end bool) {
	return buf.Bytes()[0] == TransactionLine
}

// getKV will extract the key and value from a payload
// Note: Will ignore the first byte as it's the line-type indicator. Feel free to pass the entire payload
func getKV(b []byte) (key, value []byte) {
	// Set index at 9 to accomodate 1 byte for line type and 8 bytes for key length
	idx := uint64(9)
	// Get key length
	lv := binary.LittleEndian.Uint64(b[1:idx])
	key = b[idx : lv+idx]

	// Increment index past our key bytes
	idx += lv
	// Get value length
	lv = binary.LittleEndian.Uint64(b[idx : idx+8])
	// Increment our index past the value length
	idx += 8

	// Get upper range in case we need to pack in data after the value
	value = b[idx : lv+idx]
	return
}

// getKVSafe will extract the key and value from a payload and apply copy on read
func getKVSafe(b []byte) (key, value []byte) {
	key, value = getKV(b)
	key = append([]byte{}, key...)
	value = append([]byte{}, value...)
	return
}
<commit_msg>Fixing spelling mistake<commit_after>package mrT

import (
	"bytes"
	"encoding/binary"
	"io"
	"os"
	"path"
	"sync"

	"github.com/itsmontoya/async/file"
	"github.com/itsmontoya/seeker"
	"github.com/missionMeteora/journaler"
	"github.com/missionMeteora/toolkit/errors"
	"github.com/missionMeteora/uuid"
)

const (
	// NilLine represents a zero-value for line types
	NilLine byte = iota
	// TransactionLine is the line with the transaction tag for the data lines following it
	TransactionLine
	// CommentLine is a comment line, will be ignored when parsing data
	// Note: This line type will always ignore middleware
	CommentLine
	// PutLine is for setting data
	PutLine
	// DeleteLine is for removing data
	DeleteLine
)

const (
	// ErrInvalidLine is returned when an invalid line is encountered while parsing
	ErrInvalidLine = errors.Error("invalid line")
)

// New will return a new instance of MrT
func New(dir, name string) (mp *MrT, err error) {
	var mrT MrT
	// Make the dirs needed for file
	if err = os.MkdirAll(dir, 0755); err != nil {
		return
	}

	if mrT.f, err = file.OpenFile(path.Join(dir, name+".tdb"), os.O_RDWR|os.O_CREATE, 0644); err != nil {
		return
	}

	mrT.dir = dir
	mrT.name = name

	mrT.ug = uuid.NewGen()
	mrT.buf = bytes.NewBuffer(nil)
	mrT.s = seeker.New(mrT.f)

	if err = mrT.s.SeekToEnd(); err != nil {
		return
	}

	mp = &mrT
	return
}

// MrT is Mister Transaction, he manages file transactions
// He also pities a fool
type MrT struct {
	mux sync.RWMutex

	dir  string
	name string
	// Copy on read
	cor bool

	f  *file.File
	s  *seeker.Seeker
	ug *uuid.Gen

	buf  *bytes.Buffer
	nbuf [8]byte

	closed bool
}

func (m *MrT) writeData(key, value []byte) {
	m.writeLine(PutLine, key, value)
}

func (m *MrT) writeBytes(b []byte) {
	binary.LittleEndian.PutUint64(m.nbuf[:], uint64(len(b)))
	m.buf.Write(m.nbuf[:])
	m.buf.Write(b)
}

func (m *MrT) writeLine(lineType byte, key, value []byte) {
	m.buf.WriteByte(lineType)
	m.writeBytes(key)
	m.writeBytes(value)
	m.buf.WriteByte('\n')
}

func (m *MrT) flush() (err error) {
	// Ensure we are at the end before flushing
	if err = m.s.SeekToEnd(); err != nil {
		return
	}

	_, err = io.Copy(m.f, m.buf)
	m.buf.Reset()
	return
}

func (m *MrT) rollback() {
	m.buf.Reset()
}

func (m *MrT) newCommitID() (commitID []byte) {
	return []byte(m.ug.New().String())
}

// Txn will create a transaction
func (m *MrT) Txn(fn TxnFn) (err error) {
	var txn Txn
	m.mux.Lock()
	defer m.mux.Unlock()

	txn.writeLine = m.writeLine
	defer txn.clear()

	m.writeLine(TransactionLine, m.newCommitID(), nil)

	if err = fn(&txn); err != nil {
		m.rollback()
		return
	}

	return m.flush()
}

// Comment will write a comment line
func (m *MrT) Comment(b []byte) (err error) {
	m.mux.Lock()
	defer m.mux.Unlock()

	// Write the comment line
	m.writeLine(CommentLine, b, nil)

	return m.flush()
}

// ForEach will iterate through all the file lines
func (m *MrT) ForEach(fn ForEachFn) (err error) {
	var key, value []byte
	m.mux.Lock()
	defer m.mux.Unlock()

	if err = m.s.SeekToStart(); err != nil {
		return
	}
	defer m.s.SeekToEnd()

	return m.s.ReadLines(func(buf *bytes.Buffer) (end bool) {
		b := buf.Bytes()
		switch b[0] {
		case TransactionLine, CommentLine:
		case PutLine, DeleteLine:
			if m.cor {
				key, value = getKVSafe(b)
			} else {
				key, value = getKV(b)
			}

			return fn(b[0], key, value)

		default:
			err = ErrInvalidLine
			return true
		}

		return
	})
}

// Close will close MrT
func (m *MrT) Close() (err error) {
	m.mux.Lock()
	defer m.mux.Unlock()
	if m.closed {
		return errors.ErrIsClosed
	}

	m.closed = true
	m.buf = nil
	m.s = nil
	m.ug = nil
	return m.f.Close()
}

// TxnInfo is information about a transaction
type TxnInfo struct {
	// Transaction id
	ID string `json:"id"`
	// Timestamp of transaction
	TS int64 `json:"ts"`
	// List of actions
	Actions []*ActionInfo `json:"actions"`
}

func newActionInfo(put bool, key, value []byte) *ActionInfo {
	var a ActionInfo
	a.Put = put
	a.Key = string(key)
	a.Value = string(value)
	return &a
}

// ActionInfo is information about an action
type ActionInfo struct {
	Put   bool   `json:"put"`
	Key   string `json:"key"`
	Value string `json:"value"`
}

// ForEachTxn will iterate through all the file transactions
func (m *MrT) ForEachTxn(fn ForEachTxnFn) (err error) {
	var (
		ti    *TxnInfo
		txnID []byte
		key   []byte
		value []byte
	)

	m.mux.Lock()
	defer m.mux.Unlock()

	if err = m.s.SeekToStart(); err != nil {
		return
	}
	defer m.s.SeekToEnd()

	if err = m.s.ReadLines(func(buf *bytes.Buffer) (end bool) {
		b := buf.Bytes()
		// Switch on the first byte (line indicator)
		switch b[0] {
		case TransactionLine:
			if ti != nil {
				// A transaction item already exists, let's pass it to the func!
				fn(ti)
			}

			// Extract transaction id from the key
			txnID, _ = getKV(b)

			// Parse uuid from transaction id
			tu, err := uuid.ParseStr(string(txnID))
			if err != nil {
				// Something is definitely wrong here
				// TODO: Handle error logging
				journaler.Error("Error parsing transaction: %v", err)
				return
			}

			ti = &TxnInfo{
				ID: string(txnID),
				TS: tu.Time().Unix(),
			}

		case CommentLine:
		case PutLine, DeleteLine:
			if ti == nil {
				return
			}

			key, value = getKV(b)
			ti.Actions = append(ti.Actions, newActionInfo(b[0] == PutLine, key, value))

		default:
			err = ErrInvalidLine
			return true
		}

		return
	}); err != nil {
		return
	}

	if ti != nil {
		fn(ti)
	}

	return
}

// Archive will archive the current data
func (m *MrT) Archive(populate TxnFn) (err error) {
	var (
		af  *file.File
		txn Txn
	)

	m.mux.Lock()
	defer m.mux.Unlock()
	defer m.buf.Reset()

	txn.writeLine = m.writeLine
	defer txn.clear()

	if err = populate(&txn); err != nil {
		return
	}

	// Open our archive file as an appending file
	if af, err = file.OpenFile(path.Join(m.dir, m.name+".archive.tdb"), os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0644); err != nil {
		return
	}
	defer af.Close()

	if err = m.s.SeekToStart(); err != nil {
		return
	}

	// Get the first commit
	if err = m.s.ReadLines(getFirstCommit); err != nil {
		return
	}

	// Move back a line so we can include the first commit
	if err = m.s.PrevLine(); err != nil {
		return
	}

	if _, err = io.Copy(af, m.f); err != nil {
		return
	}

	if err = m.f.Close(); err != nil {
		return
	}

	if m.f, err = file.Create(path.Join(m.dir, m.name+".tdb")); err != nil {
		return
	}

	return m.flush()
}

// ForEachFn is used for iterating through entries
type ForEachFn func(lineType byte, key, value []byte) (end bool)

// ForEachTxnFn is used for iterating through transactions
type ForEachTxnFn func(ti *TxnInfo) (end bool)

// TxnFn is used for transactions
type TxnFn func(txn *Txn) error

func getFirstCommit(buf *bytes.Buffer) (end bool) {
	return buf.Bytes()[0] == TransactionLine
}

// getKV will extract the key and value from a payload
// Note: Will ignore the first byte as it's the line-type indicator. Feel free to pass the entire payload
func getKV(b []byte) (key, value []byte) {
	// Set index at 9 to accommodate 1 byte for line type and 8 bytes for key length
	idx := uint64(9)
	// Get key length
	lv := binary.LittleEndian.Uint64(b[1:idx])
	key = b[idx : lv+idx]

	// Increment index past our key bytes
	idx += lv
	// Get value length
	lv = binary.LittleEndian.Uint64(b[idx : idx+8])
	// Increment our index past the value length
	idx += 8

	// Get upper range in case we need to pack in data after the value
	value = b[idx : lv+idx]
	return
}

// getKVSafe will extract the key and value from a payload and apply copy on read
func getKVSafe(b []byte) (key, value []byte) {
	key, value = getKV(b)
	key = append([]byte{}, key...)
	value = append([]byte{}, value...)
	return
}
<|endoftext|>