<commit_before>// Copyright 2015 Aaron Jacobs. All Rights Reserved.
// Author: aaronjjacobs@gmail.com (Aaron Jacobs)
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// A tool to add missing fields to an existing GCS bucket, for #18.
//
// Input is on stdin, and is of the form "<SHA-1> <CRC32C> <MD5>", e.g.:
//
//     e04b25d650dee1dff6ab1743724fa7c184282e94 0x12e9bf88 2bad5bb78f17232ef8c727f59eb82325
//
package main

import (
	"bufio"
	"crypto/md5"
	"crypto/sha1"
	"encoding/hex"
	"errors"
	"flag"
	"fmt"
	"io"
	"log"
	"net/http"
	"os"
	"regexp"
	"strconv"
	"strings"
	"time"

	"github.com/jacobsa/comeback/blob"
	"github.com/jacobsa/gcloud/gcs"
	"github.com/jacobsa/gcloud/oauthutil"
	"github.com/jacobsa/gcloud/syncutil"
	"golang.org/x/net/context"
)

var fKeyFile = flag.String("key_file", "", "")
var fBucket = flag.String("bucket", "", "")

type crc32cChecksum uint32
type md5Hash [md5.Size]byte
type sha1Hash [sha1.Size]byte

type checksums struct {
	crc32c crc32cChecksum
	md5    md5Hash
}

// A mapping from SHA-1 to CRC32C and MD5.
type checksumMap map[sha1Hash]checksums

const (
	// Cf. blob_store.go
	blobObjectNamePrefix = "blobs/"

	// Cf. gcs_store.go
	metadataKey_SHA1   = "comeback_sha1"
	metadataKey_CRC32C = "comeback_crc32c"
	metadataKey_MD5    = "comeback_md5"
)

var gInputLineRe = regexp.MustCompile(
	"^([0-9a-f]{40}) (0x[0-9a-f]{8}) ([0-9a-f]{32})$")

func parseInputLine(line []byte) (sha1 sha1Hash, c checksums, err error) {
	// Match against the regexp.
	matches := gInputLineRe.FindSubmatch(line)
	if matches == nil {
		err = errors.New("No match.")
		return
	}

	// Parse each component.
	_, err = hex.Decode(sha1[:], matches[1])
	if err != nil {
		panic(fmt.Sprintf("Unexpected decode error for %q: %v", matches[1], err))
	}

	crc32c64, err := strconv.ParseUint(string(matches[2]), 0, 32)
	if err != nil {
		panic(fmt.Sprintf("Unexpected decode error for %q: %v", matches[2], err))
	}

	c.crc32c = crc32cChecksum(crc32c64)

	_, err = hex.Decode(c.md5[:], matches[3])
	if err != nil {
		panic(fmt.Sprintf("Unexpected decode error for %q: %v", matches[3], err))
	}

	return
}

// Read the supplied input file, producing a checksum map.
func parseInput(in io.Reader) (m checksumMap, err error) {
	m = make(checksumMap)

	// Scan each line.
	scanner := bufio.NewScanner(in)
	for scanner.Scan() {
		var sha1 sha1Hash
		var c checksums
		sha1, c, err = parseInputLine(scanner.Bytes())
		if err != nil {
			err = fmt.Errorf("Parsing input line %q: %v", scanner.Text(), err)
			return
		}

		m[sha1] = c
	}

	// Was there an error scanning?
	if scanner.Err() != nil {
		err = fmt.Errorf("Scanning: %v", scanner.Err())
		return
	}

	return
}

// List all blob objects in the GCS bucket into the channel.
func listBlobObjects(
	ctx context.Context,
	bucket gcs.Bucket,
	objects chan<- *gcs.Object) (err error) {
	req := &gcs.ListObjectsRequest{
		Prefix: blobObjectNamePrefix,
	}

	// List until we run out.
	for {
		// Fetch the next batch.
		var listing *gcs.Listing
		listing, err = bucket.ListObjects(ctx, req)
		if err != nil {
			err = fmt.Errorf("ListObjects: %v", err)
			return
		}

		// Pass on each object.
		for _, o := range listing.Objects {
			// Special case: for gcsfuse compatibility, we allow blobObjectNamePrefix
			// to exist as its own object name. Skip it.
			if o.Name == blobObjectNamePrefix {
				continue
			}

			select {
			case objects <- o:

				// Cancelled?
			case <-ctx.Done():
				err = ctx.Err()
				return
			}
		}

		// Are we done?
		if listing.ContinuationToken == "" {
			break
		}

		req.ContinuationToken = listing.ContinuationToken
	}

	return
}

// Filter to names of objects that lack the appropriate metadata keys.
func filterToProblematicNames(
	ctx context.Context,
	objects <-chan *gcs.Object,
	names chan<- string) (err error) {
	for o := range objects {
		// Skip objects that already have all of the keys.
		_, ok0 := o.Metadata[metadataKey_SHA1]
		_, ok1 := o.Metadata[metadataKey_CRC32C]
		_, ok2 := o.Metadata[metadataKey_MD5]

		if ok0 && ok1 && ok2 {
			continue
		}

		// Pass on the names of others.
		select {
		case names <- o.Name:

			// Cancelled?
		case <-ctx.Done():
			err = ctx.Err()
			return
		}
	}

	return
}

// Parse the object name into its expected SHA-1 hash.
func parseObjectName(name string) (sha1 sha1Hash, err error) {
	if !strings.HasPrefix(name, blobObjectNamePrefix) {
		err = fmt.Errorf("Expected prefix")
		return
	}

	hexSha1 := strings.TrimPrefix(name, blobObjectNamePrefix)
	score, err := blob.ParseHexScore(hexSha1)
	if err != nil {
		err = fmt.Errorf("ParseHexScore: %v", err)
		return
	}

	sha1 = sha1Hash(score)
	return
}

// For each object name, issue a request to set the appropriate metadata keys
// based on the contents of the supplied map. Write out the names of the
// objects processed, and those for whom info wasn't available.
func fixProblematicObjects(
	ctx context.Context,
	bucket gcs.Bucket,
	info checksumMap,
	names <-chan string,
	processed chan<- string,
	unknown chan<- string) (err error) {
	for name := range names {
		// Parse the name.
		var sha1 sha1Hash
		sha1, err = parseObjectName(name)
		if err != nil {
			err = fmt.Errorf("Parsing object name %q: %v", name, err)
			return
		}

		// Do we have info for this object?
		c, ok := info[sha1]
		if !ok {
			select {
			case unknown <- name:

				// Cancelled?
			case <-ctx.Done():
				err = ctx.Err()
				return
			}

			continue
		}

		// Fix it up.
		//
		// Formats cf. gcs_store.go.
		sha1Str := hex.EncodeToString(sha1[:])
		crc32cStr := fmt.Sprintf("%#08x", c.crc32c)
		md5Str := hex.EncodeToString(c.md5[:])

		req := &gcs.UpdateObjectRequest{
			Name: name,
			Metadata: map[string]*string{
				metadataKey_SHA1:   &sha1Str,
				metadataKey_CRC32C: &crc32cStr,
				metadataKey_MD5:    &md5Str,
			},
		}

		_, err = bucket.UpdateObject(ctx, req)
		if err != nil {
			err = fmt.Errorf("UpdateObject: %v", err)
			return
		}

		// Pass on the name as processed.
		select {
		case processed <- name:

			// Cancelled?
		case <-ctx.Done():
			err = ctx.Err()
			return
		}
	}

	return
}

func run(
	bucket gcs.Bucket,
	info checksumMap) (err error) {
	b := syncutil.NewBundle(context.Background())
	defer func() { err = b.Join() }()

	// List all of the blob objects.
	objectRecords := make(chan *gcs.Object, 100)
	b.Add(func(ctx context.Context) (err error) {
		defer close(objectRecords)
		err = listBlobObjects(ctx, bucket, objectRecords)
		if err != nil {
			err = fmt.Errorf("listBlobObjects: %v", err)
			return
		}

		return
	})

	// Filter to the ones we need to fix up.
	problematicNames := make(chan string, 100)
	b.Add(func(ctx context.Context) (err error) {
		defer close(problematicNames)
		err = filterToProblematicNames(ctx, objectRecords, problematicNames)
		if err != nil {
			err = fmt.Errorf("filterToProblematicNames: %v", err)
			return
		}

		return
	})

	// Fix those objects with some parallelism.
	const parallelism = 128
	b.Add(func(ctx context.Context) (err error) {
		err = errors.New("TODO")
		return
	})

	// Log status updates, and at the end log the objects that were not
	// processed, returning an error if non-zero.
	b.Add(func(ctx context.Context) (err error) {
		err = errors.New("TODO")
		return
	})

	return
}

func panicIf(err *error) {
	if *err != nil {
		panic(*err)
	}
}

func getHTTPClient() (client *http.Client, err error) {
	if *fKeyFile == "" {
		err = errors.New("You must set --key_file.")
		return
	}

	const scope = gcs.Scope_FullControl
	client, err = oauthutil.NewJWTHttpClient(*fKeyFile, []string{scope})
	if err != nil {
		err = fmt.Errorf("oauthutil.NewJWTHttpClient: %v", err)
		return
	}

	return
}

func bucketName() (name string, err error) {
	name = *fBucket
	if name == "" {
		err = errors.New("You must set --bucket.")
		return
	}

	return
}

func getBucket() (b gcs.Bucket) {
	var err error
	defer panicIf(&err)

	// Get the HTTP client,
	client, err := getHTTPClient()
	if err != nil {
		err = fmt.Errorf("IntegrationTestHTTPClient: %v", err)
		return
	}

	// Find the bucket name.
	name, err := bucketName()
	if err != nil {
		err = fmt.Errorf("bucketName: %v", err)
		return
	}

	// Create a connection.
	cfg := &gcs.ConnConfig{
		HTTPClient:      client,
		MaxBackoffSleep: 30 * time.Second,
	}

	conn, err := gcs.NewConn(cfg)
	if err != nil {
		err = fmt.Errorf("gcs.NewConn: %v", err)
		return
	}

	// Extract the bucket.
	b = conn.GetBucket(name)

	return
}

func main() {
	flag.Parse()
	bucket := getBucket()
	log.SetFlags(log.Ldate | log.Ltime | log.Lmicroseconds)

	// Panic if anything below fails.
	var err error
	defer panicIf(&err)

	// Parse the input.
	log.Println("Parsing input...")
	info, err := parseInput(os.Stdin)
	if err != nil {
		err = fmt.Errorf("parseInput: %v", err)
		return
	}

	log.Println("Done parsing input.")

	// Run.
	err = run(bucket, info)
	if err != nil {
		err = fmt.Errorf("run: %v", err)
		return
	}
}
<commit_msg>Don't panic in main.<commit_after>// Copyright 2015 Aaron Jacobs. All Rights Reserved.
// Author: aaronjjacobs@gmail.com (Aaron Jacobs)
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// A tool to add missing fields to an existing GCS bucket, for #18.
//
// Input is on stdin, and is of the form "<SHA-1> <CRC32C> <MD5>", e.g.:
//
//     e04b25d650dee1dff6ab1743724fa7c184282e94 0x12e9bf88 2bad5bb78f17232ef8c727f59eb82325
//
package main

import (
	"bufio"
	"crypto/md5"
	"crypto/sha1"
	"encoding/hex"
	"errors"
	"flag"
	"fmt"
	"io"
	"log"
	"net/http"
	"os"
	"regexp"
	"strconv"
	"strings"
	"time"

	"github.com/jacobsa/comeback/blob"
	"github.com/jacobsa/gcloud/gcs"
	"github.com/jacobsa/gcloud/oauthutil"
	"github.com/jacobsa/gcloud/syncutil"
	"golang.org/x/net/context"
)

var fKeyFile = flag.String("key_file", "", "")
var fBucket = flag.String("bucket", "", "")

type crc32cChecksum uint32
type md5Hash [md5.Size]byte
type sha1Hash [sha1.Size]byte

type checksums struct {
	crc32c crc32cChecksum
	md5    md5Hash
}

// A mapping from SHA-1 to CRC32C and MD5.
type checksumMap map[sha1Hash]checksums

const (
	// Cf. blob_store.go
	blobObjectNamePrefix = "blobs/"

	// Cf. gcs_store.go
	metadataKey_SHA1   = "comeback_sha1"
	metadataKey_CRC32C = "comeback_crc32c"
	metadataKey_MD5    = "comeback_md5"
)

var gInputLineRe = regexp.MustCompile(
	"^([0-9a-f]{40}) (0x[0-9a-f]{8}) ([0-9a-f]{32})$")

func parseInputLine(line []byte) (sha1 sha1Hash, c checksums, err error) {
	// Match against the regexp.
	matches := gInputLineRe.FindSubmatch(line)
	if matches == nil {
		err = errors.New("No match.")
		return
	}

	// Parse each component.
	_, err = hex.Decode(sha1[:], matches[1])
	if err != nil {
		panic(fmt.Sprintf("Unexpected decode error for %q: %v", matches[1], err))
	}

	crc32c64, err := strconv.ParseUint(string(matches[2]), 0, 32)
	if err != nil {
		panic(fmt.Sprintf("Unexpected decode error for %q: %v", matches[2], err))
	}

	c.crc32c = crc32cChecksum(crc32c64)

	_, err = hex.Decode(c.md5[:], matches[3])
	if err != nil {
		panic(fmt.Sprintf("Unexpected decode error for %q: %v", matches[3], err))
	}

	return
}

// Read the supplied input file, producing a checksum map.
func parseInput(in io.Reader) (m checksumMap, err error) {
	m = make(checksumMap)

	// Scan each line.
	scanner := bufio.NewScanner(in)
	for scanner.Scan() {
		var sha1 sha1Hash
		var c checksums
		sha1, c, err = parseInputLine(scanner.Bytes())
		if err != nil {
			err = fmt.Errorf("Parsing input line %q: %v", scanner.Text(), err)
			return
		}

		m[sha1] = c
	}

	// Was there an error scanning?
	if scanner.Err() != nil {
		err = fmt.Errorf("Scanning: %v", scanner.Err())
		return
	}

	return
}

// List all blob objects in the GCS bucket into the channel.
func listBlobObjects(
	ctx context.Context,
	bucket gcs.Bucket,
	objects chan<- *gcs.Object) (err error) {
	req := &gcs.ListObjectsRequest{
		Prefix: blobObjectNamePrefix,
	}

	// List until we run out.
	for {
		// Fetch the next batch.
		var listing *gcs.Listing
		listing, err = bucket.ListObjects(ctx, req)
		if err != nil {
			err = fmt.Errorf("ListObjects: %v", err)
			return
		}

		// Pass on each object.
		for _, o := range listing.Objects {
			// Special case: for gcsfuse compatibility, we allow blobObjectNamePrefix
			// to exist as its own object name. Skip it.
			if o.Name == blobObjectNamePrefix {
				continue
			}

			select {
			case objects <- o:

				// Cancelled?
			case <-ctx.Done():
				err = ctx.Err()
				return
			}
		}

		// Are we done?
		if listing.ContinuationToken == "" {
			break
		}

		req.ContinuationToken = listing.ContinuationToken
	}

	return
}

// Filter to names of objects that lack the appropriate metadata keys.
func filterToProblematicNames(
	ctx context.Context,
	objects <-chan *gcs.Object,
	names chan<- string) (err error) {
	for o := range objects {
		// Skip objects that already have all of the keys.
		_, ok0 := o.Metadata[metadataKey_SHA1]
		_, ok1 := o.Metadata[metadataKey_CRC32C]
		_, ok2 := o.Metadata[metadataKey_MD5]

		if ok0 && ok1 && ok2 {
			continue
		}

		// Pass on the names of others.
		select {
		case names <- o.Name:

			// Cancelled?
		case <-ctx.Done():
			err = ctx.Err()
			return
		}
	}

	return
}

// Parse the object name into its expected SHA-1 hash.
func parseObjectName(name string) (sha1 sha1Hash, err error) {
	if !strings.HasPrefix(name, blobObjectNamePrefix) {
		err = fmt.Errorf("Expected prefix")
		return
	}

	hexSha1 := strings.TrimPrefix(name, blobObjectNamePrefix)
	score, err := blob.ParseHexScore(hexSha1)
	if err != nil {
		err = fmt.Errorf("ParseHexScore: %v", err)
		return
	}

	sha1 = sha1Hash(score)
	return
}

// For each object name, issue a request to set the appropriate metadata keys
// based on the contents of the supplied map. Write out the names of the
// objects processed, and those for whom info wasn't available.
func fixProblematicObjects(
	ctx context.Context,
	bucket gcs.Bucket,
	info checksumMap,
	names <-chan string,
	processed chan<- string,
	unknown chan<- string) (err error) {
	for name := range names {
		// Parse the name.
		var sha1 sha1Hash
		sha1, err = parseObjectName(name)
		if err != nil {
			err = fmt.Errorf("Parsing object name %q: %v", name, err)
			return
		}

		// Do we have info for this object?
		c, ok := info[sha1]
		if !ok {
			select {
			case unknown <- name:

				// Cancelled?
			case <-ctx.Done():
				err = ctx.Err()
				return
			}

			continue
		}

		// Fix it up.
		//
		// Formats cf. gcs_store.go.
		sha1Str := hex.EncodeToString(sha1[:])
		crc32cStr := fmt.Sprintf("%#08x", c.crc32c)
		md5Str := hex.EncodeToString(c.md5[:])

		req := &gcs.UpdateObjectRequest{
			Name: name,
			Metadata: map[string]*string{
				metadataKey_SHA1:   &sha1Str,
				metadataKey_CRC32C: &crc32cStr,
				metadataKey_MD5:    &md5Str,
			},
		}

		_, err = bucket.UpdateObject(ctx, req)
		if err != nil {
			err = fmt.Errorf("UpdateObject: %v", err)
			return
		}

		// Pass on the name as processed.
		select {
		case processed <- name:

			// Cancelled?
		case <-ctx.Done():
			err = ctx.Err()
			return
		}
	}

	return
}

func run(
	bucket gcs.Bucket,
	info checksumMap) (err error) {
	b := syncutil.NewBundle(context.Background())
	defer func() { err = b.Join() }()

	// List all of the blob objects.
	objectRecords := make(chan *gcs.Object, 100)
	b.Add(func(ctx context.Context) (err error) {
		defer close(objectRecords)
		err = listBlobObjects(ctx, bucket, objectRecords)
		if err != nil {
			err = fmt.Errorf("listBlobObjects: %v", err)
			return
		}

		return
	})

	// Filter to the ones we need to fix up.
	problematicNames := make(chan string, 100)
	b.Add(func(ctx context.Context) (err error) {
		defer close(problematicNames)
		err = filterToProblematicNames(ctx, objectRecords, problematicNames)
		if err != nil {
			err = fmt.Errorf("filterToProblematicNames: %v", err)
			return
		}

		return
	})

	// Fix those objects with some parallelism.
	const parallelism = 128
	b.Add(func(ctx context.Context) (err error) {
		err = errors.New("TODO")
		return
	})

	// Log status updates, and at the end log the objects that were not
	// processed, returning an error if non-zero.
	b.Add(func(ctx context.Context) (err error) {
		err = errors.New("TODO")
		return
	})

	return
}

func panicIf(err *error) {
	if *err != nil {
		panic(*err)
	}
}

func getHTTPClient() (client *http.Client, err error) {
	if *fKeyFile == "" {
		err = errors.New("You must set --key_file.")
		return
	}

	const scope = gcs.Scope_FullControl
	client, err = oauthutil.NewJWTHttpClient(*fKeyFile, []string{scope})
	if err != nil {
		err = fmt.Errorf("oauthutil.NewJWTHttpClient: %v", err)
		return
	}

	return
}

func bucketName() (name string, err error) {
	name = *fBucket
	if name == "" {
		err = errors.New("You must set --bucket.")
		return
	}

	return
}

func getBucket() (b gcs.Bucket) {
	var err error
	defer panicIf(&err)

	// Get the HTTP client,
	client, err := getHTTPClient()
	if err != nil {
		err = fmt.Errorf("IntegrationTestHTTPClient: %v", err)
		return
	}

	// Find the bucket name.
	name, err := bucketName()
	if err != nil {
		err = fmt.Errorf("bucketName: %v", err)
		return
	}

	// Create a connection.
	cfg := &gcs.ConnConfig{
		HTTPClient:      client,
		MaxBackoffSleep: 30 * time.Second,
	}

	conn, err := gcs.NewConn(cfg)
	if err != nil {
		err = fmt.Errorf("gcs.NewConn: %v", err)
		return
	}

	// Extract the bucket.
	b = conn.GetBucket(name)

	return
}

func main() {
	flag.Parse()
	bucket := getBucket()
	log.SetFlags(log.Ldate | log.Ltime | log.Lmicroseconds)

	// Panic if anything below fails.
	var err error
	defer func() {
		if err != nil {
			log.Fatal(err)
		}
	}()

	// Parse the input.
	log.Println("Parsing input...")
	info, err := parseInput(os.Stdin)
	if err != nil {
		err = fmt.Errorf("parseInput: %v", err)
		return
	}

	log.Println("Done parsing input.")

	// Run.
	err = run(bucket, info)
	if err != nil {
		err = fmt.Errorf("run: %v", err)
		return
	}
}
<|endoftext|>


<commit_before>// Copyright (c) 2015 Monetas
// Copyright 2016 Daniel Krawisz.
// Use of this source code is governed by an ISC
// license that can be found in the LICENSE file.

package wire

import (
	"fmt"
	"io"
	"io/ioutil"
	"time"

	"github.com/DanielKrawisz/bmutil"
)

// MsgMsg implements the Message interface and represents a message sent between
// two addresses. It can be decrypted only by those that have the private
// encryption key that corresponds to the destination address.
type MsgMsg struct {
	Nonce              uint64
	ExpiresTime        time.Time
	ObjectType         ObjectType
	Version            uint64
	StreamNumber       uint64
	Encrypted          []byte
	FromAddressVersion uint64
	FromStreamNumber   uint64
	Behavior           uint32
	SigningKey         *PubKey
	EncryptionKey      *PubKey
	NonceTrials        uint64
	ExtraBytes         uint64
	Destination        *RipeHash
	Encoding           uint64
	Message            []byte
	Ack                []byte
	Signature          []byte
}

// Decode decodes r using the bitmessage protocol encoding into the receiver.
// This is part of the Message interface implementation.
func (msg *MsgMsg) Decode(r io.Reader) error {
	var err error
	msg.Nonce, msg.ExpiresTime, msg.ObjectType, msg.Version,
		msg.StreamNumber, err = DecodeMsgObjectHeader(r)
	if err != nil {
		return err
	}

	if msg.ObjectType != ObjectTypeMsg {
		str := fmt.Sprintf("Object Type should be %d, but is %d",
			ObjectTypeMsg, msg.ObjectType)
		return messageError("Decode", str)
	}

	msg.Encrypted, err = ioutil.ReadAll(r)

	return err
}

// Encode encodes the receiver to w using the bitmessage protocol encoding.
// This is part of the Message interface implementation.
func (msg *MsgMsg) Encode(w io.Writer) error {
	err := EncodeMsgObjectHeader(w, msg.Nonce, msg.ExpiresTime, msg.ObjectType,
		msg.Version, msg.StreamNumber)
	if err != nil {
		return err
	}

	_, err = w.Write(msg.Encrypted)
	return err
}

// Command returns the protocol command string for the message. This is part
// of the Message interface implementation.
func (msg *MsgMsg) Command() string {
	return CmdObject
}

// MaxPayloadLength returns the maximum length the payload can be for the
// receiver. This is part of the Message interface implementation.
func (msg *MsgMsg) MaxPayloadLength() int {
	return MaxPayloadOfMsgObject
}

func (msg *MsgMsg) String() string {
	return fmt.Sprintf("msg: v%d %d %s %d %x", msg.Version, msg.Nonce, msg.ExpiresTime, msg.StreamNumber, msg.Encrypted)
}

// EncodeForSigning encodes MsgMsg so that it can be hashed and signed.
func (msg *MsgMsg) EncodeForSigning(w io.Writer) error {
	err := EncodeMsgObjectSignatureHeader(w, msg.ExpiresTime, msg.ObjectType,
		msg.Version, msg.StreamNumber)
	if err != nil {
		return err
	}
	if err = bmutil.WriteVarInt(w, msg.FromAddressVersion); err != nil {
		return err
	}
	if err = bmutil.WriteVarInt(w, msg.FromStreamNumber); err != nil {
		return err
	}
	err = writeElements(w, msg.Behavior, msg.SigningKey, msg.EncryptionKey)
	if err != nil {
		return err
	}
	if msg.FromAddressVersion >= 3 {
		if err = bmutil.WriteVarInt(w, msg.NonceTrials); err != nil {
			return err
		}
		if err = bmutil.WriteVarInt(w, msg.ExtraBytes); err != nil {
			return err
		}
	}
	err = writeElement(w, msg.Destination)
	if err != nil {
		return err
	}
	if err = bmutil.WriteVarInt(w, msg.Encoding); err != nil {
		return err
	}
	msgLength := uint64(len(msg.Message))
	if err = bmutil.WriteVarInt(w, msgLength); err != nil {
		return err
	}
	if _, err := w.Write(msg.Message); err != nil {
		return err
	}
	ackLength := uint64(len(msg.Ack))
	if err = bmutil.WriteVarInt(w, ackLength); err != nil {
		return err
	}
	if _, err := w.Write(msg.Ack); err != nil {
		return err
	}
	return nil
}

// EncodeForEncryption encodes MsgMsg so that it can be encrypted.
func (msg *MsgMsg) EncodeForEncryption(w io.Writer) error {
	if err := bmutil.WriteVarInt(w, msg.FromAddressVersion); err != nil {
		return err
	}
	if err := bmutil.WriteVarInt(w, msg.FromStreamNumber); err != nil {
		return err
	}
	err := writeElements(w, msg.Behavior, msg.SigningKey, msg.EncryptionKey)
	if err != nil {
		return err
	}
	if msg.FromAddressVersion >= 3 {
		if err = bmutil.WriteVarInt(w, msg.NonceTrials); err != nil {
			return err
		}
		if err = bmutil.WriteVarInt(w, msg.ExtraBytes); err != nil {
			return err
		}
	}
	if err = writeElement(w, msg.Destination); err != nil {
		return err
	}
	if err = bmutil.WriteVarInt(w, msg.Encoding); err != nil {
		return err
	}
	msgLength := uint64(len(msg.Message))
	if err = bmutil.WriteVarInt(w, msgLength); err != nil {
		return err
	}
	if _, err := w.Write(msg.Message); err != nil {
		return err
	}
	ackLength := uint64(len(msg.Ack))
	if err = bmutil.WriteVarInt(w, ackLength); err != nil {
		return err
	}
	if _, err := w.Write(msg.Ack); err != nil {
		return err
	}
	sigLength := uint64(len(msg.Signature))
	if err = bmutil.WriteVarInt(w, sigLength); err != nil {
		return err
	}
	if _, err = w.Write(msg.Signature); err != nil {
		return err
	}
	return nil
}

// DecodeFromDecrypted decodes MsgMsg from its decrypted form.
func (msg *MsgMsg) DecodeFromDecrypted(r io.Reader) error {
	var err error
	if msg.FromAddressVersion, err = bmutil.ReadVarInt(r); err != nil {
		return err
	}
	if msg.FromStreamNumber, err = bmutil.ReadVarInt(r); err != nil {
		return err
	}
	msg.SigningKey = &PubKey{}
	msg.EncryptionKey = &PubKey{}
	err = readElements(r, &msg.Behavior, msg.SigningKey, msg.EncryptionKey)
	if err != nil {
		return err
	}
	if msg.FromAddressVersion >= 3 {
		if msg.NonceTrials, err = bmutil.ReadVarInt(r); err != nil {
			return err
		}
		if msg.ExtraBytes, err = bmutil.ReadVarInt(r); err != nil {
			return err
		}
	}
	msg.Destination = &RipeHash{}
	if err = readElement(r, msg.Destination); err != nil {
		return err
	}
	if msg.Encoding, err = bmutil.ReadVarInt(r); err != nil {
		return err
	}
	var msgLength uint64
	if msgLength, err = bmutil.ReadVarInt(r); err != nil {
		return err
	}
	if msgLength > MaxPayloadOfMsgObject {
		str := fmt.Sprintf("message length exceeds max length - "+
			"indicates %d, but max length is %d",
			msgLength, MaxPayloadOfMsgObject)
		return messageError("DecodeFromDecrypted", str)
	}
	msg.Message = make([]byte, msgLength)
	_, err = io.ReadFull(r, msg.Message)
	if err != nil {
		return err
	}
	var ackLength uint64
	if ackLength, err = bmutil.ReadVarInt(r); err != nil {
		return err
	}
	if ackLength > MaxPayloadOfMsgObject {
		str := fmt.Sprintf("ack length exceeds max length - "+
			"indicates %d, but max length is %d",
			msgLength, MaxPayloadOfMsgObject)
		return messageError("DecodeFromDecrypted", str)
	}
	msg.Ack = make([]byte, ackLength)
	_, err = io.ReadFull(r, msg.Ack)
	if err != nil {
		return err
	}
	var sigLength uint64
	if sigLength, err = bmutil.ReadVarInt(r); err != nil {
		return err
	}
	if sigLength > signatureMaxLength {
		str := fmt.Sprintf("signature length exceeds max length - "+
			"indicates %d, but max length is %d",
			sigLength, signatureMaxLength)
		return messageError("DecodeFromDecrypted", str)
	}
	msg.Signature = make([]byte, sigLength)
	_, err = io.ReadFull(r, msg.Signature)
	return err
}

// NewMsgMsg returns a new object message that conforms to the Message interface
// using the passed parameters and defaults for the remaining fields.
func NewMsgMsg(nonce uint64, expires time.Time, version, streamNumber uint64, encrypted []byte, addressVersion, fromStreamNumber uint64, behavior uint32, signingKey, encryptKey *PubKey, nonceTrials, extraBytes uint64, destination *RipeHash, encoding uint64, message, ack, signature []byte) *MsgMsg {
	return &MsgMsg{
		Nonce:              nonce,
		ExpiresTime:        expires,
		ObjectType:         ObjectTypeMsg,
		Version:            version,
		StreamNumber:       streamNumber,
		Encrypted:          encrypted,
		FromAddressVersion: addressVersion,
		FromStreamNumber:   fromStreamNumber,
		Behavior:           behavior,
		SigningKey:         signingKey,
		EncryptionKey:      encryptKey,
		NonceTrials:        nonceTrials,
		ExtraBytes:         extraBytes,
		Destination:        destination,
		Encoding:           encoding,
		Message:            message,
		Ack:                ack,
		Signature:          signature,
	}
}
<commit_msg>Add function DecodeMsgMsg.<commit_after>// Copyright (c) 2015 Monetas
// Copyright 2016 Daniel Krawisz.
// Use of this source code is governed by an ISC
// license that can be found in the LICENSE file.

package wire

import (
	"fmt"
	"io"
	"io/ioutil"
	"time"
	"bytes"

	"github.com/DanielKrawisz/bmutil"
)

// MsgMsg implements the Message interface and represents a message sent between
// two addresses. It can be decrypted only by those that have the private
// encryption key that corresponds to the destination address.
type MsgMsg struct {
	Nonce              uint64
	ExpiresTime        time.Time
	ObjectType         ObjectType
	Version            uint64
	StreamNumber       uint64
	Encrypted          []byte
	FromAddressVersion uint64
	FromStreamNumber   uint64
	Behavior           uint32
	SigningKey         *PubKey
	EncryptionKey      *PubKey
	NonceTrials        uint64
	ExtraBytes         uint64
	Destination        *RipeHash
	Encoding           uint64
	Message            []byte
	Ack                []byte
	Signature          []byte
}

// Decode decodes r using the bitmessage protocol encoding into the receiver.
// This is part of the Message interface implementation.
func (msg *MsgMsg) Decode(r io.Reader) error {
	var err error
	msg.Nonce, msg.ExpiresTime, msg.ObjectType, msg.Version,
		msg.StreamNumber, err = DecodeMsgObjectHeader(r)
	if err != nil {
		return err
	}

	if msg.ObjectType != ObjectTypeMsg {
		str := fmt.Sprintf("Object Type should be %d, but is %d",
			ObjectTypeMsg, msg.ObjectType)
		return messageError("Decode", str)
	}

	msg.Encrypted, err = ioutil.ReadAll(r)

	return err
}

// Encode encodes the receiver to w using the bitmessage protocol encoding.
// This is part of the Message interface implementation.
func (msg *MsgMsg) Encode(w io.Writer) error {
	err := EncodeMsgObjectHeader(w, msg.Nonce, msg.ExpiresTime, msg.ObjectType,
		msg.Version, msg.StreamNumber)
	if err != nil {
		return err
	}

	_, err = w.Write(msg.Encrypted)
	return err
}

// Command returns the protocol command string for the message. This is part
// of the Message interface implementation.
func (msg *MsgMsg) Command() string {
	return CmdObject
}

// MaxPayloadLength returns the maximum length the payload can be for the
// receiver. This is part of the Message interface implementation.
func (msg *MsgMsg) MaxPayloadLength() int {
	return MaxPayloadOfMsgObject
}

func (msg *MsgMsg) String() string {
	return fmt.Sprintf("msg: v%d %d %s %d %x", msg.Version, msg.Nonce, msg.ExpiresTime, msg.StreamNumber, msg.Encrypted)
}

// EncodeForSigning encodes MsgMsg so that it can be hashed and signed.
func (msg *MsgMsg) EncodeForSigning(w io.Writer) error {
	err := EncodeMsgObjectSignatureHeader(w, msg.ExpiresTime, msg.ObjectType,
		msg.Version, msg.StreamNumber)
	if err != nil {
		return err
	}
	if err = bmutil.WriteVarInt(w, msg.FromAddressVersion); err != nil {
		return err
	}
	if err = bmutil.WriteVarInt(w, msg.FromStreamNumber); err != nil {
		return err
	}
	err = writeElements(w, msg.Behavior, msg.SigningKey, msg.EncryptionKey)
	if err != nil {
		return err
	}
	if msg.FromAddressVersion >= 3 {
		if err = bmutil.WriteVarInt(w, msg.NonceTrials); err != nil {
			return err
		}
		if err = bmutil.WriteVarInt(w, msg.ExtraBytes); err != nil {
			return err
		}
	}
	err = writeElement(w, msg.Destination)
	if err != nil {
		return err
	}
	if err = bmutil.WriteVarInt(w, msg.Encoding); err != nil {
		return err
	}
	msgLength := uint64(len(msg.Message))
	if err = bmutil.WriteVarInt(w, msgLength); err != nil {
		return err
	}
	if _, err := w.Write(msg.Message); err != nil {
		return err
	}
	ackLength := uint64(len(msg.Ack))
	if err = bmutil.WriteVarInt(w, ackLength); err != nil {
		return err
	}
	if _, err := w.Write(msg.Ack); err != nil {
		return err
	}
	return nil
}

// EncodeForEncryption encodes MsgMsg so that it can be encrypted.
func (msg *MsgMsg) EncodeForEncryption(w io.Writer) error {
	if err := bmutil.WriteVarInt(w, msg.FromAddressVersion); err != nil {
		return err
	}
	if err := bmutil.WriteVarInt(w, msg.FromStreamNumber); err != nil {
		return err
	}
	err := writeElements(w, msg.Behavior, msg.SigningKey, msg.EncryptionKey)
	if err != nil {
		return err
	}
	if msg.FromAddressVersion >= 3 {
		if err = bmutil.WriteVarInt(w, msg.NonceTrials); err != nil {
			return err
		}
		if err = bmutil.WriteVarInt(w, msg.ExtraBytes); err != nil {
			return err
		}
	}
	if err = writeElement(w, msg.Destination); err != nil {
		return err
	}
	if err = bmutil.WriteVarInt(w, msg.Encoding); err != nil {
		return err
	}
	msgLength := uint64(len(msg.Message))
	if err = bmutil.WriteVarInt(w, msgLength); err != nil {
		return err
	}
	if _, err := w.Write(msg.Message); err != nil {
		return err
	}
	ackLength := uint64(len(msg.Ack))
	if err = bmutil.WriteVarInt(w, ackLength); err != nil {
		return err
	}
	if _, err := w.Write(msg.Ack); err != nil {
		return err
	}
	sigLength := uint64(len(msg.Signature))
	if err = bmutil.WriteVarInt(w, sigLength); err != nil {
		return err
	}
	if _, err = w.Write(msg.Signature); err != nil {
		return err
	}
	return nil
}

// DecodeFromDecrypted decodes MsgMsg from its decrypted form.
func (msg *MsgMsg) DecodeFromDecrypted(r io.Reader) error {
	var err error
	if msg.FromAddressVersion, err = bmutil.ReadVarInt(r); err != nil {
		return err
	}
	if msg.FromStreamNumber, err = bmutil.ReadVarInt(r); err != nil {
		return err
	}
	msg.SigningKey = &PubKey{}
	msg.EncryptionKey = &PubKey{}
	err = readElements(r, &msg.Behavior, msg.SigningKey, msg.EncryptionKey)
	if err != nil {
		return err
	}
	if msg.FromAddressVersion >= 3 {
		if msg.NonceTrials, err = bmutil.ReadVarInt(r); err != nil {
			return err
		}
		if msg.ExtraBytes, err = bmutil.ReadVarInt(r); err != nil {
			return err
		}
	}
	msg.Destination = &RipeHash{}
	if err = readElement(r, msg.Destination); err != nil {
		return err
	}
	if msg.Encoding, err = bmutil.ReadVarInt(r); err != nil {
		return err
	}
	var msgLength uint64
	if msgLength, err = bmutil.ReadVarInt(r); err != nil {
		return err
	}
	if msgLength > MaxPayloadOfMsgObject {
		str := fmt.Sprintf("message length exceeds max length - "+
			"indicates %d, but max length is %d",
			msgLength, MaxPayloadOfMsgObject)
		return messageError("DecodeFromDecrypted", str)
	}
	msg.Message = make([]byte, msgLength)
	_, err = io.ReadFull(r, msg.Message)
	if err != nil {
		return err
	}
	var ackLength uint64
	if ackLength, err = bmutil.ReadVarInt(r); err != nil {
		return err
	}
	if ackLength > MaxPayloadOfMsgObject {
		str := fmt.Sprintf("ack length exceeds max length - "+
			"indicates %d, but max length is %d",
			msgLength, MaxPayloadOfMsgObject)
		return messageError("DecodeFromDecrypted", str)
	}
	msg.Ack = make([]byte, ackLength)
	_, err = io.ReadFull(r, msg.Ack)
	if err != nil {
		return err
	}
	var sigLength uint64
	if sigLength, err = bmutil.ReadVarInt(r); err != nil {
		return err
	}
	if sigLength > signatureMaxLength {
		str := fmt.Sprintf("signature length exceeds max length - "+
			"indicates %d, but max length is %d",
			sigLength, signatureMaxLength)
		return messageError("DecodeFromDecrypted", str)
	}
	msg.Signature = make([]byte, sigLength)
	_, err = io.ReadFull(r, msg.Signature)
	return err
}

// NewMsgMsg returns a new object message that conforms to the Message interface
// using the passed parameters and defaults for the remaining fields.
func NewMsgMsg(nonce uint64, expires time.Time, version, streamNumber uint64, encrypted []byte, addressVersion, fromStreamNumber uint64, behavior uint32, signingKey, encryptKey *PubKey, nonceTrials, extraBytes uint64, destination *RipeHash, encoding uint64, message, ack, signature []byte) *MsgMsg {
	return &MsgMsg{
		Nonce:              nonce,
		ExpiresTime:        expires,
		ObjectType:         ObjectTypeMsg,
		Version:            version,
		StreamNumber:       streamNumber,
		Encrypted:          encrypted,
		FromAddressVersion: addressVersion,
		FromStreamNumber:   fromStreamNumber,
		Behavior:           behavior,
		SigningKey:         signingKey,
		EncryptionKey:      encryptKey,
		NonceTrials:        nonceTrials,
		ExtraBytes:         extraBytes,
		Destination:        destination,
		Encoding:           encoding,
		Message:            message,
		Ack:                ack,
		Signature:          signature,
	}
}

// DecodeMsgMsg takes a byte array and turns it into an object message.
func DecodeMsgMsg(obj []byte) (*MsgMsg, error) {
	// Make sure that object type specific checks happen first.
	var msg *MsgMsg
	err := msg.Decode(bytes.NewReader(obj))
	if err != nil {
		return nil, err
	}

	return msg, nil
}
<|endoftext|>


<commit_before>// Copyright 2014 The Gogs Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package repo

import (
	"strings"

	"github.com/codegangsta/martini"
	"github.com/slene/blackfriday"

	"github.com/gogits/gogs/models"
	"github.com/gogits/gogs/modules/middleware"
)

func Single(ctx *middleware.Context, params martini.Params) {
	if !ctx.Repo.IsValid {
		return
	}

	if params["branchname"] == "" {
		params["branchname"] = "master"
	}

	// Directory and file list.
	treename := params["_1"]
	files, err := models.GetReposFiles(params["username"], params["reponame"],
		params["branchname"], treename)
	if err != nil {
		ctx.Handle(200, "repo.Single", err)
		return
	}
	ctx.Data["Username"] = params["username"]
	ctx.Data["Reponame"] = params["reponame"]
	ctx.Data["Branchname"] = params["branchname"]

	// Branches.
	brs, err := models.GetBranches(params["username"], params["reponame"])
	if err != nil {
		ctx.Handle(200, "repo.Single", err)
		return
	}
	ctx.Data["Branches"] = brs

	var treenames []string
	Paths := make([]string, 0)

	if len(treename) > 0 {
		treenames = strings.Split(treename, "/")
		for i, _ := range treenames {
			Paths = append(Paths, strings.Join(treenames[0:i+1], "/"))
		}
	}

	// Latest commit.
	commit, err := models.GetLastestCommit(params["username"], params["reponame"])
	if err != nil {
		ctx.Handle(200, "repo.Single", err)
		return
	}
	ctx.Data["LatestCommit"] = commit

	// README.
	// for _, f := range files {
	// 	if f.Name == "README.md" {
	// ctx.Data["ReadmeName"] = "README.md"
	// ctx.Data["ReadmeContent"] =
	// 		break
	// 	}
	// }

	ctx.Data["Paths"] = Paths
	ctx.Data["Treenames"] = treenames
	ctx.Data["IsRepoToolbarSource"] = true
	ctx.Data["Files"] = files
	ctx.Render.HTML(200, "repo/single", ctx.Data)
}

func Setting(ctx *middleware.Context, params martini.Params) {
	if !ctx.Repo.IsValid {
		return
	}

	var title string
	if t, ok := ctx.Data["Title"].(string); ok {
		title = t
	}

	ctx.Data["Title"] = title + " - settings"
	ctx.Data["IsRepoToolbarSetting"] = true
	ctx.Render.HTML(200, "repo/setting", ctx.Data)
}

func Commits(ctx *middleware.Context) string {
	return "This is commits page"
}

func Issues(ctx *middleware.Context) string {
	return "This is issues page"
}

func Pulls(ctx *middleware.Context) string {
	return "This is pulls page"
}
<commit_msg>Mirror change<commit_after>// Copyright 2014 The Gogs Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package repo

import (
	"strings"

	"github.com/codegangsta/martini"
	// "github.com/slene/blackfriday"

	"github.com/gogits/gogs/models"
	"github.com/gogits/gogs/modules/middleware"
)

func Single(ctx *middleware.Context, params martini.Params) {
	if !ctx.Repo.IsValid {
		return
	}

	if params["branchname"] == "" {
		params["branchname"] = "master"
	}

	// Directory and file list.
	treename := params["_1"]
	files, err := models.GetReposFiles(params["username"], params["reponame"],
		params["branchname"], treename)
	if err != nil {
		ctx.Handle(200, "repo.Single", err)
		return
	}
	ctx.Data["Username"] = params["username"]
	ctx.Data["Reponame"] = params["reponame"]
	ctx.Data["Branchname"] = params["branchname"]

	// Branches.
	brs, err := models.GetBranches(params["username"], params["reponame"])
	if err != nil {
		ctx.Handle(200, "repo.Single", err)
		return
	}
	ctx.Data["Branches"] = brs

	var treenames []string
	Paths := make([]string, 0)

	if len(treename) > 0 {
		treenames = strings.Split(treename, "/")
		for i, _ := range treenames {
			Paths = append(Paths, strings.Join(treenames[0:i+1], "/"))
		}
	}

	// Latest commit.
	commit, err := models.GetLastestCommit(params["username"], params["reponame"])
	if err != nil {
		ctx.Handle(200, "repo.Single", err)
		return
	}
	ctx.Data["LatestCommit"] = commit

	// README.
	// for _, f := range files {
	// 	if f.Name == "README.md" {
	// ctx.Data["ReadmeName"] = "README.md"
	// ctx.Data["ReadmeContent"] =
	// 		break
	// 	}
	// }

	ctx.Data["Paths"] = Paths
	ctx.Data["Treenames"] = treenames
	ctx.Data["IsRepoToolbarSource"] = true
	ctx.Data["Files"] = files
	ctx.Render.HTML(200, "repo/single", ctx.Data)
}

func Setting(ctx *middleware.Context, params martini.Params) {
	if !ctx.Repo.IsValid {
		return
	}

	var title string
	if t, ok := ctx.Data["Title"].(string); ok {
		title = t
	}

	ctx.Data["Title"] = title + " - settings"
	ctx.Data["IsRepoToolbarSetting"] = true
	ctx.Render.HTML(200, "repo/setting", ctx.Data)
}

func Commits(ctx *middleware.Context) string {
	return "This is commits page"
}

func Issues(ctx *middleware.Context) string {
	return "This is issues page"
}

func Pulls(ctx *middleware.Context) string {
	return "This is pulls page"
}
<|endoftext|>


<commit_before>/*
 * Copyright 2011 Nan Deng
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 */

package uniqush

import (
	"goconf.googlecode.com/hg"
	"io"
	"os"
	"strings"
)

func (s *UniqushSystem) loadLogInfo(c *conf.ConfigFile, field string, prefix string) (*Logger, error) {
	var loglevel string
	var logswitch bool
	var err error
	var writer io.Writer

	logswitch, err = c.GetBool(field, "log")
	if err != nil {
		logswitch = true
	}

	writer = s.logfile
	if writer == nil {
		writer = os.Stderr
	}

	loglevel, err = c.GetString(field, "loglevel")
	if err != nil {
		loglevel = "standard"
	}
	var level int

	if logswitch {
		switch strings.ToLower(loglevel) {
		case "standard":
			level = LOGLEVEL_INFO
		case "verbose":
			level = LOGLEVEL_DEBUG
		default:
			level = LOGLEVEL_INFO
		}
	} else {
		level = LOGLEVEL_FATAL
	}

	logger := NewLogger(writer, prefix, level)
	return logger, nil
}

func loadDatabaseConfig(cf *conf.ConfigFile) (*DatabaseConfig, error) {
	var err error
	c := new(DatabaseConfig)
	c.psm = GetPushServiceManager()
	c.Engine, err = cf.GetString("Database", "engine")
	if err != nil || c.Engine == "" {
		c.Engine = "redis"
	}
	c.Name, err = cf.GetString("Database", "name")
	if err != nil || c.Name == "" {
		c.Name = "0"
	}
	c.Port, err = cf.GetInt("Database", "port")
	if err != nil || c.Port <= 0 {
		c.Port = -1
	}
	c.Host, err = cf.GetString("Database", "host")
	if err != nil || c.Host == "" {
		c.Host = "localhost"
	}
	c.Password, err = cf.GetString("Database", "password")
	if err != nil {
		c.Password = ""
	}
	i, e := cf.GetInt("Database", "everysec")
	c.EverySec = int64(i)
	if e != nil || c.EverySec <= 60 {
		c.EverySec = 600
	}
	c.LeastDirty, err = cf.GetInt("Database", "leastdirty")
	if err != nil || c.LeastDirty < 0 {
		c.LeastDirty = 10
	}
	c.CacheSize, err = cf.GetInt("Database", "cachesize")
	if err != nil || c.CacheSize < 0 {
		c.CacheSize = 1024
	}

	return c, nil
}

type UniqushSystem struct {
	Frontend UniqushFrontEnd
	Backend  UniqushBackEndIf
	Stopch   chan bool
	Bridge   chan *Request
	Database DatabaseFrontDeskIf
	psm      *PushServiceManager
	logfile  io.WriteCloser
}

var (
	defaultConfigFilePath string = "/etc/uniqush/uniqush.conf"
)

func (s *UniqushSystem) Finalize() {
	s.logfile.Close()
	s.Database.FlushCache()
	s.psm.Finalize()
}

func LoadUniqushSystem(filename string) (*UniqushSystem, error) {
	if filename == "" {
		filename = defaultConfigFilePath
	}
	c, err := conf.ReadConfigFile(filename)
	if err != nil {
		return nil, err
	}

	ret := new(UniqushSystem)
	ret.Stopch = make(chan bool)
	ret.Bridge = make(chan *Request)
	ret.logfile = os.Stderr
	ew := NewEventWriter(&NullWriter{})

	logfilename, err := c.GetString("default", "logfile")
	if err == nil && logfilename != "" {
		ret.logfile, err = os.OpenFile(logfilename,
			os.O_WRONLY|
				os.O_APPEND|
				os.O_CREATE, 0600)
		if err != nil {
			ret.logfile = os.Stderr
		}
	}

	logger, e10 := ret.loadLogInfo(c, "WebFrontend", "[WebFrontend]")
	if e10 != nil {
		return nil, e10
	}
	addr, e20 := c.GetString("WebFrontend", "addr")
	if e20 != nil || addr == "" {
		addr = "localhost:9898"
	}

	psm := GetPushServiceManager()
	ret.psm = psm

	ret.Frontend = NewWebFrontEnd(ret.Bridge, logger, addr, psm)
	ret.Frontend.SetStopChannel(ret.Stopch)
	ret.Frontend.SetEventWriter(ew)

	logger, e10 = ret.loadLogInfo(c, "Backend", "[Backend]")
	if e10 != nil {
		return nil, e10
	}
	ret.Backend = NewUniqushBackEnd(ret.Bridge, logger)

	var p RequestProcessor

	// Load Database
	dbconf, e0 := loadDatabaseConfig(c)
	if e0 != nil {
		return nil, e0
	}
	//dbf, e30 := NewDatabaseFrontDeskWithoutCache(dbconf)
	dbf, e30 := NewDatabaseFrontDesk(dbconf)

	if e30 != nil {
		return nil, e30
	}
	ret.Database = dbf

	// Load Processors
	logger, e10 = ret.loadLogInfo(c, "AddPushServiceProvider", "[AddPushServiceProvider]")
	if e10 != nil {
		return nil, e10
	}
	p = NewAddPushServiceProviderProcessor(logger, ew, dbf)
	ret.Backend.SetProcessor(ACTION_ADD_PUSH_SERVICE_PROVIDER, p)

	logger, e10 = ret.loadLogInfo(c, "RemovePushServiceProvider", "[RemovePushServiceProvider]")
	if e10 != nil {
		return nil, e10
	}
	p = NewRemovePushServiceProviderProcessor(logger, ew, dbf)
	ret.Backend.SetProcessor(ACTION_REMOVE_PUSH_SERVICE_PROVIDER, p)

	logger, e10 = ret.loadLogInfo(c, "Subscribe", "[Subscribe]")
	if e10 != nil {
		return nil, e10
	}
	p = NewSubscribeProcessor(logger, ew, dbf)
	ret.Backend.SetProcessor(ACTION_SUBSCRIBE, p)

	logger, e10 = ret.loadLogInfo(c, "Unsubscribe", "[Unsubscribe]")
	if e10 != nil {
		return nil, e10
	}
	p = NewUnsubscribeProcessor(logger, ew, dbf)
	ret.Backend.SetProcessor(ACTION_UNSUBSCRIBE, p)

	logger, e10 = ret.loadLogInfo(c, "Push", "[Push]")
	if e10 != nil {
		return nil, e10
	}
	p = NewPushProcessor(logger, ew, dbf, ret.Bridge, psm)
	ret.Backend.SetProcessor(ACTION_PUSH, p)
	return ret, nil
}

func (s *UniqushSystem) Run() {
	defer s.Finalize()
	go s.Frontend.Run()
	go s.Backend.Run()
	<-s.Stopch
}
<commit_msg>changed the goconf impor to monnand-goconf<commit_after>/*
 * Copyright 2011 Nan Deng
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 */

package uniqush

import (
	//"goconf.googlecode.com/hg"
	"monnand-goconf.googlecode.com/hg"
	"io"
	"os"
	"strings"
)

func (s *UniqushSystem) loadLogInfo(c *conf.ConfigFile, field string, prefix string) (*Logger, error) {
	var loglevel string
	var logswitch bool
	var err error
	var writer io.Writer

	logswitch, err = c.GetBool(field, "log")
	if err != nil {
		logswitch = true
	}

	writer = s.logfile
	if writer == nil {
		writer = os.Stderr
	}

	loglevel, err = c.GetString(field, "loglevel")
	if err != nil {
		loglevel = "standard"
	}
	var level int

	if logswitch {
		switch strings.ToLower(loglevel) {
		case "standard":
			level = LOGLEVEL_INFO
		case "verbose":
			level = LOGLEVEL_DEBUG
		default:
			level = LOGLEVEL_INFO
		}
	} else {
		level = LOGLEVEL_FATAL
	}

	logger := NewLogger(writer, prefix, level)
	return logger, nil
}

func loadDatabaseConfig(cf *conf.ConfigFile) (*DatabaseConfig, error) {
	var err error
	c := new(DatabaseConfig)
	c.psm = GetPushServiceManager()
	c.Engine, err = cf.GetString("Database", "engine")
	if err != nil || c.Engine == "" {
		c.Engine = "redis"
	}
	c.Name, err = cf.GetString("Database", "name")
	if err != nil || c.Name == "" {
		c.Name = "0"
	}
	c.Port, err = cf.GetInt("Database", "port")
	if err != nil || c.Port <= 0 {
		c.Port = -1
	}
	c.Host, err = cf.GetString("Database", "host")
	if err != nil || c.Host == "" {
		c.Host = "localhost"
	}
	c.Password, err = cf.GetString("Database", "password")
	if err != nil {
		c.Password = ""
	}
	i, e := cf.GetInt("Database", "everysec")
	c.EverySec = int64(i)
	if e != nil || c.EverySec <= 60 {
		c.EverySec = 600
	}
	c.LeastDirty, err = cf.GetInt("Database", "leastdirty")
	if err != nil || c.LeastDirty < 0 {
		c.LeastDirty = 10
	}
	c.CacheSize, err = cf.GetInt("Database", "cachesize")
	if err != nil || c.CacheSize < 0 {
		c.CacheSize = 1024
	}

	return c, nil
}

type UniqushSystem struct {
	Frontend UniqushFrontEnd
	Backend  UniqushBackEndIf
	Stopch   chan bool
	Bridge   chan *Request
	Database DatabaseFrontDeskIf
	psm      *PushServiceManager
	logfile  io.WriteCloser
}

var (
	defaultConfigFilePath string = "/etc/uniqush/uniqush.conf"
)

func (s *UniqushSystem) Finalize() {
	s.logfile.Close()
	s.Database.FlushCache()
	s.psm.Finalize()
}

func LoadUniqushSystem(filename string) (*UniqushSystem, error) {
	if filename == "" {
		filename = defaultConfigFilePath
	}
	c, err := conf.ReadConfigFile(filename)
	if err != nil {
		return nil, err
	}

	ret := new(UniqushSystem)
	ret.Stopch = make(chan bool)
	ret.Bridge = make(chan *Request)
	ret.logfile = os.Stderr
	ew := NewEventWriter(&NullWriter{})

	logfilename, err := c.GetString("default", "logfile")
	if err == nil && logfilename != "" {
		ret.logfile, err = os.OpenFile(logfilename,
			os.O_WRONLY|
				os.O_APPEND|
				os.O_CREATE, 0600)
		if err != nil {
			ret.logfile = os.Stderr
		}
	}

	logger, e10 := ret.loadLogInfo(c, "WebFrontend", "[WebFrontend]")
	if e10 != nil {
		return nil, e10
	}
	addr, e20 := c.GetString("WebFrontend", "addr")
	if e20 != nil || addr == "" {
		addr = "localhost:9898"
	}

	psm := GetPushServiceManager()
	ret.psm = psm

	ret.Frontend = NewWebFrontEnd(ret.Bridge, logger, addr, psm)
	ret.Frontend.SetStopChannel(ret.Stopch)
	ret.Frontend.SetEventWriter(ew)

	logger, e10 = ret.loadLogInfo(c, "Backend", "[Backend]")
	if e10 != nil {
		return nil, e10
	}
	ret.Backend = NewUniqushBackEnd(ret.Bridge, logger)

	var p RequestProcessor

	// Load Database
	dbconf, e0 := loadDatabaseConfig(c)
	if e0 != nil {
		return nil, e0
	}
	//dbf, e30 := NewDatabaseFrontDeskWithoutCache(dbconf)
	dbf, e30 := NewDatabaseFrontDesk(dbconf)

	if e30 != nil {
		return nil, e30
	}
	ret.Database = dbf

	// Load Processors
	logger, e10 = ret.loadLogInfo(c, "AddPushServiceProvider", "[AddPushServiceProvider]")
	if e10 != nil {
		return nil, e10
	}
	p = NewAddPushServiceProviderProcessor(logger, ew, dbf)
	ret.Backend.SetProcessor(ACTION_ADD_PUSH_SERVICE_PROVIDER, p)

	logger, e10 = ret.loadLogInfo(c, "RemovePushServiceProvider", "[RemovePushServiceProvider]")
	if e10 != nil {
		return nil, e10
	}
	p = NewRemovePushServiceProviderProcessor(logger, ew, dbf)
	ret.Backend.SetProcessor(ACTION_REMOVE_PUSH_SERVICE_PROVIDER, p)

	logger, e10 = ret.loadLogInfo(c, "Subscribe", "[Subscribe]")
	if e10 != nil {
		return nil, e10
	}
	p = NewSubscribeProcessor(logger, ew, dbf)
	ret.Backend.SetProcessor(ACTION_SUBSCRIBE, p)

	logger, e10 = ret.loadLogInfo(c, "Unsubscribe", "[Unsubscribe]")
	if e10 != nil {
		return nil, e10
	}
	p = NewUnsubscribeProcessor(logger, ew, dbf)
	ret.Backend.SetProcessor(ACTION_UNSUBSCRIBE, p)

	logger, e10 = ret.loadLogInfo(c, "Push", "[Push]")
	if e10 != nil {
		return nil, e10
	}
	p = NewPushProcessor(logger, ew, dbf, ret.Bridge, psm)
	ret.Backend.SetProcessor(ACTION_PUSH, p)
	return ret, nil
}

func (s *UniqushSystem) Run() {
	defer s.Finalize()
	go s.Frontend.Run()
	go s.Backend.Run()
	<-s.Stopch
}
<|endoftext|>


<commit_before>package main

import (
	"log"
	"os"
	"strconv"
	"time"

	"github.com/garyburd/redigo/redis"
	"github.com/lancetw/hcfd-forecast/db"
	"github.com/lancetw/hcfd-forecast/rain"
	"github.com/line/line-bot-sdk-go/linebot"
)

const timeZone = "Asia/Taipei"

var bot *linebot.Client

func main() {
	strID := os.Getenv("ChannelID")
	numID, err := strconv.ParseInt(strID, 10, 64)
	if err != nil {
		log.Fatal("Wrong environment setting about ChannelID")
	}
	bot, err = linebot.NewClient(numID, os.Getenv("ChannelSecret"), os.Getenv("MID"))
	if err != nil {
		log.Println("Bot:", bot, " err:", err)
	}

	for {
		log.Println("=== 查詢。開始 ===")

		c := db.Connect(os.Getenv("REDISTOGO_URL"))

		targets0 := []string{"新竹市"}
		msgs0, token0 := rain.GetRainingInfo(targets0, false)

		status0, getErr := redis.Int(c.Do("SISMEMBER", "token0", token0))
		if getErr != nil {
			if err != nil {
				log.Println(err)
			}
		}

		if status0 == 0 {
			users0, smembersErr := redis.Strings(c.Do("SMEMBERS", "user"))

			if smembersErr != nil {
				log.Println("GetRainingInfo SMEMBERS redis error", smembersErr)
			} else {
				local := time.Now()
				location, timeZoneErr := time.LoadLocation(timeZone)
				if timeZoneErr == nil {
					local = local.In(location)
				}
				for _, contentTo := range users0 {
					for _, msg := range msgs0 {
						_, err = bot.SendText([]string{contentTo}, msg)
						if err != nil {
							log.Println(err)
						}
					}
				}
			}
		}

		n0, addErr := c.Do("SADD", "token0", token0)
		if addErr != nil {
			log.Println("GetRainingInfo SADD to redis error", addErr, n0)
		}

		targets1 := []string{"新竹市", "新竹縣"}
		msgs1, token1 := rain.GetWarningInfo(targets1)

		status1, getErr := redis.Int(c.Do("SISMEMBER", "token1", token1))
		if getErr != nil {
			if err != nil {
				log.Println(err)
			}
		}

		if status1 == 0 {
			users1, smembersErr := redis.Strings(c.Do("SMEMBERS", "user"))

			if smembersErr != nil {
				log.Println("GetWarningInfo SMEMBERS redis error", smembersErr)
			} else {
				local := time.Now()
				location, err := time.LoadLocation(timeZone)
				if err == nil {
					local = local.In(location)
				}
				for _, contentTo := range users1 {
					for _, msg := range msgs1 {
						_, err = bot.SendText([]string{contentTo}, msg)
						if err != nil {
							log.Println(err)
						}
					}
				}
			}
		}

		n, addErr := c.Do("SADD", "token1", token1)
		if addErr != nil {
			log.Println("GetWarningInfo SADD to redis error", addErr, n)
		}

		defer c.Close()

		log.Println("=== 查詢。結束 ===")

		time.Sleep(60 * time.Second)
	}
}
<commit_msg>updated<commit_after>package main

import (
	"log"
	"os"
	"strconv"
	"time"

	"github.com/garyburd/redigo/redis"
	"github.com/lancetw/hcfd-forecast/db"
	"github.com/lancetw/hcfd-forecast/rain"
	"github.com/line/line-bot-sdk-go/linebot"
)

const timeZone = "Asia/Taipei"

var bot *linebot.Client

func main() {
	strID := os.Getenv("ChannelID")
	numID, err := strconv.ParseInt(strID, 10, 64)
	if err != nil {
		log.Fatal("Wrong environment setting about ChannelID")
	}
	bot, err = linebot.NewClient(numID, os.Getenv("ChannelSecret"), os.Getenv("MID"))
	if err != nil {
		log.Println("Bot:", bot, " err:", err)
	}

	for {
		log.Println("=== 查詢。開始 ===")

		c := db.Connect(os.Getenv("REDISTOGO_URL"))

		targets0 := []string{"新竹市"}
		msgs0, token0 := rain.GetRainingInfo(targets0, false)

		status0, getErr := redis.Int(c.Do("SISMEMBER", "token0", token0))
		if getErr != nil {
			if err != nil {
				log.Println(err)
			}
		}

		if status0 == 0 {
			users0, smembersErr := redis.Strings(c.Do("SMEMBERS", "user"))

			if smembersErr != nil {
				log.Println("GetRainingInfo SMEMBERS redis error", smembersErr)
			} else {
				local := time.Now()
				location, timeZoneErr := time.LoadLocation(timeZone)
				if timeZoneErr == nil {
					local = local.In(location)
				}
				for _, contentTo := range users0 {
					for _, msg := range msgs0 {
						_, err = bot.SendText([]string{contentTo}, msg)
						if err != nil {
							log.Println(err)
						}
					}
				}
			}
		}

		n0, addErr := c.Do("SADD", "token0", token0)
		if addErr != nil {
			log.Println("GetRainingInfo SADD to redis error", addErr, n0)
		}

		targets1 := []string{"新竹市", "新竹縣"}
		msgs1, token1 := rain.GetWarningInfo(targets1)

		if token1 != "" {
			status1, getErr := redis.Int(c.Do("SISMEMBER", "token1", token1))
			if getErr != nil {
				if err != nil {
					log.Println(err)
				}
			}

			if status1 == 0 {
				users1, smembersErr := redis.Strings(c.Do("SMEMBERS", "user"))

				if smembersErr != nil {
					log.Println("GetWarningInfo SMEMBERS redis error", smembersErr)
				} else {
					local := time.Now()
					location, locationErr := time.LoadLocation(timeZone)
					if locationErr == nil {
						local = local.In(location)
					}
					for _, contentTo := range users1 {
						for _, msg := range msgs1 {
							_, msgErr := bot.SendText([]string{contentTo}, msg)
							if msgErr != nil {
								log.Println(err)
							}
						}
					}
				}
			}
		}

		if token1 != "" {
			n, addErr := c.Do("SADD", "token1", token1)
			if addErr != nil {
				log.Println("GetWarningInfo SADD to redis error", addErr, n)
			}
		}

		defer c.Close()

		log.Println("=== 查詢。結束 ===")

		time.Sleep(60 * time.Second)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"log"
	"os"
	"strconv"
	"time"

	"github.com/garyburd/redigo/redis"
	"github.com/lancetw/hcfd-forecast/db"
	"github.com/lancetw/hcfd-forecast/rain"
	"github.com/line/line-bot-sdk-go/linebot"
)

const timeZone = "Asia/Taipei"

var bot *linebot.Client

func main() {
	strID := os.Getenv("ChannelID")
	numID, err := strconv.ParseInt(strID, 10, 64)
	if err != nil {
		log.Fatal("Wrong environment setting about ChannelID")
	}
	bot, err = linebot.NewClient(numID, os.Getenv("ChannelSecret"), os.Getenv("MID"))
	if err != nil {
		log.Println("Bot:", bot, " err:", err)
	}

	for {
		log.Println("=== 查詢。開始 ===")

		c := db.Connect(os.Getenv("REDISTOGO_URL"))

		targets0 := []string{"新竹市"}
		msgs0, token0 := rain.GetRainingInfo(targets0, false)

		status0, getErr := redis.Int(c.Do("SISMEMBER", "token0", token0))
		if getErr != nil {
			if err != nil {
				log.Println(err)
			}
		}

		if status0 == 0 {
			users0, smembersErr := redis.Strings(c.Do("SMEMBERS", "user"))

			if smembersErr != nil {
				log.Println("GetRainingInfo SMEMBERS redis error", smembersErr)
			} else {
				local := time.Now()
				location, timeZoneErr := time.LoadLocation(timeZone)
				if timeZoneErr == nil {
					local = local.In(location)
				}
				for _, contentTo := range users0 {
					for _, msg := range msgs0 {
						_, err = bot.SendText([]string{contentTo}, msg)
						if err != nil {
							log.Println(err)
						}
					}
				}
			}
		}

		n0, addErr := c.Do("SADD", "token0", token0)
		if addErr != nil {
			log.Println("GetRainingInfo SADD to redis error", addErr, n0)
		}

		targets1 := []string{"新竹市", "新竹縣"}
		msgs1, token1 := rain.GetWarningInfo(targets1)

		status1, getErr := redis.Int(c.Do("SISMEMBER", "token1", token1))
		if getErr != nil {
			if err != nil {
				log.Println(err)
			}
		}

		if status1 == 0 {
			users1, smembersErr := redis.Strings(c.Do("SMEMBERS", "user"))

			if smembersErr != nil {
				log.Println("GetWarningInfo SMEMBERS redis error", smembersErr)
			} else {
				local := time.Now()
				location, err := time.LoadLocation(timeZone)
				if err == nil {
					local = local.In(location)
				}
				for _, contentTo := range users1 {
					for _, msg := range msgs1 {
						_, err = bot.SendText([]string{contentTo}, msg)
						if err != nil {
							log.Println(err)
						}
					}
				}
			}
		}

		n, addErr := c.Do("SADD", "token1", token1)
		if addErr != nil {
			log.Println("GetWarningInfo SADD to redis error", addErr, n)
		}

		defer c.Close()

		log.Println("=== 查詢。結束 ===")

		time.Sleep(90 * time.Second)
	}
}
<commit_msg>update<commit_after>package main

import (
	"log"
	"os"
	"strconv"
	"time"

	"github.com/garyburd/redigo/redis"
	"github.com/lancetw/hcfd-forecast/db"
	"github.com/lancetw/hcfd-forecast/rain"
	"github.com/line/line-bot-sdk-go/linebot"
)

const timeZone = "Asia/Taipei"

var bot *linebot.Client

func main() {
	strID := os.Getenv("ChannelID")
	numID, err := strconv.ParseInt(strID, 10, 64)
	if err != nil {
		log.Fatal("Wrong environment setting about ChannelID")
	}
	bot, err = linebot.NewClient(numID, os.Getenv("ChannelSecret"), os.Getenv("MID"))
	if err != nil {
		log.Println("Bot:", bot, " err:", err)
	}

	for {
		log.Println("=== 查詢。開始 ===")

		c := db.Connect(os.Getenv("REDISTOGO_URL"))

		targets0 := []string{"新竹市"}
		msgs0, token0 := rain.GetRainingInfo(targets0, false)

		status0, getErr := redis.Int(c.Do("SISMEMBER", "token0", token0))
		if getErr != nil {
			if err != nil {
				log.Println(err)
			}
		}

		if status0 == 0 {
			users0, smembersErr := redis.Strings(c.Do("SMEMBERS", "user"))

			if smembersErr != nil {
				log.Println("GetRainingInfo SMEMBERS redis error", smembersErr)
			} else {
				local := time.Now()
				location, timeZoneErr := time.LoadLocation(timeZone)
				if timeZoneErr == nil {
					local = local.In(location)
				}
				for _, contentTo := range users0 {
					for _, msg := range msgs0 {
						_, err = bot.SendText([]string{contentTo}, msg)
						if err != nil {
							log.Println(err)
						}
					}
				}
			}
		}

		n0, addErr := c.Do("SADD", "token0", token0)
		if addErr != nil {
			log.Println("GetRainingInfo SADD to redis error", addErr, n0)
		}

		targets1 := []string{"新竹市", "新竹縣"}
		msgs1, token1 := rain.GetWarningInfo(targets1)

		status1, getErr := redis.Int(c.Do("SISMEMBER", "token1", token1))
		if getErr != nil {
			if err != nil {
				log.Println(err)
			}
		}

		if status1 == 0 {
			users1, smembersErr := redis.Strings(c.Do("SMEMBERS", "user"))

			if smembersErr != nil {
				log.Println("GetWarningInfo SMEMBERS redis error", smembersErr)
			} else {
				local := time.Now()
				location, err := time.LoadLocation(timeZone)
				if err == nil {
					local = local.In(location)
				}
				for _, contentTo := range users1 {
					for _, msg := range msgs1 {
						_, err = bot.SendText([]string{contentTo}, msg)
						if err != nil {
							log.Println(err)
						}
					}
				}
			}
		}

		n, addErr := c.Do("SADD", "token1", token1)
		if addErr != nil {
			log.Println("GetWarningInfo SADD to redis error", addErr, n)
		}

		defer c.Close()

		log.Println("=== 查詢。結束 ===")

		time.Sleep(30 * time.Second)
	}
}
<|endoftext|>


<commit_before>package stringsutil

import (
	"regexp"
	"strings"
	"time"
)

type MatchType int

const (
	Exact MatchType = iota
	TrimSpace
	TrimSpaceLower
	Regexp
	TimeGTE
	TimeLTE
)

type MatchInfo struct {
	MatchType  MatchType
	String     string
	Regexp     *regexp.Regexp
	TimeLayout string
	TimeMin    time.Time
	TimeMax    time.Time
}

// Match provides an canonical way to match strings using multiple
// approaches
func Match(s string, matchInfo MatchInfo) (bool, error) {
	switch matchInfo.MatchType {
	case Exact:
		if s == matchInfo.String {
			return true, nil
		}
		return false, nil
	case TrimSpace:
		m := strings.TrimSpace(s)
		if m == strings.TrimSpace(matchInfo.String) {
			return true, nil
		}
		return false, nil
	case TrimSpaceLower:
		m := strings.ToLower(strings.TrimSpace(s))
		if m == strings.ToLower(strings.TrimSpace(matchInfo.String)) {
			return true, nil
		}
		return false, nil
	case Regexp:
		if matchInfo.Regexp == nil {
			return false, nil
		}
		return matchInfo.Regexp.MatchString(s), nil
	case TimeGTE:
		t, err := time.Parse(matchInfo.TimeLayout, s)
		if err != nil {
			return false, err
		}
		if t.After(matchInfo.TimeMin) || t.Equal(matchInfo.TimeMin) {
			return true, nil
		} else {
			return false, nil
		}
	default:
		return false, nil
	}
}
<commit_msg>add stringsutil.Match conditions<commit_after>package stringsutil

import (
	"regexp"
	"strings"
	"time"
)

type MatchType int

const (
	Exact MatchType = iota
	TrimSpace
	TrimSpaceLower
	Regexp
	TimeGT
	TimeGTE
	TimeLT
	TimeLTE
)

type MatchInfo struct {
	MatchType  MatchType
	String     string
	Regexp     *regexp.Regexp
	TimeLayout string
	TimeMin    time.Time
	TimeMax    time.Time
}

// Match provides an canonical way to match strings using multiple
// approaches
func Match(s string, matchInfo MatchInfo) (bool, error) {
	switch matchInfo.MatchType {
	case Exact:
		if s == matchInfo.String {
			return true, nil
		}
		return false, nil
	case TrimSpace:
		m := strings.TrimSpace(s)
		if m == strings.TrimSpace(matchInfo.String) {
			return true, nil
		}
		return false, nil
	case TrimSpaceLower:
		m := strings.ToLower(strings.TrimSpace(s))
		if m == strings.ToLower(strings.TrimSpace(matchInfo.String)) {
			return true, nil
		}
		return false, nil
	case Regexp:
		if matchInfo.Regexp == nil {
			return false, nil
		}
		return matchInfo.Regexp.MatchString(s), nil
	case TimeGT:
		t, err := time.Parse(matchInfo.TimeLayout, s)
		if err != nil {
			return false, err
		}
		if t.After(matchInfo.TimeMin) {
			return true, nil
		} else {
			return false, nil
		}
	case TimeGTE:
		t, err := time.Parse(matchInfo.TimeLayout, s)
		if err != nil {
			return false, err
		}
		if t.After(matchInfo.TimeMin) || t.Equal(matchInfo.TimeMin) {
			return true, nil
		} else {
			return false, nil
		}
	case TimeLT:
		t, err := time.Parse(matchInfo.TimeLayout, s)
		if err != nil {
			return false, err
		}
		if t.Before(matchInfo.TimeMin) {
			return true, nil
		} else {
			return false, nil
		}
	case TimeLTE:
		t, err := time.Parse(matchInfo.TimeLayout, s)
		if err != nil {
			return false, err
		}
		if t.Before(matchInfo.TimeMin) || t.Equal(matchInfo.TimeMin) {
			return true, nil
		} else {
			return false, nil
		}
	default:
		return false, nil
	}
}
<|endoftext|>


<commit_before>// Copyright 2014 Brighcove Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License"): you may
// not use this file except in compliance with the License. You may obtain
// a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
// License for the specific language governing permissions and limitations
// under the License.

// Package rq provides a simple queue abstraction that is backed by Redis.
package rq

import (
	"testing"
)

func TestQueueConnectSuccessful(t *testing.T) {
	_, err := QueueConnect(":6379", "rq_test_queue")
	if err != nil {
		t.Error("Error while connecting to Redis", err)
	}
}

func TestQueueConnectFailure(t *testing.T) {
	_, err := QueueConnect(":123", "rq_test_queue")
	if err == nil {
		t.Error("Expected error connecting to Redis")
	}
}

func TestQueueDisconnectSuccessful(t *testing.T) {
	q, err := QueueConnect(":6379", "rq_test_queue")
	q.Disconnect()
	if err != nil {
		t.Error("Error while disconnecting from Redis", err)
	}
}

func TestQueuePushSuccessful(t *testing.T) {
	q, _ := QueueConnect(":6379", "rq_test_queue")
	err := q.Push("foo")
	if err != nil {
		t.Error("Error while pushing to Redis queue", err)
	}
	q.Disconnect()
}

func TestQueuePopSuccessful(t *testing.T) {
	q, _ := QueueConnect(":6379", "rq_test_pop_queue")
	q.Push("foo")
	q.Push("bar")

	var value string
	var err error
	value, err = q.Pop(1)
	if value != "foo" {
		t.Error("Expected foo but got: ", value)
	}
	if err != nil {
		t.Error("Unexpected error: ", err)
	}

	value, err = q.Pop(1)
	if value != "bar" {
		t.Error("Expected bar but got: ", value)
	}
	if err != nil {
		t.Error("Unexpected error: ", err)
	}
	q.Disconnect()
}

func BenchmarkQueueConnectDisconnect(b *testing.B) {
	for i := 0; i < b.N; i++ {
		q, _ := QueueConnect(":6379", "rq_test_queue")
		q.Disconnect()
	}
}

func BenchmarkQueuePush(b *testing.B) {
	q, _ := QueueConnect(":6379", "rq_test_queue")
	for i := 0; i < b.N; i++ {
		q.Push("foo")
	}
	q.Disconnect()
}
<commit_msg>More tests<commit_after>// Copyright 2014 Brighcove Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License"): you may
// not use this file except in compliance with the License. You may obtain
// a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
// License for the specific language governing permissions and limitations
// under the License.

// Package rq provides a simple queue abstraction that is backed by Redis.
package rq

import (
	"testing"
)

func TestQueueConnectSuccessful(t *testing.T) {
	_, err := QueueConnect(":6379", "rq_test_queue")
	if err != nil {
		t.Error("Error while connecting to Redis", err)
	}
}

func TestQueueConnectFailure(t *testing.T) {
	_, err := QueueConnect(":123", "rq_test_queue")
	if err == nil {
		t.Error("Expected error connecting to Redis")
	}
}

func TestQueueDisconnectSuccessful(t *testing.T) {
	q, err := QueueConnect(":6379", "rq_test_queue")
	q.Disconnect()
	if err != nil {
		t.Error("Error while disconnecting from Redis", err)
	}
}

func TestQueuePushSuccessful(t *testing.T) {
	q, _ := QueueConnect(":6379", "rq_test_queue")
	err := q.Push("foo")
	if err != nil {
		t.Error("Error while pushing to Redis queue", err)
	}
	q.Disconnect()
}

func TestQueuePopSuccessful(t *testing.T) {
	q, _ := QueueConnect(":6379", "rq_test_pop_queue")
	q.Push("foo")
	q.Push("bar")

	var value string
	var err error
	value, err = q.Pop(1)
	if value != "foo" {
		t.Error("Expected foo but got: ", value)
	}
	if err != nil {
		t.Error("Unexpected error: ", err)
	}

	value, err = q.Pop(1)
	if value != "bar" {
		t.Error("Expected bar but got: ", value)
	}
	if err != nil {
		t.Error("Unexpected error: ", err)
	}
	q.Disconnect()
}

func TestQueueLengthSuccessful(t *testing.T) {
	q, _ := QueueConnect(":6379", "rq_test_queue_length")

	l, err := q.Length()
	if l != 0 {
		t.Error("Expect length to be 0, was: ", l)
	}
	if err != nil {
		t.Error("Error while getting length of Redis queue", err)
	}

	q.Push("foo")
	l, err = q.Length()

	if l != 1 {
		t.Error("Expect length to be 1, was: ", l)
	}
	if err != nil {
		t.Error("Error while getting length of Redis queue", err)
	}

	q.Pop(1)
	l, err = q.Length()
	if l != 0 {
		t.Error("Expect length to be 0, was: ", l)
	}
	if err != nil {
		t.Error("Error while getting length of Redis queue", err)
	}

	q.Disconnect()
}

func BenchmarkQueueConnectDisconnect(b *testing.B) {
	for i := 0; i < b.N; i++ {
		q, _ := QueueConnect(":6379", "rq_test_queue")
		q.Disconnect()
	}
}

func BenchmarkQueuePush(b *testing.B) {
	q, _ := QueueConnect(":6379", "rq_test_queue_pushpop_bench")
	for i := 0; i < b.N; i++ {
		q.Push("foo")
	}
	q.Disconnect()
}

func BenchmarkQueueLength(b *testing.B) {
	q, _ := QueueConnect(":6379", "rq_test_queue_length_bench")
	q.Push("foo")
	for i := 0; i < b.N; i++ {
		q.Length()
	}
	q.Pop(1)
	q.Disconnect()
}
<|endoftext|>


<commit_before>package speed

import (
	"fmt"
	"os"
	"testing"
)

func TestMmvFileLocation(t *testing.T) {
	l := config["PCP_TMP_DIR"]

	loc, _ := mmvFileLocation("test")
	expected := fmt.Sprintf("%v%cmmv%c%v", l, os.PathSeparator, os.PathSeparator, "test")
	if loc != expected {
		t.Errorf("location not expected value, expected %v, got %v", expected, loc)
	}

	delete(config, "PCP_TMP_DIR")
	loc, _ = mmvFileLocation("test")
	expected = fmt.Sprintf("%v%cmmv%c%v", os.TempDir(), os.PathSeparator, os.PathSeparator, "test")
	if loc != expected {
		t.Errorf("location not expected value, expected %v, got %v", expected, loc)
	}

	config["PCP_TMP_DIR"] = l
	loc, err := mmvFileLocation(fmt.Sprintf("%v%c", "test", os.PathSeparator))
	if err == nil {
		t.Errorf("expected error, instead got path %v", loc)
	}
}

func TestTocCountAndLength(t *testing.T) {
	w, err := NewPCPWriter("test", ProcessFlag)
	if err != nil {
		t.Errorf("cannot create writer, error: %v", err)
	}

	if w.tocCount() != 2 {
		t.Errorf("expected tocCount to be 2, got %v", w.tocCount())
	}

	expectedLength := HeaderLength + 2*TocLength
	if w.Length() != expectedLength {
		t.Errorf("expected Length to be %v, got %v", expectedLength, w.Length())
	}

	m, err := NewPCPSingletonMetric(10, "test", Int32Type, CounterSemantics, OneUnit, "test", "")
	if err != nil {
		t.Error("Cannot create a metric")
	}

	w.Register(m)
	if w.tocCount() != 3 {
		t.Errorf("expected tocCount to be 3, got %v", w.tocCount())
	}

	expectedLength = HeaderLength + 3*TocLength + 1*MetricLength + 1*ValueLength + 1*StringLength
	if w.Length() != expectedLength {
		t.Errorf("expected Length to be %v, got %v", expectedLength, w.Length())
	}

	indom, _ := NewPCPInstanceDomain("testindom", []string{"test"}, "", "")
	w.RegisterIndom(indom)

	m2, err := NewPCPInstanceMetric(
		Instances{
			"test": 1,
		},
		"test2", indom, Int32Type, CounterSemantics, OneUnit, "", "",
	)
	if err != nil {
		t.Error("Cannot create a metric")
	}

	err = w.Register(m2)
	if err != nil {
		t.Error("Cannot register m2")
	}

	if w.tocCount() != 5 {
		t.Errorf("expected tocCount to be 5, got %v", w.tocCount())
	}
}
<commit_msg>writer: test mapping credibility<commit_after>package speed

import (
	"fmt"
	"os"
	"testing"
)

func TestMmvFileLocation(t *testing.T) {
	l := config["PCP_TMP_DIR"]

	loc, _ := mmvFileLocation("test")
	expected := fmt.Sprintf("%v%cmmv%c%v", l, os.PathSeparator, os.PathSeparator, "test")
	if loc != expected {
		t.Errorf("location not expected value, expected %v, got %v", expected, loc)
	}

	delete(config, "PCP_TMP_DIR")
	loc, _ = mmvFileLocation("test")
	expected = fmt.Sprintf("%v%cmmv%c%v", os.TempDir(), os.PathSeparator, os.PathSeparator, "test")
	if loc != expected {
		t.Errorf("location not expected value, expected %v, got %v", expected, loc)
	}

	config["PCP_TMP_DIR"] = l
	loc, err := mmvFileLocation(fmt.Sprintf("%v%c", "test", os.PathSeparator))
	if err == nil {
		t.Errorf("expected error, instead got path %v", loc)
	}
}

func TestTocCountAndLength(t *testing.T) {
	w, err := NewPCPWriter("test", ProcessFlag)
	if err != nil {
		t.Errorf("cannot create writer, error: %v", err)
	}

	if w.tocCount() != 2 {
		t.Errorf("expected tocCount to be 2, got %v", w.tocCount())
	}

	expectedLength := HeaderLength + 2*TocLength
	if w.Length() != expectedLength {
		t.Errorf("expected Length to be %v, got %v", expectedLength, w.Length())
	}

	m, err := NewPCPSingletonMetric(10, "test", Int32Type, CounterSemantics, OneUnit, "test", "")
	if err != nil {
		t.Error("Cannot create a metric")
	}

	w.Register(m)
	if w.tocCount() != 3 {
		t.Errorf("expected tocCount to be 3, got %v", w.tocCount())
	}

	expectedLength = HeaderLength + 3*TocLength + 1*MetricLength + 1*ValueLength + 1*StringLength
	if w.Length() != expectedLength {
		t.Errorf("expected Length to be %v, got %v", expectedLength, w.Length())
	}

	indom, _ := NewPCPInstanceDomain("testindom", []string{"test"}, "", "")
	w.RegisterIndom(indom)

	m2, err := NewPCPInstanceMetric(
		Instances{
			"test": 1,
		},
		"test2", indom, Int32Type, CounterSemantics, OneUnit, "", "",
	)
	if err != nil {
		t.Error("Cannot create a metric")
	}

	err = w.Register(m2)
	if err != nil {
		t.Error("Cannot register m2")
	}

	if w.tocCount() != 5 {
		t.Errorf("expected tocCount to be 5, got %v", w.tocCount())
	}
}

func TestMapping(t *testing.T) {
	w, err := NewPCPWriter("test", ProcessFlag)
	_, err = w.RegisterString("test.1", 2, CounterSemantics, Int32Type, OneUnit)
	if err != nil {
		t.Error("Cannot Register")
	}

	w.Start()
	loc, _ := mmvFileLocation("test")
	if _, err = os.Stat(loc); err != nil {
		t.Error("expected a MMV file to be created on startup")
	}

	_, err = w.RegisterString("test.2", 2, CounterSemantics, Int32Type, OneUnit)
	if err == nil {
		t.Error("expected registration to fail when a mapping is active")
	}

	EraseFileOnStop = true
	err = w.Stop()
	if err != nil {
		t.Error("Cannot stop a mapping")
	}

	if _, err = os.Stat(loc); err == nil {
		t.Error("expected the MMV file be deleted after stopping")
	}

	EraseFileOnStop = false
}
<|endoftext|>


<commit_before>package unsafe_test

import (
	"testing"
	"unsafe"
)

// Note: I tried this test only on my 64bit macbook
func TestSizeof(t *testing.T) {
	if unsafe.Sizeof(int32(12345)) != 4 {
		t.Errorf("int32 is not 4 bytes?")
	}
	if unsafe.Sizeof(int64(0x12393254234)) != 8 {
		t.Errorf("int64 is not 8 bytes?")
	}
	if unsafe.Sizeof(rune('x')) != 4 {
		t.Errorf("rune is not 4 bytes?")
	}
	if unsafe.Sizeof(false) != 1 {
		t.Errorf("bool is not a byte?")
	}
	int_size := unsafe.Sizeof(int(12345))
	str := "hello"
	ptr_size := unsafe.Sizeof(&str)
	if ptr_size != int_size {
		t.Errorf("pointer size != int size") // this can actually happen on some architectures
	}
	if unsafe.Sizeof("hello") != int_size+ptr_size {
		t.Errorf("string is not int size + ptr size?")
	}
	slice := []int{}
	slice_size := unsafe.Sizeof(slice)
	if slice_size != ptr_size+int_size+int_size {
		t.Errorf("slice is not a pointer + len + cap?")
	}
	mapp := make(map[string]int)
	if unsafe.Sizeof(mapp) != ptr_size {
		t.Errorf("map is not a pointer?")
	}
	ch := make(chan int)
	if unsafe.Sizeof(ch) != ptr_size {
		t.Errorf("channel is not a pointer?")
	}
	add := func(a, b int) int {
		return a + b
	}
	if unsafe.Sizeof(add) != ptr_size {
		t.Errorf("function pointer != data pointer?") // this can also happen on some architectures
	}
	if unsafe.Sizeof(uintptr(0)) != ptr_size {
		t.Errorf("universal pointer != data pointer?") // this can also happen on some architectures
	}
	s1 := struct {
		b bool
		c int
	}{false, 3}
	if unsafe.Sizeof(s1) != 2 * int_size {
		t.Errorf("struct did not use alignment?")
	}
	s2 := struct {
		a int16
		b bool
		c int32
		d int
	}{0, false, 0, 0}
	if unsafe.Sizeof(s2) != 2 * int_size {
		t.Errorf("struct did not use field packing?")
	}
}

func TestAlighAndOffset(t *testing.T) {
	s := struct {
		a bool  // 0: 1
		// pad 3
		b int32 // 4: 4
		c int16 // 8: 2
		// pad 6
		d string  // 16: 16
	}{}
	ptr_size := unsafe.Sizeof(&s)
	if unsafe.Alignof(s) != ptr_size {
		t.Errorf("struct not aligned to biggest field (string data pointer)")
	}
	if unsafe.Alignof(s.b) != 4 {
		t.Errorf("struct has no padding")
	}
	if unsafe.Alignof(s.c) != 2 {
		t.Errorf("struct not packed")
	}
	if unsafe.Offsetof(s.b) != 4 {
		t.Errorf("no padding by int32")
	}
	if unsafe.Offsetof(s.c) != 8 {
		t.Errorf("no padding by int36")
	}
	if unsafe.Offsetof(s.d) != 16 {
		t.Errorf("no padding by string")
	}
}
<commit_msg>unsafe slice<commit_after>package unsafe_test

import (
	"testing"
	"unsafe"
)

// Note: I tried this test only on my 64bit macbook
func TestSizeof(t *testing.T) {
	if unsafe.Sizeof(int32(12345)) != 4 {
		t.Errorf("int32 is not 4 bytes?")
	}
	if unsafe.Sizeof(int64(0x12393254234)) != 8 {
		t.Errorf("int64 is not 8 bytes?")
	}
	if unsafe.Sizeof(rune('x')) != 4 {
		t.Errorf("rune is not 4 bytes?")
	}
	if unsafe.Sizeof(false) != 1 {
		t.Errorf("bool is not a byte?")
	}
	int_size := unsafe.Sizeof(int(12345))
	str := "hello"
	ptr_size := unsafe.Sizeof(&str)
	if ptr_size != int_size {
		t.Errorf("pointer size != int size") // this can actually happen on some architectures
	}
	if unsafe.Sizeof("hello") != int_size+ptr_size {
		t.Errorf("string is not int size + ptr size?")
	}
	slice := []int{}
	slice_size := unsafe.Sizeof(slice)
	if slice_size != ptr_size+int_size+int_size {
		t.Errorf("slice is not a pointer + len + cap?")
	}
	mapp := make(map[string]int)
	if unsafe.Sizeof(mapp) != ptr_size {
		t.Errorf("map is not a pointer?")
	}
	ch := make(chan int)
	if unsafe.Sizeof(ch) != ptr_size {
		t.Errorf("channel is not a pointer?")
	}
	add := func(a, b int) int {
		return a + b
	}
	if unsafe.Sizeof(add) != ptr_size {
		t.Errorf("function pointer != data pointer?") // this can also happen on some architectures
	}
	if unsafe.Sizeof(uintptr(0)) != ptr_size {
		t.Errorf("universal pointer != data pointer?") // this can also happen on some architectures
	}
	s1 := struct {
		b bool
		c int
	}{false, 3}
	if unsafe.Sizeof(s1) != 2 * int_size {
		t.Errorf("struct did not use alignment?")
	}
	s2 := struct {
		a int16
		b bool
		c int32
		d int
	}{0, false, 0, 0}
	if unsafe.Sizeof(s2) != 2 * int_size {
		t.Errorf("struct did not use field packing?")
	}
}

func TestAlighAndOffset(t *testing.T) {
	s := struct {
		a bool  // 0: 1
		// pad 3
		b int32 // 4: 4
		c int16 // 8: 2
		// pad 6
		d string  // 16: 16
	}{}
	ptr_size := unsafe.Sizeof(&s)
	if unsafe.Alignof(s) != ptr_size {
		t.Errorf("struct not aligned to biggest field (string data pointer)")
	}
	if unsafe.Alignof(s.b) != 4 {
		t.Errorf("struct has no padding")
	}
	if unsafe.Alignof(s.c) != 2 {
		t.Errorf("struct not packed")
	}
	if unsafe.Alignof(s.d) != ptr_size {
		t.Errorf("struct not packed")
	}
	if unsafe.Offsetof(s.b) != 4 {
		t.Errorf("no padding by int32")
	}
	if unsafe.Offsetof(s.c) != 8 {
		t.Errorf("no padding by int36")
	}
	if unsafe.Offsetof(s.d) != 16 {
		t.Errorf("no padding by string")
	}
	str := "hello"
	if unsafe.Alignof(str) != ptr_size {
		t.Errorf("string alignment is not internal pointer?")
	}
}

type StringStruct struct {
	data *byte
	size int
}

func TestStringPointer(t *testing.T) {
	h := "Hello"
	p := (*StringStruct)(unsafe.Pointer(&h))
	if p.size != len(h) {
		t.Errorf("size not decoded properly")
	}
	if byte('H') != *p.data {
		t.Errorf("string not decoded properly")
	}
	if byte('e') != *(*byte)(unsafe.Pointer((uintptr(unsafe.Pointer(p.data)) + 1))) {
		t.Errorf("string not decoded properly")
	}
	if byte('l') != *(*byte)(unsafe.Pointer((uintptr(unsafe.Pointer(p.data)) + 2))) {
		t.Errorf("string not decoded properly")
	}
	if byte('l') != *(*byte)(unsafe.Pointer((uintptr(unsafe.Pointer(p.data)) + 3))) {
		t.Errorf("string not decoded properly")
	}
	if byte('o') != *(*byte)(unsafe.Pointer((uintptr(unsafe.Pointer(p.data)) + 4))) {
		t.Errorf("string not decoded properly")
	}
}

type SliceStruct struct {
	data *byte
	len, cap int
}

func TestSlicePointer(t *testing.T) {
	s := make([]string, 0x55, 0x77)
	p := (*SliceStruct)(unsafe.Pointer(&s))
	if p.len != 0x55 {
		t.Errorf("len not found")
	}
	if p.cap != 0x77 {
		t.Errorf("cap not found")
	}
	h := "hello"
	s[0] = "meh"
	p.data = (*byte)(unsafe.Pointer(&h))
	if s[0] != "hello" {
		t.Errorf("setting failed")
	}
}
<|endoftext|>


<commit_before>package lfs

import (
	"bufio"
	"bytes"
	"fmt"
	"io"
	"io/ioutil"
	"regexp"
	"time"

	"github.com/git-lfs/git-lfs/git"
	"github.com/git-lfs/git-lfs/tools"
	"github.com/rubyist/tracerx"
)

// When scanning diffs e.g. parseLogOutputToPointers, which direction of diff to include
// data from, i.e. '+' or '-'. Depending on what you're scanning for either might be useful
type LogDiffDirection byte

const (
	LogDiffAdditions = LogDiffDirection('+') // include '+' diffs
	LogDiffDeletions = LogDiffDirection('-') // include '-' diffs
)

var (
	// Arguments to append to a git log call which will limit the output to
	// lfs changes and format the output suitable for parseLogOutput.. method(s)
	logLfsSearchArgs = []string{
		"-G", "oid sha256:", // only diffs which include an lfs file SHA change
		"-p",   // include diff so we can read the SHA
		"-U12", // Make sure diff context is always big enough to support 10 extension lines to get whole pointer
		`--format=lfs-commit-sha: %H %P`, // just a predictable commit header we can detect
	}
)

func scanUnpushed(remote string) (*PointerChannelWrapper, error) {
	logArgs := []string{"log",
		"--branches", "--tags", // include all locally referenced commits
		"--not"} // but exclude everything that comes after

	if len(remote) == 0 {
		logArgs = append(logArgs, "--remotes")
	} else {
		logArgs = append(logArgs, fmt.Sprintf("--remotes=%v", remote))
	}

	// Add standard search args to find lfs references
	logArgs = append(logArgs, logLfsSearchArgs...)

	cmd, err := startCommand("git", logArgs...)
	if err != nil {
		return nil, err
	}

	cmd.Stdin.Close()

	pchan := make(chan *WrappedPointer, chanBufSize)
	errchan := make(chan error, 1)

	go func() {
		parseLogOutputToPointers(cmd.Stdout, LogDiffAdditions, nil, nil, pchan)
		stderr, _ := ioutil.ReadAll(cmd.Stderr)
		err := cmd.Wait()
		if err != nil {
			errchan <- fmt.Errorf("Error in git log: %v %v", err, string(stderr))
		}
		close(pchan)
		close(errchan)
	}()

	return NewPointerChannelWrapper(pchan, errchan), nil
}

// logPreviousVersions scans history for all previous versions of LFS pointers
// from 'since' up to (but not including) the final state at ref
func logPreviousSHAs(ref string, since time.Time) (*PointerChannelWrapper, error) {
	logArgs := []string{"log",
		fmt.Sprintf("--since=%v", git.FormatGitDate(since)),
	}
	// Add standard search args to find lfs references
	logArgs = append(logArgs, logLfsSearchArgs...)
	// ending at ref
	logArgs = append(logArgs, ref)

	cmd, err := startCommand("git", logArgs...)
	if err != nil {
		return nil, err
	}

	cmd.Stdin.Close()

	pchan := make(chan *WrappedPointer, chanBufSize)
	errchan := make(chan error, 1)

	// we pull out deletions, since we want the previous SHAs at commits in the range
	// this means we pick up all previous versions that could have been checked
	// out in the date range, not just if the commit which *introduced* them is in the range
	go func() {
		parseLogOutputToPointers(cmd.Stdout, LogDiffDeletions, nil, nil, pchan)
		stderr, _ := ioutil.ReadAll(cmd.Stderr)
		err := cmd.Wait()
		if err != nil {
			errchan <- fmt.Errorf("Error in git log: %v %v", err, string(stderr))
		}
		close(pchan)
		close(errchan)
	}()

	return NewPointerChannelWrapper(pchan, errchan), nil
}

func parseLogOutputToPointers(log io.Reader, dir LogDiffDirection,
	includePaths, excludePaths []string, results chan *WrappedPointer) {
	scanner := newLogScanner(log, dir, includePaths, excludePaths)
	for scanner.Scan() {
		if p := scanner.Pointer(); p != nil {
			results <- p
		}
	}
}

// logScanner parses log output formatted as per logLfsSearchArgs & returns
// pointers.
type logScanner struct {
	s            *bufio.Scanner
	dir          LogDiffDirection
	includePaths []string
	excludePaths []string
	pointer      *WrappedPointer

	pointerData         *bytes.Buffer
	currentFilename     string
	currentFileIncluded bool

	commitHeaderRegex    *regexp.Regexp
	fileHeaderRegex      *regexp.Regexp
	fileMergeHeaderRegex *regexp.Regexp
	pointerDataRegex     *regexp.Regexp
}

// r: a stream of output from git log with at least logLfsSearchArgs specified
// dir: whether to include results from + or - diffs
// includePaths, excludePaths: filter the results by filename
func newLogScanner(r io.Reader, dir LogDiffDirection, includePaths, excludePaths []string) *logScanner {
	return &logScanner{
		s:                   bufio.NewScanner(r),
		dir:                 dir,
		includePaths:        includePaths,
		excludePaths:        excludePaths,
		pointerData:         &bytes.Buffer{},
		currentFileIncluded: true,

		// no need to compile these regexes on every `git-lfs` call, just ones that
		// use the scanner.
		commitHeaderRegex:    regexp.MustCompile(`^lfs-commit-sha: ([A-Fa-f0-9]{40})(?: ([A-Fa-f0-9]{40}))*`),
		fileHeaderRegex:      regexp.MustCompile(`diff --git a\/(.+?)\s+b\/(.+)`),
		fileMergeHeaderRegex: regexp.MustCompile(`diff --cc (.+)`),
		pointerDataRegex:     regexp.MustCompile(`^([\+\- ])(version https://git-lfs|oid sha256|size|ext-).*$`),
	}
}

func (s *logScanner) Pointer() *WrappedPointer {
	return s.pointer
}

func (s *logScanner) Err() error {
	return s.s.Err()
}

func (s *logScanner) Scan() bool {
	s.pointer = nil
	p, canScan := s.scan()
	s.pointer = p
	return canScan
}

// Utility func used at several points below (keep in narrow scope)
func (s *logScanner) finishLastPointer() *WrappedPointer {
	if s.pointerData.Len() > 0 {
		if s.currentFileIncluded {
			p, err := DecodePointer(s.pointerData)
			if err == nil {
				return &WrappedPointer{Name: s.currentFilename, Pointer: p}
			} else {
				tracerx.Printf("Unable to parse pointer from log: %v", err)
			}
		}
		s.pointerData.Reset()
	}
	return nil
}

// For each commit we'll get something like this:
/*
	lfs-commit-sha: 60fde3d23553e10a55e2a32ed18c20f65edd91e7 e2eaf1c10b57da7b98eb5d722ec5912ddeb53ea1

	diff --git a/1D_Noise.png b/1D_Noise.png
	new file mode 100644
	index 0000000..2622b4a
	--- /dev/null
	+++ b/1D_Noise.png
	@@ -0,0 +1,3 @@
	+version https://git-lfs.github.com/spec/v1
	+oid sha256:f5d84da40ab1f6aa28df2b2bf1ade2cdcd4397133f903c12b4106641b10e1ed6
	+size 1289
*/
// There can be multiple diffs per commit (multiple binaries)
// Also when a binary is changed the diff will include a '-' line for the old SHA
func (s *logScanner) scan() (*WrappedPointer, bool) {
	for s.s.Scan() {
		line := s.s.Text()

		if match := s.commitHeaderRegex.FindStringSubmatch(line); match != nil {
			// Currently we're not pulling out commit groupings, but could if we wanted
			// This just acts as a delimiter for finishing a multiline pointer
			if p := s.finishLastPointer(); p != nil {
				return p, true
			}
		} else if match := s.fileHeaderRegex.FindStringSubmatch(line); match != nil {
			// Finding a regular file header
			p := s.finishLastPointer()
			// Pertinent file name depends on whether we're listening to additions or removals
			if s.dir == LogDiffAdditions {
				s.currentFilename = match[2]
			} else {
				s.currentFilename = match[1]
			}
			s.currentFileIncluded = tools.FilenamePassesIncludeExcludeFilter(s.currentFilename, s.includePaths, s.excludePaths)

			if p != nil {
				return p, true
			}
		} else if match := s.fileMergeHeaderRegex.FindStringSubmatch(line); match != nil {
			// Git merge file header is a little different, only one file
			p := s.finishLastPointer()
			s.currentFilename = match[1]
			s.currentFileIncluded = tools.FilenamePassesIncludeExcludeFilter(s.currentFilename, s.includePaths, s.excludePaths)

			if p != nil {
				return p, true
			}
		} else if s.currentFileIncluded {
			if match := s.pointerDataRegex.FindStringSubmatch(line); match != nil {
				// An LFS pointer data line
				// Include only the entirety of one side of the diff
				// -U3 will ensure we always get all of it, even if only
				// the SHA changed (version & size the same)
				changeType := match[1][0]

				// Always include unchanged context lines (normally just the version line)
				if LogDiffDirection(changeType) == s.dir || changeType == ' ' {
					// Must skip diff +/- marker
					s.pointerData.WriteString(line[1:])
					s.pointerData.WriteString("\n") // newline was stripped off by scanner
				}
			}
		}
	}

	if p := s.finishLastPointer(); p != nil {
		return p, true
	}

	return nil, false
}
<commit_msg>remove nested ifs<commit_after>package lfs

import (
	"bufio"
	"bytes"
	"fmt"
	"io"
	"io/ioutil"
	"regexp"
	"time"

	"github.com/git-lfs/git-lfs/git"
	"github.com/git-lfs/git-lfs/tools"
	"github.com/rubyist/tracerx"
)

// When scanning diffs e.g. parseLogOutputToPointers, which direction of diff to include
// data from, i.e. '+' or '-'. Depending on what you're scanning for either might be useful
type LogDiffDirection byte

const (
	LogDiffAdditions = LogDiffDirection('+') // include '+' diffs
	LogDiffDeletions = LogDiffDirection('-') // include '-' diffs
)

var (
	// Arguments to append to a git log call which will limit the output to
	// lfs changes and format the output suitable for parseLogOutput.. method(s)
	logLfsSearchArgs = []string{
		"-G", "oid sha256:", // only diffs which include an lfs file SHA change
		"-p",   // include diff so we can read the SHA
		"-U12", // Make sure diff context is always big enough to support 10 extension lines to get whole pointer
		`--format=lfs-commit-sha: %H %P`, // just a predictable commit header we can detect
	}
)

func scanUnpushed(remote string) (*PointerChannelWrapper, error) {
	logArgs := []string{"log",
		"--branches", "--tags", // include all locally referenced commits
		"--not"} // but exclude everything that comes after

	if len(remote) == 0 {
		logArgs = append(logArgs, "--remotes")
	} else {
		logArgs = append(logArgs, fmt.Sprintf("--remotes=%v", remote))
	}

	// Add standard search args to find lfs references
	logArgs = append(logArgs, logLfsSearchArgs...)

	cmd, err := startCommand("git", logArgs...)
	if err != nil {
		return nil, err
	}

	cmd.Stdin.Close()

	pchan := make(chan *WrappedPointer, chanBufSize)
	errchan := make(chan error, 1)

	go func() {
		parseLogOutputToPointers(cmd.Stdout, LogDiffAdditions, nil, nil, pchan)
		stderr, _ := ioutil.ReadAll(cmd.Stderr)
		err := cmd.Wait()
		if err != nil {
			errchan <- fmt.Errorf("Error in git log: %v %v", err, string(stderr))
		}
		close(pchan)
		close(errchan)
	}()

	return NewPointerChannelWrapper(pchan, errchan), nil
}

// logPreviousVersions scans history for all previous versions of LFS pointers
// from 'since' up to (but not including) the final state at ref
func logPreviousSHAs(ref string, since time.Time) (*PointerChannelWrapper, error) {
	logArgs := []string{"log",
		fmt.Sprintf("--since=%v", git.FormatGitDate(since)),
	}
	// Add standard search args to find lfs references
	logArgs = append(logArgs, logLfsSearchArgs...)
	// ending at ref
	logArgs = append(logArgs, ref)

	cmd, err := startCommand("git", logArgs...)
	if err != nil {
		return nil, err
	}

	cmd.Stdin.Close()

	pchan := make(chan *WrappedPointer, chanBufSize)
	errchan := make(chan error, 1)

	// we pull out deletions, since we want the previous SHAs at commits in the range
	// this means we pick up all previous versions that could have been checked
	// out in the date range, not just if the commit which *introduced* them is in the range
	go func() {
		parseLogOutputToPointers(cmd.Stdout, LogDiffDeletions, nil, nil, pchan)
		stderr, _ := ioutil.ReadAll(cmd.Stderr)
		err := cmd.Wait()
		if err != nil {
			errchan <- fmt.Errorf("Error in git log: %v %v", err, string(stderr))
		}
		close(pchan)
		close(errchan)
	}()

	return NewPointerChannelWrapper(pchan, errchan), nil
}

func parseLogOutputToPointers(log io.Reader, dir LogDiffDirection,
	includePaths, excludePaths []string, results chan *WrappedPointer) {
	scanner := newLogScanner(log, dir, includePaths, excludePaths)
	for scanner.Scan() {
		if p := scanner.Pointer(); p != nil {
			results <- p
		}
	}
}

// logScanner parses log output formatted as per logLfsSearchArgs & returns
// pointers.
type logScanner struct {
	s            *bufio.Scanner
	dir          LogDiffDirection
	includePaths []string
	excludePaths []string
	pointer      *WrappedPointer

	pointerData         *bytes.Buffer
	currentFilename     string
	currentFileIncluded bool

	commitHeaderRegex    *regexp.Regexp
	fileHeaderRegex      *regexp.Regexp
	fileMergeHeaderRegex *regexp.Regexp
	pointerDataRegex     *regexp.Regexp
}

// r: a stream of output from git log with at least logLfsSearchArgs specified
// dir: whether to include results from + or - diffs
// includePaths, excludePaths: filter the results by filename
func newLogScanner(r io.Reader, dir LogDiffDirection, includePaths, excludePaths []string) *logScanner {
	return &logScanner{
		s:                   bufio.NewScanner(r),
		dir:                 dir,
		includePaths:        includePaths,
		excludePaths:        excludePaths,
		pointerData:         &bytes.Buffer{},
		currentFileIncluded: true,

		// no need to compile these regexes on every `git-lfs` call, just ones that
		// use the scanner.
		commitHeaderRegex:    regexp.MustCompile(`^lfs-commit-sha: ([A-Fa-f0-9]{40})(?: ([A-Fa-f0-9]{40}))*`),
		fileHeaderRegex:      regexp.MustCompile(`diff --git a\/(.+?)\s+b\/(.+)`),
		fileMergeHeaderRegex: regexp.MustCompile(`diff --cc (.+)`),
		pointerDataRegex:     regexp.MustCompile(`^([\+\- ])(version https://git-lfs|oid sha256|size|ext-).*$`),
	}
}

func (s *logScanner) Pointer() *WrappedPointer {
	return s.pointer
}

func (s *logScanner) Err() error {
	return s.s.Err()
}

func (s *logScanner) Scan() bool {
	s.pointer = nil
	p, canScan := s.scan()
	s.pointer = p
	return canScan
}

// Utility func used at several points below (keep in narrow scope)
func (s *logScanner) finishLastPointer() *WrappedPointer {
	if s.pointerData.Len() == 0 {
		return nil
	}

	if !s.currentFileIncluded {
		return nil
	}

	p, err := DecodePointer(s.pointerData)
	s.pointerData.Reset()

	if err == nil {
		return &WrappedPointer{Name: s.currentFilename, Pointer: p}
	} else {
		tracerx.Printf("Unable to parse pointer from log: %v", err)
		return nil
	}
}

// For each commit we'll get something like this:
/*
	lfs-commit-sha: 60fde3d23553e10a55e2a32ed18c20f65edd91e7 e2eaf1c10b57da7b98eb5d722ec5912ddeb53ea1

	diff --git a/1D_Noise.png b/1D_Noise.png
	new file mode 100644
	index 0000000..2622b4a
	--- /dev/null
	+++ b/1D_Noise.png
	@@ -0,0 +1,3 @@
	+version https://git-lfs.github.com/spec/v1
	+oid sha256:f5d84da40ab1f6aa28df2b2bf1ade2cdcd4397133f903c12b4106641b10e1ed6
	+size 1289
*/
// There can be multiple diffs per commit (multiple binaries)
// Also when a binary is changed the diff will include a '-' line for the old SHA
func (s *logScanner) scan() (*WrappedPointer, bool) {
	for s.s.Scan() {
		line := s.s.Text()

		if match := s.commitHeaderRegex.FindStringSubmatch(line); match != nil {
			// Currently we're not pulling out commit groupings, but could if we wanted
			// This just acts as a delimiter for finishing a multiline pointer
			if p := s.finishLastPointer(); p != nil {
				return p, true
			}
		} else if match := s.fileHeaderRegex.FindStringSubmatch(line); match != nil {
			// Finding a regular file header
			p := s.finishLastPointer()
			// Pertinent file name depends on whether we're listening to additions or removals
			if s.dir == LogDiffAdditions {
				s.currentFilename = match[2]
			} else {
				s.currentFilename = match[1]
			}
			s.currentFileIncluded = tools.FilenamePassesIncludeExcludeFilter(s.currentFilename, s.includePaths, s.excludePaths)

			if p != nil {
				return p, true
			}
		} else if match := s.fileMergeHeaderRegex.FindStringSubmatch(line); match != nil {
			// Git merge file header is a little different, only one file
			p := s.finishLastPointer()
			s.currentFilename = match[1]
			s.currentFileIncluded = tools.FilenamePassesIncludeExcludeFilter(s.currentFilename, s.includePaths, s.excludePaths)

			if p != nil {
				return p, true
			}
		} else if s.currentFileIncluded {
			if match := s.pointerDataRegex.FindStringSubmatch(line); match != nil {
				// An LFS pointer data line
				// Include only the entirety of one side of the diff
				// -U3 will ensure we always get all of it, even if only
				// the SHA changed (version & size the same)
				changeType := match[1][0]

				// Always include unchanged context lines (normally just the version line)
				if LogDiffDirection(changeType) == s.dir || changeType == ' ' {
					// Must skip diff +/- marker
					s.pointerData.WriteString(line[1:])
					s.pointerData.WriteString("\n") // newline was stripped off by scanner
				}
			}
		}
	}

	if p := s.finishLastPointer(); p != nil {
		return p, true
	}

	return nil, false
}
<|endoftext|>