<commit_before>// Copyright 2015 Aaron Jacobs. All Rights Reserved.
// Author: aaronjjacobs@gmail.com (Aaron Jacobs)
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package blob

import (
	"bytes"
	"crypto/md5"
	"crypto/sha1"
	"encoding/hex"
	"fmt"
	"io/ioutil"
	"strconv"
	"strings"

	"golang.org/x/net/context"

	"github.com/jacobsa/gcloud/gcs"
	"github.com/jacobsa/gcloud/gcs/gcsutil"
	"github.com/jacobsa/gcloud/syncutil"
)

// A key placed in GCS object metadata by GCSStore containing the hex SHA-1
// expected for the object contents. This is of course redundant with the
// object name; we use it as a paranoid check against GCS returning the
// metadata or contents for the wrong object.
const metadataKey_SHA1 = "comeback_sha1"

// A key placed in GCS object metadata by GCSStore containing the CRC32C
// checksum expected for the object contents. If GCS reports a different
// checksum or returns contents with a different checksum, we know something
// screwy has happened.
//
// See here for more info: https://github.com/jacobsa/comeback/issues/18
const metadataKey_CRC32C = "comeback_crc32c"

// A key placed in GCS object metadata by GCSStore containing the hex MD5 sum
// expected for the object contents. If GCS reports a different MD5 sum or
// returns contents with a different MD5 sum, we know something screwy has
// happened.
//
// See here for more info: https://github.com/jacobsa/comeback/issues/18
const metadataKey_MD5 = "comeback_md5"

// Return a blob store that stores blobs in the supplied GCS bucket. GCS object
// names look like:
//
//     <prefix><score>
//
// where <score> is the result of calling Score.Hex.
//
// The blob store trusts that it has full ownership of this portion of the
// bucket's namespace -- if a score name exists, then it points to the correct
// data.
//
// The returned store does not support Flush or Contains; these methods must
// not be called.
func NewGCSStore(
	bucket gcs.Bucket,
	prefix string) (store *GCSStore) {
	store = &GCSStore{
		bucket:     bucket,
		namePrefix: prefix,
	}

	return
}

type GCSStore struct {
	bucket     gcs.Bucket
	namePrefix string
}

// Parse and verify the internal consistency of the supplied object record in
// the same manner that a GCSStore configured with the supplied object name
// prefix would. Return the score of the blob that the object contains.
func ParseObjectRecord(
	o *gcs.Object,
	namePrefix string) (score Score, err error) {
	// Is the name of the appropriate form?
	if !strings.HasPrefix(o.Name, namePrefix) {
		err = fmt.Errorf("Unexpected object name: %q", o.Name)
		return
	}

	// Parse the hex score.
	hexScore := strings.TrimPrefix(o.Name, namePrefix)
	score, err = ParseHexScore(hexScore)
	if err != nil {
		err = fmt.Errorf(
			"Unexpected hex score %q for object %q: %v",
			hexScore,
			o.Name,
			err)
		return
	}

	// We expect the hex score to match the hex SHA-1 in the metadata.
	hexSHA1, ok := o.Metadata[metadataKey_SHA1]
	if !ok {
		err = fmt.Errorf(
			"Object %q is missing metadata key %q",
			o.Name,
			metadataKey_SHA1)
		return
	}

	if hexSHA1 != hexScore {
		err = fmt.Errorf(
			"Score/SHA-1 metadata mismatch for object %q: %q",
			o.Name,
			hexSHA1)
		return
	}

	// We expect the hex CRC32C in the object metadata to match what GCS says the
	// object's checksum is.
	hexCRC32C, ok := o.Metadata[metadataKey_CRC32C]
	if !ok {
		err = fmt.Errorf(
			"Object %q is missing metadata key %q",
			o.Name,
			metadataKey_CRC32C)
		return
	}

	crc32Uint64, err := strconv.ParseUint(hexCRC32C, 0, 32)
	if err != nil {
		err = fmt.Errorf(
			"Object %q has invalid hex CRC32C %q: %v",
			o.Name,
			hexCRC32C,
			err)
		return
	}

	if uint32(crc32Uint64) != o.CRC32C {
		err = fmt.Errorf(
			"CRC32C mismatch for object %q: %#08x vs. %#08x",
			o.Name,
			crc32Uint64,
			o.CRC32C)
		return
	}

	// We expect the hex MD5 in the object metadata to match what GCS says the
	// object's MD5 sum is.
	hexMD5, ok := o.Metadata[metadataKey_MD5]
	if !ok {
		err = fmt.Errorf(
			"Object %q is missing metadata key %q",
			o.Name,
			metadataKey_MD5)
		return
	}

	if hex.DecodedLen(len(hexMD5)) != md5.Size {
		err = fmt.Errorf(
			"Object %q has weird hex MD5 metadata: %q",
			o.Name,
			hexMD5)
		return
	}

	var md5 [md5.Size]byte
	_, err = hex.Decode(md5[:], []byte(hexMD5))
	if err != nil {
		err = fmt.Errorf(
			"Object %q has invalid hex MD5 in metadata: %q",
			o.Name,
			hexMD5)
		return
	}

	if md5 != o.MD5 {
		err = fmt.Errorf(
			"MD5 mismatch for object %q: %s vs. %s",
			o.Name,
			hex.EncodeToString(md5[:]),
			hex.EncodeToString(o.MD5[:]))
		return
	}

	return
}

// Write object records for all of the blob objects in the supplied bucket into
// the given channel, without closing it. The order of records is undefined.
// The caller will likely want to call ParseObjectRecord for each record.
func ListBlobObjects(
	ctx context.Context,
	bucket gcs.Bucket,
	namePrefix string,
	objects chan<- *gcs.Object) (err error) {
	req := &gcs.ListObjectsRequest{
		Prefix: namePrefix,
	}

	// List until we run out.
	for {
		// Fetch the next batch.
		var listing *gcs.Listing
		listing, err = bucket.ListObjects(ctx, req)
		if err != nil {
			err = fmt.Errorf("ListObjects: %v", err)
			return
		}

		// Pass on each object.
		for _, o := range listing.Objects {
			// Special case: for gcsfuse compatibility, we allow namePrefix to exist
			// as its own object name. Skip it.
			if o.Name == namePrefix {
				continue
			}

			select {
			case objects <- o:

				// Cancelled?
			case <-ctx.Done():
				err = ctx.Err()
				return
			}
		}

		// Are we done?
		if listing.ContinuationToken == "" {
			break
		}

		req.ContinuationToken = listing.ContinuationToken
	}

	return
}

// Feed the output of ListBlobObjects into ParseObjectRecord, passing on the
// scores to the supplied channel without closing it.
func ListScores(
	ctx context.Context,
	bucket gcs.Bucket,
	namePrefix string,
	scores chan<- Score) (err error) {
	b := syncutil.NewBundle(ctx)

	// List object records into a channel.
	objects := make(chan *gcs.Object, 100)
	b.Add(func(ctx context.Context) (err error) {
		defer close(objects)
		err = ListBlobObjects(ctx, bucket, namePrefix, objects)
		if err != nil {
			err = fmt.Errorf("ListBlobObjects: %v", err)
			return
		}

		return
	})

	// Parse and verify records, and write out scores.
	b.Add(func(ctx context.Context) (err error) {
		for o := range objects {
			// Parse and verify.
			var score Score
			score, err = ParseObjectRecord(o, namePrefix)
			if err != nil {
				err = fmt.Errorf("ParseObjectRecord: %v", err)
				return
			}

			// Send on the score.
			select {
			case scores <- score:

			// Cancelled?
			case <-ctx.Done():
				err = ctx.Err()
				return
			}
		}

		return
	})

	err = b.Join()
	return
}

////////////////////////////////////////////////////////////////////////
// Helpers
////////////////////////////////////////////////////////////////////////

func (s *GCSStore) makeName(score Score) (name string) {
	name = s.namePrefix + score.Hex()
	return
}

////////////////////////////////////////////////////////////////////////
// Public interface
////////////////////////////////////////////////////////////////////////

func (s *GCSStore) Store(blob []byte) (score Score, err error) {
	// Compute a score and an object name.
	score = ComputeScore(blob)
	name := s.makeName(score)

	// Create the object.
	crc32c := *gcsutil.CRC32C(blob)
	md5 := *gcsutil.MD5(blob)
	sha1 := sha1.Sum(blob)

	req := &gcs.CreateObjectRequest{
		Name:     name,
		Contents: bytes.NewReader(blob),
		CRC32C:   &crc32c,
		MD5:      &md5,

		Metadata: map[string]string{
			metadataKey_SHA1:   hex.EncodeToString(sha1[:]),
			metadataKey_CRC32C: fmt.Sprintf("%#08x", crc32c),
			metadataKey_MD5:    hex.EncodeToString(md5[:]),
		},
	}

	o, err := s.bucket.CreateObject(context.Background(), req)
	if err != nil {
		err = fmt.Errorf("CreateObject: %v", err)
		return
	}

	// Paranoid check: what we get back from GCS should match what we put in.
	if o.CRC32C != crc32c {
		panic(fmt.Sprintf(
			"CRC32C mismatch for object %q: 0x%08xv vs. 0x%08x",
			o.Name,
			o.CRC32C,
			crc32c))
	}

	if o.MD5 != md5 {
		panic(fmt.Sprintf(
			"MD5 mismatch for object %q: %s vs. %s",
			o.Name,
			hex.EncodeToString(o.MD5[:]),
			hex.EncodeToString(md5[:])))
	}

	return
}

func (s *GCSStore) Flush() (err error) {
	panic("GCSStore.Flush not supported; wiring code bug?")
}

func (s *GCSStore) Contains(score Score) (b bool) {
	panic("GCSStore.Contains not supported; wiring code bug?")
}

func (s *GCSStore) Load(score Score) (blob []byte, err error) {
	// Create a ReadCloser.
	req := &gcs.ReadObjectRequest{
		Name: s.makeName(score),
	}

	rc, err := s.bucket.NewReader(context.Background(), req)
	if err != nil {
		err = fmt.Errorf("NewReader: %v", err)
		return
	}

	// Read from it.
	blob, err = ioutil.ReadAll(rc)
	if err != nil {
		rc.Close()
		err = fmt.Errorf("ReadAll: %v", err)
		return
	}

	// Close it.
	err = rc.Close()
	if err != nil {
		err = fmt.Errorf("Close: %v", err)
		return
	}

	return
}

// List all of the blobs that are known to be durable in the bucket.
func (s *GCSStore) List() (scores []Score, err error) {
	b := syncutil.NewBundle(context.Background())

	// List into a channel.
	objects := make(chan *gcs.Object, 100)
	b.Add(func(ctx context.Context) (err error) {
		defer close(objects)
		err = ListBlobObjects(ctx, s.bucket, s.namePrefix, objects)
		if err != nil {
			err = fmt.Errorf("ListBlobObjects: %v", err)
			return
		}

		return
	})

	// Parse and verify records, and accumulate into the slice.
	b.Add(func(ctx context.Context) (err error) {
		for o := range objects {
			var score Score
			score, err = ParseObjectRecord(o, s.namePrefix)
			if err != nil {
				err = fmt.Errorf("ParseObjectRecord: %v", err)
				return
			}

			scores = append(scores, score)
		}

		return
	})

	err = b.Join()
	return
}
<commit_msg>Use for GCSStore.List.<commit_after>// Copyright 2015 Aaron Jacobs. All Rights Reserved.
// Author: aaronjjacobs@gmail.com (Aaron Jacobs)
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package blob

import (
	"bytes"
	"crypto/md5"
	"crypto/sha1"
	"encoding/hex"
	"fmt"
	"io/ioutil"
	"strconv"
	"strings"

	"golang.org/x/net/context"

	"github.com/jacobsa/gcloud/gcs"
	"github.com/jacobsa/gcloud/gcs/gcsutil"
	"github.com/jacobsa/gcloud/syncutil"
)

// A key placed in GCS object metadata by GCSStore containing the hex SHA-1
// expected for the object contents. This is of course redundant with the
// object name; we use it as a paranoid check against GCS returning the
// metadata or contents for the wrong object.
const metadataKey_SHA1 = "comeback_sha1"

// A key placed in GCS object metadata by GCSStore containing the CRC32C
// checksum expected for the object contents. If GCS reports a different
// checksum or returns contents with a different checksum, we know something
// screwy has happened.
//
// See here for more info: https://github.com/jacobsa/comeback/issues/18
const metadataKey_CRC32C = "comeback_crc32c"

// A key placed in GCS object metadata by GCSStore containing the hex MD5 sum
// expected for the object contents. If GCS reports a different MD5 sum or
// returns contents with a different MD5 sum, we know something screwy has
// happened.
//
// See here for more info: https://github.com/jacobsa/comeback/issues/18
const metadataKey_MD5 = "comeback_md5"

// Return a blob store that stores blobs in the supplied GCS bucket. GCS object
// names look like:
//
//     <prefix><score>
//
// where <score> is the result of calling Score.Hex.
//
// The blob store trusts that it has full ownership of this portion of the
// bucket's namespace -- if a score name exists, then it points to the correct
// data.
//
// The returned store does not support Flush or Contains; these methods must
// not be called.
func NewGCSStore(
	bucket gcs.Bucket,
	prefix string) (store *GCSStore) {
	store = &GCSStore{
		bucket:     bucket,
		namePrefix: prefix,
	}

	return
}

type GCSStore struct {
	bucket     gcs.Bucket
	namePrefix string
}

// Parse and verify the internal consistency of the supplied object record in
// the same manner that a GCSStore configured with the supplied object name
// prefix would. Return the score of the blob that the object contains.
func ParseObjectRecord(
	o *gcs.Object,
	namePrefix string) (score Score, err error) {
	// Is the name of the appropriate form?
	if !strings.HasPrefix(o.Name, namePrefix) {
		err = fmt.Errorf("Unexpected object name: %q", o.Name)
		return
	}

	// Parse the hex score.
	hexScore := strings.TrimPrefix(o.Name, namePrefix)
	score, err = ParseHexScore(hexScore)
	if err != nil {
		err = fmt.Errorf(
			"Unexpected hex score %q for object %q: %v",
			hexScore,
			o.Name,
			err)
		return
	}

	// We expect the hex score to match the hex SHA-1 in the metadata.
	hexSHA1, ok := o.Metadata[metadataKey_SHA1]
	if !ok {
		err = fmt.Errorf(
			"Object %q is missing metadata key %q",
			o.Name,
			metadataKey_SHA1)
		return
	}

	if hexSHA1 != hexScore {
		err = fmt.Errorf(
			"Score/SHA-1 metadata mismatch for object %q: %q",
			o.Name,
			hexSHA1)
		return
	}

	// We expect the hex CRC32C in the object metadata to match what GCS says the
	// object's checksum is.
	hexCRC32C, ok := o.Metadata[metadataKey_CRC32C]
	if !ok {
		err = fmt.Errorf(
			"Object %q is missing metadata key %q",
			o.Name,
			metadataKey_CRC32C)
		return
	}

	crc32Uint64, err := strconv.ParseUint(hexCRC32C, 0, 32)
	if err != nil {
		err = fmt.Errorf(
			"Object %q has invalid hex CRC32C %q: %v",
			o.Name,
			hexCRC32C,
			err)
		return
	}

	if uint32(crc32Uint64) != o.CRC32C {
		err = fmt.Errorf(
			"CRC32C mismatch for object %q: %#08x vs. %#08x",
			o.Name,
			crc32Uint64,
			o.CRC32C)
		return
	}

	// We expect the hex MD5 in the object metadata to match what GCS says the
	// object's MD5 sum is.
	hexMD5, ok := o.Metadata[metadataKey_MD5]
	if !ok {
		err = fmt.Errorf(
			"Object %q is missing metadata key %q",
			o.Name,
			metadataKey_MD5)
		return
	}

	if hex.DecodedLen(len(hexMD5)) != md5.Size {
		err = fmt.Errorf(
			"Object %q has weird hex MD5 metadata: %q",
			o.Name,
			hexMD5)
		return
	}

	var md5 [md5.Size]byte
	_, err = hex.Decode(md5[:], []byte(hexMD5))
	if err != nil {
		err = fmt.Errorf(
			"Object %q has invalid hex MD5 in metadata: %q",
			o.Name,
			hexMD5)
		return
	}

	if md5 != o.MD5 {
		err = fmt.Errorf(
			"MD5 mismatch for object %q: %s vs. %s",
			o.Name,
			hex.EncodeToString(md5[:]),
			hex.EncodeToString(o.MD5[:]))
		return
	}

	return
}

// Write object records for all of the blob objects in the supplied bucket into
// the given channel, without closing it. The order of records is undefined.
// The caller will likely want to call ParseObjectRecord for each record.
func ListBlobObjects(
	ctx context.Context,
	bucket gcs.Bucket,
	namePrefix string,
	objects chan<- *gcs.Object) (err error) {
	req := &gcs.ListObjectsRequest{
		Prefix: namePrefix,
	}

	// List until we run out.
	for {
		// Fetch the next batch.
		var listing *gcs.Listing
		listing, err = bucket.ListObjects(ctx, req)
		if err != nil {
			err = fmt.Errorf("ListObjects: %v", err)
			return
		}

		// Pass on each object.
		for _, o := range listing.Objects {
			// Special case: for gcsfuse compatibility, we allow namePrefix to exist
			// as its own object name. Skip it.
			if o.Name == namePrefix {
				continue
			}

			select {
			case objects <- o:

				// Cancelled?
			case <-ctx.Done():
				err = ctx.Err()
				return
			}
		}

		// Are we done?
		if listing.ContinuationToken == "" {
			break
		}

		req.ContinuationToken = listing.ContinuationToken
	}

	return
}

// Feed the output of ListBlobObjects into ParseObjectRecord, passing on the
// scores to the supplied channel without closing it.
func ListScores(
	ctx context.Context,
	bucket gcs.Bucket,
	namePrefix string,
	scores chan<- Score) (err error) {
	b := syncutil.NewBundle(ctx)

	// List object records into a channel.
	objects := make(chan *gcs.Object, 100)
	b.Add(func(ctx context.Context) (err error) {
		defer close(objects)
		err = ListBlobObjects(ctx, bucket, namePrefix, objects)
		if err != nil {
			err = fmt.Errorf("ListBlobObjects: %v", err)
			return
		}

		return
	})

	// Parse and verify records, and write out scores.
	b.Add(func(ctx context.Context) (err error) {
		for o := range objects {
			// Parse and verify.
			var score Score
			score, err = ParseObjectRecord(o, namePrefix)
			if err != nil {
				err = fmt.Errorf("ParseObjectRecord: %v", err)
				return
			}

			// Send on the score.
			select {
			case scores <- score:

			// Cancelled?
			case <-ctx.Done():
				err = ctx.Err()
				return
			}
		}

		return
	})

	err = b.Join()
	return
}

////////////////////////////////////////////////////////////////////////
// Helpers
////////////////////////////////////////////////////////////////////////

func (s *GCSStore) makeName(score Score) (name string) {
	name = s.namePrefix + score.Hex()
	return
}

////////////////////////////////////////////////////////////////////////
// Public interface
////////////////////////////////////////////////////////////////////////

func (s *GCSStore) Store(blob []byte) (score Score, err error) {
	// Compute a score and an object name.
	score = ComputeScore(blob)
	name := s.makeName(score)

	// Create the object.
	crc32c := *gcsutil.CRC32C(blob)
	md5 := *gcsutil.MD5(blob)
	sha1 := sha1.Sum(blob)

	req := &gcs.CreateObjectRequest{
		Name:     name,
		Contents: bytes.NewReader(blob),
		CRC32C:   &crc32c,
		MD5:      &md5,

		Metadata: map[string]string{
			metadataKey_SHA1:   hex.EncodeToString(sha1[:]),
			metadataKey_CRC32C: fmt.Sprintf("%#08x", crc32c),
			metadataKey_MD5:    hex.EncodeToString(md5[:]),
		},
	}

	o, err := s.bucket.CreateObject(context.Background(), req)
	if err != nil {
		err = fmt.Errorf("CreateObject: %v", err)
		return
	}

	// Paranoid check: what we get back from GCS should match what we put in.
	if o.CRC32C != crc32c {
		panic(fmt.Sprintf(
			"CRC32C mismatch for object %q: 0x%08xv vs. 0x%08x",
			o.Name,
			o.CRC32C,
			crc32c))
	}

	if o.MD5 != md5 {
		panic(fmt.Sprintf(
			"MD5 mismatch for object %q: %s vs. %s",
			o.Name,
			hex.EncodeToString(o.MD5[:]),
			hex.EncodeToString(md5[:])))
	}

	return
}

func (s *GCSStore) Flush() (err error) {
	panic("GCSStore.Flush not supported; wiring code bug?")
}

func (s *GCSStore) Contains(score Score) (b bool) {
	panic("GCSStore.Contains not supported; wiring code bug?")
}

func (s *GCSStore) Load(score Score) (blob []byte, err error) {
	// Create a ReadCloser.
	req := &gcs.ReadObjectRequest{
		Name: s.makeName(score),
	}

	rc, err := s.bucket.NewReader(context.Background(), req)
	if err != nil {
		err = fmt.Errorf("NewReader: %v", err)
		return
	}

	// Read from it.
	blob, err = ioutil.ReadAll(rc)
	if err != nil {
		rc.Close()
		err = fmt.Errorf("ReadAll: %v", err)
		return
	}

	// Close it.
	err = rc.Close()
	if err != nil {
		err = fmt.Errorf("Close: %v", err)
		return
	}

	return
}

// List all of the blobs that are known to be durable in the bucket.
func (s *GCSStore) List() (scores []Score, err error) {
	b := syncutil.NewBundle(context.Background())

	// List into a channel.
	scoreChan := make(chan Score, 100)
	b.Add(func(ctx context.Context) (err error) {
		defer close(scoreChan)
		err = ListScores(ctx, s.bucket, s.namePrefix, scoreChan)
		if err != nil {
			err = fmt.Errorf("ListScores: %v", err)
			return
		}

		return
	})

	// Accumulate into the slice.
	b.Add(func(ctx context.Context) (err error) {
		for score := range scoreChan {
			scores = append(scores, score)
		}

		return
	})

	err = b.Join()
	return
}
<|endoftext|>


<commit_before>package blocks

import (
	"github.com/bitly/go-nsq"
	"github.com/bitly/go-simplejson"
	"log"
)

type readWriteHandler struct {
	OutChans map[string]chan *simplejson.Json
}

func (self readWriteHandler) HandleMessage(message *nsq.Message) error {
	out, err := simplejson.NewJson(message.Body)
	if err != nil {
		log.Fatal(err.Error())
	}
	broadcast(self.OutChans, out)
	return nil
}

func FromNSQ(b *Block) {

	type fromNSQRule struct {
		ReadTopic   string
		LookupdAddr string
		MaxInFlight int
		ReadChannel string
	}

	rule := &fromNSQRule{}

	unmarshal(<-b.Routes["set_rule"], &rule)

	reader, err := nsq.NewReader(rule.ReadTopic, rule.ReadChannel)
	if err != nil {
		log.Fatal(err.Error())
	}
	reader.SetMaxInFlight(rule.MaxInFlight)

	h := readWriteHandler{b.OutChans}
	reader.AddHandler(h)
	err = reader.ConnectToLookupd(rule.LookupdAddr)
	if err != nil {
		log.Fatal(err.Error())
	}
	for {
		select {
		case <-reader.ExitChan:
			break
		}
	}
}
<commit_msg>made sure fromNSQ works<commit_after>package blocks

import (
	"github.com/bitly/go-nsq"
	"github.com/bitly/go-simplejson"
	"log"
)

type readWriteHandler struct {
	OutChans map[string]chan *simplejson.Json
}

func (self readWriteHandler) HandleMessage(message *nsq.Message) error {
	out, err := simplejson.NewJson(message.Body)
	if err != nil {
		log.Fatal(err.Error())
	}
	broadcast(self.OutChans, out)
	return nil
}

func FromNSQ(b *Block) {

	type fromNSQRule struct {
		ReadTopic   string
		LookupdAddr string
		MaxInFlight int
		ReadChannel string
	}

	rule := &fromNSQRule{}

	unmarshal(<-b.Routes["set_rule"], &rule)

	reader, err := nsq.NewReader(rule.ReadTopic, rule.ReadChannel)
	if err != nil {
		log.Fatal(err.Error())
	}
	reader.SetMaxInFlight(rule.MaxInFlight)

	h := readWriteHandler{b.OutChans}
	reader.AddHandler(h)
	err = reader.ConnectToLookupd(rule.LookupdAddr)
	if err != nil {
		log.Fatal(err.Error())
	}
	for {
		select {
		case <-reader.ExitChan:
			break
		case msg := <-b.AddChan:
			updateOutChans(msg, b)
		}
	}
}
<|endoftext|>


<commit_before>package ensure

import (
	"log"
	"net/http"
)

func CloseNotifier(next http.HandlerFunc) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		_, ok := w.(http.CloseNotifier)
		if !ok {
			http.Error(w, "unable to handle client close notifications", http.StatusInternalServerError)
			return
		}
		next.ServeHTTP(w, r)
	}
}

func Post(next http.HandlerFunc) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		if r.Method != http.MethodPost {
			http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
			return
		}
		next.ServeHTTP(w, r)
	}
}
<commit_msg>Fixed imports.<commit_after>package ensure

import "net/http"

func CloseNotifier(next http.HandlerFunc) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		_, ok := w.(http.CloseNotifier)
		if !ok {
			http.Error(w, "unable to handle client close notifications", http.StatusInternalServerError)
			return
		}
		next.ServeHTTP(w, r)
	}
}

func Post(next http.HandlerFunc) http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		if r.Method != http.MethodPost {
			http.Error(w, "Method not allowed", http.StatusMethodNotAllowed)
			return
		}
		next.ServeHTTP(w, r)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"os"
	"text/tabwriter"
)

var cmdLogins = &Command{
	Run:   runLogins,
	Usage: "logins",
	Short: "List force.com logins used",
	Long: `
List force.com accounts

Examples:

  force logins
`,
}

func runLogins(cmd *Command, args []string) {
	active, _ := ActiveLogin()
	accounts, _ := Config.List("accounts")
	if len(accounts) == 0 {
		fmt.Println("no logins")
	} else {
		w := new(tabwriter.Writer)
		w.Init(os.Stdout, 1, 0, 1, ' ', 0)

		for _, account := range accounts {
			var creds ForceCredentials
			data, err := Config.Load("accounts", account)
			json.Unmarshal([]byte(data), &creds)

			if err != nil {
				return
			}

			var banner = fmt.Sprintf("\t%s", creds.InstanceUrl)
			if account == active {
				account = fmt.Sprintf("\x1b[31;1m%s (active)\x1b[0m", account)
			} else {
				account = fmt.Sprintf("%s \x1b[31;1m\x1b[0m", account)
			}
			fmt.Fprintln(w, fmt.Sprintf("%s%s", account, banner))
		}
		fmt.Fprintln(w)
		w.Flush()
	}

}

func ActiveLogin() (account string, err error) {
	account, err = Config.Load("current", "account")
	if err != nil {
		accounts, _ := Config.List("accounts")
		if len(accounts) > 0 {
			SetActiveLoginDefault()
		}
	}
	return
}

func ActiveCredentials() (creds ForceCredentials, err error) {
	account, err := ActiveLogin()
	if err != nil {
		return
	}
	data, err := Config.Load("accounts", account)
	json.Unmarshal([]byte(data), &creds)
	return
}

func ActiveForce() (force *Force, err error) {
	creds, err := ActiveCredentials()
	if err != nil {
		return
	}
	force = NewForce(creds)
	return
}

func SetActiveLoginDefault() (account string) {
	accounts, _ := Config.List("accounts")
	if len(accounts) > 0 {
		account = accounts[0]
		SetActiveLogin(account)
	}
	return
}

func SetActiveLogin(account string) (err error) {
	err = Config.Save("current", "account", account)
	return
}
<commit_msg>login update terminal title bar<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"os"
	"text/tabwriter"
	"strings"
)

var cmdLogins = &Command{
	Run:   runLogins,
	Usage: "logins",
	Short: "List force.com logins used",
	Long: `
List force.com accounts

Examples:

  force logins
`,
}

func runLogins(cmd *Command, args []string) {
	active, _ := ActiveLogin()
	accounts, _ := Config.List("accounts")
	if len(accounts) == 0 {
		fmt.Println("no logins")
	} else {
		w := new(tabwriter.Writer)
		w.Init(os.Stdout, 1, 0, 1, ' ', 0)

		for _, account := range accounts {
			if !strings.HasPrefix(account, "."){
				var creds ForceCredentials
				data, err := Config.Load("accounts", account)
				json.Unmarshal([]byte(data), &creds)

				if err != nil {
					return
				}

				var banner = fmt.Sprintf("\t%s", creds.InstanceUrl)
				if account == active {
					account = fmt.Sprintf("\x1b[31;1m%s (active)\x1b[0m", account)
				} else {
					account = fmt.Sprintf("%s \x1b[31;1m\x1b[0m", account)
				}
				fmt.Fprintln(w, fmt.Sprintf("%s%s", account, banner))
			}
		}
		fmt.Fprintln(w)
		w.Flush()
	}

}

func ActiveLogin() (account string, err error) {
	account, err = Config.Load("current", "account")
	if err != nil {
		accounts, _ := Config.List("accounts")
		if len(accounts) > 0 {
			SetActiveLoginDefault()
		}
	}
	return
}

func ActiveCredentials() (creds ForceCredentials, err error) {
	account, err := ActiveLogin()
	if err != nil {
		return
	}
	data, err := Config.Load("accounts", account)
	json.Unmarshal([]byte(data), &creds)
	return
}

func ActiveForce() (force *Force, err error) {
	creds, err := ActiveCredentials()
	if err != nil {
		return
	}
	force = NewForce(creds)
	return
}

func SetActiveLoginDefault() (account string) {
	accounts, _ := Config.List("accounts")
	if len(accounts) > 0 {
		account = accounts[0]
		SetActiveLogin(account)
	}
	return
}

func SetActiveLogin(account string) (err error) {
	err = Config.Save("current", "account", account)
	return
}
<|endoftext|>


<commit_before>package main

import (
	"os"
	"path/filepath"
	"runtime"
	"sort"
	"time"

	config "github.com/qiniu/logkit/conf"
	"github.com/qiniu/logkit/mgr"
	"github.com/qiniu/logkit/times"
	"github.com/qiniu/logkit/utils"

	_ "net/http/pprof"

	"github.com/qiniu/log"
)

//Config of logkit
type Config struct {
	MaxProcs         int      `json:"max_procs"`
	DebugLevel       int      `json:"debug_level"`
	BindHost         string   `json:"bind_host"`
	ConfsPath        []string `json:"confs_path"`
	CleanSelfLog     bool     `json:"clean_self_log"`
	CleanSelfDir     string   `json:"clean_self_dir"`
	CleanSelfPattern string   `json:"clean_self_pattern"`
	TimeLayouts      []string `json:"timeformat_layouts"`
	CleanSelfLogCnt  int      `json:"clean_self_cnt"`
	mgr.ManagerConfig
}

var conf Config

const defaultReserveCnt = 5
const defaultLogDir = "./run"
const defaultLogPattern = "*.log-*"

func getValidPath(confPaths []string) (paths []string) {
	paths = make([]string, 0)
	exits := make(map[string]bool)
	for _, v := range confPaths {
		rp, err := filepath.Abs(v)
		if err != nil {
			log.Errorf("Get real path of ConfsPath %v error %v, ignore it", v, rp)
			continue
		}
		if _, ok := exits[rp]; ok {
			log.Errorf("ConfsPath %v duplicated, ignore", rp)
			continue
		}
		exits[rp] = true
		paths = append(paths, rp)
	}
	return
}

func cleanLogkitLog(dir, pattern string, reserveCnt int) {
	var err error
	path := filepath.Join(dir, pattern)
	matches, err := filepath.Glob(path)
	if err != nil {
		log.Errorf("filepath.Glob path %v error %v", path, err)
		return
	}
	if len(matches) <= reserveCnt {
		return
	}
	sort.Strings(matches)
	for _, f := range matches[0 : len(matches)-reserveCnt] {
		err := os.Remove(f)
		if err != nil {
			log.Errorf("Remove %s failed , error: %v", f, err)
			continue
		}
	}
	return
}

func loopCleanLogkitLog(dir, pattern string, reserveCnt int, exitchan chan struct{}) {
	if len(dir) <= 0 {
		dir = defaultLogDir
	}
	if len(pattern) <= 0 {
		pattern = defaultLogPattern
	}
	if reserveCnt <= 0 {
		reserveCnt = defaultReserveCnt
	}
	ticker := time.NewTicker(time.Minute * 10)
	for {
		select {
		case <-exitchan:
			return
		case <-ticker.C:
			cleanLogkitLog(dir, pattern, reserveCnt)
		}
	}
}

func main() {
	config.Init("f", "qbox", "qboxlogexporter.conf")
	if err := config.Load(&conf); err != nil {
		log.Fatal("config.Load failed:", err)
	}
	log.Printf("Config: %#v", conf)
	if conf.TimeLayouts != nil {
		times.AddLayout(conf.TimeLayouts)
	}
	if conf.MaxProcs == 0 {
		conf.MaxProcs = runtime.NumCPU()
	}
	runtime.GOMAXPROCS(conf.MaxProcs)
	log.SetOutputLevel(conf.DebugLevel)

	m, err := mgr.NewManager(conf.ManagerConfig)
	if err != nil {
		log.Fatalf("NewManager: %v", err)
	}
	paths := getValidPath(conf.ConfsPath)
	if len(paths) <= 0 {
		log.Fatalf("Cannot read or create any ConfsPath %v", conf.ConfsPath)
	}
	if err = m.Watch(paths); err != nil {
		log.Fatalf("watch path error %v", err)
	}
	stopClean := make(chan struct{}, 0)
	if conf.CleanSelfLog {
		go loopCleanLogkitLog(conf.CleanSelfDir, conf.CleanSelfPattern, conf.CleanSelfLogCnt, stopClean)
	}
	if len(conf.BindHost) > 0 {
		m.BindHost = conf.BindHost
	}
	// start rest service
	rs := mgr.NewRestService(m)

	utils.WaitForInterrupt(func() {
		rs.Stop()
		stopClean <- struct{}{}
		m.Stop()
	})
}
<commit_msg>fix logkit hang without conf<commit_after>package main

import (
	"os"
	"path/filepath"
	"runtime"
	"sort"
	"time"

	config "github.com/qiniu/logkit/conf"
	"github.com/qiniu/logkit/mgr"
	"github.com/qiniu/logkit/times"
	"github.com/qiniu/logkit/utils"

	_ "net/http/pprof"

	"github.com/qiniu/log"
)

//Config of logkit
type Config struct {
	MaxProcs         int      `json:"max_procs"`
	DebugLevel       int      `json:"debug_level"`
	BindHost         string   `json:"bind_host"`
	ConfsPath        []string `json:"confs_path"`
	CleanSelfLog     bool     `json:"clean_self_log"`
	CleanSelfDir     string   `json:"clean_self_dir"`
	CleanSelfPattern string   `json:"clean_self_pattern"`
	TimeLayouts      []string `json:"timeformat_layouts"`
	CleanSelfLogCnt  int      `json:"clean_self_cnt"`
	mgr.ManagerConfig
}

var conf Config

const defaultReserveCnt = 5
const defaultLogDir = "./run"
const defaultLogPattern = "*.log-*"

func getValidPath(confPaths []string) (paths []string) {
	paths = make([]string, 0)
	exits := make(map[string]bool)
	for _, v := range confPaths {
		rp, err := filepath.Abs(v)
		if err != nil {
			log.Errorf("Get real path of ConfsPath %v error %v, ignore it", v, rp)
			continue
		}
		if _, ok := exits[rp]; ok {
			log.Errorf("ConfsPath %v duplicated, ignore", rp)
			continue
		}
		exits[rp] = true
		paths = append(paths, rp)
	}
	return
}

func cleanLogkitLog(dir, pattern string, reserveCnt int) {
	var err error
	path := filepath.Join(dir, pattern)
	matches, err := filepath.Glob(path)
	if err != nil {
		log.Errorf("filepath.Glob path %v error %v", path, err)
		return
	}
	if len(matches) <= reserveCnt {
		return
	}
	sort.Strings(matches)
	for _, f := range matches[0 : len(matches)-reserveCnt] {
		err := os.Remove(f)
		if err != nil {
			log.Errorf("Remove %s failed , error: %v", f, err)
			continue
		}
	}
	return
}

func loopCleanLogkitLog(dir, pattern string, reserveCnt int, exitchan chan struct{}) {
	if len(dir) <= 0 {
		dir = defaultLogDir
	}
	if len(pattern) <= 0 {
		pattern = defaultLogPattern
	}
	if reserveCnt <= 0 {
		reserveCnt = defaultReserveCnt
	}
	ticker := time.NewTicker(time.Minute * 10)
	for {
		select {
		case <-exitchan:
			return
		case <-ticker.C:
			cleanLogkitLog(dir, pattern, reserveCnt)
		}
	}
}

func main() {
	config.Init("f", "qbox", "qboxlogexporter.conf")
	if err := config.Load(&conf); err != nil {
		log.Fatal("config.Load failed:", err)
	}
	log.Printf("Config: %#v", conf)
	if conf.TimeLayouts != nil {
		times.AddLayout(conf.TimeLayouts)
	}
	if conf.MaxProcs == 0 {
		conf.MaxProcs = runtime.NumCPU()
	}
	runtime.GOMAXPROCS(conf.MaxProcs)
	log.SetOutputLevel(conf.DebugLevel)

	m, err := mgr.NewManager(conf.ManagerConfig)
	if err != nil {
		log.Fatalf("NewManager: %v", err)
	}
	paths := getValidPath(conf.ConfsPath)
	if len(paths) <= 0 {
		log.Fatalf("Cannot read or create any ConfsPath %v", conf.ConfsPath)
	}
	if err = m.Watch(paths); err != nil {
		log.Fatalf("watch path error %v", err)
	}
	stopClean := make(chan struct{}, 0)
	defer close(stopClean)
	if conf.CleanSelfLog {
		go loopCleanLogkitLog(conf.CleanSelfDir, conf.CleanSelfPattern, conf.CleanSelfLogCnt, stopClean)
	}
	if len(conf.BindHost) > 0 {
		m.BindHost = conf.BindHost
	}
	// start rest service
	rs := mgr.NewRestService(m)

	utils.WaitForInterrupt(func() {
		rs.Stop()
		if conf.CleanSelfLog {
			stopClean <- struct{}{}
		}
		m.Stop()
	})
}
<|endoftext|>


<commit_before>package whitelist

import (
	"errors"
	"log"
	"net"
	"net/http"
)

// A Lookup is a means of getting an IP from some data, such as an
// http request or network connection.
type Lookup interface {
	Address(...interface{}) (net.IP, error)
}

// NetConnLookup implements the Lookup interface for net.Conn connections.
type NetConnLookup struct{}

// Address extracts an IP from the remote address in the net.Conn. A
// single net.Conn should be passed to Address.
func (lu NetConnLookup) Address(args ...interface{}) (net.IP, error) {
	if len(args) != 1 {
		return nil, errors.New("whitelist: lookup requires a net.Conn")
	}

	var conn net.Conn
	switch arg := args[0].(type) {
	case net.Conn:
		conn = arg
	default:
		return nil, errors.New("whitelist: lookup requires a net.Conn")
	}

	netAddr := conn.RemoteAddr()
	if netAddr == nil {
		return nil, errors.New("No error returned")
	}

	addr, _, err := net.SplitHostPort(netAddr.String())
	if err != nil {
		return nil, err
	}

	ip := net.ParseIP(addr)
	return ip, nil
}

// HTTPRequestLookup implements the Lookup interface for http.Requests.
type HTTPRequestLookup struct{}

// Address extracts an IP from the remote address in a *http.Request. A
// single *http.Request should be passed to Address.
func (lu HTTPRequestLookup) Address(args ...interface{}) (net.IP, error) {
	if len(args) != 1 {
		return nil, errors.New("whitelist: lookup requires a *http.Request")
	}

	var req *http.Request
	switch arg := args[0].(type) {
	case *http.Request:
		req = arg
	default:
		return nil, errors.New("whitelist: lookup requires a *http.Request")
	}

	addr, _, err := net.SplitHostPort(req.RemoteAddr)
	if err != nil {
		return nil, err
	}

	ip := net.ParseIP(addr)
	return ip, nil

}

// Handler wraps an HTTP handler with IP whitelisting.
type Handler struct {
	allowHandler http.Handler
	denyHandler  http.Handler
	whitelist    Whitelist
	lookup       Lookup
}

// NewHandler returns a new whitelisting-wrapped HTTP handler. The
// allow handler should contain a handler that will be called if the
// request is whitelisted; the deny handler should contain a handler
// that will be called in the request is not whitelisted.
func NewHandler(allow, deny http.Handler, wl Whitelist) http.Handler {
	return &Handler{
		allowHandler: allow,
		denyHandler:  deny,
		whitelist:    wl,
		lookup:       HTTPRequestLookup{},
	}
}

// ServeHTTP wraps the request in a whitelist check.
func (h *Handler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
	ip, err := h.lookup.Address(req)
	if err != nil {
		log.Printf("failed to lookup request address: %v", err)
		status := http.StatusInternalServerError
		http.Error(w, http.StatusText(status), status)
		return
	}

	if h.whitelist.Permitted(ip) {
		h.allowHandler.ServeHTTP(w, req)
	} else {
		if h.denyHandler == nil {
			status := http.StatusUnauthorized
			http.Error(w, http.StatusText(status), status)
		} else {
			h.denyHandler.ServeHTTP(w, req)
		}
	}
}
<commit_msg>Untypo error message<commit_after>package whitelist

import (
	"errors"
	"log"
	"net"
	"net/http"
)

// A Lookup is a means of getting an IP from some data, such as an
// http request or network connection.
type Lookup interface {
	Address(...interface{}) (net.IP, error)
}

// NetConnLookup implements the Lookup interface for net.Conn connections.
type NetConnLookup struct{}

// Address extracts an IP from the remote address in the net.Conn. A
// single net.Conn should be passed to Address.
func (lu NetConnLookup) Address(args ...interface{}) (net.IP, error) {
	if len(args) != 1 {
		return nil, errors.New("whitelist: lookup requires a net.Conn")
	}

	var conn net.Conn
	switch arg := args[0].(type) {
	case net.Conn:
		conn = arg
	default:
		return nil, errors.New("whitelist: lookup requires a net.Conn")
	}

	netAddr := conn.RemoteAddr()
	if netAddr == nil {
		return nil, errors.New("whitelist: no address returned")
	}

	addr, _, err := net.SplitHostPort(netAddr.String())
	if err != nil {
		return nil, err
	}

	ip := net.ParseIP(addr)
	return ip, nil
}

// HTTPRequestLookup implements the Lookup interface for http.Requests.
type HTTPRequestLookup struct{}

// Address extracts an IP from the remote address in a *http.Request. A
// single *http.Request should be passed to Address.
func (lu HTTPRequestLookup) Address(args ...interface{}) (net.IP, error) {
	if len(args) != 1 {
		return nil, errors.New("whitelist: lookup requires a *http.Request")
	}

	var req *http.Request
	switch arg := args[0].(type) {
	case *http.Request:
		req = arg
	default:
		return nil, errors.New("whitelist: lookup requires a *http.Request")
	}

	addr, _, err := net.SplitHostPort(req.RemoteAddr)
	if err != nil {
		return nil, err
	}

	ip := net.ParseIP(addr)
	return ip, nil

}

// Handler wraps an HTTP handler with IP whitelisting.
type Handler struct {
	allowHandler http.Handler
	denyHandler  http.Handler
	whitelist    Whitelist
	lookup       Lookup
}

// NewHandler returns a new whitelisting-wrapped HTTP handler. The
// allow handler should contain a handler that will be called if the
// request is whitelisted; the deny handler should contain a handler
// that will be called in the request is not whitelisted.
func NewHandler(allow, deny http.Handler, wl Whitelist) http.Handler {
	return &Handler{
		allowHandler: allow,
		denyHandler:  deny,
		whitelist:    wl,
		lookup:       HTTPRequestLookup{},
	}
}

// ServeHTTP wraps the request in a whitelist check.
func (h *Handler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
	ip, err := h.lookup.Address(req)
	if err != nil {
		log.Printf("failed to lookup request address: %v", err)
		status := http.StatusInternalServerError
		http.Error(w, http.StatusText(status), status)
		return
	}

	if h.whitelist.Permitted(ip) {
		h.allowHandler.ServeHTTP(w, req)
	} else {
		if h.denyHandler == nil {
			status := http.StatusUnauthorized
			http.Error(w, http.StatusText(status), status)
		} else {
			h.denyHandler.ServeHTTP(w, req)
		}
	}
}
<|endoftext|>


<commit_before>package handler

import (
	"github.com/it512/sqlt"
	"github.com/jmoiron/sqlx"
)

type (
	SliceMapRowHandler struct {
		Data [][]map[string]interface{}
	}
)

func (h *SliceMapRowHandler) AddResultSet() {
	h.Data = append(h.Data, make([]map[string]interface{}, 0, 10))
}

func (h *SliceMapRowHandler) HandleRow(r sqlt.RowScanner) {
	idx := len(h.Data) - 1
	m := make(map[string]interface{})
	if e := sqlx.MapScan(r, m); e == nil {
		h.Data[idx] = append(h.Data[idx], m)
	}
}
<commit_msg>fix name<commit_after>package handler

import (
	"github.com/it512/sqlt"
	"github.com/jmoiron/sqlx"
)

type (
	SliceMapRowsHandler struct {
		Data [][]map[string]interface{}
	}
)

func (h *SliceMapRowsHandler) AddResultSet() {
	h.Data = append(h.Data, make([]map[string]interface{}, 0, 10))
}

func (h *SliceMapRowsHandler) HandleRow(r sqlt.RowScanner) {
	idx := len(h.Data) - 1
	m := make(map[string]interface{})
	if e := sqlx.MapScan(r, m); e == nil {
		h.Data[idx] = append(h.Data[idx], m)
	}
}
<|endoftext|>


<commit_before>package handler

import (
	"net/http"
	"errors"
	"strconv"
	"fmt"
	"net"
	"time"

	"github.com/m-lab/annotation-service/metrics"
)

func init() {
	// TODO: load tables here
	http.HandleFunc("/annotate", Annotate)
	metrics.SetupPrometheus()
}

// Annotate looks up IP address and returns geodata.
func Annotate(w http.ResponseWriter, r *http.Request){
	_, _, _, err := validate(w, r)
	if err != nil {
		fmt.Fprintf(w,"Invalid request")
	}else{
		// Fake response
		fmt.Fprintf(w, "[\n  {\"ip\": \"%s\", \"type\": \"STRING\"},\n  {\"country\": \"%s\", \"type\": \"STRING\"},\n  {\"countryAbrv\": \"%s\", \"type\": \"STRING\"},\n]", "1.4.128.0", "Thailand", "TH")
		// TODO: Figure out which table to use
		// TODO: Handle request
	}
}

// validates request syntax
// parses request and returns parameters
func validate(w http.ResponseWriter, r *http.Request) (IPversion int, s string, num time.Time, err error) {
	// Setup timers and counters for prometheus metrics.
	timerStart := time.Now()
	defer func(tStart time.Time) {
		metrics.Metrics_requestTimes.Observe(float64(time.Since(tStart).Nanoseconds()))
	}(timerStart)

	metrics.Metrics_activeRequests.Inc()
	defer metrics.Metrics_activeRequests.Dec()

	query := r.URL.Query()

	time_milli, err := strconv.ParseInt(query.Get("since_epoch"), 10, 64)
	if err != nil {
		return 0, s, num, errors.New("Invalid time")
	}

	ip := query.Get("ip_addr")

	newIP := net.ParseIP(ip)
	if newIP == nil {
		return 0, s, num, errors.New("Invalid IP address.")
	}
	if newIP.To4() != nil{
		return 4, ip, time.Unix(time_milli, 0), nil
	}
	return 6, ip, time.Unix(time_milli, 0), nil
}
<commit_msg>change handler comment<commit_after>package handler

import (
	"net/http"
	"errors"
	"strconv"
	"fmt"
	"net"
	"time"

	"github.com/m-lab/annotation-service/metrics"
)

func init() {
	// TODO: load tables here
	http.HandleFunc("/annotate", Annotate)
	metrics.SetupPrometheus()
}

// Annotate looks up IP address and returns geodata.
func Annotate(w http.ResponseWriter, r *http.Request){
	_, _, _, err := validate(w, r)
	if err != nil {
		fmt.Fprintf(w,"Invalid request")
	}else{
		// Fake response
		fmt.Fprintf(w, "[\n  {\"ip\": \"%s\", \"type\": \"STRING\"},\n  {\"country\": \"%s\", \"type\": \"STRING\"},\n  {\"countryAbrv\": \"%s\", \"type\": \"STRING\"},\n]", "1.4.128.0", "Thailand", "TH")
		// TODO: Figure out which table to use
		// TODO: Handle request
	}
}

// validates request syntax
// parses request and returns parameters
// 0 for IPversion means that there was an error.
func validate(w http.ResponseWriter, r *http.Request) (IPversion int, s string, num time.Time, err error) {
	// Setup timers and counters for prometheus metrics.
	timerStart := time.Now()
	defer func(tStart time.Time) {
		metrics.Metrics_requestTimes.Observe(float64(time.Since(tStart).Nanoseconds()))
	}(timerStart)

	metrics.Metrics_activeRequests.Inc()
	defer metrics.Metrics_activeRequests.Dec()

	query := r.URL.Query()

	time_milli, err := strconv.ParseInt(query.Get("since_epoch"), 10, 64)
	if err != nil {
		return 0, s, num, errors.New("Invalid time")
	}

	ip := query.Get("ip_addr")

	newIP := net.ParseIP(ip)
	if newIP == nil {
		return 0, s, num, errors.New("Invalid IP address.")
	}
	if newIP.To4() != nil{
		return 4, ip, time.Unix(time_milli, 0), nil
	}
	return 6, ip, time.Unix(time_milli, 0), nil
}
<|endoftext|>


<commit_before>package handlers

import (
	"fmt"
	"net/http"
	"strconv"

	"github.com/jrzimmerman/bestrida-server-go/models"
	log "github.com/sirupsen/logrus"
	"github.com/strava/go.strava"
)

// AuthHandler Route to display Auth button
func AuthHandler(w http.ResponseWriter, r *http.Request) {
	// you should make this a template in your real application
	fmt.Fprintf(w, `<a href="%s">`, authenticator.AuthorizationURL("", strava.Permissions.Public, false))
	fmt.Fprint(w, `<img src="/assets/img/btn_strava_connectwith_orange.png" />`)
	fmt.Fprint(w, `</a>`)
}

func oAuthSuccess(auth *strava.AuthorizationResponse, w http.ResponseWriter, r *http.Request) {
	userToken := string(auth.AccessToken)
	userID := strconv.Itoa(int(auth.Athlete.Id))
	log.WithField("USER TOKEN", userToken).Info("user token on oAuth success")
	log.WithField("USER ID", userID).Info("user id on oAuth success")
	url := "/login.html?oauth_token=" + userToken + "&userId=" + userID
	http.Redirect(w, r, url, http.StatusFound)
	_, err := models.RegisterUser(auth)
	if err != nil {
		log.Errorf("error registering user %d", auth.Athlete.Id)
	}

	GetFriendsFromStrava(auth.Athlete.Id)

	// page refers to the number of activities to collect for a user
	var page = 3
	GetUserSegmentsFromStrava(auth.Athlete.Id, page)
}

func oAuthFailure(err error, w http.ResponseWriter, r *http.Request) {
	if err == strava.OAuthAuthorizationDeniedErr {
		log.WithError(err).Error("The user clicked the 'Do not Authorize' button on the previous page.")
	} else if err == strava.OAuthInvalidCredentialsErr {
		log.WithError(err).Error("You provided an incorrect client_id or client_secret.")
	} else if err == strava.OAuthInvalidCodeErr {
		log.WithError(err).Error("The temporary token was not recognized, this shouldn't happen normally")
	} else if err == strava.OAuthServerErr {
		log.WithError(err).Error("There was some sort of server error, try again to see if the problem continues")
	} else {
		log.WithError(err).Error("authorization failure")
	}
}
<commit_msg>update auth handler to use goroutines<commit_after>package handlers

import (
	"fmt"
	"net/http"
	"strconv"

	"github.com/jrzimmerman/bestrida-server-go/models"
	log "github.com/sirupsen/logrus"
	"github.com/strava/go.strava"
)

// AuthHandler Route to display Auth button
func AuthHandler(w http.ResponseWriter, r *http.Request) {
	// you should make this a template in your real application
	fmt.Fprintf(w, `<a href="%s">`, authenticator.AuthorizationURL("", strava.Permissions.Public, false))
	fmt.Fprint(w, `<img src="/assets/img/btn_strava_connectwith_orange.png" />`)
	fmt.Fprint(w, `</a>`)
}

func oAuthSuccess(auth *strava.AuthorizationResponse, w http.ResponseWriter, r *http.Request) {
	userToken := string(auth.AccessToken)
	userID := strconv.Itoa(int(auth.Athlete.Id))
	log.WithField("USER TOKEN", userToken).Info("user token on oAuth success")
	log.WithField("USER ID", userID).Info("user id on oAuth success")
	url := "/login.html?oauth_token=" + userToken + "&userId=" + userID
	http.Redirect(w, r, url, http.StatusFound)
	_, err := models.RegisterUser(auth)
	if err != nil {
		log.Errorf("error registering user %d", auth.Athlete.Id)
	}

	go GetFriendsFromStrava(auth.Athlete.Id)

	// page refers to the number of activities to collect for a user
	var page = 3
	go GetUserSegmentsFromStrava(auth.Athlete.Id, page)
}

func oAuthFailure(err error, w http.ResponseWriter, r *http.Request) {
	if err == strava.OAuthAuthorizationDeniedErr {
		log.WithError(err).Error("The user clicked the 'Do not Authorize' button on the previous page.")
	} else if err == strava.OAuthInvalidCredentialsErr {
		log.WithError(err).Error("You provided an incorrect client_id or client_secret.")
	} else if err == strava.OAuthInvalidCodeErr {
		log.WithError(err).Error("The temporary token was not recognized, this shouldn't happen normally")
	} else if err == strava.OAuthServerErr {
		log.WithError(err).Error("There was some sort of server error, try again to see if the problem continues")
	} else {
		log.WithError(err).Error("authorization failure")
	}
}
<|endoftext|>


<commit_before>// Copyright (c) 2020-2021 Tigera, Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package xdp

import (
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path"
	"strconv"
	"strings"

	"github.com/projectcalico/felix/bpf"
	log "github.com/sirupsen/logrus"
)

type AttachPoint struct {
	Iface    string
	LogLevel string
	Modes    []bpf.XDPMode
}

func (ap *AttachPoint) IfaceName() string {
	return ap.Iface
}

func (ap *AttachPoint) JumpMapFDMapKey() string {
	return "xdp"
}

func (ap *AttachPoint) FileName() string {
	logLevel := strings.ToLower(ap.LogLevel)
	if logLevel == "off" {
		logLevel = "no_log"
	}
	return "xdp_" + logLevel + ".o"
}

func (ap *AttachPoint) SectionName() string {
	return "calico_entrypoint_xdp"
}

func (ap *AttachPoint) Log() *log.Entry {
	return log.WithFields(log.Fields{
		"iface":    ap.Iface,
		"modes":    ap.Modes,
		"logLevel": ap.LogLevel,
	})
}

func (ap *AttachPoint) AttachProgram() error {
	preCompiledBinary := path.Join(bpf.ObjectDir, ap.FileName())
	sectionName := ap.SectionName()

	// Patch the binary so that its log prefix is like "eth0------X".
	tempDir, err := ioutil.TempDir("", "calico-xdp")
	if err != nil {
		return fmt.Errorf("failed to create temporary directory: %w", err)
	}
	defer func() {
		_ = os.RemoveAll(tempDir)
	}()
	tempBinary := path.Join(tempDir, ap.FileName())
	err = ap.patchBinary(preCompiledBinary, tempBinary)
	if err != nil {
		ap.Log().WithError(err).Error("Failed to patch binary")
		return err
	}

	var errs []error
	for _, mode := range ap.Modes {
		ap.Log().Debugf("Attempt XDP attach with mode %v", mode)

		// First remove any existing program.
		cmd := exec.Command("ip", "link", "set", "dev", ap.Iface, mode.String(), "off")
		ap.Log().Debugf("Running: %v %v", cmd.Path, cmd.Args)
		out, err := cmd.CombinedOutput()
		ap.Log().WithField("mode", mode).Debugf("Result: err=%v out=\n%v", err, string(out))

		// Now attach the program we want.
		cmd = exec.Command("ip", "link", "set", "dev", ap.Iface, mode.String(), "object", tempBinary, "section", sectionName)
		ap.Log().Debugf("Running: %v %v", cmd.Path, cmd.Args)
		out, err = cmd.CombinedOutput()
		ap.Log().WithField("mode", mode).Debugf("Result: err=%v out=\n%v", err, string(out))
		if err == nil {
			ap.Log().Infof("Successful XDP attachment with mode %v", mode)
			return nil
		}
		errs = append(errs, err)
	}
	return fmt.Errorf("Couldn't attach XDP program %v section %v to iface %v; modes=%v errs=%v", tempBinary, sectionName, ap.Iface, ap.Modes, errs)
}

func (ap *AttachPoint) patchBinary(ifile, ofile string) error {
	b, err := bpf.BinaryFromFile(ifile)
	if err != nil {
		return fmt.Errorf("failed to read pre-compiled BPF binary: %w", err)
	}

	b.PatchLogPrefix(ap.Iface)

	err = b.WriteToFile(ofile)
	if err != nil {
		return fmt.Errorf("failed to write pre-compiled BPF binary: %w", err)
	}

	return nil
}

func (ap *AttachPoint) IsAttached() (bool, error) {
	_, err := ap.ProgramID()
	return err == nil, err
}

func (ap *AttachPoint) ProgramID() (string, error) {
	cmd := exec.Command("ip", "link", "show", "dev", ap.Iface)
	ap.Log().Debugf("Running: %v %v", cmd.Path, cmd.Args)
	out, err := cmd.CombinedOutput()
	ap.Log().Debugf("Result: err=%v out=\n%v", err, string(out))
	if err != nil {
		return "", fmt.Errorf("Couldn't check for XDP program on iface %v: %v", ap.Iface, err)
	}
	s := strings.Fields(string(out))
	for i := range s {
		// Example of output:
		//
		// 196: test_A@test_B: <BROADCAST,MULTICAST> mtu 1500 xdpgeneric qdisc noop state DOWN mode DEFAULT group default qlen 1000
		//    link/ether 1a:d0:df:a5:12:59 brd ff:ff:ff:ff:ff:ff
		//    prog/xdp id 175 tag 5199fa060702bbff jited
		if s[i] == "prog/xdp" && len(s) > i+2 && s[i+1] == "id" {
			_, err := strconv.Atoi(s[i+2])
			if err != nil {
				return "", fmt.Errorf("Couldn't parse ID following 'prog/xdp' err=%v out=\n%v", err, string(out))
			}
			return s[i+2], nil
		}
	}
	return "", fmt.Errorf("Couldn't find 'prog/xdp id <ID>' err=%v out=\n%v", err, string(out))
}
<commit_msg>XDP attach: use -force and make sure always to clean up<commit_after>// Copyright (c) 2020-2021 Tigera, Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package xdp

import (
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path"
	"strconv"
	"strings"

	"github.com/projectcalico/felix/bpf"
	log "github.com/sirupsen/logrus"
)

type AttachPoint struct {
	Iface    string
	LogLevel string
	Modes    []bpf.XDPMode
}

func (ap *AttachPoint) IfaceName() string {
	return ap.Iface
}

func (ap *AttachPoint) JumpMapFDMapKey() string {
	return "xdp"
}

func (ap *AttachPoint) FileName() string {
	logLevel := strings.ToLower(ap.LogLevel)
	if logLevel == "off" {
		logLevel = "no_log"
	}
	return "xdp_" + logLevel + ".o"
}

func (ap *AttachPoint) SectionName() string {
	return "calico_entrypoint_xdp"
}

func (ap *AttachPoint) Log() *log.Entry {
	return log.WithFields(log.Fields{
		"iface":    ap.Iface,
		"modes":    ap.Modes,
		"logLevel": ap.LogLevel,
	})
}

func (ap *AttachPoint) AttachProgram() error {
	preCompiledBinary := path.Join(bpf.ObjectDir, ap.FileName())
	sectionName := ap.SectionName()

	// Patch the binary so that its log prefix is like "eth0------X".
	tempDir, err := ioutil.TempDir("", "calico-xdp")
	if err != nil {
		return fmt.Errorf("failed to create temporary directory: %w", err)
	}
	defer func() {
		_ = os.RemoveAll(tempDir)
	}()
	tempBinary := path.Join(tempDir, ap.FileName())
	err = ap.patchBinary(preCompiledBinary, tempBinary)
	if err != nil {
		ap.Log().WithError(err).Error("Failed to patch binary")
		return err
	}

	// Note that there are a few considerations here.
	//
	// Firstly, we use -force when attaching, so as to minimise any flap in the XDP program when
	// restarting or upgrading Felix.
	//
	// Secondly, we need to consider any other XDP programs that might be there at start of day.
	// (Either 3rd party, or maybe some earlier version of Felix.)  We fundamentally have to
	// clean those up, as our XDP usage can coexist with anyone else's.
	//
	// Thirdly, for a given host interface, is it possible for our best attachment mode to
	// change, other than at start of day?  In principle it's possible if we do a patch that
	// alters some conditionally included code.  We don't have that for XDP right now, but we
	// conceivably could in future.  So safest to assume it's possible for the attachment mode
	// to improve, which means needing to do `off` for subsequent modes.
	//
	// Hence this logic:
	// - Loop through the modes.
	//   - If we haven't yet successfully attached (for this loop), do the attachment (with
	//     -force).
	//   - If that failed, or we attached with an earlier mode, do `off` to remove any existing
	//     program with this mode.
	//   - Continue through remaining modes even if attachment already successful.
	var errs []error
	attachmentSucceeded := false
	for _, mode := range ap.Modes {
		ap.Log().Debugf("XDP remove/attach with mode %v", mode)

		// We will need to do "ip link set dev <dev> xdp off" if we've successfully attached
		// with an earlier mode, or if we fail to attach with this mode.  So we only _don't_
		// need "off" if we successfully attach in this iteration.
		offNeeded := true

		if !attachmentSucceeded {
			// Try to attach our XDP program in this mode.
			cmd := exec.Command("ip", "-force", "link", "set", "dev", ap.Iface, mode.String(), "object", tempBinary, "section", sectionName)
			ap.Log().Debugf("Running: %v %v", cmd.Path, cmd.Args)
			out, err := cmd.CombinedOutput()
			ap.Log().WithField("mode", mode).Debugf("Result: err=%v out=\n%v", err, string(out))
			if err == nil {
				ap.Log().Infof("Successful XDP attachment with mode %v", mode)
				attachmentSucceeded = true
				offNeeded = false
			} else {
				errs = append(errs, err)
			}
		}

		if offNeeded {
			// Remove any existing program for this mode.
			cmd := exec.Command("ip", "link", "set", "dev", ap.Iface, mode.String(), "off")
			ap.Log().Debugf("Running: %v %v", cmd.Path, cmd.Args)
			out, err := cmd.CombinedOutput()
			ap.Log().WithField("mode", mode).Debugf("Result: err=%v out=\n%v", err, string(out))
		}
	}
	if !attachmentSucceeded {
		return fmt.Errorf("Couldn't attach XDP program %v section %v to iface %v; modes=%v errs=%v", tempBinary, sectionName, ap.Iface, ap.Modes, errs)
	}
	return nil
}

func (ap *AttachPoint) patchBinary(ifile, ofile string) error {
	b, err := bpf.BinaryFromFile(ifile)
	if err != nil {
		return fmt.Errorf("failed to read pre-compiled BPF binary: %w", err)
	}

	b.PatchLogPrefix(ap.Iface)

	err = b.WriteToFile(ofile)
	if err != nil {
		return fmt.Errorf("failed to write pre-compiled BPF binary: %w", err)
	}

	return nil
}

func (ap *AttachPoint) IsAttached() (bool, error) {
	_, err := ap.ProgramID()
	return err == nil, err
}

func (ap *AttachPoint) ProgramID() (string, error) {
	cmd := exec.Command("ip", "link", "show", "dev", ap.Iface)
	ap.Log().Debugf("Running: %v %v", cmd.Path, cmd.Args)
	out, err := cmd.CombinedOutput()
	ap.Log().Debugf("Result: err=%v out=\n%v", err, string(out))
	if err != nil {
		return "", fmt.Errorf("Couldn't check for XDP program on iface %v: %v", ap.Iface, err)
	}
	s := strings.Fields(string(out))
	for i := range s {
		// Example of output:
		//
		// 196: test_A@test_B: <BROADCAST,MULTICAST> mtu 1500 xdpgeneric qdisc noop state DOWN mode DEFAULT group default qlen 1000
		//    link/ether 1a:d0:df:a5:12:59 brd ff:ff:ff:ff:ff:ff
		//    prog/xdp id 175 tag 5199fa060702bbff jited
		if s[i] == "prog/xdp" && len(s) > i+2 && s[i+1] == "id" {
			_, err := strconv.Atoi(s[i+2])
			if err != nil {
				return "", fmt.Errorf("Couldn't parse ID following 'prog/xdp' err=%v out=\n%v", err, string(out))
			}
			return s[i+2], nil
		}
	}
	return "", fmt.Errorf("Couldn't find 'prog/xdp id <ID>' err=%v out=\n%v", err, string(out))
}
<|endoftext|>


<commit_before>package cas

import "bytes"
import "fmt"
import "math/big"
import "sort"

type Expression struct {
	Parts []Ex
}

func HeadAssertion(ex Ex, head string) (*Expression, bool) {
	expr, isExpr := ex.(*Expression)
	if isExpr {
		sym, isSym := expr.Parts[0].(*Symbol)
		if isSym {
			if sym.Name == head {
				return expr, true
			}
		}
	}
	return &Expression{}, false
}

func (this *Expression) mergeSequences(es *EvalState, headStr string, shouldEval bool) {
	// TODO: I should not be attempting to merge the head if it happens to be
	// a Sequence type
	origLen := len(this.Parts)
	offset := 0
	for i := 0; i < origLen; i++ {
		j := i + offset
		e := this.Parts[j]
		seq, isseq := HeadAssertion(e, headStr)
		if shouldEval {
			for j := 1; j < len(seq.Parts); j++ {
				seq.Parts[j] = seq.Parts[j].Eval(es)
			}
		}
		if isseq {
			start := j
			end := j + 1
			if j == 0 {
				this.Parts = append(seq.Parts[1:], this.Parts[end:]...)
			} else if j == len(this.Parts)-1 {
				this.Parts = append(this.Parts[:start], seq.Parts[1:]...)
			} else {
				// All of these deep copies may not be needed.
				this.Parts = append(append(this.DeepCopy().(*Expression).Parts[:start], seq.DeepCopy().(*Expression).Parts[1:]...), this.DeepCopy().(*Expression).Parts[end:]...)
			}
			offset += len(seq.Parts[1:]) - 1
		}
	}
}

func (this *Expression) Eval(es *EvalState) Ex {
	shouldEval := true
	var lastEx Ex = this.DeepCopy()
	var currEx Ex = this.DeepCopy()
	for shouldEval {
		curr, isExpr := currEx.(*Expression)
		// Transition to the right Eval() if this is no longer an Expression
		if !isExpr {
			return currEx.Eval(es)
		}

		// Start by evaluating each argument
		headSym, headIsSym := &Symbol{}, false
		if len(curr.Parts) > 0 {
			headSym, headIsSym = curr.Parts[0].(*Symbol)
		}
		for i := range curr.Parts {
			if headIsSym && i == 1 && IsHoldFirst(headSym) {
				continue
			}
			if headIsSym && i > 1 && IsHoldRest(headSym) {
				continue
			}
			//if headIsSym && IsAttribute(headSym, "HoldAll", es) {
			if headIsSym && IsHoldAll(headSym) {
				continue
			}
			curr.Parts[i] = curr.Parts[i].Eval(es)
		}

		// If any of the parts are Sequence, merge them with parts
		curr.mergeSequences(es, "Sequence", false)
		curr.mergeSequences(es, "Evaluate", true)

		headAsSym, isHeadSym := curr.Parts[0].(*Symbol)
		if isHeadSym {
			if IsFlat(headAsSym) {
				curr.mergeSequences(es, headAsSym.Name, false)
			}
			if IsOrderless(headAsSym) {
				sort.Sort(curr)
			}
		}
		pureFunction, isPureFunction := HeadAssertion(curr.Parts[0], "Function")
		if isHeadSym {
			headStr := headAsSym.Name

			theRes, isDefined := es.GetDef(headStr, curr)
			legacyEvalFn, hasLegacyEvalFn := es.legacyEvalFns[headStr]
			if isDefined {
				//fmt.Printf("%v, %v, %v\n", headStr, curr, theRes)
				currEx = theRes
			} else if hasLegacyEvalFn {
				currEx = legacyEvalFn(curr, es)
			} else if headStr == "Apply" {
				currEx = curr.EvalApply(es)
			}
		} else if isPureFunction {
			currEx = pureFunction.EvalFunction(es, curr.Parts[1:])
		}
		if IsSameQ(currEx, lastEx, &es.CASLogger) {
			shouldEval = false
		}
		lastEx = currEx
	}
	return currEx
}

func (this *Expression) EvalFunction(es *EvalState, args []Ex) Ex {
	if len(this.Parts) == 2 {
		toReturn := this.Parts[1].DeepCopy()
		for i, arg := range args {
			toReturn = ReplaceAll(toReturn,
				&Expression{[]Ex{
					&Symbol{"Rule"},
					&Expression{[]Ex{
						&Symbol{"Slot"},
						&Integer{big.NewInt(int64(i + 1))},
					}},
					arg,
				}}, &es.CASLogger, EmptyPD())
		}
		return toReturn
	} else if len(this.Parts) == 3 {
		repSym, ok := this.Parts[1].(*Symbol)
		if !ok {
			return this
		}
		toReturn := this.Parts[2].DeepCopy()
		toReturn = ReplaceAll(toReturn,
			&Expression{[]Ex{
				&Symbol{"Rule"},
				repSym,
				args[0],
			}}, &es.CASLogger, EmptyPD())
		return toReturn
	}
	return this
}

func (this *Expression) ReplaceAll(r *Expression, cl *CASLogger) Ex {
	cl.Debugf("In Expression.ReplaceAll. First trying IsMatchQ(this, r.Parts[1], es).")
	cl.Debugf("Rule r is: %s", r)

	matchq, matches := IsMatchQ(this, r.Parts[1], EmptyPD(), cl)
	toreturn := ReplacePD(r.Parts[2].DeepCopy(), cl, matches)
	if matchq {
		cl.Debugf("After MatchQ, rule is: %s", r)
		cl.Debugf("MatchQ succeeded. Returning r.Parts[2]: %s", r.Parts[2])
		return toreturn
	}

	thisSym, thisSymOk := this.Parts[0].(*Symbol)
	lhsExpr, lhsExprOk := r.Parts[1].(*Expression)
	if lhsExprOk {
		otherSym, otherSymOk := lhsExpr.Parts[0].(*Symbol)
		if thisSymOk && otherSymOk {
			if thisSym.Name == otherSym.Name {
				if IsOrderless(thisSym) {
					cl.Debugf("r.Parts[1] is Orderless. Now running CommutativeReplace")
					replaced := this.Parts[1:len(this.Parts)]
					CommutativeReplace(&replaced, lhsExpr.Parts[1:len(lhsExpr.Parts)], r.Parts[2], cl)
					this.Parts = this.Parts[0:1]
					this.Parts = append(this.Parts, replaced...)
				}
			}
		}
	}

	for i := range this.Parts {
		this.Parts[i] = ReplaceAll(this.Parts[i], r, cl, EmptyPD())
	}
	return this
}

func (this *Expression) String() string {
	headAsSym, isHeadSym := this.Parts[0].(*Symbol)
	fullForm := false
	if isHeadSym && !fullForm {
		res, ok := "", false
		headStr := headAsSym.Name
		toStringFn, hasToStringFn := toStringFns[headStr]
		if hasToStringFn {
			ok, res = toStringFn(this)
		} else if headStr == "Times" {
			return this.ToStringTimes()
		} else if headStr == "Plus" {
			return this.ToStringPlus()
		} else if headStr == "Power" {
			return this.ToStringPower()
		} else if headStr == "Equal" {
			return this.ToStringEqual()
		} else if headStr == "SameQ" {
			return this.ToStringSameQ()
		} else if headStr == "ReplaceAll" {
			return this.ToStringReplaceAll()
		} else if headStr == "ReplaceRepeated" {
			return this.ToStringReplaceRepeated()
		} else if headStr == "Pattern" {
			return this.ToStringPattern()
		} else if headStr == "Rule" {
			return this.ToStringRule()
		} else if headStr == "RuleDelayed" {
			return this.ToStringRuleDelayed()
		} else if headStr == "Set" {
			return this.ToStringSet()
		} else if headStr == "SetDelayed" {
			return this.ToStringSetDelayed()
		} else if headStr == "List" {
			return this.ToStringList()
		}
		if ok {
			return res
		}
	}

	// Default printing format
	var buffer bytes.Buffer
	buffer.WriteString(this.Parts[0].String())
	buffer.WriteString("[")
	for i, e := range this.Parts {
		if i == 0 {
			continue
		}
		buffer.WriteString(e.String())
		if i != len(this.Parts)-1 {
			buffer.WriteString(", ")
		}
	}
	buffer.WriteString("]")
	return buffer.String()
}

func IsAttribute(sm *Symbol, attr string, es *EvalState) bool {
	if sm.Name == "MemberQ" {
		return attr == "Protected"
	} else if sm.Name == "Attributes" {
		return attr == "Protected" || attr == "HoldAll" || attr == "Listable"
	} else if sm.Name == "List" {
		return attr == "Protected" || attr == "Locked"
	} else if sm.Name == "Pattern" {
		return attr == "Protected" || attr == "HoldFirst"
	} else if sm.Name == "Blank" {
		return attr == "Protected"
	} else if sm.Name == "Rule" {
		return attr == "Protected" || attr == "SequenceHold"
	} else if sm.Name == "Times" || sm.Name == "Plus" {
		return attr == "Flat" || attr == "Listable" || attr == "NumericFunction" || attr == "OneIdentity" || attr == "Orderless" || attr == "Protected"
	} else if sm.Name == "Power" {
		return attr == "Listable" || attr == "NumericFunction" || attr == "OneIdentity" || attr == "Protected"
	} else if sm.Name == "ReplaceRepeated" {
		return attr == "Protected"
	} else if sm.Name == "Equal" {
		return attr == "Protected"
	} // This is probably slow because it requires tons of Defined copies
	fmt.Printf("IsAttribute(%v, %v)\n", sm.Name, attr)
	res := (&Expression{[]Ex{
		&Symbol{"MemberQ"},
		&Expression{[]Ex{
			&Symbol{"Attributes"},
			sm,
		}},
		&Symbol{attr},
	}}).Eval(es)
	resSym, resIsSym := res.(*Symbol)
	if resIsSym {
		return resSym.Name == "True"
	}
	return false
}

// TODO: convert to a map
func IsOrderless(sym *Symbol) bool {
	if sym.Name == "Times" {
		return true
	} else if sym.Name == "Plus" {
		return true
	}
	return false
}

// TODO: convert to a map
func IsFlat(sym *Symbol) bool {
	if sym.Name == "Times" {
		return true
	} else if sym.Name == "Plus" {
		return true
	}
	return false
}

// TODO: convert to a map
func IsHoldFirst(sym *Symbol) bool {
	if sym.Name == "Set" {
		return true
	} else if sym.Name == "Pattern" {
		return true
	}
	return false
}

// TODO: convert to a map
func IsHoldRest(sym *Symbol) bool {
	if sym.Name == "If" {
		return true
	} else if sym.Name == "RuleDelayed" {
		return true
	}
	return false
}

// TODO: convert to a map
func IsHoldAll(sym *Symbol) bool {
	if sym.Name == "SetDelayed" {
		return true
	}
	if sym.Name == "Table" {
		return true
	}
	if sym.Name == "Clear" {
		return true
	}
	if sym.Name == "Timing" {
		return true
	}
	if sym.Name == "Hold" {
		return true
	}
	if sym.Name == "CompoundExpression" {
		return true
	}
	if sym.Name == "Condition" {
		return true
	}
	if sym.Name == "Function" {
		return true
	}
	return false
}

func (this *Expression) IsEqual(otherEx Ex, cl *CASLogger) string {
	//thisEvaled := this.Eval(es)
	//otherEx = otherEx.Eval(es)
	//this, ok := thisEvaled.(*Expression)
	//if !ok {
	//return thisEvaled.IsEqual(otherEx, cl)
	//}

	other, ok := otherEx.(*Expression)
	if !ok {
		return "EQUAL_UNK"
	}

	thisSym, thisSymOk := this.Parts[0].(*Symbol)
	otherSym, otherSymOk := other.Parts[0].(*Symbol)
	if thisSymOk && otherSymOk {
		if thisSym.Name == otherSym.Name {
			if IsOrderless(thisSym) {
				return CommutativeIsEqual(this.Parts[1:len(this.Parts)], other.Parts[1:len(other.Parts)], cl)
			}
		}
	}

	return FunctionIsEqual(this.Parts, other.Parts, cl)
}

func (this *Expression) DeepCopy() Ex {
	var thiscopy = &Expression{}
	for i := range this.Parts {
		thiscopy.Parts = append(thiscopy.Parts, this.Parts[i].DeepCopy())
	}
	return thiscopy
}

// Implement the sort.Interface
func (this *Expression) Len() int {
	return len(this.Parts) - 1
}

func (this *Expression) Less(i, j int) bool {
	return ExOrder(this.Parts[i+1], this.Parts[j+1]) == 1
}

func (this *Expression) Swap(i, j int) {
	this.Parts[j+1], this.Parts[i+1] = this.Parts[i+1], this.Parts[j+1]
}

// Apply
func (this *Expression) EvalApply(es *EvalState) Ex {
	if len(this.Parts) != 3 {
		return this
	}

	sym, isSym := this.Parts[1].(*Symbol)
	list, isList := HeadAssertion(this.Parts[2].DeepCopy(), "List")
	if isSym && isList {
		toReturn := &Expression{[]Ex{sym}}
		toReturn.Parts = append(toReturn.Parts, list.Parts[1:]...)
		return toReturn.Eval(es)
	}
	return this.Parts[2]
}

func GetExpressionDefinitions() (defs []Definition) {
	return
}
<commit_msg>Move Apply function to the definition registry.<commit_after>package cas

import "bytes"
import "fmt"
import "math/big"
import "sort"

type Expression struct {
	Parts []Ex
}

func HeadAssertion(ex Ex, head string) (*Expression, bool) {
	expr, isExpr := ex.(*Expression)
	if isExpr {
		sym, isSym := expr.Parts[0].(*Symbol)
		if isSym {
			if sym.Name == head {
				return expr, true
			}
		}
	}
	return &Expression{}, false
}

func (this *Expression) mergeSequences(es *EvalState, headStr string, shouldEval bool) {
	// TODO: I should not be attempting to merge the head if it happens to be
	// a Sequence type
	origLen := len(this.Parts)
	offset := 0
	for i := 0; i < origLen; i++ {
		j := i + offset
		e := this.Parts[j]
		seq, isseq := HeadAssertion(e, headStr)
		if shouldEval {
			for j := 1; j < len(seq.Parts); j++ {
				seq.Parts[j] = seq.Parts[j].Eval(es)
			}
		}
		if isseq {
			start := j
			end := j + 1
			if j == 0 {
				this.Parts = append(seq.Parts[1:], this.Parts[end:]...)
			} else if j == len(this.Parts)-1 {
				this.Parts = append(this.Parts[:start], seq.Parts[1:]...)
			} else {
				// All of these deep copies may not be needed.
				this.Parts = append(append(this.DeepCopy().(*Expression).Parts[:start], seq.DeepCopy().(*Expression).Parts[1:]...), this.DeepCopy().(*Expression).Parts[end:]...)
			}
			offset += len(seq.Parts[1:]) - 1
		}
	}
}

func (this *Expression) Eval(es *EvalState) Ex {
	shouldEval := true
	var lastEx Ex = this.DeepCopy()
	var currEx Ex = this.DeepCopy()
	for shouldEval {
		curr, isExpr := currEx.(*Expression)
		// Transition to the right Eval() if this is no longer an Expression
		if !isExpr {
			return currEx.Eval(es)
		}

		// Start by evaluating each argument
		headSym, headIsSym := &Symbol{}, false
		if len(curr.Parts) > 0 {
			headSym, headIsSym = curr.Parts[0].(*Symbol)
		}
		for i := range curr.Parts {
			if headIsSym && i == 1 && IsHoldFirst(headSym) {
				continue
			}
			if headIsSym && i > 1 && IsHoldRest(headSym) {
				continue
			}
			//if headIsSym && IsAttribute(headSym, "HoldAll", es) {
			if headIsSym && IsHoldAll(headSym) {
				continue
			}
			curr.Parts[i] = curr.Parts[i].Eval(es)
		}

		// If any of the parts are Sequence, merge them with parts
		curr.mergeSequences(es, "Sequence", false)
		curr.mergeSequences(es, "Evaluate", true)

		headAsSym, isHeadSym := curr.Parts[0].(*Symbol)
		if isHeadSym {
			if IsFlat(headAsSym) {
				curr.mergeSequences(es, headAsSym.Name, false)
			}
			if IsOrderless(headAsSym) {
				sort.Sort(curr)
			}
		}
		pureFunction, isPureFunction := HeadAssertion(curr.Parts[0], "Function")
		if isHeadSym {
			headStr := headAsSym.Name

			theRes, isDefined := es.GetDef(headStr, curr)
			legacyEvalFn, hasLegacyEvalFn := es.legacyEvalFns[headStr]
			if isDefined {
				//fmt.Printf("%v, %v, %v\n", headStr, curr, theRes)
				currEx = theRes
			} else if hasLegacyEvalFn {
				currEx = legacyEvalFn(curr, es)
			}
		} else if isPureFunction {
			currEx = pureFunction.EvalFunction(es, curr.Parts[1:])
		}
		if IsSameQ(currEx, lastEx, &es.CASLogger) {
			shouldEval = false
		}
		lastEx = currEx
	}
	return currEx
}

func (this *Expression) EvalFunction(es *EvalState, args []Ex) Ex {
	if len(this.Parts) == 2 {
		toReturn := this.Parts[1].DeepCopy()
		for i, arg := range args {
			toReturn = ReplaceAll(toReturn,
				&Expression{[]Ex{
					&Symbol{"Rule"},
					&Expression{[]Ex{
						&Symbol{"Slot"},
						&Integer{big.NewInt(int64(i + 1))},
					}},
					arg,
				}}, &es.CASLogger, EmptyPD())
		}
		return toReturn
	} else if len(this.Parts) == 3 {
		repSym, ok := this.Parts[1].(*Symbol)
		if !ok {
			return this
		}
		toReturn := this.Parts[2].DeepCopy()
		toReturn = ReplaceAll(toReturn,
			&Expression{[]Ex{
				&Symbol{"Rule"},
				repSym,
				args[0],
			}}, &es.CASLogger, EmptyPD())
		return toReturn
	}
	return this
}

func (this *Expression) ReplaceAll(r *Expression, cl *CASLogger) Ex {
	cl.Debugf("In Expression.ReplaceAll. First trying IsMatchQ(this, r.Parts[1], es).")
	cl.Debugf("Rule r is: %s", r)

	matchq, matches := IsMatchQ(this, r.Parts[1], EmptyPD(), cl)
	toreturn := ReplacePD(r.Parts[2].DeepCopy(), cl, matches)
	if matchq {
		cl.Debugf("After MatchQ, rule is: %s", r)
		cl.Debugf("MatchQ succeeded. Returning r.Parts[2]: %s", r.Parts[2])
		return toreturn
	}

	thisSym, thisSymOk := this.Parts[0].(*Symbol)
	lhsExpr, lhsExprOk := r.Parts[1].(*Expression)
	if lhsExprOk {
		otherSym, otherSymOk := lhsExpr.Parts[0].(*Symbol)
		if thisSymOk && otherSymOk {
			if thisSym.Name == otherSym.Name {
				if IsOrderless(thisSym) {
					cl.Debugf("r.Parts[1] is Orderless. Now running CommutativeReplace")
					replaced := this.Parts[1:len(this.Parts)]
					CommutativeReplace(&replaced, lhsExpr.Parts[1:len(lhsExpr.Parts)], r.Parts[2], cl)
					this.Parts = this.Parts[0:1]
					this.Parts = append(this.Parts, replaced...)
				}
			}
		}
	}

	for i := range this.Parts {
		this.Parts[i] = ReplaceAll(this.Parts[i], r, cl, EmptyPD())
	}
	return this
}

func (this *Expression) String() string {
	headAsSym, isHeadSym := this.Parts[0].(*Symbol)
	fullForm := false
	if isHeadSym && !fullForm {
		res, ok := "", false
		headStr := headAsSym.Name
		toStringFn, hasToStringFn := toStringFns[headStr]
		if hasToStringFn {
			ok, res = toStringFn(this)
		} else if headStr == "Times" {
			return this.ToStringTimes()
		} else if headStr == "Plus" {
			return this.ToStringPlus()
		} else if headStr == "Power" {
			return this.ToStringPower()
		} else if headStr == "Equal" {
			return this.ToStringEqual()
		} else if headStr == "SameQ" {
			return this.ToStringSameQ()
		} else if headStr == "ReplaceAll" {
			return this.ToStringReplaceAll()
		} else if headStr == "ReplaceRepeated" {
			return this.ToStringReplaceRepeated()
		} else if headStr == "Pattern" {
			return this.ToStringPattern()
		} else if headStr == "Rule" {
			return this.ToStringRule()
		} else if headStr == "RuleDelayed" {
			return this.ToStringRuleDelayed()
		} else if headStr == "Set" {
			return this.ToStringSet()
		} else if headStr == "SetDelayed" {
			return this.ToStringSetDelayed()
		} else if headStr == "List" {
			return this.ToStringList()
		}
		if ok {
			return res
		}
	}

	// Default printing format
	var buffer bytes.Buffer
	buffer.WriteString(this.Parts[0].String())
	buffer.WriteString("[")
	for i, e := range this.Parts {
		if i == 0 {
			continue
		}
		buffer.WriteString(e.String())
		if i != len(this.Parts)-1 {
			buffer.WriteString(", ")
		}
	}
	buffer.WriteString("]")
	return buffer.String()
}

func IsAttribute(sm *Symbol, attr string, es *EvalState) bool {
	if sm.Name == "MemberQ" {
		return attr == "Protected"
	} else if sm.Name == "Attributes" {
		return attr == "Protected" || attr == "HoldAll" || attr == "Listable"
	} else if sm.Name == "List" {
		return attr == "Protected" || attr == "Locked"
	} else if sm.Name == "Pattern" {
		return attr == "Protected" || attr == "HoldFirst"
	} else if sm.Name == "Blank" {
		return attr == "Protected"
	} else if sm.Name == "Rule" {
		return attr == "Protected" || attr == "SequenceHold"
	} else if sm.Name == "Times" || sm.Name == "Plus" {
		return attr == "Flat" || attr == "Listable" || attr == "NumericFunction" || attr == "OneIdentity" || attr == "Orderless" || attr == "Protected"
	} else if sm.Name == "Power" {
		return attr == "Listable" || attr == "NumericFunction" || attr == "OneIdentity" || attr == "Protected"
	} else if sm.Name == "ReplaceRepeated" {
		return attr == "Protected"
	} else if sm.Name == "Equal" {
		return attr == "Protected"
	} // This is probably slow because it requires tons of Defined copies
	fmt.Printf("IsAttribute(%v, %v)\n", sm.Name, attr)
	res := (&Expression{[]Ex{
		&Symbol{"MemberQ"},
		&Expression{[]Ex{
			&Symbol{"Attributes"},
			sm,
		}},
		&Symbol{attr},
	}}).Eval(es)
	resSym, resIsSym := res.(*Symbol)
	if resIsSym {
		return resSym.Name == "True"
	}
	return false
}

// TODO: convert to a map
func IsOrderless(sym *Symbol) bool {
	if sym.Name == "Times" {
		return true
	} else if sym.Name == "Plus" {
		return true
	}
	return false
}

// TODO: convert to a map
func IsFlat(sym *Symbol) bool {
	if sym.Name == "Times" {
		return true
	} else if sym.Name == "Plus" {
		return true
	}
	return false
}

// TODO: convert to a map
func IsHoldFirst(sym *Symbol) bool {
	if sym.Name == "Set" {
		return true
	} else if sym.Name == "Pattern" {
		return true
	}
	return false
}

// TODO: convert to a map
func IsHoldRest(sym *Symbol) bool {
	if sym.Name == "If" {
		return true
	} else if sym.Name == "RuleDelayed" {
		return true
	}
	return false
}

// TODO: convert to a map
func IsHoldAll(sym *Symbol) bool {
	if sym.Name == "SetDelayed" {
		return true
	}
	if sym.Name == "Table" {
		return true
	}
	if sym.Name == "Clear" {
		return true
	}
	if sym.Name == "Timing" {
		return true
	}
	if sym.Name == "Hold" {
		return true
	}
	if sym.Name == "CompoundExpression" {
		return true
	}
	if sym.Name == "Condition" {
		return true
	}
	if sym.Name == "Function" {
		return true
	}
	return false
}

func (this *Expression) IsEqual(otherEx Ex, cl *CASLogger) string {
	//thisEvaled := this.Eval(es)
	//otherEx = otherEx.Eval(es)
	//this, ok := thisEvaled.(*Expression)
	//if !ok {
	//return thisEvaled.IsEqual(otherEx, cl)
	//}

	other, ok := otherEx.(*Expression)
	if !ok {
		return "EQUAL_UNK"
	}

	thisSym, thisSymOk := this.Parts[0].(*Symbol)
	otherSym, otherSymOk := other.Parts[0].(*Symbol)
	if thisSymOk && otherSymOk {
		if thisSym.Name == otherSym.Name {
			if IsOrderless(thisSym) {
				return CommutativeIsEqual(this.Parts[1:len(this.Parts)], other.Parts[1:len(other.Parts)], cl)
			}
		}
	}

	return FunctionIsEqual(this.Parts, other.Parts, cl)
}

func (this *Expression) DeepCopy() Ex {
	var thiscopy = &Expression{}
	for i := range this.Parts {
		thiscopy.Parts = append(thiscopy.Parts, this.Parts[i].DeepCopy())
	}
	return thiscopy
}

// Implement the sort.Interface
func (this *Expression) Len() int {
	return len(this.Parts) - 1
}

func (this *Expression) Less(i, j int) bool {
	return ExOrder(this.Parts[i+1], this.Parts[j+1]) == 1
}

func (this *Expression) Swap(i, j int) {
	this.Parts[j+1], this.Parts[i+1] = this.Parts[i+1], this.Parts[j+1]
}

func GetExpressionDefinitions() (defs []Definition) {
	defs = append(defs, Definition{
		name:      "Apply",
		legacyEvalFn: func(this *Expression, es *EvalState) Ex {
			if len(this.Parts) != 3 {
				return this
			}

			sym, isSym := this.Parts[1].(*Symbol)
			list, isList := HeadAssertion(this.Parts[2].DeepCopy(), "List")
			if isSym && isList {
				toReturn := &Expression{[]Ex{sym}}
				toReturn.Parts = append(toReturn.Parts, list.Parts[1:]...)
				return toReturn.Eval(es)
			}
			return this.Parts[2]
		},
	})
	return
}
<|endoftext|>