<commit_before>package async

/*
  Shorthand to List.RunParallel without having to manually create a new
  list, add the routines, etc.
*/
func Parallel(routines []Routine, callbacks ...Done) {
  l := New()
  l.Multiple(routines...)

  l.RunParallel(callbacks...)
}

/*
  Run all of the Routine functions in parallel from the current list.

  All of the arguments returned in a Routine's Done function will be combined
  and returned in the callbacks that are provided.

  If there is an error, any further results will be discarded but it will not
  immediately exit. It will continue to run all of the other Routine functions
  that were passed into it. This is because by the time the error is sent, the
  goroutines have already been started. At this current time, there is no way
  to cancel a sleep timer in Go.

  For example:
    async.Parallel([]async.Routine{
      func(done async.Done, args ...interface{}) {
        time.Sleep(20 * time.Second)
        done(nil, "Won't trigger the callbacks because error has been sent")
      },
      func(done async.Done, args ...interface{}) {
        done(fmt.Errorf("Test error"))
      }
    }, func(err error, results ...interface{}) {
      if err != nil {
        fmt.Printf("Error: %s", err)
        return
      }

      fmt.Printf("Args: %s", args)
    })

  If you were to run this example, you would see the error happen immediately.
  However, you would also notice that the program doesn't immediately exit.
  That is because it is still waiting for responses that it silently discards,
  since an error has already occurred.
*/
func (l *List) RunParallel(callbacks ...Done) {
  var (
    results = make([]interface{}, 0)

    result = make(chan interface{})

    _error error

    final = func(err error, results ...interface{}) {
      for i := 0; i < len(callbacks); i++ {
        if err != nil {
          callbacks[i](err)
        } else {
          callbacks[i](err, results...)
        }
      }
    }
  )

  l.Wait.Add(l.Len())

  go func() {
    for {
      r := <-result

      if _error != nil {
        continue
      }

      switch r.(type) {
      case error:
        _error = r.(error)
        final(_error)

      case []interface{}:
        results = append(results, r.([]interface{})...)
      }
    }
  }()

  for l.Len() > 0 {
    e := l.Front()
    _, r := l.Remove(e)

    go r(func(err error, args ...interface{}) {
      defer l.Wait.Done()

      if err != nil {
        result <- err
        return
      }

      result <- args
    })
  }

  l.Wait.Wait()

  close(result)

  if _error == nil {
    final(nil, results...)
  }
}
<commit_msg>Defer the close(result) in Parallel<commit_after>package async

/*
  Shorthand to List.RunParallel without having to manually create a new
  list, add the routines, etc.
*/
func Parallel(routines []Routine, callbacks ...Done) {
  l := New()
  l.Multiple(routines...)

  l.RunParallel(callbacks...)
}

/*
  Run all of the Routine functions in parallel from the current list.

  All of the arguments returned in a Routine's Done function will be combined
  and returned in the callbacks that are provided.

  If there is an error, any further results will be discarded but it will not
  immediately exit. It will continue to run all of the other Routine functions
  that were passed into it. This is because by the time the error is sent, the
  goroutines have already been started. At this current time, there is no way
  to cancel a sleep timer in Go.

  For example:
    async.Parallel([]async.Routine{
      func(done async.Done, args ...interface{}) {
        time.Sleep(20 * time.Second)
        done(nil, "Won't trigger the callbacks because error has been sent")
      },
      func(done async.Done, args ...interface{}) {
        done(fmt.Errorf("Test error"))
      }
    }, func(err error, results ...interface{}) {
      if err != nil {
        fmt.Printf("Error: %s", err)
        return
      }

      fmt.Printf("Args: %s", args)
    })

  If you were to run this example, you would see the error happen immediately.
  However, you would also notice that the program doesn't immediately exit.
  That is because it is still waiting for responses that it silently discards,
  since an error has already occurred.
*/
func (l *List) RunParallel(callbacks ...Done) {
  var (
    results = make([]interface{}, 0)

    result = make(chan interface{})

    _error error

    final = func(err error, results ...interface{}) {
      for i := 0; i < len(callbacks); i++ {
        if err != nil {
          callbacks[i](err)
        } else {
          callbacks[i](err, results...)
        }
      }
    }
  )

  defer close(result)

  l.Wait.Add(l.Len())

  go func() {
    for {
      r := <-result

      if _error != nil {
        continue
      }

      switch r.(type) {
      case error:
        _error = r.(error)
        final(_error)

      case []interface{}:
        results = append(results, r.([]interface{})...)
      }
    }
  }()

  for l.Len() > 0 {
    e := l.Front()
    _, r := l.Remove(e)

    go r(func(err error, args ...interface{}) {
      defer l.Wait.Done()

      if err != nil {
        result <- err
        return
      }

      result <- args
    })
  }

  l.Wait.Wait()

  if _error == nil {
    final(nil, results...)
  }
}
<|endoftext|>


<commit_before>package application_test

import (
	"io/ioutil"
	"os"
	"path/filepath"
	"runtime"
	"strings"
	"time"

	"github.com/cloudfoundry-incubator/cf-test-helpers/cf"
	"github.com/cloudfoundry-incubator/cf-test-helpers/generator"
	catshelpers "github.com/cloudfoundry-incubator/cf-test-helpers/helpers"
	"github.com/cloudfoundry/cli-acceptance-tests/helpers"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"github.com/onsi/gomega/gexec"
)

var _ = Describe("Push", func() {
	var (
		assets        helpers.Assets
		setupTimeout  time.Duration
		targetTimeout time.Duration
		pushTimeout   time.Duration
		context       *catshelpers.ConfiguredContext
		env           *catshelpers.Environment
	)

	BeforeEach(func() {
		assets = helpers.NewAssets()
		setupTimeout = 20 * time.Second
		targetTimeout = 10 * time.Second
		pushTimeout = 1 * time.Minute

		config := catshelpers.LoadConfig()
		context = catshelpers.NewContext(config)
		env = catshelpers.NewEnvironment(context)

		env.Setup()
	})

	AfterEach(func() {
		env.Teardown()
	})

	Context("when pushing an app with >248 character paths, which are .cfignored", func() {
		var (
			longPath     string
			cfIgnorePath string
		)

		BeforeEach(func() {
			longDirName := strings.Repeat("i", 247)
			longPath = filepath.Join(assets.ServiceBroker, longDirName)

			if runtime.GOOS == "windows" {
				cwd, err := os.Getwd()
				Expect(err).NotTo(HaveOccurred())

				// `\\?\` is used to skip Windows' file name processor, which imposes
				// length limits. Search MSDN for 'Maximum Path Length Limitation' for
				// more.
				err := os.MkdirAll(`\\?\`+filepath.Join(cwd, longPath), os.ModeDir|os.ModePerm)
				Expect(err).NotTo(HaveOccurred())
			} else {
				err := os.MkdirAll(longPath, os.ModeDir|os.ModePerm)
				Expect(err).NotTo(HaveOccurred())
			}

			cfIgnorePath = filepath.Join(assets.ServiceBroker, ".cfignore")
			cfIgnoreContents := []byte(longDirName + "\n")
			err = ioutil.WriteFile(cfIgnorePath, cfIgnoreContents, 0644)
			Expect(err).NotTo(HaveOccurred())
		})

		AfterEach(func() {
			err := os.RemoveAll(cfIgnorePath)
			Expect(err).NotTo(HaveOccurred())
			err = os.RemoveAll(longPath)
			Expect(err).NotTo(HaveOccurred())
		})

		It("is successful", func() {
			cf.AsUser(context.RegularUserContext(), setupTimeout, func() {
				space := context.RegularUserContext().Space
				org := context.RegularUserContext().Org

				target := cf.Cf("target", "-o", org, "-s", space).Wait(targetTimeout)
				Expect(target.ExitCode()).To(Equal(0))

				appName := generator.RandomName()
				session := cf.Cf("push", appName, "-p", assets.ServiceBroker).Wait(pushTimeout)
				Expect(session).To(gexec.Exit(0))
			})
		})
	})
})
<commit_msg>Clean up longPath in push test properly<commit_after>package application_test

import (
	"io/ioutil"
	"os"
	"path/filepath"
	"runtime"
	"strings"
	"time"

	"github.com/cloudfoundry-incubator/cf-test-helpers/cf"
	"github.com/cloudfoundry-incubator/cf-test-helpers/generator"
	catshelpers "github.com/cloudfoundry-incubator/cf-test-helpers/helpers"
	"github.com/cloudfoundry/cli-acceptance-tests/helpers"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"github.com/onsi/gomega/gexec"
)

var _ = Describe("Push", func() {
	var (
		assets        helpers.Assets
		setupTimeout  time.Duration
		targetTimeout time.Duration
		pushTimeout   time.Duration
		context       *catshelpers.ConfiguredContext
		env           *catshelpers.Environment
	)

	BeforeEach(func() {
		assets = helpers.NewAssets()
		setupTimeout = 20 * time.Second
		targetTimeout = 10 * time.Second
		pushTimeout = 1 * time.Minute

		config := catshelpers.LoadConfig()
		context = catshelpers.NewContext(config)
		env = catshelpers.NewEnvironment(context)

		env.Setup()
	})

	AfterEach(func() {
		env.Teardown()
	})

	Context("when pushing an app with >248 character paths, which are .cfignored", func() {
		var (
			longPath     string
			cfIgnorePath string
		)

		BeforeEach(func() {
			longDirName := strings.Repeat("i", 247)
			longPath = filepath.Join(assets.ServiceBroker, longDirName)

			if runtime.GOOS == "windows" {
				cwd, err := os.Getwd()
				Expect(err).NotTo(HaveOccurred())

				// `\\?\` is used to skip Windows' file name processor, which imposes
				// length limits. Search MSDN for 'Maximum Path Length Limitation' for
				// more.
				err := os.MkdirAll(`\\?\`+filepath.Join(cwd, longPath), os.ModeDir|os.ModePerm)
				Expect(err).NotTo(HaveOccurred())
			} else {
				err := os.MkdirAll(longPath, os.ModeDir|os.ModePerm)
				Expect(err).NotTo(HaveOccurred())
			}

			cfIgnorePath = filepath.Join(assets.ServiceBroker, ".cfignore")
			cfIgnoreContents := []byte(longDirName + "\n")
			err = ioutil.WriteFile(cfIgnorePath, cfIgnoreContents, 0644)
			Expect(err).NotTo(HaveOccurred())
		})

		AfterEach(func() {
			err := os.RemoveAll(cfIgnorePath)
			Expect(err).NotTo(HaveOccurred())
			if runtime.GOOS == "windows" {
				// `\\?\` is used to skip Windows' file name processor, which imposes
				// length limits. Search MSDN for 'Maximum Path Length Limitation' for
				// more.
				err = os.RemoveAll(`\\?\` + filepath.Join(cwd, longPath))
				Expect(err).NotTo(HaveOccurred())
			} else {
				err = os.RemoveAll(longPath)
				Expect(err).NotTo(HaveOccurred())
			}
		})

		It("is successful", func() {
			cf.AsUser(context.RegularUserContext(), setupTimeout, func() {
				space := context.RegularUserContext().Space
				org := context.RegularUserContext().Org

				target := cf.Cf("target", "-o", org, "-s", space).Wait(targetTimeout)
				Expect(target.ExitCode()).To(Equal(0))

				appName := generator.RandomName()
				session := cf.Cf("push", appName, "-p", assets.ServiceBroker).Wait(pushTimeout)
				Expect(session).To(gexec.Exit(0))
			})
		})
	})
})
<|endoftext|>


<commit_before>// Copyright 2015 Google Inc. All Rights Reserved.
// Author: jacobsa@google.com (Aaron Jacobs)

package gcsutil

import (
	"github.com/jacobsa/gcloud/gcs"
	"github.com/jacobsa/gcloud/syncutil"
	"golang.org/x/net/context"
	"google.golang.org/cloud/storage"
)

type ObjectInfo struct {
	attrs    storage.ObjectAttrs
	contents string
}

// Create multiple objects with some parallelism, returning corresponding
// storage.Object records for the objects created.
func CreateObjects(
	ctx context.Context,
	bucket gcs.Bucket,
	input []*ObjectInfo) (objects []*storage.Object, err error) {
	bundle := syncutil.NewBundle(ctx)

	// Size the output slice.
	objects = make([]*storage.Object, len(input))

	// Feed ObjectInfo records into a channel.
	type record struct {
		index      int
		objectInfo *ObjectInfo
	}

	recordChan := make(chan record, len(input))
	for i, o := range input {
		recordChan <- record{i, o}
	}

	close(recordChan)

	// Create the objects in parallel, writing to the output slice as we go.
	const parallelism = 10
	for i := 0; i < 10; i++ {
		bundle.Add(func(ctx context.Context) error {
			for r := range recordChan {
				o, err := CreateObject(ctx, bucket, &r.objectInfo.attrs, r.objectInfo.contents)
				if err != nil {
					return err
				}

				objects[r.index] = o
			}

			return nil
		})
	}

	err = bundle.Join()
	return
}
<commit_msg>Bumped up parallelism.<commit_after>// Copyright 2015 Google Inc. All Rights Reserved.
// Author: jacobsa@google.com (Aaron Jacobs)

package gcsutil

import (
	"github.com/jacobsa/gcloud/gcs"
	"github.com/jacobsa/gcloud/syncutil"
	"golang.org/x/net/context"
	"google.golang.org/cloud/storage"
)

type ObjectInfo struct {
	attrs    storage.ObjectAttrs
	contents string
}

// Create multiple objects with some parallelism, returning corresponding
// storage.Object records for the objects created.
func CreateObjects(
	ctx context.Context,
	bucket gcs.Bucket,
	input []*ObjectInfo) (objects []*storage.Object, err error) {
	bundle := syncutil.NewBundle(ctx)

	// Size the output slice.
	objects = make([]*storage.Object, len(input))

	// Feed ObjectInfo records into a channel.
	type record struct {
		index      int
		objectInfo *ObjectInfo
	}

	recordChan := make(chan record, len(input))
	for i, o := range input {
		recordChan <- record{i, o}
	}

	close(recordChan)

	// Create the objects in parallel, writing to the output slice as we go.
	const parallelism = 64
	for i := 0; i < 10; i++ {
		bundle.Add(func(ctx context.Context) error {
			for r := range recordChan {
				o, err := CreateObject(ctx, bucket, &r.objectInfo.attrs, r.objectInfo.contents)
				if err != nil {
					return err
				}

				objects[r.index] = o
			}

			return nil
		})
	}

	err = bundle.Join()
	return
}
<|endoftext|>


<commit_before>package multipass

import (
	"bytes"
	"crypto/aes"
	"crypto/cipher"
	"crypto/md5"
	"crypto/sha1"
	"encoding/base64"
	"encoding/json"
	"errors"
	"io/ioutil"
	"path"

	"code.google.com/p/go.crypto/pbkdf2"
)

func clear(b []byte) {
	for i := 0; i < len(b); i++ {
		b[i] = 0
	}
}

type KeyChain interface {
	Open(password []byte) error
	IsOpen() bool
	Close()
	ForEachItem(callback ItemCallback) error
	Keys() map[string][]byte
}

type AgileKeyChain struct {
	dir  string
	keys map[string][]byte
}

func NewAgileKeyChain(dir string) KeyChain {
	// TODO check it is a 1Password dir
	return &AgileKeyChain{dir: dir}
}

func (chain *AgileKeyChain) Close() {
	chain.keys = nil
}

func (chain *AgileKeyChain) IsOpen() bool {
	return chain.keys != nil
}

func (chain *AgileKeyChain) Keys() map[string][]byte {
	return chain.keys
}

func (chain *AgileKeyChain) ForEachItem(callback ItemCallback) error {
	return ForEachItem(chain.dir, callback)
}

func (chain *AgileKeyChain) Open(password []byte) error {
	file, err := ioutil.ReadFile(path.Join(chain.dir, "encryptionKeys.js"))
	if err != nil {
		return err
	}

	var keys Keys
	err = json.Unmarshal(file, &keys)
	if err != nil {
		return err
	}

	//fmt.Printf("%+v\n", keys)

	keyMap := make(map[string][]byte)
	for _, passKey := range keys.List {
		err = DecryptKey(password, passKey, keyMap)
		if err != nil {
			return err
		}
	}

	chain.keys = keyMap

	return nil
}

type ItemCallback func(path string, file []byte) error

func ForEachItem(onePasswordDir string, itemCallback ItemCallback) error {
	dirListing, e := ioutil.ReadDir(onePasswordDir)
	if e != nil {
		return e
	}
	for _, f := range dirListing {
		if !f.IsDir() && path.Ext(f.Name()) == ".1password" {
			p := path.Join(onePasswordDir, f.Name())
			file, err := ioutil.ReadFile(p)
			if err != nil {
				return err
			}
			err = itemCallback(p, file)
			if err != nil {
				return err
			}
		}
	}
	return nil
}

type PassKey struct {
	// Base64 encoded encryption key, AES-128 encrypted using a PBKDF2 key
	// derived from the user's password
	Data       string
	Identifier string
	Iterations int
	Level      string
	// Base64 encoded copy of the encryption key, AES-128 encrypted by itself
	Validation string
}

type Keys struct {
	List []PassKey
}

var saltMarker []byte = []byte("Salted__")
var saltLen int = len(saltMarker)

const saltDataLen = 8

func IsSalted(encryptedData []byte) bool {
	if len(encryptedData) < saltLen {
		return false
	}

	return bytes.Equal(encryptedData[:saltLen], saltMarker)
}

func DecryptKey(pass []byte, passKey PassKey, keyMap map[string][]byte) error {
	encryptedEncryptionKey, er := Base64Decode(passKey.Data)

	if er != nil {
		return er
	}

	var salt []byte
	if IsSalted(encryptedEncryptionKey) {
		// salt marker (8 bytes) | salt (8 bytes) | data
		skip := saltLen + saltDataLen
		salt = []byte(encryptedEncryptionKey[saltLen:skip])
		encryptedEncryptionKey = encryptedEncryptionKey[skip:]
	}

	// first 16 bytes are the key, last 16 bytes are the IV
	bytez := pbkdf2.Key(pass, salt, passKey.Iterations, aes.BlockSize*2, sha1.New)
	key := bytez[:16] // aes key size must be 16, 24 or 32
	iv := bytez[16:]  // IV byte length is equal to aes.BlockSize

	// 16 byte key, so AES-128
	b, e := DecryptAes(key, iv, encryptedEncryptionKey)
	if e != nil {
		return e
	}

	validationData := passKey.Validation
	validation, er := base64.StdEncoding.DecodeString(validationData[0 : len(validationData)-1])
	v, _ := DecryptData(b, validation)

	if !bytes.Equal(b, v) {
		return errors.New("encryption key validation failed")
	}

	keyMap[passKey.Level] = b

	return nil
}

type Item struct {
	Title       string
	Payload     string
	TypeName    string
	LocationKey string
	Location    string
}

func DecryptFile(file []byte, keyMap map[string][]byte) (Item, error) {
	type EncryptedItem struct {
		Title     string
		Encrypted string
		// base URL, e.g. hipchat.com
		LocationKey string
		// detailed URL e.g. https://hipchat.com/login
		Location string

		// determine which encryption key to use; values are SL3 or SL5 in my keychain
		// currently we assume SL5 if this field is empty
		// TODO add openContents.securityLevel field, it seems to be an altenative
		SecurityLevel string

		// Known values (there are probably others):
		//   passwords.Password
		//   webforms.WebForm
		//   system.folder.SavedSearch
		//   securenotes.SecureNote
		//   wallet.financial.CreditCard
		//   wallet.computer.License
		//   identities.Identity
		TypeName string
	}

	var item EncryptedItem
	err := json.Unmarshal(file, &item)
	if err != nil {
		return Item{}, err
	}

	//fmt.Printf("%+v\n", item)

	decoded, er := Base64Decode(item.Encrypted)

	if er != nil {
		return Item{}, er
	}

	securityLevel := item.SecurityLevel
	if len(securityLevel) == 0 {
		securityLevel = "SL5"
	}

	decrypted, e := DecryptData(keyMap[securityLevel], decoded)

	if e != nil {
		return Item{}, e
	}

	retVal := Item{
		Title:       item.Title,
		Payload:     string(decrypted),
		TypeName:    item.TypeName,
		Location:    item.Location,
		LocationKey: item.LocationKey,
	}
	return retVal, nil
}

func Base64Decode(data string) ([]byte, error) {
	actualData := data
	if data[len(data)-1] == 0 {
		actualData = data[:len(data)-1]
	}

	decoded, err := base64.StdEncoding.DecodeString(actualData)

	if err != nil {
		return nil, err
	}

	return decoded, nil
}

func DecryptData(key, data []byte) ([]byte, error) {
	if !IsSalted(data) {
		return nil, errors.New("unsalted data not supported")
	}

	// assuming salt
	salt := data[saltLen : saltLen+saltDataLen]
	data = data[saltLen+saltDataLen:]

	// poor man's MD5 based PBKDF1
	nkey := md5.Sum(append(key, salt...))
	iv := md5.Sum(append(append(nkey[:], key...), salt...))

	// 16 byte key, so AES-128
	result, err := DecryptAes(nkey[:], iv[:], data)

	if err != nil {
		return nil, err
	}

	return result, nil
}

// https://leanpub.com/gocrypto/read#leanpub-auto-encrypting-and-decrypting-data-with-aes-cbc
// https://github.com/kisom/gocrypto/blob/master/chapter2/aescbc/aescbc.go
// Decrypt decrypts the message and removes any padding.
func DecryptAes(k, iv, in []byte) ([]byte, error) {
	if len(in) == 0 || len(in)%aes.BlockSize != 0 {
		return nil, errors.New("input block size not a multiple of aes.BlockSize")
	}

	c, err := aes.NewCipher(k)
	if err != nil {
		// potentially wrong key size, must be one of 16/24/32
		// to select AES-128, AES-192 or AES-256 respectively
		return nil, err
	}

	// iv must be == aes.BlockSize
	cbc := cipher.NewCBCDecrypter(c, iv)
	cbc.CryptBlocks(in, in)

	out := Unpad(in)
	if out == nil {
		return nil, errors.New("failed to remove padding")
	}

	return out, nil
}

// Pad applies the PKCS #7 padding scheme on the buffer.
func Pad(in []byte) []byte {
	padding := 16 - (len(in) % 16)
	if padding == 0 {
		padding = 16
	}
	for i := 0; i < padding; i++ {
		in = append(in, byte(padding))
	}
	return in
}

// Unpad strips the PKCS #7 padding on a buffer. If the padding is
// invalid, nil is returned.
func Unpad(in []byte) []byte {
	if len(in) == 0 {
		return nil
	}

	padding := in[len(in)-1]
	if int(padding) > len(in) || padding > aes.BlockSize {
		return nil
	} else if padding == 0 {
		return nil
	}

	for i := len(in) - 1; i > len(in)-int(padding)-1; i-- {
		if in[i] != padding {
			return nil
		}
	}
	return in[:len(in)-int(padding)]
}
<commit_msg>remove unused function<commit_after>package multipass

import (
  "bytes"
  "crypto/aes"
  "crypto/cipher"
  "crypto/md5"
  "crypto/sha1"
  "encoding/base64"
  "encoding/json"
  "errors"
  "io/ioutil"
  "path"

  "code.google.com/p/go.crypto/pbkdf2"
)

type KeyChain interface {
  Open(password []byte) error
  IsOpen() bool
  Close()
  ForEachItem(callback ItemCallback) error
  Keys() map[string][]byte
}

type AgileKeyChain struct {
  dir  string
  keys map[string][]byte
}

func NewAgileKeyChain(dir string) KeyChain {
  // TODO check it is a 1Password dir
  return &AgileKeyChain{dir: dir}
}

func (chain *AgileKeyChain) Close() {
  chain.keys = nil
}

func (chain *AgileKeyChain) IsOpen() bool {
  return chain.keys != nil
}

func (chain *AgileKeyChain) Keys() map[string][]byte {
  return chain.keys
}

func (chain *AgileKeyChain) ForEachItem(callback ItemCallback) error {
  return ForEachItem(chain.dir, callback)
}

func (chain *AgileKeyChain) Open(password []byte) error {
  file, err := ioutil.ReadFile(path.Join(chain.dir, "encryptionKeys.js"))
  if err != nil {
    return err
  }

  var keys Keys
  err = json.Unmarshal(file, &keys)
  if err != nil {
    return err
  }

  //fmt.Printf("%+v\n", keys)

  keyMap := make(map[string][]byte)
  for _, passKey := range keys.List {
    err = DecryptKey(password, passKey, keyMap)
    if err != nil {
      return err
    }
  }

  chain.keys = keyMap

  return nil
}

type ItemCallback func(path string, file []byte) error

func ForEachItem(onePasswordDir string, itemCallback ItemCallback) error {
  dirListing, e := ioutil.ReadDir(onePasswordDir)
  if e != nil {
    return e
  }
  for _, f := range dirListing {
    if !f.IsDir() && path.Ext(f.Name()) == ".1password" {
      p := path.Join(onePasswordDir, f.Name())
      file, err := ioutil.ReadFile(p)
      if err != nil {
        return err
      }
      err = itemCallback(p, file)
      if err != nil {
        return err
      }
    }
  }
  return nil
}

type PassKey struct {
  // Base64 encoded encryption key, AES-128 encrypted using a PBKDF2 key
  // derived from the user's password
  Data       string
  Identifier string
  Iterations int
  Level      string
  // Base64 encoded copy of the encryption key, AES-128 encrypted by itself
  Validation string
}

type Keys struct {
  List []PassKey
}

var saltMarker []byte = []byte("Salted__")
var saltLen int = len(saltMarker)

const saltDataLen = 8

func IsSalted(encryptedData []byte) bool {
  if len(encryptedData) < saltLen {
    return false
  }

  return bytes.Equal(encryptedData[:saltLen], saltMarker)
}

func DecryptKey(pass []byte, passKey PassKey, keyMap map[string][]byte) error {
  encryptedEncryptionKey, er := Base64Decode(passKey.Data)

  if er != nil {
    return er
  }

  var salt []byte
  if IsSalted(encryptedEncryptionKey) {
    // salt marker (8 bytes) | salt (8 bytes) | data
    skip := saltLen + saltDataLen
    salt = []byte(encryptedEncryptionKey[saltLen:skip])
    encryptedEncryptionKey = encryptedEncryptionKey[skip:]
  }

  // first 16 bytes are the key, last 16 bytes are the IV
  bytez := pbkdf2.Key(pass, salt, passKey.Iterations, aes.BlockSize*2, sha1.New)
  key := bytez[:16] // aes key size must be 16, 24 or 32
  iv := bytez[16:]  // IV byte length is equal to aes.BlockSize

  // 16 byte key, so AES-128
  b, e := DecryptAes(key, iv, encryptedEncryptionKey)
  if e != nil {
    return e
  }

  validationData := passKey.Validation
  validation, er := base64.StdEncoding.DecodeString(validationData[0 : len(validationData)-1])
  v, _ := DecryptData(b, validation)

  if !bytes.Equal(b, v) {
    return errors.New("encryption key validation failed")
  }

  keyMap[passKey.Level] = b

  return nil
}

type Item struct {
  Title       string
  Payload     string
  TypeName    string
  LocationKey string
  Location    string
}

func DecryptFile(file []byte, keyMap map[string][]byte) (Item, error) {
  type EncryptedItem struct {
    Title     string
    Encrypted string
    // base URL, e.g. hipchat.com
    LocationKey string
    // detailed URL e.g. https://hipchat.com/login
    Location string

    // determine which encryption key to use; values are SL3 or SL5 in my keychain
    // currently we assume SL5 if this field is empty
    // TODO add openContents.securityLevel field, it seems to be an altenative
    SecurityLevel string

    // Known values (there are probably others):
    //   passwords.Password
    //   webforms.WebForm
    //   system.folder.SavedSearch
    //   securenotes.SecureNote
    //   wallet.financial.CreditCard
    //   wallet.computer.License
    //   identities.Identity
    TypeName string
  }

  var item EncryptedItem
  err := json.Unmarshal(file, &item)
  if err != nil {
    return Item{}, err
  }

  //fmt.Printf("%+v\n", item)

  decoded, er := Base64Decode(item.Encrypted)

  if er != nil {
    return Item{}, er
  }

  securityLevel := item.SecurityLevel
  if len(securityLevel) == 0 {
    securityLevel = "SL5"
  }

  decrypted, e := DecryptData(keyMap[securityLevel], decoded)

  if e != nil {
    return Item{}, e
  }

  retVal := Item{
    Title:       item.Title,
    Payload:     string(decrypted),
    TypeName:    item.TypeName,
    Location:    item.Location,
    LocationKey: item.LocationKey,
  }
  return retVal, nil
}

func Base64Decode(data string) ([]byte, error) {
  actualData := data
  if data[len(data)-1] == 0 {
    actualData = data[:len(data)-1]
  }

  decoded, err := base64.StdEncoding.DecodeString(actualData)

  if err != nil {
    return nil, err
  }

  return decoded, nil
}

func DecryptData(key, data []byte) ([]byte, error) {
  if !IsSalted(data) {
    return nil, errors.New("unsalted data not supported")
  }

  // assuming salt
  salt := data[saltLen : saltLen+saltDataLen]
  data = data[saltLen+saltDataLen:]

  // poor man's MD5 based PBKDF1
  nkey := md5.Sum(append(key, salt...))
  iv := md5.Sum(append(append(nkey[:], key...), salt...))

  // 16 byte key, so AES-128
  result, err := DecryptAes(nkey[:], iv[:], data)

  if err != nil {
    return nil, err
  }

  return result, nil
}

// https://leanpub.com/gocrypto/read#leanpub-auto-encrypting-and-decrypting-data-with-aes-cbc
// https://github.com/kisom/gocrypto/blob/master/chapter2/aescbc/aescbc.go
// Decrypt decrypts the message and removes any padding.
func DecryptAes(k, iv, in []byte) ([]byte, error) {
  if len(in) == 0 || len(in)%aes.BlockSize != 0 {
    return nil, errors.New("input block size not a multiple of aes.BlockSize")
  }

  c, err := aes.NewCipher(k)
  if err != nil {
    // potentially wrong key size, must be one of 16/24/32
    // to select AES-128, AES-192 or AES-256 respectively
    return nil, err
  }

  // iv must be == aes.BlockSize
  cbc := cipher.NewCBCDecrypter(c, iv)
  cbc.CryptBlocks(in, in)

  out := Unpad(in)
  if out == nil {
    return nil, errors.New("failed to remove padding")
  }

  return out, nil
}

// Pad applies the PKCS #7 padding scheme on the buffer.
func Pad(in []byte) []byte {
  padding := 16 - (len(in) % 16)
  if padding == 0 {
    padding = 16
  }
  for i := 0; i < padding; i++ {
    in = append(in, byte(padding))
  }
  return in
}

// Unpad strips the PKCS #7 padding on a buffer. If the padding is
// invalid, nil is returned.
func Unpad(in []byte) []byte {
  if len(in) == 0 {
    return nil
  }

  padding := in[len(in)-1]
  if int(padding) > len(in) || padding > aes.BlockSize {
    return nil
  } else if padding == 0 {
    return nil
  }

  for i := len(in) - 1; i > len(in)-int(padding)-1; i-- {
    if in[i] != padding {
      return nil
    }
  }
  return in[:len(in)-int(padding)]
}
<|endoftext|>


<commit_before>package multipass

import (
	"crypto/rand"
	"crypto/rsa"
	"crypto/x509"
	"encoding/json"
	"encoding/pem"
	"errors"
	"fmt"
	"html/template"
	"log"
	"net/http"
	"net/url"
	"path"
	"strings"
	"time"

	"github.com/mholt/caddy/caddyhttp/httpserver"

	jose "gopkg.in/square/go-jose.v1"
)

var ErrInvalidToken = errors.New("invalid token")

type Auth struct {
	*Multipass
	Next httpserver.Handler
}

type Rule struct {
	Basepath  string
	Expires   time.Duration
	Resources []string
	Handles   []string

	SMTPAddr, SMTPUser, SMTPPass string
	MailFrom, MailTmpl           string
}

type Multipass struct {
	Resources []string
	Basepath  string
	SiteAddr  string
	Expires   time.Duration

	Handler HandleService
	signer  jose.Signer
	key     *rsa.PrivateKey
	tmpl    *template.Template
}

func NewMultipassFromRule(r Rule) (*Multipass, error) {
	m, err := NewMultipass()
	if err != nil {
		return nil, err
	}
	if len(r.Resources) > 0 {
		m.Resources = r.Resources
	}
	if len(r.Basepath) > 0 {
		m.Basepath = path.Join("/", r.Basepath)
	}
	if r.Expires > 0 {
		m.Expires = r.Expires
	}

	// Set EmailHandler options
	opt := &EmailOptions{
		r.SMTPAddr,
		r.SMTPUser,
		r.SMTPPass,
		r.MailFrom,
	}
	handler, err := NewEmailHandler(opt)
	if err != nil {
		return nil, err
	}

	for _, handle := range r.Handles {
		handler.Register(handle)
	}
	m.Handler = handler

	tmpl, err := loadTemplates()
	if err != nil {
		return nil, err
	}
	m.tmpl = tmpl

	return m, nil
}

func NewMultipass() (*Multipass, error) {
	pk, err := rsa.GenerateKey(rand.Reader, 2048)
	if err != nil {
		return nil, err
	}
	signer, err := jose.NewSigner(jose.PS512, pk)
	if err != nil {
		return nil, err
	}
	return &Multipass{
		Resources: []string{"/"},
		Basepath:  "/",
		Expires:   time.Hour * 24,
		key:       pk,
		signer:    signer,
	}, nil
}

// Claims are part of the JSON web token
type Claims struct {
	Handle    string   `json:"handle"`
	Resources []string `json:"resources"`
	Expires   int64    `json:"exp"`
}

func (m *Multipass) AccessToken(handle string) (tokenStr string, err error) {
	exp := time.Now().Add(m.Expires)
	claims := &Claims{
		Handle:    handle,
		Resources: m.Resources,
		Expires:   exp.Unix(),
	}
	payload, err := json.Marshal(claims)
	if err != nil {
		return "", err
	}
	jws, err := m.signer.Sign(payload)
	if err != nil {
		return "", err
	}

	return jws.CompactSerialize()
}

func NewLoginURL(siteaddr, basepath, token string, v url.Values) (*url.URL, error) {
	u, err := url.Parse(siteaddr)
	if err != nil {
		return u, err
	}
	u.Path = path.Join(basepath, "login")
	v.Set("token", token)
	u.RawQuery = v.Encode()
	return u, nil
}

func loginHandler(w http.ResponseWriter, r *http.Request, m *Multipass) (int, error) {
	if r.Method == "POST" {
		r.ParseForm()
		handle := r.PostForm.Get("handle")
		if len(handle) > 0 {
			if m.Handler.Listed(handle) {
				token, err := m.AccessToken(handle)
				if err != nil {
					log.Print(err)
				}
				values := url.Values{}
				if s := r.PostForm.Get("url"); len(s) > 0 {
					values.Set("url", s)
				}
				loginURL, err := NewLoginURL(m.SiteAddr, m.Basepath, token, values)
				if err != nil {
					log.Print(err)
				}
				if err := m.Handler.Notify(handle, loginURL.String()); err != nil {
					log.Print(err)
				}
			}
			// Redirect even when the handle is not listed to prevent guessing
			location := path.Join(m.Basepath, "login/confirm")
			http.Redirect(w, r, location, http.StatusSeeOther)
			return http.StatusSeeOther, nil
		}
		var location string
		if s := r.PostForm.Get("url"); len(s) > 0 {
			location = s
		} else {
			location = path.Join(m.Basepath, "login")
		}
		http.Redirect(w, r, location, http.StatusSeeOther)
		return http.StatusSeeOther, nil
	}
	if r.Method == "GET" {
		if tokenStr := r.URL.Query().Get("token"); len(tokenStr) > 0 {
			cookie := &http.Cookie{
				Name:  "jwt_token",
				Value: tokenStr,
				Path:  "/",
			}
			http.SetCookie(w, cookie)
			nexturl := r.URL.Query().Get("url")
			if len(nexturl) == 0 || httpserver.Path(nexturl).Matches(m.Basepath) {
				nexturl = m.SiteAddr
			}
			p := &page{
				Page:        continueOrSignoutPage,
				SignoutPath: path.Join(m.Basepath, "signout"),
				NextURL:     nexturl,
			}
			m.tmpl.ExecuteTemplate(w, "page", p)
			return http.StatusOK, nil
		}
		p := &page{
			Page:        loginPage,
			LoginPath:   path.Join(m.Basepath, "login"),
			NextURL:     r.URL.String(),
			SignoutPath: path.Join(m.Basepath, "signout"),
		}
		m.tmpl.ExecuteTemplate(w, "page", p)
		return http.StatusOK, nil
	}
	return http.StatusMethodNotAllowed, nil
}

func confirmHandler(w http.ResponseWriter, r *http.Request, m *Multipass) (int, error) {
	w.Header().Add("Content-Type", "text/html; charset=utf-8")
	p := &page{
		Page: tokenSentPage,
	}
	m.tmpl.ExecuteTemplate(w, "page", p)
	return http.StatusOK, nil
}

func signoutHandler(w http.ResponseWriter, r *http.Request, m *Multipass) (int, error) {
	if r.Method == "POST" {
		if cookie, err := r.Cookie("jwt_token"); err == nil {
			cookie.Expires = time.Now().AddDate(-1, 0, 0)
			cookie.MaxAge = -1
			cookie.Path = "/"
			http.SetCookie(w, cookie)
		}
		loc := path.Join(m.Basepath, "login")
		http.Redirect(w, r, loc, http.StatusSeeOther)
		return http.StatusSeeOther, nil
	}
	return http.StatusMethodNotAllowed, nil
}

func publickeyHandler(w http.ResponseWriter, r *http.Request, m *Multipass) (int, error) {
	data, err := x509.MarshalPKIXPublicKey(&m.key.PublicKey)
	if err != nil {
		return http.StatusInternalServerError, err
	}
	block := &pem.Block{
		Type:  "PUBLIC KEY",
		Bytes: data,
	}
	w.Header().Set("Content-Type", "application/pkix-cert")
	if err := pem.Encode(w, block); err != nil {
		return http.StatusInternalServerError, err
	}
	return http.StatusOK, nil
}

func tokenHandler(w http.ResponseWriter, r *http.Request, m *Multipass) (int, error) {
	// Extract token from HTTP header, query parameter or cookie
	tokenStr, err := extractToken(r)
	if err != nil {
		return http.StatusUnauthorized, ErrInvalidToken
	}
	var claims *Claims
	if claims, err = validateToken(tokenStr, m.key.PublicKey); err != nil {
		return http.StatusUnauthorized, ErrInvalidToken
	}
	// Authorize handle claim
	if ok := m.Handler.Listed(claims.Handle); !ok {
		return http.StatusUnauthorized, ErrInvalidToken
	}
	// Verify path claim
	var match bool
	for _, path := range claims.Resources {
		if httpserver.Path(r.URL.Path).Matches(path) {
			match = true
			continue
		}
	}
	if !match {
		return http.StatusUnauthorized, ErrInvalidToken
	}

	// Pass on authorized handle to downstream handlers
	r.Header.Set("Multipass-Handle", claims.Handle)
	return http.StatusOK, nil
}

func (a *Auth) ServeHTTP(w http.ResponseWriter, r *http.Request) (int, error) {
	m := a.Multipass

	if httpserver.Path(r.URL.Path).Matches(m.Basepath) {
		switch r.URL.Path {
		case path.Join(m.Basepath, "pub.cer"):
			return publickeyHandler(w, r, m)
		case path.Join(m.Basepath, "login"):
			return loginHandler(w, r, m)
		case path.Join(m.Basepath, "login/confirm"):
			return confirmHandler(w, r, m)
		case path.Join(m.Basepath, "signout"):
			return signoutHandler(w, r, m)
		default:
			return http.StatusNotFound, nil
		}
	}

	for _, path := range m.Resources {
		if httpserver.Path(r.URL.Path).Matches(path) {
			if _, err := tokenHandler(w, r, m); err != nil {
				return loginHandler(w, r, m)
			}
		}
	}

	return a.Next.ServeHTTP(w, r)
}

// extractToken returns the JWT token embedded in the given request.
// JWT tokens can be embedded in the header prefixed with "Bearer ", with a
// "token" key query parameter or a cookie named "jwt_token".
func extractToken(r *http.Request) (string, error) {
	//from header
	if h := r.Header.Get("Authorization"); strings.HasPrefix(h, "Bearer ") {
		if len(h) > 7 {
			return h[7:], nil
		}
	}

	//from query parameter
	if token := r.URL.Query().Get("token"); len(token) > 0 {
		return token, nil
	}

	//from cookie
	if cookie, err := r.Cookie("jwt_token"); err == nil {
		return cookie.Value, nil
	}

	return "", fmt.Errorf("no token found")
}

func validateToken(token string, key rsa.PublicKey) (*Claims, error) {
	claims := &Claims{}

	// Verify token signature
	payload, err := verifyToken(token, key)
	if err != nil {
		return nil, err
	}
	// Unmarshal token claims
	if err := json.Unmarshal(payload, claims); err != nil {
		return nil, err
	}
	// Verify expire claim
	if time.Unix(claims.Expires, 0).Before(time.Now()) {
		return nil, errors.New("Token expired")
	}
	return claims, nil
}

func verifyToken(token string, key rsa.PublicKey) ([]byte, error) {
	var data []byte

	obj, err := jose.ParseSigned(token)
	if err != nil {
		return data, err
	}
	data, err = obj.Verify(&key)
	if err != nil {
		return data, err
	}
	return data, nil
}
<commit_msg>Refactor handlers, integrate into Multipass instance<commit_after>package multipass

import (
	"crypto/rand"
	"crypto/rsa"
	"crypto/x509"
	"encoding/json"
	"encoding/pem"
	"errors"
	"fmt"
	"html/template"
	"log"
	"net/http"
	"net/url"
	"path"
	"strings"
	"time"

	"github.com/mholt/caddy/caddyhttp/httpserver"

	jose "gopkg.in/square/go-jose.v1"
)

var ErrInvalidToken = errors.New("invalid token")

type Auth struct {
	*Multipass
	Next httpserver.Handler
}

type Rule struct {
	Basepath  string
	Expires   time.Duration
	Resources []string
	Handles   []string

	SMTPAddr, SMTPUser, SMTPPass string
	MailFrom, MailTmpl           string
}

type Multipass struct {
	Resources []string
	Basepath  string
	SiteAddr  string
	Expires   time.Duration

	Handler HandleService
	signer  jose.Signer
	key     *rsa.PrivateKey
	tmpl    *template.Template
	mux     *http.ServeMux
}

func NewMultipassFromRule(r Rule) (*Multipass, error) {
	m, err := NewMultipass()
	if err != nil {
		return nil, err
	}
	if len(r.Resources) > 0 {
		m.Resources = r.Resources
	}
	if len(r.Basepath) > 0 {
		m.Basepath = path.Join("/", r.Basepath)
	}
	if r.Expires > 0 {
		m.Expires = r.Expires
	}

	// Set EmailHandler options
	opt := &EmailOptions{
		r.SMTPAddr,
		r.SMTPUser,
		r.SMTPPass,
		r.MailFrom,
	}
	handler, err := NewEmailHandler(opt)
	if err != nil {
		return nil, err
	}

	for _, handle := range r.Handles {
		handler.Register(handle)
	}
	m.Handler = handler

	mux := http.NewServeMux()
	mux.HandleFunc(path.Join(m.Basepath, "/"), m.rootHandler)
	mux.HandleFunc(path.Join(m.Basepath, "/login"), m.loginHandler)
	mux.HandleFunc(path.Join(m.Basepath, "/confirm"), m.confirmHandler)
	mux.HandleFunc(path.Join(m.Basepath, "/signout"), m.signoutHandler)
	mux.HandleFunc(path.Join(m.Basepath, "/pub.cer"), m.publickeyHandler)
	m.mux = mux

	tmpl, err := loadTemplates()
	if err != nil {
		return nil, err
	}
	m.tmpl = tmpl

	return m, nil
}

func NewMultipass() (*Multipass, error) {
	pk, err := rsa.GenerateKey(rand.Reader, 2048)
	if err != nil {
		return nil, err
	}
	signer, err := jose.NewSigner(jose.PS512, pk)
	if err != nil {
		return nil, err
	}
	return &Multipass{
		Resources: []string{"/"},
		Basepath:  "/",
		Expires:   time.Hour * 24,
		key:       pk,
		signer:    signer,
	}, nil
}

// Claims are part of the JSON web token
type Claims struct {
	Handle    string   `json:"handle"`
	Resources []string `json:"resources"`
	Expires   int64    `json:"exp"`
}

// ServeHTTP satisfies the ServeHTTP interface
func (m *Multipass) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	if h, p := m.mux.Handler(r); len(p) > 0 {
		h.ServeHTTP(w, r)
		return
	}
	http.NotFound(w, r)
}

// rootHandler handles the "/" path of the Multipass handler.
// Shows login page when no JWT present
// Show continue or signout page when JWT is valid
// Show token invalid page when JWT is invalid
func (m *Multipass) rootHandler(w http.ResponseWriter, r *http.Request) {
	if r.Method == "GET" {

		// Regular login page
		p := &page{
			Page:        loginPage,
			LoginPath:   path.Join(m.Basepath, "login"),
			SignoutPath: path.Join(m.Basepath, "signout"),
		}

		// Show login page when there is no token
		tokenStr, err := extractToken(r)
		if err != nil {
			if s := r.Referer(); !httpserver.Path(s).Matches(m.Basepath) {
				p.NextURL = s
			}
			m.tmpl.ExecuteTemplate(w, "page", p)
			return
		}
		var claims *Claims
		if claims, err = validateToken(tokenStr, m.key.PublicKey); err != nil {
			p.Page = tokenInvalidPage
			m.tmpl.ExecuteTemplate(w, "page", p)
			return
		}
		// Authorize handle claim
		if ok := m.Handler.Listed(claims.Handle); !ok {
			p.Page = tokenInvalidPage
			m.tmpl.ExecuteTemplate(w, "page", p)
			return
		}
		if cookie, err := r.Cookie("next_url"); err == nil {
			p.NextURL = cookie.Value
		}
		p.Page = continueOrSignoutPage
		m.tmpl.ExecuteTemplate(w, "page", p)
		return
	}
	w.WriteHeader(http.StatusMethodNotAllowed)
}

func (m *Multipass) loginHandler(w http.ResponseWriter, r *http.Request) {
	if r.Method == "GET" {
		if tokenStr := r.URL.Query().Get("token"); len(tokenStr) > 0 {
			cookie := &http.Cookie{
				Name:  "jwt_token",
				Value: tokenStr,
				Path:  "/",
			}
			http.SetCookie(w, cookie)
		}
		if nexturl := r.URL.Query().Get("url"); len(nexturl) > 0 {
			cookie := &http.Cookie{
				Name:  "next_url",
				Value: nexturl,
				Path:  "/",
			}
			http.SetCookie(w, cookie)
		}
		http.Redirect(w, r, m.Basepath, http.StatusSeeOther)
		return
	}
	if r.Method == "POST" {
		r.ParseForm()
		handle := r.PostForm.Get("handle")
		if len(handle) > 0 {
			if m.Handler.Listed(handle) {
				token, err := m.AccessToken(handle)
				if err != nil {
					log.Print(err)
				}
				values := url.Values{}
				if s := r.PostForm.Get("url"); len(s) > 0 {
					values.Set("url", s)
				}
				loginURL, err := NewLoginURL(m.SiteAddr, m.Basepath, token, values)
				if err != nil {
					log.Print(err)
				}
				if err := m.Handler.Notify(handle, loginURL.String()); err != nil {
					log.Print(err)
				}
			}
			// Redirect even when the handle is not listed in order to prevent guessing
			location := path.Join(m.Basepath, "confirm")
			http.Redirect(w, r, location, http.StatusSeeOther)
			return
		}
		http.Redirect(w, r, m.Basepath, http.StatusSeeOther)
		return
	}
	w.WriteHeader(http.StatusMethodNotAllowed)
}

func (m *Multipass) confirmHandler(w http.ResponseWriter, r *http.Request) {
	if r.Method == "GET" {
		w.Header().Add("Content-Type", "text/html; charset=utf-8")
		p := &page{
			Page: tokenSentPage,
		}
		m.tmpl.ExecuteTemplate(w, "page", p)
		return
	}
	w.WriteHeader(http.StatusMethodNotAllowed)
}

// signoutHandler deletes the jwt_token cookie and redirect to the login
// location.
func (m *Multipass) signoutHandler(w http.ResponseWriter, r *http.Request) {
	if r.Method == "POST" {
		if cookie, err := r.Cookie("jwt_token"); err == nil {
			cookie.Expires = time.Now().AddDate(-1, 0, 0)
			cookie.MaxAge = -1
			cookie.Path = "/"
			http.SetCookie(w, cookie)
		}
		http.Redirect(w, r, m.Basepath, http.StatusSeeOther)
		return
	}
	w.WriteHeader(http.StatusMethodNotAllowed)
}

// publickeyHandler writes the public key to the given ResponseWriter to allow
// other to validate Multipass signed tokens.
func (m *Multipass) publickeyHandler(w http.ResponseWriter, r *http.Request) {
	data, err := x509.MarshalPKIXPublicKey(&m.key.PublicKey)
	if err != nil {
		w.WriteHeader(http.StatusInternalServerError)
		return
	}
	block := &pem.Block{
		Type:  "PUBLIC KEY",
		Bytes: data,
	}
	w.Header().Set("Content-Type", "application/pkix-cert")
	if err := pem.Encode(w, block); err != nil {
		w.WriteHeader(http.StatusInternalServerError)
		return
	}
}

func (m *Multipass) AccessToken(handle string) (tokenStr string, err error) {
	exp := time.Now().Add(m.Expires)
	claims := &Claims{
		Handle:    handle,
		Resources: m.Resources,
		Expires:   exp.Unix(),
	}
	payload, err := json.Marshal(claims)
	if err != nil {
		return "", err
	}
	jws, err := m.signer.Sign(payload)
	if err != nil {
		return "", err
	}

	return jws.CompactSerialize()
}

func NewLoginURL(siteaddr, basepath, token string, v url.Values) (*url.URL, error) {
	u, err := url.Parse(siteaddr)
	if err != nil {
		return u, err
	}
	u.Path = path.Join(basepath, "login")
	v.Set("token", token)
	u.RawQuery = v.Encode()
	return u, nil
}

func tokenHandler(w http.ResponseWriter, r *http.Request, m *Multipass) (int, error) {
	// Extract token from HTTP header, query parameter or cookie
	tokenStr, err := extractToken(r)
	if err != nil {
		return http.StatusUnauthorized, ErrInvalidToken
	}
	var claims *Claims
	if claims, err = validateToken(tokenStr, m.key.PublicKey); err != nil {
		return http.StatusUnauthorized, ErrInvalidToken
	}
	// Authorize handle claim
	if ok := m.Handler.Listed(claims.Handle); !ok {
		return http.StatusUnauthorized, ErrInvalidToken
	}
	// Verify path claim
	var match bool
	for _, path := range claims.Resources {
		if httpserver.Path(r.URL.Path).Matches(path) {
			match = true
			continue
		}
	}
	if !match {
		return http.StatusUnauthorized, ErrInvalidToken
	}

	// Pass on authorized handle to downstream handlers
	r.Header.Set("Multipass-Handle", claims.Handle)
	return http.StatusOK, nil
}

// ServeHTTP implements the httpserver.ServeHTTP interface from caddy.
func (a *Auth) ServeHTTP(w http.ResponseWriter, r *http.Request) (int, error) {
	m := a.Multipass

	if httpserver.Path(r.URL.Path).Matches(m.Basepath) {
		m.ServeHTTP(w, r)
		return 0, nil
	}

	for _, path := range m.Resources {
		if httpserver.Path(r.URL.Path).Matches(path) {
			if _, err := tokenHandler(w, r, m); err != nil {
				r.Header.Set("Referer", r.URL.String())
				m.rootHandler(w, r)
				return 0, nil
			}
		}
	}

	return a.Next.ServeHTTP(w, r)
}

// extractToken returns the JWT token embedded in the given request.
// JWT tokens can be embedded in the header prefixed with "Bearer ", with a
// "token" key query parameter or a cookie named "jwt_token".
func extractToken(r *http.Request) (string, error) {
	//from header
	if h := r.Header.Get("Authorization"); strings.HasPrefix(h, "Bearer ") {
		if len(h) > 7 {
			return h[7:], nil
		}
	}

	//from query parameter
	if token := r.URL.Query().Get("token"); len(token) > 0 {
		return token, nil
	}

	//from cookie
	if cookie, err := r.Cookie("jwt_token"); err == nil {
		return cookie.Value, nil
	}

	return "", fmt.Errorf("no token found")
}

func validateToken(token string, key rsa.PublicKey) (*Claims, error) {
	claims := &Claims{}

	// Verify token signature
	payload, err := verifyToken(token, key)
	if err != nil {
		return nil, err
	}
	// Unmarshal token claims
	if err := json.Unmarshal(payload, claims); err != nil {
		return nil, err
	}
	// Verify expire claim
	if time.Unix(claims.Expires, 0).Before(time.Now()) {
		return nil, errors.New("Token expired")
	}
	return claims, nil
}

func verifyToken(token string, key rsa.PublicKey) ([]byte, error) {
	var data []byte

	obj, err := jose.ParseSigned(token)
	if err != nil {
		return data, err
	}
	data, err = obj.Verify(&key)
	if err != nil {
		return data, err
	}
	return data, nil
}
<|endoftext|>


<commit_before>package add

import "github.com/catorpilor/leetcode/utils"

func addTwo(l1, l2 *utils.ListNode) *utils.ListNode {
	return useReverse(l1, l2)
}

// useReverse time complexity O(M+N), space complexity O(N)
func useReverse(l1, l2 *utils.ListNode) *utils.ListNode {
	if l1 == nil {
		return l2
	}
	if l2 == nil {
		return nil
	}

	// reverse l1 & l2
	l1, l2 = reverse(l1), reverse(l2)
	dummy := &utils.ListNode{}
	cur := dummy
	carrier := 0
	for l1 != nil || l2 != nil || carrier != 0 {
		sum := carrier
		if l1 != nil {
			sum += l1.Val
			l1 = l1.Next
		}
		if l2 != nil {
			sum += l2.Val
			l2 = l2.Next
		}
		carrier = 0
		if sum >= 10 {
			sum %= 10
			carrier = 1
		}
		node := &utils.ListNode{Val: sum}
		cur.Next = node
		cur = node
	}
	return reverse(dummy.Next)
}

func reverse(head *utils.ListNode) *utils.ListNode {
	dummy := &utils.ListNode{}
	cur := head
	var pre, next *utils.ListNode
	for cur != nil {
		next = cur.Next
		cur.Next = pre
		pre = cur
		cur = next
	}
	dummy.Next = pre
	return dummy.Next
}

// useStack time complexity O(M+N), space complexity O(M+N)
func useStack(l1, l2 *utils.ListNode) *utils.ListNode {
	if l1 == nil {
		return l2
	}
	if l2 == nil {
		return l1
	}
	a1, a2 := make([]int, 0, 65536), make([]int, 0, 65536)
	for l1 != nil {
		a1 = append(a1, l1.Val)
		l1 = l1.Next
	}
	for l2 != nil {
		a2 = append(a2, l2.Val)
		l2 = l2.Next
	}
	n1, n2 := len(a1), len(a2)
	sum := 0
	dummy := &utils.ListNode{}
	pre := dummy.Next
	for n1 != 0 || n2 != 0 || sum != 0 {
		if n1 > 0 {
			sum += a1[n1-1]
			a1 = a1[:n1-1]
		}
		if n2 > 0 {
			sum += a2[n2-1]
			a2 = a2[:n2-1]
		}
		node := &utils.ListNode{Val: sum % 10, Next: pre}
		pre = node
		n1, n2 = len(a1), len(a2)
		sum /= 10
	}
	// skip 0 at the highest bit
	// if pre.Val == 0 {
	// 	pre = pre.Next
	// }
	dummy.Next = pre
	return dummy.Next

}
<commit_msg>solve 445 use stack v2<commit_after>package add

import "github.com/catorpilor/leetcode/utils"

func addTwo(l1, l2 *utils.ListNode) *utils.ListNode {
	return useReverse(l1, l2)
}

// useReverse time complexity O(M+N), space complexity O(N)
func useReverse(l1, l2 *utils.ListNode) *utils.ListNode {
	if l1 == nil {
		return l2
	}
	if l2 == nil {
		return nil
	}

	// reverse l1 & l2
	l1, l2 = reverse(l1), reverse(l2)
	dummy := &utils.ListNode{}
	cur := dummy
	carrier := 0
	for l1 != nil || l2 != nil || carrier != 0 {
		sum := carrier
		if l1 != nil {
			sum += l1.Val
			l1 = l1.Next
		}
		if l2 != nil {
			sum += l2.Val
			l2 = l2.Next
		}
		carrier = 0
		if sum >= 10 {
			sum %= 10
			carrier = 1
		}
		node := &utils.ListNode{Val: sum}
		cur.Next = node
		cur = node
	}
	return reverse(dummy.Next)
}

func reverse(head *utils.ListNode) *utils.ListNode {
	dummy := &utils.ListNode{}
	cur := head
	var pre, next *utils.ListNode
	for cur != nil {
		next = cur.Next
		cur.Next = pre
		pre = cur
		cur = next
	}
	dummy.Next = pre
	return dummy.Next
}

// useStack time complexity O(M+N), space complexity O(M+N)
func useStack(l1, l2 *utils.ListNode) *utils.ListNode {
	if l1 == nil {
		return l2
	}
	if l2 == nil {
		return l1
	}
	a1, a2 := make([]int, 0, 65536), make([]int, 0, 65536)
	for l1 != nil {
		a1 = append(a1, l1.Val)
		l1 = l1.Next
	}
	for l2 != nil {
		a2 = append(a2, l2.Val)
		l2 = l2.Next
	}
	n1, n2 := len(a1), len(a2)
	sum := 0
	dummy := &utils.ListNode{}
	pre := dummy.Next
	for n1 != 0 || n2 != 0 || sum != 0 {
		if n1 > 0 {
			sum += a1[n1-1]
			a1 = a1[:n1-1]
		}
		if n2 > 0 {
			sum += a2[n2-1]
			a2 = a2[:n2-1]
		}
		node := &utils.ListNode{Val: sum % 10, Next: pre}
		pre = node
		n1, n2 = len(a1), len(a2)
		sum /= 10
	}
	// skip 0 at the highest bit
	// if pre.Val == 0 {
	// 	pre = pre.Next
	// }
	dummy.Next = pre
	return dummy.Next

}

// (7 -> 2 -> 4 -> 3) + (5 -> 6 -> 4)
// useStackV2 time complexity O(N), space complexity O(N)
func useStackV2(l1, l2 *utils.ListNode) *utils.ListNode {
	var s1, s2 []int
	for l1 != nil {
		s1 = append(s1, l1.Val)
		l1 = l1.Next
	}
	for l2 != nil {
		s2 = append(s2, l2.Val)
	}
	var sum int
	dummy := &utils.ListNode{}
	n1, n2 := len(s1), len(s2)
	for n1 > 0 || n2 > 0 {
		if n1 > 0 {
			sum += s1[n1-1]
			n1--
			s1 = s1[:n1]
		}
		if n2 > 0 {
			sum += s2[n2-1]
			n2--
			s2 = s2[:n2]
		}
		dummy.Val = sum % 10
		nh := &utils.ListNode{Val: sum / 10, Next: dummy}
		dummy = nh
		sum /= 10
	}
	if dummy.Val == 0 {
		return dummy.Next
	}
	return dummy
}
<|endoftext|>


<commit_before>package ethutil

import (
	"math"
	"testing"
)

func TestCompile(t *testing.T) {
	instr, err := CompileInstr("PUSH")

	if err != nil {
		t.Error("Failed compiling instruction")
	}

	calc := (48 + 0*256 + 0*int64(math.Pow(256, 2)))
	if Big(instr).Int64() != calc {
		t.Error("Expected", calc, ", got:", instr)
	}
}

func TestValidInstr(t *testing.T) {
	/*
	  op, args, err := Instr("68163")
	  if err != nil {
	    t.Error("Error decoding instruction")
	  }
	*/

}

func TestInvalidInstr(t *testing.T) {
}
<commit_msg>parse now returns byte[] instead of string<commit_after>package ethutil

import (
	"math"
	"testing"
)

func TestCompile(t *testing.T) {
	instr, err := CompileInstr("PUSH")

	if err != nil {
		t.Error("Failed compiling instruction")
	}

	calc := (48 + 0*256 + 0*int64(math.Pow(256, 2)))
	if BigD(instr).Int64() != calc {
		t.Error("Expected", calc, ", got:", instr)
	}
}

func TestValidInstr(t *testing.T) {
	/*
	  op, args, err := Instr("68163")
	  if err != nil {
	    t.Error("Error decoding instruction")
	  }
	*/

}

func TestInvalidInstr(t *testing.T) {
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"go/build"
	"io"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"strings"

	"code.google.com/p/go.tools/go/loader"
	"code.google.com/p/go.tools/godoc/vfs"

	"sourcegraph.com/sourcegraph/srclib-go/gog"
	"sourcegraph.com/sourcegraph/srclib-go/gog/definfo"
	defpkg "sourcegraph.com/sourcegraph/srclib-go/golang_def"
	"sourcegraph.com/sourcegraph/srclib/graph"
	"sourcegraph.com/sourcegraph/srclib/grapher"
	"sourcegraph.com/sourcegraph/srclib/repo"
	"sourcegraph.com/sourcegraph/srclib/unit"
)

func init() {
	_, err := parser.AddCommand("graph",
		"graph a Go package",
		"Graph a Go package, producing all defs, refs, and docs.",
		&graphCmd,
	)
	if err != nil {
		log.Fatal(err)
	}
}

type GraphCmd struct{}

var graphCmd GraphCmd

func (c *GraphCmd) Execute(args []string) error {
	var unit *unit.SourceUnit
	if err := json.NewDecoder(os.Stdin).Decode(&unit); err != nil {
		return err
	}
	if err := os.Stdin.Close(); err != nil {
		return err
	}

	if err := unmarshalTypedConfig(unit.Config); err != nil {
		return err
	}
	if err := config.apply(); err != nil {
		return err
	}

	if os.Getenv("IN_DOCKER_CONTAINER") != "" {
		buildPkg, err := UnitDataAsBuildPackage(unit)
		if err != nil {
			return err
		}

		// Make a new GOPATH.
		buildContext.GOPATH = "/tmp/gopath"

		// Set up GOPATH so it has this repo.
		log.Printf("Setting up a new GOPATH at %s", buildContext.GOPATH)
		dir := filepath.Join(buildContext.GOPATH, "src", string(unit.Repo))
		if err := os.MkdirAll(filepath.Dir(dir), 0700); err != nil {
			return err
		}
		log.Printf("Creating symlink to oldname %q at newname %q.", cwd, dir)
		if err := os.Symlink(cwd, dir); err != nil {
			return err
		}

		log.Printf("Changing directory to %q.", dir)
		if err := os.Chdir(dir); err != nil {
			return err
		}
		dockerCWD = cwd

		if config.GOROOT == "" {
			cwd = dir
		}

		// Get and install deps. (Only deps not in this repo; if we call `go
		// get` on this repo, we will either try to check out a different
		// version or fail with 'stale checkout?' because the .dockerignore
		// doesn't copy the .git dir.)
		var externalDeps []string
		for _, dep := range unit.Dependencies {
			importPath := dep.(string)
			if !strings.HasPrefix(importPath, string(unit.Repo)) && importPath != "C" {
				externalDeps = append(externalDeps, importPath)
			}
		}
		cmd := exec.Command("go", "get", "-d", "-t", "-v", "./"+buildPkg.Dir)
		cmd.Args = append(cmd.Args, externalDeps...)
		cmd.Env = config.env()
		cmd.Stdout, cmd.Stderr = os.Stderr, os.Stderr
		log.Printf("Downloading import dependencies: %v (env vars: %v).", cmd.Args, cmd.Env)
		if err := cmd.Run(); err != nil {
			return err
		}
		log.Printf("Finished downloading dependencies.")
	}

	out, err := Graph(unit)
	if err != nil {
		return err
	}

	// Make paths relative to repo.
	for _, gs := range out.Defs {
		if gs.File == "" {
			log.Printf("no file %+v", gs)
		}
		if gs.File != "" {
			gs.File = relPath(cwd, gs.File)
		}
	}
	for _, gr := range out.Refs {
		if gr.File != "" {
			gr.File = relPath(cwd, gr.File)
		}
	}
	for _, gd := range out.Docs {
		if gd.File != "" {
			gd.File = relPath(cwd, gd.File)
		}
	}

	if err := json.NewEncoder(os.Stdout).Encode(out); err != nil {
		return err
	}
	return nil
}

func relPath(base, path string) string {
	rp, err := filepath.Rel(base, path)
	if err != nil {
		log.Fatalf("Failed to make path %q relative to %q: %s", path, base, err)
	}

	// TODO(sqs): hack
	if strings.HasPrefix(rp, "../../../") && dockerCWD != "" {
		rp, err = filepath.Rel(dockerCWD, path)
		if err != nil {
			log.Fatalf("Failed to make path %q relative to %q: %s", path, cwd, err)
		}
	}

	return rp
}

func Graph(unit *unit.SourceUnit) (*grapher.Output, error) {
	pkg, err := UnitDataAsBuildPackage(unit)
	if err != nil {
		return nil, err
	}

	o, err := doGraph(pkg)
	if err != nil {
		return nil, err
	}

	o2 := grapher.Output{}

	uri := string(unit.Repo)

	for _, gs := range o.Defs {
		d, err := convertGoDef(gs, uri)
		if err != nil {
			return nil, err
		}
		if d != nil {
			o2.Defs = append(o2.Defs, d)
		}
	}
	for _, gr := range o.Refs {
		r, err := convertGoRef(gr, uri)
		if err != nil {
			return nil, err
		}
		if r != nil {
			o2.Refs = append(o2.Refs, r)
		}
	}
	for _, gd := range o.Docs {
		d, err := convertGoDoc(gd, uri)
		if err != nil {
			return nil, err
		}
		if d != nil {
			o2.Docs = append(o2.Docs, d)
		}
	}

	return &o2, nil
}

func convertGoDef(gs *gog.Def, repoURI string) (*graph.Def, error) {
	resolvedTarget, err := ResolveDep(gs.DefKey.PackageImportPath, repoURI)
	if err != nil {
		return nil, err
	}
	path := graph.DefPath(pathOrDot(strings.Join(gs.Path, "/")))
	treePath := treePath(string(path))
	if !treePath.IsValid() {
		return nil, fmt.Errorf("'%s' is not a valid tree-path", treePath)
	}

	def := &graph.Def{
		DefKey: graph.DefKey{
			Unit:     resolvedTarget.ToUnit,
			UnitType: resolvedTarget.ToUnitType,
			Path:     path,
		},
		TreePath: treePath,

		Name: gs.Name,
		Kind: definfo.GeneralKindMap[gs.Kind],

		File:     gs.File,
		DefStart: gs.DeclSpan[0],
		DefEnd:   gs.DeclSpan[1],

		Exported: gs.DefInfo.Exported,
		Test:     strings.HasSuffix(gs.File, "_test.go"),
	}

	d := defpkg.DefData{
		PackageImportPath: gs.DefKey.PackageImportPath,
		DefInfo:           gs.DefInfo,
	}
	def.Data, err = json.Marshal(d)
	if err != nil {
		return nil, err
	}

	if def.File == "" {
		// some cgo defs have empty File; omit them
		return nil, nil
	}

	return def, nil
}

func convertGoRef(gr *gog.Ref, repoURI string) (*graph.Ref, error) {
	resolvedTarget, err := ResolveDep(gr.Def.PackageImportPath, repoURI)
	if err != nil {
		return nil, err
	}
	if resolvedTarget == nil {
		return nil, nil
	}

	return &graph.Ref{
		DefRepo:     uriOrEmpty(resolvedTarget.ToRepoCloneURL),
		DefPath:     graph.DefPath(pathOrDot(strings.Join(gr.Def.Path, "/"))),
		DefUnit:     resolvedTarget.ToUnit,
		DefUnitType: resolvedTarget.ToUnitType,
		Def:         gr.IsDef,
		File:        gr.File,
		Start:       gr.Span[0],
		End:         gr.Span[1],
	}, nil
}

func convertGoDoc(gd *gog.Doc, repoURI string) (*graph.Doc, error) {
	resolvedTarget, err := ResolveDep(gd.PackageImportPath, repoURI)
	if err != nil {
		return nil, err
	}
	return &graph.Doc{
		DefKey: graph.DefKey{
			Path:     graph.DefPath(pathOrDot(strings.Join(gd.Path, "/"))),
			Unit:     resolvedTarget.ToUnit,
			UnitType: resolvedTarget.ToUnitType,
		},
		Format: gd.Format,
		Data:   gd.Data,
		File:   gd.File,
		Start:  gd.Span[0],
		End:    gd.Span[1],
	}, nil
}

func uriOrEmpty(cloneURL string) repo.URI {
	if cloneURL == "" {
		return ""
	}
	return repo.MakeURI(cloneURL)
}

func pathOrDot(path string) string {
	if path == "" {
		return "."
	}
	return path
}

func treePath(path string) graph.TreePath {
	if path == "" || path == "." {
		return graph.TreePath(".")
	}
	return graph.TreePath(fmt.Sprintf("./%s", path))
}

func doGraph(pkg *build.Package) (*gog.Output, error) {
	importPath := pkg.ImportPath

	// If we've overridden GOROOT and we're building a package not in
	// $GOROOT/src/pkg (such as "cmd/go"), then we need to virtualize GOROOT
	// because we can't set GOPATH=GOROOT (go/build ignores GOPATH in that
	// case).
	if config.GOROOT != "" && strings.HasPrefix(importPath, "cmd/") {
		// Unset our custom GOROOT (since we're routing FS ops to it using
		// vfs) and set it as our GOPATH.
		buildContext.GOROOT = build.Default.GOROOT
		buildContext.GOPATH = config.GOROOT

		virtualCWD = build.Default.GOROOT

		ns := vfs.NameSpace{}
		ns.Bind(filepath.Join(buildContext.GOROOT, "src/pkg"), vfs.OS(filepath.Join(config.GOROOT, "src/pkg")), "/", vfs.BindBefore)
		ns.Bind("/", vfs.OS("/"), "/", vfs.BindAfter)
		buildContext.IsDir = func(path string) bool {
			fi, err := ns.Stat(path)
			return err == nil && fi.Mode().IsDir()
		}
		buildContext.HasSubdir = func(root, dir string) (rel string, ok bool) { panic("unexpected") }
		buildContext.OpenFile = func(path string) (io.ReadCloser, error) {
			f, err := ns.Open(path)
			return f, err
		}
		buildContext.ReadDir = ns.ReadDir
	}

	if !loaderConfig.SourceImports {
		tmpfile, err := ioutil.TempFile("", filepath.Base(importPath))
		if err != nil {
			return nil, err
		}

		// Install pkg.
		cmd := exec.Command("go", "build", "-o", tmpfile.Name(), "-i", "-v", importPath)
		cmd.Env = config.env()
		cmd.Stdout, cmd.Stderr = os.Stderr, os.Stderr
		log.Printf("Install %q: %v (env vars: %v)", importPath, cmd.Args, cmd.Env)
		if err := cmd.Run(); err != nil {
			return nil, err
		}
		if err := tmpfile.Close(); err != nil {
			return nil, err
		}
		if err := os.Remove(tmpfile.Name()); err != nil {
			return nil, err
		}
	}

	importUnsafe := importPath == "unsafe"

	// Special-case: if this is a Cgo package, treat the CgoFiles as GoFiles or
	// else the character offsets will be junk.
	//
	// See https://codereview.appspot.com/86140043.
	loaderConfig.Build.CgoEnabled = false
	build.Default = *loaderConfig.Build
	if len(pkg.CgoFiles) > 0 {
		var allGoFiles []string
		allGoFiles = append(allGoFiles, pkg.GoFiles...)
		allGoFiles = append(allGoFiles, pkg.CgoFiles...)
		allGoFiles = append(allGoFiles, pkg.TestGoFiles...)
		for i, f := range allGoFiles {
			allGoFiles[i] = filepath.Join(cwd, pkg.Dir, f)
		}
		if err := loaderConfig.CreateFromFilenames(pkg.ImportPath, allGoFiles...); err != nil {
			return nil, err
		}
	} else {
		// Normal import
		if err := loaderConfig.ImportWithTests(importPath); err != nil {
			return nil, err
		}
	}

	if importUnsafe {
		// Special-case "unsafe" because go/loader does not let you load it
		// directly.
		if loaderConfig.ImportPkgs == nil {
			loaderConfig.ImportPkgs = make(map[string]bool)
		}
		loaderConfig.ImportPkgs["unsafe"] = true
	}

	prog, err := loaderConfig.Load()
	if err != nil {
		return nil, err
	}

	g := gog.New(prog)

	var pkgs []*loader.PackageInfo
	for _, pkg := range prog.Created {
		if strings.HasSuffix(pkg.Pkg.Name(), "_test") {
			// ignore xtest packages
			continue
		}
		pkgs = append(pkgs, pkg)
	}
	for _, pkg := range prog.Imported {
		pkgs = append(pkgs, pkg)
	}

	for _, pkg := range pkgs {
		if err := g.Graph(pkg); err != nil {
			return nil, err
		}
	}

	return &g.Output, nil
}
<commit_msg>Check for '-i' flag before running 'go build'<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"go/build"
	"io"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"strings"

	"code.google.com/p/go.tools/go/loader"
	"code.google.com/p/go.tools/godoc/vfs"

	"sourcegraph.com/sourcegraph/srclib-go/gog"
	"sourcegraph.com/sourcegraph/srclib-go/gog/definfo"
	defpkg "sourcegraph.com/sourcegraph/srclib-go/golang_def"
	"sourcegraph.com/sourcegraph/srclib/graph"
	"sourcegraph.com/sourcegraph/srclib/grapher"
	"sourcegraph.com/sourcegraph/srclib/repo"
	"sourcegraph.com/sourcegraph/srclib/unit"
)

func init() {
	_, err := parser.AddCommand("graph",
		"graph a Go package",
		"Graph a Go package, producing all defs, refs, and docs.",
		&graphCmd,
	)
	if err != nil {
		log.Fatal(err)
	}
}

type GraphCmd struct{}

var graphCmd GraphCmd

func (c *GraphCmd) Execute(args []string) error {
	var unit *unit.SourceUnit
	if err := json.NewDecoder(os.Stdin).Decode(&unit); err != nil {
		return err
	}
	if err := os.Stdin.Close(); err != nil {
		return err
	}

	if err := unmarshalTypedConfig(unit.Config); err != nil {
		return err
	}
	if err := config.apply(); err != nil {
		return err
	}

	if os.Getenv("IN_DOCKER_CONTAINER") != "" {
		buildPkg, err := UnitDataAsBuildPackage(unit)
		if err != nil {
			return err
		}

		// Make a new GOPATH.
		buildContext.GOPATH = "/tmp/gopath"

		// Set up GOPATH so it has this repo.
		log.Printf("Setting up a new GOPATH at %s", buildContext.GOPATH)
		dir := filepath.Join(buildContext.GOPATH, "src", string(unit.Repo))
		if err := os.MkdirAll(filepath.Dir(dir), 0700); err != nil {
			return err
		}
		log.Printf("Creating symlink to oldname %q at newname %q.", cwd, dir)
		if err := os.Symlink(cwd, dir); err != nil {
			return err
		}

		log.Printf("Changing directory to %q.", dir)
		if err := os.Chdir(dir); err != nil {
			return err
		}
		dockerCWD = cwd

		if config.GOROOT == "" {
			cwd = dir
		}

		// Get and install deps. (Only deps not in this repo; if we call `go
		// get` on this repo, we will either try to check out a different
		// version or fail with 'stale checkout?' because the .dockerignore
		// doesn't copy the .git dir.)
		var externalDeps []string
		for _, dep := range unit.Dependencies {
			importPath := dep.(string)
			if !strings.HasPrefix(importPath, string(unit.Repo)) && importPath != "C" {
				externalDeps = append(externalDeps, importPath)
			}
		}
		cmd := exec.Command("go", "get", "-d", "-t", "-v", "./"+buildPkg.Dir)
		cmd.Args = append(cmd.Args, externalDeps...)
		cmd.Env = config.env()
		cmd.Stdout, cmd.Stderr = os.Stderr, os.Stderr
		log.Printf("Downloading import dependencies: %v (env vars: %v).", cmd.Args, cmd.Env)
		if err := cmd.Run(); err != nil {
			return err
		}
		log.Printf("Finished downloading dependencies.")
	}

	out, err := Graph(unit)
	if err != nil {
		return err
	}

	// Make paths relative to repo.
	for _, gs := range out.Defs {
		if gs.File == "" {
			log.Printf("no file %+v", gs)
		}
		if gs.File != "" {
			gs.File = relPath(cwd, gs.File)
		}
	}
	for _, gr := range out.Refs {
		if gr.File != "" {
			gr.File = relPath(cwd, gr.File)
		}
	}
	for _, gd := range out.Docs {
		if gd.File != "" {
			gd.File = relPath(cwd, gd.File)
		}
	}

	if err := json.NewEncoder(os.Stdout).Encode(out); err != nil {
		return err
	}
	return nil
}

func relPath(base, path string) string {
	rp, err := filepath.Rel(base, path)
	if err != nil {
		log.Fatalf("Failed to make path %q relative to %q: %s", path, base, err)
	}

	// TODO(sqs): hack
	if strings.HasPrefix(rp, "../../../") && dockerCWD != "" {
		rp, err = filepath.Rel(dockerCWD, path)
		if err != nil {
			log.Fatalf("Failed to make path %q relative to %q: %s", path, cwd, err)
		}
	}

	return rp
}

func Graph(unit *unit.SourceUnit) (*grapher.Output, error) {
	pkg, err := UnitDataAsBuildPackage(unit)
	if err != nil {
		return nil, err
	}

	o, err := doGraph(pkg)
	if err != nil {
		return nil, err
	}

	o2 := grapher.Output{}

	uri := string(unit.Repo)

	for _, gs := range o.Defs {
		d, err := convertGoDef(gs, uri)
		if err != nil {
			return nil, err
		}
		if d != nil {
			o2.Defs = append(o2.Defs, d)
		}
	}
	for _, gr := range o.Refs {
		r, err := convertGoRef(gr, uri)
		if err != nil {
			return nil, err
		}
		if r != nil {
			o2.Refs = append(o2.Refs, r)
		}
	}
	for _, gd := range o.Docs {
		d, err := convertGoDoc(gd, uri)
		if err != nil {
			return nil, err
		}
		if d != nil {
			o2.Docs = append(o2.Docs, d)
		}
	}

	return &o2, nil
}

func convertGoDef(gs *gog.Def, repoURI string) (*graph.Def, error) {
	resolvedTarget, err := ResolveDep(gs.DefKey.PackageImportPath, repoURI)
	if err != nil {
		return nil, err
	}
	path := graph.DefPath(pathOrDot(strings.Join(gs.Path, "/")))
	treePath := treePath(string(path))
	if !treePath.IsValid() {
		return nil, fmt.Errorf("'%s' is not a valid tree-path", treePath)
	}

	def := &graph.Def{
		DefKey: graph.DefKey{
			Unit:     resolvedTarget.ToUnit,
			UnitType: resolvedTarget.ToUnitType,
			Path:     path,
		},
		TreePath: treePath,

		Name: gs.Name,
		Kind: definfo.GeneralKindMap[gs.Kind],

		File:     gs.File,
		DefStart: gs.DeclSpan[0],
		DefEnd:   gs.DeclSpan[1],

		Exported: gs.DefInfo.Exported,
		Test:     strings.HasSuffix(gs.File, "_test.go"),
	}

	d := defpkg.DefData{
		PackageImportPath: gs.DefKey.PackageImportPath,
		DefInfo:           gs.DefInfo,
	}
	def.Data, err = json.Marshal(d)
	if err != nil {
		return nil, err
	}

	if def.File == "" {
		// some cgo defs have empty File; omit them
		return nil, nil
	}

	return def, nil
}

func convertGoRef(gr *gog.Ref, repoURI string) (*graph.Ref, error) {
	resolvedTarget, err := ResolveDep(gr.Def.PackageImportPath, repoURI)
	if err != nil {
		return nil, err
	}
	if resolvedTarget == nil {
		return nil, nil
	}

	return &graph.Ref{
		DefRepo:     uriOrEmpty(resolvedTarget.ToRepoCloneURL),
		DefPath:     graph.DefPath(pathOrDot(strings.Join(gr.Def.Path, "/"))),
		DefUnit:     resolvedTarget.ToUnit,
		DefUnitType: resolvedTarget.ToUnitType,
		Def:         gr.IsDef,
		File:        gr.File,
		Start:       gr.Span[0],
		End:         gr.Span[1],
	}, nil
}

func convertGoDoc(gd *gog.Doc, repoURI string) (*graph.Doc, error) {
	resolvedTarget, err := ResolveDep(gd.PackageImportPath, repoURI)
	if err != nil {
		return nil, err
	}
	return &graph.Doc{
		DefKey: graph.DefKey{
			Path:     graph.DefPath(pathOrDot(strings.Join(gd.Path, "/"))),
			Unit:     resolvedTarget.ToUnit,
			UnitType: resolvedTarget.ToUnitType,
		},
		Format: gd.Format,
		Data:   gd.Data,
		File:   gd.File,
		Start:  gd.Span[0],
		End:    gd.Span[1],
	}, nil
}

func uriOrEmpty(cloneURL string) repo.URI {
	if cloneURL == "" {
		return ""
	}
	return repo.MakeURI(cloneURL)
}

func pathOrDot(path string) string {
	if path == "" {
		return "."
	}
	return path
}

func treePath(path string) graph.TreePath {
	if path == "" || path == "." {
		return graph.TreePath(".")
	}
	return graph.TreePath(fmt.Sprintf("./%s", path))
}

func doGraph(pkg *build.Package) (*gog.Output, error) {
	importPath := pkg.ImportPath

	// If we've overridden GOROOT and we're building a package not in
	// $GOROOT/src/pkg (such as "cmd/go"), then we need to virtualize GOROOT
	// because we can't set GOPATH=GOROOT (go/build ignores GOPATH in that
	// case).
	if config.GOROOT != "" && strings.HasPrefix(importPath, "cmd/") {
		// Unset our custom GOROOT (since we're routing FS ops to it using
		// vfs) and set it as our GOPATH.
		buildContext.GOROOT = build.Default.GOROOT
		buildContext.GOPATH = config.GOROOT

		virtualCWD = build.Default.GOROOT

		ns := vfs.NameSpace{}
		ns.Bind(filepath.Join(buildContext.GOROOT, "src/pkg"), vfs.OS(filepath.Join(config.GOROOT, "src/pkg")), "/", vfs.BindBefore)
		ns.Bind("/", vfs.OS("/"), "/", vfs.BindAfter)
		buildContext.IsDir = func(path string) bool {
			fi, err := ns.Stat(path)
			return err == nil && fi.Mode().IsDir()
		}
		buildContext.HasSubdir = func(root, dir string) (rel string, ok bool) { panic("unexpected") }
		buildContext.OpenFile = func(path string) (io.ReadCloser, error) {
			f, err := ns.Open(path)
			return f, err
		}
		buildContext.ReadDir = ns.ReadDir
	}

	if !loaderConfig.SourceImports {
		tmpfile, err := ioutil.TempFile("", filepath.Base(importPath))
		if err != nil {
			return nil, err
		}

		// Check that we have the '-i' flag.
		cmd := exec.Command("go", "help", "build")
		o, err := cmd.Output()
		if err != nil {
			log.Fatal(err)
		}
		usage := strings.Split(string(o), "\n")[0] // The usage is on the first line.
		matched, err := regexp.MatchString("-i", usage)
		if err != nil {
			log.Fatal(err)
		}
		if !matched {
			log.Fatal("'go build' does not have the '-i' flag. Please upgrade to go1.3+.")
		}

		// Install pkg.
		cmd := exec.Command("go", "build", "-o", tmpfile.Name(), "-i", "-v", importPath)
		cmd.Env = config.env()
		cmd.Stdout, cmd.Stderr = os.Stderr, os.Stderr
		log.Printf("Install %q: %v (env vars: %v)", importPath, cmd.Args, cmd.Env)
		if err := cmd.Run(); err != nil {
			return nil, err
		}
		if err := tmpfile.Close(); err != nil {
			return nil, err
		}
		if err := os.Remove(tmpfile.Name()); err != nil {
			return nil, err
		}
	}

	importUnsafe := importPath == "unsafe"

	// Special-case: if this is a Cgo package, treat the CgoFiles as GoFiles or
	// else the character offsets will be junk.
	//
	// See https://codereview.appspot.com/86140043.
	loaderConfig.Build.CgoEnabled = false
	build.Default = *loaderConfig.Build
	if len(pkg.CgoFiles) > 0 {
		var allGoFiles []string
		allGoFiles = append(allGoFiles, pkg.GoFiles...)
		allGoFiles = append(allGoFiles, pkg.CgoFiles...)
		allGoFiles = append(allGoFiles, pkg.TestGoFiles...)
		for i, f := range allGoFiles {
			allGoFiles[i] = filepath.Join(cwd, pkg.Dir, f)
		}
		if err := loaderConfig.CreateFromFilenames(pkg.ImportPath, allGoFiles...); err != nil {
			return nil, err
		}
	} else {
		// Normal import
		if err := loaderConfig.ImportWithTests(importPath); err != nil {
			return nil, err
		}
	}

	if importUnsafe {
		// Special-case "unsafe" because go/loader does not let you load it
		// directly.
		if loaderConfig.ImportPkgs == nil {
			loaderConfig.ImportPkgs = make(map[string]bool)
		}
		loaderConfig.ImportPkgs["unsafe"] = true
	}

	prog, err := loaderConfig.Load()
	if err != nil {
		return nil, err
	}

	g := gog.New(prog)

	var pkgs []*loader.PackageInfo
	for _, pkg := range prog.Created {
		if strings.HasSuffix(pkg.Pkg.Name(), "_test") {
			// ignore xtest packages
			continue
		}
		pkgs = append(pkgs, pkg)
	}
	for _, pkg := range prog.Imported {
		pkgs = append(pkgs, pkg)
	}

	for _, pkg := range pkgs {
		if err := g.Graph(pkg); err != nil {
			return nil, err
		}
	}

	return &g.Output, nil
}
<|endoftext|>


<commit_before>package types

// Defines the basic types such as ID types, and world units.

type TimeOfDay int64

type RandomSeed int64

// Which 'world'?
type DimensionID int8

const (
    DimensionNether = DimensionID(-1)
    DimensionNormal = DimensionID(0)
)

// Item-related types

// Item type ID
type ItemID int16

// Number of times that an item has been used
type ItemUses int16

// How many items are in a stack/slot etc.
type ItemCount byte

// Entity-related types

type EntityID int32

// The type of mob
type EntityMobType byte

type EntityStatus byte

type PlayerAnimation byte

// Block-related types

type BlockID byte

// Block face (0-5)
type Face byte

// Action-related types and constants

type DigStatus byte

const (
    DigStarted    = DigStatus(0)
    DigDigging    = DigStatus(1)
    DigStopped    = DigStatus(2)
    DigBlockBroke = DigStatus(3)
)

// Window/inventory-related types and constants

// ID specifying which slotted window, such as inventory
// TODO get numeric constants for these
type WindowID byte

// ID of the slow in inventory or other item-slotted window element
type SlotID int16

// Transaction ID
type TxID int16

// Movement-related types and constants

type VelocityComponent int16

type Velocity struct {
    X, Y, Z VelocityComponent
}

type RelMoveCoord byte

type RelMove struct {
    X, Y, Z RelMoveCoord
}

// Angle-related types and constants

const (
    DegreesToBytes = 256 / 360
)

// An angle, where there are 256 units in a circle.
type AngleBytes byte

// An angle in degrees
type AngleDegrees float32

type LookDegrees struct {
    Yaw, Pitch AngleDegrees
}

func (l *LookDegrees) ToLookBytes() *LookBytes {
    return &LookBytes{
        AngleBytes(l.Yaw * DegreesToBytes),
        AngleBytes(l.Pitch * DegreesToBytes),
    }
}

type LookBytes struct {
    Yaw, Pitch AngleBytes
}

type OrientationDegrees struct {
    Yaw, Pitch, Roll AngleDegrees
}

type OrientationBytes struct {
    Yaw, Pitch, Roll AngleBytes
}

// Location-related types and constants

const (
    // Chunk coordinates can be converted to block coordinates
    ChunkSizeX = 16
    ChunkSizeY = 128
    ChunkSizeZ = 16

    // The area within which a client receives updates
    ChunkRadius = 10

    // Sometimes it is useful to convert block coordinates to pixels
    PixelsPerBlock = 32
)

// Specifies exact world distance in blocks (floating point)
type AbsCoord float64

type AbsXYZ struct {
    X, Y, Z AbsCoord
}

func (p *AbsXYZ) ToAbsIntXYZ() *AbsIntXYZ {
    return &AbsIntXYZ{
        AbsIntCoord(p.X * PixelsPerBlock),
        AbsIntCoord(p.Y * PixelsPerBlock),
        AbsIntCoord(p.Z * PixelsPerBlock),
    }
}

func (p *AbsXYZ) ToBlockXYZ() *BlockXYZ {
    return &BlockXYZ{
        BlockCoord(p.X),
        BlockYCoord(p.Y),
        BlockCoord(p.Z),
    }
}

// Specifies approximate world distance in pixels (absolute / PixelsPerBlock)
type AbsIntCoord int32

type AbsIntXYZ struct {
    X, Y, Z AbsIntCoord
}

// Coordinate of a chunk in the world (block / 16)
type ChunkCoord int32

type ChunkXZ struct {
    X, Z ChunkCoord
}

// Returns the world BlockXYZ position of the (0, 0, 0) block in the chunk
func (chunkLoc *ChunkXZ) GetChunkCornerBlockXY() *BlockXYZ {
    return &BlockXYZ{
        BlockCoord(chunkLoc.X) * ChunkSizeX,
        0,
        BlockCoord(chunkLoc.Z) * ChunkSizeZ,
    }
}

// Convert a position within a chunk to a block position within the world
func (chunkLoc *ChunkXZ) ToBlockXY(subLoc *SubChunkXYZ) *BlockXYZ {
    return &BlockXYZ{
        BlockCoord(chunkLoc.X)*ChunkSizeX + BlockCoord(subLoc.X),
        BlockYCoord(subLoc.Y),
        BlockCoord(chunkLoc.Z)*ChunkSizeZ + BlockCoord(subLoc.Z),
    }
}

// Size of a sub-chunk
type SubChunkSizeCoord byte

// Coordinate of a block within a chunk
type SubChunkCoord byte

type SubChunkSize struct {
    X, Y, Z SubChunkSizeCoord
}

type SubChunkXYZ struct {
    X, Y, Z SubChunkCoord
}

// Coordinate of a block within the world
type BlockCoord int32
type BlockYCoord byte

type BlockXYZ struct {
    X   BlockCoord
    Y   BlockYCoord
    Z   BlockCoord
}

// Convert an (x, z) absolute coordinate pair to chunk coordinates
func (abs *AbsXYZ) ToChunkXZ() (chunkXz ChunkXZ) {
    return ChunkXZ{
        ChunkCoord(abs.X / ChunkSizeX),
        ChunkCoord(abs.Z / ChunkSizeZ),
    }
}

// Convert (x, z) absolute integer coordinates to chunk coordinates
func (abs *AbsIntXYZ) ToChunkXZ() ChunkXZ {
    // TODO check this conversion
    return ChunkXZ{
        ChunkCoord(abs.X / ChunkSizeX),
        ChunkCoord(abs.Z / ChunkSizeZ),
    }
}

func coordDivMod(num, denom int32) (div, mod int32) {
    div = num / denom
    mod = num % denom
    if mod < 0 {
        mod += denom
        div -= 1
    }
    return
}

// Convert an (x, z) block coordinate pair to chunk coordinates and the
// coordinates of the block within the chunk
func (blockLoc *BlockXYZ) ToChunkLocal() (chunkLoc ChunkXZ, subLoc SubChunkXYZ) {
    chunkX, subX := coordDivMod(int32(blockLoc.X), ChunkSizeX)
    chunkZ, subZ := coordDivMod(int32(blockLoc.Z), ChunkSizeZ)

    chunkLoc = ChunkXZ{ChunkCoord(chunkX), ChunkCoord(chunkZ)}
    subLoc = SubChunkXYZ{SubChunkCoord(subX), SubChunkCoord(blockLoc.Y), SubChunkCoord(subZ)}
    return
}

func (blockLoc *BlockXYZ) ToXYZInteger() AbsIntXYZ {
    // TODO check this conversion
    return AbsIntXYZ{
        AbsIntCoord(blockLoc.X * PixelsPerBlock),
        AbsIntCoord(blockLoc.Y * PixelsPerBlock),
        AbsIntCoord(blockLoc.Z * PixelsPerBlock),
    }
}

// Misc. types and constants

type ChunkLoadMode byte

const (
    // Client should unload the chunk
    ChunkUnload = ChunkLoadMode(0)

    // Client should initialise the chunk
    ChunkInit = ChunkLoadMode(1)
)
<commit_msg>Probably fixed a coordinate conversion, but should test.<commit_after>package types

// Defines the basic types such as ID types, and world units.

type TimeOfDay int64

type RandomSeed int64

// Which 'world'?
type DimensionID int8

const (
    DimensionNether = DimensionID(-1)
    DimensionNormal = DimensionID(0)
)

// Item-related types

// Item type ID
type ItemID int16

// Number of times that an item has been used
type ItemUses int16

// How many items are in a stack/slot etc.
type ItemCount byte

// Entity-related types

type EntityID int32

// The type of mob
type EntityMobType byte

type EntityStatus byte

type PlayerAnimation byte

// Block-related types

type BlockID byte

// Block face (0-5)
type Face byte

// Action-related types and constants

type DigStatus byte

const (
    DigStarted    = DigStatus(0)
    DigDigging    = DigStatus(1)
    DigStopped    = DigStatus(2)
    DigBlockBroke = DigStatus(3)
)

// Window/inventory-related types and constants

// ID specifying which slotted window, such as inventory
// TODO get numeric constants for these
type WindowID byte

// ID of the slow in inventory or other item-slotted window element
type SlotID int16

// Transaction ID
type TxID int16

// Movement-related types and constants

type VelocityComponent int16

type Velocity struct {
    X, Y, Z VelocityComponent
}

type RelMoveCoord byte

type RelMove struct {
    X, Y, Z RelMoveCoord
}

// Angle-related types and constants

const (
    DegreesToBytes = 256 / 360
)

// An angle, where there are 256 units in a circle.
type AngleBytes byte

// An angle in degrees
type AngleDegrees float32

type LookDegrees struct {
    Yaw, Pitch AngleDegrees
}

func (l *LookDegrees) ToLookBytes() *LookBytes {
    return &LookBytes{
        AngleBytes(l.Yaw * DegreesToBytes),
        AngleBytes(l.Pitch * DegreesToBytes),
    }
}

type LookBytes struct {
    Yaw, Pitch AngleBytes
}

type OrientationDegrees struct {
    Yaw, Pitch, Roll AngleDegrees
}

type OrientationBytes struct {
    Yaw, Pitch, Roll AngleBytes
}

// Location-related types and constants

const (
    // Chunk coordinates can be converted to block coordinates
    ChunkSizeX = 16
    ChunkSizeY = 128
    ChunkSizeZ = 16

    // The area within which a client receives updates
    ChunkRadius = 10

    // Sometimes it is useful to convert block coordinates to pixels
    PixelsPerBlock = 32
)

// Specifies exact world distance in blocks (floating point)
type AbsCoord float64

type AbsXYZ struct {
    X, Y, Z AbsCoord
}

func (p *AbsXYZ) ToAbsIntXYZ() *AbsIntXYZ {
    return &AbsIntXYZ{
        AbsIntCoord(p.X * PixelsPerBlock),
        AbsIntCoord(p.Y * PixelsPerBlock),
        AbsIntCoord(p.Z * PixelsPerBlock),
    }
}

func (p *AbsXYZ) ToBlockXYZ() *BlockXYZ {
    return &BlockXYZ{
        BlockCoord(p.X),
        BlockYCoord(p.Y),
        BlockCoord(p.Z),
    }
}

// Specifies approximate world distance in pixels (absolute / PixelsPerBlock)
type AbsIntCoord int32

type AbsIntXYZ struct {
    X, Y, Z AbsIntCoord
}

// Coordinate of a chunk in the world (block / 16)
type ChunkCoord int32

type ChunkXZ struct {
    X, Z ChunkCoord
}

// Returns the world BlockXYZ position of the (0, 0, 0) block in the chunk
func (chunkLoc *ChunkXZ) GetChunkCornerBlockXY() *BlockXYZ {
    return &BlockXYZ{
        BlockCoord(chunkLoc.X) * ChunkSizeX,
        0,
        BlockCoord(chunkLoc.Z) * ChunkSizeZ,
    }
}

// Convert a position within a chunk to a block position within the world
func (chunkLoc *ChunkXZ) ToBlockXY(subLoc *SubChunkXYZ) *BlockXYZ {
    return &BlockXYZ{
        BlockCoord(chunkLoc.X)*ChunkSizeX + BlockCoord(subLoc.X),
        BlockYCoord(subLoc.Y),
        BlockCoord(chunkLoc.Z)*ChunkSizeZ + BlockCoord(subLoc.Z),
    }
}

// Size of a sub-chunk
type SubChunkSizeCoord byte

// Coordinate of a block within a chunk
type SubChunkCoord byte

type SubChunkSize struct {
    X, Y, Z SubChunkSizeCoord
}

type SubChunkXYZ struct {
    X, Y, Z SubChunkCoord
}

// Coordinate of a block within the world
type BlockCoord int32
type BlockYCoord byte

type BlockXYZ struct {
    X   BlockCoord
    Y   BlockYCoord
    Z   BlockCoord
}

// Convert an (x, z) absolute coordinate pair to chunk coordinates
func (abs *AbsXYZ) ToChunkXZ() (chunkXz ChunkXZ) {
    return ChunkXZ{
        ChunkCoord(abs.X / ChunkSizeX),
        ChunkCoord(abs.Z / ChunkSizeZ),
    }
}

// Convert (x, z) absolute integer coordinates to chunk coordinates
func (abs *AbsIntXYZ) ToChunkXZ() ChunkXZ {
    // TODO unit tests on this
    return ChunkXZ{
        PixelsPerBlock * ChunkCoord(abs.X / ChunkSizeX),
        PixelsPerBlock * ChunkCoord(abs.Z / ChunkSizeZ),
    }
}

func coordDivMod(num, denom int32) (div, mod int32) {
    div = num / denom
    mod = num % denom
    if mod < 0 {
        mod += denom
        div -= 1
    }
    return
}

// Convert an (x, z) block coordinate pair to chunk coordinates and the
// coordinates of the block within the chunk
func (blockLoc *BlockXYZ) ToChunkLocal() (chunkLoc ChunkXZ, subLoc SubChunkXYZ) {
    chunkX, subX := coordDivMod(int32(blockLoc.X), ChunkSizeX)
    chunkZ, subZ := coordDivMod(int32(blockLoc.Z), ChunkSizeZ)

    chunkLoc = ChunkXZ{ChunkCoord(chunkX), ChunkCoord(chunkZ)}
    subLoc = SubChunkXYZ{SubChunkCoord(subX), SubChunkCoord(blockLoc.Y), SubChunkCoord(subZ)}
    return
}

func (blockLoc *BlockXYZ) ToXYZInteger() AbsIntXYZ {
    // TODO check this conversion
    return AbsIntXYZ{
        AbsIntCoord(blockLoc.X * PixelsPerBlock),
        AbsIntCoord(blockLoc.Y * PixelsPerBlock),
        AbsIntCoord(blockLoc.Z * PixelsPerBlock),
    }
}

// Misc. types and constants

type ChunkLoadMode byte

const (
    // Client should unload the chunk
    ChunkUnload = ChunkLoadMode(0)

    // Client should initialise the chunk
    ChunkInit = ChunkLoadMode(1)
)
<|endoftext|>


<commit_before>// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package pilosa

import (
	"io/ioutil"
	"runtime"
	"testing"
	"time"
)

// Ensure the file handle count is working
func TestCountOpenFiles(t *testing.T) {
	// Windows is not supported yet
	if runtime.GOOS == "windows" {
		t.Skip("Skipping unsupported countOpenFiles test on Windows.")
	}
	count, err := countOpenFiles()
	if err != nil {
		t.Errorf("countOpenFiles failed: %s", err)
	}
	if count == 0 {
		t.Error("countOpenFiles returned invalid value 0.")
	}
}

func TestMonitorAntiEntropyZero(t *testing.T) {

	td, err := ioutil.TempDir(*TempDir, "")
	if err != nil {
		t.Fatalf("getting temp dir: %v", err)
	}
	s, err := NewServer(OptServerDataDir(td),
		OptServerAntiEntropyInterval(0))
	if err != nil {
		t.Fatalf("making new server: %v", err)
	}

	ch := make(chan struct{})
	go func() {
		s.monitorAntiEntropy()
		close(ch)
	}()

	select {
	case <-ch:
	case <-time.After(time.Second):
		t.Fatalf("monitorAntiEntropy should have returned immediately with duration 0")
	}
}
<commit_msg>add test for only opening owned shards<commit_after>// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package pilosa

import (
	"io/ioutil"
	"os"
	"runtime"
	"testing"
	"time"

	"github.com/pilosa/pilosa/roaring"
)

// Ensure the file handle count is working
func TestCountOpenFiles(t *testing.T) {
	// Windows is not supported yet
	if runtime.GOOS == "windows" {
		t.Skip("Skipping unsupported countOpenFiles test on Windows.")
	}
	count, err := countOpenFiles()
	if err != nil {
		t.Errorf("countOpenFiles failed: %s", err)
	}
	if count == 0 {
		t.Error("countOpenFiles returned invalid value 0.")
	}
}

func TestMonitorAntiEntropyZero(t *testing.T) {

	td, err := ioutil.TempDir(*TempDir, "")
	if err != nil {
		t.Fatalf("getting temp dir: %v", err)
	}
	s, err := NewServer(OptServerDataDir(td),
		OptServerAntiEntropyInterval(0))
	if err != nil {
		t.Fatalf("making new server: %v", err)
	}

	ch := make(chan struct{})
	go func() {
		s.monitorAntiEntropy()
		close(ch)
	}()

	select {
	case <-ch:
	case <-time.After(time.Second):
		t.Fatalf("monitorAntiEntropy should have returned immediately with duration 0")
	}
}

func TestOnlyOpenOwnedFiles(t *testing.T) {
	path, err := ioutil.TempDir("", "pilosa")
	if err != nil {
		t.Fatalf("getting temp dir: %v", err)
	}
	defer func() {
		err := os.RemoveAll(path)
		if err != nil {
			t.Logf("cleaning up temp dir: %v", err)
		}
	}()
	bm := roaring.NewFileBitmap(1, 2, 3)
	err = os.MkdirAll(path+"/i/f/views/standard/fragments", os.ModeDir|os.ModePerm)
	if err != nil {
		t.Fatalf("mkdirall: %v", err)
	}
	one, err := os.Create(path + "/i/f/views/standard/fragments/1")
	if err != nil {
		t.Fatalf("creating one: %v", err)
	}
	two, err := os.Create(path + "/i/f/views/standard/fragments/2")
	if err != nil {
		t.Fatalf("creating two: %v", err)
	}
	_, err = bm.WriteTo(one)
	if err != nil {
		t.Fatalf("writing to one: %v", err)
	}
	_, err = bm.WriteTo(two)
	if err != nil {
		t.Fatalf("writing to two: %v", err)
	}

	h := NewHolder()
	h.Path = path
	h.shardValidatorFunc = func(index string, shard uint64) bool {
		return shard == 1
	}

	err = h.Open()
	if err != nil {
		t.Fatalf("opening holder: %v", err)
	}

	view := h.Index("i").Field("f").view("standard")

	if len(view.fragments) != 1 {
		t.Errorf("should have one fragment, but have: %d", len(view.fragments))
	}

	if _, ok := view.fragments[1]; !ok {
		t.Errorf("should have fragment 1, but fragments: %#v", view.fragments)
	}
}
<|endoftext|>


<commit_before>// Package httpd provides web service
package httpd

import (
	"fmt"
	"github.com/gorilla/mux"
	"github.com/rakyll/statik/fs"
	"net/http"

	"github.com/mehrdadrad/mylg/cli"
	"github.com/mehrdadrad/mylg/icmp"
	// statik is single binary including all web stuff
	_ "github.com/mehrdadrad/mylg/services/dashboard/statik"
)

// Route represents a HTTP route
type Route struct {
	Name        string
	Method      string
	Path        string
	HandlerFunc http.HandlerFunc
}

type APIHandler func(w http.ResponseWriter, r *http.Request, cfg cli.Config)

func APIWarpper(handler APIHandler, cfg cli.Config) func(w http.ResponseWriter, r *http.Request) {
	return func(w http.ResponseWriter, r *http.Request) {
		handler(w, r, cfg)
	}
}

// API handles API routes
func API(w http.ResponseWriter, r *http.Request, cfg cli.Config) {
	var f = mux.Vars(r)["name"]
	switch f {
	case "ping":
		r.ParseForm()
		host := r.FormValue("host")
		p, err := icmp.NewPing(host+" -c 1", cfg)
		if err != nil {
			return
		}
		resp := p.Run()
		r := <-resp
		w.Write([]byte(fmt.Sprintf(`{"rtt": %.2f,"pl":0}`, r.RTT)))
	}
}

// Run starts web service
func Run(cfg cli.Config) {
	statikFS, _ := fs.New()
	router := mux.NewRouter().StrictSlash(true)
	routes := []Route{
		Route{
			"API",
			"POST",
			"/api/{name}",
			APIWarpper(API, cfg),
		},
	}

	for _, route := range routes {
		router.
			Path(route.Path).
			Methods(route.Method).
			Name(route.Name).
			Handler(route.HandlerFunc)
	}
	router.PathPrefix("/").Handler(http.FileServer(statikFS))
	err := http.ListenAndServe(fmt.Sprintf("%s:%d", cfg.Web.Address, cfg.Web.Port), router)
	if err != nil {
		println(err.Error())
	}
}
<commit_msg>added icmp error to ping api<commit_after>// Package httpd provides web service
package httpd

import (
	"fmt"
	"github.com/gorilla/mux"
	"github.com/rakyll/statik/fs"
	"net/http"

	"github.com/mehrdadrad/mylg/cli"
	"github.com/mehrdadrad/mylg/icmp"
	// statik is single binary including all web stuff
	_ "github.com/mehrdadrad/mylg/services/dashboard/statik"
)

// Route represents a HTTP route
type Route struct {
	Name        string
	Method      string
	Path        string
	HandlerFunc http.HandlerFunc
}

type APIHandler func(w http.ResponseWriter, r *http.Request, cfg cli.Config)

func APIWarpper(handler APIHandler, cfg cli.Config) func(w http.ResponseWriter, r *http.Request) {
	return func(w http.ResponseWriter, r *http.Request) {
		handler(w, r, cfg)
	}
}

// API handles API routes
func API(w http.ResponseWriter, r *http.Request, cfg cli.Config) {
	var (
		f      = mux.Vars(r)["name"]
		errStr string
	)
	switch f {
	case "ping":
		r.ParseForm()
		host := r.FormValue("host")
		p, err := icmp.NewPing(host+" -c 1", cfg)
		if err != nil {
			return
		}
		resp := p.Run()
		r := <-resp
		if r.Error != nil {
			errStr = r.Error.Error()
		}
		w.Write([]byte(fmt.Sprintf(`{"rtt": %.2f,"pl":0, "err": "%s"}`, r.RTT, errStr)))
	}
}

// Run starts web service
func Run(cfg cli.Config) {
	statikFS, _ := fs.New()
	router := mux.NewRouter().StrictSlash(true)
	routes := []Route{
		Route{
			"API",
			"POST",
			"/api/{name}",
			APIWarpper(API, cfg),
		},
	}

	for _, route := range routes {
		router.
			Path(route.Path).
			Methods(route.Method).
			Name(route.Name).
			Handler(route.HandlerFunc)
	}
	router.PathPrefix("/").Handler(http.FileServer(statikFS))
	err := http.ListenAndServe(fmt.Sprintf("%s:%d", cfg.Web.Address, cfg.Web.Port), router)
	if err != nil {
		println(err.Error())
	}
}
<|endoftext|>