<commit_before>package gocassa

import (
	"errors"
	"fmt"
	g "github.com/hailocab/gocassa/generate"
	r "github.com/hailocab/gocassa/reflect"
	"reflect"
	"strings"
)

type t struct {
	keySpace *k
	info     *tableInfo
}

// Contains mostly analyzed information about the entity
type tableInfo struct {
	keyspace, name string
	marshalSource  interface{}
	fieldSource    map[string]interface{}
	keys           Keys
	fieldNames     map[string]struct{} // This is here only to check containment
	fields         []string
	fieldValues    []interface{}
}

func newTableInfo(keyspace, name string, keys Keys, entity interface{}, fieldSource map[string]interface{}) *tableInfo {
	cinf := &tableInfo{
		keyspace:      keyspace,
		name:          name,
		marshalSource: entity,
		keys:          keys,
		fieldSource:   fieldSource,
	}
	fields := []string{}
	values := []interface{}{}
	for k, v := range fieldSource {
		fields = append(fields, k)
		values = append(values, v)
	}
	cinf.fieldNames = map[string]struct{}{}
	for _, v := range fields {
		cinf.fieldNames[v] = struct{}{}
	}
	cinf.fields = fields
	cinf.fieldValues = values
	return cinf
}

func (t *t) zero() interface{} {
	return reflect.New(reflect.TypeOf(t.info.marshalSource)).Interface()
}

// Since we cant have Map -> [(k, v)] we settle for Map -> ([k], [v])
// #tuplelessLifeSucks
func keyValues(m map[string]interface{}) ([]string, []interface{}) {
	keys := []string{}
	values := []interface{}{}
	for k, v := range m {
		keys = append(keys, k)
		values = append(values, v)
	}
	return keys, values
}

func toMap(i interface{}) (map[string]interface{}, bool) {
	switch v := i.(type) {
	//case M:
	//	return map[string]interface{}(v), true
	case map[string]interface{}:
		return v, true
	}
	return r.StructToMap(i)
}

func (t t) Where(rs ...Relation) Filter {
	return filter{
		t:  t,
		rs: rs,
	}
}

func (t t) generateFieldNames() string {
	xs := []string{}
	for _, v := range t.info.fields {
		xs = append(xs, strings.ToLower(v))
	}
	return strings.Join(xs, ", ")
}

// INSERT INTO Hollywood.NerdMovies (user_uuid, fan)
//   VALUES ('cfd66ccc-d857-4e90-b1e5-df98a3d40cd6', 'johndoe')
//
// Gotcha: primkey must be first
func insert(cfName string, fieldNames []string) string {
	placeHolders := []string{}
	for i := 0; i < len(fieldNames); i++ {
		placeHolders = append(placeHolders, "?")
	}
	lowerFieldNames := []string{}
	for _, v := range fieldNames {
		lowerFieldNames = append(lowerFieldNames, strings.ToLower(v))
	}
	return fmt.Sprintf("INSERT INTO %v ("+strings.Join(lowerFieldNames, ", ")+") VALUES ("+strings.Join(placeHolders, ", ")+")", cfName)
}

func (t t) Set(i interface{}) error {
	m, ok := toMap(i)
	if !ok {
		return errors.New("Can't create: value not understood")
	}
	fields, values := keyValues(m)
	stmt := insert(t.info.name, fields)
	sess := t.keySpace.session
	if t.keySpace.debugMode {
		fmt.Println(stmt, values)
	}
	return sess.Query(stmt, values...).Exec()
}

func (t t) Create() error {
	stmt, err := t.CreateStatement()
	if err != nil {
		return err
	}
	return t.keySpace.session.Query(stmt).Exec()
}

// Drop table if exists and create it again
func (t t) Recreate() error {
	if ex, err := t.keySpace.Exists(t.info.name); ex && err == nil {
		err = t.keySpace.DropTable(t.info.name)
		if err != nil {
			return err
		}
	} else if err != nil {
		return err
	}
	return t.Create()
}

func (t t) CreateStatement() (string, error) {
	return g.CreateTable(t.keySpace.name, t.info.name, t.info.keys.PartitionKeys, t.info.keys.ClusteringColumns, t.info.fields, t.info.fieldValues)
}

//const (
//	asc	 = iota
//	desc
//)
//
//type Ordering struct {
//	fieldName string
//	order int
//}
//
//func ASC(fieldName string) Ordering {
//	return Ordering{
//		fieldName: fieldName,
//		order: asc,
//	}
//}
//
//func DESC(fieldName string) Ordering {
//	return Ordering{
//		fieldName: fieldName,
//		order: asc,
//	}
//}
<commit_msg>Gofmt<commit_after>package gocassa

import (
	"errors"
	"fmt"
	g "github.com/hailocab/gocassa/generate"
	r "github.com/hailocab/gocassa/reflect"
	"reflect"
	"strings"
)

type t struct {
	keySpace *k
	info     *tableInfo
}

// Contains mostly analyzed information about the entity
type tableInfo struct {
	keyspace, name string
	marshalSource  interface{}
	fieldSource    map[string]interface{}
	keys           Keys
	fieldNames     map[string]struct{} // This is here only to check containment
	fields         []string
	fieldValues    []interface{}
}

func newTableInfo(keyspace, name string, keys Keys, entity interface{}, fieldSource map[string]interface{}) *tableInfo {
	cinf := &tableInfo{
		keyspace:      keyspace,
		name:          name,
		marshalSource: entity,
		keys:          keys,
		fieldSource:   fieldSource,
	}
	fields := []string{}
	values := []interface{}{}
	for k, v := range fieldSource {
		fields = append(fields, k)
		values = append(values, v)
	}
	cinf.fieldNames = map[string]struct{}{}
	for _, v := range fields {
		cinf.fieldNames[v] = struct{}{}
	}
	cinf.fields = fields
	cinf.fieldValues = values
	return cinf
}

func (t *t) zero() interface{} {
	return reflect.New(reflect.TypeOf(t.info.marshalSource)).Interface()
}

// Since we cant have Map -> [(k, v)] we settle for Map -> ([k], [v])
// #tuplelessLifeSucks
func keyValues(m map[string]interface{}) ([]string, []interface{}) {
	keys := []string{}
	values := []interface{}{}
	for k, v := range m {
		keys = append(keys, k)
		values = append(values, v)
	}
	return keys, values
}

func toMap(i interface{}) (map[string]interface{}, bool) {
	switch v := i.(type) {
	//case M:
	//	return map[string]interface{}(v), true
	case map[string]interface{}:
		return v, true
	}
	return r.StructToMap(i)
}

func (t t) Where(rs ...Relation) Filter {
	return filter{
		t:  t,
		rs: rs,
	}
}

func (t t) generateFieldNames() string {
	xs := []string{}
	for _, v := range t.info.fields {
		xs = append(xs, strings.ToLower(v))
	}
	return strings.Join(xs, ", ")
}

// INSERT INTO Hollywood.NerdMovies (user_uuid, fan)
//   VALUES ('cfd66ccc-d857-4e90-b1e5-df98a3d40cd6', 'johndoe')
//
// Gotcha: primkey must be first
func insert(cfName string, fieldNames []string) string {
	placeHolders := []string{}
	for i := 0; i < len(fieldNames); i++ {
		placeHolders = append(placeHolders, "?")
	}
	lowerFieldNames := []string{}
	for _, v := range fieldNames {
		lowerFieldNames = append(lowerFieldNames, strings.ToLower(v))
	}
	return fmt.Sprintf("INSERT INTO %v ("+strings.Join(lowerFieldNames, ", ")+") VALUES ("+strings.Join(placeHolders, ", ")+")", cfName)
}

func (t t) Set(i interface{}) error {
	m, ok := toMap(i)
	if !ok {
		return errors.New("Can't create: value not understood")
	}
	fields, values := keyValues(m)
	stmt := insert(t.info.name, fields)
	sess := t.keySpace.session
	if t.keySpace.debugMode {
		fmt.Println(stmt, values)
	}
	return sess.Query(stmt, values...).Exec()
}

func (t t) Create() error {
	stmt, err := t.CreateStatement()
	if err != nil {
		return err
	}
	return t.keySpace.session.Query(stmt).Exec()
}

// Drop table if exists and create it again
func (t t) Recreate() error {
	if ex, err := t.keySpace.Exists(t.info.name); ex && err == nil {
		err = t.keySpace.DropTable(t.info.name)
		if err != nil {
			return err
		}
	} else if err != nil {
		return err
	}
	return t.Create()
}

func (t t) CreateStatement() (string, error) {
	return g.CreateTable(t.keySpace.name, t.info.name, t.info.keys.PartitionKeys, t.info.keys.ClusteringColumns, t.info.fields, t.info.fieldValues)
}

//const (
//	asc	 = iota
//	desc
//)
//
//type Ordering struct {
//	fieldName string
//	order int
//}
//
//func ASC(fieldName string) Ordering {
//	return Ordering{
//		fieldName: fieldName,
//		order: asc,
//	}
//}
//
//func DESC(fieldName string) Ordering {
//	return Ordering{
//		fieldName: fieldName,
//		order: asc,
//	}
//}
<|endoftext|>


<commit_before>// Copyright 2015 Robin Hahling. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package targo provides functions to create or extract tar archives.
package targo

import (
	"archive/tar"
	"errors"
	"io"
	"os"
	"path/filepath"
	"strings"
)

// Create creates a tar archive from the directory specified by dirPath.
// The resulting tar archive format is in POSIX.1 format.
// Calling this function with dirPath having a trailing slash results in the
// content of dirPath to be used at the root level of the archive rather than
// the directory pointed out by dirPath itself.
func Create(destPath, dirPath string) error {
	fi, err := os.Stat(dirPath)
	if err != nil {
		return err
	}

	if !fi.IsDir() {
		return errors.New("given path is not a directory: " + dirPath)
	}

	file, err := os.Create(destPath)
	if err != nil {
		return err
	}
	defer file.Close()

	tw := tar.NewWriter(file)
	defer tw.Close()

	err = filepath.Walk(dirPath, func(path string, info os.FileInfo, err error) error {
		if err != nil {
			return nil
		}

		var link string
		mode := info.Mode()
		switch {
		// symlinks need special treatment
		case mode&os.ModeSymlink != 0:
			linkDest, _ := os.Readlink(path)
			if link, err = filepath.EvalSymlinks(path); err == nil {
				if rel, err := filepath.Rel(filepath.Dir(path), link); err == nil {
					link = rel
				} else {
					link = linkDest
				}
			} else {
				// it may be a broken symlink, simply attempt to read it
				link = linkDest
			}

		// we don't want to tar these sort of files
		case mode&(os.ModeNamedPipe|os.ModeSocket|os.ModeDevice) != 0:
			return nil
		}

		hdr, err := tar.FileInfoHeader(info, link)
		if err != nil {
			return err
		}
		// Name is usually only the basename when created with FileInfoHeader()
		hdr.Name, err = filepath.Rel(filepath.Dir(dirPath), path)
		if err != nil {
			return err
		}

		// Some pre-POSIX.1-1988 tar implementations indicated a directory by
		// having a trailing slash in the name. Honor that here.
		if info.IsDir() {
			hdr.Name += "/"
		}

		if err := tw.WriteHeader(hdr); err != nil {
			return err
		}

		// no content to write if it is a directory or symlink
		if !info.Mode().IsRegular() {
			return nil
		}

		return func() error {
			f, err := os.Open(path)
			if err != nil {
				return err
			}
			defer f.Close()

			if _, err = io.Copy(tw, f); err != nil {
				return err
			}
			return nil
		}()
	})

	return err
}

// CreateInPlace behaves just as Create but it creates archive in place. This
// means that the original directory specified by dirPath is removed after the
// tar archive is created. The .tar suffix is automatically added to dirPath
// and is used as the name of the newly created archive.
func CreateInPlace(dirPath string) error {
	if err := Create(dirPath+".tar", dirPath); err != nil {
		return err
	}
	return os.RemoveAll(dirPath)
}

// Extract extracts a tar archive given its path.
func Extract(destPath, archivePath string) error {
	fi, err := os.Stat(archivePath)
	if err != nil {
		return err
	}

	if fi.IsDir() {
		return errors.New("given path is a directory: " + archivePath)
	}

	if err := os.MkdirAll(destPath, os.ModePerm); err != nil {
		return err
	}

	archiveFile, err := os.Open(archivePath)
	if err != nil {
		return err
	}
	defer archiveFile.Close()

	tr := tar.NewReader(archiveFile)

	for {
		hdr, err := tr.Next()
		if err == io.EOF {
			break
		}
		if err != nil {
			return err
		}

		mode := hdr.FileInfo().Mode()
		switch {
		case mode&os.ModeDir != 0:
			if err := os.Mkdir(filepath.Join(destPath, hdr.Name), mode); err != nil {
				return err
			}
		case mode&os.ModeSymlink != 0:
			os.Symlink(hdr.Linkname, filepath.Join(destPath, hdr.Name))
		default: // consider it a regular file
			createFile := func() error {
				f, err := os.Create(filepath.Join(destPath, hdr.Name))
				if err != nil {
					return err
				}
				defer f.Close()

				if _, err := io.Copy(f, tr); err != nil {
					return err
				}
				return nil
			}

			if err = createFile(); err != nil {
				return err
			}
		}
	}

	return nil
}

// ExtractInPlace extracts a tar archive, in place, given its path. The
// original tar archive is removed after extraction and only its content
// remains.
// Note that archivePath is expected to contain a file extension.
func ExtractInPlace(archivePath string) error {
	ext := filepath.Ext(archivePath)
	if ext == "" {
		return errors.New("expected a file extension (" + archivePath + ")")
	}
	destPath := filepath.Dir(strings.TrimSuffix(archivePath, ext))
	if err := Extract(destPath, archivePath); err != nil {
		return err
	}
	return os.Remove(archivePath)
}
<commit_msg>doc: fix a typo<commit_after>// Copyright 2015 Robin Hahling. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package targo provides functions to create or extract tar archives.
package targo

import (
	"archive/tar"
	"errors"
	"io"
	"os"
	"path/filepath"
	"strings"
)

// Create creates a tar archive from the directory specified by dirPath.
// The resulting tar archive format is in POSIX.1 format.
// Calling this function with dirPath having a trailing slash results in the
// content of dirPath to be used at the root level of the archive rather than
// the directory pointed out by dirPath itself.
func Create(destPath, dirPath string) error {
	fi, err := os.Stat(dirPath)
	if err != nil {
		return err
	}

	if !fi.IsDir() {
		return errors.New("given path is not a directory: " + dirPath)
	}

	file, err := os.Create(destPath)
	if err != nil {
		return err
	}
	defer file.Close()

	tw := tar.NewWriter(file)
	defer tw.Close()

	err = filepath.Walk(dirPath, func(path string, info os.FileInfo, err error) error {
		if err != nil {
			return nil
		}

		var link string
		mode := info.Mode()
		switch {
		// symlinks need special treatment
		case mode&os.ModeSymlink != 0:
			linkDest, _ := os.Readlink(path)
			if link, err = filepath.EvalSymlinks(path); err == nil {
				if rel, err := filepath.Rel(filepath.Dir(path), link); err == nil {
					link = rel
				} else {
					link = linkDest
				}
			} else {
				// it may be a broken symlink, simply attempt to read it
				link = linkDest
			}

		// we don't want to tar these sort of files
		case mode&(os.ModeNamedPipe|os.ModeSocket|os.ModeDevice) != 0:
			return nil
		}

		hdr, err := tar.FileInfoHeader(info, link)
		if err != nil {
			return err
		}
		// Name is usually only the basename when created with FileInfoHeader()
		hdr.Name, err = filepath.Rel(filepath.Dir(dirPath), path)
		if err != nil {
			return err
		}

		// Some pre-POSIX.1-1988 tar implementations indicated a directory by
		// having a trailing slash in the name. Honor that here.
		if info.IsDir() {
			hdr.Name += "/"
		}

		if err := tw.WriteHeader(hdr); err != nil {
			return err
		}

		// no content to write if it is a directory or symlink
		if !info.Mode().IsRegular() {
			return nil
		}

		return func() error {
			f, err := os.Open(path)
			if err != nil {
				return err
			}
			defer f.Close()

			if _, err = io.Copy(tw, f); err != nil {
				return err
			}
			return nil
		}()
	})

	return err
}

// CreateInPlace behaves just as Create but it creates the archive in place.
// This means that the original directory specified by dirPath is removed after
// the tar archive is created. The .tar suffix is automatically added to
// dirPath and is used as the name of the newly created archive.
func CreateInPlace(dirPath string) error {
	if err := Create(dirPath+".tar", dirPath); err != nil {
		return err
	}
	return os.RemoveAll(dirPath)
}

// Extract extracts a tar archive given its path.
func Extract(destPath, archivePath string) error {
	fi, err := os.Stat(archivePath)
	if err != nil {
		return err
	}

	if fi.IsDir() {
		return errors.New("given path is a directory: " + archivePath)
	}

	if err := os.MkdirAll(destPath, os.ModePerm); err != nil {
		return err
	}

	archiveFile, err := os.Open(archivePath)
	if err != nil {
		return err
	}
	defer archiveFile.Close()

	tr := tar.NewReader(archiveFile)

	for {
		hdr, err := tr.Next()
		if err == io.EOF {
			break
		}
		if err != nil {
			return err
		}

		mode := hdr.FileInfo().Mode()
		switch {
		case mode&os.ModeDir != 0:
			if err := os.Mkdir(filepath.Join(destPath, hdr.Name), mode); err != nil {
				return err
			}
		case mode&os.ModeSymlink != 0:
			os.Symlink(hdr.Linkname, filepath.Join(destPath, hdr.Name))
		default: // consider it a regular file
			createFile := func() error {
				f, err := os.Create(filepath.Join(destPath, hdr.Name))
				if err != nil {
					return err
				}
				defer f.Close()

				if _, err := io.Copy(f, tr); err != nil {
					return err
				}
				return nil
			}

			if err = createFile(); err != nil {
				return err
			}
		}
	}

	return nil
}

// ExtractInPlace extracts a tar archive, in place, given its path. The
// original tar archive is removed after extraction and only its content
// remains.
// Note that archivePath is expected to contain a file extension.
func ExtractInPlace(archivePath string) error {
	ext := filepath.Ext(archivePath)
	if ext == "" {
		return errors.New("expected a file extension (" + archivePath + ")")
	}
	destPath := filepath.Dir(strings.TrimSuffix(archivePath, ext))
	if err := Extract(destPath, archivePath); err != nil {
		return err
	}
	return os.Remove(archivePath)
}
<|endoftext|>


<commit_before>package archiver

import (
	"archive/tar"
	"compress/gzip"
	"fmt"
	"os"
	"strings"
)

// TarGz is for TarGz format
var TarGz tarGzFormat

func init() {
	RegisterFormat("TarGz", TarGz)
}

type tarGzFormat struct{}

func (tarGzFormat) Match(filename string) bool {
	// TODO: read file header to identify the format
	return strings.HasSuffix(strings.ToLower(filename), ".tar.gz") ||
		strings.HasSuffix(strings.ToLower(filename), ".tgz")
}

// Make creates a .tar.gz file at targzPath containing
// the contents of files listed in filePaths. It works
// the same way Tar does, but with gzip compression.
func (tarGzFormat) Make(targzPath string, filePaths []string) error {
	out, err := os.Create(targzPath)
	if err != nil {
		return fmt.Errorf("error creating %s: %v", targzPath, err)
	}
	defer out.Close()

	gzWriter := gzip.NewWriter(out)
	defer gzWriter.Close()

	tarWriter := tar.NewWriter(gzWriter)
	defer tarWriter.Close()

	return tarball(filePaths, tarWriter, targzPath)
}

// Open untars source and decompresses the contents into destination.
func (tarGzFormat) Open(source, destination string) error {
	f, err := os.Open(source)
	if err != nil {
		return fmt.Errorf("%s: failed to open archive: %v", source, err)
	}
	defer f.Close()

	gzr, err := gzip.NewReader(f)
	if err != nil {
		return fmt.Errorf("%s: create new gzip reader: %v", source, err)
	}
	defer gzr.Close()

	return untar(tar.NewReader(gzr), destination)
}
<commit_msg>Identify .tar.gz file by reading file header<commit_after>package archiver

import (
	"archive/tar"
	"compress/gzip"
	"fmt"
	"os"
	"strings"
)

// TarGz is for TarGz format
var TarGz tarGzFormat

func init() {
	RegisterFormat("TarGz", TarGz)
}

type tarGzFormat struct{}

func (tarGzFormat) Match(filename string) bool {
	return strings.HasSuffix(strings.ToLower(filename), ".tar.gz") ||
		strings.HasSuffix(strings.ToLower(filename), ".tgz") ||
		isTarGz(filename)
}

// isTarGz checks the file has the gzip compressed Tar format header by reading
// its beginning block.
func isTarGz(targzPath string) bool {
	f, err := os.Open(targzPath)
	if err != nil {
		return false
	}
	defer f.Close()

	gzr, err := gzip.NewReader(f)
	if err != nil {
		return false
	}
	defer gzr.Close()

	buf := make([]byte, tarBlockSize)
	n, err := gzr.Read(buf)
	if err != nil || n < tarBlockSize {
		return false
	}

	return hasTarHeader(buf)
}

// Make creates a .tar.gz file at targzPath containing
// the contents of files listed in filePaths. It works
// the same way Tar does, but with gzip compression.
func (tarGzFormat) Make(targzPath string, filePaths []string) error {
	out, err := os.Create(targzPath)
	if err != nil {
		return fmt.Errorf("error creating %s: %v", targzPath, err)
	}
	defer out.Close()

	gzWriter := gzip.NewWriter(out)
	defer gzWriter.Close()

	tarWriter := tar.NewWriter(gzWriter)
	defer tarWriter.Close()

	return tarball(filePaths, tarWriter, targzPath)
}

// Open untars source and decompresses the contents into destination.
func (tarGzFormat) Open(source, destination string) error {
	f, err := os.Open(source)
	if err != nil {
		return fmt.Errorf("%s: failed to open archive: %v", source, err)
	}
	defer f.Close()

	gzr, err := gzip.NewReader(f)
	if err != nil {
		return fmt.Errorf("%s: create new gzip reader: %v", source, err)
	}
	defer gzr.Close()

	return untar(tar.NewReader(gzr), destination)
}
<|endoftext|>


<commit_before>package cryptostore

import (
	"github.com/pkg/errors"

	crypto "github.com/tendermint/go-crypto"
	"github.com/tendermint/go-crypto/bcrypt"
)

var (
	// SecretBox uses the algorithm from NaCL to store secrets securely
	SecretBox Encoder = secretbox{}
	// Noop doesn't do any encryption, should only be used in test code
	Noop Encoder = noop{}
)

// Encoder is used to encrypt any key with a passphrase for storage.
//
// This should use a well-designed symetric encryption algorithm
type Encoder interface {
	Encrypt(privKey crypto.PrivKey, passphrase string) (saltBytes []byte, encBytes []byte, err error)
	Decrypt(saltBytes []byte, encBytes []byte, passphrase string) (privKey crypto.PrivKey, err error)
}

type secretbox struct{}

func (e secretbox) Encrypt(privKey crypto.PrivKey, passphrase string) (saltBytes []byte, encBytes []byte, err error) {
	if passphrase == "" {
		return nil, privKey.Bytes(), nil
	}

	saltBytes = crypto.CRandBytes(16)
	key, err := bcrypt.GenerateFromPassword(saltBytes, []byte(passphrase), 14) // TODO parameterize.  14 is good today (2016)
	if err != nil {
		return nil, nil, errors.Wrap(err, "Couldn't generate bcrypt key from passphrase.")
	}
	key = crypto.Sha256(key) // Get 32 bytes
	privKeyBytes := privKey.Bytes()
	return saltBytes, crypto.EncryptSymmetric(privKeyBytes, key), nil
}

func (e secretbox) Decrypt(saltBytes []byte, encBytes []byte, passphrase string) (privKey crypto.PrivKey, err error) {
	privKeyBytes := encBytes
	// NOTE: Some keys weren't encrypted with a passphrase and hence we have the conditional
	if passphrase != "" {
		key, err := bcrypt.GenerateFromPassword(saltBytes, []byte(passphrase), 14) // TODO parameterize.  14 is good today (2016)
		if err != nil {
			return crypto.PrivKey{}, errors.Wrap(err, "Invalid Passphrase")
		}
		key = crypto.Sha256(key) // Get 32 bytes
		privKeyBytes, err = crypto.DecryptSymmetric(encBytes, key)
		if err != nil {
			return crypto.PrivKey{}, errors.Wrap(err, "Invalid Passphrase")
		}
	}
	privKey, err = crypto.PrivKeyFromBytes(privKeyBytes)
	if err != nil {
		return crypto.PrivKey{}, errors.Wrap(err, "Couldn't get privKey from bytes")
	}
	return privKey, nil
}

type noop struct{}

func (n noop) Encrypt(key crypto.PrivKey, passphrase string) (saltBytes []byte, encBytes []byte, err error) {
	return []byte{}, key.Bytes(), nil
}

func (n noop) Decrypt(saltBytes []byte, encBytes []byte, passphrase string) (privKey crypto.PrivKey, err error) {
	return crypto.PrivKeyFromBytes(encBytes)
}
<commit_msg>Improve error message<commit_after>package cryptostore

import (
	"github.com/pkg/errors"

	crypto "github.com/tendermint/go-crypto"
	"github.com/tendermint/go-crypto/bcrypt"
)

var (
	// SecretBox uses the algorithm from NaCL to store secrets securely
	SecretBox Encoder = secretbox{}
	// Noop doesn't do any encryption, should only be used in test code
	Noop Encoder = noop{}
)

// Encoder is used to encrypt any key with a passphrase for storage.
//
// This should use a well-designed symetric encryption algorithm
type Encoder interface {
	Encrypt(privKey crypto.PrivKey, passphrase string) (saltBytes []byte, encBytes []byte, err error)
	Decrypt(saltBytes []byte, encBytes []byte, passphrase string) (privKey crypto.PrivKey, err error)
}

type secretbox struct{}

func (e secretbox) Encrypt(privKey crypto.PrivKey, passphrase string) (saltBytes []byte, encBytes []byte, err error) {
	if passphrase == "" {
		return nil, privKey.Bytes(), nil
	}

	saltBytes = crypto.CRandBytes(16)
	key, err := bcrypt.GenerateFromPassword(saltBytes, []byte(passphrase), 14) // TODO parameterize.  14 is good today (2016)
	if err != nil {
		return nil, nil, errors.Wrap(err, "Couldn't generate bcrypt key from passphrase.")
	}
	key = crypto.Sha256(key) // Get 32 bytes
	privKeyBytes := privKey.Bytes()
	return saltBytes, crypto.EncryptSymmetric(privKeyBytes, key), nil
}

func (e secretbox) Decrypt(saltBytes []byte, encBytes []byte, passphrase string) (privKey crypto.PrivKey, err error) {
	privKeyBytes := encBytes
	// NOTE: Some keys weren't encrypted with a passphrase and hence we have the conditional
	if passphrase != "" {
		key, err := bcrypt.GenerateFromPassword(saltBytes, []byte(passphrase), 14) // TODO parameterize.  14 is good today (2016)
		if err != nil {
			return crypto.PrivKey{}, errors.Wrap(err, "Invalid Passphrase")
		}
		key = crypto.Sha256(key) // Get 32 bytes
		privKeyBytes, err = crypto.DecryptSymmetric(encBytes, key)
		if err != nil {
			return crypto.PrivKey{}, errors.Wrap(err, "Invalid Passphrase")
		}
	}
	privKey, err = crypto.PrivKeyFromBytes(privKeyBytes)
	if err != nil {
		return crypto.PrivKey{}, errors.Wrap(err, "Private Key")
	}
	return privKey, nil
}

type noop struct{}

func (n noop) Encrypt(key crypto.PrivKey, passphrase string) (saltBytes []byte, encBytes []byte, err error) {
	return []byte{}, key.Bytes(), nil
}

func (n noop) Decrypt(saltBytes []byte, encBytes []byte, passphrase string) (privKey crypto.PrivKey, err error) {
	return crypto.PrivKeyFromBytes(encBytes)
}
<|endoftext|>


<commit_before>/*
guest-stats - returns disk and memory stats from guest

Example:
        { "execute": "guest-stats", "arguments": {}}
*/
package guest_stats

import (
	"bufio"
	"bytes"
	"io/ioutil"
	"strconv"
	"strings"
	"syscall"

	"github.com/vtolstov/qemu-ga/qga"
)

func init() {
	qga.RegisterCommand(&qga.Command{
		Name:    "guest-stats",
		Func:    fnGuestStats,
		Enabled: true,
		Returns: true,
	})
}

func fnGuestStats(req *qga.Request) *qga.Response {
	res := &qga.Response{Id: req.Id}
	var st syscall.Statfs_t

	resData := struct {
		MemoryTotal uint64
		MemoryFree  uint64
		SwapTotal   uint64
		SwapFree    uint64
		BlkTotal    uint64
		BlkFree     uint64
		InodeTotal  uint64
		InodeFree   uint64
	}{}

	buf, err := ioutil.ReadFile("/proc/meminfo")
	if err != nil {
		res.Error = &qga.Error{Code: -1, Desc: err.Error()}
		return res
	}

	reader := bufio.NewReader(bytes.NewBuffer(buf))

	for {
		line, _, err := reader.ReadLine()
		if err != nil {
			break
		}
		fields := strings.Fields(string(line))
		value, err := strconv.ParseUint(strings.TrimSpace(fields[1]), 10, 64)
		if err != nil {
			continue
		}
		switch strings.TrimSpace(fields[0]) {
		case "MemTotal:":
			resData.MemoryTotal = value * 1024
		case "MemFree:", "Cached:", "Buffers:":
			resData.MemoryFree += value * 1024
		case "SwapTotal:":
			resData.SwapTotal = value * 1024
		case "SwapFree:":
			resData.SwapFree = value * 1024
		}
	}

	err = syscall.Statfs("/", &st)
	if err != nil {
		res.Error = &qga.Error{Code: -1, Desc: err.Error()}
		return res
	}

	resData.BlkTotal = st.Blocks * uint64(st.Frsize)
	resData.BlkFree = st.Bavail * uint64(st.Frsize)

	resData.InodeTotal = st.Files
	resData.InodeFree = st.Ffree

	res.Return = resData
	return res
}
<commit_msg>add la<commit_after>/*
guest-stats - returns disk and memory stats from guest

Example:
        { "execute": "guest-stats", "arguments": {}}
*/
package guest_stats

import (
	"bufio"
	"bytes"
	"io/ioutil"
	"strconv"
	"strings"
	"syscall"

	"github.com/vtolstov/qemu-ga/qga"
)

func init() {
	qga.RegisterCommand(&qga.Command{
		Name:    "guest-stats",
		Func:    fnGuestStats,
		Enabled: true,
		Returns: true,
	})
}

func fnGuestStats(req *qga.Request) *qga.Response {
	res := &qga.Response{Id: req.Id}
	var st syscall.Statfs_t

	resData := struct {
		MemoryTotal uint64
		MemoryFree  uint64
		SwapTotal   uint64
		SwapFree    uint64
		BlkTotal    uint64
		BlkFree     uint64
		InodeTotal  uint64
		InodeFree   uint64
		La1         float64
		La5         float64
		La15        float64
	}{}

	buf, err := ioutil.ReadFile("/proc/loadavg")
	if err != nil {
		res.Error = &qga.Error{Code: -1, Desc: err.Error()}
		return res
	}
	fields := strings.Fields(string(buf))
	if resData.La1, err = strconv.ParseFloat(fields[0], 64); err != nil {
		resData.La1 = float64(-1)
	}
	if resData.La5, err = strconv.ParseFloat(fields[1], 64); err != nil {
		resData.La5 = float64(-1)
	}
	if resData.La15, err = strconv.ParseFloat(fields[2], 64); err != nil {
		resData.La15 = float64(-1)
	}

	buf, err = ioutil.ReadFile("/proc/meminfo")
	if err != nil {
		res.Error = &qga.Error{Code: -1, Desc: err.Error()}
		return res
	}

	reader := bufio.NewReader(bytes.NewBuffer(buf))

	for {
		line, _, err := reader.ReadLine()
		if err != nil {
			break
		}
		fields := strings.Fields(string(line))
		value, err := strconv.ParseUint(strings.TrimSpace(fields[1]), 10, 64)
		if err != nil {
			continue
		}
		switch strings.TrimSpace(fields[0]) {
		case "MemTotal:":
			resData.MemoryTotal = value * 1024
		case "MemFree:", "Cached:", "Buffers:":
			resData.MemoryFree += value * 1024
		case "SwapTotal:":
			resData.SwapTotal = value * 1024
		case "SwapFree:":
			resData.SwapFree = value * 1024
		}
	}

	err = syscall.Statfs("/", &st)
	if err != nil {
		res.Error = &qga.Error{Code: -1, Desc: err.Error()}
		return res
	}

	resData.BlkTotal = st.Blocks * uint64(st.Frsize)
	resData.BlkFree = st.Bavail * uint64(st.Frsize)

	resData.InodeTotal = st.Files
	resData.InodeFree = st.Ffree

	res.Return = resData
	return res
}
<|endoftext|>


<commit_before>// Copyright 2019 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package generator

import (
	"context"
	"errors"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"regexp"
	"strconv"
	"strings"

	"cloud.google.com/go/internal/aliasfix"
	"cloud.google.com/go/internal/aliasgen"
	"cloud.google.com/go/internal/gapicgen/execv"
	"cloud.google.com/go/internal/gapicgen/execv/gocmd"
	"cloud.google.com/go/internal/gapicgen/git"
	"golang.org/x/sync/errgroup"
)

var goPkgOptRe = regexp.MustCompile(`(?m)^option go_package = (.*);`)

// denylist is a set of clients to NOT generate.
var denylist = map[string]bool{
	// Temporarily stop generation of removed protos. Will be manually cleaned
	// up with: https://github.com/googleapis/google-cloud-go/issues/4098
	"google.golang.org/genproto/googleapis/cloud/bigquery/storage/v1alpha2": true,

	// Not properly configured:
	"google.golang.org/genproto/googleapis/cloud/ondemandscanning/v1beta1": true,
	"google.golang.org/genproto/googleapis/cloud/ondemandscanning/v1":      true,
}

// noGRPC is the set of APIs that do not need gRPC stubs.
var noGRPC = map[string]bool{
	"google.golang.org/genproto/googleapis/cloud/compute/v1": true,
}

// GenprotoGenerator is used to generate code for googleapis/go-genproto.
type GenprotoGenerator struct {
	genprotoDir     string
	googleapisDir   string
	protoSrcDir     string
	googleCloudDir  string
	gapicToGenerate string
	forceAll        bool
	genAlias        bool
}

// NewGenprotoGenerator creates a new GenprotoGenerator.
func NewGenprotoGenerator(c *Config) *GenprotoGenerator {
	return &GenprotoGenerator{
		genprotoDir:     c.GenprotoDir,
		googleapisDir:   c.GoogleapisDir,
		protoSrcDir:     filepath.Join(c.ProtoDir, "/src"),
		googleCloudDir:  c.GapicDir,
		gapicToGenerate: c.GapicToGenerate,
		forceAll:        c.ForceAll,
		genAlias:        c.GenAlias,
	}
}

var skipPrefixes = []string{
	"google.golang.org/genproto/googleapis/ads/",
	"google.golang.org/genproto/googleapis/storage/",
}

func hasPrefix(s string, prefixes []string) bool {
	for _, prefix := range prefixes {
		if strings.HasPrefix(s, prefix) {
			return true
		}
	}
	return false
}

// Regen regenerates the genproto repository.
// regenGenproto regenerates the genproto repository.
//
// regenGenproto recursively walks through each directory named by given
// arguments, looking for all .proto files. (Symlinks are not followed.) Any
// proto file without `go_package` option or whose option does not begin with
// the genproto prefix is ignored.
//
// If multiple roots contain files with the same name, eg "root1/path/to/file"
// and "root2/path/to/file", only the first file is processed; the rest are
// ignored.
//
// Protoc is executed on remaining files, one invocation per set of files
// declaring the same Go package.
func (g *GenprotoGenerator) Regen(ctx context.Context) error {
	log.Println("regenerating genproto")

	if g.genAlias {
		return g.generateAliases()
	}

	// Create space to put generated .pb.go's.
	c := execv.Command("mkdir", "-p", "generated")
	c.Dir = g.genprotoDir
	if err := c.Run(); err != nil {
		return err
	}

	// Get the last processed googleapis hash.
	lastHash, err := ioutil.ReadFile(filepath.Join(g.genprotoDir, "regen.txt"))
	if err != nil {
		return err
	}

	// TODO(noahdietz): In local mode, since it clones a shallow copy with 1 commit,
	// if the last regenerated hash is earlier than the top commit, the git diff-tree
	// command fails. This is is a bit of a rough edge. Using my local clone of
	// googleapis rectified the issue.
	pkgFiles, err := g.getUpdatedPackages(string(lastHash))
	if err != nil {
		return err
	}
	if len(pkgFiles) == 0 {
		return errors.New("couldn't find any pkgfiles")
	}

	log.Println("generating from protos")
	grp, _ := errgroup.WithContext(ctx)
	for pkg, fileNames := range pkgFiles {
		if !strings.HasPrefix(pkg, "google.golang.org/genproto") || denylist[pkg] || hasPrefix(pkg, skipPrefixes) {
			continue
		}
		grpc := !noGRPC[pkg]
		pk := pkg
		fn := fileNames

		if !isMigrated(pkg) {
			grp.Go(func() error {
				log.Println("running protoc on", pk)
				return g.protoc(fn, grpc)
			})
		} else {
			log.Printf("skipping, %q has been migrated", pkg)
		}
	}
	if err := grp.Wait(); err != nil {
		return err
	}

	if err := g.moveAndCleanupGeneratedSrc(); err != nil {
		return err
	}

	if err := gocmd.Vet(g.genprotoDir); err != nil {
		return err
	}

	if err := gocmd.Build(g.genprotoDir); err != nil {
		return err
	}

	return nil
}

// goPkg reports the import path declared in the given file's `go_package`
// option. If the option is missing, goPkg returns empty string.
func goPkg(fileName string) (string, error) {
	content, err := ioutil.ReadFile(fileName)
	if err != nil {
		return "", err
	}

	var pkgName string
	if match := goPkgOptRe.FindSubmatch(content); len(match) > 0 {
		pn, err := strconv.Unquote(string(match[1]))
		if err != nil {
			return "", err
		}
		pkgName = pn
	}
	if p := strings.IndexRune(pkgName, ';'); p > 0 {
		pkgName = pkgName[:p]
	}
	return pkgName, nil
}

// protoc executes the "protoc" command on files named in fileNames, and outputs
// to "<genprotoDir>/generated".
func (g *GenprotoGenerator) protoc(fileNames []string, grpc bool) error {
	stubs := fmt.Sprintf("--go_out=%s/generated", g.genprotoDir)
	if grpc {
		stubs = fmt.Sprintf("--go_out=plugins=grpc:%s/generated", g.genprotoDir)
	}
	args := []string{"--experimental_allow_proto3_optional", stubs, "-I", g.googleapisDir, "-I", g.protoSrcDir}
	args = append(args, fileNames...)
	c := execv.Command("protoc", args...)
	c.Dir = g.genprotoDir
	return c.Run()
}

// getUpdatedPackages parses all of the new commits to find what packages need
// to be regenerated.
func (g *GenprotoGenerator) getUpdatedPackages(googleapisHash string) (map[string][]string, error) {
	if g.forceAll {
		return g.getAllPackages()
	}
	files, err := git.UpdateFilesSinceHash(g.googleapisDir, googleapisHash)
	if err != nil {
		return nil, err
	}
	pkgFiles := make(map[string][]string)
	for _, v := range files {
		if !strings.HasSuffix(v, ".proto") {
			continue
		}
		if strings.HasSuffix(v, "compute_small.proto") {
			continue
		}
		path := filepath.Join(g.googleapisDir, v)
		pkg, err := goPkg(path)
		if err != nil {
			return nil, err
		}
		pkgFiles[pkg] = append(pkgFiles[pkg], path)
	}
	return pkgFiles, nil
}

func (g *GenprotoGenerator) getAllPackages() (map[string][]string, error) {
	seenFiles := make(map[string]bool)
	pkgFiles := make(map[string][]string)
	for _, root := range []string{g.googleapisDir} {
		walkFn := func(path string, info os.FileInfo, err error) error {
			if err != nil {
				return err
			}
			if !info.Mode().IsRegular() || !strings.HasSuffix(path, ".proto") {
				return nil
			}

			switch rel, err := filepath.Rel(root, path); {
			case err != nil:
				return err
			case seenFiles[rel]:
				return nil
			default:
				seenFiles[rel] = true
			}

			pkg, err := goPkg(path)
			if err != nil {
				return err
			}
			pkgFiles[pkg] = append(pkgFiles[pkg], path)
			return nil
		}
		if err := filepath.Walk(root, walkFn); err != nil {
			return nil, err
		}
	}
	return pkgFiles, nil
}

// moveAndCleanupGeneratedSrc moves all generated src to their correct locations
// in the repository, because protoc puts it in a folder called `generated/“.
func (g *GenprotoGenerator) moveAndCleanupGeneratedSrc() error {
	log.Println("moving generated code")
	// The period at the end is analogous to * (copy everything in this dir).
	c := execv.Command("cp", "-R", filepath.Join(g.genprotoDir, "generated", "google.golang.org", "genproto", "googleapis"), g.genprotoDir)
	if err := c.Run(); err != nil {
		return err
	}

	c = execv.Command("rm", "-rf", "generated")
	c.Dir = g.genprotoDir
	if err := c.Run(); err != nil {
		return err
	}

	return nil
}

func (g *GenprotoGenerator) generateAliases() error {
	for genprotoImport, newPkg := range aliasfix.GenprotoPkgMigration {
		if !isMigrated(genprotoImport) || g.gapicToGenerate == "" {
			continue
		}
		// remove the stubs dir segment from path
		gapicImport := newPkg.ImportPath[:strings.LastIndex(newPkg.ImportPath, "/")]
		if !strings.Contains(g.gapicToGenerate, gapicImport) {
			continue
		}
		srdDir := filepath.Join(g.googleCloudDir, strings.TrimPrefix(newPkg.ImportPath, "cloud.google.com/go/"))
		destDir := filepath.Join(g.genprotoDir, "googleapis", strings.TrimPrefix(genprotoImport, "google.golang.org/genproto/googleapis/"))
		if err := aliasgen.Run(srdDir, destDir); err != nil {
			return err
		}
	}
	return nil
}
<commit_msg>chore: no longer generate odd secretmanager client (#6888)<commit_after>// Copyright 2019 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package generator

import (
	"context"
	"errors"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"regexp"
	"strconv"
	"strings"

	"cloud.google.com/go/internal/aliasfix"
	"cloud.google.com/go/internal/aliasgen"
	"cloud.google.com/go/internal/gapicgen/execv"
	"cloud.google.com/go/internal/gapicgen/execv/gocmd"
	"cloud.google.com/go/internal/gapicgen/git"
	"golang.org/x/sync/errgroup"
)

var goPkgOptRe = regexp.MustCompile(`(?m)^option go_package = (.*);`)

// denylist is a set of clients to NOT generate.
var denylist = map[string]bool{
	// Temporarily stop generation of removed protos. Will be manually cleaned
	// up with: https://github.com/googleapis/google-cloud-go/issues/4098
	"google.golang.org/genproto/googleapis/cloud/bigquery/storage/v1alpha2": true,

	// Not properly configured:
	"google.golang.org/genproto/googleapis/cloud/ondemandscanning/v1beta1": true,
	"google.golang.org/genproto/googleapis/cloud/ondemandscanning/v1":      true,
}

// noGRPC is the set of APIs that do not need gRPC stubs.
var noGRPC = map[string]bool{
	"google.golang.org/genproto/googleapis/cloud/compute/v1": true,
}

// GenprotoGenerator is used to generate code for googleapis/go-genproto.
type GenprotoGenerator struct {
	genprotoDir     string
	googleapisDir   string
	protoSrcDir     string
	googleCloudDir  string
	gapicToGenerate string
	forceAll        bool
	genAlias        bool
}

// NewGenprotoGenerator creates a new GenprotoGenerator.
func NewGenprotoGenerator(c *Config) *GenprotoGenerator {
	return &GenprotoGenerator{
		genprotoDir:     c.GenprotoDir,
		googleapisDir:   c.GoogleapisDir,
		protoSrcDir:     filepath.Join(c.ProtoDir, "/src"),
		googleCloudDir:  c.GapicDir,
		gapicToGenerate: c.GapicToGenerate,
		forceAll:        c.ForceAll,
		genAlias:        c.GenAlias,
	}
}

var skipPrefixes = []string{
	"google.golang.org/genproto/googleapis/ads/",
	"google.golang.org/genproto/googleapis/storage/",
	"googleapis/cloud/secrets/",
}

func hasPrefix(s string, prefixes []string) bool {
	for _, prefix := range prefixes {
		if strings.HasPrefix(s, prefix) {
			return true
		}
	}
	return false
}

// Regen regenerates the genproto repository.
// regenGenproto regenerates the genproto repository.
//
// regenGenproto recursively walks through each directory named by given
// arguments, looking for all .proto files. (Symlinks are not followed.) Any
// proto file without `go_package` option or whose option does not begin with
// the genproto prefix is ignored.
//
// If multiple roots contain files with the same name, eg "root1/path/to/file"
// and "root2/path/to/file", only the first file is processed; the rest are
// ignored.
//
// Protoc is executed on remaining files, one invocation per set of files
// declaring the same Go package.
func (g *GenprotoGenerator) Regen(ctx context.Context) error {
	log.Println("regenerating genproto")

	if g.genAlias {
		return g.generateAliases()
	}

	// Create space to put generated .pb.go's.
	c := execv.Command("mkdir", "-p", "generated")
	c.Dir = g.genprotoDir
	if err := c.Run(); err != nil {
		return err
	}

	// Get the last processed googleapis hash.
	lastHash, err := ioutil.ReadFile(filepath.Join(g.genprotoDir, "regen.txt"))
	if err != nil {
		return err
	}

	// TODO(noahdietz): In local mode, since it clones a shallow copy with 1 commit,
	// if the last regenerated hash is earlier than the top commit, the git diff-tree
	// command fails. This is is a bit of a rough edge. Using my local clone of
	// googleapis rectified the issue.
	pkgFiles, err := g.getUpdatedPackages(string(lastHash))
	if err != nil {
		return err
	}
	if len(pkgFiles) == 0 {
		return errors.New("couldn't find any pkgfiles")
	}

	log.Println("generating from protos")
	grp, _ := errgroup.WithContext(ctx)
	for pkg, fileNames := range pkgFiles {
		if !strings.HasPrefix(pkg, "google.golang.org/genproto") || denylist[pkg] || hasPrefix(pkg, skipPrefixes) {
			continue
		}
		grpc := !noGRPC[pkg]
		pk := pkg
		fn := fileNames

		if !isMigrated(pkg) {
			grp.Go(func() error {
				log.Println("running protoc on", pk)
				return g.protoc(fn, grpc)
			})
		} else {
			log.Printf("skipping, %q has been migrated", pkg)
		}
	}
	if err := grp.Wait(); err != nil {
		return err
	}

	if err := g.moveAndCleanupGeneratedSrc(); err != nil {
		return err
	}

	if err := gocmd.Vet(g.genprotoDir); err != nil {
		return err
	}

	if err := gocmd.Build(g.genprotoDir); err != nil {
		return err
	}

	return nil
}

// goPkg reports the import path declared in the given file's `go_package`
// option. If the option is missing, goPkg returns empty string.
func goPkg(fileName string) (string, error) {
	content, err := ioutil.ReadFile(fileName)
	if err != nil {
		return "", err
	}

	var pkgName string
	if match := goPkgOptRe.FindSubmatch(content); len(match) > 0 {
		pn, err := strconv.Unquote(string(match[1]))
		if err != nil {
			return "", err
		}
		pkgName = pn
	}
	if p := strings.IndexRune(pkgName, ';'); p > 0 {
		pkgName = pkgName[:p]
	}
	return pkgName, nil
}

// protoc executes the "protoc" command on files named in fileNames, and outputs
// to "<genprotoDir>/generated".
func (g *GenprotoGenerator) protoc(fileNames []string, grpc bool) error {
	stubs := fmt.Sprintf("--go_out=%s/generated", g.genprotoDir)
	if grpc {
		stubs = fmt.Sprintf("--go_out=plugins=grpc:%s/generated", g.genprotoDir)
	}
	args := []string{"--experimental_allow_proto3_optional", stubs, "-I", g.googleapisDir, "-I", g.protoSrcDir}
	args = append(args, fileNames...)
	c := execv.Command("protoc", args...)
	c.Dir = g.genprotoDir
	return c.Run()
}

// getUpdatedPackages parses all of the new commits to find what packages need
// to be regenerated.
func (g *GenprotoGenerator) getUpdatedPackages(googleapisHash string) (map[string][]string, error) {
	if g.forceAll {
		return g.getAllPackages()
	}
	files, err := git.UpdateFilesSinceHash(g.googleapisDir, googleapisHash)
	if err != nil {
		return nil, err
	}
	pkgFiles := make(map[string][]string)
	for _, v := range files {
		if !strings.HasSuffix(v, ".proto") {
			continue
		}
		if strings.HasSuffix(v, "compute_small.proto") {
			continue
		}
		path := filepath.Join(g.googleapisDir, v)
		pkg, err := goPkg(path)
		if err != nil {
			return nil, err
		}
		pkgFiles[pkg] = append(pkgFiles[pkg], path)
	}
	return pkgFiles, nil
}

func (g *GenprotoGenerator) getAllPackages() (map[string][]string, error) {
	seenFiles := make(map[string]bool)
	pkgFiles := make(map[string][]string)
	for _, root := range []string{g.googleapisDir} {
		walkFn := func(path string, info os.FileInfo, err error) error {
			if err != nil {
				return err
			}
			if !info.Mode().IsRegular() || !strings.HasSuffix(path, ".proto") {
				return nil
			}

			switch rel, err := filepath.Rel(root, path); {
			case err != nil:
				return err
			case seenFiles[rel]:
				return nil
			default:
				seenFiles[rel] = true
			}

			pkg, err := goPkg(path)
			if err != nil {
				return err
			}
			pkgFiles[pkg] = append(pkgFiles[pkg], path)
			return nil
		}
		if err := filepath.Walk(root, walkFn); err != nil {
			return nil, err
		}
	}
	return pkgFiles, nil
}

// moveAndCleanupGeneratedSrc moves all generated src to their correct locations
// in the repository, because protoc puts it in a folder called `generated/“.
func (g *GenprotoGenerator) moveAndCleanupGeneratedSrc() error {
	log.Println("moving generated code")
	// The period at the end is analogous to * (copy everything in this dir).
	c := execv.Command("cp", "-R", filepath.Join(g.genprotoDir, "generated", "google.golang.org", "genproto", "googleapis"), g.genprotoDir)
	if err := c.Run(); err != nil {
		return err
	}

	c = execv.Command("rm", "-rf", "generated")
	c.Dir = g.genprotoDir
	if err := c.Run(); err != nil {
		return err
	}

	return nil
}

func (g *GenprotoGenerator) generateAliases() error {
	for genprotoImport, newPkg := range aliasfix.GenprotoPkgMigration {
		if !isMigrated(genprotoImport) || g.gapicToGenerate == "" {
			continue
		}
		// remove the stubs dir segment from path
		gapicImport := newPkg.ImportPath[:strings.LastIndex(newPkg.ImportPath, "/")]
		if !strings.Contains(g.gapicToGenerate, gapicImport) {
			continue
		}
		srdDir := filepath.Join(g.googleCloudDir, strings.TrimPrefix(newPkg.ImportPath, "cloud.google.com/go/"))
		destDir := filepath.Join(g.genprotoDir, "googleapis", strings.TrimPrefix(genprotoImport, "google.golang.org/genproto/googleapis/"))
		if err := aliasgen.Run(srdDir, destDir); err != nil {
			return err
		}
	}
	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"log"
	"os"

	"github.com/alessio/xkcdfinder/xkcd"
)

func printResults(results []xkcd.Comic) {
	for _, comic := range results {
		fmt.Printf("%4d %-40s %s\n", comic.Num, comic.Title, comic.Img)
	}
	fmt.Println("---------")
	fmt.Fprintf(os.Stderr,
		"%d results, index: %d comics, latest: %d, missing: %d\n",
		len(results), len(xkcd.ComicsIndex.Items),
		xkcd.ComicsIndex.Latest, len(xkcd.ComicsIndex.Missing))
}

func main() {
	var (
		indexFilename string
		update        bool
	)
	flag.StringVar(&indexFilename, "index", "", "Index file (default: 'index.json')")
	flag.BoolVar(&update, "update", false, "Force the update of the index")
	flag.Parse()
	if err := xkcd.LoadIndex(indexFilename); err != nil {
		log.Fatal(err)
	}
	if update {
		xkcd.UpdateIndex(indexFilename)
	}
	results := xkcd.RegexSearchComic(flag.Args())
	printResults(results)
}
<commit_msg>Small improvements to output formatting<commit_after>package main

import (
	"flag"
	"fmt"
	"log"
	"os"

	"github.com/alessio/xkcdfinder/xkcd"
)

func printResults(results []xkcd.Comic) {
	for _, comic := range results {
		fmt.Printf("%4d %-40s %s\n", comic.Num, comic.Title, comic.Img)
	}
	indexStats := fmt.Sprintf("%d results among %d comics, "+
		"index stats: latest:#%d, missing=%d\n",
		len(results),
		len(xkcd.ComicsIndex.Items),
		xkcd.ComicsIndex.Latest,
		len(xkcd.ComicsIndex.Missing))
	fmt.Fprintf(os.Stderr, indexStats)
}

func main() {
	var (
		indexFilename string
		update        bool
	)
	flag.StringVar(&indexFilename, "index", "", "Index file (default: 'index.json')")
	flag.BoolVar(&update, "update", false, "Force the update of the index")
	flag.Parse()
	if err := xkcd.LoadIndex(indexFilename); err != nil {
		log.Fatal(err)
	}
	if update {
		xkcd.UpdateIndex(indexFilename)
	}
	results := xkcd.RegexSearchComic(flag.Args())
	printResults(results)
}
<|endoftext|>


<commit_before>package scraper

import (
	"fmt"
	"log"
	"net/http"
	"os"

	"github.com/PuerkitoBio/goquery"
	"github.com/jinzhu/gorm"
	"github.com/modrzew/malusers/core"
)

// Client is a wrapper over http.Client
type Client struct {
	client *http.Client
}

var logger = log.New(os.Stdout, "[fetch] ", 0)

// Get sends GET request and converts response into goquery.Document
func (c *Client) Get(url string) *goquery.Document {
	request, err := http.NewRequest("GET", url, nil)
	// logger.Printf("GET %s\n", url)
	if err != nil {
		panic(err)
	}
	response, err := c.client.Do(request)
	if err != nil {
		panic(err)
	}
	// logger.Printf("Got response for %s\n", url)
	document, err := goquery.NewDocumentFromResponse(response)
	if err != nil {
		panic(err)
	}
	return document
}

func getClient() *Client {
	return &Client{
		client: &http.Client{},
	}
}

func getStats(username string) (*core.BasicInfo, *core.AnimeStats, *core.MangaStats) {
	client := getClient()
	url := fmt.Sprintf("https://myanimelist.net/profile/%s", username)
	document := client.Get(url)
	return ExtractBasicInfo(document.Find("ul.user-status")),
		ExtractAnimeStats(document.Find("div.stats.anime")),
		ExtractMangaStats(document.Find("div.stats.manga"))
}

func getFriends(channel chan []string, username string, offset int) {
	client := getClient()
	url := fmt.Sprintf("https://myanimelist.net/profile/%s/friends?offset=%d", username, offset)
	document := client.Get(url)
	block := document.Find("div.majorPad")
	names := ExtractFriendNames(block)
	channel <- names
	if block.Find("div.friendBlock").Length() >= 100 {
		getFriends(channel, username, offset+100)
	} else {
		close(channel)
	}
}

// GetUser obtains stats for single user and their friends
func GetUser(username string, db *gorm.DB, finished chan bool, relationsChannel chan []core.Relation) {
	user := GetOrCreateUser(username, db)
	if user.Fetched {
		return
	}
	removeFromToFetch(user.Username)
	user.Fetched = false
	user.Fetching = true
	db.Save(&user)
	basicInfo, animeStats, mangaStats := getStats(username)
	user.Birthday = basicInfo.Birthday
	user.Gender = basicInfo.Gender
	user.Location = basicInfo.Location
	animeStats.UserID = user.ID
	db.Save(animeStats)
	mangaStats.UserID = user.ID
	db.Save(mangaStats)

	friendsChannel := make(chan []string, 1)
	go getFriends(friendsChannel, username, 0)
	var relations []core.Relation
	for friendsPage := range friendsChannel {
		for i := range friendsPage {
			friendName := friendsPage[i]
			friend := GetOrCreateUser(friendName, db)
			relations = append(relations, core.NewRelation(user, friend))
		}
	}
	if len(relations) > 0 {
		relationsChannel <- relations
	}
	user.Fetched = true
	user.Fetching = false
	db.Save(&user)
	finished <- true
}
<commit_msg>Chunk relations<commit_after>package scraper

import (
	"fmt"
	"log"
	"net/http"
	"os"

	"github.com/PuerkitoBio/goquery"
	"github.com/jinzhu/gorm"
	"github.com/modrzew/malusers/core"
)

// Client is a wrapper over http.Client
type Client struct {
	client *http.Client
}

var logger = log.New(os.Stdout, "[fetch] ", 0)

// Get sends GET request and converts response into goquery.Document
func (c *Client) Get(url string) *goquery.Document {
	request, err := http.NewRequest("GET", url, nil)
	// logger.Printf("GET %s\n", url)
	if err != nil {
		panic(err)
	}
	response, err := c.client.Do(request)
	if err != nil {
		panic(err)
	}
	// logger.Printf("Got response for %s\n", url)
	document, err := goquery.NewDocumentFromResponse(response)
	if err != nil {
		panic(err)
	}
	return document
}

func getClient() *Client {
	return &Client{
		client: &http.Client{},
	}
}

func getStats(username string) (*core.BasicInfo, *core.AnimeStats, *core.MangaStats) {
	client := getClient()
	url := fmt.Sprintf("https://myanimelist.net/profile/%s", username)
	document := client.Get(url)
	return ExtractBasicInfo(document.Find("ul.user-status")),
		ExtractAnimeStats(document.Find("div.stats.anime")),
		ExtractMangaStats(document.Find("div.stats.manga"))
}

func getFriends(channel chan []string, username string, offset int) {
	client := getClient()
	url := fmt.Sprintf("https://myanimelist.net/profile/%s/friends?offset=%d", username, offset)
	document := client.Get(url)
	block := document.Find("div.majorPad")
	names := ExtractFriendNames(block)
	channel <- names
	if block.Find("div.friendBlock").Length() >= 100 {
		getFriends(channel, username, offset+100)
	} else {
		close(channel)
	}
}

// GetUser obtains stats for single user and their friends
func GetUser(username string, db *gorm.DB, finished chan bool, relationsChannel chan []core.Relation) {
	user := GetOrCreateUser(username, db)
	if user.Fetched {
		return
	}
	removeFromToFetch(user.Username)
	user.Fetched = false
	user.Fetching = true
	db.Save(&user)
	basicInfo, animeStats, mangaStats := getStats(username)
	user.Birthday = basicInfo.Birthday
	user.Gender = basicInfo.Gender
	user.Location = basicInfo.Location
	animeStats.UserID = user.ID
	db.Save(animeStats)
	mangaStats.UserID = user.ID
	db.Save(mangaStats)

	friendsChannel := make(chan []string, 1)
	go getFriends(friendsChannel, username, 0)
	var relations []core.Relation
	for friendsPage := range friendsChannel {
		for i := range friendsPage {
			friendName := friendsPage[i]
			friend := GetOrCreateUser(friendName, db)
			relations = append(relations, core.NewRelation(user, friend))
		}
	}
	relationsLen := len(relations)
	if relationsLen > 0 {
		chunkSize := 100
		for i := 0; i < relationsLen; i += chunkSize {
			end := i + chunkSize
			if end > relationsLen {
				end = relationsLen
			}
			relationsChannel <- relations[i:end]
		}
	}
	user.Fetched = true
	user.Fetching = false
	db.Save(&user)
	finished <- true
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"math/rand"
	"time"

	"github.com/docker/docker/pkg/namesgenerator"
)

func main() {
	rand.Seed(time.Now().UnixNano())
	fmt.Println(namesgenerator.GetRandomName(0))
}
<commit_msg>pkg/namesgenerator: remove names-generator binary<commit_after><|endoftext|>


<commit_before>// Copyright 2011, Bryan Matsuo. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
/*
 *  Filename:    dispatch.go
 *  Author:      Bryan Matsuo <bmatsuo@soe.ucsc.edu>
 *  Created:     Tue Jul  5 22:13:49 PDT 2011
 *  Description: Defines the Dispatch type and methods.
 */

//  Package dispatch provides goroutine dispatch and concurrency limiting.
//  It provides an object Dispatch which is a queueing system for concurrent
//  functions. It implements a dynamic limit on the number of routines that
//  it runs simultaneously. It also uses a Queue interface, allowing for
//  alternate queue implementations.
//
//  See github.com/bmatsuo/dispatch/queues for more about the Queue
//  interface and, a list of commonly used queues.
//
//  See github.com/bmatsuo/dispatch/examples for usage examples.
package dispatch

import (
    "sync"
    //"log"
    "github.com/bmatsuo/dispatch/queues"
)

//  A Dispatch is an automated function dispatch queue with a limited
//  number of concurrent gorountines. The queue can be altered with the
//  Dispatch methods Enqueue and SetKey (TODO add method Remove).
type Dispatch struct {
    // The maximum number of goroutines can be changed while the queue is
    // processing.
    MaxGo int

    // Handle waiting when the limit of concurrent goroutines has been reached.
    waitingToRun bool
    nextWait     *sync.WaitGroup

    // Handle waiting when function queue is empty.
    waitingOnQ bool
    restart    *sync.WaitGroup

    // Manage the Start()'ing of a Dispatch, avoiding race conditions.
    startLock *sync.Mutex
    started   bool

    // Handle goroutine-safe queue operations.
    qLock *sync.Mutex
    queue queues.Queue

    // Handle goroutine-safe limiting and identifier operations.
    pLock      *sync.Mutex
    pTickets   chan bool
    processing int   // Number of QueueTasks running
    idcount    int64 // pid counter

    // The longest the dispatch queue grew.
    maxlength int

    // Handle stopping of the Start() method.
    kill chan bool
}

//  Create a new Dispatch object with a specified limit on concurrency.
func New(maxroutines int) *Dispatch {
    return NewCustom(maxroutines, queues.NewFIFO())
}

//  Create a new Dispatch object with a custom backend queue satisfying
//  interface queues.Queue. It is not safe to allow non-Dispatch methods
//  any access to the object queue. This can lead to race conditions with
//  possible corruption of internal structures. So, it's considered a best
//  practice to only pass NewCustom(...) newly created queues.
//      fifoDispatch     := NewCustom(10, queues.NewFIFO())
//      priorityDispatch := NewCustom(20, queues.NewPriorityQueue())
func NewCustom(maxroutines int, queue queues.Queue) *Dispatch {
    var d = new(Dispatch)
    d.startLock = new(sync.Mutex)
    d.qLock = new(sync.Mutex)
    d.pLock = new(sync.Mutex)
    d.restart = new(sync.WaitGroup)
    d.kill = make(chan bool)
    d.nextWait = new(sync.WaitGroup)
    d.queue = queue
    d.MaxGo = maxroutines
    d.idcount = 0
    d.maxlength = 0
    d.pTickets = make(chan bool, maxroutines)
    for i := 0; i < maxroutines; i++ {
        d.pTickets <- true
    }
    return d
}

//  A simple task for use in a priority-less queue. See package
//  github.com/bmatsuo/dispatch/queues
type StdTask struct {
    F func(id int64)
}

func (dt *StdTask) Type() string {
    return "StdTask"
}
func (dt *StdTask) SetFunc(f func(id int64)) {
    dt.F = f
}
func (dt *StdTask) Func() func(id int64) {
    return dt.F
}

type dispatchTaskWrapper struct {
    id  int64
    t   queues.Task
}

func (dtw dispatchTaskWrapper) Func() func(id int64) {
    return dtw.t.Func()
}
func (dtw dispatchTaskWrapper) Id() int64 {
    return dtw.id
}
func (dtw dispatchTaskWrapper) Task() queues.Task {
    return dtw.t
}

func (gq *Dispatch) Len() int {
    return gq.queue.Len()
}
func (gq *Dispatch) MaxLen() int {
    return gq.maxlength
}
//  Enqueue a task for execution as a goroutine. The given queues.Task is
//  given a unique id (int64) and stored in the Dispatch gq's backend
//  queues.Queue object.
func (gq *Dispatch) Enqueue(t queues.Task) int64 {
    // Wrap the function so it works with the goroutine limiting code.
    var f = t.Func()
    var dtFunc = func(id int64) {
        // Run the given function.
        f(id)

        // Decrement the process counter.
        gq.pTickets <- true
        /*
        gq.pLock.Lock()
        //log.Printf("processing: %d, waiting: %v", gq.processing, gq.waitingToRun)
        gq.processing--
        if gq.waitingToRun {
            gq.waitingToRun = false
            gq.nextWait.Done()
        }
        gq.pLock.Unlock()
        */
    }
    t.SetFunc(dtFunc)

    // Lock the queue and enqueue a new task.
    gq.qLock.Lock()
    gq.idcount++
    var id = gq.idcount
    gq.queue.Enqueue(dispatchTaskWrapper{id, t})
    if gq.waitingOnQ {
        gq.waitingOnQ = false
        gq.restart.Done()
    }
    if gq.queue.Len() > gq.maxlength {
        gq.maxlength = gq.queue.Len()
    }
    gq.qLock.Unlock()

    return id
}

//  Stop the queue after gq.Start() has been called. Any goroutines which
//  have not already been dequeued will not be executed until gq.Start()
//  is called again.
func (gq *Dispatch) Stop() {
    // Lock out Start() and queue ops for the entire call.
    gq.startLock.Lock()
    defer gq.startLock.Unlock()
    gq.qLock.Lock()
    defer gq.qLock.Unlock()

    if !gq.started {
        return
    }

    // Clear channel flags and close channels, stoping further processing.
    close(gq.kill)
    gq.started = false
    if gq.waitingOnQ {
        gq.waitingOnQ = false
        gq.restart.Done()
    }
    if gq.waitingToRun {
        gq.waitingToRun = false
        gq.nextWait.Done()
    }
}

//  Start the next task in the queue. It's assumed that the queue is non-
//  empty. Furthermore, there should only be one goroutine in this method
//  (for this object) at a time. Both conditions are enforced in
//  gq.Start(), which calls gq.next() exclusively.
func (gq *Dispatch) next() {
    for true {
        // Attempt to start processing the file.
        <-gq.pTickets
        /*
        gq.pLock.Lock()
        if gq.processing >= gq.MaxGo {
            gq.waitingToRun = true
            gq.nextWait.Add(1)
            gq.pLock.Unlock()
            gq.nextWait.Wait()
            continue
        }
        // Keep the books and reset wait time before unlocking.
        gq.processing++
        gq.pLock.Unlock()
        */

        // Get an element from the queue.
        gq.qLock.Lock()
        var wrapper = gq.queue.Dequeue().(queues.RegisteredTask)
        gq.qLock.Unlock()

        // Begin processing and asyncronously return.
        //var task = taskelm.Value.(dispatchTaskWrapper)
        var task = wrapper.Func()
        go task(wrapper.Id())
        return
    }
}

//  Start executing goroutines. Don't stop until gq.Stop() is called. This
//  method will take control of the calling thread. But, it's safe to call
//  in a goroutine.
//      gq := dispatch.New()
//      go gq.Start()
//      wg := new(sync.WaitGroup)
//      for i := 0 ; i < 1000 ; i++ {
//          wg.Add(1)
//          gq.Enqueue(&dispatch.StdTask{
//              func(id int64) {
//                  log.Printf("I'm alive %d", i)
//                  wg.Done()
//              } } )
//      }
//      wg.Wait()
//      gq.Stop()
func (gq *Dispatch) Start() {
    // Avoid multiple gq.Start() methods and avoid race conditions.
    gq.startLock.Lock()
    if gq.started {
        panic("already started")
    }
    gq.started = true
    gq.startLock.Unlock()

    // Recreate any channels that were closed by a previous Stop().
    var inited = false
    for !inited {
        select {
        case _, okKill := <-gq.kill:
            if !okKill {
                gq.kill = make(chan bool)
            }
        default:
            inited = true
        }
    }

    // Process the queue
    for true {
        select {
        case die, ok := <-gq.kill:
            // If something came out of this channel, we must stop.
            if !ok {
                // Recreate the channel on a closure.
                gq.kill = make(chan bool)
                return
            }
            if die {
                return
            }
        default:
            // Check the queue size and determine if we need to wait.
            gq.qLock.Lock()
            var wait = gq.queue.Len() == 0
            if gq.waitingOnQ = wait; wait {
                gq.restart.Add(1)
            }
            gq.qLock.Unlock()

            if wait {
                // Wait for a restart signal from gq.Enqueue
                gq.restart.Wait()
            } else {
                // Process the head of the queue and start the loop again.
                gq.next()
                continue
            }
        }
    }
}
<commit_msg>Remove commented process locking stuff.<commit_after>// Copyright 2011, Bryan Matsuo. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
/*
 *  Filename:    dispatch.go
 *  Author:      Bryan Matsuo <bmatsuo@soe.ucsc.edu>
 *  Created:     Tue Jul  5 22:13:49 PDT 2011
 *  Description: Defines the Dispatch type and methods.
 */

//  Package dispatch provides goroutine dispatch and concurrency limiting.
//  It provides an object Dispatch which is a queueing system for concurrent
//  functions. It implements a dynamic limit on the number of routines that
//  it runs simultaneously. It also uses a Queue interface, allowing for
//  alternate queue implementations.
//
//  See github.com/bmatsuo/dispatch/queues for more about the Queue
//  interface and, a list of commonly used queues.
//
//  See github.com/bmatsuo/dispatch/examples for usage examples.
package dispatch

import (
    "sync"
    //"log"
    "github.com/bmatsuo/dispatch/queues"
)

//  A Dispatch is an automated function dispatch queue with a limited
//  number of concurrent gorountines. The queue can be altered with the
//  Dispatch methods Enqueue and SetKey (TODO add method Remove).
type Dispatch struct {
    // The maximum number of goroutines can be changed while the queue is
    // processing.
    MaxGo int

    // Handle waiting when the limit of concurrent goroutines has been reached.
    waitingToRun bool
    nextWait     *sync.WaitGroup

    // Handle waiting when function queue is empty.
    waitingOnQ bool
    restart    *sync.WaitGroup

    // Manage the Start()'ing of a Dispatch, avoiding race conditions.
    startLock *sync.Mutex
    started   bool

    // Handle goroutine-safe queue operations.
    qLock *sync.Mutex
    queue queues.Queue

    // Handle goroutine-safe limiting and identifier operations.
    pTickets   chan bool
    processing int   // Number of QueueTasks running
    idcount    int64 // pid counter

    // The longest the dispatch queue grew.
    maxlength int

    // Handle stopping of the Start() method.
    kill chan bool
}

//  Create a new Dispatch object with a specified limit on concurrency.
func New(maxroutines int) *Dispatch {
    return NewCustom(maxroutines, queues.NewFIFO())
}

//  Create a new Dispatch object with a custom backend queue satisfying
//  interface queues.Queue. It is not safe to allow non-Dispatch methods
//  any access to the object queue. This can lead to race conditions with
//  possible corruption of internal structures. So, it's considered a best
//  practice to only pass NewCustom(...) newly created queues.
//      fifoDispatch     := NewCustom(10, queues.NewFIFO())
//      priorityDispatch := NewCustom(20, queues.NewPriorityQueue())
func NewCustom(maxroutines int, queue queues.Queue) *Dispatch {
    var d = new(Dispatch)
    d.startLock = new(sync.Mutex)
    d.qLock = new(sync.Mutex)
    d.restart = new(sync.WaitGroup)
    d.kill = make(chan bool)
    d.nextWait = new(sync.WaitGroup)
    d.queue = queue
    d.MaxGo = maxroutines
    d.idcount = 0
    d.maxlength = 0
    d.pTickets = make(chan bool, maxroutines)
    for i := 0; i < maxroutines; i++ {
        d.pTickets <- true
    }
    return d
}

//  A simple task for use in a priority-less queue. See package
//  github.com/bmatsuo/dispatch/queues
type StdTask struct {
    F func(id int64)
}

func (dt *StdTask) Type() string {
    return "StdTask"
}
func (dt *StdTask) SetFunc(f func(id int64)) {
    dt.F = f
}
func (dt *StdTask) Func() func(id int64) {
    return dt.F
}

type dispatchTaskWrapper struct {
    id  int64
    t   queues.Task
}

func (dtw dispatchTaskWrapper) Func() func(id int64) {
    return dtw.t.Func()
}
func (dtw dispatchTaskWrapper) Id() int64 {
    return dtw.id
}
func (dtw dispatchTaskWrapper) Task() queues.Task {
    return dtw.t
}

func (gq *Dispatch) Len() int {
    return gq.queue.Len()
}
func (gq *Dispatch) MaxLen() int {
    return gq.maxlength
}
//  Enqueue a task for execution as a goroutine. The given queues.Task is
//  given a unique id (int64) and stored in the Dispatch gq's backend
//  queues.Queue object.
func (gq *Dispatch) Enqueue(t queues.Task) int64 {
    // Wrap the function so it works with the goroutine limiting code.
    var f = t.Func()
    var dtFunc = func(id int64) {
        // Run the given function.
        f(id)

        // Decrement the process counter.
        defer func() {
            recover()
        }()
        gq.pTickets <- true
    }
    t.SetFunc(dtFunc)

    // Lock the queue and enqueue a new task.
    gq.qLock.Lock()
    gq.idcount++
    var id = gq.idcount
    gq.queue.Enqueue(dispatchTaskWrapper{id, t})
    if gq.waitingOnQ {
        gq.waitingOnQ = false
        gq.restart.Done()
    }
    if gq.queue.Len() > gq.maxlength {
        gq.maxlength = gq.queue.Len()
    }
    gq.qLock.Unlock()

    return id
}

//  Stop the queue after gq.Start() has been called. Any goroutines which
//  have not already been dequeued will not be executed until gq.Start()
//  is called again.
func (gq *Dispatch) Stop() {
    // Lock out Start() and queue ops for the entire call.
    gq.startLock.Lock()
    defer gq.startLock.Unlock()
    gq.qLock.Lock()
    defer gq.qLock.Unlock()

    if !gq.started {
        return
    }

    // Clear channel flags and close channels, stoping further processing.
    close(gq.kill)
    gq.started = false
    if gq.waitingOnQ {
        gq.waitingOnQ = false
        gq.restart.Done()
    }
    close(gq.pTickets)
}

//  Start the next task in the queue. It's assumed that the queue is non-
//  empty. Furthermore, there should only be one goroutine in this method
//  (for this object) at a time. Both conditions are enforced in
//  gq.Start(), which calls gq.next() exclusively.
func (gq *Dispatch) next() {
    for true {
        // Attempt to start processing the file.
        <-gq.pTickets

        // Get an element from the queue.
        gq.qLock.Lock()
        var wrapper = gq.queue.Dequeue().(queues.RegisteredTask)
        gq.qLock.Unlock()

        // Begin processing and asyncronously return.
        //var task = taskelm.Value.(dispatchTaskWrapper)
        var task = wrapper.Func()
        go task(wrapper.Id())
        return
    }
}

//  Start executing goroutines. Don't stop until gq.Stop() is called. This
//  method will take control of the calling thread. But, it's safe to call
//  in a goroutine.
//      gq := dispatch.New()
//      go gq.Start()
//      wg := new(sync.WaitGroup)
//      for i := 0 ; i < 1000 ; i++ {
//          wg.Add(1)
//          gq.Enqueue(&dispatch.StdTask{
//              func(id int64) {
//                  log.Printf("I'm alive %d", i)
//                  wg.Done()
//              } } )
//      }
//      wg.Wait()
//      gq.Stop()
func (gq *Dispatch) Start() {
    // Avoid multiple gq.Start() methods and avoid race conditions.
    gq.startLock.Lock()
    if gq.started {
        panic("already started")
    }
    gq.started = true
    gq.startLock.Unlock()

    // Recreate any channels that were closed by a previous Stop().
    var inited = false
    for !inited {
        select {
        case _, okKill := <-gq.kill:
            if !okKill {
                gq.kill = make(chan bool)
            }
        default:
            inited = true
        }
    }

    // Process the queue
    for true {
        select {
        case die, ok := <-gq.kill:
            // If something came out of this channel, we must stop.
            if !ok {
                // Recreate the channel.
                gq.kill = make(chan bool)
                return
            }
            if die {
                return
            }
        default:
            // Check the queue size and determine if we need to wait.
            gq.qLock.Lock()
            var wait = gq.queue.Len() == 0
            if gq.waitingOnQ = wait; wait {
                gq.restart.Add(1)
            }
            gq.qLock.Unlock()

            if wait {
                // Wait for a restart signal from gq.Enqueue
                gq.restart.Wait()
            } else {
                // Process the head of the queue and start the loop again.
                gq.next()
                continue
            }
        }
    }
}
<|endoftext|>


<commit_before>package migrations

import . "github.com/grafana/grafana/pkg/services/sqlstore/migrator"

// --- Migration Guide line ---
// 1. Never change a migration that is committed and pushed to master
// 2. Always add new migrations (to change or undo previous migrations)
// 3. Some migraitons are not yet written (rename column, table, drop table, index etc)

func AddMigrations(mg *Migrator) {
	addMigrationLogMigrations(mg)
	addUserMigrations(mg)
	addTempUserMigrations(mg)
	addStarMigrations(mg)
	addOrgMigrations(mg)
	addDashboardMigration(mg)
	addDataSourceMigration(mg)
	addApiKeyMigrations(mg)
	addDashboardSnapshotMigrations(mg)
	addQuotaMigration(mg)
	addPluginBundleMigration(mg)
	addSessionMigration(mg)
  addPlaylistMigrations(mg)
}

func addMigrationLogMigrations(mg *Migrator) {
	migrationLogV1 := Table{
		Name: "migration_log",
		Columns: []*Column{
			{Name: "id", Type: DB_BigInt, IsPrimaryKey: true, IsAutoIncrement: true},
			{Name: "migration_id", Type: DB_NVarchar, Length: 255},
			{Name: "sql", Type: DB_Text},
			{Name: "success", Type: DB_Bool},
			{Name: "error", Type: DB_Text},
			{Name: "timestamp", Type: DB_DateTime},
		},
	}

	mg.AddMigration("create migration_log table", NewAddTableMigration(migrationLogV1))
}

func addStarMigrations(mg *Migrator) {
	starV1 := Table{
		Name: "star",
		Columns: []*Column{
			{Name: "id", Type: DB_BigInt, IsPrimaryKey: true, IsAutoIncrement: true},
			{Name: "user_id", Type: DB_BigInt, Nullable: false},
			{Name: "dashboard_id", Type: DB_BigInt, Nullable: false},
		},
		Indices: []*Index{
			{Cols: []string{"user_id", "dashboard_id"}, Type: UniqueIndex},
		},
	}

	mg.AddMigration("create star table", NewAddTableMigration(starV1))
	mg.AddMigration("add unique index star.user_id_dashboard_id", NewAddIndexMigration(starV1, starV1.Indices[0]))
}
<commit_msg>Fixed gofmt tests<commit_after>package migrations

import . "github.com/grafana/grafana/pkg/services/sqlstore/migrator"

// --- Migration Guide line ---
// 1. Never change a migration that is committed and pushed to master
// 2. Always add new migrations (to change or undo previous migrations)
// 3. Some migraitons are not yet written (rename column, table, drop table, index etc)

func AddMigrations(mg *Migrator) {
	addMigrationLogMigrations(mg)
	addUserMigrations(mg)
	addTempUserMigrations(mg)
	addStarMigrations(mg)
	addOrgMigrations(mg)
	addDashboardMigration(mg)
	addDataSourceMigration(mg)
	addApiKeyMigrations(mg)
	addDashboardSnapshotMigrations(mg)
	addQuotaMigration(mg)
	addPluginBundleMigration(mg)
	addSessionMigration(mg)
	addPlaylistMigrations(mg)
}

func addMigrationLogMigrations(mg *Migrator) {
	migrationLogV1 := Table{
		Name: "migration_log",
		Columns: []*Column{
			{Name: "id", Type: DB_BigInt, IsPrimaryKey: true, IsAutoIncrement: true},
			{Name: "migration_id", Type: DB_NVarchar, Length: 255},
			{Name: "sql", Type: DB_Text},
			{Name: "success", Type: DB_Bool},
			{Name: "error", Type: DB_Text},
			{Name: "timestamp", Type: DB_DateTime},
		},
	}

	mg.AddMigration("create migration_log table", NewAddTableMigration(migrationLogV1))
}

func addStarMigrations(mg *Migrator) {
	starV1 := Table{
		Name: "star",
		Columns: []*Column{
			{Name: "id", Type: DB_BigInt, IsPrimaryKey: true, IsAutoIncrement: true},
			{Name: "user_id", Type: DB_BigInt, Nullable: false},
			{Name: "dashboard_id", Type: DB_BigInt, Nullable: false},
		},
		Indices: []*Index{
			{Cols: []string{"user_id", "dashboard_id"}, Type: UniqueIndex},
		},
	}

	mg.AddMigration("create star table", NewAddTableMigration(starV1))
	mg.AddMigration("add unique index star.user_id_dashboard_id", NewAddIndexMigration(starV1, starV1.Indices[0]))
}
<|endoftext|>


<commit_before>// Copyright 2012 Aaron Jacobs. All Rights Reserved.
// Author: aaronjjacobs@gmail.com (Aaron Jacobs)
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package repr_test

import (
	"code.google.com/p/goprotobuf/proto"
	"github.com/jacobsa/comeback/blob"
	"github.com/jacobsa/comeback/repr"
	"github.com/jacobsa/comeback/repr/proto"
	. "github.com/jacobsa/oglematchers"
	. "github.com/jacobsa/ogletest"
	"testing"
)

func TestUnmarshalTest(t *testing.T) { RunTests(t) }

////////////////////////////////////////////////////////////////////////
// Helpers
////////////////////////////////////////////////////////////////////////

func makeLegalEntryProto() *repr_proto.DirectoryEntryProto {
	return &repr_proto.DirectoryEntryProto{}
}

type UnmarshalTest struct {
}

func init() { RegisterTestSuite(&UnmarshalTest{}) }

////////////////////////////////////////////////////////////////////////
// Tests
////////////////////////////////////////////////////////////////////////

func (t *UnmarshalTest) JunkWireData() {
	// Input
	d := []byte("asdf")

	// Call
	_, err := repr.Unmarshal(d)

	ExpectThat(err, Error(HasSubstr("Parsing data")))
}

func (t *UnmarshalTest) UnknownTypeValue() {
	// Input
	listingProto := &repr_proto.DirectoryListingProto{
		Entry: []*repr_proto.DirectoryEntryProto{
			makeLegalEntryProto(),
			makeLegalEntryProto(),
			makeLegalEntryProto(),
		},
	}

	listingProto.Entry[1].Type = repr_proto.DirectoryEntryProto_Type(17).Enum()

	data, err := proto.Marshal(listingProto)
	AssertEq(nil, err)

	// Call
	_, err = repr.Unmarshal(data)

	ExpectThat(err, Error(HasSubstr("Unrecognized")))
	ExpectThat(err, Error(HasSubstr("DirectoryEntryProto_Type")))
	ExpectThat(err, Error(HasSubstr("17")))
}

func (t *UnmarshalTest) HashIsTooShort() {
	// Input
	listingProto := &repr_proto.DirectoryListingProto{
		Entry: []*repr_proto.DirectoryEntryProto{
			makeLegalEntryProto(),
			makeLegalEntryProto(),
			makeLegalEntryProto(),
		},
	}

	listingProto.Entry[1].Blob = []*repr_proto.BlobInfoProto{
		&repr_proto.BlobInfoProto{Hash: blob.ComputeScore([]byte{}).Sha1Hash()},
		&repr_proto.BlobInfoProto{Hash: blob.ComputeScore([]byte{}).Sha1Hash()},
		&repr_proto.BlobInfoProto{Hash: blob.ComputeScore([]byte{}).Sha1Hash()},
	}

	blob := listingProto.Entry[1].Blob[1]
	blob.Hash = blob.Hash[0:len(blob.Hash)-1]

	data, err := proto.Marshal(listingProto)
	AssertEq(nil, err)

	// Call
	_, err = repr.Unmarshal(data)

	ExpectThat(err, Error(HasSubstr("hash length")))
	ExpectThat(err, Error(HasSubstr("19")))
}

func (t *UnmarshalTest) HashIsTooLong() {
	// Input
	listingProto := &repr_proto.DirectoryListingProto{
		Entry: []*repr_proto.DirectoryEntryProto{
			makeLegalEntryProto(),
			makeLegalEntryProto(),
			makeLegalEntryProto(),
		},
	}

	listingProto.Entry[1].Blob = []*repr_proto.BlobInfoProto{
		&repr_proto.BlobInfoProto{Hash: blob.ComputeScore([]byte{}).Sha1Hash()},
		&repr_proto.BlobInfoProto{Hash: blob.ComputeScore([]byte{}).Sha1Hash()},
		&repr_proto.BlobInfoProto{Hash: blob.ComputeScore([]byte{}).Sha1Hash()},
	}

	blob := listingProto.Entry[1].Blob[1]
	blob.Hash = append(blob.Hash, 0x01)

	data, err := proto.Marshal(listingProto)
	AssertEq(nil, err)

	// Call
	_, err = repr.Unmarshal(data)

	ExpectThat(err, Error(HasSubstr("hash length")))
	ExpectThat(err, Error(HasSubstr("21")))
}
<commit_msg>UnmarshalTest.UnknownTypeValue<commit_after>// Copyright 2012 Aaron Jacobs. All Rights Reserved.
// Author: aaronjjacobs@gmail.com (Aaron Jacobs)
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package repr_test

import (
	"code.google.com/p/goprotobuf/proto"
	"github.com/jacobsa/comeback/blob"
	"github.com/jacobsa/comeback/repr"
	"github.com/jacobsa/comeback/repr/proto"
	. "github.com/jacobsa/oglematchers"
	. "github.com/jacobsa/ogletest"
	"testing"
)

func TestUnmarshalTest(t *testing.T) { RunTests(t) }

////////////////////////////////////////////////////////////////////////
// Helpers
////////////////////////////////////////////////////////////////////////

func makeLegalEntryProto() *repr_proto.DirectoryEntryProto {
	return &repr_proto.DirectoryEntryProto{}
}

type UnmarshalTest struct {
}

func init() { RegisterTestSuite(&UnmarshalTest{}) }

////////////////////////////////////////////////////////////////////////
// Tests
////////////////////////////////////////////////////////////////////////

func (t *UnmarshalTest) JunkWireData() {
	// Input
	d := []byte("asdf")

	// Call
	_, err := repr.Unmarshal(d)

	ExpectThat(err, Error(HasSubstr("Parsing data")))
}

func (t *UnmarshalTest) InvalidTypeValue() {
	// Input
	listingProto := &repr_proto.DirectoryListingProto{
		Entry: []*repr_proto.DirectoryEntryProto{
			makeLegalEntryProto(),
			makeLegalEntryProto(),
			makeLegalEntryProto(),
		},
	}

	listingProto.Entry[1].Type = repr_proto.DirectoryEntryProto_Type(17).Enum()

	data, err := proto.Marshal(listingProto)
	AssertEq(nil, err)

	// Call
	_, err = repr.Unmarshal(data)

	ExpectThat(err, Error(HasSubstr("Unrecognized")))
	ExpectThat(err, Error(HasSubstr("DirectoryEntryProto_Type")))
	ExpectThat(err, Error(HasSubstr("17")))
}

func (t *UnmarshalTest) UnknownTypeValue() {
	// Input
	listingProto := &repr_proto.DirectoryListingProto{
		Entry: []*repr_proto.DirectoryEntryProto{
			makeLegalEntryProto(),
			makeLegalEntryProto(),
			makeLegalEntryProto(),
		},
	}

	listingProto.Entry[1].Type = repr_proto.DirectoryEntryProto_TYPE_UNKNOWN.Enum()

	data, err := proto.Marshal(listingProto)
	AssertEq(nil, err)

	// Call
	_, err = repr.Unmarshal(data)

	ExpectThat(err, Error(HasSubstr("Unrecognized")))
	ExpectThat(err, Error(HasSubstr("DirectoryEntryProto_Type")))
	ExpectThat(err, Error(HasSubstr("TYPE_UNKNOWN")))
}

func (t *UnmarshalTest) HashIsTooShort() {
	// Input
	listingProto := &repr_proto.DirectoryListingProto{
		Entry: []*repr_proto.DirectoryEntryProto{
			makeLegalEntryProto(),
			makeLegalEntryProto(),
			makeLegalEntryProto(),
		},
	}

	listingProto.Entry[1].Blob = []*repr_proto.BlobInfoProto{
		&repr_proto.BlobInfoProto{Hash: blob.ComputeScore([]byte{}).Sha1Hash()},
		&repr_proto.BlobInfoProto{Hash: blob.ComputeScore([]byte{}).Sha1Hash()},
		&repr_proto.BlobInfoProto{Hash: blob.ComputeScore([]byte{}).Sha1Hash()},
	}

	blob := listingProto.Entry[1].Blob[1]
	blob.Hash = blob.Hash[0:len(blob.Hash)-1]

	data, err := proto.Marshal(listingProto)
	AssertEq(nil, err)

	// Call
	_, err = repr.Unmarshal(data)

	ExpectThat(err, Error(HasSubstr("hash length")))
	ExpectThat(err, Error(HasSubstr("19")))
}

func (t *UnmarshalTest) HashIsTooLong() {
	// Input
	listingProto := &repr_proto.DirectoryListingProto{
		Entry: []*repr_proto.DirectoryEntryProto{
			makeLegalEntryProto(),
			makeLegalEntryProto(),
			makeLegalEntryProto(),
		},
	}

	listingProto.Entry[1].Blob = []*repr_proto.BlobInfoProto{
		&repr_proto.BlobInfoProto{Hash: blob.ComputeScore([]byte{}).Sha1Hash()},
		&repr_proto.BlobInfoProto{Hash: blob.ComputeScore([]byte{}).Sha1Hash()},
		&repr_proto.BlobInfoProto{Hash: blob.ComputeScore([]byte{}).Sha1Hash()},
	}

	blob := listingProto.Entry[1].Blob[1]
	blob.Hash = append(blob.Hash, 0x01)

	data, err := proto.Marshal(listingProto)
	AssertEq(nil, err)

	// Call
	_, err = repr.Unmarshal(data)

	ExpectThat(err, Error(HasSubstr("hash length")))
	ExpectThat(err, Error(HasSubstr("21")))
}
<|endoftext|>