<commit_before>package vfsswift

import (
	"bytes"
	"encoding/hex"
	"encoding/json"
	"path"

	"github.com/cozy/cozy-stack/model/vfs"
	"github.com/cozy/cozy-stack/pkg/consts"
	"github.com/cozy/cozy-stack/pkg/couchdb"
	"github.com/cozy/swift"
)

func (sfs *swiftVFSV3) Fsck(accumulate func(log *vfs.FsckLog)) error {
	entries := make(map[string]*vfs.TreeFile, 1024)
	tree, err := sfs.BuildTree(func(f *vfs.TreeFile) {
		if !f.IsDir {
			entries[f.DocID] = f
		}
	})
	if err != nil {
		return err
	}
	if err = sfs.CheckTreeIntegrity(tree, accumulate); err != nil {
		return err
	}
	return sfs.checkFiles(entries, accumulate)
}

func (sfs *swiftVFSV3) CheckFilesConsistency(accumulate func(log *vfs.FsckLog)) error {
	entries := make(map[string]*vfs.TreeFile, 1024)
	_, err := sfs.BuildTree(func(f *vfs.TreeFile) {
		if !f.IsDir {
			entries[f.DirID+"/"+f.DocName] = f
		}
	})
	if err != nil {
		return err
	}
	return sfs.checkFiles(entries, accumulate)
}

func (sfs *swiftVFSV3) checkFiles(entries map[string]*vfs.TreeFile, accumulate func(log *vfs.FsckLog)) (err error) {
	versions := make(map[string]*vfs.Version, 1024)
	err = couchdb.ForeachDocs(sfs, consts.FilesVersions, func(_ string, data json.RawMessage) error {
		v := &vfs.Version{}
		if erru := json.Unmarshal(data, v); erru != nil {
			return erru
		}
		versions[v.DocID] = v
		return nil
	})
	if err != nil {
		return
	}

	err = sfs.c.ObjectsWalk(sfs.container, nil, func(opts *swift.ObjectsOpts) (interface{}, error) {
		var objs []swift.Object
		objs, err = sfs.c.Objects(sfs.container, opts)
		if err != nil {
			return nil, err
		}
		for _, obj := range objs {
			docID, internalID := makeDocIDV3(obj.Name)
			if v, ok := versions[docID+"/"+internalID]; ok {
				var md5sum []byte
				md5sum, err = hex.DecodeString(obj.Hash)
				if err != nil {
					return nil, err
				}
				if !bytes.Equal(md5sum, v.MD5Sum) || v.ByteSize != obj.Bytes {
					accumulate(&vfs.FsckLog{
						Type:       vfs.ContentMismatch,
						IsVersion:  true,
						VersionDoc: v,
						ContentMismatch: &vfs.FsckContentMismatch{
							SizeFile:    obj.Bytes,
							SizeIndex:   v.ByteSize,
							MD5SumFile:  md5sum,
							MD5SumIndex: v.MD5Sum,
						},
					})
				}
				delete(versions, v.DocID)
				continue
			}
			f, ok := entries[docID]
			if !ok {
				accumulate(&vfs.FsckLog{
					Type:    vfs.IndexMissing,
					IsFile:  true,
					FileDoc: objectToFileDocV3(sfs.container, obj),
				})
			} else {
				var md5sum []byte
				md5sum, err = hex.DecodeString(obj.Hash)
				if err != nil {
					return nil, err
				}
				if !bytes.Equal(md5sum, f.MD5Sum) || f.ByteSize != obj.Bytes {
					accumulate(&vfs.FsckLog{
						Type:    vfs.ContentMismatch,
						IsFile:  true,
						FileDoc: f,
						ContentMismatch: &vfs.FsckContentMismatch{
							SizeFile:    obj.Bytes,
							SizeIndex:   f.ByteSize,
							MD5SumFile:  md5sum,
							MD5SumIndex: f.MD5Sum,
						},
					})
				}
				delete(entries, docID)
			}
		}
		return objs, nil
	})
	if err != nil {
		return
	}

	// entries should contain only data that does not contain an associated
	// index.
	for _, f := range entries {
		accumulate(&vfs.FsckLog{
			Type:    vfs.FSMissing,
			IsFile:  true,
			FileDoc: f,
		})
	}

	for _, v := range versions {
		accumulate(&vfs.FsckLog{
			Type:       vfs.FSMissing,
			IsVersion:  true,
			VersionDoc: v,
		})
	}

	return
}

func objectToFileDocV3(container string, object swift.Object) *vfs.TreeFile {
	md5sum, _ := hex.DecodeString(object.Hash)
	name := "unknown"
	mime, class := vfs.ExtractMimeAndClass(object.ContentType)
	fileID, internalID := makeDocIDV3(object.Name)
	return &vfs.TreeFile{
		DirOrFileDoc: vfs.DirOrFileDoc{
			DirDoc: &vfs.DirDoc{
				Type:      consts.FileType,
				DocID:     fileID,
				DocName:   name,
				DirID:     "",
				CreatedAt: object.LastModified,
				UpdatedAt: object.LastModified,
				Fullpath:  path.Join(vfs.OrphansDirName, name),
			},
			ByteSize:   object.Bytes,
			Mime:       mime,
			Class:      class,
			MD5Sum:     md5sum,
			InternalID: internalID,
		},
	}
}
<commit_msg>Ignore thumbnails in fsck for Swift v3 (#1973)<commit_after>package vfsswift

import (
	"bytes"
	"encoding/hex"
	"encoding/json"
	"path"
	"strings"

	"github.com/cozy/cozy-stack/model/vfs"
	"github.com/cozy/cozy-stack/pkg/consts"
	"github.com/cozy/cozy-stack/pkg/couchdb"
	"github.com/cozy/swift"
)

func (sfs *swiftVFSV3) Fsck(accumulate func(log *vfs.FsckLog)) error {
	entries := make(map[string]*vfs.TreeFile, 1024)
	tree, err := sfs.BuildTree(func(f *vfs.TreeFile) {
		if !f.IsDir {
			entries[f.DocID] = f
		}
	})
	if err != nil {
		return err
	}
	if err = sfs.CheckTreeIntegrity(tree, accumulate); err != nil {
		return err
	}
	return sfs.checkFiles(entries, accumulate)
}

func (sfs *swiftVFSV3) CheckFilesConsistency(accumulate func(log *vfs.FsckLog)) error {
	entries := make(map[string]*vfs.TreeFile, 1024)
	_, err := sfs.BuildTree(func(f *vfs.TreeFile) {
		if !f.IsDir {
			entries[f.DirID+"/"+f.DocName] = f
		}
	})
	if err != nil {
		return err
	}
	return sfs.checkFiles(entries, accumulate)
}

func (sfs *swiftVFSV3) checkFiles(entries map[string]*vfs.TreeFile, accumulate func(log *vfs.FsckLog)) (err error) {
	versions := make(map[string]*vfs.Version, 1024)
	err = couchdb.ForeachDocs(sfs, consts.FilesVersions, func(_ string, data json.RawMessage) error {
		v := &vfs.Version{}
		if erru := json.Unmarshal(data, v); erru != nil {
			return erru
		}
		versions[v.DocID] = v
		return nil
	})
	if err != nil {
		return
	}

	err = sfs.c.ObjectsWalk(sfs.container, nil, func(opts *swift.ObjectsOpts) (interface{}, error) {
		var objs []swift.Object
		objs, err = sfs.c.Objects(sfs.container, opts)
		if err != nil {
			return nil, err
		}
		for _, obj := range objs {
			if strings.HasPrefix(obj.Name, "thumbs/") {
				continue
			}
			docID, internalID := makeDocIDV3(obj.Name)
			if v, ok := versions[docID+"/"+internalID]; ok {
				var md5sum []byte
				md5sum, err = hex.DecodeString(obj.Hash)
				if err != nil {
					return nil, err
				}
				if !bytes.Equal(md5sum, v.MD5Sum) || v.ByteSize != obj.Bytes {
					accumulate(&vfs.FsckLog{
						Type:       vfs.ContentMismatch,
						IsVersion:  true,
						VersionDoc: v,
						ContentMismatch: &vfs.FsckContentMismatch{
							SizeFile:    obj.Bytes,
							SizeIndex:   v.ByteSize,
							MD5SumFile:  md5sum,
							MD5SumIndex: v.MD5Sum,
						},
					})
				}
				delete(versions, v.DocID)
				continue
			}
			f, ok := entries[docID]
			if !ok {
				accumulate(&vfs.FsckLog{
					Type:    vfs.IndexMissing,
					IsFile:  true,
					FileDoc: objectToFileDocV3(sfs.container, obj),
				})
			} else {
				var md5sum []byte
				md5sum, err = hex.DecodeString(obj.Hash)
				if err != nil {
					return nil, err
				}
				if !bytes.Equal(md5sum, f.MD5Sum) || f.ByteSize != obj.Bytes {
					accumulate(&vfs.FsckLog{
						Type:    vfs.ContentMismatch,
						IsFile:  true,
						FileDoc: f,
						ContentMismatch: &vfs.FsckContentMismatch{
							SizeFile:    obj.Bytes,
							SizeIndex:   f.ByteSize,
							MD5SumFile:  md5sum,
							MD5SumIndex: f.MD5Sum,
						},
					})
				}
				delete(entries, docID)
			}
		}
		return objs, nil
	})
	if err != nil {
		return
	}

	// entries should contain only data that does not contain an associated
	// index.
	for _, f := range entries {
		accumulate(&vfs.FsckLog{
			Type:    vfs.FSMissing,
			IsFile:  true,
			FileDoc: f,
		})
	}

	for _, v := range versions {
		accumulate(&vfs.FsckLog{
			Type:       vfs.FSMissing,
			IsVersion:  true,
			VersionDoc: v,
		})
	}

	return
}

func objectToFileDocV3(container string, object swift.Object) *vfs.TreeFile {
	md5sum, _ := hex.DecodeString(object.Hash)
	name := "unknown"
	mime, class := vfs.ExtractMimeAndClass(object.ContentType)
	fileID, internalID := makeDocIDV3(object.Name)
	return &vfs.TreeFile{
		DirOrFileDoc: vfs.DirOrFileDoc{
			DirDoc: &vfs.DirDoc{
				Type:      consts.FileType,
				DocID:     fileID,
				DocName:   name,
				DirID:     "",
				CreatedAt: object.LastModified,
				UpdatedAt: object.LastModified,
				Fullpath:  path.Join(vfs.OrphansDirName, name),
			},
			ByteSize:   object.Bytes,
			Mime:       mime,
			Class:      class,
			MD5Sum:     md5sum,
			InternalID: internalID,
		},
	}
}
<|endoftext|>


<commit_before>package article

import "strings"

type Article struct {
	Title  string
	Link   string
	Date   string
	Author string
}

type ArticleAction interface {
	ContainKeyword(keyword string) bool
}

func (a Article) ContainKeyword(keyword string) bool {
	return strings.Contains(a.Title, keyword)
}

type Articles []Article

func (as Articles) String() string {
	var content string
	for _, article := range as {
		content += article.Title + "\r\n" +
			"https://www.ptt.cc" + article.Link + "\r\n" +
			"\r\n"
	}
	return content
}
<commit_msg>make article  contain keyword incasesensetive<commit_after>package article

import "strings"

type Article struct {
	Title  string
	Link   string
	Date   string
	Author string
}

type ArticleAction interface {
	ContainKeyword(keyword string) bool
}

func (a Article) ContainKeyword(keyword string) bool {
	return strings.Contains(strings.ToLower(a.Title), strings.ToLower(keyword))
}

type Articles []Article

func (as Articles) String() string {
	var content string
	for _, article := range as {
		content += article.Title + "\r\n" +
			"https://www.ptt.cc" + article.Link + "\r\n" +
			"\r\n"
	}
	return content
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"log"
	"time"

	"github.com/kidoman/embd"
	_ "github.com/kidoman/embd/host/all"
)

// Lidar is structure to access basic functions of LidarLite
// LidarLite_V2 blue label
// Documentation on http://lidarlite.com/docs/v2/specs_and_hardware
type Lidar struct {
	bus     embd.I2CBus
	address byte
}

// Ready - Ready status. 0 - ready for a new command, 1 - busy
//          with acquisition
// RefOverflow - Overflow detected in correlation process assotieted with
//          a reference acquisition
// SignalOverflow - Overflow detected in correlation process assotieted with a signal
//          acquisition
// SignalNotValid - Indicates that the signal correlation peak is equal to or below
//          correlation record threshold
// SecondaryReturn - Secondary return detected above correlation noise floor threshold
// Health - 1 if is good, 0 if is bad
// ErrorDetection - Process error detected/measurement invalid
// EyeSafe - This bit will go high if eye-safety protection has been activated
//const (
// Ready           = 1 << iota
// RefOverflow     = 1 << iota
// SigOverflow     = 1 << iota
// SignalNotValid  = 1 << iota
// SecondatyReturn = 1 << iota
// Health          = 1 << iota
// ErrorDetection  = 1 << iota
// EyeSafe         = 1 << iota
//)

// NewLidar sets the configuration for the sensor
// Write 0x00 to Register 0x00 reset FPGA. Re-loads FPGA from internal Flash
// memory: all registers return to default values
func NewLidar(i2cbus, addr byte) *Lidar {

	lSensor := Lidar{bus: embd.NewI2CBus(i2cbus), address: byte(addr)}
	if e := lSensor.bus.WriteByteToReg(lSensor.address, 0x00, 0x00); e != nil {
		log.Panic("Write ", e)
	}
	time.Sleep(300 * time.Millisecond)
	return &lSensor

}

// GetStatus gets Mode/Status of sensor
func (ls *Lidar) GetStatus() (byte, error) {

	val, err := ls.bus.ReadByteFromReg(ls.address, 0x01)
	if err == nil {
		log.Println("Status ", val)
	} else {
		log.Panic("GetStatus ", err)
		return 0, err
	}
	return val, nil

}

// Distance reads the distance from LidarLite
// stablizePreampFlag - true - take aquisition with DC stabilisation/correction.
// false - it will read faster, but you will need to stabilize DC every once in
// awhile(ex. 1 out of every 100 readings is typically good)
func (ls *Lidar) Distance(stablizePreampFlag bool) (int, error) {

	var wErr error // Write and Read errors

	if _, errSt := ls.GetStatus(); errSt == nil {
		if stablizePreampFlag {
			wErr = ls.bus.WriteByteToReg(ls.address, 0x00, 0x04)
		} else {
			wErr = ls.bus.WriteByteToReg(ls.address, 0x00, 0x03)
		}

		if wErr != nil {
			log.Panic("Write ", wErr)
			return 0, wErr
		}

		time.Sleep(100 * time.Millisecond)
	} else {
		log.Panic(errSt)
		return 0, errSt
	}

	_, errSt := ls.GetStatus()
	if errSt == nil {
		v1, rErr := ls.bus.ReadByteFromReg(ls.address, 0x10)
		if rErr != nil {
			log.Panic("Read ", rErr)
			return 0, rErr
		}
		v2, rErr := ls.bus.ReadByteFromReg(ls.address, 0x0f)
		if rErr != nil {
			log.Panic("Read", rErr)
			return 0, rErr
		}

		return ((int(v2) << 8) + int(v1)), nil

	}
	log.Panic(errSt)
	return 0, errSt

}

func main() {
	log.SetFlags(log.Lshortfile)
	lidar := NewLidar(1, 0x62) // 0x62 the default LidarLite address
	defer lidar.bus.Close()
	for {
		if val, err := lidar.Distance(true); err == nil {
			fmt.Println(val)
		}
	}

}
<commit_msg>Add CloseLidar function<commit_after>package main

import (
	"fmt"
	"log"
	"time"

	"github.com/kidoman/embd"
	_ "github.com/kidoman/embd/host/all"
)

// Lidar is structure to access basic functions of LidarLite
// LidarLite_V2 blue label
// Documentation on http://lidarlite.com/docs/v2/specs_and_hardware
type Lidar struct {
	bus     embd.I2CBus
	address byte
}

// Ready - Ready status. 0 - ready for a new command, 1 - busy
//          with acquisition
// RefOverflow - Overflow detected in correlation process assotieted with
//          a reference acquisition
// SignalOverflow - Overflow detected in correlation process assotieted with a signal
//          acquisition
// SignalNotValid - Indicates that the signal correlation peak is equal to or below
//          correlation record threshold
// SecondaryReturn - Secondary return detected above correlation noise floor threshold
// Health - 1 if is good, 0 if is bad
// ErrorDetection - Process error detected/measurement invalid
// EyeSafe - This bit will go high if eye-safety protection has been activated
//const (
// Ready           = 1 << iota
// RefOverflow     = 1 << iota
// SigOverflow     = 1 << iota
// SignalNotValid  = 1 << iota
// SecondatyReturn = 1 << iota
// Health          = 1 << iota
// ErrorDetection  = 1 << iota
// EyeSafe         = 1 << iota
//)

// NewLidar sets the configuration for the sensor
// Write 0x00 to Register 0x00 reset FPGA. Re-loads FPGA from internal Flash
// memory: all registers return to default values
func NewLidar(i2cbus, addr byte) *Lidar {

	lSensor := Lidar{bus: embd.NewI2CBus(i2cbus), address: byte(addr)}
	if e := lSensor.bus.WriteByteToReg(lSensor.address, 0x00, 0x00); e != nil {
		log.Panic("Write ", e)
	}
	time.Sleep(300 * time.Millisecond)
	return &lSensor
}

//CloseLidar closes releases the resources associated with the bus
func (ls *Lidar) CloseLidar() {
	if err := ls.bus.Close(); err == nil {
		log.Println(err)
	}
}

// GetStatus gets Mode/Status of sensor
func (ls *Lidar) GetStatus() (byte, error) {

	val, err := ls.bus.ReadByteFromReg(ls.address, 0x01)
	if err == nil {
		log.Printf("Status: %.8b\n ", val)
	} else {
		log.Println("GetStatus ", err)
		return 0, err
	}
	return val, nil

}

// Distance reads the distance from LidarLite
// stablizePreampFlag - true - take aquisition with DC stabilisation/correction.
// false - it will read faster, but you will need to stabilize DC every once in
// awhile(ex. 1 out of every 100 readings is typically good)
func (ls *Lidar) Distance(stablizePreampFlag bool) (int, error) {

	var wErr error // Write and Read errors

	if _, errSt := ls.GetStatus(); errSt == nil {
		if stablizePreampFlag {
			wErr = ls.bus.WriteByteToReg(ls.address, 0x00, 0x04)
		} else {
			wErr = ls.bus.WriteByteToReg(ls.address, 0x00, 0x03)
		}

		if wErr != nil {
			log.Println("Write ", wErr)
			return 0, wErr
		}

		time.Sleep(100 * time.Millisecond)
	} else {
		log.Println(errSt)
		return 0, errSt
	}

	_, errSt := ls.GetStatus()
	if errSt == nil {
		v1, rErr := ls.bus.ReadByteFromReg(ls.address, 0x10)
		if rErr != nil {
			log.Println("Read ", rErr)
			return 0, rErr
		}
		v2, rErr := ls.bus.ReadByteFromReg(ls.address, 0x0f)
		if rErr != nil {
			log.Println("Read", rErr)
			return 0, rErr
		}

		return ((int(v2) << 8) + int(v1)), nil

	}
	log.Println(errSt)
	return 0, errSt
}

func main() {
	log.SetFlags(log.Lshortfile)
	lidar := NewLidar(1, 0x62) // 0x62 the default LidarLite address
	defer lidar.CloseLidar()
	for {
		if val, err := lidar.Distance(true); err == nil {
			fmt.Println(val)
		}
	}

}
<|endoftext|>


<commit_before>// Copyright 2015 Davis Webb
// Copyright 2015 Guntas Grewal
// Copyright 2015 Luke Shumaker

package store

import (
	"database/sql"
	"golang.org/x/crypto/bcrypt"
	he "httpentity"
	"strings"
)

var _ he.Entity = &User{}
var _ he.NetEntity = &User{}
var dirUsers he.Entity

func init() {
	dirUsers = newDirUsers()
}

// Model /////////////////////////////////////////////////////////////

type User struct {
	Id       string
	FullName string
	Email    string
	pwHash   []byte
}

func getUserById(con DB, id int) *User {
	var user User
	err := con.QueryRow("SELECT * FROM users WHERE id=?", id).Scan(&user)
	switch {
	case err == sql.ErrNoRows:
		// user does not exist
		return nil
	case err != nil:
		// error talking to the DB
		panic(err)
	default:
		// all ok
		return &user
	}
}

func GetUserByName(con DB, name string) *User {
	var user User
	err := con.QueryRow("SELECT * FROM users WHERE name=?", name).Scan(&user)
	switch {
	case err == sql.ErrNoRows:
		// user does not exist
		return nil
	case err != nil:
		// error talking to the DB
		panic(err)
	default:
		// all ok
		return &user
	}
}

func GetUserByEmail(con DB, address string) *User {
	var user User
	err := con.QueryRow(""+
		"SELECT users.* "+
		"FROM users JOIN user_address ON users.id=user_address.user_id "+
		"WHERE user_address.address=? AND user_address=?",
		address,
		"email").Scan(&user)
	switch {
	case err == sql.ErrNoRows:
		// user does not exist
		return nil
	case err != nil:
		// error talking to the DB
		panic(err)
	default:
		// all ok
		return &user
	}
}

func (u *User) SetPassword(password string) error {
	hash, err := bcrypt.GenerateFromPassword([]byte(password), -1)
	u.pwHash = hash
	return err
}

func (u *User) CheckPassword(password string) bool {
	err := bcrypt.CompareHashAndPassword(u.pwHash, []byte(password))
	return err != nil
}

func NewUser(con DB, name string, password string, email string) *User {
	u := &User{
		Id:       name,
		FullName: "",
		Email:    email,
	}
	u.SetPassword(password)
	_, err := con.Exec("INSERT INTO users VALUES (?,?,?,?)", u.Id, u.FullName, u.pwHash, u.Email)
	if err != nil {
		panic(err)
	}
	return u
}

func (u *User) Save() error {
	/*TODO*/
	panic("not implemented")
}

func (o *User) Subentity(name string, req he.Request) he.Entity {
	return nil
}

func (o *User) Methods() map[string]he.Handler {
	panic("not implemented")
}

// View //////////////////////////////////////////////////////////////

func (o *User) Encoders() map[string]he.Encoder {
	return defaultEncoders(o)
}

// Directory ("Controller") //////////////////////////////////////////

type t_dirUsers struct {
	methods map[string]he.Handler
}

func newDirUsers() t_dirUsers {
	r := t_dirUsers{}
	r.methods = map[string]he.Handler{
		"POST": func(req he.Request) he.Response {
			db := req.Things["db"].(DB)
			badbody := req.StatusBadRequest("submitted body not what expected")
			hash, ok := req.Entity.(map[string]interface{})
			if !ok {
				return badbody
			}
			username, ok := hash["username"].(string)
			if !ok {
				return badbody
			}
			email, ok := hash["email"].(string)
			if !ok {
				return badbody
			}
			password, ok := hash["password"].(string)
			if !ok {
				return badbody
			}

			if password2, ok := hash["password_verification"].(string); ok {
				if password != password2 {
					// Passwords don't match
					return req.StatusConflict(he.NetString("password and password_verification don't match"))
				}
			}

			username = strings.ToLower(username)

			user := NewUser(db, username, password, email)
			if user == nil {
				return req.StatusConflict(he.NetString("either that username or password is already taken"))
			} else {
				return req.StatusCreated(dirUsers, username)
			}
		},
	}
	return r
}

func (d t_dirUsers) Methods() map[string]he.Handler {
	return d.methods
}

func (d t_dirUsers) Subentity(name string, request he.Request) he.Entity {
	con := getConnection()
	defer con.Close()
	return GetUserByName(con, name)
}
<commit_msg>stub out the supported methods for users<commit_after>// Copyright 2015 Davis Webb
// Copyright 2015 Guntas Grewal
// Copyright 2015 Luke Shumaker

package store

import (
	"database/sql"
	"golang.org/x/crypto/bcrypt"
	he "httpentity"
	"strings"
)

var _ he.Entity = &User{}
var _ he.NetEntity = &User{}
var dirUsers he.Entity

func init() {
	dirUsers = newDirUsers()
}

// Model /////////////////////////////////////////////////////////////

type User struct {
	Id       string
	FullName string
	Email    string
	pwHash   []byte
}

func getUserById(con DB, id int) *User {
	var user User
	err := con.QueryRow("SELECT * FROM users WHERE id=?", id).Scan(&user)
	switch {
	case err == sql.ErrNoRows:
		// user does not exist
		return nil
	case err != nil:
		// error talking to the DB
		panic(err)
	default:
		// all ok
		return &user
	}
}

func GetUserByName(con DB, name string) *User {
	var user User
	err := con.QueryRow("SELECT * FROM users WHERE name=?", name).Scan(&user)
	switch {
	case err == sql.ErrNoRows:
		// user does not exist
		return nil
	case err != nil:
		// error talking to the DB
		panic(err)
	default:
		// all ok
		return &user
	}
}

func GetUserByEmail(con DB, address string) *User {
	var user User
	err := con.QueryRow(""+
		"SELECT users.* "+
		"FROM users JOIN user_address ON users.id=user_address.user_id "+
		"WHERE user_address.address=? AND user_address=?",
		address,
		"email").Scan(&user)
	switch {
	case err == sql.ErrNoRows:
		// user does not exist
		return nil
	case err != nil:
		// error talking to the DB
		panic(err)
	default:
		// all ok
		return &user
	}
}

func (u *User) SetPassword(password string) error {
	hash, err := bcrypt.GenerateFromPassword([]byte(password), -1)
	u.pwHash = hash
	return err
}

func (u *User) CheckPassword(password string) bool {
	err := bcrypt.CompareHashAndPassword(u.pwHash, []byte(password))
	return err != nil
}

func NewUser(con DB, name string, password string, email string) *User {
	u := &User{
		Id:       name,
		FullName: "",
		Email:    email,
	}
	u.SetPassword(password)
	_, err := con.Exec("INSERT INTO users VALUES (?,?,?,?)", u.Id, u.FullName, u.pwHash, u.Email)
	if err != nil {
		panic(err)
	}
	return u
}

func (u *User) Save() error {
	panic("not implemented")
}

func (o *User) Subentity(name string, req he.Request) he.Entity {
	return nil
}

func (o *User) Methods() map[string]he.Handler {
	return map[string]he.Handler{
		"GET": func(req he.Request) he.Response {
			panic("not implemented")
		},
		"PUT": func(req he.Request) he.Response {
			panic("not implemented")
		},
		"PATCH": func(req he.Request) he.Response {
			panic("not implemented")
		},
		"DELETE": func(req he.Request) he.Response {
			panic("not implemented")
		},
	}
}

// View //////////////////////////////////////////////////////////////

func (o *User) Encoders() map[string]he.Encoder {
	return defaultEncoders(o)
}

// Directory ("Controller") //////////////////////////////////////////

type t_dirUsers struct {
	methods map[string]he.Handler
}

func newDirUsers() t_dirUsers {
	r := t_dirUsers{}
	r.methods = map[string]he.Handler{
		"POST": func(req he.Request) he.Response {
			db := req.Things["db"].(DB)
			badbody := req.StatusBadRequest("submitted body not what expected")
			hash, ok := req.Entity.(map[string]interface{})
			if !ok {
				return badbody
			}
			username, ok := hash["username"].(string)
			if !ok {
				return badbody
			}
			email, ok := hash["email"].(string)
			if !ok {
				return badbody
			}
			password, ok := hash["password"].(string)
			if !ok {
				return badbody
			}

			if password2, ok := hash["password_verification"].(string); ok {
				if password != password2 {
					// Passwords don't match
					return req.StatusConflict(he.NetString("password and password_verification don't match"))
				}
			}

			username = strings.ToLower(username)

			user := NewUser(db, username, password, email)
			if user == nil {
				return req.StatusConflict(he.NetString("either that username or password is already taken"))
			} else {
				return req.StatusCreated(dirUsers, username)
			}
		},
	}
	return r
}

func (d t_dirUsers) Methods() map[string]he.Handler {
	return d.methods
}

func (d t_dirUsers) Subentity(name string, request he.Request) he.Entity {
	con := getConnection()
	defer con.Close()
	return GetUserByName(con, name)
}
<|endoftext|>


<commit_before>// Copyright 2010 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package json

import (
	"bytes"
	"math"
	"rand"
	"reflect"
	"testing"
)

// Tests of simple examples.

type example struct {
	compact string
	indent  string
}

var examples = []example{
	{`1`, `1`},
	{`{}`, `{}`},
	{`[]`, `[]`},
	{`{"":2}`, "{\n\t\"\": 2\n}"},
	{`[3]`, "[\n\t3\n]"},
	{`[1,2,3]`, "[\n\t1,\n\t2,\n\t3\n]"},
	{`{"x":1}`, "{\n\t\"x\": 1\n}"},
	{ex1, ex1i},
}

var ex1 = `[true,false,null,"x",1,1.5,0,-5e+2]`

var ex1i = `[
	true,
	false,
	null,
	"x",
	1,
	1.5,
	0,
	-5e+2
]`

func TestCompact(t *testing.T) {
	var buf bytes.Buffer
	for _, tt := range examples {
		buf.Reset()
		if err := Compact(&buf, []byte(tt.compact)); err != nil {
			t.Errorf("Compact(%#q): %v", tt.compact, err)
		} else if s := buf.String(); s != tt.compact {
			t.Errorf("Compact(%#q) = %#q, want original", tt.compact, s)
		}

		buf.Reset()
		if err := Compact(&buf, []byte(tt.indent)); err != nil {
			t.Errorf("Compact(%#q): %v", tt.indent, err)
			continue
		} else if s := buf.String(); s != tt.compact {
			t.Errorf("Compact(%#q) = %#q, want %#q", tt.indent, s, tt.compact)
		}
	}
}

func TestIndent(t *testing.T) {
	var buf bytes.Buffer
	for _, tt := range examples {
		buf.Reset()
		if err := Indent(&buf, []byte(tt.indent), "", "\t"); err != nil {
			t.Errorf("Indent(%#q): %v", tt.indent, err)
		} else if s := buf.String(); s != tt.indent {
			t.Errorf("Indent(%#q) = %#q, want original", tt.indent, s)
		}

		buf.Reset()
		if err := Indent(&buf, []byte(tt.compact), "", "\t"); err != nil {
			t.Errorf("Indent(%#q): %v", tt.compact, err)
			continue
		} else if s := buf.String(); s != tt.indent {
			t.Errorf("Indent(%#q) = %#q, want %#q", tt.compact, s, tt.indent)
		}
	}
}

// Tests of a large random structure.

func TestCompactBig(t *testing.T) {
	initBig()
	var buf bytes.Buffer
	if err := Compact(&buf, jsonBig); err != nil {
		t.Fatalf("Compact: %v", err)
	}
	b := buf.Bytes()
	if bytes.Compare(b, jsonBig) != 0 {
		t.Error("Compact(jsonBig) != jsonBig")
		diff(t, b, jsonBig)
		return
	}
}

func TestIndentBig(t *testing.T) {
	initBig()
	var buf bytes.Buffer
	if err := Indent(&buf, jsonBig, "", "\t"); err != nil {
		t.Fatalf("Indent1: %v", err)
	}
	b := buf.Bytes()
	if len(b) == len(jsonBig) {
		// jsonBig is compact (no unnecessary spaces);
		// indenting should make it bigger
		t.Fatalf("Indent(jsonBig) did not get bigger")
	}

	// should be idempotent
	var buf1 bytes.Buffer
	if err := Indent(&buf1, b, "", "\t"); err != nil {
		t.Fatalf("Indent2: %v", err)
	}
	b1 := buf1.Bytes()
	if bytes.Compare(b1, b) != 0 {
		t.Error("Indent(Indent(jsonBig)) != Indent(jsonBig)")
		diff(t, b1, b)
		return
	}

	// should get back to original
	buf1.Reset()
	if err := Compact(&buf1, b); err != nil {
		t.Fatalf("Compact: %v", err)
	}
	b1 = buf1.Bytes()
	if bytes.Compare(b1, jsonBig) != 0 {
		t.Error("Compact(Indent(jsonBig)) != jsonBig")
		diff(t, b1, jsonBig)
		return
	}
}

type indentErrorTest struct {
	in  string
	err error
}

var indentErrorTests = []indentErrorTest{
	{`{"X": "foo", "Y"}`, &SyntaxError{"invalid character '}' after object key", 17}},
	{`{"X": "foo" "Y": "bar"}`, &SyntaxError{"invalid character '\"' after object key:value pair", 13}},
}

func TestIdentErrors(t *testing.T) {
	for i, tt := range indentErrorTests {
		slice := make([]uint8, 0)
		buf := bytes.NewBuffer(slice)
		if err := Indent(buf, []uint8(tt.in), "", ""); err != nil {
			if !reflect.DeepEqual(err, tt.err) {
				t.Errorf("#%d: Indent: %#v", i, err)
				continue
			}
		}
	}
}

func TestNextValueBig(t *testing.T) {
	initBig()
	var scan scanner
	item, rest, err := nextValue(jsonBig, &scan)
	if err != nil {
		t.Fatalf("nextValue: %s", err)
	}
	if len(item) != len(jsonBig) || &item[0] != &jsonBig[0] {
		t.Errorf("invalid item: %d %d", len(item), len(jsonBig))
	}
	if len(rest) != 0 {
		t.Errorf("invalid rest: %d", len(rest))
	}

	item, rest, err = nextValue(append(jsonBig, "HELLO WORLD"...), &scan)
	if err != nil {
		t.Fatalf("nextValue extra: %s", err)
	}
	if len(item) != len(jsonBig) {
		t.Errorf("invalid item: %d %d", len(item), len(jsonBig))
	}
	if string(rest) != "HELLO WORLD" {
		t.Errorf("invalid rest: %d", len(rest))
	}
}

func BenchmarkSkipValue(b *testing.B) {
	initBig()
	var scan scanner
	for i := 0; i < b.N; i++ {
		nextValue(jsonBig, &scan)
	}
	b.SetBytes(int64(len(jsonBig)))
}

func diff(t *testing.T, a, b []byte) {
	for i := 0; ; i++ {
		if i >= len(a) || i >= len(b) || a[i] != b[i] {
			j := i - 10
			if j < 0 {
				j = 0
			}
			t.Errorf("diverge at %d: «%s» vs «%s»", i, trim(a[j:]), trim(b[j:]))
			return
		}
	}
}

func trim(b []byte) []byte {
	if len(b) > 20 {
		return b[0:20]
	}
	return b
}

// Generate a random JSON object.

var jsonBig []byte

const (
	big   = 10000
	small = 100
)

func initBig() {
	n := big
	if testing.Short() {
		n = small
	}
	if len(jsonBig) != n {
		b, err := Marshal(genValue(n))
		if err != nil {
			panic(err)
		}
		jsonBig = b
	}
}

func genValue(n int) interface{} {
	if n > 1 {
		switch rand.Intn(2) {
		case 0:
			return genArray(n)
		case 1:
			return genMap(n)
		}
	}
	switch rand.Intn(3) {
	case 0:
		return rand.Intn(2) == 0
	case 1:
		return rand.NormFloat64()
	case 2:
		return genString(30)
	}
	panic("unreachable")
}

func genString(stddev float64) string {
	n := int(math.Abs(rand.NormFloat64()*stddev + stddev/2))
	c := make([]rune, n)
	for i := range c {
		f := math.Abs(rand.NormFloat64()*64 + 32)
		if f > 0x10ffff {
			f = 0x10ffff
		}
		c[i] = rune(f)
	}
	return string(c)
}

func genArray(n int) []interface{} {
	f := int(math.Abs(rand.NormFloat64()) * math.Min(10, float64(n/2)))
	if f > n {
		f = n
	}
	if n > 0 && f == 0 {
		f = 1
	}
	x := make([]interface{}, f)
	for i := range x {
		x[i] = genValue(((i+1)*n)/f - (i*n)/f)
	}
	return x
}

func genMap(n int) map[string]interface{} {
	f := int(math.Abs(rand.NormFloat64()) * math.Min(10, float64(n/2)))
	if f > n {
		f = n
	}
	if n > 0 && f == 0 {
		f = 1
	}
	x := make(map[string]interface{})
	for i := 0; i < f; i++ {
		x[genString(10)] = genValue(((i+1)*n)/f - (i*n)/f)
	}
	return x
}
<commit_msg>json: fix typo in scanner_test.go.<commit_after>// Copyright 2010 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package json

import (
	"bytes"
	"math"
	"rand"
	"reflect"
	"testing"
)

// Tests of simple examples.

type example struct {
	compact string
	indent  string
}

var examples = []example{
	{`1`, `1`},
	{`{}`, `{}`},
	{`[]`, `[]`},
	{`{"":2}`, "{\n\t\"\": 2\n}"},
	{`[3]`, "[\n\t3\n]"},
	{`[1,2,3]`, "[\n\t1,\n\t2,\n\t3\n]"},
	{`{"x":1}`, "{\n\t\"x\": 1\n}"},
	{ex1, ex1i},
}

var ex1 = `[true,false,null,"x",1,1.5,0,-5e+2]`

var ex1i = `[
	true,
	false,
	null,
	"x",
	1,
	1.5,
	0,
	-5e+2
]`

func TestCompact(t *testing.T) {
	var buf bytes.Buffer
	for _, tt := range examples {
		buf.Reset()
		if err := Compact(&buf, []byte(tt.compact)); err != nil {
			t.Errorf("Compact(%#q): %v", tt.compact, err)
		} else if s := buf.String(); s != tt.compact {
			t.Errorf("Compact(%#q) = %#q, want original", tt.compact, s)
		}

		buf.Reset()
		if err := Compact(&buf, []byte(tt.indent)); err != nil {
			t.Errorf("Compact(%#q): %v", tt.indent, err)
			continue
		} else if s := buf.String(); s != tt.compact {
			t.Errorf("Compact(%#q) = %#q, want %#q", tt.indent, s, tt.compact)
		}
	}
}

func TestIndent(t *testing.T) {
	var buf bytes.Buffer
	for _, tt := range examples {
		buf.Reset()
		if err := Indent(&buf, []byte(tt.indent), "", "\t"); err != nil {
			t.Errorf("Indent(%#q): %v", tt.indent, err)
		} else if s := buf.String(); s != tt.indent {
			t.Errorf("Indent(%#q) = %#q, want original", tt.indent, s)
		}

		buf.Reset()
		if err := Indent(&buf, []byte(tt.compact), "", "\t"); err != nil {
			t.Errorf("Indent(%#q): %v", tt.compact, err)
			continue
		} else if s := buf.String(); s != tt.indent {
			t.Errorf("Indent(%#q) = %#q, want %#q", tt.compact, s, tt.indent)
		}
	}
}

// Tests of a large random structure.

func TestCompactBig(t *testing.T) {
	initBig()
	var buf bytes.Buffer
	if err := Compact(&buf, jsonBig); err != nil {
		t.Fatalf("Compact: %v", err)
	}
	b := buf.Bytes()
	if bytes.Compare(b, jsonBig) != 0 {
		t.Error("Compact(jsonBig) != jsonBig")
		diff(t, b, jsonBig)
		return
	}
}

func TestIndentBig(t *testing.T) {
	initBig()
	var buf bytes.Buffer
	if err := Indent(&buf, jsonBig, "", "\t"); err != nil {
		t.Fatalf("Indent1: %v", err)
	}
	b := buf.Bytes()
	if len(b) == len(jsonBig) {
		// jsonBig is compact (no unnecessary spaces);
		// indenting should make it bigger
		t.Fatalf("Indent(jsonBig) did not get bigger")
	}

	// should be idempotent
	var buf1 bytes.Buffer
	if err := Indent(&buf1, b, "", "\t"); err != nil {
		t.Fatalf("Indent2: %v", err)
	}
	b1 := buf1.Bytes()
	if bytes.Compare(b1, b) != 0 {
		t.Error("Indent(Indent(jsonBig)) != Indent(jsonBig)")
		diff(t, b1, b)
		return
	}

	// should get back to original
	buf1.Reset()
	if err := Compact(&buf1, b); err != nil {
		t.Fatalf("Compact: %v", err)
	}
	b1 = buf1.Bytes()
	if bytes.Compare(b1, jsonBig) != 0 {
		t.Error("Compact(Indent(jsonBig)) != jsonBig")
		diff(t, b1, jsonBig)
		return
	}
}

type indentErrorTest struct {
	in  string
	err error
}

var indentErrorTests = []indentErrorTest{
	{`{"X": "foo", "Y"}`, &SyntaxError{"invalid character '}' after object key", 17}},
	{`{"X": "foo" "Y": "bar"}`, &SyntaxError{"invalid character '\"' after object key:value pair", 13}},
}

func TestIndentErrors(t *testing.T) {
	for i, tt := range indentErrorTests {
		slice := make([]uint8, 0)
		buf := bytes.NewBuffer(slice)
		if err := Indent(buf, []uint8(tt.in), "", ""); err != nil {
			if !reflect.DeepEqual(err, tt.err) {
				t.Errorf("#%d: Indent: %#v", i, err)
				continue
			}
		}
	}
}

func TestNextValueBig(t *testing.T) {
	initBig()
	var scan scanner
	item, rest, err := nextValue(jsonBig, &scan)
	if err != nil {
		t.Fatalf("nextValue: %s", err)
	}
	if len(item) != len(jsonBig) || &item[0] != &jsonBig[0] {
		t.Errorf("invalid item: %d %d", len(item), len(jsonBig))
	}
	if len(rest) != 0 {
		t.Errorf("invalid rest: %d", len(rest))
	}

	item, rest, err = nextValue(append(jsonBig, "HELLO WORLD"...), &scan)
	if err != nil {
		t.Fatalf("nextValue extra: %s", err)
	}
	if len(item) != len(jsonBig) {
		t.Errorf("invalid item: %d %d", len(item), len(jsonBig))
	}
	if string(rest) != "HELLO WORLD" {
		t.Errorf("invalid rest: %d", len(rest))
	}
}

func BenchmarkSkipValue(b *testing.B) {
	initBig()
	var scan scanner
	for i := 0; i < b.N; i++ {
		nextValue(jsonBig, &scan)
	}
	b.SetBytes(int64(len(jsonBig)))
}

func diff(t *testing.T, a, b []byte) {
	for i := 0; ; i++ {
		if i >= len(a) || i >= len(b) || a[i] != b[i] {
			j := i - 10
			if j < 0 {
				j = 0
			}
			t.Errorf("diverge at %d: «%s» vs «%s»", i, trim(a[j:]), trim(b[j:]))
			return
		}
	}
}

func trim(b []byte) []byte {
	if len(b) > 20 {
		return b[0:20]
	}
	return b
}

// Generate a random JSON object.

var jsonBig []byte

const (
	big   = 10000
	small = 100
)

func initBig() {
	n := big
	if testing.Short() {
		n = small
	}
	if len(jsonBig) != n {
		b, err := Marshal(genValue(n))
		if err != nil {
			panic(err)
		}
		jsonBig = b
	}
}

func genValue(n int) interface{} {
	if n > 1 {
		switch rand.Intn(2) {
		case 0:
			return genArray(n)
		case 1:
			return genMap(n)
		}
	}
	switch rand.Intn(3) {
	case 0:
		return rand.Intn(2) == 0
	case 1:
		return rand.NormFloat64()
	case 2:
		return genString(30)
	}
	panic("unreachable")
}

func genString(stddev float64) string {
	n := int(math.Abs(rand.NormFloat64()*stddev + stddev/2))
	c := make([]rune, n)
	for i := range c {
		f := math.Abs(rand.NormFloat64()*64 + 32)
		if f > 0x10ffff {
			f = 0x10ffff
		}
		c[i] = rune(f)
	}
	return string(c)
}

func genArray(n int) []interface{} {
	f := int(math.Abs(rand.NormFloat64()) * math.Min(10, float64(n/2)))
	if f > n {
		f = n
	}
	if n > 0 && f == 0 {
		f = 1
	}
	x := make([]interface{}, f)
	for i := range x {
		x[i] = genValue(((i+1)*n)/f - (i*n)/f)
	}
	return x
}

func genMap(n int) map[string]interface{} {
	f := int(math.Abs(rand.NormFloat64()) * math.Min(10, float64(n/2)))
	if f > n {
		f = n
	}
	if n > 0 && f == 0 {
		f = 1
	}
	x := make(map[string]interface{})
	for i := 0; i < f; i++ {
		x[genString(10)] = genValue(((i+1)*n)/f - (i*n)/f)
	}
	return x
}
<|endoftext|>


<commit_before>// Copyright 2011 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package runtime_test

import (
	"math"
	"runtime"
	"sync/atomic"
	"testing"
	"time"
)

var stop = make(chan bool, 1)

func perpetuumMobile() {
	select {
	case <-stop:
	default:
		go perpetuumMobile()
	}
}

func TestStopTheWorldDeadlock(t *testing.T) {
	if testing.Short() {
		t.Skip("skipping during short test")
	}
	maxprocs := runtime.GOMAXPROCS(3)
	compl := make(chan bool, 2)
	go func() {
		for i := 0; i != 1000; i += 1 {
			runtime.GC()
		}
		compl <- true
	}()
	go func() {
		for i := 0; i != 1000; i += 1 {
			runtime.GOMAXPROCS(3)
		}
		compl <- true
	}()
	go perpetuumMobile()
	<-compl
	<-compl
	stop <- true
	runtime.GOMAXPROCS(maxprocs)
}

func TestYieldProgress(t *testing.T) {
	testYieldProgress(t, false)
}

func TestYieldLockedProgress(t *testing.T) {
	testYieldProgress(t, true)
}

func testYieldProgress(t *testing.T, locked bool) {
	c := make(chan bool)
	cack := make(chan bool)
	go func() {
		if locked {
			runtime.LockOSThread()
		}
		for {
			select {
			case <-c:
				cack <- true
				return
			default:
				runtime.Gosched()
			}
		}
	}()
	time.Sleep(10 * time.Millisecond)
	c <- true
	<-cack
}

func TestYieldLocked(t *testing.T) {
	const N = 10
	c := make(chan bool)
	go func() {
		runtime.LockOSThread()
		for i := 0; i < N; i++ {
			runtime.Gosched()
			time.Sleep(time.Millisecond)
		}
		c <- true
		// runtime.UnlockOSThread() is deliberately omitted
	}()
	<-c
}

func TestBlockLocked(t *testing.T) {
	const N = 10
	c := make(chan bool)
	go func() {
		runtime.LockOSThread()
		for i := 0; i < N; i++ {
			c <- true
		}
		runtime.UnlockOSThread()
	}()
	for i := 0; i < N; i++ {
		<-c
	}
}

func stackGrowthRecursive(i int) {
	var pad [128]uint64
	if i != 0 && pad[0] == 0 {
		stackGrowthRecursive(i - 1)
	}
}

func TestSchedLocalQueue(t *testing.T) {
	runtime.TestSchedLocalQueue1()
}

func TestSchedLocalQueueSteal(t *testing.T) {
	runtime.TestSchedLocalQueueSteal1()
}

func benchmarkStackGrowth(b *testing.B, rec int) {
	const CallsPerSched = 1000
	procs := runtime.GOMAXPROCS(-1)
	N := int32(b.N / CallsPerSched)
	c := make(chan bool, procs)
	for p := 0; p < procs; p++ {
		go func() {
			for atomic.AddInt32(&N, -1) >= 0 {
				runtime.Gosched()
				for g := 0; g < CallsPerSched; g++ {
					stackGrowthRecursive(rec)
				}
			}
			c <- true
		}()
	}
	for p := 0; p < procs; p++ {
		<-c
	}
}

func BenchmarkStackGrowth(b *testing.B) {
	benchmarkStackGrowth(b, 10)
}

func BenchmarkStackGrowthDeep(b *testing.B) {
	benchmarkStackGrowth(b, 1024)
}

func BenchmarkSyscall(b *testing.B) {
	const CallsPerSched = 1000
	procs := runtime.GOMAXPROCS(-1)
	N := int32(b.N / CallsPerSched)
	c := make(chan bool, procs)
	for p := 0; p < procs; p++ {
		go func() {
			for atomic.AddInt32(&N, -1) >= 0 {
				runtime.Gosched()
				for g := 0; g < CallsPerSched; g++ {
					runtime.Entersyscall()
					runtime.Exitsyscall()
				}
			}
			c <- true
		}()
	}
	for p := 0; p < procs; p++ {
		<-c
	}
}

func BenchmarkSyscallWork(b *testing.B) {
	const CallsPerSched = 1000
	const LocalWork = 100
	procs := runtime.GOMAXPROCS(-1)
	N := int32(b.N / CallsPerSched)
	c := make(chan bool, procs)
	for p := 0; p < procs; p++ {
		go func() {
			foo := 42
			for atomic.AddInt32(&N, -1) >= 0 {
				runtime.Gosched()
				for g := 0; g < CallsPerSched; g++ {
					runtime.Entersyscall()
					for i := 0; i < LocalWork; i++ {
						foo *= 2
						foo /= 2
					}
					runtime.Exitsyscall()
				}
			}
			c <- foo == 42
		}()
	}
	for p := 0; p < procs; p++ {
		<-c
	}
}

func BenchmarkCreateGoroutines(b *testing.B) {
	benchmarkCreateGoroutines(b, 1)
}

func BenchmarkCreateGoroutinesParallel(b *testing.B) {
	benchmarkCreateGoroutines(b, runtime.GOMAXPROCS(-1))
}

func benchmarkCreateGoroutines(b *testing.B, procs int) {
	c := make(chan bool)
	var f func(n int)
	f = func(n int) {
		if n == 0 {
			c <- true
			return
		}
		go f(n - 1)
	}
	for i := 0; i < procs; i++ {
		go f(b.N / procs)
	}
	for i := 0; i < procs; i++ {
		<-c
	}
}

type Matrix [][]float64

func BenchmarkMatmult(b *testing.B) {
	b.StopTimer()
	// matmult is O(N**3) but testing expects O(b.N),
	// so we need to take cube root of b.N
	n := int(math.Cbrt(float64(b.N))) + 1
	A := makeMatrix(n)
	B := makeMatrix(n)
	C := makeMatrix(n)
	b.StartTimer()
	matmult(nil, A, B, C, 0, n, 0, n, 0, n, 8)
}

func makeMatrix(n int) Matrix {
	m := make(Matrix, n)
	for i := 0; i < n; i++ {
		m[i] = make([]float64, n)
		for j := 0; j < n; j++ {
			m[i][j] = float64(i*n + j)
		}
	}
	return m
}

func matmult(done chan<- struct{}, A, B, C Matrix, i0, i1, j0, j1, k0, k1, threshold int) {
	di := i1 - i0
	dj := j1 - j0
	dk := k1 - k0
	if di >= dj && di >= dk && di >= threshold {
		// divide in two by y axis
		mi := i0 + di/2
		done1 := make(chan struct{}, 1)
		go matmult(done1, A, B, C, i0, mi, j0, j1, k0, k1, threshold)
		matmult(nil, A, B, C, mi, i1, j0, j1, k0, k1, threshold)
		<-done1
	} else if dj >= dk && dj >= threshold {
		// divide in two by x axis
		mj := j0 + dj/2
		done1 := make(chan struct{}, 1)
		go matmult(done1, A, B, C, i0, i1, j0, mj, k0, k1, threshold)
		matmult(nil, A, B, C, i0, i1, mj, j1, k0, k1, threshold)
		<-done1
	} else if dk >= threshold {
		// divide in two by "k" axis
		// deliberately not parallel because of data races
		mk := k0 + dk/2
		matmult(nil, A, B, C, i0, i1, j0, j1, k0, mk, threshold)
		matmult(nil, A, B, C, i0, i1, j0, j1, mk, k1, threshold)
	} else {
		// the matrices are small enough, compute directly
		for i := i0; i < i1; i++ {
			for j := j0; j < j1; j++ {
				for k := k0; k < k1; k++ {
					C[i][j] += A[i][k] * B[k][j]
				}
			}
		}
	}
	if done != nil {
		done <- struct{}{}
	}
}
<commit_msg>runtime: refactor Syscall benchmark And add a benchmark where #goroutines>GOMAXPROCS, because it's the most interesting case. Current results on darwin/amd64, Intel Core 2 Duo 2.13 GHz, 2 cores: BenchmarkSyscall	100000000	        56.0 ns/op BenchmarkSyscall-2	50000000	        57.2 ns/op BenchmarkSyscallWork	10000000	       635 ns/op BenchmarkSyscallWork-2	10000000	       315 ns/op BenchmarkSyscallExcess	 1000000	      2698 ns/op BenchmarkSyscallExcess-2	 5000000	      1192 ns/op BenchmarkSyscallExcessWork	 1000000	      2832 ns/op BenchmarkSyscallExcessWork-2	 2000000	      1966 ns/op<commit_after>// Copyright 2011 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package runtime_test

import (
	"math"
	"runtime"
	"sync/atomic"
	"testing"
	"time"
)

var stop = make(chan bool, 1)

func perpetuumMobile() {
	select {
	case <-stop:
	default:
		go perpetuumMobile()
	}
}

func TestStopTheWorldDeadlock(t *testing.T) {
	if testing.Short() {
		t.Skip("skipping during short test")
	}
	maxprocs := runtime.GOMAXPROCS(3)
	compl := make(chan bool, 2)
	go func() {
		for i := 0; i != 1000; i += 1 {
			runtime.GC()
		}
		compl <- true
	}()
	go func() {
		for i := 0; i != 1000; i += 1 {
			runtime.GOMAXPROCS(3)
		}
		compl <- true
	}()
	go perpetuumMobile()
	<-compl
	<-compl
	stop <- true
	runtime.GOMAXPROCS(maxprocs)
}

func TestYieldProgress(t *testing.T) {
	testYieldProgress(t, false)
}

func TestYieldLockedProgress(t *testing.T) {
	testYieldProgress(t, true)
}

func testYieldProgress(t *testing.T, locked bool) {
	c := make(chan bool)
	cack := make(chan bool)
	go func() {
		if locked {
			runtime.LockOSThread()
		}
		for {
			select {
			case <-c:
				cack <- true
				return
			default:
				runtime.Gosched()
			}
		}
	}()
	time.Sleep(10 * time.Millisecond)
	c <- true
	<-cack
}

func TestYieldLocked(t *testing.T) {
	const N = 10
	c := make(chan bool)
	go func() {
		runtime.LockOSThread()
		for i := 0; i < N; i++ {
			runtime.Gosched()
			time.Sleep(time.Millisecond)
		}
		c <- true
		// runtime.UnlockOSThread() is deliberately omitted
	}()
	<-c
}

func TestBlockLocked(t *testing.T) {
	const N = 10
	c := make(chan bool)
	go func() {
		runtime.LockOSThread()
		for i := 0; i < N; i++ {
			c <- true
		}
		runtime.UnlockOSThread()
	}()
	for i := 0; i < N; i++ {
		<-c
	}
}

func stackGrowthRecursive(i int) {
	var pad [128]uint64
	if i != 0 && pad[0] == 0 {
		stackGrowthRecursive(i - 1)
	}
}

func TestSchedLocalQueue(t *testing.T) {
	runtime.TestSchedLocalQueue1()
}

func TestSchedLocalQueueSteal(t *testing.T) {
	runtime.TestSchedLocalQueueSteal1()
}

func benchmarkStackGrowth(b *testing.B, rec int) {
	const CallsPerSched = 1000
	procs := runtime.GOMAXPROCS(-1)
	N := int32(b.N / CallsPerSched)
	c := make(chan bool, procs)
	for p := 0; p < procs; p++ {
		go func() {
			for atomic.AddInt32(&N, -1) >= 0 {
				runtime.Gosched()
				for g := 0; g < CallsPerSched; g++ {
					stackGrowthRecursive(rec)
				}
			}
			c <- true
		}()
	}
	for p := 0; p < procs; p++ {
		<-c
	}
}

func BenchmarkStackGrowth(b *testing.B) {
	benchmarkStackGrowth(b, 10)
}

func BenchmarkStackGrowthDeep(b *testing.B) {
	benchmarkStackGrowth(b, 1024)
}

func BenchmarkSyscall(b *testing.B) {
	benchmarkSyscall(b, 0, 1)
}

func BenchmarkSyscallWork(b *testing.B) {
	benchmarkSyscall(b, 100, 1)
}

func BenchmarkSyscallExcess(b *testing.B) {
	benchmarkSyscall(b, 0, 4)
}

func BenchmarkSyscallExcessWork(b *testing.B) {
	benchmarkSyscall(b, 100, 4)
}

func benchmarkSyscall(b *testing.B, work, excess int) {
	const CallsPerSched = 1000
	procs := runtime.GOMAXPROCS(-1) * excess
	N := int32(b.N / CallsPerSched)
	c := make(chan bool, procs)
	for p := 0; p < procs; p++ {
		go func() {
			foo := 42
			for atomic.AddInt32(&N, -1) >= 0 {
				runtime.Gosched()
				for g := 0; g < CallsPerSched; g++ {
					runtime.Entersyscall()
					for i := 0; i < work; i++ {
						foo *= 2
						foo /= 2
					}
					runtime.Exitsyscall()
				}
			}
			c <- foo == 42
		}()
	}
	for p := 0; p < procs; p++ {
		<-c
	}
}

func BenchmarkCreateGoroutines(b *testing.B) {
	benchmarkCreateGoroutines(b, 1)
}

func BenchmarkCreateGoroutinesParallel(b *testing.B) {
	benchmarkCreateGoroutines(b, runtime.GOMAXPROCS(-1))
}

func benchmarkCreateGoroutines(b *testing.B, procs int) {
	c := make(chan bool)
	var f func(n int)
	f = func(n int) {
		if n == 0 {
			c <- true
			return
		}
		go f(n - 1)
	}
	for i := 0; i < procs; i++ {
		go f(b.N / procs)
	}
	for i := 0; i < procs; i++ {
		<-c
	}
}

type Matrix [][]float64

func BenchmarkMatmult(b *testing.B) {
	b.StopTimer()
	// matmult is O(N**3) but testing expects O(b.N),
	// so we need to take cube root of b.N
	n := int(math.Cbrt(float64(b.N))) + 1
	A := makeMatrix(n)
	B := makeMatrix(n)
	C := makeMatrix(n)
	b.StartTimer()
	matmult(nil, A, B, C, 0, n, 0, n, 0, n, 8)
}

func makeMatrix(n int) Matrix {
	m := make(Matrix, n)
	for i := 0; i < n; i++ {
		m[i] = make([]float64, n)
		for j := 0; j < n; j++ {
			m[i][j] = float64(i*n + j)
		}
	}
	return m
}

func matmult(done chan<- struct{}, A, B, C Matrix, i0, i1, j0, j1, k0, k1, threshold int) {
	di := i1 - i0
	dj := j1 - j0
	dk := k1 - k0
	if di >= dj && di >= dk && di >= threshold {
		// divide in two by y axis
		mi := i0 + di/2
		done1 := make(chan struct{}, 1)
		go matmult(done1, A, B, C, i0, mi, j0, j1, k0, k1, threshold)
		matmult(nil, A, B, C, mi, i1, j0, j1, k0, k1, threshold)
		<-done1
	} else if dj >= dk && dj >= threshold {
		// divide in two by x axis
		mj := j0 + dj/2
		done1 := make(chan struct{}, 1)
		go matmult(done1, A, B, C, i0, i1, j0, mj, k0, k1, threshold)
		matmult(nil, A, B, C, i0, i1, mj, j1, k0, k1, threshold)
		<-done1
	} else if dk >= threshold {
		// divide in two by "k" axis
		// deliberately not parallel because of data races
		mk := k0 + dk/2
		matmult(nil, A, B, C, i0, i1, j0, j1, k0, mk, threshold)
		matmult(nil, A, B, C, i0, i1, j0, j1, mk, k1, threshold)
	} else {
		// the matrices are small enough, compute directly
		for i := i0; i < i1; i++ {
			for j := j0; j < j1; j++ {
				for k := k0; k < k1; k++ {
					C[i][j] += A[i][k] * B[k][j]
				}
			}
		}
	}
	if done != nil {
		done <- struct{}{}
	}
}
<|endoftext|>


<commit_before>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Fork, exec, wait, etc.

package syscall

import (
	"sync"
	"unsafe"
)

// Lock synchronizing creation of new file descriptors with fork.
//
// We want the child in a fork/exec sequence to inherit only the
// file descriptors we intend.  To do that, we mark all file
// descriptors close-on-exec and then, in the child, explicitly
// unmark the ones we want the exec'ed program to keep.
// Unix doesn't make this easy: there is, in general, no way to
// allocate a new file descriptor close-on-exec.  Instead you
// have to allocate the descriptor and then mark it close-on-exec.
// If a fork happens between those two events, the child's exec
// will inherit an unwanted file descriptor.
//
// This lock solves that race: the create new fd/mark close-on-exec
// operation is done holding ForkLock for reading, and the fork itself
// is done holding ForkLock for writing.  At least, that's the idea.
// There are some complications.
//
// Some system calls that create new file descriptors can block
// for arbitrarily long times: open on a hung NFS server or named
// pipe, accept on a socket, and so on.  We can't reasonably grab
// the lock across those operations.
//
// It is worse to inherit some file descriptors than others.
// If a non-malicious child accidentally inherits an open ordinary file,
// that's not a big deal.  On the other hand, if a long-lived child
// accidentally inherits the write end of a pipe, then the reader
// of that pipe will not see EOF until that child exits, potentially
// causing the parent program to hang.  This is a common problem
// in threaded C programs that use popen.
//
// Luckily, the file descriptors that are most important not to
// inherit are not the ones that can take an arbitrarily long time
// to create: pipe returns instantly, and the net package uses
// non-blocking I/O to accept on a listening socket.
// The rules for which file descriptor-creating operations use the
// ForkLock are as follows:
//
// 1) Pipe.    Does not block.  Use the ForkLock.
// 2) Socket.  Does not block.  Use the ForkLock.
// 3) Accept.  If using non-blocking mode, use the ForkLock.
//             Otherwise, live with the race.
// 4) Open.    Can block.  Use O_CLOEXEC if available (Linux).
//             Otherwise, live with the race.
// 5) Dup.     Does not block.  Use the ForkLock.
//             On Linux, could use fcntl F_DUPFD_CLOEXEC
//             instead of the ForkLock, but only for dup(fd, -1).

var ForkLock sync.RWMutex

// Convert array of string to array
// of NUL-terminated byte pointer.
func StringArrayPtr(ss []string) []*byte {
	bb := make([]*byte, len(ss)+1)
	for i := 0; i < len(ss); i++ {
		bb[i] = StringBytePtr(ss[i])
	}
	bb[len(ss)] = nil
	return bb
}

func CloseOnExec(fd int) { fcntl(fd, F_SETFD, FD_CLOEXEC) }

func SetNonblock(fd int, nonblocking bool) (errno int) {
	flag, err := fcntl(fd, F_GETFL, 0)
	if err != 0 {
		return err
	}
	if nonblocking {
		flag |= O_NONBLOCK
	} else {
		flag &= ^O_NONBLOCK
	}
	_, err = fcntl(fd, F_SETFL, flag)
	return err
}

// Fork, dup fd onto 0..len(fd), and exec(argv0, argvv, envv) in child.
// If a dup or exec fails, write the errno int to pipe.
// (Pipe is close-on-exec so if exec succeeds, it will be closed.)
// In the child, this function must not acquire any locks, because
// they might have been locked at the time of the fork.  This means
// no rescheduling, no malloc calls, and no new stack segments.
// The calls to RawSyscall are okay because they are assembly
// functions that do not grow the stack.
func forkAndExecInChild(argv0 *byte, argv []*byte, envv []*byte, traceme bool, dir *byte, fd []int, pipe int) (pid int, err int) {
	// Declare all variables at top in case any
	// declarations require heap allocation (e.g., err1).
	var r1, r2, err1 uintptr
	var nextfd int
	var i int

	darwin := OS == "darwin"

	// About to call fork.
	// No more allocation or calls of non-assembly functions.
	r1, r2, err1 = RawSyscall(SYS_FORK, 0, 0, 0)
	if err1 != 0 {
		return 0, int(err1)
	}

	// On Darwin:
	//	r1 = child pid in both parent and child.
	//	r2 = 0 in parent, 1 in child.
	// Convert to normal Unix r1 = 0 in child.
	if darwin && r2 == 1 {
		r1 = 0
	}

	if r1 != 0 {
		// parent; return PID
		return int(r1), 0
	}

	// Fork succeeded, now in child.

	// Enable tracing if requested.
	if traceme {
		_, _, err1 = RawSyscall(SYS_PTRACE, uintptr(PTRACE_TRACEME), 0, 0)
		if err1 != 0 {
			goto childerror
		}
	}

	// Chdir
	if dir != nil {
		_, _, err1 = RawSyscall(SYS_CHDIR, uintptr(unsafe.Pointer(dir)), 0, 0)
		if err1 != 0 {
			goto childerror
		}
	}

	// Pass 1: look for fd[i] < i and move those up above len(fd)
	// so that pass 2 won't stomp on an fd it needs later.
	nextfd = int(len(fd))
	if pipe < nextfd {
		_, _, err1 = RawSyscall(SYS_DUP2, uintptr(pipe), uintptr(nextfd), 0)
		if err1 != 0 {
			goto childerror
		}
		RawSyscall(SYS_FCNTL, uintptr(nextfd), F_SETFD, FD_CLOEXEC)
		pipe = nextfd
		nextfd++
	}
	for i = 0; i < len(fd); i++ {
		if fd[i] >= 0 && fd[i] < int(i) {
			_, _, err1 = RawSyscall(SYS_DUP2, uintptr(fd[i]), uintptr(nextfd), 0)
			if err1 != 0 {
				goto childerror
			}
			RawSyscall(SYS_FCNTL, uintptr(nextfd), F_SETFD, FD_CLOEXEC)
			fd[i] = nextfd
			nextfd++
			if nextfd == pipe { // don't stomp on pipe
				nextfd++
			}
		}
	}

	// Pass 2: dup fd[i] down onto i.
	for i = 0; i < len(fd); i++ {
		if fd[i] == -1 {
			RawSyscall(SYS_CLOSE, uintptr(i), 0, 0)
			continue
		}
		if fd[i] == int(i) {
			// dup2(i, i) won't clear close-on-exec flag on Linux,
			// probably not elsewhere either.
			_, _, err1 = RawSyscall(SYS_FCNTL, uintptr(fd[i]), F_SETFD, 0)
			if err1 != 0 {
				goto childerror
			}
			continue
		}
		// The new fd is created NOT close-on-exec,
		// which is exactly what we want.
		_, _, err1 = RawSyscall(SYS_DUP2, uintptr(fd[i]), uintptr(i), 0)
		if err1 != 0 {
			goto childerror
		}
	}

	// By convention, we don't close-on-exec the fds we are
	// started with, so if len(fd) < 3, close 0, 1, 2 as needed.
	// Programs that know they inherit fds >= 3 will need
	// to set them close-on-exec.
	for i = len(fd); i < 3; i++ {
		RawSyscall(SYS_CLOSE, uintptr(i), 0, 0)
	}

	// Time to exec.
	_, _, err1 = RawSyscall(SYS_EXECVE,
		uintptr(unsafe.Pointer(argv0)),
		uintptr(unsafe.Pointer(&argv[0])),
		uintptr(unsafe.Pointer(&envv[0])))

childerror:
	// send error code on pipe
	RawSyscall(SYS_WRITE, uintptr(pipe), uintptr(unsafe.Pointer(&err1)), uintptr(unsafe.Sizeof(err1)))
	for {
		RawSyscall(SYS_EXIT, 253, 0, 0)
	}

	// Calling panic is not actually safe,
	// but the for loop above won't break
	// and this shuts up the compiler.
	panic("unreached")
}

func forkExec(argv0 string, argv []string, envv []string, traceme bool, dir string, fd []int) (pid int, err int) {
	var p [2]int
	var n int
	var err1 uintptr
	var wstatus WaitStatus

	p[0] = -1
	p[1] = -1

	// Convert args to C form.
	argv0p := StringBytePtr(argv0)
	argvp := StringArrayPtr(argv)
	envvp := StringArrayPtr(envv)
	var dirp *byte
	if len(dir) > 0 {
		dirp = StringBytePtr(dir)
	}

	// Acquire the fork lock so that no other threads
	// create new fds that are not yet close-on-exec
	// before we fork.
	ForkLock.Lock()

	// Allocate child status pipe close on exec.
	if err = Pipe(p[0:]); err != 0 {
		goto error
	}
	if _, err = fcntl(p[0], F_SETFD, FD_CLOEXEC); err != 0 {
		goto error
	}
	if _, err = fcntl(p[1], F_SETFD, FD_CLOEXEC); err != 0 {
		goto error
	}

	// Kick off child.
	pid, err = forkAndExecInChild(argv0p, argvp, envvp, traceme, dirp, fd, p[1])
	if err != 0 {
	error:
		if p[0] >= 0 {
			Close(p[0])
			Close(p[1])
		}
		ForkLock.Unlock()
		return 0, err
	}
	ForkLock.Unlock()

	// Read child error status from pipe.
	Close(p[1])
	n, err = read(p[0], (*byte)(unsafe.Pointer(&err1)), unsafe.Sizeof(err1))
	Close(p[0])
	if err != 0 || n != 0 {
		if n == unsafe.Sizeof(err1) {
			err = int(err1)
		}
		if err == 0 {
			err = EPIPE
		}

		// Child failed; wait for it to exit, to make sure
		// the zombies don't accumulate.
		_, err1 := Wait4(pid, &wstatus, 0, nil)
		for err1 == EINTR {
			_, err1 = Wait4(pid, &wstatus, 0, nil)
		}
		return 0, err
	}

	// Read got EOF, so pipe closed on exec, so exec succeeded.
	return pid, 0
}

// Combination of fork and exec, careful to be thread safe.
func ForkExec(argv0 string, argv []string, envv []string, dir string, fd []int) (pid int, err int) {
	return forkExec(argv0, argv, envv, false, dir, fd)
}

// PtraceForkExec is like ForkExec, but starts the child in a traced state.
func PtraceForkExec(argv0 string, argv []string, envv []string, dir string, fd []int) (pid int, err int) {
	return forkExec(argv0, argv, envv, true, dir, fd)
}

// Ordinary exec.
func Exec(argv0 string, argv []string, envv []string) (err int) {
	_, _, err1 := RawSyscall(SYS_EXECVE,
		uintptr(unsafe.Pointer(StringBytePtr(argv0))),
		uintptr(unsafe.Pointer(&StringArrayPtr(argv)[0])),
		uintptr(unsafe.Pointer(&StringArrayPtr(envv)[0])))
	return int(err1)
}

// StartProcess wraps ForkExec for package os.
func StartProcess(argv0 string, argv []string, envv []string, dir string, fd []int) (pid, handle int, err int) {
	pid, err = forkExec(argv0, argv, envv, false, dir, fd)
	return pid, 0, err
}
<commit_msg>syscall: work around FreeBSD execve kernel bug<commit_after>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Fork, exec, wait, etc.

package syscall

import (
	"sync"
	"unsafe"
)

// Lock synchronizing creation of new file descriptors with fork.
//
// We want the child in a fork/exec sequence to inherit only the
// file descriptors we intend.  To do that, we mark all file
// descriptors close-on-exec and then, in the child, explicitly
// unmark the ones we want the exec'ed program to keep.
// Unix doesn't make this easy: there is, in general, no way to
// allocate a new file descriptor close-on-exec.  Instead you
// have to allocate the descriptor and then mark it close-on-exec.
// If a fork happens between those two events, the child's exec
// will inherit an unwanted file descriptor.
//
// This lock solves that race: the create new fd/mark close-on-exec
// operation is done holding ForkLock for reading, and the fork itself
// is done holding ForkLock for writing.  At least, that's the idea.
// There are some complications.
//
// Some system calls that create new file descriptors can block
// for arbitrarily long times: open on a hung NFS server or named
// pipe, accept on a socket, and so on.  We can't reasonably grab
// the lock across those operations.
//
// It is worse to inherit some file descriptors than others.
// If a non-malicious child accidentally inherits an open ordinary file,
// that's not a big deal.  On the other hand, if a long-lived child
// accidentally inherits the write end of a pipe, then the reader
// of that pipe will not see EOF until that child exits, potentially
// causing the parent program to hang.  This is a common problem
// in threaded C programs that use popen.
//
// Luckily, the file descriptors that are most important not to
// inherit are not the ones that can take an arbitrarily long time
// to create: pipe returns instantly, and the net package uses
// non-blocking I/O to accept on a listening socket.
// The rules for which file descriptor-creating operations use the
// ForkLock are as follows:
//
// 1) Pipe.    Does not block.  Use the ForkLock.
// 2) Socket.  Does not block.  Use the ForkLock.
// 3) Accept.  If using non-blocking mode, use the ForkLock.
//             Otherwise, live with the race.
// 4) Open.    Can block.  Use O_CLOEXEC if available (Linux).
//             Otherwise, live with the race.
// 5) Dup.     Does not block.  Use the ForkLock.
//             On Linux, could use fcntl F_DUPFD_CLOEXEC
//             instead of the ForkLock, but only for dup(fd, -1).

var ForkLock sync.RWMutex

// Convert array of string to array
// of NUL-terminated byte pointer.
func StringArrayPtr(ss []string) []*byte {
	bb := make([]*byte, len(ss)+1)
	for i := 0; i < len(ss); i++ {
		bb[i] = StringBytePtr(ss[i])
	}
	bb[len(ss)] = nil
	return bb
}

func CloseOnExec(fd int) { fcntl(fd, F_SETFD, FD_CLOEXEC) }

func SetNonblock(fd int, nonblocking bool) (errno int) {
	flag, err := fcntl(fd, F_GETFL, 0)
	if err != 0 {
		return err
	}
	if nonblocking {
		flag |= O_NONBLOCK
	} else {
		flag &= ^O_NONBLOCK
	}
	_, err = fcntl(fd, F_SETFL, flag)
	return err
}

// Fork, dup fd onto 0..len(fd), and exec(argv0, argvv, envv) in child.
// If a dup or exec fails, write the errno int to pipe.
// (Pipe is close-on-exec so if exec succeeds, it will be closed.)
// In the child, this function must not acquire any locks, because
// they might have been locked at the time of the fork.  This means
// no rescheduling, no malloc calls, and no new stack segments.
// The calls to RawSyscall are okay because they are assembly
// functions that do not grow the stack.
func forkAndExecInChild(argv0 *byte, argv []*byte, envv []*byte, traceme bool, dir *byte, fd []int, pipe int) (pid int, err int) {
	// Declare all variables at top in case any
	// declarations require heap allocation (e.g., err1).
	var r1, r2, err1 uintptr
	var nextfd int
	var i int

	darwin := OS == "darwin"

	// About to call fork.
	// No more allocation or calls of non-assembly functions.
	r1, r2, err1 = RawSyscall(SYS_FORK, 0, 0, 0)
	if err1 != 0 {
		return 0, int(err1)
	}

	// On Darwin:
	//	r1 = child pid in both parent and child.
	//	r2 = 0 in parent, 1 in child.
	// Convert to normal Unix r1 = 0 in child.
	if darwin && r2 == 1 {
		r1 = 0
	}

	if r1 != 0 {
		// parent; return PID
		return int(r1), 0
	}

	// Fork succeeded, now in child.

	// Enable tracing if requested.
	if traceme {
		_, _, err1 = RawSyscall(SYS_PTRACE, uintptr(PTRACE_TRACEME), 0, 0)
		if err1 != 0 {
			goto childerror
		}
	}

	// Chdir
	if dir != nil {
		_, _, err1 = RawSyscall(SYS_CHDIR, uintptr(unsafe.Pointer(dir)), 0, 0)
		if err1 != 0 {
			goto childerror
		}
	}

	// Pass 1: look for fd[i] < i and move those up above len(fd)
	// so that pass 2 won't stomp on an fd it needs later.
	nextfd = int(len(fd))
	if pipe < nextfd {
		_, _, err1 = RawSyscall(SYS_DUP2, uintptr(pipe), uintptr(nextfd), 0)
		if err1 != 0 {
			goto childerror
		}
		RawSyscall(SYS_FCNTL, uintptr(nextfd), F_SETFD, FD_CLOEXEC)
		pipe = nextfd
		nextfd++
	}
	for i = 0; i < len(fd); i++ {
		if fd[i] >= 0 && fd[i] < int(i) {
			_, _, err1 = RawSyscall(SYS_DUP2, uintptr(fd[i]), uintptr(nextfd), 0)
			if err1 != 0 {
				goto childerror
			}
			RawSyscall(SYS_FCNTL, uintptr(nextfd), F_SETFD, FD_CLOEXEC)
			fd[i] = nextfd
			nextfd++
			if nextfd == pipe { // don't stomp on pipe
				nextfd++
			}
		}
	}

	// Pass 2: dup fd[i] down onto i.
	for i = 0; i < len(fd); i++ {
		if fd[i] == -1 {
			RawSyscall(SYS_CLOSE, uintptr(i), 0, 0)
			continue
		}
		if fd[i] == int(i) {
			// dup2(i, i) won't clear close-on-exec flag on Linux,
			// probably not elsewhere either.
			_, _, err1 = RawSyscall(SYS_FCNTL, uintptr(fd[i]), F_SETFD, 0)
			if err1 != 0 {
				goto childerror
			}
			continue
		}
		// The new fd is created NOT close-on-exec,
		// which is exactly what we want.
		_, _, err1 = RawSyscall(SYS_DUP2, uintptr(fd[i]), uintptr(i), 0)
		if err1 != 0 {
			goto childerror
		}
	}

	// By convention, we don't close-on-exec the fds we are
	// started with, so if len(fd) < 3, close 0, 1, 2 as needed.
	// Programs that know they inherit fds >= 3 will need
	// to set them close-on-exec.
	for i = len(fd); i < 3; i++ {
		RawSyscall(SYS_CLOSE, uintptr(i), 0, 0)
	}

	// Time to exec.
	_, _, err1 = RawSyscall(SYS_EXECVE,
		uintptr(unsafe.Pointer(argv0)),
		uintptr(unsafe.Pointer(&argv[0])),
		uintptr(unsafe.Pointer(&envv[0])))

childerror:
	// send error code on pipe
	RawSyscall(SYS_WRITE, uintptr(pipe), uintptr(unsafe.Pointer(&err1)), uintptr(unsafe.Sizeof(err1)))
	for {
		RawSyscall(SYS_EXIT, 253, 0, 0)
	}

	// Calling panic is not actually safe,
	// but the for loop above won't break
	// and this shuts up the compiler.
	panic("unreached")
}

func forkExec(argv0 string, argv []string, envv []string, traceme bool, dir string, fd []int) (pid int, err int) {
	var p [2]int
	var n int
	var err1 uintptr
	var wstatus WaitStatus

	p[0] = -1
	p[1] = -1

	// Convert args to C form.
	argv0p := StringBytePtr(argv0)
	argvp := StringArrayPtr(argv)
	envvp := StringArrayPtr(envv)
	var dirp *byte
	if len(dir) > 0 {
		dirp = StringBytePtr(dir)
	}

	if OS == "freebsd" && len(argv[0]) > len(argv0) {
		argvp[0] = argv0p
	}

	// Acquire the fork lock so that no other threads
	// create new fds that are not yet close-on-exec
	// before we fork.
	ForkLock.Lock()

	// Allocate child status pipe close on exec.
	if err = Pipe(p[0:]); err != 0 {
		goto error
	}
	if _, err = fcntl(p[0], F_SETFD, FD_CLOEXEC); err != 0 {
		goto error
	}
	if _, err = fcntl(p[1], F_SETFD, FD_CLOEXEC); err != 0 {
		goto error
	}

	// Kick off child.
	pid, err = forkAndExecInChild(argv0p, argvp, envvp, traceme, dirp, fd, p[1])
	if err != 0 {
	error:
		if p[0] >= 0 {
			Close(p[0])
			Close(p[1])
		}
		ForkLock.Unlock()
		return 0, err
	}
	ForkLock.Unlock()

	// Read child error status from pipe.
	Close(p[1])
	n, err = read(p[0], (*byte)(unsafe.Pointer(&err1)), unsafe.Sizeof(err1))
	Close(p[0])
	if err != 0 || n != 0 {
		if n == unsafe.Sizeof(err1) {
			err = int(err1)
		}
		if err == 0 {
			err = EPIPE
		}

		// Child failed; wait for it to exit, to make sure
		// the zombies don't accumulate.
		_, err1 := Wait4(pid, &wstatus, 0, nil)
		for err1 == EINTR {
			_, err1 = Wait4(pid, &wstatus, 0, nil)
		}
		return 0, err
	}

	// Read got EOF, so pipe closed on exec, so exec succeeded.
	return pid, 0
}

// Combination of fork and exec, careful to be thread safe.
func ForkExec(argv0 string, argv []string, envv []string, dir string, fd []int) (pid int, err int) {
	return forkExec(argv0, argv, envv, false, dir, fd)
}

// PtraceForkExec is like ForkExec, but starts the child in a traced state.
func PtraceForkExec(argv0 string, argv []string, envv []string, dir string, fd []int) (pid int, err int) {
	return forkExec(argv0, argv, envv, true, dir, fd)
}

// Ordinary exec.
func Exec(argv0 string, argv []string, envv []string) (err int) {
	_, _, err1 := RawSyscall(SYS_EXECVE,
		uintptr(unsafe.Pointer(StringBytePtr(argv0))),
		uintptr(unsafe.Pointer(&StringArrayPtr(argv)[0])),
		uintptr(unsafe.Pointer(&StringArrayPtr(envv)[0])))
	return int(err1)
}

// StartProcess wraps ForkExec for package os.
func StartProcess(argv0 string, argv []string, envv []string, dir string, fd []int) (pid, handle int, err int) {
	pid, err = forkExec(argv0, argv, envv, false, dir, fd)
	return pid, 0, err
}
<|endoftext|>


<commit_before>package workload

import (
	"fmt"
	"io"
	"math/rand"

	"golang.org/x/net/context"

	"github.com/pachyderm/pachyderm/src/pfs"
	"github.com/pachyderm/pachyderm/src/pfs/pfsutil"
	"github.com/pachyderm/pachyderm/src/pps"
	"github.com/pachyderm/pachyderm/src/pps/ppsutil"
)

func RunWorkload(
	pfsClient pfs.APIClient,
	ppsClient pps.APIClient,
	rand *rand.Rand,
	size int,
) error {
	worker := newWorker(rand)
	for i := 0; i < size; i++ {
		if err := worker.work(pfsClient, ppsClient); err != nil {
			return err
		}
	}
	return nil
}

type worker struct {
	repos       []*pfs.Repo
	finished    []*pfs.Commit
	started     []*pfs.Commit
	files       []*pfs.File
	startedJobs []*pps.Job
	jobs        []*pps.Job
	pipelines   []*pps.Pipeline
	rand        *rand.Rand
}

func newWorker(rand *rand.Rand) *worker {
	return &worker{
		rand: rand,
	}
}

const (
	repo     float64 = .05
	commit           = .2
	file             = .9
	job              = .98
	pipeline         = 1.0
)

const maxStartedCommits = 6
const maxStartedJobs = 6

func (w *worker) work(pfsClient pfs.APIClient, ppsClient pps.APIClient) error {
	opt := w.rand.Float64()
	switch {
	case opt < repo:
		repoName := w.name()
		if err := pfsutil.CreateRepo(pfsClient, repoName); err != nil {
			return err
		}
		w.repos = append(w.repos, &pfs.Repo{Name: repoName})
		commit, err := pfsutil.StartCommit(pfsClient, repoName, "")
		if err != nil {
			return err
		}
		w.started = append(w.started, commit)
	case opt < commit:
		if len(w.started) >= maxStartedCommits || len(w.finished) == 0 {
			if len(w.started) == 0 {
				return nil
			}
			i := w.rand.Intn(len(w.started))
			commit := w.started[i]
			if err := pfsutil.FinishCommit(pfsClient, commit.Repo.Name, commit.Id); err != nil {
				return err
			}
			w.started = append(w.started[:i], w.started[i+1:]...)
			w.finished = append(w.finished, commit)
		} else {
			if len(w.finished) == 0 {
				return nil
			}
			commit := w.finished[w.rand.Intn(len(w.finished))]
			commit, err := pfsutil.StartCommit(pfsClient, commit.Repo.Name, commit.Id)
			if err != nil {
				return err
			}
			w.started = append(w.started, commit)
		}
	case opt < file:
		if len(w.started) == 0 {
			return nil
		}
		commit := w.started[w.rand.Intn(len(w.started))]
		if _, err := pfsutil.PutFile(pfsClient, commit.Repo.Name, commit.Id, w.name(), 0, w.reader()); err != nil {
			return err
		}
	case opt < job:
		if len(w.finished) == 0 {
			return nil
		}
		if len(w.startedJobs) >= maxStartedJobs {
			job := w.startedJobs[0]
			w.startedJobs = w.startedJobs[1:]
			jobInfo, err := ppsClient.InspectJob(
				context.Background(),
				&pps.InspectJobRequest{
					Job:        job,
					BlockState: true,
				},
			)
			if err != nil {
				return err
			}
			if jobInfo.State != pps.JobState_JOB_STATE_SUCCESS {
				return fmt.Errorf("job %s failed", job.Id)
			}
			w.jobs = append(w.jobs, job)
		} else {
			inputs := [5]string{}
			var inputCommits []*pfs.Commit
			for i := range inputs {
				randI := w.rand.Intn(len(w.finished))
				inputs[i] = w.finished[randI].Repo.Name
				inputCommits = append(inputCommits, w.finished[randI])
			}
			var parentJobID string
			if len(w.jobs) > 0 {
				parentJobID = w.jobs[w.rand.Intn(len(w.jobs))].Id
			}
			outFilename := w.name()
			job, err := ppsutil.CreateJob(
				ppsClient,
				"",
				[]string{"sh"},
				w.grepCmd(inputs, outFilename),
				1,
				inputCommits,
				parentJobID,
			)
			if err != nil {
				return err
			}
			w.startedJobs = append(w.startedJobs, job)
		}
	case opt < pipeline:
		if len(w.repos) == 0 {
			return nil
		}
		inputs := [5]string{}
		var inputRepos []*pfs.Repo
		for i := range inputs {
			randI := w.rand.Intn(len(w.repos))
			inputs[i] = w.repos[randI].Name
			inputRepos = append(inputRepos, w.repos[randI])
		}
		pipelineName := w.name()
		outFilename := w.name()
		if err := ppsutil.CreatePipeline(
			ppsClient,
			pipelineName,
			"",
			[]string{"sh"},
			w.grepCmd(inputs, outFilename),
			1,
			inputRepos,
		); err != nil {
			return err
		}
		w.pipelines = append(w.pipelines, ppsutil.NewPipeline(pipelineName))
	}
	return nil
}

const letters = "abcdefghijklmnopqrstuvwxyz"
const lettersAndSpaces = "abcdefghijklmnopqrstuvwxyz      "

func (w *worker) name() string {
	b := make([]byte, 20)
	for i := range b {
		b[i] = letters[w.rand.Intn(len(letters))]
	}
	return string(b)
}

type reader struct {
	rand *rand.Rand
}

func (r *reader) Read(p []byte) (int, error) {
	for i := range p {
		if i%128 == 0 {
			p[i] = '\n'
		} else {
			p[i] = lettersAndSpaces[r.rand.Intn(len(lettersAndSpaces))]
		}
	}
	if r.rand.Intn(10) == 0 {
		return len(p), io.EOF
	}
	return len(p), nil
}

func (w *worker) reader() io.Reader {
	return &reader{w.rand}
}

func (w *worker) grepCmd(inputs [5]string, outFilename string) string {
	pattern := make([]byte, 5)
	_, _ = w.reader().Read(pattern)
	return fmt.Sprintf(
		"grep %s /pfs/{%s,%s,%s,%s,%s}/* >/pfs/out/%s",
		pattern,
		inputs[0],
		inputs[1],
		inputs[2],
		inputs[3],
		inputs[4],
		outFilename,
	)
}
<commit_msg>Fix grep job a bit.<commit_after>package workload

import (
	"fmt"
	"io"
	"log"
	"math/rand"

	"golang.org/x/net/context"

	"github.com/pachyderm/pachyderm/src/pfs"
	"github.com/pachyderm/pachyderm/src/pfs/pfsutil"
	"github.com/pachyderm/pachyderm/src/pps"
	"github.com/pachyderm/pachyderm/src/pps/ppsutil"
)

func RunWorkload(
	pfsClient pfs.APIClient,
	ppsClient pps.APIClient,
	rand *rand.Rand,
	size int,
) error {
	worker := newWorker(rand)
	for i := 0; i < size; i++ {
		if err := worker.work(pfsClient, ppsClient); err != nil {
			return err
		}
	}
	return nil
}

type worker struct {
	repos       []*pfs.Repo
	finished    []*pfs.Commit
	started     []*pfs.Commit
	files       []*pfs.File
	startedJobs []*pps.Job
	jobs        []*pps.Job
	pipelines   []*pps.Pipeline
	rand        *rand.Rand
}

func newWorker(rand *rand.Rand) *worker {
	return &worker{
		rand: rand,
	}
}

const (
	repo     float64 = .05
	commit           = .2
	file             = .9
	job              = .98
	pipeline         = 1.0
)

const maxStartedCommits = 6
const maxStartedJobs = 6

func (w *worker) work(pfsClient pfs.APIClient, ppsClient pps.APIClient) error {
	opt := w.rand.Float64()
	switch {
	case opt < repo:
		repoName := w.randString(10)
		if err := pfsutil.CreateRepo(pfsClient, repoName); err != nil {
			return err
		}
		w.repos = append(w.repos, &pfs.Repo{Name: repoName})
		commit, err := pfsutil.StartCommit(pfsClient, repoName, "")
		if err != nil {
			return err
		}
		w.started = append(w.started, commit)
	case opt < commit:
		if len(w.started) >= maxStartedCommits || len(w.finished) == 0 {
			if len(w.started) == 0 {
				return nil
			}
			i := w.rand.Intn(len(w.started))
			commit := w.started[i]
			if err := pfsutil.FinishCommit(pfsClient, commit.Repo.Name, commit.Id); err != nil {
				return err
			}
			w.started = append(w.started[:i], w.started[i+1:]...)
			w.finished = append(w.finished, commit)
		} else {
			if len(w.finished) == 0 {
				return nil
			}
			commit := w.finished[w.rand.Intn(len(w.finished))]
			commit, err := pfsutil.StartCommit(pfsClient, commit.Repo.Name, commit.Id)
			if err != nil {
				return err
			}
			w.started = append(w.started, commit)
		}
	case opt < file:
		if len(w.started) == 0 {
			return nil
		}
		commit := w.started[w.rand.Intn(len(w.started))]
		if _, err := pfsutil.PutFile(pfsClient, commit.Repo.Name, commit.Id, w.randString(10), 0, w.reader()); err != nil {
			return err
		}
	case opt < job:
		if len(w.finished) == 0 {
			return nil
		}
		if len(w.startedJobs) >= maxStartedJobs {
			job := w.startedJobs[0]
			w.startedJobs = w.startedJobs[1:]
			jobInfo, err := ppsClient.InspectJob(
				context.Background(),
				&pps.InspectJobRequest{
					Job:        job,
					BlockState: true,
				},
			)
			if err != nil {
				return err
			}
			if jobInfo.State != pps.JobState_JOB_STATE_SUCCESS {
				log.Printf("failed jobInfo: %+v", jobInfo)
				return fmt.Errorf("job %s failed", job.Id)
			}
			w.jobs = append(w.jobs, job)
		} else {
			inputs := [5]string{}
			var inputCommits []*pfs.Commit
			for i := range inputs {
				randI := w.rand.Intn(len(w.finished))
				inputs[i] = w.finished[randI].Repo.Name
				inputCommits = append(inputCommits, w.finished[randI])
			}
			var parentJobID string
			if len(w.jobs) > 0 {
				parentJobID = w.jobs[w.rand.Intn(len(w.jobs))].Id
			}
			outFilename := w.randString(10)
			job, err := ppsutil.CreateJob(
				ppsClient,
				"",
				[]string{"sh"},
				w.grepCmd(inputs, outFilename),
				1,
				inputCommits,
				parentJobID,
			)
			if err != nil {
				return err
			}
			w.startedJobs = append(w.startedJobs, job)
		}
	case opt < pipeline:
		if len(w.repos) == 0 {
			return nil
		}
		inputs := [5]string{}
		var inputRepos []*pfs.Repo
		for i := range inputs {
			randI := w.rand.Intn(len(w.repos))
			inputs[i] = w.repos[randI].Name
			inputRepos = append(inputRepos, w.repos[randI])
		}
		pipelineName := w.randString(10)
		outFilename := w.randString(10)
		if err := ppsutil.CreatePipeline(
			ppsClient,
			pipelineName,
			"",
			[]string{"sh"},
			w.grepCmd(inputs, outFilename),
			1,
			inputRepos,
		); err != nil {
			return err
		}
		w.pipelines = append(w.pipelines, ppsutil.NewPipeline(pipelineName))
	}
	return nil
}

const letters = "abcdefghijklmnopqrstuvwxyz"
const lettersAndSpaces = "abcdefghijklmnopqrstuvwxyz      "

func (w *worker) randString(n int) string {
	b := make([]byte, n)
	for i := range b {
		b[i] = letters[w.rand.Intn(len(letters))]
	}
	return string(b)
}

type reader struct {
	rand *rand.Rand
}

func (r *reader) Read(p []byte) (int, error) {
	for i := range p {
		if i%128 == 127 {
			p[i] = '\n'
		} else {
			p[i] = lettersAndSpaces[r.rand.Intn(len(lettersAndSpaces))]
		}
	}
	if r.rand.Intn(10) == 0 {
		return len(p), io.EOF
	}
	return len(p), nil
}

func (w *worker) reader() io.Reader {
	return &reader{w.rand}
}

func (w *worker) grepCmd(inputs [5]string, outFilename string) string {
	return fmt.Sprintf(
		"grep %s /pfs/{%s,%s,%s,%s,%s}/* >/pfs/out/%s",
		w.randString(4),
		inputs[0],
		inputs[1],
		inputs[2],
		inputs[3],
		inputs[4],
		outFilename,
	)
}
<|endoftext|>


<commit_before>// Copyright 2020 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"log"

	cmd "github.com/apigee/apigeecli/cmd"

	"github.com/spf13/cobra/doc"
)

func main() {
	err := doc.GenMarkdownTree(cmd.RootCmd, "./docs")
	if err != nil {
		log.Fatal(err)
	}
}
<commit_msg>clean up docs docs before generate #49<commit_after>// Copyright 2020 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"log"
	"os"
	"path/filepath"

	cmd "github.com/apigee/apigeecli/cmd"

	"github.com/spf13/cobra/doc"
)

func main() {
	var err error
	var docFiles []string

	if docFiles, err = filepath.Glob("./docs/apigeecli*.md"); err != nil {
		log.Fatal(err)
	}

	for _, docFile := range docFiles {
		if err = os.Remove(docFile); err != nil {
			log.Fatal(err)
		}
	}

	if err = doc.GenMarkdownTree(cmd.RootCmd, "./docs"); err != nil {
		log.Fatal(err)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"log"
	"os"
	"regexp"
	"strconv"
	"strings"

	"github.com/andrew-d/go-termutil"
	"github.com/mgutz/ansi"
)

type tFuncAppend func([]string, string, string) []string
type tFuncFilter func(string) bool

type Lltsv struct {
	keys        []string
	no_key      bool
	filters     []string
	exprs       []string
	funcAppend  tFuncAppend
	funcFilters map[string]tFuncFilter
	exprRunners map[string]*ExprRunner
}

func newLltsv(keys []string, no_key bool, filters []string, exprs []string) *Lltsv {
	return &Lltsv{
		keys:        keys,
		no_key:      no_key,
		filters:     filters,
		exprs:       exprs,
		funcAppend:  getFuncAppend(no_key),
		funcFilters: getFuncFilters(filters),
		exprRunners: getExprRunners(exprs),
	}
}

func (lltsv *Lltsv) scanAndWrite(file *os.File) error {
	scanner := bufio.NewScanner(file)
	for scanner.Scan() {
		line := scanner.Text()
		lvs := lltsv.parseLtsv(line)
		lltsv.expr(lvs)

		if lltsv.filter(lvs) {
			ltsv := lltsv.restructLtsv(lvs)
			os.Stdout.WriteString(ltsv + "\n")
		}

	}
	return scanner.Err()
}

func (lltsv *Lltsv) filter(lvs map[string]string) bool {
	should_output := true

	for key, funcFilter := range lltsv.funcFilters {
		if !funcFilter(lvs[key]) {
			should_output = false
			break
		}
	}

	return should_output
}

func (lltsv *Lltsv) expr(lvs map[string]string) {
	ctx := &ExprContext{
		vars: lvs,
	}
	for key, runner := range lltsv.exprRunners {
		v, err := evalExpr(runner.expr, ctx)
		if err == nil {
			lvs[key] = v.String()
		}
	}
}

// lvs: label and value pairs
func (lltsv *Lltsv) restructLtsv(lvs map[string]string) string {
	// specified keys or all keys
	orders := lltsv.keys
	if len(lltsv.keys) == 0 {
		orders = keysInMap(lvs)
	}
	// make slice with enough capacity so that append does not newly create object
	// cf. http://golang.org/pkg/builtin/#append
	selected := make([]string, 0, len(orders))
	for _, label := range orders {
		value := lvs[label]
		selected = lltsv.funcAppend(selected, label, value)
	}
	return strings.Join(selected, "\t")
}

func (lltsv *Lltsv) parseLtsv(line string) map[string]string {
	columns := strings.Split(line, "\t")
	lvs := make(map[string]string)
	for _, column := range columns {
		l_v := strings.SplitN(column, ":", 2)
		if len(l_v) < 2 {
			continue
		}
		label, value := l_v[0], l_v[1]
		lvs[label] = value
	}
	return lvs
}

// Return function pointer to avoid `if` evaluation occurs in each iteration
func getFuncAppend(no_key bool) tFuncAppend {
	if no_key {
		return func(selected []string, label string, value string) []string {
			return append(selected, value)
		}
	} else {
		if termutil.Isatty(os.Stdout.Fd()) {
			return func(selected []string, label string, value string) []string {
				return append(selected, ansi.Color(label, "green")+":"+ansi.Color(value, "magenta"))
			}
		} else {
			// if pipe or redirect
			return func(selected []string, label string, value string) []string {
				return append(selected, label+":"+value)
			}
		}
	}
}

func getFuncFilters(filters []string) map[string]tFuncFilter {
	funcFilters := map[string]tFuncFilter{}
	for _, f := range filters {
		token := strings.SplitN(f, " ", 3)
		if len(token) < 3 {
			log.Fatalf("filter expression is invalid: %s\n", f)
		}
		key := token[0]
		switch token[1] {
		case ">", ">=", "<=", "<":
			r, err := strconv.ParseFloat(token[2], 64)
			if err != nil {
				log.Fatal(err)
			}

			funcFilters[key] = func(val string) bool {
				num, err := strconv.ParseFloat(val, 64)
				if err != nil {
					log.Println(err)
					return false
				}
				switch token[1] {
				case ">":
					return num > r
				case ">=":
					return num >= r
				case "<=":
					return num <= r
				case "<":
					return num < r
				default:
					return false
				}
			}
		case "==":
			funcFilters[key] = func(val string) bool {
				return val == token[2]
			}
		case "==*":
			funcFilters[key] = func(val string) bool {
				return strings.ToLower(val) == strings.ToLower(token[2])
			}
		case "!=":
			funcFilters[key] = func(val string) bool {
				return val != token[2]
			}
		case "!=*":
			funcFilters[key] = func(val string) bool {
				return strings.ToLower(val) != strings.ToLower(token[2])
			}
		case "=~", "!~", "=~*", "!~*":
			if token[1] == "=~*" || token[1] == "!~*" {
				token[2] = strings.ToLower(token[2])
			}
			re := regexp.MustCompile(token[2])
			funcFilters[key] = func(val string) bool {
				switch token[1] {
				case "=~":
					return re.MatchString(val)
				case "!~":
					return !re.MatchString(val)
				case "=~*":
					return re.MatchString(strings.ToLower(val))
				case "!~*":
					return !re.MatchString(strings.ToLower(val))
				default:
					return false
				}
			}
		}
	}
	return funcFilters
}

func getExprRunners(exprs []string) map[string]*ExprRunner {
	funcExprs := make(map[string]*ExprRunner, len(exprs))
	for _, f := range exprs {
		token := strings.SplitN(f, "=", 2)
		if len(token) != 2 {
			log.Printf("expression is invalid: %s\n", f)
			continue
		}

		expr, err := parseExpr(token[1])
		if err != nil {
			log.Printf("expression is invalid: %s\n", f)
			continue
		}

		key := strings.Trim(token[0], " ")

		funcExprs[key] = &ExprRunner{
			expr: expr,
		}
	}
	return funcExprs
}

func keysInMap(m map[string]string) []string {
	keys := make([]string, 0, len(m))
	for key := range m {
		keys = append(keys, key)
	}
	return keys
}
<commit_msg>sort keys by default.<commit_after>package main

import (
	"bufio"
	"log"
	"os"
	"regexp"
	"strconv"
	"strings"

	"github.com/andrew-d/go-termutil"
	"github.com/mgutz/ansi"
)

type tFuncAppend func([]string, string, string) []string
type tFuncFilter func(string) bool

type Lltsv struct {
	keys        []string
	no_key      bool
	filters     []string
	exprs       []string
	funcAppend  tFuncAppend
	funcFilters map[string]tFuncFilter
	exprRunners map[string]*ExprRunner
}

func newLltsv(keys []string, no_key bool, filters []string, exprs []string) *Lltsv {
	return &Lltsv{
		keys:        keys,
		no_key:      no_key,
		filters:     filters,
		exprs:       exprs,
		funcAppend:  getFuncAppend(no_key),
		funcFilters: getFuncFilters(filters),
		exprRunners: getExprRunners(exprs),
	}
}

func (lltsv *Lltsv) scanAndWrite(file *os.File) error {
	scanner := bufio.NewScanner(file)
	for scanner.Scan() {
		line := scanner.Text()
		lvs, keys := lltsv.parseLtsv(line)
		lltsv.expr(lvs)

		if lltsv.filter(lvs) {
			ltsv := lltsv.restructLtsv(lvs, keys)
			os.Stdout.WriteString(ltsv + "\n")
		}

	}
	return scanner.Err()
}

func (lltsv *Lltsv) filter(lvs map[string]string) bool {
	should_output := true

	for key, funcFilter := range lltsv.funcFilters {
		if !funcFilter(lvs[key]) {
			should_output = false
			break
		}
	}

	return should_output
}

func (lltsv *Lltsv) expr(lvs map[string]string) {
	ctx := &ExprContext{
		vars: lvs,
	}
	for key, runner := range lltsv.exprRunners {
		v, err := evalExpr(runner.expr, ctx)
		if err == nil {
			lvs[key] = v.String()
		}
	}
}

// lvs: label and value pairs
func (lltsv *Lltsv) restructLtsv(lvs map[string]string, keys []string) string {
	// specified keys or all keys
	orders := lltsv.keys
	if len(lltsv.keys) == 0 {
		orders = keys
	}
	// make slice with enough capacity so that append does not newly create object
	// cf. http://golang.org/pkg/builtin/#append
	selected := make([]string, 0, len(orders))
	for _, label := range orders {
		value := lvs[label]
		selected = lltsv.funcAppend(selected, label, value)
	}
	return strings.Join(selected, "\t")
}

func (lltsv *Lltsv) parseLtsv(line string) (map[string]string, []string) {
	columns := strings.Split(line, "\t")
	lvs := make(map[string]string)
	keys := make([]string, 0, len(columns))
	for _, column := range columns {
		l_v := strings.SplitN(column, ":", 2)
		if len(l_v) < 2 {
			continue
		}
		label, value := l_v[0], l_v[1]
		lvs[label] = value
		keys = append(keys, label)
	}
	return lvs, keys
}

// Return function pointer to avoid `if` evaluation occurs in each iteration
func getFuncAppend(no_key bool) tFuncAppend {
	if no_key {
		return func(selected []string, label string, value string) []string {
			return append(selected, value)
		}
	} else {
		if termutil.Isatty(os.Stdout.Fd()) {
			return func(selected []string, label string, value string) []string {
				return append(selected, ansi.Color(label, "green")+":"+ansi.Color(value, "magenta"))
			}
		} else {
			// if pipe or redirect
			return func(selected []string, label string, value string) []string {
				return append(selected, label+":"+value)
			}
		}
	}
}

func getFuncFilters(filters []string) map[string]tFuncFilter {
	funcFilters := map[string]tFuncFilter{}
	for _, f := range filters {
		token := strings.SplitN(f, " ", 3)
		if len(token) < 3 {
			log.Fatalf("filter expression is invalid: %s\n", f)
		}
		key := token[0]
		switch token[1] {
		case ">", ">=", "<=", "<":
			r, err := strconv.ParseFloat(token[2], 64)
			if err != nil {
				log.Fatal(err)
			}

			funcFilters[key] = func(val string) bool {
				num, err := strconv.ParseFloat(val, 64)
				if err != nil {
					log.Println(err)
					return false
				}
				switch token[1] {
				case ">":
					return num > r
				case ">=":
					return num >= r
				case "<=":
					return num <= r
				case "<":
					return num < r
				default:
					return false
				}
			}
		case "==":
			funcFilters[key] = func(val string) bool {
				return val == token[2]
			}
		case "==*":
			funcFilters[key] = func(val string) bool {
				return strings.ToLower(val) == strings.ToLower(token[2])
			}
		case "!=":
			funcFilters[key] = func(val string) bool {
				return val != token[2]
			}
		case "!=*":
			funcFilters[key] = func(val string) bool {
				return strings.ToLower(val) != strings.ToLower(token[2])
			}
		case "=~", "!~", "=~*", "!~*":
			if token[1] == "=~*" || token[1] == "!~*" {
				token[2] = strings.ToLower(token[2])
			}
			re := regexp.MustCompile(token[2])
			funcFilters[key] = func(val string) bool {
				switch token[1] {
				case "=~":
					return re.MatchString(val)
				case "!~":
					return !re.MatchString(val)
				case "=~*":
					return re.MatchString(strings.ToLower(val))
				case "!~*":
					return !re.MatchString(strings.ToLower(val))
				default:
					return false
				}
			}
		}
	}
	return funcFilters
}

func getExprRunners(exprs []string) map[string]*ExprRunner {
	funcExprs := make(map[string]*ExprRunner, len(exprs))
	for _, f := range exprs {
		token := strings.SplitN(f, "=", 2)
		if len(token) != 2 {
			log.Printf("expression is invalid: %s\n", f)
			continue
		}

		expr, err := parseExpr(token[1])
		if err != nil {
			log.Printf("expression is invalid: %s\n", f)
			continue
		}

		key := strings.Trim(token[0], " ")

		funcExprs[key] = &ExprRunner{
			expr: expr,
		}
	}
	return funcExprs
}
<|endoftext|>


<commit_before>package cfutil

import (
	"encoding/json"
	"fmt"
	log "github.com/Sirupsen/logrus"
	"github.com/jeffail/gabs"
	"github.com/satori/go.uuid"
	"os"
	"strconv"
	"strings"
	"time"
)

type vcapApplication struct {
	ApplicationName    string    `json:"application_name"`
	ApplicationVersion string    `json:"application_version"`
	ApplicationUris    []string  `json:"application_uris"`
	Host               string    `json:"host"`
	Name               string    `json:"name"`
	InstanceId         string    `json:"instance_id"`
	InstanceIndex      int       `json:"instance_index"`
	Port               int       `json:"port"`
	Start              time.Time `json:"start"`
	StartedAt          time.Time `json:"started_at"`
	StartedTimestamp   int64     `json:"started_timestamp"`
	Uris               []string  `json:"uris"`
	Users              *[]string `json:"users"`
	Version            string    `json:"version"`
}

func LocalVcapApplication() string {
	appId := uuid.NewV4().String()
	port := 8080
	host := "localhost"
	if p, err := strconv.Atoi(os.Getenv("PORT")); err == nil {
		port = p
	}
	hostWithPort := fmt.Sprintf("%s:%d", host, port)

	va := &vcapApplication{
		ApplicationName:    "appname",
		ApplicationVersion: appId,
		Host:               "0.0.0.0",
		Port:               port,
		ApplicationUris:    []string{hostWithPort},
		InstanceId:         "451f045fd16427bb99c895a2649b7b2a",
		InstanceIndex:      0,
		Name:               "appname",
		Start:              time.Now(),
		StartedAt:          time.Now(),
		StartedTimestamp:   time.Now().Unix(),
		Uris:               []string{hostWithPort},
		Version:            appId,
	}
	json, _ := json.Marshal(va)
	return string(json)
}

func LocalMemoryLimit() string {
	return "2G"
}

func LocalVcapServices() string {
	var supportedServices = []string{
		"postgres",
		"smtp",
		"rabbitmq",
	}
	jsonObj := gabs.New()
	for _, service := range supportedServices {
		env := "CF_LOCAL_" + strings.ToUpper(service)
		uri := os.Getenv(env)
		if uri != "" {
			log.Printf("Adding service '%s'", service)
			jsonObj.Array(service)
			serviceJson := gabs.New()
			serviceJson.Set(uri, "credentials", "uri")
			serviceJson.Set(service, "name")
			log.Print("Local service: ", service)
			jsonObj.ArrayAppendP(serviceJson.Data(), service)
		}
	}
	return jsonObj.String()
}
<commit_msg>Allow naming a local service<commit_after>package cfutil

import (
	"encoding/json"
	"fmt"
	log "github.com/Sirupsen/logrus"
	"github.com/jeffail/gabs"
	"github.com/satori/go.uuid"
	"os"
	"strconv"
	"strings"
	"time"
)

type vcapApplication struct {
	ApplicationName    string    `json:"application_name"`
	ApplicationVersion string    `json:"application_version"`
	ApplicationUris    []string  `json:"application_uris"`
	Host               string    `json:"host"`
	Name               string    `json:"name"`
	InstanceId         string    `json:"instance_id"`
	InstanceIndex      int       `json:"instance_index"`
	Port               int       `json:"port"`
	Start              time.Time `json:"start"`
	StartedAt          time.Time `json:"started_at"`
	StartedTimestamp   int64     `json:"started_timestamp"`
	Uris               []string  `json:"uris"`
	Users              *[]string `json:"users"`
	Version            string    `json:"version"`
}

func LocalVcapApplication() string {
	appId := uuid.NewV4().String()
	port := 8080
	host := "localhost"
	if p, err := strconv.Atoi(os.Getenv("PORT")); err == nil {
		port = p
	}
	hostWithPort := fmt.Sprintf("%s:%d", host, port)

	va := &vcapApplication{
		ApplicationName:    "appname",
		ApplicationVersion: appId,
		Host:               "0.0.0.0",
		Port:               port,
		ApplicationUris:    []string{hostWithPort},
		InstanceId:         "451f045fd16427bb99c895a2649b7b2a",
		InstanceIndex:      0,
		Name:               "appname",
		Start:              time.Now(),
		StartedAt:          time.Now(),
		StartedTimestamp:   time.Now().Unix(),
		Uris:               []string{hostWithPort},
		Version:            appId,
	}
	json, _ := json.Marshal(va)
	return string(json)
}

func LocalMemoryLimit() string {
	return "2G"
}

func LocalVcapServices() string {
	var supportedServices = []string{
		"postgres",
		"smtp",
		"rabbitmq",
	}
	jsonObj := gabs.New()
	for _, service := range supportedServices {
		env := "CF_LOCAL_" + strings.ToUpper(service)
		uri := os.Getenv(env)
		if uri != "" {
			jsonObj.Array(service)
			serviceJson := gabs.New()
			name := service
			if components := strings.Split(uri, ","); len(components) > 1 {
				name = components[0]
				uri = components[1]
			}
			serviceJson.Set(name, "name")
			serviceJson.Set(uri, "credentials", "uri")
			log.Print("Local service: ", name)
			jsonObj.ArrayAppendP(serviceJson.Data(), service)
		}
	}
	return jsonObj.String()
}
<|endoftext|>