<commit_before>// Copyright 2016 The Gogs Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package models

import (
	"fmt"
	"strings"

	"github.com/Unknwon/com"
	"github.com/gogits/git-module"

	"github.com/gogits/gogs/pkg/tool"
)

type Branch struct {
	RepoPath string
	Name     string

	IsProtected bool
	Commit      *git.Commit
}

func GetBranchesByPath(path string) ([]*Branch, error) {
	gitRepo, err := git.OpenRepository(path)
	if err != nil {
		return nil, err
	}

	brs, err := gitRepo.GetBranches()
	if err != nil {
		return nil, err
	}

	branches := make([]*Branch, len(brs))
	for i := range brs {
		branches[i] = &Branch{
			RepoPath: path,
			Name:     brs[i],
		}
	}
	return branches, nil
}

func (repo *Repository) GetBranch(br string) (*Branch, error) {
	if !git.IsBranchExist(repo.RepoPath(), br) {
		return nil, ErrBranchNotExist{br}
	}
	return &Branch{
		RepoPath: repo.RepoPath(),
		Name:     br,
	}, nil
}

func (repo *Repository) GetBranches() ([]*Branch, error) {
	return GetBranchesByPath(repo.RepoPath())
}

func (br *Branch) GetCommit() (*git.Commit, error) {
	gitRepo, err := git.OpenRepository(br.RepoPath)
	if err != nil {
		return nil, err
	}
	return gitRepo.GetBranchCommit(br.Name)
}

type ProtectBranchWhitelist struct {
	ID              int64
	ProtectBranchID int64
	RepoID          int64  `xorm:"UNIQUE(protect_branch_whitelist)"`
	Name            string `xorm:"UNIQUE(protect_branch_whitelist)"`
	UserID          int64  `xorm:"UNIQUE(protect_branch_whitelist)"`
}

// IsUserInProtectBranchWhitelist returns true if given user is in the whitelist of a branch in a repository.
func IsUserInProtectBranchWhitelist(repoID, userID int64, branch string) bool {
	has, err := x.Where("repo_id = ?", repoID).And("user_id = ?", userID).And("name = ?", branch).Get(new(ProtectBranchWhitelist))
	return has && err == nil
}

// ProtectBranch contains options of a protected branch.
type ProtectBranch struct {
	ID                 int64
	RepoID             int64  `xorm:"UNIQUE(protect_branch)"`
	Name               string `xorm:"UNIQUE(protect_branch)"`
	Protected          bool
	RequirePullRequest bool
	EnableWhitelist    bool
	WhitelistUserIDs   string `xorm:"TEXT"`
	WhitelistTeamIDs   string `xorm:"TEXT"`
}

// GetProtectBranchOfRepoByName returns *ProtectBranch by branch name in given repostiory.
func GetProtectBranchOfRepoByName(repoID int64, name string) (*ProtectBranch, error) {
	protectBranch := &ProtectBranch{
		RepoID: repoID,
		Name:   name,
	}
	has, err := x.Get(protectBranch)
	if err != nil {
		return nil, err
	} else if !has {
		return nil, ErrBranchNotExist{name}
	}
	return protectBranch, nil
}

// IsBranchOfRepoRequirePullRequest returns true if branch requires pull request in given repository.
func IsBranchOfRepoRequirePullRequest(repoID int64, name string) bool {
	protectBranch, err := GetProtectBranchOfRepoByName(repoID, name)
	if err != nil {
		return false
	}
	return protectBranch.Protected && protectBranch.RequirePullRequest
}

// UpdateProtectBranch saves branch protection options.
// If ID is 0, it creates a new record. Otherwise, updates existing record.
func UpdateProtectBranch(protectBranch *ProtectBranch) (err error) {
	sess := x.NewSession()
	defer sess.Close()
	if err = sess.Begin(); err != nil {
		return err
	}

	if protectBranch.ID == 0 {
		if _, err = sess.Insert(protectBranch); err != nil {
			return fmt.Errorf("Insert: %v", err)
		}
	}

	if _, err = sess.Id(protectBranch.ID).AllCols().Update(protectBranch); err != nil {
		return fmt.Errorf("Update: %v", err)
	}

	return sess.Commit()
}

// UpdateOrgProtectBranch saves branch protection options of organizational repository.
// If ID is 0, it creates a new record. Otherwise, updates existing record.
// This function also performs check if whitelist user and team's IDs have been changed
// to avoid unnecessary whitelist delete and regenerate.
func UpdateOrgProtectBranch(repo *Repository, protectBranch *ProtectBranch, whitelistUserIDs, whitelistTeamIDs string) (err error) {
	if err = repo.GetOwner(); err != nil {
		return fmt.Errorf("GetOwner: %v", err)
	} else if !repo.Owner.IsOrganization() {
		return fmt.Errorf("expect repository owner to be an organization")
	}

	hasUsersChanged := false
	validUserIDs := tool.StringsToInt64s(strings.Split(protectBranch.WhitelistUserIDs, ","))
	if protectBranch.WhitelistUserIDs != whitelistUserIDs {
		hasUsersChanged = true
		userIDs := tool.StringsToInt64s(strings.Split(whitelistUserIDs, ","))
		validUserIDs = make([]int64, 0, len(userIDs))
		for _, userID := range userIDs {
			has, err := HasAccess(userID, repo, ACCESS_MODE_WRITE)
			if err != nil {
				return fmt.Errorf("HasAccess [user_id: %d, repo_id: %d]: %v", userID, protectBranch.RepoID, err)
			} else if !has {
				continue // Drop invalid user ID
			}

			validUserIDs = append(validUserIDs, userID)
		}

		protectBranch.WhitelistUserIDs = strings.Join(tool.Int64sToStrings(validUserIDs), ",")
	}

	hasTeamsChanged := false
	validTeamIDs := tool.StringsToInt64s(strings.Split(protectBranch.WhitelistTeamIDs, ","))
	if protectBranch.WhitelistTeamIDs != whitelistTeamIDs {
		hasTeamsChanged = true
		teamIDs := tool.StringsToInt64s(strings.Split(whitelistTeamIDs, ","))
		teams, err := GetTeamsHaveAccessToRepo(repo.OwnerID, repo.ID, ACCESS_MODE_WRITE)
		if err != nil {
			return fmt.Errorf("GetTeamsHaveAccessToRepo [org_id: %d, repo_id: %d]: %v", repo.OwnerID, repo.ID, err)
		}
		validTeamIDs = make([]int64, 0, len(teams))
		for i := range teams {
			if teams[i].HasWriteAccess() && com.IsSliceContainsInt64(teamIDs, teams[i].ID) {
				validTeamIDs = append(validTeamIDs, teams[i].ID)
			}
		}

		protectBranch.WhitelistTeamIDs = strings.Join(tool.Int64sToStrings(validTeamIDs), ",")
	}

	// Make sure protectBranch.ID is not 0 for whitelists
	if protectBranch.ID == 0 {
		if _, err = x.Insert(protectBranch); err != nil {
			return fmt.Errorf("Insert: %v", err)
		}
	}

	// Merge users and members of teams
	var whitelists []*ProtectBranchWhitelist
	if hasUsersChanged || hasTeamsChanged {
		mergedUserIDs := make(map[int64]bool)
		for _, userID := range validUserIDs {
			// Empty whitelist users can cause an ID with 0
			if userID != 0 {
				mergedUserIDs[userID] = true
			}
		}

		for _, teamID := range validTeamIDs {
			members, err := GetTeamMembers(teamID)
			if err != nil {
				return fmt.Errorf("GetTeamMembers [team_id: %d]: %v", teamID, err)
			}

			for i := range members {
				mergedUserIDs[members[i].ID] = true
			}
		}

		whitelists = make([]*ProtectBranchWhitelist, 0, len(mergedUserIDs))
		for userID := range mergedUserIDs {
			whitelists = append(whitelists, &ProtectBranchWhitelist{
				ProtectBranchID: protectBranch.ID,
				RepoID:          repo.ID,
				Name:            protectBranch.Name,
				UserID:          userID,
			})
		}
	}

	sess := x.NewSession()
	defer sess.Close()
	if err = sess.Begin(); err != nil {
		return err
	}

	if _, err = sess.Id(protectBranch.ID).AllCols().Update(protectBranch); err != nil {
		return fmt.Errorf("Update: %v", err)
	}

	// Refresh whitelists
	if hasUsersChanged || hasTeamsChanged {
		if _, err = sess.Delete(&ProtectBranchWhitelist{ProtectBranchID: protectBranch.ID}); err != nil {
			return fmt.Errorf("delete old protect branch whitelists: %v", err)
		} else if _, err = sess.Insert(whitelists); err != nil {
			return fmt.Errorf("insert new protect branch whitelists: %v", err)
		}
	}

	return sess.Commit()
}

// GetProtectBranchesByRepoID returns a list of *ProtectBranch in given repostiory.
func GetProtectBranchesByRepoID(repoID int64) ([]*ProtectBranch, error) {
	protectBranches := make([]*ProtectBranch, 0, 2)
	return protectBranches, x.Where("repo_id = ?", repoID).Asc("name").Find(&protectBranches)
}
<commit_msg>models: bug fix for query protected branches (#5054)<commit_after>// Copyright 2016 The Gogs Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package models

import (
	"fmt"
	"strings"

	"github.com/Unknwon/com"
	"github.com/gogits/git-module"

	"github.com/gogits/gogs/pkg/tool"
)

type Branch struct {
	RepoPath string
	Name     string

	IsProtected bool
	Commit      *git.Commit
}

func GetBranchesByPath(path string) ([]*Branch, error) {
	gitRepo, err := git.OpenRepository(path)
	if err != nil {
		return nil, err
	}

	brs, err := gitRepo.GetBranches()
	if err != nil {
		return nil, err
	}

	branches := make([]*Branch, len(brs))
	for i := range brs {
		branches[i] = &Branch{
			RepoPath: path,
			Name:     brs[i],
		}
	}
	return branches, nil
}

func (repo *Repository) GetBranch(br string) (*Branch, error) {
	if !git.IsBranchExist(repo.RepoPath(), br) {
		return nil, ErrBranchNotExist{br}
	}
	return &Branch{
		RepoPath: repo.RepoPath(),
		Name:     br,
	}, nil
}

func (repo *Repository) GetBranches() ([]*Branch, error) {
	return GetBranchesByPath(repo.RepoPath())
}

func (br *Branch) GetCommit() (*git.Commit, error) {
	gitRepo, err := git.OpenRepository(br.RepoPath)
	if err != nil {
		return nil, err
	}
	return gitRepo.GetBranchCommit(br.Name)
}

type ProtectBranchWhitelist struct {
	ID              int64
	ProtectBranchID int64
	RepoID          int64  `xorm:"UNIQUE(protect_branch_whitelist)"`
	Name            string `xorm:"UNIQUE(protect_branch_whitelist)"`
	UserID          int64  `xorm:"UNIQUE(protect_branch_whitelist)"`
}

// IsUserInProtectBranchWhitelist returns true if given user is in the whitelist of a branch in a repository.
func IsUserInProtectBranchWhitelist(repoID, userID int64, branch string) bool {
	has, err := x.Where("repo_id = ?", repoID).And("user_id = ?", userID).And("name = ?", branch).Get(new(ProtectBranchWhitelist))
	return has && err == nil
}

// ProtectBranch contains options of a protected branch.
type ProtectBranch struct {
	ID                 int64
	RepoID             int64  `xorm:"UNIQUE(protect_branch)"`
	Name               string `xorm:"UNIQUE(protect_branch)"`
	Protected          bool
	RequirePullRequest bool
	EnableWhitelist    bool
	WhitelistUserIDs   string `xorm:"TEXT"`
	WhitelistTeamIDs   string `xorm:"TEXT"`
}

// GetProtectBranchOfRepoByName returns *ProtectBranch by branch name in given repostiory.
func GetProtectBranchOfRepoByName(repoID int64, name string) (*ProtectBranch, error) {
	protectBranch := &ProtectBranch{
		RepoID: repoID,
		Name:   name,
	}
	has, err := x.Get(protectBranch)
	if err != nil {
		return nil, err
	} else if !has {
		return nil, ErrBranchNotExist{name}
	}
	return protectBranch, nil
}

// IsBranchOfRepoRequirePullRequest returns true if branch requires pull request in given repository.
func IsBranchOfRepoRequirePullRequest(repoID int64, name string) bool {
	protectBranch, err := GetProtectBranchOfRepoByName(repoID, name)
	if err != nil {
		return false
	}
	return protectBranch.Protected && protectBranch.RequirePullRequest
}

// UpdateProtectBranch saves branch protection options.
// If ID is 0, it creates a new record. Otherwise, updates existing record.
func UpdateProtectBranch(protectBranch *ProtectBranch) (err error) {
	sess := x.NewSession()
	defer sess.Close()
	if err = sess.Begin(); err != nil {
		return err
	}

	if protectBranch.ID == 0 {
		if _, err = sess.Insert(protectBranch); err != nil {
			return fmt.Errorf("Insert: %v", err)
		}
	}

	if _, err = sess.Id(protectBranch.ID).AllCols().Update(protectBranch); err != nil {
		return fmt.Errorf("Update: %v", err)
	}

	return sess.Commit()
}

// UpdateOrgProtectBranch saves branch protection options of organizational repository.
// If ID is 0, it creates a new record. Otherwise, updates existing record.
// This function also performs check if whitelist user and team's IDs have been changed
// to avoid unnecessary whitelist delete and regenerate.
func UpdateOrgProtectBranch(repo *Repository, protectBranch *ProtectBranch, whitelistUserIDs, whitelistTeamIDs string) (err error) {
	if err = repo.GetOwner(); err != nil {
		return fmt.Errorf("GetOwner: %v", err)
	} else if !repo.Owner.IsOrganization() {
		return fmt.Errorf("expect repository owner to be an organization")
	}

	hasUsersChanged := false
	validUserIDs := tool.StringsToInt64s(strings.Split(protectBranch.WhitelistUserIDs, ","))
	if protectBranch.WhitelistUserIDs != whitelistUserIDs {
		hasUsersChanged = true
		userIDs := tool.StringsToInt64s(strings.Split(whitelistUserIDs, ","))
		validUserIDs = make([]int64, 0, len(userIDs))
		for _, userID := range userIDs {
			has, err := HasAccess(userID, repo, ACCESS_MODE_WRITE)
			if err != nil {
				return fmt.Errorf("HasAccess [user_id: %d, repo_id: %d]: %v", userID, protectBranch.RepoID, err)
			} else if !has {
				continue // Drop invalid user ID
			}

			validUserIDs = append(validUserIDs, userID)
		}

		protectBranch.WhitelistUserIDs = strings.Join(tool.Int64sToStrings(validUserIDs), ",")
	}

	hasTeamsChanged := false
	validTeamIDs := tool.StringsToInt64s(strings.Split(protectBranch.WhitelistTeamIDs, ","))
	if protectBranch.WhitelistTeamIDs != whitelistTeamIDs {
		hasTeamsChanged = true
		teamIDs := tool.StringsToInt64s(strings.Split(whitelistTeamIDs, ","))
		teams, err := GetTeamsHaveAccessToRepo(repo.OwnerID, repo.ID, ACCESS_MODE_WRITE)
		if err != nil {
			return fmt.Errorf("GetTeamsHaveAccessToRepo [org_id: %d, repo_id: %d]: %v", repo.OwnerID, repo.ID, err)
		}
		validTeamIDs = make([]int64, 0, len(teams))
		for i := range teams {
			if teams[i].HasWriteAccess() && com.IsSliceContainsInt64(teamIDs, teams[i].ID) {
				validTeamIDs = append(validTeamIDs, teams[i].ID)
			}
		}

		protectBranch.WhitelistTeamIDs = strings.Join(tool.Int64sToStrings(validTeamIDs), ",")
	}

	// Make sure protectBranch.ID is not 0 for whitelists
	if protectBranch.ID == 0 {
		if _, err = x.Insert(protectBranch); err != nil {
			return fmt.Errorf("Insert: %v", err)
		}
	}

	// Merge users and members of teams
	var whitelists []*ProtectBranchWhitelist
	if hasUsersChanged || hasTeamsChanged {
		mergedUserIDs := make(map[int64]bool)
		for _, userID := range validUserIDs {
			// Empty whitelist users can cause an ID with 0
			if userID != 0 {
				mergedUserIDs[userID] = true
			}
		}

		for _, teamID := range validTeamIDs {
			members, err := GetTeamMembers(teamID)
			if err != nil {
				return fmt.Errorf("GetTeamMembers [team_id: %d]: %v", teamID, err)
			}

			for i := range members {
				mergedUserIDs[members[i].ID] = true
			}
		}

		whitelists = make([]*ProtectBranchWhitelist, 0, len(mergedUserIDs))
		for userID := range mergedUserIDs {
			whitelists = append(whitelists, &ProtectBranchWhitelist{
				ProtectBranchID: protectBranch.ID,
				RepoID:          repo.ID,
				Name:            protectBranch.Name,
				UserID:          userID,
			})
		}
	}

	sess := x.NewSession()
	defer sess.Close()
	if err = sess.Begin(); err != nil {
		return err
	}

	if _, err = sess.Id(protectBranch.ID).AllCols().Update(protectBranch); err != nil {
		return fmt.Errorf("Update: %v", err)
	}

	// Refresh whitelists
	if hasUsersChanged || hasTeamsChanged {
		if _, err = sess.Delete(&ProtectBranchWhitelist{ProtectBranchID: protectBranch.ID}); err != nil {
			return fmt.Errorf("delete old protect branch whitelists: %v", err)
		} else if _, err = sess.Insert(whitelists); err != nil {
			return fmt.Errorf("insert new protect branch whitelists: %v", err)
		}
	}

	return sess.Commit()
}

// GetProtectBranchesByRepoID returns a list of *ProtectBranch in given repostiory.
func GetProtectBranchesByRepoID(repoID int64) ([]*ProtectBranch, error) {
	protectBranches := make([]*ProtectBranch, 0, 2)
	return protectBranches, x.Where("repo_id = ? and protected=true", repoID).Asc("name").Find(&protectBranches)
}
<|endoftext|>


<commit_before>package termite

import (
	"fmt"
	"github.com/hanwen/go-fuse/fuse"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"strings"
	"sync"
)

var _ = fmt.Println

// TODO - we could remember md5s for files outside the writable root
// so we don't read stuff under /usr for every master startup.

type FsServer struct {
	contentServer *ContentServer
	contentCache  *ContentCache
	Root          string
	excluded      map[string]bool

	hashCacheMutex sync.RWMutex
	// TODO - should use string (immutable) throughout for storing MD5 signatures.
	hashCache map[string][]byte
}

func NewFsServer(root string, cache *ContentCache, excluded []string) *FsServer {
	fs := &FsServer{
		contentCache:  cache,
		contentServer: &ContentServer{Cache: cache},
		Root:          root,
		hashCache:     make(map[string][]byte),
	}

	fs.excluded = make(map[string]bool)
	for _, e := range excluded {
		fs.excluded[e] = true
	}
	return fs
}

type AttrRequest struct {
	Name string
}

type AttrResponse struct {
	Path string // Used in WorkReply
	*os.FileInfo
	fuse.Status
	Hash    []byte
	Link    string
	Content []byte // optional.
}

func (me AttrResponse) String() string {
	id := ""
	if me.Hash != nil {
		id = fmt.Sprintf(" sz %d", me.FileInfo.Size)
	}
	if me.Link != "" {
		id = fmt.Sprintf(" -> %s", me.Link)
	}
	if me.Deletion() {
		id = " (del)"
	}
	return fmt.Sprintf("%s%s", me.Path, id)
}

func (me AttrResponse) Deletion() bool {
	return me.Status == fuse.ENOENT
}

type DirRequest struct {
	Name string
}

type DirResponse struct {
	NameModeMap map[string]uint32
}

func (me *FsServer) path(n string) string {
	if me.Root == "" {
		return n
	}
	return filepath.Join(me.Root, strings.TrimLeft(n, "/"))
}

func (me *FsServer) FileContent(req *ContentRequest, rep *ContentResponse) os.Error {
	return me.contentServer.FileContent(req, rep)
}

func (me *FsServer) ReadDir(req *DirRequest, r *DirResponse) os.Error {
	d, e := ioutil.ReadDir(me.path(req.Name))
	log.Println("ReadDir", req)
	r.NameModeMap = make(map[string]uint32)
	for _, v := range d {
		r.NameModeMap[v.Name] = v.Mode
	}
	return e
}

func (me *FsServer) GetAttr(req *AttrRequest, rep *AttrResponse) os.Error {
	log.Println("GetAttr req", req.Name)
	// TODO - this is not a good security measure, as we are not
	// checking the prefix; someone might directly ask for
	// /forbidden/subdir/
	if me.excluded[req.Name] {
		rep.Status = fuse.ENOENT
		return nil
	}

	fi, err := os.Lstat(me.path(req.Name))
	rep.FileInfo = fi
	rep.Status = fuse.OsErrorToErrno(err)
	if fi == nil {
		return nil
	}

	if fi.IsSymlink() {
		rep.Link, err = os.Readlink(req.Name)
	}
	if fi.IsRegular() {
		rep.Hash, rep.Content = me.getHash(req.Name)
	}
	log.Println("GetAttr", req.Name, rep)
	return nil
}

func (me *FsServer) updateHashes(infos []AttrResponse) {
	me.hashCacheMutex.Lock()
	defer me.hashCacheMutex.Unlock()

	for _, r := range infos {
		name := r.Path
		if !r.Status.Ok() || r.Link != "" {
			me.hashCache[name] = nil, false
		}
		if r.Hash != nil {
			me.hashCache[name] = r.Hash
		}
	}
}

func (me *FsServer) getHash(name string) (hash []byte, content []byte) {
	fullPath := me.path(name)

	me.hashCacheMutex.RLock()
	hash = me.hashCache[name]
	me.hashCacheMutex.RUnlock()

	if hash != nil {
		return []byte(hash), nil
	}

	me.hashCacheMutex.Lock()
	defer me.hashCacheMutex.Unlock()
	hash = me.hashCache[name]
	if hash != nil {
		return []byte(hash), nil
	}

	// TODO - would it be better to not stop other hash lookups
	// from succeeding?

	// TODO - only read the file; in many cases, we already have
	// the file int the on-disk md5 cache.
	hash, content = me.contentCache.SavePath(fullPath)
	me.hashCache[name] = hash
	return hash, content
}
<commit_msg>Drop TODO about remembering md5s for files under /usr. Reading those is less expensive than a normal compile, so it is not worth the headache of more state.<commit_after>package termite

import (
	"fmt"
	"github.com/hanwen/go-fuse/fuse"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"strings"
	"sync"
)

var _ = fmt.Println

type FsServer struct {
	contentServer *ContentServer
	contentCache  *ContentCache
	Root          string
	excluded      map[string]bool

	hashCacheMutex sync.RWMutex
	// TODO - should use string (immutable) throughout for storing MD5 signatures.
	hashCache map[string][]byte
}

func NewFsServer(root string, cache *ContentCache, excluded []string) *FsServer {
	fs := &FsServer{
		contentCache:  cache,
		contentServer: &ContentServer{Cache: cache},
		Root:          root,
		hashCache:     make(map[string][]byte),
	}

	fs.excluded = make(map[string]bool)
	for _, e := range excluded {
		fs.excluded[e] = true
	}
	return fs
}

type AttrRequest struct {
	Name string
}

type AttrResponse struct {
	Path string // Used in WorkReply
	*os.FileInfo
	fuse.Status
	Hash    []byte
	Link    string
	Content []byte // optional.
}

func (me AttrResponse) String() string {
	id := ""
	if me.Hash != nil {
		id = fmt.Sprintf(" sz %d", me.FileInfo.Size)
	}
	if me.Link != "" {
		id = fmt.Sprintf(" -> %s", me.Link)
	}
	if me.Deletion() {
		id = " (del)"
	}
	return fmt.Sprintf("%s%s", me.Path, id)
}

func (me AttrResponse) Deletion() bool {
	return me.Status == fuse.ENOENT
}

type DirRequest struct {
	Name string
}

type DirResponse struct {
	NameModeMap map[string]uint32
}

func (me *FsServer) path(n string) string {
	if me.Root == "" {
		return n
	}
	return filepath.Join(me.Root, strings.TrimLeft(n, "/"))
}

func (me *FsServer) FileContent(req *ContentRequest, rep *ContentResponse) os.Error {
	return me.contentServer.FileContent(req, rep)
}

func (me *FsServer) ReadDir(req *DirRequest, r *DirResponse) os.Error {
	d, e := ioutil.ReadDir(me.path(req.Name))
	log.Println("ReadDir", req)
	r.NameModeMap = make(map[string]uint32)
	for _, v := range d {
		r.NameModeMap[v.Name] = v.Mode
	}
	return e
}

func (me *FsServer) GetAttr(req *AttrRequest, rep *AttrResponse) os.Error {
	log.Println("GetAttr req", req.Name)
	// TODO - this is not a good security measure, as we are not
	// checking the prefix; someone might directly ask for
	// /forbidden/subdir/
	if me.excluded[req.Name] {
		rep.Status = fuse.ENOENT
		return nil
	}

	fi, err := os.Lstat(me.path(req.Name))
	rep.FileInfo = fi
	rep.Status = fuse.OsErrorToErrno(err)
	if fi == nil {
		return nil
	}

	if fi.IsSymlink() {
		rep.Link, err = os.Readlink(req.Name)
	}
	if fi.IsRegular() {
		rep.Hash, rep.Content = me.getHash(req.Name)
	}
	log.Println("GetAttr", req.Name, rep)
	return nil
}

func (me *FsServer) updateHashes(infos []AttrResponse) {
	me.hashCacheMutex.Lock()
	defer me.hashCacheMutex.Unlock()

	for _, r := range infos {
		name := r.Path
		if !r.Status.Ok() || r.Link != "" {
			me.hashCache[name] = nil, false
		}
		if r.Hash != nil {
			me.hashCache[name] = r.Hash
		}
	}
}

func (me *FsServer) getHash(name string) (hash []byte, content []byte) {
	fullPath := me.path(name)

	me.hashCacheMutex.RLock()
	hash = me.hashCache[name]
	me.hashCacheMutex.RUnlock()

	if hash != nil {
		return []byte(hash), nil
	}

	me.hashCacheMutex.Lock()
	defer me.hashCacheMutex.Unlock()
	hash = me.hashCache[name]
	if hash != nil {
		return []byte(hash), nil
	}

	// TODO - would it be better to not stop other hash lookups
	// from succeeding?

	// TODO - only read the file; in many cases, we already have
	// the file int the on-disk md5 cache.
	hash, content = me.contentCache.SavePath(fullPath)
	me.hashCache[name] = hash
	return hash, content
}
<|endoftext|>


<commit_before>package orm

import "gopkg.in/mgo.v2/bson"

import "github.com/lfq7413/tomato/utils"
import "regexp"
import "strings"
import "sort"
import "encoding/base64"

// transformKey 把 key 转换为数据库中保存的格式
func transformKey(schema *Schema, className, key string) string {
	k, _ := transformKeyValue(schema, className, key, nil, nil)
	return k
}

// transformKeyValue 把传入的键值对转换为数据库中保存的格式
func transformKeyValue(schema *Schema, className, restKey string, restValue interface{}, options bson.M) (string, interface{}) {
	if options == nil {
		options = bson.M{}
	}

	// 检测 key 是否为 内置字段
	key := restKey
	timeField := false
	switch key {
	case "objectId", "_id":
		key = "_id"
	case "createdAt", "_created_at":
		key = "_created_at"
		timeField = true
	case "updatedAt", "_updated_at":
		key = "_updated_at"
		timeField = true
	case "_email_verify_token":
		key = "_email_verify_token"
	case "_perishable_token":
		key = "_perishable_token"
	case "sessionToken", "_session_token":
		key = "_session_token"
	case "expiresAt", "_expiresAt":
		key = "_expiresAt"
		timeField = true
	case "_rperm", "_wperm":
		return key, restValue
	case "$or":
		if options["query"] == nil {
			// TODO 只有查询时才能使用 or
			return "", nil
		}
		querys := utils.SliceInterface(restValue)
		if querys == nil {
			// TODO 待转换值必须为数组类型
			return "", nil
		}
		mongoSubqueries := []interface{}{}
		for _, v := range querys {
			query := transformWhere(schema, className, utils.MapInterface(v))
			mongoSubqueries = append(mongoSubqueries, query)
		}
		return "$or", mongoSubqueries
	case "$and":
		if options["query"] == nil {
			// TODO 只有查询时才能使用 and
			return "", nil
		}
		querys := utils.SliceInterface(restValue)
		if querys == nil {
			// TODO 待转换值必须为数组类型
			return "", nil
		}
		mongoSubqueries := []interface{}{}
		for _, v := range querys {
			query := transformWhere(schema, className, utils.MapInterface(v))
			mongoSubqueries = append(mongoSubqueries, query)
		}
		return "$and", mongoSubqueries
	default:
		// 处理第三方 auth 数据
		authDataMatch, _ := regexp.MatchString(`^authData\.([a-zA-Z0-9_]+)\.id$`, key)
		if authDataMatch {
			if options["query"] != nil {
				provider := key[len("authData."):(len(key) - len(".id"))]
				return "_auth_data_" + provider + ".id", restKey
			}
			// TODO 只能将其应用查询操作
			return "", nil
		}
		// 默认处理
		if options["validate"] != nil {
			keyMatch, _ := regexp.MatchString(`^[a-zA-Z][a-zA-Z0-9_\.]*$`, key)
			if keyMatch == false {
				// TODO 无效的键名
				return "", nil
			}
		}
	}

	// 处理特殊键值
	expected := ""
	if schema != nil {
		expected = schema.getExpectedType(className, key)
	}
	// 处理指向其他对象的字段
	if expected != "" && strings.HasPrefix(expected, "*") {
		key = "_p_" + key
	}
	if expected == "" && restValue != nil {
		op := utils.MapInterface(restValue)
		if op != nil && op["__type"] != nil {
			if utils.String(op["__type"]) == "Pointer" {
				key = "_p_" + key
			}
		}
	}

	inArray := false
	if expected == "array" {
		inArray = true
	}
	// 处理查询操作
	if options["query"] != nil {
		value := transformConstraint(restValue, inArray)
		if value != cannotTransform() {
			return key, value
		}
	}
	if inArray && options["query"] != nil && utils.SliceInterface(restValue) == nil {
		return key, bson.M{"$all": []interface{}{restValue}}
	}

	// 处理原子数据
	value := transformAtom(restValue, false, options)
	if value != cannotTransform() {
		if timeField && utils.String(value) != "" {
			value, _ = utils.StringtoTime(utils.String(value))
		}
		return key, value
	}

	// ACL 在此之前处理，如果依然出现，则返回错误
	if key == "ACL" {
		// TODO 不能在此转换 ACL
		return "", nil
	}

	// 处理数组类型
	if valueArray, ok := restValue.([]interface{}); ok {
		if options["query"] != nil {
			// TODO 查询时不能为数组
			return "", nil
		}
		outValue := []interface{}{}
		for _, restObj := range valueArray {
			_, v := transformKeyValue(schema, className, restKey, restObj, bson.M{"inArray": true})
			outValue = append(outValue, v)
		}
		return key, outValue
	}

	// 处理更新操作
	var flatten bool
	if options["update"] == nil {
		flatten = true
	} else {
		flatten = false
	}
	value = transformUpdateOperator(restValue, flatten)
	if value != cannotTransform() {
		return key, value
	}

	// 处理正常的对象
	normalValue := bson.M{}
	for subRestKey, subRestValue := range utils.MapInterface(restValue) {
		k, v := transformKeyValue(schema, className, subRestKey, subRestValue, bson.M{"inObject": true})
		normalValue[k] = v
	}
	return key, normalValue
}

// transformConstraint 转换查询限制条件
func transformConstraint(constraint interface{}, inArray bool) interface{} {
	// TODO 需要根据 MongoDB 文档修正参数
	if constraint == nil && utils.MapInterface(constraint) == nil {
		return cannotTransform()
	}

	// keys is the constraints in reverse alphabetical order.
	// This is a hack so that:
	//   $regex is handled before $options
	//   $nearSphere is handled before $maxDistance
	object := utils.MapInterface(constraint)
	keys := []string{}
	for k := range object {
		keys = append(keys, k)
	}
	sort.Sort(sort.Reverse(sort.StringSlice(keys)))
	answer := bson.M{}

	for _, key := range keys {
		switch key {
		case "$lt", "$lte", "$gt", "$gte", "$exists", "$ne", "$eq":
			answer[key] = transformAtom(object[key], true, bson.M{"inArray": inArray})

		case "$in", "$nin":
			arr := utils.SliceInterface(object[key])
			if arr == nil {
				// TODO 必须为数组
				return nil
			}
			answerArr := []interface{}{}
			for _, v := range arr {
				answerArr = append(answerArr, transformAtom(v, true, bson.M{}))
			}
			answer[key] = answerArr

		case "$all":
			arr := utils.SliceInterface(object[key])
			if arr == nil {
				// TODO 必须为数组
				return nil
			}
			answerArr := []interface{}{}
			for _, v := range arr {
				answerArr = append(answerArr, transformAtom(v, true, bson.M{"inArray": true}))
			}
			answer[key] = answerArr

		case "$regex":
			s := utils.String(object[key])
			if s == "" {
				// TODO 必须为字符串
				return nil
			}
			answer[key] = s

		case "$options":
			options := utils.String(object[key])
			if answer["$regex"] == nil || options == "" {
				// TODO 无效值
				return nil
			}
			b, _ := regexp.MatchString(`^[imxs]+$`, options)
			if b == false {
				// TODO 无效值
				return nil
			}
			answer[key] = options

		case "$nearSphere":
			point := utils.MapInterface(object[key])
			answer[key] = []interface{}{point["longitude"], point["latitude"]}

		case "$maxDistance":
			answer[key] = object[key]

			// 以下三项在 SDK 中未使用，但是在 REST API 中使用了
		case "$maxDistanceInRadians":
			answer["$maxDistance"] = object[key]
		case "$maxDistanceInMiles":
			var distance float64
			if v, ok := object[key].(float64); ok {
				distance = v / 3959
			}
			answer["$maxDistance"] = distance
		case "$maxDistanceInKilometers":
			var distance float64
			if v, ok := object[key].(float64); ok {
				distance = v / 6371
			}
			answer["$maxDistance"] = distance

		case "$select", "$dontSelect":
			// TODO 暂时不支持该参数
			return nil

		case "$within":
			within := utils.MapInterface(object[key])
			box := utils.SliceInterface(within["$box"])
			if box == nil || len(box) != 2 {
				// TODO 参数不正确
				return nil
			}
			box1 := utils.MapInterface(box[0])
			box2 := utils.MapInterface(box[1])
			answer[key] = bson.M{
				"$box": []interface{}{
					[]interface{}{box1["longitude"], box1["latitude"]},
					[]interface{}{box2["longitude"], box2["latitude"]},
				},
			}

		default:
			b, _ := regexp.MatchString(`^\$+`, key)
			if b {
				// TODO 无效参数
				return nil
			}
			return cannotTransform()
		}
	}

	return answer
}

func transformAtom(atom interface{}, force bool, options bson.M) interface{} {
	// TODO 处理错误
	if options == nil {
		options = bson.M{}
	}
	inArray := false
	inObject := false
	if v, ok := options["inArray"].(bool); ok {
		inArray = v
	}
	if v, ok := options["inObject"].(bool); ok {
		inObject = v
	}

	if _, ok := atom.(string); ok {
		return atom
	}
	if _, ok := atom.(float64); ok {
		return atom
	}
	if _, ok := atom.(bool); ok {
		return atom
	}

	if object, ok := atom.(map[string]interface{}); ok {
		if atom == nil || len(object) == 0 {
			return atom
		}

		if utils.String(object["__type"]) == "Pointer" {
			if inArray == false && inObject == false {
				return utils.String(object["className"]) + "$" + utils.String(object["objectId"])
			}
			return bson.M{
				"__type":    "Pointer",
				"className": object["className"],
				"objectId":  object["objectId"],
			}
		}

		d := dateCoder{}
		if d.isValidJSON(object) {
			return d.jsonToDatabase(object)
		}
		b := bytesCoder{}
		if b.isValidJSON(object) {
			return b.jsonToDatabase(object)
		}
		g := geoPointCoder{}
		if g.isValidJSON(object) {
			if inArray || inObject {
				return object
			}
			return g.jsonToDatabase(object)
		}
		f := fileCoder{}
		if f.isValidJSON(object) {
			if inArray || inObject {
				return object
			}
			return f.jsonToDatabase(object)
		}

		if force {
			// TODO 无效类型
			return nil
		}
		return cannotTransform()
	}

	return cannotTransform()
}

func transformUpdateOperator(operator interface{}, flatten bool) interface{} {
	// TODO
	return nil
}

// transformCreate ...
func transformCreate(schema *Schema, className string, create bson.M) bson.M {
	// TODO
	return nil
}

func transformWhere(schema *Schema, className string, where bson.M) bson.M {
	// TODO
	return nil
}

func transformUpdate(schema *Schema, className string, update bson.M) bson.M {
	// TODO
	return nil
}

func untransformObjectT(schema *Schema, className string, mongoObject interface{}, isNestedObject bool) interface{} {
	// TODO
	return nil
}

func cannotTransform() interface{} {
	return nil
}

type dateCoder struct{}

func (d dateCoder) jsonToDatabase(json bson.M) interface{} {
	t, _ := utils.StringtoTime(utils.String(json["iso"]))
	return t
}

func (d dateCoder) isValidJSON(value bson.M) bool {
	return value != nil && utils.String(value["__type"]) == "Date"
}

type bytesCoder struct{}

func (b bytesCoder) databaseToJSON(object interface{}) interface{} {
	return nil
}

func (b bytesCoder) isValidDatabaseObject(object interface{}) bool {
	return false
}

func (b bytesCoder) jsonToDatabase(json bson.M) interface{} {
	by, _ := base64.StdEncoding.DecodeString(utils.String(json["base64"]))
	return by
}

func (b bytesCoder) isValidJSON(value bson.M) bool {
	return value != nil && utils.String(value["__type"]) == "Bytes"
}

type geoPointCoder struct{}

func (g geoPointCoder) databaseToJSON(object interface{}) interface{} {
	return nil
}

func (g geoPointCoder) isValidDatabaseObject(object interface{}) bool {
	return false
}

func (g geoPointCoder) jsonToDatabase(json bson.M) interface{} {
	return []interface{}{json["longitude"], json["latitude"]}
}

func (g geoPointCoder) isValidJSON(value bson.M) bool {
	return value != nil && utils.String(value["__type"]) == "GeoPoint"
}

type fileCoder struct{}

func (f fileCoder) databaseToJSON(object interface{}) interface{} {
	return nil
}

func (f fileCoder) isValidDatabaseObject(object interface{}) bool {
	return false
}

func (f fileCoder) jsonToDatabase(json bson.M) interface{} {
	return json["name"]
}

func (f fileCoder) isValidJSON(value bson.M) bool {
	return value != nil && utils.String(value["__type"]) == "File"
}
<commit_msg>完成 transformUpdateOperator<commit_after>package orm

import "gopkg.in/mgo.v2/bson"

import "github.com/lfq7413/tomato/utils"
import "regexp"
import "strings"
import "sort"
import "encoding/base64"

// transformKey 把 key 转换为数据库中保存的格式
func transformKey(schema *Schema, className, key string) string {
	k, _ := transformKeyValue(schema, className, key, nil, nil)
	return k
}

// transformKeyValue 把传入的键值对转换为数据库中保存的格式
func transformKeyValue(schema *Schema, className, restKey string, restValue interface{}, options bson.M) (string, interface{}) {
	if options == nil {
		options = bson.M{}
	}

	// 检测 key 是否为 内置字段
	key := restKey
	timeField := false
	switch key {
	case "objectId", "_id":
		key = "_id"
	case "createdAt", "_created_at":
		key = "_created_at"
		timeField = true
	case "updatedAt", "_updated_at":
		key = "_updated_at"
		timeField = true
	case "_email_verify_token":
		key = "_email_verify_token"
	case "_perishable_token":
		key = "_perishable_token"
	case "sessionToken", "_session_token":
		key = "_session_token"
	case "expiresAt", "_expiresAt":
		key = "_expiresAt"
		timeField = true
	case "_rperm", "_wperm":
		return key, restValue
	case "$or":
		if options["query"] == nil {
			// TODO 只有查询时才能使用 or
			return "", nil
		}
		querys := utils.SliceInterface(restValue)
		if querys == nil {
			// TODO 待转换值必须为数组类型
			return "", nil
		}
		mongoSubqueries := []interface{}{}
		for _, v := range querys {
			query := transformWhere(schema, className, utils.MapInterface(v))
			mongoSubqueries = append(mongoSubqueries, query)
		}
		return "$or", mongoSubqueries
	case "$and":
		if options["query"] == nil {
			// TODO 只有查询时才能使用 and
			return "", nil
		}
		querys := utils.SliceInterface(restValue)
		if querys == nil {
			// TODO 待转换值必须为数组类型
			return "", nil
		}
		mongoSubqueries := []interface{}{}
		for _, v := range querys {
			query := transformWhere(schema, className, utils.MapInterface(v))
			mongoSubqueries = append(mongoSubqueries, query)
		}
		return "$and", mongoSubqueries
	default:
		// 处理第三方 auth 数据
		authDataMatch, _ := regexp.MatchString(`^authData\.([a-zA-Z0-9_]+)\.id$`, key)
		if authDataMatch {
			if options["query"] != nil {
				provider := key[len("authData."):(len(key) - len(".id"))]
				return "_auth_data_" + provider + ".id", restKey
			}
			// TODO 只能将其应用查询操作
			return "", nil
		}
		// 默认处理
		if options["validate"] != nil {
			keyMatch, _ := regexp.MatchString(`^[a-zA-Z][a-zA-Z0-9_\.]*$`, key)
			if keyMatch == false {
				// TODO 无效的键名
				return "", nil
			}
		}
	}

	// 处理特殊键值
	expected := ""
	if schema != nil {
		expected = schema.getExpectedType(className, key)
	}
	// 处理指向其他对象的字段
	if expected != "" && strings.HasPrefix(expected, "*") {
		key = "_p_" + key
	}
	if expected == "" && restValue != nil {
		op := utils.MapInterface(restValue)
		if op != nil && op["__type"] != nil {
			if utils.String(op["__type"]) == "Pointer" {
				key = "_p_" + key
			}
		}
	}

	inArray := false
	if expected == "array" {
		inArray = true
	}
	// 处理查询操作
	if options["query"] != nil {
		value := transformConstraint(restValue, inArray)
		if value != cannotTransform() {
			return key, value
		}
	}
	if inArray && options["query"] != nil && utils.SliceInterface(restValue) == nil {
		return key, bson.M{"$all": []interface{}{restValue}}
	}

	// 处理原子数据
	value := transformAtom(restValue, false, options)
	if value != cannotTransform() {
		if timeField && utils.String(value) != "" {
			value, _ = utils.StringtoTime(utils.String(value))
		}
		return key, value
	}

	// ACL 在此之前处理，如果依然出现，则返回错误
	if key == "ACL" {
		// TODO 不能在此转换 ACL
		return "", nil
	}

	// 处理数组类型
	if valueArray, ok := restValue.([]interface{}); ok {
		if options["query"] != nil {
			// TODO 查询时不能为数组
			return "", nil
		}
		outValue := []interface{}{}
		for _, restObj := range valueArray {
			_, v := transformKeyValue(schema, className, restKey, restObj, bson.M{"inArray": true})
			outValue = append(outValue, v)
		}
		return key, outValue
	}

	// 处理更新操作
	var flatten bool
	if options["update"] == nil {
		flatten = true
	} else {
		flatten = false
	}
	value = transformUpdateOperator(restValue, flatten)
	if value != cannotTransform() {
		return key, value
	}

	// 处理正常的对象
	normalValue := bson.M{}
	for subRestKey, subRestValue := range utils.MapInterface(restValue) {
		k, v := transformKeyValue(schema, className, subRestKey, subRestValue, bson.M{"inObject": true})
		normalValue[k] = v
	}
	return key, normalValue
}

// transformConstraint 转换查询限制条件
func transformConstraint(constraint interface{}, inArray bool) interface{} {
	// TODO 需要根据 MongoDB 文档修正参数
	if constraint == nil && utils.MapInterface(constraint) == nil {
		return cannotTransform()
	}

	// keys is the constraints in reverse alphabetical order.
	// This is a hack so that:
	//   $regex is handled before $options
	//   $nearSphere is handled before $maxDistance
	object := utils.MapInterface(constraint)
	keys := []string{}
	for k := range object {
		keys = append(keys, k)
	}
	sort.Sort(sort.Reverse(sort.StringSlice(keys)))
	answer := bson.M{}

	for _, key := range keys {
		switch key {
		case "$lt", "$lte", "$gt", "$gte", "$exists", "$ne", "$eq":
			answer[key] = transformAtom(object[key], true, bson.M{"inArray": inArray})

		case "$in", "$nin":
			arr := utils.SliceInterface(object[key])
			if arr == nil {
				// TODO 必须为数组
				return nil
			}
			answerArr := []interface{}{}
			for _, v := range arr {
				answerArr = append(answerArr, transformAtom(v, true, bson.M{}))
			}
			answer[key] = answerArr

		case "$all":
			arr := utils.SliceInterface(object[key])
			if arr == nil {
				// TODO 必须为数组
				return nil
			}
			answerArr := []interface{}{}
			for _, v := range arr {
				answerArr = append(answerArr, transformAtom(v, true, bson.M{"inArray": true}))
			}
			answer[key] = answerArr

		case "$regex":
			s := utils.String(object[key])
			if s == "" {
				// TODO 必须为字符串
				return nil
			}
			answer[key] = s

		case "$options":
			options := utils.String(object[key])
			if answer["$regex"] == nil || options == "" {
				// TODO 无效值
				return nil
			}
			b, _ := regexp.MatchString(`^[imxs]+$`, options)
			if b == false {
				// TODO 无效值
				return nil
			}
			answer[key] = options

		case "$nearSphere":
			point := utils.MapInterface(object[key])
			answer[key] = []interface{}{point["longitude"], point["latitude"]}

		case "$maxDistance":
			answer[key] = object[key]

			// 以下三项在 SDK 中未使用，但是在 REST API 中使用了
		case "$maxDistanceInRadians":
			answer["$maxDistance"] = object[key]
		case "$maxDistanceInMiles":
			var distance float64
			if v, ok := object[key].(float64); ok {
				distance = v / 3959
			}
			answer["$maxDistance"] = distance
		case "$maxDistanceInKilometers":
			var distance float64
			if v, ok := object[key].(float64); ok {
				distance = v / 6371
			}
			answer["$maxDistance"] = distance

		case "$select", "$dontSelect":
			// TODO 暂时不支持该参数
			return nil

		case "$within":
			within := utils.MapInterface(object[key])
			box := utils.SliceInterface(within["$box"])
			if box == nil || len(box) != 2 {
				// TODO 参数不正确
				return nil
			}
			box1 := utils.MapInterface(box[0])
			box2 := utils.MapInterface(box[1])
			answer[key] = bson.M{
				"$box": []interface{}{
					[]interface{}{box1["longitude"], box1["latitude"]},
					[]interface{}{box2["longitude"], box2["latitude"]},
				},
			}

		default:
			b, _ := regexp.MatchString(`^\$+`, key)
			if b {
				// TODO 无效参数
				return nil
			}
			return cannotTransform()
		}
	}

	return answer
}

func transformAtom(atom interface{}, force bool, options bson.M) interface{} {
	// TODO 处理错误
	if options == nil {
		options = bson.M{}
	}
	inArray := false
	inObject := false
	if v, ok := options["inArray"].(bool); ok {
		inArray = v
	}
	if v, ok := options["inObject"].(bool); ok {
		inObject = v
	}

	if _, ok := atom.(string); ok {
		return atom
	}
	if _, ok := atom.(float64); ok {
		return atom
	}
	if _, ok := atom.(bool); ok {
		return atom
	}

	if object, ok := atom.(map[string]interface{}); ok {
		if atom == nil || len(object) == 0 {
			return atom
		}

		if utils.String(object["__type"]) == "Pointer" {
			if inArray == false && inObject == false {
				return utils.String(object["className"]) + "$" + utils.String(object["objectId"])
			}
			return bson.M{
				"__type":    "Pointer",
				"className": object["className"],
				"objectId":  object["objectId"],
			}
		}

		d := dateCoder{}
		if d.isValidJSON(object) {
			return d.jsonToDatabase(object)
		}
		b := bytesCoder{}
		if b.isValidJSON(object) {
			return b.jsonToDatabase(object)
		}
		g := geoPointCoder{}
		if g.isValidJSON(object) {
			if inArray || inObject {
				return object
			}
			return g.jsonToDatabase(object)
		}
		f := fileCoder{}
		if f.isValidJSON(object) {
			if inArray || inObject {
				return object
			}
			return f.jsonToDatabase(object)
		}

		if force {
			// TODO 无效类型
			return nil
		}
		return cannotTransform()
	}

	return cannotTransform()
}

func transformUpdateOperator(operator interface{}, flatten bool) interface{} {
	// TODO 处理错误
	operatorMap := utils.MapInterface(operator)
	if operatorMap == nil || operatorMap["__op"] == nil {
		return cannotTransform()
	}

	op := utils.String(operatorMap["__op"])
	switch op {
	case "Delete":
		if flatten {
			return nil
		}
		return bson.M{
			"__op": "$unset",
			"arg":  "",
		}

	case "Increment":
		if _, ok := operatorMap["amount"].(float64); !ok {
			// TODO 必须为数字
			return nil
		}
		if flatten {
			return operatorMap["amount"]
		}
		return bson.M{
			"__op": "$inc",
			"arg":  operatorMap["amount"],
		}

	case "Add", "AddUnique":
		objects := utils.SliceInterface(operatorMap["objects"])
		if objects == nil {
			// TODO 必须为数组
			return nil
		}
		toAdd := []interface{}{}
		for _, obj := range objects {
			o := transformAtom(obj, true, bson.M{"inArray": true})
			toAdd = append(toAdd, o)
		}
		if flatten {
			return toAdd
		}
		mongoOp := bson.M{
			"Add":       "$push",
			"AddUnique": "$addToSet",
		}[op]
		return bson.M{
			"__op": mongoOp,
			"arg": bson.M{
				"$each": toAdd,
			},
		}

	case "Remove":
		objects := utils.SliceInterface(operatorMap["objects"])
		if objects == nil {
			// TODO 必须为数组
			return nil
		}
		toRemove := []interface{}{}
		for _, obj := range objects {
			o := transformAtom(obj, true, bson.M{"inArray": true})
			toRemove = append(toRemove, o)
		}
		if flatten {
			return []interface{}{}
		}
		return bson.M{
			"__op": "$pullAll",
			"arg":  toRemove,
		}

	default:
		// TODO 不支持的类型
		return nil
	}
}

// transformCreate ...
func transformCreate(schema *Schema, className string, create bson.M) bson.M {
	// TODO
	return nil
}

func transformWhere(schema *Schema, className string, where bson.M) bson.M {
	// TODO
	return nil
}

func transformUpdate(schema *Schema, className string, update bson.M) bson.M {
	// TODO
	return nil
}

func untransformObjectT(schema *Schema, className string, mongoObject interface{}, isNestedObject bool) interface{} {
	// TODO
	return nil
}

func cannotTransform() interface{} {
	return nil
}

type dateCoder struct{}

func (d dateCoder) jsonToDatabase(json bson.M) interface{} {
	t, _ := utils.StringtoTime(utils.String(json["iso"]))
	return t
}

func (d dateCoder) isValidJSON(value bson.M) bool {
	return value != nil && utils.String(value["__type"]) == "Date"
}

type bytesCoder struct{}

func (b bytesCoder) databaseToJSON(object interface{}) interface{} {
	return nil
}

func (b bytesCoder) isValidDatabaseObject(object interface{}) bool {
	return false
}

func (b bytesCoder) jsonToDatabase(json bson.M) interface{} {
	by, _ := base64.StdEncoding.DecodeString(utils.String(json["base64"]))
	return by
}

func (b bytesCoder) isValidJSON(value bson.M) bool {
	return value != nil && utils.String(value["__type"]) == "Bytes"
}

type geoPointCoder struct{}

func (g geoPointCoder) databaseToJSON(object interface{}) interface{} {
	return nil
}

func (g geoPointCoder) isValidDatabaseObject(object interface{}) bool {
	return false
}

func (g geoPointCoder) jsonToDatabase(json bson.M) interface{} {
	return []interface{}{json["longitude"], json["latitude"]}
}

func (g geoPointCoder) isValidJSON(value bson.M) bool {
	return value != nil && utils.String(value["__type"]) == "GeoPoint"
}

type fileCoder struct{}

func (f fileCoder) databaseToJSON(object interface{}) interface{} {
	return nil
}

func (f fileCoder) isValidDatabaseObject(object interface{}) bool {
	return false
}

func (f fileCoder) jsonToDatabase(json bson.M) interface{} {
	return json["name"]
}

func (f fileCoder) isValidJSON(value bson.M) bool {
	return value != nil && utils.String(value["__type"]) == "File"
}
<|endoftext|>


<commit_before>package argon2

func fillSegment(ins *instance, pos *position) {
	var (
		refBlock, currBlock           *block
		pseudoRand, refIndex, refLane uint64
		prevOffset, currOffset        uint32
		startingIndex                 uint32
		dataIndependentAddressing     bool
		pseudoRands                   []uint64
	)
	if ins == nil {
		return
	}

	dataIndependentAddressing = (ins.variant == Argon2i)

	pseudoRands = make([]uint64, ins.segmentLength)

	if dataIndependentAddressing {
		generateAddresses(ins, pos, pseudoRands)
	}

	if pos.pass == 0 && pos.slice == 0 {
		startingIndex = 2 // We have already generated the first two blocks
	}

	// Calculate offset of the current block
	currOffset = pos.lane*ins.laneLength + uint32(pos.slice)*ins.segmentLength + startingIndex

	if currOffset%ins.laneLength == 0 {
		// Last block in this lane
		prevOffset = currOffset + ins.laneLength - 1
	} else {
		// Previous block
		prevOffset = currOffset - 1
	}

	for i := startingIndex; i < ins.segmentLength; i++ {
		/* 1.1 Rotating prev_offest if needed */
		if currOffset%ins.laneLength == 1 {
			prevOffset = currOffset - 1
		}

		/* 1.2 Computing the index of the reference block */
		/* 1.2.1 Taking pseudo-random value from the previous block */
		if dataIndependentAddressing {
			pseudoRand = pseudoRands[i]
		} else {
			pseudoRand = ins.memory[prevOffset][0]
		}

		/* 1.2.2 Computing the lane of the reference block */
		refLane = (pseudoRand >> 32) % uint64(ins.lanes)

		if (pos.pass == 0) && (pos.slice == 0) {
			// Can not reference other lanes yet
			refLane = uint64(pos.lane)
		}

		/* 1.2.3 Computing the number of possible reference blocks within lane */
		pos.index = i
		refIndex = uint64(indexAlpha(ins, pos, uint32(pseudoRand&0xFFFFFFFF), refLane == uint64(pos.lane)))

		/* 2 Creating a new block */
		refBlock =
			&ins.memory[uint64(ins.laneLength)*refLane+refIndex]
		//log.Printf("%d/%d\n", currOffset, len(ins.memory))
		currBlock = &ins.memory[currOffset]
		round(currBlock, refBlock, &ins.memory[prevOffset])
		currOffset++
		prevOffset++
	}
}

func generateAddresses(ins *instance, pos *position, pseudoRands []uint64) {
	var zeroBlock, inputBlock, addressBlock, tmpBlock block

	if ins == nil || pos == nil {
		return
	}

	inputBlock[0] = uint64(pos.pass)
	inputBlock[1] = uint64(pos.lane)
	inputBlock[2] = uint64(pos.slice)
	inputBlock[3] = uint64(ins.memoryBlocks)
	inputBlock[4] = uint64(ins.passes)
	inputBlock[5] = uint64(ins.variant)

	for i := uint32(0); i < ins.segmentLength; i++ {
		if i%addressesInBlock == 0 {
			inputBlock[6]++
			copy(tmpBlock[:], addressBlock[:])
			round(&inputBlock, &addressBlock, &zeroBlock)
			round(&addressBlock, &tmpBlock, &zeroBlock)
		}
		pseudoRands[i] = addressBlock[i%addressesInBlock]
	}
}

func xorBlock(dst, src *block) {
	for i := 0; i < qwordsInBlock; i++ {
		dst[i] ^= src[i]
	}
}
<commit_msg>Optimized Argon2d use case<commit_after>package argon2

func fillSegment(ins *instance, pos *position) {
	var (
		refBlock, currBlock           *block
		pseudoRand, refIndex, refLane uint64
		prevOffset, currOffset        uint32
		startingIndex                 uint32
		dataIndependentAddressing     bool
		pseudoRands                   []uint64
	)
	if ins == nil {
		return
	}

	dataIndependentAddressing = (ins.variant == Argon2i)

	if dataIndependentAddressing {
		pseudoRands = make([]uint64, ins.segmentLength)
		generateAddresses(ins, pos, pseudoRands)
	}

	if pos.pass == 0 && pos.slice == 0 {
		startingIndex = 2 // We have already generated the first two blocks
	}

	// Calculate offset of the current block
	currOffset = pos.lane*ins.laneLength + uint32(pos.slice)*ins.segmentLength + startingIndex

	if currOffset%ins.laneLength == 0 {
		// Last block in this lane
		prevOffset = currOffset + ins.laneLength - 1
	} else {
		// Previous block
		prevOffset = currOffset - 1
	}

	for i := startingIndex; i < ins.segmentLength; i++ {
		/* 1.1 Rotating prev_offest if needed */
		if currOffset%ins.laneLength == 1 {
			prevOffset = currOffset - 1
		}

		/* 1.2 Computing the index of the reference block */
		/* 1.2.1 Taking pseudo-random value from the previous block */
		if dataIndependentAddressing {
			pseudoRand = pseudoRands[i]
		} else {
			pseudoRand = ins.memory[prevOffset][0]
		}

		/* 1.2.2 Computing the lane of the reference block */
		refLane = (pseudoRand >> 32) % uint64(ins.lanes)

		if (pos.pass == 0) && (pos.slice == 0) {
			// Can not reference other lanes yet
			refLane = uint64(pos.lane)
		}

		/* 1.2.3 Computing the number of possible reference blocks within lane */
		pos.index = i
		refIndex = uint64(indexAlpha(ins, pos, uint32(pseudoRand&0xFFFFFFFF), refLane == uint64(pos.lane)))

		/* 2 Creating a new block */
		refBlock =
			&ins.memory[uint64(ins.laneLength)*refLane+refIndex]
		//log.Printf("%d/%d\n", currOffset, len(ins.memory))
		currBlock = &ins.memory[currOffset]
		round(currBlock, refBlock, &ins.memory[prevOffset])
		currOffset++
		prevOffset++
	}
}

func generateAddresses(ins *instance, pos *position, pseudoRands []uint64) {
	var zeroBlock, inputBlock, addressBlock, tmpBlock block

	if ins == nil || pos == nil {
		return
	}

	inputBlock[0] = uint64(pos.pass)
	inputBlock[1] = uint64(pos.lane)
	inputBlock[2] = uint64(pos.slice)
	inputBlock[3] = uint64(ins.memoryBlocks)
	inputBlock[4] = uint64(ins.passes)
	inputBlock[5] = uint64(ins.variant)

	for i := uint32(0); i < ins.segmentLength; i++ {
		if i%addressesInBlock == 0 {
			inputBlock[6]++
			copy(tmpBlock[:], addressBlock[:])
			round(&inputBlock, &addressBlock, &zeroBlock)
			round(&addressBlock, &tmpBlock, &zeroBlock)
		}
		pseudoRands[i] = addressBlock[i%addressesInBlock]
	}
}

func xorBlock(dst, src *block) {
	for i := 0; i < qwordsInBlock; i++ {
		dst[i] ^= src[i]
	}
}
<|endoftext|>


<commit_before>package mallory

import (
	"errors"
	"github.com/justmao945/mallory/ssh"
	"io/ioutil"
	"net"
	"net/http"
	"net/url"
	"os"
	"os/user"
	"sync"
)

//
type EngineSSH struct {
	Env *Env
	URL *url.URL
	Cli *ssh.Client
	Cfg *ssh.ClientConfig
	Dir *EngineDirect
	// atomic Dial
	mutex sync.RWMutex
}

// Create and initialize
func CreateEngineSSH(e *Env) (self *EngineSSH, err error) {
	self = &EngineSSH{
		Env: e,
		Cfg: &ssh.ClientConfig{},
	}
	// e.g.  ssh://user:passwd@192.168.1.1:1122
	self.URL, err = url.Parse(e.Remote)
	if err != nil {
		return
	}

	if self.URL.User != nil {
		self.Cfg.User = self.URL.User.Username()
	} else {
		u, err := user.Current()
		if err != nil {
			return self, err
		}
		// u.Name is the full name, should not be used
		self.Cfg.User = u.Username
	}

	// 1) try RSA keyring first
	for {
		id_rsa := os.ExpandEnv("$HOME/.ssh/id_rsa")
		pem, err := ioutil.ReadFile(id_rsa)
		if err != nil {
			break
		}
		signer, err := ssh.ParsePrivateKey(pem)
		if err != nil {
			break
		}
		self.Cfg.Auth = append(self.Cfg.Auth, ssh.PublicKeys(signer))
		// stop !!
		break
	}
	// 2) try password
	for {
		if self.URL.User == nil {
			break
		}
		if pass, ok := self.URL.User.Password(); ok {
			self.Cfg.Auth = append(self.Cfg.Auth, ssh.Password(pass))
		}
		// stop here!!
		break
	}

	if len(self.Cfg.Auth) == 0 {
		//TODO: keyboard intercative
		err = errors.New("Invalid auth method, please add password or generate ssh keys")
		return
	}

	self.Cli, err = ssh.Dial("tcp", self.URL.Host, self.Cfg)
	if err != nil {
		return
	}

	dial := func(network, addr string) (c net.Conn, err error) {
		for i := 0; i < 3; i++ {
			if err != nil {
				break
			}
			// need read lock, we'll reconnect Cli if is disconnected
			// use read write lock may slow down connection ?
			self.mutex.RLock()
			c, err = self.Cli.Dial(network, addr)
			saveCli := self.Cli
			self.mutex.RUnlock()

			// We want to reconnect the network when disconnected.
			// FIXME: unexported net.errClosing
			if err != nil && err.Error() == "use of closed network connection" {
				// we may change the Cli, need write lock
				self.mutex.Lock()
				if saveCli == self.Cli {
					self.Cli.Close()
					self.Cli, err = ssh.Dial("tcp", self.URL.Host, self.Cfg)
				}
				self.mutex.Unlock()
				continue
			}
			// do not reconnect when no error or other errors
			break
		}
		return
	}

	self.Dir = &EngineDirect{
		Tr: &http.Transport{Dial: dial},
	}
	return
}

func (self *EngineSSH) Serve(s *Session) {
	self.Dir.Serve(s)
}

func (self *EngineSSH) Connect(s *Session) {
	self.Dir.Connect(s)
}
<commit_msg>fix crash when multiple of reconnection after failed<commit_after>package mallory

import (
	"errors"
	"github.com/justmao945/mallory/ssh"
	"io/ioutil"
	"net"
	"net/http"
	"net/url"
	"os"
	"os/user"
	"sync"
)

//
type EngineSSH struct {
	Env *Env
	URL *url.URL
	Cli *ssh.Client
	Cfg *ssh.ClientConfig
	Dir *EngineDirect
	// atomic Dial
	mutex sync.RWMutex
}

// Create and initialize
func CreateEngineSSH(e *Env) (self *EngineSSH, err error) {
	self = &EngineSSH{
		Env: e,
		Cfg: &ssh.ClientConfig{},
	}
	// e.g.  ssh://user:passwd@192.168.1.1:1122
	self.URL, err = url.Parse(e.Remote)
	if err != nil {
		return
	}

	if self.URL.User != nil {
		self.Cfg.User = self.URL.User.Username()
	} else {
		u, err := user.Current()
		if err != nil {
			return self, err
		}
		// u.Name is the full name, should not be used
		self.Cfg.User = u.Username
	}

	// 1) try RSA keyring first
	for {
		id_rsa := os.ExpandEnv("$HOME/.ssh/id_rsa")
		pem, err := ioutil.ReadFile(id_rsa)
		if err != nil {
			break
		}
		signer, err := ssh.ParsePrivateKey(pem)
		if err != nil {
			break
		}
		self.Cfg.Auth = append(self.Cfg.Auth, ssh.PublicKeys(signer))
		// stop !!
		break
	}
	// 2) try password
	for {
		if self.URL.User == nil {
			break
		}
		if pass, ok := self.URL.User.Password(); ok {
			self.Cfg.Auth = append(self.Cfg.Auth, ssh.Password(pass))
		}
		// stop here!!
		break
	}

	if len(self.Cfg.Auth) == 0 {
		//TODO: keyboard intercative
		err = errors.New("Invalid auth method, please add password or generate ssh keys")
		return
	}

	self.Cli, err = ssh.Dial("tcp", self.URL.Host, self.Cfg)
	if err != nil {
		return
	}

	dial := func(network, addr string) (c net.Conn, err error) {
		for i := 0; i < 3; i++ {
			if err != nil {
				break
			}
			// The reason why both Cli and err are nil is that, the previous round
			// connection is failed, which keeps self.Cli nil.
			if self.Cli == nil && err == nil {
				err = errors.New("network is unreachable")
				break
			}

			// need read lock, we'll reconnect Cli if is disconnected
			// use read write lock may slow down connection ?
			self.mutex.RLock()
			c, err = self.Cli.Dial(network, addr)
			saveCli := self.Cli
			self.mutex.RUnlock()

			// We want to reconnect the network when disconnected.
			// FIXME: unexported net.errClosing
			if err != nil && err.Error() == "use of closed network connection" {
				// we may change the Cli, need write lock
				self.mutex.Lock()
				if saveCli == self.Cli {
					self.Cli.Close()
					self.Cli, err = ssh.Dial("tcp", self.URL.Host, self.Cfg)
				}
				self.mutex.Unlock()
				continue
			}
			// do not reconnect when no error or other errors
			break
		}
		return
	}

	self.Dir = &EngineDirect{
		Tr: &http.Transport{Dial: dial},
	}
	return
}

func (self *EngineSSH) Serve(s *Session) {
	self.Dir.Serve(s)
}

func (self *EngineSSH) Connect(s *Session) {
	self.Dir.Connect(s)
}
<|endoftext|>


<commit_before><commit_msg>Update dashboard_api.go<commit_after><|endoftext|>


<commit_before>// Package dashboard is an implementation of a Vanadium dashboard web
// server.
package main

// TODO(jsimsa): Move style formatting to a .css file.

import (
	"encoding/json"
	"encoding/xml"
	"flag"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"
	"os"
	"path/filepath"
	"regexp"
	"strings"
	"text/template"

	"v.io/x/devtools/lib/collect"
	"v.io/x/devtools/lib/testutil"
	"v.io/x/devtools/lib/util"
	"v.io/x/devtools/lib/xunit"
)

const (
	defaultBucket = "gs://vanadium-test-results"
)

var (
	bucketFlag  string
	cacheFlag   string
	dryRunFlag  bool
	noColorFlag bool
	portFlag    int
	verboseFlag bool
)

func init() {
	flag.StringVar(&bucketFlag, "bucket", defaultBucket, "Google Storage bucket to use for fetching files.")
	flag.StringVar(&cacheFlag, "cache", "", "Directory to use for caching files.")
	flag.BoolVar(&dryRunFlag, "n", false, "Show what commands will run but do not execute them.")
	flag.BoolVar(&noColorFlag, "nocolor", false, "Do not use color to format output.")
	flag.BoolVar(&verboseFlag, "v", false, "Print verbose output.")
	flag.IntVar(&portFlag, "port", 8000, "Port for the server.")
	flag.Parse()
}

type summaryData struct {
	Number string
	Labels []label
}

type label struct {
	Jobs []job
	Name string
}

type job struct {
	Name   string
	Result bool
}

var summaryTemplate = template.Must(template.New("summary").Parse(`
{{ $n := .Number }}

<title>Presubmit #{{ $n }} Summary</title>

Presubmit #{{ $n }}

{{ range $label := .Labels }}

{{ $label.Name }}

{{ range $job := .Jobs }}

{{ if $job.Result }}
PASS
{{ else }}
FAIL
{{ end }}
{{ $job.Name }}
{{ end }}

{{ end }}

`))

type jobData struct {
	Job    string
	Label  string
	Number string
	Output string
	Result bool
}

var jobTemplate = template.Must(template.New("job").Funcs(templateFuncMap).Parse(`

<title>Presubmit #{{ .Number }} Job Details</title>

Presubmit #{{ .Number }} (label={{ .Label }}, job={{ .Job }})

{{ if .Result }}

PASS

{{ else }}

FAIL

{{ end }}

Console Output:

{{ colors .Output }}

`))

type testData struct {
	Job      string
	Label    string
	Number   string
	TestCase xunit.TestCase
}

var testTemplate = template.Must(template.New("test").Funcs(templateFuncMap).Parse(`

<title>Presubmit #{{ .Number }} Test Details</title>

Presubmit #{{ .Number }} (label={{ .Label }}, job={{ .Job }}, suite={{ .TestCase.Classname }}, test={{ .TestCase.Name }})

{{ if eq (len .TestCase.Failures) 0 }}

PASS

{{ else }}

FAIL

Failures:

{{ range $failure := .TestCase.Failures }}

 {{ $failure.Message }}:

{{ colors $failure.Data }}

{{ end }}

{{ end }}

`))

var templateFuncMap = template.FuncMap{
	"colors": ansiColorsToHTML,
}

type ansiColor struct {
	code  string
	style string
}

var (
	ansiColors = []ansiColor{
		ansiColor{"30", "color:black"},
		ansiColor{"31", "color:red"},
		ansiColor{"32", "color:green"},
		ansiColor{"33", "color:yellow"},
		ansiColor{"34", "color:blue"},
		ansiColor{"35", "color:magenta"},
		ansiColor{"36", "color:cyan"},
		ansiColor{"37", "color:black"},
		ansiColor{"40", "background-color:black"},
		ansiColor{"41", "background-color:red"},
		ansiColor{"42", "background-color:green"},
		ansiColor{"43", "background-color:yellow"},
		ansiColor{"44", "background-color:blue"},
		ansiColor{"45", "background-color:magenta"},
		ansiColor{"46", "background-color:cyan"},
		ansiColor{"47", "background-color:black"},
	}
)

func ansiColorsToHTML(text string) (string, error) {
	for _, ansi := range ansiColors {
		re, err := regexp.Compile(fmt.Sprintf(`\[0;%sm(.*)\[0m`, ansi.code))
		if err != nil {
			return "", err
		}
		text = re.ReplaceAllString(text, fmt.Sprintf(`<font style="%s">$1</font>`, ansi.style))
	}
	return text, nil
}

func displayPresubmitPage(ctx *util.Context, w http.ResponseWriter, r *http.Request) (e error) {
	// Set up the root directory.
	root := cacheFlag
	if root == "" {
		tmpDir, err := ctx.Run().TempDir("", "")
		if err != nil {
			return err
		}
		defer collect.Error(func() error { return ctx.Run().RemoveAll(tmpDir) }, &e)
		root = tmpDir
	}

	// Fetch the presubmit test results.
	if err := ctx.Run().MkdirAll(filepath.Join(root, "presubmit"), os.FileMode(0700)); err != nil {
		return err
	}
	n := r.Form.Get("n")
	if _, err := os.Stat(filepath.Join(root, "presubmit", n)); err != nil {
		if !os.IsNotExist(err) {
			return fmt.Errorf("Stat() failed: %v", err)
		}
		bucket := bucketFlag + "/v0/presubmit/" + n
		if err := ctx.Run().Command("gsutil", "-m", "-q", "cp", "-r", bucket, filepath.Join(root, "presubmit")); err != nil {
			return err
		}
	}

	label, job, testSuite, testCase := r.Form.Get("label"), r.Form.Get("job"), r.Form.Get("suite"), r.Form.Get("test")
	switch {
	case label == "" || job == "":
		// Generate the summary page.
		path := filepath.Join(root, "presubmit", n)
		data, err := generateSummaryData(ctx, n, path)
		if err != nil {
			return err
		}
		if err := summaryTemplate.Execute(w, data); err != nil {
			return fmt.Errorf("Execute() failed: %v", err)
		}
		return nil
	case testSuite == "":
		// Generate the job detail page.
		path := filepath.Join(root, "presubmit", n, label, job)
		data, err := generateJobData(ctx, n, label, job, path)
		if err != nil {
			return err
		}
		if err := jobTemplate.Execute(w, data); err != nil {
			return fmt.Errorf("Execute() failed: %v", err)
		}
	case testSuite != "" && testCase != "":
		// Generate the test detail page.
		path := filepath.Join(root, "presubmit", n, label, job)
		data, err := generateTestData(ctx, n, label, job, testSuite, testCase, path)
		if err != nil {
			return err
		}
		if err := testTemplate.Execute(w, data); err != nil {
			return fmt.Errorf("Execute() failed: %v", err)
		}
	default:
		return fmt.Errorf("invalid combination of parameters")
	}
	return nil
}

func generateSummaryData(ctx *util.Context, n, path string) (*summaryData, error) {
	data := summaryData{n, []label{}}
	fileInfos, err := ioutil.ReadDir(path)
	if err != nil {
		return nil, fmt.Errorf("ReadDir(%v) failed: %v", path, err)
	}
	for _, fileInfo := range fileInfos {
		labelDir := filepath.Join(path, fileInfo.Name())
		fileInfos, err := ioutil.ReadDir(labelDir)
		if err != nil {
			return nil, fmt.Errorf("ReadDir(%v) failed: %v", path, err)
		}
		l := label{
			Jobs: []job{},
			Name: fileInfo.Name(),
		}
		for _, fileInfo := range fileInfos {
			jobDir := filepath.Join(labelDir, fileInfo.Name())
			bytes, err := ctx.Run().ReadFile(filepath.Join(jobDir, "result"))
			if err != nil {
				return nil, err
			}
			var r testutil.TestResult
			if err := json.Unmarshal(bytes, &r); err != nil {
				return nil, fmt.Errorf("Unmarshal(%v) failed: %v", string(bytes), err)
			}
			j := job{
				Name:   fileInfo.Name(),
				Result: r.Status == testutil.TestPassed,
			}
			l.Jobs = append(l.Jobs, j)
		}
		data.Labels = append(data.Labels, l)
	}
	return &data, nil
}

func generateJobData(ctx *util.Context, n, label, job, path string) (*jobData, error) {
	outputBytes, err := ctx.Run().ReadFile(filepath.Join(path, "output"))
	if err != nil {
		return nil, err
	}
	resultBytes, err := ctx.Run().ReadFile(filepath.Join(path, "result"))
	if err != nil {
		return nil, err
	}
	var r testutil.TestResult
	if err := json.Unmarshal(resultBytes, &r); err != nil {
		return nil, fmt.Errorf("Unmarshal(%v) failed: %v", string(resultBytes), err)
	}
	data := jobData{
		Job:    job,
		Label:  label,
		Number: n,
		Output: string(outputBytes),
		Result: r.Status == testutil.TestPassed,
	}
	return &data, nil
}

func generateTestData(ctx *util.Context, n, label, job, testSuite, testCase, path string) (*testData, error) {
	suitesBytes, err := ctx.Run().ReadFile(filepath.Join(path, "xunit.xml"))
	if err != nil {
		return nil, err
	}
	var s xunit.TestSuites
	if err := xml.Unmarshal(suitesBytes, &s); err != nil {
		return nil, fmt.Errorf("Unmarshal(%v) failed: %v", string(suitesBytes), err)
	}
	var test xunit.TestCase
	found := false
outer:
	for _, ts := range s.Suites {
		if ts.Name == testSuite {
			for _, tc := range ts.Cases {
				if tc.Name == testCase {
					test = tc
					found = true
					break outer
				}
			}
		}
	}
	if !found {
		return nil, fmt.Errorf("failed to find the test %s in test suite %s", testCase, testSuite)
	}
	data := testData{
		Job:      job,
		Label:    label,
		Number:   n,
		TestCase: test,
	}
	return &data, nil
}

func validateValues(values url.Values) error {
	ty := values.Get("type")
	if ty == "" {
		return fmt.Errorf("required parameter 'type' not found")
	}
	if ty == "presubmit" {
		if n := values.Get("n"); n == "" {
			return fmt.Errorf("required parameter 'n' not found")
		} else {
			if strings.Contains(n, "..") {
				return fmt.Errorf("parameter 'n' is not allowed to contain '..'")
			}
		}
		if job := values.Get("job"); job != "" && strings.Contains(job, "..") {
			return fmt.Errorf("parameter 'job' is not allowed to contain '..'")
		}
		if label := values.Get("label"); label != "" && strings.Contains(label, "..") {
			return fmt.Errorf("parameter 'label' is not allowed to contain '..'")
		}
		if suite := values.Get("suite"); suite != "" && strings.Contains(suite, "..") {
			return fmt.Errorf("parameter 'suite' is not allowed to contain '..'")
		}
		if test := values.Get("test"); test != "" && strings.Contains(test, "..") {
			return fmt.Errorf("parameter 'test' is not allowed to contain '..'")
		}
	}
	return nil
}

func helper(ctx *util.Context, w http.ResponseWriter, r *http.Request) {
	r.ParseForm()
	if err := validateValues(r.Form); err != nil {
		fmt.Fprintf(ctx.Stderr(), "%v", err)
		http.Error(w, "500 internal server error", http.StatusInternalServerError)
		return
	}

	switch r.Form.Get("type") {
	case "presubmit":
		if err := displayPresubmitPage(ctx, w, r); err != nil {
			fmt.Fprintf(ctx.Stderr(), "%v", err)
			http.Error(w, "500 internal server error", http.StatusInternalServerError)
		}
		// The presubmit test results data never changes, cache it in
		// the clients for up to 30 days.
		w.Header().Set("Cache-control", "public, max-age=2592000")
	default:
		fmt.Fprintf(ctx.Stderr(), "unknown type: %v", r.Form.Get("type"))
		http.NotFound(w, r)
	}
}

func main() {
	ctx := util.NewContext(nil, os.Stdin, os.Stdout, os.Stderr, !noColorFlag, dryRunFlag, verboseFlag)
	handler := func(w http.ResponseWriter, r *http.Request) {
		helper(ctx, w, r)
	}
	http.HandleFunc("/", handler)
	if err := http.ListenAndServe(fmt.Sprintf(":%d", portFlag), nil); err != nil {
		fmt.Fprintf(os.Stderr, "ListenAndServer() failed: %v", err)
		os.Exit(1)
	}
}
<commit_msg>dahsboard: adding support for a health check to the dashboard server<commit_after>// Package dashboard is an implementation of a Vanadium dashboard web
// server.
package main

// TODO(jsimsa): Move style formatting to a .css file.

import (
	"encoding/json"
	"encoding/xml"
	"flag"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"
	"os"
	"path/filepath"
	"regexp"
	"strings"
	"text/template"

	"v.io/x/devtools/lib/collect"
	"v.io/x/devtools/lib/testutil"
	"v.io/x/devtools/lib/util"
	"v.io/x/devtools/lib/xunit"
)

const (
	defaultBucket = "gs://vanadium-test-results"
)

var (
	bucketFlag  string
	cacheFlag   string
	dryRunFlag  bool
	noColorFlag bool
	portFlag    int
	verboseFlag bool
)

func init() {
	flag.StringVar(&bucketFlag, "bucket", defaultBucket, "Google Storage bucket to use for fetching files.")
	flag.StringVar(&cacheFlag, "cache", "", "Directory to use for caching files.")
	flag.BoolVar(&dryRunFlag, "n", false, "Show what commands will run but do not execute them.")
	flag.BoolVar(&noColorFlag, "nocolor", false, "Do not use color to format output.")
	flag.BoolVar(&verboseFlag, "v", false, "Print verbose output.")
	flag.IntVar(&portFlag, "port", 8000, "Port for the server.")
	flag.Parse()
}

type summaryData struct {
	Number string
	Labels []label
}

type label struct {
	Jobs []job
	Name string
}

type job struct {
	Name   string
	Result bool
}

var summaryTemplate = template.Must(template.New("summary").Parse(`
{{ $n := .Number }}

<title>Presubmit #{{ $n }} Summary</title>

Presubmit #{{ $n }}

{{ range $label := .Labels }}

{{ $label.Name }}

{{ range $job := .Jobs }}

{{ if $job.Result }}
PASS
{{ else }}
FAIL
{{ end }}
{{ $job.Name }}
{{ end }}

{{ end }}

`))

type jobData struct {
	Job    string
	Label  string
	Number string
	Output string
	Result bool
}

var jobTemplate = template.Must(template.New("job").Funcs(templateFuncMap).Parse(`

<title>Presubmit #{{ .Number }} Job Details</title>

Presubmit #{{ .Number }} (label={{ .Label }}, job={{ .Job }})

{{ if .Result }}

PASS

{{ else }}

FAIL

{{ end }}

Console Output:

{{ colors .Output }}

`))

type testData struct {
	Job      string
	Label    string
	Number   string
	TestCase xunit.TestCase
}

var testTemplate = template.Must(template.New("test").Funcs(templateFuncMap).Parse(`

<title>Presubmit #{{ .Number }} Test Details</title>

Presubmit #{{ .Number }} (label={{ .Label }}, job={{ .Job }}, suite={{ .TestCase.Classname }}, test={{ .TestCase.Name }})

{{ if eq (len .TestCase.Failures) 0 }}

PASS

{{ else }}

FAIL

Failures:

{{ range $failure := .TestCase.Failures }}

 {{ $failure.Message }}:

{{ colors $failure.Data }}

{{ end }}

{{ end }}

`))

var templateFuncMap = template.FuncMap{
	"colors": ansiColorsToHTML,
}

type ansiColor struct {
	code  string
	style string
}

var (
	ansiColors = []ansiColor{
		ansiColor{"30", "color:black"},
		ansiColor{"31", "color:red"},
		ansiColor{"32", "color:green"},
		ansiColor{"33", "color:yellow"},
		ansiColor{"34", "color:blue"},
		ansiColor{"35", "color:magenta"},
		ansiColor{"36", "color:cyan"},
		ansiColor{"37", "color:black"},
		ansiColor{"40", "background-color:black"},
		ansiColor{"41", "background-color:red"},
		ansiColor{"42", "background-color:green"},
		ansiColor{"43", "background-color:yellow"},
		ansiColor{"44", "background-color:blue"},
		ansiColor{"45", "background-color:magenta"},
		ansiColor{"46", "background-color:cyan"},
		ansiColor{"47", "background-color:black"},
	}
)

func ansiColorsToHTML(text string) (string, error) {
	for _, ansi := range ansiColors {
		re, err := regexp.Compile(fmt.Sprintf(`\[0;%sm(.*)\[0m`, ansi.code))
		if err != nil {
			return "", err
		}
		text = re.ReplaceAllString(text, fmt.Sprintf(`<font style="%s">$1</font>`, ansi.style))
	}
	return text, nil
}

func displayPresubmitPage(ctx *util.Context, w http.ResponseWriter, r *http.Request) (e error) {
	// Set up the root directory.
	root := cacheFlag
	if root == "" {
		tmpDir, err := ctx.Run().TempDir("", "")
		if err != nil {
			return err
		}
		defer collect.Error(func() error { return ctx.Run().RemoveAll(tmpDir) }, &e)
		root = tmpDir
	}

	// Fetch the presubmit test results.
	if err := ctx.Run().MkdirAll(filepath.Join(root, "presubmit"), os.FileMode(0700)); err != nil {
		return err
	}
	n := r.Form.Get("n")
	if _, err := os.Stat(filepath.Join(root, "presubmit", n)); err != nil {
		if !os.IsNotExist(err) {
			return fmt.Errorf("Stat() failed: %v", err)
		}
		bucket := bucketFlag + "/v0/presubmit/" + n
		if err := ctx.Run().Command("gsutil", "-m", "-q", "cp", "-r", bucket, filepath.Join(root, "presubmit")); err != nil {
			return err
		}
	}

	label, job, testSuite, testCase := r.Form.Get("label"), r.Form.Get("job"), r.Form.Get("suite"), r.Form.Get("test")
	switch {
	case label == "" || job == "":
		// Generate the summary page.
		path := filepath.Join(root, "presubmit", n)
		data, err := generateSummaryData(ctx, n, path)
		if err != nil {
			return err
		}
		if err := summaryTemplate.Execute(w, data); err != nil {
			return fmt.Errorf("Execute() failed: %v", err)
		}
		return nil
	case testSuite == "":
		// Generate the job detail page.
		path := filepath.Join(root, "presubmit", n, label, job)
		data, err := generateJobData(ctx, n, label, job, path)
		if err != nil {
			return err
		}
		if err := jobTemplate.Execute(w, data); err != nil {
			return fmt.Errorf("Execute() failed: %v", err)
		}
	case testSuite != "" && testCase != "":
		// Generate the test detail page.
		path := filepath.Join(root, "presubmit", n, label, job)
		data, err := generateTestData(ctx, n, label, job, testSuite, testCase, path)
		if err != nil {
			return err
		}
		if err := testTemplate.Execute(w, data); err != nil {
			return fmt.Errorf("Execute() failed: %v", err)
		}
	default:
		return fmt.Errorf("invalid combination of parameters")
	}
	return nil
}

func generateSummaryData(ctx *util.Context, n, path string) (*summaryData, error) {
	data := summaryData{n, []label{}}
	fileInfos, err := ioutil.ReadDir(path)
	if err != nil {
		return nil, fmt.Errorf("ReadDir(%v) failed: %v", path, err)
	}
	for _, fileInfo := range fileInfos {
		labelDir := filepath.Join(path, fileInfo.Name())
		fileInfos, err := ioutil.ReadDir(labelDir)
		if err != nil {
			return nil, fmt.Errorf("ReadDir(%v) failed: %v", path, err)
		}
		l := label{
			Jobs: []job{},
			Name: fileInfo.Name(),
		}
		for _, fileInfo := range fileInfos {
			jobDir := filepath.Join(labelDir, fileInfo.Name())
			bytes, err := ctx.Run().ReadFile(filepath.Join(jobDir, "result"))
			if err != nil {
				return nil, err
			}
			var r testutil.TestResult
			if err := json.Unmarshal(bytes, &r); err != nil {
				return nil, fmt.Errorf("Unmarshal(%v) failed: %v", string(bytes), err)
			}
			j := job{
				Name:   fileInfo.Name(),
				Result: r.Status == testutil.TestPassed,
			}
			l.Jobs = append(l.Jobs, j)
		}
		data.Labels = append(data.Labels, l)
	}
	return &data, nil
}

func generateJobData(ctx *util.Context, n, label, job, path string) (*jobData, error) {
	outputBytes, err := ctx.Run().ReadFile(filepath.Join(path, "output"))
	if err != nil {
		return nil, err
	}
	resultBytes, err := ctx.Run().ReadFile(filepath.Join(path, "result"))
	if err != nil {
		return nil, err
	}
	var r testutil.TestResult
	if err := json.Unmarshal(resultBytes, &r); err != nil {
		return nil, fmt.Errorf("Unmarshal(%v) failed: %v", string(resultBytes), err)
	}
	data := jobData{
		Job:    job,
		Label:  label,
		Number: n,
		Output: string(outputBytes),
		Result: r.Status == testutil.TestPassed,
	}
	return &data, nil
}

func generateTestData(ctx *util.Context, n, label, job, testSuite, testCase, path string) (*testData, error) {
	suitesBytes, err := ctx.Run().ReadFile(filepath.Join(path, "xunit.xml"))
	if err != nil {
		return nil, err
	}
	var s xunit.TestSuites
	if err := xml.Unmarshal(suitesBytes, &s); err != nil {
		return nil, fmt.Errorf("Unmarshal(%v) failed: %v", string(suitesBytes), err)
	}
	var test xunit.TestCase
	found := false
outer:
	for _, ts := range s.Suites {
		if ts.Name == testSuite {
			for _, tc := range ts.Cases {
				if tc.Name == testCase {
					test = tc
					found = true
					break outer
				}
			}
		}
	}
	if !found {
		return nil, fmt.Errorf("failed to find the test %s in test suite %s", testCase, testSuite)
	}
	data := testData{
		Job:      job,
		Label:    label,
		Number:   n,
		TestCase: test,
	}
	return &data, nil
}

func validateValues(values url.Values) error {
	ty := values.Get("type")
	if ty == "" {
		return fmt.Errorf("required parameter 'type' not found")
	}
	if ty == "presubmit" {
		if n := values.Get("n"); n == "" {
			return fmt.Errorf("required parameter 'n' not found")
		} else {
			if strings.Contains(n, "..") {
				return fmt.Errorf("parameter 'n' is not allowed to contain '..'")
			}
		}
		if job := values.Get("job"); job != "" && strings.Contains(job, "..") {
			return fmt.Errorf("parameter 'job' is not allowed to contain '..'")
		}
		if label := values.Get("label"); label != "" && strings.Contains(label, "..") {
			return fmt.Errorf("parameter 'label' is not allowed to contain '..'")
		}
		if suite := values.Get("suite"); suite != "" && strings.Contains(suite, "..") {
			return fmt.Errorf("parameter 'suite' is not allowed to contain '..'")
		}
		if test := values.Get("test"); test != "" && strings.Contains(test, "..") {
			return fmt.Errorf("parameter 'test' is not allowed to contain '..'")
		}
	}
	return nil
}

func helper(ctx *util.Context, w http.ResponseWriter, r *http.Request) {
	r.ParseForm()
	if err := validateValues(r.Form); err != nil {
		fmt.Fprintf(ctx.Stderr(), "%v", err)
		http.Error(w, "500 internal server error", http.StatusInternalServerError)
		return
	}

	switch r.Form.Get("type") {
	case "presubmit":
		if err := displayPresubmitPage(ctx, w, r); err != nil {
			fmt.Fprintf(ctx.Stderr(), "%v", err)
			http.Error(w, "500 internal server error", http.StatusInternalServerError)
		}
		// The presubmit test results data never changes, cache it in
		// the clients for up to 30 days.
		w.Header().Set("Cache-control", "public, max-age=2592000")
	default:
		fmt.Fprintf(ctx.Stderr(), "unknown type: %v", r.Form.Get("type"))
		http.NotFound(w, r)
	}
}

func main() {
	ctx := util.NewContext(nil, os.Stdin, os.Stdout, os.Stderr, !noColorFlag, dryRunFlag, verboseFlag)
	handler := func(w http.ResponseWriter, r *http.Request) {
		helper(ctx, w, r)
	}
	health := func(w http.ResponseWriter, r *http.Request) {
		w.WriteHeader(http.StatusNoContent)
	}
	http.HandleFunc("/health", health)
	http.HandleFunc("/", handler)
	if err := http.ListenAndServe(fmt.Sprintf(":%d", portFlag), nil); err != nil {
		fmt.Fprintf(os.Stderr, "ListenAndServer() failed: %v", err)
		os.Exit(1)
	}
}
<|endoftext|>


<commit_before>package tag

import (
	"crypto/sha1"
	"encoding/binary"
	"errors"
	"fmt"
	"hash"
	"io"
	"os"
)

// Sum creates a checksum of the audio file data provided by the io.ReadSeeker which is metadata
// (ID3, MP4) invariant.
func Sum(r io.ReadSeeker) (string, error) {
	b, err := readBytes(r, 11)
	if err != nil {
		return "", err
	}

	_, err = r.Seek(-11, os.SEEK_CUR)
	if err != nil {
		return "", fmt.Errorf("could not seek back to original position: %v", err)
	}

	if string(b[0:4]) == "fLaC" {
		return SumFLAC(r)
	}

	if string(b[4:11]) == "ftypM4A" {
		return SumAtoms(r)
	}

	if string(b[0:3]) == "ID3" {
		return SumID3v2(r)
	}

	h, err := SumID3v1(r)
	if err != nil {
		if err == ErrNotID3v1 {
			return SumAll(r)
		}
		return "", err
	}
	return h, nil
}

// SumAll returns a checksum of the content from the reader (until EOF).
func SumAll(r io.ReadSeeker) (string, error) {
	h := sha1.New()
	_, err := io.Copy(h, r)
	if err != nil {
		return "", nil
	}
	return hashSum(h), nil
}

// SumAtoms constructs a checksum of MP4 audio file data provided by the io.ReadSeeker which is
// metadata invariant.
func SumAtoms(r io.ReadSeeker) (string, error) {
	for {
		var size uint32
		err := binary.Read(r, binary.BigEndian, &size)
		if err != nil {
			if err == io.EOF {
				return "", fmt.Errorf("reached EOF before audio data")
			}
			return "", err
		}

		name, err := readString(r, 4)
		if err != nil {
			return "", err
		}

		switch name {
		case "meta":
			// next_item_id (int32)
			_, err := r.Seek(4, os.SEEK_CUR)
			if err != nil {
				return "", err
			}
			fallthrough

		case "moov", "udta", "ilst":
			continue

		case "mdat": // stop when we get to the data
			h := sha1.New()
			_, err := io.CopyN(h, r, int64(size-8))
			if err != nil {
				return "", fmt.Errorf("error reading audio data: %v", err)
			}
			return hashSum(h), nil
		}

		_, err = r.Seek(int64(size-8), os.SEEK_CUR)
		if err != nil {
			return "", fmt.Errorf("error reading '%v' tag: %v", name, err)
		}
	}
}

func sizeToEndOffset(r io.ReadSeeker, offset int64) (int64, error) {
	n, err := r.Seek(-128, os.SEEK_END)
	if err != nil {
		return 0, fmt.Errorf("error seeking end offset (%d bytes): %v", offset, err)
	}

	_, err = r.Seek(-n, os.SEEK_CUR)
	if err != nil {
		return 0, fmt.Errorf("error seeking back to original position: %v", err)
	}
	return n, nil
}

// SumID3v1 constructs a checksum of MP3 audio file data (assumed to have ID3v1 tags) provided
// by the io.ReadSeeker which is metadata invariant.
func SumID3v1(r io.ReadSeeker) (string, error) {
	n, err := sizeToEndOffset(r, 128)
	if err != nil {
		return "", fmt.Errorf("error determining read size to ID3v1 header: %v", err)
	}

	// TODO: improve this check???
	if n <= 0 {
		return "", fmt.Errorf("file size must be greater than 128 bytes (ID3v1 header size) for MP3")
	}

	h := sha1.New()
	_, err = io.CopyN(h, r, n)
	if err != nil {
		return "", fmt.Errorf("error reading %v bytes: %v", n, err)
	}
	return hashSum(h), nil
}

// SumID3v2 constructs a checksum of MP3 audio file data (assumed to have ID3v2 tags) provided by the
// io.ReadSeeker which is metadata invariant.
func SumID3v2(r io.ReadSeeker) (string, error) {
	header, err := readID3v2Header(r)
	if err != nil {
		return "", fmt.Errorf("error reading ID3v2 header: %v", err)
	}

	_, err = r.Seek(int64(header.Size), os.SEEK_CUR)
	if err != nil {
		return "", fmt.Errorf("error seeking to end of ID3V2 header: %v", err)
	}

	n, err := sizeToEndOffset(r, 128)
	if err != nil {
		return "", fmt.Errorf("error determining read size to ID3v1 header: %v", err)
	}

	// TODO: remove this check?????
	if n < 0 {
		return "", fmt.Errorf("file size must be greater than 128 bytes for MP3: %v bytes", n)
	}

	h := sha1.New()
	_, err = io.CopyN(h, r, n)
	if err != nil {
		return "", fmt.Errorf("error reading %v bytes: %v", n, err)
	}
	return hashSum(h), nil
}

// SumFLAC costructs a checksum of the FLAC audio file data provided by the io.ReadSeeker (ignores
// metadata fields).
func SumFLAC(r io.ReadSeeker) (string, error) {
	flac, err := readString(r, 4)
	if err != nil {
		return "", err
	}
	if flac != "fLaC" {
		return "", errors.New("expected 'fLaC'")
	}

	for {
		last, err := skipFLACMetadataBlock(r)
		if err != nil {
			return "", err
		}

		if last {
			break
		}
	}

	h := sha1.New()
	_, err = io.Copy(h, r)
	if err != nil {
		return "", fmt.Errorf("error reading data bytes from FLAC: %v", err)
	}
	return hashSum(h), nil
}

func skipFLACMetadataBlock(r io.ReadSeeker) (last bool, err error) {
	blockHeader, err := readBytes(r, 1)
	if err != nil {
		return
	}

	if getBit(blockHeader[0], 7) {
		blockHeader[0] ^= (1 << 7)
		last = true
	}

	blockLen, err := readInt(r, 3)
	if err != nil {
		return
	}

	_, err = r.Seek(int64(blockLen), os.SEEK_CUR)
	return
}

func hashSum(h hash.Hash) string {
	return fmt.Sprintf("%x", h.Sum([]byte{}))
}
<commit_msg>Use switch in place of multiple if statements.<commit_after>package tag

import (
	"crypto/sha1"
	"encoding/binary"
	"errors"
	"fmt"
	"hash"
	"io"
	"os"
)

// Sum creates a checksum of the audio file data provided by the io.ReadSeeker which is metadata
// (ID3, MP4) invariant.
func Sum(r io.ReadSeeker) (string, error) {
	b, err := readBytes(r, 11)
	if err != nil {
		return "", err
	}

	_, err = r.Seek(-11, os.SEEK_CUR)
	if err != nil {
		return "", fmt.Errorf("could not seek back to original position: %v", err)
	}

	switch {
	case string(b[0:4]) == "fLaC":
		return SumFLAC(r)

	case string(b[4:11]) == "ftypM4A":
		return SumAtoms(r)

	case string(b[0:3]) == "ID3":
		return SumID3v2(r)
	}

	h, err := SumID3v1(r)
	if err != nil {
		if err == ErrNotID3v1 {
			return SumAll(r)
		}
		return "", err
	}
	return h, nil
}

// SumAll returns a checksum of the content from the reader (until EOF).
func SumAll(r io.ReadSeeker) (string, error) {
	h := sha1.New()
	_, err := io.Copy(h, r)
	if err != nil {
		return "", nil
	}
	return hashSum(h), nil
}

// SumAtoms constructs a checksum of MP4 audio file data provided by the io.ReadSeeker which is
// metadata invariant.
func SumAtoms(r io.ReadSeeker) (string, error) {
	for {
		var size uint32
		err := binary.Read(r, binary.BigEndian, &size)
		if err != nil {
			if err == io.EOF {
				return "", fmt.Errorf("reached EOF before audio data")
			}
			return "", err
		}

		name, err := readString(r, 4)
		if err != nil {
			return "", err
		}

		switch name {
		case "meta":
			// next_item_id (int32)
			_, err := r.Seek(4, os.SEEK_CUR)
			if err != nil {
				return "", err
			}
			fallthrough

		case "moov", "udta", "ilst":
			continue

		case "mdat": // stop when we get to the data
			h := sha1.New()
			_, err := io.CopyN(h, r, int64(size-8))
			if err != nil {
				return "", fmt.Errorf("error reading audio data: %v", err)
			}
			return hashSum(h), nil
		}

		_, err = r.Seek(int64(size-8), os.SEEK_CUR)
		if err != nil {
			return "", fmt.Errorf("error reading '%v' tag: %v", name, err)
		}
	}
}

func sizeToEndOffset(r io.ReadSeeker, offset int64) (int64, error) {
	n, err := r.Seek(-128, os.SEEK_END)
	if err != nil {
		return 0, fmt.Errorf("error seeking end offset (%d bytes): %v", offset, err)
	}

	_, err = r.Seek(-n, os.SEEK_CUR)
	if err != nil {
		return 0, fmt.Errorf("error seeking back to original position: %v", err)
	}
	return n, nil
}

// SumID3v1 constructs a checksum of MP3 audio file data (assumed to have ID3v1 tags) provided
// by the io.ReadSeeker which is metadata invariant.
func SumID3v1(r io.ReadSeeker) (string, error) {
	n, err := sizeToEndOffset(r, 128)
	if err != nil {
		return "", fmt.Errorf("error determining read size to ID3v1 header: %v", err)
	}

	// TODO: improve this check???
	if n <= 0 {
		return "", fmt.Errorf("file size must be greater than 128 bytes (ID3v1 header size) for MP3")
	}

	h := sha1.New()
	_, err = io.CopyN(h, r, n)
	if err != nil {
		return "", fmt.Errorf("error reading %v bytes: %v", n, err)
	}
	return hashSum(h), nil
}

// SumID3v2 constructs a checksum of MP3 audio file data (assumed to have ID3v2 tags) provided by the
// io.ReadSeeker which is metadata invariant.
func SumID3v2(r io.ReadSeeker) (string, error) {
	header, err := readID3v2Header(r)
	if err != nil {
		return "", fmt.Errorf("error reading ID3v2 header: %v", err)
	}

	_, err = r.Seek(int64(header.Size), os.SEEK_CUR)
	if err != nil {
		return "", fmt.Errorf("error seeking to end of ID3V2 header: %v", err)
	}

	n, err := sizeToEndOffset(r, 128)
	if err != nil {
		return "", fmt.Errorf("error determining read size to ID3v1 header: %v", err)
	}

	// TODO: remove this check?????
	if n < 0 {
		return "", fmt.Errorf("file size must be greater than 128 bytes for MP3: %v bytes", n)
	}

	h := sha1.New()
	_, err = io.CopyN(h, r, n)
	if err != nil {
		return "", fmt.Errorf("error reading %v bytes: %v", n, err)
	}
	return hashSum(h), nil
}

// SumFLAC costructs a checksum of the FLAC audio file data provided by the io.ReadSeeker (ignores
// metadata fields).
func SumFLAC(r io.ReadSeeker) (string, error) {
	flac, err := readString(r, 4)
	if err != nil {
		return "", err
	}
	if flac != "fLaC" {
		return "", errors.New("expected 'fLaC'")
	}

	for {
		last, err := skipFLACMetadataBlock(r)
		if err != nil {
			return "", err
		}

		if last {
			break
		}
	}

	h := sha1.New()
	_, err = io.Copy(h, r)
	if err != nil {
		return "", fmt.Errorf("error reading data bytes from FLAC: %v", err)
	}
	return hashSum(h), nil
}

func skipFLACMetadataBlock(r io.ReadSeeker) (last bool, err error) {
	blockHeader, err := readBytes(r, 1)
	if err != nil {
		return
	}

	if getBit(blockHeader[0], 7) {
		blockHeader[0] ^= (1 << 7)
		last = true
	}

	blockLen, err := readInt(r, 3)
	if err != nil {
		return
	}

	_, err = r.Seek(int64(blockLen), os.SEEK_CUR)
	return
}

func hashSum(h hash.Hash) string {
	return fmt.Sprintf("%x", h.Sum([]byte{}))
}
<|endoftext|>


<commit_before>package amazonec2

import (
	"testing"

	"github.com/docker/machine/drivers/amazonec2/amz"
)

var (
	securityGroup = amz.SecurityGroup{
		GroupName: "test-group",
		GroupId:   "12345",
		VpcId:     "12345",
	}
)

const (
	testSshPort    = 22
	testDockerPort = 2376
)

func TestConfigureSecurityGroupPermissionsEmpty(t *testing.T) {
	group := securityGroup
	perms := configureSecurityGroupPermissions(&group)
	if len(perms) != 2 {
		t.Fatalf("expected 2 permissions; received %d", len(perms))
	}
}

func TestConfigureSecurityGroupPermissionsSshOnly(t *testing.T) {
	group := securityGroup

	group.IpPermissions = []amz.IpPermission{
		{
			IpProtocol: "tcp",
			FromPort:   testSshPort,
			ToPort:     testSshPort,
		},
	}

	perms := configureSecurityGroupPermissions(&group)
	if len(perms) != 1 {
		t.Fatalf("expected 1 permission; received %d", len(perms))
	}

	receivedPort := perms[0].FromPort
	if receivedPort != testDockerPort {
		t.Fatalf("expected permission on port %d; received port %d", testDockerPort, receivedPort)
	}
}

func TestConfigureSecurityGroupPermissionsDockerOnly(t *testing.T) {
	group := securityGroup

	group.IpPermissions = []amz.IpPermission{
		{
			IpProtocol: "tcp",
			FromPort:   testDockerPort,
			ToPort:     testDockerPort,
		},
	}

	perms := configureSecurityGroupPermissions(&group)
	if len(perms) != 1 {
		t.Fatalf("expected 1 permission; received %d", len(perms))
	}

	receivedPort := perms[0].FromPort
	if receivedPort != testSshPort {
		t.Fatalf("expected permission on port %d; received port %d", testSshPort, receivedPort)
	}
}

func TestConfigureSecurityGroupPermissionsDockerAndSsh(t *testing.T) {
	group := securityGroup

	group.IpPermissions = []amz.IpPermission{
		{
			IpProtocol: "tcp",
			FromPort:   testSshPort,
			ToPort:     testSshPort,
		},
		{
			IpProtocol: "tcp",
			FromPort:   testDockerPort,
			ToPort:     testDockerPort,
		},
	}

	perms := configureSecurityGroupPermissions(&group)
	if len(perms) != 0 {
		t.Fatalf("expected 0 permissions; received %d", len(perms))
	}
}

func TestAwsRegionList(t *testing.T) {
}

func TestValidateAwsRegionValid(t *testing.T) {
	regions := []string{"eu-west-1", "eu-central-1"}

	for _, v := range regions {
		r, err := validateAwsRegion(v)
		if err != nil {
			t.Fatal(err)
		}

		if v != r {
			t.Fatal("Wrong region returned")
		}
	}
}

func TestValidateAwsRegionInvalid(t *testing.T) {
	regions := []string{"eu-west-2", "eu-central-2"}

	for _, v := range regions {
		r, err := validateAwsRegion(v)
		if err == nil {
			t.Fatal("No error returned")
		}

		if v == r {
			t.Fatal("Wrong region returned")
		}
	}
}
<commit_msg>ec2: update tests for configuring ip permissions<commit_after>package amazonec2

import (
	"io/ioutil"
	"os"
	"testing"

	"github.com/docker/machine/drivers/amazonec2/amz"
)

const (
	testSshPort           = 22
	testDockerPort        = 2376
	testStoreDir          = ".store-test"
	machineTestName       = "test-host"
	machineTestDriverName = "none"
	machineTestStorePath  = "/test/path"
	machineTestCaCert     = "test-cert"
	machineTestPrivateKey = "test-key"
)

var (
	securityGroup = amz.SecurityGroup{
		GroupName: "test-group",
		GroupId:   "12345",
		VpcId:     "12345",
	}
)

type DriverOptionsMock struct {
	Data map[string]interface{}
}

func (d DriverOptionsMock) String(key string) string {
	return d.Data[key].(string)
}

func (d DriverOptionsMock) Int(key string) int {
	return d.Data[key].(int)
}

func (d DriverOptionsMock) Bool(key string) bool {
	return d.Data[key].(bool)
}

func cleanup() error {
	return os.RemoveAll(testStoreDir)
}

func getTestStorePath() (string, error) {
	tmpDir, err := ioutil.TempDir("", "machine-test-")
	if err != nil {
		return "", err
	}
	os.Setenv("MACHINE_STORAGE_PATH", tmpDir)
	return tmpDir, nil
}

func getDefaultTestDriverFlags() *DriverOptionsMock {
	return &DriverOptionsMock{
		Data: map[string]interface{}{
			"name":                     "test",
			"url":                      "unix:///var/run/docker.sock",
			"swarm":                    false,
			"swarm-host":               "",
			"swarm-master":             false,
			"swarm-discovery":          "",
			"amazonec2-ami":            "ami-12345",
			"amazonec2-access-key":     "abcdefg",
			"amazonec2-secret-key":     "12345",
			"amazonec2-session-token":  "",
			"amazonec2-instance-type":  "t1.micro",
			"amazonec2-vpc-id":         "vpc-12345",
			"amazonec2-subnet-id":      "subnet-12345",
			"amazonec2-security-group": "docker-machine-test",
			"amazonec2-region":         "us-east-1",
			"amazonec2-zone":           "e",
			"amazonec2-root-size":      10,
		},
	}
}

func getTestDriver() (*Driver, error) {
	storePath, err := getTestStorePath()
	if err != nil {
		return nil, err
	}
	defer cleanup()

	d, err := NewDriver(machineTestName, storePath, machineTestCaCert, machineTestPrivateKey)
	if err != nil {
		return nil, err
	}
	d.SetConfigFromFlags(getDefaultTestDriverFlags())
	drv := d.(*Driver)
	return drv, nil
}

func TestConfigureSecurityGroupPermissionsEmpty(t *testing.T) {
	d, err := getTestDriver()
	if err != nil {
		t.Fatal(err)
	}
	defer cleanup()

	group := securityGroup
	perms := d.configureSecurityGroupPermissions(&group)
	if len(perms) != 2 {
		t.Fatalf("expected 2 permissions; received %d", len(perms))
	}
}

func TestConfigureSecurityGroupPermissionsSshOnly(t *testing.T) {
	d, err := getTestDriver()
	if err != nil {
		t.Fatal(err)
	}
	defer cleanup()

	group := securityGroup

	group.IpPermissions = []amz.IpPermission{
		{
			IpProtocol: "tcp",
			FromPort:   testSshPort,
			ToPort:     testSshPort,
		},
	}

	perms := d.configureSecurityGroupPermissions(&group)
	if len(perms) != 1 {
		t.Fatalf("expected 1 permission; received %d", len(perms))
	}

	receivedPort := perms[0].FromPort
	if receivedPort != testDockerPort {
		t.Fatalf("expected permission on port %d; received port %d", testDockerPort, receivedPort)
	}
}

func TestConfigureSecurityGroupPermissionsDockerOnly(t *testing.T) {
	d, err := getTestDriver()
	if err != nil {
		t.Fatal(err)
	}
	defer cleanup()

	group := securityGroup

	group.IpPermissions = []amz.IpPermission{
		{
			IpProtocol: "tcp",
			FromPort:   testDockerPort,
			ToPort:     testDockerPort,
		},
	}

	perms := d.configureSecurityGroupPermissions(&group)
	if len(perms) != 1 {
		t.Fatalf("expected 1 permission; received %d", len(perms))
	}

	receivedPort := perms[0].FromPort
	if receivedPort != testSshPort {
		t.Fatalf("expected permission on port %d; received port %d", testSshPort, receivedPort)
	}
}

func TestConfigureSecurityGroupPermissionsDockerAndSsh(t *testing.T) {
	d, err := getTestDriver()
	if err != nil {
		t.Fatal(err)
	}
	defer cleanup()

	group := securityGroup

	group.IpPermissions = []amz.IpPermission{
		{
			IpProtocol: "tcp",
			FromPort:   testSshPort,
			ToPort:     testSshPort,
		},
		{
			IpProtocol: "tcp",
			FromPort:   testDockerPort,
			ToPort:     testDockerPort,
		},
	}

	perms := d.configureSecurityGroupPermissions(&group)
	if len(perms) != 0 {
		t.Fatalf("expected 0 permissions; received %d", len(perms))
	}
}

func TestAwsRegionList(t *testing.T) {
}

func TestValidateAwsRegionValid(t *testing.T) {
	regions := []string{"eu-west-1", "eu-central-1"}

	for _, v := range regions {
		r, err := validateAwsRegion(v)
		if err != nil {
			t.Fatal(err)
		}

		if v != r {
			t.Fatal("Wrong region returned")
		}
	}
}

func TestValidateAwsRegionInvalid(t *testing.T) {
	regions := []string{"eu-west-2", "eu-central-2"}

	for _, v := range regions {
		r, err := validateAwsRegion(v)
		if err == nil {
			t.Fatal("No error returned")
		}

		if v == r {
			t.Fatal("Wrong region returned")
		}
	}
}
<|endoftext|>