<commit_before>// Project gonder.
// Author Supme
// Copyright Supme 2016
// License http://opensource.org/licenses/MIT MIT License
//
//  THE SOFTWARE AND DOCUMENTATION ARE PROVIDED "AS IS" WITHOUT WARRANTY OF
//  ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
//  IMPLIED WARRANTIES OF MERCHANTABILITY AND/OR FITNESS FOR A PARTICULAR
//  PURPOSE.
//
// Please see the License.txt file for more information.
//
package models

import (
	"log"
	"net"
	"strconv"
	"strings"
	"sync"
	"time"
)

type mxStor struct {
	records []*net.MX
	update  time.Time
}

var mx = struct {
	stor map[string]mxStor
	sync.Mutex
}{
	stor: make(map[string]mxStor),
}

func DomainGetMX(domain string) ([]*net.MX, error) {
	var (
		record []*net.MX
		err    error
	)

	if Config.DnsCache {
		mx.Lock()
		defer mx.Unlock()
		if _, ok := mx.stor[domain]; !ok || time.Since(mx.stor[domain].update) > 15*time.Minute {
			record, err = net.LookupMX(domain)
			if err == nil {
				mx.stor[domain] = mxStor{
					records: record,
					update:  time.Now(),
				}
			}
		} else {
			record = mx.stor[domain].records
		}
	} else {
		record, err = net.LookupMX(domain)
	}

	return record, err
}

type (
	profileData struct {
		iface, host          string
		streamNow, streamMax int
		lastUpdate           time.Time
	}
)

var (
	profileStor  = map[int]profileData{}
	profileGroup = map[int]int{}
	profileMutex sync.Mutex
)

func ProfileNext(id int) (int, string, string) {
	var res profileData

	profileMutex.Lock()

	// Если есть в массиве, недавно обновлялось
	_, ok := profileStor[id]
	if ok && time.Since(profileStor[id].lastUpdate) < 60*time.Second {

		// Если это группа кампаний
		if strings.ToLower(strings.TrimSpace(profileStor[id].host)) == "group" {
			if _, gok := profileGroup[id]; !gok {
				profileGroup[id] = 0
			}
			gIfaces := strings.Split(profileStor[id].iface, ",")
			if profileGroup[id]+1 > len(gIfaces) {
				profileGroup[id] = 0
			}
			i, e := strconv.Atoi(strings.TrimSpace(gIfaces[profileGroup[id]]))
			if e != nil {
				log.Print(e)
			}
			profileGroup[id]++

			profileMutex.Unlock()
			return ProfileNext(i)
		}

		// Не достигли максимума потоков
		if profileStor[id].streamNow < profileStor[id].streamMax {
			res = profileStor[id]
			res.streamNow++
			profileStor[id] = res
			profileMutex.Unlock()
			return id, res.iface, res.host
		} else {
			// достигли максимума потоков, ждём освобождения
			profileMutex.Unlock()
			for !profileCheck(id) {
			}
			return ProfileNext(id)
		}
	}

	// В остальных случаях обновляем данные
	err := Db.QueryRow("SELECT `iface`,`host`,`stream` FROM `profile` WHERE `id`=?", id).Scan(&res.iface, &res.host, &res.streamMax)
	if err != nil {
		log.Print(err)
	}
	// если уже существовало, сохраним
	if ok {
		res.streamNow = profileStor[id].streamNow
	}
	res.lastUpdate = time.Now()
	profileStor[id] = res

	// и повторяем действие
	profileMutex.Unlock()
	return ProfileNext(id)

}

func profileCheck(id int) bool {
	var free bool
	profileMutex.Lock()
	free = profileStor[id].streamNow < profileStor[id].streamMax
	profileMutex.Unlock()
	return free
}

func ProfileFree(id int) {
	var res profileData

	profileMutex.Lock()
	res = profileStor[id]
	res.streamNow--
	profileStor[id] = res
	profileMutex.Unlock()
	log.Println("profile id =", id, " connection count =", res.streamNow)
}
<commit_msg>dns cache fix problem dns error resolv<commit_after>// Project gonder.
// Author Supme
// Copyright Supme 2016
// License http://opensource.org/licenses/MIT MIT License
//
//  THE SOFTWARE AND DOCUMENTATION ARE PROVIDED "AS IS" WITHOUT WARRANTY OF
//  ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
//  IMPLIED WARRANTIES OF MERCHANTABILITY AND/OR FITNESS FOR A PARTICULAR
//  PURPOSE.
//
// Please see the License.txt file for more information.
//
package models

import (
	"log"
	"net"
	"strconv"
	"strings"
	"sync"
	"time"
)

type mxStor struct {
	records []*net.MX
	update  time.Time
}

var mx = struct {
	stor map[string]mxStor
	sync.Mutex
}{
	stor: make(map[string]mxStor),
}

func DomainGetMX(domain string) ([]*net.MX, error) {
	var (
		record []*net.MX
		err    error
	)

	if Config.DnsCache {
		mx.Lock()
		defer mx.Unlock()
		if _, ok := mx.stor[domain]; !ok || time.Since(mx.stor[domain].update) > 15*time.Minute {
			record, err = net.LookupMX(domain)
			if err == nil {
				mx.stor[domain] = mxStor{
					records: record,
					update:  time.Now(),
				}
			} else if  _, ok := mx.stor[domain]; ok {
				record = mx.stor[domain].records
			}
		} else {
			record = mx.stor[domain].records
		}
	} else {
		record, err = net.LookupMX(domain)
	}

	return record, err
}

type (
	profileData struct {
		iface, host          string
		streamNow, streamMax int
		lastUpdate           time.Time
	}
)

var (
	profileStor  = map[int]profileData{}
	profileGroup = map[int]int{}
	profileMutex sync.Mutex
)

func ProfileNext(id int) (int, string, string) {
	var res profileData

	profileMutex.Lock()

	// Если есть в массиве, недавно обновлялось
	_, ok := profileStor[id]
	if ok && time.Since(profileStor[id].lastUpdate) < 60*time.Second {

		// Если это группа кампаний
		if strings.ToLower(strings.TrimSpace(profileStor[id].host)) == "group" {
			if _, gok := profileGroup[id]; !gok {
				profileGroup[id] = 0
			}
			gIfaces := strings.Split(profileStor[id].iface, ",")
			if profileGroup[id]+1 > len(gIfaces) {
				profileGroup[id] = 0
			}
			i, e := strconv.Atoi(strings.TrimSpace(gIfaces[profileGroup[id]]))
			if e != nil {
				log.Print(e)
			}
			profileGroup[id]++

			profileMutex.Unlock()
			return ProfileNext(i)
		}

		// Не достигли максимума потоков
		if profileStor[id].streamNow < profileStor[id].streamMax {
			res = profileStor[id]
			res.streamNow++
			profileStor[id] = res
			profileMutex.Unlock()
			return id, res.iface, res.host
		} else {
			// достигли максимума потоков, ждём освобождения
			profileMutex.Unlock()
			for !profileCheck(id) {
			}
			return ProfileNext(id)
		}
	}

	// В остальных случаях обновляем данные
	err := Db.QueryRow("SELECT `iface`,`host`,`stream` FROM `profile` WHERE `id`=?", id).Scan(&res.iface, &res.host, &res.streamMax)
	if err != nil {
		log.Print(err)
	}
	// если уже существовало, сохраним
	if ok {
		res.streamNow = profileStor[id].streamNow
	}
	res.lastUpdate = time.Now()
	profileStor[id] = res

	// и повторяем действие
	profileMutex.Unlock()
	return ProfileNext(id)

}

func profileCheck(id int) bool {
	var free bool
	profileMutex.Lock()
	free = profileStor[id].streamNow < profileStor[id].streamMax
	profileMutex.Unlock()
	return free
}

func ProfileFree(id int) {
	var res profileData

	profileMutex.Lock()
	res = profileStor[id]
	res.streamNow--
	profileStor[id] = res
	profileMutex.Unlock()
	log.Println("profile id =", id, " connection count =", res.streamNow)
}
<|endoftext|>


<commit_before>// Copyright 2015 Unknwon
//
// Licensed under the Apache License, Version 2.0 (the "License"): you may
// not use this file except in compliance with the License. You may obtain
// a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
// License for the specific language governing permissions and limitations
// under the License.

package models

import (
	"bytes"
	"errors"
	"fmt"
	"path"
	"strings"
	"time"

	"github.com/Unknwon/com"

	"github.com/Unknwon/gowalker/modules/base"
	"github.com/Unknwon/gowalker/modules/setting"
)

var (
	ErrEmptyPackagePath     = errors.New("Package import path is empty")
	ErrPackageNotFound      = errors.New("Package does not found")
	ErrPackageVersionTooOld = errors.New("Package version is too old")
)

// PkgInfo represents the package information.
type PkgInfo struct {
	ID         int64  `xorm:"pk autoincr"`
	Name       string `xorm:"-"`
	ImportPath string `xorm:"UNIQUE"`
	Etag       string

	ProjectPath string
	ViewDirPath string
	Synopsis    string

	IsCmd       bool
	IsCgo       bool
	IsGoRepo    bool
	IsGoSubrepo bool
	IsGaeRepo   bool

	PkgVer int

	Priority int `xorm:" NOT NULL"`
	Views    int64
	// Indicate how many JS should be downloaded(JsNum=total num - 1)
	JsNum int

	ImportNum int64
	ImportIDs string `xorm:"import_ids LONGTEXT"`
	// Import num usually is small so save it to reduce a database query.
	ImportPaths string `xorm:"LONGTEXT"`

	RefNum int64
	RefIDs string `xorm:"ref_ids LONGTEXT"`

	Subdirs string `xorm:"TEXT"`

	LastView int64 `xorm:"-"`
	Created  int64
}

func (p *PkgInfo) JSPath() string {
	return path.Join(setting.DocsJsPath, p.ImportPath) + ".js"
}

// CanRefresh returns true if package is available to refresh.
func (p *PkgInfo) CanRefresh() bool {
	return time.Now().UTC().Add(-1*setting.RefreshInterval).Unix() > p.Created
}

// GetRefs returns a list of packages that import this one.
func (p *PkgInfo) GetRefs() []*PkgInfo {
	pinfos := make([]*PkgInfo, 0, p.RefNum)
	refIDs := strings.Split(p.RefIDs, "|")
	for i := range refIDs {
		if len(refIDs[i]) == 0 {
			continue
		}

		id := com.StrTo(refIDs[i][1:]).MustInt64()
		if pinfo, _ := GetPkgInfoById(id); pinfo != nil {
			pinfo.Name = path.Base(pinfo.ImportPath)
			pinfos = append(pinfos, pinfo)
		}
	}
	return pinfos
}

// PACKAGE_VER is modified when previously stored packages are invalid.
const PACKAGE_VER = 1

// PkgRef represents temporary reference information of a package.
type PkgRef struct {
	ID         int64  `xorm:"pk autoincr"`
	ImportPath string `xorm:"UNIQUE"`
	RefNum     int64
	RefIDs     string `xorm:"ref_ids LONGTEXT"`
}

func updatePkgRef(pid int64, refPath string) error {
	if base.IsGoRepoPath(refPath) ||
		refPath == "C" ||
		refPath[1] == '.' {
		return nil
	}

	ref := new(PkgRef)
	has, err := x.Where("import_path=?", refPath).Get(ref)
	if err != nil {
		return fmt.Errorf("get PkgRef: %v", err)
	}

	queryStr := "$" + com.ToStr(pid) + "|"
	if !has {
		if _, err = x.Insert(&PkgRef{
			ImportPath: refPath,
			RefNum:     1,
			RefIDs:     queryStr,
		}); err != nil {
			return fmt.Errorf("insert PkgRef: %v", err)
		}
		return nil
	}

	i := strings.Index(ref.RefIDs, queryStr)
	if i > -1 {
		return nil
	}

	ref.RefIDs += queryStr
	ref.RefNum++
	_, err = x.Id(ref.ID).AllCols().Update(ref)
	return err
}

// checkRefs checks if given packages are still referencing this one.
func checkRefs(pinfo *PkgInfo) {
	var buf bytes.Buffer
	pinfo.RefNum = 0
	refIDs := strings.Split(pinfo.RefIDs, "|")
	for i := range refIDs {
		if len(refIDs[i]) == 0 {
			continue
		}

		fmt.Println(com.StrTo(refIDs[i][1:]).MustInt64())
		pkg, _ := GetPkgInfoById(com.StrTo(refIDs[i][1:]).MustInt64())
		if pkg == nil {
			continue
		}

		if strings.Index(pkg.ImportIDs, "$"+com.ToStr(pinfo.ID)+"|") == -1 {
			continue
		}

		buf.WriteString("$")
		buf.WriteString(com.ToStr(pkg.ID))
		buf.WriteString("|")
		pinfo.RefNum++
	}
	pinfo.RefIDs = buf.String()
}

// updateRef updates or crates corresponding reference import information.
func updateRef(pid int64, refPath string) (int64, error) {
	pinfo, err := GetPkgInfo(refPath)
	if err != nil && pinfo == nil {
		if err == ErrPackageNotFound ||
			err == ErrPackageVersionTooOld {
			// Package hasn't existed yet, save to temporary place.
			return 0, updatePkgRef(pid, refPath)
		}
		return 0, fmt.Errorf("GetPkgInfo(%s): %v", refPath, err)
	}

	// Check if reference information has beed recorded.
	queryStr := "$" + com.ToStr(pid) + "|"
	i := strings.Index(pinfo.RefIDs, queryStr)
	if i > -1 {
		return pinfo.ID, nil
	}

	// Add new as needed.
	pinfo.RefIDs += queryStr
	pinfo.RefNum++
	_, err = x.Id(pinfo.ID).AllCols().Update(pinfo)
	return pinfo.ID, err
}

// SavePkgInfo saves package information.
func SavePkgInfo(pinfo *PkgInfo, updateRefs bool) (err error) {
	pinfo.PkgVer = PACKAGE_VER

	switch {
	case pinfo.IsGaeRepo:
		pinfo.Priority = 70
	case pinfo.IsGoSubrepo:
		pinfo.Priority = 80
	case pinfo.IsGoRepo:
		pinfo.Priority = 99
	}

	// Create or update package info itself.
	// Note(Unknwon): do this because we need ID field later.
	if pinfo.ID == 0 {
		pinfo.Views = 1

		// First time created, check PkgRef.
		ref := new(PkgRef)
		has, err := x.Where("import_path=?", pinfo.ImportPath).Get(ref)
		if err != nil {
			return fmt.Errorf("get PkgRef: %v", err)
		} else if has {
			pinfo.RefNum = ref.RefNum
			pinfo.RefIDs = ref.RefIDs
			if _, err = x.Id(ref.ID).Delete(ref); err != nil {
				return fmt.Errorf("delete PkgRef: %v", err)
			}
		}

		_, err = x.Insert(pinfo)
	} else {
		_, err = x.Id(pinfo.ID).AllCols().Update(pinfo)
	}
	if err != nil {
		return fmt.Errorf("update package info: %v", err)
	}

	// Update package import references.
	// Note(Unknwon): I just don't see the value of who imports STD
	//	when you don't even import and uses what objects.
	if updateRefs && !pinfo.IsGoRepo {
		var buf bytes.Buffer
		paths := strings.Split(pinfo.ImportPaths, "|")
		for i := range paths {
			if base.IsGoRepoPath(paths[i]) {
				continue
			}

			refID, err := updateRef(pinfo.ID, paths[i])
			if err != nil {
				return fmt.Errorf("updateRef: %v", err)
			} else if refID == 0 {
				continue
			}
			buf.WriteString("$")
			buf.WriteString(com.ToStr(refID))
			buf.WriteString("|")
		}
		pinfo.ImportIDs = buf.String()

		// Check packages who import this is still importing.
		checkRefs(pinfo)
		_, err = x.Id(pinfo.ID).AllCols().Update(pinfo)
		return err
	}
	return nil
}

// GetPkgInfo returns package information by given import path.
func GetPkgInfo(importPath string) (*PkgInfo, error) {
	if len(importPath) == 0 {
		return nil, ErrEmptyPackagePath
	}

	pinfo := new(PkgInfo)
	has, err := x.Where("import_path=?", importPath).Get(pinfo)
	if err != nil {
		return nil, err
	} else if !has {
		return nil, ErrPackageNotFound
	} else if pinfo.PkgVer < PACKAGE_VER {
		pinfo.Etag = ""
		return pinfo, ErrPackageVersionTooOld
	}

	if !com.IsFile(pinfo.JSPath()) {
		pinfo.Etag = ""
		return pinfo, ErrPackageVersionTooOld
	}

	return pinfo, nil
}

// GetSubPkgs returns sub-projects by given sub-directories.
func GetSubPkgs(importPath string, dirs []string) []*PkgInfo {
	pinfos := make([]*PkgInfo, 0, len(dirs))
	for _, dir := range dirs {
		if len(dir) == 0 {
			continue
		}

		fullPath := importPath + "/" + dir
		if pinfo, err := GetPkgInfo(fullPath); err == nil {
			pinfo.Name = dir
			pinfos = append(pinfos, pinfo)
		} else {
			pinfos = append(pinfos, &PkgInfo{
				Name:       dir,
				ImportPath: fullPath,
			})
		}
	}
	return pinfos
}

// GetPkgInfosByPaths returns a list of packages by given import paths.
func GetPkgInfosByPaths(paths []string) []*PkgInfo {
	pinfos := make([]*PkgInfo, 0, len(paths))
	for _, p := range paths {
		if len(p) == 0 {
			continue
		}

		if pinfo, err := GetPkgInfo(p); err == nil {
			pinfo.Name = path.Base(p)
			pinfos = append(pinfos, pinfo)
		} else {
			pinfos = append(pinfos, &PkgInfo{
				Name:       path.Base(p),
				ImportPath: p,
			})
		}
	}
	return pinfos
}

// GetPkgInfoById returns package information by given ID.
func GetPkgInfoById(id int64) (*PkgInfo, error) {
	pinfo := new(PkgInfo)
	has, err := x.Id(id).Get(pinfo)
	if err != nil {
		return nil, err
	} else if !has {
		return nil, ErrPackageNotFound
	} else if pinfo.PkgVer < PACKAGE_VER {
		return pinfo, ErrPackageVersionTooOld
	}

	if !com.IsFile(pinfo.JSPath()) {
		return pinfo, ErrPackageVersionTooOld
	}

	return pinfo, nil
}

// SearchPkgInfo searches package information by given keyword.
func SearchPkgInfo(limit int, keyword string) ([]*PkgInfo, error) {
	if len(keyword) == 0 {
		return nil, nil
	}
	pkgs := make([]*PkgInfo, 0, limit)
	return pkgs, x.Limit(limit).Desc("priority").Desc("views").Where("import_path like ?", "%"+keyword+"%").Find(&pkgs)
}
<commit_msg>fix pkg_ref<commit_after>// Copyright 2015 Unknwon
//
// Licensed under the Apache License, Version 2.0 (the "License"): you may
// not use this file except in compliance with the License. You may obtain
// a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
// License for the specific language governing permissions and limitations
// under the License.

package models

import (
	"bytes"
	"errors"
	"fmt"
	"path"
	"strings"
	"time"

	"github.com/Unknwon/com"

	"github.com/Unknwon/gowalker/modules/base"
	"github.com/Unknwon/gowalker/modules/setting"
)

var (
	ErrEmptyPackagePath     = errors.New("Package import path is empty")
	ErrPackageNotFound      = errors.New("Package does not found")
	ErrPackageVersionTooOld = errors.New("Package version is too old")
)

// PkgInfo represents the package information.
type PkgInfo struct {
	ID         int64  `xorm:"pk autoincr"`
	Name       string `xorm:"-"`
	ImportPath string `xorm:"UNIQUE"`
	Etag       string

	ProjectPath string
	ViewDirPath string
	Synopsis    string

	IsCmd       bool
	IsCgo       bool
	IsGoRepo    bool
	IsGoSubrepo bool
	IsGaeRepo   bool

	PkgVer int

	Priority int `xorm:" NOT NULL"`
	Views    int64
	// Indicate how many JS should be downloaded(JsNum=total num - 1)
	JsNum int

	ImportNum int64
	ImportIDs string `xorm:"import_ids LONGTEXT"`
	// Import num usually is small so save it to reduce a database query.
	ImportPaths string `xorm:"LONGTEXT"`

	RefNum int64
	RefIDs string `xorm:"ref_ids LONGTEXT"`

	Subdirs string `xorm:"TEXT"`

	LastView int64 `xorm:"-"`
	Created  int64
}

func (p *PkgInfo) JSPath() string {
	return path.Join(setting.DocsJsPath, p.ImportPath) + ".js"
}

// CanRefresh returns true if package is available to refresh.
func (p *PkgInfo) CanRefresh() bool {
	return time.Now().UTC().Add(-1*setting.RefreshInterval).Unix() > p.Created
}

// GetRefs returns a list of packages that import this one.
func (p *PkgInfo) GetRefs() []*PkgInfo {
	pinfos := make([]*PkgInfo, 0, p.RefNum)
	refIDs := strings.Split(p.RefIDs, "|")
	for i := range refIDs {
		if len(refIDs[i]) == 0 {
			continue
		}

		id := com.StrTo(refIDs[i][1:]).MustInt64()
		if pinfo, _ := GetPkgInfoById(id); pinfo != nil {
			pinfo.Name = path.Base(pinfo.ImportPath)
			pinfos = append(pinfos, pinfo)
		}
	}
	return pinfos
}

// PACKAGE_VER is modified when previously stored packages are invalid.
const PACKAGE_VER = 1

// PkgRef represents temporary reference information of a package.
type PkgRef struct {
	ID         int64  `xorm:"pk autoincr"`
	ImportPath string `xorm:"UNIQUE"`
	RefNum     int64
	RefIDs     string `xorm:"ref_ids LONGTEXT"`
}

func updatePkgRef(pid int64, refPath string) error {
	if base.IsGoRepoPath(refPath) ||
		refPath == "C" ||
		refPath[1] == '.' ||
		!base.IsValidRemotePath(refPath) {
		return nil
	}

	ref := new(PkgRef)
	has, err := x.Where("import_path=?", refPath).Get(ref)
	if err != nil {
		return fmt.Errorf("get PkgRef: %v", err)
	}

	queryStr := "$" + com.ToStr(pid) + "|"
	if !has {
		if _, err = x.Insert(&PkgRef{
			ImportPath: refPath,
			RefNum:     1,
			RefIDs:     queryStr,
		}); err != nil {
			return fmt.Errorf("insert PkgRef: %v", err)
		}
		return nil
	}

	i := strings.Index(ref.RefIDs, queryStr)
	if i > -1 {
		return nil
	}

	ref.RefIDs += queryStr
	ref.RefNum++
	_, err = x.Id(ref.ID).AllCols().Update(ref)
	return err
}

// checkRefs checks if given packages are still referencing this one.
func checkRefs(pinfo *PkgInfo) {
	var buf bytes.Buffer
	pinfo.RefNum = 0
	refIDs := strings.Split(pinfo.RefIDs, "|")
	for i := range refIDs {
		if len(refIDs[i]) == 0 {
			continue
		}

		fmt.Println(com.StrTo(refIDs[i][1:]).MustInt64())
		pkg, _ := GetPkgInfoById(com.StrTo(refIDs[i][1:]).MustInt64())
		if pkg == nil {
			continue
		}

		if strings.Index(pkg.ImportIDs, "$"+com.ToStr(pinfo.ID)+"|") == -1 {
			continue
		}

		buf.WriteString("$")
		buf.WriteString(com.ToStr(pkg.ID))
		buf.WriteString("|")
		pinfo.RefNum++
	}
	pinfo.RefIDs = buf.String()
}

// updateRef updates or crates corresponding reference import information.
func updateRef(pid int64, refPath string) (int64, error) {
	pinfo, err := GetPkgInfo(refPath)
	if err != nil && pinfo == nil {
		if err == ErrPackageNotFound ||
			err == ErrPackageVersionTooOld {
			// Package hasn't existed yet, save to temporary place.
			return 0, updatePkgRef(pid, refPath)
		}
		return 0, fmt.Errorf("GetPkgInfo(%s): %v", refPath, err)
	}

	// Check if reference information has beed recorded.
	queryStr := "$" + com.ToStr(pid) + "|"
	i := strings.Index(pinfo.RefIDs, queryStr)
	if i > -1 {
		return pinfo.ID, nil
	}

	// Add new as needed.
	pinfo.RefIDs += queryStr
	pinfo.RefNum++
	_, err = x.Id(pinfo.ID).AllCols().Update(pinfo)
	return pinfo.ID, err
}

// SavePkgInfo saves package information.
func SavePkgInfo(pinfo *PkgInfo, updateRefs bool) (err error) {
	pinfo.PkgVer = PACKAGE_VER

	switch {
	case pinfo.IsGaeRepo:
		pinfo.Priority = 70
	case pinfo.IsGoSubrepo:
		pinfo.Priority = 80
	case pinfo.IsGoRepo:
		pinfo.Priority = 99
	}

	// Create or update package info itself.
	// Note(Unknwon): do this because we need ID field later.
	if pinfo.ID == 0 {
		pinfo.Views = 1

		// First time created, check PkgRef.
		ref := new(PkgRef)
		has, err := x.Where("import_path=?", pinfo.ImportPath).Get(ref)
		if err != nil {
			return fmt.Errorf("get PkgRef: %v", err)
		} else if has {
			pinfo.RefNum = ref.RefNum
			pinfo.RefIDs = ref.RefIDs
			if _, err = x.Id(ref.ID).Delete(ref); err != nil {
				return fmt.Errorf("delete PkgRef: %v", err)
			}
		}

		_, err = x.Insert(pinfo)
	} else {
		_, err = x.Id(pinfo.ID).AllCols().Update(pinfo)
	}
	if err != nil {
		return fmt.Errorf("update package info: %v", err)
	}

	// Update package import references.
	// Note(Unknwon): I just don't see the value of who imports STD
	//	when you don't even import and uses what objects.
	if updateRefs && !pinfo.IsGoRepo {
		var buf bytes.Buffer
		paths := strings.Split(pinfo.ImportPaths, "|")
		for i := range paths {
			if base.IsGoRepoPath(paths[i]) {
				continue
			}

			refID, err := updateRef(pinfo.ID, paths[i])
			if err != nil {
				return fmt.Errorf("updateRef: %v", err)
			} else if refID == 0 {
				continue
			}
			buf.WriteString("$")
			buf.WriteString(com.ToStr(refID))
			buf.WriteString("|")
		}
		pinfo.ImportIDs = buf.String()

		// Check packages who import this is still importing.
		checkRefs(pinfo)
		_, err = x.Id(pinfo.ID).AllCols().Update(pinfo)
		return err
	}
	return nil
}

// GetPkgInfo returns package information by given import path.
func GetPkgInfo(importPath string) (*PkgInfo, error) {
	if len(importPath) == 0 {
		return nil, ErrEmptyPackagePath
	}

	pinfo := new(PkgInfo)
	has, err := x.Where("import_path=?", importPath).Get(pinfo)
	if err != nil {
		return nil, err
	} else if !has {
		return nil, ErrPackageNotFound
	} else if pinfo.PkgVer < PACKAGE_VER {
		pinfo.Etag = ""
		return pinfo, ErrPackageVersionTooOld
	}

	if !com.IsFile(pinfo.JSPath()) {
		pinfo.Etag = ""
		return pinfo, ErrPackageVersionTooOld
	}

	return pinfo, nil
}

// GetSubPkgs returns sub-projects by given sub-directories.
func GetSubPkgs(importPath string, dirs []string) []*PkgInfo {
	pinfos := make([]*PkgInfo, 0, len(dirs))
	for _, dir := range dirs {
		if len(dir) == 0 {
			continue
		}

		fullPath := importPath + "/" + dir
		if pinfo, err := GetPkgInfo(fullPath); err == nil {
			pinfo.Name = dir
			pinfos = append(pinfos, pinfo)
		} else {
			pinfos = append(pinfos, &PkgInfo{
				Name:       dir,
				ImportPath: fullPath,
			})
		}
	}
	return pinfos
}

// GetPkgInfosByPaths returns a list of packages by given import paths.
func GetPkgInfosByPaths(paths []string) []*PkgInfo {
	pinfos := make([]*PkgInfo, 0, len(paths))
	for _, p := range paths {
		if len(p) == 0 {
			continue
		}

		if pinfo, err := GetPkgInfo(p); err == nil {
			pinfo.Name = path.Base(p)
			pinfos = append(pinfos, pinfo)
		} else {
			pinfos = append(pinfos, &PkgInfo{
				Name:       path.Base(p),
				ImportPath: p,
			})
		}
	}
	return pinfos
}

// GetPkgInfoById returns package information by given ID.
func GetPkgInfoById(id int64) (*PkgInfo, error) {
	pinfo := new(PkgInfo)
	has, err := x.Id(id).Get(pinfo)
	if err != nil {
		return nil, err
	} else if !has {
		return nil, ErrPackageNotFound
	} else if pinfo.PkgVer < PACKAGE_VER {
		return pinfo, ErrPackageVersionTooOld
	}

	if !com.IsFile(pinfo.JSPath()) {
		return pinfo, ErrPackageVersionTooOld
	}

	return pinfo, nil
}

// SearchPkgInfo searches package information by given keyword.
func SearchPkgInfo(limit int, keyword string) ([]*PkgInfo, error) {
	if len(keyword) == 0 {
		return nil, nil
	}
	pkgs := make([]*PkgInfo, 0, limit)
	return pkgs, x.Limit(limit).Desc("priority").Desc("views").Where("import_path like ?", "%"+keyword+"%").Find(&pkgs)
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"bytes"
	"flag"
	"fmt"
	"io"
	"log"
	"os"
	"strings"

	"github.com/Shopify/sarama"
)

var (
	fPeers     = flag.String("peers", os.Getenv("KAFKA_PEERS"), "List of Kafka peer addresses (Defaults to KAFKA_PEERS env)")
	fPartition = flag.Int("partition", -1, "Partition to send on")
	fTopic     = flag.String("topic", "", "Topic to send on")
	fVerbose   = flag.Bool("verbose", false, "Print message details")

	logger = log.New(os.Stderr, "producer", log.LstdFlags)
)

func flagbad(f string, i ...interface{}) {
	fmt.Fprintf(os.Stderr, f, i...)
	flag.PrintDefaults()
	os.Exit(1)
}

func main() {
	flag.Parse()
	if *fPeers == "" {
		flagbad("-peers is empty\n")
	}
	if *fPartition == -1 {
		flagbad("-partition is empty\n")
	}
	if *fTopic == "" {
		flagbad("-topic is empty\n")
	}

	config := sarama.NewConfig()
	config.Producer.RequiredAcks = sarama.WaitForAll
	config.ClientID = "kafkaproc.producer"

	client, err := sarama.NewClient(strings.Split(*fPeers, ","), config)
	if err != nil {
		logger.Panicf("Creating sarama client: %v", err)
	}
	defer client.Close()

	producer, err := sarama.NewSyncProducerFromClient(client)
	if err != nil {
		logger.Panicf("Creating sarama syncproducer: %v", err)
	}
	defer producer.Close()

	br := bufio.NewReader(os.Stdin)
	for {
		line, err := br.ReadBytes('\n')
		if err == io.EOF && len(line) == 0 {
			break
		}
		if err != nil {
			logger.Panicf("Reading from stdin: %v", err)
		}
		line = bytes.TrimRight(line, "\n")
		message := &sarama.ProducerMessage{
			Topic:     *fTopic,
			Partition: int32(*fPartition),
			Value:     sarama.ByteEncoder(line),
		}
		part, offset, err := producer.SendMessage(message)
		if err != nil {
			logger.Panicf("Sending message: %v", err)
		}
		if *fVerbose {
			fmt.Printf("send (len=%d, part=%d, offset=%d)\n", len(line), part, offset)
		}
	}
}
<commit_msg>Cannot set partition<commit_after>package main

import (
	"bufio"
	"bytes"
	"flag"
	"fmt"
	"io"
	"log"
	"os"
	"strings"

	"github.com/Shopify/sarama"
)

var (
	fPeers   = flag.String("peers", os.Getenv("KAFKA_PEERS"), "List of Kafka peer addresses (Defaults to KAFKA_PEERS env)")
	fTopic   = flag.String("topic", "", "Topic to send on")
	fVerbose = flag.Bool("verbose", false, "Print message details")

	logger = log.New(os.Stderr, "producer", log.LstdFlags)
)

func flagbad(f string, i ...interface{}) {
	fmt.Fprintf(os.Stderr, f, i...)
	flag.PrintDefaults()
	os.Exit(1)
}

func main() {
	flag.Parse()
	if *fPeers == "" {
		flagbad("-peers is empty\n")
	}
	if *fTopic == "" {
		flagbad("-topic is empty\n")
	}

	config := sarama.NewConfig()
	config.Producer.RequiredAcks = sarama.WaitForAll
	config.ClientID = "kafkaproc.producer"

	client, err := sarama.NewClient(strings.Split(*fPeers, ","), config)
	if err != nil {
		logger.Panicf("Creating sarama client: %v", err)
	}
	defer client.Close()

	producer, err := sarama.NewSyncProducerFromClient(client)
	if err != nil {
		logger.Panicf("Creating sarama syncproducer: %v", err)
	}
	defer producer.Close()

	br := bufio.NewReader(os.Stdin)
	for {
		line, err := br.ReadBytes('\n')
		if err == io.EOF && len(line) == 0 {
			break
		}
		if err != nil {
			logger.Panicf("Reading from stdin: %v", err)
		}
		line = bytes.TrimRight(line, "\n")
		message := &sarama.ProducerMessage{
			Topic: *fTopic,
			Value: sarama.ByteEncoder(line),
		}
		part, offset, err := producer.SendMessage(message)
		if err != nil {
			logger.Panicf("Sending message: %v", err)
		}
		if *fVerbose {
			fmt.Printf("send (len=%d, part=%d, offset=%d)\n", len(line), part, offset)
		}
	}
}
<|endoftext|>


<commit_before>package api

import (
	"encoding/json"
	"io/ioutil"
	"net/http"
	"time"

	"github.com/julienschmidt/httprouter"

	"github.com/octavore/press/db"
	"github.com/octavore/press/db/bolt"
	"github.com/octavore/press/proto/press/api"
	"github.com/octavore/press/proto/press/models"
	"github.com/octavore/press/server/router"
	"github.com/octavore/press/util/errors"
)

func (m *Module) getPage(par httprouter.Params, fn func(*models.Page) error) (*models.Page, error) {
	uuid := par.ByName("uuid")
	if uuid == "" {
		return nil, router.ErrNotFound
	}
	page, err := m.DB.GetPage(uuid)
	if _, ok := err.(bolt.ErrNoKey); ok {
		return nil, router.ErrNotFound
	}
	if err != nil {
		return nil, err
	}
	err = fn(page)
	if err != nil {
		return nil, err
	}
	return page, nil
}

func (m *Module) GetPage(rw http.ResponseWriter, req *http.Request, par httprouter.Params) error {
	page, err := m.getPage(par, func(*models.Page) error { return nil })
	if err != nil {
		return err
	}
	return router.Proto(rw, page)
}

func (m *Module) ListPages(rw http.ResponseWriter, req *http.Request, par httprouter.Params) error {
	pages, err := m.DB.ListPages()
	if err != nil {
		return err
	}
	db.SortPagesByUpdatedAt(pages, false)
	return router.Proto(rw, &api.ListPageResponse{
		Pages: pages,
	})
}

func (m *Module) UpdatePage(rw http.ResponseWriter, req *http.Request, par httprouter.Params) error {
	page := &models.Page{}
	b, err := ioutil.ReadAll(req.Body)
	if err != nil {
		return errors.Wrap(err)
	}
	err = json.Unmarshal(b, page)
	if err != nil {
		return err
	}
	err = m.DB.UpdatePage(page)
	if err != nil {
		return err
	}
	return router.Proto(rw, page)
}

func (m *Module) PublishPage(rw http.ResponseWriter, req *http.Request, par httprouter.Params) error {
	page, err := m.getPage(par, func(page *models.Page) error {
		now := time.Now().Unix()
		page.PublishedAt = &now
		err := m.DB.UpdatePage(page)
		if err != nil {
			return err
		}
		return m.Content.ReloadRouter()
	})
	if err != nil {
		return err
	}
	return router.Proto(rw, page)
}

func (m *Module) UnpublishPage(rw http.ResponseWriter, req *http.Request, par httprouter.Params) error {
	page, err := m.getPage(par, func(page *models.Page) error {
		page.PublishedAt = nil
		err := m.DB.UpdatePage(page)
		if err != nil {
			return err
		}
		return m.Content.ReloadRouter()
	})
	if err != nil {
		return err
	}
	return router.Proto(rw, page)
}

func (m *Module) DeletePage(rw http.ResponseWriter, req *http.Request, par httprouter.Params) error {
	uuid := par.ByName("uuid")
	page, err := m.DB.GetPage(uuid)
	if _, ok := err.(bolt.ErrNoKey); ok {
		return router.ErrNotFound
	}
	if err != nil {
		return err
	}
	return m.DB.DeletePage(page)
}
<commit_msg>server: Add comments to pages api, and fix unmarshalling and published at.<commit_after>package api

import (
	"net/http"
	"time"

	"github.com/golang/protobuf/jsonpb"
	"github.com/julienschmidt/httprouter"

	"github.com/octavore/press/db"
	"github.com/octavore/press/db/bolt"
	"github.com/octavore/press/proto/press/api"
	"github.com/octavore/press/proto/press/models"
	"github.com/octavore/press/server/router"
)

func (m *Module) getPage(par httprouter.Params, fn func(*models.Page) error) (*models.Page, error) {
	uuid := par.ByName("uuid")
	if uuid == "" {
		return nil, router.ErrNotFound
	}
	page, err := m.DB.GetPage(uuid)
	if _, ok := err.(bolt.ErrNoKey); ok {
		return nil, router.ErrNotFound
	}
	if err != nil {
		return nil, err
	}
	err = fn(page)
	if err != nil {
		return nil, err
	}
	return page, nil
}

// GetPage gets a page by UUID.
// todo: nest response?
func (m *Module) GetPage(rw http.ResponseWriter, req *http.Request, par httprouter.Params) error {
	page, err := m.getPage(par, func(*models.Page) error { return nil })
	if err != nil {
		return err
	}
	return router.Proto(rw, page)
}

// ListPages returns all pages, sorted by updated at.
// todo: pagination, filtering
func (m *Module) ListPages(rw http.ResponseWriter, req *http.Request, par httprouter.Params) error {
	pages, err := m.DB.ListPages()
	if err != nil {
		return err
	}
	db.SortPagesByUpdatedAt(pages, false)
	return router.Proto(rw, &api.ListPageResponse{
		Pages: pages,
	})
}

// UpdatePage saves the given page to the DB.
// todo: nest response?
func (m *Module) UpdatePage(rw http.ResponseWriter, req *http.Request, par httprouter.Params) error {
	page := &models.Page{}
	// use jsonpb.unmarshal to correct unmarshal int64 e.g. PublishedAt
	err := jsonpb.Unmarshal(req.Body, page)
	if err != nil {
		return err
	}
	err = m.DB.UpdatePage(page)
	if err != nil {
		return err
	}
	return router.Proto(rw, page)
}

// PublishPage sets the published time on a page to the current time.
func (m *Module) PublishPage(rw http.ResponseWriter, req *http.Request, par httprouter.Params) error {
	page, err := m.getPage(par, func(page *models.Page) error {
		// already published
		if page.PublishedAt != nil {
			return nil
		}

		// set published at to current time
		now := time.Now().Unix()
		page.PublishedAt = &now
		err := m.DB.UpdatePage(page)
		if err != nil {
			return err
		}
		return m.Content.ReloadRouter()
	})
	if err != nil {
		return err
	}
	return router.Proto(rw, page)
}

// UnpublishPage sets published at to null, effectively unpublishing the page.
func (m *Module) UnpublishPage(rw http.ResponseWriter, req *http.Request, par httprouter.Params) error {
	page, err := m.getPage(par, func(page *models.Page) error {
		page.PublishedAt = nil
		err := m.DB.UpdatePage(page)
		if err != nil {
			return err
		}
		return m.Content.ReloadRouter()
	})
	if err != nil {
		return err
	}
	return router.Proto(rw, page)
}

// DeletePage deletes the given page.
func (m *Module) DeletePage(rw http.ResponseWriter, req *http.Request, par httprouter.Params) error {
	uuid := par.ByName("uuid")
	page, err := m.DB.GetPage(uuid)
	if _, ok := err.(bolt.ErrNoKey); ok {
		return router.ErrNotFound
	}
	if err != nil {
		return err
	}
	return m.DB.DeletePage(page)
}
<|endoftext|>


<commit_before>package chroot

import (
	"github.com/mitchellh/packer/packer"
	"io"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"syscall"
)

// Communicator is a special communicator that works by executing
// commands locally but within a chroot.
type Communicator struct {
	Chroot string
}

func (c *Communicator) Start(cmd *packer.RemoteCmd) error {
	chrootCmdPath, err := exec.LookPath("chroot")
	if err != nil {
		return err
	}

	localCmd := exec.Command(chrootCmdPath, c.Chroot, "/bin/sh", "-c", cmd.Command)
	localCmd.Stdin = cmd.Stdin
	localCmd.Stdout = cmd.Stdout
	localCmd.Stderr = cmd.Stderr
	log.Printf("Executing: %s %#v", localCmd.Path, localCmd.Args)
	if err := localCmd.Start(); err != nil {
		return err
	}

	go func() {
		exitStatus := 0
		if err := localCmd.Wait(); err != nil {
			if exitErr, ok := err.(*exec.ExitError); ok {
				exitStatus = 1

				// There is no process-independent way to get the REAL
				// exit status so we just try to go deeper.
				if status, ok := exitErr.Sys().(syscall.WaitStatus); ok {
					exitStatus = status.ExitStatus()
				}
			}
		}

		cmd.SetExited(exitStatus)
	}()

	return nil
}

func (c *Communicator) Upload(dst string, r io.Reader) error {
	dst = filepath.Join(c.Chroot, dst)
	log.Printf("Uploading to chroot dir: %s", dst)
	f, err := os.Create(dst)
	if err != nil {
		return err
	}
	defer f.Close()

	if _, err := io.Copy(f, r); err != nil {
		return err
	}

	return nil
}

func (c *Communicator) Download(src string, w io.Writer) error {
	src = filepath.Join(c.Chroot, src)
	log.Printf("Downloading from chroot dir: %s", src)
	f, err := os.Open(src)
	if err != nil {
		return err
	}
	defer f.Close()

	if _, err := io.Copy(w, f); err != nil {
		return err
	}

	return nil
}
<commit_msg>builder/amazon/chroot: log the exit code for the chroot communicator<commit_after>package chroot

import (
	"github.com/mitchellh/packer/packer"
	"io"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"syscall"
)

// Communicator is a special communicator that works by executing
// commands locally but within a chroot.
type Communicator struct {
	Chroot string
}

func (c *Communicator) Start(cmd *packer.RemoteCmd) error {
	chrootCmdPath, err := exec.LookPath("chroot")
	if err != nil {
		return err
	}

	localCmd := exec.Command(chrootCmdPath, c.Chroot, "/bin/sh", "-c", cmd.Command)
	localCmd.Stdin = cmd.Stdin
	localCmd.Stdout = cmd.Stdout
	localCmd.Stderr = cmd.Stderr
	log.Printf("Executing: %s %#v", localCmd.Path, localCmd.Args)
	if err := localCmd.Start(); err != nil {
		return err
	}

	go func() {
		exitStatus := 0
		if err := localCmd.Wait(); err != nil {
			if exitErr, ok := err.(*exec.ExitError); ok {
				exitStatus = 1

				// There is no process-independent way to get the REAL
				// exit status so we just try to go deeper.
				if status, ok := exitErr.Sys().(syscall.WaitStatus); ok {
					exitStatus = status.ExitStatus()
				}
			}
		}

		log.Printf(
			"Chroot executation ended with '%d': '%s'",
			exitStatus, cmd.Command)
		cmd.SetExited(exitStatus)
	}()

	return nil
}

func (c *Communicator) Upload(dst string, r io.Reader) error {
	dst = filepath.Join(c.Chroot, dst)
	log.Printf("Uploading to chroot dir: %s", dst)
	f, err := os.Create(dst)
	if err != nil {
		return err
	}
	defer f.Close()

	if _, err := io.Copy(f, r); err != nil {
		return err
	}

	return nil
}

func (c *Communicator) Download(src string, w io.Writer) error {
	src = filepath.Join(c.Chroot, src)
	log.Printf("Downloading from chroot dir: %s", src)
	f, err := os.Open(src)
	if err != nil {
		return err
	}
	defer f.Close()

	if _, err := io.Copy(w, f); err != nil {
		return err
	}

	return nil
}
<|endoftext|>


<commit_before>package common

import (
	"fmt"

	"github.com/mitchellh/goamz/ec2"
	"github.com/mitchellh/packer/packer"
)

// BlockDevice
type BlockDevice struct {
	DeleteOnTermination bool   `mapstructure:"delete_on_termination"`
	DeviceName          string `mapstructure:"device_name"`
	Encrypted           bool   `mapstructure:"encrypted"`
	IOPS                int64  `mapstructure:"iops"`
	NoDevice            bool   `mapstructure:"no_device"`
	SnapshotId          string `mapstructure:"snapshot_id"`
	VirtualName         string `mapstructure:"virtual_name"`
	VolumeType          string `mapstructure:"volume_type"`
	VolumeSize          int64  `mapstructure:"volume_size"`
}

type BlockDevices struct {
	AMIMappings    []BlockDevice `mapstructure:"ami_block_device_mappings"`
	LaunchMappings []BlockDevice `mapstructure:"launch_block_device_mappings"`
}

func buildBlockDevices(b []BlockDevice) []ec2.BlockDeviceMapping {
	var blockDevices []ec2.BlockDeviceMapping

	for _, blockDevice := range b {
		blockDevices = append(blockDevices, ec2.BlockDeviceMapping{
			DeviceName:          blockDevice.DeviceName,
			VirtualName:         blockDevice.VirtualName,
			SnapshotId:          blockDevice.SnapshotId,
			VolumeType:          blockDevice.VolumeType,
			VolumeSize:          blockDevice.VolumeSize,
			DeleteOnTermination: blockDevice.DeleteOnTermination,
			IOPS:                blockDevice.IOPS,
			NoDevice:            blockDevice.NoDevice,
			Encrypted:           blockDevice.Encrypted,
		})
	}
	return blockDevices
}

func (b *BlockDevices) Prepare(t *packer.ConfigTemplate) []error {
	if t == nil {
		var err error
		t, err = packer.NewConfigTemplate()
		if err != nil {
			return []error{err}
		}
	}

	lists := map[string][]BlockDevice{
		"ami_block_device_mappings":    b.AMIMappings,
		"launch_block_device_mappings": b.LaunchMappings,
	}

	var errs []error
	for outer, bds := range lists {
		for i, bd := range bds {
			templates := map[string]*string{
				"device_name":  &bd.DeviceName,
				"snapshot_id":  &bd.SnapshotId,
				"virtual_name": &bd.VirtualName,
				"volume_type":  &bd.VolumeType,
			}

			errs := make([]error, 0)
			for n, ptr := range templates {
				var err error
				*ptr, err = t.Process(*ptr, nil)
				if err != nil {
					errs = append(
						errs, fmt.Errorf(
							"Error processing %s[%d].%s: %s",
							outer, i, n, err))
				}
			}
		}
	}

	if len(errs) > 0 {
		return errs
	}

	return nil
}

func (b *BlockDevices) BuildAMIDevices() []ec2.BlockDeviceMapping {
	return buildBlockDevices(b.AMIMappings)
}

func (b *BlockDevices) BuildLaunchDevices() []ec2.BlockDeviceMapping {
	return buildBlockDevices(b.LaunchMappings)
}
<commit_msg>Use an index loop as range loops over copies, not references<commit_after>package common

import (
	"fmt"

	"github.com/mitchellh/goamz/ec2"
	"github.com/mitchellh/packer/packer"
)

// BlockDevice
type BlockDevice struct {
	DeleteOnTermination bool   `mapstructure:"delete_on_termination"`
	DeviceName          string `mapstructure:"device_name"`
	Encrypted           bool   `mapstructure:"encrypted"`
	IOPS                int64  `mapstructure:"iops"`
	NoDevice            bool   `mapstructure:"no_device"`
	SnapshotId          string `mapstructure:"snapshot_id"`
	VirtualName         string `mapstructure:"virtual_name"`
	VolumeType          string `mapstructure:"volume_type"`
	VolumeSize          int64  `mapstructure:"volume_size"`
}

type BlockDevices struct {
	AMIMappings    []BlockDevice `mapstructure:"ami_block_device_mappings"`
	LaunchMappings []BlockDevice `mapstructure:"launch_block_device_mappings"`
}

func buildBlockDevices(b []BlockDevice) []ec2.BlockDeviceMapping {
	var blockDevices []ec2.BlockDeviceMapping

	for _, blockDevice := range b {
		blockDevices = append(blockDevices, ec2.BlockDeviceMapping{
			DeviceName:          blockDevice.DeviceName,
			VirtualName:         blockDevice.VirtualName,
			SnapshotId:          blockDevice.SnapshotId,
			VolumeType:          blockDevice.VolumeType,
			VolumeSize:          blockDevice.VolumeSize,
			DeleteOnTermination: blockDevice.DeleteOnTermination,
			IOPS:                blockDevice.IOPS,
			NoDevice:            blockDevice.NoDevice,
			Encrypted:           blockDevice.Encrypted,
		})
	}
	return blockDevices
}

func (b *BlockDevices) Prepare(t *packer.ConfigTemplate) []error {
	if t == nil {
		var err error
		t, err = packer.NewConfigTemplate()
		if err != nil {
			return []error{err}
		}
	}

	lists := map[string][]BlockDevice{
		"ami_block_device_mappings":    b.AMIMappings,
		"launch_block_device_mappings": b.LaunchMappings,
	}

	var errs []error
	for outer, bds := range lists {
		for i := 0; i < len(bds); i++ {
			templates := map[string]*string{
				"device_name":  &bds[i].DeviceName,
				"snapshot_id":  &bds[i].SnapshotId,
				"virtual_name": &bds[i].VirtualName,
				"volume_type":  &bds[i].VolumeType,
			}

			errs := make([]error, 0)
			for n, ptr := range templates {
				var err error
				*ptr, err = t.Process(*ptr, nil)
				if err != nil {
					errs = append(
						errs, fmt.Errorf(
							"Error processing %s[%d].%s: %s",
							outer, i, n, err))
				}
			}
		}
	}

	if len(errs) > 0 {
		return errs
	}

	return nil
}

func (b *BlockDevices) BuildAMIDevices() []ec2.BlockDeviceMapping {
	return buildBlockDevices(b.AMIMappings)
}

func (b *BlockDevices) BuildLaunchDevices() []ec2.BlockDeviceMapping {
	return buildBlockDevices(b.LaunchMappings)
}
<|endoftext|>


<commit_before>// Package write implements writing basic Go types as external Erlang terms.
package write

import (
	"bytes"
	"fmt"
	t "github.com/goerlang/etf/types"
	"io"
	"math"
	"math/big"
	"reflect"
)

type ErrUnknownType struct {
	t reflect.Type
}

func (e *ErrUnknownType) Error() string {
	return fmt.Sprintf("write: can't encode type \"%s\"", e.t.Name())
}

func Atom(w io.Writer, atom t.Atom) (err error) {
	switch size := len(atom); {
	case size <= 0xff:
		// $sL…
		if _, err = w.Write([]byte{t.EttSmallAtom, byte(size)}); err == nil {
			_, err = w.Write([]byte(atom))
		}

	case size <= 0xffff:
		// $dLL…
		_, err = w.Write([]byte{byte(t.EttAtom), byte(size >> 8), byte(size)})
		if err == nil {
			_, err = w.Write([]byte(atom))
		}

	default:
		err = fmt.Errorf("atom is too big (%d bytes)", size)
	}

	return
}

func BigInt(w io.Writer, x *big.Int) (err error) {
	sign := 0
	if x.Sign() < 0 {
		sign = 1
	}

	bytes := reverse(new(big.Int).Abs(x).Bytes())

	switch size := len(bytes); {
	case size <= 0xff:
		// $nAS…
		_, err = w.Write([]byte{t.EttSmallBig, byte(size), byte(sign)})

	case int(uint32(size)) == size:
		// $oAAAAS…
		_, err = w.Write([]byte{
			t.EttLargeBig,
			byte(size >> 24), byte(size >> 16), byte(size >> 8), byte(size),
			byte(sign),
		})

	default:
		err = fmt.Errorf("bad big int size (%d)", size)
	}

	if err == nil {
		_, err = w.Write(bytes)
	}

	return
}

func Binary(w io.Writer, bytes []byte) (err error) {
	switch size := len(bytes); {
	case int(uint32(size)) == size:
		// $mLLLL…
		data := []byte{
			t.EttBinary,
			byte(size >> 24), byte(size >> 16), byte(size >> 8), byte(size),
		}
		if _, err = w.Write(data); err == nil {
			_, err = w.Write(bytes)
		}

	default:
		err = fmt.Errorf("bad binary size (%d)", size)
	}

	return
}

func Bool(w io.Writer, b bool) (err error) {
	// $sL…
	if b {
		_, err = w.Write([]byte{t.EttSmallAtom, 4, 't', 'r', 'u', 'e'})
	} else {
		_, err = w.Write([]byte{t.EttSmallAtom, 5, 'f', 'a', 'l', 's', 'e'})
	}

	return
}

func Float(w io.Writer, f float64) (err error) {
	if _, err = w.Write([]byte{t.EttNewFloat}); err == nil {
		fb := math.Float64bits(f)
		_, err = w.Write([]byte{
			byte(fb >> 56), byte(fb >> 48), byte(fb >> 40), byte(fb >> 32),
			byte(fb >> 24), byte(fb >> 16), byte(fb >> 8), byte(fb),
		})
	}
	return
}

func Int(w io.Writer, x int64) (err error) {
	switch {
	case x >= 0 && x <= math.MaxUint8:
		// $aI
		_, err = w.Write([]byte{t.EttSmallInteger, byte(x)})

	case x >= math.MinInt32 && x <= math.MaxInt32:
		// $bIIII
		x := int32(x)
		_, err = w.Write([]byte{
			t.EttInteger,
			byte(x >> 24), byte(x >> 16), byte(x >> 8), byte(x),
		})

	default:
		err = BigInt(w, big.NewInt(x))
	}

	return
}

func Uint(w io.Writer, x uint64) (err error) {
	switch {
	case x <= math.MaxUint8:
		// $aI
		_, err = w.Write([]byte{t.EttSmallInteger, byte(x)})

	case x <= math.MaxInt32:
		// $bIIII
		_, err = w.Write([]byte{
			t.EttInteger,
			byte(x >> 24), byte(x >> 16), byte(x >> 8), byte(x),
		})

	default:
		err = BigInt(w, new(big.Int).SetUint64(x))
	}

	return
}

func Pid(w io.Writer, p t.Pid) (err error) {
	if _, err = w.Write([]byte{t.EttPid}); err != nil {
		return
	} else if err = Atom(w, p.Node); err != nil {
		return
	}

	_, err = w.Write([]byte{
		0, 0, byte(p.Id >> 8), byte(p.Id),
		byte(p.Serial >> 24),
		byte(p.Serial >> 16),
		byte(p.Serial >> 8),
		byte(p.Serial),
		p.Creation,
	})

	return
}

func String(w io.Writer, s string) (err error) {
	switch size := len(s); {
	case size <= 0xffff:
		// $kLL…
		_, err = w.Write([]byte{t.EttString, byte(size >> 8), byte(size)})
		if err == nil {
			_, err = w.Write([]byte(s))
		}

	default:
		err = fmt.Errorf("string is too big (%d bytes)", size)
	}

	return
}

func List(w io.Writer, l interface{}) (err error) {
	rv := reflect.ValueOf(l)
	n := rv.Len()
	_, err = w.Write([]byte{
		t.EttList,
		byte(n >> 24),
		byte(n >> 16),
		byte(n >> 8),
		byte(n),
	})

	if err != nil {
		return
	}

	for i := 0; i < n; i++ {
		v := rv.Index(i).Interface()
		if err = Term(w, v); err != nil {
			return
		}
	}

	_, err = w.Write([]byte{t.EttNil})

	return
}

func Record(w io.Writer, r interface{}) (err error) {
	rv := reflect.ValueOf(r)
	n := rv.NumField()
	buf := new(bytes.Buffer)
	arity := 0

	for i := 0; i < n; i++ {
		if f := rv.Field(i); f.CanInterface() {
			if err = Term(buf, f.Interface()); err != nil {
				return
			}
			arity++
		}
	}

	if arity <= math.MaxUint8 {
		_, err = w.Write([]byte{t.EttSmallTuple, byte(arity)})
	} else {
		_, err = w.Write([]byte{
			t.EttLargeTuple,
			byte(arity >> 24),
			byte(arity >> 16),
			byte(arity >> 8),
			byte(arity),
		})
	}

	if err == nil {
		_, err = buf.WriteTo(w)
	}

	return
}

func Ref(w io.Writer, ref t.Ref) (err error) {
	n := len(ref.Id)
	_, err = w.Write([]byte{t.EttNewReference, byte(n >> 8), byte(n)})
	if err != nil {
		return
	}
	if err = Atom(w, ref.Node); err != nil {
		return
	}
	if _, err = w.Write([]byte{ref.Creation}); err != nil {
		return
	}
	for _, v := range ref.Id {
		b := []byte{
			byte(v >> 24),
			byte(v >> 16),
			byte(v >> 8),
			byte(v),
		}
		if _, err = w.Write(b); err != nil {
			return
		}
	}

	return
}

func Tuple(w io.Writer, tuple t.Tuple) (err error) {
	n := len(tuple)
	if n <= math.MaxUint8 {
		_, err = w.Write([]byte{t.EttSmallTuple, byte(n)})
	} else {
		_, err = w.Write([]byte{
			t.EttLargeTuple,
			byte(n >> 24),
			byte(n >> 16),
			byte(n >> 8),
			byte(n),
		})
	}

	if err != nil {
		return
	}

	for _, v := range tuple {
		if err = Term(w, v); err != nil {
			return
		}
	}

	return
}

func Term(w io.Writer, term t.Term) (err error) {
	switch v := term.(type) {
	case bool:
		err = Bool(w, v)
	case int8, int16, int32, int64, int:
		err = Int(w, reflect.ValueOf(term).Int())
	case uint8, uint16, uint32, uint64, uintptr, uint:
		err = Uint(w, reflect.ValueOf(term).Uint())
	case *big.Int:
		err = BigInt(w, v)
	case string:
		err = String(w, v)
	case []byte:
		err = Binary(w, v)
	case float64:
		err = Float(w, v)
	case float32:
		err = Float(w, float64(v))
	case t.Atom:
		err = Atom(w, v)
	case t.Pid:
		err = Pid(w, v)
	case t.Tuple:
		err = Tuple(w, v)
	case t.Ref:
		err = Ref(w, v)
	default:
		rv := reflect.ValueOf(v)
		switch rv.Kind() {
		case reflect.Struct:
			err = Record(w, term)
		case reflect.Array, reflect.Slice:
			err = List(w, term)
		case reflect.Ptr:
			err = Term(w, rv.Elem())
		//case reflect.Map // FIXME
		default:
			err = &ErrUnknownType{rv.Type()}
		}
	}

	return
}

func reverse(b []byte) []byte {
	size := len(b)
	hsize := size >> 1

	for i := 0; i < hsize; i++ {
		b[i], b[size-i-1] = b[size-i-1], b[i]
	}

	return b
}
<commit_msg>write: use math.* consts instead of magic numbers and type conversion<commit_after>// Package write implements writing basic Go types as external Erlang terms.
package write

import (
	"bytes"
	"fmt"
	t "github.com/goerlang/etf/types"
	"io"
	"math"
	"math/big"
	"reflect"
)

type ErrUnknownType struct {
	t reflect.Type
}

func (e *ErrUnknownType) Error() string {
	return fmt.Sprintf("write: can't encode type \"%s\"", e.t.Name())
}

func Atom(w io.Writer, atom t.Atom) (err error) {
	switch size := len(atom); {
	case size <= math.MaxUint8:
		// $sL…
		if _, err = w.Write([]byte{t.EttSmallAtom, byte(size)}); err == nil {
			_, err = io.WriteString(w, string(atom))
		}

	case size <= math.MaxUint16:
		// $dLL…
		_, err = w.Write([]byte{byte(t.EttAtom), byte(size >> 8), byte(size)})
		if err == nil {
			_, err = io.WriteString(w, string(atom))
		}

	default:
		err = fmt.Errorf("atom is too big (%d bytes)", size)
	}

	return
}

func BigInt(w io.Writer, x *big.Int) (err error) {
	sign := 0
	if x.Sign() < 0 {
		sign = 1
	}

	bytes := reverse(new(big.Int).Abs(x).Bytes())

	switch size := len(bytes); {
	case size <= math.MaxUint8:
		// $nAS…
		_, err = w.Write([]byte{t.EttSmallBig, byte(size), byte(sign)})

	case size <= math.MaxUint32:
		// $oAAAAS…
		_, err = w.Write([]byte{
			t.EttLargeBig,
			byte(size >> 24), byte(size >> 16), byte(size >> 8), byte(size),
			byte(sign),
		})

	default:
		err = fmt.Errorf("bad big int size (%d)", size)
	}

	if err == nil {
		_, err = w.Write(bytes)
	}

	return
}

func Binary(w io.Writer, bytes []byte) (err error) {
	switch size := len(bytes); {
	case size <= math.MaxUint32:
		// $mLLLL…
		data := []byte{
			t.EttBinary,
			byte(size >> 24), byte(size >> 16), byte(size >> 8), byte(size),
		}
		if _, err = w.Write(data); err == nil {
			_, err = w.Write(bytes)
		}

	default:
		err = fmt.Errorf("bad binary size (%d)", size)
	}

	return
}

func Bool(w io.Writer, b bool) (err error) {
	// $sL…
	if b {
		_, err = w.Write([]byte{t.EttSmallAtom, 4, 't', 'r', 'u', 'e'})
	} else {
		_, err = w.Write([]byte{t.EttSmallAtom, 5, 'f', 'a', 'l', 's', 'e'})
	}

	return
}

func Float(w io.Writer, f float64) (err error) {
	if _, err = w.Write([]byte{t.EttNewFloat}); err == nil {
		fb := math.Float64bits(f)
		_, err = w.Write([]byte{
			byte(fb >> 56), byte(fb >> 48), byte(fb >> 40), byte(fb >> 32),
			byte(fb >> 24), byte(fb >> 16), byte(fb >> 8), byte(fb),
		})
	}
	return
}

func Int(w io.Writer, x int64) (err error) {
	switch {
	case x >= 0 && x <= math.MaxUint8:
		// $aI
		_, err = w.Write([]byte{t.EttSmallInteger, byte(x)})

	case x >= math.MinInt32 && x <= math.MaxInt32:
		// $bIIII
		x := int32(x)
		_, err = w.Write([]byte{
			t.EttInteger,
			byte(x >> 24), byte(x >> 16), byte(x >> 8), byte(x),
		})

	default:
		err = BigInt(w, big.NewInt(x))
	}

	return
}

func Uint(w io.Writer, x uint64) (err error) {
	switch {
	case x <= math.MaxUint8:
		// $aI
		_, err = w.Write([]byte{t.EttSmallInteger, byte(x)})

	case x <= math.MaxInt32:
		// $bIIII
		_, err = w.Write([]byte{
			t.EttInteger,
			byte(x >> 24), byte(x >> 16), byte(x >> 8), byte(x),
		})

	default:
		err = BigInt(w, new(big.Int).SetUint64(x))
	}

	return
}

func Pid(w io.Writer, p t.Pid) (err error) {
	if _, err = w.Write([]byte{t.EttPid}); err != nil {
		return
	} else if err = Atom(w, p.Node); err != nil {
		return
	}

	_, err = w.Write([]byte{
		0, 0, byte(p.Id >> 8), byte(p.Id),
		byte(p.Serial >> 24),
		byte(p.Serial >> 16),
		byte(p.Serial >> 8),
		byte(p.Serial),
		p.Creation,
	})

	return
}

func String(w io.Writer, s string) (err error) {
	switch size := len(s); {
	case size <= math.MaxUint16:
		// $kLL…
		_, err = w.Write([]byte{t.EttString, byte(size >> 8), byte(size)})
		if err == nil {
			_, err = w.Write([]byte(s))
		}

	default:
		err = fmt.Errorf("string is too big (%d bytes)", size)
	}

	return
}

func List(w io.Writer, l interface{}) (err error) {
	rv := reflect.ValueOf(l)
	n := rv.Len()
	_, err = w.Write([]byte{
		t.EttList,
		byte(n >> 24),
		byte(n >> 16),
		byte(n >> 8),
		byte(n),
	})

	if err != nil {
		return
	}

	for i := 0; i < n; i++ {
		v := rv.Index(i).Interface()
		if err = Term(w, v); err != nil {
			return
		}
	}

	_, err = w.Write([]byte{t.EttNil})

	return
}

func Record(w io.Writer, r interface{}) (err error) {
	rv := reflect.ValueOf(r)
	n := rv.NumField()
	buf := new(bytes.Buffer)
	arity := 0

	for i := 0; i < n; i++ {
		if f := rv.Field(i); f.CanInterface() {
			if err = Term(buf, f.Interface()); err != nil {
				return
			}
			arity++
		}
	}

	if arity <= math.MaxUint8 {
		_, err = w.Write([]byte{t.EttSmallTuple, byte(arity)})
	} else {
		_, err = w.Write([]byte{
			t.EttLargeTuple,
			byte(arity >> 24),
			byte(arity >> 16),
			byte(arity >> 8),
			byte(arity),
		})
	}

	if err == nil {
		_, err = buf.WriteTo(w)
	}

	return
}

func Ref(w io.Writer, ref t.Ref) (err error) {
	n := len(ref.Id)
	_, err = w.Write([]byte{t.EttNewReference, byte(n >> 8), byte(n)})
	if err != nil {
		return
	}
	if err = Atom(w, ref.Node); err != nil {
		return
	}
	if _, err = w.Write([]byte{ref.Creation}); err != nil {
		return
	}
	for _, v := range ref.Id {
		b := []byte{
			byte(v >> 24),
			byte(v >> 16),
			byte(v >> 8),
			byte(v),
		}
		if _, err = w.Write(b); err != nil {
			return
		}
	}

	return
}

func Tuple(w io.Writer, tuple t.Tuple) (err error) {
	n := len(tuple)
	if n <= math.MaxUint8 {
		_, err = w.Write([]byte{t.EttSmallTuple, byte(n)})
	} else {
		_, err = w.Write([]byte{
			t.EttLargeTuple,
			byte(n >> 24),
			byte(n >> 16),
			byte(n >> 8),
			byte(n),
		})
	}

	if err != nil {
		return
	}

	for _, v := range tuple {
		if err = Term(w, v); err != nil {
			return
		}
	}

	return
}

func Term(w io.Writer, term t.Term) (err error) {
	switch v := term.(type) {
	case bool:
		err = Bool(w, v)
	case int8, int16, int32, int64, int:
		err = Int(w, reflect.ValueOf(term).Int())
	case uint8, uint16, uint32, uint64, uintptr, uint:
		err = Uint(w, reflect.ValueOf(term).Uint())
	case *big.Int:
		err = BigInt(w, v)
	case string:
		err = String(w, v)
	case []byte:
		err = Binary(w, v)
	case float64:
		err = Float(w, v)
	case float32:
		err = Float(w, float64(v))
	case t.Atom:
		err = Atom(w, v)
	case t.Pid:
		err = Pid(w, v)
	case t.Tuple:
		err = Tuple(w, v)
	case t.Ref:
		err = Ref(w, v)
	default:
		rv := reflect.ValueOf(v)
		switch rv.Kind() {
		case reflect.Struct:
			err = Record(w, term)
		case reflect.Array, reflect.Slice:
			err = List(w, term)
		case reflect.Ptr:
			err = Term(w, rv.Elem())
		//case reflect.Map // FIXME
		default:
			err = &ErrUnknownType{rv.Type()}
		}
	}

	return
}

func reverse(b []byte) []byte {
	size := len(b)
	hsize := size >> 1

	for i := 0; i < hsize; i++ {
		b[i], b[size-i-1] = b[size-i-1], b[i]
	}

	return b
}
<|endoftext|>


<commit_before>package filer

import (
	"context"
	"fmt"
	"io"
	"math/rand"
	"sync"

	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
	"github.com/chrislusf/seaweedfs/weed/util"
	"github.com/chrislusf/seaweedfs/weed/util/chunk_cache"
	"github.com/chrislusf/seaweedfs/weed/wdclient"
	"github.com/golang/groupcache/singleflight"
)

type ChunkReadAt struct {
	masterClient *wdclient.MasterClient
	chunkViews   []*ChunkView
	lookupFileId LookupFileIdFunctionType
	readerLock   sync.Mutex
	fileSize     int64

	fetchGroup      singleflight.Group
	lastChunkFileId string
	lastChunkData   []byte
	chunkCache      chunk_cache.ChunkCache
}

// var _ = io.ReaderAt(&ChunkReadAt{})

type LookupFileIdFunctionType func(fileId string) (targetUrls []string, err error)

func LookupFn(filerClient filer_pb.FilerClient) LookupFileIdFunctionType {

	vidCache := make(map[string]*filer_pb.Locations)
	var vicCacheLock sync.RWMutex
	return func(fileId string) (targetUrls []string, err error) {
		vid := VolumeId(fileId)
		vicCacheLock.RLock()
		locations, found := vidCache[vid]
		vicCacheLock.RUnlock()

		if !found {
			util.Retry("lookup volume "+vid, func() error {
				err = filerClient.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error {
					resp, err := client.LookupVolume(context.Background(), &filer_pb.LookupVolumeRequest{
						VolumeIds: []string{vid},
					})
					if err != nil {
						return err
					}

					locations = resp.LocationsMap[vid]
					if locations == nil || len(locations.Locations) == 0 {
						glog.V(0).Infof("failed to locate %s", fileId)
						return fmt.Errorf("failed to locate %s", fileId)
					}
					vicCacheLock.Lock()
					vidCache[vid] = locations
					vicCacheLock.Unlock()

					return nil
				})
				return err
			})
		}

		if err != nil {
			return nil, err
		}

		for _, loc := range locations.Locations {
			volumeServerAddress := filerClient.AdjustedUrl(loc)
			targetUrl := fmt.Sprintf("http://%s/%s", volumeServerAddress, fileId)
			targetUrls = append(targetUrls, targetUrl)
		}

		for i := len(targetUrls) - 1; i > 0; i-- {
			j := rand.Intn(i + 1)
			targetUrls[i], targetUrls[j] = targetUrls[j], targetUrls[i]
		}

		return
	}
}

func NewChunkReaderAtFromClient(filerClient filer_pb.FilerClient, chunkViews []*ChunkView, chunkCache chunk_cache.ChunkCache, fileSize int64) *ChunkReadAt {

	return &ChunkReadAt{
		chunkViews:   chunkViews,
		lookupFileId: LookupFn(filerClient),
		chunkCache:   chunkCache,
		fileSize:     fileSize,
	}
}

func (c *ChunkReadAt) ReadAt(p []byte, offset int64) (n int, err error) {

	c.readerLock.Lock()
	defer c.readerLock.Unlock()

	glog.V(4).Infof("ReadAt [%d,%d) of total file size %d bytes %d chunk views", offset, offset+int64(len(p)), c.fileSize, len(c.chunkViews))
	return c.doReadAt(p[n:], offset+int64(n))
}

func (c *ChunkReadAt) doReadAt(p []byte, offset int64) (n int, err error) {

	var buffer []byte
	startOffset, remaining := offset, int64(len(p))
	var nextChunk *ChunkView
	for i, chunk := range c.chunkViews {
		if remaining <= 0 {
			break
		}
		if i+1 < len(c.chunkViews) {
			nextChunk = c.chunkViews[i+1]
		} else {
			nextChunk = nil
		}
		if startOffset < chunk.LogicOffset {
			gap := int(chunk.LogicOffset - startOffset)
			glog.V(4).Infof("zero [%d,%d)", startOffset, startOffset+int64(gap))
			n += int(min(int64(gap), remaining))
			startOffset, remaining = chunk.LogicOffset, remaining-int64(gap)
			if remaining <= 0 {
				break
			}
		}
		// fmt.Printf(">>> doReadAt [%d,%d), chunk[%d,%d)\n", offset, offset+int64(len(p)), chunk.LogicOffset, chunk.LogicOffset+int64(chunk.Size))
		chunkStart, chunkStop := max(chunk.LogicOffset, startOffset), min(chunk.LogicOffset+int64(chunk.Size), startOffset+remaining)
		if chunkStart >= chunkStop {
			continue
		}
		glog.V(4).Infof("read [%d,%d), %d/%d chunk %s [%d,%d)", chunkStart, chunkStop, i, len(c.chunkViews), chunk.FileId, chunk.LogicOffset-chunk.Offset, chunk.LogicOffset-chunk.Offset+int64(chunk.Size))
		buffer, err = c.readFromWholeChunkData(chunk, nextChunk)
		if err != nil {
			glog.Errorf("fetching chunk %+v: %v\n", chunk, err)
			return
		}
		bufferOffset := chunkStart - chunk.LogicOffset + chunk.Offset
		copied := copy(p[startOffset-offset:chunkStop-chunkStart+startOffset-offset], buffer[bufferOffset:bufferOffset+chunkStop-chunkStart])
		n += copied
		startOffset, remaining = startOffset+int64(copied), remaining-int64(copied)
	}

	glog.V(4).Infof("doReadAt [%d,%d), n:%v, err:%v", offset, offset+int64(len(p)), n, err)

	if err == nil && remaining > 0 && c.fileSize > startOffset {
		delta := int(min(remaining, c.fileSize-startOffset))
		glog.V(4).Infof("zero2 [%d,%d) of file size %d bytes", startOffset, startOffset+int64(delta), c.fileSize)
		n += delta
	}

	if err == nil && offset+int64(len(p)) >= c.fileSize {
		err = io.EOF
	}
	// fmt.Printf("~~~ filled %d, err: %v\n\n", n, err)

	return

}

func (c *ChunkReadAt) readFromWholeChunkData(chunkView *ChunkView, nextChunkViews ...*ChunkView) (chunkData []byte, err error) {

	if c.lastChunkFileId == chunkView.FileId {
		return c.lastChunkData, nil
	}

	v, doErr := c.readOneWholeChunk(chunkView)

	if doErr != nil {
		return nil, doErr
	}

	chunkData = v.([]byte)

	c.lastChunkData = chunkData
	c.lastChunkFileId = chunkView.FileId

	for _, nextChunkView := range nextChunkViews {
		if c.chunkCache != nil && nextChunkView != nil {
			go c.readOneWholeChunk(nextChunkView)
		}
	}

	return
}

func (c *ChunkReadAt) readOneWholeChunk(chunkView *ChunkView) (interface{}, error) {

	var err error

	return c.fetchGroup.Do(chunkView.FileId, func() (interface{}, error) {

		glog.V(4).Infof("readFromWholeChunkData %s offset %d [%d,%d) size at least %d", chunkView.FileId, chunkView.Offset, chunkView.LogicOffset, chunkView.LogicOffset+int64(chunkView.Size), chunkView.ChunkSize)

		data := c.chunkCache.GetChunk(chunkView.FileId, chunkView.ChunkSize)
		if data != nil {
			glog.V(4).Infof("cache hit %s [%d,%d)", chunkView.FileId, chunkView.LogicOffset-chunkView.Offset, chunkView.LogicOffset-chunkView.Offset+int64(len(data)))
		} else {
			var err error
			data, err = c.doFetchFullChunkData(chunkView)
			if err != nil {
				return data, err
			}
			c.chunkCache.SetChunk(chunkView.FileId, data)
		}
		return data, err
	})
}

func (c *ChunkReadAt) doFetchFullChunkData(chunkView *ChunkView) ([]byte, error) {

	glog.V(4).Infof("+ doFetchFullChunkData %s", chunkView.FileId)

	data, err := fetchChunk(c.lookupFileId, chunkView.FileId, chunkView.CipherKey, chunkView.IsGzipped)

	glog.V(4).Infof("- doFetchFullChunkData %s", chunkView.FileId)

	return data, err

}
<commit_msg>mount: avoid memory leaking read buffer<commit_after>package filer

import (
	"context"
	"fmt"
	"io"
	"math/rand"
	"sync"

	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
	"github.com/chrislusf/seaweedfs/weed/util"
	"github.com/chrislusf/seaweedfs/weed/util/chunk_cache"
	"github.com/chrislusf/seaweedfs/weed/wdclient"
	"github.com/golang/groupcache/singleflight"
)

type ChunkReadAt struct {
	masterClient *wdclient.MasterClient
	chunkViews   []*ChunkView
	lookupFileId LookupFileIdFunctionType
	readerLock   sync.Mutex
	fileSize     int64

	fetchGroup      singleflight.Group
	chunkCache      chunk_cache.ChunkCache
}

// var _ = io.ReaderAt(&ChunkReadAt{})

type LookupFileIdFunctionType func(fileId string) (targetUrls []string, err error)

func LookupFn(filerClient filer_pb.FilerClient) LookupFileIdFunctionType {

	vidCache := make(map[string]*filer_pb.Locations)
	var vicCacheLock sync.RWMutex
	return func(fileId string) (targetUrls []string, err error) {
		vid := VolumeId(fileId)
		vicCacheLock.RLock()
		locations, found := vidCache[vid]
		vicCacheLock.RUnlock()

		if !found {
			util.Retry("lookup volume "+vid, func() error {
				err = filerClient.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error {
					resp, err := client.LookupVolume(context.Background(), &filer_pb.LookupVolumeRequest{
						VolumeIds: []string{vid},
					})
					if err != nil {
						return err
					}

					locations = resp.LocationsMap[vid]
					if locations == nil || len(locations.Locations) == 0 {
						glog.V(0).Infof("failed to locate %s", fileId)
						return fmt.Errorf("failed to locate %s", fileId)
					}
					vicCacheLock.Lock()
					vidCache[vid] = locations
					vicCacheLock.Unlock()

					return nil
				})
				return err
			})
		}

		if err != nil {
			return nil, err
		}

		for _, loc := range locations.Locations {
			volumeServerAddress := filerClient.AdjustedUrl(loc)
			targetUrl := fmt.Sprintf("http://%s/%s", volumeServerAddress, fileId)
			targetUrls = append(targetUrls, targetUrl)
		}

		for i := len(targetUrls) - 1; i > 0; i-- {
			j := rand.Intn(i + 1)
			targetUrls[i], targetUrls[j] = targetUrls[j], targetUrls[i]
		}

		return
	}
}

func NewChunkReaderAtFromClient(filerClient filer_pb.FilerClient, chunkViews []*ChunkView, chunkCache chunk_cache.ChunkCache, fileSize int64) *ChunkReadAt {

	return &ChunkReadAt{
		chunkViews:   chunkViews,
		lookupFileId: LookupFn(filerClient),
		chunkCache:   chunkCache,
		fileSize:     fileSize,
	}
}

func (c *ChunkReadAt) ReadAt(p []byte, offset int64) (n int, err error) {

	c.readerLock.Lock()
	defer c.readerLock.Unlock()

	glog.V(4).Infof("ReadAt [%d,%d) of total file size %d bytes %d chunk views", offset, offset+int64(len(p)), c.fileSize, len(c.chunkViews))
	return c.doReadAt(p[n:], offset+int64(n))
}

func (c *ChunkReadAt) doReadAt(p []byte, offset int64) (n int, err error) {

	startOffset, remaining := offset, int64(len(p))
	var nextChunk *ChunkView
	for i, chunk := range c.chunkViews {
		if remaining <= 0 {
			break
		}
		if i+1 < len(c.chunkViews) {
			nextChunk = c.chunkViews[i+1]
		} else {
			nextChunk = nil
		}
		if startOffset < chunk.LogicOffset {
			gap := int(chunk.LogicOffset - startOffset)
			glog.V(4).Infof("zero [%d,%d)", startOffset, startOffset+int64(gap))
			n += int(min(int64(gap), remaining))
			startOffset, remaining = chunk.LogicOffset, remaining-int64(gap)
			if remaining <= 0 {
				break
			}
		}
		// fmt.Printf(">>> doReadAt [%d,%d), chunk[%d,%d)\n", offset, offset+int64(len(p)), chunk.LogicOffset, chunk.LogicOffset+int64(chunk.Size))
		chunkStart, chunkStop := max(chunk.LogicOffset, startOffset), min(chunk.LogicOffset+int64(chunk.Size), startOffset+remaining)
		if chunkStart >= chunkStop {
			continue
		}
		glog.V(4).Infof("read [%d,%d), %d/%d chunk %s [%d,%d)", chunkStart, chunkStop, i, len(c.chunkViews), chunk.FileId, chunk.LogicOffset-chunk.Offset, chunk.LogicOffset-chunk.Offset+int64(chunk.Size))
		var buffer []byte
		buffer, err = c.readFromWholeChunkData(chunk, nextChunk)
		if err != nil {
			glog.Errorf("fetching chunk %+v: %v\n", chunk, err)
			return
		}
		bufferOffset := chunkStart - chunk.LogicOffset + chunk.Offset
		copied := copy(p[startOffset-offset:chunkStop-chunkStart+startOffset-offset], buffer[bufferOffset:bufferOffset+chunkStop-chunkStart])
		n += copied
		startOffset, remaining = startOffset+int64(copied), remaining-int64(copied)
	}

	glog.V(4).Infof("doReadAt [%d,%d), n:%v, err:%v", offset, offset+int64(len(p)), n, err)

	if err == nil && remaining > 0 && c.fileSize > startOffset {
		delta := int(min(remaining, c.fileSize-startOffset))
		glog.V(4).Infof("zero2 [%d,%d) of file size %d bytes", startOffset, startOffset+int64(delta), c.fileSize)
		n += delta
	}

	if err == nil && offset+int64(len(p)) >= c.fileSize {
		err = io.EOF
	}
	// fmt.Printf("~~~ filled %d, err: %v\n\n", n, err)

	return

}

func (c *ChunkReadAt) readFromWholeChunkData(chunkView *ChunkView, nextChunkViews ...*ChunkView) (chunkData []byte, err error) {

	v, doErr := c.readOneWholeChunk(chunkView)

	if doErr != nil {
		return nil, doErr
	}

	chunkData = v.([]byte)

	for _, nextChunkView := range nextChunkViews {
		if c.chunkCache != nil && nextChunkView != nil {
			go c.readOneWholeChunk(nextChunkView)
		}
	}

	return
}

func (c *ChunkReadAt) readOneWholeChunk(chunkView *ChunkView) (interface{}, error) {

	var err error

	return c.fetchGroup.Do(chunkView.FileId, func() (interface{}, error) {

		glog.V(4).Infof("readFromWholeChunkData %s offset %d [%d,%d) size at least %d", chunkView.FileId, chunkView.Offset, chunkView.LogicOffset, chunkView.LogicOffset+int64(chunkView.Size), chunkView.ChunkSize)

		data := c.chunkCache.GetChunk(chunkView.FileId, chunkView.ChunkSize)
		if data != nil {
			glog.V(4).Infof("cache hit %s [%d,%d)", chunkView.FileId, chunkView.LogicOffset-chunkView.Offset, chunkView.LogicOffset-chunkView.Offset+int64(len(data)))
		} else {
			var err error
			data, err = c.doFetchFullChunkData(chunkView)
			if err != nil {
				return data, err
			}
			c.chunkCache.SetChunk(chunkView.FileId, data)
		}
		return data, err
	})
}

func (c *ChunkReadAt) doFetchFullChunkData(chunkView *ChunkView) ([]byte, error) {

	glog.V(4).Infof("+ doFetchFullChunkData %s", chunkView.FileId)

	data, err := fetchChunk(c.lookupFileId, chunkView.FileId, chunkView.CipherKey, chunkView.IsGzipped)

	glog.V(4).Infof("- doFetchFullChunkData %s", chunkView.FileId)

	return data, err

}
<|endoftext|>


<commit_before>package server

import (
	"fmt"
	"github.com/centurylinkcloud/clc-go-cli/models/customfields"
	"time"
)

const (
	timeFormat = "2006-01-02 15:04:05"
)

type CreateReq struct {
	Name                   string `valid:"required"`
	Description            string `json:",omitempty"`
	GroupId                string
	GroupName              string `json:",omitempty"`
	SourceServerId         string
	TemplateId             string
	TemplateName           string             `json:",omitempty"`
	IsManagedOs            bool               `json:",omitempty"`
	IsManagedBackup        bool               `json:",omitempty"`
	PrimaryDns             string             `json:",omitempty"`
	SecondaryDns           string             `json:",omitempty"`
	NetworkId              string             `json:",omitempty"`
	IpAddress              string             `json:",omitempty"`
	RootPassword           string             `json:",omitempty"`
	SourceServerPassword   string             `json:",omitempty"`
	Cpu                    int64              `valid:"required"`
	CpuAutoscalePolicyId   string             `json:",omitempty"`
	MemoryGb               int64              `valid:"required"`
	Type                   string             `valid:"required" oneOf:"standard,hyperscale,bareMetal"`
	StorageType            string             `json:",omitempty" oneOf:"standard,premium,hyperscale"`
	AntiAffinityPolicyId   string             `json:",omitempty"`
	AntiAffinityPolicyName string             `json:",omitempty"`
	CustomFields           []customfields.Def `json:",omitempty"`
	AdditionalDisks        []AddDiskRequest   `json:",omitempty"`
	Ttl                    time.Time          `json:"-"`
	TtlString              string             `json:"Ttl,omitempty"`
	Packages               []PackageDef       `json:",omitempty"`
	ConfigurationId        string             `json:",omitempty"`
	OsType                 string             `json:",omitempty"`
}

func (c *CreateReq) Validate() error {
	serverIdValues := []string{c.SourceServerId, c.TemplateId, c.TemplateName}
	numNonEmpty := 0
	for _, item := range serverIdValues {
		if item != "" {
			numNonEmpty++
		}
	}
	if numNonEmpty > 1 || numNonEmpty == 0 {
		return fmt.Errorf("Exactly one parameter from the following: source-server-id, source-server-name, template-id, template-name must be specified.")
	}

	if (c.GroupName == "") == (c.GroupId == "") {
		return fmt.Errorf("Exactly one parameter from the following: group-id, group-name must be specified.")
	}

	if c.Type == "bareMetal" {
		if c.ConfigurationId == "" {
			return fmt.Errorf("ConfigurationId: required for bare metal servers.")
		}
		if c.OsType == "" {
			return fmt.Errorf("OsType: required for bare metal servers.")
		}
	}

	return nil
}

func (c *CreateReq) ApplyDefaultBehaviour() error {
	if c.TemplateId != "" {
		c.SourceServerId = c.TemplateId
	}

	zeroTime := time.Time{}
	if c.Ttl != zeroTime {
		c.TtlString = c.Ttl.Format(timeFormat)
	}
	return nil

	//TODO: implement searching templates by name
	//TODO: implement searching groups by names
}
<commit_msg>Get rid of the --template-id property<commit_after>package server

import (
	"fmt"
	"github.com/centurylinkcloud/clc-go-cli/models/customfields"
	"time"
)

const (
	timeFormat = "2006-01-02 15:04:05"
)

type CreateReq struct {
	Name                   string `valid:"required"`
	Description            string `json:",omitempty"`
	GroupId                string
	GroupName              string `json:",omitempty"`
	SourceServerId         string
	TemplateName           string             `json:",omitempty"`
	IsManagedOs            bool               `json:",omitempty"`
	IsManagedBackup        bool               `json:",omitempty"`
	PrimaryDns             string             `json:",omitempty"`
	SecondaryDns           string             `json:",omitempty"`
	NetworkId              string             `json:",omitempty"`
	IpAddress              string             `json:",omitempty"`
	RootPassword           string             `json:",omitempty"`
	SourceServerPassword   string             `json:",omitempty"`
	Cpu                    int64              `valid:"required"`
	CpuAutoscalePolicyId   string             `json:",omitempty"`
	MemoryGb               int64              `valid:"required"`
	Type                   string             `valid:"required" oneOf:"standard,hyperscale,bareMetal"`
	StorageType            string             `json:",omitempty" oneOf:"standard,premium,hyperscale"`
	AntiAffinityPolicyId   string             `json:",omitempty"`
	AntiAffinityPolicyName string             `json:",omitempty"`
	CustomFields           []customfields.Def `json:",omitempty"`
	AdditionalDisks        []AddDiskRequest   `json:",omitempty"`
	Ttl                    time.Time          `json:"-"`
	TtlString              string             `json:"Ttl,omitempty"`
	Packages               []PackageDef       `json:",omitempty"`
	ConfigurationId        string             `json:",omitempty"`
	OsType                 string             `json:",omitempty"`
}

func (c *CreateReq) Validate() error {
	serverIdValues := []string{c.SourceServerId, c.TemplateName}
	numNonEmpty := 0
	for _, item := range serverIdValues {
		if item != "" {
			numNonEmpty++
		}
	}
	if numNonEmpty > 1 || numNonEmpty == 0 {
		return fmt.Errorf("Exactly one parameter from the following: source-server-id, source-server-name, template-name must be specified.")
	}

	if (c.GroupName == "") == (c.GroupId == "") {
		return fmt.Errorf("Exactly one parameter from the following: group-id, group-name must be specified.")
	}

	if c.Type == "bareMetal" {
		if c.ConfigurationId == "" {
			return fmt.Errorf("ConfigurationId: required for bare metal servers.")
		}
		if c.OsType == "" {
			return fmt.Errorf("OsType: required for bare metal servers.")
		}
	}

	return nil
}

func (c *CreateReq) ApplyDefaultBehaviour() error {
	if c.TemplateName != "" {
		c.SourceServerId = c.TemplateName
	}

	zeroTime := time.Time{}
	if c.Ttl != zeroTime {
		c.TtlString = c.Ttl.Format(timeFormat)
	}
	return nil

	//TODO: implement searching templates by name
	//TODO: implement searching groups by names
}
<|endoftext|>


<commit_before>package master_ui

import (
	"html/template"
)

var StatusTpl = template.Must(template.New("status").Parse(`<!DOCTYPE html>

	<title>SeaweedFS Filer</title>
	<link rel="icon" href="http://7viirv.com1.z0.glb.clouddn.com/seaweed_favicon.png" sizes="32x32" />
	<link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.1/css/bootstrap.min.css">

				SeaweedFS Filer

			{{ range $entry := .Breadcrumbs }}

					{{ $entry.Name }}

			{{ end }}

				{{$path := .Path }}
				{{ range $entry_index, $entry := .Entries }}

					{{if $entry.IsDirectory}}

							{{ $entry.Name }}

					{{else}}

							{{ $entry.Name }}

					{{end}}

					{{if $entry.IsDirectory}}
					{{else}}
						{{ $entry.Mime }}
					{{end}}

					{{if $entry.IsDirectory}}
					{{else}}
						{{ $entry.Size }} bytes

					{{end}}

						{{ $entry.Timestamp.Format "2006-01-02 15:04" }}

				{{ end }}

		{{if .ShouldDisplayLoadMore}}

		Load more

		{{end}}

`))
<commit_msg>readable file size<commit_after>package master_ui

import (
	"html/template"
	"github.com/dustin/go-humanize"
)

var funcMap = template.FuncMap{
	"humanizeBytes": humanize.Bytes,
}

var StatusTpl = template.Must(template.New("status").Funcs(funcMap).Parse(`<!DOCTYPE html>

	<title>SeaweedFS Filer</title>
	<link rel="icon" href="http://7viirv.com1.z0.glb.clouddn.com/seaweed_favicon.png" sizes="32x32" />
	<link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.1/css/bootstrap.min.css">

				SeaweedFS Filer

			{{ range $entry := .Breadcrumbs }}

					{{ $entry.Name }}

			{{ end }}

				{{$path := .Path }}
				{{ range $entry_index, $entry := .Entries }}

					{{if $entry.IsDirectory}}

							{{ $entry.Name }}

					{{else}}

							{{ $entry.Name }}

					{{end}}

					{{if $entry.IsDirectory}}
					{{else}}
						{{ $entry.Mime }}
					{{end}}

					{{if $entry.IsDirectory}}
					{{else}}
						{{ $entry.Size | humanizeBytes }}

					{{end}}

						{{ $entry.Timestamp.Format "2006-01-02 15:04" }}

				{{ end }}

		{{if .ShouldDisplayLoadMore}}

		Load more

		{{end}}

`))
<|endoftext|>


<commit_before>// Shows example use of the keyring package
//
// May need to be built with a platform-specific build flag to specify a
// provider.
//
// For Example, on Linux, to use the gnome-keyring provider:
// 	$ go build +gnome_keyring github.com/tmc/keyring/example
// 	$ ./example
//
package main

import (
	"os"
	"fmt"
	"code.google.com/p/gopass"
	"github.com/tmc/keyring"
)

func main() {
	if pw, err := keyring.Get("keyring_example", "jack"); err == nil {
		fmt.Println("current stored password:", pw)
	} else if err == keyring.ErrNotFound {
		fmt.Println("no password stored yet")
	} else {
		fmt.Println("got unexpected error:", err)
		os.Exit(1)
	}
	pw, err := gopass.GetPass("enter new password: ")
	if err != nil {
		fmt.Println(err)
		os.Exit(1)
	}
	keyring.Set("keyring_example", "jack", pw)
	if pw, err := keyring.Get("keyring_example", "jack"); err == nil {
		fmt.Println("stored", pw)
	} else {
		fmt.Println("error:", err)
	}
}
<commit_msg>Make example more verbose<commit_after>// Shows example use of the keyring package
//
// May need to be built with a platform-specific build flag to specify a
// provider.
//
// For Example, on Linux, to use the gnome-keyring provider:
// 	$ go build +gnome_keyring github.com/tmc/keyring/example
// 	$ ./example
//
package main

import (
	"os"
	"fmt"
	"code.google.com/p/gopass"
	"github.com/tmc/keyring"
)

func main() {
	if pw, err := keyring.Get("keyring_example", "jack"); err == nil {
		fmt.Println("current stored password:", pw)
	} else if err == keyring.ErrNotFound {
		fmt.Println("no password stored yet")
	} else {
		fmt.Println("got unexpected error:", err)
		os.Exit(1)
	}
	pw, err := gopass.GetPass("enter new password: ")
	if err != nil {
		fmt.Println(err)
		os.Exit(1)
	}
	fmt.Println("setting keyring_example/jack to..", pw)
	err = keyring.Set("keyring_example", "jack", pw)
	if err != nil {
		fmt.Println(err)
		os.Exit(1)
	}
	fmt.Println("fetching keyring_example/jack..")
	if pw, err := keyring.Get("keyring_example", "jack"); err == nil {
		fmt.Println("got", pw)
	} else {
		fmt.Println("error:", err)
	}
}
<|endoftext|>


<commit_before>package provision

import (
	"bytes"
	"errors"
	"fmt"
	"os/exec"
	"path"

	log "github.com/Sirupsen/logrus"
	"github.com/docker/machine/drivers"
	"github.com/docker/machine/libmachine/auth"
	"github.com/docker/machine/libmachine/provision/pkgaction"
	"github.com/docker/machine/libmachine/swarm"
	"github.com/docker/machine/state"
	"github.com/docker/machine/utils"
)

func init() {
	Register("boot2docker", &RegisteredProvisioner{
		New: NewBoot2DockerProvisioner,
	})
}

func NewBoot2DockerProvisioner(d drivers.Driver) Provisioner {
	return &Boot2DockerProvisioner{
		Driver: d,
	}
}

type Boot2DockerProvisioner struct {
	OsReleaseInfo *OsRelease
	Driver        drivers.Driver
	SwarmOptions  swarm.SwarmOptions
}

func (provisioner *Boot2DockerProvisioner) Service(name string, action pkgaction.ServiceAction) error {
	var (
		cmd *exec.Cmd
		err error
	)
	cmd, err = provisioner.SSHCommand(fmt.Sprintf("sudo /etc/init.d/%s %s", name, action.String()))
	if err != nil {
		return err
	}
	if err := cmd.Run(); err != nil {
		return err
	}
	return nil
}

func (provisioner *Boot2DockerProvisioner) upgradeIso() error {
	log.Infof("Stopping machine to do the upgrade...")

	switch provisioner.Driver.DriverName() {
	case "vmwarefusion", "vmwarevsphere":
		return errors.New("Upgrade functionality is currently not supported for these providers, as they use a custom ISO.")
	}

	if err := provisioner.Driver.Stop(); err != nil {
		return err
	}

	if err := utils.WaitFor(drivers.MachineInState(provisioner.Driver, state.Stopped)); err != nil {
		return err
	}

	machineName := provisioner.GetDriver().GetMachineName()

	log.Infof("Upgrading machine %s...", machineName)

	b2dutils := utils.NewB2dUtils("", "")

	// Usually we call this implicitly, but call it here explicitly to get
	// the latest boot2docker ISO.
	if err := b2dutils.DownloadLatestBoot2Docker(); err != nil {
		return err
	}

	// Copy the latest version of boot2docker ISO to the machine's directory
	if err := b2dutils.CopyIsoToMachineDir("", machineName); err != nil {
		return err
	}

	log.Infof("Starting machine back up...")

	if err := provisioner.Driver.Start(); err != nil {
		return err
	}

	return utils.WaitFor(drivers.MachineInState(provisioner.Driver, state.Running))
}

func (provisioner *Boot2DockerProvisioner) Package(name string, action pkgaction.PackageAction) error {
	if name == "docker" && action == pkgaction.Upgrade {
		if err := provisioner.upgradeIso(); err != nil {
			return err
		}
	}
	return nil
}

func (provisioner *Boot2DockerProvisioner) Hostname() (string, error) {
	cmd, err := provisioner.SSHCommand(fmt.Sprintf("hostname"))
	if err != nil {
		return "", err
	}

	var so bytes.Buffer
	cmd.Stdout = &so

	if err := cmd.Run(); err != nil {
		return "", err
	}

	return so.String(), nil
}

func (provisioner *Boot2DockerProvisioner) SetHostname(hostname string) error {
	cmd, err := provisioner.SSHCommand(fmt.Sprintf(
		"sudo hostname %s && echo %q | sudo tee /var/lib/boot2docker/etc/hostname",
		hostname,
		hostname,
	))
	if err != nil {
		return err
	}

	return cmd.Run()
}

func (provisioner *Boot2DockerProvisioner) GetDockerOptionsDir() string {
	return "/var/lib/boot2docker"
}

func (provisioner *Boot2DockerProvisioner) GenerateDockerOptions(dockerPort int, authOptions auth.AuthOptions) (*DockerOptions, error) {
	defaultDaemonOpts := getDefaultDaemonOpts(provisioner.Driver.DriverName(), authOptions)
	daemonOpts := fmt.Sprintf("-H tcp://0.0.0.0:%d", dockerPort)
	daemonOptsDir := path.Join(provisioner.GetDockerOptionsDir(), "profile")
	opts := fmt.Sprintf("%s %s", defaultDaemonOpts, daemonOpts)
	daemonCfg := fmt.Sprintf(`EXTRA_ARGS='%s'
CACERT=%s
SERVERCERT=%s
SERVERKEY=%s
DOCKER_TLS=no`, opts, authOptions.CaCertRemotePath, authOptions.ServerCertRemotePath, authOptions.ServerKeyRemotePath)
	return &DockerOptions{
		EngineOptions:     daemonCfg,
		EngineOptionsPath: daemonOptsDir,
	}, nil
}

func (provisioner *Boot2DockerProvisioner) CompatibleWithHost() bool {
	return provisioner.OsReleaseInfo.Id == "boot2docker"
}

func (provisioner *Boot2DockerProvisioner) SetOsReleaseInfo(info *OsRelease) {
	provisioner.OsReleaseInfo = info
}

func (provisioner *Boot2DockerProvisioner) Provision(swarmOptions swarm.SwarmOptions, authOptions auth.AuthOptions) error {
	if err := provisioner.SetHostname(provisioner.Driver.GetMachineName()); err != nil {
		return err
	}

	if err := installDockerGeneric(provisioner); err != nil {
		return err
	}

	ip, err := provisioner.GetDriver().GetIP()
	if err != nil {
		return err
	}

	// b2d hosts need to wait for the daemon to be up
	// before continuing with provisioning
	if err := utils.WaitForDocker(ip, 2376); err != nil {
		return err
	}

	if err := ConfigureAuth(provisioner, authOptions); err != nil {
		return err
	}

	if err := configureSwarm(provisioner, swarmOptions); err != nil {
		return err
	}

	return nil
}

func (provisioner *Boot2DockerProvisioner) SSHCommand(args ...string) (*exec.Cmd, error) {
	return drivers.GetSSHCommandFromDriver(provisioner.Driver, args...)
}

func (provisioner *Boot2DockerProvisioner) GetDriver() drivers.Driver {
	return provisioner.Driver
}
<commit_msg>Moves the 'stopping machine' log below error log<commit_after>package provision

import (
	"bytes"
	"errors"
	"fmt"
	"os/exec"
	"path"

	log "github.com/Sirupsen/logrus"
	"github.com/docker/machine/drivers"
	"github.com/docker/machine/libmachine/auth"
	"github.com/docker/machine/libmachine/provision/pkgaction"
	"github.com/docker/machine/libmachine/swarm"
	"github.com/docker/machine/state"
	"github.com/docker/machine/utils"
)

func init() {
	Register("boot2docker", &RegisteredProvisioner{
		New: NewBoot2DockerProvisioner,
	})
}

func NewBoot2DockerProvisioner(d drivers.Driver) Provisioner {
	return &Boot2DockerProvisioner{
		Driver: d,
	}
}

type Boot2DockerProvisioner struct {
	OsReleaseInfo *OsRelease
	Driver        drivers.Driver
	SwarmOptions  swarm.SwarmOptions
}

func (provisioner *Boot2DockerProvisioner) Service(name string, action pkgaction.ServiceAction) error {
	var (
		cmd *exec.Cmd
		err error
	)
	cmd, err = provisioner.SSHCommand(fmt.Sprintf("sudo /etc/init.d/%s %s", name, action.String()))
	if err != nil {
		return err
	}
	if err := cmd.Run(); err != nil {
		return err
	}
	return nil
}

func (provisioner *Boot2DockerProvisioner) upgradeIso() error {
	switch provisioner.Driver.DriverName() {
	case "vmwarefusion", "vmwarevsphere":
		return errors.New("Upgrade functionality is currently not supported for these providers, as they use a custom ISO.")
	}

	log.Info("Stopping machine to do the upgrade...")

	if err := provisioner.Driver.Stop(); err != nil {
		return err
	}

	if err := utils.WaitFor(drivers.MachineInState(provisioner.Driver, state.Stopped)); err != nil {
		return err
	}

	machineName := provisioner.GetDriver().GetMachineName()

	log.Infof("Upgrading machine %s...", machineName)

	b2dutils := utils.NewB2dUtils("", "")

	// Usually we call this implicitly, but call it here explicitly to get
	// the latest boot2docker ISO.
	if err := b2dutils.DownloadLatestBoot2Docker(); err != nil {
		return err
	}

	// Copy the latest version of boot2docker ISO to the machine's directory
	if err := b2dutils.CopyIsoToMachineDir("", machineName); err != nil {
		return err
	}

	log.Infof("Starting machine back up...")

	if err := provisioner.Driver.Start(); err != nil {
		return err
	}

	return utils.WaitFor(drivers.MachineInState(provisioner.Driver, state.Running))
}

func (provisioner *Boot2DockerProvisioner) Package(name string, action pkgaction.PackageAction) error {
	if name == "docker" && action == pkgaction.Upgrade {
		if err := provisioner.upgradeIso(); err != nil {
			return err
		}
	}
	return nil
}

func (provisioner *Boot2DockerProvisioner) Hostname() (string, error) {
	cmd, err := provisioner.SSHCommand(fmt.Sprintf("hostname"))
	if err != nil {
		return "", err
	}

	var so bytes.Buffer
	cmd.Stdout = &so

	if err := cmd.Run(); err != nil {
		return "", err
	}

	return so.String(), nil
}

func (provisioner *Boot2DockerProvisioner) SetHostname(hostname string) error {
	cmd, err := provisioner.SSHCommand(fmt.Sprintf(
		"sudo hostname %s && echo %q | sudo tee /var/lib/boot2docker/etc/hostname",
		hostname,
		hostname,
	))
	if err != nil {
		return err
	}

	return cmd.Run()
}

func (provisioner *Boot2DockerProvisioner) GetDockerOptionsDir() string {
	return "/var/lib/boot2docker"
}

func (provisioner *Boot2DockerProvisioner) GenerateDockerOptions(dockerPort int, authOptions auth.AuthOptions) (*DockerOptions, error) {
	defaultDaemonOpts := getDefaultDaemonOpts(provisioner.Driver.DriverName(), authOptions)
	daemonOpts := fmt.Sprintf("-H tcp://0.0.0.0:%d", dockerPort)
	daemonOptsDir := path.Join(provisioner.GetDockerOptionsDir(), "profile")
	opts := fmt.Sprintf("%s %s", defaultDaemonOpts, daemonOpts)
	daemonCfg := fmt.Sprintf(`EXTRA_ARGS='%s'
CACERT=%s
SERVERCERT=%s
SERVERKEY=%s
DOCKER_TLS=no`, opts, authOptions.CaCertRemotePath, authOptions.ServerCertRemotePath, authOptions.ServerKeyRemotePath)
	return &DockerOptions{
		EngineOptions:     daemonCfg,
		EngineOptionsPath: daemonOptsDir,
	}, nil
}

func (provisioner *Boot2DockerProvisioner) CompatibleWithHost() bool {
	return provisioner.OsReleaseInfo.Id == "boot2docker"
}

func (provisioner *Boot2DockerProvisioner) SetOsReleaseInfo(info *OsRelease) {
	provisioner.OsReleaseInfo = info
}

func (provisioner *Boot2DockerProvisioner) Provision(swarmOptions swarm.SwarmOptions, authOptions auth.AuthOptions) error {
	if err := provisioner.SetHostname(provisioner.Driver.GetMachineName()); err != nil {
		return err
	}

	if err := installDockerGeneric(provisioner); err != nil {
		return err
	}

	ip, err := provisioner.GetDriver().GetIP()
	if err != nil {
		return err
	}

	// b2d hosts need to wait for the daemon to be up
	// before continuing with provisioning
	if err := utils.WaitForDocker(ip, 2376); err != nil {
		return err
	}

	if err := ConfigureAuth(provisioner, authOptions); err != nil {
		return err
	}

	if err := configureSwarm(provisioner, swarmOptions); err != nil {
		return err
	}

	return nil
}

func (provisioner *Boot2DockerProvisioner) SSHCommand(args ...string) (*exec.Cmd, error) {
	return drivers.GetSSHCommandFromDriver(provisioner.Driver, args...)
}

func (provisioner *Boot2DockerProvisioner) GetDriver() drivers.Driver {
	return provisioner.Driver
}
<|endoftext|>