<commit_before>package mode

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"time"
)

const (
	defaultHTTPTimeout          = time.Second * 10
	defaultHTTPKeepAliveTimeout = time.Minute * 5
	defaultHTTPKeepAliveMaxConn = 5
	defaultUserAgent            = "GoDeviceSDK/" + sdkVersion
)

var (
	httpTransport = &http.Transport{
		MaxIdleConnsPerHost: defaultHTTPKeepAliveMaxConn,
		IdleConnTimeout:     defaultHTTPKeepAliveTimeout,
	}

	httpClient = &http.Client{
		Timeout: defaultHTTPTimeout,
	}
)

// RESTError represents an error returned by the MODE REST API.
type RESTError struct {
	statusCode int
	reason     string
	data       map[string]interface{}
}

func newRESTError(statusCode int, jsonBytes []byte) *RESTError {
	restErr := &RESTError{statusCode: statusCode}

	var fields struct {
		Reason string                 `json:"reason"`
		Data   map[string]interface{} `json:"data"`
	}

	if err := json.Unmarshal(jsonBytes, &fields); err == nil {
		restErr.reason = fields.Reason
		restErr.data = fields.Data
	} else {
		restErr.reason = "UNKNOWN"
	}

	return restErr
}

// Error returns a summary of the error.
func (e *RESTError) Error() string {
	return fmt.Sprintf("server returned status %d (reason: %s)", e.statusCode, e.reason)
}

// Reason returns the specific reason for the error.
func (e *RESTError) Reason() string {
	return e.reason
}

// StatusCode returns the HTTP status code provided by the API server.
func (e *RESTError) StatusCode() int {
	return e.statusCode
}

// Data returns any additional data associated with this error, or nil.
func (e *RESTError) Data() map[string]interface{} {
	return e.data
}

func makeRESTCallURL(path string) string {
	var proto string
	if restTLS {
		proto = "https"
	} else {
		proto = "http"
	}

	return fmt.Sprintf("%s://%s:%d%s", proto, restHost, restPort, path)
}

func makeRESTAuth(authToken string) string {
	return fmt.Sprintf("ModeCloud %s", authToken)
}

func makeRESTCall(method string, path string, authToken string, data interface{}, resData interface{}) error {
	url := makeRESTCallURL(path)

	var body io.Reader
	if data != nil {
		jsonBytes, err := json.Marshal(data)
		if err != nil {
			logError("Failed to encode data to JSON: %s", err.Error())
			return err
		}

		body = bytes.NewReader(jsonBytes)
	}

	request, err := http.NewRequest(method, url, body)
	if err != nil {
		logError("Failed to set up HTTP request: %s", err.Error())
		return err
	}

	request.Header.Add("User-Agent", defaultUserAgent)

	if authToken != "" {
		request.Header.Add("Authorization", makeRESTAuth(authToken))
	}

	if body != nil {
		request.Header.Set("Content-Type", "application/json")
	}

	res, err := httpClient.Do(request)
	if err != nil {
		logError("Failed to make HTTP request: %s", err.Error())
		return err
	}
	defer res.Body.Close()

	jsonBytes, err := ioutil.ReadAll(res.Body)
	if err != nil {
		logError("Failed to read HTTP response body: %s", err.Error())
		return err
	}

	logInfo("REST API response code %d", res.StatusCode)

	if res.StatusCode == http.StatusNoContent {
		return nil
	}

	if res.StatusCode == http.StatusOK || res.StatusCode == http.StatusCreated {
		if resData == nil {
			return nil
		} else {
			if err := json.Unmarshal(jsonBytes, resData); err == nil {
				return nil
			} else {
				logError("Failed to parse JSON response: %s", err.Error())
				return err
			}
		}
	}

	return newRESTError(res.StatusCode, jsonBytes)
}
<commit_msg>Use custom http transport for REST calls.<commit_after>package mode

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"time"
)

const (
	defaultHTTPTimeout          = time.Second * 10
	defaultHTTPKeepAliveTimeout = time.Minute * 5
	defaultHTTPKeepAliveMaxConn = 5
	defaultUserAgent            = "GoDeviceSDK/" + sdkVersion
)

var (
	httpTransport = &http.Transport{
		MaxIdleConnsPerHost: defaultHTTPKeepAliveMaxConn,
		IdleConnTimeout:     defaultHTTPKeepAliveTimeout,
	}

	httpClient = &http.Client{
		Transport: httpTransport,
		Timeout:   defaultHTTPTimeout,
	}
)

// RESTError represents an error returned by the MODE REST API.
type RESTError struct {
	statusCode int
	reason     string
	data       map[string]interface{}
}

func newRESTError(statusCode int, jsonBytes []byte) *RESTError {
	restErr := &RESTError{statusCode: statusCode}

	var fields struct {
		Reason string                 `json:"reason"`
		Data   map[string]interface{} `json:"data"`
	}

	if err := json.Unmarshal(jsonBytes, &fields); err == nil {
		restErr.reason = fields.Reason
		restErr.data = fields.Data
	} else {
		restErr.reason = "UNKNOWN"
	}

	return restErr
}

// Error returns a summary of the error.
func (e *RESTError) Error() string {
	return fmt.Sprintf("server returned status %d (reason: %s)", e.statusCode, e.reason)
}

// Reason returns the specific reason for the error.
func (e *RESTError) Reason() string {
	return e.reason
}

// StatusCode returns the HTTP status code provided by the API server.
func (e *RESTError) StatusCode() int {
	return e.statusCode
}

// Data returns any additional data associated with this error, or nil.
func (e *RESTError) Data() map[string]interface{} {
	return e.data
}

func makeRESTCallURL(path string) string {
	var proto string
	if restTLS {
		proto = "https"
	} else {
		proto = "http"
	}

	return fmt.Sprintf("%s://%s:%d%s", proto, restHost, restPort, path)
}

func makeRESTAuth(authToken string) string {
	return fmt.Sprintf("ModeCloud %s", authToken)
}

func makeRESTCall(method string, path string, authToken string, data interface{}, resData interface{}) error {
	url := makeRESTCallURL(path)

	var body io.Reader
	if data != nil {
		jsonBytes, err := json.Marshal(data)
		if err != nil {
			logError("Failed to encode data to JSON: %s", err.Error())
			return err
		}

		body = bytes.NewReader(jsonBytes)
	}

	request, err := http.NewRequest(method, url, body)
	if err != nil {
		logError("Failed to set up HTTP request: %s", err.Error())
		return err
	}

	request.Header.Add("User-Agent", defaultUserAgent)

	if authToken != "" {
		request.Header.Add("Authorization", makeRESTAuth(authToken))
	}

	if body != nil {
		request.Header.Set("Content-Type", "application/json")
	}

	res, err := httpClient.Do(request)
	if err != nil {
		logError("Failed to make HTTP request: %s", err.Error())
		return err
	}
	defer res.Body.Close()

	jsonBytes, err := ioutil.ReadAll(res.Body)
	if err != nil {
		logError("Failed to read HTTP response body: %s", err.Error())
		return err
	}

	logInfo("REST API response code %d", res.StatusCode)

	if res.StatusCode == http.StatusNoContent {
		return nil
	}

	if res.StatusCode == http.StatusOK || res.StatusCode == http.StatusCreated {
		if resData == nil {
			return nil
		} else {
			if err := json.Unmarshal(jsonBytes, resData); err == nil {
				return nil
			} else {
				logError("Failed to parse JSON response: %s", err.Error())
				return err
			}
		}
	}

	return newRESTError(res.StatusCode, jsonBytes)
}
<|endoftext|>


<commit_before>package urknall

import (
	"fmt"
	"strings"
)

// A role is a function that adds packages to a host.
type Role func(host *Host)

// A role registry holds information on the available roles and applies all those roles registered for a given host
// (hosts have a list of tags that should have the form `role:<name>` for the roles to be used for it).
type RoleRegistry map[string]Role

// Create a new role registry.
func NewRoleRegistry() *RoleRegistry {
	return &RoleRegistry{}
}

// Add a role with the given name.
func (rr RoleRegistry) Add(name string, role Role) (e error) {
	if _, found := rr[name]; found {
		return fmt.Errorf("Role %q already registered", name)
	}
	rr[name] = role
	return nil
}

// Add a single package as a role (creating the boilerplate for you).
func (rr RoleRegistry) AddPackage(name string, pkg Package) (e error) {
	return rr.Add(name, func(host *Host) { host.Add(name, pkg) })
}

// Apply all matching roles to the given host. The matching criteria is the existence of a host tag of the form
// "role:<rolename>".
func (rr RoleRegistry) ApplyRoles(host *Host) (e error) {
	for _, tag := range host.Tags {
		if strings.HasPrefix(tag, "role:") {
			name := strings.TrimPrefix(tag, "role:")
			role, found := rr[name]
			if !found {
				return fmt.Errorf("role %q unknown", name)
			}
			role(host)
		}
	}
	return nil
}
<commit_msg>removed role concept<commit_after><|endoftext|>


<commit_before>package dnsr

var root = `
;       This file holds the information on root name servers needed to
;       initialize cache of Internet domain name servers
;       (e.g. reference this file in the "cache  .  <file>"
;       configuration file of BIND domain name servers).
;
;       This file is made available by InterNIC
;       under anonymous FTP as
;           file                /domain/named.cache
;           on server           FTP.INTERNIC.NET
;       -OR-                    RS.INTERNIC.NET
;
;       last update:     January 11, 2021
;       related version of root zone:     2021011101
;
; FORMERLY NS.INTERNIC.NET
;
.                        3600000      NS    A.ROOT-SERVERS.NET.
A.ROOT-SERVERS.NET.      3600000      A     198.41.0.4
A.ROOT-SERVERS.NET.      3600000      AAAA  2001:503:ba3e::2:30
;
; FORMERLY NS1.ISI.EDU
;
.                        3600000      NS    B.ROOT-SERVERS.NET.
B.ROOT-SERVERS.NET.      3600000      A     199.9.14.201
B.ROOT-SERVERS.NET.      3600000      AAAA  2001:500:200::b
;
; FORMERLY C.PSI.NET
;
.                        3600000      NS    C.ROOT-SERVERS.NET.
C.ROOT-SERVERS.NET.      3600000      A     192.33.4.12
C.ROOT-SERVERS.NET.      3600000      AAAA  2001:500:2::c
;
; FORMERLY TERP.UMD.EDU
;
.                        3600000      NS    D.ROOT-SERVERS.NET.
D.ROOT-SERVERS.NET.      3600000      A     199.7.91.13
D.ROOT-SERVERS.NET.      3600000      AAAA  2001:500:2d::d
;
; FORMERLY NS.NASA.GOV
;
.                        3600000      NS    E.ROOT-SERVERS.NET.
E.ROOT-SERVERS.NET.      3600000      A     192.203.230.10
E.ROOT-SERVERS.NET.      3600000      AAAA  2001:500:a8::e
;
; FORMERLY NS.ISC.ORG
;
.                        3600000      NS    F.ROOT-SERVERS.NET.
F.ROOT-SERVERS.NET.      3600000      A     192.5.5.241
F.ROOT-SERVERS.NET.      3600000      AAAA  2001:500:2f::f
;
; FORMERLY NS.NIC.DDN.MIL
;
.                        3600000      NS    G.ROOT-SERVERS.NET.
G.ROOT-SERVERS.NET.      3600000      A     192.112.36.4
G.ROOT-SERVERS.NET.      3600000      AAAA  2001:500:12::d0d
;
; FORMERLY AOS.ARL.ARMY.MIL
;
.                        3600000      NS    H.ROOT-SERVERS.NET.
H.ROOT-SERVERS.NET.      3600000      A     198.97.190.53
H.ROOT-SERVERS.NET.      3600000      AAAA  2001:500:1::53
;
; FORMERLY NIC.NORDU.NET
;
.                        3600000      NS    I.ROOT-SERVERS.NET.
I.ROOT-SERVERS.NET.      3600000      A     192.36.148.17
I.ROOT-SERVERS.NET.      3600000      AAAA  2001:7fe::53
;
; OPERATED BY VERISIGN, INC.
;
.                        3600000      NS    J.ROOT-SERVERS.NET.
J.ROOT-SERVERS.NET.      3600000      A     192.58.128.30
J.ROOT-SERVERS.NET.      3600000      AAAA  2001:503:c27::2:30
;
; OPERATED BY RIPE NCC
;
.                        3600000      NS    K.ROOT-SERVERS.NET.
K.ROOT-SERVERS.NET.      3600000      A     193.0.14.129
K.ROOT-SERVERS.NET.      3600000      AAAA  2001:7fd::1
;
; OPERATED BY ICANN
;
.                        3600000      NS    L.ROOT-SERVERS.NET.
L.ROOT-SERVERS.NET.      3600000      A     199.7.83.42
L.ROOT-SERVERS.NET.      3600000      AAAA  2001:500:9f::42
;
; OPERATED BY WIDE
;
.                        3600000      NS    M.ROOT-SERVERS.NET.
M.ROOT-SERVERS.NET.      3600000      A     202.12.27.33
M.ROOT-SERVERS.NET.      3600000      AAAA  2001:dc3::35
; End of file`
<commit_msg>Automatic update for Thu Feb 18 08:33:44 UTC 2021<commit_after>package dnsr

var root = `
;       This file holds the information on root name servers needed to
;       initialize cache of Internet domain name servers
;       (e.g. reference this file in the "cache  .  <file>"
;       configuration file of BIND domain name servers).
;
;       This file is made available by InterNIC
;       under anonymous FTP as
;           file                /domain/named.cache
;           on server           FTP.INTERNIC.NET
;       -OR-                    RS.INTERNIC.NET
;
;       last update:     February 17, 2021
;       related version of root zone:     2021021701
;
; FORMERLY NS.INTERNIC.NET
;
.                        3600000      NS    A.ROOT-SERVERS.NET.
A.ROOT-SERVERS.NET.      3600000      A     198.41.0.4
A.ROOT-SERVERS.NET.      3600000      AAAA  2001:503:ba3e::2:30
;
; FORMERLY NS1.ISI.EDU
;
.                        3600000      NS    B.ROOT-SERVERS.NET.
B.ROOT-SERVERS.NET.      3600000      A     199.9.14.201
B.ROOT-SERVERS.NET.      3600000      AAAA  2001:500:200::b
;
; FORMERLY C.PSI.NET
;
.                        3600000      NS    C.ROOT-SERVERS.NET.
C.ROOT-SERVERS.NET.      3600000      A     192.33.4.12
C.ROOT-SERVERS.NET.      3600000      AAAA  2001:500:2::c
;
; FORMERLY TERP.UMD.EDU
;
.                        3600000      NS    D.ROOT-SERVERS.NET.
D.ROOT-SERVERS.NET.      3600000      A     199.7.91.13
D.ROOT-SERVERS.NET.      3600000      AAAA  2001:500:2d::d
;
; FORMERLY NS.NASA.GOV
;
.                        3600000      NS    E.ROOT-SERVERS.NET.
E.ROOT-SERVERS.NET.      3600000      A     192.203.230.10
E.ROOT-SERVERS.NET.      3600000      AAAA  2001:500:a8::e
;
; FORMERLY NS.ISC.ORG
;
.                        3600000      NS    F.ROOT-SERVERS.NET.
F.ROOT-SERVERS.NET.      3600000      A     192.5.5.241
F.ROOT-SERVERS.NET.      3600000      AAAA  2001:500:2f::f
;
; FORMERLY NS.NIC.DDN.MIL
;
.                        3600000      NS    G.ROOT-SERVERS.NET.
G.ROOT-SERVERS.NET.      3600000      A     192.112.36.4
G.ROOT-SERVERS.NET.      3600000      AAAA  2001:500:12::d0d
;
; FORMERLY AOS.ARL.ARMY.MIL
;
.                        3600000      NS    H.ROOT-SERVERS.NET.
H.ROOT-SERVERS.NET.      3600000      A     198.97.190.53
H.ROOT-SERVERS.NET.      3600000      AAAA  2001:500:1::53
;
; FORMERLY NIC.NORDU.NET
;
.                        3600000      NS    I.ROOT-SERVERS.NET.
I.ROOT-SERVERS.NET.      3600000      A     192.36.148.17
I.ROOT-SERVERS.NET.      3600000      AAAA  2001:7fe::53
;
; OPERATED BY VERISIGN, INC.
;
.                        3600000      NS    J.ROOT-SERVERS.NET.
J.ROOT-SERVERS.NET.      3600000      A     192.58.128.30
J.ROOT-SERVERS.NET.      3600000      AAAA  2001:503:c27::2:30
;
; OPERATED BY RIPE NCC
;
.                        3600000      NS    K.ROOT-SERVERS.NET.
K.ROOT-SERVERS.NET.      3600000      A     193.0.14.129
K.ROOT-SERVERS.NET.      3600000      AAAA  2001:7fd::1
;
; OPERATED BY ICANN
;
.                        3600000      NS    L.ROOT-SERVERS.NET.
L.ROOT-SERVERS.NET.      3600000      A     199.7.83.42
L.ROOT-SERVERS.NET.      3600000      AAAA  2001:500:9f::42
;
; OPERATED BY WIDE
;
.                        3600000      NS    M.ROOT-SERVERS.NET.
M.ROOT-SERVERS.NET.      3600000      A     202.12.27.33
M.ROOT-SERVERS.NET.      3600000      AAAA  2001:dc3::35
; End of file`
<|endoftext|>


<commit_before>// Go MySQL Driver - A MySQL-Driver for Go's database/sql package
//
// Copyright 2012 The Go-MySQL-Driver Authors. All rights reserved.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at http://mozilla.org/MPL/2.0/.

package mysql

import (
	"database/sql/driver"
	"io"
)

type mysqlField struct {
	fieldType byte
	flags     fieldFlag
	name      string
}

type mysqlRows struct {
	mc      *mysqlConn
	columns []mysqlField
	binary  bool
}

func (rows *mysqlRows) Columns() []string {
	columns := make([]string, len(rows.columns))
	for i := range columns {
		columns[i] = rows.columns[i].name
	}
	return columns
}

func (rows *mysqlRows) Close() error {
	mc := rows.mc
	if mc == nil {
		return nil
	}
	if mc.netConn == nil {
		return errInvalidConn
	}
	// Remove unread packets from stream
	err := mc.readUntilEOF()
	rows.mc = nil
	return err
}

func (rows *mysqlRows) Next(dest []driver.Value) error {
	mc := rows.mc
	if mc == nil {
		return io.EOF
	}
	if mc.netConn == nil {
		return errInvalidConn
	}
	var err error
	// Fetch next row from stream
	if rows.binary {
		err = rows.readBinaryRow(dest)
	} else {
		err = rows.readRow(dest)
	}
	if err == io.EOF {
		rows.mc = nil
	}
	return err
}
<commit_msg>removed internal err declaration<commit_after>// Go MySQL Driver - A MySQL-Driver for Go's database/sql package
//
// Copyright 2012 The Go-MySQL-Driver Authors. All rights reserved.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at http://mozilla.org/MPL/2.0/.

package mysql

import (
	"database/sql/driver"
	"io"
)

type mysqlField struct {
	fieldType byte
	flags     fieldFlag
	name      string
}

type mysqlRows struct {
	mc      *mysqlConn
	columns []mysqlField
	binary  bool
}

func (rows *mysqlRows) Columns() []string {
	columns := make([]string, len(rows.columns))
	for i := range columns {
		columns[i] = rows.columns[i].name
	}
	return columns
}

func (rows *mysqlRows) Close() error {
	mc := rows.mc
	if mc == nil {
		return nil
	}
	if mc.netConn == nil {
		return errInvalidConn
	}
	// Remove unread packets from stream
	err := mc.readUntilEOF()
	rows.mc = nil
	return err
}

func (rows *mysqlRows) Next(dest []driver.Value) (err error) {
	mc := rows.mc
	if mc == nil {
		return io.EOF
	}
	if mc.netConn == nil {
		return errInvalidConn
	}
	// Fetch next row from stream
	if rows.binary {
		err = rows.readBinaryRow(dest)
	} else {
		err = rows.readRow(dest)
	}
	if err == io.EOF {
		rows.mc = nil
	}
	return
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"crypto/rand"
	"encoding/hex"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"net"
	"os"
	"os/exec"
	"path"
	"path/filepath"
	"strconv"

	"github.com/hyperhq/runv/driverloader"
	"github.com/hyperhq/runv/hypervisor"
	"github.com/hyperhq/runv/hypervisor/pod"
	"github.com/hyperhq/runv/hypervisor/types"
	"github.com/hyperhq/runv/lib/term"

	"github.com/opencontainers/specs"
)

const (
	_ = iota
	RUNV_ACK
	RUNV_EXECCMD
	RUNV_WINSIZE
)

const shortLen = 12

func TruncateID(id string) string {
	trimTo := shortLen
	if len(id) < shortLen {
		trimTo = len(id)
	}
	return id[:trimTo]
}

// GenerateRandomID returns an unique id
func GenerateRandomID() string {
	for {
		id := make([]byte, 32)
		if _, err := io.ReadFull(rand.Reader, id); err != nil {
			panic(err) // This shouldn't happen
		}
		value := hex.EncodeToString(id)
		// if we try to parse the truncated for as an int and we don't have
		// an error then the value is all numberic and causes issues when
		// used as a hostname. ref #3869
		if _, err := strconv.ParseInt(TruncateID(value), 10, 64); err == nil {
			continue
		}
		return value
	}
}

func removeState(container, root string, sock net.Listener) {
	os.RemoveAll(path.Join(root, container))
	sock.Close()
}

func saveState(container, root string, state *specs.State) (net.Listener, error) {
	stateDir := path.Join(root, container)

	_, err := os.Stat(stateDir)
	if err == nil {
		return nil, fmt.Errorf("Container %s exists\n", container)
	}

	err = os.MkdirAll(stateDir, 0644)
	if err != nil {
		fmt.Printf("%s\n", err.Error())
		return nil, err
	}

	defer func() {
		if err != nil {
			os.RemoveAll(stateDir)
		}
	}()

	stateData, err := json.MarshalIndent(state, "", "\t")
	if err != nil {
		fmt.Printf("%s\n", err.Error())
		return nil, err
	}

	stateFile := path.Join(stateDir, "state.json")
	err = ioutil.WriteFile(stateFile, stateData, 0644)
	if err != nil {
		fmt.Printf("%s\n", err.Error())
		return nil, err
	}

	sock, err := net.Listen("unix", path.Join(stateDir, "runv.sock"))
	if err != nil {
		fmt.Printf("%s\n", err.Error())
		return nil, err
	}

	return sock, nil
}

func execHook(hook specs.Hook, state *specs.State) error {
	b, err := json.Marshal(state)
	if err != nil {
		return err
	}
	cmd := exec.Cmd{
		Path:  hook.Path,
		Args:  hook.Args,
		Env:   hook.Env,
		Stdin: bytes.NewReader(b),
	}
	return cmd.Run()
}

func execPrestartHooks(rt *specs.RuntimeSpec, state *specs.State) error {
	for _, hook := range rt.Hooks.Prestart {
		err := execHook(hook, state)
		if err != nil {
			return err
		}
	}

	return nil
}

func execPoststartHooks(rt *specs.RuntimeSpec, state *specs.State) error {
	for _, hook := range rt.Hooks.Poststart {
		err := execHook(hook, state)
		if err != nil {
			fmt.Printf("exec Poststart hook %s failed %s", hook.Path, err.Error())
		}
	}

	return nil
}

func execPoststopHooks(rt *specs.RuntimeSpec, state *specs.State) error {
	for _, hook := range rt.Hooks.Poststop {
		err := execHook(hook, state)
		if err != nil {
			fmt.Printf("exec Poststop hook %s failed %s", hook.Path, err.Error())
		}
	}

	return nil
}

func prepareInfo(config *startConfig, c *pod.UserContainer, vmId string) (*hypervisor.ContainerInfo, error) {
	var root string
	var err error

	containerId := GenerateRandomID()
	sharedDir := path.Join(hypervisor.BaseDir, vmId, hypervisor.ShareDirTag)
	rootDir := path.Join(sharedDir, containerId)
	os.MkdirAll(rootDir, 0755)

	rootDir = path.Join(rootDir, "rootfs")

	if !filepath.IsAbs(c.Image) {
		root = path.Join(config.BundlePath, c.Image)
	} else {
		root = c.Image
	}

	err = mount(root, rootDir)
	if err != nil {
		fmt.Printf("mount %s to %s failed: %s\n", root, rootDir, err.Error())
		return nil, err
	}

	containerInfo := &hypervisor.ContainerInfo{
		Id:     containerId,
		Rootfs: "rootfs",
		Image:  containerId,
		Fstype: "dir",
	}

	return containerInfo, nil
}

func startVm(config *startConfig, userPod *pod.UserPod, vmId string) (*hypervisor.Vm, error) {
	var (
		err error
		cpu = 1
		mem = 128
	)

	vbox := config.Vbox
	if _, err = os.Stat(vbox); err == nil {
		vbox, err = filepath.Abs(vbox)
		if err != nil {
			fmt.Printf("Cannot get abs path for vbox: %s\n", err.Error())
			return nil, err
		}
	}

	kernel := config.Kernel
	if _, err = os.Stat(kernel); err == nil {
		kernel, err = filepath.Abs(kernel)
		if err != nil {
			fmt.Printf("Cannot get abs path for kernel: %s\n", err.Error())
			return nil, err
		}
	}

	initrd := config.Initrd
	if _, err = os.Stat(initrd); err == nil {
		initrd, err = filepath.Abs(initrd)
		if err != nil {
			fmt.Printf("Cannot get abs path for initrd: %s\n", err.Error())
			return nil, err
		}
	}

	if userPod.Resource.Vcpu > 0 {
		cpu = userPod.Resource.Vcpu
	}

	if userPod.Resource.Memory > 0 {
		mem = userPod.Resource.Memory
	}

	b := &hypervisor.BootConfig{
		Kernel: kernel,
		Initrd: initrd,
		Bios:   "",
		Cbfs:   "",
		Vbox:   vbox,
		CPU:    cpu,
		Memory: mem,
	}

	vm := hypervisor.NewVm(vmId, cpu, mem, false, types.VM_KEEP_NONE)
	err = vm.Launch(b)
	if err != nil {
		fmt.Printf("%s\n", err.Error())
		return nil, err
	}

	return vm, nil
}

// pod context for runv daemon
type runvPodContext struct {
	podId     string
	vmId      string
	userPod   *pod.UserPod
	podStatus *hypervisor.PodStatus
	vm        *hypervisor.Vm
}

var daemonPodContext runvPodContext

func startRunvPod(config *startConfig) (context *runvPodContext, err error) {
	context = &daemonPodContext

	hypervisor.InterfaceCount = 0

	driver := config.Driver
	if hypervisor.HDriver, err = driverloader.Probe(driver); err != nil {
		fmt.Printf("%s\n", err.Error())
		return nil, err
	}

	context.podId = fmt.Sprintf("pod-%s", pod.RandStr(10, "alpha"))
	context.vmId = fmt.Sprintf("vm-%s", pod.RandStr(10, "alpha"))

	context.userPod = pod.ConvertOCF2PureUserPod(&config.LinuxSpec, &config.LinuxRuntimeSpec)
	context.podStatus = hypervisor.NewPod(context.podId, context.userPod)
	context.vm, err = startVm(config, context.userPod, context.vmId)
	if err != nil {
		fmt.Printf("%s\n", err.Error())
		return nil, err
	}

	Response := context.vm.StartPod(context.podStatus, context.userPod, nil, nil)
	if Response.Data == nil {
		fmt.Printf("StartPod fail: QEMU response data is nil\n")
		return nil, fmt.Errorf("StartPod fail")
	}
	fmt.Printf("result: code %d %s\n", Response.Code, Response.Cause)

	return context, nil
}

func cleanupRunvPod(context *runvPodContext) {
	Response := context.vm.StopPod(context.podStatus, "yes")

	if Response.Data == nil {
		fmt.Printf("StopPod fail: QEMU response data is nil\n")
		return
	}
	fmt.Printf("result: code %d %s\n", Response.Code, Response.Cause)
	os.RemoveAll(path.Join(hypervisor.BaseDir, context.vmId))
}

func startVContainer(config *startConfig) {
	context, err := startRunvPod(config)
	if err != nil {
		fmt.Printf("Start Pod failed: %s\n", err.Error())
		return
	}
	defer cleanupRunvPod(context)

	root := config.Root
	container := config.Name
	fmt.Printf("runv container id: %s\n", container)

	state := &specs.State{
		Version:    config.LinuxSpec.Spec.Version,
		ID:         container,
		Pid:        -1,
		BundlePath: config.BundlePath,
	}

	sock, err := saveState(container, root, state)
	if err != nil {
		fmt.Printf("%s\n", err.Error())
		return
	}

	defer removeState(container, root, sock)

	err = execPrestartHooks(&config.LinuxRuntimeSpec.RuntimeSpec, state)
	if err != nil {
		fmt.Printf("execute Prestart hooks failed, %s\n", err.Error())
		return
	}

	inFd, _ := term.GetFdInfo(os.Stdin)
	outFd, isTerminalOut := term.GetFdInfo(os.Stdout)

	oldState, err := term.SetRawTerminal(inFd)
	if err != nil {
		return
	}

	defer term.RestoreTerminal(inFd, oldState)

	tag := pod.RandStr(8, "alphanum")
	ttyCallback := make(chan *types.VmResponse, 1)

	// using pipes in vm.Attach to avoid the stdio to be closed
	fromStd, toVm := io.Pipe()
	fromVm, toStd := io.Pipe()
	go io.Copy(toVm, os.Stdin)
	go io.Copy(os.Stdout, fromVm)

	userContainer := pod.ConvertOCF2UserContainer(&config.LinuxSpec, &config.LinuxRuntimeSpec)
	info, err := prepareInfo(config, userContainer, context.vmId)
	if err != nil {
		fmt.Printf("%s\n", err.Error())
		return
	}

	defer func() {
		rootDir := path.Join(hypervisor.BaseDir, context.vmId, hypervisor.ShareDirTag, info.Id, "rootfs")
		umount(rootDir)
		os.RemoveAll(path.Join(hypervisor.BaseDir, context.vmId, hypervisor.ShareDirTag, info.Id))
	}()

	err = context.vm.Attach(fromStd, toStd, tag, info.Id, ttyCallback, nil)
	if err != nil {
		fmt.Printf("StartPod fail: fail to set up tty connection.\n")
		return
	}

	context.podStatus.AddContainer(info.Id, context.podId, "", []string{}, types.S_POD_CREATED)
	context.vm.NewContainer(userContainer, info)
	ListenAndHandleRunvRequests(context.vm, sock)

	err = execPoststartHooks(&config.LinuxRuntimeSpec.RuntimeSpec, state)
	if err != nil {
		fmt.Printf("execute Poststart hooks failed %s\n", err.Error())
	}

	newTty(context.vm, path.Join(root, container), tag, outFd, isTerminalOut).monitorTtySize()
	<-ttyCallback

	err = execPoststopHooks(&config.LinuxRuntimeSpec.RuntimeSpec, state)
	if err != nil {
		fmt.Printf("execute Poststop hooks failed %s\n", err.Error())
		return
	}
}

func runvGetTag(conn net.Conn) (tag string, err error) {
	msg, err := hypervisor.ReadVmMessage(conn.(*net.UnixConn))
	if err != nil {
		fmt.Printf("read runv server data failed: %v\n", err)
		return "", err
	}

	if msg.Code != RUNV_ACK {
		return "", fmt.Errorf("unexpected respond code")
	}

	return string(msg.Message), nil
}

func HandleRunvRequest(vm *hypervisor.Vm, conn net.Conn) {
	defer conn.Close()

	msg, err := hypervisor.ReadVmMessage(conn.(*net.UnixConn))
	if err != nil {
		fmt.Printf("read runv client data failed: %v\n", err)
		return
	}

	switch msg.Code {
	case RUNV_EXECCMD:
		{
			tag := pod.RandStr(8, "alphanum")
			m := &hypervisor.DecodedMessage{
				Code:    RUNV_ACK,
				Message: []byte(tag),
			}
			data := hypervisor.NewVmMessage(m)
			conn.Write(data)

			fmt.Printf("client exec cmd request %s\n", msg.Message[:])
			err = vm.Exec(conn, conn, string(msg.Message[:]), tag, vm.Pod.Containers[0].Id)

			if err != nil {
				fmt.Printf("read runv client data failed: %v\n", err)
			}
		}
	case RUNV_WINSIZE:
		{
			var winSize ttyWinSize
			json.Unmarshal(msg.Message, &winSize)
			//fmt.Printf("client exec winsize request %v\n", winSize)
			vm.Tty(winSize.Tag, winSize.Height, winSize.Width)
		}
	default:
		fmt.Printf("unknown cient request\n")
	}
}

func ListenAndHandleRunvRequests(vm *hypervisor.Vm, sock net.Listener) {
	go func() {
		for {
			conn, err := sock.Accept()
			if err != nil {
				fmt.Printf("accept on runv Socket err: %v\n", err)
				break
			}

			go HandleRunvRequest(vm, conn)
		}
	}()
}
<commit_msg>move tag allocation and respond code to runvAllocAndRespondTag()<commit_after>package main

import (
	"bytes"
	"crypto/rand"
	"encoding/hex"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"net"
	"os"
	"os/exec"
	"path"
	"path/filepath"
	"strconv"

	"github.com/hyperhq/runv/driverloader"
	"github.com/hyperhq/runv/hypervisor"
	"github.com/hyperhq/runv/hypervisor/pod"
	"github.com/hyperhq/runv/hypervisor/types"
	"github.com/hyperhq/runv/lib/term"

	"github.com/opencontainers/specs"
)

const (
	_ = iota
	RUNV_ACK
	RUNV_EXECCMD
	RUNV_WINSIZE
)

const shortLen = 12

func TruncateID(id string) string {
	trimTo := shortLen
	if len(id) < shortLen {
		trimTo = len(id)
	}
	return id[:trimTo]
}

// GenerateRandomID returns an unique id
func GenerateRandomID() string {
	for {
		id := make([]byte, 32)
		if _, err := io.ReadFull(rand.Reader, id); err != nil {
			panic(err) // This shouldn't happen
		}
		value := hex.EncodeToString(id)
		// if we try to parse the truncated for as an int and we don't have
		// an error then the value is all numberic and causes issues when
		// used as a hostname. ref #3869
		if _, err := strconv.ParseInt(TruncateID(value), 10, 64); err == nil {
			continue
		}
		return value
	}
}

func removeState(container, root string, sock net.Listener) {
	os.RemoveAll(path.Join(root, container))
	sock.Close()
}

func saveState(container, root string, state *specs.State) (net.Listener, error) {
	stateDir := path.Join(root, container)

	_, err := os.Stat(stateDir)
	if err == nil {
		return nil, fmt.Errorf("Container %s exists\n", container)
	}

	err = os.MkdirAll(stateDir, 0644)
	if err != nil {
		fmt.Printf("%s\n", err.Error())
		return nil, err
	}

	defer func() {
		if err != nil {
			os.RemoveAll(stateDir)
		}
	}()

	stateData, err := json.MarshalIndent(state, "", "\t")
	if err != nil {
		fmt.Printf("%s\n", err.Error())
		return nil, err
	}

	stateFile := path.Join(stateDir, "state.json")
	err = ioutil.WriteFile(stateFile, stateData, 0644)
	if err != nil {
		fmt.Printf("%s\n", err.Error())
		return nil, err
	}

	sock, err := net.Listen("unix", path.Join(stateDir, "runv.sock"))
	if err != nil {
		fmt.Printf("%s\n", err.Error())
		return nil, err
	}

	return sock, nil
}

func execHook(hook specs.Hook, state *specs.State) error {
	b, err := json.Marshal(state)
	if err != nil {
		return err
	}
	cmd := exec.Cmd{
		Path:  hook.Path,
		Args:  hook.Args,
		Env:   hook.Env,
		Stdin: bytes.NewReader(b),
	}
	return cmd.Run()
}

func execPrestartHooks(rt *specs.RuntimeSpec, state *specs.State) error {
	for _, hook := range rt.Hooks.Prestart {
		err := execHook(hook, state)
		if err != nil {
			return err
		}
	}

	return nil
}

func execPoststartHooks(rt *specs.RuntimeSpec, state *specs.State) error {
	for _, hook := range rt.Hooks.Poststart {
		err := execHook(hook, state)
		if err != nil {
			fmt.Printf("exec Poststart hook %s failed %s", hook.Path, err.Error())
		}
	}

	return nil
}

func execPoststopHooks(rt *specs.RuntimeSpec, state *specs.State) error {
	for _, hook := range rt.Hooks.Poststop {
		err := execHook(hook, state)
		if err != nil {
			fmt.Printf("exec Poststop hook %s failed %s", hook.Path, err.Error())
		}
	}

	return nil
}

func prepareInfo(config *startConfig, c *pod.UserContainer, vmId string) (*hypervisor.ContainerInfo, error) {
	var root string
	var err error

	containerId := GenerateRandomID()
	sharedDir := path.Join(hypervisor.BaseDir, vmId, hypervisor.ShareDirTag)
	rootDir := path.Join(sharedDir, containerId)
	os.MkdirAll(rootDir, 0755)

	rootDir = path.Join(rootDir, "rootfs")

	if !filepath.IsAbs(c.Image) {
		root = path.Join(config.BundlePath, c.Image)
	} else {
		root = c.Image
	}

	err = mount(root, rootDir)
	if err != nil {
		fmt.Printf("mount %s to %s failed: %s\n", root, rootDir, err.Error())
		return nil, err
	}

	containerInfo := &hypervisor.ContainerInfo{
		Id:     containerId,
		Rootfs: "rootfs",
		Image:  containerId,
		Fstype: "dir",
	}

	return containerInfo, nil
}

func startVm(config *startConfig, userPod *pod.UserPod, vmId string) (*hypervisor.Vm, error) {
	var (
		err error
		cpu = 1
		mem = 128
	)

	vbox := config.Vbox
	if _, err = os.Stat(vbox); err == nil {
		vbox, err = filepath.Abs(vbox)
		if err != nil {
			fmt.Printf("Cannot get abs path for vbox: %s\n", err.Error())
			return nil, err
		}
	}

	kernel := config.Kernel
	if _, err = os.Stat(kernel); err == nil {
		kernel, err = filepath.Abs(kernel)
		if err != nil {
			fmt.Printf("Cannot get abs path for kernel: %s\n", err.Error())
			return nil, err
		}
	}

	initrd := config.Initrd
	if _, err = os.Stat(initrd); err == nil {
		initrd, err = filepath.Abs(initrd)
		if err != nil {
			fmt.Printf("Cannot get abs path for initrd: %s\n", err.Error())
			return nil, err
		}
	}

	if userPod.Resource.Vcpu > 0 {
		cpu = userPod.Resource.Vcpu
	}

	if userPod.Resource.Memory > 0 {
		mem = userPod.Resource.Memory
	}

	b := &hypervisor.BootConfig{
		Kernel: kernel,
		Initrd: initrd,
		Bios:   "",
		Cbfs:   "",
		Vbox:   vbox,
		CPU:    cpu,
		Memory: mem,
	}

	vm := hypervisor.NewVm(vmId, cpu, mem, false, types.VM_KEEP_NONE)
	err = vm.Launch(b)
	if err != nil {
		fmt.Printf("%s\n", err.Error())
		return nil, err
	}

	return vm, nil
}

// pod context for runv daemon
type runvPodContext struct {
	podId     string
	vmId      string
	userPod   *pod.UserPod
	podStatus *hypervisor.PodStatus
	vm        *hypervisor.Vm
}

var daemonPodContext runvPodContext

func startRunvPod(config *startConfig) (context *runvPodContext, err error) {
	context = &daemonPodContext

	hypervisor.InterfaceCount = 0

	driver := config.Driver
	if hypervisor.HDriver, err = driverloader.Probe(driver); err != nil {
		fmt.Printf("%s\n", err.Error())
		return nil, err
	}

	context.podId = fmt.Sprintf("pod-%s", pod.RandStr(10, "alpha"))
	context.vmId = fmt.Sprintf("vm-%s", pod.RandStr(10, "alpha"))

	context.userPod = pod.ConvertOCF2PureUserPod(&config.LinuxSpec, &config.LinuxRuntimeSpec)
	context.podStatus = hypervisor.NewPod(context.podId, context.userPod)
	context.vm, err = startVm(config, context.userPod, context.vmId)
	if err != nil {
		fmt.Printf("%s\n", err.Error())
		return nil, err
	}

	Response := context.vm.StartPod(context.podStatus, context.userPod, nil, nil)
	if Response.Data == nil {
		fmt.Printf("StartPod fail: QEMU response data is nil\n")
		return nil, fmt.Errorf("StartPod fail")
	}
	fmt.Printf("result: code %d %s\n", Response.Code, Response.Cause)

	return context, nil
}

func cleanupRunvPod(context *runvPodContext) {
	Response := context.vm.StopPod(context.podStatus, "yes")

	if Response.Data == nil {
		fmt.Printf("StopPod fail: QEMU response data is nil\n")
		return
	}
	fmt.Printf("result: code %d %s\n", Response.Code, Response.Cause)
	os.RemoveAll(path.Join(hypervisor.BaseDir, context.vmId))
}

func startVContainer(config *startConfig) {
	context, err := startRunvPod(config)
	if err != nil {
		fmt.Printf("Start Pod failed: %s\n", err.Error())
		return
	}
	defer cleanupRunvPod(context)

	root := config.Root
	container := config.Name
	fmt.Printf("runv container id: %s\n", container)

	state := &specs.State{
		Version:    config.LinuxSpec.Spec.Version,
		ID:         container,
		Pid:        -1,
		BundlePath: config.BundlePath,
	}

	sock, err := saveState(container, root, state)
	if err != nil {
		fmt.Printf("%s\n", err.Error())
		return
	}

	defer removeState(container, root, sock)

	err = execPrestartHooks(&config.LinuxRuntimeSpec.RuntimeSpec, state)
	if err != nil {
		fmt.Printf("execute Prestart hooks failed, %s\n", err.Error())
		return
	}

	inFd, _ := term.GetFdInfo(os.Stdin)
	outFd, isTerminalOut := term.GetFdInfo(os.Stdout)

	oldState, err := term.SetRawTerminal(inFd)
	if err != nil {
		return
	}

	defer term.RestoreTerminal(inFd, oldState)

	tag := pod.RandStr(8, "alphanum")
	ttyCallback := make(chan *types.VmResponse, 1)

	// using pipes in vm.Attach to avoid the stdio to be closed
	fromStd, toVm := io.Pipe()
	fromVm, toStd := io.Pipe()
	go io.Copy(toVm, os.Stdin)
	go io.Copy(os.Stdout, fromVm)

	userContainer := pod.ConvertOCF2UserContainer(&config.LinuxSpec, &config.LinuxRuntimeSpec)
	info, err := prepareInfo(config, userContainer, context.vmId)
	if err != nil {
		fmt.Printf("%s\n", err.Error())
		return
	}

	defer func() {
		rootDir := path.Join(hypervisor.BaseDir, context.vmId, hypervisor.ShareDirTag, info.Id, "rootfs")
		umount(rootDir)
		os.RemoveAll(path.Join(hypervisor.BaseDir, context.vmId, hypervisor.ShareDirTag, info.Id))
	}()

	err = context.vm.Attach(fromStd, toStd, tag, info.Id, ttyCallback, nil)
	if err != nil {
		fmt.Printf("StartPod fail: fail to set up tty connection.\n")
		return
	}

	context.podStatus.AddContainer(info.Id, context.podId, "", []string{}, types.S_POD_CREATED)
	context.vm.NewContainer(userContainer, info)
	ListenAndHandleRunvRequests(context.vm, sock)

	err = execPoststartHooks(&config.LinuxRuntimeSpec.RuntimeSpec, state)
	if err != nil {
		fmt.Printf("execute Poststart hooks failed %s\n", err.Error())
	}

	newTty(context.vm, path.Join(root, container), tag, outFd, isTerminalOut).monitorTtySize()
	<-ttyCallback

	err = execPoststopHooks(&config.LinuxRuntimeSpec.RuntimeSpec, state)
	if err != nil {
		fmt.Printf("execute Poststop hooks failed %s\n", err.Error())
		return
	}
}

func runvAllocAndRespondTag(conn net.Conn) (tag string, err error) {
	tag = pod.RandStr(8, "alphanum")
	m := &hypervisor.DecodedMessage{
		Code:    RUNV_ACK,
		Message: []byte(tag),
	}
	data := hypervisor.NewVmMessage(m)
	conn.Write(data)

	return tag, nil
}

func runvGetTag(conn net.Conn) (tag string, err error) {
	msg, err := hypervisor.ReadVmMessage(conn.(*net.UnixConn))
	if err != nil {
		fmt.Printf("read runv server data failed: %v\n", err)
		return "", err
	}

	if msg.Code != RUNV_ACK {
		return "", fmt.Errorf("unexpected respond code")
	}

	return string(msg.Message), nil
}

func HandleRunvRequest(vm *hypervisor.Vm, conn net.Conn) {
	defer conn.Close()

	msg, err := hypervisor.ReadVmMessage(conn.(*net.UnixConn))
	if err != nil {
		fmt.Printf("read runv client data failed: %v\n", err)
		return
	}

	switch msg.Code {
	case RUNV_EXECCMD:
		{
			tag, _ := runvAllocAndRespondTag(conn)

			fmt.Printf("client exec cmd request %s\n", msg.Message[:])
			err = vm.Exec(conn, conn, string(msg.Message[:]), tag, vm.Pod.Containers[0].Id)

			if err != nil {
				fmt.Printf("read runv client data failed: %v\n", err)
			}
		}
	case RUNV_WINSIZE:
		{
			var winSize ttyWinSize
			json.Unmarshal(msg.Message, &winSize)
			//fmt.Printf("client exec winsize request %v\n", winSize)
			vm.Tty(winSize.Tag, winSize.Height, winSize.Width)
		}
	default:
		fmt.Printf("unknown cient request\n")
	}
}

func ListenAndHandleRunvRequests(vm *hypervisor.Vm, sock net.Listener) {
	go func() {
		for {
			conn, err := sock.Accept()
			if err != nil {
				fmt.Printf("accept on runv Socket err: %v\n", err)
				break
			}

			go HandleRunvRequest(vm, conn)
		}
	}()
}
<|endoftext|>


<commit_before>package main

import (
	"errors"
	"github.com/kr/fs"
	"io"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"sort"
	"strings"
)

var cmdSave = &Command{
	Usage: "save [-copy=false] [packages]",
	Short: "list and copy dependencies into Godeps",
	Long: `
Save writes a list of the dependencies of the named packages along
with the exact source control revision of each dependency, and copies
their source code into a subdirectory.

The dependency list is a JSON document with the following structure:

	type Godeps struct {
		ImportPath string
		GoVersion  string   // Abridged output of 'go version'.
		Packages   []string // Arguments to godep save, if any.
		Deps       []struct {
			ImportPath string
			Comment    string // Tag or description of commit.
			Rev        string // VCS-specific commit ID.
		}
	}

If -copy=false is given, the list alone is written to file Godeps.

Otherwise, the list is written to Godeps/Godeps.json, and source
code for all dependencies is copied into Godeps/_workspace.

For more about specifying packages, see 'go help packages'.
`,
	Run: runSave,
}

var flagCopy = true

func init() {
	cmdSave.Flag.BoolVar(&flagCopy, "copy", true, "copy source code")
}

func runSave(cmd *Command, args []string) {
	// Remove Godeps before listing packages, so that args
	// such as ./... don't match anything in there.
	if err := os.RemoveAll("Godeps"); err != nil {
		log.Fatalln(err)
	}
	g := &Godeps{
		ImportPath: MustLoadPackages(".")[0].ImportPath,
		GoVersion:  mustGoVersion(),
	}
	if len(args) > 0 {
		g.Packages = args
	} else {
		args = []string{"."}
	}
	a := MustLoadPackages(args...)
	err := g.Load(a)
	if err != nil {
		log.Fatalln(err)
	}
	if a := badSandboxVCS(g.Deps); a != nil && !flagCopy {
		log.Println("Unsupported sandbox VCS:", strings.Join(a, ", "))
		log.Printf("Instead, run: godep save -copy %s", strings.Join(args, " "))
		os.Exit(1)
	}
	if g.Deps == nil {
		g.Deps = make([]Dependency, 0) // produce json [], not null
	}
	manifest := "Godeps"
	if flagCopy {
		manifest = filepath.Join("Godeps", "Godeps.json")
		// We use a name starting with "_" so the go tool
		// ignores this directory when traversing packages
		// starting at the project's root. For example,
		//   godep go list ./...
		workspace := filepath.Join("Godeps", "_workspace")
		err = copySrc(workspace, g)
		if err != nil {
			log.Fatalln(err)
		}
		path := filepath.Join("Godeps", "Readme")
		err = writeFile(path, strings.TrimSpace(Readme)+"\n")
		if err != nil {
			log.Println(err)
		}
		writeVCSIgnore(workspace)
	}
	f, err := os.Create(manifest)
	if err != nil {
		log.Fatalln(err)
	}
	_, err = g.WriteTo(f)
	if err != nil {
		log.Fatalln(err)
	}
	err = f.Close()
	if err != nil {
		log.Fatalln(err)
	}
}

// badSandboxVCS returns a list of VCSes that don't work
// with the `godep go` sandbox code.
func badSandboxVCS(deps []Dependency) (a []string) {
	for _, d := range deps {
		if d.vcs.CreateCmd == "" {
			a = append(a, d.vcs.vcs.Name)
		}
	}
	sort.Strings(a)
	return uniq(a)
}

func copySrc(dir string, g *Godeps) error {
	ok := true
	for _, dep := range g.Deps {
		w := fs.Walk(dep.dir)
		for w.Step() {
			if w.Err() != nil {
				log.Println(w.Err())
				ok = false
				continue
			}
			if c := w.Stat().Name()[0]; c == '.' || c == '_' {
				// Skip directories using a rule similar to how
				// the go tool enumerates packages.
				// See $GOROOT/src/cmd/go/main.go:/matchPackagesInFs
				w.SkipDir()
			}
			if w.Stat().IsDir() {
				continue
			}
			dst := filepath.Join(dir, w.Path()[len(dep.ws)+1:])
			if err := copyFile(dst, w.Path()); err != nil {
				log.Println(err)
				ok = false
			}
		}
	}
	if !ok {
		return errors.New("error copying source code")
	}
	return nil
}

// copyFile copies a regular file from src to dst.
// dst is opened with os.Create.
func copyFile(dst, src string) error {
	err := os.MkdirAll(filepath.Dir(dst), 0777)
	if err != nil {
		return err
	}

	linkDst, err := os.Readlink(src)
	if err == nil {
		return os.Symlink(linkDst, dst)
	}

	r, err := os.Open(src)
	if err != nil {
		return err
	}
	defer r.Close()

	w, err := os.Create(dst)
	if err != nil {
		return err
	}

	_, err = io.Copy(w, r)
	err1 := w.Close()
	if err == nil {
		err = err1
	}

	return err
}

// Func writeVCSIgnore writes "ignore" files inside dir for known VCSs,
// so that dir/pkg and dir/bin don't accidentally get committed.
// It logs any errors it encounters.
func writeVCSIgnore(dir string) {
	// Currently git is the only VCS for which we know how to do this.
	// Mercurial and Bazaar have similar mechasims, but they apparently
	// require writing files outside of dir.
	const ignore = "/pkg\n/bin\n"
	name := filepath.Join(dir, ".gitignore")
	err := writeFile(name, ignore)
	if err != nil {
		log.Println(err)
	}
}

// writeFile is like ioutil.WriteFile but it creates
// intermediate directories with os.MkdirAll.
func writeFile(name, body string) error {
	err := os.MkdirAll(filepath.Dir(name), 0777)
	if err != nil {
		return err
	}
	return ioutil.WriteFile(name, []byte(body), 0666)
}

const Readme = `
This directory tree is generated automatically by godep.

Please do not edit.

See https://github.com/kr/godep for more information.
`
<commit_msg>rename to allow for more flags cleanly<commit_after>package main

import (
	"errors"
	"github.com/kr/fs"
	"io"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"sort"
	"strings"
)

var cmdSave = &Command{
	Usage: "save [-copy=false] [packages]",
	Short: "list and copy dependencies into Godeps",
	Long: `
Save writes a list of the dependencies of the named packages along
with the exact source control revision of each dependency, and copies
their source code into a subdirectory.

The dependency list is a JSON document with the following structure:

	type Godeps struct {
		ImportPath string
		GoVersion  string   // Abridged output of 'go version'.
		Packages   []string // Arguments to godep save, if any.
		Deps       []struct {
			ImportPath string
			Comment    string // Tag or description of commit.
			Rev        string // VCS-specific commit ID.
		}
	}

If -copy=false is given, the list alone is written to file Godeps.

Otherwise, the list is written to Godeps/Godeps.json, and source
code for all dependencies is copied into Godeps/_workspace.

For more about specifying packages, see 'go help packages'.
`,
	Run: runSave,
}

var saveCopy = true

func init() {
	cmdSave.Flag.BoolVar(&saveCopy, "copy", true, "copy source code")
}

func runSave(cmd *Command, args []string) {
	// Remove Godeps before listing packages, so that args
	// such as ./... don't match anything in there.
	if err := os.RemoveAll("Godeps"); err != nil {
		log.Fatalln(err)
	}
	g := &Godeps{
		ImportPath: MustLoadPackages(".")[0].ImportPath,
		GoVersion:  mustGoVersion(),
	}
	if len(args) > 0 {
		g.Packages = args
	} else {
		args = []string{"."}
	}
	a := MustLoadPackages(args...)
	err := g.Load(a)
	if err != nil {
		log.Fatalln(err)
	}
	if a := badSandboxVCS(g.Deps); a != nil && !saveCopy {
		log.Println("Unsupported sandbox VCS:", strings.Join(a, ", "))
		log.Printf("Instead, run: godep save -copy %s", strings.Join(args, " "))
		os.Exit(1)
	}
	if g.Deps == nil {
		g.Deps = make([]Dependency, 0) // produce json [], not null
	}
	manifest := "Godeps"
	if saveCopy {
		manifest = filepath.Join("Godeps", "Godeps.json")
		// We use a name starting with "_" so the go tool
		// ignores this directory when traversing packages
		// starting at the project's root. For example,
		//   godep go list ./...
		workspace := filepath.Join("Godeps", "_workspace")
		err = copySrc(workspace, g)
		if err != nil {
			log.Fatalln(err)
		}
		path := filepath.Join("Godeps", "Readme")
		err = writeFile(path, strings.TrimSpace(Readme)+"\n")
		if err != nil {
			log.Println(err)
		}
		writeVCSIgnore(workspace)
	}
	f, err := os.Create(manifest)
	if err != nil {
		log.Fatalln(err)
	}
	_, err = g.WriteTo(f)
	if err != nil {
		log.Fatalln(err)
	}
	err = f.Close()
	if err != nil {
		log.Fatalln(err)
	}
}

// badSandboxVCS returns a list of VCSes that don't work
// with the `godep go` sandbox code.
func badSandboxVCS(deps []Dependency) (a []string) {
	for _, d := range deps {
		if d.vcs.CreateCmd == "" {
			a = append(a, d.vcs.vcs.Name)
		}
	}
	sort.Strings(a)
	return uniq(a)
}

func copySrc(dir string, g *Godeps) error {
	ok := true
	for _, dep := range g.Deps {
		w := fs.Walk(dep.dir)
		for w.Step() {
			if w.Err() != nil {
				log.Println(w.Err())
				ok = false
				continue
			}
			if c := w.Stat().Name()[0]; c == '.' || c == '_' {
				// Skip directories using a rule similar to how
				// the go tool enumerates packages.
				// See $GOROOT/src/cmd/go/main.go:/matchPackagesInFs
				w.SkipDir()
			}
			if w.Stat().IsDir() {
				continue
			}
			dst := filepath.Join(dir, w.Path()[len(dep.ws)+1:])
			if err := copyFile(dst, w.Path()); err != nil {
				log.Println(err)
				ok = false
			}
		}
	}
	if !ok {
		return errors.New("error copying source code")
	}
	return nil
}

// copyFile copies a regular file from src to dst.
// dst is opened with os.Create.
func copyFile(dst, src string) error {
	err := os.MkdirAll(filepath.Dir(dst), 0777)
	if err != nil {
		return err
	}

	linkDst, err := os.Readlink(src)
	if err == nil {
		return os.Symlink(linkDst, dst)
	}

	r, err := os.Open(src)
	if err != nil {
		return err
	}
	defer r.Close()

	w, err := os.Create(dst)
	if err != nil {
		return err
	}

	_, err = io.Copy(w, r)
	err1 := w.Close()
	if err == nil {
		err = err1
	}

	return err
}

// Func writeVCSIgnore writes "ignore" files inside dir for known VCSs,
// so that dir/pkg and dir/bin don't accidentally get committed.
// It logs any errors it encounters.
func writeVCSIgnore(dir string) {
	// Currently git is the only VCS for which we know how to do this.
	// Mercurial and Bazaar have similar mechasims, but they apparently
	// require writing files outside of dir.
	const ignore = "/pkg\n/bin\n"
	name := filepath.Join(dir, ".gitignore")
	err := writeFile(name, ignore)
	if err != nil {
		log.Println(err)
	}
}

// writeFile is like ioutil.WriteFile but it creates
// intermediate directories with os.MkdirAll.
func writeFile(name, body string) error {
	err := os.MkdirAll(filepath.Dir(name), 0777)
	if err != nil {
		return err
	}
	return ioutil.WriteFile(name, []byte(body), 0666)
}

const Readme = `
This directory tree is generated automatically by godep.

Please do not edit.

See https://github.com/kr/godep for more information.
`
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"go/build"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"regexp"
	"sort"
	"strings"

	"sourcegraph.com/sourcegraph/srclib"
	"sourcegraph.com/sourcegraph/srclib/unit"
)

func init() {
	_, err := parser.AddCommand("scan",
		"scan for Go packages",
		"Scan the directory tree rooted at the current directory for Go packages.",
		&scanCmd,
	)
	if err != nil {
		log.Fatal(err)
	}
}

type ScanCmd struct {
	Repo   string `long:"repo" description:"repository URI" value-name:"URI"`
	Subdir string `long:"subdir" description:"subdirectory in repository" value-name:"DIR"`
}

var scanCmd ScanCmd

func (c *ScanCmd) Execute(args []string) error {
	if c.Repo == "" && os.Getenv("IN_DOCKER_CONTAINER") != "" {
		log.Println("Warning: no --repo specified, and tool is running in a Docker container (i.e., without awareness of host's GOPATH). Go import paths in source units produced by the scanner may be inaccurate. To fix this, ensure that the --repo URI is specified. Report this issue if you are seeing it unexpectedly.")
	}

	if err := json.NewDecoder(os.Stdin).Decode(&config); err != nil {
		return err
	}
	if err := os.Stdin.Close(); err != nil {
		return err
	}

	// Automatically detect vendored dirs (check for vendor/src and
	// Godeps/_workspace/src) and set up GOPATH pointing to them if
	// they exist.
	var setAutoGOPATH bool
	if config.GOPATH == "" {
		vendorDirs := []string{"vendor", "Godeps/_workspace"}
		var foundGOPATHs []string
		for _, vdir := range vendorDirs {
			if fi, err := os.Stat(filepath.Join(cwd, vdir, "src")); err == nil && fi.Mode().IsDir() {
				foundGOPATHs = append(foundGOPATHs, vdir)
				setAutoGOPATH = true
				log.Printf("Adding %s to GOPATH (auto-detected Go vendored dependencies source dir %s). If you don't want this, make a Srcfile with a GOPATH property set to something other than the empty string.", vdir, filepath.Join(vdir, "src"))
			}
		}
		config.GOPATH = strings.Join(foundGOPATHs, string(filepath.ListSeparator))
	}

	if err := config.apply(); err != nil {
		return err
	}

	cwd, err := filepath.EvalSymlinks(getCWD())
	if err != nil {
		return err
	}
	scanDir := cwd
	if !isInGopath(scanDir) {
		scanDir = filepath.Join(cwd, srclibGopath, "src", filepath.FromSlash(c.Repo))
		buildContext.GOPATH = filepath.Join(cwd, srclibGopath) + string(os.PathListSeparator) + buildContext.GOPATH

		os.RemoveAll(srclibGopath) // ignore error
		if err := os.MkdirAll(filepath.Dir(scanDir), 0777); err != nil {
			return err
		}
		rel, err := filepath.Rel(filepath.Dir(scanDir), cwd)
		if err != nil {
			return err
		}
		if err := os.Symlink(rel, scanDir); err != nil {
			return err
		}
	}

	units, err := scan(scanDir)
	if err != nil {
		return err
	}

	if len(config.PkgPatterns) != 0 {
		matchers := make([]func(name string) bool, len(config.PkgPatterns))
		for i, pattern := range config.PkgPatterns {
			matchers[i] = matchPattern(pattern)
		}

		var filteredUnits []*unit.SourceUnit
		for _, unit := range units {
			for _, m := range matchers {
				if m(unit.Name) {
					filteredUnits = append(filteredUnits, unit)
					break
				}
			}
		}
		units = filteredUnits
	}

	// Fix up import paths to be consistent when running as a program and as
	// a Docker container. But if a GOROOT is set, then we probably want import
	// paths to not contain the repo, so only do this if there's no GOROOT set
	// in the Srcfile.
	if os.Getenv("IN_DOCKER_CONTAINER") != "" && config.GOROOT == "" {
		for _, u := range units {
			pkg := u.Data.(*build.Package)
			pkg.ImportPath = filepath.Join(c.Repo, c.Subdir, pkg.Dir)
			u.Name = pkg.ImportPath
		}
	}

	// Make vendored dep unit names (package import paths) relative to
	// vendored src dir, not to top-level dir.
	if config.GOPATH != "" {
		dirs := filepath.SplitList(config.GOPATH)
		for _, dir := range dirs {
			relDir, err := filepath.Rel(cwd, dir)
			if err != nil {
				return err
			}
			srcDir := filepath.Join(relDir, "src")
			for _, u := range units {
				pkg := u.Data.(*build.Package)
				if strings.HasPrefix(pkg.Dir, srcDir) {
					relImport, err := filepath.Rel(srcDir, pkg.Dir)
					if err != nil {
						return err
					}
					pkg.ImportPath = relImport
					u.Name = pkg.ImportPath
				}
			}
		}
	}

	// make files relative to repository root
	for _, u := range units {
		pkgSubdir := filepath.Join(c.Subdir, u.Data.(*build.Package).Dir)
		for i, f := range u.Files {
			u.Files[i] = filepath.ToSlash(filepath.Join(pkgSubdir, f))
		}
	}

	// If we automatically set the GOPATH based on the presence of
	// vendor dirs, then we need to pass the GOPATH to the units
	// because it is not persisted in the Srcfile. Otherwise the other
	// tools would never see the auto-set GOPATH.
	if setAutoGOPATH {
		for _, u := range units {
			if u.Config == nil {
				u.Config = map[string]interface{}{}
			}

			dirs := filepath.SplitList(config.GOPATH)
			for i, dir := range dirs {
				relDir, err := filepath.Rel(cwd, dir)
				if err != nil {
					return err
				}
				dirs[i] = relDir
			}
			u.Config["GOPATH"] = strings.Join(dirs, string(filepath.ListSeparator))
		}
	}

	b, err := json.MarshalIndent(units, "", "  ")
	if err != nil {
		return err
	}
	if _, err := os.Stdout.Write(b); err != nil {
		return err
	}
	return nil
}

func isInGopath(path string) bool {
	for _, gopath := range filepath.SplitList(buildContext.GOPATH) {
		if strings.HasPrefix(evalSymlinks(path), filepath.Join(evalSymlinks(gopath), "src")) {
			return true
		}
	}
	return false
}

func scan(scanDir string) ([]*unit.SourceUnit, error) {
	// TODO(sqs): include xtest, but we'll have to make them have a distinctly
	// namespaced def path from the non-xtest pkg.

	pkgs, err := scanForPackages(scanDir)
	if err != nil {
		return nil, err
	}

	var units []*unit.SourceUnit
	for _, pkg := range pkgs {
		// Collect all files
		var files []string
		files = append(files, pkg.GoFiles...)
		files = append(files, pkg.CgoFiles...)
		files = append(files, pkg.IgnoredGoFiles...)
		files = append(files, pkg.CFiles...)
		files = append(files, pkg.CXXFiles...)
		files = append(files, pkg.MFiles...)
		files = append(files, pkg.HFiles...)
		files = append(files, pkg.SFiles...)
		files = append(files, pkg.SwigFiles...)
		files = append(files, pkg.SwigCXXFiles...)
		files = append(files, pkg.SysoFiles...)
		files = append(files, pkg.TestGoFiles...)
		files = append(files, pkg.XTestGoFiles...)

		// Collect all imports. We use a map to remove duplicates.
		var imports []string
		imports = append(imports, pkg.Imports...)
		imports = append(imports, pkg.TestImports...)
		imports = append(imports, pkg.XTestImports...)
		imports = uniq(imports)
		sort.Strings(imports)

		// Create appropriate type for (unit).SourceUnit
		deps := make([]interface{}, len(imports))
		for i, imp := range imports {
			deps[i] = imp
		}

		pkg.Dir, err = filepath.Rel(scanDir, pkg.Dir)
		if err != nil {
			return nil, err
		}
		pkg.Dir = filepath.ToSlash(pkg.Dir)
		pkg.BinDir = ""
		pkg.ConflictDir = ""

		// Root differs depending on the system, so it's hard to compare results
		// across environments (when running as a program). Clear it so we can
		// compare results in tests more easily.
		pkg.Root = ""
		pkg.SrcRoot = ""
		pkg.PkgRoot = ""

		pkg.ImportPos = nil
		pkg.TestImportPos = nil
		pkg.XTestImportPos = nil

		units = append(units, &unit.SourceUnit{
			Name:         pkg.ImportPath,
			Type:         "GoPackage",
			Dir:          pkg.Dir,
			Files:        files,
			Data:         pkg,
			Dependencies: deps,
			Ops:          map[string]*srclib.ToolRef{"depresolve": nil, "graph": nil},
		})
	}

	return units, nil
}

func scanForPackages(dir string) ([]*build.Package, error) {
	var pkgs []*build.Package

	pkg, err := buildContext.ImportDir(dir, 0)
	if _, isNoGoError := err.(*build.NoGoError); err != nil && !isNoGoError {
		return nil, err
	}
	if err == nil {
		pkgs = append(pkgs, pkg)
	}

	infos, err := ioutil.ReadDir(dir)
	if err != nil {
		return nil, err
	}
	for _, info := range infos {
		name := info.Name()
		fullPath := filepath.Join(dir, name)
		if info.IsDir() && ((name[0] != '.' && name[0] != '_' && name != "testdata") || (strings.HasSuffix(filepath.ToSlash(fullPath), "/Godeps/_workspace") && !config.SkipGodeps)) {
			subPkgs, err := scanForPackages(fullPath)
			if err != nil {
				return nil, err
			}
			pkgs = append(pkgs, subPkgs...)
		}
	}

	return pkgs, nil
}

// matchPattern(pattern)(name) reports whether
// name matches pattern.  Pattern is a limited glob
// pattern in which '...' means 'any string' and there
// is no other special syntax.
func matchPattern(pattern string) func(name string) bool {
	re := regexp.QuoteMeta(pattern)
	re = strings.Replace(re, `\.\.\.`, `.*`, -1)
	// Special case: foo/... matches foo too.
	if strings.HasSuffix(re, `/.*`) {
		re = re[:len(re)-len(`/.*`)] + `(/.*)?`
	}
	reg := regexp.MustCompile(`^` + re + `$`)
	return func(name string) bool {
		return reg.MatchString(name)
	}
}
<commit_msg>scan: ignore dirs with multiple packages<commit_after>package main

import (
	"encoding/json"
	"go/build"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"regexp"
	"sort"
	"strings"

	"sourcegraph.com/sourcegraph/srclib"
	"sourcegraph.com/sourcegraph/srclib/unit"
)

func init() {
	_, err := parser.AddCommand("scan",
		"scan for Go packages",
		"Scan the directory tree rooted at the current directory for Go packages.",
		&scanCmd,
	)
	if err != nil {
		log.Fatal(err)
	}
}

type ScanCmd struct {
	Repo   string `long:"repo" description:"repository URI" value-name:"URI"`
	Subdir string `long:"subdir" description:"subdirectory in repository" value-name:"DIR"`
}

var scanCmd ScanCmd

func (c *ScanCmd) Execute(args []string) error {
	if c.Repo == "" && os.Getenv("IN_DOCKER_CONTAINER") != "" {
		log.Println("Warning: no --repo specified, and tool is running in a Docker container (i.e., without awareness of host's GOPATH). Go import paths in source units produced by the scanner may be inaccurate. To fix this, ensure that the --repo URI is specified. Report this issue if you are seeing it unexpectedly.")
	}

	if err := json.NewDecoder(os.Stdin).Decode(&config); err != nil {
		return err
	}
	if err := os.Stdin.Close(); err != nil {
		return err
	}

	// Automatically detect vendored dirs (check for vendor/src and
	// Godeps/_workspace/src) and set up GOPATH pointing to them if
	// they exist.
	var setAutoGOPATH bool
	if config.GOPATH == "" {
		vendorDirs := []string{"vendor", "Godeps/_workspace"}
		var foundGOPATHs []string
		for _, vdir := range vendorDirs {
			if fi, err := os.Stat(filepath.Join(cwd, vdir, "src")); err == nil && fi.Mode().IsDir() {
				foundGOPATHs = append(foundGOPATHs, vdir)
				setAutoGOPATH = true
				log.Printf("Adding %s to GOPATH (auto-detected Go vendored dependencies source dir %s). If you don't want this, make a Srcfile with a GOPATH property set to something other than the empty string.", vdir, filepath.Join(vdir, "src"))
			}
		}
		config.GOPATH = strings.Join(foundGOPATHs, string(filepath.ListSeparator))
	}

	if err := config.apply(); err != nil {
		return err
	}

	cwd, err := filepath.EvalSymlinks(getCWD())
	if err != nil {
		return err
	}
	scanDir := cwd
	if !isInGopath(scanDir) {
		scanDir = filepath.Join(cwd, srclibGopath, "src", filepath.FromSlash(c.Repo))
		buildContext.GOPATH = filepath.Join(cwd, srclibGopath) + string(os.PathListSeparator) + buildContext.GOPATH

		os.RemoveAll(srclibGopath) // ignore error
		if err := os.MkdirAll(filepath.Dir(scanDir), 0777); err != nil {
			return err
		}
		rel, err := filepath.Rel(filepath.Dir(scanDir), cwd)
		if err != nil {
			return err
		}
		if err := os.Symlink(rel, scanDir); err != nil {
			return err
		}
	}

	units, err := scan(scanDir)
	if err != nil {
		return err
	}

	if len(config.PkgPatterns) != 0 {
		matchers := make([]func(name string) bool, len(config.PkgPatterns))
		for i, pattern := range config.PkgPatterns {
			matchers[i] = matchPattern(pattern)
		}

		var filteredUnits []*unit.SourceUnit
		for _, unit := range units {
			for _, m := range matchers {
				if m(unit.Name) {
					filteredUnits = append(filteredUnits, unit)
					break
				}
			}
		}
		units = filteredUnits
	}

	// Fix up import paths to be consistent when running as a program and as
	// a Docker container. But if a GOROOT is set, then we probably want import
	// paths to not contain the repo, so only do this if there's no GOROOT set
	// in the Srcfile.
	if os.Getenv("IN_DOCKER_CONTAINER") != "" && config.GOROOT == "" {
		for _, u := range units {
			pkg := u.Data.(*build.Package)
			pkg.ImportPath = filepath.Join(c.Repo, c.Subdir, pkg.Dir)
			u.Name = pkg.ImportPath
		}
	}

	// Make vendored dep unit names (package import paths) relative to
	// vendored src dir, not to top-level dir.
	if config.GOPATH != "" {
		dirs := filepath.SplitList(config.GOPATH)
		for _, dir := range dirs {
			relDir, err := filepath.Rel(cwd, dir)
			if err != nil {
				return err
			}
			srcDir := filepath.Join(relDir, "src")
			for _, u := range units {
				pkg := u.Data.(*build.Package)
				if strings.HasPrefix(pkg.Dir, srcDir) {
					relImport, err := filepath.Rel(srcDir, pkg.Dir)
					if err != nil {
						return err
					}
					pkg.ImportPath = relImport
					u.Name = pkg.ImportPath
				}
			}
		}
	}

	// make files relative to repository root
	for _, u := range units {
		pkgSubdir := filepath.Join(c.Subdir, u.Data.(*build.Package).Dir)
		for i, f := range u.Files {
			u.Files[i] = filepath.ToSlash(filepath.Join(pkgSubdir, f))
		}
	}

	// If we automatically set the GOPATH based on the presence of
	// vendor dirs, then we need to pass the GOPATH to the units
	// because it is not persisted in the Srcfile. Otherwise the other
	// tools would never see the auto-set GOPATH.
	if setAutoGOPATH {
		for _, u := range units {
			if u.Config == nil {
				u.Config = map[string]interface{}{}
			}

			dirs := filepath.SplitList(config.GOPATH)
			for i, dir := range dirs {
				relDir, err := filepath.Rel(cwd, dir)
				if err != nil {
					return err
				}
				dirs[i] = relDir
			}
			u.Config["GOPATH"] = strings.Join(dirs, string(filepath.ListSeparator))
		}
	}

	b, err := json.MarshalIndent(units, "", "  ")
	if err != nil {
		return err
	}
	if _, err := os.Stdout.Write(b); err != nil {
		return err
	}
	return nil
}

func isInGopath(path string) bool {
	for _, gopath := range filepath.SplitList(buildContext.GOPATH) {
		if strings.HasPrefix(evalSymlinks(path), filepath.Join(evalSymlinks(gopath), "src")) {
			return true
		}
	}
	return false
}

func scan(scanDir string) ([]*unit.SourceUnit, error) {
	// TODO(sqs): include xtest, but we'll have to make them have a distinctly
	// namespaced def path from the non-xtest pkg.

	pkgs, err := scanForPackages(scanDir)
	if err != nil {
		return nil, err
	}

	var units []*unit.SourceUnit
	for _, pkg := range pkgs {
		// Collect all files
		var files []string
		files = append(files, pkg.GoFiles...)
		files = append(files, pkg.CgoFiles...)
		files = append(files, pkg.IgnoredGoFiles...)
		files = append(files, pkg.CFiles...)
		files = append(files, pkg.CXXFiles...)
		files = append(files, pkg.MFiles...)
		files = append(files, pkg.HFiles...)
		files = append(files, pkg.SFiles...)
		files = append(files, pkg.SwigFiles...)
		files = append(files, pkg.SwigCXXFiles...)
		files = append(files, pkg.SysoFiles...)
		files = append(files, pkg.TestGoFiles...)
		files = append(files, pkg.XTestGoFiles...)

		// Collect all imports. We use a map to remove duplicates.
		var imports []string
		imports = append(imports, pkg.Imports...)
		imports = append(imports, pkg.TestImports...)
		imports = append(imports, pkg.XTestImports...)
		imports = uniq(imports)
		sort.Strings(imports)

		// Create appropriate type for (unit).SourceUnit
		deps := make([]interface{}, len(imports))
		for i, imp := range imports {
			deps[i] = imp
		}

		pkg.Dir, err = filepath.Rel(scanDir, pkg.Dir)
		if err != nil {
			return nil, err
		}
		pkg.Dir = filepath.ToSlash(pkg.Dir)
		pkg.BinDir = ""
		pkg.ConflictDir = ""

		// Root differs depending on the system, so it's hard to compare results
		// across environments (when running as a program). Clear it so we can
		// compare results in tests more easily.
		pkg.Root = ""
		pkg.SrcRoot = ""
		pkg.PkgRoot = ""

		pkg.ImportPos = nil
		pkg.TestImportPos = nil
		pkg.XTestImportPos = nil

		units = append(units, &unit.SourceUnit{
			Name:         pkg.ImportPath,
			Type:         "GoPackage",
			Dir:          pkg.Dir,
			Files:        files,
			Data:         pkg,
			Dependencies: deps,
			Ops:          map[string]*srclib.ToolRef{"depresolve": nil, "graph": nil},
		})
	}

	return units, nil
}

func scanForPackages(dir string) ([]*build.Package, error) {
	var pkgs []*build.Package

	pkg, err := buildContext.ImportDir(dir, 0)
	if _, isNoGoError := err.(*build.NoGoError); err != nil && !isNoGoError {
		return nil, err
	}
	if err == nil {
		pkgs = append(pkgs, pkg)
	}

	infos, err := ioutil.ReadDir(dir)
	if err != nil {
		return nil, err
	}
	for _, info := range infos {
		name := info.Name()
		fullPath := filepath.Join(dir, name)
		if info.IsDir() && ((name[0] != '.' && name[0] != '_' && name != "testdata") || (strings.HasSuffix(filepath.ToSlash(fullPath), "/Godeps/_workspace") && !config.SkipGodeps)) {
			subPkgs, err := scanForPackages(fullPath)
			if err != nil {
				// if dir contains multiple packages, ignore it and continue.
				if _, ok := err.(*build.MultiplePackageError); !ok {
					return nil, err
				}
			}
			pkgs = append(pkgs, subPkgs...)
		}
	}

	return pkgs, nil
}

// matchPattern(pattern)(name) reports whether
// name matches pattern.  Pattern is a limited glob
// pattern in which '...' means 'any string' and there
// is no other special syntax.
func matchPattern(pattern string) func(name string) bool {
	re := regexp.QuoteMeta(pattern)
	re = strings.Replace(re, `\.\.\.`, `.*`, -1)
	// Special case: foo/... matches foo too.
	if strings.HasSuffix(re, `/.*`) {
		re = re[:len(re)-len(`/.*`)] + `(/.*)?`
	}
	reg := regexp.MustCompile(`^` + re + `$`)
	return func(name string) bool {
		return reg.MatchString(name)
	}
}
<|endoftext|>


<commit_before>// goscws是C语言版本的scws分词库的go封装，使之更易于在Go程序中使用。
package goscws

/*
#include <stdio.h>
#include <stdlib.h>
#include <scws/scws.h>
#define SCWS_PREFIX     "/usr/local"
#cgo LDFLAGS: -lscws
*/
import "C"
import (
	"errors"
	"unsafe"
)

const (
	SCWS_XDICT_XDB     = C.SCWS_XDICT_XDB     // 表示直接读取 xdb 文件
	SCWS_XDICT_MEM     = C.SCWS_XDICT_MEM     // 表示将 xdb 文件全部加载到内存中，以 XTree 结构存放，可用异或结合另外2个使用。
	SCWS_XDICT_TXT     = C.SCWS_XDICT_TXT     // 表示要读取的词典文件是文本格式，可以和后2项结合用
	SCWS_MULTI_SHORT   = C.SCWS_MULTI_SHORT   // 短词
	SCWS_MULTI_DUALITY = C.SCWS_MULTI_DUALITY // 二元（将相邻的2个单字组合成一个词）
	SCWS_MULTI_ZMAIN   = C.SCWS_MULTI_ZMAIN   // 重要单字
	SCWS_MULTI_ZALL    = C.SCWS_MULTI_ZALL    // 全部单字
)

// scws主结构体
// typedef struct scws_st {
//   struct scws_st *p;
//   xdict_t d; // 词典指针，可检测是否为 NULL 来判断是否加载成功
//   rule_t r; // 规则集指针，可检测是否为 NULL 来判断是否加载成功
//   unsigned char *mblen;
//   unsigned int mode;
//   unsigned char *txt;
//   int len;
//   int off;
//   int wend;
//   scws_res_t res0; // scws_res_t 解释见后面
//   scws_res_t res1;
//   word_t **wmap;
//   struct scws_zchar *zmap;
// } scws_st, *scws_t;
type Scws struct {
	s    C.scws_t     // scws的C对象
	res  C.scws_res_t // C语言 分词结果集
	cur  C.scws_res_t
	text string //分词的内容
	rs   Res    //分词结果
}

// 分词结果
type Res struct {
	String string  //分词的结果
	Attr   string  //词性
	Idf    float64 //idf值
}

//  分配或初始化与 scws 系列操作的 scws_st 对象。该函数将自动分配、初始化、并返回新对象的指针。 只能通过调用 scws_free() 来释放该对象
func (s *Scws) New() (err error) {
	s.s = C.scws_new()
	if s.s == nil {
		err = errors.New("内存不足")
	}
	return
}

// 设置C语言里的scws_t，用于Fork
func (s *Scws) SetScws(scws C.scws_t) {
	s.s = scws
}

// 释放对象
func (s *Scws) Free() {
	C.scws_free(s.s)
}

// Fork一个新的对象，共享原来的字典和规则
func (s *Scws) Fork() (scws Scws, err error) {
	rs := C.scws_fork(s.s)
	if rs == nil {
		err = errors.New("内存不足")
	}
	scws = Scws{}
	scws.SetScws(rs)
	return
}

// 设定字符编码，支持gbk和utf8两种，默认是gbk
func (s *Scws) SetCharset(charset string) {
	c_charset := C.CString(charset)
	C.scws_set_charset(s.s, c_charset)
	C.free(unsafe.Pointer(c_charset))
}

// 清除并设定当前 scws 操作所有的词典文件
//
// 参数 fpath 词典的文件路径，词典格式是 XDB或TXT 格式。
// 参数 mode 有3种值，参见 AddDict。
// 返回值 成功返回 0，失败返回 -1。
// 注意 若此前 scws 句柄已经加载过词典，则此调用会先释放已经加载的全部词典。和 AddDict 的区别在于会覆盖已有词典。
func (s *Scws) SetDict(fpath string, mode C.int) (err error) {
	c_fpath := C.CString(fpath)
	if int(C.scws_set_dict(s.s, c_fpath, mode)) == -1 {
		err = errors.New("加载词典失败")
	}
	C.free(unsafe.Pointer(c_fpath))
	return
}

// 添加词典文件到当前 scws 对象。
// 若此前 scws 句柄已经加载过词典，则新加入的词典具有更高的优先权。
func (s *Scws) AddDict(fpath string, mode C.int) (err error) {
	c_fpath := C.CString(fpath)
	if int(C.scws_add_dict(s.s, c_fpath, mode)) == -1 {
		err = errors.New("加载词典失败")
	}
	C.free(unsafe.Pointer(c_fpath))
	return
}

// 设定是否将闲散文字自动以二字分词法聚合。
//
// 参数 yes 如果为 1 表示执行二分聚合，0 表示不处理，缺省为 0。
func (s *Scws) SetDuality(yes int) {
	c_yes := C.int(yes)
	C.scws_set_duality(s.s, c_yes)
	C.free(unsafe.Pointer(&c_yes))
}

// 设定规则集文件
// 参数 fpath 规则集文件的路径。若此前 scws 句柄已经加载过规则集，则此调用会先释放已经加载的规则集。
// 错误 加载失败，scws_t 结构中的 r 元素为 NULL，即通过 s->r == NULL 与否来判断加载的失败与成功。
// 注意 规则集定义了一些新词自动识别规则，包括常见的人名、地区、数字年代等。规则编写方法另行参考其它部分。
func (s *Scws) SetRule(fpath string) (err error) {
	c_fpath := C.CString(fpath)
	C.scws_set_rule(s.s, c_fpath)
	if s.s.r == nil {
		err = errors.New("加载失败")
	}
	C.free(unsafe.Pointer(c_fpath))
	return
}

// 设定分词执行时是否执行针对长词复合切分。（例：“中国人”分为“中国”、“人”、“中国人”）。

// 参数 mode 复合分词法的级别，缺省不复合分词。取值由下面几个常量异或组合：

// SCWS_MULTI_SHORT 短词
// SCWS_MULTI_DUALITY 二元（将相邻的2个单字组合成一个词）
// SCWS_MULTI_ZMAIN 重要单字
// SCWS_MULTI_ZALL 全部单字
func (s *Scws) SetMulti(mode C.int) {
	C.scws_set_multi(s.s, mode)
}

// 设定分词结果是否忽略所有的标点等特殊符号（不会忽略\r和\n）。
//
// 参数 yes 1 表示忽略，0 表示不忽略，缺省情况为不忽略。
func (s *Scws) SetIgnore(yes int) {
	c_yes := C.int(yes)
	C.scws_set_ignore(s.s, c_yes)
	C.free(unsafe.Pointer(&c_yes))
}

// C.scws_send_text(s, text, C.int(len(C.GoString(text))))
// void scws_send_text(scws_t s, const char *text, int len) 设定要切分的文本数据。
// 参数 text 文本字符串指针。
// 参数 len 文本的长度。
// 注意 该函数可安全用于二进制数据，不会因为字符串中包括 \0 而停止切分。 这个函数应在 scws_get_result() 和 scws_get_tops() 之前调用。
// scws 结构内部维护着该字符串的指针和相应的偏移及长度，连续调用后会覆盖之前的设定；故不应在多次的 scws_get_result 循环中再调用 scws_send_text() 以免出错。
func (s *Scws) SendText(text string) {
	s.text = text
	c_text := C.CString(text)
	c_len := C.int(len(text))
	C.scws_send_text(s.s, c_text, c_len)
	C.free(unsafe.Pointer(c_text))
	C.free(unsafe.Pointer(&c_len))
}

// 获取下一个分词结果
func (s *Scws) Next() bool {
	// cur为nil说明一个res已经结束
	if s.cur == nil {
		C.scws_free_result(s.res)
		s.res = C.scws_get_result(s.s)
		s.cur = s.res
	}
	// res为nil代表分词已结束
	if s.res == nil {
		return false
	}
	// 生成分词结果
	s.rs = Res{}
	s.rs.String = s.text[s.cur.off : int(s.cur.off)+int(s.cur.len)]
	// 将C语言的char数组转成字符串
	goArray := make([]byte, len(s.cur.attr))
	p := uintptr(unsafe.Pointer(&s.cur.attr[0]))
	for i := 0; i < len(s.cur.attr); i++ {
		j := *(*byte)(unsafe.Pointer(p))
		goArray[i] = j
		p += unsafe.Sizeof(j)
	}
	s.rs.Attr = string(goArray)
	s.rs.Idf = float64(s.cur.idf)
	s.cur = s.cur.next
	return true
}

// 返回结果
func (s *Scws) GetRes() Res {
	return s.rs
}
<commit_msg>修正内存泄漏bug<commit_after>// goscws是C语言版本的scws分词库的go封装，使之更易于在Go程序中使用。
package goscws

/*
#include <stdio.h>
#include <stdlib.h>
#include <scws/scws.h>
#define SCWS_PREFIX     "/usr/local"
#cgo LDFLAGS: -lscws
*/
import "C"
import (
	"errors"
	"unsafe"
)

const (
	SCWS_XDICT_XDB     = C.SCWS_XDICT_XDB     // 表示直接读取 xdb 文件
	SCWS_XDICT_MEM     = C.SCWS_XDICT_MEM     // 表示将 xdb 文件全部加载到内存中，以 XTree 结构存放，可用异或结合另外2个使用。
	SCWS_XDICT_TXT     = C.SCWS_XDICT_TXT     // 表示要读取的词典文件是文本格式，可以和后2项结合用
	SCWS_MULTI_SHORT   = C.SCWS_MULTI_SHORT   // 短词
	SCWS_MULTI_DUALITY = C.SCWS_MULTI_DUALITY // 二元（将相邻的2个单字组合成一个词）
	SCWS_MULTI_ZMAIN   = C.SCWS_MULTI_ZMAIN   // 重要单字
	SCWS_MULTI_ZALL    = C.SCWS_MULTI_ZALL    // 全部单字
)

// scws主结构体
// typedef struct scws_st {
//   struct scws_st *p;
//   xdict_t d; // 词典指针，可检测是否为 NULL 来判断是否加载成功
//   rule_t r; // 规则集指针，可检测是否为 NULL 来判断是否加载成功
//   unsigned char *mblen;
//   unsigned int mode;
//   unsigned char *txt;
//   int len;
//   int off;
//   int wend;
//   scws_res_t res0; // scws_res_t 解释见后面
//   scws_res_t res1;
//   word_t **wmap;
//   struct scws_zchar *zmap;
// } scws_st, *scws_t;
type Scws struct {
	s      C.scws_t     // scws的C对象
	res    C.scws_res_t // C语言 分词结果集
	cur    C.scws_res_t
	text   string  //分词的内容
	c_text *C.char //c分词的内容
	rs     Res     //分词结果
}

// 分词结果
type Res struct {
	String string  //分词的结果
	Attr   string  //词性
	Idf    float64 //idf值
}

//  分配或初始化与 scws 系列操作的 scws_st 对象。该函数将自动分配、初始化、并返回新对象的指针。 只能通过调用 scws_free() 来释放该对象
func (s *Scws) New() (err error) {
	s.s = C.scws_new()
	if s.s == nil {
		err = errors.New("内存不足")
	}
	return
}

// 设置C语言里的scws_t，用于Fork
func (s *Scws) SetScws(scws C.scws_t) {
	s.s = scws
}

// 释放对象
func (s *Scws) Free() {
	// 释放文本对象
	if s.c_text != nil {
		C.free(unsafe.Pointer(s.c_text))
	}
	// 释放scws对象
	if s.s != nil {
		C.scws_free(s.s)
	}
}

// Fork一个新的对象，共享原来的字典和规则
func (s *Scws) Fork() (scws Scws, err error) {
	rs := C.scws_fork(s.s)
	if rs == nil {
		err = errors.New("内存不足")
	}
	scws = Scws{}
	scws.SetScws(rs)
	return
}

// 设定字符编码，支持gbk和utf8两种，默认是gbk
func (s *Scws) SetCharset(charset string) {
	c_charset := C.CString(charset)
	C.scws_set_charset(s.s, c_charset)
	C.free(unsafe.Pointer(c_charset))
}

// 清除并设定当前 scws 操作所有的词典文件
//
// 参数 fpath 词典的文件路径，词典格式是 XDB或TXT 格式。
// 参数 mode 有3种值，参见 AddDict。
// 返回值 成功返回 0，失败返回 -1。
// 注意 若此前 scws 句柄已经加载过词典，则此调用会先释放已经加载的全部词典。和 AddDict 的区别在于会覆盖已有词典。
func (s *Scws) SetDict(fpath string, mode C.int) (err error) {
	c_fpath := C.CString(fpath)
	if int(C.scws_set_dict(s.s, c_fpath, mode)) == -1 {
		err = errors.New("加载词典失败")
	}
	C.free(unsafe.Pointer(c_fpath))
	return
}

// 添加词典文件到当前 scws 对象。
// 若此前 scws 句柄已经加载过词典，则新加入的词典具有更高的优先权。
func (s *Scws) AddDict(fpath string, mode C.int) (err error) {
	c_fpath := C.CString(fpath)
	if int(C.scws_add_dict(s.s, c_fpath, mode)) == -1 {
		err = errors.New("加载词典失败")
	}
	C.free(unsafe.Pointer(c_fpath))
	return
}

// 设定是否将闲散文字自动以二字分词法聚合。
//
// 参数 yes 如果为 1 表示执行二分聚合，0 表示不处理，缺省为 0。
func (s *Scws) SetDuality(yes int) {
	C.scws_set_duality(s.s, C.int(yes))
}

// 设定规则集文件
// 参数 fpath 规则集文件的路径。若此前 scws 句柄已经加载过规则集，则此调用会先释放已经加载的规则集。
// 错误 加载失败，scws_t 结构中的 r 元素为 NULL，即通过 s->r == NULL 与否来判断加载的失败与成功。
// 注意 规则集定义了一些新词自动识别规则，包括常见的人名、地区、数字年代等。规则编写方法另行参考其它部分。
func (s *Scws) SetRule(fpath string) (err error) {
	c_fpath := C.CString(fpath)
	C.scws_set_rule(s.s, c_fpath)
	if s.s.r == nil {
		err = errors.New("加载失败")
	}
	C.free(unsafe.Pointer(c_fpath))
	return
}

// 设定分词执行时是否执行针对长词复合切分。（例：“中国人”分为“中国”、“人”、“中国人”）。

// 参数 mode 复合分词法的级别，缺省不复合分词。取值由下面几个常量异或组合：

// SCWS_MULTI_SHORT 短词
// SCWS_MULTI_DUALITY 二元（将相邻的2个单字组合成一个词）
// SCWS_MULTI_ZMAIN 重要单字
// SCWS_MULTI_ZALL 全部单字
func (s *Scws) SetMulti(mode C.int) {
	C.scws_set_multi(s.s, mode)
}

// 设定分词结果是否忽略所有的标点等特殊符号（不会忽略\r和\n）。
//
// 参数 yes 1 表示忽略，0 表示不忽略，缺省情况为不忽略。
func (s *Scws) SetIgnore(yes int) {
	C.scws_set_ignore(s.s, C.int(yes))
}

// C.scws_send_text(s, text, C.int(len(C.GoString(text))))
// void scws_send_text(scws_t s, const char *text, int len) 设定要切分的文本数据。
// 参数 text 文本字符串指针。
// 参数 len 文本的长度。
// 注意 该函数可安全用于二进制数据，不会因为字符串中包括 \0 而停止切分。 这个函数应在 scws_get_result() 和 scws_get_tops() 之前调用。
// scws 结构内部维护着该字符串的指针和相应的偏移及长度，连续调用后会覆盖之前的设定；故不应在多次的 scws_get_result 循环中再调用 scws_send_text() 以免出错。
func (s *Scws) SendText(text string) {
	s.text = text
	if s.c_text != nil {
		C.free(unsafe.Pointer(s.c_text))
	}
	s.c_text = C.CString(text)
	C.scws_send_text(s.s, s.c_text, C.int(len(text)))
}

// 获取下一个分词结果
func (s *Scws) Next() bool {
	// cur为nil说明一个res已经结束
	if s.cur == nil {
		C.scws_free_result(s.res)
		s.res = C.scws_get_result(s.s)
		s.cur = s.res
	}
	// res为nil代表分词已结束
	if s.res == nil {
		if s.c_text != nil {
			C.free(unsafe.Pointer(s.c_text))
		}
		return false
	}
	// 生成分词结果
	s.rs = Res{}
	s.rs.String = s.text[s.cur.off : int(s.cur.off)+int(s.cur.len)]
	// 将C语言的char数组转成字符串
	goArray := make([]byte, len(s.cur.attr))
	p := uintptr(unsafe.Pointer(&s.cur.attr[0]))
	for i := 0; i < len(s.cur.attr); i++ {
		j := *(*byte)(unsafe.Pointer(p))
		goArray[i] = j
		p += unsafe.Sizeof(j)
	}
	s.rs.Attr = string(goArray)
	s.rs.Idf = float64(s.cur.idf)
	s.cur = s.cur.next
	return true
}

// 返回结果
func (s *Scws) GetRes() Res {
	return s.rs
}
<|endoftext|>


<commit_before>package gophermail

import (
	"net/smtp"
)

// SendMail connects to the server at addr, switches to TLS if possible,
// authenticates with mechanism a if possible, and then sends an email from
// address from, to addresses to, with message msg.
//
// Based heavily on smtp.SendMail().
func SendMail(addr string, a smtp.Auth, msg *Message) error {

	msgBytes, err := msg.Bytes()
	if err != nil {
		return err
	}

	c, err := smtp.Dial(addr)
	if err != nil {
		return err
	}

	// TODO(JPOEHLS): hello() is private and it looks like other things call Hello implicitly, do we really need this?
	// if err := c.hello(); err != nil {
	// 	return err
	// }
	if ok, _ := c.Extension("STARTTLS"); ok {
		if err = c.StartTLS(nil); err != nil {
			return err
		}
	}
	if a != nil {
		if ok, _ := c.Extension("AUTH"); ok {
			if err = c.Auth(a); err != nil {
				return err
			}
		}
	}
	if err = c.Mail(msg.Sender); err != nil {
		return err
	}
	for _, rcptAddr := range msg.To {
		if err = c.Rcpt(rcptAddr); err != nil {
			return err
		}
	}
	for _, rcptAddr := range msg.Cc {
		if err = c.Rcpt(rcptAddr); err != nil {
			return err
		}
	}
	for _, rcptAddr := range msg.Bcc {
		if err = c.Rcpt(rcptAddr); err != nil {
			return err
		}
	}
	w, err := c.Data()
	if err != nil {
		return err
	}
	_, err = w.Write(msgBytes)
	if err != nil {
		return err
	}
	err = w.Close()
	if err != nil {
		return err
	}
	return c.Quit()
}
<commit_msg>Tweaked comment.<commit_after>package gophermail

import (
	"net/smtp"
)

// SendMail connects to the server at addr, switches to TLS if possible,
// authenticates with mechanism a if possible, and then sends the given Message.
//
// Based heavily on smtp.SendMail().
func SendMail(addr string, a smtp.Auth, msg *Message) error {

	msgBytes, err := msg.Bytes()
	if err != nil {
		return err
	}

	c, err := smtp.Dial(addr)
	if err != nil {
		return err
	}

	// TODO(JPOEHLS): hello() is private and it looks like other things call Hello implicitly, do we really need this?
	// if err := c.hello(); err != nil {
	// 	return err
	// }
	if ok, _ := c.Extension("STARTTLS"); ok {
		if err = c.StartTLS(nil); err != nil {
			return err
		}
	}
	if a != nil {
		if ok, _ := c.Extension("AUTH"); ok {
			if err = c.Auth(a); err != nil {
				return err
			}
		}
	}
	if err = c.Mail(msg.Sender); err != nil {
		return err
	}
	for _, rcptAddr := range msg.To {
		if err = c.Rcpt(rcptAddr); err != nil {
			return err
		}
	}
	for _, rcptAddr := range msg.Cc {
		if err = c.Rcpt(rcptAddr); err != nil {
			return err
		}
	}
	for _, rcptAddr := range msg.Bcc {
		if err = c.Rcpt(rcptAddr); err != nil {
			return err
		}
	}
	w, err := c.Data()
	if err != nil {
		return err
	}
	_, err = w.Write(msgBytes)
	if err != nil {
		return err
	}
	err = w.Close()
	if err != nil {
		return err
	}
	return c.Quit()
}
<|endoftext|>


<commit_before>package dmgo

import (
	"bytes"
	"compress/gzip"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
)

const currentSnapshotVersion = 3

const infoString = "dmgo snapshot"

type snapshot struct {
	Version int
	Info    string
	State   json.RawMessage
	MBC     marshalledMBC
}

func (cs *cpuState) loadSnapshot(snapBytes []byte) (*cpuState, error) {
	var err error
	var reader io.Reader
	var unpackedBytes []byte
	var snap snapshot
	if reader, err = gzip.NewReader(bytes.NewReader(snapBytes)); err != nil {
		return nil, err
	} else if unpackedBytes, err = ioutil.ReadAll(reader); err != nil {
		return nil, err
	} else if err = json.Unmarshal(unpackedBytes, &snap); err != nil {
		return nil, err
	} else if snap.Version < currentSnapshotVersion {
		return cs.convertOldSnapshot(&snap)
	} else if snap.Version > currentSnapshotVersion {
		return nil, fmt.Errorf("this version of dmgo is too old to open this snapshot")
	}

	// NOTE: what about external RAM? Doesn't this overwrite .sav files with whatever's in the snapshot?

	return cs.convertLatestSnapshot(&snap)
}

func (cs *cpuState) convertLatestSnapshot(snap *snapshot) (*cpuState, error) {
	var err error
	var newState cpuState
	if err = json.Unmarshal(snap.State, &newState); err != nil {
		return nil, err
	}
	if newState.Mem.mbc, err = unmarshalMBC(snap.MBC); err != nil {
		return nil, err
	}
	newState.Mem.cart = cs.Mem.cart
	return &newState, nil
}

var snapshotConverters = map[int]func(map[string]interface{}) error{

	// NOTE: Be careful with the json here. use/read the pack.go functions.
	// golang's json marshalling can sometimes do the unexpected, e.g. byte
	// slices must be packed as base64 strings.

	// NOTE: If an MBC ever has incompatible changes, the marshalledMBC will have to
	// be passed through all these conv fns as well.

	// NOTE: If new field can be zero, no need for converter.

	// added 2017-03-01
	1: func(state map[string]interface{}) error {

		if vramStr, lcd, err := followJSON(state, "LCD", "VideoRAM"); err == nil {
			if vram, err := getByteSliceFromJSON(vramStr); err == nil {
				newVRAM := [0x4000]byte{}
				copy(newVRAM[:], vram)
				lcd["VideoRAM"] = newVRAM
			} else {
				return fmt.Errorf("could not convert old v1 snapshot: %v", err)
			}
		} else {
			return fmt.Errorf("could not convert old v1 snapshot: %v", err)
		}

		if ramStr, mem, err := followJSON(state, "Mem", "InternalRAM"); err == nil {
			if ram, err := getByteSliceFromJSON(ramStr); err == nil {
				var newRAM [0x8000]byte
				copy(newRAM[:], ram)
				mem["InternalRAM"] = newRAM
				mem["InternalRAMBankNumber"] = 1
			} else {
				return fmt.Errorf("could not convert old v1 snapshot: %v", err)
			}
		} else {
			return fmt.Errorf("could not convert old v1 snapshot: %v", err)
		}

		return nil
	},

	// added 2018-12-21
	2: func(state map[string]interface{}) error {
		if sounds, _, err := followJSON(state, "APU", "Sounds"); err == nil {
			if soundsArr, ok := sounds.([]interface{}); ok {
				for _, sound := range soundsArr {
					if soundMap, ok := sound.(map[string]interface{}); ok {
						soundMap["T"] = 0
						soundMap["PolySample"] = 0
					} else {
						return fmt.Errorf("could not convert old v2 snapshot: Sound var is of unknown type")
					}
				}
			} else {
				return fmt.Errorf("could not convert old v2 snapshot: Sounds var is of unknown type")
			}
		} else {
			return fmt.Errorf("could not convert old v2 snapshot: %v", err)
		}
		return nil
	},
}

func (cs *cpuState) convertOldSnapshot(snap *snapshot) (*cpuState, error) {

	var state map[string]interface{}
	if err := json.Unmarshal(snap.State, &state); err != nil {
		return nil, fmt.Errorf("json unpack err: %v", err)
	}

	for i := snap.Version; i < currentSnapshotVersion; i++ {
		if converterFn, ok := snapshotConverters[i]; !ok {
			return nil, fmt.Errorf("could not find converter for snapshot version: %v", i)
		} else if err := converterFn(state); err != nil {
			return nil, fmt.Errorf("error converting snapshot version %v: %v", i, err)
		}
	}

	var err error
	if snap.State, err = json.Marshal(state); err != nil {
		return nil, fmt.Errorf("json pack err: %v", err)
	}

	return cs.convertLatestSnapshot(snap)
}

func (cs *cpuState) makeSnapshot() []byte {
	var err error
	var csJSON []byte
	var snapJSON []byte
	if csJSON, err = json.Marshal(cs); err != nil {
		panic(err)
	}
	snap := snapshot{
		Version: currentSnapshotVersion,
		Info:    infoString,
		State:   json.RawMessage(csJSON),
		MBC:     cs.Mem.mbc.Marshal(),
	}
	if snapJSON, err = json.Marshal(&snap); err != nil {
		panic(err)
	}
	buf := &bytes.Buffer{}
	writer := gzip.NewWriter(buf)
	writer.Write(snapJSON)
	writer.Close()
	return buf.Bytes()
}
<commit_msg>Keep same devmode setting on snap load<commit_after>package dmgo

import (
	"bytes"
	"compress/gzip"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
)

const currentSnapshotVersion = 3

const infoString = "dmgo snapshot"

type snapshot struct {
	Version int
	Info    string
	State   json.RawMessage
	MBC     marshalledMBC
}

func (cs *cpuState) loadSnapshot(snapBytes []byte) (*cpuState, error) {
	var err error
	var reader io.Reader
	var unpackedBytes []byte
	var snap snapshot
	if reader, err = gzip.NewReader(bytes.NewReader(snapBytes)); err != nil {
		return nil, err
	} else if unpackedBytes, err = ioutil.ReadAll(reader); err != nil {
		return nil, err
	} else if err = json.Unmarshal(unpackedBytes, &snap); err != nil {
		return nil, err
	} else if snap.Version < currentSnapshotVersion {
		return cs.convertOldSnapshot(&snap)
	} else if snap.Version > currentSnapshotVersion {
		return nil, fmt.Errorf("this version of dmgo is too old to open this snapshot")
	}

	// NOTE: what about external RAM? Doesn't this overwrite .sav files with whatever's in the snapshot?

	return cs.convertLatestSnapshot(&snap)
}

func (cs *cpuState) convertLatestSnapshot(snap *snapshot) (*cpuState, error) {
	var err error
	var newState cpuState
	if err = json.Unmarshal(snap.State, &newState); err != nil {
		return nil, err
	}
	if newState.Mem.mbc, err = unmarshalMBC(snap.MBC); err != nil {
		return nil, err
	}
	newState.Mem.cart = cs.Mem.cart

	newState.devMode = cs.devMode

	return &newState, nil
}

var snapshotConverters = map[int]func(map[string]interface{}) error{

	// NOTE: Be careful with the json here. use/read the pack.go functions.
	// golang's json marshalling can sometimes do the unexpected, e.g. byte
	// slices must be packed as base64 strings.

	// NOTE: If an MBC ever has incompatible changes, the marshalledMBC will have to
	// be passed through all these conv fns as well.

	// NOTE: If new field can be zero, no need for converter.

	// added 2017-03-01
	1: func(state map[string]interface{}) error {

		if vramStr, lcd, err := followJSON(state, "LCD", "VideoRAM"); err == nil {
			if vram, err := getByteSliceFromJSON(vramStr); err == nil {
				newVRAM := [0x4000]byte{}
				copy(newVRAM[:], vram)
				lcd["VideoRAM"] = newVRAM
			} else {
				return fmt.Errorf("could not convert old v1 snapshot: %v", err)
			}
		} else {
			return fmt.Errorf("could not convert old v1 snapshot: %v", err)
		}

		if ramStr, mem, err := followJSON(state, "Mem", "InternalRAM"); err == nil {
			if ram, err := getByteSliceFromJSON(ramStr); err == nil {
				var newRAM [0x8000]byte
				copy(newRAM[:], ram)
				mem["InternalRAM"] = newRAM
				mem["InternalRAMBankNumber"] = 1
			} else {
				return fmt.Errorf("could not convert old v1 snapshot: %v", err)
			}
		} else {
			return fmt.Errorf("could not convert old v1 snapshot: %v", err)
		}

		return nil
	},

	// added 2018-12-21
	2: func(state map[string]interface{}) error {
		if sounds, _, err := followJSON(state, "APU", "Sounds"); err == nil {
			if soundsArr, ok := sounds.([]interface{}); ok {
				for _, sound := range soundsArr {
					if soundMap, ok := sound.(map[string]interface{}); ok {
						soundMap["T"] = 0
						soundMap["PolySample"] = 0
					} else {
						return fmt.Errorf("could not convert old v2 snapshot: Sound var is of unknown type")
					}
				}
			} else {
				return fmt.Errorf("could not convert old v2 snapshot: Sounds var is of unknown type")
			}
		} else {
			return fmt.Errorf("could not convert old v2 snapshot: %v", err)
		}
		return nil
	},
}

func (cs *cpuState) convertOldSnapshot(snap *snapshot) (*cpuState, error) {

	var state map[string]interface{}
	if err := json.Unmarshal(snap.State, &state); err != nil {
		return nil, fmt.Errorf("json unpack err: %v", err)
	}

	for i := snap.Version; i < currentSnapshotVersion; i++ {
		if converterFn, ok := snapshotConverters[i]; !ok {
			return nil, fmt.Errorf("could not find converter for snapshot version: %v", i)
		} else if err := converterFn(state); err != nil {
			return nil, fmt.Errorf("error converting snapshot version %v: %v", i, err)
		}
	}

	var err error
	if snap.State, err = json.Marshal(state); err != nil {
		return nil, fmt.Errorf("json pack err: %v", err)
	}

	return cs.convertLatestSnapshot(snap)
}

func (cs *cpuState) makeSnapshot() []byte {
	var err error
	var csJSON []byte
	var snapJSON []byte
	if csJSON, err = json.Marshal(cs); err != nil {
		panic(err)
	}
	snap := snapshot{
		Version: currentSnapshotVersion,
		Info:    infoString,
		State:   json.RawMessage(csJSON),
		MBC:     cs.Mem.mbc.Marshal(),
	}
	if snapJSON, err = json.Marshal(&snap); err != nil {
		panic(err)
	}
	buf := &bytes.Buffer{}
	writer := gzip.NewWriter(buf)
	writer.Write(snapJSON)
	writer.Close()
	return buf.Bytes()
}
<|endoftext|>


<commit_before>// Copyright 2017 The OpenSDS Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
/*
This module implements a entry into the OpenSDS CLI service.

*/

package cli

import (
	"os"

	"fmt"
	c "github.com/opensds/opensds/client"
	"github.com/spf13/cobra"
)

var (
	client      *c.Client
	rootCommand = &cobra.Command{
		Use:   "osdsctl",
		Short: "Administer the opensds storage cluster",
		Long:  `Admin utility for the opensds unified storage cluster.`,
		Run: func(cmd *cobra.Command, args []string) {
			cmd.Usage()
			os.Exit(1)
		},
	}
)

func init() {
	ep, ok := os.LookupEnv("OPENSDS_ENDPOINT")
	if !ok {
		fmt.Println("ERROR: You must provide the endpoint by setting " +
			"the environment variable OPENSDS_ENDPOINT")
		os.Exit(1)
	}
	client = c.NewClient(&c.Config{Endpoint: ep})
	//	rootCommand.AddCommand(versionCommand)
	rootCommand.AddCommand(volumeCommand)
	rootCommand.AddCommand(dockCommand)
	rootCommand.AddCommand(poolCommand)
	rootCommand.AddCommand(profileCommand)
}

func Run() error {
	return rootCommand.Execute()
}
<commit_msg>Update osdsctl.go<commit_after>// Copyright 2017 The OpenSDS Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
/*
This module implements a entry into the OpenSDS CLI service.

*/

package cli

import (
	"os"

	"fmt"
	c "github.com/opensds/opensds/client"
	"github.com/spf13/cobra"
)

var (
	client      *c.Client
	rootCommand = &cobra.Command{
		Use:   "osdsctl",
		Short: "Administer the opensds storage cluster",
		Long:  `Admin utility for the opensds unified storage cluster.`,
		Run: func(cmd *cobra.Command, args []string) {
			cmd.Usage()
			os.Exit(1)
		},
	}
)

func init() {
	// rootCommand.AddCommand(versionCommand)
	rootCommand.AddCommand(volumeCommand)
	rootCommand.AddCommand(dockCommand)
	rootCommand.AddCommand(poolCommand)
	rootCommand.AddCommand(profileCommand)
}

func Run() error {
	ep, ok := os.LookupEnv("OPENSDS_ENDPOINT")
	if !ok {
		fmt.Println("ERROR: You must provide the endpoint by setting " +
			    "the environment variable OPENSDS_ENDPOINT")
		os.Exit(1)
	}
	client = c.NewClient(&c.Config{Endpoint: ep})

	return rootCommand.Execute()
}
<|endoftext|>


<commit_before>package gcssync

import (
	"fmt"
	"os"
	"path/filepath"
	"sync"
)

const (
	fileWalkerBufferSize = 20
	fileWalkerEstimate   = 500
)

type fileInfo struct {
	info os.FileInfo
	path string
}

func getLocalFiles(dirname string) []fileInfo {
	rawFiles := make(chan fileInfo, fileWalkerBufferSize)
	var done sync.WaitGroup

	done.Add(1)
	go func() {
		filepath.Walk(dirname, func(path string, f os.FileInfo, err error) error {
			if err == nil && !f.IsDir() {
				rel, _ := filepath.Rel(dirname, path)
				rawFiles <- fileInfo{
					info: f,
					path: rel,
				}
			}
			return nil
		})
		close(rawFiles)
		done.Done()
	}()

	result := make([]fileInfo, 0, fileWalkerEstimate)

	// this is unnecessary, but left there in case filters for files will be implemented
	done.Add(1)
	go func() {
		for {
			f, more := <-rawFiles
			if more {
				result = append(result, f)
			} else {
				done.Done()
				return
			}
		}
	}()
	done.Wait()
	return result
}

func (c *Client) SyncFolder(from, to string) {
	fmt.Println(from, to)
	localfiles := getLocalFiles(from)
	fmt.Printf("Found %d local files\n", len(localfiles))
}
<commit_msg>feat(sync): fetch local and remote file list in parallel<commit_after>package gcssync

import (
	"code.google.com/p/google-api-go-client/storage/v1"
	"fmt"
	"os"
	"path/filepath"
	"sync"
)

const (
	fileWalkerBufferSize = 20
	fileWalkerEstimate   = 500
)

type fileInfo struct {
	info os.FileInfo
	path string
}

func getLocalFiles(dirname string) []fileInfo {
	rawFiles := make(chan fileInfo, fileWalkerBufferSize)
	var done sync.WaitGroup

	done.Add(1)
	go func() {
		filepath.Walk(dirname, func(path string, f os.FileInfo, err error) error {
			if err == nil && !f.IsDir() {
				rel, _ := filepath.Rel(dirname, path)
				rawFiles <- fileInfo{
					info: f,
					path: rel,
				}
			}
			return nil
		})
		close(rawFiles)
		done.Done()
	}()

	result := make([]fileInfo, 0, fileWalkerEstimate)

	// this is unnecessary, but left there in case filters for files will be implemented
	done.Add(1)
	go func() {
		for {
			f, more := <-rawFiles
			if more {
				result = append(result, f)
			} else {
				done.Done()
				return
			}
		}
	}()
	done.Wait()
	return result
}

func (c *Client) SyncFolder(from, to string) {
	fmt.Println(from, to)
	var localFiles []fileInfo
	var listsFetched sync.WaitGroup

	listsFetched.Add(1)
	go func() {
		localFiles = getLocalFiles(from)
		fmt.Println("Done local")
		listsFetched.Done()
	}()

	var objects *storage.Objects
	var err error

	listsFetched.Add(1)
	go func() {
		objects, err = c.ListFiles()
		if err != nil {
			fmt.Println(err)
		}
		fmt.Println("Done remote")
		listsFetched.Done()
	}()

	listsFetched.Wait()
	fmt.Printf("Found %d local files\n", len(localFiles))
	fmt.Printf("Found %d remote files\n", len(objects.Items))

	// for _, file := range localfiles {
	// 	fmt.Println(file.path)
	// }
}
<|endoftext|>