<commit_before>package woocommerce

import (
	"crypto/hmac"
	"crypto/rand"
	"crypto/sha1"
	"crypto/sha256"
	"crypto/tls"
	"encoding/base64"
	"fmt"
	"io"
	"net/http"
	"net/url"
	"sort"
	"strconv"
	"strings"
	"time"
)

const (
	Version       = "1.0.0"
	UserAgent     = "WooCommerce API Client-PHP/" + Version
	HashAlgorithm = "HMAC-SHA256"
)

type Client struct {
	storeURL  *url.URL
	ck        string
	cs        string
	option    *Option
	rawClient *http.Client
}

func NewClient(store, ck, cs string, option *Option) (*Client, error) {
	storeURL, err := url.Parse(store)
	if err != nil {
		return nil, err
	}

	if option == nil {
		option = &Option{}
	}
	if option.OauthTimestamp.IsZero() {
		option.OauthTimestamp = time.Now()
	}

	ver := "v3"
	if option.Version != "" {
		ver = option.Version
	}
	path := "/wc-api/"
	if option.API {
		path = option.APIPrefix
	}
	path = path + ver + "/"
	storeURL.Path = path

	rawClient := &http.Client{}
	if !option.VerifySSL {
		rawClient.Transport = &http.Transport{
			TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
		}
	}
	return &Client{
		storeURL:  storeURL,
		ck:        ck,
		cs:        cs,
		option:    option,
		rawClient: rawClient,
	}, nil
}

func (c *Client) basicAuth(params url.Values) string {
	params.Add("consumer_key", c.ck)
	params.Add("consumer_secret", c.cs)
	return params.Encode()
}

func (c *Client) oauth(method, urlStr string, params url.Values) string {
	if params == nil {
		params = make(url.Values)
	}
	params.Add("oauth_consumer_key", c.ck)
	params.Add("oauth_timestamp", strconv.Itoa(int(c.option.OauthTimestamp.Unix())))
	nonce := make([]byte, 16)
	rand.Read(nonce)
	sha1Nonce := fmt.Sprintf("%x", sha1.Sum(nonce))
	params.Add("oauth_nonce", sha1Nonce)
	params.Add("oauth_signature_method", HashAlgorithm)
	var keys []string
	for k, _ := range params {
		keys = append(keys, k)
	}
	sort.Strings(keys)
	var paramStrs []string
	for _, key := range keys {
		paramStrs = append(paramStrs, fmt.Sprintf("%s=%s", key, params.Get(key)))
	}
	paramStr := strings.Join(paramStrs, "&")
	params.Add("oauth_signature", c.oauthSign(method, urlStr, paramStr))
	return params.Encode()
}

func (c *Client) oauthSign(method, endpoint, params string) string {
	signingKey := c.cs
	if c.option.Version != "v1" || c.option.Version != "v2" {
		signingKey = signingKey + "&"
	}

	a := strings.Join([]string{method, url.QueryEscape(endpoint), url.QueryEscape(params)}, "&")
	mac := hmac.New(sha256.New, []byte(signingKey))
	mac.Write([]byte(a))
	signatureBytes := mac.Sum(nil)
	return base64.StdEncoding.EncodeToString(signatureBytes)
}

func (c *Client) request(method, endpoint string, params url.Values, data interface{}) (io.ReadCloser, error) {
	urlstr := c.storeURL.String() + endpoint
	var body io.Reader
	if c.storeURL.Scheme == "https" {
		urlstr += "?" + c.basicAuth(params)
	} else {
		urlstr += "?" + c.oauth(method, urlstr, params)
	}
	switch method {
	case http.MethodPost, http.MethodPut:
	case http.MethodDelete, http.MethodGet, http.MethodOptions:
	default:
		return nil, fmt.Errorf("Method is not recognised: %s", method)
	}
	req, err := http.NewRequest(method, urlstr, body)
	if err != nil {
		return nil, err
	}
	resp, err := c.rawClient.Do(req)
	if err != nil {
		return nil, err
	}
	if resp.StatusCode != http.StatusOK {
		return nil, fmt.Errorf("Request failed: %s", resp.Status)
	}
	return resp.Body, nil
}

func (c *Client) Post(endpoint string, data interface{}) (io.ReadCloser, error) {
	return c.request("POST", endpoint, nil, data)
}

func (c *Client) Put(endpoint string, data interface{}) (io.ReadCloser, error) {
	return c.request("PUT", endpoint, nil, data)
}

func (c *Client) Get(endpoint string, params url.Values) (io.ReadCloser, error) {
	return c.request("GET", endpoint, params, nil)
}

func (c *Client) Delete(endpoint string, params url.Values) (io.ReadCloser, error) {
	return c.request("POST", endpoint, params, nil)
}

func (c *Client) Options(endpoint string) (io.ReadCloser, error) {
	return c.request("OPTIONS", endpoint, nil, nil)
}
<commit_msg>fixed Option(s) name<commit_after>package woocommerce

import (
	"crypto/hmac"
	"crypto/rand"
	"crypto/sha1"
	"crypto/sha256"
	"crypto/tls"
	"encoding/base64"
	"fmt"
	"io"
	"net/http"
	"net/url"
	"sort"
	"strconv"
	"strings"
	"time"
)

const (
	Version       = "1.0.0"
	UserAgent     = "WooCommerce API Client-PHP/" + Version
	HashAlgorithm = "HMAC-SHA256"
)

type Client struct {
	storeURL  *url.URL
	ck        string
	cs        string
	option    *Options
	rawClient *http.Client
}

func NewClient(store, ck, cs string, option *Options) (*Client, error) {
	storeURL, err := url.Parse(store)
	if err != nil {
		return nil, err
	}

	if option == nil {
		option = &Options{}
	}
	if option.OauthTimestamp.IsZero() {
		option.OauthTimestamp = time.Now()
	}

	ver := "v3"
	if option.Version != "" {
		ver = option.Version
	}
	path := "/wc-api/"
	if option.API {
		path = option.APIPrefix
	}
	path = path + ver + "/"
	storeURL.Path = path

	rawClient := &http.Client{}
	if !option.VerifySSL {
		rawClient.Transport = &http.Transport{
			TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
		}
	}
	return &Client{
		storeURL:  storeURL,
		ck:        ck,
		cs:        cs,
		option:    option,
		rawClient: rawClient,
	}, nil
}

func (c *Client) basicAuth(params url.Values) string {
	params.Add("consumer_key", c.ck)
	params.Add("consumer_secret", c.cs)
	return params.Encode()
}

func (c *Client) oauth(method, urlStr string, params url.Values) string {
	if params == nil {
		params = make(url.Values)
	}
	params.Add("oauth_consumer_key", c.ck)
	params.Add("oauth_timestamp", strconv.Itoa(int(c.option.OauthTimestamp.Unix())))
	nonce := make([]byte, 16)
	rand.Read(nonce)
	sha1Nonce := fmt.Sprintf("%x", sha1.Sum(nonce))
	params.Add("oauth_nonce", sha1Nonce)
	params.Add("oauth_signature_method", HashAlgorithm)
	var keys []string
	for k, _ := range params {
		keys = append(keys, k)
	}
	sort.Strings(keys)
	var paramStrs []string
	for _, key := range keys {
		paramStrs = append(paramStrs, fmt.Sprintf("%s=%s", key, params.Get(key)))
	}
	paramStr := strings.Join(paramStrs, "&")
	params.Add("oauth_signature", c.oauthSign(method, urlStr, paramStr))
	return params.Encode()
}

func (c *Client) oauthSign(method, endpoint, params string) string {
	signingKey := c.cs
	if c.option.Version != "v1" || c.option.Version != "v2" {
		signingKey = signingKey + "&"
	}

	a := strings.Join([]string{method, url.QueryEscape(endpoint), url.QueryEscape(params)}, "&")
	mac := hmac.New(sha256.New, []byte(signingKey))
	mac.Write([]byte(a))
	signatureBytes := mac.Sum(nil)
	return base64.StdEncoding.EncodeToString(signatureBytes)
}

func (c *Client) request(method, endpoint string, params url.Values, data interface{}) (io.ReadCloser, error) {
	urlstr := c.storeURL.String() + endpoint
	var body io.Reader
	if c.storeURL.Scheme == "https" {
		urlstr += "?" + c.basicAuth(params)
	} else {
		urlstr += "?" + c.oauth(method, urlstr, params)
	}
	switch method {
	case http.MethodPost, http.MethodPut:
	case http.MethodDelete, http.MethodGet, http.MethodOptions:
	default:
		return nil, fmt.Errorf("Method is not recognised: %s", method)
	}
	req, err := http.NewRequest(method, urlstr, body)
	if err != nil {
		return nil, err
	}
	resp, err := c.rawClient.Do(req)
	if err != nil {
		return nil, err
	}
	if resp.StatusCode != http.StatusOK {
		return nil, fmt.Errorf("Request failed: %s", resp.Status)
	}
	return resp.Body, nil
}

func (c *Client) Post(endpoint string, data interface{}) (io.ReadCloser, error) {
	return c.request("POST", endpoint, nil, data)
}

func (c *Client) Put(endpoint string, data interface{}) (io.ReadCloser, error) {
	return c.request("PUT", endpoint, nil, data)
}

func (c *Client) Get(endpoint string, params url.Values) (io.ReadCloser, error) {
	return c.request("GET", endpoint, params, nil)
}

func (c *Client) Delete(endpoint string, params url.Values) (io.ReadCloser, error) {
	return c.request("POST", endpoint, params, nil)
}

func (c *Client) Options(endpoint string) (io.ReadCloser, error) {
	return c.request("OPTIONS", endpoint, nil, nil)
}
<|endoftext|>


<commit_before>package console

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"regexp"
	"strings"
	"syscall"

	log "github.com/Sirupsen/logrus"
	"github.com/rancher/os/cmd/cloudinitexecute"
	"github.com/rancher/os/config"
	"github.com/rancher/os/util"
)

const (
	consoleDone = "/run/console-done"
	dockerHome  = "/home/docker"
	gettyCmd    = "/sbin/agetty"
	rancherHome = "/home/rancher"
	startScript = "/opt/rancher/bin/start.sh"
)

type symlink struct {
	oldname, newname string
}

func Main() {
	cfg := config.LoadConfig()

	if _, err := os.Stat(rancherHome); os.IsNotExist(err) {
		if err := os.MkdirAll(rancherHome, 0755); err != nil {
			log.Error(err)
		}
		if err := os.Chown(rancherHome, 1100, 1100); err != nil {
			log.Error(err)
		}
	}

	if _, err := os.Stat(dockerHome); os.IsNotExist(err) {
		if err := os.MkdirAll(dockerHome, 0755); err != nil {
			log.Error(err)
		}
		if err := os.Chown(dockerHome, 1101, 1101); err != nil {
			log.Error(err)
		}
	}

	password := config.GetCmdline("rancher.password")
	cmd := exec.Command("chpasswd")
	cmd.Stdin = strings.NewReader(fmt.Sprint("rancher:", password))
	if err := cmd.Run(); err != nil {
		log.Error(err)
	}

	cmd = exec.Command("bash", "-c", `sed -E -i 's/(rancher:.*:).*(:.*:.*:.*:.*:.*:.*)$/\1\2/' /etc/shadow`)
	if err := cmd.Run(); err != nil {
		log.Error(err)
	}

	if err := setupSSH(cfg); err != nil {
		log.Error(err)
	}

	cmdline, err := ioutil.ReadFile("/proc/cmdline")
	if err != nil {
		log.Error(err)
	}

	respawnConf := generateRespawnConf(string(cmdline))

	if err = ioutil.WriteFile("/etc/respawn.conf", respawnConf, 0644); err != nil {
		log.Error(err)
	}

	if err = modifySshdConfig(); err != nil {
		log.Error(err)
	}

	if err = writeOsRelease(); err != nil {
		log.Error(err)
	}

	for _, link := range []symlink{
		{"/var/lib/rancher/engine/docker", "/usr/bin/docker"},
		{"/var/lib/rancher/engine/docker-containerd", "/usr/bin/docker-containerd"},
		{"/var/lib/rancher/engine/docker-containerd-ctr", "/usr/bin/docker-containerd-ctr"},
		{"/var/lib/rancher/engine/docker-containerd-shim", "/usr/bin/docker-containerd-shim"},
		{"/var/lib/rancher/engine/dockerd", "/usr/bin/dockerd"},
		{"/var/lib/rancher/engine/docker-runc", "/usr/bin/docker-runc"},
	} {
		syscall.Unlink(link.newname)
		if err = os.Symlink(link.oldname, link.newname); err != nil {
			log.Error(err)
		}
	}

	cmd = exec.Command("bash", "-c", `echo 'RancherOS \n \l' > /etc/issue`)
	if err = cmd.Run(); err != nil {
		log.Error(err)
	}

	cmd = exec.Command("bash", "-c", `echo $(/sbin/ifconfig | grep -B1 "inet addr" |awk '{ if ( $1 == "inet" ) { print $2 } else if ( $2 == "Link" ) { printf "%s:" ,$1 } }' |awk -F: '{ print $1 ": " $3}') >> /etc/issue`)
	if err = cmd.Run(); err != nil {
		log.Error(err)
	}

	cloudinitexecute.ApplyConsole(cfg)

	if util.ExistsAndExecutable(config.CloudConfigScriptFile) {
		cmd := exec.Command(config.CloudConfigScriptFile)
		if err = cmd.Run(); err != nil {
			log.Error(err)
		}
	}

	if util.ExistsAndExecutable(startScript) {
		cmd := exec.Command(startScript)
		if err = cmd.Run(); err != nil {
			log.Error(err)
		}
	}

	if util.ExistsAndExecutable("/etc/rc.local") {
		cmd := exec.Command("/etc/rc.local")
		if err = cmd.Run(); err != nil {
			log.Error(err)
		}
	}

	os.Setenv("TERM", "linux")

	respawnBinPath, err := exec.LookPath("respawn")
	if err != nil {
		log.Fatal(err)
	}

	if err := ioutil.WriteFile(consoleDone, []byte(cfg.Rancher.Console), 0644); err != nil {
		log.Error(err)
	}

	log.Fatal(syscall.Exec(respawnBinPath, []string{"respawn", "-f", "/etc/respawn.conf"}, os.Environ()))
}

func generateRespawnConf(cmdline string) []byte {
	var respawnConf bytes.Buffer

	for i := 1; i < 7; i++ {
		tty := fmt.Sprintf("tty%d", i)

		respawnConf.WriteString(gettyCmd)
		if strings.Contains(cmdline, fmt.Sprintf("rancher.autologin=%s", tty)) {
			respawnConf.WriteString(" --autologin rancher")
		}
		respawnConf.WriteString(fmt.Sprintf(" 115200 %s\n", tty))
	}

	for _, tty := range []string{"ttyS0", "ttyS1", "ttyS2", "ttyS3", "ttyAMA0"} {
		if !strings.Contains(cmdline, fmt.Sprintf("console=%s", tty)) {
			continue
		}

		respawnConf.WriteString(gettyCmd)
		if strings.Contains(cmdline, fmt.Sprintf("rancher.autologin=%s", tty)) {
			respawnConf.WriteString(" --autologin rancher")
		}
		respawnConf.WriteString(fmt.Sprintf(" 115200 %s\n", tty))
	}

	respawnConf.WriteString("/usr/sbin/sshd -D")

	return respawnConf.Bytes()
}

func modifySshdConfig() error {
	sshdConfig, err := ioutil.ReadFile("/etc/ssh/sshd_config")
	if err != nil {
		return err
	}
	sshdConfigString := string(sshdConfig)

	for _, item := range []string{
		"UseDNS no",
		"PermitRootLogin no",
		"ServerKeyBits 2048",
		"AllowGroups docker",
	} {
		match, err := regexp.Match("^"+item, sshdConfig)
		if err != nil {
			return err
		}
		if !match {
			sshdConfigString += fmt.Sprintf("%s\n", item)
		}
	}

	return ioutil.WriteFile("/etc/ssh/sshd_config", []byte(sshdConfigString), 0644)
}

func writeOsRelease() error {
	idLike := "busybox"
	if osRelease, err := ioutil.ReadFile("/etc/os-release"); err == nil {
		for _, line := range strings.Split(string(osRelease), "\n") {
			if strings.HasPrefix(line, "ID_LIKE") {
				split := strings.Split(line, "ID_LIKE")
				if len(split) > 1 {
					idLike = split[1]
				}
			}
		}
	}

	return ioutil.WriteFile("/etc/os-release", []byte(fmt.Sprintf(`
NAME="RancherOS"
VERSION=%s
ID=rancheros
ID_LIKE=%s
VERSION_ID=%s
PRETTY_NAME="RancherOS %s"
HOME_URL=
SUPPORT_URL=
BUG_REPORT_URL=
BUILD_ID=
`, config.VERSION, idLike, config.VERSION, config.VERSION)), 0644)
}

func setupSSH(cfg *config.CloudConfig) error {
	for _, keyType := range []string{"rsa", "dsa", "ecdsa", "ed25519"} {
		outputFile := fmt.Sprintf("/etc/ssh/ssh_host_%s_key", keyType)
		outputFilePub := fmt.Sprintf("/etc/ssh/ssh_host_%s_key.pub", keyType)

		if _, err := os.Stat(outputFile); err == nil {
			continue
		}

		saved, savedExists := cfg.Rancher.Ssh.Keys[keyType]
		pub, pubExists := cfg.Rancher.Ssh.Keys[keyType+"-pub"]

		if savedExists && pubExists {
			// TODO check permissions
			if err := util.WriteFileAtomic(outputFile, []byte(saved), 0600); err != nil {
				return err
			}
			if err := util.WriteFileAtomic(outputFilePub, []byte(pub), 0600); err != nil {
				return err
			}
			continue
		}

		cmd := exec.Command("bash", "-c", fmt.Sprintf("ssh-keygen -f %s -N '' -t %s", outputFile, keyType))
		if err := cmd.Run(); err != nil {
			return err
		}

		savedBytes, err := ioutil.ReadFile(outputFile)
		if err != nil {
			return err
		}

		pubBytes, err := ioutil.ReadFile(outputFilePub)
		if err != nil {
			return err
		}

		config.Set(fmt.Sprintf("rancher.ssh.keys.%s", keyType), string(savedBytes))
		config.Set(fmt.Sprintf("rancher.ssh.keys.%s-pub", keyType), string(pubBytes))
	}

	return os.MkdirAll("/var/run/sshd", 0644)
}
<commit_msg>Make respawn slightly extensible<commit_after>package console

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"regexp"
	"strings"
	"syscall"

	log "github.com/Sirupsen/logrus"
	"github.com/rancher/os/cmd/cloudinitexecute"
	"github.com/rancher/os/config"
	"github.com/rancher/os/util"
)

const (
	consoleDone = "/run/console-done"
	dockerHome  = "/home/docker"
	gettyCmd    = "/sbin/agetty"
	rancherHome = "/home/rancher"
	startScript = "/opt/rancher/bin/start.sh"
)

type symlink struct {
	oldname, newname string
}

func Main() {
	cfg := config.LoadConfig()

	if _, err := os.Stat(rancherHome); os.IsNotExist(err) {
		if err := os.MkdirAll(rancherHome, 0755); err != nil {
			log.Error(err)
		}
		if err := os.Chown(rancherHome, 1100, 1100); err != nil {
			log.Error(err)
		}
	}

	if _, err := os.Stat(dockerHome); os.IsNotExist(err) {
		if err := os.MkdirAll(dockerHome, 0755); err != nil {
			log.Error(err)
		}
		if err := os.Chown(dockerHome, 1101, 1101); err != nil {
			log.Error(err)
		}
	}

	password := config.GetCmdline("rancher.password")
	cmd := exec.Command("chpasswd")
	cmd.Stdin = strings.NewReader(fmt.Sprint("rancher:", password))
	if err := cmd.Run(); err != nil {
		log.Error(err)
	}

	cmd = exec.Command("bash", "-c", `sed -E -i 's/(rancher:.*:).*(:.*:.*:.*:.*:.*:.*)$/\1\2/' /etc/shadow`)
	if err := cmd.Run(); err != nil {
		log.Error(err)
	}

	if err := setupSSH(cfg); err != nil {
		log.Error(err)
	}

	if err := writeRespawn(); err != nil {
		log.Error(err)
	}

	if err := modifySshdConfig(); err != nil {
		log.Error(err)
	}

	if err := writeOsRelease(); err != nil {
		log.Error(err)
	}

	for _, link := range []symlink{
		{"/var/lib/rancher/engine/docker", "/usr/bin/docker"},
		{"/var/lib/rancher/engine/docker-containerd", "/usr/bin/docker-containerd"},
		{"/var/lib/rancher/engine/docker-containerd-ctr", "/usr/bin/docker-containerd-ctr"},
		{"/var/lib/rancher/engine/docker-containerd-shim", "/usr/bin/docker-containerd-shim"},
		{"/var/lib/rancher/engine/dockerd", "/usr/bin/dockerd"},
		{"/var/lib/rancher/engine/docker-runc", "/usr/bin/docker-runc"},
	} {
		syscall.Unlink(link.newname)
		if err := os.Symlink(link.oldname, link.newname); err != nil {
			log.Error(err)
		}
	}

	cmd = exec.Command("bash", "-c", `echo 'RancherOS \n \l' > /etc/issue`)
	if err := cmd.Run(); err != nil {
		log.Error(err)
	}

	cmd = exec.Command("bash", "-c", `echo $(/sbin/ifconfig | grep -B1 "inet addr" |awk '{ if ( $1 == "inet" ) { print $2 } else if ( $2 == "Link" ) { printf "%s:" ,$1 } }' |awk -F: '{ print $1 ": " $3}') >> /etc/issue`)
	if err := cmd.Run(); err != nil {
		log.Error(err)
	}

	cloudinitexecute.ApplyConsole(cfg)

	if util.ExistsAndExecutable(config.CloudConfigScriptFile) {
		cmd := exec.Command(config.CloudConfigScriptFile)
		if err := cmd.Run(); err != nil {
			log.Error(err)
		}
	}

	if util.ExistsAndExecutable(startScript) {
		cmd := exec.Command(startScript)
		if err := cmd.Run(); err != nil {
			log.Error(err)
		}
	}

	if util.ExistsAndExecutable("/etc/rc.local") {
		cmd := exec.Command("/etc/rc.local")
		if err := cmd.Run(); err != nil {
			log.Error(err)
		}
	}

	os.Setenv("TERM", "linux")

	respawnBinPath, err := exec.LookPath("respawn")
	if err != nil {
		log.Fatal(err)
	}

	if err := ioutil.WriteFile(consoleDone, []byte(cfg.Rancher.Console), 0644); err != nil {
		log.Error(err)
	}

	log.Fatal(syscall.Exec(respawnBinPath, []string{"respawn", "-f", "/etc/respawn.conf"}, os.Environ()))
}

func generateRespawnConf(cmdline string) string {
	var respawnConf bytes.Buffer

	for i := 1; i < 7; i++ {
		tty := fmt.Sprintf("tty%d", i)

		respawnConf.WriteString(gettyCmd)
		if strings.Contains(cmdline, fmt.Sprintf("rancher.autologin=%s", tty)) {
			respawnConf.WriteString(" --autologin rancher")
		}
		respawnConf.WriteString(fmt.Sprintf(" 115200 %s\n", tty))
	}

	for _, tty := range []string{"ttyS0", "ttyS1", "ttyS2", "ttyS3", "ttyAMA0"} {
		if !strings.Contains(cmdline, fmt.Sprintf("console=%s", tty)) {
			continue
		}

		respawnConf.WriteString(gettyCmd)
		if strings.Contains(cmdline, fmt.Sprintf("rancher.autologin=%s", tty)) {
			respawnConf.WriteString(" --autologin rancher")
		}
		respawnConf.WriteString(fmt.Sprintf(" 115200 %s\n", tty))
	}

	respawnConf.WriteString("/usr/sbin/sshd -D")

	return respawnConf.String()
}

func writeRespawn() error {
	cmdline, err := ioutil.ReadFile("/proc/cmdline")
	if err != nil {
		return err
	}

	respawn := generateRespawnConf(string(cmdline))

	files, err := ioutil.ReadDir("/etc/respawn.conf.d")
	if err == nil {
		for _, f := range files {
			content, err := ioutil.ReadFile(f.Name())
			if err != nil {
				return err
			}
			respawn += fmt.Sprintf("\n%s", string(content))
		}
	} else if !os.IsNotExist(err) {
		log.Error(err)
	}

	return ioutil.WriteFile("/etc/respawn.conf", []byte(respawn), 0644)
}

func modifySshdConfig() error {
	sshdConfig, err := ioutil.ReadFile("/etc/ssh/sshd_config")
	if err != nil {
		return err
	}
	sshdConfigString := string(sshdConfig)

	for _, item := range []string{
		"UseDNS no",
		"PermitRootLogin no",
		"ServerKeyBits 2048",
		"AllowGroups docker",
	} {
		match, err := regexp.Match("^"+item, sshdConfig)
		if err != nil {
			return err
		}
		if !match {
			sshdConfigString += fmt.Sprintf("%s\n", item)
		}
	}

	return ioutil.WriteFile("/etc/ssh/sshd_config", []byte(sshdConfigString), 0644)
}

func writeOsRelease() error {
	idLike := "busybox"
	if osRelease, err := ioutil.ReadFile("/etc/os-release"); err == nil {
		for _, line := range strings.Split(string(osRelease), "\n") {
			if strings.HasPrefix(line, "ID_LIKE") {
				split := strings.Split(line, "ID_LIKE")
				if len(split) > 1 {
					idLike = split[1]
				}
			}
		}
	}

	return ioutil.WriteFile("/etc/os-release", []byte(fmt.Sprintf(`
NAME="RancherOS"
VERSION=%s
ID=rancheros
ID_LIKE=%s
VERSION_ID=%s
PRETTY_NAME="RancherOS %s"
HOME_URL=
SUPPORT_URL=
BUG_REPORT_URL=
BUILD_ID=
`, config.VERSION, idLike, config.VERSION, config.VERSION)), 0644)
}

func setupSSH(cfg *config.CloudConfig) error {
	for _, keyType := range []string{"rsa", "dsa", "ecdsa", "ed25519"} {
		outputFile := fmt.Sprintf("/etc/ssh/ssh_host_%s_key", keyType)
		outputFilePub := fmt.Sprintf("/etc/ssh/ssh_host_%s_key.pub", keyType)

		if _, err := os.Stat(outputFile); err == nil {
			continue
		}

		saved, savedExists := cfg.Rancher.Ssh.Keys[keyType]
		pub, pubExists := cfg.Rancher.Ssh.Keys[keyType+"-pub"]

		if savedExists && pubExists {
			// TODO check permissions
			if err := util.WriteFileAtomic(outputFile, []byte(saved), 0600); err != nil {
				return err
			}
			if err := util.WriteFileAtomic(outputFilePub, []byte(pub), 0600); err != nil {
				return err
			}
			continue
		}

		cmd := exec.Command("bash", "-c", fmt.Sprintf("ssh-keygen -f %s -N '' -t %s", outputFile, keyType))
		if err := cmd.Run(); err != nil {
			return err
		}

		savedBytes, err := ioutil.ReadFile(outputFile)
		if err != nil {
			return err
		}

		pubBytes, err := ioutil.ReadFile(outputFilePub)
		if err != nil {
			return err
		}

		config.Set(fmt.Sprintf("rancher.ssh.keys.%s", keyType), string(savedBytes))
		config.Set(fmt.Sprintf("rancher.ssh.keys.%s-pub", keyType), string(pubBytes))
	}

	return os.MkdirAll("/var/run/sshd", 0644)
}
<|endoftext|>


<commit_before>// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package cmd

import (
	"context"
	"io"
	"os"

	"github.com/spf13/cobra"

	"github.com/pilosa/pilosa/ctl"
)

var generateConf *ctl.GenerateConfigCommand

func NewGenerateConfigCommand(stdin io.Reader, stdout, stderr io.Writer) *cobra.Command {
	generateConf = ctl.NewGenerateConfigCommand(os.Stdin, os.Stdout, os.Stderr)
	confCmd := &cobra.Command{
		Use:   "generate-config",
		Short: "Print the default configuration.",
		Long: `generate-config prints the default configuration to stdout
`,
		RunE: func(cmd *cobra.Command, args []string) error {
			if err := generateConf.Run(context.Background()); err != nil {
				return err
			}
			return nil
		},
	}

	return confCmd
}

func init() {
	subcommandFns["generate-config"] = NewGenerateConfigCommand
}
<commit_msg>Unexport cmd.NewGenerateConfigCommand<commit_after>// Copyright 2017 Pilosa Corp.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package cmd

import (
	"context"
	"io"
	"os"

	"github.com/spf13/cobra"

	"github.com/pilosa/pilosa/ctl"
)

var generateConf *ctl.GenerateConfigCommand

func newGenerateConfigCommand(stdin io.Reader, stdout, stderr io.Writer) *cobra.Command {
	generateConf = ctl.NewGenerateConfigCommand(os.Stdin, os.Stdout, os.Stderr)
	confCmd := &cobra.Command{
		Use:   "generate-config",
		Short: "Print the default configuration.",
		Long: `generate-config prints the default configuration to stdout
`,
		RunE: func(cmd *cobra.Command, args []string) error {
			if err := generateConf.Run(context.Background()); err != nil {
				return err
			}
			return nil
		},
	}

	return confCmd
}

func init() {
	subcommandFns["generate-config"] = newGenerateConfigCommand
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"launchpad.net/gnuflag"
	"launchpad.net/juju-core/cmd"
	"launchpad.net/juju-core/environs"
	"launchpad.net/juju-core/log"
	"launchpad.net/juju-core/version"
)

// SyncToolsCommand copies all the tools from the us-east-1 bucket to the local
// bucket
type SyncToolsCommand struct {
	EnvCommandBase
	toolsList    *environs.ToolsList
	agentVersion version.Number
}

var _ cmd.Command = (*SyncToolsCommand)(nil)

func (c *SyncToolsCommand) Info() *cmd.Info {
	return &cmd.Info{
		Name:    "sync-tools",
		Purpose: "copy tools from another public bucket",
	}
}

func (c *SyncToolsCommand) SetFlags(f *gnuflag.FlagSet) {
	c.EnvCommandBase.SetFlags(f)
	// f.BoolVar(&c.Development, "dev", false, "allow development versions to be chosen")

}

func (c *SyncToolsCommand) Init(args []string) error {
	return nil
}

var officialBucketAttrs = map[string]interface{}{
	"name":           "juju-public",
	"type":           "ec2",
	"control-bucket": "juju-dist",
}

func (c *SyncToolsCommand) Run(_ *cmd.Context) error {
	officialEnviron, err := environs.NewFromAttrs(officialBucketAttrs)
	if err != nil {
		log.Infof("Failed to create officialEnviron")
		return err
	}
	c.toolsList, err = environs.ListTools(officialEnviron, version.Current.Major)
	if err != nil {
		return err
	}
	for _, tool := range c.toolsList.Public {
		fmt.Printf("Found: %s\n", tool.URL)
	}
	return nil
}
<commit_msg>Some tweaks<commit_after>package main

import (
	"fmt"
	"launchpad.net/gnuflag"
	"launchpad.net/juju-core/cmd"
	"launchpad.net/juju-core/environs"
	"launchpad.net/juju-core/log"
	"launchpad.net/juju-core/version"
)

// SyncToolsCommand copies all the tools from the us-east-1 bucket to the local
// bucket
type SyncToolsCommand struct {
	EnvCommandBase
	sourceToolsList *environs.ToolsList
	targetToolsList *environs.ToolsList
	agentVersion    version.Number
}

var _ cmd.Command = (*SyncToolsCommand)(nil)

func (c *SyncToolsCommand) Info() *cmd.Info {
	return &cmd.Info{
		Name:    "sync-tools",
		Purpose: "copy tools from another public bucket",
	}
}

func (c *SyncToolsCommand) SetFlags(f *gnuflag.FlagSet) {
	c.EnvCommandBase.SetFlags(f)
	// f.BoolVar(&c.Development, "dev", false, "allow development versions to be chosen")

}

func (c *SyncToolsCommand) Init(args []string) error {
	return nil
}

var officialBucketAttrs = map[string]interface{}{
	"name":           "juju-public",
	"type":           "ec2",
	"control-bucket": "juju-dist",
	"access-key":     "",
	"secret-key":     "",
}

func (c *SyncToolsCommand) Run(_ *cmd.Context) error {
	officialEnviron, err := environs.NewFromAttrs(officialBucketAttrs)
	if err != nil {
		log.Infof("Failed to create officialEnviron")
		return err
	}
	c.sourceToolsList, err = environs.ListTools(officialEnviron, version.Current.Major)
	if err != nil {
		return err
	}
	for _, tool := range c.sourceToolsList.Public {
		fmt.Printf("Found: %s\n", tool.URL)
	}
	_, err = environs.NewFromName(c.EnvName)
	if err != nil {
		return err
	}
	return nil
}
<|endoftext|>


<commit_before>/*
 * Copyright (C) 2015-2019 Virgil Security Inc.
 *
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are
 * met:
 *
 *     (1) Redistributions of source code must retain the above copyright
 *     notice, this list of conditions and the following disclaimer.
 *
 *     (2) Redistributions in binary form must reproduce the above copyright
 *     notice, this list of conditions and the following disclaimer in
 *     the documentation and/or other materials provided with the
 *     distribution.
 *
 *     (3) Neither the name of the copyright holder nor the names of its
 *     contributors may be used to endorse or promote products derived from
 *     this software without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ''AS IS'' AND ANY EXPRESS OR
 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
 * DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT,
 * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
 * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
 * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
 * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 * POSSIBILITY OF SUCH DAMAGE.
 *
 * Lead Maintainer: Virgil Security Inc. <support@virgilsecurity.com>
 */

package pure

import (
	"encoding/base64"
	"fmt"
	"strings"

	"github.com/VirgilSecurity/virgil-sdk-go/v6/crypto/wrapper/phe"
	"github.com/pkg/errors"
	"github.com/urfave/cli/v2"

	"github.com/VirgilSecurity/virgil-cli/cmd/kms"
	"github.com/VirgilSecurity/virgil-cli/utils"
)

const (
	OldKeyPartsCount = 3
)

// UpdateKeys updates secret key and public key using update token
func UpdateKeys() *cli.Command {
	return &cli.Command{
		Name:      "update-keys",
		Aliases:   []string{"u"},
		ArgsUsage: "phe_service_public_key secret_key update_token",
		Usage:     "Update secret key and phe service public key using update token",
		Action:    updateFunc,
	}
}
func updateFunc(context *cli.Context) error {
	if context.NArg() < OldKeyPartsCount {
		return errors.New("invalid number of arguments")
	}

	pkStr := context.Args().First()
	skStr := context.Args().Get(1)
	tokenStr := context.Args().Get(2)

	if isNewKeysVersion(pkStr, skStr, tokenStr) {
		err := rotate(pkStr, skStr, tokenStr)
		if err != nil {
			return utils.CliExit(err)
		}
		return err
	} else {
		err := oldRotate(pkStr, skStr, tokenStr)
		if err != nil {
			return utils.CliExit(err)
		}
		return err
	}
}

func isNewKeysVersion(pkStr, skStr, tokenStr string) bool {
	pkPartsCount := len(strings.Split(pkStr, "."))
	skPartsCount := len(strings.Split(skStr, "."))
	tokenPartsCount := len(strings.Split(tokenStr, "."))

	return pkPartsCount > OldKeyPartsCount && skPartsCount > OldKeyPartsCount && tokenPartsCount > OldKeyPartsCount
}

func oldRotate(pkStr, skStr, tokenStr string) error {
	pkVersion, pk, err := utils.ParseVersionAndContent("PK", pkStr)
	if err != nil {
		return errors.Wrapf(err, "parse public key failed: ")
	}
	skVersion, sk, err := utils.ParseVersionAndContent("SK", skStr)
	if err != nil {
		return errors.Wrapf(err, "parse private key failed: ")
	}
	tokenVersion, updateToken, err := utils.ParseVersionAndContent("UT", tokenStr)

	if err != nil {
		return errors.Wrapf(err, "parse update token failed: ")
	}

	if (pkVersion+1) != tokenVersion || (skVersion+1) != tokenVersion {
		return errors.New("Key version must be 1 less than token version")
	}

	pheClient := phe.NewPheClient()
	if err := pheClient.SetKeys(sk, pk); err != nil {
		return err
	}
	if err := pheClient.SetupDefaults(); err != nil {
		return err
	}
	newSk, newPk, err := pheClient.RotateKeys(updateToken)
	if err != nil {
		return err
	}

	fmt.Printf("New server public key:\nPK.%d.%s\nNew client private key:\nSK.%d.%s\n",
		tokenVersion, base64.StdEncoding.EncodeToString(newPk),
		tokenVersion, base64.StdEncoding.EncodeToString(newSk),
	)
	return nil
}

func rotate(pkStr, skStr, tokenStr string) error {
	pkVersion, pkPhe, pkKMS, _, err := utils.ParseCombinedEntities("PK", pkStr)
	if err != nil {
		return errors.Wrapf(err, "parse public key failed: ")
	}
	skVersion, skPhe, skKMS, skAuth, err := utils.ParseCombinedEntities("SK", skStr)
	if err != nil {
		return errors.Wrapf(err, "parse private key failed: ")
	}
	tokenVersion, updateTokenPhe, updateTokenKMS, updateTokenAuth, err := utils.ParseCombinedEntities("UT", tokenStr)

	if err != nil {
		return errors.Wrapf(err, "parse update token failed: ")
	}

	if (pkVersion+1) != tokenVersion || (skVersion+1) != tokenVersion {
		return errors.New("Key version must be 1 less than token version")
	}

	pheClient := phe.NewPheClient()
	if err := pheClient.SetKeys(skPhe, pkPhe); err != nil {
		return err
	}
	if err := pheClient.SetupDefaults(); err != nil {
		return err
	}
	newPheSk, newPhePk, err := pheClient.RotateKeys(updateTokenPhe)
	if err != nil {
		return err
	}

	newKMSSk, newKMSPk, err := kms.RotateKMSKeys(skKMS, pkKMS, updateTokenKMS)
	if err != nil {
		return err
	}

	uokmsClient := phe.NewUokmsClient()
	err = uokmsClient.SetupDefaults()
	if err != nil {
		return err
	}
	if err := uokmsClient.SetKeysOneparty(skAuth); err != nil {
		return err
	}
	newSkAuth, err := uokmsClient.RotateKeysOneparty(updateTokenAuth)
	if err != nil {
		return err
	}

	fmt.Println(utils.PureKeysRotateSuccess)
	fmt.Printf("New service public key:\nPK.%d.%s.%s\nNew secret key:\nSK.%d.%s.%s.%s\n",
		tokenVersion,
		base64.StdEncoding.EncodeToString(newPhePk),
		base64.StdEncoding.EncodeToString(newKMSPk),
		tokenVersion,
		base64.StdEncoding.EncodeToString(newPheSk),
		base64.StdEncoding.EncodeToString(newKMSSk),
		base64.StdEncoding.EncodeToString(newSkAuth),
	)
	return nil
}
<commit_msg>Fix test in pure update keys usage<commit_after>/*
 * Copyright (C) 2015-2019 Virgil Security Inc.
 *
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are
 * met:
 *
 *     (1) Redistributions of source code must retain the above copyright
 *     notice, this list of conditions and the following disclaimer.
 *
 *     (2) Redistributions in binary form must reproduce the above copyright
 *     notice, this list of conditions and the following disclaimer in
 *     the documentation and/or other materials provided with the
 *     distribution.
 *
 *     (3) Neither the name of the copyright holder nor the names of its
 *     contributors may be used to endorse or promote products derived from
 *     this software without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ''AS IS'' AND ANY EXPRESS OR
 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
 * DISCLAIMED. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT,
 * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
 * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
 * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
 * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING
 * IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 * POSSIBILITY OF SUCH DAMAGE.
 *
 * Lead Maintainer: Virgil Security Inc. <support@virgilsecurity.com>
 */

package pure

import (
	"encoding/base64"
	"fmt"
	"strings"

	"github.com/VirgilSecurity/virgil-sdk-go/v6/crypto/wrapper/phe"
	"github.com/pkg/errors"
	"github.com/urfave/cli/v2"

	"github.com/VirgilSecurity/virgil-cli/cmd/kms"
	"github.com/VirgilSecurity/virgil-cli/utils"
)

const (
	OldKeyPartsCount = 3
)

// UpdateKeys updates secret key and public key using update token
func UpdateKeys() *cli.Command {
	return &cli.Command{
		Name:      "update-keys",
		Aliases:   []string{"u"},
		ArgsUsage: "service_public_key secret_key update_token",
		Usage:     "Update secret key and service public key using update token",
		Action:    updateFunc,
	}
}
func updateFunc(context *cli.Context) error {
	if context.NArg() < OldKeyPartsCount {
		return errors.New("invalid number of arguments")
	}

	pkStr := context.Args().First()
	skStr := context.Args().Get(1)
	tokenStr := context.Args().Get(2)

	if isNewKeysVersion(pkStr, skStr, tokenStr) {
		err := rotate(pkStr, skStr, tokenStr)
		if err != nil {
			return utils.CliExit(err)
		}
		return err
	} else {
		err := oldRotate(pkStr, skStr, tokenStr)
		if err != nil {
			return utils.CliExit(err)
		}
		return err
	}
}

func isNewKeysVersion(pkStr, skStr, tokenStr string) bool {
	pkPartsCount := len(strings.Split(pkStr, "."))
	skPartsCount := len(strings.Split(skStr, "."))
	tokenPartsCount := len(strings.Split(tokenStr, "."))

	return pkPartsCount > OldKeyPartsCount && skPartsCount > OldKeyPartsCount && tokenPartsCount > OldKeyPartsCount
}

func oldRotate(pkStr, skStr, tokenStr string) error {
	pkVersion, pk, err := utils.ParseVersionAndContent("PK", pkStr)
	if err != nil {
		return errors.Wrapf(err, "parse public key failed: ")
	}
	skVersion, sk, err := utils.ParseVersionAndContent("SK", skStr)
	if err != nil {
		return errors.Wrapf(err, "parse private key failed: ")
	}
	tokenVersion, updateToken, err := utils.ParseVersionAndContent("UT", tokenStr)

	if err != nil {
		return errors.Wrapf(err, "parse update token failed: ")
	}

	if (pkVersion+1) != tokenVersion || (skVersion+1) != tokenVersion {
		return errors.New("Key version must be 1 less than token version")
	}

	pheClient := phe.NewPheClient()
	if err := pheClient.SetKeys(sk, pk); err != nil {
		return err
	}
	if err := pheClient.SetupDefaults(); err != nil {
		return err
	}
	newSk, newPk, err := pheClient.RotateKeys(updateToken)
	if err != nil {
		return err
	}

	fmt.Printf("New server public key:\nPK.%d.%s\nNew client private key:\nSK.%d.%s\n",
		tokenVersion, base64.StdEncoding.EncodeToString(newPk),
		tokenVersion, base64.StdEncoding.EncodeToString(newSk),
	)
	return nil
}

func rotate(pkStr, skStr, tokenStr string) error {
	pkVersion, pkPhe, pkKMS, _, err := utils.ParseCombinedEntities("PK", pkStr)
	if err != nil {
		return errors.Wrapf(err, "parse public key failed: ")
	}
	skVersion, skPhe, skKMS, skAuth, err := utils.ParseCombinedEntities("SK", skStr)
	if err != nil {
		return errors.Wrapf(err, "parse private key failed: ")
	}
	tokenVersion, updateTokenPhe, updateTokenKMS, updateTokenAuth, err := utils.ParseCombinedEntities("UT", tokenStr)

	if err != nil {
		return errors.Wrapf(err, "parse update token failed: ")
	}

	if (pkVersion+1) != tokenVersion || (skVersion+1) != tokenVersion {
		return errors.New("Key version must be 1 less than token version")
	}

	pheClient := phe.NewPheClient()
	if err := pheClient.SetKeys(skPhe, pkPhe); err != nil {
		return err
	}
	if err := pheClient.SetupDefaults(); err != nil {
		return err
	}
	newPheSk, newPhePk, err := pheClient.RotateKeys(updateTokenPhe)
	if err != nil {
		return err
	}

	newKMSSk, newKMSPk, err := kms.RotateKMSKeys(skKMS, pkKMS, updateTokenKMS)
	if err != nil {
		return err
	}

	uokmsClient := phe.NewUokmsClient()
	err = uokmsClient.SetupDefaults()
	if err != nil {
		return err
	}
	if err := uokmsClient.SetKeysOneparty(skAuth); err != nil {
		return err
	}
	newSkAuth, err := uokmsClient.RotateKeysOneparty(updateTokenAuth)
	if err != nil {
		return err
	}

	fmt.Println(utils.PureKeysRotateSuccess)
	fmt.Printf("New service public key:\nPK.%d.%s.%s\nNew secret key:\nSK.%d.%s.%s.%s\n",
		tokenVersion,
		base64.StdEncoding.EncodeToString(newPhePk),
		base64.StdEncoding.EncodeToString(newKMSPk),
		tokenVersion,
		base64.StdEncoding.EncodeToString(newPheSk),
		base64.StdEncoding.EncodeToString(newKMSSk),
		base64.StdEncoding.EncodeToString(newSkAuth),
	)
	return nil
}
<|endoftext|>


<commit_before>package main

// Regenerate the JSON-encoded history records on stdin
// by extracting their Data fields and decoding them again.

import (
	"encoding/base64"
	"encoding/json"
	"flag"
	"log"
	"os"

	"github.com/ecc1/medtronic"
)

var (
	model = flag.Int("m", 523, "pump model")
)

func main() {
	flag.Parse()
	family := medtronic.Family(*model % 100)
	d := json.NewDecoder(os.Stdin)
	var maps []interface{}
	if err := d.Decode(&maps); err != nil {
		log.Fatal(err)
	}
	var records medtronic.History
	for _, v := range maps {
		m := v.(map[string]interface{})
		base64data := m["Data"].(string)
		var data []byte
		data, err := base64.StdEncoding.DecodeString(base64data)
		if err != nil {
			log.Fatal(err)
		}
		r, err := medtronic.DecodeHistoryRecord(data, family)
		if err != nil {
			log.Fatal(err)
		}
		records = append(records, r)
	}
	e := json.NewEncoder(os.Stdout)
	e.SetIndent("", "  ")
	if err := e.Encode(records); err != nil {
		log.Fatal(err)
	}
}
<commit_msg>Remove var declaration<commit_after>package main

// Regenerate the JSON-encoded history records on stdin
// by extracting their Data fields and decoding them again.

import (
	"encoding/base64"
	"encoding/json"
	"flag"
	"log"
	"os"

	"github.com/ecc1/medtronic"
)

var (
	model = flag.Int("m", 523, "pump model")
)

func main() {
	flag.Parse()
	family := medtronic.Family(*model % 100)
	d := json.NewDecoder(os.Stdin)
	var maps []interface{}
	if err := d.Decode(&maps); err != nil {
		log.Fatal(err)
	}
	var records medtronic.History
	for _, v := range maps {
		m := v.(map[string]interface{})
		base64data := m["Data"].(string)
		data, err := base64.StdEncoding.DecodeString(base64data)
		if err != nil {
			log.Fatal(err)
		}
		r, err := medtronic.DecodeHistoryRecord(data, family)
		if err != nil {
			log.Fatal(err)
		}
		records = append(records, r)
	}
	e := json.NewEncoder(os.Stdout)
	e.SetIndent("", "  ")
	if err := e.Encode(records); err != nil {
		log.Fatal(err)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"context"
	"fmt"
	"os"

	"github.com/restic/restic/internal/debug"
	"github.com/restic/restic/internal/errors"
	"github.com/restic/restic/internal/restic"

	"github.com/spf13/cobra"
)

var cmdCopy = &cobra.Command{
	Use:   "copy [flags] [snapshotID ...]",
	Short: "Copy snapshots from one repository to another",
	Long: `
The "copy" command copies one or more snapshots from one repository to another
repository. Note that this will have to read (download) and write (upload) the
entire snapshot(s) due to the different encryption keys on the source and
destination, and that transferred files are not re-chunked, which may break
their deduplication.
`,
	RunE: func(cmd *cobra.Command, args []string) error {
		return runCopy(copyOptions, globalOptions, args)
	},
}

// CopyOptions bundles all options for the copy command.
type CopyOptions struct {
	Repo            string
	PasswordFile    string
	PasswordCommand string
	KeyHint         string
	Hosts           []string
	Tags            restic.TagLists
	Paths           []string
}

var copyOptions CopyOptions

func init() {
	cmdRoot.AddCommand(cmdCopy)

	f := cmdCopy.Flags()
	f.StringVarP(&copyOptions.Repo, "repo2", "", os.Getenv("RESTIC_REPOSITORY2"), "destination repository to copy snapshots to (default: $RESTIC_REPOSITORY2)")
	f.StringVarP(&copyOptions.PasswordFile, "password-file2", "", os.Getenv("RESTIC_PASSWORD_FILE2"), "read the destination repository password from a file (default: $RESTIC_PASSWORD_FILE2)")
	f.StringVarP(&copyOptions.KeyHint, "key-hint2", "", os.Getenv("RESTIC_KEY_HINT2"), "key ID of key to try decrypting the destination repository first (default: $RESTIC_KEY_HINT2)")
	f.StringVarP(&copyOptions.PasswordCommand, "password-command2", "", os.Getenv("RESTIC_PASSWORD_COMMAND2"), "specify a shell command to obtain a password for the destination repository (default: $RESTIC_PASSWORD_COMMAND2)")

	f.StringArrayVarP(&copyOptions.Hosts, "host", "H", nil, "only consider snapshots for this `host`, when no snapshot ID is given (can be specified multiple times)")
	f.Var(&copyOptions.Tags, "tag", "only consider snapshots which include this `taglist`, when no snapshot ID is given")
	f.StringArrayVar(&copyOptions.Paths, "path", nil, "only consider snapshots which include this (absolute) `path`, when no snapshot ID is given")
}

func runCopy(opts CopyOptions, gopts GlobalOptions, args []string) error {
	if opts.Repo == "" {
		return errors.Fatal("Please specify a destination repository location (--repo2)")
	}
	var err error
	dstGopts := gopts
	dstGopts.Repo = opts.Repo
	dstGopts.PasswordFile = opts.PasswordFile
	dstGopts.PasswordCommand = opts.PasswordCommand
	dstGopts.KeyHint = opts.KeyHint
	dstGopts.password, err = resolvePassword(dstGopts, "RESTIC_PASSWORD2")
	if err != nil {
		return err
	}
	dstGopts.password, err = ReadPassword(dstGopts, "enter password for destination repository: ")
	if err != nil {
		return err
	}

	ctx, cancel := context.WithCancel(gopts.ctx)
	defer cancel()

	srcRepo, err := OpenRepository(gopts)
	if err != nil {
		return err
	}

	dstRepo, err := OpenRepository(dstGopts)
	if err != nil {
		return err
	}

	srcLock, err := lockRepo(srcRepo)
	defer unlockRepo(srcLock)
	if err != nil {
		return err
	}

	dstLock, err := lockRepo(dstRepo)
	defer unlockRepo(dstLock)
	if err != nil {
		return err
	}

	debug.Log("Loading source index")
	if err := srcRepo.LoadIndex(ctx); err != nil {
		return err
	}

	debug.Log("Loading destination index")
	if err := dstRepo.LoadIndex(ctx); err != nil {
		return err
	}

	for sn := range FindFilteredSnapshots(ctx, srcRepo, opts.Hosts, opts.Tags, opts.Paths, args) {
		Verbosef("snapshot %s of %v at %s)\n", sn.ID().Str(), sn.Paths, sn.Time)
		Verbosef("  copy started, this may take a while...\n")

		if err := copyTree(ctx, srcRepo, dstRepo, *sn.Tree); err != nil {
			return err
		}
		debug.Log("tree copied")

		if err = dstRepo.Flush(ctx); err != nil {
			return err
		}
		debug.Log("flushed packs and saved index")

		// save snapshot
		sn.Parent = nil   // Parent does not have relevance in the new repo.
		sn.Original = nil // Original does not have relevance in the new repo.
		newID, err := dstRepo.SaveJSONUnpacked(ctx, restic.SnapshotFile, sn)
		if err != nil {
			return err
		}
		Verbosef("snapshot %s saved\n", newID.Str())
	}
	return nil
}

func copyTree(ctx context.Context, srcRepo, dstRepo restic.Repository, treeID restic.ID) error {
	tree, err := srcRepo.LoadTree(ctx, treeID)
	if err != nil {
		return fmt.Errorf("LoadTree(%v) returned error %v", treeID.Str(), err)
	}
	// Do we already have this tree blob?
	if !dstRepo.Index().Has(treeID, restic.TreeBlob) {
		newTreeID, err := dstRepo.SaveTree(ctx, tree)
		if err != nil {
			return fmt.Errorf("SaveTree(%v) returned error %v", treeID.Str(), err)
		}
		// Assurance only.
		if newTreeID != treeID {
			return fmt.Errorf("SaveTree(%v) returned unexpected id %s", treeID.Str(), newTreeID.Str())
		}
	}

	// TODO: keep only one (big) buffer around.
	// TODO: parellize this stuff, likely only needed inside a tree.

	for _, entry := range tree.Nodes {
		// If it is a directory, recurse
		if entry.Type == "dir" && entry.Subtree != nil {
			if err := copyTree(ctx, srcRepo, dstRepo, *entry.Subtree); err != nil {
				return err
			}
		}
		// Copy the blobs for this file.
		for _, blobID := range entry.Content {
			// Do we already have this data blob?
			if dstRepo.Index().Has(blobID, restic.DataBlob) {
				continue
			}
			debug.Log("Copying blob %s\n", blobID.Str())
			buf, err := srcRepo.LoadBlob(ctx, restic.DataBlob, blobID, nil)
			if err != nil {
				return fmt.Errorf("LoadBlob(%v) returned error %v", blobID, err)
			}

			_, _, err = dstRepo.SaveBlob(ctx, restic.DataBlob, buf, blobID, false)
			if err != nil {
				return fmt.Errorf("SaveBlob(%v) returned error %v", blobID, err)
			}
		}
	}

	return nil
}
<commit_msg>copy: Only process each tree once<commit_after>package main

import (
	"context"
	"fmt"
	"os"

	"github.com/restic/restic/internal/debug"
	"github.com/restic/restic/internal/errors"
	"github.com/restic/restic/internal/restic"

	"github.com/spf13/cobra"
)

var cmdCopy = &cobra.Command{
	Use:   "copy [flags] [snapshotID ...]",
	Short: "Copy snapshots from one repository to another",
	Long: `
The "copy" command copies one or more snapshots from one repository to another
repository. Note that this will have to read (download) and write (upload) the
entire snapshot(s) due to the different encryption keys on the source and
destination, and that transferred files are not re-chunked, which may break
their deduplication.
`,
	RunE: func(cmd *cobra.Command, args []string) error {
		return runCopy(copyOptions, globalOptions, args)
	},
}

// CopyOptions bundles all options for the copy command.
type CopyOptions struct {
	Repo            string
	PasswordFile    string
	PasswordCommand string
	KeyHint         string
	Hosts           []string
	Tags            restic.TagLists
	Paths           []string
}

var copyOptions CopyOptions

func init() {
	cmdRoot.AddCommand(cmdCopy)

	f := cmdCopy.Flags()
	f.StringVarP(&copyOptions.Repo, "repo2", "", os.Getenv("RESTIC_REPOSITORY2"), "destination repository to copy snapshots to (default: $RESTIC_REPOSITORY2)")
	f.StringVarP(&copyOptions.PasswordFile, "password-file2", "", os.Getenv("RESTIC_PASSWORD_FILE2"), "read the destination repository password from a file (default: $RESTIC_PASSWORD_FILE2)")
	f.StringVarP(&copyOptions.KeyHint, "key-hint2", "", os.Getenv("RESTIC_KEY_HINT2"), "key ID of key to try decrypting the destination repository first (default: $RESTIC_KEY_HINT2)")
	f.StringVarP(&copyOptions.PasswordCommand, "password-command2", "", os.Getenv("RESTIC_PASSWORD_COMMAND2"), "specify a shell command to obtain a password for the destination repository (default: $RESTIC_PASSWORD_COMMAND2)")

	f.StringArrayVarP(&copyOptions.Hosts, "host", "H", nil, "only consider snapshots for this `host`, when no snapshot ID is given (can be specified multiple times)")
	f.Var(&copyOptions.Tags, "tag", "only consider snapshots which include this `taglist`, when no snapshot ID is given")
	f.StringArrayVar(&copyOptions.Paths, "path", nil, "only consider snapshots which include this (absolute) `path`, when no snapshot ID is given")
}

func runCopy(opts CopyOptions, gopts GlobalOptions, args []string) error {
	if opts.Repo == "" {
		return errors.Fatal("Please specify a destination repository location (--repo2)")
	}
	var err error
	dstGopts := gopts
	dstGopts.Repo = opts.Repo
	dstGopts.PasswordFile = opts.PasswordFile
	dstGopts.PasswordCommand = opts.PasswordCommand
	dstGopts.KeyHint = opts.KeyHint
	dstGopts.password, err = resolvePassword(dstGopts, "RESTIC_PASSWORD2")
	if err != nil {
		return err
	}
	dstGopts.password, err = ReadPassword(dstGopts, "enter password for destination repository: ")
	if err != nil {
		return err
	}

	ctx, cancel := context.WithCancel(gopts.ctx)
	defer cancel()

	srcRepo, err := OpenRepository(gopts)
	if err != nil {
		return err
	}

	dstRepo, err := OpenRepository(dstGopts)
	if err != nil {
		return err
	}

	srcLock, err := lockRepo(srcRepo)
	defer unlockRepo(srcLock)
	if err != nil {
		return err
	}

	dstLock, err := lockRepo(dstRepo)
	defer unlockRepo(dstLock)
	if err != nil {
		return err
	}

	debug.Log("Loading source index")
	if err := srcRepo.LoadIndex(ctx); err != nil {
		return err
	}

	debug.Log("Loading destination index")
	if err := dstRepo.LoadIndex(ctx); err != nil {
		return err
	}

	visitedTrees := restic.NewIDSet()

	for sn := range FindFilteredSnapshots(ctx, srcRepo, opts.Hosts, opts.Tags, opts.Paths, args) {
		Verbosef("snapshot %s of %v at %s)\n", sn.ID().Str(), sn.Paths, sn.Time)
		Verbosef("  copy started, this may take a while...\n")

		if err := copyTree(ctx, srcRepo, dstRepo, *sn.Tree, visitedTrees); err != nil {
			return err
		}
		debug.Log("tree copied")

		if err = dstRepo.Flush(ctx); err != nil {
			return err
		}
		debug.Log("flushed packs and saved index")

		// save snapshot
		sn.Parent = nil   // Parent does not have relevance in the new repo.
		sn.Original = nil // Original does not have relevance in the new repo.
		newID, err := dstRepo.SaveJSONUnpacked(ctx, restic.SnapshotFile, sn)
		if err != nil {
			return err
		}
		Verbosef("snapshot %s saved\n", newID.Str())
	}
	return nil
}

func copyTree(ctx context.Context, srcRepo, dstRepo restic.Repository, treeID restic.ID, visitedTrees restic.IDSet) error {
	// We have already processed this tree
	if visitedTrees.Has(treeID) {
		return nil
	}

	tree, err := srcRepo.LoadTree(ctx, treeID)
	if err != nil {
		return fmt.Errorf("LoadTree(%v) returned error %v", treeID.Str(), err)
	}
	visitedTrees.Insert(treeID)

	// Do we already have this tree blob?
	if !dstRepo.Index().Has(treeID, restic.TreeBlob) {
		newTreeID, err := dstRepo.SaveTree(ctx, tree)
		if err != nil {
			return fmt.Errorf("SaveTree(%v) returned error %v", treeID.Str(), err)
		}
		// Assurance only.
		if newTreeID != treeID {
			return fmt.Errorf("SaveTree(%v) returned unexpected id %s", treeID.Str(), newTreeID.Str())
		}
	}

	// TODO: keep only one (big) buffer around.
	// TODO: parellize this stuff, likely only needed inside a tree.

	for _, entry := range tree.Nodes {
		// If it is a directory, recurse
		if entry.Type == "dir" && entry.Subtree != nil {
			if err := copyTree(ctx, srcRepo, dstRepo, *entry.Subtree, visitedTrees); err != nil {
				return err
			}
		}
		// Copy the blobs for this file.
		for _, blobID := range entry.Content {
			// Do we already have this data blob?
			if dstRepo.Index().Has(blobID, restic.DataBlob) {
				continue
			}
			debug.Log("Copying blob %s\n", blobID.Str())
			buf, err := srcRepo.LoadBlob(ctx, restic.DataBlob, blobID, nil)
			if err != nil {
				return fmt.Errorf("LoadBlob(%v) returned error %v", blobID, err)
			}

			_, _, err = dstRepo.SaveBlob(ctx, restic.DataBlob, buf, blobID, false)
			if err != nil {
				return fmt.Errorf("SaveBlob(%v) returned error %v", blobID, err)
			}
		}
	}

	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"context"
	"path"
	"reflect"
	"sort"

	"github.com/restic/restic/internal/debug"
	"github.com/restic/restic/internal/errors"
	"github.com/restic/restic/internal/repository"
	"github.com/restic/restic/internal/restic"
	"github.com/spf13/cobra"
)

var cmdDiff = &cobra.Command{
	Use:   "diff [flags] snapshot-ID snapshot-ID",
	Short: "Show differences between two snapshots",
	Long: `
The "diff" command shows differences from the first to the second snapshot. The
first characters in each line display what has happened to a particular file or
directory:

* +  The item was added
* -  The item was removed
* U  The metadata (access mode, timestamps, ...) for the item was updated
* M  The file's content was modified
* T  The type was changed, e.g. a file was made a symlink

EXIT STATUS
===========

Exit status is 0 if the command was successful, and non-zero if there was any error.
`,
	DisableAutoGenTag: true,
	RunE: func(cmd *cobra.Command, args []string) error {
		return runDiff(diffOptions, globalOptions, args)
	},
}

// DiffOptions collects all options for the diff command.
type DiffOptions struct {
	ShowMetadata bool
}

var diffOptions DiffOptions

func init() {
	cmdRoot.AddCommand(cmdDiff)

	f := cmdDiff.Flags()
	f.BoolVar(&diffOptions.ShowMetadata, "metadata", false, "print changes in metadata")
}

func loadSnapshot(ctx context.Context, repo *repository.Repository, desc string) (*restic.Snapshot, error) {
	id, err := restic.FindSnapshot(repo, desc)
	if err != nil {
		return nil, err
	}

	return restic.LoadSnapshot(ctx, repo, id)
}

// Comparer collects all things needed to compare two snapshots.
type Comparer struct {
	repo restic.Repository
	opts DiffOptions
}

// DiffStat collects stats for all types of items.
type DiffStat struct {
	Files, Dirs, Others  int
	DataBlobs, TreeBlobs int
	Bytes                uint64
}

// Add adds stats information for node to s.
func (s *DiffStat) Add(node *restic.Node) {
	if node == nil {
		return
	}

	switch node.Type {
	case "file":
		s.Files++
	case "dir":
		s.Dirs++
	default:
		s.Others++
	}
}

// addBlobs adds the blobs of node to s.
func addBlobs(bs restic.BlobSet, node *restic.Node) {
	if node == nil {
		return
	}

	switch node.Type {
	case "file":
		for _, blob := range node.Content {
			h := restic.BlobHandle{
				ID:   blob,
				Type: restic.DataBlob,
			}
			bs.Insert(h)
		}
	case "dir":
		h := restic.BlobHandle{
			ID:   *node.Subtree,
			Type: restic.TreeBlob,
		}
		bs.Insert(h)
	}
}

// DiffStats collects the differences between two snapshots.
type DiffStats struct {
	ChangedFiles            int
	Added                   DiffStat
	Removed                 DiffStat
	BlobsBefore, BlobsAfter restic.BlobSet
}

// NewDiffStats creates new stats for a diff run.
func NewDiffStats() *DiffStats {
	return &DiffStats{
		BlobsBefore: restic.NewBlobSet(),
		BlobsAfter:  restic.NewBlobSet(),
	}
}

// updateBlobs updates the blob counters in the stats struct.
func updateBlobs(repo restic.Repository, blobs restic.BlobSet, stats *DiffStat) {
	for h := range blobs {
		switch h.Type {
		case restic.DataBlob:
			stats.DataBlobs++
		case restic.TreeBlob:
			stats.TreeBlobs++
		}

		size, found := repo.LookupBlobSize(h.ID, h.Type)
		if !found {
			Warnf("unable to find blob size for %v\n", h)
			continue
		}

		stats.Bytes += uint64(size)
	}
}

func (c *Comparer) printDir(ctx context.Context, mode string, stats *DiffStat, blobs restic.BlobSet, prefix string, id restic.ID) error {
	debug.Log("print %v tree %v", mode, id)
	tree, err := c.repo.LoadTree(ctx, id)
	if err != nil {
		return err
	}

	for _, node := range tree.Nodes {
		name := path.Join(prefix, node.Name)
		if node.Type == "dir" {
			name += "/"
		}
		Printf("%-5s%v\n", mode, name)
		stats.Add(node)
		addBlobs(blobs, node)

		if node.Type == "dir" {
			err := c.printDir(ctx, mode, stats, blobs, name, *node.Subtree)
			if err != nil {
				Warnf("error: %v\n", err)
			}
		}
	}

	return nil
}

func uniqueNodeNames(tree1, tree2 *restic.Tree) (tree1Nodes, tree2Nodes map[string]*restic.Node, uniqueNames []string) {
	names := make(map[string]struct{})
	tree1Nodes = make(map[string]*restic.Node)
	for _, node := range tree1.Nodes {
		tree1Nodes[node.Name] = node
		names[node.Name] = struct{}{}
	}

	tree2Nodes = make(map[string]*restic.Node)
	for _, node := range tree2.Nodes {
		tree2Nodes[node.Name] = node
		names[node.Name] = struct{}{}
	}

	uniqueNames = make([]string, 0, len(names))
	for name := range names {
		uniqueNames = append(uniqueNames, name)
	}

	sort.Strings(uniqueNames)
	return tree1Nodes, tree2Nodes, uniqueNames
}

func (c *Comparer) diffTree(ctx context.Context, stats *DiffStats, prefix string, id1, id2 restic.ID) error {
	debug.Log("diffing %v to %v", id1, id2)
	tree1, err := c.repo.LoadTree(ctx, id1)
	if err != nil {
		return err
	}

	tree2, err := c.repo.LoadTree(ctx, id2)
	if err != nil {
		return err
	}

	tree1Nodes, tree2Nodes, names := uniqueNodeNames(tree1, tree2)

	for _, name := range names {
		node1, t1 := tree1Nodes[name]
		node2, t2 := tree2Nodes[name]

		addBlobs(stats.BlobsBefore, node1)
		addBlobs(stats.BlobsAfter, node2)

		switch {
		case t1 && t2:
			name := path.Join(prefix, name)
			mod := ""

			if node1.Type != node2.Type {
				mod += "T"
			}

			if node2.Type == "dir" {
				name += "/"
			}

			if node1.Type == "file" &&
				node2.Type == "file" &&
				!reflect.DeepEqual(node1.Content, node2.Content) {
				mod += "M"
				stats.ChangedFiles++
			} else if c.opts.ShowMetadata && !node1.Equals(*node2) {
				mod += "U"
			}

			if mod != "" {
				Printf("%-5s%v\n", mod, name)
			}

			if node1.Type == "dir" && node2.Type == "dir" {
				err := c.diffTree(ctx, stats, name, *node1.Subtree, *node2.Subtree)
				if err != nil {
					Warnf("error: %v\n", err)
				}
			}
		case t1 && !t2:
			prefix := path.Join(prefix, name)
			if node1.Type == "dir" {
				prefix += "/"
			}
			Printf("%-5s%v\n", "-", prefix)
			stats.Removed.Add(node1)

			if node1.Type == "dir" {
				err := c.printDir(ctx, "-", &stats.Removed, stats.BlobsBefore, prefix, *node1.Subtree)
				if err != nil {
					Warnf("error: %v\n", err)
				}
			}
		case !t1 && t2:
			prefix := path.Join(prefix, name)
			if node2.Type == "dir" {
				prefix += "/"
			}
			Printf("%-5s%v\n", "+", prefix)
			stats.Added.Add(node2)

			if node2.Type == "dir" {
				err := c.printDir(ctx, "+", &stats.Added, stats.BlobsAfter, prefix, *node2.Subtree)
				if err != nil {
					Warnf("error: %v\n", err)
				}
			}
		}
	}

	return nil
}

func runDiff(opts DiffOptions, gopts GlobalOptions, args []string) error {
	if len(args) != 2 {
		return errors.Fatalf("specify two snapshot IDs")
	}

	ctx, cancel := context.WithCancel(gopts.ctx)
	defer cancel()

	repo, err := OpenRepository(gopts)
	if err != nil {
		return err
	}

	if err = repo.LoadIndex(ctx); err != nil {
		return err
	}

	if !gopts.NoLock {
		lock, err := lockRepo(repo)
		defer unlockRepo(lock)
		if err != nil {
			return err
		}
	}

	sn1, err := loadSnapshot(ctx, repo, args[0])
	if err != nil {
		return err
	}

	sn2, err := loadSnapshot(ctx, repo, args[1])
	if err != nil {
		return err
	}

	Verbosef("comparing snapshot %v to %v:\n\n", sn1.ID().Str(), sn2.ID().Str())

	if sn1.Tree == nil {
		return errors.Errorf("snapshot %v has nil tree", sn1.ID().Str())
	}

	if sn2.Tree == nil {
		return errors.Errorf("snapshot %v has nil tree", sn2.ID().Str())
	}

	c := &Comparer{
		repo: repo,
		opts: diffOptions,
	}

	stats := NewDiffStats()

	err = c.diffTree(ctx, stats, "/", *sn1.Tree, *sn2.Tree)
	if err != nil {
		return err
	}

	both := stats.BlobsBefore.Intersect(stats.BlobsAfter)
	updateBlobs(repo, stats.BlobsBefore.Sub(both), &stats.Removed)
	updateBlobs(repo, stats.BlobsAfter.Sub(both), &stats.Added)

	Printf("\n")
	Printf("Files:       %5d new, %5d removed, %5d changed\n", stats.Added.Files, stats.Removed.Files, stats.ChangedFiles)
	Printf("Dirs:        %5d new, %5d removed\n", stats.Added.Dirs, stats.Removed.Dirs)
	Printf("Others:      %5d new, %5d removed\n", stats.Added.Others, stats.Removed.Others)
	Printf("Data Blobs:  %5d new, %5d removed\n", stats.Added.DataBlobs, stats.Removed.DataBlobs)
	Printf("Tree Blobs:  %5d new, %5d removed\n", stats.Added.TreeBlobs, stats.Removed.TreeBlobs)
	Printf("  Added:   %-5s\n", formatBytes(uint64(stats.Added.Bytes)))
	Printf("  Removed: %-5s\n", formatBytes(uint64(stats.Removed.Bytes)))

	return nil
}
<commit_msg>diff: Optimize diff calculation for shared subtrees<commit_after>package main

import (
	"context"
	"path"
	"reflect"
	"sort"

	"github.com/restic/restic/internal/debug"
	"github.com/restic/restic/internal/errors"
	"github.com/restic/restic/internal/repository"
	"github.com/restic/restic/internal/restic"
	"github.com/spf13/cobra"
)

var cmdDiff = &cobra.Command{
	Use:   "diff [flags] snapshot-ID snapshot-ID",
	Short: "Show differences between two snapshots",
	Long: `
The "diff" command shows differences from the first to the second snapshot. The
first characters in each line display what has happened to a particular file or
directory:

* +  The item was added
* -  The item was removed
* U  The metadata (access mode, timestamps, ...) for the item was updated
* M  The file's content was modified
* T  The type was changed, e.g. a file was made a symlink

EXIT STATUS
===========

Exit status is 0 if the command was successful, and non-zero if there was any error.
`,
	DisableAutoGenTag: true,
	RunE: func(cmd *cobra.Command, args []string) error {
		return runDiff(diffOptions, globalOptions, args)
	},
}

// DiffOptions collects all options for the diff command.
type DiffOptions struct {
	ShowMetadata bool
}

var diffOptions DiffOptions

func init() {
	cmdRoot.AddCommand(cmdDiff)

	f := cmdDiff.Flags()
	f.BoolVar(&diffOptions.ShowMetadata, "metadata", false, "print changes in metadata")
}

func loadSnapshot(ctx context.Context, repo *repository.Repository, desc string) (*restic.Snapshot, error) {
	id, err := restic.FindSnapshot(repo, desc)
	if err != nil {
		return nil, err
	}

	return restic.LoadSnapshot(ctx, repo, id)
}

// Comparer collects all things needed to compare two snapshots.
type Comparer struct {
	repo restic.Repository
	opts DiffOptions
}

// DiffStat collects stats for all types of items.
type DiffStat struct {
	Files, Dirs, Others  int
	DataBlobs, TreeBlobs int
	Bytes                uint64
}

// Add adds stats information for node to s.
func (s *DiffStat) Add(node *restic.Node) {
	if node == nil {
		return
	}

	switch node.Type {
	case "file":
		s.Files++
	case "dir":
		s.Dirs++
	default:
		s.Others++
	}
}

// addBlobs adds the blobs of node to s.
func addBlobs(bs restic.BlobSet, node *restic.Node) {
	if node == nil {
		return
	}

	switch node.Type {
	case "file":
		for _, blob := range node.Content {
			h := restic.BlobHandle{
				ID:   blob,
				Type: restic.DataBlob,
			}
			bs.Insert(h)
		}
	case "dir":
		h := restic.BlobHandle{
			ID:   *node.Subtree,
			Type: restic.TreeBlob,
		}
		bs.Insert(h)
	}
}

// DiffStats collects the differences between two snapshots.
type DiffStats struct {
	ChangedFiles                         int
	Added                                DiffStat
	Removed                              DiffStat
	BlobsBefore, BlobsAfter, BlobsCommon restic.BlobSet
}

// NewDiffStats creates new stats for a diff run.
func NewDiffStats() *DiffStats {
	return &DiffStats{
		BlobsBefore: restic.NewBlobSet(),
		BlobsAfter:  restic.NewBlobSet(),
		BlobsCommon: restic.NewBlobSet(),
	}
}

// updateBlobs updates the blob counters in the stats struct.
func updateBlobs(repo restic.Repository, blobs restic.BlobSet, stats *DiffStat) {
	for h := range blobs {
		switch h.Type {
		case restic.DataBlob:
			stats.DataBlobs++
		case restic.TreeBlob:
			stats.TreeBlobs++
		}

		size, found := repo.LookupBlobSize(h.ID, h.Type)
		if !found {
			Warnf("unable to find blob size for %v\n", h)
			continue
		}

		stats.Bytes += uint64(size)
	}
}

func (c *Comparer) printDir(ctx context.Context, mode string, stats *DiffStat, blobs restic.BlobSet, prefix string, id restic.ID) error {
	debug.Log("print %v tree %v", mode, id)
	tree, err := c.repo.LoadTree(ctx, id)
	if err != nil {
		return err
	}

	for _, node := range tree.Nodes {
		name := path.Join(prefix, node.Name)
		if node.Type == "dir" {
			name += "/"
		}
		Printf("%-5s%v\n", mode, name)
		stats.Add(node)
		addBlobs(blobs, node)

		if node.Type == "dir" {
			err := c.printDir(ctx, mode, stats, blobs, name, *node.Subtree)
			if err != nil {
				Warnf("error: %v\n", err)
			}
		}
	}

	return nil
}

func (c *Comparer) collectDir(ctx context.Context, blobs restic.BlobSet, id restic.ID) error {
	debug.Log("print tree %v", id)
	tree, err := c.repo.LoadTree(ctx, id)
	if err != nil {
		return err
	}

	for _, node := range tree.Nodes {
		addBlobs(blobs, node)

		if node.Type == "dir" {
			err := c.collectDir(ctx, blobs, *node.Subtree)
			if err != nil {
				Warnf("error: %v\n", err)
			}
		}
	}

	return nil
}

func uniqueNodeNames(tree1, tree2 *restic.Tree) (tree1Nodes, tree2Nodes map[string]*restic.Node, uniqueNames []string) {
	names := make(map[string]struct{})
	tree1Nodes = make(map[string]*restic.Node)
	for _, node := range tree1.Nodes {
		tree1Nodes[node.Name] = node
		names[node.Name] = struct{}{}
	}

	tree2Nodes = make(map[string]*restic.Node)
	for _, node := range tree2.Nodes {
		tree2Nodes[node.Name] = node
		names[node.Name] = struct{}{}
	}

	uniqueNames = make([]string, 0, len(names))
	for name := range names {
		uniqueNames = append(uniqueNames, name)
	}

	sort.Strings(uniqueNames)
	return tree1Nodes, tree2Nodes, uniqueNames
}

func (c *Comparer) diffTree(ctx context.Context, stats *DiffStats, prefix string, id1, id2 restic.ID) error {
	debug.Log("diffing %v to %v", id1, id2)
	tree1, err := c.repo.LoadTree(ctx, id1)
	if err != nil {
		return err
	}

	tree2, err := c.repo.LoadTree(ctx, id2)
	if err != nil {
		return err
	}

	tree1Nodes, tree2Nodes, names := uniqueNodeNames(tree1, tree2)

	for _, name := range names {
		node1, t1 := tree1Nodes[name]
		node2, t2 := tree2Nodes[name]

		addBlobs(stats.BlobsBefore, node1)
		addBlobs(stats.BlobsAfter, node2)

		switch {
		case t1 && t2:
			name := path.Join(prefix, name)
			mod := ""

			if node1.Type != node2.Type {
				mod += "T"
			}

			if node2.Type == "dir" {
				name += "/"
			}

			if node1.Type == "file" &&
				node2.Type == "file" &&
				!reflect.DeepEqual(node1.Content, node2.Content) {
				mod += "M"
				stats.ChangedFiles++
			} else if c.opts.ShowMetadata && !node1.Equals(*node2) {
				mod += "U"
			}

			if mod != "" {
				Printf("%-5s%v\n", mod, name)
			}

			if node1.Type == "dir" && node2.Type == "dir" {
				var err error
				if (*node1.Subtree).Equal(*node2.Subtree) {
					err = c.collectDir(ctx, stats.BlobsCommon, *node1.Subtree)
				} else {
					err = c.diffTree(ctx, stats, name, *node1.Subtree, *node2.Subtree)
				}
				if err != nil {
					Warnf("error: %v\n", err)
				}
			}
		case t1 && !t2:
			prefix := path.Join(prefix, name)
			if node1.Type == "dir" {
				prefix += "/"
			}
			Printf("%-5s%v\n", "-", prefix)
			stats.Removed.Add(node1)

			if node1.Type == "dir" {
				err := c.printDir(ctx, "-", &stats.Removed, stats.BlobsBefore, prefix, *node1.Subtree)
				if err != nil {
					Warnf("error: %v\n", err)
				}
			}
		case !t1 && t2:
			prefix := path.Join(prefix, name)
			if node2.Type == "dir" {
				prefix += "/"
			}
			Printf("%-5s%v\n", "+", prefix)
			stats.Added.Add(node2)

			if node2.Type == "dir" {
				err := c.printDir(ctx, "+", &stats.Added, stats.BlobsAfter, prefix, *node2.Subtree)
				if err != nil {
					Warnf("error: %v\n", err)
				}
			}
		}
	}

	return nil
}

func runDiff(opts DiffOptions, gopts GlobalOptions, args []string) error {
	if len(args) != 2 {
		return errors.Fatalf("specify two snapshot IDs")
	}

	ctx, cancel := context.WithCancel(gopts.ctx)
	defer cancel()

	repo, err := OpenRepository(gopts)
	if err != nil {
		return err
	}

	if err = repo.LoadIndex(ctx); err != nil {
		return err
	}

	if !gopts.NoLock {
		lock, err := lockRepo(repo)
		defer unlockRepo(lock)
		if err != nil {
			return err
		}
	}

	sn1, err := loadSnapshot(ctx, repo, args[0])
	if err != nil {
		return err
	}

	sn2, err := loadSnapshot(ctx, repo, args[1])
	if err != nil {
		return err
	}

	Verbosef("comparing snapshot %v to %v:\n\n", sn1.ID().Str(), sn2.ID().Str())

	if sn1.Tree == nil {
		return errors.Errorf("snapshot %v has nil tree", sn1.ID().Str())
	}

	if sn2.Tree == nil {
		return errors.Errorf("snapshot %v has nil tree", sn2.ID().Str())
	}

	c := &Comparer{
		repo: repo,
		opts: diffOptions,
	}

	stats := NewDiffStats()

	err = c.diffTree(ctx, stats, "/", *sn1.Tree, *sn2.Tree)
	if err != nil {
		return err
	}

	both := stats.BlobsBefore.Intersect(stats.BlobsAfter)
	updateBlobs(repo, stats.BlobsBefore.Sub(both).Sub(stats.BlobsCommon), &stats.Removed)
	updateBlobs(repo, stats.BlobsAfter.Sub(both).Sub(stats.BlobsCommon), &stats.Added)

	Printf("\n")
	Printf("Files:       %5d new, %5d removed, %5d changed\n", stats.Added.Files, stats.Removed.Files, stats.ChangedFiles)
	Printf("Dirs:        %5d new, %5d removed\n", stats.Added.Dirs, stats.Removed.Dirs)
	Printf("Others:      %5d new, %5d removed\n", stats.Added.Others, stats.Removed.Others)
	Printf("Data Blobs:  %5d new, %5d removed\n", stats.Added.DataBlobs, stats.Removed.DataBlobs)
	Printf("Tree Blobs:  %5d new, %5d removed\n", stats.Added.TreeBlobs, stats.Removed.TreeBlobs)
	Printf("  Added:   %-5s\n", formatBytes(uint64(stats.Added.Bytes)))
	Printf("  Removed: %-5s\n", formatBytes(uint64(stats.Removed.Bytes)))

	return nil
}
<|endoftext|>


<commit_before>// Copyright (C) 2019 Google Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// The smoketests command runs a series of smoke tests.
package main

import (
	"context"
	"errors"
	"flag"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"os/signal"
	"path/filepath"
	"strings"

	"github.com/google/gapid/core/app"
	"github.com/google/gapid/core/log"
)

var (
	gapitArg  = flag.String("gapit", "gapit", "Path to gapit executable")
	keepArg   = flag.Bool("keep", false, "Keep the temporary directory even if no errors are found")
	tracesArg = flag.String("traces", "traces", "The directory containing traces to run smoke tests on")
)

func main() {
	app.ShortHelp = "smoketests runs a series of smoke tests"
	app.Name = "smoketests"
	app.Run(run)
}

func sigInt(ctx context.Context, c chan os.Signal) {
	s := <-c
	log.F(ctx, true, "Received signal: ", s)
}

func run(ctx context.Context) error {

	// Register SIGINT handler
	signalChan := make(chan os.Signal, 1)
	signal.Notify(signalChan, os.Interrupt)
	go sigInt(ctx, signalChan)

	// Record starting working directory
	startwd, err := os.Getwd()
	if err != nil {
		return err
	}

	// Trace directory argument
	traceFilemode, err := os.Lstat(*tracesArg)
	if err != nil {
		return err
	}
	if !traceFilemode.IsDir() {
		return errors.New("Trace path given in argument is not a directory")
	}
	traceDir := *tracesArg
	if !filepath.IsAbs(traceDir) {
		traceDir = filepath.Join(startwd, traceDir)
	}

	// Gapit path argument
	gapitPath := *gapitArg
	// We assume gapitPath is found in PATH environment unless it
	// contains a separator, in which case we treat it as a path
	hasSeparator := strings.ContainsAny(gapitPath, string(filepath.Separator))
	if hasSeparator && !filepath.IsAbs(gapitPath) {
		gapitPath = filepath.Join(startwd, gapitPath)
	}

	// Create temporary directory
	tmpdir, err := ioutil.TempDir(startwd, "smoketests.")
	if err != nil {
		return err
	}

	// For each trace, run gapit tests
	traces, err := ioutil.ReadDir(traceDir)
	atLeastOneTraceFound := false
	nbErr := 0

	for _, t := range traces {

		// Filter out non-trace files
		if t.IsDir() {
			continue
		}
		trace := t.Name()
		if !strings.HasSuffix(trace, ".gfxtrace") {
			continue
		}

		atLeastOneTraceFound = true

		// Run smoke tests on trace under temporary directory
		tracewd := filepath.Join(tmpdir, trace)
		tracepath := filepath.Join(traceDir, trace)
		if err := os.Mkdir(tracewd, 0777); err != nil {
			return err
		}
		os.Chdir(tracewd)
		if err := testTrace(ctx, &nbErr, gapitPath, tracepath); err != nil {
			return err
		}
		os.Chdir(startwd)
	}

	if !atLeastOneTraceFound {
		return errors.New("No file ending with '.gfxtrace' found in trace directory")
	}

	// Print number of errors
	errStr := "errors"
	if nbErr == 1 {
		errStr = "error"
	}
	log.I(ctx, "%d %s found", nbErr, errStr)

	// Temporary directory is kept if there are errors or if the user
	// explicitely asked to keep it
	if nbErr > 0 || *keepArg {
		log.I(ctx, "See logs in %s", tmpdir)
	} else {
		os.RemoveAll(tmpdir)
	}

	if nbErr > 0 {
		// Return a non-nil error to force a non-zero exit value
		return fmt.Errorf("Smoketests: %d %s found", nbErr, errStr)
	}
	return nil
}

func testTrace(ctx context.Context, nbErr *int, gapitPath string, tracepath string) error {

	// The trace basename is used for some commands argument
	trace := filepath.Base(tracepath)

	tests := [][]string{

		{"commands", tracepath},
		{"commands", "-groupbyapi", tracepath},
		{"commands", "-groupbydrawcall", tracepath},
		{"commands", "-groupbyframe", tracepath},
		{"commands", "-groupbythread", tracepath},
		{"commands", "-groupbyusermarkers", tracepath},
		{"commands", "-groupbysubmission", tracepath},
		{"commands", "-maxchildren", "1", tracepath},
		{"commands", "-name", "glBindFramebuffer", tracepath},
		{"commands", "-observations-ranges", tracepath},
		{"commands", "-observations-data", tracepath},
		{"commands", "-raw", tracepath},
		{"create_graph_visualization", "-format", "dot", "-out", trace + ".dot", tracepath},
		{"dump", tracepath},
		{"dump_fbo", tracepath},
		{"dump_pipeline", tracepath},
		{"dump_resources", tracepath},
		{"memory", tracepath},
		{"trim", tracepath},
		{"unpack", tracepath},
	}

	for _, test := range tests {
		if err := gapit(ctx, nbErr, gapitPath, test...); err != nil {
			return err
		}
	}

	return nil
}

func gapit(ctx context.Context, nbErr *int, gapitPath string, args ...string) error {
	// Print command description
	arglen := len(args)
	argsWithoutTrace := args[:arglen-1]
	trace := filepath.Base(args[arglen-1])
	printCmd := "gapit " + strings.Join(argsWithoutTrace, " ") + " " + trace

	// Execute, check error, print status
	cmd := exec.Command(gapitPath, args...)
	output, err := cmd.CombinedOutput()
	if err != nil {
		if _, ok := err.(*exec.ExitError); ok {
			// Here the gapit command raised an error
			fmt.Printf("FAIL %s\n", printCmd)
			fmt.Println("===============================================")
			fmt.Println(output)
			fmt.Println("===============================================")
			*nbErr++
		} else {
			// Here the error comes from somewhere else
			return err
		}
	} else {
		fmt.Printf("PASS %s\n", printCmd)
	}

	// Write output in log
	logFilename := strings.Join(argsWithoutTrace, "_") + ".log"
	if err := ioutil.WriteFile(logFilename, output, 0666); err != nil {
		return err
	}

	return nil
}
<commit_msg>Smoketest: look up Vulkan command name (#675)<commit_after>// Copyright (C) 2019 Google Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// The smoketests command runs a series of smoke tests.
package main

import (
	"context"
	"errors"
	"flag"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"os/signal"
	"path/filepath"
	"strings"

	"github.com/google/gapid/core/app"
	"github.com/google/gapid/core/log"
)

var (
	gapitArg  = flag.String("gapit", "gapit", "Path to gapit executable")
	keepArg   = flag.Bool("keep", false, "Keep the temporary directory even if no errors are found")
	tracesArg = flag.String("traces", "traces", "The directory containing traces to run smoke tests on")
)

func main() {
	app.ShortHelp = "smoketests runs a series of smoke tests"
	app.Name = "smoketests"
	app.Run(run)
}

func sigInt(ctx context.Context, c chan os.Signal) {
	s := <-c
	log.F(ctx, true, "Received signal: ", s)
}

func run(ctx context.Context) error {

	// Register SIGINT handler
	signalChan := make(chan os.Signal, 1)
	signal.Notify(signalChan, os.Interrupt)
	go sigInt(ctx, signalChan)

	// Record starting working directory
	startwd, err := os.Getwd()
	if err != nil {
		return err
	}

	// Trace directory argument
	traceFilemode, err := os.Lstat(*tracesArg)
	if err != nil {
		return err
	}
	if !traceFilemode.IsDir() {
		return errors.New("Trace path given in argument is not a directory")
	}
	traceDir := *tracesArg
	if !filepath.IsAbs(traceDir) {
		traceDir = filepath.Join(startwd, traceDir)
	}

	// Gapit path argument
	gapitPath := *gapitArg
	// We assume gapitPath is found in PATH environment unless it
	// contains a separator, in which case we treat it as a path
	hasSeparator := strings.ContainsAny(gapitPath, string(filepath.Separator))
	if hasSeparator && !filepath.IsAbs(gapitPath) {
		gapitPath = filepath.Join(startwd, gapitPath)
	}

	// Create temporary directory
	tmpdir, err := ioutil.TempDir(startwd, "smoketests.")
	if err != nil {
		return err
	}

	// For each trace, run gapit tests
	traces, err := ioutil.ReadDir(traceDir)
	atLeastOneTraceFound := false
	nbErr := 0

	for _, t := range traces {

		// Filter out non-trace files
		if t.IsDir() {
			continue
		}
		trace := t.Name()
		if !strings.HasSuffix(trace, ".gfxtrace") {
			continue
		}

		atLeastOneTraceFound = true

		// Run smoke tests on trace under temporary directory
		tracewd := filepath.Join(tmpdir, trace)
		tracepath := filepath.Join(traceDir, trace)
		if err := os.Mkdir(tracewd, 0777); err != nil {
			return err
		}
		os.Chdir(tracewd)
		if err := testTrace(ctx, &nbErr, gapitPath, tracepath); err != nil {
			return err
		}
		os.Chdir(startwd)
	}

	if !atLeastOneTraceFound {
		return errors.New("No file ending with '.gfxtrace' found in trace directory")
	}

	// Print number of errors
	errStr := "errors"
	if nbErr == 1 {
		errStr = "error"
	}
	log.I(ctx, "%d %s found", nbErr, errStr)

	// Temporary directory is kept if there are errors or if the user
	// explicitely asked to keep it
	if nbErr > 0 || *keepArg {
		log.I(ctx, "See logs in %s", tmpdir)
	} else {
		os.RemoveAll(tmpdir)
	}

	if nbErr > 0 {
		// Return a non-nil error to force a non-zero exit value
		return fmt.Errorf("Smoketests: %d %s found", nbErr, errStr)
	}
	return nil
}

func testTrace(ctx context.Context, nbErr *int, gapitPath string, tracepath string) error {

	// The trace basename is used for some commands argument
	trace := filepath.Base(tracepath)

	tests := [][]string{

		{"commands", tracepath},
		{"commands", "-groupbyapi", tracepath},
		{"commands", "-groupbydrawcall", tracepath},
		{"commands", "-groupbyframe", tracepath},
		{"commands", "-groupbythread", tracepath},
		{"commands", "-groupbyusermarkers", tracepath},
		{"commands", "-groupbysubmission", tracepath},
		{"commands", "-maxchildren", "1", tracepath},
		{"commands", "-name", "vkQueueSubmit", tracepath},
		{"commands", "-observations-ranges", tracepath},
		{"commands", "-observations-data", tracepath},
		{"commands", "-raw", tracepath},
		{"create_graph_visualization", "-format", "dot", "-out", trace + ".dot", tracepath},
		{"dump", tracepath},
		{"dump_fbo", tracepath},
		{"dump_pipeline", tracepath},
		{"dump_resources", tracepath},
		{"memory", tracepath},
		{"trim", tracepath},
		{"unpack", tracepath},
	}

	for _, test := range tests {
		if err := gapit(ctx, nbErr, gapitPath, test...); err != nil {
			return err
		}
	}

	return nil
}

func gapit(ctx context.Context, nbErr *int, gapitPath string, args ...string) error {
	// Print command description
	arglen := len(args)
	argsWithoutTrace := args[:arglen-1]
	trace := filepath.Base(args[arglen-1])
	printCmd := "gapit " + strings.Join(argsWithoutTrace, " ") + " " + trace

	// Execute, check error, print status
	cmd := exec.Command(gapitPath, args...)
	output, err := cmd.CombinedOutput()
	if err != nil {
		if _, ok := err.(*exec.ExitError); ok {
			// Here the gapit command raised an error
			fmt.Printf("FAIL %s\n", printCmd)
			fmt.Println("===============================================")
			fmt.Println(output)
			fmt.Println("===============================================")
			*nbErr++
		} else {
			// Here the error comes from somewhere else
			return err
		}
	} else {
		fmt.Printf("PASS %s\n", printCmd)
	}

	// Write output in log
	logFilename := strings.Join(argsWithoutTrace, "_") + ".log"
	if err := ioutil.WriteFile(logFilename, output, 0666); err != nil {
		return err
	}

	return nil
}
<|endoftext|>


<commit_before>package update

import (
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"net/http"
)

type Version struct {
	Major int `json:"major"`
	Minor int `json:"minor"`
	Patch int `json:"patch"`
}

const format = "v%d.%d.%d"

func Check(cur Version, url string) (rel Release, err error) {
	resp, err := http.Get(url)
	if err != nil {
		return
	}
	defer resp.Body.Close()

	dat, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return
	}

	json.Unmarshal(dat, &rel)
	if rel.Version.after(cur) {
		err = errors.New("No update available")
	}
	return
}

func ParseVersion(s string) Version {
	var ver Version
	fmt.Sscanf(format, s, &ver.Major, &ver.Minor, &ver.Patch)
	return ver
}

func (v Version) String() string {
	return fmt.Sprintf(format, v.Major, v.Minor, v.Patch)
}

func (v Version) after(o Version) bool {
	if v.Major != o.Major {
		return v.Major > o.Major
	} else if v.Minor != o.Minor {
		return v.Minor > o.Minor
	}
	return v.Patch > o.Patch
}
<commit_msg>Fix typo in ParseVersion<commit_after>package update

import (
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"net/http"
)

type Version struct {
	Major int `json:"major"`
	Minor int `json:"minor"`
	Patch int `json:"patch"`
}

const format = "v%d.%d.%d"

func Check(cur Version, url string) (rel Release, err error) {
	resp, err := http.Get(url)
	if err != nil {
		return
	}
	defer resp.Body.Close()

	dat, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return
	}

	json.Unmarshal(dat, &rel)
	if rel.Version.after(cur) {
		err = errors.New("No update available")
	}
	return
}

func ParseVersion(s string) Version {
	var ver Version
	fmt.Sscanf(s, format, &ver.Major, &ver.Minor, &ver.Patch)
	return ver
}

func (v Version) String() string {
	return fmt.Sprintf(format, v.Major, v.Minor, v.Patch)
}

func (v Version) after(o Version) bool {
	if v.Major != o.Major {
		return v.Major > o.Major
	} else if v.Minor != o.Minor {
		return v.Minor > o.Minor
	}
	return v.Patch > o.Patch
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"os"
	"os/signal"
	"regexp"
	"strings"
	"syscall"
	"time"

	"github.com/gentlemanautomaton/volmgmt/fileattr"
	"github.com/gentlemanautomaton/volmgmt/usn"
)

func usage(errmsg string) {
	fmt.Fprintf(os.Stderr, "%s\n\n", errmsg)
	flag.Usage()
	os.Exit(2)
}

func main() {
	flag.Usage = func() {
		fmt.Fprintf(os.Stderr, "usage: %s [-t type[,type...]] [-i regexp] [-e regexp] <volume>\n", os.Args[0])
		flag.PrintDefaults()
	}

	var (
		reasonStr  string
		reason     usn.Reason
		includeStr string
		include    *regexp.Regexp
		excludeStr string
		exclude    *regexp.Regexp
	)

	flag.StringVar(&reasonStr, "t", "*", "journal record types to include (comma-separated)")
	flag.StringVar(&includeStr, "i", "", "regular expression for file match (inclusion)")
	flag.StringVar(&excludeStr, "e", "", "regular expression for file match (exclusion)")
	flag.Parse()

	if flag.NArg() == 0 {
		usage("No volume specified.")
	}
	if flag.NArg() > 1 {
		usage("Only a single volume may be specified.")
	}

	path := flag.Arg(0)

	reason, err := usn.ParseReason(reasonStr)
	if err != nil {
		usage(fmt.Sprintf("%v", err))
	}

	include = compileRegex(includeStr)
	exclude = compileRegex(excludeStr)

	journal, err := usn.NewJournal(path)
	if err != nil {
		fmt.Printf("Unable to create monitor: %v\n", err)
		os.Exit(2)
	}
	defer journal.Close()

	data, err := journal.Query()
	if err != nil {
		fmt.Printf("Unable to access USN Journal: %v\n", err)
		os.Exit(2)
	}

	location, err := time.LoadLocation("Local")
	if err != nil {
		fmt.Printf("Unable to load local timezone information: %v\n", err)
		os.Exit(2)
	}

	monitor := journal.Monitor()
	defer monitor.Close()

	feed := monitor.Listen(64) // Register the feed before starting the monitor

	err = monitor.Run(data.NextUSN, time.Millisecond*100, reason)
	if err != nil {
		fmt.Printf("Unable to monitor USN Journal: %v\n", err)
		os.Exit(2)
	}

	done := make(chan struct{})
	go run(feed, location, include, exclude, done)

	ch := make(chan os.Signal)
	signal.Notify(ch, syscall.SIGINT, syscall.SIGTERM)

	select {
	case <-ch:
	case <-done:
	}
}

func run(feed <-chan usn.Record, location *time.Location, include, exclude *regexp.Regexp, done chan struct{}) {
	defer close(done)

	for record := range feed {
		if include != nil && !include.MatchString(record.FileName) {
			continue
		}

		if exclude != nil && exclude.MatchString(record.FileName) {
			continue
		}

		id := record.FileReferenceNumber.String()
		when := record.TimeStamp.In(location).Format("2006-01-02 15:04:05.000000 MST")
		attr := record.FileAttributes.Join("", fileattr.FormatCode)
		action := strings.ToUpper(record.Reason.Join("|", usn.ReasonFormatShort))

		fmt.Printf("%s  %d.%d  %-5s  %20s  \"%s\"  %s  %s\n", when, record.MajorVersion, record.MinorVersion, record.SourceInfo, id, record.FileName, attr, action)
	}
}

func compileRegex(re string) *regexp.Regexp {
	if re == "" {
		return nil
	}

	// Force case-insensitive matching
	if !strings.HasPrefix(re, "(?i)") {
		re = "(?i)" + re
	}

	c, err := regexp.Compile(re)
	if err != nil {
		usage(fmt.Sprintf("Unable to compile regular expression \"%s\": %v\n", re, err))
	}
	return c
}
<commit_msg>usnjournal: Output now includes full file paths<commit_after>package main

import (
	"flag"
	"fmt"
	"os"
	"os/signal"
	"regexp"
	"strings"
	"syscall"
	"time"

	"github.com/gentlemanautomaton/volmgmt/fileattr"
	"github.com/gentlemanautomaton/volmgmt/usn"
)

func usage(errmsg string) {
	fmt.Fprintf(os.Stderr, "%s\n\n", errmsg)
	flag.Usage()
	os.Exit(2)
}

func main() {
	flag.Usage = func() {
		fmt.Fprintf(os.Stderr, "usage: %s [-t type[,type...]] [-i regexp] [-e regexp] <volume>\n", os.Args[0])
		flag.PrintDefaults()
	}

	var (
		reasonStr  string
		reason     usn.Reason
		includeStr string
		include    *regexp.Regexp
		excludeStr string
		exclude    *regexp.Regexp
	)

	flag.StringVar(&reasonStr, "t", "*", "journal record types to include (comma-separated)")
	flag.StringVar(&includeStr, "i", "", "regular expression for file match (inclusion)")
	flag.StringVar(&excludeStr, "e", "", "regular expression for file match (exclusion)")
	flag.Parse()

	if flag.NArg() == 0 {
		usage("No volume specified.")
	}
	if flag.NArg() > 1 {
		usage("Only a single volume may be specified.")
	}

	path := flag.Arg(0)

	reason, err := usn.ParseReason(reasonStr)
	if err != nil {
		usage(fmt.Sprintf("%v", err))
	}

	include = compileRegex(includeStr)
	exclude = compileRegex(excludeStr)

	journal, err := usn.NewJournal(path)
	if err != nil {
		fmt.Printf("Unable to create monitor: %v\n", err)
		os.Exit(2)
	}
	defer journal.Close()

	data, err := journal.Query()
	if err != nil {
		fmt.Printf("Unable to access USN Journal: %v\n", err)
		os.Exit(2)
	}

	location, err := time.LoadLocation("Local")
	if err != nil {
		fmt.Printf("Unable to load local timezone information: %v\n", err)
		os.Exit(2)
	}

	monitor := journal.Monitor()
	defer monitor.Close()

	feed := monitor.Listen(64) // Register the feed before starting the monitor

	err = monitor.Run(data.NextUSN, time.Millisecond*100, reason)
	if err != nil {
		fmt.Printf("Unable to monitor USN Journal: %v\n", err)
		os.Exit(2)
	}

	done := make(chan struct{})
	go run(feed, location, include, exclude, done)

	ch := make(chan os.Signal)
	signal.Notify(ch, syscall.SIGINT, syscall.SIGTERM)

	select {
	case <-ch:
	case <-done:
	}
}

func run(feed <-chan usn.Record, location *time.Location, include, exclude *regexp.Regexp, done chan struct{}) {
	defer close(done)

	for record := range feed {
		if include != nil && !include.MatchString(record.FileName) {
			continue
		}

		if exclude != nil && exclude.MatchString(record.FileName) {
			continue
		}

		id := record.FileReferenceNumber.String()
		when := record.TimeStamp.In(location).Format("2006-01-02 15:04:05.000000 MST")
		attr := record.FileAttributes.Join("", fileattr.FormatCode)
		action := strings.ToUpper(record.Reason.Join("|", usn.ReasonFormatShort))

		fmt.Printf("%s  %d.%d  %-5s  %20s  \"%s\"  %s  %s\n", when, record.MajorVersion, record.MinorVersion, record.SourceInfo, id, record.Path, attr, action)
	}
}

func compileRegex(re string) *regexp.Regexp {
	if re == "" {
		return nil
	}

	// Force case-insensitive matching
	if !strings.HasPrefix(re, "(?i)") {
		re = "(?i)" + re
	}

	c, err := regexp.Compile(re)
	if err != nil {
		usage(fmt.Sprintf("Unable to compile regular expression \"%s\": %v\n", re, err))
	}
	return c
}
<|endoftext|>