<commit_before>package main

import "github.com/jpki/myna/libmyna"

import (
	"crypto/rsa"
	"crypto/x509"
	"encoding/json"
	"encoding/pem"
	"errors"
	"fmt"
	"github.com/howeyc/gopass"
	"github.com/ianmcmahon/encoding_ssh"
	"github.com/urfave/cli"
	"os"
	"strings"
)

func main() {
	app := cli.NewApp()
	app.Name = "myna"
	app.Usage = "マイナクライアント"
	app.Author = "HAMANO Tsukasa"
	app.Email = "hamano@osstech.co.jp"
	app.Version = libmyna.Version
	app.Flags = []cli.Flag{
		cli.BoolFlag{
			Name:  "debug, d",
			Usage: "詳細出力",
		},
	}
	app.Commands = []cli.Command{
		{
			Name:   "test",
			Usage:  "動作確認",
			Action: testCard,
		},
		{
			Name:   "card",
			Usage:  "券面事項入力補助AP",
			Action: showCardInfo,
			Before: checkCard,
			Flags: []cli.Flag{
				cli.StringFlag{
					Name:  "pin",
					Usage: "暗証番号(4桁)",
				},
				cli.StringFlag{
					Name:  "form",
					Usage: "出力形式(txt,json)",
				},
			},
		},
		{
			Name:        "cert",
			Usage:       "証明書を表示",
			Subcommands: certCommands,
		},
		{
			Name:   "pin_status",
			Usage:  "PINステータス",
			Action: showPinStatus,
			Before: checkCard,
		},
		{
			Name:  "auth_change_pin",
			Usage: "利用者認証用PINを変更",
			Flags: []cli.Flag{
				cli.StringFlag{
					Name:  "pin",
					Usage: "暗証番号(4桁)",
				},
				cli.StringFlag{
					Name:  "newpin",
					Usage: "新しい暗証番号(4桁)",
				},
			},
			Action: changeAuthPIN,
		},
		{
			Name:   "sign",
			Usage:  "署名用証明書で署名",
			Action: sign,
			Flags: []cli.Flag{
				cli.StringFlag{
					Name:  "pin",
					Usage: "署名用パスワード(6-16桁)",
				},
				cli.StringFlag{
					Name:  "in,i",
					Usage: "署名対象ファイル",
				},
				cli.StringFlag{
					Name:  "out,o",
					Usage: "署名対象ファイル",
				},
			},
		},
		{
			Name:        "tool",
			Usage:       "種々様々なツール",
			Subcommands: toolCommands,
		},
	}
	err := app.Run(os.Args)
	if err != nil {
		fmt.Fprintf(os.Stderr, "エラー: %s\n", err)
		os.Exit(1)
	}
}

var certFormFlag = cli.StringFlag{
	Name:  "form",
	Usage: "出力形式(text,pem,der,ssh)",
}

var certCommands = []cli.Command{
	{
		Name:   "auth",
		Usage:  "利用者認証用証明書を表示",
		Action: showAuthCert,
		Flags: []cli.Flag{
			certFormFlag,
		},
	},
	{
		Name:   "auth_ca",
		Usage:  "利用者認証用CA証明書を表示",
		Action: showAuthCACert,
		Flags: []cli.Flag{
			certFormFlag,
		},
	},
	{
		Name:   "sign",
		Usage:  "署名用証明書を表示",
		Action: showSignCert,
		Flags: []cli.Flag{
			certFormFlag,
			cli.StringFlag{
				Name:  "pin",
				Usage: "署名用パスワード(6-16桁)",
			},
		},
	},
	{
		Name:   "sign_ca",
		Usage:  "署名用CA証明書を表示",
		Action: showSignCACert,
		Flags: []cli.Flag{
			certFormFlag,
		},
	},
}

func checkCard(c *cli.Context) error {
	err := libmyna.CheckCard(c)
	if err != nil {
		return err
	}
	return nil
}

func testCard(c *cli.Context) error {
	err := libmyna.CheckCard(c)
	if err != nil {
		return err
	}
	fmt.Printf("マイナンバーカードです。\n")
	return nil
}

func printCert(c *cli.Context, cert *x509.Certificate) {
	form := c.String("form")
	if form == "pem" {
		var block pem.Block
		block.Type = "CERTIFICATE"
		block.Bytes = cert.Raw
		pem.Encode(os.Stdout, &block)
	} else if form == "der" {
		fmt.Println("not implement yet")
		/*
			    fp, _ := os.Create("cert.der")
				defer fp.Close()
				fp.Write(data)
		*/
	} else if form == "ssh" {
		rsaPubkey := cert.PublicKey.(*rsa.PublicKey)
		sshPubkey, _ := ssh.EncodePublicKey(*rsaPubkey, "")
		fmt.Println(sshPubkey)
	} else {
		fmt.Printf("Subject: %s\n", libmyna.Name2String(cert.Subject))
		fmt.Printf("Issuer: %s\n", libmyna.Name2String(cert.Issuer))
	}
}

func showAuthCert(c *cli.Context) error {
	cert, err := libmyna.GetCert(c, "00 0A", "")
	if err != nil {
		return err
	}
	printCert(c, cert)
	return nil
}

func showAuthCACert(c *cli.Context) error {
	cert, err := libmyna.GetCert(c, "00 0B", "")
	if err != nil {
		return err
	}
	printCert(c, cert)
	return nil
}

func changeAuthPIN(c *cli.Context) error {
	reader := libmyna.NewReader(c)
	if reader == nil {
		os.Exit(1)
	}
	defer reader.Finalize()
	reader.WaitForCard()

	pin := c.String("pin")
	if len(pin) == 0 {
		fmt.Printf("認証用PIN(4桁): ")
		input, _ := gopass.GetPasswdMasked()
		pin = string(input)
	}

	newpin := c.String("newpin")
	if len(newpin) == 0 {
		fmt.Printf("新しい認証用PIN(4桁): ")
		input, _ := gopass.GetPasswdMasked()
		newpin = string(input)
	}

	reader.SelectAP("D3 92 f0 00 26 01 00 00 00 01")
	reader.SelectEF("00 18")
	apdu := "00 20 00 80"
	reader.Tx(apdu)
	apdu = "00 20 00 80 " + fmt.Sprintf("%02X % X", len(pin), pin)
	reader.Tx(apdu)
	apdu = "00 24 01 80 " + fmt.Sprintf("%02X % X", len(newpin), newpin)
	reader.Tx(apdu)
	return nil
}

func showSignCert(c *cli.Context) error {
	pin := c.String("pin")
	if len(pin) == 0 {
		fmt.Printf("署名用パスワード(6-16桁): ")
		input, _ := gopass.GetPasswdMasked()
		pin = string(input)
	}
	pass := strings.ToUpper(pin)

	if len(pass) < 6 || 16 < len(pass) {
		return errors.New("署名用パスワード(6-16桁)を入力してください。")
	}
	cert, err := libmyna.GetCert(c, "00 01", pass)
	if err != nil {
		return err
	}
	printCert(c, cert)
	return nil
}

func showSignCACert(c *cli.Context) error {
	cert, err := libmyna.GetCert(c, "00 02", "")
	if err != nil {
		return err
	}
	printCert(c, cert)
	return nil
}

func showCardInfo(c *cli.Context) error {
	pin := []byte(c.String("pin"))
	if len(pin) == 0 {
		fmt.Printf("暗証番号(4桁): ")
		pin, _ = gopass.GetPasswdMasked()
	}
	if len(pin) != 4 {
		return errors.New("暗証番号(4桁)を入力してください。")
	}

	info, err := libmyna.GetCardInfo(c, string(pin))
	if err != nil {
		return err
	}

	if c.String("form") == "json" {
		out, _ := json.MarshalIndent(info, "", "  ")
		fmt.Printf("%s", out)
	} else {
		fmt.Printf("個人番号: %s\n", info["number"])
		fmt.Printf("謎ヘッダ: %s\n", info["header"])
		fmt.Printf("氏名:     %s\n", info["name"])
		fmt.Printf("住所:     %s\n", info["address"])
		fmt.Printf("生年月日: %s\n", info["birth"])
		fmt.Printf("性別:     %s\n", info["sex"])
	}
	return nil
}

func showPinStatus(c *cli.Context) error {
	status, err := libmyna.GetPinStatus(c)
	if err != nil {
		return err
	}

	fmt.Printf("認証用PIN: のこり%d回\n", status["auth"])
	fmt.Printf("署名用PIN: のこり%d回\n", status["sign"])
	fmt.Printf("券面入力補助PIN: のこり%d回\n", status["card"])
	fmt.Printf("謎のPIN1: のこり%d回\n", status["unknown1"])
	fmt.Printf("謎のPIN2: のこり%d回\n", status["unknown2"])

	return nil
}

func sign(c *cli.Context) error {
	pin := c.String("pin")
	if len(pin) == 0 {
		fmt.Printf("署名用パスワード(6-16桁): ")
		input, _ := gopass.GetPasswdMasked()
		pin = string(input)
	}
	pass := strings.ToUpper(pin)

	if len(pin) < 6 || 16 < len(pin) {
		return errors.New("署名用パスワード(6-16桁)を入力してください。")
	}
	in := c.String("in")
	if in == "" {
		return errors.New("署名対象ファイルを指定してください。")
	}

	out := c.String("out")
	if out == "" {
		return errors.New("出力ファイルを指定してください。")
	}

	err := libmyna.Sign(c, pass, in, out)
	if err != nil {
		return err
	}
	return nil
}
<commit_msg>print certificate info<commit_after>package main

import "github.com/jpki/myna/libmyna"

import (
	"crypto/rsa"
	"crypto/x509"
	"encoding/json"
	"encoding/pem"
	"errors"
	"fmt"
	"github.com/howeyc/gopass"
	"github.com/ianmcmahon/encoding_ssh"
	"github.com/urfave/cli"
	"os"
	"strings"
)

func main() {
	app := cli.NewApp()
	app.Name = "myna"
	app.Usage = "マイナクライアント"
	app.Author = "HAMANO Tsukasa"
	app.Email = "hamano@osstech.co.jp"
	app.Version = libmyna.Version
	app.Flags = []cli.Flag{
		cli.BoolFlag{
			Name:  "debug, d",
			Usage: "詳細出力",
		},
	}
	app.Commands = []cli.Command{
		{
			Name:   "test",
			Usage:  "動作確認",
			Action: testCard,
		},
		{
			Name:   "card",
			Usage:  "券面事項入力補助AP",
			Action: showCardInfo,
			Before: checkCard,
			Flags: []cli.Flag{
				cli.StringFlag{
					Name:  "pin",
					Usage: "暗証番号(4桁)",
				},
				cli.StringFlag{
					Name:  "form",
					Usage: "出力形式(txt,json)",
				},
			},
		},
		{
			Name:        "cert",
			Usage:       "証明書を表示",
			Subcommands: certCommands,
		},
		{
			Name:   "pin_status",
			Usage:  "PINステータス",
			Action: showPinStatus,
			Before: checkCard,
		},
		{
			Name:  "auth_change_pin",
			Usage: "利用者認証用PINを変更",
			Flags: []cli.Flag{
				cli.StringFlag{
					Name:  "pin",
					Usage: "暗証番号(4桁)",
				},
				cli.StringFlag{
					Name:  "newpin",
					Usage: "新しい暗証番号(4桁)",
				},
			},
			Action: changeAuthPIN,
		},
		{
			Name:   "sign",
			Usage:  "署名用証明書で署名",
			Action: sign,
			Flags: []cli.Flag{
				cli.StringFlag{
					Name:  "pin",
					Usage: "署名用パスワード(6-16桁)",
				},
				cli.StringFlag{
					Name:  "in,i",
					Usage: "署名対象ファイル",
				},
				cli.StringFlag{
					Name:  "out,o",
					Usage: "署名対象ファイル",
				},
			},
		},
		{
			Name:        "tool",
			Usage:       "種々様々なツール",
			Subcommands: toolCommands,
		},
	}
	err := app.Run(os.Args)
	if err != nil {
		fmt.Fprintf(os.Stderr, "エラー: %s\n", err)
		os.Exit(1)
	}
}

var certFormFlag = cli.StringFlag{
	Name:  "form",
	Usage: "出力形式(text,pem,der,ssh)",
}

var certCommands = []cli.Command{
	{
		Name:   "auth",
		Usage:  "利用者認証用証明書を表示",
		Action: showAuthCert,
		Flags: []cli.Flag{
			certFormFlag,
		},
	},
	{
		Name:   "auth_ca",
		Usage:  "利用者認証用CA証明書を表示",
		Action: showAuthCACert,
		Flags: []cli.Flag{
			certFormFlag,
		},
	},
	{
		Name:   "sign",
		Usage:  "署名用証明書を表示",
		Action: showSignCert,
		Flags: []cli.Flag{
			certFormFlag,
			cli.StringFlag{
				Name:  "pin",
				Usage: "署名用パスワード(6-16桁)",
			},
		},
	},
	{
		Name:   "sign_ca",
		Usage:  "署名用CA証明書を表示",
		Action: showSignCACert,
		Flags: []cli.Flag{
			certFormFlag,
		},
	},
}

func checkCard(c *cli.Context) error {
	err := libmyna.CheckCard(c)
	if err != nil {
		return err
	}
	return nil
}

func testCard(c *cli.Context) error {
	err := libmyna.CheckCard(c)
	if err != nil {
		return err
	}
	fmt.Printf("マイナンバーカードです。\n")
	return nil
}

func printCert(c *cli.Context, cert *x509.Certificate) {
	form := c.String("form")
	if form == "pem" {
		var block pem.Block
		block.Type = "CERTIFICATE"
		block.Bytes = cert.Raw
		pem.Encode(os.Stdout, &block)
	} else if form == "der" {
		fmt.Println("not implement yet")
		/*
			    fp, _ := os.Create("cert.der")
				defer fp.Close()
				fp.Write(data)
		*/
	} else if form == "ssh" {
		rsaPubkey := cert.PublicKey.(*rsa.PublicKey)
		sshPubkey, _ := ssh.EncodePublicKey(*rsaPubkey, "")
		fmt.Println(sshPubkey)
	} else {
		fmt.Printf("SerialNumber: %s\n", cert.SerialNumber)
		fmt.Printf("Subject: %s\n", libmyna.Name2String(cert.Subject))
		fmt.Printf("Issuer: %s\n", libmyna.Name2String(cert.Issuer))
		fmt.Printf("NotBefore: %s\n", cert.NotBefore)
		fmt.Printf("NotAfter: %s\n", cert.NotAfter)
		fmt.Printf("KeyUsage: %v\n", cert.KeyUsage)
	}
}

func showAuthCert(c *cli.Context) error {
	cert, err := libmyna.GetCert(c, "00 0A", "")
	if err != nil {
		return err
	}
	printCert(c, cert)
	return nil
}

func showAuthCACert(c *cli.Context) error {
	cert, err := libmyna.GetCert(c, "00 0B", "")
	if err != nil {
		return err
	}
	printCert(c, cert)
	return nil
}

func changeAuthPIN(c *cli.Context) error {
	reader := libmyna.NewReader(c)
	if reader == nil {
		os.Exit(1)
	}
	defer reader.Finalize()
	reader.WaitForCard()

	pin := c.String("pin")
	if len(pin) == 0 {
		fmt.Printf("認証用PIN(4桁): ")
		input, _ := gopass.GetPasswdMasked()
		pin = string(input)
	}

	newpin := c.String("newpin")
	if len(newpin) == 0 {
		fmt.Printf("新しい認証用PIN(4桁): ")
		input, _ := gopass.GetPasswdMasked()
		newpin = string(input)
	}

	reader.SelectAP("D3 92 f0 00 26 01 00 00 00 01")
	reader.SelectEF("00 18")
	apdu := "00 20 00 80"
	reader.Tx(apdu)
	apdu = "00 20 00 80 " + fmt.Sprintf("%02X % X", len(pin), pin)
	reader.Tx(apdu)
	apdu = "00 24 01 80 " + fmt.Sprintf("%02X % X", len(newpin), newpin)
	reader.Tx(apdu)
	return nil
}

func showSignCert(c *cli.Context) error {
	pin := c.String("pin")
	if len(pin) == 0 {
		fmt.Printf("署名用パスワード(6-16桁): ")
		input, _ := gopass.GetPasswdMasked()
		pin = string(input)
	}
	pass := strings.ToUpper(pin)

	if len(pass) < 6 || 16 < len(pass) {
		return errors.New("署名用パスワード(6-16桁)を入力してください。")
	}
	cert, err := libmyna.GetCert(c, "00 01", pass)
	if err != nil {
		return err
	}
	printCert(c, cert)
	return nil
}

func showSignCACert(c *cli.Context) error {
	cert, err := libmyna.GetCert(c, "00 02", "")
	if err != nil {
		return err
	}
	printCert(c, cert)
	return nil
}

func showCardInfo(c *cli.Context) error {
	pin := []byte(c.String("pin"))
	if len(pin) == 0 {
		fmt.Printf("暗証番号(4桁): ")
		pin, _ = gopass.GetPasswdMasked()
	}
	if len(pin) != 4 {
		return errors.New("暗証番号(4桁)を入力してください。")
	}

	info, err := libmyna.GetCardInfo(c, string(pin))
	if err != nil {
		return err
	}

	if c.String("form") == "json" {
		out, _ := json.MarshalIndent(info, "", "  ")
		fmt.Printf("%s", out)
	} else {
		fmt.Printf("個人番号: %s\n", info["number"])
		fmt.Printf("謎ヘッダ: %s\n", info["header"])
		fmt.Printf("氏名:     %s\n", info["name"])
		fmt.Printf("住所:     %s\n", info["address"])
		fmt.Printf("生年月日: %s\n", info["birth"])
		fmt.Printf("性別:     %s\n", info["sex"])
	}
	return nil
}

func showPinStatus(c *cli.Context) error {
	status, err := libmyna.GetPinStatus(c)
	if err != nil {
		return err
	}

	fmt.Printf("認証用PIN: のこり%d回\n", status["auth"])
	fmt.Printf("署名用PIN: のこり%d回\n", status["sign"])
	fmt.Printf("券面入力補助PIN: のこり%d回\n", status["card"])
	fmt.Printf("謎のPIN1: のこり%d回\n", status["unknown1"])
	fmt.Printf("謎のPIN2: のこり%d回\n", status["unknown2"])

	return nil
}

func sign(c *cli.Context) error {
	pin := c.String("pin")
	if len(pin) == 0 {
		fmt.Printf("署名用パスワード(6-16桁): ")
		input, _ := gopass.GetPasswdMasked()
		pin = string(input)
	}
	pass := strings.ToUpper(pin)

	if len(pin) < 6 || 16 < len(pin) {
		return errors.New("署名用パスワード(6-16桁)を入力してください。")
	}
	in := c.String("in")
	if in == "" {
		return errors.New("署名対象ファイルを指定してください。")
	}

	out := c.String("out")
	if out == "" {
		return errors.New("出力ファイルを指定してください。")
	}

	err := libmyna.Sign(c, pass, in, out)
	if err != nil {
		return err
	}
	return nil
}
<|endoftext|>


<commit_before>package restic

import (
	"encoding/json"
	"fmt"
	"os"
	"strconv"
	"syscall"
	"time"

	"github.com/juju/arrar"
	"github.com/restic/restic/backend"
)

type Node struct {
	Name       string       `json:"name"`
	Type       string       `json:"type"`
	Mode       os.FileMode  `json:"mode,omitempty"`
	ModTime    time.Time    `json:"mtime,omitempty"`
	AccessTime time.Time    `json:"atime,omitempty"`
	ChangeTime time.Time    `json:"ctime,omitempty"`
	UID        uint32       `json:"uid"`
	GID        uint32       `json:"gid"`
	User       string       `json:"user,omitempty"`
	Group      string       `json:"group,omitempty"`
	Inode      uint64       `json:"inode,omitempty"`
	Size       uint64       `json:"size,omitempty"`
	Links      uint64       `json:"links,omitempty"`
	LinkTarget string       `json:"linktarget,omitempty"`
	Device     uint64       `json:"device,omitempty"`
	Content    []backend.ID `json:"content"`
	Subtree    backend.ID   `json:"subtree,omitempty"`

	Error string `json:"error,omitempty"`

	tree *Tree

	path string
	err  error
}

func (n Node) String() string {
	switch n.Type {
	case "file":
		return fmt.Sprintf("%s %5d %5d %6d %s %s",
			n.Mode, n.UID, n.GID, n.Size, n.ModTime, n.Name)
	case "dir":
		return fmt.Sprintf("%s %5d %5d %6d %s %s",
			n.Mode|os.ModeDir, n.UID, n.GID, n.Size, n.ModTime, n.Name)
	}

	return fmt.Sprintf("<Node(%s) %s>", n.Type, n.Name)
}

func (node Node) Tree() *Tree {
	return node.tree
}

func NodeFromFileInfo(path string, fi os.FileInfo) (*Node, error) {
	node := GetNode()
	node.path = path
	node.Name = fi.Name()
	node.Mode = fi.Mode() & os.ModePerm
	node.ModTime = fi.ModTime()

	switch fi.Mode() & (os.ModeType | os.ModeCharDevice) {
	case 0:
		node.Type = "file"
	case os.ModeDir:
		node.Type = "dir"
	case os.ModeSymlink:
		node.Type = "symlink"
	case os.ModeDevice | os.ModeCharDevice:
		node.Type = "chardev"
	case os.ModeDevice:
		node.Type = "dev"
	case os.ModeNamedPipe:
		node.Type = "fifo"
	case os.ModeSocket:
		node.Type = "socket"
	}

	err := node.fill_extra(path, fi)
	return node, err
}

func CreateNodeAt(node *Node, m *Map, s Server, path string) error {
	switch node.Type {
	case "dir":
		err := os.Mkdir(path, node.Mode)
		if err != nil {
			return arrar.Annotate(err, "Mkdir")
		}

		err = os.Lchown(path, int(node.UID), int(node.GID))
		if err != nil {
			return arrar.Annotate(err, "Lchown")
		}

		var utimes = []syscall.Timespec{
			syscall.NsecToTimespec(node.AccessTime.UnixNano()),
			syscall.NsecToTimespec(node.ModTime.UnixNano()),
		}
		err = syscall.UtimesNano(path, utimes)
		if err != nil {
			return arrar.Annotate(err, "Utimesnano")
		}
	case "file":
		// TODO: handle hard links
		f, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY, 0600)
		defer f.Close()
		if err != nil {
			return arrar.Annotate(err, "OpenFile")
		}

		for _, blobid := range node.Content {
			blob, err := m.FindID(blobid)
			if err != nil {
				return arrar.Annotate(err, "Find Blob")
			}

			buf, err := s.Load(backend.Data, blob)
			if err != nil {
				return arrar.Annotate(err, "Load")
			}

			_, err = f.Write(buf)
			if err != nil {
				return arrar.Annotate(err, "Write")
			}
		}

		f.Close()

		err = os.Lchown(path, int(node.UID), int(node.GID))
		if err != nil {
			return arrar.Annotate(err, "Lchown")
		}

		var utimes = []syscall.Timespec{
			syscall.NsecToTimespec(node.AccessTime.UnixNano()),
			syscall.NsecToTimespec(node.ModTime.UnixNano()),
		}
		err = syscall.UtimesNano(path, utimes)
		if err != nil {
			return arrar.Annotate(err, "Utimesnano")
		}
	case "symlink":
		err := os.Symlink(node.LinkTarget, path)
		if err != nil {
			return arrar.Annotate(err, "Symlink")
		}

		err = os.Lchown(path, int(node.UID), int(node.GID))
		if err != nil {
			return arrar.Annotate(err, "Lchown")
		}

		// f, err := os.OpenFile(path, O_PATH|syscall.O_NOFOLLOW, 0600)
		// defer f.Close()
		// if err != nil {
		// 	return arrar.Annotate(err, "OpenFile")
		// }

		// TODO: Get Futimes() working on older Linux kernels (fails with 3.2.0)
		// var utimes = []syscall.Timeval{
		// 	syscall.NsecToTimeval(node.AccessTime.UnixNano()),
		// 	syscall.NsecToTimeval(node.ModTime.UnixNano()),
		// }
		// err = syscall.Futimes(int(f.Fd()), utimes)
		// if err != nil {
		// 	return arrar.Annotate(err, "Futimes")
		// }

		return nil
	case "dev":
		err := node.createDevAt(path)
		if err != nil {
			return arrar.Annotate(err, "Mknod")
		}
	case "chardev":
		err := node.createCharDevAt(path)
		if err != nil {
			return arrar.Annotate(err, "Mknod")
		}
	case "fifo":
		err := node.createFifoAt(path)
		if err != nil {
			return arrar.Annotate(err, "Mkfifo")
		}
	case "socket":
		// nothing to do, we do not restore sockets
		return nil
	default:
		return fmt.Errorf("filetype %q not implemented!\n", node.Type)
	}

	err := os.Chmod(path, node.Mode)
	if err != nil {
		return arrar.Annotate(err, "Chmod")
	}

	err = os.Chown(path, int(node.UID), int(node.GID))
	if err != nil {
		return arrar.Annotate(err, "Chown")
	}

	err = os.Chtimes(path, node.AccessTime, node.ModTime)
	if err != nil {
		return arrar.Annotate(err, "Chtimes")
	}

	return nil
}

func (node Node) SameContent(olderNode *Node) bool {
	// if this node has a type other than "file", treat as if content has changed
	if node.Type != "file" {
		return false
	}

	// if the name or type has changed, this is surely something different
	if node.Name != olderNode.Name || node.Type != olderNode.Type {
		return false
	}

	// if timestamps or inodes differ, content has changed
	if node.ModTime != olderNode.ModTime ||
		node.ChangeTime != olderNode.ChangeTime ||
		node.Inode != olderNode.Inode {
		return false
	}

	// otherwise the node is assumed to have the same content
	return true
}

func (node Node) MarshalJSON() ([]byte, error) {
	type nodeJSON Node
	nj := nodeJSON(node)
	name := strconv.Quote(node.Name)
	nj.Name = name[1 : len(name)-1]

	return json.Marshal(nj)
}

func (node *Node) UnmarshalJSON(data []byte) error {
	type nodeJSON Node
	var nj *nodeJSON = (*nodeJSON)(node)

	err := json.Unmarshal(data, nj)
	if err != nil {
		return err
	}

	nj.Name, err = strconv.Unquote(`"` + nj.Name + `"`)
	return err
}

func (node Node) Equals(other Node) bool {
	// TODO: add generatored code for this
	if node.Name != other.Name {
		return false
	}
	if node.Type != other.Type {
		return false
	}
	if node.Mode != other.Mode {
		return false
	}
	if node.ModTime != other.ModTime {
		return false
	}
	if node.AccessTime != other.AccessTime {
		return false
	}
	if node.ChangeTime != other.ChangeTime {
		return false
	}
	if node.UID != other.UID {
		return false
	}
	if node.GID != other.GID {
		return false
	}
	if node.User != other.User {
		return false
	}
	if node.Group != other.Group {
		return false
	}
	if node.Inode != other.Inode {
		return false
	}
	if node.Size != other.Size {
		return false
	}
	if node.Links != other.Links {
		return false
	}
	if node.LinkTarget != other.LinkTarget {
		return false
	}
	if node.Device != other.Device {
		return false
	}
	if node.Content != nil && other.Content == nil {
		return false
	} else if node.Content == nil && other.Content != nil {
		return false
	} else if node.Content != nil && other.Content != nil {
		if len(node.Content) != len(other.Content) {
			return false
		}

		for i := 0; i < len(node.Content); i++ {
			if !node.Content[i].Equal(other.Content[i]) {
				return false
			}
		}
	}

	if !node.Subtree.Equal(other.Subtree) {
		return false
	}

	if node.Error != other.Error {
		return false
	}

	return true
}
<commit_msg>Use file size from FileInfo<commit_after>package restic

import (
	"encoding/json"
	"fmt"
	"os"
	"strconv"
	"syscall"
	"time"

	"github.com/juju/arrar"
	"github.com/restic/restic/backend"
)

type Node struct {
	Name       string       `json:"name"`
	Type       string       `json:"type"`
	Mode       os.FileMode  `json:"mode,omitempty"`
	ModTime    time.Time    `json:"mtime,omitempty"`
	AccessTime time.Time    `json:"atime,omitempty"`
	ChangeTime time.Time    `json:"ctime,omitempty"`
	UID        uint32       `json:"uid"`
	GID        uint32       `json:"gid"`
	User       string       `json:"user,omitempty"`
	Group      string       `json:"group,omitempty"`
	Inode      uint64       `json:"inode,omitempty"`
	Size       uint64       `json:"size,omitempty"`
	Links      uint64       `json:"links,omitempty"`
	LinkTarget string       `json:"linktarget,omitempty"`
	Device     uint64       `json:"device,omitempty"`
	Content    []backend.ID `json:"content"`
	Subtree    backend.ID   `json:"subtree,omitempty"`

	Error string `json:"error,omitempty"`

	tree *Tree

	path string
	err  error
}

func (n Node) String() string {
	switch n.Type {
	case "file":
		return fmt.Sprintf("%s %5d %5d %6d %s %s",
			n.Mode, n.UID, n.GID, n.Size, n.ModTime, n.Name)
	case "dir":
		return fmt.Sprintf("%s %5d %5d %6d %s %s",
			n.Mode|os.ModeDir, n.UID, n.GID, n.Size, n.ModTime, n.Name)
	}

	return fmt.Sprintf("<Node(%s) %s>", n.Type, n.Name)
}

func (node Node) Tree() *Tree {
	return node.tree
}

func NodeFromFileInfo(path string, fi os.FileInfo) (*Node, error) {
	node := GetNode()
	node.path = path
	node.Name = fi.Name()
	node.Mode = fi.Mode() & os.ModePerm
	node.ModTime = fi.ModTime()

	switch fi.Mode() & (os.ModeType | os.ModeCharDevice) {
	case 0:
		node.Type = "file"
		node.Size = uint64(fi.Size())
	case os.ModeDir:
		node.Type = "dir"
	case os.ModeSymlink:
		node.Type = "symlink"
	case os.ModeDevice | os.ModeCharDevice:
		node.Type = "chardev"
	case os.ModeDevice:
		node.Type = "dev"
	case os.ModeNamedPipe:
		node.Type = "fifo"
	case os.ModeSocket:
		node.Type = "socket"
	}

	err := node.fill_extra(path, fi)
	return node, err
}

func CreateNodeAt(node *Node, m *Map, s Server, path string) error {
	switch node.Type {
	case "dir":
		err := os.Mkdir(path, node.Mode)
		if err != nil {
			return arrar.Annotate(err, "Mkdir")
		}

		err = os.Lchown(path, int(node.UID), int(node.GID))
		if err != nil {
			return arrar.Annotate(err, "Lchown")
		}

		var utimes = []syscall.Timespec{
			syscall.NsecToTimespec(node.AccessTime.UnixNano()),
			syscall.NsecToTimespec(node.ModTime.UnixNano()),
		}
		err = syscall.UtimesNano(path, utimes)
		if err != nil {
			return arrar.Annotate(err, "Utimesnano")
		}
	case "file":
		// TODO: handle hard links
		f, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY, 0600)
		defer f.Close()
		if err != nil {
			return arrar.Annotate(err, "OpenFile")
		}

		for _, blobid := range node.Content {
			blob, err := m.FindID(blobid)
			if err != nil {
				return arrar.Annotate(err, "Find Blob")
			}

			buf, err := s.Load(backend.Data, blob)
			if err != nil {
				return arrar.Annotate(err, "Load")
			}

			_, err = f.Write(buf)
			if err != nil {
				return arrar.Annotate(err, "Write")
			}
		}

		f.Close()

		err = os.Lchown(path, int(node.UID), int(node.GID))
		if err != nil {
			return arrar.Annotate(err, "Lchown")
		}

		var utimes = []syscall.Timespec{
			syscall.NsecToTimespec(node.AccessTime.UnixNano()),
			syscall.NsecToTimespec(node.ModTime.UnixNano()),
		}
		err = syscall.UtimesNano(path, utimes)
		if err != nil {
			return arrar.Annotate(err, "Utimesnano")
		}
	case "symlink":
		err := os.Symlink(node.LinkTarget, path)
		if err != nil {
			return arrar.Annotate(err, "Symlink")
		}

		err = os.Lchown(path, int(node.UID), int(node.GID))
		if err != nil {
			return arrar.Annotate(err, "Lchown")
		}

		// f, err := os.OpenFile(path, O_PATH|syscall.O_NOFOLLOW, 0600)
		// defer f.Close()
		// if err != nil {
		// 	return arrar.Annotate(err, "OpenFile")
		// }

		// TODO: Get Futimes() working on older Linux kernels (fails with 3.2.0)
		// var utimes = []syscall.Timeval{
		// 	syscall.NsecToTimeval(node.AccessTime.UnixNano()),
		// 	syscall.NsecToTimeval(node.ModTime.UnixNano()),
		// }
		// err = syscall.Futimes(int(f.Fd()), utimes)
		// if err != nil {
		// 	return arrar.Annotate(err, "Futimes")
		// }

		return nil
	case "dev":
		err := node.createDevAt(path)
		if err != nil {
			return arrar.Annotate(err, "Mknod")
		}
	case "chardev":
		err := node.createCharDevAt(path)
		if err != nil {
			return arrar.Annotate(err, "Mknod")
		}
	case "fifo":
		err := node.createFifoAt(path)
		if err != nil {
			return arrar.Annotate(err, "Mkfifo")
		}
	case "socket":
		// nothing to do, we do not restore sockets
		return nil
	default:
		return fmt.Errorf("filetype %q not implemented!\n", node.Type)
	}

	err := os.Chmod(path, node.Mode)
	if err != nil {
		return arrar.Annotate(err, "Chmod")
	}

	err = os.Chown(path, int(node.UID), int(node.GID))
	if err != nil {
		return arrar.Annotate(err, "Chown")
	}

	err = os.Chtimes(path, node.AccessTime, node.ModTime)
	if err != nil {
		return arrar.Annotate(err, "Chtimes")
	}

	return nil
}

func (node Node) SameContent(olderNode *Node) bool {
	// if this node has a type other than "file", treat as if content has changed
	if node.Type != "file" {
		return false
	}

	// if the name or type has changed, this is surely something different
	if node.Name != olderNode.Name || node.Type != olderNode.Type {
		return false
	}

	// if timestamps or inodes differ, content has changed
	if node.ModTime != olderNode.ModTime ||
		node.ChangeTime != olderNode.ChangeTime ||
		node.Inode != olderNode.Inode {
		return false
	}

	// otherwise the node is assumed to have the same content
	return true
}

func (node Node) MarshalJSON() ([]byte, error) {
	type nodeJSON Node
	nj := nodeJSON(node)
	name := strconv.Quote(node.Name)
	nj.Name = name[1 : len(name)-1]

	return json.Marshal(nj)
}

func (node *Node) UnmarshalJSON(data []byte) error {
	type nodeJSON Node
	var nj *nodeJSON = (*nodeJSON)(node)

	err := json.Unmarshal(data, nj)
	if err != nil {
		return err
	}

	nj.Name, err = strconv.Unquote(`"` + nj.Name + `"`)
	return err
}

func (node Node) Equals(other Node) bool {
	// TODO: add generatored code for this
	if node.Name != other.Name {
		return false
	}
	if node.Type != other.Type {
		return false
	}
	if node.Mode != other.Mode {
		return false
	}
	if node.ModTime != other.ModTime {
		return false
	}
	if node.AccessTime != other.AccessTime {
		return false
	}
	if node.ChangeTime != other.ChangeTime {
		return false
	}
	if node.UID != other.UID {
		return false
	}
	if node.GID != other.GID {
		return false
	}
	if node.User != other.User {
		return false
	}
	if node.Group != other.Group {
		return false
	}
	if node.Inode != other.Inode {
		return false
	}
	if node.Size != other.Size {
		return false
	}
	if node.Links != other.Links {
		return false
	}
	if node.LinkTarget != other.LinkTarget {
		return false
	}
	if node.Device != other.Device {
		return false
	}
	if node.Content != nil && other.Content == nil {
		return false
	} else if node.Content == nil && other.Content != nil {
		return false
	} else if node.Content != nil && other.Content != nil {
		if len(node.Content) != len(other.Content) {
			return false
		}

		for i := 0; i < len(node.Content); i++ {
			if !node.Content[i].Equal(other.Content[i]) {
				return false
			}
		}
	}

	if !node.Subtree.Equal(other.Subtree) {
		return false
	}

	if node.Error != other.Error {
		return false
	}

	return true
}
<|endoftext|>


<commit_before>package nvml

/*
// #cgo CPPFLAGS: -I/path/to/install
// #cgo LDFLAGS: -L/path/to/install -l nvidia-ml

#include <nvml.h>
#include <stdlib.h>
#include <stddef.h>
#include <string.h>

// Not every function can be genericized in this way because of all the custom structs,
// but there are several nvmlGet functions we want that take a nvmlDevice_t, *char, and
// a length as arguments. These are trivial to pass as function pointers along with their,
// arguments, so we might as well save some effort.
//
typedef int (*gettextProperty) (nvmlDevice_t device , char *buf, uint length);
int bridge_get_text_property(gettextProperty f,
                             nvmlDevice_t device,
                             char *buf,
                             uint length)
{
    nvmlReturn_t ret;

    ret = f(device, buf, length);

    if (ret == NVML_SUCCESS) {
        return(EXIT_SUCCESS);
    } else {
        return(EXIT_FAILURE);
    }
}

// Same as above, but for integer properties
typedef int (*getintProperty) (nvmlDevice_t device , uint *property);
int bridge_get_int_property(getintProperty f,
                             nvmlDevice_t device,
                             uint *property)
{
    nvmlReturn_t ret;

    ret = f(device, property);

    if (ret == NVML_SUCCESS) {
        return(EXIT_SUCCESS);
    } else {
        return(EXIT_FAILURE);
    }
}

*/
import "C"

import (
	"errors"
	"log"
	"unsafe"
)

type Device struct {
	nvmldevice C.nvmlDevice_t
	pcibus     string
	name       string
	uuid       string
}

// NewDevice is a contstructor function for Device structs. Given an nvmlDevice_t
// object as input, it populates some static property fields and returns a Device
func NewDevice(cdevice C.nvmlDevice_t) (*Device, error) {
	device := Device{
		nvmldevice: cdevice,
	}

	uuid, err := device.UUID()
	if err != nil {
		return nil, errors.New("Cannot retrieve UUID property")
	}
	device.uuid = uuid

	name, err := device.Name()
	if err != nil {
		return nil, errors.New("Cannot retrieve Name property")
	}
	device.name = name

	return &device, nil
}

func (gpu *Device) PowerState() (int, error) {
	var pstate C.nvmlPstates_t
	var result C.nvmlReturn_t

	result = C.nvmlDeviceGetPowerState(gpu.nvmldevice, &pstate)
	if result != C.NVML_SUCCESS {
		return -1, errors.New("GetPowerState returned error")
	}

	return int(pstate), nil
}

// Temp returns the current temperature of the card in degrees Celsius
func (gpu *Device) Temp() (uint, error) {
	var result C.nvmlReturn_t
	var ctemp C.uint

	result = C.nvmlDeviceGetTemperature(gpu.nvmldevice, C.NVML_TEMPERATURE_GPU, &ctemp)
	if result != C.NVML_SUCCESS {
		return 0, errors.New("GetPowerState returned error")
	}

	return uint(ctemp), nil
}

type cIntPropFunc struct {
	f C.getintProperty
}

var intpropfunctions = map[string]*cIntPropFunc{
	"Index":                        {C.getintProperty(C.nvmlDeviceGetIndex)},
	"MinorNumber":                  {C.getintProperty(C.nvmlDeviceGetMinorNumber)},
	"InforomConfigurationChecksum": {C.getintProperty(C.nvmlDeviceGetInforomConfigurationChecksum)},
	"MaxPcieLinkGeneration":        {C.getintProperty(C.nvmlDeviceGetMaxPcieLinkGeneration)},
	"MaxPcieLinkWidth":             {C.getintProperty(C.nvmlDeviceGetMaxPcieLinkWidth)},
	"CurrPcieLinkGeneration":       {C.getintProperty(C.nvmlDeviceGetCurrPcieLinkGeneration)},
	"CurrPcieLinkWidth":            {C.getintProperty(C.nvmlDeviceGetCurrPcieLinkWidth)},
	"PcieReplayCounter":            {C.getintProperty(C.nvmlDeviceGetPcieReplayCounter)},
	"FanSpeed":                     {C.getintProperty(C.nvmlDeviceGetFanSpeed)},
	"PowerManagementLimit":         {C.getintProperty(C.nvmlDeviceGetPowerManagementLimit)},
	"PowerManagementDefaultLimit":  {C.getintProperty(C.nvmlDeviceGetPowerManagementDefaultLimit)},
	"PowerUsage":                   {C.getintProperty(C.nvmlDeviceGetPowerUsage)},
	"EnforcedPowerLimit":           {C.getintProperty(C.nvmlDeviceGetEnforcedPowerLimit)},
	"BoardId":                      {C.getintProperty(C.nvmlDeviceGetBoardId)},
	"MultiGpuBoard":                {C.getintProperty(C.nvmlDeviceGetMultiGpuBoard)},
	"AccountingBufferSize":         {C.getintProperty(C.nvmlDeviceGetAccountingBufferSize)},
}

func (gpu *Device) intProperty(property string) (uint, error) {
	var cuintproperty C.uint

	ipf, ok := intpropfunctions[property]
	if ok == false {
		return 0, errors.New("property not found")
	}

	result := C.bridge_get_int_property(ipf.f, gpu.nvmldevice, &cuintproperty)
	if result != C.EXIT_SUCCESS {
		return 0, errors.New("getintProperty bridge returned error")
	}

	return uint(cuintproperty), nil
}

func (gpu *Device) Index() (uint, error) {
	return gpu.intProperty("Index")
}

func (gpu *Device) MinorNumber() (uint, error) {
	return gpu.intProperty("MinorNumber")
}

func (gpu *Device) InforomConfigurationChecksum() (uint, error) {
	return gpu.intProperty("InforomConfigurationChecksum")
}

func (gpu *Device) MaxPcieLinkGeneration() (uint, error) {
	return gpu.intProperty("MaxPcieLinkGeneration")
}

func (gpu *Device) MaxPcieLinkWidth() (uint, error) {
	return gpu.intProperty("MaxPcieLinkWidth")
}

func (gpu *Device) CurrPcieLinkGeneration() (uint, error) {
	return gpu.intProperty("CurrPcieLinkGeneration")
}

func (gpu *Device) CurrPcieLinkWidth() (uint, error) {
	return gpu.intProperty("CurrPcieLinkWidth")
}

func (gpu *Device) PcieReplayCounter() (uint, error) {
	return gpu.intProperty("PcieReplayCounter")
}

func (gpu *Device) FanSpeed() (uint, error) {
	return gpu.intProperty("FanSpeed")
}

func (gpu *Device) PowerManagementLimit() (uint, error) {
	return gpu.intProperty("PowerManagementLimit")
}

func (gpu *Device) PowerManagementDefaultLimit() (uint, error) {
	return gpu.intProperty("PowerManagementDefaultLimit")
}

func (gpu *Device) PowerUsage() (uint, error) {
	return gpu.intProperty("PowerUsage")
}

func (gpu *Device) EnforcedPowerLimit() (uint, error) {
	return gpu.intProperty("EnforcedPowerLimit")
}

func (gpu *Device) BoardId() (uint, error) {
	return gpu.intProperty("BoardId")
}

func (gpu *Device) MultiGpuBoard() (uint, error) {
	return gpu.intProperty("MultiGpuBoard")
}

func (gpu *Device) AccountingBufferSize() (uint, error) {
	return gpu.intProperty("AccountingBufferSize")
}

type cTextPropFunc struct {
	f      C.gettextProperty
	length C.uint
}

var textpropfunctions = map[string]*cTextPropFunc{
	"Name":                {C.gettextProperty(C.nvmlDeviceGetName), C.NVML_DEVICE_NAME_BUFFER_SIZE},
	"Serial":              {C.gettextProperty(C.nvmlDeviceGetSerial), C.NVML_DEVICE_SERIAL_BUFFER_SIZE},
	"UUID":                {C.gettextProperty(C.nvmlDeviceGetUUID), C.NVML_DEVICE_UUID_BUFFER_SIZE},
	"InforomImageVersion": {C.gettextProperty(C.nvmlDeviceGetInforomImageVersion), C.NVML_DEVICE_INFOROM_VERSION_BUFFER_SIZE},
	"VbiosVersion":        {C.gettextProperty(C.nvmlDeviceGetVbiosVersion), C.NVML_DEVICE_VBIOS_VERSION_BUFFER_SIZE},
}

// textProperty takes a propertyname as input and then runs the corresponding
// function in the textpropfunctions map, returning the result as a Go string.
//
// textProperty takes care of allocating (and freeing) the text buffers of
// proper size.
func (gpu *Device) textProperty(property string) (string, error) {
	var propvalue string

	// If there isn't a valid entry for this property in the map, there's no reason
	// to process any further
	tpf, ok := textpropfunctions[property]
	if ok == false {
		return "", errors.New("property not found")
	}

	var buf *C.char = genCStringBuffer(uint(tpf.length))
	defer C.free(unsafe.Pointer(buf))

	result := C.bridge_get_text_property(tpf.f, gpu.nvmldevice, buf, tpf.length)
	if result != C.EXIT_SUCCESS {
		return propvalue, errors.New("gettextProperty bridge returned error")
	}

	propvalue = strndup(buf, int(tpf.length))
	if len(propvalue) > 0 {
		return propvalue, nil
	} else {
		return "", errors.New("textProperty returned empty string")
	}
}

func (gpu *Device) InforomImageVersion() (string, error) {
	return gpu.textProperty("InforomImageVersion")
}

func (gpu *Device) VbiosVersion() (string, error) {
	return gpu.textProperty("VbiosVersion")
}

// Return the product name of the device, e.g. "Tesla K40m"
func (gpu *Device) Name() (string, error) {
	return gpu.textProperty("Name")
}

// Return the UUID of the device
func (gpu *Device) UUID() (string, error) {
	return gpu.textProperty("UUID")
}

// Return the serial number of the device
func (gpu *Device) Serial() (string, error) {
	return gpu.textProperty("Serial")
}

// Go correspondent of the C.nvmlMemory_t struct. Memory in bytes
type NvmlMemory struct {
	Free  uint64
	Total uint64
	Used  uint64
}

// MemoryInfo returns a NvmlMemory struct populated with the amount of memory used,
// free, and in total on the device, in bytes.
func (gpu *Device) MemoryInfo() (NvmlMemory, error) {
	var result C.nvmlReturn_t
	var cmeminfo C.nvmlMemory_t
	var meminfo NvmlMemory

	result = C.nvmlDeviceGetMemoryInfo(gpu.nvmldevice, &cmeminfo)
	if result != C.NVML_SUCCESS {
		return meminfo, errors.New("GetPowerState returned error")
	}

	meminfo.Free = uint64(cmeminfo.free)
	meminfo.Total = uint64(cmeminfo.total)
	meminfo.Used = uint64(cmeminfo.used)

	return meminfo, nil
}

// Return a proper golang error of representation of the nvmlReturn_t error
func (gpu *Device) Error(cerror C.nvmlReturn_t) error {
	var cerrorstring *C.char

	// No need to process anything further if the nvml call succeeded
	if cerror == C.NVML_SUCCESS {
		return nil
	}

	cerrorstring = C.nvmlErrorString(cerror)
	if cerrorstring == nil {
		// I'm not sure how this could happen, but it's easy to check for
		return errors.New("Error not found in nvml.h")
	}

	return errors.New(C.GoString(cerrorstring))
}

// Initialize the NVML session
func nvmlInit() error {
	var result C.nvmlReturn_t

	result = C.nvmlInit()
	if result != C.NVML_SUCCESS {
		return errors.New("nvmlInit returned error")
	}

	return nil
}

func nvmlDeviceGetCount() (int, error) {
	var count C.uint

	result := C.nvmlDeviceGetCount(&count)
	if result != C.NVML_SUCCESS {
		return -1, errors.New("nvmlDeviceGetCount failed")
	}

	return int(count), nil
}

// GetAllGPUs will return a slice of type Device for all NVML devices present on
// the host system
func GetAllGPUs() ([]Device, error) {
	var devices []Device
	cdevices, err := getAllDevices()
	if err != nil {
		return devices, err
	}

	for _, cdevice := range cdevices {
		device, err := NewDevice(cdevice)
		if err != nil {
			break
		}

		devices = append(devices, *device)
	}

	return devices, nil
}

// getAllDevices returns an array of nvmlDevice_t structs representing all GPU
// devices in the system.
func getAllDevices() ([]C.nvmlDevice_t, error) {
	var devices []C.nvmlDevice_t

	device_count, err := nvmlDeviceGetCount()
	if err != nil {
		log.Fatal(err)
	}

	for i := 0; i < device_count; i++ {
		var device C.nvmlDevice_t
		result := C.nvmlDeviceGetHandleByIndex(C.uint(i), &device)
		if result != C.NVML_SUCCESS {
			return devices, errors.New("nvmlDeviceGetHandleByIndex returns error")
		}

		devices = append(devices, device)
	}

	if len(devices) == 0 {
		return devices, errors.New("No devices found")
	}

	return devices, nil
}

// lots of the nvml functions require an allocated *char into which to place
// strings. genCStringBuffer() allocates this buffer and returns it.
//
// IMPORTANT: These buffers need to be freed! It is strongly advised to put:
//
//            defer C.free(unsafe.Pointer(buffer))
//
// immediately after the allocation of this buffer!
//
func genCStringBuffer(size uint) *C.char {
	buf := make([]byte, size)
	return C.CString(string(buf))
}

// strndup replicates the functionality of strndup from string.h in go to
// convert *C.char to string, respecting null-termination in the original.
// C.GoStringN doesn't stop at null characters.
//
// h/t: https://utcc.utoronto.ca/~cks/space/blog/programming/GoCGoStringFunctions
//
func strndup(cstring *C.char, length int) string {
	clength := C.int(C.strnlen(cstring, C.size_t(length)))
	gostring := C.GoStringN(cstring, clength)
	return gostring
}

func init() {
	err := nvmlInit()
	if err != nil {
		log.Fatal("Could not initialize NVML interface: %s\n", err)
	}
}
<commit_msg>Standard NVML include/lib location<commit_after>package nvml

/*
// #cgo CPPFLAGS: -I/path/to/install
#cgo LDFLAGS: -L/usr/src/gdk/nvml/lib/ -l nvidia-ml

#include <nvidia/gdk/nvml.h>
#include <stdlib.h>
#include <stddef.h>
#include <string.h>

// Not every function can be genericized in this way because of all the custom structs,
// but there are several nvmlGet functions we want that take a nvmlDevice_t, *char, and
// a length as arguments. These are trivial to pass as function pointers along with their,
// arguments, so we might as well save some effort.
//
typedef int (*gettextProperty) (nvmlDevice_t device , char *buf, uint length);
int bridge_get_text_property(gettextProperty f,
                             nvmlDevice_t device,
                             char *buf,
                             uint length)
{
    nvmlReturn_t ret;

    ret = f(device, buf, length);

    if (ret == NVML_SUCCESS) {
        return(EXIT_SUCCESS);
    } else {
        return(EXIT_FAILURE);
    }
}

// Same as above, but for integer properties
typedef int (*getintProperty) (nvmlDevice_t device , uint *property);
int bridge_get_int_property(getintProperty f,
                             nvmlDevice_t device,
                             uint *property)
{
    nvmlReturn_t ret;

    ret = f(device, property);

    if (ret == NVML_SUCCESS) {
        return(EXIT_SUCCESS);
    } else {
        return(EXIT_FAILURE);
    }
}

*/
import "C"

import (
	"errors"
	"log"
	"unsafe"
)

type Device struct {
	nvmldevice C.nvmlDevice_t
	pcibus     string
	name       string
	uuid       string
}

// NewDevice is a contstructor function for Device structs. Given an nvmlDevice_t
// object as input, it populates some static property fields and returns a Device
func NewDevice(cdevice C.nvmlDevice_t) (*Device, error) {
	device := Device{
		nvmldevice: cdevice,
	}

	uuid, err := device.UUID()
	if err != nil {
		return nil, errors.New("Cannot retrieve UUID property")
	}
	device.uuid = uuid

	name, err := device.Name()
	if err != nil {
		return nil, errors.New("Cannot retrieve Name property")
	}
	device.name = name

	return &device, nil
}

func (gpu *Device) PowerState() (int, error) {
	var pstate C.nvmlPstates_t
	var result C.nvmlReturn_t

	result = C.nvmlDeviceGetPowerState(gpu.nvmldevice, &pstate)
	if result != C.NVML_SUCCESS {
		return -1, errors.New("GetPowerState returned error")
	}

	return int(pstate), nil
}

// Temp returns the current temperature of the card in degrees Celsius
func (gpu *Device) Temp() (uint, error) {
	var result C.nvmlReturn_t
	var ctemp C.uint

	result = C.nvmlDeviceGetTemperature(gpu.nvmldevice, C.NVML_TEMPERATURE_GPU, &ctemp)
	if result != C.NVML_SUCCESS {
		return 0, errors.New("GetPowerState returned error")
	}

	return uint(ctemp), nil
}

type cIntPropFunc struct {
	f C.getintProperty
}

var intpropfunctions = map[string]*cIntPropFunc{
	"Index":                        {C.getintProperty(C.nvmlDeviceGetIndex)},
	"MinorNumber":                  {C.getintProperty(C.nvmlDeviceGetMinorNumber)},
	"InforomConfigurationChecksum": {C.getintProperty(C.nvmlDeviceGetInforomConfigurationChecksum)},
	"MaxPcieLinkGeneration":        {C.getintProperty(C.nvmlDeviceGetMaxPcieLinkGeneration)},
	"MaxPcieLinkWidth":             {C.getintProperty(C.nvmlDeviceGetMaxPcieLinkWidth)},
	"CurrPcieLinkGeneration":       {C.getintProperty(C.nvmlDeviceGetCurrPcieLinkGeneration)},
	"CurrPcieLinkWidth":            {C.getintProperty(C.nvmlDeviceGetCurrPcieLinkWidth)},
	"PcieReplayCounter":            {C.getintProperty(C.nvmlDeviceGetPcieReplayCounter)},
	"FanSpeed":                     {C.getintProperty(C.nvmlDeviceGetFanSpeed)},
	"PowerManagementLimit":         {C.getintProperty(C.nvmlDeviceGetPowerManagementLimit)},
	"PowerManagementDefaultLimit":  {C.getintProperty(C.nvmlDeviceGetPowerManagementDefaultLimit)},
	"PowerUsage":                   {C.getintProperty(C.nvmlDeviceGetPowerUsage)},
	"EnforcedPowerLimit":           {C.getintProperty(C.nvmlDeviceGetEnforcedPowerLimit)},
	"BoardId":                      {C.getintProperty(C.nvmlDeviceGetBoardId)},
	"MultiGpuBoard":                {C.getintProperty(C.nvmlDeviceGetMultiGpuBoard)},
	"AccountingBufferSize":         {C.getintProperty(C.nvmlDeviceGetAccountingBufferSize)},
}

func (gpu *Device) intProperty(property string) (uint, error) {
	var cuintproperty C.uint

	ipf, ok := intpropfunctions[property]
	if ok == false {
		return 0, errors.New("property not found")
	}

	result := C.bridge_get_int_property(ipf.f, gpu.nvmldevice, &cuintproperty)
	if result != C.EXIT_SUCCESS {
		return 0, errors.New("getintProperty bridge returned error")
	}

	return uint(cuintproperty), nil
}

func (gpu *Device) Index() (uint, error) {
	return gpu.intProperty("Index")
}

func (gpu *Device) MinorNumber() (uint, error) {
	return gpu.intProperty("MinorNumber")
}

func (gpu *Device) InforomConfigurationChecksum() (uint, error) {
	return gpu.intProperty("InforomConfigurationChecksum")
}

func (gpu *Device) MaxPcieLinkGeneration() (uint, error) {
	return gpu.intProperty("MaxPcieLinkGeneration")
}

func (gpu *Device) MaxPcieLinkWidth() (uint, error) {
	return gpu.intProperty("MaxPcieLinkWidth")
}

func (gpu *Device) CurrPcieLinkGeneration() (uint, error) {
	return gpu.intProperty("CurrPcieLinkGeneration")
}

func (gpu *Device) CurrPcieLinkWidth() (uint, error) {
	return gpu.intProperty("CurrPcieLinkWidth")
}

func (gpu *Device) PcieReplayCounter() (uint, error) {
	return gpu.intProperty("PcieReplayCounter")
}

func (gpu *Device) FanSpeed() (uint, error) {
	return gpu.intProperty("FanSpeed")
}

func (gpu *Device) PowerManagementLimit() (uint, error) {
	return gpu.intProperty("PowerManagementLimit")
}

func (gpu *Device) PowerManagementDefaultLimit() (uint, error) {
	return gpu.intProperty("PowerManagementDefaultLimit")
}

func (gpu *Device) PowerUsage() (uint, error) {
	return gpu.intProperty("PowerUsage")
}

func (gpu *Device) EnforcedPowerLimit() (uint, error) {
	return gpu.intProperty("EnforcedPowerLimit")
}

func (gpu *Device) BoardId() (uint, error) {
	return gpu.intProperty("BoardId")
}

func (gpu *Device) MultiGpuBoard() (uint, error) {
	return gpu.intProperty("MultiGpuBoard")
}

func (gpu *Device) AccountingBufferSize() (uint, error) {
	return gpu.intProperty("AccountingBufferSize")
}

type cTextPropFunc struct {
	f      C.gettextProperty
	length C.uint
}

var textpropfunctions = map[string]*cTextPropFunc{
	"Name":                {C.gettextProperty(C.nvmlDeviceGetName), C.NVML_DEVICE_NAME_BUFFER_SIZE},
	"Serial":              {C.gettextProperty(C.nvmlDeviceGetSerial), C.NVML_DEVICE_SERIAL_BUFFER_SIZE},
	"UUID":                {C.gettextProperty(C.nvmlDeviceGetUUID), C.NVML_DEVICE_UUID_BUFFER_SIZE},
	"InforomImageVersion": {C.gettextProperty(C.nvmlDeviceGetInforomImageVersion), C.NVML_DEVICE_INFOROM_VERSION_BUFFER_SIZE},
	"VbiosVersion":        {C.gettextProperty(C.nvmlDeviceGetVbiosVersion), C.NVML_DEVICE_VBIOS_VERSION_BUFFER_SIZE},
}

// textProperty takes a propertyname as input and then runs the corresponding
// function in the textpropfunctions map, returning the result as a Go string.
//
// textProperty takes care of allocating (and freeing) the text buffers of
// proper size.
func (gpu *Device) textProperty(property string) (string, error) {
	var propvalue string

	// If there isn't a valid entry for this property in the map, there's no reason
	// to process any further
	tpf, ok := textpropfunctions[property]
	if ok == false {
		return "", errors.New("property not found")
	}

	var buf *C.char = genCStringBuffer(uint(tpf.length))
	defer C.free(unsafe.Pointer(buf))

	result := C.bridge_get_text_property(tpf.f, gpu.nvmldevice, buf, tpf.length)
	if result != C.EXIT_SUCCESS {
		return propvalue, errors.New("gettextProperty bridge returned error")
	}

	propvalue = strndup(buf, int(tpf.length))
	if len(propvalue) > 0 {
		return propvalue, nil
	} else {
		return "", errors.New("textProperty returned empty string")
	}
}

func (gpu *Device) InforomImageVersion() (string, error) {
	return gpu.textProperty("InforomImageVersion")
}

func (gpu *Device) VbiosVersion() (string, error) {
	return gpu.textProperty("VbiosVersion")
}

// Return the product name of the device, e.g. "Tesla K40m"
func (gpu *Device) Name() (string, error) {
	return gpu.textProperty("Name")
}

// Return the UUID of the device
func (gpu *Device) UUID() (string, error) {
	return gpu.textProperty("UUID")
}

// Return the serial number of the device
func (gpu *Device) Serial() (string, error) {
	return gpu.textProperty("Serial")
}

// Go correspondent of the C.nvmlMemory_t struct. Memory in bytes
type NvmlMemory struct {
	Free  uint64
	Total uint64
	Used  uint64
}

// MemoryInfo returns a NvmlMemory struct populated with the amount of memory used,
// free, and in total on the device, in bytes.
func (gpu *Device) MemoryInfo() (NvmlMemory, error) {
	var result C.nvmlReturn_t
	var cmeminfo C.nvmlMemory_t
	var meminfo NvmlMemory

	result = C.nvmlDeviceGetMemoryInfo(gpu.nvmldevice, &cmeminfo)
	if result != C.NVML_SUCCESS {
		return meminfo, errors.New("GetPowerState returned error")
	}

	meminfo.Free = uint64(cmeminfo.free)
	meminfo.Total = uint64(cmeminfo.total)
	meminfo.Used = uint64(cmeminfo.used)

	return meminfo, nil
}

// Return a proper golang error of representation of the nvmlReturn_t error
func (gpu *Device) Error(cerror C.nvmlReturn_t) error {
	var cerrorstring *C.char

	// No need to process anything further if the nvml call succeeded
	if cerror == C.NVML_SUCCESS {
		return nil
	}

	cerrorstring = C.nvmlErrorString(cerror)
	if cerrorstring == nil {
		// I'm not sure how this could happen, but it's easy to check for
		return errors.New("Error not found in nvml.h")
	}

	return errors.New(C.GoString(cerrorstring))
}

// Initialize the NVML session
func nvmlInit() error {
	var result C.nvmlReturn_t

	result = C.nvmlInit()
	if result != C.NVML_SUCCESS {
		return errors.New("nvmlInit returned error")
	}

	return nil
}

func nvmlDeviceGetCount() (int, error) {
	var count C.uint

	result := C.nvmlDeviceGetCount(&count)
	if result != C.NVML_SUCCESS {
		return -1, errors.New("nvmlDeviceGetCount failed")
	}

	return int(count), nil
}

// GetAllGPUs will return a slice of type Device for all NVML devices present on
// the host system
func GetAllGPUs() ([]Device, error) {
	var devices []Device
	cdevices, err := getAllDevices()
	if err != nil {
		return devices, err
	}

	for _, cdevice := range cdevices {
		device, err := NewDevice(cdevice)
		if err != nil {
			break
		}

		devices = append(devices, *device)
	}

	return devices, nil
}

// getAllDevices returns an array of nvmlDevice_t structs representing all GPU
// devices in the system.
func getAllDevices() ([]C.nvmlDevice_t, error) {
	var devices []C.nvmlDevice_t

	device_count, err := nvmlDeviceGetCount()
	if err != nil {
		log.Fatal(err)
	}

	for i := 0; i < device_count; i++ {
		var device C.nvmlDevice_t
		result := C.nvmlDeviceGetHandleByIndex(C.uint(i), &device)
		if result != C.NVML_SUCCESS {
			return devices, errors.New("nvmlDeviceGetHandleByIndex returns error")
		}

		devices = append(devices, device)
	}

	if len(devices) == 0 {
		return devices, errors.New("No devices found")
	}

	return devices, nil
}

// lots of the nvml functions require an allocated *char into which to place
// strings. genCStringBuffer() allocates this buffer and returns it.
//
// IMPORTANT: These buffers need to be freed! It is strongly advised to put:
//
//            defer C.free(unsafe.Pointer(buffer))
//
// immediately after the allocation of this buffer!
//
func genCStringBuffer(size uint) *C.char {
	buf := make([]byte, size)
	return C.CString(string(buf))
}

// strndup replicates the functionality of strndup from string.h in go to
// convert *C.char to string, respecting null-termination in the original.
// C.GoStringN doesn't stop at null characters.
//
// h/t: https://utcc.utoronto.ca/~cks/space/blog/programming/GoCGoStringFunctions
//
func strndup(cstring *C.char, length int) string {
	clength := C.int(C.strnlen(cstring, C.size_t(length)))
	gostring := C.GoStringN(cstring, clength)
	return gostring
}

func init() {
	err := nvmlInit()
	if err != nil {
		log.Fatal("Could not initialize NVML interface: %s\n", err)
	}
}
<|endoftext|>


<commit_before>package main

import (
    "net"
    "bufio"
    "io"
    "encoding/json"
    "encoding/csv"
    "errors"
    "strings"
)

func MakeJsonExtractor(field string) Extractor {
    return func(s string) (net.IP, error) {
        var f interface{}
        b := []byte(s)
        if err := json.Unmarshal(b, &f); err != nil {
            return nil, err
        }
        m := f.(map[string]interface{})
        v, ok := m[field];
        if !ok {
            return nil, errors.New("missing field: " + field)
        }
        switch v.(type) {
        case string:
            ipString := v.(string)
            if ip := net.ParseIP(ipString); ip == nil {
                return nil, errors.New("invalid IP address: " + ipString)
            } else {
                return ip, nil
            }
        default:
            return nil, errors.New("invalid field: " + field)
        }
    }
}

func MakeCsvExtractor(column int) Extractor {
    return func(s string) (net.IP, error) {
        sr := strings.NewReader(s)
        cr := csv.NewReader(sr)
        records, err := cr.Read()
        if err != nil {
            return nil, err
        }
        ipString := records[column]
        ip := net.ParseIP(ipString)
        if ip == nil {
            return nil, errors.New("invalid IP address: " + ipString)
        }
        return ip, nil
    }
}

func MakeLineSplitter(extract Extractor) func(lineChan chan string, ipChan chan net.IP, rewriteChan chan string) {
    return func(lineChan chan string, ipChan chan net.IP, rewriteChan chan string) {
        for line := range lineChan {
            // Push the line to the output file rewriter
            rewriteChan <- line
            // Grab the IP from the line, send it to stdout writer
            if (config.SuccessOnly) {
                if (strings.Contains(line, "synack")) {
                    if ip, err := extract(line); err == nil {
                        ipChan <- ip
                    }
                }
            } else {
                if ip, err := extract(line); err == nil {
                    ipChan <- ip
                }
            }
        }
        close(ipChan)
        close(rewriteChan)
    }
}

func GobbleInput(inputFile io.Reader, lineChan chan string) {
    scanner := bufio.NewScanner(inputFile)
    for scanner.Scan() {
        line := scanner.Text()
        lineChan <- line
    }
    close(lineChan)
}

type Extractor func(string) (net.IP, error)
<commit_msg>Making success_only cmd line generic.<commit_after>package main

import (
    "net"
    "bufio"
    "io"
    "encoding/json"
    "encoding/csv"
    "errors"
    "strings"
)

func MakeJsonExtractor(field string) Extractor {
    return func(s string) (net.IP, error) {
        var f interface{}
        b := []byte(s)
        if err := json.Unmarshal(b, &f); err != nil {
            return nil, err
        }
        m := f.(map[string]interface{})
        v, ok := m[field];
        if !ok {
            return nil, errors.New("missing field: " + field)
        }
        switch v.(type) {
        case string:
            ipString := v.(string)
            if ip := net.ParseIP(ipString); ip == nil {
                return nil, errors.New("invalid IP address: " + ipString)
            } else {
                return ip, nil
            }
        default:
            return nil, errors.New("invalid field: " + field)
        }
    }
}

func MakeCsvExtractor(column int) Extractor {
    return func(s string) (net.IP, error) {
        sr := strings.NewReader(s)
        cr := csv.NewReader(sr)
        records, err := cr.Read()
        if err != nil {
            return nil, err
        }
        ipString := records[column]
        ip := net.ParseIP(ipString)
        if ip == nil {
            return nil, errors.New("invalid IP address: " + ipString)
        }
        return ip, nil
    }
}

func MakeLineSplitter(extract Extractor) func(lineChan chan string, ipChan chan net.IP, rewriteChan chan string) {
    return func(lineChan chan string, ipChan chan net.IP, rewriteChan chan string) {
        for line := range lineChan {
            // Push the line to the output file rewriter
            rewriteChan <- line
            // Grab the IP from the line, send it to stdout writer
            if (config.SuccessOnly) {
                splitString := strings.Split(line, ",")
                if (splitString[12] == "1") {
                    if ip, err := extract(line); err == nil {
                        ipChan <- ip
                    }
                }
            } else {
                if ip, err := extract(line); err == nil {
                    ipChan <- ip
                }
            }
        }
        close(ipChan)
        close(rewriteChan)
    }
}

func GobbleInput(inputFile io.Reader, lineChan chan string) {
    scanner := bufio.NewScanner(inputFile)
    for scanner.Scan() {
        line := scanner.Text()
        lineChan <- line
    }
    close(lineChan)
}

type Extractor func(string) (net.IP, error)
<|endoftext|>


<commit_before>// Package ipify provides a single function for retrieving your computer's
// public IP address from the ipify service: http://www.ipify.org
package pubip

import (
	"errors"
	"fmt"
	"io/ioutil"
	"net"
	"net/http"
	"strconv"
	"strings"
	"time"

	"github.com/jpillora/backoff"
)

// GetIp queries the ipify service (http://www.ipify.org) to retrieve this
// machine's public IP address.  Returns your public IP address as a string, and
// any error encountered.  By default, this function will run using exponential
// backoff -- if this function fails for any reason, the request will be retried
// up to 3 times.
//
// Usage:
//
//		package main
//
//		import (
//			"fmt"
//			"github.com/rdegges/go-ipify"
//		)
//
//		func main() {
//			ip, err := ipify.GetIp()
//			if err != nil {
//				fmt.Println("Couldn't get my IP address:", err)
//			} else {
//				fmt.Println("My IP address is:", ip)
//			}
//		}
func GetIp(dest string) (string, error) {
	b := &backoff.Backoff{
		Jitter: true,
	}
	client := &http.Client{}

	req, err := http.NewRequest("GET", dest, nil)
	if err != nil {
		return "", err
	}

	//req.Header.Add("User-Agent", USER_AGENT)

	for tries := 0; tries < MaxTries; tries++ {
		resp, err := client.Do(req)
		if err != nil {
			d := b.Duration()
			time.Sleep(d)
			continue
		}

		defer resp.Body.Close()

		out, err := ioutil.ReadAll(resp.Body)
		if err != nil {
			return "", err
		}

		if resp.StatusCode != 200 {
			return "", errors.New(dest + " status code" + strconv.Itoa(resp.StatusCode) + ", body: " + string(out))
		}

		tout := strings.TrimSpace(string(out))
		ip := net.ParseIP(tout)
		if ip == nil {
			return "", errors.New("IP address not valid: " + tout)
		}
		return ip.String(), nil
	}

	return "", errors.New("Failed to reach " + dest)
}

func isValid(rs []string) bool {
	if len(rs) < 2 {
		return false
	}
	for _, s := range rs {
		if s != rs[0] {
			return false
		}
	}
	return true
}

func validateAppend(rs []string, r string) ([]string, error) {
	if len(rs) < 2 {
		return rs, nil
	}
	for _, s := range rs {
		if r != s {
			return nil, errors.New("not identical")
		}
	}
	return append(rs, r), nil
}

func validate(rs []string) (string, error) {
	if len(rs) < 3 {
		return "", fmt.Errorf("Less than %d results from %d APIs", 3, len(APIURIs))
	}
	first := rs[0]
	for i := 1; i < len(rs); i++ {
		if first != rs[i] {
			return "", errors.New("Results are not identical")
		}
	}
	return first, nil
}

func worker(d string, r chan<- string, e chan<- error) {
	ip, err := GetIp(d)
	if err != nil {
		e <- err
		return
	}
	r <- ip
}

func Get() (string, error) {
	var results []string
	var errs []string
	var err error
	resultCh := make(chan string, len(APIURIs))
	errCh := make(chan error, len(APIURIs))

	for _, d := range APIURIs {
		go worker(d, resultCh, errCh)
	}
	for {
		select {
		case err = <-errCh:
			errs = append(errs, err.Error())
		case r := <-resultCh:
			//results = append(results, r)
			//if isValid(results) {
			//	return results[0], nil
			//}

			//results, err = validateAppend(results, r)
			//if err != nil {
			//	errs = append(errs, err.Error())
			//} else {
			//	return r, nil
			//}

			results = append(results, r)
		case <-time.After(Timeout):
			r, err := validate(results)
			if err != nil {
				return "", err
			}
			//return "", errors.New(strings.Join(errs, "\n"))
			return r, nil
		}
	}
}
<commit_msg>Update the error message<commit_after>// Package ipify provides a single function for retrieving your computer's
// public IP address from the ipify service: http://www.ipify.org
package pubip

import (
	"errors"
	"fmt"
	"io/ioutil"
	"net"
	"net/http"
	"strconv"
	"strings"
	"time"

	"github.com/jpillora/backoff"
)

// GetIp queries the ipify service (http://www.ipify.org) to retrieve this
// machine's public IP address.  Returns your public IP address as a string, and
// any error encountered.  By default, this function will run using exponential
// backoff -- if this function fails for any reason, the request will be retried
// up to 3 times.
//
// Usage:
//
//		package main
//
//		import (
//			"fmt"
//			"github.com/rdegges/go-ipify"
//		)
//
//		func main() {
//			ip, err := ipify.GetIp()
//			if err != nil {
//				fmt.Println("Couldn't get my IP address:", err)
//			} else {
//				fmt.Println("My IP address is:", ip)
//			}
//		}
func GetIp(dest string) (string, error) {
	b := &backoff.Backoff{
		Jitter: true,
	}
	client := &http.Client{}

	req, err := http.NewRequest("GET", dest, nil)
	if err != nil {
		return "", err
	}

	//req.Header.Add("User-Agent", USER_AGENT)

	for tries := 0; tries < MaxTries; tries++ {
		resp, err := client.Do(req)
		if err != nil {
			d := b.Duration()
			time.Sleep(d)
			continue
		}

		defer resp.Body.Close()

		out, err := ioutil.ReadAll(resp.Body)
		if err != nil {
			return "", err
		}

		if resp.StatusCode != 200 {
			return "", errors.New(dest + " status code " + strconv.Itoa(resp.StatusCode) + ", body: " + string(out))
		}

		tout := strings.TrimSpace(string(out))
		ip := net.ParseIP(tout)
		if ip == nil {
			return "", errors.New("IP address not valid: " + tout)
		}
		return ip.String(), nil
	}

	return "", errors.New("Failed to reach " + dest)
}

func isValid(rs []string) bool {
	if len(rs) < 2 {
		return false
	}
	for _, s := range rs {
		if s != rs[0] {
			return false
		}
	}
	return true
}

func validateAppend(rs []string, r string) ([]string, error) {
	if len(rs) < 2 {
		return rs, nil
	}
	for _, s := range rs {
		if r != s {
			return nil, errors.New("not identical")
		}
	}
	return append(rs, r), nil
}

func detailErr(err error, errs []error) error {
	errStrs := []string{err.Error()}
	for _, e := range errs {
		errStrs = append(errStrs, e.Error())
	}
	j := strings.Join(errStrs, "\n")
	return errors.New(j)
}

func validate(rs []string) (string, error) {
	if len(rs) < 3 {
		return "", fmt.Errorf("Less than %d results from %d APIs", 3, len(APIURIs))
	}
	first := rs[0]
	for i := 1; i < len(rs); i++ {
		if first != rs[i] {
			return "", fmt.Errorf("Results are not identical: %s", rs)
		}
	}
	return first, nil
}

func worker(d string, r chan<- string, e chan<- error) {
	ip, err := GetIp(d)
	if err != nil {
		e <- err
		return
	}
	r <- ip
}

func Get() (string, error) {
	var results []string
	resultCh := make(chan string, len(APIURIs))
	var errs []error
	errCh := make(chan error, len(APIURIs))

	for _, d := range APIURIs {
		go worker(d, resultCh, errCh)
	}
	for {
		select {
		case err := <-errCh:
			errs = append(errs, err)
		case r := <-resultCh:
			results = append(results, r)
		case <-time.After(Timeout):
			r, err := validate(results)
			if err != nil {
				return "", detailErr(err, errs)
			}
			return r, nil
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"time"

	"github.com/fatih/color"
)

// Issue is the internal representation of an issue
type Issue struct {
	ID       string    `json:"id"`
	Number   int       `json:"number"`
	Status   string    `json:"status"`
	Title    string    `json:"title,omitempty"`
	Body     string    `json:"body,omitempty"`
	Assignee string    `json:"assignee,omitempty"`
	Comments []Comment `json:"comments,omitempty"`
	URL      string    `json:"url,omitempty"`
	Repo     string    `json:"repo"`
	Owner    string    `json:"owner"`
}

func (i *Issue) Path() string {
	return i.Owner + "/" + i.Repo + "/" + i.ID
}

// Comments is a collection/slice of comments
type Comments []Comment

// Comment is the internal representation of an issue comment
type Comment struct {
	ID        int       `json:"id,omitempty"`
	Body      string    `json:"body,omitempty"`
	User      string    `json:"user,omitempty"`
	CreatedAt time.Time `json:"created_at,omitempty"`
	UpdatedAt time.Time `json:"updated_at,omitempty"`
	URL       string    `json:"url,omitempty"`
	HTMLURL   string    `json:"html_url,omitempty"`
	IssueURL  string    `json:"issue_url,omitempty"`
}

// Issues is a collection/slice of issues
type Issues []*Issue

type Error struct {
	Message string `json:"error"`
}

func printIssuesList(body []byte) error {
	issues := Issues{}
	err := json.Unmarshal(body, &issues)
	if err != nil {
		return err
	}

	for _, issue := range issues {
		title := color.YellowString(issue.Title)
		fmt.Printf("\n" + title + " - [" + issue.Path() + "]")
	}

	return nil
}

func printIssueDetails(body []byte) {
	issue := Issue{}
	err := json.Unmarshal(body, &issue)
	if err != nil {
		color.Red(err.Error())
		return
	}

	color.Blue("[ #" + issue.Path() + "] " + issue.Title)
	fmt.Println(issue.Body)
	fmt.Println(issue.Status)
}
<commit_msg>Fix responses org + repo<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"time"

	"github.com/fatih/color"
)

// Issue is the internal representation of an issue
type Issue struct {
	ID       string    `json:"id"`
	Number   int       `json:"number"`
	Status   string    `json:"status"`
	Title    string    `json:"title,omitempty"`
	Body     string    `json:"body,omitempty"`
	Assignee string    `json:"assignee,omitempty"`
	Comments []Comment `json:"comments,omitempty"`
	URL      string    `json:"url,omitempty"`
	Repo     string    `json:"repo"`
	Owner    string    `json:"owner"`
}

func (i *Issue) Path() string {
	return i.Owner + "/" + i.Repo + "/" + i.ID
}

// Comments is a collection/slice of comments
type Comments []Comment

// Comment is the internal representation of an issue comment
type Comment struct {
	ID        int       `json:"id,omitempty"`
	Body      string    `json:"body,omitempty"`
	User      string    `json:"user,omitempty"`
	CreatedAt time.Time `json:"created_at,omitempty"`
	UpdatedAt time.Time `json:"updated_at,omitempty"`
	URL       string    `json:"url,omitempty"`
	HTMLURL   string    `json:"html_url,omitempty"`
	IssueURL  string    `json:"issue_url,omitempty"`
}

// Issues is a collection/slice of issues
type Issues []*Issue

type Error struct {
	Message string `json:"error"`
}

func printIssuesList(body []byte) error {
	issues := Issues{}
	err := json.Unmarshal(body, &issues)
	if err != nil {
		return err
	}

	for _, issue := range issues {
		title := color.YellowString(issue.Title)
		fmt.Printf("\n" + title + " - [#" + issue.Path() + "]")
	}

	return nil
}

func printIssueDetails(body []byte) {
	issue := Issue{}
	err := json.Unmarshal(body, &issue)
	if err != nil {
		color.Red(err.Error())
		return
	}

	if issue.Title != "" {
		color.Blue("[ #" + issue.Path() + "] " + issue.Title)
		fmt.Println(issue.Body)
		fmt.Println(issue.Status)
	} else {
		err := Error{}
		json.Unmarshal(body, &err)
		color.Red("ERROR: " + err.Message)

	}
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"io/ioutil"
	"os"
	"path"

	"github.com/schollz/versionedtext"
)

type Page struct {
	Name                    string
	Text                    versionedtext.VersionedText
	RenderedPage            string
	IsLocked                bool
	PassphraseToUnlock      string
	IsEncrypted             bool
	IsPrimedForSelfDestruct bool
}

func Open(name string) (p *Page) {
	p = new(Page)
	p.Name = name
	p.Text = versionedtext.NewVersionedText("")
	p.Render()
	bJSON, err := ioutil.ReadFile(path.Join(pathToData, encodeToBase32(name)+".json"))
	if err != nil {
		return
	}
	err = json.Unmarshal(bJSON, &p)
	if err != nil {
		p = new(Page)
	}
	return p
}

func (p *Page) Update(newText string) error {
	p.Text.Update(newText)
	p.Render()
	return p.Save()
}

func (p *Page) Render() {
	if p.IsEncrypted {
		p.RenderedPage = "" + p.Text.GetCurrent() + ""
		return
	}
	p.RenderedPage = MarkdownToHtml(p.Text.GetCurrent())
}

func (p *Page) Save() error {
	bJSON, err := json.MarshalIndent(p, "", " ")
	if err != nil {
		return err
	}
	return ioutil.WriteFile(path.Join(pathToData, encodeToBase32(p.Name)+".json"), bJSON, 0755)
}

func (p *Page) Erase() error {
	return os.Remove(path.Join(pathToData, encodeToBase32(p.Name)+".json"))
}
<commit_msg>Allow [[link]] for internal links<commit_after>package main

import (
	"encoding/json"
	"io/ioutil"
	"os"
	"path"
	"regexp"
	"strings"

	"github.com/schollz/versionedtext"
)

// Page is the basic struct
type Page struct {
	Name                    string
	Text                    versionedtext.VersionedText
	RenderedPage            string
	IsLocked                bool
	PassphraseToUnlock      string
	IsEncrypted             bool
	IsPrimedForSelfDestruct bool
}

func Open(name string) (p *Page) {
	p = new(Page)
	p.Name = name
	p.Text = versionedtext.NewVersionedText("")
	p.Render()
	bJSON, err := ioutil.ReadFile(path.Join(pathToData, encodeToBase32(name)+".json"))
	if err != nil {
		return
	}
	err = json.Unmarshal(bJSON, &p)
	if err != nil {
		p = new(Page)
	}
	return p
}

func (p *Page) Update(newText string) error {
	p.Text.Update(newText)
	p.Render()
	return p.Save()
}

func (p *Page) Render() {
	if p.IsEncrypted {
		p.RenderedPage = "" + p.Text.GetCurrent() + ""
		return
	}

	// Convert [[page]] to [page](/page/view)
	r, _ := regexp.Compile("\\[\\[(.*?)\\]\\]")
	currentText := p.Text.GetCurrent()
	for _, s := range r.FindAllString(currentText, -1) {
		currentText = strings.Replace(currentText, s, "["+s[2:len(s)-2]+"](/"+s[2:len(s)-2]+"/view)", 1)
	}
	p.Text.Update(currentText)
	p.RenderedPage = MarkdownToHtml(p.Text.GetCurrent())
}

func (p *Page) Save() error {
	bJSON, err := json.MarshalIndent(p, "", " ")
	if err != nil {
		return err
	}
	return ioutil.WriteFile(path.Join(pathToData, encodeToBase32(p.Name)+".json"), bJSON, 0755)
}

func (p *Page) Erase() error {
	return os.Remove(path.Join(pathToData, encodeToBase32(p.Name)+".json"))
}
<|endoftext|>


<commit_before>package jsonq

import (
	"fmt"
	"strconv"
)

// JsonQuery is an object that enables querying of a Go map with a simple
// positional query language.
type JsonQuery struct {
	blob map[string]interface{}
}

// stringFromInterface converts an interface{} to a string and returns an error if types don't match.
func stringFromInterface(val interface{}) (string, error) {
	switch val.(type) {
	case string:
		return val.(string), nil
	}
	return "", fmt.Errorf("Expected string value for String, got \"%v\"\n", val)
}

// boolFromInterface converts an interface{} to a bool and returns an error if types don't match.
func boolFromInterface(val interface{}) (bool, error) {
	switch val.(type) {
	case bool:
		return val.(bool), nil
	}
	return false, fmt.Errorf("Expected boolean value for Bool, got \"%v\"\n", val)
}

// floatFromInterface converts an interface{} to a float64 and returns an error if types don't match.
func floatFromInterface(val interface{}) (float64, error) {
	switch val.(type) {
	case float64:
		return val.(float64), nil
	case int:
		return float64(val.(int)), nil
	case string:
		fval, err := strconv.ParseFloat(val.(string), 64)
		if err == nil {
			return fval, nil
		}
	}
	return 0.0, fmt.Errorf("Expected numeric value for Float, got \"%v\"\n", val)
}

// intFromInterface converts an interface{} to an int and returns an error if types don't match.
func intFromInterface(val interface{}) (int, error) {
	switch val.(type) {
	case float64:
		return int(val.(float64)), nil
	case string:
		ival, err := strconv.ParseFloat(val.(string), 64)
		if err == nil {
			return int(ival), nil
		}
	case int:
		return val.(int), nil
	}
	return 0, fmt.Errorf("Expected numeric value for Int, got \"%v\"\n", val)
}

// objectFromInterface converts an interface{} to a map[string]interface{} and returns an error if types don't match.
func objectFromInterface(val interface{}) (map[string]interface{}, error) {
	switch val.(type) {
	case map[string]interface{}:
		return val.(map[string]interface{}), nil
	}
	return map[string]interface{}{}, fmt.Errorf("Expected json object for Object, got \"%v\"\n", val)
}

// arrayFromInterface converts an interface{} to an []interface{} and returns an error if types don't match.
func arrayFromInterface(val interface{}) ([]interface{}, error) {
	switch val.(type) {
	case []interface{}:
		return val.([]interface{}), nil
	}
	return []interface{}{}, fmt.Errorf("Expected json array for Array, got \"%v\"\n", val)
}

// NewQuery creates a new JsonQuery obj from an interface{}.
func NewQuery(data interface{}) *JsonQuery {
	j := new(JsonQuery)
	j.blob = data.(map[string]interface{})
	return j
}

// Bool extracts a bool the JsonQuery
func (j *JsonQuery) Bool(s ...string) (bool, error) {
	val, err := rquery(j.blob, s...)
	if err != nil {
		return false, err
	}
	return boolFromInterface(val)
}

// Float extracts a float from the JsonQuery
func (j *JsonQuery) Float(s ...string) (float64, error) {
	val, err := rquery(j.blob, s...)
	if err != nil {
		return 0.0, err
	}
	return floatFromInterface(val)
}

// Int extracts an int from the JsonQuery
func (j *JsonQuery) Int(s ...string) (int, error) {
	val, err := rquery(j.blob, s...)
	if err != nil {
		return 0, err
	}
	return intFromInterface(val)
}

// String extracts a string from the JsonQuery
func (j *JsonQuery) String(s ...string) (string, error) {
	val, err := rquery(j.blob, s...)
	if err != nil {
		return "", err
	}
	return stringFromInterface(val)
}

// Object extracts a json object from the JsonQuery
func (j *JsonQuery) Object(s ...string) (map[string]interface{}, error) {
	val, err := rquery(j.blob, s...)
	if err != nil {
		return map[string]interface{}{}, err
	}
	return objectFromInterface(val)
}

// Array extracts a []interface{} from the JsonQuery
func (j *JsonQuery) Array(s ...string) ([]interface{}, error) {
	val, err := rquery(j.blob, s...)
	if err != nil {
		return []interface{}{}, err
	}
	return arrayFromInterface(val)
}

// Interface extracts an interface{} from the JsonQuery
func (j *JsonQuery) Interface(s ...string) (interface{}, error) {
	val, err := rquery(j.blob, s...)
	if err != nil {
		return nil, err
	}
	return val, nil
}

// ArrayOfStrings extracts an array of strings from some json
func (j *JsonQuery) ArrayOfStrings(s ...string) ([]string, error) {
	array, err := j.Array(s...)
	if err != nil {
		return []string{}, err
	}
	toReturn := make([]string, len(array))
	for index, val := range array {
		toReturn[index], err = stringFromInterface(val)
		if err != nil {
			return toReturn, err
		}
	}
	return toReturn, nil
}

// ArrayOfInts extracts an array of ints from some json
func (j *JsonQuery) ArrayOfInts(s ...string) ([]int, error) {
	array, err := j.Array(s...)
	if err != nil {
		return []int{}, err
	}
	toReturn := make([]int, len(array))
	for index, val := range array {
		toReturn[index], err = intFromInterface(val)
		if err != nil {
			return toReturn, err
		}
	}
	return toReturn, nil
}

// ArrayOfFloats extracts an array of float64s from some json
func (j *JsonQuery) ArrayOfFloats(s ...string) ([]float64, error) {
	array, err := j.Array(s...)
	if err != nil {
		return []float64{}, err
	}
	toReturn := make([]float64, len(array))
	for index, val := range array {
		toReturn[index], err = floatFromInterface(val)
		if err != nil {
			return toReturn, err
		}
	}
	return toReturn, nil
}

// ArrayOfBools extracts an array of bools from some json
func (j *JsonQuery) ArrayOfBools(s ...string) ([]bool, error) {
	array, err := j.Array(s...)
	if err != nil {
		return []bool{}, err
	}
	toReturn := make([]bool, len(array))
	for index, val := range array {
		toReturn[index], err = boolFromInterface(val)
		if err != nil {
			return toReturn, err
		}
	}
	return toReturn, nil
}

// ArrayOfObjects extracts an array of map[string]interface{} (objects) from some json
func (j *JsonQuery) ArrayOfObjects(s ...string) ([]map[string]interface{}, error) {
	array, err := j.Array(s...)
	if err != nil {
		return []map[string]interface{}{}, err
	}
	toReturn := make([]map[string]interface{}, len(array))
	for index, val := range array {
		toReturn[index], err = objectFromInterface(val)
		if err != nil {
			return toReturn, err
		}
	}
	return toReturn, nil
}

// ArrayOfArrays extracts an array of []interface{} (arrays) from some json
func (j *JsonQuery) ArrayOfArrays(s ...string) ([][]interface{}, error) {
	array, err := j.Array(s...)
	if err != nil {
		return [][]interface{}{}, err
	}
	toReturn := make([][]interface{}, len(array))
	for index, val := range array {
		toReturn[index], err = arrayFromInterface(val)
		if err != nil {
			return toReturn, err
		}
	}
	return toReturn, nil
}

// Matrix2D is an alias for ArrayOfArrays
func (j *JsonQuery) Matrix2D(s ...string) ([][]interface{}, error) {
	return j.ArrayOfArrays(s...)
}

// Recursively query a decoded json blob
func rquery(blob interface{}, s ...string) (interface{}, error) {
	var (
		val interface{}
		err error
	)
	val = blob
	for _, q := range s {
		val, err = query(val, q)
		if err != nil {
			return nil, err
		}
	}
	switch val.(type) {
	case nil:
		return nil, fmt.Errorf("Nil value found at %s\n", s[len(s)-1])
	}
	return val, nil
}

// query a json blob for a single field or index.  If query is a string, then
// the blob is treated as a json object (map[string]interface{}).  If query is
// an integer, the blob is treated as a json array ([]interface{}).  Any kind
// of key or index error will result in a nil return value with an error set.
func query(blob interface{}, query string) (interface{}, error) {
	index, err := strconv.Atoi(query)
	// if it's an integer, then we treat the current interface as an array
	if err == nil {
		switch blob.(type) {
		case []interface{}:
		default:
			return nil, fmt.Errorf("Array index on non-array %v\n", blob)
		}
		if len(blob.([]interface{})) > index {
			return blob.([]interface{})[index], nil
		}
		return nil, fmt.Errorf("Array index %d on array %v out of bounds\n", index, blob)
	}

	// blob is likely an object, but verify first
	switch blob.(type) {
	case map[string]interface{}:
	default:
		return nil, fmt.Errorf("Object lookup \"%s\" on non-object %v\n", query, blob)
	}

	val, ok := blob.(map[string]interface{})[query]
	if !ok {
		return nil, fmt.Errorf("Object %v does not contain field %s\n", blob, query)
	}
	return val, nil
}
<commit_msg>accounting for json.Number<commit_after>package jsonq

import (
	"encoding/json"
	"fmt"
	"strconv"
)

// JsonQuery is an object that enables querying of a Go map with a simple
// positional query language.
type JsonQuery struct {
	blob map[string]interface{}
}

// stringFromInterface converts an interface{} to a string and returns an error if types don't match.
func stringFromInterface(val interface{}) (string, error) {
	switch val.(type) {
	case string:
		return val.(string), nil
	}
	return "", fmt.Errorf("Expected string value for String, got \"%v\"\n", val)
}

// boolFromInterface converts an interface{} to a bool and returns an error if types don't match.
func boolFromInterface(val interface{}) (bool, error) {
	switch val.(type) {
	case bool:
		return val.(bool), nil
	}
	return false, fmt.Errorf("Expected boolean value for Bool, got \"%v\"\n", val)
}

// floatFromInterface converts an interface{} to a float64 and returns an error if types don't match.
func floatFromInterface(val interface{}) (float64, error) {
	switch val.(type) {
	case float64:
		return val.(float64), nil
	case int:
		return float64(val.(int)), nil
	case string:
		fval, err := strconv.ParseFloat(val.(string), 64)
		if err == nil {
			return fval, nil
		}
	case json.Number:
		return val.(json.Number).Float64()
	}
	return 0.0, fmt.Errorf("Expected numeric value for Float, got \"%v\"\n", val)
}

// intFromInterface converts an interface{} to an int and returns an error if types don't match.
func intFromInterface(val interface{}) (int, error) {
	switch val.(type) {
	case float64:
		return int(val.(float64)), nil
	case string:
		ival, err := strconv.ParseFloat(val.(string), 64)
		if err == nil {
			return int(ival), nil
		}
	case int:
		return val.(int), nil
	case json.Number:
		i, err := val.(json.Number).Int64()
		return int(i), err
	}
	return 0, fmt.Errorf("Expected numeric value for Int, got \"%v\"\n", val)
}

// objectFromInterface converts an interface{} to a map[string]interface{} and returns an error if types don't match.
func objectFromInterface(val interface{}) (map[string]interface{}, error) {
	switch val.(type) {
	case map[string]interface{}:
		return val.(map[string]interface{}), nil
	}
	return map[string]interface{}{}, fmt.Errorf("Expected json object for Object, got \"%v\"\n", val)
}

// arrayFromInterface converts an interface{} to an []interface{} and returns an error if types don't match.
func arrayFromInterface(val interface{}) ([]interface{}, error) {
	switch val.(type) {
	case []interface{}:
		return val.([]interface{}), nil
	}
	return []interface{}{}, fmt.Errorf("Expected json array for Array, got \"%v\"\n", val)
}

// NewQuery creates a new JsonQuery obj from an interface{}.
func NewQuery(data interface{}) *JsonQuery {
	j := new(JsonQuery)
	j.blob = data.(map[string]interface{})
	return j
}

// Bool extracts a bool the JsonQuery
func (j *JsonQuery) Bool(s ...string) (bool, error) {
	val, err := rquery(j.blob, s...)
	if err != nil {
		return false, err
	}
	return boolFromInterface(val)
}

// Float extracts a float from the JsonQuery
func (j *JsonQuery) Float(s ...string) (float64, error) {
	val, err := rquery(j.blob, s...)
	if err != nil {
		return 0.0, err
	}
	return floatFromInterface(val)
}

// Int extracts an int from the JsonQuery
func (j *JsonQuery) Int(s ...string) (int, error) {
	val, err := rquery(j.blob, s...)
	if err != nil {
		return 0, err
	}
	return intFromInterface(val)
}

// String extracts a string from the JsonQuery
func (j *JsonQuery) String(s ...string) (string, error) {
	val, err := rquery(j.blob, s...)
	if err != nil {
		return "", err
	}
	return stringFromInterface(val)
}

// Object extracts a json object from the JsonQuery
func (j *JsonQuery) Object(s ...string) (map[string]interface{}, error) {
	val, err := rquery(j.blob, s...)
	if err != nil {
		return map[string]interface{}{}, err
	}
	return objectFromInterface(val)
}

// Array extracts a []interface{} from the JsonQuery
func (j *JsonQuery) Array(s ...string) ([]interface{}, error) {
	val, err := rquery(j.blob, s...)
	if err != nil {
		return []interface{}{}, err
	}
	return arrayFromInterface(val)
}

// Interface extracts an interface{} from the JsonQuery
func (j *JsonQuery) Interface(s ...string) (interface{}, error) {
	val, err := rquery(j.blob, s...)
	if err != nil {
		return nil, err
	}
	return val, nil
}

// ArrayOfStrings extracts an array of strings from some json
func (j *JsonQuery) ArrayOfStrings(s ...string) ([]string, error) {
	array, err := j.Array(s...)
	if err != nil {
		return []string{}, err
	}
	toReturn := make([]string, len(array))
	for index, val := range array {
		toReturn[index], err = stringFromInterface(val)
		if err != nil {
			return toReturn, err
		}
	}
	return toReturn, nil
}

// ArrayOfInts extracts an array of ints from some json
func (j *JsonQuery) ArrayOfInts(s ...string) ([]int, error) {
	array, err := j.Array(s...)
	if err != nil {
		return []int{}, err
	}
	toReturn := make([]int, len(array))
	for index, val := range array {
		toReturn[index], err = intFromInterface(val)
		if err != nil {
			return toReturn, err
		}
	}
	return toReturn, nil
}

// ArrayOfFloats extracts an array of float64s from some json
func (j *JsonQuery) ArrayOfFloats(s ...string) ([]float64, error) {
	array, err := j.Array(s...)
	if err != nil {
		return []float64{}, err
	}
	toReturn := make([]float64, len(array))
	for index, val := range array {
		toReturn[index], err = floatFromInterface(val)
		if err != nil {
			return toReturn, err
		}
	}
	return toReturn, nil
}

// ArrayOfBools extracts an array of bools from some json
func (j *JsonQuery) ArrayOfBools(s ...string) ([]bool, error) {
	array, err := j.Array(s...)
	if err != nil {
		return []bool{}, err
	}
	toReturn := make([]bool, len(array))
	for index, val := range array {
		toReturn[index], err = boolFromInterface(val)
		if err != nil {
			return toReturn, err
		}
	}
	return toReturn, nil
}

// ArrayOfObjects extracts an array of map[string]interface{} (objects) from some json
func (j *JsonQuery) ArrayOfObjects(s ...string) ([]map[string]interface{}, error) {
	array, err := j.Array(s...)
	if err != nil {
		return []map[string]interface{}{}, err
	}
	toReturn := make([]map[string]interface{}, len(array))
	for index, val := range array {
		toReturn[index], err = objectFromInterface(val)
		if err != nil {
			return toReturn, err
		}
	}
	return toReturn, nil
}

// ArrayOfArrays extracts an array of []interface{} (arrays) from some json
func (j *JsonQuery) ArrayOfArrays(s ...string) ([][]interface{}, error) {
	array, err := j.Array(s...)
	if err != nil {
		return [][]interface{}{}, err
	}
	toReturn := make([][]interface{}, len(array))
	for index, val := range array {
		toReturn[index], err = arrayFromInterface(val)
		if err != nil {
			return toReturn, err
		}
	}
	return toReturn, nil
}

// Matrix2D is an alias for ArrayOfArrays
func (j *JsonQuery) Matrix2D(s ...string) ([][]interface{}, error) {
	return j.ArrayOfArrays(s...)
}

// Recursively query a decoded json blob
func rquery(blob interface{}, s ...string) (interface{}, error) {
	var (
		val interface{}
		err error
	)
	val = blob
	for _, q := range s {
		val, err = query(val, q)
		if err != nil {
			return nil, err
		}
	}
	switch val.(type) {
	case nil:
		return nil, fmt.Errorf("Nil value found at %s\n", s[len(s)-1])
	}
	return val, nil
}

// query a json blob for a single field or index.  If query is a string, then
// the blob is treated as a json object (map[string]interface{}).  If query is
// an integer, the blob is treated as a json array ([]interface{}).  Any kind
// of key or index error will result in a nil return value with an error set.
func query(blob interface{}, query string) (interface{}, error) {
	index, err := strconv.Atoi(query)
	// if it's an integer, then we treat the current interface as an array
	if err == nil {
		switch blob.(type) {
		case []interface{}:
		default:
			return nil, fmt.Errorf("Array index on non-array %v\n", blob)
		}
		if len(blob.([]interface{})) > index {
			return blob.([]interface{})[index], nil
		}
		return nil, fmt.Errorf("Array index %d on array %v out of bounds\n", index, blob)
	}

	// blob is likely an object, but verify first
	switch blob.(type) {
	case map[string]interface{}:
	default:
		return nil, fmt.Errorf("Object lookup \"%s\" on non-object %v\n", query, blob)
	}

	val, ok := blob.(map[string]interface{})[query]
	if !ok {
		return nil, fmt.Errorf("Object %v does not contain field %s\n", blob, query)
	}
	return val, nil
}
<|endoftext|>


<commit_before>package main

import (
	"strings"
	"time"

	"github.com/redBorder/rbforwarder"
	"gopkg.in/Shopify/sarama.v1"
	"gopkg.in/bsm/sarama-cluster.v2"
)

// KafkaConsumer get messages from multiple kafka topics
type KafkaConsumer struct {
	backend  *rbforwarder.RBForwarder // The backend to send messages
	consumer *cluster.Consumer
	Config   KafkaConfig // Cofiguration after the parsing
}

// KafkaConfig stores the configuration for the Kafka source
type KafkaConfig struct {
	topics              []string // Topics where listen for messages
	brokers             []string // Brokers to connect
	consumergroup       string   // ID for the consumer
	consumerGroupConfig *cluster.Config
}

// Start starts reading messages from kafka and pushing them to the pipeline
func (k *KafkaConsumer) Start() {
	var err error

	// Init consumer, consume errors & messages
	k.consumer, err = cluster.NewConsumer(
		k.Config.brokers,
		k.Config.consumergroup,
		k.Config.topics,
		k.Config.consumerGroupConfig,
	)
	if err != nil {
		logger.Fatal("Failed to start consumer: ", err)
	}

	var offset uint64
	var eventsReported uint64
	var eventsSent uint64

	// Start processing reports
	done := make(chan struct{})
	go func() {
		for report := range k.backend.GetOrderedReports() {
			message := report.Metadata["sarama_message"].(*sarama.ConsumerMessage)

			if offset != report.ID {
				logger.Fatalf("Unexpected offset. Expected %d, found %d.",
					offset, report.ID)
			}

			if report.StatusCode != 0 {
				logger.
					WithField("ID", report.ID).
					WithField("STATUS", report.Status).
					WithField("OFFSET", message.Offset).
					Errorf("REPORT")
			} else {
				logger.
					WithField("ID", report.ID).
					WithField("STATUS", report.Status).
					WithField("OFFSET", message.Offset).
					Debugf("REPORT")
			}

			k.consumer.MarkOffset(message, "")
			offset++
			eventsReported++
		}

		done <- struct{}{}
	}()

	// Start consuming messages
	for message := range k.consumer.Messages() {
		msg, err := k.backend.TakeMessage()
		if err != nil {
			break
		}

		if _, err := msg.InputBuffer.Write(message.Value); err != nil {
			logger.Error("Error writing buffer: ", err)
		}

		msg.Metadata["sarama_message"] = message
		msg.Metadata["topic"] = message.Topic
		if err := msg.Produce(); err != nil {
			break
		}

		eventsSent++
	}

	logger.Info("Consumer terminated")

	<-done
	logger.Infof("TOTAL SENT MESSAGES: %d", eventsSent)
	logger.Infof("TOTAL REPORTS: %d", eventsReported)

	return
}

// Close closes the connection with Kafka
func (k *KafkaConsumer) Close() {
	if err := k.consumer.Close(); err != nil {
		logger.Println("Failed to close consumer: ", err)
	}
}

// ParseKafkaConfig reads the configuration from the YAML config file and store it
// on the instance
func (k *KafkaConsumer) ParseKafkaConfig(config map[string]interface{}) {

	// Create the config
	k.Config.consumerGroupConfig = cluster.NewConfig()
	k.Config.consumerGroupConfig.Config.Consumer.Offsets.CommitInterval = 1 * time.Second
	k.Config.consumerGroupConfig.Consumer.Offsets.Initial = sarama.OffsetNewest
	k.Config.consumerGroupConfig.Consumer.MaxProcessingTime = 3 * time.Second

	sarama.Logger = logger

	// Parse the brokers addresses
	if config["broker"] != nil {
		k.Config.brokers = strings.Split(config["broker"].(string), ",")
	}

	// Parse topics
	if config["topics"] != nil {
		topics := config["topics"].([]interface{})

		for _, topic := range topics {
			k.Config.topics = append(k.Config.topics, topic.(string))
		}
	}

	// Parse consumergroup
	if config["consumergroup"] != nil {
		k.Config.consumergroup = config["consumergroup"].(string)
	}
}
<commit_msg>Reload consumer on fail<commit_after>package main

import (
	"bytes"
	"strings"
	"time"

	"github.com/redBorder/rbforwarder"
	"gopkg.in/Shopify/sarama.v1"
	"gopkg.in/bsm/sarama-cluster.v2"
)

// KafkaConsumer get messages from multiple kafka topics
type KafkaConsumer struct {
	backend  *rbforwarder.RBForwarder // The backend to send messages
	consumer *cluster.Consumer
	closed   bool
	Config   KafkaConfig // Cofiguration after the parsing
}

// KafkaConfig stores the configuration for the Kafka source
type KafkaConfig struct {
	topics              []string // Topics where listen for messages
	brokers             []string // Brokers to connect
	consumergroup       string   // ID for the consumer
	consumerGroupConfig *cluster.Config
}

// Start starts reading messages from kafka and pushing them to the pipeline
func (k *KafkaConsumer) Start() {
	var err error

	var offset uint64
	var eventsReported uint64
	var eventsSent uint64

	// Start processing reports
	done := make(chan struct{})
	go func() {
		for report := range k.backend.GetOrderedReports() {
			message := report.Metadata["sarama_message"].(*sarama.ConsumerMessage)

			if offset != report.ID {
				logger.Fatalf("Unexpected offset. Expected %d, found %d.",
					offset, report.ID)
			}

			if report.StatusCode != 0 {
				logger.
					WithField("ID", report.ID).
					WithField("STATUS", report.Status).
					WithField("OFFSET", message.Offset).
					Errorf("REPORT")
			} else {
				logger.
					WithField("ID", report.ID).
					WithField("STATUS", report.Status).
					WithField("OFFSET", message.Offset).
					Debugf("REPORT")
			}

			k.consumer.MarkOffset(message, "")
			offset++
			eventsReported++
		}

		done <- struct{}{}
	}()

	// Init consumer, consume errors & messages
mainLoop:
	for {
		k.consumer, err = cluster.NewConsumer(
			k.Config.brokers,
			k.Config.consumergroup,
			k.Config.topics,
			k.Config.consumerGroupConfig,
		)
		if err != nil {
			logger.Fatal("Failed to start consumer: ", err)
			break
		}

		// Start consuming messages
		for message := range k.consumer.Messages() {
			if message == nil {
				k.Config.consumerGroupConfig.Consumer.Offsets.Initial = sarama.OffsetOldest
				if err := k.consumer.Close(); err != nil {
					logger.Error("Failed to close consumer: ", err)
				}
				break
			}

			msg, err := k.backend.TakeMessage()
			if err != nil {
				break mainLoop
			}

			msg.InputBuffer = bytes.NewBuffer(message.Value)
			msg.Metadata["sarama_message"] = message
			msg.Metadata["topic"] = message.Topic
			if err := msg.Produce(); err != nil {
				break mainLoop
			}

			eventsSent++
		}

		if k.closed {
			break
		}
	}

	logger.Info("Consumer terminated")

	<-done
	logger.Infof("TOTAL SENT MESSAGES: %d", eventsSent)
	logger.Infof("TOTAL REPORTS: %d", eventsReported)

	return
}

// Close closes the connection with Kafka
func (k *KafkaConsumer) Close() {
	k.closed = true
	if err := k.consumer.Close(); err != nil {
		logger.Println("Failed to close consumer: ", err)
	}
}

// ParseKafkaConfig reads the configuration from the YAML config file and store it
// on the instance
func (k *KafkaConsumer) ParseKafkaConfig(config map[string]interface{}) {

	// Create the config
	k.Config.consumerGroupConfig = cluster.NewConfig()
	k.Config.consumerGroupConfig.Config.Consumer.Offsets.CommitInterval = 1 * time.Second
	k.Config.consumerGroupConfig.Consumer.Offsets.Initial = sarama.OffsetNewest
	k.Config.consumerGroupConfig.Consumer.MaxProcessingTime = 3 * time.Second

	sarama.Logger = logger

	// Parse the brokers addresses
	if config["broker"] != nil {
		k.Config.brokers = strings.Split(config["broker"].(string), ",")
	}

	// Parse topics
	if config["topics"] != nil {
		topics := config["topics"].([]interface{})

		for _, topic := range topics {
			k.Config.topics = append(k.Config.topics, topic.(string))
		}
	}

	// Parse consumergroup
	if config["consumergroup"] != nil {
		k.Config.consumergroup = config["consumergroup"].(string)
	}
}
<|endoftext|>