<commit_before>package main

import (
	"bufio"
	"fmt"
	"os"
	"path/filepath"
	"restic"
	"strings"
	"time"

	"golang.org/x/crypto/ssh/terminal"

	"github.com/spf13/cobra"

	"restic/archiver"
	"restic/debug"
	"restic/errors"
	"restic/filter"
	"restic/fs"
)

var cmdBackup = &cobra.Command{
	Use:   "backup [flags] FILE/DIR [FILE/DIR] ...",
	Short: "create a new backup of files and/or directories",
	Long: `
The "backup" command creates a new snapshot and saves the files and directories
given as the arguments.
`,
	RunE: func(cmd *cobra.Command, args []string) error {
		if backupOptions.Stdin {
			return readBackupFromStdin(backupOptions, globalOptions, args)
		}

		return runBackup(backupOptions, globalOptions, args)
	},
}

// BackupOptions bundles all options for the backup command.
type BackupOptions struct {
	Parent         string
	Force          bool
	Excludes       []string
	ExcludeFile    string
	ExcludeOtherFS bool
	Stdin          bool
	StdinFilename  string
	Tags           []string
	FilesFrom      string
}

var backupOptions BackupOptions

func init() {
	cmdRoot.AddCommand(cmdBackup)

	f := cmdBackup.Flags()
	f.StringVar(&backupOptions.Parent, "parent", "", "use this parent snapshot (default: last snapshot in the repo that has the same target files/directories)")
	f.BoolVarP(&backupOptions.Force, "force", "f", false, `force re-reading the target files/directories. Overrides the "parent" flag`)
	f.StringSliceVarP(&backupOptions.Excludes, "exclude", "e", []string{}, "exclude a `pattern` (can be specified multiple times)")
	f.StringVar(&backupOptions.ExcludeFile, "exclude-file", "", "read exclude patterns from a file")
	f.BoolVarP(&backupOptions.ExcludeOtherFS, "one-file-system", "x", false, "Exclude other file systems")
	f.BoolVar(&backupOptions.Stdin, "stdin", false, "read backup from stdin")
	f.StringVar(&backupOptions.StdinFilename, "stdin-filename", "", "file name to use when reading from stdin")
	f.StringSliceVar(&backupOptions.Tags, "tag", []string{}, "add a `tag` for the new snapshot (can be specified multiple times)")
	f.StringVar(&backupOptions.FilesFrom, "files-from", "", "read the files to backup from file (can be combined with file args)")
}

func newScanProgress(gopts GlobalOptions) *restic.Progress {
	if gopts.Quiet {
		return nil
	}

	p := restic.NewProgress()
	p.OnUpdate = func(s restic.Stat, d time.Duration, ticker bool) {
		if IsProcessBackground() {
			return
		}

		PrintProgress("[%s] %d directories, %d files, %s", formatDuration(d), s.Dirs, s.Files, formatBytes(s.Bytes))
	}

	p.OnDone = func(s restic.Stat, d time.Duration, ticker bool) {
		PrintProgress("scanned %d directories, %d files in %s\n", s.Dirs, s.Files, formatDuration(d))
	}

	return p
}

func newArchiveProgress(gopts GlobalOptions, todo restic.Stat) *restic.Progress {
	if gopts.Quiet {
		return nil
	}

	archiveProgress := restic.NewProgress()

	var bps, eta uint64
	itemsTodo := todo.Files + todo.Dirs

	archiveProgress.OnUpdate = func(s restic.Stat, d time.Duration, ticker bool) {
		if IsProcessBackground() {
			return
		}

		sec := uint64(d / time.Second)
		if todo.Bytes > 0 && sec > 0 && ticker {
			bps = s.Bytes / sec
			if s.Bytes >= todo.Bytes {
				eta = 0
			} else if bps > 0 {
				eta = (todo.Bytes - s.Bytes) / bps
			}
		}

		itemsDone := s.Files + s.Dirs

		status1 := fmt.Sprintf("[%s] %s  %s/s  %s / %s  %d / %d items  %d errors  ",
			formatDuration(d),
			formatPercent(s.Bytes, todo.Bytes),
			formatBytes(bps),
			formatBytes(s.Bytes), formatBytes(todo.Bytes),
			itemsDone, itemsTodo,
			s.Errors)
		status2 := fmt.Sprintf("ETA %s ", formatSeconds(eta))

		w, _, err := terminal.GetSize(int(os.Stdout.Fd()))
		if err == nil {
			maxlen := w - len(status2) - 1

			if maxlen < 4 {
				status1 = ""
			} else if len(status1) > maxlen {
				status1 = status1[:maxlen-4]
				status1 += "... "
			}
		}

		PrintProgress("%s%s", status1, status2)
	}

	archiveProgress.OnDone = func(s restic.Stat, d time.Duration, ticker bool) {
		fmt.Printf("\nduration: %s, %s\n", formatDuration(d), formatRate(todo.Bytes, d))
	}

	return archiveProgress
}

func newArchiveStdinProgress(gopts GlobalOptions) *restic.Progress {
	if gopts.Quiet {
		return nil
	}

	archiveProgress := restic.NewProgress()

	var bps uint64

	archiveProgress.OnUpdate = func(s restic.Stat, d time.Duration, ticker bool) {
		if IsProcessBackground() {
			return
		}

		sec := uint64(d / time.Second)
		if s.Bytes > 0 && sec > 0 && ticker {
			bps = s.Bytes / sec
		}

		status1 := fmt.Sprintf("[%s] %s  %s/s", formatDuration(d),
			formatBytes(s.Bytes),
			formatBytes(bps))

		w, _, err := terminal.GetSize(int(os.Stdout.Fd()))
		if err == nil {
			maxlen := w - len(status1)

			if maxlen < 4 {
				status1 = ""
			} else if len(status1) > maxlen {
				status1 = status1[:maxlen-4]
				status1 += "... "
			}
		}

		PrintProgress("%s", status1)
	}

	archiveProgress.OnDone = func(s restic.Stat, d time.Duration, ticker bool) {
		fmt.Printf("\nduration: %s, %s\n", formatDuration(d), formatRate(s.Bytes, d))
	}

	return archiveProgress
}

// filterExisting returns a slice of all existing items, or an error if no
// items exist at all.
func filterExisting(items []string) (result []string, err error) {
	for _, item := range items {
		_, err := fs.Lstat(item)
		if err != nil && os.IsNotExist(errors.Cause(err)) {
			continue
		}

		result = append(result, item)
	}

	if len(result) == 0 {
		return nil, errors.Fatal("all target directories/files do not exist")
	}

	return
}

// gatherDevices returns the set of unique device ids of the files and/or
// directory paths listed in "items".
func gatherDevices(items []string) (deviceMap map[uint64]struct{}, err error) {
	deviceMap = make(map[uint64]struct{})
	for _, item := range items {
		fi, err := fs.Lstat(item)
		if err != nil {
			return nil, err
		}
		id, err := fs.DeviceID(fi)
		if err != nil {
			return nil, err
		}
		deviceMap[id] = struct{}{}
	}
	if len(deviceMap) == 0 {
		return nil, errors.New("zero allowed devices")
	}
	return deviceMap, nil
}

func readBackupFromStdin(opts BackupOptions, gopts GlobalOptions, args []string) error {
	if len(args) != 0 {
		return errors.Fatalf("when reading from stdin, no additional files can be specified")
	}

	repo, err := OpenRepository(gopts)
	if err != nil {
		return err
	}

	lock, err := lockRepo(repo)
	defer unlockRepo(lock)
	if err != nil {
		return err
	}

	err = repo.LoadIndex()
	if err != nil {
		return err
	}

	_, id, err := archiver.ArchiveReader(repo, newArchiveStdinProgress(gopts), os.Stdin, opts.StdinFilename, opts.Tags)
	if err != nil {
		return err
	}

	Verbosef("archived as %v\n", id.Str())
	return nil
}

// readFromFile will read all lines from the given filename and write them to a
// string array, if filename is empty readFromFile returns and empty string
// array
func readLinesFromFile(filename string) ([]string, error) {
	if filename == "" {
		return nil, nil
	}

	file, err := os.Open(filename)
	if err != nil {
		return nil, err
	}
	defer file.Close()

	var lines []string

	scanner := bufio.NewScanner(file)
	for scanner.Scan() {
		lines = append(lines, scanner.Text())
	}

	if err := scanner.Err(); err != nil {
		return nil, err
	}

	return lines, nil
}

func runBackup(opts BackupOptions, gopts GlobalOptions, args []string) error {
	target, err := readLinesFromFile(opts.FilesFrom)
	if err != nil {
		return err
	}

	// merge files from files-from into normal args so we can reuse the normal
	// args checks and have the ability to use both files-from and args at the
	// same time
	args = append(args, target...)
	if len(args) == 0 {
		return errors.Fatalf("wrong number of parameters")
	}

	for _, d := range args {
		if a, err := filepath.Abs(d); err == nil {
			d = a
		}
		target = append(target, d)
	}

	target, err = filterExisting(target)
	if err != nil {
		return err
	}

	// allowed devices
	var allowedDevs map[uint64]struct{}
	if opts.ExcludeOtherFS {
		allowedDevs, err = gatherDevices(target)
		if err != nil {
			return err
		}
		debug.Log("allowed devices: %v\n", allowedDevs)
	}

	repo, err := OpenRepository(gopts)
	if err != nil {
		return err
	}

	lock, err := lockRepo(repo)
	defer unlockRepo(lock)
	if err != nil {
		return err
	}

	err = repo.LoadIndex()
	if err != nil {
		return err
	}

	var parentSnapshotID *restic.ID

	// Force using a parent
	if !opts.Force && opts.Parent != "" {
		id, err := restic.FindSnapshot(repo, opts.Parent)
		if err != nil {
			return errors.Fatalf("invalid id %q: %v", opts.Parent, err)
		}

		parentSnapshotID = &id
	}

	// Find last snapshot to set it as parent, if not already set
	if !opts.Force && parentSnapshotID == nil {
		hostname, err := os.Hostname()
		if err != nil {
			debug.Log("os.Hostname() returned err: %v", err)
			hostname = ""
		}

		id, err := restic.FindLatestSnapshot(repo, target, hostname)
		if err == nil {
			parentSnapshotID = &id
		} else if err != restic.ErrNoSnapshotFound {
			return err
		}
	}

	if parentSnapshotID != nil {
		Verbosef("using parent snapshot %v\n", parentSnapshotID.Str())
	}

	Verbosef("scan %v\n", target)

	// add patterns from file
	if opts.ExcludeFile != "" {
		file, err := fs.Open(opts.ExcludeFile)
		if err != nil {
			Warnf("error reading exclude patterns: %v", err)
			return nil
		}

		scanner := bufio.NewScanner(file)
		for scanner.Scan() {
			line := scanner.Text()
			if !strings.HasPrefix(line, "#") {
				line = os.ExpandEnv(line)
				opts.Excludes = append(opts.Excludes, line)
			}
		}
	}

	selectFilter := func(item string, fi os.FileInfo) bool {
		matched, err := filter.List(opts.Excludes, item)
		if err != nil {
			Warnf("error for exclude pattern: %v", err)
		}

		if matched {
			debug.Log("path %q excluded by a filter", item)
			return false
		}

		if !opts.ExcludeOtherFS || fi == nil {
			return true
		}

		id, err := fs.DeviceID(fi)
		if err != nil {
			// This should never happen because gatherDevices() would have
			// errored out earlier. If it still does that's a reason to panic.
			panic(err)
		}
		_, found := allowedDevs[id]
		if !found {
			debug.Log("path %q on disallowed device %d", item, id)
			return false
		}

		return true
	}

	stat, err := archiver.Scan(target, selectFilter, newScanProgress(gopts))
	if err != nil {
		return err
	}

	arch := archiver.New(repo)
	arch.Excludes = opts.Excludes
	arch.SelectFilter = selectFilter

	arch.Warn = func(dir string, fi os.FileInfo, err error) {
		// TODO: make ignoring errors configurable
		Warnf("%s\rwarning for %s: %v\n", ClearLine(), dir, err)
	}

	_, id, err := arch.Snapshot(newArchiveProgress(gopts, stat), target, opts.Tags, parentSnapshotID)
	if err != nil {
		return err
	}

	Verbosef("snapshot %s saved\n", id.Str())

	return nil
}
<commit_msg>Avoid duplicate backup paths<commit_after>package main

import (
	"bufio"
	"fmt"
	"os"
	"path/filepath"
	"restic"
	"strings"
	"time"

	"golang.org/x/crypto/ssh/terminal"

	"github.com/spf13/cobra"

	"restic/archiver"
	"restic/debug"
	"restic/errors"
	"restic/filter"
	"restic/fs"
)

var cmdBackup = &cobra.Command{
	Use:   "backup [flags] FILE/DIR [FILE/DIR] ...",
	Short: "create a new backup of files and/or directories",
	Long: `
The "backup" command creates a new snapshot and saves the files and directories
given as the arguments.
`,
	RunE: func(cmd *cobra.Command, args []string) error {
		if backupOptions.Stdin {
			return readBackupFromStdin(backupOptions, globalOptions, args)
		}

		return runBackup(backupOptions, globalOptions, args)
	},
}

// BackupOptions bundles all options for the backup command.
type BackupOptions struct {
	Parent         string
	Force          bool
	Excludes       []string
	ExcludeFile    string
	ExcludeOtherFS bool
	Stdin          bool
	StdinFilename  string
	Tags           []string
	FilesFrom      string
}

var backupOptions BackupOptions

func init() {
	cmdRoot.AddCommand(cmdBackup)

	f := cmdBackup.Flags()
	f.StringVar(&backupOptions.Parent, "parent", "", "use this parent snapshot (default: last snapshot in the repo that has the same target files/directories)")
	f.BoolVarP(&backupOptions.Force, "force", "f", false, `force re-reading the target files/directories. Overrides the "parent" flag`)
	f.StringSliceVarP(&backupOptions.Excludes, "exclude", "e", []string{}, "exclude a `pattern` (can be specified multiple times)")
	f.StringVar(&backupOptions.ExcludeFile, "exclude-file", "", "read exclude patterns from a file")
	f.BoolVarP(&backupOptions.ExcludeOtherFS, "one-file-system", "x", false, "Exclude other file systems")
	f.BoolVar(&backupOptions.Stdin, "stdin", false, "read backup from stdin")
	f.StringVar(&backupOptions.StdinFilename, "stdin-filename", "", "file name to use when reading from stdin")
	f.StringSliceVar(&backupOptions.Tags, "tag", []string{}, "add a `tag` for the new snapshot (can be specified multiple times)")
	f.StringVar(&backupOptions.FilesFrom, "files-from", "", "read the files to backup from file (can be combined with file args)")
}

func newScanProgress(gopts GlobalOptions) *restic.Progress {
	if gopts.Quiet {
		return nil
	}

	p := restic.NewProgress()
	p.OnUpdate = func(s restic.Stat, d time.Duration, ticker bool) {
		if IsProcessBackground() {
			return
		}

		PrintProgress("[%s] %d directories, %d files, %s", formatDuration(d), s.Dirs, s.Files, formatBytes(s.Bytes))
	}

	p.OnDone = func(s restic.Stat, d time.Duration, ticker bool) {
		PrintProgress("scanned %d directories, %d files in %s\n", s.Dirs, s.Files, formatDuration(d))
	}

	return p
}

func newArchiveProgress(gopts GlobalOptions, todo restic.Stat) *restic.Progress {
	if gopts.Quiet {
		return nil
	}

	archiveProgress := restic.NewProgress()

	var bps, eta uint64
	itemsTodo := todo.Files + todo.Dirs

	archiveProgress.OnUpdate = func(s restic.Stat, d time.Duration, ticker bool) {
		if IsProcessBackground() {
			return
		}

		sec := uint64(d / time.Second)
		if todo.Bytes > 0 && sec > 0 && ticker {
			bps = s.Bytes / sec
			if s.Bytes >= todo.Bytes {
				eta = 0
			} else if bps > 0 {
				eta = (todo.Bytes - s.Bytes) / bps
			}
		}

		itemsDone := s.Files + s.Dirs

		status1 := fmt.Sprintf("[%s] %s  %s/s  %s / %s  %d / %d items  %d errors  ",
			formatDuration(d),
			formatPercent(s.Bytes, todo.Bytes),
			formatBytes(bps),
			formatBytes(s.Bytes), formatBytes(todo.Bytes),
			itemsDone, itemsTodo,
			s.Errors)
		status2 := fmt.Sprintf("ETA %s ", formatSeconds(eta))

		w, _, err := terminal.GetSize(int(os.Stdout.Fd()))
		if err == nil {
			maxlen := w - len(status2) - 1

			if maxlen < 4 {
				status1 = ""
			} else if len(status1) > maxlen {
				status1 = status1[:maxlen-4]
				status1 += "... "
			}
		}

		PrintProgress("%s%s", status1, status2)
	}

	archiveProgress.OnDone = func(s restic.Stat, d time.Duration, ticker bool) {
		fmt.Printf("\nduration: %s, %s\n", formatDuration(d), formatRate(todo.Bytes, d))
	}

	return archiveProgress
}

func newArchiveStdinProgress(gopts GlobalOptions) *restic.Progress {
	if gopts.Quiet {
		return nil
	}

	archiveProgress := restic.NewProgress()

	var bps uint64

	archiveProgress.OnUpdate = func(s restic.Stat, d time.Duration, ticker bool) {
		if IsProcessBackground() {
			return
		}

		sec := uint64(d / time.Second)
		if s.Bytes > 0 && sec > 0 && ticker {
			bps = s.Bytes / sec
		}

		status1 := fmt.Sprintf("[%s] %s  %s/s", formatDuration(d),
			formatBytes(s.Bytes),
			formatBytes(bps))

		w, _, err := terminal.GetSize(int(os.Stdout.Fd()))
		if err == nil {
			maxlen := w - len(status1)

			if maxlen < 4 {
				status1 = ""
			} else if len(status1) > maxlen {
				status1 = status1[:maxlen-4]
				status1 += "... "
			}
		}

		PrintProgress("%s", status1)
	}

	archiveProgress.OnDone = func(s restic.Stat, d time.Duration, ticker bool) {
		fmt.Printf("\nduration: %s, %s\n", formatDuration(d), formatRate(s.Bytes, d))
	}

	return archiveProgress
}

// filterExisting returns a slice of all existing items, or an error if no
// items exist at all.
func filterExisting(items []string) (result []string, err error) {
	for _, item := range items {
		_, err := fs.Lstat(item)
		if err != nil && os.IsNotExist(errors.Cause(err)) {
			continue
		}

		result = append(result, item)
	}

	if len(result) == 0 {
		return nil, errors.Fatal("all target directories/files do not exist")
	}

	return
}

// gatherDevices returns the set of unique device ids of the files and/or
// directory paths listed in "items".
func gatherDevices(items []string) (deviceMap map[uint64]struct{}, err error) {
	deviceMap = make(map[uint64]struct{})
	for _, item := range items {
		fi, err := fs.Lstat(item)
		if err != nil {
			return nil, err
		}
		id, err := fs.DeviceID(fi)
		if err != nil {
			return nil, err
		}
		deviceMap[id] = struct{}{}
	}
	if len(deviceMap) == 0 {
		return nil, errors.New("zero allowed devices")
	}
	return deviceMap, nil
}

func readBackupFromStdin(opts BackupOptions, gopts GlobalOptions, args []string) error {
	if len(args) != 0 {
		return errors.Fatalf("when reading from stdin, no additional files can be specified")
	}

	repo, err := OpenRepository(gopts)
	if err != nil {
		return err
	}

	lock, err := lockRepo(repo)
	defer unlockRepo(lock)
	if err != nil {
		return err
	}

	err = repo.LoadIndex()
	if err != nil {
		return err
	}

	_, id, err := archiver.ArchiveReader(repo, newArchiveStdinProgress(gopts), os.Stdin, opts.StdinFilename, opts.Tags)
	if err != nil {
		return err
	}

	Verbosef("archived as %v\n", id.Str())
	return nil
}

// readFromFile will read all lines from the given filename and write them to a
// string array, if filename is empty readFromFile returns and empty string
// array
func readLinesFromFile(filename string) ([]string, error) {
	if filename == "" {
		return nil, nil
	}

	file, err := os.Open(filename)
	if err != nil {
		return nil, err
	}
	defer file.Close()

	var lines []string

	scanner := bufio.NewScanner(file)
	for scanner.Scan() {
		lines = append(lines, scanner.Text())
	}

	if err := scanner.Err(); err != nil {
		return nil, err
	}

	return lines, nil
}

func runBackup(opts BackupOptions, gopts GlobalOptions, args []string) error {
	fromfile, err := readLinesFromFile(opts.FilesFrom)
	if err != nil {
		return err
	}

	// merge files from files-from into normal args so we can reuse the normal
	// args checks and have the ability to use both files-from and args at the
	// same time
	args = append(args, fromfile...)
	if len(args) == 0 {
		return errors.Fatalf("wrong number of parameters")
	}

	target := make([]string, 0, len(args))
	for _, d := range args {
		if a, err := filepath.Abs(d); err == nil {
			d = a
		}
		target = append(target, d)
	}

	target, err = filterExisting(target)
	if err != nil {
		return err
	}

	// allowed devices
	var allowedDevs map[uint64]struct{}
	if opts.ExcludeOtherFS {
		allowedDevs, err = gatherDevices(target)
		if err != nil {
			return err
		}
		debug.Log("allowed devices: %v\n", allowedDevs)
	}

	repo, err := OpenRepository(gopts)
	if err != nil {
		return err
	}

	lock, err := lockRepo(repo)
	defer unlockRepo(lock)
	if err != nil {
		return err
	}

	err = repo.LoadIndex()
	if err != nil {
		return err
	}

	var parentSnapshotID *restic.ID

	// Force using a parent
	if !opts.Force && opts.Parent != "" {
		id, err := restic.FindSnapshot(repo, opts.Parent)
		if err != nil {
			return errors.Fatalf("invalid id %q: %v", opts.Parent, err)
		}

		parentSnapshotID = &id
	}

	// Find last snapshot to set it as parent, if not already set
	if !opts.Force && parentSnapshotID == nil {
		hostname, err := os.Hostname()
		if err != nil {
			debug.Log("os.Hostname() returned err: %v", err)
			hostname = ""
		}

		id, err := restic.FindLatestSnapshot(repo, target, hostname)
		if err == nil {
			parentSnapshotID = &id
		} else if err != restic.ErrNoSnapshotFound {
			return err
		}
	}

	if parentSnapshotID != nil {
		Verbosef("using parent snapshot %v\n", parentSnapshotID.Str())
	}

	Verbosef("scan %v\n", target)

	// add patterns from file
	if opts.ExcludeFile != "" {
		file, err := fs.Open(opts.ExcludeFile)
		if err != nil {
			Warnf("error reading exclude patterns: %v", err)
			return nil
		}

		scanner := bufio.NewScanner(file)
		for scanner.Scan() {
			line := scanner.Text()
			if !strings.HasPrefix(line, "#") {
				line = os.ExpandEnv(line)
				opts.Excludes = append(opts.Excludes, line)
			}
		}
	}

	selectFilter := func(item string, fi os.FileInfo) bool {
		matched, err := filter.List(opts.Excludes, item)
		if err != nil {
			Warnf("error for exclude pattern: %v", err)
		}

		if matched {
			debug.Log("path %q excluded by a filter", item)
			return false
		}

		if !opts.ExcludeOtherFS || fi == nil {
			return true
		}

		id, err := fs.DeviceID(fi)
		if err != nil {
			// This should never happen because gatherDevices() would have
			// errored out earlier. If it still does that's a reason to panic.
			panic(err)
		}
		_, found := allowedDevs[id]
		if !found {
			debug.Log("path %q on disallowed device %d", item, id)
			return false
		}

		return true
	}

	stat, err := archiver.Scan(target, selectFilter, newScanProgress(gopts))
	if err != nil {
		return err
	}

	arch := archiver.New(repo)
	arch.Excludes = opts.Excludes
	arch.SelectFilter = selectFilter

	arch.Warn = func(dir string, fi os.FileInfo, err error) {
		// TODO: make ignoring errors configurable
		Warnf("%s\rwarning for %s: %v\n", ClearLine(), dir, err)
	}

	_, id, err := arch.Snapshot(newArchiveProgress(gopts, stat), target, opts.Tags, parentSnapshotID)
	if err != nil {
		return err
	}

	Verbosef("snapshot %s saved\n", id.Str())

	return nil
}
<|endoftext|>


<commit_before>package gmail

import (
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"net/http"
)

type AttachmentResponse struct {
	Data string  `json:"data"`
	Size float64 `json:"size"`
}

func (g *Gmail) GetAttachmentById(messageId string, attachmentId string) (response AttachmentResponse, err error) {
	toReturn := AttachmentResponse{}
	if !(len(messageId) > 0) || !(len(attachmentId) > 0) {
		return toReturn, errors.New("Message Id, or attachment Id missing.")
	}
	if len(g.AccessToken) > 0 {
		URL := BASEURL + "gmail/v1/users/me/messages/" + messageId + "/attachments/" + attachmentId + "?access_token=" + g.AccessToken
		response, err := http.Get(URL)
		if err != nil {
			fmt.Printf("%s", err)
			return toReturn, err
		} else {
			defer response.Body.Close()
			contents, err := ioutil.ReadAll(response.Body)
			if err != nil {
				return toReturn, err
			}
			err = json.Unmarshal(contents, &toReturn)
			if err != nil {
				return toReturn, err
			}
			if toReturn.Size == 0 {
				return toReturn, errors.New("Incorrect API key, message Id, or attachment Id.")
			}
			return toReturn, nil
		}
	}
	return toReturn, errors.New("Missing API key")
}
<commit_msg>Removing check condition from Attachments to formalize it better next time<commit_after>package gmail

import (
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"net/http"
)

type AttachmentResponse struct {
	Data string  `json:"data"`
	Size float64 `json:"size"`
}

func (g *Gmail) GetAttachmentById(messageId string, attachmentId string) (response AttachmentResponse, err error) {
	toReturn := AttachmentResponse{}
	if len(g.AccessToken) > 0 {
		URL := BASEURL + "gmail/v1/users/me/messages/" + messageId + "/attachments/" + attachmentId + "?access_token=" + g.AccessToken
		response, err := http.Get(URL)
		if err != nil {
			fmt.Printf("%s", err)
			return toReturn, err
		} else {
			defer response.Body.Close()
			contents, err := ioutil.ReadAll(response.Body)
			if err != nil {
				return toReturn, err
			}
			err = json.Unmarshal(contents, &toReturn)
			if err != nil {
				return toReturn, err
			}
			if toReturn.Size == 0 {
				return toReturn, errors.New("Incorrect API key, message Id, or attachment Id.")
			}
			return toReturn, nil
		}
	}
	return toReturn, errors.New("Missing API key")
}
<|endoftext|>


<commit_before>package byzq

import (
	"crypto/ecdsa"
	"crypto/rand"
	"crypto/sha256"
	"fmt"
	"log"
	"math/big"
	"sync"
	"time"
)

// AuthDataQ is the quorum specification for the Authenticated-Data Byzantine quorum
// algorithm described in RSDP, Algorithm 4.15, page 181.
type AuthDataQ struct {
	n    int               // size of system
	f    int               // tolerable number of failures
	q    int               // quorum size
	priv *ecdsa.PrivateKey // writer's private key for signing
	pub  *ecdsa.PublicKey  // public key of the writer (used by readers)
	wts  int64             // writer's timestamp
}

// NewAuthDataQ returns a Byzantine masking quorum specification or nil and an error
// if the quorum requirements are not satisifed.
func NewAuthDataQ(n int, priv *ecdsa.PrivateKey, pub *ecdsa.PublicKey) (*AuthDataQ, error) {
	f := (n - 1) / 3
	if f < 1 {
		return nil, fmt.Errorf("Byzantine quorum require n>3f replicas; only got n=%d, yielding f=%d", n, f)
	}
	return &AuthDataQ{n, f, (n + f) / 2, priv, pub, 0}, nil
}

// IncWTS updates the writer's timestamp wts. This is not thread safe.
func (aq *AuthDataQ) IncWTS() int64 {
	aq.wts++
	return aq.wts
}

// NewTS reads the system clock updates the writer's timestamp wts. This is not thread safe.
func (aq *AuthDataQ) NewTS() int64 {
	aq.wts = time.Now().UnixNano()
	return aq.wts
}

// Sign signs the provided content and returns a value to be passed into Write.
// (This function must currently be exported since our writer client code is not
// in the byzq package.)
func (aq *AuthDataQ) Sign(content *Content) (*Value, error) {
	msg, err := content.Marshal()
	if err != nil {
		return nil, err
	}
	hash := sha256.Sum256(msg)
	r, s, err := ecdsa.Sign(rand.Reader, aq.priv, hash[:])
	if err != nil {
		return nil, err
	}
	return &Value{C: content, SignatureR: r.Bytes(), SignatureS: s.Bytes()}, nil
}

func (aq *AuthDataQ) verify(reply *Value) bool {
	msg, err := reply.C.Marshal()
	if err != nil {
		log.Printf("failed to marshal msg for verify: %v", err)
		return false
	}
	msgHash := sha256.Sum256(msg)
	r := new(big.Int).SetBytes(reply.SignatureR)
	s := new(big.Int).SetBytes(reply.SignatureS)
	return ecdsa.Verify(aq.pub, msgHash[:], r, s)
}

// ReadQF returns nil and false until the supplied replies
// constitute a Byzantine quorum, at which point the
// method returns the single highest value and true.
func (aq *AuthDataQ) ReadQF(replies []*Value) (*Value, bool) {
	if len(replies) <= aq.q {
		// not enough replies yet; need at least bq.q=(n+2f)/2 replies
		return nil, false
	}
	var highest *Value
	for _, reply := range replies {
		if aq.verify(reply) {
			if highest != nil && reply.C.Timestamp <= highest.C.Timestamp {
				continue
			}
			highest = reply
		}
	}
	// returns reply with the highest timestamp, or nil if no replies were verified
	return highest, true
}

// HReadQF returns Hein's QFunc version 1
func (aq *AuthDataQ) HReadQF(replies []*Value) (*Value, bool) {
	if len(replies) <= aq.q {
		// not enough replies yet; need at least bq.q=(n+2f)/2 replies
		return nil, false
	}
	verified := make([]bool, len(replies))
	wg := &sync.WaitGroup{}
	for i, reply := range replies {
		wg.Add(1)
		go func(i int, r *Value) {
			verified[i] = aq.verify(r)
			wg.Done()
		}(i, reply)
	}
	wg.Wait()
	cnt := 0
	var highest *Value
	for i, v := range verified {
		if !v {
			// some signature could not be verified:
			cnt++
			if len(replies)-cnt <= aq.q {
				return nil, false
			}
		}
		if highest != nil && replies[i].C.Timestamp <= highest.C.Timestamp {
			continue
		}
		highest = replies[i]
	}

	// returns reply with the highest timestamp, or nil if no replies were verified
	return highest, true
}

// LReadQF returns Leanders QFunc version 1
func (aq *AuthDataQ) LReadQF(replies []*Value) (*Value, bool) {
	if len(replies) <= aq.q {
		// not enough replies yet; need at least bq.q=(n+2f)/2 replies
		return nil, false
	}

	veriresult := make(chan int, len(replies))
	for i, reply := range replies {
		go func(i int, r *Value) {
			if !aq.verify(r) {
				i = -1
			}
			veriresult <- i
		}(i, reply)
	}

	cnt := 0
	var highest *Value
	for j := 0; j < len(replies); j++ {
		i := <-veriresult
		if i == -1 {
			// some signature could not be verified:
			cnt++
			if len(replies)-cnt <= aq.q {
				return nil, false
			}
		}
		if highest != nil && replies[i].C.Timestamp <= highest.C.Timestamp {
			continue
		}
		highest = replies[i]
	}
	// returns reply with the highest timestamp, or nil if no replies were verified
	return highest, true
}

// L2ReadQF returns Leanders QFunc version 2
func (aq *AuthDataQ) L2ReadQF(replies []*Value) (*Value, bool) {
	if len(replies) < 1 {
		return nil, false
	}
	if !aq.verify(replies[len(replies)-1]) {
		// return if last reply failed to verify
		replies[len(replies)-1] = nil
		return nil, false
	}
	if len(replies) <= aq.q {
		// not enough replies yet; need at least bq.q=(n+2f)/2 replies
		return nil, false
	}

	var highest *Value
	cntnotnil := 0
	for _, reply := range replies {
		if reply == nil {
			continue
		}
		cntnotnil++
		// select reply with highest timestamp
		if highest != nil && reply.C.Timestamp <= highest.C.Timestamp {
			continue
		}
		highest = reply
	}

	if cntnotnil <= aq.q {
		// not enough replies yet; need at least bq.q=(n+2f)/2 replies
		return nil, false
	}
	// returns reply with the highest timestamp, or nil if no replies were verified
	return highest, true
}

// WriteQF returns nil and false until it is possible to check for a quorum.
// If enough replies with the same timestamp is found, we return true.
func (aq *AuthDataQ) WriteQF(replies []*WriteResponse) (*WriteResponse, bool) {
	if len(replies) <= aq.q {
		return nil, false
	}
	cnt := 0
	var reply *WriteResponse
	for _, r := range replies {
		if aq.wts == r.Timestamp {
			cnt++
			reply = r
		}
	}
	if cnt < aq.q {
		return nil, false
	}
	return reply, true
}
<commit_msg>switched to new auth QF which is faster by processing last reply first<commit_after>package byzq

import (
	"crypto/ecdsa"
	"crypto/rand"
	"crypto/sha256"
	"fmt"
	"log"
	"math/big"
	"sync"
	"time"
)

// AuthDataQ is the quorum specification for the Authenticated-Data Byzantine quorum
// algorithm described in RSDP, Algorithm 4.15, page 181.
type AuthDataQ struct {
	n    int               // size of system
	f    int               // tolerable number of failures
	q    int               // quorum size
	priv *ecdsa.PrivateKey // writer's private key for signing
	pub  *ecdsa.PublicKey  // public key of the writer (used by readers)
	wts  int64             // writer's timestamp
}

// NewAuthDataQ returns a Byzantine masking quorum specification or nil and an error
// if the quorum requirements are not satisifed.
func NewAuthDataQ(n int, priv *ecdsa.PrivateKey, pub *ecdsa.PublicKey) (*AuthDataQ, error) {
	f := (n - 1) / 3
	if f < 1 {
		return nil, fmt.Errorf("Byzantine quorum require n>3f replicas; only got n=%d, yielding f=%d", n, f)
	}
	return &AuthDataQ{n, f, (n + f) / 2, priv, pub, 0}, nil
}

// IncWTS updates the writer's timestamp wts. This is not thread safe.
func (aq *AuthDataQ) IncWTS() int64 {
	aq.wts++
	return aq.wts
}

// NewTS reads the system clock updates the writer's timestamp wts. This is not thread safe.
func (aq *AuthDataQ) NewTS() int64 {
	aq.wts = time.Now().UnixNano()
	return aq.wts
}

// Sign signs the provided content and returns a value to be passed into Write.
// (This function must currently be exported since our writer client code is not
// in the byzq package.)
func (aq *AuthDataQ) Sign(content *Content) (*Value, error) {
	msg, err := content.Marshal()
	if err != nil {
		return nil, err
	}
	hash := sha256.Sum256(msg)
	r, s, err := ecdsa.Sign(rand.Reader, aq.priv, hash[:])
	if err != nil {
		return nil, err
	}
	return &Value{C: content, SignatureR: r.Bytes(), SignatureS: s.Bytes()}, nil
}

func (aq *AuthDataQ) verify(reply *Value) bool {
	msg, err := reply.C.Marshal()
	if err != nil {
		log.Printf("failed to marshal msg for verify: %v", err)
		return false
	}
	msgHash := sha256.Sum256(msg)
	r := new(big.Int).SetBytes(reply.SignatureR)
	s := new(big.Int).SetBytes(reply.SignatureS)
	return ecdsa.Verify(aq.pub, msgHash[:], r, s)
}

// ReadQF returns nil and false until the supplied replies
// constitute a Byzantine quorum, at which point the
// method returns the single highest value and true.
func (aq *AuthDataQ) OrigReadQF(replies []*Value) (*Value, bool) {
	if len(replies) <= aq.q {
		// not enough replies yet; need at least bq.q=(n+2f)/2 replies
		return nil, false
	}
	var highest *Value
	for _, reply := range replies {
		if aq.verify(reply) {
			if highest != nil && reply.C.Timestamp <= highest.C.Timestamp {
				continue
			}
			highest = reply
		}
	}
	// returns reply with the highest timestamp, or nil if no replies were verified
	return highest, true
}

// HReadQF returns Hein's QFunc version 1
func (aq *AuthDataQ) HReadQF(replies []*Value) (*Value, bool) {
	if len(replies) <= aq.q {
		// not enough replies yet; need at least bq.q=(n+2f)/2 replies
		return nil, false
	}
	verified := make([]bool, len(replies))
	wg := &sync.WaitGroup{}
	for i, reply := range replies {
		wg.Add(1)
		go func(i int, r *Value) {
			verified[i] = aq.verify(r)
			wg.Done()
		}(i, reply)
	}
	wg.Wait()
	cnt := 0
	var highest *Value
	for i, v := range verified {
		if !v {
			// some signature could not be verified:
			cnt++
			if len(replies)-cnt <= aq.q {
				return nil, false
			}
		}
		if highest != nil && replies[i].C.Timestamp <= highest.C.Timestamp {
			continue
		}
		highest = replies[i]
	}

	// returns reply with the highest timestamp, or nil if no replies were verified
	return highest, true
}

// LReadQF returns Leanders QFunc version 1
func (aq *AuthDataQ) LReadQF(replies []*Value) (*Value, bool) {
	if len(replies) <= aq.q {
		// not enough replies yet; need at least bq.q=(n+2f)/2 replies
		return nil, false
	}

	veriresult := make(chan int, len(replies))
	for i, reply := range replies {
		go func(i int, r *Value) {
			if !aq.verify(r) {
				i = -1
			}
			veriresult <- i
		}(i, reply)
	}

	cnt := 0
	var highest *Value
	for j := 0; j < len(replies); j++ {
		i := <-veriresult
		if i == -1 {
			// some signature could not be verified:
			cnt++
			if len(replies)-cnt <= aq.q {
				return nil, false
			}
		}
		if highest != nil && replies[i].C.Timestamp <= highest.C.Timestamp {
			continue
		}
		highest = replies[i]
	}
	// returns reply with the highest timestamp, or nil if no replies were verified
	return highest, true
}

// L2ReadQF returns Leanders QFunc version 2
func (aq *AuthDataQ) ReadQF(replies []*Value) (*Value, bool) {
	if len(replies) < 1 {
		return nil, false
	}
	if !aq.verify(replies[len(replies)-1]) {
		// return if last reply failed to verify
		replies[len(replies)-1] = nil
		return nil, false
	}
	if len(replies) <= aq.q {
		// not enough replies yet; need at least bq.q=(n+2f)/2 replies
		return nil, false
	}

	var highest *Value
	cntnotnil := 0
	for _, reply := range replies {
		if reply == nil {
			continue
		}
		cntnotnil++
		// select reply with highest timestamp
		if highest != nil && reply.C.Timestamp <= highest.C.Timestamp {
			continue
		}
		highest = reply
	}

	if cntnotnil <= aq.q {
		// not enough replies yet; need at least bq.q=(n+2f)/2 replies
		return nil, false
	}
	// returns reply with the highest timestamp, or nil if no replies were verified
	return highest, true
}

// WriteQF returns nil and false until it is possible to check for a quorum.
// If enough replies with the same timestamp is found, we return true.
func (aq *AuthDataQ) WriteQF(replies []*WriteResponse) (*WriteResponse, bool) {
	if len(replies) <= aq.q {
		return nil, false
	}
	cnt := 0
	var reply *WriteResponse
	for _, r := range replies {
		if aq.wts == r.Timestamp {
			cnt++
			reply = r
		}
	}
	if cnt < aq.q {
		return nil, false
	}
	return reply, true
}
<|endoftext|>


<commit_before>// Package auth handles authentication of Upspin users.
package auth

import (
	"fmt"
	"net/http"
	"net/url"
	"time"

	"upspin.googlesource.com/upspin.git/cloud/netutil"
	"upspin.googlesource.com/upspin.git/upspin"
)

// HTTPClient is a thin wrapper around a standard HTTP Client that implements authentication transparently. It caches state
// so that not every request needs to be signed. HTTPClient is optimized to work with a single server endpoint.
// It will work with any number of servers, but it keeps state about the last one, so using it with many servers will
// decrease its performance.
type HTTPClient struct {
	// Caches the base URL of the last server connected with.
	url *url.URL

	// Records the time we last authenticated with the server.
	// NOTE: this may seem like a premature optimization, but a comment in tls.ConnectionState indicates that
	// resumed connections don't get a TLS unique token, which prevents us from implicitly authenticating the
	// connection. To prevent a round-trip to the server, we preemptly re-auth every AuthIntervalSec
	timeLastAuth time.Time

	// The user we authenticate for.
	user upspin.UserName

	// The user's keys.
	factotum *Factotum

	// The underlying HTTP client
	client netutil.HTTPClientInterface
}

var _ netutil.HTTPClientInterface = (*HTTPClient)(nil)

const (
	// AuthIntervalSec is the maximum allowed time between unauthenticated requests to the same server.
	AuthIntervalSec = 5 * 60 // 5 minutes
)

var (
	errNoUser = &clientError{"no user set"}
	errNoKeys = &clientError{"no keys set"}
)

// NewClient returns a new HTTPClient that handles auth for the named user and underlying HTTP client.
func NewClient(user upspin.UserName, factotum *Factotum, httClient netutil.HTTPClientInterface) *HTTPClient {
	return &HTTPClient{
		user:     user,
		factotum: factotum,
		client:   httClient,
	}
}

// NewAnonymousClient returns a new HTTPClient that does not yet know about the user name.
// To complete setup, use SetUserName and SetUserKeys.
func NewAnonymousClient(httClient netutil.HTTPClientInterface) *HTTPClient {
	return &HTTPClient{
		client: httClient,
	}
}

// SetUserName sets the user name for this HTTPClient instance.
func (c *HTTPClient) SetUserName(user upspin.UserName) {
	c.user = user
}

// SetUserKeys sets the factotum for this HTTPClient instance.
func (c *HTTPClient) SetUserKeys(factotum *Factotum) {
	c.factotum = factotum
}

// Do implements netutil.HTTPClientInterface.
func (c *HTTPClient) Do(req *http.Request) (resp *http.Response, err error) {
	c.prepareRequest(req)
	if req.URL == nil {
		// Let the native client handle this weirdness.
		return c.doWithoutSign(req)
	}
	if req.URL.Scheme != "https" {
		// No point in doing authentication.
		return c.doWithoutSign(req)
	}
	if c.url == nil || c.url.Host != req.URL.Host {
		// Must sign gain.
		return c.doWithSign(req)
	}
	// It's better to avoid a round trip and sign requests that can't be played back.
	if !isReplayable(req) {
		return c.doWithSign(req)
	}
	now := time.Now()
	if c.timeLastAuth.Add(time.Duration(AuthIntervalSec) * time.Second).Before(now) {
		return c.doWithSign(req)
	}
	return c.doWithoutSign(req)
}

// prepareRequest sets the necessary fields for auth on the request, common to both signed and unsigned requests.
func (c *HTTPClient) prepareRequest(req *http.Request) {
	req.Header.Set(userNameHeader, string(c.user)) // Set the username
	req.Header.Set("Date", time.Now().Format(time.RFC850))
}

// isReplayable reports whether the request can be played back to the server safely.
func isReplayable(req *http.Request) bool {
	if req.Body == nil && req.Method != "POST" {
		// A request without payload is always replayable
		return true
	}
	// Note: In general, if the body can seek to the beginning again, it should be replayable. However, Go's HTTP
	// client peeks inside the buffer, making it hard for us to wrap another buffer with an implentation of
	// io.ReadSeeker (it can be resolved, but not without reverse-engineering the native HTTP client, which is a bad idea).
	return false
}

// doWithoutSign does not initially sign the request, but if the request fails with error code 401, we try up to one more
// time with signing, if possible.
func (c *HTTPClient) doWithoutSign(req *http.Request) (*http.Response, error) {
	resp, err := c.client.Do(req)
	if err != nil {
		return resp, newError(err)
	}
	if resp.StatusCode == http.StatusUnauthorized && req.URL.Scheme == "https" {
		if isReplayable(req) {
			return c.doWithSign(req)
		}
	}
	return resp, err
}

// doAuth performs signature authentication and caches the server and time of this last signed request.
func (c *HTTPClient) doWithSign(req *http.Request) (*http.Response, error) {
	if c.user == "" {
		return nil, errNoUser
	}
	if c.factotum.PackingString() == "" {
		return nil, errNoKeys
	}
	err := signRequest(c.user, c.factotum, req)
	if err != nil {
		return nil, newError(err)
	}
	c.url = req.URL
	c.timeLastAuth = time.Now()
	return c.client.Do(req)
}

// signRequest sets the necessary headers in the HTTP request to authenticate a user, by signing the request with the given key.
func signRequest(userName upspin.UserName, factotum *Factotum, req *http.Request) error {
	req.Header.Set(signatureTypeHeader, factotum.PackingString())
	// The hash includes the user name and the key type.
	hash := hashUserRequest(userName, req)
	sig, err := factotum.UserSign(hash)
	if err != nil {
		return err
	}
	req.Header.Set(signatureHeader, fmt.Sprintf("%s %s", sig.R, sig.S))
	return nil
}

type clientError struct {
	errorMsg string
}

// Error implements error
func (c *clientError) Error() string {
	return fmt.Sprintf("HTTPClient: %s", c.errorMsg)
}

func newError(err error) error {
	return &clientError{err.Error()}
}
<commit_msg>auth: add mutex to protect concurrent access.<commit_after>// Package auth handles authentication of Upspin users.
package auth

import (
	"fmt"
	"net/http"
	"net/url"
	"sync"
	"time"

	"upspin.googlesource.com/upspin.git/cloud/netutil"
	"upspin.googlesource.com/upspin.git/upspin"
)

// HTTPClient is a thin wrapper around a standard HTTP Client that implements authentication transparently. It caches state
// so that not every request needs to be signed. HTTPClient is optimized to work with a single server endpoint.
// It will work with any number of servers, but it keeps state about the last one, so using it with many servers will
// decrease its performance.
type HTTPClient struct {
	// Protects all fields from concurrent access, except client, which does not need locking.
	sync.Mutex

	// Caches the base URL of the last server connected with.
	url *url.URL

	// Records the time we last authenticated with the server.
	// NOTE: this may seem like a premature optimization, but a comment in tls.ConnectionState indicates that
	// resumed connections don't get a TLS unique token, which prevents us from implicitly authenticating the
	// connection. To prevent a round-trip to the server, we preemptly re-auth every AuthIntervalSec
	timeLastAuth time.Time

	// The user we authenticate for.
	user upspin.UserName

	// The user's keys.
	factotum *Factotum

	// The underlying HTTP client
	client netutil.HTTPClientInterface
}

var _ netutil.HTTPClientInterface = (*HTTPClient)(nil)

const (
	// AuthIntervalSec is the maximum allowed time between unauthenticated requests to the same server.
	AuthIntervalSec = 5 * 60 // 5 minutes
)

var (
	errNoUser = &clientError{"no user set"}
	errNoKeys = &clientError{"no keys set"}
)

// NewClient returns a new HTTPClient that handles auth for the named user and underlying HTTP client.
func NewClient(user upspin.UserName, factotum *Factotum, httClient netutil.HTTPClientInterface) *HTTPClient {
	return &HTTPClient{
		user:     user,
		factotum: factotum,
		client:   httClient,
	}
}

// NewAnonymousClient returns a new HTTPClient that does not yet know about the user name.
// To complete setup, use SetUserName and SetUserKeys.
func NewAnonymousClient(httClient netutil.HTTPClientInterface) *HTTPClient {
	return &HTTPClient{
		client: httClient,
	}
}

// SetUserName sets the user name for this HTTPClient instance.
func (c *HTTPClient) SetUserName(user upspin.UserName) {
	c.Lock()
	c.user = user
	c.Unlock()
}

// SetUserKeys sets the factotum for this HTTPClient instance.
func (c *HTTPClient) SetUserKeys(factotum *Factotum) {
	c.Lock()
	c.factotum = factotum
	c.Unlock()
}

// Do implements netutil.HTTPClientInterface.
func (c *HTTPClient) Do(req *http.Request) (resp *http.Response, err error) {
	c.prepareRequest(req)
	if req.URL == nil {
		// Let the native client handle this weirdness.
		return c.doWithoutSign(req)
	}
	if req.URL.Scheme != "https" {
		// No point in doing authentication.
		return c.doWithoutSign(req)
	}
	c.Lock() // Will be unlocked by doWithSign.
	if c.url == nil || c.url.Host != req.URL.Host {
		// Must sign gain.
		return c.doWithSign(req)
	}
	// It's better to avoid a round trip and sign requests that can't be played back.
	if !isReplayable(req) {
		return c.doWithSign(req)
	}
	now := time.Now()
	if c.timeLastAuth.Add(time.Duration(AuthIntervalSec) * time.Second).Before(now) {
		return c.doWithSign(req)
	}
	c.Unlock()
	return c.doWithoutSign(req)
}

// prepareRequest sets the necessary fields for auth on the request, common to both signed and unsigned requests.
func (c *HTTPClient) prepareRequest(req *http.Request) {
	req.Header.Set(userNameHeader, string(c.user)) // Set the username
	req.Header.Set("Date", time.Now().Format(time.RFC850))
}

// isReplayable reports whether the request can be played back to the server safely.
func isReplayable(req *http.Request) bool {
	if req.Body == nil && req.Method != "POST" {
		// A request without payload is always replayable
		return true
	}
	// Note: In general, if the body can seek to the beginning again, it should be replayable. However, Go's HTTP
	// client peeks inside the buffer, making it hard for us to wrap another buffer with an implentation of
	// io.ReadSeeker (it can be resolved, but not without reverse-engineering the native HTTP client, which is a bad idea).
	return false
}

// doWithoutSign does not initially sign the request, but if the request fails with error code 401, we try up to one more
// time with signing, if possible. It must be called with the mutex NOT held.
func (c *HTTPClient) doWithoutSign(req *http.Request) (*http.Response, error) {
	resp, err := c.client.Do(req)
	if err != nil {
		return resp, newError(err)
	}
	if resp.StatusCode == http.StatusUnauthorized && req.URL.Scheme == "https" {
		if isReplayable(req) {
			c.Lock()
			return c.doWithSign(req)
		}
	}
	return resp, err
}

// doAuth performs signature authentication and caches the server and time of this last signed request.
// It must be called with the mutex held.
func (c *HTTPClient) doWithSign(req *http.Request) (*http.Response, error) {
	if c.user == "" {
		c.Unlock()
		return nil, errNoUser
	}
	if c.factotum.PackingString() == "" {
		c.Unlock()
		return nil, errNoKeys
	}
	err := signRequest(c.user, c.factotum, req)
	if err != nil {
		c.Unlock()
		return nil, newError(err)
	}
	c.url = req.URL
	c.timeLastAuth = time.Now()
	c.Unlock()
	return c.client.Do(req)
}

// signRequest sets the necessary headers in the HTTP request to authenticate a user, by signing the request with the given key.
func signRequest(userName upspin.UserName, factotum *Factotum, req *http.Request) error {
	req.Header.Set(signatureTypeHeader, factotum.PackingString())
	// The hash includes the user name and the key type.
	hash := hashUserRequest(userName, req)
	sig, err := factotum.UserSign(hash)
	if err != nil {
		return err
	}
	req.Header.Set(signatureHeader, fmt.Sprintf("%s %s", sig.R, sig.S))
	return nil
}

type clientError struct {
	errorMsg string
}

// Error implements error
func (c *clientError) Error() string {
	return fmt.Sprintf("HTTPClient: %s", c.errorMsg)
}

func newError(err error) error {
	return &clientError{err.Error()}
}
<|endoftext|>


<commit_before>package titan

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"time"

	"github.com/dgrijalva/jwt-go"
	"github.com/neptulon/neptulon"
)

type gProfile struct {
	Name    string
	Email   string
	Picture []byte
}

type tokenContainer struct {
	Token string `json:"token"`
}

// googleAuth authenticates a user with Google+ using provided OAuth 2.0 access token.
// If authenticated successfully, user profile is retrieved from Google+ and user is given a JWT token in return.
func googleAuth(ctx *neptulon.ReqCtx, db DB, pass string) error {
	var r tokenContainer
	if err := ctx.Params(&r); err != nil || r.Token == "" {
		ctx.Err = &neptulon.ResError{Code: 666, Message: "Malformed or null Google oauth access token was provided."}
		return fmt.Errorf("middleware: auth: google: malformed or null Google oauth token '%v' was provided: %v", r.Token, err)
	}

	p, err := getTokenInfo(r.Token)
	if err != nil {
		ctx.Err = &neptulon.ResError{Code: 666, Message: "Failed to authenticated with the given Google oauth access token."}
		return fmt.Errorf("middleware: auth: google: error during Google+ profile call using provided access token: %v with error: %v", r.Token, err)
	}

	// retrieve user information
	user, ok := db.GetByMail(p.Email)
	if !ok {
		// this is a first-time registration so create user profile via Google+ profile info
		user = &User{Email: p.Email, Name: p.Name, Picture: p.Picture}

		// save the user information for user ID to be generated by the database
		if err := db.SaveUser(user); err != nil {
			return fmt.Errorf("middleware: auth: google: failed to persist user information: %v", err)
		}

		// create the JWT token
		token := jwt.New(jwt.SigningMethodHS256)
		token.Claims["userid"] = user.ID
		token.Claims["created"] = time.Now().Unix()
		user.JWTToken, err = token.SignedString([]byte(pass))
		if err != nil {
			return fmt.Errorf("middleware: auth: google: jwt signing error: %v", err)
		}

		// now save the full user info
		if err := db.SaveUser(user); err != nil {
			return fmt.Errorf("middleware: auth: google: failed to persist user information: %v", err)
		}
	}

	ctx.Res = tokenContainer{Token: user.JWTToken}
	return nil
}

// ################ Google OAuth2 TokenInfo API Call ################

// verifyIDToken verifies and returns ID token info as described in:
// https://developers.google.com/identity/sign-in/android/backend-auth#send-the-id-token-to-your-server
func getTokenInfo(idToken string) (profile *gProfile, err error) {
	uri := fmt.Sprintf("https://www.googleapis.com/oauth2/v3/tokeninfo?id_token=%s", idToken)
	res, err := http.Get(uri)
	if err != nil || res.StatusCode >= http.StatusBadRequest {
		err = fmt.Errorf("failed to call google oauth2 api with error: %v, and response: %+v", err, res)
		return
	}

	resBody, err := ioutil.ReadAll(res.Body)
	res.Body.Close()
	if err != nil {
		err = fmt.Errorf("failed to read response from google oauth2 api with error: %v", err)
		return
	}

	var ti gTokenInfo
	if err = json.Unmarshal(resBody, &ti); err != nil {
		err = fmt.Errorf("failed to deserialize google oauth2 api response with error: %v", err)
		return
	}

	// check that 'aud' claim contains our client id
	if ti.AUD != gServerClient {
		err = fmt.Errorf("given google oauth2 id token belongs to another app id: %v", ti.AUD)
		return
	}

	// retrieve profile image
	uri = ti.Picture
	res, err = http.Get(uri)
	if err != nil {
		err = fmt.Errorf("failed to call google oauth2 api to get user image with error: %v", err)
		return
	}

	profilePic, err := ioutil.ReadAll(res.Body)
	res.Body.Close()
	if err != nil {
		err = fmt.Errorf("failed to read google oauth2 api user profile pic response with error: %v", err)
		return
	}

	profile = &gProfile{Name: ti.GivenName + " " + ti.FamilyName, Email: ti.Email, Picture: profilePic}
	return
}

var gServerClient = "218602439235-6g09g0ap6i8v25v3rel49rtqjcu9ppj0.apps.googleusercontent.com"

type gTokenInfo struct {
	ISS string
	SUB string
	AZP string
	AUD string
	IAT string
	EXP string

	Email         string
	EmailVerified bool `json:"email_verified"`
	Name          string
	Picture       string // profile pic URL
	GivenName     string `json:"given_name"`
	FamilyName    string `json:"family_name"`
	Locale        string
}

// ################ Google+ API Call ################

// getGPProfile retrieves user info (display name, e-mail, profile pic) using an oauth2 access token that has 'profile' and 'email' scopes.
// Also retrieves user profile image via profile image URL provided the response.
func getGPProfile(oauthToken string) (profile *gProfile, err error) {
	// retrieve profile info from Google
	uri := fmt.Sprintf("https://www.googleapis.com/plus/v1/people/me?access_token=%s", oauthToken)
	res, err := http.Get(uri)
	if err != nil || res.StatusCode >= http.StatusBadRequest {
		err = fmt.Errorf("failed to call google+ api with error: %v, and response: %+v", err, res)
		return
	}

	resBody, err := ioutil.ReadAll(res.Body)
	res.Body.Close()
	if err != nil {
		err = fmt.Errorf("failed to read response from google+ api with error: %v", err)
		return
	}

	var p gPlusProfile
	if err = json.Unmarshal(resBody, &p); err != nil {
		err = fmt.Errorf("failed to deserialize google+ api response with error: %v", err)
		return
	}

	// retrieve profile image
	uri = p.Image.URL
	res, err = http.Get(uri)
	if err != nil {
		err = fmt.Errorf("failed to call google+ api to get user image with error: %v", err)
		return
	}

	profilePic, err := ioutil.ReadAll(res.Body)
	res.Body.Close()
	if err != nil {
		err = fmt.Errorf("failed to read google+ api user profile pic response with error: %v", err)
		return
	}

	profile = &gProfile{Name: p.DisplayName, Email: p.Emails[0].Value, Picture: profilePic}
	return
}

// Response from GET https://www.googleapis.com/plus/v1/people/me?access_token=... (with scope 'profile' and 'email')
// has the following structure with denoted fields of interest (rest is left out):
type gPlusProfile struct {
	Emails      []gPlusEmail
	DisplayName string
	Image       gPlusImage
}

type gPlusEmail struct {
	Value string
}

type gPlusImage struct {
	URL string
}
<commit_msg>auth: google: add more logging<commit_after>package titan

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"time"

	"github.com/dgrijalva/jwt-go"
	"github.com/neptulon/neptulon"
)

type gProfile struct {
	Name    string
	Email   string
	Picture []byte
}

type tokenContainer struct {
	Token string `json:"token"`
}

// googleAuth authenticates a user with Google+ using provided OAuth 2.0 access token.
// If authenticated successfully, user profile is retrieved from Google+ and user is given a JWT token in return.
func googleAuth(ctx *neptulon.ReqCtx, db DB, pass string) error {
	var r tokenContainer
	if err := ctx.Params(&r); err != nil || r.Token == "" {
		ctx.Err = &neptulon.ResError{Code: 666, Message: "Malformed or null Google oauth access token was provided."}
		return fmt.Errorf("middleware: auth: google: malformed or null Google oauth token '%v' was provided: %v", r.Token, err)
	}

	p, err := getTokenInfo(r.Token)
	if err != nil {
		ctx.Err = &neptulon.ResError{Code: 666, Message: "Failed to authenticated with the given Google oauth access token."}
		return fmt.Errorf("middleware: auth: google: error during Google API call using provided token: %v with error: %v", r.Token, err)
	}

	// retrieve user information
	user, ok := db.GetByMail(p.Email)
	if !ok {
		// this is a first-time registration so create user profile via Google+ profile info
		user = &User{Email: p.Email, Name: p.Name, Picture: p.Picture}

		// save the user information for user ID to be generated by the database
		if err := db.SaveUser(user); err != nil {
			return fmt.Errorf("middleware: auth: google: failed to persist user information: %v", err)
		}

		// create the JWT token
		token := jwt.New(jwt.SigningMethodHS256)
		token.Claims["userid"] = user.ID
		token.Claims["created"] = time.Now().Unix()
		user.JWTToken, err = token.SignedString([]byte(pass))
		if err != nil {
			return fmt.Errorf("middleware: auth: google: jwt signing error: %v", err)
		}

		// now save the full user info
		if err := db.SaveUser(user); err != nil {
			return fmt.Errorf("middleware: auth: google: failed to persist user information: %v", err)
		}
	}

	ctx.Res = tokenContainer{Token: user.JWTToken}
	log.Printf("auth: google: logged in: %v, %v", p.Name, p.Email)
	return nil
}

// ################ Google OAuth2 TokenInfo API Call ################

// verifyIDToken verifies and returns ID token info as described in:
// https://developers.google.com/identity/sign-in/android/backend-auth#send-the-id-token-to-your-server
func getTokenInfo(idToken string) (profile *gProfile, err error) {
	uri := fmt.Sprintf("https://www.googleapis.com/oauth2/v3/tokeninfo?id_token=%s", idToken)
	res, err := http.Get(uri)
	if err != nil || res.StatusCode >= http.StatusBadRequest {
		err = fmt.Errorf("failed to call google oauth2 api with error: %v, and response: %+v", err, res)
		return
	}

	resBody, err := ioutil.ReadAll(res.Body)
	res.Body.Close()
	if err != nil {
		err = fmt.Errorf("failed to read response from google oauth2 api with error: %v", err)
		return
	}

	var ti gTokenInfo
	if err = json.Unmarshal(resBody, &ti); err != nil {
		err = fmt.Errorf("failed to deserialize google oauth2 api response with error: %v", err)
		return
	}

	// check that 'aud' claim contains our client id
	if ti.AUD != gServerClient {
		err = fmt.Errorf("given google oauth2 id token belongs to another app id: %v", ti.AUD)
		return
	}

	// retrieve profile image
	uri = ti.Picture
	res, err = http.Get(uri)
	if err != nil {
		err = fmt.Errorf("failed to call google oauth2 api to get user image with error: %v", err)
		return
	}

	profilePic, err := ioutil.ReadAll(res.Body)
	res.Body.Close()
	if err != nil {
		err = fmt.Errorf("failed to read google oauth2 api user profile pic response with error: %v", err)
		return
	}

	profile = &gProfile{Name: ti.GivenName + " " + ti.FamilyName, Email: ti.Email, Picture: profilePic}
	return
}

var gServerClient = "218602439235-6g09g0ap6i8v25v3rel49rtqjcu9ppj0.apps.googleusercontent.com"

type gTokenInfo struct {
	ISS string
	SUB string
	AZP string
	AUD string
	IAT string
	EXP string

	Email         string
	EmailVerified string `json:"email_verified"`
	Name          string
	Picture       string // profile pic URL
	GivenName     string `json:"given_name"`
	FamilyName    string `json:"family_name"`
	Locale        string
}

// ################ Google+ API Call ################

// getGPProfile retrieves user info (display name, e-mail, profile pic) using an oauth2 access token that has 'profile' and 'email' scopes.
// Also retrieves user profile image via profile image URL provided the response.
func getGPProfile(oauthToken string) (profile *gProfile, err error) {
	// retrieve profile info from Google
	uri := fmt.Sprintf("https://www.googleapis.com/plus/v1/people/me?access_token=%s", oauthToken)
	res, err := http.Get(uri)
	if err != nil || res.StatusCode >= http.StatusBadRequest {
		err = fmt.Errorf("failed to call google+ api with error: %v, and response: %+v", err, res)
		return
	}

	resBody, err := ioutil.ReadAll(res.Body)
	res.Body.Close()
	if err != nil {
		err = fmt.Errorf("failed to read response from google+ api with error: %v", err)
		return
	}

	var p gPlusProfile
	if err = json.Unmarshal(resBody, &p); err != nil {
		err = fmt.Errorf("failed to deserialize google+ api response with error: %v", err)
		return
	}

	// retrieve profile image
	uri = p.Image.URL
	res, err = http.Get(uri)
	if err != nil {
		err = fmt.Errorf("failed to call google+ api to get user image with error: %v", err)
		return
	}

	profilePic, err := ioutil.ReadAll(res.Body)
	res.Body.Close()
	if err != nil {
		err = fmt.Errorf("failed to read google+ api user profile pic response with error: %v", err)
		return
	}

	profile = &gProfile{Name: p.DisplayName, Email: p.Emails[0].Value, Picture: profilePic}
	return
}

// Response from GET https://www.googleapis.com/plus/v1/people/me?access_token=... (with scope 'profile' and 'email')
// has the following structure with denoted fields of interest (rest is left out):
type gPlusProfile struct {
	Emails      []gPlusEmail
	DisplayName string
	Image       gPlusImage
}

type gPlusEmail struct {
	Value string
}

type gPlusImage struct {
	URL string
}
<|endoftext|>


<commit_before>// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// A CLI application to monitor one or more stocks.
package main

import (
	"encoding/json"
	"errors"
	"flag"
	"fmt"
	"log"
	"net/http"
	"os"
	"os/exec"
	"regexp"
	"sort"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/mattn/go-runewidth"
	"github.com/nsf/termbox-go"
)

const TIMEOUT = time.Duration(time.Second * 10)
const URL = "http://finance.yahoo.com/webservice/v1/symbols/%s/quote?format=json"
const UP = "↑"   // rune 8593
const DOWN = "↓" // rune 8595

var re = regexp.MustCompile(`^\d.+\.\d{2}`) // this is to have only 2 decimal places
var signalChan = make(chan os.Signal, 1)    // channel to catch ctrl-c

// Flag variables to hold CLI arguments
var (
	symbolFlag   = flag.String("s", "", "Symbols for ticker, comma seperate (no spaces)")
	intervalFlag = flag.Int("i", 1, "Interval for stock data to be updated in seconds")
)

type Stock struct {
	List List `json:"list"`
}

type List struct {
	Meta      Meta        `json:"meta"`
	Resources []Resources `json:"resources"`
}

type Meta struct {
	Type  string `json:"type"`
	Start uint   `json:"start"`
	Count uint   `json:"count"`
}

type Resources struct {
	Resource Resource `json:"resource"`
}

type Resource struct {
	Classname string `json:"classname"`
	Fields    Fields `json:"fields"`
}

// Fields holds all of the retrieved data from the API call
type Fields struct {
	Name    string `json:"name"`    // name of company
	Price   string `json:"price"`   // current price
	Symbol  string `json:"symbol"`  // stock symbol
	TS      string `json:"ts"`      //
	Type    string `json:"type"`    // type of stock (equity, etc...)
	UTCTime string `json:"utctime"` // time in UTC
	Volume  string `json:"volume"`  // shares traded
}

// stockwatcher holds the relevant data for the running instance
type stockwatcher struct {
	quotes   map[string]map[string]float64
	interval time.Duration
	m        *sync.Mutex
}

// NewStockWatcher returns a new instance of stockwatcher with the given parameters
func NewStockWatcher(i time.Duration) *stockwatcher {
	return &stockwatcher{
		quotes:   make(map[string]map[string]float64),
		interval: i,
		m:        &sync.Mutex{},
	}
}

// add takes the given symbol and populates a key in the quotes map
func (t *stockwatcher) add(symbol string) {
	t.m.Lock()
	defer t.m.Unlock()
	if _, ok := t.quotes[symbol]; !ok {
		t.quotes[symbol] = map[string]float64{}
	}
}

// updateStock populates stockwatcher struct with stock prices
func (t *stockwatcher) updateStock(symbol string, price float64) {
	t.m.Lock()
	defer t.m.Unlock()
	t.quotes[symbol] = map[string]float64{
		"previous": t.quotes[symbol]["current"],
		"current":  price,
	}
}

// query will retrieve data for a given symbol
// TODO(briandowns) Instead of dying on failure, set the symbol data to nil values
func query(symbol string) (*Stock, error) {
	data := &Stock{}
	client := http.Client{
		Timeout: TIMEOUT,
	}

	resp, err := client.Get(fmt.Sprintf(URL, symbol))
	if err != nil {
		return nil, errors.New("unable to retrive symbol data")
	}
	defer resp.Body.Close()

	if err := json.NewDecoder(resp.Body).Decode(&data); err != nil {
		log.Fatalln(err)
		//os.Exit(1)
	}
	return data, nil
}

// convertPrice converts the given string to a float64 value
func convertPrice(p string) float64 {
	price, err := strconv.ParseFloat(p, 64)
	if err != nil {
		log.Fatalln(err)
		os.Exit(1)
	}
	return price
}

// runner goes through and gets the data for each symbol
func (t *stockwatcher) runner() {
	var wg sync.WaitGroup
	for k, _ := range t.quotes {
		wg.Add(1)
		go func(k string) {
			defer wg.Done()
			stock, err := query(k)
			if err != nil { // if we can't get a response from the API, put 0.00 in and keep going
				t.updateStock(k, 0.00)
				return
			}
			t.updateStock(stock.List.Resources[0].Resource.Fields.Symbol,
				convertPrice(re.FindString(stock.List.Resources[0].Resource.Fields.Price)),
			)
		}(k)
	}
	wg.Wait()
}

// formatData formats the given data for printing
func (t *stockwatcher) formatData() {
	// populate list with keys from quote map
	var keys []string
	for k := range t.quotes {
		keys = append(keys, k)
	}
	sort.Strings(keys) // alphabetize keys

	pos := 1
	for _, k := range keys {
		// print format for first run or if not change detected from previous run
		if t.quotes[k]["previous"] == 0.00 || t.quotes[k]["previous"] == t.quotes[k]["current"] {
			printTb(1,
				pos,
				fmt.Sprintf("%-6s %-7v %11s %-4s\n", k, t.quotes[k]["current"], "%", "-"),
				termbox.ColorWhite, termbox.ColorDefault,
			)
			pos++
			// print format in green if current price being is greater than previous price
		} else if t.quotes[k]["current"] > t.quotes[k]["previous"] {
			printTb(1,
				pos,
				fmt.Sprintf("%-6s %-7v +%-.6f %% %-4s\n", k, t.quotes[k]["current"], t.quotes[k]["current"]/t.quotes[k]["previous"], UP),
				termbox.ColorGreen,
				termbox.ColorDefault,
			)
			pos++
			// print format in red if current price being is lesser than previous price
		} else {
			printTb(1,
				pos,
				fmt.Sprintf("%-6s %-7v -%-.6f %% %-4s\n", k, t.quotes[k]["current"], t.quotes[k]["current"]/t.quotes[k]["previous"], DOWN),
				termbox.ColorRed,
				termbox.ColorDefault,
			)
			pos++
		}
	}
}

// printTb prints the given data out to the screen
func printTb(x, y int, msg string, fg, bg termbox.Attribute) {
	for _, c := range []rune(msg) {
		termbox.SetCell(x, y, c, fg, bg)
		x += runewidth.RuneWidth(c)
	}
	termbox.Flush()
}

func main() {
	flag.Parse()

	// make sure we got what was expected from the CLI
	if flag.NFlag() != 2 || *symbolFlag == "" {
		flag.Usage()
		os.Exit(1)
	}

	t := NewStockWatcher(time.Duration(*intervalFlag) * time.Second)

	// check if more than one symbol has been given
	switch {
	case strings.Contains(*symbolFlag, ","):
		for _, a := range strings.Split(*symbolFlag, ",") {
			t.add(a)
		}
	default:
		t.add(*symbolFlag)
	}

	// initialize termbox
	err := termbox.Init()
	if err != nil {
		log.Fatal(err)
	}
	termbox.Clear(termbox.ColorDefault, termbox.ColorDefault)

	event := make(chan termbox.Event)
	go func() {
		for {
			// Post events to channel
			event <- termbox.PollEvent()
		}
	}()

loop:
	for {
		t.runner()
		t.formatData()

		// Poll key event or timeout (maybe)
		select {
		case <-event:
			break loop
		case <-time.After(t.interval):
			continue loop
		}
	}
	close(event)
	time.Sleep(1 * time.Second)
	termbox.Close()
	os.Exit(0)
}
<commit_msg>add some more comments<commit_after>// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// A CLI application to monitor one or more stocks.
package main

import (
	"encoding/json"
	"errors"
	"flag"
	"fmt"
	"log"
	"net/http"
	"os"
	"os/exec"
	"regexp"
	"sort"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/mattn/go-runewidth"
	"github.com/nsf/termbox-go"
)

const TIMEOUT = time.Duration(time.Second * 10)
const URL = "http://finance.yahoo.com/webservice/v1/symbols/%s/quote?format=json"
const UP = "↑"   // rune 8593
const DOWN = "↓" // rune 8595

var re = regexp.MustCompile(`^\d.+\.\d{2}`) // this is to have only 2 decimal places
var signalChan = make(chan os.Signal, 1)    // channel to catch ctrl-c

// Flag variables to hold CLI arguments
var (
	symbolFlag   = flag.String("s", "", "Symbols for ticker, comma seperate (no spaces)")
	intervalFlag = flag.Int("i", 1, "Interval for stock data to be updated in seconds")
)

// Stock is the top level of the returned JSON
type Stock struct {
	List List `json:"list"`
}

// List hold the metadata and list of returned symbol data
type List struct {
	Meta      Meta        `json:"meta"`
	Resources []Resources `json:"resources"`
}

// Meta is the calls metadata
type Meta struct {
	Type  string `json:"type"`
	Start uint   `json:"start"`
	Count uint   `json:"count"`
}

// Resources holds a JSON obj with the symbol data
type Resources struct {
	Resource Resource `json:"resource"`
}

// Resource contains the actual JSON obj with the symbol data
type Resource struct {
	Classname string `json:"classname"`
	Fields    Fields `json:"fields"`
}

// Fields holds all of the retrieved data from the API call
type Fields struct {
	Name    string `json:"name"`    // name of company
	Price   string `json:"price"`   // current price
	Symbol  string `json:"symbol"`  // stock symbol
	TS      string `json:"ts"`      //
	Type    string `json:"type"`    // type of stock (equity, etc...)
	UTCTime string `json:"utctime"` // time in UTC
	Volume  string `json:"volume"`  // shares traded
}

// stockwatcher holds the relevant data for the running instance
type stockwatcher struct {
	quotes   map[string]map[string]float64
	interval time.Duration
	m        *sync.Mutex
}

// NewStockWatcher returns a new instance of stockwatcher with the given parameters
func NewStockWatcher(i time.Duration) *stockwatcher {
	return &stockwatcher{
		quotes:   make(map[string]map[string]float64),
		interval: i,
		m:        &sync.Mutex{},
	}
}

// add takes the given symbol and populates a key in the quotes map
func (t *stockwatcher) add(symbol string) {
	t.m.Lock()
	defer t.m.Unlock()
	if _, ok := t.quotes[symbol]; !ok {
		t.quotes[symbol] = map[string]float64{}
	}
}

// updateStock populates stockwatcher struct with stock prices
func (t *stockwatcher) updateStock(symbol string, price float64) {
	t.m.Lock()
	defer t.m.Unlock()
	t.quotes[symbol] = map[string]float64{
		"previous": t.quotes[symbol]["current"],
		"current":  price,
	}
}

// query will retrieve data for a given symbol
// TODO(briandowns) Instead of dying on failure, set the symbol data to nil values
func query(symbol string) (*Stock, error) {
	data := &Stock{}
	client := http.Client{
		Timeout: TIMEOUT,
	}

	resp, err := client.Get(fmt.Sprintf(URL, symbol))
	if err != nil {
		return nil, errors.New("unable to retrive symbol data")
	}
	defer resp.Body.Close()

	if err := json.NewDecoder(resp.Body).Decode(&data); err != nil {
		log.Fatalln(err)
		//os.Exit(1)
	}
	return data, nil
}

// convertPrice converts the given string to a float64 value
func convertPrice(p string) float64 {
	price, err := strconv.ParseFloat(p, 64)
	if err != nil {
		log.Fatalln(err)
		os.Exit(1)
	}
	return price
}

// runner goes through and gets the data for each symbol
func (t *stockwatcher) runner() {
	var wg sync.WaitGroup
	for k, _ := range t.quotes {
		wg.Add(1)
		go func(k string) {
			defer wg.Done()
			stock, err := query(k)
			if err != nil { // if we can't get a response from the API, put 0.00 in and keep going
				t.updateStock(k, 0.00)
				return
			}
			t.updateStock(stock.List.Resources[0].Resource.Fields.Symbol,
				convertPrice(re.FindString(stock.List.Resources[0].Resource.Fields.Price)),
			)
		}(k)
	}
	wg.Wait()
}

// formatData formats the given data for printing
func (t *stockwatcher) formatData() {
	// populate list with keys from quote map
	var keys []string
	for k := range t.quotes {
		keys = append(keys, k)
	}
	sort.Strings(keys) // alphabetize keys

	pos := 1
	for _, k := range keys {
		// print format for first run or if not change detected from previous run
		if t.quotes[k]["previous"] == 0.00 || t.quotes[k]["previous"] == t.quotes[k]["current"] {
			printTb(1,
				pos,
				fmt.Sprintf("%-6s %-7v %11s %-4s\n", k, t.quotes[k]["current"], "%", "-"),
				termbox.ColorWhite, termbox.ColorDefault,
			)
			pos++
			// print format in green if current price being is greater than previous price
		} else if t.quotes[k]["current"] > t.quotes[k]["previous"] {
			printTb(1,
				pos,
				fmt.Sprintf("%-6s %-7v +%-.6f %% %-4s\n", k, t.quotes[k]["current"], t.quotes[k]["current"]/t.quotes[k]["previous"], UP),
				termbox.ColorGreen,
				termbox.ColorDefault,
			)
			pos++
			// print format in red if current price being is lesser than previous price
		} else {
			printTb(1,
				pos,
				fmt.Sprintf("%-6s %-7v -%-.6f %% %-4s\n", k, t.quotes[k]["current"], t.quotes[k]["current"]/t.quotes[k]["previous"], DOWN),
				termbox.ColorRed,
				termbox.ColorDefault,
			)
			pos++
		}
	}
}

// printTb prints the given data out to the screen
func printTb(x, y int, msg string, fg, bg termbox.Attribute) {
	for _, c := range []rune(msg) {
		termbox.SetCell(x, y, c, fg, bg)
		x += runewidth.RuneWidth(c)
	}
	termbox.Flush()
}

func main() {
	flag.Parse()

	// make sure we got what was expected from the CLI
	if flag.NFlag() != 2 || *symbolFlag == "" {
		flag.Usage()
		os.Exit(1)
	}

	t := NewStockWatcher(time.Duration(*intervalFlag) * time.Second)

	// check if more than one symbol has been given
	switch {
	case strings.Contains(*symbolFlag, ","):
		for _, a := range strings.Split(*symbolFlag, ",") {
			t.add(a)
		}
	default:
		t.add(*symbolFlag)
	}

	// initialize termbox
	err := termbox.Init()
	if err != nil {
		log.Fatal(err)
	}
	termbox.Clear(termbox.ColorDefault, termbox.ColorDefault)

	event := make(chan termbox.Event)
	go func() {
		for {
			// Post events to channel
			event <- termbox.PollEvent()
		}
	}()

loop:
	for {
		t.runner()
		t.formatData()

		// Poll key event or timeout (maybe)
		select {
		case <-event:
			break loop
		case <-time.After(t.interval):
			continue loop
		}
	}
	close(event)
	time.Sleep(1 * time.Second)
	termbox.Close()
	os.Exit(0)
}
<|endoftext|>


<commit_before>package persistence

var Expressions = map[string]string{
	"CreditCard":           "(?:4[0-9]{12}(?:[0-9]{3})?|5[1-5][0-9]{14}|6011[0-9]{12}|622((12[6-9]|1[3-9][0-9])|([2-8][0-9][0-9])|(9(([0-1][0-9])|(2[0-5]))))[0-9]{10}|64[4-9][0-9]{13}|65[0-9]{14}|3(?:0[0-5]|[68][0-9])[0-9]{11}|3[47][0-9]{13})+",
	"NorthAmericanPhone":   "((([0-9]{1})*[- .(]*([0-9]{3})[- .)]*[0-9]{3}[- .]*[0-9]{4})+)+",
	"SocialSecurityNumber": "([0-9]{3}[-]*[0-9]{2}[-]*[0-9]{4})+",
}
<commit_msg>refactor: Add extra validation for regular expressions<commit_after>package persistence

var Expressions = map[string]string{
	"CreditCard":           `\b(?:4[0-9]{12}(?:[0-9]{3})?|5[1-5][0-9]{14}|6011[0-9]{12}|622((12[6-9]|1[3-9][0-9])|([2-8][0-9][0-9])|(9(([0-1][0-9])|(2[0-5]))))[0-9]{10}|64[4-9][0-9]{13}|65[0-9]{14}|3(?:0[0-5]|[68][0-9])[0-9]{11}|3[47][0-9]{13})+\b`,
	"NorthAmericanPhone":   `\b((([0-9]{1})*[- .(]*([0-9]{3})[- .)]*[0-9]{3}[- .]*[0-9]{4})+)+\b`,
	"SocialSecurityNumber": `\b([0-9]{3}[-]*[0-9]{2}[-]*[0-9]{4})+\b`,
}
<|endoftext|>


<commit_before>package main

import (
	"common"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"strings"
	"time"
)

type orchestrator struct {
	repoip      chan string
	deploystate chan map[string]common.DockerInfo
	addip       chan string
}

func (o *orchestrator) pollDocker(ip string, update chan common.DockerInfo) {
	for ; ; time.Sleep(60 * time.Second) {
		c, err := common.ListContainers(ip)
		if err != nil {
			log.Print(err)
			continue
		}
		d := common.DockerInfo{ip, c, nil, time.Now()}
		update <- d
	}
}

func (o *orchestrator) StartState() {
	d := make(map[string]common.DockerInfo)
	o.deploystate = make(chan map[string]common.DockerInfo)
	o.addip = make(chan string)
	updatechan := make(chan common.DockerInfo)
	for {
		select {
		case o.deploystate <- d:

		case ip := <-o.addip:
			_, exist := d[ip]
			if !exist {
				d[ip] = common.DockerInfo{}
			}
			go o.pollDocker(ip, updatechan)

		case up := <-updatechan:
			d[up.Ip] = up
		}
	}
}

func (o *orchestrator) WaitRefresh(t time.Time) {
	for {
		s := <-o.deploystate
		good := false
		for _, v := range s {
			if v.Updated.After(t) {
				good = true
				break

			}
		}
		if good {
			break
		}
		time.Sleep(10 * time.Second)
	}
}

func (o *orchestrator) StartRepository() {
	log.Print("index setup")
	registry_name := "samalba/docker-registry"
	host := os.Getenv("HOST")
	// So that id is passed out of the function
	id := ""
	var err error
	var running bool
	for ; ; time.Sleep(10 * time.Second) {
		running, id, err = common.ImageRunning(host, registry_name)
		if err != nil {
			log.Print(err)
			continue
		}
		if !running {
			log.Print("index not running")
			err := common.LoadImage(host, registry_name)
			if err != nil {
				log.Print(err)
				continue
			}
			id, err = common.RunImage(host, registry_name, false)
			if err != nil {
				log.Print(err)
				continue
			}
		}
		break
	}
	log.Print("index running id: ", id)
	config, err := common.InspectContainer(host, id)
	log.Print("fetched config")
	ip := config.NetworkSettings.PortMapping.Tcp["5000"]

	host = host + ":" + ip

	if err != nil {
		log.Print(err)
	}

	o.repoip = make(chan string)
	for {
		o.repoip <- host
	}

}

func (o *orchestrator) handleImage(w http.ResponseWriter, r *http.Request) {
	if o.repoip == nil {
		io.WriteString(w, "Waiting for index to be downloaded, this may take a while")
	}
	repoip := <-o.repoip
	io.WriteString(w, "Recieved\n")
	tag := r.URL.Query()["name"]
	if len(tag) > 0 {
		io.WriteString(w, "Building image\n")
		err := common.BuildImage(os.Getenv("HOST"), r.Body, tag[0])
		if err != nil {
			io.WriteString(w, err.Error()+"\n")
			return
		}

		io.WriteString(w, "Tagging\n")
		repo_tag := repoip + "/" + tag[0]
		err = common.TagImage(os.Getenv("HOST"), tag[0], repo_tag)
		if err != nil {
			io.WriteString(w, err.Error())
			return
		}
		io.WriteString(w, "Pushing to index\n")
		err = common.PushImage(os.Getenv("HOST"), w, repo_tag)
		if err != nil {
			io.WriteString(w, err.Error())
			return
		}
	}
	io.WriteString(w, "built\n")
}

func (o *orchestrator) calcUpdate(w io.Writer, desired common.SkeletonDeployment, current map[string]common.DockerInfo) (update map[string][]string) {
	c := fmt.Sprint(current)
	io.WriteString(w, c)
	io.WriteString(w, "\n")
	// Maps IP's to lists of containers to deploy
	update = make(map[string][]string)
	// For each container we want to deploy
	for container, _ := range desired.Containers {
		// Assuming granularity machine

		// For each machine check for container
		for ip, mInfo := range current {

			//Have we found the container
			found := false

			//Check if the container is running
			for _, checkContainer := range mInfo.Containers {

				imageName := checkContainer.Image

				//Get the actual name
				io.WriteString(w, "Image name before proc: ")
				io.WriteString(w, imageName)
				io.WriteString(w, "\n")
				if strings.Contains(imageName, "/") {
					imageName = strings.SplitN(imageName, "/", 2)[1]
				}
				if strings.Contains(imageName, ":") {
					imageName = strings.SplitN(imageName, ":", 2)[0]
				}
				io.WriteString(w, "Image name after proc: ")
				io.WriteString(w, imageName)
				io.WriteString(w, "\n")

				if imageName == container {
					found = true
					break
				}
			}

			//Do we need to deploy a image?
			if !found {
				update[ip] = append(update[ip], container)
			}

		}
	}

	return update

}

func (o *orchestrator) deploy(w http.ResponseWriter, r *http.Request) {

	io.WriteString(w, "Starting deploy\n")
	d := &common.SkeletonDeployment{}
	c, err := ioutil.ReadAll(r.Body)

	if err != nil {
		io.WriteString(w, err.Error())
		return
	}
	err = json.Unmarshal(c, d)
	if err != nil {
		io.WriteString(w, err.Error())
		return
	}

	for _, ip := range d.Machines.Ip {
		io.WriteString(w, "Adding ip\n")
		io.WriteString(w, ip)
		io.WriteString(w, "\n")
		o.addip <- ip
	}

	io.WriteString(w, "Waiting for image refreshes\n")
	o.WaitRefresh(time.Now())
	io.WriteString(w, "waited\n")

	current := <-o.deploystate

	diff := o.calcUpdate(w, *d, current)

	sdiff := fmt.Sprint(diff)
	io.WriteString(w, sdiff)
	io.WriteString(w, "\n")

	indexip := <-o.repoip

	io.WriteString(w, "Deploying diff\n")
	for ip, images := range diff {
		for _, container := range images {
			io.WriteString(w, "Deploying "+container+" on "+ip+"\n")
			err := common.LoadImage(ip, indexip+"/"+container)
			if err != nil {
				io.WriteString(w, err.Error())
				continue
			}
			id, err := common.RunImage(ip, indexip+"/"+container, false)
			io.WriteString(w, "Deployed \n")
			io.WriteString(w, id)
			io.WriteString(w, "\n")
			if err != nil {
				io.WriteString(w, err.Error())
			}
			io.WriteString(w, "\n")
		}
	}
}

func NewOrchestrator() (o *orchestrator) {
	o = new(orchestrator)
	go o.StartRepository()
	go o.StartState()
	return o
}

func main() {

	o := NewOrchestrator()

	http.HandleFunc("/version", func(w http.ResponseWriter, r *http.Request) {
		io.WriteString(w, "orchestrator v0")
	})

	http.HandleFunc("/image", o.handleImage)

	http.HandleFunc("/deploy", o.deploy)

	log.Fatal(http.ListenAndServe(":900", nil))
}
<commit_msg>orchestrator: Don't block badly on initial deploy<commit_after>package main

import (
	"common"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"strings"
	"time"
)

type orchestrator struct {
	repoip      chan string
	deploystate chan map[string]common.DockerInfo
	addip       chan string
}

func (o *orchestrator) pollDocker(ip string, update chan common.DockerInfo) {
	for ; ; time.Sleep(60 * time.Second) {
		c, err := common.ListContainers(ip)
		if err != nil {
			log.Print(err)
			continue
		}
		d := common.DockerInfo{ip, c, nil, time.Now()}
		update <- d
	}
}

func (o *orchestrator) StartState() {
	d := make(map[string]common.DockerInfo)
	o.deploystate = make(chan map[string]common.DockerInfo)
	o.addip = make(chan string)
	updatechan := make(chan common.DockerInfo)
	for {
		select {
		case o.deploystate <- d:

		case ip := <-o.addip:
			_, exist := d[ip]
			if !exist {
				d[ip] = common.DockerInfo{}
			}
			go o.pollDocker(ip, updatechan)

		case up := <-updatechan:
			d[up.Ip] = up
		}
	}
}

func (o *orchestrator) WaitRefresh(t time.Time) {
	for {
		s := <-o.deploystate
		good := false
		for _, v := range s {
			if v.Updated.After(t) {
				good = true
				break

			}
		}
		if good {
			break
		}
		time.Sleep(10 * time.Second)
	}
}

func (o *orchestrator) StartRepository() {
	log.Print("index setup")
	registry_name := "samalba/docker-registry"
	host := os.Getenv("HOST")
	// So that id is passed out of the function
	id := ""
	var err error
	var running bool
	for ; ; time.Sleep(10 * time.Second) {
		running, id, err = common.ImageRunning(host, registry_name)
		if err != nil {
			log.Print(err)
			continue
		}
		if !running {
			log.Print("index not running")
			err := common.LoadImage(host, registry_name)
			if err != nil {
				log.Print(err)
				continue
			}
			id, err = common.RunImage(host, registry_name, false)
			if err != nil {
				log.Print(err)
				continue
			}
		}
		break
	}
	log.Print("index running id: ", id)
	config, err := common.InspectContainer(host, id)
	log.Print("fetched config")
	ip := config.NetworkSettings.PortMapping.Tcp["5000"]

	host = host + ":" + ip

	if err != nil {
		log.Print(err)
	}

	for {
		o.repoip <- host
	}

}

func (o *orchestrator) handleImage(w http.ResponseWriter, r *http.Request) {
	io.WriteString(w, "Waiting for index to be downloaded, this may take a while")
	repoip := <-o.repoip
	io.WriteString(w, "Recieved\n")
	tag := r.URL.Query()["name"]
	if len(tag) > 0 {
		io.WriteString(w, "Building image\n")
		err := common.BuildImage(os.Getenv("HOST"), r.Body, tag[0])
		if err != nil {
			io.WriteString(w, err.Error()+"\n")
			return
		}

		io.WriteString(w, "Tagging\n")
		repo_tag := repoip + "/" + tag[0]
		err = common.TagImage(os.Getenv("HOST"), tag[0], repo_tag)
		if err != nil {
			io.WriteString(w, err.Error())
			return
		}
		io.WriteString(w, "Pushing to index\n")
		err = common.PushImage(os.Getenv("HOST"), w, repo_tag)
		if err != nil {
			io.WriteString(w, err.Error())
			return
		}
	}
	io.WriteString(w, "built\n")
}

func (o *orchestrator) calcUpdate(w io.Writer, desired common.SkeletonDeployment, current map[string]common.DockerInfo) (update map[string][]string) {
	c := fmt.Sprint(current)
	io.WriteString(w, c)
	io.WriteString(w, "\n")
	// Maps IP's to lists of containers to deploy
	update = make(map[string][]string)
	// For each container we want to deploy
	for container, _ := range desired.Containers {
		// Assuming granularity machine

		// For each machine check for container
		for ip, mInfo := range current {

			//Have we found the container
			found := false

			//Check if the container is running
			for _, checkContainer := range mInfo.Containers {

				imageName := checkContainer.Image

				//Get the actual name
				io.WriteString(w, "Image name before proc: ")
				io.WriteString(w, imageName)
				io.WriteString(w, "\n")
				if strings.Contains(imageName, "/") {
					imageName = strings.SplitN(imageName, "/", 2)[1]
				}
				if strings.Contains(imageName, ":") {
					imageName = strings.SplitN(imageName, ":", 2)[0]
				}
				io.WriteString(w, "Image name after proc: ")
				io.WriteString(w, imageName)
				io.WriteString(w, "\n")

				if imageName == container {
					found = true
					break
				}
			}

			//Do we need to deploy a image?
			if !found {
				update[ip] = append(update[ip], container)
			}

		}
	}

	return update

}

func (o *orchestrator) deploy(w http.ResponseWriter, r *http.Request) {

	io.WriteString(w, "Starting deploy\n")
	d := &common.SkeletonDeployment{}
	c, err := ioutil.ReadAll(r.Body)

	if err != nil {
		io.WriteString(w, err.Error())
		return
	}
	err = json.Unmarshal(c, d)
	if err != nil {
		io.WriteString(w, err.Error())
		return
	}

	for _, ip := range d.Machines.Ip {
		io.WriteString(w, "Adding ip\n")
		io.WriteString(w, ip)
		io.WriteString(w, "\n")
		o.addip <- ip
	}

	io.WriteString(w, "Waiting for image refreshes\n")
	o.WaitRefresh(time.Now())
	io.WriteString(w, "waited\n")

	current := <-o.deploystate

	diff := o.calcUpdate(w, *d, current)

	sdiff := fmt.Sprint(diff)
	io.WriteString(w, sdiff)
	io.WriteString(w, "\n")

	indexip := <-o.repoip

	io.WriteString(w, "Deploying diff\n")
	for ip, images := range diff {
		for _, container := range images {
			io.WriteString(w, "Deploying "+container+" on "+ip+"\n")
			err := common.LoadImage(ip, indexip+"/"+container)
			if err != nil {
				io.WriteString(w, err.Error())
				continue
			}
			id, err := common.RunImage(ip, indexip+"/"+container, false)
			io.WriteString(w, "Deployed \n")
			io.WriteString(w, id)
			io.WriteString(w, "\n")
			if err != nil {
				io.WriteString(w, err.Error())
			}
			io.WriteString(w, "\n")
		}
	}
}

func NewOrchestrator() (o *orchestrator) {
	o = new(orchestrator)
	o.repoip = make(chan string)
	go o.StartState()
	go o.StartRepository()
	return o
}

func main() {

	o := NewOrchestrator()

	http.HandleFunc("/version", func(w http.ResponseWriter, r *http.Request) {
		io.WriteString(w, "orchestrator v0")
	})

	http.HandleFunc("/image", o.handleImage)

	http.HandleFunc("/deploy", o.deploy)

	log.Fatal(http.ListenAndServe(":900", nil))
}
<|endoftext|>


<commit_before>// Copyright 2011 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package html

import (
	"exp/html/atom"
)

// A NodeType is the type of a Node.
type NodeType uint32

const (
	ErrorNode NodeType = iota
	TextNode
	DocumentNode
	ElementNode
	CommentNode
	DoctypeNode
	scopeMarkerNode
)

// Section 12.2.3.3 says "scope markers are inserted when entering applet
// elements, buttons, object elements, marquees, table cells, and table
// captions, and are used to prevent formatting from 'leaking'".
var scopeMarker = Node{Type: scopeMarkerNode}

// A Node consists of a NodeType and some Data (tag name for element nodes,
// content for text) and are part of a tree of Nodes. Element nodes may also
// have a Namespace and contain a slice of Attributes. Data is unescaped, so
// that it looks like "a<b" rather than "a<b". For element nodes, DataAtom
// is the atom for Data, or zero if Data is not a known tag name.
//
// An empty Namespace implies a "http://www.w3.org/1999/xhtml" namespace.
// Similarly, "math" is short for "http://www.w3.org/1998/Math/MathML", and
// "svg" is short for "http://www.w3.org/2000/svg".
type Node struct {
	Parent    *Node
	Child     []*Node
	Type      NodeType
	DataAtom  atom.Atom
	Data      string
	Namespace string
	Attr      []Attribute
}

// Add adds a node as a child of n.
// It will panic if the child's parent is not nil.
func (n *Node) Add(child *Node) {
	if child.Parent != nil {
		panic("html: Node.Add called for a child Node that already has a parent")
	}
	child.Parent = n
	n.Child = append(n.Child, child)
}

// Remove removes a node as a child of n.
// It will panic if the child's parent is not n.
func (n *Node) Remove(child *Node) {
	if child.Parent == n {
		child.Parent = nil
		for i, m := range n.Child {
			if m == child {
				copy(n.Child[i:], n.Child[i+1:])
				j := len(n.Child) - 1
				n.Child[j] = nil
				n.Child = n.Child[:j]
				return
			}
		}
	}
	panic("html: Node.Remove called for a non-child Node")
}

// reparentChildren reparents all of src's child nodes to dst.
func reparentChildren(dst, src *Node) {
	for _, n := range src.Child {
		if n.Parent != src {
			panic("html: nodes have an inconsistent parent/child relationship")
		}
		n.Parent = dst
	}
	dst.Child = append(dst.Child, src.Child...)
	src.Child = nil
}

// clone returns a new node with the same type, data and attributes.
// The clone has no parent and no children.
func (n *Node) clone() *Node {
	m := &Node{
		Type:     n.Type,
		DataAtom: n.DataAtom,
		Data:     n.Data,
		Attr:     make([]Attribute, len(n.Attr)),
	}
	copy(m.Attr, n.Attr)
	return m
}

// nodeStack is a stack of nodes.
type nodeStack []*Node

// pop pops the stack. It will panic if s is empty.
func (s *nodeStack) pop() *Node {
	i := len(*s)
	n := (*s)[i-1]
	*s = (*s)[:i-1]
	return n
}

// top returns the most recently pushed node, or nil if s is empty.
func (s *nodeStack) top() *Node {
	if i := len(*s); i > 0 {
		return (*s)[i-1]
	}
	return nil
}

// index returns the index of the top-most occurrence of n in the stack, or -1
// if n is not present.
func (s *nodeStack) index(n *Node) int {
	for i := len(*s) - 1; i >= 0; i-- {
		if (*s)[i] == n {
			return i
		}
	}
	return -1
}

// insert inserts a node at the given index.
func (s *nodeStack) insert(i int, n *Node) {
	(*s) = append(*s, nil)
	copy((*s)[i+1:], (*s)[i:])
	(*s)[i] = n
}

// remove removes a node from the stack. It is a no-op if n is not present.
func (s *nodeStack) remove(n *Node) {
	i := s.index(n)
	if i == -1 {
		return
	}
	copy((*s)[i:], (*s)[i+1:])
	j := len(*s) - 1
	(*s)[j] = nil
	*s = (*s)[:j]
}

// TODO(nigeltao): forTag no longer used. Should it be deleted?

// forTag returns the top-most element node with the given tag.
func (s *nodeStack) forTag(tag string) *Node {
	for i := len(*s) - 1; i >= 0; i-- {
		n := (*s)[i]
		if n.Type == ElementNode && n.Data == tag {
			return n
		}
	}
	return nil
}
<commit_msg>exp/html: remove unused forTag function.<commit_after>// Copyright 2011 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package html

import (
	"exp/html/atom"
)

// A NodeType is the type of a Node.
type NodeType uint32

const (
	ErrorNode NodeType = iota
	TextNode
	DocumentNode
	ElementNode
	CommentNode
	DoctypeNode
	scopeMarkerNode
)

// Section 12.2.3.3 says "scope markers are inserted when entering applet
// elements, buttons, object elements, marquees, table cells, and table
// captions, and are used to prevent formatting from 'leaking'".
var scopeMarker = Node{Type: scopeMarkerNode}

// A Node consists of a NodeType and some Data (tag name for element nodes,
// content for text) and are part of a tree of Nodes. Element nodes may also
// have a Namespace and contain a slice of Attributes. Data is unescaped, so
// that it looks like "a<b" rather than "a<b". For element nodes, DataAtom
// is the atom for Data, or zero if Data is not a known tag name.
//
// An empty Namespace implies a "http://www.w3.org/1999/xhtml" namespace.
// Similarly, "math" is short for "http://www.w3.org/1998/Math/MathML", and
// "svg" is short for "http://www.w3.org/2000/svg".
type Node struct {
	Parent    *Node
	Child     []*Node
	Type      NodeType
	DataAtom  atom.Atom
	Data      string
	Namespace string
	Attr      []Attribute
}

// Add adds a node as a child of n.
// It will panic if the child's parent is not nil.
func (n *Node) Add(child *Node) {
	if child.Parent != nil {
		panic("html: Node.Add called for a child Node that already has a parent")
	}
	child.Parent = n
	n.Child = append(n.Child, child)
}

// Remove removes a node as a child of n.
// It will panic if the child's parent is not n.
func (n *Node) Remove(child *Node) {
	if child.Parent == n {
		child.Parent = nil
		for i, m := range n.Child {
			if m == child {
				copy(n.Child[i:], n.Child[i+1:])
				j := len(n.Child) - 1
				n.Child[j] = nil
				n.Child = n.Child[:j]
				return
			}
		}
	}
	panic("html: Node.Remove called for a non-child Node")
}

// reparentChildren reparents all of src's child nodes to dst.
func reparentChildren(dst, src *Node) {
	for _, n := range src.Child {
		if n.Parent != src {
			panic("html: nodes have an inconsistent parent/child relationship")
		}
		n.Parent = dst
	}
	dst.Child = append(dst.Child, src.Child...)
	src.Child = nil
}

// clone returns a new node with the same type, data and attributes.
// The clone has no parent and no children.
func (n *Node) clone() *Node {
	m := &Node{
		Type:     n.Type,
		DataAtom: n.DataAtom,
		Data:     n.Data,
		Attr:     make([]Attribute, len(n.Attr)),
	}
	copy(m.Attr, n.Attr)
	return m
}

// nodeStack is a stack of nodes.
type nodeStack []*Node

// pop pops the stack. It will panic if s is empty.
func (s *nodeStack) pop() *Node {
	i := len(*s)
	n := (*s)[i-1]
	*s = (*s)[:i-1]
	return n
}

// top returns the most recently pushed node, or nil if s is empty.
func (s *nodeStack) top() *Node {
	if i := len(*s); i > 0 {
		return (*s)[i-1]
	}
	return nil
}

// index returns the index of the top-most occurrence of n in the stack, or -1
// if n is not present.
func (s *nodeStack) index(n *Node) int {
	for i := len(*s) - 1; i >= 0; i-- {
		if (*s)[i] == n {
			return i
		}
	}
	return -1
}

// insert inserts a node at the given index.
func (s *nodeStack) insert(i int, n *Node) {
	(*s) = append(*s, nil)
	copy((*s)[i+1:], (*s)[i:])
	(*s)[i] = n
}

// remove removes a node from the stack. It is a no-op if n is not present.
func (s *nodeStack) remove(n *Node) {
	i := s.index(n)
	if i == -1 {
		return
	}
	copy((*s)[i:], (*s)[i+1:])
	j := len(*s) - 1
	(*s)[j] = nil
	*s = (*s)[:j]
}
<|endoftext|>