<commit_before>package zmq

import (
	"bufio"
	"encoding/binary"
	"http"
	"io"
	"io/ioutil"
	"net"
	"os"
	"sync"
)

const (
	flagMore = 1
)

const (
	SOCK_PAIR = iota
	SOCK_PUB
	SOCK_SUB
	SOCK_REQ
	SOCK_REP
	//SOCK_XREQ
	//SOCK_XREP
	SOCK_PULL
	SOCK_PUSH
)

type nilWAdder struct {}

func (b nilWAdder) addConn(wc io.WriteCloser) {}

type nilRAdder struct {}

func (b nilRAdder) addConn(fr *frameReader) {}

type reader interface {
	addConn(fr *frameReader)
	RecvMsg() (io.ReadCloser, os.Error)
}

type bindWriter interface {
	io.Writer
	addConn(wc io.WriteCloser)
}

type Socket struct {
	identity string
	r reader
	w bindWriter
}

func NewSocket(typ int, identity string) (*Socket, os.Error) {
	var r reader
	var w bindWriter
	switch typ {
	case SOCK_PAIR:
	case SOCK_PUB:
		mw := make(multiWriter, 0, 5)
		w = newFrameWriter(mw)
	case SOCK_SUB:
		r = newQueuedReader()
	case SOCK_REQ:
	case SOCK_REP:
	case SOCK_PULL:
		r = newQueuedReader()
	case SOCK_PUSH:
		w = newLbWriter()
	default:
	}
	return &Socket{identity, r, w}, nil
}

func (s *Socket) RecvMsg() (io.ReadCloser, os.Error) {
	if s.r == nil {
		return nil, os.NewError("socket is not readable")
	}
	return s.r.RecvMsg()
}

func (s *Socket) Write(b []byte) (int, os.Error) {
	if s.w == nil {
		return 0, os.NewError("socket is not writable")
	}
	return s.w.Write(b)
}

func (s *Socket) Connect(addr string) os.Error {
	url, err := http.ParseURL(addr)
	if err != nil {
		return err
	}
	var conn net.Conn
	switch url.Scheme {
	case "ipc":
		conn, err = net.Dial("unix", url.Host + url.Path)
	case "tcp":
		conn, err = net.Dial("tcp", url.Host)
	default:
		err = os.NewError("unsupported URL scheme")
	}
	if err != nil {
		return err
	}
	// TODO: avoid making extra frameWriters and frameReaders
	if s.w != nil {
		fw := newFrameWriter(conn)
		// TODO: write identity properly
		b := make([]byte, len(s.identity)+2)
		b[0] = byte(len(s.identity))
		b[1] = 0
		copy(b[2:], s.identity)
		fw.Write(b)
		fr := newFrameReader(conn)
		msg, err := fr.RecvMsg()
		if err != nil {
			return err
		}
		io.Copy(ioutil.Discard, msg)
		msg.Close()
		s.w.addConn(conn)
	}
	if s.r != nil {
		fw := newFrameWriter(conn)
		fw.Write(nil)
		fr := newFrameReader(conn)
		msg, err := fr.RecvMsg()
		if err != nil {
			return err
		}
		io.Copy(ioutil.Discard, msg)
		msg.Close()
		s.r.addConn(fr)
	}
	return nil
}

// Similar to io.MultiWriter, but we have access to its internals and it has a Close method.
type multiWriter []io.WriteCloser

func (mw multiWriter) Write(p []byte) (n int, err os.Error) {
	n = len(p)
	for _, w := range mw {
		n2, err2 := w.Write(p)
		if err2 != nil {
			n = n2
			err = err2
		}
	}
	return
}

func (mw multiWriter) Close() (err os.Error) {
	for _, w := range mw {
		err2 := w.Close()
		if err2 != nil {
			err = err2
		}
	}
	return
}

func (mw multiWriter) addConn(wc io.WriteCloser) {
	mw = append(mw, wc)
}

// A load-balanced WriteCloser
type lbWriter struct {
	w []io.WriteCloser
	c chan []byte
}

func newLbWriter() *lbWriter {
	c := make(chan []byte, 10)
	return &lbWriter{nil, c}
}

func (w *lbWriter) addConn(wc io.WriteCloser) {
	go writeListen(wc, w.c)
	// TODO: figure out a better way to keep track of writers
	w.w = append(w.w, wc)
}

func writeListen(w io.WriteCloser, c chan []byte) {
	for {
		b, ok := <-c
		if !ok {
			w.Close()
			break
		}
		if _, err := w.Write(b); err != nil {
			// pass it on to a different writer
			c <- b
			break
		}
	}
}

func (w *lbWriter) Write(b []byte) (int, os.Error) {
	w.c <- b
	// TODO: can we do better?
	return len(b), nil
}

func (w *lbWriter) Close() os.Error {
	close(w.c)
	return nil
}

type queuedReader struct {
	fr []*frameReader
	c chan io.ReadCloser
}

func newQueuedReader() *queuedReader {
	c := make(chan io.ReadCloser, 10)
	return &queuedReader{nil, c}
}

func (r *queuedReader) addConn(fr *frameReader) {
	go readListen(fr, r.c)
	// TODO: figure out a better way to keep track of readers
	r.fr = append(r.fr, fr)
}

func readListen(fr *frameReader, c chan io.ReadCloser) {
	for {
		mr, err := fr.RecvMsg()
		if err != nil {
			break
		}
		c <- mr
	}
}

func (r *queuedReader) RecvMsg() (io.ReadCloser, os.Error) {
	mr := <-r.c
	return mr, nil
}

func (r *queuedReader) Close() os.Error {
	for _, r := range r.fr {
		r.Close()
	}
	return nil
}

type frameWriter struct {
	nilWAdder
	wc io.WriteCloser
	buf *bufio.Writer
}

func newFrameWriter(wc io.WriteCloser) *frameWriter {
	w := &frameWriter{wc: wc, buf: bufio.NewWriter(wc)}
	return w
}

func (fc *frameWriter) Write(b []byte) (n int, err os.Error) {
	// + 1 for flags
	l := len(b) + 1
	if l < 255 {
		n, err = fc.buf.Write([]byte{byte(l)})
	} else {
		var length [9]byte
		length[0] = 255
		binary.BigEndian.PutUint64(length[1:], uint64(l))
		n, err = fc.buf.Write(length[:])
	}
	if err != nil {
		return
	}
	// flags; it’s impossible to have a slice with len > 2^64-1, so the MORE flag is always 0
	// All other flag bits are reserved.
	nn, err := fc.buf.Write([]byte{0})
	n += nn
	if err != nil {
		return
	}
	nn, err = fc.buf.Write(b)
	n += nn
	fc.buf.Flush()
	return
}

func (fc *frameWriter) Close() os.Error {
	return fc.wc.Close()
}

type frameReader struct {
	nilRAdder
	lock sync.Mutex
	rc io.ReadCloser
	buf *bufio.Reader
}

type msgReader struct {
	length uint64 // length of the current frame
	more bool // whether there are more frames after this one
	buf *bufio.Reader
	lock *sync.Mutex
}

func newMsgReader(buf *bufio.Reader, lock *sync.Mutex) (*msgReader, os.Error) {
	r := &msgReader{buf: buf, lock: lock}
	err := r.readHeader()
	return r, err
}

func (r *msgReader) readHeader() os.Error {
	var b [8]byte
	if _, err := r.buf.Read(b[:1]); err != nil {
		return err
	}
	if b[0] == 255 {
		if _, err := r.buf.Read(b[:]); err != nil {
			return err
		}
		r.length = binary.BigEndian.Uint64(b[:])
	} else {
		r.length = uint64(b[0])
	}
	r.length--
	flags, err := r.buf.ReadByte()
	if err != nil {
		return err
	}
	r.more = flags & flagMore != 0
	return nil
}

func (r *msgReader) Read(b []byte) (n int, err os.Error) {
	for n < len(b) {
		l := uint64(len(b) - n)
		if r.length < l {
			l = r.length
		}
		nn, err := r.buf.Read(b[n:n+int(l)])
		n += nn
		r.length -= uint64(nn)
		if err != nil {
			return n, err
		}
		if r.length == 0 {
			if r.more {
				r.readHeader()
			} else {
				return n, os.EOF
			}
		}
	}
	return
}

func (r *msgReader) Close() os.Error {
	r.lock.Unlock()
	return nil
}

func newFrameReader(rc io.ReadCloser) *frameReader {
	r := &frameReader{rc: rc, buf: bufio.NewReader(rc)}
	return r
}

func (fr *frameReader) RecvMsg() (io.ReadCloser, os.Error) {
	fr.lock.Lock()
	return newMsgReader(fr.buf, &fr.lock)
}

func (fr *frameReader) Close() os.Error {
	return fr.rc.Close()
}
<commit_msg>Fix several bugs<commit_after>package zmq

import (
	"bufio"
	"encoding/binary"
	"http"
	"io"
	"io/ioutil"
	"net"
	"os"
	"sync"
)

const (
	flagMore = 1
)

const (
	SOCK_PAIR = iota
	SOCK_PUB
	SOCK_SUB
	SOCK_REQ
	SOCK_REP
	//SOCK_XREQ
	//SOCK_XREP
	SOCK_PULL
	SOCK_PUSH
)

type nilWAdder struct {
	net.Conn
}

func (b nilWAdder) addConn(wc io.WriteCloser) {}

type nilRAdder struct {}

func (b nilRAdder) addConn(fr *frameReader) {}

type reader interface {
	addConn(fr *frameReader)
	RecvMsg() (io.ReadCloser, os.Error)
}

type bindWriter interface {
	io.WriteCloser
	addConn(wc io.WriteCloser)
}

type Socket struct {
	identity string
	r reader
	w *frameWriter
}

func NewSocket(typ int, identity string) (*Socket, os.Error) {
	var r reader
	var w *frameWriter
	switch typ {
	case SOCK_PAIR:
	case SOCK_PUB:
		mw := newMultiWriter()
		w = newFrameWriter(mw)
	case SOCK_SUB:
		r = newQueuedReader()
	case SOCK_REQ:
	case SOCK_REP:
	case SOCK_PULL:
		r = newQueuedReader()
	case SOCK_PUSH:
		lbw := newLbWriter()
		w = newFrameWriter(lbw)
	default:
	}
	return &Socket{identity, r, w}, nil
}

func (s *Socket) RecvMsg() (io.ReadCloser, os.Error) {
	if s.r == nil {
		return nil, os.NewError("socket is not readable")
	}
	return s.r.RecvMsg()
}

func (s *Socket) Write(b []byte) (int, os.Error) {
	if s.w == nil {
		return 0, os.NewError("socket is not writable")
	}
	return s.w.Write(b)
}

func (s *Socket) Connect(addr string) os.Error {
	url, err := http.ParseURL(addr)
	if err != nil {
		return err
	}
	var conn net.Conn
	switch url.Scheme {
	case "ipc":
		conn, err = net.Dial("unix", url.Host + url.Path)
	case "tcp":
		conn, err = net.Dial("tcp", url.Host)
	default:
		err = os.NewError("unsupported URL scheme")
	}
	if err != nil {
		return err
	}
	// TODO: avoid making extra frameWriters and frameReaders
	fw := newFrameWriter(nilWAdder{conn})
	fw.sendIdentity(s.identity)

	fr := newFrameReader(conn)
	msg, err := fr.RecvMsg()
	if err != nil {
		return err
	}
	io.Copy(ioutil.Discard, msg)
	msg.Close()

	if s.w != nil {
		s.w.addConn(conn)
	}
	if s.r != nil {
		s.r.addConn(fr)
	}
	return nil
}

// Similar to io.MultiWriter, but we have access to its internals and it has a Close method.
type multiWriter []io.WriteCloser

func newMultiWriter() *multiWriter {
	mw := make(multiWriter, 0, 5)
	return &mw
}

func (mw *multiWriter) Write(p []byte) (n int, err os.Error) {
	n = len(p)
	for _, w := range *mw {
		n2, err2 := w.Write(p)
		if err2 != nil {
			n = n2
			err = err2
		}
	}
	return
}

func (mw *multiWriter) Close() (err os.Error) {
	for _, w := range *mw {
		err2 := w.Close()
		if err2 != nil {
			err = err2
		}
	}
	return
}

func (mw *multiWriter) addConn(wc io.WriteCloser) {
	*mw = append(*mw, wc)
}

// A load-balanced WriteCloser
type lbWriter struct {
	w []io.WriteCloser
	c chan []byte
}

func newLbWriter() *lbWriter {
	c := make(chan []byte, 10)
	return &lbWriter{nil, c}
}

func (w *lbWriter) addConn(wc io.WriteCloser) {
	go writeListen(wc, w.c)
	// TODO: figure out a better way to keep track of writers
	w.w = append(w.w, wc)
}

func writeListen(w io.WriteCloser, c chan []byte) {
	for {
		b, ok := <-c
		if !ok {
			w.Close()
			break
		}
		if _, err := w.Write(b); err != nil {
			// pass it on to a different writer
			c <- b
			break
		}
	}
}

func (w *lbWriter) Write(b []byte) (int, os.Error) {
	w.c <- b
	// TODO: can we do better?
	return len(b), nil
}

func (w *lbWriter) Close() os.Error {
	close(w.c)
	return nil
}

type queuedReader struct {
	fr []*frameReader
	c chan io.ReadCloser
}

func newQueuedReader() *queuedReader {
	c := make(chan io.ReadCloser, 10)
	return &queuedReader{nil, c}
}

func (r *queuedReader) addConn(fr *frameReader) {
	go readListen(fr, r.c)
	// TODO: figure out a better way to keep track of readers
	r.fr = append(r.fr, fr)
}

func readListen(fr *frameReader, c chan io.ReadCloser) {
	for {
		mr, err := fr.RecvMsg()
		if err != nil {
			break
		}
		c <- mr
	}
}

func (r *queuedReader) RecvMsg() (io.ReadCloser, os.Error) {
	mr := <-r.c
	return mr, nil
}

func (r *queuedReader) Close() os.Error {
	for _, r := range r.fr {
		r.Close()
	}
	return nil
}

type frameWriter struct {
	bindWriter
	buf *bufio.Writer
}

func newFrameWriter(wc bindWriter) *frameWriter {
	w := &frameWriter{wc, bufio.NewWriter(wc)}
	return w
}

func (fw *frameWriter) sendIdentity(id string) os.Error {
	var b []byte
	if id != "" {
		b = []byte(id)
	}
	_, err := fw.Write(b)
	return err
}

func (fc *frameWriter) Write(b []byte) (n int, err os.Error) {
	// + 1 for flags
	l := len(b) + 1
	if l < 255 {
		n, err = fc.buf.Write([]byte{byte(l)})
	} else {
		var length [9]byte
		length[0] = 255
		binary.BigEndian.PutUint64(length[1:], uint64(l))
		n, err = fc.buf.Write(length[:])
	}
	if err != nil {
		return
	}
	// flags; it’s impossible to have a slice with len > 2^64-1, so the MORE flag is always 0
	// All other flag bits are reserved.
	nn, err := fc.buf.Write([]byte{0})
	n += nn
	if err != nil {
		return
	}
	nn, err = fc.buf.Write(b)
	n += nn
	fc.buf.Flush()
	return
}

type frameReader struct {
	nilRAdder
	lock sync.Mutex
	rc io.ReadCloser
	buf *bufio.Reader
}

type msgReader struct {
	length uint64 // length of the current frame
	more bool // whether there are more frames after this one
	buf *bufio.Reader
	lock *sync.Mutex
}

func newMsgReader(buf *bufio.Reader, lock *sync.Mutex) (*msgReader, os.Error) {
	r := &msgReader{buf: buf, lock: lock}
	err := r.readHeader()
	return r, err
}

func (r *msgReader) readHeader() os.Error {
	var b [8]byte
	if _, err := r.buf.Read(b[:1]); err != nil {
		return err
	}
	if b[0] == 255 {
		if _, err := r.buf.Read(b[:]); err != nil {
			return err
		}
		r.length = binary.BigEndian.Uint64(b[:])
	} else {
		r.length = uint64(b[0])
	}
	r.length--
	flags, err := r.buf.ReadByte()
	if err != nil {
		return err
	}
	r.more = flags & flagMore != 0
	return nil
}

func (r *msgReader) Read(b []byte) (n int, err os.Error) {
	for n < len(b) {
		l := uint64(len(b) - n)
		if r.length < l {
			l = r.length
		}
		nn, err := r.buf.Read(b[n:n+int(l)])
		n += nn
		r.length -= uint64(nn)
		if err != nil {
			return n, err
		}
		if r.length == 0 {
			if r.more {
				r.readHeader()
			} else {
				return n, os.EOF
			}
		}
	}
	return
}

func (r *msgReader) Close() os.Error {
	r.lock.Unlock()
	return nil
}

func newFrameReader(rc io.ReadCloser) *frameReader {
	r := &frameReader{rc: rc, buf: bufio.NewReader(rc)}
	return r
}

func (fr *frameReader) RecvMsg() (io.ReadCloser, os.Error) {
	fr.lock.Lock()
	return newMsgReader(fr.buf, &fr.lock)
}

func (fr *frameReader) Close() os.Error {
	return fr.rc.Close()
}
<|endoftext|>


<commit_before>// Copyright 2014 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

/*
Git-codereview manages the code review process for Git changes using a Gerrit
server.

The git-codereview tool manages ``change branches'' in the local git repository.
Each such branch tracks a single commit, or ``pending change'',
that is reviewed using a Gerrit server; the Gerrit remote must be
named ``origin'' in the local git repo.

Modifications to the pending change are applied by amending the commit.
This process implements the ``single-commit feature branch'' model.
Creating multiple-commit feature branches, for example to break a large
change into a reviewable sequence, is also supported; see the discussion below.

Once installed as git-codereview, the tool's commands are available through git
either by running

	git codereview <command>

or, if aliases are installed, as

	git <command>

The review tool's command names do not conflict with any extant git commands.
This document uses the first form for clarity but most users install these
aliases in their .gitconfig file:

	[alias]
		change = codereview change
		gofmt = codereview gofmt
		mail = codereview mail
		pending = codereview pending
		rebase-work = codereview rebase-work
		submit = codereview submit
		sync = codereview sync

Single-Commit Work Branches

For simple, unrelated changes, the typical usage of the git-codereview tool
is to place each pending change in its own Git branch.
In this workflow, the work branch contains
either no pending change beyond origin/master (when there's no local work)
or exactly one pending change beyond origin/master (the change being developed).

When there is no pending change on the work branch,
``git codereview change'' creates one by running ``git commit''.
Otherwise, when there is already a pending change,
``git codereview change'' revises it by running ``git commit --amend''.

The ``git codereview mail'' and ``git codereview submit'' commands
implicitly operate on the lone pending change.

Multiple-Commit Work Branches

Of course, it is not always feasible to put each pending change in a separate branch.
A sequence of changes that build on one another is more easily
managed as multiple commits on a single branch, and the git-codereview tool
supports this workflow as well.
To add a new pending change, invoke ``git commit'' directly,
instead of ``git codereview change''.
The git-codereview tool adjusts its behavior when there are
multiple pending changes.

The ``git codereview change'' command amends the top commit in the stack (HEAD).
To amend a commit further down the stack, use Git's rebase support,
for example by using ``git commit --fixup'' followed by ``git codereview rebase-work''.

The ``git codereview mail'' command requires an explicit revision argument,
but note that since ``git codereview mail'' is implemented as a ``git push'',
any commits earlier in the stack are necessarily also mailed.

The ``git codereview submit'' command also requires an explicit revision argument,
and while earlier commits are necessarily still uploaded and mailed,
only the named revision or revisions are submitted (merged into origin/master).
In a single-commit work branch, a successful ``git codereview submit''
effectively runs ``git codereview sync'' automatically.
In a multiple-commit work branch, it does not, because
the implied ``git rebase'' may conflict with the remaining pending commits.
Instead it is necessary to run ``git codereview sync'' explicitly
(when ready) after ``git codereview submit''.

Reusing Work Branches

Although one common practice is to create a new branch for each pending change,
running ``git codereview submit'' (and possibly ``git codereview sync'')
leaves the current branch ready for reuse with a future change.
Some developers find it helpful to create a single work branch
(``git change work'') and then do all work in that branch,
possibly in the multiple-commit mode, never changing between branches.

Command Details

All commands accept these global flags:

The -v flag prints all commands that make changes. Multiple occurrences
trigger more verbosity in some commands, including sync.

The -n flag prints all commands that would be run, but does not run them.

Descriptions of each command follow.

Branchpoint

	git codereview branchpoint

The branchpoint command prints the commit hash of the most recent commit
on the current branch that is shared with the Gerrit server. This is the point
where local work branched from the published tree. The command is intended
mainly for use in scripts. For example, ``git diff $(git codereview branchpoint)''
or ``git log $(git codereview branchpoint)..HEAD''.

Change

The change command creates and moves between Git branches and maintains the
pending changes on work branches.

	git codereview change [-a] [-q] [branchname]

Given a branch name as an argument, the change command switches to the named
branch, creating it if necessary. If the branch is created and there are staged
changes, it will commit the changes to the branch, creating a new pending
change.

With no argument, the change command creates a new pending change from the
staged changes in the current branch or, if there is already a pending change,
amends that change.

The -q option skips the editing of an extant pending change's commit message.

The -a option automatically adds any unstaged edits in tracked files during
commit; it is equivalent to the 'git commit' -a option.

Gofmt

The gofmt command applies the gofmt program to all files modified in the
current work branch, both in the staging area (index) and the working tree
(local directory).

	git codereview gofmt [-l]

The -l option causes the command to list the files that need reformatting but
not reformat them. Otherwise, the gofmt command reformats modified files in
place. That is, files in the staging area are reformatted in the staging area,
and files in the working tree are reformatted in the working tree.

Help

The help command displays basic usage instructions.

	git codereview help

Hooks

The hooks command installs the Git hooks to enforce code review conventions.

	git codereview hooks

The pre-commit hook checks that all Go code is formatted with gofmt and that
the commit is not being made directly to the master branch.

The commit-msg hook adds the Gerrit ``Change-Id'' line to the commit message if
not present. It also checks that the message uses the convention established by
the Go project that the first line has the form, pkg/path: summary.

The hooks command will not overwrite an existing hook.
If it is not installing hooks, use ``git codereview hooks -v'' for details.
This hook installation is also done at startup by all other git codereview
commands, except ``git codereview help''.

Hook-Invoke

The hook-invoke command is an internal command that invokes the named Git hook.

	git codereview hook-invoke <hook> [args]

It is run by the shell scripts installed by the ``git codereview hooks'' command.

Mail

The mail command starts the code review process for the pending change.

	git codereview mail [-f] [-r email] [-cc email] [-trybot] [revision]

It pushes the pending change commit in the current branch to the Gerrit code
review server and prints the URL for the change on the server.
If the change already exists on the server, the mail command updates that
change with a new changeset.

The -r and -cc flags identify the email addresses of people to do the code
review and to be CC'ed about the code review.
Multiple addresses are given as a comma-separated list.

An email address passed to -r or -cc can be shortened from name@domain to name.
The mail command resolves such shortenings by reading the list of past reviewers
from the git repository log to find email addresses of the form name@somedomain
and then, in case of ambiguity, using the reviewer who appears most often.

The -trybot flag runs the trybots on all new or updated changes. It is
equivalent to setting the Run-Trybot+1 label from Gerrit.

The mail command fails if there are staged edits that are not committed.
The -f flag overrides this behavior.

The mail command updates the tag <branchname>.mailed to refer to the
commit that was most recently mailed, so running ``git diff <branchname>.mailed''
shows diffs between what is on the Gerrit server and the current directory.

If there are multiple pending commits, the revision argument is mandatory.
If no revision is specified, the mail command prints a short summary of
the pending commits for use in deciding which to mail.

If any commit that would be pushed to the server contains the text
"DO NOT MAIL" (case insensitive) in its commit message, the mail command
will refuse to send the commit to the server.

Pending

The pending command prints to standard output the status of all pending changes
and staged, unstaged, and untracked files in the local repository.

	git codereview pending [-c] [-l] [-s]

The -c flag causes the command to show pending changes only on the current branch.

The -l flag causes the command to use only locally available information.
By default, it fetches recent commits and code review information from the
Gerrit server.

The -s flag causes the command to print abbreviated (short) output.

Common shorter aliases include ``git p'' for ``git pending''
and ``git pl'' for ``git pending -l'' (notably faster but without Gerrit information).

Rebase-work

The rebase-work command runs git rebase in interactive mode over pending changes.
It is shorthand for ``git rebase -i $(git codereview branchpoint)''.
It differs from plain ``git rebase -i'' in that the latter will try to incorporate
new commits from the origin branch during the rebase;
``git codereview rebase-work'' does not.

In multiple-commit workflows, rebase-work is used so often
that it can be helpful to alias it to ``git rw''.

Submit

The submit command pushes the pending change to the Gerrit server and tells
Gerrit to submit it to the master branch.

	git codereview submit [-i | revision...]

The command fails if there are modified files (staged or unstaged) that are not
part of the pending change.

The -i option causes the submit command to open a list of commits to submit
in the configured text editor, similar to ``git rebase -i''.

If multiple revisions are specified, the submit command submits each one in turn,
stopping at the first failure.

When run in a multiple-commit work branch,
either the -i option or the revision argument is mandatory.
If both are omitted, the submit command prints a short summary of
the pending commits for use in deciding which to submit.

After submitting the pending changes, the submit command tries to synchronize the
current branch to the submitted commit, if it can do so cleanly.
If not, it will prompt the user to run ``git codereview sync'' manually.

After a successful sync, the branch can be used to prepare a new change.

Sync

The sync command updates the local repository.

	git codereview sync

It fetches commits from the remote repository and merges them from the
upstream branch to the current branch, rebasing any pending changes.

Configuration

If a file named codereview.cfg is present in the repository root,
git-codereview will use it for configuration. It should contain lines
of this format:

	key: value

The ``gerrit'' key sets the Gerrit URL for this project. Git-codereview
automatically derives the Gerrit URL from repositories hosted in
*.googlesource.com. If not set or derived, the repository is assumed to
not have Gerrit, and certain features won't work.

The ``issuerepo'' key specifies the GitHub repository to use for issues, if
different from the source repository. If set to ``golang/go'', for example,
lines such as ``Fixes #123'' in a commit message will be rewritten to ``Fixes
golang/go#123''.

*/
package main
<commit_msg>git-codereview: highlight branchpoint command<commit_after>// Copyright 2014 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

/*
Git-codereview manages the code review process for Git changes using a Gerrit
server.

The git-codereview tool manages ``change branches'' in the local git repository.
Each such branch tracks a single commit, or ``pending change'',
that is reviewed using a Gerrit server; the Gerrit remote must be
named ``origin'' in the local git repo.

Modifications to the pending change are applied by amending the commit.
This process implements the ``single-commit feature branch'' model.
Creating multiple-commit feature branches, for example to break a large
change into a reviewable sequence, is also supported; see the discussion below.

Once installed as git-codereview, the tool's commands are available through git
either by running

	git codereview <command>

or, if aliases are installed, as

	git <command>

The review tool's command names do not conflict with any extant git commands.
This document uses the first form for clarity but most users install these
aliases in their .gitconfig file:

	[alias]
		change = codereview change
		gofmt = codereview gofmt
		mail = codereview mail
		pending = codereview pending
		rebase-work = codereview rebase-work
		submit = codereview submit
		sync = codereview sync

Single-Commit Work Branches

For simple, unrelated changes, the typical usage of the git-codereview tool
is to place each pending change in its own Git branch.
In this workflow, the work branch contains
either no pending change beyond origin/master (when there's no local work)
or exactly one pending change beyond origin/master (the change being developed).

When there is no pending change on the work branch,
``git codereview change'' creates one by running ``git commit''.
Otherwise, when there is already a pending change,
``git codereview change'' revises it by running ``git commit --amend''.

The ``git codereview mail'' and ``git codereview submit'' commands
implicitly operate on the lone pending change.

Multiple-Commit Work Branches

Of course, it is not always feasible to put each pending change in a separate branch.
A sequence of changes that build on one another is more easily
managed as multiple commits on a single branch, and the git-codereview tool
supports this workflow as well.
To add a new pending change, invoke ``git commit'' directly,
instead of ``git codereview change''.
The git-codereview tool adjusts its behavior when there are
multiple pending changes.

The ``git codereview change'' command amends the top commit in the stack (HEAD).
To amend a commit further down the stack, use Git's rebase support,
for example by using ``git commit --fixup'' followed by ``git codereview rebase-work''.

The ``git codereview mail'' command requires an explicit revision argument,
but note that since ``git codereview mail'' is implemented as a ``git push'',
any commits earlier in the stack are necessarily also mailed.

The ``git codereview submit'' command also requires an explicit revision argument,
and while earlier commits are necessarily still uploaded and mailed,
only the named revision or revisions are submitted (merged into origin/master).
In a single-commit work branch, a successful ``git codereview submit''
effectively runs ``git codereview sync'' automatically.
In a multiple-commit work branch, it does not, because
the implied ``git rebase'' may conflict with the remaining pending commits.
Instead it is necessary to run ``git codereview sync'' explicitly
(when ready) after ``git codereview submit''.

Reusing Work Branches

Although one common practice is to create a new branch for each pending change,
running ``git codereview submit'' (and possibly ``git codereview sync'')
leaves the current branch ready for reuse with a future change.
Some developers find it helpful to create a single work branch
(``git change work'') and then do all work in that branch,
possibly in the multiple-commit mode, never changing between branches.

Command Details

All commands accept these global flags:

The -v flag prints all commands that make changes. Multiple occurrences
trigger more verbosity in some commands, including sync.

The -n flag prints all commands that would be run, but does not run them.

Descriptions of each command follow.

Branchpoint

The branchpoint command prints the commit hash of the most recent commit
on the current branch that is shared with the Gerrit server.

	git codereview branchpoint

This commit is the point where local work branched from the published tree.
The command is intended mainly for use in scripts. For example,
``git diff $(git codereview branchpoint)'' or
``git log $(git codereview branchpoint)..HEAD''.

Change

The change command creates and moves between Git branches and maintains the
pending changes on work branches.

	git codereview change [-a] [-q] [branchname]

Given a branch name as an argument, the change command switches to the named
branch, creating it if necessary. If the branch is created and there are staged
changes, it will commit the changes to the branch, creating a new pending
change.

With no argument, the change command creates a new pending change from the
staged changes in the current branch or, if there is already a pending change,
amends that change.

The -q option skips the editing of an extant pending change's commit message.

The -a option automatically adds any unstaged edits in tracked files during
commit; it is equivalent to the 'git commit' -a option.

Gofmt

The gofmt command applies the gofmt program to all files modified in the
current work branch, both in the staging area (index) and the working tree
(local directory).

	git codereview gofmt [-l]

The -l option causes the command to list the files that need reformatting but
not reformat them. Otherwise, the gofmt command reformats modified files in
place. That is, files in the staging area are reformatted in the staging area,
and files in the working tree are reformatted in the working tree.

Help

The help command displays basic usage instructions.

	git codereview help

Hooks

The hooks command installs the Git hooks to enforce code review conventions.

	git codereview hooks

The pre-commit hook checks that all Go code is formatted with gofmt and that
the commit is not being made directly to the master branch.

The commit-msg hook adds the Gerrit ``Change-Id'' line to the commit message if
not present. It also checks that the message uses the convention established by
the Go project that the first line has the form, pkg/path: summary.

The hooks command will not overwrite an existing hook.
If it is not installing hooks, use ``git codereview hooks -v'' for details.
This hook installation is also done at startup by all other git codereview
commands, except ``git codereview help''.

Hook-Invoke

The hook-invoke command is an internal command that invokes the named Git hook.

	git codereview hook-invoke <hook> [args]

It is run by the shell scripts installed by the ``git codereview hooks'' command.

Mail

The mail command starts the code review process for the pending change.

	git codereview mail [-f] [-r email] [-cc email] [-trybot] [revision]

It pushes the pending change commit in the current branch to the Gerrit code
review server and prints the URL for the change on the server.
If the change already exists on the server, the mail command updates that
change with a new changeset.

The -r and -cc flags identify the email addresses of people to do the code
review and to be CC'ed about the code review.
Multiple addresses are given as a comma-separated list.

An email address passed to -r or -cc can be shortened from name@domain to name.
The mail command resolves such shortenings by reading the list of past reviewers
from the git repository log to find email addresses of the form name@somedomain
and then, in case of ambiguity, using the reviewer who appears most often.

The -trybot flag runs the trybots on all new or updated changes. It is
equivalent to setting the Run-Trybot+1 label from Gerrit.

The mail command fails if there are staged edits that are not committed.
The -f flag overrides this behavior.

The mail command updates the tag <branchname>.mailed to refer to the
commit that was most recently mailed, so running ``git diff <branchname>.mailed''
shows diffs between what is on the Gerrit server and the current directory.

If there are multiple pending commits, the revision argument is mandatory.
If no revision is specified, the mail command prints a short summary of
the pending commits for use in deciding which to mail.

If any commit that would be pushed to the server contains the text
"DO NOT MAIL" (case insensitive) in its commit message, the mail command
will refuse to send the commit to the server.

Pending

The pending command prints to standard output the status of all pending changes
and staged, unstaged, and untracked files in the local repository.

	git codereview pending [-c] [-l] [-s]

The -c flag causes the command to show pending changes only on the current branch.

The -l flag causes the command to use only locally available information.
By default, it fetches recent commits and code review information from the
Gerrit server.

The -s flag causes the command to print abbreviated (short) output.

Common shorter aliases include ``git p'' for ``git pending''
and ``git pl'' for ``git pending -l'' (notably faster but without Gerrit information).

Rebase-work

The rebase-work command runs git rebase in interactive mode over pending changes.
It is shorthand for ``git rebase -i $(git codereview branchpoint)''.
It differs from plain ``git rebase -i'' in that the latter will try to incorporate
new commits from the origin branch during the rebase;
``git codereview rebase-work'' does not.

In multiple-commit workflows, rebase-work is used so often
that it can be helpful to alias it to ``git rw''.

Submit

The submit command pushes the pending change to the Gerrit server and tells
Gerrit to submit it to the master branch.

	git codereview submit [-i | revision...]

The command fails if there are modified files (staged or unstaged) that are not
part of the pending change.

The -i option causes the submit command to open a list of commits to submit
in the configured text editor, similar to ``git rebase -i''.

If multiple revisions are specified, the submit command submits each one in turn,
stopping at the first failure.

When run in a multiple-commit work branch,
either the -i option or the revision argument is mandatory.
If both are omitted, the submit command prints a short summary of
the pending commits for use in deciding which to submit.

After submitting the pending changes, the submit command tries to synchronize the
current branch to the submitted commit, if it can do so cleanly.
If not, it will prompt the user to run ``git codereview sync'' manually.

After a successful sync, the branch can be used to prepare a new change.

Sync

The sync command updates the local repository.

	git codereview sync

It fetches commits from the remote repository and merges them from the
upstream branch to the current branch, rebasing any pending changes.

Configuration

If a file named codereview.cfg is present in the repository root,
git-codereview will use it for configuration. It should contain lines
of this format:

	key: value

The ``gerrit'' key sets the Gerrit URL for this project. Git-codereview
automatically derives the Gerrit URL from repositories hosted in
*.googlesource.com. If not set or derived, the repository is assumed to
not have Gerrit, and certain features won't work.

The ``issuerepo'' key specifies the GitHub repository to use for issues, if
different from the source repository. If set to ``golang/go'', for example,
lines such as ``Fixes #123'' in a commit message will be rewritten to ``Fixes
golang/go#123''.

*/
package main
<|endoftext|>


<commit_before>package main

import (
	"errors"
	"fmt"
	"io"
	"log"
	"os"
	"path/filepath"
	"sort"
	"strings"
	"sync"
	"time"

	"github.com/rjw57/aonui"
)

const maximumSimultaneousDownloads = 5

// Global semaphore used to limit the number of simultaneous downloads
var fetchSem = make(chan int, maximumSimultaneousDownloads)

// Command-line flags
var (
	syncBaseDir string
	syncHighRes bool
	syncMaxRuns int
)

var cmdSync = &Command{
	UsageLine: "sync [-basedir directory] [-highres] [-maxruns number]",
	Short:     "fetch wind data from the GFS",
	Long: `
Sync will fetch wind data from the Global Forecast System (GFS) servers in
GRIB2 data. It will only fetch the subset of the data needed. It knows how to
fetch both the current 0.5 degree resolution data and the forthcoming 0.25
degree data.

Data is saved to the file gfs.YYYMMDDHH.grib2 where YYYY, MM, DD and HH are the
year, month, day and hour of the run with an appropriate number of leading
zeros.

The -basedir option specifies the directory data should be downloaded to. If
omitted, the current working directory is used.

If the -highres option is present, 0.25 degree data will be downloaded. If
omitted, the 0.5 degree data is downloaded.

The -maxruns options controls how far into the past sync will look for data
before stopping. The default value of 3 means examine the 3 newest runs on the
server starting with the newest. If any run is a) incomplete on the server or
b) already downloaded proceed to the next until the list of runs is exhausted.

The utility attempts to be robust in the face of flaky network connections or a
flaky server by re-trying failed downloads.
`,
}

func init() {
	cmdSync.Run = runSync // break init cycle
	cmdSync.Flag.StringVar(&syncBaseDir, "basedir", ".",
		"directory to download data to")
	cmdSync.Flag.BoolVar(&syncHighRes, "highres", false,
		"download 0.25deg data as opposed to 0.5deg")
	cmdSync.Flag.IntVar(&syncMaxRuns, "maxruns", 3,
		"maximum number of runs to examine before giving up")
}

func runSync(cmd *Command, args []string) {
	baseDir, highRes, maxRuns := syncBaseDir, syncHighRes, syncMaxRuns

	// Which source to use?
	src := aonui.GFSHalfDegreeDataset
	if highRes {
		src = aonui.GFSQuarterDegreeDataset
	}

	// Fetch all of the runs
	runs, err := src.FetchRuns()
	if err != nil {
		log.Fatal(err)
	}

	// Sort by *descending* date
	sort.Sort(sort.Reverse(ByDate(runs)))

	succeeded := false
	for _, run := range runs[:maxRuns] {
		destFn := filepath.Join(baseDir, run.Identifier+".grib2")

		if _, err := os.Stat(destFn); err == nil {
			log.Print("not overwriting ", destFn)
			continue
		}

		if err := syncRun(run, destFn); err != nil {
			log.Print("error syncing run: ", err)

			// ensure we remove destFn if we created it
			if os.IsExist(err) {
				log.Print("Removing ", destFn)
				os.Remove(destFn)
			}
		} else {
			// success!
			log.Print("run downloaded successfully")
			succeeded = true
			break
		}
	}

	if !succeeded {
		log.Fatal("no runs were downloaded")
	}
}

func syncRun(run *aonui.Run, destFn string) error {
	log.Print("Fetching data for run at ", run.When)

	// Get datasets for this run
	datasets, err := run.FetchDatasets()
	if err != nil {
		return err
	}
	log.Print("Run has ", len(datasets), " dataset(s)")

	if len(datasets) < run.Source.MinDatasets {
		log.Print("Run has too few, expecting at least ", run.Source.MinDatasets)
		return errors.New("too few datasets in source")
	}

	// File source for temporary files
	tfs := TemporaryFileSource{BaseDir: syncBaseDir, Prefix: "dataset-"}
	defer tfs.RemoveAll()

	// Make sure to remove temporary files on keyboard interrupt
	atexit(func() { tfs.RemoveAll() })

	// Open the output file
	log.Print("Fetching run to ", destFn)
	output, err := os.Create(destFn)
	if err != nil {
		log.Print("Error creating output: ", err)
		return err
	}

	// Ensure the file is closed on function exit
	defer output.Close()

	// Concatenate temporary files as they are finished
	fetchStart := time.Now()
	for f := range fetchDatasetsData(&tfs, datasets) {
		if input, err := os.Open(f.Name()); err != nil {
			log.Print("Error copying temporary file: ", err)
		} else {
			io.Copy(output, input)
			input.Close()
		}
		tfs.Remove(f)
	}

	fetchDuration := time.Since(fetchStart)
	fi, err := output.Stat()
	if err != nil {
		log.Print("Error: ", err)
		return err
	}
	log.Print(fmt.Sprintf("Overall download speed: %v/sec",
		ByteCount(float64(fi.Size())/fetchDuration.Seconds())))

	return nil
}

func fetchDatasetsData(tfs *TemporaryFileSource, datasets []*aonui.Dataset) chan *os.File {
	// Which records are we interested in?
	paramsOfInterest := []string{"HGT", "UGRD", "VGRD"}

	var wg sync.WaitGroup
	tmpFilesChan := make(chan *os.File)

	trySleepDuration, err := time.ParseDuration("10s")
	if err != nil {
		log.Fatal(err)
	}

	for _, ds := range datasets {
		// If we have a max forecast hour, and this dataset is later, skip
		if ds.Run.Source.MaxForecastHour > 0 && ds.ForecastHour > ds.Run.Source.MaxForecastHour {
			continue
		}

		wg.Add(1)

		go func(dataset *aonui.Dataset) {
			defer wg.Done()

			fetchSem <- 1
			defer func() { <-fetchSem }()

			// Perform download. Attempt download repeatedly
			maximumTries := dataset.Run.Source.FetchStrategy.MaximumRetries
			var tmpFile *os.File
			for tries := 0; tries < maximumTries; tries++ {
				// Create a temporary file for output
				tmpFile, err = tfs.Create()
				if err != nil {
					log.Print("Error creating temporary file: ", err)
				}

				log.Print("Fetching ", dataset.Identifier,
					" (try ", tries+1, " of ", maximumTries, ")")
				err := fetchDataset(tmpFile, dataset, paramsOfInterest)
				if err == nil {
					break
				} else {
					log.Print("Error fetching dataset: ", err)
				}

				// Remove this temporary file
				tmpFile.Close()
				tfs.Remove(tmpFile)
				tmpFile = nil

				// Sleep until the next try
				time.Sleep(trySleepDuration)
			}

			if tmpFile == nil {
				log.Print("error: failed to download ", dataset.Identifier)
			} else {
				tmpFile.Close()
				tmpFilesChan <- tmpFile
			}
		}(ds)
	}

	// Launch a goroutine to wait for all datasets to be downloaded and
	// then close the channel.
	go func() {
		wg.Wait()
		close(tmpFilesChan)
	}()

	return tmpFilesChan
}

func fetchDataset(output io.Writer, dataset *aonui.Dataset, paramsOfInterest []string) error {
	// Fetch inventory for this dataset
	inventory, err := dataset.FetchInventory()
	if err != nil {
		return err
	}

	// Calculate which items to save
	var (
		totalToFetch int64
		fetchItems   []*aonui.InventoryItem
	)
	for _, item := range inventory {
		saveItem := false
		for _, poi := range paramsOfInterest {
			for _, p := range item.Parameters {
				saveItem = saveItem || poi == p
			}
		}

		// HACK: we also are only interested in wind velocities at a
		// particular pressure. (i.e. ones whose "LayerName" field is of
		// the form "XXX mb".)
		saveItem = saveItem && strings.HasSuffix(item.LayerName, " mb")

		if saveItem {
			fetchItems = append(fetchItems, item)
			totalToFetch += item.Extent
		}
	}

	if len(fetchItems) == 0 {
		log.Print("No items to fetch")
		return nil
	}

	log.Print(fmt.Sprintf("Fetching %d records from %v (%v)",
		len(fetchItems), dataset.Identifier, ByteCount(totalToFetch)))
	if _, err := dataset.FetchAndWriteRecords(output, fetchItems); err != nil {
		return err
	}

	return nil
}
<commit_msg>sync: add ability to specify which parameters get downloaded<commit_after>package main

import (
	"errors"
	"fmt"
	"io"
	"log"
	"os"
	"path/filepath"
	"sort"
	"strings"
	"sync"
	"time"

	"github.com/rjw57/aonui"
)

const maximumSimultaneousDownloads = 5

// Global semaphore used to limit the number of simultaneous downloads
var fetchSem = make(chan int, maximumSimultaneousDownloads)

// A StringListValue wraps a slice of strings and implements the Value intrface
// for flag allowing it to be used as a command line flag.
type StringListValue []string

func (sl StringListValue) String() string      { return strings.Join(sl, ",") }
func (sl StringListValue) Get() interface{}    { return sl }
func (sl *StringListValue) Set(s string) error { *sl = strings.Split(s, ","); return nil }

// Command-line flags
var (
	syncBaseDir    string
	syncHighRes    bool
	syncMaxRuns    int
	syncParameters StringListValue = []string{"HGT", "UGRD", "VGRD"}
)

var cmdSync = &Command{
	UsageLine: "sync [flags]",
	Short:     "fetch wind data from the GFS",
	Long: `
Sync will fetch wind data from the Global Forecast System (GFS) servers in
GRIB2 data. It will only fetch the subset of the data needed. It knows how to
fetch both the current 0.5 degree resolution data and the forthcoming 0.25
degree data.

Data is saved to the file gfs.YYYMMDDHH.grib2 where YYYY, MM, DD and HH are the
year, month, day and hour of the run with an appropriate number of leading
zeros.

Setting base directory

The -basedir flag specifies the directory data should be downloaded to. If
omitted, the current working directory is used.

Downloading high reolsution data

If the -highres flag is present, 0.25 degree data will be downloaded. If
omitted, the 0.5 degree data is downloaded.

Specifying the oldest run to sync

The -maxruns flag controls how far into the past sync will look for data before
stopping. The default value of 3 means examine the 3 newest runs on the server
starting with the newest. If any run is a) incomplete on the server or b)
already downloaded proceed to the next until the list of runs is exhausted.

The utility attempts to be robust in the face of flaky network connections or a
flaky server by re-trying failed downloads.

Specifing which parameters to download

By default, aonui sync will download the HGT, UGRD and VGRD parameters from the
dataset. Use the -params flag to specify an alternate set. The set of
parameters to download should be a comma-separated lists.

`,
}

func init() {
	cmdSync.Run = runSync // break init cycle
	cmdSync.Flag.StringVar(&syncBaseDir, "basedir", ".",
		"directory to download data to")
	cmdSync.Flag.BoolVar(&syncHighRes, "highres", false,
		"download 0.25deg data as opposed to 0.5deg")
	cmdSync.Flag.IntVar(&syncMaxRuns, "maxruns", 3,
		"maximum number of runs to examine before giving up")
	cmdSync.Flag.Var(&syncParameters, "params", "list of parameters to download")
}

func runSync(cmd *Command, args []string) {
	baseDir, highRes, maxRuns := syncBaseDir, syncHighRes, syncMaxRuns

	// Which source to use?
	src := aonui.GFSHalfDegreeDataset
	if highRes {
		src = aonui.GFSQuarterDegreeDataset
	}

	// Fetch all of the runs
	runs, err := src.FetchRuns()
	if err != nil {
		log.Fatal(err)
	}

	// Sort by *descending* date
	sort.Sort(sort.Reverse(ByDate(runs)))

	succeeded := false
	for _, run := range runs[:maxRuns] {
		destFn := filepath.Join(baseDir, run.Identifier+".grib2")

		if _, err := os.Stat(destFn); err == nil {
			log.Print("not overwriting ", destFn)
			continue
		}

		if err := syncRun(run, destFn); err != nil {
			log.Print("error syncing run: ", err)

			// ensure we remove destFn if we created it
			if os.IsExist(err) {
				log.Print("Removing ", destFn)
				os.Remove(destFn)
			}
		} else {
			// success!
			log.Print("run downloaded successfully")
			succeeded = true
			break
		}
	}

	if !succeeded {
		log.Fatal("no runs were downloaded")
	}
}

func syncRun(run *aonui.Run, destFn string) error {
	log.Print("Fetching data for run at ", run.When)

	// Get datasets for this run
	datasets, err := run.FetchDatasets()
	if err != nil {
		return err
	}
	log.Print("Run has ", len(datasets), " dataset(s)")

	if len(datasets) < run.Source.MinDatasets {
		log.Print("Run has too few, expecting at least ", run.Source.MinDatasets)
		return errors.New("too few datasets in source")
	}

	// File source for temporary files
	tfs := TemporaryFileSource{BaseDir: syncBaseDir, Prefix: "dataset-"}
	defer tfs.RemoveAll()

	// Make sure to remove temporary files on keyboard interrupt
	atexit(func() { tfs.RemoveAll() })

	// Open the output file
	log.Print("Fetching run to ", destFn)
	output, err := os.Create(destFn)
	if err != nil {
		log.Print("Error creating output: ", err)
		return err
	}

	// Ensure the file is closed on function exit
	defer output.Close()

	// Concatenate temporary files as they are finished
	fetchStart := time.Now()
	for f := range fetchDatasetsData(&tfs, datasets) {
		if input, err := os.Open(f.Name()); err != nil {
			log.Print("Error copying temporary file: ", err)
		} else {
			io.Copy(output, input)
			input.Close()
		}
		tfs.Remove(f)
	}

	fetchDuration := time.Since(fetchStart)
	fi, err := output.Stat()
	if err != nil {
		log.Print("Error: ", err)
		return err
	}
	log.Print(fmt.Sprintf("Overall download speed: %v/sec",
		ByteCount(float64(fi.Size())/fetchDuration.Seconds())))

	return nil
}

func fetchDatasetsData(tfs *TemporaryFileSource, datasets []*aonui.Dataset) chan *os.File {
	// Which records are we interested in?
	paramsOfInterest := syncParameters

	var wg sync.WaitGroup
	tmpFilesChan := make(chan *os.File)

	trySleepDuration, err := time.ParseDuration("10s")
	if err != nil {
		log.Fatal(err)
	}

	for _, ds := range datasets {
		// If we have a max forecast hour, and this dataset is later, skip
		if ds.Run.Source.MaxForecastHour > 0 && ds.ForecastHour > ds.Run.Source.MaxForecastHour {
			continue
		}

		wg.Add(1)

		go func(dataset *aonui.Dataset) {
			defer wg.Done()

			fetchSem <- 1
			defer func() { <-fetchSem }()

			// Perform download. Attempt download repeatedly
			maximumTries := dataset.Run.Source.FetchStrategy.MaximumRetries
			var tmpFile *os.File
			for tries := 0; tries < maximumTries; tries++ {
				// Create a temporary file for output
				tmpFile, err = tfs.Create()
				if err != nil {
					log.Print("Error creating temporary file: ", err)
				}

				log.Print("Fetching ", dataset.Identifier,
					" (try ", tries+1, " of ", maximumTries, ")")
				err := fetchDataset(tmpFile, dataset, paramsOfInterest)
				if err == nil {
					break
				} else {
					log.Print("Error fetching dataset: ", err)
				}

				// Remove this temporary file
				tmpFile.Close()
				tfs.Remove(tmpFile)
				tmpFile = nil

				// Sleep until the next try
				time.Sleep(trySleepDuration)
			}

			if tmpFile == nil {
				log.Print("error: failed to download ", dataset.Identifier)
			} else {
				tmpFile.Close()
				tmpFilesChan <- tmpFile
			}
		}(ds)
	}

	// Launch a goroutine to wait for all datasets to be downloaded and
	// then close the channel.
	go func() {
		wg.Wait()
		close(tmpFilesChan)
	}()

	return tmpFilesChan
}

func fetchDataset(output io.Writer, dataset *aonui.Dataset, paramsOfInterest []string) error {
	// Fetch inventory for this dataset
	inventory, err := dataset.FetchInventory()
	if err != nil {
		return err
	}

	// Calculate which items to save
	var (
		totalToFetch int64
		fetchItems   []*aonui.InventoryItem
	)
	for _, item := range inventory {
		saveItem := false
		for _, poi := range paramsOfInterest {
			for _, p := range item.Parameters {
				saveItem = saveItem || poi == p
			}
		}

		// HACK: we also are only interested in wind velocities at a
		// particular pressure. (i.e. ones whose "LayerName" field is of
		// the form "XXX mb".)
		saveItem = saveItem && strings.HasSuffix(item.LayerName, " mb")

		if saveItem {
			fetchItems = append(fetchItems, item)
			totalToFetch += item.Extent
		}
	}

	if len(fetchItems) == 0 {
		log.Print("No items to fetch")
		return nil
	}

	log.Print(fmt.Sprintf("Fetching %d records from %v (%v)",
		len(fetchItems), dataset.Identifier, ByteCount(totalToFetch)))
	if _, err := dataset.FetchAndWriteRecords(output, fetchItems); err != nil {
		return err
	}

	return nil
}
<|endoftext|>


<commit_before>package git

import (
	"bufio"
	"bytes"
	"fmt"
	"io"
	"io/ioutil"
	"os/exec"
	"strconv"

	"github.com/git-lfs/git-lfs/errors"
	"github.com/rubyist/tracerx"
)

// object represents a generic Git object of any type.
type object struct {
	// Contents reads Git's internal object representation.
	Contents *io.LimitedReader
	// Oid is the ID of the object.
	Oid string
	// Size is the size in bytes of the object.
	Size int64
	// Type is the type of the object being held.
	Type string
}

// ObjectScanner is a scanner type that scans for Git objects reference-able in
// Git's object database by their unique OID.
type ObjectScanner struct {
	// object is the object that the ObjectScanner last scanned, or nil.
	object *object
	// err is the error (if any) that the ObjectScanner encountered during
	// its last scan, or nil.
	err error

	// from is the buffered source of input to the *ObjectScanner. It
	// expects input in the form described by
	// https://git-scm.com/docs/git-cat-file.
	from *bufio.Reader
	// to is a writer which accepts the object's OID to be scanned.
	to io.Writer
	// closeFn is an optional function that is run before the ObjectScanner
	// is closed. It is designated to clean up and close any resources held
	// by the ObjectScanner during runtime.
	closeFn func() error
}

// NewObjectScanner constructs a new instance of the `*ObjectScanner` type and
// returns it. It backs the ObjectScanner with an invocation of the `git
// cat-file --batch` command. If any errors were encountered while starting that
// command, they will be returned immediately.
//
// Otherwise, an `*ObjectScanner` is returned with no error.
func NewObjectScanner() (*ObjectScanner, error) {
	cmd := exec.Command("git", "cat-file", "--batch")
	stdout, err := cmd.StdoutPipe()
	if err != nil {
		return nil, errors.Wrap(err, "open stdout")
	}
	stdin, err := cmd.StdinPipe()
	if err != nil {
		return nil, errors.Wrap(err, "open stdin")
	}

	stderr, err := cmd.StderrPipe()
	if err != nil {
		return nil, errors.Wrap(err, "open stderr")
	}

	closeFn := func() error {
		if err := stdin.Close(); err != nil {
			return err
		}

		msg, _ := ioutil.ReadAll(stderr)
		if err = cmd.Wait(); err != nil {
			return errors.Errorf("Error in git cat-file --batch: %v %v", err, string(msg))
		}

		return nil
	}

	tracerx.Printf("run_command: git cat-file --batch")
	if err := cmd.Start(); err != nil {
		return nil, err
	}

	return &ObjectScanner{
		from: bufio.NewReaderSize(stdout, 16384),
		to:   stdin,

		closeFn: closeFn,
	}, nil
}

// NewObjectScannerFrom returns a new `*ObjectScanner` populated with data from
// the given `io.Reader`, "r". It supplies no close function, and discards any
// input given to the Scan() function.
func NewObjectScannerFrom(r io.Reader) *ObjectScanner {
	return &ObjectScanner{
		from: bufio.NewReader(r),
		to:   ioutil.Discard,
	}
}

// Scan scans for a particular object given by the "oid" parameter. Once the
// scan is complete, the Contents(), Sha1(), Size() and Type() functions may be
// called and will return data corresponding to the given OID.
//
// Scan() returns whether the scan was successful, or in other words, whether or
// not the scanner can continue to progress.
func (s *ObjectScanner) Scan(oid string) bool {
	if err := s.reset(); err != nil {
		s.err = err
		return false
	}

	obj, err := s.scan(oid)
	s.object = obj

	if err != nil {
		if err != io.EOF {
			s.err = err
		}
		return false
	}
	return true
}

// Close closes and frees any resources owned by the *ObjectScanner that it is
// called upon. If there were any errors in freeing that (those) resource(s), it
// it will be returned, otherwise nil.
func (s *ObjectScanner) Close() error {
	if s.closeFn != nil {
		return s.closeFn()
	}
	return nil
}

// Contents returns an io.Reader which reads Git's representation of the object
// that was last scanned for.
func (s *ObjectScanner) Contents() io.Reader {
	return s.object.Contents
}

// Sha1 returns the SHA1 object ID of the object that was last scanned for.
func (s *ObjectScanner) Sha1() string {
	return s.object.Oid
}

// Size returns the size in bytes of the object that was last scanned for.
func (s *ObjectScanner) Size() int64 {
	return s.object.Size
}

// Type returns the type of the object that was last scanned for.
func (s *ObjectScanner) Type() string {
	return s.object.Type
}

// Err returns the error (if any) that was encountered during the last Scan()
// operation.
func (s *ObjectScanner) Err() error { return s.err }

// reset resets the `*ObjectScanner` to scan again by advancing the reader (if
// necessary) and clearing both the object and error fields on the
// `*ObjectScanner` instance.
func (s *ObjectScanner) reset() error {
	if s.object != nil {
		if s.object.Contents != nil {
			remaining := s.object.Contents.N
			if _, err := io.CopyN(ioutil.Discard, s.object.Contents, remaining); err != nil {
				return errors.Wrap(err, "unwind contents")
			}
		}

		// Consume extra LF inserted by cat-file
		if _, err := s.from.ReadByte(); err != nil {
			return err
		}
	}

	s.object, s.err = nil, nil

	return nil
}

// scan scans for and populates a new Git object given an OID.
func (s *ObjectScanner) scan(oid string) (*object, error) {
	if _, err := fmt.Fprintln(s.to, oid); err != nil {
		return nil, err
	}

	l, err := s.from.ReadBytes('\n')
	if err != nil {
		return nil, err
	}

	fields := bytes.Fields(l)
	if len(fields) < 3 {
		return nil, errors.Errorf("invalid line: %q", l)
	}

	oid = string(fields[0])
	typ := string(fields[1])
	size, _ := strconv.Atoi(string(fields[2]))
	contents := io.LimitReader(s.from, int64(size))

	return &object{
		Contents: contents.(*io.LimitedReader),
		Oid:      oid,
		Size:     int64(size),
		Type:     typ,
	}, nil
}
<commit_msg>git: teach 'missingErr' type and IsMissingObject()<commit_after>package git

import (
	"bufio"
	"bytes"
	"fmt"
	"io"
	"io/ioutil"
	"os/exec"
	"strconv"

	"github.com/git-lfs/git-lfs/errors"
	"github.com/rubyist/tracerx"
)

// object represents a generic Git object of any type.
type object struct {
	// Contents reads Git's internal object representation.
	Contents *io.LimitedReader
	// Oid is the ID of the object.
	Oid string
	// Size is the size in bytes of the object.
	Size int64
	// Type is the type of the object being held.
	Type string
}

// ObjectScanner is a scanner type that scans for Git objects reference-able in
// Git's object database by their unique OID.
type ObjectScanner struct {
	// object is the object that the ObjectScanner last scanned, or nil.
	object *object
	// err is the error (if any) that the ObjectScanner encountered during
	// its last scan, or nil.
	err error

	// from is the buffered source of input to the *ObjectScanner. It
	// expects input in the form described by
	// https://git-scm.com/docs/git-cat-file.
	from *bufio.Reader
	// to is a writer which accepts the object's OID to be scanned.
	to io.Writer
	// closeFn is an optional function that is run before the ObjectScanner
	// is closed. It is designated to clean up and close any resources held
	// by the ObjectScanner during runtime.
	closeFn func() error
}

// NewObjectScanner constructs a new instance of the `*ObjectScanner` type and
// returns it. It backs the ObjectScanner with an invocation of the `git
// cat-file --batch` command. If any errors were encountered while starting that
// command, they will be returned immediately.
//
// Otherwise, an `*ObjectScanner` is returned with no error.
func NewObjectScanner() (*ObjectScanner, error) {
	cmd := exec.Command("git", "cat-file", "--batch")
	stdout, err := cmd.StdoutPipe()
	if err != nil {
		return nil, errors.Wrap(err, "open stdout")
	}
	stdin, err := cmd.StdinPipe()
	if err != nil {
		return nil, errors.Wrap(err, "open stdin")
	}

	stderr, err := cmd.StderrPipe()
	if err != nil {
		return nil, errors.Wrap(err, "open stderr")
	}

	closeFn := func() error {
		if err := stdin.Close(); err != nil {
			return err
		}

		msg, _ := ioutil.ReadAll(stderr)
		if err = cmd.Wait(); err != nil {
			return errors.Errorf("Error in git cat-file --batch: %v %v", err, string(msg))
		}

		return nil
	}

	tracerx.Printf("run_command: git cat-file --batch")
	if err := cmd.Start(); err != nil {
		return nil, err
	}

	return &ObjectScanner{
		from: bufio.NewReaderSize(stdout, 16384),
		to:   stdin,

		closeFn: closeFn,
	}, nil
}

// NewObjectScannerFrom returns a new `*ObjectScanner` populated with data from
// the given `io.Reader`, "r". It supplies no close function, and discards any
// input given to the Scan() function.
func NewObjectScannerFrom(r io.Reader) *ObjectScanner {
	return &ObjectScanner{
		from: bufio.NewReader(r),
		to:   ioutil.Discard,
	}
}

// Scan scans for a particular object given by the "oid" parameter. Once the
// scan is complete, the Contents(), Sha1(), Size() and Type() functions may be
// called and will return data corresponding to the given OID.
//
// Scan() returns whether the scan was successful, or in other words, whether or
// not the scanner can continue to progress.
func (s *ObjectScanner) Scan(oid string) bool {
	if err := s.reset(); err != nil {
		s.err = err
		return false
	}

	obj, err := s.scan(oid)
	s.object = obj

	if err != nil {
		if err != io.EOF {
			s.err = err
		}
		return false
	}
	return true
}

// Close closes and frees any resources owned by the *ObjectScanner that it is
// called upon. If there were any errors in freeing that (those) resource(s), it
// it will be returned, otherwise nil.
func (s *ObjectScanner) Close() error {
	if s.closeFn != nil {
		return s.closeFn()
	}
	return nil
}

// Contents returns an io.Reader which reads Git's representation of the object
// that was last scanned for.
func (s *ObjectScanner) Contents() io.Reader {
	return s.object.Contents
}

// Sha1 returns the SHA1 object ID of the object that was last scanned for.
func (s *ObjectScanner) Sha1() string {
	return s.object.Oid
}

// Size returns the size in bytes of the object that was last scanned for.
func (s *ObjectScanner) Size() int64 {
	return s.object.Size
}

// Type returns the type of the object that was last scanned for.
func (s *ObjectScanner) Type() string {
	return s.object.Type
}

// Err returns the error (if any) that was encountered during the last Scan()
// operation.
func (s *ObjectScanner) Err() error { return s.err }

// reset resets the `*ObjectScanner` to scan again by advancing the reader (if
// necessary) and clearing both the object and error fields on the
// `*ObjectScanner` instance.
func (s *ObjectScanner) reset() error {
	if s.object != nil {
		if s.object.Contents != nil {
			remaining := s.object.Contents.N
			if _, err := io.CopyN(ioutil.Discard, s.object.Contents, remaining); err != nil {
				return errors.Wrap(err, "unwind contents")
			}
		}

		// Consume extra LF inserted by cat-file
		if _, err := s.from.ReadByte(); err != nil {
			return err
		}
	}

	s.object, s.err = nil, nil

	return nil
}

type missingErr struct {
	oid string
}

func (m *missingErr) Error() string {
	return fmt.Sprintf("missing object: %s", m.oid)
}

func IsMissingObject(err error) bool {
	_, ok := err.(*missingErr)
	return ok
}

// scan scans for and populates a new Git object given an OID.
func (s *ObjectScanner) scan(oid string) (*object, error) {
	if _, err := fmt.Fprintln(s.to, oid); err != nil {
		return nil, err
	}

	l, err := s.from.ReadBytes('\n')
	if err != nil {
		return nil, err
	}

	fields := bytes.Fields(l)
	if len(fields) < 3 {
		return nil, errors.Errorf("invalid line: %q", l)
	}

	oid = string(fields[0])
	typ := string(fields[1])
	size, _ := strconv.Atoi(string(fields[2]))
	contents := io.LimitReader(s.from, int64(size))

	return &object{
		Contents: contents.(*io.LimitedReader),
		Oid:      oid,
		Size:     int64(size),
		Type:     typ,
	}, nil
}
<|endoftext|>


<commit_before>// Program bpf2go embeds eBPF in Go.
//
// It compiles a C source file into eBPF bytecode and then emits a
// Go file containing the eBPF. The goal is to avoid loading the
// eBPF from disk at runtime and to minimise the amount of manual
// work required to interact with eBPF programs. It takes inspiration
// from `bpftool gen skeleton`.
//
// Invoke the program using go generate:
//    //go:generate go run github.com/cilium/ebpf/cmd/bpf2go foo path/to/src.c -- -I/path/to/include
// This will emit foo_bpfel.go and foo_bpfeb.go, with types using `foo`
// as a stem. The two files contain compiled BPF for little and big
// endian systems, respectively.
//
// You can use environment variables to affect all bpf2go invocations
// across a project, e.g. to set specific C flags:
//    //go:generate go run github.com/cilium/ebpf/cmd/bpf2go -cflags "$BPF_CFLAGS" foo path/to/src.c
// By exporting $BPF_CFLAGS from your build system you can then control
// all builds from a single location.
//
// For a full list of accepted options check the `-help` output. There is a
// fully worked example at https://github.com/cilium/ebpf/tree/master/cmd/bpf2go/example.
package main
<commit_msg>cmd/bpf2go: document required clang version<commit_after>// Program bpf2go embeds eBPF in Go.
//
// It compiles a C source file into eBPF bytecode and then emits a
// Go file containing the eBPF. The goal is to avoid loading the
// eBPF from disk at runtime and to minimise the amount of manual
// work required to interact with eBPF programs. It takes inspiration
// from `bpftool gen skeleton`.
//
// Invoke the program using go generate:
//    //go:generate go run github.com/cilium/ebpf/cmd/bpf2go foo path/to/src.c -- -I/path/to/include
// This will emit foo_bpfel.go and foo_bpfeb.go, with types using `foo`
// as a stem. The two files contain compiled BPF for little and big
// endian systems, respectively.
//
// You can use environment variables to affect all bpf2go invocations
// across a project, e.g. to set specific C flags:
//    //go:generate go run github.com/cilium/ebpf/cmd/bpf2go -cflags "$BPF_CFLAGS" foo path/to/src.c
// By exporting $BPF_CFLAGS from your build system you can then control
// all builds from a single location.
//
// Requires at least clang 9.
//
// For a full list of accepted options check the `-help` output. There is a
// fully worked example at https://github.com/cilium/ebpf/tree/master/cmd/bpf2go/example.
package main
<|endoftext|>


<commit_before>// Copyright 2013 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"bufio"
	"bytes"
	"fmt"
	"go/build"
	"html/template"
	"io"
	"io/ioutil"
	"math"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"runtime"
	"sort"
	"strconv"
)

// htmlOutput reads the profile data from profile and generates an HTML
// coverage report, writing it to outfile. If outfile is empty,
// it writes the report to a temporary file and opens it in a web browser.
func htmlOutput(profile, outfile string) error {
	pf, err := os.Open(profile)
	if err != nil {
		return err
	}
	defer pf.Close()

	profiles, err := ParseProfiles(pf)
	if err != nil {
		return err
	}

	var files []*templateFile

	for fn, profile := range profiles {
		dir, file := filepath.Split(fn)
		pkg, err := build.Import(dir, ".", build.FindOnly)
		if err != nil {
			return fmt.Errorf("can't find %q: %v", fn, err)
		}
		src, err := ioutil.ReadFile(filepath.Join(pkg.Dir, file))
		if err != nil {
			return fmt.Errorf("can't read %q: %v", fn, err)
		}
		var buf bytes.Buffer
		err = htmlGen(&buf, src, profile.Tokens(src))
		if err != nil {
			return err
		}
		files = append(files, &templateFile{
			Name: fn,
			Body: template.HTML(buf.String()),
		})
	}

	var out *os.File
	if outfile == "" {
		var dir string
		dir, err = ioutil.TempDir("", "cover")
		if err != nil {
			return err
		}
		out, err = os.Create(filepath.Join(dir, "coverage.html"))
	} else {
		out, err = os.Create(outfile)
	}
	err = htmlTemplate.Execute(out, templateData{Files: files})
	if err == nil {
		err = out.Close()
	}
	if err != nil {
		return err
	}

	if outfile == "" {
		if !startBrowser("file://" + out.Name()) {
			fmt.Fprintf(os.Stderr, "HTML output written to %s\n", out.Name())
		}
	}

	return nil
}

// Profile represents the profiling data for a specific file.
type Profile struct {
	Blocks []ProfileBlock
}

// ProfileBlock represents a single block of profiling data.
type ProfileBlock struct {
	StartLine, StartCol int
	EndLine, EndCol     int
	NumStmt, Count      int
}

// ParseProfiles parses profile data from the given Reader and returns a
// Profile for each file.
func ParseProfiles(r io.Reader) (map[string]*Profile, error) {
	files := make(map[string]*Profile)
	buf := bufio.NewReader(r)
	// First line is mode.
	mode, err := buf.ReadString('\n')
	if err != nil {
		return nil, err
	}
	_ = mode // TODO: Use the mode to affect the display.
	// Rest of file is in the format
	//	encoding/base64/base64.go:34.44,37.40 3 1
	// where the fields are: name.go:line.column,line.column numberOfStatements count
	s := bufio.NewScanner(buf)
	for s.Scan() {
		line := s.Text()
		m := lineRe.FindStringSubmatch(line)
		if m == nil {
			return nil, fmt.Errorf("line %q doesn't match expected format: %v", m, lineRe)
		}
		fn := m[1]
		p := files[fn]
		if p == nil {
			p = new(Profile)
			files[fn] = p
		}
		p.Blocks = append(p.Blocks, ProfileBlock{
			StartLine: toInt(m[2]),
			StartCol:  toInt(m[3]),
			EndLine:   toInt(m[4]),
			EndCol:    toInt(m[5]),
			NumStmt:   toInt(m[6]),
			Count:     toInt(m[7]),
		})
	}
	if err := s.Err(); err != nil {
		return nil, err
	}
	for _, p := range files {
		sort.Sort(blocksByStart(p.Blocks))
	}
	return files, nil
}

type blocksByStart []ProfileBlock

func (b blocksByStart) Len() int      { return len(b) }
func (b blocksByStart) Swap(i, j int) { b[i], b[j] = b[j], b[i] }
func (b blocksByStart) Less(i, j int) bool {
	return b[i].StartLine < b[j].StartLine || b[i].StartLine == b[j].StartLine && b[i].StartCol < b[j].StartCol
}

var lineRe = regexp.MustCompile(`^(.+):([0-9]+).([0-9]+),([0-9]+).([0-9]+) ([0-9]+) ([0-9]+)$`)

func toInt(s string) int {
	i, err := strconv.ParseInt(s, 10, 64)
	if err != nil {
		panic(err)
	}
	return int(i)
}

// Token represents the position in a source file of an opening or closing
//  tag. These are used to colorize the source.
type Token struct {
	Pos   int
	Start bool
	Count int
	Norm  float64 // count normalized to 0-1
}

// Tokens returns a Profile as a set of Tokens within the provided src.
func (p *Profile) Tokens(src []byte) (tokens []Token) {
	// Find maximum counts.
	max := 0
	for _, b := range p.Blocks {
		if b.Count > max {
			max = b.Count
		}
	}
	// Divisor for normalization.
	divisor := math.Log(float64(max + 1))

	// tok returns a Token, populating the Norm field with a normalized Count.
	tok := func(pos int, start bool, count int) Token {
		t := Token{Pos: pos, Start: start, Count: count}
		if !start || count == 0 {
			return t
		}
		if max == 1 {
			t.Norm = 0.4 // "set" mode; use pale color
		} else {
			t.Norm = math.Log(float64(count+1)) / divisor
		}
		return t
	}

	line, col := 1, 2
	for si, bi := 0, 0; si < len(src) && bi < len(p.Blocks); {
		b := p.Blocks[bi]
		if b.StartLine == line && b.StartCol == col {
			tokens = append(tokens, tok(si, true, b.Count))
		}
		if b.EndLine == line && b.EndCol == col {
			tokens = append(tokens, tok(si, false, 0))
			bi++
			continue // Don't advance through src; maybe the next block starts here.
		}
		if src[si] == '\n' {
			line++
			col = 0
		}
		col++
		si++
	}
	sort.Sort(tokensByPos(tokens))
	return
}

type tokensByPos []Token

func (t tokensByPos) Len() int      { return len(t) }
func (t tokensByPos) Swap(i, j int) { t[i], t[j] = t[j], t[i] }
func (t tokensByPos) Less(i, j int) bool {
	if t[i].Pos == t[j].Pos {
		return !t[i].Start && t[j].Start
	}
	return t[i].Pos < t[j].Pos
}

// htmlGen generates an HTML coverage report with the provided filename,
// source code, and tokens, and writes it to the given Writer.
func htmlGen(w io.Writer, src []byte, tokens []Token) error {
	dst := bufio.NewWriter(w)
	for i := range src {
		for len(tokens) > 0 && tokens[0].Pos == i {
			t := tokens[0]
			if t.Start {
				n := 0
				if t.Count > 0 {
					n = int(math.Floor(t.Norm*10)) + 1
				}
				fmt.Fprintf(dst, `<span class="cov%v" title="%v">`, n, t.Count)
			} else {
				dst.WriteString("")
			}
			tokens = tokens[1:]
		}
		switch b := src[i]; b {
		case '>':
			dst.WriteString(">")
		case '<':
			dst.WriteString("<")
		case '&':
			dst.WriteString("&")
		case '\t':
			dst.WriteString("        ")
		default:
			dst.WriteByte(b)
		}
	}
	return dst.Flush()
}

// startBrowser tries to open the URL in a browser
// and returns whether it succeed.
func startBrowser(url string) bool {
	// try to start the browser
	var args []string
	switch runtime.GOOS {
	case "darwin":
		args = []string{"open"}
	case "windows":
		args = []string{"cmd", "/c", "start"}
	default:
		args = []string{"xdg-open"}
	}
	cmd := exec.Command(args[0], append(args[1:], url)...)
	return cmd.Start() == nil
}

// rgb returns an rgb value for the specified coverage value
// between 0 (no coverage) and 11 (max coverage).
func rgb(n int) string {
	if n == 0 {
		return "rgb(255, 0, 0)" // Red
	}
	// Gradient from pale green (low count)
	// to bright green (high count)
	r := 240 - 22*(n-1)
	b := 170 + r/3
	return fmt.Sprintf("rgb(%v, 255, %v)", r, b)
}

// colors generates the CSS rules for coverage colors.
func colors() template.CSS {
	var buf bytes.Buffer
	for i := 0; i < 12; i++ {
		fmt.Fprintf(&buf, ".cov%v { color: %v }\n", i, rgb(i))
	}
	return template.CSS(buf.String())
}

var htmlTemplate = template.Must(template.New("html").Funcs(template.FuncMap{
	"colors": colors,
}).Parse(tmplHTML))

type templateData struct {
	Files []*templateFile
}

type templateFile struct {
	Name string
	Body template.HTML
}

const tmplHTML = `
<!DOCTYPE html>

			<select id="files">
			{{range $i, $f := .Files}}
			<option value="file{{$i}}">{{$f.Name}}</option>
			{{end}}
			</select>

		{{range $i, $f := .Files}}

{{$f.Body}}

		{{end}}

`
<commit_msg>go.tools/cmd/cover: change color scheme and add legend<commit_after>// Copyright 2013 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"bufio"
	"bytes"
	"fmt"
	"go/build"
	"html/template"
	"io"
	"io/ioutil"
	"math"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"runtime"
	"sort"
	"strconv"
)

// htmlOutput reads the profile data from profile and generates an HTML
// coverage report, writing it to outfile. If outfile is empty,
// it writes the report to a temporary file and opens it in a web browser.
func htmlOutput(profile, outfile string) error {
	pf, err := os.Open(profile)
	if err != nil {
		return err
	}
	defer pf.Close()

	profiles, err := ParseProfiles(pf)
	if err != nil {
		return err
	}

	var files []*templateFile

	for fn, profile := range profiles {
		dir, file := filepath.Split(fn)
		pkg, err := build.Import(dir, ".", build.FindOnly)
		if err != nil {
			return fmt.Errorf("can't find %q: %v", fn, err)
		}
		src, err := ioutil.ReadFile(filepath.Join(pkg.Dir, file))
		if err != nil {
			return fmt.Errorf("can't read %q: %v", fn, err)
		}
		var buf bytes.Buffer
		err = htmlGen(&buf, src, profile.Tokens(src))
		if err != nil {
			return err
		}
		files = append(files, &templateFile{
			Name: fn,
			Body: template.HTML(buf.String()),
		})
	}

	var out *os.File
	if outfile == "" {
		var dir string
		dir, err = ioutil.TempDir("", "cover")
		if err != nil {
			return err
		}
		out, err = os.Create(filepath.Join(dir, "coverage.html"))
	} else {
		out, err = os.Create(outfile)
	}
	err = htmlTemplate.Execute(out, templateData{Files: files})
	if err == nil {
		err = out.Close()
	}
	if err != nil {
		return err
	}

	if outfile == "" {
		if !startBrowser("file://" + out.Name()) {
			fmt.Fprintf(os.Stderr, "HTML output written to %s\n", out.Name())
		}
	}

	return nil
}

// Profile represents the profiling data for a specific file.
type Profile struct {
	Blocks []ProfileBlock
}

// ProfileBlock represents a single block of profiling data.
type ProfileBlock struct {
	StartLine, StartCol int
	EndLine, EndCol     int
	NumStmt, Count      int
}

// ParseProfiles parses profile data from the given Reader and returns a
// Profile for each file.
func ParseProfiles(r io.Reader) (map[string]*Profile, error) {
	files := make(map[string]*Profile)
	buf := bufio.NewReader(r)
	// First line is mode.
	mode, err := buf.ReadString('\n')
	if err != nil {
		return nil, err
	}
	_ = mode // TODO: Use the mode to affect the display.
	// Rest of file is in the format
	//	encoding/base64/base64.go:34.44,37.40 3 1
	// where the fields are: name.go:line.column,line.column numberOfStatements count
	s := bufio.NewScanner(buf)
	for s.Scan() {
		line := s.Text()
		m := lineRe.FindStringSubmatch(line)
		if m == nil {
			return nil, fmt.Errorf("line %q doesn't match expected format: %v", m, lineRe)
		}
		fn := m[1]
		p := files[fn]
		if p == nil {
			p = new(Profile)
			files[fn] = p
		}
		p.Blocks = append(p.Blocks, ProfileBlock{
			StartLine: toInt(m[2]),
			StartCol:  toInt(m[3]),
			EndLine:   toInt(m[4]),
			EndCol:    toInt(m[5]),
			NumStmt:   toInt(m[6]),
			Count:     toInt(m[7]),
		})
	}
	if err := s.Err(); err != nil {
		return nil, err
	}
	for _, p := range files {
		sort.Sort(blocksByStart(p.Blocks))
	}
	return files, nil
}

type blocksByStart []ProfileBlock

func (b blocksByStart) Len() int      { return len(b) }
func (b blocksByStart) Swap(i, j int) { b[i], b[j] = b[j], b[i] }
func (b blocksByStart) Less(i, j int) bool {
	return b[i].StartLine < b[j].StartLine || b[i].StartLine == b[j].StartLine && b[i].StartCol < b[j].StartCol
}

var lineRe = regexp.MustCompile(`^(.+):([0-9]+).([0-9]+),([0-9]+).([0-9]+) ([0-9]+) ([0-9]+)$`)

func toInt(s string) int {
	i, err := strconv.ParseInt(s, 10, 64)
	if err != nil {
		panic(err)
	}
	return int(i)
}

// Token represents the position in a source file of an opening or closing
//  tag. These are used to colorize the source.
type Token struct {
	Pos   int
	Start bool
	Count int
	Norm  float64 // count normalized to 0-1
}

// Tokens returns a Profile as a set of Tokens within the provided src.
func (p *Profile) Tokens(src []byte) (tokens []Token) {
	// Find maximum counts.
	max := 0
	for _, b := range p.Blocks {
		if b.Count > max {
			max = b.Count
		}
	}
	// Divisor for normalization.
	divisor := math.Log(float64(max + 1))

	// tok returns a Token, populating the Norm field with a normalized Count.
	tok := func(pos int, start bool, count int) Token {
		t := Token{Pos: pos, Start: start, Count: count}
		if !start || count == 0 {
			return t
		}
		if max == 1 {
			t.Norm = 0.4 // "set" mode; use pale color
		} else {
			t.Norm = math.Log(float64(count+1)) / divisor
		}
		return t
	}

	line, col := 1, 2
	for si, bi := 0, 0; si < len(src) && bi < len(p.Blocks); {
		b := p.Blocks[bi]
		if b.StartLine == line && b.StartCol == col {
			tokens = append(tokens, tok(si, true, b.Count))
		}
		if b.EndLine == line && b.EndCol == col {
			tokens = append(tokens, tok(si, false, 0))
			bi++
			continue // Don't advance through src; maybe the next block starts here.
		}
		if src[si] == '\n' {
			line++
			col = 0
		}
		col++
		si++
	}
	sort.Sort(tokensByPos(tokens))
	return
}

type tokensByPos []Token

func (t tokensByPos) Len() int      { return len(t) }
func (t tokensByPos) Swap(i, j int) { t[i], t[j] = t[j], t[i] }
func (t tokensByPos) Less(i, j int) bool {
	if t[i].Pos == t[j].Pos {
		return !t[i].Start && t[j].Start
	}
	return t[i].Pos < t[j].Pos
}

// htmlGen generates an HTML coverage report with the provided filename,
// source code, and tokens, and writes it to the given Writer.
func htmlGen(w io.Writer, src []byte, tokens []Token) error {
	dst := bufio.NewWriter(w)
	for i := range src {
		for len(tokens) > 0 && tokens[0].Pos == i {
			t := tokens[0]
			if t.Start {
				n := 0
				if t.Count > 0 {
					n = int(math.Floor(t.Norm*10)) + 1
				}
				fmt.Fprintf(dst, `<span class="cov%v" title="%v">`, n, t.Count)
			} else {
				dst.WriteString("")
			}
			tokens = tokens[1:]
		}
		switch b := src[i]; b {
		case '>':
			dst.WriteString(">")
		case '<':
			dst.WriteString("<")
		case '&':
			dst.WriteString("&")
		case '\t':
			dst.WriteString("        ")
		default:
			dst.WriteByte(b)
		}
	}
	return dst.Flush()
}

// startBrowser tries to open the URL in a browser
// and returns whether it succeed.
func startBrowser(url string) bool {
	// try to start the browser
	var args []string
	switch runtime.GOOS {
	case "darwin":
		args = []string{"open"}
	case "windows":
		args = []string{"cmd", "/c", "start"}
	default:
		args = []string{"xdg-open"}
	}
	cmd := exec.Command(args[0], append(args[1:], url)...)
	return cmd.Start() == nil
}

// rgb returns an rgb value for the specified coverage value
// between 0 (no coverage) and 11 (max coverage).
func rgb(n int) string {
	if n == 0 {
		return "rgb(255, 0, 0)" // Red
	}
	// Gradient from pale blue (low count) to yellow (high count)
	r := 185 + 7*(n-1)
	g := 185 + 7*(n-1)
	b := 5 + 25*(11-n)
	return fmt.Sprintf("rgb(%v, %v, %v)", r, g, b)
}

// colors generates the CSS rules for coverage colors.
func colors() template.CSS {
	var buf bytes.Buffer
	for i := 0; i < 12; i++ {
		fmt.Fprintf(&buf, ".cov%v { color: %v }\n", i, rgb(i))
	}
	return template.CSS(buf.String())
}

var htmlTemplate = template.Must(template.New("html").Funcs(template.FuncMap{
	"colors": colors,
}).Parse(tmplHTML))

type templateData struct {
	Files []*templateFile
}

type templateFile struct {
	Name string
	Body template.HTML
}

const tmplHTML = `
<!DOCTYPE html>

			<select id="files">
			{{range $i, $f := .Files}}
			<option value="file{{$i}}">{{$f.Name}}</option>
			{{end}}
			</select>

				not tracked
				no coverage
				low coverage
				*
				*
				*
				*
				*
				*
				*
				*
				*
				high coverage

		{{range $i, $f := .Files}}

{{$f.Body}}

		{{end}}

`
<|endoftext|>