<commit_before>// Copyright 2015 Aaron Jacobs. All Rights Reserved.
// Author: aaronjjacobs@gmail.com (Aaron Jacobs)
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package graph

import (
	"errors"
	"fmt"
	"sync"

	"github.com/jacobsa/gcloud/syncutil"
	"golang.org/x/net/context"
)

// A visitor in a directed graph whose nodes are identified by strings.
type Visitor interface {
	// Process the supplied node and return a list of direct successors.
	Visit(ctx context.Context, node string) (adjacent []string, err error)
}

// Invoke v.Visit on each node reachable from the supplied search roots,
// including the roots themselves. Use the supplied degree of parallelism.
//
// It is guaranteed that if a node N is fed to v.Visit, then either:
//
//  *  N is an element of roots, or
//  *  There exists a direct predecessor N' of N such that v.Visit(N') was
//     called and returned successfully.
//
// In particular, if the graph is a rooted tree and searching starts at the
// root, then parents will be successfully visited before children are visited.
// However note that in arbitrary DAGs it is *not* guaranteed that all of a
// node's predecessors have been visited before it is.
func Traverse(
	ctx context.Context,
	parallelism int,
	roots []string,
	v Visitor) (err error) {
	b := syncutil.NewBundle(ctx)
	defer func() { err = b.Join() }()

	// Set up initial state.
	ts := &traverseState{
		admitted: make(map[string]struct{}),
	}

	ts.mu = syncutil.NewInvariantMutex(ts.checkInvariants)
	ts.cond.L = &ts.mu

	for _, r := range roots {
		ts.admitted[r] = struct{}{}
		ts.toVisit = append(ts.toVisit, r)
	}

	// Ensure that ts.cancelled is set when the context is cancelled (or when we
	// return from this function, if the context will never be cancelled).
	done := ctx.Done()
	if done == nil {
		doneChan := make(chan struct{})
		defer close(doneChan)

		done = doneChan
	}

	go watchForCancel(done, ts)

	// Run the appropriate number of workers.
	for i := 0; i < parallelism; i++ {
		b.Add(func(ctx context.Context) (err error) {
			err = traverse(ctx, ts, v)
			return
		})
	}

	return
}

////////////////////////////////////////////////////////////////////////
// traverseState
////////////////////////////////////////////////////////////////////////

// State shared by each traverse worker.
type traverseState struct {
	mu syncutil.InvariantMutex

	// All nodes that have ever been seen. If a node is in this map, it will
	// eventually be visted (barring errors returned by the visitor).
	//
	// GUARDED_BY(mu)
	admitted map[string]struct{}

	// Admitted nodes that have yet to be visted.
	//
	// INVARIANT: For each n in toVisit, n is a key of admitted.
	//
	// GUARDED_BY(mu)
	toVisit []string

	// Set to true if the context has been cancelled. All workers should return
	// when this happens.
	//
	// GUARDED_BY(mu)
	cancelled bool

	// The number of workers that are doing something besides waiting on a node
	// to visit. If this hits zero with toVisit empty, it means that there is
	// nothing further to do.
	//
	// GUARDED_BY(mu)
	busyWorkers int

	// Signalled with mu held when any of the following state changes:
	//
	//  *  toVisit
	//  *  cancelled
	//  *  busyWorkers
	//
	// GUARDED_BY(mu)
	cond sync.Cond
}

// LOCKS_REQUIRED(ts.mu)
func (ts *traverseState) checkInvariants() {
	// INVARIANT: For each n in toVisit, n is a key of admitted.
	for _, n := range ts.toVisit {
		if _, ok := ts.admitted[n]; !ok {
			panic(fmt.Sprintf("Expected %q to be in admitted map", n))
		}
	}
}

// Is there anything that needs a worker's attention?
//
// LOCKS_REQUIRED(ts.mu)
func (ts *traverseState) shouldWake() bool {
	return len(ts.toVisit) != 0 || ts.cancelled || ts.busyWorkers == 0
}

// Sleep until there's something interesting for a traverse worker.
//
// LOCKS_REQUIRED(ts.mu)
func (ts *traverseState) waitForSomethingToDo() {
	for !ts.shouldWake() {
		ts.cond.Wait()
	}
}

////////////////////////////////////////////////////////////////////////
// Helpers
////////////////////////////////////////////////////////////////////////

// REQUIRES: len(ts.toVisit) > 0
//
// LOCKS_REQUIRED(ts.mu)
func visitOne(
	ctx context.Context,
	ts *traverseState,
	v Visitor) (err error) {
	// Mark this worker as busy for the duration of this function.
	ts.busyWorkers++
	defer func() { ts.busyWorkers-- }()

	// Extract the node to visit.
	l := len(ts.toVisit)
	node := ts.toVisit[l-1]
	ts.toVisit = ts.toVisit[:l-1]

	// Unlock while visiting.
	ts.mu.Unlock()
	adjacent, err := v.Visit(ctx, node)
	ts.mu.Lock()

	// Enqueue the adjacent nodes that we haven't already admitted.
	for _, n := range adjacent {
		if _, ok := ts.admitted[n]; !ok {
			ts.toVisit = append(ts.toVisit, n)
			ts.admitted[n] = struct{}{}
		}
	}

	return
}

// A single traverse worker.
func traverse(
	ctx context.Context,
	ts *traverseState,
	v Visitor) (err error) {
	ts.mu.Lock()
	defer ts.mu.Unlock()

	for {
		// Wait for something to do.
		ts.waitForSomethingToDo()

		// Why were we awoken?
		switch {
		// Return immediately if cancelled.
		case ts.cancelled:
			err = errors.New("Cancelled")

		// Otherwise, handle work if it exists.
		case len(ts.toVisit) != 0:
			err = visitOne(ctx, ts, v)
			if err != nil {
				return
			}

		// Otherwise, are we done?
		case ts.busyWorkers == 0:
			return

		default:
			panic("Unexpected wake-up")
		}
	}
}

// Bridge context cancellation with traverseState.cancelled.
func watchForCancel(
	done <-chan struct{},
	ts *traverseState) {
	<-done

	ts.mu.Lock()
	defer ts.mu.Unlock()
	ts.cancelled = true
}
<commit_msg>Fixed wakeup bugs.<commit_after>// Copyright 2015 Aaron Jacobs. All Rights Reserved.
// Author: aaronjjacobs@gmail.com (Aaron Jacobs)
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package graph

import (
	"errors"
	"fmt"
	"sync"

	"github.com/jacobsa/gcloud/syncutil"
	"golang.org/x/net/context"
)

// A visitor in a directed graph whose nodes are identified by strings.
type Visitor interface {
	// Process the supplied node and return a list of direct successors.
	Visit(ctx context.Context, node string) (adjacent []string, err error)
}

// Invoke v.Visit on each node reachable from the supplied search roots,
// including the roots themselves. Use the supplied degree of parallelism.
//
// It is guaranteed that if a node N is fed to v.Visit, then either:
//
//  *  N is an element of roots, or
//  *  There exists a direct predecessor N' of N such that v.Visit(N') was
//     called and returned successfully.
//
// In particular, if the graph is a rooted tree and searching starts at the
// root, then parents will be successfully visited before children are visited.
// However note that in arbitrary DAGs it is *not* guaranteed that all of a
// node's predecessors have been visited before it is.
func Traverse(
	ctx context.Context,
	parallelism int,
	roots []string,
	v Visitor) (err error) {
	b := syncutil.NewBundle(ctx)
	defer func() { err = b.Join() }()

	// Set up initial state.
	ts := &traverseState{
		admitted: make(map[string]struct{}),
	}

	ts.mu = syncutil.NewInvariantMutex(ts.checkInvariants)
	ts.cond.L = &ts.mu

	for _, r := range roots {
		ts.admitted[r] = struct{}{}
		ts.toVisit = append(ts.toVisit, r)
	}

	// Ensure that ts.cancelled is set when the context is cancelled (or when we
	// return from this function, if the context will never be cancelled).
	done := ctx.Done()
	if done == nil {
		doneChan := make(chan struct{})
		defer close(doneChan)

		done = doneChan
	}

	go watchForCancel(done, ts)

	// Run the appropriate number of workers.
	for i := 0; i < parallelism; i++ {
		b.Add(func(ctx context.Context) (err error) {
			err = traverse(ctx, ts, v)
			return
		})
	}

	return
}

////////////////////////////////////////////////////////////////////////
// traverseState
////////////////////////////////////////////////////////////////////////

// State shared by each traverse worker.
type traverseState struct {
	mu syncutil.InvariantMutex

	// All nodes that have ever been seen. If a node is in this map, it will
	// eventually be visted (barring errors returned by the visitor).
	//
	// GUARDED_BY(mu)
	admitted map[string]struct{}

	// Admitted nodes that have yet to be visted.
	//
	// INVARIANT: For each n in toVisit, n is a key of admitted.
	//
	// GUARDED_BY(mu)
	toVisit []string

	// Set to true if the context has been cancelled. All workers should return
	// when this happens.
	//
	// GUARDED_BY(mu)
	cancelled bool

	// The number of workers that are doing something besides waiting on a node
	// to visit. If this hits zero with toVisit empty, it means that there is
	// nothing further to do.
	//
	// GUARDED_BY(mu)
	busyWorkers int

	// Broadcasted to with mu held when any of the following state changes:
	//
	//  *  toVisit
	//  *  cancelled
	//  *  busyWorkers
	//
	// GUARDED_BY(mu)
	cond sync.Cond
}

// LOCKS_REQUIRED(ts.mu)
func (ts *traverseState) checkInvariants() {
	// INVARIANT: For each n in toVisit, n is a key of admitted.
	for _, n := range ts.toVisit {
		if _, ok := ts.admitted[n]; !ok {
			panic(fmt.Sprintf("Expected %q to be in admitted map", n))
		}
	}
}

// Is there anything that needs a worker's attention?
//
// LOCKS_REQUIRED(ts.mu)
func (ts *traverseState) shouldWake() bool {
	return len(ts.toVisit) != 0 || ts.cancelled || ts.busyWorkers == 0
}

// Sleep until there's something interesting for a traverse worker.
//
// LOCKS_REQUIRED(ts.mu)
func (ts *traverseState) waitForSomethingToDo() {
	for !ts.shouldWake() {
		ts.cond.Wait()
	}
}

////////////////////////////////////////////////////////////////////////
// Helpers
////////////////////////////////////////////////////////////////////////

// REQUIRES: len(ts.toVisit) > 0
//
// LOCKS_REQUIRED(ts.mu)
func visitOne(
	ctx context.Context,
	ts *traverseState,
	v Visitor) (err error) {
	// Mark this worker as busy for the duration of this function.
	ts.busyWorkers++
	defer func() { ts.busyWorkers-- }()

	// Extract the node to visit.
	l := len(ts.toVisit)
	node := ts.toVisit[l-1]
	ts.toVisit = ts.toVisit[:l-1]
	ts.cond.Broadcast()

	// Unlock while visiting.
	ts.mu.Unlock()
	adjacent, err := v.Visit(ctx, node)
	ts.mu.Lock()

	// Enqueue the adjacent nodes that we haven't already admitted.
	for _, n := range adjacent {
		if _, ok := ts.admitted[n]; !ok {
			ts.toVisit = append(ts.toVisit, n)
			ts.admitted[n] = struct{}{}
			ts.cond.Broadcast()
		}
	}

	return
}

// A single traverse worker.
func traverse(
	ctx context.Context,
	ts *traverseState,
	v Visitor) (err error) {
	ts.mu.Lock()
	defer ts.mu.Unlock()

	for {
		// Wait for something to do.
		ts.waitForSomethingToDo()

		// Why were we awoken?
		switch {
		// Return immediately if cancelled.
		case ts.cancelled:
			err = errors.New("Cancelled")

		// Otherwise, handle work if it exists.
		case len(ts.toVisit) != 0:
			err = visitOne(ctx, ts, v)
			if err != nil {
				return
			}

		// Otherwise, are we done?
		case ts.busyWorkers == 0:
			return

		default:
			panic("Unexpected wake-up")
		}
	}
}

// Bridge context cancellation with traverseState.cancelled.
func watchForCancel(
	done <-chan struct{},
	ts *traverseState) {
	<-done

	ts.mu.Lock()
	defer ts.mu.Unlock()
	ts.cancelled = true
	ts.cond.Broadcast()
}
<|endoftext|>


<commit_before>// Package irc implements parsing and encoding of IRC protocol messages.
package irc

import (
	"fmt"
	"strings"
)

// MaxLineLength is the maximum protocol message line length.
// From RFC 2812 section 2.3. It includes CRLF.
const MaxLineLength = 512

// Message holds a protocol message.
// See section 2.3.1 in RFC 2812.
type Message struct {
	// Prefix may be blank. It's optional.
	Prefix string

	Command string

	// There are at most 15 parameters.
	Params []string
}

func (m Message) String() string {
	return fmt.Sprintf("Prefix [%s] Command [%s] Params%q", m.Prefix, m.Command,
		m.Params)
}

// Encode turns the in memory representation of the memory into the IRC
// protocol message string.
//
// It does not enforce command specific semantics. It is instead responsible
// only for placing prefix, command, and parameters.
func (m Message) Encode() (string, error) {
	s := ""

	if len(m.Prefix) > 0 {
		s += ":" + m.Prefix + " "
	}

	s += m.Command

	for i, param := range m.Params {
		// If we have a ":" or " ", then we need to prefix the parameter with ":".
		// This is only valid for the last ("trailing") parameter.
		//
		// Also, the trailing parameter may be an empty string. Ensure it shows up
		// by adding a :. This can happen e.g. from ircd-ratbox in TOPIC unset
		// command // (server protocol). Modern IRC and ircv3 both specify this as
		// valid (that the last parameter may be empty). Also, RFC 2812's grammar
		// permits it. The ":" is optional only if this is the 15th parameter. But
		// it is valid.
		if idx := strings.IndexAny(param, ": "); idx != -1 || len(param) == 0 {
			s += " :" + param

			// This must be the last parameter.
			if i+1 != len(m.Params) {
				return "", fmt.Errorf("Parameter problem: ':' or ' ' outside last parameter.")
			}
			continue
		}

		s += " " + param
	}

	s += "\r\n"

	if len(s) > MaxLineLength {
		return "", fmt.Errorf("Message after encoding is too long (%d bytes)",
			len(s))
	}

	return s, nil
}

// ParseMessage parses a protocol message from the client/server.
//
// See RFC 2812 Section 2.3.1.
//
// line ends with \n.
func ParseMessage(line string) (Message, error) {
	if len(line) > MaxLineLength {
		return Message{}, fmt.Errorf("Line exceeds maximum length: %d vs. %d",
			len(line), MaxLineLength)
	}

	line, err := fixLineEnding(line)
	if err != nil {
		return Message{}, fmt.Errorf("Line does not have a valid ending: %s", line)
	}

	message := Message{}
	index := 0

	// It is optional to have a prefix.
	if line[0] == ':' {
		prefix, prefixIndex, err := parsePrefix(line)
		if err != nil {
			return Message{}, fmt.Errorf("Problem parsing prefix: %s", err.Error())
		}
		index = prefixIndex

		message.Prefix = prefix

		if index >= len(line) {
			return Message{}, fmt.Errorf("Malformed message. Prefix only.")
		}
	}

	// We've either parsed a prefix out or have no prefix.
	command, index, err := parseCommand(line, index)
	if err != nil {
		return Message{}, fmt.Errorf("Problem parsing command: %s", err)
	}

	message.Command = command

	// May have params.
	params, index, err := parseParams(line, index)
	if err != nil {
		return Message{}, fmt.Errorf("Problem parsing params: %s", err)
	}

	message.Params = params

	// We should now have CRLF.
	// index should be pointing at the CR after parsing params.
	if index != len(line)-2 || line[index] != '\r' || line[index+1] != '\n' {
		return Message{}, fmt.Errorf("Malformed message. No CRLF found. Looking for end at position %d.", index)
	}

	return message, nil
}

// fixLineEnding tries to ensure the line ends with CRLF.
//
// If it ends with only LF, add a CR.
func fixLineEnding(line string) (string, error) {
	if len(line) == 0 {
		return "", fmt.Errorf("Line is blank")
	}

	if len(line) == 1 {
		if line[0] == '\n' {
			return "\r\n", nil
		}

		return "", fmt.Errorf("Line does not end with LF")
	}

	lastIndex := len(line) - 1
	secondLastIndex := lastIndex - 1

	if line[secondLastIndex] == '\r' && line[lastIndex] == '\n' {
		return line, nil
	}

	if line[lastIndex] == '\n' {
		return line[:lastIndex] + "\r\n", nil
	}

	return "", fmt.Errorf("Line has no ending CRLF or LF")
}

// parsePrefix parses out the prefix portion of a string.
//
// line begins with : and ends with \n.
//
// If there is no error we return the prefix and the position after
// the SPACE.
// This means the index points to the first character of the command (in a well
// formed message). We do not confirm there actually is a character.
//
// We are parsing this:
// message    =  [ ":" prefix SPACE ] command [ params ] crlf
// prefix     =  servername / ( nickname [ [ "!" user ] "@" host ] )
//
// TODO: Enforce length limits
// TODO: Enforce character / format more strictly.
//   Right now I don't do much other than ensure there is no space.
func parsePrefix(line string) (string, int, error) {
	pos := 0

	if line[pos] != ':' {
		return "", -1, fmt.Errorf("Line does not start with :")
	}

	for pos < len(line) {
		// Prefix ends with a space.
		if line[pos] == ' ' {
			break
		}

		// Basic character check.
		// I'm being very lenient here right now. Servername and hosts should only
		// allow [a-zA-Z0-9]. Nickname can have any except NUL, CR, LF, " ". I
		// choose to accept anything nicks can.
		if line[pos] == '\x00' || line[pos] == '\n' || line[pos] == '\r' {
			return "", -1, fmt.Errorf("Invalid character found: [%q]", line[pos])
		}

		pos++
	}

	// We didn't find a space.
	if pos == len(line) {
		return "", -1, fmt.Errorf("No space found")
	}

	// Ensure we have at least one character in the prefix.
	if pos == 1 {
		return "", -1, fmt.Errorf("Prefix is zero length")
	}

	// Return the prefix without the space.
	// New index is after the space.
	return line[1:pos], pos + 1, nil
}

// parseCommand parses the command portion of a message from the server.
//
// We start parsing at the given index in the string.
//
// We return the command portion and the index just after the command.
//
// ABNF:
// message    =  [ ":" prefix SPACE ] command [ params ] crlf
// command    =  1*letter / 3digit
// params     =  *14( SPACE middle ) [ SPACE ":" trailing ]
//            =/ 14( SPACE middle ) [ SPACE [ ":" ] trailing ]
func parseCommand(line string, index int) (string, int, error) {
	newIndex := index

	// Parse until we hit a non-letter or non-digit.
	for newIndex < len(line) {
		// Digit
		if line[newIndex] >= 48 && line[newIndex] <= 57 {
			newIndex++
			continue
		}

		// Letter
		if line[newIndex] >= 65 && line[newIndex] <= 122 {
			newIndex++
			continue
		}

		// Must be a space or CR.
		if line[newIndex] != ' ' &&
			line[newIndex] != '\r' {
			return "", -1, fmt.Errorf("Unexpected character after command: [%q]",
				line[newIndex])
		}
		break
	}

	// 0 length command is not valid.
	if newIndex == index {
		return "", -1, fmt.Errorf("0 length command found")
	}

	// TODO: Enforce that we either have 3 digits or all letters.

	// Return command string without space or CR.
	// New index is at the CR or space.
	return strings.ToUpper(line[index:newIndex]), newIndex, nil
}

// parseParams parses the params part of a message.
//
// The given index points to the first character in the params.
// There may not actually be any params. In that case we don't raise an
// error but simply do not return any.
//
// We return each param (stripped of : in the case of 'trailing') and the
// index after the params end.
//
// Note there may be blank parameters in some cases. Specifically since
// trailing accepts 0 length as valid.
//
// Relevant parts of ABNF from RFC 2812 section 2.3.1:
// message    =  [ ":" prefix SPACE ] command [ params ] crlf
// params     =  *14( SPACE middle ) [ SPACE ":" trailing ]
//            =/ 14( SPACE middle ) [ SPACE [ ":" ] trailing ]
// middle     =  nospcrlfcl *( ":" / nospcrlfcl )
// trailing   =  *( ":" / " " / nospcrlfcl )
// nospcrlfcl =  %x01-09 / %x0B-0C / %x0E-1F / %x21-39 / %x3B-FF
//               ; any octet except NUL, CR, LF, " " and ":"
func parseParams(line string, index int) ([]string, int, error) {
	newIndex := index
	var params []string

	for newIndex < len(line) {
		if line[newIndex] != ' ' {
			return params, newIndex, nil
		}

		if len(params) < 14 {
			param, paramIndex, err := parseParam(line, newIndex)
			if err != nil {
				// At this point we should always have at least one character in the
				// param. However it is common in the wild (ratbox, quassel) for there
				// to be a single space character before CRLF. Permit it here.
				// Set index to point after the spurious " " as though we consumed it.
				if err.Error() == "Param with zero characters" &&
					newIndex+3 == len(line) && line[newIndex] == ' ' &&
					line[newIndex+1] == '\r' && line[newIndex+2] == '\n' {
					return params, newIndex + 1, nil
				}
				return nil, -1, fmt.Errorf("Problem parsing parameter: %s", err)
			}
			newIndex = paramIndex

			params = append(params, param)

			continue
		}

		param, newIndex, err := parseParamLast(line, newIndex)
		if err != nil {
			return nil, -1, fmt.Errorf("Problem parsing last parameter: %s", err)
		}

		params = append(params, param)

		return params, newIndex, nil
	}

	return nil, -1, fmt.Errorf("Malformed params. Not terminated properly.")
}

// parseParam parses out a single parameter term.
//
// index points to a space.
//
// We return the parameter (stripped of : in the case of trailing) and the
// index after the parameter ends.
func parseParam(line string, index int) (string, int, error) {
	newIndex := index

	if line[newIndex] != ' ' {
		return "", -1, fmt.Errorf("Malformed param. No leading space.")
	}

	newIndex++

	if len(line) == newIndex {
		return "", -1, fmt.Errorf("Malformed parameter. End of string after space.")
	}

	// SPACE ":" trailing
	if line[newIndex] == ':' {
		newIndex++

		if len(line) == newIndex {
			return "", -1, fmt.Errorf("Malformed parameter. End of string after :.")
		}

		// It is valid for there to be no characters.
		// Because: trailing   =  *( ":" / " " / nospcrlfcl )

		paramIndexStart := newIndex

		for newIndex < len(line) {
			if line[newIndex] == '\x00' || line[newIndex] == '\r' ||
				line[newIndex] == '\n' {
				break
			}
			newIndex++
		}

		return line[paramIndexStart:newIndex], newIndex, nil
	}

	// SPACE middle
	// middle     =  nospcrlfcl *( ":" / nospcrlfcl )
	// nospcrlfcl = any octet except NUL, CR, LF, " ", ":"
	// This means the first character must not be any of those, but afterwards :
	// may appear.
	//
	// We know from the above check (for SPACE ":" trailing) that it is NOT ":",
	// so we can take all except NUL, CR, LF, " ".

	// paramIndexStart points at the character after the space.
	paramIndexStart := newIndex

	for newIndex < len(line) {
		if line[newIndex] == '\x00' || line[newIndex] == '\r' ||
			line[newIndex] == '\n' || line[newIndex] == ' ' {
			break
		}
		newIndex++
	}

	// Must have at least one character in this case. See grammar for 'middle'.
	if paramIndexStart == newIndex {
		return "", -1, fmt.Errorf("Param with zero characters")
	}

	return line[paramIndexStart:newIndex], newIndex, nil
}

// parseParamLast takes the final case when parsing params.
//
// Specifically, the [ SPACE [ ":" ] trailing ] case.
func parseParamLast(line string, index int) (string, int, error) {
	newIndex := index

	if line[newIndex] != ' ' {
		return "", -1, fmt.Errorf("Malformed param. No leading space.")
	}

	newIndex++

	// If we're at the end of the string, then something is wrong. While the
	// parameter may be blank, there should at least be CRLF remaining.
	// It's valid for there to be no characters.
	if newIndex == len(line) {
		return "", -1, fmt.Errorf("Malformed param. Space ends message.")
	}

	// It is valid for there to be no :
	if line[newIndex] == ':' {
		newIndex++

		// See above. We should have at least CRLF.
		if newIndex == len(line) {
			return "", -1, fmt.Errorf("Malformed param. : ends message.")
		}
	}

	// It is valid for there to be no characters.

	paramStartIndex := newIndex

	for newIndex < len(line) {
		if line[newIndex] == '\x00' || line[newIndex] == '\r' ||
			line[newIndex] == '\n' {
			break
		}
		newIndex++
	}

	return line[paramStartIndex:newIndex], newIndex, nil
}
<commit_msg>Error messages should not start with capital<commit_after>// Package irc implements parsing and encoding of IRC protocol messages.
package irc

import (
	"fmt"
	"strings"
)

// MaxLineLength is the maximum protocol message line length.
// From RFC 2812 section 2.3. It includes CRLF.
const MaxLineLength = 512

// Message holds a protocol message.
// See section 2.3.1 in RFC 2812.
type Message struct {
	// Prefix may be blank. It's optional.
	Prefix string

	Command string

	// There are at most 15 parameters.
	Params []string
}

func (m Message) String() string {
	return fmt.Sprintf("Prefix [%s] Command [%s] Params%q", m.Prefix, m.Command,
		m.Params)
}

// Encode turns the in memory representation of the memory into the IRC
// protocol message string.
//
// It does not enforce command specific semantics. It is instead responsible
// only for placing prefix, command, and parameters.
func (m Message) Encode() (string, error) {
	s := ""

	if len(m.Prefix) > 0 {
		s += ":" + m.Prefix + " "
	}

	s += m.Command

	for i, param := range m.Params {
		// If we have a ":" or " ", then we need to prefix the parameter with ":".
		// This is only valid for the last ("trailing") parameter.
		//
		// Also, the trailing parameter may be an empty string. Ensure it shows up
		// by adding a :. This can happen e.g. from ircd-ratbox in TOPIC unset
		// command // (server protocol). Modern IRC and ircv3 both specify this as
		// valid (that the last parameter may be empty). Also, RFC 2812's grammar
		// permits it. The ":" is optional only if this is the 15th parameter. But
		// it is valid.
		if idx := strings.IndexAny(param, ": "); idx != -1 || len(param) == 0 {
			s += " :" + param

			// This must be the last parameter.
			if i+1 != len(m.Params) {
				return "", fmt.Errorf("parameter problem: ':' or ' ' outside last parameter")
			}
			continue
		}

		s += " " + param
	}

	s += "\r\n"

	if len(s) > MaxLineLength {
		return "", fmt.Errorf("message after encoding is too long (%d bytes)",
			len(s))
	}

	return s, nil
}

// ParseMessage parses a protocol message from the client/server.
//
// See RFC 2812 Section 2.3.1.
//
// line ends with \n.
func ParseMessage(line string) (Message, error) {
	if len(line) > MaxLineLength {
		return Message{}, fmt.Errorf("line exceeds maximum length: %d vs. %d",
			len(line), MaxLineLength)
	}

	line, err := fixLineEnding(line)
	if err != nil {
		return Message{}, fmt.Errorf("line does not have a valid ending: %s", line)
	}

	message := Message{}
	index := 0

	// It is optional to have a prefix.
	if line[0] == ':' {
		prefix, prefixIndex, err := parsePrefix(line)
		if err != nil {
			return Message{}, fmt.Errorf("problem parsing prefix: %s", err)
		}
		index = prefixIndex

		message.Prefix = prefix

		if index >= len(line) {
			return Message{}, fmt.Errorf("malformed message. Prefix only")
		}
	}

	// We've either parsed a prefix out or have no prefix.
	command, index, err := parseCommand(line, index)
	if err != nil {
		return Message{}, fmt.Errorf("problem parsing command: %s", err)
	}

	message.Command = command

	// May have params.
	params, index, err := parseParams(line, index)
	if err != nil {
		return Message{}, fmt.Errorf("problem parsing params: %s", err)
	}

	message.Params = params

	// We should now have CRLF.
	// index should be pointing at the CR after parsing params.
	if index != len(line)-2 || line[index] != '\r' || line[index+1] != '\n' {
		return Message{}, fmt.Errorf("malformed message. No CRLF found. Looking for end at position %d", index)
	}

	return message, nil
}

// fixLineEnding tries to ensure the line ends with CRLF.
//
// If it ends with only LF, add a CR.
func fixLineEnding(line string) (string, error) {
	if len(line) == 0 {
		return "", fmt.Errorf("line is blank")
	}

	if len(line) == 1 {
		if line[0] == '\n' {
			return "\r\n", nil
		}

		return "", fmt.Errorf("line does not end with LF")
	}

	lastIndex := len(line) - 1
	secondLastIndex := lastIndex - 1

	if line[secondLastIndex] == '\r' && line[lastIndex] == '\n' {
		return line, nil
	}

	if line[lastIndex] == '\n' {
		return line[:lastIndex] + "\r\n", nil
	}

	return "", fmt.Errorf("line has no ending CRLF or LF")
}

// parsePrefix parses out the prefix portion of a string.
//
// line begins with : and ends with \n.
//
// If there is no error we return the prefix and the position after
// the SPACE.
// This means the index points to the first character of the command (in a well
// formed message). We do not confirm there actually is a character.
//
// We are parsing this:
// message    =  [ ":" prefix SPACE ] command [ params ] crlf
// prefix     =  servername / ( nickname [ [ "!" user ] "@" host ] )
//
// TODO: Enforce length limits
// TODO: Enforce character / format more strictly.
//   Right now I don't do much other than ensure there is no space.
func parsePrefix(line string) (string, int, error) {
	pos := 0

	if line[pos] != ':' {
		return "", -1, fmt.Errorf("line does not start with ':'")
	}

	for pos < len(line) {
		// Prefix ends with a space.
		if line[pos] == ' ' {
			break
		}

		// Basic character check.
		// I'm being very lenient here right now. Servername and hosts should only
		// allow [a-zA-Z0-9]. Nickname can have any except NUL, CR, LF, " ". I
		// choose to accept anything nicks can.
		if line[pos] == '\x00' || line[pos] == '\n' || line[pos] == '\r' {
			return "", -1, fmt.Errorf("invalid character found: %q", line[pos])
		}

		pos++
	}

	// We didn't find a space.
	if pos == len(line) {
		return "", -1, fmt.Errorf("no space found")
	}

	// Ensure we have at least one character in the prefix.
	if pos == 1 {
		return "", -1, fmt.Errorf("prefix is zero length")
	}

	// Return the prefix without the space.
	// New index is after the space.
	return line[1:pos], pos + 1, nil
}

// parseCommand parses the command portion of a message from the server.
//
// We start parsing at the given index in the string.
//
// We return the command portion and the index just after the command.
//
// ABNF:
// message    =  [ ":" prefix SPACE ] command [ params ] crlf
// command    =  1*letter / 3digit
// params     =  *14( SPACE middle ) [ SPACE ":" trailing ]
//            =/ 14( SPACE middle ) [ SPACE [ ":" ] trailing ]
func parseCommand(line string, index int) (string, int, error) {
	newIndex := index

	// Parse until we hit a non-letter or non-digit.
	for newIndex < len(line) {
		// Digit
		if line[newIndex] >= 48 && line[newIndex] <= 57 {
			newIndex++
			continue
		}

		// Letter
		if line[newIndex] >= 65 && line[newIndex] <= 122 {
			newIndex++
			continue
		}

		// Must be a space or CR.
		if line[newIndex] != ' ' &&
			line[newIndex] != '\r' {
			return "", -1, fmt.Errorf("unexpected character after command: %q",
				line[newIndex])
		}
		break
	}

	// 0 length command is not valid.
	if newIndex == index {
		return "", -1, fmt.Errorf("0 length command found")
	}

	// TODO: Enforce that we either have 3 digits or all letters.

	// Return command string without space or CR.
	// New index is at the CR or space.
	return strings.ToUpper(line[index:newIndex]), newIndex, nil
}

// parseParams parses the params part of a message.
//
// The given index points to the first character in the params.
// There may not actually be any params. In that case we don't raise an
// error but simply do not return any.
//
// We return each param (stripped of : in the case of 'trailing') and the
// index after the params end.
//
// Note there may be blank parameters in some cases. Specifically since
// trailing accepts 0 length as valid.
//
// Relevant parts of ABNF from RFC 2812 section 2.3.1:
// message    =  [ ":" prefix SPACE ] command [ params ] crlf
// params     =  *14( SPACE middle ) [ SPACE ":" trailing ]
//            =/ 14( SPACE middle ) [ SPACE [ ":" ] trailing ]
// middle     =  nospcrlfcl *( ":" / nospcrlfcl )
// trailing   =  *( ":" / " " / nospcrlfcl )
// nospcrlfcl =  %x01-09 / %x0B-0C / %x0E-1F / %x21-39 / %x3B-FF
//               ; any octet except NUL, CR, LF, " " and ":"
func parseParams(line string, index int) ([]string, int, error) {
	newIndex := index
	var params []string

	for newIndex < len(line) {
		if line[newIndex] != ' ' {
			return params, newIndex, nil
		}

		if len(params) < 14 {
			param, paramIndex, err := parseParam(line, newIndex)
			if err != nil {
				// At this point we should always have at least one character in the
				// param. However it is common in the wild (ratbox, quassel) for there
				// to be a single space character before CRLF. Permit it here.
				// Set index to point after the spurious " " as though we consumed it.
				if err.Error() == "param with zero characters" &&
					newIndex+3 == len(line) && line[newIndex] == ' ' &&
					line[newIndex+1] == '\r' && line[newIndex+2] == '\n' {
					return params, newIndex + 1, nil
				}
				return nil, -1, fmt.Errorf("problem parsing parameter: %s", err)
			}
			newIndex = paramIndex

			params = append(params, param)

			continue
		}

		param, newIndex, err := parseParamLast(line, newIndex)
		if err != nil {
			return nil, -1, fmt.Errorf("problem parsing last parameter: %s", err)
		}

		params = append(params, param)

		return params, newIndex, nil
	}

	return nil, -1, fmt.Errorf("malformed params. Not terminated properly")
}

// parseParam parses out a single parameter term.
//
// index points to a space.
//
// We return the parameter (stripped of : in the case of trailing) and the
// index after the parameter ends.
func parseParam(line string, index int) (string, int, error) {
	newIndex := index

	if line[newIndex] != ' ' {
		return "", -1, fmt.Errorf("malformed param. No leading space")
	}

	newIndex++

	if len(line) == newIndex {
		return "", -1, fmt.Errorf("malformed parameter. End of string after space")
	}

	// SPACE ":" trailing
	if line[newIndex] == ':' {
		newIndex++

		if len(line) == newIndex {
			return "", -1, fmt.Errorf("malformed parameter. End of string after ':'")
		}

		// It is valid for there to be no characters.
		// Because: trailing   =  *( ":" / " " / nospcrlfcl )

		paramIndexStart := newIndex

		for newIndex < len(line) {
			if line[newIndex] == '\x00' || line[newIndex] == '\r' ||
				line[newIndex] == '\n' {
				break
			}
			newIndex++
		}

		return line[paramIndexStart:newIndex], newIndex, nil
	}

	// SPACE middle
	// middle     =  nospcrlfcl *( ":" / nospcrlfcl )
	// nospcrlfcl = any octet except NUL, CR, LF, " ", ":"
	// This means the first character must not be any of those, but afterwards :
	// may appear.
	//
	// We know from the above check (for SPACE ":" trailing) that it is NOT ":",
	// so we can take all except NUL, CR, LF, " ".

	// paramIndexStart points at the character after the space.
	paramIndexStart := newIndex

	for newIndex < len(line) {
		if line[newIndex] == '\x00' || line[newIndex] == '\r' ||
			line[newIndex] == '\n' || line[newIndex] == ' ' {
			break
		}
		newIndex++
	}

	// Must have at least one character in this case. See grammar for 'middle'.
	if paramIndexStart == newIndex {
		return "", -1, fmt.Errorf("param with zero characters")
	}

	return line[paramIndexStart:newIndex], newIndex, nil
}

// parseParamLast takes the final case when parsing params.
//
// Specifically, the [ SPACE [ ":" ] trailing ] case.
func parseParamLast(line string, index int) (string, int, error) {
	newIndex := index

	if line[newIndex] != ' ' {
		return "", -1, fmt.Errorf("malformed param. No leading space")
	}

	newIndex++

	// If we're at the end of the string, then something is wrong. While the
	// parameter may be blank, there should at least be CRLF remaining.
	// It's valid for there to be no characters.
	if newIndex == len(line) {
		return "", -1, fmt.Errorf("malformed param. Space ends message")
	}

	// It is valid for there to be no :
	if line[newIndex] == ':' {
		newIndex++

		// See above. We should have at least CRLF.
		if newIndex == len(line) {
			return "", -1, fmt.Errorf("malformed param. : ends message")
		}
	}

	// It is valid for there to be no characters.

	paramStartIndex := newIndex

	for newIndex < len(line) {
		if line[newIndex] == '\x00' || line[newIndex] == '\r' ||
			line[newIndex] == '\n' {
			break
		}
		newIndex++
	}

	return line[paramStartIndex:newIndex], newIndex, nil
}
<|endoftext|>


<commit_before>// Copyright (c) 2017-2018 Intel Corporation
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// +build linux

package main

/*
#include <linux/kvm.h>

const int ioctl_KVM_CREATE_VM = KVM_CREATE_VM;
*/
import "C"

import (
	"fmt"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"strings"
	"syscall"

	vc "github.com/containers/virtcontainers"
	"github.com/sirupsen/logrus"
	"github.com/urfave/cli"
)

type kernelModule struct {
	// description
	desc string

	// maps parameter names to values
	parameters map[string]string
}

const (
	moduleParamDir        = "parameters"
	cpuFlagsTag           = "flags"
	successMessageCapable = "System is capable of running " + project
	successMessageCreate  = "System can currently create " + project
	failMessage           = "System is not capable of running " + project
	kernelPropertyCorrect = "Kernel property value correct"
)

// variables rather than consts to allow tests to modify them
var (
	procCPUInfo  = "/proc/cpuinfo"
	sysModuleDir = "/sys/module"
	modInfoCmd   = "modinfo"
	kvmDevice    = "/dev/kvm"
)

// requiredCPUFlags maps a CPU flag value to search for and a
// human-readable description of that value.
var requiredCPUFlags = map[string]string{
	"vmx":    "Virtualization support",
	"lm":     "64Bit CPU",
	"sse4_1": "SSE4.1",
}

// requiredCPUAttribs maps a CPU (non-CPU flag) attribute value to search for
// and a human-readable description of that value.
var requiredCPUAttribs = map[string]string{
	"GenuineIntel": "Intel Architecture CPU",
}

// requiredKernelModules maps a required module name to a human-readable
// description of the modules functionality and an optional list of
// required module parameters.
var requiredKernelModules = map[string]kernelModule{
	"kvm": {
		desc: "Kernel-based Virtual Machine",
	},
	"kvm_intel": {
		desc: "Intel KVM",
		parameters: map[string]string{
			"nested":             "Y",
			"unrestricted_guest": "Y",
		},
	},
	"vhost": {
		desc: "Host kernel accelerator for virtio",
	},
	"vhost_net": {
		desc: "Host kernel accelerator for virtio network",
	},
}

// getCPUInfo returns details of the first CPU
func getCPUInfo(cpuInfoFile string) (string, error) {
	text, err := getFileContents(cpuInfoFile)
	if err != nil {
		return "", err
	}

	cpus := strings.SplitAfter(text, "\n\n")

	trimmed := strings.TrimSpace(cpus[0])
	if trimmed == "" {
		return "", fmt.Errorf("Cannot determine CPU details")
	}

	return trimmed, nil
}

func findAnchoredString(haystack, needle string) bool {
	if haystack == "" || needle == "" {
		return false
	}

	// Ensure the search string is anchored
	pattern := regexp.MustCompile(`\b` + needle + `\b`)

	return pattern.MatchString(haystack)
}

func getCPUFlags(cpuinfo string) string {
	for _, line := range strings.Split(cpuinfo, "\n") {
		if strings.HasPrefix(line, cpuFlagsTag) {
			fields := strings.Split(line, ":")
			if len(fields) == 2 {
				return strings.TrimSpace(fields[1])
			}
		}
	}

	return ""
}

func haveKernelModule(module string) bool {
	// First, check to see if the module is already loaded
	path := filepath.Join(sysModuleDir, module)
	if fileExists(path) {
		return true
	}

	// Now, check if the module is unloaded, but available
	cmd := exec.Command(modInfoCmd, module)
	err := cmd.Run()
	return err == nil
}

// checkCPU checks all required CPU attributes modules and returns a count of
// the number of CPU attribute errors (all of which are logged by this
// function). Only fatal errors result in an error return.
func checkCPU(tag, cpuinfo string, attribs map[string]string) (count uint32) {
	if cpuinfo == "" {
		return 0
	}

	for attrib, desc := range attribs {
		fields := logrus.Fields{
			"type":        tag,
			"name":        attrib,
			"description": desc,
		}

		found := findAnchoredString(cpuinfo, attrib)
		if !found {
			ccLog.WithFields(fields).Errorf("CPU property not found")
			count++
			continue

		}

		ccLog.WithFields(fields).Infof("CPU property found")
	}

	return count
}

func checkCPUFlags(cpuflags string, required map[string]string) uint32 {
	return checkCPU("flag", cpuflags, required)
}

func checkCPUAttribs(cpuinfo string, attribs map[string]string) uint32 {
	return checkCPU("attribute", cpuinfo, attribs)
}

// checkKernelModules checks all required kernel modules modules and returns a count of
// the number of module errors (all of which are logged by this
// function). Only fatal errors result in an error return.
func checkKernelModules(modules map[string]kernelModule) (count uint32, err error) {
	onVMM, err := vc.RunningOnVMM(procCPUInfo)
	if err != nil {
		return 0, err
	}

	for module, details := range modules {
		fields := logrus.Fields{
			"type":        "module",
			"name":        module,
			"description": details.desc,
		}

		if !haveKernelModule(module) {
			ccLog.WithFields(fields).Error("kernel property not found")
			count++
			continue
		}

		ccLog.WithFields(fields).Infof("kernel property found")

		for param, expected := range details.parameters {
			path := filepath.Join(sysModuleDir, module, moduleParamDir, param)
			value, err := getFileContents(path)
			if err != nil {
				return 0, err
			}

			value = strings.TrimRight(value, "\n\r")

			if value != expected {
				fields["expected"] = expected
				fields["actual"] = value
				fields["parameter"] = param

				msg := "kernel module parameter has unexpected value"

				// this option is not required when
				// already running under a hypervisor.
				if param == "unrestricted_guest" && onVMM {
					ccLog.WithFields(fields).Warn(kernelPropertyCorrect)
					continue
				}

				if param == "nested" {
					ccLog.WithFields(fields).Warn(msg)
					continue
				}

				ccLog.WithFields(fields).Error(msg)
				count++
			}

			ccLog.WithFields(fields).Info(kernelPropertyCorrect)
		}
	}

	return count, nil
}

// hostIsClearContainersCapable determines if the system is capable of
// running Clear Containers.
func hostIsClearContainersCapable(cpuinfoFile string) error {
	cpuinfo, err := getCPUInfo(cpuinfoFile)
	if err != nil {
		return err
	}

	cpuFlags := getCPUFlags(cpuinfo)
	if cpuFlags == "" {
		return fmt.Errorf("Cannot find CPU flags")
	}

	// Keep a track of the error count, but don't error until all tests
	// have been performed!
	errorCount := uint32(0)

	count := checkCPUAttribs(cpuinfo, requiredCPUAttribs)

	errorCount += count

	count = checkCPUFlags(cpuFlags, requiredCPUFlags)

	errorCount += count

	count, err = checkKernelModules(requiredKernelModules)
	if err != nil {
		return err
	}

	errorCount += count

	if errorCount == 0 {
		return nil
	}

	return fmt.Errorf("ERROR: %s", failMessage)
}

// kvmIsUsable determines if it will be possible to create a virtual machine.
func kvmIsUsable() error {
	flags := syscall.O_RDWR | syscall.O_CLOEXEC

	f, err := syscall.Open(kvmDevice, flags, 0)
	if err != nil {
		return err
	}
	defer syscall.Close(f)

	fieldLogger := ccLog.WithField("check-type", "full")

	fieldLogger.WithField("device", kvmDevice).Info("device available")

	vm, _, errno := syscall.Syscall(syscall.SYS_IOCTL,
		uintptr(f),
		uintptr(C.ioctl_KVM_CREATE_VM),
		0)
	if errno != 0 {
		if errno == syscall.EBUSY {
			fieldLogger.WithField("reason", "another hypervisor running").Error("cannot create VM")
		}

		return errno
	}
	defer syscall.Close(int(vm))

	fieldLogger.WithField("feature", "create-vm").Info("feature available")

	return nil
}

var ccCheckCLICommand = cli.Command{
	Name:  checkCmd,
	Usage: "tests if system can run " + project,
	Action: func(context *cli.Context) error {
		err := hostIsClearContainersCapable(procCPUInfo)
		if err != nil {
			return err
		}

		ccLog.Info(successMessageCapable)

		if os.Geteuid() == 0 {
			// If running as the superuser, perform additional
			// checks.
			err = kvmIsUsable()
			if err != nil {
				return err
			}

			ccLog.Info(successMessageCreate)
		}

		return nil
	},
}
<commit_msg>cc-check: Document "unrestricted_guest" requirement<commit_after>// Copyright (c) 2017-2018 Intel Corporation
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// +build linux

package main

/*
#include <linux/kvm.h>

const int ioctl_KVM_CREATE_VM = KVM_CREATE_VM;
*/
import "C"

import (
	"fmt"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"strings"
	"syscall"

	vc "github.com/containers/virtcontainers"
	"github.com/sirupsen/logrus"
	"github.com/urfave/cli"
)

type kernelModule struct {
	// description
	desc string

	// maps parameter names to values
	parameters map[string]string
}

const (
	moduleParamDir        = "parameters"
	cpuFlagsTag           = "flags"
	successMessageCapable = "System is capable of running " + project
	successMessageCreate  = "System can currently create " + project
	failMessage           = "System is not capable of running " + project
	kernelPropertyCorrect = "Kernel property value correct"
)

// variables rather than consts to allow tests to modify them
var (
	procCPUInfo  = "/proc/cpuinfo"
	sysModuleDir = "/sys/module"
	modInfoCmd   = "modinfo"
	kvmDevice    = "/dev/kvm"
)

// requiredCPUFlags maps a CPU flag value to search for and a
// human-readable description of that value.
var requiredCPUFlags = map[string]string{
	"vmx":    "Virtualization support",
	"lm":     "64Bit CPU",
	"sse4_1": "SSE4.1",
}

// requiredCPUAttribs maps a CPU (non-CPU flag) attribute value to search for
// and a human-readable description of that value.
var requiredCPUAttribs = map[string]string{
	"GenuineIntel": "Intel Architecture CPU",
}

// requiredKernelModules maps a required module name to a human-readable
// description of the modules functionality and an optional list of
// required module parameters.
var requiredKernelModules = map[string]kernelModule{
	"kvm": {
		desc: "Kernel-based Virtual Machine",
	},
	"kvm_intel": {
		desc: "Intel KVM",
		parameters: map[string]string{
			"nested": "Y",
			// "VMX Unrestricted mode support". This is used
			// as a heuristic to determine if the system is
			// "new enough" to run a Clear Container
			// (atleast a Westmere).
			"unrestricted_guest": "Y",
		},
	},
	"vhost": {
		desc: "Host kernel accelerator for virtio",
	},
	"vhost_net": {
		desc: "Host kernel accelerator for virtio network",
	},
}

// getCPUInfo returns details of the first CPU
func getCPUInfo(cpuInfoFile string) (string, error) {
	text, err := getFileContents(cpuInfoFile)
	if err != nil {
		return "", err
	}

	cpus := strings.SplitAfter(text, "\n\n")

	trimmed := strings.TrimSpace(cpus[0])
	if trimmed == "" {
		return "", fmt.Errorf("Cannot determine CPU details")
	}

	return trimmed, nil
}

func findAnchoredString(haystack, needle string) bool {
	if haystack == "" || needle == "" {
		return false
	}

	// Ensure the search string is anchored
	pattern := regexp.MustCompile(`\b` + needle + `\b`)

	return pattern.MatchString(haystack)
}

func getCPUFlags(cpuinfo string) string {
	for _, line := range strings.Split(cpuinfo, "\n") {
		if strings.HasPrefix(line, cpuFlagsTag) {
			fields := strings.Split(line, ":")
			if len(fields) == 2 {
				return strings.TrimSpace(fields[1])
			}
		}
	}

	return ""
}

func haveKernelModule(module string) bool {
	// First, check to see if the module is already loaded
	path := filepath.Join(sysModuleDir, module)
	if fileExists(path) {
		return true
	}

	// Now, check if the module is unloaded, but available
	cmd := exec.Command(modInfoCmd, module)
	err := cmd.Run()
	return err == nil
}

// checkCPU checks all required CPU attributes modules and returns a count of
// the number of CPU attribute errors (all of which are logged by this
// function). Only fatal errors result in an error return.
func checkCPU(tag, cpuinfo string, attribs map[string]string) (count uint32) {
	if cpuinfo == "" {
		return 0
	}

	for attrib, desc := range attribs {
		fields := logrus.Fields{
			"type":        tag,
			"name":        attrib,
			"description": desc,
		}

		found := findAnchoredString(cpuinfo, attrib)
		if !found {
			ccLog.WithFields(fields).Errorf("CPU property not found")
			count++
			continue

		}

		ccLog.WithFields(fields).Infof("CPU property found")
	}

	return count
}

func checkCPUFlags(cpuflags string, required map[string]string) uint32 {
	return checkCPU("flag", cpuflags, required)
}

func checkCPUAttribs(cpuinfo string, attribs map[string]string) uint32 {
	return checkCPU("attribute", cpuinfo, attribs)
}

// checkKernelModules checks all required kernel modules modules and returns a count of
// the number of module errors (all of which are logged by this
// function). Only fatal errors result in an error return.
func checkKernelModules(modules map[string]kernelModule) (count uint32, err error) {
	onVMM, err := vc.RunningOnVMM(procCPUInfo)
	if err != nil {
		return 0, err
	}

	for module, details := range modules {
		fields := logrus.Fields{
			"type":        "module",
			"name":        module,
			"description": details.desc,
		}

		if !haveKernelModule(module) {
			ccLog.WithFields(fields).Error("kernel property not found")
			count++
			continue
		}

		ccLog.WithFields(fields).Infof("kernel property found")

		for param, expected := range details.parameters {
			path := filepath.Join(sysModuleDir, module, moduleParamDir, param)
			value, err := getFileContents(path)
			if err != nil {
				return 0, err
			}

			value = strings.TrimRight(value, "\n\r")

			if value != expected {
				fields["expected"] = expected
				fields["actual"] = value
				fields["parameter"] = param

				msg := "kernel module parameter has unexpected value"

				// this option is not required when
				// already running under a hypervisor.
				if param == "unrestricted_guest" && onVMM {
					ccLog.WithFields(fields).Warn(kernelPropertyCorrect)
					continue
				}

				if param == "nested" {
					ccLog.WithFields(fields).Warn(msg)
					continue
				}

				ccLog.WithFields(fields).Error(msg)
				count++
			}

			ccLog.WithFields(fields).Info(kernelPropertyCorrect)
		}
	}

	return count, nil
}

// hostIsClearContainersCapable determines if the system is capable of
// running Clear Containers.
func hostIsClearContainersCapable(cpuinfoFile string) error {
	cpuinfo, err := getCPUInfo(cpuinfoFile)
	if err != nil {
		return err
	}

	cpuFlags := getCPUFlags(cpuinfo)
	if cpuFlags == "" {
		return fmt.Errorf("Cannot find CPU flags")
	}

	// Keep a track of the error count, but don't error until all tests
	// have been performed!
	errorCount := uint32(0)

	count := checkCPUAttribs(cpuinfo, requiredCPUAttribs)

	errorCount += count

	count = checkCPUFlags(cpuFlags, requiredCPUFlags)

	errorCount += count

	count, err = checkKernelModules(requiredKernelModules)
	if err != nil {
		return err
	}

	errorCount += count

	if errorCount == 0 {
		return nil
	}

	return fmt.Errorf("ERROR: %s", failMessage)
}

// kvmIsUsable determines if it will be possible to create a virtual machine.
func kvmIsUsable() error {
	flags := syscall.O_RDWR | syscall.O_CLOEXEC

	f, err := syscall.Open(kvmDevice, flags, 0)
	if err != nil {
		return err
	}
	defer syscall.Close(f)

	fieldLogger := ccLog.WithField("check-type", "full")

	fieldLogger.WithField("device", kvmDevice).Info("device available")

	vm, _, errno := syscall.Syscall(syscall.SYS_IOCTL,
		uintptr(f),
		uintptr(C.ioctl_KVM_CREATE_VM),
		0)
	if errno != 0 {
		if errno == syscall.EBUSY {
			fieldLogger.WithField("reason", "another hypervisor running").Error("cannot create VM")
		}

		return errno
	}
	defer syscall.Close(int(vm))

	fieldLogger.WithField("feature", "create-vm").Info("feature available")

	return nil
}

var ccCheckCLICommand = cli.Command{
	Name:  checkCmd,
	Usage: "tests if system can run " + project,
	Action: func(context *cli.Context) error {
		err := hostIsClearContainersCapable(procCPUInfo)
		if err != nil {
			return err
		}

		ccLog.Info(successMessageCapable)

		if os.Geteuid() == 0 {
			// If running as the superuser, perform additional
			// checks.
			err = kvmIsUsable()
			if err != nil {
				return err
			}

			ccLog.Info(successMessageCreate)
		}

		return nil
	},
}
<|endoftext|>


<commit_before>package gpgpdump

import (
	"io"
	"io/ioutil"

	"github.com/spiegel-im-spiegel/gpgpdump/info"
	"github.com/spiegel-im-spiegel/gpgpdump/options"
	"github.com/spiegel-im-spiegel/gpgpdump/packet"
)

//Parse returns packet info (from io.Reader stream).
func Parse(r io.Reader, o *options.Options) (*info.Info, error) {
	data, err := ioutil.ReadAll(r) //buffering to []byte
	if err != nil {
		return nil, err
	}
	return ParseByte(data, o)
}

//ParseByte returns packet info (from []byte data).
func ParseByte(data []byte, o *options.Options) (*info.Info, error) {
	parser, err := packet.NewParser(data, o)
	if err != nil {
		return nil, err
	}
	return parser.Parse()
}

/* Copyright 2017 Spiegel
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * 	http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
<commit_msg>Update function interface<commit_after>package gpgpdump

import (
	"bytes"
	"io"

	"github.com/spiegel-im-spiegel/gpgpdump/info"
	"github.com/spiegel-im-spiegel/gpgpdump/options"
	"github.com/spiegel-im-spiegel/gpgpdump/packet"
)

//Parse returns packet info (from io.Reader stream).
func Parse(r io.Reader, o *options.Options) (*info.Info, error) {
	parser, err := packet.NewParser(r, o)
	if err != nil {
		return nil, err
	}
	return parser.Parse()
}

//ParseByte returns packet info (from []byte data).
func ParseByte(data []byte, o *options.Options) (*info.Info, error) {
	return Parse(bytes.NewReader(data), o)
}

/* Copyright 2017 Spiegel
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * 	http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
<|endoftext|>


<commit_before>package printer

import (
	"bytes"
	"encoding/binary"
	"fmt"
	"io/ioutil"
	"strconv"

	"github.com/davyxu/tabtoy/exportorv2/model"
)

type BinaryFile struct {
	buf bytes.Buffer
}

func (self *BinaryFile) Buffer() *bytes.Buffer {
	return &self.buf
}

func (self *BinaryFile) Printf(format string, args ...interface{}) {
	self.buf.WriteString(fmt.Sprintf(format, args...))
}

func (self *BinaryFile) Write(outfile string) bool {
	err := ioutil.WriteFile(outfile, self.buf.Bytes(), 0666)
	if err != nil {
		log.Errorln(err.Error())
		return false
	}

	return true
}

func (self *BinaryFile) WriteInt32(v int32) {
	binary.Write(&self.buf, binary.LittleEndian, v)
}

func (self *BinaryFile) WriteString(v string) {
	rawStr := []byte(v)

	binary.Write(&self.buf, binary.LittleEndian, int32(len(rawStr)))
	binary.Write(&self.buf, binary.LittleEndian, rawStr)
}

func (self *BinaryFile) WriteNodeValue(ft model.FieldType, value *model.Node) {

	switch ft {
	case model.FieldType_Int32:
		v, _ := strconv.ParseInt(value.Value, 10, 32)
		binary.Write(&self.buf, binary.LittleEndian, int32(v))
	case model.FieldType_UInt32:
		v, _ := strconv.ParseUint(value.Value, 10, 32)

		binary.Write(&self.buf, binary.LittleEndian, uint32(v))
	case model.FieldType_Int64:
		v, _ := strconv.ParseInt(value.Value, 10, 64)

		binary.Write(&self.buf, binary.LittleEndian, int64(v))
	case model.FieldType_UInt64:
		v, _ := strconv.ParseUint(value.Value, 10, 64)

		binary.Write(&self.buf, binary.LittleEndian, uint64(v))
	case model.FieldType_Float:
		v, _ := strconv.ParseFloat(value.Value, 32)

		binary.Write(&self.buf, binary.LittleEndian, float32(v))
	case model.FieldType_Bool:
		v, _ := strconv.ParseBool(value.Value)

		binary.Write(&self.buf, binary.LittleEndian, v)
	case model.FieldType_String:
		self.WriteString(value.Value)
	case model.FieldType_Enum:
		binary.Write(&self.buf, binary.LittleEndian, value.EnumValue)
	default:
		panic("unsupport type" + model.FieldTypeToString(ft))
	}

}

func NewBinaryFile() *BinaryFile {
	return &BinaryFile{}
}
<commit_msg>Fix `binary.Write: invalid type bool`<commit_after>package printer

import (
	"bytes"
	"encoding/binary"
	"fmt"
	"io/ioutil"
	"strconv"

	"github.com/davyxu/tabtoy/exportorv2/model"
)

type BinaryFile struct {
	buf bytes.Buffer
}

func (self *BinaryFile) Buffer() *bytes.Buffer {
	return &self.buf
}

func (self *BinaryFile) Printf(format string, args ...interface{}) {
	self.buf.WriteString(fmt.Sprintf(format, args...))
}

func (self *BinaryFile) Write(outfile string) bool {
	err := ioutil.WriteFile(outfile, self.buf.Bytes(), 0666)
	if err != nil {
		log.Errorln(err.Error())
		return false
	}

	return true
}

func (self *BinaryFile) WriteInt32(v int32) {
	binary.Write(&self.buf, binary.LittleEndian, v)
}

func (self *BinaryFile) WriteString(v string) {
	rawStr := []byte(v)

	binary.Write(&self.buf, binary.LittleEndian, int32(len(rawStr)))
	binary.Write(&self.buf, binary.LittleEndian, rawStr)
}

func (self *BinaryFile) WriteNodeValue(ft model.FieldType, value *model.Node) {

	switch ft {
	case model.FieldType_Int32:
		v, _ := strconv.ParseInt(value.Value, 10, 32)
		binary.Write(&self.buf, binary.LittleEndian, int32(v))
	case model.FieldType_UInt32:
		v, _ := strconv.ParseUint(value.Value, 10, 32)

		binary.Write(&self.buf, binary.LittleEndian, uint32(v))
	case model.FieldType_Int64:
		v, _ := strconv.ParseInt(value.Value, 10, 64)

		binary.Write(&self.buf, binary.LittleEndian, int64(v))
	case model.FieldType_UInt64:
		v, _ := strconv.ParseUint(value.Value, 10, 64)

		binary.Write(&self.buf, binary.LittleEndian, uint64(v))
	case model.FieldType_Float:
		v, _ := strconv.ParseFloat(value.Value, 32)

		binary.Write(&self.buf, binary.LittleEndian, float32(v))
	case model.FieldType_Bool:
		v, _ := strconv.ParseBool(value.Value)
		boolByte := []byte{0}
		if v {
			boolByte = []byte{1}
		}
		binary.Write(&self.buf, binary.LittleEndian, boolByte)
	case model.FieldType_String:
		self.WriteString(value.Value)
	case model.FieldType_Enum:
		binary.Write(&self.buf, binary.LittleEndian, value.EnumValue)
	default:
		panic("unsupport type" + model.FieldTypeToString(ft))
	}

}

func NewBinaryFile() *BinaryFile {
	return &BinaryFile{}
}
<|endoftext|>


<commit_before>package erpel

import (
	"bytes"
	"io/ioutil"
	"os"
	"strings"
	"testing"
)

func tempfile(t *testing.T) string {
	f, err := ioutil.TempFile("", "erpel-test-")
	if err != nil {
		t.Fatalf("TempFile(): %v", err)
	}

	name := f.Name()

	if err = f.Close(); err != nil {
		t.Fatalf("Close(): %v", err)
	}

	return name
}

func log(t *testing.T, filename string, data string) Marker {
	f, err := os.OpenFile(filename, os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0644)
	if err != nil {
		t.Fatalf("open(): %v", err)
	}

	if _, err = f.Write([]byte(data)); err != nil {
		t.Errorf("write(): %v", err)
	}

	m, err := Position(f)
	if err != nil {
		t.Errorf("Position(): %v", err)
	}

	if err = f.Close(); err != nil {
		t.Fatalf("close(): %v", err)
	}

	return m
}

func rm(t *testing.T, filename string) {
	if err := os.Remove(filename); err != nil {
		t.Errorf("removing %q failed: %v", filename, err)
	}
}

func writeMessages(t *testing.T, filename string, m []string) []Marker {
	var markers []Marker
	for _, msg := range messages {
		m := log(t, filename, msg)
		markers = append(markers, m)
	}

	return markers
}

func readRemainingData(t *testing.T, filename string, m Marker) []byte {
	fd, err := os.Open(filename)
	if err != nil {
		t.Fatalf("open(%v): %v", filename, err)
	}

	if err = m.Seek(fd); err != nil {
		t.Fatalf("Marker.Seek(): %v", err)
	}

	buf, err := ioutil.ReadAll(fd)
	if err != nil {
		t.Fatalf("read: %v", err)
	}

	if err = fd.Close(); err != nil {
		t.Fatalf("close(%v): %v", filename, err)
	}

	return buf
}

var messages = []string{
	"foobar baz message\n",
	"Jun 30 21:57:10 mopped sudo[19517]: pam_unix(sudo:session): session opened for user root by fd0(uid=0)\n",
	"foobar message2\n",
	"Jun 30 21:57:15 mopped sudo[19517]: pam_unix(sudo:session): session closed for user root\n",
}

func TestMarker(t *testing.T) {
	f := tempfile(t)
	t.Logf("using tempfile %v", f)

	markers := writeMessages(t, f, messages)
	for i, m := range markers {
		var data []byte

		for j := i + 1; j < len(messages); j++ {
			data = append(data, []byte(messages[j])...)
		}

		buf := readRemainingData(t, f, m)

		if !bytes.Equal(buf, data) {
			t.Errorf("marker %d returned wrong data, want:\n  %q\ngot:\n  %q", i, data, buf)
		}
	}

	rm(t, f)
}

func writeFile(t *testing.T, filename string, data []byte) {
	fd, err := os.Create(filename)
	if err != nil {
		t.Fatalf("create() %v", err)
	}

	_, err = fd.Write(data)
	if err != nil {
		t.Fatalf("write() %v", err)
	}

	if err = fd.Close(); err != nil {
		t.Fatalf("Close(): %v", err)
	}
}

func TestMarkerNewFile(t *testing.T) {
	f := tempfile(t)
	t.Logf("using tempfile %v", f)

	markers := writeMessages(t, f, messages)
	data := []byte(strings.Join(messages, ""))

	rm(t, f)
	writeFile(t, f, data)

	for i, m := range markers {
		buf := readRemainingData(t, f, m)

		if !bytes.Equal(buf, data) {
			t.Errorf("marker %d returned wrong data, want:\n  %q\ngot:\n  %q", i, data, buf)
		}
	}

}
<commit_msg>Add debug log to TestMarkerNewFile<commit_after>package erpel

import (
	"bytes"
	"io/ioutil"
	"os"
	"strings"
	"testing"
)

func tempfile(t *testing.T) string {
	f, err := ioutil.TempFile("", "erpel-test-")
	if err != nil {
		t.Fatalf("TempFile(): %v", err)
	}

	name := f.Name()

	if err = f.Close(); err != nil {
		t.Fatalf("Close(): %v", err)
	}

	return name
}

func log(t *testing.T, filename string, data string) Marker {
	f, err := os.OpenFile(filename, os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0644)
	if err != nil {
		t.Fatalf("open(): %v", err)
	}

	if _, err = f.Write([]byte(data)); err != nil {
		t.Errorf("write(): %v", err)
	}

	m, err := Position(f)
	if err != nil {
		t.Errorf("Position(): %v", err)
	}

	if err = f.Close(); err != nil {
		t.Fatalf("close(): %v", err)
	}

	return m
}

func rm(t *testing.T, filename string) {
	if err := os.Remove(filename); err != nil {
		t.Errorf("removing %q failed: %v", filename, err)
	}
}

func writeMessages(t *testing.T, filename string, m []string) []Marker {
	var markers []Marker
	for _, msg := range messages {
		m := log(t, filename, msg)
		markers = append(markers, m)
	}

	return markers
}

func readRemainingData(t *testing.T, filename string, m Marker) []byte {
	fd, err := os.Open(filename)
	if err != nil {
		t.Fatalf("open(%v): %v", filename, err)
	}

	if err = m.Seek(fd); err != nil {
		t.Fatalf("Marker.Seek(): %v", err)
	}

	buf, err := ioutil.ReadAll(fd)
	if err != nil {
		t.Fatalf("read: %v", err)
	}

	if err = fd.Close(); err != nil {
		t.Fatalf("close(%v): %v", filename, err)
	}

	return buf
}

var messages = []string{
	"foobar baz message\n",
	"Jun 30 21:57:10 mopped sudo[19517]: pam_unix(sudo:session): session opened for user root by fd0(uid=0)\n",
	"foobar message2\n",
	"Jun 30 21:57:15 mopped sudo[19517]: pam_unix(sudo:session): session closed for user root\n",
}

func TestMarker(t *testing.T) {
	f := tempfile(t)
	t.Logf("using tempfile %v", f)

	markers := writeMessages(t, f, messages)
	for i, m := range markers {
		var data []byte

		for j := i + 1; j < len(messages); j++ {
			data = append(data, []byte(messages[j])...)
		}

		buf := readRemainingData(t, f, m)

		if !bytes.Equal(buf, data) {
			t.Errorf("marker %d returned wrong data, want:\n  %q\ngot:\n  %q", i, data, buf)
		}
	}

	rm(t, f)
}

func writeFile(t *testing.T, filename string, data []byte) {
	fd, err := os.Create(filename)
	if err != nil {
		t.Fatalf("create() %v", err)
	}

	_, err = fd.Write(data)
	if err != nil {
		t.Fatalf("write() %v", err)
	}

	if err = fd.Close(); err != nil {
		t.Fatalf("Close(): %v", err)
	}
}

func TestMarkerNewFile(t *testing.T) {
	f := tempfile(t)
	t.Logf("using tempfile %v", f)

	markers := writeMessages(t, f, messages)
	data := []byte(strings.Join(messages, ""))

	rm(t, f)
	writeFile(t, f, data)

	for i, m := range markers {
		buf := readRemainingData(t, f, m)

		t.Logf("marker %d: %v", i, m)

		if !bytes.Equal(buf, data) {
			t.Errorf("marker %d returned wrong data, want:\n  %q\ngot:\n  %q", i, data, buf)
		}
	}

}
<|endoftext|>


<commit_before>// Client executable to send file to the upd daemon.
// Copyright © 2015 - Rémy MATHIEU

package client

import (
	"bytes"
	"crypto/tls"
	"crypto/x509"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"mime/multipart"
	"net/http"
	"net/url"
	"os"

	"server"
)

const (
	ROUTE_SEND = "/1.0/send"
)

type Client struct {
	Flags Flags
}

func NewClient(flags Flags) *Client {
	return &Client{Flags: flags}
}

// Send sends the given file to the upd server.
func (c *Client) Send(filename string) error {
	// first, we need to read the data
	data, err := c.readFile(filename)
	if err != nil {
		return err
	}

	// and now to send it the servee
	return c.sendData(filename, data)
}

func (c *Client) createClient() *http.Client {
	if c.Flags.CA == "unsafe" {
		tr := &http.Transport{
			TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
		}
		return &http.Client{Transport: tr}
	} else if len(c.Flags.CA) > 0 && c.Flags.CA != "none" {
		// reads the CA
		certs := x509.NewCertPool()
		pemData, err := ioutil.ReadFile(c.Flags.CA)
		if err != nil {
			log.Println("[err] Can't read the CA.")
		}
		certs.AppendCertsFromPEM(pemData)
		tr := &http.Transport{
			TLSClientConfig: &tls.Config{RootCAs: certs},
		}
		return &http.Client{Transport: tr}
	}

	// No HTTPS support
	return &http.Client{}
}

// sendData sends the data to the upd server.
func (c *Client) sendData(filename string, data []byte) error {
	// Prepare the multipart content
	body := &bytes.Buffer{}
	writer := multipart.NewWriter(body)

	part, err := writer.CreateFormFile("data", "file")
	if err != nil {
		log.Println("[err] Unable to prepare the multipart content (CreateFormFile)")
		return err
	}

	_, err = io.Copy(part, bytes.NewReader(data))
	if err != nil {
		log.Println("[err] Unable to prepare the multipart content (Copy)")
		return err
	}

	err = writer.Close()
	if err != nil {
		log.Println("[err] Unable to prepare the multipart content (Close)")
		return err
	}

	// create the request
	client := c.createClient()

	uri := c.Flags.ServerUrl + ROUTE_SEND

	params := make(map[string]string)
	params["ttl"] = c.Flags.TTL
	params["name"] = filename

	uri = c.buildParams(uri, params)

	req, err := http.NewRequest("POST", uri, body)
	req.Header.Add("Content-Type", writer.FormDataContentType())
	if err != nil {
		log.Println("[err] Unable to create the request to send the file.")
		return err
	}

	// adds the secret key if any
	if len(c.Flags.SecretKey) > 0 {
		req.Header.Set(server.SECRET_KEY_HEADER, c.Flags.SecretKey)
	}

	// execute
	resp, err := client.Do(req)
	if err != nil {
		log.Println("[err] Unable to execut the request to send the file.")
		return err
	}

	if resp.StatusCode != 200 {
		return fmt.Errorf("[err] Received a %d while sending: %s", resp.StatusCode, filename)
	}

	// read the name given by the server
	defer resp.Body.Close()
	readBody, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		log.Println("[err] Unable to read the body returned by the server.")
		return err
	}

	// decodes the json
	var sendResponse server.SendResponse
	err = json.Unmarshal(readBody, &sendResponse)
	if err != nil {
		log.Println("[err] Unable to read the returned JSON.")
	}

	fmt.Println("For file :", filename)
	fmt.Println("URL:", c.Flags.ServerUrl+"/"+sendResponse.Name)
	fmt.Println("Delete URL:", c.Flags.ServerUrl+"/"+sendResponse.Name+"/"+sendResponse.DeleteKey)

	// compute until when it'll be available
	fmt.Println("Available until:", sendResponse.DeletionTime)
	fmt.Println("--")

	return nil
}

// buildParams adds the GET parameters to the given uri.
func (c *Client) buildParams(uri string, params map[string]string) string {
	if len(params) == 0 {
		return uri
	}
	atLeastOne := false

	ret := uri
	ret += "?"

	for k, v := range params {
		if len(v) > 0 {
			ret = fmt.Sprintf("%s%s=%s&", ret, k, url.QueryEscape(v))
			atLeastOne = true
		}
	}

	// there were parameters but they're all empty
	if !atLeastOne {
		return uri
	}

	ret = ret[0 : len(ret)-1]
	return ret
}

// readFile reads the content of the given file.
func (c *Client) readFile(filename string) ([]byte, error) {
	result := make([]byte, 0)

	// opening
	file, err := os.Open(filename)
	if err != nil {
		return result, err
	}

	// reading
	result, err = ioutil.ReadAll(file)
	if err != nil {
		return result, err
	}

	return result, nil
}
<commit_msg>client: Better display of the TTL value.<commit_after>// Client executable to send file to the upd daemon.
// Copyright © 2015 - Rémy MATHIEU

package client

import (
	"bytes"
	"crypto/tls"
	"crypto/x509"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"mime/multipart"
	"net/http"
	"net/url"
	"os"

	"server"
)

const (
	ROUTE_SEND = "/1.0/send"
)

type Client struct {
	Flags Flags
}

func NewClient(flags Flags) *Client {
	return &Client{Flags: flags}
}

// Send sends the given file to the upd server.
func (c *Client) Send(filename string) error {
	// first, we need to read the data
	data, err := c.readFile(filename)
	if err != nil {
		return err
	}

	// and now to send it the servee
	return c.sendData(filename, data)
}

func (c *Client) createClient() *http.Client {
	if c.Flags.CA == "unsafe" {
		tr := &http.Transport{
			TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
		}
		return &http.Client{Transport: tr}
	} else if len(c.Flags.CA) > 0 && c.Flags.CA != "none" {
		// reads the CA
		certs := x509.NewCertPool()
		pemData, err := ioutil.ReadFile(c.Flags.CA)
		if err != nil {
			log.Println("[err] Can't read the CA.")
		}
		certs.AppendCertsFromPEM(pemData)
		tr := &http.Transport{
			TLSClientConfig: &tls.Config{RootCAs: certs},
		}
		return &http.Client{Transport: tr}
	}

	// No HTTPS support
	return &http.Client{}
}

// sendData sends the data to the upd server.
func (c *Client) sendData(filename string, data []byte) error {
	// Prepare the multipart content
	body := &bytes.Buffer{}
	writer := multipart.NewWriter(body)

	part, err := writer.CreateFormFile("data", "file")
	if err != nil {
		log.Println("[err] Unable to prepare the multipart content (CreateFormFile)")
		return err
	}

	_, err = io.Copy(part, bytes.NewReader(data))
	if err != nil {
		log.Println("[err] Unable to prepare the multipart content (Copy)")
		return err
	}

	err = writer.Close()
	if err != nil {
		log.Println("[err] Unable to prepare the multipart content (Close)")
		return err
	}

	// create the request
	client := c.createClient()

	uri := c.Flags.ServerUrl + ROUTE_SEND

	params := make(map[string]string)
	params["ttl"] = c.Flags.TTL
	params["name"] = filename

	uri = c.buildParams(uri, params)

	req, err := http.NewRequest("POST", uri, body)
	req.Header.Add("Content-Type", writer.FormDataContentType())
	if err != nil {
		log.Println("[err] Unable to create the request to send the file.")
		return err
	}

	// adds the secret key if any
	if len(c.Flags.SecretKey) > 0 {
		req.Header.Set(server.SECRET_KEY_HEADER, c.Flags.SecretKey)
	}

	// execute
	resp, err := client.Do(req)
	if err != nil {
		log.Println("[err] Unable to execut the request to send the file.")
		return err
	}

	if resp.StatusCode != 200 {
		return fmt.Errorf("[err] Received a %d while sending: %s", resp.StatusCode, filename)
	}

	// read the name given by the server
	defer resp.Body.Close()
	readBody, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		log.Println("[err] Unable to read the body returned by the server.")
		return err
	}

	// decodes the json
	var sendResponse server.SendResponse
	err = json.Unmarshal(readBody, &sendResponse)
	if err != nil {
		log.Println("[err] Unable to read the returned JSON.")
	}

	fmt.Println("For file :", filename)
	fmt.Println("URL:", c.Flags.ServerUrl+"/"+sendResponse.Name)
	fmt.Println("Delete URL:", c.Flags.ServerUrl+"/"+sendResponse.Name+"/"+sendResponse.DeleteKey)

	// compute until when it'll be available
	if sendResponse.DeletionTime.IsZero() {
		fmt.Println("Available forever.")
	} else {
		fmt.Println("Available until:", sendResponse.DeletionTime)
	}
	fmt.Println("--")

	return nil
}

// buildParams adds the GET parameters to the given uri.
func (c *Client) buildParams(uri string, params map[string]string) string {
	if len(params) == 0 {
		return uri
	}
	atLeastOne := false

	ret := uri
	ret += "?"

	for k, v := range params {
		if len(v) > 0 {
			ret = fmt.Sprintf("%s%s=%s&", ret, k, url.QueryEscape(v))
			atLeastOne = true
		}
	}

	// there were parameters but they're all empty
	if !atLeastOne {
		return uri
	}

	ret = ret[0 : len(ret)-1]
	return ret
}

// readFile reads the content of the given file.
func (c *Client) readFile(filename string) ([]byte, error) {
	result := make([]byte, 0)

	// opening
	file, err := os.Open(filename)
	if err != nil {
		return result, err
	}

	// reading
	result, err = ioutil.ReadAll(file)
	if err != nil {
		return result, err
	}

	return result, nil
}
<|endoftext|>


<commit_before>// this short program iterate over the certificates index
// to extract unique domains and IPs from recorded certificates
// and write them into two files in /tmp/

// Julien Vehent - Feb. 2015
package main

import (
	"encoding/json"
	"flag"
	"fmt"
	elastigo "github.com/mattbaird/elastigo/lib"
	"github.com/streadway/amqp"
	"time"
)

type StoredCertificate struct {
	Domains []string `json:"domains,omitempty"`
}

func main() {
	var (
		esaddr, mqaddr, window string
		es                     *elastigo.Conn
		from, to               time.Time
	)
	flag.StringVar(&esaddr, "e", "localhost:9200", "Address of the ElasticSearch database")
	flag.StringVar(&mqaddr, "m", "amqp://guest:guest@localhost:5672/", "Address of the RabbitMQ broker")
	flag.StringVar(&window, "w", "360h", "Time window to cover, in hours (360h = 15 days)")
	flag.Parse()

	es = elastigo.NewConn()
	es.Domain = esaddr

	mqconn, err := amqp.Dial(mqaddr)
	if err != nil {
		panic(err)
	}
	defer mqconn.Close()

	mqch, err := mqconn.Channel()
	if err != nil {
		panic(err)
	}
	defer mqch.Close()

	dctr := 0
	domains := make(map[string]int)

	// start the search at now - window, iterate over all certs by chunks of 5 minutes,
	// and build a list of unique domains
	dur, err := time.ParseDuration(window)
	if err != nil {
		panic(err)
	}
	to = time.Now().Add(-dur)
	for {
		// advance the window by 5 minutes
		from = to
		to = from.Add(5 * time.Minute)
		if from.After(time.Now()) {
			break
		}
		// ES query date is in format 2015-02-12T07:45:22
		filter := fmt.Sprintf(`{
			"from" : 0, "size" : 100000,
			"query": {
				"range": {
					"lastSeenTimestamp": {
						"from": "%s",
						"to": "%s"
					}
				}
			}
		}`, from.Format("2006-01-02T15:04:05"), to.Format("2006-01-02T15:04:05"))
		res, err := es.Search("certificates", "certificateInfo", nil, filter)
		if err != nil {
			panic(err)
		}
		fmt.Println(len(res.Hits.Hits), "records found for time window [", from.String(), ",", to.String(), "]")
		for _, storedCert := range res.Hits.Hits {
			cert := new(StoredCertificate)
			err = json.Unmarshal(*storedCert.Source, cert)
			if err != nil {
				panic(err)
			}
			for _, d := range cert.Domains {
				if _, ok := domains[d]; !ok {
					err = mqch.Publish(
						"",                 // exchange
						"scan_ready_queue", // routing key
						false,              // mandatory
						false,
						amqp.Publishing{
							DeliveryMode: amqp.Persistent,
							ContentType:  "text/plain",
							Body:         []byte(d),
						})
					if err != nil {
						panic(err)
					}
					dctr++
					domains[d] = dctr
					fmt.Println("send domain", d, "to scan_ready_queue")
				}
			}
		}
	}
	fmt.Println("Done.", dctr, "domains sent to scanning queue.")
}
<commit_msg>change rescanDomains to pull the list of domains from CN & SAN<commit_after>// this short program iterate over the certificates index
// to extract unique domains and IPs from recorded certificates
// and write them into two files in /tmp/

// Julien Vehent - Feb. 2015
package main

import (
	"encoding/json"
	"flag"
	"fmt"
	elastigo "github.com/mattbaird/elastigo/lib"
	"github.com/streadway/amqp"
	"time"
)

// limit the structs to just the amount of data we need: domain names and CA status
type StoredCertificate struct {
	Subject          certSubject    `json:"subject"`
	X509v3Extensions certExtensions `json:"x509v3Extensions"`
	CA               bool           `json:"ca"`
}
type certSubject struct {
	CommonName string `json:"cn"`
}
type certExtensions struct {
	SubjectAlternativeName []string `json:"subjectAlternativeName"`
}

func main() {
	var (
		esaddr, mqaddr, window string
		es                     *elastigo.Conn
		from, to               time.Time
	)
	flag.StringVar(&esaddr, "e", "localhost:9200", "Address of the ElasticSearch database")
	flag.StringVar(&mqaddr, "m", "amqp://guest:guest@localhost:5672/", "Address of the RabbitMQ broker")
	flag.StringVar(&window, "w", "360h", "Time window to cover, in hours (360h = 15 days)")
	flag.Parse()

	es = elastigo.NewConn()
	es.Domain = esaddr

	mqconn, err := amqp.Dial(mqaddr)
	if err != nil {
		panic(err)
	}
	defer mqconn.Close()

	mqch, err := mqconn.Channel()
	if err != nil {
		panic(err)
	}
	defer mqch.Close()

	dctr := 0
	domains := make(map[string]int)

	// start the search at now - window, iterate over all certs by chunks of 5 minutes,
	// and build a list of unique domains
	dur, err := time.ParseDuration(window)
	if err != nil {
		panic(err)
	}
	to = time.Now().Add(-dur)
	for {
		// advance the window by 5 minutes
		from = to
		to = from.Add(5 * time.Minute)
		if from.After(time.Now()) {
			break
		}
		// ES query date is in format 2015-02-12T07:45:22
		filter := fmt.Sprintf(`{
			"from" : 0, "size" : 100000,
			"query": {
				"range": {
					"lastSeenTimestamp": {
						"from": "%s",
						"to": "%s"
					}
				}
			}
		}`, from.Format("2006-01-02T15:04:05"), to.Format("2006-01-02T15:04:05"))
		res, err := es.Search("certificates", "certificateInfo", nil, filter)
		if err != nil {
			panic(err)
		}
		fmt.Println(len(res.Hits.Hits), "records found for time window [", from.String(), ",", to.String(), "]")
		for _, storedCert := range res.Hits.Hits {
			cert := new(StoredCertificate)
			err = json.Unmarshal(*storedCert.Source, cert)
			if err != nil {
				panic(err)
			}
			// skip certificate authorities
			if cert.CA {
				continue
			}
			// build a list of domains from the certs SAN records and its subject.CN
			// then scan the ones that haven't been scanned yet
			dlist := cert.X509v3Extensions.SubjectAlternativeName
			dlist = append(dlist, cert.Subject.CommonName)
			for _, d := range dlist {
				if _, ok := domains[d]; !ok {
					err = mqch.Publish(
						"",                 // exchange
						"scan_ready_queue", // routing key
						false,              // mandatory
						false,
						amqp.Publishing{
							DeliveryMode: amqp.Persistent,
							ContentType:  "text/plain",
							Body:         []byte(d),
						})
					if err != nil {
						panic(err)
					}
					dctr++
					domains[d] = dctr
					fmt.Println("send domain", d, "to scan_ready_queue")
				}
			}
		}
	}
	fmt.Println("Done.", dctr, "domains sent to scanning queue.")
}
<|endoftext|>


<commit_before>package network

import (
	"common"
	"encoding/binary"
	"net"
	"strconv"
)

// TCPServer is a MessageSender that communicates over TCP.
type TCPServer struct {
	Addr            common.Address
	MessageHandlers []common.MessageHandler
	Listener        net.Listener
}

func (tcp *TCPServer) Address() common.Address {
	return tcp.Addr
}

// AddMessageHandler adds a MessageHandler to the MessageHandlers slice.
// It creates an identifier associated with that MessageHandler, and returns
// an Address incorporating the identifier.
func (tcp *TCPServer) AddMessageHandler(mh common.MessageHandler) common.Address {
	tcp.MessageHandlers = append(tcp.MessageHandlers, mh)
	addr := tcp.Addr
	addr.Id = common.Identifier(len(tcp.MessageHandlers) - 1)
	return addr
}

// SendMessage transmits the payload of a message to its intended recipient.
// It marshalls the Message struct using a length-prefix scheme.
// It does not wait for a response.
func (tcp *TCPServer) SendMessage(m *common.Message) (err error) {
	conn, err := net.Dial("tcp", net.JoinHostPort(m.Destination.Host, strconv.Itoa(m.Destination.Port)))
	if err != nil {
		return
	}
	defer conn.Close()

	// construct stream to be transmitted
	// bytes 0:3 are the payload length
	// byte 4 is the destination identifier
	// the remainder is the payload
	payloadLength := make([]byte, 4)
	binary.PutUvarint(payloadLength, uint64(len(m.Payload)))
	stream := append(payloadLength, byte(m.Destination.Id))
	stream = append(stream, m.Payload...)

	// transmit stream
	_, err = conn.Write(stream)
	if err != nil {
		return
	}

	return
}

// NewTCPServer creates and initializes a server that listens for TCP connections on a specified port.
// It then spawns a serverHandler with a specified message.
// It is the serverHandler's responsibility to close the TCP connection.
func NewTCPServer(port int) (tcp *TCPServer, err error) {
	tcp = new(TCPServer)
	tcpServ, err := net.Listen("tcp", ":"+strconv.Itoa(port))
	if err != nil {
		return
	}

	// initialize struct fields
	tcp.Addr = common.Address{0, "localhost", port}
	// MessageHandlers[0] is reserved for the MessageHandler of the TCPServer
	tcp.MessageHandlers = make([]common.MessageHandler, 1)
	tcp.Listener = tcpServ

	go tcp.serverHandler()
	return
}

// Close closes the connection associated with the TCP server.
// This causes tcpServ.Accept() to return an err, ending the serverHandler process
func (tcp *TCPServer) Close() {
	tcp.Listener.Close()
}

// serverHandler accepts incoming connections and spawns a clientHandler for each.
func (tcp *TCPServer) serverHandler() {
	for {
		conn, err := tcp.Listener.Accept()
		if err != nil {
			return
		} else {
			tcp.clientHandler(conn)
			conn.Close()
		}
	}
}

// clientHandler reads data sent by a client and processes it.
func (tcp *TCPServer) clientHandler(conn net.Conn) {
	// read first 1024 bytes
	buffer := make([]byte, 1024)

	// read first 1024 bytes
	b, err := conn.Read(buffer)
	if err != nil {
		return
	}

	// split message into payload length, identifier, and payload
	payloadLength, _ := binary.Uvarint(buffer[:4])
	id := int(buffer[4])
	payload := make([]byte, b-5)
	copy(payload, buffer[5:b])

	// read rest of payload, 1024 bytes at a time
	// TODO: add a timeout
	bytesRead := len(payload)
	for uint64(bytesRead) != payloadLength {
		b, err = conn.Read(buffer)
		if err != nil {
			return
		}
		payload = append(payload, buffer[:b]...)
		bytesRead += b
	}

	// Message sent directly to TCPServer
	// for now, just send it to the first message handler
	if id == 0 {
		tcp.MessageHandlers[1].HandleMessage(payload)
		return
	}

	// look up message handler and call it
	if id < len(tcp.MessageHandlers) {
		go tcp.MessageHandlers[id].HandleMessage(payload)
	}
}
<commit_msg>initialize TCPServer struct directly<commit_after>package network

import (
	"common"
	"encoding/binary"
	"net"
	"strconv"
)

// TCPServer is a MessageSender that communicates over TCP.
type TCPServer struct {
	addr            common.Address
	MessageHandlers []common.MessageHandler
	listener        net.Listener
}

func (tcp *TCPServer) Address() common.Address {
	return tcp.addr
}

// AddMessageHandler adds a MessageHandler to the MessageHandlers slice.
// It creates an identifier associated with that MessageHandler, and returns
// an Address incorporating the identifier.
func (tcp *TCPServer) AddMessageHandler(mh common.MessageHandler) common.Address {
	tcp.MessageHandlers = append(tcp.MessageHandlers, mh)
	addr := tcp.addr
	addr.Id = common.Identifier(len(tcp.MessageHandlers) - 1)
	return addr
}

// SendMessage transmits the payload of a message to its intended recipient.
// It marshalls the Message struct using a length-prefix scheme.
// It does not wait for a response.
func (tcp *TCPServer) SendMessage(m *common.Message) (err error) {
	conn, err := net.Dial("tcp", net.JoinHostPort(m.Destination.Host, strconv.Itoa(m.Destination.Port)))
	if err != nil {
		return
	}
	defer conn.Close()

	// construct stream to be transmitted
	// bytes 0:3 are the payload length
	// byte 4 is the destination identifier
	// the remainder is the payload
	payloadLength := make([]byte, 4)
	binary.PutUvarint(payloadLength, uint64(len(m.Payload)))
	stream := append(payloadLength, byte(m.Destination.Id))
	stream = append(stream, m.Payload...)

	// transmit stream
	_, err = conn.Write(stream)
	if err != nil {
		return
	}

	return
}

// NewTCPServer creates and initializes a server that listens for TCP connections on a specified port.
// It then spawns a serverHandler with a specified message.
// It is the serverHandler's responsibility to close the TCP connection.
func NewTCPServer(port int) (tcp *TCPServer, err error) {
	tcpServ, err := net.Listen("tcp", ":"+strconv.Itoa(port))
	if err != nil {
		return
	}

	tcp = &TCPServer{
		common.Address{0, "localhost", port},
		make([]common.MessageHandler, 1),
		tcpServ,
	}

	go tcp.serverHandler()
	return
}

// Close closes the connection associated with the TCP server.
// This causes tcpServ.Accept() to return an err, ending the serverHandler process
func (tcp *TCPServer) Close() {
	tcp.listener.Close()
}

// serverHandler accepts incoming connections and spawns a clientHandler for each.
func (tcp *TCPServer) serverHandler() {
	for {
		conn, err := tcp.listener.Accept()
		if err != nil {
			return
		} else {
			tcp.clientHandler(conn)
			conn.Close()
		}
	}
}

// clientHandler reads data sent by a client and processes it.
func (tcp *TCPServer) clientHandler(conn net.Conn) {
	// read first 1024 bytes
	buffer := make([]byte, 1024)

	// read first 1024 bytes
	b, err := conn.Read(buffer)
	if err != nil {
		return
	}

	// split message into payload length, identifier, and payload
	payloadLength, _ := binary.Uvarint(buffer[:4])
	id := int(buffer[4])
	payload := make([]byte, b-5)
	copy(payload, buffer[5:b])

	// read rest of payload, 1024 bytes at a time
	// TODO: add a timeout
	bytesRead := len(payload)
	for uint64(bytesRead) != payloadLength {
		b, err = conn.Read(buffer)
		if err != nil {
			return
		}
		payload = append(payload, buffer[:b]...)
		bytesRead += b
	}

	// Message sent directly to TCPServer
	// for now, just send it to the first message handler
	if id == 0 {
		tcp.MessageHandlers[1].HandleMessage(payload)
		return
	}

	// look up message handler and call it
	if id < len(tcp.MessageHandlers) {
		go tcp.MessageHandlers[id].HandleMessage(payload)
	}
}
<|endoftext|>