<commit_before>// Package schemas provides a container for Cap'n Proto reflection data.
// The code generated by capnpc-go will register its schema in the
// default registry (unless disabled at generation time).
//
// Most programs will use the default registry.  However, a program
// could dynamically build up a registry, perhaps by invoking the capnp
// tool or querying a service.
package schemas

import (
	"bytes"
	"compress/zlib"
	"errors"
	"io"
	"io/ioutil"
	"strconv"
	"strings"
	"sync"

	"zombiezen.com/go/capnproto2/internal/packed"
)

// A Schema is a collection of schema nodes parsed by the capnp tool.
type Schema struct {
	// Either String or Bytes must be populated with a CodeGeneratorRequest
	// message in the standard Cap'n Proto framing format.
	String string
	Bytes  []byte

	// If true, the input is assumed to be zlib-compressed and packed.
	Compressed bool

	// Node IDs that are contained in this schema.
	Nodes []uint64
}

// A Registry is a mapping of IDs to schema blobs.  It is safe to read
// from multiple goroutines.  The zero value is an empty registry.
type Registry struct {
	m map[uint64]*record
}

// Register indexes a schema in the registry.  It is an error to
// register schemas with overlapping IDs.
func (reg *Registry) Register(s *Schema) error {
	if len(s.String) > 0 && len(s.Bytes) > 0 {
		return errors.New("schemas: schema should have only one of string or bytes")
	}
	r := &record{
		s:          s.String,
		data:       s.Bytes,
		compressed: s.Compressed,
	}
	if reg.m == nil {
		reg.m = make(map[uint64]*record)
	}
	for _, id := range s.Nodes {
		if _, dup := reg.m[id]; dup {
			return errors.New("schemas: registered ID @0x" + strconv.FormatUint(id, 16) + " twice")
		}
		reg.m[id] = r
	}
	return nil
}

// Find returns the CodeGeneratorRequest message for the given ID,
// suitable for capnp.Unmarshal.  If the ID is not found, Find returns
// nil, nil.  The returned byte slice should not be modified.
func (reg *Registry) Find(id uint64) ([]byte, error) {
	r := reg.m[id]
	if r == nil {
		return nil, nil
	}
	b, err := r.read()
	if err != nil {
		return nil, decompressError(id, err)
	}
	return b, nil
}

type record struct {
	// All the fields are protected by once.
	once       sync.Once
	s          string // input
	compressed bool
	data       []byte // input and result
	err        error  // result
}

func (r *record) read() ([]byte, error) {
	r.once.Do(func() {
		if !r.compressed {
			if r.s != "" {
				r.data = []byte(r.s)
				r.s = ""
			}
			return
		}
		var in io.Reader
		if r.s != "" {
			in = strings.NewReader(r.s)
			r.s = ""
		} else {
			in = bytes.NewReader(r.data)
		}
		z, err := zlib.NewReader(in)
		if err != nil {
			r.data, r.err = nil, err
			return
		}
		p := packed.NewReader(z)
		r.data, r.err = ioutil.ReadAll(p)
		if err != nil {
			r.data = nil
			return
		}
	})
	return r.data, r.err
}

// DefaultRegistry is the process-wide registry used by Register and Find.
var DefaultRegistry Registry

// Register is called by generated code to associate a blob of zlib-
// compressed, packed Cap'n Proto data for a CodeGeneratorRequest with
// the IDs it contains.  It should only be called during init().
func Register(data string, ids ...uint64) {
	err := DefaultRegistry.Register(&Schema{
		String:     data,
		Nodes:      ids,
		Compressed: true,
	})
	if err != nil {
		panic(err)
	}
}

// Find returns the CodeGeneratorRequest message for the given ID,
// suitable for capnp.Unmarshal, or nil if the ID was not found.
// It is safe to call Find from multiple goroutines, so the returned
// byte slice should not be modified.  However, it is not safe to
// call Find concurrently with Register.
func Find(id uint64) []byte {
	b, err := DefaultRegistry.Find(id)
	if err != nil {
		panic(err)
	}
	return b
}

func decompressError(id uint64, e error) error {
	return errors.New("schemas: decompressing schema for @0x" + strconv.FormatUint(id, 16) + ": " + e.Error())
}
<commit_msg>schemas: add IsNotFound function<commit_after>// Package schemas provides a container for Cap'n Proto reflection data.
// The code generated by capnpc-go will register its schema in the
// default registry (unless disabled at generation time).
//
// Most programs will use the default registry.  However, a program
// could dynamically build up a registry, perhaps by invoking the capnp
// tool or querying a service.
package schemas

import (
	"bytes"
	"compress/zlib"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"strings"
	"sync"

	"zombiezen.com/go/capnproto2/internal/packed"
)

// A Schema is a collection of schema nodes parsed by the capnp tool.
type Schema struct {
	// Either String or Bytes must be populated with a CodeGeneratorRequest
	// message in the standard Cap'n Proto framing format.
	String string
	Bytes  []byte

	// If true, the input is assumed to be zlib-compressed and packed.
	Compressed bool

	// Node IDs that are contained in this schema.
	Nodes []uint64
}

// A Registry is a mapping of IDs to schema blobs.  It is safe to read
// from multiple goroutines.  The zero value is an empty registry.
type Registry struct {
	m map[uint64]*record
}

// Register indexes a schema in the registry.  It is an error to
// register schemas with overlapping IDs.
func (reg *Registry) Register(s *Schema) error {
	if len(s.String) > 0 && len(s.Bytes) > 0 {
		return errors.New("schemas: schema should have only one of string or bytes")
	}
	r := &record{
		s:          s.String,
		data:       s.Bytes,
		compressed: s.Compressed,
	}
	if reg.m == nil {
		reg.m = make(map[uint64]*record)
	}
	for _, id := range s.Nodes {
		if _, dup := reg.m[id]; dup {
			return &dupeError{id: id}
		}
		reg.m[id] = r
	}
	return nil
}

// Find returns the CodeGeneratorRequest message for the given ID,
// suitable for capnp.Unmarshal.  If the ID is not found, Find returns
// an error that can be identified with IsNotFound.  The returned byte
// slice should not be modified.
func (reg *Registry) Find(id uint64) ([]byte, error) {
	r := reg.m[id]
	if r == nil {
		return nil, nil
	}
	b, err := r.read()
	if err != nil {
		return nil, &decompressError{id, err}
	}
	return b, nil
}

type record struct {
	// All the fields are protected by once.
	once       sync.Once
	s          string // input
	compressed bool
	data       []byte // input and result
	err        error  // result
}

func (r *record) read() ([]byte, error) {
	r.once.Do(func() {
		if !r.compressed {
			if r.s != "" {
				r.data = []byte(r.s)
				r.s = ""
			}
			return
		}
		var in io.Reader
		if r.s != "" {
			in = strings.NewReader(r.s)
			r.s = ""
		} else {
			in = bytes.NewReader(r.data)
		}
		z, err := zlib.NewReader(in)
		if err != nil {
			r.data, r.err = nil, err
			return
		}
		p := packed.NewReader(z)
		r.data, r.err = ioutil.ReadAll(p)
		if err != nil {
			r.data = nil
			return
		}
	})
	return r.data, r.err
}

// DefaultRegistry is the process-wide registry used by Register and Find.
var DefaultRegistry Registry

// Register is called by generated code to associate a blob of zlib-
// compressed, packed Cap'n Proto data for a CodeGeneratorRequest with
// the IDs it contains.  It should only be called during init().
func Register(data string, ids ...uint64) {
	err := DefaultRegistry.Register(&Schema{
		String:     data,
		Nodes:      ids,
		Compressed: true,
	})
	if err != nil {
		panic(err)
	}
}

// Find returns the CodeGeneratorRequest message for the given ID,
// suitable for capnp.Unmarshal, or nil if the ID was not found.
// It is safe to call Find from multiple goroutines, so the returned
// byte slice should not be modified.  However, it is not safe to
// call Find concurrently with Register.
func Find(id uint64) []byte {
	b, err := DefaultRegistry.Find(id)
	if IsNotFound(err) {
		return nil
	}
	if err != nil {
		panic(err)
	}
	return b
}

// IsNotFound reports whether e indicates a failure to find a schema.
func IsNotFound(e error) bool {
	_, ok := e.(*notFoundError)
	return ok
}

type dupeError struct {
	id uint64
}

func (e *dupeError) Error() string {
	return fmt.Sprintf("schemas: registered @%#x twice", e.id)
}

type notFoundError struct {
	id uint64
}

func (e *notFoundError) Error() string {
	return fmt.Sprintf("schemas: could not find @%#x", e.id)
}

type decompressError struct {
	id  uint64
	err error
}

func (e *decompressError) Error() string {
	return fmt.Sprintf("schemas: decompressing schema for @%#x: %v", e.id, e.err)
}
<|endoftext|>


<commit_before>/*
	Used by the Leasing Server to poll swarming.
*/

package main

import (
	"fmt"
	"strconv"
	"time"

	"cloud.google.com/go/datastore"
	"google.golang.org/api/iterator"

	"go.skia.org/infra/go/sklog"
	"go.skia.org/infra/go/swarming"
)

// populateRunningTask updates the provided Task struct with a new state, botId, lease start/end times,
// and sends a start email.
func populateRunningTask(newState, botId string, t *Task, k *datastore.Key) error {
	// Update the state and add the bot name.
	t.SwarmingTaskState = newState
	t.SwarmingBotId = botId

	// Add the start and end lease times.
	durationHrs, err := strconv.Atoi(t.InitialDurationHrs)
	if err != nil {
		return fmt.Errorf("Failed to parse %s", t.InitialDurationHrs)
	}
	t.LeaseStartTime = time.Now()
	t.LeaseEndTime = time.Now().Add(time.Hour * time.Duration(durationHrs))

	if _, err := UpdateDSTask(k, t); err != nil {
		return fmt.Errorf("Error updating task in datastore: %v", err)
	}

	// Inform the requester that the task has been picked up
	if err := SendStartEmail(t.Requester, t.SwarmingServer, t.SwarmingTaskId, t.SwarmingBotId, t.TaskIdForIsolates); err != nil {
		return fmt.Errorf("Error sending start email: %s", err)
	}

	return nil
}

// expireTask marks the provided Task struct as Done and sends a completion email.
func expireTask(t *Task, k *datastore.Key) error {
	t.Done = true
	t.SwarmingTaskState = getCompletedStateStr(false)
	if _, err := UpdateDSTask(k, t); err != nil {
		return fmt.Errorf("Error updating task in datastore: %v", err)
	}
	sklog.Infof("Marked as expired task %v in the datastore with key %d", t, k.ID)
	// Inform the requester that the task has completed.
	if err := SendCompletionEmail(t.Requester, t.SwarmingServer, t.SwarmingTaskId, t.SwarmingBotId); err != nil {
		return fmt.Errorf("Error sending completion email: %s", err)
	}
	return nil
}

// taskExpiringSoon sends a warning email and updates the WarningSent field in the Task struct.
func taskExpiringSoon(t *Task, k *datastore.Key) error {
	if err := SendWarningEmail(t.Requester, t.SwarmingServer, t.SwarmingTaskId, t.SwarmingBotId); err != nil {
		return fmt.Errorf("Error sending 15m warning email: %s", err)
	}
	t.WarningSent = true
	if _, err := UpdateDSTask(k, t); err != nil {
		return fmt.Errorf("Error updating task in datastore: %v", err)
	}
	return nil
}

func getCompletedStateStr(failure bool) string {
	if failure {
		return swarming.TASK_STATE_COMPLETED + " (FAILURE)"
	} else {
		return swarming.TASK_STATE_COMPLETED + " (SUCCESS)"
	}
}

// checkForUnexpectedStates checks to see if the new state falls in a list of unexpected states.
// If it does then the Task is marked as Done, the lease ended and a failure email it sent.
func checkForUnexpectedStates(newState string, failure bool, t *Task, k *datastore.Key) error {
	unexpectedStates := []string{
		swarming.TASK_STATE_BOT_DIED,
		swarming.TASK_STATE_CANCELED,
		swarming.TASK_STATE_COMPLETED,
		swarming.TASK_STATE_EXPIRED,
		swarming.TASK_STATE_TIMED_OUT,
	}
	for _, unexpectedState := range unexpectedStates {
		if newState == unexpectedState {
			// Update the state.
			if newState == swarming.TASK_STATE_COMPLETED {
				newState = getCompletedStateStr(failure)
			}
			t.SwarmingTaskState = newState
			// Something unexpected happened so mark the leasing task as done and end the lease.
			t.Done = true
			t.LeaseEndTime = time.Now()

			if _, err := UpdateDSTask(k, t); err != nil {
				return fmt.Errorf("Error updating task in datastore: %v", err)
			}

			// Inform the requester that something went wrong.
			if err := SendFailureEmail(t.Requester, t.SwarmingServer, t.SwarmingTaskId, t.SwarmingBotId, t.SwarmingTaskState); err != nil {
				return fmt.Errorf("Error sending failure email: %s", err)
			}
			break
		}
	}
	return nil
}

// pollSwarmingTasks gets all running tasks from the Datastore, polls the equivalent
// tasks in swarming, and updates the tasks in the Datastore accordingly.
func pollSwarmingTasks() error {

	it := GetRunningDSTasks()
	for {
		t := &Task{}
		k, err := it.Next(t)
		if err == iterator.Done {
			break
		} else if err != nil {
			return fmt.Errorf("Failed to retrieve list of tasks: %s", err)
		}

		if t.SwarmingTaskId == "" {
			// This task is not ready to be looked at yet.
			continue
		}

		// Get the swarming task from swarming server.
		swarmingTask, err := GetSwarmingTask(t.SwarmingPool, t.SwarmingTaskId)
		if err != nil {
			return fmt.Errorf("Failed to retrieve swarming task %s: %s", t.SwarmingTaskId, err)
		}

		if swarmingTask.State == swarming.TASK_STATE_PENDING {
			// If the swarming task is still pending then there is nothing to do here.
			continue
		}

		// Check for 4 things-
		//
		// Check 1: If the previous state was pending and the new state is not then
		//          update the state, add botId and populate the lease start/end time.
		// Check 2: If lease has already expired then set Done=true.
		// Check 3: If lease is expiring in 15 mins then send a warning email if we
		//          have not done so already.
		// Check 4: If new state is unexpected then set Done=true and send failure email
		//          to the requester.

		if t.SwarmingTaskState == swarming.TASK_STATE_PENDING /* Check 1 */ {
			// The previous task state was pending but this has now changed.
			// Populate the datastore with running task values.
			if err := populateRunningTask(swarmingTask.State, swarmingTask.BotId, t, k); err != nil {
				return fmt.Errorf("Error populating running task: %s", err)
			}
		} else if t.LeaseEndTime.Before(time.Now()) /* Check 2*/ {
			// The task has expired.
			if err := expireTask(t, k); err != nil {
				return fmt.Errorf("Error when expiring task: %s", err)
			}
		} else if swarmingTask.State == swarming.TASK_STATE_RUNNING && !t.WarningSent && t.LeaseEndTime.Before(time.Now().Add(time.Minute*15)) /* Check 3 */ {
			if err := taskExpiringSoon(t, k); err != nil {
				return fmt.Errorf("Error when warning task expiring soon: %s", err)
			}
		} else /* Check 4 */ {

			if err := checkForUnexpectedStates(swarmingTask.State, swarmingTask.Failure, t, k); err != nil {
				return fmt.Errorf("Error when warning task expiring soon: %s", err)
			}
		}
	}

	return nil
}
<commit_msg>[Leasing Server] Handle case where leasing tasks are cancelled<commit_after>/*
	Used by the Leasing Server to poll swarming.
*/

package main

import (
	"fmt"
	"strconv"
	"time"

	"cloud.google.com/go/datastore"
	"google.golang.org/api/iterator"

	"go.skia.org/infra/go/sklog"
	"go.skia.org/infra/go/swarming"
)

// populateRunningTask updates the provided Task struct with a new state, botId, lease start/end times,
// and sends a start email.
func populateRunningTask(newState, botId string, k *datastore.Key, t *Task) error {
	// Update the state and add the bot name.
	t.SwarmingTaskState = newState
	t.SwarmingBotId = botId

	// Add the start and end lease times.
	durationHrs, err := strconv.Atoi(t.InitialDurationHrs)
	if err != nil {
		return fmt.Errorf("Failed to parse %s", t.InitialDurationHrs)
	}
	t.LeaseStartTime = time.Now()
	t.LeaseEndTime = time.Now().Add(time.Hour * time.Duration(durationHrs))

	if _, err := UpdateDSTask(k, t); err != nil {
		return fmt.Errorf("Error updating task in datastore: %v", err)
	}

	// Inform the requester that the task has been picked up
	if err := SendStartEmail(t.Requester, t.SwarmingServer, t.SwarmingTaskId, t.SwarmingBotId, t.TaskIdForIsolates); err != nil {
		return fmt.Errorf("Error sending start email: %s", err)
	}

	return nil
}

// expireTask marks the provided Task struct as Done and sends a completion email.
func expireTask(k *datastore.Key, t *Task) error {
	t.Done = true
	t.SwarmingTaskState = getCompletedStateStr(false)
	if _, err := UpdateDSTask(k, t); err != nil {
		return fmt.Errorf("Error updating task in datastore: %v", err)
	}
	sklog.Infof("Marked as expired task %v in the datastore with key %d", t, k.ID)
	// Inform the requester that the task has completed.
	if err := SendCompletionEmail(t.Requester, t.SwarmingServer, t.SwarmingTaskId, t.SwarmingBotId); err != nil {
		return fmt.Errorf("Error sending completion email: %s", err)
	}
	return nil
}

// taskExpiringSoon sends a warning email and updates the WarningSent field in the Task struct.
func taskExpiringSoon(k *datastore.Key, t *Task) error {
	if err := SendWarningEmail(t.Requester, t.SwarmingServer, t.SwarmingTaskId, t.SwarmingBotId); err != nil {
		return fmt.Errorf("Error sending 15m warning email: %s", err)
	}
	t.WarningSent = true
	if _, err := UpdateDSTask(k, t); err != nil {
		return fmt.Errorf("Error updating task in datastore: %v", err)
	}
	return nil
}

// taskCancelled marks the provided Task struct as cancelled.
func taskCancelled(k *datastore.Key, t *Task) error {
	t.Done = true
	t.SwarmingTaskState = getCompletedStateStr(true)
	t.LeaseStartTime = time.Now()
	t.LeaseEndTime = time.Now()
	if _, err := UpdateDSTask(k, t); err != nil {
		return fmt.Errorf("Error updating task in datastore: %v", err)
	}
	return nil
}

func getCompletedStateStr(failure bool) string {
	if failure {
		return swarming.TASK_STATE_COMPLETED + " (FAILURE)"
	} else {
		return swarming.TASK_STATE_COMPLETED + " (SUCCESS)"
	}
}

// checkForUnexpectedStates checks to see if the new state falls in a list of unexpected states.
// If it does then the Task is marked as Done, the lease ended and a failure email it sent.
func checkForUnexpectedStates(newState string, failure bool, k *datastore.Key, t *Task) error {
	unexpectedStates := []string{
		swarming.TASK_STATE_BOT_DIED,
		swarming.TASK_STATE_CANCELED,
		swarming.TASK_STATE_COMPLETED,
		swarming.TASK_STATE_EXPIRED,
		swarming.TASK_STATE_TIMED_OUT,
	}
	for _, unexpectedState := range unexpectedStates {
		if newState == unexpectedState {
			// Update the state.
			if newState == swarming.TASK_STATE_COMPLETED {
				newState = getCompletedStateStr(failure)
			}
			t.SwarmingTaskState = newState
			// Something unexpected happened so mark the leasing task as done and end the lease.
			t.Done = true
			t.LeaseEndTime = time.Now()

			if _, err := UpdateDSTask(k, t); err != nil {
				return fmt.Errorf("Error updating task in datastore: %v", err)
			}

			// Inform the requester that something went wrong.
			if err := SendFailureEmail(t.Requester, t.SwarmingServer, t.SwarmingTaskId, t.SwarmingBotId, t.SwarmingTaskState); err != nil {
				return fmt.Errorf("Error sending failure email: %s", err)
			}
			break
		}
	}
	return nil
}

// pollSwarmingTasks gets all running tasks from the Datastore, polls the equivalent
// tasks in swarming, and updates the tasks in the Datastore accordingly.
func pollSwarmingTasks() error {

	it := GetRunningDSTasks()
	for {
		t := &Task{}
		k, err := it.Next(t)
		if err == iterator.Done {
			break
		} else if err != nil {
			return fmt.Errorf("Failed to retrieve list of tasks: %s", err)
		}

		if t.SwarmingTaskId == "" {
			// This task is not ready to be looked at yet.
			continue
		}

		// Get the swarming task from swarming server.
		swarmingTask, err := GetSwarmingTask(t.SwarmingPool, t.SwarmingTaskId)
		if err != nil {
			return fmt.Errorf("Failed to retrieve swarming task %s: %s", t.SwarmingTaskId, err)
		}

		if swarmingTask.State == swarming.TASK_STATE_PENDING {
			// If the swarming task is still pending then there is nothing to do here.
			continue
		} else if swarmingTask.State == swarming.TASK_STATE_CANCELED {
			// If the swarming task has been cancelled then mark it as such in the DS.
			if err := taskCancelled(k, t); err != nil {
				return fmt.Errorf("Failed to mark task as cancelled: %s", err)
			}
			continue
		}

		// Check for 4 things-
		//
		// Check 1: If the previous state was pending and the new state is not then
		//          update the state, add botId and populate the lease start/end time.
		// Check 2: If lease has already expired then set Done=true.
		// Check 3: If lease is expiring in 15 mins then send a warning email if we
		//          have not done so already.
		// Check 4: If new state is unexpected then set Done=true and send failure email
		//          to the requester.

		if t.SwarmingTaskState == swarming.TASK_STATE_PENDING /* Check 1 */ {
			// The previous task state was pending but this has now changed.
			// Populate the datastore with running task values.
			if err := populateRunningTask(swarmingTask.State, swarmingTask.BotId, k, t); err != nil {
				return fmt.Errorf("Error populating running task: %s", err)
			}
		} else if t.LeaseEndTime.Before(time.Now()) /* Check 2*/ {
			// The task has expired.
			if err := expireTask(k, t); err != nil {
				return fmt.Errorf("Error when expiring task: %s", err)
			}
		} else if swarmingTask.State == swarming.TASK_STATE_RUNNING && !t.WarningSent && t.LeaseEndTime.Before(time.Now().Add(time.Minute*15)) /* Check 3 */ {
			if err := taskExpiringSoon(k, t); err != nil {
				return fmt.Errorf("Error when warning task expiring soon: %s", err)
			}
		} else /* Check 4 */ {

			if err := checkForUnexpectedStates(swarmingTask.State, swarmingTask.Failure, k, t); err != nil {
				return fmt.Errorf("Error when warning task expiring soon: %s", err)
			}
		}
	}

	return nil
}
<|endoftext|>


<commit_before>// +build js

package os

import (
	"github.com/gopherjs/gopherjs/js"
)

func init() {
	process := js.Global.Get("process")
	if !process.IsUndefined() {
		args := process.Get("argv")
		Args = make([]string, args.Length()-1)
		for i := 0; i < args.Length()-1; i++ {
			Args[i] = args.Index(i + 1).Str()
		}
	}
}
<commit_msg>os.Args needs to have at least a command name in the browser<commit_after>// +build js

package os

import (
	"github.com/gopherjs/gopherjs/js"
)

func init() {
	process := js.Global.Get("process")
	if process.IsUndefined() {
		Args = []string{"browser"}
		return
	}
	args := process.Get("argv")
	Args = make([]string, args.Length()-1)
	for i := 0; i < args.Length()-1; i++ {
		Args[i] = args.Index(i + 1).Str()
	}
}
<|endoftext|>


<commit_before>package main

// Command chatsvr starts the web-based chat server.
//
// Usage: chatsvr [host:port] [chat URL]
//
// If [host:port] is not specified, then chatsvr defaults to localhost:8083
// If [chat URL] is not specified, then chat server defaults to host:port/chat
import (
	"fmt"
	"html/template"
	"io"
	"net/http"
	"os"

	"golang.org/x/net/websocket"
)

// Host and port on which to listen for incoming HTTP requests
var listenAddr string = "localhost:8083"
var chatURL string = listenAddr + "/chat"

// ingoreOriginHandler is a simple interface to a WebSocket browser client.
// It ingnores the Origin request header by default.
// See websocket.OriginHandler
type ingoreOriginHandler func(*websocket.Conn)

// ServeHTTP implements the http.Handler interface for a WebSocket but does
// not check the origin of the request header. Useful when testing on localhost
func (h ingoreOriginHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
	s := websocket.Server{Handler: websocket.Handler(h)}
	s.ServeHTTP(w, req)
}

// Start a chat. (For now, only echoing)
func chatHandler(ws *websocket.Conn) {
	io.Copy(ws, ws)
}

func rootHandler(w http.ResponseWriter, req *http.Request) {
	t, err := template.ParseFiles("index.html")
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	err = t.Execute(w, chatURL)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
	}
}

func main() {
	if len(os.Args) >= 2 {
		listenAddr = os.Args[1]
		if len(os.Args) > 2 {
			chatURL = os.Args[2]
		}
	}

	fmt.Printf("chatsvr listening at %v\n", listenAddr)

	http.HandleFunc("/", rootHandler)
	http.Handle("/chat", ingoreOriginHandler(chatHandler))

	err := http.ListenAndServe(listenAddr, nil)
	if err != nil {
		panic("ListenAndServe: " + err.Error())
	}
}
<commit_msg>Fix spelling<commit_after>package main

// Command chatsvr starts the web-based chat server.
//
// Usage: chatsvr [host:port] [chat URL]
//
// If [host:port] is not specified, then chatsvr defaults to localhost:8083
// If [chat URL] is not specified, then chat server defaults to host:port/chat
import (
	"fmt"
	"html/template"
	"io"
	"net/http"
	"os"

	"golang.org/x/net/websocket"
)

// Host and port on which to listen for incoming HTTP requests
var listenAddr string = "localhost:8083"
var chatURL string = listenAddr + "/chat"

// ingoreOriginHandler is a simple interface to a WebSocket browser client.
// It ignores the Origin request header by default.
// See websocket.OriginHandler
type ingoreOriginHandler func(*websocket.Conn)

// ServeHTTP implements the http.Handler interface for a WebSocket but does
// not check the origin of the request header. Useful when testing on localhost
func (h ingoreOriginHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
	s := websocket.Server{Handler: websocket.Handler(h)}
	s.ServeHTTP(w, req)
}

// Start a chat. (For now, only echoing)
func chatHandler(ws *websocket.Conn) {
	io.Copy(ws, ws)
}

func rootHandler(w http.ResponseWriter, req *http.Request) {
	t, err := template.ParseFiles("index.html")
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	err = t.Execute(w, chatURL)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
	}
}

func main() {
	if len(os.Args) >= 2 {
		listenAddr = os.Args[1]
		if len(os.Args) > 2 {
			chatURL = os.Args[2]
		}
	}

	fmt.Printf("chatsvr listening at %v\n", listenAddr)

	http.HandleFunc("/", rootHandler)
	http.Handle("/chat", ingoreOriginHandler(chatHandler))

	err := http.ListenAndServe(listenAddr, nil)
	if err != nil {
		panic("ListenAndServe: " + err.Error())
	}
}
<|endoftext|>


<commit_before>// Copyright 2016 The Vulcan Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package scraper

import (
	"sync"

	log "github.com/Sirupsen/logrus"
)

// Scraper answers to a query on which targets are running and groups them
// by job.
type Scraper struct {
	Targeter TargetWatcher
	Writer   Writer
	running  map[string]*Worker
	done     chan struct{}
	once     sync.Once
}

// Config represents an instance of a scraper configuration.
// It contains a Targeter interface and a Writer interface.
type Config struct {
	Targeter TargetWatcher
	Writer   Writer
}

// NewScraper returns a new Scraper instance from the provided Config.
func NewScraper(config *Config) *Scraper {
	return &Scraper{
		Targeter: config.Targeter,
		Writer:   config.Writer,
		running:  map[string]*Worker{},
		done:     make(chan struct{}),
	}
}

// Run ranges over the Scraper's targets and does work on them.
func (s *Scraper) Run() {
	for {
		select {
		case target := <-s.Targeter.Targets():
			s.set(target)

			log.WithFields(log.Fields{
				"scraper":      "run",
				"target_count": len(target),
			}).Info("scraping targets")

		case <-s.done:
			return

		default:
		}
	}
}

func (s *Scraper) set(targets []Targeter) {
	next := make(map[string]*Worker, len(targets))

	for _, target := range targets {
		worker, ok := s.running[target.Key()]
		if !ok {
			next[target.Key()] = NewWorker(&WorkerConfig{
				Key:    target.Key(),
				Target: target,
				Writer: s.Writer,
			})
			continue
		}

		if !worker.Target.Equals(target) {
			worker.Retarget(target)
		}
		// Set found worker to next map of workers and remove reference from
		// existing running.
		next[target.Key()] = worker
		delete(s.running, target.Key())
	}
	// Stop runnning jobs that are no longer active targets targets.
	for _, w := range s.running {
		w.Stop()
	}

	s.running = next
}

// Stop signals the Scraper instance to stop running.
func (s *Scraper) Stop() {
	s.once.Do(func() {
		close(s.done)
	})
}
<commit_msg>scraper: removed unnecessary default case in run (#41)<commit_after>// Copyright 2016 The Vulcan Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package scraper

import (
	"sync"

	log "github.com/Sirupsen/logrus"
)

// Scraper answers to a query on which targets are running and groups them
// by job.
type Scraper struct {
	Targeter TargetWatcher
	Writer   Writer
	running  map[string]*Worker
	done     chan struct{}
	once     sync.Once
}

// Config represents an instance of a scraper configuration.
// It contains a Targeter interface and a Writer interface.
type Config struct {
	Targeter TargetWatcher
	Writer   Writer
}

// NewScraper returns a new Scraper instance from the provided Config.
func NewScraper(config *Config) *Scraper {
	return &Scraper{
		Targeter: config.Targeter,
		Writer:   config.Writer,
		running:  map[string]*Worker{},
		done:     make(chan struct{}),
	}
}

// Run ranges over the Scraper's targets and does work on them.
func (s *Scraper) Run() {
	for {
		select {
		case target := <-s.Targeter.Targets():
			s.set(target)

			log.WithFields(log.Fields{
				"scraper":      "run",
				"target_count": len(target),
			}).Info("scraping targets")

		case <-s.done:
			return
		}
	}
}

func (s *Scraper) set(targets []Targeter) {
	next := make(map[string]*Worker, len(targets))

	for _, target := range targets {
		worker, ok := s.running[target.Key()]
		if !ok {
			next[target.Key()] = NewWorker(&WorkerConfig{
				Key:    target.Key(),
				Target: target,
				Writer: s.Writer,
			})
			continue
		}

		if !worker.Target.Equals(target) {
			worker.Retarget(target)
		}
		// Set found worker to next map of workers and remove reference from
		// existing running.
		next[target.Key()] = worker
		delete(s.running, target.Key())
	}
	// Stop runnning jobs that are no longer active targets targets.
	for _, w := range s.running {
		w.Stop()
	}

	s.running = next
}

// Stop signals the Scraper instance to stop running.
func (s *Scraper) Stop() {
	s.once.Do(func() {
		close(s.done)
	})
}
<|endoftext|>


<commit_before>package scraper

import (
	"hnews/services"
	"log"
	"net/http"
	"strconv"
	"strings"
	"time"

	"hnews/Godeps/_workspace/src/github.com/yhat/scrape"
	"hnews/Godeps/_workspace/src/golang.org/x/net/html"
	"hnews/Godeps/_workspace/src/golang.org/x/net/html/atom"
)

// StartScraper starts the scraping and never returns, run as a goroutine.
func StartScraper() {
	newsCh := make(chan []services.News)
	commentsCh := make(chan []services.Comment)
	go scrapeFrontPage(newsCh)
	go scrapeComments(commentsCh)

	for {
		select {
		case newNews := <-newsCh:
			// log.Println(len(newNews), "new news.") // DEBUG
			go services.SaveNews(newNews)
		case newComments := <-commentsCh:
			// log.Println(len(newComments), "new comments.") // DEBUG
			go services.SaveComments(newComments)
		}
	}
}

/********************** News **********************/
// Parses the front page and sends a []News of the content
func scrapeFrontPage(newsCh chan []services.News) {
	for {
		for i := 1; i <= 16; i++ {
			var news []services.News

			url := "https://news.ycombinator.com/news?p=" + strconv.Itoa(i)
			resp, err := http.Get(url)
			if err != nil {
				log.Println(err)
				continue
			}

			root, err := html.Parse(resp.Body)
			if err != nil {
				log.Println(err)
				continue
			}

			pointsCh := make(chan []int)
			ranksCh := make(chan []int)
			titlesCh := make(chan []string)
			linksCh := make(chan []string)
			authorsCh := make(chan []string)
			timesCh := make(chan []time.Time)
			commentsCh := make(chan []int)
			idsCh := make(chan []int)

			go parsePoints(root, pointsCh)
			go parseRanks(root, ranksCh)
			go parseArticles(root, titlesCh, linksCh)
			go parseAuthors(root, authorsCh)
			go parseTimes(root, timesCh)
			go parseNumComments(root, commentsCh)
			go parseIDs(root, idsCh)

			points := <-pointsCh
			ranks := <-ranksCh
			titles := <-titlesCh
			links := <-linksCh
			authors := <-authorsCh
			times := <-timesCh
			comments := <-commentsCh
			ids := <-idsCh

			for i := 0; i < len(ranks); i++ {
				rank := int32(ranks[i])
				title := titles[i]

				var time time.Time
				if i < len(times) {
					time = times[i]
				}

				var link string
				if i < len(links) {
					link = links[i]
				}

				var author string
				if i < len(authors) {
					author = authors[i]
				}

				var numPoints int32
				if i < len(points) {
					numPoints = int32(points[i])
				}

				var numComments int32
				if i < len(comments) {
					numComments = int32(comments[i])
				}

				var id int32
				if i < len(ids) {
					id = int32(ids[i])
				}

				news = append(news, services.News{id, rank, title, link, author, numPoints, time, numComments})
			}
			newsCh <- news
		}
	}
}

// Parses out the rank of the articles.
func parseRanks(root *html.Node, ranksCh chan []int) {
	var ranks []int

	rankMatcher := func(n *html.Node) bool {
		if n.DataAtom == atom.Span && n.Parent != nil && n.Parent.Parent != nil {
			grandparent := scrape.Attr(n.Parent.Parent, "class") == "athing"
			self := scrape.Attr(n, "class") == "rank"
			return grandparent && self
		}
		return false
	}

	rankNodes := scrape.FindAll(root, rankMatcher)
	for _, rankNode := range rankNodes {
		text := strings.Replace(scrape.Text(rankNode), ".", "", -1)
		rank, err := strconv.Atoi(text)
		if err != nil {
			rank = 0
		}
		ranks = append(ranks, rank)
	}
	ranksCh <- ranks
}

// Parses the authors of each Story on the frontpage
func parseAuthors(root *html.Node, authorsCh chan []string) {
	matcher := func(n *html.Node) bool {
		if n.DataAtom == atom.A && n.Parent != nil {
			parent := scrape.Attr(n.Parent, "class") == "subtext"
			href := scrape.Attr(n, "href")
			if len(href) > 4 && href[0:4] == "user" {
				return parent && true
			}
		}
		return false
	}

	var authors []string
	authorNodes := scrape.FindAll(root, matcher)
	for _, authorNode := range authorNodes {
		authors = append(authors, scrape.Text(authorNode))
	}
	authorsCh <- authors
}

// Parses the number of comments on each Story on the frontpage
func parseNumComments(root *html.Node, commentsCh chan []int) {
	matcher := func(n *html.Node) bool {
		if n.DataAtom == atom.A && n.Parent != nil {
			parent := scrape.Attr(n.Parent, "class") == "subtext"
			href := scrape.Attr(n, "href")
			if len(href) > 4 && href[0:4] == "item" {
				return parent && true
			}
		}
		return false
	}

	var numComms []int // Number of comments for each Story
	commentNodes := scrape.FindAll(root, matcher)
	for _, commentNode := range commentNodes {
		text := scrape.Text(commentNode)
		words := strings.Fields(text)
		numComm, err := strconv.Atoi(words[0])
		if err != nil {
			numComm = 0
		}
		numComms = append(numComms, numComm)
	}
	commentsCh <- numComms
}

// Parses the timestamps from the HTML of the frontpage
func parseTimes(root *html.Node, timesCh chan []time.Time) {
	matcher := func(n *html.Node) bool {
		if n.DataAtom == atom.A && n.Parent != nil {
			return scrape.Attr(n.Parent, "class") == "age"
		}
		return false
	}

	var dates []time.Time
	timeNodes := scrape.FindAll(root, matcher)
	for _, timeNode := range timeNodes {
		text := scrape.Text(timeNode)
		date, err := parseTimeString(text)
		if err != nil {
			date = time.Now()
		}
		dates = append(dates, date)
	}
	timesCh <- dates
}

// Parses the id of each Story on the page
func parseIDs(root *html.Node, idsCh chan []int) {
	matcher := func(n *html.Node) bool {
		if n.DataAtom == atom.Span && n.Parent != nil {
			parent := scrape.Attr(n.Parent, "class") == "subtext"
			self := scrape.Attr(n, "class") == "score"
			return parent && self
		}
		return false
	}

	var ids []int
	idNodes := scrape.FindAll(root, matcher)
	for _, idNode := range idNodes {
		var id int
		idTemp := scrape.Attr(idNode, "id")
		if len(idTemp) <= 6 {
			id = 0
			ids = append(ids, id)
			continue
		}
		id, err := strconv.Atoi(idTemp[6:len(idTemp)])
		if err != nil {
			id = 0
		}
		ids = append(ids, id)
	}
	idsCh <- ids
}

// Quantity is of "hours"/"days"/"minutes"
// Text is of "4 hours ago", "41 days ago", etc
func parseTimeString(text string) (time.Time, error) {
	words := strings.Fields(text)

	timeAgo, err := strconv.Atoi(words[0])
	if err != nil {
		return time.Time{}, err
	}

	now := time.Now()
	var result time.Time
	switch words[1] {
	case "minutes":
		result = now.Add(time.Duration(-timeAgo) * time.Minute)
	case "hours", "hour":
		result = now.Add(time.Duration(-timeAgo) * time.Hour)
	case "days", "day":
		result = now.Add(time.Duration(-timeAgo*24) * time.Hour)
	}
	return result, err
}

// Parses the title and the link of each Story on the page
func parseArticles(root *html.Node, titleCh chan []string, linksCh chan []string) {
	matcher := func(n *html.Node) bool {
		if n.DataAtom == atom.A && n.Parent != nil && n.Parent.Parent != nil {
			return scrape.Attr(n.Parent.Parent, "class") == "athing"
		}
		return false
	}

	var titles []string
	var links []string

	articles := scrape.FindAll(root, matcher)
	for _, article := range articles {
		title := scrape.Text(article)
		link := scrape.Attr(article, "href")
		titles = append(titles, title)
		links = append(links, link)
	}
	titleCh <- titles
	linksCh <- links
}

func parsePoints(root *html.Node, pointsCh chan []int) {
	matcher := func(n *html.Node) bool {
		if n.DataAtom == atom.Span && n.Parent != nil {
			parent := scrape.Attr(n.Parent, "class") == "subtext"
			self := scrape.Attr(n, "class") == "score"
			return parent && self
		}
		return false
	}

	var points []int

	pointsNodes := scrape.FindAll(root, matcher)
	for _, pointsNode := range pointsNodes {
		pointS := strings.Replace(scrape.Text(pointsNode), " points", "", -1)
		point, err := strconv.Atoi(pointS)
		if err != nil {
			point = 0
		}
		points = append(points, point)
	}
	pointsCh <- points
}

/********************** News **********************/

/******************** Comments ********************/
// Scrapes the Comments for every News item currently in the database.
func scrapeComments(commentsCh chan []services.Comment) {
	for {
		ids := services.ReadNewsIds()
		for _, id := range ids {
			url := "https://news.ycombinator.com/item?id=" + strconv.Itoa(int(id))

			resp, err := http.Get(url)
			if err != nil {
				log.Println(err)
				continue
			}

			root, err := html.Parse(resp.Body)
			if err != nil {
				log.Println(err)
				continue
			}
			go parseComments(root, id, commentsCh)
		}
	}
}

// Parses all the Comments for a particular News item.
func parseComments(root *html.Node, newsid int32, commentsCh chan []services.Comment) {
	offsetsCh := make(chan []int)
	idsCh := make(chan []int)
	authorsCh := make(chan []string)
	timesCh := make(chan []time.Time)
	textsCh := make(chan []string)
	go parseOffsets(root, offsetsCh)
	go parseCommentIDs(root, idsCh)
	go parseCommentAuthors(root, authorsCh)
	go parseCommentTimes(root, timesCh)
	go parseCommentText(root, textsCh)
	offsets := <-offsetsCh
	ids := <-idsCh
	authors := <-authorsCh
	times := <-timesCh
	texts := <-textsCh

	var comments []services.Comment
	for i := range authors {
		comment := services.Comment{int32(i + 1), newsid, int32(ids[i]), int32(offsets[i]),
			times[i], authors[i], texts[i]}
		comments = append(comments, comment)
	}
	commentsCh <- comments
}

// Parses the level for each Comment in the comment tree. Interval: 0-inf.
func parseOffsets(root *html.Node, ch chan []int) {
	offsetMatcher := func(n *html.Node) bool {
		if n.DataAtom == atom.Img && n.Parent != nil {
			parent := scrape.Attr(n.Parent, "class") == "ind"
			return parent
		}
		return false
	}

	offsets := scrape.FindAll(root, offsetMatcher)

	var norm []int
	for _, offset := range offsets {
		lvl, _ := strconv.Atoi(scrape.Attr(offset, "width"))
		norm = append(norm, int(lvl/40))
	}
	ch <- norm
}

// Parses all the comments authors
func parseCommentAuthors(root *html.Node, ch chan []string) {
	matcher := func(n *html.Node) bool {
		if n.DataAtom == atom.A && n.Parent != nil {
			return scrape.Attr(n.Parent, "class") == "comhead"
		}
		return false
	}

	var authors []string
	authorNodes := scrape.FindAll(root, matcher)
	for _, authorNode := range authorNodes {
		authors = append(authors, scrape.Text(authorNode))
	}
	ch <- authors
}

// Parses all the comments itemids
func parseCommentIDs(root *html.Node, ch chan []int) {
	matcher := func(n *html.Node) bool {
		if n.DataAtom == atom.A && n.Parent != nil {
			return scrape.Attr(n.Parent, "class") == "age"
		}
		return false
	}

	var ids []int
	idNodes := scrape.FindAll(root, matcher)
	for _, idNode := range idNodes {
		var id int
		href := scrape.Attr(idNode, "href")
		if len(href) <= 8 {
			id = 0
			ids = append(ids, id)
			continue
		}
		id, err := strconv.Atoi(href[8:len(href)])
		if err != nil {
			id = 0
		}
		ids = append(ids, id)
	}
	ch <- ids
}

// Parses all the comments timestamps
func parseCommentTimes(root *html.Node, ch chan []time.Time) {
	matcher := func(n *html.Node) bool {
		if n.DataAtom == atom.A && n.Parent != nil {
			return scrape.Attr(n.Parent, "class") == "age"
		}
		return false
	}

	var dates []time.Time
	timeNodes := scrape.FindAll(root, matcher)
	for _, timeNode := range timeNodes {
		date, err := parseTimeString(scrape.Text(timeNode))
		if err != nil {
			date = time.Now()
		}
		dates = append(dates, date)
	}
	ch <- dates
}

// Parses the text of all Comments for a News
func parseCommentText(root *html.Node, ch chan []string) {
	textMatcher := func(n *html.Node) bool {
		if n.DataAtom == atom.Span && n.Parent != nil {
			parent := scrape.Attr(n.Parent, "class") == "comment"
			return parent
		}
		return false
	}

	textNodes := scrape.FindAll(root, textMatcher)

	var texts []string
	for _, text := range textNodes {
		content := scrape.Text(text)
		// TODO: Removes trailing trash from the 'Reply' HTML node ...
		texts = append(texts, content[0:len(content)-5])
	}
	ch <- texts
}

/******************** Comments ********************/
<commit_msg>Removed TODO<commit_after>package scraper

import (
	"hnews/services"
	"log"
	"net/http"
	"strconv"
	"strings"
	"time"

	"hnews/Godeps/_workspace/src/github.com/yhat/scrape"
	"hnews/Godeps/_workspace/src/golang.org/x/net/html"
	"hnews/Godeps/_workspace/src/golang.org/x/net/html/atom"
)

// StartScraper starts the scraping and never returns, run as a goroutine.
func StartScraper() {
	newsCh := make(chan []services.News)
	commentsCh := make(chan []services.Comment)
	go scrapeFrontPage(newsCh)
	go scrapeComments(commentsCh)

	for {
		select {
		case newNews := <-newsCh:
			// log.Println(len(newNews), "new news.") // DEBUG
			services.SaveNews(newNews)
		case newComments := <-commentsCh:
			// log.Println(len(newComments), "new comments.") // DEBUG
			services.SaveComments(newComments)
		}
	}
}

/********************** News **********************/
// Parses the front page and sends a []News of the content
func scrapeFrontPage(newsCh chan []services.News) {
	for {
		for i := 1; i <= 16; i++ {
			var news []services.News

			url := "https://news.ycombinator.com/news?p=" + strconv.Itoa(i)
			resp, err := http.Get(url)
			if err != nil {
				log.Println(err)
				continue
			}

			root, err := html.Parse(resp.Body)
			if err != nil {
				log.Println(err)
				continue
			}

			pointsCh := make(chan []int)
			ranksCh := make(chan []int)
			titlesCh := make(chan []string)
			linksCh := make(chan []string)
			authorsCh := make(chan []string)
			timesCh := make(chan []time.Time)
			commentsCh := make(chan []int)
			idsCh := make(chan []int)

			go parsePoints(root, pointsCh)
			go parseRanks(root, ranksCh)
			go parseArticles(root, titlesCh, linksCh)
			go parseAuthors(root, authorsCh)
			go parseTimes(root, timesCh)
			go parseNumComments(root, commentsCh)
			go parseIDs(root, idsCh)

			points := <-pointsCh
			ranks := <-ranksCh
			titles := <-titlesCh
			links := <-linksCh
			authors := <-authorsCh
			times := <-timesCh
			comments := <-commentsCh
			ids := <-idsCh

			for i := 0; i < len(ranks); i++ {
				rank := int32(ranks[i])
				title := titles[i]

				var time time.Time
				if i < len(times) {
					time = times[i]
				}

				var link string
				if i < len(links) {
					link = links[i]
				}

				var author string
				if i < len(authors) {
					author = authors[i]
				}

				var numPoints int32
				if i < len(points) {
					numPoints = int32(points[i])
				}

				var numComments int32
				if i < len(comments) {
					numComments = int32(comments[i])
				}

				var id int32
				if i < len(ids) {
					id = int32(ids[i])
				}

				news = append(news, services.News{id, rank, title, link, author, numPoints, time, numComments})
			}
			newsCh <- news
		}
	}
}

// Parses out the rank of the articles.
func parseRanks(root *html.Node, ranksCh chan []int) {
	var ranks []int

	rankMatcher := func(n *html.Node) bool {
		if n.DataAtom == atom.Span && n.Parent != nil && n.Parent.Parent != nil {
			grandparent := scrape.Attr(n.Parent.Parent, "class") == "athing"
			self := scrape.Attr(n, "class") == "rank"
			return grandparent && self
		}
		return false
	}

	rankNodes := scrape.FindAll(root, rankMatcher)
	for _, rankNode := range rankNodes {
		text := strings.Replace(scrape.Text(rankNode), ".", "", -1)
		rank, err := strconv.Atoi(text)
		if err != nil {
			rank = 0
		}
		ranks = append(ranks, rank)
	}
	ranksCh <- ranks
}

// Parses the authors of each Story on the frontpage
func parseAuthors(root *html.Node, authorsCh chan []string) {
	matcher := func(n *html.Node) bool {
		if n.DataAtom == atom.A && n.Parent != nil {
			parent := scrape.Attr(n.Parent, "class") == "subtext"
			href := scrape.Attr(n, "href")
			if len(href) > 4 && href[0:4] == "user" {
				return parent && true
			}
		}
		return false
	}

	var authors []string
	authorNodes := scrape.FindAll(root, matcher)
	for _, authorNode := range authorNodes {
		authors = append(authors, scrape.Text(authorNode))
	}
	authorsCh <- authors
}

// Parses the number of comments on each Story on the frontpage
func parseNumComments(root *html.Node, commentsCh chan []int) {
	matcher := func(n *html.Node) bool {
		if n.DataAtom == atom.A && n.Parent != nil {
			parent := scrape.Attr(n.Parent, "class") == "subtext"
			href := scrape.Attr(n, "href")
			if len(href) > 4 && href[0:4] == "item" {
				return parent && true
			}
		}
		return false
	}

	var numComms []int // Number of comments for each Story
	commentNodes := scrape.FindAll(root, matcher)
	for _, commentNode := range commentNodes {
		text := scrape.Text(commentNode)
		words := strings.Fields(text)
		numComm, err := strconv.Atoi(words[0])
		if err != nil {
			numComm = 0
		}
		numComms = append(numComms, numComm)
	}
	commentsCh <- numComms
}

// Parses the timestamps from the HTML of the frontpage
func parseTimes(root *html.Node, timesCh chan []time.Time) {
	matcher := func(n *html.Node) bool {
		if n.DataAtom == atom.A && n.Parent != nil {
			return scrape.Attr(n.Parent, "class") == "age"
		}
		return false
	}

	var dates []time.Time
	timeNodes := scrape.FindAll(root, matcher)
	for _, timeNode := range timeNodes {
		text := scrape.Text(timeNode)
		date, err := parseTimeString(text)
		if err != nil {
			date = time.Now()
		}
		dates = append(dates, date)
	}
	timesCh <- dates
}

// Parses the id of each Story on the page
func parseIDs(root *html.Node, idsCh chan []int) {
	matcher := func(n *html.Node) bool {
		if n.DataAtom == atom.Span && n.Parent != nil {
			parent := scrape.Attr(n.Parent, "class") == "subtext"
			self := scrape.Attr(n, "class") == "score"
			return parent && self
		}
		return false
	}

	var ids []int
	idNodes := scrape.FindAll(root, matcher)
	for _, idNode := range idNodes {
		var id int
		idTemp := scrape.Attr(idNode, "id")
		if len(idTemp) <= 6 {
			id = 0
			ids = append(ids, id)
			continue
		}
		id, err := strconv.Atoi(idTemp[6:len(idTemp)])
		if err != nil {
			id = 0
		}
		ids = append(ids, id)
	}
	idsCh <- ids
}

// Quantity is of "hours"/"days"/"minutes"
// Text is of "4 hours ago", "41 days ago", etc
func parseTimeString(text string) (time.Time, error) {
	words := strings.Fields(text)

	timeAgo, err := strconv.Atoi(words[0])
	if err != nil {
		return time.Time{}, err
	}

	now := time.Now()
	var result time.Time
	switch words[1] {
	case "minutes":
		result = now.Add(time.Duration(-timeAgo) * time.Minute)
	case "hours", "hour":
		result = now.Add(time.Duration(-timeAgo) * time.Hour)
	case "days", "day":
		result = now.Add(time.Duration(-timeAgo*24) * time.Hour)
	}
	return result, err
}

// Parses the title and the link of each Story on the page
func parseArticles(root *html.Node, titleCh chan []string, linksCh chan []string) {
	matcher := func(n *html.Node) bool {
		if n.DataAtom == atom.A && n.Parent != nil && n.Parent.Parent != nil {
			return scrape.Attr(n.Parent.Parent, "class") == "athing"
		}
		return false
	}

	var titles []string
	var links []string

	articles := scrape.FindAll(root, matcher)
	for _, article := range articles {
		title := scrape.Text(article)
		link := scrape.Attr(article, "href")
		titles = append(titles, title)
		links = append(links, link)
	}
	titleCh <- titles
	linksCh <- links
}

func parsePoints(root *html.Node, pointsCh chan []int) {
	matcher := func(n *html.Node) bool {
		if n.DataAtom == atom.Span && n.Parent != nil {
			parent := scrape.Attr(n.Parent, "class") == "subtext"
			self := scrape.Attr(n, "class") == "score"
			return parent && self
		}
		return false
	}

	var points []int

	pointsNodes := scrape.FindAll(root, matcher)
	for _, pointsNode := range pointsNodes {
		pointS := strings.Replace(scrape.Text(pointsNode), " points", "", -1)
		point, err := strconv.Atoi(pointS)
		if err != nil {
			point = 0
		}
		points = append(points, point)
	}
	pointsCh <- points
}

/********************** News **********************/

/******************** Comments ********************/
// Scrapes the Comments for every News item currently in the database.
func scrapeComments(commentsCh chan []services.Comment) {
	for {
		ids := services.ReadNewsIds()
		for _, id := range ids {
			url := "https://news.ycombinator.com/item?id=" + strconv.Itoa(int(id))

			resp, err := http.Get(url)
			if err != nil {
				log.Println(err)
				continue
			}

			root, err := html.Parse(resp.Body)
			if err != nil {
				log.Println(err)
				continue
			}
			go parseComments(root, id, commentsCh)
		}
	}
}

// Parses all the Comments for a particular News item.
func parseComments(root *html.Node, newsid int32, commentsCh chan []services.Comment) {
	offsetsCh := make(chan []int)
	idsCh := make(chan []int)
	authorsCh := make(chan []string)
	timesCh := make(chan []time.Time)
	textsCh := make(chan []string)
	go parseOffsets(root, offsetsCh)
	go parseCommentIDs(root, idsCh)
	go parseCommentAuthors(root, authorsCh)
	go parseCommentTimes(root, timesCh)
	go parseCommentText(root, textsCh)
	offsets := <-offsetsCh
	ids := <-idsCh
	authors := <-authorsCh
	times := <-timesCh
	texts := <-textsCh

	var comments []services.Comment
	for i := range authors {
		comment := services.Comment{int32(i + 1), newsid, int32(ids[i]), int32(offsets[i]),
			times[i], authors[i], texts[i]}
		comments = append(comments, comment)
	}
	commentsCh <- comments
}

// Parses the level for each Comment in the comment tree. Interval: 0-inf.
func parseOffsets(root *html.Node, ch chan []int) {
	offsetMatcher := func(n *html.Node) bool {
		if n.DataAtom == atom.Img && n.Parent != nil {
			parent := scrape.Attr(n.Parent, "class") == "ind"
			return parent
		}
		return false
	}

	offsets := scrape.FindAll(root, offsetMatcher)

	var norm []int
	for _, offset := range offsets {
		lvl, _ := strconv.Atoi(scrape.Attr(offset, "width"))
		norm = append(norm, int(lvl/40))
	}
	ch <- norm
}

// Parses all the comments authors
func parseCommentAuthors(root *html.Node, ch chan []string) {
	matcher := func(n *html.Node) bool {
		if n.DataAtom == atom.A && n.Parent != nil {
			return scrape.Attr(n.Parent, "class") == "comhead"
		}
		return false
	}

	var authors []string
	authorNodes := scrape.FindAll(root, matcher)
	for _, authorNode := range authorNodes {
		authors = append(authors, scrape.Text(authorNode))
	}
	ch <- authors
}

// Parses all the comments itemids
func parseCommentIDs(root *html.Node, ch chan []int) {
	matcher := func(n *html.Node) bool {
		if n.DataAtom == atom.A && n.Parent != nil {
			return scrape.Attr(n.Parent, "class") == "age"
		}
		return false
	}

	var ids []int
	idNodes := scrape.FindAll(root, matcher)
	for _, idNode := range idNodes {
		var id int
		href := scrape.Attr(idNode, "href")
		if len(href) <= 8 {
			id = 0
			ids = append(ids, id)
			continue
		}
		id, err := strconv.Atoi(href[8:len(href)])
		if err != nil {
			id = 0
		}
		ids = append(ids, id)
	}
	ch <- ids
}

// Parses all the comments timestamps
func parseCommentTimes(root *html.Node, ch chan []time.Time) {
	matcher := func(n *html.Node) bool {
		if n.DataAtom == atom.A && n.Parent != nil {
			return scrape.Attr(n.Parent, "class") == "age"
		}
		return false
	}

	var dates []time.Time
	timeNodes := scrape.FindAll(root, matcher)
	for _, timeNode := range timeNodes {
		date, err := parseTimeString(scrape.Text(timeNode))
		if err != nil {
			date = time.Now()
		}
		dates = append(dates, date)
	}
	ch <- dates
}

// Parses the text of all Comments for a News
func parseCommentText(root *html.Node, ch chan []string) {
	textMatcher := func(n *html.Node) bool {
		if n.DataAtom == atom.Span && n.Parent != nil {
			parent := scrape.Attr(n.Parent, "class") == "comment"
			return parent
		}
		return false
	}

	textNodes := scrape.FindAll(root, textMatcher)

	var texts []string
	for _, text := range textNodes {
		content := scrape.Text(text)
		// Removes trailing trash from the 'Reply' HTML node ...
		texts = append(texts, content[0:len(content)-5])
	}
	ch <- texts
}

/******************** Comments ********************/
<|endoftext|>


<commit_before>// build +linux

package specconv

import (
	"strings"
	"testing"

	"github.com/opencontainers/runtime-spec/specs-go"
)

func TestLinuxCgroupsPathSpecified(t *testing.T) {
	cgroupsPath := "/user/cgroups/path/id"

	spec := &specs.Spec{}
	spec.Linux.CgroupsPath = &cgroupsPath

	cgroup, err := createCgroupConfig("ContainerID", false, spec)
	if err != nil {
		t.Errorf("Couldn't create Cgroup config: %v", err)
	}

	if cgroup.Path != cgroupsPath {
		t.Errorf("Wrong cgroupsPath, expected '%s' got '%s'", cgroupsPath, cgroup.Path)
	}
}

func TestLinuxCgroupsPathNotSpecified(t *testing.T) {
	spec := &specs.Spec{}

	cgroup, err := createCgroupConfig("ContainerID", false, spec)
	if err != nil {
		t.Errorf("Couldn't create Cgroup config: %v", err)
	}

	if !strings.HasSuffix(cgroup.Path, "/ContainerID") {
		t.Errorf("Wrong cgroupsPath, expected it to have suffix '%s' got '%s'", "/ContainerID", cgroup.Path)
	}
}
<commit_msg>Fixed typo in build constraint.<commit_after>// +build linux

package specconv

import (
	"strings"
	"testing"

	"github.com/opencontainers/runtime-spec/specs-go"
)

func TestLinuxCgroupsPathSpecified(t *testing.T) {
	cgroupsPath := "/user/cgroups/path/id"

	spec := &specs.Spec{}
	spec.Linux.CgroupsPath = &cgroupsPath

	cgroup, err := createCgroupConfig("ContainerID", false, spec)
	if err != nil {
		t.Errorf("Couldn't create Cgroup config: %v", err)
	}

	if cgroup.Path != cgroupsPath {
		t.Errorf("Wrong cgroupsPath, expected '%s' got '%s'", cgroupsPath, cgroup.Path)
	}
}

func TestLinuxCgroupsPathNotSpecified(t *testing.T) {
	spec := &specs.Spec{}

	cgroup, err := createCgroupConfig("ContainerID", false, spec)
	if err != nil {
		t.Errorf("Couldn't create Cgroup config: %v", err)
	}

	if !strings.HasSuffix(cgroup.Path, "/ContainerID") {
		t.Errorf("Wrong cgroupsPath, expected it to have suffix '%s' got '%s'", "/ContainerID", cgroup.Path)
	}
}
<|endoftext|>


<commit_before>package server

import (
	"bufio"
	"bytes"
	"fmt"
	"hash/fnv"
	"io"
	"math/rand"
	"os"
	"path"
	"reflect"
	"strings"
	"sync"
	"testing"

	"github.com/pachyderm/pachyderm/src/client"
	"github.com/pachyderm/pachyderm/src/client/pfs"
	"github.com/pachyderm/pachyderm/src/client/pkg/require"
	"github.com/pachyderm/pachyderm/src/client/pps"
	versionlib "github.com/pachyderm/pachyderm/src/client/version"
	"github.com/pachyderm/pachyderm/src/server/pkg/backoff"
	tu "github.com/pachyderm/pachyderm/src/server/pkg/testutil"
	"github.com/pachyderm/pachyderm/src/server/pkg/workload"

	"github.com/golang/snappy"
)

const (
	KB = 1024
	MB = 1024 * KB
)

var pachClient *client.APIClient
var getPachClientOnce sync.Once

func getPachClient(t testing.TB) *client.APIClient {
	getPachClientOnce.Do(func() {
		var err error
		if addr := os.Getenv("PACHD_PORT_650_TCP_ADDR"); addr != "" {
			pachClient, err = client.NewInCluster()
		} else {
			pachClient, err = client.NewOnUserMachine(false, false, "user")
		}
		require.NoError(t, err)
	})
	return pachClient
}

func collectCommitInfos(t testing.TB, commitInfoIter client.CommitInfoIterator) []*pfs.CommitInfo {
	var commitInfos []*pfs.CommitInfo
	for {
		commitInfo, err := commitInfoIter.Next()
		if err == io.EOF {
			return commitInfos
		}
		require.NoError(t, err)
		commitInfos = append(commitInfos, commitInfo)
	}
}

// TODO(msteffen) equivalent to funciton in src/server/auth/server/admin_test.go.
// These should be unified.
func RepoInfoToName(repoInfo interface{}) interface{} {
	return repoInfo.(*pfs.RepoInfo).Repo.Name
}

// testExtractRestored effectively implements both TestExtractRestoreObjects
// TestExtractRestoreNoObjects, as their logic is mostly the same
func testExtractRestore(t *testing.T, testObjects bool) {
	if testing.Short() {
		t.Skip("Skipping integration tests in short mode")
	}

	c := getPachClient(t)
	require.NoError(t, c.DeleteAll())

	dataRepo := tu.UniqueString("TestExtractRestoreObjects-in-")
	require.NoError(t, c.CreateRepo(dataRepo))

	// Create input data
	nCommits := 2
	r := rand.New(rand.NewSource(45))
	fileHashes := make([]string, 0, nCommits)
	for i := 0; i < nCommits; i++ {
		hash := fnv.New64a()
		fileContent := workload.RandString(r, 40*MB)
		_, err := c.PutFile(dataRepo, "master", fmt.Sprintf("file-%d", i),
			io.TeeReader(strings.NewReader(fileContent), hash))
		require.NoError(t, err)
		fileHashes = append(fileHashes, string(hash.Sum(nil)))
	}

	// Create test pipelines
	numPipelines := 3
	var input, pipeline string
	input = dataRepo
	for i := 0; i < numPipelines; i++ {
		pipeline = tu.UniqueString(fmt.Sprintf("TestExtractRestoreObjects-P%d-", i))
		require.NoError(t, c.CreatePipeline(
			pipeline,
			"",
			[]string{"bash"},
			[]string{
				fmt.Sprintf("cp /pfs/%s/* /pfs/out/", input),
			},
			&pps.ParallelismSpec{
				Constant: 1,
			},
			client.NewPFSInput(input, "/*"),
			"",
			false,
		))
		input = pipeline
	}

	// Wait for pipelines to process input data
	commitIter, err := c.FlushCommit([]*pfs.Commit{client.NewCommit(dataRepo, "master")}, nil)
	require.NoError(t, err)
	commitInfos := collectCommitInfos(t, commitIter)
	require.Equal(t, numPipelines, len(commitInfos))

	// Extract existing cluster state
	ops, err := c.ExtractAll(testObjects)
	require.NoError(t, err)

	// Delete existing metadata
	require.NoError(t, c.DeleteAll())

	if testObjects {
		// Delete existing objects
		require.NoError(t, c.GarbageCollect(10000))
	}

	// Restore metadata and possibly objects
	require.NoError(t, c.Restore(ops))

	// Wait for re-created pipelines to process recreated input data
	commitIter, err = c.FlushCommit([]*pfs.Commit{client.NewCommit(dataRepo, "master")}, nil)
	require.NoError(t, err)
	commitInfos = collectCommitInfos(t, commitIter)
	require.Equal(t, numPipelines, len(commitInfos))

	// Make sure recreated jobs all succeeded
	jis, err := c.ListJob("", nil, nil) // make sure jobs all succeeded
	require.NoError(t, err)
	for _, ji := range jis {
		// prevent race--we may call listJob between when a job's output commit is
		// closed and when its state is updated
		backoff.Retry(func() error {
			if ji.State.String() == "JOB_MERGING" {
				return fmt.Errorf("output commit is closed but job state hasn't been updated")
			}
			return nil
		}, backoff.NewTestingBackOff())
		// Job must ultimately succeed
		require.Equal(t, "JOB_SUCCESS", ji.State.String())
	}

	// Make sure all branches were recreated
	bis, err := c.ListBranch(dataRepo)
	require.NoError(t, err)
	require.Equal(t, 1, len(bis))

	// Check input data
	// This check uses a backoff because sometimes GetFile causes pachd to OOM
	var restoredFileHashes []string
	require.NoError(t, backoff.Retry(func() error {
		restoredFileHashes = make([]string, 0, nCommits)
		for i := 0; i < nCommits; i++ {
			hash := fnv.New64a()
			err := c.GetFile(dataRepo, "master", fmt.Sprintf("file-%d", i), 0, 0, hash)
			if err != nil {
				return err
			}
			restoredFileHashes = append(restoredFileHashes, string(hash.Sum(nil)))
		}
		return nil
	}, backoff.NewTestingBackOff()))
	require.ElementsEqual(t, fileHashes, restoredFileHashes)

	// Check output data
	// This check uses a backoff because sometimes GetFile causes pachd to OOM
	require.NoError(t, backoff.Retry(func() error {
		restoredFileHashes = make([]string, 0, nCommits)
		for i := 0; i < nCommits; i++ {
			hash := fnv.New64a()
			err := c.GetFile(pipeline, "master", fmt.Sprintf("file-%d", i), 0, 0, hash)
			if err != nil {
				return err
			}
			restoredFileHashes = append(restoredFileHashes, string(hash.Sum(nil)))
		}
		return nil
	}, backoff.NewTestingBackOff()))
	require.ElementsEqual(t, fileHashes, restoredFileHashes)
}

// TestExtractRestoreNoObjects tests extraction and restoration in the case
// where existing objects are re-used (common for cloud deployments, as objects
// are stored outside of kubernetes, in object store)
func TestExtractRestoreNoObjects(t *testing.T) {
	testExtractRestore(t, false)
}

// TestExtractRestoreObjects tests extraction and restoration of objects. Note
// that since 1.8, only data in input repos is referenced by objects, so this
// tests extracting/restoring an input repo.
func TestExtractRestoreObjects(t *testing.T) {
	testExtractRestore(t, true)
}

func TestExtractRestoreHeadlessBranches(t *testing.T) {
	if testing.Short() {
		t.Skip("Skipping integration tests in short mode")
	}

	c := getPachClient(t)
	require.NoError(t, c.DeleteAll())

	dataRepo := tu.UniqueString("TestExtractRestore_data")
	require.NoError(t, c.CreateRepo(dataRepo))

	// create a headless branch
	require.NoError(t, c.CreateBranch(dataRepo, "headless", "", nil))

	ops, err := c.ExtractAll(false)
	require.NoError(t, err)
	require.NoError(t, c.DeleteAll())
	require.NoError(t, c.Restore(ops))

	bis, err := c.ListBranch(dataRepo)
	require.NoError(t, err)
	require.Equal(t, 1, len(bis))
	require.Equal(t, "headless", bis[0].Branch.Name)
}

func TestExtractVersion(t *testing.T) {
	if testing.Short() {
		t.Skip("Skipping integration tests in short mode")
	}

	c := getPachClient(t)
	require.NoError(t, c.DeleteAll())

	dataRepo := tu.UniqueString("TestExtractRestore_data")
	require.NoError(t, c.CreateRepo(dataRepo))

	r := rand.New(rand.NewSource(45))
	_, err := c.PutFile(dataRepo, "master", "file", strings.NewReader(workload.RandString(r, 40*MB)))
	require.NoError(t, err)

	pipeline := tu.UniqueString("TestExtractRestore")
	require.NoError(t, c.CreatePipeline(
		pipeline,
		"",
		[]string{"bash"},
		[]string{
			fmt.Sprintf("cp /pfs/%s/* /pfs/out/", dataRepo),
		},
		&pps.ParallelismSpec{
			Constant: 1,
		},
		client.NewPFSInput(dataRepo, "/*"),
		"",
		false,
	))

	ops, err := c.ExtractAll(false)
	require.NoError(t, err)
	require.True(t, len(ops) > 0)

	// Check that every Op looks right; the version set matches pachd's version
	for _, op := range ops {
		opV := reflect.ValueOf(op).Elem()
		var versions, nonemptyVersions int
		for i := 0; i < opV.NumField(); i++ {
			fDesc := opV.Type().Field(i)
			if !strings.HasPrefix(fDesc.Name, "Op") {
				continue
			}
			versions++
			if strings.HasSuffix(fDesc.Name,
				fmt.Sprintf("%d_%d", versionlib.MajorVersion, versionlib.MinorVersion)) {
				require.False(t, opV.Field(i).IsNil())
				nonemptyVersions++
			} else {
				require.True(t, opV.Field(i).IsNil())
			}
		}
		require.Equal(t, 1, nonemptyVersions)
		require.True(t, versions > 1)
	}
}

func TestMigrateFrom1_7(t *testing.T) {
	if testing.Short() {
		t.Skip("Skipping integration tests in short mode")
	}

	// Clear pachyderm cluster (so that next cluster starts up in a clean environment)
	c := getPachClient(t)
	require.NoError(t, c.DeleteAll())

	// Restore dumped metadata (now that objects are present)
	md, err := os.Open(path.Join(os.Getenv("GOPATH"),
		"src/github.com/pachyderm/pachyderm/etc/testing/migration/1_7/sort.metadata"))
	require.NoError(t, err)
	require.NoError(t, c.RestoreReader(snappy.NewReader(md)))
	require.NoError(t, md.Close())

	// Wait for final imported commit to be processed
	commitIter, err := c.FlushCommit([]*pfs.Commit{client.NewCommit("left", "master")}, nil)
	require.NoError(t, err)
	commitInfos := collectCommitInfos(t, commitIter)
	// filter-left and filter-right both compute a join of left and
	// right--depending on when the final commit to 'left' was added, it may have
	// been processed multiple times (should be n * 3, as there are 3 pipelines)
	require.True(t, len(commitInfos) >= 3)

	// Inspect input
	commits, err := c.ListCommit("left", "master", "", 0)
	require.NoError(t, err)
	require.Equal(t, 3, len(commits))
	commits, err = c.ListCommit("right", "master", "", 0)
	require.NoError(t, err)
	require.Equal(t, 3, len(commits))

	// Inspect output
	repos, err := c.ListRepo()
	require.NoError(t, err)
	require.ElementsEqualUnderFn(t,
		[]string{"left", "right", "copy", "sort"},
		repos, RepoInfoToName)

	// make sure all numbers 0-99 are in /nums
	var buf bytes.Buffer
	require.NoError(t, c.GetFile("sort", "master", "/nums", 0, 0, &buf))
	s := bufio.NewScanner(&buf)
	numbers := make(map[string]struct{})
	for s.Scan() {
		numbers[s.Text()] = struct{}{}
	}
	require.Equal(t, 100, len(numbers)) // job processed all inputs

	// Confirm stats commits are present
	commits, err = c.ListCommit("sort", "stats", "", 0)
	require.NoError(t, err)
	require.Equal(t, 6, len(commits))
}

func int64p(i int64) *int64 {
	return &i
}
<commit_msg>Wrap ListJob call in backoff loop<commit_after>package server

import (
	"bufio"
	"bytes"
	"fmt"
	"hash/fnv"
	"io"
	"math/rand"
	"os"
	"path"
	"reflect"
	"strings"
	"sync"
	"testing"

	"github.com/pachyderm/pachyderm/src/client"
	"github.com/pachyderm/pachyderm/src/client/pfs"
	"github.com/pachyderm/pachyderm/src/client/pkg/require"
	"github.com/pachyderm/pachyderm/src/client/pps"
	versionlib "github.com/pachyderm/pachyderm/src/client/version"
	"github.com/pachyderm/pachyderm/src/server/pkg/backoff"
	tu "github.com/pachyderm/pachyderm/src/server/pkg/testutil"
	"github.com/pachyderm/pachyderm/src/server/pkg/workload"

	"github.com/golang/snappy"
)

const (
	KB = 1024
	MB = 1024 * KB
)

var pachClient *client.APIClient
var getPachClientOnce sync.Once

func getPachClient(t testing.TB) *client.APIClient {
	getPachClientOnce.Do(func() {
		var err error
		if addr := os.Getenv("PACHD_PORT_650_TCP_ADDR"); addr != "" {
			pachClient, err = client.NewInCluster()
		} else {
			pachClient, err = client.NewOnUserMachine(false, false, "user")
		}
		require.NoError(t, err)
	})
	return pachClient
}

func collectCommitInfos(t testing.TB, commitInfoIter client.CommitInfoIterator) []*pfs.CommitInfo {
	var commitInfos []*pfs.CommitInfo
	for {
		commitInfo, err := commitInfoIter.Next()
		if err == io.EOF {
			return commitInfos
		}
		require.NoError(t, err)
		commitInfos = append(commitInfos, commitInfo)
	}
}

// TODO(msteffen) equivalent to funciton in src/server/auth/server/admin_test.go.
// These should be unified.
func RepoInfoToName(repoInfo interface{}) interface{} {
	return repoInfo.(*pfs.RepoInfo).Repo.Name
}

// testExtractRestored effectively implements both TestExtractRestoreObjects
// TestExtractRestoreNoObjects, as their logic is mostly the same
func testExtractRestore(t *testing.T, testObjects bool) {
	if testing.Short() {
		t.Skip("Skipping integration tests in short mode")
	}

	c := getPachClient(t)
	require.NoError(t, c.DeleteAll())

	dataRepo := tu.UniqueString("TestExtractRestoreObjects-in-")
	require.NoError(t, c.CreateRepo(dataRepo))

	// Create input data
	nCommits := 2
	r := rand.New(rand.NewSource(45))
	fileHashes := make([]string, 0, nCommits)
	for i := 0; i < nCommits; i++ {
		hash := fnv.New64a()
		fileContent := workload.RandString(r, 40*MB)
		_, err := c.PutFile(dataRepo, "master", fmt.Sprintf("file-%d", i),
			io.TeeReader(strings.NewReader(fileContent), hash))
		require.NoError(t, err)
		fileHashes = append(fileHashes, string(hash.Sum(nil)))
	}

	// Create test pipelines
	numPipelines := 3
	var input, pipeline string
	input = dataRepo
	for i := 0; i < numPipelines; i++ {
		pipeline = tu.UniqueString(fmt.Sprintf("TestExtractRestoreObjects-P%d-", i))
		require.NoError(t, c.CreatePipeline(
			pipeline,
			"",
			[]string{"bash"},
			[]string{
				fmt.Sprintf("cp /pfs/%s/* /pfs/out/", input),
			},
			&pps.ParallelismSpec{
				Constant: 1,
			},
			client.NewPFSInput(input, "/*"),
			"",
			false,
		))
		input = pipeline
	}

	// Wait for pipelines to process input data
	commitIter, err := c.FlushCommit([]*pfs.Commit{client.NewCommit(dataRepo, "master")}, nil)
	require.NoError(t, err)
	commitInfos := collectCommitInfos(t, commitIter)
	require.Equal(t, numPipelines, len(commitInfos))

	// Extract existing cluster state
	ops, err := c.ExtractAll(testObjects)
	require.NoError(t, err)

	// Delete existing metadata
	require.NoError(t, c.DeleteAll())

	if testObjects {
		// Delete existing objects
		require.NoError(t, c.GarbageCollect(10000))
	}

	// Restore metadata and possibly objects
	require.NoError(t, c.Restore(ops))

	// Wait for re-created pipelines to process recreated input data
	commitIter, err = c.FlushCommit([]*pfs.Commit{client.NewCommit(dataRepo, "master")}, nil)
	require.NoError(t, err)
	commitInfos = collectCommitInfos(t, commitIter)
	require.Equal(t, numPipelines, len(commitInfos))

	// Make sure recreated jobs all succeeded
	backoff.Retry(func() error {
		jis, err := c.ListJob("", nil, nil) // make sure jobs all succeeded
		require.NoError(t, err)
		for _, ji := range jis {
			// race--we may call listJob between when a job's output commit is closed
			// and when its state is updated
			if ji.State.String() == "JOB_RUNNING" || ji.State.String() == "JOB_MERGING" {
				return fmt.Errorf("output commit is closed but job state hasn't been updated")
			}
			// Job must ultimately succeed
			require.Equal(t, "JOB_SUCCESS", ji.State.String())
		}
		return nil
	}, backoff.NewTestingBackOff())

	// Make sure all branches were recreated
	bis, err := c.ListBranch(dataRepo)
	require.NoError(t, err)
	require.Equal(t, 1, len(bis))

	// Check input data
	// This check uses a backoff because sometimes GetFile causes pachd to OOM
	var restoredFileHashes []string
	require.NoError(t, backoff.Retry(func() error {
		restoredFileHashes = make([]string, 0, nCommits)
		for i := 0; i < nCommits; i++ {
			hash := fnv.New64a()
			err := c.GetFile(dataRepo, "master", fmt.Sprintf("file-%d", i), 0, 0, hash)
			if err != nil {
				return err
			}
			restoredFileHashes = append(restoredFileHashes, string(hash.Sum(nil)))
		}
		return nil
	}, backoff.NewTestingBackOff()))
	require.ElementsEqual(t, fileHashes, restoredFileHashes)

	// Check output data
	// This check uses a backoff because sometimes GetFile causes pachd to OOM
	require.NoError(t, backoff.Retry(func() error {
		restoredFileHashes = make([]string, 0, nCommits)
		for i := 0; i < nCommits; i++ {
			hash := fnv.New64a()
			err := c.GetFile(pipeline, "master", fmt.Sprintf("file-%d", i), 0, 0, hash)
			if err != nil {
				return err
			}
			restoredFileHashes = append(restoredFileHashes, string(hash.Sum(nil)))
		}
		return nil
	}, backoff.NewTestingBackOff()))
	require.ElementsEqual(t, fileHashes, restoredFileHashes)
}

// TestExtractRestoreNoObjects tests extraction and restoration in the case
// where existing objects are re-used (common for cloud deployments, as objects
// are stored outside of kubernetes, in object store)
func TestExtractRestoreNoObjects(t *testing.T) {
	testExtractRestore(t, false)
}

// TestExtractRestoreObjects tests extraction and restoration of objects. Note
// that since 1.8, only data in input repos is referenced by objects, so this
// tests extracting/restoring an input repo.
func TestExtractRestoreObjects(t *testing.T) {
	testExtractRestore(t, true)
}

func TestExtractRestoreHeadlessBranches(t *testing.T) {
	if testing.Short() {
		t.Skip("Skipping integration tests in short mode")
	}

	c := getPachClient(t)
	require.NoError(t, c.DeleteAll())

	dataRepo := tu.UniqueString("TestExtractRestore_data")
	require.NoError(t, c.CreateRepo(dataRepo))

	// create a headless branch
	require.NoError(t, c.CreateBranch(dataRepo, "headless", "", nil))

	ops, err := c.ExtractAll(false)
	require.NoError(t, err)
	require.NoError(t, c.DeleteAll())
	require.NoError(t, c.Restore(ops))

	bis, err := c.ListBranch(dataRepo)
	require.NoError(t, err)
	require.Equal(t, 1, len(bis))
	require.Equal(t, "headless", bis[0].Branch.Name)
}

func TestExtractVersion(t *testing.T) {
	if testing.Short() {
		t.Skip("Skipping integration tests in short mode")
	}

	c := getPachClient(t)
	require.NoError(t, c.DeleteAll())

	dataRepo := tu.UniqueString("TestExtractRestore_data")
	require.NoError(t, c.CreateRepo(dataRepo))

	r := rand.New(rand.NewSource(45))
	_, err := c.PutFile(dataRepo, "master", "file", strings.NewReader(workload.RandString(r, 40*MB)))
	require.NoError(t, err)

	pipeline := tu.UniqueString("TestExtractRestore")
	require.NoError(t, c.CreatePipeline(
		pipeline,
		"",
		[]string{"bash"},
		[]string{
			fmt.Sprintf("cp /pfs/%s/* /pfs/out/", dataRepo),
		},
		&pps.ParallelismSpec{
			Constant: 1,
		},
		client.NewPFSInput(dataRepo, "/*"),
		"",
		false,
	))

	ops, err := c.ExtractAll(false)
	require.NoError(t, err)
	require.True(t, len(ops) > 0)

	// Check that every Op looks right; the version set matches pachd's version
	for _, op := range ops {
		opV := reflect.ValueOf(op).Elem()
		var versions, nonemptyVersions int
		for i := 0; i < opV.NumField(); i++ {
			fDesc := opV.Type().Field(i)
			if !strings.HasPrefix(fDesc.Name, "Op") {
				continue
			}
			versions++
			if strings.HasSuffix(fDesc.Name,
				fmt.Sprintf("%d_%d", versionlib.MajorVersion, versionlib.MinorVersion)) {
				require.False(t, opV.Field(i).IsNil())
				nonemptyVersions++
			} else {
				require.True(t, opV.Field(i).IsNil())
			}
		}
		require.Equal(t, 1, nonemptyVersions)
		require.True(t, versions > 1)
	}
}

func TestMigrateFrom1_7(t *testing.T) {
	if testing.Short() {
		t.Skip("Skipping integration tests in short mode")
	}

	// Clear pachyderm cluster (so that next cluster starts up in a clean environment)
	c := getPachClient(t)
	require.NoError(t, c.DeleteAll())

	// Restore dumped metadata (now that objects are present)
	md, err := os.Open(path.Join(os.Getenv("GOPATH"),
		"src/github.com/pachyderm/pachyderm/etc/testing/migration/1_7/sort.metadata"))
	require.NoError(t, err)
	require.NoError(t, c.RestoreReader(snappy.NewReader(md)))
	require.NoError(t, md.Close())

	// Wait for final imported commit to be processed
	commitIter, err := c.FlushCommit([]*pfs.Commit{client.NewCommit("left", "master")}, nil)
	require.NoError(t, err)
	commitInfos := collectCommitInfos(t, commitIter)
	// filter-left and filter-right both compute a join of left and
	// right--depending on when the final commit to 'left' was added, it may have
	// been processed multiple times (should be n * 3, as there are 3 pipelines)
	require.True(t, len(commitInfos) >= 3)

	// Inspect input
	commits, err := c.ListCommit("left", "master", "", 0)
	require.NoError(t, err)
	require.Equal(t, 3, len(commits))
	commits, err = c.ListCommit("right", "master", "", 0)
	require.NoError(t, err)
	require.Equal(t, 3, len(commits))

	// Inspect output
	repos, err := c.ListRepo()
	require.NoError(t, err)
	require.ElementsEqualUnderFn(t,
		[]string{"left", "right", "copy", "sort"},
		repos, RepoInfoToName)

	// make sure all numbers 0-99 are in /nums
	var buf bytes.Buffer
	require.NoError(t, c.GetFile("sort", "master", "/nums", 0, 0, &buf))
	s := bufio.NewScanner(&buf)
	numbers := make(map[string]struct{})
	for s.Scan() {
		numbers[s.Text()] = struct{}{}
	}
	require.Equal(t, 100, len(numbers)) // job processed all inputs

	// Confirm stats commits are present
	commits, err = c.ListCommit("sort", "stats", "", 0)
	require.NoError(t, err)
	require.Equal(t, 6, len(commits))
}

func int64p(i int64) *int64 {
	return &i
}
<|endoftext|>


<commit_before>/**
Copyright (c) 2015 The ConnectorDB Contributors (see AUTHORS)
Licensed under the MIT license.
**/
package website

import (
	"fmt"
	"html/template"
	"io"
	"io/ioutil"
	"os"
	"path"
	"sync"
	"time"

	"github.com/kardianos/osext"
	"gopkg.in/fsnotify.v1"

	log "github.com/Sirupsen/logrus"
)

var (
	//The prefix to use for the paths in web server
	WWWPrefix = "www"
	AppPrefix = "app"

	// WWWPath is the path to the not-logged-in website files
	WWWPath string
	// AppPath is the path to the logged-in user website files
	AppPath string

	//These are the pre-loaded templates for non-logged in users
	WWWIndex *FileTemplate
	WWWLogin *FileTemplate
	WWWJoin  *FileTemplate
	WWW404   *FileTemplate

	//These are the pre-loaded templates for logged in users
	AppIndex  *FileTemplate
	AppUser   *FileTemplate
	AppDevice *FileTemplate
	AppStream *FileTemplate
	App404    *FileTemplate
	AppError  *FileTemplate
)

//FileTemplate implements all the necessary logic to read/write a "special" templated file
// as well as to update it from the folder in real time as it is modified.
type FileTemplate struct {
	sync.RWMutex //RWMutex allows for writing the template during runtime

	FilePath string
	Template *template.Template

	Watcher *fsnotify.Watcher
	done    chan bool
}

//NewFileTemplate loads a template from file and subscribes to changes from the file system
func NewFileTemplate(fpath string, err error) (*FileTemplate, error) {
	if err != nil {
		return nil, err
	}
	f, err := ioutil.ReadFile(fpath)
	if err != nil {
		log.Warnf("Could not read '%s'", fpath)
		return nil, err
	}
	tmpl, err := template.New(fpath).Parse(string(f))
	if err != nil {
		log.Warnf("Failed to parse '%s'", fpath)
		return nil, err
	}

	watch, err := fsnotify.NewWatcher()
	if err != nil {
		return nil, err
	}

	err = watch.Add(fpath)
	if err != nil {
		watch.Close()
	}

	done := make(chan bool)

	ft := &FileTemplate{
		RWMutex:  sync.RWMutex{},
		FilePath: fpath,
		Template: tmpl,
		Watcher:  watch,
		done:     done,
	}

	//Run the file watch in the background
	go ft.Watch()

	return ft, nil
}

// Reload loads up the template from the file path
func (f *FileTemplate) Reload() error {
	file, err := ioutil.ReadFile(f.FilePath)
	if err != nil {
		return err
	}

	tmpl, err := template.New(f.FilePath).Parse(string(file))
	if err != nil {
		return fmt.Errorf("Failed to parse '%s': %v", f.FilePath, err.Error())
	}
	f.Lock()
	f.Template = tmpl
	f.Unlock()

	return nil
}

//Watch is run in the background to watch for changes in the template files
func (f *FileTemplate) Watch() {
	for {
		select {
		case event := <-f.Watcher.Events:
			if event.Op&fsnotify.Write == fsnotify.Write {
				//We reload the file
				log.Infof("Reloading file: '%s'", f.FilePath)
				err := f.Reload()
				if err != nil {
					log.Warn(err.Error())
				}
			} else if event.Op&fsnotify.Remove == fsnotify.Remove || event.Op&fsnotify.Rename == fsnotify.Rename {
				log.Warningf("File '%s' removed. Using cached version.", f.FilePath)
				f.Watcher.Remove(f.FilePath)

				// Keep trying to see if the file exists until it is found again
				for {
					time.Sleep(2 * time.Second)
					v, err := os.Stat(f.FilePath)
					if err == nil && !v.IsDir() {

						err = f.Watcher.Add(f.FilePath)
						if err == nil {
							log.Infof("Reloading file: '%s'", f.FilePath)
							err := f.Reload()
							if err == nil {
								go f.Watch()
								return
							}
							log.Warn(err.Error())
						}

					}
				}

			}
		case err := <-f.Watcher.Errors:
			log.Errorf("Watcher for '%s' failed: %s", f.FilePath, err.Error())
			return
		case <-f.done:
			return
		}
	}
}

//Execute the template
func (f *FileTemplate) Execute(w io.Writer, data interface{}) error {
	f.RLock()
	err := f.Template.Execute(w, data)
	f.RUnlock()
	return err
}

// Close shuts down the file template
func (f *FileTemplate) Close() {
	f.Watcher.Close()
	f.done <- true
}

//LoadFiles sets up all the necessary files
func LoadFiles() error {
	//Now set up the app and www folder paths and make sure they exist
	exefolder, err := osext.ExecutableFolder()
	if err != nil {
		return err
	}
	WWWPath = path.Join(exefolder, WWWPrefix)
	log.Debugf("Hosting www from '%s'", WWWPath)

	AppPath = path.Join(exefolder, AppPrefix)
	log.Debugf("Hosting app from '%s'", AppPath)

	WWWIndex, err = NewFileTemplate(path.Join(WWWPath, "index.html"), err)
	WWWLogin, err = NewFileTemplate(path.Join(WWWPath, "login.html"), err)
	WWW404, err = NewFileTemplate(path.Join(WWWPath, "404.html"), err)
	WWWJoin, err = NewFileTemplate(path.Join(WWWPath, "join.html"), err)

	AppIndex, err = NewFileTemplate(path.Join(AppPath, "index.html"), err)
	AppUser, err = NewFileTemplate(path.Join(AppPath, "user.html"), err)
	AppDevice, err = NewFileTemplate(path.Join(AppPath, "device.html"), err)
	AppStream, err = NewFileTemplate(path.Join(AppPath, "stream.html"), err)
	App404, err = NewFileTemplate(path.Join(AppPath, "404.html"), err)
	AppError, err = NewFileTemplate(path.Join(AppPath, "error.html"), err)

	return err
}
<commit_msg>Added Joseph's feedback<commit_after>/**
Copyright (c) 2015 The ConnectorDB Contributors (see AUTHORS)
Licensed under the MIT license.
**/
package website

import (
	"fmt"
	"html/template"
	"io"
	"io/ioutil"
	"os"
	"path"
	"sync"
	"time"

	"github.com/kardianos/osext"
	"gopkg.in/fsnotify.v1"

	log "github.com/Sirupsen/logrus"
)

var (
	//The prefix to use for the paths in web server
	WWWPrefix = "www"
	AppPrefix = "app"

	// WWWPath is the path to the not-logged-in website files
	WWWPath string
	// AppPath is the path to the logged-in user website files
	AppPath string

	//These are the pre-loaded templates for non-logged in users
	WWWIndex *FileTemplate
	WWWLogin *FileTemplate
	WWWJoin  *FileTemplate
	WWW404   *FileTemplate

	//These are the pre-loaded templates for logged in users
	AppIndex  *FileTemplate
	AppUser   *FileTemplate
	AppDevice *FileTemplate
	AppStream *FileTemplate
	App404    *FileTemplate
	AppError  *FileTemplate
)

//FileTemplate implements all the necessary logic to read/write a "special" templated file
// as well as to update it from the folder in real time as it is modified.
type FileTemplate struct {
	sync.RWMutex //RWMutex allows for writing the template during runtime

	FilePath string
	Template *template.Template

	Watcher *fsnotify.Watcher
	done    chan bool
}

//NewFileTemplate loads a template from file and subscribes to changes from the file system
func NewFileTemplate(fpath string, err error) (*FileTemplate, error) {
	if err != nil {
		return nil, err
	}
	f, err := ioutil.ReadFile(fpath)
	if err != nil {
		log.Warnf("Could not read '%s'", fpath)
		return nil, err
	}
	tmpl, err := template.New(fpath).Parse(string(f))
	if err != nil {
		log.Warnf("Failed to parse '%s'", fpath)
		return nil, err
	}

	watch, err := fsnotify.NewWatcher()
	if err != nil {
		return nil, err
	}

	err = watch.Add(fpath)
	if err != nil {
		watch.Close()
	}

	done := make(chan bool)

	ft := &FileTemplate{
		RWMutex:  sync.RWMutex{},
		FilePath: fpath,
		Template: tmpl,
		Watcher:  watch,
		done:     done,
	}

	//Run the file watch in the background
	go ft.Watch()

	return ft, nil
}

// Reload loads up the template from the file path
func (f *FileTemplate) Reload() error {
	log.Infof("Reloading file: '%s'", f.FilePath)
	file, err := ioutil.ReadFile(f.FilePath)
	if err != nil {
		log.Warn(err.Error())
		return err
	}

	tmpl, err := template.New(f.FilePath).Parse(string(file))
	if err != nil {
		err = fmt.Errorf("Failed to parse '%s': %v", f.FilePath, err.Error())
		log.Warn(err.Error())
		return err
	}
	f.Lock()
	f.Template = tmpl
	f.Unlock()

	return nil
}

//Watch is run in the background to watch for changes in the template files
func (f *FileTemplate) Watch() {
	for {
		select {
		case event := <-f.Watcher.Events:
			if event.Op&fsnotify.Write == fsnotify.Write {
				f.Reload()
			} else if event.Op&fsnotify.Remove == fsnotify.Remove || event.Op&fsnotify.Rename == fsnotify.Rename {
				log.Warningf("File '%s' removed. Using cached version.", f.FilePath)
				f.Watcher.Remove(f.FilePath)

				// Keep trying to see if the file exists until it is found again
				for {
					time.Sleep(2 * time.Second)
					v, err := os.Stat(f.FilePath)
					if err == nil && !v.IsDir() {

						err = f.Watcher.Add(f.FilePath)
						if err == nil {
							err = f.Reload()
							if err == nil {
								break
							}
							log.Warn(err.Error())
						}

					}
				}

			}
		case err := <-f.Watcher.Errors:
			log.Errorf("Watcher for '%s' failed: %s", f.FilePath, err.Error())
			return
		case <-f.done:
			return
		}
	}
}

//Execute the template
func (f *FileTemplate) Execute(w io.Writer, data interface{}) error {
	f.RLock()
	err := f.Template.Execute(w, data)
	f.RUnlock()
	return err
}

// Close shuts down the file template
func (f *FileTemplate) Close() {
	f.Watcher.Close()
	f.done <- true
}

//LoadFiles sets up all the necessary files
func LoadFiles() error {
	//Now set up the app and www folder paths and make sure they exist
	exefolder, err := osext.ExecutableFolder()
	if err != nil {
		return err
	}
	WWWPath = path.Join(exefolder, WWWPrefix)
	log.Debugf("Hosting www from '%s'", WWWPath)

	AppPath = path.Join(exefolder, AppPrefix)
	log.Debugf("Hosting app from '%s'", AppPath)

	WWWIndex, err = NewFileTemplate(path.Join(WWWPath, "index.html"), err)
	WWWLogin, err = NewFileTemplate(path.Join(WWWPath, "login.html"), err)
	WWW404, err = NewFileTemplate(path.Join(WWWPath, "404.html"), err)
	WWWJoin, err = NewFileTemplate(path.Join(WWWPath, "join.html"), err)

	AppIndex, err = NewFileTemplate(path.Join(AppPath, "index.html"), err)
	AppUser, err = NewFileTemplate(path.Join(AppPath, "user.html"), err)
	AppDevice, err = NewFileTemplate(path.Join(AppPath, "device.html"), err)
	AppStream, err = NewFileTemplate(path.Join(AppPath, "stream.html"), err)
	App404, err = NewFileTemplate(path.Join(AppPath, "404.html"), err)
	AppError, err = NewFileTemplate(path.Join(AppPath, "error.html"), err)

	return err
}
<|endoftext|>


<commit_before>package sipparser

import (
	//"bytes"
	"fmt"
	"testing"
)

func TestSipHeaderCallIdParse(t *testing.T) {

	testdata := []struct {
		src    string
		ok     bool
		newPos int
		encode string
	}{
		{"Call-ID: abc123@a.com", true, len("Call-ID: abc123@a.com"), "Call-ID: abc123@a.com"},
		{"Call-ID: abc123", true, len("Call-ID: abc123"), "Call-ID: abc123"},
		{"Call-ID: abc123\r\n", true, len("Call-ID: abc123"), "Call-ID: abc123"},

		{" Call-ID: abc123@", false, 0, ""},
		{"Call-ID1: abc123@", false, len("Call-ID1: "), ""},
		{"Call-ID: abc123@", false, len("Call-ID: abc123@"), ""},
		{"Call-ID: @abc", false, len("Call-ID: "), ""},
	}

	context := NewParseContext()
	context.allocator = NewMemAllocator(1024 * 30)
	prefix := FuncName()

	for i, v := range testdata {
		addr := NewSipHeaderCallId(context)
		header := addr.GetSipHeaderCallId(context)
		newPos, err := header.Parse(context, []byte(v.src), 0)

		if v.ok && err != nil {
			t.Errorf("%s[%d] failed: err = %s\n", prefix, i, err)
			continue
		}

		if !v.ok && err == nil {
			t.Errorf("%s[%d] failed: should parse failed", prefix, i)
			continue
		}

		if v.newPos != newPos {
			t.Errorf("%s[%d] failed: newPos = %d, wanted = %d\n", prefix, i, newPos, v.newPos)
		}

		if v.ok && v.encode != header.String(context) {
			t.Errorf("%s[%d] failed: encode = %s, wanted = %s\n", prefix, i, header.String(context), v.encode)
			continue
		}
	}

}

func BenchmarkSipHeaderCallIdParse(b *testing.B) {
	b.StopTimer()
	//v := []byte("Call-ID: abc123@a.com")
	//v := []byte("Call-ID: 0009b7da-0352000f-30a69b83-0e7b53d6@24.15.255.101")
	v := []byte("Call-ID: 0009b7da-0352000f-30a69b83-0e7b53d6@24.15.255.101")
	context := NewParseContext()
	context.allocator = NewMemAllocator(1024 * 30)
	addr := NewSipHeaderCallId(context)
	header := addr.GetSipHeaderCallId(context)
	remain := context.allocator.Used()
	b.ReportAllocs()
	b.SetBytes(2)
	b.StartTimer()

	var i int
	for i = 0; i < b.N; i++ {
		context.allocator.ClearAllocNum()
		context.allocator.FreePart(remain)
		header.Parse(context, v, 0)
	}
	//fmt.Printf("header = %s\n", header.String(context))
	//fmt.Printf("allocator.AllocNum = %d, i= %d\n", context.allocator.AllocNum(), i)
	//fmt.Printf("allocator.Used = %d, i= %d\n", context.allocator.Used(), i)
	//fmt.Printf("")
}

func BenchmarkSipHeaderCallIdEncode(b *testing.B) {
	b.StopTimer()
	//v := []byte("Call-ID: abc123@a.com")
	v := []byte("Call-ID: 0009b7da-0352000f-30a69b83-0e7b53d6@24.15.255.101")
	context := NewParseContext()
	context.allocator = NewMemAllocator(1024 * 30)
	addr := NewSipHeaderCallId(context)
	header := addr.GetSipHeaderCallId(context)
	header.Parse(context, v, 0)
	remain := context.allocator.Used()
	//buf := bytes.NewBuffer(make([]byte, 1024*1024))
	buf := &AbnfByteBuffer{}
	b.SetBytes(2)
	b.ReportAllocs()
	b.StartTimer()

	for i := 0; i < b.N; i++ {
		buf.Reset()
		context.allocator.ClearAllocNum()
		context.allocator.FreePart(remain)
		header.Encode(context, buf)
	}
	fmt.Printf("")
}
<commit_msg>modify testcase<commit_after>package sipparser

import (
	//"bytes"
	"fmt"
	"testing"
)

func TestSipHeaderCallIdParse(t *testing.T) {

	testdata := []struct {
		src    string
		ok     bool
		newPos int
		encode string
	}{
		{"Call-ID: abc123@a.com", true, len("Call-ID: abc123@a.com"), "Call-ID: abc123@a.com"},
		{"Call-ID: abc123", true, len("Call-ID: abc123"), "Call-ID: abc123"},
		{"Call-ID: abc123\r\n", true, len("Call-ID: abc123"), "Call-ID: abc123"},

		{" Call-ID: abc123@", false, 0, ""},
		{"Call-ID1: abc123@", false, len("Call-ID1: "), ""},
		{"Call-ID: abc123@", false, len("Call-ID: abc123@"), ""},
		{"Call-ID: @abc", false, len("Call-ID: "), ""},
	}

	context := NewParseContext()
	context.allocator = NewMemAllocator(1024 * 30)
	prefix := FuncName()

	for i, v := range testdata {
		addr := NewSipHeaderCallId(context)
		header := addr.GetSipHeaderCallId(context)
		newPos, err := header.Parse(context, []byte(v.src), 0)

		if v.ok && err != nil {
			t.Errorf("%s[%d] failed: err = %s\n", prefix, i, err)
			continue
		}

		if !v.ok && err == nil {
			t.Errorf("%s[%d] failed: should parse failed", prefix, i)
			continue
		}

		if v.newPos != newPos {
			t.Errorf("%s[%d] failed: newPos = %d, wanted = %d\n", prefix, i, newPos, v.newPos)
		}

		if v.ok && v.encode != header.String(context) {
			t.Errorf("%s[%d] failed: encode = %s, wanted = %s\n", prefix, i, header.String(context), v.encode)
			continue
		}
	}

}

func BenchmarkSipHeaderCallIdParse(b *testing.B) {
	b.StopTimer()
	//v := []byte("Call-ID: abc123@a.com")
	//v := []byte("Call-ID: 0009b7da-0352000f-30a69b83-0e7b53d6@24.15.255.101")
	v := []byte("Call-ID: 0009b7da-0352000f-30a69b83-0e7b53d6@24.15.255.101")
	context := NewParseContext()
	context.allocator = NewMemAllocator(1024 * 30)
	addr := NewSipHeaderCallId(context)
	header := addr.GetSipHeaderCallId(context)
	remain := context.allocator.Used()
	b.ReportAllocs()
	b.SetBytes(2)
	b.StartTimer()

	var i int
	for i = 0; i < b.N; i++ {
		context.allocator.ClearAllocNum()
		context.allocator.FreePart(remain)
		header.Parse(context, v, 0)
	}
	//fmt.Printf("header = %s\n", header.String(context))
	//fmt.Printf("allocator.AllocNum = %d, i= %d\n", context.allocator.AllocNum(), i)
	//fmt.Printf("allocator.Used = %d, i= %d\n", context.allocator.Used(), i)
	//fmt.Printf("")
}

func BenchmarkSipHeaderCallIdEncode(b *testing.B) {
	b.StopTimer()
	//v := []byte("Call-ID: abc123@a.com")
	v := []byte("Call-ID: 0009b7da-0352000f-30a69b83-0e7b53d6@24.15.255.101")
	context := NewParseContext()
	context.allocator = NewMemAllocator(1024 * 30)
	addr := NewSipHeaderCallId(context)
	header := addr.GetSipHeaderCallId(context)
	header.Parse(context, v, 0)
	remain := context.allocator.Used()
	//buf := bytes.NewBuffer(make([]byte, 1024*1024))
	buf := &AbnfByteBuffer{}
	b.SetBytes(2)
	b.ReportAllocs()
	b.StartTimer()

	for i := 0; i < b.N; i++ {
		buf.Reset()
		context.allocator.ClearAllocNum()
		context.allocator.FreePart(remain)
		header.Encode(context, buf)
	}
	fmt.Printf("")
	//fmt.Println("header =", header.String(context))
}
<|endoftext|>


<commit_before>
package models

import "testing"<commit_msg>Added testing skeleton for initial test of Database<commit_after>
package models

import "testing"

// TestCollectionResponse tests connections with existing collections.
// This includes Create, Read, Update, and Destroy operations.
func TestCollectionResponse(t *testing.T) {
  	// Make a colReq channel
  	colReq := make(chan ColRequest)

  	// Activate database
	go Database(colReq)
  	close(colReq)

  	// Connect to existing collection
  		// Build connection request
  		// Send connection request

  // The following tests should be in separate functions:

  	// Test Read:
  		// Positive: To test Read we try to read something we know exists in
  		// the collection and check that it matches.
  		// Negative: Also try to read for a document that doesn't exist.

  	// Test Create:
  		// Positive: To test create, generate a random document, check that it is
  		// unique and then put it in the database. Read to see that it is
  		// there and matches the input.

	// Test Update:
  		// Positive: To test update, pick a document, update it in a random
  		// way, read it by id and then compare to see that the changes
  		// are correct.
  		// Negative: Also try to update an non existent document.

  	// Test Destroy:
  		// Positive: To test destroy, pick a document to destroy, and destroy it.
  		// Read to make sure that it is not there.
 	 	// Negative: Also try to destroy a non existent document.

}<|endoftext|>


<commit_before>package lord

import (
	"time"
	"github.com/concourse/time-resource/models"
)

type TimeLord struct {
	PreviousTime time.Time
	Location     *models.Location
	Start        *models.TimeOfDay
	Stop         *models.TimeOfDay
	Interval     *models.Interval
	Days         []models.Weekday
}

func (tl TimeLord) Check(now time.Time) bool {

	start, stop := tl.latestRangeBefore(now)

	if !tl.daysMatch(now) {
		return false
	}

	if !start.IsZero() && (now.Before(start) || !now.Before(stop)) {
		return false
	}

	if tl.PreviousTime.IsZero() {
		return true
	}

	if tl.Interval != nil {
		if now.Sub(tl.PreviousTime) >= time.Duration(*tl.Interval) {
			return true
		}
	} else if !start.IsZero() {
		return tl.PreviousTime.Before(start)
	}

	return false
}

func (tl TimeLord) daysMatch(now time.Time) bool {
	if len(tl.Days) == 0 {
		return true
	}

	todayInLoc := models.Weekday(now.In(tl.loc()).Weekday())

	for _, day := range tl.Days {
		if day == todayInLoc {
			return true
		}
	}

	return false
}

func (tl TimeLord) latestRangeBefore(reference time.Time) (time.Time, time.Time) {

	if tl.Start == nil || tl.Stop == nil {
		return time.Time{}, time.Time{}
	}

	refInLoc := reference.In(tl.loc())

	start := time.Date(refInLoc.Year(), refInLoc.Month(), refInLoc.Day(),
		tl.Start.Hour(), tl.Start.Minute(), 0, 0, tl.loc())

	if start.After(refInLoc) {
		start = start.AddDate(0, 0, -1)
	}

	stop := time.Date(start.Year(), start.Month(), start.Day(),
		tl.Stop.Hour(), tl.Stop.Minute(), 0, 0, tl.loc())

	if stop.Before(start) {
		stop = stop.AddDate(0, 0, 1)
	}

	return start, stop
}

func (tl TimeLord) loc() *time.Location {
	if tl.Location != nil {
		return (*time.Location)(tl.Location)
	}

	return time.UTC
}
<commit_msg>Codify implicit defaults for start and stop<commit_after>package lord

import (
	"time"

	"github.com/concourse/time-resource/models"
)

var DEFAULT_TIME_OF_DAY = models.TimeOfDay(time.Duration(0))

type TimeLord struct {
	PreviousTime time.Time
	Location     *models.Location
	Start        *models.TimeOfDay
	Stop         *models.TimeOfDay
	Interval     *models.Interval
	Days         []models.Weekday
}

func (tl TimeLord) Check(now time.Time) bool {

	start, stop := tl.latestRangeBefore(now)

	if !tl.daysMatch(now) {
		return false
	}

	if !start.IsZero() && (now.Before(start) || !now.Before(stop)) {
		return false
	}

	if tl.PreviousTime.IsZero() {
		return true
	}

	if tl.Interval != nil {
		if now.Sub(tl.PreviousTime) >= time.Duration(*tl.Interval) {
			return true
		}
	} else if !start.IsZero() {
		return tl.PreviousTime.Before(start)
	}

	return false
}

func (tl TimeLord) daysMatch(now time.Time) bool {
	if len(tl.Days) == 0 {
		return true
	}

	todayInLoc := models.Weekday(now.In(tl.loc()).Weekday())

	for _, day := range tl.Days {
		if day == todayInLoc {
			return true
		}
	}

	return false
}

func (tl TimeLord) latestRangeBefore(reference time.Time) (time.Time, time.Time) {

	tlStart := DEFAULT_TIME_OF_DAY
	if tl.Start != nil {
		tlStart = *tl.Start
	}
	tlStop := DEFAULT_TIME_OF_DAY
	if tl.Stop != nil {
		tlStop = *tl.Stop
	}

	refInLoc := reference.In(tl.loc())

	start := time.Date(refInLoc.Year(), refInLoc.Month(), refInLoc.Day(),
		tlStart.Hour(), tlStart.Minute(), 0, 0, tl.loc())

	if start.After(refInLoc) {
		start = start.AddDate(0, 0, -1)
	}

	stop := time.Date(start.Year(), start.Month(), start.Day(),
		tlStop.Hour(), tlStop.Minute(), 0, 0, tl.loc())

	if !stop.After(start) {
		stop = stop.AddDate(0, 0, 1)
	}

	return start, stop
}

func (tl TimeLord) loc() *time.Location {
	if tl.Location != nil {
		return (*time.Location)(tl.Location)
	}

	return time.UTC
}
<|endoftext|>