<commit_before>package schedule

import (
	"testing"
	"time"
)

func XTestBuildCommitScheduleFullWeeks(t *testing.T) {
	var tests = []struct {
		startDay time.Time
		numDays  int
	}{
		{time.Date(2009, time.November, 9, 0, 0, 0, 0, time.UTC), 7},
		{time.Date(2015, time.November, 9, 0, 0, 0, 0, time.UTC), 21},
		{time.Date(2014, time.July, 7, 0, 0, 0, 0, time.UTC), 365},
	}
	for _, test := range tests {
		days := getTestDays(test.startDay, test.numDays)
		schedule := BuildCommitSchedule(days)
		for r, row := range schedule {
			for c, col := range row {
				if col != EMPTY {
					fmt := "Expected only EMPTY values, but got %d at (%d,%d)"
					t.Errorf(fmt, col, r, c)
				}
			}
		}
	}
}

func XTestBuildCommitScheduleWednesdayStart(t *testing.T) {
	var tests = []struct {
		startDay time.Time
		numDays  int
	}{
		{time.Date(2009, time.November, 11, 0, 0, 0, 0, time.UTC), 9},
		{time.Date(2015, time.November, 11, 0, 0, 0, 0, time.UTC), 23},
		{time.Date(2014, time.July, 9, 0, 0, 0, 0, time.UTC), 367},
	}
	for _, test := range tests {
		days := getTestDays(test.startDay, test.numDays)
		schedule := BuildCommitSchedule(days)
		for r, row := range schedule {
			for c, col := range row {
				firstWeekMondayOrTuesDay := (c == 0 && r < 2)
				if firstWeekMondayOrTuesDay {
					if col != NOT_A_FIELD {
						fmt := "Expected NOT_A_FIELD values, but got %d at (%d,%d)"
						t.Errorf(fmt, col, r, c)
					}
				} else if col != EMPTY {
					fmt := "Expected only EMPTY values, but got %d at (%d,%d)"
					t.Errorf(fmt, col, r, c)
				}
			}
		}
	}
}

func XTestBuildCommitScheduleThrusdayEnd(t *testing.T) {
	var tests = []struct {
		startDay time.Time
		numDays  int
	}{
		{time.Date(2009, time.November, 9, 0, 0, 0, 0, time.UTC), 4},
		{time.Date(2015, time.November, 9, 0, 0, 0, 0, time.UTC), 19},
		{time.Date(2014, time.July, 7, 0, 0, 0, 0, time.UTC), 362},
	}
	for _, test := range tests {
		days := getTestDays(test.startDay, test.numDays)
		schedule := BuildCommitSchedule(days)
		for r, row := range schedule {
			for c, col := range row {
				lastWeekFridayOrSaturdayOrSunday := (c == len(row) && r > 4)
				if lastWeekFridayOrSaturdayOrSunday {
					if col != NOT_A_FIELD {
						fmt := "Expected NOT_A_FIELD values, but got %d at (%d,%d)"
						t.Errorf(fmt, col, r, c)
					}
				} else if col != EMPTY {
					fmt := "Expected only EMPTY values, but got %d at (%d,%d)"
					t.Errorf(fmt, col, r, c)
				}
			}
		}
	}
}

func getTestDays(startDay time.Time, numDays int) []time.Time {
	var resultingDays []time.Time
	for i := 0; i < numDays; i++ {
		resultingDays = append(resultingDays, startDay.AddDate(0, 0, i))
	}
	return resultingDays
}

func TestBuildFirstWeek(t *testing.T) {
	var tests = []struct {
		day              time.Weekday
		expectedSchedule []ScheduleEntries
	}{
		{time.Sunday, []ScheduleEntries{0, 0, 0, 0, 0, 0, 0}},
		{time.Monday, []ScheduleEntries{-1, 0, 0, 0, 0, 0, 0}},
		{time.Tuesday, []ScheduleEntries{-1, -1, 0, 0, 0, 0, 0}},
		{time.Wednesday, []ScheduleEntries{-1, -1, -1, 0, 0, 0, 0}},
		{time.Thursday, []ScheduleEntries{-1, -1, -1, -1, 0, 0, 0}},
		{time.Friday, []ScheduleEntries{-1, -1, -1, -1, -1, 0, 0}},
		{time.Saturday, []ScheduleEntries{-1, -1, -1, -1, -1, -1, 0}},
	}
	for _, test := range tests {
		actualSchedule := buildFirstWeek(test.day)
		if !sliceEqual(actualSchedule, test.expectedSchedule) {
			fmt := "Expected %v as schedule, but got %v"
			t.Errorf(fmt, test.expectedSchedule, actualSchedule)
		}
	}
}

func TestBuildLastWeek(t *testing.T) {
	var tests = []struct {
		day              time.Weekday
		expectedSchedule []ScheduleEntries
	}{
		{time.Sunday, []ScheduleEntries{0, -1, -1, -1, -1, -1, -1}},
		{time.Monday, []ScheduleEntries{0, 0, -1, -1, -1, -1, -1}},
		{time.Tuesday, []ScheduleEntries{0, 0, 0, -1, -1, -1, -1}},
		{time.Wednesday, []ScheduleEntries{0, 0, 0, 0, -1, -1, -1}},
		{time.Thursday, []ScheduleEntries{0, 0, 0, 0, 0, -1, -1}},
		{time.Friday, []ScheduleEntries{0, 0, 0, 0, 0, 0, -1}},
		{time.Saturday, []ScheduleEntries{0, 0, 0, 0, 0, 0, 0}},
	}
	for _, test := range tests {
		actualSchedule := buildLastWeek(test.day)
		if !sliceEqual(actualSchedule, test.expectedSchedule) {
			fmt := "Expected %v as schedule, but got %v"
			t.Errorf(fmt, test.expectedSchedule, actualSchedule)
		}
	}
}

func sliceEqual(sliceA, sliceB []ScheduleEntries) bool {
	if len(sliceA) != len(sliceB) {
		return false
	}
	for i := 0; i < len(sliceA); i++ {
		if sliceA[i] != sliceB[i] {
			return false
		}
	}
	return true
}

func TestConnectWeeksToSchedule(t *testing.T) {
	var tests = []struct {
		firstDay   time.Weekday
		lastDay    time.Weekday
		numEntries int
	}{
		{time.Sunday, time.Saturday, 371},
	}
	for _, test := range tests {
		firstWeek := buildFirstWeek(test.firstDay)
		lastWeek := buildLastWeek(test.lastDay)
		schedule := connectWeeksToSchedule(firstWeek, lastWeek)
		length := 0
		for _, row := range schedule {
			for _, entry := range row {
				if entry != EMPTY || entry != NOT_A_FIELD {
					t.Errorf("Entry should be EMPTY or NOT_A_FIELD, but was %v", entry)
				}
				length++
			}
		}
		if length != test.numEntries {
			t.Errorf("Expected length was %d, but got %d", test.numEntries, length)
		}
	}
}
<commit_msg>Fix test logic for connect weeks and add notafield check.<commit_after>package schedule

import (
	"testing"
	"time"
)

func XTestBuildCommitScheduleFullWeeks(t *testing.T) {
	var tests = []struct {
		startDay time.Time
		numDays  int
	}{
		{time.Date(2009, time.November, 9, 0, 0, 0, 0, time.UTC), 7},
		{time.Date(2015, time.November, 9, 0, 0, 0, 0, time.UTC), 21},
		{time.Date(2014, time.July, 7, 0, 0, 0, 0, time.UTC), 365},
	}
	for _, test := range tests {
		days := getTestDays(test.startDay, test.numDays)
		schedule := BuildCommitSchedule(days)
		for r, row := range schedule {
			for c, col := range row {
				if col != EMPTY {
					fmt := "Expected only EMPTY values, but got %d at (%d,%d)"
					t.Errorf(fmt, col, r, c)
				}
			}
		}
	}
}

func XTestBuildCommitScheduleWednesdayStart(t *testing.T) {
	var tests = []struct {
		startDay time.Time
		numDays  int
	}{
		{time.Date(2009, time.November, 11, 0, 0, 0, 0, time.UTC), 9},
		{time.Date(2015, time.November, 11, 0, 0, 0, 0, time.UTC), 23},
		{time.Date(2014, time.July, 9, 0, 0, 0, 0, time.UTC), 367},
	}
	for _, test := range tests {
		days := getTestDays(test.startDay, test.numDays)
		schedule := BuildCommitSchedule(days)
		for r, row := range schedule {
			for c, col := range row {
				firstWeekMondayOrTuesDay := (c == 0 && r < 2)
				if firstWeekMondayOrTuesDay {
					if col != NOT_A_FIELD {
						fmt := "Expected NOT_A_FIELD values, but got %d at (%d,%d)"
						t.Errorf(fmt, col, r, c)
					}
				} else if col != EMPTY {
					fmt := "Expected only EMPTY values, but got %d at (%d,%d)"
					t.Errorf(fmt, col, r, c)
				}
			}
		}
	}
}

func XTestBuildCommitScheduleThrusdayEnd(t *testing.T) {
	var tests = []struct {
		startDay time.Time
		numDays  int
	}{
		{time.Date(2009, time.November, 9, 0, 0, 0, 0, time.UTC), 4},
		{time.Date(2015, time.November, 9, 0, 0, 0, 0, time.UTC), 19},
		{time.Date(2014, time.July, 7, 0, 0, 0, 0, time.UTC), 362},
	}
	for _, test := range tests {
		days := getTestDays(test.startDay, test.numDays)
		schedule := BuildCommitSchedule(days)
		for r, row := range schedule {
			for c, col := range row {
				lastWeekFridayOrSaturdayOrSunday := (c == len(row) && r > 4)
				if lastWeekFridayOrSaturdayOrSunday {
					if col != NOT_A_FIELD {
						fmt := "Expected NOT_A_FIELD values, but got %d at (%d,%d)"
						t.Errorf(fmt, col, r, c)
					}
				} else if col != EMPTY {
					fmt := "Expected only EMPTY values, but got %d at (%d,%d)"
					t.Errorf(fmt, col, r, c)
				}
			}
		}
	}
}

func getTestDays(startDay time.Time, numDays int) []time.Time {
	var resultingDays []time.Time
	for i := 0; i < numDays; i++ {
		resultingDays = append(resultingDays, startDay.AddDate(0, 0, i))
	}
	return resultingDays
}

func TestBuildFirstWeek(t *testing.T) {
	var tests = []struct {
		day              time.Weekday
		expectedSchedule []ScheduleEntries
	}{
		{time.Sunday, []ScheduleEntries{0, 0, 0, 0, 0, 0, 0}},
		{time.Monday, []ScheduleEntries{-1, 0, 0, 0, 0, 0, 0}},
		{time.Tuesday, []ScheduleEntries{-1, -1, 0, 0, 0, 0, 0}},
		{time.Wednesday, []ScheduleEntries{-1, -1, -1, 0, 0, 0, 0}},
		{time.Thursday, []ScheduleEntries{-1, -1, -1, -1, 0, 0, 0}},
		{time.Friday, []ScheduleEntries{-1, -1, -1, -1, -1, 0, 0}},
		{time.Saturday, []ScheduleEntries{-1, -1, -1, -1, -1, -1, 0}},
	}
	for _, test := range tests {
		actualSchedule := buildFirstWeek(test.day)
		if !sliceEqual(actualSchedule, test.expectedSchedule) {
			fmt := "Expected %v as schedule, but got %v"
			t.Errorf(fmt, test.expectedSchedule, actualSchedule)
		}
	}
}

func TestBuildLastWeek(t *testing.T) {
	var tests = []struct {
		day              time.Weekday
		expectedSchedule []ScheduleEntries
	}{
		{time.Sunday, []ScheduleEntries{0, -1, -1, -1, -1, -1, -1}},
		{time.Monday, []ScheduleEntries{0, 0, -1, -1, -1, -1, -1}},
		{time.Tuesday, []ScheduleEntries{0, 0, 0, -1, -1, -1, -1}},
		{time.Wednesday, []ScheduleEntries{0, 0, 0, 0, -1, -1, -1}},
		{time.Thursday, []ScheduleEntries{0, 0, 0, 0, 0, -1, -1}},
		{time.Friday, []ScheduleEntries{0, 0, 0, 0, 0, 0, -1}},
		{time.Saturday, []ScheduleEntries{0, 0, 0, 0, 0, 0, 0}},
	}
	for _, test := range tests {
		actualSchedule := buildLastWeek(test.day)
		if !sliceEqual(actualSchedule, test.expectedSchedule) {
			fmt := "Expected %v as schedule, but got %v"
			t.Errorf(fmt, test.expectedSchedule, actualSchedule)
		}
	}
}

func sliceEqual(sliceA, sliceB []ScheduleEntries) bool {
	if len(sliceA) != len(sliceB) {
		return false
	}
	for i := 0; i < len(sliceA); i++ {
		if sliceA[i] != sliceB[i] {
			return false
		}
	}
	return true
}

func TestConnectWeeksToSchedule(t *testing.T) {
	var tests = []struct {
		firstDay            time.Weekday
		lastDay             time.Weekday
		numEntries          int
		numNotAFieldEntries int
	}{
		{time.Sunday, time.Saturday, 371, 0},
		{time.Monday, time.Saturday, 371, 1},
		{time.Wednesday, time.Wednesday, 371, 6},
		{time.Saturday, time.Sunday, 371, 12},
	}
	for _, test := range tests {
		firstWeek := buildFirstWeek(test.firstDay)
		lastWeek := buildLastWeek(test.lastDay)
		schedule := connectWeeksToSchedule(firstWeek, lastWeek)
		length := 0
		numNotAFieldEntries := 0
		for _, row := range schedule {
			for _, entry := range row {
				if entry != EMPTY && entry != NOT_A_FIELD {
					t.Errorf("Entry should be EMPTY or NOT_A_FIELD, but was %v", entry)
				}
				if entry == NOT_A_FIELD {
					numNotAFieldEntries++
				}
				length++
			}
		}

		if length != test.numEntries {
			t.Errorf("Expected length was %d, but got %d", test.numEntries, length)
		}

		if numNotAFieldEntries != test.numNotAFieldEntries {
			t.Errorf("Expected %d NOT_A_FIELD, but got %d", test.numNotAFieldEntries, numNotAFieldEntries)
		}
	}
}
<|endoftext|>


<commit_before>package ozero

import (
	"fmt"
	"time"
)

// WorkerFunc defines a function that receives a job and processes it.
type WorkerFunc func(interface{})

// DefaultWorkerID is the default WorkerID when posting jobs without defined WorkerID.
const DefaultWorkerID string = "_DEFAULT"

func (pool *Pool) worker() {
	var job job

	// Catch panics and notify of exit.
	defer func() {
		if r := recover(); nil != r {
			err, ok := r.(error)
			if !ok {
				err = fmt.Errorf("pkg: %v", r)
			}
			pool.mutex.RLock()
			defer pool.mutex.RUnlock()
			if nil != pool.errorFunc {
				pool.errorFunc(job.Data, err)
			}
		}
		pool.workerExitedCh <- struct{}{}
	}()

	for job = range pool.jobsCh {
		pool.mutex.RLock()
		f := pool.workers[job.WorkerID]
		tries := pool.totalTryCount
		delay := pool.retryTimeout
		pool.mutex.RUnlock()

		if nil != f {
			var err error
			for i := 0; ; i++ {
				err = pool.work(f, job.Data)
				if nil == err { // Executed successfully
					break
				}
				if (0 != tries) && (i >= tries) { // Maximum try count not infinite and exceeded.
					break
				}
				time.Sleep(delay)
			}
			if nil != err {
				panic(err)
			}
		}
	}
}

func (pool *Pool) work(f WorkerFunc, data interface{}) (ret error) {
	// Catch panics and notify of exit.
	defer func() {
		if r := recover(); nil != r {
			err, ok := r.(error)
			if !ok {
				err = fmt.Errorf("pkg: %v", r)
			}
			ret = err
		}
	}()

	f(data)

	return nil
}

// AddWorkerFunc adds the function to be processed when sending jobs to the default workerId.
func (pool *Pool) AddWorkerFunc(f WorkerFunc) *Pool {
	pool.mutex.Lock()
	defer pool.mutex.Unlock()

	pool.workers["_DEFAULT"] = f
	return pool
}

// AddWorkerFuncForWorkerID adds the function to be processed when sending jobs to the specified workerId.
func (pool *Pool) AddWorkerFuncForWorkerID(workerID string, f WorkerFunc) *Pool {
	pool.mutex.Lock()
	defer pool.mutex.Unlock()

	pool.workers[workerID] = f
	return pool
}

// SetErrorFunc sets the function to be executed when a panic occurrs in a worker.
// If nil, nothing gets executed on panic.
func (pool *Pool) SetErrorFunc(f ErrorFunc) *Pool {
	pool.mutex.Lock()
	defer pool.mutex.Unlock()

	pool.errorFunc = f
	return pool
}
<commit_msg>Fixed retry count trying once more than needed.<commit_after>package ozero

import (
	"fmt"
	"time"
)

// WorkerFunc defines a function that receives a job and processes it.
type WorkerFunc func(interface{})

// DefaultWorkerID is the default WorkerID when posting jobs without defined WorkerID.
const DefaultWorkerID string = "_DEFAULT"

func (pool *Pool) worker() {
	var job job

	// Catch panics and notify of exit.
	defer func() {
		if r := recover(); nil != r {
			err, ok := r.(error)
			if !ok {
				err = fmt.Errorf("pkg: %v", r)
			}
			pool.mutex.RLock()
			defer pool.mutex.RUnlock()
			if nil != pool.errorFunc {
				pool.errorFunc(job.Data, err)
			}
		}
		pool.workerExitedCh <- struct{}{}
	}()

	for job = range pool.jobsCh {
		pool.mutex.RLock()
		f := pool.workers[job.WorkerID]
		tries := pool.totalTryCount
		delay := pool.retryTimeout
		pool.mutex.RUnlock()

		if nil != f {
			var err error
			var currentTry = 0
			for {
				err = pool.work(f, job.Data)
				if nil == err { // Executed successfully
					break
				}
				if 0 != tries { // Maximum try count not infinite.
					currentTry++
					if currentTry >= tries { // Maximum try count exceeded.
						break
					}
				}
				time.Sleep(delay)
			}
			if nil != err {
				panic(err)
			}
		}
	}
}

func (pool *Pool) work(f WorkerFunc, data interface{}) (ret error) {
	// Catch panics and notify of exit.
	defer func() {
		if r := recover(); nil != r {
			err, ok := r.(error)
			if !ok {
				err = fmt.Errorf("pkg: %v", r)
			}
			ret = err
		}
	}()

	f(data)

	return nil
}

// AddWorkerFunc adds the function to be processed when sending jobs to the default workerId.
func (pool *Pool) AddWorkerFunc(f WorkerFunc) *Pool {
	pool.mutex.Lock()
	defer pool.mutex.Unlock()

	pool.workers["_DEFAULT"] = f
	return pool
}

// AddWorkerFuncForWorkerID adds the function to be processed when sending jobs to the specified workerId.
func (pool *Pool) AddWorkerFuncForWorkerID(workerID string, f WorkerFunc) *Pool {
	pool.mutex.Lock()
	defer pool.mutex.Unlock()

	pool.workers[workerID] = f
	return pool
}

// SetErrorFunc sets the function to be executed when a panic occurrs in a worker.
// If nil, nothing gets executed on panic.
func (pool *Pool) SetErrorFunc(f ErrorFunc) *Pool {
	pool.mutex.Lock()
	defer pool.mutex.Unlock()

	pool.errorFunc = f
	return pool
}
<|endoftext|>


<commit_before>package v8worker

/*
#cgo CXXFLAGS: -std=c++11
#cgo pkg-config: v8.pc
#include <stdlib.h>
#include "binding.h"
*/
import "C"
import "errors"
import "unsafe"
import "sync"

// To receive messages from javascript...
type ReceiveMessageCallback func(msg string)

// Don't init V8 more than once.
var initV8Once sync.Once

// This is a golang wrapper around a single V8 Isolate.
type Worker struct {
	cWorker *C.worker
	cb      ReceiveMessageCallback
}

// Return the V8 version E.G. "4.3.59"
func Version() string {
	return C.GoString(C.worker_version())
}

//export recvCb
func recvCb(msg_s *C.char, ptr unsafe.Pointer) {
	msg := C.GoString(msg_s)
	worker := (*Worker)(ptr)
	worker.cb(msg)
}

// Creates a new worker, which corresponds to a V8 isolate. A single threaded
// standalone execution context.
func New(cb ReceiveMessageCallback) *Worker {
	worker := &Worker{
		cb: cb,
	}

	initV8Once.Do(func() {
		C.v8_init()
	})

	callback := C.worker_recv_cb(C.go_recv_cb)

	worker.cWorker = C.worker_new(callback, unsafe.Pointer(worker))
	return worker
}

// Load and executes a javascript file with the filename specified by
// scriptName and the contents of the file specified by the param code.
func (w *Worker) Load(scriptName string, code string) error {
	scriptName_s := C.CString(scriptName)
	code_s := C.CString(code)
	r := C.worker_load(w.cWorker, scriptName_s, code_s)
	if r != 0 {
		errStr := C.GoString(C.worker_last_exception(w.cWorker))
		return errors.New(errStr)
	}
	return nil
}

// Sends a message to a worker. The $recv callback in js will be called.
func (w *Worker) Send(msg string) error {
	msg_s := C.CString(string(msg))
	defer C.free(unsafe.Pointer(msg_s))

	r := C.worker_send(w.cWorker, msg_s)
	if r != 0 {
		errStr := C.GoString(C.worker_last_exception(w.cWorker))
		return errors.New(errStr)
	}

	return nil
}
<commit_msg>fixed memory leak<commit_after>package v8worker

/*
#cgo CXXFLAGS: -std=c++11
#cgo pkg-config: v8.pc
#include <stdlib.h>
#include "binding.h"
*/
import "C"
import "errors"
import "unsafe"
import "sync"

// To receive messages from javascript...
type ReceiveMessageCallback func(msg string)

// Don't init V8 more than once.
var initV8Once sync.Once

// This is a golang wrapper around a single V8 Isolate.
type Worker struct {
	cWorker *C.worker
	cb      ReceiveMessageCallback
}

// Return the V8 version E.G. "4.3.59"
func Version() string {
	return C.GoString(C.worker_version())
}

//export recvCb
func recvCb(msg_s *C.char, ptr unsafe.Pointer) {
	msg := C.GoString(msg_s)
	worker := (*Worker)(ptr)
	worker.cb(msg)
}

// Creates a new worker, which corresponds to a V8 isolate. A single threaded
// standalone execution context.
func New(cb ReceiveMessageCallback) *Worker {
	worker := &Worker{
		cb: cb,
	}

	initV8Once.Do(func() {
		C.v8_init()
	})

	callback := C.worker_recv_cb(C.go_recv_cb)

	worker.cWorker = C.worker_new(callback, unsafe.Pointer(worker))
	return worker
}

// Load and executes a javascript file with the filename specified by
// scriptName and the contents of the file specified by the param code.
func (w *Worker) Load(scriptName string, code string) error {
	scriptName_s := C.CString(scriptName)
	code_s := C.CString(code)
	defer C.free(unsafe.Pointer(scriptName_s))
	defer C.free(unsafe.Pointer(code_s))

	r := C.worker_load(w.cWorker, scriptName_s, code_s)
	if r != 0 {
		errStr := C.GoString(C.worker_last_exception(w.cWorker))
		return errors.New(errStr)
	}
	return nil
}

// Sends a message to a worker. The $recv callback in js will be called.
func (w *Worker) Send(msg string) error {
	msg_s := C.CString(string(msg))
	defer C.free(unsafe.Pointer(msg_s))

	r := C.worker_send(w.cWorker, msg_s)
	if r != 0 {
		errStr := C.GoString(C.worker_last_exception(w.cWorker))
		return errors.New(errStr)
	}

	return nil
}
<|endoftext|>


<commit_before>package gocrawl

import (
	"bytes"
	"io/ioutil"
	"net/http"
	"net/url"
	"strings"
	"sync"
	"time"

	"github.com/PuerkitoBio/goquery"
	"github.com/temoto/robotstxt.go"
	"golang.org/x/net/html"
)

// The worker is dedicated to fetching and visiting a given host, respecting
// this host's robots.txt crawling policies.
type worker struct {
	// Worker identification
	host  string
	index int

	// Communication channels and sync
	push    chan<- *workerResponse
	pop     popChannel
	stop    chan struct{}
	enqueue chan<- interface{}
	wg      *sync.WaitGroup

	// Robots validation
	robotsGroup *robotstxt.Group

	// Logging
	logFunc func(LogFlags, string, ...interface{})

	// Implementation fields
	wait           <-chan time.Time
	lastFetch      *FetchInfo
	lastCrawlDelay time.Duration
	opts           *Options
}

// Start crawling the host.
func (this *worker) run() {
	defer func() {
		this.logFunc(LogInfo, "worker done.")
		this.wg.Done()
	}()

	// Enter loop to process URLs until stop signal is received
	for {
		var idleChan <-chan time.Time

		this.logFunc(LogInfo, "waiting for pop...")

		// Initialize the idle timeout channel, if required
		if this.opts.WorkerIdleTTL > 0 {
			idleChan = time.After(this.opts.WorkerIdleTTL)
		}

		select {
		case <-this.stop:
			this.logFunc(LogInfo, "stop signal received.")
			return

		case <-idleChan:
			this.logFunc(LogInfo, "idle timeout received.")
			this.sendResponse(nil, false, nil, true)
			return

		case batch := <-this.pop:

			// Got a batch of urls to crawl, loop and check at each iteration if a stop
			// is received.
			for _, ctx := range batch {
				this.logFunc(LogInfo, "popped: %s", ctx.url)

				if ctx.IsRobotsURL() {
					this.requestRobotsTxt(ctx)
				} else if this.isAllowedPerRobotsPolicies(ctx.url) {
					this.requestUrl(ctx, ctx.HeadBeforeGet)
				} else {
					// Must still notify Crawler that this URL was processed, although not visited
					this.opts.Extender.Disallowed(ctx)
					this.sendResponse(ctx, false, nil, false)
				}

				// No need to check for idle timeout here, no idling while looping through
				// a batch of URLs.
				select {
				case <-this.stop:
					this.logFunc(LogInfo, "stop signal received.")
					return
				default:
					// Nothing, just continue...
				}
			}
		}
	}
}

// Checks if the given URL can be fetched based on robots.txt policies.
func (this *worker) isAllowedPerRobotsPolicies(u *url.URL) bool {
	if this.robotsGroup != nil {
		// Is this URL allowed per robots.txt policy?
		ok := this.robotsGroup.Test(u.Path)
		if !ok {
			this.logFunc(LogIgnored, "ignored on robots.txt policy: %s", u.String())
		}
		return ok
	}

	// No robots.txt = everything is allowed
	return true
}

// Process the specified URL.
func (this *worker) requestUrl(ctx *URLContext, headRequest bool) {
	if res, ok := this.fetchUrl(ctx, this.opts.UserAgent, headRequest); ok {
		var harvested interface{}
		var visited bool

		// Close the body on function end
		defer res.Body.Close()

		// Any 2xx status code is good to go
		if res.StatusCode >= 200 && res.StatusCode < 300 {
			// Success, visit the URL
			harvested = this.visitUrl(ctx, res)
			visited = true
		} else {
			// Error based on status code received
			this.opts.Extender.Error(newCrawlErrorMessage(ctx, res.Status, CekHttpStatusCode))
			this.logFunc(LogError, "ERROR status code for %s: %s", ctx.url, res.Status)
		}
		this.sendResponse(ctx, visited, harvested, false)
	}
}

// Process the robots.txt URL.
func (this *worker) requestRobotsTxt(ctx *URLContext) {
	// Ask if it should be fetched
	if robData, reqRob := this.opts.Extender.RequestRobots(ctx, this.opts.RobotUserAgent); !reqRob {
		this.logFunc(LogInfo, "using robots.txt from cache")
		this.robotsGroup = this.getRobotsTxtGroup(ctx, robData, nil)

	} else {
		// Fetch the document, using the robot user agent,
		// so that the host admin can see what robots are doing requests.
		if res, ok := this.fetchUrl(ctx, this.opts.RobotUserAgent, false); ok {
			// Close the body on function end
			defer res.Body.Close()
			this.robotsGroup = this.getRobotsTxtGroup(ctx, nil, res)
		}
	}
}

// Get the robots.txt group for this crawler.
func (this *worker) getRobotsTxtGroup(ctx *URLContext, b []byte, res *http.Response) (g *robotstxt.Group) {
	var data *robotstxt.RobotsData
	var e error

	if res != nil {
		// Get the bytes from the response body
		b, e = ioutil.ReadAll(res.Body)
		// Rewind the res.Body (by re-creating it from the bytes)
		res.Body = ioutil.NopCloser(bytes.NewBuffer(b))
		// Error or not, the robots.txt has been fetched, so notify
		this.opts.Extender.FetchedRobots(ctx, res)
	}

	if e == nil {
		data, e = robotstxt.FromBytes(b)
	}

	// If robots data cannot be parsed, will return nil, which will allow access by default.
	// Reasonable, since by default no robots.txt means full access, so invalid
	// robots.txt is similar behavior.
	if e != nil {
		this.opts.Extender.Error(newCrawlError(nil, e, CekParseRobots))
		this.logFunc(LogError, "ERROR parsing robots.txt for host %s: %s", this.host, e)
	} else {
		g = data.FindGroup(this.opts.RobotUserAgent)
	}
	return
}

// Set the crawl delay between this request and the next.
func (this *worker) setCrawlDelay() {
	var robDelay time.Duration

	if this.robotsGroup != nil {
		robDelay = this.robotsGroup.CrawlDelay
	}
	this.lastCrawlDelay = this.opts.Extender.ComputeDelay(this.host,
		&DelayInfo{
			this.opts.CrawlDelay,
			robDelay,
			this.lastCrawlDelay,
		},
		this.lastFetch)
	this.logFunc(LogInfo, "using crawl-delay: %v", this.lastCrawlDelay)
}

// Request the specified URL and return the response.
func (this *worker) fetchUrl(ctx *URLContext, agent string, headRequest bool) (res *http.Response, ok bool) {
	var e error
	var silent bool

	for {
		// Wait for crawl delay, if one is pending.
		this.logFunc(LogTrace, "waiting for crawl delay")
		if this.wait != nil {
			<-this.wait
			this.wait = nil
		}

		// Compute the next delay
		this.setCrawlDelay()

		// Compute the fetch duration
		now := time.Now()

		// Request the URL
		if res, e = this.opts.Extender.Fetch(ctx, agent, headRequest); e != nil {
			// Check if this is an ErrEnqueueRedirect, in which case we will enqueue
			// the redirect-to URL.
			if ue, y := e.(*url.Error); y {
				// We have a *url.Error, check if it was returned because of an ErrEnqueueRedirect
				if ue.Err == ErrEnqueueRedirect {
					// Do not notify this error outside of this if block, this is not a
					// "real" error. We either enqueue the new URL, or fail to parse it,
					// and then stop processing the current URL.
					silent = true
					// Parse the URL in the context of the original URL (so that relative URLs are ok).
					// Absolute URLs that point to another host are ok too.
					if ur, e := ctx.url.Parse(ue.URL); e != nil {
						// Notify error
						this.opts.Extender.Error(newCrawlError(nil, e, CekParseRedirectURL))
						this.logFunc(LogError, "ERROR parsing redirect URL %s: %s", ue.URL, e)
					} else {
						// Enqueue the redirect-to URL
						this.logFunc(LogTrace, "redirect to %s", ur)
						this.enqueue <- ur
					}
				}
			}

			// No fetch, so set to nil
			this.lastFetch = nil

			if !silent {
				// Notify error
				this.opts.Extender.Error(newCrawlError(ctx, e, CekFetch))
				this.logFunc(LogError, "ERROR fetching %s: %s", ctx.url, e)
			}

			// Return from this URL crawl
			this.sendResponse(ctx, false, nil, false)
			return nil, false

		} else {
			// Get the fetch duration
			fetchDuration := now.Sub(time.Now())
			// Crawl delay starts now.
			this.wait = time.After(this.lastCrawlDelay)

			// Keep trace of this last fetch info
			this.lastFetch = &FetchInfo{
				ctx,
				fetchDuration,
				res.StatusCode,
				headRequest,
			}
		}

		if headRequest {
			// Close the HEAD request's body
			defer res.Body.Close()
			// Next up is GET request, maybe
			headRequest = false
			// Ask caller if we should proceed with a GET
			if !this.opts.Extender.RequestGet(ctx, res) {
				this.logFunc(LogIgnored, "ignored on HEAD filter policy: %s", ctx.url)
				this.sendResponse(ctx, false, nil, false)
				ok = false
				break
			}
		} else {
			ok = true
			break
		}
	}
	return
}

// Send a response to the crawler.
func (this *worker) sendResponse(ctx *URLContext, visited bool, harvested interface{}, idleDeath bool) {
	// Push harvested urls back to crawler, even if empty (uses the channel communication
	// to decrement reference count of pending URLs)
	if ctx == nil || !isRobotsURL(ctx.url) {
		// If a stop signal has been received, ignore the response, since the push
		// channel may be full and could block indefinitely.
		select {
		case <-this.stop:
			this.logFunc(LogInfo, "ignoring send response, will stop.")
			return
		default:
			// Nothing, just continue...
		}

		// No stop signal, send the response
		res := &workerResponse{
			ctx,
			visited,
			harvested,
			this.host,
			idleDeath,
		}
		this.push <- res
	}
}

// Process the response for a URL.
func (this *worker) visitUrl(ctx *URLContext, res *http.Response) interface{} {
	var doc *goquery.Document
	var harvested interface{}
	var doLinks bool

	// Load a goquery document and call the visitor function
	if bd, e := ioutil.ReadAll(res.Body); e != nil {
		this.opts.Extender.Error(newCrawlError(ctx, e, CekReadBody))
		this.logFunc(LogError, "ERROR reading body %s: %s", ctx.url, e)
	} else {
		if node, e := html.Parse(bytes.NewBuffer(bd)); e != nil {
			this.opts.Extender.Error(newCrawlError(ctx, e, CekParseBody))
			this.logFunc(LogError, "ERROR parsing %s: %s", ctx.url, e)
		} else {
			doc = goquery.NewDocumentFromNode(node)
			doc.Url = res.Request.URL
		}
		// Re-assign the body so it can be consumed by the visitor function
		res.Body = ioutil.NopCloser(bytes.NewBuffer(bd))
	}

	// Visit the document (with nil goquery doc if failed to load)
	if harvested, doLinks = this.opts.Extender.Visit(ctx, res, doc); doLinks {
		// Links were not processed by the visitor, so process links
		if doc != nil {
			harvested = this.processLinks(doc)
		} else {
			this.opts.Extender.Error(newCrawlErrorMessage(ctx, "No goquery document to process links.", CekProcessLinks))
			this.logFunc(LogError, "ERROR processing links %s", ctx.url)
		}
	}
	// Notify that this URL has been visited
	this.opts.Extender.Visited(ctx, harvested)

	return harvested
}

// Scrape the document's content to gather all links
func (this *worker) processLinks(doc *goquery.Document) (result []*url.URL) {
	baseUrl := doc.Find("base[href]").FirstChild
	urls := doc.Find("a[href]").Map(func(_ int, s *goquery.Selection) string {
		val, _ := s.Attr("href")
		if baseUrl != nil {
			val = baseUrl + val
		}
		return val
	})
	for _, s := range urls {
		// If href starts with "#", then it points to this same exact URL, ignore (will fail to parse anyway)
		if len(s) > 0 && !strings.HasPrefix(s, "#") {
			if parsed, e := url.Parse(s); e == nil {
				parsed = doc.Url.ResolveReference(parsed)
				result = append(result, parsed)
			} else {
				this.logFunc(LogIgnored, "ignore on unparsable policy %s: %s", s, e.Error())
			}
		}
	}
	return
}
<commit_msg>Properly get the base's href<commit_after>package gocrawl

import (
	"bytes"
	"io/ioutil"
	"net/http"
	"net/url"
	"strings"
	"sync"
	"time"

	"github.com/PuerkitoBio/goquery"
	"github.com/temoto/robotstxt.go"
	"golang.org/x/net/html"
)

// The worker is dedicated to fetching and visiting a given host, respecting
// this host's robots.txt crawling policies.
type worker struct {
	// Worker identification
	host  string
	index int

	// Communication channels and sync
	push    chan<- *workerResponse
	pop     popChannel
	stop    chan struct{}
	enqueue chan<- interface{}
	wg      *sync.WaitGroup

	// Robots validation
	robotsGroup *robotstxt.Group

	// Logging
	logFunc func(LogFlags, string, ...interface{})

	// Implementation fields
	wait           <-chan time.Time
	lastFetch      *FetchInfo
	lastCrawlDelay time.Duration
	opts           *Options
}

// Start crawling the host.
func (this *worker) run() {
	defer func() {
		this.logFunc(LogInfo, "worker done.")
		this.wg.Done()
	}()

	// Enter loop to process URLs until stop signal is received
	for {
		var idleChan <-chan time.Time

		this.logFunc(LogInfo, "waiting for pop...")

		// Initialize the idle timeout channel, if required
		if this.opts.WorkerIdleTTL > 0 {
			idleChan = time.After(this.opts.WorkerIdleTTL)
		}

		select {
		case <-this.stop:
			this.logFunc(LogInfo, "stop signal received.")
			return

		case <-idleChan:
			this.logFunc(LogInfo, "idle timeout received.")
			this.sendResponse(nil, false, nil, true)
			return

		case batch := <-this.pop:

			// Got a batch of urls to crawl, loop and check at each iteration if a stop
			// is received.
			for _, ctx := range batch {
				this.logFunc(LogInfo, "popped: %s", ctx.url)

				if ctx.IsRobotsURL() {
					this.requestRobotsTxt(ctx)
				} else if this.isAllowedPerRobotsPolicies(ctx.url) {
					this.requestUrl(ctx, ctx.HeadBeforeGet)
				} else {
					// Must still notify Crawler that this URL was processed, although not visited
					this.opts.Extender.Disallowed(ctx)
					this.sendResponse(ctx, false, nil, false)
				}

				// No need to check for idle timeout here, no idling while looping through
				// a batch of URLs.
				select {
				case <-this.stop:
					this.logFunc(LogInfo, "stop signal received.")
					return
				default:
					// Nothing, just continue...
				}
			}
		}
	}
}

// Checks if the given URL can be fetched based on robots.txt policies.
func (this *worker) isAllowedPerRobotsPolicies(u *url.URL) bool {
	if this.robotsGroup != nil {
		// Is this URL allowed per robots.txt policy?
		ok := this.robotsGroup.Test(u.Path)
		if !ok {
			this.logFunc(LogIgnored, "ignored on robots.txt policy: %s", u.String())
		}
		return ok
	}

	// No robots.txt = everything is allowed
	return true
}

// Process the specified URL.
func (this *worker) requestUrl(ctx *URLContext, headRequest bool) {
	if res, ok := this.fetchUrl(ctx, this.opts.UserAgent, headRequest); ok {
		var harvested interface{}
		var visited bool

		// Close the body on function end
		defer res.Body.Close()

		// Any 2xx status code is good to go
		if res.StatusCode >= 200 && res.StatusCode < 300 {
			// Success, visit the URL
			harvested = this.visitUrl(ctx, res)
			visited = true
		} else {
			// Error based on status code received
			this.opts.Extender.Error(newCrawlErrorMessage(ctx, res.Status, CekHttpStatusCode))
			this.logFunc(LogError, "ERROR status code for %s: %s", ctx.url, res.Status)
		}
		this.sendResponse(ctx, visited, harvested, false)
	}
}

// Process the robots.txt URL.
func (this *worker) requestRobotsTxt(ctx *URLContext) {
	// Ask if it should be fetched
	if robData, reqRob := this.opts.Extender.RequestRobots(ctx, this.opts.RobotUserAgent); !reqRob {
		this.logFunc(LogInfo, "using robots.txt from cache")
		this.robotsGroup = this.getRobotsTxtGroup(ctx, robData, nil)

	} else {
		// Fetch the document, using the robot user agent,
		// so that the host admin can see what robots are doing requests.
		if res, ok := this.fetchUrl(ctx, this.opts.RobotUserAgent, false); ok {
			// Close the body on function end
			defer res.Body.Close()
			this.robotsGroup = this.getRobotsTxtGroup(ctx, nil, res)
		}
	}
}

// Get the robots.txt group for this crawler.
func (this *worker) getRobotsTxtGroup(ctx *URLContext, b []byte, res *http.Response) (g *robotstxt.Group) {
	var data *robotstxt.RobotsData
	var e error

	if res != nil {
		// Get the bytes from the response body
		b, e = ioutil.ReadAll(res.Body)
		// Rewind the res.Body (by re-creating it from the bytes)
		res.Body = ioutil.NopCloser(bytes.NewBuffer(b))
		// Error or not, the robots.txt has been fetched, so notify
		this.opts.Extender.FetchedRobots(ctx, res)
	}

	if e == nil {
		data, e = robotstxt.FromBytes(b)
	}

	// If robots data cannot be parsed, will return nil, which will allow access by default.
	// Reasonable, since by default no robots.txt means full access, so invalid
	// robots.txt is similar behavior.
	if e != nil {
		this.opts.Extender.Error(newCrawlError(nil, e, CekParseRobots))
		this.logFunc(LogError, "ERROR parsing robots.txt for host %s: %s", this.host, e)
	} else {
		g = data.FindGroup(this.opts.RobotUserAgent)
	}
	return
}

// Set the crawl delay between this request and the next.
func (this *worker) setCrawlDelay() {
	var robDelay time.Duration

	if this.robotsGroup != nil {
		robDelay = this.robotsGroup.CrawlDelay
	}
	this.lastCrawlDelay = this.opts.Extender.ComputeDelay(this.host,
		&DelayInfo{
			this.opts.CrawlDelay,
			robDelay,
			this.lastCrawlDelay,
		},
		this.lastFetch)
	this.logFunc(LogInfo, "using crawl-delay: %v", this.lastCrawlDelay)
}

// Request the specified URL and return the response.
func (this *worker) fetchUrl(ctx *URLContext, agent string, headRequest bool) (res *http.Response, ok bool) {
	var e error
	var silent bool

	for {
		// Wait for crawl delay, if one is pending.
		this.logFunc(LogTrace, "waiting for crawl delay")
		if this.wait != nil {
			<-this.wait
			this.wait = nil
		}

		// Compute the next delay
		this.setCrawlDelay()

		// Compute the fetch duration
		now := time.Now()

		// Request the URL
		if res, e = this.opts.Extender.Fetch(ctx, agent, headRequest); e != nil {
			// Check if this is an ErrEnqueueRedirect, in which case we will enqueue
			// the redirect-to URL.
			if ue, y := e.(*url.Error); y {
				// We have a *url.Error, check if it was returned because of an ErrEnqueueRedirect
				if ue.Err == ErrEnqueueRedirect {
					// Do not notify this error outside of this if block, this is not a
					// "real" error. We either enqueue the new URL, or fail to parse it,
					// and then stop processing the current URL.
					silent = true
					// Parse the URL in the context of the original URL (so that relative URLs are ok).
					// Absolute URLs that point to another host are ok too.
					if ur, e := ctx.url.Parse(ue.URL); e != nil {
						// Notify error
						this.opts.Extender.Error(newCrawlError(nil, e, CekParseRedirectURL))
						this.logFunc(LogError, "ERROR parsing redirect URL %s: %s", ue.URL, e)
					} else {
						// Enqueue the redirect-to URL
						this.logFunc(LogTrace, "redirect to %s", ur)
						this.enqueue <- ur
					}
				}
			}

			// No fetch, so set to nil
			this.lastFetch = nil

			if !silent {
				// Notify error
				this.opts.Extender.Error(newCrawlError(ctx, e, CekFetch))
				this.logFunc(LogError, "ERROR fetching %s: %s", ctx.url, e)
			}

			// Return from this URL crawl
			this.sendResponse(ctx, false, nil, false)
			return nil, false

		} else {
			// Get the fetch duration
			fetchDuration := now.Sub(time.Now())
			// Crawl delay starts now.
			this.wait = time.After(this.lastCrawlDelay)

			// Keep trace of this last fetch info
			this.lastFetch = &FetchInfo{
				ctx,
				fetchDuration,
				res.StatusCode,
				headRequest,
			}
		}

		if headRequest {
			// Close the HEAD request's body
			defer res.Body.Close()
			// Next up is GET request, maybe
			headRequest = false
			// Ask caller if we should proceed with a GET
			if !this.opts.Extender.RequestGet(ctx, res) {
				this.logFunc(LogIgnored, "ignored on HEAD filter policy: %s", ctx.url)
				this.sendResponse(ctx, false, nil, false)
				ok = false
				break
			}
		} else {
			ok = true
			break
		}
	}
	return
}

// Send a response to the crawler.
func (this *worker) sendResponse(ctx *URLContext, visited bool, harvested interface{}, idleDeath bool) {
	// Push harvested urls back to crawler, even if empty (uses the channel communication
	// to decrement reference count of pending URLs)
	if ctx == nil || !isRobotsURL(ctx.url) {
		// If a stop signal has been received, ignore the response, since the push
		// channel may be full and could block indefinitely.
		select {
		case <-this.stop:
			this.logFunc(LogInfo, "ignoring send response, will stop.")
			return
		default:
			// Nothing, just continue...
		}

		// No stop signal, send the response
		res := &workerResponse{
			ctx,
			visited,
			harvested,
			this.host,
			idleDeath,
		}
		this.push <- res
	}
}

// Process the response for a URL.
func (this *worker) visitUrl(ctx *URLContext, res *http.Response) interface{} {
	var doc *goquery.Document
	var harvested interface{}
	var doLinks bool

	// Load a goquery document and call the visitor function
	if bd, e := ioutil.ReadAll(res.Body); e != nil {
		this.opts.Extender.Error(newCrawlError(ctx, e, CekReadBody))
		this.logFunc(LogError, "ERROR reading body %s: %s", ctx.url, e)
	} else {
		if node, e := html.Parse(bytes.NewBuffer(bd)); e != nil {
			this.opts.Extender.Error(newCrawlError(ctx, e, CekParseBody))
			this.logFunc(LogError, "ERROR parsing %s: %s", ctx.url, e)
		} else {
			doc = goquery.NewDocumentFromNode(node)
			doc.Url = res.Request.URL
		}
		// Re-assign the body so it can be consumed by the visitor function
		res.Body = ioutil.NopCloser(bytes.NewBuffer(bd))
	}

	// Visit the document (with nil goquery doc if failed to load)
	if harvested, doLinks = this.opts.Extender.Visit(ctx, res, doc); doLinks {
		// Links were not processed by the visitor, so process links
		if doc != nil {
			harvested = this.processLinks(doc)
		} else {
			this.opts.Extender.Error(newCrawlErrorMessage(ctx, "No goquery document to process links.", CekProcessLinks))
			this.logFunc(LogError, "ERROR processing links %s", ctx.url)
		}
	}
	// Notify that this URL has been visited
	this.opts.Extender.Visited(ctx, harvested)

	return harvested
}

// Scrape the document's content to gather all links
func (this *worker) processLinks(doc *goquery.Document) (result []*url.URL) {
	baseUrl, _ := doc.Find("base[href]").Attr("href")
	urls := doc.Find("a[href]").Map(func(_ int, s *goquery.Selection) string {
		val, _ := s.Attr("href")
		if baseUrl != nil {
			val = baseUrl + val
		}
		return val
	})
	for _, s := range urls {
		// If href starts with "#", then it points to this same exact URL, ignore (will fail to parse anyway)
		if len(s) > 0 && !strings.HasPrefix(s, "#") {
			if parsed, e := url.Parse(s); e == nil {
				parsed = doc.Url.ResolveReference(parsed)
				result = append(result, parsed)
			} else {
				this.logFunc(LogIgnored, "ignore on unparsable policy %s: %s", s, e.Error())
			}
		}
	}
	return
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"syscall"
	"time"

	"gopkg.in/yaml.v2"
	//"github.com/garyburd/redigo/redis"
	log "github.com/Sirupsen/logrus"

	"github.com/ArchCI/simple-worker/fileutil"
)

// The task struct to run test
type Task struct {
	Id      float64 `json:"id"`
	Commit  string  `json:"commit"`
	Public  bool    `json:"is_public"`
	Type    string  `json:"type"`
	Project string  `json:"project`
	Url     string  `json:"url"`
}

// Check if the worker can run task or not
func checkRequirement() bool {
	// TODO: it should not work for mac os but docker may be installed
	_, err := exec.LookPath("docker")
	if err != nil {
		return false
	} else {
		return true
	}
}

// Execute the command to replace current process. TODO: not used yet
func Exec(args []string) {
	path, err := exec.LookPath(args[0])
	if err != nil {
		os.Exit(1)
	}
	err = syscall.Exec(path, args, os.Environ())
	if err != nil {
		os.Exit(1)
	}
}

// Struct for archci.yml
type ArchciConfig struct {
	Image  string   `yaml:"image"`
	Script []string `yaml:"script"`
}

// Parse archci.yml to struct
func ParseYaml(filename string) ArchciConfig {
	// fmt.Println("Start parse yaml") // TODO: Make it as debug log

	var archciConfig ArchciConfig
	file, err := ioutil.ReadFile(filename)
	if err != nil {
		panic(err)
	}

	err = yaml.Unmarshal(file, &archciConfig)
	if err != nil {
		panic(err)
	}

	// fmt.Printf("Value: %#v\n", config.Script[0])
	return archciConfig
}

// Use archci.yml struct to generate archci.sh
func GenerateArchciShellContent(archciConfig ArchciConfig) string {
	// Add this and user's scripts into archci.sh
	baseShellContent := `#!/bin/bash
set -e
cd /project
`

	archciShellContent := baseShellContent
	for _, script := range archciConfig.Script {
		archciShellContent += script + "\n"
	}

	return archciShellContent
}

// The simple worker to pull task and run
func main() {
	fmt.Println("Start ArchCI simple worker")
	log.Info("Start simple-worker")

	// TODO: Support get parameter from command-line(server url, interval time and task number)

	// Exit if it doesn't meet the requirement
	if checkRequirement() == false {
		// TODO: Add more info to install requirement
		fmt.Println("Need some requirements, exit now")
		return
	}

	// Loop to pull task and run test
	for {

		// HTTP request to get task array
		// TODO: change to http://archci.com/tasks?number=1
		task := Task{Id: 123, Commit: "commit", Public: true, Type: "github", Project: "test-project", Url: "https://github.com/tobegit3hub/test-project.git"}
		tasks := []Task{task}

		// If no task, sleep and wait for next
		if len(tasks) == 0 {
			fmt.Println("Sleep 5 seconds then pull task again")
			time.Sleep(5 * time.Second)
			continue
		}

		// Get the task and run test
		task = tasks[0]
		fmt.Println(task.Project)

		// 1. Clone the code in specified directory
		// TODO: support user defined directory, avoid the name conflict
		// TODO: Support other command than "git clone"
		cloneCmd := exec.Command("git", "clone", task.Url)
		cloneOut, err := cloneCmd.Output()
		if err != nil {
			// TODO: Don't be so easy to exit
			os.Exit(1)
		}
		fmt.Println(string(cloneOut)) // Nothing to output if success
		fmt.Println("Success to clone the code")

		// 2. Parse archci.yaml file for base image and test scripts
		archciConfig := ParseYaml(task.Project + "/archci.yml")
		// fmt.Printf("Value: %#v\n", archciConfig.Image)
		dockerImage := archciConfig.Image
		fmt.Printf("Docker image: %#v\n", dockerImage)

		// 3. Generate archci.sh to "cd", combine user scipts and redirect STDOUT to file, this file should put into user's root directory
		// TODO: Make sure that the archci.sh is not conflict or just rm the user's one
		archciShellContent := GenerateArchciShellContent(archciConfig)
		archciShellFile, err2 := os.OpenFile(task.Project+"/archci.sh", os.O_RDWR|os.O_CREATE|os.O_EXCL, 0755) // TODO: Make it a constant
		if err2 != nil {
			panic(err2)
		}
		defer archciShellFile.Close()
		_, err2 = archciShellFile.WriteString(archciShellContent)
		if err2 != nil {
			panic(err2)
		}
		archciShellFile.Sync()
		archciShellFile.Close()
		fmt.Println("Success to create archci.sh")

		// 4. Docker run the base image and put the code into container to test
		// docker run --rm -v $PWD:/project golan:1.4 /project/archci.sh > docker.log 2>&1 ; echo $? > exit_code.log
		cpuLimit := "" // " -c 2 "
		memoryLimit := "" // " -m 100m "
		dockerCommand := "docker run --rm " + cpuLimit + memoryLimit + "-v $PWD/" + task.Project + ":/project " + dockerImage + " /project/archci.sh > docker.log 2>&1 ; echo $? > exit_code.log"
		dockerCmd := exec.Command("sh", "-c", dockerCommand)
		dockerOut, err := dockerCmd.Output()
		if err != nil {
			os.Exit(1)
		}
		fmt.Println(string(dockerOut))
		fmt.Println("Success to run " + dockerCommand)

		// 5. Non-block read the log and exit_code file and put them into redis
		fileutil.NonblockReadFile("docker.log");

		// 6. Delete the code
		// TODO: make it a function to call
		rmCmd := exec.Command("rm", "-rf", task.Project)
		rmOut, err := rmCmd.Output()
		if err != nil {
			// TODO: Don't be so easy to exit
			os.Exit(1)
		}
		fmt.Println(string(rmOut))
		fmt.Println("Success to delete the code")

		// Sleep for next task
		fmt.Println("Sleep 60 seconds for next task")
		time.Sleep(60 * time.Second)
	}

	fmt.Println("Simple worker exists")

}
<commit_msg>Support post data<commit_after>package main

import (
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"strings"
	"syscall"
	"time"

	"github.com/gorilla/http"

	"gopkg.in/yaml.v2"
	//"github.com/garyburd/redigo/redis"
	log "github.com/Sirupsen/logrus"

	"github.com/ArchCI/simple-worker/fileutil"
)

// The task struct to run test
type Task struct {
	Id      float64 `json:"id"`
	Commit  string  `json:"commit"`
	Public  bool    `json:"is_public"`
	Type    string  `json:"type"`
	Project string  `json:"project`
	Url     string  `json:"url"`
}

// Check if the worker can run task or not
func checkRequirement() bool {
	// TODO: it should not work for mac os but docker may be installed
	_, err := exec.LookPath("docker")
	if err != nil {
		return false
	} else {
		return true
	}
}

// Execute the command to replace current process. TODO: not used yet
func Exec(args []string) {
	path, err := exec.LookPath(args[0])
	if err != nil {
		os.Exit(1)
	}
	err = syscall.Exec(path, args, os.Environ())
	if err != nil {
		os.Exit(1)
	}
}

// Struct for archci.yml
type ArchciConfig struct {
	Image  string   `yaml:"image"`
	Script []string `yaml:"script"`
}

// Parse archci.yml to struct
func ParseYaml(filename string) ArchciConfig {
	// fmt.Println("Start parse yaml") // TODO: Make it as debug log

	var archciConfig ArchciConfig
	file, err := ioutil.ReadFile(filename)
	if err != nil {
		panic(err)
	}

	err = yaml.Unmarshal(file, &archciConfig)
	if err != nil {
		panic(err)
	}

	// fmt.Printf("Value: %#v\n", config.Script[0])
	return archciConfig
}

// Use archci.yml struct to generate archci.sh
func GenerateArchciShellContent(archciConfig ArchciConfig) string {
	// Add this and user's scripts into archci.sh
	baseShellContent := `#!/bin/bash
set -e
cd /project
`

	archciShellContent := baseShellContent
	for _, script := range archciConfig.Script {
		archciShellContent += script + "\n"
	}

	return archciShellContent
}

func PostString(url string, data string) {
	if err := http.Post(url, strings.NewReader(data)); err != nil {
		log.Fatal("could not post")
	}
}

// The simple worker to pull task and run
func main() {
	fmt.Println("Start ArchCI simple worker")

	log.Info("Start simple-worker")

	// TODO: Support get parameter from command-line(server url, interval time and task number)

	// Exit if it doesn't meet the requirement
	if checkRequirement() == false {
		// TODO: Add more info to install requirement
		fmt.Println("Need some requirements, exit now")
		return
	}

	// Loop to pull task and run test
	for {

		// HTTP request to get task array
		// TODO: change to http://archci.com/tasks?number=1
		task := Task{Id: 123, Commit: "commit", Public: true, Type: "github", Project: "test-project", Url: "https://github.com/tobegit3hub/test-project.git"}
		tasks := []Task{task}

		// If no task, sleep and wait for next
		if len(tasks) == 0 {
			fmt.Println("Sleep 5 seconds then pull task again")
			time.Sleep(5 * time.Second)
			continue
		}

		// Get the task and run test
		task = tasks[0]
		fmt.Println(task.Project)

		// 1. Clone the code in specified directory
		// TODO: support user defined directory, avoid the name conflict
		// TODO: Support other command than "git clone"
		cloneCmd := exec.Command("git", "clone", task.Url)
		cloneOut, err := cloneCmd.Output()
		if err != nil {
			// TODO: Don't be so easy to exit
			os.Exit(1)
		}
		fmt.Println(string(cloneOut)) // Nothing to output if success
		fmt.Println("Success to clone the code")

		// 2. Parse archci.yaml file for base image and test scripts
		archciConfig := ParseYaml(task.Project + "/archci.yml")
		// fmt.Printf("Value: %#v\n", archciConfig.Image)
		dockerImage := archciConfig.Image
		fmt.Printf("Docker image: %#v\n", dockerImage)

		// 3. Generate archci.sh to "cd", combine user scipts and redirect STDOUT to file, this file should put into user's root directory
		// TODO: Make sure that the archci.sh is not conflict or just rm the user's one
		archciShellContent := GenerateArchciShellContent(archciConfig)
		archciShellFile, err2 := os.OpenFile(task.Project+"/archci.sh", os.O_RDWR|os.O_CREATE|os.O_EXCL, 0755) // TODO: Make it a constant
		if err2 != nil {
			panic(err2)
		}
		defer archciShellFile.Close()
		_, err2 = archciShellFile.WriteString(archciShellContent)
		if err2 != nil {
			panic(err2)
		}
		archciShellFile.Sync()
		archciShellFile.Close()
		fmt.Println("Success to create archci.sh")

		// 4. Docker run the base image and put the code into container to test
		// docker run --rm -v $PWD:/project golan:1.4 /project/archci.sh > docker.log 2>&1 ; echo $? > exit_code.log
		cpuLimit := ""    // " -c 2 "
		memoryLimit := "" // " -m 100m "
		dockerCommand := "docker run --rm " + cpuLimit + memoryLimit + "-v $PWD/" + task.Project + ":/project " + dockerImage + " /project/archci.sh > docker.log 2>&1 ; echo $? > exit_code.log"
		dockerCmd := exec.Command("sh", "-c", dockerCommand)
		dockerOut, err := dockerCmd.Output()
		if err != nil {
			os.Exit(1)
		}
		fmt.Println(string(dockerOut))
		fmt.Println("Success to run " + dockerCommand)

		// 5. Non-block read the log and exit_code file and put them into redis
		fileutil.NonblockReadFile("docker.log")
		// PostString("http://127.0.0.1:8080/v1/account", "my log one")

		// 6. Delete the code
		// TODO: make it a function to call
		rmCmd := exec.Command("rm", "-rf", task.Project)
		rmOut, err := rmCmd.Output()
		if err != nil {
			// TODO: Don't be so easy to exit
			os.Exit(1)
		}
		fmt.Println(string(rmOut))
		fmt.Println("Success to delete the code")

		// Sleep for next task
		fmt.Println("Sleep 60 seconds for next task")
		time.Sleep(60 * time.Second)
	}

	fmt.Println("Simple worker exists")

}
<|endoftext|>


<commit_before>/*
Copyright 2014 Google Inc. All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package api

import (
	"encoding/json"
	"fmt"
	"reflect"

	"gopkg.in/v1/yaml"
)

var knownTypes = map[string]reflect.Type{}

func init() {
	AddKnownTypes(
		PodList{}, Pod{},
		ReplicationControllerList{}, ReplicationController{},
		ServiceList{}, Service{},
		Status{},
	)
}

func AddKnownTypes(types ...interface{}) {
	for _, obj := range types {
		t := reflect.TypeOf(obj)
		knownTypes[t.Name()] = t
	}
}

// Encode turns the given api object into an appropriate JSON string.
// Will return an error if the object doesn't have an embedded JSONBase.
// Obj may be a pointer to a struct, or a struct. If a struct, a copy
// will be made so that the object's Kind field can be set. If a pointer,
// we change the Kind field, marshal, and then set the kind field back to
// "". Having to keep track of the kind field makes tests very annoying,
// so the rule is it's set only in wire format (json), not when in native
// format.
func Encode(obj interface{}) (data []byte, err error) {
	obj = checkPtr(obj)
	fieldToReset, err := prepareEncode(obj)
	if err != nil {
		return nil, err
	}
	data, err = json.Marshal(obj)
	fieldToReset.SetString("")
	return
}

// Just like Encode, but produces indented output.
func EncodeIndent(obj interface{}) (data []byte, err error) {
	obj = checkPtr(obj)
	fieldToReset, err := prepareEncode(obj)
	if err != nil {
		return nil, err
	}
	data, err = json.MarshalIndent(obj, "", "	")
	fieldToReset.SetString("")
	return
}

func checkPtr(obj interface{}) interface{} {
	v := reflect.ValueOf(obj)
	if v.Kind() == reflect.Ptr {
		return obj
	}
	v2 := reflect.New(v.Type())
	v2.Elem().Set(v)
	return v2.Interface()
}

func prepareEncode(obj interface{}) (reflect.Value, error) {
	name, jsonBase, err := nameAndJSONBase(obj)
	if err != nil {
		return reflect.Value{}, err
	}
	if _, contains := knownTypes[name]; !contains {
		return reflect.Value{}, fmt.Errorf("struct %v won't be unmarshalable because it's not in knownTypes", name)
	}
	kind := jsonBase.FieldByName("Kind")
	kind.SetString(name)
	return kind, nil
}

// Returns the name of the type (sans pointer), and its kind field. Takes pointer-to-struct..
func nameAndJSONBase(obj interface{}) (string, reflect.Value, error) {
	v := reflect.ValueOf(obj)
	if v.Kind() != reflect.Ptr {
		return "", reflect.Value{}, fmt.Errorf("expected pointer, but got %v", v.Type().Name())
	}
	v = v.Elem()
	name := v.Type().Name()
	if v.Kind() != reflect.Struct {
		return "", reflect.Value{}, fmt.Errorf("expected struct, but got %v", name)
	}
	jsonBase := v.FieldByName("JSONBase")
	if !jsonBase.IsValid() {
		return "", reflect.Value{}, fmt.Errorf("struct %v lacks embedded JSON type", name)
	}
	return name, jsonBase, nil
}

// Decode converts a JSON string back into a pointer to an api object. Deduces the type
// based upon the Kind field (set by encode).
func Decode(data []byte) (interface{}, error) {
	findKind := struct {
		Kind string `json:"kind,omitempty" yaml:"kind,omitempty"`
	}{}
	// yaml is a superset of json, so we use it to decode here. That way, we understand both.
	err := yaml.Unmarshal(data, &findKind)
	if err != nil {
		return nil, fmt.Errorf("Couldn't get kind: %#v", err)
	}
	objType, found := knownTypes[findKind.Kind]
	if !found {
		return nil, fmt.Errorf("%v is not a known type", findKind.Kind)
	}
	obj := reflect.New(objType).Interface()
	err = yaml.Unmarshal(data, obj)
	if err != nil {
		return nil, err
	}
	_, jsonBase, err := nameAndJSONBase(obj)
	if err != nil {
		return nil, err
	}
	// Don't leave these set. Track type with go's type.
	jsonBase.FieldByName("Kind").SetString("")
	return obj, nil
}

// DecodeInto parses a JSON string and stores it in obj. Returns an error
// if data.Kind is set and doesn't match the type of obj. Obj should be a
// pointer to an api type.
func DecodeInto(data []byte, obj interface{}) error {
	err := yaml.Unmarshal(data, obj)
	if err != nil {
		return err
	}
	name, jsonBase, err := nameAndJSONBase(obj)
	if err != nil {
		return err
	}
	foundName := jsonBase.FieldByName("Kind").Interface().(string)
	if foundName != "" && foundName != name {
		return fmt.Errorf("data had kind %v, but passed object was of type %v", foundName, name)
	}
	// Don't leave these set. Track type with go's type.
	jsonBase.FieldByName("Kind").SetString("")
	return nil
}
<commit_msg>Address comments; Also internally start using JSONBase instead of reflect.Value of JSONBase.<commit_after>/*
Copyright 2014 Google Inc. All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package api

import (
	"encoding/json"
	"fmt"
	"reflect"

	"gopkg.in/v1/yaml"
)

var knownTypes = map[string]reflect.Type{}

func init() {
	AddKnownTypes(
		PodList{},
		Pod{},
		ReplicationControllerList{},
		ReplicationController{},
		ServiceList{},
		Service{},
		Status{},
	)
}

func AddKnownTypes(types ...interface{}) {
	for _, obj := range types {
		t := reflect.TypeOf(obj)
		knownTypes[t.Name()] = t
	}
}

// Encode turns the given api object into an appropriate JSON string.
// Will return an error if the object doesn't have an embedded JSONBase.
// Obj may be a pointer to a struct, or a struct. If a struct, a copy
// will be made so that the object's Kind field can be set. If a pointer,
// we change the Kind field, marshal, and then set the kind field back to
// "". Having to keep track of the kind field makes tests very annoying,
// so the rule is it's set only in wire format (json), not when in native
// format.
func Encode(obj interface{}) (data []byte, err error) {
	obj = checkPtr(obj)
	jsonBase, err := prepareEncode(obj)
	if err != nil {
		return nil, err
	}
	data, err = json.MarshalIndent(obj, "", "	")
	jsonBase.Kind = ""
	return data, err
}

func checkPtr(obj interface{}) interface{} {
	v := reflect.ValueOf(obj)
	if v.Kind() == reflect.Ptr {
		return obj
	}
	v2 := reflect.New(v.Type())
	v2.Elem().Set(v)
	return v2.Interface()
}

func prepareEncode(obj interface{}) (*JSONBase, error) {
	name, jsonBase, err := nameAndJSONBase(obj)
	if err != nil {
		return nil, err
	}
	if _, contains := knownTypes[name]; !contains {
		return nil, fmt.Errorf("struct %v won't be unmarshalable because it's not in knownTypes", name)
	}
	jsonBase.Kind = name
	return jsonBase, nil
}

// Returns the name of the type (sans pointer), and its kind field. Takes pointer-to-struct..
func nameAndJSONBase(obj interface{}) (string, *JSONBase, error) {
	v := reflect.ValueOf(obj)
	if v.Kind() != reflect.Ptr {
		return "", nil, fmt.Errorf("expected pointer, but got %v", v.Type().Name())
	}
	v = v.Elem()
	name := v.Type().Name()
	if v.Kind() != reflect.Struct {
		return "", nil, fmt.Errorf("expected struct, but got %v", name)
	}
	jsonBase := v.FieldByName("JSONBase")
	if !jsonBase.IsValid() {
		return "", nil, fmt.Errorf("struct %v lacks embedded JSON type", name)
	}
	return name, jsonBase.Addr().Interface().(*JSONBase), nil
}

// Decode converts a JSON string back into a pointer to an api object. Deduces the type
// based upon the Kind field (set by encode).
func Decode(data []byte) (interface{}, error) {
	findKind := struct {
		Kind string `json:"kind,omitempty" yaml:"kind,omitempty"`
	}{}
	// yaml is a superset of json, so we use it to decode here. That way, we understand both.
	err := yaml.Unmarshal(data, &findKind)
	if err != nil {
		return nil, fmt.Errorf("Couldn't get kind: %#v", err)
	}
	objType, found := knownTypes[findKind.Kind]
	if !found {
		return nil, fmt.Errorf("%v is not a known type", findKind.Kind)
	}
	obj := reflect.New(objType).Interface()
	err = yaml.Unmarshal(data, obj)
	if err != nil {
		return nil, err
	}
	_, jsonBase, err := nameAndJSONBase(obj)
	if err != nil {
		return nil, err
	}
	// Don't leave these set. Track type with go's type.
	jsonBase.Kind = ""
	return obj, nil
}

// DecodeInto parses a JSON string and stores it in obj. Returns an error
// if data.Kind is set and doesn't match the type of obj. Obj should be a
// pointer to an api type.
func DecodeInto(data []byte, obj interface{}) error {
	err := yaml.Unmarshal(data, obj)
	if err != nil {
		return err
	}
	name, jsonBase, err := nameAndJSONBase(obj)
	if err != nil {
		return err
	}
	if jsonBase.Kind != "" && jsonBase.Kind != name {
		return fmt.Errorf("data had kind %v, but passed object was of type %v", jsonBase.Kind, name)
	}
	// Don't leave these set. Track type with go's type.
	jsonBase.Kind = ""
	return nil
}
<|endoftext|>


<commit_before>package client

import (
	"encoding/xml"
	"fmt"
)

type ConsoleButtonType int

const (
	ConsleButtonTypeDestination ConsoleButtonType = 1
	ConsoleButtonTypeSource                       = 4
	ConsoleButtonTypeUserKey                      = 6
	ConsoleButtonTypePreset                       = 7
)

func (value ConsoleButtonType) String() string {
	switch (value) {
	case ConsleButtonTypeDestination:
		return "destination"
	case ConsoleButtonTypeSource:
		return "source"
	case ConsoleButtonTypeUserKey:
		return "user-key"
	case ConsoleButtonTypePreset:
		return "preset"
	default:
		return fmt.Sprintf("%d", value)
	}
}

func (value ConsoleButtonType) MarshalJSON() ([]byte, error) {
	return marshalJSONString(value)
}

type ConsoleButton struct {
	ID                     int		`xml:"id,attr"`
	ConsoleButtonType      ConsoleButtonType
	ConsoleButtonTypeIndex int
}

// Presets
type PresetBusColl map[int]ConsoleButton

func (col *PresetBusColl) UnmarshalXML(d *xml.Decoder, e xml.StartElement) error {
	return unmarshalXMLCol(col, d, e)
}

func (col PresetBusColl) MarshalJSON() ([]byte, error) {
	return marshalJSONMap(col)
}

// UserKeys
type UserKeyBusColl map[int]ConsoleButton

func (col *UserKeyBusColl) UnmarshalXML(d *xml.Decoder, e xml.StartElement) error {
	return unmarshalXMLCol(col, d, e)
}

func (col UserKeyBusColl) MarshalJSON() ([]byte, error) {
	return marshalJSONMap(col)
}

// Destinations
type DestinationBusColl map[int]ConsoleButton

func (col *DestinationBusColl) UnmarshalXML(d *xml.Decoder, e xml.StartElement) error {
	return unmarshalXMLCol(col, d, e)
}

func (col DestinationBusColl) MarshalJSON() ([]byte, error) {
	return marshalJSONMap(col)
}

// InputSource
type InputSourceBusColl map[int]ConsoleButton

func (col *InputSourceBusColl) UnmarshalXML(d *xml.Decoder, e xml.StartElement) error {
	return unmarshalXMLCol(col, d, e)
}

func (col InputSourceBusColl) MarshalJSON() ([]byte, error) {
	return marshalJSONMap(col)
}

type ConsoleLayout struct {
	PresetBusColl	   PresetBusColl
	UserKeyBusColl     UserKeyBusColl
	DestinationBusColl DestinationBusColl
	InputSourceBusColl InputSourceBusColl
}

type ConsoleLayoutMgr struct {
	ConsoleLayout ConsoleLayout
}
<commit_msg>client: ConsoleButton.ID is JSON .id<commit_after>package client

import (
	"encoding/xml"
	"fmt"
)

type ConsoleButtonType int

const (
	ConsleButtonTypeDestination ConsoleButtonType = 1
	ConsoleButtonTypeSource                       = 4
	ConsoleButtonTypeUserKey                      = 6
	ConsoleButtonTypePreset                       = 7
)

func (value ConsoleButtonType) String() string {
	switch (value) {
	case ConsleButtonTypeDestination:
		return "destination"
	case ConsoleButtonTypeSource:
		return "source"
	case ConsoleButtonTypeUserKey:
		return "user-key"
	case ConsoleButtonTypePreset:
		return "preset"
	default:
		return fmt.Sprintf("%d", value)
	}
}

func (value ConsoleButtonType) MarshalJSON() ([]byte, error) {
	return marshalJSONString(value)
}

type ConsoleButton struct {
	ID                     int		`xml:"id,attr" json:"id"`
	ConsoleButtonType      ConsoleButtonType
	ConsoleButtonTypeIndex int
}

// Presets
type PresetBusColl map[int]ConsoleButton

func (col *PresetBusColl) UnmarshalXML(d *xml.Decoder, e xml.StartElement) error {
	return unmarshalXMLCol(col, d, e)
}

func (col PresetBusColl) MarshalJSON() ([]byte, error) {
	return marshalJSONMap(col)
}

// UserKeys
type UserKeyBusColl map[int]ConsoleButton

func (col *UserKeyBusColl) UnmarshalXML(d *xml.Decoder, e xml.StartElement) error {
	return unmarshalXMLCol(col, d, e)
}

func (col UserKeyBusColl) MarshalJSON() ([]byte, error) {
	return marshalJSONMap(col)
}

// Destinations
type DestinationBusColl map[int]ConsoleButton

func (col *DestinationBusColl) UnmarshalXML(d *xml.Decoder, e xml.StartElement) error {
	return unmarshalXMLCol(col, d, e)
}

func (col DestinationBusColl) MarshalJSON() ([]byte, error) {
	return marshalJSONMap(col)
}

// InputSource
type InputSourceBusColl map[int]ConsoleButton

func (col *InputSourceBusColl) UnmarshalXML(d *xml.Decoder, e xml.StartElement) error {
	return unmarshalXMLCol(col, d, e)
}

func (col InputSourceBusColl) MarshalJSON() ([]byte, error) {
	return marshalJSONMap(col)
}

type ConsoleLayout struct {
	PresetBusColl	   PresetBusColl
	UserKeyBusColl     UserKeyBusColl
	DestinationBusColl DestinationBusColl
	InputSourceBusColl InputSourceBusColl
}

type ConsoleLayoutMgr struct {
	ConsoleLayout ConsoleLayout
}
<|endoftext|>


<commit_before>package client

import (
	"encoding/json"
	"fmt"
	"net/http"

	"github.com/julienschmidt/httprouter"
	"github.com/ory/herodot"
	"github.com/ory/hydra/firewall"
	"github.com/ory/hydra/rand/sequence"
	"github.com/ory/ladon"
	"github.com/pkg/errors"
)

type Handler struct {
	Manager Manager
	H       herodot.Writer
	W       firewall.Firewall
}

const (
	ClientsHandlerPath = "/clients"
)

const (
	ClientsResource = "rn:hydra:clients"
	ClientResource  = "rn:hydra:clients:%s"
	Scope           = "hydra.clients"
)

func (h *Handler) SetRoutes(r *httprouter.Router) {
	r.GET(ClientsHandlerPath, h.List)
	r.POST(ClientsHandlerPath, h.Create)
	r.GET(ClientsHandlerPath+"/:id", h.Get)
	r.PUT(ClientsHandlerPath+"/:id", h.Update)
	r.DELETE(ClientsHandlerPath+"/:id", h.Delete)
}

// swagger:route POST /clients oauth2 clients createOAuthClient
//
// Creates an OAuth 2.0 Client
//
// Be aware that an OAuth 2.0 Client may gain highly priviledged access if configured that way. This
// endpoint should be well protected and only called by code you trust.
//
// The subject making the request needs to be assigned to a policy containing:
//
//  ```
//  {
//    "resources": ["rn:hydra:clients"],
//    "actions": ["create"],
//    "effect": "allow"
//  }
//  ```
//
//  Additionally, the context key "owner" is set to the owner of the client, allowing policies such as:
//
//  ```
//  {
//    "resources": ["rn:hydra:clients"],
//    "actions": ["create"],
//    "effect": "allow",
//    "conditions": { "owner": { "type": "EqualsSubjectCondition" } }
//  }
//  ```
//
//     Consumes:
//     - application/json
//
//     Produces:
//     - application/json
//
//     Schemes: http, https
//
//     Security:
//       oauth2: hydra.clients
//
//     Responses:
//       200: oauthClient
//       401: genericError
//       403: genericError
//       500: genericError
func (h *Handler) Create(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
	var c Client
	var ctx = r.Context()

	if err := json.NewDecoder(r.Body).Decode(&c); err != nil {
		h.H.WriteError(w, r, errors.WithStack(err))
		return
	}

	if _, err := h.W.TokenAllowed(ctx, h.W.TokenFromRequest(r), &firewall.TokenAccessRequest{
		Resource: ClientsResource,
		Action:   "create",
		Context: map[string]interface{}{
			"owner": c.Owner,
		},
	}, Scope); err != nil {
		h.H.WriteError(w, r, err)
		return
	}

	if len(c.Secret) == 0 {
		secret, err := sequence.RuneSequence(12, []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890_-.~"))
		if err != nil {
			h.H.WriteError(w, r, errors.WithStack(err))
			return
		}
		c.Secret = string(secret)
	} else if len(c.Secret) < 6 {
		h.H.WriteError(w, r, errors.New("The client secret must be at least 6 characters long"))
	}

	secret := c.Secret
	if err := h.Manager.CreateClient(&c); err != nil {
		h.H.WriteError(w, r, err)
		return
	}

	c.Secret = secret
	h.H.WriteCreated(w, r, ClientsHandlerPath+"/"+c.GetID(), &c)
}

// swagger:route PUT /clients/{id} oauth2 clients updateOAuthClient
//
// Updates an OAuth 2.0 Client
//
// Be aware that an OAuth 2.0 Client may gain highly priviledged access if configured that way. This
// endpoint should be well protected and only called by code you trust.
//
// The subject making the request needs to be assigned to a policy containing:
//
//  ```
//  {
//    "resources": ["rn:hydra:clients"],
//    "actions": ["update"],
//    "effect": "allow"
//  }
//  ```
//
//  Additionally, the context key "owner" is set to the owner of the client, allowing policies such as:
//
//  ```
//  {
//    "resources": ["rn:hydra:clients"],
//    "actions": ["update"],
//    "effect": "allow",
//    "conditions": { "owner": { "type": "EqualsSubjectCondition" } }
//  }
//  ```
//
//     Consumes:
//     - application/json
//
//     Produces:
//     - application/json
//
//     Schemes: http, https
//
//     Security:
//       oauth2: hydra.clients
//
//     Responses:
//       200: oauthClient
//       401: genericError
//       403: genericError
//       500: genericError
func (h *Handler) Update(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
	var c Client
	var ctx = r.Context()

	if err := json.NewDecoder(r.Body).Decode(&c); err != nil {
		h.H.WriteError(w, r, errors.WithStack(err))
		return
	}

	o, err := h.Manager.GetConcreteClient(ps.ByName("id"))
	if err != nil {
		h.H.WriteError(w, r, err)
		return
	}

	if _, err := h.W.TokenAllowed(ctx, h.W.TokenFromRequest(r), &firewall.TokenAccessRequest{
		Resource: ClientsResource,
		Action:   "update",
		Context: ladon.Context{
			"owner": o.Owner,
		},
	}, Scope); err != nil {
		h.H.WriteError(w, r, err)
		return
	}

	if len(c.Secret) > 0 && len(c.Secret) < 6 {
		h.H.WriteError(w, r, errors.New("The client secret must be at least 6 characters long"))
	}

	c.ID = ps.ByName("id")
	if err := h.Manager.UpdateClient(&c); err != nil {
		h.H.WriteError(w, r, err)
		return
	}

	h.H.WriteCreated(w, r, ClientsHandlerPath+"/"+c.GetID(), &c)
}

// swagger:route GET /clients oauth2 clients listOAuthClients
//
// Lists OAuth 2.0 Clients
//
// Never returns a client's secret.
//
// The subject making the request needs to be assigned to a policy containing:
//
// ```
// {
//   "resources": ["rn:hydra:clients"],
//   "actions": ["get"],
//   "effect": "allow"
// }
// ```
//
//     Consumes:
//     - application/json
//
//     Produces:
//     - application/json
//
//     Schemes: http, https
//
//     Security:
//       oauth2: hydra.clients
//
//     Responses:
//       200: clientsList
//       401: genericError
//       403: genericError
//       500: genericError
func (h *Handler) List(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
	var ctx = r.Context()

	if _, err := h.W.TokenAllowed(ctx, h.W.TokenFromRequest(r), &firewall.TokenAccessRequest{
		Resource: ClientsResource,
		Action:   "get",
	}, Scope); err != nil {
		h.H.WriteError(w, r, err)
		return
	}

	c, err := h.Manager.GetClients()
	if err != nil {
		h.H.WriteError(w, r, err)
		return
	}

	for k, cc := range c {
		cc.Secret = ""
		c[k] = cc
	}

	h.H.Write(w, r, c)
}

// swagger:route GET /clients/{id} oauth2 clients getOAuthClient
//
// Fetches an OAuth 2.0 Client.
//
// Never returns the client's secret.
//
// The subject making the request needs to be assigned to a policy containing:
//
//  ```
//  {
//    "resources": ["rn:hydra:clients:<some-id>"],
//    "actions": ["get"],
//    "effect": "allow"
//  }
//  ```
//
//  Additionally, the context key "owner" is set to the owner of the client, allowing policies such as:
//
//  ```
//  {
//    "resources": ["rn:hydra:clients:<some-id>"],
//    "actions": ["get"],
//    "effect": "allow",
//    "conditions": { "owner": { "type": "EqualsSubjectCondition" } }
//  }
//  ```
//
//     Consumes:
//     - application/json
//
//     Produces:
//     - application/json
//
//     Schemes: http, https
//
//     Security:
//       oauth2: hydra.clients
//
//     Responses:
//       200: oauthClient
//       401: genericError
//       403: genericError
//       500: genericError
func (h *Handler) Get(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
	var ctx = r.Context()
	var id = ps.ByName("id")

	c, err := h.Manager.GetConcreteClient(id)
	if err != nil {
		h.H.WriteError(w, r, err)
		return
	}

	if _, err := h.W.TokenAllowed(ctx, h.W.TokenFromRequest(r), &firewall.TokenAccessRequest{
		Resource: fmt.Sprintf(ClientResource, id),
		Action:   "get",
		Context: ladon.Context{
			"owner": c.GetOwner(),
		},
	}, Scope); err != nil {
		h.H.WriteError(w, r, err)
		return
	}

	c.Secret = ""
	h.H.Write(w, r, c)
}

// swagger:route DELETE /clients/{id} oauth2 clients deleteOAuthClient
//
// Deletes an OAuth 2.0 Client
//
// The subject making the request needs to be assigned to a policy containing:
//
//  ```
//  {
//    "resources": ["rn:hydra:clients:<some-id>"],
//    "actions": ["delete"],
//    "effect": "allow"
//  }
//  ```
//
//  Additionally, the context key "owner" is set to the owner of the client, allowing policies such as:
//
//  ```
//  {
//    "resources": ["rn:hydra:clients:<some-id>"],
//    "actions": ["delete"],
//    "effect": "allow",
//    "conditions": { "owner": { "type": "EqualsSubjectCondition" } }
//  }
//  ```
//
//     Consumes:
//     - application/json
//
//     Produces:
//     - application/json
//
//     Schemes: http, https
//
//     Security:
//       oauth2: hydra.clients
//
//     Responses:
//       204: emptyResponse
//       401: genericError
//       403: genericError
//       500: genericError
func (h *Handler) Delete(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
	var ctx = r.Context()
	var id = ps.ByName("id")

	c, err := h.Manager.GetConcreteClient(id)
	if err != nil {
		h.H.WriteError(w, r, err)
		return
	}

	if _, err := h.W.TokenAllowed(ctx, h.W.TokenFromRequest(r), &firewall.TokenAccessRequest{
		Resource: fmt.Sprintf(ClientResource, id),
		Action:   "delete",
		Context: ladon.Context{
			"owner": c.GetOwner(),
		},
	}, Scope); err != nil {
		h.H.WriteError(w, r, err)
		return
	}

	if err := h.Manager.DeleteClient(id); err != nil {
		h.H.WriteError(w, r, err)
		return
	}

	w.WriteHeader(http.StatusNoContent)
}
<commit_msg>client: appropriately handle client secret responses<commit_after>package client

import (
	"encoding/json"
	"fmt"
	"net/http"

	"github.com/julienschmidt/httprouter"
	"github.com/ory/herodot"
	"github.com/ory/hydra/firewall"
	"github.com/ory/hydra/rand/sequence"
	"github.com/ory/ladon"
	"github.com/pkg/errors"
)

type Handler struct {
	Manager Manager
	H       herodot.Writer
	W       firewall.Firewall
}

const (
	ClientsHandlerPath = "/clients"
)

const (
	ClientsResource = "rn:hydra:clients"
	ClientResource  = "rn:hydra:clients:%s"
	Scope           = "hydra.clients"
)

func (h *Handler) SetRoutes(r *httprouter.Router) {
	r.GET(ClientsHandlerPath, h.List)
	r.POST(ClientsHandlerPath, h.Create)
	r.GET(ClientsHandlerPath+"/:id", h.Get)
	r.PUT(ClientsHandlerPath+"/:id", h.Update)
	r.DELETE(ClientsHandlerPath+"/:id", h.Delete)
}

// swagger:route POST /clients oauth2 clients createOAuthClient
//
// Creates an OAuth 2.0 Client
//
// Be aware that an OAuth 2.0 Client may gain highly priviledged access if configured that way. This
// endpoint should be well protected and only called by code you trust.
//
// The subject making the request needs to be assigned to a policy containing:
//
//  ```
//  {
//    "resources": ["rn:hydra:clients"],
//    "actions": ["create"],
//    "effect": "allow"
//  }
//  ```
//
//  Additionally, the context key "owner" is set to the owner of the client, allowing policies such as:
//
//  ```
//  {
//    "resources": ["rn:hydra:clients"],
//    "actions": ["create"],
//    "effect": "allow",
//    "conditions": { "owner": { "type": "EqualsSubjectCondition" } }
//  }
//  ```
//
//     Consumes:
//     - application/json
//
//     Produces:
//     - application/json
//
//     Schemes: http, https
//
//     Security:
//       oauth2: hydra.clients
//
//     Responses:
//       200: oauthClient
//       401: genericError
//       403: genericError
//       500: genericError
func (h *Handler) Create(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
	var c Client
	var ctx = r.Context()

	if err := json.NewDecoder(r.Body).Decode(&c); err != nil {
		h.H.WriteError(w, r, errors.WithStack(err))
		return
	}

	if _, err := h.W.TokenAllowed(ctx, h.W.TokenFromRequest(r), &firewall.TokenAccessRequest{
		Resource: ClientsResource,
		Action:   "create",
		Context: map[string]interface{}{
			"owner": c.Owner,
		},
	}, Scope); err != nil {
		h.H.WriteError(w, r, err)
		return
	}

	if len(c.Secret) == 0 {
		secret, err := sequence.RuneSequence(12, []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890_-.~"))
		if err != nil {
			h.H.WriteError(w, r, errors.WithStack(err))
			return
		}
		c.Secret = string(secret)
	} else if len(c.Secret) < 6 {
		h.H.WriteError(w, r, errors.New("The client secret must be at least 6 characters long"))
	}

	secret := c.Secret
	if err := h.Manager.CreateClient(&c); err != nil {
		h.H.WriteError(w, r, err)
		return
	}

	c.Secret = secret
	h.H.WriteCreated(w, r, ClientsHandlerPath+"/"+c.GetID(), &c)
}

// swagger:route PUT /clients/{id} oauth2 clients updateOAuthClient
//
// Updates an OAuth 2.0 Client
//
// Be aware that an OAuth 2.0 Client may gain highly priviledged access if configured that way. This
// endpoint should be well protected and only called by code you trust.
//
// The subject making the request needs to be assigned to a policy containing:
//
//  ```
//  {
//    "resources": ["rn:hydra:clients"],
//    "actions": ["update"],
//    "effect": "allow"
//  }
//  ```
//
//  Additionally, the context key "owner" is set to the owner of the client, allowing policies such as:
//
//  ```
//  {
//    "resources": ["rn:hydra:clients"],
//    "actions": ["update"],
//    "effect": "allow",
//    "conditions": { "owner": { "type": "EqualsSubjectCondition" } }
//  }
//  ```
//
//     Consumes:
//     - application/json
//
//     Produces:
//     - application/json
//
//     Schemes: http, https
//
//     Security:
//       oauth2: hydra.clients
//
//     Responses:
//       200: oauthClient
//       401: genericError
//       403: genericError
//       500: genericError
func (h *Handler) Update(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
	var c Client
	var ctx = r.Context()

	if err := json.NewDecoder(r.Body).Decode(&c); err != nil {
		h.H.WriteError(w, r, errors.WithStack(err))
		return
	}

	o, err := h.Manager.GetConcreteClient(ps.ByName("id"))
	if err != nil {
		h.H.WriteError(w, r, err)
		return
	}

	if _, err := h.W.TokenAllowed(ctx, h.W.TokenFromRequest(r), &firewall.TokenAccessRequest{
		Resource: ClientsResource,
		Action:   "update",
		Context: ladon.Context{
			"owner": o.Owner,
		},
	}, Scope); err != nil {
		h.H.WriteError(w, r, err)
		return
	}

	var secret string
	if len(c.Secret) > 0 && len(c.Secret) < 6 {
		h.H.WriteError(w, r, errors.New("The client secret must be at least 6 characters long"))
		return
	} else {
		secret = c.Secret
	}

	c.ID = ps.ByName("id")
	if err := h.Manager.UpdateClient(&c); err != nil {
		h.H.WriteError(w, r, err)
		return
	}

	c.Secret = secret
	h.H.WriteCreated(w, r, ClientsHandlerPath+"/"+c.GetID(), &c)
}

// swagger:route GET /clients oauth2 clients listOAuthClients
//
// Lists OAuth 2.0 Clients
//
// Never returns a client's secret.
//
// The subject making the request needs to be assigned to a policy containing:
//
// ```
// {
//   "resources": ["rn:hydra:clients"],
//   "actions": ["get"],
//   "effect": "allow"
// }
// ```
//
//     Consumes:
//     - application/json
//
//     Produces:
//     - application/json
//
//     Schemes: http, https
//
//     Security:
//       oauth2: hydra.clients
//
//     Responses:
//       200: clientsList
//       401: genericError
//       403: genericError
//       500: genericError
func (h *Handler) List(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
	var ctx = r.Context()

	if _, err := h.W.TokenAllowed(ctx, h.W.TokenFromRequest(r), &firewall.TokenAccessRequest{
		Resource: ClientsResource,
		Action:   "get",
	}, Scope); err != nil {
		h.H.WriteError(w, r, err)
		return
	}

	c, err := h.Manager.GetClients()
	if err != nil {
		h.H.WriteError(w, r, err)
		return
	}

	for k, cc := range c {
		cc.Secret = ""
		c[k] = cc
	}

	h.H.Write(w, r, c)
}

// swagger:route GET /clients/{id} oauth2 clients getOAuthClient
//
// Fetches an OAuth 2.0 Client.
//
// Never returns the client's secret.
//
// The subject making the request needs to be assigned to a policy containing:
//
//  ```
//  {
//    "resources": ["rn:hydra:clients:<some-id>"],
//    "actions": ["get"],
//    "effect": "allow"
//  }
//  ```
//
//  Additionally, the context key "owner" is set to the owner of the client, allowing policies such as:
//
//  ```
//  {
//    "resources": ["rn:hydra:clients:<some-id>"],
//    "actions": ["get"],
//    "effect": "allow",
//    "conditions": { "owner": { "type": "EqualsSubjectCondition" } }
//  }
//  ```
//
//     Consumes:
//     - application/json
//
//     Produces:
//     - application/json
//
//     Schemes: http, https
//
//     Security:
//       oauth2: hydra.clients
//
//     Responses:
//       200: oauthClient
//       401: genericError
//       403: genericError
//       500: genericError
func (h *Handler) Get(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
	var ctx = r.Context()
	var id = ps.ByName("id")

	c, err := h.Manager.GetConcreteClient(id)
	if err != nil {
		h.H.WriteError(w, r, err)
		return
	}

	if _, err := h.W.TokenAllowed(ctx, h.W.TokenFromRequest(r), &firewall.TokenAccessRequest{
		Resource: fmt.Sprintf(ClientResource, id),
		Action:   "get",
		Context: ladon.Context{
			"owner": c.GetOwner(),
		},
	}, Scope); err != nil {
		h.H.WriteError(w, r, err)
		return
	}

	c.Secret = ""
	h.H.Write(w, r, c)
}

// swagger:route DELETE /clients/{id} oauth2 clients deleteOAuthClient
//
// Deletes an OAuth 2.0 Client
//
// The subject making the request needs to be assigned to a policy containing:
//
//  ```
//  {
//    "resources": ["rn:hydra:clients:<some-id>"],
//    "actions": ["delete"],
//    "effect": "allow"
//  }
//  ```
//
//  Additionally, the context key "owner" is set to the owner of the client, allowing policies such as:
//
//  ```
//  {
//    "resources": ["rn:hydra:clients:<some-id>"],
//    "actions": ["delete"],
//    "effect": "allow",
//    "conditions": { "owner": { "type": "EqualsSubjectCondition" } }
//  }
//  ```
//
//     Consumes:
//     - application/json
//
//     Produces:
//     - application/json
//
//     Schemes: http, https
//
//     Security:
//       oauth2: hydra.clients
//
//     Responses:
//       204: emptyResponse
//       401: genericError
//       403: genericError
//       500: genericError
func (h *Handler) Delete(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
	var ctx = r.Context()
	var id = ps.ByName("id")

	c, err := h.Manager.GetConcreteClient(id)
	if err != nil {
		h.H.WriteError(w, r, err)
		return
	}

	if _, err := h.W.TokenAllowed(ctx, h.W.TokenFromRequest(r), &firewall.TokenAccessRequest{
		Resource: fmt.Sprintf(ClientResource, id),
		Action:   "delete",
		Context: ladon.Context{
			"owner": c.GetOwner(),
		},
	}, Scope); err != nil {
		h.H.WriteError(w, r, err)
		return
	}

	if err := h.Manager.DeleteClient(id); err != nil {
		h.H.WriteError(w, r, err)
		return
	}

	w.WriteHeader(http.StatusNoContent)
}
<|endoftext|>


<commit_before>package sys

import "io/ioutil"

// HasShabang checks whether the named file is a script by checking the shabang directive
func FileHasShabang(filename string) (bool, error) {
	// TODO: Don't read the complete file, only the first line or few bytes.
	byteContent, _ := ioutil.ReadFile(filename)

	content := string(byteContent)

	return content[0] == '#' && content[1] == '!', nil
}

// WriteScript writes the content of the byte slice into the named file
func WriteScript(filename string, contents []byte) error {
	// TODO: Variable environment preparation (e.g. temporarily Chdir)
	// BUG: The contents are appended if file/script already exists.
	//		Arbitrary code could be execution can be performed.
	return ioutil.WriteFile(filename, contents, 0744)
}
<commit_msg>Read first 2 bytes of a file to check for shabang<commit_after>package sys

import (
	"io/ioutil"
	"os"
)

// HasShabang checks whether the named file is a script by checking the shabang directive
func FileHasShabang(filename string) (bool, error) {
	f, err := os.Open(filename)
	if err != nil {
		return false, err
	}

	twoBytes := make([]byte, 2)

	_, err = f.Read(twoBytes)
	if err != nil {
		return false, err
	}

	return twoBytes[0] == '#' && twoBytes[1] == '!', nil
}

// WriteScript writes the content of the byte slice into the named file
func WriteScript(filename string, contents []byte) error {
	// TODO: Variable environment preparation (e.g. temporarily Chdir)
	// BUG: The contents are appended if file/script already exists.
	//		Arbitrary code could be execution can be performed.
	return ioutil.WriteFile(filename, contents, 0744)
}
<|endoftext|>


<commit_before>package httphead

import "io"

var (
	comma     = []byte{','}
	equality  = []byte{'='}
	semicolon = []byte{';'}
	quote     = []byte{'"'}
	escape    = []byte{'\\'}
)

// WriteOptions write options list to the dest.
// It uses the same form as {Scan,Parse}Options functions:
// values = 1#value
// value = token *( ";" param )
// param = token [ "=" (token | quoted-string) ]
//
// It wraps valuse into the quoted-string sequence if it contains any
// non-token characters.
func WriteOptions(dest io.Writer, options []Option) {
	for i, opt := range options {
		if i > 0 {
			dest.Write(comma)
		}

		writeTokenSanitized(dest, opt.Name)

		for _, p := range opt.Parameters.data() {
			dest.Write(semicolon)
			writeTokenSanitized(dest, p.key)
			if len(p.value) != 0 {
				dest.Write(equality)
				writeTokenSanitized(dest, p.value)
			}
		}
	}
}

// writeTokenSanitized writes token as is or as quouted string if it contains
// non-token characters.
//
// Note that is is not expects LWS sequnces be in s, cause LWS is used only as
// header field continuation:
// "A CRLF is allowed in the definition of TEXT only as part of a header field
// continuation. It is expected that the folding LWS will be replaced with a
// single SP before interpretation of the TEXT value."
// See https://tools.ietf.org/html/rfc2616#section-2
//
// That is we sanitizing s for writing, so there could not be any header field
// continuation.
// That is any CRLF will be escaped as any other control characters not allowd in TEXT.
func writeTokenSanitized(bw io.Writer, bts []byte) {
	var qt bool
	var pos int
	for i := 0; i < len(bts); i++ {
		c := bts[i]
		if !OctetTypes[c].IsToken() && !qt {
			qt = true
			bw.Write(quote)
		}
		if OctetTypes[c].IsControl() || c == '"' {
			if !qt {
				qt = true
				bw.Write(quote)
			}
			bw.Write(bts[pos:i])
			bw.Write(escape)
			bw.Write(bts[i : i+1])
			pos = i + 1
		}
	}
	if !qt {
		bw.Write(bts)
	} else {
		bw.Write(bts[pos:])
		bw.Write(quote)
	}
}
<commit_msg>collect io.Writer stats<commit_after>package httphead

import "io"

var (
	comma     = []byte{','}
	equality  = []byte{'='}
	semicolon = []byte{';'}
	quote     = []byte{'"'}
	escape    = []byte{'\\'}
)

// WriteOptions write options list to the dest.
// It uses the same form as {Scan,Parse}Options functions:
// values = 1#value
// value = token *( ";" param )
// param = token [ "=" (token | quoted-string) ]
//
// It wraps valuse into the quoted-string sequence if it contains any
// non-token characters.
func WriteOptions(dest io.Writer, options []Option) (n int, err error) {
	w := &writerErrHolder{w: dest}
	for i, opt := range options {
		if i > 0 {
			w.Write(comma)
		}

		writeTokenSanitized(w, opt.Name)

		for _, p := range opt.Parameters.data() {
			w.Write(semicolon)
			writeTokenSanitized(w, p.key)
			if len(p.value) != 0 {
				w.Write(equality)
				writeTokenSanitized(w, p.value)
			}
		}
	}
	return w.n, w.err
}

// writeTokenSanitized writes token as is or as quouted string if it contains
// non-token characters.
//
// Note that is is not expects LWS sequnces be in s, cause LWS is used only as
// header field continuation:
// "A CRLF is allowed in the definition of TEXT only as part of a header field
// continuation. It is expected that the folding LWS will be replaced with a
// single SP before interpretation of the TEXT value."
// See https://tools.ietf.org/html/rfc2616#section-2
//
// That is we sanitizing s for writing, so there could not be any header field
// continuation.
// That is any CRLF will be escaped as any other control characters not allowd in TEXT.
func writeTokenSanitized(bw io.Writer, bts []byte) {
	var qt bool
	var pos int
	for i := 0; i < len(bts); i++ {
		c := bts[i]
		if !OctetTypes[c].IsToken() && !qt {
			qt = true
			bw.Write(quote)
		}
		if OctetTypes[c].IsControl() || c == '"' {
			if !qt {
				qt = true
				bw.Write(quote)
			}
			bw.Write(bts[pos:i])
			bw.Write(escape)
			bw.Write(bts[i : i+1])
			pos = i + 1
		}
	}
	if !qt {
		bw.Write(bts)
	} else {
		bw.Write(bts[pos:])
		bw.Write(quote)
	}
}

type writerErrHolder struct {
	w io.Writer

	n   int
	err error
}

func (w *writerErrHolder) Write(p []byte) (n int, err error) {
	if w.err != nil {
		return 0, w.err
	}
	n, err = w.w.Write(p)
	w.err = err
	w.n += n
	return
}
<|endoftext|>


<commit_before>package quicktemplate

import (
	"fmt"
	"io"
	"strconv"
	"sync"
)

// Writer implements auxiliary writer used by quicktemplate functions.
//
// Use AcquireWriter for creating new writers.
type Writer struct {
	e QWriter
	n QWriter
}

// W returns the underlying writer passed to AcquireWriter.
func (qw *Writer) W() io.Writer {
	return qw.n.w
}

// E returns QWriter with enabled html escaping.
func (qw *Writer) E() *QWriter {
	return &qw.e
}

// N returns QWriter without html escaping.
func (qw *Writer) N() *QWriter {
	return &qw.n
}

// AcquireWriter returns new writer from the pool.
//
// Return unneeded writer to the pool by calling ReleaseWriter
// in order to reduce memory allocations.
func AcquireWriter(w io.Writer) *Writer {
	v := writerPool.Get()
	if v == nil {
		qw := &Writer{}
		qw.e.w = &htmlEscapeWriter{}
		v = qw
	}
	qw := v.(*Writer)
	qw.e.w.(*htmlEscapeWriter).w = w
	qw.n.w = w
	return qw
}

// ReleaseWriter returns the writer to the pool.
//
// Do not access released writer, otherwise data races may occur.
func ReleaseWriter(qw *Writer) {
	hw := qw.e.w.(*htmlEscapeWriter)
	hw.w = nil
	qw.e.Reset()
	qw.e.w = hw

	qw.n.Reset()

	writerPool.Put(qw)
}

var writerPool sync.Pool

// QWriter is auxiliary writer used by Writer.
type QWriter struct {
	w   io.Writer
	err error
	b   []byte
}

// Write implements io.Writer.
func (w *QWriter) Write(p []byte) (int, error) {
	if w.err != nil {
		return 0, w.err
	}
	n, err := w.w.Write(p)
	if err != nil {
		w.err = err
	}
	return n, err
}

// Reset resets QWriter to the original state.
func (w *QWriter) Reset() {
	w.w = nil
	w.err = nil
}

// S writes s to w.
func (w *QWriter) S(s string) {
	w.Write(unsafeStrToBytes(s))
}

// Z writes z to w.
func (w *QWriter) Z(z []byte) {
	w.Write(z)
}

// SZ is a synonym to Z.
func (w *QWriter) SZ(z []byte) {
	w.Write(z)
}

// D writes n to w.
func (w *QWriter) D(n int) {
	bb, ok := w.w.(*ByteBuffer)
	if ok {
		bb.B = strconv.AppendInt(bb.B, int64(n), 10)
	} else {
		w.b = strconv.AppendInt(w.b, int64(n), 10)
		w.Write(w.b)
	}
}

// F writes f to w.
func (w *QWriter) F(f float64) {
	n := int(f)
	if float64(n) == f {
		// Fast path - just int.
		w.D(n)
		return
	}

	// Slow path.
	w.FPrec(f, -1)
}

// FPrec writes f to w using the given floating point precision.
func (w *QWriter) FPrec(f float64, prec int) {
	bb, ok := w.w.(*ByteBuffer)
	if ok {
		bb.B = strconv.AppendFloat(bb.B, f, 'f', prec, 64)
	} else {
		w.b = strconv.AppendFloat(w.b[:0], f, 'f', prec, 64)
		w.Write(w.b)
	}
}

// Q writes quoted json-safe s to w.
func (w *QWriter) Q(s string) {
	w.Write(strQuote)
	writeJSONString(w, s)
	w.Write(strQuote)
}

var strQuote = []byte(`"`)

// QZ writes quoted json-safe z to w.
func (w *QWriter) QZ(z []byte) {
	w.Q(unsafeBytesToStr(z))
}

// J writes json-safe s to w.
//
// Unlike Q it doesn't qoute resulting s.
func (w *QWriter) J(s string) {
	writeJSONString(w, s)
}

// JZ writes json-safe z to w.
//
// Unlike Q it doesn't qoute resulting z.
func (w *QWriter) JZ(z []byte) {
	w.J(unsafeBytesToStr(z))
}

// V writes v to w.
func (w *QWriter) V(v interface{}) {
	fmt.Fprintf(w, "%v", v)
}

// U writes url-encoded s to w.
func (w *QWriter) U(s string) {
	bb, ok := w.w.(*ByteBuffer)
	if ok {
		bb.B = appendURLEncode(bb.B, s)
	} else {
		w.b = appendURLEncode(w.b[:0], s)
		w.Write(w.b)
	}
}

// UZ writes url-encoded z to w.
func (w *QWriter) UZ(z []byte) {
	w.U(unsafeBytesToStr(z))
}
<commit_msg>typo fix<commit_after>package quicktemplate

import (
	"fmt"
	"io"
	"strconv"
	"sync"
)

// Writer implements auxiliary writer used by quicktemplate functions.
//
// Use AcquireWriter for creating new writers.
type Writer struct {
	e QWriter
	n QWriter
}

// W returns the underlying writer passed to AcquireWriter.
func (qw *Writer) W() io.Writer {
	return qw.n.w
}

// E returns QWriter with enabled html escaping.
func (qw *Writer) E() *QWriter {
	return &qw.e
}

// N returns QWriter without html escaping.
func (qw *Writer) N() *QWriter {
	return &qw.n
}

// AcquireWriter returns new writer from the pool.
//
// Return unneeded writer to the pool by calling ReleaseWriter
// in order to reduce memory allocations.
func AcquireWriter(w io.Writer) *Writer {
	v := writerPool.Get()
	if v == nil {
		qw := &Writer{}
		qw.e.w = &htmlEscapeWriter{}
		v = qw
	}
	qw := v.(*Writer)
	qw.e.w.(*htmlEscapeWriter).w = w
	qw.n.w = w
	return qw
}

// ReleaseWriter returns the writer to the pool.
//
// Do not access released writer, otherwise data races may occur.
func ReleaseWriter(qw *Writer) {
	hw := qw.e.w.(*htmlEscapeWriter)
	hw.w = nil
	qw.e.Reset()
	qw.e.w = hw

	qw.n.Reset()

	writerPool.Put(qw)
}

var writerPool sync.Pool

// QWriter is auxiliary writer used by Writer.
type QWriter struct {
	w   io.Writer
	err error
	b   []byte
}

// Write implements io.Writer.
func (w *QWriter) Write(p []byte) (int, error) {
	if w.err != nil {
		return 0, w.err
	}
	n, err := w.w.Write(p)
	if err != nil {
		w.err = err
	}
	return n, err
}

// Reset resets QWriter to the original state.
func (w *QWriter) Reset() {
	w.w = nil
	w.err = nil
}

// S writes s to w.
func (w *QWriter) S(s string) {
	w.Write(unsafeStrToBytes(s))
}

// Z writes z to w.
func (w *QWriter) Z(z []byte) {
	w.Write(z)
}

// SZ is a synonym to Z.
func (w *QWriter) SZ(z []byte) {
	w.Write(z)
}

// D writes n to w.
func (w *QWriter) D(n int) {
	bb, ok := w.w.(*ByteBuffer)
	if ok {
		bb.B = strconv.AppendInt(bb.B, int64(n), 10)
	} else {
		w.b = strconv.AppendInt(w.b[:0], int64(n), 10)
		w.Write(w.b)
	}
}

// F writes f to w.
func (w *QWriter) F(f float64) {
	n := int(f)
	if float64(n) == f {
		// Fast path - just int.
		w.D(n)
		return
	}

	// Slow path.
	w.FPrec(f, -1)
}

// FPrec writes f to w using the given floating point precision.
func (w *QWriter) FPrec(f float64, prec int) {
	bb, ok := w.w.(*ByteBuffer)
	if ok {
		bb.B = strconv.AppendFloat(bb.B, f, 'f', prec, 64)
	} else {
		w.b = strconv.AppendFloat(w.b[:0], f, 'f', prec, 64)
		w.Write(w.b)
	}
}

// Q writes quoted json-safe s to w.
func (w *QWriter) Q(s string) {
	w.Write(strQuote)
	writeJSONString(w, s)
	w.Write(strQuote)
}

var strQuote = []byte(`"`)

// QZ writes quoted json-safe z to w.
func (w *QWriter) QZ(z []byte) {
	w.Q(unsafeBytesToStr(z))
}

// J writes json-safe s to w.
//
// Unlike Q it doesn't qoute resulting s.
func (w *QWriter) J(s string) {
	writeJSONString(w, s)
}

// JZ writes json-safe z to w.
//
// Unlike Q it doesn't qoute resulting z.
func (w *QWriter) JZ(z []byte) {
	w.J(unsafeBytesToStr(z))
}

// V writes v to w.
func (w *QWriter) V(v interface{}) {
	fmt.Fprintf(w, "%v", v)
}

// U writes url-encoded s to w.
func (w *QWriter) U(s string) {
	bb, ok := w.w.(*ByteBuffer)
	if ok {
		bb.B = appendURLEncode(bb.B, s)
	} else {
		w.b = appendURLEncode(w.b[:0], s)
		w.Write(w.b)
	}
}

// UZ writes url-encoded z to w.
func (w *QWriter) UZ(z []byte) {
	w.U(unsafeBytesToStr(z))
}
<|endoftext|>