<commit_before>package timeouts_test

import (
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"os/exec"
	"policy-server/config"
	"policy-server/integration/helpers"
	"strconv"
	"strings"

	"code.cloudfoundry.org/cf-networking-helpers/db"
	"code.cloudfoundry.org/cf-networking-helpers/metrics"
	"code.cloudfoundry.org/cf-networking-helpers/testsupport"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"github.com/onsi/gomega/gexec"
)

const testTimeoutInSeconds = 5

var policiesBody = `{
	"policies": [{
		"source": { "id": "some-app-guid" },
		"destination": { "id": "some-other-app-guid", "protocol": "tcp", "ports": {"start": 8090, "end": 8090} }
	}]
}`

var _ = Describe("Timeout", func() {
	var (
		session *gexec.Session
		conf    config.Config
		dbConf  db.Config
		headers map[string]string

		fakeMetron      metrics.FakeMetron
		policyServerURL string
	)
	BeforeEach(func() {
		dbConf = testsupport.GetDBConfig()
		if dbConf.Type == "postgres" {
			Skip("skipping timeout tests on postgres; only supported by mysql")
		}

		dbConf.DatabaseName = fmt.Sprintf("test_timeouts_node_%d", GinkgoParallelNode())
		dbConf.Timeout = 1
		testsupport.CreateDatabase(dbConf)

		fakeMetron = metrics.NewFakeMetron()

		conf = helpers.DefaultTestConfig(dbConf, fakeMetron.Address(), "../fixtures")
		session = helpers.StartPolicyServer(policyServerPath, conf)
		policyServerURL = fmt.Sprintf("http://%s:%d", conf.ListenHost, conf.ListenPort)

		resp := helpers.MakeAndDoRequest("GET", fmt.Sprintf("%s/%s", policyServerURL, "networking/v0/external/policies"), headers, nil)
		defer resp.Body.Close()
		Expect(resp.StatusCode).To(Equal(http.StatusOK))
		Expect(ioutil.ReadAll(resp.Body)).To(MatchJSON(`{ "total_policies": 0, "policies": [] }`))
	})

	AfterEach(func() {
		session.Interrupt()
		Eventually(session, helpers.DEFAULT_TIMEOUT).Should(gexec.Exit())

		testsupport.RemoveDatabase(dbConf)

		Expect(fakeMetron.Close()).To(Succeed())
	})

	Context("when the database is unreachable", func() {
		BeforeEach(func() {
			By("blocking access to port " + strconv.Itoa(int(dbConf.Port)))
			mustSucceed("iptables", "-w", "-A", "INPUT", "-p", "tcp", "--dport", strconv.Itoa(int(dbConf.Port)), "-j", "DROP")
		})
		AfterEach(func() {
			By("allowing access to port " + strconv.Itoa(int(dbConf.Port)))
			mustSucceed("iptables", "-w", "-D", "INPUT", "-p", "tcp", "--dport", strconv.Itoa(int(dbConf.Port)), "-j", "DROP")
		})

		itTimesOut := func(description string, endpointMethod string, endpointPath string, bodyString string, failureJSON string) {
			It(fmt.Sprintf("times out %s", description), func(done Done) {
				var body io.Reader
				if bodyString != "" {
					body = strings.NewReader(bodyString)
				}
				resp := helpers.MakeAndDoRequest(
					endpointMethod,
					fmt.Sprintf("%s/%s", policyServerURL, endpointPath),
					headers,
					body,
				)
				defer resp.Body.Close()
				Expect(resp.StatusCode).To(Equal(http.StatusInternalServerError))
				Expect(ioutil.ReadAll(resp.Body)).To(MatchJSON(failureJSON))

				close(done)
			}, float64(testTimeoutInSeconds))
		}

		itTimesOut("getting policies",
			"GET", "networking/v0/external/policies", "",
			`{ "error": "policies-index: database read failed" }`,
		)
		itTimesOut("creating policies",
			"POST", "networking/v0/external/policies", policiesBody,
			`{ "error": "policies-create: database create failed" }`,
		)
		itTimesOut("deleting policies",
			"POST", "networking/v0/external/policies/delete", policiesBody,
			`{ "error": "delete-policies: database delete failed" }`,
		)
		itTimesOut("getting tags",
			"GET", "networking/v0/external/tags", "",
			`{ "error": "tags-index: database read failed" }`,
		)
		itTimesOut("cleaning up",
			"POST", "networking/v0/external/policies/cleanup", "",
			`{ "error": "policies-cleanup: policies cleanup failed" }`,
		)
		itTimesOut("checking health",
			"GET", "health", "",
			`{ "error": "health: check database failed" }`,
		)
	})
})

func mustSucceed(binary string, args ...string) string {
	cmd := exec.Command(binary, args...)
	sess, err := gexec.Start(cmd, GinkgoWriter, GinkgoWriter)
	Expect(err).NotTo(HaveOccurred())
	Eventually(sess, helpers.DEFAULT_TIMEOUT).Should(gexec.Exit(0))
	return string(sess.Out.Contents())
}
<commit_msg>Backfill v0 tests in timeout test suite<commit_after>package timeouts_test

import (
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"os/exec"
	"policy-server/config"
	"policy-server/integration/helpers"
	"strconv"
	"strings"

	"code.cloudfoundry.org/cf-networking-helpers/db"
	"code.cloudfoundry.org/cf-networking-helpers/metrics"
	"code.cloudfoundry.org/cf-networking-helpers/testsupport"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"github.com/onsi/gomega/gexec"
)

const testTimeoutInSeconds = 5

var policiesBodyV0 = `{
	"policies": [{
		"source": { "id": "some-app-guid" },
		"destination": { "id": "some-other-app-guid", "protocol": "tcp", "port": 1234 }
	}]
}`

var policiesBodyV1 = `{
	"policies": [{
		"source": { "id": "some-app-guid" },
		"destination": { "id": "some-other-app-guid", "protocol": "tcp", "ports": {"start": 8090, "end": 8090} }
	}]
}`

var _ = Describe("Timeout", func() {
	var (
		session *gexec.Session
		conf    config.Config
		dbConf  db.Config
		headers map[string]string

		fakeMetron      metrics.FakeMetron
		policyServerURL string
	)
	BeforeEach(func() {
		dbConf = testsupport.GetDBConfig()
		if dbConf.Type == "postgres" {
			Skip("skipping timeout tests on postgres; only supported by mysql")
		}

		dbConf.DatabaseName = fmt.Sprintf("test_timeouts_node_%d", GinkgoParallelNode())
		dbConf.Timeout = 1
		testsupport.CreateDatabase(dbConf)

		fakeMetron = metrics.NewFakeMetron()

		conf = helpers.DefaultTestConfig(dbConf, fakeMetron.Address(), "../fixtures")
		session = helpers.StartPolicyServer(policyServerPath, conf)
		policyServerURL = fmt.Sprintf("http://%s:%d", conf.ListenHost, conf.ListenPort)

		resp := helpers.MakeAndDoRequest("GET", fmt.Sprintf("%s/%s", policyServerURL, "networking/v1/external/policies"), headers, nil)
		defer resp.Body.Close()
		Expect(resp.StatusCode).To(Equal(http.StatusOK))
		Expect(ioutil.ReadAll(resp.Body)).To(MatchJSON(`{ "total_policies": 0, "policies": [] }`))
	})

	AfterEach(func() {
		session.Interrupt()
		Eventually(session, helpers.DEFAULT_TIMEOUT).Should(gexec.Exit())

		testsupport.RemoveDatabase(dbConf)

		Expect(fakeMetron.Close()).To(Succeed())
	})

	Context("when the database is unreachable", func() {
		BeforeEach(func() {
			By("blocking access to port " + strconv.Itoa(int(dbConf.Port)))
			mustSucceed("iptables", "-w", "-A", "INPUT", "-p", "tcp", "--dport", strconv.Itoa(int(dbConf.Port)), "-j", "DROP")
		})
		AfterEach(func() {
			By("allowing access to port " + strconv.Itoa(int(dbConf.Port)))
			mustSucceed("iptables", "-w", "-D", "INPUT", "-p", "tcp", "--dport", strconv.Itoa(int(dbConf.Port)), "-j", "DROP")
		})

		itTimesOut := func(description string, endpointMethod string, endpointPath string, bodyString string, failureJSON string) {
			It(fmt.Sprintf("times out %s", description), func(done Done) {
				var body io.Reader
				if bodyString != "" {
					body = strings.NewReader(bodyString)
				}
				resp := helpers.MakeAndDoRequest(
					endpointMethod,
					fmt.Sprintf("%s/%s", policyServerURL, endpointPath),
					headers,
					body,
				)
				defer resp.Body.Close()
				Expect(resp.StatusCode).To(Equal(http.StatusInternalServerError))
				Expect(ioutil.ReadAll(resp.Body)).To(MatchJSON(failureJSON))

				close(done)
			}, float64(testTimeoutInSeconds))
		}

		// v1
		itTimesOut("V1 getting policies",
			"GET", "networking/v1/external/policies", "",
			`{ "error": "policies-index: database read failed" }`,
		)
		itTimesOut("V1 creating policies",
			"POST", "networking/v1/external/policies", policiesBodyV1,
			`{ "error": "policies-create: database create failed" }`,
		)
		itTimesOut("V1 deleting policies",
			"POST", "networking/v1/external/policies/delete", policiesBodyV1,
			`{ "error": "delete-policies: database delete failed" }`,
		)
		itTimesOut("V1 getting tags",
			"GET", "networking/v1/external/tags", "",
			`{ "error": "tags-index: database read failed" }`,
		)
		itTimesOut("V1 cleaning up",
			"POST", "networking/v1/external/policies/cleanup", "",
			`{ "error": "policies-cleanup: policies cleanup failed" }`,
		)

		// v0
		itTimesOut("V0 getting policies",
			"GET", "networking/v0/external/policies", "",
			`{ "error": "policies-index: database read failed" }`,
		)
		itTimesOut("V0 creating policies",
			"POST", "networking/v0/external/policies", policiesBodyV0,
			`{ "error": "policies-create: database create failed" }`,
		)
		itTimesOut("V0 deleting policies",
			"POST", "networking/v0/external/policies/delete", policiesBodyV0,
			`{ "error": "delete-policies: database delete failed" }`,
		)
		itTimesOut("V0 getting tags",
			"GET", "networking/v0/external/tags", "",
			`{ "error": "tags-index: database read failed" }`,
		)
		itTimesOut("V0 cleaning up",
			"POST", "networking/v0/external/policies/cleanup", "",
			`{ "error": "policies-cleanup: policies cleanup failed" }`,
		)

		itTimesOut("checking health",
			"GET", "health", "",
			`{ "error": "health: check database failed" }`,
		)
	})
})

func mustSucceed(binary string, args ...string) string {
	cmd := exec.Command(binary, args...)
	sess, err := gexec.Start(cmd, GinkgoWriter, GinkgoWriter)
	Expect(err).NotTo(HaveOccurred())
	Eventually(sess, helpers.DEFAULT_TIMEOUT).Should(gexec.Exit(0))
	return string(sess.Out.Contents())
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"strings"
)

type Renamer struct {
	Needle string
	Out    string
	Rel    string
	// Assets map[string]Asset
	Searcher
}

func NewRenamer(needle string, inputPath string, out string, rel string) *Renamer {

	var r Renamer

	r.Needle = needle
	r.Out = out
	r.Rel = rel

	r.Searcher.InputPath = inputPath
	r.Searcher.Data = struct{}{}
	r.Searcher.SearchCell = r.SearchCell

	return &r
}

func (r Renamer) SearchCell(cell string) {
	for _, s := range strings.Split(cell, ",") {
		// r.SearchCell(strings.TrimSpace(s))
		println(strings.TrimSpace(s))
	}
}

func (r Renamer) readJSONFromFile(inputPath string) (map[string]interface{}, error) {

	contents, err := ioutil.ReadFile(inputPath)
	if err != nil {
		return nil, err
	}

	var c map[string]interface{}
	json.Unmarshal(contents, &c)

	return c, err
}

func (r Renamer) Run() error {

	contents, err := ioutil.ReadFile(r.InputPath)
	if err != nil {
		return err
	}

	c, err := r.readJSONFromFile(r.InputPath)

	println(c)
	fmt.Printf("%v", c)
	r.searchMap(c)

	err = ioutil.WriteFile(r.Out, contents, 0644)
	return err
}
<commit_msg>move renamer write JSON to file into separate method<commit_after>package main

import (
	"encoding/json"
	"io/ioutil"
	"strings"
)

type Renamer struct {
	Needle string
	Out    string
	Rel    string
	// Assets map[string]Asset
	Searcher
}

func NewRenamer(needle string, inputPath string, out string, rel string) *Renamer {

	var r Renamer

	r.Needle = needle
	r.Out = out
	r.Rel = rel

	r.Searcher.InputPath = inputPath
	r.Searcher.Data = struct{}{}
	r.Searcher.SearchCell = r.SearchCell

	return &r
}

func (r Renamer) SearchCell(cell string) {
	for _, s := range strings.Split(cell, ",") {
		// r.SearchCell(strings.TrimSpace(s))
		println(strings.TrimSpace(s))
	}
}

func (r Renamer) readJSONFromFile(inputPath string) (map[string]interface{}, error) {

	contents, err := ioutil.ReadFile(inputPath)
	if err != nil {
		return nil, err
	}

	var c map[string]interface{}
	json.Unmarshal(contents, &c)

	return c, err
}

func (r Renamer) writeJSONToFile(c interface{}) error {
	contents, err := json.Marshal(c)
	if err != nil {
		return err
	}
	return ioutil.WriteFile(r.Out, contents, 0644)
}

func (r Renamer) Run() error {
	c, err := r.readJSONFromFile(r.InputPath)
	if err != nil {
		return err
	}
	r.searchMap(c)
	return r.writeJSONToFile(c)
}
<|endoftext|>


<commit_before>package falcore

import (
	"container/list"
	"fmt"
	"hash"
	"hash/crc32"
	"math/rand"
	"net"
	"net/http"
	"reflect"
	"time"
)

// Request wrapper
//
// The request is wrapped so that useful information can be kept
// with the request as it moves through the pipeline.
//
// A pointer is kept to the originating Connection.
//
// There is a unique ID assigned to each request.  This ID is not
// globally unique to keep it shorter for logging purposes.  It is
// possible to have duplicates though very unlikely over the period
// of a day or so.  It is a good idea to log the ID in any custom
// log statements so that individual requests can easily be grepped
// from busy log files.
//
// Falcore collects performance statistics on every stage of the
// pipeline.  The stats for the request are kept in PipelineStageStats.
// This structure will only be complete in the Request passed to the
// pipeline RequestDoneCallback.  Overhead will only be available in
// the RequestDoneCallback and it's the difference between the total
// request time and the sums of the stage times.  It will include things
// like pipeline iteration and the stat collection itself.
//
// See falcore.PipelineStageStat docs for more info.
//
// The Signature is also a cool feature. See the
type Request struct {
	ID                 string
	StartTime          time.Time
	EndTime            time.Time
	HttpRequest        *http.Request
	Connection         net.Conn
	RemoteAddr         *net.TCPAddr
	PipelineStageStats *list.List
	CurrentStage       *PipelineStageStat
	pipelineHash       hash.Hash32
	piplineTot         time.Duration
	Overhead           time.Duration
	Context            map[string]interface{}
}

// Used internally to create and initialize a new request.
func newRequest(request *http.Request, conn net.Conn, startTime time.Time) *Request {
	fReq := new(Request)
	fReq.Context = make(map[string]interface{})
	fReq.HttpRequest = request
	fReq.StartTime = startTime
	fReq.Connection = conn
	if conn != nil {
		fReq.RemoteAddr = conn.RemoteAddr().(*net.TCPAddr)
	}
	// create a semi-unique id to track a connection in the logs
	// ID is the least significant decimal digits of time with some randomization
	// the last 3 zeros of time.Nanoseconds appear to always be zero
	var ut = fReq.StartTime.UnixNano()
	fReq.ID = fmt.Sprintf("%010x", (ut-(ut-(ut%1e12)))+int64(rand.Intn(999)))
	fReq.PipelineStageStats = list.New()
	fReq.pipelineHash = crc32.NewIEEE()
	return fReq
}

// Returns a completed falcore.Request and response after running the single filter stage
// The PipelineStageStats is completed in the returned Request
// The falcore.Request.Connection and falcore.Request.RemoteAddr are nil
func TestWithRequest(request *http.Request, filter RequestFilter, context map[string]interface{}) (*Request, *http.Response) {
	r := newRequest(request, nil, time.Now())
	if context == nil {
		context = make(map[string]interface{})
	}
	r.Context = context
	t := reflect.TypeOf(filter)
	r.startPipelineStage(t.String())
	res := filter.FilterRequest(r)
	r.finishPipelineStage()
	r.finishRequest()
	return r, res
}

// Starts a new pipeline stage and makes it the CurrentStage.
func (fReq *Request) startPipelineStage(name string) {
	fReq.CurrentStage = NewPiplineStage(name)
	fReq.PipelineStageStats.PushBack(fReq.CurrentStage)
}

// Finishes the CurrentStage.
func (fReq *Request) finishPipelineStage() {
	fReq.CurrentStage.EndTime = time.Now()
	fReq.finishCommon()
}

// Appends an already completed PipelineStageStat directly to the list
func (fReq *Request) appendPipelineStage(pss *PipelineStageStat) {
	fReq.PipelineStageStats.PushBack(pss)
	fReq.CurrentStage = pss
	fReq.finishCommon()
}

// Does some required bookeeping for the pipeline and the pipeline signature
func (fReq *Request) finishCommon() {
	fReq.pipelineHash.Write([]byte(fReq.CurrentStage.Name))
	fReq.pipelineHash.Write([]byte{fReq.CurrentStage.Status})
	fReq.piplineTot += fReq.CurrentStage.EndTime.Sub(fReq.CurrentStage.StartTime)
}

// The Signature will only be complete in the RequestDoneCallback.  At
// any given time, the Signature is a crc32 sum of all the finished
// pipeline stages combining PipelineStageStat.Name and PipelineStageStat.Status.
// This gives a unique signature for each unique path through the pipeline.
// To modify the signature for your own use, just set the
// request.CurrentStage.Status in your RequestFilter or ResponseFilter.
func (fReq *Request) Signature() string {
	return fmt.Sprintf("%X", fReq.pipelineHash.Sum32())
}

// Call from RequestDoneCallback.  Logs a bunch of information about the
// request to the falcore logger. This is a pretty big hit to performance
// so it should only be used for debugging or development.  The source is a
// good example of how to get useful information out of the Request.
func (fReq *Request) Trace(res *http.Response) {
	reqTime := TimeDiff(fReq.StartTime, fReq.EndTime)
	req := fReq.HttpRequest
	Trace("%s [%s] %s%s Res=%i Sig=%s Tot=%.4f", fReq.ID, req.Method, req.Host, req.URL, res.StatusCode, fReq.Signature(), reqTime)
	l := fReq.PipelineStageStats
	for e := l.Front(); e != nil; e = e.Next() {
		pss, _ := e.Value.(*PipelineStageStat)
		dur := TimeDiff(pss.StartTime, pss.EndTime)
		Trace("%s %-30s S=%d Tot=%.4f %%=%.2f", fReq.ID, pss.Name, pss.Status, dur, dur/reqTime*100.0)
	}
	Trace("%s %-30s S=0 Tot=%.4f %%=%.2f", fReq.ID, "Overhead", float32(fReq.Overhead)/1.0e9, float32(fReq.Overhead)/1.0e9/reqTime*100.0)
}

func (fReq *Request) finishRequest() {
	fReq.EndTime = time.Now()
	fReq.Overhead = fReq.EndTime.Sub(fReq.StartTime) - fReq.piplineTot
}

// Container for keeping stats per pipeline stage
// Name for filter stages is reflect.TypeOf(filter).String()[1:] and the Status is 0 unless
// it is changed explicitly in the Filter or Router.
//
// For the Status, the falcore library will not apply any specific meaning to the status
// codes but the following are suggested conventional usages that we have found useful
//
//   type PipelineStatus byte
//   const (
// 	    Success PipelineStatus = iota	// General Run successfully
//	    Skip								// Skipped (all or most of the work of this stage)
//	    Fail								// General Fail
//	    // All others may be used as custom status codes
//   )
type PipelineStageStat struct {
	Name      string
	Status    byte
	StartTime time.Time
	EndTime   time.Time
}

func NewPiplineStage(name string) *PipelineStageStat {
	pss := new(PipelineStageStat)
	pss.Name = name
	pss.StartTime = time.Now()
	return pss
}
<commit_msg>fixed format code<commit_after>package falcore

import (
	"container/list"
	"fmt"
	"hash"
	"hash/crc32"
	"math/rand"
	"net"
	"net/http"
	"reflect"
	"time"
)

// Request wrapper
//
// The request is wrapped so that useful information can be kept
// with the request as it moves through the pipeline.
//
// A pointer is kept to the originating Connection.
//
// There is a unique ID assigned to each request.  This ID is not
// globally unique to keep it shorter for logging purposes.  It is
// possible to have duplicates though very unlikely over the period
// of a day or so.  It is a good idea to log the ID in any custom
// log statements so that individual requests can easily be grepped
// from busy log files.
//
// Falcore collects performance statistics on every stage of the
// pipeline.  The stats for the request are kept in PipelineStageStats.
// This structure will only be complete in the Request passed to the
// pipeline RequestDoneCallback.  Overhead will only be available in
// the RequestDoneCallback and it's the difference between the total
// request time and the sums of the stage times.  It will include things
// like pipeline iteration and the stat collection itself.
//
// See falcore.PipelineStageStat docs for more info.
//
// The Signature is also a cool feature. See the
type Request struct {
	ID                 string
	StartTime          time.Time
	EndTime            time.Time
	HttpRequest        *http.Request
	Connection         net.Conn
	RemoteAddr         *net.TCPAddr
	PipelineStageStats *list.List
	CurrentStage       *PipelineStageStat
	pipelineHash       hash.Hash32
	piplineTot         time.Duration
	Overhead           time.Duration
	Context            map[string]interface{}
}

// Used internally to create and initialize a new request.
func newRequest(request *http.Request, conn net.Conn, startTime time.Time) *Request {
	fReq := new(Request)
	fReq.Context = make(map[string]interface{})
	fReq.HttpRequest = request
	fReq.StartTime = startTime
	fReq.Connection = conn
	if conn != nil {
		fReq.RemoteAddr = conn.RemoteAddr().(*net.TCPAddr)
	}
	// create a semi-unique id to track a connection in the logs
	// ID is the least significant decimal digits of time with some randomization
	// the last 3 zeros of time.Nanoseconds appear to always be zero
	var ut = fReq.StartTime.UnixNano()
	fReq.ID = fmt.Sprintf("%010x", (ut-(ut-(ut%1e12)))+int64(rand.Intn(999)))
	fReq.PipelineStageStats = list.New()
	fReq.pipelineHash = crc32.NewIEEE()
	return fReq
}

// Returns a completed falcore.Request and response after running the single filter stage
// The PipelineStageStats is completed in the returned Request
// The falcore.Request.Connection and falcore.Request.RemoteAddr are nil
func TestWithRequest(request *http.Request, filter RequestFilter, context map[string]interface{}) (*Request, *http.Response) {
	r := newRequest(request, nil, time.Now())
	if context == nil {
		context = make(map[string]interface{})
	}
	r.Context = context
	t := reflect.TypeOf(filter)
	r.startPipelineStage(t.String())
	res := filter.FilterRequest(r)
	r.finishPipelineStage()
	r.finishRequest()
	return r, res
}

// Starts a new pipeline stage and makes it the CurrentStage.
func (fReq *Request) startPipelineStage(name string) {
	fReq.CurrentStage = NewPiplineStage(name)
	fReq.PipelineStageStats.PushBack(fReq.CurrentStage)
}

// Finishes the CurrentStage.
func (fReq *Request) finishPipelineStage() {
	fReq.CurrentStage.EndTime = time.Now()
	fReq.finishCommon()
}

// Appends an already completed PipelineStageStat directly to the list
func (fReq *Request) appendPipelineStage(pss *PipelineStageStat) {
	fReq.PipelineStageStats.PushBack(pss)
	fReq.CurrentStage = pss
	fReq.finishCommon()
}

// Does some required bookeeping for the pipeline and the pipeline signature
func (fReq *Request) finishCommon() {
	fReq.pipelineHash.Write([]byte(fReq.CurrentStage.Name))
	fReq.pipelineHash.Write([]byte{fReq.CurrentStage.Status})
	fReq.piplineTot += fReq.CurrentStage.EndTime.Sub(fReq.CurrentStage.StartTime)
}

// The Signature will only be complete in the RequestDoneCallback.  At
// any given time, the Signature is a crc32 sum of all the finished
// pipeline stages combining PipelineStageStat.Name and PipelineStageStat.Status.
// This gives a unique signature for each unique path through the pipeline.
// To modify the signature for your own use, just set the
// request.CurrentStage.Status in your RequestFilter or ResponseFilter.
func (fReq *Request) Signature() string {
	return fmt.Sprintf("%X", fReq.pipelineHash.Sum32())
}

// Call from RequestDoneCallback.  Logs a bunch of information about the
// request to the falcore logger. This is a pretty big hit to performance
// so it should only be used for debugging or development.  The source is a
// good example of how to get useful information out of the Request.
func (fReq *Request) Trace(res *http.Response) {
	reqTime := TimeDiff(fReq.StartTime, fReq.EndTime)
	req := fReq.HttpRequest
	Trace("%s [%s] %s%s Res=%d Sig=%s Tot=%.4f", fReq.ID, req.Method, req.Host, req.URL, res.StatusCode, fReq.Signature(), reqTime)
	l := fReq.PipelineStageStats
	for e := l.Front(); e != nil; e = e.Next() {
		pss, _ := e.Value.(*PipelineStageStat)
		dur := TimeDiff(pss.StartTime, pss.EndTime)
		Trace("%s %-30s S=%d Tot=%.4f %%=%.2f", fReq.ID, pss.Name, pss.Status, dur, dur/reqTime*100.0)
	}
	Trace("%s %-30s S=0 Tot=%.4f %%=%.2f", fReq.ID, "Overhead", float32(fReq.Overhead)/1.0e9, float32(fReq.Overhead)/1.0e9/reqTime*100.0)
}

func (fReq *Request) finishRequest() {
	fReq.EndTime = time.Now()
	fReq.Overhead = fReq.EndTime.Sub(fReq.StartTime) - fReq.piplineTot
}

// Container for keeping stats per pipeline stage
// Name for filter stages is reflect.TypeOf(filter).String()[1:] and the Status is 0 unless
// it is changed explicitly in the Filter or Router.
//
// For the Status, the falcore library will not apply any specific meaning to the status
// codes but the following are suggested conventional usages that we have found useful
//
//   type PipelineStatus byte
//   const (
// 	    Success PipelineStatus = iota	// General Run successfully
//	    Skip								// Skipped (all or most of the work of this stage)
//	    Fail								// General Fail
//	    // All others may be used as custom status codes
//   )
type PipelineStageStat struct {
	Name      string
	Status    byte
	StartTime time.Time
	EndTime   time.Time
}

func NewPiplineStage(name string) *PipelineStageStat {
	pss := new(PipelineStageStat)
	pss.Name = name
	pss.StartTime = time.Now()
	return pss
}
<|endoftext|>


<commit_before>package maodou

import (
	"fmt"
	"log"
	"net/http"
	urlpkg "net/url"
	"strconv"
	"time"

	"github.com/PuerkitoBio/goquery"
	"github.com/franela/goreq"

	// "github.com/mnhkahn/maodou/proxy"
)

type Request struct {
	goreq.Request
	proxies     []*ProxyConfig
	proxy_index int
	root        string
	Interval    time.Duration
}

func NewRequest(interval time.Duration) *Request {
	goreq.SetConnectTimeout(time.Duration(60) * time.Second)
	req := new(Request)
	req.Method = "GET"
	req.UserAgent = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2272.89 Safari/537.36"
	req.Timeout = time.Duration(60) * time.Second
	req.AddHeader("Accept-Language", "zh-CN,zh;q=0.8,en;q=0.6,zh-TW;q=0.4")
	req.AddHeader("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8")
	req.Interval = interval
	// req.ShowDebug = true

	if len(req.proxies) == 0 {
		resp, _ := req.Cawl("http://www.xici.net.co/")
		if resp != nil {
			resp.Doc(`#ip_list > tbody > tr`).Each(func(i int, s *goquery.Selection) {
				if i > 1 && len(req.proxies) < 10 && s.Children().Length() == 7 {
					p := new(ProxyConfig)
					p.ip = s.Children().Get(1).FirstChild.Data
					p.port, _ = strconv.Atoi(s.Children().Get(2).FirstChild.Data)
					if s.Children().Get(3).FirstChild != nil {
						p.location = s.Children().Get(3).FirstChild.Data
					}
					p.safe = s.Children().Get(4).FirstChild.Data == "高匿"
					p.schema = s.Children().Get(5).FirstChild.Data
					p.verifytime = s.Children().Get(6).FirstChild.Data
					req.proxies = append(req.proxies, p)
				}
			})
		}
	}

	return req
}

const (
	CAWL_NOPROXY = 0
	CAWL_PROXY   = 1
)

func (this *Request) Cawl(paras ...interface{}) (*Response, error) {
	this.Uri = paras[0].(string)

	// Add referer
	if this.root == "" {
		this.root = this.Uri
	} else {
		this.AddHeader("Referer", this.root)
	}

	if len(this.proxies) > 0 && (len(paras) == 1 || (len(paras) == 2 && paras[1].(int) == CAWL_PROXY)) {
		u := new(urlpkg.URL)
		p := this.proxies[this.proxy_index]
		u.Scheme = "http"
		u.Host = fmt.Sprintf("%s:%d", p.ip, p.port)
		this.Proxy = u.String()
	}

	log.Println("Start to Parse: ", this.Uri, "proxy:", this.Proxy)

	http_resp, err := this.Do()
	if err != nil {
		this.proxy_index = (this.proxy_index + 1) % len(this.proxies)
		log.Printf("Cawl Error: %s\n", err.Error())
		return nil, err
	}

	var resp *Response
	if http_resp.StatusCode == http.StatusOK {
		resp, err = NewResponse(http_resp.Body, this.Uri)
		if err != nil {
			log.Printf("Cawl Error: %s.\n", err.Error())
			return resp, err
		} else {
			log.Println("Cawl Success.")
		}
	} else {
		this.proxy_index = (this.proxy_index + 1) % len(this.proxies)
		if http_resp.StatusCode == http.StatusMovedPermanently || http_resp.StatusCode == http.StatusFound {
			log.Println(this.Uri, http_resp.StatusCode)
			return this.Cawl(http_resp.Header.Get("Location"))
		} else {
			log.Printf("Cawl Got Status Code %d.\n", http_resp.StatusCode)
			return resp, fmt.Errorf("Cawl Got Status Code %d.", http_resp.StatusCode)
		}
	}

	if this.Interval > 0 {
		time.Sleep(this.Interval)
	}
	return resp, nil
}

func (this *Request) DumpRequest() string {
	return this.Uri + "?" + urlpkg.Values(this.QueryString.(urlpkg.Values)).Encode()
}

type ProxyConfig struct {
	ip         string
	port       int
	location   string
	safe       bool
	schema     string
	verifytime string
}
<commit_msg>修复代理错乱的问题<commit_after>package maodou

import (
	"fmt"
	"log"
	"net/http"
	urlpkg "net/url"
	"strconv"
	"time"

	"github.com/PuerkitoBio/goquery"
	"github.com/franela/goreq"
)

type Request struct {
	goreq.Request
	proxies     []*ProxyConfig
	proxy_index int
	root        string
	Interval    time.Duration
}

func NewRequest(interval time.Duration) *Request {
	goreq.SetConnectTimeout(time.Duration(60) * time.Second)
	req := new(Request)
	req.Method = "GET"
	req.UserAgent = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2272.89 Safari/537.36"
	req.Timeout = time.Duration(60) * time.Second
	req.AddHeader("Accept-Language", "zh-CN,zh;q=0.8,en;q=0.6,zh-TW;q=0.4")
	req.AddHeader("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8")
	req.Interval = interval
	// req.ShowDebug = true

	if len(req.proxies) == 0 {
		resp, _ := req.Cawl("http://www.xici.net.co/")
		if resp != nil {
			resp.Doc(`#ip_list > tbody > tr`).Each(func(i int, s *goquery.Selection) {
				if i > 1 && len(req.proxies) < 10 && s.Children().Length() == 7 {
					p := new(ProxyConfig)
					p.ip = s.Children().Get(1).FirstChild.Data
					p.port, _ = strconv.Atoi(s.Children().Get(2).FirstChild.Data)
					if s.Children().Get(3).FirstChild != nil {
						p.location = s.Children().Get(3).FirstChild.Data
					}
					p.safe = s.Children().Get(4).FirstChild.Data == "高匿"
					p.schema = s.Children().Get(5).FirstChild.Data
					p.verifytime = s.Children().Get(6).FirstChild.Data
					req.proxies = append(req.proxies, p)
				}
			})
		}
	}

	return req
}

const (
	CAWL_NOPROXY = 0
	CAWL_PROXY   = 1
)

func (this *Request) Cawl(paras ...interface{}) (*Response, error) {
	this.Uri = paras[0].(string)

	// Add referer
	if this.root == "" {
		this.root = this.Uri
	} else {
		this.AddHeader("Referer", this.root)
	}

	if len(this.proxies) > 0 && (len(paras) == 1 || (len(paras) == 2 && paras[1].(int) == CAWL_PROXY)) {
		u := new(urlpkg.URL)
		p := this.proxies[this.proxy_index]
		u.Scheme = "http"
		u.Host = fmt.Sprintf("%s:%d", p.ip, p.port)
		this.Proxy = u.String()
	}

	log.Println("Start to Parse: ", this.Uri, "proxy:", this.Proxy)

	http_resp, err := this.Do()
	// 修复代理错乱的问题，需要重置代理
	this.Proxy = ""
	if err != nil {
		this.proxy_index = (this.proxy_index + 1) % len(this.proxies)
		log.Printf("Cawl Error: %s\n", err.Error())
		return nil, err
	}

	var resp *Response
	if http_resp.StatusCode == http.StatusOK {
		resp, err = NewResponse(http_resp.Body, this.Uri)
		if err != nil {
			log.Printf("Cawl Error: %s.\n", err.Error())
			return resp, err
		} else {
			log.Println("Cawl Success.")
		}
	} else {
		this.proxy_index = (this.proxy_index + 1) % len(this.proxies)
		if http_resp.StatusCode == http.StatusMovedPermanently || http_resp.StatusCode == http.StatusFound {
			log.Println(this.Uri, http_resp.StatusCode)
			return this.Cawl(http_resp.Header.Get("Location"))
		} else {
			log.Printf("Cawl Got Status Code %d.\n", http_resp.StatusCode)
			return resp, fmt.Errorf("Cawl Got Status Code %d.", http_resp.StatusCode)
		}
	}

	if this.Interval > 0 {
		time.Sleep(this.Interval)
	}
	return resp, nil
}

func (this *Request) DumpRequest() string {
	return this.Uri + "?" + urlpkg.Values(this.QueryString.(urlpkg.Values)).Encode()
}

type ProxyConfig struct {
	ip         string
	port       int
	location   string
	safe       bool
	schema     string
	verifytime string
}
<|endoftext|>


<commit_before>package main

import (
	"crypto/tls"
	"crypto/x509"
	"encoding/json"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"path/filepath"
	"strings"
	"time"

	"github.com/franela/goreq"
)

func init() {
	goreq.SetConnectTimeout(15 * time.Second)
	goreq.DefaultTransport.(*http.Transport).TLSClientConfig = &tls.Config{RootCAs: getCACerts()}
}

func apiRequestBase(authToken string) *goreq.Request {
	req := goreq.Request{
		Uri:       apiURL(),
		ShowDebug: debugging,
		Insecure:  !shouldVerifyHost(apiURL()),
	}
	if authToken != "" {
		req.AddHeader("Authorization", "Bearer "+authToken)
	}
	if os.Getenv("HEROKU_HEADERS") != "" {
		var h map[string]string
		json.Unmarshal([]byte(os.Getenv("HEROKU_HEADERS")), &h)
		for k, v := range h {
			req.AddHeader(k, v)
		}
	}
	return &req
}

func apiRequest(authToken string) *goreq.Request {
	req := apiRequestBase(authToken)
	req.AddHeader("Accept", "application/vnd.heroku+json; version=3")
	return req
}

func shouldVerifyHost(host string) bool {
	return !strings.HasSuffix(host, "herokudev.com")
}

func getCACerts() *x509.CertPool {
	certs := x509.NewCertPool()
	path := filepath.Join(AppDir(), "cacert.pem")
	data, err := ioutil.ReadFile(path)
	if err != nil {
		downloadCert(path)
		data, err = ioutil.ReadFile(path)
		if err != nil {
			PrintError(err)
			return nil
		}
	}
	ok := certs.AppendCertsFromPEM(data)
	if !ok {
		Warn("Error parsing " + path)
		return nil
	}
	return certs
}

func downloadCert(path string) {
	f, err := os.Create(path)
	if err != nil {
		PrintError(err)
		return
	}
	res, err := goreq.Request{
		Uri:       "https://raw.githubusercontent.com/bagder/ca-bundle/master/ca-bundle.crt",
		ShowDebug: debugging,
	}.Do()
	if err != nil {
		PrintError(err)
		return
	}
	defer res.Body.Close()
	defer f.Close()
	io.Copy(f, res.Body)
}
<commit_msg>use HEROKU_SSL_VERIFY env var<commit_after>package main

import (
	"crypto/tls"
	"crypto/x509"
	"encoding/json"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"path/filepath"
	"strings"
	"time"

	"github.com/franela/goreq"
)

func init() {
	goreq.SetConnectTimeout(15 * time.Second)
	goreq.DefaultTransport.(*http.Transport).TLSClientConfig = &tls.Config{RootCAs: getCACerts()}
}

func apiRequestBase(authToken string) *goreq.Request {
	req := goreq.Request{
		Uri:       apiURL(),
		ShowDebug: debugging,
		Insecure:  !shouldVerifyHost(apiURL()),
	}
	if authToken != "" {
		req.AddHeader("Authorization", "Bearer "+authToken)
	}
	if os.Getenv("HEROKU_HEADERS") != "" {
		var h map[string]string
		json.Unmarshal([]byte(os.Getenv("HEROKU_HEADERS")), &h)
		for k, v := range h {
			req.AddHeader(k, v)
		}
	}
	return &req
}

func apiRequest(authToken string) *goreq.Request {
	req := apiRequestBase(authToken)
	req.AddHeader("Accept", "application/vnd.heroku+json; version=3")
	return req
}

func shouldVerifyHost(host string) bool {
	return !(os.Getenv("HEROKU_SSL_VERIFY") == "disable" || strings.HasSuffix(host, "herokudev.com"))
}

func getCACerts() *x509.CertPool {
	certs := x509.NewCertPool()
	path := filepath.Join(AppDir(), "cacert.pem")
	data, err := ioutil.ReadFile(path)
	if err != nil {
		downloadCert(path)
		data, err = ioutil.ReadFile(path)
		if err != nil {
			PrintError(err)
			return nil
		}
	}
	ok := certs.AppendCertsFromPEM(data)
	if !ok {
		Warn("Error parsing " + path)
		return nil
	}
	return certs
}

func downloadCert(path string) {
	f, err := os.Create(path)
	if err != nil {
		PrintError(err)
		return
	}
	res, err := goreq.Request{
		Uri:       "https://raw.githubusercontent.com/bagder/ca-bundle/master/ca-bundle.crt",
		ShowDebug: debugging,
	}.Do()
	if err != nil {
		PrintError(err)
		return
	}
	defer res.Body.Close()
	defer f.Close()
	io.Copy(f, res.Body)
}
<|endoftext|>


<commit_before>package request

import (
	"compress/gzip"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"net/http/cookiejar"
	"net/url"
	"strings"

	"github.com/bitly/go-simplejson"
	"golang.org/x/net/publicsuffix"
)

type Request struct {
	*http.Request
}

type Response struct {
	*http.Response
	content []byte
}

func (resp *Response) Json() (*simplejson.Json, error) {
	b, err := resp.Content()
	if err != nil {
		return nil, err
	}
	return simplejson.NewJson(b)
}

func (resp *Response) Content() (b []byte, err error) {
	if resp.content != nil {
		return resp.content, nil
	}

	var reader io.ReadCloser
	switch resp.Header.Get("Content-Encoding") {
	case "gzip":
		reader, err := gzip.NewReader(resp.Body)
		if err != nil {
			log.Fatal(err)
		}
		b, err = ioutil.ReadAll(reader)
		defer reader.Close()
	default:
		reader = resp.Body
		b, err = ioutil.ReadAll(reader)
	}

	if err != nil {
		return nil, err
	}
	resp.content = b
	return b, err
}

func (resp *Response) Text() (string, error) {
	b, err := resp.Content()
	s := string(b)
	return s, err
}

func (resp *Response) Ok() bool {
	return resp.StatusCode < 400
}

func (resp *Response) Reason() string {
	return resp.Status
}

type Args struct {
	Client  *http.Client
	Headers map[string]string
	Cookies map[string]string
	Data    map[string]string
	Params  map[string]string
	Files   map[string]string
}

var defaultHeaders = map[string]string{
	"Connection":      "keep-alive",
	"Accept-Encoding": "gzip, deflate",
	"Accept":          "*/*",
	"User-Agent":      "go-request/0.1.0",
}
var defaultBodyType = "application/x-www-form-urlencoded"

func NewArgs(c *http.Client) *Args {
	if c.Jar == nil {
		options := cookiejar.Options{
			PublicSuffixList: publicsuffix.List,
		}
		jar, _ := cookiejar.New(&options)
		c.Jar = jar
	}

	return &Args{
		Client:  c,
		Headers: defaultHeaders,
		Cookies: nil,
		Data:    nil,
		Params:  nil,
		Files:   nil,
	}
}

func newBody(data map[string]string) (body io.Reader) {
	if data == nil {
		return nil
	}

	d := url.Values{}
	for k, v := range data {
		d.Set(k, v)
	}
	return strings.NewReader(d.Encode())
}

func newURL(u string, params map[string]string) string {
	if params == nil {
		return u
	}

	p := url.Values{}
	for k, v := range params {
		p.Set(k, v)
	}
	if strings.Contains(u, "?") {
		return u + "&" + p.Encode()
	}
	return u + "?" + p.Encode()
}

func newRequest(method string, url string, a *Args) (resp *Response, err error) {
	client := a.Client
	body := newBody(a.Data)
	u := newURL(url, a.Params)
	req, err := http.NewRequest(method, u, body)
	if err != nil {
		log.Fatal(err)
		return
	}
	// apply defaultHeaders
	for k, v := range defaultHeaders {
		_, ok := a.Headers[k]
		if !ok {
			req.Header.Set(k, v)
		}
	}
	// apply custom Headers
	for k, v := range a.Headers {
		req.Header.Set(k, v)
	}
	// apply "Content-Type" Headers
	_, ok := a.Headers["Content-Type"]
	if !ok {
		req.Header.Set("Content-Type", defaultBodyType)
	}

	s, err := client.Do(req)
	resp = &Response{s, nil}
	return
}

func Get(url string, a *Args) (resp *Response, err error) {
	resp, err = newRequest("GET", url, a)
	return
}

func Head(url string, a *Args) (resp *Response, err error) {
	resp, err = newRequest("HEAD", url, a)
	return
}

func Post(url string, a *Args) (resp *Response, err error) {
	resp, err = newRequest("POST", url, a)
	return
}

func Put(url string, a *Args) (resp *Response, err error) {
	resp, err = newRequest("PUT", url, a)
	return
}

func Patch(url string, a *Args) (resp *Response, err error) {
	resp, err = newRequest("PATCH", url, a)
	return
}

func Delete(url string, a *Args) (resp *Response, err error) {
	resp, err = newRequest("DELETE", url, a)
	return
}

func Options(url string, a *Args) (resp *Response, err error) {
	resp, err = newRequest("OPTIONS", url, a)
	return
}
<commit_msg>add applyHeaders<commit_after>package request

import (
	"compress/gzip"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"net/http/cookiejar"
	"net/url"
	"strings"

	"github.com/bitly/go-simplejson"
	"golang.org/x/net/publicsuffix"
)

// type Request struct {
// 	*http.Request
// }

type Response struct {
	*http.Response
	content []byte
}

func (resp *Response) Json() (*simplejson.Json, error) {
	b, err := resp.Content()
	if err != nil {
		return nil, err
	}
	return simplejson.NewJson(b)
}

func (resp *Response) Content() (b []byte, err error) {
	if resp.content != nil {
		return resp.content, nil
	}

	var reader io.ReadCloser
	switch resp.Header.Get("Content-Encoding") {
	case "gzip":
		reader, err := gzip.NewReader(resp.Body)
		if err != nil {
			log.Fatal(err)
		}
		b, err = ioutil.ReadAll(reader)
		defer reader.Close()
	default:
		reader = resp.Body
		b, err = ioutil.ReadAll(reader)
	}

	if err != nil {
		return nil, err
	}
	resp.content = b
	return b, err
}

func (resp *Response) Text() (string, error) {
	b, err := resp.Content()
	s := string(b)
	return s, err
}

func (resp *Response) Ok() bool {
	return resp.StatusCode < 400
}

func (resp *Response) Reason() string {
	return resp.Status
}

type Args struct {
	Client  *http.Client
	Headers map[string]string
	Cookies map[string]string
	Data    map[string]string
	Params  map[string]string
	Files   map[string][]byte
}

var defaultHeaders = map[string]string{
	"Connection":      "keep-alive",
	"Accept-Encoding": "gzip, deflate",
	"Accept":          "*/*",
	"User-Agent":      "go-request/0.1.0",
}
var defaultBodyType = "application/x-www-form-urlencoded"

func NewArgs(c *http.Client) *Args {
	if c.Jar == nil {
		options := cookiejar.Options{
			PublicSuffixList: publicsuffix.List,
		}
		jar, _ := cookiejar.New(&options)
		c.Jar = jar
	}

	return &Args{
		Client:  c,
		Headers: defaultHeaders,
		Cookies: nil,
		Data:    nil,
		Params:  nil,
		Files:   nil,
	}
}

func applyHeaders(a *Args, req *http.Request) {
	// apply defaultHeaders
	for k, v := range defaultHeaders {
		_, ok := a.Headers[k]
		if !ok {
			req.Header.Set(k, v)
		}
	}
	// apply custom Headers
	for k, v := range a.Headers {
		req.Header.Set(k, v)
	}
	// apply "Content-Type" Headers
	_, ok := a.Headers["Content-Type"]
	if !ok {
		req.Header.Set("Content-Type", defaultBodyType)
	}
}

func newURL(u string, params map[string]string) string {
	if params == nil {
		return u
	}

	p := url.Values{}
	for k, v := range params {
		p.Set(k, v)
	}
	if strings.Contains(u, "?") {
		return u + "&" + p.Encode()
	}
	return u + "?" + p.Encode()
}

func newBody(data map[string]string) (body io.Reader) {
	if data == nil {
		return nil
	}

	d := url.Values{}
	for k, v := range data {
		d.Set(k, v)
	}
	return strings.NewReader(d.Encode())
}

func newRequest(method string, url string, a *Args) (resp *Response, err error) {
	client := a.Client
	body := newBody(a.Data)
	u := newURL(url, a.Params)
	req, err := http.NewRequest(method, u, body)
	if err != nil {
		log.Fatal(err)
		return
	}
	applyHeaders(a, req)

	s, err := client.Do(req)
	resp = &Response{s, nil}
	return
}

func Get(url string, a *Args) (resp *Response, err error) {
	resp, err = newRequest("GET", url, a)
	return
}

func Head(url string, a *Args) (resp *Response, err error) {
	resp, err = newRequest("HEAD", url, a)
	return
}

func Post(url string, a *Args) (resp *Response, err error) {
	resp, err = newRequest("POST", url, a)
	return
}

func Put(url string, a *Args) (resp *Response, err error) {
	resp, err = newRequest("PUT", url, a)
	return
}

func Patch(url string, a *Args) (resp *Response, err error) {
	resp, err = newRequest("PATCH", url, a)
	return
}

func Delete(url string, a *Args) (resp *Response, err error) {
	resp, err = newRequest("DELETE", url, a)
	return
}

func Options(url string, a *Args) (resp *Response, err error) {
	resp, err = newRequest("OPTIONS", url, a)
	return
}
<|endoftext|>


<commit_before>// +build linux

package main

import (
	"os"
	"syscall"

	"github.com/Sirupsen/logrus"
	"github.com/opencontainers/runc/libcontainer"
	"github.com/opencontainers/runc/libcontainer/configs"
	"github.com/opencontainers/runc/libcontainer/specconv"
	"github.com/opencontainers/runtime-spec/specs-go"
	"github.com/urfave/cli"
)

var restoreCommand = cli.Command{
	Name:  "restore",
	Usage: "restore a container from a previous checkpoint",
	ArgsUsage: `<container-id>

Where "<container-id>" is the name for the instance of the container to be
restored.`,
	Description: `Restores the saved state of the container instance that was previously saved
using the runc checkpoint command.`,
	Flags: []cli.Flag{
		cli.StringFlag{
			Name:  "image-path",
			Value: "",
			Usage: "path to criu image files for restoring",
		},
		cli.StringFlag{
			Name:  "work-path",
			Value: "",
			Usage: "path for saving work files and logs",
		},
		cli.BoolFlag{
			Name:  "tcp-established",
			Usage: "allow open tcp connections",
		},
		cli.BoolFlag{
			Name:  "ext-unix-sk",
			Usage: "allow external unix sockets",
		},
		cli.BoolFlag{
			Name:  "shell-job",
			Usage: "allow shell jobs",
		},
		cli.BoolFlag{
			Name:  "file-locks",
			Usage: "handle file locks, for safety",
		},
		cli.StringFlag{
			Name:  "manage-cgroups-mode",
			Value: "",
			Usage: "cgroups mode: 'soft' (default), 'full' and 'strict'",
		},
		cli.StringFlag{
			Name:  "bundle, b",
			Value: "",
			Usage: "path to the root of the bundle directory",
		},
		cli.BoolFlag{
			Name:  "detach,d",
			Usage: "detach from the container's process",
		},
		cli.StringFlag{
			Name:  "pid-file",
			Value: "",
			Usage: "specify the file to write the process id to",
		},
		cli.BoolFlag{
			Name:  "no-subreaper",
			Usage: "disable the use of the subreaper used to reap reparented processes",
		},
		cli.BoolFlag{
			Name:  "no-pivot",
			Usage: "do not use pivot root to jail process inside rootfs.  This should be used whenever the rootfs is on top of a ramdisk",
		},
		cli.StringSliceFlag{
			Name:  "empty-ns",
			Usage: "create a namespace, but don't restore its properies",
		},
	},
	Action: func(context *cli.Context) error {
		imagePath := context.String("image-path")
		id := context.Args().First()
		if id == "" {
			return errEmptyID
		}
		if imagePath == "" {
			imagePath = getDefaultImagePath(context)
		}
		bundle := context.String("bundle")
		if bundle != "" {
			if err := os.Chdir(bundle); err != nil {
				return err
			}
		}
		spec, err := loadSpec(specConfig)
		if err != nil {
			return err
		}
		config, err := specconv.CreateLibcontainerConfig(&specconv.CreateOpts{
			CgroupName:       id,
			UseSystemdCgroup: context.GlobalBool("systemd-cgroup"),
			NoPivotRoot:      context.Bool("no-pivot"),
			Spec:             spec,
		})
		if err != nil {
			return err
		}
		status, err := restoreContainer(context, spec, config, imagePath)
		if err == nil {
			os.Exit(status)
		}
		return err
	},
}

func restoreContainer(context *cli.Context, spec *specs.Spec, config *configs.Config, imagePath string) (code int, err error) {
	var (
		rootuid = 0
		rootgid = 0
		id      = context.Args().First()
	)
	factory, err := loadFactory(context)
	if err != nil {
		return -1, err
	}
	container, err := factory.Load(id)
	if err != nil {
		container, err = factory.Create(id, config)
		if err != nil {
			return -1, err
		}
	}
	options := criuOptions(context)

	status, err := container.Status()
	if err != nil {
		logrus.Error(err)
	}
	if status == libcontainer.Running {
		fatalf("Container with id %s already running", id)
	}

	setManageCgroupsMode(context, options)

	if err := setEmptyNsMask(context, options); err != nil {
		return -1, err
	}

	// ensure that the container is always removed if we were the process
	// that created it.
	detach := context.Bool("detach")
	if !detach {
		defer destroy(container)
	}
	process := &libcontainer.Process{}
	tty, err := setupIO(process, rootuid, rootgid, "", false, detach)
	if err != nil {
		return -1, err
	}
	defer tty.Close()
	handler := newSignalHandler(tty, !context.Bool("no-subreaper"))
	if err := container.Restore(process, options); err != nil {
		return -1, err
	}
	if err := tty.ClosePostStart(); err != nil {
		return -1, err
	}
	if pidFile := context.String("pid-file"); pidFile != "" {
		if err := createPidFile(pidFile, process); err != nil {
			process.Signal(syscall.SIGKILL)
			process.Wait()
			return -1, err
		}
	}
	if detach {
		return 0, nil
	}
	return handler.forward(process)
}

func criuOptions(context *cli.Context) *libcontainer.CriuOpts {
	imagePath := getCheckpointImagePath(context)
	if err := os.MkdirAll(imagePath, 0655); err != nil {
		fatal(err)
	}
	return &libcontainer.CriuOpts{
		ImagesDirectory:         imagePath,
		WorkDirectory:           context.String("work-path"),
		LeaveRunning:            context.Bool("leave-running"),
		TcpEstablished:          context.Bool("tcp-established"),
		ExternalUnixConnections: context.Bool("ext-unix-sk"),
		ShellJob:                context.Bool("shell-job"),
		FileLocks:               context.Bool("file-locks"),
	}
}
<commit_msg>remove unused returned variables name<commit_after>// +build linux

package main

import (
	"os"
	"syscall"

	"github.com/Sirupsen/logrus"
	"github.com/opencontainers/runc/libcontainer"
	"github.com/opencontainers/runc/libcontainer/configs"
	"github.com/opencontainers/runc/libcontainer/specconv"
	"github.com/opencontainers/runtime-spec/specs-go"
	"github.com/urfave/cli"
)

var restoreCommand = cli.Command{
	Name:  "restore",
	Usage: "restore a container from a previous checkpoint",
	ArgsUsage: `<container-id>

Where "<container-id>" is the name for the instance of the container to be
restored.`,
	Description: `Restores the saved state of the container instance that was previously saved
using the runc checkpoint command.`,
	Flags: []cli.Flag{
		cli.StringFlag{
			Name:  "image-path",
			Value: "",
			Usage: "path to criu image files for restoring",
		},
		cli.StringFlag{
			Name:  "work-path",
			Value: "",
			Usage: "path for saving work files and logs",
		},
		cli.BoolFlag{
			Name:  "tcp-established",
			Usage: "allow open tcp connections",
		},
		cli.BoolFlag{
			Name:  "ext-unix-sk",
			Usage: "allow external unix sockets",
		},
		cli.BoolFlag{
			Name:  "shell-job",
			Usage: "allow shell jobs",
		},
		cli.BoolFlag{
			Name:  "file-locks",
			Usage: "handle file locks, for safety",
		},
		cli.StringFlag{
			Name:  "manage-cgroups-mode",
			Value: "",
			Usage: "cgroups mode: 'soft' (default), 'full' and 'strict'",
		},
		cli.StringFlag{
			Name:  "bundle, b",
			Value: "",
			Usage: "path to the root of the bundle directory",
		},
		cli.BoolFlag{
			Name:  "detach,d",
			Usage: "detach from the container's process",
		},
		cli.StringFlag{
			Name:  "pid-file",
			Value: "",
			Usage: "specify the file to write the process id to",
		},
		cli.BoolFlag{
			Name:  "no-subreaper",
			Usage: "disable the use of the subreaper used to reap reparented processes",
		},
		cli.BoolFlag{
			Name:  "no-pivot",
			Usage: "do not use pivot root to jail process inside rootfs.  This should be used whenever the rootfs is on top of a ramdisk",
		},
		cli.StringSliceFlag{
			Name:  "empty-ns",
			Usage: "create a namespace, but don't restore its properies",
		},
	},
	Action: func(context *cli.Context) error {
		imagePath := context.String("image-path")
		id := context.Args().First()
		if id == "" {
			return errEmptyID
		}
		if imagePath == "" {
			imagePath = getDefaultImagePath(context)
		}
		bundle := context.String("bundle")
		if bundle != "" {
			if err := os.Chdir(bundle); err != nil {
				return err
			}
		}
		spec, err := loadSpec(specConfig)
		if err != nil {
			return err
		}
		config, err := specconv.CreateLibcontainerConfig(&specconv.CreateOpts{
			CgroupName:       id,
			UseSystemdCgroup: context.GlobalBool("systemd-cgroup"),
			NoPivotRoot:      context.Bool("no-pivot"),
			Spec:             spec,
		})
		if err != nil {
			return err
		}
		status, err := restoreContainer(context, spec, config, imagePath)
		if err == nil {
			os.Exit(status)
		}
		return err
	},
}

func restoreContainer(context *cli.Context, spec *specs.Spec, config *configs.Config, imagePath string) (int, error) {
	var (
		rootuid = 0
		rootgid = 0
		id      = context.Args().First()
	)
	factory, err := loadFactory(context)
	if err != nil {
		return -1, err
	}
	container, err := factory.Load(id)
	if err != nil {
		container, err = factory.Create(id, config)
		if err != nil {
			return -1, err
		}
	}
	options := criuOptions(context)

	status, err := container.Status()
	if err != nil {
		logrus.Error(err)
	}
	if status == libcontainer.Running {
		fatalf("Container with id %s already running", id)
	}

	setManageCgroupsMode(context, options)

	if err := setEmptyNsMask(context, options); err != nil {
		return -1, err
	}

	// ensure that the container is always removed if we were the process
	// that created it.
	detach := context.Bool("detach")
	if !detach {
		defer destroy(container)
	}
	process := &libcontainer.Process{}
	tty, err := setupIO(process, rootuid, rootgid, "", false, detach)
	if err != nil {
		return -1, err
	}
	defer tty.Close()
	handler := newSignalHandler(tty, !context.Bool("no-subreaper"))
	if err := container.Restore(process, options); err != nil {
		return -1, err
	}
	if err := tty.ClosePostStart(); err != nil {
		return -1, err
	}
	if pidFile := context.String("pid-file"); pidFile != "" {
		if err := createPidFile(pidFile, process); err != nil {
			process.Signal(syscall.SIGKILL)
			process.Wait()
			return -1, err
		}
	}
	if detach {
		return 0, nil
	}
	return handler.forward(process)
}

func criuOptions(context *cli.Context) *libcontainer.CriuOpts {
	imagePath := getCheckpointImagePath(context)
	if err := os.MkdirAll(imagePath, 0655); err != nil {
		fatal(err)
	}
	return &libcontainer.CriuOpts{
		ImagesDirectory:         imagePath,
		WorkDirectory:           context.String("work-path"),
		LeaveRunning:            context.Bool("leave-running"),
		TcpEstablished:          context.Bool("tcp-established"),
		ExternalUnixConnections: context.Bool("ext-unix-sk"),
		ShellJob:                context.Bool("shell-job"),
		FileLocks:               context.Bool("file-locks"),
	}
}
<|endoftext|>


<commit_before>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Marshalling and unmarshalling of
// JSON data into Go structs using reflection.

package json

import (
	"bytes"
	"fmt"
	"io"
	"os"
	"reflect"
	"strings"
)

type structBuilder struct {
	val reflect.Value

	// if map_ != nil, write val to map_[key] on each change
	map_ *reflect.MapValue
	key  reflect.Value
}

var nobuilder *structBuilder

func isfloat(v reflect.Value) bool {
	switch v.(type) {
	case *reflect.FloatValue, *reflect.Float32Value, *reflect.Float64Value:
		return true
	}
	return false
}

func setfloat(v reflect.Value, f float64) {
	switch v := v.(type) {
	case *reflect.FloatValue:
		v.Set(float(f))
	case *reflect.Float32Value:
		v.Set(float32(f))
	case *reflect.Float64Value:
		v.Set(float64(f))
	}
}

func setint(v reflect.Value, i int64) {
	switch v := v.(type) {
	case *reflect.IntValue:
		v.Set(int(i))
	case *reflect.Int8Value:
		v.Set(int8(i))
	case *reflect.Int16Value:
		v.Set(int16(i))
	case *reflect.Int32Value:
		v.Set(int32(i))
	case *reflect.Int64Value:
		v.Set(int64(i))
	case *reflect.UintValue:
		v.Set(uint(i))
	case *reflect.Uint8Value:
		v.Set(uint8(i))
	case *reflect.Uint16Value:
		v.Set(uint16(i))
	case *reflect.Uint32Value:
		v.Set(uint32(i))
	case *reflect.Uint64Value:
		v.Set(uint64(i))
	}
}

// If updating b.val is not enough to update the original,
// copy a changed b.val out to the original.
func (b *structBuilder) Flush() {
	if b == nil {
		return
	}
	if b.map_ != nil {
		b.map_.SetElem(b.key, b.val)
	}
}

func (b *structBuilder) Int64(i int64) {
	if b == nil {
		return
	}
	v := b.val
	if isfloat(v) {
		setfloat(v, float64(i))
	} else {
		setint(v, i)
	}
}

func (b *structBuilder) Uint64(i uint64) {
	if b == nil {
		return
	}
	v := b.val
	if isfloat(v) {
		setfloat(v, float64(i))
	} else {
		setint(v, int64(i))
	}
}

func (b *structBuilder) Float64(f float64) {
	if b == nil {
		return
	}
	v := b.val
	if isfloat(v) {
		setfloat(v, f)
	} else {
		setint(v, int64(f))
	}
}

func (b *structBuilder) Null() {}

func (b *structBuilder) String(s string) {
	if b == nil {
		return
	}
	if v, ok := b.val.(*reflect.StringValue); ok {
		v.Set(s)
	}
}

func (b *structBuilder) Bool(tf bool) {
	if b == nil {
		return
	}
	if v, ok := b.val.(*reflect.BoolValue); ok {
		v.Set(tf)
	}
}

func (b *structBuilder) Array() {
	if b == nil {
		return
	}
	if v, ok := b.val.(*reflect.SliceValue); ok {
		if v.IsNil() {
			v.Set(reflect.MakeSlice(v.Type().(*reflect.SliceType), 0, 8))
		}
	}
}

func (b *structBuilder) Elem(i int) Builder {
	if b == nil || i < 0 {
		return nobuilder
	}
	switch v := b.val.(type) {
	case *reflect.ArrayValue:
		if i < v.Len() {
			return &structBuilder{val: v.Elem(i)}
		}
	case *reflect.SliceValue:
		if i >= v.Cap() {
			n := v.Cap()
			if n < 8 {
				n = 8
			}
			for n <= i {
				n *= 2
			}
			nv := reflect.MakeSlice(v.Type().(*reflect.SliceType), v.Len(), n)
			reflect.ArrayCopy(nv, v)
			v.Set(nv)
		}
		if v.Len() <= i && i < v.Cap() {
			v.SetLen(i + 1)
		}
		if i < v.Len() {
			return &structBuilder{val: v.Elem(i)}
		}
	}
	return nobuilder
}

func (b *structBuilder) Map() {
	if b == nil {
		return
	}
	if v, ok := b.val.(*reflect.PtrValue); ok && v.IsNil() {
		if v.IsNil() {
			v.PointTo(reflect.MakeZero(v.Type().(*reflect.PtrType).Elem()))
			b.Flush()
		}
		b.map_ = nil
		b.val = v.Elem()
	}
	if v, ok := b.val.(*reflect.MapValue); ok && v.IsNil() {
		v.Set(reflect.MakeMap(v.Type().(*reflect.MapType)))
	}
}

func (b *structBuilder) Key(k string) Builder {
	if b == nil {
		return nobuilder
	}
	switch v := reflect.Indirect(b.val).(type) {
	case *reflect.StructValue:
		t := v.Type().(*reflect.StructType)
		// Case-insensitive field lookup.
		k = strings.ToLower(k)
		for i := 0; i < t.NumField(); i++ {
			if strings.ToLower(t.Field(i).Name) == k {
				return &structBuilder{val: v.Field(i)}
			}
		}
	case *reflect.MapValue:
		t := v.Type().(*reflect.MapType)
		if t.Key() != reflect.Typeof(k) {
			break
		}
		key := reflect.NewValue(k)
		elem := v.Elem(key)
		if elem == nil {
			v.SetElem(key, reflect.MakeZero(t.Elem()))
			elem = v.Elem(key)
		}
		return &structBuilder{val: elem, map_: v, key: key}
	}
	return nobuilder
}

// Unmarshal parses the JSON syntax string s and fills in
// an arbitrary struct or slice pointed at by val.
// It uses the reflect package to assign to fields
// and arrays embedded in val.  Well-formed data that does not fit
// into the struct is discarded.
//
// For example, given these definitions:
//
//	type Email struct {
//		Where string;
//		Addr string;
//	}
//
//	type Result struct {
//		Name string;
//		Phone string;
//		Email []Email
//	}
//
//	var r = Result{ "name", "phone", nil }
//
// unmarshalling the JSON syntax string
//
//	{
//	  "email": [
//	    {
//	      "where": "home",
//	      "addr": "gre@example.com"
//	    },
//	    {
//	      "where": "work",
//	      "addr": "gre@work.com"
//	    }
//	  ],
//	  "name": "Grace R. Emlin",
//	  "address": "123 Main Street"
//	}
//
// via Unmarshal(s, &r) is equivalent to assigning
//
//	r = Result{
//		"Grace R. Emlin",	// name
//		"phone",		// no phone given
//		[]Email{
//			Email{ "home", "gre@example.com" },
//			Email{ "work", "gre@work.com" }
//		}
//	}
//
// Note that the field r.Phone has not been modified and
// that the JSON field "address" was discarded.
//
// Because Unmarshal uses the reflect package, it can only
// assign to upper case fields.  Unmarshal uses a case-insensitive
// comparison to match JSON field names to struct field names.
//
// To unmarshal a top-level JSON array, pass in a pointer to an empty
// slice of the correct type.
//
// On success, Unmarshal returns with ok set to true.
// On a syntax error, it returns with ok set to false and errtok
// set to the offending token.
func Unmarshal(s string, val interface{}) (ok bool, errtok string) {
	v := reflect.NewValue(val)
	var b *structBuilder

	// If val is a pointer to a slice, we append to the slice.
	if ptr, ok := v.(*reflect.PtrValue); ok {
		if slice, ok := ptr.Elem().(*reflect.SliceValue); ok {
			b = &structBuilder{val: slice}
		}
	}

	if b == nil {
		b = &structBuilder{val: v}
	}

	ok, _, errtok = Parse(s, b)
	if !ok {
		return false, errtok
	}
	return true, ""
}

type MarshalError struct {
	T reflect.Type
}

func (e *MarshalError) String() string {
	return "json cannot encode value of type " + e.T.String()
}

type writeState struct {
	bytes.Buffer
	indent   string
	newlines bool
	depth    int
}

func (s *writeState) descend(bra byte) {
	s.depth++
	s.WriteByte(bra)
}

func (s *writeState) ascend(ket byte) {
	s.depth--
	s.writeIndent()
	s.WriteByte(ket)
}

func (s *writeState) writeIndent() {
	if s.newlines {
		s.WriteByte('\n')
	}
	for i := 0; i < s.depth; i++ {
		s.WriteString(s.indent)
	}
}

func (s *writeState) writeArrayOrSlice(val reflect.ArrayOrSliceValue) {
	s.descend('[')

	for i := 0; i < val.Len(); i++ {
		s.writeIndent()
		s.writeValue(val.Elem(i))
		if i < val.Len()-1 {
			s.WriteByte(',')
		}
	}

	s.ascend(']')
}

func (s *writeState) writeMap(val *reflect.MapValue) {
	key := val.Type().(*reflect.MapType).Key()
	if _, ok := key.(*reflect.StringType); !ok {
		panic(&MarshalError{val.Type()})
	}

	s.descend('{')

	keys := val.Keys()
	for i := 0; i < len(keys); i++ {
		s.writeIndent()
		fmt.Fprintf(s, "%s:", Quote(keys[i].(*reflect.StringValue).Get()))
		s.writeValue(val.Elem(keys[i]))
		if i < len(keys)-1 {
			s.WriteByte(',')
		}
	}

	s.ascend('}')
}

func (s *writeState) writeStruct(val *reflect.StructValue) {
	s.descend('{')

	typ := val.Type().(*reflect.StructType)

	for i := 0; i < val.NumField(); i++ {
		s.writeIndent()
		fmt.Fprintf(s, "%s:", Quote(typ.Field(i).Name))
		s.writeValue(val.Field(i))
		if i < val.NumField()-1 {
			s.WriteByte(',')
		}
	}

	s.ascend('}')
}

func (s *writeState) writeValue(val reflect.Value) {
	if val == nil {
		fmt.Fprint(s, "null")
		return
	}

	switch v := val.(type) {
	case *reflect.StringValue:
		fmt.Fprint(s, Quote(v.Get()))
	case *reflect.ArrayValue:
		s.writeArrayOrSlice(v)
	case *reflect.SliceValue:
		s.writeArrayOrSlice(v)
	case *reflect.MapValue:
		s.writeMap(v)
	case *reflect.StructValue:
		s.writeStruct(v)
	case *reflect.ChanValue,
		*reflect.UnsafePointerValue,
		*reflect.FuncValue:
		panic(&MarshalError{val.Type()})
	case *reflect.InterfaceValue:
		if v.IsNil() {
			fmt.Fprint(s, "null")
		} else {
			s.writeValue(v.Elem())
		}
	case *reflect.PtrValue:
		if v.IsNil() {
			fmt.Fprint(s, "null")
		} else {
			s.writeValue(v.Elem())
		}
	case *reflect.UintptrValue:
		fmt.Fprintf(s, "%d", v.Get())
	case *reflect.Uint64Value:
		fmt.Fprintf(s, "%d", v.Get())
	case *reflect.Uint32Value:
		fmt.Fprintf(s, "%d", v.Get())
	case *reflect.Uint16Value:
		fmt.Fprintf(s, "%d", v.Get())
	case *reflect.Uint8Value:
		fmt.Fprintf(s, "%d", v.Get())
	default:
		value := val.(reflect.Value)
		fmt.Fprintf(s, "%#v", value.Interface())
	}
}

func (s *writeState) marshal(w io.Writer, val interface{}) (err os.Error) {
	defer func() {
		if e := recover(); e != nil {
			err = e.(*MarshalError)
		}
	}()
	s.writeValue(reflect.NewValue(val))
	if s.newlines {
		s.WriteByte('\n')
	}
	_, err = s.WriteTo(w)
	return
}

// Marshal writes the JSON encoding of val to w.
//
// Due to limitations in JSON, val cannot include cyclic data
// structures, channels, functions, or maps.
func Marshal(w io.Writer, val interface{}) os.Error {
	s := &writeState{indent: "", newlines: false, depth: 0}
	return s.marshal(w, val)
}

// MarshalIndent writes the JSON encoding of val to w,
// indenting nested values using the indent string.
//
// Due to limitations in JSON, val cannot include cyclic data
// structures, channels, functions, or maps.
func MarshalIndent(w io.Writer, val interface{}, indent string) os.Error {
	s := &writeState{indent: indent, newlines: true, depth: 0}
	return s.marshal(w, val)
}
<commit_msg>json: update documentation to match current coding style<commit_after>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Marshalling and unmarshalling of
// JSON data into Go structs using reflection.

package json

import (
	"bytes"
	"fmt"
	"io"
	"os"
	"reflect"
	"strings"
)

type structBuilder struct {
	val reflect.Value

	// if map_ != nil, write val to map_[key] on each change
	map_ *reflect.MapValue
	key  reflect.Value
}

var nobuilder *structBuilder

func isfloat(v reflect.Value) bool {
	switch v.(type) {
	case *reflect.FloatValue, *reflect.Float32Value, *reflect.Float64Value:
		return true
	}
	return false
}

func setfloat(v reflect.Value, f float64) {
	switch v := v.(type) {
	case *reflect.FloatValue:
		v.Set(float(f))
	case *reflect.Float32Value:
		v.Set(float32(f))
	case *reflect.Float64Value:
		v.Set(float64(f))
	}
}

func setint(v reflect.Value, i int64) {
	switch v := v.(type) {
	case *reflect.IntValue:
		v.Set(int(i))
	case *reflect.Int8Value:
		v.Set(int8(i))
	case *reflect.Int16Value:
		v.Set(int16(i))
	case *reflect.Int32Value:
		v.Set(int32(i))
	case *reflect.Int64Value:
		v.Set(int64(i))
	case *reflect.UintValue:
		v.Set(uint(i))
	case *reflect.Uint8Value:
		v.Set(uint8(i))
	case *reflect.Uint16Value:
		v.Set(uint16(i))
	case *reflect.Uint32Value:
		v.Set(uint32(i))
	case *reflect.Uint64Value:
		v.Set(uint64(i))
	}
}

// If updating b.val is not enough to update the original,
// copy a changed b.val out to the original.
func (b *structBuilder) Flush() {
	if b == nil {
		return
	}
	if b.map_ != nil {
		b.map_.SetElem(b.key, b.val)
	}
}

func (b *structBuilder) Int64(i int64) {
	if b == nil {
		return
	}
	v := b.val
	if isfloat(v) {
		setfloat(v, float64(i))
	} else {
		setint(v, i)
	}
}

func (b *structBuilder) Uint64(i uint64) {
	if b == nil {
		return
	}
	v := b.val
	if isfloat(v) {
		setfloat(v, float64(i))
	} else {
		setint(v, int64(i))
	}
}

func (b *structBuilder) Float64(f float64) {
	if b == nil {
		return
	}
	v := b.val
	if isfloat(v) {
		setfloat(v, f)
	} else {
		setint(v, int64(f))
	}
}

func (b *structBuilder) Null() {}

func (b *structBuilder) String(s string) {
	if b == nil {
		return
	}
	if v, ok := b.val.(*reflect.StringValue); ok {
		v.Set(s)
	}
}

func (b *structBuilder) Bool(tf bool) {
	if b == nil {
		return
	}
	if v, ok := b.val.(*reflect.BoolValue); ok {
		v.Set(tf)
	}
}

func (b *structBuilder) Array() {
	if b == nil {
		return
	}
	if v, ok := b.val.(*reflect.SliceValue); ok {
		if v.IsNil() {
			v.Set(reflect.MakeSlice(v.Type().(*reflect.SliceType), 0, 8))
		}
	}
}

func (b *structBuilder) Elem(i int) Builder {
	if b == nil || i < 0 {
		return nobuilder
	}
	switch v := b.val.(type) {
	case *reflect.ArrayValue:
		if i < v.Len() {
			return &structBuilder{val: v.Elem(i)}
		}
	case *reflect.SliceValue:
		if i >= v.Cap() {
			n := v.Cap()
			if n < 8 {
				n = 8
			}
			for n <= i {
				n *= 2
			}
			nv := reflect.MakeSlice(v.Type().(*reflect.SliceType), v.Len(), n)
			reflect.ArrayCopy(nv, v)
			v.Set(nv)
		}
		if v.Len() <= i && i < v.Cap() {
			v.SetLen(i + 1)
		}
		if i < v.Len() {
			return &structBuilder{val: v.Elem(i)}
		}
	}
	return nobuilder
}

func (b *structBuilder) Map() {
	if b == nil {
		return
	}
	if v, ok := b.val.(*reflect.PtrValue); ok && v.IsNil() {
		if v.IsNil() {
			v.PointTo(reflect.MakeZero(v.Type().(*reflect.PtrType).Elem()))
			b.Flush()
		}
		b.map_ = nil
		b.val = v.Elem()
	}
	if v, ok := b.val.(*reflect.MapValue); ok && v.IsNil() {
		v.Set(reflect.MakeMap(v.Type().(*reflect.MapType)))
	}
}

func (b *structBuilder) Key(k string) Builder {
	if b == nil {
		return nobuilder
	}
	switch v := reflect.Indirect(b.val).(type) {
	case *reflect.StructValue:
		t := v.Type().(*reflect.StructType)
		// Case-insensitive field lookup.
		k = strings.ToLower(k)
		for i := 0; i < t.NumField(); i++ {
			if strings.ToLower(t.Field(i).Name) == k {
				return &structBuilder{val: v.Field(i)}
			}
		}
	case *reflect.MapValue:
		t := v.Type().(*reflect.MapType)
		if t.Key() != reflect.Typeof(k) {
			break
		}
		key := reflect.NewValue(k)
		elem := v.Elem(key)
		if elem == nil {
			v.SetElem(key, reflect.MakeZero(t.Elem()))
			elem = v.Elem(key)
		}
		return &structBuilder{val: elem, map_: v, key: key}
	}
	return nobuilder
}

// Unmarshal parses the JSON syntax string s and fills in
// an arbitrary struct or slice pointed at by val.
// It uses the reflect package to assign to fields
// and arrays embedded in val.  Well-formed data that does not fit
// into the struct is discarded.
//
// For example, given these definitions:
//
//	type Email struct {
//		Where string
//		Addr string
//	}
//
//	type Result struct {
//		Name string
//		Phone string
//		Email []Email
//	}
//
//	var r = Result{ "name", "phone", nil }
//
// unmarshalling the JSON syntax string
//
//	{
//	  "email": [
//	    {
//	      "where": "home",
//	      "addr": "gre@example.com"
//	    },
//	    {
//	      "where": "work",
//	      "addr": "gre@work.com"
//	    }
//	  ],
//	  "name": "Grace R. Emlin",
//	  "address": "123 Main Street"
//	}
//
// via Unmarshal(s, &r) is equivalent to assigning
//
//	r = Result{
//		"Grace R. Emlin",	// name
//		"phone",		// no phone given
//		[]Email{
//			Email{ "home", "gre@example.com" },
//			Email{ "work", "gre@work.com" },
//		},
//	}
//
// Note that the field r.Phone has not been modified and
// that the JSON field "address" was discarded.
//
// Because Unmarshal uses the reflect package, it can only
// assign to upper case fields.  Unmarshal uses a case-insensitive
// comparison to match JSON field names to struct field names.
//
// To unmarshal a top-level JSON array, pass in a pointer to an empty
// slice of the correct type.
//
// On success, Unmarshal returns with ok set to true.
// On a syntax error, it returns with ok set to false and errtok
// set to the offending token.
func Unmarshal(s string, val interface{}) (ok bool, errtok string) {
	v := reflect.NewValue(val)
	var b *structBuilder

	// If val is a pointer to a slice, we append to the slice.
	if ptr, ok := v.(*reflect.PtrValue); ok {
		if slice, ok := ptr.Elem().(*reflect.SliceValue); ok {
			b = &structBuilder{val: slice}
		}
	}

	if b == nil {
		b = &structBuilder{val: v}
	}

	ok, _, errtok = Parse(s, b)
	if !ok {
		return false, errtok
	}
	return true, ""
}

type MarshalError struct {
	T reflect.Type
}

func (e *MarshalError) String() string {
	return "json cannot encode value of type " + e.T.String()
}

type writeState struct {
	bytes.Buffer
	indent   string
	newlines bool
	depth    int
}

func (s *writeState) descend(bra byte) {
	s.depth++
	s.WriteByte(bra)
}

func (s *writeState) ascend(ket byte) {
	s.depth--
	s.writeIndent()
	s.WriteByte(ket)
}

func (s *writeState) writeIndent() {
	if s.newlines {
		s.WriteByte('\n')
	}
	for i := 0; i < s.depth; i++ {
		s.WriteString(s.indent)
	}
}

func (s *writeState) writeArrayOrSlice(val reflect.ArrayOrSliceValue) {
	s.descend('[')

	for i := 0; i < val.Len(); i++ {
		s.writeIndent()
		s.writeValue(val.Elem(i))
		if i < val.Len()-1 {
			s.WriteByte(',')
		}
	}

	s.ascend(']')
}

func (s *writeState) writeMap(val *reflect.MapValue) {
	key := val.Type().(*reflect.MapType).Key()
	if _, ok := key.(*reflect.StringType); !ok {
		panic(&MarshalError{val.Type()})
	}

	s.descend('{')

	keys := val.Keys()
	for i := 0; i < len(keys); i++ {
		s.writeIndent()
		fmt.Fprintf(s, "%s:", Quote(keys[i].(*reflect.StringValue).Get()))
		s.writeValue(val.Elem(keys[i]))
		if i < len(keys)-1 {
			s.WriteByte(',')
		}
	}

	s.ascend('}')
}

func (s *writeState) writeStruct(val *reflect.StructValue) {
	s.descend('{')

	typ := val.Type().(*reflect.StructType)

	for i := 0; i < val.NumField(); i++ {
		s.writeIndent()
		fmt.Fprintf(s, "%s:", Quote(typ.Field(i).Name))
		s.writeValue(val.Field(i))
		if i < val.NumField()-1 {
			s.WriteByte(',')
		}
	}

	s.ascend('}')
}

func (s *writeState) writeValue(val reflect.Value) {
	if val == nil {
		fmt.Fprint(s, "null")
		return
	}

	switch v := val.(type) {
	case *reflect.StringValue:
		fmt.Fprint(s, Quote(v.Get()))
	case *reflect.ArrayValue:
		s.writeArrayOrSlice(v)
	case *reflect.SliceValue:
		s.writeArrayOrSlice(v)
	case *reflect.MapValue:
		s.writeMap(v)
	case *reflect.StructValue:
		s.writeStruct(v)
	case *reflect.ChanValue,
		*reflect.UnsafePointerValue,
		*reflect.FuncValue:
		panic(&MarshalError{val.Type()})
	case *reflect.InterfaceValue:
		if v.IsNil() {
			fmt.Fprint(s, "null")
		} else {
			s.writeValue(v.Elem())
		}
	case *reflect.PtrValue:
		if v.IsNil() {
			fmt.Fprint(s, "null")
		} else {
			s.writeValue(v.Elem())
		}
	case *reflect.UintptrValue:
		fmt.Fprintf(s, "%d", v.Get())
	case *reflect.Uint64Value:
		fmt.Fprintf(s, "%d", v.Get())
	case *reflect.Uint32Value:
		fmt.Fprintf(s, "%d", v.Get())
	case *reflect.Uint16Value:
		fmt.Fprintf(s, "%d", v.Get())
	case *reflect.Uint8Value:
		fmt.Fprintf(s, "%d", v.Get())
	default:
		value := val.(reflect.Value)
		fmt.Fprintf(s, "%#v", value.Interface())
	}
}

func (s *writeState) marshal(w io.Writer, val interface{}) (err os.Error) {
	defer func() {
		if e := recover(); e != nil {
			err = e.(*MarshalError)
		}
	}()
	s.writeValue(reflect.NewValue(val))
	if s.newlines {
		s.WriteByte('\n')
	}
	_, err = s.WriteTo(w)
	return
}

// Marshal writes the JSON encoding of val to w.
//
// Due to limitations in JSON, val cannot include cyclic data
// structures, channels, functions, or maps.
func Marshal(w io.Writer, val interface{}) os.Error {
	s := &writeState{indent: "", newlines: false, depth: 0}
	return s.marshal(w, val)
}

// MarshalIndent writes the JSON encoding of val to w,
// indenting nested values using the indent string.
//
// Due to limitations in JSON, val cannot include cyclic data
// structures, channels, functions, or maps.
func MarshalIndent(w io.Writer, val interface{}, indent string) os.Error {
	s := &writeState{indent: indent, newlines: true, depth: 0}
	return s.marshal(w, val)
}
<|endoftext|>


<commit_before>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// HTTP file system request handler

package http

import (
	"errors"
	"fmt"
	"io"
	"mime"
	"os"
	"path"
	"path/filepath"
	"strconv"
	"strings"
	"time"
)

// A Dir implements http.FileSystem using the native file
// system restricted to a specific directory tree.
//
// An empty Dir is treated as ".".
type Dir string

func (d Dir) Open(name string) (File, error) {
	if filepath.Separator != '/' && strings.IndexRune(name, filepath.Separator) >= 0 {
		return nil, errors.New("http: invalid character in file path")
	}
	dir := string(d)
	if dir == "" {
		dir = "."
	}
	f, err := os.Open(filepath.Join(dir, filepath.FromSlash(path.Clean("/"+name))))
	if err != nil {
		return nil, err
	}
	return f, nil
}

// A FileSystem implements access to a collection of named files.
// The elements in a file path are separated by slash ('/', U+002F)
// characters, regardless of host operating system convention.
type FileSystem interface {
	Open(name string) (File, error)
}

// A File is returned by a FileSystem's Open method and can be
// served by the FileServer implementation.
type File interface {
	Close() error
	Stat() (os.FileInfo, error)
	Readdir(count int) ([]os.FileInfo, error)
	Read([]byte) (int, error)
	Seek(offset int64, whence int) (int64, error)
}

func dirList(w ResponseWriter, f File) {
	w.Header().Set("Content-Type", "text/html; charset=utf-8")
	fmt.Fprintf(w, "
\n")
	for {
		dirs, err := f.Readdir(100)
		if err != nil || len(dirs) == 0 {
			break
		}
		for _, d := range dirs {
			name := d.Name()
			if d.IsDir() {
				name += "/"
			}
			// TODO htmlescape
			fmt.Fprintf(w, "%s\n", name, name)
		}
	}
	fmt.Fprintf(w, "
\n")
}

// ServeContent replies to the request using the content in the
// provided ReadSeeker.  The main benefit of ServeContent over io.Copy
// is that it handles Range requests properly, sets the MIME type, and
// handles If-Modified-Since requests.
//
// If the response's Content-Type header is not set, ServeContent
// first tries to deduce the type from name's file extension and,
// if that fails, falls back to reading the first block of the content
// and passing it to DetectContentType.
// The name is otherwise unused; in particular it can be empty and is
// never sent in the response.
//
// If modtime is not the zero time, ServeContent includes it in a
// Last-Modified header in the response.  If the request includes an
// If-Modified-Since header, ServeContent uses modtime to decide
// whether the content needs to be sent at all.
//
// The content's Seek method must work: ServeContent uses
// a seek to the end of the content to determine its size.
//
// Note that *os.File implements the io.ReadSeeker interface.
func ServeContent(w ResponseWriter, req *Request, name string, modtime time.Time, content io.ReadSeeker) {
	size, err := content.Seek(0, os.SEEK_END)
	if err != nil {
		Error(w, "seeker can't seek", StatusInternalServerError)
		return
	}
	_, err = content.Seek(0, os.SEEK_SET)
	if err != nil {
		Error(w, "seeker can't seek", StatusInternalServerError)
		return
	}
	serveContent(w, req, name, modtime, size, content)
}

// if name is empty, filename is unknown. (used for mime type, before sniffing)
// if modtime.IsZero(), modtime is unknown.
// content must be seeked to the beginning of the file.
func serveContent(w ResponseWriter, r *Request, name string, modtime time.Time, size int64, content io.ReadSeeker) {
	if checkLastModified(w, r, modtime) {
		return
	}

	code := StatusOK

	// If Content-Type isn't set, use the file's extension to find it.
	if w.Header().Get("Content-Type") == "" {
		ctype := mime.TypeByExtension(filepath.Ext(name))
		if ctype == "" {
			// read a chunk to decide between utf-8 text and binary
			var buf [1024]byte
			n, _ := io.ReadFull(content, buf[:])
			b := buf[:n]
			ctype = DetectContentType(b)
			_, err := content.Seek(0, os.SEEK_SET) // rewind to output whole file
			if err != nil {
				Error(w, "seeker can't seek", StatusInternalServerError)
				return
			}
		}
		w.Header().Set("Content-Type", ctype)
	}

	// handle Content-Range header.
	// TODO(adg): handle multiple ranges
	sendSize := size
	if size >= 0 {
		ranges, err := parseRange(r.Header.Get("Range"), size)
		if err == nil && len(ranges) > 1 {
			err = errors.New("multiple ranges not supported")
		}
		if err != nil {
			Error(w, err.Error(), StatusRequestedRangeNotSatisfiable)
			return
		}
		if len(ranges) == 1 {
			ra := ranges[0]
			if _, err := content.Seek(ra.start, os.SEEK_SET); err != nil {
				Error(w, err.Error(), StatusRequestedRangeNotSatisfiable)
				return
			}
			sendSize = ra.length
			code = StatusPartialContent
			w.Header().Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", ra.start, ra.start+ra.length-1, size))
		}

		w.Header().Set("Accept-Ranges", "bytes")
		if w.Header().Get("Content-Encoding") == "" {
			w.Header().Set("Content-Length", strconv.FormatInt(sendSize, 10))
		}
	}

	w.WriteHeader(code)

	if r.Method != "HEAD" {
		if sendSize == -1 {
			io.Copy(w, content)
		} else {
			io.CopyN(w, content, sendSize)
		}
	}
}

// modtime is the modification time of the resource to be served, or IsZero().
// return value is whether this request is now complete.
func checkLastModified(w ResponseWriter, r *Request, modtime time.Time) bool {
	if modtime.IsZero() {
		return false
	}
	if t, err := time.Parse(TimeFormat, r.Header.Get("If-Modified-Since")); err == nil && modtime.After(t) {
		w.WriteHeader(StatusNotModified)
		return true
	}
	w.Header().Set("Last-Modified", modtime.UTC().Format(TimeFormat))
	return false
}

// name is '/'-separated, not filepath.Separator.
func serveFile(w ResponseWriter, r *Request, fs FileSystem, name string, redirect bool) {
	const indexPage = "/index.html"

	// redirect .../index.html to .../
	// can't use Redirect() because that would make the path absolute,
	// which would be a problem running under StripPrefix
	if strings.HasSuffix(r.URL.Path, indexPage) {
		localRedirect(w, r, "./")
		return
	}

	f, err := fs.Open(name)
	if err != nil {
		// TODO expose actual error?
		NotFound(w, r)
		return
	}
	defer f.Close()

	d, err1 := f.Stat()
	if err1 != nil {
		// TODO expose actual error?
		NotFound(w, r)
		return
	}

	if redirect {
		// redirect to canonical path: / at end of directory url
		// r.URL.Path always begins with /
		url := r.URL.Path
		if d.IsDir() {
			if url[len(url)-1] != '/' {
				localRedirect(w, r, path.Base(url)+"/")
				return
			}
		} else {
			if url[len(url)-1] == '/' {
				localRedirect(w, r, "../"+path.Base(url))
				return
			}
		}
	}

	// use contents of index.html for directory, if present
	if d.IsDir() {
		if checkLastModified(w, r, d.ModTime()) {
			return
		}
		index := name + indexPage
		ff, err := fs.Open(index)
		if err == nil {
			defer ff.Close()
			dd, err := ff.Stat()
			if err == nil {
				name = index
				d = dd
				f = ff
			}
		}
	}

	if d.IsDir() {
		dirList(w, f)
		return
	}

	serveContent(w, r, d.Name(), d.ModTime(), d.Size(), f)
}

// localRedirect gives a Moved Permanently response.
// It does not convert relative paths to absolute paths like Redirect does.
func localRedirect(w ResponseWriter, r *Request, newPath string) {
	if q := r.URL.RawQuery; q != "" {
		newPath += "?" + q
	}
	w.Header().Set("Location", newPath)
	w.WriteHeader(StatusMovedPermanently)
}

// ServeFile replies to the request with the contents of the named file or directory.
func ServeFile(w ResponseWriter, r *Request, name string) {
	dir, file := filepath.Split(name)
	serveFile(w, r, Dir(dir), file, false)
}

type fileHandler struct {
	root FileSystem
}

// FileServer returns a handler that serves HTTP requests
// with the contents of the file system rooted at root.
//
// To use the operating system's file system implementation,
// use http.Dir:
//
//     http.Handle("/", http.FileServer(http.Dir("/tmp")))
func FileServer(root FileSystem) Handler {
	return &fileHandler{root}
}

func (f *fileHandler) ServeHTTP(w ResponseWriter, r *Request) {
	upath := r.URL.Path
	if !strings.HasPrefix(upath, "/") {
		upath = "/" + upath
		r.URL.Path = upath
	}
	serveFile(w, r, f.root, path.Clean(upath), true)
}

// httpRange specifies the byte range to be sent to the client.
type httpRange struct {
	start, length int64
}

// parseRange parses a Range header string as per RFC 2616.
func parseRange(s string, size int64) ([]httpRange, error) {
	if s == "" {
		return nil, nil // header not present
	}
	const b = "bytes="
	if !strings.HasPrefix(s, b) {
		return nil, errors.New("invalid range")
	}
	var ranges []httpRange
	for _, ra := range strings.Split(s[len(b):], ",") {
		i := strings.Index(ra, "-")
		if i < 0 {
			return nil, errors.New("invalid range")
		}
		start, end := ra[:i], ra[i+1:]
		var r httpRange
		if start == "" {
			// If no start is specified, end specifies the
			// range start relative to the end of the file.
			i, err := strconv.ParseInt(end, 10, 64)
			if err != nil {
				return nil, errors.New("invalid range")
			}
			if i > size {
				i = size
			}
			r.start = size - i
			r.length = size - r.start
		} else {
			i, err := strconv.ParseInt(start, 10, 64)
			if err != nil || i > size || i < 0 {
				return nil, errors.New("invalid range")
			}
			r.start = i
			if end == "" {
				// If no end is specified, range extends to end of the file.
				r.length = size - r.start
			} else {
				i, err := strconv.ParseInt(end, 10, 64)
				if err != nil || r.start > i {
					return nil, errors.New("invalid range")
				}
				if i >= size {
					i = size - 1
				}
				r.length = i - r.start + 1
			}
		}
		ranges = append(ranges, r)
	}
	return ranges, nil
}
<commit_msg>net/http: use mtime < t+1s to check for unmodified<commit_after>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// HTTP file system request handler

package http

import (
	"errors"
	"fmt"
	"io"
	"mime"
	"os"
	"path"
	"path/filepath"
	"strconv"
	"strings"
	"time"
)

// A Dir implements http.FileSystem using the native file
// system restricted to a specific directory tree.
//
// An empty Dir is treated as ".".
type Dir string

func (d Dir) Open(name string) (File, error) {
	if filepath.Separator != '/' && strings.IndexRune(name, filepath.Separator) >= 0 {
		return nil, errors.New("http: invalid character in file path")
	}
	dir := string(d)
	if dir == "" {
		dir = "."
	}
	f, err := os.Open(filepath.Join(dir, filepath.FromSlash(path.Clean("/"+name))))
	if err != nil {
		return nil, err
	}
	return f, nil
}

// A FileSystem implements access to a collection of named files.
// The elements in a file path are separated by slash ('/', U+002F)
// characters, regardless of host operating system convention.
type FileSystem interface {
	Open(name string) (File, error)
}

// A File is returned by a FileSystem's Open method and can be
// served by the FileServer implementation.
type File interface {
	Close() error
	Stat() (os.FileInfo, error)
	Readdir(count int) ([]os.FileInfo, error)
	Read([]byte) (int, error)
	Seek(offset int64, whence int) (int64, error)
}

func dirList(w ResponseWriter, f File) {
	w.Header().Set("Content-Type", "text/html; charset=utf-8")
	fmt.Fprintf(w, "
\n")
	for {
		dirs, err := f.Readdir(100)
		if err != nil || len(dirs) == 0 {
			break
		}
		for _, d := range dirs {
			name := d.Name()
			if d.IsDir() {
				name += "/"
			}
			// TODO htmlescape
			fmt.Fprintf(w, "%s\n", name, name)
		}
	}
	fmt.Fprintf(w, "
\n")
}

// ServeContent replies to the request using the content in the
// provided ReadSeeker.  The main benefit of ServeContent over io.Copy
// is that it handles Range requests properly, sets the MIME type, and
// handles If-Modified-Since requests.
//
// If the response's Content-Type header is not set, ServeContent
// first tries to deduce the type from name's file extension and,
// if that fails, falls back to reading the first block of the content
// and passing it to DetectContentType.
// The name is otherwise unused; in particular it can be empty and is
// never sent in the response.
//
// If modtime is not the zero time, ServeContent includes it in a
// Last-Modified header in the response.  If the request includes an
// If-Modified-Since header, ServeContent uses modtime to decide
// whether the content needs to be sent at all.
//
// The content's Seek method must work: ServeContent uses
// a seek to the end of the content to determine its size.
//
// Note that *os.File implements the io.ReadSeeker interface.
func ServeContent(w ResponseWriter, req *Request, name string, modtime time.Time, content io.ReadSeeker) {
	size, err := content.Seek(0, os.SEEK_END)
	if err != nil {
		Error(w, "seeker can't seek", StatusInternalServerError)
		return
	}
	_, err = content.Seek(0, os.SEEK_SET)
	if err != nil {
		Error(w, "seeker can't seek", StatusInternalServerError)
		return
	}
	serveContent(w, req, name, modtime, size, content)
}

// if name is empty, filename is unknown. (used for mime type, before sniffing)
// if modtime.IsZero(), modtime is unknown.
// content must be seeked to the beginning of the file.
func serveContent(w ResponseWriter, r *Request, name string, modtime time.Time, size int64, content io.ReadSeeker) {
	if checkLastModified(w, r, modtime) {
		return
	}

	code := StatusOK

	// If Content-Type isn't set, use the file's extension to find it.
	if w.Header().Get("Content-Type") == "" {
		ctype := mime.TypeByExtension(filepath.Ext(name))
		if ctype == "" {
			// read a chunk to decide between utf-8 text and binary
			var buf [1024]byte
			n, _ := io.ReadFull(content, buf[:])
			b := buf[:n]
			ctype = DetectContentType(b)
			_, err := content.Seek(0, os.SEEK_SET) // rewind to output whole file
			if err != nil {
				Error(w, "seeker can't seek", StatusInternalServerError)
				return
			}
		}
		w.Header().Set("Content-Type", ctype)
	}

	// handle Content-Range header.
	// TODO(adg): handle multiple ranges
	sendSize := size
	if size >= 0 {
		ranges, err := parseRange(r.Header.Get("Range"), size)
		if err == nil && len(ranges) > 1 {
			err = errors.New("multiple ranges not supported")
		}
		if err != nil {
			Error(w, err.Error(), StatusRequestedRangeNotSatisfiable)
			return
		}
		if len(ranges) == 1 {
			ra := ranges[0]
			if _, err := content.Seek(ra.start, os.SEEK_SET); err != nil {
				Error(w, err.Error(), StatusRequestedRangeNotSatisfiable)
				return
			}
			sendSize = ra.length
			code = StatusPartialContent
			w.Header().Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", ra.start, ra.start+ra.length-1, size))
		}

		w.Header().Set("Accept-Ranges", "bytes")
		if w.Header().Get("Content-Encoding") == "" {
			w.Header().Set("Content-Length", strconv.FormatInt(sendSize, 10))
		}
	}

	w.WriteHeader(code)

	if r.Method != "HEAD" {
		if sendSize == -1 {
			io.Copy(w, content)
		} else {
			io.CopyN(w, content, sendSize)
		}
	}
}

// modtime is the modification time of the resource to be served, or IsZero().
// return value is whether this request is now complete.
func checkLastModified(w ResponseWriter, r *Request, modtime time.Time) bool {
	if modtime.IsZero() {
		return false
	}

	// The Date-Modified header truncates sub-second precision, so
	// use mtime < t+1s instead of mtime <= t to check for unmodified.
	if t, err := time.Parse(TimeFormat, r.Header.Get("If-Modified-Since")); err == nil && modtime.Before(t.Add(1*time.Second)) {
		w.WriteHeader(StatusNotModified)
		return true
	}
	w.Header().Set("Last-Modified", modtime.UTC().Format(TimeFormat))
	return false
}

// name is '/'-separated, not filepath.Separator.
func serveFile(w ResponseWriter, r *Request, fs FileSystem, name string, redirect bool) {
	const indexPage = "/index.html"

	// redirect .../index.html to .../
	// can't use Redirect() because that would make the path absolute,
	// which would be a problem running under StripPrefix
	if strings.HasSuffix(r.URL.Path, indexPage) {
		localRedirect(w, r, "./")
		return
	}

	f, err := fs.Open(name)
	if err != nil {
		// TODO expose actual error?
		NotFound(w, r)
		return
	}
	defer f.Close()

	d, err1 := f.Stat()
	if err1 != nil {
		// TODO expose actual error?
		NotFound(w, r)
		return
	}

	if redirect {
		// redirect to canonical path: / at end of directory url
		// r.URL.Path always begins with /
		url := r.URL.Path
		if d.IsDir() {
			if url[len(url)-1] != '/' {
				localRedirect(w, r, path.Base(url)+"/")
				return
			}
		} else {
			if url[len(url)-1] == '/' {
				localRedirect(w, r, "../"+path.Base(url))
				return
			}
		}
	}

	// use contents of index.html for directory, if present
	if d.IsDir() {
		if checkLastModified(w, r, d.ModTime()) {
			return
		}
		index := name + indexPage
		ff, err := fs.Open(index)
		if err == nil {
			defer ff.Close()
			dd, err := ff.Stat()
			if err == nil {
				name = index
				d = dd
				f = ff
			}
		}
	}

	if d.IsDir() {
		dirList(w, f)
		return
	}

	serveContent(w, r, d.Name(), d.ModTime(), d.Size(), f)
}

// localRedirect gives a Moved Permanently response.
// It does not convert relative paths to absolute paths like Redirect does.
func localRedirect(w ResponseWriter, r *Request, newPath string) {
	if q := r.URL.RawQuery; q != "" {
		newPath += "?" + q
	}
	w.Header().Set("Location", newPath)
	w.WriteHeader(StatusMovedPermanently)
}

// ServeFile replies to the request with the contents of the named file or directory.
func ServeFile(w ResponseWriter, r *Request, name string) {
	dir, file := filepath.Split(name)
	serveFile(w, r, Dir(dir), file, false)
}

type fileHandler struct {
	root FileSystem
}

// FileServer returns a handler that serves HTTP requests
// with the contents of the file system rooted at root.
//
// To use the operating system's file system implementation,
// use http.Dir:
//
//     http.Handle("/", http.FileServer(http.Dir("/tmp")))
func FileServer(root FileSystem) Handler {
	return &fileHandler{root}
}

func (f *fileHandler) ServeHTTP(w ResponseWriter, r *Request) {
	upath := r.URL.Path
	if !strings.HasPrefix(upath, "/") {
		upath = "/" + upath
		r.URL.Path = upath
	}
	serveFile(w, r, f.root, path.Clean(upath), true)
}

// httpRange specifies the byte range to be sent to the client.
type httpRange struct {
	start, length int64
}

// parseRange parses a Range header string as per RFC 2616.
func parseRange(s string, size int64) ([]httpRange, error) {
	if s == "" {
		return nil, nil // header not present
	}
	const b = "bytes="
	if !strings.HasPrefix(s, b) {
		return nil, errors.New("invalid range")
	}
	var ranges []httpRange
	for _, ra := range strings.Split(s[len(b):], ",") {
		i := strings.Index(ra, "-")
		if i < 0 {
			return nil, errors.New("invalid range")
		}
		start, end := ra[:i], ra[i+1:]
		var r httpRange
		if start == "" {
			// If no start is specified, end specifies the
			// range start relative to the end of the file.
			i, err := strconv.ParseInt(end, 10, 64)
			if err != nil {
				return nil, errors.New("invalid range")
			}
			if i > size {
				i = size
			}
			r.start = size - i
			r.length = size - r.start
		} else {
			i, err := strconv.ParseInt(start, 10, 64)
			if err != nil || i > size || i < 0 {
				return nil, errors.New("invalid range")
			}
			r.start = i
			if end == "" {
				// If no end is specified, range extends to end of the file.
				r.length = size - r.start
			} else {
				i, err := strconv.ParseInt(end, 10, 64)
				if err != nil || r.start > i {
					return nil, errors.New("invalid range")
				}
				if i >= size {
					i = size - 1
				}
				r.length = i - r.start + 1
			}
		}
		ranges = append(ranges, r)
	}
	return ranges, nil
}
<|endoftext|>


<commit_before>package slackbot

const VERSION = "0.3"

const WIKI_URL = "http://wiki.apollo-community.org/index.php?title="

const GOON_QUOTES_FILE = "./goon_quotes.txt"

const APOLLO_QUOTES_FILE = "./apollo_quotes.txt"

const TALK_CHANNEL = "C0JPGPM9T" // The dumpster
<commit_msg>Bump version.<commit_after>package slackbot

const VERSION = "0.4"

const WIKI_URL = "http://wiki.apollo-community.org/index.php?title="

const GOON_QUOTES_FILE = "./goon_quotes.txt"

const APOLLO_QUOTES_FILE = "./apollo_quotes.txt"

const TALK_CHANNEL = "C0JPGPM9T" // The dumpster
<|endoftext|>


<commit_before>package internal

import (
	"reflect"
	"unsafe"
)

const (
	sizeOfUint64 = 8
)

// Hash is a structure for keeping track of unique indices.
type Hash struct {
	ni      uint
	mapping map[string]bool
}

func NewHash(ni uint) *Hash {
	return &Hash{
		ni:      ni,
		mapping: make(map[string]bool),
	}
}

func (self *Hash) Filter(indices []uint64) []uint64 {
	ni := self.ni
	nn := uint(len(indices)) / ni
	nb := ni * sizeOfUint64

	var bytes []byte

	header := (*reflect.SliceHeader)(unsafe.Pointer(&bytes))
	header.Cap = int(nb)
	header.Len = int(nb)

	offset := ((*reflect.SliceHeader)(unsafe.Pointer(&indices))).Data

	na, ne := uint(0), nn
	for i, j := uint(0), uint(0); i < nn; i++ {
		header.Data = offset + uintptr(j*nb)
		key := string(bytes)
		if _, ok := self.mapping[key]; ok {
			j++
			continue
		}
		self.mapping[key] = true
		if j > na {
			copy(indices[na*ni:], indices[j*ni:ne*ni])
			ne -= j - na
			j = na
		}
		na++
		j++
	}

	return indices[:na*ni]
}
<commit_msg>a/internal: add more documentation<commit_after>package internal

import (
	"reflect"
	"unsafe"
)

const (
	sizeOfUint64 = 8
)

// Hash is a structure for keeping track of unique indices.
type Hash struct {
	ni      uint
	mapping map[string]bool
}

// NewHash creates an index hash.
func NewHash(ni uint) *Hash {
	return &Hash{
		ni:      ni,
		mapping: make(map[string]bool),
	}
}

// Filter eliminates the indices that have already been seen.
func (self *Hash) Filter(indices []uint64) []uint64 {
	ni := self.ni
	nn := uint(len(indices)) / ni
	nb := ni * sizeOfUint64

	var bytes []byte

	header := (*reflect.SliceHeader)(unsafe.Pointer(&bytes))
	header.Cap = int(nb)
	header.Len = int(nb)

	offset := ((*reflect.SliceHeader)(unsafe.Pointer(&indices))).Data

	na, ne := uint(0), nn
	for i, j := uint(0), uint(0); i < nn; i++ {
		header.Data = offset + uintptr(j*nb)
		key := string(bytes)
		if _, ok := self.mapping[key]; ok {
			j++
			continue
		}
		self.mapping[key] = true
		if j > na {
			copy(indices[na*ni:], indices[j*ni:ne*ni])
			ne -= j - na
			j = na
		}
		na++
		j++
	}

	return indices[:na*ni]
}
<|endoftext|>


<commit_before>package internal

// Basis is a functional basis.
type Basis interface {
	// Compute evaluates the value of a basis function.
	Compute([]uint64, []float64) float64
}
<commit_msg>a/internal: add a description<commit_after>// Package internal contains types and functions shared by the interpolation
// algorithms.
package internal

// Basis is a functional basis.
type Basis interface {
	// Compute evaluates the value of a basis function.
	Compute([]uint64, []float64) float64
}
<|endoftext|>


<commit_before>package alignmentprofile

// This package contains procedures for serializing
// rawAlignmentProfile structs. It uses the built-in template package
// instead of YAML serialization because it needs tighter control of
// the resulting data's formatting than our YAML parser has.

import (
	"bytes"
	"text/template"
)

var profileTemplateSrc = `StopCodonPenalty: {{.StopCodonPenalty}}
GapOpeningPenalty: {{.GapOpeningPenalty}}
GapExtensionPenalty: {{.GapExtensionPenalty}}
IndelCodonOpeningBonus: {{.IndelCodonOpeningBonus}}
IndelCodonExtensionBonus: {{.IndelCodonExtensionBonus}}
ReferenceSequences:
{{ range $gene, $seq := .ReferenceSequences }}  {{$gene}}:
    {{$seq}}
{{end -}}
PositionalIndelScores:
{{range $gene, $rawIndels := .RawIndelScores}}  {{$gene}}:
{{- range $rawIndels}}
    - [ {{.Kind}}, {{.Position}}, {{.Open}}, {{.Extend}} ]
{{- end}}
{{end}}`

var profileTemplate *template.Template

func init() {
	profileTemplate = template.Must(template.New("alignmentprofile").Parse(profileTemplateSrc))

}

func Format(ap AlignmentProfile) string {
	rawProfile := ap.asRaw()
	var buff bytes.Buffer
	err := profileTemplate.Execute(&buff, rawProfile)
	if err != nil {
		panic(err)
	}
	return buff.String()
}
<commit_msg>Don't print PositionalIndelScores if it's nil<commit_after>package alignmentprofile

// This package contains procedures for serializing
// rawAlignmentProfile structs. It uses the built-in template package
// instead of YAML serialization because it needs tighter control of
// the resulting data's formatting than our YAML parser has.

import (
	"bytes"
	"text/template"
)

var profileTemplateSrc = `StopCodonPenalty: {{.StopCodonPenalty}}
GapOpeningPenalty: {{.GapOpeningPenalty}}
GapExtensionPenalty: {{.GapExtensionPenalty}}
IndelCodonOpeningBonus: {{.IndelCodonOpeningBonus}}
IndelCodonExtensionBonus: {{.IndelCodonExtensionBonus}}
ReferenceSequences:
{{ range $gene, $seq := .ReferenceSequences }}  {{$gene}}:
    {{$seq}}
{{end -}}
{{ if .RawIndelScores -}} PositionalIndelScores: {{- end }}
{{range $gene, $rawIndels := .RawIndelScores}}  {{$gene}}:
{{- range $rawIndels}}
    - [ {{.Kind}}, {{.Position}}, {{.Open}}, {{.Extend}} ]
{{- end}}
{{end}}`

var profileTemplate *template.Template

func init() {
	profileTemplate = template.Must(template.New("alignmentprofile").Parse(profileTemplateSrc))

}

func Format(ap AlignmentProfile) string {
	rawProfile := ap.asRaw()
	var buff bytes.Buffer
	err := profileTemplate.Execute(&buff, rawProfile)
	if err != nil {
		panic(err)
	}
	return buff.String()
}
<|endoftext|>