<commit_before>//  Copyright (c) 2014 Couchbase, Inc.
//  Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
//  except in compliance with the License. You may obtain a copy of the License at
//    http://www.apache.org/licenses/LICENSE-2.0
//  Unless required by applicable law or agreed to in writing, software distributed under the
//  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
//  either express or implied. See the License for the specific language governing permissions
//  and limitations under the License.

package cbft

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
	"path/filepath"
	"sort"
	"strings"
	"time"

	bleveHttp "github.com/blevesearch/bleve/http"

	"github.com/gorilla/mux"
)

type DiagGetHandler struct {
	versionMain string
	mgr         *Manager
	mr          *MsgRing
}

func NewDiagGetHandler(versionMain string,
	mgr *Manager, mr *MsgRing) *DiagGetHandler {
	return &DiagGetHandler{versionMain: versionMain, mgr: mgr, mr: mr}
}

func (h *DiagGetHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
	handlers := []struct {
		Name        string
		Handler     http.Handler
		HandlerFunc http.HandlerFunc
	}{
		{"/api/cfg", NewCfgGetHandler(h.mgr), nil},
		{"/api/index", NewListIndexHandler(h.mgr), nil},
		{"/api/log", NewLogGetHandler(h.mgr, h.mr), nil},
		{"/api/managerMeta", NewManagerMetaHandler(h.mgr, nil), nil},
		{"/api/pindex", NewListPIndexHandler(h.mgr), nil},
		{"/api/pindex-bleve", bleveHttp.NewListIndexesHandler(), nil},
		{"/api/runtime", NewRuntimeGetHandler(h.versionMain, h.mgr), nil},
		{"/api/runtime/args", nil, restGetRuntimeArgs},
		{"/api/runtime/stats", nil, restGetRuntimeStats},
		{"/api/runtime/statsMem", nil, restGetRuntimeStatsMem},
		{"/api/stats", NewStatsHandler(h.mgr), nil},
	}

	w.Write(jsonOpenBrace)
	for i, handler := range handlers {
		if i > 0 {
			w.Write(jsonComma)
		}
		w.Write([]byte(fmt.Sprintf(`"%s":`, handler.Name)))
		if handler.Handler != nil {
			handler.Handler.ServeHTTP(w, req)
		}
		if handler.HandlerFunc != nil {
			handler.HandlerFunc.ServeHTTP(w, req)
		}
	}

	// TODO: We should include contents of some of the smaller config
	// files that we recognize from the dataDir.
	var first = true
	var visit func(path string, f os.FileInfo, err error) error
	visit = func(path string, f os.FileInfo, err error) error {
		m := map[string]interface{}{
			"Path":    path,
			"Name":    f.Name(),
			"Size":    f.Size(),
			"Mode":    f.Mode(),
			"ModTime": f.ModTime().Format(time.RFC3339Nano),
			"IsDir":   f.IsDir(),
		}
		if strings.HasPrefix(f.Name(), "PINDEX_") || // Matches PINDEX_xxx_META.
			strings.HasSuffix(f.Name(), "_META") || // Matches PINDEX_META.
			strings.HasSuffix(f.Name(), ".json") { // Matches index_meta.json.
			b, err := ioutil.ReadFile(path)
			if err == nil {
				m["Contents"] = string(b)
			}
		}
		buf, err := json.Marshal(m)
		if err == nil {
			if !first {
				w.Write(jsonComma)
			}
			w.Write(buf)
			first = false
		}
		return nil
	}

	w.Write([]byte(`,"dataDir":[`))
	filepath.Walk(h.mgr.dataDir, visit)
	w.Write([]byte(`]`))

	w.Write(jsonCloseBrace)
}

// ---------------------------------------------------

type StatsHandler struct {
	mgr *Manager
}

func NewStatsHandler(mgr *Manager) *StatsHandler {
	return &StatsHandler{mgr: mgr}
}

var statsFeedsPrefix = []byte("\"feeds\":{")
var statsPIndexesPrefix = []byte("\"pindexes\":{")
var statsManagerPrefix = []byte(",\"manager\":")
var statsNamePrefix = []byte("\"")
var statsNameSuffix = []byte("\":")

func (h *StatsHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
	indexName := mux.Vars(req)["indexName"]

	feeds, pindexes := h.mgr.CurrentMaps()
	feedNames := make([]string, 0, len(feeds))
	for feedName := range feeds {
		feedNames = append(feedNames, feedName)
	}
	sort.Strings(feedNames)

	pindexNames := make([]string, 0, len(pindexes))
	for pindexName := range pindexes {
		pindexNames = append(pindexNames, pindexName)
	}
	sort.Strings(pindexNames)

	w.Write(jsonOpenBrace)

	if indexName == "" {
		first := true
		w.Write(statsFeedsPrefix)
		for _, feedName := range feedNames {
			if !first {
				w.Write(jsonComma)
			}
			first = false
			w.Write(statsNamePrefix)
			w.Write([]byte(feedName))
			w.Write(statsNameSuffix)
			feeds[feedName].Stats(w)
		}
		w.Write(jsonCloseBraceComma)
	}

	first := true
	w.Write(statsPIndexesPrefix)
	for _, pindexName := range pindexNames {
		if indexName == "" || indexName == pindexes[pindexName].IndexName {
			if !first {
				w.Write(jsonComma)
			}
			first = false
			w.Write(statsNamePrefix)
			w.Write([]byte(pindexName))
			w.Write(statsNameSuffix)
			pindexes[pindexName].Dest.Stats(w)
		}
	}
	w.Write(jsonCloseBrace)

	if indexName == "" {
		w.Write(statsManagerPrefix)
		var mgrStats ManagerStats
		h.mgr.stats.AtomicCopyTo(&mgrStats)
		mgrStatsJSON, err := json.Marshal(&mgrStats)
		if err == nil && len(mgrStatsJSON) > 0 {
			w.Write(mgrStatsJSON)
		} else {
			w.Write(jsonNULL)
		}
	}

	w.Write(jsonCloseBrace)
}

// ---------------------------------------------------

type ManagerKickHandler struct {
	mgr *Manager
}

func NewManagerKickHandler(mgr *Manager) *ManagerKickHandler {
	return &ManagerKickHandler{mgr: mgr}
}

func (h *ManagerKickHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
	h.mgr.Kick(req.FormValue("msg"))
	mustEncode(w, struct {
		Status string `json:"status"`
	}{Status: "ok"})
}

// ---------------------------------------------------

type CfgGetHandler struct {
	mgr *Manager
}

func NewCfgGetHandler(mgr *Manager) *CfgGetHandler {
	return &CfgGetHandler{mgr: mgr}
}

func (h *CfgGetHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
	// TODO: Might need to scrub auth passwords from this output.
	cfg := h.mgr.Cfg()
	indexDefs, indexDefsCAS, indexDefsErr :=
		CfgGetIndexDefs(cfg)
	nodeDefsWanted, nodeDefsWantedCAS, nodeDefsWantedErr :=
		CfgGetNodeDefs(cfg, NODE_DEFS_WANTED)
	nodeDefsKnown, nodeDefsKnownCAS, nodeDefsKnownErr :=
		CfgGetNodeDefs(cfg, NODE_DEFS_KNOWN)
	planPIndexes, planPIndexesCAS, planPIndexesErr :=
		CfgGetPlanPIndexes(cfg)
	mustEncode(w, struct {
		Status            string        `json:"status"`
		IndexDefs         *IndexDefs    `json:"indexDefs"`
		IndexDefsCAS      uint64        `json:"indexDefsCAS"`
		IndexDefsErr      error         `json:"indexDefsErr"`
		NodeDefsWanted    *NodeDefs     `json:"nodeDefsWanted"`
		NodeDefsWantedCAS uint64        `json:"nodeDefsWantedCAS"`
		NodeDefsWantedErr error         `json:"nodeDefsWantedErr"`
		NodeDefsKnown     *NodeDefs     `json:"nodeDefsKnown"`
		NodeDefsKnownCAS  uint64        `json:"nodeDefsKnownCAS"`
		NodeDefsKnownErr  error         `json:"nodeDefsKnownErr"`
		PlanPIndexes      *PlanPIndexes `json:"planPIndexes"`
		PlanPIndexesCAS   uint64        `json:"planPIndexesCAS"`
		PlanPIndexesErr   error         `json:"planPIndexesErr"`
	}{
		Status:            "ok",
		IndexDefs:         indexDefs,
		IndexDefsCAS:      indexDefsCAS,
		IndexDefsErr:      indexDefsErr,
		NodeDefsWanted:    nodeDefsWanted,
		NodeDefsWantedCAS: nodeDefsWantedCAS,
		NodeDefsWantedErr: nodeDefsWantedErr,
		NodeDefsKnown:     nodeDefsKnown,
		NodeDefsKnownCAS:  nodeDefsKnownCAS,
		NodeDefsKnownErr:  nodeDefsKnownErr,
		PlanPIndexes:      planPIndexes,
		PlanPIndexesCAS:   planPIndexesCAS,
		PlanPIndexesErr:   planPIndexesErr,
	})
}

// ---------------------------------------------------

type CfgRefreshHandler struct {
	mgr *Manager
}

func NewCfgRefreshHandler(mgr *Manager) *CfgRefreshHandler {
	return &CfgRefreshHandler{mgr: mgr}
}

func (h *CfgRefreshHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
	h.mgr.Cfg().Refresh()
	h.mgr.GetIndexDefs(true)
	h.mgr.GetPlanPIndexes(true)
	mustEncode(w, struct {
		Status string `json:"status"`
	}{Status: "ok"})
}
<commit_msg>REST /api/stats/index/{indexName} includes related feeds<commit_after>//  Copyright (c) 2014 Couchbase, Inc.
//  Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
//  except in compliance with the License. You may obtain a copy of the License at
//    http://www.apache.org/licenses/LICENSE-2.0
//  Unless required by applicable law or agreed to in writing, software distributed under the
//  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
//  either express or implied. See the License for the specific language governing permissions
//  and limitations under the License.

package cbft

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
	"path/filepath"
	"sort"
	"strings"
	"time"

	bleveHttp "github.com/blevesearch/bleve/http"

	"github.com/gorilla/mux"
)

type DiagGetHandler struct {
	versionMain string
	mgr         *Manager
	mr          *MsgRing
}

func NewDiagGetHandler(versionMain string,
	mgr *Manager, mr *MsgRing) *DiagGetHandler {
	return &DiagGetHandler{versionMain: versionMain, mgr: mgr, mr: mr}
}

func (h *DiagGetHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
	handlers := []struct {
		Name        string
		Handler     http.Handler
		HandlerFunc http.HandlerFunc
	}{
		{"/api/cfg", NewCfgGetHandler(h.mgr), nil},
		{"/api/index", NewListIndexHandler(h.mgr), nil},
		{"/api/log", NewLogGetHandler(h.mgr, h.mr), nil},
		{"/api/managerMeta", NewManagerMetaHandler(h.mgr, nil), nil},
		{"/api/pindex", NewListPIndexHandler(h.mgr), nil},
		{"/api/pindex-bleve", bleveHttp.NewListIndexesHandler(), nil},
		{"/api/runtime", NewRuntimeGetHandler(h.versionMain, h.mgr), nil},
		{"/api/runtime/args", nil, restGetRuntimeArgs},
		{"/api/runtime/stats", nil, restGetRuntimeStats},
		{"/api/runtime/statsMem", nil, restGetRuntimeStatsMem},
		{"/api/stats", NewStatsHandler(h.mgr), nil},
	}

	w.Write(jsonOpenBrace)
	for i, handler := range handlers {
		if i > 0 {
			w.Write(jsonComma)
		}
		w.Write([]byte(fmt.Sprintf(`"%s":`, handler.Name)))
		if handler.Handler != nil {
			handler.Handler.ServeHTTP(w, req)
		}
		if handler.HandlerFunc != nil {
			handler.HandlerFunc.ServeHTTP(w, req)
		}
	}

	// TODO: We should include contents of some of the smaller config
	// files that we recognize from the dataDir.
	var first = true
	var visit func(path string, f os.FileInfo, err error) error
	visit = func(path string, f os.FileInfo, err error) error {
		m := map[string]interface{}{
			"Path":    path,
			"Name":    f.Name(),
			"Size":    f.Size(),
			"Mode":    f.Mode(),
			"ModTime": f.ModTime().Format(time.RFC3339Nano),
			"IsDir":   f.IsDir(),
		}
		if strings.HasPrefix(f.Name(), "PINDEX_") || // Matches PINDEX_xxx_META.
			strings.HasSuffix(f.Name(), "_META") || // Matches PINDEX_META.
			strings.HasSuffix(f.Name(), ".json") { // Matches index_meta.json.
			b, err := ioutil.ReadFile(path)
			if err == nil {
				m["Contents"] = string(b)
			}
		}
		buf, err := json.Marshal(m)
		if err == nil {
			if !first {
				w.Write(jsonComma)
			}
			w.Write(buf)
			first = false
		}
		return nil
	}

	w.Write([]byte(`,"dataDir":[`))
	filepath.Walk(h.mgr.dataDir, visit)
	w.Write([]byte(`]`))

	w.Write(jsonCloseBrace)
}

// ---------------------------------------------------

type StatsHandler struct {
	mgr *Manager
}

func NewStatsHandler(mgr *Manager) *StatsHandler {
	return &StatsHandler{mgr: mgr}
}

var statsFeedsPrefix = []byte("\"feeds\":{")
var statsPIndexesPrefix = []byte("\"pindexes\":{")
var statsManagerPrefix = []byte(",\"manager\":")
var statsNamePrefix = []byte("\"")
var statsNameSuffix = []byte("\":")

func (h *StatsHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
	indexName := mux.Vars(req)["indexName"]

	feeds, pindexes := h.mgr.CurrentMaps()
	feedNames := make([]string, 0, len(feeds))
	for feedName := range feeds {
		feedNames = append(feedNames, feedName)
	}
	sort.Strings(feedNames)

	pindexNames := make([]string, 0, len(pindexes))
	for pindexName := range pindexes {
		pindexNames = append(pindexNames, pindexName)
	}
	sort.Strings(pindexNames)

	w.Write(jsonOpenBrace)

	first := true
	w.Write(statsFeedsPrefix)
	for _, feedName := range feedNames {
		if indexName == "" || indexName == feeds[feedName].IndexName() {
			if !first {
				w.Write(jsonComma)
			}
			first = false
			w.Write(statsNamePrefix)
			w.Write([]byte(feedName))
			w.Write(statsNameSuffix)
			feeds[feedName].Stats(w)
		}
	}
	w.Write(jsonCloseBraceComma)

	first = true
	w.Write(statsPIndexesPrefix)
	for _, pindexName := range pindexNames {
		if indexName == "" || indexName == pindexes[pindexName].IndexName {
			if !first {
				w.Write(jsonComma)
			}
			first = false
			w.Write(statsNamePrefix)
			w.Write([]byte(pindexName))
			w.Write(statsNameSuffix)
			pindexes[pindexName].Dest.Stats(w)
		}
	}
	w.Write(jsonCloseBrace)

	if indexName == "" {
		w.Write(statsManagerPrefix)
		var mgrStats ManagerStats
		h.mgr.stats.AtomicCopyTo(&mgrStats)
		mgrStatsJSON, err := json.Marshal(&mgrStats)
		if err == nil && len(mgrStatsJSON) > 0 {
			w.Write(mgrStatsJSON)
		} else {
			w.Write(jsonNULL)
		}
	}

	w.Write(jsonCloseBrace)
}

// ---------------------------------------------------

type ManagerKickHandler struct {
	mgr *Manager
}

func NewManagerKickHandler(mgr *Manager) *ManagerKickHandler {
	return &ManagerKickHandler{mgr: mgr}
}

func (h *ManagerKickHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
	h.mgr.Kick(req.FormValue("msg"))
	mustEncode(w, struct {
		Status string `json:"status"`
	}{Status: "ok"})
}

// ---------------------------------------------------

type CfgGetHandler struct {
	mgr *Manager
}

func NewCfgGetHandler(mgr *Manager) *CfgGetHandler {
	return &CfgGetHandler{mgr: mgr}
}

func (h *CfgGetHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
	// TODO: Might need to scrub auth passwords from this output.
	cfg := h.mgr.Cfg()
	indexDefs, indexDefsCAS, indexDefsErr :=
		CfgGetIndexDefs(cfg)
	nodeDefsWanted, nodeDefsWantedCAS, nodeDefsWantedErr :=
		CfgGetNodeDefs(cfg, NODE_DEFS_WANTED)
	nodeDefsKnown, nodeDefsKnownCAS, nodeDefsKnownErr :=
		CfgGetNodeDefs(cfg, NODE_DEFS_KNOWN)
	planPIndexes, planPIndexesCAS, planPIndexesErr :=
		CfgGetPlanPIndexes(cfg)
	mustEncode(w, struct {
		Status            string        `json:"status"`
		IndexDefs         *IndexDefs    `json:"indexDefs"`
		IndexDefsCAS      uint64        `json:"indexDefsCAS"`
		IndexDefsErr      error         `json:"indexDefsErr"`
		NodeDefsWanted    *NodeDefs     `json:"nodeDefsWanted"`
		NodeDefsWantedCAS uint64        `json:"nodeDefsWantedCAS"`
		NodeDefsWantedErr error         `json:"nodeDefsWantedErr"`
		NodeDefsKnown     *NodeDefs     `json:"nodeDefsKnown"`
		NodeDefsKnownCAS  uint64        `json:"nodeDefsKnownCAS"`
		NodeDefsKnownErr  error         `json:"nodeDefsKnownErr"`
		PlanPIndexes      *PlanPIndexes `json:"planPIndexes"`
		PlanPIndexesCAS   uint64        `json:"planPIndexesCAS"`
		PlanPIndexesErr   error         `json:"planPIndexesErr"`
	}{
		Status:            "ok",
		IndexDefs:         indexDefs,
		IndexDefsCAS:      indexDefsCAS,
		IndexDefsErr:      indexDefsErr,
		NodeDefsWanted:    nodeDefsWanted,
		NodeDefsWantedCAS: nodeDefsWantedCAS,
		NodeDefsWantedErr: nodeDefsWantedErr,
		NodeDefsKnown:     nodeDefsKnown,
		NodeDefsKnownCAS:  nodeDefsKnownCAS,
		NodeDefsKnownErr:  nodeDefsKnownErr,
		PlanPIndexes:      planPIndexes,
		PlanPIndexesCAS:   planPIndexesCAS,
		PlanPIndexesErr:   planPIndexesErr,
	})
}

// ---------------------------------------------------

type CfgRefreshHandler struct {
	mgr *Manager
}

func NewCfgRefreshHandler(mgr *Manager) *CfgRefreshHandler {
	return &CfgRefreshHandler{mgr: mgr}
}

func (h *CfgRefreshHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
	h.mgr.Cfg().Refresh()
	h.mgr.GetIndexDefs(true)
	h.mgr.GetPlanPIndexes(true)
	mustEncode(w, struct {
		Status string `json:"status"`
	}{Status: "ok"})
}
<|endoftext|>


<commit_before><commit_msg>auto-updated agent/uiserver/bindata_assetfs.go from commit 5a39be47d<commit_after><|endoftext|>


<commit_before><commit_msg>auto-updated agent/uiserver/bindata_assetfs.go from commit cfbd1bb84<commit_after><|endoftext|>


<commit_before>package hookshot

import (
	"bytes"
	"crypto/hmac"
	"crypto/sha1"
	"fmt"
	"io/ioutil"
	"net/http"
)

const (
	// HeaderEvent is the name of the header that contains the type of event.
	HeaderEvent = "X-GitHub-Event"

	// HeaderSignature is the name of the header that contains the signature.
	HeaderSignature = "X-Hub-Signature"
)

// Router demultiplexes github hooks.
type Router struct {
	NotFoundHandler     http.Handler
	UnauthorizedHandler http.Handler

	routes routes
	secret string
}

// NewRouter returns a new Router.
func NewRouter(secret string) *Router {
	return &Router{
		routes: make(routes),
		secret: secret,
	}
}

// Handle maps a github event to an http.Handler.
func (r *Router) Handle(event string, h http.Handler) *Route {
	route := &Route{Secret: r.secret, event: event, handler: h}
	r.routes[event] = route
	return route
}

// ServeHTTP implements the http.Handler interface.
func (r *Router) ServeHTTP(w http.ResponseWriter, req *http.Request) {
	event := req.Header.Get(HeaderEvent)

	route := r.routes[event]
	if route == nil {
		r.notFound(w, req)
		return
	}

	if !authorized(req, route.Secret) {
		r.unauthorized(w, req)
		return
	}

	route.ServeHTTP(w, req)
}

func (r *Router) notFound(w http.ResponseWriter, req *http.Request) {
	if r.NotFoundHandler == nil {
		r.NotFoundHandler = http.HandlerFunc(http.NotFound)
	}
	r.NotFoundHandler.ServeHTTP(w, req)
}

func (r *Router) unauthorized(w http.ResponseWriter, req *http.Request) {
	if r.UnauthorizedHandler == nil {
		r.UnauthorizedHandler = http.HandlerFunc(unauthorized)
	}
	r.UnauthorizedHandler.ServeHTTP(w, req)
}

// Route represents the http.Handler for a github event.
type Route struct {
	Secret string

	handler http.Handler
	event   string
}

// ServeHTTP implements the http.Handler interface.
func (r *Route) ServeHTTP(w http.ResponseWriter, req *http.Request) {
	r.handler.ServeHTTP(w, req)
}

// routes maps a github event to a Route.
type routes map[string]*Route

// Signature calculates the SHA1 HMAC signature of in using the secret.
func Signature(in []byte, secret string) string {
	mac := hmac.New(sha1.New, []byte(secret))
	mac.Write(in)
	return fmt.Sprintf("%x", mac.Sum(nil))
}

// authorized checks that the calculated signature for the request matches the provided signature in
// the request headers.
func authorized(r *http.Request, secret string) bool {
	raw, er := ioutil.ReadAll(r.Body)
	if er != nil {
		return false
	}
	r.Body = ioutil.NopCloser(bytes.NewReader(raw))

	if len(r.Header[HeaderSignature]) == 0 {
		return true
	}

	return r.Header.Get(HeaderSignature) == "sha1="+Signature(raw, secret)
}

// unauthorized is the default UnauthorizedHandler.
func unauthorized(w http.ResponseWriter, r *http.Request) {
	http.Error(w, "The provided signature in the "+HeaderSignature+" header does not match.", 403)
}
<commit_msg>Comment.<commit_after>package hookshot

import (
	"bytes"
	"crypto/hmac"
	"crypto/sha1"
	"fmt"
	"io/ioutil"
	"net/http"
)

const (
	// HeaderEvent is the name of the header that contains the type of event.
	HeaderEvent = "X-GitHub-Event"

	// HeaderSignature is the name of the header that contains the signature.
	HeaderSignature = "X-Hub-Signature"
)

// Router demultiplexes github hooks.
type Router struct {
	NotFoundHandler     http.Handler
	UnauthorizedHandler http.Handler

	routes routes
	secret string
}

// NewRouter returns a new Router.
func NewRouter(secret string) *Router {
	return &Router{
		routes: make(routes),
		secret: secret,
	}
}

// Handle maps a github event to an http.Handler.
func (r *Router) Handle(event string, h http.Handler) *Route {
	route := &Route{Secret: r.secret, event: event, handler: h}
	r.routes[event] = route
	return route
}

// ServeHTTP implements the http.Handler interface.
func (r *Router) ServeHTTP(w http.ResponseWriter, req *http.Request) {
	event := req.Header.Get(HeaderEvent)

	route := r.routes[event]
	if route == nil {
		r.notFound(w, req)
		return
	}

	if !authorized(req, route.Secret) {
		r.unauthorized(w, req)
		return
	}

	route.ServeHTTP(w, req)
}

func (r *Router) notFound(w http.ResponseWriter, req *http.Request) {
	if r.NotFoundHandler == nil {
		r.NotFoundHandler = http.HandlerFunc(http.NotFound)
	}
	r.NotFoundHandler.ServeHTTP(w, req)
}

func (r *Router) unauthorized(w http.ResponseWriter, req *http.Request) {
	if r.UnauthorizedHandler == nil {
		r.UnauthorizedHandler = http.HandlerFunc(unauthorized)
	}
	r.UnauthorizedHandler.ServeHTTP(w, req)
}

// Route represents the http.Handler for a github event.
type Route struct {
	Secret string

	handler http.Handler
	event   string
}

// ServeHTTP implements the http.Handler interface.
func (r *Route) ServeHTTP(w http.ResponseWriter, req *http.Request) {
	r.handler.ServeHTTP(w, req)
}

// routes maps a github event to a Route.
type routes map[string]*Route

// Signature calculates the SHA1 HMAC signature of in using the secret.
func Signature(in []byte, secret string) string {
	mac := hmac.New(sha1.New, []byte(secret))
	mac.Write(in)
	return fmt.Sprintf("%x", mac.Sum(nil))
}

// authorized checks that the calculated signature for the request matches the provided signature in
// the request headers.
func authorized(r *http.Request, secret string) bool {
	raw, er := ioutil.ReadAll(r.Body)
	if er != nil {
		return false
	}

	// Since we're reading the request from the network, r.Body will return EOF if any
	// downstream http.Handler attempts to read it. We set it to a new io.ReadCloser
	// that will read from the bytes in memory.
	r.Body = ioutil.NopCloser(bytes.NewReader(raw))

	if len(r.Header[HeaderSignature]) == 0 {
		return true
	}

	return r.Header.Get(HeaderSignature) == "sha1="+Signature(raw, secret)
}

// unauthorized is the default UnauthorizedHandler.
func unauthorized(w http.ResponseWriter, r *http.Request) {
	http.Error(w, "The provided signature in the "+HeaderSignature+" header does not match.", 403)
}
<|endoftext|>


<commit_before>package main

import (
	log "github.com/Sirupsen/logrus"
	"github.com/elazarl/goproxy"

	"bufio"
	"flag"
	"fmt"
	"net"
	"net/http"
	"os"
	"regexp"
)

// VirtualizeMode - default mode when Hoverfly looks for captured requests to respond
const VirtualizeMode = "virtualize"

// SynthesizeMode - all requests are sent to middleware to create response
const SynthesizeMode = "synthesize"

// ModifyMode - middleware is applied to outgoing and incoming traffic
const ModifyMode = "modify"

// CaptureMode - requests are captured and stored in cache
const CaptureMode = "capture"

// orPanic - wrapper for logging errors
func orPanic(err error) {
	if err != nil {
		log.WithFields(log.Fields{
			"error": err.Error(),
		}).Panic("Got error.")
	}
}

func main() {
	// Output to stderr instead of stdout, could also be a file.
	log.SetOutput(os.Stderr)
	log.SetFormatter(&log.TextFormatter{})

	// getting proxy configuration
	verbose := flag.Bool("v", false, "should every proxy request be logged to stdout")
	// modes
	capture := flag.Bool("capture", false, "should proxy capture requests")
	synthesize := flag.Bool("synthesize", false, "should proxy capture requests")
	modify := flag.Bool("modify", false, "should proxy only modify requests")

	destination := flag.String("destination", ".", "destination URI to catch")
	middleware := flag.String("middleware", "", "should proxy use middleware")

	// proxy port
	proxyPort := flag.String("pp", "", "proxy port - run proxy on another port (i.e. '-pp 9999' to run proxy on port 9999)")
	// admin port
	adminPort := flag.String("ap", "", "admin port - run admin interface on another port (i.e. '-ap 1234' to run admin UI on port 1234)")

	flag.Parse()

	// getting settings
	cfg := InitSettings()

	if *verbose {
		// Only log the warning severity or above.
		log.SetLevel(log.DebugLevel)
	}
	cfg.verbose = *verbose

	// overriding environment variables (proxy and admin ports)
	if *proxyPort != "" {
		cfg.proxyPort = *proxyPort
	}
	if *adminPort != "" {
		cfg.adminPort = *adminPort
	}

	// overriding default middleware setting
	cfg.middleware = *middleware

	// setting default mode
	mode := VirtualizeMode

	if *capture {
		mode = CaptureMode
		// checking whether user supplied other modes
		if *synthesize == true || *modify == true {
			log.Fatal("Two or more modes supplied, check your flags")
		}
	} else if *synthesize {
		mode = SynthesizeMode

		if cfg.middleware == "" {
			log.Fatal("Synthesize mode chosen although middleware not supplied")
		}

		if *capture == true || *modify == true {
			log.Fatal("Two or more modes supplied, check your flags")
		}
	} else if *modify {
		mode = ModifyMode

		if cfg.middleware == "" {
			log.Fatal("Modify mode chosen although middleware not supplied")
		}

		if *capture == true || *synthesize == true {
			log.Fatal("Two or more modes supplied, check your flags")
		}
	}

	// overriding default settings
	cfg.mode = mode

	// overriding destination
	cfg.destination = *destination

	proxy, dbClient := getNewHoverfly(cfg)
	defer dbClient.cache.db.Close()

	log.Warn(http.ListenAndServe(fmt.Sprintf(":%s", cfg.proxyPort), proxy))
}

// getNewHoverfly returns a configured ProxyHttpServer and DBClient, also starts admin interface on configured port
func getNewHoverfly(cfg *Configuration) (*goproxy.ProxyHttpServer, DBClient) {

	// getting boltDB
	db := getDB(cfg.databaseName)

	cache := Cache{
		db:             db,
		requestsBucket: []byte(requestsBucketName),
	}

	// getting connections
	d := DBClient{
		cache: cache,
		http:  &http.Client{},
		cfg:   cfg,
	}

	// creating proxy
	proxy := goproxy.NewProxyHttpServer()

	proxy.OnRequest(goproxy.ReqHostMatches(regexp.MustCompile(d.cfg.destination))).
		HandleConnect(goproxy.AlwaysMitm)

	// enable curl -p for all hosts on port 80
	proxy.OnRequest(goproxy.ReqHostMatches(regexp.MustCompile(d.cfg.destination))).
		HijackConnect(func(req *http.Request, client net.Conn, ctx *goproxy.ProxyCtx) {
		defer func() {
			if e := recover(); e != nil {
				ctx.Logf("error connecting to remote: %v", e)
				client.Write([]byte("HTTP/1.1 500 Cannot reach destination\r\n\r\n"))
			}
			client.Close()
		}()
		clientBuf := bufio.NewReadWriter(bufio.NewReader(client), bufio.NewWriter(client))
		remote, err := net.Dial("tcp", req.URL.Host)
		orPanic(err)
		remoteBuf := bufio.NewReadWriter(bufio.NewReader(remote), bufio.NewWriter(remote))
		for {
			req, err := http.ReadRequest(clientBuf.Reader)
			orPanic(err)
			orPanic(req.Write(remoteBuf))
			orPanic(remoteBuf.Flush())
			resp, err := http.ReadResponse(remoteBuf.Reader, req)

			orPanic(err)
			orPanic(resp.Write(clientBuf.Writer))
			orPanic(clientBuf.Flush())
		}
	})

	// processing connections
	proxy.OnRequest(goproxy.ReqHostMatches(regexp.MustCompile(cfg.destination))).DoFunc(
		func(r *http.Request, ctx *goproxy.ProxyCtx) (*http.Request, *http.Response) {
			return d.processRequest(r)
		})

	go d.startAdminInterface()

	proxy.Verbose = d.cfg.verbose
	// proxy starting message
	log.WithFields(log.Fields{
		"Destination": d.cfg.destination,
		"ProxyPort":   d.cfg.proxyPort,
		"Mode":        d.cfg.GetMode(),
	}).Info("Proxy prepared...")

	return proxy, d
}

// processRequest - processes incoming requests and based on proxy state (record/playback)
// returns HTTP response.
func (d *DBClient) processRequest(req *http.Request) (*http.Request, *http.Response) {

	mode := d.cfg.GetMode()
	if mode == CaptureMode {
		log.Info("*** Capture ***")
		newResponse, err := d.captureRequest(req)
		if err != nil {
			// something bad happened, passing through
			return req, nil
		}
		// discarding original requests and returns supplied response
		return req, newResponse

	} else if mode == SynthesizeMode {
		log.Info("*** Sinthesize ***")
		response := synthesizeResponse(req, d.cfg.middleware)
		return req, response

	} else if mode == ModifyMode {
		log.Info("*** Modify ***")
		response, err := d.modifyRequestResponse(req, d.cfg.middleware)

		if err != nil {
			log.WithFields(log.Fields{
				"error":      err.Error(),
				"middleware": d.cfg.middleware,
			}).Error("Got error when performing request modification")
			return req, nil
		}

		// returning modified response
		return req, response

	}

	log.Info("*** Virtualize ***")
	newResponse := d.getResponse(req)
	return req, newResponse

}
<commit_msg>json formatter for logrus (proxy)<commit_after>package main

import (
	log "github.com/Sirupsen/logrus"
	"github.com/elazarl/goproxy"

	"bufio"
	"flag"
	"fmt"
	"net"
	"net/http"
	"os"
	"regexp"
)

// VirtualizeMode - default mode when Hoverfly looks for captured requests to respond
const VirtualizeMode = "virtualize"

// SynthesizeMode - all requests are sent to middleware to create response
const SynthesizeMode = "synthesize"

// ModifyMode - middleware is applied to outgoing and incoming traffic
const ModifyMode = "modify"

// CaptureMode - requests are captured and stored in cache
const CaptureMode = "capture"

// orPanic - wrapper for logging errors
func orPanic(err error) {
	if err != nil {
		log.WithFields(log.Fields{
			"error": err.Error(),
		}).Panic("Got error.")
	}
}

func main() {
	// Output to stderr instead of stdout, could also be a file.
	//	log.SetOutput(os.Stderr)
	//	log.SetFormatter(&log.TextFormatter{})
	log.SetFormatter(&log.JSONFormatter{})

	// getting proxy configuration
	verbose := flag.Bool("v", false, "should every proxy request be logged to stdout")
	// modes
	capture := flag.Bool("capture", false, "should proxy capture requests")
	synthesize := flag.Bool("synthesize", false, "should proxy capture requests")
	modify := flag.Bool("modify", false, "should proxy only modify requests")

	destination := flag.String("destination", ".", "destination URI to catch")
	middleware := flag.String("middleware", "", "should proxy use middleware")

	// proxy port
	proxyPort := flag.String("pp", "", "proxy port - run proxy on another port (i.e. '-pp 9999' to run proxy on port 9999)")
	// admin port
	adminPort := flag.String("ap", "", "admin port - run admin interface on another port (i.e. '-ap 1234' to run admin UI on port 1234)")

	flag.Parse()

	// getting settings
	cfg := InitSettings()

	if *verbose {
		// Only log the warning severity or above.
		log.SetLevel(log.DebugLevel)
	}
	cfg.verbose = *verbose

	// overriding environment variables (proxy and admin ports)
	if *proxyPort != "" {
		cfg.proxyPort = *proxyPort
	}
	if *adminPort != "" {
		cfg.adminPort = *adminPort
	}

	// overriding default middleware setting
	cfg.middleware = *middleware

	// setting default mode
	mode := VirtualizeMode

	if *capture {
		mode = CaptureMode
		// checking whether user supplied other modes
		if *synthesize == true || *modify == true {
			log.Fatal("Two or more modes supplied, check your flags")
		}
	} else if *synthesize {
		mode = SynthesizeMode

		if cfg.middleware == "" {
			log.Fatal("Synthesize mode chosen although middleware not supplied")
		}

		if *capture == true || *modify == true {
			log.Fatal("Two or more modes supplied, check your flags")
		}
	} else if *modify {
		mode = ModifyMode

		if cfg.middleware == "" {
			log.Fatal("Modify mode chosen although middleware not supplied")
		}

		if *capture == true || *synthesize == true {
			log.Fatal("Two or more modes supplied, check your flags")
		}
	}

	// overriding default settings
	cfg.mode = mode

	// overriding destination
	cfg.destination = *destination

	proxy, dbClient := getNewHoverfly(cfg)
	defer dbClient.cache.db.Close()

	log.Warn(http.ListenAndServe(fmt.Sprintf(":%s", cfg.proxyPort), proxy))
}

// getNewHoverfly returns a configured ProxyHttpServer and DBClient, also starts admin interface on configured port
func getNewHoverfly(cfg *Configuration) (*goproxy.ProxyHttpServer, DBClient) {

	// getting boltDB
	db := getDB(cfg.databaseName)

	cache := Cache{
		db:             db,
		requestsBucket: []byte(requestsBucketName),
	}

	// getting connections
	d := DBClient{
		cache: cache,
		http:  &http.Client{},
		cfg:   cfg,
	}

	// creating proxy
	proxy := goproxy.NewProxyHttpServer()

	proxy.OnRequest(goproxy.ReqHostMatches(regexp.MustCompile(d.cfg.destination))).
		HandleConnect(goproxy.AlwaysMitm)

	// enable curl -p for all hosts on port 80
	proxy.OnRequest(goproxy.ReqHostMatches(regexp.MustCompile(d.cfg.destination))).
		HijackConnect(func(req *http.Request, client net.Conn, ctx *goproxy.ProxyCtx) {
		defer func() {
			if e := recover(); e != nil {
				ctx.Logf("error connecting to remote: %v", e)
				client.Write([]byte("HTTP/1.1 500 Cannot reach destination\r\n\r\n"))
			}
			client.Close()
		}()
		clientBuf := bufio.NewReadWriter(bufio.NewReader(client), bufio.NewWriter(client))
		remote, err := net.Dial("tcp", req.URL.Host)
		orPanic(err)
		remoteBuf := bufio.NewReadWriter(bufio.NewReader(remote), bufio.NewWriter(remote))
		for {
			req, err := http.ReadRequest(clientBuf.Reader)
			orPanic(err)
			orPanic(req.Write(remoteBuf))
			orPanic(remoteBuf.Flush())
			resp, err := http.ReadResponse(remoteBuf.Reader, req)

			orPanic(err)
			orPanic(resp.Write(clientBuf.Writer))
			orPanic(clientBuf.Flush())
		}
	})

	// processing connections
	proxy.OnRequest(goproxy.ReqHostMatches(regexp.MustCompile(cfg.destination))).DoFunc(
		func(r *http.Request, ctx *goproxy.ProxyCtx) (*http.Request, *http.Response) {
			return d.processRequest(r)
		})

	go d.startAdminInterface()

	proxy.Verbose = d.cfg.verbose
	// proxy starting message
	log.WithFields(log.Fields{
		"Destination": d.cfg.destination,
		"ProxyPort":   d.cfg.proxyPort,
		"Mode":        d.cfg.GetMode(),
	}).Info("Proxy prepared...")

	return proxy, d
}

// processRequest - processes incoming requests and based on proxy state (record/playback)
// returns HTTP response.
func (d *DBClient) processRequest(req *http.Request) (*http.Request, *http.Response) {

	mode := d.cfg.GetMode()
	if mode == CaptureMode {
		log.Info("*** Capture ***")
		newResponse, err := d.captureRequest(req)
		if err != nil {
			// something bad happened, passing through
			return req, nil
		}
		// discarding original requests and returns supplied response
		return req, newResponse

	} else if mode == SynthesizeMode {
		log.Info("*** Sinthesize ***")
		response := synthesizeResponse(req, d.cfg.middleware)
		return req, response

	} else if mode == ModifyMode {
		log.Info("*** Modify ***")
		response, err := d.modifyRequestResponse(req, d.cfg.middleware)

		if err != nil {
			log.WithFields(log.Fields{
				"error":      err.Error(),
				"middleware": d.cfg.middleware,
			}).Error("Got error when performing request modification")
			return req, nil
		}

		// returning modified response
		return req, response

	}

	log.Info("*** Virtualize ***")
	newResponse := d.getResponse(req)
	return req, newResponse

}
<|endoftext|>


<commit_before>package httpclient

import (
	"fmt"
	"io"
	"net/http"
	"os"
)

// HttpFile is a file-like object that allows reading and seeking from an
// http resources (via an HTTP GET with Range request)
type HttpFile struct {
	Url     string
	Headers map[string]string
	Debug   bool

	Buffer []byte

	client *http.Client
	pos    int64
	len    int64

	bpos   int64 // seek position for buffered reads
	bstart int   // first available byte in buffer
	bend   int   // last available byte in buffer
}

// HttpFileError wraps a network error
type HttpFileError struct {
	Err error
}

func (e *HttpFileError) Error() string {
	return "HttpFileError: " + e.Err.Error()
}

type headersType map[string]string

var HttpFileNoHead = false

// Creates an HttpFile object. At this point the "file" is "open"
func OpenHttpFile(url string, headers map[string]string) (*HttpFile, error) {
	f := HttpFile{Url: url, Headers: headers, client: &http.Client{}, pos: 0, len: -1}

	hmethod := "HEAD"
	var hheaders map[string]string

	if HttpFileNoHead { // some servers don't support HEAD, try with a GET of 0 bytes (actually 1)
		hmethod = "GET"
		hheaders = headersType{"Range": "bytes=0-0"}
	}

	resp, err := f.do(hmethod, hheaders)
	defer CloseResponse(resp)

	if err != nil {
		return nil, &HttpFileError{Err: err}
	}
	if resp.StatusCode == http.StatusNotFound {
		return nil, os.ErrNotExist
	}
	if resp.StatusCode == http.StatusOK {
		f.len = resp.ContentLength
	} else if resp.StatusCode == http.StatusPartialContent {
		_, _, clen, err := f.getContentRange(resp)
		if err != nil {
			return nil, err
		}

		f.len = clen
	} else {
		return nil, &HttpFileError{Err: fmt.Errorf("Unexpected Status %s", resp.Status)}
	}

	return &f, nil
}

func (f *HttpFile) do(method string, headers map[string]string) (*http.Response, error) {
	req, err := http.NewRequest(method, f.Url, nil)
	if err != nil {
		return nil, err
	}

	for k, v := range f.Headers {
		req.Header.Set(k, v)
	}

	for k, v := range headers {
		req.Header.Set(k, v)
	}

	return f.client.Do(req)
}

func (f *HttpFile) getContentRange(resp *http.Response) (first, last, total int64, err error) {
	content_range := resp.Header.Get("Content-Range")

	n, err := fmt.Sscanf(content_range, "bytes %d-%d/%d", &first, &last, &total)
	if err != nil {
		DebugLog(f.Debug).Println("Error", err)
		return -1, -1, -1, err
	}
	if n != 3 {
		return -1, -1, -1, &HttpFileError{Err: fmt.Errorf("Unexpected Content-Range %q (%d)", content_range, n)}
	}

	return first, last, total, nil
}

// Returns the file size
func (f *HttpFile) Size() int64 {
	DebugLog(f.Debug).Println("Size", f.len)
	return f.len
}

func (f *HttpFile) readAt(p []byte, off int64) (int, error) {
	DebugLog(f.Debug).Println("readAt", off, len(p))

	if f.client == nil {
		return 0, os.ErrInvalid
	}

	plen := len(p)
	if plen <= 0 {
		return plen, nil
	}

	end := off + int64(plen)
	if end > f.len {
		end = f.len
	}

	bytes_range := fmt.Sprintf("bytes=%d-%d", off, end-1)
	resp, err := f.do("GET", headersType{"Range": bytes_range})
	defer CloseResponse(resp)

	switch {
	case err != nil:
		DebugLog(f.Debug).Println("readAt error", err)
		return 0, &HttpFileError{Err: err}

	case resp.StatusCode == http.StatusRequestedRangeNotSatisfiable:
		DebugLog(f.Debug).Println("readAt http.StatusRequestedRangeNotSatisfiable")
		return 0, io.EOF

	case resp.StatusCode != http.StatusPartialContent:
		DebugLog(f.Debug).Println("readAt error", resp.Status)
		return 0, &HttpFileError{Err: fmt.Errorf("Unexpected Status %s", resp.Status)}
	}

	first, last, total, err := f.getContentRange(resp)
	DebugLog(f.Debug).Println("Range", bytes_range, "Content-Range", first, last, total)

	n, err := io.ReadFull(resp.Body, p)
	if n > 0 && err == io.EOF {
		// read reached EOF, but archive/zip doesn't like this!
		DebugLog(f.Debug).Println("readAt", n, "reached EOF")
		err = nil
	}

	DebugLog(f.Debug).Println("readAt", n, err)
	return n, err
}

func (f *HttpFile) readFromBuffer(p []byte, off int64) (int, error) {
	ppos := 0
	plen := len(p)

	if off != f.bpos {
		blen := f.bend - f.bstart

		if blen == 0 {
			f.bstart = 0
			f.bend = 0
		} else if f.bpos < off && f.bpos+int64(blen) > off {
			drop := int(off - f.bpos)
			f.bstart += drop

			DebugLog(f.Debug).Println("readFrom", off, "pos", f.bpos, "drop", drop, "bytes, saved", blen-drop, "bytes")
		} else {
			DebugLog(f.Debug).Println("readFrom", off, "pos", f.bpos, "dropping", blen, "bytes")

			f.bstart = 0
			f.bend = 0
		}

		f.bpos = off
	}

	for ppos < plen {
		DebugLog(f.Debug).Println("readFromBuffer", ppos, plen, "pos", f.bpos)

		if f.bstart < f.bend { // there is already some data
			n := copy(p[ppos:], f.Buffer[f.bstart:f.bend])

			f.bstart += n
			f.bpos += int64(n)
			ppos += n

			if ppos >= plen {
				DebugLog(f.Debug).Println("readFromBuffer", ppos, "done", "pos", f.bpos)
				return ppos, nil
			}
		}

		if plen-ppos > len(f.Buffer) { // no need to read in buffer
			f.bstart = 0
			f.bend = 0

			return f.readAt(p[ppos:], f.bpos)
		}

		n, err := f.readAt(f.Buffer, f.bpos)

		f.bstart = 0
		f.bend = n

		if err != nil && n == 0 { // don't return an error if we read something
			DebugLog(f.Debug).Println("readFromBuffer", "error", err)
			return 0, err
		}
	}

	panic("should not get here")
	return 0, nil
}

// The ReaderAt interface
func (f *HttpFile) ReadAt(p []byte, off int64) (int, error) {
	DebugLog(f.Debug).Println("ReadAt", off, "len", len(p))

	if f.Buffer != nil {
		return f.readFromBuffer(p, off)
	}

	return f.readAt(p, off)
}

// The Reader interface
func (f *HttpFile) Read(p []byte) (int, error) {
	DebugLog(f.Debug).Println("Read from", f.pos, "len", len(p))

	n, err := f.ReadAt(p, f.pos)
	if n > 0 {
		f.pos += int64(n)
	}

	DebugLog(f.Debug).Println("Read", n, err)
	return n, err
}

// The Closer interface
func (f *HttpFile) Close() error {
	DebugLog(f.Debug).Println("Close")
	f.client = nil
	f.pos = -1
	f.len = -1
	return nil
}

// The Seeker interface
func (f *HttpFile) Seek(offset int64, whence int) (int64, error) {
	DebugLog(f.Debug).Println("Seek", offset, whence)

	var newpos int64 = -1

	if f.client != nil {
		switch whence {
		case 0: // from 0
			newpos = offset

		case 1: // from current
			newpos = f.pos + offset

		case 2: // from end
			newpos = f.len + offset
		}
	}

	if newpos < 0 {
		return 0, os.ErrInvalid
	} else {
		if f.pos != newpos {
			f.pos = newpos
		}

		return f.pos, nil
	}
}
<commit_msg>Added retries for Cloudfront 403 errors (too many requests)<commit_after>package httpclient

import (
	"fmt"
	"io"
	"log"
	"net/http"
	"os"
	"time"
)

// HttpFile is a file-like object that allows reading and seeking from an
// http resources (via an HTTP GET with Range request)
type HttpFile struct {
	Url     string
	Headers map[string]string
	Debug   bool

	Buffer []byte

	client *http.Client
	pos    int64
	len    int64

	bpos   int64 // seek position for buffered reads
	bstart int   // first available byte in buffer
	bend   int   // last available byte in buffer
}

// HttpFileError wraps a network error
type HttpFileError struct {
	Err error
}

func (e *HttpFileError) Error() string {
	return "HttpFileError: " + e.Err.Error()
}

type headersType map[string]string

var HttpFileNoHead = false
var HttpFileRetries = 10
var HttpFileRetryWait = 60 * time.Second

// Creates an HttpFile object. At this point the "file" is "open"
func OpenHttpFile(url string, headers map[string]string) (*HttpFile, error) {
	f := HttpFile{Url: url, Headers: headers, client: &http.Client{}, pos: 0, len: -1}

	hmethod := "HEAD"
	var hheaders map[string]string

	if HttpFileNoHead { // some servers don't support HEAD, try with a GET of 0 bytes (actually 1)
		hmethod = "GET"
		hheaders = headersType{"Range": "bytes=0-0"}
	}

	resp, err := f.do(hmethod, hheaders)
	defer CloseResponse(resp)

	if err != nil {
		return nil, &HttpFileError{Err: err}
	}
	if resp.StatusCode == http.StatusNotFound {
		return nil, os.ErrNotExist
	}
	if resp.StatusCode == http.StatusOK {
		f.len = resp.ContentLength
	} else if resp.StatusCode == http.StatusPartialContent {
		_, _, clen, err := f.getContentRange(resp)
		if err != nil {
			return nil, err
		}

		f.len = clen
	} else {
		return nil, &HttpFileError{Err: fmt.Errorf("Unexpected Status %s", resp.Status)}
	}

	return &f, nil
}

func (f *HttpFile) do(method string, headers map[string]string) (*http.Response, error) {
	req, err := http.NewRequest(method, f.Url, nil)
	if err != nil {
		return nil, err
	}

	for k, v := range f.Headers {
		req.Header.Set(k, v)
	}

	for k, v := range headers {
		req.Header.Set(k, v)
	}

	retry := 0

	for {
		res, err := f.client.Do(req)
		if err != nil {
			return res, err
		}

		if res.StatusCode == 403 && res.Header.Get("X-Cache") == "Error from cloudfront" {
			log.Println(req, err)

			retry++

			if retry < HttpFileRetries {
				log.Println("Retry", retry, "Sleep...")
				time.Sleep(HttpFileRetryWait)
				continue
			}
		}

		return res, err
	}
}

func (f *HttpFile) getContentRange(resp *http.Response) (first, last, total int64, err error) {
	content_range := resp.Header.Get("Content-Range")

	n, err := fmt.Sscanf(content_range, "bytes %d-%d/%d", &first, &last, &total)
	if err != nil {
		DebugLog(f.Debug).Println("Error", err)
		return -1, -1, -1, err
	}
	if n != 3 {
		return -1, -1, -1, &HttpFileError{Err: fmt.Errorf("Unexpected Content-Range %q (%d)", content_range, n)}
	}

	return first, last, total, nil
}

// Returns the file size
func (f *HttpFile) Size() int64 {
	DebugLog(f.Debug).Println("Size", f.len)
	return f.len
}

func (f *HttpFile) readAt(p []byte, off int64) (int, error) {
	DebugLog(f.Debug).Println("readAt", off, len(p))

	if f.client == nil {
		return 0, os.ErrInvalid
	}

	plen := len(p)
	if plen <= 0 {
		return plen, nil
	}

	end := off + int64(plen)
	if end > f.len {
		end = f.len
	}

	bytes_range := fmt.Sprintf("bytes=%d-%d", off, end-1)
	resp, err := f.do("GET", headersType{"Range": bytes_range})
	defer CloseResponse(resp)

	switch {
	case err != nil:
		DebugLog(f.Debug).Println("readAt error", err)
		return 0, &HttpFileError{Err: err}

	case resp.StatusCode == http.StatusRequestedRangeNotSatisfiable:
		DebugLog(f.Debug).Println("readAt http.StatusRequestedRangeNotSatisfiable")
		return 0, io.EOF

	case resp.StatusCode != http.StatusPartialContent:
		DebugLog(f.Debug).Println("readAt error", resp.Status)
		return 0, &HttpFileError{Err: fmt.Errorf("Unexpected Status %s", resp.Status)}
	}

	first, last, total, err := f.getContentRange(resp)
	DebugLog(f.Debug).Println("Range", bytes_range, "Content-Range", first, last, total)

	n, err := io.ReadFull(resp.Body, p)
	if n > 0 && err == io.EOF {
		// read reached EOF, but archive/zip doesn't like this!
		DebugLog(f.Debug).Println("readAt", n, "reached EOF")
		err = nil
	}

	DebugLog(f.Debug).Println("readAt", n, err)
	return n, err
}

func (f *HttpFile) readFromBuffer(p []byte, off int64) (int, error) {
	ppos := 0
	plen := len(p)

	if off != f.bpos {
		blen := f.bend - f.bstart

		if blen == 0 {
			f.bstart = 0
			f.bend = 0
		} else if f.bpos < off && f.bpos+int64(blen) > off {
			drop := int(off - f.bpos)
			f.bstart += drop

			DebugLog(f.Debug).Println("readFrom", off, "pos", f.bpos, "drop", drop, "bytes, saved", blen-drop, "bytes")
		} else {
			DebugLog(f.Debug).Println("readFrom", off, "pos", f.bpos, "dropping", blen, "bytes")

			f.bstart = 0
			f.bend = 0
		}

		f.bpos = off
	}

	for ppos < plen {
		DebugLog(f.Debug).Println("readFromBuffer", ppos, plen, "pos", f.bpos)

		if f.bstart < f.bend { // there is already some data
			n := copy(p[ppos:], f.Buffer[f.bstart:f.bend])

			f.bstart += n
			f.bpos += int64(n)
			ppos += n

			if ppos >= plen {
				DebugLog(f.Debug).Println("readFromBuffer", ppos, "done", "pos", f.bpos)
				return ppos, nil
			}
		}

		if plen-ppos > len(f.Buffer) { // no need to read in buffer
			f.bstart = 0
			f.bend = 0

			return f.readAt(p[ppos:], f.bpos)
		}

		n, err := f.readAt(f.Buffer, f.bpos)

		f.bstart = 0
		f.bend = n

		if err != nil && n == 0 { // don't return an error if we read something
			DebugLog(f.Debug).Println("readFromBuffer", "error", err)
			return 0, err
		}
	}

	panic("should not get here")
	return 0, nil
}

// The ReaderAt interface
func (f *HttpFile) ReadAt(p []byte, off int64) (int, error) {
	DebugLog(f.Debug).Println("ReadAt", off, "len", len(p))

	if f.Buffer != nil {
		return f.readFromBuffer(p, off)
	}

	return f.readAt(p, off)
}

// The Reader interface
func (f *HttpFile) Read(p []byte) (int, error) {
	DebugLog(f.Debug).Println("Read from", f.pos, "len", len(p))

	n, err := f.ReadAt(p, f.pos)
	if n > 0 {
		f.pos += int64(n)
	}

	DebugLog(f.Debug).Println("Read", n, err)
	return n, err
}

// The Closer interface
func (f *HttpFile) Close() error {
	DebugLog(f.Debug).Println("Close")
	f.client = nil
	f.pos = -1
	f.len = -1
	return nil
}

// The Seeker interface
func (f *HttpFile) Seek(offset int64, whence int) (int64, error) {
	DebugLog(f.Debug).Println("Seek", offset, whence)

	var newpos int64 = -1

	if f.client != nil {
		switch whence {
		case 0: // from 0
			newpos = offset

		case 1: // from current
			newpos = f.pos + offset

		case 2: // from end
			newpos = f.len + offset
		}
	}

	if newpos < 0 {
		return 0, os.ErrInvalid
	} else {
		if f.pos != newpos {
			f.pos = newpos
		}

		return f.pos, nil
	}
}
<|endoftext|>


<commit_before>package ethui

import (
	"bitbucket.org/kardianos/osext"
	"fmt"
	"github.com/ethereum/eth-go"
	"github.com/ethereum/eth-go/ethchain"
	"github.com/ethereum/eth-go/ethutil"
	"github.com/ethereum/go-ethereum/utils"
	"github.com/niemeyer/qml"
	"math/big"
	"os"
	"path"
	"path/filepath"
	"runtime"
)

type memAddr struct {
	Num   string
	Value string
}

// UI Library that has some basic functionality exposed
type UiLib struct {
	engine    *qml.Engine
	eth       *eth.Ethereum
	connected bool
	assetPath string
	// The main application window
	win *qml.Window
	Db  *Debugger
}

func NewUiLib(engine *qml.Engine, eth *eth.Ethereum, assetPath string) *UiLib {
	if assetPath == "" {
		assetPath = DefaultAssetPath()
	}
	return &UiLib{engine: engine, eth: eth, assetPath: assetPath}
}

// Opens a QML file (external application)
func (ui *UiLib) Open(path string) {
	component, err := ui.engine.LoadFile(path[7:])
	if err != nil {
		ethutil.Config.Log.Debugln(err)
	}
	win := component.CreateWindow(nil)

	go func() {
		win.Show()
		win.Wait()
	}()
}

func (ui *UiLib) Connect(button qml.Object) {
	if !ui.connected {
		ui.eth.Start()
		ui.connected = true
		button.Set("enabled", false)
	}
}

func (ui *UiLib) ConnectToPeer(addr string) {
	ui.eth.ConnectToPeer(addr)
}

func (ui *UiLib) AssetPath(p string) string {
	return path.Join(ui.assetPath, p)
}

func DefaultAssetPath() string {
	var base string
	// If the current working directory is the go-ethereum dir
	// assume a debug build and use the source directory as
	// asset directory.
	pwd, _ := os.Getwd()
	if pwd == path.Join(os.Getenv("GOPATH"), "src", "github.com", "ethereum", "go-ethereum", "ethereal") {
		base = path.Join(pwd, "assets")
	} else {
		switch runtime.GOOS {
		case "darwin":
			// Get Binary Directory
			exedir, _ := osext.ExecutableFolder()
			base = filepath.Join(exedir, "../Resources")
		case "linux":
			base = "/usr/share/ethereal"
		case "window":
			fallthrough
		default:
			base = "."
		}
	}

	return base
}

func (ui *UiLib) DebugTx(recipient, valueStr, gasStr, gasPriceStr, data string) {
	state := ui.eth.BlockChain().CurrentBlock.State()

	mainInput, _ := ethutil.PreProcess(data)
	callerScript, err := utils.Compile(mainInput)
	if err != nil {
		ethutil.Config.Log.Debugln(err)

		return
	}

	dis := ethchain.Disassemble(callerScript)
	ui.win.Root().Call("clearAsm")

	for _, str := range dis {
		ui.win.Root().Call("setAsm", str)
	}
	callerTx := ethchain.NewContractCreationTx(ethutil.Big(valueStr), ethutil.Big(gasPriceStr), callerScript, nil)

	// Contract addr as test address
	keyPair := ethutil.Config.Db.GetKeys()[0]
	account := ui.eth.StateManager().GetAddrState(keyPair.Address()).Object
	c := ethchain.MakeContract(callerTx, state)
	callerClosure := ethchain.NewClosure(account, c, c.Script(), state, ethutil.Big(gasStr), new(big.Int))

	block := ui.eth.BlockChain().CurrentBlock
	vm := ethchain.NewVm(state, ethchain.RuntimeVars{
		Origin:      account.Address(),
		BlockNumber: block.BlockInfo().Number,
		PrevHash:    block.PrevHash,
		Coinbase:    block.Coinbase,
		Time:        block.Time,
		Diff:        block.Difficulty,
		TxData:      nil,
	})

	go func() {
		callerClosure.Call(vm, nil, ui.Db.halting)

		state.Reset()
	}()
}

func (ui *UiLib) Next() {
	ui.Db.Next()
}

type Debugger struct {
	win *qml.Window
	N   chan bool
}

func (d *Debugger) halting(pc int, op ethchain.OpCode, mem *ethchain.Memory, stack *ethchain.Stack) {
	d.win.Root().Call("setInstruction", pc)
	d.win.Root().Call("clearMem")
	d.win.Root().Call("clearStack")

	addr := 0
	for i := 0; i+32 <= mem.Len(); i += 32 {
		d.win.Root().Call("setMem", memAddr{fmt.Sprintf("%03d", addr), fmt.Sprintf("% x", mem.Data()[i:i+32])})
		addr++
	}

	for _, val := range stack.Data() {
		d.win.Root().Call("setStack", val.String())
	}

out:
	for {
		select {
		case <-d.N:
			break out
		default:
		}
	}
}

func (d *Debugger) Next() {
	d.N <- true
}
<commit_msg>Updated closure call<commit_after>package ethui

import (
	"bitbucket.org/kardianos/osext"
	"fmt"
	"github.com/ethereum/eth-go"
	"github.com/ethereum/eth-go/ethchain"
	"github.com/ethereum/eth-go/ethutil"
	"github.com/ethereum/go-ethereum/utils"
	"github.com/niemeyer/qml"
	"os"
	"path"
	"path/filepath"
	"runtime"
)

type memAddr struct {
	Num   string
	Value string
}

// UI Library that has some basic functionality exposed
type UiLib struct {
	engine    *qml.Engine
	eth       *eth.Ethereum
	connected bool
	assetPath string
	// The main application window
	win *qml.Window
	Db  *Debugger
}

func NewUiLib(engine *qml.Engine, eth *eth.Ethereum, assetPath string) *UiLib {
	if assetPath == "" {
		assetPath = DefaultAssetPath()
	}
	return &UiLib{engine: engine, eth: eth, assetPath: assetPath}
}

// Opens a QML file (external application)
func (ui *UiLib) Open(path string) {
	component, err := ui.engine.LoadFile(path[7:])
	if err != nil {
		ethutil.Config.Log.Debugln(err)
	}
	win := component.CreateWindow(nil)

	go func() {
		win.Show()
		win.Wait()
	}()
}

func (ui *UiLib) Connect(button qml.Object) {
	if !ui.connected {
		ui.eth.Start()
		ui.connected = true
		button.Set("enabled", false)
	}
}

func (ui *UiLib) ConnectToPeer(addr string) {
	ui.eth.ConnectToPeer(addr)
}

func (ui *UiLib) AssetPath(p string) string {
	return path.Join(ui.assetPath, p)
}

func DefaultAssetPath() string {
	var base string
	// If the current working directory is the go-ethereum dir
	// assume a debug build and use the source directory as
	// asset directory.
	pwd, _ := os.Getwd()
	if pwd == path.Join(os.Getenv("GOPATH"), "src", "github.com", "ethereum", "go-ethereum", "ethereal") {
		base = path.Join(pwd, "assets")
	} else {
		switch runtime.GOOS {
		case "darwin":
			// Get Binary Directory
			exedir, _ := osext.ExecutableFolder()
			base = filepath.Join(exedir, "../Resources")
		case "linux":
			base = "/usr/share/ethereal"
		case "window":
			fallthrough
		default:
			base = "."
		}
	}

	return base
}

func (ui *UiLib) DebugTx(recipient, valueStr, gasStr, gasPriceStr, data string) {
	state := ui.eth.BlockChain().CurrentBlock.State()

	mainInput, _ := ethutil.PreProcess(data)
	callerScript, err := utils.Compile(mainInput)
	if err != nil {
		ethutil.Config.Log.Debugln(err)

		return
	}

	dis := ethchain.Disassemble(callerScript)
	ui.win.Root().Call("clearAsm")

	for _, str := range dis {
		ui.win.Root().Call("setAsm", str)
	}
	callerTx := ethchain.NewContractCreationTx(ethutil.Big(valueStr), ethutil.Big(gasPriceStr), callerScript, nil)

	// Contract addr as test address
	keyPair := ethutil.Config.Db.GetKeys()[0]
	account := ui.eth.StateManager().GetAddrState(keyPair.Address()).Object
	c := ethchain.MakeContract(callerTx, state)
	callerClosure := ethchain.NewClosure(account, c, c.Script(), state, ethutil.Big(gasStr), ethutil.Big(gasPriceStr), ethutil.Big(valueStr))

	block := ui.eth.BlockChain().CurrentBlock
	vm := ethchain.NewVm(state, ethchain.RuntimeVars{
		Origin:      account.Address(),
		BlockNumber: block.BlockInfo().Number,
		PrevHash:    block.PrevHash,
		Coinbase:    block.Coinbase,
		Time:        block.Time,
		Diff:        block.Difficulty,
		TxData:      nil,
	})

	go func() {
		callerClosure.Call(vm, nil, ui.Db.halting)

		state.Reset()
	}()
}

func (ui *UiLib) Next() {
	ui.Db.Next()
}

type Debugger struct {
	win *qml.Window
	N   chan bool
}

func (d *Debugger) halting(pc int, op ethchain.OpCode, mem *ethchain.Memory, stack *ethchain.Stack) {
	d.win.Root().Call("setInstruction", pc)
	d.win.Root().Call("clearMem")
	d.win.Root().Call("clearStack")

	addr := 0
	for i := 0; i+32 <= mem.Len(); i += 32 {
		d.win.Root().Call("setMem", memAddr{fmt.Sprintf("%03d", addr), fmt.Sprintf("% x", mem.Data()[i:i+32])})
		addr++
	}

	for _, val := range stack.Data() {
		d.win.Root().Call("setStack", val.String())
	}

out:
	for {
		select {
		case <-d.N:
			break out
		default:
		}
	}
}

func (d *Debugger) Next() {
	d.N <- true
}
<|endoftext|>


<commit_before>package event

import (
	"testing"

	"github.com/stretchr/testify/assert"
)

func TestHandler(t *testing.T) {
	updateCh := make(chan bool)
	UpdateLoop(60, updateCh)
	triggers := 0
	Bind(func(int, interface{}) int {
		triggers++
		return 0
	}, Enter, 0)
	sleep()
	assert.Equal(t, 1, triggers)
	<-updateCh
	sleep()
	assert.Equal(t, 2, triggers)
	assert.Equal(t, 2, FramesElapsed())
	assert.Nil(t, SetTick(1))
	<-updateCh
	assert.Nil(t, Stop())
	sleep()
	sleep()
	select {
	case <-updateCh:
		t.Fatal("Handler should be closed")
	default:
	}
	Update()
	sleep()
	assert.Equal(t, 3, triggers)
	assert.Nil(t, Flush())

	BindPriority(func(int, interface{}) int {
		triggers = 100
		return 0
	}, BindingOption{
		Event: Event{
			Name:     Enter,
			CallerID: 0,
		},
		Priority: 4,
	})

	BindPriority(func(int, interface{}) int {
		if triggers != 100 {
			t.Fatal("Wrong call order")
		}
		return 0
	}, BindingOption{
		Event: Event{
			Name:     Enter,
			CallerID: 0,
		},
		Priority: 3,
	})

	Flush()
	sleep()
	Update()
	sleep()
	sleep()
	Reset()
}
<commit_msg>Add a benchmark for the logic loop<commit_after>package event

import (
	"testing"

	"github.com/stretchr/testify/assert"
)

func TestHandler(t *testing.T) {
	updateCh := make(chan bool)
	UpdateLoop(60, updateCh)
	triggers := 0
	Bind(func(int, interface{}) int {
		triggers++
		return 0
	}, Enter, 0)
	sleep()
	assert.Equal(t, 1, triggers)
	<-updateCh
	sleep()
	assert.Equal(t, 2, triggers)
	assert.Equal(t, 2, FramesElapsed())
	assert.Nil(t, SetTick(1))
	<-updateCh
	assert.Nil(t, Stop())
	sleep()
	sleep()
	select {
	case <-updateCh:
		t.Fatal("Handler should be closed")
	default:
	}
	Update()
	sleep()
	assert.Equal(t, 3, triggers)
	assert.Nil(t, Flush())

	BindPriority(func(int, interface{}) int {
		triggers = 100
		return 0
	}, BindingOption{
		Event: Event{
			Name:     Enter,
			CallerID: 0,
		},
		Priority: 4,
	})

	BindPriority(func(int, interface{}) int {
		if triggers != 100 {
			t.Fatal("Wrong call order")
		}
		return 0
	}, BindingOption{
		Event: Event{
			Name:     Enter,
			CallerID: 0,
		},
		Priority: 3,
	})

	Flush()
	sleep()
	Update()
	sleep()
	sleep()
	Reset()
}

func BenchmarkHandler(b *testing.B) {
	triggers := 0
	entities := 10
	go DefaultBus.ResolvePending()
	for i := 0; i < entities; i++ {
		DefaultBus.GlobalBind(func(int, interface{}) int {
			triggers++
			return 0
		}, Enter)
	}
	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		<-DefaultBus.TriggerBack(Enter, DefaultBus.framesElapsed)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"log"
	"os"
	"runtime/pprof"
	"time"

	"github.com/mrfuxi/neural"
	"github.com/petar/GoMNIST"
)

var (
	cpuprofile = flag.String("cpuprofile", "", "write cpu profile to file")
	nnSaveFile = flag.String("save-file", "", "Save neural network to file")
	nnLoadFile = flag.String("load-file", "", "Load neural network to file")
	inputSize  = GoMNIST.Width * GoMNIST.Height
)

func prepareMnistData(rawData *GoMNIST.Set) []neural.TrainExample {
	trainData := make([]neural.TrainExample, rawData.Count())
	for i := range trainData {
		image, label := rawData.Get(i)
		trainData[i].Input = make([]float64, inputSize, inputSize)
		trainData[i].Output = make([]float64, 10, 10)
		for j, pix := range image {
			trainData[i].Input[j] = (float64(pix)/255)*0.9 + 0.1
		}

		for j := range trainData[i].Output {
			trainData[i].Output[j] = 0.1
		}
		trainData[i].Output[label] = 0.9
	}
	return trainData
}

func loadTestData() ([]neural.TrainExample, []neural.TrainExample) {
	train, test, err := GoMNIST.Load("./data")
	if err != nil {
		panic(err)
	}

	trainData := prepareMnistData(train)
	testData := prepareMnistData(test)
	return trainData, testData
}

func epocheCallback(nn neural.Evaluator, cost neural.Cost, trainData, testData []neural.TrainExample) neural.EpocheCallback {
	return func(epoche int, dt time.Duration) {
		avgCost, errors := neural.CalculateCorrectness(nn, cost, testData)
		fmt.Printf("%v,%v,%v\n", epoche, avgCost, errors)
	}
}

func main() {
	trainData, testData := loadTestData()

	activator := neural.NewSigmoidActivator()
	nn := neural.NewNeuralNetwork(
		[]int{inputSize, 100, 10},
		neural.NewFullyConnectedLayer(activator),
		neural.NewFullyConnectedLayer(activator),
	)

	flag.Parse()

	if *nnLoadFile != "" {
		fn, err := os.Open(*nnLoadFile)
		if err != nil {
			log.Fatalln(err)
		}
		if err := neural.Load(nn, fn); err != nil {
			log.Fatalln(err)
		}
	}

	if *cpuprofile != "" {
		f, err := os.Create(*cpuprofile)
		if err != nil {
			log.Fatal(err)
		}
		pprof.StartCPUProfile(f)
		defer pprof.StopCPUProfile()
	}

	cost := neural.NewQuadraticCost()
	options := neural.TrainOptions{
		Epochs:         10,
		MiniBatchSize:  10,
		LearningRate:   4,
		Cost:           cost,
		TrainerFactory: neural.NewBackwardPropagationTrainer,
		EpocheCallback: epocheCallback(nn, cost, trainData, testData),
	}

	t0 := time.Now()
	neural.Train(nn, trainData, options)
	dt := time.Since(t0)

	fmt.Println("Training complete in", dt)

	if *nnSaveFile != "" {
		fn, err := os.OpenFile(*nnSaveFile, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0666)
		if err != nil {
			log.Fatalln(err)
		}
		if err := neural.Save(nn, fn); err != nil {
			log.Fatalln(err)
		}
	}
}
<commit_msg>Fix example<commit_after>package main

import (
	"flag"
	"fmt"
	"log"
	"os"
	"runtime/pprof"
	"time"

	"github.com/mrfuxi/neural"
	"github.com/petar/GoMNIST"
)

var (
	cpuprofile = flag.String("cpuprofile", "", "write cpu profile to file")
	nnSaveFile = flag.String("save-file", "", "Save neural network to file")
	nnLoadFile = flag.String("load-file", "", "Load neural network to file")
	inputSize  = GoMNIST.Width * GoMNIST.Height
)

func prepareMnistData(rawData *GoMNIST.Set) []neural.TrainExample {
	trainData := make([]neural.TrainExample, rawData.Count())
	for i := range trainData {
		image, label := rawData.Get(i)
		trainData[i].Input = make([]float64, inputSize, inputSize)
		trainData[i].Output = make([]float64, 10, 10)
		for j, pix := range image {
			trainData[i].Input[j] = (float64(pix)/255)*0.9 + 0.1
		}

		for j := range trainData[i].Output {
			trainData[i].Output[j] = 0.1
		}
		trainData[i].Output[label] = 0.9
	}
	return trainData
}

func loadTestData() ([]neural.TrainExample, []neural.TrainExample) {
	train, test, err := GoMNIST.Load("./data")
	if err != nil {
		panic(err)
	}

	trainData := prepareMnistData(train)
	testData := prepareMnistData(test)
	return trainData, testData
}

func epocheCallback(nn neural.Evaluator, cost neural.Cost, trainData, testData []neural.TrainExample) neural.EpocheCallback {
	return func(epoche int, dt time.Duration) {
		avgCost, errors := neural.CalculateCorrectness(nn, cost, testData)
		fmt.Printf("%v,%v,%v\n", epoche, avgCost, errors)
	}
}

func main() {
	trainData, testData := loadTestData()

	activator := neural.NewSigmoidActivator()
	nn := neural.NewNeuralNetwork(
		[]int{inputSize, 100, 10},
		neural.NewFullyConnectedLayer(activator),
		neural.NewFullyConnectedLayer(activator),
	)

	flag.Parse()

	if *nnLoadFile != "" {
		fn, err := os.Open(*nnLoadFile)
		if err != nil {
			log.Fatalln(err)
		}
		if err := neural.Load(nn, fn); err != nil {
			log.Fatalln(err)
		}
	}

	if *cpuprofile != "" {
		f, err := os.Create(*cpuprofile)
		if err != nil {
			log.Fatal(err)
		}
		pprof.StartCPUProfile(f)
		defer pprof.StopCPUProfile()
	}

	trainer := neural.NewQuadraticCostTrainer
	options := neural.TrainOptions{
		Epochs:         10,
		MiniBatchSize:  10,
		LearningRate:   4,
		TrainerFactory: trainer,
		EpocheCallback: epocheCallback(nn, trainer(nn), trainData, testData),
	}

	t0 := time.Now()
	neural.Train(nn, trainData, options)
	dt := time.Since(t0)

	fmt.Println("Training complete in", dt)

	if *nnSaveFile != "" {
		fn, err := os.OpenFile(*nnSaveFile, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0666)
		if err != nil {
			log.Fatalln(err)
		}
		if err := neural.Save(nn, fn); err != nil {
			log.Fatalln(err)
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"net/http"
	"time"

	"github.com/mreithub/goref"
)

func indexHTML(w http.ResponseWriter, r *http.Request) {
	ref := goref.Ref("/")
	defer ref.Deref()

	w.Write([]byte(`
Index

  delayed.html

  goref.json`))
}

func delayedHTML(w http.ResponseWriter, r *http.Request) {
	ref := goref.Ref("/delayed.html")
	defer ref.Deref()

	time.Sleep(200 * time.Millisecond)
	msg := fmt.Sprintf("The time is %s", time.Now().String())
	w.Write([]byte(msg))
}

func gorefJSON(w http.ResponseWriter, r *http.Request) {
	ref := goref.Ref("/goref.json")
	defer ref.Deref()

	data, _ := json.Marshal(goref.Clone().Data)

	w.Header().Add("Content-type", "application/json")
	w.Write(data)
}

func main() {
	http.HandleFunc("/", indexHTML)
	http.HandleFunc("/delayed.html", delayedHTML)
	http.HandleFunc("/goref.json", gorefJSON)

	http.ListenAndServe("localhost:1234", nil)
}
<commit_msg>updated examples/webserver.go<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"net/http"
	"time"

	"github.com/mreithub/goref"
)

func indexHTML(w http.ResponseWriter, r *http.Request) {
	ref := goref.Ref("/")
	defer ref.Deref()

	w.Write([]byte(`
Index

  delayed.html

  goref.json`))
}

func delayedHTML(w http.ResponseWriter, r *http.Request) {
	ref := goref.Ref("/delayed.html")
	defer ref.Deref()

	time.Sleep(200 * time.Millisecond)
	msg := fmt.Sprintf("The time is %s", time.Now().String())
	w.Write([]byte(msg))
}

func gorefJSON(w http.ResponseWriter, r *http.Request) {
	ref := goref.Ref("/goref.json")
	defer ref.Deref()

	data, _ := json.Marshal(goref.GetSnapshot().Data)

	w.Header().Add("Content-type", "application/json")
	w.Write(data)
}

func main() {
	http.HandleFunc("/", indexHTML)
	http.HandleFunc("/delayed.html", delayedHTML)
	http.HandleFunc("/goref.json", gorefJSON)

	http.ListenAndServe("localhost:1234", nil)
}
<|endoftext|>


<commit_before>package ini

import (
	"reflect"
	"strings"
	"syscall"
	"testing"
)

func TestLoad(t *testing.T) {
	src := `
  # Comments are ignored

  herp = derp

  [foo]
  hello=world
  whitespace should   =   not matter
  ; sneaky semicolon-style comment
  multiple = equals = signs

  [bar]
  this = that`

	file, err := Load(strings.NewReader(src))
	if err != nil {
		t.Fatal(err)
	}
	check := func(section, key, expect string) {
		if value, _ := file.Get(section, key); value != expect {
			t.Errorf("Get(%q, %q): expected %q, got %q", section, key, expect, value)
		}
	}

	check("", "herp", "derp")
	check("foo", "hello", "world")
	check("foo", "whitespace should", "not matter")
	check("foo", "multiple", "equals = signs")
	check("bar", "this", "that")
}

func TestSyntaxError(t *testing.T) {
	src := `
  # Line 2
  [foo]
  bar = baz
  # Here's an error on line 6:
  wut?
  herp = derp`
	_, err := Load(strings.NewReader(src))
	t.Logf("%T: %v", err, err)
	if err == nil {
		t.Fatal("expected an error, got nil")
	}
	syntaxErr, ok := err.(ErrSyntax)
	if !ok {
		t.Fatal("expected an error of type ErrSyntax")
	}
	if syntaxErr.Line != 6 {
		t.Fatal("incorrect line number")
	}
	if syntaxErr.Source != "wut?" {
		t.Fatal("incorrect source")
	}
}

func TestDefinedSectionBehaviour(t *testing.T) {
	check := func(src string, expect File) {
		file, err := Load(strings.NewReader(src))
		if err != nil {
			t.Fatal(err)
		}
		if !reflect.DeepEqual(file, expect) {
			t.Errorf("expected %v, got %v", expect, file)
		}
	}
	// No sections for an empty file
	check("", File{})
	// Default section only if there are actually values for it
	check("foo=bar", File{"": {"foo": "bar"}})
	// User-defined sections should always be present, even if empty
	check("[a]\n[b]\nfoo=bar", File{
		"a": {},
		"b": {"foo": "bar"},
	})
	check("foo=bar\n[a]\nthis=that", File{
		"":  {"foo": "bar"},
		"a": {"this": "that"},
	})
}

func TestLoadFile(t *testing.T) {
	originalOpenFiles := numFilesOpen()

	file, err := LoadFile("test.ini")
	if err != nil {
		t.Fatal(err)
	}

	if originalOpenFiles != numFilesOpen() {
		t.Error("test.ini not closed")
	}

	if !reflect.DeepEqual(file, File{"default": {"stuff": "things"}}) {
		t.Error("file not read correctly")
	}
}

func numFilesOpen() (num uint64) {
	var rlimit syscall.Rlimit
	syscall.Getrlimit(syscall.RLIMIT_NOFILE, &rlimit)
	maxFds := int(rlimit.Cur)

	var stat syscall.Stat_t
	for i := 0; i < maxFds; i++ {
		if syscall.Fstat(i, &stat) == nil {
			num++
		} else {
			return
		}
	}
	return
}
<commit_msg>Don't ignore Getrlimit error in test<commit_after>package ini

import (
	"reflect"
	"strings"
	"syscall"
	"testing"
)

func TestLoad(t *testing.T) {
	src := `
  # Comments are ignored

  herp = derp

  [foo]
  hello=world
  whitespace should   =   not matter
  ; sneaky semicolon-style comment
  multiple = equals = signs

  [bar]
  this = that`

	file, err := Load(strings.NewReader(src))
	if err != nil {
		t.Fatal(err)
	}
	check := func(section, key, expect string) {
		if value, _ := file.Get(section, key); value != expect {
			t.Errorf("Get(%q, %q): expected %q, got %q", section, key, expect, value)
		}
	}

	check("", "herp", "derp")
	check("foo", "hello", "world")
	check("foo", "whitespace should", "not matter")
	check("foo", "multiple", "equals = signs")
	check("bar", "this", "that")
}

func TestSyntaxError(t *testing.T) {
	src := `
  # Line 2
  [foo]
  bar = baz
  # Here's an error on line 6:
  wut?
  herp = derp`
	_, err := Load(strings.NewReader(src))
	t.Logf("%T: %v", err, err)
	if err == nil {
		t.Fatal("expected an error, got nil")
	}
	syntaxErr, ok := err.(ErrSyntax)
	if !ok {
		t.Fatal("expected an error of type ErrSyntax")
	}
	if syntaxErr.Line != 6 {
		t.Fatal("incorrect line number")
	}
	if syntaxErr.Source != "wut?" {
		t.Fatal("incorrect source")
	}
}

func TestDefinedSectionBehaviour(t *testing.T) {
	check := func(src string, expect File) {
		file, err := Load(strings.NewReader(src))
		if err != nil {
			t.Fatal(err)
		}
		if !reflect.DeepEqual(file, expect) {
			t.Errorf("expected %v, got %v", expect, file)
		}
	}
	// No sections for an empty file
	check("", File{})
	// Default section only if there are actually values for it
	check("foo=bar", File{"": {"foo": "bar"}})
	// User-defined sections should always be present, even if empty
	check("[a]\n[b]\nfoo=bar", File{
		"a": {},
		"b": {"foo": "bar"},
	})
	check("foo=bar\n[a]\nthis=that", File{
		"":  {"foo": "bar"},
		"a": {"this": "that"},
	})
}

func TestLoadFile(t *testing.T) {
	originalOpenFiles := numFilesOpen(t)

	file, err := LoadFile("test.ini")
	if err != nil {
		t.Fatal(err)
	}

	if originalOpenFiles != numFilesOpen(t) {
		t.Error("test.ini not closed")
	}

	if !reflect.DeepEqual(file, File{"default": {"stuff": "things"}}) {
		t.Error("file not read correctly")
	}
}

func numFilesOpen(t *testing.T) (num uint64) {
	var rlimit syscall.Rlimit
	err := syscall.Getrlimit(syscall.RLIMIT_NOFILE, &rlimit)
	if err != nil {
		t.Fatal(err)
	}
	maxFds := int(rlimit.Cur)

	var stat syscall.Stat_t
	for i := 0; i < maxFds; i++ {
		if syscall.Fstat(i, &stat) == nil {
			num++
		} else {
			return
		}
	}
	return
}
<|endoftext|>


<commit_before>package iniflags

import (
	"bufio"
	"flag"
	"fmt"
	"io"
	"log"
	"net/http"
	"net/url"
	"os"
	"os/signal"
	"path"
	"strings"
	"syscall"
	"time"
)

var (
	config               = flag.String("config", "", "Path to ini config for using in go flags. May be relative to the current executable path.")
	configUpdateInterval = flag.Duration("configUpdateInterval", 0, "Update interval for re-reading config file set via -config flag. Zero disables config file re-reading.")
	dumpflags            = flag.Bool("dumpflags", false, "Dumps values for all flags defined in the app into stdout in ini-compatible syntax and terminates the app.")
)

var (
	flagChangeCallbacks = make(map[string][]FlagChangeCallback)
	importStack         []string
	parsed              bool
)

// Generation is flags' generation number.
//
// It is modified on each flags' modification
// via either -configUpdateInterval or SIGHUP.
var Generation int

// Parse obtains flag values from config file set via -config.
//
// It obtains flag values from command line like flag.Parse(), then overrides
// them by values parsed from config file set via -config.
//
// Path to config file can also be set via SetConfigFile() before Parse() call.
func Parse() {
	if parsed {
		panic("iniflags: duplicate call to iniflags.Parse() detected")
	}

	parsed = true
	flag.Parse()
	_, ok := parseConfigFlags()
	if !ok {
		os.Exit(1)
	}

	if *dumpflags {
		dumpFlags()
		os.Exit(0)
	}

	for flagName := range flagChangeCallbacks {
		verifyFlagChangeFlagName(flagName)
	}
	Generation++
	issueAllFlagChangeCallbacks()

	ch := make(chan os.Signal)
	signal.Notify(ch, syscall.SIGHUP)
	go sighupHandler(ch)

	go configUpdater()
}

func configUpdater() {
	if *configUpdateInterval != 0 {
		for {
			// Use time.Sleep() instead of time.Tick() for the sake of dynamic flag update.
			time.Sleep(*configUpdateInterval)
			updateConfig()
		}
	}
}

func updateConfig() {
	if oldFlagValues, ok := parseConfigFlags(); ok && len(oldFlagValues) > 0 {
		modifiedFlags := make(map[string]string)
		for k := range oldFlagValues {
			modifiedFlags[k] = flag.Lookup(k).Value.String()
		}
		log.Printf("iniflags: read updated config. Modified flags are: %v\n", modifiedFlags)
		Generation++
		issueFlagChangeCallbacks(oldFlagValues)
	}
}

// FlagChangeCallback is called when the given flag is changed.
//
// The callback may be registered for any flag via OnFlagChange().
type FlagChangeCallback func()

// OnFlagChange registers the callback, which is called after the given flag
// value is initialized and/or changed.
//
// Flag values are initialized during iniflags.Parse() call.
// Flag value can be changed on config re-read after obtaining SIGHUP signal
// or if periodic config re-read is enabled with -configUpdateInterval flag.
//
// Note that flags set via command-line cannot be overriden via config file modifications.
func OnFlagChange(flagName string, callback FlagChangeCallback) {
	if parsed {
		verifyFlagChangeFlagName(flagName)
	}
	flagChangeCallbacks[flagName] = append(flagChangeCallbacks[flagName], callback)
}

func verifyFlagChangeFlagName(flagName string) {
	if flag.Lookup(flagName) == nil {
		log.Fatalf("iniflags: cannot register FlagChangeCallback for non-existing flag [%s]\n", flagName)
	}
}

func issueFlagChangeCallbacks(oldFlagValues map[string]string) {
	for flagName := range oldFlagValues {
		if fs, ok := flagChangeCallbacks[flagName]; ok {
			for _, f := range fs {
				f()
			}
		}
	}
}

func issueAllFlagChangeCallbacks() {
	for _, fs := range flagChangeCallbacks {
		for _, f := range fs {
			f()
		}
	}
}

func sighupHandler(ch <-chan os.Signal) {
	for range ch {
		updateConfig()
	}
}

func parseConfigFlags() (oldFlagValues map[string]string, ok bool) {
	configPath := *config
	if !strings.HasPrefix(configPath, "./") {
		if configPath, ok = combinePath(os.Args[0], *config); !ok {
			return nil, false
		}
	}
	if configPath == "" {
		return nil, true
	}
	parsedArgs, ok := getArgsFromConfig(configPath)
	if !ok {
		return nil, false
	}
	missingFlags := getMissingFlags()

	ok = true
	oldFlagValues = make(map[string]string)
	for _, arg := range parsedArgs {
		f := flag.Lookup(arg.Key)
		if f == nil {
			log.Printf("iniflags: unknown flag name=[%s] found at line [%d] of file [%s]\n", arg.Key, arg.LineNum, arg.FilePath)
			ok = false
			continue
		}

		if _, found := missingFlags[f.Name]; found {
			oldValue := f.Value.String()
			if oldValue == arg.Value {
				continue
			}
			if err := f.Value.Set(arg.Value); err != nil {
				log.Printf("iniflags: error when parsing flag [%s] value [%s] at line [%d] of file [%s]: [%s]\n", arg.Key, arg.Value, arg.LineNum, arg.FilePath, err)
				ok = false
				continue
			}
			if oldValue != f.Value.String() {
				oldFlagValues[arg.Key] = oldValue
			}
		}
	}

	if !ok {
		// restore old flag values
		for k, v := range oldFlagValues {
			flag.Set(k, v)
		}
		oldFlagValues = nil
	}

	return oldFlagValues, ok
}

func checkImportRecursion(configPath string) bool {
	for _, path := range importStack {
		if path == configPath {
			log.Printf("iniflags: import recursion found for [%s]: %v\n", configPath, importStack)
			return false
		}
	}
	return true
}

type flagArg struct {
	Key      string
	Value    string
	FilePath string
	LineNum  int
}

func stripBOM(s string) string {
	if len(s) < 3 {
		return s
	}
	bom := s[:3]
	if bom == "\ufeff" || bom == "\ufffe" {
		return s[3:]
	}
	return s
}

func getArgsFromConfig(configPath string) (args []flagArg, ok bool) {
	if !checkImportRecursion(configPath) {
		return nil, false
	}
	importStack = append(importStack, configPath)
	defer func() {
		importStack = importStack[:len(importStack)-1]
	}()

	file := openConfigFile(configPath)
	if file == nil {
		return nil, false
	}
	defer file.Close()
	r := bufio.NewReader(file)

	var lineNum int
	for {
		lineNum++
		line, err := r.ReadString('\n')
		if err != nil && line == "" {
			if err == io.EOF {
				break
			}
			log.Printf("iniflags: error when reading file [%s] at line %d: [%s]\n", configPath, lineNum, err)
			return nil, false
		}
		if lineNum == 1 {
			line = stripBOM(line)
		}
		line = strings.TrimSpace(line)
		if strings.HasPrefix(line, "#import ") {
			importPath, ok := unquoteValue(line[7:], lineNum, configPath)
			if !ok {
				return nil, false
			}
			if importPath, ok = combinePath(configPath, importPath); !ok {
				return nil, false
			}
			importArgs, ok := getArgsFromConfig(importPath)
			if !ok {
				return nil, false
			}
			args = append(args, importArgs...)
			continue
		}
		if line == "" || line[0] == ';' || line[0] == '#' || line[0] == '[' {
			continue
		}
		parts := strings.SplitN(line, "=", 2)
		if len(parts) != 2 {
			log.Printf("iniflags: cannot split [%s] at line %d into key and value in config file [%s]\n", line, lineNum, configPath)
			return nil, false
		}
		key := strings.TrimSpace(parts[0])
		value, ok := unquoteValue(parts[1], lineNum, configPath)
		if !ok {
			return nil, false
		}
		args = append(args, flagArg{Key: key, Value: value, FilePath: configPath, LineNum: lineNum})
	}

	return args, true
}

func openConfigFile(path string) io.ReadCloser {
	if isHTTP(path) {
		resp, err := http.Get(path)
		if err != nil {
			log.Printf("iniflags: cannot load config file at [%s]: [%s]\n", path, err)
			return nil
		}
		if resp.StatusCode != http.StatusOK {
			log.Printf("iniflags: unexpected http status code when obtaining config file [%s]: %d. Expected %d\n", path, resp.StatusCode, http.StatusOK)
			return nil
		}
		return resp.Body
	}

	file, err := os.Open(path)
	if err != nil {
		log.Printf("iniflags: cannot open config file at [%s]: [%s]\n", path, err)
		return nil
	}
	return file
}

func combinePath(basePath, relPath string) (string, bool) {
	if isHTTP(basePath) {
		base, err := url.Parse(basePath)
		if err != nil {
			log.Printf("iniflags: error when parsing http base path [%s]: %s\n", basePath, err)
			return "", false
		}
		rel, err := url.Parse(relPath)
		if err != nil {
			log.Printf("iniflags: error when parsing http rel path [%s] for base [%s]: %s\n", relPath, basePath, err)
			return "", false
		}
		return base.ResolveReference(rel).String(), true
	}

	if relPath == "" || relPath[0] == '/' || isHTTP(relPath) {
		return relPath, true
	}
	return path.Join(path.Dir(basePath), relPath), true
}

func isHTTP(path string) bool {
	return strings.HasPrefix(strings.ToLower(path), "http://") || strings.HasPrefix(strings.ToLower(path), "https://")
}

func getMissingFlags() map[string]bool {
	setFlags := make(map[string]bool)
	flag.Visit(func(f *flag.Flag) {
		setFlags[f.Name] = true
	})

	missingFlags := make(map[string]bool)
	flag.VisitAll(func(f *flag.Flag) {
		if _, ok := setFlags[f.Name]; !ok {
			missingFlags[f.Name] = true
		}
	})
	return missingFlags
}

func dumpFlags() {
	flag.VisitAll(func(f *flag.Flag) {
		if f.Name != "config" && f.Name != "dumpflags" {
			fmt.Printf("%s = %s  # %s\n", f.Name, quoteValue(f.Value.String()), escapeUsage(f.Usage))
		}
	})
}

func escapeUsage(s string) string {
	return strings.Replace(s, "\n", "\n    # ", -1)
}

func quoteValue(v string) string {
	if !strings.ContainsAny(v, "\n#;") && strings.TrimSpace(v) == v {
		return v
	}
	v = strings.Replace(v, "\\", "\\\\", -1)
	v = strings.Replace(v, "\n", "\\n", -1)
	v = strings.Replace(v, "\"", "\\\"", -1)
	return fmt.Sprintf("\"%s\"", v)
}

func unquoteValue(v string, lineNum int, configPath string) (string, bool) {
	v = strings.TrimSpace(v)
	if len(v) == 0 {
		return "", true
	}
	if v[0] != '"' {
		return removeTrailingComments(v), true
	}
	n := strings.LastIndex(v, "\"")
	if n == -1 {
		log.Printf("iniflags: unclosed string found [%s] at line %d in config file [%s]\n", v, lineNum, configPath)
		return "", false
	}
	v = v[1:n]
	v = strings.Replace(v, "\\\"", "\"", -1)
	v = strings.Replace(v, "\\n", "\n", -1)
	return strings.Replace(v, "\\\\", "\\", -1), true
}

func removeTrailingComments(v string) string {
	v = strings.Split(v, "#")[0]
	v = strings.Split(v, ";")[0]
	return strings.TrimSpace(v)
}

// SetConfigFile sets path to config file.
//
// Call this function before Parse() if you need default path to config file
// when -config command-line flag is not set.
func SetConfigFile(path string) {
	if parsed {
		panic("iniflags: SetConfigFile() must be called before Parse()")
	}
	*config = path
}
<commit_msg>Fixing for older go versions<commit_after>package iniflags

import (
	"bufio"
	"flag"
	"fmt"
	"io"
	"log"
	"net/http"
	"net/url"
	"os"
	"os/signal"
	"path"
	"strings"
	"syscall"
	"time"
)

var (
	config               = flag.String("config", "", "Path to ini config for using in go flags. May be relative to the current executable path.")
	configUpdateInterval = flag.Duration("configUpdateInterval", 0, "Update interval for re-reading config file set via -config flag. Zero disables config file re-reading.")
	dumpflags            = flag.Bool("dumpflags", false, "Dumps values for all flags defined in the app into stdout in ini-compatible syntax and terminates the app.")
)

var (
	flagChangeCallbacks = make(map[string][]FlagChangeCallback)
	importStack         []string
	parsed              bool
)

// Generation is flags' generation number.
//
// It is modified on each flags' modification
// via either -configUpdateInterval or SIGHUP.
var Generation int

// Parse obtains flag values from config file set via -config.
//
// It obtains flag values from command line like flag.Parse(), then overrides
// them by values parsed from config file set via -config.
//
// Path to config file can also be set via SetConfigFile() before Parse() call.
func Parse() {
	if parsed {
		panic("iniflags: duplicate call to iniflags.Parse() detected")
	}

	parsed = true
	flag.Parse()
	_, ok := parseConfigFlags()
	if !ok {
		os.Exit(1)
	}

	if *dumpflags {
		dumpFlags()
		os.Exit(0)
	}

	for flagName := range flagChangeCallbacks {
		verifyFlagChangeFlagName(flagName)
	}
	Generation++
	issueAllFlagChangeCallbacks()

	ch := make(chan os.Signal)
	signal.Notify(ch, syscall.SIGHUP)
	go sighupHandler(ch)

	go configUpdater()
}

func configUpdater() {
	if *configUpdateInterval != 0 {
		for {
			// Use time.Sleep() instead of time.Tick() for the sake of dynamic flag update.
			time.Sleep(*configUpdateInterval)
			updateConfig()
		}
	}
}

func updateConfig() {
	if oldFlagValues, ok := parseConfigFlags(); ok && len(oldFlagValues) > 0 {
		modifiedFlags := make(map[string]string)
		for k := range oldFlagValues {
			modifiedFlags[k] = flag.Lookup(k).Value.String()
		}
		log.Printf("iniflags: read updated config. Modified flags are: %v\n", modifiedFlags)
		Generation++
		issueFlagChangeCallbacks(oldFlagValues)
	}
}

// FlagChangeCallback is called when the given flag is changed.
//
// The callback may be registered for any flag via OnFlagChange().
type FlagChangeCallback func()

// OnFlagChange registers the callback, which is called after the given flag
// value is initialized and/or changed.
//
// Flag values are initialized during iniflags.Parse() call.
// Flag value can be changed on config re-read after obtaining SIGHUP signal
// or if periodic config re-read is enabled with -configUpdateInterval flag.
//
// Note that flags set via command-line cannot be overriden via config file modifications.
func OnFlagChange(flagName string, callback FlagChangeCallback) {
	if parsed {
		verifyFlagChangeFlagName(flagName)
	}
	flagChangeCallbacks[flagName] = append(flagChangeCallbacks[flagName], callback)
}

func verifyFlagChangeFlagName(flagName string) {
	if flag.Lookup(flagName) == nil {
		log.Fatalf("iniflags: cannot register FlagChangeCallback for non-existing flag [%s]\n", flagName)
	}
}

func issueFlagChangeCallbacks(oldFlagValues map[string]string) {
	for flagName := range oldFlagValues {
		if fs, ok := flagChangeCallbacks[flagName]; ok {
			for _, f := range fs {
				f()
			}
		}
	}
}

func issueAllFlagChangeCallbacks() {
	for _, fs := range flagChangeCallbacks {
		for _, f := range fs {
			f()
		}
	}
}

func sighupHandler(ch <-chan os.Signal) {
	for _ = range ch {
		updateConfig()
	}
}

func parseConfigFlags() (oldFlagValues map[string]string, ok bool) {
	configPath := *config
	if !strings.HasPrefix(configPath, "./") {
		if configPath, ok = combinePath(os.Args[0], *config); !ok {
			return nil, false
		}
	}
	if configPath == "" {
		return nil, true
	}
	parsedArgs, ok := getArgsFromConfig(configPath)
	if !ok {
		return nil, false
	}
	missingFlags := getMissingFlags()

	ok = true
	oldFlagValues = make(map[string]string)
	for _, arg := range parsedArgs {
		f := flag.Lookup(arg.Key)
		if f == nil {
			log.Printf("iniflags: unknown flag name=[%s] found at line [%d] of file [%s]\n", arg.Key, arg.LineNum, arg.FilePath)
			ok = false
			continue
		}

		if _, found := missingFlags[f.Name]; found {
			oldValue := f.Value.String()
			if oldValue == arg.Value {
				continue
			}
			if err := f.Value.Set(arg.Value); err != nil {
				log.Printf("iniflags: error when parsing flag [%s] value [%s] at line [%d] of file [%s]: [%s]\n", arg.Key, arg.Value, arg.LineNum, arg.FilePath, err)
				ok = false
				continue
			}
			if oldValue != f.Value.String() {
				oldFlagValues[arg.Key] = oldValue
			}
		}
	}

	if !ok {
		// restore old flag values
		for k, v := range oldFlagValues {
			flag.Set(k, v)
		}
		oldFlagValues = nil
	}

	return oldFlagValues, ok
}

func checkImportRecursion(configPath string) bool {
	for _, path := range importStack {
		if path == configPath {
			log.Printf("iniflags: import recursion found for [%s]: %v\n", configPath, importStack)
			return false
		}
	}
	return true
}

type flagArg struct {
	Key      string
	Value    string
	FilePath string
	LineNum  int
}

func stripBOM(s string) string {
	if len(s) < 3 {
		return s
	}
	bom := s[:3]
	if bom == "\ufeff" || bom == "\ufffe" {
		return s[3:]
	}
	return s
}

func getArgsFromConfig(configPath string) (args []flagArg, ok bool) {
	if !checkImportRecursion(configPath) {
		return nil, false
	}
	importStack = append(importStack, configPath)
	defer func() {
		importStack = importStack[:len(importStack)-1]
	}()

	file := openConfigFile(configPath)
	if file == nil {
		return nil, false
	}
	defer file.Close()
	r := bufio.NewReader(file)

	var lineNum int
	for {
		lineNum++
		line, err := r.ReadString('\n')
		if err != nil && line == "" {
			if err == io.EOF {
				break
			}
			log.Printf("iniflags: error when reading file [%s] at line %d: [%s]\n", configPath, lineNum, err)
			return nil, false
		}
		if lineNum == 1 {
			line = stripBOM(line)
		}
		line = strings.TrimSpace(line)
		if strings.HasPrefix(line, "#import ") {
			importPath, ok := unquoteValue(line[7:], lineNum, configPath)
			if !ok {
				return nil, false
			}
			if importPath, ok = combinePath(configPath, importPath); !ok {
				return nil, false
			}
			importArgs, ok := getArgsFromConfig(importPath)
			if !ok {
				return nil, false
			}
			args = append(args, importArgs...)
			continue
		}
		if line == "" || line[0] == ';' || line[0] == '#' || line[0] == '[' {
			continue
		}
		parts := strings.SplitN(line, "=", 2)
		if len(parts) != 2 {
			log.Printf("iniflags: cannot split [%s] at line %d into key and value in config file [%s]\n", line, lineNum, configPath)
			return nil, false
		}
		key := strings.TrimSpace(parts[0])
		value, ok := unquoteValue(parts[1], lineNum, configPath)
		if !ok {
			return nil, false
		}
		args = append(args, flagArg{Key: key, Value: value, FilePath: configPath, LineNum: lineNum})
	}

	return args, true
}

func openConfigFile(path string) io.ReadCloser {
	if isHTTP(path) {
		resp, err := http.Get(path)
		if err != nil {
			log.Printf("iniflags: cannot load config file at [%s]: [%s]\n", path, err)
			return nil
		}
		if resp.StatusCode != http.StatusOK {
			log.Printf("iniflags: unexpected http status code when obtaining config file [%s]: %d. Expected %d\n", path, resp.StatusCode, http.StatusOK)
			return nil
		}
		return resp.Body
	}

	file, err := os.Open(path)
	if err != nil {
		log.Printf("iniflags: cannot open config file at [%s]: [%s]\n", path, err)
		return nil
	}
	return file
}

func combinePath(basePath, relPath string) (string, bool) {
	if isHTTP(basePath) {
		base, err := url.Parse(basePath)
		if err != nil {
			log.Printf("iniflags: error when parsing http base path [%s]: %s\n", basePath, err)
			return "", false
		}
		rel, err := url.Parse(relPath)
		if err != nil {
			log.Printf("iniflags: error when parsing http rel path [%s] for base [%s]: %s\n", relPath, basePath, err)
			return "", false
		}
		return base.ResolveReference(rel).String(), true
	}

	if relPath == "" || relPath[0] == '/' || isHTTP(relPath) {
		return relPath, true
	}
	return path.Join(path.Dir(basePath), relPath), true
}

func isHTTP(path string) bool {
	return strings.HasPrefix(strings.ToLower(path), "http://") || strings.HasPrefix(strings.ToLower(path), "https://")
}

func getMissingFlags() map[string]bool {
	setFlags := make(map[string]bool)
	flag.Visit(func(f *flag.Flag) {
		setFlags[f.Name] = true
	})

	missingFlags := make(map[string]bool)
	flag.VisitAll(func(f *flag.Flag) {
		if _, ok := setFlags[f.Name]; !ok {
			missingFlags[f.Name] = true
		}
	})
	return missingFlags
}

func dumpFlags() {
	flag.VisitAll(func(f *flag.Flag) {
		if f.Name != "config" && f.Name != "dumpflags" {
			fmt.Printf("%s = %s  # %s\n", f.Name, quoteValue(f.Value.String()), escapeUsage(f.Usage))
		}
	})
}

func escapeUsage(s string) string {
	return strings.Replace(s, "\n", "\n    # ", -1)
}

func quoteValue(v string) string {
	if !strings.ContainsAny(v, "\n#;") && strings.TrimSpace(v) == v {
		return v
	}
	v = strings.Replace(v, "\\", "\\\\", -1)
	v = strings.Replace(v, "\n", "\\n", -1)
	v = strings.Replace(v, "\"", "\\\"", -1)
	return fmt.Sprintf("\"%s\"", v)
}

func unquoteValue(v string, lineNum int, configPath string) (string, bool) {
	v = strings.TrimSpace(v)
	if len(v) == 0 {
		return "", true
	}
	if v[0] != '"' {
		return removeTrailingComments(v), true
	}
	n := strings.LastIndex(v, "\"")
	if n == -1 {
		log.Printf("iniflags: unclosed string found [%s] at line %d in config file [%s]\n", v, lineNum, configPath)
		return "", false
	}
	v = v[1:n]
	v = strings.Replace(v, "\\\"", "\"", -1)
	v = strings.Replace(v, "\\n", "\n", -1)
	return strings.Replace(v, "\\\\", "\\", -1), true
}

func removeTrailingComments(v string) string {
	v = strings.Split(v, "#")[0]
	v = strings.Split(v, ";")[0]
	return strings.TrimSpace(v)
}

// SetConfigFile sets path to config file.
//
// Call this function before Parse() if you need default path to config file
// when -config command-line flag is not set.
func SetConfigFile(path string) {
	if parsed {
		panic("iniflags: SetConfigFile() must be called before Parse()")
	}
	*config = path
}
<|endoftext|>