<commit_before>package bsdiff

import (
	"bytes"
	"fmt"
	"io"
	"io/ioutil"
	"math"
	"runtime"
	"time"

	humanize "github.com/dustin/go-humanize"
	"github.com/golang/protobuf/proto"
	"github.com/itchio/wharf/state"
	"github.com/jgallagher/gosaca"
)

// MaxFileSize is the largest size bsdiff will diff (for both old and new file): 2GB - 1 bytes
// a different codepath could be used for larger files, at the cost of unreasonable memory usage
// (even in 2016). If a big corporate user is willing to sponsor that part of the code, get in touch!
// Fair warning though: it won't be covered, our CI workers don't have that much RAM :)
const MaxFileSize = int64(math.MaxInt32 - 1)

// MaxMessageSize is the maximum amount of bytes that will be stored
// in a protobuf message generated by bsdiff. This enable friendlier streaming apply
// at a small storage cost
// TODO: actually use
const MaxMessageSize int64 = 16 * 1024 * 1024

type DiffStats struct {
	TimeSpentSorting  time.Duration
	TimeSpentScanning time.Duration
	BiggestAdd        int64
}

// DiffContext holds settings for the diff process, along with some
// internal storage: re-using a diff context is good to avoid GC thrashing
// (but never do it concurrently!)
type DiffContext struct {
	// SuffixSortConcurrency specifies the number of workers to use for suffix sorting.
	// Exceeding the number of cores will only slow it down. A 0 value (default) uses
	// sequential suffix sorting, which uses less RAM and has less overhead (might be faster
	// in some scenarios). A negative value means (number of cores - value).
	SuffixSortConcurrency int

	// MeasureMem enables printing memory usage statistics at various points in the
	// diffing process.
	MeasureMem bool

	// MeasureParallelOverhead prints some stats on the overhead of parallel suffix sorting
	MeasureParallelOverhead bool

	Stats *DiffStats

	db bytes.Buffer
}

// WriteMessageFunc should write a given protobuf message and relay any errors
// No reference to the given message can be kept, as its content may be modified
// after WriteMessageFunc returns. See the `wire` package for an example implementation.
type WriteMessageFunc func(msg proto.Message) (err error)

// Do computes the difference between old and new, according to the bsdiff
// algorithm, and writes the result to patch.
func (ctx *DiffContext) Do(old, new io.Reader, writeMessage WriteMessageFunc, consumer *state.Consumer) error {
	var memstats *runtime.MemStats

	if ctx.MeasureMem {
		memstats = &runtime.MemStats{}
		runtime.ReadMemStats(memstats)
		consumer.Debugf("Allocated bytes at start of bsdiff: %s (%s total)", humanize.IBytes(uint64(memstats.Alloc)), humanize.IBytes(uint64(memstats.TotalAlloc)))
	}

	obuf, err := ioutil.ReadAll(old)
	if err != nil {
		return err
	}

	obuflen := len(obuf)

	nbuf, err := ioutil.ReadAll(new)
	if err != nil {
		return err
	}
	nbuflen := len(nbuf)

	if ctx.MeasureMem {
		runtime.ReadMemStats(memstats)
		consumer.Debugf("Allocated bytes after ReadAll: %s (%s total)", humanize.IBytes(uint64(memstats.Alloc)), humanize.IBytes(uint64(memstats.TotalAlloc)))
	}

	var lenf int
	startTime := time.Now()

	I := make([]int, len(obuf)+1)
	ws := &gosaca.WorkSpace{}
	ws.ComputeSuffixArray(obuf, I[:len(obuf)])

	if ctx.Stats != nil {
		ctx.Stats.TimeSpentSorting += time.Since(startTime)
	}

	if ctx.MeasureMem {
		runtime.ReadMemStats(memstats)
		consumer.Debugf("Allocated bytes after qsufsort: %s (%s total)", humanize.IBytes(uint64(memstats.Alloc)), humanize.IBytes(uint64(memstats.TotalAlloc)))
	}

	bsdc := &Control{}

	consumer.ProgressLabel(fmt.Sprintf("Scanning %s...", humanize.IBytes(uint64(nbuflen))))

	var lastProgressUpdate int
	var updateEvery = 64 * 1024 * 1046 // 64MB

	startTime = time.Now()

	// Compute the differences, writing ctrl as we go
	var scan, pos, length int
	var lastscan, lastpos, lastoffset int
	for scan < nbuflen {
		var oldscore int
		scan += length

		if scan-lastProgressUpdate > updateEvery {
			lastProgressUpdate = scan
			progress := float64(scan) / float64(nbuflen)
			consumer.Progress(progress)
		}

		for scsc := scan; scan < nbuflen; scan++ {
			pos, length = search(I, obuf, nbuf[scan:], 0, obuflen)

			for ; scsc < scan+length; scsc++ {
				if scsc+lastoffset < obuflen &&
					obuf[scsc+lastoffset] == nbuf[scsc] {
					oldscore++
				}
			}

			if (length == oldscore && length != 0) || length > oldscore+8 {
				break
			}

			if scan+lastoffset < obuflen && obuf[scan+lastoffset] == nbuf[scan] {
				oldscore--
			}
		}

		if length != oldscore || scan == nbuflen {
			var s, Sf int
			lenf = 0
			for i := int(0); lastscan+i < scan && lastpos+i < obuflen; {
				if obuf[lastpos+i] == nbuf[lastscan+i] {
					s++
				}
				i++
				if s*2-i > Sf*2-lenf {
					Sf = s
					lenf = i
				}
			}

			lenb := 0
			if scan < nbuflen {
				var s, Sb int
				for i := int(1); (scan >= lastscan+i) && (pos >= i); i++ {
					if obuf[pos-i] == nbuf[scan-i] {
						s++
					}
					if s*2-i > Sb*2-lenb {
						Sb = s
						lenb = i
					}
				}
			}

			if lastscan+lenf > scan-lenb {
				overlap := (lastscan + lenf) - (scan - lenb)
				s := int(0)
				Ss := int(0)
				lens := int(0)
				for i := int(0); i < overlap; i++ {
					if nbuf[lastscan+lenf-overlap+i] == obuf[lastpos+lenf-overlap+i] {
						s++
					}
					if nbuf[scan-lenb+i] == obuf[pos-lenb+i] {
						s--
					}
					if s > Ss {
						Ss = s
						lens = i + 1
					}
				}

				lenf += lens - overlap
				lenb -= lens
			}

			ctx.db.Reset()
			ctx.db.Grow(int(lenf))

			for i := int(0); i < lenf; i++ {
				ctx.db.WriteByte(nbuf[lastscan+i] - obuf[lastpos+i])
			}

			bsdc.Add = ctx.db.Bytes()
			bsdc.Copy = nbuf[(lastscan + lenf):(scan - lenb)]
			bsdc.Seek = int64((pos - lenb) - (lastpos + lenf))

			err := writeMessage(bsdc)
			if err != nil {
				return err
			}

			if ctx.Stats != nil && ctx.Stats.BiggestAdd < int64(lenf) {
				ctx.Stats.BiggestAdd = int64(lenf)
			}

			lastscan = scan - lenb
			lastpos = pos - lenb
			lastoffset = pos - scan
		}
	}

	if ctx.Stats != nil {
		ctx.Stats.TimeSpentScanning += time.Since(startTime)
	}

	if ctx.MeasureMem {
		runtime.ReadMemStats(memstats)
		consumer.Debugf("Allocated bytes after scan: %s (%s total)", humanize.IBytes(uint64(memstats.Alloc)), humanize.IBytes(uint64(memstats.TotalAlloc)))
	}

	bsdc.Reset()
	bsdc.Eof = true
	err = writeMessage(bsdc)
	if err != nil {
		return err
	}

	return nil
}
<commit_msg>Re-use gosaca workspace + obuf/nbuf across runs<commit_after>package bsdiff

import (
	"bytes"
	"fmt"
	"io"
	"math"
	"runtime"
	"time"

	humanize "github.com/dustin/go-humanize"
	"github.com/golang/protobuf/proto"
	"github.com/itchio/wharf/state"
	"github.com/jgallagher/gosaca"
)

// MaxFileSize is the largest size bsdiff will diff (for both old and new file): 2GB - 1 bytes
// a different codepath could be used for larger files, at the cost of unreasonable memory usage
// (even in 2016). If a big corporate user is willing to sponsor that part of the code, get in touch!
// Fair warning though: it won't be covered, our CI workers don't have that much RAM :)
const MaxFileSize = int64(math.MaxInt32 - 1)

// MaxMessageSize is the maximum amount of bytes that will be stored
// in a protobuf message generated by bsdiff. This enable friendlier streaming apply
// at a small storage cost
// TODO: actually use
const MaxMessageSize int64 = 16 * 1024 * 1024

type DiffStats struct {
	TimeSpentSorting  time.Duration
	TimeSpentScanning time.Duration
	BiggestAdd        int64
}

// DiffContext holds settings for the diff process, along with some
// internal storage: re-using a diff context is good to avoid GC thrashing
// (but never do it concurrently!)
type DiffContext struct {
	// SuffixSortConcurrency specifies the number of workers to use for suffix sorting.
	// Exceeding the number of cores will only slow it down. A 0 value (default) uses
	// sequential suffix sorting, which uses less RAM and has less overhead (might be faster
	// in some scenarios). A negative value means (number of cores - value).
	SuffixSortConcurrency int

	// MeasureMem enables printing memory usage statistics at various points in the
	// diffing process.
	MeasureMem bool

	// MeasureParallelOverhead prints some stats on the overhead of parallel suffix sorting
	MeasureParallelOverhead bool

	Stats *DiffStats

	db bytes.Buffer
	ws *gosaca.WorkSpace

	obuf bytes.Buffer
	nbuf bytes.Buffer
}

// WriteMessageFunc should write a given protobuf message and relay any errors
// No reference to the given message can be kept, as its content may be modified
// after WriteMessageFunc returns. See the `wire` package for an example implementation.
type WriteMessageFunc func(msg proto.Message) (err error)

// Do computes the difference between old and new, according to the bsdiff
// algorithm, and writes the result to patch.
func (ctx *DiffContext) Do(old, new io.Reader, writeMessage WriteMessageFunc, consumer *state.Consumer) error {
	var memstats *runtime.MemStats
	var err error

	if ctx.MeasureMem {
		memstats = &runtime.MemStats{}
		runtime.ReadMemStats(memstats)
		consumer.Debugf("Allocated bytes at start of bsdiff: %s (%s total)", humanize.IBytes(uint64(memstats.Alloc)), humanize.IBytes(uint64(memstats.TotalAlloc)))
	}

	ctx.obuf.Reset()
	_, err = io.Copy(&ctx.obuf, old)
	if err != nil {
		return err
	}

	obuf := ctx.obuf.Bytes()
	obuflen := ctx.obuf.Len()

	ctx.nbuf.Reset()
	_, err = io.Copy(&ctx.nbuf, new)
	if err != nil {
		return err
	}

	nbuf := ctx.nbuf.Bytes()
	nbuflen := ctx.nbuf.Len()

	if ctx.MeasureMem {
		runtime.ReadMemStats(memstats)
		consumer.Debugf("Allocated bytes after ReadAll: %s (%s total)", humanize.IBytes(uint64(memstats.Alloc)), humanize.IBytes(uint64(memstats.TotalAlloc)))
	}

	var lenf int
	startTime := time.Now()

	I := make([]int, obuflen+1)
	if ctx.ws == nil {
		ctx.ws = &gosaca.WorkSpace{}
	}

	if obuflen > 0 {
		ctx.ws.ComputeSuffixArray(obuf, I[:obuflen])
	}

	if ctx.Stats != nil {
		ctx.Stats.TimeSpentSorting += time.Since(startTime)
	}

	if ctx.MeasureMem {
		runtime.ReadMemStats(memstats)
		consumer.Debugf("Allocated bytes after qsufsort: %s (%s total)", humanize.IBytes(uint64(memstats.Alloc)), humanize.IBytes(uint64(memstats.TotalAlloc)))
	}

	bsdc := &Control{}

	consumer.ProgressLabel(fmt.Sprintf("Scanning %s...", humanize.IBytes(uint64(nbuflen))))

	var lastProgressUpdate int
	var updateEvery = 64 * 1024 * 1046 // 64MB

	startTime = time.Now()

	// Compute the differences, writing ctrl as we go
	var scan, pos, length int
	var lastscan, lastpos, lastoffset int
	for scan < nbuflen {
		var oldscore int
		scan += length

		if scan-lastProgressUpdate > updateEvery {
			lastProgressUpdate = scan
			progress := float64(scan) / float64(nbuflen)
			consumer.Progress(progress)
		}

		for scsc := scan; scan < nbuflen; scan++ {
			pos, length = search(I, obuf, nbuf[scan:], 0, obuflen)

			for ; scsc < scan+length; scsc++ {
				if scsc+lastoffset < obuflen &&
					obuf[scsc+lastoffset] == nbuf[scsc] {
					oldscore++
				}
			}

			if (length == oldscore && length != 0) || length > oldscore+8 {
				break
			}

			if scan+lastoffset < obuflen && obuf[scan+lastoffset] == nbuf[scan] {
				oldscore--
			}
		}

		if length != oldscore || scan == nbuflen {
			var s, Sf int
			lenf = 0
			for i := int(0); lastscan+i < scan && lastpos+i < obuflen; {
				if obuf[lastpos+i] == nbuf[lastscan+i] {
					s++
				}
				i++
				if s*2-i > Sf*2-lenf {
					Sf = s
					lenf = i
				}
			}

			lenb := 0
			if scan < nbuflen {
				var s, Sb int
				for i := int(1); (scan >= lastscan+i) && (pos >= i); i++ {
					if obuf[pos-i] == nbuf[scan-i] {
						s++
					}
					if s*2-i > Sb*2-lenb {
						Sb = s
						lenb = i
					}
				}
			}

			if lastscan+lenf > scan-lenb {
				overlap := (lastscan + lenf) - (scan - lenb)
				s := int(0)
				Ss := int(0)
				lens := int(0)
				for i := int(0); i < overlap; i++ {
					if nbuf[lastscan+lenf-overlap+i] == obuf[lastpos+lenf-overlap+i] {
						s++
					}
					if nbuf[scan-lenb+i] == obuf[pos-lenb+i] {
						s--
					}
					if s > Ss {
						Ss = s
						lens = i + 1
					}
				}

				lenf += lens - overlap
				lenb -= lens
			}

			ctx.db.Reset()
			ctx.db.Grow(int(lenf))

			for i := int(0); i < lenf; i++ {
				ctx.db.WriteByte(nbuf[lastscan+i] - obuf[lastpos+i])
			}

			bsdc.Add = ctx.db.Bytes()
			bsdc.Copy = nbuf[(lastscan + lenf):(scan - lenb)]
			bsdc.Seek = int64((pos - lenb) - (lastpos + lenf))

			err := writeMessage(bsdc)
			if err != nil {
				return err
			}

			if ctx.Stats != nil && ctx.Stats.BiggestAdd < int64(lenf) {
				ctx.Stats.BiggestAdd = int64(lenf)
			}

			lastscan = scan - lenb
			lastpos = pos - lenb
			lastoffset = pos - scan
		}
	}

	if ctx.Stats != nil {
		ctx.Stats.TimeSpentScanning += time.Since(startTime)
	}

	if ctx.MeasureMem {
		runtime.ReadMemStats(memstats)
		consumer.Debugf("Allocated bytes after scan: %s (%s total)", humanize.IBytes(uint64(memstats.Alloc)), humanize.IBytes(uint64(memstats.TotalAlloc)))
	}

	bsdc.Reset()
	bsdc.Eof = true
	err = writeMessage(bsdc)
	if err != nil {
		return err
	}

	return nil
}
<|endoftext|>


<commit_before>package workflow

import (
	"fmt"
	"net/http"

	"github.com/go-gorp/gorp"

	"github.com/ovh/cds/engine/api/cache"
	"github.com/ovh/cds/engine/api/repositoriesmanager"
	"github.com/ovh/cds/engine/api/services"
	"github.com/ovh/cds/sdk"
	"github.com/ovh/cds/sdk/log"
)

// HookRegistration ensures hooks registration on Hook µService
func HookRegistration(db gorp.SqlExecutor, store cache.Store, oldW *sdk.Workflow, wf sdk.Workflow, p *sdk.Project) (*sdk.WorkflowNodeContextDefaultPayloadVCS, error) {
	var hookToUpdate map[string]sdk.WorkflowNodeHook
	var hookToDelete map[string]sdk.WorkflowNodeHook

	if oldW != nil {
		hookToUpdate, hookToDelete = diffHook(oldW.GetHooks(), wf.GetHooks())
	} else {
		hookToUpdate = wf.GetHooks()
	}

	var defaultPayload *sdk.WorkflowNodeContextDefaultPayloadVCS

	if len(hookToUpdate) > 0 {
		//Push the hook to hooks µService
		dao := services.Querier(db, store)
		//Load service "hooks"
		srvs, err := dao.FindByType("hooks")
		if err != nil {
			return nil, sdk.WrapError(err, "HookRegistration> Unable to get services dao")
		}

		// Update in VCS
		for i := range hookToUpdate {
			h := hookToUpdate[i]
			if oldW != nil && wf.Name != oldW.Name {
				configValue := h.Config["workflow"]
				configValue.Value = wf.Name
				h.Config["workflow"] = configValue
				hookToUpdate[i] = h
			}
		}

		//Perform the request on one off the hooks service
		if len(srvs) < 1 {
			return nil, sdk.WrapError(fmt.Errorf("HookRegistration> No hooks service available, please try again"), "Unable to get services dao")
		}

		var hooksUpdated map[string]sdk.WorkflowNodeHook
		code, errHooks := services.DoJSONRequest(srvs, http.MethodPost, "/task/bulk", hookToUpdate, &hooksUpdated)
		if errHooks != nil || code >= 400 {
			return nil, sdk.WrapError(errHooks, "HookRegistration> Unable to create hooks [%d]", code)
		}

		for i := range hooksUpdated {
			h := hooksUpdated[i]
			if h.Config["vcsServer"].Value != "" {
				if err := createVCSConfiguration(db, store, p, &h); err != nil {
					return nil, sdk.WrapError(err, "HookRegistration> Cannot update vcs configuration")
				}
				defaultPayload = &sdk.WorkflowNodeContextDefaultPayloadVCS{
					GitRepository: h.Config["repoFullName"].Value,
				}
			}
			if err := UpdateHook(db, &h); err != nil {
				return nil, sdk.WrapError(err, "HookRegistration> Cannot update hook")
			}
		}
	}

	if len(hookToDelete) > 0 {
		if err := deleteHookConfiguration(db, store, p, hookToDelete); err != nil {
			return nil, sdk.WrapError(err, "HookRegistration> Cannot remove hook configuration")
		}
	}
	return defaultPayload, nil
}

func deleteHookConfiguration(db gorp.SqlExecutor, store cache.Store, p *sdk.Project, hookToDelete map[string]sdk.WorkflowNodeHook) error {
	// Delete from vcs configuration if needed
	for _, h := range hookToDelete {
		if h.WorkflowHookModel.Name == RepositoryWebHookModel.Name {
			// Call VCS to know if repository allows webhook and get the configuration fields
			projectVCSServer := repositoriesmanager.GetProjectVCSServer(p, h.Config["vcsServer"].Value)
			if projectVCSServer != nil {
				client, errclient := repositoriesmanager.AuthorizedClient(db, store, projectVCSServer)
				if errclient != nil {
					return sdk.WrapError(errclient, "deleteHookConfiguration> Cannot get vcs client")
				}
				vcsHook := sdk.VCSHook{
					Method:   "POST",
					URL:      h.Config["webHookURL"].Value,
					Workflow: true,
					ID:       h.Config["webHookID"].Value,
				}
				if err := client.DeleteHook(h.Config["repoFullName"].Value, vcsHook); err != nil {
					return sdk.WrapError(err, "deleteHookConfiguration> Cannot delete hook on repository")
				}
				h.Config["webHookID"] = sdk.WorkflowNodeHookConfigValue{
					Value:        vcsHook.ID,
					Configurable: false,
				}
			}
		}
	}

	//Push the hook to hooks µService
	dao := services.Querier(db, store)
	//Load service "hooks"
	srvs, err := dao.FindByType("hooks")
	if err != nil {
		return sdk.WrapError(err, "HookRegistration> Unable to get services dao")
	}
	code, errHooks := services.DoJSONRequest(srvs, http.MethodDelete, "/task/bulk", hookToDelete, nil)
	if errHooks != nil || code >= 400 {
		// if we return an error, transaction will be rollbacked => hook will in database be not anymore on gitlab/bitbucket/github.
		// so, it's just a warn log
		log.Warning("HookRegistration> Unable to delete old hooks [%d]: %s", code, errHooks)
	}
	return nil
}

func createVCSConfiguration(db gorp.SqlExecutor, store cache.Store, p *sdk.Project, h *sdk.WorkflowNodeHook) error {
	// Call VCS to know if repository allows webhook and get the configuration fields
	projectVCSServer := repositoriesmanager.GetProjectVCSServer(p, h.Config["vcsServer"].Value)
	if projectVCSServer == nil {
		return nil
	}

	client, errclient := repositoriesmanager.AuthorizedClient(db, store, projectVCSServer)
	if errclient != nil {
		return sdk.WrapError(errclient, "createVCSConfiguration> Cannot get vcs client")
	}
	webHookInfo, errWH := repositoriesmanager.GetWebhooksInfos(client)
	if errWH != nil {
		return sdk.WrapError(errWH, "createVCSConfiguration> Cannot get vcs web hook info")
	}
	if !webHookInfo.WebhooksSupported || webHookInfo.WebhooksDisabled {
		return sdk.WrapError(sdk.ErrForbidden, "createVCSConfiguration> hook creation are forbidden")
	}
	vcsHook := sdk.VCSHook{
		Method:   "POST",
		URL:      h.Config["webHookURL"].Value,
		Workflow: true,
	}
	if err := client.CreateHook(h.Config["repoFullName"].Value, &vcsHook); err != nil {
		return sdk.WrapError(err, "createVCSConfiguration> Cannot create hook on repository: %+v", vcsHook)
	}
	h.Config["webHookID"] = sdk.WorkflowNodeHookConfigValue{
		Value:        vcsHook.ID,
		Configurable: false,
	}

	return nil
}

func diffHook(oldHooks map[string]sdk.WorkflowNodeHook, newHooks map[string]sdk.WorkflowNodeHook) (hookToUpdate map[string]sdk.WorkflowNodeHook, hookToDelete map[string]sdk.WorkflowNodeHook) {
	hookToUpdate = make(map[string]sdk.WorkflowNodeHook)
	hookToDelete = make(map[string]sdk.WorkflowNodeHook)

	for kNew := range newHooks {
		hold, ok := oldHooks[kNew]
		// if new hook
		if !ok {
			hookToUpdate[kNew] = newHooks[kNew]
			continue
		}

	next:
		for k, v := range newHooks[kNew].Config {
			for kold, vold := range hold.Config {
				if kold == k && v != vold {
					hookToUpdate[kNew] = newHooks[kNew]
					break next
				}
			}
		}
	}

	for kHold := range oldHooks {
		if _, ok := newHooks[kHold]; !ok {
			hookToDelete[kHold] = oldHooks[kHold]
		}
	}
	return
}
<commit_msg>fix: add debug log (#1932)<commit_after>package workflow

import (
	"fmt"
	"net/http"

	"github.com/go-gorp/gorp"

	"github.com/ovh/cds/engine/api/cache"
	"github.com/ovh/cds/engine/api/repositoriesmanager"
	"github.com/ovh/cds/engine/api/services"
	"github.com/ovh/cds/sdk"
	"github.com/ovh/cds/sdk/log"
)

// HookRegistration ensures hooks registration on Hook µService
func HookRegistration(db gorp.SqlExecutor, store cache.Store, oldW *sdk.Workflow, wf sdk.Workflow, p *sdk.Project) (*sdk.WorkflowNodeContextDefaultPayloadVCS, error) {
	var hookToUpdate map[string]sdk.WorkflowNodeHook
	var hookToDelete map[string]sdk.WorkflowNodeHook

	if oldW != nil {
		hookToUpdate, hookToDelete = diffHook(oldW.GetHooks(), wf.GetHooks())
	} else {
		hookToUpdate = wf.GetHooks()
	}

	var defaultPayload *sdk.WorkflowNodeContextDefaultPayloadVCS

	if len(hookToUpdate) > 0 {

		if oldW != nil {
			log.Info("HookRegistration> Merge: %+v, New: %+v, OLD: %+v", hookToUpdate, wf.GetHooks(), oldW.GetHooks())
		}

		//Push the hook to hooks µService
		dao := services.Querier(db, store)
		//Load service "hooks"
		srvs, err := dao.FindByType("hooks")
		if err != nil {
			return nil, sdk.WrapError(err, "HookRegistration> Unable to get services dao")
		}

		// Update in VCS
		for i := range hookToUpdate {
			h := hookToUpdate[i]
			if oldW != nil && wf.Name != oldW.Name {
				configValue := h.Config["workflow"]
				configValue.Value = wf.Name
				h.Config["workflow"] = configValue
				hookToUpdate[i] = h
			}
		}

		//Perform the request on one off the hooks service
		if len(srvs) < 1 {
			return nil, sdk.WrapError(fmt.Errorf("HookRegistration> No hooks service available, please try again"), "Unable to get services dao")
		}

		var hooksUpdated map[string]sdk.WorkflowNodeHook
		code, errHooks := services.DoJSONRequest(srvs, http.MethodPost, "/task/bulk", hookToUpdate, &hooksUpdated)
		if errHooks != nil || code >= 400 {
			return nil, sdk.WrapError(errHooks, "HookRegistration> Unable to create hooks [%d]", code)
		}

		for i := range hooksUpdated {
			h := hooksUpdated[i]
			if h.Config["vcsServer"].Value != "" {
				if err := createVCSConfiguration(db, store, p, &h); err != nil {
					return nil, sdk.WrapError(err, "HookRegistration> Cannot update vcs configuration")
				}
				defaultPayload = &sdk.WorkflowNodeContextDefaultPayloadVCS{
					GitRepository: h.Config["repoFullName"].Value,
				}
			}
			if err := UpdateHook(db, &h); err != nil {
				return nil, sdk.WrapError(err, "HookRegistration> Cannot update hook")
			}
		}
	}

	if len(hookToDelete) > 0 {
		if err := deleteHookConfiguration(db, store, p, hookToDelete); err != nil {
			return nil, sdk.WrapError(err, "HookRegistration> Cannot remove hook configuration")
		}
	}
	return defaultPayload, nil
}

func deleteHookConfiguration(db gorp.SqlExecutor, store cache.Store, p *sdk.Project, hookToDelete map[string]sdk.WorkflowNodeHook) error {
	// Delete from vcs configuration if needed
	for _, h := range hookToDelete {
		if h.WorkflowHookModel.Name == RepositoryWebHookModel.Name {
			// Call VCS to know if repository allows webhook and get the configuration fields
			projectVCSServer := repositoriesmanager.GetProjectVCSServer(p, h.Config["vcsServer"].Value)
			if projectVCSServer != nil {
				client, errclient := repositoriesmanager.AuthorizedClient(db, store, projectVCSServer)
				if errclient != nil {
					return sdk.WrapError(errclient, "deleteHookConfiguration> Cannot get vcs client")
				}
				vcsHook := sdk.VCSHook{
					Method:   "POST",
					URL:      h.Config["webHookURL"].Value,
					Workflow: true,
					ID:       h.Config["webHookID"].Value,
				}
				if err := client.DeleteHook(h.Config["repoFullName"].Value, vcsHook); err != nil {
					return sdk.WrapError(err, "deleteHookConfiguration> Cannot delete hook on repository")
				}
				h.Config["webHookID"] = sdk.WorkflowNodeHookConfigValue{
					Value:        vcsHook.ID,
					Configurable: false,
				}
			}
		}
	}

	//Push the hook to hooks µService
	dao := services.Querier(db, store)
	//Load service "hooks"
	srvs, err := dao.FindByType("hooks")
	if err != nil {
		return sdk.WrapError(err, "HookRegistration> Unable to get services dao")
	}
	code, errHooks := services.DoJSONRequest(srvs, http.MethodDelete, "/task/bulk", hookToDelete, nil)
	if errHooks != nil || code >= 400 {
		// if we return an error, transaction will be rollbacked => hook will in database be not anymore on gitlab/bitbucket/github.
		// so, it's just a warn log
		log.Warning("HookRegistration> Unable to delete old hooks [%d]: %s", code, errHooks)
	}
	return nil
}

func createVCSConfiguration(db gorp.SqlExecutor, store cache.Store, p *sdk.Project, h *sdk.WorkflowNodeHook) error {
	// Call VCS to know if repository allows webhook and get the configuration fields
	projectVCSServer := repositoriesmanager.GetProjectVCSServer(p, h.Config["vcsServer"].Value)
	if projectVCSServer == nil {
		return nil
	}

	client, errclient := repositoriesmanager.AuthorizedClient(db, store, projectVCSServer)
	if errclient != nil {
		return sdk.WrapError(errclient, "createVCSConfiguration> Cannot get vcs client")
	}
	webHookInfo, errWH := repositoriesmanager.GetWebhooksInfos(client)
	if errWH != nil {
		return sdk.WrapError(errWH, "createVCSConfiguration> Cannot get vcs web hook info")
	}
	if !webHookInfo.WebhooksSupported || webHookInfo.WebhooksDisabled {
		return sdk.WrapError(sdk.ErrForbidden, "createVCSConfiguration> hook creation are forbidden")
	}
	vcsHook := sdk.VCSHook{
		Method:   "POST",
		URL:      h.Config["webHookURL"].Value,
		Workflow: true,
	}
	if err := client.CreateHook(h.Config["repoFullName"].Value, &vcsHook); err != nil {
		return sdk.WrapError(err, "createVCSConfiguration> Cannot create hook on repository: %+v", vcsHook)
	}
	h.Config["webHookID"] = sdk.WorkflowNodeHookConfigValue{
		Value:        vcsHook.ID,
		Configurable: false,
	}

	return nil
}

func diffHook(oldHooks map[string]sdk.WorkflowNodeHook, newHooks map[string]sdk.WorkflowNodeHook) (hookToUpdate map[string]sdk.WorkflowNodeHook, hookToDelete map[string]sdk.WorkflowNodeHook) {
	hookToUpdate = make(map[string]sdk.WorkflowNodeHook)
	hookToDelete = make(map[string]sdk.WorkflowNodeHook)

	for kNew := range newHooks {
		hold, ok := oldHooks[kNew]
		// if new hook
		if !ok {
			hookToUpdate[kNew] = newHooks[kNew]
			continue
		}

	next:
		for k, v := range newHooks[kNew].Config {
			for kold, vold := range hold.Config {
				if kold == k && v != vold {
					hookToUpdate[kNew] = newHooks[kNew]
					break next
				}
			}
		}
	}

	for kHold := range oldHooks {
		if _, ok := newHooks[kHold]; !ok {
			hookToDelete[kHold] = oldHooks[kHold]
		}
	}
	return
}
<|endoftext|>


<commit_before>/*
   Copyright 2014 CoreOS, Inc.

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
*/

package etcdserver

import (
	"bytes"
	"encoding/json"
	"fmt"
	"log"
	"net/http"
	"strconv"
	"time"

	etcdErr "github.com/coreos/etcd/error"

	"github.com/coreos/etcd/etcdserver/stats"
	"github.com/coreos/etcd/raft/raftpb"
	"github.com/coreos/etcd/store"
)

const (
	raftPrefix = "/raft"

	raftAttributesSuffix = "/raftAttributes"
	attributesSuffix     = "/attributes"
)

type ClusterStore interface {
	Add(m Member)
	Get() Cluster
	Remove(id uint64)
}

type clusterStore struct {
	Store store.Store
	// TODO: write the id into the actual store?
	// TODO: save the id as string?
	id uint64
}

// Add puts a new Member into the store.
// A Member with a matching id must not exist.
func (s *clusterStore) Add(m Member) {
	b, err := json.Marshal(m.RaftAttributes)
	if err != nil {
		log.Panicf("marshal error: %v", err)
	}
	if _, err := s.Store.Create(m.storeKey()+raftAttributesSuffix, false, string(b), false, store.Permanent); err != nil {
		log.Panicf("add raftAttributes should never fail: %v", err)
	}

	b, err = json.Marshal(m.Attributes)
	if err != nil {
		log.Panicf("marshal error: %v", err)
	}
	if _, err := s.Store.Create(m.storeKey()+attributesSuffix, false, string(b), false, store.Permanent); err != nil {
		log.Panicf("add attributes should never fail: %v", err)
	}
}

// TODO(philips): keep the latest copy without going to the store to avoid the
// lock here.
func (s *clusterStore) Get() Cluster {
	c := NewCluster()
	c.id = s.id
	e, err := s.Store.Get(membersKVPrefix, true, true)
	if err != nil {
		if v, ok := err.(*etcdErr.Error); ok && v.ErrorCode == etcdErr.EcodeKeyNotFound {
			return *c
		}
		log.Panicf("get member should never fail: %v", err)
	}
	for _, n := range e.Node.Nodes {
		m, err := nodeToMember(n)
		if err != nil {
			log.Panicf("unexpected nodeToMember error: %v", err)
		}
		if err := c.Add(m); err != nil {
			log.Panicf("add member to cluster should never fail: %v", err)
		}
	}
	return *c
}

// nodeToMember builds member through a store node.
// the child nodes of the given node should be sorted by key.
func nodeToMember(n *store.NodeExtern) (Member, error) {
	m := Member{ID: parseMemberID(n.Key)}
	if len(n.Nodes) != 2 {
		return m, fmt.Errorf("len(nodes) = %d, want 2", len(n.Nodes))
	}
	if w := n.Key + attributesSuffix; n.Nodes[0].Key != w {
		return m, fmt.Errorf("key = %v, want %v", n.Nodes[0].Key, w)
	}
	if err := json.Unmarshal([]byte(*n.Nodes[0].Value), &m.Attributes); err != nil {
		return m, fmt.Errorf("unmarshal attributes error: %v", err)
	}
	if w := n.Key + raftAttributesSuffix; n.Nodes[1].Key != w {
		return m, fmt.Errorf("key = %v, want %v", n.Nodes[1].Key, w)
	}
	if err := json.Unmarshal([]byte(*n.Nodes[1].Value), &m.RaftAttributes); err != nil {
		return m, fmt.Errorf("unmarshal raftAttributes error: %v", err)
	}
	return m, nil
}

// Remove removes a member from the store.
// The given id MUST exist.
func (s *clusterStore) Remove(id uint64) {
	p := s.Get().FindID(id).storeKey()
	if _, err := s.Store.Delete(p, true, true); err != nil {
		log.Panicf("delete peer should never fail: %v", err)
	}
}

// Sender creates the default production sender used to transport raft messages
// in the cluster. The returned sender will update the given ServerStats and
// LeaderStats appropriately.
func Sender(t *http.Transport, cls ClusterStore, ss *stats.ServerStats, ls *stats.LeaderStats) func(msgs []raftpb.Message) {
	c := &http.Client{Transport: t}

	return func(msgs []raftpb.Message) {
		for _, m := range msgs {
			// TODO: reuse go routines
			// limit the number of outgoing connections for the same receiver
			go send(c, cls, m, ss, ls)
		}
	}
}

// send uses the given client to send a message to a member in the given
// ClusterStore, retrying up to 3 times for each message. The given
// ServerStats and LeaderStats are updated appropriately
func send(c *http.Client, cls ClusterStore, m raftpb.Message, ss *stats.ServerStats, ls *stats.LeaderStats) {
	cid := cls.Get().ID()
	// TODO (xiangli): reasonable retry logic
	for i := 0; i < 3; i++ {
		u := cls.Get().Pick(m.To)
		if u == "" {
			// TODO: unknown peer id.. what do we do? I
			// don't think his should ever happen, need to
			// look into this further.
			log.Printf("etcdhttp: no addr for %d", m.To)
			return
		}
		u = fmt.Sprintf("%s%s", u, raftPrefix)

		// TODO: don't block. we should be able to have 1000s
		// of messages out at a time.
		data, err := m.Marshal()
		if err != nil {
			log.Println("etcdhttp: dropping message:", err)
			return // drop bad message
		}
		if m.Type == raftpb.MsgApp {
			ss.SendAppendReq(len(data))
		}
		to := idAsHex(m.To)
		fs := ls.Follower(to)

		start := time.Now()
		sent := httpPost(c, u, cid, data)
		end := time.Now()
		if sent {
			fs.Succ(end.Sub(start))
			return
		}
		fs.Fail()
		// TODO: backoff
	}
}

// httpPost POSTs a data payload to a url using the given client. Returns true
// if the POST succeeds, false on any failure.
func httpPost(c *http.Client, url string, cid uint64, data []byte) bool {
	req, err := http.NewRequest("POST", url, bytes.NewBuffer(data))
	if err != nil {
		// TODO: log the error?
		return false
	}
	req.Header.Set("Content-Type", "application/protobuf")
	req.Header.Set("X-Etcd-Cluster-ID", strconv.FormatUint(cid, 16))
	resp, err := c.Do(req)
	if err != nil {
		// TODO: log the error?
		return false
	}

	resp.Body.Close()
	if resp.StatusCode != http.StatusNoContent {
		// TODO: log the error?
		return false
	}
	return true
}
<commit_msg>etcdserver: panic on clusterid mismatch<commit_after>/*
   Copyright 2014 CoreOS, Inc.

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
*/

package etcdserver

import (
	"bytes"
	"encoding/json"
	"fmt"
	"log"
	"net/http"
	"strconv"
	"time"

	etcdErr "github.com/coreos/etcd/error"

	"github.com/coreos/etcd/etcdserver/stats"
	"github.com/coreos/etcd/raft/raftpb"
	"github.com/coreos/etcd/store"
)

const (
	raftPrefix = "/raft"

	raftAttributesSuffix = "/raftAttributes"
	attributesSuffix     = "/attributes"
)

type ClusterStore interface {
	Add(m Member)
	Get() Cluster
	Remove(id uint64)
}

type clusterStore struct {
	Store store.Store
	// TODO: write the id into the actual store?
	// TODO: save the id as string?
	id uint64
}

// Add puts a new Member into the store.
// A Member with a matching id must not exist.
func (s *clusterStore) Add(m Member) {
	b, err := json.Marshal(m.RaftAttributes)
	if err != nil {
		log.Panicf("marshal error: %v", err)
	}
	if _, err := s.Store.Create(m.storeKey()+raftAttributesSuffix, false, string(b), false, store.Permanent); err != nil {
		log.Panicf("add raftAttributes should never fail: %v", err)
	}

	b, err = json.Marshal(m.Attributes)
	if err != nil {
		log.Panicf("marshal error: %v", err)
	}
	if _, err := s.Store.Create(m.storeKey()+attributesSuffix, false, string(b), false, store.Permanent); err != nil {
		log.Panicf("add attributes should never fail: %v", err)
	}
}

// TODO(philips): keep the latest copy without going to the store to avoid the
// lock here.
func (s *clusterStore) Get() Cluster {
	c := NewCluster()
	c.id = s.id
	e, err := s.Store.Get(membersKVPrefix, true, true)
	if err != nil {
		if v, ok := err.(*etcdErr.Error); ok && v.ErrorCode == etcdErr.EcodeKeyNotFound {
			return *c
		}
		log.Panicf("get member should never fail: %v", err)
	}
	for _, n := range e.Node.Nodes {
		m, err := nodeToMember(n)
		if err != nil {
			log.Panicf("unexpected nodeToMember error: %v", err)
		}
		if err := c.Add(m); err != nil {
			log.Panicf("add member to cluster should never fail: %v", err)
		}
	}
	return *c
}

// nodeToMember builds member through a store node.
// the child nodes of the given node should be sorted by key.
func nodeToMember(n *store.NodeExtern) (Member, error) {
	m := Member{ID: parseMemberID(n.Key)}
	if len(n.Nodes) != 2 {
		return m, fmt.Errorf("len(nodes) = %d, want 2", len(n.Nodes))
	}
	if w := n.Key + attributesSuffix; n.Nodes[0].Key != w {
		return m, fmt.Errorf("key = %v, want %v", n.Nodes[0].Key, w)
	}
	if err := json.Unmarshal([]byte(*n.Nodes[0].Value), &m.Attributes); err != nil {
		return m, fmt.Errorf("unmarshal attributes error: %v", err)
	}
	if w := n.Key + raftAttributesSuffix; n.Nodes[1].Key != w {
		return m, fmt.Errorf("key = %v, want %v", n.Nodes[1].Key, w)
	}
	if err := json.Unmarshal([]byte(*n.Nodes[1].Value), &m.RaftAttributes); err != nil {
		return m, fmt.Errorf("unmarshal raftAttributes error: %v", err)
	}
	return m, nil
}

// Remove removes a member from the store.
// The given id MUST exist.
func (s *clusterStore) Remove(id uint64) {
	p := s.Get().FindID(id).storeKey()
	if _, err := s.Store.Delete(p, true, true); err != nil {
		log.Panicf("delete peer should never fail: %v", err)
	}
}

// Sender creates the default production sender used to transport raft messages
// in the cluster. The returned sender will update the given ServerStats and
// LeaderStats appropriately.
func Sender(t *http.Transport, cls ClusterStore, ss *stats.ServerStats, ls *stats.LeaderStats) func(msgs []raftpb.Message) {
	c := &http.Client{Transport: t}

	return func(msgs []raftpb.Message) {
		for _, m := range msgs {
			// TODO: reuse go routines
			// limit the number of outgoing connections for the same receiver
			go send(c, cls, m, ss, ls)
		}
	}
}

// send uses the given client to send a message to a member in the given
// ClusterStore, retrying up to 3 times for each message. The given
// ServerStats and LeaderStats are updated appropriately
func send(c *http.Client, cls ClusterStore, m raftpb.Message, ss *stats.ServerStats, ls *stats.LeaderStats) {
	cid := cls.Get().ID()
	// TODO (xiangli): reasonable retry logic
	for i := 0; i < 3; i++ {
		u := cls.Get().Pick(m.To)
		if u == "" {
			// TODO: unknown peer id.. what do we do? I
			// don't think his should ever happen, need to
			// look into this further.
			log.Printf("etcdhttp: no addr for %d", m.To)
			return
		}
		u = fmt.Sprintf("%s%s", u, raftPrefix)

		// TODO: don't block. we should be able to have 1000s
		// of messages out at a time.
		data, err := m.Marshal()
		if err != nil {
			log.Println("etcdhttp: dropping message:", err)
			return // drop bad message
		}
		if m.Type == raftpb.MsgApp {
			ss.SendAppendReq(len(data))
		}
		to := idAsHex(m.To)
		fs := ls.Follower(to)

		start := time.Now()
		sent := httpPost(c, u, cid, data)
		end := time.Now()
		if sent {
			fs.Succ(end.Sub(start))
			return
		}
		fs.Fail()
		// TODO: backoff
	}
}

// httpPost POSTs a data payload to a url using the given client. Returns true
// if the POST succeeds, false on any failure.
func httpPost(c *http.Client, url string, cid uint64, data []byte) bool {
	req, err := http.NewRequest("POST", url, bytes.NewBuffer(data))
	if err != nil {
		// TODO: log the error?
		return false
	}
	req.Header.Set("Content-Type", "application/protobuf")
	req.Header.Set("X-Etcd-Cluster-ID", strconv.FormatUint(cid, 16))
	resp, err := c.Do(req)
	if err != nil {
		// TODO: log the error?
		return false
	}
	resp.Body.Close()

	switch resp.StatusCode {
	case http.StatusPreconditionFailed:
		// TODO: shutdown the etcdserver gracefully?
		log.Panicf("clusterID mismatch")
		return false
	case http.StatusNoContent:
		return true
	default:
		return false
	}
}
<|endoftext|>


<commit_before>// Copyright 2016 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package prometheus

import "time"

// Observer is the interface that wraps the Observe method, used by Histogram
// and Summary to add observations.
type Observer interface {
	Observe(float64)
}

type Timer struct {
	begin    time.Time
	observer Observer
	gauge    Gauge
}

func StartTimer() *Timer {
	return &Timer{begin: time.Now()}
}

func (t *Timer) With(o Observer) *Timer {
	t.observer = o
	return t
}

func (t *Timer) WithGauge(g Gauge) *Timer {
	t.gauge = g
	return t
}

func (t *Timer) Stop() {
	if t.observer != nil {
		t.observer.Observe(time.Since(t.begin).Seconds())
	}
	if t.gauge != nil {
		t.gauge.Set(time.Since(t.begin).Seconds())
	}
}
<commit_msg>Add observerFunc to observe with a Gauge<commit_after>// Copyright 2016 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package prometheus

import "time"

// Observer is the interface that wraps the Observe method, used by Histogram
// and Summary to add observations.
type Observer interface {
	Observe(float64)
}

type observerFunc func(float64)

func (o observerFunc) Observe(value float64) {
	o(value)
}

type Timer struct {
	begin    time.Time
	observer Observer
}

func StartTimer() *Timer {
	return &Timer{begin: time.Now()}
}

func (t *Timer) With(o Observer) *Timer {
	t.observer = o
	return t
}

func (t *Timer) WithGauge(g Gauge) *Timer {
	t.observer = observerFunc(g.Set)
	return t
}

func (t *Timer) Stop() {
	if t.observer != nil {
		t.observer.Observe(time.Since(t.begin).Seconds())
	}
}
<|endoftext|>


<commit_before>package provider

import (
	"io/ioutil"
	"net"
	"net/http"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"time"

	fsnotify "gopkg.in/fsnotify.v1"

	"github.com/BurntSushi/toml"
	"github.com/Nitro/sidecar/catalog"
	"github.com/Nitro/sidecar/service"
	"github.com/containous/traefik/log"
	"github.com/containous/traefik/safe"
	"github.com/containous/traefik/types"
)

const (
	method         = "drr"
	sticky         = false
	circuitBreaker = "ResponseCodeRatio(500, 600, 0, 600) > 0.3"
	weight         = 0
)

var _ Provider = (*Sidecar)(nil)

// Sidecar holds configurations of the Sidecar provider
type Sidecar struct {
	BaseProvider      `mapstructure:",squash"`
	Endpoint          string `description:"Sidecar URL"`
	Frontend          string `description:"Configuration file for frontend"`
	configurationChan chan<- types.ConfigMessage
	RefreshConn       time.Duration `description:"How often to refresh the connection to Sidecar backend"`
	connTimer         *time.Timer
}

type callback func(map[string][]*service.Service, error)

// Provide allows the provider to provide configurations to traefik
// using the given configuration channel.
func (provider *Sidecar) Provide(configurationChan chan<- types.ConfigMessage, pool *safe.Pool, constraints types.Constraints) error {
	provider.configurationChan = configurationChan
	if provider.Watch {
		safe.Go(func() { provider.sidecarWatcher() })

		watcher, err := fsnotify.NewWatcher()
		if err != nil {
			log.Errorln("Error creating file watcher", err)
			return err
		}

		file, err := os.Open(provider.Frontend)
		if err != nil {
			log.Errorln("Error opening file", err)
			return err
		}
		defer file.Close()
		pool.Go(func(stop chan bool) {
			defer watcher.Close()
			for {
				select {
				case <-stop:
					return
				case event := <-watcher.Events:
					if strings.Contains(event.Name, file.Name()) {
						log.Debug("Sidecar Frontend File event:", event)
						states, errState := provider.fetchState()
						if errState != nil {
							log.Errorln("Error reloading Sidecar config", errState)
						}
						provider.loadSidecarConfig(states.ByService())
					}
				case errWatcher := <-watcher.Errors:
					log.Errorln("Watcher event error", errWatcher)
				}
			}
		})
		err = watcher.Add(filepath.Dir(file.Name()))
		if err != nil {
			log.Error("Error adding file watcher", err)
			return err
		}
	}
	states, err := provider.fetchState()
	if err != nil {
		log.Fatalln("Error reloading Sidecar config", err)
	}
	err = provider.loadSidecarConfig(states.ByService())
	if err != nil {
		return err
	}
	return nil
}

func (provider *Sidecar) constructConfig(sidecarStates map[string][]*service.Service) (*types.Configuration, error) {
	sidecarConfig := types.Configuration{}
	log.Infoln("loading sidecar config")
	sidecarConfig.Backends = provider.makeBackends(sidecarStates)
	log.Infoln("loading frontend config from file: ", provider.Frontend)
	var err error
	sidecarConfig.Frontends, err = provider.makeFrontend()
	if err != nil {
		return nil, err
	}
	return &sidecarConfig, nil
}

func (provider *Sidecar) loadSidecarConfig(sidecarStates map[string][]*service.Service) error {
	conf, err := provider.constructConfig(sidecarStates)
	if err != nil {
		return err
	}
	provider.configurationChan <- types.ConfigMessage{
		ProviderName:  "sidecar",
		Configuration: conf,
	}
	return nil
}

func (provider *Sidecar) sidecarWatcher() error {
	//set timeout to be just a bot more than connection refresh interval
	provider.connTimer = time.NewTimer(provider.RefreshConn * time.Second)
	tr := &http.Transport{ResponseHeaderTimeout: 0}
	client := &http.Client{
		Timeout:   0,
		Transport: tr}
	log.Infof("Using %s Sidecar connection refresh interval", provider.RefreshConn)
	provider.recycleConn(client, tr)
	return nil
}

func (provider *Sidecar) recycleConn(client *http.Client, tr *http.Transport) {
	var err error
	var resp *http.Response
	var req *http.Request
	for { //use refresh interval to occasionally reconnect to Sidecar in case the stream connection is lost
		req, err = http.NewRequest("GET", provider.Endpoint+"/watch", nil)
		if err != nil {
			log.Errorf("Error creating http request to Sidecar: %s, Error: %s", provider.Endpoint, err)
			continue
		}
		resp, err = client.Do(req)
		if err != nil {
			log.Errorf("Error connecting to Sidecar: %s, Error: %s", provider.Endpoint, err)
			time.Sleep(5 * time.Second)
			continue
		}
		go catalog.DecodeStream(resp.Body, provider.callbackLoader)

		//wait on refresh connection timer.  If this expires we haven't seen an update in a
		//while and should cancel the request, reset the time, and reconnect just in case
		<-provider.connTimer.C
		provider.connTimer.Reset(provider.RefreshConn * time.Second)
		tr.CancelRequest(req)
	}
}

func (provider *Sidecar) callbackLoader(sidecarStates map[string][]*service.Service, err error) error {
	//load config regardless
	provider.loadSidecarConfig(sidecarStates)

	if err != nil {
		return err
	}
	//else reset connection timer
	if !provider.connTimer.Stop() {
		<-provider.connTimer.C
	}
	provider.connTimer.Reset(provider.RefreshConn * time.Second)
	return nil
}

func (provider *Sidecar) makeFrontend() (map[string]*types.Frontend, error) {
	configuration := new(types.Configuration)
	if _, err := toml.DecodeFile(provider.Frontend, configuration); err != nil {
		log.Errorf("Error reading file: %s", err)
		return nil, err
	}
	return configuration.Frontends, nil
}

func (provider *Sidecar) makeBackends(sidecarStates map[string][]*service.Service) map[string]*types.Backend {
	sidecarBacks := make(map[string]*types.Backend)
	for serviceName, services := range sidecarStates {
		newServers := make(map[string]types.Server)
		newBackend := &types.Backend{LoadBalancer: &types.LoadBalancer{Method: method, Sticky: sticky},
			CircuitBreaker: &types.CircuitBreaker{Expression: circuitBreaker},
			Servers:        newServers}
		for _, serv := range services {
			if serv.IsAlive() {
				for i := 0; i < len(serv.Ports); i++ {
					ipAddr, err := net.LookupIP(serv.Hostname)
					if err != nil {
						log.Errorln("Error resolving Ip address, ", err)
						newBackend.Servers[serv.Hostname] = types.Server{URL: "http://" + serv.Hostname + ":" + strconv.FormatInt(serv.Ports[i].Port, 10), Weight: weight}
					} else {
						newBackend.Servers[serv.Hostname] = types.Server{URL: "http://" + ipAddr[0].String() + ":" + strconv.FormatInt(serv.Ports[i].Port, 10), Weight: weight}
					}
				}
			}
		}
		sidecarBacks[serviceName] = newBackend
	}
	return sidecarBacks
}

func (provider *Sidecar) fetchState() (*catalog.ServicesState, error) {
	client := &http.Client{Timeout: 5 * time.Second}
	resp, err := client.Get(provider.Endpoint + "/state.json")
	if err != nil {
		return nil, err
	}

	bytes, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}

	state, err := catalog.Decode(bytes)
	if err != nil {
		return nil, err
	}
	return state, nil
}
<commit_msg>follow safe goroutine<commit_after>package provider

import (
	"io/ioutil"
	"net"
	"net/http"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"time"

	fsnotify "gopkg.in/fsnotify.v1"

	"github.com/BurntSushi/toml"
	"github.com/Nitro/sidecar/catalog"
	"github.com/Nitro/sidecar/service"
	"github.com/containous/traefik/log"
	"github.com/containous/traefik/safe"
	"github.com/containous/traefik/types"
)

const (
	method         = "drr"
	sticky         = false
	circuitBreaker = "ResponseCodeRatio(500, 600, 0, 600) > 0.3"
)

var _ Provider = (*Sidecar)(nil)

// Sidecar holds configurations of the Sidecar provider
type Sidecar struct {
	BaseProvider      `mapstructure:",squash"`
	Endpoint          string `description:"Sidecar URL"`
	Frontend          string `description:"Configuration file for frontend"`
	configurationChan chan<- types.ConfigMessage
	RefreshConn       time.Duration `description:"How often to refresh the connection to Sidecar backend"`
	connTimer         *time.Timer
}

type callback func(map[string][]*service.Service, error)

// Provide allows the provider to provide configurations to traefik
// using the given configuration channel.
func (provider *Sidecar) Provide(configurationChan chan<- types.ConfigMessage, pool *safe.Pool, constraints types.Constraints) error {
	provider.configurationChan = configurationChan
	if provider.Watch {
		safe.Go(func() { provider.sidecarWatcher() })

		watcher, err := fsnotify.NewWatcher()
		if err != nil {
			log.Errorln("Error creating file watcher", err)
			return err
		}

		file, err := os.Open(provider.Frontend)
		if err != nil {
			log.Errorln("Error opening file", err)
			return err
		}
		defer file.Close()
		pool.Go(func(stop chan bool) {
			defer watcher.Close()
			for {
				select {
				case <-stop:
					return
				case event := <-watcher.Events:
					if strings.Contains(event.Name, file.Name()) {
						log.Debug("Sidecar Frontend File event:", event)
						states, errState := provider.fetchState()
						if errState != nil {
							log.Errorln("Error reloading Sidecar config", errState)
						}
						provider.loadSidecarConfig(states.ByService())
					}
				case errWatcher := <-watcher.Errors:
					log.Errorln("Watcher event error", errWatcher)
				}
			}
		})
		err = watcher.Add(filepath.Dir(file.Name()))
		if err != nil {
			log.Error("Error adding file watcher", err)
			return err
		}
	}
	states, err := provider.fetchState()
	if err != nil {
		log.Fatalln("Error reloading Sidecar config", err)
	}
	err = provider.loadSidecarConfig(states.ByService())
	if err != nil {
		return err
	}
	return nil
}

func (provider *Sidecar) constructConfig(sidecarStates map[string][]*service.Service) (*types.Configuration, error) {
	sidecarConfig := types.Configuration{}
	log.Infoln("loading sidecar config")
	sidecarConfig.Backends = provider.makeBackends(sidecarStates)
	log.Infoln("loading frontend config from file: ", provider.Frontend)
	var err error
	sidecarConfig.Frontends, err = provider.makeFrontend()
	if err != nil {
		return nil, err
	}
	return &sidecarConfig, nil
}

func (provider *Sidecar) loadSidecarConfig(sidecarStates map[string][]*service.Service) error {
	conf, err := provider.constructConfig(sidecarStates)
	if err != nil {
		return err
	}
	provider.configurationChan <- types.ConfigMessage{
		ProviderName:  "sidecar",
		Configuration: conf,
	}
	return nil
}

func (provider *Sidecar) sidecarWatcher() error {
	//set timeout to be just a bot more than connection refresh interval
	provider.connTimer = time.NewTimer(provider.RefreshConn * time.Second)
	tr := &http.Transport{ResponseHeaderTimeout: 0}
	client := &http.Client{
		Timeout:   0,
		Transport: tr}
	log.Infof("Using %s Sidecar connection refresh interval", provider.RefreshConn)
	provider.recycleConn(client, tr)
	return nil
}

func (provider *Sidecar) recycleConn(client *http.Client, tr *http.Transport) {
	var err error
	var resp *http.Response
	var req *http.Request
	for { //use refresh interval to occasionally reconnect to Sidecar in case the stream connection is lost
		req, err = http.NewRequest("GET", provider.Endpoint+"/watch", nil)
		if err != nil {
			log.Errorf("Error creating http request to Sidecar: %s, Error: %s", provider.Endpoint, err)
			continue
		}
		resp, err = client.Do(req)
		if err != nil {
			log.Errorf("Error connecting to Sidecar: %s, Error: %s", provider.Endpoint, err)
			time.Sleep(5 * time.Second)
			continue
		}
		safe.Go(func() { catalog.DecodeStream(resp.Body, provider.callbackLoader) })

		//wait on refresh connection timer.  If this expires we haven't seen an update in a
		//while and should cancel the request, reset the time, and reconnect just in case
		<-provider.connTimer.C
		provider.connTimer.Reset(provider.RefreshConn * time.Second)
		tr.CancelRequest(req)
	}
}

func (provider *Sidecar) callbackLoader(sidecarStates map[string][]*service.Service, err error) error {
	//load config regardless
	provider.loadSidecarConfig(sidecarStates)

	if err != nil {
		return err
	}
	//else reset connection timer
	if !provider.connTimer.Stop() {
		<-provider.connTimer.C
	}
	provider.connTimer.Reset(provider.RefreshConn * time.Second)
	return nil
}

func (provider *Sidecar) makeFrontend() (map[string]*types.Frontend, error) {
	configuration := new(types.Configuration)
	if _, err := toml.DecodeFile(provider.Frontend, configuration); err != nil {
		log.Errorf("Error reading file: %s", err)
		return nil, err
	}
	return configuration.Frontends, nil
}

func (provider *Sidecar) makeBackends(sidecarStates map[string][]*service.Service) map[string]*types.Backend {
	sidecarBacks := make(map[string]*types.Backend)
	for serviceName, services := range sidecarStates {
		newServers := make(map[string]types.Server)
		newBackend := &types.Backend{LoadBalancer: &types.LoadBalancer{Method: method, Sticky: sticky},
			CircuitBreaker: &types.CircuitBreaker{Expression: circuitBreaker},
			Servers:        newServers}
		for _, serv := range services {
			if serv.IsAlive() {
				for i := 0; i < len(serv.Ports); i++ {
					ipAddr, err := net.LookupIP(serv.Hostname)
					if err != nil {
						log.Errorln("Error resolving Ip address, ", err)
						newBackend.Servers[serv.Hostname] = types.Server{URL: "http://" + serv.Hostname + ":" + strconv.FormatInt(serv.Ports[i].Port, 10)}
					} else {
						newBackend.Servers[serv.Hostname] = types.Server{URL: "http://" + ipAddr[0].String() + ":" + strconv.FormatInt(serv.Ports[i].Port, 10)}
					}
				}
			}
		}
		sidecarBacks[serviceName] = newBackend
	}
	return sidecarBacks
}

func (provider *Sidecar) fetchState() (*catalog.ServicesState, error) {
	client := &http.Client{Timeout: 5 * time.Second}
	resp, err := client.Get(provider.Endpoint + "/state.json")
	if err != nil {
		return nil, err
	}

	bytes, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}

	state, err := catalog.Decode(bytes)
	if err != nil {
		return nil, err
	}
	return state, nil
}
<|endoftext|>


<commit_before>// Tests of call chaining f(g()) when g has multiple return values (MRVs).
// See https://code.google.com/p/go/issues/detail?id=4573.

package main

func assert(actual, expected int) {
	if actual != expected {
		panic(actual)
	}
}

func g() (int, int) {
	return 5, 7
}

func g2() (float64, float64) {
	return 5, 7
}

func f1v(x int, v ...int) {
	assert(x, 5)
	assert(v[0], 7)
}

func f2(x, y int) {
	assert(x, 5)
	assert(y, 7)
}

func f2v(x, y int, v ...int) {
	assert(x, 5)
	assert(y, 7)
	assert(len(v), 0)
}

func complexArgs() (float64, float64) {
	return 5, 7
}

func appendArgs() ([]string, string) {
	return []string{"foo"}, "bar"
}

func h() (i interface{}, ok bool) {
	m := map[int]string{1: "hi"}
	i, ok = m[1] // string->interface{} conversion within multi-valued expression
	return
}

func main() {
	f1v(g())
	f2(g())
	f2v(g())
	// TODO(gri): the typechecker still doesn't support these cases correctly.
	// if c := complex(complexArgs()); c != 5+7i {
	// 	panic(c)
	// }
	// if s := append(appendArgs()); len(s) != 2 || s[0] != "foo" || s[1] != "bar" {
	// 	panic(s)
	// }

	i, ok := h()
	if !ok || i.(string) != "hi" {
		panic(i)
	}
}
<commit_msg>go.tools/ssa/interp: enable tests of builtin(f()) where f has multiple results.<commit_after>// Tests of call chaining f(g()) when g has multiple return values (MRVs).
// See https://code.google.com/p/go/issues/detail?id=4573.

package main

func assert(actual, expected int) {
	if actual != expected {
		panic(actual)
	}
}

func g() (int, int) {
	return 5, 7
}

func g2() (float64, float64) {
	return 5, 7
}

func f1v(x int, v ...int) {
	assert(x, 5)
	assert(v[0], 7)
}

func f2(x, y int) {
	assert(x, 5)
	assert(y, 7)
}

func f2v(x, y int, v ...int) {
	assert(x, 5)
	assert(y, 7)
	assert(len(v), 0)
}

func complexArgs() (float64, float64) {
	return 5, 7
}

func appendArgs() ([]string, string) {
	return []string{"foo"}, "bar"
}

func h() (i interface{}, ok bool) {
	m := map[int]string{1: "hi"}
	i, ok = m[1] // string->interface{} conversion within multi-valued expression
	return
}

func main() {
	f1v(g())
	f2(g())
	f2v(g())
	if c := complex(complexArgs()); c != 5+7i {
		panic(c)
	}
	if s := append(appendArgs()); len(s) != 2 || s[0] != "foo" || s[1] != "bar" {
		panic(s)
	}
	i, ok := h()
	if !ok || i.(string) != "hi" {
		panic(i)
	}
}
<|endoftext|>


<commit_before>// Copyright The OpenTelemetry Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package oterror

// Handler performs a required function when an irremediable event is
// encountered.
type Handler interface {
	// Handle handles any error that irremediable by an OpenTelemetry
	// component.
	Handle(error)
}
<commit_msg>Update Handler comments<commit_after>// Copyright The OpenTelemetry Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package oterror

// Handler handles irremediable events.
type Handler interface {
	// Handle handles any error deemed irremediable by an OpenTelemetry
	// component.
	Handle(error)
}
<|endoftext|>


<commit_before>package main

import (
	"database/sql"
	"encoding/json"
	"fmt"
	"github.com/lib/pq"
	"log"
	"time"
)

type ServiceRequest struct {
	Lat, Long                                                                                float64
	Ward, Ward2015, Police_district, Transition_area_id                                                     int
	Service_request_id, Status, Service_name, Service_code, Agency_responsible, Address, Channel, Media_url string
	Requested_datetime, Updated_datetime                                                                    time.Time // FIXME: should these be proper time objects?
	Extended_attributes                                                                                map[string]interface{}
	Notes                                                                                []map[string]interface{}
}

type ServiceRequestDB struct {
	InsertStmt *sql.Stmt
	UpdateStmt *sql.Stmt
	db         *sql.DB
}

func (srdb *ServiceRequestDB) Init(db *sql.DB) error {
	srdb.db = db
	srdb.SetupStmts()

	return nil
}

func (srdb *ServiceRequestDB) Close() error {
	log.Printf("Closing ServiceRequestDB database connection.")
	srdb.db.Close()

	return nil
}

func (srdb *ServiceRequestDB) SetupStmts() {
	insert, err := srdb.db.Prepare(`INSERT INTO service_requests(service_request_id,
		status, service_name, service_code, agency_responsible,
		address, requested_datetime, updated_datetime, lat, long,
		ward, police_district, media_url, channel, duplicate, parent_service_request_id, closed_datetime, notes, ward_2015, transition_area_id)
		VALUES ($1::varchar, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20);`)

	if err != nil {
		log.Fatal("error preparing insert statement ", err)
	}
	srdb.InsertStmt = insert

	update, err := srdb.db.Prepare(`UPDATE service_requests SET
		status = $2, service_name = $3, service_code = $4, agency_responsible = $5,
		address = $6, requested_datetime = $7, updated_datetime = $8, lat = $9, long = $10,
		ward = $11, police_district = $12, media_url = $13, channel = $14, duplicate = $15,
		parent_service_request_id = $16, updated_at = NOW(), closed_datetime = $17, notes = $18, ward_2015 = $19, transition_area_id = $20
		WHERE service_request_id = $1;`)

	if err != nil {
		log.Fatal("error preparing update statement ", err)
	}
	srdb.UpdateStmt = update
}

func (req ServiceRequest) String() string {
	// pretty print SR information
	return fmt.Sprintf("%s: %s at %s %f,%f, last update %s", req.Service_request_id, req.Service_name, req.Address, req.Lat, req.Long, req.Updated_datetime)
}

func (srdb *ServiceRequestDB) Newest() (*ServiceRequest, error) {
	var newest ServiceRequest
	if err := srdb.db.QueryRow("SELECT MAX(updated_datetime) FROM service_requests;").Scan(&newest.Updated_datetime); err != nil {
		log.Print("error loading most recent SR", err)
	}
	return &newest, nil
}

func (srdb *ServiceRequestDB) Oldest() (*ServiceRequest, error) {
	var oldest ServiceRequest
	if err := srdb.db.QueryRow("SELECT MIN(updated_datetime) FROM service_requests;").Scan(&oldest.Updated_datetime); err != nil {
		log.Print("error loading oldest SR", err)
	}
	return &oldest, nil
}

func (srdb *ServiceRequestDB) Save(req *ServiceRequest) (persisted bool) { // FIXME: should return error, too
	persisted = false

	// open311 says we should always ignore a SR that does not have a SR# assigned
	if req.Service_request_id == "" {
		log.Printf("cowardly refusing to create a new SR record because of empty SR#. Request type is %s", req.Service_name)
		return persisted
	}

	// find existing record if exists
	var existing_id int
	err := srdb.db.QueryRow("SELECT id FROM service_requests WHERE service_request_id = $1", req.Service_request_id).Scan(&existing_id)
	switch {
	case err == sql.ErrNoRows:
		// log.Printf("did not find existing record %s", req.Service_request_id)
	case err != nil:
		log.Fatal("error searching for existing SR", err)
	default:
		persisted = true
		// log.Printf("found existing sr %s", req.Service_request_id)
	}

	var stmt *sql.Stmt

	if !persisted {
		stmt = srdb.InsertStmt
	} else {
		stmt = srdb.UpdateStmt
	}

	t := req.ExtractClosedDatetime()
	closed_time := pq.NullTime{Time: t, Valid: !t.IsZero()}
	notes_as_json, err := json.Marshal(req.Notes)
	if err != nil {
		log.Print("error marshaling notes to JSON: ", err)
	}
	new_ward := srdb.Ward(req, 2015)
	transition_area_id := srdb.TransitionArea(req)

	_, err = stmt.Exec(req.Service_request_id,
		req.Status,
		req.Service_name,
		req.Service_code,
		req.Agency_responsible,
		req.Address,
		req.Requested_datetime,
		req.Updated_datetime,
		req.Lat,
		req.Long,
		req.Extended_attributes["ward"],
		req.Extended_attributes["police_district"],
		req.Media_url,
		req.Extended_attributes["channel"],
		req.Extended_attributes["duplicate"],
		req.Extended_attributes["parent_service_request_id"],
		closed_time,
		notes_as_json,
		new_ward,
		transition_area_id)

	if err != nil {
		log.Printf("[error] could not update %s because %s", req.Service_request_id, err)
	} else {
		var verb string
		switch {
		case !persisted && closed_time.Time.IsZero():
			verb = "CREATED"
		case !persisted && !closed_time.Time.IsZero():
			verb = "CREATED/CLOSED"
		case persisted && closed_time.Time.IsZero():
			verb = "UPDATED"
		case persisted && !closed_time.Time.IsZero():
			verb = "UPDATED/CLOSED"
		}

		log.Printf("[%s] %s", verb, req)
		persisted = true
	}

	return persisted
}

func (req ServiceRequest) ExtractClosedDatetime() time.Time {
	// given an extended_attributes JSON blob, pluck out the closed time, if present
	// req.PrintNotes()

	var closed_at time.Time
	for _, note := range req.Notes {
		if note["type"] == "closed" {
			parsed_date, err := time.Parse("2006-01-02T15:04:05-07:00", note["datetime"].(string))
			if err != nil {
				log.Print("error parsing date", err)
			}
			log.Printf("SR %s closed at: %s", req, parsed_date)
			closed_at = parsed_date
		}
	}

	return closed_at
}

func (req ServiceRequest) PrintNotes() {
	fmt.Printf("Notes for SR %s:\n", req.Service_request_id)

	for _, note := range req.Notes {
		fmt.Printf("%+v\n", note)
	}
}

func (srdb *ServiceRequestDB) Ward(sr *ServiceRequest, year int) (ward int) {
	// given a year, return the ward containing the SR

	var boundaries_table string

	switch year {
	case 2013:
		boundaries_table = "ward_boundaries_2013"
	case 2015:
		boundaries_table = "ward_boundaries_2015"
	}

	query := fmt.Sprintf("SELECT ward FROM %s WHERE ST_Contains(boundary, ST_PointFromText('POINT(%f %f)', 4326))", boundaries_table, sr.Long, sr.Lat)

	err := srdb.db.QueryRow(query).Scan(&ward)
	if err != nil {
		log.Print(err)
	}

	return
}

func (srdb *ServiceRequestDB) TransitionArea(sr *ServiceRequest) (transition_area int) {
	// given an SR, return the transition area it exists in, if any

	query := fmt.Sprintf("SELECT id FROM transition_areas WHERE ST_Contains(boundary, ST_PointFromText('POINT(%f %f)', 4326))", sr.Long, sr.Lat)

	err := srdb.db.QueryRow(query).Scan(&transition_area)
	if err != nil {
		log.Print(err)
	}

	return
}
<commit_msg>allow for nil transition_area_id<commit_after>package main

import (
	"database/sql"
	"encoding/json"
	"fmt"
	"github.com/lib/pq"
	"log"
	"time"
)

type ServiceRequest struct {
	Lat, Long                                                                                float64
	Ward, Ward2015, Police_district, Transition_area_id                                                     int
	Service_request_id, Status, Service_name, Service_code, Agency_responsible, Address, Channel, Media_url string
	Requested_datetime, Updated_datetime                                                                    time.Time // FIXME: should these be proper time objects?
	Extended_attributes                                                                                map[string]interface{}
	Notes                                                                                []map[string]interface{}
}

type ServiceRequestDB struct {
	InsertStmt *sql.Stmt
	UpdateStmt *sql.Stmt
	db         *sql.DB
}

func (srdb *ServiceRequestDB) Init(db *sql.DB) error {
	srdb.db = db
	srdb.SetupStmts()

	return nil
}

func (srdb *ServiceRequestDB) Close() error {
	log.Printf("Closing ServiceRequestDB database connection.")
	srdb.db.Close()

	return nil
}

func (srdb *ServiceRequestDB) SetupStmts() {
	insert, err := srdb.db.Prepare(`INSERT INTO service_requests(service_request_id,
		status, service_name, service_code, agency_responsible,
		address, requested_datetime, updated_datetime, lat, long,
		ward, police_district, media_url, channel, duplicate, parent_service_request_id, closed_datetime, notes, ward_2015, transition_area_id)
		VALUES ($1::varchar, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20);`)

	if err != nil {
		log.Fatal("error preparing insert statement ", err)
	}
	srdb.InsertStmt = insert

	update, err := srdb.db.Prepare(`UPDATE service_requests SET
		status = $2, service_name = $3, service_code = $4, agency_responsible = $5,
		address = $6, requested_datetime = $7, updated_datetime = $8, lat = $9, long = $10,
		ward = $11, police_district = $12, media_url = $13, channel = $14, duplicate = $15,
		parent_service_request_id = $16, updated_at = NOW(), closed_datetime = $17, notes = $18, ward_2015 = $19, transition_area_id = $20
		WHERE service_request_id = $1;`)

	if err != nil {
		log.Fatal("error preparing update statement ", err)
	}
	srdb.UpdateStmt = update
}

func (req ServiceRequest) String() string {
	// pretty print SR information
	return fmt.Sprintf("%s: %s at %s %f,%f, last update %s", req.Service_request_id, req.Service_name, req.Address, req.Lat, req.Long, req.Updated_datetime)
}

func (srdb *ServiceRequestDB) Newest() (*ServiceRequest, error) {
	var newest ServiceRequest
	if err := srdb.db.QueryRow("SELECT MAX(updated_datetime) FROM service_requests;").Scan(&newest.Updated_datetime); err != nil {
		log.Print("error loading most recent SR", err)
	}
	return &newest, nil
}

func (srdb *ServiceRequestDB) Oldest() (*ServiceRequest, error) {
	var oldest ServiceRequest
	if err := srdb.db.QueryRow("SELECT MIN(updated_datetime) FROM service_requests;").Scan(&oldest.Updated_datetime); err != nil {
		log.Print("error loading oldest SR", err)
	}
	return &oldest, nil
}

func (srdb *ServiceRequestDB) Save(req *ServiceRequest) (persisted bool) { // FIXME: should return error, too
	persisted = false

	// open311 says we should always ignore a SR that does not have a SR# assigned
	if req.Service_request_id == "" {
		log.Printf("cowardly refusing to create a new SR record because of empty SR#. Request type is %s", req.Service_name)
		return persisted
	}

	// find existing record if exists
	var existing_id int
	err := srdb.db.QueryRow("SELECT id FROM service_requests WHERE service_request_id = $1", req.Service_request_id).Scan(&existing_id)
	switch {
	case err == sql.ErrNoRows:
		// log.Printf("did not find existing record %s", req.Service_request_id)
	case err != nil:
		log.Fatal("error searching for existing SR", err)
	default:
		persisted = true
		// log.Printf("found existing sr %s", req.Service_request_id)
	}

	var stmt *sql.Stmt

	if !persisted {
		stmt = srdb.InsertStmt
	} else {
		stmt = srdb.UpdateStmt
	}

	// preprocessing some attrs
	// closed time
	t := req.ExtractClosedDatetime()
	closed_time := pq.NullTime{Time: t, Valid: !t.IsZero()}
	// notes
	notes_as_json, err := json.Marshal(req.Notes)
	if err != nil {
		log.Print("error marshaling notes to JSON: ", err)
	}
	// 2015 wards
	new_ward := srdb.Ward(req, 2015)
	// transition areas
	transition_area_id, err := srdb.TransitionArea(req)

	_, err = stmt.Exec(req.Service_request_id,
		req.Status,
		req.Service_name,
		req.Service_code,
		req.Agency_responsible,
		req.Address,
		req.Requested_datetime,
		req.Updated_datetime,
		req.Lat,
		req.Long,
		req.Extended_attributes["ward"],
		req.Extended_attributes["police_district"],
		req.Media_url,
		req.Extended_attributes["channel"],
		req.Extended_attributes["duplicate"],
		req.Extended_attributes["parent_service_request_id"],
		closed_time,
		notes_as_json,
		new_ward,
		transition_area_id)

	if err != nil {
		log.Printf("[error] could not update %s because %s", req.Service_request_id, err)
	} else {
		var verb string
		switch {
		case !persisted && closed_time.Time.IsZero():
			verb = "CREATED"
		case !persisted && !closed_time.Time.IsZero():
			verb = "CREATED/CLOSED"
		case persisted && closed_time.Time.IsZero():
			verb = "UPDATED"
		case persisted && !closed_time.Time.IsZero():
			verb = "UPDATED/CLOSED"
		}

		log.Printf("[%s] %s", verb, req)
		persisted = true
	}

	return persisted
}

func (req ServiceRequest) ExtractClosedDatetime() time.Time {
	// given an extended_attributes JSON blob, pluck out the closed time, if present
	// req.PrintNotes()

	var closed_at time.Time
	for _, note := range req.Notes {
		if note["type"] == "closed" {
			parsed_date, err := time.Parse("2006-01-02T15:04:05-07:00", note["datetime"].(string))
			if err != nil {
				log.Print("error parsing date", err)
			}
			log.Printf("SR %s closed at: %s", req, parsed_date)
			closed_at = parsed_date
		}
	}

	return closed_at
}

func (req ServiceRequest) PrintNotes() {
	fmt.Printf("Notes for SR %s:\n", req.Service_request_id)

	for _, note := range req.Notes {
		fmt.Printf("%+v\n", note)
	}
}

func (srdb *ServiceRequestDB) Ward(sr *ServiceRequest, year int) (ward int) {
	// given a year, return the ward containing the SR

	var boundaries_table string

	switch year {
	case 2013:
		boundaries_table = "ward_boundaries_2013"
	case 2015:
		boundaries_table = "ward_boundaries_2015"
	}

	query := fmt.Sprintf("SELECT ward FROM %s WHERE ST_Contains(boundary, ST_PointFromText('POINT(%f %f)', 4326))", boundaries_table, sr.Long, sr.Lat)

	err := srdb.db.QueryRow(query).Scan(&ward)
	if err != nil {
		log.Print(err)
	}

	return
}

func (srdb *ServiceRequestDB) TransitionArea(sr *ServiceRequest) (sql.NullInt64, error) {
	// given an SR, return the transition area it exists in, if any
	var transition_area sql.NullInt64

	query := fmt.Sprintf("SELECT id FROM transition_areas WHERE ST_Contains(boundary, ST_PointFromText('POINT(%f %f)', 4326))", sr.Long, sr.Lat)

	err := srdb.db.QueryRow(query).Scan(&transition_area)
	if err != nil {
		return transition_area, err
	}

	return transition_area, nil
}
<|endoftext|>


<commit_before>// Copyright 2015 The Vanadium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build mojo

package main

// To build:
// cd $JIRI_ROOT/release/projects/mojo/syncbase
// make build

import (
	"log"

	"mojo/public/go/application"
	"mojo/public/go/bindings"
	"mojo/public/go/system"

	"mojom/syncbase"

	"v.io/v23"
	"v.io/v23/context"
	"v.io/v23/rpc"
	"v.io/x/ref/services/syncbase/server"
)

//#include "mojo/public/c/system/types.h"
import "C"

type delegate struct {
	ctx      *context.T
	disp     rpc.Dispatcher
	shutdown func()
	srv      rpc.Server
	cleanup  func()
	stubs    []*bindings.Stub
}

func (d *delegate) Initialize(actx application.Context) {
	d.ctx, d.shutdown = v23.Init(actx)
	if err := setBlessings(d.ctx, actx); err != nil {
		panic(err)
	}
	d.srv, d.disp, d.cleanup = Serve(d.ctx)
}

func (d *delegate) Create(req syncbase.Syncbase_Request) {
	impl := server.NewMojoImpl(d.ctx, d.srv, d.disp)
	stub := syncbase.NewSyncbaseStub(req, impl, bindings.GetAsyncWaiter())
	d.stubs = append(d.stubs, stub)
	go func() {
		for {
			if err := stub.ServeRequest(); err != nil {
				connErr, ok := err.(*bindings.ConnectionError)
				if !ok || !connErr.Closed() {
					log.Println(err)
				}
				break
			}
		}
	}()
}

func (d *delegate) AcceptConnection(conn *application.Connection) {
	conn.ProvideServices(&syncbase.Syncbase_ServiceFactory{d})
}

func (d *delegate) Quit() {
	for _, stub := range d.stubs {
		stub.Close()
	}
	d.cleanup()
	d.shutdown()
}

//export MojoMain
func MojoMain(handle C.MojoHandle) C.MojoResult {
	application.Run(&delegate{}, system.MojoHandle(handle))
	return C.MOJO_RESULT_OK
}

// NOTE(nlacasse): Mojo runs Go code by calling MojoMain().  The main() method
// below is still needed because the Go tool won't build without it.
func main() {}
<commit_msg>syncbase-mojo: spawn 100 workers for parallelism<commit_after>// Copyright 2015 The Vanadium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build mojo

package main

// To build:
// cd $JIRI_ROOT/release/projects/mojo/syncbase
// make build

import (
	"log"

	"mojo/public/go/application"
	"mojo/public/go/bindings"
	"mojo/public/go/system"
	"mojom/syncbase"

	"v.io/v23"
	"v.io/v23/context"
	"v.io/v23/rpc"
	"v.io/x/ref/services/syncbase/server"
)

//#include "mojo/public/c/system/types.h"
import "C"

type delegate struct {
	ctx      *context.T
	disp     rpc.Dispatcher
	shutdown func()
	srv      rpc.Server
	cleanup  func()
	stubs    []*bindings.Stub
}

func (d *delegate) Initialize(actx application.Context) {
	d.ctx, d.shutdown = v23.Init(actx)
	if err := setBlessings(d.ctx, actx); err != nil {
		panic(err)
	}
	d.srv, d.disp, d.cleanup = Serve(d.ctx)
}

const numGoroutines = 100

func (d *delegate) Create(req syncbase.Syncbase_Request) {
	impl := server.NewMojoImpl(d.ctx, d.srv, d.disp)
	stub := syncbase.NewSyncbaseStub(req, impl, bindings.GetAsyncWaiter())
	d.stubs = append(d.stubs, stub)
	// Spawn a bunch of goroutines to handle incoming requests concurrently.
	// Note: It would be better to spawn a goroutine per request as requests
	// arrive, but Mojo's Go interfaces make this difficult at best. Discussions
	// with Mojo team ongoing (as of Nov 25, 2015).
	for i := 0; i < numGoroutines; i++ {
		go func(i int) {
			for {
				if err := stub.ServeRequest(); err != nil {
					connErr, ok := err.(*bindings.ConnectionError)
					if !ok || !connErr.Closed() {
						log.Println(i, err)
					}
					break
				}
			}
		}(i)
	}
}

func (d *delegate) AcceptConnection(conn *application.Connection) {
	conn.ProvideServices(&syncbase.Syncbase_ServiceFactory{d})
}

func (d *delegate) Quit() {
	for _, stub := range d.stubs {
		stub.Close()
	}
	d.cleanup()
	d.shutdown()
}

//export MojoMain
func MojoMain(handle C.MojoHandle) C.MojoResult {
	application.Run(&delegate{}, system.MojoHandle(handle))
	return C.MOJO_RESULT_OK
}

// NOTE(nlacasse): Mojo runs Go code by calling MojoMain().  The main() method
// below is still needed because the Go tool won't build without it.
func main() {}
<|endoftext|>


<commit_before>package jsval

import "errors"

func (c *comboconstraint) Add(v Constraint) {
	c.constraints = append(c.constraints, v)
}

func (c *comboconstraint) Constraints() []Constraint {
	return c.constraints
}

func reduceCombined(cc interface{ Constraint; Constraints() []Constraint }) Constraint {
	l := cc.Constraints()
	if len(l) == 1 {
		return l[0]
	}
	return cc
}

func Any() *AnyConstraint {
	return &AnyConstraint{}
}

func (c *AnyConstraint) Reduce() Constraint {
	return reduceCombined(c)
}

func (c *AnyConstraint) Add(c2 Constraint) *AnyConstraint {
	c.comboconstraint.Add(c2)
	return c
}

func (c *AnyConstraint) Validate(v interface{}) error {
	for _, celem := range c.constraints {
		if err := celem.Validate(v); err == nil {
			return nil
		}
	}
	return errors.New("could not validate against any of the constraints")
}

func All() *AllConstraint {
	return &AllConstraint{}
}

func (c *AllConstraint) Reduce() Constraint {
	return reduceCombined(c)
}

func (c *AllConstraint) Add(c2 Constraint) *AllConstraint {
	c.comboconstraint.Add(c2)
	return c
}

func (c *AllConstraint) Validate(v interface{}) error {
	for _, celem := range c.constraints {
		if err := celem.Validate(v); err != nil {
			return err
		}
	}
	return nil
}

func OneOf() *OneOfConstraint {
	return &OneOfConstraint{}
}

func (c *OneOfConstraint) Reduce() Constraint {
	return reduceCombined(c)
}

func (c *OneOfConstraint) Add(c2 Constraint) *OneOfConstraint {
	c.comboconstraint.Add(c2)
	return c
}

func (c *OneOfConstraint) Validate(v interface{}) error {
	count := 0
	for _, celem := range c.constraints {
		if err := celem.Validate(v); err == nil {
			count++
		}
	}

	if count == 0 {
		return errors.New("none of the constraints passed")
	} else if count > 1 {
		return errors.New("more than 1 of the constraints passed")
	}
	return nil // Yes!
}
<commit_msg>golint<commit_after>package jsval

import "errors"

func (c *comboconstraint) Add(v Constraint) {
	c.constraints = append(c.constraints, v)
}

func (c *comboconstraint) Constraints() []Constraint {
	return c.constraints
}

func reduceCombined(cc interface{ Constraint; Constraints() []Constraint }) Constraint {
	l := cc.Constraints()
	if len(l) == 1 {
		return l[0]
	}
	return cc
}

// Any creates a new AnyConstraint
func Any() *AnyConstraint {
	return &AnyConstraint{}
}

// Reduce returns the child Constraint, if this constraint
// has only 1 child constraint.
func (c *AnyConstraint) Reduce() Constraint {
	return reduceCombined(c)
}

// Add appends a new Constraint
func (c *AnyConstraint) Add(c2 Constraint) *AnyConstraint {
	c.comboconstraint.Add(c2)
	return c
}

// Validate validates the value against the input value.
// For AnyConstraints, it will return success the moment
// one child Constraint succeeds. It will return an error
// if none of the child Constraints succeeds
func (c *AnyConstraint) Validate(v interface{}) error {
	for _, celem := range c.constraints {
		if err := celem.Validate(v); err == nil {
			return nil
		}
	}
	return errors.New("could not validate against any of the constraints")
}

// All creates a new AllConstraint
func All() *AllConstraint {
	return &AllConstraint{}
}

// Reduce returns the child Constraint, if this constraint
// has only 1 child constraint.
func (c *AllConstraint) Reduce() Constraint {
	return reduceCombined(c)
}

// Add appends a new Constraint
func (c *AllConstraint) Add(c2 Constraint) *AllConstraint {
	c.comboconstraint.Add(c2)
	return c
}

// Validate validates the value against the input value.
// For AllConstraints, it will only return success if
// all of the child Constraints succeeded.
func (c *AllConstraint) Validate(v interface{}) error {
	for _, celem := range c.constraints {
		if err := celem.Validate(v); err != nil {
			return err
		}
	}
	return nil
}

// OneOf creates a new OneOfConstraint
func OneOf() *OneOfConstraint {
	return &OneOfConstraint{}
}

// Reduce returns the child Constraint, if this constraint
// has only 1 child constraint.
func (c *OneOfConstraint) Reduce() Constraint {
	return reduceCombined(c)
}

// Add appends a new Constraint
func (c *OneOfConstraint) Add(c2 Constraint) *OneOfConstraint {
	c.comboconstraint.Add(c2)
	return c
}

// Validate validates the value against the input value.
// For OneOfConstraints, it will return success only if
// exactly 1 child Constraint succeeds.
func (c *OneOfConstraint) Validate(v interface{}) error {
	count := 0
	for _, celem := range c.constraints {
		if err := celem.Validate(v); err == nil {
			count++
		}
	}

	if count == 0 {
		return errors.New("none of the constraints passed")
	} else if count > 1 {
		return errors.New("more than 1 of the constraints passed")
	}
	return nil // Yes!
}
<|endoftext|>


<commit_before>package command

import (
	"fmt"
	"github.com/davidnix/ffdraft/players"
	"strconv"
	"strings"
	"errors"
)

func Pick(r *players.Repo, args []string) {
	name := strings.Join(args, " ")

	p, err := choose(r.FindAvailable(name))
	if err != nil {
		fmt.Println(err)
		return
	}

	if err := r.Pick(p); err != nil {
		fmt.Println(err)
		return
	}
	fmt.Println(p.ShortDesc(), "was picked.", len(r.UnDrafted), "available players remaining.")
}

func UnPick(r *players.Repo, args []string) {
	name := strings.Join(args, " ")
	p, err := choose(r.FindUnavailable(name))
	if err != nil {
		fmt.Println(err)
		return
	}

	if err := r.UnPick(p); err != nil {
		fmt.Println(err)
		return
	}
	fmt.Println(p.ShortDesc(), "is now available.", len(r.UnDrafted), "available players remaining.")
}

func choose(choices []players.Player) (players.Player, error) {
	for i, p := range choices {
		fmt.Printf("%v: %s", i+1, p.ShortDesc() + "\n")
	}

	fmt.Print("Choose player:")
	selection, err := strconv.ParseInt(GetInput('\n'), 0, 0)
	if err != nil {
		return players.Player{}, err
	}
	index := int(selection - 1)
	if index < 0 || index >= len(choices) {
		return players.Player{}, errors.New("Invalid choice. No player selected.")
	}
	return choices[index], nil
}<commit_msg>better error messaging<commit_after>package command

import (
	"fmt"
	"github.com/davidnix/ffdraft/players"
	"strconv"
	"strings"
	"errors"
)

var invalidErr = errors.New("Invalid choice. No player selected.")

func Pick(r *players.Repo, args []string) {
	name := strings.Join(args, " ")

	p, err := choose(r.FindAvailable(name))
	if err != nil {
		fmt.Println(err)
		return
	}

	if err := r.Pick(p); err != nil {
		fmt.Println(err)
		return
	}
	fmt.Println(p.ShortDesc(), "was picked.", len(r.UnDrafted), "available players remaining.")
}

func UnPick(r *players.Repo, args []string) {
	name := strings.Join(args, " ")
	p, err := choose(r.FindUnavailable(name))
	if err != nil {
		fmt.Println(err)
		return
	}

	if err := r.UnPick(p); err != nil {
		fmt.Println(err)
		return
	}
	fmt.Println(p.ShortDesc(), "is now available.", len(r.UnDrafted), "available players remaining.")
}

func choose(choices []players.Player) (players.Player, error) {
	for i, p := range choices {
		fmt.Printf("%v: %s", i+1, p.ShortDesc() + "\n")
	}

	fmt.Print("Choose player:")
	selection, err := strconv.ParseInt(GetInput('\n'), 0, 0)
	if err != nil {
		return players.Player{}, invalidErr
	}
	index := int(selection - 1)
	if index < 0 || index >= len(choices) {
		return players.Player{}, invalidErr
	}
	return choices[index], nil
}<|endoftext|>