<commit_before>package service

import (
	"errors"
	"sync"

	msg "github.com/jbenet/go-ipfs/net/message"
	u "github.com/jbenet/go-ipfs/util"

	context "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/go.net/context"
)

// Handler is an interface that objects must implement in order to handle
// a service's requests.
type Handler interface {

	// HandleMessage receives an incoming message, and potentially returns
	// a response message to send back.
	HandleMessage(context.Context, msg.NetMessage) (msg.NetMessage, error)
}

// Service is a networking component that protocols can use to multiplex
// messages over the same channel, and to issue + handle requests.
type Service struct {
	// Handler is the object registered to handle incoming requests.
	Handler Handler

	// Requests are all the pending requests on this service.
	Requests     RequestMap
	RequestsLock sync.RWMutex

	// cancel is the function to stop the Service
	cancel context.CancelFunc

	// Message Pipe (connected to the outside world)
	*msg.Pipe
}

// NewService creates a service object with given type ID and Handler
func NewService(h Handler) *Service {
	return &Service{
		Handler:  h,
		Requests: RequestMap{},
		Pipe:     msg.NewPipe(10),
	}
}

// Start kicks off the Service goroutines.
func (s *Service) Start(ctx context.Context) error {
	if s.cancel != nil {
		return errors.New("Service already started.")
	}

	// make a cancellable context.
	ctx, s.cancel = context.WithCancel(ctx)

	go s.handleIncomingMessages(ctx)
	return nil
}

// Stop stops Service activity.
func (s *Service) Stop() {
	s.cancel()
	s.cancel = context.CancelFunc(nil)
}

// GetPipe implements the mux.Protocol interface
func (s *Service) GetPipe() *msg.Pipe {
	return s.Pipe
}

// sendMessage sends a message out (actual leg work. SendMessage is to export w/o rid)
func (s *Service) sendMessage(ctx context.Context, m msg.NetMessage, rid RequestID) error {

	// serialize ServiceMessage wrapper
	data, err := wrapData(m.Data(), rid)
	if err != nil {
		return err
	}

	// send message
	m2 := msg.New(m.Peer(), data)
	select {
	case s.Outgoing <- m2:
	case <-ctx.Done():
		return ctx.Err()
	}

	return nil
}

// SendMessage sends a message out
func (s *Service) SendMessage(ctx context.Context, m msg.NetMessage) error {
	return s.sendMessage(ctx, m, nil)
}

// SendRequest sends a request message out and awaits a response.
func (s *Service) SendRequest(ctx context.Context, m msg.NetMessage) (msg.NetMessage, error) {

	// create a request
	r, err := NewRequest(m.Peer().ID)
	if err != nil {
		return nil, err
	}

	// register Request
	s.RequestsLock.Lock()
	s.Requests[r.Key()] = r
	s.RequestsLock.Unlock()

	// defer deleting this request
	defer func() {
		s.RequestsLock.Lock()
		delete(s.Requests, r.Key())
		s.RequestsLock.Unlock()
	}()

	// check if we should bail after waiting for mutex
	select {
	default:
	case <-ctx.Done():
		return nil, ctx.Err()
	}

	// Send message
	s.sendMessage(ctx, m, r.ID)

	// wait for response
	m = nil
	err = nil
	select {
	case m = <-r.Response:
	case <-ctx.Done():
		err = ctx.Err()
	}

	return m, err
}

// handleIncoming consumes the messages on the s.Incoming channel and
// routes them appropriately (to requests, or handler).
func (s *Service) handleIncomingMessages(ctx context.Context) {
	for {
		select {
		case m := <-s.Incoming:
			go s.handleIncomingMessage(ctx, m)

		case <-ctx.Done():
			return
		}
	}
}

func (s *Service) handleIncomingMessage(ctx context.Context, m msg.NetMessage) {

	// unwrap the incoming message
	data, rid, err := unwrapData(m.Data())
	if err != nil {
		u.PErr("de-serializing error: %v\n", err)
	}
	m2 := msg.New(m.Peer(), data)

	// if it's a request (or has no RequestID), handle it
	if rid == nil || rid.IsRequest() {
		if s.Handler == nil {
			u.PErr("service dropped msg: %v\n", m)
			return // no handler, drop it.
		}

		// should this be "go HandleMessage ... ?"
		r1, err := s.Handler.HandleMessage(ctx, m2)
		if err != nil {
			u.PErr("handled message yielded error %v\n", err)
			return
		}

		// if handler gave us a response, send it back out!
		if r1 != nil {
			err := s.sendMessage(ctx, r1, rid.Response())
			if err != nil {
				u.PErr("error sending response message: %v\n", err)
			}
		}
		return
	}

	// Otherwise, it is a response. handle it.
	if !rid.IsResponse() {
		u.PErr("RequestID should identify a response here.\n")
	}

	key := RequestKey(m.Peer().ID, RequestID(rid))
	s.RequestsLock.RLock()
	r, found := s.Requests[key]
	s.RequestsLock.RUnlock()

	if !found {
		u.PErr("no request key %v (timeout?)\n", []byte(key))
		return
	}

	select {
	case r.Response <- m2:
	case <-ctx.Done():
	}
}
<commit_msg>feat(net:service) add SetHandler method<commit_after>package service

import (
	"errors"
	"sync"

	msg "github.com/jbenet/go-ipfs/net/message"
	u "github.com/jbenet/go-ipfs/util"

	context "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/go.net/context"
)

// Handler is an interface that objects must implement in order to handle
// a service's requests.
type Handler interface {

	// HandleMessage receives an incoming message, and potentially returns
	// a response message to send back.
	HandleMessage(context.Context, msg.NetMessage) (msg.NetMessage, error)
}

// Service is a networking component that protocols can use to multiplex
// messages over the same channel, and to issue + handle requests.
type Service struct {
	// Handler is the object registered to handle incoming requests.
	Handler Handler

	// Requests are all the pending requests on this service.
	Requests     RequestMap
	RequestsLock sync.RWMutex

	// cancel is the function to stop the Service
	cancel context.CancelFunc

	// Message Pipe (connected to the outside world)
	*msg.Pipe
}

// NewService creates a service object with given type ID and Handler
func NewService(h Handler) *Service {
	return &Service{
		Handler:  h,
		Requests: RequestMap{},
		Pipe:     msg.NewPipe(10),
	}
}

// Start kicks off the Service goroutines.
func (s *Service) Start(ctx context.Context) error {
	if s.cancel != nil {
		return errors.New("Service already started.")
	}

	// make a cancellable context.
	ctx, s.cancel = context.WithCancel(ctx)

	go s.handleIncomingMessages(ctx)
	return nil
}

// Stop stops Service activity.
func (s *Service) Stop() {
	s.cancel()
	s.cancel = context.CancelFunc(nil)
}

// GetPipe implements the mux.Protocol interface
func (s *Service) GetPipe() *msg.Pipe {
	return s.Pipe
}

// sendMessage sends a message out (actual leg work. SendMessage is to export w/o rid)
func (s *Service) sendMessage(ctx context.Context, m msg.NetMessage, rid RequestID) error {

	// serialize ServiceMessage wrapper
	data, err := wrapData(m.Data(), rid)
	if err != nil {
		return err
	}

	// send message
	m2 := msg.New(m.Peer(), data)
	select {
	case s.Outgoing <- m2:
	case <-ctx.Done():
		return ctx.Err()
	}

	return nil
}

// SendMessage sends a message out
func (s *Service) SendMessage(ctx context.Context, m msg.NetMessage) error {
	return s.sendMessage(ctx, m, nil)
}

// SendRequest sends a request message out and awaits a response.
func (s *Service) SendRequest(ctx context.Context, m msg.NetMessage) (msg.NetMessage, error) {

	// create a request
	r, err := NewRequest(m.Peer().ID)
	if err != nil {
		return nil, err
	}

	// register Request
	s.RequestsLock.Lock()
	s.Requests[r.Key()] = r
	s.RequestsLock.Unlock()

	// defer deleting this request
	defer func() {
		s.RequestsLock.Lock()
		delete(s.Requests, r.Key())
		s.RequestsLock.Unlock()
	}()

	// check if we should bail after waiting for mutex
	select {
	default:
	case <-ctx.Done():
		return nil, ctx.Err()
	}

	// Send message
	s.sendMessage(ctx, m, r.ID)

	// wait for response
	m = nil
	err = nil
	select {
	case m = <-r.Response:
	case <-ctx.Done():
		err = ctx.Err()
	}

	return m, err
}

// handleIncoming consumes the messages on the s.Incoming channel and
// routes them appropriately (to requests, or handler).
func (s *Service) handleIncomingMessages(ctx context.Context) {
	for {
		select {
		case m := <-s.Incoming:
			go s.handleIncomingMessage(ctx, m)

		case <-ctx.Done():
			return
		}
	}
}

func (s *Service) handleIncomingMessage(ctx context.Context, m msg.NetMessage) {

	// unwrap the incoming message
	data, rid, err := unwrapData(m.Data())
	if err != nil {
		u.PErr("de-serializing error: %v\n", err)
	}
	m2 := msg.New(m.Peer(), data)

	// if it's a request (or has no RequestID), handle it
	if rid == nil || rid.IsRequest() {
		if s.Handler == nil {
			u.PErr("service dropped msg: %v\n", m)
			return // no handler, drop it.
		}

		// should this be "go HandleMessage ... ?"
		r1, err := s.Handler.HandleMessage(ctx, m2)
		if err != nil {
			u.PErr("handled message yielded error %v\n", err)
			return
		}

		// if handler gave us a response, send it back out!
		if r1 != nil {
			err := s.sendMessage(ctx, r1, rid.Response())
			if err != nil {
				u.PErr("error sending response message: %v\n", err)
			}
		}
		return
	}

	// Otherwise, it is a response. handle it.
	if !rid.IsResponse() {
		u.PErr("RequestID should identify a response here.\n")
	}

	key := RequestKey(m.Peer().ID, RequestID(rid))
	s.RequestsLock.RLock()
	r, found := s.Requests[key]
	s.RequestsLock.RUnlock()

	if !found {
		u.PErr("no request key %v (timeout?)\n", []byte(key))
		return
	}

	select {
	case r.Response <- m2:
	case <-ctx.Done():
	}
}

func (s *Service) SetHandler(h Handler) {
	s.Handler = h
}
<|endoftext|>


<commit_before>// Copyright Microsoft Corp.
// All rights reserved.

package network

import (
	"fmt"
	"net/http"
	"net/http/httptest"
	"os"
	"testing"
)

var plugin NetPlugin
var mux *http.ServeMux

func TestMain(m *testing.M) {
	var err error

	// Create the plugin.
	plugin, err = NewPlugin("test", "")
	if err != nil {
		fmt.Printf("Failed to create network plugin %v\n", err)
		return
	}

	err = plugin.Start(nil)
	if err != nil {
		fmt.Printf("Failed to start network plugin %v\n", err)
		return
	}

	mux = plugin.GetListener().GetMux()

	// Run tests.
	exitCode := m.Run()

	// Cleanup.
	plugin.Stop()

	os.Exit(exitCode)
}

//
// Libnetwork remote API compliance tests
// github.com/docker/libnetwork/drivers/remote/api
//

func TestActivate(t *testing.T) {
	fmt.Println("Test: Activate")

	req, err := http.NewRequest(http.MethodGet, "/Plugin.Activate", nil)
	if err != nil {
		t.Fatal(err)
	}

	w := httptest.NewRecorder()
	mux.ServeHTTP(w, req)

	if w.Code != http.StatusOK {
		t.Error("Activate request failed")
	}

	//fmt.Printf("%d - %s", w.Code, w.Body.String())
}

func TestGetCapabilities(t *testing.T) {
	fmt.Println("Test: GetCapabilities")

	req, err := http.NewRequest(http.MethodGet, "/NetworkDriver.GetCapabilities", nil)
	if err != nil {
		t.Fatal(err)
	}

	w := httptest.NewRecorder()
	mux.ServeHTTP(w, req)

	if w.Code != http.StatusOK {
		t.Error("GetCapabilities request failed")
	}

	//resp api.GetCapabilityResponse
}
<commit_msg>Added network plugin tests for basic remote API validation<commit_after>// Copyright Microsoft Corp.
// All rights reserved.

package network

import (
	"bytes"
	"encoding/json"
	"fmt"
	"net/http"
	"net/http/httptest"
	"os"
	"testing"

	remoteApi "github.com/docker/libnetwork/drivers/remote/api"
)

var plugin NetPlugin
var mux *http.ServeMux

// Wraps the test run with plugin setup and teardown.
func TestMain(m *testing.M) {
	var err error

	// Create the plugin.
	plugin, err = NewPlugin("test", "")
	if err != nil {
		fmt.Printf("Failed to create network plugin %v\n", err)
		return
	}

	err = plugin.Start(nil)
	if err != nil {
		fmt.Printf("Failed to start network plugin %v\n", err)
		return
	}

	mux = plugin.GetListener().GetMux()

	// Run tests.
	exitCode := m.Run()

	// Cleanup.
	plugin.Stop()

	os.Exit(exitCode)
}

// Decodes plugin's responses to test requests.
func decodeResponse(w *httptest.ResponseRecorder, response interface{}) error {
	if w.Code != http.StatusOK {
		return fmt.Errorf("Request failed with HTTP error %s", w.Code)
	}

	if w.Body == nil {
		return fmt.Errorf("Response body is empty")
	}

	return json.NewDecoder(w.Body).Decode(&response)
}

//
// Libnetwork remote API compliance tests
// https://github.com/docker/libnetwork/blob/master/docs/remote.md
//

// Tests Plugin.Activate functionality.
func TestActivate(t *testing.T) {
	fmt.Println("Test: Activate")

	var resp struct {
		Implements []string
	}

	req, err := http.NewRequest(http.MethodGet, "/Plugin.Activate", nil)
	if err != nil {
		t.Fatal(err)
	}

	w := httptest.NewRecorder()
	mux.ServeHTTP(w, req)

	err = decodeResponse(w, &resp)

	if err != nil || resp.Implements[0] != "NetworkDriver" {
		t.Errorf("Activate response is invalid %+v", resp)
	}
}

// Tests NetworkDriver.GetCapabilities functionality.
func TestGetCapabilities(t *testing.T) {
	fmt.Println("Test: GetCapabilities")

	var resp remoteApi.GetCapabilityResponse

	req, err := http.NewRequest(http.MethodGet, "/NetworkDriver.GetCapabilities", nil)
	if err != nil {
		t.Fatal(err)
	}

	w := httptest.NewRecorder()
	mux.ServeHTTP(w, req)

	err = decodeResponse(w, &resp)

	if err != nil || resp.Err != "" || resp.Scope != "local" {
		t.Errorf("GetCapabilities response is invalid %+v", resp)
	}
}

// Tests NetworkDriver.CreateNetwork functionality.
func TestCreateNetwork(t *testing.T) {
	fmt.Println("Test: CreateNetwork")

	var body bytes.Buffer
	var resp remoteApi.CreateNetworkResponse

	info := &remoteApi.CreateNetworkRequest{
		NetworkID:  "N1",
	}

	json.NewEncoder(&body).Encode(info)

	req, err := http.NewRequest(http.MethodGet, "/NetworkDriver.CreateNetwork", &body)
	if err != nil {
		t.Fatal(err)
	}

	w := httptest.NewRecorder()
	mux.ServeHTTP(w, req)

	err = decodeResponse(w, &resp)

	if err != nil || resp.Err != "" {
		t.Errorf("CreateNetwork response is invalid %+v", resp)
	}
}

// Tests NetworkDriver.DeleteNetwork functionality.
func TestDeleteNetwork(t *testing.T) {
	fmt.Println("Test: DeleteNetwork")

	var body bytes.Buffer
	var resp remoteApi.DeleteNetworkResponse

	info := &remoteApi.DeleteNetworkRequest{
		NetworkID:  "N1",
	}

	json.NewEncoder(&body).Encode(info)

	req, err := http.NewRequest(http.MethodGet, "/NetworkDriver.DeleteNetwork", &body)
	if err != nil {
		t.Fatal(err)
	}

	w := httptest.NewRecorder()
	mux.ServeHTTP(w, req)

	err = decodeResponse(w, &resp)

	if err != nil || resp.Err != "" {
		t.Errorf("DeleteNetwork response is invalid %+v", resp)
	}
}

// Tests NetworkDriver.EndpointOperInfo functionality.
func TestEndpointOperInfo(t *testing.T) {
	fmt.Println("Test: EndpointOperInfo")

	var body bytes.Buffer
	var resp remoteApi.EndpointInfoResponse

	info := &remoteApi.EndpointInfoRequest{
		NetworkID:  "N1",
		EndpointID: "E1",
	}

	json.NewEncoder(&body).Encode(info)

	req, err := http.NewRequest(http.MethodGet, "/NetworkDriver.EndpointOperInfo", &body)
	if err != nil {
		t.Fatal(err)
	}

	w := httptest.NewRecorder()
	mux.ServeHTTP(w, req)

	err = decodeResponse(w, &resp)

	if err != nil {
		t.Errorf("EndpointOperInfo response is invalid %+v", resp)
	}
}
<|endoftext|>


<commit_before>/*
Copyright 2014 Google Inc. All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package playlistimporter

import (
	"fmt"
	"net/http"
	"net/url"

	"appengine"
	"appengine/urlfetch"

	"playlistimporter/unwrap"
)

const pllimit = "500"

func indexPlaylistsHandler(w http.ResponseWriter, r *http.Request) {
	r.ParseForm()
	c := appengine.NewContext(r)
	playlistTitles := r.Form["titles"]
	args := url.Values{
		"action": []string{"query"},
		"plnamespace": []string{"0"},
		"pllimit": []string{pllimit},
		"prop": []string{"links"},
		"titles": playlistTitles,
	}
	client := urlfetch.Client(c)
	userAgentString := makeUserAgentString(appengine.VersionID(c))
	jsonRsp, err := queryWikipediaAPI(client, userAgentString, args)
	if err != nil {
		http.Error(
			w,
			"I can't believe this is happening.",
			http.StatusInternalServerError,
		)
		return
	}
	pageIDToPageUntyped, err := unwrap.Unwrap(jsonRsp, ".query.pages")
	if err != nil {
		c.Errorf("%v", jsonRsp)
		http.Error(w, "Badness", http.StatusInternalServerError)
		return
	}
	pageIDToPage, ok := pageIDToPageUntyped.(map[string]interface{})
	if !ok {
		http.Error(w, "Curses", http.StatusInternalServerError)
		return
	}
	for _, page := range pageIDToPage {
		titleUntyped, _ := unwrap.Unwrap(page, ".title")
		title, _ := titleUntyped.(string)
		linkedTitlesUntyped, err := unwrap.Unwrap(page, ".links[:].title")
		if err != nil {
			http.Error(w, err.Error(), http.StatusInternalServerError)
			return
		}
		linkedTitles, ok := linkedTitlesUntyped.([]interface{})
		if !ok {
			http.Error(
				w,
				"linkedTitles type conversion failed",
				http.StatusInternalServerError,
			)
		}
		for _, linkedTitleUntyped := range(linkedTitles) {
			linkedTitle, _ := linkedTitleUntyped.(string)
			fmt.Fprintln(w, url.Values{
				"parent": []string{title},
				"linkedTitles": []string{linkedTitle},
			})
		}
		c.Infof("%v links found on page \"%v\"", len(linkedTitles), title)
	}
}
<commit_msg>Changed Content-Type of indexplaylists debugging to text/plain.<commit_after>/*
Copyright 2014 Google Inc. All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package playlistimporter

import (
	"fmt"
	"net/http"
	"net/url"

	"appengine"
	"appengine/urlfetch"

	"playlistimporter/unwrap"
)

const pllimit = "500"

func indexPlaylistsHandler(w http.ResponseWriter, r *http.Request) {
	w.Header().Add("Content-Type", "text/html; charset=utf-8")
	r.ParseForm()
	c := appengine.NewContext(r)
	playlistTitles := r.Form["titles"]
	args := url.Values{
		"action": []string{"query"},
		"plnamespace": []string{"0"},
		"pllimit": []string{pllimit},
		"prop": []string{"links"},
		"titles": playlistTitles,
	}
	client := urlfetch.Client(c)
	userAgentString := makeUserAgentString(appengine.VersionID(c))
	jsonRsp, err := queryWikipediaAPI(client, userAgentString, args)
	if err != nil {
		http.Error(
			w,
			"I can't believe this is happening.",
			http.StatusInternalServerError,
		)
		return
	}
	pageIDToPageUntyped, err := unwrap.Unwrap(jsonRsp, ".query.pages")
	if err != nil {
		c.Errorf("%v", jsonRsp)
		http.Error(w, "Badness", http.StatusInternalServerError)
		return
	}
	pageIDToPage, ok := pageIDToPageUntyped.(map[string]interface{})
	if !ok {
		http.Error(w, "Curses", http.StatusInternalServerError)
		return
	}
	for _, page := range pageIDToPage {
		titleUntyped, _ := unwrap.Unwrap(page, ".title")
		title, _ := titleUntyped.(string)
		linkedTitlesUntyped, err := unwrap.Unwrap(page, ".links[:].title")
		if err != nil {
			http.Error(w, err.Error(), http.StatusInternalServerError)
			return
		}
		linkedTitles, ok := linkedTitlesUntyped.([]interface{})
		if !ok {
			http.Error(
				w,
				"linkedTitles type conversion failed",
				http.StatusInternalServerError,
			)
		}
		for _, linkedTitleUntyped := range(linkedTitles) {
			linkedTitle, _ := linkedTitleUntyped.(string)
			fmt.Fprintln(w, url.Values{
				"parent": []string{title},
				"linkedTitles": []string{linkedTitle},
			})
		}
		c.Infof("%v links found on page \"%v\"", len(linkedTitles), title)
	}
}
<|endoftext|>


<commit_before>//  Copyright (c) 2019 Cisco and/or its affiliates.
//
//  Licensed under the Apache License, Version 2.0 (the "License");
//  you may not use this file except in compliance with the License.
//  You may obtain a copy of the License at:
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
//  Unless required by applicable law or agreed to in writing, software
//  distributed under the License is distributed on an "AS IS" BASIS,
//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//  See the License for the specific language governing permissions and
//  limitations under the License.

package orchestrator

import (
	"context"
	"fmt"
	"runtime/trace"
	"sync"
	"time"

	"github.com/golang/protobuf/proto"
	"github.com/pkg/errors"
	"go.ligato.io/cn-infra/v2/logging"

	"go.ligato.io/vpp-agent/v3/pkg/models"
	kvs "go.ligato.io/vpp-agent/v3/plugins/kvscheduler/api"
	"go.ligato.io/vpp-agent/v3/proto/ligato/kvscheduler"
)

// KeyVal associates value with its key.
type KeyVal struct {
	Key string
	Val proto.Message
}

// KVPairs represents key-value pairs.
type KVPairs map[string]proto.Message

type Status = kvscheduler.ValueStatus

type Result struct {
	Key    string
	Status *Status
}

type Dispatcher interface {
	ListData() KVPairs
	PushData(context.Context, []KeyVal) ([]Result, error)
	GetStatus(key string) (*Status, error)
	ListState() (KVPairs, error)
}

type dispatcher struct {
	log logging.Logger
	kvs kvs.KVScheduler
	mu  sync.Mutex
	db  KVStore
}

// ListData retrieves actual data.
func (p *dispatcher) ListData() KVPairs {
	p.mu.Lock()
	defer p.mu.Unlock()

	return p.db.ListAll()
}

func (p *dispatcher) GetStatus(key string) (*Status, error) {
	s := p.kvs.GetValueStatus(key)
	status := s.GetValue()
	if status == nil {
		return nil, errors.Errorf("status for key %q not found", key)
	}
	return status, nil
}

// PushData updates actual data.
func (p *dispatcher) PushData(ctx context.Context, kvPairs []KeyVal) (results []Result, err error) {
	trace.Logf(ctx, "pushData", "%d KV pairs", len(kvPairs))

	// validate key-value pairs
	uniq := make(map[string]struct{})
	for _, kv := range kvPairs {
		if kv.Val != nil {
			// check if given key matches the key generated from value
			if k := models.Key(kv.Val); k != kv.Key {
				return nil, errors.Errorf("given key %q does not match with key generated from value: %q (value: %#v)", kv.Key, k, kv.Val)
			}
		}
		// check if key is unique
		if _, ok := uniq[kv.Key]; ok {
			return nil, errors.Errorf("found multiple key-value pairs with same key: ")
		}
		uniq[kv.Key] = struct{}{}
	}

	p.mu.Lock()
	defer p.mu.Unlock()

	pr := trace.StartRegion(ctx, "prepare kv data")

	dataSrc, ok := DataSrcFromContext(ctx)
	if !ok {
		dataSrc = "global"
	}

	p.log.Debugf("Push data with %d KV pairs (source: %s)", len(kvPairs), dataSrc)

	txn := p.kvs.StartNBTransaction()

	if typ, _ := kvs.IsResync(ctx); typ == kvs.FullResync {
		trace.Log(ctx, "resyncType", typ.String())
		p.db.Reset(dataSrc)
		for _, kv := range kvPairs {
			if kv.Val == nil {
				p.log.Debugf(" - PUT: %q (skipped nil value for resync)", kv.Key)
				continue
			}
			p.log.Debugf(" - PUT: %q ", kv.Key)
			p.db.Update(dataSrc, kv.Key, kv.Val)
		}
		allPairs := p.db.ListAll()
		p.log.Debugf("will resync %d pairs", len(allPairs))
		for k, v := range allPairs {
			txn.SetValue(k, v)
		}
	} else {
		for _, kv := range kvPairs {
			if kv.Val == nil {
				p.log.Debugf(" - DELETE: %q", kv.Key)
				txn.SetValue(kv.Key, nil)
				p.db.Delete(dataSrc, kv.Key)
			} else {
				p.log.Debugf(" - UPDATE: %q ", kv.Key)
				txn.SetValue(kv.Key, kv.Val)
				p.db.Update(dataSrc, kv.Key, kv.Val)
			}
		}
	}

	pr.End()

	t := time.Now()

	seqID, err := txn.Commit(ctx)
	if err != nil {
		if txErr, ok := err.(*kvs.TransactionError); ok && len(txErr.GetKVErrors()) > 0 {
			kvErrs := txErr.GetKVErrors()
			var errInfo = ""
			for i, kvErr := range kvErrs {
				errInfo += fmt.Sprintf(" - %3d. error (%s) %s - %v\n", i+1, kvErr.TxnOperation, kvErr.Key, kvErr.Error)
			}
			p.log.Errorf("Transaction #%d finished with %d errors", seqID, len(kvErrs))
			fmt.Println(errInfo)
		} else {
			p.log.Errorf("Transaction failed: %v", err)
			return nil, err
		}
		return nil, err
	}

	p.kvs.TransactionBarrier()

	for key := range uniq {
		s := p.kvs.GetValueStatus(key)
		/*results = append(results, KeyVal{
			Key: key,
			Val: s.Value,
		})*/
		results = append(results, Result{
			Key:    key,
			Status: s.GetValue(),
		})
	}

	took := time.Since(t).Round(time.Microsecond * 100)
	p.log.Infof("Transaction #%d successful! (took %v)", seqID, took)

	return results, nil
}

// ListState retrieves running state.
func (p *dispatcher) ListState() (KVPairs, error) {
	p.mu.Lock()
	defer p.mu.Unlock()

	pairs := KVPairs{}
	for _, prefix := range p.kvs.GetRegisteredNBKeyPrefixes() {
		data, err := p.kvs.DumpValuesByKeyPrefix(prefix, kvs.CachedView)
		if err != nil {
			return nil, err
		}
		for _, d := range data {
			//status := p.kvs.GetValueStatus(d.Key)
			pairs[d.Key] = d.Value
		}
	}

	return pairs, nil
}
<commit_msg>fix: Fix error message for duplicate keys (#1659)<commit_after>//  Copyright (c) 2019 Cisco and/or its affiliates.
//
//  Licensed under the Apache License, Version 2.0 (the "License");
//  you may not use this file except in compliance with the License.
//  You may obtain a copy of the License at:
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
//  Unless required by applicable law or agreed to in writing, software
//  distributed under the License is distributed on an "AS IS" BASIS,
//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//  See the License for the specific language governing permissions and
//  limitations under the License.

package orchestrator

import (
	"context"
	"fmt"
	"runtime/trace"
	"sync"
	"time"

	"github.com/golang/protobuf/proto"
	"github.com/pkg/errors"
	"go.ligato.io/cn-infra/v2/logging"

	"go.ligato.io/vpp-agent/v3/pkg/models"
	kvs "go.ligato.io/vpp-agent/v3/plugins/kvscheduler/api"
	"go.ligato.io/vpp-agent/v3/proto/ligato/kvscheduler"
)

// KeyVal associates value with its key.
type KeyVal struct {
	Key string
	Val proto.Message
}

// KVPairs represents key-value pairs.
type KVPairs map[string]proto.Message

type Status = kvscheduler.ValueStatus

type Result struct {
	Key    string
	Status *Status
}

type Dispatcher interface {
	ListData() KVPairs
	PushData(context.Context, []KeyVal) ([]Result, error)
	GetStatus(key string) (*Status, error)
	ListState() (KVPairs, error)
}

type dispatcher struct {
	log logging.Logger
	kvs kvs.KVScheduler
	mu  sync.Mutex
	db  KVStore
}

// ListData retrieves actual data.
func (p *dispatcher) ListData() KVPairs {
	p.mu.Lock()
	defer p.mu.Unlock()

	return p.db.ListAll()
}

func (p *dispatcher) GetStatus(key string) (*Status, error) {
	s := p.kvs.GetValueStatus(key)
	status := s.GetValue()
	if status == nil {
		return nil, errors.Errorf("status for key %q not found", key)
	}
	return status, nil
}

// PushData updates actual data.
func (p *dispatcher) PushData(ctx context.Context, kvPairs []KeyVal) (results []Result, err error) {
	trace.Logf(ctx, "pushData", "%d KV pairs", len(kvPairs))

	// validate key-value pairs
	uniq := make(map[string]struct{})
	for _, kv := range kvPairs {
		if kv.Val != nil {
			// check if given key matches the key generated from value
			if k := models.Key(kv.Val); k != kv.Key {
				return nil, errors.Errorf("given key %q does not match with key generated from value: %q (value: %#v)", kv.Key, k, kv.Val)
			}
		}
		// check if key is unique
		if _, ok := uniq[kv.Key]; ok {
			return nil, errors.Errorf("found multiple key-value pairs with same key: %q", kv.Key)
		}
		uniq[kv.Key] = struct{}{}
	}

	p.mu.Lock()
	defer p.mu.Unlock()

	pr := trace.StartRegion(ctx, "prepare kv data")

	dataSrc, ok := DataSrcFromContext(ctx)
	if !ok {
		dataSrc = "global"
	}

	p.log.Debugf("Push data with %d KV pairs (source: %s)", len(kvPairs), dataSrc)

	txn := p.kvs.StartNBTransaction()

	if typ, _ := kvs.IsResync(ctx); typ == kvs.FullResync {
		trace.Log(ctx, "resyncType", typ.String())
		p.db.Reset(dataSrc)
		for _, kv := range kvPairs {
			if kv.Val == nil {
				p.log.Debugf(" - PUT: %q (skipped nil value for resync)", kv.Key)
				continue
			}
			p.log.Debugf(" - PUT: %q ", kv.Key)
			p.db.Update(dataSrc, kv.Key, kv.Val)
		}
		allPairs := p.db.ListAll()
		p.log.Debugf("will resync %d pairs", len(allPairs))
		for k, v := range allPairs {
			txn.SetValue(k, v)
		}
	} else {
		for _, kv := range kvPairs {
			if kv.Val == nil {
				p.log.Debugf(" - DELETE: %q", kv.Key)
				txn.SetValue(kv.Key, nil)
				p.db.Delete(dataSrc, kv.Key)
			} else {
				p.log.Debugf(" - UPDATE: %q ", kv.Key)
				txn.SetValue(kv.Key, kv.Val)
				p.db.Update(dataSrc, kv.Key, kv.Val)
			}
		}
	}

	pr.End()

	t := time.Now()

	seqID, err := txn.Commit(ctx)
	if err != nil {
		if txErr, ok := err.(*kvs.TransactionError); ok && len(txErr.GetKVErrors()) > 0 {
			kvErrs := txErr.GetKVErrors()
			var errInfo = ""
			for i, kvErr := range kvErrs {
				errInfo += fmt.Sprintf(" - %3d. error (%s) %s - %v\n", i+1, kvErr.TxnOperation, kvErr.Key, kvErr.Error)
			}
			p.log.Errorf("Transaction #%d finished with %d errors", seqID, len(kvErrs))
			fmt.Println(errInfo)
		} else {
			p.log.Errorf("Transaction failed: %v", err)
			return nil, err
		}
		return nil, err
	}

	p.kvs.TransactionBarrier()

	for key := range uniq {
		s := p.kvs.GetValueStatus(key)
		/*results = append(results, KeyVal{
			Key: key,
			Val: s.Value,
		})*/
		results = append(results, Result{
			Key:    key,
			Status: s.GetValue(),
		})
	}

	took := time.Since(t).Round(time.Microsecond * 100)
	p.log.Infof("Transaction #%d successful! (took %v)", seqID, took)

	return results, nil
}

// ListState retrieves running state.
func (p *dispatcher) ListState() (KVPairs, error) {
	p.mu.Lock()
	defer p.mu.Unlock()

	pairs := KVPairs{}
	for _, prefix := range p.kvs.GetRegisteredNBKeyPrefixes() {
		data, err := p.kvs.DumpValuesByKeyPrefix(prefix, kvs.CachedView)
		if err != nil {
			return nil, err
		}
		for _, d := range data {
			//status := p.kvs.GetValueStatus(d.Key)
			pairs[d.Key] = d.Value
		}
	}

	return pairs, nil
}
<|endoftext|>


<commit_before>// Copyright 2013 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package scrape

import (
	"reflect"
	"sync"
	"time"

	"github.com/go-kit/kit/log"
	"github.com/go-kit/kit/log/level"

	"github.com/prometheus/prometheus/config"
	"github.com/prometheus/prometheus/discovery/targetgroup"
	"github.com/prometheus/prometheus/storage"
)

// Appendable returns an Appender.
type Appendable interface {
	Appender() (storage.Appender, error)
}

// NewManager is the Manager constructor
func NewManager(logger log.Logger, app Appendable) *Manager {
	if logger == nil {
		logger = log.NewNopLogger()
	}
	return &Manager{
		append:        app,
		logger:        logger,
		scrapeConfigs: make(map[string]*config.ScrapeConfig),
		scrapePools:   make(map[string]*scrapePool),
		graceShut:     make(chan struct{}),
		triggerReload: make(chan struct{}, 1),
	}
}

// Manager maintains a set of scrape pools and manages start/stop cycles
// when receiving new target groups form the discovery manager.
type Manager struct {
	logger    log.Logger
	append    Appendable
	graceShut chan struct{}

	mtxScrape     sync.Mutex // Guards the fields below.
	scrapeConfigs map[string]*config.ScrapeConfig
	scrapePools   map[string]*scrapePool
	targetSets    map[string][]*targetgroup.Group

	triggerReload chan struct{}
}

// Run receives and saves target set updates and triggers the scraping loops reloading.
// Reloading happens in the background so that it doesn't block receiving targets updates.
func (m *Manager) Run(tsets <-chan map[string][]*targetgroup.Group) error {
	go m.reloader()
	for {
		select {
		case ts := <-tsets:
			m.updateTsets(ts)

			select {
			case m.triggerReload <- struct{}{}:
			default:
			}

		case <-m.graceShut:
			return nil
		}
	}
}

func (m *Manager) reloader() {
	ticker := time.NewTicker(5 * time.Second)
	defer ticker.Stop()

	for {
		select {
		case <-m.graceShut:
			return
		case <-ticker.C:
			select {
			case <-m.triggerReload:
				m.reload()
			case <-m.graceShut:
				return
			}
		}
	}
}

func (m *Manager) reload() {
	m.mtxScrape.Lock()
	var wg sync.WaitGroup
	for setName, groups := range m.targetSets {
		var sp *scrapePool
		existing, ok := m.scrapePools[setName]
		if !ok {
			scrapeConfig, ok := m.scrapeConfigs[setName]
			if !ok {
				level.Error(m.logger).Log("msg", "error reloading target set", "err", "invalid config id:"+setName)
				return
			}
			sp = newScrapePool(scrapeConfig, m.append, log.With(m.logger, "scrape_pool", setName))
			m.scrapePools[setName] = sp
		} else {
			sp = existing
		}

		wg.Add(1)
		// Run the sync in parallel as these take a while and at high load can't catch up.
		go func(sp *scrapePool, groups []*targetgroup.Group) {
			sp.Sync(groups)
			wg.Done()
		}(sp, groups)

	}
	m.mtxScrape.Unlock()
	wg.Wait()
}

// Stop cancels all running scrape pools and blocks until all have exited.
func (m *Manager) Stop() {
	m.mtxScrape.Lock()
	defer m.mtxScrape.Unlock()

	for _, sp := range m.scrapePools {
		sp.stop()
	}
	close(m.graceShut)
}

func (m *Manager) updateTsets(tsets map[string][]*targetgroup.Group) {
	m.mtxScrape.Lock()
	m.targetSets = tsets
	m.mtxScrape.Unlock()
}

// ApplyConfig resets the manager's target providers and job configurations as defined by the new cfg.
func (m *Manager) ApplyConfig(cfg *config.Config) error {
	m.mtxScrape.Lock()
	defer m.mtxScrape.Unlock()

	c := make(map[string]*config.ScrapeConfig)
	for _, scfg := range cfg.ScrapeConfigs {
		c[scfg.JobName] = scfg
	}
	m.scrapeConfigs = c

	// Cleanup and reload pool if config has changed.
	for name, sp := range m.scrapePools {
		if cfg, ok := m.scrapeConfigs[name]; !ok {
			sp.stop()
			delete(m.scrapePools, name)
		} else if !reflect.DeepEqual(sp.config, cfg) {
			sp.reload(cfg)
		}
	}

	return nil
}

// TargetsAll returns active and dropped targets grouped by job_name.
func (m *Manager) TargetsAll() map[string][]*Target {
	m.mtxScrape.Lock()
	defer m.mtxScrape.Unlock()

	targets := make(map[string][]*Target, len(m.scrapePools))
	for tset, sp := range m.scrapePools {
		targets[tset] = append(sp.ActiveTargets(), sp.DroppedTargets()...)

	}
	return targets
}

// TargetsActive returns the active targets currently being scraped.
func (m *Manager) TargetsActive() map[string][]*Target {
	m.mtxScrape.Lock()
	defer m.mtxScrape.Unlock()

	targets := make(map[string][]*Target, len(m.scrapePools))
	for tset, sp := range m.scrapePools {
		targets[tset] = sp.ActiveTargets()
	}
	return targets
}

// TargetsDropped returns the dropped targets during relabelling.
func (m *Manager) TargetsDropped() map[string][]*Target {
	m.mtxScrape.Lock()
	defer m.mtxScrape.Unlock()

	targets := make(map[string][]*Target, len(m.scrapePools))
	for tset, sp := range m.scrapePools {
		targets[tset] = sp.DroppedTargets()
	}
	return targets
}
<commit_msg>fix deadlock in scrape manager (#4894)<commit_after>// Copyright 2013 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package scrape

import (
	"reflect"
	"sync"
	"time"

	"github.com/go-kit/kit/log"
	"github.com/go-kit/kit/log/level"

	"github.com/prometheus/prometheus/config"
	"github.com/prometheus/prometheus/discovery/targetgroup"
	"github.com/prometheus/prometheus/storage"
)

// Appendable returns an Appender.
type Appendable interface {
	Appender() (storage.Appender, error)
}

// NewManager is the Manager constructor
func NewManager(logger log.Logger, app Appendable) *Manager {
	if logger == nil {
		logger = log.NewNopLogger()
	}
	return &Manager{
		append:        app,
		logger:        logger,
		scrapeConfigs: make(map[string]*config.ScrapeConfig),
		scrapePools:   make(map[string]*scrapePool),
		graceShut:     make(chan struct{}),
		triggerReload: make(chan struct{}, 1),
	}
}

// Manager maintains a set of scrape pools and manages start/stop cycles
// when receiving new target groups form the discovery manager.
type Manager struct {
	logger    log.Logger
	append    Appendable
	graceShut chan struct{}

	mtxScrape     sync.Mutex // Guards the fields below.
	scrapeConfigs map[string]*config.ScrapeConfig
	scrapePools   map[string]*scrapePool
	targetSets    map[string][]*targetgroup.Group

	triggerReload chan struct{}
}

// Run receives and saves target set updates and triggers the scraping loops reloading.
// Reloading happens in the background so that it doesn't block receiving targets updates.
func (m *Manager) Run(tsets <-chan map[string][]*targetgroup.Group) error {
	go m.reloader()
	for {
		select {
		case ts := <-tsets:
			m.updateTsets(ts)

			select {
			case m.triggerReload <- struct{}{}:
			default:
			}

		case <-m.graceShut:
			return nil
		}
	}
}

func (m *Manager) reloader() {
	ticker := time.NewTicker(5 * time.Second)
	defer ticker.Stop()

	for {
		select {
		case <-m.graceShut:
			return
		case <-ticker.C:
			select {
			case <-m.triggerReload:
				m.reload()
			case <-m.graceShut:
				return
			}
		}
	}
}

func (m *Manager) reload() {
	m.mtxScrape.Lock()
	var wg sync.WaitGroup
	for setName, groups := range m.targetSets {
		var sp *scrapePool
		existing, ok := m.scrapePools[setName]
		if !ok {
			scrapeConfig, ok := m.scrapeConfigs[setName]
			if !ok {
				level.Error(m.logger).Log("msg", "error reloading target set", "err", "invalid config id:"+setName)
				continue
			}
			sp = newScrapePool(scrapeConfig, m.append, log.With(m.logger, "scrape_pool", setName))
			m.scrapePools[setName] = sp
		} else {
			sp = existing
		}

		wg.Add(1)
		// Run the sync in parallel as these take a while and at high load can't catch up.
		go func(sp *scrapePool, groups []*targetgroup.Group) {
			sp.Sync(groups)
			wg.Done()
		}(sp, groups)

	}
	m.mtxScrape.Unlock()
	wg.Wait()
}

// Stop cancels all running scrape pools and blocks until all have exited.
func (m *Manager) Stop() {
	m.mtxScrape.Lock()
	defer m.mtxScrape.Unlock()

	for _, sp := range m.scrapePools {
		sp.stop()
	}
	close(m.graceShut)
}

func (m *Manager) updateTsets(tsets map[string][]*targetgroup.Group) {
	m.mtxScrape.Lock()
	m.targetSets = tsets
	m.mtxScrape.Unlock()
}

// ApplyConfig resets the manager's target providers and job configurations as defined by the new cfg.
func (m *Manager) ApplyConfig(cfg *config.Config) error {
	m.mtxScrape.Lock()
	defer m.mtxScrape.Unlock()

	c := make(map[string]*config.ScrapeConfig)
	for _, scfg := range cfg.ScrapeConfigs {
		c[scfg.JobName] = scfg
	}
	m.scrapeConfigs = c

	// Cleanup and reload pool if config has changed.
	for name, sp := range m.scrapePools {
		if cfg, ok := m.scrapeConfigs[name]; !ok {
			sp.stop()
			delete(m.scrapePools, name)
		} else if !reflect.DeepEqual(sp.config, cfg) {
			sp.reload(cfg)
		}
	}

	return nil
}

// TargetsAll returns active and dropped targets grouped by job_name.
func (m *Manager) TargetsAll() map[string][]*Target {
	m.mtxScrape.Lock()
	defer m.mtxScrape.Unlock()

	targets := make(map[string][]*Target, len(m.scrapePools))
	for tset, sp := range m.scrapePools {
		targets[tset] = append(sp.ActiveTargets(), sp.DroppedTargets()...)

	}
	return targets
}

// TargetsActive returns the active targets currently being scraped.
func (m *Manager) TargetsActive() map[string][]*Target {
	m.mtxScrape.Lock()
	defer m.mtxScrape.Unlock()

	targets := make(map[string][]*Target, len(m.scrapePools))
	for tset, sp := range m.scrapePools {
		targets[tset] = sp.ActiveTargets()
	}
	return targets
}

// TargetsDropped returns the dropped targets during relabelling.
func (m *Manager) TargetsDropped() map[string][]*Target {
	m.mtxScrape.Lock()
	defer m.mtxScrape.Unlock()

	targets := make(map[string][]*Target, len(m.scrapePools))
	for tset, sp := range m.scrapePools {
		targets[tset] = sp.DroppedTargets()
	}
	return targets
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"github.com/dim13/gold"
	"log"
	"net/http"
	"net/url"
	"sort"
	"strconv"
	"time"
)

type Page struct {
	Config   *gold.Config
	Title    string
	Articles gold.Articles
	Article  *gold.Article
	Error    error
	PrevPage int
	NextPage int
	TagCloud gold.TagCloud
	Match    []string
	Year     int
	Month    time.Month
	Archive  []Archive
}

type ByYear []Archive
type Archive struct {
	Year  int
	Count int
	Month []Month
}

type ByMonth []Month
type Month struct {
	Month    time.Month
	Count    int
	Articles gold.Articles
}

func (m ByMonth) Len() int           { return len(m) }
func (m ByMonth) Swap(i, j int)      { m[i], m[j] = m[j], m[i] }
func (m ByMonth) Less(i, j int) bool { return m[i].Month < m[j].Month }

func (y ByYear) Len() int           { return len(y) }
func (y ByYear) Swap(i, j int)      { y[i], y[j] = y[j], y[i] }
func (y ByYear) Less(i, j int) bool { return y[i].Year < y[j].Year }

func (m Month) IntMonth() string {
	return fmt.Sprintf("%.2d", int(m.Month))
}

func atoiMust(s string) int {
	i, err := strconv.Atoi(s)
	if err != nil {
		log.Fatal(err)
	}
	return i
}

func parsePage(u url.URL) int {
	if page, ok := u.Query()["page"]; ok {
		if pg, err := strconv.Atoi(page[0]); err == nil {
			return pg
		}
	}
	return 1
}

func (p *Page) StoreMatch(s []string) { p.Match = s }

func (p *Page) MakeArchive() {
	ym := data.Articles.Enabled().YearMap()
	for y, v := range ym {
		year := Archive{
			Year:  y,
			Count: len(v),
		}
		mm := v.MonthMap()
		if p.Year == y {
			for m, v := range mm {
				month := Month{
					Month: time.Month(m),
					Count: len(v),
				}
				if p.Month == time.Month(m) {
					month.Articles = v
				}
				year.Month = append(year.Month, month)
			}
			sort.Sort(ByMonth(year.Month))
		}
		p.Archive = append(p.Archive, year)
	}
	sort.Sort(sort.Reverse(ByYear(p.Archive)))
}

func (p Page) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	pg := parsePage(*r.URL)
	app := conf.Blog.ArticlesPerPage
	p.Articles, p.NextPage, p.PrevPage = p.Articles.Page(pg, app)
	p.TagCloud = data.Articles.TagCloud()
	p.Config = conf
	if p.Year == 0 && p.Month == 0 {
		p.Month = time.Now().Month()
	}
	if p.Year == 0 {
		p.Year = time.Now().Year()
	}
	p.MakeArchive()

	err := tmpl.ExecuteTemplate(w, "index.tmpl", p)
	if err != nil {
		log.Fatal(err)
	}
}

type TagPage struct{ Page }

func (p TagPage) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	s := p.Page.Match[0]
	p.Articles = data.Articles.Tag(s)
	p.Title = fmt.Sprint(conf.Blog.Title, " - ", s)
	p.Page.ServeHTTP(w, r)
}

type IndexPage struct{ Page }

func (p IndexPage) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	p.Articles = data.Articles.Enabled()
	p.Title = conf.Blog.Title
	p.Page.ServeHTTP(w, r)
}

type SlugPage struct{ Page }

func (p SlugPage) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	a, err := data.Articles.Find(p.Match[0])
	if err != nil {
		http.NotFound(w, r)
		return
	}
	p.Title = a.Title
	p.Articles = gold.Articles{a}
	p.Year = a.Year()
	p.Month = a.Month()
	p.Page.ServeHTTP(w, r)
}

type YearPage struct{ Page }

func (p YearPage) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	p.Year = atoiMust(p.Match[0])
	p.Articles = data.Articles.Year(p.Year)
	p.Title = fmt.Sprint(conf.Blog.Title, " - ", p.Year)
	p.Page.ServeHTTP(w, r)
}

type MonthPage struct{ Page }

func (p MonthPage) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	p.Year = atoiMust(p.Match[0])
	p.Month = time.Month(atoiMust(p.Match[1]))
	p.Articles = data.Articles.Year(p.Year).Month(p.Month)
	p.Title = fmt.Sprint(conf.Blog.Title, " - ", p.Year, p.Month)
	p.Page.ServeHTTP(w, r)
}
<commit_msg>cleanup<commit_after>package main

import (
	"fmt"
	"github.com/dim13/gold"
	"log"
	"net/http"
	"net/url"
	"sort"
	"strconv"
	"time"
)

type Page struct {
	Config   *gold.Config
	Title    string
	Articles gold.Articles
	Article  *gold.Article
	Error    error
	PrevPage int
	NextPage int
	TagCloud gold.TagCloud
	Match    []string
	Year     int
	Month    time.Month
	Archive  []Archive
}

type ByYear []Archive
type Archive struct {
	Year  int
	Count int
	Month []Month
}

type ByMonth []Month
type Month struct {
	Month    time.Month
	Count    int
	Articles gold.Articles
}

func (m ByMonth) Len() int           { return len(m) }
func (m ByMonth) Swap(i, j int)      { m[i], m[j] = m[j], m[i] }
func (m ByMonth) Less(i, j int) bool { return m[i].Month < m[j].Month }

func (y ByYear) Len() int           { return len(y) }
func (y ByYear) Swap(i, j int)      { y[i], y[j] = y[j], y[i] }
func (y ByYear) Less(i, j int) bool { return y[i].Year < y[j].Year }

func (m Month) IntMonth() string {
	return fmt.Sprintf("%.2d", int(m.Month))
}

func atoiMust(s string) int {
	i, err := strconv.Atoi(s)
	if err != nil {
		log.Fatal(err)
	}
	return i
}

func parsePage(u url.URL) int {
	if page, ok := u.Query()["page"]; ok {
		if pg, err := strconv.Atoi(page[0]); err == nil {
			return pg
		}
	}
	return 1
}

func (p *Page) StoreMatch(s []string) { p.Match = s }

func (p *Page) MakeArchive() {
	for y, v := range data.Articles.Enabled().YearMap() {
		year := Archive{
			Year:  y,
			Count: len(v),
		}
		if p.Year == y {
			for m, v := range v.MonthMap() {
				month := Month{
					Month: time.Month(m),
					Count: len(v),
				}
				if p.Month == time.Month(m) {
					month.Articles = v
				}
				year.Month = append(year.Month, month)
			}
			sort.Sort(ByMonth(year.Month))
		}
		p.Archive = append(p.Archive, year)
	}
	sort.Sort(sort.Reverse(ByYear(p.Archive)))
}

func (p Page) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	pg := parsePage(*r.URL)
	app := conf.Blog.ArticlesPerPage
	p.Articles, p.NextPage, p.PrevPage = p.Articles.Page(pg, app)
	p.TagCloud = data.Articles.TagCloud()
	p.Config = conf
	if p.Year == 0 && p.Month == 0 {
		p.Month = time.Now().Month()
	}
	if p.Year == 0 {
		p.Year = time.Now().Year()
	}
	p.MakeArchive()

	err := tmpl.ExecuteTemplate(w, "index.tmpl", p)
	if err != nil {
		log.Fatal(err)
	}
}

type TagPage struct{ Page }

func (p TagPage) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	s := p.Page.Match[0]
	p.Articles = data.Articles.Tag(s)
	p.Title = fmt.Sprint(conf.Blog.Title, " - ", s)
	p.Page.ServeHTTP(w, r)
}

type IndexPage struct{ Page }

func (p IndexPage) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	p.Articles = data.Articles.Enabled()
	p.Title = conf.Blog.Title
	p.Page.ServeHTTP(w, r)
}

type SlugPage struct{ Page }

func (p SlugPage) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	a, err := data.Articles.Find(p.Match[0])
	if err != nil {
		http.NotFound(w, r)
		return
	}
	p.Title = a.Title
	p.Articles = gold.Articles{a}
	p.Year = a.Year()
	p.Month = a.Month()
	p.Page.ServeHTTP(w, r)
}

type YearPage struct{ Page }

func (p YearPage) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	p.Year = atoiMust(p.Match[0])
	p.Articles = data.Articles.Year(p.Year)
	p.Title = fmt.Sprint(conf.Blog.Title, " - ", p.Year)
	p.Page.ServeHTTP(w, r)
}

type MonthPage struct{ Page }

func (p MonthPage) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	p.Year = atoiMust(p.Match[0])
	p.Month = time.Month(atoiMust(p.Match[1]))
	p.Articles = data.Articles.Year(p.Year).Month(p.Month)
	p.Title = fmt.Sprint(conf.Blog.Title, " - ", p.Year, p.Month)
	p.Page.ServeHTTP(w, r)
}
<|endoftext|>


<commit_before>/*
	Copyright (C) 2016-2017 vLife Systems Ltd <http://vlifesystems.com>
	This file is part of rhkit.

	rhkit is free software: you can redistribute it and/or modify
	it under the terms of the GNU General Public License as published by
	the Free Software Foundation, either version 3 of the License, or
	(at your option) any later version.

	rhkit is distributed in the hope that it will be useful,
	but WITHOUT ANY WARRANTY; without even the implied warranty of
	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
	GNU General Public License for more details.

	You should have received a copy of the GNU General Public License
	along with rhkit; see the file COPYING.  If not, see
	<http://www.gnu.org/licenses/>.
*/

// package rhkit is used to find rules in a Dataset to satisfy user defined
// goals
package rhkit

import (
	"errors"
	"github.com/vlifesystems/rhkit/assessment"
	"github.com/vlifesystems/rhkit/description"
	"github.com/vlifesystems/rhkit/experiment"
	"github.com/vlifesystems/rhkit/rule"
)

var ErrNoRulesGenerated = errors.New("no rules generated")

type DescribingError struct {
	Err error
}

func (e DescribingError) Error() string {
	return "problem describing dataset: " + e.Err.Error()
}

type AssessError struct {
	Err error
}

func (e AssessError) Error() string {
	return "problem assessing rules: " + e.Err.Error()
}

type MergeError struct {
	Err error
}

func (e MergeError) Error() string {
	return "problem merging rules: " + e.Err.Error()
}

// Process processes an Experiment and returns an assessment
func Process(
	experiment *experiment.Experiment,
	maxNumRules int,
) (*assessment.Assessment, error) {
	var ass *assessment.Assessment
	var newAss *assessment.Assessment
	var err error
	fieldDescriptions, err := description.DescribeDataset(experiment.Dataset)
	if err != nil {
		return nil, DescribingError{Err: err}
	}
	rules := rule.Generate(
		fieldDescriptions,
		experiment.RuleFields,
		experiment.RuleComplexity,
	)
	if len(rules) < 2 {
		return nil, ErrNoRulesGenerated
	}

	ass, err = assessment.AssessRules(rules, experiment)
	if err != nil {
		return nil, AssessError{Err: err}
	}

	ass.Sort(experiment.SortOrder)
	ass.Refine()
	rules = ass.Rules()

	tweakableRules := rule.Tweak(
		1,
		rules,
		fieldDescriptions,
	)

	newAss, err = assessment.AssessRules(tweakableRules, experiment)
	if err != nil {
		return nil, AssessError{Err: err}
	}

	ass, err = ass.Merge(newAss)
	if err != nil {
		return nil, MergeError{Err: err}
	}
	ass.Sort(experiment.SortOrder)
	ass.Refine()

	rules = ass.Rules()
	reducedDPRules := rule.ReduceDP(rules)

	newAss, err = assessment.AssessRules(reducedDPRules, experiment)
	if err != nil {
		return nil, AssessError{Err: err}
	}

	ass, err = ass.Merge(newAss)
	if err != nil {
		return nil, MergeError{Err: err}
	}
	ass.Sort(experiment.SortOrder)
	ass.Refine()

	numRulesToCombine := 50
	bestNonCombinedRules := ass.Rules(numRulesToCombine)
	combinedRules := rule.Combine(bestNonCombinedRules)

	newAss, err = assessment.AssessRules(combinedRules, experiment)
	if err != nil {
		return nil, AssessError{Err: err}
	}

	ass, err = ass.Merge(newAss)
	if err != nil {
		return nil, MergeError{Err: err}
	}
	ass.Sort(experiment.SortOrder)
	ass.Refine()

	return ass.TruncateRuleAssessments(maxNumRules), nil
}
<commit_msg>s/DescribingError/DescribeError/<commit_after>/*
	Copyright (C) 2016-2017 vLife Systems Ltd <http://vlifesystems.com>
	This file is part of rhkit.

	rhkit is free software: you can redistribute it and/or modify
	it under the terms of the GNU General Public License as published by
	the Free Software Foundation, either version 3 of the License, or
	(at your option) any later version.

	rhkit is distributed in the hope that it will be useful,
	but WITHOUT ANY WARRANTY; without even the implied warranty of
	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
	GNU General Public License for more details.

	You should have received a copy of the GNU General Public License
	along with rhkit; see the file COPYING.  If not, see
	<http://www.gnu.org/licenses/>.
*/

// package rhkit is used to find rules in a Dataset to satisfy user defined
// goals
package rhkit

import (
	"errors"
	"github.com/vlifesystems/rhkit/assessment"
	"github.com/vlifesystems/rhkit/description"
	"github.com/vlifesystems/rhkit/experiment"
	"github.com/vlifesystems/rhkit/rule"
)

var ErrNoRulesGenerated = errors.New("no rules generated")

// DescribeError indicates an error describing a Dataset
type DescribeError struct {
	Err error
}

func (e DescribeError) Error() string {
	return "problem describing dataset: " + e.Err.Error()
}

type AssessError struct {
	Err error
}

func (e AssessError) Error() string {
	return "problem assessing rules: " + e.Err.Error()
}

type MergeError struct {
	Err error
}

func (e MergeError) Error() string {
	return "problem merging rules: " + e.Err.Error()
}

// Process processes an Experiment and returns an assessment
func Process(
	experiment *experiment.Experiment,
	maxNumRules int,
) (*assessment.Assessment, error) {
	var ass *assessment.Assessment
	var newAss *assessment.Assessment
	var err error
	fieldDescriptions, err := description.DescribeDataset(experiment.Dataset)
	if err != nil {
		return nil, DescribeError{Err: err}
	}
	rules := rule.Generate(
		fieldDescriptions,
		experiment.RuleFields,
		experiment.RuleComplexity,
	)
	if len(rules) < 2 {
		return nil, ErrNoRulesGenerated
	}

	ass, err = assessment.AssessRules(rules, experiment)
	if err != nil {
		return nil, AssessError{Err: err}
	}

	ass.Sort(experiment.SortOrder)
	ass.Refine()
	rules = ass.Rules()

	tweakableRules := rule.Tweak(
		1,
		rules,
		fieldDescriptions,
	)

	newAss, err = assessment.AssessRules(tweakableRules, experiment)
	if err != nil {
		return nil, AssessError{Err: err}
	}

	ass, err = ass.Merge(newAss)
	if err != nil {
		return nil, MergeError{Err: err}
	}
	ass.Sort(experiment.SortOrder)
	ass.Refine()

	rules = ass.Rules()
	reducedDPRules := rule.ReduceDP(rules)

	newAss, err = assessment.AssessRules(reducedDPRules, experiment)
	if err != nil {
		return nil, AssessError{Err: err}
	}

	ass, err = ass.Merge(newAss)
	if err != nil {
		return nil, MergeError{Err: err}
	}
	ass.Sort(experiment.SortOrder)
	ass.Refine()

	numRulesToCombine := 50
	bestNonCombinedRules := ass.Rules(numRulesToCombine)
	combinedRules := rule.Combine(bestNonCombinedRules)

	newAss, err = assessment.AssessRules(combinedRules, experiment)
	if err != nil {
		return nil, AssessError{Err: err}
	}

	ass, err = ass.Merge(newAss)
	if err != nil {
		return nil, MergeError{Err: err}
	}
	ass.Sort(experiment.SortOrder)
	ass.Refine()

	return ass.TruncateRuleAssessments(maxNumRules), nil
}
<|endoftext|>


<commit_before>package main

import (
	"errors"
	"fmt"
	"log"
	"strings"
	"time"
)

// Match represents a game being played
type Match struct {
	Players    []Player    `json:"players"`
	Judges     []Judge     `json:"judges"`
	Kind       string      `json:"kind"`
	Index      int         `json:"index"`
	Started    time.Time   `json:"started"`
	Ended      time.Time   `json:"ended"`
	Tournament *Tournament `json:"-"`
}

// NewMatch creates a new Match for usage!
func NewMatch(t *Tournament, index int, kind string) *Match {
	m := Match{
		Index:      index,
		Kind:       kind,
		Tournament: t,
	}
	m.Prefill()
	return &m
}

func (m *Match) String() string {
	var tempo string
	var name string

	if !m.IsStarted() {
		tempo = "not started"
	} else if m.IsEnded() {
		tempo = "ended"
	} else {
		tempo = "playing"
	}

	if m.Kind == "final" {
		name = "Final"
	} else {
		name = fmt.Sprintf("%s %d", strings.Title(m.Kind), m.Index+1)
	}

	names := make([]string, 0, len(m.Players))
	for _, p := range m.Players {
		names = append(names, p.Name)
	}

	return fmt.Sprintf(
		"<%s: %s - %s>",
		name,
		strings.Join(names, " / "),
		tempo,
	)
}

// Title returns a title string
func (m *Match) Title() string {
	l := 2
	if m.Kind == "final" {
		return "Final"
	} else if m.Kind == "tryout" {
		l = len(m.Tournament.Tryouts)
	}

	out := fmt.Sprintf(
		"%s %d/%d",
		strings.Title(m.Kind),
		m.Index+1,
		l,
	)
	return out
}

// URL builds the URL to the match
func (m *Match) URL() string {
	out := fmt.Sprintf(
		"/%s/%s/%d",
		m.Tournament.ID,
		m.Kind,
		m.Index,
	)
	return out

}

// AddPlayer adds a player to the match
func (m *Match) AddPlayer(p Player) error {
	if m.ActualPlayers() == 4 {
		return errors.New("cannot add fifth player")
	}

	if len(m.Players) == 4 {
		// Loop through the players and replace the first prefill player that can be found with
		// the actual player.
		for i, o := range m.Players {
			if o.IsPrefill() {
				m.Players[i] = p
				break
			}
		}
	} else {
		m.Players = append(m.Players, p)
	}

	p.Match = m

	return nil
}

// Prefill fills remaining player slots with nil players
func (m *Match) Prefill() error {
	for i := len(m.Players); i < 4; i++ {
		err := m.AddPlayer(Player{})
		if err != nil {
			log.Fatal(err)
		}

	}
	return nil
}

// ActualPlayers returns the number of actual players set in the match
func (m *Match) ActualPlayers() int {
	ret := 0
	for _, p := range m.Players {
		if !p.IsPrefill() {
			ret++
		}
	}

	return ret
}

// Start starts the match
func (m *Match) Start() error {
	if !m.Started.IsZero() {
		return errors.New("match already started")
	}

	// If there are not four players in the match, we need to populate
	// the match with runnerups from the tournament
	if m.ActualPlayers() != 4 {
		m.Tournament.PopulateRunnerups(m)
	}

	for i := range m.Players {
		m.Players[i].Reset()
		m.Players[i].Match = m
	}

	m.Started = time.Now()
	if m.Tournament != nil {
		m.Tournament.Persist()
	}
	return nil
}

// End signals that the match has ended
//
// It is also the place that moves players into either the Runnerup bracket
// or into their place in the semis.
func (m *Match) End() error {
	if !m.Ended.IsZero() {
		return errors.New("match already ended")
	}

	// Give the winner one last shot
	ps := ByScore(m.Players)
	winner := ps[0].Name
	for i, p := range m.Players {
		if p.Name == winner {
			m.Players[i].AddShot()
			break
		}
	}

	m.Ended = time.Now()
	// TODO: This is for the tests not to break. Fix by setting up better tests.
	if m.Tournament != nil {
		if m.Kind == "final" {
			m.Tournament.AwardMedals(m)
		} else {
			m.Tournament.MovePlayers(m)
		}

		m.Tournament.Persist()
	}
	return nil
}

// IsStarted returns boolean whether the match has started or not
func (m *Match) IsStarted() bool {
	return !m.Started.IsZero()
}

// IsEnded returns boolean whether the match has ended or not
func (m *Match) IsEnded() bool {
	return !m.Ended.IsZero()
}

// CanEnd returns boolean whether the match can be ended or not
func (m *Match) CanEnd() bool {
	for _, p := range m.Players {
		if p.Kills >= m.Length() {
			return true
		}
	}
	return false
}

// IsOpen returns boolean the match can be controlled or not
func (m *Match) IsOpen() bool {
	return !m.IsEnded()
}

// Length returns the length of the match
func (m *Match) Length() int {
	if m.Kind == "final" {
		return 20
	}
	return 10
}
<commit_msg>Add resetting of player before adding to next match<commit_after>package main

import (
	"errors"
	"fmt"
	"log"
	"strings"
	"time"
)

// Match represents a game being played
type Match struct {
	Players    []Player    `json:"players"`
	Judges     []Judge     `json:"judges"`
	Kind       string      `json:"kind"`
	Index      int         `json:"index"`
	Started    time.Time   `json:"started"`
	Ended      time.Time   `json:"ended"`
	Tournament *Tournament `json:"-"`
}

// NewMatch creates a new Match for usage!
func NewMatch(t *Tournament, index int, kind string) *Match {
	m := Match{
		Index:      index,
		Kind:       kind,
		Tournament: t,
	}
	m.Prefill()
	return &m
}

func (m *Match) String() string {
	var tempo string
	var name string

	if !m.IsStarted() {
		tempo = "not started"
	} else if m.IsEnded() {
		tempo = "ended"
	} else {
		tempo = "playing"
	}

	if m.Kind == "final" {
		name = "Final"
	} else {
		name = fmt.Sprintf("%s %d", strings.Title(m.Kind), m.Index+1)
	}

	names := make([]string, 0, len(m.Players))
	for _, p := range m.Players {
		names = append(names, p.Name)
	}

	return fmt.Sprintf(
		"<%s: %s - %s>",
		name,
		strings.Join(names, " / "),
		tempo,
	)
}

// Title returns a title string
func (m *Match) Title() string {
	l := 2
	if m.Kind == "final" {
		return "Final"
	} else if m.Kind == "tryout" {
		l = len(m.Tournament.Tryouts)
	}

	out := fmt.Sprintf(
		"%s %d/%d",
		strings.Title(m.Kind),
		m.Index+1,
		l,
	)
	return out
}

// URL builds the URL to the match
func (m *Match) URL() string {
	out := fmt.Sprintf(
		"/%s/%s/%d",
		m.Tournament.ID,
		m.Kind,
		m.Index,
	)
	return out

}

// AddPlayer adds a player to the match
func (m *Match) AddPlayer(p Player) error {
	if m.ActualPlayers() == 4 {
		return errors.New("cannot add fifth player")
	}

	p.Reset()
	p.Match = m

	if len(m.Players) == 4 {
		// Loop through the players and replace the first prefill player that can be found with
		// the actual player.
		for i, o := range m.Players {
			if o.IsPrefill() {
				m.Players[i] = p
				break
			}
		}
	} else {
		m.Players = append(m.Players, p)
	}

	return nil
}

// Prefill fills remaining player slots with nil players
func (m *Match) Prefill() error {
	for i := len(m.Players); i < 4; i++ {
		err := m.AddPlayer(Player{})
		if err != nil {
			log.Fatal(err)
		}

	}
	return nil
}

// ActualPlayers returns the number of actual players set in the match
func (m *Match) ActualPlayers() int {
	ret := 0
	for _, p := range m.Players {
		if !p.IsPrefill() {
			ret++
		}
	}

	return ret
}

// Start starts the match
func (m *Match) Start() error {
	if !m.Started.IsZero() {
		return errors.New("match already started")
	}

	// If there are not four players in the match, we need to populate
	// the match with runnerups from the tournament
	if m.ActualPlayers() != 4 {
		m.Tournament.PopulateRunnerups(m)
	}

	for i := range m.Players {
		m.Players[i].Reset()
		m.Players[i].Match = m
	}

	m.Started = time.Now()
	if m.Tournament != nil {
		m.Tournament.Persist()
	}
	return nil
}

// End signals that the match has ended
//
// It is also the place that moves players into either the Runnerup bracket
// or into their place in the semis.
func (m *Match) End() error {
	if !m.Ended.IsZero() {
		return errors.New("match already ended")
	}

	// Give the winner one last shot
	ps := ByScore(m.Players)
	winner := ps[0].Name
	for i, p := range m.Players {
		if p.Name == winner {
			m.Players[i].AddShot()
			break
		}
	}

	m.Ended = time.Now()
	// TODO: This is for the tests not to break. Fix by setting up better tests.
	if m.Tournament != nil {
		if m.Kind == "final" {
			m.Tournament.AwardMedals(m)
		} else {
			m.Tournament.MovePlayers(m)
		}

		m.Tournament.Persist()
	}
	return nil
}

// IsStarted returns boolean whether the match has started or not
func (m *Match) IsStarted() bool {
	return !m.Started.IsZero()
}

// IsEnded returns boolean whether the match has ended or not
func (m *Match) IsEnded() bool {
	return !m.Ended.IsZero()
}

// CanEnd returns boolean whether the match can be ended or not
func (m *Match) CanEnd() bool {
	for _, p := range m.Players {
		if p.Kills >= m.Length() {
			return true
		}
	}
	return false
}

// IsOpen returns boolean the match can be controlled or not
func (m *Match) IsOpen() bool {
	return !m.IsEnded()
}

// Length returns the length of the match
func (m *Match) Length() int {
	if m.Kind == "final" {
		return 20
	}
	return 10
}
<|endoftext|>


<commit_before>package collectors

import "fmt"
import "regexp"
import "strings"
import "github.com/josegonzalez/go-radixurl"
import "github.com/josegonzalez/metricsd/mappings"
import "github.com/josegonzalez/metricsd/structs"
import "github.com/josegonzalez/metricsd/utils"
import "github.com/vaughan0/go-ini"

type RedisCollector struct {
	enabled bool
	url     string
}

func (this *RedisCollector) Enabled() bool {
	return this.enabled
}

func (this *RedisCollector) State(state bool) {
	this.enabled = state
}

func (this *RedisCollector) Setup(conf ini.File) {
	this.State(true)

	useRedisUrl, ok := conf.Get("RedisCollector", "url")
	if ok {
		this.url = useRedisUrl
	} else {
		this.url = "redis://127.0.0.1:6379/0"
	}
}

func (this *RedisCollector) Report() (structs.MetricSlice, error) {
	var report structs.MetricSlice
	data, _ := this.collect()

	if data != nil {
		// TODO: _ is a prefix
		for _, values := range data {
			for k, v := range values {
				metric := structs.BuildMetric("RedisCollector", "redis", "gauge", k, v, structs.FieldsMap{
					"raw_key":   k,
					"raw_value": v,
				})
				report = append(report, metric)
			}
		}
	}

	return report, nil
}

func (this *RedisCollector) collect() (map[string]mappings.MetricMap, error) {
	c, err := radixurl.ConnectToURL(this.url)
	if err != nil {
		return nil, err
	}
	defer c.Close()

	r := c.Cmd("info")
	if r.Err != nil {
		return nil, r.Err
	}

	s, err := r.Str()
	if err != nil {
		return nil, err
	}

	dbRegexp := regexp.MustCompile("^db(\\d+):keys=(\\d+),expires=(\\d+)")

	redisMapping := map[string]mappings.MetricMap{}
	values := map[string]string{}

	for _, line := range strings.Split(s, "\n") {
		line = strings.TrimSpace(line)
		if strings.HasPrefix(line, "#") || len(line) < 2 {
			continue
		}
		res := dbRegexp.FindStringSubmatch(line)
		if len(res) == 4 {
			redisMapping[fmt.Sprintf("db%d", res[1])] = mappings.MetricMap{
				"expires": utils.ParseInt64(res[2]),
				"keys":    utils.ParseInt64(res[3]),
			}
			continue
		}
		chunks := strings.Split(line, ":")
		if len(chunks) > 1 {
			values[chunks[0]] = strings.Join(chunks[1:], ":")
		}
	}

	// redisMapping["UptimeInSeconds", parseInt64(values["uptime_in_seconds"]))
	redisMapping["clients"] = mappings.MetricMap{
		"biggest_input_buf":   values["client_biggest_input_buf"],
		"blocked":             values["blocked_clients"],
		"connected":           values["connected_clients"],
		"longest_output_list": values["client_longest_output_list"],
	}

	// TODO
	// 'cpu.parent.sys': 'used_cpu_sys',
	// 'cpu.children.sys': 'used_cpu_sys_children',
	// 'cpu.parent.user': 'used_cpu_user',
	// 'cpu.children.user': 'used_cpu_user_children',

	redisMapping["hash_max_zipmap"] = mappings.MetricMap{
		"entries": values["hash_max_zipmap_entries"],
		"value":   values["hash_max_zipmap_value"],
	}
	redisMapping["keys"] = mappings.MetricMap{
		"evicted": values["evicted_keys"],
		"expired": values["expired_keys"],
	}
	redisMapping["keyspace"] = mappings.MetricMap{
		"hits":   values["keyspace_hits"],
		"misses": values["keyspace_misses"],
	}
	redisMapping["last_save"] = mappings.MetricMap{
		"changes_since": values["changes_since_last_save"],
		"time":          values["last_save_time"],
	}
	redisMapping["memory"] = mappings.MetricMap{
		"internal_view":       values["used_memory"],
		"external_view":       values["used_memory_rss"],
		"fragmentation_ratio": values["mem_fragmentation_ratio"],
	}
	redisMapping["process"] = mappings.MetricMap{
		"commands_processed":   values["total_commands_processed"],
		"connections_received": values["total_connections_received"],
		"uptime":               values["uptime_in_seconds"],
	}
	redisMapping["pubsub"] = mappings.MetricMap{
		"channels": values["pubsub_channels"],
		"patterns": values["pubsub_patterns"],
	}
	redisMapping["slaves"] = mappings.MetricMap{
		"connected": values["connected_slaves"],
		"last_io":   values["master_last_io_seconds_ago"],
	}

	return redisMapping, nil
}
<commit_msg>Add all missing redis metrics<commit_after>package collectors

import "fmt"
import "regexp"
import "strings"
import "github.com/josegonzalez/go-radixurl"
import "github.com/josegonzalez/metricsd/mappings"
import "github.com/josegonzalez/metricsd/structs"
import "github.com/josegonzalez/metricsd/utils"
import "github.com/vaughan0/go-ini"

type RedisCollector struct {
	enabled bool
	url     string
}

func (this *RedisCollector) Enabled() bool {
	return this.enabled
}

func (this *RedisCollector) State(state bool) {
	this.enabled = state
}

func (this *RedisCollector) Setup(conf ini.File) {
	this.State(true)

	useRedisUrl, ok := conf.Get("RedisCollector", "url")
	if ok {
		this.url = useRedisUrl
	} else {
		this.url = "redis://127.0.0.1:6379/0"
	}
}

func (this *RedisCollector) Report() (structs.MetricSlice, error) {
	return this.collect()
}

func (this *RedisCollector) collect() (structs.MetricSlice, error) {
	c, err := radixurl.ConnectToURL(this.url)
	if err != nil {
		return nil, err
	}
	defer c.Close()

	r := c.Cmd("info")
	if r.Err != nil {
		return nil, r.Err
	}

	s, err := r.Str()
	if err != nil {
		return nil, err
	}

	dbRegexp := regexp.MustCompile("^db(\\d+):keys=(\\d+),expires=(\\d+)")

	redisMapping := map[string]mappings.MetricMap{}
	values := map[string]string{}

	for _, line := range strings.Split(s, "\n") {
		line = strings.TrimSpace(line)
		if strings.HasPrefix(line, "#") || len(line) < 2 {
			continue
		}
		res := dbRegexp.FindStringSubmatch(line)
		if len(res) == 4 {
			redisMapping[fmt.Sprintf("db%s", res[1])] = mappings.MetricMap{
				"expires": utils.ParseInt64(res[2]),
				"keys":    utils.ParseInt64(res[3]),
			}
			continue
		}
		chunks := strings.Split(line, ":")
		if len(chunks) > 1 {
			values[chunks[0]] = strings.Join(chunks[1:], ":")
		}
	}

	redisMapping = this.processValues(redisMapping, values)

	var report structs.MetricSlice
	for prefix, values := range redisMapping {
		for k, v := range values {
			metric := structs.BuildMetric("RedisCollector", "redis", "gauge", k, v, structs.FieldsMap{
				"raw_key":   k,
				"raw_value": v,
			})
			metric.Path = fmt.Sprintf("redis.%s", prefix)
			report = append(report, metric)
		}
	}

	metric := structs.BuildMetric("RedisCollector", "redis", "gauge", "sys", values["used_cpu_sys"], structs.FieldsMap{
		"raw_key":   "sys",
		"raw_value": values["used_cpu_sys"],
	})
	metric.Path = "redis.cpu.parent"
	report = append(report, metric)

	metric = structs.BuildMetric("RedisCollector", "redis", "gauge", "sys", values["used_cpu_sys_children"], structs.FieldsMap{
		"raw_key":   "sys",
		"raw_value": values["used_cpu_sys_children"],
	})
	metric.Path = "redis.cpu.children"
	report = append(report, metric)

	metric = structs.BuildMetric("RedisCollector", "redis", "gauge", "user", values["used_cpu_user"], structs.FieldsMap{
		"raw_key":   "user",
		"raw_value": values["used_cpu_user"],
	})
	metric.Path = "redis.cpu.parent"
	report = append(report, metric)

	metric = structs.BuildMetric("RedisCollector", "redis", "gauge", "user", values["used_cpu_user_children"], structs.FieldsMap{
		"raw_key":   "user",
		"raw_value": values["used_cpu_user_children"],
	})
	metric.Path = "redis.cpu.children"
	report = append(report, metric)

	return report, nil
}

func (this *RedisCollector) processValues(redisMapping map[string]mappings.MetricMap, values map[string]string) map[string]mappings.MetricMap {
	redisMapping["clients"] = mappings.MetricMap{
		"biggest_input_buf":   values["client_biggest_input_buf"],
		"blocked":             values["blocked_clients"],
		"connected":           values["connected_clients"],
		"longest_output_list": values["client_longest_output_list"],
	}

	redisMapping["hash_max_zipmap"] = mappings.MetricMap{
		"entries": values["hash_max_zipmap_entries"],
		"value":   values["hash_max_zipmap_value"],
	}
	redisMapping["keys"] = mappings.MetricMap{
		"evicted": values["evicted_keys"],
		"expired": values["expired_keys"],
	}
	redisMapping["keyspace"] = mappings.MetricMap{
		"hits":   values["keyspace_hits"],
		"misses": values["keyspace_misses"],
	}
	redisMapping["last_save"] = mappings.MetricMap{
		"changes_since": values["changes_since_last_save"],
		"time":          values["last_save_time"],
	}
	redisMapping["memory"] = mappings.MetricMap{
		"internal_view":       values["used_memory"],
		"external_view":       values["used_memory_rss"],
		"fragmentation_ratio": values["mem_fragmentation_ratio"],
	}
	redisMapping["process"] = mappings.MetricMap{
		"commands_processed":   values["total_commands_processed"],
		"connections_received": values["total_connections_received"],
		"uptime":               values["uptime_in_seconds"],
	}
	redisMapping["pubsub"] = mappings.MetricMap{
		"channels": values["pubsub_channels"],
		"patterns": values["pubsub_patterns"],
	}
	redisMapping["slaves"] = mappings.MetricMap{
		"connected": values["connected_slaves"],
		"last_io":   values["master_last_io_seconds_ago"],
	}

	return redisMapping
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"crypto/md5"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"sync"
	"syscall"

	"github.com/flynn/go-crypto-ssh"
	"github.com/flynn/go-shlex"
)

const PrereceiveHook = `#!/bin/bash
set -eo pipefail; while read oldrev newrev refname; do
[[ $refname = "refs/heads/master" ]] && git archive $newrev | {{RECEIVER}} "$RECEIVE_USER" "$RECEIVE_REPO" "$RECEIVE_KEYNAME" "$RECEIVE_FINGERPRINT" | sed -$([[ $(uname) == "Darwin" ]] && echo l || echo u) "s/^/"$'\e[1G'"/"
done
`

var port *string = flag.String("p", "22", "port to listen on")
var repoPath *string = flag.String("r", "/tmp/repos", "path to repo cache")
var keyPath *string = flag.String("k", "/tmp/keys", "path to named keys")
var noAuth *bool = flag.Bool("n", false, "no client authentication")

var receiver string
var privateKey string
var keyNames = make(map[string]string)

func init() {
	flag.Usage = func() {
		fmt.Fprintf(os.Stderr, "Usage:  %v [options] <privatekey> <receiver>\n\n", os.Args[0])
		flag.PrintDefaults()
	}
}

func main() {
	flag.Parse()
	if len(os.Args) < 2 {
		flag.Usage()
		return
	}
	privateKey = flag.Arg(0)
	receiver = flag.Arg(1)

	var config *ssh.ServerConfig
	if *noAuth {
		config = &ssh.ServerConfig{NoClientAuth: true}
	} else {
		config = &ssh.ServerConfig{PublicKeyCallback: keyCallback}
	}

	pemBytes, err := ioutil.ReadFile(privateKey)
	if err != nil {
		log.Fatal("Failed to load private key:", err)
	}
	if err = config.SetRSAPrivateKey(pemBytes); err != nil {
		log.Fatal("Failed to parse private key:", err)
	}

	listener, err := ssh.Listen("tcp", "0.0.0.0:"+*port, config)
	if err != nil {
		log.Fatal("failed to listen for connection")
	}
	for {
		// SSH connections just house multiplexed connections
		conn, err := listener.Accept()
		if err != nil {
			log.Println("failed to accept incoming connection:", err)
			continue
		}
		if err := conn.Handshake(); err != nil {
			log.Println("failed to handshake:", err)
			continue
		}
		go handleConnection(conn)
	}
}

func keyCallback(conn *ssh.ServerConn, user, algo string, pubkey []byte) bool {
	clientkey, _, ok := ssh.ParsePublicKey(pubkey)
	if !ok {
		return false
	}
	os.MkdirAll(*keyPath, 0755)
	files, err := ioutil.ReadDir(*keyPath)
	if err != nil {
		log.Println("keydir read failed:", err)
		return false
	}
	for _, file := range files {
		if !file.IsDir() {
			data, err := ioutil.ReadFile(*keyPath + "/" + file.Name())
			if err != nil {
				log.Println("key read failed:", err)
				return false
			}
			filekey, _, _, _, ok := ssh.ParseAuthorizedKey(data)
			if !ok {
				continue
			}
			if bytes.Equal(clientkey.Marshal(), filekey.Marshal()) {
				keyNames[publicKeyFingerprint(clientkey)] = file.Name()
				return true
			}
		}
	}
	return false
}

func handleConnection(conn *ssh.ServerConn) {
	defer conn.Close()
	for {
		// Accept reads from the connection, demultiplexes packets
		// to their corresponding channels and returns when a new
		// channel request is seen. Some goroutine must always be
		// calling Accept; otherwise no messages will be forwarded
		// to the channels.
		ch, err := conn.Accept()
		if err == io.EOF {
			return
		}
		if err != nil {
			log.Println("handleConnection Accept:", err)
			break
		}
		if ch.ChannelType() != "session" {
			ch.Reject(ssh.UnknownChannelType, "unknown channel type")
			break
		}
		go handleChannel(conn, ch)
	}
}

func handleChannel(conn *ssh.ServerConn, ch ssh.Channel) {
	defer ch.Close()
	if err := ch.Accept(); err != nil {
		log.Println("ch.Accept failed:", err)
		return
	}
	for {
		req, err := ch.ReadRequest()
		if err == io.EOF {
			return
		}
		if err != nil {
			log.Println("handleChannel read request err:", err)
			continue
		}
		switch req.Request {
		case "exec":
			fail := func(at string, err error) {
				log.Printf("%s failed: %s", at, err)
				ch.Stderr().Write([]byte("Internal error."))
			}
			if req.WantReply {
				ch.AckRequest(true)
			}
			cmdline := string(req.Payload[4:])
			cmdargs, err := shlex.Split(cmdline)
			if err != nil || len(cmdargs) != 2 {
				ch.Stderr().Write([]byte("Invalid arguments."))
				return
			}
			if cmdargs[0] != "git-receive-pack" {
				ch.Stderr().Write([]byte("Only `git push` is supported."))
				return
			}
			cmdargs[1] = strings.TrimSuffix(strings.TrimPrefix(cmdargs[1], "/"), ".git")
			if strings.Contains(cmdargs[1], "/") {
				ch.Stderr().Write([]byte("Invalid repo."))
				return
			}
			if err := ensureCacheRepo(cmdargs[1]); err != nil {
				fail("ensureCacheRepo", err)
				return
			}
			var keyname, fingerprint string
			if *noAuth {
				fingerprint = ""
				keyname = ""
			} else {
				fingerprint = publicKeyFingerprint(conn.PublicKey)
				keyname = keyNames[fingerprint]
			}
			cmd := exec.Command("git-shell", "-c", cmdargs[0]+" '"+cmdargs[1]+"'")
			cmd.Dir = *repoPath
			cmd.Env = []string{
				"RECEIVE_USER=" + conn.User,
				"RECEIVE_REPO=" + cmdargs[1],
				"RECEIVE_KEYNAME=" + keyname,
				"RECEIVE_FINGERPRINT=" + fingerprint,
			}
			done, err := attachCmd(cmd, ch, ch.Stderr(), ch)
			if err != nil {
				fail("attachCmd", err)
				return
			}
			if err := cmd.Start(); err != nil {
				fail("cmd.Start", err)
				return
			}
			done.Wait()
			status, err := exitStatus(cmd)
			if err != nil {
				fail("exitStatus", err)
				return
			}
			ch.Exit(uint(status))
		case "env":
			if req.WantReply {
				ch.AckRequest(true)
			}
		}
	}
}

func attachCmd(cmd *exec.Cmd, stdout, stderr io.Writer, stdin io.Reader) (*sync.WaitGroup, error) {
	var wg sync.WaitGroup
	wg.Add(2)

	stdinIn, err := cmd.StdinPipe()
	if err != nil {
		return nil, err
	}
	stdoutOut, err := cmd.StdoutPipe()
	if err != nil {
		return nil, err
	}
	stderrOut, err := cmd.StderrPipe()
	if err != nil {
		return nil, err
	}

	go func() {
		io.Copy(stdinIn, stdin)
		stdinIn.Close()
	}()
	go func() {
		io.Copy(stdout, stdoutOut)
		wg.Done()
	}()
	go func() {
		io.Copy(stderr, stderrOut)
		wg.Done()
	}()

	return &wg, nil
}

func exitStatus(cmd *exec.Cmd) (int, error) {
	err := cmd.Wait()
	if err != nil {
		if exiterr, ok := err.(*exec.ExitError); ok {
			// There is no platform independent way to retrieve
			// the exit code, but the following will work on Unix
			if status, ok := exiterr.Sys().(syscall.WaitStatus); ok {
				return status.ExitStatus(), nil
			}
		}
		return 0, err
	}
	return 0, nil
}

var cacheMtx sync.Mutex

func ensureCacheRepo(path string) error {
	cacheMtx.Lock()
	defer cacheMtx.Unlock()

	cachePath := *repoPath + "/" + path
	if _, err := os.Stat(cachePath); os.IsNotExist(err) {
		os.MkdirAll(cachePath, 0755)
		cmd := exec.Command("git", "init", "--bare")
		cmd.Dir = cachePath
		err = cmd.Run()
		if err != nil {
			return err
		}
	}
	receiver, err := filepath.Abs(receiver)
	if err != nil {
		return err
	}
	return ioutil.WriteFile(
		cachePath+"/hooks/pre-receive",
		[]byte(strings.Replace(PrereceiveHook, "{{RECEIVER}}", receiver, 1)),
		0755,
	)
}

func publicKeyFingerprint(key ssh.PublicKey) string {
	var values []string
	h := md5.New()
	h.Write(ssh.MarshalPublicKey(key))
	for _, value := range h.Sum(nil) {
		values = append(values, fmt.Sprintf("%x", value))
	}
	return strings.Join(values, ":")
}
<commit_msg>Add newlines to errors<commit_after>package main

import (
	"bytes"
	"crypto/md5"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"sync"
	"syscall"

	"github.com/flynn/go-crypto-ssh"
	"github.com/flynn/go-shlex"
)

const PrereceiveHook = `#!/bin/bash
set -eo pipefail; while read oldrev newrev refname; do
[[ $refname = "refs/heads/master" ]] && git archive $newrev | {{RECEIVER}} "$RECEIVE_USER" "$RECEIVE_REPO" "$RECEIVE_KEYNAME" "$RECEIVE_FINGERPRINT" | sed -$([[ $(uname) == "Darwin" ]] && echo l || echo u) "s/^/"$'\e[1G'"/"
done
`

var port *string = flag.String("p", "22", "port to listen on")
var repoPath *string = flag.String("r", "/tmp/repos", "path to repo cache")
var keyPath *string = flag.String("k", "/tmp/keys", "path to named keys")
var noAuth *bool = flag.Bool("n", false, "no client authentication")

var receiver string
var privateKey string
var keyNames = make(map[string]string)

func init() {
	flag.Usage = func() {
		fmt.Fprintf(os.Stderr, "Usage:  %v [options] <privatekey> <receiver>\n\n", os.Args[0])
		flag.PrintDefaults()
	}
}

func main() {
	flag.Parse()
	if len(os.Args) < 2 {
		flag.Usage()
		return
	}
	privateKey = flag.Arg(0)
	receiver = flag.Arg(1)

	var config *ssh.ServerConfig
	if *noAuth {
		config = &ssh.ServerConfig{NoClientAuth: true}
	} else {
		config = &ssh.ServerConfig{PublicKeyCallback: keyCallback}
	}

	pemBytes, err := ioutil.ReadFile(privateKey)
	if err != nil {
		log.Fatal("Failed to load private key:", err)
	}
	if err = config.SetRSAPrivateKey(pemBytes); err != nil {
		log.Fatal("Failed to parse private key:", err)
	}

	listener, err := ssh.Listen("tcp", "0.0.0.0:"+*port, config)
	if err != nil {
		log.Fatal("failed to listen for connection")
	}
	for {
		// SSH connections just house multiplexed connections
		conn, err := listener.Accept()
		if err != nil {
			log.Println("failed to accept incoming connection:", err)
			continue
		}
		if err := conn.Handshake(); err != nil {
			log.Println("failed to handshake:", err)
			continue
		}
		go handleConnection(conn)
	}
}

func keyCallback(conn *ssh.ServerConn, user, algo string, pubkey []byte) bool {
	clientkey, _, ok := ssh.ParsePublicKey(pubkey)
	if !ok {
		return false
	}
	os.MkdirAll(*keyPath, 0755)
	files, err := ioutil.ReadDir(*keyPath)
	if err != nil {
		log.Println("keydir read failed:", err)
		return false
	}
	for _, file := range files {
		if !file.IsDir() {
			data, err := ioutil.ReadFile(*keyPath + "/" + file.Name())
			if err != nil {
				log.Println("key read failed:", err)
				return false
			}
			filekey, _, _, _, ok := ssh.ParseAuthorizedKey(data)
			if !ok {
				continue
			}
			if bytes.Equal(clientkey.Marshal(), filekey.Marshal()) {
				keyNames[publicKeyFingerprint(clientkey)] = file.Name()
				return true
			}
		}
	}
	return false
}

func handleConnection(conn *ssh.ServerConn) {
	defer conn.Close()
	for {
		// Accept reads from the connection, demultiplexes packets
		// to their corresponding channels and returns when a new
		// channel request is seen. Some goroutine must always be
		// calling Accept; otherwise no messages will be forwarded
		// to the channels.
		ch, err := conn.Accept()
		if err == io.EOF {
			return
		}
		if err != nil {
			log.Println("handleConnection Accept:", err)
			break
		}
		if ch.ChannelType() != "session" {
			ch.Reject(ssh.UnknownChannelType, "unknown channel type")
			break
		}
		go handleChannel(conn, ch)
	}
}

func handleChannel(conn *ssh.ServerConn, ch ssh.Channel) {
	defer ch.Close()
	if err := ch.Accept(); err != nil {
		log.Println("ch.Accept failed:", err)
		return
	}
	for {
		req, err := ch.ReadRequest()
		if err == io.EOF {
			return
		}
		if err != nil {
			log.Println("handleChannel read request err:", err)
			continue
		}
		switch req.Request {
		case "exec":
			fail := func(at string, err error) {
				log.Printf("%s failed: %s", at, err)
				ch.Stderr().Write([]byte("Internal error.\n"))
			}
			if req.WantReply {
				ch.AckRequest(true)
			}
			cmdline := string(req.Payload[4:])
			cmdargs, err := shlex.Split(cmdline)
			if err != nil || len(cmdargs) != 2 {
				ch.Stderr().Write([]byte("Invalid arguments.\n"))
				return
			}
			if cmdargs[0] != "git-receive-pack" {
				ch.Stderr().Write([]byte("Only `git push` is supported.\n"))
				return
			}
			cmdargs[1] = strings.TrimSuffix(strings.TrimPrefix(cmdargs[1], "/"), ".git")
			if strings.Contains(cmdargs[1], "/") {
				ch.Stderr().Write([]byte("Invalid repo."))
				return
			}
			if err := ensureCacheRepo(cmdargs[1]); err != nil {
				fail("ensureCacheRepo", err)
				return
			}
			var keyname, fingerprint string
			if *noAuth {
				fingerprint = ""
				keyname = ""
			} else {
				fingerprint = publicKeyFingerprint(conn.PublicKey)
				keyname = keyNames[fingerprint]
			}
			cmd := exec.Command("git-shell", "-c", cmdargs[0]+" '"+cmdargs[1]+"'")
			cmd.Dir = *repoPath
			cmd.Env = []string{
				"RECEIVE_USER=" + conn.User,
				"RECEIVE_REPO=" + cmdargs[1],
				"RECEIVE_KEYNAME=" + keyname,
				"RECEIVE_FINGERPRINT=" + fingerprint,
			}
			done, err := attachCmd(cmd, ch, ch.Stderr(), ch)
			if err != nil {
				fail("attachCmd", err)
				return
			}
			if err := cmd.Start(); err != nil {
				fail("cmd.Start", err)
				return
			}
			done.Wait()
			status, err := exitStatus(cmd)
			if err != nil {
				fail("exitStatus", err)
				return
			}
			ch.Exit(uint(status))
		case "env":
			if req.WantReply {
				ch.AckRequest(true)
			}
		}
	}
}

func attachCmd(cmd *exec.Cmd, stdout, stderr io.Writer, stdin io.Reader) (*sync.WaitGroup, error) {
	var wg sync.WaitGroup
	wg.Add(2)

	stdinIn, err := cmd.StdinPipe()
	if err != nil {
		return nil, err
	}
	stdoutOut, err := cmd.StdoutPipe()
	if err != nil {
		return nil, err
	}
	stderrOut, err := cmd.StderrPipe()
	if err != nil {
		return nil, err
	}

	go func() {
		io.Copy(stdinIn, stdin)
		stdinIn.Close()
	}()
	go func() {
		io.Copy(stdout, stdoutOut)
		wg.Done()
	}()
	go func() {
		io.Copy(stderr, stderrOut)
		wg.Done()
	}()

	return &wg, nil
}

func exitStatus(cmd *exec.Cmd) (int, error) {
	err := cmd.Wait()
	if err != nil {
		if exiterr, ok := err.(*exec.ExitError); ok {
			// There is no platform independent way to retrieve
			// the exit code, but the following will work on Unix
			if status, ok := exiterr.Sys().(syscall.WaitStatus); ok {
				return status.ExitStatus(), nil
			}
		}
		return 0, err
	}
	return 0, nil
}

var cacheMtx sync.Mutex

func ensureCacheRepo(path string) error {
	cacheMtx.Lock()
	defer cacheMtx.Unlock()

	cachePath := *repoPath + "/" + path
	if _, err := os.Stat(cachePath); os.IsNotExist(err) {
		os.MkdirAll(cachePath, 0755)
		cmd := exec.Command("git", "init", "--bare")
		cmd.Dir = cachePath
		err = cmd.Run()
		if err != nil {
			return err
		}
	}
	receiver, err := filepath.Abs(receiver)
	if err != nil {
		return err
	}
	return ioutil.WriteFile(
		cachePath+"/hooks/pre-receive",
		[]byte(strings.Replace(PrereceiveHook, "{{RECEIVER}}", receiver, 1)),
		0755,
	)
}

func publicKeyFingerprint(key ssh.PublicKey) string {
	var values []string
	h := md5.New()
	h.Write(ssh.MarshalPublicKey(key))
	for _, value := range h.Sum(nil) {
		values = append(values, fmt.Sprintf("%x", value))
	}
	return strings.Join(values, ":")
}
<|endoftext|>