<commit_before>package services

import (
	"context"
	"encoding/json"
	"fmt"
	"sync"
	"time"

	"go.uber.org/zap"

	"github.com/swagchat/chat-api/datastore"
	"github.com/swagchat/chat-api/models"
	"github.com/swagchat/chat-api/notification"
	"github.com/swagchat/chat-api/utils"
)

func GetDevices(userId string) (*models.Devices, *models.ProblemDetail) {
	dRes := datastore.GetProvider().SelectDevices(userId)
	if dRes.ProblemDetail != nil {
		return nil, dRes.ProblemDetail
	}

	devices := &models.Devices{
		Devices: dRes.Data.([]*models.Device),
	}
	return devices, nil
}

func GetDevice(userId string, platform int) (*models.Device, *models.ProblemDetail) {
	user, pd := SelectDevice(userId, platform)
	return user, pd
}

func PutDevice(put *models.Device) (*models.Device, *models.ProblemDetail) {
	if pd := put.IsValid(); pd != nil {
		return nil, pd
	}

	// User existence check
	_, pd := selectUser(put.UserId)
	if pd != nil {
		return nil, pd
	}

	isExist := true
	device, pd := SelectDevice(put.UserId, put.Platform)
	if device == nil {
		isExist = false
	}

	if !isExist || (device.Token != put.Token) {
		ctx, _ := context.WithCancel(context.Background())

		// When using another user on the same device, delete the notification information
		// of the olderuser in order to avoid duplication of the device token
		dRes := datastore.GetProvider().SelectDevicesByToken(put.Token)
		if dRes.ProblemDetail != nil {
			return nil, dRes.ProblemDetail
		}
		if dRes.Data != nil {
			wg := &sync.WaitGroup{}
			deleteDevices := dRes.Data.([]*models.Device)
			for _, deleteDevice := range deleteDevices {
				nRes := <-notification.GetProvider().DeleteEndpoint(deleteDevice.NotificationDeviceId)
				if nRes.ProblemDetail != nil {
					return nil, nRes.ProblemDetail
				}
				dRes := datastore.GetProvider().DeleteDevice(deleteDevice.UserId, deleteDevice.Platform)
				if dRes.ProblemDetail != nil {
					return nil, dRes.ProblemDetail
				}
				wg.Add(1)
				go unsubscribeByDevice(ctx, deleteDevice, wg)
			}
			wg.Wait()
		}

		nRes := <-notification.GetProvider().CreateEndpoint(put.UserId, put.Platform, put.Token)
		if nRes.ProblemDetail != nil {
			return nil, nRes.ProblemDetail
		}
		put.NotificationDeviceId = put.Token
		if nRes.Data != nil {
			put.NotificationDeviceId = *nRes.Data.(*string)
		}

		if isExist {
			dRes := datastore.GetProvider().UpdateDevice(put)
			if dRes.ProblemDetail != nil {
				return nil, dRes.ProblemDetail
			}
			nRes = <-notification.GetProvider().DeleteEndpoint(device.NotificationDeviceId)
			if nRes.ProblemDetail != nil {
				return nil, nRes.ProblemDetail
			}
			go func() {
				wg := &sync.WaitGroup{}
				wg.Add(1)
				go unsubscribeByDevice(ctx, device, wg)
				wg.Wait()
				go subscribeByDevice(ctx, put, nil)
			}()
		} else {
			dRes := datastore.GetProvider().InsertDevice(put)
			if dRes.ProblemDetail != nil {
				return nil, dRes.ProblemDetail
			}
			go subscribeByDevice(ctx, put, nil)
		}
		return put, nil
	} else {
		return nil, nil
	}
}

func DeleteDevice(userId string, platform int) *models.ProblemDetail {
	// User existence check
	_, pd := selectUser(userId)
	if pd != nil {
		return pd
	}

	device, pd := SelectDevice(userId, platform)
	if pd != nil {
		return pd
	}

	np := notification.GetProvider()
	nRes := <-np.DeleteEndpoint(device.NotificationDeviceId)
	if nRes.ProblemDetail != nil {
		return nRes.ProblemDetail
	}

	dRes := datastore.GetProvider().DeleteDevice(userId, platform)
	if dRes.ProblemDetail != nil {
		return dRes.ProblemDetail
	}

	ctx, _ := context.WithCancel(context.Background())
	go unsubscribeByDevice(ctx, device, nil)

	return nil
}

func SelectDevice(userId string, platform int) (*models.Device, *models.ProblemDetail) {
	dRes := datastore.GetProvider().SelectDevice(userId, platform)
	if dRes.ProblemDetail != nil {
		return nil, dRes.ProblemDetail
	}
	if dRes.Data == nil {
		return nil, nil
	}
	return dRes.Data.(*models.Device), nil
}

func subscribeByDevice(ctx context.Context, device *models.Device, wg *sync.WaitGroup) {
	dRes := datastore.GetProvider().SelectRoomUsersByUserId(device.UserId)
	if dRes.ProblemDetail != nil {
		pdBytes, _ := json.Marshal(dRes.ProblemDetail)
		utils.AppLogger.Error("",
			zap.String("problemDetail", string(pdBytes)),
			zap.String("err", fmt.Sprintf("%+v", dRes.ProblemDetail.Error)),
		)
	}
	if dRes.Data != nil {
		<-subscribe(ctx, dRes.Data.([]*models.RoomUser), device)
	}
	if wg != nil {
		wg.Done()
	}
}

func unsubscribeByDevice(ctx context.Context, device *models.Device, wg *sync.WaitGroup) {
	dRes := datastore.GetProvider().SelectDeletedSubscriptionsByUserIdAndPlatform(device.UserId, device.Platform)
	if dRes.ProblemDetail != nil {
		pdBytes, _ := json.Marshal(dRes.ProblemDetail)
		utils.AppLogger.Error("",
			zap.String("problemDetail", string(pdBytes)),
			zap.String("err", fmt.Sprintf("%+v", dRes.ProblemDetail.Error)),
		)
	}
	<-unsubscribe(ctx, dRes.Data.([]*models.Subscription))
	if wg != nil {
		wg.Done()
	}
}

func subscribe(ctx context.Context, roomUsers []*models.RoomUser, device *models.Device) chan bool {
	np := notification.GetProvider()
	dp := datastore.GetProvider()
	doneCh := make(chan bool, 1)
	pdCh := make(chan *models.ProblemDetail, 1)
	finishCh := make(chan bool, 1)

	d := utils.NewDispatcher(10)
	for _, roomUser := range roomUsers {
		ctx = context.WithValue(ctx, "roomUser", roomUser)
		d.Work(ctx, func(ctx context.Context) {
			ru := ctx.Value("roomUser").(*models.RoomUser)
			dRes := dp.SelectRoom(ru.RoomId)
			if dRes.ProblemDetail != nil {
				pdCh <- dRes.ProblemDetail
			} else {
				room := dRes.Data.(*models.Room)
				if room.NotificationTopicId == "" {
					notificationTopicId, pd := createTopic(room.RoomId)
					if pd != nil {
						pdCh <- pd
					}

					room.NotificationTopicId = notificationTopicId
					room.Modified = time.Now().Unix()
					dRes := datastore.GetProvider().UpdateRoom(room)
					if dRes.ProblemDetail != nil {
						pdCh <- dRes.ProblemDetail
					}
				}
				nRes := <-np.Subscribe(room.NotificationTopicId, device.NotificationDeviceId)
				if nRes.ProblemDetail != nil {
					pdCh <- nRes.ProblemDetail
				} else {
					if nRes.Data != nil {
						notificationSubscriptionId := nRes.Data.(*string)
						subscription := &models.Subscription{
							RoomId:                     ru.RoomId,
							UserId:                     ru.UserId,
							Platform:                   device.Platform,
							NotificationSubscriptionId: *notificationSubscriptionId,
						}
						dRes := dp.InsertSubscription(subscription)
						if dRes.ProblemDetail != nil {
							pdCh <- dRes.ProblemDetail
						} else {
							doneCh <- true
						}
					}
				}
			}

			select {
			case <-ctx.Done():
				return
			case <-doneCh:
				return
			case pd := <-pdCh:
				pdBytes, _ := json.Marshal(pd)
				utils.AppLogger.Error("",
					zap.String("problemDetail", string(pdBytes)),
					zap.String("err", fmt.Sprintf("%+v", pd.Error)),
				)
				return
			}
		})
	}
	d.Wait()
	finishCh <- true
	return finishCh
}

func unsubscribe(ctx context.Context, subscriptions []*models.Subscription) chan bool {
	np := notification.GetProvider()
	dp := datastore.GetProvider()
	doneCh := make(chan bool, 1)
	pdCh := make(chan *models.ProblemDetail, 1)
	finishCh := make(chan bool, 1)

	d := utils.NewDispatcher(10)
	for _, subscription := range subscriptions {
		ctx = context.WithValue(ctx, "subscription", subscription)
		d.Work(ctx, func(ctx context.Context) {
			targetSubscription := ctx.Value("subscription").(*models.Subscription)
			nRes := <-np.Unsubscribe(targetSubscription.NotificationSubscriptionId)
			if nRes.ProblemDetail != nil {
				pdCh <- nRes.ProblemDetail
			}
			dRes := dp.DeleteSubscription(targetSubscription)
			if dRes.ProblemDetail != nil {
				pdCh <- dRes.ProblemDetail
			} else {
				doneCh <- true
			}

			select {
			case <-ctx.Done():
				return
			case <-doneCh:
				return
			case pd := <-pdCh:
				pdBytes, _ := json.Marshal(pd)
				utils.AppLogger.Error("",
					zap.String("problemDetail", string(pdBytes)),
					zap.String("err", fmt.Sprintf("%+v", pd.Error)),
				)
				return
			}
		})
	}
	d.Wait()
	finishCh <- true
	return finishCh
}
<commit_msg>Add 404 status when device is not found<commit_after>package services

import (
	"context"
	"encoding/json"
	"fmt"
	"net/http"
	"sync"
	"time"

	"go.uber.org/zap"

	"github.com/swagchat/chat-api/datastore"
	"github.com/swagchat/chat-api/models"
	"github.com/swagchat/chat-api/notification"
	"github.com/swagchat/chat-api/utils"
)

func GetDevices(userId string) (*models.Devices, *models.ProblemDetail) {
	dRes := datastore.GetProvider().SelectDevices(userId)
	if dRes.ProblemDetail != nil {
		return nil, dRes.ProblemDetail
	}

	devices := &models.Devices{
		Devices: dRes.Data.([]*models.Device),
	}
	return devices, nil
}

func GetDevice(userId string, platform int) (*models.Device, *models.ProblemDetail) {
	devide, pd := SelectDevice(userId, platform)
	if devide == nil {
		return nil, &models.ProblemDetail{
			Status: http.StatusNotFound,
		}
	}
	return devide, pd
}

func PutDevice(put *models.Device) (*models.Device, *models.ProblemDetail) {
	if pd := put.IsValid(); pd != nil {
		return nil, pd
	}

	// User existence check
	_, pd := selectUser(put.UserId)
	if pd != nil {
		return nil, pd
	}

	isExist := true
	device, pd := SelectDevice(put.UserId, put.Platform)
	if device == nil {
		isExist = false
	}

	if !isExist || (device.Token != put.Token) {
		ctx, _ := context.WithCancel(context.Background())

		// When using another user on the same device, delete the notification information
		// of the olderuser in order to avoid duplication of the device token
		dRes := datastore.GetProvider().SelectDevicesByToken(put.Token)
		if dRes.ProblemDetail != nil {
			return nil, dRes.ProblemDetail
		}
		if dRes.Data != nil {
			wg := &sync.WaitGroup{}
			deleteDevices := dRes.Data.([]*models.Device)
			for _, deleteDevice := range deleteDevices {
				nRes := <-notification.GetProvider().DeleteEndpoint(deleteDevice.NotificationDeviceId)
				if nRes.ProblemDetail != nil {
					return nil, nRes.ProblemDetail
				}
				dRes := datastore.GetProvider().DeleteDevice(deleteDevice.UserId, deleteDevice.Platform)
				if dRes.ProblemDetail != nil {
					return nil, dRes.ProblemDetail
				}
				wg.Add(1)
				go unsubscribeByDevice(ctx, deleteDevice, wg)
			}
			wg.Wait()
		}

		nRes := <-notification.GetProvider().CreateEndpoint(put.UserId, put.Platform, put.Token)
		if nRes.ProblemDetail != nil {
			return nil, nRes.ProblemDetail
		}
		put.NotificationDeviceId = put.Token
		if nRes.Data != nil {
			put.NotificationDeviceId = *nRes.Data.(*string)
		}

		if isExist {
			dRes := datastore.GetProvider().UpdateDevice(put)
			if dRes.ProblemDetail != nil {
				return nil, dRes.ProblemDetail
			}
			nRes = <-notification.GetProvider().DeleteEndpoint(device.NotificationDeviceId)
			if nRes.ProblemDetail != nil {
				return nil, nRes.ProblemDetail
			}
			go func() {
				wg := &sync.WaitGroup{}
				wg.Add(1)
				go unsubscribeByDevice(ctx, device, wg)
				wg.Wait()
				go subscribeByDevice(ctx, put, nil)
			}()
		} else {
			dRes := datastore.GetProvider().InsertDevice(put)
			if dRes.ProblemDetail != nil {
				return nil, dRes.ProblemDetail
			}
			go subscribeByDevice(ctx, put, nil)
		}
		return put, nil
	} else {
		return nil, nil
	}
}

func DeleteDevice(userId string, platform int) *models.ProblemDetail {
	// User existence check
	_, pd := selectUser(userId)
	if pd != nil {
		return pd
	}

	device, pd := SelectDevice(userId, platform)
	if pd != nil {
		return pd
	}

	np := notification.GetProvider()
	nRes := <-np.DeleteEndpoint(device.NotificationDeviceId)
	if nRes.ProblemDetail != nil {
		return nRes.ProblemDetail
	}

	dRes := datastore.GetProvider().DeleteDevice(userId, platform)
	if dRes.ProblemDetail != nil {
		return dRes.ProblemDetail
	}

	ctx, _ := context.WithCancel(context.Background())
	go unsubscribeByDevice(ctx, device, nil)

	return nil
}

func SelectDevice(userId string, platform int) (*models.Device, *models.ProblemDetail) {
	dRes := datastore.GetProvider().SelectDevice(userId, platform)
	if dRes.ProblemDetail != nil {
		return nil, dRes.ProblemDetail
	}
	if dRes.Data == nil {
		return nil, nil
	}
	return dRes.Data.(*models.Device), nil
}

func subscribeByDevice(ctx context.Context, device *models.Device, wg *sync.WaitGroup) {
	dRes := datastore.GetProvider().SelectRoomUsersByUserId(device.UserId)
	if dRes.ProblemDetail != nil {
		pdBytes, _ := json.Marshal(dRes.ProblemDetail)
		utils.AppLogger.Error("",
			zap.String("problemDetail", string(pdBytes)),
			zap.String("err", fmt.Sprintf("%+v", dRes.ProblemDetail.Error)),
		)
	}
	if dRes.Data != nil {
		<-subscribe(ctx, dRes.Data.([]*models.RoomUser), device)
	}
	if wg != nil {
		wg.Done()
	}
}

func unsubscribeByDevice(ctx context.Context, device *models.Device, wg *sync.WaitGroup) {
	dRes := datastore.GetProvider().SelectDeletedSubscriptionsByUserIdAndPlatform(device.UserId, device.Platform)
	if dRes.ProblemDetail != nil {
		pdBytes, _ := json.Marshal(dRes.ProblemDetail)
		utils.AppLogger.Error("",
			zap.String("problemDetail", string(pdBytes)),
			zap.String("err", fmt.Sprintf("%+v", dRes.ProblemDetail.Error)),
		)
	}
	<-unsubscribe(ctx, dRes.Data.([]*models.Subscription))
	if wg != nil {
		wg.Done()
	}
}

func subscribe(ctx context.Context, roomUsers []*models.RoomUser, device *models.Device) chan bool {
	np := notification.GetProvider()
	dp := datastore.GetProvider()
	doneCh := make(chan bool, 1)
	pdCh := make(chan *models.ProblemDetail, 1)
	finishCh := make(chan bool, 1)

	d := utils.NewDispatcher(10)
	for _, roomUser := range roomUsers {
		ctx = context.WithValue(ctx, "roomUser", roomUser)
		d.Work(ctx, func(ctx context.Context) {
			ru := ctx.Value("roomUser").(*models.RoomUser)
			dRes := dp.SelectRoom(ru.RoomId)
			if dRes.ProblemDetail != nil {
				pdCh <- dRes.ProblemDetail
			} else {
				room := dRes.Data.(*models.Room)
				if room.NotificationTopicId == "" {
					notificationTopicId, pd := createTopic(room.RoomId)
					if pd != nil {
						pdCh <- pd
					}

					room.NotificationTopicId = notificationTopicId
					room.Modified = time.Now().Unix()
					dRes := datastore.GetProvider().UpdateRoom(room)
					if dRes.ProblemDetail != nil {
						pdCh <- dRes.ProblemDetail
					}
				}
				nRes := <-np.Subscribe(room.NotificationTopicId, device.NotificationDeviceId)
				if nRes.ProblemDetail != nil {
					pdCh <- nRes.ProblemDetail
				} else {
					if nRes.Data != nil {
						notificationSubscriptionId := nRes.Data.(*string)
						subscription := &models.Subscription{
							RoomId:                     ru.RoomId,
							UserId:                     ru.UserId,
							Platform:                   device.Platform,
							NotificationSubscriptionId: *notificationSubscriptionId,
						}
						dRes := dp.InsertSubscription(subscription)
						if dRes.ProblemDetail != nil {
							pdCh <- dRes.ProblemDetail
						} else {
							doneCh <- true
						}
					}
				}
			}

			select {
			case <-ctx.Done():
				return
			case <-doneCh:
				return
			case pd := <-pdCh:
				pdBytes, _ := json.Marshal(pd)
				utils.AppLogger.Error("",
					zap.String("problemDetail", string(pdBytes)),
					zap.String("err", fmt.Sprintf("%+v", pd.Error)),
				)
				return
			}
		})
	}
	d.Wait()
	finishCh <- true
	return finishCh
}

func unsubscribe(ctx context.Context, subscriptions []*models.Subscription) chan bool {
	np := notification.GetProvider()
	dp := datastore.GetProvider()
	doneCh := make(chan bool, 1)
	pdCh := make(chan *models.ProblemDetail, 1)
	finishCh := make(chan bool, 1)

	d := utils.NewDispatcher(10)
	for _, subscription := range subscriptions {
		ctx = context.WithValue(ctx, "subscription", subscription)
		d.Work(ctx, func(ctx context.Context) {
			targetSubscription := ctx.Value("subscription").(*models.Subscription)
			nRes := <-np.Unsubscribe(targetSubscription.NotificationSubscriptionId)
			if nRes.ProblemDetail != nil {
				pdCh <- nRes.ProblemDetail
			}
			dRes := dp.DeleteSubscription(targetSubscription)
			if dRes.ProblemDetail != nil {
				pdCh <- dRes.ProblemDetail
			} else {
				doneCh <- true
			}

			select {
			case <-ctx.Done():
				return
			case <-doneCh:
				return
			case pd := <-pdCh:
				pdBytes, _ := json.Marshal(pd)
				utils.AppLogger.Error("",
					zap.String("problemDetail", string(pdBytes)),
					zap.String("err", fmt.Sprintf("%+v", pd.Error)),
				)
				return
			}
		})
	}
	d.Wait()
	finishCh <- true
	return finishCh
}
<|endoftext|>


<commit_before>// Copyright 2011 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build appengine

package main

import (
	"bufio"
	"bytes"
	"io"
	"net/http"
	"strings"

	"appengine"

	_ "golang.org/x/tools/playground"
)

const runUrl = "http://golang.org/compile"

func init() {
	http.HandleFunc("/lesson/", lessonHandler)
	http.HandleFunc("/", rootHandler)

	if err := initTour(".", "HTTPTransport"); err != nil {
		panic(err)
	}
}

func rootHandler(w http.ResponseWriter, r *http.Request) {
	c := appengine.NewContext(r)
	if err := renderUI(w); err != nil {
		c.Criticalf("UI render: %v", err)
	}
}

func lessonHandler(w http.ResponseWriter, r *http.Request) {
	c := appengine.NewContext(r)
	lesson := strings.TrimPrefix(r.URL.Path, "/lesson/")
	if err := writeLesson(lesson, w); err != nil {
		if err == lessonNotFound {
			http.NotFound(w, r)
		} else {
			c.Criticalf("tour render: %v", err)
		}
	}
}

// prepContent returns a Reader that produces the content from the given
// Reader, but strips the prefix "#appengine: " from each line. It also drops
// any non-blank like that follows a series of 1 or more lines with the prefix.
func prepContent(in io.Reader) io.Reader {
	var prefix = []byte("#appengine: ")
	out, w := io.Pipe()
	go func() {
		r := bufio.NewReader(in)
		drop := false
		for {
			b, err := r.ReadBytes('\n')
			if err != nil && err != io.EOF {
				w.CloseWithError(err)
				return
			}
			if bytes.HasPrefix(b, prefix) {
				b = b[len(prefix):]
				drop = true
			} else if drop {
				if len(b) > 1 {
					b = nil
				}
				drop = false
			}
			if len(b) > 0 {
				w.Write(b)
			}
			if err == io.EOF {
				w.Close()
				return
			}
		}
	}()
	return out
}

// socketAddr returns the WebSocket handler address.
// The App Engine version does not provide a WebSocket handler.
func socketAddr() string { return "" }
<commit_msg>[x/tour] gotour: set Strict-Transport-Security header on App Engine<commit_after>// Copyright 2011 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build appengine

package main

import (
	"bufio"
	"bytes"
	"io"
	"net/http"
	"strings"

	"appengine"

	_ "golang.org/x/tools/playground"
)

const runUrl = "http://golang.org/compile"

func init() {
	http.Handle("/lesson/", hstsHandler(lessonHandler))
	http.Handle("/", hstsHandler(rootHandler))

	if err := initTour(".", "HTTPTransport"); err != nil {
		panic(err)
	}
}

func rootHandler(w http.ResponseWriter, r *http.Request) {
	c := appengine.NewContext(r)
	if err := renderUI(w); err != nil {
		c.Criticalf("UI render: %v", err)
	}
}

func lessonHandler(w http.ResponseWriter, r *http.Request) {
	c := appengine.NewContext(r)
	lesson := strings.TrimPrefix(r.URL.Path, "/lesson/")
	if err := writeLesson(lesson, w); err != nil {
		if err == lessonNotFound {
			http.NotFound(w, r)
		} else {
			c.Criticalf("tour render: %v", err)
		}
	}
}

// prepContent returns a Reader that produces the content from the given
// Reader, but strips the prefix "#appengine: " from each line. It also drops
// any non-blank like that follows a series of 1 or more lines with the prefix.
func prepContent(in io.Reader) io.Reader {
	var prefix = []byte("#appengine: ")
	out, w := io.Pipe()
	go func() {
		r := bufio.NewReader(in)
		drop := false
		for {
			b, err := r.ReadBytes('\n')
			if err != nil && err != io.EOF {
				w.CloseWithError(err)
				return
			}
			if bytes.HasPrefix(b, prefix) {
				b = b[len(prefix):]
				drop = true
			} else if drop {
				if len(b) > 1 {
					b = nil
				}
				drop = false
			}
			if len(b) > 0 {
				w.Write(b)
			}
			if err == io.EOF {
				w.Close()
				return
			}
		}
	}()
	return out
}

// socketAddr returns the WebSocket handler address.
// The App Engine version does not provide a WebSocket handler.
func socketAddr() string { return "" }

// hstsHandler wraps an http.HandlerFunc such that it sets the HSTS header.
func hstsHandler(fn http.HandlerFunc) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		w.Header().Set("Strict-Transport-Security", "max-age=31536000; preload")
		fn(w, r)
	})
}
<|endoftext|>


<commit_before>// +build linux

package resource

import (
	"errors"
	"fmt"

	"github.com/coreos/go-systemd/dbus"
	"github.com/coreos/go-systemd/util"
)

// Service type is a resource which manages
// services on a GNU/Linux system running with systemd
type Service struct {
	BaseResource

	// Name of the service
	Name string `luar:"name"`

	// If true then enable service during boot-time
	Enable bool `luar:"enable"`

	// Systemd unit name
	unit string `luar:"-"`
}

// NewService creates a new resource for managing services
// using systemd on a GNU/Linux system
func NewService(title string) (Resource, error) {
	s := &Service{
		BaseResource: BaseResource{
			Title: title,
			Type:  "service",
			State: StateRunning,
		},
		Name:   title,
		Enable: false,
		unit:   fmt.Sprintf("%s.service", title),
	}

	return &s, nil
}

// unitProperty retrieves the requested property for the service unit
func (s *Service) unitProperty(propertyName string) (*dbus.Property, error) {
	conn, err := dbus.New()
	if err != nil {
		return nil, err
	}
	defer conn.Close()

	property, err := conn.GetUnitProperty(s.unit, propertyName)

	return property, err
}

// unitIsEnabled checks if the unit is enabled or disabled
func (s *Service) unitIsEnabled() (bool, error) {
	unitState, err := s.unitProperty("UnitFileState")
	if err != nil {
		return false, err
	}

	value := unitState.Value.Value().(string)
	switch value {
	case "enabled", "static", "enabled-runtime", "linked", "linked-runtime":
		return true, nil
	case "disabled", "masked", "masked-runtime":
		return false, nil
	case "invalid":
		fallthrough
	default:
		return false, errors.New("Invalid unit state")
	}
}

// enableUnit enables the service unit during boot-time
func (s *Service) enableUnit() error {
	conn, err := dbus.New()
	if err != nil {
		return err
	}
	defer conn.Close()

	s.Printf("enabling service\n")

	units := []string{s.unit}
	_, changes, err := conn.EnableUnitFiles(units, false, false)
	if err != nil {
		return err
	}

	for _, change := range changes {
		s.Printf("%s %s -> %s\n", change.Type, change.Filename, change.Destination)
	}

	return nil
}

// disableUnit disables the service unit during boot-time
func (s *Service) disableUnit() error {
	conn, err := dbus.New()
	if err != nil {
		return err
	}
	defer conn.Close()

	s.Printf("disabling service\n")

	units := []string{s.unit}
	changes, err := conn.DisableUnitFiles(units, false)
	if err != nil {
		return err
	}

	for _, change := range changes {
		s.Printf("%s %s\n", change.Type, change.Filename)
	}

	return nil
}

// daemonReload instructs systemd to scan for and reload unit files
func (s *Service) daemonReload() error {
	conn, err := dbus.New()
	if err != nil {
		return err
	}
	defer conn.Close()

	return conn.Reload()
}

// Evaluate evaluates the state of the resource
func (s *Service) Evaluate() (State, error) {
	rs := State{
		Current: StateUnknown,
		Want:    s.State,
		Update:  false,
	}

	// Check if the unit is started/stopped
	activeState, err := s.unitProperty("ActiveState")
	if err != nil {
		return rs, err
	}

	// TODO: Handle cases where the unit is not found

	value := activeState.Value.Value().(string)
	switch value {
	case "active", "reloading", "activating":
		rs.Current = StateRunning
	case "inactive", "failed", "deactivating":
		rs.Current = StateStopped
	}

	enabled, err := s.unitIsEnabled()
	if err != nil {
		return rs, err
	}

	if s.Enable != enabled {
		rs.Update = true
	}

	return rs, nil
}

// Create starts the service unit
func (s *Service) Create() error {
	conn, err := dbus.New()
	if err != nil {
		return err
	}
	defer conn.Close()

	s.Printf("starting service\n")

	ch := make(chan string)
	jobID, err := conn.StartUnit(s.unit, "replace", ch)
	if err != nil {
		return err
	}

	result := <-ch
	s.Printf("systemd job id %d result: %s\n", jobID, result)

	return nil
}

// Delete stops the service unit
func (s *Service) Delete() error {
	conn, err := dbus.New()
	if err != nil {
		return err
	}
	defer conn.Close()

	s.Printf("stopping service\n")

	ch := make(chan string)
	jobID, err := conn.StopUnit(s.unit, "replace", ch)
	if err != nil {
		return err
	}

	result := <-ch
	s.Printf("systemd job id %d result: %s\n", jobID, result)

	return nil
}

// Update updates the service unit state
func (s *Service) Update() error {
	enabled, err := s.unitIsEnabled()
	if err != nil {
		return err
	}

	if s.Enable && !enabled {
		s.enableUnit()
	} else {
		s.disableUnit()
	}

	return s.daemonReload()
}

func init() {
	if util.IsRunningSystemd() {
		item := RegistryItem{
			Name:        serviceResourceType,
			Description: serviceResourceDesc,
			Provider:    NewService,
		}

		Register(item)
	}
}
<commit_msg>resource: refactor Service resource<commit_after>// +build linux

package resource

import (
	"errors"
	"fmt"

	"github.com/coreos/go-systemd/dbus"
	"github.com/coreos/go-systemd/util"
)

// Service type is a resource which manages
// services on a GNU/Linux system running with systemd
type Service struct {
	BaseResource

	// If true then enable the service during boot-time
	Enable bool `luar:"enable"`

	// Systemd unit name
	unit string `luar:"-"`
}

// NewService creates a new resource for managing services
// using systemd on a GNU/Linux system
func NewService(name string) (Resource, error) {
	s := &Service{
		BaseResource: BaseResource{
			Name:  name,
			Type:  "service",
			State: StateRunning,
		},
		Enable: false,
		unit:   fmt.Sprintf("%s.service", name),
	}

	return &s, nil
}

// unitProperty retrieves the requested property for the service unit
func (s *Service) unitProperty(name string) (*dbus.Property, error) {
	conn, err := dbus.New()
	if err != nil {
		return nil, err
	}
	defer conn.Close()

	property, err := conn.GetUnitProperty(s.unit, name)

	return property, err
}

// unitIsEnabled checks if the unit is enabled or disabled
func (s *Service) unitIsEnabled() (bool, error) {
	unitState, err := s.unitProperty("UnitFileState")
	if err != nil {
		return false, err
	}

	value := unitState.Value.Value().(string)
	switch value {
	case "enabled", "static", "enabled-runtime", "linked", "linked-runtime":
		return true, nil
	case "disabled", "masked", "masked-runtime":
		return false, nil
	case "invalid":
		fallthrough
	default:
		return false, errors.New("Invalid unit state")
	}
}

// enableUnit enables the service unit during boot-time
func (s *Service) enableUnit() error {
	conn, err := dbus.New()
	if err != nil {
		return err
	}
	defer conn.Close()

	s.Printf("enabling service\n")

	units := []string{s.unit}
	_, changes, err := conn.EnableUnitFiles(units, false, false)
	if err != nil {
		return err
	}

	for _, change := range changes {
		s.Printf("%s %s -> %s\n", change.Type, change.Filename, change.Destination)
	}

	return nil
}

// disableUnit disables the service unit during boot-time
func (s *Service) disableUnit() error {
	conn, err := dbus.New()
	if err != nil {
		return err
	}
	defer conn.Close()

	s.Printf("disabling service\n")

	units := []string{s.unit}
	changes, err := conn.DisableUnitFiles(units, false)
	if err != nil {
		return err
	}

	for _, change := range changes {
		s.Printf("%s %s\n", change.Type, change.Filename)
	}

	return nil
}

// daemonReload instructs systemd to reload it's configuration
func (s *Service) daemonReload() error {
	conn, err := dbus.New()
	if err != nil {
		return err
	}
	defer conn.Close()

	return conn.Reload()
}

// Evaluate evaluates the state of the resource
func (s *Service) Evaluate() (State, error) {
	state := State{
		Current: StateUnknown,
		Want:    s.State,
		Update:  false,
	}

	// Check if the unit is started/stopped
	activeState, err := s.unitProperty("ActiveState")
	if err != nil {
		return state, err
	}

	// TODO: Handle cases where the unit is not found

	value := activeState.Value.Value().(string)
	switch value {
	case "active", "reloading", "activating":
		state.Current = StateRunning
	case "inactive", "failed", "deactivating":
		state.Current = StateStopped
	}

	enabled, err := s.unitIsEnabled()
	if err != nil {
		return state, err
	}

	if s.Enable != enabled {
		state.Update = true
	}

	return state, nil
}

// Create starts the service unit
func (s *Service) Create() error {
	conn, err := dbus.New()
	if err != nil {
		return err
	}
	defer conn.Close()

	s.Printf("starting service\n")

	ch := make(chan string)
	jobID, err := conn.StartUnit(s.unit, "replace", ch)
	if err != nil {
		return err
	}

	result := <-ch
	s.Printf("systemd job id %d result: %s\n", jobID, result)

	return nil
}

// Delete stops the service unit
func (s *Service) Delete() error {
	conn, err := dbus.New()
	if err != nil {
		return err
	}
	defer conn.Close()

	s.Printf("stopping service\n")

	ch := make(chan string)
	jobID, err := conn.StopUnit(s.unit, "replace", ch)
	if err != nil {
		return err
	}

	result := <-ch
	s.Printf("systemd job id %d result: %s\n", jobID, result)

	return nil
}

// Update updates the service unit state
func (s *Service) Update() error {
	enabled, err := s.unitIsEnabled()
	if err != nil {
		return err
	}

	if s.Enable && !enabled {
		s.enableUnit()
	} else {
		s.disableUnit()
	}

	return s.daemonReload()
}

func init() {
	if util.IsRunningSystemd() {
		RegisterProvider("service", NewService)
	}
}
<|endoftext|>


<commit_before>package main_test

import (
	. "github.com/alphagov/cloudflare-configure"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"

	"io/ioutil"
	"net/http"
	"strings"
)

const (
	headerEmail = "X-Auth-Email"
	headerKey   = "X-Auth-Key"
	authEmail   = "user@example.com"
	authKey     = "abc123"
)

var _ = Describe("CloudFlareQuery", func() {
	var query CloudFlareQuery

	BeforeEach(func() {
		query = CloudFlareQuery{
			RootURL:   "https://example.com/api",
			AuthEmail: authEmail,
			AuthKey:   authKey,
		}
	})

	Describe("MakeRequest", func() {
		var (
			req *http.Request
			err error
		)

		BeforeEach(func() {
			req, err = query.NewRequest("GET", "/zones")
		})

		It("should return request and no errors", func() {
			Expect(req).ToNot(BeNil())
			Expect(err).To(BeNil())
		})

		It("should set method and path", func() {
			Expect(req.Method).To(Equal("GET"))
			Expect(req.URL.String()).To(Equal("https://example.com/api/zones"))
		})

		It("should not set request body", func() {
			Expect(req.Body).To(BeNil())
		})

		It("should set Content-Type to JSON", func() {
			Expect(req.Header.Get("Content-Type")).To(Equal("application/json"))
		})

		It("should set authentication email and key header", func() {
			Expect(req.Header.Get(headerEmail)).To(Equal(authEmail))
			Expect(req.Header.Get(headerKey)).To(Equal(authKey))
		})
	})

	Describe("MakeRequestBody", func() {
		var (
			body = `{"foo": "bar"}`
			req  *http.Request
			err  error
		)

		BeforeEach(func() {
			req, err = query.NewRequestBody(
				"PATCH", "/settings", strings.NewReader(body),
			)
		})

		It("should return request and no errors", func() {
			Expect(req).ToNot(BeNil())
			Expect(err).To(BeNil())
		})

		It("should set method and URL", func() {
			Expect(req.Method).To(Equal("PATCH"))
			Expect(req.URL.String()).To(Equal("https://example.com/api/settings"))
		})

		It("should set request body", func() {
			buf, err := ioutil.ReadAll(req.Body)
			defer req.Body.Close()

			Expect(err).To(BeNil())
			Expect(buf).To(MatchJSON(body))
		})
	})
})
<commit_msg>Reduce duplication of error and *http.Request object vars<commit_after>package main_test

import (
	. "github.com/alphagov/cloudflare-configure"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"

	"io/ioutil"
	"net/http"
	"strings"
)

const (
	headerEmail = "X-Auth-Email"
	headerKey   = "X-Auth-Key"
	authEmail   = "user@example.com"
	authKey     = "abc123"
)

var _ = Describe("CloudFlareQuery", func() {
	var (
		err   error
		query CloudFlareQuery
		req   *http.Request
	)

	BeforeEach(func() {
		query = CloudFlareQuery{
			RootURL:   "https://example.com/api",
			AuthEmail: authEmail,
			AuthKey:   authKey,
		}
	})

	Describe("MakeRequest", func() {
		BeforeEach(func() {
			req, err = query.NewRequest("GET", "/zones")
		})

		It("should return request and no errors", func() {
			Expect(req).ToNot(BeNil())
			Expect(err).To(BeNil())
		})

		It("should set method and path", func() {
			Expect(req.Method).To(Equal("GET"))
			Expect(req.URL.String()).To(Equal("https://example.com/api/zones"))
		})

		It("should not set request body", func() {
			Expect(req.Body).To(BeNil())
		})

		It("should set Content-Type to JSON", func() {
			Expect(req.Header.Get("Content-Type")).To(Equal("application/json"))
		})

		It("should set authentication email and key header", func() {
			Expect(req.Header.Get(headerEmail)).To(Equal(authEmail))
			Expect(req.Header.Get(headerKey)).To(Equal(authKey))
		})
	})

	Describe("MakeRequestBody", func() {
		const body = `{"foo": "bar"}`

		BeforeEach(func() {
			req, err = query.NewRequestBody(
				"PATCH", "/settings", strings.NewReader(body),
			)
		})

		It("should return request and no errors", func() {
			Expect(req).ToNot(BeNil())
			Expect(err).To(BeNil())
		})

		It("should set method and URL", func() {
			Expect(req.Method).To(Equal("PATCH"))
			Expect(req.URL.String()).To(Equal("https://example.com/api/settings"))
		})

		It("should set request body", func() {
			buf, err := ioutil.ReadAll(req.Body)
			defer req.Body.Close()

			Expect(err).To(BeNil())
			Expect(buf).To(MatchJSON(body))
		})
	})
})
<|endoftext|>


<commit_before>/*
Copyright 2014 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package admission

import (
	"io"
	"os"
	"sync"

	"github.com/golang/glog"
	client "k8s.io/kubernetes/pkg/client/unversioned"
)

// Factory is a function that returns an Interface for admission decisions.
// The config parameter provides an io.Reader handler to the factory in
// order to load specific configurations. If no configuration is provided
// the parameter is nil.
type Factory func(client client.Interface, config io.Reader) (Interface, error)

// All registered admission options.
var (
	pluginsMutex sync.Mutex
	plugins      = make(map[string]Factory)
)

// GetPlugins enumerates the
func GetPlugins() []string {
	pluginsMutex.Lock()
	defer pluginsMutex.Unlock()
	keys := []string{}
	for k := range plugins {
		keys = append(keys, k)
	}
	return keys
}

// RegisterPlugin registers a plugin Factory by name.  This
// is expected to happen during app startup.
func RegisterPlugin(name string, plugin Factory) {
	pluginsMutex.Lock()
	defer pluginsMutex.Unlock()
	_, found := plugins[name]
	if found {
		glog.Fatalf("Admission plugin %q was registered twice", name)
	}
	glog.V(1).Infof("Registered admission plugin %q", name)
	plugins[name] = plugin
}

// GetPlugin creates an instance of the named plugin, or nil if
// the name is not known.  The error return is only used if the named provider
// was known but failed to initialize. The config parameter specifies the
// io.Reader handler of the configuration file for the cloud provider, or nil
// for no configuration.
func GetPlugin(name string, client client.Interface, config io.Reader) (Interface, error) {
	pluginsMutex.Lock()
	defer pluginsMutex.Unlock()
	f, found := plugins[name]
	if !found {
		return nil, nil
	}
	return f(client, config)
}

// InitPlugin creates an instance of the named interface.
func InitPlugin(name string, client client.Interface, configFilePath string) Interface {
	var (
		config *os.File
		err    error
	)

	if name == "" {
		glog.Info("No admission plugin specified.")
		return nil
	}

	if configFilePath != "" {
		config, err = os.Open(configFilePath)
		if err != nil {
			glog.Fatalf("Couldn't open admission plugin configuration %s: %#v",
				configFilePath, err)
		}

		defer config.Close()
	}

	plugin, err := GetPlugin(name, client, config)
	if err != nil {
		glog.Fatalf("Couldn't init admission plugin %q: %v", name, err)
	}
	if plugin == nil {
		glog.Fatalf("Unknown admission plugin: %s", name)
	}

	return plugin
}
<commit_msg>Doc fix for admission/plugin.go<commit_after>/*
Copyright 2014 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package admission

import (
	"io"
	"os"
	"sync"

	"github.com/golang/glog"
	client "k8s.io/kubernetes/pkg/client/unversioned"
)

// Factory is a function that returns an Interface for admission decisions.
// The config parameter provides an io.Reader handler to the factory in
// order to load specific configurations. If no configuration is provided
// the parameter is nil.
type Factory func(client client.Interface, config io.Reader) (Interface, error)

// All registered admission options.
var (
	pluginsMutex sync.Mutex
	plugins      = make(map[string]Factory)
)

// GetPlugins enumerates the names of all registered plugins.
func GetPlugins() []string {
	pluginsMutex.Lock()
	defer pluginsMutex.Unlock()
	keys := []string{}
	for k := range plugins {
		keys = append(keys, k)
	}
	return keys
}

// RegisterPlugin registers a plugin Factory by name. This
// is expected to happen during app startup.
func RegisterPlugin(name string, plugin Factory) {
	pluginsMutex.Lock()
	defer pluginsMutex.Unlock()
	_, found := plugins[name]
	if found {
		glog.Fatalf("Admission plugin %q was registered twice", name)
	}
	glog.V(1).Infof("Registered admission plugin %q", name)
	plugins[name] = plugin
}

// GetPlugin creates an instance of the named plugin, or nil if the name is not
// known. The error is returned only when the named provider was known but failed
// to initialize. The config parameter specifies the io.Reader handler of the
// configuration file for the cloud provider, or nil for no configuration.
func GetPlugin(name string, client client.Interface, config io.Reader) (Interface, error) {
	pluginsMutex.Lock()
	defer pluginsMutex.Unlock()
	f, found := plugins[name]
	if !found {
		return nil, nil
	}
	return f(client, config)
}

// InitPlugin creates an instance of the named interface.
func InitPlugin(name string, client client.Interface, configFilePath string) Interface {
	var (
		config *os.File
		err    error
	)

	if name == "" {
		glog.Info("No admission plugin specified.")
		return nil
	}

	if configFilePath != "" {
		config, err = os.Open(configFilePath)
		if err != nil {
			glog.Fatalf("Couldn't open admission plugin configuration %s: %#v",
				configFilePath, err)
		}

		defer config.Close()
	}

	plugin, err := GetPlugin(name, client, config)
	if err != nil {
		glog.Fatalf("Couldn't init admission plugin %q: %v", name, err)
	}
	if plugin == nil {
		glog.Fatalf("Unknown admission plugin: %s", name)
	}

	return plugin
}
<|endoftext|>


<commit_before>package main

import (
	"crypto/md5"
	"crypto/sha1"
	"crypto/sha256"
	"crypto/sha512"
	"flag"
	"fmt"
	"hash"
	"io"
	"os"
	"runtime"
	"strings"
	"sync"
)

var fHash = flag.String("h", "sha256", "valid hashes: md5, sha1, sha224, sha256, sha384, sha512")
var fConcurrent = flag.Int("j", runtime.NumCPU()*2, "Maximum number of files processed concurrently.")

type fileInput struct {
	fileName string
	data     io.ReadCloser
}

func main() {
	flag.Usage = func() {
		fmt.Fprintf(os.Stderr, "%s v1.0 Copyright (c) 2014, Gregory L. Dietsche.\n", os.Args[0])
		fmt.Fprintf(os.Stderr, "Usage of %s: [OPTION]... [FILE]...\n", os.Args[0])
		flag.PrintDefaults()
	}
	flag.Parse()

	if *fConcurrent <= 0 {
		*fConcurrent = 1
	}

	in := make(chan fileInput, *fConcurrent*10)
	out := make(chan *string, *fConcurrent*10)

	go func() {
		if flag.NFlag() == 0 {
			in <- fileInput{"", os.Stdin}
		} else {
			for _, file := range flag.Args() {
				stream, err := os.Open(file)
				if err == nil {
					in <- fileInput{file, stream}
				} else {
					fmt.Println(err.Error())
				}
			}
		}
		close(in)
	}()

	go func() {
		defer close(out)
		var wg sync.WaitGroup
		*fHash = strings.ToLower(*fHash)
		for i := 0; i < *fConcurrent; i++ {
			var hash hash.Hash
			switch *fHash {
			case "md5":
				hash = md5.New()
			case "sha1":
				hash = sha1.New()
			case "sha224":
				hash = sha256.New224()
			case "sha256":
				hash = sha256.New()
			case "sha384":
				hash = sha512.New384()
			case "sha512":
				hash = sha512.New()
			default:
				fmt.Fprintf(os.Stderr, "I don't know how to compute a %s hash!", *fHash)
				return
			}
			wg.Add(1)
			go digester(&wg, &hash, out, in)
		}
		wg.Wait()
	}()

	for curResult := range out {
		fmt.Println(*curResult)
	}
}

func digester(wg *sync.WaitGroup, h *hash.Hash, out chan *string, streams chan fileInput) {
	for stream := range streams {
		(*h).Reset()

		io.Copy(*h, stream.data)
		stream.data.Close()

		var message string

		if len(stream.fileName) > 0 {
			message = fmt.Sprintf("%08x\t%s", (*h).Sum(nil), stream.fileName)
		} else {
			message = fmt.Sprintf("%08x", (*h).Sum(nil))
		}

		out <- &message
	}
	wg.Done()
}
<commit_msg>set max procs equal to number of CPUs available<commit_after>package main

import (
	"crypto/md5"
	"crypto/sha1"
	"crypto/sha256"
	"crypto/sha512"
	"flag"
	"fmt"
	"hash"
	"io"
	"os"
	"runtime"
	"strings"
	"sync"
)

var fHash = flag.String("h", "sha256", "valid hashes: md5, sha1, sha224, sha256, sha384, sha512")
var fConcurrent = flag.Int("j", runtime.NumCPU()*2, "Maximum number of files processed concurrently.")

type fileInput struct {
	fileName string
	data     io.ReadCloser
}

func main() {
	flag.Usage = func() {
		fmt.Fprintf(os.Stderr, "%s v1.0 Copyright (c) 2014, Gregory L. Dietsche.\n", os.Args[0])
		fmt.Fprintf(os.Stderr, "Usage of %s: [OPTION]... [FILE]...\n", os.Args[0])
		flag.PrintDefaults()
	}
	flag.Parse()

	if *fConcurrent <= 0 {
		*fConcurrent = 1
	}

	runtime.GOMAXPROCS(runtime.NumCPU())
	in := make(chan fileInput, *fConcurrent*10)
	out := make(chan *string, *fConcurrent*10)

	go func() {
		if flag.NFlag() == 0 {
			in <- fileInput{"", os.Stdin}
		} else {
			for _, file := range flag.Args() {
				stream, err := os.Open(file)
				if err == nil {
					in <- fileInput{file, stream}
				} else {
					fmt.Println(err.Error())
				}
			}
		}
		close(in)
	}()

	go func() {
		defer close(out)
		var wg sync.WaitGroup
		*fHash = strings.ToLower(*fHash)
		for i := 0; i < *fConcurrent; i++ {
			var hash hash.Hash
			switch *fHash {
			case "md5":
				hash = md5.New()
			case "sha1":
				hash = sha1.New()
			case "sha224":
				hash = sha256.New224()
			case "sha256":
				hash = sha256.New()
			case "sha384":
				hash = sha512.New384()
			case "sha512":
				hash = sha512.New()
			default:
				fmt.Fprintf(os.Stderr, "I don't know how to compute a %s hash!", *fHash)
				return
			}
			wg.Add(1)
			go digester(&wg, &hash, out, in)
		}
		wg.Wait()
	}()

	for curResult := range out {
		fmt.Println(*curResult)
	}
}

func digester(wg *sync.WaitGroup, h *hash.Hash, out chan *string, streams chan fileInput) {
	for stream := range streams {
		(*h).Reset()

		io.Copy(*h, stream.data)
		stream.data.Close()

		var message string

		if len(stream.fileName) > 0 {
			message = fmt.Sprintf("%08x\t%s", (*h).Sum(nil), stream.fileName)
		} else {
			message = fmt.Sprintf("%08x", (*h).Sum(nil))
		}

		out <- &message
	}
	wg.Done()
}
<|endoftext|>


<commit_before>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package rbac

import (
	"fmt"
	"strings"

	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/runtime/schema"
	"k8s.io/apimachinery/pkg/util/sets"
)

func RoleRefGroupKind(roleRef RoleRef) schema.GroupKind {
	return schema.GroupKind{Group: roleRef.APIGroup, Kind: roleRef.Kind}
}

func VerbMatches(rule PolicyRule, requestedVerb string) bool {
	for _, ruleVerb := range rule.Verbs {
		if ruleVerb == VerbAll {
			return true
		}
		if ruleVerb == requestedVerb {
			return true
		}
	}

	return false
}

func APIGroupMatches(rule PolicyRule, requestedGroup string) bool {
	for _, ruleGroup := range rule.APIGroups {
		if ruleGroup == APIGroupAll {
			return true
		}
		if ruleGroup == requestedGroup {
			return true
		}
	}

	return false
}

func ResourceMatches(rule PolicyRule, requestedResource string) bool {
	for _, ruleResource := range rule.Resources {
		if ruleResource == ResourceAll {
			return true
		}
		if ruleResource == requestedResource {
			return true
		}
	}

	return false
}

func ResourceNameMatches(rule PolicyRule, requestedName string) bool {
	if len(rule.ResourceNames) == 0 {
		return true
	}

	for _, ruleName := range rule.ResourceNames {
		if ruleName == requestedName {
			return true
		}
	}

	return false
}

func NonResourceURLMatches(rule PolicyRule, requestedURL string) bool {
	for _, ruleURL := range rule.NonResourceURLs {
		if ruleURL == NonResourceAll {
			return true
		}
		if ruleURL == requestedURL {
			return true
		}
		if strings.HasSuffix(ruleURL, "*") && strings.HasPrefix(requestedURL, strings.TrimRight(ruleURL, "*")) {
			return true
		}
	}

	return false
}

// subjectsStrings returns users, groups, serviceaccounts, unknown for display purposes.
func SubjectsStrings(subjects []Subject) ([]string, []string, []string, []string) {
	users := []string{}
	groups := []string{}
	sas := []string{}
	others := []string{}

	for _, subject := range subjects {
		switch subject.Kind {
		case ServiceAccountKind:
			sas = append(sas, fmt.Sprintf("%s/%s", subject.Namespace, subject.Name))

		case UserKind:
			users = append(users, subject.Name)

		case GroupKind:
			groups = append(groups, subject.Name)

		default:
			others = append(others, fmt.Sprintf("%s/%s/%s", subject.Kind, subject.Namespace, subject.Name))
		}
	}

	return users, groups, sas, others
}

// +k8s:deepcopy-gen=false
// PolicyRuleBuilder let's us attach methods.  A no-no for API types.
// We use it to construct rules in code.  It's more compact than trying to write them
// out in a literal and allows us to perform some basic checking during construction
type PolicyRuleBuilder struct {
	PolicyRule PolicyRule
}

func NewRule(verbs ...string) *PolicyRuleBuilder {
	return &PolicyRuleBuilder{
		PolicyRule: PolicyRule{Verbs: sets.NewString(verbs...).List()},
	}
}

func (r *PolicyRuleBuilder) Groups(groups ...string) *PolicyRuleBuilder {
	r.PolicyRule.APIGroups = combine(r.PolicyRule.APIGroups, groups)
	return r
}

func (r *PolicyRuleBuilder) Resources(resources ...string) *PolicyRuleBuilder {
	r.PolicyRule.Resources = combine(r.PolicyRule.Resources, resources)
	return r
}

func (r *PolicyRuleBuilder) Names(names ...string) *PolicyRuleBuilder {
	r.PolicyRule.ResourceNames = combine(r.PolicyRule.ResourceNames, names)
	return r
}

func (r *PolicyRuleBuilder) URLs(urls ...string) *PolicyRuleBuilder {
	r.PolicyRule.NonResourceURLs = combine(r.PolicyRule.NonResourceURLs, urls)
	return r
}

func (r *PolicyRuleBuilder) RuleOrDie() PolicyRule {
	ret, err := r.Rule()
	if err != nil {
		panic(err)
	}
	return ret
}

func combine(s1, s2 []string) []string {
	s := sets.NewString(s1...)
	s.Insert(s2...)
	return s.List()
}

func (r *PolicyRuleBuilder) Rule() (PolicyRule, error) {
	if len(r.PolicyRule.Verbs) == 0 {
		return PolicyRule{}, fmt.Errorf("verbs are required: %#v", r.PolicyRule)
	}

	switch {
	case len(r.PolicyRule.NonResourceURLs) > 0:
		if len(r.PolicyRule.APIGroups) != 0 || len(r.PolicyRule.Resources) != 0 || len(r.PolicyRule.ResourceNames) != 0 {
			return PolicyRule{}, fmt.Errorf("non-resource rule may not have apiGroups, resources, or resourceNames: %#v", r.PolicyRule)
		}
	case len(r.PolicyRule.Resources) > 0:
		if len(r.PolicyRule.NonResourceURLs) != 0 {
			return PolicyRule{}, fmt.Errorf("resource rule may not have nonResourceURLs: %#v", r.PolicyRule)
		}
		if len(r.PolicyRule.APIGroups) == 0 {
			// this a common bug
			return PolicyRule{}, fmt.Errorf("resource rule must have apiGroups: %#v", r.PolicyRule)
		}
	default:
		return PolicyRule{}, fmt.Errorf("a rule must have either nonResourceURLs or resources: %#v", r.PolicyRule)
	}

	return r.PolicyRule, nil
}

// +k8s:deepcopy-gen=false
// ClusterRoleBindingBuilder let's us attach methods.  A no-no for API types.
// We use it to construct bindings in code.  It's more compact than trying to write them
// out in a literal.
type ClusterRoleBindingBuilder struct {
	ClusterRoleBinding ClusterRoleBinding
}

func NewClusterBinding(clusterRoleName string) *ClusterRoleBindingBuilder {
	return &ClusterRoleBindingBuilder{
		ClusterRoleBinding: ClusterRoleBinding{
			ObjectMeta: metav1.ObjectMeta{Name: clusterRoleName},
			RoleRef: RoleRef{
				APIGroup: GroupName,
				Kind:     "ClusterRole",
				Name:     clusterRoleName,
			},
		},
	}
}

func (r *ClusterRoleBindingBuilder) Groups(groups ...string) *ClusterRoleBindingBuilder {
	for _, group := range groups {
		r.ClusterRoleBinding.Subjects = append(r.ClusterRoleBinding.Subjects, Subject{Kind: GroupKind, APIGroup: GroupName, Name: group})
	}
	return r
}

func (r *ClusterRoleBindingBuilder) Users(users ...string) *ClusterRoleBindingBuilder {
	for _, user := range users {
		r.ClusterRoleBinding.Subjects = append(r.ClusterRoleBinding.Subjects, Subject{Kind: UserKind, APIGroup: GroupName, Name: user})
	}
	return r
}

func (r *ClusterRoleBindingBuilder) SAs(namespace string, serviceAccountNames ...string) *ClusterRoleBindingBuilder {
	for _, saName := range serviceAccountNames {
		r.ClusterRoleBinding.Subjects = append(r.ClusterRoleBinding.Subjects, Subject{Kind: ServiceAccountKind, Namespace: namespace, Name: saName})
	}
	return r
}

func (r *ClusterRoleBindingBuilder) BindingOrDie() ClusterRoleBinding {
	ret, err := r.Binding()
	if err != nil {
		panic(err)
	}
	return ret
}

func (r *ClusterRoleBindingBuilder) Binding() (ClusterRoleBinding, error) {
	if len(r.ClusterRoleBinding.Subjects) == 0 {
		return ClusterRoleBinding{}, fmt.Errorf("subjects are required: %#v", r.ClusterRoleBinding)
	}

	return r.ClusterRoleBinding, nil
}

// +k8s:deepcopy-gen=false
// RoleBindingBuilder let's us attach methods. It is similar to
// ClusterRoleBindingBuilder above.
type RoleBindingBuilder struct {
	RoleBinding RoleBinding
}

// NewRoleBinding creates a RoleBinding builder that can be used
// to define the subjects of a role binding. At least one of
// the `Groups`, `Users` or `SAs` method must be called before
// calling the `Binding*` methods.
func NewRoleBinding(roleName, namespace string) *RoleBindingBuilder {
	return &RoleBindingBuilder{
		RoleBinding: RoleBinding{
			ObjectMeta: metav1.ObjectMeta{
				Name:      roleName,
				Namespace: namespace,
			},
			RoleRef: RoleRef{
				APIGroup: GroupName,
				Kind:     "Role",
				Name:     roleName,
			},
		},
	}
}

func NewRoleBindingForClusterRole(roleName, namespace string) *RoleBindingBuilder {
	return &RoleBindingBuilder{
		RoleBinding: RoleBinding{
			ObjectMeta: metav1.ObjectMeta{
				Name:      roleName,
				Namespace: namespace,
			},
			RoleRef: RoleRef{
				APIGroup: GroupName,
				Kind:     "ClusterRole",
				Name:     roleName,
			},
		},
	}
}

// Groups adds the specified groups as the subjects of the RoleBinding.
func (r *RoleBindingBuilder) Groups(groups ...string) *RoleBindingBuilder {
	for _, group := range groups {
		r.RoleBinding.Subjects = append(r.RoleBinding.Subjects, Subject{Kind: GroupKind, Name: group})
	}
	return r
}

// Users adds the specified users as the subjects of the RoleBinding.
func (r *RoleBindingBuilder) Users(users ...string) *RoleBindingBuilder {
	for _, user := range users {
		r.RoleBinding.Subjects = append(r.RoleBinding.Subjects, Subject{Kind: UserKind, Name: user})
	}
	return r
}

// SAs adds the specified service accounts as the subjects of the
// RoleBinding.
func (r *RoleBindingBuilder) SAs(namespace string, serviceAccountNames ...string) *RoleBindingBuilder {
	for _, saName := range serviceAccountNames {
		r.RoleBinding.Subjects = append(r.RoleBinding.Subjects, Subject{Kind: ServiceAccountKind, Namespace: namespace, Name: saName})
	}
	return r
}

// BindingOrDie calls the binding method and panics if there is an error.
func (r *RoleBindingBuilder) BindingOrDie() RoleBinding {
	ret, err := r.Binding()
	if err != nil {
		panic(err)
	}
	return ret
}

// Binding builds and returns the RoleBinding API object from the builder
// object.
func (r *RoleBindingBuilder) Binding() (RoleBinding, error) {
	if len(r.RoleBinding.Subjects) == 0 {
		return RoleBinding{}, fmt.Errorf("subjects are required: %#v", r.RoleBinding)
	}

	return r.RoleBinding, nil
}
<commit_msg>Switch to pointer to policy rule, visit and short circuit during authorization<commit_after>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package rbac

import (
	"fmt"
	"strings"

	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/runtime/schema"
	"k8s.io/apimachinery/pkg/util/sets"
)

func RoleRefGroupKind(roleRef RoleRef) schema.GroupKind {
	return schema.GroupKind{Group: roleRef.APIGroup, Kind: roleRef.Kind}
}

func VerbMatches(rule *PolicyRule, requestedVerb string) bool {
	for _, ruleVerb := range rule.Verbs {
		if ruleVerb == VerbAll {
			return true
		}
		if ruleVerb == requestedVerb {
			return true
		}
	}

	return false
}

func APIGroupMatches(rule *PolicyRule, requestedGroup string) bool {
	for _, ruleGroup := range rule.APIGroups {
		if ruleGroup == APIGroupAll {
			return true
		}
		if ruleGroup == requestedGroup {
			return true
		}
	}

	return false
}

func ResourceMatches(rule *PolicyRule, requestedResource string) bool {
	for _, ruleResource := range rule.Resources {
		if ruleResource == ResourceAll {
			return true
		}
		if ruleResource == requestedResource {
			return true
		}
	}

	return false
}

func ResourceNameMatches(rule *PolicyRule, requestedName string) bool {
	if len(rule.ResourceNames) == 0 {
		return true
	}

	for _, ruleName := range rule.ResourceNames {
		if ruleName == requestedName {
			return true
		}
	}

	return false
}

func NonResourceURLMatches(rule *PolicyRule, requestedURL string) bool {
	for _, ruleURL := range rule.NonResourceURLs {
		if ruleURL == NonResourceAll {
			return true
		}
		if ruleURL == requestedURL {
			return true
		}
		if strings.HasSuffix(ruleURL, "*") && strings.HasPrefix(requestedURL, strings.TrimRight(ruleURL, "*")) {
			return true
		}
	}

	return false
}

// subjectsStrings returns users, groups, serviceaccounts, unknown for display purposes.
func SubjectsStrings(subjects []Subject) ([]string, []string, []string, []string) {
	users := []string{}
	groups := []string{}
	sas := []string{}
	others := []string{}

	for _, subject := range subjects {
		switch subject.Kind {
		case ServiceAccountKind:
			sas = append(sas, fmt.Sprintf("%s/%s", subject.Namespace, subject.Name))

		case UserKind:
			users = append(users, subject.Name)

		case GroupKind:
			groups = append(groups, subject.Name)

		default:
			others = append(others, fmt.Sprintf("%s/%s/%s", subject.Kind, subject.Namespace, subject.Name))
		}
	}

	return users, groups, sas, others
}

// +k8s:deepcopy-gen=false
// PolicyRuleBuilder let's us attach methods.  A no-no for API types.
// We use it to construct rules in code.  It's more compact than trying to write them
// out in a literal and allows us to perform some basic checking during construction
type PolicyRuleBuilder struct {
	PolicyRule PolicyRule
}

func NewRule(verbs ...string) *PolicyRuleBuilder {
	return &PolicyRuleBuilder{
		PolicyRule: PolicyRule{Verbs: sets.NewString(verbs...).List()},
	}
}

func (r *PolicyRuleBuilder) Groups(groups ...string) *PolicyRuleBuilder {
	r.PolicyRule.APIGroups = combine(r.PolicyRule.APIGroups, groups)
	return r
}

func (r *PolicyRuleBuilder) Resources(resources ...string) *PolicyRuleBuilder {
	r.PolicyRule.Resources = combine(r.PolicyRule.Resources, resources)
	return r
}

func (r *PolicyRuleBuilder) Names(names ...string) *PolicyRuleBuilder {
	r.PolicyRule.ResourceNames = combine(r.PolicyRule.ResourceNames, names)
	return r
}

func (r *PolicyRuleBuilder) URLs(urls ...string) *PolicyRuleBuilder {
	r.PolicyRule.NonResourceURLs = combine(r.PolicyRule.NonResourceURLs, urls)
	return r
}

func (r *PolicyRuleBuilder) RuleOrDie() PolicyRule {
	ret, err := r.Rule()
	if err != nil {
		panic(err)
	}
	return ret
}

func combine(s1, s2 []string) []string {
	s := sets.NewString(s1...)
	s.Insert(s2...)
	return s.List()
}

func (r *PolicyRuleBuilder) Rule() (PolicyRule, error) {
	if len(r.PolicyRule.Verbs) == 0 {
		return PolicyRule{}, fmt.Errorf("verbs are required: %#v", r.PolicyRule)
	}

	switch {
	case len(r.PolicyRule.NonResourceURLs) > 0:
		if len(r.PolicyRule.APIGroups) != 0 || len(r.PolicyRule.Resources) != 0 || len(r.PolicyRule.ResourceNames) != 0 {
			return PolicyRule{}, fmt.Errorf("non-resource rule may not have apiGroups, resources, or resourceNames: %#v", r.PolicyRule)
		}
	case len(r.PolicyRule.Resources) > 0:
		if len(r.PolicyRule.NonResourceURLs) != 0 {
			return PolicyRule{}, fmt.Errorf("resource rule may not have nonResourceURLs: %#v", r.PolicyRule)
		}
		if len(r.PolicyRule.APIGroups) == 0 {
			// this a common bug
			return PolicyRule{}, fmt.Errorf("resource rule must have apiGroups: %#v", r.PolicyRule)
		}
	default:
		return PolicyRule{}, fmt.Errorf("a rule must have either nonResourceURLs or resources: %#v", r.PolicyRule)
	}

	return r.PolicyRule, nil
}

// +k8s:deepcopy-gen=false
// ClusterRoleBindingBuilder let's us attach methods.  A no-no for API types.
// We use it to construct bindings in code.  It's more compact than trying to write them
// out in a literal.
type ClusterRoleBindingBuilder struct {
	ClusterRoleBinding ClusterRoleBinding
}

func NewClusterBinding(clusterRoleName string) *ClusterRoleBindingBuilder {
	return &ClusterRoleBindingBuilder{
		ClusterRoleBinding: ClusterRoleBinding{
			ObjectMeta: metav1.ObjectMeta{Name: clusterRoleName},
			RoleRef: RoleRef{
				APIGroup: GroupName,
				Kind:     "ClusterRole",
				Name:     clusterRoleName,
			},
		},
	}
}

func (r *ClusterRoleBindingBuilder) Groups(groups ...string) *ClusterRoleBindingBuilder {
	for _, group := range groups {
		r.ClusterRoleBinding.Subjects = append(r.ClusterRoleBinding.Subjects, Subject{Kind: GroupKind, APIGroup: GroupName, Name: group})
	}
	return r
}

func (r *ClusterRoleBindingBuilder) Users(users ...string) *ClusterRoleBindingBuilder {
	for _, user := range users {
		r.ClusterRoleBinding.Subjects = append(r.ClusterRoleBinding.Subjects, Subject{Kind: UserKind, APIGroup: GroupName, Name: user})
	}
	return r
}

func (r *ClusterRoleBindingBuilder) SAs(namespace string, serviceAccountNames ...string) *ClusterRoleBindingBuilder {
	for _, saName := range serviceAccountNames {
		r.ClusterRoleBinding.Subjects = append(r.ClusterRoleBinding.Subjects, Subject{Kind: ServiceAccountKind, Namespace: namespace, Name: saName})
	}
	return r
}

func (r *ClusterRoleBindingBuilder) BindingOrDie() ClusterRoleBinding {
	ret, err := r.Binding()
	if err != nil {
		panic(err)
	}
	return ret
}

func (r *ClusterRoleBindingBuilder) Binding() (ClusterRoleBinding, error) {
	if len(r.ClusterRoleBinding.Subjects) == 0 {
		return ClusterRoleBinding{}, fmt.Errorf("subjects are required: %#v", r.ClusterRoleBinding)
	}

	return r.ClusterRoleBinding, nil
}

// +k8s:deepcopy-gen=false
// RoleBindingBuilder let's us attach methods. It is similar to
// ClusterRoleBindingBuilder above.
type RoleBindingBuilder struct {
	RoleBinding RoleBinding
}

// NewRoleBinding creates a RoleBinding builder that can be used
// to define the subjects of a role binding. At least one of
// the `Groups`, `Users` or `SAs` method must be called before
// calling the `Binding*` methods.
func NewRoleBinding(roleName, namespace string) *RoleBindingBuilder {
	return &RoleBindingBuilder{
		RoleBinding: RoleBinding{
			ObjectMeta: metav1.ObjectMeta{
				Name:      roleName,
				Namespace: namespace,
			},
			RoleRef: RoleRef{
				APIGroup: GroupName,
				Kind:     "Role",
				Name:     roleName,
			},
		},
	}
}

func NewRoleBindingForClusterRole(roleName, namespace string) *RoleBindingBuilder {
	return &RoleBindingBuilder{
		RoleBinding: RoleBinding{
			ObjectMeta: metav1.ObjectMeta{
				Name:      roleName,
				Namespace: namespace,
			},
			RoleRef: RoleRef{
				APIGroup: GroupName,
				Kind:     "ClusterRole",
				Name:     roleName,
			},
		},
	}
}

// Groups adds the specified groups as the subjects of the RoleBinding.
func (r *RoleBindingBuilder) Groups(groups ...string) *RoleBindingBuilder {
	for _, group := range groups {
		r.RoleBinding.Subjects = append(r.RoleBinding.Subjects, Subject{Kind: GroupKind, Name: group})
	}
	return r
}

// Users adds the specified users as the subjects of the RoleBinding.
func (r *RoleBindingBuilder) Users(users ...string) *RoleBindingBuilder {
	for _, user := range users {
		r.RoleBinding.Subjects = append(r.RoleBinding.Subjects, Subject{Kind: UserKind, Name: user})
	}
	return r
}

// SAs adds the specified service accounts as the subjects of the
// RoleBinding.
func (r *RoleBindingBuilder) SAs(namespace string, serviceAccountNames ...string) *RoleBindingBuilder {
	for _, saName := range serviceAccountNames {
		r.RoleBinding.Subjects = append(r.RoleBinding.Subjects, Subject{Kind: ServiceAccountKind, Namespace: namespace, Name: saName})
	}
	return r
}

// BindingOrDie calls the binding method and panics if there is an error.
func (r *RoleBindingBuilder) BindingOrDie() RoleBinding {
	ret, err := r.Binding()
	if err != nil {
		panic(err)
	}
	return ret
}

// Binding builds and returns the RoleBinding API object from the builder
// object.
func (r *RoleBindingBuilder) Binding() (RoleBinding, error) {
	if len(r.RoleBinding.Subjects) == 0 {
		return RoleBinding{}, fmt.Errorf("subjects are required: %#v", r.RoleBinding)
	}

	return r.RoleBinding, nil
}
<|endoftext|>


<commit_before>package main

import (
  "net/http"
  "log"
  "io/ioutil"
  "encoding/json"
  "os/exec"
  "flag"
  "strconv"
  "crypto/md5"
  "text/template"
  "bytes"
  "gopkg.in/fsnotify.v1"
)

// A map of repos => (a map of branches => (a list of actions))
// Todo: Idiomatic way of doing this?
type Config map[string]map[string][]string

type HookMsg struct {
  CanonicalUrl string `json:"canon_url"`

  Commits []struct {
    Branch string
  }

  Repository struct {
    AbsoluteUrl string `json:"absolute_url"`
  }
}

type Commit struct {
  Repository string
  Branch string
}

type Job struct {
  Commit Commit
  Action string
}

var listenPort = flag.Int("port", 8080, "portnumber to listen on")
var configFile = flag.String("config", "golive.json", "the configfile to read")
var verbose = flag.Bool("v", false, "print more output")
var bootstrap = flag.Bool("bootstrap", false, "run all non-wildcard jobs on startup")

var jobTemplates = make(map[[16]byte]template.Template)
var config Config

func main() {
	flag.Parse()

  parseConfig(*configFile)
  go watchConfig(*configFile)

  msgs := make(chan HookMsg, 100)
  commits := make(chan Commit, 100)
  jobs := make(chan Job, 100)
  actions := make(chan string, 100)

  if *bootstrap {
    performBootstrap(config, jobs)
  }

  go hookWrangler(msgs, commits)
  go commitWrangler(commits, jobs, config)
  go jobWrangler(jobs, actions)
  go actionRunner(actions)

  log.Print("Starting golive server on port ", *listenPort)

  http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
    log.Print("Received request")

    payload := r.FormValue("payload")

    var hookmsg HookMsg
    if err := json.Unmarshal([]byte(payload), &hookmsg); err != nil {
      http.Error(w, "Could not decode json", 500)
      log.Fatal(err)
    }

    if *verbose {
      log.Print("Received commit: ", hookmsg)
    }

    msgs <- hookmsg
  })

  log.Fatal(http.ListenAndServe(":" + strconv.Itoa(*listenPort), nil))
}

func performBootstrap(config Config, results chan<- Job) {
  if *verbose {
    log.Println("Starting bootstrap")
  }

  for repository, branches := range config {
    for branch, actions := range branches {
      if (branch == "*") {
        continue
      }

      for _, action := range actions {
        fakecommit := Commit{ repository, branch }

        if *verbose {
          log.Println("Creating bootstrap job: ", fakecommit, action)
        }

        results <- Job{fakecommit, string(action)}
      }
    }
  }

  if *verbose {
    log.Println("Bootstrap done")
  }
}

func watchConfig(configFile string) {
  watcher, err := fsnotify.NewWatcher()
  if err != nil {
    log.Fatal(err)
  }

  log.Print("Watching config file: ", configFile)

  defer watcher.Close()

	done := make(chan bool)

  go func() {
    for {
      select {
        case event := <-watcher.Events:
          if event.Name == "" {
            continue
          }

          if *verbose {
            log.Println(event.Name, event.Op, event)
          }

          // The watching doesn't seem to follow renames, which happen i.e.
          // during Vim editing, so set up a new one
          if event.Op & fsnotify.Rename == fsnotify.Rename {
            addWatcher(watcher, configFile)
          }

          log.Print("Relading config file ", configFile)

          parseConfig(configFile)

        case err := <-watcher.Errors:
          if err == nil {
            continue
          }

          done <- true
          log.Fatal("Error when watching config file: ", err)
      }
    }
  }()

  addWatcher(watcher, configFile)

  <- done
}

func addWatcher(watcher *fsnotify.Watcher, file string) {
  err := watcher.Add(file)
  if err != nil {
    log.Fatal(err)
  }
}

func parseConfig(configFile string) {
    config_raw, err := ioutil.ReadFile(configFile)
    if err != nil {
      log.Fatal(err)
    }

    var newConfig Config
    json.Unmarshal(config_raw, &newConfig)

    config = newConfig

    if *verbose {
      log.Print("Loaded config: ", config)
    }
}

func hookWrangler(msgs <-chan HookMsg, results chan<- Commit) {
  for msg := range msgs {
    repository := msg.CanonicalUrl + msg.Repository.AbsoluteUrl

    for _, commit := range msg.Commits {
      results <- Commit{ repository, commit.Branch}
    }
  }
}

func commitWrangler(commits <-chan Commit, results chan<- Job, config Config) {
  for commit := range commits {
    if *verbose {
      log.Print("Received commit struct: ", commit)
    }

    if commit.Branch == "" || commit.Repository == "" {
      continue
    }

    if branches, ok := config[commit.Repository]; ok {
      if *verbose {
        log.Print("Repository present in config: ", commit.Repository)
      }

      runactions := make([]string, 0)

      if actions, ok := branches[commit.Branch]; ok {
        if *verbose {
          log.Print("Branch present in config: ", commit.Branch)
        }

        runactions = actions
      } else if actions, ok := branches["*"]; ok {
        if *verbose {
          log.Print("Wildcard branch present in config: ", commit.Branch)
        }

        runactions = actions
      }

      for _, action := range runactions {
        results <- Job{commit, string(action)}
      }
    }
  }
}

func jobWrangler(jobs <-chan Job, actions chan<- string) {
  for job := range jobs {
    hash := md5.Sum([]byte(job.Action))

    if _, ok := jobTemplates[hash]; !ok {
      log.Print("Template not found, compiling: ", job.Action)

      tpl, err := template.New(string(hash[:])).Parse(job.Action)
      if err != nil {
        log.Fatal("Could not compile template: ", job.Action, " - ", err)
      }

      jobTemplates[hash] = *tpl
    }

    tpl := jobTemplates[hash]

    var buff bytes.Buffer
    (&tpl).Execute(&buff, job.Commit)
    str := buff.String()

    actions <- str
  }
}

func actionRunner(actions <-chan string) {
  for action := range actions {
    if *verbose {
      log.Print("Running action: ", action)
    }

    command := exec.Command("bash", "-c", action)

    command.Run()
  }
}
<commit_msg>Log more stuff when JSON decoding fails<commit_after>package main

import (
  "net/http"
  "log"
  "io/ioutil"
  "encoding/json"
  "os/exec"
  "flag"
  "strconv"
  "crypto/md5"
  "text/template"
  "bytes"
  "gopkg.in/fsnotify.v1"
)

// A map of repos => (a map of branches => (a list of actions))
// Todo: Idiomatic way of doing this?
type Config map[string]map[string][]string

type HookMsg struct {
  CanonicalUrl string `json:"canon_url"`

  Commits []struct {
    Branch string
  }

  Repository struct {
    AbsoluteUrl string `json:"absolute_url"`
  }
}

type Commit struct {
  Repository string
  Branch string
}

type Job struct {
  Commit Commit
  Action string
}

var listenPort = flag.Int("port", 8080, "portnumber to listen on")
var configFile = flag.String("config", "golive.json", "the configfile to read")
var verbose = flag.Bool("v", false, "print more output")
var bootstrap = flag.Bool("bootstrap", false, "run all non-wildcard jobs on startup")

var jobTemplates = make(map[[16]byte]template.Template)
var config Config

func main() {
	flag.Parse()

  parseConfig(*configFile)
  go watchConfig(*configFile)

  msgs := make(chan HookMsg, 100)
  commits := make(chan Commit, 100)
  jobs := make(chan Job, 100)
  actions := make(chan string, 100)

  if *bootstrap {
    performBootstrap(config, jobs)
  }

  go hookWrangler(msgs, commits)
  go commitWrangler(commits, jobs, config)
  go jobWrangler(jobs, actions)
  go actionRunner(actions)

  log.Print("Starting golive server on port ", *listenPort)

  http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
    log.Print("Received request")

    payload := r.FormValue("payload")

    var hookmsg HookMsg
    if err := json.Unmarshal([]byte(payload), &hookmsg); err != nil {
      http.Error(w, "Could not decode json", 500)
      log.Fatal(err, " Payload: ", payload)
    }

    if *verbose {
      log.Print("Received commit: ", hookmsg)
    }

    msgs <- hookmsg
  })

  log.Fatal(http.ListenAndServe(":" + strconv.Itoa(*listenPort), nil))
}

func performBootstrap(config Config, results chan<- Job) {
  if *verbose {
    log.Println("Starting bootstrap")
  }

  for repository, branches := range config {
    for branch, actions := range branches {
      if (branch == "*") {
        continue
      }

      for _, action := range actions {
        fakecommit := Commit{ repository, branch }

        if *verbose {
          log.Println("Creating bootstrap job: ", fakecommit, action)
        }

        results <- Job{fakecommit, string(action)}
      }
    }
  }

  if *verbose {
    log.Println("Bootstrap done")
  }
}

func watchConfig(configFile string) {
  watcher, err := fsnotify.NewWatcher()
  if err != nil {
    log.Fatal(err)
  }

  log.Print("Watching config file: ", configFile)

  defer watcher.Close()

	done := make(chan bool)

  go func() {
    for {
      select {
        case event := <-watcher.Events:
          if event.Name == "" {
            continue
          }

          if *verbose {
            log.Println(event.Name, event.Op, event)
          }

          // The watching doesn't seem to follow renames, which happen i.e.
          // during Vim editing, so set up a new one
          if event.Op & fsnotify.Rename == fsnotify.Rename {
            addWatcher(watcher, configFile)
          }

          log.Print("Relading config file ", configFile)

          parseConfig(configFile)

        case err := <-watcher.Errors:
          if err == nil {
            continue
          }

          done <- true
          log.Fatal("Error when watching config file: ", err)
      }
    }
  }()

  addWatcher(watcher, configFile)

  <- done
}

func addWatcher(watcher *fsnotify.Watcher, file string) {
  err := watcher.Add(file)
  if err != nil {
    log.Fatal(err)
  }
}

func parseConfig(configFile string) {
    config_raw, err := ioutil.ReadFile(configFile)
    if err != nil {
      log.Fatal(err)
    }

    var newConfig Config
    json.Unmarshal(config_raw, &newConfig)

    config = newConfig

    if *verbose {
      log.Print("Loaded config: ", config)
    }
}

func hookWrangler(msgs <-chan HookMsg, results chan<- Commit) {
  for msg := range msgs {
    repository := msg.CanonicalUrl + msg.Repository.AbsoluteUrl

    for _, commit := range msg.Commits {
      results <- Commit{ repository, commit.Branch}
    }
  }
}

func commitWrangler(commits <-chan Commit, results chan<- Job, config Config) {
  for commit := range commits {
    if *verbose {
      log.Print("Received commit struct: ", commit)
    }

    if commit.Branch == "" || commit.Repository == "" {
      continue
    }

    if branches, ok := config[commit.Repository]; ok {
      if *verbose {
        log.Print("Repository present in config: ", commit.Repository)
      }

      runactions := make([]string, 0)

      if actions, ok := branches[commit.Branch]; ok {
        if *verbose {
          log.Print("Branch present in config: ", commit.Branch)
        }

        runactions = actions
      } else if actions, ok := branches["*"]; ok {
        if *verbose {
          log.Print("Wildcard branch present in config: ", commit.Branch)
        }

        runactions = actions
      }

      for _, action := range runactions {
        results <- Job{commit, string(action)}
      }
    }
  }
}

func jobWrangler(jobs <-chan Job, actions chan<- string) {
  for job := range jobs {
    hash := md5.Sum([]byte(job.Action))

    if _, ok := jobTemplates[hash]; !ok {
      log.Print("Template not found, compiling: ", job.Action)

      tpl, err := template.New(string(hash[:])).Parse(job.Action)
      if err != nil {
        log.Fatal("Could not compile template: ", job.Action, " - ", err)
      }

      jobTemplates[hash] = *tpl
    }

    tpl := jobTemplates[hash]

    var buff bytes.Buffer
    (&tpl).Execute(&buff, job.Commit)
    str := buff.String()

    actions <- str
  }
}

func actionRunner(actions <-chan string) {
  for action := range actions {
    if *verbose {
      log.Print("Running action: ", action)
    }

    command := exec.Command("bash", "-c", action)

    command.Run()
  }
}
<|endoftext|>


<commit_before>//
// Copyright 2022 The GUAC Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package assembler

import (
	"fmt"
	"strings"

	"github.com/neo4j/neo4j-go-driver/v4/neo4j"
	"github.com/guacsec/guac/pkg/assembler/graphdb"
)

// Note: This module is experimental and might change often!

// StoreSubgraph stores a Graph to the graph database given by Client
func StoreGraph(g Graph, client graphdb.Client) error {
	session := client.NewSession(neo4j.SessionConfig{})
	defer session.Close()

	node_queries := make([]string, len(g.Nodes))
	node_dicts := make([]map[string]interface{}, len(g.Nodes))
	for i, n := range g.Nodes {
		var sb strings.Builder
		if err := queryPartForMergeNode(&sb, n, "n"); err != nil {
			return err
		}
		queryPartForNodeAttributes(&sb, true, n, "n")
		queryPartForNodeAttributes(&sb, false, n, "n")
		node_queries[i] = sb.String()
		node_dicts[i] = map[string]interface{}{}
		for k, v := range n.Properties() {
			node_dicts[i]["n_" + k] = v
		}
	}

	edge_queries := make([]string, len(g.Edges))
	edge_dicts := make([]map[string]interface{}, len(g.Nodes))
	for i, e := range g.Edges {
		a, b := e.Nodes()
		var sb strings.Builder
		if err := queryPartForMergeNode(&sb, a, "a"); err != nil {
			return err
		}
		if err := queryPartForMergeNode(&sb, b, "b"); err != nil {
			return err
		}
		queryPartForEdgeConnection(&sb, e)
		edge_queries[i] = sb.String()
		edge_dicts[i] = map[string]interface{}{}
		for k, v := range a.Properties() {
			edge_dicts[i]["a_" + k] = v
		}
		for k, v := range b.Properties() {
			edge_dicts[i]["b_" + k] = v
		}
		for k, v := range e.Properties() {
			edge_dicts[i]["e_" + k] = v
		}
	}

	queries := append(node_queries, edge_queries...)
	params := append(node_dicts, edge_dicts...)
	_, err := session.WriteTransaction(
		func (tx graphdb.Transaction) (interface{}, error) {
			for i, query := range queries {
				fmt.Printf("%v(where: %v)\n\n", query, params[i])
				if _, err := tx.Run(query, params[i]); err != nil {
					return nil, err
				}
			}
			return nil, nil
		})

	return err
}

// Creates the "MERGE (n:${NODE_TYPE} {${ATTR}:${VALUE}, ...})" part of the query
func queryPartForMergeNode(sb *strings.Builder, n GuacNode, label string) error {
	node_data := n.Properties()
	sb.WriteString("MERGE (")
	sb.WriteString(label)    // not user controlled
	sb.WriteString(":")
	sb.WriteString(n.Type()) // not user controlled
	sb.WriteString(" {")
	for ix, key := range n.IdentifiablePropertyNames() {
		if _, ok := node_data[key]; ok {
			writeKeyValToQuery(sb, key, label, false, ix == 0)
		} else {
			return fmt.Errorf("Node %v has no value for property %v", n, key)
		}
	}
	sb.WriteString("})\n")

	return nil
}

// Creates the "ON CREATE SET ${ATTR}=${VALUE}, ..." part of the query
// Creates the "ON MATCH SET ${ATTR}=${VALUE}, ..." part of the query
func queryPartForNodeAttributes(sb *strings.Builder, onCreate bool, n GuacNode, label string) {
	node_data := n.Properties()
	if onCreate {
		sb.WriteString("ON CREATE SET ")
	} else {
		sb.WriteString("ON MATCH SET ")
	}
	first := true
	for key := range node_data {
		writeKeyValToQuery(sb, key, label, true, first)
		first = false
	}
	sb.WriteString("\n")
}

// Creates the "(a) -[e:${EDGE_TYPE}] -> (b)" part of the query and sets the edge attributes
func queryPartForEdgeConnection(sb *strings.Builder, e GuacEdge) {
	sb.WriteString("MERGE (a) -[e:")
	sb.WriteString(e.Type()) // not user controlled
	sb.WriteString("]-> (b)")
	if edge_data := e.Properties(); len(edge_data) > 0 {
		sb.WriteString("\nSET ")
		first := true
		for key := range edge_data {
			writeKeyValToQuery(sb, key, "e", true, first)
			first = false
		}
	}
	sb.WriteString("\n")
}

// Creates either the "${ATTR}:${VALUE}" part (set=false) or the "n.${ATTR}=${VALUE}" one (set=true).
// Uses first to determine if we need to add comma from what comes before
func writeKeyValToQuery(sb *strings.Builder, key string, label string, set bool, first bool) {
	if !first {
		sb.WriteString(", ")
	}
	if set {
		sb.WriteString(label) // not user controlled
		sb.WriteString(".")
	}
	sb.WriteString(key) // not user controlled
	if set {
		sb.WriteString("=$")
	} else {
		sb.WriteString(":$")
	}
	sb.WriteString(label) // not user controlled
	sb.WriteString("_")
	sb.WriteString(key) // not user controlled, will be as a prepared statement parameter
}
<commit_msg>Fix fmt/goimports<commit_after>//
// Copyright 2022 The GUAC Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package assembler

import (
	"fmt"
	"strings"

	"github.com/guacsec/guac/pkg/assembler/graphdb"
	"github.com/neo4j/neo4j-go-driver/v4/neo4j"
)

// Note: This module is experimental and might change often!

// StoreSubgraph stores a Graph to the graph database given by Client
func StoreGraph(g Graph, client graphdb.Client) error {
	session := client.NewSession(neo4j.SessionConfig{})
	defer session.Close()

	node_queries := make([]string, len(g.Nodes))
	node_dicts := make([]map[string]interface{}, len(g.Nodes))
	for i, n := range g.Nodes {
		var sb strings.Builder
		if err := queryPartForMergeNode(&sb, n, "n"); err != nil {
			return err
		}
		queryPartForNodeAttributes(&sb, true, n, "n")
		queryPartForNodeAttributes(&sb, false, n, "n")
		node_queries[i] = sb.String()
		node_dicts[i] = map[string]interface{}{}
		for k, v := range n.Properties() {
			node_dicts[i]["n_"+k] = v
		}
	}

	edge_queries := make([]string, len(g.Edges))
	edge_dicts := make([]map[string]interface{}, len(g.Nodes))
	for i, e := range g.Edges {
		a, b := e.Nodes()
		var sb strings.Builder
		if err := queryPartForMergeNode(&sb, a, "a"); err != nil {
			return err
		}
		if err := queryPartForMergeNode(&sb, b, "b"); err != nil {
			return err
		}
		queryPartForEdgeConnection(&sb, e)
		edge_queries[i] = sb.String()
		edge_dicts[i] = map[string]interface{}{}
		for k, v := range a.Properties() {
			edge_dicts[i]["a_"+k] = v
		}
		for k, v := range b.Properties() {
			edge_dicts[i]["b_"+k] = v
		}
		for k, v := range e.Properties() {
			edge_dicts[i]["e_"+k] = v
		}
	}

	queries := append(node_queries, edge_queries...)
	params := append(node_dicts, edge_dicts...)
	_, err := session.WriteTransaction(
		func(tx graphdb.Transaction) (interface{}, error) {
			for i, query := range queries {
				fmt.Printf("%v(where: %v)\n\n", query, params[i])
				if _, err := tx.Run(query, params[i]); err != nil {
					return nil, err
				}
			}
			return nil, nil
		})

	return err
}

// Creates the "MERGE (n:${NODE_TYPE} {${ATTR}:${VALUE}, ...})" part of the query
func queryPartForMergeNode(sb *strings.Builder, n GuacNode, label string) error {
	node_data := n.Properties()
	sb.WriteString("MERGE (")
	sb.WriteString(label) // not user controlled
	sb.WriteString(":")
	sb.WriteString(n.Type()) // not user controlled
	sb.WriteString(" {")
	for ix, key := range n.IdentifiablePropertyNames() {
		if _, ok := node_data[key]; ok {
			writeKeyValToQuery(sb, key, label, false, ix == 0)
		} else {
			return fmt.Errorf("Node %v has no value for property %v", n, key)
		}
	}
	sb.WriteString("})\n")

	return nil
}

// Creates the "ON CREATE SET ${ATTR}=${VALUE}, ..." part of the query
// Creates the "ON MATCH SET ${ATTR}=${VALUE}, ..." part of the query
func queryPartForNodeAttributes(sb *strings.Builder, onCreate bool, n GuacNode, label string) {
	node_data := n.Properties()
	if onCreate {
		sb.WriteString("ON CREATE SET ")
	} else {
		sb.WriteString("ON MATCH SET ")
	}
	first := true
	for key := range node_data {
		writeKeyValToQuery(sb, key, label, true, first)
		first = false
	}
	sb.WriteString("\n")
}

// Creates the "(a) -[e:${EDGE_TYPE}] -> (b)" part of the query and sets the edge attributes
func queryPartForEdgeConnection(sb *strings.Builder, e GuacEdge) {
	sb.WriteString("MERGE (a) -[e:")
	sb.WriteString(e.Type()) // not user controlled
	sb.WriteString("]-> (b)")
	if edge_data := e.Properties(); len(edge_data) > 0 {
		sb.WriteString("\nSET ")
		first := true
		for key := range edge_data {
			writeKeyValToQuery(sb, key, "e", true, first)
			first = false
		}
	}
	sb.WriteString("\n")
}

// Creates either the "${ATTR}:${VALUE}" part (set=false) or the "n.${ATTR}=${VALUE}" one (set=true).
// Uses first to determine if we need to add comma from what comes before
func writeKeyValToQuery(sb *strings.Builder, key string, label string, set bool, first bool) {
	if !first {
		sb.WriteString(", ")
	}
	if set {
		sb.WriteString(label) // not user controlled
		sb.WriteString(".")
	}
	sb.WriteString(key) // not user controlled
	if set {
		sb.WriteString("=$")
	} else {
		sb.WriteString(":$")
	}
	sb.WriteString(label) // not user controlled
	sb.WriteString("_")
	sb.WriteString(key) // not user controlled, will be as a prepared statement parameter
}
<|endoftext|>


<commit_before>package gomapr

import (
	"errors"
	"log"
	"reflect"
	"sync"
)

var (
	EndOfEmit = errors.New("Nothing left to emit")
)

type Event interface{}
type Partial interface{}
type ReduceKey interface{}

type MapReduce interface {
	Emit() (Event, error)
	Map(Event) (ReduceKey, Partial)
	Reduce(ReduceKey, []Partial) (ReduceKey, Partial)
}

// Corresponds to a set of values that a reducer can join.
type partialGroup struct {
	values []Partial
	mutex  *sync.Mutex
}

func newPartialGroup() *partialGroup {
	return &partialGroup{
		values: make([]Partial, 0),
		mutex:  &sync.Mutex{},
	}
}

// Adds a value to the group.
func (p *partialGroup) add(v interface{}) {
	p.values = append(p.values, v)
}

// Replaces the contents of the partial group.
func (p *partialGroup) replace(v interface{}) {
	p.values = []Partial{v}
}

// Contains all partial groups.
type reduceWorkspace struct {
	groups map[ReduceKey]*partialGroup
	mutex  *sync.Mutex
}

func newReduceWorkspace() *reduceWorkspace {
	return &reduceWorkspace{
		make(map[ReduceKey]*partialGroup),
		&sync.Mutex{},
	}
}

// Returns a partial group by its key.
func (r *reduceWorkspace) getPartialGroup(key ReduceKey) *partialGroup {
	r.mutex.Lock()
	defer r.mutex.Unlock()

	partialGroup, ok := r.groups[key]
	if !ok {
		partialGroup = newPartialGroup()
		r.groups[key] = partialGroup
	}

	return partialGroup
}

// Adds a key-value pair to its appropriate partial group.
func (r *reduceWorkspace) add(key ReduceKey, value Partial) {
	partialGroup := r.getPartialGroup(key)

	partialGroup.mutex.Lock()
	partialGroup.add(value)
	partialGroup.mutex.Unlock()
}

// Replaces an existing partial group with the input arguments.
func (r *reduceWorkspace) replace(key ReduceKey, value Partial) {
	partialGroup := r.getPartialGroup(key)
	partialGroup.replace(value)
}

// Contains configuration for a MapReduce task.
type Runner struct {
	reduceWorkspace *reduceWorkspace
	mr              MapReduce
	wg              *sync.WaitGroup
}

func NewRunner(m MapReduce) *Runner {
	return &Runner{
		reduceWorkspace: newReduceWorkspace(),
		mr:              m,
		wg:              &sync.WaitGroup{},
	}
}

// Returns the map containing all groups.
func (r *Runner) Groups() map[ReduceKey]Partial {
	groups := make(map[ReduceKey]Partial)
	for k, v := range r.reduceWorkspace.groups {
		groups[k] = v.values[0]
	}
	return groups
}

// Maps the input it receives on its emitted channel, spawning
// a reduce task when appropriate.
func (r *Runner) mapWorker(emitted chan Event) {
	for val := range emitted {
		key, mapped := r.mr.Map(val)
		r.reduceWorkspace.add(key, mapped)
		r.wg.Add(1)
		go r.reduce(key)
	}

	r.wg.Done()
}

// Reduces the partial group with the matching input key.
func (r *Runner) reduce(key ReduceKey) {
	partialGroup := r.reduceWorkspace.getPartialGroup(key)

	partialGroup.mutex.Lock()
	defer partialGroup.mutex.Unlock()

	if len(partialGroup.values) > 1 {
		newKey, partial := r.mr.Reduce(key, partialGroup.values)

		if reflect.DeepEqual(key, newKey) {
			partialGroup.replace(partial)
		} else {
			r.reduceWorkspace.replace(key, partial)
		}
	}

	r.wg.Done()
}

// Starts the MapReduce task.
func (r *Runner) Run(mappers int) {
	emit := make(chan Event, mappers)

	// Create background mapping workers.
	for i := 0; i < mappers; i++ {
		r.wg.Add(1)
		go r.mapWorker(emit)
	}

	// Emit all events.
	go func() {
		for {
			emitted, err := r.mr.Emit()
			if err != EndOfEmit && err != nil {
				log.Printf("Error emitting: %v", err)
				break
			}

			emit <- emitted

			if err == EndOfEmit {
				break
			}

		}
		close(emit)
	}()

	r.wg.Wait()
}
<commit_msg>Cleaner locking code<commit_after>package gomapr

import (
	"errors"
	"log"
	"reflect"
	"sync"
)

var (
	EndOfEmit = errors.New("Nothing left to emit")
)

type Event interface{}
type Partial interface{}
type ReduceKey interface{}

type MapReduce interface {
	Emit() (Event, error)
	Map(Event) (ReduceKey, Partial)
	Reduce(ReduceKey, []Partial) (ReduceKey, Partial)
}

// Corresponds to a set of values that a reducer can join.
type partialGroup struct {
	values []Partial
	l      *sync.RWMutex
}

func newPartialGroup() *partialGroup {
	return &partialGroup{
		values: make([]Partial, 0),
		l:      &sync.RWMutex{},
	}
}

// Adds a value to the group.
func (p *partialGroup) add(v interface{}) {
	p.values = append(p.values, v)
}

// Replaces the contents of the partial group.
func (p *partialGroup) replace(v interface{}) {
	p.values = []Partial{v}
}

// Contains all partial groups.
type reduceWorkspace struct {
	groups map[ReduceKey]*partialGroup
	l      *sync.RWMutex
}

func newReduceWorkspace() *reduceWorkspace {
	return &reduceWorkspace{
		make(map[ReduceKey]*partialGroup),
		&sync.RWMutex{},
	}
}

// Returns a partial group by its key.
func (r *reduceWorkspace) getPartialGroup(key ReduceKey) *partialGroup {
	r.l.Lock()
	defer r.l.Unlock()

	partialGroup, ok := r.groups[key]
	if !ok {
		partialGroup = newPartialGroup()
		r.groups[key] = partialGroup
	}

	return partialGroup
}

// Adds a key-value pair to its appropriate partial group.
func (r *reduceWorkspace) add(key ReduceKey, value Partial) {
	partialGroup := r.getPartialGroup(key)

	partialGroup.l.Lock()
	partialGroup.add(value)
	partialGroup.l.Unlock()
}

// Replaces an existing partial group with the input arguments.
func (r *reduceWorkspace) replace(key ReduceKey, value Partial) {
	partialGroup := r.getPartialGroup(key)

	partialGroup.l.Lock()
	partialGroup.replace(value)
	partialGroup.l.Unlock()
}

// Contains configuration for a MapReduce task.
type Runner struct {
	reduceWorkspace *reduceWorkspace
	mr              MapReduce
	wg              *sync.WaitGroup
}

func NewRunner(m MapReduce) *Runner {
	return &Runner{
		reduceWorkspace: newReduceWorkspace(),
		mr:              m,
		wg:              &sync.WaitGroup{},
	}
}

// Returns the map containing all groups. Only safe to call after
// the task has completed.
func (r *Runner) Groups() map[ReduceKey]Partial {
	r.reduceWorkspace.l.Lock()
	defer r.reduceWorkspace.l.Unlock()

	groups := make(map[ReduceKey]Partial)

	for k, v := range r.reduceWorkspace.groups {
		groups[k] = v.values[0]
	}

	return groups
}

// Maps the input it receives on its emitted channel, spawning
// a reduce task when appropriate.
func (r *Runner) mapWorker(emitted chan Event) {
	for val := range emitted {
		key, mapped := r.mr.Map(val)
		r.reduceWorkspace.add(key, mapped)
		r.wg.Add(1)
		go r.reduce(key)
	}

	r.wg.Done()
}

// Reduces the partial group with the matching input key.
func (r *Runner) reduce(key ReduceKey) {
	partialGroup := r.reduceWorkspace.getPartialGroup(key)

	partialGroup.l.Lock()
	defer partialGroup.l.Unlock()

	if len(partialGroup.values) > 1 {
		newKey, partial := r.mr.Reduce(key, partialGroup.values)

		if reflect.DeepEqual(key, newKey) {
			partialGroup.replace(partial)
		} else {
			r.reduceWorkspace.replace(key, partial)
		}
	}

	r.wg.Done()
}

// Starts the MapReduce task.
func (r *Runner) Run(mappers int) {
	emit := make(chan Event, mappers)

	// Create background mapping workers.
	for i := 0; i < mappers; i++ {
		r.wg.Add(1)
		go r.mapWorker(emit)
	}

	// Emit all events.
	go func() {
		for {
			emitted, err := r.mr.Emit()
			if err != EndOfEmit && err != nil {
				log.Printf("Error emitting: %v", err)
				break
			}

			emit <- emitted

			if err == EndOfEmit {
				break
			}

		}
		close(emit)
	}()

	r.wg.Wait()
}
<|endoftext|>