<commit_before>package cloudstorage

import (
	"compress/gzip"
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"net/http"
	"path"
	"strings"

	"cloud.google.com/go/storage"
	"github.com/estafette/estafette-ci-api/config"
	contracts "github.com/estafette/estafette-ci-contracts"
	foundation "github.com/estafette/estafette-foundation"
	"github.com/rs/zerolog/log"
	"google.golang.org/api/iterator"
)

var (
	// ErrLogNotExist is returned when a log cannot be found
	ErrLogNotExist = errors.New("The log does not exist")
)

// Client is the interface for connecting to google cloud storage
type Client interface {
	InsertBuildLog(ctx context.Context, buildLog contracts.BuildLog) (err error)
	InsertReleaseLog(ctx context.Context, releaseLog contracts.ReleaseLog) (err error)
	GetPipelineBuildLogs(ctx context.Context, buildLog contracts.BuildLog, acceptGzipEncoding bool, responseWriter http.ResponseWriter) (err error)
	GetPipelineReleaseLogs(ctx context.Context, releaseLog contracts.ReleaseLog, acceptGzipEncoding bool, responseWriter http.ResponseWriter) (err error)
	Rename(ctx context.Context, fromRepoSource, fromRepoOwner, fromRepoName, toRepoSource, toRepoOwner, toRepoName string) (err error)
}

// NewClient returns new cloudstorage.Client
func NewClient(config *config.CloudStorageConfig, storageClient *storage.Client) Client {

	if config == nil {
		return &client{
			config: config,
		}
	}

	return &client{
		client: storageClient,
		config: config,
	}
}

type client struct {
	client *storage.Client
	config *config.CloudStorageConfig
}

func (c *client) InsertBuildLog(ctx context.Context, buildLog contracts.BuildLog) (err error) {

	logPath := c.getBuildLogPath(buildLog)

	return foundation.Retry(func() error {
		return c.insertLog(ctx, logPath, buildLog.Steps)
	})
}

func (c *client) InsertReleaseLog(ctx context.Context, releaseLog contracts.ReleaseLog) (err error) {

	logPath := c.getReleaseLogPath(releaseLog)

	return foundation.Retry(func() error {
		return c.insertLog(ctx, logPath, releaseLog.Steps)
	})
}

func (c *client) insertLog(ctx context.Context, path string, steps []*contracts.BuildLogStep) (err error) {

	bucket := c.client.Bucket(c.config.Bucket)

	// marshal json
	jsonBytes, err := json.Marshal(steps)
	if err != nil {
		return err
	}

	// create writer for cloud storage object
	logObject := bucket.Object(path)

	// don't allow overwrites, return when file already exists
	_, err = logObject.Attrs(ctx)
	if err == nil {
		// log file already exists, return
		return nil
	}
	if err != nil && err != storage.ErrObjectNotExist {
		// some other error happened, return it
		return err
	}

	// object doesn't exist, okay to write it
	writer := logObject.NewWriter(ctx)
	if writer == nil {
		return fmt.Errorf("Writer for logobject %v is nil", path)
	}

	// write compressed bytes
	gz, err := gzip.NewWriterLevel(writer, gzip.BestSpeed)
	if err != nil {
		_ = writer.Close()
		return err
	}
	_, err = gz.Write(jsonBytes)
	if err != nil {
		_ = writer.Close()
		return err
	}
	err = gz.Close()
	if err != nil {
		_ = writer.Close()
		return err
	}
	err = writer.Close()
	if err != nil {
		return err
	}

	return nil
}

func (c *client) GetPipelineBuildLogs(ctx context.Context, buildLog contracts.BuildLog, acceptGzipEncoding bool, responseWriter http.ResponseWriter) (err error) {

	logPath := c.getBuildLogPath(buildLog)

	return c.getLog(ctx, logPath, acceptGzipEncoding, responseWriter)
}

func (c *client) GetPipelineReleaseLogs(ctx context.Context, releaseLog contracts.ReleaseLog, acceptGzipEncoding bool, responseWriter http.ResponseWriter) (err error) {

	logPath := c.getReleaseLogPath(releaseLog)

	return c.getLog(ctx, logPath, acceptGzipEncoding, responseWriter)
}

func (c *client) getLog(ctx context.Context, path string, acceptGzipEncoding bool, responseWriter http.ResponseWriter) (err error) {

	bucket := c.client.Bucket(c.config.Bucket)

	// create reader for cloud storage object
	logObject := bucket.Object(path).ReadCompressed(true)
	reader, err := logObject.NewReader(ctx)
	if err != nil {
		if errors.Is(err, storage.ErrObjectNotExist) {
			return ErrLogNotExist
		}

		return err
	}
	defer reader.Close()

	// create source reader to either copy compressed bytes or decompress them first
	sourceReader := io.Reader(reader)
	if acceptGzipEncoding {
		responseWriter.Header().Set("Content-Encoding", "gzip")
		responseWriter.Header().Set("Vary", "Accept-Encoding")
	} else {
		gzr, err := gzip.NewReader(reader)
		if err != nil {
			return err
		}
		defer gzr.Close()
		sourceReader = io.Reader(gzr)
	}

	responseWriter.Header().Set("Content-Type", "application/json; charset=utf-8")

	writtenBytes, err := io.Copy(responseWriter, sourceReader)
	if err != nil {
		return err
	}

	responseWriter.Header().Set("Content-Length", fmt.Sprint(writtenBytes))

	return nil
}

func (c *client) getBuildLogPath(buildLog contracts.BuildLog) (logPath string) {

	logDirectory := c.getLogDirectory(buildLog.RepoSource, buildLog.RepoOwner, buildLog.RepoName, "builds")

	logPath = path.Join(logDirectory, fmt.Sprintf("%v.log", buildLog.ID))

	return logPath
}

func (c *client) getReleaseLogPath(releaseLog contracts.ReleaseLog) (logPath string) {

	logDirectory := c.getLogDirectory(releaseLog.RepoSource, releaseLog.RepoOwner, releaseLog.RepoName, "releases")

	logPath = path.Join(logDirectory, fmt.Sprintf("%v.log", releaseLog.ID))

	return logPath
}

func (c *client) getLogDirectory(repoSource, repoOwner, repoName, logType string) (logDirectory string) {
	logDirectory = path.Join(c.config.LogsDirectory, repoSource, repoOwner, repoName, logType)

	return logDirectory
}

func (c *client) Rename(ctx context.Context, fromRepoSource, fromRepoOwner, fromRepoName, toRepoSource, toRepoOwner, toRepoName string) (err error) {

	log.Info().Msgf("Renaming cloud storage logs from %v/%v/%v to %v/%v/%v", fromRepoSource, fromRepoOwner, fromRepoName, toRepoSource, toRepoOwner, toRepoName)

	bucket := c.client.Bucket(c.config.Bucket)

	// list all build log files in old location, rename to new location
	fromBuildLogDirectory := c.getLogDirectory(fromRepoSource, fromRepoOwner, fromRepoName, "builds")
	toBuildLogDirectory := c.getLogDirectory(toRepoSource, toRepoOwner, toRepoName, "builds")

	err = c.renameFilesInDirectory(ctx, bucket, fromBuildLogDirectory, toBuildLogDirectory)
	if err != nil {
		return err
	}

	// list all release log files in old location, rename to new location
	fromReleaseLogDirectory := c.getLogDirectory(fromRepoSource, fromRepoOwner, fromRepoName, "releases")
	toReleaseLogDirectory := c.getLogDirectory(toRepoSource, toRepoOwner, toRepoName, "releases")

	err = c.renameFilesInDirectory(ctx, bucket, fromReleaseLogDirectory, toReleaseLogDirectory)
	if err != nil {
		return err
	}

	return nil
}

func (c *client) renameFilesInDirectory(ctx context.Context, bucket *storage.BucketHandle, fromLogFileDirectory, toLogFileDirectory string) (err error) {
	it := bucket.Objects(ctx, &storage.Query{
		Prefix:    fromLogFileDirectory,
		Delimiter: "/",
	})
	for {
		attrs, err := it.Next()
		if err == iterator.Done {
			break
		}
		if err != nil {
			return err
		}
		fromLogFilePath := attrs.Name
		toLogFilePath := strings.Replace(fromLogFilePath, fromLogFileDirectory, toLogFileDirectory, 1)

		err = c.renameFile(ctx, bucket, fromLogFilePath, toLogFilePath)
		if err != nil {
			return err
		}
	}

	return nil
}

func (c *client) renameFile(ctx context.Context, bucket *storage.BucketHandle, fromLogFilePath, toLogFilePath string) (err error) {
	src := bucket.Object(fromLogFilePath)
	dst := bucket.Object(toLogFilePath)

	if _, err := dst.CopierFrom(src).Run(ctx); err != nil {
		return err
	}
	if err := src.Delete(ctx); err != nil {
		return err
	}

	return nil
}
<commit_msg>additional logging for cloud storage logs rename<commit_after>package cloudstorage

import (
	"compress/gzip"
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"net/http"
	"path"
	"strings"

	"cloud.google.com/go/storage"
	"github.com/estafette/estafette-ci-api/config"
	contracts "github.com/estafette/estafette-ci-contracts"
	foundation "github.com/estafette/estafette-foundation"
	"github.com/rs/zerolog/log"
	"google.golang.org/api/iterator"
)

var (
	// ErrLogNotExist is returned when a log cannot be found
	ErrLogNotExist = errors.New("The log does not exist")
)

// Client is the interface for connecting to google cloud storage
type Client interface {
	InsertBuildLog(ctx context.Context, buildLog contracts.BuildLog) (err error)
	InsertReleaseLog(ctx context.Context, releaseLog contracts.ReleaseLog) (err error)
	GetPipelineBuildLogs(ctx context.Context, buildLog contracts.BuildLog, acceptGzipEncoding bool, responseWriter http.ResponseWriter) (err error)
	GetPipelineReleaseLogs(ctx context.Context, releaseLog contracts.ReleaseLog, acceptGzipEncoding bool, responseWriter http.ResponseWriter) (err error)
	Rename(ctx context.Context, fromRepoSource, fromRepoOwner, fromRepoName, toRepoSource, toRepoOwner, toRepoName string) (err error)
}

// NewClient returns new cloudstorage.Client
func NewClient(config *config.CloudStorageConfig, storageClient *storage.Client) Client {

	if config == nil {
		return &client{
			config: config,
		}
	}

	return &client{
		client: storageClient,
		config: config,
	}
}

type client struct {
	client *storage.Client
	config *config.CloudStorageConfig
}

func (c *client) InsertBuildLog(ctx context.Context, buildLog contracts.BuildLog) (err error) {

	logPath := c.getBuildLogPath(buildLog)

	return foundation.Retry(func() error {
		return c.insertLog(ctx, logPath, buildLog.Steps)
	})
}

func (c *client) InsertReleaseLog(ctx context.Context, releaseLog contracts.ReleaseLog) (err error) {

	logPath := c.getReleaseLogPath(releaseLog)

	return foundation.Retry(func() error {
		return c.insertLog(ctx, logPath, releaseLog.Steps)
	})
}

func (c *client) insertLog(ctx context.Context, path string, steps []*contracts.BuildLogStep) (err error) {

	bucket := c.client.Bucket(c.config.Bucket)

	// marshal json
	jsonBytes, err := json.Marshal(steps)
	if err != nil {
		return err
	}

	// create writer for cloud storage object
	logObject := bucket.Object(path)

	// don't allow overwrites, return when file already exists
	_, err = logObject.Attrs(ctx)
	if err == nil {
		// log file already exists, return
		return nil
	}
	if err != nil && err != storage.ErrObjectNotExist {
		// some other error happened, return it
		return err
	}

	// object doesn't exist, okay to write it
	writer := logObject.NewWriter(ctx)
	if writer == nil {
		return fmt.Errorf("Writer for logobject %v is nil", path)
	}

	// write compressed bytes
	gz, err := gzip.NewWriterLevel(writer, gzip.BestSpeed)
	if err != nil {
		_ = writer.Close()
		return err
	}
	_, err = gz.Write(jsonBytes)
	if err != nil {
		_ = writer.Close()
		return err
	}
	err = gz.Close()
	if err != nil {
		_ = writer.Close()
		return err
	}
	err = writer.Close()
	if err != nil {
		return err
	}

	return nil
}

func (c *client) GetPipelineBuildLogs(ctx context.Context, buildLog contracts.BuildLog, acceptGzipEncoding bool, responseWriter http.ResponseWriter) (err error) {

	logPath := c.getBuildLogPath(buildLog)

	return c.getLog(ctx, logPath, acceptGzipEncoding, responseWriter)
}

func (c *client) GetPipelineReleaseLogs(ctx context.Context, releaseLog contracts.ReleaseLog, acceptGzipEncoding bool, responseWriter http.ResponseWriter) (err error) {

	logPath := c.getReleaseLogPath(releaseLog)

	return c.getLog(ctx, logPath, acceptGzipEncoding, responseWriter)
}

func (c *client) getLog(ctx context.Context, path string, acceptGzipEncoding bool, responseWriter http.ResponseWriter) (err error) {

	bucket := c.client.Bucket(c.config.Bucket)

	// create reader for cloud storage object
	logObject := bucket.Object(path).ReadCompressed(true)
	reader, err := logObject.NewReader(ctx)
	if err != nil {
		if errors.Is(err, storage.ErrObjectNotExist) {
			return ErrLogNotExist
		}

		return err
	}
	defer reader.Close()

	// create source reader to either copy compressed bytes or decompress them first
	sourceReader := io.Reader(reader)
	if acceptGzipEncoding {
		responseWriter.Header().Set("Content-Encoding", "gzip")
		responseWriter.Header().Set("Vary", "Accept-Encoding")
	} else {
		gzr, err := gzip.NewReader(reader)
		if err != nil {
			return err
		}
		defer gzr.Close()
		sourceReader = io.Reader(gzr)
	}

	responseWriter.Header().Set("Content-Type", "application/json; charset=utf-8")

	writtenBytes, err := io.Copy(responseWriter, sourceReader)
	if err != nil {
		return err
	}

	responseWriter.Header().Set("Content-Length", fmt.Sprint(writtenBytes))

	return nil
}

func (c *client) getBuildLogPath(buildLog contracts.BuildLog) (logPath string) {

	logDirectory := c.getLogDirectory(buildLog.RepoSource, buildLog.RepoOwner, buildLog.RepoName, "builds")

	logPath = path.Join(logDirectory, fmt.Sprintf("%v.log", buildLog.ID))

	return logPath
}

func (c *client) getReleaseLogPath(releaseLog contracts.ReleaseLog) (logPath string) {

	logDirectory := c.getLogDirectory(releaseLog.RepoSource, releaseLog.RepoOwner, releaseLog.RepoName, "releases")

	logPath = path.Join(logDirectory, fmt.Sprintf("%v.log", releaseLog.ID))

	return logPath
}

func (c *client) getLogDirectory(repoSource, repoOwner, repoName, logType string) (logDirectory string) {
	logDirectory = path.Join(c.config.LogsDirectory, repoSource, repoOwner, repoName, logType)

	return logDirectory
}

func (c *client) Rename(ctx context.Context, fromRepoSource, fromRepoOwner, fromRepoName, toRepoSource, toRepoOwner, toRepoName string) (err error) {

	log.Info().Msgf("Renaming cloud storage logs from %v/%v/%v to %v/%v/%v for bucket %v", fromRepoSource, fromRepoOwner, fromRepoName, toRepoSource, toRepoOwner, toRepoName, c.config.Bucket)

	bucket := c.client.Bucket(c.config.Bucket)

	// list all build log files in old location, rename to new location
	fromBuildLogDirectory := c.getLogDirectory(fromRepoSource, fromRepoOwner, fromRepoName, "builds")
	toBuildLogDirectory := c.getLogDirectory(toRepoSource, toRepoOwner, toRepoName, "builds")

	err = c.renameFilesInDirectory(ctx, bucket, fromBuildLogDirectory, toBuildLogDirectory)
	if err != nil {
		return err
	}

	// list all release log files in old location, rename to new location
	fromReleaseLogDirectory := c.getLogDirectory(fromRepoSource, fromRepoOwner, fromRepoName, "releases")
	toReleaseLogDirectory := c.getLogDirectory(toRepoSource, toRepoOwner, toRepoName, "releases")

	err = c.renameFilesInDirectory(ctx, bucket, fromReleaseLogDirectory, toReleaseLogDirectory)
	if err != nil {
		return err
	}

	return nil
}

func (c *client) renameFilesInDirectory(ctx context.Context, bucket *storage.BucketHandle, fromLogFileDirectory, toLogFileDirectory string) (err error) {

	bucketName := ""
	attrs, attrErr := bucket.Attrs(ctx)
	if attrErr == nil {
		bucketName = attrs.Name
	}

	log.Info().Msgf("Renaming cloud storage logs in bucket %v from directory %v to %v", bucketName, fromLogFileDirectory, toLogFileDirectory)

	it := bucket.Objects(ctx, &storage.Query{
		Prefix:    fromLogFileDirectory,
		Delimiter: "/",
	})
	for {
		attrs, err := it.Next()
		if err == iterator.Done {
			break
		}
		if err != nil {
			return err
		}
		fromLogFilePath := attrs.Name
		toLogFilePath := strings.Replace(fromLogFilePath, fromLogFileDirectory, toLogFileDirectory, 1)

		err = c.renameFile(ctx, bucket, fromLogFilePath, toLogFilePath)
		if err != nil {
			return err
		}
	}

	return nil
}

func (c *client) renameFile(ctx context.Context, bucket *storage.BucketHandle, fromLogFilePath, toLogFilePath string) (err error) {
	src := bucket.Object(fromLogFilePath)
	dst := bucket.Object(toLogFilePath)

	if _, err := dst.CopierFrom(src).Run(ctx); err != nil {
		return err
	}
	if err := src.Delete(ctx); err != nil {
		return err
	}

	return nil
}
<|endoftext|>


<commit_before>// Copyright (c) 2012-2014 Jeremy Latt
// Copyright (c) 2016- Daniel Oaks <daniel@danieloaks.net>
// released under the MIT license

package irc

import (
	"strings"

	"github.com/DanielOaks/girc-go/ircmsg"
)

// NICK <nickname>
func nickHandler(server *Server, client *Client, msg ircmsg.IrcMessage) bool {
	if !client.authorized {
		client.Quit("Bad password")
		return true
	}

	nickname, err := CasefoldName(msg.Params[0])

	if len(strings.TrimSpace(msg.Params[0])) < 1 {
		client.Send(nil, server.name, ERR_NONICKNAMEGIVEN, client.nick, "No nickname given")
		return false
	}

	if err != nil || len(strings.TrimSpace(msg.Params[0])) > server.limits.NickLen {
		client.Send(nil, server.name, ERR_ERRONEUSNICKNAME, client.nick, msg.Params[0], "Erroneous nickname")
		return false
	}

	if client.nick == nickname {
		return false
	}

	//TODO(dan): There's probably some races here, we should be changing this in the primary server thread
	target := server.clients.Get(nickname)
	if target != nil && target != client {
		client.Send(nil, server.name, ERR_NICKNAMEINUSE, client.nick, msg.Params[0], "Nickname is already in use")
		return false
	}

	if client.registered {
		client.ChangeNickname(nickname)
	} else {
		client.SetNickname(nickname)
	}
	server.tryRegister(client)
	return false
}

// SANICK <oldnick> <nickname>
func sanickHandler(server *Server, client *Client, msg ircmsg.IrcMessage) bool {
	if !client.authorized {
		client.Quit("Bad password")
		return true
	}

	oldnick, oerr := CasefoldName(msg.Params[0])
	casefoldedNickname, err := CasefoldName(msg.Params[1])

	if len(casefoldedNickname) < 1 {
		client.Send(nil, server.name, ERR_NONICKNAMEGIVEN, client.nick, "No nickname given")
		return false
	}

	if oerr != nil || err != nil || len(strings.TrimSpace(msg.Params[1])) > server.limits.NickLen {
		client.Send(nil, server.name, ERR_ERRONEUSNICKNAME, client.nick, msg.Params[0], "Erroneous nickname")
		return false
	}

	if client.nick == msg.Params[1] {
		return false
	}

	target := server.clients.Get(oldnick)
	if target == nil {
		client.Send(nil, server.name, ERR_NOSUCHNICK, msg.Params[0], "No such nick")
		return false
	}

	//TODO(dan): There's probably some races here, we should be changing this in the primary server thread
	if server.clients.Get(casefoldedNickname) != nil || server.clients.Get(casefoldedNickname) != target {
		client.Send(nil, server.name, ERR_NICKNAMEINUSE, client.nick, msg.Params[0], "Nickname is already in use")
		return false
	}

	target.ChangeNickname(msg.Params[1])
	return false
}
<commit_msg>nickname: Preserve case when setting NICK<commit_after>// Copyright (c) 2012-2014 Jeremy Latt
// Copyright (c) 2016- Daniel Oaks <daniel@danieloaks.net>
// released under the MIT license

package irc

import (
	"strings"

	"github.com/DanielOaks/girc-go/ircmsg"
)

// NICK <nickname>
func nickHandler(server *Server, client *Client, msg ircmsg.IrcMessage) bool {
	if !client.authorized {
		client.Quit("Bad password")
		return true
	}

	nicknameRaw := strings.TrimSpace(msg.Params[0])
	nickname, err := CasefoldName(nicknameRaw)

	if len(nicknameRaw) < 1 {
		client.Send(nil, server.name, ERR_NONICKNAMEGIVEN, client.nick, "No nickname given")
		return false
	}

	if err != nil || len(nicknameRaw) > server.limits.NickLen {
		client.Send(nil, server.name, ERR_ERRONEUSNICKNAME, client.nick, nicknameRaw, "Erroneous nickname")
		return false
	}

	if client.nick == nickname {
		return false
	}

	//TODO(dan): There's probably some races here, we should be changing this in the primary server thread
	target := server.clients.Get(nickname)
	if target != nil && target != client {
		client.Send(nil, server.name, ERR_NICKNAMEINUSE, client.nick, nicknameRaw, "Nickname is already in use")
		return false
	}

	if client.registered {
		client.ChangeNickname(nicknameRaw)
	} else {
		client.SetNickname(nicknameRaw)
	}
	server.tryRegister(client)
	return false
}

// SANICK <oldnick> <nickname>
func sanickHandler(server *Server, client *Client, msg ircmsg.IrcMessage) bool {
	if !client.authorized {
		client.Quit("Bad password")
		return true
	}

	oldnick, oerr := CasefoldName(msg.Params[0])
	casefoldedNickname, err := CasefoldName(msg.Params[1])

	if len(casefoldedNickname) < 1 {
		client.Send(nil, server.name, ERR_NONICKNAMEGIVEN, client.nick, "No nickname given")
		return false
	}

	if oerr != nil || err != nil || len(strings.TrimSpace(msg.Params[1])) > server.limits.NickLen {
		client.Send(nil, server.name, ERR_ERRONEUSNICKNAME, client.nick, msg.Params[0], "Erroneous nickname")
		return false
	}

	if client.nick == msg.Params[1] {
		return false
	}

	target := server.clients.Get(oldnick)
	if target == nil {
		client.Send(nil, server.name, ERR_NOSUCHNICK, msg.Params[0], "No such nick")
		return false
	}

	//TODO(dan): There's probably some races here, we should be changing this in the primary server thread
	if server.clients.Get(casefoldedNickname) != nil || server.clients.Get(casefoldedNickname) != target {
		client.Send(nil, server.name, ERR_NICKNAMEINUSE, client.nick, msg.Params[0], "Nickname is already in use")
		return false
	}

	target.ChangeNickname(msg.Params[1])
	return false
}
<|endoftext|>


<commit_before>// Copyright 2015 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build windows

package windriver

// #include "windriver.h"
import "C"

import (
	"runtime"

	"golang.org/x/exp/shiny/screen"
)

// TODO(andlabs): Should the Windows API code be split into a
// separate package internal/winbackend so gldriver can use it too?

// Main is called by the program's main function to run the graphical
// application.
//
// It calls f on the Screen, possibly in a separate goroutine, as some OS-
// specific libraries require being on 'the main thread'. It returns when f
// returns.
func Main(f func(screen.Screen)) {
	if err := main(f); err != nil {
		f(errscreen.Stub(err))
	}
}

func main(f func(screen.Screen)) (retErr error) {
	// It does not matter which OS thread we are on.
	// All that matters is that we confine all UI operations
	// to the thread that created the respective window.
	runtime.LockOSThread()

	hr := C.initUtilityWindow()
	if hr != C.S_OK {
		return winerror("failed to create utility window", hr)
	}
	defer func() {
		// TODO(andlabs): log an error if this fails?
		C.DestroyWindow(C.utilityWindow)
		// TODO(andlabs): unregister window class
	}()

	hr = C.initWindowClass()
	if hr != C.S_OK {
		return winerror("failed to create Window window class", hr)
	}
	// TODO(andlabs): uninit

	s := newScreenImpl()
	go f(s)

	mainMessagePump()
	return nil
}

func mainMessagePump() {
	var m msg
	for {
		// This GetMessage cannot fail: http://blogs.msdn.com/b/oldnewthing/archive/2013/03/22/10404367.aspx
		// TODO(andlabs): besides, what should we do if a future Windows change makes it fail for some other reason? we can't return an error because it's too late to stop the main function
		_, err := getMessage(&m, 0, 0, 0)
		if err != nil {
			return
		}
		translateMessage(&m)
		dispatchMessage(&m)
	}
}
<commit_msg>shiny/driver/windriver: fix build on windows<commit_after>// Copyright 2015 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build windows

package windriver

// #include "windriver.h"
import "C"

import (
	"runtime"

	"golang.org/x/exp/shiny/driver/internal/errscreen"
	"golang.org/x/exp/shiny/screen"
)

// TODO(andlabs): Should the Windows API code be split into a
// separate package internal/winbackend so gldriver can use it too?

// Main is called by the program's main function to run the graphical
// application.
//
// It calls f on the Screen, possibly in a separate goroutine, as some OS-
// specific libraries require being on 'the main thread'. It returns when f
// returns.
func Main(f func(screen.Screen)) {
	if err := main(f); err != nil {
		f(errscreen.Stub(err))
	}
}

func main(f func(screen.Screen)) (retErr error) {
	// It does not matter which OS thread we are on.
	// All that matters is that we confine all UI operations
	// to the thread that created the respective window.
	runtime.LockOSThread()

	hr := C.initUtilityWindow()
	if hr != C.S_OK {
		return winerror("failed to create utility window", hr)
	}
	defer func() {
		// TODO(andlabs): log an error if this fails?
		C.DestroyWindow(C.utilityWindow)
		// TODO(andlabs): unregister window class
	}()

	hr = C.initWindowClass()
	if hr != C.S_OK {
		return winerror("failed to create Window window class", hr)
	}
	// TODO(andlabs): uninit

	s := newScreenImpl()
	go f(s)

	mainMessagePump()
	return nil
}

func mainMessagePump() {
	var m msg
	for {
		// This GetMessage cannot fail: http://blogs.msdn.com/b/oldnewthing/archive/2013/03/22/10404367.aspx
		// TODO(andlabs): besides, what should we do if a future Windows change makes it fail for some other reason? we can't return an error because it's too late to stop the main function
		_, err := getMessage(&m, 0, 0, 0)
		if err != nil {
			return
		}
		translateMessage(&m)
		dispatchMessage(&m)
	}
}
<|endoftext|>


<commit_before>// Copyright 2016 The Upspin Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

/*
Command upbox builds and runs Upspin servers as specified by a configuration
file and provides an upspin shell acting as the first user specified by the
configuration.

Configuration files must be in YAML format, of this general form:

	users:
	- name: joe
	- name: jess@example.net
	  storeserver: store.upspin.io
	  dirserver: dir.upspin.io
	  packing: ee
	servers:
	- name: storeserver
	- name: dirserver
	  user: joe
	- name: myserver
	  importpath: github.com/user/myserver
	  flags:
	    debug: cockroach
	keyserver: key.uspin.io
	domain: exmaple.com

The Users and Servers lists specify the users and servers to create within this
configuration.

Users

Name specifies the user name of this user.
It must be non-empty.
It can be a full email address, or just the user component.
In the latter case, the top-level domain field must be set.

StoreServer and DirServer specify the store and directory endpoints for this
user. If empty, they default to the servers "storeserver" and "dirserver",
respectively. If they are of the form "$servername" then the address of the
server "servername" is used.

Packing specifies the packing method for this user.
If empty, it defaults to "ee".

Servers

Name specifies a short name for this server. It must be non-empty.
The names "keyserver", "storeserver", and "dirserver" represent useful
defaults.

User specifies the user to run this server as.
It can be a full email address, or just the user component.
If empty, the Name of the server is combined with the
Config's Domain and a user is created with that name.
In the latter cases, the top-level Domain field must be set.

ImportPath specifies the import path for this server that is built before
starting the server. If empty, the server Name is appended to the string
"upspin.io/cmd/".

Other top-level fields

KeyServer specifies the KeyServer that each user in the cluster
should use. If it is empty, then a Server named "keyserver" must
be included in the list of Servers, and the address of that server
is used.

Domain specifies a domain that is appended to any user names that do
not include a domain component.
Domain must be specified if any domain suffixes are omitted from
User Names or if a Servers is specified with an empty User field.

Default configuration

If no config is specified, the default configuration is used:

	users:
	  - name: user
	servers:
	  - name: keyserver
	  - name: storeserver
	  - name: dirserver
	domain: example.com

This creates the users user@example.com, keyserver@example.com,
storeserver@example.com, and dirserver@example.com, builds and runs
the servers keyserver, storeserver, and dirserver (running as their
respective users), and runs "upspin shell" as user@example.com.

*/
package main

import (
	"bufio"
	"bytes"
	"crypto/tls"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"time"

	yaml "gopkg.in/yaml.v2"

	"upspin.io/upspin"
)

var (
	logLevel = flag.String("log", "info", "log `level`")
	basePort = flag.Int("port", 8000, "base `port` number for upspin servers")
	config   = flag.String("config", "", "configuration `file` name")
)

func main() {
	flag.Parse()

	cfg, err := ConfigFromFile(*config)
	if err != nil {
		fmt.Fprintln(os.Stderr, "upbox: error parsing config:", err)
		os.Exit(1)
	}

	if err := cfg.Run(); err != nil {
		fmt.Fprintln(os.Stderr, "upbox:", err)
		os.Exit(1)
	}
}

func (cfg *Config) Run() error {
	// Build servers and commands.
	args := []string{"install", "upspin.io/cmd/upspin"}
	for _, s := range cfg.Servers {
		args = append(args, s.ImportPath)
	}
	cmd := exec.Command("go", args...)
	cmd.Stdout = prefix("build: ", os.Stdout)
	cmd.Stderr = prefix("build: ", os.Stderr)
	if err := cmd.Run(); err != nil {
		return fmt.Errorf("build error: %v", err)
	}

	// Create temporary directory.
	tmpDir, err := ioutil.TempDir("", "upbox")
	if err != nil {
		return err
	}
	defer os.RemoveAll(tmpDir)
	userDir := func(user string) string { return filepath.Join(tmpDir, user) }

	// Generate TLS certificates.
	if err := generateCert(tmpDir); err != nil {
		return err
	}

	// Generate keys.
	// Write an empty  file for use by 'upspin keygen'.
	configKeygen := filepath.Join(tmpDir, "config.keygen")
	if err := ioutil.WriteFile(configKeygen, []byte("secrets: none"), 0644); err != nil {
		return err
	}
	for _, u := range cfg.Users {
		fmt.Fprintf(os.Stderr, "upbox: generating keys for user %q\n", u.Name)
		dir := userDir(u.Name)
		if err := os.MkdirAll(dir, 0700); err != nil {
			return err
		}
		keygen := exec.Command("upspin", "-config="+configKeygen, "keygen", "-where="+dir)
		keygen.Stdout = prefix("keygen: ", os.Stdout)
		keygen.Stderr = prefix("keygen: ", os.Stderr)
		if err := keygen.Run(); err != nil {
			return err
		}
		u.secrets = dir
	}

	writeConfig := func(server, user string) (string, error) {
		u, ok := cfg.user[user]
		if !ok {
			return "", fmt.Errorf("unknown user %q", user)
		}

		configContent := []string{
			"username: " + u.Name,
			"secrets: " + userDir(user),
			"tlscerts: " + tmpDir,
			"packing: " + u.Packing,
			"storeserver: " + u.StoreServer,
			"dirserver: " + u.DirServer,
		}
		switch server {
		case "keyserver":
			configContent = append(configContent,
				"keyserver: inprocess,",
			)
		default:
			configContent = append(configContent,
				"keyserver: remote,"+cfg.KeyServer,
			)
		}
		configFile := filepath.Join(tmpDir, "config."+server)
		if err := ioutil.WriteFile(configFile, []byte(strings.Join(configContent, "\n")), 0644); err != nil {
			return "", err
		}
		return configFile, nil
	}

	// Start servers.
	for i := range cfg.Servers {
		s := cfg.Servers[i]

		configFile, err := writeConfig(s.Name, s.User)
		if err != nil {
			return fmt.Errorf("writing config for %v: %v", s.Name, err)
		}

		args := []string{
			"-config=" + configFile,
			"-log=" + *logLevel,
			"-tls_cert=" + filepath.Join(tmpDir, "cert.pem"),
			"-tls_key=" + filepath.Join(tmpDir, "key.pem"),
			"-letscache=", // disable
			"-https=" + s.addr,
			"-addr=" + s.addr,
		}
		if s.Name == "keyserver" {
			args = append(args,
				"-test_user="+s.User,
				"-test_secrets="+userDir(s.User),
			)
		}
		for k, v := range s.Flags {
			args = append(args, fmt.Sprintf("-%s=%v", k, v))
		}
		cmd := exec.Command(s.Name, args...)
		cmd.Stdout = prefix(s.Name+":\t", os.Stdout)
		cmd.Stderr = prefix(s.Name+":\t", os.Stderr)
		if err := cmd.Start(); err != nil {
			return fmt.Errorf("starting %v: %v", s.Name, err)
		}
		defer kill(cmd)
	}

	// Wait for the keyserver to start and add the users to it.
	if err := waitReady(cfg.KeyServer); err != nil {
		return err
	}
	keyUser := cfg.Users[0].Name
	if s, ok := cfg.server["keyserver"]; ok {
		keyUser = s.User
	}
	configFile, err := writeConfig("key-bootstrap", keyUser)
	if err != nil {
		return err
	}
	for _, u := range cfg.Users {
		pk, err := ioutil.ReadFile(filepath.Join(userDir(u.Name), "public.upspinkey"))
		if err != nil {
			return err
		}
		dir, err := upspin.ParseEndpoint(u.DirServer)
		if err != nil {
			return err
		}
		store, err := upspin.ParseEndpoint(u.StoreServer)
		if err != nil {
			return err
		}
		user := &upspin.User{
			Name:      upspin.UserName(u.Name),
			Dirs:      []upspin.Endpoint{*dir},
			Stores:    []upspin.Endpoint{*store},
			PublicKey: upspin.PublicKey(pk),
		}
		userYAML, err := yaml.Marshal(user)
		if err != nil {
			return err
		}
		cmd := exec.Command("upspin",
			"-config="+configFile,
			"-log="+*logLevel,
			"user", "-put",
		)
		cmd.Stdin = bytes.NewReader(userYAML)
		cmd.Stdout = prefix("key-bootstrap:\t", os.Stdout)
		cmd.Stderr = prefix("key-bootstrap:\t", os.Stderr)
		if err := cmd.Run(); err != nil {
			return err
		}
	}

	// Wait for the other services to start.
	for _, s := range cfg.Servers {
		if s.addr == cfg.KeyServer {
			continue
		}
		if err := waitReady(s.addr); err != nil {
			return err
		}
	}

	// Start a shell as the first user.
	configFile, err = writeConfig("shell", cfg.Users[0].Name)
	if err != nil {
		return err
	}
	args = []string{
		"-config=" + configFile,
		"-log=" + *logLevel,
		"shell",
	}
	fmt.Fprintf(os.Stderr, "upbox: upspin %s\n", strings.Join(args, " "))
	shell := exec.Command("upspin", args...)
	shell.Stdin = os.Stdin
	shell.Stdout = os.Stdout
	shell.Stderr = os.Stderr
	return shell.Run()
}

func kill(cmd *exec.Cmd) {
	if cmd.Process != nil {
		cmd.Process.Kill()
	}
}

func prefix(p string, out io.Writer) io.Writer {
	r, w := io.Pipe()
	go func() {
		s := bufio.NewScanner(r)
		for s.Scan() {
			fmt.Fprintf(out, "%s%s\n", p, s.Bytes())
		}
	}()
	return w
}

func waitReady(addr string) error {
	rt := &http.Transport{
		TLSClientConfig: &tls.Config{
			InsecureSkipVerify: true,
		},
	}
	req, _ := http.NewRequest("GET", "https://"+addr, nil)
	for i := 0; i < 10; i++ {
		_, err := rt.RoundTrip(req)
		if err != nil {
			time.Sleep(time.Second)
			continue
		}
		return nil
	}
	return fmt.Errorf("timed out waiting for %q to come up", addr)
}
<commit_msg>exp/cmd/upbox: start keyserver before other servers<commit_after>// Copyright 2016 The Upspin Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

/*
Command upbox builds and runs Upspin servers as specified by a configuration
file and provides an upspin shell acting as the first user specified by the
configuration.

Configuration files must be in YAML format, of this general form:

	users:
	- name: joe
	- name: jess@example.net
	  storeserver: store.upspin.io
	  dirserver: dir.upspin.io
	  packing: ee
	servers:
	- name: storeserver
	- name: dirserver
	  user: joe
	- name: myserver
	  importpath: github.com/user/myserver
	  flags:
	    debug: cockroach
	keyserver: key.uspin.io
	domain: exmaple.com

The Users and Servers lists specify the users and servers to create within this
configuration.

Users

Name specifies the user name of this user.
It must be non-empty.
It can be a full email address, or just the user component.
In the latter case, the top-level domain field must be set.

StoreServer and DirServer specify the store and directory endpoints for this
user. If empty, they default to the servers "storeserver" and "dirserver",
respectively. If they are of the form "$servername" then the address of the
server "servername" is used.

Packing specifies the packing method for this user.
If empty, it defaults to "ee".

Servers

Name specifies a short name for this server. It must be non-empty.
The names "keyserver", "storeserver", and "dirserver" represent useful
defaults.

User specifies the user to run this server as.
It can be a full email address, or just the user component.
If empty, the Name of the server is combined with the
Config's Domain and a user is created with that name.
In the latter cases, the top-level Domain field must be set.

ImportPath specifies the import path for this server that is built before
starting the server. If empty, the server Name is appended to the string
"upspin.io/cmd/".

Other top-level fields

KeyServer specifies the KeyServer that each user in the cluster
should use. If it is empty, then a Server named "keyserver" must
be included in the list of Servers, and the address of that server
is used.

Domain specifies a domain that is appended to any user names that do
not include a domain component.
Domain must be specified if any domain suffixes are omitted from
User Names or if a Servers is specified with an empty User field.

Default configuration

If no config is specified, the default configuration is used:

	users:
	  - name: user
	servers:
	  - name: keyserver
	  - name: storeserver
	  - name: dirserver
	domain: example.com

This creates the users user@example.com, keyserver@example.com,
storeserver@example.com, and dirserver@example.com, builds and runs
the servers keyserver, storeserver, and dirserver (running as their
respective users), and runs "upspin shell" as user@example.com.

*/
package main

import (
	"bufio"
	"bytes"
	"crypto/tls"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"time"

	yaml "gopkg.in/yaml.v2"

	"upspin.io/upspin"
)

var (
	logLevel = flag.String("log", "info", "log `level`")
	basePort = flag.Int("port", 8000, "base `port` number for upspin servers")
	config   = flag.String("config", "", "configuration `file` name")
)

func main() {
	flag.Parse()

	cfg, err := ConfigFromFile(*config)
	if err != nil {
		fmt.Fprintln(os.Stderr, "upbox: error parsing config:", err)
		os.Exit(1)
	}

	if err := cfg.Run(); err != nil {
		fmt.Fprintln(os.Stderr, "upbox:", err)
		os.Exit(1)
	}
}

func (cfg *Config) Run() error {
	// Build servers and commands.
	args := []string{"install", "upspin.io/cmd/upspin"}
	for _, s := range cfg.Servers {
		args = append(args, s.ImportPath)
	}
	cmd := exec.Command("go", args...)
	cmd.Stdout = prefix("build: ", os.Stdout)
	cmd.Stderr = prefix("build: ", os.Stderr)
	if err := cmd.Run(); err != nil {
		return fmt.Errorf("build error: %v", err)
	}

	// Create temporary directory.
	tmpDir, err := ioutil.TempDir("", "upbox")
	if err != nil {
		return err
	}
	defer os.RemoveAll(tmpDir)
	userDir := func(user string) string { return filepath.Join(tmpDir, user) }

	// Generate TLS certificates.
	if err := generateCert(tmpDir); err != nil {
		return err
	}

	// Generate keys.
	// Write an empty  file for use by 'upspin keygen'.
	configKeygen := filepath.Join(tmpDir, "config.keygen")
	if err := ioutil.WriteFile(configKeygen, []byte("secrets: none"), 0644); err != nil {
		return err
	}
	for _, u := range cfg.Users {
		fmt.Fprintf(os.Stderr, "upbox: generating keys for user %q\n", u.Name)
		dir := userDir(u.Name)
		if err := os.MkdirAll(dir, 0700); err != nil {
			return err
		}
		keygen := exec.Command("upspin", "-config="+configKeygen, "keygen", "-where="+dir)
		keygen.Stdout = prefix("keygen: ", os.Stdout)
		keygen.Stderr = prefix("keygen: ", os.Stderr)
		if err := keygen.Run(); err != nil {
			return err
		}
		u.secrets = dir
	}

	// TODO(adg): make these closures methods on *Config
	writeConfig := func(server, user string) (string, error) {
		u, ok := cfg.user[user]
		if !ok {
			return "", fmt.Errorf("unknown user %q", user)
		}

		configContent := []string{
			"username: " + u.Name,
			"secrets: " + userDir(user),
			"tlscerts: " + tmpDir,
			"packing: " + u.Packing,
			"storeserver: " + u.StoreServer,
			"dirserver: " + u.DirServer,
		}
		switch server {
		case "keyserver":
			configContent = append(configContent,
				"keyserver: inprocess,",
			)
		default:
			configContent = append(configContent,
				"keyserver: remote,"+cfg.KeyServer,
			)
		}
		configFile := filepath.Join(tmpDir, "config."+server)
		if err := ioutil.WriteFile(configFile, []byte(strings.Join(configContent, "\n")), 0644); err != nil {
			return "", err
		}
		return configFile, nil
	}

	startServer := func(s *Server) (*exec.Cmd, error) {
		configFile, err := writeConfig(s.Name, s.User)
		if err != nil {
			return nil, fmt.Errorf("writing config for %v: %v", s.Name, err)
		}

		args := []string{
			"-config=" + configFile,
			"-log=" + *logLevel,
			"-tls_cert=" + filepath.Join(tmpDir, "cert.pem"),
			"-tls_key=" + filepath.Join(tmpDir, "key.pem"),
			"-letscache=", // disable
			"-https=" + s.addr,
			"-addr=" + s.addr,
		}
		if s.Name == "keyserver" {
			args = append(args,
				"-test_user="+s.User,
				"-test_secrets="+userDir(s.User),
			)
		}
		for k, v := range s.Flags {
			args = append(args, fmt.Sprintf("-%s=%v", k, v))
		}
		cmd := exec.Command(s.Name, args...)
		cmd.Stdout = prefix(s.Name+":\t", os.Stdout)
		cmd.Stderr = prefix(s.Name+":\t", os.Stderr)
		if err := cmd.Start(); err != nil {
			return nil, fmt.Errorf("starting %v: %v", s.Name, err)
		}
		return cmd, nil
	}

	keyUser := cfg.Users[0].Name
	if s, ok := cfg.server["keyserver"]; ok {
		keyUser = s.User
		// Start keyserver.
		cmd, err := startServer(s)
		if err != nil {
			return err
		}
		defer kill(cmd)
	}
	// Wait for the keyserver to start and add the users to it.
	if err := waitReady(cfg.KeyServer); err != nil {
		return err
	}
	configFile, err := writeConfig("key-bootstrap", keyUser)
	if err != nil {
		return err
	}
	for _, u := range cfg.Users {
		pk, err := ioutil.ReadFile(filepath.Join(userDir(u.Name), "public.upspinkey"))
		if err != nil {
			return err
		}
		dir, err := upspin.ParseEndpoint(u.DirServer)
		if err != nil {
			return err
		}
		store, err := upspin.ParseEndpoint(u.StoreServer)
		if err != nil {
			return err
		}
		user := &upspin.User{
			Name:      upspin.UserName(u.Name),
			Dirs:      []upspin.Endpoint{*dir},
			Stores:    []upspin.Endpoint{*store},
			PublicKey: upspin.PublicKey(pk),
		}
		userYAML, err := yaml.Marshal(user)
		if err != nil {
			return err
		}
		cmd := exec.Command("upspin",
			"-config="+configFile,
			"-log="+*logLevel,
			"user", "-put",
		)
		cmd.Stdin = bytes.NewReader(userYAML)
		cmd.Stdout = prefix("key-bootstrap:\t", os.Stdout)
		cmd.Stderr = prefix("key-bootstrap:\t", os.Stderr)
		if err := cmd.Run(); err != nil {
			return err
		}
	}

	// Start other servers.
	for i := range cfg.Servers {
		s := cfg.Servers[i]
		if s.Name == "keyserver" {
			continue
		}

		cmd, err := startServer(s)
		if err != nil {
			return err
		}
		defer kill(cmd)
	}
	// Wait for the other servers to start.
	for _, s := range cfg.Servers {
		if s.Name == "keyserver" {
			continue
		}
		if err := waitReady(s.addr); err != nil {
			return err
		}
	}

	// Start a shell as the first user.
	configFile, err = writeConfig("shell", cfg.Users[0].Name)
	if err != nil {
		return err
	}
	args = []string{
		"-config=" + configFile,
		"-log=" + *logLevel,
		"shell",
	}
	fmt.Fprintf(os.Stderr, "upbox: upspin %s\n", strings.Join(args, " "))
	shell := exec.Command("upspin", args...)
	shell.Stdin = os.Stdin
	shell.Stdout = os.Stdout
	shell.Stderr = os.Stderr
	return shell.Run()
}

func kill(cmd *exec.Cmd) {
	if cmd.Process != nil {
		cmd.Process.Kill()
	}
}

func prefix(p string, out io.Writer) io.Writer {
	r, w := io.Pipe()
	go func() {
		s := bufio.NewScanner(r)
		for s.Scan() {
			fmt.Fprintf(out, "%s%s\n", p, s.Bytes())
		}
	}()
	return w
}

func waitReady(addr string) error {
	rt := &http.Transport{
		TLSClientConfig: &tls.Config{
			InsecureSkipVerify: true,
		},
	}
	req, _ := http.NewRequest("GET", "https://"+addr, nil)
	for i := 0; i < 10; i++ {
		_, err := rt.RoundTrip(req)
		if err != nil {
			time.Sleep(time.Second)
			continue
		}
		return nil
	}
	return fmt.Errorf("timed out waiting for %q to come up", addr)
}
<|endoftext|>


<commit_before>package log

import (
	"io"
	"os"
)

// Sink defines a log sink with a target io.Writer.
type Sink struct {
	Level  Level     // The minimum log level before using the sink
	Writer io.Writer // The io.Writer to use for writing messages
}

// Default sinks.
var (
	Stdout = &Sink{Level: LevelWarning, Writer: os.Stdout}
)

// NewSink returns a new Sink.
func NewSink(level Level, writer io.Writer) *Sink {
	return &Sink{Level: level, Writer: writer}
}

// Write writes a message to the log sink io.Writer if the level is equal to
// or higher than the minumum level set on the sink.
func (s *Sink) Write(level Level, message string) error {
	if level < s.Level {
		return nil
	}

	_, err := s.Writer.Write([]byte(message))
	return err
}
<commit_msg>Ensure atomic write to sinks<commit_after>package log

import (
	"io"
	"os"
	"sync"
)

// Sink defines a log sink with a target io.Writer.
type Sink struct {
	mutex  sync.Mutex // Ensure atomic writes
	Level  Level      // The minimum log level before using the sink
	Writer io.Writer  // The io.Writer to use for writing messages
}

// Default sinks.
var (
	Stdout = &Sink{Level: LevelWarning, Writer: os.Stdout}
)

// NewSink returns a new Sink.
func NewSink(level Level, writer io.Writer) *Sink {
	return &Sink{Level: level, Writer: writer}
}

// Write writes a message to the log sink io.Writer if the level is equal to
// or higher than the minumum level set on the sink.
func (s *Sink) Write(level Level, message string) error {
	if level < s.Level {
		return nil
	}

	s.mutex.Lock()
	defer s.mutex.Unlock()

	_, err := s.Writer.Write([]byte(message))
	return err
}
<|endoftext|>


<commit_before>package calcium

import (
	"bytes"
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"
	"text/template"

	log "github.com/Sirupsen/logrus"
	enginetypes "github.com/docker/docker/api/types"
	"github.com/docker/docker/pkg/archive"
	"github.com/projecteru2/core/types"
	"github.com/projecteru2/core/utils"
)

// FIXME in alpine, useradd rename as adduser
const (
	fromAsTmpl = "FROM %s as %s"
	commonTmpl = `{{ range $k, $v:= .Args -}}
{{ printf "ARG %s=%q" $k $v }}
{{ end -}}
{{ range $k, $v:= .Envs -}}
{{ printf "ENV %s %q" $k $v }}
{{ end -}}
{{ range $k, $v:= .Labels -}}
{{ printf "LABEL %s=%s" $k $v }}
{{ end -}}
{{- if .Dir}}RUN mkdir -p {{.Dir}}
WORKDIR {{.Dir}}{{ end }}
{{ if .Repo }}ADD {{.Repo}} .{{ end }}`
	copyTmpl = "COPY --from=%s %s %s"
	runTmpl  = "RUN %s"
	userTmpl = `RUN echo "{{.User}}::{{.UID}}:{{.UID}}::/home/{{.User}}:/sbin/nologin" >> /etc/passwd && \
echo "{{.User}}:x:{{.UID}}:" >> /etc/group && \
echo "{{.User}}:!::0:::::" >> /etc/shadow
USER {{.User}}
`
)

func (c *calcium) preparedSource(build *types.Build, buildDir string) (string, error) {
	// parse repository name
	// code locates under /:repositoryname
	var cloneDir string
	var err error
	reponame := build.Repo
	if build.Repo != "" && build.Version != "" {
		reponame, err = utils.GetGitRepoName(build.Repo)
		if err != nil {
			return "", err
		}

		// clone code into cloneDir
		// which is under buildDir and named as repository name
		cloneDir = filepath.Join(buildDir, reponame)
		if err := c.source.SourceCode(build.Repo, cloneDir, build.Version); err != nil {
			return "", err
		}

		// ensure source code is safe
		// we don't want any history files to be retrieved
		if err := c.source.Security(cloneDir); err != nil {
			return "", err
		}
	}

	// if artifact download url is provided, remove all source code to
	// improve security
	if len(build.Artifacts) > 0 {
		artifactsDir := buildDir
		if cloneDir != "" {
			os.RemoveAll(cloneDir)
			os.MkdirAll(cloneDir, os.ModeDir)
			artifactsDir = cloneDir
		}
		for _, artifact := range build.Artifacts {
			if err := c.source.Artifact(artifact, artifactsDir); err != nil {
				return "", err
			}
		}
	}

	return reponame, nil
}

// BuildImage will build image for repository
// since we wanna set UID for the user inside container, we have to know the uid parameter
//
// build directory is like:
//
//    buildDir ├─ :appname ├─ code
//             ├─ Dockerfile
func (c *calcium) BuildImage(ctx context.Context, opts *types.BuildOptions) (chan *types.BuildImageMessage, error) {
	ch := make(chan *types.BuildImageMessage)

	// get pod from config
	buildPodname := c.config.Docker.BuildPod
	if buildPodname == "" {
		return ch, fmt.Errorf("No build pod set in config")
	}

	// get node by scheduler
	nodes, err := c.ListPodNodes(buildPodname, false)
	if err != nil {
		return ch, err
	}
	if len(nodes) == 0 {
		return ch, errors.New("No node to build")
	}
	node := c.scheduler.MaxIdleNode(nodes)

	// make build dir
	buildDir, err := ioutil.TempDir(os.TempDir(), "corebuild-")
	if err != nil {
		return ch, err
	}
	defer os.RemoveAll(buildDir)

	// create dockerfile
	if err := c.makeDockerFile(opts, buildDir); err != nil {
		return ch, err
	}

	// tag of image, later this will be used to push image to hub
	tag := createImageTag(c.config.Docker, opts.Name, opts.Tag)

	// create tar stream for Build API
	buildContext, err := createTarStream(buildDir)
	if err != nil {
		return ch, err
	}

	// must be put here because of that `defer os.RemoveAll(buildDir)`
	buildOptions := enginetypes.ImageBuildOptions{
		Tags:           []string{tag},
		SuppressOutput: false,
		NoCache:        true,
		Remove:         true,
		ForceRemove:    true,
		PullParent:     true,
	}

	log.Infof("[BuildImage] Building image %v at %v:%v", tag, buildPodname, node.Name)
	resp, err := node.Engine.ImageBuild(ctx, buildContext, buildOptions)
	if err != nil {
		return ch, err
	}

	go func() {
		defer resp.Body.Close()
		defer close(ch)
		decoder := json.NewDecoder(resp.Body)
		var err error
		var lastMessage *types.BuildImageMessage
		for {
			message := &types.BuildImageMessage{}
			err := decoder.Decode(message)
			if err != nil {
				if err == io.EOF {
					break
				}
				if err == context.Canceled || err == context.DeadlineExceeded {
					lastMessage.ErrorDetail.Code = -1
					lastMessage.Error = err.Error()
					break
				}
				log.Errorf("[BuildImage] Decode build image message failed %v", err)
				return
			}
			ch <- message
			lastMessage = message
		}

		if lastMessage.ErrorDetail.Code != 0 {
			log.Errorf("[BuildImage] Build image failed %v", lastMessage.ErrorDetail.Message)
			return
		}
		// About this "Khadgar", https://github.com/docker/docker/issues/10983#issuecomment-85892396
		// Just because Ben Schnetzer's cute Khadgar...
		rc, err := node.Engine.ImagePush(ctx, tag, enginetypes.ImagePushOptions{RegistryAuth: "Khadgar"})
		if err != nil {
			ch <- makeErrorBuildImageMessage(err)
			return
		}

		defer rc.Close()
		decoder2 := json.NewDecoder(rc)
		for {
			message := &types.BuildImageMessage{}
			err := decoder2.Decode(message)
			if err != nil {
				if err == io.EOF {
					break
				}
				log.Errorf("[BuildImage] Decode push image message failed %v", err)
				return
			}
			ch <- message
		}

		// 无论如何都删掉build机器的
		// 事实上他不会跟cached pod一样
		// 一样就砍死
		go func() {
			_, err := node.Engine.ImageRemove(context.Background(), tag, enginetypes.ImageRemoveOptions{
				Force:         false,
				PruneChildren: true,
			})
			if err != nil {
				log.Errorf("[BuildImage] Remove image error: %s", err)
			}
		}()

		ch <- &types.BuildImageMessage{Stream: fmt.Sprintf("finished %s\n", tag)}
	}()

	return ch, nil
}

func makeErrorBuildImageMessage(err error) *types.BuildImageMessage {
	return &types.BuildImageMessage{Error: err.Error()}
}

func createTarStream(path string) (io.ReadCloser, error) {
	tarOpts := &archive.TarOptions{
		ExcludePatterns: []string{},
		IncludeFiles:    []string{"."},
		Compression:     archive.Uncompressed,
		NoLchown:        true,
	}
	return archive.TarWithOptions(path, tarOpts)
}

func makeCommonPart(build *types.Build) (string, error) {
	tmpl := template.Must(template.New("common").Parse(commonTmpl))
	out := bytes.Buffer{}
	if err := tmpl.Execute(&out, build); err != nil {
		return "", err
	}
	return out.String(), nil
}

func makeUserPart(opts *types.BuildOptions) (string, error) {
	tmpl := template.Must(template.New("user").Parse(userTmpl))
	out := bytes.Buffer{}
	if err := tmpl.Execute(&out, opts); err != nil {
		return "", err
	}
	return out.String(), nil
}

func makeMainPart(opts *types.BuildOptions, build *types.Build, from string, commands, copys []string) (string, error) {
	var buildTmpl []string
	common, err := makeCommonPart(build)
	if err != nil {
		return "", err
	}
	buildTmpl = append(buildTmpl, from, common)
	if len(copys) > 0 {
		buildTmpl = append(buildTmpl, copys...)
	}
	if len(commands) > 0 {
		buildTmpl = append(buildTmpl, commands...)
	}
	return strings.Join(buildTmpl, "\n"), nil
}

func (c *calcium) makeDockerFile(opts *types.BuildOptions, buildDir string) error {
	var preCache map[string]string
	var preStage string
	var buildTmpl []string

	for _, stage := range opts.Builds.Stages {
		build, ok := opts.Builds.Builds[stage]
		if !ok {
			log.Warnf("[makeDockerFile] Builds stage %s not defined", stage)
			continue
		}

		// get source or artifacts
		reponame, err := c.preparedSource(build, buildDir)
		if err != nil {
			return err
		}
		build.Repo = reponame

		// get header
		from := fmt.Sprintf(fromAsTmpl, build.Base, stage)

		// get multiple stags
		copys := []string{}
		for src, dst := range preCache {
			copys = append(copys, fmt.Sprintf(copyTmpl, preStage, src, dst))
		}

		// get commands
		commands := []string{}
		for _, command := range build.Commands {
			commands = append(commands, fmt.Sprintf(runTmpl, command))
		}

		// decide add source or not
		mainPart, err := makeMainPart(opts, build, from, commands, copys)
		if err != nil {
			return err
		}
		buildTmpl = append(buildTmpl, mainPart)
		preStage = stage
		preCache = build.Cache
	}

	if opts.User != "" && opts.UID != 0 {
		userPart, err := makeUserPart(opts)
		if err != nil {
			return err
		}
		buildTmpl = append(buildTmpl, userPart)
	}
	dockerfile := strings.Join(buildTmpl, "\n")
	return createDockerfile(dockerfile, buildDir)
}

// Dockerfile
func createDockerfile(dockerfile, buildDir string) error {
	f, err := os.Create(filepath.Join(buildDir, "Dockerfile"))
	if err != nil {
		return err
	}
	defer f.Close()
	_, err = f.WriteString(dockerfile)
	return err
}

// Image tag
// 格式严格按照 Hub/HubPrefix/appname:version 来
func createImageTag(config types.DockerConfig, appname, version string) string {
	prefix := strings.Trim(config.Namespace, "/")
	if prefix == "" {
		return fmt.Sprintf("%s/%s:%s", config.Hub, appname, version)
	}
	return fmt.Sprintf("%s/%s/%s:%s", config.Hub, prefix, appname, version)
}
<commit_msg>emit a final BuildImageMessage that indicates build success<commit_after>package calcium

import (
	"bytes"
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"
	"text/template"

	log "github.com/Sirupsen/logrus"
	enginetypes "github.com/docker/docker/api/types"
	"github.com/docker/docker/pkg/archive"
	"github.com/projecteru2/core/types"
	"github.com/projecteru2/core/utils"
)

// FIXME in alpine, useradd rename as adduser
const (
	fromAsTmpl = "FROM %s as %s"
	commonTmpl = `{{ range $k, $v:= .Args -}}
{{ printf "ARG %s=%q" $k $v }}
{{ end -}}
{{ range $k, $v:= .Envs -}}
{{ printf "ENV %s %q" $k $v }}
{{ end -}}
{{ range $k, $v:= .Labels -}}
{{ printf "LABEL %s=%s" $k $v }}
{{ end -}}
{{- if .Dir}}RUN mkdir -p {{.Dir}}
WORKDIR {{.Dir}}{{ end }}
{{ if .Repo }}ADD {{.Repo}} .{{ end }}`
	copyTmpl = "COPY --from=%s %s %s"
	runTmpl  = "RUN %s"
	userTmpl = `RUN echo "{{.User}}::{{.UID}}:{{.UID}}::/home/{{.User}}:/sbin/nologin" >> /etc/passwd && \
echo "{{.User}}:x:{{.UID}}:" >> /etc/group && \
echo "{{.User}}:!::0:::::" >> /etc/shadow
USER {{.User}}
`
)

func (c *calcium) preparedSource(build *types.Build, buildDir string) (string, error) {
	// parse repository name
	// code locates under /:repositoryname
	var cloneDir string
	var err error
	reponame := build.Repo
	if build.Repo != "" && build.Version != "" {
		reponame, err = utils.GetGitRepoName(build.Repo)
		if err != nil {
			return "", err
		}

		// clone code into cloneDir
		// which is under buildDir and named as repository name
		cloneDir = filepath.Join(buildDir, reponame)
		if err := c.source.SourceCode(build.Repo, cloneDir, build.Version); err != nil {
			return "", err
		}

		// ensure source code is safe
		// we don't want any history files to be retrieved
		if err := c.source.Security(cloneDir); err != nil {
			return "", err
		}
	}

	// if artifact download url is provided, remove all source code to
	// improve security
	if len(build.Artifacts) > 0 {
		artifactsDir := buildDir
		if cloneDir != "" {
			os.RemoveAll(cloneDir)
			os.MkdirAll(cloneDir, os.ModeDir)
			artifactsDir = cloneDir
		}
		for _, artifact := range build.Artifacts {
			if err := c.source.Artifact(artifact, artifactsDir); err != nil {
				return "", err
			}
		}
	}

	return reponame, nil
}

// BuildImage will build image for repository
// since we wanna set UID for the user inside container, we have to know the uid parameter
//
// build directory is like:
//
//    buildDir ├─ :appname ├─ code
//             ├─ Dockerfile
func (c *calcium) BuildImage(ctx context.Context, opts *types.BuildOptions) (chan *types.BuildImageMessage, error) {
	ch := make(chan *types.BuildImageMessage)

	// get pod from config
	buildPodname := c.config.Docker.BuildPod
	if buildPodname == "" {
		return ch, fmt.Errorf("No build pod set in config")
	}

	// get node by scheduler
	nodes, err := c.ListPodNodes(buildPodname, false)
	if err != nil {
		return ch, err
	}
	if len(nodes) == 0 {
		return ch, errors.New("No node to build")
	}
	node := c.scheduler.MaxIdleNode(nodes)

	// make build dir
	buildDir, err := ioutil.TempDir(os.TempDir(), "corebuild-")
	if err != nil {
		return ch, err
	}
	defer os.RemoveAll(buildDir)

	// create dockerfile
	if err := c.makeDockerFile(opts, buildDir); err != nil {
		return ch, err
	}

	// tag of image, later this will be used to push image to hub
	tag := createImageTag(c.config.Docker, opts.Name, opts.Tag)

	// create tar stream for Build API
	buildContext, err := createTarStream(buildDir)
	if err != nil {
		return ch, err
	}

	// must be put here because of that `defer os.RemoveAll(buildDir)`
	buildOptions := enginetypes.ImageBuildOptions{
		Tags:           []string{tag},
		SuppressOutput: false,
		NoCache:        true,
		Remove:         true,
		ForceRemove:    true,
		PullParent:     true,
	}

	log.Infof("[BuildImage] Building image %v at %v:%v", tag, buildPodname, node.Name)
	resp, err := node.Engine.ImageBuild(ctx, buildContext, buildOptions)
	if err != nil {
		return ch, err
	}

	go func() {
		defer resp.Body.Close()
		defer close(ch)
		decoder := json.NewDecoder(resp.Body)
		var err error
		var lastMessage *types.BuildImageMessage
		for {
			message := &types.BuildImageMessage{}
			err := decoder.Decode(message)
			if err != nil {
				if err == io.EOF {
					break
				}
				if err == context.Canceled || err == context.DeadlineExceeded {
					lastMessage.ErrorDetail.Code = -1
					lastMessage.Error = err.Error()
					break
				}
				log.Errorf("[BuildImage] Decode build image message failed %v", err)
				return
			}
			ch <- message
			lastMessage = message
		}

		if lastMessage.ErrorDetail.Code != 0 {
			log.Errorf("[BuildImage] Build image failed %v", lastMessage.ErrorDetail.Message)
			return
		}
		// About this "Khadgar", https://github.com/docker/docker/issues/10983#issuecomment-85892396
		// Just because Ben Schnetzer's cute Khadgar...
		rc, err := node.Engine.ImagePush(ctx, tag, enginetypes.ImagePushOptions{RegistryAuth: "Khadgar"})
		if err != nil {
			ch <- makeErrorBuildImageMessage(err)
			return
		}

		defer rc.Close()
		decoder2 := json.NewDecoder(rc)
		for {
			message := &types.BuildImageMessage{}
			err := decoder2.Decode(message)
			if err != nil {
				if err == io.EOF {
					break
				}
				log.Errorf("[BuildImage] Decode push image message failed %v", err)
				return
			}
			ch <- message
		}

		// 无论如何都删掉build机器的
		// 事实上他不会跟cached pod一样
		// 一样就砍死
		go func() {
			_, err := node.Engine.ImageRemove(context.Background(), tag, enginetypes.ImageRemoveOptions{
				Force:         false,
				PruneChildren: true,
			})
			if err != nil {
				log.Errorf("[BuildImage] Remove image error: %s", err)
			}
		}()

		ch <- &types.BuildImageMessage{Stream: fmt.Sprintf("finished %s\n", tag), Status: "finished", Progress: tag}
	}()

	return ch, nil
}

func makeErrorBuildImageMessage(err error) *types.BuildImageMessage {
	return &types.BuildImageMessage{Error: err.Error()}
}

func createTarStream(path string) (io.ReadCloser, error) {
	tarOpts := &archive.TarOptions{
		ExcludePatterns: []string{},
		IncludeFiles:    []string{"."},
		Compression:     archive.Uncompressed,
		NoLchown:        true,
	}
	return archive.TarWithOptions(path, tarOpts)
}

func makeCommonPart(build *types.Build) (string, error) {
	tmpl := template.Must(template.New("common").Parse(commonTmpl))
	out := bytes.Buffer{}
	if err := tmpl.Execute(&out, build); err != nil {
		return "", err
	}
	return out.String(), nil
}

func makeUserPart(opts *types.BuildOptions) (string, error) {
	tmpl := template.Must(template.New("user").Parse(userTmpl))
	out := bytes.Buffer{}
	if err := tmpl.Execute(&out, opts); err != nil {
		return "", err
	}
	return out.String(), nil
}

func makeMainPart(opts *types.BuildOptions, build *types.Build, from string, commands, copys []string) (string, error) {
	var buildTmpl []string
	common, err := makeCommonPart(build)
	if err != nil {
		return "", err
	}
	buildTmpl = append(buildTmpl, from, common)
	if len(copys) > 0 {
		buildTmpl = append(buildTmpl, copys...)
	}
	if len(commands) > 0 {
		buildTmpl = append(buildTmpl, commands...)
	}
	return strings.Join(buildTmpl, "\n"), nil
}

func (c *calcium) makeDockerFile(opts *types.BuildOptions, buildDir string) error {
	var preCache map[string]string
	var preStage string
	var buildTmpl []string

	for _, stage := range opts.Builds.Stages {
		build, ok := opts.Builds.Builds[stage]
		if !ok {
			log.Warnf("[makeDockerFile] Builds stage %s not defined", stage)
			continue
		}

		// get source or artifacts
		reponame, err := c.preparedSource(build, buildDir)
		if err != nil {
			return err
		}
		build.Repo = reponame

		// get header
		from := fmt.Sprintf(fromAsTmpl, build.Base, stage)

		// get multiple stags
		copys := []string{}
		for src, dst := range preCache {
			copys = append(copys, fmt.Sprintf(copyTmpl, preStage, src, dst))
		}

		// get commands
		commands := []string{}
		for _, command := range build.Commands {
			commands = append(commands, fmt.Sprintf(runTmpl, command))
		}

		// decide add source or not
		mainPart, err := makeMainPart(opts, build, from, commands, copys)
		if err != nil {
			return err
		}
		buildTmpl = append(buildTmpl, mainPart)
		preStage = stage
		preCache = build.Cache
	}

	if opts.User != "" && opts.UID != 0 {
		userPart, err := makeUserPart(opts)
		if err != nil {
			return err
		}
		buildTmpl = append(buildTmpl, userPart)
	}
	dockerfile := strings.Join(buildTmpl, "\n")
	return createDockerfile(dockerfile, buildDir)
}

// Dockerfile
func createDockerfile(dockerfile, buildDir string) error {
	f, err := os.Create(filepath.Join(buildDir, "Dockerfile"))
	if err != nil {
		return err
	}
	defer f.Close()
	_, err = f.WriteString(dockerfile)
	return err
}

// Image tag
// 格式严格按照 Hub/HubPrefix/appname:version 来
func createImageTag(config types.DockerConfig, appname, version string) string {
	prefix := strings.Trim(config.Namespace, "/")
	if prefix == "" {
		return fmt.Sprintf("%s/%s:%s", config.Hub, appname, version)
	}
	return fmt.Sprintf("%s/%s/%s:%s", config.Hub, prefix, appname, version)
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"log"
	"net"
	"net/url"
	"os"
	"os/signal"
	"strings"
	"syscall"

	"github.com/CMGS/go-dockerclient"
)

var debugMode bool

func debug(v ...interface{}) {
	if debugMode {
		log.Println(v...)
	}
}

func assert(err error, context string) {
	if err != nil {
		log.Fatal(context+": ", err)
	}
}

func udpStreamer(target Target, types []string, logstream chan *Log) {
	typestr := "," + strings.Join(types, ",") + ","
	addr, err := net.ResolveUDPAddr("udp", target.Addr)
	assert(err, "resolve udp failed")
	conn, err := net.DialUDP("udp", nil, addr)
	assert(err, "connect udp failed")
	encoder := json.NewEncoder(conn)
	defer conn.Close()
	for logline := range logstream {
		if typestr != ",," && !strings.Contains(typestr, logline.Type) {
			continue
		}
		appinfo := strings.SplitN(logline.Name, "_", 2)
		logline.Appname = appinfo[0]
		logline.Port = appinfo[1]
		logline.Tag = target.AppendTag
		encoder.Encode(logline)
	}
}

func main() {
	flag.BoolVar(&debugMode, "DEBUG", false, "enable debug")
	endpoint := flag.String("docker", "unix:///var/run/docker.sock", "docker location")
	routes := flag.String("routes", "/var/lib/logspout", "routes path")
	forwarder := flag.String("forwarder", "udp://127.0.0.1:20000", "log forward dest")
	flag.Parse()

	client, err := docker.NewClient(*endpoint)
	assert(err, "docker")
	attacher := NewAttachManager(client)
	router := NewRouteManager(attacher)

	u, err := url.Parse(*forwarder)
	assert(err, "url")
	log.Println("routing all to " + *forwarder)
	router.Add(&Route{Target: Target{Type: u.Scheme, Addr: u.Host}})

	if _, err := os.Stat(*routes); err == nil {
		log.Println("loading and persisting routes in " + *routes)
		assert(router.Load(RouteFileStore(*routes)), "persistor")
	}

	var c = make(chan os.Signal, 1)
	signal.Notify(c, os.Interrupt)
	signal.Notify(c, syscall.SIGTERM)
	for {
		fmt.Println("Catch", <-c)
		break
	}
}
<commit_msg>catch hup signal<commit_after>package main

import (
	"encoding/json"
	"flag"
	"log"
	"net"
	"net/url"
	"os"
	"os/signal"
	"strings"
	"syscall"

	"github.com/CMGS/go-dockerclient"
)

var debugMode bool

func debug(v ...interface{}) {
	if debugMode {
		log.Println(v...)
	}
}

func assert(err error, context string) {
	if err != nil {
		log.Fatal(context+": ", err)
	}
}

func udpStreamer(target Target, types []string, logstream chan *Log) {
	typestr := "," + strings.Join(types, ",") + ","
	addr, err := net.ResolveUDPAddr("udp", target.Addr)
	assert(err, "resolve udp failed")
	conn, err := net.DialUDP("udp", nil, addr)
	assert(err, "connect udp failed")
	encoder := json.NewEncoder(conn)
	defer conn.Close()
	for logline := range logstream {
		if typestr != ",," && !strings.Contains(typestr, logline.Type) {
			continue
		}
		appinfo := strings.SplitN(logline.Name, "_", 2)
		logline.Appname = appinfo[0]
		logline.Port = appinfo[1]
		logline.Tag = target.AppendTag
		encoder.Encode(logline)
	}
}

func main() {
	flag.BoolVar(&debugMode, "DEBUG", false, "enable debug")
	endpoint := flag.String("docker", "unix:///var/run/docker.sock", "docker location")
	routes := flag.String("routes", "/var/lib/logspout", "routes path")
	forwarder := flag.String("forwarder", "udp://127.0.0.1:20000", "log forward dest")
	flag.Parse()

	client, err := docker.NewClient(*endpoint)
	assert(err, "docker")
	attacher := NewAttachManager(client)
	router := NewRouteManager(attacher)

	u, err := url.Parse(*forwarder)
	assert(err, "url")
	log.Println("routing all to " + *forwarder)
	router.Add(&Route{Target: Target{Type: u.Scheme, Addr: u.Host}})

	if _, err := os.Stat(*routes); err == nil {
		log.Println("loading and persisting routes in " + *routes)
		assert(router.Load(RouteFileStore(*routes)), "persistor")
	}

	var c = make(chan os.Signal, 1)
	signal.Notify(c, os.Interrupt)
	signal.Notify(c, syscall.SIGTERM)
	signal.Notify(c, syscall.SIGHUP)
	for {
		log.Println("Catch", <-c)
		break
	}
}
<|endoftext|>


<commit_before>package logstash

import (
	"encoding/json"
	"errors"
	"log"
	"net"
	"regexp"
	"strings"
	"os"

	"github.com/gliderlabs/logspout/router"
)

func init() {
	router.AdapterFactories.Register(NewAdapter, "logstash")
}

// Adapter is an adapter that streams UDP JSON to Logstash.
type Adapter struct {
	conn  net.Conn
	route *router.Route
}

// NewAdapter creates an Adapter with UDP as the default transport.
func NewAdapter(route *router.Route) (router.LogAdapter, error) {
	transport, found := router.AdapterTransports.Lookup(route.AdapterTransport("udp"))
	if !found {
		return nil, errors.New("unable to find adapter: " + route.Adapter)
	}

	conn, err := transport.Dial(route.Address, route.Options)
	if err != nil {
		return nil, err
	}

	return &Adapter{
		route: route,
		conn:  conn,
	}, nil
}

// MergeMessages merges an array of Message into a string
func MergeMessages(messages []Message) string {
	var strs = make([]string, 0)

	for _, x := range messages {
		strs = append(strs, x.Message)
	}

	return strings.Join(strs, "\n")
}

// GetTags decides if a message array should be tagged multiline.
func GetTags (messages []Message) []string {
	var tags = make([]string, 0)

	if len(messages) > 1 {
		tags = append(tags, "multiline")
	} else {
		tags = append(tags, "")
	}

	return tags
}

// GetHostname gets the HOSTNAME variable or the container's hostname.
func GetHostname() string {
	hostname := os.Getenv("HOSTNAME")

	if hostname == "" {
		log.Println("logstash: Defaulting to container hostname.")
		hostname, err := os.Hostname()
		if err != nil {
			log.Println("logstash_hostname:", err)
		}
		return hostname
	}
	return hostname
}

// Stream implements the router.LogAdapter interface.
func (a *Adapter) Stream(logstream chan *router.Message) {
	queue := make(map[string][]Message)

	hostname := GetHostname()

	for m := range logstream {
		rawMessage := Message{
			Message:  m.Data,
		}
		finalMessage := Message{}

		// Internal hardcoded multiline. This is terrible, I know.
		matched, err := regexp.Match("^\\s", []byte(m.Data))
		if err != nil {
			log.Println("logstash_regex:", err)
			continue
		}

		_, existing := queue[m.Container.ID];

		// Create an empty slice if there is no queue slice.
		if !existing {
			queue[m.Container.ID] = []Message{}
		}

		if matched {
			queue[m.Container.ID] = append(queue[m.Container.ID], rawMessage)
			continue
		} else {
			if len(queue[m.Container.ID]) == 0 {
				queue[m.Container.ID] = append(queue[m.Container.ID], rawMessage)
				continue
			} else {
				finalMessage = Message{
					Message: MergeMessages(queue[m.Container.ID]),
					Name: m.Container.Name,
					ID: m.Container.ID,
					Image: m.Container.Config.Image,
					Hostname: m.Container.Config.Hostname,
					Stream: m.Source,
					Tags: GetTags(queue[m.Container.ID]),
					Host: hostname,
				}
				queue[m.Container.ID] = []Message{}
			}
		}

		// Mashal the message into JSON.
		js, err := json.Marshal(finalMessage)
		if err != nil {
			log.Println("logstash_marshal:", err)
			continue
		}

		// Write the message to the Logstash server.
		_, err = a.conn.Write(js)
		if err != nil {
			log.Println("logstash_write:", err)
			continue
		}
	}
}

// Message is a simple JSON input to Logstash.
type Message struct {
	Message  string   `json:"message"`
	Name     string   `json:"container_name"`
	ID       string   `json:"container_id"`
	Image    string   `json:"image_name"`
	Hostname string   `json:"container_hostname"`
	Host     string   `json:"host"`
	Stream   string   `json:"stream"`
	Tags     []string `json:"tags"`
}
<commit_msg>Remove container name trailing slash.<commit_after>package logstash

import (
	"encoding/json"
	"errors"
	"log"
	"net"
	"regexp"
	"strings"
	"os"

	"github.com/gliderlabs/logspout/router"
)

func init() {
	router.AdapterFactories.Register(NewAdapter, "logstash")
}

// Adapter is an adapter that streams UDP JSON to Logstash.
type Adapter struct {
	conn  net.Conn
	route *router.Route
}

// NewAdapter creates an Adapter with UDP as the default transport.
func NewAdapter(route *router.Route) (router.LogAdapter, error) {
	transport, found := router.AdapterTransports.Lookup(route.AdapterTransport("udp"))
	if !found {
		return nil, errors.New("unable to find adapter: " + route.Adapter)
	}

	conn, err := transport.Dial(route.Address, route.Options)
	if err != nil {
		return nil, err
	}

	return &Adapter{
		route: route,
		conn:  conn,
	}, nil
}

// MergeMessages merges an array of Message into a string
func MergeMessages(messages []Message) string {
	var strs = make([]string, 0)

	for _, x := range messages {
		strs = append(strs, x.Message)
	}

	return strings.Join(strs, "\n")
}

// GetTags decides if a message array should be tagged multiline.
func GetTags (messages []Message) []string {
	var tags = make([]string, 0)

	if len(messages) > 1 {
		tags = append(tags, "multiline")
	} else {
		tags = append(tags, "")
	}

	return tags
}

// GetHostname gets the HOSTNAME variable or the container's hostname.
func GetHostname() string {
	hostname := os.Getenv("HOSTNAME")

	if hostname == "" {
		log.Println("logstash: Defaulting to container hostname.")
		hostname, err := os.Hostname()
		if err != nil {
			log.Println("logstash_hostname:", err)
		}
		return hostname
	}
	return hostname
}

// Stream implements the router.LogAdapter interface.
func (a *Adapter) Stream(logstream chan *router.Message) {
	queue := make(map[string][]Message)

	hostname := GetHostname()

	for m := range logstream {
		rawMessage := Message{
			Message:  m.Data,
		}
		finalMessage := Message{}

		// Internal hardcoded multiline. This is terrible, I know.
		matched, err := regexp.Match("^\\s", []byte(m.Data))
		if err != nil {
			log.Println("logstash_regex:", err)
			continue
		}

		_, existing := queue[m.Container.ID];

		// Create an empty slice if there is no queue slice.
		if !existing {
			queue[m.Container.ID] = []Message{}
		}

		if matched {
			queue[m.Container.ID] = append(queue[m.Container.ID], rawMessage)
			continue
		} else {
			if len(queue[m.Container.ID]) == 0 {
				queue[m.Container.ID] = append(queue[m.Container.ID], rawMessage)
				continue
			} else {
				// remove trailing slash from container name
				containerName := strings.TrimLeft(m.Container.Name, "/")

				finalMessage = Message{
					Message: MergeMessages(queue[m.Container.ID]),
					Name: containerName,
					ID: m.Container.ID,
					Image: m.Container.Config.Image,
					Hostname: m.Container.Config.Hostname,
					Stream: m.Source,
					Tags: GetTags(queue[m.Container.ID]),
					Host: hostname,
				}
				queue[m.Container.ID] = []Message{}
			}
		}

		// Mashal the message into JSON.
		js, err := json.Marshal(finalMessage)
		if err != nil {
			log.Println("logstash_marshal:", err)
			continue
		}

		// Write the message to the Logstash server.
		_, err = a.conn.Write(js)
		if err != nil {
			log.Println("logstash_write:", err)
			continue
		}
	}
}

// Message is a simple JSON input to Logstash.
type Message struct {
	Message  string   `json:"message"`
	Name     string   `json:"container_name"`
	ID       string   `json:"container_id"`
	Image    string   `json:"image_name"`
	Hostname string   `json:"container_hostname"`
	Host     string   `json:"host"`
	Stream   string   `json:"stream"`
	Tags     []string `json:"tags"`
}
<|endoftext|>


<commit_before>package policy

import (
	"fmt"
	"time"

	"github.com/kopia/kopia/snapshot"
)

// RetentionPolicy describes snapshot retention policy.
type RetentionPolicy struct {
	KeepLatest  *int `json:"keepLatest,omitempty"`
	KeepHourly  *int `json:"keepHourly,omitempty"`
	KeepDaily   *int `json:"keepDaily,omitempty"`
	KeepWeekly  *int `json:"keepWeekly,omitempty"`
	KeepMonthly *int `json:"keepMonthly,omitempty"`
	KeepAnnual  *int `json:"keepAnnual,omitempty"`
}

// ComputeRetentionReasons computes the reasons why each snapshot is retained, based on
// the settings in retention policy and stores them in RetentionReason field.
func (r *RetentionPolicy) ComputeRetentionReasons(manifests []*snapshot.Manifest) {
	now := time.Now()
	maxTime := now.Add(365 * 24 * time.Hour)

	cutoffTime := func(setting *int, add func(time.Time, int) time.Time) time.Time {
		if setting != nil {
			return add(now, *setting)
		}

		return maxTime
	}

	cutoff := cutoffTimes{
		annual:  cutoffTime(r.KeepAnnual, yearsAgo),
		monthly: cutoffTime(r.KeepMonthly, monthsAgo),
		daily:   cutoffTime(r.KeepDaily, daysAgo),
		hourly:  cutoffTime(r.KeepHourly, hoursAgo),
		weekly:  cutoffTime(r.KeepHourly, weeksAgo),
	}

	ids := make(map[string]bool)
	idCounters := make(map[string]int)

	for i, s := range snapshot.SortByTime(manifests, true) {
		s.RetentionReasons = r.getRetentionReasons(i, s, cutoff, ids, idCounters)
	}
}

func (r *RetentionPolicy) getRetentionReasons(i int, s *snapshot.Manifest, cutoff cutoffTimes, ids map[string]bool, idCounters map[string]int) []string {
	if s.IncompleteReason != "" {
		return nil
	}

	keepReasons := []string{}
	var zeroTime time.Time

	yyyy, wk := s.StartTime.ISOWeek()

	cases := []struct {
		cutoffTime     time.Time
		timePeriodID   string
		timePeriodType string
		max            *int
	}{
		{zeroTime, fmt.Sprintf("%v", i), "latest", r.KeepLatest},
		{cutoff.annual, s.StartTime.Format("2006"), "annual", r.KeepAnnual},
		{cutoff.monthly, s.StartTime.Format("2006-01"), "monthly", r.KeepMonthly},
		{cutoff.weekly, fmt.Sprintf("%04v-%02v", yyyy, wk), "weekly", r.KeepWeekly},
		{cutoff.daily, s.StartTime.Format("2006-01-02"), "daily", r.KeepDaily},
		{cutoff.hourly, s.StartTime.Format("2006-01-02 15"), "hourly", r.KeepHourly},
	}

	for _, c := range cases {
		if c.max == nil {
			continue
		}
		if s.StartTime.Before(c.cutoffTime) {
			continue
		}

		if _, exists := ids[c.timePeriodID]; exists {
			continue
		}

		if idCounters[c.timePeriodType] < *c.max {
			ids[c.timePeriodID] = true
			idCounters[c.timePeriodType]++
			keepReasons = append(keepReasons, fmt.Sprintf("%v-%v", c.timePeriodType, idCounters[c.timePeriodType]))
		}
	}

	return keepReasons
}

type cutoffTimes struct {
	annual  time.Time
	monthly time.Time
	daily   time.Time
	hourly  time.Time
	weekly  time.Time
}

func yearsAgo(base time.Time, n int) time.Time {
	return base.AddDate(-n, 0, 0)
}

func monthsAgo(base time.Time, n int) time.Time {
	return base.AddDate(0, -n, 0)
}

func daysAgo(base time.Time, n int) time.Time {
	return base.AddDate(0, 0, -n)
}

func weeksAgo(base time.Time, n int) time.Time {
	return base.AddDate(0, 0, -n*7)
}

func hoursAgo(base time.Time, n int) time.Time {
	return base.Add(time.Duration(-n) * time.Hour)
}

var defaultRetentionPolicy = RetentionPolicy{
	KeepLatest:  intPtr(1),
	KeepHourly:  intPtr(48),
	KeepDaily:   intPtr(7),
	KeepWeekly:  intPtr(4),
	KeepMonthly: intPtr(4),
	KeepAnnual:  intPtr(0),
}

// Merge applies default values from the provided policy.
func (r *RetentionPolicy) Merge(src RetentionPolicy) {
	if r.KeepLatest == nil {
		r.KeepLatest = src.KeepLatest
	}
	if r.KeepHourly == nil {
		r.KeepHourly = src.KeepHourly
	}
	if r.KeepDaily == nil {
		r.KeepDaily = src.KeepDaily
	}
	if r.KeepWeekly == nil {
		r.KeepWeekly = src.KeepWeekly
	}
	if r.KeepMonthly == nil {
		r.KeepMonthly = src.KeepMonthly
	}
	if r.KeepAnnual == nil {
		r.KeepAnnual = src.KeepAnnual
	}
}
<commit_msg>snapshot: automatically retain the most recent incomplete snapshot<commit_after>package policy

import (
	"fmt"
	"time"

	"github.com/kopia/kopia/snapshot"
)

// RetentionPolicy describes snapshot retention policy.
type RetentionPolicy struct {
	KeepLatest  *int `json:"keepLatest,omitempty"`
	KeepHourly  *int `json:"keepHourly,omitempty"`
	KeepDaily   *int `json:"keepDaily,omitempty"`
	KeepWeekly  *int `json:"keepWeekly,omitempty"`
	KeepMonthly *int `json:"keepMonthly,omitempty"`
	KeepAnnual  *int `json:"keepAnnual,omitempty"`
}

// ComputeRetentionReasons computes the reasons why each snapshot is retained, based on
// the settings in retention policy and stores them in RetentionReason field.
func (r *RetentionPolicy) ComputeRetentionReasons(manifests []*snapshot.Manifest) {
	now := time.Now()
	maxTime := now.Add(365 * 24 * time.Hour)

	cutoffTime := func(setting *int, add func(time.Time, int) time.Time) time.Time {
		if setting != nil {
			return add(now, *setting)
		}

		return maxTime
	}

	cutoff := cutoffTimes{
		annual:  cutoffTime(r.KeepAnnual, yearsAgo),
		monthly: cutoffTime(r.KeepMonthly, monthsAgo),
		daily:   cutoffTime(r.KeepDaily, daysAgo),
		hourly:  cutoffTime(r.KeepHourly, hoursAgo),
		weekly:  cutoffTime(r.KeepHourly, weeksAgo),
	}

	ids := make(map[string]bool)
	idCounters := make(map[string]int)

	sorted := snapshot.SortByTime(manifests, true)
	for i, s := range sorted {
		s.RetentionReasons = r.getRetentionReasons(i, s, cutoff, ids, idCounters)
	}
	if len(sorted) > 0 && sorted[0].IncompleteReason != "" {
		sorted[0].RetentionReasons = append(sorted[0].RetentionReasons, "last-incomplete")
	}
}

func (r *RetentionPolicy) getRetentionReasons(i int, s *snapshot.Manifest, cutoff cutoffTimes, ids map[string]bool, idCounters map[string]int) []string {
	if s.IncompleteReason != "" {
		return nil
	}

	keepReasons := []string{}
	var zeroTime time.Time

	yyyy, wk := s.StartTime.ISOWeek()

	cases := []struct {
		cutoffTime     time.Time
		timePeriodID   string
		timePeriodType string
		max            *int
	}{
		{zeroTime, fmt.Sprintf("%v", i), "latest", r.KeepLatest},
		{cutoff.annual, s.StartTime.Format("2006"), "annual", r.KeepAnnual},
		{cutoff.monthly, s.StartTime.Format("2006-01"), "monthly", r.KeepMonthly},
		{cutoff.weekly, fmt.Sprintf("%04v-%02v", yyyy, wk), "weekly", r.KeepWeekly},
		{cutoff.daily, s.StartTime.Format("2006-01-02"), "daily", r.KeepDaily},
		{cutoff.hourly, s.StartTime.Format("2006-01-02 15"), "hourly", r.KeepHourly},
	}

	for _, c := range cases {
		if c.max == nil {
			continue
		}
		if s.StartTime.Before(c.cutoffTime) {
			continue
		}

		if _, exists := ids[c.timePeriodID]; exists {
			continue
		}

		if idCounters[c.timePeriodType] < *c.max {
			ids[c.timePeriodID] = true
			idCounters[c.timePeriodType]++
			keepReasons = append(keepReasons, fmt.Sprintf("%v-%v", c.timePeriodType, idCounters[c.timePeriodType]))
		}
	}

	return keepReasons
}

type cutoffTimes struct {
	annual  time.Time
	monthly time.Time
	daily   time.Time
	hourly  time.Time
	weekly  time.Time
}

func yearsAgo(base time.Time, n int) time.Time {
	return base.AddDate(-n, 0, 0)
}

func monthsAgo(base time.Time, n int) time.Time {
	return base.AddDate(0, -n, 0)
}

func daysAgo(base time.Time, n int) time.Time {
	return base.AddDate(0, 0, -n)
}

func weeksAgo(base time.Time, n int) time.Time {
	return base.AddDate(0, 0, -n*7)
}

func hoursAgo(base time.Time, n int) time.Time {
	return base.Add(time.Duration(-n) * time.Hour)
}

var defaultRetentionPolicy = RetentionPolicy{
	KeepLatest:  intPtr(1),
	KeepHourly:  intPtr(48),
	KeepDaily:   intPtr(7),
	KeepWeekly:  intPtr(4),
	KeepMonthly: intPtr(4),
	KeepAnnual:  intPtr(0),
}

// Merge applies default values from the provided policy.
func (r *RetentionPolicy) Merge(src RetentionPolicy) {
	if r.KeepLatest == nil {
		r.KeepLatest = src.KeepLatest
	}
	if r.KeepHourly == nil {
		r.KeepHourly = src.KeepHourly
	}
	if r.KeepDaily == nil {
		r.KeepDaily = src.KeepDaily
	}
	if r.KeepWeekly == nil {
		r.KeepWeekly = src.KeepWeekly
	}
	if r.KeepMonthly == nil {
		r.KeepMonthly = src.KeepMonthly
	}
	if r.KeepAnnual == nil {
		r.KeepAnnual = src.KeepAnnual
	}
}
<|endoftext|>


<commit_before>package executor

import (
	"fmt"
	"sync"
	"time"

	"github.com/funkygao/fae/servant/mysql"
	"github.com/funkygao/gafka/cmd/kateway/job"
	jm "github.com/funkygao/gafka/cmd/kateway/job/mysql"
	"github.com/funkygao/gafka/cmd/kateway/manager"
	"github.com/funkygao/gafka/cmd/kateway/store"
	log "github.com/funkygao/log4go"
)

const (
	LagThreshold = 3 // in sec
)

// JobExecutor polls a single JobQueue and handle each Job.
type JobExecutor struct {
	parentId       string // controller short id
	cluster, topic string
	mc             *mysql.MysqlCluster
	stopper        <-chan struct{}
	dueJobs        chan job.JobItem
	auditor        log.Logger

	// cached values
	appid string
	aid   int
	table string
	ident string
}

func NewJobExecutor(parentId, cluster, topic string, mc *mysql.MysqlCluster,
	stopper <-chan struct{}, auditor log.Logger) *JobExecutor {
	this := &JobExecutor{
		parentId: parentId,
		cluster:  cluster,
		topic:    topic,
		mc:       mc,
		stopper:  stopper,
		dueJobs:  make(chan job.JobItem, 200),
		auditor:  auditor,
	}

	return this
}

// poll mysql for due jobs and send to kafka.
func (this *JobExecutor) Run() {
	this.appid = manager.Default.TopicAppid(this.topic)
	if this.appid == "" {
		log.Warn("invalid topic: %s", this.topic)
		return
	}
	this.aid = jm.App_id(this.appid)
	this.table = jm.JobTable(this.topic)
	this.ident = this.topic

	log.Trace("starting %s", this.Ident())

	var (
		wg   sync.WaitGroup
		item job.JobItem
		tick = time.NewTicker(time.Second)
		sql  = fmt.Sprintf("SELECT job_id,payload,ctime,due_time FROM %s WHERE due_time<=?", this.table)
	)

	// handler pool, currently to guarantee the order, we use pool=1
	for i := 0; i < 1; i++ {
		wg.Add(1)
		go this.handleDueJobs(&wg)
	}

	for {
		select {
		case <-this.stopper:
			log.Debug("%s stopping", this.ident)
			wg.Wait()
			return

		case now := <-tick.C:
			rows, err := this.mc.Query(jm.AppPool, this.topic, this.aid, sql, now.Unix())
			if err != nil {
				log.Error("%s: %v", this.ident, err)
				continue
			}

			for rows.Next() {
				err = rows.Scan(&item.JobId, &item.Payload, &item.Ctime, &item.DueTime)
				if err == nil {
					log.Debug("%s due %s", this.ident, item)
					if lag := now.Unix() - item.DueTime; lag > LagThreshold {
						log.Warn("%s lag %ds %s", this.ident, lag, item)
					}

					this.dueJobs <- item
				} else {
					log.Error("%s: %s", this.ident, err)
				}
			}

			if err = rows.Err(); err != nil {
				log.Error("%s: %s", this.ident, err)
			}

			rows.Close()
		}
	}

}

// TODO batch DELETE/INSERT for better performance.
func (this *JobExecutor) handleDueJobs(wg *sync.WaitGroup) {
	defer wg.Done()

	var (
		// zabbix maintains a in-memory delete queue
		// delete from history_uint where itemid=? and clock<min_clock
		sqlDeleteJob = fmt.Sprintf("DELETE FROM %s WHERE job_id=?", this.table)

		sqlInsertArchive = fmt.Sprintf("INSERT INTO %s(job_id,payload,ctime,due_time,etime,actor_id) VALUES(?,?,?,?,?,?)",
			jm.HistoryTable(this.topic))
		sqlReinject = fmt.Sprintf("INSERT INTO %s(job_id, payload, ctime, due_time) VALUES(?,?,?,?)", this.table)
	)
	for {
		select {
		case <-this.stopper:
			return

		case item := <-this.dueJobs:
			now := time.Now()
			affectedRows, _, err := this.mc.Exec(jm.AppPool, this.table, this.aid, sqlDeleteJob, item.JobId)
			if err != nil {
				log.Error("%s: %s", this.ident, err)
				continue
			}
			if affectedRows == 0 {
				// 2 possibilities:
				// - client Cancel job wins
				// - this handler is too slow and the job fetched twice in tick
				continue
			}

			log.Debug("%s land %s", this.ident, item)
			_, _, err = store.DefaultPubStore.SyncPub(this.cluster, this.topic, nil, item.Payload)
			if err != nil {
				log.Error("%s: %s", this.ident, err)
				this.mc.Exec(jm.AppPool, this.table, this.aid, sqlReinject,
					item.JobId, item.Payload, item.Ctime, item.DueTime)
				continue
			}

			log.Debug("%s fired %s", this.ident, item)
			this.auditor.Trace(item.String())

			// mv job to archive table
			_, _, err = this.mc.Exec(jm.AppPool, this.table, this.aid, sqlInsertArchive,
				item.JobId, item.Payload, item.Ctime, item.DueTime, now.Unix(), this.parentId)
			if err != nil {
				log.Error("%s: %s", this.ident, err)
			} else {
				log.Debug("%s archived %s", this.ident, item)
			}

		}
	}
}

func (this *JobExecutor) Ident() string {
	return this.ident
}
<commit_msg>FIXME due jobs handler concurrency breaks order guarantee<commit_after>package executor

import (
	"fmt"
	"sync"
	"time"

	"github.com/funkygao/fae/servant/mysql"
	"github.com/funkygao/gafka/cmd/kateway/job"
	jm "github.com/funkygao/gafka/cmd/kateway/job/mysql"
	"github.com/funkygao/gafka/cmd/kateway/manager"
	"github.com/funkygao/gafka/cmd/kateway/store"
	log "github.com/funkygao/log4go"
)

const (
	LagWarnThreshold   = 3  // in sec
	HandlerConcurrentN = 10 // FIXME breaks the delivery order guarantee
)

// JobExecutor polls a single JobQueue and handle each Job.
type JobExecutor struct {
	parentId       string // controller short id
	cluster, topic string
	mc             *mysql.MysqlCluster
	stopper        <-chan struct{}
	dueJobs        chan job.JobItem
	auditor        log.Logger

	// cached values
	appid string
	aid   int
	table string
	ident string
}

func NewJobExecutor(parentId, cluster, topic string, mc *mysql.MysqlCluster,
	stopper <-chan struct{}, auditor log.Logger) *JobExecutor {
	this := &JobExecutor{
		parentId: parentId,
		cluster:  cluster,
		topic:    topic,
		mc:       mc,
		stopper:  stopper,
		dueJobs:  make(chan job.JobItem, 200),
		auditor:  auditor,
	}

	return this
}

// poll mysql for due jobs and send to kafka.
func (this *JobExecutor) Run() {
	this.appid = manager.Default.TopicAppid(this.topic)
	if this.appid == "" {
		log.Warn("invalid topic: %s", this.topic)
		return
	}
	this.aid = jm.App_id(this.appid)
	this.table = jm.JobTable(this.topic)
	this.ident = this.topic

	log.Trace("starting %s", this.Ident())

	var (
		wg   sync.WaitGroup
		item job.JobItem
		tick = time.NewTicker(time.Second)
		sql  = fmt.Sprintf("SELECT job_id,payload,ctime,due_time FROM %s WHERE due_time<=?", this.table)
	)

	for i := 0; i < HandlerConcurrentN; i++ {
		wg.Add(1)
		go this.handleDueJobs(&wg)
	}

	for {
		select {
		case <-this.stopper:
			log.Debug("%s stopping", this.ident)
			wg.Wait()
			return

		case now := <-tick.C:
			rows, err := this.mc.Query(jm.AppPool, this.topic, this.aid, sql, now.Unix())
			if err != nil {
				log.Error("%s: %v", this.ident, err)
				continue
			}

			for rows.Next() {
				err = rows.Scan(&item.JobId, &item.Payload, &item.Ctime, &item.DueTime)
				if err == nil {
					log.Debug("%s due %s", this.ident, item)
					if lag := now.Unix() - item.DueTime; lag > LagWarnThreshold {
						log.Warn("%s lag %ds %s", this.ident, lag, item)
					}

					this.dueJobs <- item
				} else {
					log.Error("%s: %s", this.ident, err)
				}
			}

			if err = rows.Err(); err != nil {
				log.Error("%s: %s", this.ident, err)
			}

			rows.Close()
		}
	}

}

// TODO batch DELETE/INSERT for better performance.
func (this *JobExecutor) handleDueJobs(wg *sync.WaitGroup) {
	defer wg.Done()

	var (
		// zabbix maintains a in-memory delete queue
		// delete from history_uint where itemid=? and clock<min_clock
		sqlDeleteJob = fmt.Sprintf("DELETE FROM %s WHERE job_id=?", this.table)

		sqlInsertArchive = fmt.Sprintf("INSERT INTO %s(job_id,payload,ctime,due_time,etime,actor_id) VALUES(?,?,?,?,?,?)",
			jm.HistoryTable(this.topic))
		sqlReinject = fmt.Sprintf("INSERT INTO %s(job_id, payload, ctime, due_time) VALUES(?,?,?,?)", this.table)
	)
	for {
		select {
		case <-this.stopper:
			return

		case item := <-this.dueJobs:
			now := time.Now()
			affectedRows, _, err := this.mc.Exec(jm.AppPool, this.table, this.aid, sqlDeleteJob, item.JobId)
			if err != nil {
				log.Error("%s: %s", this.ident, err)
				continue
			}
			if affectedRows == 0 {
				// 2 possibilities:
				// - client Cancel job wins
				// - this handler is too slow and the job fetched twice in tick
				continue
			}

			log.Debug("%s land %s", this.ident, item)
			_, _, err = store.DefaultPubStore.SyncPub(this.cluster, this.topic, nil, item.Payload)
			if err != nil {
				log.Error("%s: %s", this.ident, err)
				this.mc.Exec(jm.AppPool, this.table, this.aid, sqlReinject,
					item.JobId, item.Payload, item.Ctime, item.DueTime)
				continue
			}

			log.Debug("%s fired %s", this.ident, item)
			this.auditor.Trace(item.String())

			// mv job to archive table
			_, _, err = this.mc.Exec(jm.AppPool, this.table, this.aid, sqlInsertArchive,
				item.JobId, item.Payload, item.Ctime, item.DueTime, now.Unix(), this.parentId)
			if err != nil {
				log.Error("%s: %s", this.ident, err)
			} else {
				log.Debug("%s archived %s", this.ident, item)
			}

		}
	}
}

func (this *JobExecutor) Ident() string {
	return this.ident
}
<|endoftext|>