<commit_before>// Package telegram_bot_api provides methods for communicating with Telegram Bot API.
package telegram_bot_api

import (
    "sync"
    "log"
    "net/http"
    // "net/url"
    "io/ioutil"
    "strconv"
    "mime/multipart"
    "bytes"
    "path/filepath"
    "io"
    // "reflect"
)

type (
    Param struct {
        Key string
        Value string
        FileUrl string
    }

    ApiManager struct {
        baseApi string
        secretToken string
        DebugMode bool
    }
)

var (
    once sync.Once
    instance *ApiManager
)

// NewApiManager returns a singleton API manager.
func NewApiManager(baseApi string, secretToken string, debugMode bool) *ApiManager {
    once.Do(func() {
        instance = &ApiManager{
            baseApi: baseApi,
            secretToken: secretToken,
            DebugMode: debugMode,
        }
    })
    return instance
}

func (this *ApiManager) SendMessage(chatId int, text string) {
    url := this.getUrl("sendMessage")

    params := []*Param{
        &Param{Key: "text", FileUrl: text},
        &Param{Key: "chat_id", Value: strconv.Itoa(chatId)},
    }

    // reply_to_message_id

    _, err := this.sendPost(url, params)
    if err != nil {
        log.Println("ERROR", err)
    }
}

func (this *ApiManager) GetMe() {
    url := this.getUrl("getMe")

    this.sendGet(url, nil)
}

func (this *ApiManager) SendPhoto(chatId int, fileUrl string) {
    url := this.getUrl("sendPhoto")

    params := []*Param{
        &Param{Key: "photo", FileUrl: fileUrl},
        &Param{Key: "chat_id", Value: strconv.Itoa(chatId)},
    }

    this.sendPost(url, params)
}

func (this *ApiManager) SetWebhook(webhookUrl string) {
    url := this.getUrl("setWebhook")

    params := []*Param{
        &Param{Key: "url", Value: webhookUrl},
    }

    this.sendGet(url, params)
}

func (this *ApiManager) sendPost(url string, params []*Param) ([]int, error) {
    body := &bytes.Buffer{}
    writer := multipart.NewWriter(body)

    for _, param := range params {
        if param.FileUrl != "" {
            res, err := http.Get(param.FileUrl)

            if err != nil { return nil, err }
            defer res.Body.Close()

            file := res.Body
            part, err := writer.CreateFormFile(param.Key, filepath.Base(param.FileUrl))
            if err != nil { return nil, err }

            _, err = io.Copy(part, file)
            if err != nil { return nil, err }
        } else {
            writer.WriteField(param.Key, param.Value)
        }
    }

    err := writer.Close()
    if err != nil { return nil, err }

    req, _ := http.NewRequest("POST", url, body)
    req.Header.Set("Content-Type", writer.FormDataContentType())

    client := &http.Client{}
    resp, err := client.Do(req)

    if err != nil { return nil, err }
    defer resp.Body.Close()

    resBody, _ := ioutil.ReadAll(resp.Body)

    if this.DebugMode {
        log.Println("response Status:", resp.Status)
        log.Println("response Headers:", resp.Header)
        log.Println("response Body:", string(resBody))
    }

    return nil, nil
}

func (this *ApiManager) sendGet(url string, params []*Param) {
    url += "?"

    for i, param := range params {
        if i > 0 { url += "&" }
        url += param.Key + "=" + param.Value
    }

    resp, err := http.Get(url)

    if err != nil {
    	log.Println(err)
    }

    defer resp.Body.Close()

    body, _ := ioutil.ReadAll(resp.Body)

    if this.DebugMode {
        log.Println("response Status:", resp.Status)
        log.Println("response Headers:", resp.Header)
        log.Println("response Body:", string(body))
    }
}

func (this *ApiManager) getUrl(method string) (url string) {
    url = this.baseApi + "/" + this.secretToken + "/" + method
    return
}
<commit_msg>fix param key in SendMessage<commit_after>// Package telegram_bot_api provides methods for communicating with Telegram Bot API.
package telegram_bot_api

import (
    "sync"
    "log"
    "net/http"
    // "net/url"
    "io/ioutil"
    "strconv"
    "mime/multipart"
    "bytes"
    "path/filepath"
    "io"
    // "reflect"
)

type (
    Param struct {
        Key string
        Value string
        FileUrl string
    }

    ApiManager struct {
        baseApi string
        secretToken string
        DebugMode bool
    }
)

var (
    once sync.Once
    instance *ApiManager
)

// NewApiManager returns a singleton API manager.
func NewApiManager(baseApi string, secretToken string, debugMode bool) *ApiManager {
    once.Do(func() {
        instance = &ApiManager{
            baseApi: baseApi,
            secretToken: secretToken,
            DebugMode: debugMode,
        }
    })
    return instance
}

func (this *ApiManager) SendMessage(chatId int, text string) {
    url := this.getUrl("sendMessage")

    params := []*Param{
        &Param{Key: "text", Value: text},
        &Param{Key: "chat_id", Value: strconv.Itoa(chatId)},
    }

    // reply_to_message_id

    _, err := this.sendPost(url, params)
    if err != nil {
        log.Println("ERROR", err)
    }
}

func (this *ApiManager) GetMe() {
    url := this.getUrl("getMe")

    this.sendGet(url, nil)
}

func (this *ApiManager) SendPhoto(chatId int, fileUrl string) {
    url := this.getUrl("sendPhoto")

    params := []*Param{
        &Param{Key: "photo", FileUrl: fileUrl},
        &Param{Key: "chat_id", Value: strconv.Itoa(chatId)},
    }

    this.sendPost(url, params)
}

func (this *ApiManager) SetWebhook(webhookUrl string) {
    url := this.getUrl("setWebhook")

    params := []*Param{
        &Param{Key: "url", Value: webhookUrl},
    }

    this.sendGet(url, params)
}

func (this *ApiManager) sendPost(url string, params []*Param) ([]int, error) {
    body := &bytes.Buffer{}
    writer := multipart.NewWriter(body)

    for _, param := range params {
        if param.FileUrl != "" {
            res, err := http.Get(param.FileUrl)

            if err != nil { return nil, err }
            defer res.Body.Close()

            file := res.Body
            part, err := writer.CreateFormFile(param.Key, filepath.Base(param.FileUrl))
            if err != nil { return nil, err }

            _, err = io.Copy(part, file)
            if err != nil { return nil, err }
        } else {
            writer.WriteField(param.Key, param.Value)
        }
    }

    err := writer.Close()
    if err != nil { return nil, err }

    req, _ := http.NewRequest("POST", url, body)
    req.Header.Set("Content-Type", writer.FormDataContentType())

    client := &http.Client{}
    resp, err := client.Do(req)

    if err != nil { return nil, err }
    defer resp.Body.Close()

    resBody, _ := ioutil.ReadAll(resp.Body)

    if this.DebugMode {
        log.Println("response Status:", resp.Status)
        log.Println("response Headers:", resp.Header)
        log.Println("response Body:", string(resBody))
    }

    return nil, nil
}

func (this *ApiManager) sendGet(url string, params []*Param) {
    url += "?"

    for i, param := range params {
        if i > 0 { url += "&" }
        url += param.Key + "=" + param.Value
    }

    resp, err := http.Get(url)

    if err != nil {
    	log.Println(err)
    }

    defer resp.Body.Close()

    body, _ := ioutil.ReadAll(resp.Body)

    if this.DebugMode {
        log.Println("response Status:", resp.Status)
        log.Println("response Headers:", resp.Header)
        log.Println("response Body:", string(body))
    }
}

func (this *ApiManager) getUrl(method string) (url string) {
    url = this.baseApi + "/" + this.secretToken + "/" + method
    return
}
<|endoftext|>


<commit_before>package main

import (
  "encoding/json"
  "fmt"
  "net/http"
  "strconv"
  "github.com/rcrowley/go-tigertonic"
  "github.com/rkbodenner/parallel_universe/collection"
  "github.com/rkbodenner/parallel_universe/game"
  "github.com/rkbodenner/parallel_universe/session"
)

var players = []*game.Player{
  &game.Player{1, "Player One"},
  &game.Player{2, "Player Two"},
}

var gameCollection = collection.NewCollection()
var gameIndex = make(map[uint64]*game.Game)

func initGameData() {
  for i,game := range gameCollection.Games {
    game.Id = (uint)(i+1)
    gameIndex[(uint64)(i+1)] = game
  }
}

var sessions []*session.Session
var sessionIndex = make(map[uint64]*session.Session)

func initSessionData() {
  sessions = make([]*session.Session, 2)
  sessions[0] = session.NewSession(gameCollection.Games[0], 2)
  sessions[1] = session.NewSession(gameCollection.Games[1], 2)

  for i,session := range sessions {
    session.Id = (uint)(i+1)
    sessionIndex[(uint64)(i+1)] = session
  }
}

func corsHandler(handler func(http.ResponseWriter, *http.Request)) func(http.ResponseWriter, *http.Request) {
  return func(w http.ResponseWriter, r *http.Request) {
    header := w.Header()
    header.Add("Access-Control-Allow-Origin", "http://localhost:8000")
    handler(w, r)
  }
}

func collectionHandler(w http.ResponseWriter, r *http.Request) {
  err := json.NewEncoder(w).Encode(gameCollection)
  if ( nil != err ) {
    fmt.Fprintln(w, err)
  }
}

func gameHandler(w http.ResponseWriter, r *http.Request) {
  id_str := r.URL.Query().Get("id")
  id, err := strconv.ParseUint(id_str, 10, 64)
  if nil != err {
    http.Error(w, "Not found", http.StatusNotFound)
    return
  }

  game, ok := gameIndex[id]
  if ok {
    err := json.NewEncoder(w).Encode(game)
    if ( nil != err ) {
      http.Error(w, "Error", http.StatusInternalServerError)
    }
  } else {
    http.Error(w, "Not found", http.StatusNotFound)
  }
}

func playersHandler(w http.ResponseWriter, r *http.Request) {
  err := json.NewEncoder(w).Encode(players)
  if ( nil != err ) {
    fmt.Fprintln(w, err)
  }
}

func sessionsHandler(w http.ResponseWriter, r *http.Request) {
  err := json.NewEncoder(w).Encode(sessions)
  if ( nil != err ) {
    http.Error(w, "Error", http.StatusInternalServerError)
  }
}

func sessionHandler(w http.ResponseWriter, r *http.Request) {
  id_str := r.URL.Query().Get("id")
  id, err := strconv.ParseUint(id_str, 10, 64)
  if nil != err {
    http.Error(w, "Not found", http.StatusNotFound)
    return
  }

  session, ok := sessionIndex[id]
  if ok {
    err := json.NewEncoder(w).Encode(session)
    if ( nil != err ) {
      http.Error(w, "Error", http.StatusInternalServerError)
    }
  } else {
    http.Error(w, "Not found", http.StatusNotFound)
  }
}

func main() {
  initGameData()
  initSessionData()

  mux := tigertonic.NewTrieServeMux()
  mux.HandleFunc("GET", "/games", corsHandler(collectionHandler))
  mux.HandleFunc("GET", "/games/{id}", corsHandler(gameHandler))
  mux.HandleFunc("GET", "/players", corsHandler(playersHandler))
  mux.HandleFunc("GET", "/sessions", corsHandler(sessionsHandler))
  mux.HandleFunc("GET", "/sessions/{id}", corsHandler(sessionHandler))
  http.ListenAndServe(":8080", mux)
}
<commit_msg>Create session with player objects<commit_after>package main

import (
  "encoding/json"
  "fmt"
  "net/http"
  "strconv"
  "github.com/rcrowley/go-tigertonic"
  "github.com/rkbodenner/parallel_universe/collection"
  "github.com/rkbodenner/parallel_universe/game"
  "github.com/rkbodenner/parallel_universe/session"
)

var players = []*game.Player{
  &game.Player{1, "Player One"},
  &game.Player{2, "Player Two"},
}

var gameCollection = collection.NewCollection()
var gameIndex = make(map[uint64]*game.Game)

func initGameData() {
  for i,game := range gameCollection.Games {
    game.Id = (uint)(i+1)
    gameIndex[(uint64)(i+1)] = game
  }
}

var sessions []*session.Session
var sessionIndex = make(map[uint64]*session.Session)

func initSessionData() {
  sessions = make([]*session.Session, 2)
  sessions[0] = session.NewSession(gameCollection.Games[0], players)
  sessions[0].Step(players[0])
  sessions[0].Step(players[1])

  sessions[1] = session.NewSession(gameCollection.Games[1], players)
  sessions[1].Step(players[0])
  sessions[1].Step(players[1])

  for i,session := range sessions {
    session.Id = (uint)(i+1)
    sessionIndex[(uint64)(i+1)] = session
  }
}

func corsHandler(handler func(http.ResponseWriter, *http.Request)) func(http.ResponseWriter, *http.Request) {
  return func(w http.ResponseWriter, r *http.Request) {
    header := w.Header()
    header.Add("Access-Control-Allow-Origin", "http://localhost:8000")
    handler(w, r)
  }
}

func collectionHandler(w http.ResponseWriter, r *http.Request) {
  err := json.NewEncoder(w).Encode(gameCollection)
  if ( nil != err ) {
    fmt.Fprintln(w, err)
  }
}

func gameHandler(w http.ResponseWriter, r *http.Request) {
  id_str := r.URL.Query().Get("id")
  id, err := strconv.ParseUint(id_str, 10, 64)
  if nil != err {
    http.Error(w, "Not found", http.StatusNotFound)
    return
  }

  game, ok := gameIndex[id]
  if ok {
    err := json.NewEncoder(w).Encode(game)
    if ( nil != err ) {
      http.Error(w, "Error", http.StatusInternalServerError)
    }
  } else {
    http.Error(w, "Not found", http.StatusNotFound)
  }
}

func playersHandler(w http.ResponseWriter, r *http.Request) {
  err := json.NewEncoder(w).Encode(players)
  if ( nil != err ) {
    fmt.Fprintln(w, err)
  }
}

func sessionsHandler(w http.ResponseWriter, r *http.Request) {
  err := json.NewEncoder(w).Encode(sessions)
  if ( nil != err ) {
    http.Error(w, "Error", http.StatusInternalServerError)
  }
}

func sessionHandler(w http.ResponseWriter, r *http.Request) {
  id_str := r.URL.Query().Get("id")
  id, err := strconv.ParseUint(id_str, 10, 64)
  if nil != err {
    http.Error(w, "Not found", http.StatusNotFound)
    return
  }

  session, ok := sessionIndex[id]
  if ok {
    err := json.NewEncoder(w).Encode(session)
    if ( nil != err ) {
      http.Error(w, "Error", http.StatusInternalServerError)
    }
  } else {
    http.Error(w, "Not found", http.StatusNotFound)
  }
}

func main() {
  initGameData()
  initSessionData()

  mux := tigertonic.NewTrieServeMux()
  mux.HandleFunc("GET", "/games", corsHandler(collectionHandler))
  mux.HandleFunc("GET", "/games/{id}", corsHandler(gameHandler))
  mux.HandleFunc("GET", "/players", corsHandler(playersHandler))
  mux.HandleFunc("GET", "/sessions", corsHandler(sessionsHandler))
  mux.HandleFunc("GET", "/sessions/{id}", corsHandler(sessionHandler))
  http.ListenAndServe(":8080", mux)
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"log"
	"os"
	"path/filepath"
	"strings"
	"time"

	"github.com/jingweno/travisarchive/db"
	"github.com/jingweno/travisarchive/export/uploader"
	"github.com/joho/godotenv"
)

var (
	execDir  string
	mongoURL string
)

func init() {
	godotenv.Load("../.env")
	flag.StringVar(&execDir, "e", "", "dir to the mongoexport executable")
	flag.StringVar(&mongoURL, "u", os.Getenv("MONGOHQ_URL"), "URL of the Mongo server")
}

func main() {
	flag.Parse()

	if execDir == "" {
		log.Fatal(fmt.Errorf("specify the dir to the mongoexport executable with -e"))
	}

	if mongoURL == "" {
		log.Fatal(fmt.Errorf("specify the URL of Mongo server with -u"))
	}

	db, err := db.Connect()
	if err != nil {
		log.Fatal(err)
	}

	cols, err := db.DB().CollectionNames()
	if err != nil {
		log.Fatal(err)
	}

	exportBuilds(cols)
}

func exportBuilds(cols []string) {
	oneDayAgo := time.Now().Add(-24 * time.Hour).UTC()
	for _, col := range cols {
		d, err := parseDate(col)
		if err != nil {
			continue
		}

		if d.UTC().After(oneDayAgo) {
			continue
		}

		log.Printf("exporting %s...\n", col)
		cmd := &MongoExport{ExecDir: execDir, URL: mongoURL, ColName: col}
		outfile, err := cmd.Run()
		if err != nil {
			log.Println(err)
			continue
		}

		log.Printf("exported to %s\n", outfile)

		archiver := &Archiver{outfile}
		outzip, err := archiver.Archive()
		if err != nil {
			log.Println(err)
			continue
		}

		log.Printf("archived to %s\n", outzip)

		zipfile, err := os.Open(outzip)
		if err != nil {
			log.Println(err)
			continue
		}
		defer zipfile.Close()

		filename := fmt.Sprintf("/builds/%s", filepath.Base(outzip))
		u, err := uploader.New("s3")
		if err != nil {
			log.Println(err)
			continue
		}
		err = u.Upload(filename, "application/zip", zipfile)
		if err != nil {
			log.Println(err)
			continue
		}

		log.Printf("uploaded to s3 %s", filename)
	}
}

func parseDate(col string) (time.Time, error) {
	if !strings.HasPrefix(col, "builds_") {
		return time.Time{}, fmt.Errorf("input doesn't include the right prefix")
	}

	timePart := strings.SplitN(col, "_", 2)[1]
	form := "2006_01_02"

	return time.Parse(form, timePart)
}
<commit_msg>Use MONGO_URL<commit_after>package main

import (
	"flag"
	"fmt"
	"log"
	"os"
	"path/filepath"
	"strings"
	"time"

	"github.com/jingweno/travisarchive/db"
	"github.com/jingweno/travisarchive/export/uploader"
	"github.com/joho/godotenv"
)

var (
	execDir  string
	mongoURL string
)

func init() {
	godotenv.Load("../.env")
	flag.StringVar(&execDir, "e", "", "dir to the mongoexport executable")
	flag.StringVar(&mongoURL, "u", os.Getenv("MONGO_URL"), "URL of the Mongo server")
}

func main() {
	flag.Parse()

	if execDir == "" {
		log.Fatal(fmt.Errorf("specify the dir to the mongoexport executable with -e"))
	}

	if mongoURL == "" {
		log.Fatal(fmt.Errorf("specify the URL of Mongo server with -u"))
	}

	db, err := db.Connect()
	if err != nil {
		log.Fatal(err)
	}

	cols, err := db.DB().CollectionNames()
	if err != nil {
		log.Fatal(err)
	}

	exportBuilds(cols)
}

func exportBuilds(cols []string) {
	oneDayAgo := time.Now().Add(-24 * time.Hour).UTC()
	for _, col := range cols {
		d, err := parseDate(col)
		if err != nil {
			continue
		}

		if d.UTC().After(oneDayAgo) {
			continue
		}

		log.Printf("exporting %s...\n", col)
		cmd := &MongoExport{ExecDir: execDir, URL: mongoURL, ColName: col}
		outfile, err := cmd.Run()
		if err != nil {
			log.Println(err)
			continue
		}

		log.Printf("exported to %s\n", outfile)

		archiver := &Archiver{outfile}
		outzip, err := archiver.Archive()
		if err != nil {
			log.Println(err)
			continue
		}

		log.Printf("archived to %s\n", outzip)

		zipfile, err := os.Open(outzip)
		if err != nil {
			log.Println(err)
			continue
		}
		defer zipfile.Close()

		filename := fmt.Sprintf("/builds/%s", filepath.Base(outzip))
		u, err := uploader.New("s3")
		if err != nil {
			log.Println(err)
			continue
		}
		err = u.Upload(filename, "application/zip", zipfile)
		if err != nil {
			log.Println(err)
			continue
		}

		log.Printf("uploaded to s3 %s", filename)
	}
}

func parseDate(col string) (time.Time, error) {
	if !strings.HasPrefix(col, "builds_") {
		return time.Time{}, fmt.Errorf("input doesn't include the right prefix")
	}

	timePart := strings.SplitN(col, "_", 2)[1]
	form := "2006_01_02"

	return time.Parse(form, timePart)
}
<|endoftext|>


<commit_before>// +build go1.5,cgo

package plugin // import "collectd.org/plugin"

// #cgo CPPFLAGS: -DHAVE_CONFIG_H
// #cgo LDFLAGS: -ldl
// #include <stdlib.h>
// #include <dlfcn.h>
// #include "plugin.h"
//
// static void (*plugin_log_) (int, char const *, ...) = NULL;
// void wrap_plugin_log(int severity, char *msg) {
//   if (plugin_log_ == NULL) {
//     void *hnd = dlopen(NULL, RTLD_LAZY);
//     plugin_log_ = dlsym(hnd, "plugin_log");
//     dlclose(hnd);
//   }
//   (*plugin_log_) (severity, "%s", msg);
// }
import "C"

import (
	"fmt"
	"unsafe"
)

// Severity is the severity of log messages. These are well-known constants
// within collectd, so don't define your own. Use the constants provided by
// this package instead.
type Severity int

// Predefined severities for collectd log functions.
const (
	SeverityError   Severity = 3
	SeverityWarning Severity = 4
	SeverityNotice  Severity = 5
	SeverityInfo    Severity = 6
	SeverityDebug   Severity = 7
)

func log(s Severity, msg string) error {
	ptr := C.CString(msg)
	defer C.free(unsafe.Pointer(ptr))

	_, err := C.wrap_plugin_log(C.int(s), ptr)
	return err
}

// Error logs an error using plugin_log(). Arguments are handled in the manner
// of fmt.Print.
func Error(v ...interface{}) error {
	return log(SeverityError, fmt.Sprint(v...))
}

// Errorf logs an error using plugin_log(). Arguments are handled in the manner
// of fmt.Printf.
func Errorf(format string, v ...interface{}) error {
	return Error(fmt.Sprintf(format, v...))
}

// Warning logs a warning using plugin_log(). Arguments are handled in the
// manner of fmt.Print.
func Warning(v ...interface{}) error {
	return log(SeverityWarning, fmt.Sprint(v...))
}

// Warningf logs a warning using plugin_log(). Arguments are handled in the
// manner of fmt.Printf.
func Warningf(format string, v ...interface{}) error {
	return Warning(fmt.Sprintf(format, v...))
}

// Notice logs a notice using plugin_log(). Arguments are handled in the manner
// of fmt.Print.
func Notice(v ...interface{}) error {
	return log(SeverityNotice, fmt.Sprint(v...))
}

// Noticef logs a notice using plugin_log(). Arguments are handled in the
// manner of fmt.Printf.
func Noticef(format string, v ...interface{}) error {
	return Notice(fmt.Sprintf(format, v...))
}

// Info logs a purely informal message using plugin_log(). Arguments are
// handled in the manner of fmt.Print.
func Info(v ...interface{}) error {
	return log(SeverityInfo, fmt.Sprint(v...))
}

// Infof logs a purely informal message using plugin_log(). Arguments are
// handled in the manner of fmt.Printf.
func Infof(format string, v ...interface{}) error {
	return Info(fmt.Sprintf(format, v...))
}

// Debug logs a debugging message using plugin_log(). Arguments are handled in
// the manner of fmt.Print.
func Debug(v ...interface{}) error {
	return log(SeverityDebug, fmt.Sprint(v...))
}

// Debugf logs a debugging message using plugin_log(). Arguments are handled in
// the manner of fmt.Printf.
func Debugf(format string, v ...interface{}) error {
	return Debug(fmt.Sprintf(format, v...))
}
<commit_msg>Package plugin: Trim trailing whitespace from log messages.<commit_after>// +build go1.5,cgo

package plugin // import "collectd.org/plugin"

// #cgo CPPFLAGS: -DHAVE_CONFIG_H
// #cgo LDFLAGS: -ldl
// #include <stdlib.h>
// #include <dlfcn.h>
// #include "plugin.h"
//
// static void (*plugin_log_) (int, char const *, ...) = NULL;
// void wrap_plugin_log(int severity, char *msg) {
//   if (plugin_log_ == NULL) {
//     void *hnd = dlopen(NULL, RTLD_LAZY);
//     plugin_log_ = dlsym(hnd, "plugin_log");
//     dlclose(hnd);
//   }
//   (*plugin_log_) (severity, "%s", msg);
// }
import "C"

import (
	"fmt"
	"strings"
	"unicode"
	"unsafe"
)

// Severity is the severity of log messages. These are well-known constants
// within collectd, so don't define your own. Use the constants provided by
// this package instead.
type Severity int

// Predefined severities for collectd log functions.
const (
	SeverityError   Severity = 3
	SeverityWarning Severity = 4
	SeverityNotice  Severity = 5
	SeverityInfo    Severity = 6
	SeverityDebug   Severity = 7
)

func log(s Severity, msg string) error {
	// Trim trailing whitespace.
	msg = strings.TrimRightFunc(msg, unicode.IsSpace)

	ptr := C.CString(msg)
	defer C.free(unsafe.Pointer(ptr))

	_, err := C.wrap_plugin_log(C.int(s), ptr)
	return wrapCError(0, err, "plugin_log")
}

// Error logs an error using plugin_log(). Arguments are handled in the manner
// of fmt.Print.
func Error(v ...interface{}) error {
	return log(SeverityError, fmt.Sprint(v...))
}

// Errorf logs an error using plugin_log(). Arguments are handled in the manner
// of fmt.Printf.
func Errorf(format string, v ...interface{}) error {
	return Error(fmt.Sprintf(format, v...))
}

// Warning logs a warning using plugin_log(). Arguments are handled in the
// manner of fmt.Print.
func Warning(v ...interface{}) error {
	return log(SeverityWarning, fmt.Sprint(v...))
}

// Warningf logs a warning using plugin_log(). Arguments are handled in the
// manner of fmt.Printf.
func Warningf(format string, v ...interface{}) error {
	return Warning(fmt.Sprintf(format, v...))
}

// Notice logs a notice using plugin_log(). Arguments are handled in the manner
// of fmt.Print.
func Notice(v ...interface{}) error {
	return log(SeverityNotice, fmt.Sprint(v...))
}

// Noticef logs a notice using plugin_log(). Arguments are handled in the
// manner of fmt.Printf.
func Noticef(format string, v ...interface{}) error {
	return Notice(fmt.Sprintf(format, v...))
}

// Info logs a purely informal message using plugin_log(). Arguments are
// handled in the manner of fmt.Print.
func Info(v ...interface{}) error {
	return log(SeverityInfo, fmt.Sprint(v...))
}

// Infof logs a purely informal message using plugin_log(). Arguments are
// handled in the manner of fmt.Printf.
func Infof(format string, v ...interface{}) error {
	return Info(fmt.Sprintf(format, v...))
}

// Debug logs a debugging message using plugin_log(). Arguments are handled in
// the manner of fmt.Print.
func Debug(v ...interface{}) error {
	return log(SeverityDebug, fmt.Sprint(v...))
}

// Debugf logs a debugging message using plugin_log(). Arguments are handled in
// the manner of fmt.Printf.
func Debugf(format string, v ...interface{}) error {
	return Debug(fmt.Sprintf(format, v...))
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"log"
	"net/http"
	"os"

	"github.com/fweikert/continuous-integration/metrics/clients"
	"github.com/fweikert/continuous-integration/metrics/metrics"
	"github.com/fweikert/continuous-integration/metrics/publishers"
	"github.com/fweikert/continuous-integration/metrics/service"
)

var (
	projectID             = flag.String("project_id", "bazel-untrusted", "ID of the GCP project.")
	datastoreSettingsName = flag.String("datastore_settings_name", "MetricSettings", "Name of the settings entity in Datastore.")
	testMode              = flag.Bool("test", false, "If true, the service will collect and publish all metrics immediately and only once.")
)

const megaByte = 1024 * 1024

func handleError(metricName string, err error) {
	fmt.Printf("[%s] %v\n", metricName, err)
}

func handleRequest(w http.ResponseWriter, r *http.Request) {
	fmt.Fprintln(w, "You should try https://bazel.build")
}

func main() {
	flag.Parse()

	settings, err := ReadSettingsFromDatastore(*projectID, *datastoreSettingsName)
	if err != nil {
		log.Fatalf("Could not read settings from Datastore: %v", err)
	}
	pipelines, err := settings.GetPipelineIDs()
	if err != nil {
		log.Fatalf("Could not get Buildkite pipeline IDs from Datastore: %v", err)
	}
	if len(pipelines) == 0 {
		log.Fatalf("No pipelines were specified.")
	}

	bk, err := clients.CreateBuildkiteClient(settings.BuildkiteApiToken, settings.BuildkiteDebug)
	if err != nil {
		log.Fatalf("Cannot create Buildkite client: %v", err)
	}

	gcs, err := clients.CreateGcsClient()
	if err != nil {
		log.Fatalf("Cannot create GCS client: %v", err)
	}

	/*
		stackdriverClient, err := clients.CreateStackdriverClient()
		if err != nil {
			log.Fatalf("Cannot create Stackdriver client: %v", err)
		}

		stackdriver := publishers.CreateStackdriverPublisher(stackdriverClient, *projectID)
	*/

	cloudSql, err := publishers.CreateCloudSqlPublisher(settings.CloudSqlUser, settings.CloudSqlPassword, settings.CloudSqlInstance, settings.CloudSqlDatabase, settings.CloudSqlLocalPort)
	if err != nil {
		log.Fatalf("Failed to set up Cloud SQL publisher: %v", err)
	}

	/*
		stdout := publishers.CreateStdoutPublisher(publishers.Csv)
	*/

	srv := service.CreateService(handleError)

	buildsPerChange := metrics.CreateBuildsPerChange(bk, 500, pipelines...)
	srv.AddMetric(buildsPerChange, 60, cloudSql)

	buildSuccess := metrics.CreateBuildSuccess(bk, 200, pipelines...)
	srv.AddMetric(buildSuccess, 60, cloudSql)

	flakiness := metrics.CreateFlakiness(gcs, "bazel-buildkite-stats", "flaky-tests-bep", pipelines...)
	srv.AddMetric(flakiness, 60, cloudSql)

	macPerformance := metrics.CreateMacPerformance(bk, 20, pipelines...)
	srv.AddMetric(macPerformance, 60, cloudSql)

	pipelinePerformance := metrics.CreatePipelinePerformance(bk, 20, pipelines...)
	srv.AddMetric(pipelinePerformance, 60, cloudSql)

	platformLoad := metrics.CreatePlatformLoad(bk, 100, settings.BuildkiteOrgs...)
	srv.AddMetric(platformLoad, 60, cloudSql)

	platformSignificance := metrics.CreatePlatformSignificance(bk, 100, pipelines...)
	srv.AddMetric(platformSignificance, 24*60, cloudSql)

	platformUsage := metrics.CreatePlatformUsage(bk, 100, settings.BuildkiteOrgs...)
	srv.AddMetric(platformUsage, 60, cloudSql)

	releaseDownloads := metrics.CreateReleaseDownloads(settings.GitHubOrg,
		settings.GitHubRepo,
		settings.GitHubApiToken, megaByte)
	srv.AddMetric(releaseDownloads, 12*60, cloudSql)

	workerAvailability := metrics.CreateWorkerAvailability(bk, settings.BuildkiteOrgs...)
	srv.AddMetric(workerAvailability, 60, cloudSql)

	if *testMode {
		log.Println("[Test mode] Running all jobs exactly once...")
		srv.RunJobsOnce()
		os.Exit(0)
	}

	srv.Start()
	http.HandleFunc("/", handleRequest)

	port := os.Getenv("PORT")
	if port == "" {
		port = "8080"
	}
	log.Fatal(http.ListenAndServe(fmt.Sprintf(":%s", port), nil))
}
<commit_msg>Collect platform_load and worker_availability every five minutes<commit_after>package main

import (
	"flag"
	"fmt"
	"log"
	"net/http"
	"os"

	"github.com/fweikert/continuous-integration/metrics/clients"
	"github.com/fweikert/continuous-integration/metrics/metrics"
	"github.com/fweikert/continuous-integration/metrics/publishers"
	"github.com/fweikert/continuous-integration/metrics/service"
)

var (
	projectID             = flag.String("project_id", "bazel-untrusted", "ID of the GCP project.")
	datastoreSettingsName = flag.String("datastore_settings_name", "MetricSettings", "Name of the settings entity in Datastore.")
	testMode              = flag.Bool("test", false, "If true, the service will collect and publish all metrics immediately and only once.")
)

const megaByte = 1024 * 1024

func handleError(metricName string, err error) {
	fmt.Printf("[%s] %v\n", metricName, err)
}

func handleRequest(w http.ResponseWriter, r *http.Request) {
	fmt.Fprintln(w, "You should try https://bazel.build")
}

func main() {
	flag.Parse()

	settings, err := ReadSettingsFromDatastore(*projectID, *datastoreSettingsName)
	if err != nil {
		log.Fatalf("Could not read settings from Datastore: %v", err)
	}
	pipelines, err := settings.GetPipelineIDs()
	if err != nil {
		log.Fatalf("Could not get Buildkite pipeline IDs from Datastore: %v", err)
	}
	if len(pipelines) == 0 {
		log.Fatalf("No pipelines were specified.")
	}

	bk, err := clients.CreateBuildkiteClient(settings.BuildkiteApiToken, settings.BuildkiteDebug)
	if err != nil {
		log.Fatalf("Cannot create Buildkite client: %v", err)
	}

	gcs, err := clients.CreateGcsClient()
	if err != nil {
		log.Fatalf("Cannot create GCS client: %v", err)
	}

	/*
		stackdriverClient, err := clients.CreateStackdriverClient()
		if err != nil {
			log.Fatalf("Cannot create Stackdriver client: %v", err)
		}

		stackdriver := publishers.CreateStackdriverPublisher(stackdriverClient, *projectID)
	*/

	cloudSql, err := publishers.CreateCloudSqlPublisher(settings.CloudSqlUser, settings.CloudSqlPassword, settings.CloudSqlInstance, settings.CloudSqlDatabase, settings.CloudSqlLocalPort)
	if err != nil {
		log.Fatalf("Failed to set up Cloud SQL publisher: %v", err)
	}

	/*
		stdout := publishers.CreateStdoutPublisher(publishers.Csv)
	*/

	srv := service.CreateService(handleError)

	buildsPerChange := metrics.CreateBuildsPerChange(bk, 500, pipelines...)
	srv.AddMetric(buildsPerChange, 60, cloudSql)

	buildSuccess := metrics.CreateBuildSuccess(bk, 200, pipelines...)
	srv.AddMetric(buildSuccess, 60, cloudSql)

	flakiness := metrics.CreateFlakiness(gcs, "bazel-buildkite-stats", "flaky-tests-bep", pipelines...)
	srv.AddMetric(flakiness, 60, cloudSql)

	macPerformance := metrics.CreateMacPerformance(bk, 20, pipelines...)
	srv.AddMetric(macPerformance, 60, cloudSql)

	pipelinePerformance := metrics.CreatePipelinePerformance(bk, 20, pipelines...)
	srv.AddMetric(pipelinePerformance, 60, cloudSql)

	platformLoad := metrics.CreatePlatformLoad(bk, 100, settings.BuildkiteOrgs...)
	srv.AddMetric(platformLoad, 5, cloudSql)

	platformSignificance := metrics.CreatePlatformSignificance(bk, 100, pipelines...)
	srv.AddMetric(platformSignificance, 24*60, cloudSql)

	platformUsage := metrics.CreatePlatformUsage(bk, 100, settings.BuildkiteOrgs...)
	srv.AddMetric(platformUsage, 60, cloudSql)

	releaseDownloads := metrics.CreateReleaseDownloads(settings.GitHubOrg,
		settings.GitHubRepo,
		settings.GitHubApiToken, megaByte)
	srv.AddMetric(releaseDownloads, 12*60, cloudSql)

	workerAvailability := metrics.CreateWorkerAvailability(bk, settings.BuildkiteOrgs...)
	srv.AddMetric(workerAvailability, 5, cloudSql)

	if *testMode {
		log.Println("[Test mode] Running all jobs exactly once...")
		srv.RunJobsOnce()
		os.Exit(0)
	}

	srv.Start()
	http.HandleFunc("/", handleRequest)

	port := os.Getenv("PORT")
	if port == "" {
		port = "8080"
	}
	log.Fatal(http.ListenAndServe(fmt.Sprintf(":%s", port), nil))
}
<|endoftext|>


<commit_before>package cfclient

import (
	"encoding/json"
	"net/url"
)

// ProcessListResponse is the json body returned from the API
type ProcessListResponse struct {
	Pagination Pagination `json:"pagination"`
	Processes  []Process  `json:"resources"`
}

// Process represents a running process in a container.
type Process struct {
	GUID        string `json:"guid"`
	Type        string `json:"type"`
	Instances   int    `json:"instances"`
	MemoryInMB  int    `json:"memory_in_mb"`
	DiskInMB    int    `json:"disk_in_mb"`
	Ports       []int  `json:"ports,omitempty"`
	CreatedAt   string `json:"created_at"`
	UpdatedAt   string `json:"updated_at"`
	HealthCheck struct {
		Type string `json:"type"`
		Data struct {
			Timeout           int    `json:"timeout"`
			InvocationTimeout int    `json:"invocation_timeout"`
			Endpoint          string `json:"endpoint"`
		} `json:"data"`
	} `json:"health_check"`
	Links struct {
		Self  Link `json:"self"`
		Scale Link `json:"scale"`
		App   Link `json:"app"`
		Space Link `json:"space"`
		Stats Link `json:"stats"`
	} `json:"links"`
}

// ListAllProcesses will call the v3 processes api
func (c *Client) ListAllProcesses() ([]Process, error) {
	return c.ListAllProcessesByQuery(url.Values{})
}

// ListAllProcessesByQuery will call the v3 processes api
func (c *Client) ListAllProcessesByQuery(query url.Values) ([]Process, error) {
	var allProcesses []Process

	requestURL := "/v3/processes"
	if e := query.Encode(); len(e) > 0 {
		requestURL += "?" + e
	}
	for {
		resp, err := c.getProcessPage(requestURL)
		if err != nil {
			return nil, err
		}

		if resp.Pagination.TotalResults == 0 {
			return nil, nil
		}

		if allProcesses == nil {
			allProcesses = make([]Process, 0, resp.Pagination.TotalResults)
		}

		allProcesses = append(allProcesses, resp.Processes...)
		if resp.Pagination.Next.Href == "" {
			break
		}

		requestURL := resp.Pagination.Next.Href
		if requestURL == "" {
			return allProcesses, nil
		}
		requestURL, err = extractPathFromURL(requestURL)
		if err != nil {
			return nil, err
		}
	}

	return allProcesses, nil
}

func (c *Client) getProcessPage(requestURL string) (*ProcessListResponse, error) {
	req := c.NewRequest("GET", requestURL)

	resp, err := c.DoRequest(req)
	if err != nil {
		return nil, err
	}

	procResp := new(ProcessListResponse)
	defer resp.Body.Close()
	err = json.NewDecoder(resp.Body).Decode(procResp)
	if err != nil {
		return nil, err
	}

	return procResp, nil
}
<commit_msg>Prevent requestURL from being a shadow variable<commit_after>package cfclient

import (
	"encoding/json"
	"net/url"
)

// ProcessListResponse is the json body returned from the API
type ProcessListResponse struct {
	Pagination Pagination `json:"pagination"`
	Processes  []Process  `json:"resources"`
}

// Process represents a running process in a container.
type Process struct {
	GUID        string `json:"guid"`
	Type        string `json:"type"`
	Instances   int    `json:"instances"`
	MemoryInMB  int    `json:"memory_in_mb"`
	DiskInMB    int    `json:"disk_in_mb"`
	Ports       []int  `json:"ports,omitempty"`
	CreatedAt   string `json:"created_at"`
	UpdatedAt   string `json:"updated_at"`
	HealthCheck struct {
		Type string `json:"type"`
		Data struct {
			Timeout           int    `json:"timeout"`
			InvocationTimeout int    `json:"invocation_timeout"`
			Endpoint          string `json:"endpoint"`
		} `json:"data"`
	} `json:"health_check"`
	Links struct {
		Self  Link `json:"self"`
		Scale Link `json:"scale"`
		App   Link `json:"app"`
		Space Link `json:"space"`
		Stats Link `json:"stats"`
	} `json:"links"`
}

// ListAllProcesses will call the v3 processes api
func (c *Client) ListAllProcesses() ([]Process, error) {
	return c.ListAllProcessesByQuery(url.Values{})
}

// ListAllProcessesByQuery will call the v3 processes api
func (c *Client) ListAllProcessesByQuery(query url.Values) ([]Process, error) {
	var allProcesses []Process

	requestURL := "/v3/processes"
	if e := query.Encode(); len(e) > 0 {
		requestURL += "?" + e
	}
	for {
		resp, err := c.getProcessPage(requestURL)
		if err != nil {
			return nil, err
		}

		if resp.Pagination.TotalResults == 0 {
			return nil, nil
		}

		if allProcesses == nil {
			allProcesses = make([]Process, 0, resp.Pagination.TotalResults)
		}

		allProcesses = append(allProcesses, resp.Processes...)
		if resp.Pagination.Next.Href == "" {
			break
		}

		requestURL = resp.Pagination.Next.Href
		if requestURL == "" {
			return allProcesses, nil
		}
		requestURL, err = extractPathFromURL(requestURL)
		if err != nil {
			return nil, err
		}
	}

	return allProcesses, nil
}

func (c *Client) getProcessPage(requestURL string) (*ProcessListResponse, error) {
	req := c.NewRequest("GET", requestURL)

	resp, err := c.DoRequest(req)
	if err != nil {
		return nil, err
	}

	procResp := new(ProcessListResponse)
	defer resp.Body.Close()
	err = json.NewDecoder(resp.Body).Decode(procResp)
	if err != nil {
		return nil, err
	}

	return procResp, nil
}
<|endoftext|>


<commit_before>package nakadi

import (
	"context"
	"math/rand"
	"sync"
	"time"

	"github.com/pkg/errors"
)

// ProcessorOptions contains optional parameters that are used to create a Processor.
type ProcessorOptions struct {
	// The maximum number of Events in each chunk (and therefore per partition) of the stream (default: 1)
	BatchLimit uint
	// Maximum time in seconds to wait for the flushing of each chunk (per partition).(default: 30)
	FlushTimeout uint
	// The number of parallel streams the Processor will use to consume events (default: 1)
	StreamCount uint
	// Limits the number of events that the processor will handle per minute. This value represents an
	// upper bound, if some streams are not healthy e.g. if StreamCount exceeds the number of partitions,
	// or the actual batch size is lower than BatchLimit the actual number of processed events can be
	// much lower. 0 is interpreted as no limit at all (default: no limit)
	EventsPerMinute uint
	// The amount of uncommitted events Nakadi will stream before pausing the stream. When in paused
	// state and commit comes - the stream will resume. If MaxUncommittedEvents is lower than BatchLimit,
	// effective batch size will be upperbound by MaxUncommittedEvents. (default: 10, minimum: 1)
	MaxUncommittedEvents uint
	// The initial (minimal) retry interval used for the exponential backoff. This value is applied for
	// stream initialization as well as for cursor commits.
	InitialRetryInterval time.Duration
	// MaxRetryInterval the maximum retry interval. Once the exponential backoff reaches this value
	// the retry intervals remain constant. This value is applied for stream initialization as well as
	// for cursor commits.
	MaxRetryInterval time.Duration
	// CommitMaxElapsedTime is the maximum time spent on retries when committing a cursor. Once this value
	// was reached the exponential backoff is halted and the cursor will not be committed.
	CommitMaxElapsedTime time.Duration
	// NotifyErr is called when an error occurs that leads to a retry. This notify function can be used to
	// detect unhealthy streams. The first parameter indicates the stream No that encountered the error.
	NotifyErr func(uint, error, time.Duration)
	// NotifyOK is called whenever a successful operation was completed. This notify function can be used
	// to detect that a stream is healthy again. The first parameter indicates the stream No that just
	// regained health.
	NotifyOK func(uint)
}

func (o *ProcessorOptions) withDefaults() *ProcessorOptions {
	var copyOptions ProcessorOptions
	if o != nil {
		copyOptions = *o
	}
	if copyOptions.BatchLimit == 0 {
		copyOptions.BatchLimit = 1
	}
	if copyOptions.FlushTimeout == 0 {
		copyOptions.FlushTimeout = 30
	}
	if copyOptions.StreamCount == 0 {
		copyOptions.StreamCount = 1
	}
	if copyOptions.EventsPerMinute == 0 {
		copyOptions.EventsPerMinute = 0
	}
	if copyOptions.InitialRetryInterval == 0 {
		copyOptions.InitialRetryInterval = defaultInitialRetryInterval
	}
	if copyOptions.MaxRetryInterval == 0 {
		copyOptions.MaxRetryInterval = defaultMaxRetryInterval
	}
	if copyOptions.CommitMaxElapsedTime == 0 {
		copyOptions.CommitMaxElapsedTime = defaultMaxElapsedTime
	}
	if copyOptions.NotifyErr == nil {
		copyOptions.NotifyErr = func(_ uint, _ error, _ time.Duration) {}
	}
	if copyOptions.NotifyOK == nil {
		copyOptions.NotifyOK = func(_ uint) {}
	}
	if copyOptions.MaxUncommittedEvents == 0 {
		copyOptions.MaxUncommittedEvents = 10
	}
	return &copyOptions
}

// streamAPI is a contract that is used internally in order to be able to mock StreamAPI
type streamAPI interface {
	NextEvents() (Cursor, []byte, error)
	CommitCursor(cursor Cursor) error
	Close() error
}

// NewProcessor creates a new processor for a given subscription ID.  The constructor receives a
// configured Nakadi client as first parameter. Furthermore a valid subscription ID must be
// provided. The last parameter is a struct containing only optional parameters. The options may be
// nil, in this case the processor falls back to the defaults defined in the ProcessorOptions.
func NewProcessor(client *Client, subscriptionID string, options *ProcessorOptions) *Processor {
	options = options.withDefaults()

	var timePerBatchPerStream int64
	if options.EventsPerMinute > 0 {
		timePerBatchPerStream = int64(time.Minute) / int64(options.EventsPerMinute) * int64(options.StreamCount) * int64(options.BatchLimit)
	}

	ctx, cancel := context.WithCancel(context.Background())
	processor := &Processor{
		client:         client,
		subscriptionID: subscriptionID,
		ctx:            ctx,
		cancel:         cancel,
		newStream: func(client *Client, id string, options *StreamOptions) streamAPI {
			return NewStream(client, id, options)
		},
		timePerBatchPerStream: time.Duration(timePerBatchPerStream),
		closeErrorCh:          make(chan error)}

	for i := uint(0); i < options.StreamCount; i++ {
		streamOptions := StreamOptions{
			BatchLimit:           options.BatchLimit,
			FlushTimeout:         options.FlushTimeout,
			MaxUncommittedEvents: options.MaxUncommittedEvents,
			InitialRetryInterval: options.InitialRetryInterval,
			MaxRetryInterval:     options.MaxRetryInterval,
			CommitMaxElapsedTime: options.CommitMaxElapsedTime,
			NotifyErr:            func(err error, duration time.Duration) { options.NotifyErr(i, err, duration) },
			NotifyOK:             func() { options.NotifyOK(i) }}

		processor.streamOptions = append(processor.streamOptions, streamOptions)
	}

	return processor
}

// A Processor for nakadi events. The Processor is a high level API for consuming events from
// Nakadi. It can process event batches from multiple partitions (streams) and can be configured
// with a certain event rate, that limits the number of processed events per minute. The cursors of
// event batches that were successfully processed are automatically committed.
type Processor struct {
	sync.Mutex
	client                *Client
	subscriptionID        string
	streamOptions         []StreamOptions
	newStream             func(*Client, string, *StreamOptions) streamAPI
	timePerBatchPerStream time.Duration
	isStarted             bool
	ctx                   context.Context
	cancel                context.CancelFunc
	closeErrorCh          chan error
}

// Operation defines a certain procedure that consumes the event data from a processor. An operation,
// can either be successful or may fail with an error. Operation receives three parameters: the stream
// number or position in the processor, the Nakadi stream id of the underlying stream, and the json
// encoded event payload.
type Operation func(int, string, []byte) error

// Start begins event processing. All event batches received from the underlying streams are passed to
// the operation function. If the operation function terminates without error the respective cursor will
// be automatically committed to Nakadi. If the operations terminates with an error, the underlying stream
// will be halted and a new stream will continue to pass event batches to the operation function.
//
// Event processing will go on indefinitely unless the processor is stopped via its Stop method. Star will
// return an error if the processor is already running.
func (p *Processor) Start(operation Operation) error {
	p.Lock()
	defer p.Unlock()

	if p.isStarted {
		return errors.New("processor was already started")
	}
	p.isStarted = true

	for streamNo, options := range p.streamOptions {
		go p.startSingleStream(operation, streamNo, options)
	}

	return nil
}

// startSingleStream starts a single stream with a given stream number / position. After the stream has been
// started it consumes events. In cases of errors the stream is closed and a new stream will be opened.
func (p *Processor) startSingleStream(operation Operation, streamNo int, options StreamOptions) {
	stream := p.newStream(p.client, p.subscriptionID, &options)

	if p.timePerBatchPerStream > 0 {
		initialWait := rand.Int63n(int64(p.timePerBatchPerStream))
		select {
		case <-p.ctx.Done():
			p.closeErrorCh <- stream.Close()
			return
		case <-time.After(time.Duration(initialWait)):
			// nothing
		}
	}

	for {
		start := time.Now()

		select {
		case <-p.ctx.Done():
			p.closeErrorCh <- stream.Close()
			return
		default:
			cursor, events, err := stream.NextEvents()
			if err != nil {
				continue
			}

			err = operation(streamNo, cursor.NakadiStreamID, events)
			if err != nil {
				err = stream.Close()
				if err != nil {
					options.NotifyErr(err, 0)
				}
				stream = p.newStream(p.client, p.subscriptionID, &options)
				continue
			}

			stream.CommitCursor(cursor)
		}

		elapsed := time.Since(start)
		if elapsed < p.timePerBatchPerStream {
			select {
			case <-p.ctx.Done():
				p.closeErrorCh <- stream.Close()
				return
			case <-time.After(p.timePerBatchPerStream - elapsed):
				// nothing
			}
		}
	}
}

// Stop halts all steams and terminates event processing. Stop will return with an error if the processor
// is not running.
func (p *Processor) Stop() error {
	p.Lock()
	defer p.Unlock()

	if !p.isStarted {
		return errors.New("processor is not running")
	}

	p.cancel()

	var errCount int
	for i := 0; i < len(p.streamOptions); i++ {
		err := <-p.closeErrorCh
		if err != nil {
			errCount++
		}
	}
	if errCount > 0 {
		return errors.Errorf("%d streams had errors while closing the stream", errCount)
	}

	return nil
}
<commit_msg>fixed creation of notifyOK and NotifyErr functions in processor<commit_after>package nakadi

import (
	"context"
	"math/rand"
	"sync"
	"time"

	"github.com/pkg/errors"
)

// ProcessorOptions contains optional parameters that are used to create a Processor.
type ProcessorOptions struct {
	// The maximum number of Events in each chunk (and therefore per partition) of the stream (default: 1)
	BatchLimit uint
	// Maximum time in seconds to wait for the flushing of each chunk (per partition).(default: 30)
	FlushTimeout uint
	// The number of parallel streams the Processor will use to consume events (default: 1)
	StreamCount uint
	// Limits the number of events that the processor will handle per minute. This value represents an
	// upper bound, if some streams are not healthy e.g. if StreamCount exceeds the number of partitions,
	// or the actual batch size is lower than BatchLimit the actual number of processed events can be
	// much lower. 0 is interpreted as no limit at all (default: no limit)
	EventsPerMinute uint
	// The amount of uncommitted events Nakadi will stream before pausing the stream. When in paused
	// state and commit comes - the stream will resume. If MaxUncommittedEvents is lower than BatchLimit,
	// effective batch size will be upperbound by MaxUncommittedEvents. (default: 10, minimum: 1)
	MaxUncommittedEvents uint
	// The initial (minimal) retry interval used for the exponential backoff. This value is applied for
	// stream initialization as well as for cursor commits.
	InitialRetryInterval time.Duration
	// MaxRetryInterval the maximum retry interval. Once the exponential backoff reaches this value
	// the retry intervals remain constant. This value is applied for stream initialization as well as
	// for cursor commits.
	MaxRetryInterval time.Duration
	// CommitMaxElapsedTime is the maximum time spent on retries when committing a cursor. Once this value
	// was reached the exponential backoff is halted and the cursor will not be committed.
	CommitMaxElapsedTime time.Duration
	// NotifyErr is called when an error occurs that leads to a retry. This notify function can be used to
	// detect unhealthy streams. The first parameter indicates the stream No that encountered the error.
	NotifyErr func(uint, error, time.Duration)
	// NotifyOK is called whenever a successful operation was completed. This notify function can be used
	// to detect that a stream is healthy again. The first parameter indicates the stream No that just
	// regained health.
	NotifyOK func(uint)
}

func (o *ProcessorOptions) withDefaults() *ProcessorOptions {
	var copyOptions ProcessorOptions
	if o != nil {
		copyOptions = *o
	}
	if copyOptions.BatchLimit == 0 {
		copyOptions.BatchLimit = 1
	}
	if copyOptions.FlushTimeout == 0 {
		copyOptions.FlushTimeout = 30
	}
	if copyOptions.StreamCount == 0 {
		copyOptions.StreamCount = 1
	}
	if copyOptions.EventsPerMinute == 0 {
		copyOptions.EventsPerMinute = 0
	}
	if copyOptions.InitialRetryInterval == 0 {
		copyOptions.InitialRetryInterval = defaultInitialRetryInterval
	}
	if copyOptions.MaxRetryInterval == 0 {
		copyOptions.MaxRetryInterval = defaultMaxRetryInterval
	}
	if copyOptions.CommitMaxElapsedTime == 0 {
		copyOptions.CommitMaxElapsedTime = defaultMaxElapsedTime
	}
	if copyOptions.NotifyErr == nil {
		copyOptions.NotifyErr = func(_ uint, _ error, _ time.Duration) {}
	}
	if copyOptions.NotifyOK == nil {
		copyOptions.NotifyOK = func(_ uint) {}
	}
	if copyOptions.MaxUncommittedEvents == 0 {
		copyOptions.MaxUncommittedEvents = 10
	}
	return &copyOptions
}

// streamAPI is a contract that is used internally in order to be able to mock StreamAPI
type streamAPI interface {
	NextEvents() (Cursor, []byte, error)
	CommitCursor(cursor Cursor) error
	Close() error
}

// NewProcessor creates a new processor for a given subscription ID.  The constructor receives a
// configured Nakadi client as first parameter. Furthermore a valid subscription ID must be
// provided. The last parameter is a struct containing only optional parameters. The options may be
// nil, in this case the processor falls back to the defaults defined in the ProcessorOptions.
func NewProcessor(client *Client, subscriptionID string, options *ProcessorOptions) *Processor {
	options = options.withDefaults()

	var timePerBatchPerStream int64
	if options.EventsPerMinute > 0 {
		timePerBatchPerStream = int64(time.Minute) / int64(options.EventsPerMinute) * int64(options.StreamCount) * int64(options.BatchLimit)
	}

	ctx, cancel := context.WithCancel(context.Background())
	processor := &Processor{
		client:         client,
		subscriptionID: subscriptionID,
		ctx:            ctx,
		cancel:         cancel,
		newStream: func(client *Client, id string, options *StreamOptions) streamAPI {
			return NewStream(client, id, options)
		},
		timePerBatchPerStream: time.Duration(timePerBatchPerStream),
		closeErrorCh:          make(chan error)}

	for i := uint(0); i < options.StreamCount; i++ {
		streamOptions := StreamOptions{
			BatchLimit:           options.BatchLimit,
			FlushTimeout:         options.FlushTimeout,
			MaxUncommittedEvents: options.MaxUncommittedEvents,
			InitialRetryInterval: options.InitialRetryInterval,
			MaxRetryInterval:     options.MaxRetryInterval,
			CommitMaxElapsedTime: options.CommitMaxElapsedTime,
			NotifyErr: func(streamNo uint) func(err error, duration time.Duration) {
				return func(err error, duration time.Duration) { options.NotifyErr(streamNo, err, duration) }
			}(i),
			NotifyOK: func(streamNo uint) func() {
				return func() { options.NotifyOK(streamNo) }
			}(i),
		}
		processor.streamOptions = append(processor.streamOptions, streamOptions)
	}

	return processor
}

// A Processor for nakadi events. The Processor is a high level API for consuming events from
// Nakadi. It can process event batches from multiple partitions (streams) and can be configured
// with a certain event rate, that limits the number of processed events per minute. The cursors of
// event batches that were successfully processed are automatically committed.
type Processor struct {
	sync.Mutex
	client                *Client
	subscriptionID        string
	streamOptions         []StreamOptions
	newStream             func(*Client, string, *StreamOptions) streamAPI
	timePerBatchPerStream time.Duration
	isStarted             bool
	ctx                   context.Context
	cancel                context.CancelFunc
	closeErrorCh          chan error
}

// Operation defines a certain procedure that consumes the event data from a processor. An operation,
// can either be successful or may fail with an error. Operation receives three parameters: the stream
// number or position in the processor, the Nakadi stream id of the underlying stream, and the json
// encoded event payload.
type Operation func(int, string, []byte) error

// Start begins event processing. All event batches received from the underlying streams are passed to
// the operation function. If the operation function terminates without error the respective cursor will
// be automatically committed to Nakadi. If the operations terminates with an error, the underlying stream
// will be halted and a new stream will continue to pass event batches to the operation function.
//
// Event processing will go on indefinitely unless the processor is stopped via its Stop method. Star will
// return an error if the processor is already running.
func (p *Processor) Start(operation Operation) error {
	p.Lock()
	defer p.Unlock()

	if p.isStarted {
		return errors.New("processor was already started")
	}
	p.isStarted = true

	for streamNo, options := range p.streamOptions {
		go p.startSingleStream(operation, streamNo, options)
	}

	return nil
}

// startSingleStream starts a single stream with a given stream number / position. After the stream has been
// started it consumes events. In cases of errors the stream is closed and a new stream will be opened.
func (p *Processor) startSingleStream(operation Operation, streamNo int, options StreamOptions) {
	stream := p.newStream(p.client, p.subscriptionID, &options)

	if p.timePerBatchPerStream > 0 {
		initialWait := rand.Int63n(int64(p.timePerBatchPerStream))
		select {
		case <-p.ctx.Done():
			p.closeErrorCh <- stream.Close()
			return
		case <-time.After(time.Duration(initialWait)):
			// nothing
		}
	}

	for {
		start := time.Now()

		select {
		case <-p.ctx.Done():
			p.closeErrorCh <- stream.Close()
			return
		default:
			cursor, events, err := stream.NextEvents()
			if err != nil {
				continue
			}

			err = operation(streamNo, cursor.NakadiStreamID, events)
			if err != nil {
				err = stream.Close()
				if err != nil {
					options.NotifyErr(err, 0)
				}
				stream = p.newStream(p.client, p.subscriptionID, &options)
				continue
			}

			stream.CommitCursor(cursor)
		}

		elapsed := time.Since(start)
		if elapsed < p.timePerBatchPerStream {
			select {
			case <-p.ctx.Done():
				p.closeErrorCh <- stream.Close()
				return
			case <-time.After(p.timePerBatchPerStream - elapsed):
				// nothing
			}
		}
	}
}

// Stop halts all steams and terminates event processing. Stop will return with an error if the processor
// is not running.
func (p *Processor) Stop() error {
	p.Lock()
	defer p.Unlock()

	if !p.isStarted {
		return errors.New("processor is not running")
	}

	p.cancel()

	var errCount int
	for i := 0; i < len(p.streamOptions); i++ {
		err := <-p.closeErrorCh
		if err != nil {
			errCount++
		}
	}
	if errCount > 0 {
		return errors.Errorf("%d streams had errors while closing the stream", errCount)
	}

	return nil
}
<|endoftext|>


<commit_before>package thuder

import (
	"bytes"
	"fmt"
	"io"
	"os"
	"path/filepath"
	"time"
)

//PullAndPush does push and pulls based on given configurations, it uses Processors
//
//TODO: push filtering
func PullAndPush(hc *HostConfig, mc *MediaConfig) error {

	err := PrepareFilters(hc.Filters)
	if err != nil {
		return fmt.Errorf("HostConfig PrepareFilters error: %v", err)
	}
	err = PrepareFilters(mc.Filters)
	if err != nil {
		return fmt.Errorf("MediaConfig PrepareFilters error: %v", err)
	}
	isPush := false
	now := time.Now()
	accept := func(n *Node) bool {
		_, a := MatchFilters(hc.Filters, n, isPush, now)
		if !a {
			return false
		}
		_, a = MatchFilters(mc.Filters, n, isPush, now)
		return a
	}

	actions := make(chan action, 8)
	apply := func(p *Processor) {
		go p.Do()
		for {
			a := <-actions
			if len(a.from) == 0 {
				return
			}
			LogP("Appling %v actions to %v.\n", len(a.from), a.to)
			errs := applyAction(a)
			if len(errs) != 0 {
				p.logErrors(a.to, errs)
			}
		}
	}

	p, err := NewPullingProcessor(mc.Pulls, hc.PullTarget(), actions, accept)
	if err != nil {
		return err
	}
	apply(p)

	isPush = true
	p, err = NewPushingProcessor(hc, actions, accept)
	if err != nil {
		return err
	}
	apply(p)

	syncWriteCache()

	return nil
}

//Processor does the recursive, depth first, processing of directories
type Processor struct {
	stack   []layer
	actions chan<- action // a buffered channal of queued actions to take
	accept  func(n *Node) bool
}

//joinSub is filePath.Join with additional special charcter handling
func joinSub(parent, sub string) string {
	if filepath.Separator == '\\' && len(sub) > 1 && sub[1] == ':' {
		if len(sub) > 2 {
			sub = sub[0:1] + sub[2:]
		} else {
			sub = sub[0:1]
		}
	}
	return filepath.Join(parent, sub)
}

//NewPullingProcessor create a new Processor for pulling dirs from host to media.
func NewPullingProcessor(dirs []string, pullTo string, actions chan<- action, accept func(n *Node) bool) (*Processor, error) {
	var stack []layer
	for _, fullname := range dirs {
		rootNode, err := NewRootNode(fullname, false)
		if err != nil {
			return nil, err
		}
		to := joinSub(pullTo, fullname)
		LogP("Pulls dir %v to %v.\n", fullname, to)
		err = fs.MkdirAll(to, 0755)
		if err != nil {
			return nil, err
		}
		stack = append(stack, layer{from: []Node{*rootNode}, to: to})
	}
	p := Processor{
		stack:   stack,
		actions: actions,
		accept:  accept,
	}
	return &p, nil
}

//NewPushingProcessor create a new Processor for pushing dirs from media to host.
func NewPushingProcessor(hc *HostConfig, actions chan<- action, accept func(n *Node) bool) (*Processor, error) {
	var stack []layer
	sources, isDeletes := hc.PushSources()
	for _, root := range hc.PushRoots() {
		var nodes []Node
		for i := 0; i < len(sources); i++ {
			from := joinSub(sources[i], root)
			node, err := NewRootNode(from, isDeletes[i])
			if err != nil {
				//LogP("Pushes skipped from %v because error %v.\n", from, err)
				continue
			} else {
				LogP("pushes from %v to %v.\n", from, root)
			}
			nodes = append(nodes, *node)
		}
		stack = append(stack, layer{from: nodes, to: root})
	}
	p := Processor{
		stack:   stack,
		actions: actions,
		accept:  accept,
	}
	return &p, nil
}

//NewProcessor create a new Processor
func newProcessor(dirs []string, to string, actions chan<- action, accept func(n *Node) bool) (*Processor, error) {
	var sources []Node
	for _, fullname := range dirs {
		rootNode, err := NewRootNode(fullname, false)
		if err != nil {
			return nil, err
		}
		sources = append(sources, *rootNode)
	}
	p := Processor{
		stack: []layer{
			layer{from: sources, to: to},
		},
		actions: actions,
		accept:  accept,
	}
	return &p, nil
}

//String returns string debugging
func (p *Processor) String() string {
	b := bytes.NewBufferString("{stack:[")
	for _, l := range p.stack {
		b.WriteString("\n\t" + l.String())
	}
	b.WriteString("]")
	return b.String()
}

//layer is a layer in a Processor's stack
type layer struct {
	from []Node
	to   string
}

//String returns string for debugging
func (l layer) String() string {
	b := bytes.NewBufferString("{from:[")
	for _, n := range l.from {
		b.WriteString(n.String() + " ")
	}
	b.WriteString("] to:" + l.to + "}")
	return b.String()
}

///action is an action to be done to the file system
type action struct {
	from []Node
	to   string
}

func applyAction(a action) []error {
	var errs []error
	for i := len(a.from) - 1; i >= 0; i-- {
		n := a.from[i]
		err := applyNode(n, a.to)
		if err != nil {
			errs = append(errs, err)
		}
	}
	return errs
}

func applyNode(n Node, to string) error {
	target := filepath.Join(to, n.info.Name())
	if n.IsDelete() {
		//fmt.Println("remove", n.info.Name())
		return fs.RemoveAll(target)
	}
	if n.IsDir() {
		//fmt.Println("mkdir", n.info.Name())
		return fs.Mkdir(target, n.FileMode())
	}
	//fmt.Println("copy", n.info.Name())
	return atomicCopy(n, to)
}

//Do make the Processor process the stack until done
func (p *Processor) Do() {
	for p.doOnce() {
	}
	p.actions <- action{} //empty action means done
}

// returns true until there is nothing left to do
func (p *Processor) doOnce() bool {
	//fmt.Println(p)
	top := len(p.stack) - 1
	if top < 0 {
		return false
	}
	FlashLED()
	var l layer
	p.stack, l = p.stack[:top], p.stack[top] //pop from stack

	c := NewCollection(p.accept)
	for _, node := range l.from {
		err := c.Add(&node)
		if err != nil {
			p.logError(node.FullName(), err)
		}
	}
	deletes, changedfiles, dirs, err := c.GetAppliedTo(l.to)
	if err != nil {
		p.logError(l.to, err)
		// continue even on error
	}
	a := action{to: l.to}
	if len(deletes) > 0 {
		a.from = deletes
		p.actions <- a
	}
	if len(changedfiles) > 0 {
		a.from = changedfiles
		p.actions <- a
	}
	if len(dirs) > 0 {
		for _, d := range dirs {
			last := d[len(d)-1]

			newLayer := layer{
				from: d,
				to:   filepath.Join(l.to, last.info.Name()),
			}
			p.stack = append(p.stack, newLayer)
		}
	}
	return true
}

//LogErrorOut used to redirect error logs
var LogErrorOut io.Writer = os.Stderr

func (p *Processor) logError(dir string, err error) {
	fmt.Fprintln(LogErrorOut, "Processor Error: ", dir, err)
}
func (p *Processor) logErrors(dir string, errs []error) {
	fmt.Fprintln(LogErrorOut, "Processor Errors: ", dir, errs)
}

//LogVerbosOut used to redirect verbos logs
var LogVerbosOut io.Writer = os.Stdout

//LogP is the handler for logging live progress, in the form of fmt.Printf
var LogP = func(format string, a ...interface{}) {
	fmt.Fprintf(LogVerbosOut, format, a...)
}
<commit_msg>add flash at end<commit_after>package thuder

import (
	"bytes"
	"fmt"
	"io"
	"os"
	"path/filepath"
	"time"
)

//PullAndPush does push and pulls based on given configurations, it uses Processors
//
//TODO: push filtering
func PullAndPush(hc *HostConfig, mc *MediaConfig) error {

	err := PrepareFilters(hc.Filters)
	if err != nil {
		return fmt.Errorf("HostConfig PrepareFilters error: %v", err)
	}
	err = PrepareFilters(mc.Filters)
	if err != nil {
		return fmt.Errorf("MediaConfig PrepareFilters error: %v", err)
	}
	isPush := false
	now := time.Now()
	accept := func(n *Node) bool {
		_, a := MatchFilters(hc.Filters, n, isPush, now)
		if !a {
			return false
		}
		_, a = MatchFilters(mc.Filters, n, isPush, now)
		return a
	}

	actions := make(chan action, 8)
	apply := func(p *Processor) {
		go p.Do()
		for {
			a := <-actions
			if len(a.from) == 0 {
				return
			}
			LogP("Appling %v actions to %v.\n", len(a.from), a.to)
			errs := applyAction(a)
			if len(errs) != 0 {
				p.logErrors(a.to, errs)
			}
		}
	}

	p, err := NewPullingProcessor(mc.Pulls, hc.PullTarget(), actions, accept)
	if err != nil {
		return err
	}
	apply(p)

	isPush = true
	p, err = NewPushingProcessor(hc, actions, accept)
	if err != nil {
		return err
	}
	apply(p)

	FlashLED()

	syncWriteCache()

	return nil
}

//Processor does the recursive, depth first, processing of directories
type Processor struct {
	stack   []layer
	actions chan<- action // a buffered channal of queued actions to take
	accept  func(n *Node) bool
}

//joinSub is filePath.Join with additional special charcter handling
func joinSub(parent, sub string) string {
	if filepath.Separator == '\\' && len(sub) > 1 && sub[1] == ':' {
		if len(sub) > 2 {
			sub = sub[0:1] + sub[2:]
		} else {
			sub = sub[0:1]
		}
	}
	return filepath.Join(parent, sub)
}

//NewPullingProcessor create a new Processor for pulling dirs from host to media.
func NewPullingProcessor(dirs []string, pullTo string, actions chan<- action, accept func(n *Node) bool) (*Processor, error) {
	var stack []layer
	for _, fullname := range dirs {
		rootNode, err := NewRootNode(fullname, false)
		if err != nil {
			return nil, err
		}
		to := joinSub(pullTo, fullname)
		LogP("Pulls dir %v to %v.\n", fullname, to)
		err = fs.MkdirAll(to, 0755)
		if err != nil {
			return nil, err
		}
		stack = append(stack, layer{from: []Node{*rootNode}, to: to})
	}
	p := Processor{
		stack:   stack,
		actions: actions,
		accept:  accept,
	}
	return &p, nil
}

//NewPushingProcessor create a new Processor for pushing dirs from media to host.
func NewPushingProcessor(hc *HostConfig, actions chan<- action, accept func(n *Node) bool) (*Processor, error) {
	var stack []layer
	sources, isDeletes := hc.PushSources()
	for _, root := range hc.PushRoots() {
		var nodes []Node
		for i := 0; i < len(sources); i++ {
			from := joinSub(sources[i], root)
			node, err := NewRootNode(from, isDeletes[i])
			if err != nil {
				//LogP("Pushes skipped from %v because error %v.\n", from, err)
				continue
			} else {
				LogP("pushes from %v to %v.\n", from, root)
			}
			nodes = append(nodes, *node)
		}
		stack = append(stack, layer{from: nodes, to: root})
	}
	p := Processor{
		stack:   stack,
		actions: actions,
		accept:  accept,
	}
	return &p, nil
}

//NewProcessor create a new Processor
func newProcessor(dirs []string, to string, actions chan<- action, accept func(n *Node) bool) (*Processor, error) {
	var sources []Node
	for _, fullname := range dirs {
		rootNode, err := NewRootNode(fullname, false)
		if err != nil {
			return nil, err
		}
		sources = append(sources, *rootNode)
	}
	p := Processor{
		stack: []layer{
			layer{from: sources, to: to},
		},
		actions: actions,
		accept:  accept,
	}
	return &p, nil
}

//String returns string debugging
func (p *Processor) String() string {
	b := bytes.NewBufferString("{stack:[")
	for _, l := range p.stack {
		b.WriteString("\n\t" + l.String())
	}
	b.WriteString("]")
	return b.String()
}

//layer is a layer in a Processor's stack
type layer struct {
	from []Node
	to   string
}

//String returns string for debugging
func (l layer) String() string {
	b := bytes.NewBufferString("{from:[")
	for _, n := range l.from {
		b.WriteString(n.String() + " ")
	}
	b.WriteString("] to:" + l.to + "}")
	return b.String()
}

///action is an action to be done to the file system
type action struct {
	from []Node
	to   string
}

func applyAction(a action) []error {
	var errs []error
	for i := len(a.from) - 1; i >= 0; i-- {
		n := a.from[i]
		err := applyNode(n, a.to)
		if err != nil {
			errs = append(errs, err)
		}
	}
	return errs
}

func applyNode(n Node, to string) error {
	target := filepath.Join(to, n.info.Name())
	if n.IsDelete() {
		//fmt.Println("remove", n.info.Name())
		return fs.RemoveAll(target)
	}
	if n.IsDir() {
		//fmt.Println("mkdir", n.info.Name())
		return fs.Mkdir(target, n.FileMode())
	}
	//fmt.Println("copy", n.info.Name())
	return atomicCopy(n, to)
}

//Do make the Processor process the stack until done
func (p *Processor) Do() {
	for p.doOnce() {
	}
	p.actions <- action{} //empty action means done
}

// returns true until there is nothing left to do
func (p *Processor) doOnce() bool {
	//fmt.Println(p)
	top := len(p.stack) - 1
	if top < 0 {
		return false
	}
	FlashLED()
	var l layer
	p.stack, l = p.stack[:top], p.stack[top] //pop from stack

	c := NewCollection(p.accept)
	for _, node := range l.from {
		err := c.Add(&node)
		if err != nil {
			p.logError(node.FullName(), err)
		}
	}
	deletes, changedfiles, dirs, err := c.GetAppliedTo(l.to)
	if err != nil {
		p.logError(l.to, err)
		// continue even on error
	}
	a := action{to: l.to}
	if len(deletes) > 0 {
		a.from = deletes
		p.actions <- a
	}
	if len(changedfiles) > 0 {
		a.from = changedfiles
		p.actions <- a
	}
	if len(dirs) > 0 {
		for _, d := range dirs {
			last := d[len(d)-1]

			newLayer := layer{
				from: d,
				to:   filepath.Join(l.to, last.info.Name()),
			}
			p.stack = append(p.stack, newLayer)
		}
	}
	return true
}

//LogErrorOut used to redirect error logs
var LogErrorOut io.Writer = os.Stderr

func (p *Processor) logError(dir string, err error) {
	fmt.Fprintln(LogErrorOut, "Processor Error: ", dir, err)
}
func (p *Processor) logErrors(dir string, errs []error) {
	fmt.Fprintln(LogErrorOut, "Processor Errors: ", dir, errs)
}

//LogVerbosOut used to redirect verbos logs
var LogVerbosOut io.Writer = os.Stdout

//LogP is the handler for logging live progress, in the form of fmt.Printf
var LogP = func(format string, a ...interface{}) {
	fmt.Fprintf(LogVerbosOut, format, a...)
}
<|endoftext|>


<commit_before>package mpb

import "context"

func WithContext(ctx context.Context) ProgressOption {
	return func(c *pConf) {
		if ctx != nil {
			c.cancel = ctx.Done()
		}
	}
}
<commit_msg>+build go1.7<commit_after>//+build go1.7

package mpb

import "context"

func WithContext(ctx context.Context) ProgressOption {
	return func(c *pConf) {
		if ctx != nil {
			c.cancel = ctx.Done()
		}
	}
}
<|endoftext|>


<commit_before>// Copyright (C) 2014 Constantin Schomburg <me@cschomburg.com>
//
// Use of this source code is governed by an MIT-style
// license that can be found in the LICENSE file.

package events

import (
	"database/sql"
	"time"

	"github.com/xconstruct/stark/core"
	"github.com/xconstruct/stark/proto"
)

var Module = core.Module{
	Name:        "events",
	Version:     "1.0",
	NewInstance: newInstance,
}

func init() {
	core.RegisterModule(Module)
}

func newInstance(ctx *core.Context) (core.ModuleInstance, error) {
	return NewService(ctx)
}

type Service struct {
	DB    Database
	ctx   *core.Context
	proto *proto.Client
}

func NewService(ctx *core.Context) (*Service, error) {
	s := &Service{
		DB:    &sqlDatabase{ctx.Database.Driver(), ctx.Database.DB},
		ctx:   ctx,
		proto: proto.NewClient("events", ctx.Proto),
	}
	return s, nil
}

func (s *Service) Enable() error {
	if err := s.DB.Setup(); err != nil {
		return err
	}
	if err := s.proto.Subscribe("event/new", "", s.handleEventNew); err != nil {
		return err
	}
	if err := s.proto.Subscribe("event/last", "", s.handleEventLast); err != nil {
		return err
	}
	if err := s.proto.Subscribe("location/fence", "", s.handleLocationFence); err != nil {
		return err
	}
	if err := s.proto.Subscribe("location/fence", "", s.handleLocationFence); err != nil {
		return err
	}
	return nil
}

func (s *Service) Disable() error { return nil }

var MessageEventNotFound = proto.Message{
	Action: "event/notfound",
	Payload: map[string]interface{}{
		"text": "No event found.",
	},
}

func fixEvent(e *Event) {
	if e.Timestamp.IsZero() {
		e.Timestamp = time.Now()
	}
	if e.Subject == "i" || e.Subject == "I" {
		e.Subject = "user"
	}
}

func (s *Service) handleEventNew(msg proto.Message) {
	var e Event
	if err := msg.DecodePayload(&e); err != nil {
		s.ctx.Log.Warnln("[events] received bad payload:", err)
		s.publish(msg.Reply(proto.BadRequest(err)))
		return
	}
	fixEvent(&e)

	s.ctx.Log.Infoln("[events] new event:", e)

	if err := s.DB.StoreEvent(e); err != nil {
		s.ctx.Log.Errorln("[vents] could not store event:", err)
		s.publish(msg.Reply(proto.InternalError(err)))
		return
	}

	reply := proto.Message{Action: "event/created"}
	if err := reply.EncodePayload(e); err != nil {
		s.ctx.Log.Errorln("[events] could not encode reply:", err)
		s.publish(msg.Reply(proto.InternalError(err)))
		return
	}
	reply.Payload["text"] = "New event: " + e.String()

	s.publish(msg.Reply(reply))
}

func (s *Service) handleEventLast(msg proto.Message) {
	var filter Event
	if err := msg.DecodePayload(&filter); err != nil {
		s.ctx.Log.Warnln("[events] received bad payload:", err)
		s.publish(msg.Reply(proto.BadRequest(err)))
		return
	}
	fixEvent(&filter)

	s.ctx.Log.Infoln("[events] get last by filter:", filter)
	last, err := s.DB.GetLastEvent(filter)
	if err != nil {
		if err == sql.ErrNoRows {
			s.publish(msg.Reply(MessageEventNotFound))
			return
		}
		s.ctx.Log.Errorln("[events] could not get events:", err)
		s.publish(msg.Reply(proto.InternalError(err)))
		return
	}
	s.ctx.Log.Infoln("[events] found", last)

	reply := proto.Message{Action: "event/found"}
	if err := reply.EncodePayload(last); err != nil {
		s.ctx.Log.Errorln("[events] could not encode reply:", err)
		s.publish(msg.Reply(proto.InternalError(err)))
		return
	}
	reply.Payload["text"] = last.String()

	s.publish(msg.Reply(reply))
}

func (s *Service) publish(msg proto.Message) {
	if err := s.proto.Publish(msg); err != nil {
		s.ctx.Log.Errorln(err)
	}
}
<commit_msg>Events: Do not subscribe twice to location/fence.<commit_after>// Copyright (C) 2014 Constantin Schomburg <me@cschomburg.com>
//
// Use of this source code is governed by an MIT-style
// license that can be found in the LICENSE file.

package events

import (
	"database/sql"
	"time"

	"github.com/xconstruct/stark/core"
	"github.com/xconstruct/stark/proto"
)

var Module = core.Module{
	Name:        "events",
	Version:     "1.0",
	NewInstance: newInstance,
}

func init() {
	core.RegisterModule(Module)
}

func newInstance(ctx *core.Context) (core.ModuleInstance, error) {
	return NewService(ctx)
}

type Service struct {
	DB    Database
	ctx   *core.Context
	proto *proto.Client
}

func NewService(ctx *core.Context) (*Service, error) {
	s := &Service{
		DB:    &sqlDatabase{ctx.Database.Driver(), ctx.Database.DB},
		ctx:   ctx,
		proto: proto.NewClient("events", ctx.Proto),
	}
	return s, nil
}

func (s *Service) Enable() error {
	if err := s.DB.Setup(); err != nil {
		return err
	}
	if err := s.proto.Subscribe("event/new", "", s.handleEventNew); err != nil {
		return err
	}
	if err := s.proto.Subscribe("event/last", "", s.handleEventLast); err != nil {
		return err
	}
	if err := s.proto.Subscribe("location/fence", "", s.handleLocationFence); err != nil {
		return err
	}
	return nil
}

func (s *Service) Disable() error { return nil }

var MessageEventNotFound = proto.Message{
	Action: "event/notfound",
	Payload: map[string]interface{}{
		"text": "No event found.",
	},
}

func fixEvent(e *Event) {
	if e.Timestamp.IsZero() {
		e.Timestamp = time.Now()
	}
	if e.Subject == "i" || e.Subject == "I" {
		e.Subject = "user"
	}
}

func (s *Service) handleEventNew(msg proto.Message) {
	var e Event
	if err := msg.DecodePayload(&e); err != nil {
		s.ctx.Log.Warnln("[events] received bad payload:", err)
		s.publish(msg.Reply(proto.BadRequest(err)))
		return
	}
	fixEvent(&e)

	s.ctx.Log.Infoln("[events] new event:", e)

	if err := s.DB.StoreEvent(e); err != nil {
		s.ctx.Log.Errorln("[vents] could not store event:", err)
		s.publish(msg.Reply(proto.InternalError(err)))
		return
	}

	reply := proto.Message{Action: "event/created"}
	if err := reply.EncodePayload(e); err != nil {
		s.ctx.Log.Errorln("[events] could not encode reply:", err)
		s.publish(msg.Reply(proto.InternalError(err)))
		return
	}
	reply.Payload["text"] = "New event: " + e.String()

	s.publish(msg.Reply(reply))
}

func (s *Service) handleEventLast(msg proto.Message) {
	var filter Event
	if err := msg.DecodePayload(&filter); err != nil {
		s.ctx.Log.Warnln("[events] received bad payload:", err)
		s.publish(msg.Reply(proto.BadRequest(err)))
		return
	}
	fixEvent(&filter)

	s.ctx.Log.Infoln("[events] get last by filter:", filter)
	last, err := s.DB.GetLastEvent(filter)
	if err != nil {
		if err == sql.ErrNoRows {
			s.publish(msg.Reply(MessageEventNotFound))
			return
		}
		s.ctx.Log.Errorln("[events] could not get events:", err)
		s.publish(msg.Reply(proto.InternalError(err)))
		return
	}
	s.ctx.Log.Infoln("[events] found", last)

	reply := proto.Message{Action: "event/found"}
	if err := reply.EncodePayload(last); err != nil {
		s.ctx.Log.Errorln("[events] could not encode reply:", err)
		s.publish(msg.Reply(proto.InternalError(err)))
		return
	}
	reply.Payload["text"] = last.String()

	s.publish(msg.Reply(reply))
}

func (s *Service) publish(msg proto.Message) {
	if err := s.proto.Publish(msg); err != nil {
		s.ctx.Log.Errorln(err)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"io"
	"log"
	"net"
	"os"
)

/**
 * Launch two read-write goroutines and waits for signal from them
 */
func TransferStreams(con net.Conn) {
	c1 := readAndWrite(con, os.Stdout)
	c2 := readAndWrite(os.Stdin, con)
	select {
	case <-c1:
		log.Println("Remote connection is closed")
	case <-c2:
		log.Println("Local program is terminated")
	}
}

/**
 * Launch receive goroutine first, wait for address from it (if needed), launch send goroutine then.
 */
func TransferPackets(con net.Conn) {
	c1 := readAndWrite(con, os.Stdout)
	// If connection hasn't got remote address then wait for it from receiver goroutine
	ra := con.RemoteAddr()
	if ra == nil {
		ra = <-c1
		log.Println("Connect from", ra)
	}
	c2 := readAndWriteToAddr(os.Stdin, con, ra)
	select {
	case <-c1:
		log.Println("Remote connection is closed")
	case <-c2:
		log.Println("Local program is terminated")
	}
}

/**
 * Read from Reader and write to Writer until EOF.
 */
func readAndWrite(r io.Reader, w io.Writer) <-chan net.Addr {
	return readAndWriteToAddr(r, w, nil)
}

/**
 * Read from Reader and write to Writer until EOF.
 * ra is an address to whom packets must be sent in UDP listen mode.
 */
func readAndWriteToAddr(r io.Reader, w io.Writer, ra net.Addr) <-chan net.Addr {
	buf := make([]byte, 1024)
	c := make(chan net.Addr)
	go func() {
		defer func() {
			if con, ok := w.(net.Conn); ok {
				con.Close()
				if _, ok := con.(*net.UDPConn); ok {
					log.Printf("Stop receiving flow from %v\n", ra)
				} else {
					log.Printf("Connection from %v is closed\n", con.RemoteAddr())
				}
			}
			c <- ra
		}()

		for {
			var n int
			var err error

			// Read
			if con, ok := r.(*net.UDPConn); ok {
				var addr net.Addr
				n, addr, err = con.ReadFrom(buf)
				// Inform caller function with remote address once
				// (for UDP in listen mode only)
				if con.RemoteAddr() == nil && ra == nil {
					ra = addr
					c <- ra
				}
			} else {
				n, err = r.Read(buf)
			}
			if err != nil {
				if err != io.EOF {
					log.Printf("Read error: %s\n", err)
				}
				break
			}

			// Write
			if con, ok := w.(*net.UDPConn); ok && con.RemoteAddr() == nil {
				// Special case for UDP in listen mode otherwise
				// net.ErrWriteToConnected will be thrown
				_, err = con.WriteTo(buf[0:n], ra)
			} else {
				_, err = w.Write(buf[0:n])
			}
			if err != nil {
				log.Fatalf("Write error: %s\n", err)
			}
		}
	}()
	return c
}

func main() {
	var host, port, proto string
	var listen bool
	flag.StringVar(&host, "host", "", "Remote host to connect, i.e. 127.0.0.1")
	flag.StringVar(&proto, "proto", "tcp", "TCP/UDP mode")
	flag.BoolVar(&listen, "listen", false, "Listen mode")
	flag.StringVar(&port, "port", "", "Port to listen on or connect to (prepended by colon), i.e. :9999")
	flag.Parse()

	if proto == "tcp" {
		if listen {
			ln, err := net.Listen(proto, port)
			if err != nil {
				log.Fatalln(err)
			}
			log.Println("Listening on", port)
			con, err := ln.Accept()
			if err != nil {
				log.Fatalln(err)
			}
			log.Println("Connect from", con.RemoteAddr())
			TransferStreams(con)
		} else if host != "" {
			con, err := net.Dial(proto, host+port)
			if err != nil {
				log.Fatalln(err)
			}
			log.Println("Connected to", host+port)
			TransferStreams(con)
		} else {
			flag.Usage()
		}
	} else if proto == "udp" {
		if listen {
			addr, err := net.ResolveUDPAddr(proto, port)
			if err != nil {
				log.Fatalln(err)
			}
			con, err := net.ListenUDP(proto, addr)
			if err != nil {
				log.Fatalln(err)
			}
			log.Println("Listening on", port)
			TransferPackets(con)
		} else if host != "" {
			addr, err := net.ResolveUDPAddr(proto, host+port)
			if err != nil {
				log.Fatalln(err)
			}
			con, err := net.DialUDP(proto, nil, addr)
			if err != nil {
				log.Fatalln(err)
			}
			TransferPackets(con)
		} else {
			flag.Usage()
		}
	}
}
<commit_msg>Increase buffer size<commit_after>package main

import (
	"flag"
	"io"
	"log"
	"net"
	"os"
)

// Ready to handle full-size UDP datagram or TCP segment in one step
const (
	BUFFER_LIMIT = 2<<16 - 1
)

/**
 * Launch two read-write goroutines and waits for signal from them
 */
func TransferStreams(con net.Conn) {
	c1 := readAndWrite(con, os.Stdout)
	c2 := readAndWrite(os.Stdin, con)
	select {
	case <-c1:
		log.Println("Remote connection is closed")
	case <-c2:
		log.Println("Local program is terminated")
	}
}

/**
 * Launch receive goroutine first, wait for address from it (if needed), launch send goroutine then.
 */
func TransferPackets(con net.Conn) {
	c1 := readAndWrite(con, os.Stdout)
	// If connection hasn't got remote address then wait for it from receiver goroutine
	ra := con.RemoteAddr()
	if ra == nil {
		ra = <-c1
		log.Println("Connect from", ra)
	}
	c2 := readAndWriteToAddr(os.Stdin, con, ra)
	select {
	case <-c1:
		log.Println("Remote connection is closed")
	case <-c2:
		log.Println("Local program is terminated")
	}
}

/**
 * Read from Reader and write to Writer until EOF.
 */
func readAndWrite(r io.Reader, w io.Writer) <-chan net.Addr {
	return readAndWriteToAddr(r, w, nil)
}

/**
 * Read from Reader and write to Writer until EOF.
 * ra is an address to whom packets must be sent in UDP listen mode.
 */
func readAndWriteToAddr(r io.Reader, w io.Writer, ra net.Addr) <-chan net.Addr {
	buf := make([]byte, BUFFER_LIMIT)
	c := make(chan net.Addr)
	go func() {
		defer func() {
			if con, ok := w.(net.Conn); ok {
				con.Close()
				if _, ok := con.(*net.UDPConn); ok {
					log.Printf("Stop receiving flow from %v\n", ra)
				} else {
					log.Printf("Connection from %v is closed\n", con.RemoteAddr())
				}
			}
			c <- ra
		}()

		for {
			var n int
			var err error

			// Read
			if con, ok := r.(*net.UDPConn); ok {
				var addr net.Addr
				n, addr, err = con.ReadFrom(buf)
				// Inform caller function with remote address once
				// (for UDP in listen mode only)
				if con.RemoteAddr() == nil && ra == nil {
					ra = addr
					c <- ra
				}
			} else {
				n, err = r.Read(buf)
			}
			if err != nil {
				if err != io.EOF {
					log.Printf("Read error: %s\n", err)
				}
				break
			}

			// Write
			if con, ok := w.(*net.UDPConn); ok && con.RemoteAddr() == nil {
				// Special case for UDP in listen mode otherwise
				// net.ErrWriteToConnected will be thrown
				_, err = con.WriteTo(buf[0:n], ra)
			} else {
				_, err = w.Write(buf[0:n])
			}
			if err != nil {
				log.Fatalf("Write error: %s\n", err)
			}
		}
	}()
	return c
}

func main() {
	var host, port, proto string
	var listen bool
	flag.StringVar(&host, "host", "", "Remote host to connect, i.e. 127.0.0.1")
	flag.StringVar(&proto, "proto", "tcp", "TCP/UDP mode")
	flag.BoolVar(&listen, "listen", false, "Listen mode")
	flag.StringVar(&port, "port", "", "Port to listen on or connect to (prepended by colon), i.e. :9999")
	flag.Parse()

	if proto == "tcp" {
		if listen {
			ln, err := net.Listen(proto, port)
			if err != nil {
				log.Fatalln(err)
			}
			log.Println("Listening on", port)
			con, err := ln.Accept()
			if err != nil {
				log.Fatalln(err)
			}
			log.Println("Connect from", con.RemoteAddr())
			TransferStreams(con)
		} else if host != "" {
			con, err := net.Dial(proto, host+port)
			if err != nil {
				log.Fatalln(err)
			}
			log.Println("Connected to", host+port)
			TransferStreams(con)
		} else {
			flag.Usage()
		}
	} else if proto == "udp" {
		if listen {
			addr, err := net.ResolveUDPAddr(proto, port)
			if err != nil {
				log.Fatalln(err)
			}
			con, err := net.ListenUDP(proto, addr)
			if err != nil {
				log.Fatalln(err)
			}
			log.Println("Listening on", port)
			TransferPackets(con)
		} else if host != "" {
			addr, err := net.ResolveUDPAddr(proto, host+port)
			if err != nil {
				log.Fatalln(err)
			}
			con, err := net.DialUDP(proto, nil, addr)
			if err != nil {
				log.Fatalln(err)
			}
			TransferPackets(con)
		} else {
			flag.Usage()
		}
	}
}
<|endoftext|>