<commit_before>package qb

import (
	"strings"
	//	"fmt"
	"errors"
	"fmt"
)

// Tag is the base abstraction of qbit tag
type Tag struct {

	// contains default, null, notnull, unique, primary_key, foreign_key(table.column), check(condition > 0)
	Constraints []string

	// contains type(size) or type parameters
	Type string
}

// ParseTag parses raw qbit tag and builds a Tag object
func ParseTag(rawTag string) (*Tag, error) {

	tag := &Tag{
		Constraints: []string{},
	}

	if strings.Trim(rawTag, " ") == "" {
		return tag, nil
	}

	tags := strings.Split(rawTag, ";")
	for _, t := range tags {
		tagKey := strings.Split(t, ":")
		if len(tagKey) == 2 {
			if tagKey[0] == "type" {
				tag.Type = tagKey[1]
			} else if tagKey[0] == "constraints" || tagKey[0] == "constraint" {
				tag.Constraints = strings.Split(tagKey[1], ",")
			} else {
				return nil, errors.New("Invalid keyword in struct tag")
			}
		} else {
			return nil, fmt.Errorf("Invalid tag key length, tag: %v", tag)
		}
	}

	return tag, nil
}
<commit_msg>add simplicity, stabilization improvements to ParseTag, remove unknown tag key error<commit_after>package qb

import (
	"strings"
	//	"fmt"
	"errors"
	"fmt"
)

// Tag is the base abstraction of qbit tag
type Tag struct {

	// contains default, null, notnull, unique, primary_key, foreign_key(table.column), check(condition > 0)
	Constraints []string

	// contains type(size) or type parameters
	Type string
}

// ParseTag parses raw qbit tag and builds a Tag object
func ParseTag(rawTag string) (*Tag, error) {

	rawTag = strings.Trim(rawTag, " ")

	tag := &Tag{
		Constraints: []string{},
	}

	if rawTag == "" {
		return tag, nil
	}

	tags := strings.Split(rawTag, ";")
	for _, t := range tags {
		tagKeyVal := strings.Split(t, ":")
		if len(tagKeyVal) != 2 {
			return nil, errors.New(fmt.Sprintf("Invalid tag key length, tag: %v", tag))
		}

		if tagKeyVal[0] == "type" {
			tag.Type = tagKeyVal[1]
		} else if tagKeyVal[0] == "constraints" || tagKeyVal[0] == "constraint" {
			for _, c := range strings.Split(tagKeyVal[1], ",") {
				if c != "" {
					tag.Constraints = append(tag.Constraints, c)
				}
			}
		}
	}

	return tag, nil
}
<|endoftext|>


<commit_before>package tango

import (
	"net/http"
	"os"
)

const (
	Dev = iota
	Prod
)

var (
	Env = Dev

	modes = []string{
		"Dev",
		"Product",
	}
)

func Version() string {
	return "0.2.7.0119"
}

type Tango struct {
	Router
	Mode       int
	handlers   []Handler
	logger     Logger
	ErrHandler Handler
}

func (t *Tango) Logger() Logger {
	return t.logger
}

func (t *Tango) Get(url string, c interface{}) {
	t.Route([]string{"GET", "HEAD"}, url, c)
}

func (t *Tango) Post(url string, c interface{}) {
	t.Route([]string{"POST"}, url, c)
}

func (t *Tango) Head(url string, c interface{}) {
	t.Route([]string{"HEAD"}, url, c)
}

func (t *Tango) Options(url string, c interface{}) {
	t.Route([]string{"OPTIONS"}, url, c)
}

func (t *Tango) Trace(url string, c interface{}) {
	t.Route([]string{"TRACE"}, url, c)
}

func (t *Tango) Patch(url string, c interface{}) {
	t.Route([]string{"PATCH"}, url, c)
}

func (t *Tango) Delete(url string, c interface{}) {
	t.Route([]string{"DELETE"}, url, c)
}

func (t *Tango) Put(url string, c interface{}) {
	t.Route([]string{"PUT"}, url, c)
}

func (t *Tango) Any(url string, c interface{}) {
	t.Route(SupportMethods, url, c)
}

func (t *Tango) Use(handlers ...Handler) {
	for _, handler := range handlers {
		t.handlers = append(t.handlers, handler)
	}
}

func (t *Tango) Run(addrs ...string) {
	var addr string
	if len(addrs) == 0 {
		addr = ":8000"
	} else {
		addr = addrs[0]
	}

	t.logger.Info("listening on", addr, modes[t.Mode])

	err := http.ListenAndServe(addr, t)
	if err != nil {
		t.logger.Error(err)
	}
}

func (t *Tango) RunTLS(certFile, keyFile string, addrs ...string) {
	var addr string
	if len(addrs) == 0 {
		addr = ":8000"
	} else {
		addr = addrs[0]
	}

	t.logger.Info("listening on https", addr, modes[t.Mode])

	err := http.ListenAndServeTLS(addr, certFile, keyFile, t)
	if err != nil {
		t.logger.Error(err)
	}
}

type HandlerFunc func(ctx *Context)

func (h HandlerFunc) Handle(ctx *Context) {
	h(ctx)
}

func WrapBefore(handler http.Handler) HandlerFunc {
	return func(ctx *Context) {
		handler.ServeHTTP(ctx.ResponseWriter, ctx.Req())

		ctx.Next()
	}
}

func WrapAfter(handler http.Handler) HandlerFunc {
	return func(ctx *Context) {
		ctx.Next()

		handler.ServeHTTP(ctx.ResponseWriter, ctx.Req())
	}
}

func (t *Tango) UseHandler(handler http.Handler) {
	t.Use(WrapBefore(handler))
}

func (t *Tango) ServeHTTP(w http.ResponseWriter, req *http.Request) {
	ctx := NewContext(
		t,
		req,
		NewResponseWriter(w),
		t.logger,
	)

	ctx.Invoke()

	// if there is no logging or error handle, so the last written check.
	if !ctx.Written() {
		if ctx.Result == nil {
			ctx.Result = NotFound()
		}
		ctx.HandleError()
		p := req.URL.Path
		if len(req.URL.RawQuery) > 0 {
			p = p + "?" + req.URL.RawQuery
		}

		t.logger.Error(req.Method, ctx.Status(), p)
	}
}

func NewWithLog(logger Logger, handlers ...Handler) *Tango {
	tango := &Tango{
		Router:     NewRouter(),
		Mode:       Env,
		logger:     logger,
		handlers:   make([]Handler, 0),
		ErrHandler: Errors(),
	}

	tango.Use(handlers...)

	return tango
}

func New(handlers ...Handler) *Tango {
	return NewWithLog(NewLogger(os.Stdout), handlers...)
}

func Classic(l ...Logger) *Tango {
	var logger Logger
	if len(l) == 0 {
		logger = NewLogger(os.Stdout)
	} else {
		logger = l[0]
	}

	return NewWithLog(
		logger,
		Logging(),
		Recovery(true),
		Compresses([]string{".js", ".css", ".html", ".htm"}),
		Static(StaticOptions{Prefix: "public"}),
		Return(),
		Responses(),
		Requests(),
		Param(),
		Contexts(),
	)
}
<commit_msg>add ClassicHandlers<commit_after>package tango

import (
	"net/http"
	"os"
)

const (
	Dev = iota
	Prod
)

var (
	Env = Dev

	modes = []string{
		"Dev",
		"Product",
	}
)

func Version() string {
	return "0.2.7.0119"
}

type Tango struct {
	Router
	Mode       int
	handlers   []Handler
	logger     Logger
	ErrHandler Handler
}

var (
	ClassicHandlers = []Handler{
		Logging(),
		Recovery(true),
		Compresses([]string{".js", ".css", ".html", ".htm"}),
		Static(StaticOptions{Prefix: "public"}),
		Return(),
		Responses(),
		Requests(),
		Param(),
		Contexts(),
	}
)

func (t *Tango) Logger() Logger {
	return t.logger
}

func (t *Tango) Get(url string, c interface{}) {
	t.Route([]string{"GET", "HEAD"}, url, c)
}

func (t *Tango) Post(url string, c interface{}) {
	t.Route([]string{"POST"}, url, c)
}

func (t *Tango) Head(url string, c interface{}) {
	t.Route([]string{"HEAD"}, url, c)
}

func (t *Tango) Options(url string, c interface{}) {
	t.Route([]string{"OPTIONS"}, url, c)
}

func (t *Tango) Trace(url string, c interface{}) {
	t.Route([]string{"TRACE"}, url, c)
}

func (t *Tango) Patch(url string, c interface{}) {
	t.Route([]string{"PATCH"}, url, c)
}

func (t *Tango) Delete(url string, c interface{}) {
	t.Route([]string{"DELETE"}, url, c)
}

func (t *Tango) Put(url string, c interface{}) {
	t.Route([]string{"PUT"}, url, c)
}

func (t *Tango) Any(url string, c interface{}) {
	t.Route(SupportMethods, url, c)
}

func (t *Tango) Use(handlers ...Handler) {
	for _, handler := range handlers {
		t.handlers = append(t.handlers, handler)
	}
}

func (t *Tango) Run(addrs ...string) {
	var addr string
	if len(addrs) == 0 {
		addr = ":8000"
	} else {
		addr = addrs[0]
	}

	t.logger.Info("listening on", addr, modes[t.Mode])

	err := http.ListenAndServe(addr, t)
	if err != nil {
		t.logger.Error(err)
	}
}

func (t *Tango) RunTLS(certFile, keyFile string, addrs ...string) {
	var addr string
	if len(addrs) == 0 {
		addr = ":8000"
	} else {
		addr = addrs[0]
	}

	t.logger.Info("listening on https", addr, modes[t.Mode])

	err := http.ListenAndServeTLS(addr, certFile, keyFile, t)
	if err != nil {
		t.logger.Error(err)
	}
}

type HandlerFunc func(ctx *Context)

func (h HandlerFunc) Handle(ctx *Context) {
	h(ctx)
}

func WrapBefore(handler http.Handler) HandlerFunc {
	return func(ctx *Context) {
		handler.ServeHTTP(ctx.ResponseWriter, ctx.Req())

		ctx.Next()
	}
}

func WrapAfter(handler http.Handler) HandlerFunc {
	return func(ctx *Context) {
		ctx.Next()

		handler.ServeHTTP(ctx.ResponseWriter, ctx.Req())
	}
}

func (t *Tango) UseHandler(handler http.Handler) {
	t.Use(WrapBefore(handler))
}

func (t *Tango) ServeHTTP(w http.ResponseWriter, req *http.Request) {
	ctx := NewContext(
		t,
		req,
		NewResponseWriter(w),
		t.logger,
	)

	ctx.Invoke()

	// if there is no logging or error handle, so the last written check.
	if !ctx.Written() {
		if ctx.Result == nil {
			ctx.Result = NotFound()
		}
		ctx.HandleError()
		p := req.URL.Path
		if len(req.URL.RawQuery) > 0 {
			p = p + "?" + req.URL.RawQuery
		}

		t.logger.Error(req.Method, ctx.Status(), p)
	}
}

func NewWithLog(logger Logger, handlers ...Handler) *Tango {
	tango := &Tango{
		Router:     NewRouter(),
		Mode:       Env,
		logger:     logger,
		handlers:   make([]Handler, 0),
		ErrHandler: Errors(),
	}

	tango.Use(handlers...)

	return tango
}

func New(handlers ...Handler) *Tango {
	return NewWithLog(NewLogger(os.Stdout), handlers...)
}

func Classic(l ...Logger) *Tango {
	var logger Logger
	if len(l) == 0 {
		logger = NewLogger(os.Stdout)
	} else {
		logger = l[0]
	}

	return NewWithLog(
		logger,
		ClassicHandlers...,
	)
}
<|endoftext|>


<commit_before>package tap // import "github.com/mndrix/tap-go"

import "fmt"
import "testing/quick"

type T struct {
	nextTestNumber int
}

// New creates a new Tap value
func New() *T {
	return &T{
		nextTestNumber: 1,
	}
}

// Header displays a TAP header including version number and expected
// number of tests to run.  For an unknown number of tests, set
// testCount to zero (in which case the plan is not written); this is
// useful with AutoPlan.
func (t *T) Header(testCount int) {
	fmt.Printf("TAP version 13\n")
	if testCount > 0 {
		fmt.Printf("1..%d\n", testCount)
	}
}

// Ok generates TAP output indicating whether a test passed or failed.
func (t *T) Ok(test bool, description string) {
	// did the test pass or not?
	ok := "ok"
	if !test {
		ok = "not ok"
	}

	fmt.Printf("%s %d - %s\n", ok, t.nextTestNumber, description)
	t.nextTestNumber++
}

// Check runs randomized tests against a function just as "testing/quick.Check"
// does.  Success or failure generate appropriate TAP output.
func (t *T) Check(function interface{}, description string) {
	err := quick.Check(function, nil)
	if err == nil {
		t.Ok(true, description)
		return
	}

	fmt.Printf("# %s\n", err)
	t.Ok(false, description)
}

// Count returns the number of tests completed so far.
func (t *T) Count() int {
	return t.nextTestNumber - 1
}

// AutoPlan generates a test plan based on the number of tests that were run.
func (t *T) AutoPlan() {
	fmt.Printf("1..%d\n", t.Count())
}
<commit_msg>More documentation<commit_after>// Package tap provides support for automated Test Anything Protocol ("TAP")
// tests in Go.  For example:
//
// 		package main
//
// 		import "github.com/mndrix/tap-go"
//
// 		func main() {
// 			t := tap.New()
// 			t.Header(2)
// 			t.Ok(true, "first test")
// 			t.Ok(true, "second test")
// 		}
//
// generates the following output
//
// 		TAP version 13
// 		1..2
// 		ok 1 - first test
// 		ok 2 - second test
package tap // import "github.com/mndrix/tap-go"

import "fmt"
import "testing/quick"

// T is a type to encapsulate test state.  Methods on this type generate TAP
// output.
type T struct {
	nextTestNumber int
}

// New creates a new Tap value
func New() *T {
	return &T{
		nextTestNumber: 1,
	}
}

// Header displays a TAP header including version number and expected
// number of tests to run.  For an unknown number of tests, set
// testCount to zero (in which case the plan is not written); this is
// useful with AutoPlan.
func (t *T) Header(testCount int) {
	fmt.Printf("TAP version 13\n")
	if testCount > 0 {
		fmt.Printf("1..%d\n", testCount)
	}
}

// Ok generates TAP output indicating whether a test passed or failed.
func (t *T) Ok(test bool, description string) {
	// did the test pass or not?
	ok := "ok"
	if !test {
		ok = "not ok"
	}

	fmt.Printf("%s %d - %s\n", ok, t.nextTestNumber, description)
	t.nextTestNumber++
}

// Check runs randomized tests against a function just as "testing/quick.Check"
// does.  Success or failure generate appropriate TAP output.
func (t *T) Check(function interface{}, description string) {
	err := quick.Check(function, nil)
	if err == nil {
		t.Ok(true, description)
		return
	}

	fmt.Printf("# %s\n", err)
	t.Ok(false, description)
}

// Count returns the number of tests completed so far.
func (t *T) Count() int {
	return t.nextTestNumber - 1
}

// AutoPlan generates a test plan based on the number of tests that were run.
func (t *T) AutoPlan() {
	fmt.Printf("1..%d\n", t.Count())
}
<|endoftext|>


<commit_before>// Copyright (c) 2014-2015 Ludovic Fauvet
// Licensed under the MIT license

package scan

import (
	"errors"
	"fmt"
	. "github.com/etix/mirrorbits/config"
	"github.com/etix/mirrorbits/database"
	"github.com/etix/mirrorbits/filesystem"
	"github.com/etix/mirrorbits/utils"
	"github.com/garyburd/redigo/redis"
	"github.com/op/go-logging"
	"os"
	"path/filepath"
	"strconv"
	"time"
)

var (
	ScanAborted    = errors.New("scan aborted")
	ScanInProgress = errors.New("scan already in progress")

	log = logging.MustGetLogger("main")
)

type ScannerType int8

const (
	RSYNC ScannerType = iota
	FTP
)

type Scanner interface {
	Scan(url, identifier string, conn redis.Conn, stop chan bool) error
}

type filedata struct {
	path    string
	sha1    string
	sha256  string
	md5     string
	size    int64
	modTime time.Time
}

type scan struct {
	redis           *database.Redis
	walkSourceFiles []*filedata
	walkRedisConn   redis.Conn

	conn        redis.Conn
	identifier  string
	filesKey    string
	filesTmpKey string
	count       uint
}

func IsScanning(conn redis.Conn, identifier string) (bool, error) {
	return redis.Bool(conn.Do("EXISTS", fmt.Sprintf("SCANNING_%s", identifier)))
}

func Scan(typ ScannerType, r *database.Redis, url, identifier string, stop chan bool) error {
	s := &scan{
		redis:      r,
		identifier: identifier,
	}

	var scanner Scanner
	switch typ {
	case RSYNC:
		scanner = &RsyncScanner{
			scan: s,
		}
	case FTP:
		scanner = &FTPScanner{
			scan: s,
		}
	default:
		panic(fmt.Sprintf("Unknown scanner"))
	}

	// Connect to the database
	conn := s.redis.Get()
	defer conn.Close()

	s.conn = conn

	lockKey := fmt.Sprintf("SCANNING_%s", identifier)

	// Try to aquire a lock so we don't have a scanning race
	// from different nodes.
	lock, err := redis.Bool(conn.Do("SETNX", lockKey, 1))
	if err != nil {
		return err
	}
	if lock {
		// Lock aquired.
		defer conn.Do("DEL", lockKey)
		// Make the key expire automatically in case our process gets killed
		conn.Do("EXPIRE", lockKey, 600)
	} else {
		return ScanInProgress
	}

	s.setLastSync(conn, identifier, false)

	conn.Send("MULTI")

	s.filesKey = fmt.Sprintf("MIRROR_%s_FILES", identifier)
	s.filesTmpKey = fmt.Sprintf("MIRROR_%s_FILES_TMP", identifier)

	// Remove any left over
	conn.Send("DEL", s.filesTmpKey)

	err = scanner.Scan(url, identifier, conn, stop)
	if err != nil {
		// Discard MULTI
		s.ScannerDiscard()

		// Remove the temporary key
		conn.Do("DEL", s.filesTmpKey)

		log.Error("[%s] %s", identifier, err.Error())
		return err
	}

	// Exec multi
	s.ScannerCommit()

	// Get the list of files no more present on this mirror
	toremove, err := redis.Values(conn.Do("SDIFF", s.filesKey, s.filesTmpKey))
	if err != nil {
		return err
	}

	// Remove this mirror from the given file SET
	if len(toremove) > 0 {
		conn.Send("MULTI")
		for _, e := range toremove {
			log.Debug("[%s] Removing %s from mirror", identifier, e)
			conn.Send("SREM", fmt.Sprintf("FILEMIRRORS_%s", e), identifier)
			conn.Send("DEL", fmt.Sprintf("FILEINFO_%s_%s", identifier, e))
			// Publish update
			database.SendPublish(conn, database.MIRROR_FILE_UPDATE, fmt.Sprintf("%s %s", identifier, e))

		}
		_, err = conn.Do("EXEC")
		if err != nil {
			return err
		}
	}

	// Finally rename the temporary sets containing the list
	// of files for this mirror to the production key
	_, err = conn.Do("RENAME", s.filesTmpKey, s.filesKey)
	if err != nil {
		return err
	}

	sinterKey := fmt.Sprintf("HANDLEDFILES_%s", identifier)

	// Count the number of files known on the remote end
	common, _ := redis.Int64(conn.Do("SINTERSTORE", sinterKey, "FILES", s.filesKey))

	if err != nil {
		return err
	}

	s.setLastSync(conn, identifier, true)
	log.Info("[%s] Indexed %d files (%d known), %d removed", identifier, s.count, common, len(toremove))
	return nil
}

func (s *scan) ScannerAddFile(f filedata) {
	s.count++

	// Add all the files to a temporary key
	s.conn.Send("SADD", s.filesTmpKey, f.path)

	// Mark the file as being supported by this mirror
	rk := fmt.Sprintf("FILEMIRRORS_%s", f.path)
	s.conn.Send("SADD", rk, s.identifier)

	// Save the size of the current file found on this mirror
	ik := fmt.Sprintf("FILEINFO_%s_%s", s.identifier, f.path)
	s.conn.Send("HSET", ik, "size", f.size)

	// Publish update
	database.SendPublish(s.conn, database.MIRROR_FILE_UPDATE, fmt.Sprintf("%s %s", s.identifier, f.path))
}

func (s *scan) ScannerDiscard() {
	s.conn.Do("DISCARD")
}

func (s *scan) ScannerCommit() error {
	_, err := s.conn.Do("EXEC")
	return err
}

func (s *scan) setLastSync(conn redis.Conn, identifier string, successful bool) error {
	now := time.Now().UTC().Unix()

	conn.Send("MULTI")

	// Set the last sync time
	conn.Send("HSET", fmt.Sprintf("MIRROR_%s", identifier), "lastSync", now)

	// Set the last successful sync time
	if successful {
		conn.Send("HSET", fmt.Sprintf("MIRROR_%s", identifier), "lastSuccessfulSync", now)
	}

	_, err := conn.Do("EXEC")

	// Publish an update on redis
	database.Publish(conn, database.MIRROR_UPDATE, identifier)

	return err
}

// Walk inside the source/reference repository
func (s *scan) walkSource(path string, f os.FileInfo, err error) error {
	if f == nil || f.IsDir() || f.Mode()&os.ModeSymlink != 0 {
		return nil
	}

	d := new(filedata)
	d.path = path[len(GetConfig().Repository):]
	d.size = f.Size()
	d.modTime = f.ModTime()

	// Get the previous file properties
	properties, err := redis.Strings(s.walkRedisConn.Do("HMGET", fmt.Sprintf("FILE_%s", d.path), "size", "modTime", "sha1", "sha256", "md5"))
	if err != nil && err != redis.ErrNil {
		return err
	} else if len(properties) < 5 {
		// This will force a rehash
		properties = make([]string, 5)
	}

	size, _ := strconv.ParseInt(properties[0], 10, 64)
	modTime, _ := time.Parse("2006-01-02 15:04:05.999999999 -0700 MST", properties[1])
	sha1 := properties[2]
	sha256 := properties[3]
	md5 := properties[4]

	rehash := (GetConfig().Hashes.SHA1 && len(sha1) == 0) ||
		(GetConfig().Hashes.SHA256 && len(sha256) == 0) ||
		(GetConfig().Hashes.MD5 && len(md5) == 0)

	if rehash || size != d.size || !modTime.Equal(d.modTime) {
		h, err := filesystem.HashFile(GetConfig().Repository + d.path)
		if err != nil {
			log.Warning("%s: hashing failed: %s", d.path, err.Error())
		} else {
			d.sha1 = h.Sha1
			d.sha256 = h.Sha256
			d.md5 = h.Md5
			if len(d.sha1) > 0 {
				log.Info("%s: SHA1 %s", d.path, d.sha1)
			}
			if len(d.sha256) > 0 {
				log.Info("%s: SHA256 %s", d.path, d.sha256)
			}
			if len(d.md5) > 0 {
				log.Info("%s: MD5 %s", d.path, d.md5)
			}
		}
	} else {
		d.sha1 = sha1
		d.sha256 = sha256
		d.md5 = md5
	}

	s.walkSourceFiles = append(s.walkSourceFiles, d)
	return nil
}

func ScanSource(r *database.Redis, stop chan bool) (err error) {
	s := &scan{
		redis: r,
	}

	s.walkRedisConn = s.redis.Get()
	defer s.walkRedisConn.Close()
	if err != nil {
		return fmt.Errorf("redis %s", err.Error())
	}
	defer s.walkRedisConn.Close()

	s.walkSourceFiles = make([]*filedata, 0, 1000)
	defer func() {
		// Reset the slice so it can be garbage collected
		s.walkSourceFiles = nil
	}()

	//TODO lock atomically inside redis to avoid two simultanous scan

	if _, err := os.Stat(GetConfig().Repository); os.IsNotExist(err) {
		return fmt.Errorf("%s: No such file or directory", GetConfig().Repository)
	}

	log.Info("[source] Scanning the filesystem...")
	err = filepath.Walk(GetConfig().Repository, s.walkSource)
	if utils.IsStopped(stop) {
		return ScanAborted
	}
	if err != nil {
		return err
	}
	log.Info("[source] Indexing the files...")

	s.walkRedisConn.Send("MULTI")

	// Remove any left over
	s.walkRedisConn.Send("DEL", "FILES_TMP")

	// Add all the files to a temporary key
	count := 0
	for _, e := range s.walkSourceFiles {
		s.walkRedisConn.Send("SADD", "FILES_TMP", e.path)
		count++
	}

	_, err = s.walkRedisConn.Do("EXEC")
	if err != nil {
		return err
	}

	// Do a diff between the sets to get the removed files
	toremove, err := redis.Values(s.walkRedisConn.Do("SDIFF", "FILES", "FILES_TMP"))

	// Create/Update the files' hash keys with the fresh infos
	s.walkRedisConn.Send("MULTI")
	for _, e := range s.walkSourceFiles {
		s.walkRedisConn.Send("HMSET", fmt.Sprintf("FILE_%s", e.path),
			"size", e.size,
			"modTime", e.modTime,
			"sha1", e.sha1,
			"sha256", e.sha256,
			"md5", e.md5)

		// Publish update
		database.SendPublish(s.walkRedisConn, database.FILE_UPDATE, e.path)
	}

	// Remove old keys
	if len(toremove) > 0 {
		for _, e := range toremove {
			s.walkRedisConn.Send("DEL", fmt.Sprintf("FILE_%s", e))

			// Publish update
			database.SendPublish(s.walkRedisConn, database.FILE_UPDATE, fmt.Sprintf("%s", e))
		}
	}

	// Finally rename the temporary sets containing the list
	// of files to the production key
	s.walkRedisConn.Send("RENAME", "FILES_TMP", "FILES")

	_, err = s.walkRedisConn.Do("EXEC")
	if err != nil {
		return err
	}

	log.Info("[source] Scanned %d files", count)

	return nil
}
<commit_msg>scan: don't close the connection twice<commit_after>// Copyright (c) 2014-2015 Ludovic Fauvet
// Licensed under the MIT license

package scan

import (
	"errors"
	"fmt"
	. "github.com/etix/mirrorbits/config"
	"github.com/etix/mirrorbits/database"
	"github.com/etix/mirrorbits/filesystem"
	"github.com/etix/mirrorbits/utils"
	"github.com/garyburd/redigo/redis"
	"github.com/op/go-logging"
	"os"
	"path/filepath"
	"strconv"
	"time"
)

var (
	ScanAborted    = errors.New("scan aborted")
	ScanInProgress = errors.New("scan already in progress")

	log = logging.MustGetLogger("main")
)

type ScannerType int8

const (
	RSYNC ScannerType = iota
	FTP
)

type Scanner interface {
	Scan(url, identifier string, conn redis.Conn, stop chan bool) error
}

type filedata struct {
	path    string
	sha1    string
	sha256  string
	md5     string
	size    int64
	modTime time.Time
}

type scan struct {
	redis           *database.Redis
	walkSourceFiles []*filedata
	walkRedisConn   redis.Conn

	conn        redis.Conn
	identifier  string
	filesKey    string
	filesTmpKey string
	count       uint
}

func IsScanning(conn redis.Conn, identifier string) (bool, error) {
	return redis.Bool(conn.Do("EXISTS", fmt.Sprintf("SCANNING_%s", identifier)))
}

func Scan(typ ScannerType, r *database.Redis, url, identifier string, stop chan bool) error {
	s := &scan{
		redis:      r,
		identifier: identifier,
	}

	var scanner Scanner
	switch typ {
	case RSYNC:
		scanner = &RsyncScanner{
			scan: s,
		}
	case FTP:
		scanner = &FTPScanner{
			scan: s,
		}
	default:
		panic(fmt.Sprintf("Unknown scanner"))
	}

	// Connect to the database
	conn := s.redis.Get()
	defer conn.Close()

	s.conn = conn

	lockKey := fmt.Sprintf("SCANNING_%s", identifier)

	// Try to aquire a lock so we don't have a scanning race
	// from different nodes.
	lock, err := redis.Bool(conn.Do("SETNX", lockKey, 1))
	if err != nil {
		return err
	}
	if lock {
		// Lock aquired.
		defer conn.Do("DEL", lockKey)
		// Make the key expire automatically in case our process gets killed
		conn.Do("EXPIRE", lockKey, 600)
	} else {
		return ScanInProgress
	}

	s.setLastSync(conn, identifier, false)

	conn.Send("MULTI")

	s.filesKey = fmt.Sprintf("MIRROR_%s_FILES", identifier)
	s.filesTmpKey = fmt.Sprintf("MIRROR_%s_FILES_TMP", identifier)

	// Remove any left over
	conn.Send("DEL", s.filesTmpKey)

	err = scanner.Scan(url, identifier, conn, stop)
	if err != nil {
		// Discard MULTI
		s.ScannerDiscard()

		// Remove the temporary key
		conn.Do("DEL", s.filesTmpKey)

		log.Error("[%s] %s", identifier, err.Error())
		return err
	}

	// Exec multi
	s.ScannerCommit()

	// Get the list of files no more present on this mirror
	toremove, err := redis.Values(conn.Do("SDIFF", s.filesKey, s.filesTmpKey))
	if err != nil {
		return err
	}

	// Remove this mirror from the given file SET
	if len(toremove) > 0 {
		conn.Send("MULTI")
		for _, e := range toremove {
			log.Debug("[%s] Removing %s from mirror", identifier, e)
			conn.Send("SREM", fmt.Sprintf("FILEMIRRORS_%s", e), identifier)
			conn.Send("DEL", fmt.Sprintf("FILEINFO_%s_%s", identifier, e))
			// Publish update
			database.SendPublish(conn, database.MIRROR_FILE_UPDATE, fmt.Sprintf("%s %s", identifier, e))

		}
		_, err = conn.Do("EXEC")
		if err != nil {
			return err
		}
	}

	// Finally rename the temporary sets containing the list
	// of files for this mirror to the production key
	_, err = conn.Do("RENAME", s.filesTmpKey, s.filesKey)
	if err != nil {
		return err
	}

	sinterKey := fmt.Sprintf("HANDLEDFILES_%s", identifier)

	// Count the number of files known on the remote end
	common, _ := redis.Int64(conn.Do("SINTERSTORE", sinterKey, "FILES", s.filesKey))

	if err != nil {
		return err
	}

	s.setLastSync(conn, identifier, true)
	log.Info("[%s] Indexed %d files (%d known), %d removed", identifier, s.count, common, len(toremove))
	return nil
}

func (s *scan) ScannerAddFile(f filedata) {
	s.count++

	// Add all the files to a temporary key
	s.conn.Send("SADD", s.filesTmpKey, f.path)

	// Mark the file as being supported by this mirror
	rk := fmt.Sprintf("FILEMIRRORS_%s", f.path)
	s.conn.Send("SADD", rk, s.identifier)

	// Save the size of the current file found on this mirror
	ik := fmt.Sprintf("FILEINFO_%s_%s", s.identifier, f.path)
	s.conn.Send("HSET", ik, "size", f.size)

	// Publish update
	database.SendPublish(s.conn, database.MIRROR_FILE_UPDATE, fmt.Sprintf("%s %s", s.identifier, f.path))
}

func (s *scan) ScannerDiscard() {
	s.conn.Do("DISCARD")
}

func (s *scan) ScannerCommit() error {
	_, err := s.conn.Do("EXEC")
	return err
}

func (s *scan) setLastSync(conn redis.Conn, identifier string, successful bool) error {
	now := time.Now().UTC().Unix()

	conn.Send("MULTI")

	// Set the last sync time
	conn.Send("HSET", fmt.Sprintf("MIRROR_%s", identifier), "lastSync", now)

	// Set the last successful sync time
	if successful {
		conn.Send("HSET", fmt.Sprintf("MIRROR_%s", identifier), "lastSuccessfulSync", now)
	}

	_, err := conn.Do("EXEC")

	// Publish an update on redis
	database.Publish(conn, database.MIRROR_UPDATE, identifier)

	return err
}

// Walk inside the source/reference repository
func (s *scan) walkSource(path string, f os.FileInfo, err error) error {
	if f == nil || f.IsDir() || f.Mode()&os.ModeSymlink != 0 {
		return nil
	}

	d := new(filedata)
	d.path = path[len(GetConfig().Repository):]
	d.size = f.Size()
	d.modTime = f.ModTime()

	// Get the previous file properties
	properties, err := redis.Strings(s.walkRedisConn.Do("HMGET", fmt.Sprintf("FILE_%s", d.path), "size", "modTime", "sha1", "sha256", "md5"))
	if err != nil && err != redis.ErrNil {
		return err
	} else if len(properties) < 5 {
		// This will force a rehash
		properties = make([]string, 5)
	}

	size, _ := strconv.ParseInt(properties[0], 10, 64)
	modTime, _ := time.Parse("2006-01-02 15:04:05.999999999 -0700 MST", properties[1])
	sha1 := properties[2]
	sha256 := properties[3]
	md5 := properties[4]

	rehash := (GetConfig().Hashes.SHA1 && len(sha1) == 0) ||
		(GetConfig().Hashes.SHA256 && len(sha256) == 0) ||
		(GetConfig().Hashes.MD5 && len(md5) == 0)

	if rehash || size != d.size || !modTime.Equal(d.modTime) {
		h, err := filesystem.HashFile(GetConfig().Repository + d.path)
		if err != nil {
			log.Warning("%s: hashing failed: %s", d.path, err.Error())
		} else {
			d.sha1 = h.Sha1
			d.sha256 = h.Sha256
			d.md5 = h.Md5
			if len(d.sha1) > 0 {
				log.Info("%s: SHA1 %s", d.path, d.sha1)
			}
			if len(d.sha256) > 0 {
				log.Info("%s: SHA256 %s", d.path, d.sha256)
			}
			if len(d.md5) > 0 {
				log.Info("%s: MD5 %s", d.path, d.md5)
			}
		}
	} else {
		d.sha1 = sha1
		d.sha256 = sha256
		d.md5 = md5
	}

	s.walkSourceFiles = append(s.walkSourceFiles, d)
	return nil
}

func ScanSource(r *database.Redis, stop chan bool) (err error) {
	s := &scan{
		redis: r,
	}

	s.walkRedisConn = s.redis.Get()
	defer s.walkRedisConn.Close()
	if err != nil {
		return fmt.Errorf("redis %s", err.Error())
	}

	s.walkSourceFiles = make([]*filedata, 0, 1000)
	defer func() {
		// Reset the slice so it can be garbage collected
		s.walkSourceFiles = nil
	}()

	//TODO lock atomically inside redis to avoid two simultanous scan

	if _, err := os.Stat(GetConfig().Repository); os.IsNotExist(err) {
		return fmt.Errorf("%s: No such file or directory", GetConfig().Repository)
	}

	log.Info("[source] Scanning the filesystem...")
	err = filepath.Walk(GetConfig().Repository, s.walkSource)
	if utils.IsStopped(stop) {
		return ScanAborted
	}
	if err != nil {
		return err
	}
	log.Info("[source] Indexing the files...")

	s.walkRedisConn.Send("MULTI")

	// Remove any left over
	s.walkRedisConn.Send("DEL", "FILES_TMP")

	// Add all the files to a temporary key
	count := 0
	for _, e := range s.walkSourceFiles {
		s.walkRedisConn.Send("SADD", "FILES_TMP", e.path)
		count++
	}

	_, err = s.walkRedisConn.Do("EXEC")
	if err != nil {
		return err
	}

	// Do a diff between the sets to get the removed files
	toremove, err := redis.Values(s.walkRedisConn.Do("SDIFF", "FILES", "FILES_TMP"))

	// Create/Update the files' hash keys with the fresh infos
	s.walkRedisConn.Send("MULTI")
	for _, e := range s.walkSourceFiles {
		s.walkRedisConn.Send("HMSET", fmt.Sprintf("FILE_%s", e.path),
			"size", e.size,
			"modTime", e.modTime,
			"sha1", e.sha1,
			"sha256", e.sha256,
			"md5", e.md5)

		// Publish update
		database.SendPublish(s.walkRedisConn, database.FILE_UPDATE, e.path)
	}

	// Remove old keys
	if len(toremove) > 0 {
		for _, e := range toremove {
			s.walkRedisConn.Send("DEL", fmt.Sprintf("FILE_%s", e))

			// Publish update
			database.SendPublish(s.walkRedisConn, database.FILE_UPDATE, fmt.Sprintf("%s", e))
		}
	}

	// Finally rename the temporary sets containing the list
	// of files to the production key
	s.walkRedisConn.Send("RENAME", "FILES_TMP", "FILES")

	_, err = s.walkRedisConn.Do("EXEC")
	if err != nil {
		return err
	}

	log.Info("[source] Scanned %d files", count)

	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"auto_updater"
	"cron_eval"
	"database/sql"
	"fmt"
	_ "github.com/mattn/go-sqlite3"
	"os"
	"os/exec"
	"parse_update_script"
	"path/filepath"
	"strconv"
	"time"
	"util"
)

type UpdateScriptResponse struct {
	ret_code      int
	system_id     int
	stdout        string
	stderr        string
	is_start      bool
	is_end        bool
	update_script *parse_update_script.UpdateScript
	db            *sql.DB
	api_url       string
}

// SystemReboot executes a shell command to reboot the host
func SystemReboot() {
	// Sleep for 2 seconds to give time for the API post
	time.Sleep(2 * time.Second)
	cmd := exec.Command("/sbin/shutdown", "-r", "now")
	err := cmd.Run()
	if err != nil {
		panic(err)
	}
}

// Main entry point
func main() {
	log := auto_updater.GetLogger()

	exec_path, _ := os.Getwd()
	HOSTNAME, _ := os.Hostname()
	config, config_err := auto_updater.ParseConfig()
	if config_err != nil {
		log.Error("Unable to open config file")
	}
	log.Debug("Config Loaded")
	log.Debug(HOSTNAME)
	var DB_FILE = fmt.Sprintf("%s/%s", exec_path, config.Main.Dbfile)
	log.Debug("DB_FILE: ", DB_FILE)
	var CRONFILE = fmt.Sprintf("%s/%s", exec_path, config.Main.Cronfile)
	var SCRIPTPATH = fmt.Sprintf("%s/%s", exec_path, config.Main.Scriptpath)
	var APIURL = config.Main.APIUrl
	system_id, system_id_err := GetSystemId(APIURL, HOSTNAME)
	if system_id_err != nil {
		log.Error("Could not retrieve system ID")
	}
	log.Error("System ID:", system_id)
	log.Debug("GUIDHash: ", GUIDHash(HOSTNAME))
	db_created := auto_updater.CreateDbIfNotExists(DB_FILE)
	if db_created {
		log.Info("DB Created at path: ", DB_FILE)
	}
	//	db, db_open_err := sql.Open("sqlite3", DB_FILE)
	db1, db_open_err := sql.Open("sqlite3", fmt.Sprintf("file:%s?cache=shared&mode=rwc", DB_FILE))
	//db2, db_open_err2 := sql.Open("sqlite3", fmt.Sprintf("file:%s?cache=shared&mode=rwc", DB_FILE))
	if db_open_err != nil {
		panic("Unable to open existing database")
	}
	/*if db_open_err2 != nil {
		panic("Unable to open existing database")
	}*/
	state, _ := auto_updater.GetMostRecentState(db1)
	log.Debug(state)
	var current_locked = false
	start_state, start_state_err := auto_updater.GetMostRecentState(db1)
	if start_state.Finished == 0 && start_state.Id > 0 {
		log.Error("asdf", start_state.Id)
		current_locked = true
	}
	if start_state_err != nil && start_state.Finished == 0 {
		if start_state_err.Error() != "sql: no rows in result set" {
			current_locked = true
		}
	}
	log.Debug("At INIT: current_locked:", current_locked)
	if !util.HasScriptPath(SCRIPTPATH) {
		log.Error(fmt.Sprintf("Script Path %s does not exist.", SCRIPTPATH))
		os.Exit(2)
	}
	go auto_updater.DBPoll(db1, HOSTNAME, APIURL, 0)
	for {
		log.Debug("In LOOP: current_locked:", current_locked)
		cron_line, cron_err := util.ReadCronFile(CRONFILE)
		if cron_err != nil {
			log.Debug(fmt.Sprintf("%s", cron_err))
			os.Exit(2)
		}

		run_now, run_after, sleep_seconds := cron_eval.EvalCronLine(cron_line)
		if run_now == false && run_after == false {
			time.Sleep(time.Duration(sleep_seconds) * time.Second)
			continue
		}

		if run_now == false && run_after == true {
			time.Sleep(time.Duration(sleep_seconds) * time.Second)
		}

		scripts, _ := filepath.Glob(fmt.Sprintf("%s/*", SCRIPTPATH))
		UpdateScripts := []parse_update_script.UpdateScript{}

		var run_next = false
		for i := 0; i < len(scripts); i++ {
			script_path := scripts[i]
			if !util.ScriptValid(script_path) {
				continue
			}
			if run_next == false && current_locked && script_path != start_state.Last_script_completed && start_state.Last_script_completed != "" {
				// We are locked from running scripts
				run_next = false
				continue
			} else if current_locked && script_path == start_state.Last_script_completed && start_state.Last_script_completed != "" {
				// We are locked from running scripts
				// The last ran script is the current in the stack
				// unlock and continue to the next runnable script
				run_next = true
				continue
			} else {
				run_next = true
			}
			var uf parse_update_script.UpdateScriptFile
			var us parse_update_script.UpdateScript
			uf.FilePath = script_path
			parse_update_script.ReadFile(&uf)
			us.ParseScript(&uf)
			UpdateScripts = append(UpdateScripts, us)
		}
		exec_chan := make(chan UpdateScriptResponse)
		go ProcessEntry(exec_chan, db1)
		should_reboot := false
		for i := 0; i < len(UpdateScripts); i++ {
			if should_reboot == true {
				continue
			}
			exec_script := UpdateScripts[i].FilePath
			ret_code, stdout, stderr := auto_updater.ExecCommand(exec_script)
			usr := UpdateScriptResponse{}
			usr.ret_code = ret_code
			usr.stderr = stderr
			usr.stdout = stdout
			usr.update_script = &UpdateScripts[i]
			usr.db = db1
			usr.system_id = system_id
			usr.api_url = APIURL
			log.Error("current_locked:", current_locked)
			if i == 0 && !current_locked {
				usr.is_start = true
			} else {
				usr.is_start = false
			}
			if i == len(UpdateScripts)-1 {
				usr.is_end = true
			} else {
				usr.is_end = false
			}
			exec_chan <- usr
			go ProcessEntry(exec_chan, db1)
			//auto_updater.ProcessLog(update_guid, ret_code, stdout, stderr, &UpdateScripts[i], db2)
			exit_code_to_i, exit_code_to_i_err := strconv.Atoi(UpdateScripts[i].ScriptExitCodeReboot)
			//state.SetLastScriptCompleted(db1, exec_script)

			if exit_code_to_i_err != nil {
				panic(exit_code_to_i_err)
			}
			if exit_code_to_i == ret_code {
				SystemReboot()
				should_reboot = true
			}

		}
		current_locked = false
		//state.Finish(db1)
		//time.Sleep(10 * time.Minute)
	}
}
<commit_msg>Checking for current_locked with sleep. Issue #5<commit_after>package main

import (
	"auto_updater"
	"cron_eval"
	"database/sql"
	"fmt"
	_ "github.com/mattn/go-sqlite3"
	"os"
	"os/exec"
	"parse_update_script"
	"path/filepath"
	"strconv"
	"time"
	"util"
)

type UpdateScriptResponse struct {
	ret_code      int
	system_id     int
	stdout        string
	stderr        string
	is_start      bool
	is_end        bool
	update_script *parse_update_script.UpdateScript
	db            *sql.DB
	api_url       string
}

// SystemReboot executes a shell command to reboot the host
func SystemReboot() {
	// Sleep for 2 seconds to give time for the API post
	time.Sleep(2 * time.Second)
	cmd := exec.Command("/sbin/shutdown", "-r", "now")
	err := cmd.Run()
	if err != nil {
		panic(err)
	}
}

// Main entry point
func main() {
	log := auto_updater.GetLogger()

	exec_path, _ := os.Getwd()
	HOSTNAME, _ := os.Hostname()
	config, config_err := auto_updater.ParseConfig()
	if config_err != nil {
		log.Error("Unable to open config file")
	}
	log.Debug("Config Loaded")
	log.Debug(HOSTNAME)
	var DB_FILE = fmt.Sprintf("%s/%s", exec_path, config.Main.Dbfile)
	log.Debug("DB_FILE: ", DB_FILE)
	var CRONFILE = fmt.Sprintf("%s/%s", exec_path, config.Main.Cronfile)
	var SCRIPTPATH = fmt.Sprintf("%s/%s", exec_path, config.Main.Scriptpath)
	var APIURL = config.Main.APIUrl
	system_id, system_id_err := GetSystemId(APIURL, HOSTNAME)
	if system_id_err != nil {
		log.Error("Could not retrieve system ID")
	}
	log.Error("System ID:", system_id)
	log.Debug("GUIDHash: ", GUIDHash(HOSTNAME))
	db_created := auto_updater.CreateDbIfNotExists(DB_FILE)
	if db_created {
		log.Info("DB Created at path: ", DB_FILE)
	}
	//	db, db_open_err := sql.Open("sqlite3", DB_FILE)
	db1, db_open_err := sql.Open("sqlite3", fmt.Sprintf("file:%s?cache=shared&mode=rwc", DB_FILE))
	//db2, db_open_err2 := sql.Open("sqlite3", fmt.Sprintf("file:%s?cache=shared&mode=rwc", DB_FILE))
	if db_open_err != nil {
		panic("Unable to open existing database")
	}
	/*if db_open_err2 != nil {
		panic("Unable to open existing database")
	}*/
	state, _ := auto_updater.GetMostRecentState(db1)
	log.Debug(state)
	var current_locked = false
	start_state, start_state_err := auto_updater.GetMostRecentState(db1)
	if start_state.Finished == 0 && start_state.Id > 0 {
		log.Error("asdf", start_state.Id)
		current_locked = true
	}
	if start_state_err != nil && start_state.Finished == 0 {
		if start_state_err.Error() != "sql: no rows in result set" {
			current_locked = true
		}
	}
	log.Debug("At INIT: current_locked:", current_locked)
	if !util.HasScriptPath(SCRIPTPATH) {
		log.Error(fmt.Sprintf("Script Path %s does not exist.", SCRIPTPATH))
		os.Exit(2)
	}
	go auto_updater.DBPoll(db1, HOSTNAME, APIURL, 0)
	for {
		log.Debug("In LOOP: current_locked:", current_locked)
		cron_line, cron_err := util.ReadCronFile(CRONFILE)
		if cron_err != nil {
			log.Debug(fmt.Sprintf("%s", cron_err))
			os.Exit(2)
		}

		run_now, run_after, sleep_seconds := cron_eval.EvalCronLine(cron_line)
		if run_now == false && run_after == false && current_locked == false {
			time.Sleep(time.Duration(sleep_seconds) * time.Second)
			continue
		}

		if run_now == false && run_after == true && current_locked == false {
			time.Sleep(time.Duration(sleep_seconds) * time.Second)
		}

		scripts, _ := filepath.Glob(fmt.Sprintf("%s/*", SCRIPTPATH))
		UpdateScripts := []parse_update_script.UpdateScript{}

		var run_next = false
		for i := 0; i < len(scripts); i++ {
			script_path := scripts[i]
			if !util.ScriptValid(script_path) {
				continue
			}
			if run_next == false && current_locked && script_path != start_state.Last_script_completed && start_state.Last_script_completed != "" {
				// We are locked from running scripts
				run_next = false
				continue
			} else if current_locked && script_path == start_state.Last_script_completed && start_state.Last_script_completed != "" {
				// We are locked from running scripts
				// The last ran script is the current in the stack
				// unlock and continue to the next runnable script
				run_next = true
				continue
			} else {
				run_next = true
			}
			var uf parse_update_script.UpdateScriptFile
			var us parse_update_script.UpdateScript
			uf.FilePath = script_path
			parse_update_script.ReadFile(&uf)
			us.ParseScript(&uf)
			UpdateScripts = append(UpdateScripts, us)
		}
		exec_chan := make(chan UpdateScriptResponse)
		go ProcessEntry(exec_chan, db1)
		should_reboot := false
		for i := 0; i < len(UpdateScripts); i++ {
			if should_reboot == true {
				continue
			}
			exec_script := UpdateScripts[i].FilePath
			ret_code, stdout, stderr := auto_updater.ExecCommand(exec_script)
			usr := UpdateScriptResponse{}
			usr.ret_code = ret_code
			usr.stderr = stderr
			usr.stdout = stdout
			usr.update_script = &UpdateScripts[i]
			usr.db = db1
			usr.system_id = system_id
			usr.api_url = APIURL
			log.Error("current_locked:", current_locked)
			if i == 0 && !current_locked {
				usr.is_start = true
			} else {
				usr.is_start = false
			}
			if i == len(UpdateScripts)-1 {
				usr.is_end = true
			} else {
				usr.is_end = false
			}
			exec_chan <- usr
			go ProcessEntry(exec_chan, db1)
			//auto_updater.ProcessLog(update_guid, ret_code, stdout, stderr, &UpdateScripts[i], db2)
			exit_code_to_i, exit_code_to_i_err := strconv.Atoi(UpdateScripts[i].ScriptExitCodeReboot)
			//state.SetLastScriptCompleted(db1, exec_script)

			if exit_code_to_i_err != nil {
				panic(exit_code_to_i_err)
			}
			if exit_code_to_i == ret_code {
				SystemReboot()
				should_reboot = true
			}

		}
		current_locked = false
		//state.Finish(db1)
		//time.Sleep(10 * time.Minute)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"crypto/rand"
	"crypto/sha512"
	"encoding/base64"
	"fmt"
	"net/http"
	"strings"
	"text/template"
	"time"

	"github.com/codegangsta/martini"

	"google.golang.org/appengine"
	"google.golang.org/appengine/datastore"
	"google.golang.org/appengine/user"
)

func PublishPost(rw http.ResponseWriter, req *http.Request, params martini.Params) {
	c := appengine.NewContext(req)
	u := user.Current(c)
	if u == nil {
		http.Error(rw, fmt.Sprintf("wat %s", u), http.StatusForbidden)
		return
	}

	if !isAuthedToMakeChanges(u) {
		http.Error(rw, fmt.Sprintf("wat? %s", u), http.StatusForbidden)
		return
	}

	req.ParseForm()
	postslug := req.PostFormValue("slug")
	if postslug == "" {
		postslug = fmt.Sprintf("DRAFT-%s", RandString(10))
	}

	k := datastore.NewKey(c, "Post", postslug, 0, nil)

	postdate, err := time.Parse("2006-01-02 15:04:05", req.PostFormValue("date"))

	if err != nil {
		http.Error(rw, err.Error(), http.StatusInternalServerError)
	}

	NP := Post{
		Author:       "Benjojo",
		Content:      base64.StdEncoding.EncodeToString([]byte(req.PostFormValue("post"))),
		Date:         postdate,
		Slug:         postslug,
		Title:        strings.Split(req.PostFormValue("post"), "\n")[0],
		Type:         req.PostFormValue("cata"),
		R1:           req.PostFormValue("R1"),
		R2:           req.PostFormValue("R2"),
		FeatureImage: req.PostFormValue("featureimage"),
	}
	_, err = datastore.Put(c, k, &NP)
	if err != nil {
		http.Error(rw, err.Error(), http.StatusInternalServerError)
	} else {
		http.Error(rw, fmt.Sprintf("/post/%s", postslug), http.StatusCreated)
	}
	forceUpdatePostTitleCache(rw, req)
	return
}

func isAuthedToMakeChanges(u *user.User) bool {
	emailHash := sha512.New()
	emailHashString := fmt.Sprintf("%x", emailHash.Sum([]byte(fmt.Sprintf("%s", u))))

	if emailHashString == "8270bbd8cfc0ff367556e4ee1ec05d7f5873b65d71001c154efbb03dca232307c2616fa81ca7bacbe7e1739b8f49cf9cb4a3c3905df9faebf75992233ff4d170" {
		return true
	}
	if emailHashString == "4d98aa710d2be9770a0d173718a85dfb7b527d6c1aa7952fa3011cc2fcefa5cc22745c563faa95c5dd4a8b616f42b815f00527d4bdd033959dcd443fd3dfd8b2" {
		return true
	}
	if emailHashString == "cffd49359ad9dcadd350f9cbb913f210303d0c20c1a57eaf59c81a3301abee39377095e8ae5b827633ca7624d473361d5c37ff848418185892b7b58cca1b677a" {
		return true
	}
	return false
}

func RemovePost(rw http.ResponseWriter, req *http.Request, params martini.Params) {
	c := appengine.NewContext(req)

	u := user.Current(c)
	if u == nil {
		http.Error(rw, fmt.Sprintf("wat %s", u), http.StatusForbidden)
		return
	}

	if !isAuthedToMakeChanges(u) {
		http.Error(rw, fmt.Sprintf("wat? %s", u), http.StatusForbidden)
		return
	}

	k := datastore.NewKey(c, "Post", params["name"], 0, nil)
	err := datastore.Delete(c, k)
	if err == nil {
		rw.Write([]byte("the deed has been done\r\n"))
	}
}

func Run_GC(rw http.ResponseWriter, req *http.Request) {
	c := appengine.NewContext(req)
	u := user.Current(c)
	if u == nil {
		http.Error(rw, fmt.Sprintf("wat %s", u), http.StatusForbidden)
		return
	}

	if !isAuthedToMakeChanges(u) {
		http.Error(rw, fmt.Sprintf("wat? %s", u), http.StatusForbidden)
		return
	}

	q := datastore.NewQuery("Post").Order("-Date").Limit(100)

	for t := q.Run(c); ; {
		var e Post
		key, err := t.Next(&e)
		if err == datastore.Done {
			break
		}
		if err != nil {
			// return nil, err
			break
		}
		if strings.HasPrefix(e.Slug, "DRAFT-") || e.Title == "" {
			// DESTROY
			err := datastore.Delete(c, key)
			if err == nil {
				rw.Write([]byte("Killed Draft\r\n"))
			}
		}
	}

}

func Admin(rw http.ResponseWriter, req *http.Request, params martini.Params) {

	layoutData := struct {
		Date string
	}{
		Date: time.Now().Format("2006-01-02 15:04:05"),
	}
	var AdminPage = template.Must(template.ParseFiles("public2/admin.html"))

	err := AdminPage.Execute(rw, layoutData)
	if err != nil {
		http.Error(rw, err.Error(), http.StatusInternalServerError)
	}
}

func RandString(n int) string {
	const alphanum = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"
	var bytes = make([]byte, n)
	rand.Read(bytes)
	for i, b := range bytes {
		bytes[i] = alphanum[b%byte(len(alphanum))]
	}
	return string(bytes)
}
<commit_msg>Better auditing?<commit_after>package main

import (
	"crypto/rand"
	"crypto/sha512"
	"encoding/base64"
	"fmt"
	"log"
	"net/http"
	"strings"
	"text/template"
	"time"

	"github.com/codegangsta/martini"

	"google.golang.org/appengine"
	"google.golang.org/appengine/datastore"
	"google.golang.org/appengine/user"
)

func PublishPost(rw http.ResponseWriter, req *http.Request, params martini.Params) {
	c := appengine.NewContext(req)
	u := user.Current(c)
	if u == nil {
		http.Error(rw, fmt.Sprintf("wat %s", u), http.StatusForbidden)
		return
	}

	if !isAuthedToMakeChanges(u) {
		http.Error(rw, fmt.Sprintf("wat? %s", u), http.StatusForbidden)
		return
	}

	req.ParseForm()
	postslug := req.PostFormValue("slug")
	if postslug == "" {
		postslug = fmt.Sprintf("DRAFT-%s", RandString(10))
	}

	k := datastore.NewKey(c, "Post", postslug, 0, nil)

	postdate, err := time.Parse("2006-01-02 15:04:05", req.PostFormValue("date"))

	if err != nil {
		http.Error(rw, err.Error(), http.StatusInternalServerError)
	}

	NP := Post{
		Author:       "Benjojo",
		Content:      base64.StdEncoding.EncodeToString([]byte(req.PostFormValue("post"))),
		Date:         postdate,
		Slug:         postslug,
		Title:        strings.Split(req.PostFormValue("post"), "\n")[0],
		Type:         req.PostFormValue("cata"),
		R1:           req.PostFormValue("R1"),
		R2:           req.PostFormValue("R2"),
		FeatureImage: req.PostFormValue("featureimage"),
	}
	_, err = datastore.Put(c, k, &NP)
	if err != nil {
		http.Error(rw, err.Error(), http.StatusInternalServerError)
	} else {
		http.Error(rw, fmt.Sprintf("/post/%s", postslug), http.StatusCreated)
	}
	forceUpdatePostTitleCache(rw, req)
	return
}

func isAuthedToMakeChanges(u *user.User) bool {
	emailHash := sha512.New()
	emailHashString := fmt.Sprintf("%x", emailHash.Sum([]byte(fmt.Sprintf("%s", u))))
	log.Printf("Email Hash Attempt %s / %s", u, emailHashString)
	if emailHashString == "8270bbd8cfc0ff367556e4ee1ec05d7f5873b65d71001c154efbb03dca232307c2616fa81ca7bacbe7e1739b8f49cf9cb4a3c3905df9faebf75992233ff4d170" {
		return true
	}
	if emailHashString == "4d98aa710d2be9770a0d173718a85dfb7b527d6c1aa7952fa3011cc2fcefa5cc22745c563faa95c5dd4a8b616f42b815f00527d4bdd033959dcd443fd3dfd8b2" {
		return true
	}
	if emailHashString == "cffd49359ad9dcadd350f9cbb913f210303d0c20c1a57eaf59c81a3301abee39377095e8ae5b827633ca7624d473361d5c37ff848418185892b7b58cca1b677a" {
		return true
	}
	return false
}

func RemovePost(rw http.ResponseWriter, req *http.Request, params martini.Params) {
	c := appengine.NewContext(req)

	u := user.Current(c)
	if u == nil {
		http.Error(rw, fmt.Sprintf("wat %s", u), http.StatusForbidden)
		return
	}

	if !isAuthedToMakeChanges(u) {
		http.Error(rw, fmt.Sprintf("wat? %s", u), http.StatusForbidden)
		return
	}

	k := datastore.NewKey(c, "Post", params["name"], 0, nil)
	err := datastore.Delete(c, k)
	if err == nil {
		rw.Write([]byte("the deed has been done\r\n"))
	}
}

func Run_GC(rw http.ResponseWriter, req *http.Request) {
	c := appengine.NewContext(req)
	u := user.Current(c)
	if u == nil {
		http.Error(rw, fmt.Sprintf("wat %s", u), http.StatusForbidden)
		return
	}

	if !isAuthedToMakeChanges(u) {
		http.Error(rw, fmt.Sprintf("wat? %s", u), http.StatusForbidden)
		return
	}

	q := datastore.NewQuery("Post").Order("-Date").Limit(100)

	for t := q.Run(c); ; {
		var e Post
		key, err := t.Next(&e)
		if err == datastore.Done {
			break
		}
		if err != nil {
			// return nil, err
			break
		}
		if strings.HasPrefix(e.Slug, "DRAFT-") || e.Title == "" {
			// DESTROY
			err := datastore.Delete(c, key)
			if err == nil {
				rw.Write([]byte("Killed Draft\r\n"))
			}
		}
	}

}

func Admin(rw http.ResponseWriter, req *http.Request, params martini.Params) {

	layoutData := struct {
		Date string
	}{
		Date: time.Now().Format("2006-01-02 15:04:05"),
	}
	var AdminPage = template.Must(template.ParseFiles("public2/admin.html"))

	err := AdminPage.Execute(rw, layoutData)
	if err != nil {
		http.Error(rw, err.Error(), http.StatusInternalServerError)
	}
}

func RandString(n int) string {
	const alphanum = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"
	var bytes = make([]byte, n)
	rand.Read(bytes)
	for i, b := range bytes {
		bytes[i] = alphanum[b%byte(len(alphanum))]
	}
	return string(bytes)
}
<|endoftext|>


<commit_before>package hoverfly

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"time"

	// static assets
	_ "github.com/SpectoLabs/hoverfly/statik"
	"github.com/rakyll/statik/fs"

	log "github.com/Sirupsen/logrus"
	"github.com/codegangsta/negroni"
	"github.com/go-zoo/bone"
	"github.com/gorilla/websocket"
	"github.com/meatballhat/negroni-logrus"
)

// recordedRequests struct encapsulates payload data
type recordedRequests struct {
	Data []Payload `json:"data"`
}

type recordsCount struct {
	Count int `json:"count"`
}

type statsResponse struct {
	Stats        Stats `json:"stats"`
	RecordsCount int   `json:"recordsCount"`
}

type stateRequest struct {
	Mode        string `json:"mode"`
	Destination string `json:"destination"`
}

type messageResponse struct {
	Message string `json:"message"`
}

// StartAdminInterface - starts admin interface web server
func (d *DBClient) StartAdminInterface() {
	go func() {
		// starting admin interface
		mux := getBoneRouter(*d)
		n := negroni.Classic()

		logLevel := log.WarnLevel

		if d.Cfg.Verbose {
			logLevel = log.DebugLevel
		}

		n.Use(negronilogrus.NewCustomMiddleware(logLevel, &log.JSONFormatter{}, "admin"))
		n.UseHandler(mux)

		// admin interface starting message
		log.WithFields(log.Fields{
			"AdminPort": d.Cfg.AdminPort,
		}).Info("Admin interface is starting...")

		n.Run(fmt.Sprintf(":%s", d.Cfg.AdminPort))
	}()
}

// getBoneRouter returns mux for admin interface
func getBoneRouter(d DBClient) *bone.Mux {
	mux := bone.New()

	// preparing static assets for embedded admin
	statikFS, err := fs.New()
	if err != nil {
		log.WithFields(log.Fields{
			"Error": err.Error(),
		}).Error("Failed to load statikFS, admin UI might not work :(")
	}

	mux.Get("/records", http.HandlerFunc(d.AllRecordsHandler))
	mux.Delete("/records", http.HandlerFunc(d.DeleteAllRecordsHandler))
	mux.Post("/records", http.HandlerFunc(d.ImportRecordsHandler))

	mux.Get("/count", http.HandlerFunc(d.RecordsCount))
	mux.Get("/stats", http.HandlerFunc(d.StatsHandler))
	mux.Get("/statsws", http.HandlerFunc(d.StatsWSHandler))

	mux.Get("/state", http.HandlerFunc(d.CurrentStateHandler))
	mux.Post("/state", http.HandlerFunc(d.StateHandler))

	if d.Cfg.Development {
		mux.Handle("/*", http.FileServer(http.Dir("static/dist")))
	} else {
		mux.Handle("/*", http.FileServer(statikFS))
	}

	return mux
}

// AllRecordsHandler returns JSON content type http response
func (d *DBClient) AllRecordsHandler(w http.ResponseWriter, req *http.Request) {
	records, err := d.Cache.GetAllRequests()

	if err == nil {

		w.Header().Set("Content-Type", "application/json")

		var response recordedRequests
		response.Data = records
		b, err := json.Marshal(response)

		if err != nil {
			log.Error(err)
			http.Error(w, err.Error(), http.StatusInternalServerError)
		} else {
			w.Write(b)
			return
		}
	} else {
		log.WithFields(log.Fields{
			"Error": err.Error(),
		}).Error("Failed to get data from cache!")

		w.Header().Set("Content-Type", "application/json; charset=UTF-8")
		w.WriteHeader(500) // can't process this entity
		return
	}
}

// RecordsCount returns number of captured requests as a JSON payload
func (d *DBClient) RecordsCount(w http.ResponseWriter, req *http.Request) {
	count, err := d.Cache.RecordsCount()

	if err == nil {

		w.Header().Set("Content-Type", "application/json")

		var response recordsCount
		response.Count = count
		b, err := json.Marshal(response)

		if err != nil {
			log.Error(err)
			http.Error(w, err.Error(), http.StatusInternalServerError)
		} else {
			w.Write(b)
			return
		}
	} else {
		log.WithFields(log.Fields{
			"Error": err.Error(),
		}).Error("Failed to get data from cache!")

		w.Header().Set("Content-Type", "application/json; charset=UTF-8")
		w.WriteHeader(500) // can't process this entity
		return
	}
}

// StatsHandler - returns current stats about Hoverfly (request counts, record count)
func (d *DBClient) StatsHandler(w http.ResponseWriter, req *http.Request) {
	stats := d.Counter.Flush()

	count, err := d.Cache.RecordsCount()

	if err != nil {
		log.Error(err)
		http.Error(w, err.Error(), http.StatusInternalServerError)
	}

	var sr statsResponse
	sr.Stats = stats
	sr.RecordsCount = count

	w.Header().Set("Content-Type", "application/json")

	b, err := json.Marshal(sr)

	if err != nil {
		log.Error(err)
		http.Error(w, err.Error(), http.StatusInternalServerError)
	} else {
		w.Write(b)
		return
	}

}

var upgrader = websocket.Upgrader{
	ReadBufferSize:  1024,
	WriteBufferSize: 1024,
	CheckOrigin: func(r *http.Request) bool {
		return true
	},
}

// StatsWSHandler - returns current stats about Hoverfly (request counts, record count) through the websocket
func (d *DBClient) StatsWSHandler(w http.ResponseWriter, r *http.Request) {
	conn, err := upgrader.Upgrade(w, r, nil)
	if err != nil {
		log.Println(err)
		return
	}

	for {
		messageType, p, err := conn.ReadMessage()
		if err != nil {
			return
		}
		log.WithFields(log.Fields{
			"message": string(p),
		}).Info("Got message...")

		for _ = range time.Tick(1 * time.Second) {

			count, err := d.Cache.RecordsCount()

			if err != nil {
				log.WithFields(log.Fields{
					"message": p,
					"error":   err.Error(),
				}).Error("got error while trying to get records count")
				return
			}

			stats := d.Counter.Flush()

			var sr statsResponse
			sr.Stats = stats
			sr.RecordsCount = count

			b, err := json.Marshal(sr)

			if err = conn.WriteMessage(messageType, b); err != nil {
				log.WithFields(log.Fields{
					"message": p,
					"error":   err.Error(),
				}).Error("Got error when writing message...")
				return
			}
		}

	}

}

// ImportRecordsHandler - accepts JSON payload and saves it to cache
func (d *DBClient) ImportRecordsHandler(w http.ResponseWriter, req *http.Request) {

	var requests recordedRequests

	defer req.Body.Close()
	body, err := ioutil.ReadAll(req.Body)

	w.Header().Set("Content-Type", "application/json; charset=UTF-8")
	var response messageResponse

	if err != nil {
		// failed to read response body
		log.WithFields(log.Fields{
			"error": err.Error(),
		}).Error("Could not read response body!")
		response.Message = "Bad request. Nothing to import!"
		http.Error(w, "Failed to read request body.", 400)
		return
	}

	err = json.Unmarshal(body, &requests)

	if err != nil {
		w.WriteHeader(422) // can't process this entity
		return
	}

	payloads := requests.Data
	if len(payloads) > 0 {
		for _, pl := range payloads {
			bts, err := pl.Encode()
			if err != nil {
				log.WithFields(log.Fields{
					"error": err.Error(),
				}).Error("Failed to encode payload")
			} else {
				// recalculating request hash and storing it in database
				r := RequestContainer{Details: pl.Request}
				d.Cache.Set([]byte(r.Hash()), bts)
			}
		}
		response.Message = fmt.Sprintf("%d requests imported successfully", len(payloads))
	} else {
		response.Message = "Bad request. Nothing to import!"
		w.WriteHeader(400)
	}

	b, err := json.Marshal(response)
	w.Write(b)

}

// DeleteAllRecordsHandler - deletes all captured requests
func (d *DBClient) DeleteAllRecordsHandler(w http.ResponseWriter, req *http.Request) {
	err := d.Cache.DeleteBucket(d.Cache.RequestsBucket)

	w.Header().Set("Content-Type", "application/json")

	var response messageResponse
	if err != nil {
		if err.Error() == "bucket not found" {
			response.Message = fmt.Sprintf("No records found")
			w.WriteHeader(200)
		} else {
			response.Message = fmt.Sprintf("Something went wrong: %s", err.Error())
			w.WriteHeader(500)
		}
	} else {
		response.Message = "Proxy cache deleted successfuly"
		w.WriteHeader(200)
	}
	b, err := json.Marshal(response)

	w.Write(b)
	return
}

// CurrentStateHandler returns current state
func (d *DBClient) CurrentStateHandler(w http.ResponseWriter, req *http.Request) {
	var resp stateRequest
	resp.Mode = d.Cfg.GetMode()
	resp.Destination = d.Cfg.Destination

	b, _ := json.Marshal(resp)
	w.Header().Set("Content-Type", "application/json; charset=UTF-8")
	w.Write(b)
}

// StateHandler handles current proxy state
func (d *DBClient) StateHandler(w http.ResponseWriter, r *http.Request) {
	var sr stateRequest

	// this is mainly for testing, since when you create
	if r.Body == nil {
		r.Body = ioutil.NopCloser(bytes.NewBuffer([]byte("")))
	}

	defer r.Body.Close()
	body, err := ioutil.ReadAll(r.Body)

	if err != nil {
		// failed to read response body
		log.WithFields(log.Fields{
			"error": err.Error(),
		}).Error("Could not read response body!")
		http.Error(w, "Failed to read request body.", 400)
		return
	}

	err = json.Unmarshal(body, &sr)

	if err != nil {
		w.Header().Set("Content-Type", "application/json; charset=UTF-8")
		w.WriteHeader(400) // can't process this entity
		return
	}

	availableModes := map[string]bool{
		"virtualize": true,
		"capture":    true,
		"modify":     true,
		"synthesize": true,
	}

	if !availableModes[sr.Mode] {
		log.WithFields(log.Fields{
			"suppliedMode": sr.Mode,
		}).Error("Wrong mode found, can't change state")
		http.Error(w, "Bad mode supplied, available modes: virtualize, capture, modify, synthesize.", 400)
		return
	}

	log.WithFields(log.Fields{
		"newState": sr.Mode,
		"body":     string(body),
	}).Info("Handling state change request!")

	// setting new state
	d.Cfg.SetMode(sr.Mode)

	var resp stateRequest
	resp.Mode = d.Cfg.GetMode()
	resp.Destination = d.Cfg.Destination
	b, _ := json.Marshal(resp)
	w.Header().Set("Content-Type", "application/json; charset=UTF-8")
	w.Write(b)

}
<commit_msg>since executable binary is now in cmd/hoverfly/ - adjusting static files directory during development<commit_after>package hoverfly

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"time"

	// static assets
	_ "github.com/SpectoLabs/hoverfly/statik"
	"github.com/rakyll/statik/fs"

	log "github.com/Sirupsen/logrus"
	"github.com/codegangsta/negroni"
	"github.com/go-zoo/bone"
	"github.com/gorilla/websocket"
	"github.com/meatballhat/negroni-logrus"
)

// recordedRequests struct encapsulates payload data
type recordedRequests struct {
	Data []Payload `json:"data"`
}

type recordsCount struct {
	Count int `json:"count"`
}

type statsResponse struct {
	Stats        Stats `json:"stats"`
	RecordsCount int   `json:"recordsCount"`
}

type stateRequest struct {
	Mode        string `json:"mode"`
	Destination string `json:"destination"`
}

type messageResponse struct {
	Message string `json:"message"`
}

// StartAdminInterface - starts admin interface web server
func (d *DBClient) StartAdminInterface() {
	go func() {
		// starting admin interface
		mux := getBoneRouter(*d)
		n := negroni.Classic()

		logLevel := log.WarnLevel

		if d.Cfg.Verbose {
			logLevel = log.DebugLevel
		}

		n.Use(negronilogrus.NewCustomMiddleware(logLevel, &log.JSONFormatter{}, "admin"))
		n.UseHandler(mux)

		// admin interface starting message
		log.WithFields(log.Fields{
			"AdminPort": d.Cfg.AdminPort,
		}).Info("Admin interface is starting...")

		n.Run(fmt.Sprintf(":%s", d.Cfg.AdminPort))
	}()
}

// getBoneRouter returns mux for admin interface
func getBoneRouter(d DBClient) *bone.Mux {
	mux := bone.New()

	mux.Get("/records", http.HandlerFunc(d.AllRecordsHandler))
	mux.Delete("/records", http.HandlerFunc(d.DeleteAllRecordsHandler))
	mux.Post("/records", http.HandlerFunc(d.ImportRecordsHandler))

	mux.Get("/count", http.HandlerFunc(d.RecordsCount))
	mux.Get("/stats", http.HandlerFunc(d.StatsHandler))
	mux.Get("/statsws", http.HandlerFunc(d.StatsWSHandler))

	mux.Get("/state", http.HandlerFunc(d.CurrentStateHandler))
	mux.Post("/state", http.HandlerFunc(d.StateHandler))

	if d.Cfg.Development {
		// since hoverfly is not started from cmd/hoverfly/hoverfly
		// we have to target to that directory
		log.Warn("Hoverfly is serving files from /static/dist instead of statik binary!")
		mux.Handle("/*", http.FileServer(http.Dir("../../static/dist")))
	} else {
		// preparing static assets for embedded admin
		statikFS, err := fs.New()

		if err != nil {
			log.WithFields(log.Fields{
				"Error": err.Error(),
			}).Error("Failed to load statikFS, admin UI might not work :(")
		}

		mux.Handle("/*", http.FileServer(statikFS))
	}

	return mux
}

// AllRecordsHandler returns JSON content type http response
func (d *DBClient) AllRecordsHandler(w http.ResponseWriter, req *http.Request) {
	records, err := d.Cache.GetAllRequests()

	if err == nil {

		w.Header().Set("Content-Type", "application/json")

		var response recordedRequests
		response.Data = records
		b, err := json.Marshal(response)

		if err != nil {
			log.Error(err)
			http.Error(w, err.Error(), http.StatusInternalServerError)
		} else {
			w.Write(b)
			return
		}
	} else {
		log.WithFields(log.Fields{
			"Error": err.Error(),
		}).Error("Failed to get data from cache!")

		w.Header().Set("Content-Type", "application/json; charset=UTF-8")
		w.WriteHeader(500) // can't process this entity
		return
	}
}

// RecordsCount returns number of captured requests as a JSON payload
func (d *DBClient) RecordsCount(w http.ResponseWriter, req *http.Request) {
	count, err := d.Cache.RecordsCount()

	if err == nil {

		w.Header().Set("Content-Type", "application/json")

		var response recordsCount
		response.Count = count
		b, err := json.Marshal(response)

		if err != nil {
			log.Error(err)
			http.Error(w, err.Error(), http.StatusInternalServerError)
		} else {
			w.Write(b)
			return
		}
	} else {
		log.WithFields(log.Fields{
			"Error": err.Error(),
		}).Error("Failed to get data from cache!")

		w.Header().Set("Content-Type", "application/json; charset=UTF-8")
		w.WriteHeader(500) // can't process this entity
		return
	}
}

// StatsHandler - returns current stats about Hoverfly (request counts, record count)
func (d *DBClient) StatsHandler(w http.ResponseWriter, req *http.Request) {
	stats := d.Counter.Flush()

	count, err := d.Cache.RecordsCount()

	if err != nil {
		log.Error(err)
		http.Error(w, err.Error(), http.StatusInternalServerError)
	}

	var sr statsResponse
	sr.Stats = stats
	sr.RecordsCount = count

	w.Header().Set("Content-Type", "application/json")

	b, err := json.Marshal(sr)

	if err != nil {
		log.Error(err)
		http.Error(w, err.Error(), http.StatusInternalServerError)
	} else {
		w.Write(b)
		return
	}

}

var upgrader = websocket.Upgrader{
	ReadBufferSize:  1024,
	WriteBufferSize: 1024,
	CheckOrigin: func(r *http.Request) bool {
		return true
	},
}

// StatsWSHandler - returns current stats about Hoverfly (request counts, record count) through the websocket
func (d *DBClient) StatsWSHandler(w http.ResponseWriter, r *http.Request) {
	conn, err := upgrader.Upgrade(w, r, nil)
	if err != nil {
		log.Println(err)
		return
	}

	for {
		messageType, p, err := conn.ReadMessage()
		if err != nil {
			return
		}
		log.WithFields(log.Fields{
			"message": string(p),
		}).Info("Got message...")

		for _ = range time.Tick(1 * time.Second) {

			count, err := d.Cache.RecordsCount()

			if err != nil {
				log.WithFields(log.Fields{
					"message": p,
					"error":   err.Error(),
				}).Error("got error while trying to get records count")
				return
			}

			stats := d.Counter.Flush()

			var sr statsResponse
			sr.Stats = stats
			sr.RecordsCount = count

			b, err := json.Marshal(sr)

			if err = conn.WriteMessage(messageType, b); err != nil {
				log.WithFields(log.Fields{
					"message": p,
					"error":   err.Error(),
				}).Error("Got error when writing message...")
				return
			}
		}

	}

}

// ImportRecordsHandler - accepts JSON payload and saves it to cache
func (d *DBClient) ImportRecordsHandler(w http.ResponseWriter, req *http.Request) {

	var requests recordedRequests

	defer req.Body.Close()
	body, err := ioutil.ReadAll(req.Body)

	w.Header().Set("Content-Type", "application/json; charset=UTF-8")
	var response messageResponse

	if err != nil {
		// failed to read response body
		log.WithFields(log.Fields{
			"error": err.Error(),
		}).Error("Could not read response body!")
		response.Message = "Bad request. Nothing to import!"
		http.Error(w, "Failed to read request body.", 400)
		return
	}

	err = json.Unmarshal(body, &requests)

	if err != nil {
		w.WriteHeader(422) // can't process this entity
		return
	}

	payloads := requests.Data
	if len(payloads) > 0 {
		for _, pl := range payloads {
			bts, err := pl.Encode()
			if err != nil {
				log.WithFields(log.Fields{
					"error": err.Error(),
				}).Error("Failed to encode payload")
			} else {
				// recalculating request hash and storing it in database
				r := RequestContainer{Details: pl.Request}
				d.Cache.Set([]byte(r.Hash()), bts)
			}
		}
		response.Message = fmt.Sprintf("%d requests imported successfully", len(payloads))
	} else {
		response.Message = "Bad request. Nothing to import!"
		w.WriteHeader(400)
	}

	b, err := json.Marshal(response)
	w.Write(b)

}

// DeleteAllRecordsHandler - deletes all captured requests
func (d *DBClient) DeleteAllRecordsHandler(w http.ResponseWriter, req *http.Request) {
	err := d.Cache.DeleteBucket(d.Cache.RequestsBucket)

	w.Header().Set("Content-Type", "application/json")

	var response messageResponse
	if err != nil {
		if err.Error() == "bucket not found" {
			response.Message = fmt.Sprintf("No records found")
			w.WriteHeader(200)
		} else {
			response.Message = fmt.Sprintf("Something went wrong: %s", err.Error())
			w.WriteHeader(500)
		}
	} else {
		response.Message = "Proxy cache deleted successfuly"
		w.WriteHeader(200)
	}
	b, err := json.Marshal(response)

	w.Write(b)
	return
}

// CurrentStateHandler returns current state
func (d *DBClient) CurrentStateHandler(w http.ResponseWriter, req *http.Request) {
	var resp stateRequest
	resp.Mode = d.Cfg.GetMode()
	resp.Destination = d.Cfg.Destination

	b, _ := json.Marshal(resp)
	w.Header().Set("Content-Type", "application/json; charset=UTF-8")
	w.Write(b)
}

// StateHandler handles current proxy state
func (d *DBClient) StateHandler(w http.ResponseWriter, r *http.Request) {
	var sr stateRequest

	// this is mainly for testing, since when you create
	if r.Body == nil {
		r.Body = ioutil.NopCloser(bytes.NewBuffer([]byte("")))
	}

	defer r.Body.Close()
	body, err := ioutil.ReadAll(r.Body)

	if err != nil {
		// failed to read response body
		log.WithFields(log.Fields{
			"error": err.Error(),
		}).Error("Could not read response body!")
		http.Error(w, "Failed to read request body.", 400)
		return
	}

	err = json.Unmarshal(body, &sr)

	if err != nil {
		w.Header().Set("Content-Type", "application/json; charset=UTF-8")
		w.WriteHeader(400) // can't process this entity
		return
	}

	availableModes := map[string]bool{
		"virtualize": true,
		"capture":    true,
		"modify":     true,
		"synthesize": true,
	}

	if !availableModes[sr.Mode] {
		log.WithFields(log.Fields{
			"suppliedMode": sr.Mode,
		}).Error("Wrong mode found, can't change state")
		http.Error(w, "Bad mode supplied, available modes: virtualize, capture, modify, synthesize.", 400)
		return
	}

	log.WithFields(log.Fields{
		"newState": sr.Mode,
		"body":     string(body),
	}).Info("Handling state change request!")

	// setting new state
	d.Cfg.SetMode(sr.Mode)

	var resp stateRequest
	resp.Mode = d.Cfg.GetMode()
	resp.Destination = d.Cfg.Destination
	b, _ := json.Marshal(resp)
	w.Header().Set("Content-Type", "application/json; charset=UTF-8")
	w.Write(b)

}
<|endoftext|>


<commit_before>// +build integration

package riak

import (
	"bytes"
	rpb_riak "github.com/basho-labs/riak-go-client/rpb/riak"
	"github.com/golang/protobuf/proto"
	"io"
	"net"
	"reflect"
	"testing"
	"time"
)

func TestSuccessfulConnection(t *testing.T) {
	ln, err := net.Listen("tcp4", "127.0.0.1:1337")
	if err != nil {
		t.Error(err)
	}
	defer ln.Close()

	sawConnection := false

	go func() {
		c, err := ln.Accept()
		defer c.Close()
		if err != nil {
			t.Error(err.Error())
		}
		sawConnection = true
	}()

	addr, err := net.ResolveTCPAddr("tcp4", "127.0.0.1:1337")
	if err != nil {
		t.Error(err.Error())
	}

	opts := &connectionOptions{
		remoteAddress: addr,
	}

	conn, err := newConnection(opts)
	if err != nil {
		t.Error(err)
	}

	if err := conn.connect(); err != nil {
		t.Error(err)
	}

	if err := conn.close(); err != nil {
		t.Error(err)
	}

	if !sawConnection {
		t.Error("did not connect")
	}
}

func TestConnectionClosed(t *testing.T) {
	ln, err := net.Listen("tcp", "127.0.0.1:1338")
	if err != nil {
		t.Error(err)
	}
	defer ln.Close()

	go func() {
		c, err := ln.Accept()
		if err != nil {
			t.Error(err)
		}
		if err := c.Close(); err != nil {
			t.Error(err)
		}
	}()

	addr, err := net.ResolveTCPAddr("tcp4", "127.0.0.1:1338")
	if err != nil {
		t.Error(err.Error())
	}

	opts := &connectionOptions{
		remoteAddress: addr,
	}

	conn, err := newConnection(opts)
	if err != nil {
		t.Error(err)
	}

	if err := conn.connect(); err != nil {
		t.Error("unexpected error in connect", err)
	} else {
		if err := ln.Close(); err != nil {
			t.Error(err)
		}
		cmd := &PingCommand{}
		if err := conn.execute(cmd); err != nil {
			if operr, ok := err.(*net.OpError); ok {
				t.Log("op error", operr, operr.Op)
			} else if err == io.EOF {
				t.Log("saw EOF")
			} else {
				t.Errorf("expected to see net.OpError or io.EOF, but got '%s' (type: %v)", err.Error(), reflect.TypeOf(err))
			}
		} else {
			t.Error("expected error in execute")
		}
	}
}

func TestConnectionTimeout(t *testing.T) {
	addr, err := net.ResolveTCPAddr("tcp4", "127.0.0.1:65535")
	if err != nil {
		t.Error(err.Error())
	}

	opts := &connectionOptions{
		remoteAddress:  addr,
		connectTimeout: time.Millisecond * 150,
	}

	if conn, err := newConnection(opts); err == nil {
		if err := conn.connect(); err == nil {
			t.Error("expected to see timeout error")
		} else {
			if neterr, ok := err.(net.Error); ok && neterr.Timeout() {
				t.Log("timeout error", neterr)
			} else {
				t.Error("expected to see timeout error")
			}
		}
	} else {
		t.Error(err)
	}
}

func TestHealthCheckFail(t *testing.T) {
	ln, err := net.Listen("tcp", "127.0.0.1:1339")
	if err != nil {
		t.Error(err)
	}
	defer ln.Close()

	go func() {
		c, err := ln.Accept()
		if err != nil {
			t.Error(err)
		}
		defer c.Close()

		var errcode uint32 = 1
		errmsg := bytes.NewBufferString("this is an error")
		rpbErr := &rpb_riak.RpbErrorResp{
			Errcode: &errcode,
			Errmsg:  errmsg.Bytes(),
		}

		encoded, err := proto.Marshal(rpbErr)
		if err != nil {
			t.Error(err)
		}

		data := buildRiakMessage(rpbCode_RpbErrorResp, encoded)
		count, err := c.Write(data)
		if err != nil {
			t.Error(err)
		}
		if count != len(data) {
			t.Errorf("expected to write %v bytes, wrote %v bytes", len(data), count)
		}
	}()

	addr, err := net.ResolveTCPAddr("tcp4", "127.0.0.1:1339")
	if err != nil {
		t.Error(err.Error())
	}

	opts := &connectionOptions{
		remoteAddress:  addr,
		connectTimeout: thirtySeconds,
		healthCheck:    &PingCommand{},
	}

	if conn, err := newConnection(opts); err == nil {
		if err := conn.connect(); err == nil {
			t.Error("expected to see error")
		} else {
			if riakError, ok := err.(Error); ok == true {
				if expected, actual := "riak error - errcode '1', errmsg 'this is an error'", riakError.Error(); expected != actual {
					t.Errorf("expected %v, got %v", expected, actual)
				}
			} else {
				t.Error("expected to see Riak error, got:", err)
			}
		}
	} else {
		t.Error(err)
	}
}

func TestHealthCheckSuccess(t *testing.T) {
	ln, err := net.Listen("tcp", "127.0.0.1:1340")
	if err != nil {
		t.Error(err)
	}
	defer ln.Close()

	go func() {
		c, err := ln.Accept()
		defer c.Close()
		if err != nil {
			t.Error(err)
		}
		writePingResp(t, c)
	}()

	addr, err := net.ResolveTCPAddr("tcp4", "127.0.0.1:1340")
	if err != nil {
		t.Error(err.Error())
	}

	opts := &connectionOptions{
		remoteAddress:  addr,
		connectTimeout: thirtySeconds,
		healthCheck:    &PingCommand{},
	}

	if conn, err := newConnection(opts); err == nil {
		if err := conn.connect(); err != nil {
			t.Error("unexpected error:", err)
		}
	} else {
		t.Error("unexpected error:", err)
	}
}
<commit_msg>Fix integration test<commit_after>// +build integration

package riak

import (
	"bytes"
	rpb_riak "github.com/basho-labs/riak-go-client/rpb/riak"
	"github.com/golang/protobuf/proto"
	"io"
	"net"
	"reflect"
	"testing"
	"time"
)

func TestSuccessfulConnection(t *testing.T) {
	ln, err := net.Listen("tcp4", "127.0.0.1:1337")
	if err != nil {
		t.Error(err)
	}
	defer ln.Close()

	sawConnection := false

	go func() {
		c, err := ln.Accept()
		defer c.Close()
		if err != nil {
			t.Error(err.Error())
		}
		sawConnection = true
	}()

	addr, err := net.ResolveTCPAddr("tcp4", "127.0.0.1:1337")
	if err != nil {
		t.Error(err.Error())
	}

	opts := &connectionOptions{
		remoteAddress: addr,
	}

	conn, err := newConnection(opts)
	if err != nil {
		t.Error(err)
	}

	if err := conn.connect(); err != nil {
		t.Error(err)
	}

	if err := conn.close(); err != nil {
		t.Error(err)
	}

	if !sawConnection {
		t.Error("did not connect")
	}
}

func TestConnectionClosed(t *testing.T) {
	ln, err := net.Listen("tcp", "127.0.0.1:1338")
	if err != nil {
		t.Error(err)
	}
	defer ln.Close()

	go func() {
		c, err := ln.Accept()
		if err != nil {
			t.Error(err)
		}
		if err := c.Close(); err != nil {
			t.Error(err)
		}
	}()

	addr, err := net.ResolveTCPAddr("tcp4", "127.0.0.1:1338")
	if err != nil {
		t.Error(err.Error())
	}

	opts := &connectionOptions{
		remoteAddress: addr,
	}

	conn, err := newConnection(opts)
	if err != nil {
		t.Error(err)
	}

	if err := conn.connect(); err != nil {
		t.Error("unexpected error in connect", err)
	} else {
		if err := ln.Close(); err != nil {
			t.Error(err)
		}
		cmd := &PingCommand{}
		if err := conn.execute(cmd); err != nil {
			if operr, ok := err.(*net.OpError); ok {
				t.Log("op error", operr, operr.Op)
			} else if err == io.EOF {
				t.Log("saw EOF")
			} else {
				t.Errorf("expected to see net.OpError or io.EOF, but got '%s' (type: %v)", err.Error(), reflect.TypeOf(err))
			}
		} else {
			t.Error("expected error in execute")
		}
	}
}

func TestConnectionTimeout(t *testing.T) {
	addr, err := net.ResolveTCPAddr("tcp4", "10.255.255.1:65535")
	if err != nil {
		t.Error(err.Error())
	}

	opts := &connectionOptions{
		remoteAddress:  addr,
		connectTimeout: time.Millisecond * 150,
	}

	if conn, err := newConnection(opts); err == nil {
		if err := conn.connect(); err == nil {
			t.Error("expected to see timeout error")
		} else {
			if neterr, ok := err.(net.Error); ok && neterr.Timeout() {
				t.Log("timeout error", neterr)
			} else {
				t.Errorf("expected to see timeout error, but got '%s' (type: %v)", err.Error(), reflect.TypeOf(err))
			}
		}
	} else {
		t.Error(err)
	}
}

func TestHealthCheckFail(t *testing.T) {
	ln, err := net.Listen("tcp", "127.0.0.1:1339")
	if err != nil {
		t.Error(err)
	}
	defer ln.Close()

	go func() {
		c, err := ln.Accept()
		if err != nil {
			t.Error(err)
		}
		defer c.Close()

		var errcode uint32 = 1
		errmsg := bytes.NewBufferString("this is an error")
		rpbErr := &rpb_riak.RpbErrorResp{
			Errcode: &errcode,
			Errmsg:  errmsg.Bytes(),
		}

		encoded, err := proto.Marshal(rpbErr)
		if err != nil {
			t.Error(err)
		}

		data := buildRiakMessage(rpbCode_RpbErrorResp, encoded)
		count, err := c.Write(data)
		if err != nil {
			t.Error(err)
		}
		if count != len(data) {
			t.Errorf("expected to write %v bytes, wrote %v bytes", len(data), count)
		}
	}()

	addr, err := net.ResolveTCPAddr("tcp4", "127.0.0.1:1339")
	if err != nil {
		t.Error(err.Error())
	}

	opts := &connectionOptions{
		remoteAddress:  addr,
		connectTimeout: thirtySeconds,
		healthCheck:    &PingCommand{},
	}

	if conn, err := newConnection(opts); err == nil {
		if err := conn.connect(); err == nil {
			t.Error("expected to see error")
		} else {
			if riakError, ok := err.(Error); ok == true {
				if expected, actual := "riak error - errcode '1', errmsg 'this is an error'", riakError.Error(); expected != actual {
					t.Errorf("expected %v, got %v", expected, actual)
				}
			} else {
				t.Error("expected to see Riak error, got:", err)
			}
		}
	} else {
		t.Error(err)
	}
}

func TestHealthCheckSuccess(t *testing.T) {
	ln, err := net.Listen("tcp", "127.0.0.1:1340")
	if err != nil {
		t.Error(err)
	}
	defer ln.Close()

	go func() {
		c, err := ln.Accept()
		defer c.Close()
		if err != nil {
			t.Error(err)
		}
		writePingResp(t, c)
	}()

	addr, err := net.ResolveTCPAddr("tcp4", "127.0.0.1:1340")
	if err != nil {
		t.Error(err.Error())
	}

	opts := &connectionOptions{
		remoteAddress:  addr,
		connectTimeout: thirtySeconds,
		healthCheck:    &PingCommand{},
	}

	if conn, err := newConnection(opts); err == nil {
		if err := conn.connect(); err != nil {
			t.Error("unexpected error:", err)
		}
	} else {
		t.Error("unexpected error:", err)
	}
}
<|endoftext|>


<commit_before>package modules

import (
	// Stdlib
	"fmt"

	// Internal
	"github.com/salsaflow/salsaflow/errs"
	"github.com/salsaflow/salsaflow/modules/common"

	// Internal: modules
	github "github.com/salsaflow/salsaflow/modules/github/releasenotes"
)

// ReleaseNotesManager instantiation --------------------------------------------------

var notesManagerFactories = map[string]common.ReleaseNotesManagerFactory{
	github.Id: github.NewFactory(),
}

func AvailableReleaseNotesManagerKeys() []string {
	keys := make([]string, 0, len(notesManagerFactories))
	for key := range notesManagerFactories {
		keys = append(keys, key)
	}
	return keys
}

func GetReleaseNotesManagerFactory(id string) (common.ReleaseNotesManagerFactory, error) {
	// Choose the release notes manager based on the configuration.
	var task = "Instantiate the selected release notes manager plugin"
	// In case the id is not set, we simply return nil.
	// This means that this module is disabled.
	if id == "" {
		return nil, nil
	}
	factory, ok := notesManagerFactories[id]
	if !ok {
		// Collect the available tracker ids.
		ids := make([]string, 0, len(notesManagerFactories))
		for id := range notesManagerFactories {
			ids = append(ids, id)
		}

		hint := fmt.Sprintf("\nAvailable release notes managers: %v\n\n", ids)
		return nil, errs.NewErrorWithHint(
			task, fmt.Errorf("unknown release notes manager: '%v'", id), hint)
	}

	return factory, nil
}

func GetReleaseNotesManager() (common.ReleaseNotesManager, error) {
	// Load configuration.
	config, err := common.LoadConfig()
	if err != nil && config == nil {
		return nil, err
	}

	// Get the factory.
	factory, err := GetReleaseNotesManagerFactory(config.ReleaseNotesManagerId())
	if err != nil {
		return nil, err
	}

	// Return a new module instance.
	return factory.NewReleaseNotesManager()
}
<commit_msg>modules: Fix GetReleaseNotesManager<commit_after>package modules

import (
	// Stdlib
	"fmt"

	// Internal
	"github.com/salsaflow/salsaflow/errs"
	"github.com/salsaflow/salsaflow/modules/common"

	// Internal: modules
	github "github.com/salsaflow/salsaflow/modules/github/releasenotes"
)

// ReleaseNotesManager instantiation --------------------------------------------------

var notesManagerFactories = map[string]common.ReleaseNotesManagerFactory{
	github.Id: github.NewFactory(),
}

func AvailableReleaseNotesManagerKeys() []string {
	keys := make([]string, 0, len(notesManagerFactories))
	for key := range notesManagerFactories {
		keys = append(keys, key)
	}
	return keys
}

func GetReleaseNotesManagerFactory(id string) (common.ReleaseNotesManagerFactory, error) {
	// Choose the release notes manager based on the configuration.
	var task = "Instantiate the selected release notes manager plugin"
	// In case the id is not set, we simply return nil.
	// This means that this module is disabled.
	if id == "" {
		return nil, nil
	}
	factory, ok := notesManagerFactories[id]
	if !ok {
		// Collect the available tracker ids.
		ids := make([]string, 0, len(notesManagerFactories))
		for id := range notesManagerFactories {
			ids = append(ids, id)
		}

		hint := fmt.Sprintf("\nAvailable release notes managers: %v\n\n", ids)
		return nil, errs.NewErrorWithHint(
			task, fmt.Errorf("unknown release notes manager: '%v'", id), hint)
	}

	return factory, nil
}

func GetReleaseNotesManager() (common.ReleaseNotesManager, error) {
	// Load configuration.
	config, err := common.LoadConfig()
	if err != nil && config == nil {
		return nil, err
	}

	// Get the factory.
	factory, err := GetReleaseNotesManagerFactory(config.ReleaseNotesManagerId())
	if err != nil {
		return nil, err
	}
	if factory == nil {
		return nil, nil
	}

	// Return a new module instance.
	return factory.NewReleaseNotesManager()
}
<|endoftext|>


<commit_before>package configuration

import (
	"encoding/json"
	"fmt"

	"github.com/bborbe/monitoring/check"
	"github.com/bborbe/monitoring/check/http"
	"github.com/bborbe/monitoring/check/tcp"
	"github.com/bborbe/monitoring/node"
)

type Node interface {
	Check() check.Check
	Nodes() []Node
}

type Configuration interface {
	Checks() []check.Check
	Nodes() []node.Node
}

type configuration struct {
}

func New() Configuration {
	return new(configuration)
}

func (c *configuration) Checks() []check.Check {
	list := make([]check.Check, 0)
	list = addChecksToList(c.Nodes(), list)
	return list
}

func addChecksToList(nodes []node.Node, checks []check.Check) []check.Check {
	if nodes != nil {
		for _, n := range nodes {
			if n.Check() != nil {
				checks = append(checks, n.Check())
			}
			checks = addChecksToList(n.Nodes(), checks)
		}
	}
	return checks
}

func (c *configuration) Nodes() []node.Node {
	list := make([]node.Node, 0)
	list = append(list, createNodeInternetAvaiable())
	return list
}

func createNodeInternetAvaiable() node.Node {
	return node.New(tcp.New("www.google.com", 80), createNodeRocketsourceAvaiable())
}

func createNodeRocketsourceAvaiable() node.Node {
	list := make([]node.Node, 0)

	list = append(list, node.New(tcp.New("144.76.187.199", 22)))
	list = append(list, node.New(tcp.New("144.76.187.200", 22)))
	list = append(list, node.New(tcp.New("144.76.187.199", 80)))
	list = append(list, node.New(tcp.New("144.76.187.200", 80)))
	list = append(list, node.New(tcp.New("144.76.187.199", 443)))
	list = append(list, node.New(tcp.New("144.76.187.200", 443)))

	list = append(list, node.New(http.New("http://www.benjamin-borbe.de").ExpectTitle("Benjamin Borbe Fotografie")))
	list = append(list, node.New(http.New("https://www.benjamin-borbe.de").ExpectTitle("Benjamin Borbe Fotografie")))
	list = append(list, node.New(http.New("http://www.benjaminborbe.de").ExpectTitle("Benjamin Borbe Fotografie")))
	list = append(list, node.New(http.New("https://www.benjamin-borbe.de/confluence/").ExpectTitle("Dashboard - Confluence")))

	list = append(list, node.New(http.New("http://www.harteslicht.de").ExpectTitle("www.Harteslicht.com | Fotografieren das Spass macht.")))
	list = append(list, node.New(http.New("http://www.harteslicht.com").ExpectTitle("www.Harteslicht.com | Fotografieren das Spass macht.")))

	list = append(list, node.New(http.New("http://jenkins.benjamin-borbe.de").ExpectTitle("Dashboard [Jenkins]")))
	list = append(list, node.New(http.New("http://kickstart.benjamin-borbe.de").ExpectBody("ks.cfg")))

	list = append(list, node.New(http.New("http://ip.benjamin-borbe.de")))
	list = append(list, node.New(http.New("http://slideshow.benjamin-borbe.de").ExpectBody("go.html")))
	list = append(list, node.New(http.New("http://apt.benjamin-borbe.de/bborbe-unstable/Sources").ExpectContent("bborbe-unstable")))
	list = append(list, node.New(http.New("http://blog.benjamin-borbe.de").ExpectTitle("Benjamin Borbe Fotografie")))

	list = append(list, createBackupStatusNode())

	return node.New(tcp.New("host.rocketsource.de", 22), list...)
}

func createBackupStatusNode() node.Node {
	list := make([]node.Node, 0)
	list = append(list, node.New(http.New("http://backup.pn.benjamin-borbe.de:7777?status=false").AddExpectation(checkBackupJson)))
	return node.New(tcp.New("backup.pn.benjamin-borbe.de", 7777), list...)
}

func checkBackupJson(content []byte) error {
	var data []interface{}
	err := json.Unmarshal(content, &data)
	if err != nil {
		return fmt.Errorf("parse json failed")
	}
	if len(data) > 0 {
		return fmt.Errorf("found false backups")
	}
	return nil
}
<commit_msg>add nexus check<commit_after>package configuration

import (
	"encoding/json"
	"fmt"

	"github.com/bborbe/monitoring/check"
	"github.com/bborbe/monitoring/check/http"
	"github.com/bborbe/monitoring/check/tcp"
	"github.com/bborbe/monitoring/node"
)

type Node interface {
	Check() check.Check
	Nodes() []Node
}

type Configuration interface {
	Checks() []check.Check
	Nodes() []node.Node
}

type configuration struct {
}

func New() Configuration {
	return new(configuration)
}

func (c *configuration) Checks() []check.Check {
	list := make([]check.Check, 0)
	list = addChecksToList(c.Nodes(), list)
	return list
}

func addChecksToList(nodes []node.Node, checks []check.Check) []check.Check {
	if nodes != nil {
		for _, n := range nodes {
			if n.Check() != nil {
				checks = append(checks, n.Check())
			}
			checks = addChecksToList(n.Nodes(), checks)
		}
	}
	return checks
}

func (c *configuration) Nodes() []node.Node {
	list := make([]node.Node, 0)
	list = append(list, createNodeInternetAvaiable())
	return list
}

func createNodeInternetAvaiable() node.Node {
	return node.New(tcp.New("www.google.com", 80), createHmNode(), createRnNode(), createPnNode())
}

func createRnNode() node.Node {
	list := make([]node.Node, 0)

	list = append(list, node.New(tcp.New("144.76.187.199", 22)))
	list = append(list, node.New(tcp.New("144.76.187.200", 22)))
	list = append(list, node.New(tcp.New("144.76.187.199", 80)))
	list = append(list, node.New(tcp.New("144.76.187.200", 80)))
	list = append(list, node.New(tcp.New("144.76.187.199", 443)))
	list = append(list, node.New(tcp.New("144.76.187.200", 443)))

	list = append(list, node.New(http.New("http://www.benjamin-borbe.de").ExpectTitle("Benjamin Borbe Fotografie")))
	list = append(list, node.New(http.New("https://www.benjamin-borbe.de").ExpectTitle("Benjamin Borbe Fotografie")))
	list = append(list, node.New(http.New("http://www.benjaminborbe.de").ExpectTitle("Benjamin Borbe Fotografie")))
	list = append(list, node.New(http.New("https://www.benjamin-borbe.de/confluence/").ExpectTitle("Dashboard - Confluence")))

	list = append(list, node.New(http.New("http://www.harteslicht.de").ExpectTitle("www.Harteslicht.com | Fotografieren das Spass macht.")))
	list = append(list, node.New(http.New("http://www.harteslicht.com").ExpectTitle("www.Harteslicht.com | Fotografieren das Spass macht.")))
	list = append(list, node.New(http.New("http://nexus.benjamin-borbe.de").ExpectTitle("Sonatype Nexus")))
	list = append(list, node.New(http.New("http://nexus.benjamin-borbe.de/nexus/content/groups/public").ExpectTitle("Index of /groups/public")))

	list = append(list, node.New(http.New("http://jenkins.benjamin-borbe.de").ExpectTitle("Dashboard [Jenkins]")))
	list = append(list, node.New(http.New("http://kickstart.benjamin-borbe.de").ExpectBody("ks.cfg")))

	list = append(list, node.New(http.New("http://ip.benjamin-borbe.de")))
	list = append(list, node.New(http.New("http://slideshow.benjamin-borbe.de").ExpectBody("go.html")))
	list = append(list, node.New(http.New("http://apt.benjamin-borbe.de/bborbe-unstable/Sources").ExpectContent("bborbe-unstable")))
	list = append(list, node.New(http.New("http://blog.benjamin-borbe.de").ExpectTitle("Benjamin Borbe Fotografie")))

	return node.New(tcp.New("host.rocketsource.de", 22), list...)
}

func createPnNode() node.Node {
	list := make([]node.Node, 0)
	list = append(list, node.New(http.New("http://backup.pn.benjamin-borbe.de:7777?status=false").AddExpectation(checkBackupJson)))
	return node.New(tcp.New("backup.pn.benjamin-borbe.de", 7777), list...)
}

func createHmNode() node.Node {
	list := make([]node.Node, 0)
	return node.New(tcp.New("home.benjamin-borbe.de", 443), list...)
}

func checkBackupJson(content []byte) error {
	var data []interface{}
	err := json.Unmarshal(content, &data)
	if err != nil {
		return fmt.Errorf("parse json failed")
	}
	if len(data) > 0 {
		return fmt.Errorf("found false backups")
	}
	return nil
}
<|endoftext|>