<commit_before>package projects

import (
	"database/sql"

	database "github.com/ansible-semaphore/semaphore/db"
	"github.com/ansible-semaphore/semaphore/models"
	"github.com/ansible-semaphore/semaphore/util"
	"github.com/gin-gonic/gin"
	"github.com/masterminds/squirrel"
)

func EnvironmentMiddleware(c *gin.Context) {
	project := c.MustGet("project").(models.Project)
	envID, err := util.GetIntParam("environment_id", c)
	if err != nil {
		return
	}

	query, args, _ := squirrel.Select("*").
		From("project__environment").
		Where("project_id=?", project.ID).
		Where("id=?", envID).
		ToSql()

	var env models.Environment
	if err := database.Mysql.SelectOne(&env, query, args...); err != nil {
		if err == sql.ErrNoRows {
			c.AbortWithStatus(404)
			return
		}

		panic(err)
	}

	c.Set("environment", env)
	c.Next()
}

func GetEnvironment(c *gin.Context) {
	project := c.MustGet("project").(models.Project)
	var env []models.Environment

	q := squirrel.Select("*").
		From("project__environment").
		Where("project_id=?", project.ID)

	query, args, _ := q.ToSql()

	if _, err := database.Mysql.Select(&env, query, args...); err != nil {
		panic(err)
	}

	c.JSON(200, env)
}

func UpdateEnvironment(c *gin.Context) {
	oldEnv := c.MustGet("environment").(models.Environment)
	var env models.Environment
	if err := c.Bind(&env); err != nil {
		return
	}

	if _, err := database.Mysql.Exec("update project__environment set name=?, json=? where id=?", env.Name, env.JSON, oldEnv.ID); err != nil {
		panic(err)
	}

	c.AbortWithStatus(204)
}

func AddEnvironment(c *gin.Context) {
	project := c.MustGet("project").(models.Project)
	var env models.Environment

	if err := c.Bind(&env); err != nil {
		return
	}

	res, err := database.Mysql.Exec("insert into project__environment set project_id=?, name=?, json=?, password=?", project.ID, env.Name, env.JSON, env.Password)
	if err != nil {
		panic(err)
	}

	insertID, _ := res.LastInsertId()
	insertIDInt := int(insertID)
	objType := "environment"

	desc := "Environment " + env.Name + " created"
	if err := (models.Event{
		ProjectID:   &project.ID,
		ObjectType:  &objType,
		ObjectID:    &insertIDInt,
		Description: &desc,
	}.Insert()); err != nil {
		panic(err)
	}

	c.AbortWithStatus(204)
}

func RemoveEnvironment(c *gin.Context) {
	env := c.MustGet("environment").(models.Environment)

	templatesC, err := database.Mysql.SelectInt("select count(1) from project__template where project_id=? and environment_id=?", env.ProjectID, env.ID)
	if err != nil {
		panic(err)
	}

	if templatesC > 0 {
		if len(c.Query("setRemoved")) == 0 {
			c.JSON(400, map[string]interface{}{
				"error": "Environment is in use by one or more templates",
				"inUse": true,
			})

			return
		}

		if _, err := database.Mysql.Exec("update project__environment set removed=1 where id=?", env.ID); err != nil {
			panic(err)
		}

		c.AbortWithStatus(204)
		return
	}

	if _, err := database.Mysql.Exec("delete from project__environment where id=?", env.ID); err != nil {
		panic(err)
	}

	desc := "Environment " + env.Name + " deleted"
	if err := (models.Event{
		ProjectID:   &env.ProjectID,
		Description: &desc,
	}.Insert()); err != nil {
		panic(err)
	}

	c.AbortWithStatus(204)
}
<commit_msg>Add JSON validation in environment model<commit_after>package projects

import (
	"database/sql"

	database "github.com/ansible-semaphore/semaphore/db"
	"github.com/ansible-semaphore/semaphore/models"
	"github.com/ansible-semaphore/semaphore/util"
	"github.com/gin-gonic/gin"
	"github.com/masterminds/squirrel"
)

func EnvironmentMiddleware(c *gin.Context) {
	project := c.MustGet("project").(models.Project)
	envID, err := util.GetIntParam("environment_id", c)
	if err != nil {
		return
	}

	query, args, _ := squirrel.Select("*").
		From("project__environment").
		Where("project_id=?", project.ID).
		Where("id=?", envID).
		ToSql()

	var env models.Environment
	if err := database.Mysql.SelectOne(&env, query, args...); err != nil {
		if err == sql.ErrNoRows {
			c.AbortWithStatus(404)
			return
		}

		panic(err)
	}

	c.Set("environment", env)
	c.Next()
}

func GetEnvironment(c *gin.Context) {
	project := c.MustGet("project").(models.Project)
	var env []models.Environment

	q := squirrel.Select("*").
		From("project__environment").
		Where("project_id=?", project.ID)

	query, args, _ := q.ToSql()

	if _, err := database.Mysql.Select(&env, query, args...); err != nil {
		panic(err)
	}

	c.JSON(200, env)
}

func UpdateEnvironment(c *gin.Context) {
	oldEnv := c.MustGet("environment").(models.Environment)
	var env models.Environment
	if err := c.Bind(&env); err != nil {
		return
	}

	var js map[string]interface{}
	if json.Unmarshal([]byte(env.JSON), &js) != nil {
		c.JSON(400, map[string]string{
			"error": "JSON is not valid",
		})
		return
	}

	if _, err := database.Mysql.Exec("update project__environment set name=?, json=? where id=?", env.Name, env.JSON, oldEnv.ID); err != nil {
		panic(err)
	}

	c.AbortWithStatus(204)
}

func AddEnvironment(c *gin.Context) {
	project := c.MustGet("project").(models.Project)
	var env models.Environment

	if err := c.Bind(&env); err != nil {
		return
	}

	var js map[string]interface{}
	if json.Unmarshal([]byte(env.JSON), &js) != nil {
		c.JSON(400, map[string]string{
			"error": "JSON is not valid",
		})
		return
	}

	res, err := database.Mysql.Exec("insert into project__environment set project_id=?, name=?, json=?, password=?", project.ID, env.Name, env.JSON, env.Password)
	if err != nil {
		panic(err)
	}

	insertID, _ := res.LastInsertId()
	insertIDInt := int(insertID)
	objType := "environment"

	desc := "Environment " + env.Name + " created"
	if err := (models.Event{
		ProjectID:   &project.ID,
		ObjectType:  &objType,
		ObjectID:    &insertIDInt,
		Description: &desc,
	}.Insert()); err != nil {
		panic(err)
	}

	c.AbortWithStatus(204)
}

func RemoveEnvironment(c *gin.Context) {
	env := c.MustGet("environment").(models.Environment)

	templatesC, err := database.Mysql.SelectInt("select count(1) from project__template where project_id=? and environment_id=?", env.ProjectID, env.ID)
	if err != nil {
		panic(err)
	}

	if templatesC > 0 {
		if len(c.Query("setRemoved")) == 0 {
			c.JSON(400, map[string]interface{}{
				"error": "Environment is in use by one or more templates",
				"inUse": true,
			})

			return
		}

		if _, err := database.Mysql.Exec("update project__environment set removed=1 where id=?", env.ID); err != nil {
			panic(err)
		}

		c.AbortWithStatus(204)
		return
	}

	if _, err := database.Mysql.Exec("delete from project__environment where id=?", env.ID); err != nil {
		panic(err)
	}

	desc := "Environment " + env.Name + " deleted"
	if err := (models.Event{
		ProjectID:   &env.ProjectID,
		Description: &desc,
	}.Insert()); err != nil {
		panic(err)
	}

	c.AbortWithStatus(204)
}
<|endoftext|>


<commit_before>/*
Copyright 2015 All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"fmt"
	"net/http"
	"net/url"
	"os"
	"sync"

	log "github.com/Sirupsen/logrus"
	"github.com/gambol99/go-oidc/oidc"
	"github.com/gin-gonic/gin"
)

// KeycloakProxy is the server component
type KeycloakProxy struct {
	config *Config
	// the gin service
	router *gin.Engine
	// the oidc provider config
	openIDConfig oidc.ClientConfig
	// the oidc client
	openIDClient *oidc.Client
	// the proxy client
	proxy reverseProxy
	// the upstream endpoint
	upstreamURL *url.URL
}

type reverseProxy interface {
	ServeHTTP(rw http.ResponseWriter, req *http.Request)
}

// newKeycloakProxy create's a new keycloak proxy from configuration
func newKeycloakProxy(cfg *Config) (*KeycloakProxy, error) {
	// step: set the logging level
	if cfg.LogJSONFormat {
		log.SetFormatter(&log.JSONFormatter{})
	}
	if cfg.Verbose {
		log.SetLevel(log.DebugLevel)
	}

	log.Infof("starting %s, version: %s, author: %s", prog, version, author)

	// step: parse the upstream endpoint
	upstreamURL, err := url.Parse(cfg.Upstream)
	if err != nil {
		return nil, err
	}

	// step: initialize the reverse http proxy
	reverse, err := initializeReverseProxy(upstreamURL)
	if err != nil {
		return nil, err
	}

	// step: create a proxy service
	service := &KeycloakProxy{
		config:      cfg,
		proxy:       reverse,
		upstreamURL: upstreamURL,
	}

	// step: initialize the openid client
	if cfg.SkipTokenVerification {
		log.Infof("TESTING ONLY CONFIG - the verification of the token have been disabled")

	} else {
		client, clientCfg, err := initializeOpenID(cfg.DiscoveryURL, cfg.ClientID, cfg.Secret, cfg.RedirectionURL, cfg.Scopes)
		if err != nil {
			return nil, err
		}
		service.openIDConfig = clientCfg
		service.openIDClient = client
	}

	// step: initialize the gin router
	router := gin.New()
	service.router = router

	// step: load the templates
	service.initializeTemplates()
	for _, resource := range cfg.Resources {
		log.Infof("protecting resources under uri: %s", resource)
	}
	for name, value := range cfg.ClaimsMatch {
		log.Infof("the token must container the claim: %s, required: %s", name, value)
	}

	router.Use(gin.Recovery())
	// step: are we logging the traffic?
	if cfg.LogRequests {
		router.Use(service.loggingHandler())
	}

	// step: if gin release production
	if os.Getenv("GIN_MODE") == "release" {
		log.Infof("enabling the security handler for release mode")
		router.Use(service.securityHandler())
	}

	// step: add the routing
	router.GET(authorizationURL, service.oauthAuthorizationHandler)
	router.GET(callbackURL, service.oauthCallbackHandler)
	router.GET(healthURL, service.healthHandler)
	router.Use(service.entryPointHandler(), service.authenticationHandler(), service.admissionHandler())

	return service, nil
}

func (r *KeycloakProxy) abortAll() gin.HandlerFunc {
	return func(cx *gin.Context) {
		fmt.Println("HELLO")
		cx.Next()
		cx.Abort()
	}
}

// initializeTemplates loads the custom template
func (r *KeycloakProxy) initializeTemplates() {
	var list []string

	if r.config.SignInPage != "" {
		log.Debugf("loading the custom sign in page: %s", r.config.SignInPage)
		list = append(list, r.config.SignInPage)
	}
	if r.config.ForbiddenPage != "" {
		log.Debugf("loading the custom sign forbidden page: %s", r.config.ForbiddenPage)
		list = append(list, r.config.ForbiddenPage)
	}

	if len(list) > 0 {
		r.router.LoadHTMLFiles(list...)
	}
}

// Run starts the proxy service
func (r *KeycloakProxy) Run() error {
	go func() {
		log.Infof("keycloak proxy service starting on %s", r.config.Listen)

		var err error
		if r.config.TLSCertificate == "" {
			err = r.router.Run(r.config.Listen)
		} else {
			err = r.router.RunTLS(r.config.Listen, r.config.TLSCertificate, r.config.TLSPrivateKey)
		}
		if err != nil {
			log.WithFields(log.Fields{"error": err.Error()}).Fatalf("failed to start the service")
		}
	}()

	return nil
}

// redirectToURL redirects the user and aborts the context
func (r KeycloakProxy) redirectToURL(url string, cx *gin.Context) {
	cx.Redirect(http.StatusTemporaryRedirect, url)
	cx.Abort()
}

// accessForbidden redirects the user to the forbidden page
func (r KeycloakProxy) accessForbidden(cx *gin.Context) {
	// step: do we have a custom forbidden page
	if r.config.hasForbiddenPage() {
		cx.HTML(http.StatusForbidden, r.config.ForbiddenPage, r.config.TagData)
		cx.Abort()
		return
	}

	cx.AbortWithStatus(http.StatusForbidden)
	cx.Abort()
}

// redirectToAuthorization redirects the user to authorization handler
func (r KeycloakProxy) redirectToAuthorization(cx *gin.Context) {
	// step: add a state referrer to the authorization page
	authQuery := fmt.Sprintf("?state=%s", cx.Request.URL.String())

	// step: if verification is switched off, we can't authorization
	if r.config.SkipTokenVerification {
		log.Errorf("refusing to redirection to authorization endpoint, skip token verification switched on")
		r.accessForbidden(cx)
		return
	}

	r.redirectToURL(authorizationURL+authQuery, cx)
}

// tryUpdateConnection attempt to upgrade the connection to a http pdy stream
func (r *KeycloakProxy) tryUpdateConnection(cx *gin.Context) error {
	// step: dial the endpoint
	tlsConn, err := tryDialEndpoint(r.upstreamURL)
	if err != nil {
		return err
	}
	defer tlsConn.Close()

	// step: we need to hijack the underlining client connection
	clientConn, _, err := cx.Writer.(http.Hijacker).Hijack()
	if err != nil {

	}
	defer clientConn.Close()

	// step: write the request to upstream
	if err = cx.Request.Write(tlsConn); err != nil {
		return err
	}

	// step: copy the date between client and upstream endpoint
	var wg sync.WaitGroup
	wg.Add(2)
	go transferBytes(tlsConn, clientConn, &wg)
	go transferBytes(clientConn, tlsConn, &wg)
	wg.Wait()

	return nil
}
<commit_msg>- removing the debug method, no longer required<commit_after>/*
Copyright 2015 All rights reserved.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"fmt"
	"net/http"
	"net/url"
	"os"
	"sync"

	log "github.com/Sirupsen/logrus"
	"github.com/gambol99/go-oidc/oidc"
	"github.com/gin-gonic/gin"
)

// KeycloakProxy is the server component
type KeycloakProxy struct {
	config *Config
	// the gin service
	router *gin.Engine
	// the oidc provider config
	openIDConfig oidc.ClientConfig
	// the oidc client
	openIDClient *oidc.Client
	// the proxy client
	proxy reverseProxy
	// the upstream endpoint
	upstreamURL *url.URL
}

type reverseProxy interface {
	ServeHTTP(rw http.ResponseWriter, req *http.Request)
}

// newKeycloakProxy create's a new keycloak proxy from configuration
func newKeycloakProxy(cfg *Config) (*KeycloakProxy, error) {
	// step: set the logging level
	if cfg.LogJSONFormat {
		log.SetFormatter(&log.JSONFormatter{})
	}
	if cfg.Verbose {
		log.SetLevel(log.DebugLevel)
	}

	log.Infof("starting %s, version: %s, author: %s", prog, version, author)

	// step: parse the upstream endpoint
	upstreamURL, err := url.Parse(cfg.Upstream)
	if err != nil {
		return nil, err
	}

	// step: initialize the reverse http proxy
	reverse, err := initializeReverseProxy(upstreamURL)
	if err != nil {
		return nil, err
	}

	// step: create a proxy service
	service := &KeycloakProxy{
		config:      cfg,
		proxy:       reverse,
		upstreamURL: upstreamURL,
	}

	// step: initialize the openid client
	if cfg.SkipTokenVerification {
		log.Infof("TESTING ONLY CONFIG - the verification of the token have been disabled")

	} else {
		client, clientCfg, err := initializeOpenID(cfg.DiscoveryURL, cfg.ClientID, cfg.Secret, cfg.RedirectionURL, cfg.Scopes)
		if err != nil {
			return nil, err
		}
		service.openIDConfig = clientCfg
		service.openIDClient = client
	}

	// step: initialize the gin router
	router := gin.New()
	service.router = router

	// step: load the templates
	service.initializeTemplates()
	for _, resource := range cfg.Resources {
		log.Infof("protecting resources under uri: %s", resource)
	}
	for name, value := range cfg.ClaimsMatch {
		log.Infof("the token must container the claim: %s, required: %s", name, value)
	}

	router.Use(gin.Recovery())
	// step: are we logging the traffic?
	if cfg.LogRequests {
		router.Use(service.loggingHandler())
	}

	// step: if gin release production
	if os.Getenv("GIN_MODE") == "release" {
		log.Infof("enabling the security handler for release mode")
		router.Use(service.securityHandler())
	}

	// step: add the routing
	router.GET(authorizationURL, service.oauthAuthorizationHandler)
	router.GET(callbackURL, service.oauthCallbackHandler)
	router.GET(healthURL, service.healthHandler)
	router.Use(service.entryPointHandler(), service.authenticationHandler(), service.admissionHandler())

	return service, nil
}

// initializeTemplates loads the custom template
func (r *KeycloakProxy) initializeTemplates() {
	var list []string

	if r.config.SignInPage != "" {
		log.Debugf("loading the custom sign in page: %s", r.config.SignInPage)
		list = append(list, r.config.SignInPage)
	}
	if r.config.ForbiddenPage != "" {
		log.Debugf("loading the custom sign forbidden page: %s", r.config.ForbiddenPage)
		list = append(list, r.config.ForbiddenPage)
	}

	if len(list) > 0 {
		r.router.LoadHTMLFiles(list...)
	}
}

// Run starts the proxy service
func (r *KeycloakProxy) Run() error {
	go func() {
		log.Infof("keycloak proxy service starting on %s", r.config.Listen)

		var err error
		if r.config.TLSCertificate == "" {
			err = r.router.Run(r.config.Listen)
		} else {
			err = r.router.RunTLS(r.config.Listen, r.config.TLSCertificate, r.config.TLSPrivateKey)
		}
		if err != nil {
			log.WithFields(log.Fields{"error": err.Error()}).Fatalf("failed to start the service")
		}
	}()

	return nil
}

// redirectToURL redirects the user and aborts the context
func (r KeycloakProxy) redirectToURL(url string, cx *gin.Context) {
	cx.Redirect(http.StatusTemporaryRedirect, url)
	cx.Abort()
}

// accessForbidden redirects the user to the forbidden page
func (r KeycloakProxy) accessForbidden(cx *gin.Context) {
	// step: do we have a custom forbidden page
	if r.config.hasForbiddenPage() {
		cx.HTML(http.StatusForbidden, r.config.ForbiddenPage, r.config.TagData)
		cx.Abort()
		return
	}

	cx.AbortWithStatus(http.StatusForbidden)
	cx.Abort()
}

// redirectToAuthorization redirects the user to authorization handler
func (r KeycloakProxy) redirectToAuthorization(cx *gin.Context) {
	// step: add a state referrer to the authorization page
	authQuery := fmt.Sprintf("?state=%s", cx.Request.URL.String())

	// step: if verification is switched off, we can't authorization
	if r.config.SkipTokenVerification {
		log.Errorf("refusing to redirection to authorization endpoint, skip token verification switched on")
		r.accessForbidden(cx)
		return
	}

	r.redirectToURL(authorizationURL+authQuery, cx)
}

// tryUpdateConnection attempt to upgrade the connection to a http pdy stream
func (r *KeycloakProxy) tryUpdateConnection(cx *gin.Context) error {
	// step: dial the endpoint
	tlsConn, err := tryDialEndpoint(r.upstreamURL)
	if err != nil {
		return err
	}
	defer tlsConn.Close()

	// step: we need to hijack the underlining client connection
	clientConn, _, err := cx.Writer.(http.Hijacker).Hijack()
	if err != nil {

	}
	defer clientConn.Close()

	// step: write the request to upstream
	if err = cx.Request.Write(tlsConn); err != nil {
		return err
	}

	// step: copy the date between client and upstream endpoint
	var wg sync.WaitGroup
	wg.Add(2)
	go transferBytes(tlsConn, clientConn, &wg)
	go transferBytes(clientConn, tlsConn, &wg)
	wg.Wait()

	return nil
}
<|endoftext|>


<commit_before>package delayd

import (
	"log"
	"os"
	"path/filepath"
	"sync"
	"time"

	"github.com/streadway/amqp"
)

// a generous 60 seconds to apply raft commands
const raftMaxTime = time.Duration(60) * time.Second
const routingKey = "delayd"

// Server is the delayd server. It handles the server lifecycle (startup, clean shutdown)
type Server struct {
	sender     Sender
	receiver   Receiver
	raft       *Raft
	timer      *Timer
	shutdownCh chan bool
	leader     bool
	mu         sync.Mutex
}

// NewServer initialize Server instance.
func NewServer(c Config) (*Server, error) {
	if len(c.LogDir) != 0 {
		logFile := filepath.Join(c.LogDir, "delayd.log")
		logOutput, err := os.OpenFile(logFile, os.O_CREATE|os.O_RDWR|os.O_APPEND, 0666)
		if err != nil {
			return nil, err
		}
		log.SetOutput(logOutput)
	}

	receiver, err := NewAMQPReceiver(c.AMQP, routingKey)
	if err != nil {
		return nil, err
	}

	sender, err := NewAMQPSender(c.AMQP.URL)
	if err != nil {
		return nil, err
	}

	raft, err := NewRaft(c.Raft, c.DataDir, c.LogDir)
	if err != nil {
		return nil, err
	}

	return &Server{
		sender:     sender,
		receiver:   receiver,
		raft:       raft,
		shutdownCh: make(chan bool),
	}, nil
}

// Run starts server and begins its main loop.
func (s *Server) Run() {
	Info("server: starting delayd")

	s.timer = NewTimer(s.timerSend)

	go s.observeLeaderChanges()
	go s.observeNextTime()

	for {
		msg, ok := <-s.receiver.MessageCh()
		entry := msg.Entry
		// XXX cleanup needed here before exit
		if !ok {
			continue
		}

		Debug("server: got new request entry:", entry)
		b, err := entry.ToBytes()
		if err != nil {
			Error("server: error encoding entry: ", err)
			continue
		}

		if err := s.raft.Add(b, raftMaxTime); err != nil {
			Error("server: failed to add: ", err)
			msg.Nack()
			continue
		}

		msg.Ack()
	}
}

// Stop shuts down the Server cleanly. Order of the Close calls is important.
func (s *Server) Stop() {
	Info("server: shutting down gracefully.")

	// stop triggering new changes to the FSM
	s.receiver.Close()
	s.timer.Stop()

	close(s.shutdownCh)

	// with no FSM stimulus, we don't need to send.
	s.sender.Close()

	s.raft.Close()

	Info("server: terminated.")
}

func (s *Server) resetTimer() {
	ok, t, err := s.raft.fsm.store.NextTime()
	if err != nil {
		Fatal("server: could not read initial send time from storage:", err)
	}
	if ok {
		s.timer.Reset(t, true)
	}
}

// Listen for changes to raft for when we transition to and from the leader state
// and react accordingly.
func (s *Server) observeLeaderChanges() {
	for isLeader := range s.raft.LeaderCh() {
		s.mu.Lock()
		s.leader = isLeader
		s.mu.Unlock()

		if isLeader {
			Debug("server: became raft leader")
			s.resetTimer()
			err := s.receiver.Start()
			if err != nil {
				Fatal("server: error while starting receiver:", err)
			}
		} else {
			Debug("server: lost raft leadership")
			s.timer.Pause()
			err := s.receiver.Pause()
			if err != nil {
				Fatal("server: error while starting receiver:", err)
			}
		}
	}
}

// Listen to storage for next time changes on entry commits
func (s *Server) observeNextTime() {
	for {
		select {
		case <-s.shutdownCh:
			return
		case t := <-s.raft.fsm.store.C:
			s.mu.Lock()
			if s.leader {
				Debug("server: got time: ", t)
				s.timer.Reset(t, false)
			}
			s.mu.Unlock()
		}
	}
}

func (s *Server) timerSend(t time.Time) {
	// FIXME: In the case of error, we don't remove a log from raft log.
	uuids, entries, err := s.raft.fsm.store.Get(t)
	if err != nil {
		Fatal("server: could not read entries from db:", err)
	}

	if len(entries) > 1 {
		Infof("server: sending %d entries", len(entries))
	}

	for i, e := range entries {
		if err := s.sender.Send(e); err != nil {
			if err, ok := err.(*amqp.Error); ok && err.Code == 504 {
				// error 504 code means that the exchange we were trying
				// to send on didnt exist.  In the case of delayd this usually
				// means that a consumer didn't set up the exchange they wish
				// to be notified on. We do not attempt to make this for them,
				// as we don't know what exchange options they would want, we
				// simply drop this message, other errors are fatal
				Warnf("server: channel/connection not set up for exchange `%s`, message will be deleted", e.Target, err)
			} else {
				Fatal("server: could not send entry:", err)
			}
		}

		if err := s.raft.Remove(uuids[i], raftMaxTime); err != nil {
			// This node is no longer the leader. give up on other amqp sends,
			// and scheduling the next emission
			Warnf("server: lost raft leadership during remove. AMQP send will be a duplicate. uuid=%x", uuids[i])
			break
		}
	}

	// ensure everyone is up to date
	Debug("server: syncing raft after send.")
	err = s.raft.SyncAll()
	if err != nil {
		Warn("server: lost raft leadership during sync after send.")
		return
	}

	Debug("server: synced raft after send.")
}
<commit_msg>server: Use err in if<commit_after>package delayd

import (
	"log"
	"os"
	"path/filepath"
	"sync"
	"time"

	"github.com/streadway/amqp"
)

// a generous 60 seconds to apply raft commands
const raftMaxTime = time.Duration(60) * time.Second
const routingKey = "delayd"

// Server is the delayd server. It handles the server lifecycle (startup, clean shutdown)
type Server struct {
	sender     Sender
	receiver   Receiver
	raft       *Raft
	timer      *Timer
	shutdownCh chan bool
	leader     bool
	mu         sync.Mutex
}

// NewServer initialize Server instance.
func NewServer(c Config) (*Server, error) {
	if len(c.LogDir) != 0 {
		logFile := filepath.Join(c.LogDir, "delayd.log")
		logOutput, err := os.OpenFile(logFile, os.O_CREATE|os.O_RDWR|os.O_APPEND, 0666)
		if err != nil {
			return nil, err
		}
		log.SetOutput(logOutput)
	}

	receiver, err := NewAMQPReceiver(c.AMQP, routingKey)
	if err != nil {
		return nil, err
	}

	sender, err := NewAMQPSender(c.AMQP.URL)
	if err != nil {
		return nil, err
	}

	raft, err := NewRaft(c.Raft, c.DataDir, c.LogDir)
	if err != nil {
		return nil, err
	}

	return &Server{
		sender:     sender,
		receiver:   receiver,
		raft:       raft,
		shutdownCh: make(chan bool),
	}, nil
}

// Run starts server and begins its main loop.
func (s *Server) Run() {
	Info("server: starting delayd")

	s.timer = NewTimer(s.timerSend)

	go s.observeLeaderChanges()
	go s.observeNextTime()

	for {
		msg, ok := <-s.receiver.MessageCh()
		entry := msg.Entry
		// XXX cleanup needed here before exit
		if !ok {
			continue
		}

		Debug("server: got new request entry:", entry)
		b, err := entry.ToBytes()
		if err != nil {
			Error("server: error encoding entry: ", err)
			continue
		}

		if err := s.raft.Add(b, raftMaxTime); err != nil {
			Error("server: failed to add: ", err)
			msg.Nack()
			continue
		}

		msg.Ack()
	}
}

// Stop shuts down the Server cleanly. Order of the Close calls is important.
func (s *Server) Stop() {
	Info("server: shutting down gracefully.")

	// stop triggering new changes to the FSM
	s.receiver.Close()
	s.timer.Stop()

	close(s.shutdownCh)

	// with no FSM stimulus, we don't need to send.
	s.sender.Close()

	s.raft.Close()

	Info("server: terminated.")
}

func (s *Server) resetTimer() {
	ok, t, err := s.raft.fsm.store.NextTime()
	if err != nil {
		Fatal("server: could not read initial send time from storage:", err)
	}
	if ok {
		s.timer.Reset(t, true)
	}
}

// Listen for changes to raft for when we transition to and from the leader state
// and react accordingly.
func (s *Server) observeLeaderChanges() {
	for isLeader := range s.raft.LeaderCh() {
		s.mu.Lock()
		s.leader = isLeader
		s.mu.Unlock()

		if isLeader {
			Debug("server: became raft leader")
			s.resetTimer()
			if err := s.receiver.Start(); err != nil {
				Fatal("server: error while starting receiver:", err)
			}
		} else {
			Debug("server: lost raft leadership")
			s.timer.Pause()
			if err := s.receiver.Pause(); err != nil {
				Fatal("server: error while starting receiver:", err)
			}
		}
	}
}

// Listen to storage for next time changes on entry commits
func (s *Server) observeNextTime() {
	for {
		select {
		case <-s.shutdownCh:
			return
		case t := <-s.raft.fsm.store.C:
			s.mu.Lock()
			if s.leader {
				Debug("server: got time: ", t)
				s.timer.Reset(t, false)
			}
			s.mu.Unlock()
		}
	}
}

func (s *Server) timerSend(t time.Time) {
	// FIXME: In the case of error, we don't remove a log from raft log.
	uuids, entries, err := s.raft.fsm.store.Get(t)
	if err != nil {
		Fatal("server: could not read entries from db:", err)
	}

	if len(entries) > 1 {
		Infof("server: sending %d entries", len(entries))
	}

	for i, e := range entries {
		if err := s.sender.Send(e); err != nil {
			if err, ok := err.(*amqp.Error); ok && err.Code == 504 {
				// error 504 code means that the exchange we were trying
				// to send on didnt exist.  In the case of delayd this usually
				// means that a consumer didn't set up the exchange they wish
				// to be notified on. We do not attempt to make this for them,
				// as we don't know what exchange options they would want, we
				// simply drop this message, other errors are fatal
				Warnf("server: channel/connection not set up for exchange `%s`, message will be deleted", e.Target, err)
			} else {
				Fatal("server: could not send entry:", err)
			}
		}

		if err := s.raft.Remove(uuids[i], raftMaxTime); err != nil {
			// This node is no longer the leader. give up on other amqp sends,
			// and scheduling the next emission
			Warnf("server: lost raft leadership during remove. AMQP send will be a duplicate. uuid=%x", uuids[i])
			break
		}
	}

	// ensure everyone is up to date
	Debug("server: syncing raft after send.")
	err = s.raft.SyncAll()
	if err != nil {
		Warn("server: lost raft leadership during sync after send.")
		return
	}

	Debug("server: synced raft after send.")
}
<|endoftext|>


<commit_before>package main

import (
	"errors"
	"fmt"
	"net/http"
	"runtime"
	"time"

	"gopkg.in/fsnotify.v1"

	"github.com/Sirupsen/logrus"
	"github.com/dysolution/airstrike"
	"github.com/dysolution/espsdk"
	"github.com/spf13/viper"
)

func serve() {
	http.HandleFunc("/", mw(status))
	http.HandleFunc("/config", mw(showConfig))
	http.HandleFunc("/example", mw(showExampleConfig))

	http.HandleFunc("/attack", mw(attack))
	http.HandleFunc("/cease_fire", mw(pause))
	http.HandleFunc("/pause", mw(pause))

	http.HandleFunc("/faster", mw(faster))
	http.HandleFunc("/slower", mw(slower))
	http.HandleFunc("/backoff", mw(backoff))
	http.HandleFunc("/speedup", mw(speedup))
	// TODO http.HandleFunc("/refresh_token", refreshToken)

	go beginMission(&cfg.Mission, reporter)

	tcpSocket := ":8080"
	log.Infof("listening on %s", tcpSocket)
	http.ListenAndServe(tcpSocket, nil)
}

// runs in a goroutine
func beginMission(mission *airstrike.Mission, reporter airstrike.Reporter) {
	desc := "beginMission"

	// set up the reporter for logging and console output
	logFields := make(chan map[string]interface{})
	newThreshold := make(chan time.Duration)

	reporter.ThresholdReceiver = newThreshold

	go reporter.Run(logFields)

	logCh <- map[string]interface{}{
		"severity": "info",
		"source":   desc,
		"interval": mission.Interval,
	}

	viper.WatchConfig()
	viper.OnConfigChange(func(e fsnotify.Event) {
		fmt.Println("Config file changed:", e.Name)
		fmt.Println(viper.GetString("foo"))
	})

	for {
		select {
		case mission.Enabled = <-toggle:
		case d := <-intervalDelta:
			setInterval(logFields, d, mission)
			reporter.ThresholdReceiver <- time.Duration(mission.Interval) * time.Millisecond
		default:
		}

		if mission.Enabled {
			logFields <- map[string]interface{}{
				"msg":    "conducting raid",
				"source": desc,
			}

			config.Conduct(log, espsdk.APIInvariant, logFields)
			mission.RaidCount++
			pauseBetweenRaids(logFields, mission)
		}
	}
}

func pauseBetweenRaids(logFields chan map[string]interface{}, mission *airstrike.Mission) {
	desc := "pauseBetweenRaids"
	if mission.Interval <= 0 {
		logFields <- map[string]interface{}{
			"severity": "error",
			"err":      errors.New("non-positive interval; defaulting to 1s"),
			"interval": mission.Interval,
			"source":   desc,
		}
		time.Sleep(1000 * time.Millisecond)
	} else {
		logFields <- map[string]interface{}{
			"msg":      "sleeping",
			"interval": mission.Interval,
			"source":   desc,
		}

		time.Sleep(time.Duration(mission.Interval) * time.Millisecond)

		logFields <- map[string]interface{}{
			"msg":      "waking up",
			"interval": mission.Interval,
			"source":   desc,
		}
	}
}

func logger(ch chan map[string]interface{}, log *logrus.Logger) {
	for {
		select {
		case fields := <-ch:
			desc := fmt.Sprintf("photobomb.logger (gr: %v)", runtime.NumGoroutine())
			switch fields["severity"] {
			case "INFO", "info":
				log.WithFields(fields).Info(desc)
			case "WARN", "warn":
				log.WithFields(fields).Warn(desc)
			default:
				log.WithFields(fields).Debug(desc)
			}
		default:
		}
	}
}
<commit_msg>fix: decrease CPU load while paused<commit_after>package main

import (
	"errors"
	"fmt"
	"net/http"
	"runtime"
	"time"

	"gopkg.in/fsnotify.v1"

	"github.com/Sirupsen/logrus"
	"github.com/dysolution/airstrike"
	"github.com/dysolution/espsdk"
	"github.com/spf13/viper"
)

func serve() {
	http.HandleFunc("/", mw(status))
	http.HandleFunc("/config", mw(showConfig))
	http.HandleFunc("/example", mw(showExampleConfig))

	http.HandleFunc("/attack", mw(attack))
	http.HandleFunc("/cease_fire", mw(pause))
	http.HandleFunc("/pause", mw(pause))

	http.HandleFunc("/faster", mw(faster))
	http.HandleFunc("/slower", mw(slower))
	http.HandleFunc("/backoff", mw(backoff))
	http.HandleFunc("/speedup", mw(speedup))
	// TODO http.HandleFunc("/refresh_token", refreshToken)

	go beginMission(&cfg.Mission, reporter)

	tcpSocket := ":8080"
	log.Infof("listening on %s", tcpSocket)
	http.ListenAndServe(tcpSocket, nil)
}

// runs in a goroutine
func beginMission(mission *airstrike.Mission, reporter airstrike.Reporter) {
	desc := "beginMission"

	// set up the reporter for logging and console output
	logFields := make(chan map[string]interface{})
	newThreshold := make(chan time.Duration)

	reporter.ThresholdReceiver = newThreshold

	go reporter.Run(logFields)

	logCh <- map[string]interface{}{
		"severity": "info",
		"source":   desc,
		"interval": mission.Interval,
	}

	viper.WatchConfig()
	viper.OnConfigChange(func(e fsnotify.Event) {
		fmt.Println("Config file changed:", e.Name)
		fmt.Println(viper.GetString("foo"))
	})

	for {
		select {
		case mission.Enabled = <-toggle:
		case d := <-intervalDelta:
			setInterval(logFields, d, mission)
			reporter.ThresholdReceiver <- time.Duration(mission.Interval) * time.Millisecond
		default:
		}

		if mission.Enabled {
			logFields <- map[string]interface{}{
				"msg":    "conducting raid",
				"source": desc,
			}

			config.Conduct(log, espsdk.APIInvariant, logFields)
			mission.RaidCount++
		}
		pauseBetweenRaids(logFields, mission)
	}
}

func pauseBetweenRaids(logFields chan map[string]interface{}, mission *airstrike.Mission) {
	desc := "pauseBetweenRaids"
	if mission.Interval <= 0 {
		logFields <- map[string]interface{}{
			"severity": "error",
			"err":      errors.New("non-positive interval; defaulting to 1s"),
			"interval": mission.Interval,
			"source":   desc,
		}
		time.Sleep(1000 * time.Millisecond)
	} else {
		// logFields <- map[string]interface{}{
		// 	"msg":      "sleeping",
		// 	"interval": mission.Interval,
		// 	"source":   desc,
		// }

		time.Sleep(time.Duration(mission.Interval) * time.Millisecond)

		// logFields <- map[string]interface{}{
		// 	"msg":      "waking up",
		// 	"interval": mission.Interval,
		// 	"source":   desc,
		// }
	}
}

func logger(ch chan map[string]interface{}, log *logrus.Logger) {
	for {
		select {
		case fields := <-ch:
			desc := fmt.Sprintf("photobomb.logger (gr: %v)", runtime.NumGoroutine())
			switch fields["severity"] {
			case "INFO", "info":
				log.WithFields(fields).Info(desc)
			case "WARN", "warn":
				log.WithFields(fields).Warn(desc)
			default:
				log.WithFields(fields).Debug(desc)
			}
		default:
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"crypto/md5"
	"encoding/hex"
	"fmt"
	"github.com/codegangsta/negroni"
	"github.com/fzzy/radix/extra/pool"
	"github.com/fzzy/radix/redis"
	"github.com/gorilla/mux"
	"github.com/unrolled/render"
	"net/http"
	"os"
	"runtime"
	// "io"
	// "time"
	// "log"
	"strconv"
	"strings"
	// "regexp"
)

func cleanWords(query string) []string {
	// r := regexp.MustCompile(regexp.QuoteMeta("[]()"))
	// query = r.ReplaceAllString(query, " ")
	query = strings.Replace(query, "\"", " ", -1)
	query = strings.Replace(query, "[", " ", -1)
	query = strings.Replace(query, "]", " ", -1)
	query = strings.Replace(query, "(", " ", -1)
	query = strings.Replace(query, ")", " ", -1)
	query = strings.Replace(query, "?", " ", -1)
	query = strings.Replace(query, "!", " ", -1)
	query = strings.Replace(query, ",", " ", -1)
	query = strings.Replace(query, "-", " ", -1)
	query = strings.Replace(query, ":", " ", -1)
	query = strings.Replace(query, ";", " ", -1)
	query = strings.Replace(query, ",", " ", -1)
	query = strings.Replace(query, "'", " ", -1)
	split := strings.Fields(strings.Trim(query, " "))
	terms := make([]string, len(split))
	for i, term := range split {
		terms[i] = strings.ToLower(strings.Trim(strings.Trim(term, " "), "."))
	}
	return terms
}

func encodeString(str string) string {
	h := md5.New()
	h.Write([]byte(str))
	return hex.EncodeToString(h.Sum(nil))[0:8]
}

func getPrefixes(title string) []string {
	var prefixes []string
	for _, word := range cleanWords(title) {
		for i := 1; i <= len(word); i++ {
			prefixes = append(prefixes, word[0:i])
		}
	}
	return prefixes
}

// func QueryScore(terms []string, title) float32 {
// 	return 1.0
// }

func errHndlr(err error) {
	if err != nil {
		fmt.Println("error:", err)
		panic(err)
		os.Exit(1)
	}
}

func IndexHandler(w http.ResponseWriter, req *http.Request) {
	// this assumes there's a `templates/index.tmpl` file
	renderer.HTML(w, http.StatusOK, "index", nil)
}

/* POSTING new stuff in */
func UpdateHandler(w http.ResponseWriter, req *http.Request) {
	values := make(map[string]string)

	required := []string{"domain", "url", "title"}
	errors := make(map[string]string)
	var value string
	for _, key := range required {
		value = strings.Trim(req.FormValue(key), " ")
		values[key] = value
		if value == "" {
			errors[key] = "Missing"
		}
	}

	optional := []string{"groups", "popularity"}
	for _, key := range optional {
		value = strings.Trim(req.FormValue(key), " ")
		if key == "groups" {
			// values[key] = strings.Split(value, ",")
			values[key] = value
		} else if key == "popularity" {
			if value == "" {
				values[key] = "0"
			} else {
				_, err := strconv.ParseFloat(value, 0)
				if err != nil {
					errors[key] = "Not a number"
				} else {
					values[key] = value
				}
			}
		} else {
			values[key] = value
		}
	}
	// fmt.Println("errors=", errors, len(errors))
	if len(errors) > 0 {
		error := make(map[string]interface{})
		error["error"] = errors
		renderer.JSON(w, http.StatusBadRequest, error)
		return
	}
	// popularity, _ := strconv.ParseInt(values["popularity"], 10, 0)
	popularity, _ := strconv.ParseFloat(values["popularity"], 0)

	encoded := encodeString(values["domain"])
	values["url_encoded"] = encodeString(values["url"])

	c, err := redis_pool.Get()
	errHndlr(err)
	defer redis_pool.Put(c)
	// c.Cmd("FLUSHALL")
	// fmt.Println("CAREFUL! Always flushing the database")
	piped_commands := 0
	for _, prefix := range getPrefixes(values["title"]) {
		c.Append("ZADD", encoded+prefix, popularity, values["url_encoded"])
		piped_commands += 1
	}
	c.Append("HSET", encoded+"$titles", values["url_encoded"], values["title"])
	piped_commands += 1
	c.Append("HSET", encoded+"$urls", values["url_encoded"], values["url"])
	piped_commands += 1
	for i := 1; i <= piped_commands; i++ {
		if err := c.GetReply().Err; err != nil {
			errHndlr(err)
		}
	}

	output := map[string]string{"message": "OK"}
	renderer.JSON(w, http.StatusCreated, output)
}

func FetchHandler(w http.ResponseWriter, req *http.Request) {

	n_str := req.FormValue("n")

	var n int64
	if n_str == "" {
		n = 10
	} else {
		n, err := strconv.ParseInt(n_str, 10, 0)
		if err != nil {
			// error := make(map[string]string{"error": "Not a number"})
			error := map[string]string{"error": "Not a number"}
			renderer.JSON(w, http.StatusBadRequest, error)
			return
		}
		if n <= 0 {
			error := map[string]string{"error": "Number too small"}
			renderer.JSON(w, http.StatusBadRequest, error)
			return
		}
		if n > 100 {
			error := map[string]string{"error": "Number too big"}
			renderer.JSON(w, http.StatusBadRequest, error)
			return
		}
	}

	domain := strings.Trim(req.FormValue("domain"), " ")
	if domain == "" {
		error := map[string]string{"error": "'domain' missing"}
		renderer.JSON(w, http.StatusBadRequest, error)
		return
	}

	encoded := encodeString(domain)
	// fmt.Println(domain, encoded)

	q := strings.Trim(req.FormValue("q"), " ")
	terms := cleanWords(q)

	c, err := redis_pool.Get()
	errHndlr(err)
	defer redis_pool.CarefullyPut(c, &err)

	encoded_terms := make([]string, len(terms))
	for i, term := range terms {
		encoded_terms[i] = encoded + term
	}
	// NOTE! Maybe we don't need the ZINTERSTORE if there's only 1 command
	c.Append("ZINTERSTORE", "$tmp", len(terms), encoded_terms, "AGGREGATE", "max")
	c.Append("ZREVRANGE", "$tmp", 0, n-1, "WITHSCORES")

	c.GetReply() // the ZINTERSTORE
	replies, err := c.GetReply().List()
	// fmt.Println("replies", replies, len(replies))
	errHndlr(err)

	encoded_urls := make([]string, n+1)
	scores := make([]string, n+1)
	evens := 0
	for i, element := range replies {
		if i%2 == 0 {
			encoded_urls[evens] = element
			evens = evens + 1
		} else {
			scores[evens-1] = element
		}
	}
	encoded_urls = encoded_urls[:evens]
	scores = scores[:evens]

	var titles []string
	var urls []string
	if len(encoded_urls) == 0 {
	} else {
		titles, err = c.Cmd("HMGET", encoded+"$titles", encoded_urls).List()
		errHndlr(err)
		urls, err = c.Cmd("HMGET", encoded+"$urls", encoded_urls).List()
		errHndlr(err)
	}
	rows := make([]interface{}, len(titles))
	for i, title := range titles {
		row := make([]string, 2)
		row[0] = urls[i]
		// fmt.Println("scores", scores[i]+ 1000)
		// row[1] = scores[i] * QueryScore(terms, title)
		row[1] = title
		rows[i] = row
	}
	rows = rows[:len(titles)]

	output := make(map[string]interface{})
	output["terms"] = terms
	output["results"] = rows
	// fmt.Println(output)
	w.Header().Set("Access-Control-Allow-Origin", "*")
	renderer.JSON(w, http.StatusOK, output)
}

var (
	renderer   *render.Render
	redis_pool *pool.Pool
	procs      int64 // because that's what ParseInt() gives us
	debug      = true
)

func main() {
	// Figuring out how many processors to use.
	procs_str := os.Getenv("PROCS")
	max_procs := int64(runtime.NumCPU())
	if procs_str == "" {
		procs_str = "1"
	}
	var err error
	if procs_str == "0" {
		procs = int64(runtime.NumCPU())
	} else {
		procs, err = strconv.ParseInt(procs_str, 10, 0)
		errHndlr(err)
	}
	if procs < 0 {
		panic("PROCS < 0")
	} else if procs > max_procs {
		panic(fmt.Sprintf("PROCS > max (%v)", max_procs))
	}
	runtime.GOMAXPROCS(int(procs))

	// Figuring out if this is debug mode or not.
	if os.Getenv("DEBUG") == "" ||
		os.Getenv("DEBUG") == "false" ||
		os.Getenv("DEBUG") == "0" {
		debug = false
	}

	renderer = render.New(render.Options{
		IndentJSON:    debug,
		IsDevelopment: debug,
	})

	fmt.Println("DEBUG?", debug)

	REDIS_URL := os.Getenv("REDIS_URL")
	if REDIS_URL == "" {
		REDIS_URL = "127.0.0.1:6379"
	}
	REDIS_DATABASE := os.Getenv("REDIS_DATABASE")
	if REDIS_DATABASE == "" {
		REDIS_DATABASE = "0"
	}

	df := func(network, addr string) (*redis.Client, error) {
		client, err := redis.Dial(network, addr)
		// fmt.Println("DIaling")
		if err != nil {
			return nil, err
		}
		err = client.Cmd("SELECT", REDIS_DATABASE).Err
		if err != nil {
			return nil, err
		}
		// if err = client.Cmd("AUTH", "SUPERSECRET").Err; err != nil {
		// 	client.Close()
		// 	return nil, err
		// }
		return client, nil
	}

	redis_pool, err = pool.NewCustomPool("tcp", REDIS_URL, int(procs)*10, df)
	errHndlr(err)

	mux := mux.NewRouter()
	mux.HandleFunc("/", IndexHandler).Methods("GET", "HEAD")
	mux.HandleFunc("/v1", FetchHandler).Methods("GET", "HEAD")
	mux.HandleFunc("/v1", UpdateHandler).Methods("POST", "PUT")

	n := negroni.Classic()
	n.UseHandler(mux)
	port := os.Getenv("PORT")
	if port == "" {
		port = "3001"
	}
	n.Run(":" + port)
}
<commit_msg>removed debug printing<commit_after>package main

import (
	"crypto/md5"
	"encoding/hex"
	"fmt"
	"github.com/codegangsta/negroni"
	"github.com/fzzy/radix/extra/pool"
	"github.com/fzzy/radix/redis"
	"github.com/gorilla/mux"
	"github.com/unrolled/render"
	"net/http"
	"os"
	"runtime"
	// "io"
	// "time"
	// "log"
	"strconv"
	"strings"
	// "regexp"
)

func cleanWords(query string) []string {
	// r := regexp.MustCompile(regexp.QuoteMeta("[]()"))
	// query = r.ReplaceAllString(query, " ")
	query = strings.Replace(query, "\"", " ", -1)
	query = strings.Replace(query, "[", " ", -1)
	query = strings.Replace(query, "]", " ", -1)
	query = strings.Replace(query, "(", " ", -1)
	query = strings.Replace(query, ")", " ", -1)
	query = strings.Replace(query, "?", " ", -1)
	query = strings.Replace(query, "!", " ", -1)
	query = strings.Replace(query, ",", " ", -1)
	query = strings.Replace(query, "-", " ", -1)
	query = strings.Replace(query, ":", " ", -1)
	query = strings.Replace(query, ";", " ", -1)
	query = strings.Replace(query, ",", " ", -1)
	query = strings.Replace(query, "'", " ", -1)
	split := strings.Fields(strings.Trim(query, " "))
	terms := make([]string, len(split))
	for i, term := range split {
		terms[i] = strings.ToLower(strings.Trim(strings.Trim(term, " "), "."))
	}
	return terms
}

func encodeString(str string) string {
	h := md5.New()
	h.Write([]byte(str))
	return hex.EncodeToString(h.Sum(nil))[0:8]
}

func getPrefixes(title string) []string {
	var prefixes []string
	for _, word := range cleanWords(title) {
		for i := 1; i <= len(word); i++ {
			prefixes = append(prefixes, word[0:i])
		}
	}
	return prefixes
}

// func QueryScore(terms []string, title) float32 {
// 	return 1.0
// }

func errHndlr(err error) {
	if err != nil {
		fmt.Println("error:", err)
		panic(err)
		os.Exit(1)
	}
}

func IndexHandler(w http.ResponseWriter, req *http.Request) {
	// this assumes there's a `templates/index.tmpl` file
	renderer.HTML(w, http.StatusOK, "index", nil)
}

/* POSTING new stuff in */
func UpdateHandler(w http.ResponseWriter, req *http.Request) {
	values := make(map[string]string)

	required := []string{"domain", "url", "title"}
	errors := make(map[string]string)
	var value string
	for _, key := range required {
		value = strings.Trim(req.FormValue(key), " ")
		values[key] = value
		if value == "" {
			errors[key] = "Missing"
		}
	}

	optional := []string{"groups", "popularity"}
	for _, key := range optional {
		value = strings.Trim(req.FormValue(key), " ")
		if key == "groups" {
			// values[key] = strings.Split(value, ",")
			values[key] = value
		} else if key == "popularity" {
			if value == "" {
				values[key] = "0"
			} else {
				_, err := strconv.ParseFloat(value, 0)
				if err != nil {
					errors[key] = "Not a number"
				} else {
					values[key] = value
				}
			}
		} else {
			values[key] = value
		}
	}
	// fmt.Println("errors=", errors, len(errors))
	if len(errors) > 0 {
		error := make(map[string]interface{})
		error["error"] = errors
		renderer.JSON(w, http.StatusBadRequest, error)
		return
	}
	// popularity, _ := strconv.ParseInt(values["popularity"], 10, 0)
	popularity, _ := strconv.ParseFloat(values["popularity"], 0)

	encoded := encodeString(values["domain"])
	values["url_encoded"] = encodeString(values["url"])

	c, err := redis_pool.Get()
	errHndlr(err)
	defer redis_pool.Put(c)
	// c.Cmd("FLUSHALL")
	// fmt.Println("CAREFUL! Always flushing the database")
	piped_commands := 0
	for _, prefix := range getPrefixes(values["title"]) {
		c.Append("ZADD", encoded+prefix, popularity, values["url_encoded"])
		piped_commands += 1
	}
	c.Append("HSET", encoded+"$titles", values["url_encoded"], values["title"])
	piped_commands += 1
	c.Append("HSET", encoded+"$urls", values["url_encoded"], values["url"])
	piped_commands += 1
	for i := 1; i <= piped_commands; i++ {
		if err := c.GetReply().Err; err != nil {
			errHndlr(err)
		}
	}

	output := map[string]string{"message": "OK"}
	renderer.JSON(w, http.StatusCreated, output)
}

func FetchHandler(w http.ResponseWriter, req *http.Request) {

	n_str := req.FormValue("n")

	var n int64
	if n_str == "" {
		n = 10
	} else {
		n, err := strconv.ParseInt(n_str, 10, 0)
		if err != nil {
			// error := make(map[string]string{"error": "Not a number"})
			error := map[string]string{"error": "Not a number"}
			renderer.JSON(w, http.StatusBadRequest, error)
			return
		}
		if n <= 0 {
			error := map[string]string{"error": "Number too small"}
			renderer.JSON(w, http.StatusBadRequest, error)
			return
		}
		if n > 100 {
			error := map[string]string{"error": "Number too big"}
			renderer.JSON(w, http.StatusBadRequest, error)
			return
		}
	}

	domain := strings.Trim(req.FormValue("domain"), " ")
	if domain == "" {
		error := map[string]string{"error": "'domain' missing"}
		renderer.JSON(w, http.StatusBadRequest, error)
		return
	}

	encoded := encodeString(domain)
	// fmt.Println(domain, encoded)

	q := strings.Trim(req.FormValue("q"), " ")
	terms := cleanWords(q)

	c, err := redis_pool.Get()
	errHndlr(err)
	defer redis_pool.CarefullyPut(c, &err)

	encoded_terms := make([]string, len(terms))
	for i, term := range terms {
		encoded_terms[i] = encoded + term
	}
	// NOTE! Maybe we don't need the ZINTERSTORE if there's only 1 command
	c.Append("ZINTERSTORE", "$tmp", len(terms), encoded_terms, "AGGREGATE", "max")
	c.Append("ZREVRANGE", "$tmp", 0, n-1, "WITHSCORES")

	c.GetReply() // the ZINTERSTORE
	replies, err := c.GetReply().List()
	// fmt.Println("replies", replies, len(replies))
	errHndlr(err)

	encoded_urls := make([]string, n+1)
	scores := make([]string, n+1)
	evens := 0
	for i, element := range replies {
		if i%2 == 0 {
			encoded_urls[evens] = element
			evens = evens + 1
		} else {
			scores[evens-1] = element
		}
	}
	encoded_urls = encoded_urls[:evens]
	scores = scores[:evens]

	var titles []string
	var urls []string
	if len(encoded_urls) == 0 {
	} else {
		titles, err = c.Cmd("HMGET", encoded+"$titles", encoded_urls).List()
		errHndlr(err)
		urls, err = c.Cmd("HMGET", encoded+"$urls", encoded_urls).List()
		errHndlr(err)
	}
	rows := make([]interface{}, len(titles))
	for i, title := range titles {
		row := make([]string, 2)
		row[0] = urls[i]
		// fmt.Println("scores", scores[i]+ 1000)
		// row[1] = scores[i] * QueryScore(terms, title)
		row[1] = title
		rows[i] = row
	}
	rows = rows[:len(titles)]

	output := make(map[string]interface{})
	output["terms"] = terms
	output["results"] = rows
	// fmt.Println(output)
	w.Header().Set("Access-Control-Allow-Origin", "*")
	renderer.JSON(w, http.StatusOK, output)
}

var (
	renderer   *render.Render
	redis_pool *pool.Pool
	procs      int64 // because that's what ParseInt() gives us
	debug      = true
)

func main() {
	// Figuring out how many processors to use.
	procs_str := os.Getenv("PROCS")
	max_procs := int64(runtime.NumCPU())
	if procs_str == "" {
		procs_str = "1"
	}
	var err error
	if procs_str == "0" {
		procs = int64(runtime.NumCPU())
	} else {
		procs, err = strconv.ParseInt(procs_str, 10, 0)
		errHndlr(err)
	}
	if procs < 0 {
		panic("PROCS < 0")
	} else if procs > max_procs {
		panic(fmt.Sprintf("PROCS > max (%v)", max_procs))
	}
	runtime.GOMAXPROCS(int(procs))

	// Figuring out if this is debug mode or not.
	if os.Getenv("DEBUG") == "" ||
		os.Getenv("DEBUG") == "false" ||
		os.Getenv("DEBUG") == "0" {
		debug = false
	}

	renderer = render.New(render.Options{
		IndentJSON:    debug,
		IsDevelopment: debug,
	})

	REDIS_URL := os.Getenv("REDIS_URL")
	if REDIS_URL == "" {
		REDIS_URL = "127.0.0.1:6379"
	}
	REDIS_DATABASE := os.Getenv("REDIS_DATABASE")
	if REDIS_DATABASE == "" {
		REDIS_DATABASE = "0"
	}

	df := func(network, addr string) (*redis.Client, error) {
		client, err := redis.Dial(network, addr)
		// fmt.Println("DIaling")
		if err != nil {
			return nil, err
		}
		err = client.Cmd("SELECT", REDIS_DATABASE).Err
		if err != nil {
			return nil, err
		}
		// if err = client.Cmd("AUTH", "SUPERSECRET").Err; err != nil {
		// 	client.Close()
		// 	return nil, err
		// }
		return client, nil
	}

	redis_pool, err = pool.NewCustomPool("tcp", REDIS_URL, int(procs)*10, df)
	errHndlr(err)

	mux := mux.NewRouter()
	mux.HandleFunc("/", IndexHandler).Methods("GET", "HEAD")
	mux.HandleFunc("/v1", FetchHandler).Methods("GET", "HEAD")
	mux.HandleFunc("/v1", UpdateHandler).Methods("POST", "PUT")

	n := negroni.Classic()
	n.UseHandler(mux)
	port := os.Getenv("PORT")
	if port == "" {
		port = "3001"
	}
	n.Run(":" + port)
}
<|endoftext|>


<commit_before>package main

import (
	"database/sql"
	"flag"
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
	"strconv"
	"time"

	"github.com/cloudfoundry-incubator/galera-healthcheck/config"

	"github.com/cloudfoundry-incubator/cf-lager"
	"github.com/cloudfoundry-incubator/galera-healthcheck/healthcheck"
	"github.com/pivotal-cf-experimental/service-config"
	"github.com/pivotal-golang/lager"

	_ "github.com/go-sql-driver/mysql"
)

var healthchecker *healthcheck.Healthchecker

func handler(w http.ResponseWriter, r *http.Request, logger lager.Logger) {
	result, msg := healthchecker.Check()
	if result {
		w.WriteHeader(http.StatusOK)
	} else {
		w.WriteHeader(http.StatusServiceUnavailable)
	}

	body := fmt.Sprintf("Galera Cluster Node Status: %s", msg)
	fmt.Fprint(w, body)

	logger.Debug(fmt.Sprintf("Healhcheck Response Body: %s", body))
}

func main() {

	serviceConfig := service_config.New()

	flags := flag.NewFlagSet(os.Args[0], flag.ExitOnError)
	pidFile := flags.String("pidFile", "", "Path to create a pid file when the healthcheck server has started")
	statusEndpoint := flags.String("statusEndpoint", "", "Http Endpoint to get healthcheck of server")
	serviceConfig.AddFlags(flags)
	var defaults = config.Config{
		Host:           "0.0.0.0",
		Port:           8080,
		StatusEndpoint: *statusEndpoint,
		DB: config.DBConfig{
			Host:     "0.0.0.0",
			Port:     3306,
			User:     "root",
			Password: "",
		},
		AvailableWhenDonor:    true,
		AvailableWhenReadOnly: false,
	}
	serviceConfig.AddDefaults(defaults)
	cf_lager.AddFlags(flags)

	flags.Parse(os.Args[1:])
	logger, _ := cf_lager.New("Galera Healthcheck")

	logger.Info("Starting galera healthcheck...")

	var config config.Config
	err := serviceConfig.Read(&config)
	if err != nil && err != service_config.NoConfigError {
		logger.Fatal("Failed to read config", err)
	}

	db, err := sql.Open("mysql", fmt.Sprintf("%s:%s@tcp(%s:%d)/", config.DB.User, config.DB.Password, config.DB.Host, config.DB.Port))
	if err != nil {
		// sql.Open may not actually check that the DB is reachable
		err = db.Ping()
	}
	if err != nil {
		logger.Fatal("Failed to open DB connection", err, lager.Data{
			"dbHost": config.DB.Host,
			"dbPort": config.DB.Port,
			"dbUser": config.DB.User,
		})
	}

	logger.Info("Opened DB connection", lager.Data{
		"dbHost": config.DB.Host,
		"dbPort": config.DB.Port,
		"dbUser": config.DB.User,
	})

	healthchecker = healthcheck.New(db, config, logger)
	StatusEndpointHandler := fmt.Sprintf("/%s", *statusEndpoint)
	http.HandleFunc(StatusEndpointHandler, func(w http.ResponseWriter, r *http.Request) {
		handler(w, r, logger)
	})

	address := fmt.Sprintf("%s:%d", config.Host, config.Port)
	url := fmt.Sprintf("http://%s/", address)
	logger.Info("Serving healthcheck endpoint", lager.Data{
		"url": url,
	})

	go func() {
		client := &http.Client{
			Timeout: 10 * time.Second,
		}

		logger.Info("Attempting to GET endpoint...", lager.Data{
			"url": url,
		})

		var resp *http.Response
		retryAttemptsRemaining := 3
		for ; retryAttemptsRemaining > 0; retryAttemptsRemaining-- {
			resp, err = client.Get(url)
			if err != nil {
				logger.Info("GET endpoint failed, retrying...", lager.Data{
					"url": url,
					"err": err,
				})
				time.Sleep(time.Second * 10)
			} else {
				break
			}
		}
		if retryAttemptsRemaining == 0 {
			logger.Fatal(
				"Initialization failed: Coudn't GET endpoint",
				err,
				lager.Data{
					"url":     url,
					"retries": retryAttemptsRemaining,
				})
		}
		logger.Info("GET endpoint succeeded, now accepting connections", lager.Data{
			"url":        url,
			"statusCode": resp.StatusCode,
		})

		defer resp.Body.Close()
		body, err := ioutil.ReadAll(resp.Body)
		if err != nil {
			logger.Fatal("Initialization failed: reading response body", err, lager.Data{
				"url":         url,
				"status-code": resp.StatusCode,
			})
		}
		logger.Info(fmt.Sprintf("Initial Response: %s", body))

		if *pidFile != "" {
			// existence of pid file means the server is running
			pid := os.Getpid()
			err = ioutil.WriteFile(*pidFile, []byte(strconv.Itoa(os.Getpid())), 0644)
			if err != nil {
				logger.Fatal("Failed to write pid file", err, lager.Data{
					"pid":     pid,
					"pidFile": *pidFile,
				})
			}
		}

		// Used by tests to deterministically know that the healthcheck is accepting incoming connections
		logger.Info("Healthcheck Started")
	}()

	server := &http.Server{Addr: address}
	server.ListenAndServe()
}
<commit_msg>Add sequence_number endpoint to galera server<commit_after>package main

import (
	"database/sql"
	"flag"
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
	"strconv"
	"time"

	"github.com/cloudfoundry-incubator/galera-healthcheck/config"
	"github.com/cloudfoundry-incubator/galera-healthcheck/sequence_number"

	"github.com/cloudfoundry-incubator/cf-lager"
	"github.com/cloudfoundry-incubator/galera-healthcheck/healthcheck"
	"github.com/pivotal-cf-experimental/service-config"
	"github.com/pivotal-golang/lager"

	_ "github.com/go-sql-driver/mysql"
)

var healthchecker *healthcheck.Healthchecker
var sequence_number_checker *sequence_number.SequenceNumberchecker

func handler(w http.ResponseWriter, r *http.Request, logger lager.Logger) {
	result, msg := healthchecker.Check()
	if result {
		w.WriteHeader(http.StatusOK)
	} else {
		w.WriteHeader(http.StatusServiceUnavailable)
	}

	body := fmt.Sprintf("Galera Cluster Node Status: %s", msg)
	fmt.Fprint(w, body)

	logger.Debug(fmt.Sprintf("Healhcheck Response Body: %s", body))
}

func main() {

	serviceConfig := service_config.New()

	flags := flag.NewFlagSet(os.Args[0], flag.ExitOnError)
	pidFile := flags.String("pidFile", "", "Path to create a pid file when the healthcheck server has started")
	statusEndpoint := flags.String("statusEndpoint", "", "Http Endpoint to get healthcheck of server")
	serviceConfig.AddFlags(flags)
	var defaults = config.Config{
		Host:           "0.0.0.0",
		Port:           8080,
		StatusEndpoint: *statusEndpoint,
		DB: config.DBConfig{
			Host:     "0.0.0.0",
			Port:     3306,
			User:     "root",
			Password: "",
		},
		AvailableWhenDonor:    true,
		AvailableWhenReadOnly: false,
	}
	serviceConfig.AddDefaults(defaults)
	cf_lager.AddFlags(flags)

	flags.Parse(os.Args[1:])
	logger, _ := cf_lager.New("Galera Healthcheck")

	logger.Info("Starting galera healthcheck...")

	var config config.Config
	err := serviceConfig.Read(&config)
	if err != nil && err != service_config.NoConfigError {
		logger.Fatal("Failed to read config", err)
	}

	db, err := sql.Open("mysql", fmt.Sprintf("%s:%s@tcp(%s:%d)/", config.DB.User, config.DB.Password, config.DB.Host, config.DB.Port))
	if err != nil {
		// sql.Open may not actually check that the DB is reachable
		err = db.Ping()
	}
	if err != nil {
		logger.Fatal("Failed to open DB connection", err, lager.Data{
			"dbHost": config.DB.Host,
			"dbPort": config.DB.Port,
			"dbUser": config.DB.User,
		})
	}

	logger.Info("Opened DB connection", lager.Data{
		"dbHost": config.DB.Host,
		"dbPort": config.DB.Port,
		"dbUser": config.DB.User,
	})

	healthchecker = healthcheck.New(db, config, logger)
	StatusEndpointHandler := fmt.Sprintf("/%s", *statusEndpoint)
	http.HandleFunc(StatusEndpointHandler, func(w http.ResponseWriter, r *http.Request) {
		handler(w, r, logger)
	})

	sequence_number_checker = sequence_number.New(db, config, logger)
	http.HandleFunc("sequence_number", func(w http.ResponseWriter, r *http.Request) {
		handler(w, r, logger)
	})

	address := fmt.Sprintf("%s:%d", config.Host, config.Port)
	url := fmt.Sprintf("http://%s/", address)
	logger.Info("Serving healthcheck endpoint", lager.Data{
		"url": url,
	})

	go func() {
		client := &http.Client{
			Timeout: 10 * time.Second,
		}

		logger.Info("Attempting to GET endpoint...", lager.Data{
			"url": url,
		})

		var resp *http.Response
		retryAttemptsRemaining := 3
		for ; retryAttemptsRemaining > 0; retryAttemptsRemaining-- {
			resp, err = client.Get(url)
			if err != nil {
				logger.Info("GET endpoint failed, retrying...", lager.Data{
					"url": url,
					"err": err,
				})
				time.Sleep(time.Second * 10)
			} else {
				break
			}
		}
		if retryAttemptsRemaining == 0 {
			logger.Fatal(
				"Initialization failed: Coudn't GET endpoint",
				err,
				lager.Data{
					"url":     url,
					"retries": retryAttemptsRemaining,
				})
		}
		logger.Info("GET endpoint succeeded, now accepting connections", lager.Data{
			"url":        url,
			"statusCode": resp.StatusCode,
		})

		defer resp.Body.Close()
		body, err := ioutil.ReadAll(resp.Body)
		if err != nil {
			logger.Fatal("Initialization failed: reading response body", err, lager.Data{
				"url":         url,
				"status-code": resp.StatusCode,
			})
		}
		logger.Info(fmt.Sprintf("Initial Response: %s", body))

		if *pidFile != "" {
			// existence of pid file means the server is running
			pid := os.Getpid()
			err = ioutil.WriteFile(*pidFile, []byte(strconv.Itoa(os.Getpid())), 0644)
			if err != nil {
				logger.Fatal("Failed to write pid file", err, lager.Data{
					"pid":     pid,
					"pidFile": *pidFile,
				})
			}
		}

		// Used by tests to deterministically know that the healthcheck is accepting incoming connections
		logger.Info("Healthcheck Started")
	}()

	server := &http.Server{Addr: address}
	server.ListenAndServe()
}
<|endoftext|>


<commit_before>package devd

import (
	"crypto/tls"
	"fmt"
	"html/template"
	"net"
	"net/http"
	"os"
	"os/signal"
	"regexp"
	"strings"
	"syscall"
	"time"

	"golang.org/x/net/context"

	"github.com/GeertJohan/go.rice"
	"github.com/goji/httpauth"

	"github.com/cortesi/devd/httpctx"
	"github.com/cortesi/devd/inject"
	"github.com/cortesi/devd/livereload"
	"github.com/cortesi/devd/ricetemp"
	"github.com/cortesi/devd/slowdown"
	"github.com/cortesi/devd/timer"
	"github.com/cortesi/termlog"
)

const (
	// Version is the current version of devd
	Version  = "0.4"
	portLow  = 8000
	portHigh = 10000
)

func pickPort(addr string, low int, high int, tls bool) (net.Listener, error) {
	firstTry := 80
	if tls {
		firstTry = 443
	}
	hl, err := net.Listen("tcp", fmt.Sprintf("%v:%d", addr, firstTry))
	if err == nil {
		return hl, nil
	}
	for i := low; i < high; i++ {
		hl, err := net.Listen("tcp", fmt.Sprintf("%v:%d", addr, i))
		if err == nil {
			return hl, nil
		}
	}
	return nil, fmt.Errorf("Could not find open port.")
}

func getTLSConfig(path string) (t *tls.Config, err error) {
	config := &tls.Config{}
	if config.NextProtos == nil {
		config.NextProtos = []string{"http/1.1"}
	}
	config.Certificates = make([]tls.Certificate, 1)
	config.Certificates[0], err = tls.LoadX509KeyPair(path, path)
	if err != nil {
		return nil, err
	}
	return config, nil
}

// This filthy hack works in conjunction with hostPortStrip to restore the
// original request host after mux match.
func revertOriginalHost(r *http.Request) {
	original := r.Header.Get("_devd_original_host")
	if original != "" {
		r.Host = original
		r.Header.Del("_devd_original_host")
	}
}

// We can remove the mangling once this is fixed:
// 		https://github.com/golang/go/issues/10463
func hostPortStrip(next http.Handler) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		host, _, err := net.SplitHostPort(r.Host)
		if err == nil {
			original := r.Host
			r.Host = host
			r.Header.Set("_devd_original_host", original)
		}
		next.ServeHTTP(w, r)
	})
}

func matchStringAny(regexps []*regexp.Regexp, s string) bool {
	for _, r := range regexps {
		if r.MatchString(s) {
			return true
		}
	}
	return false
}

func formatURL(tls bool, httpIP string, port int) string {
	proto := "http"
	if tls {
		proto = "https"
	}
	host := httpIP
	if httpIP == "0.0.0.0" || httpIP == "127.0.0.1" {
		host = "devd.io"
	}
	if port == 443 && tls {
		return fmt.Sprintf("https://%s", host)
	}
	if port == 80 && !tls {
		return fmt.Sprintf("http://%s", host)
	}
	return fmt.Sprintf("%s://%s:%d", proto, host, port)
}

// Credentials is a simple username/password pair
type Credentials struct {
	username string
	password string
}

// CredentialsFromSpec creates a set of credentials from a spec
func CredentialsFromSpec(spec string) (*Credentials, error) {
	parts := strings.SplitN(spec, ":", 2)
	if len(parts) != 2 || parts[0] == "" || parts[1] == "" {
		return nil, fmt.Errorf("Invalid credential spec: %s", spec)
	}
	return &Credentials{parts[0], parts[1]}, nil
}

// Devd represents the devd server options
type Devd struct {
	Routes RouteCollection

	// Shaping
	Latency  int
	DownKbps uint
	UpKbps   uint

	// Livereload and watch static routes
	LivereloadRoutes bool
	// Livereload, but don't watch static routes
	Livereload bool
	WatchPaths []string
	Excludes   []string

	// Logging
	IgnoreLogs []*regexp.Regexp

	// Password protection
	Credentials *Credentials

	lrserver *livereload.Server
}

// WrapHandler wraps an httpctx.Handler in the paraphernalia needed by devd for
// logging, latency, and so forth.
func (dd *Devd) WrapHandler(log termlog.TermLog, next httpctx.Handler) http.Handler {
	h := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		revertOriginalHost(r)
		timr := timer.Timer{}
		sublog := log.Group()
		defer func() {
			timing := termlog.DefaultPalette.Timestamp.SprintFunc()("timing: ")
			sublog.SayAs("timer", timing+timr.String())
			sublog.Done()
		}()
		if matchStringAny(dd.IgnoreLogs, fmt.Sprintf("%s%s", r.URL.Host, r.RequestURI)) {
			sublog.Quiet()
		}
		timr.RequestHeaders()
		time.Sleep(time.Millisecond * time.Duration(dd.Latency))
		sublog.Say("%s %s", r.Method, r.URL)
		LogHeader(sublog, r.Header)
		ctx := timr.NewContext(context.Background())
		ctx = termlog.NewContext(ctx, sublog)
		next.ServeHTTPContext(
			ctx,
			&ResponseLogWriter{Log: sublog, Resp: w, Timer: &timr},
			r,
		)
	})
	return h
}

// HasLivereload tells us if liverload is enabled
func (dd *Devd) HasLivereload() bool {
	if dd.Livereload || dd.LivereloadRoutes || len(dd.WatchPaths) > 0 {
		return true
	}
	return false
}

// AddRoutes adds route specifications to the server
func (dd *Devd) AddRoutes(specs []string) error {
	dd.Routes = make(RouteCollection)
	for _, s := range specs {
		err := dd.Routes.Add(s)
		if err != nil {
			return fmt.Errorf("Invalid route specification: %s", err)
		}
	}
	return nil
}

// AddIgnores adds log ignore patterns to the server
func (dd *Devd) AddIgnores(specs []string) error {
	dd.IgnoreLogs = make([]*regexp.Regexp, 0, 0)
	for _, expr := range specs {
		v, err := regexp.Compile(expr)
		if err != nil {
			return fmt.Errorf("%s", err)
		}
		dd.IgnoreLogs = append(dd.IgnoreLogs, v)
	}
	return nil
}

// HandleNotFound handles pages not found. In particular, this handler is used
// when we have no matching route for a request. This also means it's not
// useful to inject the livereload paraphernalia here.
func HandleNotFound(templates *template.Template) httpctx.Handler {
	return httpctx.HandlerFunc(func(ctx context.Context, w http.ResponseWriter, r *http.Request) {
		w.WriteHeader(http.StatusNotFound)
		err := templates.Lookup("404.html").Execute(w, nil)
		if err != nil {
			logger := termlog.FromContext(ctx)
			logger.Shout("Could not execute template: %s", err)
		}
	})
}

// Router constructs the main Devd router that serves all requests
func (dd *Devd) Router(logger termlog.TermLog, templates *template.Template) (http.Handler, error) {
	mux := http.NewServeMux()
	hasGlobal := false

	ci := inject.CopyInject{}
	if dd.HasLivereload() {
		ci = livereload.Injector
	}

	for match, route := range dd.Routes {
		if match == "/" {
			hasGlobal = true
		}
		handler := dd.WrapHandler(
			logger,
			route.Endpoint.Handler(templates, ci),
		)
		handler = http.StripPrefix(route.Path, handler)
		mux.Handle(match, handler)
	}
	if dd.HasLivereload() {
		lr := livereload.NewServer("livereload", logger)
		mux.Handle(livereload.EndpointPath, lr)
		mux.Handle(livereload.ScriptPath, http.HandlerFunc(lr.ServeScript))
		if dd.LivereloadRoutes {
			err := WatchRoutes(dd.Routes, lr, dd.Excludes, logger)
			if err != nil {
				return nil, fmt.Errorf("Could not watch routes for livereload: %s", err)
			}
		}
		if len(dd.WatchPaths) > 0 {
			err := WatchPaths(dd.WatchPaths, dd.Excludes, lr, logger)
			if err != nil {
				return nil, fmt.Errorf("Could not watch path for livereload: %s", err)
			}
		}
		dd.lrserver = lr
	}
	if !hasGlobal {
		mux.Handle(
			"/",
			dd.WrapHandler(logger, HandleNotFound(templates)),
		)
	}
	var h = http.Handler(mux)
	if dd.Credentials != nil {
		h = httpauth.SimpleBasicAuth(
			dd.Credentials.username, dd.Credentials.password,
		)(h)
	}
	return hostPortStrip(h), nil
}

// Serve starts the devd server. The callback is called with the serving URL
// just before service starts.
func (dd *Devd) Serve(address string, port int, certFile string, logger termlog.TermLog, callback func(string)) error {
	templates, err := ricetemp.MakeTemplates(rice.MustFindBox("templates"))
	if err != nil {
		return fmt.Errorf("Error loading templates: %s", err)
	}
	mux, err := dd.Router(logger, templates)
	if err != nil {
		return err
	}
	var tlsConfig *tls.Config
	var tlsEnabled bool
	if certFile != "" {
		tlsConfig, err = getTLSConfig(certFile)
		if err != nil {
			return fmt.Errorf("Could not load certs: %s", err)
		}
		tlsEnabled = true
	}

	var hl net.Listener
	if port > 0 {
		hl, err = net.Listen("tcp", fmt.Sprintf("%v:%d", address, port))
	} else {
		hl, err = pickPort(address, portLow, portHigh, tlsEnabled)
	}
	if err != nil {
		return err
	}

	if tlsConfig != nil {
		hl = tls.NewListener(hl, tlsConfig)
	}

	hl = slowdown.NewSlowListener(hl, dd.UpKbps*1024, dd.DownKbps*1024)
	url := formatURL(tlsEnabled, address, hl.Addr().(*net.TCPAddr).Port)
	logger.Say("Listening on %s (%s)", url, hl.Addr().String())
	server := &http.Server{Addr: hl.Addr().String(), Handler: mux}
	callback(url)

	if dd.HasLivereload() {
		c := make(chan os.Signal, 1)
		signal.Notify(c, syscall.SIGHUP)
		go func() {
			for {
				<-c
				logger.Say("Received signal - reloading")
				dd.lrserver.Reload([]string{"*"})
			}
		}()
	}

	err = server.Serve(hl)
	logger.Shout("Server stopped: %v", err)
	return nil
}
<commit_msg>Fix path display in logs<commit_after>package devd

import (
	"crypto/tls"
	"fmt"
	"html/template"
	"net"
	"net/http"
	"os"
	"os/signal"
	"regexp"
	"strings"
	"syscall"
	"time"

	"golang.org/x/net/context"

	"github.com/GeertJohan/go.rice"
	"github.com/goji/httpauth"

	"github.com/cortesi/devd/httpctx"
	"github.com/cortesi/devd/inject"
	"github.com/cortesi/devd/livereload"
	"github.com/cortesi/devd/ricetemp"
	"github.com/cortesi/devd/slowdown"
	"github.com/cortesi/devd/timer"
	"github.com/cortesi/termlog"
)

const (
	// Version is the current version of devd
	Version  = "0.4"
	portLow  = 8000
	portHigh = 10000
)

func pickPort(addr string, low int, high int, tls bool) (net.Listener, error) {
	firstTry := 80
	if tls {
		firstTry = 443
	}
	hl, err := net.Listen("tcp", fmt.Sprintf("%v:%d", addr, firstTry))
	if err == nil {
		return hl, nil
	}
	for i := low; i < high; i++ {
		hl, err := net.Listen("tcp", fmt.Sprintf("%v:%d", addr, i))
		if err == nil {
			return hl, nil
		}
	}
	return nil, fmt.Errorf("Could not find open port.")
}

func getTLSConfig(path string) (t *tls.Config, err error) {
	config := &tls.Config{}
	if config.NextProtos == nil {
		config.NextProtos = []string{"http/1.1"}
	}
	config.Certificates = make([]tls.Certificate, 1)
	config.Certificates[0], err = tls.LoadX509KeyPair(path, path)
	if err != nil {
		return nil, err
	}
	return config, nil
}

// This filthy hack works in conjunction with hostPortStrip to restore the
// original request host after mux match.
func revertOriginalHost(r *http.Request) {
	original := r.Header.Get("_devd_original_host")
	if original != "" {
		r.Host = original
		r.Header.Del("_devd_original_host")
	}
}

// We can remove the mangling once this is fixed:
// 		https://github.com/golang/go/issues/10463
func hostPortStrip(next http.Handler) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		host, _, err := net.SplitHostPort(r.Host)
		if err == nil {
			original := r.Host
			r.Host = host
			r.Header.Set("_devd_original_host", original)
		}
		next.ServeHTTP(w, r)
	})
}

func matchStringAny(regexps []*regexp.Regexp, s string) bool {
	for _, r := range regexps {
		if r.MatchString(s) {
			return true
		}
	}
	return false
}

func formatURL(tls bool, httpIP string, port int) string {
	proto := "http"
	if tls {
		proto = "https"
	}
	host := httpIP
	if httpIP == "0.0.0.0" || httpIP == "127.0.0.1" {
		host = "devd.io"
	}
	if port == 443 && tls {
		return fmt.Sprintf("https://%s", host)
	}
	if port == 80 && !tls {
		return fmt.Sprintf("http://%s", host)
	}
	return fmt.Sprintf("%s://%s:%d", proto, host, port)
}

// Credentials is a simple username/password pair
type Credentials struct {
	username string
	password string
}

// CredentialsFromSpec creates a set of credentials from a spec
func CredentialsFromSpec(spec string) (*Credentials, error) {
	parts := strings.SplitN(spec, ":", 2)
	if len(parts) != 2 || parts[0] == "" || parts[1] == "" {
		return nil, fmt.Errorf("Invalid credential spec: %s", spec)
	}
	return &Credentials{parts[0], parts[1]}, nil
}

// Devd represents the devd server options
type Devd struct {
	Routes RouteCollection

	// Shaping
	Latency  int
	DownKbps uint
	UpKbps   uint

	// Livereload and watch static routes
	LivereloadRoutes bool
	// Livereload, but don't watch static routes
	Livereload bool
	WatchPaths []string
	Excludes   []string

	// Logging
	IgnoreLogs []*regexp.Regexp

	// Password protection
	Credentials *Credentials

	lrserver *livereload.Server
}

// WrapHandler wraps an httpctx.Handler in the paraphernalia needed by devd for
// logging, latency, and so forth.
func (dd *Devd) WrapHandler(log termlog.TermLog, next httpctx.Handler) http.Handler {
	h := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		revertOriginalHost(r)
		timr := timer.Timer{}
		sublog := log.Group()
		defer func() {
			timing := termlog.DefaultPalette.Timestamp.SprintFunc()("timing: ")
			sublog.SayAs("timer", timing+timr.String())
			sublog.Done()
		}()
		if matchStringAny(dd.IgnoreLogs, fmt.Sprintf("%s%s", r.URL.Host, r.RequestURI)) {
			sublog.Quiet()
		}
		timr.RequestHeaders()
		time.Sleep(time.Millisecond * time.Duration(dd.Latency))

		dpath := r.URL.String()
		if !strings.HasPrefix(dpath, "/") {
			dpath = "/" + dpath
		}
		sublog.Say("%s %s", r.Method, dpath)
		LogHeader(sublog, r.Header)
		ctx := timr.NewContext(context.Background())
		ctx = termlog.NewContext(ctx, sublog)
		next.ServeHTTPContext(
			ctx,
			&ResponseLogWriter{Log: sublog, Resp: w, Timer: &timr},
			r,
		)
	})
	return h
}

// HasLivereload tells us if liverload is enabled
func (dd *Devd) HasLivereload() bool {
	if dd.Livereload || dd.LivereloadRoutes || len(dd.WatchPaths) > 0 {
		return true
	}
	return false
}

// AddRoutes adds route specifications to the server
func (dd *Devd) AddRoutes(specs []string) error {
	dd.Routes = make(RouteCollection)
	for _, s := range specs {
		err := dd.Routes.Add(s)
		if err != nil {
			return fmt.Errorf("Invalid route specification: %s", err)
		}
	}
	return nil
}

// AddIgnores adds log ignore patterns to the server
func (dd *Devd) AddIgnores(specs []string) error {
	dd.IgnoreLogs = make([]*regexp.Regexp, 0, 0)
	for _, expr := range specs {
		v, err := regexp.Compile(expr)
		if err != nil {
			return fmt.Errorf("%s", err)
		}
		dd.IgnoreLogs = append(dd.IgnoreLogs, v)
	}
	return nil
}

// HandleNotFound handles pages not found. In particular, this handler is used
// when we have no matching route for a request. This also means it's not
// useful to inject the livereload paraphernalia here.
func HandleNotFound(templates *template.Template) httpctx.Handler {
	return httpctx.HandlerFunc(func(ctx context.Context, w http.ResponseWriter, r *http.Request) {
		w.WriteHeader(http.StatusNotFound)
		err := templates.Lookup("404.html").Execute(w, nil)
		if err != nil {
			logger := termlog.FromContext(ctx)
			logger.Shout("Could not execute template: %s", err)
		}
	})
}

// Router constructs the main Devd router that serves all requests
func (dd *Devd) Router(logger termlog.TermLog, templates *template.Template) (http.Handler, error) {
	mux := http.NewServeMux()
	hasGlobal := false

	ci := inject.CopyInject{}
	if dd.HasLivereload() {
		ci = livereload.Injector
	}

	for match, route := range dd.Routes {
		if match == "/" {
			hasGlobal = true
		}
		handler := dd.WrapHandler(
			logger,
			route.Endpoint.Handler(templates, ci),
		)
		handler = http.StripPrefix(route.Path, handler)
		mux.Handle(match, handler)
	}
	if dd.HasLivereload() {
		lr := livereload.NewServer("livereload", logger)
		mux.Handle(livereload.EndpointPath, lr)
		mux.Handle(livereload.ScriptPath, http.HandlerFunc(lr.ServeScript))
		if dd.LivereloadRoutes {
			err := WatchRoutes(dd.Routes, lr, dd.Excludes, logger)
			if err != nil {
				return nil, fmt.Errorf("Could not watch routes for livereload: %s", err)
			}
		}
		if len(dd.WatchPaths) > 0 {
			err := WatchPaths(dd.WatchPaths, dd.Excludes, lr, logger)
			if err != nil {
				return nil, fmt.Errorf("Could not watch path for livereload: %s", err)
			}
		}
		dd.lrserver = lr
	}
	if !hasGlobal {
		mux.Handle(
			"/",
			dd.WrapHandler(logger, HandleNotFound(templates)),
		)
	}
	var h = http.Handler(mux)
	if dd.Credentials != nil {
		h = httpauth.SimpleBasicAuth(
			dd.Credentials.username, dd.Credentials.password,
		)(h)
	}
	return hostPortStrip(h), nil
}

// Serve starts the devd server. The callback is called with the serving URL
// just before service starts.
func (dd *Devd) Serve(address string, port int, certFile string, logger termlog.TermLog, callback func(string)) error {
	templates, err := ricetemp.MakeTemplates(rice.MustFindBox("templates"))
	if err != nil {
		return fmt.Errorf("Error loading templates: %s", err)
	}
	mux, err := dd.Router(logger, templates)
	if err != nil {
		return err
	}
	var tlsConfig *tls.Config
	var tlsEnabled bool
	if certFile != "" {
		tlsConfig, err = getTLSConfig(certFile)
		if err != nil {
			return fmt.Errorf("Could not load certs: %s", err)
		}
		tlsEnabled = true
	}

	var hl net.Listener
	if port > 0 {
		hl, err = net.Listen("tcp", fmt.Sprintf("%v:%d", address, port))
	} else {
		hl, err = pickPort(address, portLow, portHigh, tlsEnabled)
	}
	if err != nil {
		return err
	}

	if tlsConfig != nil {
		hl = tls.NewListener(hl, tlsConfig)
	}

	hl = slowdown.NewSlowListener(hl, dd.UpKbps*1024, dd.DownKbps*1024)
	url := formatURL(tlsEnabled, address, hl.Addr().(*net.TCPAddr).Port)
	logger.Say("Listening on %s (%s)", url, hl.Addr().String())
	server := &http.Server{Addr: hl.Addr().String(), Handler: mux}
	callback(url)

	if dd.HasLivereload() {
		c := make(chan os.Signal, 1)
		signal.Notify(c, syscall.SIGHUP)
		go func() {
			for {
				<-c
				logger.Say("Received signal - reloading")
				dd.lrserver.Reload([]string{"*"})
			}
		}()
	}

	err = server.Serve(hl)
	logger.Shout("Server stopped: %v", err)
	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"sync"
)

type comment struct {
	Author string `json:"author"`
	Text   string `json:"text"`
}

const dataFile = "./_comments.json"

var commentMutex = new(sync.Mutex)

// Handle comments
func handleComments(w http.ResponseWriter, r *http.Request) {
	// Since multiple requests could come in at once, ensure we have a lock
	// around all file operations
	commentMutex.Lock()
	defer commentMutex.Unlock()

	// Stat the file, so we can find it's current permissions
	fi, err := os.Stat(dataFile)
	if err != nil {
		http.Error(w, fmt.Sprintf("Unable to stat the data file (%s): %s", dataFile, err), http.StatusInternalServerError)
		return
	}

	// Read the comments from the file.
	commentData, err := ioutil.ReadFile(dataFile)
	if err != nil {
		http.Error(w, fmt.Sprintf("Unable to read the data file (%s): %s", dataFile, err), http.StatusInternalServerError)
		return
	}

	switch r.Method {
	case "POST":
		// Decode the JSON data
		comments := make([]comment, 0)
		if err := json.Unmarshal(commentData, &comments); err != nil {
			http.Error(w, fmt.Sprintf("Unable to Unmarshal comments from data file (%s): %s", dataFile, err), http.StatusInternalServerError)
			return
		}

		// Add a new comment to the in memory slice of comments
		comments = append(comments, comment{Author: r.FormValue("author"), Text: r.FormValue("text")})

		// Marshal the comments to indented json.
		commentData, err = json.MarshalIndent(comments, "", "    ")
		if err != nil {
			http.Error(w, fmt.Sprintf("Unable to marshal comments to json: %s", err), http.StatusInternalServerError)
			return
		}

		// Write out the comments to the file, preserving permissions
		err := ioutil.WriteFile(dataFile, commentData, fi.Mode())
		if err != nil {
			http.Error(w, fmt.Sprintf("Unable to write comments to data file (%s): %s", dataFile, err), http.StatusInternalServerError)
			return
		}

	case "GET":
		// stream the contents of the file to the response
		io.Copy(w, bytes.NewReader(commentData))

	default:
		// Don't know the method, so error
		http.Error(w, fmt.Sprintf("Unsupported method: %s", r.Method), http.StatusMethodNotAllowed)
	}
}

func main() {
	http.HandleFunc("/comments.json", handleComments)
	http.Handle("/", http.FileServer(http.Dir("./public")))
	log.Fatal(http.ListenAndServe(":3000", nil))
}
<commit_msg>Improve Go server<commit_after>package main

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"sync"
)

type comment struct {
	Author string `json:"author"`
	Text   string `json:"text"`
}

const dataFile = "./_comments.json"

var commentMutex = new(sync.Mutex)

// Handle comments
func handleComments(w http.ResponseWriter, r *http.Request) {
	// Since multiple requests could come in at once, ensure we have a lock
	// around all file operations
	commentMutex.Lock()
	defer commentMutex.Unlock()

	// Stat the file, so we can find it's current permissions
	fi, err := os.Stat(dataFile)
	if err != nil {
		http.Error(w, fmt.Sprintf("Unable to stat the data file (%s): %s", dataFile, err), http.StatusInternalServerError)
		return
	}

	// Read the comments from the file.
	commentData, err := ioutil.ReadFile(dataFile)
	if err != nil {
		http.Error(w, fmt.Sprintf("Unable to read the data file (%s): %s", dataFile, err), http.StatusInternalServerError)
		return
	}

	switch r.Method {
	case "POST":
		// Decode the JSON data
		comments := make([]comment, 0)
		if err := json.Unmarshal(commentData, &comments); err != nil {
			http.Error(w, fmt.Sprintf("Unable to Unmarshal comments from data file (%s): %s", dataFile, err), http.StatusInternalServerError)
			return
		}

		// Add a new comment to the in memory slice of comments
		comments = append(comments, comment{Author: r.FormValue("author"), Text: r.FormValue("text")})

		// Marshal the comments to indented json.
		commentData, err = json.MarshalIndent(comments, "", "    ")
		if err != nil {
			http.Error(w, fmt.Sprintf("Unable to marshal comments to json: %s", err), http.StatusInternalServerError)
			return
		}

		// Write out the comments to the file, preserving permissions
		err := ioutil.WriteFile(dataFile, commentData, fi.Mode())
		if err != nil {
			http.Error(w, fmt.Sprintf("Unable to write comments to data file (%s): %s", dataFile, err), http.StatusInternalServerError)
			return
		}

		w.Header().Set("Content-Type", "application/json")
		io.Copy(w, bytes.NewReader(commentData))

	case "GET":
		w.Header().Set("Content-Type", "application/json")
		// stream the contents of the file to the response
		io.Copy(w, bytes.NewReader(commentData))

	default:
		// Don't know the method, so error
		http.Error(w, fmt.Sprintf("Unsupported method: %s", r.Method), http.StatusMethodNotAllowed)
	}
}

func main() {
	http.HandleFunc("/comments.json", handleComments)
	http.Handle("/", http.FileServer(http.Dir("./public")))
	log.Println("Server started: http://localhost:3000")
	log.Fatal(http.ListenAndServe(":3000", nil))
}
<|endoftext|>


<commit_before>package main

import (
	"crypto/hmac"
	"crypto/sha1"
	"encoding/hex"
	"encoding/json"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"sync"

	"github.com/google/go-github/github"
	mgo "gopkg.in/mgo.v2"
	"gopkg.in/mgo.v2/bson"
)

var dbSession *mgo.Session
var users *mgo.Collection
var repos *mgo.Collection

var specialBranch string
var specialRef string

var handlers map[string]*comm
var handlersMtx *sync.RWMutex

// Repo is a github repo
type Repo struct {
	ID              string `bson:"_id,omitempty"`
	Username        string
	AccessToken     string
	LastBuildOutput string
	LastBuildStatus string
}

// Comm is used to communicate to a worker goroutine
// stop is sent 'true' to ask the goroutine to stop
// stopped is sent 'true' when it stops
type comm struct {
	Stop    chan bool
	Stopped chan bool
}

func main() {
	config := loadConfig()
	config.print()

	specialBranch = config.SpecialBranch
	specialRef = "refs/heads/" + config.SpecialBranch

	dbSession, err := mgo.Dial(config.MongoURL)
	if err != nil {
		panic(err)
	}
	defer dbSession.Close()

	users = dbSession.DB("hugo-pages").C("users")
	repos = dbSession.DB("hugo-pages").C("repos")

	handlers = make(map[string]*comm)
	handlersMtx = &sync.RWMutex{}

	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		payload := validate(r)
		if payload == nil {
			return
		}

		fullname := *payload.Repo.FullName
		url := *payload.Repo.URL

		go func() {
			log.Printf("Got: %s", fullname)
			workerComm := synchronize(fullname)
			w := newWorker(fullname, url, workerComm)
			w.work()
			change := bson.M{
				"$set": bson.M{
					"lastbuildoutput": w.log,
					"lastbuildstatus": w.status,
				},
			}
			repos.UpdateId(fullname, change)
		}()
	})

	log.Println("Listening")
	log.Fatal(http.ListenAndServe(config.Address, nil))
}

// synchronize is a step to see if another goroutine is handling a request
// from the same repo
func synchronize(fullname string) *comm {
	// TODO: Make sure that the goroutine that overtakes is actually a new one
	handlersMtx.RLock()
	currentComm := handlers[fullname]
	handlersMtx.RUnlock()

	if currentComm == nil {
		// There is no goroutine handling a request for this repo
		handlersMtx.Lock()
		currentComm = handlers[fullname]
		if currentComm == nil {
			// Still there is no goroutine handling a request for this repo
			currentComm = &comm{
				Stop:    make(chan bool, 3), // Can buffer stop signals upto 3
				Stopped: make(chan bool, 1),
			}
			handlers[fullname] = currentComm
			handlersMtx.Unlock()
		} else {
			// Some goroutine has entered while we were creating a new channel
			handlersMtx.Unlock()
			_waitAndStart(currentComm, fullname)
		}
	} else {
		_waitAndStart(currentComm, fullname)
	}
	return currentComm
}

func _waitAndStart(currentComm *comm, fullname string) {
	currentComm.Stop <- true
	val := <-currentComm.Stopped
	if val == true {
		// This means previous goroutine does not know it was interrupted
		// So clear the signal it was sent
		<-currentComm.Stop
		currentComm.Stopped <- true
		//and resynchronize
		synchronize(fullname)
	}
}

type worker struct {
	stop     chan bool
	stopped  chan bool
	fullname string
	url      string
	log      string
	status   string
}

func newWorker(fullname, url string, workerComm *comm) *worker {
	return &worker{
		stop:     workerComm.Stop,
		stopped:  workerComm.Stopped,
		fullname: fullname,
		url:      url,
		log:      "",
		status:   "incomplete",
	}
}

func (w *worker) work() {
	w.checkAndContinue("starting processing")

	repo := getRepo(w.fullname)
	if repo == nil {
		w.checkAndStop("repo not known")
		return
	}

	path := "/tmp/" + w.fullname
	defer cleanUp(path)

	_, err := Clone(w.url, path, specialBranch)
	if err != nil {
		w.checkAndStop("could not clone")
		return
	}

	w.checkAndContinue("building")
	out, err := HugoBuild(path)
	if err != nil {
		cleanUp(path)
		w.checkAndStop("could not build")
		return
	}
	log.Printf(out)
	w.checkAndContinue(out)

	subrepo, err := Checkout(path + "/public/") // trailing / important
	if err != nil {
		cleanUp(path)
		w.checkAndStop("Could not checkout: " + err.Error())
		return
	}

	w.checkAndContinue("pushing")

	err = Push(formatPushURL(repo.AccessToken, repo.Username, w.fullname), subrepo)
	if err != nil {
		cleanUp(path)
		log.Printf("Could not push: %s", err)
		return
	}
	cleanUp(path)
	w.checkAndStop("finishing up")
	w.status = "complete"
}

func (w *worker) checkAndContinue(msg string) {
	select {
	case <-w.stop:
		log.Printf("stopping: %s %s", msg, w.fullname)
		w.log = w.log + "stopping: " + msg + "\n"
		w.stopped <- false
	default:
		log.Printf("continuing %s %s", msg, w.fullname)
		w.log = w.log + "continuing: " + msg + "\n"
	}
}

func (w *worker) checkAndStop(msg string) {
	select {
	case <-w.stop:
		log.Printf("stopping: %s %s", msg, w.fullname)
		w.log = w.log + "stopping: " + msg + "\n"
		w.stopped <- false
	default:
		handlersMtx.Lock()
		delete(handlers, w.fullname)
		handlersMtx.Unlock()
		log.Printf("stopping (completed): %s %s", msg, w.fullname)
		w.log = w.log + "stopping (completed): " + msg + "\n"
		w.stopped <- true
	}
}

func validate(r *http.Request) *github.WebHookPayload {
	if r.Method != "POST" {
		log.Println("Not POST")
		return nil
	}

	b, err := ioutil.ReadAll(r.Body)
	if err != nil {
		return nil
	}

	var t github.WebHookPayload
	err = json.Unmarshal(b, &t)
	if err != nil {
		return nil
	}

	if *t.Ref != specialRef {
		log.Println("Not hugo pages:", *t.Ref)
		return nil
	}

	mac := hmac.New(sha1.New, []byte("hugopagessecret"))
	mac.Write(b)
	if err != nil {
		log.Println(err)
		return nil
	}

	expectedMAC := mac.Sum(nil)
	log.Println(r.Header)
	signature := r.Header["X-Hub-Signature"][0]
	log.Println(signature)
	expected := "sha1=" + hex.EncodeToString(expectedMAC)
	log.Println(expected)

	if !hmac.Equal([]byte(signature), []byte(expected)) {
		return nil
	}

	return &t
}

func formatPushURL(accessToken, username, fullname string) string {
	// TODO: optimize string concat
	return "https://" + username + ":" + accessToken + "@github.com/" + fullname
}

func getRepo(fullname string) *Repo {
	result := &Repo{}
	err := repos.Find(bson.M{"_id": fullname}).One(result)
	if err != nil {
		log.Println(err)
		return nil
	}

	return result
}

func cleanUp(path string) {
	err := os.RemoveAll(path)
	if err != nil {
		log.Println(err)
	}
}
<commit_msg>Fix errors in code<commit_after>package main

import (
	"crypto/hmac"
	"crypto/sha1"
	"encoding/hex"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"strings"
	"sync"

	"github.com/google/go-github/github"
	mgo "gopkg.in/mgo.v2"
	"gopkg.in/mgo.v2/bson"
)

var dbSession *mgo.Session
var users *mgo.Collection
var repos *mgo.Collection

var specialBranch string
var specialRef string

var handlers map[int]*comm
var handlersMtx *sync.RWMutex

// Repo is a github repo
type Repo struct {
	ID              string `bson:"_id,omitempty"`
	Username        string
	AccessToken     string
	LastBuildOutput string
	LastBuildStatus string
}

// Comm is used to communicate to a worker goroutine
// stop is sent 'true' to ask the goroutine to stop
// stopped is sent 'true' when it stops
type comm struct {
	Stop    chan bool
	Stopped chan bool
}

func main() {
	config := loadConfig()
	config.print()

	specialBranch = config.SpecialBranch
	specialRef = "refs/heads/" + config.SpecialBranch

	dbSession, err := mgo.Dial(config.MongoURL)
	if err != nil {
		panic(err)
	}
	defer dbSession.Close()

	users = dbSession.DB("hugo-pages").C("users")
	repos = dbSession.DB("hugo-pages").C("repos")

	handlers = make(map[int]*comm)
	handlersMtx = &sync.RWMutex{}

	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		payload := validate(r)
		if payload == nil {
			return
		}

		fullname := *payload.Repo.FullName
		id := *payload.Repo.ID
		url := *payload.Repo.URL

		go func() {
			log.Printf("Got: %s", fullname)
			workerComm := synchronize(id)
			w := newWorker(id, url, workerComm)
			w.work()
			change := bson.M{
				"$set": bson.M{
					"lastbuildoutput": w.log,
					"lastbuildstatus": w.status,
				},
			}
			repos.UpdateId(fullname, change)
		}()
	})

	log.Println("Listening")
	log.Fatal(http.ListenAndServe(config.Address, nil))
}

// synchronize is a step to see if another goroutine is handling a request
// from the same repo
func synchronize(id int) *comm {
	handlersMtx.RLock()
	currentComm := handlers[id]
	handlersMtx.RUnlock()

	if currentComm == nil {
		// There is no goroutine handling a request for this repo
		handlersMtx.Lock()
		currentComm = handlers[id]
		if currentComm == nil {
			// Still there is no goroutine handling a request for this repo
			currentComm = &comm{
				Stop:    make(chan bool, 3), // Can buffer stop signals upto 3
				Stopped: make(chan bool, 1),
			}
			handlers[id] = currentComm
			handlersMtx.Unlock()
		} else {
			// Some goroutine has entered while we were creating a new channel
			handlersMtx.Unlock()
			_waitAndStart(currentComm, id)
		}
	} else {
		_waitAndStart(currentComm, id)
	}
	return currentComm
}

func _waitAndStart(currentComm *comm, id int) {
	currentComm.Stop <- true
	val := <-currentComm.Stopped
	if val == true {
		// This means previous goroutine does not know it was interrupted
		// So clear the signal it was sent
		<-currentComm.Stop
		currentComm.Stopped <- true
		//and resynchronize
		synchronize(id)
	}
}

type worker struct {
	stop    chan bool
	stopped chan bool
	id      int
	url     string
	path    string
	log     string
	status  string
}

func newWorker(id int, url string, workerComm *comm) *worker {
	return &worker{
		stop:    workerComm.Stop,
		stopped: workerComm.Stopped,
		id:      id,
		path:    fmt.Sprintf("/tmp/%d", id),
		url:     url,
		log:     "",
		status:  "incomplete",
	}
}

func (w *worker) work() {
	if !w.checkAndContinue("Starting build") {
		return
	}

	repo := getRepo(w.id)
	if repo == nil {
		w.checkAndStop("Repo not known")
		return
	}

	_, err := Clone(w.url, w.path, specialBranch)
	if err != nil {
		w.checkAndStop("Could not clone")
		return
	}

	if !w.checkAndContinue("building") {
		return
	}

	out, err := HugoBuild(w.path)
	if err != nil {
		cleanUp(w.path)
		w.checkAndStop("Could not build")
		return
	}
	log.Printf(out)

	if !w.checkAndContinue(out) {
		return
	}

	subrepo, err := Checkout(w.path + "/public/") // trailing / important
	if err != nil {
		cleanUp(w.path)
		w.checkAndStop("Could not checkout: " + err.Error())
		return
	}

	if !w.checkAndContinue("pushing") {
		return
	}

	err = Push(formatPushURL(repo.AccessToken, repo.Username, w.url), subrepo)
	if err != nil {
		cleanUp(w.path)
		log.Printf("Could not push: %s", err)
		return
	}
	cleanUp(w.path)
	w.checkAndStop("Finishing up")
	w.status = "complete"
}

func (w *worker) checkAndContinue(msg string) bool {
	select {
	case <-w.stop:
		log.Printf("Stopping: %s %s", msg, w.url)
		w.log = w.log + "Stopping: " + msg + "\n"
		cleanUp(w.path)
		w.stopped <- false
		return false
	default:
		log.Printf("Continuing %s", msg)
		w.log = w.log + msg + "\n"
		return true
	}
}

func (w *worker) checkAndStop(msg string) {
	select {
	case <-w.stop:
		log.Printf("Stopping: %s %s", msg, w.url)
		w.log = w.log + "Stopping: " + msg + "\n"
		cleanUp(w.path)
		w.stopped <- false
	default:
		handlersMtx.Lock()
		delete(handlers, w.id)
		handlersMtx.Unlock()
		log.Printf("Stopping (completed): %s", msg)
		w.log = w.log + "Stopping (completed): " + msg + "\n"
		cleanUp(w.path)
		w.stopped <- true
	}
}

func validate(r *http.Request) *github.WebHookPayload {
	if r.Method != "POST" {
		log.Println("Not POST")
		return nil
	}

	b, err := ioutil.ReadAll(r.Body)
	if err != nil {
		return nil
	}

	var t github.WebHookPayload
	err = json.Unmarshal(b, &t)
	if err != nil {
		return nil
	}

	if t.Ref == nil {
		return nil
	}

	if *t.Ref != specialRef {
		log.Println("Not hugo pages:", *t.Ref)
		return nil
	}

	mac := hmac.New(sha1.New, []byte("hugopagessecret"))
	mac.Write(b)
	if err != nil {
		log.Println(err)
		return nil
	}

	expectedMAC := mac.Sum(nil)
	signature := r.Header["X-Hub-Signature"][0]
	expected := "sha1=" + hex.EncodeToString(expectedMAC)

	if !hmac.Equal([]byte(signature), []byte(expected)) {
		log.Println("unknown origin")
		return nil
	}

	return &t
}

func formatPushURL(accessToken, username, url string) string {
	// TODO: optimize string concat
	parts := strings.Split(url, "://")
	return parts[0] + "://" + username + ":" + accessToken + parts[1]
}

func getRepo(id int) *Repo {
	result := &Repo{}
	err := repos.FindId(id).One(result)
	if err != nil {
		log.Println(err)
		return nil
	}

	return result
}

func cleanUp(path string) {
	err := os.RemoveAll(path)
	if err != nil {
		log.Println(err)
	}
}
<|endoftext|>