<commit_before>package app

import (
	"bytes"
	"fmt"
	"github.com/go-martini/martini"
	"github.com/hashicorp/memberlist"
	"github.com/martini-contrib/render"
	"log"
	"net/http"
	"strconv"
)

//TODO make message definitions more explicit

func InitWebserver(list *memberlist.Memberlist, cfg Config) {
	//init the connectionPool
	riakPool := InitRiakPool(cfg)
	// tieing our Queue to HTTP interface == bad we should move this somewhere else
	queues := InitQueues(riakPool)
	// also tieing topics this is next for refactor
	topics := InitTopics(cfg, riakPool, queues)

	m := martini.Classic()
	m.Use(render.Renderer())
	m.Get("/status/servers", func() string {
		return_string := ""
		for _, member := range list.Members() {
			return_string += fmt.Sprintf("Member: %s %s\n", member.Name, member.Addr)
		}
		return return_string
	})
	m.Get("/status/:queue/partitions", func(r render.Render, params martini.Params) {

		//check if we've initialized this queue yet
		var present bool
		_, present = queues.QueueMap[params["queue"]]
		if present != true {
			queues.InitQueue(cfg, params["queue"])
		}
		r.JSON(200, map[string]interface{}{"paritions": queues.QueueMap[params["queue"]].Parts.PartitionCount()})
	})
	m.Get("/topics", func(r render.Render) {
		topicList := make([]string, 0, 10)
		for topicName, _ := range topics.TopicMap {
			topicList = append(topicList, topicName)
		}
		r.JSON(200, map[string]interface{}{"topics": topicList})
	})
	m.Get("/topics/:topic", func(r render.Render, params martini.Params) {
		var present bool
		_, present = topics.TopicMap[params["topic"]]
		if present != true {
			topics.InitTopic(params["topic"])
		}

		r.JSON(200, map[string]interface{}{"Queues": topics.TopicMap[params["topic"]].ListQueues()})
	})

	m.Put("/topics/:topic/queues/:queue", func(r render.Render, params martini.Params) {
		var present bool
		_, present = topics.TopicMap[params["topic"]]
		if present != true {
			topics.InitTopic(params["topic"])
		}
		topics.TopicMap[params["topic"]].AddQueue(params["queue"])
		r.JSON(200, map[string]interface{}{"Queues": topics.TopicMap[params["topic"]].ListQueues()})
	})

	// neeeds a little work....
	m.Delete("/topics/:topic/queues/:queue", func(r render.Render, params martini.Params) {
		var present bool
		_, present = topics.TopicMap[params["topic"]]
		if present != true {
			topics.InitTopic(params["topic"])
		}
		topics.TopicMap[params["topic"]].DeleteQueue(params["queue"])
		r.JSON(200, map[string]interface{}{"Queues": topics.TopicMap[params["topic"]].ListQueues()})
	})

	m.Delete("/topics/:topic", func(r render.Render, params martini.Params) {

		var present bool
		_, present = topics.TopicMap[params["topic"]]
		if present != true {
			topics.InitTopic(params["topic"])
		}
		r.JSON(200, map[string]interface{}{"Deleted": topics.DeleteTopic(params["topic"])})
	})

	m.Put("/topics/:topic/message", func(r render.Render, params martini.Params, req *http.Request) {

		var present bool
		_, present = topics.TopicMap[params["topic"]]
		if present != true {
			topics.InitTopic(params["topic"])
		}
		var buf bytes.Buffer
		buf.ReadFrom(req.Body)

		response := topics.TopicMap[params["topic"]].Broadcast(cfg, buf.String())
		r.JSON(200, response)
	})

	m.Get("/queues/:queue/messages/:batchSize", func(r render.Render, params martini.Params) {
		//check if we've initialized this queue yet
		var present bool
		_, present = queues.QueueMap[params["queue"]]
		if present != true {
			queues.InitQueue(cfg, params["queue"])
		}
		batchSize, err := strconv.ParseUint(params["batchSize"], 10, 32)
		if err != nil {
			//log the error for unparsable input
			log.Println(err)
			r.JSON(422, err.Error())
		}
		messages, err := queues.QueueMap[params["queue"]].Get(cfg, list, uint32(batchSize))
		//TODO move this into the Queue.Get code
		messageReturn := make(map[string][]map[string]interface{})
		messageList := make([]map[string]interface{}, 0, 10)
		for _, object := range messages {
			//messageReturn[object.Key] = string(object.Data[:])
			message := make(map[string]interface{})
			message["id"] = object.Key
			message["body"] = string(object.Data[:])
			messageList = append(messageList, message)
		}
		messageReturn["messages"] = messageList
		if err != nil {
			log.Println(err)
			r.JSON(204, err.Error())
		} else {
			r.JSON(200, messageReturn)
		}
	})
	m.Put("/queues/:queue/messages", func(params martini.Params, req *http.Request) string {
		var present bool
		_, present = queues.QueueMap[params["queue"]]
		if present != true {
			queues.InitQueue(cfg, params["queue"])
		}

		// parse the request body into a sting
		// TODO clean this up, full json api?
		var buf bytes.Buffer
		buf.ReadFrom(req.Body)
		uuid := queues.QueueMap[params["queue"]].Put(cfg, buf.String())

		return uuid
	})
	m.Delete("/queues/:queue/message/:messageId", func(r render.Render, params martini.Params) {
		var present bool
		_, present = queues.QueueMap[params["queue"]]
		if present != true {
			queues.InitQueue(cfg, params["queue"])
		}

		r.JSON(200, queues.QueueMap[params["queue"]].Delete(cfg, params["messageId"]))
	})
	log.Fatal(http.ListenAndServe(":"+strconv.Itoa(cfg.Core.HttpPort), m))
}
<commit_msg>whitespace!!!<commit_after>package app

import (
  "bytes"
  "fmt"
  "github.com/go-martini/martini"
  "github.com/hashicorp/memberlist"
  "github.com/martini-contrib/render"
  "log"
  "net/http"
  "strconv"
)

//TODO make message definitions more explicit

func InitWebserver(list *memberlist.Memberlist, cfg Config) {
  //init the connectionPool
  riakPool := InitRiakPool(cfg)
  // tieing our Queue to HTTP interface == bad we should move this somewhere else
  queues := InitQueues(riakPool)
  // also tieing topics this is next for refactor
  topics := InitTopics(cfg, riakPool, queues)

  m := martini.Classic()
  m.Use(render.Renderer())
  m.Get("/status/servers", func() string {
    return_string := ""
    for _, member := range list.Members() {
      return_string += fmt.Sprintf("Member: %s %s\n", member.Name, member.Addr)
    }
    return return_string
  })
  m.Get("/status/:queue/partitions", func(r render.Render, params martini.Params) {

    //check if we've initialized this queue yet
    var present bool
    _, present = queues.QueueMap[params["queue"]]
    if present != true {
      queues.InitQueue(cfg, params["queue"])
    }
    r.JSON(200, map[string]interface{}{"paritions": queues.QueueMap[params["queue"]].Parts.PartitionCount()})
  })
  m.Get("/topics", func(r render.Render) {
    topicList := make([]string, 0, 10)
    for topicName, _ := range topics.TopicMap {
      topicList = append(topicList, topicName)
    }
    r.JSON(200, map[string]interface{}{"topics": topicList})
  })
  m.Get("/topics/:topic", func(r render.Render, params martini.Params) {
    var present bool
    _, present = topics.TopicMap[params["topic"]]
    if present != true {
      topics.InitTopic(params["topic"])
    }

    r.JSON(200, map[string]interface{}{"Queues": topics.TopicMap[params["topic"]].ListQueues()})
  })

  m.Put("/topics/:topic/queues/:queue", func(r render.Render, params martini.Params) {
    var present bool
    _, present = topics.TopicMap[params["topic"]]
    if present != true {
      topics.InitTopic(params["topic"])
    }
    topics.TopicMap[params["topic"]].AddQueue(params["queue"])
    r.JSON(200, map[string]interface{}{"Queues": topics.TopicMap[params["topic"]].ListQueues()})
  })

  // neeeds a little work....
  m.Delete("/topics/:topic/queues/:queue", func(r render.Render, params martini.Params) {
    var present bool
    _, present = topics.TopicMap[params["topic"]]
    if present != true {
      topics.InitTopic(params["topic"])
    }
    topics.TopicMap[params["topic"]].DeleteQueue(params["queue"])
    r.JSON(200, map[string]interface{}{"Queues": topics.TopicMap[params["topic"]].ListQueues()})
  })

  m.Delete("/topics/:topic", func(r render.Render, params martini.Params) {

    var present bool
    _, present = topics.TopicMap[params["topic"]]
    if present != true {
      topics.InitTopic(params["topic"])
    }
    r.JSON(200, map[string]interface{}{"Deleted": topics.DeleteTopic(params["topic"])})
  })

  m.Put("/topics/:topic/message", func(r render.Render, params martini.Params, req *http.Request) {

    var present bool
    _, present = topics.TopicMap[params["topic"]]
    if present != true {
      topics.InitTopic(params["topic"])
    }
    var buf bytes.Buffer
    buf.ReadFrom(req.Body)

    response := topics.TopicMap[params["topic"]].Broadcast(cfg, buf.String())
    r.JSON(200, response)
  })

  m.Get("/queues/:queue/messages/:batchSize", func(r render.Render, params martini.Params) {
    //check if we've initialized this queue yet
    var present bool
    _, present = queues.QueueMap[params["queue"]]
    if present != true {
      queues.InitQueue(cfg, params["queue"])
    }
    batchSize, err := strconv.ParseUint(params["batchSize"], 10, 32)
    if err != nil {
      //log the error for unparsable input
      log.Println(err)
      r.JSON(422, err.Error())
    }
    messages, err := queues.QueueMap[params["queue"]].Get(cfg, list, uint32(batchSize))
    //TODO move this into the Queue.Get code
    messageReturn := make(map[string][]map[string]interface{})
    messageList := make([]map[string]interface{}, 0, 10)
    for _, object := range messages {
      //messageReturn[object.Key] = string(object.Data[:])
      message := make(map[string]interface{})
      message["id"] = object.Key
      message["body"] = string(object.Data[:])
      messageList = append(messageList, message)
    }
    messageReturn["messages"] = messageList
    if err != nil {
      log.Println(err)
      r.JSON(204, err.Error())
    } else {
      r.JSON(200, messageReturn)
    }
  })
  m.Put("/queues/:queue/messages", func(params martini.Params, req *http.Request) string {
    var present bool
    _, present = queues.QueueMap[params["queue"]]
    if present != true {
      queues.InitQueue(cfg, params["queue"])
    }

    // parse the request body into a sting
    // TODO clean this up, full json api?
    var buf bytes.Buffer
    buf.ReadFrom(req.Body)
    uuid := queues.QueueMap[params["queue"]].Put(cfg, buf.String())

    return uuid
  })
  m.Delete("/queues/:queue/message/:messageId", func(r render.Render, params martini.Params) {
    var present bool
    _, present = queues.QueueMap[params["queue"]]
    if present != true {
      queues.InitQueue(cfg, params["queue"])
    }

    r.JSON(200, queues.QueueMap[params["queue"]].Delete(cfg, params["messageId"]))
  })
  log.Fatal(http.ListenAndServe(":"+strconv.Itoa(cfg.Core.HttpPort), m))
}
<|endoftext|>


<commit_before>package signals

import "encoding/gob"

import (
	"math"
)

func init() {
	gob.Register(Constant{})
	gob.Register(Sine{})
	gob.Register(Pulse{})
	gob.Register(Square{})
	gob.Register(RampUp{})
	gob.Register(RampDown{})
	gob.Register(Heavyside{})
	gob.Register(Sigmoid{})
}

func DB(vol float64) float64 {
	return 6 * math.Log2(vol)
}
func Vol(DB float64) float64 {
	return math.Pow(2, DB/6)
}

// a Signal with constant value
type Constant struct {
	Constant y
}

func NewConstant(DB float64) Constant {
	return Constant{y(unitYfloat64 * Vol(DB))}
}

func (s Constant) property(p x) y {
	return s.Constant
}

// a PeriodicSignal that varies sinusoidally, repeating with Cycle width.
type Sine struct {
	Cycle x
}

func (s Sine) property(p x) y {
	return y(math.Sin(float64(p)/float64(s.Cycle)*2*math.Pi) * unitYfloat64)
}

func (s Sine) Period() x {
	return s.Cycle
}

// a Signal that varies sinusoidally, repeating with Cycle width.
type Sinc struct {
	Cycle x
}

func (s Sinc) property(p x) y {
	if p==0 {return unitY}
	xp:=float64(p)/float64(s.Cycle)*2*math.Pi
	return y(math.Sin(xp)/xp * unitYfloat64)
}

// a Signal that varies sinusoidally, repeating with Cycle width.
type Gauss struct {
	Q22 float64   // 2 * q squared
}

func (s Gauss) property(p x) y {
	return y(math.Exp(-float64(p)*float64(p)/s.Q22) * unitYfloat64)
}

// a LimitedSignal that produces unitY for a Width, zero otherwise.
type Pulse struct {
	Width x
}

func (s Pulse) property(p x) y {
	if p > s.Width || p < 0 {
		return 0
	} else {
		return unitY
	}
}

func (s Pulse) MaxX() x {
	return s.Width
}

// a PeriodicSignal that produces equal regions of +unitY then -unitY, repeating with Cycle width.
type Square struct {
	Cycle x
}

func (s Square) property(p x) y {
	if p%s.Cycle >= s.Cycle/2 {
		return -unitY
	} else {
		return unitY
	}
}

func (s Square) Period() x {
	return s.Cycle
}

// a Signal which ramps from zero to unitY over a Period width.
type RampUp struct {
	Period x
}

func (s RampUp) property(p x) y {
	if p < 0 {
		return 0
	} else if p > s.Period {
		return unitY
	} else {
		return y(x(unitY) / s.Period * p)
	}
}

// a Signal wcich ramps from unitY to zero, over a Period width.
type RampDown struct {
	Period x
}

func (s RampDown) property(p x) y {
	if p < 0 {
		return unitY
	} else if p > s.Period {
		return 0
	} else {
		return y(x(unitY) / s.Period * (s.Period - p))
	}
}

// a Signal that returns +unitY for positive x and zero for negative.
type Heavyside struct {
}

func (s Heavyside) property(p x) y {
	if p < 0 {
		return 0
	}
	return unitY
}

// a Signal that smoothly transitions from 0 to +unitY.
// with a maximium gradient (first derivative) at x=0, of Steepness.
type Sigmoid struct {
	Steepness x
}

func (s Sigmoid) property(p x) y {
	return y(unitYfloat64 / (1 + math.Exp(-float64(p)/float64(s.Steepness))))
}

<commit_msg>comment<commit_after>package signals

import "encoding/gob"

import (
	"math"
)

func init() {
	gob.Register(Constant{})
	gob.Register(Sine{})
	gob.Register(Pulse{})
	gob.Register(Square{})
	gob.Register(RampUp{})
	gob.Register(RampDown{})
	gob.Register(Heavyside{})
	gob.Register(Sigmoid{})
}

func DB(vol float64) float64 {
	return 6 * math.Log2(vol)
}
func Vol(DB float64) float64 {
	return math.Pow(2, DB/6)
}

// a Signal with constant value
type Constant struct {
	Constant y
}

func NewConstant(DB float64) Constant {
	return Constant{y(unitYfloat64 * Vol(DB))}
}

func (s Constant) property(p x) y {
	return s.Constant
}

// a PeriodicSignal that varies sinusoidally, repeating with Cycle width.
type Sine struct {
	Cycle x
}

func (s Sine) property(p x) y {
	return y(math.Sin(float64(p)/float64(s.Cycle)*2*math.Pi) * unitYfloat64)
}

func (s Sine) Period() x {
	return s.Cycle
}

// a Signal that is a wave 'packet', with fundamental (central) wavelength of Cycle.
type Sinc struct {
	Cycle x
}

func (s Sinc) property(p x) y {
	if p==0 {return unitY}
	xp:=float64(p)/float64(s.Cycle)*2*math.Pi
	return y(math.Sin(xp)/xp * unitYfloat64)
}

// a Signal that peaks, centred on zero, as a gaussian distribution, width q.
type Gauss struct {
	Q22 float64   // 2 * q squared
}

func (s Gauss) property(p x) y {
	return y(math.Exp(-float64(p)*float64(p)/s.Q22) * unitYfloat64)
}

// a LimitedSignal that produces unitY for a Width, zero otherwise.
type Pulse struct {
	Width x
}

func (s Pulse) property(p x) y {
	if p > s.Width || p < 0 {
		return 0
	} else {
		return unitY
	}
}

func (s Pulse) MaxX() x {
	return s.Width
}

// a PeriodicSignal that produces equal regions of +unitY then -unitY, repeating with Cycle width.
type Square struct {
	Cycle x
}

func (s Square) property(p x) y {
	if p%s.Cycle >= s.Cycle/2 {
		return -unitY
	} else {
		return unitY
	}
}

func (s Square) Period() x {
	return s.Cycle
}

// a Signal which ramps from zero to unitY over a Period width.
type RampUp struct {
	Period x
}

func (s RampUp) property(p x) y {
	if p < 0 {
		return 0
	} else if p > s.Period {
		return unitY
	} else {
		return y(x(unitY) / s.Period * p)
	}
}

// a Signal wcich ramps from unitY to zero, over a Period width.
type RampDown struct {
	Period x
}

func (s RampDown) property(p x) y {
	if p < 0 {
		return unitY
	} else if p > s.Period {
		return 0
	} else {
		return y(x(unitY) / s.Period * (s.Period - p))
	}
}

// a Signal that returns +unitY for positive x and zero for negative.
type Heavyside struct {
}

func (s Heavyside) property(p x) y {
	if p < 0 {
		return 0
	}
	return unitY
}

// a Signal that smoothly transitions from 0 to +unitY.
// with a maximium gradient (first derivative) at x=0, of Steepness.
type Sigmoid struct {
	Steepness x
}

func (s Sigmoid) property(p x) y {
	return y(unitYfloat64 / (1 + math.Exp(-float64(p)/float64(s.Steepness))))
}

<|endoftext|>


<commit_before>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package fs

import (
	"fmt"
	"sort"
	"sync"

	"github.com/googlecloudplatform/gcsfuse/fs/inode"
	"github.com/jacobsa/fuse"
	"github.com/jacobsa/fuse/fuseops"
	"github.com/jacobsa/fuse/fuseutil"
	"github.com/jacobsa/gcloud/gcs"
	"github.com/jacobsa/gcloud/syncutil"
	"golang.org/x/net/context"
)

// State required for reading from directories.
type dirHandle struct {
	/////////////////////////
	// Constant data
	/////////////////////////

	in           *inode.DirInode
	implicitDirs bool

	/////////////////////////
	// Mutable state
	/////////////////////////

	Mu syncutil.InvariantMutex

	// All entries in the directory. Populated the first time we need one.
	//
	// INVARIANT: For each i, entries[i+1].Offset == entries[i].Offset + 1
	//
	// GUARDED_BY(Mu)
	entries []fuseutil.Dirent

	// Has entries yet been populated?
	//
	// INVARIANT: If !entriesValid, then len(entries) == 0
	//
	// GUARDED_BY(Mu)
	entriesValid bool
}

// Create a directory handle that obtains listings from the supplied inode.
func newDirHandle(
	in *inode.DirInode,
	implicitDirs bool) (dh *dirHandle) {
	// Set up the basic struct.
	dh = &dirHandle{
		in:           in,
		implicitDirs: implicitDirs,
	}

	// Set up invariant checking.
	dh.Mu = syncutil.NewInvariantMutex(dh.checkInvariants)

	return
}

////////////////////////////////////////////////////////////////////////
// Helpers
////////////////////////////////////////////////////////////////////////

// Dirents, sorted by name.
type sortedDirents []fuseutil.Dirent

func (p sortedDirents) Len() int           { return len(p) }
func (p sortedDirents) Less(i, j int) bool { return p[i].Name < p[j].Name }
func (p sortedDirents) Swap(i, j int)      { p[i], p[j] = p[j], p[i] }

func (dh *dirHandle) checkInvariants() {
	// INVARIANT: For each i, entries[i+1].Offset == entries[i].Offset + 1
	for i := 0; i < len(dh.entries)-1; i++ {
		if !(dh.entries[i+1].Offset == dh.entries[i].Offset+1) {
			panic(
				fmt.Sprintf(
					"Unexpected offset sequence: %v, %v",
					dh.entries[i].Offset,
					dh.entries[i+1].Offset))
		}
	}

	// INVARIANT: If !entriesValid, then len(entries) == 0
	if !dh.entriesValid && len(dh.entries) != 0 {
		panic("Unexpected non-empty entries slice")
	}
}

// Read some entries from the directory inode. Return newTok == "" (possibly
// with a non-empty list of entries) when the end of the directory has been
// hit.
//
// The contents of the entries' Offset fields are undefined. This function
// always behaves as if implicit directories are defined; see notes on
// DirInode.ReadEntries.
//
// LOCKS_EXCLUDED(in)
func readSomeEntries(
	ctx context.Context,
	in *inode.DirInode,
	tok string) (entries []fuseutil.Dirent, newTok string, err error) {
	in.Lock()
	defer in.Unlock()

	entries, newTok, err = in.ReadEntries(ctx, tok)
	if err != nil {
		err = fmt.Errorf("ReadEntries: %v", err)
		return
	}

	// Return a bogus inode ID for each entry, but not the root inode ID.
	//
	// NOTE(jacobsa): As far as I can tell this is harmless. Minting and
	// returning a real inode ID is difficult because fuse does not count
	// readdir as an operation that increases the inode ID's lookup count and
	// we therefore don't get a forget for it later, but we would like to not
	// have to remember every inode ID that we've ever minted for readdir.
	//
	// If it turns out this is not harmless, we'll need to switch to something
	// like inode IDs based on (object name, generation) hashes. But then what
	// about the birthday problem? And more importantly, what about our
	// semantic of not minting a new inode ID when the generation changes due
	// to a local action?
	for i, _ := range entries {
		entries[i].Inode = fuseops.RootInodeID + 1
	}

	return
}

// Read all entries for the directory, making no effort to deal with
// conflicting names or the lack of implicit directories (see notes on
// DirInode.ReadEntries).
//
// Write entries to the supplied channel, without closing. Entry Offset fields
// have unspecified contents.
//
// LOCKS_EXCLUDED(in)
func readEntries(
	ctx context.Context,
	in *inode.DirInode,
	entries chan<- fuseutil.Dirent) (err error) {
	var tok string
	for {
		// Read a batch.
		var batch []fuseutil.Dirent

		batch, tok, err = readSomeEntries(ctx, in, tok)
		if err != nil {
			return
		}

		// Write each to the channel.
		for _, e := range batch {
			select {
			case <-ctx.Done():
				err = ctx.Err()
				return

			case entries <- e:
			}
		}

		// Are we done?
		if tok == "" {
			break
		}
	}

	return
}

// Filter out entries with type DT_Directory for which in.LookUpChild does not
// return a directory. This can be used to implicit directories without a
// matching backing object from the output of DirInode.ReadDir, which always
// behaves as if implicit directories are enabled.
//
// LOCKS_EXCLUDED(in)
func filterMissingDirectories(
	ctx context.Context,
	in *inode.DirInode,
	entriesIn <-chan fuseutil.Dirent,
	entriesOut chan<- fuseutil.Dirent) (err error) {
	for e := range entriesIn {
		// If this is a directory, confirm it actually exists.
		if e.Type == fuseutil.DT_Directory {
			var o *gcs.Object
			in.Lock()
			o, err = in.LookUpChild(ctx, e.Name)
			defer in.Unlock()
			if err != nil {
				err = fmt.Errorf("LookUpChild: %v", err)
				return
			}

			// Skip this entry if the result is not an extant directory.
			if o == nil || !isDirName(o.Name) {
				continue
			}
		}

		// Pass on the entry.
		select {
		case <-ctx.Done():
			err = ctx.Err()
			return

		case entriesOut <- e:
		}
	}

	return
}

// Resolve name conflicts between file objects and directory objects (e.g. the
// objects "foo/bar" and "foo/bar/") by appending U+000A, which is illegal in
// GCS object names, to conflicting file names.
//
// Input must be sorted by name.
func fixConflictingNames(entries []fuseutil.Dirent) (err error) {
	// Sanity check.
	if !sort.IsSorted(sortedDirents(entries)) {
		err = fmt.Errorf("Expected sorted input")
		return
	}

	// Examine each adjacent pair of names.
	for i, _ := range entries {
		e := &entries[i]

		// Find the previous entry.
		if i == 0 {
			continue
		}

		prev := &entries[i-1]

		// Does the pair have matching names?
		if e.Name != prev.Name {
			continue
		}

		// Repair whichever is the file, remembering that there's no way to get any
		// other type in the mix.
		if e.Type == fuseutil.DT_File {
			e.Name += inode.ConflictingFileNameSuffix
		} else {
			if prev.Type != fuseutil.DT_File {
				panic(fmt.Sprintf("Unexpected type for entry: %v", prev))
			}
			prev.Name += inode.ConflictingFileNameSuffix
		}
	}

	return
}

// LOCKS_EXCLUDED(dh.in)
func (dh *dirHandle) readAllEntries(
	ctx context.Context) (entries []fuseutil.Dirent, err error) {
	b := syncutil.NewBundle(ctx)

	// Read entries into a channel.
	unfiltered := make(chan fuseutil.Dirent, 100)
	b.Add(func(ctx context.Context) (err error) {
		defer close(unfiltered)
		err = readEntries(ctx, dh.in, unfiltered)
		return
	})

	// If implicit directories are disabled, filter out entries for
	// sub-directories without a matching backing object.
	var filtered chan fuseutil.Dirent
	if dh.implicitDirs {
		filtered = unfiltered
	} else {
		filtered = make(chan fuseutil.Dirent, 100)

		var wg sync.WaitGroup
		f := func(ctx context.Context) error {
			defer wg.Done()
			return filterMissingDirectories(ctx, dh.in, unfiltered, filtered)
		}

		// Bound the parallelism with which we call the inode.
		const filterWorkers = 32
		for i := 0; i < filterWorkers; i++ {
			wg.Add(1)
			b.Add(f)
		}

		go func() {
			wg.Wait()
			close(filtered)
		}()
	}

	// Accumulate entries into the slice.
	b.Add(func(ctx context.Context) (err error) {
		for e := range filtered {
			entries = append(entries, e)
		}

		return
	})

	// Wait.
	err = b.Join()
	if err != nil {
		return
	}

	// Ensure that the entries are sorted, for use in fixConflictingNames
	// below.
	sort.Sort(sortedDirents(entries))

	// Fix name conflicts.
	err = fixConflictingNames(entries)
	if err != nil {
		err = fmt.Errorf("fixConflictingNames: %v", err)
		return
	}

	// Fix up offset fields.
	for i := 0; i < len(entries); i++ {
		entries[i].Offset = fuseops.DirOffset(i) + 1
	}

	return
}

// LOCKS_REQUIRED(dh.Mu)
// LOCKS_EXCLUDED(dh.in)
func (dh *dirHandle) ensureEntries(ctx context.Context) (err error) {
	// Read entries.
	var entries []fuseutil.Dirent
	entries, err = dh.readAllEntries(ctx)
	if err != nil {
		err = fmt.Errorf("readAllEntries: %v", err)
		return
	}

	// Update state.
	dh.entries = entries
	dh.entriesValid = true

	return
}

////////////////////////////////////////////////////////////////////////
// Public interface
////////////////////////////////////////////////////////////////////////

// Handle a request to read from the directory, without responding.
//
// Special case: we assume that a zero offset indicates that rewinddir has been
// called (since fuse gives us no way to intercept and know for sure), and
// start the listing process over again.
//
// LOCKS_REQUIRED(dh.Mu)
// LOCKS_EXCLUDED(du.in)
func (dh *dirHandle) ReadDir(
	op *fuseops.ReadDirOp) (err error) {
	// If the request is for offset zero, we assume that either this is the first
	// call or rewinddir has been called. Reset state.
	if op.Offset == 0 {
		dh.entries = nil
		dh.entriesValid = false
	}

	// Do we need to read entries from GCS?
	if !dh.entriesValid {
		err = dh.ensureEntries(op.Context())
		if err != nil {
			return
		}
	}

	// Is the offset past the end of what we have buffered? If so, this must be
	// an invalid seekdir according to posix.
	index := int(op.Offset)
	if index > len(dh.entries) {
		err = fuse.EINVAL
		return
	}

	// We copy out entries until we run out of entries or space.
	for i := index; i < len(dh.entries); i++ {
		op.Data = fuseutil.AppendDirent(op.Data, dh.entries[i])
		if len(op.Data) > op.Size {
			op.Data = op.Data[:op.Size]
			break
		}
	}

	return
}
<commit_msg>Fixed a deadlock in filterMissingDirectories.<commit_after>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package fs

import (
	"fmt"
	"sort"
	"sync"

	"github.com/googlecloudplatform/gcsfuse/fs/inode"
	"github.com/jacobsa/fuse"
	"github.com/jacobsa/fuse/fuseops"
	"github.com/jacobsa/fuse/fuseutil"
	"github.com/jacobsa/gcloud/gcs"
	"github.com/jacobsa/gcloud/syncutil"
	"golang.org/x/net/context"
)

// State required for reading from directories.
type dirHandle struct {
	/////////////////////////
	// Constant data
	/////////////////////////

	in           *inode.DirInode
	implicitDirs bool

	/////////////////////////
	// Mutable state
	/////////////////////////

	Mu syncutil.InvariantMutex

	// All entries in the directory. Populated the first time we need one.
	//
	// INVARIANT: For each i, entries[i+1].Offset == entries[i].Offset + 1
	//
	// GUARDED_BY(Mu)
	entries []fuseutil.Dirent

	// Has entries yet been populated?
	//
	// INVARIANT: If !entriesValid, then len(entries) == 0
	//
	// GUARDED_BY(Mu)
	entriesValid bool
}

// Create a directory handle that obtains listings from the supplied inode.
func newDirHandle(
	in *inode.DirInode,
	implicitDirs bool) (dh *dirHandle) {
	// Set up the basic struct.
	dh = &dirHandle{
		in:           in,
		implicitDirs: implicitDirs,
	}

	// Set up invariant checking.
	dh.Mu = syncutil.NewInvariantMutex(dh.checkInvariants)

	return
}

////////////////////////////////////////////////////////////////////////
// Helpers
////////////////////////////////////////////////////////////////////////

// Dirents, sorted by name.
type sortedDirents []fuseutil.Dirent

func (p sortedDirents) Len() int           { return len(p) }
func (p sortedDirents) Less(i, j int) bool { return p[i].Name < p[j].Name }
func (p sortedDirents) Swap(i, j int)      { p[i], p[j] = p[j], p[i] }

func (dh *dirHandle) checkInvariants() {
	// INVARIANT: For each i, entries[i+1].Offset == entries[i].Offset + 1
	for i := 0; i < len(dh.entries)-1; i++ {
		if !(dh.entries[i+1].Offset == dh.entries[i].Offset+1) {
			panic(
				fmt.Sprintf(
					"Unexpected offset sequence: %v, %v",
					dh.entries[i].Offset,
					dh.entries[i+1].Offset))
		}
	}

	// INVARIANT: If !entriesValid, then len(entries) == 0
	if !dh.entriesValid && len(dh.entries) != 0 {
		panic("Unexpected non-empty entries slice")
	}
}

// Read some entries from the directory inode. Return newTok == "" (possibly
// with a non-empty list of entries) when the end of the directory has been
// hit.
//
// The contents of the entries' Offset fields are undefined. This function
// always behaves as if implicit directories are defined; see notes on
// DirInode.ReadEntries.
//
// LOCKS_EXCLUDED(in)
func readSomeEntries(
	ctx context.Context,
	in *inode.DirInode,
	tok string) (entries []fuseutil.Dirent, newTok string, err error) {
	in.Lock()
	defer in.Unlock()

	entries, newTok, err = in.ReadEntries(ctx, tok)
	if err != nil {
		err = fmt.Errorf("ReadEntries: %v", err)
		return
	}

	// Return a bogus inode ID for each entry, but not the root inode ID.
	//
	// NOTE(jacobsa): As far as I can tell this is harmless. Minting and
	// returning a real inode ID is difficult because fuse does not count
	// readdir as an operation that increases the inode ID's lookup count and
	// we therefore don't get a forget for it later, but we would like to not
	// have to remember every inode ID that we've ever minted for readdir.
	//
	// If it turns out this is not harmless, we'll need to switch to something
	// like inode IDs based on (object name, generation) hashes. But then what
	// about the birthday problem? And more importantly, what about our
	// semantic of not minting a new inode ID when the generation changes due
	// to a local action?
	for i, _ := range entries {
		entries[i].Inode = fuseops.RootInodeID + 1
	}

	return
}

// Read all entries for the directory, making no effort to deal with
// conflicting names or the lack of implicit directories (see notes on
// DirInode.ReadEntries).
//
// Write entries to the supplied channel, without closing. Entry Offset fields
// have unspecified contents.
//
// LOCKS_EXCLUDED(in)
func readEntries(
	ctx context.Context,
	in *inode.DirInode,
	entries chan<- fuseutil.Dirent) (err error) {
	var tok string
	for {
		// Read a batch.
		var batch []fuseutil.Dirent

		batch, tok, err = readSomeEntries(ctx, in, tok)
		if err != nil {
			return
		}

		// Write each to the channel.
		for _, e := range batch {
			select {
			case <-ctx.Done():
				err = ctx.Err()
				return

			case entries <- e:
			}
		}

		// Are we done?
		if tok == "" {
			break
		}
	}

	return
}

// Filter out entries with type DT_Directory for which in.LookUpChild does not
// return a directory. This can be used to implicit directories without a
// matching backing object from the output of DirInode.ReadDir, which always
// behaves as if implicit directories are enabled.
//
// LOCKS_EXCLUDED(in)
func filterMissingDirectories(
	ctx context.Context,
	in *inode.DirInode,
	entriesIn <-chan fuseutil.Dirent,
	entriesOut chan<- fuseutil.Dirent) (err error) {
	for e := range entriesIn {
		// If this is a directory, confirm it actually exists.
		if e.Type == fuseutil.DT_Directory {
			var o *gcs.Object
			in.Lock()
			o, err = in.LookUpChild(ctx, e.Name)
			in.Unlock()
			if err != nil {
				err = fmt.Errorf("LookUpChild: %v", err)
				return
			}

			// Skip this entry if the result is not an extant directory.
			if o == nil || !isDirName(o.Name) {
				continue
			}
		}

		// Pass on the entry.
		select {
		case <-ctx.Done():
			err = ctx.Err()
			return

		case entriesOut <- e:
		}
	}

	return
}

// Resolve name conflicts between file objects and directory objects (e.g. the
// objects "foo/bar" and "foo/bar/") by appending U+000A, which is illegal in
// GCS object names, to conflicting file names.
//
// Input must be sorted by name.
func fixConflictingNames(entries []fuseutil.Dirent) (err error) {
	// Sanity check.
	if !sort.IsSorted(sortedDirents(entries)) {
		err = fmt.Errorf("Expected sorted input")
		return
	}

	// Examine each adjacent pair of names.
	for i, _ := range entries {
		e := &entries[i]

		// Find the previous entry.
		if i == 0 {
			continue
		}

		prev := &entries[i-1]

		// Does the pair have matching names?
		if e.Name != prev.Name {
			continue
		}

		// Repair whichever is the file, remembering that there's no way to get any
		// other type in the mix.
		if e.Type == fuseutil.DT_File {
			e.Name += inode.ConflictingFileNameSuffix
		} else {
			if prev.Type != fuseutil.DT_File {
				panic(fmt.Sprintf("Unexpected type for entry: %v", prev))
			}
			prev.Name += inode.ConflictingFileNameSuffix
		}
	}

	return
}

// LOCKS_EXCLUDED(dh.in)
func (dh *dirHandle) readAllEntries(
	ctx context.Context) (entries []fuseutil.Dirent, err error) {
	b := syncutil.NewBundle(ctx)

	// Read entries into a channel.
	unfiltered := make(chan fuseutil.Dirent, 100)
	b.Add(func(ctx context.Context) (err error) {
		defer close(unfiltered)
		err = readEntries(ctx, dh.in, unfiltered)
		return
	})

	// If implicit directories are disabled, filter out entries for
	// sub-directories without a matching backing object.
	var filtered chan fuseutil.Dirent
	if dh.implicitDirs {
		filtered = unfiltered
	} else {
		filtered = make(chan fuseutil.Dirent, 100)

		var wg sync.WaitGroup
		f := func(ctx context.Context) error {
			defer wg.Done()
			return filterMissingDirectories(ctx, dh.in, unfiltered, filtered)
		}

		// Bound the parallelism with which we call the inode.
		const filterWorkers = 32
		for i := 0; i < filterWorkers; i++ {
			wg.Add(1)
			b.Add(f)
		}

		go func() {
			wg.Wait()
			close(filtered)
		}()
	}

	// Accumulate entries into the slice.
	b.Add(func(ctx context.Context) (err error) {
		for e := range filtered {
			entries = append(entries, e)
		}

		return
	})

	// Wait.
	err = b.Join()
	if err != nil {
		return
	}

	// Ensure that the entries are sorted, for use in fixConflictingNames
	// below.
	sort.Sort(sortedDirents(entries))

	// Fix name conflicts.
	err = fixConflictingNames(entries)
	if err != nil {
		err = fmt.Errorf("fixConflictingNames: %v", err)
		return
	}

	// Fix up offset fields.
	for i := 0; i < len(entries); i++ {
		entries[i].Offset = fuseops.DirOffset(i) + 1
	}

	return
}

// LOCKS_REQUIRED(dh.Mu)
// LOCKS_EXCLUDED(dh.in)
func (dh *dirHandle) ensureEntries(ctx context.Context) (err error) {
	// Read entries.
	var entries []fuseutil.Dirent
	entries, err = dh.readAllEntries(ctx)
	if err != nil {
		err = fmt.Errorf("readAllEntries: %v", err)
		return
	}

	// Update state.
	dh.entries = entries
	dh.entriesValid = true

	return
}

////////////////////////////////////////////////////////////////////////
// Public interface
////////////////////////////////////////////////////////////////////////

// Handle a request to read from the directory, without responding.
//
// Special case: we assume that a zero offset indicates that rewinddir has been
// called (since fuse gives us no way to intercept and know for sure), and
// start the listing process over again.
//
// LOCKS_REQUIRED(dh.Mu)
// LOCKS_EXCLUDED(du.in)
func (dh *dirHandle) ReadDir(
	op *fuseops.ReadDirOp) (err error) {
	// If the request is for offset zero, we assume that either this is the first
	// call or rewinddir has been called. Reset state.
	if op.Offset == 0 {
		dh.entries = nil
		dh.entriesValid = false
	}

	// Do we need to read entries from GCS?
	if !dh.entriesValid {
		err = dh.ensureEntries(op.Context())
		if err != nil {
			return
		}
	}

	// Is the offset past the end of what we have buffered? If so, this must be
	// an invalid seekdir according to posix.
	index := int(op.Offset)
	if index > len(dh.entries) {
		err = fuse.EINVAL
		return
	}

	// We copy out entries until we run out of entries or space.
	for i := index; i < len(dh.entries); i++ {
		op.Data = fuseutil.AppendDirent(op.Data, dh.entries[i])
		if len(op.Data) > op.Size {
			op.Data = op.Data[:op.Size]
			break
		}
	}

	return
}
<|endoftext|>


<commit_before>package github

import (
	"crypto/hmac"
	"crypto/sha1"
	"encoding/hex"
	"encoding/json"
	"io/ioutil"
	"net/http"
	"strings"

	"github.com/gin-gonic/gin"
	"github.com/prometheus/client_golang/prometheus"
	"github.com/rs/zerolog/log"
)

// EventHandler handles http events for Github integration
type EventHandler interface {
	Handle(*gin.Context)
	HandlePushEvent(PushEvent)
}

type eventHandlerImpl struct {
	eventsChannel                chan PushEvent
	githubWebhookSecret          string
	prometheusInboundEventTotals *prometheus.CounterVec
}

// NewGithubEventHandler returns a github.EventHandler to handle incoming webhook events
func NewGithubEventHandler(eventsChannel chan PushEvent, githubWebhookSecret string, prometheusInboundEventTotals *prometheus.CounterVec) EventHandler {
	return &eventHandlerImpl{
		eventsChannel:                eventsChannel,
		githubWebhookSecret:          githubWebhookSecret,
		prometheusInboundEventTotals: prometheusInboundEventTotals,
	}
}

func (h *eventHandlerImpl) Handle(c *gin.Context) {

	// https://developer.github.com/webhooks/
	eventType := c.GetHeader("X-Github-Event")
	h.prometheusInboundEventTotals.With(prometheus.Labels{"event": eventType, "source": "github"}).Inc()

	body, err := ioutil.ReadAll(c.Request.Body)
	if err != nil {
		log.Error().Err(err).Msg("Reading body from Github webhook failed")
		c.String(http.StatusInternalServerError, "Reading body from Github webhook failed")
		return
	}

	// https://developer.github.com/webhooks/securing/
	signature := c.GetHeader("X-Hub-Signature")
	signature = strings.Replace(signature, "sha1=", "", 1)
	actualMAC, err := hex.DecodeString(signature)
	if err != nil {
		log.Error().Err(err).Msg("Decoding hexadecimal X-Hub-Signature to byte array failed")
	}

	// calculate expected MAC
	mac := hmac.New(sha1.New, []byte(h.githubWebhookSecret))
	mac.Write(body)
	expectedMAC := mac.Sum(nil)

	// compare actual and expected MAC
	if hmac.Equal(actualMAC, expectedMAC) {
		log.Debug().Str("expectedMAC", hex.EncodeToString(expectedMAC)).Str("actualMAC", hex.EncodeToString(actualMAC)).Msg("Valid HMAC signature")
	} else {
		log.Warn().Str("expectedMAC", hex.EncodeToString(expectedMAC)).Str("actualMAC", hex.EncodeToString(actualMAC)).Msg("Invalid HMAC signature")
	}

	// unmarshal json body
	var b interface{}
	err = json.Unmarshal(body, &b)
	if err != nil {
		log.Error().Err(err).Str("body", string(body)).Msg("Deserializing body from Github webhook failed")
		c.String(http.StatusInternalServerError, "Deserializing body from Github webhook failed")
		return
	}

	log.Debug().
		Str("method", c.Request.Method).
		Str("url", c.Request.URL.String()).
		Interface("headers", c.Request.Header).
		Interface("body", b).
		Msgf("Received webhook event of type '%v' from GitHub...", eventType)

	switch eventType {
	case "push": // Any Git push to a Repository, including editing tags or branches. Commits via API actions that update references are also counted. This is the default event.

		// unmarshal json body
		var pushEvent PushEvent
		err := json.Unmarshal(body, &pushEvent)
		if err != nil {
			log.Error().Err(err).Str("body", string(body)).Msg("Deserializing body to GithubPushEvent failed")
			return
		}

		h.HandlePushEvent(pushEvent)

	case
		"commit_comment",              // Any time a Commit is commented on.
		"create",                      // Any time a Branch or Tag is created.
		"delete",                      // Any time a Branch or Tag is deleted.
		"deployment",                  // Any time a Repository has a new deployment created from the API.
		"deployment_status",           // Any time a deployment for a Repository has a status update from the API.
		"fork",                        // Any time a Repository is forked.
		"gollum",                      // Any time a Wiki page is updated.
		"installation",                // Any time a GitHub App is installed or uninstalled.
		"installation_repositories",   // Any time a repository is added or removed from an installation.
		"issue_comment",               // Any time a comment on an issue is created, edited, or deleted.
		"issues",                      // Any time an Issue is assigned, unassigned, labeled, unlabeled, opened, edited, milestoned, demilestoned, closed, or reopened.
		"label",                       // Any time a Label is created, edited, or deleted.
		"marketplace_purchase",        // Any time a user purchases, cancels, or changes their GitHub Marketplace plan.
		"member",                      // Any time a User is added or removed as a collaborator to a Repository, or has their permissions modified.
		"membership",                  // Any time a User is added or removed from a team. Organization hooks only.
		"milestone",                   // Any time a Milestone is created, closed, opened, edited, or deleted.
		"organization",                // Any time a user is added, removed, or invited to an Organization. Organization hooks only.
		"org_block",                   // Any time an organization blocks or unblocks a user. Organization hooks only.
		"page_build",                  // Any time a Pages site is built or results in a failed build.
		"project_card",                // Any time a Project Card is created, edited, moved, converted to an issue, or deleted.
		"project_column",              // Any time a Project Column is created, edited, moved, or deleted.
		"project",                     // Any time a Project is created, edited, closed, reopened, or deleted.
		"public",                      // Any time a Repository changes from private to public.
		"pull_request_review_comment", // Any time a comment on a pull request's unified diff is created, edited, or deleted (in the Files Changed tab).
		"pull_request_review",         // Any time a pull request review is submitted, edited, or dismissed.
		"pull_request",                // Any time a pull request is assigned, unassigned, labeled, unlabeled, opened, edited, closed, reopened, or synchronized (updated due to a new push in the branch that the pull request is tracking). Also any time a pull request review is requested, or a review request is removed.
		"repository",                  // Any time a Repository is created, deleted (organization hooks only), made public, or made private.
		"release",                     // Any time a Release is published in a Repository.
		"status",                      // Any time a Repository has a status update from the API
		"team",                        // Any time a team is created, deleted, modified, or added to or removed from a repository. Organization hooks only
		"team_add",                    // Any time a team is added or modified on a Repository.
		"watch",                       // Any time a User stars a Repository.
		"integration_installation_repositories": // ?
		log.Debug().Str("event", eventType).Msgf("Not implemented Github webhook event of type '%v'", eventType)

	default:
		log.Warn().Str("event", eventType).Msgf("Unsupported Github webhook event of type '%v'", eventType)
	}

	c.String(http.StatusOK, "Aye aye!")
}

func (h *eventHandlerImpl) HandlePushEvent(pushEvent PushEvent) {

	log.Debug().Interface("pushEvent", pushEvent).Msgf("Deserialized GitHub push event for repository %v", pushEvent.Repository.FullName)

	// test making api calls for github app in the background
	h.eventsChannel <- pushEvent
}
<commit_msg>verify hmac and fail the request if it's invalid<commit_after>package github

import (
	"crypto/hmac"
	"crypto/sha1"
	"encoding/hex"
	"encoding/json"
	"errors"
	"io/ioutil"
	"net/http"
	"strings"

	"github.com/gin-gonic/gin"
	"github.com/prometheus/client_golang/prometheus"
	"github.com/rs/zerolog/log"
)

// EventHandler handles http events for Github integration
type EventHandler interface {
	Handle(*gin.Context)
	HandlePushEvent(PushEvent)
	HasValidSignature([]byte, string) (bool, error)
}

type eventHandlerImpl struct {
	eventsChannel                chan PushEvent
	githubWebhookSecret          string
	prometheusInboundEventTotals *prometheus.CounterVec
}

// NewGithubEventHandler returns a github.EventHandler to handle incoming webhook events
func NewGithubEventHandler(eventsChannel chan PushEvent, githubWebhookSecret string, prometheusInboundEventTotals *prometheus.CounterVec) EventHandler {
	return &eventHandlerImpl{
		eventsChannel:                eventsChannel,
		githubWebhookSecret:          githubWebhookSecret,
		prometheusInboundEventTotals: prometheusInboundEventTotals,
	}
}

func (h *eventHandlerImpl) Handle(c *gin.Context) {

	// https://developer.github.com/webhooks/
	eventType := c.GetHeader("X-Github-Event")
	h.prometheusInboundEventTotals.With(prometheus.Labels{"event": eventType, "source": "github"}).Inc()

	body, err := ioutil.ReadAll(c.Request.Body)
	if err != nil {
		log.Error().Err(err).Msg("Reading body from Github webhook failed")
		c.String(http.StatusInternalServerError, "Reading body from Github webhook failed")
		return
	}

	// verify hmac signature
	hasValidSignature, err := h.HasValidSignature(body, c.GetHeader("X-Hub-Signature"))
	if err != nil {
		log.Error().Err(err).Msg("Verifying signature from Github webhook failed")
		c.String(http.StatusInternalServerError, "Verifying signature from Github webhook failed")
		return
	}
	if !hasValidSignature {
		log.Warn().Msg("Signature from Github webhook is invalid")
		c.String(http.StatusBadRequest, "Signature from Github webhook is invalid")
		return
	}

	// unmarshal json body
	var b interface{}
	err = json.Unmarshal(body, &b)
	if err != nil {
		log.Error().Err(err).Str("body", string(body)).Msg("Deserializing body from Github webhook failed")
		c.String(http.StatusInternalServerError, "Deserializing body from Github webhook failed")
		return
	}

	log.Debug().
		Str("method", c.Request.Method).
		Str("url", c.Request.URL.String()).
		Interface("headers", c.Request.Header).
		Interface("body", b).
		Msgf("Received webhook event of type '%v' from GitHub...", eventType)

	switch eventType {
	case "push": // Any Git push to a Repository, including editing tags or branches. Commits via API actions that update references are also counted. This is the default event.

		// unmarshal json body
		var pushEvent PushEvent
		err := json.Unmarshal(body, &pushEvent)
		if err != nil {
			log.Error().Err(err).Str("body", string(body)).Msg("Deserializing body to GithubPushEvent failed")
			return
		}

		h.HandlePushEvent(pushEvent)

	case
		"commit_comment",              // Any time a Commit is commented on.
		"create",                      // Any time a Branch or Tag is created.
		"delete",                      // Any time a Branch or Tag is deleted.
		"deployment",                  // Any time a Repository has a new deployment created from the API.
		"deployment_status",           // Any time a deployment for a Repository has a status update from the API.
		"fork",                        // Any time a Repository is forked.
		"gollum",                      // Any time a Wiki page is updated.
		"installation",                // Any time a GitHub App is installed or uninstalled.
		"installation_repositories",   // Any time a repository is added or removed from an installation.
		"issue_comment",               // Any time a comment on an issue is created, edited, or deleted.
		"issues",                      // Any time an Issue is assigned, unassigned, labeled, unlabeled, opened, edited, milestoned, demilestoned, closed, or reopened.
		"label",                       // Any time a Label is created, edited, or deleted.
		"marketplace_purchase",        // Any time a user purchases, cancels, or changes their GitHub Marketplace plan.
		"member",                      // Any time a User is added or removed as a collaborator to a Repository, or has their permissions modified.
		"membership",                  // Any time a User is added or removed from a team. Organization hooks only.
		"milestone",                   // Any time a Milestone is created, closed, opened, edited, or deleted.
		"organization",                // Any time a user is added, removed, or invited to an Organization. Organization hooks only.
		"org_block",                   // Any time an organization blocks or unblocks a user. Organization hooks only.
		"page_build",                  // Any time a Pages site is built or results in a failed build.
		"project_card",                // Any time a Project Card is created, edited, moved, converted to an issue, or deleted.
		"project_column",              // Any time a Project Column is created, edited, moved, or deleted.
		"project",                     // Any time a Project is created, edited, closed, reopened, or deleted.
		"public",                      // Any time a Repository changes from private to public.
		"pull_request_review_comment", // Any time a comment on a pull request's unified diff is created, edited, or deleted (in the Files Changed tab).
		"pull_request_review",         // Any time a pull request review is submitted, edited, or dismissed.
		"pull_request",                // Any time a pull request is assigned, unassigned, labeled, unlabeled, opened, edited, closed, reopened, or synchronized (updated due to a new push in the branch that the pull request is tracking). Also any time a pull request review is requested, or a review request is removed.
		"repository",                  // Any time a Repository is created, deleted (organization hooks only), made public, or made private.
		"release",                     // Any time a Release is published in a Repository.
		"status",                      // Any time a Repository has a status update from the API
		"team",                        // Any time a team is created, deleted, modified, or added to or removed from a repository. Organization hooks only
		"team_add",                    // Any time a team is added or modified on a Repository.
		"watch",                       // Any time a User stars a Repository.
		"integration_installation_repositories": // ?
		log.Debug().Str("event", eventType).Msgf("Not implemented Github webhook event of type '%v'", eventType)

	default:
		log.Warn().Str("event", eventType).Msgf("Unsupported Github webhook event of type '%v'", eventType)
	}

	c.String(http.StatusOK, "Aye aye!")
}

func (h *eventHandlerImpl) HandlePushEvent(pushEvent PushEvent) {

	log.Debug().Interface("pushEvent", pushEvent).Msgf("Deserialized GitHub push event for repository %v", pushEvent.Repository.FullName)

	// test making api calls for github app in the background
	h.eventsChannel <- pushEvent
}

func (h *eventHandlerImpl) HasValidSignature(body []byte, signatureHeader string) (bool, error) {

	// https://developer.github.com/webhooks/securing/
	signature := strings.Replace(signatureHeader, "sha1=", "", 1)
	actualMAC, err := hex.DecodeString(signature)
	if err != nil {
		return false, errors.New("Decoding hexadecimal X-Hub-Signature to byte array failed")
	}

	// calculate expected MAC
	mac := hmac.New(sha1.New, []byte(h.githubWebhookSecret))
	mac.Write(body)
	expectedMAC := mac.Sum(nil)

	// compare actual and expected MAC
	if hmac.Equal(actualMAC, expectedMAC) {
		return true, nil
	}

	log.Debug().
		Str("expectedMAC", hex.EncodeToString(expectedMAC)).
		Str("actualMAC", hex.EncodeToString(actualMAC)).
		Msg("Expected and actual MAC do not match")

	return false, nil
}
<|endoftext|>


<commit_before>// Copyright 2011 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package runtime

import "unsafe"

const (
	_ESRCH     = 3
	_ETIMEDOUT = 60

	// From NetBSD's <sys/time.h>
	_CLOCK_REALTIME  = 0
	_CLOCK_VIRTUAL   = 1
	_CLOCK_PROF      = 2
	_CLOCK_MONOTONIC = 3
)

var sigset_all = sigset{[4]uint32{^uint32(0), ^uint32(0), ^uint32(0), ^uint32(0)}}

// From NetBSD's <sys/sysctl.h>
const (
	_CTL_HW  = 6
	_HW_NCPU = 3
)

func getncpu() int32 {
	mib := [2]uint32{_CTL_HW, _HW_NCPU}
	out := uint32(0)
	nout := unsafe.Sizeof(out)
	ret := sysctl(&mib[0], 2, (*byte)(unsafe.Pointer(&out)), &nout, nil, 0)
	if ret >= 0 {
		return int32(out)
	}
	return 1
}

//go:nosplit
func semacreate() uintptr {
	return 1
}

//go:nosplit
func semasleep(ns int64) int32 {
	_g_ := getg()

	// Compute sleep deadline.
	var tsp *timespec
	if ns >= 0 {
		var ts timespec
		var nsec int32
		ns += nanotime()
		ts.set_sec(timediv(ns, 1000000000, &nsec))
		ts.set_nsec(nsec)
		tsp = &ts
	}

	for {
		v := atomicload(&_g_.m.waitsemacount)
		if v > 0 {
			if cas(&_g_.m.waitsemacount, v, v-1) {
				return 0 // semaphore acquired
			}
			continue
		}

		// Sleep until unparked by semawakeup or timeout.
		ret := lwp_park(tsp, 0, unsafe.Pointer(&_g_.m.waitsemacount), nil)
		if ret == _ETIMEDOUT {
			return -1
		}
	}
}

//go:nosplit
func semawakeup(mp *m) {
	xadd(&mp.waitsemacount, 1)
	// From NetBSD's _lwp_unpark(2) manual:
	// "If the target LWP is not currently waiting, it will return
	// immediately upon the next call to _lwp_park()."
	ret := lwp_unpark(int32(mp.procid), unsafe.Pointer(&mp.waitsemacount))
	if ret != 0 && ret != _ESRCH {
		// semawakeup can be called on signal stack.
		systemstack(func() {
			print("thrwakeup addr=", &mp.waitsemacount, " sem=", mp.waitsemacount, " ret=", ret, "\n")
		})
	}
}

// May run with m.p==nil, so write barriers are not allowed.
//go:nowritebarrier
func newosproc(mp *m, stk unsafe.Pointer) {
	if false {
		print("newosproc stk=", stk, " m=", mp, " g=", mp.g0, " id=", mp.id, "/", int32(mp.tls[0]), " ostk=", &mp, "\n")
	}

	mp.tls[0] = uintptr(mp.id) // so 386 asm can find it

	var uc ucontextt
	getcontext(unsafe.Pointer(&uc))

	uc.uc_flags = _UC_SIGMASK | _UC_CPU
	uc.uc_link = nil
	uc.uc_sigmask = sigset_all

	lwp_mcontext_init(&uc.uc_mcontext, stk, mp, mp.g0, funcPC(mstart))

	ret := lwp_create(unsafe.Pointer(&uc), 0, unsafe.Pointer(&mp.procid))
	if ret < 0 {
		print("runtime: failed to create new OS thread (have ", mcount()-1, " already; errno=", -ret, ")\n")
		throw("runtime.newosproc")
	}
}

func osinit() {
	ncpu = getncpu()
}

var urandom_dev = []byte("/dev/urandom\x00")

//go:nosplit
func getRandomData(r []byte) {
	fd := open(&urandom_dev[0], 0 /* O_RDONLY */, 0)
	n := read(fd, unsafe.Pointer(&r[0]), int32(len(r)))
	closefd(fd)
	extendRandom(r, int(n))
}

func goenvs() {
	goenvs_unix()
}

// Called to initialize a new m (including the bootstrap m).
// Called on the parent thread (main thread in case of bootstrap), can allocate memory.
func mpreinit(mp *m) {
	mp.gsignal = malg(32 * 1024)
	mp.gsignal.m = mp
}

func msigsave(mp *m) {
	smask := (*sigset)(unsafe.Pointer(&mp.sigmask))
	if unsafe.Sizeof(*smask) > unsafe.Sizeof(mp.sigmask) {
		throw("insufficient storage for signal mask")
	}
	sigprocmask(_SIG_SETMASK, nil, smask)
}

// Called to initialize a new m (including the bootstrap m).
// Called on the new thread, can not allocate memory.
func minit() {
	_g_ := getg()
	_g_.m.procid = uint64(lwp_self())

	// Initialize signal handling
	signalstack(&_g_.m.gsignal.stack)

	// restore signal mask from m.sigmask and unblock essential signals
	nmask := *(*sigset)(unsafe.Pointer(&_g_.m.sigmask))
	for i := range sigtable {
		if sigtable[i].flags&_SigUnblock != 0 {
			nmask.__bits[(i-1)/32] &^= 1 << ((uint32(i) - 1) & 31)
		}
	}
	sigprocmask(_SIG_SETMASK, &nmask, nil)
}

// Called from dropm to undo the effect of an minit.
func unminit() {
	_g_ := getg()
	smask := (*sigset)(unsafe.Pointer(&_g_.m.sigmask))
	sigprocmask(_SIG_SETMASK, smask, nil)

	signalstack(nil)
}

func memlimit() uintptr {
	return 0
}

func sigtramp()

type sigactiont struct {
	sa_sigaction uintptr
	sa_mask      sigset
	sa_flags     int32
}

func setsig(i int32, fn uintptr, restart bool) {
	var sa sigactiont
	sa.sa_flags = _SA_SIGINFO | _SA_ONSTACK
	if restart {
		sa.sa_flags |= _SA_RESTART
	}
	sa.sa_mask = sigset_all
	if fn == funcPC(sighandler) {
		fn = funcPC(sigtramp)
	}
	sa.sa_sigaction = fn
	sigaction(i, &sa, nil)
}

func setsigstack(i int32) {
	throw("setsigstack")
}

func getsig(i int32) uintptr {
	var sa sigactiont
	sigaction(i, nil, &sa)
	if sa.sa_sigaction == funcPC(sigtramp) {
		return funcPC(sighandler)
	}
	return sa.sa_sigaction
}

func signalstack(s *stack) {
	var st sigaltstackt
	if s == nil {
		st.ss_flags = _SS_DISABLE
	} else {
		st.ss_sp = s.lo
		st.ss_size = s.hi - s.lo
		st.ss_flags = 0
	}
	sigaltstack(&st, nil)
}

func updatesigmask(m sigmask) {
	var mask sigset
	copy(mask.__bits[:], m[:])
	sigprocmask(_SIG_SETMASK, &mask, nil)
}

func unblocksig(sig int32) {
	var mask sigset
	mask.__bits[(sig-1)/32] |= 1 << ((uint32(sig) - 1) & 31)
	sigprocmask(_SIG_UNBLOCK, &mask, nil)
}
<commit_msg>os1_netbsd.go: eliminate signal stuff (for Rumprun)<commit_after>// Copyright 2011 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package runtime

import "unsafe"

const (
	_ESRCH     = 3
	_ETIMEDOUT = 60

	// From NetBSD's <sys/time.h>
	_CLOCK_REALTIME  = 0
	_CLOCK_VIRTUAL   = 1
	_CLOCK_PROF      = 2
	_CLOCK_MONOTONIC = 3
)

var sigset_all = sigset{[4]uint32{^uint32(0), ^uint32(0), ^uint32(0), ^uint32(0)}}

// From NetBSD's <sys/sysctl.h>
const (
	_CTL_HW  = 6
	_HW_NCPU = 3
)

func getncpu() int32 {
	mib := [2]uint32{_CTL_HW, _HW_NCPU}
	out := uint32(0)
	nout := unsafe.Sizeof(out)
	ret := sysctl(&mib[0], 2, (*byte)(unsafe.Pointer(&out)), &nout, nil, 0)
	if ret >= 0 {
		return int32(out)
	}
	return 1
}

//go:nosplit
func semacreate() uintptr {
	return 1
}

//go:nosplit
func semasleep(ns int64) int32 {
	_g_ := getg()

	// Compute sleep deadline.
	var tsp *timespec
	if ns >= 0 {
		var ts timespec
		var nsec int32
		ns += nanotime()
		ts.set_sec(timediv(ns, 1000000000, &nsec))
		ts.set_nsec(nsec)
		tsp = &ts
	}

	for {
		v := atomicload(&_g_.m.waitsemacount)
		if v > 0 {
			if cas(&_g_.m.waitsemacount, v, v-1) {
				return 0 // semaphore acquired
			}
			continue
		}

		// Sleep until unparked by semawakeup or timeout.
		ret := lwp_park(tsp, 0, unsafe.Pointer(&_g_.m.waitsemacount), nil)
		if ret == _ETIMEDOUT {
			return -1
		}
	}
}

//go:nosplit
func semawakeup(mp *m) {
	xadd(&mp.waitsemacount, 1)
	// From NetBSD's _lwp_unpark(2) manual:
	// "If the target LWP is not currently waiting, it will return
	// immediately upon the next call to _lwp_park()."
	ret := lwp_unpark(int32(mp.procid), unsafe.Pointer(&mp.waitsemacount))
	if ret != 0 && ret != _ESRCH {
		// semawakeup can be called on signal stack.
		systemstack(func() {
			print("thrwakeup addr=", &mp.waitsemacount, " sem=", mp.waitsemacount, " ret=", ret, "\n")
		})
	}
}

// May run with m.p==nil, so write barriers are not allowed.
//go:nowritebarrier
func newosproc(mp *m, stk unsafe.Pointer) {
	if false {
		print("newosproc stk=", stk, " m=", mp, " g=", mp.g0, " id=", mp.id, "/", int32(mp.tls[0]), " ostk=", &mp, "\n")
	}

	mp.tls[0] = uintptr(mp.id) // so 386 asm can find it

	var uc ucontextt
	getcontext(unsafe.Pointer(&uc))

	uc.uc_flags = _UC_SIGMASK | _UC_CPU
	uc.uc_link = nil
	uc.uc_sigmask = sigset_all

	lwp_mcontext_init(&uc.uc_mcontext, stk, mp, mp.g0, funcPC(mstart))

	ret := lwp_create(unsafe.Pointer(&uc), 0, unsafe.Pointer(&mp.procid))
	if ret < 0 {
		print("runtime: failed to create new OS thread (have ", mcount()-1, " already; errno=", -ret, ")\n")
		throw("runtime.newosproc")
	}
}

func osinit() {
	ncpu = getncpu()
}

var urandom_dev = []byte("/dev/urandom\x00")

//go:nosplit
func getRandomData(r []byte) {
	fd := open(&urandom_dev[0], 0 /* O_RDONLY */, 0)
	n := read(fd, unsafe.Pointer(&r[0]), int32(len(r)))
	closefd(fd)
	extendRandom(r, int(n))
}

func goenvs() {
	goenvs_unix()
}

// Called to initialize a new m (including the bootstrap m).
// Called on the parent thread (main thread in case of bootstrap), can allocate memory.
func mpreinit(mp *m) {
	mp.gsignal = malg(32 * 1024)
	mp.gsignal.m = mp
}

func msigsave(mp *m) {
}

// Called to initialize a new m (including the bootstrap m).
// Called on the new thread, can not allocate memory.
func minit() {
	_g_ := getg()
	_g_.m.procid = uint64(lwp_self())
}

// Called from dropm to undo the effect of an minit.
func unminit() {
}

func memlimit() uintptr {
	return 0
}

func sigtramp()

func setsig(i int32, fn uintptr, restart bool) {
}

func setsigstack(i int32) {
	throw("setsigstack")
}

func getsig(i int32) uintptr {
}

func signalstack(s *stack) {
}

func updatesigmask(m sigmask) {
}

func unblocksig(sig int32) {
}
<|endoftext|>


<commit_before>/*
Server.

Accepts client connections. Runs predefined processing function
for each incoming request and send the answer back to the client side.

Author: Aleksey Morarash <aleksey.morarash@gmail.com>
Since: 4 Sep 2016
Copyright: 2016, Aleksey Morarash <aleksey.morarash@gmail.com>
*/

package tcpcall

import (
	"bytes"
	"fmt"
	"log"
	"net"
	"sync"
	"tcpcall/pools"
	"tcpcall/proto"
	"time"
)

// Server counter indices.
const (
	// when new TCP connection established
	SC_ACCEPTED = iota
	// failed to accept new incoming connection
	SC_ACCEPT_ERRORS
	// too many incoming connections already exist
	SC_ACCEPT_OVERFLOWS
	// client connection handler creation error
	SC_HANDLER_CREATE_ERRORS
	// new packet received from the client
	SC_PACKET_INPUT
	// too many request processors (incoming packet
	// will be dropped without processing)
	SC_CONCURRENCY_OVERFLOWS
	// we're about sending suspend signal to the client
	SC_SUSPEND_REQUESTED
	// suspend signal not sent due to error
	SC_SUSPEND_REQUEST_ERRORS
	// we're about sending resume signal to the client
	SC_RESUME_REQUESTED
	// resume signal not sent due to error
	SC_RESUME_REQUEST_ERRORS
	// we're about sending packet back to the client
	SC_PACKET_WRITE
	// packet not sent due to error
	SC_PACKET_WRITE_ERRORS
	// new request processing worker (goroutine) created
	SC_WORKER_ADDED
	// request processing worker (goroutine) finished
	SC_WORKER_REMOVED
	// failed to decode packet received from the client
	SC_PACKET_DECODE_ERRORS
	// sync request received
	SC_REQUESTS
	// async request received
	SC_CASTS
	// unknown request received (and left without processing)
	SC_UNKNOWN
	// we're about sending uplink cast packet to the client
	SC_UPLINK_CAST_REQUESTED
	// uplink cast packet not sent due to error
	SC_UPLINK_CAST_ERRORS
	SC_COUNT // special value - count of all counters
)

// Server state
type Server struct {
	// IP address and TCP port number used for listening
	// incoming connections (colon separated).
	bindAddr string
	// Configuration used for server instance creation.
	config ServerConf
	// Server socket.
	socket net.Listener
	// List of established client connections.
	connections map[*ServerConn]bool
	// Controls access to client connections map.
	lock sync.Mutex
	// Set to truth when server is about to terminate.
	stopFlag bool
	// Counters array
	counters   []int
	countersMu sync.RWMutex
}

// Server configuration
type ServerConf struct {
	// TCP port number to listen
	PortNumber uint16
	// Maximum simultaneous connections to accept
	MaxConnections int
	// Maximum request processing concurrency per connection.
	Concurrency int
	// Max request packet size, in bytes. 0 means no limit.
	MaxRequestSize int
	// Minimum flush period for socket writer
	MinFlushPeriod time.Duration
	// Socket write buffer size
	WriteBufferSize int
	// Request processing function.
	// Each request will be processed in parallel with others.
	RequestCallback func([]byte) []byte
	// Asynchronous request (cast) processing function.
	// Each request will be processed in parallel with others.
	CastCallback func([]byte)
	// Called with encoded network packet when there are no workers
	// left to process the packet.
	OnDrop func([]byte)
	// Duration for suspend signals
	SuspendDuration time.Duration
	// Enable debug logging or not
	Trace bool
}

// Server instance information returned by Info() method.
// Added mostly for debugging.
type ServerInfo struct {
	// Configuration used to create Server instance
	Config ServerConf
	// Bind address and port number actually used
	BindAddr string
	// Truth means server is terminating (or terminated)
	StopFlag bool
	// Count of established connections from clients
	Connections int
	// Counters array
	Counters []int
}

// Connection handler state
type ServerConn struct {
	// IP address and TCP port number of the other
	// side of connection (client side).
	peer string
	// Message oriented connection
	conn *MsgConn
	// Number of currently running workers
	// (actually worker is a goroutine handling one
	// request received from the client).
	workers int
	// Controls access to conn object.
	lock sync.Mutex
	// Link to the Server instance this conection
	// is originated from.
	server *Server
}

// Start new server.
func Listen(conf ServerConf) (*Server, error) {
	bindAddr := fmt.Sprintf(":%d", conf.PortNumber)
	socket, err := net.Listen("tcp", bindAddr)
	if err == nil {
		server := &Server{
			bindAddr:    bindAddr,
			config:      conf,
			socket:      socket,
			connections: make(map[*ServerConn]bool, conf.MaxConnections),
			counters:    make([]int, SC_COUNT),
		}
		go server.acceptLoop()
		return server, nil
	}
	return nil, err
}

// Create new default configuration for server.
func NewServerConf() ServerConf {
	return ServerConf{
		MaxConnections:  500,
		Concurrency:     1000,
		SuspendDuration: time.Second,
		MinFlushPeriod:  gMinFlushPeriod,
		WriteBufferSize: gWriteBufferSize,
		Trace:           gTraceServer,
	}
}

// Stop the server.
func (s *Server) Stop() {
	s.stopFlag = true
	s.lock.Lock()
	for h, _ := range s.connections {
		go h.close()
	}
	s.lock.Unlock()
}

// Send 'suspend' signal to all connected clients.
func (s *Server) Suspend(duration time.Duration) {
	s.lock.Lock()
	for conn := range s.connections {
		conn.suspend(duration)
	}
	s.lock.Unlock()
}

// Send 'resume' signal to all connected clients.
func (s *Server) Resume() {
	s.lock.Lock()
	for conn := range s.connections {
		conn.resume()
	}
	s.lock.Unlock()
}

// Send uplink cast packet to all connected clients.
func (s *Server) UplinkCast(data []byte) {
	s.lock.Lock()
	for conn := range s.connections {
		conn.uplinkCast(data)
	}
	s.lock.Unlock()
}

// Goroutine.
// Accept incoming connections.
func (s *Server) acceptLoop() {
	defer s.socket.Close()
	s.log("daemon started")
	defer s.log("daemon terminated")
	for !s.stopFlag {
		if s.config.MaxConnections <= len(s.connections) {
			s.counters[SC_ACCEPT_OVERFLOWS]++
			time.Sleep(time.Millisecond * 200)
			continue
		}
		socket, err := s.socket.Accept()
		s.counters[SC_ACCEPTED]++
		if err != nil {
			s.counters[SC_ACCEPT_ERRORS]++
			time.Sleep(time.Millisecond * 200)
			continue
		}
		if s.stopFlag {
			socket.Close()
			return
		}
		h := &ServerConn{
			peer:   socket.RemoteAddr().String(),
			server: s,
		}
		msgConn, err := NewMsgConn(socket,
			s.config.MinFlushPeriod,
			s.config.WriteBufferSize,
			h.onRecv, h.onClose)
		if err != nil {
			s.counters[SC_HANDLER_CREATE_ERRORS]++
			socket.Close()
			continue
		}
		h.conn = msgConn
		h.log("accepted")
		s.lock.Lock()
		s.connections[h] = true
		s.lock.Unlock()
	}
}

// Return count of active client connections opened.
func (s *Server) GetConnections() int {
	s.lock.Lock()
	res := len(s.connections)
	s.lock.Unlock()
	return res
}

// Return total count of running requests.
func (s *Server) GetWorkers() int {
	s.lock.Lock()
	workers := 0
	for conn := range s.connections {
		workers += conn.workers
	}
	s.lock.Unlock()
	return workers
}

// Print message to the stdout if verbose mode is enabled.
func (s *Server) log(format string, args ...interface{}) {
	if s.config.Trace {
		prefix := fmt.Sprintf("tcpcall srv%s> ", s.bindAddr)
		log.Printf(prefix+format, args...)
	}
}

// Thread safe counter increment.
func (s *Server) hit(counter int) {
	s.countersMu.Lock()
	s.counters[counter]++
	s.countersMu.Unlock()
}

// Return snapshot of server internal counters.
func (s *Server) Counters() []int {
	res := make([]int, SC_COUNT)
	s.countersMu.RLock()
	copy(res, s.counters)
	s.countersMu.RUnlock()
	return res
}

// Return Server's info and stats.
func (s *Server) Info() ServerInfo {
	return ServerInfo{
		Config:      s.config,
		BindAddr:    s.bindAddr,
		StopFlag:    s.stopFlag,
		Connections: s.GetConnections(),
		Counters:    s.Counters(),
	}
}

// Callback for message-oriented socket.
// Handles connection close event.
func (h *ServerConn) onClose() {
	h.server.lock.Lock()
	delete(h.server.connections, h)
	h.server.lock.Unlock()
	h.log("connection closed")
}

// Callback for message-oriented socket.
// Handles incoming message from the remote side.
func (h *ServerConn) onRecv(packet []byte) {
	h.server.hit(SC_PACKET_INPUT)
	if h.server.config.Concurrency < h.workers {
		// max workers count reached
		h.server.hit(SC_CONCURRENCY_OVERFLOWS)
		if f := h.server.config.OnDrop; f != nil {
			f(packet)
		}
		if err := h.suspend(h.server.config.SuspendDuration); err != nil {
			h.log("suspend send: %v", err)
			h.close()
		}
		return
	}
	h.incrementWorkers()
	go func() {
		// decode packet
		ptype, data, err := proto.Decode(packet)
		if err != nil {
			h.server.hit(SC_PACKET_DECODE_ERRORS)
			h.log("packet decode failed: %v", err)
			h.close()
			h.decrementWorkers()
			return
		}
		h.log("got packet of type %d: %v", ptype, data)
		switch ptype {
		case proto.REQUEST:
			h.server.hit(SC_REQUESTS)
			h.processRequest(data.(*proto.PacketRequest))
		case proto.CAST:
			h.server.hit(SC_CASTS)
			if f := h.server.config.CastCallback; f != nil {
				f(bytes.Join(
					(data.(*proto.PacketCast)).Request,
					[]byte{}))
			}
		default:
			// ignore packet
			h.server.hit(SC_UNKNOWN)
		}
		h.decrementWorkers()
	}()
}

// Send 'suspend' signal to the connected client.
func (h *ServerConn) suspend(duration time.Duration) error {
	h.server.hit(SC_SUSPEND_REQUESTED)
	packet := proto.PacketFlowControlSuspend{duration}
	if err := h.writePacket(packet); err != nil {
		h.server.hit(SC_SUSPEND_REQUEST_ERRORS)
		return err
	}
	return nil
}

// Send 'resume' signal to the connected client.
func (h *ServerConn) resume() error {
	h.server.hit(SC_RESUME_REQUESTED)
	packet := proto.PacketFlowControlResume{}
	if err := h.writePacket(packet); err != nil {
		h.server.hit(SC_RESUME_REQUEST_ERRORS)
		return err
	}
	return nil
}

// Send uplink cast packet to client side.
func (h *ServerConn) uplinkCast(data []byte) error {
	h.server.hit(SC_UPLINK_CAST_REQUESTED)
	packet := proto.PacketUplinkCast{[][]byte{data}}
	if err := h.writePacket(packet); err != nil {
		h.server.hit(SC_UPLINK_CAST_ERRORS)
		return err
	}
	return nil
}

// Force close connection to the client.
func (h *ServerConn) close() {
	h.conn.Close()
	h.log("stopped")
}

// Send packet back to the client side.
func (h *ServerConn) writePacket(packet proto.Packet) error {
	h.server.hit(SC_PACKET_WRITE)
	encoded := packet.Encode()
	if err := h.conn.Send(encoded); err != nil {
		h.server.hit(SC_PACKET_WRITE_ERRORS)
		h.log("packet write: %v", err)
		pools.ReleaseBuffer(encoded[0])
		return err
	}
	pools.ReleaseBuffer(encoded[0])
	return nil
}

// Process synchronous request from the client.
func (h *ServerConn) processRequest(req *proto.PacketRequest) {
	var res []byte
	if f := h.server.config.RequestCallback; f != nil {
		res = f(bytes.Join(req.Request, []byte{}))
	} else {
		res = []byte{}
	}
	replyPacket := proto.PacketReply{req.SeqNum, [][]byte{res}}
	if err := h.writePacket(replyPacket); err == nil {
		h.log("sent reply: %v", replyPacket)
	} else {
		h.log("sent reply failed: %v", err)
	}
}

// Safely increment workers count.
func (h *ServerConn) incrementWorkers() {
	h.lock.Lock()
	h.workers++
	h.lock.Unlock()
	h.server.hit(SC_WORKER_ADDED)
}

// Safely decrement workers count.
func (h *ServerConn) decrementWorkers() {
	h.lock.Lock()
	h.workers--
	h.lock.Unlock()
	h.server.hit(SC_WORKER_REMOVED)
}

// Print message to the stdout if verbose mode is enabled.
func (h *ServerConn) log(format string, args ...interface{}) {
	if h.server.config.Trace {
		prefix := fmt.Sprintf("tcpcall srv%s %s> ", h.server.bindAddr, h.peer)
		log.Printf(prefix+format, args...)
	}
}
<commit_msg>Simplify map used as set<commit_after>/*
Server.

Accepts client connections. Runs predefined processing function
for each incoming request and send the answer back to the client side.

Author: Aleksey Morarash <aleksey.morarash@gmail.com>
Since: 4 Sep 2016
Copyright: 2016, Aleksey Morarash <aleksey.morarash@gmail.com>
*/

package tcpcall

import (
	"bytes"
	"fmt"
	"log"
	"net"
	"sync"
	"tcpcall/pools"
	"tcpcall/proto"
	"time"
)

// Server counter indices.
const (
	// when new TCP connection established
	SC_ACCEPTED = iota
	// failed to accept new incoming connection
	SC_ACCEPT_ERRORS
	// too many incoming connections already exist
	SC_ACCEPT_OVERFLOWS
	// client connection handler creation error
	SC_HANDLER_CREATE_ERRORS
	// new packet received from the client
	SC_PACKET_INPUT
	// too many request processors (incoming packet
	// will be dropped without processing)
	SC_CONCURRENCY_OVERFLOWS
	// we're about sending suspend signal to the client
	SC_SUSPEND_REQUESTED
	// suspend signal not sent due to error
	SC_SUSPEND_REQUEST_ERRORS
	// we're about sending resume signal to the client
	SC_RESUME_REQUESTED
	// resume signal not sent due to error
	SC_RESUME_REQUEST_ERRORS
	// we're about sending packet back to the client
	SC_PACKET_WRITE
	// packet not sent due to error
	SC_PACKET_WRITE_ERRORS
	// new request processing worker (goroutine) created
	SC_WORKER_ADDED
	// request processing worker (goroutine) finished
	SC_WORKER_REMOVED
	// failed to decode packet received from the client
	SC_PACKET_DECODE_ERRORS
	// sync request received
	SC_REQUESTS
	// async request received
	SC_CASTS
	// unknown request received (and left without processing)
	SC_UNKNOWN
	// we're about sending uplink cast packet to the client
	SC_UPLINK_CAST_REQUESTED
	// uplink cast packet not sent due to error
	SC_UPLINK_CAST_ERRORS
	SC_COUNT // special value - count of all counters
)

// Server state
type Server struct {
	// IP address and TCP port number used for listening
	// incoming connections (colon separated).
	bindAddr string
	// Configuration used for server instance creation.
	config ServerConf
	// Server socket.
	socket net.Listener
	// List of established client connections.
	connections map[*ServerConn]struct{}
	// Controls access to client connections map.
	lock sync.Mutex
	// Set to truth when server is about to terminate.
	stopFlag bool
	// Counters array
	counters   []int
	countersMu sync.RWMutex
}

// Server configuration
type ServerConf struct {
	// TCP port number to listen
	PortNumber uint16
	// Maximum simultaneous connections to accept
	MaxConnections int
	// Maximum request processing concurrency per connection.
	Concurrency int
	// Max request packet size, in bytes. 0 means no limit.
	MaxRequestSize int
	// Minimum flush period for socket writer
	MinFlushPeriod time.Duration
	// Socket write buffer size
	WriteBufferSize int
	// Request processing function.
	// Each request will be processed in parallel with others.
	RequestCallback func([]byte) []byte
	// Asynchronous request (cast) processing function.
	// Each request will be processed in parallel with others.
	CastCallback func([]byte)
	// Called with encoded network packet when there are no workers
	// left to process the packet.
	OnDrop func([]byte)
	// Duration for suspend signals
	SuspendDuration time.Duration
	// Enable debug logging or not
	Trace bool
}

// Server instance information returned by Info() method.
// Added mostly for debugging.
type ServerInfo struct {
	// Configuration used to create Server instance
	Config ServerConf
	// Bind address and port number actually used
	BindAddr string
	// Truth means server is terminating (or terminated)
	StopFlag bool
	// Count of established connections from clients
	Connections int
	// Counters array
	Counters []int
}

// Connection handler state
type ServerConn struct {
	// IP address and TCP port number of the other
	// side of connection (client side).
	peer string
	// Message oriented connection
	conn *MsgConn
	// Number of currently running workers
	// (actually worker is a goroutine handling one
	// request received from the client).
	workers int
	// Controls access to conn object.
	lock sync.Mutex
	// Link to the Server instance this conection
	// is originated from.
	server *Server
}

// Start new server.
func Listen(conf ServerConf) (*Server, error) {
	bindAddr := fmt.Sprintf(":%d", conf.PortNumber)
	socket, err := net.Listen("tcp", bindAddr)
	if err == nil {
		server := &Server{
			bindAddr:    bindAddr,
			config:      conf,
			socket:      socket,
			connections: map[*ServerConn]struct{}{},
			counters:    make([]int, SC_COUNT),
		}
		go server.acceptLoop()
		return server, nil
	}
	return nil, err
}

// Create new default configuration for server.
func NewServerConf() ServerConf {
	return ServerConf{
		MaxConnections:  500,
		Concurrency:     1000,
		SuspendDuration: time.Second,
		MinFlushPeriod:  gMinFlushPeriod,
		WriteBufferSize: gWriteBufferSize,
		Trace:           gTraceServer,
	}
}

// Stop the server.
func (s *Server) Stop() {
	s.stopFlag = true
	s.lock.Lock()
	for h, _ := range s.connections {
		go h.close()
	}
	s.lock.Unlock()
}

// Send 'suspend' signal to all connected clients.
func (s *Server) Suspend(duration time.Duration) {
	s.lock.Lock()
	for conn := range s.connections {
		conn.suspend(duration)
	}
	s.lock.Unlock()
}

// Send 'resume' signal to all connected clients.
func (s *Server) Resume() {
	s.lock.Lock()
	for conn := range s.connections {
		conn.resume()
	}
	s.lock.Unlock()
}

// Send uplink cast packet to all connected clients.
func (s *Server) UplinkCast(data []byte) {
	s.lock.Lock()
	for conn := range s.connections {
		conn.uplinkCast(data)
	}
	s.lock.Unlock()
}

// Goroutine.
// Accept incoming connections.
func (s *Server) acceptLoop() {
	defer s.socket.Close()
	s.log("daemon started")
	defer s.log("daemon terminated")
	for !s.stopFlag {
		if s.config.MaxConnections <= len(s.connections) {
			s.counters[SC_ACCEPT_OVERFLOWS]++
			time.Sleep(time.Millisecond * 200)
			continue
		}
		socket, err := s.socket.Accept()
		s.counters[SC_ACCEPTED]++
		if err != nil {
			s.counters[SC_ACCEPT_ERRORS]++
			time.Sleep(time.Millisecond * 200)
			continue
		}
		if s.stopFlag {
			socket.Close()
			return
		}
		h := &ServerConn{
			peer:   socket.RemoteAddr().String(),
			server: s,
		}
		msgConn, err := NewMsgConn(socket,
			s.config.MinFlushPeriod,
			s.config.WriteBufferSize,
			h.onRecv, h.onClose)
		if err != nil {
			s.counters[SC_HANDLER_CREATE_ERRORS]++
			socket.Close()
			continue
		}
		h.conn = msgConn
		h.log("accepted")
		s.lock.Lock()
		s.connections[h] = struct{}{}
		s.lock.Unlock()
	}
}

// Return count of active client connections opened.
func (s *Server) GetConnections() int {
	s.lock.Lock()
	res := len(s.connections)
	s.lock.Unlock()
	return res
}

// Return total count of running requests.
func (s *Server) GetWorkers() int {
	s.lock.Lock()
	workers := 0
	for conn := range s.connections {
		workers += conn.workers
	}
	s.lock.Unlock()
	return workers
}

// Print message to the stdout if verbose mode is enabled.
func (s *Server) log(format string, args ...interface{}) {
	if s.config.Trace {
		prefix := fmt.Sprintf("tcpcall srv%s> ", s.bindAddr)
		log.Printf(prefix+format, args...)
	}
}

// Thread safe counter increment.
func (s *Server) hit(counter int) {
	s.countersMu.Lock()
	s.counters[counter]++
	s.countersMu.Unlock()
}

// Return snapshot of server internal counters.
func (s *Server) Counters() []int {
	res := make([]int, SC_COUNT)
	s.countersMu.RLock()
	copy(res, s.counters)
	s.countersMu.RUnlock()
	return res
}

// Return Server's info and stats.
func (s *Server) Info() ServerInfo {
	return ServerInfo{
		Config:      s.config,
		BindAddr:    s.bindAddr,
		StopFlag:    s.stopFlag,
		Connections: s.GetConnections(),
		Counters:    s.Counters(),
	}
}

// Callback for message-oriented socket.
// Handles connection close event.
func (h *ServerConn) onClose() {
	h.server.lock.Lock()
	delete(h.server.connections, h)
	h.server.lock.Unlock()
	h.log("connection closed")
}

// Callback for message-oriented socket.
// Handles incoming message from the remote side.
func (h *ServerConn) onRecv(packet []byte) {
	h.server.hit(SC_PACKET_INPUT)
	if h.server.config.Concurrency < h.workers {
		// max workers count reached
		h.server.hit(SC_CONCURRENCY_OVERFLOWS)
		if f := h.server.config.OnDrop; f != nil {
			f(packet)
		}
		if err := h.suspend(h.server.config.SuspendDuration); err != nil {
			h.log("suspend send: %v", err)
			h.close()
		}
		return
	}
	h.incrementWorkers()
	go func() {
		// decode packet
		ptype, data, err := proto.Decode(packet)
		if err != nil {
			h.server.hit(SC_PACKET_DECODE_ERRORS)
			h.log("packet decode failed: %v", err)
			h.close()
			h.decrementWorkers()
			return
		}
		h.log("got packet of type %d: %v", ptype, data)
		switch ptype {
		case proto.REQUEST:
			h.server.hit(SC_REQUESTS)
			h.processRequest(data.(*proto.PacketRequest))
		case proto.CAST:
			h.server.hit(SC_CASTS)
			if f := h.server.config.CastCallback; f != nil {
				f(bytes.Join(
					(data.(*proto.PacketCast)).Request,
					[]byte{}))
			}
		default:
			// ignore packet
			h.server.hit(SC_UNKNOWN)
		}
		h.decrementWorkers()
	}()
}

// Send 'suspend' signal to the connected client.
func (h *ServerConn) suspend(duration time.Duration) error {
	h.server.hit(SC_SUSPEND_REQUESTED)
	packet := proto.PacketFlowControlSuspend{duration}
	if err := h.writePacket(packet); err != nil {
		h.server.hit(SC_SUSPEND_REQUEST_ERRORS)
		return err
	}
	return nil
}

// Send 'resume' signal to the connected client.
func (h *ServerConn) resume() error {
	h.server.hit(SC_RESUME_REQUESTED)
	packet := proto.PacketFlowControlResume{}
	if err := h.writePacket(packet); err != nil {
		h.server.hit(SC_RESUME_REQUEST_ERRORS)
		return err
	}
	return nil
}

// Send uplink cast packet to client side.
func (h *ServerConn) uplinkCast(data []byte) error {
	h.server.hit(SC_UPLINK_CAST_REQUESTED)
	packet := proto.PacketUplinkCast{[][]byte{data}}
	if err := h.writePacket(packet); err != nil {
		h.server.hit(SC_UPLINK_CAST_ERRORS)
		return err
	}
	return nil
}

// Force close connection to the client.
func (h *ServerConn) close() {
	h.conn.Close()
	h.log("stopped")
}

// Send packet back to the client side.
func (h *ServerConn) writePacket(packet proto.Packet) error {
	h.server.hit(SC_PACKET_WRITE)
	encoded := packet.Encode()
	if err := h.conn.Send(encoded); err != nil {
		h.server.hit(SC_PACKET_WRITE_ERRORS)
		h.log("packet write: %v", err)
		pools.ReleaseBuffer(encoded[0])
		return err
	}
	pools.ReleaseBuffer(encoded[0])
	return nil
}

// Process synchronous request from the client.
func (h *ServerConn) processRequest(req *proto.PacketRequest) {
	var res []byte
	if f := h.server.config.RequestCallback; f != nil {
		res = f(bytes.Join(req.Request, []byte{}))
	} else {
		res = []byte{}
	}
	replyPacket := proto.PacketReply{req.SeqNum, [][]byte{res}}
	if err := h.writePacket(replyPacket); err == nil {
		h.log("sent reply: %v", replyPacket)
	} else {
		h.log("sent reply failed: %v", err)
	}
}

// Safely increment workers count.
func (h *ServerConn) incrementWorkers() {
	h.lock.Lock()
	h.workers++
	h.lock.Unlock()
	h.server.hit(SC_WORKER_ADDED)
}

// Safely decrement workers count.
func (h *ServerConn) decrementWorkers() {
	h.lock.Lock()
	h.workers--
	h.lock.Unlock()
	h.server.hit(SC_WORKER_REMOVED)
}

// Print message to the stdout if verbose mode is enabled.
func (h *ServerConn) log(format string, args ...interface{}) {
	if h.server.config.Trace {
		prefix := fmt.Sprintf("tcpcall srv%s %s> ", h.server.bindAddr, h.peer)
		log.Printf(prefix+format, args...)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"github.com/microcosm-cc/bluemonday"
	"github.com/russross/blackfriday"
	"io/ioutil"
	"os"
)

func main() {

	args := os.Args[1:] // Reason: index 0 contains the program path
	if len(args) != 2 {
		fmt.Println("ERROR: Wrong number of arguments provided. We're expecting:")
		fmt.Println("1. Input file")
		fmt.Println("2. Output file")
		os.Exit(1)
	}
	inputFile := args[0]
	outputFile := args[1]

	// Get the Markdown input
	rawInput, err := ioutil.ReadFile(inputFile)
	check(err)

	// Convert and sanitize our content
	unsafe := blackfriday.MarkdownCommon(rawInput)
	content := bluemonday.UGCPolicy().SanitizeBytes(unsafe)

	// Get the shared markup pieces
	topHTML, err := ioutil.ReadFile("./shared_markup/top.html")
	check(err)
	bottomHTML, err := ioutil.ReadFile("./shared_markup/bottom.html")
	check(err)

	// Incrementally building a singular byte array via append()
	finalOutput := append(topHTML[:], content[:]...)
	finalOutput = append(finalOutput, bottomHTML[:]...)

	// Write our output to an HTML file
	err = ioutil.WriteFile(outputFile, finalOutput, 0644)
	check(err)

	fmt.Println("Program finished, check result.")
}

func check(e error) {
	if e != nil {
		panic(e)
	}
}
<commit_msg>Figuring out how to walk a directory<commit_after>package main

import (
	"fmt"
	"github.com/microcosm-cc/bluemonday"
	"github.com/russross/blackfriday"
	"io/ioutil"
	"os"
	"path/filepath"
)

func main() {

	root := "./markdown_source_files"
	err := filepath.Walk(root, visit)
	fmt.Printf("filepath.Walk() returned %v\n", err)
	os.Exit(1)

	args := os.Args[1:] // Reason: index 0 contains the program path
	if len(args) != 2 {
		fmt.Println("ERROR: Wrong number of arguments provided. We're expecting:")
		fmt.Println("1. Input file")
		fmt.Println("2. Output file")
		os.Exit(1)
	}
	inputFile := args[0]
	outputFile := args[1]

	// Get the Markdown input
	rawInput, err := ioutil.ReadFile(inputFile)
	check(err)

	// Convert and sanitize our content
	unsafe := blackfriday.MarkdownCommon(rawInput)
	content := bluemonday.UGCPolicy().SanitizeBytes(unsafe)

	// Get the shared markup pieces
	topHTML, err := ioutil.ReadFile("./shared_markup/top.html")
	check(err)
	bottomHTML, err := ioutil.ReadFile("./shared_markup/bottom.html")
	check(err)

	// Incrementally building a singular byte array via append()
	finalOutput := append(topHTML[:], content[:]...)
	finalOutput = append(finalOutput, bottomHTML[:]...)

	// Write our output to an HTML file
	err = ioutil.WriteFile(outputFile, finalOutput, 0644)
	check(err)

	fmt.Println("Program finished, check result.")
}

func visit(path string, f os.FileInfo, err error) error {
	fmt.Printf("Visited: %s\n", path)
	return nil
}

func check(e error) {
	if e != nil {
		panic(e)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"github.com/microcosm-cc/bluemonday"
	"github.com/russross/blackfriday"
	"io/ioutil"
	"os"
	// "path/filepath" // NOTE: Holding this for something I'll be adding soon.
	"encoding/json"
	"net/url"
	"strings"
)

// For holding our post's configuration settings
type Config struct {
	Title     string `json:"title`
	Date      string `json:"date`
	Permalink string `json:"string`
}

func main() {

	// NOTE: Holding this for something I'll be adding soon.
	//
	// root := "./content"
	// err := filepath.Walk(root, visit)
	// fmt.Printf("filepath.Walk() returned %v\n", err)
	// os.Exit(1)

	// Note: index 0 contains the program path, so I'm excluding it from what gets passed in
	inputFile, outputFile := dealWithArgs(os.Args[1:])

	title, date, permalink := getPostConfigsJson(&inputFile)

	fmt.Println("postTitle: " + title)
	fmt.Println("postDate: " + date)
	fmt.Println("permalink: " + permalink)
	// fmt.Println("Leaving off here")
	// os.Exit(1)

	content := getContent(&inputFile)
	topHTML, bottomHTML := getSharedMarkup()
	combinedOutput := buildCombinedOutput(topHTML, content, bottomHTML)
	finalOutput := interpolateConfigVals(combinedOutput, &title, &date, &permalink)
	writeOutputFile(finalOutput, &outputFile)
	fmt.Println("Program finished, check result.")
}

func dealWithArgs(args []string) (string, string) {
	if len(args) != 2 {
		fmt.Println("ERROR: Wrong number of arguments provided. We're expecting:")
		fmt.Println("1. Input file")
		fmt.Println("2. Output file")
		os.Exit(1)
	}
	inputFile := args[0]
	outputFile := args[1]

	return inputFile, outputFile
}

func writeOutputFile(finalOutput []byte, outputFile *string) bool {
	// Write our output to an HTML file
	err := ioutil.WriteFile("./content/"+*outputFile, finalOutput, 0644)
	check(err)
	return true
}

// Interpolate config values into the output
func interpolateConfigVals(combinedOutput []byte, title *string, date *string, permalink *string) []byte {

	encodedSquigglyOpen := url.QueryEscape("{")
	encodedSquigglyClose := url.QueryEscape("}")
	permalinkPlaceHolder := encodedSquigglyOpen + encodedSquigglyOpen + "permalink" + encodedSquigglyClose + encodedSquigglyClose

	str := string(combinedOutput[:])
	str = strings.Replace(str, "{{title}}", *title, -1)
	str = strings.Replace(str, "{{date}}", *date, -1)
	str = strings.Replace(str, permalinkPlaceHolder, *permalink, -1)

	return []byte(str)
}

func buildCombinedOutput(topHTML []byte, content []byte, bottomHTML []byte) []byte {
	// Incrementally building a singular byte array via append()
	combinedOutput := append(topHTML[:], content[:]...)
	combinedOutput = append(combinedOutput, bottomHTML[:]...)
	return combinedOutput
}

func getSharedMarkup() ([]byte, []byte) {

	// Get the shared markup pieces
	topHTML, err := ioutil.ReadFile("./shared_markup/top.html")
	check(err)
	bottomHTML, err := ioutil.ReadFile("./shared_markup/bottom.html")
	check(err)

	return topHTML, bottomHTML
}

func getContent(inputFile *string) []byte {

	// Get the Markdown input
	rawInput, err := ioutil.ReadFile("./content/" + *inputFile)
	check(err)

	// Convert and sanitize our content
	unsafe := blackfriday.MarkdownCommon(rawInput)
	content := bluemonday.UGCPolicy().SanitizeBytes(unsafe)

	return content
}

func getPostConfigsJson(inputFile *string) (string, string, string) {
	// Get the configs for this page:
	// First read the post's corresponding .json file
	configPath := "./content/" + strings.Replace(*inputFile, ".md", "", 1) + ".json"
	configJson, err := ioutil.ReadFile(configPath)
	check(err)

	// Then parse out our relevant config values for later use
	var config Config
	err = json.Unmarshal(configJson, &config)
	check(err)

	return config.Title, config.Date, config.Permalink
}

func visit(path string, f os.FileInfo, err error) error {
	fmt.Printf("Visited: %s\n", path)
	return nil
}

func check(e error) {
	if e != nil {
		panic(e)
	}
}
<commit_msg>Modified the sanitizer policy to not automatically add rel="nofollow" to our links, and additionally so that we can arbitrarily add it ourselves to individual links, if we want.<commit_after>package main

import (
	"fmt"
	"github.com/microcosm-cc/bluemonday"
	"github.com/russross/blackfriday"
	"io/ioutil"
	"os"
	// "path/filepath" // NOTE: Holding this for something I'll be adding soon.
	"encoding/json"
	"net/url"
	"strings"
)

// For holding our post's configuration settings
type Config struct {
	Title     string `json:"title`
	Date      string `json:"date`
	Permalink string `json:"string`
}

func main() {

	// NOTE: Holding this for something I'll be adding soon.
	//
	// root := "./content"
	// err := filepath.Walk(root, visit)
	// fmt.Printf("filepath.Walk() returned %v\n", err)
	// os.Exit(1)

	// Note: index 0 contains the program path, so I'm excluding it from what gets passed in
	inputFile, outputFile := dealWithArgs(os.Args[1:])

	title, date, permalink := getPostConfigsJson(&inputFile)

	fmt.Println("postTitle: " + title)
	fmt.Println("postDate: " + date)
	fmt.Println("permalink: " + permalink)
	// fmt.Println("Leaving off here")
	// os.Exit(1)

	content := getContent(&inputFile)
	topHTML, bottomHTML := getSharedMarkup()
	combinedOutput := buildCombinedOutput(topHTML, content, bottomHTML)
	finalOutput := interpolateConfigVals(combinedOutput, &title, &date, &permalink)
	writeOutputFile(finalOutput, &outputFile)
	fmt.Println("Program finished, check result.")
}

func dealWithArgs(args []string) (string, string) {
	if len(args) != 2 {
		fmt.Println("ERROR: Wrong number of arguments provided. We're expecting:")
		fmt.Println("1. Input file")
		fmt.Println("2. Output file")
		os.Exit(1)
	}
	inputFile := args[0]
	outputFile := args[1]

	return inputFile, outputFile
}

func writeOutputFile(finalOutput []byte, outputFile *string) bool {
	// Write our output to an HTML file
	err := ioutil.WriteFile("./content/"+*outputFile, finalOutput, 0644)
	check(err)
	return true
}

// Interpolate config values into the output
func interpolateConfigVals(combinedOutput []byte, title *string, date *string, permalink *string) []byte {

	encodedSquigglyOpen := url.QueryEscape("{")
	encodedSquigglyClose := url.QueryEscape("}")
	permalinkPlaceHolder := encodedSquigglyOpen + encodedSquigglyOpen + "permalink" + encodedSquigglyClose + encodedSquigglyClose

	str := string(combinedOutput[:])
	str = strings.Replace(str, "{{title}}", *title, -1)
	str = strings.Replace(str, "{{date}}", *date, -1)
	str = strings.Replace(str, permalinkPlaceHolder, *permalink, -1)

	return []byte(str)
}

func buildCombinedOutput(topHTML []byte, content []byte, bottomHTML []byte) []byte {
	// Incrementally building a singular byte array via append()
	combinedOutput := append(topHTML[:], content[:]...)
	combinedOutput = append(combinedOutput, bottomHTML[:]...)
	return combinedOutput
}

func getSharedMarkup() ([]byte, []byte) {

	// Get the shared markup pieces
	topHTML, err := ioutil.ReadFile("./shared_markup/top.html")
	check(err)
	bottomHTML, err := ioutil.ReadFile("./shared_markup/bottom.html")
	check(err)

	return topHTML, bottomHTML
}

func getContent(inputFile *string) []byte {

	// Get the Markdown input
	rawInput, err := ioutil.ReadFile("./content/" + *inputFile)
	check(err)

	// Convert and sanitize our content
	unsafe := blackfriday.MarkdownCommon(rawInput)
	policy := bluemonday.UGCPolicy()
	policy.RequireNoFollowOnLinks(false)
	// This allows us to arbitrarily add "rel" attributes to our links
	policy.AllowAttrs("rel").OnElements("a", "area")

	content := policy.SanitizeBytes(unsafe)
	return content
}

func getPostConfigsJson(inputFile *string) (string, string, string) {
	// Get the configs for this page:
	// First read the post's corresponding .json file
	configPath := "./content/" + strings.Replace(*inputFile, ".md", "", 1) + ".json"
	configJson, err := ioutil.ReadFile(configPath)
	check(err)

	// Then parse out our relevant config values for later use
	var config Config
	err = json.Unmarshal(configJson, &config)
	check(err)

	return config.Title, config.Date, config.Permalink
}

func visit(path string, f os.FileInfo, err error) error {
	fmt.Printf("Visited: %s\n", path)
	return nil
}

func check(e error) {
	if e != nil {
		panic(e)
	}
}
<|endoftext|>