<commit_before>package main

import (
  "encoding/json"
  "fmt"
  "github.com/streadway/amqp"
  . "koding/db/models"
  "koding/tools/amqputil"
  "log"
  "strings"
)

type Status string

const (
  UPDATE Status = "update"
  DELETE Status = "delete"
  MERGE  Status = "merge"
)

var (
  EXCHANGE_NAME     = "topicModifierExchange"
  WORKER_QUEUE_NAME = "topicModifierWorkerQueue"
)

type Consumer struct {
  conn    *amqp.Connection
  channel *amqp.Channel
  name    string
}

type TagModifierData struct {
  OldTagId string `json:"oldTagId"`
  TagId    string `json:"tagId"`
  Status   Status `json:"status"`
}

func init() {
  declareExchange()
}

func main() {
  log.Printf("Tag Modifier Worker Started")
  consumeMessages()
  // fetch/update related posts
  // ack message
}

func declareExchange() {

  connection := amqputil.CreateConnection("exchangeDeclareConnection")

  channel := amqputil.CreateChannel(connection)

  err := channel.ExchangeDeclare(EXCHANGE_NAME, "fanout", true, false, false, false, nil)
  if err != nil {
    log.Println("exchange.declare: %s", err)
    panic(err)
  }

  //name, durable, autoDelete, exclusive, noWait, args Table
  _, err = channel.QueueDeclare(WORKER_QUEUE_NAME, true, false, false, false, nil)
  if err != nil {
    log.Println("queue.declare: %s", err)
    panic(err)
  }
  err = channel.QueueBind(WORKER_QUEUE_NAME, "", EXCHANGE_NAME, false, nil)
  if err != nil {
    log.Println("queue.bind: %s", err)
    panic(err)
  }

}

func fetchRelatedPosts() {

}

func consumeMessages() {
  c := &Consumer{
    conn:    nil,
    channel: nil,
    name:    "",
  }

  c.name = "tagModifier"
  c.conn = amqputil.CreateConnection(c.name)
  c.channel = amqputil.CreateChannel(c.conn)

  postData, err := c.channel.Consume(WORKER_QUEUE_NAME, c.name, false, false, false, false, nil)
  if err != nil {
    log.Printf("Consume error %s", err)
    panic(err)
  }

  for rawMsg := range postData {
    modifierData := &TagModifierData{}
    if err = json.Unmarshal([]byte(rawMsg.Body), modifierData); err != nil {
      log.Println("Wrong Post Format", err, rawMsg)
    }

    modifyMessage := func() {
      // get rid of this case structure.
      switch modifierData.Status {
      default:
        log.Println("Unknown modification status")
        // rawMsg.Ack(false)
      case UPDATE:
        updateTags(modifierData)
      case DELETE:
        deleteTags(modifierData)
      case MERGE:
        mergeTags(modifierData)
      }
    }

    modifyMessage()

  }
}

func isTagValid(tagId string, fn func(string) Tag) bool {
  tag := fn(tagId)
  if (tag != Tag{}) {
    return true
  }
  return false
}

func updateTags(modifierData *TagModifierData) {
  log.Println("update")
  // fetch tag
  // fetch related posts
}

func deleteTags(modifierData *TagModifierData) {
  log.Println("delete")
  if isTagValid(modifierData.TagId, FindDeletedTagById) {
    log.Println("Valid tag")
    rels := FindRelationshipsWithTagId(modifierData.TagId)
    deleteTagsFromPosts(rels)
  }
  //ack

}

func deleteTagsFromPosts(rels []Relationship) {
  for _, rel := range rels {
    tagId := rel.TargetId.Hex()
    post := FindPostWithId(rel.SourceId.Hex())
    updatePost(&post, tagId)
    RemoveTagRelationship(rel)
  }
  log.Printf("Deleted %d tags", len(rels))
}

//it could take fn parameter for deleting/appending tag
func updatePost(post *StatusUpdate, tagId string) {
  modifiedTag := fmt.Sprintf("|#:JTag:%v|", tagId)
  post.Body = strings.Replace(post.Body, modifiedTag, "", -1)
  // log.Println("PostBody:", post.Body)
  UpdatePost(post)
}

func mergeTags(modifierData *TagModifierData) {
  log.Println("merge")
}
<commit_msg>Moderation: Topic merge feature is added<commit_after>package main

import (
  "encoding/json"
  "fmt"
  "github.com/streadway/amqp"
  . "koding/db/models"
  "koding/tools/amqputil"
  "labix.org/v2/mgo/bson"
  "log"
  "strings"
)

type Status string

const (
  UPDATE Status = "update"
  DELETE Status = "delete"
  MERGE  Status = "merge"
)

var (
  EXCHANGE_NAME     = "topicModifierExchange"
  WORKER_QUEUE_NAME = "topicModifierWorkerQueue"
)

type Consumer struct {
  conn    *amqp.Connection
  channel *amqp.Channel
  name    string
}

type TagModifierData struct {
  TagId  string `json:"tagId"`
  Status Status `json:"status"`
}

func init() {
  declareExchange()
}

func main() {
  log.Printf("Tag Modifier Worker Started")
  consumeMessages()
  // ack message
}

func declareExchange() {

  connection := amqputil.CreateConnection("exchangeDeclareConnection")

  channel := amqputil.CreateChannel(connection)

  err := channel.ExchangeDeclare(EXCHANGE_NAME, "fanout", true, false, false, false, nil)
  if err != nil {
    log.Println("exchange.declare: %s", err)
    panic(err)
  }

  //name, durable, autoDelete, exclusive, noWait, args Table
  _, err = channel.QueueDeclare(WORKER_QUEUE_NAME, true, false, false, false, nil)
  if err != nil {
    log.Println("queue.declare: %s", err)
    panic(err)
  }
  err = channel.QueueBind(WORKER_QUEUE_NAME, "", EXCHANGE_NAME, false, nil)
  if err != nil {
    log.Println("queue.bind: %s", err)
    panic(err)
  }

}

func consumeMessages() {
  c := &Consumer{
    conn:    nil,
    channel: nil,
    name:    "",
  }

  c.name = "tagModifier"
  c.conn = amqputil.CreateConnection(c.name)
  c.channel = amqputil.CreateChannel(c.conn)

  postData, err := c.channel.Consume(WORKER_QUEUE_NAME, c.name, false, false, false, false, nil)
  if err != nil {
    log.Printf("Consume error %s", err)
    panic(err)
  }

  for rawMsg := range postData {
    modifierData := &TagModifierData{}
    if err = json.Unmarshal([]byte(rawMsg.Body), modifierData); err != nil {
      log.Println("Wrong Post Format", err, rawMsg)
    }

    modifyMessage := func() {
      tagId := modifierData.TagId
      switch modifierData.Status {
      default:
        log.Println("Unknown modification status")
        // rawMsg.Ack(false)
      case UPDATE:
        updateTags(tagId)
      case DELETE:
        deleteTags(tagId)
      case MERGE:
        mergeTags(tagId)
      }
      rawMsg.Ack(false)
    }

    modifyMessage()

  }
}

func updateTags(tagId string) {
  log.Println("update")
  // fetch tag
  // fetch related posts
}

func deleteTags(tagId string) {
  log.Println("delete")
  if tag := FindTagById(tagId); (tag != Tag{}) {
    log.Println("Valid tag")
    rels := FindRelationships(bson.M{"targetId": bson.ObjectIdHex(tagId), "as": "tag"})
    updatePosts(rels, "")
    updateRelationships(rels, &Tag{})
  }
  //ack

}

func mergeTags(tagId string) {
  log.Println("merge")

  if tag := FindTagById(tagId); (tag != Tag{}) {
    log.Println("Valid tag")

    synonym := FindSynonym(tagId)
    tagRels := FindRelationships(bson.M{"targetId": bson.ObjectIdHex(tagId), "as": "tag"})
    if len(tagRels) > 0 {
      updatePosts(tagRels, synonym.Id.Hex())
      updateRelationships(tagRels, &synonym)
    }

    postRels := FindRelationships(bson.M{"sourceId": bson.ObjectIdHex(tagId), "as": "post"})
    if len(postRels) > 0 {
      updateRelationships(postRels, &synonym)
      updateCounts(&tag, &synonym)
    }
    UpdateTag(&synonym)
    tag.Counts = TagCount{} // reset counts
    UpdateTag(&tag)
    updateFollowers(&tag, &synonym)
  }
}

func updatePosts(rels []Relationship, newTagId string) {
  var newTag string
  if newTagId != "" {
    newTag = fmt.Sprintf("|#:JTag:%v|", newTagId)
  }

  for _, rel := range rels {
    tagId := rel.TargetId.Hex()
    post := FindPostWithId(rel.SourceId.Hex())
    modifiedTag := fmt.Sprintf("|#:JTag:%v|", tagId)
    post.Body = strings.Replace(post.Body, modifiedTag, newTag, -1)
    UpdatePost(&post)
  }
  log.Printf("%v Posts updated", len(rels))
}

func updateRelationships(rels []Relationship, synonym *Tag) {
  for _, rel := range rels {
    removeRelationship(rel)
    if (synonym != &Tag{}) {
      if rel.TargetName == "JTag" {
        rel.TargetId = synonym.Id
      } else {
        rel.SourceId = synonym.Id
      }
      rel.Id = bson.NewObjectId()
      createRelationship(rel)
    }
  }
}

func updateCounts(tag *Tag, synonym *Tag) {
  synonym.Counts.Following += tag.Counts.Following
  synonym.Counts.Followers += tag.Counts.Followers
  synonym.Counts.Post += tag.Counts.Post
  synonym.Counts.Tagged += tag.Counts.Tagged
}

func updateFollowers(tag *Tag, synonym *Tag) {
  var arr [2]bson.M
  arr[0] = bson.M{
    "targetId": tag.Id,
    "as":       "follower",
  }
  arr[1] = bson.M{
    "sourceId": tag.Id,
    "as":       "follower",
  }
  rels := FindRelationships(bson.M{"$or": arr})
  log.Printf("%v follower rels found", len(rels))
  if len(rels) > 0 {
    updateRelationships(rels, synonym)
  }
}

func createRelationship(relationship Relationship) {
  CreateGraphRelationship(relationship)
  CreateRelationship(relationship)
}

func removeRelationship(relationship Relationship) {
  RemoveGraphRelationship(relationship)
  RemoveRelationship(relationship)

}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"log"
	"net/http"
)

func registerAccountHandlerFunc(w http.ResponseWriter, r *http.Request) {
	type inDataType struct {
		Username string `json:"username"`
		Email    string `json:"email"`
		Password string `json:"password"`
	}

	type outDataType struct {
		Success bool   `json:"success"`
		Error   string `json:"error"`
	}

	// defer sending output
	outData := &outDataType{}
	defer func() {
		err := json.NewEncoder(w).Encode(outData)
		if err != nil {
			log.Printf("could not encode data for registerAccoutn request. %s\n", err)
		}
	}()

	inData := &inDataType{}
	err := json.NewDecoder(r.Body).Decode(inData)
	r.Body.Close()
	if err != nil {
		log.Println("could not decode body for registerAccount request. %s\n", err)
		return
	}

	err = registerNewAccount(inData.Username, inData.Email, inData.Password)
	if err != nil {
		if err == errAccountUsernameNotUnique {
			outData.Error = "Username is already taken."
			return
		}
		log.Println("error creating an account: %s\n", err)
		outData.Error = "A server error occured."
		return
	}

	// all done
	outData.Success = true
}
<commit_msg>made `go vet` happy again.<commit_after>package main

import (
	"encoding/json"
	"log"
	"net/http"
)

func registerAccountHandlerFunc(w http.ResponseWriter, r *http.Request) {
	type inDataType struct {
		Username string `json:"username"`
		Email    string `json:"email"`
		Password string `json:"password"`
	}

	type outDataType struct {
		Success bool   `json:"success"`
		Error   string `json:"error"`
	}

	// defer sending output
	outData := &outDataType{}
	defer func() {
		err := json.NewEncoder(w).Encode(outData)
		if err != nil {
			log.Printf("could not encode data for registerAccoutn request. %s\n", err)
		}
	}()

	inData := &inDataType{}
	err := json.NewDecoder(r.Body).Decode(inData)
	r.Body.Close()
	if err != nil {
		log.Printf("could not decode body for registerAccount request. %s\n", err)
		return
	}

	err = registerNewAccount(inData.Username, inData.Email, inData.Password)
	if err != nil {
		if err == errAccountUsernameNotUnique {
			outData.Error = "Username is already taken."
			return
		}
		log.Printf("error creating an account: %s\n", err)
		outData.Error = "A server error occured."
		return
	}

	// all done
	outData.Success = true
}
<|endoftext|>


<commit_before>package state_machine

import "testing"

func Test_Zero(t *testing.T) {
   sm := StateMachine{0, 0}

   expected := false
   actual := sm.step(0, 0, 0, 0)

   if expected != actual {
        t.Error("Failed for all inputs as zero, single step.")
   }
}

func Test_DoubleZero(t *testing.T) {
   sm := StateMachine{0, 0}

   expected := false
   sm.step(0, 0, 0, 0)
   actual := sm.step(0, 0, 0, 0)

   if expected != actual {
        t.Error("Failed for all inputs as zero, double step.")
   }
}

func Test_MultipleZero(t *testing.T) {
   sm := StateMachine{0, 0}

   expected := false
   sm.step(0, 0, 0, 0)
   sm.step(0, 0, 0, 0)
   sm.step(0, 0, 0, 0)
   sm.step(0, 0, 0, 0)
   sm.step(0, 0, 0, 0)
   sm.step(0, 0, 0, 0)
   sm.step(0, 0, 0, 0)
   sm.step(0, 0, 0, 0)
   actual := sm.step(0, 0, 0, 0)

   if expected != actual {
        t.Error("Failed for all inputs as zero, many step.")
   }
}

func Test_One(t *testing.T) {
   sm := StateMachine{0, 0}

   expected := true
   actual := sm.step(1, 0, 0, 0)

   if expected != actual {
        t.Error("Failed for a single input as one, the rest zero, single step.")
   }

   expected = true
   actual = sm.step(0, 1, 0, 0)

   if expected != actual {
        t.Error("Failed for a single input as one, the rest zero, single step.")
   }

   expected = true
   actual = sm.step(0, 0, 1, 0)

   if expected != actual {
        t.Error("Failed for a single input as one, the rest zero, single step.")
   }

   expected = true
   actual = sm.step(0, 0, 0, 1)

   if expected != actual {
        t.Error("Failed for a single input as one, the rest zero, single step.")
   }
}

// When we only have a single bit set, the value of c will never change
// This is thanks to the /2 component
func Test_DoubleOne(t *testing.T) {
   sm := StateMachine{0, 0}

   expected := true
   sm.step(1, 0, 0, 0)
   actual := sm.step(1, 0, 0, 0)

   if expected != actual {
        t.Error("Failed for a single input as one, the rest zero, double step.")
   }

   expected = true
   sm.step(0, 1, 0, 0)
   actual = sm.step(0, 1, 0, 0)

   if expected != actual {
        t.Error("Failed for a single input as one, the rest zero, double step.")
   }

   expected = true
   sm.step(0, 0, 1, 0)
   actual = sm.step(0, 0, 1, 0)

   if expected != actual {
        t.Error("Failed for a single input as one, the rest zero, double step.")
   }

   expected = true
   sm.step(0, 0, 0, 1)
   actual = sm.step(0, 0, 0, 1)

   if expected != actual {
        t.Error("Failed for a single input as one, the rest zero, double step.")
   }
}

func Test_MultipleOne(t *testing.T) {
   sm := StateMachine{0, 0}

   expected := true
   sm.step(1, 0, 0, 0)
   sm.step(1, 0, 0, 0)
   sm.step(1, 0, 0, 0)
   sm.step(1, 0, 0, 0)
   sm.step(1, 0, 0, 0)
   actual := sm.step(1, 0, 0, 0)

   if expected != actual {
        t.Error("Failed for a single input as one, the rest zero, many step.")
   }

   expected = true
   sm.step(0, 1, 0, 0)
   sm.step(0, 1, 0, 0)
   sm.step(0, 1, 0, 0)
   sm.step(0, 1, 0, 0)
   sm.step(0, 1, 0, 0)
   actual = sm.step(0, 1, 0, 0)

   if expected != actual {
        t.Error("Failed for a single input as one, the rest zero, many step.")
   }

   expected = true
   sm.step(0, 0, 1, 0)
   sm.step(0, 0, 1, 0)
   sm.step(0, 0, 1, 0)
   sm.step(0, 0, 1, 0)
   sm.step(0, 0, 1, 0)
   actual = sm.step(0, 0, 1, 0)

   if expected != actual {
        t.Error("Failed for a single input as one, the rest zero, many step.")
   }

   expected = true
   sm.step(0, 0, 0, 1)
   sm.step(0, 0, 0, 1)
   sm.step(0, 0, 0, 1)
   sm.step(0, 0, 0, 1)
   sm.step(0, 0, 0, 1)
   actual = sm.step(0, 0, 0, 1)

   if expected != actual {
        t.Error("Failed for a single input as one, the rest zero, many step.")
   }
}

// The order of the bits doesn't actually matter...
func Test_PermutingOne(t *testing.T) {
   sm := StateMachine{0, 0}

   expected := true
   sm.step(1, 0, 0, 0)
   sm.step(0, 1, 0, 0)
   sm.step(0, 0, 1, 0)
   sm.step(0, 0, 0, 1)
   sm.step(1, 0, 0, 0)
   sm.step(1, 0, 0, 0)
   sm.step(0, 0, 1, 0)
   sm.step(1, 0, 0, 0)
   sm.step(1, 0, 0, 0)
   sm.step(1, 0, 0, 0)
   sm.step(0, 1, 0, 0)
   sm.step(0, 0, 1, 0)
   sm.step(0, 0, 0, 1)
   sm.step(0, 1, 0, 0)
   sm.step(0, 0, 1, 0)
   sm.step(0, 0, 0, 1)
   sm.step(0, 1, 0, 0)
   sm.step(0, 0, 1, 0)
   sm.step(0, 0, 0, 1)
   sm.step(0, 1, 0, 0)
   sm.step(0, 0, 1, 0)
   sm.step(0, 0, 0, 1)
   sm.step(0, 1, 0, 0)
   sm.step(0, 0, 1, 0)
   sm.step(0, 0, 0, 1)
   actual := sm.step(1, 0, 0, 0)

   if expected != actual {
        t.Error("Failed for a single input as one, the rest zero, many step, random permutations.")
   }
}

func Test_Two(t *testing.T) {
   sm := StateMachine{0, 0}

   expected := false
   actual := sm.step(1, 1, 0, 0)

   if expected != actual {
        t.Error("Failed for a two inputs as one, the rest zero, single step.")
   }
}

func Test_DoubleTwo(t *testing.T) {
   sm := StateMachine{0, 0}

   expected := true
   sm.step(1, 1, 0, 0)
   actual := sm.step(1, 1, 0, 0)

   if expected != actual {
        t.Error("Failed for a two inputs as one, the rest zero, double step.")
   }
}

func Test_MultipleTwo(t *testing.T) {
   sm := StateMachine{0, 0}

   expected := false
   sm.step(1, 1, 0, 0)
   sm.step(1, 1, 0, 0)
   actual := sm.step(1, 1, 0, 0)

   if expected != actual {
        t.Error("Failed for a two inputs as one, the rest zero, three step.")
   }

   expected = false
   actual = sm.step(1, 1, 0, 0)

   if expected != actual {
        t.Error("Failed for a two inputs as one, the rest zero, four step.")
   }

   expected = false
   actual = sm.step(1, 1, 0, 0)

   if expected != actual {
        t.Error("Failed for a two inputs as one, the rest zero, five step.")
   }

   expected = false
   actual = sm.step(1, 1, 0, 0)

   if expected != actual {
        t.Error("Failed for a two inputs as one, the rest zero, six step.")
   }
}
<commit_msg>Added some more tests<commit_after>package state_machine

import "testing"

func Test_Zero(t *testing.T) {
   sm := StateMachine{0, 0}

   expected := false
   actual := sm.step(0, 0, 0, 0)

   if expected != actual {
        t.Error("Failed for all inputs as zero, single step.")
   }
}

func Test_DoubleZero(t *testing.T) {
   sm := StateMachine{0, 0}

   expected := false
   sm.step(0, 0, 0, 0)
   actual := sm.step(0, 0, 0, 0)

   if expected != actual {
        t.Error("Failed for all inputs as zero, double step.")
   }
}

func Test_MultipleZero(t *testing.T) {
   sm := StateMachine{0, 0}

   expected := false
   sm.step(0, 0, 0, 0)
   sm.step(0, 0, 0, 0)
   sm.step(0, 0, 0, 0)
   sm.step(0, 0, 0, 0)
   sm.step(0, 0, 0, 0)
   sm.step(0, 0, 0, 0)
   sm.step(0, 0, 0, 0)
   sm.step(0, 0, 0, 0)
   actual := sm.step(0, 0, 0, 0)

   if expected != actual {
        t.Error("Failed for all inputs as zero, many step.")
   }
}

func Test_One(t *testing.T) {
   sm := StateMachine{0, 0}

   expected := true
   actual := sm.step(1, 0, 0, 0)

   if expected != actual {
        t.Error("Failed for a single input as one, the rest zero, single step.")
   }

   expected = true
   actual = sm.step(0, 1, 0, 0)

   if expected != actual {
        t.Error("Failed for a single input as one, the rest zero, single step.")
   }

   expected = true
   actual = sm.step(0, 0, 1, 0)

   if expected != actual {
        t.Error("Failed for a single input as one, the rest zero, single step.")
   }

   expected = true
   actual = sm.step(0, 0, 0, 1)

   if expected != actual {
        t.Error("Failed for a single input as one, the rest zero, single step.")
   }
}

// When we only have a single bit set, the value of c will never change
// This is thanks to the /2 component
func Test_DoubleOne(t *testing.T) {
   sm := StateMachine{0, 0}

   expected := true
   sm.step(1, 0, 0, 0)
   actual := sm.step(1, 0, 0, 0)

   if expected != actual {
        t.Error("Failed for a single input as one, the rest zero, double step.")
   }

   expected = true
   sm.step(0, 1, 0, 0)
   actual = sm.step(0, 1, 0, 0)

   if expected != actual {
        t.Error("Failed for a single input as one, the rest zero, double step.")
   }

   expected = true
   sm.step(0, 0, 1, 0)
   actual = sm.step(0, 0, 1, 0)

   if expected != actual {
        t.Error("Failed for a single input as one, the rest zero, double step.")
   }

   expected = true
   sm.step(0, 0, 0, 1)
   actual = sm.step(0, 0, 0, 1)

   if expected != actual {
        t.Error("Failed for a single input as one, the rest zero, double step.")
   }
}

func Test_MultipleOne(t *testing.T) {
   sm := StateMachine{0, 0}

   expected := true
   sm.step(1, 0, 0, 0)
   sm.step(1, 0, 0, 0)
   sm.step(1, 0, 0, 0)
   sm.step(1, 0, 0, 0)
   sm.step(1, 0, 0, 0)
   actual := sm.step(1, 0, 0, 0)

   if expected != actual {
        t.Error("Failed for a single input as one, the rest zero, many step.")
   }

   expected = true
   sm.step(0, 1, 0, 0)
   sm.step(0, 1, 0, 0)
   sm.step(0, 1, 0, 0)
   sm.step(0, 1, 0, 0)
   sm.step(0, 1, 0, 0)
   actual = sm.step(0, 1, 0, 0)

   if expected != actual {
        t.Error("Failed for a single input as one, the rest zero, many step.")
   }

   expected = true
   sm.step(0, 0, 1, 0)
   sm.step(0, 0, 1, 0)
   sm.step(0, 0, 1, 0)
   sm.step(0, 0, 1, 0)
   sm.step(0, 0, 1, 0)
   actual = sm.step(0, 0, 1, 0)

   if expected != actual {
        t.Error("Failed for a single input as one, the rest zero, many step.")
   }

   expected = true
   sm.step(0, 0, 0, 1)
   sm.step(0, 0, 0, 1)
   sm.step(0, 0, 0, 1)
   sm.step(0, 0, 0, 1)
   sm.step(0, 0, 0, 1)
   actual = sm.step(0, 0, 0, 1)

   if expected != actual {
        t.Error("Failed for a single input as one, the rest zero, many step.")
   }
}

// The order of the bits doesn't actually matter...
func Test_PermutingOne(t *testing.T) {
   sm := StateMachine{0, 0}

   expected := true
   sm.step(1, 0, 0, 0)
   sm.step(0, 1, 0, 0)
   sm.step(0, 0, 1, 0)
   sm.step(0, 0, 0, 1)
   sm.step(1, 0, 0, 0)
   sm.step(1, 0, 0, 0)
   sm.step(0, 0, 1, 0)
   sm.step(1, 0, 0, 0)
   sm.step(1, 0, 0, 0)
   sm.step(1, 0, 0, 0)
   sm.step(0, 1, 0, 0)
   sm.step(0, 0, 1, 0)
   sm.step(0, 0, 0, 1)
   sm.step(0, 1, 0, 0)
   sm.step(0, 0, 1, 0)
   sm.step(0, 0, 0, 1)
   sm.step(0, 1, 0, 0)
   sm.step(0, 0, 1, 0)
   sm.step(0, 0, 0, 1)
   sm.step(0, 1, 0, 0)
   sm.step(0, 0, 1, 0)
   sm.step(0, 0, 0, 1)
   sm.step(0, 1, 0, 0)
   sm.step(0, 0, 1, 0)
   sm.step(0, 0, 0, 1)
   actual := sm.step(1, 0, 0, 0)

   if expected != actual {
        t.Error("Failed for a single input as one, the rest zero, many step, random permutations.")
   }
}

func Test_Two(t *testing.T) {
   sm := StateMachine{0, 0}

   expected := false
   actual := sm.step(1, 1, 0, 0)

   if expected != actual {
        t.Error("Failed for a two inputs as one, the rest zero, single step.")
   }
}

func Test_DoubleTwo(t *testing.T) {
   sm := StateMachine{0, 0}

   expected := true
   sm.step(1, 1, 0, 0)
   actual := sm.step(1, 1, 0, 0)

   if expected != actual {
        t.Error("Failed for a two inputs as one, the rest zero, double step.")
   }
}

func Test_MultipleTwo(t *testing.T) {
   sm := StateMachine{0, 0}

   expected := false
   sm.step(1, 1, 0, 0)
   sm.step(1, 1, 0, 0)
   actual := sm.step(1, 1, 0, 0)

   if expected != actual {
        t.Error("Failed for a two inputs as one, the rest zero, three step.")
   }

   expected = false
   actual = sm.step(1, 1, 0, 0)

   if expected != actual {
        t.Error("Failed for a two inputs as one, the rest zero, four step.")
   }

   expected = false
   actual = sm.step(1, 1, 0, 0)

   if expected != actual {
        t.Error("Failed for a two inputs as one, the rest zero, five step.")
   }

   expected = false
   actual = sm.step(1, 1, 0, 0)

   if expected != actual {
        t.Error("Failed for a two inputs as one, the rest zero, six step.")
   }
}

func Test_Three(t *testing.T) {
   sm := StateMachine{0, 0}

   expected := true
   actual := sm.step(1, 1, 1, 0)

   if expected != actual {
        t.Error("Failed for a three inputs as one, the rest zero, single step.")
   }
}

func Test_Four(t *testing.T) {
   sm := StateMachine{0, 0}

   expected := false
   actual := sm.step(1, 1, 1, 1)

   if expected != actual {
        t.Error("Failed for a three inputs as one, the rest zero, single step.")
   }
}
<|endoftext|>


<commit_before>// Copyright 2013 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package rsyslog_test

import (
	"encoding/pem"

	jc "github.com/juju/testing/checkers"
	gc "launchpad.net/gocheck"

	"launchpad.net/juju-core/juju/testing"
	"launchpad.net/juju-core/state"
	"launchpad.net/juju-core/state/api/params"
	apirsyslog "launchpad.net/juju-core/state/api/rsyslog"
	"launchpad.net/juju-core/state/apiserver/common"
	commontesting "launchpad.net/juju-core/state/apiserver/common/testing"
	"launchpad.net/juju-core/state/apiserver/rsyslog"
	apiservertesting "launchpad.net/juju-core/state/apiserver/testing"
	coretesting "launchpad.net/juju-core/testing"
)

type rsyslogSuite struct {
	testing.JujuConnSuite
	*commontesting.EnvironWatcherTest
	authorizer apiservertesting.FakeAuthorizer
	resources  *common.Resources
	rsyslog    *rsyslog.RsyslogAPI
}

var _ = gc.Suite(&rsyslogSuite{})

func (s *rsyslogSuite) SetUpTest(c *gc.C) {
	s.JujuConnSuite.SetUpTest(c)
	s.authorizer = apiservertesting.FakeAuthorizer{
		LoggedIn:       true,
		EnvironManager: true,
	}
	s.resources = common.NewResources()
	api, err := rsyslog.NewRsyslogAPI(s.State, s.resources, s.authorizer)
	c.Assert(err, gc.IsNil)
	s.EnvironWatcherTest = commontesting.NewEnvironWatcherTest(
		api, s.State, s.resources, commontesting.NoSecrets)
}

func verifyRsyslogCACert(c *gc.C, st *apirsyslog.State, expected string) {
	cfg, err := st.GetRsyslogConfig()
	c.Assert(err, gc.IsNil)
	c.Assert(cfg.CACert, gc.DeepEquals, expected)
}

func (s *rsyslogSuite) TestSetRsyslogCert(c *gc.C) {
	st, _ := s.OpenAPIAsNewMachine(c, state.JobManageEnviron)
	err := st.Rsyslog().SetRsyslogCert(coretesting.CACert)
	c.Assert(err, gc.IsNil)
	verifyRsyslogCACert(c, st.Rsyslog(), coretesting.CACert)
}

func (s *rsyslogSuite) TestSetRsyslogCertNil(c *gc.C) {
	st, _ := s.OpenAPIAsNewMachine(c, state.JobManageEnviron)
	err := st.Rsyslog().SetRsyslogCert("")
	c.Assert(err, gc.ErrorMatches, "no certificates found")
	verifyRsyslogCACert(c, st.Rsyslog(), "")
}

func (s *rsyslogSuite) TestSetRsyslogCertInvalid(c *gc.C) {
	st, _ := s.OpenAPIAsNewMachine(c, state.JobManageEnviron)
	err := st.Rsyslog().SetRsyslogCert(string(pem.EncodeToMemory(&pem.Block{
		Type:  "CERTIFICATE",
		Bytes: []byte("not a valid certificate"),
	})))
	c.Assert(err, gc.ErrorMatches, ".*structure error.*")
	verifyRsyslogCACert(c, st.Rsyslog(), "")
}

func (s *rsyslogSuite) TestSetRsyslogCertPerms(c *gc.C) {
	st, _ := s.OpenAPIAsNewMachine(c, state.JobHostUnits)
	err := st.Rsyslog().SetRsyslogCert(coretesting.CACert)
	c.Assert(err, gc.ErrorMatches, "invalid entity name or password")
	c.Assert(err, jc.Satisfies, params.IsCodeUnauthorized)
	// Verify no change was effected.
	verifyRsyslogCACert(c, st.Rsyslog(), "")
}
<commit_msg>Test fixes<commit_after>// Copyright 2013 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package rsyslog_test

import (
	"encoding/pem"

	jc "github.com/juju/testing/checkers"
	gc "launchpad.net/gocheck"

	"launchpad.net/juju-core/instance"
	"launchpad.net/juju-core/juju/testing"
	"launchpad.net/juju-core/state"
	"launchpad.net/juju-core/state/api/params"
	apirsyslog "launchpad.net/juju-core/state/api/rsyslog"
	"launchpad.net/juju-core/state/apiserver/common"
	commontesting "launchpad.net/juju-core/state/apiserver/common/testing"
	"launchpad.net/juju-core/state/apiserver/rsyslog"
	apiservertesting "launchpad.net/juju-core/state/apiserver/testing"
	coretesting "launchpad.net/juju-core/testing"
)

type rsyslogSuite struct {
	testing.JujuConnSuite
	*commontesting.EnvironWatcherTest
	authorizer apiservertesting.FakeAuthorizer
	resources  *common.Resources
	rsyslog    *rsyslog.RsyslogAPI
}

var _ = gc.Suite(&rsyslogSuite{})

func (s *rsyslogSuite) SetUpTest(c *gc.C) {
	s.JujuConnSuite.SetUpTest(c)
	s.authorizer = apiservertesting.FakeAuthorizer{
		LoggedIn:       true,
		EnvironManager: true,
	}
	s.resources = common.NewResources()
	api, err := rsyslog.NewRsyslogAPI(s.State, s.resources, s.authorizer)
	c.Assert(err, gc.IsNil)
	s.EnvironWatcherTest = commontesting.NewEnvironWatcherTest(
		api, s.State, s.resources, commontesting.NoSecrets)
}

func verifyRsyslogCACert(c *gc.C, st *apirsyslog.State, expected string) {
	cfg, err := st.GetRsyslogConfig()
	c.Assert(err, gc.IsNil)
	c.Assert(cfg.CACert, gc.DeepEquals, expected)
}

func (s *rsyslogSuite) TestSetRsyslogCert(c *gc.C) {
	st, m := s.OpenAPIAsNewMachine(c, state.JobManageEnviron)
	err := m.SetAddresses(instance.NewAddress("0.1.2.3", instance.NetworkUnknown))
	c.Assert(err, gc.IsNil)

	err = st.Rsyslog().SetRsyslogCert(coretesting.CACert)
	c.Assert(err, gc.IsNil)
	verifyRsyslogCACert(c, st.Rsyslog(), coretesting.CACert)
}

func (s *rsyslogSuite) TestSetRsyslogCertNil(c *gc.C) {
	st, m := s.OpenAPIAsNewMachine(c, state.JobManageEnviron)
	err := m.SetAddresses(instance.NewAddress("0.1.2.3", instance.NetworkUnknown))
	c.Assert(err, gc.IsNil)

	err = st.Rsyslog().SetRsyslogCert("")
	c.Assert(err, gc.ErrorMatches, "no certificates found")
	verifyRsyslogCACert(c, st.Rsyslog(), "")
}

func (s *rsyslogSuite) TestSetRsyslogCertInvalid(c *gc.C) {
	st, m := s.OpenAPIAsNewMachine(c, state.JobManageEnviron)
	err := m.SetAddresses(instance.NewAddress("0.1.2.3", instance.NetworkUnknown))
	c.Assert(err, gc.IsNil)

	err = st.Rsyslog().SetRsyslogCert(string(pem.EncodeToMemory(&pem.Block{
		Type:  "CERTIFICATE",
		Bytes: []byte("not a valid certificate"),
	})))
	c.Assert(err, gc.ErrorMatches, ".*structure error.*")
	verifyRsyslogCACert(c, st.Rsyslog(), "")
}

func (s *rsyslogSuite) TestSetRsyslogCertPerms(c *gc.C) {
	_, m := s.OpenAPIAsNewMachine(c, state.JobManageEnviron)
	err := m.SetAddresses(instance.NewAddress("0.1.2.3", instance.NetworkUnknown))
	c.Assert(err, gc.IsNil)

	unitState, _ := s.OpenAPIAsNewMachine(c, state.JobHostUnits)
	err = unitState.Rsyslog().SetRsyslogCert(coretesting.CACert)
	c.Assert(err, gc.ErrorMatches, "invalid entity name or password")
	c.Assert(err, jc.Satisfies, params.IsCodeUnauthorized)
	// Verify no change was effected.
	verifyRsyslogCACert(c, unitState.Rsyslog(), "")
}
<|endoftext|>


<commit_before>//: ----------------------------------------------------------------------------
//: Copyright (C) 2017 Verizon.  All Rights Reserved.
//: All Rights Reserved
//:
//: file:    flow_sample.go
//: details: TODO
//: author:  Mehrdad Arshad Rad
//: date:    02/01/2017
//:
//: Licensed under the Apache License, Version 2.0 (the "License");
//: you may not use this file except in compliance with the License.
//: You may obtain a copy of the License at
//:
//:     http://www.apache.org/licenses/LICENSE-2.0
//:
//: Unless required by applicable law or agreed to in writing, software
//: distributed under the License is distributed on an "AS IS" BASIS,
//: WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//: See the License for the specific language governing permissions and
//: limitations under the License.
//: ----------------------------------------------------------------------------

package sflow

import (
	"errors"
	"io"
	"net"

	"github.com/VerizonDigital/vflow/packet"
)

const (
	// SFDataRawHeader is sFlow Raw Packet Header number
	SFDataRawHeader = 1

	// SFDataExtSwitch is sFlow Extended Switch Data number
	SFDataExtSwitch = 1001

	// SFDataExtRouter is sFlow Extended Router Data number
	SFDataExtRouter = 1002
)

// FlowSample represents single flow sample
type FlowSample struct {
	SequenceNo   uint32 // Incremented with each flow sample
	SourceID     byte   // sfSourceID
	SamplingRate uint32 // sfPacketSamplingRate
	SamplePool   uint32 // Total number of packets that could have been sampled
	Drops        uint32 // Number of times a packet was dropped due to lack of resources
	Input        uint32 // SNMP ifIndex of input interface
	Output       uint32 // SNMP ifIndex of input interface
	RecordsNo    uint32 // Number of records to follow
	Records      []Record
}

// SampledHeader represents sampled header
type SampledHeader struct {
	Protocol     uint32 // (enum SFLHeader_protocol)
	FrameLength  uint32 // Original length of packet before sampling
	Stripped     uint32 // Header/trailer bytes stripped by sender
	HeaderLength uint32 // Length of sampled header bytes to follow
	Header       []byte // Header bytes
}

// ExtSwitchData represents Extended Switch Data
type ExtSwitchData struct {
	SrcVlan     uint32 // The 802.1Q VLAN id of incoming frame
	SrcPriority uint32 // The 802.1p priority of incoming frame
	DstVlan     uint32 // The 802.1Q VLAN id of outgoing frame
	DstPriority uint32 // The 802.1p priority of outgoing frame
}

type ExtRouterData struct {
	NextHop net.IP
	SrcMask uint32
	DstMask uint32
}

var (
	errMaxOutEthernetLength = errors.New("the ethernet length is greater than 1500")
)

func (fs *FlowSample) unmarshal(r io.ReadSeeker) error {
	var err error

	if err = read(r, &fs.SequenceNo); err != nil {
		return err
	}

	if err = read(r, &fs.SourceID); err != nil {
		return err
	}

	r.Seek(3, 1) // skip counter sample decoding

	if err = read(r, &fs.SamplingRate); err != nil {
		return err
	}

	if err = read(r, &fs.SamplePool); err != nil {
		return err
	}

	if err = read(r, &fs.Drops); err != nil {
		return err
	}

	if err = read(r, &fs.Input); err != nil {
		return err
	}

	if err = read(r, &fs.Output); err != nil {
		return err
	}

	if err = read(r, &fs.RecordsNo); err != nil {
		return err
	}

	return nil
}

func (sh *SampledHeader) unmarshal(r io.Reader) error {
	var err error

	if err = read(r, &sh.Protocol); err != nil {
		return err
	}

	if err = read(r, &sh.FrameLength); err != nil {
		return err
	}

	if err = read(r, &sh.Stripped); err != nil {
		return err
	}

	if err = read(r, &sh.HeaderLength); err != nil {
		return err
	}

	if sh.HeaderLength > 1500 {
		return errMaxOutEthernetLength
	}

	// cut off a header length mod 4 == 0 number of bytes
	tmp := (4 - sh.HeaderLength) % 4
	if tmp < 0 {
		tmp += 4
	}

	sh.Header = make([]byte, sh.HeaderLength+tmp)
	if _, err = r.Read(sh.Header); err != nil {
		return err
	}

	sh.Header = sh.Header[:sh.HeaderLength]

	return nil
}

func (es *ExtSwitchData) unmarshal(r io.Reader) error {
	var err error

	if err = read(r, &es.SrcVlan); err != nil {
		return err
	}

	if err = read(r, &es.SrcPriority); err != nil {
		return err
	}

	if err = read(r, &es.DstVlan); err != nil {
		return err
	}

	if err = read(r, &es.SrcPriority); err != nil {
		return err
	}

	return nil
}

func (er *ExtRouterData) unmarshal(r io.Reader, l uint32) error {
	var err error

	buff := make([]byte, l-8)
	if err = read(r, &buff); err != nil {
		return err
	}
	er.NextHop = buff[4:]

	if err = read(r, &er.SrcMask); err != nil {
		return err
	}

	if err = read(r, &er.DstMask); err != nil {
		return err
	}

	return nil
}

func decodeFlowSample(r io.ReadSeeker) (*FlowSample, error) {
	var (
		fs          = new(FlowSample)
		rTypeFormat uint32
		rTypeLength uint32
		err         error
	)

	if err = fs.unmarshal(r); err != nil {
		return nil, err
	}

	for i := uint32(0); i < fs.RecordsNo; i++ {
		if err = read(r, &rTypeFormat); err != nil {
			return nil, err
		}
		if err = read(r, &rTypeLength); err != nil {
			return nil, err
		}

		switch rTypeFormat {
		case SFDataRawHeader:
			d, err := decodeSampledHeader(r)
			if err != nil {
				return fs, err
			}
			fs.Records = append(fs.Records, d)
		case SFDataExtSwitch:
			d, err := decodeExtSwitchData(r)
			if err != nil {
				return fs, err
			}
			fs.Records = append(fs.Records, d)
		case SFDataExtRouter:
			d, err := decodeExtRouterData(r, rTypeLength)
			if err != nil {
				return fs, err
			}
			fs.Records = append(fs.Records, d)
		default:
			r.Seek(int64(rTypeLength), 1)
		}
	}

	return fs, nil
}

func decodeSampledHeader(r io.Reader) (*packet.Packet, error) {
	var (
		h   = new(SampledHeader)
		err error
	)

	if err = h.unmarshal(r); err != nil {
		return nil, err
	}

	p := packet.NewPacket()
	d, err := p.Decoder(h.Header, h.Protocol)
	if err != nil {
		return nil, err
	}

	return d, nil
}

func decodeExtSwitchData(r io.Reader) (*ExtSwitchData, error) {
	var es = new(ExtSwitchData)

	if err := es.unmarshal(r); err != nil {
		return nil, err
	}

	return es, nil
}

func decodeExtRouterData(r io.Reader, l uint32) (*ExtRouterData, error) {
	var er = new(ExtRouterData)

	if err := er.unmarshal(r, l); err != nil {
		return nil, err
	}

	return er, nil
}
<commit_msg>change records to map<commit_after>//: ----------------------------------------------------------------------------
//: Copyright (C) 2017 Verizon.  All Rights Reserved.
//: All Rights Reserved
//:
//: file:    flow_sample.go
//: details: TODO
//: author:  Mehrdad Arshad Rad
//: date:    02/01/2017
//:
//: Licensed under the Apache License, Version 2.0 (the "License");
//: you may not use this file except in compliance with the License.
//: You may obtain a copy of the License at
//:
//:     http://www.apache.org/licenses/LICENSE-2.0
//:
//: Unless required by applicable law or agreed to in writing, software
//: distributed under the License is distributed on an "AS IS" BASIS,
//: WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//: See the License for the specific language governing permissions and
//: limitations under the License.
//: ----------------------------------------------------------------------------

package sflow

import (
	"errors"
	"io"
	"net"

	"github.com/VerizonDigital/vflow/packet"
)

const (
	// SFDataRawHeader is sFlow Raw Packet Header number
	SFDataRawHeader = 1

	// SFDataExtSwitch is sFlow Extended Switch Data number
	SFDataExtSwitch = 1001

	// SFDataExtRouter is sFlow Extended Router Data number
	SFDataExtRouter = 1002
)

// FlowSample represents single flow sample
type FlowSample struct {
	SequenceNo   uint32 // Incremented with each flow sample
	SourceID     byte   // sfSourceID
	SamplingRate uint32 // sfPacketSamplingRate
	SamplePool   uint32 // Total number of packets that could have been sampled
	Drops        uint32 // Number of times a packet was dropped due to lack of resources
	Input        uint32 // SNMP ifIndex of input interface
	Output       uint32 // SNMP ifIndex of input interface
	RecordsNo    uint32 // Number of records to follow
	Records      map[string]Record
}

// SampledHeader represents sampled header
type SampledHeader struct {
	Protocol     uint32 // (enum SFLHeader_protocol)
	FrameLength  uint32 // Original length of packet before sampling
	Stripped     uint32 // Header/trailer bytes stripped by sender
	HeaderLength uint32 // Length of sampled header bytes to follow
	Header       []byte // Header bytes
}

// ExtSwitchData represents Extended Switch Data
type ExtSwitchData struct {
	SrcVlan     uint32 // The 802.1Q VLAN id of incoming frame
	SrcPriority uint32 // The 802.1p priority of incoming frame
	DstVlan     uint32 // The 802.1Q VLAN id of outgoing frame
	DstPriority uint32 // The 802.1p priority of outgoing frame
}

type ExtRouterData struct {
	NextHop net.IP
	SrcMask uint32
	DstMask uint32
}

var (
	errMaxOutEthernetLength = errors.New("the ethernet length is greater than 1500")
)

func (fs *FlowSample) unmarshal(r io.ReadSeeker) error {
	var err error

	if err = read(r, &fs.SequenceNo); err != nil {
		return err
	}

	if err = read(r, &fs.SourceID); err != nil {
		return err
	}

	r.Seek(3, 1) // skip counter sample decoding

	if err = read(r, &fs.SamplingRate); err != nil {
		return err
	}

	if err = read(r, &fs.SamplePool); err != nil {
		return err
	}

	if err = read(r, &fs.Drops); err != nil {
		return err
	}

	if err = read(r, &fs.Input); err != nil {
		return err
	}

	if err = read(r, &fs.Output); err != nil {
		return err
	}

	if err = read(r, &fs.RecordsNo); err != nil {
		return err
	}

	return nil
}

func (sh *SampledHeader) unmarshal(r io.Reader) error {
	var err error

	if err = read(r, &sh.Protocol); err != nil {
		return err
	}

	if err = read(r, &sh.FrameLength); err != nil {
		return err
	}

	if err = read(r, &sh.Stripped); err != nil {
		return err
	}

	if err = read(r, &sh.HeaderLength); err != nil {
		return err
	}

	if sh.HeaderLength > 1500 {
		return errMaxOutEthernetLength
	}

	// cut off a header length mod 4 == 0 number of bytes
	tmp := (4 - sh.HeaderLength) % 4
	if tmp < 0 {
		tmp += 4
	}

	sh.Header = make([]byte, sh.HeaderLength+tmp)
	if _, err = r.Read(sh.Header); err != nil {
		return err
	}

	sh.Header = sh.Header[:sh.HeaderLength]

	return nil
}

func (es *ExtSwitchData) unmarshal(r io.Reader) error {
	var err error

	if err = read(r, &es.SrcVlan); err != nil {
		return err
	}

	if err = read(r, &es.SrcPriority); err != nil {
		return err
	}

	if err = read(r, &es.DstVlan); err != nil {
		return err
	}

	if err = read(r, &es.SrcPriority); err != nil {
		return err
	}

	return nil
}

func (er *ExtRouterData) unmarshal(r io.Reader, l uint32) error {
	var err error

	buff := make([]byte, l-8)
	if err = read(r, &buff); err != nil {
		return err
	}
	er.NextHop = buff[4:]

	if err = read(r, &er.SrcMask); err != nil {
		return err
	}

	if err = read(r, &er.DstMask); err != nil {
		return err
	}

	return nil
}

func decodeFlowSample(r io.ReadSeeker) (*FlowSample, error) {
	var (
		fs          = new(FlowSample)
		rTypeFormat uint32
		rTypeLength uint32
		err         error
	)

	if err = fs.unmarshal(r); err != nil {
		return nil, err
	}

	fs.Records = make(map[string]Record)

	for i := uint32(0); i < fs.RecordsNo; i++ {
		if err = read(r, &rTypeFormat); err != nil {
			return nil, err
		}
		if err = read(r, &rTypeLength); err != nil {
			return nil, err
		}

		switch rTypeFormat {
		case SFDataRawHeader:
			d, err := decodeSampledHeader(r)
			if err != nil {
				return fs, err
			}
			fs.Records["RawHeader"] = d
		case SFDataExtSwitch:
			d, err := decodeExtSwitchData(r)
			if err != nil {
				return fs, err
			}

			fs.Records["ExtSwitch"] = d
		case SFDataExtRouter:
			d, err := decodeExtRouterData(r, rTypeLength)
			if err != nil {
				return fs, err
			}

			fs.Records["ExtRouter"] = d
		default:
			r.Seek(int64(rTypeLength), 1)
		}
	}

	return fs, nil
}

func decodeSampledHeader(r io.Reader) (*packet.Packet, error) {
	var (
		h   = new(SampledHeader)
		err error
	)

	if err = h.unmarshal(r); err != nil {
		return nil, err
	}

	p := packet.NewPacket()
	d, err := p.Decoder(h.Header, h.Protocol)
	if err != nil {
		return nil, err
	}

	return d, nil
}

func decodeExtSwitchData(r io.Reader) (*ExtSwitchData, error) {
	var es = new(ExtSwitchData)

	if err := es.unmarshal(r); err != nil {
		return nil, err
	}

	return es, nil
}

func decodeExtRouterData(r io.Reader, l uint32) (*ExtRouterData, error) {
	var er = new(ExtRouterData)

	if err := er.unmarshal(r, l); err != nil {
		return nil, err
	}

	return er, nil
}
<|endoftext|>


<commit_before>package sechat

import (
	"errors"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"
	"regexp"
	"strconv"
	"strings"
	"time"
)

var conflictRegexp = regexp.MustCompile(`\d+`)

// postForm is a utility method for sending a POST request with form data. The
// fkey is automatically added to the form data sent. If a 409 Conflict
// response is received, the request is retried after the specified amount of
// time (to work around any throttle). Consequently, this method is blocking.
func (c *Conn) postForm(path string, data *url.Values) (*http.Response, error) {
	data.Set("fkey", c.fkey)
	req, err := http.NewRequest(
		http.MethodPost,
		fmt.Sprintf("http://chat.stackexchange.com%s", path),
		strings.NewReader(data.Encode()),
	)
	if err != nil {
		return nil, err
	}
	req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
	for {
		res, err := c.client.Do(req)
		if err != nil {
			return nil, err
		}
		if res.StatusCode >= 400 {
			// For HTTP 409, wait for the throttle to cool down
			if res.StatusCode == http.StatusConflict {
				b, err := ioutil.ReadAll(res.Body)
				if err == nil {
					m := conflictRegexp.FindStringSubmatch(string(b))
					if len(m) != 0 {
						i, _ := strconv.Atoi(m[0])
						c.log.Debugf("retrying %s in %d second(s)", path, i)
						time.Sleep(time.Duration(i) * time.Second)
						continue
					}
				}
			}
			return nil, errors.New(res.Status)
		}
		return res, nil
	}
}
<commit_msg>Switch retry to info log message.<commit_after>package sechat

import (
	"errors"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"
	"regexp"
	"strconv"
	"strings"
	"time"
)

var conflictRegexp = regexp.MustCompile(`\d+`)

// postForm is a utility method for sending a POST request with form data. The
// fkey is automatically added to the form data sent. If a 409 Conflict
// response is received, the request is retried after the specified amount of
// time (to work around any throttle). Consequently, this method is blocking.
func (c *Conn) postForm(path string, data *url.Values) (*http.Response, error) {
	data.Set("fkey", c.fkey)
	req, err := http.NewRequest(
		http.MethodPost,
		fmt.Sprintf("http://chat.stackexchange.com%s", path),
		strings.NewReader(data.Encode()),
	)
	if err != nil {
		return nil, err
	}
	req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
	for {
		res, err := c.client.Do(req)
		if err != nil {
			return nil, err
		}
		if res.StatusCode >= 400 {
			// For HTTP 409, wait for the throttle to cool down
			if res.StatusCode == http.StatusConflict {
				b, err := ioutil.ReadAll(res.Body)
				if err == nil {
					m := conflictRegexp.FindStringSubmatch(string(b))
					if len(m) != 0 {
						i, _ := strconv.Atoi(m[0])
						c.log.Infof("retrying %s in %d second(s)", path, i)
						time.Sleep(time.Duration(i) * time.Second)
						continue
					}
				}
			}
			return nil, errors.New(res.Status)
		}
		return res, nil
	}
}
<|endoftext|>


<commit_before>package pull

import (
	"crypto/hmac"
	"crypto/sha1"
	"encoding/hex"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"

	"github.com/google/go-github/github"
)

type Middleware func(*github.Client, *github.PullRequest)

type Handler struct {
	Client *github.Client
	*Configuration
}

type Configuration struct {
	Client       *http.Client
	Repositories []Repository
	Middlewares  []Middleware
	Secret       string
}

type Repository struct {
	Username string
	Name     string
}

func New(c Configuration) *Handler {
	return &Handler{github.NewClient(c.Client), &c}
}

func (l *Handler) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
	w := io.MultiWriter(resp, logger{})

	t := req.Header.Get("X-GitHub-Event")
	if t == "ping" {
		resp.WriteHeader(http.StatusAccepted)
		fmt.Fprintf(w, "Received ping")
		return
	}

	if t != "pull_request" {
		resp.WriteHeader(http.StatusNotImplemented)
		fmt.Fprintf(w, "Unsupported event")
		return
	}

	body, _ := ioutil.ReadAll(req.Body)

	if l.Secret != "" {
		s := req.Header.Get("X-Hub-Signature")

		if s == "" {
			resp.WriteHeader(http.StatusForbidden)
			fmt.Fprintf(w, "X-Hub-Signature required for HMAC verification")
			return
		}

		hash := hmac.New(sha1.New, []byte(l.Secret))
		hash.Write(body)
		expected := "sha1=" + hex.EncodeToString(hash.Sum(nil))

		if !hmac.Equal([]byte(expected), []byte(s)) {
			resp.WriteHeader(http.StatusForbidden)
			fmt.Fprintf(w, "HMAC verification failed")
			return
		}
	}

	var event github.PullRequestEvent
	err := json.Unmarshal(body, &event)
	//err := decoder.Decode(&event)

	if err != nil {
		resp.WriteHeader(http.StatusBadRequest)
		fmt.Fprintf(w, "Bad JSON")
		return
	}
	fmt.Printf("%s %s:%d\n", *event.Action, *event.Repo.FullName, *event.Number)

	pending(l.Client, event.PullRequest)
	go l.runner(event.PullRequest)
}

func (l *Handler) runner(pr *github.PullRequest) {
	for _, m := range l.Middlewares {
		m(l.Client, pr)
	}
}

func pending(client *github.Client, event *github.PullRequest) {
	user, repo, number := Data(event)

	commits, _, _ := client.PullRequests.ListCommits(user, repo, number, nil)
	for _, c := range commits {
		status := &github.RepoStatus{
			State:       github.String("pending"),
			Description: github.String("Running tests."),
		}

		client.Repositories.CreateStatus(user, repo, *c.SHA, status)
	}
}
<commit_msg>Refactoring and moving to HTTP middleware layout.<commit_after>package pull

import (
	"bytes"
	"crypto/hmac"
	"crypto/sha1"
	"encoding/hex"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"

	"github.com/google/go-github/github"
)

type Middleware func(*github.Client, *github.PullRequest)
type Middlewares map[string][]Middleware

type Handler struct {
	Client *github.Client
	*Configuration
}

type Configuration struct {
	Client      *http.Client
	Middlewares Middlewares
	Secret      string
}

func New(c Configuration) *Handler {
	return &Handler{github.NewClient(c.Client), &c}
}

func (l *Handler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
	pingHandler(implementsHandler(secretHandler(l.Secret,
		http.HandlerFunc(
			func(w http.ResponseWriter, req *http.Request) {
				body, _ := ioutil.ReadAll(req.Body)

				var event github.PullRequestEvent
				err := json.Unmarshal(body, &event)
				//err := decoder.Decode(&event)

				if err != nil {
					w.WriteHeader(http.StatusBadRequest)
					fmt.Fprintf(w, "Bad JSON")
					return
				}
				fmt.Printf("%s %s:%d\n", *event.Action, *event.Repo.FullName, *event.Number)

				//pending(l.Client, event.PullRequest)
				go func() {
					for _, m := range l.Middlewares[*event.Action] {
						m(l.Client, event.PullRequest)
					}
				}()
			})))).ServeHTTP(w, req)
}

func pending(client *github.Client, event *github.PullRequest) {
	user, repo, number := Data(event)

	commits, _, _ := client.PullRequests.ListCommits(user, repo, number, nil)
	for _, c := range commits {
		status := &github.RepoStatus{
			State:       github.String("pending"),
			Description: github.String("Running tests."),
		}

		client.Repositories.CreateStatus(user, repo, *c.SHA, status)
	}
}

func pingHandler(next http.Handler) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
		t := req.Header.Get("X-GitHub-Event")
		if t == "ping" {
			w.WriteHeader(http.StatusAccepted)
			fmt.Fprintf(w, "Received ping")
			return
		}
		next.ServeHTTP(w, req)
	})
}

func implementsHandler(next http.Handler) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
		t := req.Header.Get("X-GitHub-Event")
		if t != "pull_request" {
			w.WriteHeader(http.StatusNotImplemented)
			fmt.Fprintf(w, "Unsupported event")
			return
		}
		next.ServeHTTP(w, req)
	})
}

func secretHandler(secret string, next http.Handler) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {

		if secret != "" {
			s := req.Header.Get("X-Hub-Signature")

			if s == "" {
				w.WriteHeader(http.StatusForbidden)
				fmt.Fprintf(w, "X-Hub-Signature required for HMAC verification")
				return
			}

			body, _ := ioutil.ReadAll(req.Body)

			hash := hmac.New(sha1.New, []byte(secret))
			hash.Write(body)
			expected := "sha1=" + hex.EncodeToString(hash.Sum(nil))

			if !hmac.Equal([]byte(expected), []byte(s)) {
				w.WriteHeader(http.StatusForbidden)
				fmt.Fprintf(w, "HMAC verification failed")
				return
			}
			req.Body = ioutil.NopCloser(bytes.NewBuffer(body))
		}

		next.ServeHTTP(w, req)
	})
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"io"
	"math"
	"net/http"
	"time"

	"github.com/dustin/go-humanize"
	"github.com/go-errors/errors"
	"github.com/itchio/butler/comm"
	"github.com/itchio/go-itchio"
	"github.com/itchio/httpkit/uploader"
	"github.com/itchio/wharf/counter"
	"github.com/itchio/wharf/pools"
	"github.com/itchio/wharf/pwr"
	"github.com/itchio/wharf/state"
	"github.com/itchio/wharf/tlc"
	"github.com/itchio/wharf/wsync"
)

// AlmostThereThreshold is the amount of data left where the progress indicator isn't indicative anymore.
// At this point, we're basically waiting for build files to be finalized.
const AlmostThereThreshold int64 = 10 * 1024

func push(buildPath string, specStr string, userVersion string, fixPerms bool) {
	go versionCheck()
	must(doPush(buildPath, specStr, userVersion, fixPerms))
}

func doPush(buildPath string, specStr string, userVersion string, fixPerms bool) error {
	// start walking source container while waiting on auth flow
	sourceContainerChan := make(chan walkResult)
	walkErrs := make(chan error)
	go doWalk(buildPath, sourceContainerChan, walkErrs, fixPerms)

	spec, err := itchio.ParseSpec(specStr)
	if err != nil {
		return errors.Wrap(err, 1)
	}

	err = spec.EnsureChannel()
	if err != nil {
		return errors.Wrap(err, 1)
	}

	client, err := authenticateViaOauth()
	if err != nil {
		return errors.Wrap(err, 1)
	}

	newBuildRes, err := client.CreateBuild(spec.Target, spec.Channel, userVersion)
	if err != nil {
		return errors.Wrap(err, 1)
	}

	buildID := newBuildRes.Build.ID
	parentID := newBuildRes.Build.ParentBuild.ID

	var targetSignature *pwr.SignatureInfo

	if parentID == 0 {
		comm.Opf("For channel `%s`: pushing first build", spec.Channel)
		targetSignature = &pwr.SignatureInfo{
			Container: &tlc.Container{},
			Hashes:    make([]wsync.BlockHash, 0),
		}
	} else {
		comm.Opf("For channel `%s`: last build is %d, downloading its signature", spec.Channel, parentID)
		var buildFiles itchio.ListBuildFilesResponse
		buildFiles, err = client.ListBuildFiles(parentID)
		if err != nil {
			return errors.Wrap(err, 1)
		}

		signatureFile := itchio.FindBuildFile(itchio.BuildFileType_SIGNATURE, buildFiles.Files)
		if signatureFile == nil {
			comm.Dief("Could not find signature for parent build %d, aborting", parentID)
		}

		var signatureReader io.Reader
		signatureReader, err = client.DownloadBuildFile(parentID, signatureFile.ID)
		if err != nil {
			return errors.Wrap(err, 1)
		}

		targetSignature, err = pwr.ReadSignature(signatureReader)
		if err != nil {
			return errors.Wrap(err, 1)
		}
	}

	newPatchRes, newSignatureRes, err := createBothFiles(client, buildID)
	if err != nil {
		return errors.Wrap(err, 1)
	}

	uploadDone := make(chan bool)
	uploadErrs := make(chan error)

	patchWriter, err := uploader.NewResumableUpload(newPatchRes.File.UploadURL,
		uploadDone, uploadErrs, comm.NewStateConsumer())
	patchWriter.MaxChunkGroup = *appArgs.maxChunkGroup
	if err != nil {
		return errors.Wrap(err, 1)
	}

	signatureWriter, err := uploader.NewResumableUpload(newSignatureRes.File.UploadURL,
		uploadDone, uploadErrs, comm.NewStateConsumer())
	signatureWriter.MaxChunkGroup = *appArgs.maxChunkGroup
	if err != nil {
		return errors.Wrap(err, 1)
	}

	comm.Debugf("Launching patch & signature channels")

	patchCounter := counter.NewWriter(patchWriter)
	signatureCounter := counter.NewWriter(signatureWriter)

	// we started walking the source container in the beginning,
	// we actually need it now.
	// note that we could actually start diffing before all the file
	// creation & upload setup is done

	var sourceContainer *tlc.Container
	var sourcePool wsync.Pool

	comm.Debugf("Waiting for source container")
	select {
	case walkErr := <-walkErrs:
		return errors.Wrap(walkErr, 1)
	case walkies := <-sourceContainerChan:
		comm.Debugf("Got sourceContainer!")
		sourceContainer = walkies.container
		sourcePool = walkies.pool
		break
	}

	comm.Logf("")
	comm.Opf("Pushing %s (%s)", humanize.IBytes(uint64(sourceContainer.Size)), sourceContainer.Stats())

	comm.Debugf("Building diff context")
	var readBytes int64

	bytesPerSec := float64(0)
	lastUploadedBytes := int64(0)
	stopTicking := make(chan struct{})

	updateProgress := func() {
		uploadedBytes := int64(float64(patchWriter.UploadedBytes))

		// input bytes that aren't in output, for example:
		//  - bytes that have been compressed away
		//  - bytes that were in old build and were simply reused
		goneBytes := readBytes - patchWriter.TotalBytes

		conservativeTotalBytes := sourceContainer.Size - goneBytes

		leftBytes := conservativeTotalBytes - uploadedBytes
		if leftBytes > AlmostThereThreshold {
			netStatus := "- network idle"
			if bytesPerSec > 1 {
				netStatus = fmt.Sprintf("@ %s/s", humanize.IBytes(uint64(bytesPerSec)))
			}
			comm.ProgressLabel(fmt.Sprintf("%s, %s left", netStatus, humanize.IBytes(uint64(leftBytes))))
		} else {
			comm.ProgressLabel(fmt.Sprintf("- almost there"))
		}

		conservativeProgress := float64(uploadedBytes) / float64(conservativeTotalBytes)
		conservativeProgress = min(1.0, conservativeProgress)
		comm.Progress(conservativeProgress)

		comm.ProgressScale(float64(readBytes) / float64(sourceContainer.Size))
	}

	go func() {
		ticker := time.NewTicker(time.Second * time.Duration(2))
		for {
			select {
			case <-ticker.C:
				bytesPerSec = float64(patchWriter.UploadedBytes-lastUploadedBytes) / 2.0
				lastUploadedBytes = patchWriter.UploadedBytes
				updateProgress()
			case <-stopTicking:
				break
			}
		}
	}()

	patchWriter.OnProgress = updateProgress

	stateConsumer := &state.Consumer{
		OnProgress: func(progress float64) {
			readBytes = int64(float64(sourceContainer.Size) * progress)
			updateProgress()
		},
	}

	dctx := &pwr.DiffContext{
		Compression: &pwr.CompressionSettings{
			Algorithm: pwr.CompressionAlgorithm_BROTLI,
			Quality:   1,
		},

		SourceContainer: sourceContainer,
		Pool:            sourcePool,

		TargetContainer: targetSignature.Container,
		TargetSignature: targetSignature.Hashes,

		Consumer: stateConsumer,
	}

	comm.StartProgress()
	comm.ProgressScale(0.0)
	err = dctx.WritePatch(patchCounter, signatureCounter)
	if err != nil {
		return errors.Wrap(err, 1)
	}

	// close in a goroutine to avoid deadlocking
	doClose := func(c io.Closer, done chan bool, errs chan error) {
		closeErr := c.Close()
		if closeErr != nil {
			errs <- errors.Wrap(closeErr, 1)
			return
		}

		done <- true
	}

	go doClose(patchWriter, uploadDone, uploadErrs)
	go doClose(signatureWriter, uploadDone, uploadErrs)

	for c := 0; c < 4; c++ {
		select {
		case uploadErr := <-uploadErrs:
			return errors.Wrap(uploadErr, 1)
		case <-uploadDone:
			comm.Debugf("upload done")
		}
	}

	close(stopTicking)
	comm.ProgressLabel("finalizing build")

	finalDone := make(chan bool)
	finalErrs := make(chan error)

	doFinalize := func(fileID int64, fileSize int64, done chan bool, errs chan error) {
		_, err = client.FinalizeBuildFile(buildID, fileID, fileSize)
		if err != nil {
			errs <- errors.Wrap(err, 1)
			return
		}

		done <- true
	}

	go doFinalize(newPatchRes.File.ID, patchCounter.Count(), finalDone, finalErrs)
	go doFinalize(newSignatureRes.File.ID, signatureCounter.Count(), finalDone, finalErrs)

	for i := 0; i < 2; i++ {
		select {
		case err := <-finalErrs:
			return errors.Wrap(err, 1)
		case <-finalDone:
		}
	}

	comm.EndProgress()

	{
		prettyPatchSize := humanize.IBytes(uint64(patchCounter.Count()))
		percReused := 100.0 * float64(dctx.ReusedBytes) / float64(dctx.FreshBytes+dctx.ReusedBytes)
		relToNew := 100.0 * float64(patchCounter.Count()) / float64(sourceContainer.Size)
		prettyFreshSize := humanize.IBytes(uint64(dctx.FreshBytes))
		savings := 100.0 - relToNew

		if dctx.ReusedBytes > 0 {
			comm.Statf("Re-used %.2f%% of old, added %s fresh data", percReused, prettyFreshSize)
		} else {
			comm.Statf("Added %s fresh data", prettyFreshSize)
		}

		if savings > 0 && !math.IsNaN(savings) {
			comm.Statf("%s patch (%.2f%% savings)", prettyPatchSize, 100.0-relToNew)
		} else {
			comm.Statf("%s patch (no savings)", prettyPatchSize)
		}
	}
	comm.Opf("Build is now processing, should be up in a bit (see `butler status`)")

	return nil
}

type fileSlot struct {
	Type     itchio.BuildFileType
	Response itchio.NewBuildFileResponse
}

func createBothFiles(client *itchio.Client, buildID int64) (patch itchio.NewBuildFileResponse, signature itchio.NewBuildFileResponse, err error) {
	createFile := func(buildType itchio.BuildFileType, done chan fileSlot, errs chan error) {
		var res itchio.NewBuildFileResponse
		res, err = client.CreateBuildFile(buildID, buildType, itchio.BuildFileSubType_DEFAULT, itchio.UploadType_DEFERRED_RESUMABLE)
		if err != nil {
			errs <- errors.Wrap(err, 1)
		}
		comm.Debugf("Created %s build file: %+v", buildType, res.File)

		// TODO: resumable upload session creation sounds like it belongs in an external lib, go-itchio maybe?
		req, reqErr := http.NewRequest("POST", res.File.UploadURL, nil)
		if reqErr != nil {
			errs <- errors.Wrap(reqErr, 1)
		}

		req.ContentLength = 0

		for k, v := range res.File.UploadHeaders {
			req.Header.Add(k, v)
		}

		gcsRes, gcsErr := client.HTTPClient.Do(req)
		if gcsErr != nil {
			errs <- errors.Wrap(gcsErr, 1)
		}

		if gcsRes.StatusCode != 201 {
			errs <- errors.Wrap(fmt.Errorf("could not create resumable upload session (got HTTP %d)", gcsRes.StatusCode), 1)
		}

		comm.Debugf("Started resumable upload session %s", gcsRes.Header.Get("Location"))

		res.File.UploadHeaders = nil
		res.File.UploadURL = gcsRes.Header.Get("Location")

		done <- fileSlot{buildType, res}
	}

	done := make(chan fileSlot)
	errs := make(chan error)

	go createFile(itchio.BuildFileType_PATCH, done, errs)
	go createFile(itchio.BuildFileType_SIGNATURE, done, errs)

	for i := 0; i < 2; i++ {
		select {
		case err = <-errs:
			err = errors.Wrap(err, 1)
			return
		case slot := <-done:
			switch slot.Type {
			case itchio.BuildFileType_PATCH:
				patch = slot.Response
			case itchio.BuildFileType_SIGNATURE:
				signature = slot.Response
			}
		}
	}

	return
}

type walkResult struct {
	container *tlc.Container
	pool      wsync.Pool
}

func doWalk(path string, out chan walkResult, errs chan error, fixPerms bool) {
	container, err := tlc.WalkAny(path, filterPaths)
	if err != nil {
		errs <- errors.Wrap(err, 1)
		return
	}

	pool, err := pools.New(container, path)
	if err != nil {
		errs <- errors.Wrap(err, 1)
		return
	}

	result := walkResult{
		container: container,
		pool:      pool,
	}

	if fixPerms {
		result.container.FixPermissions(result.pool)
	}
	out <- result
}

func min(a, b float64) float64 {
	if a < b {
		return a
	}
	return b
}
<commit_msg>Indulge shadi, closes #88<commit_after>package main

import (
	"fmt"
	"io"
	"math"
	"net/http"
	"time"

	"github.com/dustin/go-humanize"
	"github.com/go-errors/errors"
	"github.com/itchio/butler/comm"
	"github.com/itchio/go-itchio"
	"github.com/itchio/httpkit/uploader"
	"github.com/itchio/wharf/counter"
	"github.com/itchio/wharf/pools"
	"github.com/itchio/wharf/pwr"
	"github.com/itchio/wharf/state"
	"github.com/itchio/wharf/tlc"
	"github.com/itchio/wharf/wsync"
)

// AlmostThereThreshold is the amount of data left where the progress indicator isn't indicative anymore.
// At this point, we're basically waiting for build files to be finalized.
const AlmostThereThreshold int64 = 10 * 1024

func push(buildPath string, specStr string, userVersion string, fixPerms bool) {
	go versionCheck()
	must(doPush(buildPath, specStr, userVersion, fixPerms))
}

func doPush(buildPath string, specStr string, userVersion string, fixPerms bool) error {
	// start walking source container while waiting on auth flow
	sourceContainerChan := make(chan walkResult)
	walkErrs := make(chan error)
	go doWalk(buildPath, sourceContainerChan, walkErrs, fixPerms)

	spec, err := itchio.ParseSpec(specStr)
	if err != nil {
		return errors.Wrap(err, 1)
	}

	err = spec.EnsureChannel()
	if err != nil {
		return errors.Wrap(err, 1)
	}

	client, err := authenticateViaOauth()
	if err != nil {
		return errors.Wrap(err, 1)
	}

	newBuildRes, err := client.CreateBuild(spec.Target, spec.Channel, userVersion)
	if err != nil {
		return errors.Wrap(err, 1)
	}

	buildID := newBuildRes.Build.ID
	parentID := newBuildRes.Build.ParentBuild.ID

	var targetSignature *pwr.SignatureInfo

	if parentID == 0 {
		comm.Opf("For channel `%s`: pushing first build", spec.Channel)
		targetSignature = &pwr.SignatureInfo{
			Container: &tlc.Container{},
			Hashes:    make([]wsync.BlockHash, 0),
		}
	} else {
		comm.Opf("For channel `%s`: last build is %d, downloading its signature", spec.Channel, parentID)
		var buildFiles itchio.ListBuildFilesResponse
		buildFiles, err = client.ListBuildFiles(parentID)
		if err != nil {
			return errors.Wrap(err, 1)
		}

		signatureFile := itchio.FindBuildFile(itchio.BuildFileType_SIGNATURE, buildFiles.Files)
		if signatureFile == nil {
			comm.Dief("Could not find signature for parent build %d, aborting", parentID)
		}

		var signatureReader io.Reader
		signatureReader, err = client.DownloadBuildFile(parentID, signatureFile.ID)
		if err != nil {
			return errors.Wrap(err, 1)
		}

		targetSignature, err = pwr.ReadSignature(signatureReader)
		if err != nil {
			return errors.Wrap(err, 1)
		}
	}

	newPatchRes, newSignatureRes, err := createBothFiles(client, buildID)
	if err != nil {
		return errors.Wrap(err, 1)
	}

	uploadDone := make(chan bool)
	uploadErrs := make(chan error)

	patchWriter, err := uploader.NewResumableUpload(newPatchRes.File.UploadURL,
		uploadDone, uploadErrs, comm.NewStateConsumer())
	patchWriter.MaxChunkGroup = *appArgs.maxChunkGroup
	if err != nil {
		return errors.Wrap(err, 1)
	}

	signatureWriter, err := uploader.NewResumableUpload(newSignatureRes.File.UploadURL,
		uploadDone, uploadErrs, comm.NewStateConsumer())
	signatureWriter.MaxChunkGroup = *appArgs.maxChunkGroup
	if err != nil {
		return errors.Wrap(err, 1)
	}

	comm.Debugf("Launching patch & signature channels")

	patchCounter := counter.NewWriter(patchWriter)
	signatureCounter := counter.NewWriter(signatureWriter)

	// we started walking the source container in the beginning,
	// we actually need it now.
	// note that we could actually start diffing before all the file
	// creation & upload setup is done

	var sourceContainer *tlc.Container
	var sourcePool wsync.Pool

	comm.Debugf("Waiting for source container")
	select {
	case walkErr := <-walkErrs:
		return errors.Wrap(walkErr, 1)
	case walkies := <-sourceContainerChan:
		comm.Debugf("Got sourceContainer!")
		sourceContainer = walkies.container
		sourcePool = walkies.pool
		break
	}

	comm.Opf("Pushing %s (%s)", humanize.IBytes(uint64(sourceContainer.Size)), sourceContainer.Stats())

	comm.Debugf("Building diff context")
	var readBytes int64

	bytesPerSec := float64(0)
	lastUploadedBytes := int64(0)
	stopTicking := make(chan struct{})

	updateProgress := func() {
		uploadedBytes := int64(float64(patchWriter.UploadedBytes))

		// input bytes that aren't in output, for example:
		//  - bytes that have been compressed away
		//  - bytes that were in old build and were simply reused
		goneBytes := readBytes - patchWriter.TotalBytes

		conservativeTotalBytes := sourceContainer.Size - goneBytes

		leftBytes := conservativeTotalBytes - uploadedBytes
		if leftBytes > AlmostThereThreshold {
			netStatus := "- network idle"
			if bytesPerSec > 1 {
				netStatus = fmt.Sprintf("@ %s/s", humanize.IBytes(uint64(bytesPerSec)))
			}
			comm.ProgressLabel(fmt.Sprintf("%s, %s left", netStatus, humanize.IBytes(uint64(leftBytes))))
		} else {
			comm.ProgressLabel(fmt.Sprintf("- almost there"))
		}

		conservativeProgress := float64(uploadedBytes) / float64(conservativeTotalBytes)
		conservativeProgress = min(1.0, conservativeProgress)
		comm.Progress(conservativeProgress)

		comm.ProgressScale(float64(readBytes) / float64(sourceContainer.Size))
	}

	go func() {
		ticker := time.NewTicker(time.Second * time.Duration(2))
		for {
			select {
			case <-ticker.C:
				bytesPerSec = float64(patchWriter.UploadedBytes-lastUploadedBytes) / 2.0
				lastUploadedBytes = patchWriter.UploadedBytes
				updateProgress()
			case <-stopTicking:
				break
			}
		}
	}()

	patchWriter.OnProgress = updateProgress

	stateConsumer := &state.Consumer{
		OnProgress: func(progress float64) {
			readBytes = int64(float64(sourceContainer.Size) * progress)
			updateProgress()
		},
	}

	dctx := &pwr.DiffContext{
		Compression: &pwr.CompressionSettings{
			Algorithm: pwr.CompressionAlgorithm_BROTLI,
			Quality:   1,
		},

		SourceContainer: sourceContainer,
		Pool:            sourcePool,

		TargetContainer: targetSignature.Container,
		TargetSignature: targetSignature.Hashes,

		Consumer: stateConsumer,
	}

	comm.StartProgress()
	comm.ProgressScale(0.0)
	err = dctx.WritePatch(patchCounter, signatureCounter)
	if err != nil {
		return errors.Wrap(err, 1)
	}

	// close in a goroutine to avoid deadlocking
	doClose := func(c io.Closer, done chan bool, errs chan error) {
		closeErr := c.Close()
		if closeErr != nil {
			errs <- errors.Wrap(closeErr, 1)
			return
		}

		done <- true
	}

	go doClose(patchWriter, uploadDone, uploadErrs)
	go doClose(signatureWriter, uploadDone, uploadErrs)

	for c := 0; c < 4; c++ {
		select {
		case uploadErr := <-uploadErrs:
			return errors.Wrap(uploadErr, 1)
		case <-uploadDone:
			comm.Debugf("upload done")
		}
	}

	close(stopTicking)
	comm.ProgressLabel("finalizing build")

	finalDone := make(chan bool)
	finalErrs := make(chan error)

	doFinalize := func(fileID int64, fileSize int64, done chan bool, errs chan error) {
		_, err = client.FinalizeBuildFile(buildID, fileID, fileSize)
		if err != nil {
			errs <- errors.Wrap(err, 1)
			return
		}

		done <- true
	}

	go doFinalize(newPatchRes.File.ID, patchCounter.Count(), finalDone, finalErrs)
	go doFinalize(newSignatureRes.File.ID, signatureCounter.Count(), finalDone, finalErrs)

	for i := 0; i < 2; i++ {
		select {
		case err := <-finalErrs:
			return errors.Wrap(err, 1)
		case <-finalDone:
		}
	}

	comm.EndProgress()

	{
		prettyPatchSize := humanize.IBytes(uint64(patchCounter.Count()))
		percReused := 100.0 * float64(dctx.ReusedBytes) / float64(dctx.FreshBytes+dctx.ReusedBytes)
		relToNew := 100.0 * float64(patchCounter.Count()) / float64(sourceContainer.Size)
		prettyFreshSize := humanize.IBytes(uint64(dctx.FreshBytes))
		savings := 100.0 - relToNew

		if dctx.ReusedBytes > 0 {
			comm.Statf("Re-used %.2f%% of old, added %s fresh data", percReused, prettyFreshSize)
		} else {
			comm.Statf("Added %s fresh data", prettyFreshSize)
		}

		if savings > 0 && !math.IsNaN(savings) {
			comm.Statf("%s patch (%.2f%% savings)", prettyPatchSize, 100.0-relToNew)
		} else {
			comm.Statf("%s patch (no savings)", prettyPatchSize)
		}
	}
	comm.Opf("Build is now processing, should be up in a bit (see `butler status`)")
	comm.Logf("")

	return nil
}

type fileSlot struct {
	Type     itchio.BuildFileType
	Response itchio.NewBuildFileResponse
}

func createBothFiles(client *itchio.Client, buildID int64) (patch itchio.NewBuildFileResponse, signature itchio.NewBuildFileResponse, err error) {
	createFile := func(buildType itchio.BuildFileType, done chan fileSlot, errs chan error) {
		var res itchio.NewBuildFileResponse
		res, err = client.CreateBuildFile(buildID, buildType, itchio.BuildFileSubType_DEFAULT, itchio.UploadType_DEFERRED_RESUMABLE)
		if err != nil {
			errs <- errors.Wrap(err, 1)
		}
		comm.Debugf("Created %s build file: %+v", buildType, res.File)

		// TODO: resumable upload session creation sounds like it belongs in an external lib, go-itchio maybe?
		req, reqErr := http.NewRequest("POST", res.File.UploadURL, nil)
		if reqErr != nil {
			errs <- errors.Wrap(reqErr, 1)
		}

		req.ContentLength = 0

		for k, v := range res.File.UploadHeaders {
			req.Header.Add(k, v)
		}

		gcsRes, gcsErr := client.HTTPClient.Do(req)
		if gcsErr != nil {
			errs <- errors.Wrap(gcsErr, 1)
		}

		if gcsRes.StatusCode != 201 {
			errs <- errors.Wrap(fmt.Errorf("could not create resumable upload session (got HTTP %d)", gcsRes.StatusCode), 1)
		}

		comm.Debugf("Started resumable upload session %s", gcsRes.Header.Get("Location"))

		res.File.UploadHeaders = nil
		res.File.UploadURL = gcsRes.Header.Get("Location")

		done <- fileSlot{buildType, res}
	}

	done := make(chan fileSlot)
	errs := make(chan error)

	go createFile(itchio.BuildFileType_PATCH, done, errs)
	go createFile(itchio.BuildFileType_SIGNATURE, done, errs)

	for i := 0; i < 2; i++ {
		select {
		case err = <-errs:
			err = errors.Wrap(err, 1)
			return
		case slot := <-done:
			switch slot.Type {
			case itchio.BuildFileType_PATCH:
				patch = slot.Response
			case itchio.BuildFileType_SIGNATURE:
				signature = slot.Response
			}
		}
	}

	return
}

type walkResult struct {
	container *tlc.Container
	pool      wsync.Pool
}

func doWalk(path string, out chan walkResult, errs chan error, fixPerms bool) {
	container, err := tlc.WalkAny(path, filterPaths)
	if err != nil {
		errs <- errors.Wrap(err, 1)
		return
	}

	pool, err := pools.New(container, path)
	if err != nil {
		errs <- errors.Wrap(err, 1)
		return
	}

	result := walkResult{
		container: container,
		pool:      pool,
	}

	if fixPerms {
		result.container.FixPermissions(result.pool)
	}
	out <- result
}

func min(a, b float64) float64 {
	if a < b {
		return a
	}
	return b
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"crypto/tls"
	"io"
	"io/ioutil"
	"math/rand"
	"net"
	"net/http"
	"strings"
	"time"

	quic "github.com/lucas-clemente/quic-go"
	"github.com/lucas-clemente/quic-go/h2quic"
)

var errNoSuchBucket = []byte("<?xml version='1.0' encoding='UTF-8'?><Error>NoSuchBucket<Message>The specified bucket does not exist.</Message></Error>")

func testQuic(ip string, config *ScanConfig, record *ScanRecord) bool {
	start := time.Now()

	udpConn, err := net.ListenUDP("udp", &net.UDPAddr{IP: net.IPv4zero, Port: 0})
	if err != nil {
		return false
	}
	udpConn.SetDeadline(time.Now().Add(config.ScanMaxRTT))
	defer udpConn.Close()

	quicCfg := &quic.Config{
		HandshakeTimeout: config.HandshakeTimeout,
		KeepAlive:        false,
	}

	var serverName string
	if len(config.ServerName) == 0 {
		serverName = randomHost()
	} else {
		serverName = config.ServerName[rand.Intn(len(config.ServerName))]
	}

	tlsCfg := &tls.Config{
		InsecureSkipVerify: true,
		ServerName:         serverName,
	}

	udpAddr := &net.UDPAddr{IP: net.ParseIP(ip), Port: 443}
	addr := net.JoinHostPort(serverName, "443")
	quicSessn, err := quic.Dial(udpConn, udpAddr, addr, tlsCfg, quicCfg)
	if err != nil {
		return false
	}
	defer quicSessn.Close(nil)

	// lv1 只会验证证书是否存在
	cs := quicSessn.ConnectionState()
	if !cs.HandshakeComplete {
		return false
	}
	pcs := cs.PeerCertificates
	if len(pcs) < 2 {
		return false
	}

	// lv2 验证证书是否正确
	if config.Level > 1 {
		pkp := pcs[1].RawSubjectPublicKeyInfo
		if !bytes.Equal(g2pkp, pkp) && !bytes.Equal(g3pkp, pkp) { // && !bytes.Equal(g3ecc, pkp[:]) {
			return false
		}
	}

	// lv3 使用 http 访问来验证
	if config.Level > 2 {
		tr := &h2quic.RoundTripper{DisableCompression: true}
		defer tr.Close()

		tr.Dial = func(network, addr string, tlsCfg *tls.Config, cfg *quic.Config) (quic.Session, error) {
			return quicSessn, err
		}
		// 设置超时
		hclient := &http.Client{
			Transport: tr,
			CheckRedirect: func(req *http.Request, via []*http.Request) error {
				return http.ErrUseLastResponse
			},
			Timeout: config.ScanMaxRTT - time.Since(start),
		}
		url := "https://" + config.HTTPVerifyHosts[rand.Intn(len(config.HTTPVerifyHosts))]
		req, _ := http.NewRequest(http.MethodGet, url, nil)
		req.Close = true
		resp, _ := hclient.Do(req)
		if resp == nil || (resp.StatusCode < 200 || resp.StatusCode >= 400) || !strings.Contains(resp.Header.Get("Alt-Svc"), `quic=":443"`) {
			return false
		}
		if resp.Body != nil {
			defer resp.Body.Close()
			// lv4 验证是否是 NoSuchBucket 错误
			if config.Level > 3 && resp.Header.Get("Content-Type") == "application/xml; charset=UTF-8" { // 也许条件改为 || 更好
				body, err := ioutil.ReadAll(resp.Body)
				if err != nil || bytes.Equal(body, errNoSuchBucket) {
					return false
				}
			} else {
				io.Copy(ioutil.Discard, resp.Body)
			}
		}
	}

	if rtt := time.Since(start); rtt > config.ScanMinRTT {
		record.RTT += rtt
		return true
	}
	return false
}
<commit_msg>dependency moved (#213)<commit_after>package main

import (
	"bytes"
	"crypto/tls"
	"io"
	"io/ioutil"
	"math/rand"
	"net"
	"net/http"
	"strings"
	"time"

	quic "github.com/ipsn/go-ipfs/gxlibs/github.com/lucas-clemente/quic-go"
	"github.com/ipsn/go-ipfs/gxlibs/github.com/lucas-clemente/quic-go/h2quic"
)

var errNoSuchBucket = []byte("<?xml version='1.0' encoding='UTF-8'?><Error>NoSuchBucket<Message>The specified bucket does not exist.</Message></Error>")

func testQuic(ip string, config *ScanConfig, record *ScanRecord) bool {
	start := time.Now()

	udpConn, err := net.ListenUDP("udp", &net.UDPAddr{IP: net.IPv4zero, Port: 0})
	if err != nil {
		return false
	}
	udpConn.SetDeadline(time.Now().Add(config.ScanMaxRTT))
	defer udpConn.Close()

	quicCfg := &quic.Config{
		HandshakeTimeout: config.HandshakeTimeout,
		KeepAlive:        false,
	}

	var serverName string
	if len(config.ServerName) == 0 {
		serverName = randomHost()
	} else {
		serverName = config.ServerName[rand.Intn(len(config.ServerName))]
	}

	tlsCfg := &tls.Config{
		InsecureSkipVerify: true,
		ServerName:         serverName,
	}

	udpAddr := &net.UDPAddr{IP: net.ParseIP(ip), Port: 443}
	addr := net.JoinHostPort(serverName, "443")
	quicSessn, err := quic.Dial(udpConn, udpAddr, addr, tlsCfg, quicCfg)
	if err != nil {
		return false
	}
	defer quicSessn.Close()

	// lv1 只会验证证书是否存在
	cs := quicSessn.ConnectionState()
	if !cs.HandshakeComplete {
		return false
	}
	pcs := cs.PeerCertificates
	if len(pcs) < 2 {
		return false
	}

	// lv2 验证证书是否正确
	if config.Level > 1 {
		pkp := pcs[1].RawSubjectPublicKeyInfo
		if !bytes.Equal(g2pkp, pkp) && !bytes.Equal(g3pkp, pkp) { // && !bytes.Equal(g3ecc, pkp[:]) {
			return false
		}
	}

	// lv3 使用 http 访问来验证
	if config.Level > 2 {
		tr := &h2quic.RoundTripper{DisableCompression: true}
		defer tr.Close()

		tr.Dial = func(network, addr string, tlsCfg *tls.Config, cfg *quic.Config) (quic.Session, error) {
			return quicSessn, err
		}
		// 设置超时
		hclient := &http.Client{
			Transport: tr,
			CheckRedirect: func(req *http.Request, via []*http.Request) error {
				return http.ErrUseLastResponse
			},
			Timeout: config.ScanMaxRTT - time.Since(start),
		}
		url := "https://" + config.HTTPVerifyHosts[rand.Intn(len(config.HTTPVerifyHosts))]
		req, _ := http.NewRequest(http.MethodGet, url, nil)
		req.Close = true
		resp, _ := hclient.Do(req)
		if resp == nil || (resp.StatusCode < 200 || resp.StatusCode >= 400) || !strings.Contains(resp.Header.Get("Alt-Svc"), `quic=":443"`) {
			return false
		}
		if resp.Body != nil {
			defer resp.Body.Close()
			// lv4 验证是否是 NoSuchBucket 错误
			if config.Level > 3 && resp.Header.Get("Content-Type") == "application/xml; charset=UTF-8" { // 也许条件改为 || 更好
				body, err := ioutil.ReadAll(resp.Body)
				if err != nil || bytes.Equal(body, errNoSuchBucket) {
					return false
				}
			} else {
				io.Copy(ioutil.Discard, resp.Body)
			}
		}
	}

	if rtt := time.Since(start); rtt > config.ScanMinRTT {
		record.RTT += rtt
		return true
	}
	return false
}
<|endoftext|>


<commit_before>package rate // import "gopkg.in/go-redis/rate.v4"

import (
	"fmt"
	"time"

	redis "gopkg.in/redis.v4"

	timerate "golang.org/x/time/rate"
)

const redisPrefix = "rate"

type rediser interface {
	Pipelined(func(pipe *redis.Pipeline) error) ([]redis.Cmder, error)
}

// Limiter controls how frequently events are allowed to happen.
// It uses redis to store data and fallbacks to the fallbackLimiter
// when Redis Server is not available.
type Limiter struct {
	fallbackLimiter *timerate.Limiter
	redis           rediser
}

func NewLimiter(redis rediser, fallbackLimiter *timerate.Limiter) *Limiter {
	return &Limiter{
		fallbackLimiter: fallbackLimiter,
		redis:           redis,
	}
}

// AllowN reports whether an event with given name may happen at time now.
// It allows up to maxn events within duration dur, with each interaction
// incrementing the limit by n.
func (l *Limiter) AllowN(name string, maxn int64, dur time.Duration, n int64) (count, reset int64, allow bool) {
	udur := int64(dur / time.Second)
	slot := time.Now().Unix() / udur
	reset = (slot + 1) * udur
	allow = l.fallbackLimiter.Allow()

	name = fmt.Sprintf("%s:%s-%d", redisPrefix, name, slot)
	count, err := l.incr(name, dur, n)
	if err == nil {
		allow = count <= maxn
	}

	return count, reset, allow
}

// Allow is shorthand for AllowN(name, max, dur, 1).
func (l *Limiter) Allow(name string, maxn int64, dur time.Duration) (count, reset int64, allow bool) {
	return l.AllowN(name, maxn, dur, 1)
}

// AllowMinute is shorthand for Allow(name, maxn, time.Minute).
func (l *Limiter) AllowMinute(name string, maxn int64) (int64, int64, bool) {
	return l.Allow(name, maxn, time.Minute)
}

// AllowHour is shorthand for Allow(name, maxn, time.Hour).
func (l *Limiter) AllowHour(name string, maxn int64) (int64, int64, bool) {
	return l.Allow(name, maxn, time.Hour)
}

// AllowRate reports whether an event may happen at time now.
// It allows up to rateLimit events each second.
func (l *Limiter) AllowRate(name string, rateLimit timerate.Limit) (delay time.Duration, allow bool) {
	if rateLimit == 0 {
		return 0, false
	}
	if rateLimit == timerate.Inf {
		return 0, true
	}

	dur := time.Second
	limit := int64(rateLimit)
	if limit == 0 {
		limit = 1
		dur *= time.Duration(1 / rateLimit)
	}

	now := time.Now()
	slot := now.UnixNano() / dur.Nanoseconds()
	allow = l.fallbackLimiter.Allow()

	name = fmt.Sprintf("%s:%s-%d-%d", redisPrefix, name, dur, slot)
	count, err := l.incr(name, dur, 1)
	if err == nil {
		allow = count <= limit
	}

	if !allow {
		delay = time.Duration(slot+1)*dur - time.Duration(now.UnixNano())
	}

	return delay, allow
}

func (l *Limiter) incr(name string, dur time.Duration, n int64) (int64, error) {
	var incr *redis.IntCmd
	_, err := l.redis.Pipelined(func(pipe *redis.Pipeline) error {
		incr = pipe.IncrBy(name, n)
		pipe.Expire(name, dur)
		return nil
	})

	rate, _ := incr.Result()
	return rate, err
}
<commit_msg>teardown ability<commit_after>package rate // import "gopkg.in/go-redis/rate.v4"

import (
	"fmt"
	"time"
	redis "gopkg.in/redis.v4"
	timerate "golang.org/x/time/rate"
)

const redisPrefix = "rate"

type rediser interface {
	Pipelined(func(pipe *redis.Pipeline) error) ([]redis.Cmder, error)
}

// Limiter controls how frequently events are allowed to happen.
// It uses redis to store data and fallbacks to the fallbackLimiter
// when Redis Server is not available.
type Limiter struct {
	fallbackLimiter *timerate.Limiter
	redis           rediser
}

func NewLimiter(redis rediser, fallbackLimiter *timerate.Limiter) *Limiter {
	return &Limiter{
		fallbackLimiter: fallbackLimiter,
		redis:           redis,
	}
}

//Delete deletes everything attached to a key.
//Will delete all rate limiting attached to any given key.
func (l *Limiter) Teardown(name string) (err error) {
	keys, err := l.keys(name)
	if err != nil {
		return
	}

	for _, k := range keys {
		err = l.delete(k)
	}

	return
}

// AllowN reports whether an event with given name may happen at time now.
// It allows up to maxn events within duration dur, with each interaction
// incrementing the limit by n.
func (l *Limiter) AllowN(name string, maxn int64, dur time.Duration, n int64) (count, reset int64, allow bool) {
	udur := int64(dur / time.Second)
	slot := time.Now().Unix() / udur
	reset = (slot + 1) * udur
	allow = l.fallbackLimiter.Allow()

	name = fmt.Sprintf("%s:%s-%d", redisPrefix, name, slot)
	count, err := l.incr(name, dur, n)
	if err == nil {
		allow = count <= maxn
	}

	return count, reset, allow
}

// Allow is shorthand for AllowN(name, max, dur, 1).
func (l *Limiter) Allow(name string, maxn int64, dur time.Duration) (count, reset int64, allow bool) {
	return l.AllowN(name, maxn, dur, 1)
}

// AllowMinute is shorthand for Allow(name, maxn, time.Minute).
func (l *Limiter) AllowMinute(name string, maxn int64) (int64, int64, bool) {
	return l.Allow(name, maxn, time.Minute)
}

// AllowHour is shorthand for Allow(name, maxn, time.Hour).
func (l *Limiter) AllowHour(name string, maxn int64) (int64, int64, bool) {
	return l.Allow(name, maxn, time.Hour)
}

// AllowRate reports whether an event may happen at time now.
// It allows up to rateLimit events each second.
func (l *Limiter) AllowRate(name string, rateLimit timerate.Limit) (delay time.Duration, allow bool) {
	if rateLimit == 0 {
		return 0, false
	}
	if rateLimit == timerate.Inf {
		return 0, true
	}

	dur := time.Second
	limit := int64(rateLimit)
	if limit == 0 {
		limit = 1
		dur *= time.Duration(1 / rateLimit)
	}

	now := time.Now()
	slot := now.UnixNano() / dur.Nanoseconds()
	allow = l.fallbackLimiter.Allow()

	name = fmt.Sprintf("%s:%s-%d-%d", redisPrefix, name, dur, slot)
	count, err := l.incr(name, dur, 1)
	if err == nil {
		allow = count <= limit
	}

	if !allow {
		delay = time.Duration(slot+1)*dur - time.Duration(now.UnixNano())
	}

	return delay, allow
}

func (l *Limiter) delete(names ...string) (err error) {
	var del *redis.IntCmd
	_, err = l.redis.Pipelined(func(pipe *redis.Pipeline) error {
		del = pipe.Del(names...)
		return nil
	})

	if err != nil {
		return
	}

	err = del.Err()

	return
}

func (l *Limiter) keys(name string) ([]string, error) {
	var keys *redis.StringSliceCmd
	_, err := l.redis.Pipelined(func(pipe *redis.Pipeline) error {
		//hard coded since we know the format
		keys = pipe.Keys("rate:" + name + "*")
		return nil
	})

	if err != nil {
		return []string{}, err
	}

	return keys.Result()
}

func (l *Limiter) incr(name string, dur time.Duration, n int64) (int64, error) {
	var incr *redis.IntCmd
	_, err := l.redis.Pipelined(func(pipe *redis.Pipeline) error {
		incr = pipe.IncrBy(name, n)
		pipe.Expire(name, dur)
		return nil
	})

	rate, _ := incr.Result()
	return rate, err
}
<|endoftext|>


<commit_before>/*

config is a simple library that manages config set-up for boardgame-util and
friends, reading from config.json and config.SECRET.json files. See boardgame-
util/README.md for more on the structure of config.json files.

*/
package config

import (
	"encoding/json"
	"errors"
	"io/ioutil"
	"log"
	"net/url"
	"os"
	"strings"
)

type Config struct {
	Base *ConfigMode
	Dev  *ConfigMode
	Prod *ConfigMode
}

type ConfigMode struct {
	AllowedOrigins    string
	DefaultPort       string
	FirebaseProjectId string
	AdminUserIds      []string
	//This is a dangerous config. Only enable in Dev!
	DisableAdminChecking bool
	StorageConfig        map[string]string
	Games                *GameNode
	//GamesList is not intended to be inflated from JSON, but rather is
	//derived based on the contents of Games.
	GamesList []string
}

//derive takes a raw input and creates a struct with fully derived values in
//Dev/Prod ready for use.
func (c *Config) derive() {
	if c.Base == nil {
		return
	}

	c.Prod = c.Base.extend(c.Prod)
	c.Dev = c.Base.extend(c.Dev)

	c.Prod.derive()
	c.Dev.derive()

}

func (c *Config) copy() *Config {
	return &Config{
		c.Base.copy(),
		c.Dev.copy(),
		c.Prod.copy(),
	}
}

//extend takes an other config and returns a *new* config where any non-zero
//value for other extends base.
func (c *Config) extend(other *Config) *Config {

	result := c.copy()

	if other == nil {
		return result
	}

	result.Base = c.Base.extend(other.Base)
	result.Dev = c.Dev.extend(other.Dev)
	result.Prod = c.Prod.extend(other.Prod)

	return result

}

func (c *Config) validate() error {

	if c.Dev == nil && c.Prod == nil {
		return errors.New("Neither dev nor prod configuration was valid")
	}

	if c.Dev != nil {
		if err := c.Dev.validate(true); err != nil {
			return err
		}
	}
	if c.Prod != nil {
		return c.Prod.validate(false)
	}
	return nil
}

//derive tells the ConfigMode to do its final processing to create any derived
//fields, like GamesList.
func (c *ConfigMode) derive() {

	if c == nil {
		return
	}

	c.GamesList = c.Games.List()

}

func (c *ConfigMode) String() string {
	blob, err := json.MarshalIndent(c, "", "\t")
	if err != nil {
		return "ERROR, couldn't unmarshal: " + err.Error()
	}
	return string(blob)
}

func (c *ConfigMode) validate(isDev bool) error {
	if c.DefaultPort == "" {
		return errors.New("No default port provided")
	}
	//AllowedOrigins will just be default allow
	if c.AllowedOrigins == "" {
		log.Println("No AllowedOrigins found. Defaulting to '*'")
		c.AllowedOrigins = "*"
	}
	if c.StorageConfig == nil {
		c.StorageConfig = make(map[string]string)
	}
	if c.DisableAdminChecking && !isDev {
		return errors.New("DisableAdminChecking enabled in prod, which is illegal")
	}
	return nil
}

//copy returns a deep copy of the config mode.
func (c *ConfigMode) copy() *ConfigMode {

	if c == nil {
		return nil
	}

	result := &ConfigMode{}

	(*result) = *c
	result.AdminUserIds = make([]string, len(c.AdminUserIds))
	copy(result.AdminUserIds, c.AdminUserIds)
	result.StorageConfig = make(map[string]string, len(c.StorageConfig))
	for key, val := range c.StorageConfig {
		result.StorageConfig[key] = val
	}

	result.Games = result.Games.copy()

	return result

}

//mergedStrList returns a list where base is concatenated with the non-
//duplicates in other.
func mergedStrList(base, other []string) []string {

	if base == nil {
		return other
	}

	if other == nil {
		return base
	}

	result := make([]string, len(base))

	for i, str := range base {
		result[i] = str
	}

	strSet := make(map[string]bool, len(base))
	for _, key := range base {
		strSet[key] = true
	}

	for _, key := range other {
		if strSet[key] {
			//Already in the set, don't add a duplicate
			continue
		}
		result = append(result, key)
	}

	return result
}

//extend takes a given base config mode, extends it with properties set in
//other (with any non-zero value overwriting the base values) and returns a
//*new* config representing the merged one.
func (c *ConfigMode) extend(other *ConfigMode) *ConfigMode {

	result := c.copy()

	if other == nil {
		return result
	}

	if other.AllowedOrigins != "" {
		result.AllowedOrigins = other.AllowedOrigins
	}

	if other.DefaultPort != "" {
		result.DefaultPort = other.DefaultPort
	}

	if other.FirebaseProjectId != "" {
		result.FirebaseProjectId = other.FirebaseProjectId
	}

	if other.DisableAdminChecking {
		result.DisableAdminChecking = true
	}

	result.AdminUserIds = mergedStrList(c.AdminUserIds, other.AdminUserIds)

	for key, val := range other.StorageConfig {
		result.StorageConfig[key] = val
	}

	result.Games = result.Games.extend(other.Games)

	return result

}

func (c *ConfigMode) OriginAllowed(origin string) bool {

	originUrl, err := url.Parse(origin)

	if err != nil {
		return false
	}

	if c.AllowedOrigins == "" {
		return false
	}
	if c.AllowedOrigins == "*" {
		return true
	}
	allowedOrigins := strings.Split(c.AllowedOrigins, ",")
	for _, allowedOrigin := range allowedOrigins {
		u, err := url.Parse(allowedOrigin)

		if err != nil {
			continue
		}

		if u.Scheme == originUrl.Scheme && u.Host == originUrl.Host {
			return true
		}
	}
	return false
}

const (
	privateConfigFileName = "config.SECRET.json"
	publicConfigFileName  = "config.PUBLIC.json"
)

func fileNamesToUse() (publicConfig, privateConfig string) {

	if _, err := os.Stat(privateConfigFileName); err == nil {
		privateConfig = privateConfigFileName
	}

	infos, err := ioutil.ReadDir(".")

	if err != nil {
		return "", ""
	}

	foundNames := make(map[string]bool)

	for _, info := range infos {
		if info.Name() == privateConfigFileName {
			continue
		}
		if strings.HasPrefix(info.Name(), "config.") && strings.HasSuffix(info.Name(), ".json") {
			foundNames[info.Name()] = true
		}
	}

	prioritizedNames := []string{
		publicConfigFileName,
		"config.json",
	}

	for _, name := range prioritizedNames {
		if foundNames[name] {
			return name, privateConfig
		}
	}

	//Whatever, return the first one
	for name, _ := range foundNames {
		return name, privateConfig
	}

	//None of the preferred names were found, just return whatever is in
	//publicConfig, privateConfig.
	return

}

func getConfig(filename string) (*Config, error) {

	if filename == "" {
		return nil, nil
	}

	contents, err := ioutil.ReadFile(filename)

	if err != nil {
		return nil, errors.New("Couldn't read config file: " + err.Error())
	}

	var config Config

	if err := json.Unmarshal(contents, &config); err != nil {
		return nil, errors.New("couldn't unmarshal config file: " + err.Error())
	}

	return &config, nil
}

func combinedConfig() (*Config, error) {
	publicConfigName, privateConfigName := fileNamesToUse()

	publicConfig, err := getConfig(publicConfigName)

	if err != nil {
		return nil, errors.New("Couldn't get public config: " + err.Error())
	}

	privateConfig, err := getConfig(privateConfigName)

	if err != nil {
		return nil, errors.New("Couldn't get private config: " + err.Error())
	}

	return publicConfig.extend(privateConfig), nil

}

func Get() (*Config, error) {

	config, err := combinedConfig()

	if err != nil {
		return nil, err
	}

	config.derive()

	if err := config.validate(); err != nil {
		return nil, errors.New("Couldn't validate config: " + err.Error())
	}

	return config, nil

}
<commit_msg>Fixed long-standing bug in config.Derive for files without a base. Part of #655.<commit_after>/*

config is a simple library that manages config set-up for boardgame-util and
friends, reading from config.json and config.SECRET.json files. See boardgame-
util/README.md for more on the structure of config.json files.

*/
package config

import (
	"encoding/json"
	"errors"
	"io/ioutil"
	"log"
	"net/url"
	"os"
	"strings"
)

type Config struct {
	Base *ConfigMode
	Dev  *ConfigMode
	Prod *ConfigMode
}

type ConfigMode struct {
	AllowedOrigins    string
	DefaultPort       string
	FirebaseProjectId string
	AdminUserIds      []string
	//This is a dangerous config. Only enable in Dev!
	DisableAdminChecking bool
	StorageConfig        map[string]string
	Games                *GameNode
	//GamesList is not intended to be inflated from JSON, but rather is
	//derived based on the contents of Games.
	GamesList []string
}

//derive takes a raw input and creates a struct with fully derived values in
//Dev/Prod ready for use.
func (c *Config) derive() {
	if c.Base != nil {
		c.Prod = c.Base.extend(c.Prod)
		c.Dev = c.Base.extend(c.Dev)
	}

	c.Prod.derive()
	c.Dev.derive()

}

func (c *Config) copy() *Config {
	return &Config{
		c.Base.copy(),
		c.Dev.copy(),
		c.Prod.copy(),
	}
}

//extend takes an other config and returns a *new* config where any non-zero
//value for other extends base.
func (c *Config) extend(other *Config) *Config {

	result := c.copy()

	if other == nil {
		return result
	}

	result.Base = c.Base.extend(other.Base)
	result.Dev = c.Dev.extend(other.Dev)
	result.Prod = c.Prod.extend(other.Prod)

	return result

}

func (c *Config) validate() error {

	if c.Dev == nil && c.Prod == nil {
		return errors.New("Neither dev nor prod configuration was valid")
	}

	if c.Dev != nil {
		if err := c.Dev.validate(true); err != nil {
			return err
		}
	}
	if c.Prod != nil {
		return c.Prod.validate(false)
	}
	return nil
}

//derive tells the ConfigMode to do its final processing to create any derived
//fields, like GamesList.
func (c *ConfigMode) derive() {

	if c == nil {
		return
	}

	c.GamesList = c.Games.List()

}

func (c *ConfigMode) String() string {
	blob, err := json.MarshalIndent(c, "", "\t")
	if err != nil {
		return "ERROR, couldn't unmarshal: " + err.Error()
	}
	return string(blob)
}

func (c *ConfigMode) validate(isDev bool) error {
	if c.DefaultPort == "" {
		return errors.New("No default port provided")
	}
	//AllowedOrigins will just be default allow
	if c.AllowedOrigins == "" {
		log.Println("No AllowedOrigins found. Defaulting to '*'")
		c.AllowedOrigins = "*"
	}
	if c.StorageConfig == nil {
		c.StorageConfig = make(map[string]string)
	}
	if c.DisableAdminChecking && !isDev {
		return errors.New("DisableAdminChecking enabled in prod, which is illegal")
	}
	return nil
}

//copy returns a deep copy of the config mode.
func (c *ConfigMode) copy() *ConfigMode {

	if c == nil {
		return nil
	}

	result := &ConfigMode{}

	(*result) = *c
	result.AdminUserIds = make([]string, len(c.AdminUserIds))
	copy(result.AdminUserIds, c.AdminUserIds)
	result.StorageConfig = make(map[string]string, len(c.StorageConfig))
	for key, val := range c.StorageConfig {
		result.StorageConfig[key] = val
	}

	result.Games = result.Games.copy()

	return result

}

//mergedStrList returns a list where base is concatenated with the non-
//duplicates in other.
func mergedStrList(base, other []string) []string {

	if base == nil {
		return other
	}

	if other == nil {
		return base
	}

	result := make([]string, len(base))

	for i, str := range base {
		result[i] = str
	}

	strSet := make(map[string]bool, len(base))
	for _, key := range base {
		strSet[key] = true
	}

	for _, key := range other {
		if strSet[key] {
			//Already in the set, don't add a duplicate
			continue
		}
		result = append(result, key)
	}

	return result
}

//extend takes a given base config mode, extends it with properties set in
//other (with any non-zero value overwriting the base values) and returns a
//*new* config representing the merged one.
func (c *ConfigMode) extend(other *ConfigMode) *ConfigMode {

	result := c.copy()

	if other == nil {
		return result
	}

	if other.AllowedOrigins != "" {
		result.AllowedOrigins = other.AllowedOrigins
	}

	if other.DefaultPort != "" {
		result.DefaultPort = other.DefaultPort
	}

	if other.FirebaseProjectId != "" {
		result.FirebaseProjectId = other.FirebaseProjectId
	}

	if other.DisableAdminChecking {
		result.DisableAdminChecking = true
	}

	result.AdminUserIds = mergedStrList(c.AdminUserIds, other.AdminUserIds)

	for key, val := range other.StorageConfig {
		result.StorageConfig[key] = val
	}

	result.Games = result.Games.extend(other.Games)

	return result

}

func (c *ConfigMode) OriginAllowed(origin string) bool {

	originUrl, err := url.Parse(origin)

	if err != nil {
		return false
	}

	if c.AllowedOrigins == "" {
		return false
	}
	if c.AllowedOrigins == "*" {
		return true
	}
	allowedOrigins := strings.Split(c.AllowedOrigins, ",")
	for _, allowedOrigin := range allowedOrigins {
		u, err := url.Parse(allowedOrigin)

		if err != nil {
			continue
		}

		if u.Scheme == originUrl.Scheme && u.Host == originUrl.Host {
			return true
		}
	}
	return false
}

const (
	privateConfigFileName = "config.SECRET.json"
	publicConfigFileName  = "config.PUBLIC.json"
)

func fileNamesToUse() (publicConfig, privateConfig string) {

	if _, err := os.Stat(privateConfigFileName); err == nil {
		privateConfig = privateConfigFileName
	}

	infos, err := ioutil.ReadDir(".")

	if err != nil {
		return "", ""
	}

	foundNames := make(map[string]bool)

	for _, info := range infos {
		if info.Name() == privateConfigFileName {
			continue
		}
		if strings.HasPrefix(info.Name(), "config.") && strings.HasSuffix(info.Name(), ".json") {
			foundNames[info.Name()] = true
		}
	}

	prioritizedNames := []string{
		publicConfigFileName,
		"config.json",
	}

	for _, name := range prioritizedNames {
		if foundNames[name] {
			return name, privateConfig
		}
	}

	//Whatever, return the first one
	for name, _ := range foundNames {
		return name, privateConfig
	}

	//None of the preferred names were found, just return whatever is in
	//publicConfig, privateConfig.
	return

}

func getConfig(filename string) (*Config, error) {

	if filename == "" {
		return nil, nil
	}

	contents, err := ioutil.ReadFile(filename)

	if err != nil {
		return nil, errors.New("Couldn't read config file: " + err.Error())
	}

	var config Config

	if err := json.Unmarshal(contents, &config); err != nil {
		return nil, errors.New("couldn't unmarshal config file: " + err.Error())
	}

	return &config, nil
}

func combinedConfig() (*Config, error) {
	publicConfigName, privateConfigName := fileNamesToUse()

	publicConfig, err := getConfig(publicConfigName)

	if err != nil {
		return nil, errors.New("Couldn't get public config: " + err.Error())
	}

	privateConfig, err := getConfig(privateConfigName)

	if err != nil {
		return nil, errors.New("Couldn't get private config: " + err.Error())
	}

	return publicConfig.extend(privateConfig), nil

}

func Get() (*Config, error) {

	config, err := combinedConfig()

	if err != nil {
		return nil, err
	}

	config.derive()

	if err := config.validate(); err != nil {
		return nil, errors.New("Couldn't validate config: " + err.Error())
	}

	return config, nil

}
<|endoftext|>