<commit_before>package main

import (
  "encoding/json"
  "fmt"
  logging "github.com/op/go-logging"
  "github.com/streadway/amqp"
  . "koding/db/models"
  helper "koding/db/mongodb/modelhelper"
  "koding/messaging/rabbitmq"
  "labix.org/v2/mgo"
  stdlog "log"
  "os"
  "strings"
)

type Status string

const (
  DELETE Status = "delete"
  MERGE  Status = "merge"
)

var (
  EXCHANGE_NAME     = "topicModifierExchange"
  WORKER_QUEUE_NAME = "topicModifierWorkerQueue"
  log               = logging.MustGetLogger("TopicModifier")
)

type TagModifierData struct {
  TagId  string `json:"tagId"`
  Status Status `json:"status"`
}

func init() {
  configureLogger()
}

func main() {
  exchange := rabbitmq.Exchange{
    Name:    EXCHANGE_NAME,
    Type:    "fanout",
    Durable: true,
  }

  queue := rabbitmq.Queue{
    Name:    WORKER_QUEUE_NAME,
    Durable: true,
  }

  binding := rabbitmq.BindingOptions{
    RoutingKey: "",
  }

  consumerOptions := rabbitmq.ConsumerOptions{
    Tag: "TopicModifier",
  }

  consumer, err := rabbitmq.NewConsumer(exchange, queue, binding, consumerOptions)
  if err != nil {
    log.Error("%v", err)
    return
  }

  defer consumer.Shutdown()
  err = consumer.QOS(3)
  if err != nil {
    panic(err)
  }

  defer PUBLISHER.Shutdown()

  log.Info("Topic Modifier worker started")
  consumer.RegisterSignalHandler()
  consumer.Consume(messageConsumer)
}

func configureLogger() {
  logging.SetLevel(logging.INFO, "TopicModifier")
  log.Module = "TopicModifier"
  logging.SetFormatter(logging.MustStringFormatter("%{level:-3s} ▶ %{message}"))
  stderrBackend := logging.NewLogBackend(os.Stderr, "", stdlog.LstdFlags|stdlog.Lshortfile)
  stderrBackend.Color = true
  logging.SetBackend(stderrBackend)
}

var messageConsumer = func(delivery amqp.Delivery) {

  modifierData := &TagModifierData{}
  if err := json.Unmarshal([]byte(delivery.Body), modifierData); err != nil {
    log.Error("Wrong Post Format", err, delivery)
  }

  tagId := modifierData.TagId
  switch modifierData.Status {
  default:
    log.Error("Unknown modification status %s", modifierData.Status)
  case DELETE:
    deleteTags(tagId)
  case MERGE:
    mergeTags(tagId)
  }
  delivery.Ack(false)

}

//Deletes given tags. Tags are removed from post bodies and collections.
//Tag relations are also removed.
func deleteTags(tagId string) {
  log.Info("Deleting obsolete tag")
  tag, err := helper.GetTagById(tagId)
  if err != nil {
    log.Error("Tag not found - Id: ", tagId)
    return
  }

  selector := helper.Selector{"targetId": helper.GetObjectId(tagId), "as": "tag"}

  rels := helper.GetRelationships(selector)
  updatePosts(rels, "")
  updateTagRelationships(rels, &Tag{})

  postRels := convertTagRelationships(rels)
  updateTagRelationships(postRels, &Tag{})

  tag.Counts = TagCount{}
  helper.UpdateTag(tag)
}

func mergeTags(tagId string) {
  log.Info("Merging topics")

  tag, err := helper.GetTagById(tagId)
  if err != nil {
    log.Error("Tag not found - Id: ", tagId)
    return
  }

  synonym, err := FindSynonym(tagId)
  if err != nil {
    log.Error("Synonym not found - Id %s", tagId)
    return
  }
  log.Info("Merging Topic %s into %s", tag.Title, synonym.Title)

  selector := helper.Selector{"targetId": helper.GetObjectId(tagId), "as": "tag"}
  tagRels := helper.GetRelationships(selector)

  taggedPostCount := len(tagRels)
  log.Info("%v tagged posts found", taggedPostCount)
  if taggedPostCount > 0 {
    updatedPostRels := updatePosts(tagRels, synonym.Id.Hex())
    postCount := len(updatedPostRels)
    log.Info("Merged Post count %d", postCount)
    synonym.Counts.Post += postCount

    updateTagRelationships(updatedPostRels, synonym)
    postRels := convertTagRelationships(updatedPostRels)
    updateTagRelationships(postRels, synonym)
  }

  updateCounts(tag, synonym)
  synonym.Counts.Followers += updateFollowers(tag, synonym)
  helper.UpdateTag(synonym)
  tag.Counts = TagCount{} // reset counts
  helper.UpdateTag(tag)
}

func convertTagRelationships(tagRels []Relationship) (postRelationships []Relationship) {
  for _, tagRel := range tagRels {
    postRelationships = append(postRelationships, swapTagRelation(&tagRel, "post"))
  }

  return postRelationships
}

//Update post tags with new ones. When newTagId = "" or post already
//includes new tag, then it just removes old tag and also removes tag relationship
//Returns Filtered Relationships
func updatePosts(rels []Relationship, newTagId string) (filteredRels []Relationship) {
  for _, rel := range rels {
    tagId := rel.TargetId.Hex()
    post, err := helper.GetStatusUpdateById(rel.SourceId.Hex())
    if err != nil {
      log.Error("Status Update Not Found - Id: %s, Err: %s", rel.SourceId.Hex(), err)
      continue
    }
    tagIncluded := updatePostBody(post, tagId, newTagId)
    err = helper.UpdateStatusUpdate(post)

    if err != nil {
      log.Error(err.Error())
      continue
    }

    if !tagIncluded {
      filteredRels = append(filteredRels, rel)
    } else {
      removeRelationship(&rel)
      postRel := swapTagRelation(&rel, "post")
      removeRelationship(&postRel)
    }
  }

  return filteredRels
}

//Replaces given post tagId with new one. If new tag is already included
//then it just removes old one.
//Returns tag included information
func updatePostBody(s *StatusUpdate, tagId string, newTagId string) (tagIncluded bool) {
  var newTag string
  tagIncluded = false
  if newTagId != "" {
    newTag = fmt.Sprintf("|#:JTag:%v|", newTagId)
    //new tag already included in post
    if strings.Index(s.Body, newTag) != -1 {
      tagIncluded = true
      newTag = ""
    }
  }

  modifiedTag := fmt.Sprintf("|#:JTag:%v|", tagId)
  s.Body = strings.Replace(s.Body, modifiedTag, newTag, -1)
  return tagIncluded
}

//Removes old tag relationships and creates new ones if synonym tag does exists
func updateTagRelationships(rels []Relationship, synonym *Tag) {
  for _, rel := range rels {
    removeRelationship(&rel)
    if synonym.Id.Hex() != "" {
      if rel.TargetName == "JTag" {
        rel.TargetId = synonym.Id
      } else {
        rel.SourceId = synonym.Id
      }
      rel.Id = helper.NewObjectId()
      createRelationship(&rel)
    }
  }
}

func updateCounts(tag *Tag, synonym *Tag) {
  synonym.Counts.Following += tag.Counts.Following // does this have any meaning?
  synonym.Counts.Tagged += tag.Counts.Tagged
}

//Moves follower information under the new topic. If user is already following
//new topic, then she is not added as follower.
func updateFollowers(tag *Tag, synonym *Tag) int {
  selector := helper.Selector{
    "sourceId":   tag.Id,
    "as":         "follower",
    "targetName": "JAccount",
  }

  rels := helper.GetRelationships(selector)
  var oldFollowers []Relationship
  var newFollowers []Relationship

  for _, rel := range rels {
    selector["sourceId"] = synonym.Id
    selector["targetId"] = rel.TargetId

    // checking if relationship already exists for the synonym
    _, err := helper.GetRelationship(selector)
    //because there are two relations as account -> follower -> tag and
    //tag -> follower -> account, we have added
    if err != nil {
      if err == mgo.ErrNotFound {
        newFollowers = append(newFollowers, rel)
      } else {
        log.Error(err.Error())
        return 0
      }
    } else {
      oldFollowers = append(oldFollowers, rel)
    }
  }

  log.Info("%v users are already following new topic", len(oldFollowers))
  if len(oldFollowers) > 0 {
    updateTagRelationships(oldFollowers, &Tag{})
  }
  log.Info("%v users followed new topic", len(newFollowers))
  if len(newFollowers) > 0 {
    updateTagRelationships(newFollowers, synonym)
  }

  return len(newFollowers)
}

func swapTagRelation(r *Relationship, as string) Relationship {
  return Relationship{
    As:         as,
    SourceId:   r.TargetId,
    SourceName: r.TargetName,
    TargetId:   r.SourceId,
    TargetName: r.SourceName,
    TimeStamp:  r.TimeStamp,
  }
}
<commit_msg>Moderation: post is deleted when a post body becomes empty after deleting topics<commit_after>package main

import (
  "encoding/json"
  "fmt"
  logging "github.com/op/go-logging"
  "github.com/streadway/amqp"
  . "koding/db/models"
  helper "koding/db/mongodb/modelhelper"
  "koding/messaging/rabbitmq"
  "labix.org/v2/mgo"
  stdlog "log"
  "os"
  "strings"
)

type Status string

const (
  DELETE Status = "delete"
  MERGE  Status = "merge"
)

var (
  EXCHANGE_NAME     = "topicModifierExchange"
  WORKER_QUEUE_NAME = "topicModifierWorkerQueue"
  log               = logging.MustGetLogger("TopicModifier")
)

type TagModifierData struct {
  TagId  string `json:"tagId"`
  Status Status `json:"status"`
}

func init() {
  configureLogger()
}

func main() {
  exchange := rabbitmq.Exchange{
    Name:    EXCHANGE_NAME,
    Type:    "fanout",
    Durable: true,
  }

  queue := rabbitmq.Queue{
    Name:    WORKER_QUEUE_NAME,
    Durable: true,
  }

  binding := rabbitmq.BindingOptions{
    RoutingKey: "",
  }

  consumerOptions := rabbitmq.ConsumerOptions{
    Tag: "TopicModifier",
  }

  consumer, err := rabbitmq.NewConsumer(exchange, queue, binding, consumerOptions)
  if err != nil {
    log.Error("%v", err)
    return
  }

  defer consumer.Shutdown()
  err = consumer.QOS(3)
  if err != nil {
    panic(err)
  }

  defer PUBLISHER.Shutdown()

  log.Info("Topic Modifier worker started")
  consumer.RegisterSignalHandler()
  consumer.Consume(messageConsumer)
}

func configureLogger() {
  logging.SetLevel(logging.INFO, "TopicModifier")
  log.Module = "TopicModifier"
  logging.SetFormatter(logging.MustStringFormatter("%{level:-3s} ▶ %{message}"))
  stderrBackend := logging.NewLogBackend(os.Stderr, "", stdlog.LstdFlags|stdlog.Lshortfile)
  stderrBackend.Color = true
  logging.SetBackend(stderrBackend)
}

var messageConsumer = func(delivery amqp.Delivery) {

  modifierData := &TagModifierData{}
  if err := json.Unmarshal([]byte(delivery.Body), modifierData); err != nil {
    log.Error("Wrong Post Format", err, delivery)
  }

  tagId := modifierData.TagId
  switch modifierData.Status {
  default:
    log.Error("Unknown modification status %s", modifierData.Status)
  case DELETE:
    deleteTags(tagId)
  case MERGE:
    mergeTags(tagId)
  }
  delivery.Ack(false)

}

//Deletes given tags. Tags are removed from post bodies and collections.
//Tag relations are also removed.
func deleteTags(tagId string) {
  log.Info("Deleting obsolete tag")
  tag, err := helper.GetTagById(tagId)
  if err != nil {
    log.Error("Tag not found - Id: ", tagId)
    return
  }

  selector := helper.Selector{"targetId": helper.GetObjectId(tagId), "as": "tag"}

  rels := helper.GetRelationships(selector)
  updatePosts(rels, "")
  updateTagRelationships(rels, &Tag{})

  postRels := convertTagRelationships(rels)
  updateTagRelationships(postRels, &Tag{})

  tag.Counts = TagCount{}
  helper.UpdateTag(tag)
}

func mergeTags(tagId string) {
  log.Info("Merging topics")

  tag, err := helper.GetTagById(tagId)
  if err != nil {
    log.Error("Tag not found - Id: ", tagId)
    return
  }

  synonym, err := FindSynonym(tagId)
  if err != nil {
    log.Error("Synonym not found - Id %s", tagId)
    return
  }
  log.Info("Merging Topic %s into %s", tag.Title, synonym.Title)

  selector := helper.Selector{"targetId": helper.GetObjectId(tagId), "as": "tag"}
  tagRels := helper.GetRelationships(selector)

  taggedPostCount := len(tagRels)
  log.Info("%v tagged posts found", taggedPostCount)
  if taggedPostCount > 0 {
    updatedPostRels := updatePosts(tagRels, synonym.Id.Hex())
    postCount := len(updatedPostRels)
    log.Info("Merged Post count %d", postCount)
    synonym.Counts.Post += postCount

    updateTagRelationships(updatedPostRels, synonym)
    postRels := convertTagRelationships(updatedPostRels)
    updateTagRelationships(postRels, synonym)
  }

  updateCounts(tag, synonym)
  synonym.Counts.Followers += updateFollowers(tag, synonym)
  helper.UpdateTag(synonym)
  tag.Counts = TagCount{} // reset counts
  helper.UpdateTag(tag)
}

func convertTagRelationships(tagRels []Relationship) (postRelationships []Relationship) {
  for _, tagRel := range tagRels {
    postRelationships = append(postRelationships, swapTagRelation(&tagRel, "post"))
  }

  return postRelationships
}

//Update post tags with new ones. When newTagId = "" or post already
//includes new tag, then it just removes old tag and also removes tag relationship
//Returns Filtered Relationships
func updatePosts(rels []Relationship, newTagId string) (filteredRels []Relationship) {
  for _, rel := range rels {
    tagId := rel.TargetId.Hex()
    post, err := helper.GetStatusUpdateById(rel.SourceId.Hex())
    if err != nil {
      log.Error("Status Update Not Found - Id: %s, Err: %s", rel.SourceId.Hex(), err)
      continue
    }

    tagIncluded := updatePostBody(post, tagId, newTagId)
    if strings.TrimSpace(post.Body) == "" {
      DeleteStatusUpdate(post.Id.Hex())
    } else {
      err = helper.UpdateStatusUpdate(post)
    }

    if err != nil {
      log.Error(err.Error())
      continue
    }

    if !tagIncluded {
      filteredRels = append(filteredRels, rel)
    } else {
      RemoveRelationship(&rel)
      postRel := swapTagRelation(&rel, "post")
      RemoveRelationship(&postRel)
    }
  }

  return filteredRels
}

//Replaces given post tagId with new one. If new tag is already included
//then it just removes old one.
//Returns tag included information
func updatePostBody(s *StatusUpdate, tagId string, newTagId string) (tagIncluded bool) {
  var newTag string
  tagIncluded = false
  if newTagId != "" {
    newTag = fmt.Sprintf("|#:JTag:%v|", newTagId)
    //new tag already included in post
    if strings.Index(s.Body, newTag) != -1 {
      tagIncluded = true
      newTag = ""
    }
  }

  modifiedTag := fmt.Sprintf("|#:JTag:%v|", tagId)
  s.Body = strings.Replace(s.Body, modifiedTag, newTag, -1)
  return tagIncluded
}

//Removes old tag relationships and creates new ones if synonym tag does exists
func updateTagRelationships(rels []Relationship, synonym *Tag) {
  for _, rel := range rels {
    RemoveRelationship(&rel)
    if synonym.Id.Hex() != "" {
      if rel.TargetName == "JTag" {
        rel.TargetId = synonym.Id
      } else {
        rel.SourceId = synonym.Id
      }
      rel.Id = helper.NewObjectId()
      CreateRelationship(&rel)
    }
  }
}

func updateCounts(tag *Tag, synonym *Tag) {
  synonym.Counts.Following += tag.Counts.Following // does this have any meaning?
  synonym.Counts.Tagged += tag.Counts.Tagged
}

//Moves follower information under the new topic. If user is already following
//new topic, then she is not added as follower.
func updateFollowers(tag *Tag, synonym *Tag) int {
  selector := helper.Selector{
    "sourceId":   tag.Id,
    "as":         "follower",
    "targetName": "JAccount",
  }

  rels := helper.GetRelationships(selector)
  var oldFollowers []Relationship
  var newFollowers []Relationship

  for _, rel := range rels {
    selector["sourceId"] = synonym.Id
    selector["targetId"] = rel.TargetId

    // checking if relationship already exists for the synonym
    _, err := helper.GetRelationship(selector)
    //because there are two relations as account -> follower -> tag and
    //tag -> follower -> account, we have added
    if err != nil {
      if err == mgo.ErrNotFound {
        newFollowers = append(newFollowers, rel)
      } else {
        log.Error(err.Error())
        return 0
      }
    } else {
      oldFollowers = append(oldFollowers, rel)
    }
  }

  log.Info("%v users are already following new topic", len(oldFollowers))
  if len(oldFollowers) > 0 {
    updateTagRelationships(oldFollowers, &Tag{})
  }
  log.Info("%v users followed new topic", len(newFollowers))
  if len(newFollowers) > 0 {
    updateTagRelationships(newFollowers, synonym)
  }

  return len(newFollowers)
}

func swapTagRelation(r *Relationship, as string) Relationship {
  return Relationship{
    As:         as,
    SourceId:   r.TargetId,
    SourceName: r.TargetName,
    TargetId:   r.SourceId,
    TargetName: r.SourceName,
    TimeStamp:  r.TimeStamp,
  }
}
<|endoftext|>


<commit_before>package log

import (
	"fmt"
	"io/ioutil"
	"net/http"
	"net/http/httptest"
	"os"
	"regexp"
	"testing"
	"time"
)

func validate(key string, logFunc func(msg ...interface{}), valueExpected string) (err error) {
	rescueStdout := os.Stdout
	defer func() { os.Stdout = rescueStdout }()

	r, w, err := os.Pipe()
	if err != nil {
		return
	}
	os.Stdout = w

	logFunc("log test")

	err = w.Close()
	if err != nil {
		return
	}

	out, err := ioutil.ReadAll(r)
	if err != nil {
		return
	}

	if string(out) != valueExpected {
		err = fmt.Errorf("Error, '%s' printed %q, expected %q", key, string(out), valueExpected)
	}
	return
}

func TestLog(t *testing.T) {
	now = func() time.Time { return time.Unix(1498405744, 0) }
	DebugMode = false

	data := []struct {
		key           string
		logFunc       func(msg ...interface{})
		expectedValue string
	}{
		{"Println", Println, "\x1b[37m2017/06/25 15:49:04 [msg] log test\x1b[0;00m\n"},
		{"Errorln", Errorln, "\x1b[91m2017/06/25 15:49:04 [error] log test\x1b[0;00m\n"},
		{"Warningln", Warningln, "\x1b[93m2017/06/25 15:49:04 [warning] log test\x1b[0;00m\n"},
		{"Debugln", Debugln, ""},
	}

	for _, v := range data {
		err := validate(v.key, v.logFunc, v.expectedValue)
		if err != nil {
			t.Fatal(err.Error())
		}
	}
	DebugMode = true

	rescueStdout := os.Stdout
	defer func() {
		os.Stdout = rescueStdout
		DebugMode = false
	}()

	r, w, err := os.Pipe()
	if err != nil {
		return
	}
	os.Stdout = w

	Debugln("log test")

	os.Stdout = rescueStdout

	err = w.Close()
	if err != nil {
		t.Fatal(err.Error())
	}

	out, err := ioutil.ReadAll(r)
	if err != nil {
		t.Fatal(err.Error())
	}
	rstr := "\x1b\\[96m2017/06/25 15:49:04 \\[debug\\] logsys_test.go:\\d+ log test\x1b\\[0;00m\n"
	match, err := regexp.Match(rstr, out)
	if err != nil {
		t.Fatal(err.Error())
	}

	if !match {
		t.Fatalf("Error, 'Debugln' printed %q, not match with expected", string(out))
	}

}

func TestHTTPError(t *testing.T) {
	now = func() time.Time { return time.Unix(1498405744, 0) }

	rescueStdout := os.Stdout
	DebugMode = false
	defer func() { os.Stdout = rescueStdout }()

	r, w, err := os.Pipe()
	if err != nil {
		return
	}
	os.Stdout = w

	handler := func(w http.ResponseWriter, r *http.Request) {
		HTTPError(w, http.StatusBadRequest)
	}

	req := httptest.NewRequest("GET", "http://example.com/foo", nil)
	httpw := httptest.NewRecorder()
	handler(httpw, req)

	resp := httpw.Result()
	body, _ := ioutil.ReadAll(resp.Body)

	os.Stdout = rescueStdout
	err = w.Close()
	if err != nil {
		return
	}

	out, err := ioutil.ReadAll(r)
	if err != nil {
		return
	}

	valueExpected := "\x1b[91m2017/06/25 15:49:04 [error] Bad Request\x1b[0;00m\n"
	if string(out) != valueExpected {
		t.Fatalf("Error, 'HTTPError' printed %q, expected %q", string(out), valueExpected)
	}

	if resp.StatusCode != http.StatusBadRequest {
		t.Fatalf("Error, 'HTTPError' status code %v, expected 400", resp.StatusCode)
	}

	valueExpected = "{\n\t\"error\": \"Bad Request\",\n\t\"status\": \"error\"\n}\n"
	if string(body) != valueExpected {
		t.Fatalf("Error, 'HTTPError' write to client %q, expected %q", string(body), valueExpected)
	}

}
<commit_msg>Fixing broken test<commit_after>package log

import (
	"fmt"
	"io/ioutil"
	"net/http"
	"net/http/httptest"
	"os"
	"regexp"
	"testing"
	"time"
)

func validate(key string, logFunc func(msg ...interface{}), valueExpected string) (err error) {
	rescueStdout := os.Stdout
	defer func() { os.Stdout = rescueStdout }()

	r, w, err := os.Pipe()
	if err != nil {
		return
	}
	os.Stdout = w

	logFunc("log test")

	err = w.Close()
	if err != nil {
		return
	}

	out, err := ioutil.ReadAll(r)
	if err != nil {
		return
	}

	if string(out) != valueExpected {
		err = fmt.Errorf("Error, '%s' printed %q, expected %q", key, string(out), valueExpected)
	}
	return
}

func TestLog(t *testing.T) {
	now = func() time.Time { return time.Unix(1498405744, 0) }
	DebugMode = false

	data := []struct {
		key           string
		logFunc       func(msg ...interface{})
		expectedValue string
	}{
		{"Println", Println, "\x1b[37m2017/06/25 15:49:04 [msg] log test\x1b[0;00m\n"},
		{"Errorln", Errorln, "\x1b[91m2017/06/25 15:49:04 [error] log test\x1b[0;00m\n"},
		{"Warningln", Warningln, "\x1b[93m2017/06/25 15:49:04 [warning] log test\x1b[0;00m\n"},
		{"Debugln", Debugln, ""},
	}

	for _, v := range data {
		err := validate(v.key, v.logFunc, v.expectedValue)
		if err != nil {
			t.Fatal(err.Error())
		}
	}
	DebugMode = true

	rescueStdout := os.Stdout
	defer func() {
		os.Stdout = rescueStdout
		DebugMode = false
	}()

	r, w, err := os.Pipe()
	if err != nil {
		return
	}
	os.Stdout = w

	Debugln("log test")

	os.Stdout = rescueStdout

	err = w.Close()
	if err != nil {
		t.Fatal(err.Error())
	}

	out, err := ioutil.ReadAll(r)
	if err != nil {
		t.Fatal(err.Error())
	}
	rstr := "\x1b\\[96m2017/06/25 15:49:04 \\[debug\\] log_test.go:\\d+ log test\x1b\\[0;00m\n"
	match, err := regexp.Match(rstr, out)
	if err != nil {
		t.Fatal(err.Error())
	}

	if !match {
		t.Fatalf("Error, 'Debugln' printed %q, not match with expected", string(out))
	}

}

func TestHTTPError(t *testing.T) {
	now = func() time.Time { return time.Unix(1498405744, 0) }

	rescueStdout := os.Stdout
	DebugMode = false
	defer func() { os.Stdout = rescueStdout }()

	r, w, err := os.Pipe()
	if err != nil {
		return
	}
	os.Stdout = w

	handler := func(w http.ResponseWriter, r *http.Request) {
		HTTPError(w, http.StatusBadRequest)
	}

	req := httptest.NewRequest("GET", "http://example.com/foo", nil)
	httpw := httptest.NewRecorder()
	handler(httpw, req)

	resp := httpw.Result()
	body, _ := ioutil.ReadAll(resp.Body)

	os.Stdout = rescueStdout
	err = w.Close()
	if err != nil {
		return
	}

	out, err := ioutil.ReadAll(r)
	if err != nil {
		return
	}

	valueExpected := "\x1b[91m2017/06/25 15:49:04 [error] Bad Request\x1b[0;00m\n"
	if string(out) != valueExpected {
		t.Fatalf("Error, 'HTTPError' printed %q, expected %q", string(out), valueExpected)
	}

	if resp.StatusCode != http.StatusBadRequest {
		t.Fatalf("Error, 'HTTPError' status code %v, expected 400", resp.StatusCode)
	}

	valueExpected = "{\n\t\"error\": \"Bad Request\",\n\t\"status\": \"error\"\n}\n"
	if string(body) != valueExpected {
		t.Fatalf("Error, 'HTTPError' write to client %q, expected %q", string(body), valueExpected)
	}

}
<|endoftext|>


<commit_before>package throttler

import (
	"fmt"
	"time"

	"github.com/youtube/vitess/go/vt/proto/throttlerdata"
)

// MaxReplicationLagModuleConfig stores all configuration parameters for
// MaxReplicationLagModule. Internally, the parameters are represented by a
// protobuf message. This message is also used to update the parameters.
type MaxReplicationLagModuleConfig struct {
	throttlerdata.MaxReplicationLagModuleConfig
}

var defaultMaxReplicationLagModuleConfig = MaxReplicationLagModuleConfig{
	throttlerdata.MaxReplicationLagModuleConfig{
		TargetReplicationLagSec: 1,
		MaxReplicationLagSec:    ReplicationLagModuleDisabled,

		InitialRate: 100,
		// 1 means 100% i.e. double rates by default.
		MaxIncrease:       1,
		EmergencyDecrease: 0.5,

		MinDurationBetweenChangesSec:   10,
		MaxDurationBetweenIncreasesSec: 61,
	},
}

// NewMaxReplicationLagModuleConfig returns a default configuration where
// only "maxReplicationLag" is set.
func NewMaxReplicationLagModuleConfig(maxReplicationLag int64) MaxReplicationLagModuleConfig {
	config := defaultMaxReplicationLagModuleConfig
	config.MaxReplicationLagSec = maxReplicationLag
	return config
}

// TODO(mberlin): Add method which updates the config using a (partially) filled
// in protobuf.

// Verify returns an error if the config is invalid.
func (c MaxReplicationLagModuleConfig) Verify() error {
	if c.TargetReplicationLagSec > c.MaxReplicationLagSec {
		return fmt.Errorf("target replication lag must not be higher than the configured max replication lag: invalid: %v > %v",
			c.TargetReplicationLagSec, c.MaxReplicationLagSec)
	}
	return nil
}

// MinDurationBetweenChanges is a helper function which returns the respective
// protobuf field as native Go type.
func (c MaxReplicationLagModuleConfig) MinDurationBetweenChanges() time.Duration {
	return time.Duration(c.MinDurationBetweenChangesSec) * time.Second
}

// MaxDurationBetweenIncreases is a helper function which returns the respective
// protobuf field as native Go type.
func (c MaxReplicationLagModuleConfig) MaxDurationBetweenIncreases() time.Duration {
	return time.Duration(c.MaxDurationBetweenIncreasesSec) * time.Second
}
<commit_msg>throttler: MaxReplicationLagModule: Increase default target replication lag from 1 to 2.<commit_after>package throttler

import (
	"fmt"
	"time"

	"github.com/youtube/vitess/go/vt/proto/throttlerdata"
)

// MaxReplicationLagModuleConfig stores all configuration parameters for
// MaxReplicationLagModule. Internally, the parameters are represented by a
// protobuf message. This message is also used to update the parameters.
type MaxReplicationLagModuleConfig struct {
	throttlerdata.MaxReplicationLagModuleConfig
}

var defaultMaxReplicationLagModuleConfig = MaxReplicationLagModuleConfig{
	throttlerdata.MaxReplicationLagModuleConfig{
		TargetReplicationLagSec: 2,
		MaxReplicationLagSec:    ReplicationLagModuleDisabled,

		InitialRate: 100,
		// 1 means 100% i.e. double rates by default.
		MaxIncrease:       1,
		EmergencyDecrease: 0.5,

		MinDurationBetweenChangesSec:   10,
		MaxDurationBetweenIncreasesSec: 61,
	},
}

// NewMaxReplicationLagModuleConfig returns a default configuration where
// only "maxReplicationLag" is set.
func NewMaxReplicationLagModuleConfig(maxReplicationLag int64) MaxReplicationLagModuleConfig {
	config := defaultMaxReplicationLagModuleConfig
	config.MaxReplicationLagSec = maxReplicationLag
	return config
}

// TODO(mberlin): Add method which updates the config using a (partially) filled
// in protobuf.

// Verify returns an error if the config is invalid.
func (c MaxReplicationLagModuleConfig) Verify() error {
	if c.TargetReplicationLagSec > c.MaxReplicationLagSec {
		return fmt.Errorf("target replication lag must not be higher than the configured max replication lag: invalid: %v > %v",
			c.TargetReplicationLagSec, c.MaxReplicationLagSec)
	}
	return nil
}

// MinDurationBetweenChanges is a helper function which returns the respective
// protobuf field as native Go type.
func (c MaxReplicationLagModuleConfig) MinDurationBetweenChanges() time.Duration {
	return time.Duration(c.MinDurationBetweenChangesSec) * time.Second
}

// MaxDurationBetweenIncreases is a helper function which returns the respective
// protobuf field as native Go type.
func (c MaxReplicationLagModuleConfig) MaxDurationBetweenIncreases() time.Duration {
	return time.Duration(c.MaxDurationBetweenIncreasesSec) * time.Second
}
<|endoftext|>


<commit_before>package logplexc

import (
	"bytes"
	"fmt"
	"net/http"
	"net/url"
	"strconv"
	"sync"
	"time"
)

// Running statistics on Client operation
type Stats struct {
	NumberFramed uint64
	Buffered     int
}

// Configuration of a Client.
//
// The configuration is by-value to prevent most kinds of accidental
// sharing of modifications between clients.  Also, modification of
// the URL is compulsary in the constructor, and it's not desirable to
// modify the version passed by the user as a side effect of
// constructing a client instance.
type Config struct {
	Logplex    url.URL
	Token      string
	HttpClient http.Client
}

// A bundle of messages that are either being accrued to or in the
// progress of being sent.
//
// This is packaged into its own type as it is handy to be able to
// manipulate bundles to pipeline buffering new logs to be written
// with bundles that have I/O in progress.
type bundle struct {
	nFramed uint64
	outbox  bytes.Buffer
}

// Client context: generally, at a minimum, one should exist per
// Logplex credential serviced by the program.
type Client struct {
	Stats

	// Configuration that should not be mutated after creation
	Config

	reqInFlight sync.WaitGroup

	// Messages that have been collected but not yet sent.
	bSwapLock sync.Mutex
	b         *bundle
}

func NewClient(cfg *Config) (client *Client, err error) {
	c := Client{}

	c.b = &bundle{outbox: bytes.Buffer{}}

	// Make a private copy
	c.Config = *cfg

	// If the username and password weren't part of the URL, use
	// the logplex-token as the password
	if c.Logplex.User == nil {
		c.Logplex.User = url.UserPassword("token", c.Token)
	}

	return &c, nil
}

// Unsynchronized statistics gathering function
//
// Useful as a subroutine for procedures that already have taken care
// of synchronization.
func unsyncStats(b *bundle) Stats {
	return Stats{
		NumberFramed: b.nFramed,
		Buffered:     b.outbox.Len(),
	}
}

// Copy the statistics structure embedded in the client.
func (c *Client) Statistics() Stats {
	c.bSwapLock.Lock()
	defer c.bSwapLock.Unlock()

	return unsyncStats(c.b)
}

// Buffer a message for best-effort delivery to Logplex
//
// Return the critical statistics on what has been buffered so far so
// that the caller can opt to PostMessages() and empty the buffer.
//
// No effort is expended to clean up bad bytes disallowed by syslog,
// as Logplex has a length-prefixed format and the intention that each
// Client will only process messages for a single user/security
// context, so at worst it seems a buggy or malicious emitter of logs
// can cause problems for themselves only.
func (c *Client) BufferMessage(when time.Time, procId string, log []byte) Stats {
	// Avoid racing against other operations that may want to swap
	// out client's current bundle.
	c.bSwapLock.Lock()
	defer c.bSwapLock.Unlock()

	ts := when.UTC().Format(time.RFC3339)
	syslogPrefix := "<134>1 " + ts + " 1234 " +
		c.Token + " " + procId + " - - "
	msgLen := len(syslogPrefix) + len(log)

	fmt.Fprintf(&c.b.outbox, "%d %s%s", msgLen, syslogPrefix, log)
	c.b.nFramed += 1

	return unsyncStats(c.b)
}

// Post messages that are pending being posted.
func (c *Client) PostMessages() (*http.Response, Stats, error) {
	// Swap out the bundle that is about to go through a long I/O
	// operation for a fresh one, so that buffering can continue
	// again immediately.
	s, b := func() (Stats, bundle) {
		c.bSwapLock.Lock()
		defer c.bSwapLock.Unlock()

		s := c.Stats
		b := *c.b
		c.b.outbox = bytes.Buffer{}
		c.b.nFramed = 0

		return s, b
	}()

	// Record that a request is in progress so that a clean
	// shutdown can wait for it to complete.
	c.reqInFlight.Add(1)
	defer c.reqInFlight.Done()

	req, _ := http.NewRequest("POST", c.Logplex.String(), &b.outbox)
	req.Header.Add("Content-Type", "application/logplex-1")
	req.Header.Add("Logplex-Msg-Count", strconv.FormatUint(b.nFramed, 10))

	resp, err := c.HttpClient.Do(req)
	if err != nil {
		return nil, s, err
	}

	return resp, s, nil
}
<commit_msg>Fix Client statistics gathering<commit_after>package logplexc

import (
	"bytes"
	"fmt"
	"net/http"
	"net/url"
	"strconv"
	"sync"
	"time"
)

// Running statistics on Client operation
type Stats struct {
	NumberFramed uint64
	Buffered     int
}

// Configuration of a Client.
//
// The configuration is by-value to prevent most kinds of accidental
// sharing of modifications between clients.  Also, modification of
// the URL is compulsary in the constructor, and it's not desirable to
// modify the version passed by the user as a side effect of
// constructing a client instance.
type Config struct {
	Logplex    url.URL
	Token      string
	HttpClient http.Client
}

// A bundle of messages that are either being accrued to or in the
// progress of being sent.
//
// This is packaged into its own type as it is handy to be able to
// manipulate bundles to pipeline buffering new logs to be written
// with bundles that have I/O in progress.
type bundle struct {
	Stats
	outbox bytes.Buffer
}

// Client context: generally, at a minimum, one should exist per
// Logplex credential serviced by the program.
type Client struct {
	// Configuration that should not be mutated after creation
	Config

	reqInFlight sync.WaitGroup

	// Messages that have been collected but not yet sent.
	bSwapLock sync.Mutex
	b         *bundle
}

func NewClient(cfg *Config) (client *Client, err error) {
	c := Client{}

	c.b = &bundle{outbox: bytes.Buffer{}}

	// Make a private copy
	c.Config = *cfg

	// If the username and password weren't part of the URL, use
	// the logplex-token as the password
	if c.Logplex.User == nil {
		c.Logplex.User = url.UserPassword("token", c.Token)
	}

	return &c, nil
}

// Unsynchronized statistics gathering function
//
// Useful as a subroutine for procedures that already have taken care
// of synchronization.
func unsyncStats(b *bundle) Stats {
	return b.Stats
}

// Copy the statistics structure embedded in the client.
func (c *Client) Statistics() Stats {
	c.bSwapLock.Lock()
	defer c.bSwapLock.Unlock()

	return unsyncStats(c.b)
}

// Buffer a message for best-effort delivery to Logplex
//
// Return the critical statistics on what has been buffered so far so
// that the caller can opt to PostMessages() and empty the buffer.
//
// No effort is expended to clean up bad bytes disallowed by syslog,
// as Logplex has a length-prefixed format and the intention that each
// Client will only process messages for a single user/security
// context, so at worst it seems a buggy or malicious emitter of logs
// can cause problems for themselves only.
func (c *Client) BufferMessage(when time.Time, procId string, log []byte) Stats {
	// Avoid racing against other operations that may want to swap
	// out client's current bundle.
	c.bSwapLock.Lock()
	defer c.bSwapLock.Unlock()

	ts := when.UTC().Format(time.RFC3339)
	syslogPrefix := "<134>1 " + ts + " 1234 " +
		c.Token + " " + procId + " - - "
	msgLen := len(syslogPrefix) + len(log)

	fmt.Fprintf(&c.b.outbox, "%d %s%s", msgLen, syslogPrefix, log)
	c.b.NumberFramed += 1
	c.b.Buffered = c.b.outbox.Len()

	return unsyncStats(c.b)
}

// Post messages that are pending being posted.
func (c *Client) PostMessages() (*http.Response, Stats, error) {
	// Swap out the bundle that is about to go through a long I/O
	// operation for a fresh one, so that buffering can continue
	// again immediately.
	b := func() bundle {
		c.bSwapLock.Lock()
		defer c.bSwapLock.Unlock()

		b := *c.b
		c.b.outbox = bytes.Buffer{}
		c.b.Stats = Stats{}

		return b
	}()

	// Record that a request is in progress so that a clean
	// shutdown can wait for it to complete.
	c.reqInFlight.Add(1)
	defer c.reqInFlight.Done()

	req, _ := http.NewRequest("POST", c.Logplex.String(), &b.outbox)
	req.Header.Add("Content-Type", "application/logplex-1")
	req.Header.Add("Logplex-Msg-Count",
		strconv.FormatUint(b.NumberFramed, 10))

	resp, err := c.HttpClient.Do(req)
	if err != nil {
		return nil, b.Stats, err
	}

	return resp, b.Stats, nil
}
<|endoftext|>


<commit_before>package vkapi

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"net/url"
	"strconv"
	"time"
)

const (
	LPFlagMessageUnread = 1 << iota
	LPFlagMessageOutBox
	LPFlagMessageReplied
	LPFlagMessageImportant
	LPFlagMessageChat
	LPFlagMessageFriends
	LPFlagMessageSpam
	LPFlagMessageDeleted
	LPFlagMessageFixed
	LPFlagMessageMedia
	LPFlagMessageHidden = 65536
)

const (
	LPModeAttachments   = 2
	LPModeExtendedEvent = 8
	LPModePts           = 32
	LPModeExtra         = 64
	LPModeRandomID      = 128
)

const (
	LPCodeNewMessage    = 4
	LPCodeFriendOnline  = 8
	LPCodeFriendOffline = 9
)

const (
	LPPlatformUndefined = iota
	LPPlatformMobile
	LPPlatformIPhone
	LPPlatformIPad
	LPPlatformAndroid
	LPPlatformWPhone
	LPPlatformWindows
	LPPlatformWeb
)

// Timestamp is the wrapper of int64.
type Timestamp int64

func (ts Timestamp) String() string {
	return time.Unix(int64(ts), 0).Format("15:04:05 02/01/2006")
}

// LongPoll allow you to interact with long poll server.
type LongPoll struct {
	Host      string    `json:"server"`
	Path      string    `json:"path"`
	Key       string    `json:"key"`
	Timestamp Timestamp `json:"ts"`
	LPVersion int       `json:"-"`
	NeedPts   int       `json:"-"`
}

// LPUpdate stores response from a long poll server.
type LPUpdate struct {
	Code               int64
	Update             []interface{}
	Message            *LPMessage
	FriendNotification *LPFriendNotification
}

// Event returns event as a string.
func (update *LPUpdate) Event() (event string) {
	switch update.Code {
	case LPCodeNewMessage:
		event = "New message"
	case LPCodeFriendOnline:
		event = "Friend online"
	case LPCodeFriendOffline:
		event = "Friend offline"
	default:
		event = "Undefined event"
	}

	return
}

// UnmarshalUpdate unmarshal a LPUpdate.
func (update *LPUpdate) UnmarshalUpdate(mode int) error {
	update.Code = int64(update.Update[0].(float64))
	switch update.Code {
	case LPCodeNewMessage:
		message := new(LPMessage)

		message.ID = int64(update.Update[1].(float64))
		message.Flags = int64(update.Update[2].(float64))
		message.FromID = int64(update.Update[3].(float64))
		message.Timestamp = Timestamp(update.Update[4].(float64))
		message.Text = update.Update[5].(string)

		if mode&LPModeAttachments == LPModeAttachments {
			message.Attachments = make(map[string]string)
			for key, value := range update.Update[6].(map[string]interface{}) {
				message.Attachments[key] = value.(string)
			}
		}

		if mode&LPModeRandomID&LPModeRandomID == (LPModeAttachments | LPModeRandomID) {
			message.RandomID = int64(update.Update[7].(float64))
		} else {
			if mode&LPModeRandomID == LPModeRandomID {
				message.RandomID = int64(update.Update[6].(float64))
			}
		}

		update.Message = message
	case LPCodeFriendOnline, LPCodeFriendOffline:
		if len(update.Update) < 3 {
			return errors.New("(" + string(update.Code) + ") invalid update size.")
		}

		friend := new(LPFriendNotification)
		friend.Code = update.Code
		friend.ID = -int64(update.Update[1].(float64))
		friend.Arg = int(update.Update[2].(float64)) & 0xFF
		friend.Timestamp = Timestamp(update.Update[3].(float64))

		update.FriendNotification = friend
	}

	return nil
}

// LPMessage is new messages
// that come from long poll server.
type LPMessage struct {
	ID          int64
	Flags       int64
	FromID      int64
	Timestamp   Timestamp
	Text        string
	Attachments map[string]string
	RandomID    int64
}

func (message *LPMessage) String() string {
	return fmt.Sprintf("Message (%d):`%s` from (%d) at %s", message.ID, message.Text, message.FromID, message.Timestamp)
}

// LPFriendNotification is a notification
// that a friend has become online or offline.
type LPFriendNotification struct {
	ID int64

	// If friend is online,
	// then Arg is equal to platform.
	//
	// If the friend offline, then
	// 0 - friend logout,
	// 1 - offline by timeout.
	Arg       int
	Timestamp Timestamp
	Code      int64
}

// Status returns event as a string.
func (friend *LPFriendNotification) Status() (status string) {
	switch friend.Code {
	case LPCodeFriendOnline:
		status = "Online"
	case LPCodeFriendOffline:
		status = "Offline"
	default:
		status = "Undefined event"
	}

	return
}

func (friend *LPFriendNotification) String() string {
	return fmt.Sprintf("Friend (%d) was %s at %s", friend.ID, friend.Status(), friend.Timestamp)
}

// LPAnswer is response from long poll server.
type LPAnswer struct {
	Failed    int64           `json:"failed"`
	Timestamp Timestamp       `json:"ts"`
	Updates   [][]interface{} `json:"updates"`
}

// LPChan allows to receive new LPUpdate.
type LPChan <-chan LPUpdate

// InitLongPoll establishes a new connection
// to long poll server.
func (client *Client) InitLongPoll(needPts int, lpVersion int) *Error {
	values := url.Values{}
	values.Add("need_pts", strconv.FormatInt(int64(needPts), 10))
	values.Add("lp_version", strconv.FormatInt(int64(lpVersion), 10))

	res, err := client.Do(NewRequest("messages.getLongPollServer", "", values))
	if err != nil {
		return err
	}

	client.LongPoll = new(LongPoll)
	if err := res.To(&client.LongPoll); err != nil {
		return NewError(ErrBadCode, err.Error())
	}

	u, error := url.Parse(client.LongPoll.Host)
	if error != nil {
		return NewError(ErrBadCode, error.Error())
	}

	client.LongPoll.Host = u.Host
	client.LongPoll.Path = u.Path
	client.LongPoll.LPVersion = lpVersion
	client.LongPoll.NeedPts = needPts

	return nil
}

// LPConfig stores data to connect to long poll server.
type LPConfig struct {
	Wait int
	Mode int
}

// GetLPAnswer makes a query with parameters
// from LPConfig to long poll server
// and returns a LPAnswer in case of success.
func (client *Client) GetLPAnswer(config LPConfig) (LPAnswer, error) {
	if client.apiClient == nil {
		return LPAnswer{}, errors.New(ErrApiClientNotFound)
	}

	if client.LongPoll == nil {
		return LPAnswer{}, errors.New("A long poll was not initialized")
	}

	values := url.Values{}
	values.Add("act", "a_check")
	values.Add("key", client.LongPoll.Key)
	values.Add("ts", strconv.FormatInt(int64(client.LongPoll.Timestamp), 10))
	values.Add("wait", strconv.FormatInt(int64(config.Wait), 10))
	values.Add("mode", strconv.FormatInt(int64(config.Mode), 10))
	values.Add("version", strconv.FormatInt(int64(client.LongPoll.LPVersion), 10))

	if client.apiClient.Log {
		client.apiClient.Logger.Printf("Request: %s", NewRequest("getLongPoll", "", values).JS())
	}

	u := url.URL{}
	u.Host = client.LongPoll.Host
	u.Path = client.LongPoll.Path
	u.Scheme = "https"
	u.RawQuery = values.Encode()

	req, err := http.NewRequest("GET", u.String(), nil)
	if err != nil {
		return LPAnswer{}, err
	}

	res, err := client.apiClient.httpClient.Do(req)
	if err != nil {
		client.apiClient.logPrintf("Response error: %s", err.Error())
		return LPAnswer{}, err
	}

	var reader io.Reader
	reader = res.Body

	if client.apiClient.Log {
		b, err := ioutil.ReadAll(res.Body)
		if err != nil {
			panic(err)
		}

		client.apiClient.Logger.Printf("Response: %s", string(b))
		reader = bytes.NewReader(b)
	}

	if res.StatusCode != http.StatusOK {
		client.apiClient.Logger.Printf("Response error: %s", res.Status)
		return LPAnswer{}, errors.New(res.Status)
	}

	var answer LPAnswer
	if err = json.NewDecoder(reader).Decode(&answer); err != nil {
		return LPAnswer{}, err
	}

	return answer, nil
}

// GetLPUpdates makes a query with parameters
// from LPConfig to long poll server
// and returns array LPUpdate in case of success.
func (client *Client) GetLPUpdates(config LPConfig) ([]LPUpdate, error) {
	answer, err := client.GetLPAnswer(config)
	if err != nil {
		return []LPUpdate{}, err
	}

	var LPUpdates []LPUpdate

	switch answer.Failed {
	case 0:
		for i := len(answer.Updates) - 1; i >= 0; i-- {
			var LPUpdate LPUpdate
			LPUpdate.Update = answer.Updates[i]
			if err := LPUpdate.UnmarshalUpdate(config.Mode); err != nil {
				client.apiClient.logPrintf("%s", err.Error())
			}

			LPUpdates = append(LPUpdates, LPUpdate)
		}

		client.LongPoll.Timestamp = answer.Timestamp
		return LPUpdates, nil
	case 1:
		client.LongPoll.Timestamp = answer.Timestamp
		client.apiClient.logPrintf("Timestamp updated")
	case 2, 3:
		if err := client.InitLongPoll(client.LongPoll.NeedPts, client.LongPoll.LPVersion); err != nil {
			client.apiClient.logPrintf("Long poll update error: %s", err.Error())

			return []LPUpdate{}, err
		}

		client.apiClient.logPrintf("Long poll config updated")
	}

	return []LPUpdate{}, nil
}

// GetLPUpdatesChan makes a query with parameters
// from LPConfig to long poll server
// and returns LPChan in case of success.
func (client *Client) GetLPUpdatesChan(bufSize int, config LPConfig) (LPChan, *bool, error) {
	ch := make(chan LPUpdate, bufSize)
	run := true

	go func() {
		for run {
			updates, err := client.GetLPUpdates(config)
			if err != nil {
				log.Print("Failed to get updates, retrying in 3 seconds...")
				time.Sleep(time.Second * 3)

				continue
			}

			for _, u := range updates {
				ch <- u
			}
		}

		close(ch)
	}()

	return ch, &run, nil
}
<commit_msg>Added funtions for checking LPMessageFlags.<commit_after>package vkapi

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"html"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"net/url"
	"strconv"
	"time"
)

const (
	LPMessageFlagUnread = 1 << iota
	LPMessageFlagOutBox
	LPMessageFlagReplied
	LPMessageFlagImportant
	LPMessageFlagChat
	LPMessageFlagFriends
	LPMessageFlagSpam
	LPMessageFlagDeleted
	LPMessageFlagFixed
	LPMessageFlagMedia
	LPMessageFlagHidden = 65536
)

const (
	LPModeAttachments   = 2
	LPModeExtendedEvent = 8
	LPModePts           = 32
	LPModeExtra         = 64
	LPModeRandomID      = 128
)

const (
	LPCodeNewMessage    = 4
	LPCodeFriendOnline  = 8
	LPCodeFriendOffline = 9
)

const (
	LPPlatformUndefined = iota
	LPPlatformMobile
	LPPlatformIPhone
	LPPlatformIPad
	LPPlatformAndroid
	LPPlatformWPhone
	LPPlatformWindows
	LPPlatformWeb
)

// Timestamp is the wrapper of int64.
type Timestamp int64

func (ts Timestamp) String() string {
	return time.Unix(int64(ts), 0).Format("15:04:05 02/01/2006")
}

// LongPoll allow you to interact with long poll server.
type LongPoll struct {
	Host      string    `json:"server"`
	Path      string    `json:"path"`
	Key       string    `json:"key"`
	Timestamp Timestamp `json:"ts"`
	LPVersion int       `json:"-"`
	NeedPts   int       `json:"-"`
}

// LPUpdate stores response from a long poll server.
type LPUpdate struct {
	Code               int64
	Update             []interface{}
	Message            *LPMessage
	FriendNotification *LPFriendNotification
}

// Event returns event as a string.
func (update *LPUpdate) Event() (event string) {
	switch update.Code {
	case LPCodeNewMessage:
		event = "New message"
	case LPCodeFriendOnline:
		event = "Friend online"
	case LPCodeFriendOffline:
		event = "Friend offline"
	default:
		event = "Undefined event"
	}

	return
}

// UnmarshalUpdate unmarshal a LPUpdate.
func (update *LPUpdate) UnmarshalUpdate(mode int) error {
	update.Code = int64(update.Update[0].(float64))
	switch update.Code {
	case LPCodeNewMessage:
		message := new(LPMessage)

		message.ID = int64(update.Update[1].(float64))
		message.Flags = int64(update.Update[2].(float64))
		message.FromID = int64(update.Update[3].(float64))
		message.Timestamp = Timestamp(update.Update[4].(float64))
		message.Text = html.UnescapeString(update.Update[5].(string))

		if mode&LPModeAttachments == LPModeAttachments {
			message.Attachments = make(map[string]string)
			for key, value := range update.Update[6].(map[string]interface{}) {
				message.Attachments[key] = value.(string)
			}
		}

		if mode&LPModeRandomID&LPModeRandomID == (LPModeAttachments | LPModeRandomID) {
			message.RandomID = int64(update.Update[7].(float64))
		} else {
			if mode&LPModeRandomID == LPModeRandomID {
				message.RandomID = int64(update.Update[6].(float64))
			}
		}

		update.Message = message
	case LPCodeFriendOnline, LPCodeFriendOffline:
		if len(update.Update) < 3 {
			return errors.New("(" + string(update.Code) + ") invalid update size.")
		}

		friend := new(LPFriendNotification)
		friend.Code = update.Code
		friend.ID = -int64(update.Update[1].(float64))
		friend.Arg = int(update.Update[2].(float64)) & 0xFF
		friend.Timestamp = Timestamp(update.Update[3].(float64))

		update.FriendNotification = friend
	}

	return nil
}

// LPMessage is new messages
// that come from long poll server.
type LPMessage struct {
	ID          int64
	Flags       int64
	FromID      int64
	Timestamp   Timestamp
	Text        string
	Attachments map[string]string
	RandomID    int64
}

func (message *LPMessage) String() string {
	return fmt.Sprintf("Message (%d):`%s` from (%d) at %s", message.ID, message.Text, message.FromID, message.Timestamp)
}

// Unread will return true if the message is not read.
func (message *LPMessage) Unread() bool {
	return message.Flags&LPMessageFlagUnread != 0
}

// Outbox will return true if this is an outgoing message.
func (message *LPMessage) Outbox() bool {
	return message.Flags&LPMessageFlagOutBox != 0
}

// Replied will be returned true if an answer was created to the message.
func (message *LPMessage) Replied() bool {
	return message.Flags&LPMessageFlagReplied != 0
}

// Important will return true if this is a marked message.
func (message *LPMessage) Important() bool {
	return message.Flags&LPMessageFlagImportant != 0
}

// FromChat will return true if this message was sent via chat.
func (message *LPMessage) FromChat() bool {
	return message.Flags&LPMessageFlagChat != 0
}

// FromFriends will return true if this message was sent from friends.
// Not applicable for messages from group conversations.
func (message *LPMessage) FromFriends() bool {
	return message.Flags&LPMessageFlagFriends != 0
}

// IsSpam will return true if it is spam.
func (message *LPMessage) IsSpam() bool {
	return message.Flags&LPMessageFlagSpam != 0
}

// Deleted will return true if the message was deleted (in the Recycle Bin).
func (message *LPMessage) Deleted() bool {
	return message.Flags&LPMessageFlagDeleted != 0
}

// Fixed will return true if the message has been scanned by the user for spam.
func (message *LPMessage) Fixed() bool {
	return message.Flags&LPMessageFlagFixed != 0
}

// ContainsMedia will return true if the message contains multimedia content.
func (message *LPMessage) ContainsMedia() bool {
	return message.Flags&LPMessageFlagMedia != 0
}

// IsHidden will return true if it is a welcome message from the community.
func (message *LPMessage) IsHidden() bool {
	return message.Flags&LPMessageFlagHidden != 0
}

// LPFriendNotification is a notification
// that a friend has become online or offline.
type LPFriendNotification struct {
	ID int64

	// If friend is online,
	// then Arg is equal to platform.
	//
	// If the friend offline, then
	// 0 - friend logout,
	// 1 - offline by timeout.
	Arg       int
	Timestamp Timestamp
	Code      int64
}

// Status returns event as a string.
func (friend *LPFriendNotification) Status() (status string) {
	switch friend.Code {
	case LPCodeFriendOnline:
		status = "Online"
	case LPCodeFriendOffline:
		status = "Offline"
	default:
		status = "Undefined event"
	}

	return
}

func (friend *LPFriendNotification) String() string {
	return fmt.Sprintf("Friend (%d) was %s at %s", friend.ID, friend.Status(), friend.Timestamp)
}

// LPAnswer is response from long poll server.
type LPAnswer struct {
	Failed    int64           `json:"failed"`
	Timestamp Timestamp       `json:"ts"`
	Updates   [][]interface{} `json:"updates"`
}

// LPChan allows to receive new LPUpdate.
type LPChan <-chan LPUpdate

// InitLongPoll establishes a new connection
// to long poll server.
func (client *Client) InitLongPoll(needPts int, lpVersion int) *Error {
	values := url.Values{}
	values.Add("need_pts", strconv.FormatInt(int64(needPts), 10))
	values.Add("lp_version", strconv.FormatInt(int64(lpVersion), 10))

	res, err := client.Do(NewRequest("messages.getLongPollServer", "", values))
	if err != nil {
		return err
	}

	client.LongPoll = new(LongPoll)
	if err := res.To(&client.LongPoll); err != nil {
		return NewError(ErrBadCode, err.Error())
	}

	u, error := url.Parse(client.LongPoll.Host)
	if error != nil {
		return NewError(ErrBadCode, error.Error())
	}

	client.LongPoll.Host = u.Host
	client.LongPoll.Path = u.Path
	client.LongPoll.LPVersion = lpVersion
	client.LongPoll.NeedPts = needPts

	return nil
}

// LPConfig stores data to connect to long poll server.
type LPConfig struct {
	Wait int
	Mode int
}

// GetLPAnswer makes a query with parameters
// from LPConfig to long poll server
// and returns a LPAnswer in case of success.
func (client *Client) GetLPAnswer(config LPConfig) (LPAnswer, error) {
	if client.apiClient == nil {
		return LPAnswer{}, errors.New(ErrApiClientNotFound)
	}

	if client.LongPoll == nil {
		return LPAnswer{}, errors.New("A long poll was not initialized")
	}

	values := url.Values{}
	values.Add("act", "a_check")
	values.Add("key", client.LongPoll.Key)
	values.Add("ts", strconv.FormatInt(int64(client.LongPoll.Timestamp), 10))
	values.Add("wait", strconv.FormatInt(int64(config.Wait), 10))
	values.Add("mode", strconv.FormatInt(int64(config.Mode), 10))
	values.Add("version", strconv.FormatInt(int64(client.LongPoll.LPVersion), 10))

	if client.apiClient.Log {
		client.apiClient.Logger.Printf("Request: %s", NewRequest("getLongPoll", "", values).JS())
	}

	u := url.URL{}
	u.Host = client.LongPoll.Host
	u.Path = client.LongPoll.Path
	u.Scheme = "https"
	u.RawQuery = values.Encode()

	req, err := http.NewRequest("GET", u.String(), nil)
	if err != nil {
		return LPAnswer{}, err
	}

	res, err := client.apiClient.httpClient.Do(req)
	if err != nil {
		client.apiClient.logPrintf("Response error: %s", err.Error())
		return LPAnswer{}, err
	}

	var reader io.Reader
	reader = res.Body

	if client.apiClient.Log {
		b, err := ioutil.ReadAll(res.Body)
		if err != nil {
			panic(err)
		}

		client.apiClient.Logger.Printf("Response: %s", string(b))
		reader = bytes.NewReader(b)
	}

	if res.StatusCode != http.StatusOK {
		client.apiClient.Logger.Printf("Response error: %s", res.Status)
		return LPAnswer{}, errors.New(res.Status)
	}

	var answer LPAnswer
	if err = json.NewDecoder(reader).Decode(&answer); err != nil {
		return LPAnswer{}, err
	}

	return answer, nil
}

// GetLPUpdates makes a query with parameters
// from LPConfig to long poll server
// and returns array LPUpdate in case of success.
func (client *Client) GetLPUpdates(config LPConfig) ([]LPUpdate, error) {
	answer, err := client.GetLPAnswer(config)
	if err != nil {
		return []LPUpdate{}, err
	}

	var LPUpdates []LPUpdate

	switch answer.Failed {
	case 0:
		for i := len(answer.Updates) - 1; i >= 0; i-- {
			var LPUpdate LPUpdate
			LPUpdate.Update = answer.Updates[i]
			if err := LPUpdate.UnmarshalUpdate(config.Mode); err != nil {
				client.apiClient.logPrintf("%s", err.Error())
			}

			LPUpdates = append(LPUpdates, LPUpdate)
		}

		client.LongPoll.Timestamp = answer.Timestamp
		return LPUpdates, nil
	case 1:
		client.LongPoll.Timestamp = answer.Timestamp
		client.apiClient.logPrintf("Timestamp updated")
	case 2, 3:
		if err := client.InitLongPoll(client.LongPoll.NeedPts, client.LongPoll.LPVersion); err != nil {
			client.apiClient.logPrintf("Long poll update error: %s", err.Error())

			return []LPUpdate{}, err
		}

		client.apiClient.logPrintf("Long poll config updated")
	}

	return []LPUpdate{}, nil
}

// GetLPUpdatesChan makes a query with parameters
// from LPConfig to long poll server
// and returns LPChan in case of success.
func (client *Client) GetLPUpdatesChan(bufSize int, config LPConfig) (LPChan, *bool, error) {
	ch := make(chan LPUpdate, bufSize)
	run := true

	go func() {
		for run {
			updates, err := client.GetLPUpdates(config)
			if err != nil {
				log.Print("Failed to get updates, retrying in 3 seconds...")
				time.Sleep(time.Second * 3)

				continue
			}

			for _, u := range updates {
				ch <- u
			}
		}

		close(ch)
	}()

	return ch, &run, nil
}
<|endoftext|>


<commit_before>/*
Copyright 2015 ENDOH takanao.
<https://github.com/MiCHiLU/go-lru-cache-stats>

Copyright 2012 Google Inc.
<https://github.com/golang/groupcache>

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

     http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// Tests for groupcache.

package lru

import (
	"errors"
	"fmt"
	"hash/crc32"
	"math/rand"
	"reflect"
	"sync"
	"testing"
	"time"

	"github.com/golang/protobuf/proto"

	pb "github.com/golang/groupcache/groupcachepb"
	testpb "github.com/golang/groupcache/testpb"
)

var (
	once                    sync.Once
	stringGroup, protoGroup Getter

	stringc = make(chan string)

	dummyCtx Context

	// cacheFills is the number of times stringGroup or
	// protoGroup's Getter have been called. Read using the
	// cacheFills function.
	cacheFills AtomicInt
)

const (
	stringGroupName = "string-group"
	protoGroupName  = "proto-group"
	testMessageType = "google3/net/groupcache/go/test_proto.TestMessage"
	fromChan        = "from-chan"
	cacheSize       = 1 << 20
)

func testSetup() {
	stringGroup = NewGroup(stringGroupName, cacheSize, GetterFunc(func(_ Context, key string, dest Sink) error {
		if key == fromChan {
			key = <-stringc
		}
		cacheFills.Add(1)
		return dest.SetString("ECHO:" + key)
	}))

	protoGroup = NewGroup(protoGroupName, cacheSize, GetterFunc(func(_ Context, key string, dest Sink) error {
		if key == fromChan {
			key = <-stringc
		}
		cacheFills.Add(1)
		return dest.SetProto(&testpb.TestMessage{
			Name: proto.String("ECHO:" + key),
			City: proto.String("SOME-CITY"),
		})
	}))
}

// tests that a Getter's Get method is only called once with two
// outstanding callers.  This is the string variant.
func TestGetDupSuppressString(t *testing.T) {
	once.Do(testSetup)
	// Start two getters. The first should block (waiting reading
	// from stringc) and the second should latch on to the first
	// one.
	resc := make(chan string, 2)
	for i := 0; i < 2; i++ {
		go func() {
			var s string
			if err := stringGroup.Get(dummyCtx, fromChan, StringSink(&s)); err != nil {
				resc <- "ERROR:" + err.Error()
				return
			}
			resc <- s
		}()
	}

	// Wait a bit so both goroutines get merged together via
	// singleflight.
	// TODO(bradfitz): decide whether there are any non-offensive
	// debug/test hooks that could be added to singleflight to
	// make a sleep here unnecessary.
	time.Sleep(250 * time.Millisecond)

	// Unblock the first getter, which should unblock the second
	// as well.
	stringc <- "foo"

	for i := 0; i < 2; i++ {
		select {
		case v := <-resc:
			if v != "ECHO:foo" {
				t.Errorf("got %q; want %q", v, "ECHO:foo")
			}
		case <-time.After(5 * time.Second):
			t.Errorf("timeout waiting on getter #%d of 2", i+1)
		}
	}
}

// tests that a Getter's Get method is only called once with two
// outstanding callers.  This is the proto variant.
func TestGetDupSuppressProto(t *testing.T) {
	once.Do(testSetup)
	// Start two getters. The first should block (waiting reading
	// from stringc) and the second should latch on to the first
	// one.
	resc := make(chan *testpb.TestMessage, 2)
	for i := 0; i < 2; i++ {
		go func() {
			tm := new(testpb.TestMessage)
			if err := protoGroup.Get(dummyCtx, fromChan, ProtoSink(tm)); err != nil {
				tm.Name = proto.String("ERROR:" + err.Error())
			}
			resc <- tm
		}()
	}

	// Wait a bit so both goroutines get merged together via
	// singleflight.
	// TODO(bradfitz): decide whether there are any non-offensive
	// debug/test hooks that could be added to singleflight to
	// make a sleep here unnecessary.
	time.Sleep(250 * time.Millisecond)

	// Unblock the first getter, which should unblock the second
	// as well.
	stringc <- "Fluffy"
	want := &testpb.TestMessage{
		Name: proto.String("ECHO:Fluffy"),
		City: proto.String("SOME-CITY"),
	}
	for i := 0; i < 2; i++ {
		select {
		case v := <-resc:
			if !reflect.DeepEqual(v, want) {
				t.Errorf(" Got: %v\nWant: %v", proto.CompactTextString(v), proto.CompactTextString(want))
			}
		case <-time.After(5 * time.Second):
			t.Errorf("timeout waiting on getter #%d of 2", i+1)
		}
	}
}

func countFills(f func()) int64 {
	fills0 := cacheFills.Get()
	f()
	return cacheFills.Get() - fills0
}

func TestCaching(t *testing.T) {
	once.Do(testSetup)
	fills := countFills(func() {
		for i := 0; i < 10; i++ {
			var s string
			if err := stringGroup.Get(dummyCtx, "TestCaching-key", StringSink(&s)); err != nil {
				t.Fatal(err)
			}
		}
	})
	if fills != 1 {
		t.Errorf("expected 1 cache fill; got %d", fills)
	}
}

func TestCacheEviction(t *testing.T) {
	once.Do(testSetup)
	testKey := "TestCacheEviction-key"
	getTestKey := func() {
		var res string
		for i := 0; i < 10; i++ {
			if err := stringGroup.Get(dummyCtx, testKey, StringSink(&res)); err != nil {
				t.Fatal(err)
			}
		}
	}
	fills := countFills(getTestKey)
	if fills != 1 {
		t.Fatalf("expected 1 cache fill; got %d", fills)
	}

	g := stringGroup.(*Group)
	evict0 := g.mainCache.nevict

	// Trash the cache with other keys.
	var bytesFlooded int64
	// cacheSize/len(testKey) is approximate
	for bytesFlooded < cacheSize+1024 {
		var res string
		key := fmt.Sprintf("dummy-key-%d", bytesFlooded)
		stringGroup.Get(dummyCtx, key, StringSink(&res))
		bytesFlooded += int64(len(key) + len(res))
	}
	evicts := g.mainCache.nevict - evict0
	if evicts <= 0 {
		t.Errorf("evicts = %v; want more than 0", evicts)
	}

	// Test that the key is gone.
	fills = countFills(getTestKey)
	if fills != 1 {
		t.Fatalf("expected 1 cache fill after cache trashing; got %d", fills)
	}
}

type fakePeer struct {
	hits int
	fail bool
}

func (p *fakePeer) Get(_ Context, in *pb.GetRequest, out *pb.GetResponse) error {
	p.hits++
	if p.fail {
		return errors.New("simulated error from peer")
	}
	out.Value = []byte("got:" + in.GetKey())
	return nil
}

type fakePeers []ProtoGetter

func (p fakePeers) PickPeer(key string) (peer ProtoGetter, ok bool) {
	if len(p) == 0 {
		return
	}
	n := crc32.Checksum([]byte(key), crc32.IEEETable) % uint32(len(p))
	return p[n], p[n] != nil
}

// tests that peers (virtual, in-process) are hit, and how much.
func TestPeers(t *testing.T) {
	once.Do(testSetup)
	rand.Seed(123)
	peer0 := &fakePeer{}
	peer1 := &fakePeer{}
	peer2 := &fakePeer{}
	peerList := fakePeers([]ProtoGetter{peer0, peer1, peer2, nil})
	const cacheSize = 0 // disabled
	localHits := 0
	getter := func(_ Context, key string, dest Sink) error {
		localHits++
		return dest.SetString("got:" + key)
	}
	testGroup := newGroup("TestPeers-group", cacheSize, GetterFunc(getter))
	run := func(name string, n int, wantSummary string) {
		// Reset counters
		localHits = 0
		for _, p := range []*fakePeer{peer0, peer1, peer2} {
			p.hits = 0
		}

		for i := 0; i < n; i++ {
			key := fmt.Sprintf("key-%d", i)
			want := "got:" + key
			var got string
			err := testGroup.Get(dummyCtx, key, StringSink(&got))
			if err != nil {
				t.Errorf("%s: error on key %q: %v", name, key, err)
				continue
			}
			if got != want {
				t.Errorf("%s: for key %q, got %q; want %q", name, key, got, want)
			}
		}
		summary := func() string {
			return fmt.Sprintf("localHits = %d, peers = %d %d %d", localHits, peer0.hits, peer1.hits, peer2.hits)
		}
		if got := summary(); got != wantSummary {
			t.Errorf("%s: got %q; want %q", name, got, wantSummary)
		}
	}
	resetCacheSize := func(maxBytes int64) {
		g := testGroup
		g.cacheBytes = maxBytes
		g.mainCache = cache{}
	}

	// Base case; peers all up, with no problems.
	resetCacheSize(1 << 20)
	run("base", 200, "localHits = 200, peers = 0 0 0")

	// Verify cache was hit.  All localHits are gone, and some of
	// the peer hits (the ones randomly selected to be maybe hot)
	run("cached_base", 200, "localHits = 0, peers = 0 0 0")
	resetCacheSize(0)

	// With one of the peers being down.
	// TODO(bradfitz): on a peer number being unavailable, the
	// consistent hashing should maybe keep trying others to
	// spread the load out. Currently it fails back to local
	// execution if the first consistent-hash slot is unavailable.
	peerList[0] = nil
	run("one_peer_down", 200, "localHits = 200, peers = 0 0 0")

	// Failing peer
	peerList[0] = peer0
	peer0.fail = true
	run("peer0_failing", 200, "localHits = 200, peers = 0 0 0")
}

func TestTruncatingByteSliceTarget(t *testing.T) {
	var buf [100]byte
	s := buf[:]
	if err := stringGroup.Get(dummyCtx, "short", TruncatingByteSliceSink(&s)); err != nil {
		t.Fatal(err)
	}
	if want := "ECHO:short"; string(s) != want {
		t.Errorf("short key got %q; want %q", s, want)
	}

	s = buf[:6]
	if err := stringGroup.Get(dummyCtx, "truncated", TruncatingByteSliceSink(&s)); err != nil {
		t.Fatal(err)
	}
	if want := "ECHO:t"; string(s) != want {
		t.Errorf("truncated key got %q; want %q", s, want)
	}
}

func TestAllocatingByteSliceTarget(t *testing.T) {
	var dst []byte
	sink := AllocatingByteSliceSink(&dst)

	inBytes := []byte("some bytes")
	sink.SetBytes(inBytes)
	if want := "some bytes"; string(dst) != want {
		t.Errorf("SetBytes resulted in %q; want %q", dst, want)
	}
	v, err := sink.view()
	if err != nil {
		t.Fatalf("view after SetBytes failed: %v", err)
	}
	if &inBytes[0] == &dst[0] {
		t.Error("inBytes and dst share memory")
	}
	if &inBytes[0] == &v.b[0] {
		t.Error("inBytes and view share memory")
	}
	if &dst[0] == &v.b[0] {
		t.Error("dst and view share memory")
	}
}

// TODO(bradfitz): port the Google-internal full integration test into here,
// using HTTP requests instead of our RPC system.
<commit_msg>fix tests<commit_after>/*
Copyright 2015 ENDOH takanao.
<https://github.com/MiCHiLU/go-lru-cache-stats>

Copyright 2012 Google Inc.
<https://github.com/golang/groupcache>

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

     http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// Tests for groupcache.

package lru

import (
	"errors"
	"fmt"
	"hash/crc32"
	"math/rand"
	"reflect"
	"sync"
	"testing"
	"time"

	"github.com/golang/protobuf/proto"

	pb "github.com/golang/groupcache/groupcachepb"
	testpb "github.com/golang/groupcache/testpb"
)

var (
	once                    sync.Once
	stringGroup, protoGroup Getter

	stringc = make(chan string)

	dummyCtx Context

	// cacheFills is the number of times stringGroup or
	// protoGroup's Getter have been called. Read using the
	// cacheFills function.
	cacheFills AtomicInt
)

const (
	stringGroupName = "string-group"
	protoGroupName  = "proto-group"
	testMessageType = "google3/net/groupcache/go/test_proto.TestMessage"
	fromChan        = "from-chan"
	cacheSize       = 1 << 20
	stats           = true
)

func testSetup() {
	stringGroup = (*NewGroup(stringGroupName, cacheSize, GetterFunc(func(_ Context, key string, dest Sink) error {
		if key == fromChan {
			key = <-stringc
		}
		cacheFills.Add(1)
		return dest.SetString("ECHO:" + key)
	}), stats))

	protoGroup = (*NewGroup(protoGroupName, cacheSize, GetterFunc(func(_ Context, key string, dest Sink) error {
		if key == fromChan {
			key = <-stringc
		}
		cacheFills.Add(1)
		return dest.SetProto(&testpb.TestMessage{
			Name: proto.String("ECHO:" + key),
			City: proto.String("SOME-CITY"),
		})
	}), stats))
}

// tests that a Getter's Get method is only called once with two
// outstanding callers.  This is the string variant.
func TestGetDupSuppressString(t *testing.T) {
	once.Do(testSetup)
	// Start two getters. The first should block (waiting reading
	// from stringc) and the second should latch on to the first
	// one.
	resc := make(chan string, 2)
	for i := 0; i < 2; i++ {
		go func() {
			var s string
			if err := stringGroup.Get(dummyCtx, fromChan, StringSink(&s)); err != nil {
				resc <- "ERROR:" + err.Error()
				return
			}
			resc <- s
		}()
	}

	// Wait a bit so both goroutines get merged together via
	// singleflight.
	// TODO(bradfitz): decide whether there are any non-offensive
	// debug/test hooks that could be added to singleflight to
	// make a sleep here unnecessary.
	time.Sleep(250 * time.Millisecond)

	// Unblock the first getter, which should unblock the second
	// as well.
	stringc <- "foo"

	for i := 0; i < 2; i++ {
		select {
		case v := <-resc:
			if v != "ECHO:foo" {
				t.Errorf("got %q; want %q", v, "ECHO:foo")
			}
		case <-time.After(5 * time.Second):
			t.Errorf("timeout waiting on getter #%d of 2", i+1)
		}
	}
}

// tests that a Getter's Get method is only called once with two
// outstanding callers.  This is the proto variant.
func TestGetDupSuppressProto(t *testing.T) {
	once.Do(testSetup)
	// Start two getters. The first should block (waiting reading
	// from stringc) and the second should latch on to the first
	// one.
	resc := make(chan *testpb.TestMessage, 2)
	for i := 0; i < 2; i++ {
		go func() {
			tm := new(testpb.TestMessage)
			if err := protoGroup.Get(dummyCtx, fromChan, ProtoSink(tm)); err != nil {
				tm.Name = proto.String("ERROR:" + err.Error())
			}
			resc <- tm
		}()
	}

	// Wait a bit so both goroutines get merged together via
	// singleflight.
	// TODO(bradfitz): decide whether there are any non-offensive
	// debug/test hooks that could be added to singleflight to
	// make a sleep here unnecessary.
	time.Sleep(250 * time.Millisecond)

	// Unblock the first getter, which should unblock the second
	// as well.
	stringc <- "Fluffy"
	want := &testpb.TestMessage{
		Name: proto.String("ECHO:Fluffy"),
		City: proto.String("SOME-CITY"),
	}
	for i := 0; i < 2; i++ {
		select {
		case v := <-resc:
			if !reflect.DeepEqual(v, want) {
				t.Errorf(" Got: %v\nWant: %v", proto.CompactTextString(v), proto.CompactTextString(want))
			}
		case <-time.After(5 * time.Second):
			t.Errorf("timeout waiting on getter #%d of 2", i+1)
		}
	}
}

func countFills(f func()) int64 {
	fills0 := cacheFills.Get()
	f()
	return cacheFills.Get() - fills0
}

func TestCaching(t *testing.T) {
	once.Do(testSetup)
	fills := countFills(func() {
		for i := 0; i < 10; i++ {
			var s string
			if err := stringGroup.Get(dummyCtx, "TestCaching-key", StringSink(&s)); err != nil {
				t.Fatal(err)
			}
		}
	})
	if fills != 1 {
		t.Errorf("expected 1 cache fill; got %d", fills)
	}
}

func TestCacheEviction(t *testing.T) {
	once.Do(testSetup)
	testKey := "TestCacheEviction-key"
	getTestKey := func() {
		var res string
		for i := 0; i < 10; i++ {
			if err := stringGroup.Get(dummyCtx, testKey, StringSink(&res)); err != nil {
				t.Fatal(err)
			}
		}
	}
	fills := countFills(getTestKey)
	if fills != 1 {
		t.Fatalf("expected 1 cache fill; got %d", fills)
	}

	g := stringGroup.(*GroupWithStats)
	evict0 := g.mainCache.nevict

	// Trash the cache with other keys.
	var bytesFlooded int64
	// cacheSize/len(testKey) is approximate
	for bytesFlooded < cacheSize+1024 {
		var res string
		key := fmt.Sprintf("dummy-key-%d", bytesFlooded)
		stringGroup.Get(dummyCtx, key, StringSink(&res))
		bytesFlooded += int64(len(key) + len(res))
	}
	evicts := g.mainCache.nevict - evict0
	if evicts <= 0 {
		t.Errorf("evicts = %v; want more than 0", evicts)
	}

	// Test that the key is gone.
	fills = countFills(getTestKey)
	if fills != 1 {
		t.Fatalf("expected 1 cache fill after cache trashing; got %d", fills)
	}
}

type fakePeer struct {
	hits int
	fail bool
}

func (p *fakePeer) Get(_ Context, in *pb.GetRequest, out *pb.GetResponse) error {
	p.hits++
	if p.fail {
		return errors.New("simulated error from peer")
	}
	out.Value = []byte("got:" + in.GetKey())
	return nil
}

type fakePeers []ProtoGetter

func (p fakePeers) PickPeer(key string) (peer ProtoGetter, ok bool) {
	if len(p) == 0 {
		return
	}
	n := crc32.Checksum([]byte(key), crc32.IEEETable) % uint32(len(p))
	return p[n], p[n] != nil
}

// tests that peers (virtual, in-process) are hit, and how much.
func TestPeers(t *testing.T) {
	once.Do(testSetup)
	rand.Seed(123)
	peer0 := &fakePeer{}
	peer1 := &fakePeer{}
	peer2 := &fakePeer{}
	peerList := fakePeers([]ProtoGetter{peer0, peer1, peer2, nil})
	const cacheSize = 0 // disabled
	localHits := 0
	getter := func(_ Context, key string, dest Sink) error {
		localHits++
		return dest.SetString("got:" + key)
	}
	var testGroup GroupInterface
	run := func(name string, n int, wantSummary string) {
		// Reset counters
		localHits = 0
		for _, p := range []*fakePeer{peer0, peer1, peer2} {
			p.hits = 0
		}

		for i := 0; i < n; i++ {
			key := fmt.Sprintf("key-%d", i)
			want := "got:" + key
			var got string
			err := testGroup.Get(dummyCtx, key, StringSink(&got))
			if err != nil {
				t.Errorf("%s: error on key %q: %v", name, key, err)
				continue
			}
			if got != want {
				t.Errorf("%s: for key %q, got %q; want %q", name, key, got, want)
			}
		}
		summary := func() string {
			return fmt.Sprintf("localHits = %d, peers = %d %d %d", localHits, peer0.hits, peer1.hits, peer2.hits)
		}
		if got := summary(); got != wantSummary {
			t.Errorf("%s: got %q; want %q", name, got, wantSummary)
		}
	}
	resetCacheSize := func(maxBytes int64) {
		testGroup = (*newGroup(fmt.Sprintf("TestPeers-group-%s", maxBytes), maxBytes, GetterFunc(getter), stats))
	}

	// Base case; peers all up, with no problems.
	resetCacheSize(1 << 20)
	run("base", 200, "localHits = 200, peers = 0 0 0")

	// Verify cache was hit.  All localHits are gone, and some of
	// the peer hits (the ones randomly selected to be maybe hot)
	run("cached_base", 200, "localHits = 0, peers = 0 0 0")
	resetCacheSize(0)

	// With one of the peers being down.
	// TODO(bradfitz): on a peer number being unavailable, the
	// consistent hashing should maybe keep trying others to
	// spread the load out. Currently it fails back to local
	// execution if the first consistent-hash slot is unavailable.
	peerList[0] = nil
	run("one_peer_down", 200, "localHits = 200, peers = 0 0 0")

	// Failing peer
	peerList[0] = peer0
	peer0.fail = true
	run("peer0_failing", 200, "localHits = 200, peers = 0 0 0")
}

func TestTruncatingByteSliceTarget(t *testing.T) {
	var buf [100]byte
	s := buf[:]
	if err := stringGroup.Get(dummyCtx, "short", TruncatingByteSliceSink(&s)); err != nil {
		t.Fatal(err)
	}
	if want := "ECHO:short"; string(s) != want {
		t.Errorf("short key got %q; want %q", s, want)
	}

	s = buf[:6]
	if err := stringGroup.Get(dummyCtx, "truncated", TruncatingByteSliceSink(&s)); err != nil {
		t.Fatal(err)
	}
	if want := "ECHO:t"; string(s) != want {
		t.Errorf("truncated key got %q; want %q", s, want)
	}
}

func TestAllocatingByteSliceTarget(t *testing.T) {
	var dst []byte
	sink := AllocatingByteSliceSink(&dst)

	inBytes := []byte("some bytes")
	sink.SetBytes(inBytes)
	if want := "some bytes"; string(dst) != want {
		t.Errorf("SetBytes resulted in %q; want %q", dst, want)
	}
	v, err := sink.view()
	if err != nil {
		t.Fatalf("view after SetBytes failed: %v", err)
	}
	if &inBytes[0] == &dst[0] {
		t.Error("inBytes and dst share memory")
	}
	if &inBytes[0] == &v.b[0] {
		t.Error("inBytes and view share memory")
	}
	if &dst[0] == &v.b[0] {
		t.Error("dst and view share memory")
	}
}

// TODO(bradfitz): port the Google-internal full integration test into here,
// using HTTP requests instead of our RPC system.
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"github.com/mitchellh/packer/packer"
	"time"
)

type Builder struct {
	config config
	cancel chan int
	done   bool
	ui     packer.Ui
}

type config struct {
	period   uint
	duration uint
}

func (b *Builder) Prepare(raws ...interface{}) ([]string, error) {
	b.config.period = 1
	b.config.duration = 5

	for _, e := range raws {
		var m map[string]interface{} = *e.(*map[string]interface{})

		if v, ok := m["period"]; ok {
			b.config.period = uint(v.(float64))
		}

		if v, ok := m["duration"]; ok {
			b.config.duration = uint(v.(float64))
		}
	}

	b.cancel = make(chan int, 1)

	return nil, nil
}

func (b *Builder) Run(ui packer.Ui, _ packer.Hook, _ packer.Cache) (packer.Artifact, error) {
	ui.Say(fmt.Sprintf("Running(%d, %d)...", b.config.period, b.config.duration))

	b.ui = ui

	tick := time.Tick(time.Duration(b.config.period) * time.Second)
	stop := time.After(time.Duration(b.config.duration) * time.Second)
	start := time.Now()

	for !b.done {
		select {
		case <-tick:
			ui.Say(fmt.Sprintf("Building... %d", uint(time.Since(start).Seconds())))
		case <-stop:
			ui.Say("Done! Stopping...")
			b.done = true
		case <-b.cancel:
			ui.Say("Cancelled! Stopping...")
			b.done = true
		}
	}

	ui.Say("Stopped!")
	return nil, nil
}

func (b *Builder) Cancel() {
	if b.done {
		return
	}

	b.ui.Say("Cancelling...")
	b.cancel <- 1
}
<commit_msg>Update Run; improve output message<commit_after>package main

import (
	"fmt"
	"github.com/mitchellh/packer/packer"
	"time"
)

type Builder struct {
	config config
	cancel chan int
	done   bool
	ui     packer.Ui
}

type config struct {
	period   uint
	duration uint
}

func (b *Builder) Prepare(raws ...interface{}) ([]string, error) {
	b.config.period = 1
	b.config.duration = 5

	for _, e := range raws {
		var m map[string]interface{} = *e.(*map[string]interface{})

		if v, ok := m["period"]; ok {
			b.config.period = uint(v.(float64))
		}

		if v, ok := m["duration"]; ok {
			b.config.duration = uint(v.(float64))
		}
	}

	b.cancel = make(chan int, 1)

	return nil, nil
}

func (b *Builder) Run(ui packer.Ui, _ packer.Hook, _ packer.Cache) (packer.Artifact, error) {
	ui.Say(fmt.Sprintf("Running for %d second(s), ticking every %d second(s)...", b.config.duration, b.config.period))

	b.ui = ui

	tick := time.Tick(time.Duration(b.config.period) * time.Second)
	stop := time.After(time.Duration(b.config.duration) * time.Second)
	start := time.Now()

	for !b.done {
		select {
		case <-tick:
			ui.Say(fmt.Sprintf("Building... %d", uint(time.Since(start).Seconds())))
		case <-stop:
			ui.Say("Done! Stopping...")
			b.done = true
		case <-b.cancel:
			ui.Say("Cancelled! Stopping...")
			b.done = true
		}
	}

	ui.Say("Stopped!")
	return nil, nil
}

func (b *Builder) Cancel() {
	if b.done {
		return
	}

	b.ui.Say("Cancelling...")
	b.cancel <- 1
}
<|endoftext|>


<commit_before>//: ----------------------------------------------------------------------------
//: Copyright (C) 2017 Verizon.  All Rights Reserved.
//: All Rights Reserved
//:
//: file:    flow_counter.go
//: details: TODO
//: author:  Mehrdad Arshad Rad
//: date:    08/08/2018
//:
//: Licensed under the Apache License, Version 2.0 (the "License");
//: you may not use this file except in compliance with the License.
//: You may obtain a copy of the License at
//:
//:     http://www.apache.org/licenses/LICENSE-2.0
//:
//: Unless required by applicable law or agreed to in writing, software
//: distributed under the License is distributed on an "AS IS" BASIS,
//: WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//: See the License for the specific language governing permissions and
//: limitations under the License.
//: ----------------------------------------------------------------------------

package sflow

import "io"

const (
	// GenericInterfaceCounters is Generic interface counters - see RFC 2233
	SFGenericInterfaceCounters = 1

	// EthernetInterfaceCounters is Ethernet interface counters - see RFC 2358
	SFEthernetInterfaceCounters = 2

	// SFTokenRingInterfaceCounters is Token ring counters - see RFC 1748
	SFTokenRingInterfaceCounters = 3

	// SF100BaseVGInterfaceCounters is 100 BaseVG interface counters - see RFC 2020
	SF100BaseVGInterfaceCounters = 4

	// SFVLANCounters is VLAN counters
	SFVLANCounters = 5

	// SFProcessorCounters is processor counters
	SFProcessorCounters = 1001
)

// GenericInterfaceCounters represents Generic Interface Counters RFC2233
type GenericInterfaceCounters struct {
	Index               uint32
	Type                uint32
	Speed               uint64
	Direction           uint32
	Status              uint32
	InOctets            uint64
	InUnicastPackets    uint32
	InMulticastPackets  uint32
	InBroadcastPackets  uint32
	InDiscards          uint32
	InErrors            uint32
	InUnknownProtocols  uint32
	OutOctets           uint64
	OutUnicastPackets   uint32
	OutMulticastPackets uint32
	OutBroadcastPackets uint32
	OutDiscards         uint32
	OutErrors           uint32
	PromiscuousMode     uint32
}

// EthernetInterfaceCounters represents Ethernet Interface Counters RFC2358
type EthernetInterfaceCounters struct {
	AlignmentErrors           uint32
	FCSErrors                 uint32
	SingleCollisionFrames     uint32
	MultipleCollisionFrames   uint32
	SQETestErrors             uint32
	DeferredTransmissions     uint32
	LateCollisions            uint32
	ExcessiveCollisions       uint32
	InternalMACTransmitErrors uint32
	CarrierSenseErrors        uint32
	FrameTooLongs             uint32
	InternalMACReceiveErrors  uint32
	SymbolErrors              uint32
}

// TokenRingCounters represents Token Ring Counters - see RFC 1748
type TokenRingCounters struct {
	LineErrors         uint32
	BurstErrors        uint32
	ACErrors           uint32
	AbortTransErrors   uint32
	InternalErrors     uint32
	LostFrameErrors    uint32
	ReceiveCongestions uint32
	FrameCopiedErrors  uint32
	TokenErrors        uint32
	SoftErrors         uint32
	HardErrors         uint32
	SignalLoss         uint32
	TransmitBeacons    uint32
	Recoverys          uint32
	LobeWires          uint32
	Removes            uint32
	Singles            uint32
	FreqErrors         uint32
}

// VGCounters represents 100 BaseVG interface counters - see RFC 2020
type VGCounters struct {
	InHighPriorityFrames    uint32
	InHighPriorityOctets    uint64
	InNormPriorityFrames    uint32
	InNormPriorityOctets    uint64
	InIPMErrors             uint32
	InOversizeFrameErrors   uint32
	InDataErrors            uint32
	InNullAddressedFrames   uint32
	OutHighPriorityFrames   uint32
	OutHighPriorityOctets   uint64
	TransitionIntoTrainings uint32
	HCInHighPriorityOctets  uint64
	HCInNormPriorityOctets  uint64
	HCOutHighPriorityOctets uint64
}

// VlanCounters represents VLAN Counters
type VlanCounters struct {
	ID               uint32
	Octets           uint64
	UnicastPackets   uint32
	MulticastPackets uint32
	BroadcastPackets uint32
	Discards         uint32
}

// ProcessorCounters represents Processor Information
type ProcessorCounters struct {
	CPU5s       uint32
	CPU1m       uint32
	CPU5m       uint32
	TotalMemory uint64
	FreeMemory  uint64
}

type CounterSample struct {
	SequenceNo   uint32
	SourceIdType byte
	SourceIdIdx  uint32
	RecordsNo    uint32
	Records      map[string]Record
}

func decodeFlowCounter(r io.ReadSeeker) (*CounterSample, error) {
	var (
		cs          = new(CounterSample)
		rTypeFormat uint32
		rTypeLength uint32
		err         error
	)

	if err = cs.unmarshal(r); err != nil {
		return nil, err
	}

	cs.Records = make(map[string]Record)

	for i := uint32(0); i < cs.RecordsNo; i++ {
		if err = read(r, &rTypeFormat); err != nil {
			return nil, err
		}
		if err = read(r, &rTypeLength); err != nil {
			return nil, err
		}

		switch rTypeFormat {

		case SFGenericInterfaceCounters:
			d, err := decodeGenericIntCounters(r)
			if err != nil {
				return cs, err
			}
			cs.Records["GenInt"] = d
		case SFEthernetInterfaceCounters:
			d, err := decodeEthIntCounters(r)
			if err != nil {
				return cs, err
			}
			cs.Records["EthInt"] = d
		case SFVLANCounters:
			d, err := decodeVlanCounters(r)
			if err != nil {
				return cs, err
			}
			cs.Records["Vlan"] = d
		case SFProcessorCounters:
			d, err := decodedProcessorCounters(r)
			if err != nil {
				return cs, err
			}
			cs.Records["Proc"] = d
		default:
			r.Seek(int64(rTypeLength), 1)
		}
	}

	return cs, nil
}

func decodeGenericIntCounters(r io.Reader) (*GenericInterfaceCounters, error) {
	var gic = new(GenericInterfaceCounters)

	if err := gic.unmarshal(r); err != nil {
		return nil, err
	}

	return gic, nil
}

func (gic *GenericInterfaceCounters) unmarshal(r io.Reader) error {
	var err error

	fields := []interface{}{
		&gic.Index,
		&gic.Type,
		&gic.Speed,
		&gic.Direction,
		&gic.Status,
		&gic.InOctets,
		&gic.InUnicastPackets,
		&gic.InMulticastPackets,
		&gic.InBroadcastPackets,
		&gic.InDiscards,
		&gic.InErrors,
		&gic.InUnknownProtocols,
		&gic.OutOctets,
		&gic.OutUnicastPackets,
		&gic.OutMulticastPackets,
		&gic.OutBroadcastPackets,
		&gic.OutDiscards,
		&gic.OutErrors,
		&gic.PromiscuousMode,
	}

	for _, field := range fields {
		if err = read(r, field); err != nil {
			return err
		}
	}

	return nil
}
func decodeEthIntCounters(r io.Reader) (*EthernetInterfaceCounters, error) {
	var eic = new(EthernetInterfaceCounters)

	if err := eic.unmarshal(r); err != nil {
		return nil, err
	}

	return eic, nil
}

func (eic *EthernetInterfaceCounters) unmarshal(r io.Reader) error {
	var err error

	fields := []interface{}{
		&eic.AlignmentErrors,
		&eic.FCSErrors,
		&eic.SingleCollisionFrames,
		&eic.MultipleCollisionFrames,
		&eic.SQETestErrors,
		&eic.DeferredTransmissions,
		&eic.LateCollisions,
		&eic.ExcessiveCollisions,
		&eic.InternalMACTransmitErrors,
		&eic.CarrierSenseErrors,
		&eic.FrameTooLongs,
		&eic.InternalMACReceiveErrors,
		&eic.SymbolErrors,
	}

	for _, field := range fields {
		if err = read(r, field); err != nil {
			return err
		}
	}

	return nil
}
func decodeTokenRingCounters(r io.Reader) (*TokenRingCounters, error) {
	var tr = new(TokenRingCounters)

	if err := tr.unmarshal(r); err != nil {
		return nil, err
	}

	return tr, nil
}

func (tr *TokenRingCounters) unmarshal(r io.Reader) error {
	var err error

	fields := []interface{}{
		&tr.LineErrors,
		&tr.BurstErrors,
		&tr.ACErrors,
		&tr.AbortTransErrors,
		&tr.InternalErrors,
		&tr.LostFrameErrors,
		&tr.ReceiveCongestions,
		&tr.FrameCopiedErrors,
		&tr.TokenErrors,
		&tr.SoftErrors,
		&tr.HardErrors,
		&tr.SignalLoss,
		&tr.TransmitBeacons,
		&tr.Recoverys,
		&tr.LobeWires,
		&tr.Removes,
		&tr.Singles,
		&tr.FreqErrors,
	}

	for _, field := range fields {
		if err = read(r, field); err != nil {
			return err
		}
	}

	return nil
}

func decodeVGCounters(r io.Reader) (*VGCounters, error) {
	var vg = new(VGCounters)

	if err := vg.unmarshal(r); err != nil {
		return nil, err
	}

	return vg, nil
}

func (vg *VGCounters) unmarshal(r io.Reader) error {
	var err error

	fields := []interface{}{
		&vg.InHighPriorityFrames,
		&vg.InHighPriorityOctets,
		&vg.InNormPriorityFrames,
		&vg.InNormPriorityOctets,
		&vg.InIPMErrors,
		&vg.InOversizeFrameErrors,
		&vg.InDataErrors,
		&vg.InNullAddressedFrames,
		&vg.OutHighPriorityFrames,
		&vg.OutHighPriorityOctets,
		&vg.TransitionIntoTrainings,
		&vg.HCInHighPriorityOctets,
		&vg.HCInNormPriorityOctets,
		&vg.HCOutHighPriorityOctets,
	}

	for _, field := range fields {
		if err = read(r, field); err != nil {
			return err
		}
	}

	return nil
}

func decodeVlanCounters(r io.Reader) (*VlanCounters, error) {
	var vc = new(VlanCounters)

	if err := vc.unmarshal(r); err != nil {
		return nil, err
	}

	return vc, nil
}

func (vc *VlanCounters) unmarshal(r io.Reader) error {
	var err error
	fields := []interface{}{
		&vc.ID,
		&vc.Octets,
		&vc.UnicastPackets,
		&vc.MulticastPackets,
		&vc.BroadcastPackets,
		&vc.Discards,
	}

	for _, field := range fields {
		if err = read(r, field); err != nil {
			return err
		}
	}

	return nil
}

func decodedProcessorCounters(r io.Reader) (*ProcessorCounters, error) {
	var pc = new(ProcessorCounters)

	if err := pc.unmarshal(r); err != nil {
		return nil, err
	}

	return pc, nil
}

func (pc *ProcessorCounters) unmarshal(r io.Reader) error {
	var err error
	fields := []interface{}{
		&pc.CPU5s,
		&pc.CPU1m,
		&pc.CPU5m,
		&pc.TotalMemory,
		&pc.FreeMemory,
	}

	for _, field := range fields {
		if err = read(r, field); err != nil {
			return err
		}
	}

	return nil
}

func (cs *CounterSample) unmarshal(r io.Reader) error {

	var err error

	if err = read(r, &cs.SequenceNo); err != nil {
		return err
	}

	if err = read(r, &cs.SourceIdType); err != nil {
		return err
	}

	buf := make([]byte, 3)
	if err = read(r, &buf); err != nil {
		return err
	}
	cs.SourceIdIdx = uint32(buf[2]) | uint32(buf[1])<<8 | uint32(buf[0])<<16

	if err = read(r, &cs.RecordsNo); err != nil {
		return err
	}

	return nil
}
<commit_msg>add sflow counter token ring and SF100Base decoders<commit_after>//: ----------------------------------------------------------------------------
//: Copyright (C) 2017 Verizon.  All Rights Reserved.
//: All Rights Reserved
//:
//: file:    flow_counter.go
//: details: TODO
//: author:  Mehrdad Arshad Rad
//: date:    08/08/2018
//:
//: Licensed under the Apache License, Version 2.0 (the "License");
//: you may not use this file except in compliance with the License.
//: You may obtain a copy of the License at
//:
//:     http://www.apache.org/licenses/LICENSE-2.0
//:
//: Unless required by applicable law or agreed to in writing, software
//: distributed under the License is distributed on an "AS IS" BASIS,
//: WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//: See the License for the specific language governing permissions and
//: limitations under the License.
//: ----------------------------------------------------------------------------

package sflow

import "io"

const (
	// GenericInterfaceCounters is Generic interface counters - see RFC 2233
	SFGenericInterfaceCounters = 1

	// EthernetInterfaceCounters is Ethernet interface counters - see RFC 2358
	SFEthernetInterfaceCounters = 2

	// SFTokenRingInterfaceCounters is Token ring counters - see RFC 1748
	SFTokenRingInterfaceCounters = 3

	// SF100BaseVGInterfaceCounters is 100 BaseVG interface counters - see RFC 2020
	SF100BaseVGInterfaceCounters = 4

	// SFVLANCounters is VLAN counters
	SFVLANCounters = 5

	// SFProcessorCounters is processor counters
	SFProcessorCounters = 1001
)

// GenericInterfaceCounters represents Generic Interface Counters RFC2233
type GenericInterfaceCounters struct {
	Index               uint32
	Type                uint32
	Speed               uint64
	Direction           uint32
	Status              uint32
	InOctets            uint64
	InUnicastPackets    uint32
	InMulticastPackets  uint32
	InBroadcastPackets  uint32
	InDiscards          uint32
	InErrors            uint32
	InUnknownProtocols  uint32
	OutOctets           uint64
	OutUnicastPackets   uint32
	OutMulticastPackets uint32
	OutBroadcastPackets uint32
	OutDiscards         uint32
	OutErrors           uint32
	PromiscuousMode     uint32
}

// EthernetInterfaceCounters represents Ethernet Interface Counters RFC2358
type EthernetInterfaceCounters struct {
	AlignmentErrors           uint32
	FCSErrors                 uint32
	SingleCollisionFrames     uint32
	MultipleCollisionFrames   uint32
	SQETestErrors             uint32
	DeferredTransmissions     uint32
	LateCollisions            uint32
	ExcessiveCollisions       uint32
	InternalMACTransmitErrors uint32
	CarrierSenseErrors        uint32
	FrameTooLongs             uint32
	InternalMACReceiveErrors  uint32
	SymbolErrors              uint32
}

// TokenRingCounters represents Token Ring Counters - see RFC 1748
type TokenRingCounters struct {
	LineErrors         uint32
	BurstErrors        uint32
	ACErrors           uint32
	AbortTransErrors   uint32
	InternalErrors     uint32
	LostFrameErrors    uint32
	ReceiveCongestions uint32
	FrameCopiedErrors  uint32
	TokenErrors        uint32
	SoftErrors         uint32
	HardErrors         uint32
	SignalLoss         uint32
	TransmitBeacons    uint32
	Recoverys          uint32
	LobeWires          uint32
	Removes            uint32
	Singles            uint32
	FreqErrors         uint32
}

// VGCounters represents 100 BaseVG interface counters - see RFC 2020
type VGCounters struct {
	InHighPriorityFrames    uint32
	InHighPriorityOctets    uint64
	InNormPriorityFrames    uint32
	InNormPriorityOctets    uint64
	InIPMErrors             uint32
	InOversizeFrameErrors   uint32
	InDataErrors            uint32
	InNullAddressedFrames   uint32
	OutHighPriorityFrames   uint32
	OutHighPriorityOctets   uint64
	TransitionIntoTrainings uint32
	HCInHighPriorityOctets  uint64
	HCInNormPriorityOctets  uint64
	HCOutHighPriorityOctets uint64
}

// VlanCounters represents VLAN Counters
type VlanCounters struct {
	ID               uint32
	Octets           uint64
	UnicastPackets   uint32
	MulticastPackets uint32
	BroadcastPackets uint32
	Discards         uint32
}

// ProcessorCounters represents Processor Information
type ProcessorCounters struct {
	CPU5s       uint32
	CPU1m       uint32
	CPU5m       uint32
	TotalMemory uint64
	FreeMemory  uint64
}

type CounterSample struct {
	SequenceNo   uint32
	SourceIdType byte
	SourceIdIdx  uint32
	RecordsNo    uint32
	Records      map[string]Record
}

func decodeFlowCounter(r io.ReadSeeker) (*CounterSample, error) {
	var (
		cs          = new(CounterSample)
		rTypeFormat uint32
		rTypeLength uint32
		err         error
	)

	if err = cs.unmarshal(r); err != nil {
		return nil, err
	}

	cs.Records = make(map[string]Record)

	for i := uint32(0); i < cs.RecordsNo; i++ {
		if err = read(r, &rTypeFormat); err != nil {
			return nil, err
		}
		if err = read(r, &rTypeLength); err != nil {
			return nil, err
		}

		switch rTypeFormat {

		case SFGenericInterfaceCounters:
			d, err := decodeGenericIntCounters(r)
			if err != nil {
				return cs, err
			}
			cs.Records["GenInt"] = d
		case SFEthernetInterfaceCounters:
			d, err := decodeEthIntCounters(r)
			if err != nil {
				return cs, err
			}
			cs.Records["EthInt"] = d
		case SFTokenRingInterfaceCounters:
			d, err := decodeTokenRingCounters(r)
			if err != nil {
				return cs, err
			}
			cs.Records["TRInt"] = d
		case SF100BaseVGInterfaceCounters:
			d, err := decodeVGCounters(r)
			if err != nil {
				return cs, err
			}
			cs.Records["VGInt"] = d
		case SFVLANCounters:
			d, err := decodeVlanCounters(r)
			if err != nil {
				return cs, err
			}
			cs.Records["Vlan"] = d
		case SFProcessorCounters:
			d, err := decodedProcessorCounters(r)
			if err != nil {
				return cs, err
			}
			cs.Records["Proc"] = d
		default:
			r.Seek(int64(rTypeLength), 1)
		}
	}

	return cs, nil
}

func decodeGenericIntCounters(r io.Reader) (*GenericInterfaceCounters, error) {
	var gic = new(GenericInterfaceCounters)

	if err := gic.unmarshal(r); err != nil {
		return nil, err
	}

	return gic, nil
}

func (gic *GenericInterfaceCounters) unmarshal(r io.Reader) error {
	var err error

	fields := []interface{}{
		&gic.Index,
		&gic.Type,
		&gic.Speed,
		&gic.Direction,
		&gic.Status,
		&gic.InOctets,
		&gic.InUnicastPackets,
		&gic.InMulticastPackets,
		&gic.InBroadcastPackets,
		&gic.InDiscards,
		&gic.InErrors,
		&gic.InUnknownProtocols,
		&gic.OutOctets,
		&gic.OutUnicastPackets,
		&gic.OutMulticastPackets,
		&gic.OutBroadcastPackets,
		&gic.OutDiscards,
		&gic.OutErrors,
		&gic.PromiscuousMode,
	}

	for _, field := range fields {
		if err = read(r, field); err != nil {
			return err
		}
	}

	return nil
}
func decodeEthIntCounters(r io.Reader) (*EthernetInterfaceCounters, error) {
	var eic = new(EthernetInterfaceCounters)

	if err := eic.unmarshal(r); err != nil {
		return nil, err
	}

	return eic, nil
}

func (eic *EthernetInterfaceCounters) unmarshal(r io.Reader) error {
	var err error

	fields := []interface{}{
		&eic.AlignmentErrors,
		&eic.FCSErrors,
		&eic.SingleCollisionFrames,
		&eic.MultipleCollisionFrames,
		&eic.SQETestErrors,
		&eic.DeferredTransmissions,
		&eic.LateCollisions,
		&eic.ExcessiveCollisions,
		&eic.InternalMACTransmitErrors,
		&eic.CarrierSenseErrors,
		&eic.FrameTooLongs,
		&eic.InternalMACReceiveErrors,
		&eic.SymbolErrors,
	}

	for _, field := range fields {
		if err = read(r, field); err != nil {
			return err
		}
	}

	return nil
}
func decodeTokenRingCounters(r io.Reader) (*TokenRingCounters, error) {
	var tr = new(TokenRingCounters)

	if err := tr.unmarshal(r); err != nil {
		return nil, err
	}

	return tr, nil
}

func (tr *TokenRingCounters) unmarshal(r io.Reader) error {
	var err error

	fields := []interface{}{
		&tr.LineErrors,
		&tr.BurstErrors,
		&tr.ACErrors,
		&tr.AbortTransErrors,
		&tr.InternalErrors,
		&tr.LostFrameErrors,
		&tr.ReceiveCongestions,
		&tr.FrameCopiedErrors,
		&tr.TokenErrors,
		&tr.SoftErrors,
		&tr.HardErrors,
		&tr.SignalLoss,
		&tr.TransmitBeacons,
		&tr.Recoverys,
		&tr.LobeWires,
		&tr.Removes,
		&tr.Singles,
		&tr.FreqErrors,
	}

	for _, field := range fields {
		if err = read(r, field); err != nil {
			return err
		}
	}

	return nil
}

func decodeVGCounters(r io.Reader) (*VGCounters, error) {
	var vg = new(VGCounters)

	if err := vg.unmarshal(r); err != nil {
		return nil, err
	}

	return vg, nil
}

func (vg *VGCounters) unmarshal(r io.Reader) error {
	var err error

	fields := []interface{}{
		&vg.InHighPriorityFrames,
		&vg.InHighPriorityOctets,
		&vg.InNormPriorityFrames,
		&vg.InNormPriorityOctets,
		&vg.InIPMErrors,
		&vg.InOversizeFrameErrors,
		&vg.InDataErrors,
		&vg.InNullAddressedFrames,
		&vg.OutHighPriorityFrames,
		&vg.OutHighPriorityOctets,
		&vg.TransitionIntoTrainings,
		&vg.HCInHighPriorityOctets,
		&vg.HCInNormPriorityOctets,
		&vg.HCOutHighPriorityOctets,
	}

	for _, field := range fields {
		if err = read(r, field); err != nil {
			return err
		}
	}

	return nil
}

func decodeVlanCounters(r io.Reader) (*VlanCounters, error) {
	var vc = new(VlanCounters)

	if err := vc.unmarshal(r); err != nil {
		return nil, err
	}

	return vc, nil
}

func (vc *VlanCounters) unmarshal(r io.Reader) error {
	var err error
	fields := []interface{}{
		&vc.ID,
		&vc.Octets,
		&vc.UnicastPackets,
		&vc.MulticastPackets,
		&vc.BroadcastPackets,
		&vc.Discards,
	}

	for _, field := range fields {
		if err = read(r, field); err != nil {
			return err
		}
	}

	return nil
}

func decodedProcessorCounters(r io.Reader) (*ProcessorCounters, error) {
	var pc = new(ProcessorCounters)

	if err := pc.unmarshal(r); err != nil {
		return nil, err
	}

	return pc, nil
}

func (pc *ProcessorCounters) unmarshal(r io.Reader) error {
	var err error
	fields := []interface{}{
		&pc.CPU5s,
		&pc.CPU1m,
		&pc.CPU5m,
		&pc.TotalMemory,
		&pc.FreeMemory,
	}

	for _, field := range fields {
		if err = read(r, field); err != nil {
			return err
		}
	}

	return nil
}

func (cs *CounterSample) unmarshal(r io.Reader) error {

	var err error

	if err = read(r, &cs.SequenceNo); err != nil {
		return err
	}

	if err = read(r, &cs.SourceIdType); err != nil {
		return err
	}

	buf := make([]byte, 3)
	if err = read(r, &buf); err != nil {
		return err
	}
	cs.SourceIdIdx = uint32(buf[2]) | uint32(buf[1])<<8 | uint32(buf[0])<<16

	if err = read(r, &cs.RecordsNo); err != nil {
		return err
	}

	return nil
}
<|endoftext|>


<commit_before>// +build linux

package fscommon

import (
	"fmt"
	"os"
	"path/filepath"
	"strconv"
	"testing"
	"time"

	"github.com/opencontainers/runc/libcontainer/cgroups"
)

func TestWriteCgroupFileHandlesInterrupt(t *testing.T) {
	memoryCgroupMount, err := cgroups.FindCgroupMountpoint("", "memory")
	if err != nil {
		t.Fatal(err)
	}

	cgroupName := fmt.Sprintf("test-eint-%d", time.Now().Nanosecond())
	cgroupPath := filepath.Join(memoryCgroupMount, cgroupName)
	if err := os.MkdirAll(cgroupPath, 0755); err != nil {
		t.Fatal(err)
	}
	defer os.RemoveAll(cgroupPath)

	for i := 0; i < 100000; i++ {
		limit := 1024*1024 + i
		if err := WriteFile(cgroupPath, "memory.limit_in_bytes", strconv.Itoa(limit)); err != nil {
			t.Fatalf("Failed to write %d on attempt %d: %+v", limit, i, err)
		}
	}
}
<commit_msg>Skip test for cgroups v2<commit_after>// +build linux

package fscommon

import (
	"fmt"
	"os"
	"path/filepath"
	"strconv"
	"testing"
	"time"

	"github.com/opencontainers/runc/libcontainer/cgroups"
)

func TestWriteCgroupFileHandlesInterrupt(t *testing.T) {
	if cgroups.IsCgroup2UnifiedMode() {
		t.Skip("cgroup v2 is not supported")
	}

	memoryCgroupMount, err := cgroups.FindCgroupMountpoint("", "memory")
	if err != nil {
		t.Fatal(err)
	}

	cgroupName := fmt.Sprintf("test-eint-%d", time.Now().Nanosecond())
	cgroupPath := filepath.Join(memoryCgroupMount, cgroupName)
	if err := os.MkdirAll(cgroupPath, 0755); err != nil {
		t.Fatal(err)
	}
	defer os.RemoveAll(cgroupPath)

	for i := 0; i < 100000; i++ {
		limit := 1024*1024 + i
		if err := WriteFile(cgroupPath, "memory.limit_in_bytes", strconv.Itoa(limit)); err != nil {
			t.Fatalf("Failed to write %d on attempt %d: %+v", limit, i, err)
		}
	}
}
<|endoftext|>