<commit_before>package model

import (
	"fmt"

	"strings"

	"github.com/evandroflores/claimr/database"
	"github.com/jinzhu/gorm"
)

func init() {
	database.DB.AutoMigrate(&Container{})
}

// Container defines the Container information on database.
type Container struct {
	gorm.Model
	TeamID         string `gorm:"not null"`
	ChannelID      string `gorm:"not null"`
	Name           string `gorm:"not null"`
	InUseBy        string
	InUseForReason string
	CreatedByUser  string
}

// MaxNameSize is the max number of characters for a container name.
const MaxNameSize = 22

func isValidContainerInput(teamID string, channelID string, containerName string) (bool, error) {
	fields := []struct {
		name  string
		value string
	}{
		{"teamID", teamID},
		{"channelID", channelID},
		{"container name", containerName},
	}

	for _, field := range fields {
		err := checkRequired(field.name, field.value)
		if err != nil {
			return false, err
		}
	}

	if len(containerName) > MaxNameSize {
		return false, fmt.Errorf("try a smaller container name up to %d characters", MaxNameSize)
	}

	return true, nil
}

func checkRequired(fieldName string, fieldValue string) error {
	if fieldValue == "" {
		return fmt.Errorf("can not continue without a %s 🙄", fieldName)
	}
	return nil
}

// GetContainer returns a container for teamID, channelID, and name provided
func GetContainer(teamID string, channelID string, name string) (Container, error) {
	result := Container{}
	valid, err := isValidContainerInput(teamID, channelID, name)

	if !valid {
		return result, err
	}

	database.DB.Where(&Container{TeamID: teamID, ChannelID: channelID, Name: strings.ToLower(name)}).
		First(&result)

	return result, nil
}

// GetContainers returns a list of containers for the given TeamID and ChannelID
func GetContainers(teamID string, channelID string) ([]Container, error) {
	results := []Container{}
	valid, err := isValidContainerInput(teamID, channelID, ".")

	if !valid {
		return results, err
	}

	database.DB.Where(&Container{TeamID: teamID, ChannelID: channelID}).
		Find(&results)

	return results, nil
}

// Add a given Container to database
func (container Container) Add() error {
	existingContainer, err := GetContainer(container.TeamID, container.ChannelID, container.Name)

	if err != nil {
		return err
	}

	if existingContainer != (Container{}) {
		return fmt.Errorf("there is a container with the same name on this channel. Try a different one 😕")
	}
	container.Name = strings.ToLower(container.Name)
	database.DB.Create(&container)

	return nil
}

// Update a given Container
func (container Container) Update() error {
	existingContainer, err := GetContainer(container.TeamID, container.ChannelID, strings.ToLower(container.Name))

	if err != nil {
		return err
	}

	if existingContainer == (Container{}) {
		return fmt.Errorf("could not find this container on this channel. Can not update 😕")
	}

	existingContainer.InUseBy = container.InUseBy
	existingContainer.InUseForReason = container.InUseForReason

	database.DB.Save(&existingContainer)

	return nil
}

// Delete removes a Container from the database
func (container Container) Delete() error {
	existingContainer, err := GetContainer(container.TeamID, container.ChannelID, strings.ToLower(container.Name))

	if err != nil {
		return err
	}
	if existingContainer == (Container{}) {
		return fmt.Errorf("could not find this container on this channel. Can not delete 😕")
	}

	database.DB.Delete(&existingContainer)

	return nil
}

// ClearInUse removes information InUseBy and InUseForReason, for a given container
func (container Container) ClearInUse() error {
	container.InUseBy = ""
	container.InUseForReason = ""

	return container.Update()
}
<commit_msg>Creating method to set in use attributes<commit_after>package model

import (
	"fmt"

	"strings"

	"github.com/evandroflores/claimr/database"
	"github.com/jinzhu/gorm"
)

func init() {
	database.DB.AutoMigrate(&Container{})
}

// Container defines the Container information on database.
type Container struct {
	gorm.Model
	TeamID         string `gorm:"not null"`
	ChannelID      string `gorm:"not null"`
	Name           string `gorm:"not null"`
	InUseBy        string
	InUseForReason string
	CreatedByUser  string
}

// MaxNameSize is the max number of characters for a container name.
const MaxNameSize = 22

func isValidContainerInput(teamID string, channelID string, containerName string) (bool, error) {
	fields := []struct {
		name  string
		value string
	}{
		{"teamID", teamID},
		{"channelID", channelID},
		{"container name", containerName},
	}

	for _, field := range fields {
		err := checkRequired(field.name, field.value)
		if err != nil {
			return false, err
		}
	}

	if len(containerName) > MaxNameSize {
		return false, fmt.Errorf("try a smaller container name up to %d characters", MaxNameSize)
	}

	return true, nil
}

func checkRequired(fieldName string, fieldValue string) error {
	if fieldValue == "" {
		return fmt.Errorf("can not continue without a %s 🙄", fieldName)
	}
	return nil
}

// GetContainer returns a container for teamID, channelID, and name provided
func GetContainer(teamID string, channelID string, name string) (Container, error) {
	result := Container{}
	valid, err := isValidContainerInput(teamID, channelID, name)

	if !valid {
		return result, err
	}

	database.DB.Where(&Container{TeamID: teamID, ChannelID: channelID, Name: strings.ToLower(name)}).
		First(&result)

	return result, nil
}

// GetContainers returns a list of containers for the given TeamID and ChannelID
func GetContainers(teamID string, channelID string) ([]Container, error) {
	results := []Container{}
	valid, err := isValidContainerInput(teamID, channelID, ".")

	if !valid {
		return results, err
	}

	database.DB.Where(&Container{TeamID: teamID, ChannelID: channelID}).
		Find(&results)

	return results, nil
}

// Add a given Container to database
func (container Container) Add() error {
	existingContainer, err := GetContainer(container.TeamID, container.ChannelID, container.Name)

	if err != nil {
		return err
	}

	if existingContainer != (Container{}) {
		return fmt.Errorf("there is a container with the same name on this channel. Try a different one 😕")
	}
	container.Name = strings.ToLower(container.Name)
	database.DB.Create(&container)

	return nil
}

// Update a given Container
func (container Container) Update() error {
	existingContainer, err := GetContainer(container.TeamID, container.ChannelID, strings.ToLower(container.Name))

	if err != nil {
		return err
	}

	if existingContainer == (Container{}) {
		return fmt.Errorf("could not find this container on this channel. Can not update 😕")
	}

	existingContainer.InUseBy = container.InUseBy
	existingContainer.InUseForReason = container.InUseForReason

	database.DB.Save(&existingContainer)

	return nil
}

// Delete removes a Container from the database
func (container Container) Delete() error {
	existingContainer, err := GetContainer(container.TeamID, container.ChannelID, strings.ToLower(container.Name))

	if err != nil {
		return err
	}
	if existingContainer == (Container{}) {
		return fmt.Errorf("could not find this container on this channel. Can not delete 😕")
	}

	database.DB.Delete(&existingContainer)

	return nil
}

// ClearInUse removes information InUseBy and InUseForReason, for a given container
func (container Container) ClearInUse() error {
	container.InUseBy = ""
	container.InUseForReason = ""

	return container.Update()
}

func (container Container) SetInUse(by string, reason string) error {
	container.InUseBy = by
	container.InUseForReason = reason

	return container.Update()
}
<|endoftext|>


<commit_before>package model

import (
	"github.com/slyrz/newscat/html"
	"math"
	"strings"
)

// ChunkExtractor utilizes the trained model to extract relevant html.Chunks from
// an html.Article.
type ChunkExtractor struct {
	ChunkFeatures []chunkFeature
	BoostFeatures []boostFeature
}

// LinkExtractor extracts possible links to news articles from a html.Website.
type LinkExtractor struct {
}

// NewChunkExtractor creates and initalizes a new ChunkExtractor.
func NewChunkExtractor() *ChunkExtractor {
	return new(ChunkExtractor)
}

// Extract returns a list of relevant text chunks found in article.
//
// How it works
//
// This function creates a feature vector for each chunk found in article.
// A feature vector contains a numerical representation of the chunk's
// properties like HTML element type, parent element type, number of words,
// number of sentences and stuff like this.
//
// A logistic regression model is used to calculate scores based on these
// feature vectors. Then, in some kind of meta / ensemble learning approach,
// a second type of feature vector is created based on these scores.
// This feature vector is fed to our random forest and finally
// the random forest's predictions are used to generate the result.
//
// By now you might have noticed that I'm exceptionally bad at naming and
// describing things properly.
func (ext *ChunkExtractor) Extract(article *html.Article) []*html.Chunk {
	ext.ChunkFeatures = nil
	ext.BoostFeatures = nil

	// No chunks? No features.
	if len(article.Chunks) == 0 {
		return nil
	}

	// We create one feature for each chunk.
	chunkFeatures := make([]chunkFeature, len(article.Chunks))
	boostFeatures := make([]boostFeature, len(article.Chunks))

	// Count the number of words and sentences we encountered for each
	// class. This helps us to detect elements that contain the article text.
	classStats := article.GetClassStats()
	clusterStats := article.GetClusterStats()

	chunkFeatureWriter := new(chunkFeatureWriter)
	for i, chunk := range article.Chunks {
		chunkFeatureWriter.Assign(chunkFeatures[i][:])
		chunkFeatureWriter.WriteElementType(chunk)
		chunkFeatureWriter.WriteParentType(chunk)
		chunkFeatureWriter.WriteSiblingTypes(chunk)
		chunkFeatureWriter.WriteAncestors(chunk)
		chunkFeatureWriter.WriteTextStat(chunk)
		chunkFeatureWriter.WriteTextStatSiblings(chunk)
		chunkFeatureWriter.WriteClassStat(chunk, classStats)
		chunkFeatureWriter.WriteClusterStat(chunk, clusterStats)
	}

	// Detect the minimum and maximum value for each element in the
	// feature vector.
	empMin := chunkFeature{}
	empMax := chunkFeature{}
	for i := range chunkFeatures {
		for j, val := range chunkFeatures[i] {
			switch {
			case val < empMin[j]:
				empMin[j] = val
			case val > empMax[j]:
				empMax[j] = val
			}
		}
	}

	// Perform MinMax normalization.
	for i := range chunkFeatures {
		feature := &chunkFeatures[i]
		for j, val := range chunkFeatures[i] {
			// If the maximum value is not greater than one, we assume that the feature is
			// already normalized and leave it untouched.
			if empMax[j] > 1.0 {
				feature[j] = (val - empMin[j]) / (empMax[j] - empMin[j])
			}
		}
	}

	// Now we cluster Chunks by Containers to calculate average score per
	// container.
	clusterContainer := newClusterMap()
	for i, chunk := range article.Chunks {
		clusterContainer.Add(chunk.Container, chunk, chunkFeatures[i].Score())
	}

	boostFeatureWriter := new(boostFeatureWriter)
	for i, chunk := range article.Chunks {
		boostFeatureWriter.Assign(boostFeatures[i][:])
		boostFeatureWriter.WriteChunk(chunk)
		boostFeatureWriter.WriteCluster(chunk, clusterContainer[chunk.Container])
		boostFeatureWriter.WriteTitleSimilarity(chunk, article.Title)
	}

	// Cluster chunks by block and add those blocks to the result whose average
	// score is above prediction level. This makes sure that we don't split large
	// blocks.
	clusterBlock := newClusterMap()
	for i, chunk := range article.Chunks {
		clusterBlock.Add(chunk.Block, chunk, boostFeatures[i].Score(), float32(chunk.Text.Len()))
	}

	// Keep blocks together.
	result := make([]*html.Chunk, 0, 8)
	for _, chunk := range article.Chunks {
		if clusterBlock[chunk.Block].Score() > 0.5 {
			result = append(result, chunk)
		}
	}

	// Make them accessible.
	ext.ChunkFeatures = chunkFeatures
	ext.BoostFeatures = boostFeatures
	return result
}

// NewLinkExtractor creates and initalizes a new LinkExtractor.
func NewLinkExtractor() *LinkExtractor {
	return new(LinkExtractor)
}

// TODO
func (ext *LinkExtractor) Extract(website *html.Website) []*html.Link {
	if len(website.Links) == 0 {
		return nil
	}

	hostFreq := make(map[string]float64)
	for _, link := range website.Links {
		hostFreq[link.URL.Host] = hostFreq[link.URL.Host] + 1
	}

	total := float64(len(website.Links))
	for host, count := range hostFreq {
		hostFreq[host] = count / total
	}

	// TODO
	result := make([]*html.Link, 0, 8)
	for _, link := range website.Links {
		score := calcEntropy(link.URL.Path) * hostFreq[link.URL.Host]
		if score >= 3.0 {
			result = append(result, link)
		}
	}
	return result
}

var (
	entropy = map[rune]float64{
		'a': -0.10634380971489953,
		'b': -0.1530425750819142,
		'c': -0.054512771770650156,
		'd': -0.2277081908035247,
		'e': -0.019143424124981535,
		'f': -0.10631901817532326,
		'g': -0.08950958783746302,
		'h': -0.02138890867431747,
		'i': -0.2040842535452831,
		'j': -0.08066894356623576,
		'k': -0.14676065483416048,
		'l': -0.054715501033775864,
		'm': -0.22756818123774952,
		'n': -0.2036576755124059,
		'o': -0.011796162629390488,
		'p': -0.19352586509733832,
		'q': -0.061206381465562155,
		'r': -0.12701158290463668,
		's': -0.10100191941724014,
		't': -0.022872200444912123,
		'u': -0.05095402954111724,
		'v': -0.09120240867794169,
		'w': -0.20676837174037171,
		'x': -0.0856356305628621,
		'y': -0.052914228421479595,
		'z': -0.08290376493347784,
		'0': -0.20509019606654505,
		'1': -0.04936506198036805,
		'2': -0.052649170776553546,
		'3': -0.28439304075529853,
		'4': -0.11790007547597285,
		'5': -0.09902560480951333,
		'6': -0.0624913286633471,
		'7': -0.1247640940220319,
		'8': -0.17807011198199005,
		'9': -0.2233673033869976,
	}
)

// calcEntropy returns the entropy of a string.
func calcEntropy(s string) float64 {
	result := 0.0
	for _, r := range strings.ToLower(s) {
		result += entropy[r]
	}
	return math.Abs(result)
}
<commit_msg>simplified hostname frequency calculation<commit_after>package model

import (
	"github.com/slyrz/newscat/html"
	"math"
	"strings"
)

// ChunkExtractor utilizes the trained model to extract relevant html.Chunks from
// an html.Article.
type ChunkExtractor struct {
	ChunkFeatures []chunkFeature
	BoostFeatures []boostFeature
}

// LinkExtractor extracts possible links to news articles from a html.Website.
type LinkExtractor struct {
}

// NewChunkExtractor creates and initalizes a new ChunkExtractor.
func NewChunkExtractor() *ChunkExtractor {
	return new(ChunkExtractor)
}

// Extract returns a list of relevant text chunks found in article.
//
// How it works
//
// This function creates a feature vector for each chunk found in article.
// A feature vector contains a numerical representation of the chunk's
// properties like HTML element type, parent element type, number of words,
// number of sentences and stuff like this.
//
// A logistic regression model is used to calculate scores based on these
// feature vectors. Then, in some kind of meta / ensemble learning approach,
// a second type of feature vector is created based on these scores.
// This feature vector is fed to our random forest and finally
// the random forest's predictions are used to generate the result.
//
// By now you might have noticed that I'm exceptionally bad at naming and
// describing things properly.
func (ext *ChunkExtractor) Extract(article *html.Article) []*html.Chunk {
	ext.ChunkFeatures = nil
	ext.BoostFeatures = nil

	// No chunks? No features.
	if len(article.Chunks) == 0 {
		return nil
	}

	// We create one feature for each chunk.
	chunkFeatures := make([]chunkFeature, len(article.Chunks))
	boostFeatures := make([]boostFeature, len(article.Chunks))

	// Count the number of words and sentences we encountered for each
	// class. This helps us to detect elements that contain the article text.
	classStats := article.GetClassStats()
	clusterStats := article.GetClusterStats()

	chunkFeatureWriter := new(chunkFeatureWriter)
	for i, chunk := range article.Chunks {
		chunkFeatureWriter.Assign(chunkFeatures[i][:])
		chunkFeatureWriter.WriteElementType(chunk)
		chunkFeatureWriter.WriteParentType(chunk)
		chunkFeatureWriter.WriteSiblingTypes(chunk)
		chunkFeatureWriter.WriteAncestors(chunk)
		chunkFeatureWriter.WriteTextStat(chunk)
		chunkFeatureWriter.WriteTextStatSiblings(chunk)
		chunkFeatureWriter.WriteClassStat(chunk, classStats)
		chunkFeatureWriter.WriteClusterStat(chunk, clusterStats)
	}

	// Detect the minimum and maximum value for each element in the
	// feature vector.
	empMin := chunkFeature{}
	empMax := chunkFeature{}
	for i := range chunkFeatures {
		for j, val := range chunkFeatures[i] {
			switch {
			case val < empMin[j]:
				empMin[j] = val
			case val > empMax[j]:
				empMax[j] = val
			}
		}
	}

	// Perform MinMax normalization.
	for i := range chunkFeatures {
		feature := &chunkFeatures[i]
		for j, val := range chunkFeatures[i] {
			// If the maximum value is not greater than one, we assume that the feature is
			// already normalized and leave it untouched.
			if empMax[j] > 1.0 {
				feature[j] = (val - empMin[j]) / (empMax[j] - empMin[j])
			}
		}
	}

	// Now we cluster Chunks by Containers to calculate average score per
	// container.
	clusterContainer := newClusterMap()
	for i, chunk := range article.Chunks {
		clusterContainer.Add(chunk.Container, chunk, chunkFeatures[i].Score())
	}

	boostFeatureWriter := new(boostFeatureWriter)
	for i, chunk := range article.Chunks {
		boostFeatureWriter.Assign(boostFeatures[i][:])
		boostFeatureWriter.WriteChunk(chunk)
		boostFeatureWriter.WriteCluster(chunk, clusterContainer[chunk.Container])
		boostFeatureWriter.WriteTitleSimilarity(chunk, article.Title)
	}

	// Cluster chunks by block and add those blocks to the result whose average
	// score is above prediction level. This makes sure that we don't split large
	// blocks.
	clusterBlock := newClusterMap()
	for i, chunk := range article.Chunks {
		clusterBlock.Add(chunk.Block, chunk, boostFeatures[i].Score(), float32(chunk.Text.Len()))
	}

	// Keep blocks together.
	result := make([]*html.Chunk, 0, 8)
	for _, chunk := range article.Chunks {
		if clusterBlock[chunk.Block].Score() > 0.5 {
			result = append(result, chunk)
		}
	}

	// Make them accessible.
	ext.ChunkFeatures = chunkFeatures
	ext.BoostFeatures = boostFeatures
	return result
}

// NewLinkExtractor creates and initalizes a new LinkExtractor.
func NewLinkExtractor() *LinkExtractor {
	return new(LinkExtractor)
}

// TODO
func (ext *LinkExtractor) Extract(website *html.Website) []*html.Link {
	if len(website.Links) == 0 {
		return nil
	}

	// Calculate the host name frequency. We assume that the more often
	// we encounter a host name, the more important are its links.
	hostFreq := make(map[string]float64)
	unitFrac := 1.0 / float64(len(website.Links))
	for _, link := range website.Links {
		hostFreq[link.URL.Host] = hostFreq[link.URL.Host] + unitFrac
	}

	// TODO
	result := make([]*html.Link, 0, 8)
	for _, link := range website.Links {
		score := calcEntropy(link.URL.Path) * hostFreq[link.URL.Host]
		if score >= 3.0 {
			result = append(result, link)
		}
	}
	return result
}

var (
	entropy = map[rune]float64{
		'a': -0.10634380971489953,
		'b': -0.1530425750819142,
		'c': -0.054512771770650156,
		'd': -0.2277081908035247,
		'e': -0.019143424124981535,
		'f': -0.10631901817532326,
		'g': -0.08950958783746302,
		'h': -0.02138890867431747,
		'i': -0.2040842535452831,
		'j': -0.08066894356623576,
		'k': -0.14676065483416048,
		'l': -0.054715501033775864,
		'm': -0.22756818123774952,
		'n': -0.2036576755124059,
		'o': -0.011796162629390488,
		'p': -0.19352586509733832,
		'q': -0.061206381465562155,
		'r': -0.12701158290463668,
		's': -0.10100191941724014,
		't': -0.022872200444912123,
		'u': -0.05095402954111724,
		'v': -0.09120240867794169,
		'w': -0.20676837174037171,
		'x': -0.0856356305628621,
		'y': -0.052914228421479595,
		'z': -0.08290376493347784,
		'0': -0.20509019606654505,
		'1': -0.04936506198036805,
		'2': -0.052649170776553546,
		'3': -0.28439304075529853,
		'4': -0.11790007547597285,
		'5': -0.09902560480951333,
		'6': -0.0624913286633471,
		'7': -0.1247640940220319,
		'8': -0.17807011198199005,
		'9': -0.2233673033869976,
	}
)

// calcEntropy returns the entropy of a string.
func calcEntropy(s string) float64 {
	result := 0.0
	for _, r := range strings.ToLower(s) {
		result += entropy[r]
	}
	return math.Abs(result)
}
<|endoftext|>


<commit_before>package model

import (
  "appengine"
  "appengine/memcache"
  "errors"
  "fmt"
  "math"
  "time"
)

type ErrRateLimitExceeded struct {
  RateLimit *DistributedRateLimiter
  Debug error
}
func (e ErrRateLimitExceeded) Error() string {
  return fmt.Sprintf("Exceeded rate limit: %s (%v)", e.RateLimit.Name, e.Debug)
}

// Describes a rate limit that permits MaxEvents in the last IntervalSeconds.
// Ex: RateLimit{10, 100} would indicate 10 events per 100 seconds.
type RateLimit struct {
  MaxEvents       int
  IntervalSeconds int
}

type DistributedRateLimiter struct {
  Name   string
  Limits []RateLimit
}

// Creates a new DistributedRateLimiter that allows events according to the
// specified limits (or re-initializes the limits of an existing rate limiter
// with the same name).
func (r DistributedRateLimiter) Init(c appengine.Context) error {
  e := new(DistributedRateLimiterEntity)
  e.Buckets = make([]TokenBucket, len(r.Limits))
  for i, _ := range e.Buckets {
    e.Buckets[i].SetLimit(r.Limits[i])
  }

  err := memcache.JSON.Set(c, &memcache.Item{
    Key: fmt.Sprintf("DistributedRateLimiterEntity/%s", r.Name),
    Object: e,
  })
  return err
}

// Blocks for up to 1 minute.
func(r *DistributedRateLimiter) Consume(
  c appengine.Context,
  events int) error {

  tryResult := make(chan error)
  timeout := time.After(1 * time.Minute)

  try := func() {
    tryResult <- r.TryConsume(c, events)
  }

  for {
    go try()
    select {
      case err := <-tryResult:
        if _, ok := err.(ErrRateLimitExceeded); ok {
          time.Sleep(1 * time.Second)
          continue
        }
        return err
      case <-timeout:
        return ErrRateLimitExceeded{
          r,
          errors.New("Consume() timeout after 1 minute."),
        }
    }
  }
}

// Consumes tokens from the rate limiter if nil is returned.
// Otherwise an error is returned describing why tokens could not
// be consumed.
func (r *DistributedRateLimiter) TryConsume(c appengine.Context, events int) error {
  e := new(DistributedRateLimiterEntity)
  key := fmt.Sprintf("DistributedRateLimiterEntity/%s", r.Name)
  for attempt := 0; attempt < 10; attempt++ {
    item, err := memcache.JSON.Get(c, key, e)
    if err != nil {
      if err == memcache.ErrCacheMiss {
        // Put a fresh item in memcache and try again.
        r.Init(c)
        continue
      } else {
        return err
      }
    }

    e.addTokens(time.Now().UTC())

    if err := e.tryConsume(events); err != nil {
      return ErrRateLimitExceeded{r, err}
    }

    // We've consumed the tokens if and only if nothing else has written the
    // entry back to memcache since we got it.
    item.Object = e
    if err := memcache.JSON.CompareAndSwap(c, item); err != nil {
      continue
    }
    return nil
  }
  return errors.New(fmt.Sprintf("RetryLimit for memcache.CompareAndSwap reached: %s", key))
}

func (r *DistributedRateLimiter) DebugStr(c appengine.Context) string {
  e := new(DistributedRateLimiterEntity)
  key := fmt.Sprintf("DistributedRateLimiterEntity/%s", r.Name)
  _, err := memcache.JSON.Get(c, key, e)
  if err != nil {
    return err.Error()
  }
  return fmt.Sprintf("%v", e)
}

type TokenBucket struct {
  Limit RateLimit

  // The number of tokens in the bucket at LastCheckTime.
  Tokens float64

  // Tracks the last time this token bucket was processed so that we can
  // optimize adding tokens to the bucket each update instead of every
  // Rate seconds. Time is in UTC.
  LastCheckTime time.Time
}

// Computes the number of tokens to add to the bucket per second.
func (b *TokenBucket) newTokensPerSecond() float64 {
  return float64(b.Limit.MaxEvents) / float64(b.Limit.IntervalSeconds)
}

func (b *TokenBucket) AddTokens(t time.Time) {
  elapsedSeconds := t.Sub(b.LastCheckTime).Seconds()
  if elapsedSeconds < 0 {
    return
  }
  b.Tokens = math.Min(b.Tokens+elapsedSeconds*b.newTokensPerSecond(),
    float64(b.Limit.MaxEvents))
  b.LastCheckTime = t
}

// If the provided limits are the same as the existing limits this is a no-op.
// Otherwise the limits are set to the new limits and all tokens are removed from
// the bucket.
func (b *TokenBucket) SetLimit(limit RateLimit) {
  if limit == b.Limit {
    return
  }
  b.Limit = limit
  b.Tokens = 0.0
  b.LastCheckTime = time.Now().UTC()
}

// An opaque type for managing distributed rate limits with the appengine datastore.
// Use its methods to interact with it.
type DistributedRateLimiterEntity struct {
  // Counts of the total number of requests accepted.
  AcceptCount int64

  // Internal buckets.
  Buckets []TokenBucket
}

func (e *DistributedRateLimiterEntity) addTokens(t time.Time) {
  for i, _ := range e.Buckets {
    e.Buckets[i].AddTokens(t)
  }
}

func (e *DistributedRateLimiterEntity) tryConsume(numTokens int) error {
  tokens := float64(numTokens)

  // See if tokens are available from all buckets before consuming any tokens.
  for _, b := range e.Buckets {
    if tokens > b.Tokens {
      return errors.New(fmt.Sprintf("Exceeded rate limit: %+v", b.Limit))
    }
  }
  // Consume the tokens.
  for i, _ := range e.Buckets {
    e.Buckets[i].Tokens -= tokens
  }
  e.AcceptCount++
  return nil
}
<commit_msg>display rate limit considering time till now in debug<commit_after>package model

import (
  "appengine"
  "appengine/memcache"
  "errors"
  "fmt"
  "math"
  "time"
)

type ErrRateLimitExceeded struct {
  RateLimit *DistributedRateLimiter
  Debug error
}
func (e ErrRateLimitExceeded) Error() string {
  return fmt.Sprintf("Exceeded rate limit: %s (%v)", e.RateLimit.Name, e.Debug)
}

// Describes a rate limit that permits MaxEvents in the last IntervalSeconds.
// Ex: RateLimit{10, 100} would indicate 10 events per 100 seconds.
type RateLimit struct {
  MaxEvents       int
  IntervalSeconds int
}

type DistributedRateLimiter struct {
  Name   string
  Limits []RateLimit
}

// Creates a new DistributedRateLimiter that allows events according to the
// specified limits (or re-initializes the limits of an existing rate limiter
// with the same name).
func (r DistributedRateLimiter) Init(c appengine.Context) error {
  e := new(DistributedRateLimiterEntity)
  e.Buckets = make([]TokenBucket, len(r.Limits))
  for i, _ := range e.Buckets {
    e.Buckets[i].SetLimit(r.Limits[i])
  }

  err := memcache.JSON.Set(c, &memcache.Item{
    Key: fmt.Sprintf("DistributedRateLimiterEntity/%s", r.Name),
    Object: e,
  })
  return err
}

// Blocks for up to 1 minute.
func(r *DistributedRateLimiter) Consume(
  c appengine.Context,
  events int) error {

  tryResult := make(chan error)
  timeout := time.After(1 * time.Minute)

  try := func() {
    tryResult <- r.TryConsume(c, events)
  }

  for {
    go try()
    select {
      case err := <-tryResult:
        if _, ok := err.(ErrRateLimitExceeded); ok {
          time.Sleep(1 * time.Second)
          continue
        }
        return err
      case <-timeout:
        return ErrRateLimitExceeded{
          r,
          errors.New("Consume() timeout after 1 minute."),
        }
    }
  }
}

// Consumes tokens from the rate limiter if nil is returned.
// Otherwise an error is returned describing why tokens could not
// be consumed.
func (r *DistributedRateLimiter) TryConsume(c appengine.Context, events int) error {
  e := new(DistributedRateLimiterEntity)
  key := fmt.Sprintf("DistributedRateLimiterEntity/%s", r.Name)
  for attempt := 0; attempt < 10; attempt++ {
    item, err := memcache.JSON.Get(c, key, e)
    if err != nil {
      if err == memcache.ErrCacheMiss {
        // Put a fresh item in memcache and try again.
        r.Init(c)
        continue
      } else {
        return err
      }
    }

    e.addTokens(time.Now().UTC())

    if err := e.tryConsume(events); err != nil {
      return ErrRateLimitExceeded{r, err}
    }

    // We've consumed the tokens if and only if nothing else has written the
    // entry back to memcache since we got it.
    item.Object = e
    if err := memcache.JSON.CompareAndSwap(c, item); err != nil {
      continue
    }
    return nil
  }
  return errors.New(fmt.Sprintf("RetryLimit for memcache.CompareAndSwap reached: %s", key))
}

func (r *DistributedRateLimiter) DebugStr(c appengine.Context) string {
  e := new(DistributedRateLimiterEntity)
  key := fmt.Sprintf("DistributedRateLimiterEntity/%s", r.Name)
  _, err := memcache.JSON.Get(c, key, e)
  if err != nil {
    return err.Error()
  }
  e.addTokens(time.Now().UTC())
  return fmt.Sprintf("%v", e)
}

type TokenBucket struct {
  Limit RateLimit

  // The number of tokens in the bucket at LastCheckTime.
  Tokens float64

  // Tracks the last time this token bucket was processed so that we can
  // optimize adding tokens to the bucket each update instead of every
  // Rate seconds. Time is in UTC.
  LastCheckTime time.Time
}

// Computes the number of tokens to add to the bucket per second.
func (b *TokenBucket) newTokensPerSecond() float64 {
  return float64(b.Limit.MaxEvents) / float64(b.Limit.IntervalSeconds)
}

func (b *TokenBucket) AddTokens(t time.Time) {
  elapsedSeconds := t.Sub(b.LastCheckTime).Seconds()
  if elapsedSeconds < 0 {
    return
  }
  b.Tokens = math.Min(b.Tokens+elapsedSeconds*b.newTokensPerSecond(),
    float64(b.Limit.MaxEvents))
  b.LastCheckTime = t
}

// If the provided limits are the same as the existing limits this is a no-op.
// Otherwise the limits are set to the new limits and all tokens are removed from
// the bucket.
func (b *TokenBucket) SetLimit(limit RateLimit) {
  if limit == b.Limit {
    return
  }
  b.Limit = limit
  b.Tokens = 0.0
  b.LastCheckTime = time.Now().UTC()
}

// An opaque type for managing distributed rate limits with the appengine datastore.
// Use its methods to interact with it.
type DistributedRateLimiterEntity struct {
  // Counts of the total number of requests accepted.
  AcceptCount int64

  // Internal buckets.
  Buckets []TokenBucket
}

func (e *DistributedRateLimiterEntity) addTokens(t time.Time) {
  for i, _ := range e.Buckets {
    e.Buckets[i].AddTokens(t)
  }
}

func (e *DistributedRateLimiterEntity) tryConsume(numTokens int) error {
  tokens := float64(numTokens)

  // See if tokens are available from all buckets before consuming any tokens.
  for _, b := range e.Buckets {
    if tokens > b.Tokens {
      return errors.New(fmt.Sprintf("Exceeded rate limit: %+v", b.Limit))
    }
  }
  // Consume the tokens.
  for i, _ := range e.Buckets {
    e.Buckets[i].Tokens -= tokens
  }
  e.AcceptCount++
  return nil
}
<|endoftext|>


<commit_before>/*
Copyright 2017 Mirantis

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package e2e

import (
	"time"

	. "github.com/onsi/gomega"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/client-go/pkg/api/v1"

	"github.com/Mirantis/virtlet/tests/e2e/framework"
	. "github.com/Mirantis/virtlet/tests/e2e/ginkgo-ext"
)

var _ = Describe("Container volume mounts", func() {
	Context("Of raw volumes", func() {
		var (
			vm           *framework.VMInterface
			nodeExecutor framework.Executor
			devPath      string
			ssh          framework.Executor
		)

		BeforeAll(func() {
			virtletNodeName, err := controller.VirtletNodeName()
			Expect(err).NotTo(HaveOccurred())
			nodeExecutor, err = controller.DinDNodeExecutor(virtletNodeName)
			Expect(err).NotTo(HaveOccurred())

			_, err = framework.ExecSimple(nodeExecutor, "dd", "if=/dev/zero", "of=/rawdevtest", "bs=1M", "count=10")
			Expect(err).NotTo(HaveOccurred())
			_, err = framework.ExecSimple(nodeExecutor, "mkfs.ext4", "/rawdevtest")
			Expect(err).NotTo(HaveOccurred())
			devPath, err = framework.ExecSimple(nodeExecutor, "losetup", "-f", "/rawdevtest", "--show")
			Expect(err).NotTo(HaveOccurred())

			vm = makeVolumeMountVM(map[string]string{
				"type": "raw",
				"path": devPath,
				"part": "0",
			}, virtletNodeName)
		})

		AfterAll(func() {
			deleteVM(vm)
			// The loopback device is detached by itself upon
			// success (TODO: check why it happens), so we
			// ignore errors here
			framework.ExecSimple(nodeExecutor, "losetup", "-d", devPath)
		})

		scheduleWaitSSH(&vm, &ssh)

		It("Should be handled inside the VM", func() {
			shouldBeMounted(ssh)
		})
	})

	Context("Of ephemeral volumes", func() {
		var (
			vm  *framework.VMInterface
			ssh framework.Executor
		)

		BeforeAll(func() {
			vm = makeVolumeMountVM(map[string]string{
				"type":     "qcow2",
				"capacity": "10MB",
			}, "")
		})

		AfterAll(func() {
			deleteVM(vm)
		})

		scheduleWaitSSH(&vm, &ssh)
		It("Should be handled inside the VM [Conformance]", func() {
			shouldBeMounted(ssh)
		})
	})

	Context("ConfigMap files must be mounted", func() {
		var vm *framework.VMInterface

		BeforeAll(func() {
			requireCloudInit()
			cm := &v1.ConfigMap{
				ObjectMeta: metav1.ObjectMeta{
					Name: "files-cm",
				},
				Data: map[string]string{
					"file1": "Hello ",
					"file2": "world!",
				},
			}
			_, err := controller.ConfigMaps().Create(cm)
			Expect(err).NotTo(HaveOccurred())

			vm = controller.VM("cm-files")
			podCustomization := func(pod *framework.PodInterface) {
				pod.Pod.Spec.Containers[0].VolumeMounts = []v1.VolumeMount{
					{
						Name:      "vol",
						MountPath: "/tmp/vol",
					},
				}
				pod.Pod.Spec.Volumes = append(pod.Pod.Spec.Volumes, v1.Volume{
					Name: "vol",
					VolumeSource: v1.VolumeSource{
						ConfigMap: &v1.ConfigMapVolumeSource{
							LocalObjectReference: v1.LocalObjectReference{
								Name: "files-cm",
							},
							Items: []v1.KeyToPath{
								{
									Path: "test-file1",
									Key:  "file1",
								},
								{
									Path: "test-file2",
									Key:  "file2",
								},
							},
						},
					},
				})
			}

			vm.Create(VMOptions{}.applyDefaults(), time.Minute*5, podCustomization)
		})

		AfterAll(func() {
			deleteVM(vm)
			controller.ConfigMaps().Delete("files-cm", nil)
		})

		It("And files must be found on VM [Conformance]", func() {
			ssh := waitSSH(vm)
			Expect(framework.ExecSimple(ssh, "cat", "/tmp/vol/test-file1", "/tmp/vol/test-file2")).To(Equal("Hello world!"))
		})
	})

	Context("Secret files must be mounted", func() {
		var vm *framework.VMInterface

		BeforeAll(func() {
			requireCloudInit()
			secret := &v1.Secret{
				ObjectMeta: metav1.ObjectMeta{
					Name: "files-secret",
				},
				StringData: map[string]string{
					"file1": "Hello ",
					"file2": "world!",
				},
			}
			_, err := controller.Secrets().Create(secret)
			Expect(err).NotTo(HaveOccurred())

			vm = controller.VM("secret-files")
			podCustomization := func(pod *framework.PodInterface) {
				pod.Pod.Spec.Containers[0].VolumeMounts = []v1.VolumeMount{
					{
						Name:      "vol",
						MountPath: "/tmp/vol",
					},
				}
				pod.Pod.Spec.Volumes = append(pod.Pod.Spec.Volumes, v1.Volume{
					Name: "vol",
					VolumeSource: v1.VolumeSource{
						Secret: &v1.SecretVolumeSource{
							SecretName: "files-secret",
							Items: []v1.KeyToPath{
								{
									Path: "test-file1",
									Key:  "file1",
								},
								{
									Path: "test-file2",
									Key:  "file2",
								},
							},
						},
					},
				})
			}

			vm.Create(VMOptions{}.applyDefaults(), time.Minute*5, podCustomization)
		})

		AfterAll(func() {
			deleteVM(vm)
			controller.Secrets().Delete("files-secret", nil)
		})

		It("And files must be found on VM [Conformance]", func() {
			ssh := waitSSH(vm)
			Expect(framework.ExecSimple(ssh, "cat", "/tmp/vol/test-file1", "/tmp/vol/test-file2")).To(Equal("Hello world!"))
		})
	})

})

func makeVolumeMountVM(flexvolOptions map[string]string, nodeName string) *framework.VMInterface {
	podCustomization := func(pod *framework.PodInterface) {
		pod.Pod.Spec.Volumes = append(pod.Pod.Spec.Volumes, v1.Volume{
			Name: "blockdev",
			VolumeSource: v1.VolumeSource{
				FlexVolume: &v1.FlexVolumeSource{
					Driver:  "virtlet/flexvolume_driver",
					Options: flexvolOptions,
				},
			},
		})
		pod.Pod.Spec.Containers[0].VolumeMounts = append(pod.Pod.Spec.Containers[0].VolumeMounts, v1.VolumeMount{
			Name:      "blockdev",
			MountPath: "/foo",
		})
	}
	vm := controller.VM("mount-vm")
	vm.Create(VMOptions{
		NodeName: nodeName,
		// TODO: should also have an option to test using
		// ubuntu image with volumes mounted using cloud-init
		// userdata 'mounts' section
		UserDataScript: "@virtlet-mount-script@",
	}.applyDefaults(), time.Minute*5, podCustomization)
	_, err := vm.Pod()
	Expect(err).NotTo(HaveOccurred())
	return vm
}

func shouldBeMounted(ssh framework.Executor) {
	Eventually(func() (string, error) {
		return framework.ExecSimple(ssh, "ls -l /foo")
	}, 60).Should(ContainSubstring("lost+found"))
}
<commit_msg>Add multiple volume mount e2e<commit_after>/*
Copyright 2017 Mirantis

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package e2e

import (
	"time"

	. "github.com/onsi/gomega"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/client-go/pkg/api/v1"

	"github.com/Mirantis/virtlet/tests/e2e/framework"
	. "github.com/Mirantis/virtlet/tests/e2e/ginkgo-ext"
)

var _ = Describe("Container volume mounts", func() {
	Context("Of raw volumes", func() {
		var (
			err             error
			virtletNodeName string
			vm              *framework.VMInterface
			nodeExecutor    framework.Executor
			devPath         string
			ssh             framework.Executor
		)

		BeforeAll(func() {
			virtletNodeName, err = controller.VirtletNodeName()
			Expect(err).NotTo(HaveOccurred())
			nodeExecutor, err = controller.DinDNodeExecutor(virtletNodeName)
			Expect(err).NotTo(HaveOccurred())

			_, err = framework.ExecSimple(nodeExecutor, "dd", "if=/dev/zero", "of=/rawdevtest", "bs=1M", "count=10")
			Expect(err).NotTo(HaveOccurred())
			_, err = framework.ExecSimple(nodeExecutor, "mkfs.ext4", "/rawdevtest")
			Expect(err).NotTo(HaveOccurred())
			devPath, err = framework.ExecSimple(nodeExecutor, "losetup", "-f", "/rawdevtest", "--show")
			Expect(err).NotTo(HaveOccurred())
		})

		AfterEach(func() {
			if ssh != nil {
				ssh.Close()
			}
			if vm != nil {
				deleteVM(vm)
			}
		})

		AfterAll(func() {
			// The loopback device is detached by itself upon
			// success (TODO: check why it happens), so we
			// ignore errors here
			framework.ExecSimple(nodeExecutor, "losetup", "-d", devPath)
		})

		It("Should be handled inside the VM", func() {
			vm = makeVolumeMountVM(virtletNodeName, func(pod *framework.PodInterface) {
				addFlexvolMount(pod, "blockdev", "/foo", map[string]string{
					"type": "raw",
					"path": devPath,
					"part": "0",
				})
			})
			ssh = waitSSH(vm)
			shouldBeMounted(ssh, "/foo")
		})

		It("Should be handled inside the VM together with another volume mount", func() {
			vm = makeVolumeMountVM(virtletNodeName, func(pod *framework.PodInterface) {
				addFlexvolMount(pod, "blockdev1", "/foo", map[string]string{
					"type": "raw",
					"path": devPath,
					"part": "0",
				})
				addFlexvolMount(pod, "blockdev2", "/bar", map[string]string{
					"type":     "qcow2",
					"capacity": "10MB",
				})
			})
			ssh = waitSSH(vm)
			shouldBeMounted(ssh, "/foo")
			shouldBeMounted(ssh, "/bar")
		})
	})

	Context("Of ephemeral volumes", func() {
		var (
			vm  *framework.VMInterface
			ssh framework.Executor
		)

		BeforeAll(func() {
			vm = makeVolumeMountVM("", func(pod *framework.PodInterface) {
				addFlexvolMount(pod, "blockdev", "/foo", map[string]string{
					"type":     "qcow2",
					"capacity": "10MB",
				})
			})
		})

		AfterAll(func() {
			deleteVM(vm)
		})

		scheduleWaitSSH(&vm, &ssh)
		It("Should be handled inside the VM [Conformance]", func() {
			shouldBeMounted(ssh, "/foo")
		})
	})

	Context("ConfigMap files must be mounted", func() {
		var vm *framework.VMInterface

		BeforeAll(func() {
			requireCloudInit()
			cm := &v1.ConfigMap{
				ObjectMeta: metav1.ObjectMeta{
					Name: "files-cm",
				},
				Data: map[string]string{
					"file1": "Hello ",
					"file2": "world!",
				},
			}
			_, err := controller.ConfigMaps().Create(cm)
			Expect(err).NotTo(HaveOccurred())

			vm = controller.VM("cm-files")
			podCustomization := func(pod *framework.PodInterface) {
				pod.Pod.Spec.Containers[0].VolumeMounts = []v1.VolumeMount{
					{
						Name:      "vol",
						MountPath: "/tmp/vol",
					},
				}
				pod.Pod.Spec.Volumes = append(pod.Pod.Spec.Volumes, v1.Volume{
					Name: "vol",
					VolumeSource: v1.VolumeSource{
						ConfigMap: &v1.ConfigMapVolumeSource{
							LocalObjectReference: v1.LocalObjectReference{
								Name: "files-cm",
							},
							Items: []v1.KeyToPath{
								{
									Path: "test-file1",
									Key:  "file1",
								},
								{
									Path: "test-file2",
									Key:  "file2",
								},
							},
						},
					},
				})
			}

			vm.Create(VMOptions{}.applyDefaults(), time.Minute*5, podCustomization)
		})

		AfterAll(func() {
			deleteVM(vm)
			controller.ConfigMaps().Delete("files-cm", nil)
		})

		It("And files must be found on VM [Conformance]", func() {
			ssh := waitSSH(vm)
			Expect(framework.ExecSimple(ssh, "cat", "/tmp/vol/test-file1", "/tmp/vol/test-file2")).To(Equal("Hello world!"))
		})
	})

	Context("Secret files must be mounted", func() {
		var vm *framework.VMInterface

		BeforeAll(func() {
			requireCloudInit()
			secret := &v1.Secret{
				ObjectMeta: metav1.ObjectMeta{
					Name: "files-secret",
				},
				StringData: map[string]string{
					"file1": "Hello ",
					"file2": "world!",
				},
			}
			_, err := controller.Secrets().Create(secret)
			Expect(err).NotTo(HaveOccurred())

			vm = controller.VM("secret-files")
			podCustomization := func(pod *framework.PodInterface) {
				pod.Pod.Spec.Containers[0].VolumeMounts = []v1.VolumeMount{
					{
						Name:      "vol",
						MountPath: "/tmp/vol",
					},
				}
				pod.Pod.Spec.Volumes = append(pod.Pod.Spec.Volumes, v1.Volume{
					Name: "vol",
					VolumeSource: v1.VolumeSource{
						Secret: &v1.SecretVolumeSource{
							SecretName: "files-secret",
							Items: []v1.KeyToPath{
								{
									Path: "test-file1",
									Key:  "file1",
								},
								{
									Path: "test-file2",
									Key:  "file2",
								},
							},
						},
					},
				})
			}

			vm.Create(VMOptions{}.applyDefaults(), time.Minute*5, podCustomization)
		})

		AfterAll(func() {
			deleteVM(vm)
			controller.Secrets().Delete("files-secret", nil)
		})

		It("And files must be found on VM [Conformance]", func() {
			ssh := waitSSH(vm)
			Expect(framework.ExecSimple(ssh, "cat", "/tmp/vol/test-file1", "/tmp/vol/test-file2")).To(Equal("Hello world!"))
		})
	})

})

func addFlexvolMount(pod *framework.PodInterface, name string, mountPath string, flexvolOptions map[string]string) {
	pod.Pod.Spec.Volumes = append(pod.Pod.Spec.Volumes, v1.Volume{
		Name: name,
		VolumeSource: v1.VolumeSource{
			FlexVolume: &v1.FlexVolumeSource{
				Driver:  "virtlet/flexvolume_driver",
				Options: flexvolOptions,
			},
		},
	})
	pod.Pod.Spec.Containers[0].VolumeMounts = append(pod.Pod.Spec.Containers[0].VolumeMounts, v1.VolumeMount{
		Name:      name,
		MountPath: mountPath,
	})
}

func makeVolumeMountVM(nodeName string, podCustomization func(*framework.PodInterface)) *framework.VMInterface {
	vm := controller.VM("mount-vm")
	vm.Create(VMOptions{
		NodeName: nodeName,
		// TODO: should also have an option to test using
		// ubuntu image with volumes mounted using cloud-init
		// userdata 'mounts' section
		UserDataScript: "@virtlet-mount-script@",
	}.applyDefaults(), time.Minute*5, podCustomization)
	_, err := vm.Pod()
	Expect(err).NotTo(HaveOccurred())
	return vm
}

func shouldBeMounted(ssh framework.Executor, path string) {
	Eventually(func() (string, error) {
		return framework.ExecSimple(ssh, "ls -l "+path)
	}, 60).Should(ContainSubstring("lost+found"))
}
<|endoftext|>


<commit_before>package main

import (
	"eaciit/gdrj/model"
	"eaciit/gdrj/modules"
	"os"

	"github.com/eaciit/dbox"
	"github.com/eaciit/orm/v1"
	"github.com/eaciit/toolkit"
    "time"
	"strings"
)

var conn dbox.IConnection
var count int

func setinitialconnection() {
	var err error
    conn, err = modules.GetDboxIConnection("db_godrej")

	if err != nil {
		toolkit.Println("Initial connection found : ", err)
		os.Exit(1)
	}

	err = gdrj.SetDb(conn)
	if err != nil {
		toolkit.Println("Initial connection found : ", err)
		os.Exit(1)
	}
}

var (
    pcs = toolkit.M{}
    ccs = toolkit.M{}
    ledgers = toolkit.M{}
    prods = toolkit.M{}
    custs = toolkit.M{}
    vdistskus = toolkit.M{}
)

func getCursor(obj orm.IModel)dbox.ICursor{
    c, e := gdrj.Find(obj,nil,nil)
    if e!=nil{
        return nil
    }
    return c
}

func prepMaster(){
    pc:=new(gdrj.ProfitCenter)
    cc:=new(gdrj.CostCenter)
    prod:=new(gdrj.Product)
    ledger:=new(gdrj.LedgerMaster)

    cpc := getCursor(pc)
    defer cpc.Close()
    var e error
    for e=cpc.Fetch(pc,1,false);e==nil;{
        pcs.Set(pc.ID,pc)
        pc =new(gdrj.ProfitCenter)
        e=cpc.Fetch(pc,1,false)
    }

    ccc:=getCursor(cc)
    defer ccc.Close()
    for e=ccc.Fetch(cc,1,false);e==nil;{
        ccs.Set(cc.ID,cc)
        cc = new(gdrj.CostCenter)
        e=ccc.Fetch(cc,1,false)
    }

    cprod:=getCursor(prod)
    defer cprod.Close()
    for e=cprod.Fetch(prod,1,false);e==nil;{
        prods.Set(prod.ID,prod)
        prod=new(gdrj.Product)
        e=cprod.Fetch(prod,1,false)
    }

    cledger:=getCursor(ledger)
    defer cledger.Close()
    for e=cledger.Fetch(ledger,1,false);e==nil;{
        ledgers.Set(ledger.ID,ledger)
        ledger=new(gdrj.LedgerMaster)
        e=cledger.Fetch(ledger,1,false)
    }

    cust := new(gdrj.Customer)
    ccust:=getCursor(cust)
    defer ccust.Close()
    for e=ccust.Fetch(cust,1,false);e==nil;{
        custs.Set(cust.ID,cust)
        cust=new(gdrj.Customer)
        e=ccust.Fetch(cust,1,false)
    }

    sku:=new(gdrj.MappingInventory)
    cskus:=getCursor(sku)
    defer cskus.Close()
    for e=cskus.Fetch(sku,1,false);e==nil;{
        vdistskus.Set(sku.SKUID_VDIST,sku.ID)
        e=cskus.Fetch(sku,1,false)
    }
}

func main() {
	//runtime.GOMAXPROCS(runtime.NumCPU())
	setinitialconnection()
	defer gdrj.CloseDb()

    toolkit.Println("Reading Master")
    prepMaster()

    toolkit.Println("START...")

	//for i, src := range arrstring {
	//dbf := dbox.Contains("src", src)
	crx, err := gdrj.Find(new(gdrj.SalesHeader), dbox.Ne("outletid",""),
        toolkit.M{})
	if err != nil {
		toolkit.Println("Error Found : ", err.Error())
		os.Exit(1)
	}
    defer crx.Close()

    toolkit.Printf("Populating header")
    count = crx.Count()
    var e error
    i:=0
    t0 := time.Now()
    var shs []gdrj.SalesHeader
    e=crx.Fetch(&shs,0,false);
    if e!=nil {
        toolkit.Printfn("End: %s", e.Error())
    } else {
        toolkit.Printfn("Processing line")
        for _, sh := range shs{
            i++
            toolkit.Printfn("%d of %d: %s in %s", i, count, sh.ID, time.Since(t0).String())
            processHeader(&sh)
        }
    }
	toolkit.Println("END...")
}

var isales int

func processHeader(sh *gdrj.SalesHeader){
    sds:=[]*gdrj.SalesDetail{}
    var sdsi []toolkit.M

    isales++
    filter := dbox.Eq("salesheaderid",sh.ID)
    sh.OutletID = sh.BranchID + sh.OutletID
    crs, ecrs :=gdrj.Find(new(gdrj.SalesDetail), filter, nil)
    if ecrs!=nil{
        return
    }
    defer crs.Close()
    ecrs = crs.Fetch(&sds,0,false)

    //toolkit.Printfn("%d of %d - Processing %s", isales, count, sh.ID)
    func(){
        filter = dbox.Eq("SalesheaderID",sh.ID)
        crs, ecrs :=conn.NewQuery().From("rawsalesdetail_import").Where(filter).Cursor(nil)
        if ecrs!=nil{
            return
        }
        defer crs.Close()
        crs.Fetch(&sdsi,0,false)

        for _, si := range sdsi{
            sd := gdrj.SalesDetail{}
            skuid := toolkit.ToString(si.GetInt("SKUID_VDIST"))
            if !vdistskus.Has(skuid){
                continue
            }
            sd.SKUID_SAPBI = vdistskus.GetString(skuid)
            sd.BranchID = si.GetString("BranchID")
            sd.SalesQty = si.GetInt("SalesQty")
            sd.Price = si.GetFloat64("Price")
            sd.SalesGrossAmount = si.GetFloat64("SalesGrossAmount")
            sd.SalesNetAmount = si.GetFloat64("SalesNetAmount")
            sds = append(sds,&sd)
        }
    }()

    totalAmount := float64(0)
    for _, sd := range sds{
        totalAmount += sd.SalesNetAmount
    }

    //-- alloc disc
    lss := map[string]*gdrj.LedgerSummary{}

    for _, sd := range sds{
        if sh.SalesDiscountAmount != 0 {
            sd.AllocDiscAmount = sd.SalesNetAmount * sh.SalesDiscountAmount / totalAmount
        }
        if sh.SalesTaxAmount != 0 {
            sd.AllocTaxAmount = sd.SalesNetAmount * sh.SalesTaxAmount / totalAmount
        }

        gdate := gdrj.NewDate(sh.Date.Year(), int(sh.Date.Month()), sh.Date.Day())
        updatels := new(gdrj.LedgerSummary)
        updatels.CompanyCode = "ID11"
        updatels.Date = gdate
        updatels.Year = gdate.Year
        updatels.Month = gdate.Month
        updatels.SKUID = sd.SKUID_SAPBI
        updatels.OutletID = sh.OutletID
        //updatels.Value1 = sd.SalesGrossAmount
        //updatels.Value2 = float64(sd.SalesQty)

        lss_id_gross := toolkit.Sprintf("%d_%d_%s_%s_gross",  updatels.Year, updatels.Month, updatels.OutletID, updatels.SKUID)
        lss_id_return := toolkit.Sprintf("%d_%d_%s_%s_return",  updatels.Year, updatels.Month, updatels.OutletID, updatels.SKUID)
        lss_id_discount := toolkit.Sprintf("%d_%d_%s_%s_discount",  updatels.Year, updatels.Month, updatels.OutletID, updatels.SKUID)
        //lss_id_tax := toolkit.Sprintf("%d_%d_%s_%s_tax",  updatels.Year, updatels.Month, updatels.OutletID, updatels.SKUID)

        if sd.SalesGrossAmount<0{
            ls := findSales(&lss, lss_id_return, updatels)
            if ls==nil {
                continue
            }
            ls.Value1 += sd.SalesGrossAmount
            ls.Value3 += float64(sd.SalesQty)
        } else if sd.SalesGrossAmount>0 {
            ls := findSales(&lss, lss_id_gross, updatels)
            if ls==nil {
                continue
            }
            ls.Value1 += sd.SalesGrossAmount
            ls.Value3 += float64(sd.SalesQty)
        }

        if sd.AllocDiscAmount != 0{
            ls := findSales(&lss, lss_id_discount, updatels)
            if ls==nil {
                continue
            }
            ls.Value1 += sd.AllocDiscAmount
            ls.Value3 += float64(sd.SalesQty)
        }

       /*
       if sd.AllocTaxAmount != 0{
            ls := findSales(&lss, lss_id_tax)
            ls.Value1 += sd.AllocTaxAmount
            ls.Value3 += float64(sd.SalesQty)
        }
        */
    }

    for _, ls := range lss{
        toolkit.Printfn("Saving %s %s %s", ls.LedgerAccount, ls.Customer.Name, ls.Product.Name)
        gdrj.Save(ls)
    }
    //toolkit.Printfn("%s Length: %d 5s",sh.ID,len(sds),toolkit.JsonString(lss))
}

func findSales(lss *map[string]*gdrj.LedgerSummary, id string, defls *gdrj.LedgerSummary)*gdrj.LedgerSummary{
    var (
        ls *gdrj.LedgerSummary
        b bool
    )
    lssv := *lss
    if ls, b = lssv[id]; !b{
        var (
            prod *gdrj.Product
            cust *gdrj.Customer
            pcid string
        )

        if !prods.Has(defls.SKUID){
            toolkit.Printfn("Product %s is not exist", defls.SKUID)
            return nil
        }

        if !custs.Has(defls.OutletID){
            toolkit.Printfn("Outlet %s is not exist", defls.OutletID)
            return nil
        }

        prod = prods[defls.SKUID].(*gdrj.Product)
        cust = custs[defls.OutletID].(*gdrj.Customer)
        pcid = cust.BranchID + prod.BrandCategoryID

        lsnew := gdrj.GetLedgerSummaryByDetail(defls.LedgerAccount,pcid,"",defls.OutletID,defls.SKUID,
            defls.Year,defls.Month)
        if lsnew.ID=="" {
            ls = defls
        } else {
            ls = lsnew
        }

        if !pcs.Has(pcid){
            toolkit.Printfn("PC %s is not exist", pcid)
            return nil
        }

        ls.PCID = pcid
        ls.PC = pcs[ls.PCID].(*gdrj.ProfitCenter)
        ls.Product = prod
        ls.Customer = cust
        if strings.HasSuffix(id, "gross"){
            ls.LedgerAccount = "70000000"
        }  else if strings.HasSuffix(id, "return"){
            ls.LedgerAccount = "70000302"
        }  else if strings.HasSuffix(id, "tax"){
        } else if strings.HasSuffix(id, "discount"){
            ls.LedgerAccount = "75053730"
        }
        /*
        if !ledgers.Has(defls.LedgerAccount){
            toolkit.Printfn("Ledger %s id %s is not exist", defls.LedgerAccount, id)
            return nil
        }
        */
        ledg := ledgers.Get(ls.LedgerAccount).(*gdrj.LedgerMaster)
        ls.PLCode = ledg.PLCode
        ls.PLOrder = ledg.OrderIndex
        ls.PLGroup1 = ledg.H1
        ls.PLGroup2 = ledg.H2
        ls.PLGroup3 = ledg.H3
        lssv[id]=ls
    }
    //toolkit.Printfn("Findls: %s",toolkit.JsonString(ls))
    *lss = lssv
    return ls
}<commit_msg>check speed<commit_after>package main

import (
	"eaciit/gdrj/model"
	"eaciit/gdrj/modules"
	"os"

	"github.com/eaciit/dbox"
	"github.com/eaciit/orm/v1"
	"github.com/eaciit/toolkit"
    "time"
)

var conn dbox.IConnection
var count int

func setinitialconnection() {
	var err error
    conn, err = modules.GetDboxIConnection("db_godrej")

	if err != nil {
		toolkit.Println("Initial connection found : ", err)
		os.Exit(1)
	}

	err = gdrj.SetDb(conn)
	if err != nil {
		toolkit.Println("Initial connection found : ", err)
		os.Exit(1)
	}
}

func getCursor(obj orm.IModel)dbox.ICursor{
    c, e := gdrj.Find(obj,nil,nil)
    if e!=nil{
        return nil
    }
    return c
}

func main(){
    salestrx := new(gdrj.SalesTrx)
    c, e:= gdrj.Find(salestrx,nil,nil)
    defer c.Close()

    i:=0
    count:=c.Count()
    step:=count/1000
    limit:=step

    t0 := time.Now()
    for {
        salestrx = new(gdrj.SalesTrx)
        e = c.Fetch(salestrx,1,false)
        if e!=nil{
            break
        }

        i++
        if i>=limit{
            toolkit.Printfn("%d of %d [%.2f pct] in %s",
                i,count,float64(i)*100.0/float64(count),
                time.Since(t0).String())
            limit+=step
        }
    }
}<|endoftext|>


<commit_before>// Copyright 2016 The etcd Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package command

import (
	"encoding/binary"
	"encoding/json"
	"fmt"
	"io"
	"os"
	"os/exec"
	"path/filepath"
	"time"

	pb "go.etcd.io/etcd/api/v3/etcdserverpb"
	"go.etcd.io/etcd/api/v3/mvccpb"
	"go.etcd.io/etcd/client/pkg/v3/types"
	"go.etcd.io/etcd/client/v2"
	"go.etcd.io/etcd/pkg/v3/pbutil"
	"go.etcd.io/etcd/raft/v3/raftpb"
	"go.etcd.io/etcd/server/v3/etcdserver"
	"go.etcd.io/etcd/server/v3/etcdserver/api"
	"go.etcd.io/etcd/server/v3/etcdserver/api/membership"
	"go.etcd.io/etcd/server/v3/etcdserver/api/snap"
	"go.etcd.io/etcd/server/v3/etcdserver/api/v2error"
	"go.etcd.io/etcd/server/v3/etcdserver/api/v2store"
	"go.etcd.io/etcd/server/v3/etcdserver/cindex"
	"go.etcd.io/etcd/server/v3/mvcc"
	"go.etcd.io/etcd/server/v3/mvcc/backend"
	"go.etcd.io/etcd/server/v3/wal"
	"go.etcd.io/etcd/server/v3/wal/walpb"

	"github.com/gogo/protobuf/proto"
	"github.com/spf13/cobra"
	"go.uber.org/zap"
)

var (
	migrateExcludeTTLKey bool
	migrateDatadir       string
	migrateWALdir        string
	migrateTransformer   string
)

// NewMigrateCommand returns the cobra command for "migrate".
func NewMigrateCommand() *cobra.Command {
	mc := &cobra.Command{
		Use:   "migrate",
		Short: "Migrates keys in a v2 store to a mvcc store",
		Run:   migrateCommandFunc,
	}

	mc.Flags().BoolVar(&migrateExcludeTTLKey, "no-ttl", false, "Do not convert TTL keys")
	mc.Flags().StringVar(&migrateDatadir, "data-dir", "", "Path to the data directory")
	mc.Flags().StringVar(&migrateWALdir, "wal-dir", "", "Path to the WAL directory")
	mc.Flags().StringVar(&migrateTransformer, "transformer", "", "Path to the user-provided transformer program")
	return mc
}

func migrateCommandFunc(cmd *cobra.Command, args []string) {
	var (
		writer io.WriteCloser
		reader io.ReadCloser
		errc   chan error
	)
	if migrateTransformer != "" {
		writer, reader, errc = startTransformer()
	} else {
		fmt.Println("using default transformer")
		writer, reader, errc = defaultTransformer()
	}

	st, index := rebuildStoreV2()
	be := prepareBackend()
	defer be.Close()

	go func() {
		writeStore(writer, st)
		writer.Close()
	}()

	readKeys(reader, be)
	cindex.UpdateConsistentIndex(be.BatchTx(), index)
	err := <-errc
	if err != nil {
		fmt.Println("failed to transform keys")
		ExitWithError(ExitError, err)
	}

	fmt.Println("finished transforming keys")
}

func prepareBackend() backend.Backend {
	var be backend.Backend

	bch := make(chan struct{})
	dbpath := filepath.Join(migrateDatadir, "member", "snap", "db")
	go func() {
		defer close(bch)
		be = backend.NewDefaultBackend(dbpath)

	}()
	select {
	case <-bch:
	case <-time.After(time.Second):
		fmt.Fprintf(os.Stderr, "waiting for etcd to close and release its lock on %q\n", dbpath)
		<-bch
	}

	tx := be.BatchTx()
	tx.Lock()
	tx.UnsafeCreateBucket([]byte("key"))
	tx.UnsafeCreateBucket([]byte("meta"))
	tx.Unlock()
	return be
}

func rebuildStoreV2() (v2store.Store, uint64) {
	var index uint64
	cl := membership.NewCluster(zap.NewExample())

	waldir := migrateWALdir
	if len(waldir) == 0 {
		waldir = filepath.Join(migrateDatadir, "member", "wal")
	}
	snapdir := filepath.Join(migrateDatadir, "member", "snap")

	ss := snap.New(zap.NewExample(), snapdir)
	snapshot, err := ss.Load()
	if err != nil && err != snap.ErrNoSnapshot {
		ExitWithError(ExitError, err)
	}

	var walsnap walpb.Snapshot
	if snapshot != nil {
		walsnap.Index, walsnap.Term = snapshot.Metadata.Index, snapshot.Metadata.Term
		index = snapshot.Metadata.Index
	}

	w, err := wal.OpenForRead(zap.NewExample(), waldir, walsnap)
	if err != nil {
		ExitWithError(ExitError, err)
	}
	defer w.Close()

	_, _, ents, err := w.ReadAll()
	if err != nil {
		ExitWithError(ExitError, err)
	}

	st := v2store.New()
	if snapshot != nil {
		err := st.Recovery(snapshot.Data)
		if err != nil {
			ExitWithError(ExitError, err)
		}
	}

	cl.SetStore(st)
	cl.Recover(api.UpdateCapability)

	applier := etcdserver.NewApplierV2(zap.NewExample(), st, cl)
	for _, ent := range ents {
		if ent.Type == raftpb.EntryConfChange {
			var cc raftpb.ConfChange
			pbutil.MustUnmarshal(&cc, ent.Data)
			applyConf(cc, cl)
			continue
		}

		var raftReq pb.InternalRaftRequest
		if !pbutil.MaybeUnmarshal(&raftReq, ent.Data) { // backward compatible
			var r pb.Request
			pbutil.MustUnmarshal(&r, ent.Data)
			applyRequest(&r, applier)
		} else {
			if raftReq.V2 != nil {
				req := raftReq.V2
				applyRequest(req, applier)
			}
		}
		if ent.Index > index {
			index = ent.Index
		}
	}

	return st, index
}

func applyConf(cc raftpb.ConfChange, cl *membership.RaftCluster) {
	if err := cl.ValidateConfigurationChange(cc); err != nil {
		return
	}
	switch cc.Type {
	case raftpb.ConfChangeAddNode:
		m := new(membership.Member)
		if err := json.Unmarshal(cc.Context, m); err != nil {
			panic(err)
		}
		cl.AddMember(m, true)
	case raftpb.ConfChangeRemoveNode:
		cl.RemoveMember(types.ID(cc.NodeID), true)
	case raftpb.ConfChangeUpdateNode:
		m := new(membership.Member)
		if err := json.Unmarshal(cc.Context, m); err != nil {
			panic(err)
		}
		cl.UpdateRaftAttributes(m.ID, m.RaftAttributes, true)
	}
}

func applyRequest(req *pb.Request, applyV2 etcdserver.ApplierV2) {
	r := (*etcdserver.RequestV2)(req)
	r.TTLOptions()
	switch r.Method {
	case "POST":
		applyV2.Post(r)
	case "PUT":
		applyV2.Put(r)
	case "DELETE":
		applyV2.Delete(r)
	case "QGET":
		applyV2.QGet(r)
	case "SYNC":
		applyV2.Sync(r)
	default:
		panic("unknown command")
	}
}

func writeStore(w io.Writer, st v2store.Store) uint64 {
	all, err := st.Get("/1", true, true)
	if err != nil {
		if eerr, ok := err.(*v2error.Error); ok && eerr.ErrorCode == v2error.EcodeKeyNotFound {
			fmt.Println("no v2 keys to migrate")
			os.Exit(0)
		}
		ExitWithError(ExitError, err)
	}
	return writeKeys(w, all.Node)
}

func writeKeys(w io.Writer, n *v2store.NodeExtern) uint64 {
	maxIndex := n.ModifiedIndex

	nodes := n.Nodes
	// remove store v2 bucket prefix
	n.Key = n.Key[2:]
	if n.Key == "" {
		n.Key = "/"
	}
	if n.Dir {
		n.Nodes = nil
	}
	if !migrateExcludeTTLKey || n.TTL == 0 {
		b, err := json.Marshal(n)
		if err != nil {
			ExitWithError(ExitError, err)
		}
		fmt.Fprint(w, string(b))
	}
	for _, nn := range nodes {
		max := writeKeys(w, nn)
		if max > maxIndex {
			maxIndex = max
		}
	}
	return maxIndex
}

func readKeys(r io.Reader, be backend.Backend) error {
	for {
		length64, err := readInt64(r)
		if err != nil {
			if err == io.EOF {
				return nil
			}
			return err
		}

		buf := make([]byte, int(length64))
		if _, err = io.ReadFull(r, buf); err != nil {
			return err
		}

		var kv mvccpb.KeyValue
		err = proto.Unmarshal(buf, &kv)
		if err != nil {
			return err
		}

		mvcc.WriteKV(be, kv)
	}
}

func readInt64(r io.Reader) (int64, error) {
	var n int64
	err := binary.Read(r, binary.LittleEndian, &n)
	return n, err
}

func startTransformer() (io.WriteCloser, io.ReadCloser, chan error) {
	cmd := exec.Command(migrateTransformer)
	cmd.Stderr = os.Stderr

	writer, err := cmd.StdinPipe()
	if err != nil {
		ExitWithError(ExitError, err)
	}

	reader, rerr := cmd.StdoutPipe()
	if rerr != nil {
		ExitWithError(ExitError, rerr)
	}

	if err := cmd.Start(); err != nil {
		ExitWithError(ExitError, err)
	}

	errc := make(chan error, 1)

	go func() {
		errc <- cmd.Wait()
	}()

	return writer, reader, errc
}

func defaultTransformer() (io.WriteCloser, io.ReadCloser, chan error) {
	// transformer decodes v2 keys from sr
	sr, sw := io.Pipe()
	// transformer encodes v3 keys into dw
	dr, dw := io.Pipe()

	decoder := json.NewDecoder(sr)

	errc := make(chan error, 1)

	go func() {
		defer func() {
			sr.Close()
			dw.Close()
		}()

		for decoder.More() {
			node := &client.Node{}
			if err := decoder.Decode(node); err != nil {
				errc <- err
				return
			}

			kv := transform(node)
			if kv == nil {
				continue
			}

			data, err := proto.Marshal(kv)
			if err != nil {
				errc <- err
				return
			}
			buf := make([]byte, 8)
			binary.LittleEndian.PutUint64(buf, uint64(len(data)))
			if _, err := dw.Write(buf); err != nil {
				errc <- err
				return
			}
			if _, err := dw.Write(data); err != nil {
				errc <- err
				return
			}
		}

		errc <- nil
	}()

	return sw, dr, errc
}

func transform(n *client.Node) *mvccpb.KeyValue {
	const unKnownVersion = 1
	if n.Dir {
		return nil
	}
	kv := &mvccpb.KeyValue{
		Key:            []byte(n.Key),
		Value:          []byte(n.Value),
		CreateRevision: int64(n.CreatedIndex),
		ModRevision:    int64(n.ModifiedIndex),
		Version:        unKnownVersion,
	}
	return kv
}
<commit_msg>fixup! Unify shared code (and constants) with cindex package.<commit_after>// Copyright 2016 The etcd Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package command

import (
	"encoding/binary"
	"encoding/json"
	"fmt"
	"io"
	"os"
	"os/exec"
	"path/filepath"
	"time"

	pb "go.etcd.io/etcd/api/v3/etcdserverpb"
	"go.etcd.io/etcd/api/v3/mvccpb"
	"go.etcd.io/etcd/client/pkg/v3/types"
	"go.etcd.io/etcd/client/v2"
	"go.etcd.io/etcd/pkg/v3/pbutil"
	"go.etcd.io/etcd/raft/v3/raftpb"
	"go.etcd.io/etcd/server/v3/etcdserver"
	"go.etcd.io/etcd/server/v3/etcdserver/api"
	"go.etcd.io/etcd/server/v3/etcdserver/api/membership"
	"go.etcd.io/etcd/server/v3/etcdserver/api/snap"
	"go.etcd.io/etcd/server/v3/etcdserver/api/v2error"
	"go.etcd.io/etcd/server/v3/etcdserver/api/v2store"
	"go.etcd.io/etcd/server/v3/etcdserver/cindex"
	"go.etcd.io/etcd/server/v3/mvcc"
	"go.etcd.io/etcd/server/v3/mvcc/backend"
	"go.etcd.io/etcd/server/v3/wal"
	"go.etcd.io/etcd/server/v3/wal/walpb"

	"github.com/gogo/protobuf/proto"
	"github.com/spf13/cobra"
	"go.uber.org/zap"
)

var (
	migrateExcludeTTLKey bool
	migrateDatadir       string
	migrateWALdir        string
	migrateTransformer   string
)

// NewMigrateCommand returns the cobra command for "migrate".
func NewMigrateCommand() *cobra.Command {
	mc := &cobra.Command{
		Use:   "migrate",
		Short: "Migrates keys in a v2 store to a mvcc store",
		Run:   migrateCommandFunc,
	}

	mc.Flags().BoolVar(&migrateExcludeTTLKey, "no-ttl", false, "Do not convert TTL keys")
	mc.Flags().StringVar(&migrateDatadir, "data-dir", "", "Path to the data directory")
	mc.Flags().StringVar(&migrateWALdir, "wal-dir", "", "Path to the WAL directory")
	mc.Flags().StringVar(&migrateTransformer, "transformer", "", "Path to the user-provided transformer program")
	return mc
}

func migrateCommandFunc(cmd *cobra.Command, args []string) {
	var (
		writer io.WriteCloser
		reader io.ReadCloser
		errc   chan error
	)
	if migrateTransformer != "" {
		writer, reader, errc = startTransformer()
	} else {
		fmt.Println("using default transformer")
		writer, reader, errc = defaultTransformer()
	}

	st, index := rebuildStoreV2()
	be := prepareBackend()
	defer be.Close()

	go func() {
		writeStore(writer, st)
		writer.Close()
	}()

	readKeys(reader, be)
	cindex.UpdateConsistentIndex(be.BatchTx(), index, true)
	err := <-errc
	if err != nil {
		fmt.Println("failed to transform keys")
		ExitWithError(ExitError, err)
	}

	fmt.Println("finished transforming keys")
}

func prepareBackend() backend.Backend {
	var be backend.Backend

	bch := make(chan struct{})
	dbpath := filepath.Join(migrateDatadir, "member", "snap", "db")
	go func() {
		defer close(bch)
		be = backend.NewDefaultBackend(dbpath)

	}()
	select {
	case <-bch:
	case <-time.After(time.Second):
		fmt.Fprintf(os.Stderr, "waiting for etcd to close and release its lock on %q\n", dbpath)
		<-bch
	}

	tx := be.BatchTx()
	tx.Lock()
	tx.UnsafeCreateBucket([]byte("key"))
	tx.UnsafeCreateBucket([]byte("meta"))
	tx.Unlock()
	return be
}

func rebuildStoreV2() (v2store.Store, uint64) {
	var index uint64
	cl := membership.NewCluster(zap.NewExample())

	waldir := migrateWALdir
	if len(waldir) == 0 {
		waldir = filepath.Join(migrateDatadir, "member", "wal")
	}
	snapdir := filepath.Join(migrateDatadir, "member", "snap")

	ss := snap.New(zap.NewExample(), snapdir)
	snapshot, err := ss.Load()
	if err != nil && err != snap.ErrNoSnapshot {
		ExitWithError(ExitError, err)
	}

	var walsnap walpb.Snapshot
	if snapshot != nil {
		walsnap.Index, walsnap.Term = snapshot.Metadata.Index, snapshot.Metadata.Term
		index = snapshot.Metadata.Index
	}

	w, err := wal.OpenForRead(zap.NewExample(), waldir, walsnap)
	if err != nil {
		ExitWithError(ExitError, err)
	}
	defer w.Close()

	_, _, ents, err := w.ReadAll()
	if err != nil {
		ExitWithError(ExitError, err)
	}

	st := v2store.New()
	if snapshot != nil {
		err := st.Recovery(snapshot.Data)
		if err != nil {
			ExitWithError(ExitError, err)
		}
	}

	cl.SetStore(st)
	cl.Recover(api.UpdateCapability)

	applier := etcdserver.NewApplierV2(zap.NewExample(), st, cl)
	for _, ent := range ents {
		if ent.Type == raftpb.EntryConfChange {
			var cc raftpb.ConfChange
			pbutil.MustUnmarshal(&cc, ent.Data)
			applyConf(cc, cl)
			continue
		}

		var raftReq pb.InternalRaftRequest
		if !pbutil.MaybeUnmarshal(&raftReq, ent.Data) { // backward compatible
			var r pb.Request
			pbutil.MustUnmarshal(&r, ent.Data)
			applyRequest(&r, applier)
		} else {
			if raftReq.V2 != nil {
				req := raftReq.V2
				applyRequest(req, applier)
			}
		}
		if ent.Index > index {
			index = ent.Index
		}
	}

	return st, index
}

func applyConf(cc raftpb.ConfChange, cl *membership.RaftCluster) {
	if err := cl.ValidateConfigurationChange(cc); err != nil {
		return
	}
	switch cc.Type {
	case raftpb.ConfChangeAddNode:
		m := new(membership.Member)
		if err := json.Unmarshal(cc.Context, m); err != nil {
			panic(err)
		}
		cl.AddMember(m, true)
	case raftpb.ConfChangeRemoveNode:
		cl.RemoveMember(types.ID(cc.NodeID), true)
	case raftpb.ConfChangeUpdateNode:
		m := new(membership.Member)
		if err := json.Unmarshal(cc.Context, m); err != nil {
			panic(err)
		}
		cl.UpdateRaftAttributes(m.ID, m.RaftAttributes, true)
	}
}

func applyRequest(req *pb.Request, applyV2 etcdserver.ApplierV2) {
	r := (*etcdserver.RequestV2)(req)
	r.TTLOptions()
	switch r.Method {
	case "POST":
		applyV2.Post(r)
	case "PUT":
		applyV2.Put(r)
	case "DELETE":
		applyV2.Delete(r)
	case "QGET":
		applyV2.QGet(r)
	case "SYNC":
		applyV2.Sync(r)
	default:
		panic("unknown command")
	}
}

func writeStore(w io.Writer, st v2store.Store) uint64 {
	all, err := st.Get("/1", true, true)
	if err != nil {
		if eerr, ok := err.(*v2error.Error); ok && eerr.ErrorCode == v2error.EcodeKeyNotFound {
			fmt.Println("no v2 keys to migrate")
			os.Exit(0)
		}
		ExitWithError(ExitError, err)
	}
	return writeKeys(w, all.Node)
}

func writeKeys(w io.Writer, n *v2store.NodeExtern) uint64 {
	maxIndex := n.ModifiedIndex

	nodes := n.Nodes
	// remove store v2 bucket prefix
	n.Key = n.Key[2:]
	if n.Key == "" {
		n.Key = "/"
	}
	if n.Dir {
		n.Nodes = nil
	}
	if !migrateExcludeTTLKey || n.TTL == 0 {
		b, err := json.Marshal(n)
		if err != nil {
			ExitWithError(ExitError, err)
		}
		fmt.Fprint(w, string(b))
	}
	for _, nn := range nodes {
		max := writeKeys(w, nn)
		if max > maxIndex {
			maxIndex = max
		}
	}
	return maxIndex
}

func readKeys(r io.Reader, be backend.Backend) error {
	for {
		length64, err := readInt64(r)
		if err != nil {
			if err == io.EOF {
				return nil
			}
			return err
		}

		buf := make([]byte, int(length64))
		if _, err = io.ReadFull(r, buf); err != nil {
			return err
		}

		var kv mvccpb.KeyValue
		err = proto.Unmarshal(buf, &kv)
		if err != nil {
			return err
		}

		mvcc.WriteKV(be, kv)
	}
}

func readInt64(r io.Reader) (int64, error) {
	var n int64
	err := binary.Read(r, binary.LittleEndian, &n)
	return n, err
}

func startTransformer() (io.WriteCloser, io.ReadCloser, chan error) {
	cmd := exec.Command(migrateTransformer)
	cmd.Stderr = os.Stderr

	writer, err := cmd.StdinPipe()
	if err != nil {
		ExitWithError(ExitError, err)
	}

	reader, rerr := cmd.StdoutPipe()
	if rerr != nil {
		ExitWithError(ExitError, rerr)
	}

	if err := cmd.Start(); err != nil {
		ExitWithError(ExitError, err)
	}

	errc := make(chan error, 1)

	go func() {
		errc <- cmd.Wait()
	}()

	return writer, reader, errc
}

func defaultTransformer() (io.WriteCloser, io.ReadCloser, chan error) {
	// transformer decodes v2 keys from sr
	sr, sw := io.Pipe()
	// transformer encodes v3 keys into dw
	dr, dw := io.Pipe()

	decoder := json.NewDecoder(sr)

	errc := make(chan error, 1)

	go func() {
		defer func() {
			sr.Close()
			dw.Close()
		}()

		for decoder.More() {
			node := &client.Node{}
			if err := decoder.Decode(node); err != nil {
				errc <- err
				return
			}

			kv := transform(node)
			if kv == nil {
				continue
			}

			data, err := proto.Marshal(kv)
			if err != nil {
				errc <- err
				return
			}
			buf := make([]byte, 8)
			binary.LittleEndian.PutUint64(buf, uint64(len(data)))
			if _, err := dw.Write(buf); err != nil {
				errc <- err
				return
			}
			if _, err := dw.Write(data); err != nil {
				errc <- err
				return
			}
		}

		errc <- nil
	}()

	return sw, dr, errc
}

func transform(n *client.Node) *mvccpb.KeyValue {
	const unKnownVersion = 1
	if n.Dir {
		return nil
	}
	kv := &mvccpb.KeyValue{
		Key:            []byte(n.Key),
		Value:          []byte(n.Value),
		CreateRevision: int64(n.CreatedIndex),
		ModRevision:    int64(n.ModifiedIndex),
		Version:        unKnownVersion,
	}
	return kv
}
<|endoftext|>


<commit_before>package managesystemagent

import (
	"github.com/rancher/fleet/pkg/apis/fleet.cattle.io/v1alpha1"
	v3 "github.com/rancher/rancher/pkg/apis/management.cattle.io/v3"
	rancherv1 "github.com/rancher/rancher/pkg/apis/provisioning.cattle.io/v1"
	namespaces "github.com/rancher/rancher/pkg/namespace"
	"github.com/rancher/rancher/pkg/settings"
	"github.com/rancher/wrangler/pkg/name"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/runtime"
)

func (h *handler) OnChangeInstallSUC(cluster *rancherv1.Cluster, status rancherv1.ClusterStatus) ([]runtime.Object, rancherv1.ClusterStatus, error) {
	if cluster.Spec.RKEConfig == nil {
		return nil, status, nil
	}

	mcc := &v3.ManagedChart{
		ObjectMeta: metav1.ObjectMeta{
			Namespace: cluster.Namespace,
			Name:      name.SafeConcatName(cluster.Name, "managed", "system-upgrade-controller"),
		},
		Spec: v3.ManagedChartSpec{
			DefaultNamespace: namespaces.System,
			RepoName:         "rancher-charts",
			Chart:            "system-upgrade-controller",
			Version:          "0.7.001",
			Values: &v1alpha1.GenericMap{
				Data: map[string]interface{}{
					"global": map[string]interface{}{
						"systemDefaultRegistry": settings.SystemDefaultRegistry.Get(),
					},
				},
			},
			Targets: []v1alpha1.BundleTarget{
				{
					ClusterName: cluster.Name,
					ClusterSelector: &metav1.LabelSelector{
						MatchExpressions: []metav1.LabelSelectorRequirement{
							{
								Key:      "provisioning.cattle.io/unmanaged-system-agent",
								Operator: metav1.LabelSelectorOpDoesNotExist,
							},
						},
					},
				},
			},
		},
	}

	return []runtime.Object{
		mcc,
	}, status, nil
}
<commit_msg>Update to new system-upgrade-controller chart version<commit_after>package managesystemagent

import (
	"github.com/rancher/fleet/pkg/apis/fleet.cattle.io/v1alpha1"
	v3 "github.com/rancher/rancher/pkg/apis/management.cattle.io/v3"
	rancherv1 "github.com/rancher/rancher/pkg/apis/provisioning.cattle.io/v1"
	namespaces "github.com/rancher/rancher/pkg/namespace"
	"github.com/rancher/rancher/pkg/settings"
	"github.com/rancher/wrangler/pkg/name"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/runtime"
)

func (h *handler) OnChangeInstallSUC(cluster *rancherv1.Cluster, status rancherv1.ClusterStatus) ([]runtime.Object, rancherv1.ClusterStatus, error) {
	if cluster.Spec.RKEConfig == nil {
		return nil, status, nil
	}

	mcc := &v3.ManagedChart{
		ObjectMeta: metav1.ObjectMeta{
			Namespace: cluster.Namespace,
			Name:      name.SafeConcatName(cluster.Name, "managed", "system-upgrade-controller"),
		},
		Spec: v3.ManagedChartSpec{
			DefaultNamespace: namespaces.System,
			RepoName:         "rancher-charts",
			Chart:            "system-upgrade-controller",
			Version:          "100.0.0",
			Values: &v1alpha1.GenericMap{
				Data: map[string]interface{}{
					"global": map[string]interface{}{
						"systemDefaultRegistry": settings.SystemDefaultRegistry.Get(),
					},
				},
			},
			Targets: []v1alpha1.BundleTarget{
				{
					ClusterName: cluster.Name,
					ClusterSelector: &metav1.LabelSelector{
						MatchExpressions: []metav1.LabelSelectorRequirement{
							{
								Key:      "provisioning.cattle.io/unmanaged-system-agent",
								Operator: metav1.LabelSelectorOpDoesNotExist,
							},
						},
					},
				},
			},
		},
	}

	return []runtime.Object{
		mcc,
	}, status, nil
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"strconv"
	"strings"

	"github.com/lxc/lxd/lxd/device"
	"github.com/lxc/lxd/shared"
)

var networkConfigKeys = map[string]func(value string) error{
	"bridge.driver": func(value string) error {
		return shared.IsOneOf(value, []string{"native", "openvswitch"})
	},
	"bridge.external_interfaces": func(value string) error {
		if value == "" {
			return nil
		}

		for _, entry := range strings.Split(value, ",") {
			entry = strings.TrimSpace(entry)
			if networkValidName(entry) != nil {
				return fmt.Errorf("Invalid interface name '%s'", entry)
			}
		}

		return nil
	},
	"bridge.hwaddr": shared.IsAny,
	"bridge.mtu":    shared.IsInt64,
	"bridge.mode": func(value string) error {
		return shared.IsOneOf(value, []string{"standard", "fan"})
	},

	"fan.overlay_subnet": device.NetworkValidNetworkV4,
	"fan.underlay_subnet": func(value string) error {
		if value == "auto" {
			return nil
		}

		return device.NetworkValidNetworkV4(value)
	},
	"fan.type": func(value string) error {
		return shared.IsOneOf(value, []string{"vxlan", "ipip"})
	},

	"tunnel.TARGET.protocol": func(value string) error {
		return shared.IsOneOf(value, []string{"gre", "vxlan"})
	},
	"tunnel.TARGET.local":     networkValidAddress,
	"tunnel.TARGET.remote":    networkValidAddress,
	"tunnel.TARGET.port":      networkValidPort,
	"tunnel.TARGET.group":     networkValidAddress,
	"tunnel.TARGET.id":        shared.IsInt64,
	"tunnel.TARGET.interface": networkValidName,
	"tunnel.TARGET.ttl":       shared.IsUint8,

	"ipv4.address": func(value string) error {
		if shared.IsOneOf(value, []string{"none", "auto"}) == nil {
			return nil
		}

		return networkValidAddressCIDRV4(value)
	},
	"ipv4.firewall": shared.IsBool,
	"ipv4.nat":      shared.IsBool,
	"ipv4.nat.order": func(value string) error {
		return shared.IsOneOf(value, []string{"before", "after"})
	},
	"ipv4.nat.address":  device.NetworkValidAddressV4,
	"ipv4.dhcp":         shared.IsBool,
	"ipv4.dhcp.gateway": device.NetworkValidAddressV4,
	"ipv4.dhcp.expiry":  shared.IsAny,
	"ipv4.dhcp.ranges":  shared.IsAny,
	"ipv4.routes":       shared.IsAny,
	"ipv4.routing":      shared.IsBool,

	"ipv6.address": func(value string) error {
		if shared.IsOneOf(value, []string{"none", "auto"}) == nil {
			return nil
		}

		return networkValidAddressCIDRV6(value)
	},
	"ipv6.firewall": shared.IsBool,
	"ipv6.nat":      shared.IsBool,
	"ipv6.nat.order": func(value string) error {
		return shared.IsOneOf(value, []string{"before", "after"})
	},
	"ipv6.nat.address":   device.NetworkValidAddressV6,
	"ipv6.dhcp":          shared.IsBool,
	"ipv6.dhcp.expiry":   shared.IsAny,
	"ipv6.dhcp.stateful": shared.IsBool,
	"ipv6.dhcp.ranges":   shared.IsAny,
	"ipv6.routes":        shared.IsAny,
	"ipv6.routing":       shared.IsBool,

	"dns.domain": shared.IsAny,
	"dns.mode": func(value string) error {
		return shared.IsOneOf(value, []string{"dynamic", "managed", "none"})
	},

	"raw.dnsmasq": shared.IsAny,
}

func networkValidateConfig(name string, config map[string]string) error {
	bridgeMode := config["bridge.mode"]

	if bridgeMode == "fan" && len(name) > 11 {
		return fmt.Errorf("Network name too long to use with the FAN (must be 11 characters or less)")
	}

	for k, v := range config {
		key := k

		// User keys are free for all
		if strings.HasPrefix(key, "user.") {
			continue
		}

		// Tunnel keys have the remote name in their name, so extract the real key
		if strings.HasPrefix(key, "tunnel.") {
			fields := strings.Split(key, ".")
			if len(fields) != 3 {
				return fmt.Errorf("Invalid network configuration key: %s", k)
			}

			if len(name)+len(fields[1]) > 14 {
				return fmt.Errorf("Network name too long for tunnel interface: %s-%s", name, fields[1])
			}

			key = fmt.Sprintf("tunnel.TARGET.%s", fields[2])
		}

		// Then validate
		validator, ok := networkConfigKeys[key]
		if !ok {
			return fmt.Errorf("Invalid network configuration key: %s", k)
		}

		err := validator(v)
		if err != nil {
			return err
		}

		// Bridge mode checks
		if bridgeMode == "fan" && strings.HasPrefix(key, "ipv4.") && v != "" {
			return fmt.Errorf("IPv4 configuration may not be set when in 'fan' mode")
		}

		if bridgeMode == "fan" && strings.HasPrefix(key, "ipv6.") && v != "" {
			return fmt.Errorf("IPv6 configuration may not be set when in 'fan' mode")
		}

		if bridgeMode != "fan" && strings.HasPrefix(key, "fan.") && v != "" {
			return fmt.Errorf("FAN configuration may only be set when in 'fan' mode")
		}

		// MTU checks
		if key == "bridge.mtu" && v != "" {
			mtu, err := strconv.ParseInt(v, 10, 64)
			if err != nil {
				return fmt.Errorf("Invalid value for an integer: %s", v)
			}

			ipv6 := config["ipv6.address"]
			if ipv6 != "" && ipv6 != "none" && mtu < 1280 {
				return fmt.Errorf("The minimum MTU for an IPv6 network is 1280")
			}

			ipv4 := config["ipv4.address"]
			if ipv4 != "" && ipv4 != "none" && mtu < 68 {
				return fmt.Errorf("The minimum MTU for an IPv4 network is 68")
			}

			if config["bridge.mode"] == "fan" {
				if config["fan.type"] == "ipip" {
					if mtu > 1480 {
						return fmt.Errorf("Maximum MTU for an IPIP FAN bridge is 1480")
					}
				} else {
					if mtu > 1450 {
						return fmt.Errorf("Maximum MTU for a VXLAN FAN bridge is 1450")
					}
				}
			}
		}
	}

	return nil
}

func networkFillAuto(config map[string]string) error {
	if config["ipv4.address"] == "auto" {
		subnet, err := networkRandomSubnetV4()
		if err != nil {
			return err
		}

		config["ipv4.address"] = subnet
	}

	if config["ipv6.address"] == "auto" {
		subnet, err := networkRandomSubnetV6()
		if err != nil {
			return err
		}

		config["ipv6.address"] = subnet
	}

	if config["fan.underlay_subnet"] == "auto" {
		subnet, _, err := networkDefaultGatewaySubnetV4()
		if err != nil {
			return err
		}

		config["fan.underlay_subnet"] = subnet.String()
	}

	return nil
}
<commit_msg>networks/config: Updates references to NetworkValidAddress<commit_after>package main

import (
	"fmt"
	"strconv"
	"strings"

	"github.com/lxc/lxd/lxd/device"
	"github.com/lxc/lxd/shared"
)

var networkConfigKeys = map[string]func(value string) error{
	"bridge.driver": func(value string) error {
		return shared.IsOneOf(value, []string{"native", "openvswitch"})
	},
	"bridge.external_interfaces": func(value string) error {
		if value == "" {
			return nil
		}

		for _, entry := range strings.Split(value, ",") {
			entry = strings.TrimSpace(entry)
			if networkValidName(entry) != nil {
				return fmt.Errorf("Invalid interface name '%s'", entry)
			}
		}

		return nil
	},
	"bridge.hwaddr": shared.IsAny,
	"bridge.mtu":    shared.IsInt64,
	"bridge.mode": func(value string) error {
		return shared.IsOneOf(value, []string{"standard", "fan"})
	},

	"fan.overlay_subnet": device.NetworkValidNetworkV4,
	"fan.underlay_subnet": func(value string) error {
		if value == "auto" {
			return nil
		}

		return device.NetworkValidNetworkV4(value)
	},
	"fan.type": func(value string) error {
		return shared.IsOneOf(value, []string{"vxlan", "ipip"})
	},

	"tunnel.TARGET.protocol": func(value string) error {
		return shared.IsOneOf(value, []string{"gre", "vxlan"})
	},
	"tunnel.TARGET.local":     device.NetworkValidAddress,
	"tunnel.TARGET.remote":    device.NetworkValidAddress,
	"tunnel.TARGET.port":      networkValidPort,
	"tunnel.TARGET.group":     device.NetworkValidAddress,
	"tunnel.TARGET.id":        shared.IsInt64,
	"tunnel.TARGET.interface": networkValidName,
	"tunnel.TARGET.ttl":       shared.IsUint8,

	"ipv4.address": func(value string) error {
		if shared.IsOneOf(value, []string{"none", "auto"}) == nil {
			return nil
		}

		return networkValidAddressCIDRV4(value)
	},
	"ipv4.firewall": shared.IsBool,
	"ipv4.nat":      shared.IsBool,
	"ipv4.nat.order": func(value string) error {
		return shared.IsOneOf(value, []string{"before", "after"})
	},
	"ipv4.nat.address":  device.NetworkValidAddressV4,
	"ipv4.dhcp":         shared.IsBool,
	"ipv4.dhcp.gateway": device.NetworkValidAddressV4,
	"ipv4.dhcp.expiry":  shared.IsAny,
	"ipv4.dhcp.ranges":  shared.IsAny,
	"ipv4.routes":       shared.IsAny,
	"ipv4.routing":      shared.IsBool,

	"ipv6.address": func(value string) error {
		if shared.IsOneOf(value, []string{"none", "auto"}) == nil {
			return nil
		}

		return networkValidAddressCIDRV6(value)
	},
	"ipv6.firewall": shared.IsBool,
	"ipv6.nat":      shared.IsBool,
	"ipv6.nat.order": func(value string) error {
		return shared.IsOneOf(value, []string{"before", "after"})
	},
	"ipv6.nat.address":   device.NetworkValidAddressV6,
	"ipv6.dhcp":          shared.IsBool,
	"ipv6.dhcp.expiry":   shared.IsAny,
	"ipv6.dhcp.stateful": shared.IsBool,
	"ipv6.dhcp.ranges":   shared.IsAny,
	"ipv6.routes":        shared.IsAny,
	"ipv6.routing":       shared.IsBool,

	"dns.domain": shared.IsAny,
	"dns.mode": func(value string) error {
		return shared.IsOneOf(value, []string{"dynamic", "managed", "none"})
	},

	"raw.dnsmasq": shared.IsAny,
}

func networkValidateConfig(name string, config map[string]string) error {
	bridgeMode := config["bridge.mode"]

	if bridgeMode == "fan" && len(name) > 11 {
		return fmt.Errorf("Network name too long to use with the FAN (must be 11 characters or less)")
	}

	for k, v := range config {
		key := k

		// User keys are free for all
		if strings.HasPrefix(key, "user.") {
			continue
		}

		// Tunnel keys have the remote name in their name, so extract the real key
		if strings.HasPrefix(key, "tunnel.") {
			fields := strings.Split(key, ".")
			if len(fields) != 3 {
				return fmt.Errorf("Invalid network configuration key: %s", k)
			}

			if len(name)+len(fields[1]) > 14 {
				return fmt.Errorf("Network name too long for tunnel interface: %s-%s", name, fields[1])
			}

			key = fmt.Sprintf("tunnel.TARGET.%s", fields[2])
		}

		// Then validate
		validator, ok := networkConfigKeys[key]
		if !ok {
			return fmt.Errorf("Invalid network configuration key: %s", k)
		}

		err := validator(v)
		if err != nil {
			return err
		}

		// Bridge mode checks
		if bridgeMode == "fan" && strings.HasPrefix(key, "ipv4.") && v != "" {
			return fmt.Errorf("IPv4 configuration may not be set when in 'fan' mode")
		}

		if bridgeMode == "fan" && strings.HasPrefix(key, "ipv6.") && v != "" {
			return fmt.Errorf("IPv6 configuration may not be set when in 'fan' mode")
		}

		if bridgeMode != "fan" && strings.HasPrefix(key, "fan.") && v != "" {
			return fmt.Errorf("FAN configuration may only be set when in 'fan' mode")
		}

		// MTU checks
		if key == "bridge.mtu" && v != "" {
			mtu, err := strconv.ParseInt(v, 10, 64)
			if err != nil {
				return fmt.Errorf("Invalid value for an integer: %s", v)
			}

			ipv6 := config["ipv6.address"]
			if ipv6 != "" && ipv6 != "none" && mtu < 1280 {
				return fmt.Errorf("The minimum MTU for an IPv6 network is 1280")
			}

			ipv4 := config["ipv4.address"]
			if ipv4 != "" && ipv4 != "none" && mtu < 68 {
				return fmt.Errorf("The minimum MTU for an IPv4 network is 68")
			}

			if config["bridge.mode"] == "fan" {
				if config["fan.type"] == "ipip" {
					if mtu > 1480 {
						return fmt.Errorf("Maximum MTU for an IPIP FAN bridge is 1480")
					}
				} else {
					if mtu > 1450 {
						return fmt.Errorf("Maximum MTU for a VXLAN FAN bridge is 1450")
					}
				}
			}
		}
	}

	return nil
}

func networkFillAuto(config map[string]string) error {
	if config["ipv4.address"] == "auto" {
		subnet, err := networkRandomSubnetV4()
		if err != nil {
			return err
		}

		config["ipv4.address"] = subnet
	}

	if config["ipv6.address"] == "auto" {
		subnet, err := networkRandomSubnetV6()
		if err != nil {
			return err
		}

		config["ipv6.address"] = subnet
	}

	if config["fan.underlay_subnet"] == "auto" {
		subnet, _, err := networkDefaultGatewaySubnetV4()
		if err != nil {
			return err
		}

		config["fan.underlay_subnet"] = subnet.String()
	}

	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"crypto/rand"
	"fmt"
	"image"
	"image/gif"
	"io"
	"log"
	"net/http"
	"net/url"
	"strconv"
	"strings"

	"github.com/avct/uasurfer"

	"./database"

	"encoding/hex"

	influx "github.com/influxdata/influxdb/client/v2"
)

// VersionString converts a uasurfer.Version into a string
func VersionString(v uasurfer.Version) string {
	if v.Major == 0 {
		return ""
	}

	return strconv.Itoa(v.Major) + "." + strconv.Itoa(v.Minor) + "." + strconv.Itoa(v.Patch)
}

var b bytes.Buffer
var err = gif.Encode(&b, image.NewAlpha(image.Rect(0, 0, 1, 1)), nil)

// OnePixelGIF - The data for a one pixel transparent GIF
var OnePixelGIF = b.Bytes()

func handler(w http.ResponseWriter, r *http.Request) {

	tags := make(map[string]string)

	var fpt string

	// TODO: set a limit on arguments
	for key, vals := range r.URL.Query() {
		log.Printf("%s: %s\n", key, vals[0])

		if key != "fpt" {
			tags[key] = vals[0]
		} else {
			fpt = vals[0]
		}
	}

	ua := r.Header.Get("User-Agent")

	if ua != "" {
		parsedUa := uasurfer.Parse(ua)

		tags["browser"] = parsedUa.Browser.Name.String()[7:] // Remove "Browser" prefix added by uasurfer
		tags["browser_ver"] = VersionString(parsedUa.Browser.Version)

		if parsedUa.Browser.Version.Major != 0 {
			tags["browser_major"] = strconv.Itoa(parsedUa.Browser.Version.Major)
		}

		tags["os"] = parsedUa.OS.Name.String()[2:] // Remove "OS" prefix added by uasurfer

		tags["os_ver"] = VersionString(parsedUa.OS.Version)

		if parsedUa.OS.Version.Major != 0 {
			// OS X versions are weird
			if parsedUa.OS.Name == uasurfer.OSMacOSX {
				tags["os_major"] = strconv.Itoa(parsedUa.OS.Version.Minor)
			} else {
				tags["os_major"] = strconv.Itoa(parsedUa.OS.Version.Major)
			}
		}

		tags["device_type"] = parsedUa.DeviceType.String()[6:] // Remove "Device" prefix added by uasurfer

		log.Printf("%v\n", parsedUa.Browser.Name)
	}

	// TODO: utilized X-Forwarded-For

	// Remove port from IP
	ip := r.RemoteAddr[0:strings.LastIndex(r.RemoteAddr, ":")]

	// Remove square brackets from IPv6
	if ip[0] == '[' {
		ip = ip[1:(len(ip) - 1)]
	}

	tags["ip"] = ip

	// break URL into components
	if tags["url"] != "" {
		pageURL, err := url.Parse(tags["url"])
		if err == nil {
			tags["host"] = pageURL.Host
			tags["scheme"] = pageURL.Scheme
			tags["path"] = pageURL.Path
		}
	}

	connection := database.Connect("test", "test", "http://xenial.dev:8086")
	points, err := influx.NewBatchPoints(influx.BatchPointsConfig{
		Database: "test",
	})

	if err != nil {
		log.Fatal(err)
	}

	pt, _ := influx.NewPoint("hello", tags, map[string]interface{}{
		"fpt": fpt,
	})

	points.AddPoint(pt)

	err = connection.Write(points)

	if err != nil {
		log.Fatal(err)
	}

	w.Header().Add("Content-Type", "image/gif")
	io.Copy(w, bytes.NewReader(OnePixelGIF))
}

func randomFingerprint(w http.ResponseWriter, r *http.Request) {
	bytes := make([]byte, 8)

	_, err := rand.Read(bytes)

	if err != nil {
		r.Response.StatusCode = 500
	} else {
		fmt.Fprintf(w, "%s", hex.EncodeToString(bytes))
	}
}

func main() {
	http.HandleFunc("/aurora", handler)
	http.HandleFunc("/fp", randomFingerprint)
	http.ListenAndServe(":3030", nil)
}
<commit_msg>Added POST request support<commit_after>package main

import (
	"bytes"
	"crypto/rand"
	"fmt"
	"image"
	"image/gif"
	"io"
	"log"
	"net/http"
	"net/url"
	"strconv"
	"strings"

	"github.com/avct/uasurfer"

	"./database"

	"encoding/hex"

	"encoding/json"

	influx "github.com/influxdata/influxdb/client/v2"
)

// VersionString converts a uasurfer.Version into a string
func VersionString(v uasurfer.Version) string {
	if v.Major == 0 {
		return ""
	}

	return strconv.Itoa(v.Major) + "." + strconv.Itoa(v.Minor) + "." + strconv.Itoa(v.Patch)
}

var b bytes.Buffer
var err = gif.Encode(&b, image.NewAlpha(image.Rect(0, 0, 1, 1)), nil)

// OnePixelGIF - The data for a one pixel transparent GIF
var OnePixelGIF = b.Bytes()

// ActionEvent is to represent POST data
type ActionEvent struct {
	Fpt  string
	Vars map[string]string
}

func handler(w http.ResponseWriter, r *http.Request) {

	tags := make(map[string]string)

	var fpt string

	// Get variables from the request
	// TODO: set a limit on arguments

	// GET request
	if r.Method == "GET" {
		for key, vals := range r.URL.Query() {
			log.Printf("%s: %s\n", key, vals[0])

			if key != "fpt" {
				tags[key] = vals[0]
			} else {
				fpt = vals[0]
			}
		}
	} else if r.Method == "POST" {
		if !strings.HasPrefix(r.Header.Get("Content-Type"), "application/json") {
			r.Response.StatusCode = 400
			fmt.Fprint(w, "Requests must be JSON")
			return
		}

		var postData ActionEvent

		decoder := json.NewDecoder(r.Body)

		err := decoder.Decode(&postData)

		if err != nil {
			r.Response.StatusCode = 500
			log.Println(err)
			return
		}

		fpt = postData.Fpt

		for key, val := range postData.Vars {
			tags[key] = val
		}
	} else {
		r.Response.StatusCode = 405
		return
	}

	ua := r.Header.Get("User-Agent")

	if ua != "" {
		parsedUa := uasurfer.Parse(ua)

		tags["browser"] = parsedUa.Browser.Name.String()[7:] // Remove "Browser" prefix added by uasurfer
		tags["browser_ver"] = VersionString(parsedUa.Browser.Version)

		if parsedUa.Browser.Version.Major != 0 {
			tags["browser_major"] = strconv.Itoa(parsedUa.Browser.Version.Major)
		}

		tags["os"] = parsedUa.OS.Name.String()[2:] // Remove "OS" prefix added by uasurfer

		tags["os_ver"] = VersionString(parsedUa.OS.Version)

		if parsedUa.OS.Version.Major != 0 {
			// OS X versions are weird
			if parsedUa.OS.Name == uasurfer.OSMacOSX {
				tags["os_major"] = strconv.Itoa(parsedUa.OS.Version.Minor)
			} else {
				tags["os_major"] = strconv.Itoa(parsedUa.OS.Version.Major)
			}
		}

		tags["device_type"] = parsedUa.DeviceType.String()[6:] // Remove "Device" prefix added by uasurfer

		log.Printf("%v\n", parsedUa.Browser.Name)
	}

	// TODO: utilized X-Forwarded-For

	// Remove port from IP
	ip := r.RemoteAddr[0:strings.LastIndex(r.RemoteAddr, ":")]

	// Remove square brackets from IPv6
	if ip[0] == '[' {
		ip = ip[1:(len(ip) - 1)]
	}

	tags["ip"] = ip

	// break URL into components
	if tags["url"] != "" {
		pageURL, err := url.Parse(tags["url"])
		if err == nil {
			tags["host"] = pageURL.Host
			tags["scheme"] = pageURL.Scheme
			tags["path"] = pageURL.Path
		}
	}

	connection := database.Connect("test", "test", "http://xenial.dev:8086")
	points, err := influx.NewBatchPoints(influx.BatchPointsConfig{
		Database: "test",
	})

	if err != nil {
		log.Fatal(err)
	}

	pt, _ := influx.NewPoint("hello", tags, map[string]interface{}{
		"fpt": fpt,
	})

	points.AddPoint(pt)

	err = connection.Write(points)

	if err != nil {
		log.Fatal(err)
	}

	// Return a 1px GIF if this is a GET request
	if r.Method == "GET" {
		// TODO: add no-cache header
		w.Header().Add("Content-Type", "image/gif")
		io.Copy(w, bytes.NewReader(OnePixelGIF))
	}
}

func randomFingerprint(w http.ResponseWriter, r *http.Request) {
	bytes := make([]byte, 8)

	_, err := rand.Read(bytes)

	if err != nil {
		r.Response.StatusCode = 500
	} else {
		fmt.Fprintf(w, "%s", hex.EncodeToString(bytes))
	}
}

func main() {
	http.HandleFunc("/aurora", handler)
	http.HandleFunc("/fp", randomFingerprint)
	http.ListenAndServe(":3030", nil)
}
<|endoftext|>