<commit_before>package consensus

import (
	"math/big"
	"sort"
)

// SurpassThreshold is a percentage that dictates how much heavier a competing
// chain has to be before the node will switch to mining on that chain. This is
// not a consensus rule. This percentage is only applied to the most recent
// block, not the entire chain; see blockNode.heavierThan.
//
// If no threshold were in place, it would be possible to manipulate a block's
// timestamp to produce a sufficiently heavier block.
var SurpassThreshold = big.NewRat(20, 100)

// a blockNode is a node in the tree of competing blockchain forks. It contains
// the block itself, parent and child blockNodes, and context such as the
// block height, depth, and target. It also contains a set of diffs that
// dictate how the consensus set is affected by the block.
type blockNode struct {
	block    Block
	parent   *blockNode
	children []*blockNode

	height BlockHeight
	depth  Target // Cumulative weight of all parents.
	target Target // Target for next block, i.e. any child blockNodes.

	// Diffs are computationally expensive to generate, so a lazy approach is
	// taken wherein the diffs are only generated when needed. A boolean
	// prevents duplicate work from being performed.
	diffsGenerated        bool
	siafundPoolDiff       SiafundPoolDiff
	siacoinOutputDiffs    []SiacoinOutputDiff
	fileContractDiffs     []FileContractDiff
	siafundOutputDiffs    []SiafundOutputDiff
	delayedSiacoinOutputs map[SiacoinOutputID]SiacoinOutput
}

// childDepth returns the depth of a blockNode's child nodes. The depth is the
// "sum" of the current depth and current target, where sum is defined as:
//
//     sum(x,y) := 1/(1/x + 1/y)
func (bn *blockNode) childDepth() (depth Target) {
	cumulativeDifficulty := new(big.Rat).Add(bn.target.Inverse(), bn.depth.Inverse())
	return RatToTarget(new(big.Rat).Inv(cumulativeDifficulty))
}

// heavierThan returns true if the blockNode is sufficiently heavier than
// 'cmp', where "sufficient" is defined as:
//
//     (1/bn.depth) - (1/cmp.depth) > (1/cmp.target) * SurpassThreshold
func (bn *blockNode) heavierThan(cmp *blockNode) bool {
	diff := new(big.Rat).Sub(bn.depth.Inverse(), cmp.depth.Inverse())
	threshold := new(big.Rat).Mul(cmp.target.Inverse(), SurpassThreshold)
	return diff.Cmp(threshold) > 0
}

// earliestChildTimestamp returns the earliest timestamp that a child node
// can have while still being valid. See section 'Timestamp Rules' in
// Consensus.md.
func (bn *blockNode) earliestChildTimestamp() Timestamp {
	// Get the previous `MedianTimestampWindow` timestamps.
	var intTimestamps []int
	referenceNode := bn
	for i := 0; i < MedianTimestampWindow; i++ {
		intTimestamps = append(intTimestamps, int(referenceNode.block.Timestamp))
		if referenceNode.parent != nil {
			referenceNode = referenceNode.parent
		}
	}
	sort.Ints(intTimestamps)

	// Return the median of the sorted timestamps.
	return Timestamp(intTimestamps[MedianTimestampWindow/2])
}

// newChild creates a blockNode from a block and adds it to the parent's set of
// children. The new node is also returned.
func (bn *blockNode) newChild(b Block) *blockNode {
	// Sanity check - parent can't be nil.
	if DEBUG {
		if bn == nil {
			panic("can't create blockNode with nil parent")
		}
	}

	child := &blockNode{
		block:  b,
		parent: bn,

		height: bn.height + 1,
		depth:  bn.childDepth(),

		delayedSiacoinOutputs: make(map[SiacoinOutputID]SiacoinOutput),
	}

	// Calculate the target for the new node. To calculate the target, we need
	// to compare our timestamp with the timestamp of the reference node, which
	// is `TargetWindow` blocks earlier, or if the height is less than
	// `TargetWindow`, it's the genesis block.
	//
	// CONTRIBUTE: find a way to look up the correct parent without scrolling
	// through 'TargetWindow' elements in a linked list.
	var numBlocks BlockHeight
	windowStart := child
	for numBlocks = 0; numBlocks < TargetWindow && windowStart.parent != nil; numBlocks++ {
		windowStart = windowStart.parent
	}

	// Calculate the amount to adjust the target by dividing the amount of time
	// passed by the expected amount of time passed.
	timePassed := child.block.Timestamp - windowStart.block.Timestamp
	expectedTimePassed := BlockFrequency * numBlocks
	targetAdjustment := big.NewRat(int64(timePassed), int64(expectedTimePassed))

	// Clamp adjustment to reasonable values.
	if targetAdjustment.Cmp(MaxAdjustmentUp) > 0 {
		targetAdjustment = MaxAdjustmentUp
	} else if targetAdjustment.Cmp(MaxAdjustmentDown) < 0 {
		targetAdjustment = MaxAdjustmentDown
	}

	// Multiply the previous target by the adjustment to get the new target.
	newRatTarget := new(big.Rat).Mul(bn.target.Rat(), targetAdjustment)
	child.target = RatToTarget(newRatTarget)

	// add child to parent
	bn.children = append(bn.children, child)

	return child
}
<commit_msg>proper timestamp sorting<commit_after>package consensus

import (
	"math/big"
	"sort"
)

// SurpassThreshold is a percentage that dictates how much heavier a competing
// chain has to be before the node will switch to mining on that chain. This is
// not a consensus rule. This percentage is only applied to the most recent
// block, not the entire chain; see blockNode.heavierThan.
//
// If no threshold were in place, it would be possible to manipulate a block's
// timestamp to produce a sufficiently heavier block.
var SurpassThreshold = big.NewRat(20, 100)

// a blockNode is a node in the tree of competing blockchain forks. It contains
// the block itself, parent and child blockNodes, and context such as the
// block height, depth, and target. It also contains a set of diffs that
// dictate how the consensus set is affected by the block.
type blockNode struct {
	block    Block
	parent   *blockNode
	children []*blockNode

	height BlockHeight
	depth  Target // Cumulative weight of all parents.
	target Target // Target for next block, i.e. any child blockNodes.

	// Diffs are computationally expensive to generate, so a lazy approach is
	// taken wherein the diffs are only generated when needed. A boolean
	// prevents duplicate work from being performed.
	diffsGenerated        bool
	siafundPoolDiff       SiafundPoolDiff
	siacoinOutputDiffs    []SiacoinOutputDiff
	fileContractDiffs     []FileContractDiff
	siafundOutputDiffs    []SiafundOutputDiff
	delayedSiacoinOutputs map[SiacoinOutputID]SiacoinOutput
}

// childDepth returns the depth of a blockNode's child nodes. The depth is the
// "sum" of the current depth and current target, where sum is defined as:
//
//     sum(x,y) := 1/(1/x + 1/y)
func (bn *blockNode) childDepth() (depth Target) {
	cumulativeDifficulty := new(big.Rat).Add(bn.target.Inverse(), bn.depth.Inverse())
	return RatToTarget(new(big.Rat).Inv(cumulativeDifficulty))
}

// heavierThan returns true if the blockNode is sufficiently heavier than
// 'cmp', where "sufficient" is defined as:
//
//     (1/bn.depth) - (1/cmp.depth) > (1/cmp.target) * SurpassThreshold
func (bn *blockNode) heavierThan(cmp *blockNode) bool {
	diff := new(big.Rat).Sub(bn.depth.Inverse(), cmp.depth.Inverse())
	threshold := new(big.Rat).Mul(cmp.target.Inverse(), SurpassThreshold)
	return diff.Cmp(threshold) > 0
}

// timestampSlice is used for sorting.
type timestampSlice []Timestamp

func (ts timestampSlice) Len() int           { return len(ts) }
func (ts timestampSlice) Less(i, j int) bool { return ts[i] < ts[j] }
func (ts timestampSlice) Swap(i, j int)      { ts[i], ts[j] = ts[j], ts[i] }

// earliestChildTimestamp returns the earliest timestamp that a child node
// can have while still being valid. See section 'Timestamp Rules' in
// Consensus.md.
func (bn *blockNode) earliestChildTimestamp() Timestamp {
	// Get the previous MedianTimestampWindow timestamps.
	windowTimes := make(timestampSlice, MedianTimestampWindow)
	traverse := bn
	for i := 0; i < MedianTimestampWindow; i++ {
		windowTimes[i] = traverse.block.Timestamp
		if traverse.parent != nil {
			traverse = traverse.parent
		}
	}
	sort.Sort(windowTimes)

	// Return the median of the sorted timestamps.
	return windowTimes[len(windowTimes)/2]
}

// newChild creates a blockNode from a block and adds it to the parent's set of
// children. The new node is also returned.
func (bn *blockNode) newChild(b Block) *blockNode {
	// Sanity check - parent can't be nil.
	if DEBUG {
		if bn == nil {
			panic("can't create blockNode with nil parent")
		}
	}

	child := &blockNode{
		block:  b,
		parent: bn,

		height: bn.height + 1,
		depth:  bn.childDepth(),

		delayedSiacoinOutputs: make(map[SiacoinOutputID]SiacoinOutput),
	}

	// Calculate the target for the new node. To calculate the target, we need
	// to compare our timestamp with the timestamp of the reference node, which
	// is `TargetWindow` blocks earlier, or if the height is less than
	// `TargetWindow`, it's the genesis block.
	//
	// CONTRIBUTE: find a way to look up the correct parent without scrolling
	// through 'TargetWindow' elements in a linked list.
	var numBlocks BlockHeight
	windowStart := child
	for numBlocks = 0; numBlocks < TargetWindow && windowStart.parent != nil; numBlocks++ {
		windowStart = windowStart.parent
	}

	// Calculate the amount to adjust the target by dividing the amount of time
	// passed by the expected amount of time passed.
	timePassed := child.block.Timestamp - windowStart.block.Timestamp
	expectedTimePassed := BlockFrequency * numBlocks
	targetAdjustment := big.NewRat(int64(timePassed), int64(expectedTimePassed))

	// Clamp adjustment to reasonable values.
	if targetAdjustment.Cmp(MaxAdjustmentUp) > 0 {
		targetAdjustment = MaxAdjustmentUp
	} else if targetAdjustment.Cmp(MaxAdjustmentDown) < 0 {
		targetAdjustment = MaxAdjustmentDown
	}

	// Multiply the previous target by the adjustment to get the new target.
	newRatTarget := new(big.Rat).Mul(bn.target.Rat(), targetAdjustment)
	child.target = RatToTarget(newRatTarget)

	// add child to parent
	bn.children = append(bn.children, child)

	return child
}
<|endoftext|>


<commit_before>package container

import (
	log "github.com/Sirupsen/logrus"
	"github.com/samalba/dockerclient"

	"fmt"
	"strings"
	"time"
)

// Docker container labels
const (
	TugbotService     = "tugbot-service"
	TugbotTest        = "tugbot-test"
	TugbotEventDocker = "tugbot-event-docker"
	TugbotEventTimer  = "tugbot-event-timer"
	TugbotCreatedFrom = "tugbot-created-from"
	SwarmTaskID       = "com.docker.swarm.task.id"
)

// Docker Event Filter
const (
	// type filter: tugbot-event-docker-filter-type=container|image|daemon|network|volume|plugin
	TypeFilter = "tugbot-event-docker-filter-type"
	// action filter (depends on type), for 'container' type:
	//  - attach, commit, copy, create, destroy, detach, die, exec_create, exec_detach, exec_start, export,
	//  - health_status, kill, oom, pause, rename, resize, restart, start, stop, top, unpause, update
	ActionFilter = "tugbot-event-docker-filter-action"
	// container filter: use name, comma separated name list or RE2 regexp
	ContainerFilter = "tugbot-event-docker-filter-container"
	// image filter: use name, comma separated name list or RE2 regexp
	ImageFilter = "tugbot-event-docker-filter-image"
	// label filter: use key=value comma separated pairs
	LabelFilter = "tugbot-event-docker-filter-label"
)

// NewContainer returns a new Container instance instantiated with the
// specified ContainerInfo and ImageInfo structs.
func NewContainer(containerInfo *dockerclient.ContainerInfo, imageInfo *dockerclient.ImageInfo) *Container {
	return &Container{
		containerInfo: containerInfo,
		imageInfo:     imageInfo,
	}
}

// Container represents a running Docker container.
type Container struct {
	containerInfo *dockerclient.ContainerInfo
	imageInfo     *dockerclient.ImageInfo
}

// ID returns the Docker container ID.
func (c Container) ID() string {
	return c.containerInfo.Id
}

// Name returns the Docker container name.
func (c Container) Name() string {
	return strings.TrimPrefix(c.containerInfo.Name, "/")
}

// ImageID returns the ID of the Docker image that was used to start the
// container.
func (c Container) ImageID() string {
	return c.imageInfo.Id
}

// ImageName returns the name of the Docker image that was used to start the
// container. If the original image was specified without a particular tag, the
// "latest" tag is assumed.
func (c Container) ImageName() string {
	imageName := c.containerInfo.Config.Image

	if !strings.Contains(imageName, ":") {
		imageName = fmt.Sprintf("%s:latest", imageName)
	}

	return imageName
}

// IsTugbot returns whether or not the current container is the tugbot container itself.
// The tugbot container is identified by the presence of the "tugbot.service"
// label in the container metadata.
func (c Container) IsTugbot() bool {
	val, ok := c.containerInfo.Config.Labels[TugbotService]

	return ok && val == "true"
}

// IsTugbotCandidate returns whether or not a container is a candidate to run by tugbot.
// A candidate container is identified by the presence of "tugbot-test",
// it doesn't contain "tugbot.created.from" in the container metadata and it state is "Exited".
func (c Container) IsTugbotCandidate() bool {
	ret := false
	log.Info("Checking container for being candidate: ",c.containerInfo.Name)
	val, ok := c.containerInfo.Config.Labels[TugbotTest]
	log.Info("Label is ", c.containerInfo.Config.Labels[TugbotTest])
	if ok && val == "true" {
		if !c.IsCreatedByTugbot() {
			ret = c.containerInfo.State.StateString() == "exited"
		}
	}

	return ret
}

// IsCreatedByTugbot returns whether or not a container created by tugbot.
func (c Container) IsCreatedByTugbot() bool {
	val, ok := c.containerInfo.Config.Labels[TugbotCreatedFrom]

	return ok && val != ""
}

// IsEventListener returns whether or not a container should run when an event e is occurred.
func (c Container) IsEventListener(e *dockerclient.Event) bool {
	ret := false
	if e != nil {
		// check if container is subscribed to Docker events, i.e. 'tugbot-event-docker' label exists
		_, ret = c.containerInfo.Config.Labels[TugbotEventDocker]
		if ret {
			// filter by event type
			if typeFilter, ok := c.containerInfo.Config.Labels[TypeFilter]; ok {
				ret = sliceContains(e.Type, splitAndTrimSpaces(typeFilter, ","))
			}
			// filter by event action
			if actionFilter, ok := c.containerInfo.Config.Labels[ActionFilter]; ok {
				ret = ret && sliceContains(e.Action, splitAndTrimSpaces(actionFilter, ","))
			}
			// filter by container name or name regexp
			if containerFilter, ok := c.containerInfo.Config.Labels[ContainerFilter]; ok {
				ret = ret && inFilterOrList(e.Actor.Attributes["name"], containerFilter)
			}
		}
		// filter by event image
		if imageFilter, ok := c.containerInfo.Config.Labels[ImageFilter]; ok {
			// get image name from event.From field
			imageName := e.From
			// in case of "image" event.Type, event.ID contains image ID (name:tag) for 'pull' action and sha256:num for untag and delete
			if e.Type == "image" {
				imageName = e.ID
			}
			ret = ret && inFilterOrList(imageName, imageFilter)
		}
		// filter by event labels
		if labelFilter, ok := c.containerInfo.Config.Labels[LabelFilter]; ok {
			labels := splitAndTrimSpaces(labelFilter, ",")
			for _, label := range labels {
				ret = ret && mapContains(e.Actor.Attributes, splitAndTrimSpaces(label, "="))
			}
		}
	}

	return ret
}

// GetEventListenerTimer returns interval duration between a test container run and true
// if docker label exist and label value parsed into Duration, Otherwise false.
func (c Container) GetEventListenerInterval() (time.Duration, bool) {
	var ret time.Duration
	val, ok := c.containerInfo.Config.Labels[TugbotEventTimer]
	if ok {
		interval, err := time.ParseDuration(val)
		if err != nil {
			log.Errorf("Failed to parse %s docker label: %s into golang Duration (%v)", TugbotEventTimer, val, err)
			ok = false
		} else {
			ret = interval
		}
	}

	return ret, ok
}

// Any links in the HostConfig need to be re-written before they can be
// re-submitted to the Docker create API.
func (c Container) hostConfig() *dockerclient.HostConfig {
	hostConfig := c.containerInfo.HostConfig

	for i, link := range hostConfig.Links {
		name := link[0:strings.Index(link, ":")]
		alias := link[strings.LastIndex(link, "/"):]

		hostConfig.Links[i] = fmt.Sprintf("%s:%s", name, alias)
	}

	return hostConfig
}
<commit_msg>remove extra logging<commit_after>package container

import (
	log "github.com/Sirupsen/logrus"
	"github.com/samalba/dockerclient"

	"fmt"
	"strings"
	"time"
)

// Docker container labels
const (
	TugbotService     = "tugbot-service"
	TugbotTest        = "tugbot-test"
	TugbotEventDocker = "tugbot-event-docker"
	TugbotEventTimer  = "tugbot-event-timer"
	TugbotCreatedFrom = "tugbot-created-from"
	SwarmTaskID       = "com.docker.swarm.task.id"
)

// Docker Event Filter
const (
	// type filter: tugbot-event-docker-filter-type=container|image|daemon|network|volume|plugin
	TypeFilter = "tugbot-event-docker-filter-type"
	// action filter (depends on type), for 'container' type:
	//  - attach, commit, copy, create, destroy, detach, die, exec_create, exec_detach, exec_start, export,
	//  - health_status, kill, oom, pause, rename, resize, restart, start, stop, top, unpause, update
	ActionFilter = "tugbot-event-docker-filter-action"
	// container filter: use name, comma separated name list or RE2 regexp
	ContainerFilter = "tugbot-event-docker-filter-container"
	// image filter: use name, comma separated name list or RE2 regexp
	ImageFilter = "tugbot-event-docker-filter-image"
	// label filter: use key=value comma separated pairs
	LabelFilter = "tugbot-event-docker-filter-label"
)

// NewContainer returns a new Container instance instantiated with the
// specified ContainerInfo and ImageInfo structs.
func NewContainer(containerInfo *dockerclient.ContainerInfo, imageInfo *dockerclient.ImageInfo) *Container {
	return &Container{
		containerInfo: containerInfo,
		imageInfo:     imageInfo,
	}
}

// Container represents a running Docker container.
type Container struct {
	containerInfo *dockerclient.ContainerInfo
	imageInfo     *dockerclient.ImageInfo
}

// ID returns the Docker container ID.
func (c Container) ID() string {
	return c.containerInfo.Id
}

// Name returns the Docker container name.
func (c Container) Name() string {
	return strings.TrimPrefix(c.containerInfo.Name, "/")
}

// ImageID returns the ID of the Docker image that was used to start the
// container.
func (c Container) ImageID() string {
	return c.imageInfo.Id
}

// ImageName returns the name of the Docker image that was used to start the
// container. If the original image was specified without a particular tag, the
// "latest" tag is assumed.
func (c Container) ImageName() string {
	imageName := c.containerInfo.Config.Image

	if !strings.Contains(imageName, ":") {
		imageName = fmt.Sprintf("%s:latest", imageName)
	}

	return imageName
}

// IsTugbot returns whether or not the current container is the tugbot container itself.
// The tugbot container is identified by the presence of the "tugbot.service"
// label in the container metadata.
func (c Container) IsTugbot() bool {
	val, ok := c.containerInfo.Config.Labels[TugbotService]

	return ok && val == "true"
}

// IsTugbotCandidate returns whether or not a container is a candidate to run by tugbot.
// A candidate container is identified by the presence of "tugbot-test",
// it doesn't contain "tugbot.created.from" in the container metadata and it state is "Exited".
func (c Container) IsTugbotCandidate() bool {
	ret := false
	val, ok := c.containerInfo.Config.Labels[TugbotTest]
	if ok && val == "true" {
		if !c.IsCreatedByTugbot() {
			ret = c.containerInfo.State.StateString() == "exited"
		}
	}

	return ret
}

// IsCreatedByTugbot returns whether or not a container created by tugbot.
func (c Container) IsCreatedByTugbot() bool {
	val, ok := c.containerInfo.Config.Labels[TugbotCreatedFrom]

	return ok && val != ""
}

// IsEventListener returns whether or not a container should run when an event e is occurred.
func (c Container) IsEventListener(e *dockerclient.Event) bool {
	ret := false
	if e != nil {
		// check if container is subscribed to Docker events, i.e. 'tugbot-event-docker' label exists
		_, ret = c.containerInfo.Config.Labels[TugbotEventDocker]
		if ret {
			// filter by event type
			if typeFilter, ok := c.containerInfo.Config.Labels[TypeFilter]; ok {
				ret = sliceContains(e.Type, splitAndTrimSpaces(typeFilter, ","))
			}
			// filter by event action
			if actionFilter, ok := c.containerInfo.Config.Labels[ActionFilter]; ok {
				ret = ret && sliceContains(e.Action, splitAndTrimSpaces(actionFilter, ","))
			}
			// filter by container name or name regexp
			if containerFilter, ok := c.containerInfo.Config.Labels[ContainerFilter]; ok {
				ret = ret && inFilterOrList(e.Actor.Attributes["name"], containerFilter)
			}
		}
		// filter by event image
		if imageFilter, ok := c.containerInfo.Config.Labels[ImageFilter]; ok {
			// get image name from event.From field
			imageName := e.From
			// in case of "image" event.Type, event.ID contains image ID (name:tag) for 'pull' action and sha256:num for untag and delete
			if e.Type == "image" {
				imageName = e.ID
			}
			ret = ret && inFilterOrList(imageName, imageFilter)
		}
		// filter by event labels
		if labelFilter, ok := c.containerInfo.Config.Labels[LabelFilter]; ok {
			labels := splitAndTrimSpaces(labelFilter, ",")
			for _, label := range labels {
				ret = ret && mapContains(e.Actor.Attributes, splitAndTrimSpaces(label, "="))
			}
		}
	}

	return ret
}

// GetEventListenerTimer returns interval duration between a test container run and true
// if docker label exist and label value parsed into Duration, Otherwise false.
func (c Container) GetEventListenerInterval() (time.Duration, bool) {
	var ret time.Duration
	val, ok := c.containerInfo.Config.Labels[TugbotEventTimer]
	if ok {
		interval, err := time.ParseDuration(val)
		if err != nil {
			log.Errorf("Failed to parse %s docker label: %s into golang Duration (%v)", TugbotEventTimer, val, err)
			ok = false
		} else {
			ret = interval
		}
	}

	return ret, ok
}

// Any links in the HostConfig need to be re-written before they can be
// re-submitted to the Docker create API.
func (c Container) hostConfig() *dockerclient.HostConfig {
	hostConfig := c.containerInfo.HostConfig

	for i, link := range hostConfig.Links {
		name := link[0:strings.Index(link, ":")]
		alias := link[strings.LastIndex(link, "/"):]

		hostConfig.Links[i] = fmt.Sprintf("%s:%s", name, alias)
	}

	return hostConfig
}
<|endoftext|>


<commit_before>// Copyright 2017 Ole Krüger.
// Licensed under the MIT license which can be found in the LICENSE file.

package dpt

import (
	"fmt"
	"time"
)

// DPT_11001 represents DPT 11.001 / Date p 34.
// Valid years are limited to 1990 - 2089 for the Year field.
type DPT_11001 struct {
	Year  uint16
	Month uint8
	Day   uint8
}

func (d DPT_11001) Pack() []byte {
	var buf = []byte{0, 0, 0, 0}

	if d.Year >= 1990 && d.Year <= 2089 && d.IsValid() {
		buf[1] = d.Day
		buf[2] = d.Month

		if d.Year < 2000 {
			buf[3] = uint8(d.Year - 1900)
		} else {
			buf[3] = uint8(d.Year - 2000)
		}
	}
	return []byte(buf)
}

func (d *DPT_11001) Unpack(data []byte) error {
	if len(data) != 4 {
		return nil
	}
	d.Day = uint8(data[1] & 0x1F)
	d.Month = uint8(data[2] & 0xF)
	d.Year = uint16(data[3] & 0x7F)
	if d.Year > 99 {
		return fmt.Errorf("payload is out of range")
	}
	if d.Year == 0 && d.Month == 0 && d.Day == 0 {
		d.Year = 90
		d.Month = 1
		d.Day = 1
	}
	if d.Year >= 90 {
		d.Year += 1900
	} else {
		d.Year += 2000
	}
	if !d.IsValid() {
		return fmt.Errorf("payload is out of range")
	}

	return nil
}

func (d DPT_11001) Unit() string {
	return ""
}

func (d DPT_11001) IsValid() bool {
	tm := time.Date(int(d.Year), time.Month(d.Month), int(d.Day), 0, 0, 0, 0, time.UTC)
	if tm.Year() < 1990 || tm.Year() > 2089 {
		return false
	} else {
		return (int(d.Year) == tm.Year() && d.Month == uint8(tm.Month()) && int(d.Day) == tm.Day())
	}
}

func (d DPT_11001) String() string {
	return fmt.Sprintf("%04d-%02d-%02d", d.Year, d.Month, d.Day)
}
<commit_msg>Fix DPT 11<commit_after>// Copyright 2017 Ole Krüger.
// Licensed under the MIT license which can be found in the LICENSE file.

package dpt

import (
	"fmt"
	"time"
)

// DPT_11001 represents DPT 11.001 / Date p 34.
// Valid years are limited to 1990 - 2089 for the Year field.
type DPT_11001 struct {
	Year  uint16
	Month uint8
	Day   uint8
}

func (d DPT_11001) Pack() []byte {
	var buf = []byte{0, 0, 0, 0}

	if d.Year >= 1990 && d.Year <= 2089 && d.IsValid() {
		buf[1] = d.Day & 0x1F
		buf[2] = d.Month & 0xF

		if d.Year < 2000 {
			buf[3] = uint8(d.Year - 1900)
		} else {
			buf[3] = uint8(d.Year - 2000)
		}
	}

	buf[3] &= 0x7F

	return buf
}

func (d *DPT_11001) Unpack(data []byte) error {
	if len(data) != 4 {
		return fmt.Errorf("invalid payload length")
	}

	d.Day = uint8(data[1] & 0x1F)
	d.Month = uint8(data[2] & 0xF)
	d.Year = uint16(data[3] & 0x7F)

	if d.Year > 99 {
		return fmt.Errorf("payload is out of range")
	}

	if d.Year == 0 && d.Month == 0 && d.Day == 0 {
		d.Year = 90
		d.Month = 1
		d.Day = 1
	}

	if d.Year >= 90 {
		d.Year += 1900
	} else {
		d.Year += 2000
	}

	if !d.IsValid() {
		return fmt.Errorf("payload is out of range")
	}

	return nil
}

func (d DPT_11001) Unit() string {
	return ""
}

func (d DPT_11001) IsValid() bool {
	tm := time.Date(int(d.Year), time.Month(d.Month), int(d.Day), 0, 0, 0, 0, time.UTC)
	if tm.Year() < 1990 || tm.Year() > 2089 {
		return false
	} else {
		return (int(d.Year) == tm.Year() && d.Month == uint8(tm.Month()) && int(d.Day) == tm.Day())
	}
}

func (d DPT_11001) String() string {
	return fmt.Sprintf("%04d-%02d-%02d", d.Year, d.Month, d.Day)
}
<|endoftext|>


<commit_before>package kontrol

import (
	"database/sql"
	"errors"
	"fmt"
	"regexp"
	"strings"

	"github.com/hashicorp/go-version"
	_ "github.com/lib/pq"

	"github.com/koding/kite"
	kontrolprotocol "github.com/koding/kite/kontrol/protocol"
	"github.com/koding/kite/protocol"
	"github.com/koding/logging"
)

// Postgres holds Postgresql database related configuration
type PostgresConfig struct {
	Host     string
	Port     int
	Username string
	Password string
	DBName   string
}

type Postgres struct {
	DB  *sql.DB
	Log logging.Logger
}

func NewPostgres(conf *PostgresConfig, log kite.Logger) *Postgres {
	if conf.Port == 0 {
		conf.Port = 5432
	}

	if conf.Host == "" {
		conf.Host = "localhost"
	}

	if conf.DBName == "" {
		conf.DBName = "test"
	}

	connString := fmt.Sprintf(
		"host=%s port=%d dbname=%s sslmode=disable",
		conf.Host, conf.Port, conf.DBName,
	)

	if conf.Password != "" {
		connString += " password=" + conf.Password
	}

	if conf.Username != "" {
		connString += " user=" + conf.Username
	}

	fmt.Printf("connString %+v\n", connString)

	db, err := sql.Open("postgres", connString)
	if err != nil {
		panic(err)
	}

	// enable the ltree module which we are going to use, any error means it's
	// failed so there is no sense to continue, panic!
	enableTree := `CREATE EXTENSION IF NOT EXISTS ltree`
	if _, err := db.Exec(enableTree); err != nil {
		panic(err)
	}

	// create our initial kites table
	// * kite is going to be our ltree
	// * url is cointaining the kite's register url
	// * id is going to be kites' unique id (which also exists in the ltree
	//   path). We are adding it as a primary key so each kite with the full
	//   path can only exist once.
	// * created_at and updated_at are updated at creation and updating (like
	//  if the URL has changed)
	// Some notes:
	// * path label can only contain a sequence of alphanumeric characters
	//   and underscores. So for example a version string of "1.0.4" needs to
	//   be converted to "1_0_4" or uuid of 1111-2222-3333-4444 needs to be
	//   converted to 1111_2222_3333_4444.
	table := `CREATE TABLE IF NOT EXISTS kites (
		kite ltree NOT NULL,
		url text NOT NULL,
		id uuid PRIMARY KEY,
		created_at timestamptz NOT NULL DEFAULT (NOW() AT TIME ZONE 'UTC'),
		updated_at timestamptz NOT NULL DEFAULT (NOW() AT TIME ZONE 'UTC')
	);`

	if _, err := db.Exec(table); err != nil {
		panic(err)
	}

	return &Postgres{
		DB: db,
	}
}

func (p *Postgres) Get(query *protocol.KontrolQuery) (Kites, error) {
	// only let query with usernames, otherwise the whole tree will be fetched
	// which is not good for us
	if query.Username == "" {
		return nil, errors.New("username is not specified in query")
	}

	// NewVersion returns an error if it's a constraint, like: ">= 1.0, < 1.4"
	_, err := version.NewVersion(query.Version)
	if err != nil && query.Version != "" {
		return nil, errors.New("version constraint is not implemented")
	}

	path := ltreePath(query)

	// TODO: see how a prepared statements perfoms out
	rows, err := p.DB.Query(`SELECT kite, url FROM kites WHERE kite <@ $1`, path)
	if err != nil {
		return nil, err
	}
	defer rows.Close()

	var kitePath string
	var url string

	kites := make(Kites, 0)

	for rows.Next() {
		err := rows.Scan(&kitePath, &url)
		if err != nil {
			return nil, err
		}

		kiteProt, err := kiteFromPath(kitePath)
		if err != nil {
			return nil, err
		}

		kites = append(kites, &protocol.KiteWithToken{
			Kite: *kiteProt,
			URL:  url,
		})

		fmt.Printf("kitePath %+v\n", kitePath)
		fmt.Printf("url %+v\n", url)
	}

	if err := rows.Err(); err != nil {
		return nil, err
	}

	kites.Shuffle()

	return kites, nil
}

func (p *Postgres) Set(kiteProt *protocol.Kite, value *kontrolprotocol.RegisterValue) error {
	_, err := p.DB.Exec("INSERT into kites(kite, url, id) VALUES($1, $2, $3)",
		ltreePath(kiteProt.Query()),
		value.URL,
		kiteProt.ID,
	)
	return err
}

func (p *Postgres) Delete(kite *protocol.Kite) error {
	return errors.New("DELETE is not implemented")
}

// ltreeLabel satisfies a invalid ltree definition of a label in path.
// According to the definition it is: "A label is a sequence of alphanumeric
// characters and underscores (for example, in C locale the characters
// A-Za-z0-9_ are allowed). Labels must be less than 256 bytes long."
//
// We could express one character with "[A-Za-z0-9_]", a word with
// "[A-Za-z0-9_]+". However we want to catch words that are not valid labels so
// we negate them with the "^" character, so it will be : "[^[A-Za-z0-9_]]+".
// Finally we cann use the POSIX character class: [:word:] which is:
// "Alphanumeric characters plus "_"", so the final regexp will be
// "[^[:word]]+"
var invalidLabelRe = regexp.MustCompile("[^[:word:]]+")

// ltreePath returns a query path to be used with the ltree module in postgress
// in the form of "username.environment.kitename.version.region.hostname.id"
func ltreePath(query *protocol.KontrolQuery) string {
	path := ""
	fields := query.Fields()

	// we stop for the first empty value
	for _, key := range keyOrder {
		v := fields[key]
		if v == "" {
			break
		}

		// replace anything that doesn't match the definition for a ltree path
		// label with a underscore, so the version "0.0.1" will be "0_0_1", or
		// uuid of "1111-2222-3333-4444" will be converted to
		// 1111_2222_3333_4444. Strings that satisifies the requirement are
		// untouched.
		v = invalidLabelRe.ReplaceAllLiteralString(v, "_")

		path = path + v + "."
	}

	// remove the latest dot which causes an invalid query
	path = strings.TrimSuffix(path, ".")
	return path
}

// kiteFromPath returns a Query from the given ltree path label
func kiteFromPath(path string) (*protocol.Kite, error) {
	fields := strings.Split(path, ".")

	if len(fields) != 7 {
		return nil, fmt.Errorf("invalid ltree path: %s", path)
	}

	// those labels were converted by us, therefore convert them back
	version := strings.Replace(fields[3], "_", ".", -1)
	id := strings.Replace(fields[6], "_", "-", -1)

	return &protocol.Kite{
		Username:    fields[0],
		Environment: fields[1],
		Name:        fields[2],
		Version:     version,
		Region:      fields[4],
		Hostname:    fields[5],
		ID:          id,
	}, nil

}
<commit_msg>kontrol/postgres: implement Update method for postgres<commit_after>package kontrol

import (
	"database/sql"
	"errors"
	"fmt"
	"regexp"
	"strings"

	"github.com/hashicorp/go-version"
	_ "github.com/lib/pq"

	"github.com/koding/kite"
	kontrolprotocol "github.com/koding/kite/kontrol/protocol"
	"github.com/koding/kite/protocol"
	"github.com/koding/logging"
)

// Postgres holds Postgresql database related configuration
type PostgresConfig struct {
	Host     string
	Port     int
	Username string
	Password string
	DBName   string
}

type Postgres struct {
	DB  *sql.DB
	Log logging.Logger
}

func NewPostgres(conf *PostgresConfig, log kite.Logger) *Postgres {
	if conf.Port == 0 {
		conf.Port = 5432
	}

	if conf.Host == "" {
		conf.Host = "localhost"
	}

	if conf.DBName == "" {
		conf.DBName = "test"
	}

	connString := fmt.Sprintf(
		"host=%s port=%d dbname=%s sslmode=disable",
		conf.Host, conf.Port, conf.DBName,
	)

	if conf.Password != "" {
		connString += " password=" + conf.Password
	}

	if conf.Username != "" {
		connString += " user=" + conf.Username
	}

	fmt.Printf("connString %+v\n", connString)

	db, err := sql.Open("postgres", connString)
	if err != nil {
		panic(err)
	}

	// enable the ltree module which we are going to use, any error means it's
	// failed so there is no sense to continue, panic!
	enableTree := `CREATE EXTENSION IF NOT EXISTS ltree`
	if _, err := db.Exec(enableTree); err != nil {
		panic(err)
	}

	// create our initial kites table
	// * kite is going to be our ltree
	// * url is cointaining the kite's register url
	// * id is going to be kites' unique id (which also exists in the ltree
	//   path). We are adding it as a primary key so each kite with the full
	//   path can only exist once.
	// * created_at and updated_at are updated at creation and updating (like
	//  if the URL has changed)
	// Some notes:
	// * path label can only contain a sequence of alphanumeric characters
	//   and underscores. So for example a version string of "1.0.4" needs to
	//   be converted to "1_0_4" or uuid of 1111-2222-3333-4444 needs to be
	//   converted to 1111_2222_3333_4444.
	table := `CREATE TABLE IF NOT EXISTS kites (
		kite ltree NOT NULL,
		url text NOT NULL,
		id uuid PRIMARY KEY,
		created_at timestamptz NOT NULL DEFAULT (NOW() AT TIME ZONE 'UTC'),
		updated_at timestamptz NOT NULL DEFAULT (NOW() AT TIME ZONE 'UTC')
	);`

	if _, err := db.Exec(table); err != nil {
		panic(err)
	}

	return &Postgres{
		DB: db,
	}
}

func (p *Postgres) Get(query *protocol.KontrolQuery) (Kites, error) {
	// only let query with usernames, otherwise the whole tree will be fetched
	// which is not good for us
	if query.Username == "" {
		return nil, errors.New("username is not specified in query")
	}

	// NewVersion returns an error if it's a constraint, like: ">= 1.0, < 1.4"
	_, err := version.NewVersion(query.Version)
	if err != nil && query.Version != "" {
		return nil, errors.New("version constraint is not implemented")
	}

	path := ltreePath(query)

	// TODO: see how a prepared statements perfoms out
	rows, err := p.DB.Query(`SELECT kite, url FROM kites WHERE kite <@ $1`, path)
	if err != nil {
		return nil, err
	}
	defer rows.Close()

	var kitePath string
	var url string

	kites := make(Kites, 0)

	for rows.Next() {
		err := rows.Scan(&kitePath, &url)
		if err != nil {
			return nil, err
		}

		kiteProt, err := kiteFromPath(kitePath)
		if err != nil {
			return nil, err
		}

		kites = append(kites, &protocol.KiteWithToken{
			Kite: *kiteProt,
			URL:  url,
		})

		fmt.Printf("kitePath %+v\n", kitePath)
		fmt.Printf("url %+v\n", url)
	}

	if err := rows.Err(); err != nil {
		return nil, err
	}

	kites.Shuffle()

	return kites, nil
}

func (p *Postgres) Add(kiteProt *protocol.Kite, value *kontrolprotocol.RegisterValue) error {
	_, err := p.DB.Exec("INSERT into kites(kite, url, id) VALUES($1, $2, $3)",
		ltreePath(kiteProt.Query()),
		value.URL,
		kiteProt.ID,
	)
	return err
}

func (p *Postgres) Update(kiteProt *protocol.Kite, value *kontrolprotocol.RegisterValue) error {
	// TODO: also consider just usting WHERE id = kiteProt.ID, see how it's
	// perfoms out
	_, err := p.DB.Exec(`UPDATE kites SET url = $1, updated_at = (now() at time zone 'utc')
	WHERE kite ~ $2`,
		value.URL, ltreePath(kiteProt.Query()))

	return err
}

func (p *Postgres) Delete(kite *protocol.Kite) error {
	return errors.New("DELETE is not implemented")
}

// ltreeLabel satisfies a invalid ltree definition of a label in path.
// According to the definition it is: "A label is a sequence of alphanumeric
// characters and underscores (for example, in C locale the characters
// A-Za-z0-9_ are allowed). Labels must be less than 256 bytes long."
//
// We could express one character with "[A-Za-z0-9_]", a word with
// "[A-Za-z0-9_]+". However we want to catch words that are not valid labels so
// we negate them with the "^" character, so it will be : "[^[A-Za-z0-9_]]+".
// Finally we cann use the POSIX character class: [:word:] which is:
// "Alphanumeric characters plus "_"", so the final regexp will be
// "[^[:word]]+"
var invalidLabelRe = regexp.MustCompile("[^[:word:]]+")

// ltreePath returns a query path to be used with the ltree module in postgress
// in the form of "username.environment.kitename.version.region.hostname.id"
func ltreePath(query *protocol.KontrolQuery) string {
	path := ""
	fields := query.Fields()

	// we stop for the first empty value
	for _, key := range keyOrder {
		v := fields[key]
		if v == "" {
			break
		}

		// replace anything that doesn't match the definition for a ltree path
		// label with a underscore, so the version "0.0.1" will be "0_0_1", or
		// uuid of "1111-2222-3333-4444" will be converted to
		// 1111_2222_3333_4444. Strings that satisifies the requirement are
		// untouched.
		v = invalidLabelRe.ReplaceAllLiteralString(v, "_")

		path = path + v + "."
	}

	// remove the latest dot which causes an invalid query
	path = strings.TrimSuffix(path, ".")
	return path
}

// kiteFromPath returns a Query from the given ltree path label
func kiteFromPath(path string) (*protocol.Kite, error) {
	fields := strings.Split(path, ".")

	if len(fields) != 7 {
		return nil, fmt.Errorf("invalid ltree path: %s", path)
	}

	// those labels were converted by us, therefore convert them back
	version := strings.Replace(fields[3], "_", ".", -1)
	id := strings.Replace(fields[6], "_", "-", -1)

	return &protocol.Kite{
		Username:    fields[0],
		Environment: fields[1],
		Name:        fields[2],
		Version:     version,
		Region:      fields[4],
		Hostname:    fields[5],
		ID:          id,
	}, nil

}
<|endoftext|>


<commit_before>package kontrol

import (
	"database/sql"
	"errors"
	"fmt"
	"net/url"
	"os"
	"regexp"
	"strings"
	"time"

	"github.com/hashicorp/go-version"
	_ "github.com/lib/pq"

	"github.com/koding/kite"
	kontrolprotocol "github.com/koding/kite/kontrol/protocol"
	"github.com/koding/kite/protocol"
)

// Postgres holds Postgresql database related configuration
type PostgresConfig struct {
	Host     string
	Port     int
	Username string
	Password string
	DBName   string
}

type Postgres struct {
	DB  *sql.DB
	Log kite.Logger
}

func NewPostgres(conf *PostgresConfig, log kite.Logger) *Postgres {
	if conf == nil {
		conf = &PostgresConfig{}
	}

	if conf.Port == 0 {
		conf.Port = 5432
	}

	if conf.Host == "" {
		conf.Host = "localhost"
	}

	if conf.DBName == "" {
		conf.DBName = os.Getenv("KONTROL_POSTGRES_DBNAME")
		if conf.DBName == "" {
			panic("db name is not set for postgres kontrol storage")
		}
	}

	connString := fmt.Sprintf(
		"host=%s port=%d dbname=%s sslmode=disable",
		conf.Host, conf.Port, conf.DBName,
	)

	if conf.Password != "" {
		connString += " password=" + conf.Password
	}

	if conf.Username == "" {
		conf.Username = os.Getenv("KONTROL_POSTGRES_USERNAME")
		if conf.Username == "" {
			panic("username is not set for postgres kontrol storage")
		}
	}

	connString += " user=" + conf.Username

	db, err := sql.Open("postgres", connString)
	if err != nil {
		panic(err)
	}

	// add a limit so we don't hit a "too many open connections" errors. We
	// might change this in the future to tweak according to the machine and
	// usage behaviour
	db.SetMaxIdleConns(100)
	db.SetMaxOpenConns(100)

	// enable the ltree module which we are going to use, any error means it's
	// failed so there is no sense to continue, panic!
	enableTree := `CREATE EXTENSION IF NOT EXISTS ltree`
	if _, err := db.Exec(enableTree); err != nil {
		panic(err)
	}

	// create our initial kites table
	// * kite is going to be our ltree
	// * url is containing the kite's register url
	// * id is going to be kites' unique id (which also exists in the ltree
	//   path). We are adding it as a primary key so each kite with the full
	//   path can only exist once.
	// * created_at and updated_at are updated at creation and updating (like
	//  if the URL has changed)
	// Some notes:
	// * path label can only contain a sequence of alphanumeric characters
	//   and underscores. So for example a version string of "1.0.4" needs to
	//   be converted to "1_0_4" or uuid of 1111-2222-3333-4444 needs to be
	//   converted to 1111_2222_3333_4444.
	table := `CREATE TABLE IF NOT EXISTS kites (
		kite ltree NOT NULL,
		url text NOT NULL,
		id uuid PRIMARY KEY,
		created_at timestamptz NOT NULL DEFAULT (NOW() AT TIME ZONE 'UTC'),
		updated_at timestamptz NOT NULL DEFAULT (NOW() AT TIME ZONE 'UTC')
	);`

	if _, err := db.Exec(table); err != nil {
		panic(err)
	}

	// We enable index on the kite field. We don't return on errors because the
	// operator `IF NOT EXISTS` doesn't work for index creation, therefore we
	// assume the indexes might be already created.
	enableGistIndex := `CREATE INDEX kite_path_gist_idx ON kites USING GIST(kite)`
	enableBtreeIndex := `CREATE INDEX kite_path_btree_idx ON kites USING BTREE(kite)`

	if _, err := db.Exec(enableGistIndex); err != nil {
		log.Warning("postgres: enable gist index: %s", err)
	}

	if _, err := db.Exec(enableBtreeIndex); err != nil {
		log.Warning("postgres: enable btree index: %s", err)
	}

	p := &Postgres{
		DB:  db,
		Log: log,
	}

	cleanInterval := 5 * time.Second   // clean every 5 second
	expireInterval := 10 * time.Second // clean rows that are 10 second old
	go p.RunCleaner(cleanInterval, expireInterval)

	return p
}

// RunCleaner delets every "interval" duration rows which are older than
// "expire" duration based on the "updated_at" field. For more info check
// CleanExpireRows which is used to delete old rows.
func (p *Postgres) RunCleaner(interval, expire time.Duration) {
	cleanFunc := func() {
		affectedRows, err := p.CleanExpiredRows(expire)
		if err != nil {
			p.Log.Warning("postgres: cleaning old rows failed: %s", err)
		} else if affectedRows != 0 {
			p.Log.Info("postgres: cleaned up %d rows", affectedRows)
		}
	}

	cleanFunc() // run for the first time
	for _ = range time.Tick(interval) {
		cleanFunc()
	}
}

// CleanExpiredRows deletes rows that are at least "expire" duration old. So if
// say an expire duration of 10 second is given, it will delete all rows that
// were updated 10 seconds ago
func (p *Postgres) CleanExpiredRows(expire time.Duration) (int64, error) {
	// See: http://stackoverflow.com/questions/14465727/how-to-insert-things-like-now-interval-2-minutes-into-php-pdo-query
	// basically by passing an integer to INTERVAL is not possible, we need to
	// cast it. However there is a more simpler way, we can multiply INTERVAL
	// with an integer so we just declare a one second INTERVAL and multiply it
	// with the amount we want.
	cleanOldRows := `DELETE FROM kites WHERE updated_at < (now() at time zone 'utc') - ((INTERVAL '1 second') * $1)`

	rows, err := p.DB.Exec(cleanOldRows, int64(expire/time.Second))
	if err != nil {
		return 0, err
	}

	return rows.RowsAffected()
}

func (p *Postgres) Get(query *protocol.KontrolQuery) (Kites, error) {
	// only let query with usernames, otherwise the whole tree will be fetched
	// which is not good for us
	if query.Username == "" {
		return nil, errors.New("username is not specified in query")
	}

	path := ltreePath(query)

	var hasVersionConstraint bool // does query contains a constraint on version?
	var keyRest string            // query key after the version field
	var versionConstraint version.Constraints
	// NewVersion returns an error if it's a constraint, like: ">= 1.0, < 1.4"
	_, err := version.NewVersion(query.Version)
	if err != nil && query.Version != "" {
		// now parse our constraint
		versionConstraint, err = version.NewConstraint(query.Version)
		if err != nil {
			// version is a malformed, just return the error
			return nil, err
		}

		hasVersionConstraint = true
		nameQuery := &protocol.KontrolQuery{
			Username:    query.Username,
			Environment: query.Environment,
			Name:        query.Name,
		}

		// We will make a get request to all nodes under this name
		// and filter the result later.
		path = ltreePath(nameQuery)

		// Rest of the key after version field
		keyRest = "/" + strings.TrimRight(
			query.Region+"/"+query.Hostname+"/"+query.ID, "/")

	}

	rows, err := p.DB.Query(`SELECT kite, url FROM kites WHERE kite <@ $1`, path)
	if err != nil {
		return nil, err
	}
	defer rows.Close()

	var kitePath string
	var url string

	kites := make(Kites, 0)

	for rows.Next() {
		err := rows.Scan(&kitePath, &url)
		if err != nil {
			return nil, err
		}

		kiteProt, err := kiteFromPath(kitePath)
		if err != nil {
			return nil, err
		}

		kites = append(kites, &protocol.KiteWithToken{
			Kite: *kiteProt,
			URL:  url,
		})
	}

	if err := rows.Err(); err != nil {
		return nil, err
	}

	// if it's just single result there is no need to shuffle or filter
	// according to the version constraint
	if len(kites) == 1 {
		return kites, nil
	}

	// Filter kites by version constraint
	if hasVersionConstraint {
		kites.Filter(versionConstraint, keyRest)
	}

	// randomize the result
	kites.Shuffle()

	return kites, nil
}

func (p *Postgres) Add(kiteProt *protocol.Kite, value *kontrolprotocol.RegisterValue) error {
	// check that the incoming URL is valid to prevent malformed input
	_, err := url.Parse(value.URL)
	if err != nil {
		return err
	}

	_, err = p.DB.Exec("INSERT into kites(kite, url, id) VALUES($1, $2, $3)",
		ltreePath(kiteProt.Query()),
		value.URL,
		kiteProt.ID,
	)
	return err
}

func (p *Postgres) Update(kiteProt *protocol.Kite, value *kontrolprotocol.RegisterValue) error {
	// check that the incoming url is valid to prevent malformed input
	_, err := url.Parse(value.URL)
	if err != nil {
		return err
	}

	// TODO: also consider just using WHERE id = kiteProt.ID, see how it's
	// performs out
	_, err = p.DB.Exec(`UPDATE kites SET url = $1, updated_at = (now() at time zone 'utc')
	WHERE kite ~ $2`,
		value.URL, ltreePath(kiteProt.Query()))

	return err
}

func (p *Postgres) Delete(kiteProt *protocol.Kite) error {
	deleteKite := `DELETE FROM kites WHERE kite ~ $1`
	_, err := p.DB.Exec(deleteKite, ltreePath(kiteProt.Query()))
	return err
}

func (p *Postgres) Clear() error {
	_, err := p.DB.Exec(`DROP TABLE kites`)
	return err
}

// ltreeLabel satisfies a invalid ltree definition of a label in path.
// According to the definition it is: "A label is a sequence of alphanumeric
// characters and underscores (for example, in C locale the characters
// A-Za-z0-9_ are allowed). Labels must be less than 256 bytes long."
//
// We could express one character with "[A-Za-z0-9_]", a word with
// "[A-Za-z0-9_]+". However we want to catch words that are not valid labels so
// we negate them with the "^" character, so it will be : "[^[A-Za-z0-9_]]+".
// Finally we cann use the POSIX character class: [:word:] which is:
// "Alphanumeric characters plus "_"", so the final regexp will be
// "[^[:word]]+"
var invalidLabelRe = regexp.MustCompile("[^[:word:]]+")

// ltreePath returns a query path to be used with the ltree module in postgress
// in the form of "username.environment.kitename.version.region.hostname.id"
func ltreePath(query *protocol.KontrolQuery) string {
	path := ""
	fields := query.Fields()

	// we stop for the first empty value
	for _, key := range keyOrder {
		v := fields[key]
		if v == "" {
			break
		}

		// replace anything that doesn't match the definition for a ltree path
		// label with a underscore, so the version "0.0.1" will be "0_0_1", or
		// uuid of "1111-2222-3333-4444" will be converted to
		// 1111_2222_3333_4444. Strings that satisfies the requirement are
		// untouched.
		v = invalidLabelRe.ReplaceAllLiteralString(v, "_")

		path = path + v + "."
	}

	// remove the latest dot which causes an invalid query
	path = strings.TrimSuffix(path, ".")
	return path
}

// kiteFromPath returns a Query from the given ltree path label
func kiteFromPath(path string) (*protocol.Kite, error) {
	fields := strings.Split(path, ".")

	if len(fields) != 7 {
		return nil, fmt.Errorf("invalid ltree path: %s", path)
	}

	// those labels were converted by us, therefore convert them back
	version := strings.Replace(fields[3], "_", ".", -1)
	id := strings.Replace(fields[6], "_", "-", -1)

	return &protocol.Kite{
		Username:    fields[0],
		Environment: fields[1],
		Name:        fields[2],
		Version:     version,
		Region:      fields[4],
		Hostname:    fields[5],
		ID:          id,
	}, nil

}
<commit_msg>kontrol/postgres: remove limitations until we found a reason to use them<commit_after>package kontrol

import (
	"database/sql"
	"errors"
	"fmt"
	"net/url"
	"os"
	"regexp"
	"strings"
	"time"

	"github.com/hashicorp/go-version"
	_ "github.com/lib/pq"

	"github.com/koding/kite"
	kontrolprotocol "github.com/koding/kite/kontrol/protocol"
	"github.com/koding/kite/protocol"
)

// Postgres holds Postgresql database related configuration
type PostgresConfig struct {
	Host     string
	Port     int
	Username string
	Password string
	DBName   string
}

type Postgres struct {
	DB  *sql.DB
	Log kite.Logger
}

func NewPostgres(conf *PostgresConfig, log kite.Logger) *Postgres {
	if conf == nil {
		conf = &PostgresConfig{}
	}

	if conf.Port == 0 {
		conf.Port = 5432
	}

	if conf.Host == "" {
		conf.Host = "localhost"
	}

	if conf.DBName == "" {
		conf.DBName = os.Getenv("KONTROL_POSTGRES_DBNAME")
		if conf.DBName == "" {
			panic("db name is not set for postgres kontrol storage")
		}
	}

	connString := fmt.Sprintf(
		"host=%s port=%d dbname=%s sslmode=disable",
		conf.Host, conf.Port, conf.DBName,
	)

	if conf.Password != "" {
		connString += " password=" + conf.Password
	}

	if conf.Username == "" {
		conf.Username = os.Getenv("KONTROL_POSTGRES_USERNAME")
		if conf.Username == "" {
			panic("username is not set for postgres kontrol storage")
		}
	}

	connString += " user=" + conf.Username

	db, err := sql.Open("postgres", connString)
	if err != nil {
		panic(err)
	}

	// enable the ltree module which we are going to use, any error means it's
	// failed so there is no sense to continue, panic!
	enableTree := `CREATE EXTENSION IF NOT EXISTS ltree`
	if _, err := db.Exec(enableTree); err != nil {
		panic(err)
	}

	// create our initial kites table
	// * kite is going to be our ltree
	// * url is containing the kite's register url
	// * id is going to be kites' unique id (which also exists in the ltree
	//   path). We are adding it as a primary key so each kite with the full
	//   path can only exist once.
	// * created_at and updated_at are updated at creation and updating (like
	//  if the URL has changed)
	// Some notes:
	// * path label can only contain a sequence of alphanumeric characters
	//   and underscores. So for example a version string of "1.0.4" needs to
	//   be converted to "1_0_4" or uuid of 1111-2222-3333-4444 needs to be
	//   converted to 1111_2222_3333_4444.
	table := `CREATE TABLE IF NOT EXISTS kites (
		kite ltree NOT NULL,
		url text NOT NULL,
		id uuid PRIMARY KEY,
		created_at timestamptz NOT NULL DEFAULT (NOW() AT TIME ZONE 'UTC'),
		updated_at timestamptz NOT NULL DEFAULT (NOW() AT TIME ZONE 'UTC')
	);`

	if _, err := db.Exec(table); err != nil {
		panic(err)
	}

	// We enable index on the kite field. We don't return on errors because the
	// operator `IF NOT EXISTS` doesn't work for index creation, therefore we
	// assume the indexes might be already created.
	enableGistIndex := `CREATE INDEX kite_path_gist_idx ON kites USING GIST(kite)`
	enableBtreeIndex := `CREATE INDEX kite_path_btree_idx ON kites USING BTREE(kite)`

	if _, err := db.Exec(enableGistIndex); err != nil {
		log.Warning("postgres: enable gist index: %s", err)
	}

	if _, err := db.Exec(enableBtreeIndex); err != nil {
		log.Warning("postgres: enable btree index: %s", err)
	}

	p := &Postgres{
		DB:  db,
		Log: log,
	}

	cleanInterval := 5 * time.Second   // clean every 5 second
	expireInterval := 10 * time.Second // clean rows that are 10 second old
	go p.RunCleaner(cleanInterval, expireInterval)

	return p
}

// RunCleaner delets every "interval" duration rows which are older than
// "expire" duration based on the "updated_at" field. For more info check
// CleanExpireRows which is used to delete old rows.
func (p *Postgres) RunCleaner(interval, expire time.Duration) {
	cleanFunc := func() {
		affectedRows, err := p.CleanExpiredRows(expire)
		if err != nil {
			p.Log.Warning("postgres: cleaning old rows failed: %s", err)
		} else if affectedRows != 0 {
			p.Log.Info("postgres: cleaned up %d rows", affectedRows)
		}
	}

	cleanFunc() // run for the first time
	for _ = range time.Tick(interval) {
		cleanFunc()
	}
}

// CleanExpiredRows deletes rows that are at least "expire" duration old. So if
// say an expire duration of 10 second is given, it will delete all rows that
// were updated 10 seconds ago
func (p *Postgres) CleanExpiredRows(expire time.Duration) (int64, error) {
	// See: http://stackoverflow.com/questions/14465727/how-to-insert-things-like-now-interval-2-minutes-into-php-pdo-query
	// basically by passing an integer to INTERVAL is not possible, we need to
	// cast it. However there is a more simpler way, we can multiply INTERVAL
	// with an integer so we just declare a one second INTERVAL and multiply it
	// with the amount we want.
	cleanOldRows := `DELETE FROM kites WHERE updated_at < (now() at time zone 'utc') - ((INTERVAL '1 second') * $1)`

	rows, err := p.DB.Exec(cleanOldRows, int64(expire/time.Second))
	if err != nil {
		return 0, err
	}

	return rows.RowsAffected()
}

func (p *Postgres) Get(query *protocol.KontrolQuery) (Kites, error) {
	// only let query with usernames, otherwise the whole tree will be fetched
	// which is not good for us
	if query.Username == "" {
		return nil, errors.New("username is not specified in query")
	}

	path := ltreePath(query)

	var hasVersionConstraint bool // does query contains a constraint on version?
	var keyRest string            // query key after the version field
	var versionConstraint version.Constraints
	// NewVersion returns an error if it's a constraint, like: ">= 1.0, < 1.4"
	_, err := version.NewVersion(query.Version)
	if err != nil && query.Version != "" {
		// now parse our constraint
		versionConstraint, err = version.NewConstraint(query.Version)
		if err != nil {
			// version is a malformed, just return the error
			return nil, err
		}

		hasVersionConstraint = true
		nameQuery := &protocol.KontrolQuery{
			Username:    query.Username,
			Environment: query.Environment,
			Name:        query.Name,
		}

		// We will make a get request to all nodes under this name
		// and filter the result later.
		path = ltreePath(nameQuery)

		// Rest of the key after version field
		keyRest = "/" + strings.TrimRight(
			query.Region+"/"+query.Hostname+"/"+query.ID, "/")

	}

	rows, err := p.DB.Query(`SELECT kite, url FROM kites WHERE kite <@ $1`, path)
	if err != nil {
		return nil, err
	}
	defer rows.Close()

	var kitePath string
	var url string

	kites := make(Kites, 0)

	for rows.Next() {
		err := rows.Scan(&kitePath, &url)
		if err != nil {
			return nil, err
		}

		kiteProt, err := kiteFromPath(kitePath)
		if err != nil {
			return nil, err
		}

		kites = append(kites, &protocol.KiteWithToken{
			Kite: *kiteProt,
			URL:  url,
		})
	}

	if err := rows.Err(); err != nil {
		return nil, err
	}

	// if it's just single result there is no need to shuffle or filter
	// according to the version constraint
	if len(kites) == 1 {
		return kites, nil
	}

	// Filter kites by version constraint
	if hasVersionConstraint {
		kites.Filter(versionConstraint, keyRest)
	}

	// randomize the result
	kites.Shuffle()

	return kites, nil
}

func (p *Postgres) Add(kiteProt *protocol.Kite, value *kontrolprotocol.RegisterValue) error {
	// check that the incoming URL is valid to prevent malformed input
	_, err := url.Parse(value.URL)
	if err != nil {
		return err
	}

	_, err = p.DB.Exec("INSERT into kites(kite, url, id) VALUES($1, $2, $3)",
		ltreePath(kiteProt.Query()),
		value.URL,
		kiteProt.ID,
	)
	return err
}

func (p *Postgres) Update(kiteProt *protocol.Kite, value *kontrolprotocol.RegisterValue) error {
	// check that the incoming url is valid to prevent malformed input
	_, err := url.Parse(value.URL)
	if err != nil {
		return err
	}

	// TODO: also consider just using WHERE id = kiteProt.ID, see how it's
	// performs out
	_, err = p.DB.Exec(`UPDATE kites SET url = $1, updated_at = (now() at time zone 'utc')
	WHERE kite ~ $2`,
		value.URL, ltreePath(kiteProt.Query()))

	return err
}

func (p *Postgres) Delete(kiteProt *protocol.Kite) error {
	deleteKite := `DELETE FROM kites WHERE kite ~ $1`
	_, err := p.DB.Exec(deleteKite, ltreePath(kiteProt.Query()))
	return err
}

func (p *Postgres) Clear() error {
	_, err := p.DB.Exec(`DROP TABLE kites`)
	return err
}

// ltreeLabel satisfies a invalid ltree definition of a label in path.
// According to the definition it is: "A label is a sequence of alphanumeric
// characters and underscores (for example, in C locale the characters
// A-Za-z0-9_ are allowed). Labels must be less than 256 bytes long."
//
// We could express one character with "[A-Za-z0-9_]", a word with
// "[A-Za-z0-9_]+". However we want to catch words that are not valid labels so
// we negate them with the "^" character, so it will be : "[^[A-Za-z0-9_]]+".
// Finally we cann use the POSIX character class: [:word:] which is:
// "Alphanumeric characters plus "_"", so the final regexp will be
// "[^[:word]]+"
var invalidLabelRe = regexp.MustCompile("[^[:word:]]+")

// ltreePath returns a query path to be used with the ltree module in postgress
// in the form of "username.environment.kitename.version.region.hostname.id"
func ltreePath(query *protocol.KontrolQuery) string {
	path := ""
	fields := query.Fields()

	// we stop for the first empty value
	for _, key := range keyOrder {
		v := fields[key]
		if v == "" {
			break
		}

		// replace anything that doesn't match the definition for a ltree path
		// label with a underscore, so the version "0.0.1" will be "0_0_1", or
		// uuid of "1111-2222-3333-4444" will be converted to
		// 1111_2222_3333_4444. Strings that satisfies the requirement are
		// untouched.
		v = invalidLabelRe.ReplaceAllLiteralString(v, "_")

		path = path + v + "."
	}

	// remove the latest dot which causes an invalid query
	path = strings.TrimSuffix(path, ".")
	return path
}

// kiteFromPath returns a Query from the given ltree path label
func kiteFromPath(path string) (*protocol.Kite, error) {
	fields := strings.Split(path, ".")

	if len(fields) != 7 {
		return nil, fmt.Errorf("invalid ltree path: %s", path)
	}

	// those labels were converted by us, therefore convert them back
	version := strings.Replace(fields[3], "_", ".", -1)
	id := strings.Replace(fields[6], "_", "-", -1)

	return &protocol.Kite{
		Username:    fields[0],
		Environment: fields[1],
		Name:        fields[2],
		Version:     version,
		Region:      fields[4],
		Hostname:    fields[5],
		ID:          id,
	}, nil

}
<|endoftext|>


<commit_before>package server

import (
	"fmt"
	"log"
	"net/http"
	"time"

	"github.com/labstack/echo"
	"github.com/nzai/stockrecorder/market"
	"github.com/nzai/stockrecorder/server/result"
)

//	注册路由
func registerRoute(e *echo.Echo) {

	e.Get("/", welcome)

	e.Get("/1m/query", queryPeroid60)
}

func welcome(c *echo.Context) error {
	return c.String(http.StatusOK, "Welcome to stockrecorder http service!")
}

//	查询分时数据
func queryPeroid60(c *echo.Context) error {

	_market := c.Query("market")
	code := c.Query("code")
	start := c.Query("start")
	end := c.Query("end")

	log.Printf("m=%s c=%s s=%s e=%s", _market, code, start, end)
	if _market == "" || code == "" || start == "" || end == "" {
		return c.JSON(http.StatusOK, result.Failed("查询参数为空"))
	}

	_start, err := time.Parse("20060102", start)
	if err != nil {
		return c.JSON(http.StatusOK, result.Failed("查询参数不正确"))
	}

	_end, err := time.Parse("20060102", end)
	if err != nil {
		return c.JSON(http.StatusOK, result.Failed("查询参数不正确"))
	}

	//	查询
	peroids, err := market.QueryPeroid60(_market, code, _start, _end)
	if err != nil {
		log.Printf("[Query]\t查询分时数据发生错误(m=%s c=%s s=%s e=%s):%s", _market, code, start, end, err.Error())
		return c.JSON(http.StatusInternalServerError, fmt.Sprintf("查询分时数据发生错误:%s", err.Error()))
	}

	resultList := make([][]float32, 0)
	for _, p := range peroids {
		resultList = append(resultList, []float32{p.Open, p.Close, p.High, p.Low, float32(p.Volume)})
	}

	return c.JSON(http.StatusOK, result.Create(resultList))
}
<commit_msg>成交量为0的忽略<commit_after>package server

import (
	"log"
	"net/http"
	"time"

	"github.com/labstack/echo"
	"github.com/nzai/stockrecorder/market"
	"github.com/nzai/stockrecorder/server/result"
)

//	注册路由
func registerRoute(e *echo.Echo) {

	e.Get("/", welcome)

	e.Get("/1m/query", queryPeroid60)
}

func welcome(c *echo.Context) error {
	return c.String(http.StatusOK, "Welcome to stockrecorder http service!")
}

//	查询分时数据
func queryPeroid60(c *echo.Context) error {

	_market := c.Query("market")
	code := c.Query("code")
	start := c.Query("start")
	end := c.Query("end")

	//	log.Printf("m=%s c=%s s=%s e=%s", _market, code, start, end)
	if _market == "" || code == "" || start == "" || end == "" {
		return c.JSON(http.StatusOK, result.Failed("查询参数为空"))
	}

	_start, err := time.Parse("20060102", start)
	if err != nil {
		return c.JSON(http.StatusOK, result.Failed("查询参数不正确"))
	}

	_end, err := time.Parse("20060102", end)
	if err != nil {
		return c.JSON(http.StatusOK, result.Failed("查询参数不正确"))
	}

	//	查询
	peroids, err := market.QueryPeroid60(_market, code, _start, _end)
	if err != nil {
		log.Printf("[Query]\t查询分时数据发生错误(m=%s c=%s s=%s e=%s):%s", _market, code, start, end, err.Error())
		return c.JSON(http.StatusOK, result.Failed("查询分时数据发生错误"))
	}

	resultList := make([][]float32, 0)
	for _, p := range peroids {
		if p.Volume == 0 {
			continue
		}

		resultList = append(resultList, []float32{p.Open, p.Close, p.High, p.Low, float32(p.Volume)})
	}

	return c.JSON(http.StatusOK, result.Create(resultList))
}
<|endoftext|>


<commit_before><commit_msg>Multibyte Line Highlight<commit_after><|endoftext|>


<commit_before><commit_msg>add rw lock in cache<commit_after><|endoftext|>


<commit_before>package main

import (
	"bufio"
	"errors"
	"fmt"
	"os"
	"runtime"
	"strconv"
	"strings"
	"sync"
	"time"
)

func ParseLine(line string) (*Example, error) {
	tokens := strings.Split(line, "\t")
	var url string
	if len(tokens) == 1 {
		url = tokens[0]
		return NewExample(url, UNLABELED), nil
	} else if len(tokens) == 2 {
		url = tokens[0]
		label, _ := strconv.ParseInt(tokens[1], 10, 0)
		switch LabelType(label) {
		case POSITIVE, NEGATIVE, UNLABELED:
			return NewExample(url, LabelType(label)), nil
		default:
			return nil, errors.New(fmt.Sprintf("Invalid Label type %d in %s", label, line))
		}
	} else {
		return nil, errors.New(fmt.Sprintf("Invalid line: %s", line))
	}
}

func ReadExamples(filename string) ([]*Example, error) {
	fp, err := os.Open(filename)
	defer fp.Close()
	if err != nil {
		return nil, err
	}

	scanner := bufio.NewScanner(fp)
	var examples Examples
	for scanner.Scan() {
		line := scanner.Text()
		e, err := ParseLine(line)
		if err != nil {
			return nil, err
		}
		examples = append(examples, e)
	}
	if err := scanner.Err(); err != nil {
		return nil, err
	}
	return examples, nil
}

func WriteExamples(examples Examples, filename string) error {
	fp, err := os.Create(filename)
	defer fp.Close()
	if err != nil {
		return err
	}

	writer := bufio.NewWriter(fp)
	for _, e := range examples {
		if e.IsNew && e.IsLabeled() {
			_, err := writer.WriteString(e.Url + "\t" + strconv.Itoa(int(e.Label)) + "\n")
			if err != nil {
				return err
			}
		}
	}

	writer.Flush()
	return nil
}

func FilterLabeledExamples(examples Examples) Examples {
	var result Examples
	for _, e := range examples {
		if e.IsLabeled() {
			result = append(result, e)
		}
	}
	return result
}

func FilterUnlabeledExamples(examples Examples) Examples {
	result := Examples{}

	alreadyLabeledByURL := make(map[string]bool)
	alreadyLabeledByTitle := make(map[string]bool)
	for _, e := range FilterLabeledExamples(examples) {
		alreadyLabeledByURL[e.Url] = true
		alreadyLabeledByTitle[e.Title] = true
	}

	for _, e := range examples {
		if _, ok := alreadyLabeledByURL[e.Url]; ok {
			continue
		}
		if _, ok := alreadyLabeledByTitle[e.Title]; ok {
			continue
		}
		if !e.IsLabeled() {
			alreadyLabeledByURL[e.Url] = true
			alreadyLabeledByTitle[e.Title] = true
			result = append(result, e)
		}
	}
	return result
}

func removeDuplicate(args []string) []string {
	results := make([]string, 0)
	encountered := map[string]bool{}
	for i := 0; i < len(args); i++ {
		if !encountered[args[i]] {
			encountered[args[i]] = true
			results = append(results, args[i])
		}
	}
	return results
}

func AttachMetaData(cache *Cache, examples Examples) {
	oldStdout := os.Stdout
	readFile, writeFile, _ := os.Pipe()
	os.Stdout = writeFile

	defer func() {
		writeFile.Close()
		readFile.Close()
		os.Stdout = oldStdout
	}()

	shuffle(examples)

	wg := &sync.WaitGroup{}
	cpus := runtime.NumCPU()
	runtime.GOMAXPROCS(cpus)
	sem := make(chan struct{}, 20)
	for idx, e := range examples {
		wg.Add(1)
		sem <- struct{}{}
		go func(e *Example, idx int) {
			defer wg.Done()
			if example, ok := cache.Get(*e); ok {
				e.Title = example.Title
				e.FinalUrl = example.FinalUrl
				e.Description = example.Description
				e.Body = example.Body
				e.StatusCode = example.StatusCode
				e.Fv = example.Fv
			} else {
				article := GetArticle(e.Url)
				fmt.Fprintln(os.Stderr, "Fetching("+strconv.Itoa(idx)+"): "+article.Url)
				e.Title = article.Title
				e.FinalUrl = article.Url
				e.Description = article.Description
				e.Body = article.Body
				e.StatusCode = article.StatusCode
				e.Fv = removeDuplicate(ExtractFeatures(*e))
				cache.Add(*e)
			}
			<-sem
		}(e, idx)
	}
	wg.Wait()
}

func FilterStatusCodeOkExamples(examples Examples) Examples {
	result := Examples{}

	for _, e := range examples {
		if e.StatusCode == 200 {
			result = append(result, e)
		}
	}

	return result
}

func NewOutputFilename() string {
	t := time.Now()
	return fmt.Sprintf("%d-%02d-%02d-%02d-%02d.txt", t.Year(), t.Month(), t.Day(), t.Hour(), t.Minute())
}
<commit_msg>リダイレクト後のurlを出力するように<commit_after>package main

import (
	"bufio"
	"errors"
	"fmt"
	"os"
	"runtime"
	"strconv"
	"strings"
	"sync"
	"time"
)

func ParseLine(line string) (*Example, error) {
	tokens := strings.Split(line, "\t")
	var url string
	if len(tokens) == 1 {
		url = tokens[0]
		return NewExample(url, UNLABELED), nil
	} else if len(tokens) == 2 {
		url = tokens[0]
		label, _ := strconv.ParseInt(tokens[1], 10, 0)
		switch LabelType(label) {
		case POSITIVE, NEGATIVE, UNLABELED:
			return NewExample(url, LabelType(label)), nil
		default:
			return nil, errors.New(fmt.Sprintf("Invalid Label type %d in %s", label, line))
		}
	} else {
		return nil, errors.New(fmt.Sprintf("Invalid line: %s", line))
	}
}

func ReadExamples(filename string) ([]*Example, error) {
	fp, err := os.Open(filename)
	defer fp.Close()
	if err != nil {
		return nil, err
	}

	scanner := bufio.NewScanner(fp)
	var examples Examples
	for scanner.Scan() {
		line := scanner.Text()
		e, err := ParseLine(line)
		if err != nil {
			return nil, err
		}
		examples = append(examples, e)
	}
	if err := scanner.Err(); err != nil {
		return nil, err
	}
	return examples, nil
}

func WriteExamples(examples Examples, filename string) error {
	fp, err := os.Create(filename)
	defer fp.Close()
	if err != nil {
		return err
	}

	writer := bufio.NewWriter(fp)
	for _, e := range examples {
		if e.IsNew && e.IsLabeled() {
			url := e.FinalUrl
			if url == "" {
				url = e.Url
			}
			_, err := writer.WriteString(url + "\t" + strconv.Itoa(int(e.Label)) + "\n")
			if err != nil {
				return err
			}
		}
	}

	writer.Flush()
	return nil
}

func FilterLabeledExamples(examples Examples) Examples {
	var result Examples
	for _, e := range examples {
		if e.IsLabeled() {
			result = append(result, e)
		}
	}
	return result
}

func FilterUnlabeledExamples(examples Examples) Examples {
	result := Examples{}

	alreadyLabeledByURL := make(map[string]bool)
	alreadyLabeledByTitle := make(map[string]bool)
	for _, e := range FilterLabeledExamples(examples) {
		alreadyLabeledByURL[e.Url] = true
		alreadyLabeledByTitle[e.Title] = true
	}

	for _, e := range examples {
		if _, ok := alreadyLabeledByURL[e.Url]; ok {
			continue
		}
		if _, ok := alreadyLabeledByTitle[e.Title]; ok {
			continue
		}
		if !e.IsLabeled() {
			alreadyLabeledByURL[e.Url] = true
			alreadyLabeledByTitle[e.Title] = true
			result = append(result, e)
		}
	}
	return result
}

func removeDuplicate(args []string) []string {
	results := make([]string, 0)
	encountered := map[string]bool{}
	for i := 0; i < len(args); i++ {
		if !encountered[args[i]] {
			encountered[args[i]] = true
			results = append(results, args[i])
		}
	}
	return results
}

func AttachMetaData(cache *Cache, examples Examples) {
	oldStdout := os.Stdout
	readFile, writeFile, _ := os.Pipe()
	os.Stdout = writeFile

	defer func() {
		writeFile.Close()
		readFile.Close()
		os.Stdout = oldStdout
	}()

	shuffle(examples)

	wg := &sync.WaitGroup{}
	cpus := runtime.NumCPU()
	runtime.GOMAXPROCS(cpus)
	sem := make(chan struct{}, 20)
	for idx, e := range examples {
		wg.Add(1)
		sem <- struct{}{}
		go func(e *Example, idx int) {
			defer wg.Done()
			if example, ok := cache.Get(*e); ok {
				e.Title = example.Title
				e.FinalUrl = example.FinalUrl
				e.Description = example.Description
				e.Body = example.Body
				e.StatusCode = example.StatusCode
				e.Fv = example.Fv
			} else {
				article := GetArticle(e.Url)
				fmt.Fprintln(os.Stderr, "Fetching("+strconv.Itoa(idx)+"): "+article.Url)
				e.Title = article.Title
				e.FinalUrl = article.Url
				e.Description = article.Description
				e.Body = article.Body
				e.StatusCode = article.StatusCode
				e.Fv = removeDuplicate(ExtractFeatures(*e))
				cache.Add(*e)
			}
			<-sem
		}(e, idx)
	}
	wg.Wait()
}

func FilterStatusCodeOkExamples(examples Examples) Examples {
	result := Examples{}

	for _, e := range examples {
		if e.StatusCode == 200 {
			result = append(result, e)
		}
	}

	return result
}

func NewOutputFilename() string {
	t := time.Now()
	return fmt.Sprintf("%d-%02d-%02d-%02d-%02d.txt", t.Year(), t.Month(), t.Day(), t.Hour(), t.Minute())
}
<|endoftext|>


<commit_before>// Copyright 2020 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package server

import (
	"context"
	"encoding/json"
	"fmt"
	"strings"

	"cloud.google.com/go/bigtable"
	pb "github.com/datacommonsorg/mixer/proto"
	"google.golang.org/protobuf/encoding/protojson"
)

type rankKey struct {
	prov    string
	mmethod string
}

// Ranking for (import name, measurement method) combination. This is used to rank
// multiple dataset for the same StatisticalVariable, where lower value means
// higher ranking.
// The ranking score ranges from 0 to 100.
var statsRanking = map[rankKey]int{
	{"CensusPEP", "CensusPEPSurvey"}:                   0, // Population
	{"CensusACS5YearSurvey", "CensusACS5yrSurvey"}:     1, // Population
	{"EurostatData", "EurostatRegionalPopulationData"}: 2, // Population
	{"WorldDevelopmentIndicators", ""}:                 3, // Population
	{"BLS_LAUS", "BLSSeasonallyUnadjusted"}:            0, // Unemployment Rate
	{"EurostatData", ""}:                               1, // Unemployment Rate
	{"NYT_COVID19", "NYT_COVID19_GitHub"}:              0, // Covid
}

const lowestRank = 100

// Limit the concurrent channels when processing in-memory cache data.
const maxChannelSize = 50

// GetStats implements API for Mixer.GetStats.
func (s *Server) GetStats(ctx context.Context, in *pb.GetStatsRequest) (
	*pb.GetStatsResponse, error) {
	placeDcids := in.GetPlace()
	statsVarDcid := in.GetStatsVar()
	if len(placeDcids) == 0 || statsVarDcid == "" {
		return nil, fmt.Errorf("missing required arguments")
	}

	mmethod := in.GetMeasurementMethod()
	unit := in.GetUnit()
	op := in.GetObservationPeriod()

	// Read triples for stats var.
	triplesRowList := buildTriplesKey([]string{statsVarDcid})
	triples, err := readTriples(ctx, s.btTable, triplesRowList)
	if err != nil {
		return nil, err
	}
	// Get the StatisticalVariable
	if triples[statsVarDcid] == nil {
		return nil, fmt.Errorf("No stats var found for %s", statsVarDcid)
	}
	statsVar, err := triplesToStatsVar(statsVarDcid, triples[statsVarDcid])
	if err != nil {
		return nil, err
	}
	// Construct BigTable row keys.
	rowList := buildStatsKey(placeDcids, statsVar)

	result := map[string]*ObsTimeSeries{}

	// Read data from branch in-memory cache first.
	if in.GetOption().GetCacheChoice() != pb.Option_BASE_CACHE_ONLY {
		tmp := s.memcache.ReadParallel(rowList, convertToObsSeries, nil)
		for dcid, data := range tmp {
			if data == nil {
				result[dcid] = nil
			} else {
				result[dcid] = data.(*ObsTimeSeries)
			}
		}
	}
	// For each place, if the data is missing in branch cache, fetch it from the
	// base cache in Bigtable.
	if len(result) < len(placeDcids) {
		extraDcids := []string{}
		for _, dcid := range placeDcids {
			if _, ok := result[dcid]; !ok {
				extraDcids = append(extraDcids, dcid)
			}
		}
		extraData, err := readStats(ctx, s.btTable, buildStatsKey(extraDcids, statsVar))
		if err != nil {
			return nil, err
		}
		for dcid := range extraData {
			result[dcid] = extraData[dcid]
		}
	}

	// Fill missing place data and result result
	for _, dcid := range placeDcids {
		if _, ok := result[dcid]; !ok {
			result[dcid] = nil
		}
	}
	for dcid := range result {
		result[dcid] = filterAndRank(result[dcid], mmethod, unit, op)
	}
	jsonRaw, err := json.Marshal(result)
	if err != nil {
		return nil, err
	}
	return &pb.GetStatsResponse{Payload: string(jsonRaw)}, nil
}

// triplesToStatsVar converts a Triples cache into a StatisticalVarible object.
func triplesToStatsVar(
	statsVarDcid string, triples *TriplesCache) (*StatisticalVariable, error) {
	// Get constraint properties.
	propValMap := map[string]string{}
	for _, t := range triples.Triples {
		if t.Predicate == "constraintProperties" {
			propValMap[t.ObjectID] = ""
		}
	}
	statsVar := StatisticalVariable{}
	// Populate the field.
	for _, t := range triples.Triples {
		if t.SubjectID != statsVarDcid {
			continue
		}
		object := t.ObjectID
		switch t.Predicate {
		case "typeOf":
			if object != "StatisticalVariable" {
				return nil, fmt.Errorf("%s is not a StatisticalVariable", t.SubjectID)
			}
		case "statType":
			statsVar.StatType = strings.Replace(object, "Value", "", 1)
		case "populationType":
			statsVar.PopType = object
		case "measurementMethod":
			statsVar.MeasurementMethod = object
		case "measuredProperty":
			statsVar.MeasuredProp = object
		case "measurementDenominator":
			statsVar.MeasurementDenominator = object
		case "measurementQualifier":
			statsVar.MeasurementQualifier = object
		case "scalingFactor":
			statsVar.ScalingFactor = object
		case "unit":
			statsVar.Unit = object
		default:
			if _, ok := propValMap[t.Predicate]; ok {
				if statsVar.PVs == nil {
					statsVar.PVs = map[string]string{}
				}
				statsVar.PVs[t.Predicate] = object
			}
		}
	}
	return &statsVar, nil
}

func filterAndRank(
	in *ObsTimeSeries, mmethod, op, unit string) *ObsTimeSeries {
	out := &ObsTimeSeries{
		PlaceDcid: in.PlaceDcid,
		PlaceName: in.PlaceName,
	}
	filteredSeries := []*SourceSeries{}
	for _, series := range in.SourceSeries {
		if mmethod != "" && mmethod != series.MeasurementMethod {
			continue
		}
		if op != "" && op != series.ObservationPeriod {
			continue
		}
		if unit != "" && unit != series.Unit {
			continue
		}
		filteredSeries = append(filteredSeries, series)
	}
	bestScore := lowestRank
	for _, series := range filteredSeries {
		key := rankKey{series.ImportName, series.MeasurementMethod}
		score, ok := statsRanking[key]
		if !ok {
			score = lowestRank
		}
		if score <= bestScore {
			out.Data = series.Val
			// TODO(boxu): correct this when source url is populated in cache data.
			out.ProvenanceDomain = series.ProvenanceDomain
			bestScore = score
		}
	}
	out.SourceSeries = nil
	return out
}

func convertToObsSeries(dcid string, jsonRaw []byte) (interface{}, error) {
	pbData := &pb.ChartStore{}
	if err := protojson.Unmarshal(jsonRaw, pbData); err != nil {
		return nil, err
	}
	switch x := pbData.Val.(type) {
	case *pb.ChartStore_ObsTimeSeries:
		pbSourceSeries := x.ObsTimeSeries.GetSourceSeries()
		ret := &ObsTimeSeries{
			Data:         x.ObsTimeSeries.GetData(),
			PlaceName:    x.ObsTimeSeries.GetPlaceName(),
			PlaceDcid:    dcid,
			SourceSeries: make([]*SourceSeries, len(pbSourceSeries)),
		}
		for i, source := range pbSourceSeries {
			ret.SourceSeries[i] = &SourceSeries{
				ImportName:        source.GetImportName(),
				ObservationPeriod: source.GetObservationPeriod(),
				MeasurementMethod: source.GetMeasurementMethod(),
				ScalingFactor:     source.GetScalingFactor(),
				Unit:              source.GetUnit(),
				ProvenanceDomain:  source.GetProvenanceDomain(),
				Val:               source.GetVal(),
			}
		}
		ret.ProvenanceDomain = x.ObsTimeSeries.GetProvenanceDomain()
		return ret, nil
	case nil:
		return nil, fmt.Errorf("ChartStore.Val is not set")
	default:
		return nil, fmt.Errorf("ChartStore.Val has unexpected type %T", x)
	}
}

// readStats reads and process BigTable rows in parallel.
// Consider consolidate this function and bigTableReadRowsParallel.
func readStats(ctx context.Context, btTable *bigtable.Table,
	rowList bigtable.RowList) (map[string]*ObsTimeSeries, error) {

	dataMap, err := bigTableReadRowsParallel(
		ctx, btTable, rowList, convertToObsSeries, nil)
	if err != nil {
		return nil, err
	}
	result := map[string]*ObsTimeSeries{}
	for dcid, data := range dataMap {
		if data == nil {
			result[dcid] = nil
		} else {
			result[dcid] = data.(*ObsTimeSeries)
		}
	}
	return result, nil
}
<commit_msg>Check for nil pointer of stats data (#275)<commit_after>// Copyright 2020 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package server

import (
	"context"
	"encoding/json"
	"fmt"
	"strings"

	"cloud.google.com/go/bigtable"
	pb "github.com/datacommonsorg/mixer/proto"
	"google.golang.org/protobuf/encoding/protojson"
)

type rankKey struct {
	prov    string
	mmethod string
}

// Ranking for (import name, measurement method) combination. This is used to rank
// multiple dataset for the same StatisticalVariable, where lower value means
// higher ranking.
// The ranking score ranges from 0 to 100.
var statsRanking = map[rankKey]int{
	{"CensusPEP", "CensusPEPSurvey"}:                   0, // Population
	{"CensusACS5YearSurvey", "CensusACS5yrSurvey"}:     1, // Population
	{"EurostatData", "EurostatRegionalPopulationData"}: 2, // Population
	{"WorldDevelopmentIndicators", ""}:                 3, // Population
	{"BLS_LAUS", "BLSSeasonallyUnadjusted"}:            0, // Unemployment Rate
	{"EurostatData", ""}:                               1, // Unemployment Rate
	{"NYT_COVID19", "NYT_COVID19_GitHub"}:              0, // Covid
}

const lowestRank = 100

// Limit the concurrent channels when processing in-memory cache data.
const maxChannelSize = 50

// GetStats implements API for Mixer.GetStats.
func (s *Server) GetStats(ctx context.Context, in *pb.GetStatsRequest) (
	*pb.GetStatsResponse, error) {
	placeDcids := in.GetPlace()
	statsVarDcid := in.GetStatsVar()
	if len(placeDcids) == 0 || statsVarDcid == "" {
		return nil, fmt.Errorf("missing required arguments")
	}

	mmethod := in.GetMeasurementMethod()
	unit := in.GetUnit()
	op := in.GetObservationPeriod()

	// Read triples for stats var.
	triplesRowList := buildTriplesKey([]string{statsVarDcid})
	triples, err := readTriples(ctx, s.btTable, triplesRowList)
	if err != nil {
		return nil, err
	}
	// Get the StatisticalVariable
	if triples[statsVarDcid] == nil {
		return nil, fmt.Errorf("No stats var found for %s", statsVarDcid)
	}
	statsVar, err := triplesToStatsVar(statsVarDcid, triples[statsVarDcid])
	if err != nil {
		return nil, err
	}
	// Construct BigTable row keys.
	rowList := buildStatsKey(placeDcids, statsVar)

	result := map[string]*ObsTimeSeries{}

	// Read data from branch in-memory cache first.
	if in.GetOption().GetCacheChoice() != pb.Option_BASE_CACHE_ONLY {
		tmp := s.memcache.ReadParallel(rowList, convertToObsSeries, nil)
		for dcid, data := range tmp {
			if data == nil {
				result[dcid] = nil
			} else {
				result[dcid] = data.(*ObsTimeSeries)
			}
		}
	}
	// For each place, if the data is missing in branch cache, fetch it from the
	// base cache in Bigtable.
	if len(result) < len(placeDcids) {
		extraDcids := []string{}
		for _, dcid := range placeDcids {
			if _, ok := result[dcid]; !ok {
				extraDcids = append(extraDcids, dcid)
			}
		}
		extraData, err := readStats(ctx, s.btTable, buildStatsKey(extraDcids, statsVar))
		if err != nil {
			return nil, err
		}
		for dcid := range extraData {
			result[dcid] = extraData[dcid]
		}
	}

	// Fill missing place data and result result
	for _, dcid := range placeDcids {
		if _, ok := result[dcid]; !ok {
			result[dcid] = nil
		}
	}
	for dcid := range result {
		result[dcid] = filterAndRank(result[dcid], mmethod, unit, op)
	}
	jsonRaw, err := json.Marshal(result)
	if err != nil {
		return nil, err
	}
	return &pb.GetStatsResponse{Payload: string(jsonRaw)}, nil
}

// triplesToStatsVar converts a Triples cache into a StatisticalVarible object.
func triplesToStatsVar(
	statsVarDcid string, triples *TriplesCache) (*StatisticalVariable, error) {
	// Get constraint properties.
	propValMap := map[string]string{}
	for _, t := range triples.Triples {
		if t.Predicate == "constraintProperties" {
			propValMap[t.ObjectID] = ""
		}
	}
	statsVar := StatisticalVariable{}
	// Populate the field.
	for _, t := range triples.Triples {
		if t.SubjectID != statsVarDcid {
			continue
		}
		object := t.ObjectID
		switch t.Predicate {
		case "typeOf":
			if object != "StatisticalVariable" {
				return nil, fmt.Errorf("%s is not a StatisticalVariable", t.SubjectID)
			}
		case "statType":
			statsVar.StatType = strings.Replace(object, "Value", "", 1)
		case "populationType":
			statsVar.PopType = object
		case "measurementMethod":
			statsVar.MeasurementMethod = object
		case "measuredProperty":
			statsVar.MeasuredProp = object
		case "measurementDenominator":
			statsVar.MeasurementDenominator = object
		case "measurementQualifier":
			statsVar.MeasurementQualifier = object
		case "scalingFactor":
			statsVar.ScalingFactor = object
		case "unit":
			statsVar.Unit = object
		default:
			if _, ok := propValMap[t.Predicate]; ok {
				if statsVar.PVs == nil {
					statsVar.PVs = map[string]string{}
				}
				statsVar.PVs[t.Predicate] = object
			}
		}
	}
	return &statsVar, nil
}

func filterAndRank(
	in *ObsTimeSeries, mmethod, op, unit string) *ObsTimeSeries {
	if in == nil {
		return nil
	}
	out := &ObsTimeSeries{
		PlaceDcid: in.PlaceDcid,
		PlaceName: in.PlaceName,
	}
	filteredSeries := []*SourceSeries{}
	for _, series := range in.SourceSeries {
		if mmethod != "" && mmethod != series.MeasurementMethod {
			continue
		}
		if op != "" && op != series.ObservationPeriod {
			continue
		}
		if unit != "" && unit != series.Unit {
			continue
		}
		filteredSeries = append(filteredSeries, series)
	}
	bestScore := lowestRank
	for _, series := range filteredSeries {
		key := rankKey{series.ImportName, series.MeasurementMethod}
		score, ok := statsRanking[key]
		if !ok {
			score = lowestRank
		}
		if score <= bestScore {
			out.Data = series.Val
			// TODO(boxu): correct this when source url is populated in cache data.
			out.ProvenanceDomain = series.ProvenanceDomain
			bestScore = score
		}
	}
	out.SourceSeries = nil
	return out
}

func convertToObsSeries(dcid string, jsonRaw []byte) (interface{}, error) {
	pbData := &pb.ChartStore{}
	if err := protojson.Unmarshal(jsonRaw, pbData); err != nil {
		return nil, err
	}
	switch x := pbData.Val.(type) {
	case *pb.ChartStore_ObsTimeSeries:
		pbSourceSeries := x.ObsTimeSeries.GetSourceSeries()
		ret := &ObsTimeSeries{
			Data:         x.ObsTimeSeries.GetData(),
			PlaceName:    x.ObsTimeSeries.GetPlaceName(),
			PlaceDcid:    dcid,
			SourceSeries: make([]*SourceSeries, len(pbSourceSeries)),
		}
		for i, source := range pbSourceSeries {
			ret.SourceSeries[i] = &SourceSeries{
				ImportName:        source.GetImportName(),
				ObservationPeriod: source.GetObservationPeriod(),
				MeasurementMethod: source.GetMeasurementMethod(),
				ScalingFactor:     source.GetScalingFactor(),
				Unit:              source.GetUnit(),
				ProvenanceDomain:  source.GetProvenanceDomain(),
				Val:               source.GetVal(),
			}
		}
		ret.ProvenanceDomain = x.ObsTimeSeries.GetProvenanceDomain()
		return ret, nil
	case nil:
		return nil, fmt.Errorf("ChartStore.Val is not set")
	default:
		return nil, fmt.Errorf("ChartStore.Val has unexpected type %T", x)
	}
}

// readStats reads and process BigTable rows in parallel.
// Consider consolidate this function and bigTableReadRowsParallel.
func readStats(ctx context.Context, btTable *bigtable.Table,
	rowList bigtable.RowList) (map[string]*ObsTimeSeries, error) {

	dataMap, err := bigTableReadRowsParallel(
		ctx, btTable, rowList, convertToObsSeries, nil)
	if err != nil {
		return nil, err
	}
	result := map[string]*ObsTimeSeries{}
	for dcid, data := range dataMap {
		if data == nil {
			result[dcid] = nil
		} else {
			result[dcid] = data.(*ObsTimeSeries)
		}
	}
	return result, nil
}
<|endoftext|>


<commit_before>package gzip

import (
	"compress/gzip"
	"net/http"
)

const (
	HeaderAcceptEncoding  = "Accept-Encoding"
	HeaderContentEncoding = "Content-Encoding"
	HeaderContentLength   = "Content-Length"
	HeaderVary            = "Vary"
)

type gzipResponseWriter struct {
	*gzip.Writer
	w http.ResponseWriter
}

func (grw gzipResponseWriter) Header() http.Header {
	return grw.w.Header()
}

func (grw gzipResponseWriter) WriteHeader(code int) {
	grw.Header().Del(HeaderContentLength)
	grw.w.WriteHeader(code)
}

func (grw gzipResponseWriter) Write(b []byte) (int, error) {
	return grw.Writer.Write(b)
}
<commit_msg>-n [Migrated] embedding ResponseWriter from martini<commit_after>package gzip

import (
	"compress/gzip"
	"github.com/codegnagsta/martini"
	"net/http"
)

const (
	HeaderAcceptEncoding  = "Accept-Encoding"
	HeaderContentEncoding = "Content-Encoding"
	HeaderContentLength   = "Content-Length"
	HeaderVary            = "Vary"
)

type gzipResponseWriter struct {
	*gzip.Writer
	w martini.ResponseWriter
}

func (grw gzipResponseWriter) Header() http.Header {
	return grw.w.Header()
}

func (grw gzipResponseWriter) WriteHeader(code int) {
	grw.Header().Del(HeaderContentLength)
	grw.w.WriteHeader(code)
}

func (grw gzipResponseWriter) Write(b []byte) (int, error) {
	return grw.Writer.Write(b)
}
<|endoftext|>