<commit_before>package formatter

import (
	"bytes"
	"fmt"
	"strings"
	"text/template"

	"github.com/docker/distribution/reference"
	"github.com/docker/docker/api/types"
	units "github.com/docker/go-units"
)

const (
	defaultDiskUsageImageTableFormat     = "table {{.Repository}}\t{{.Tag}}\t{{.ID}}\t{{.CreatedSince}} ago\t{{.VirtualSize}}\t{{.SharedSize}}\t{{.UniqueSize}}\t{{.Containers}}"
	defaultDiskUsageContainerTableFormat = "table {{.ID}}\t{{.Image}}\t{{.Command}}\t{{.LocalVolumes}}\t{{.Size}}\t{{.RunningFor}} ago\t{{.Status}}\t{{.Names}}"
	defaultDiskUsageVolumeTableFormat    = "table {{.Name}}\t{{.Links}}\t{{.Size}}"
	defaultDiskUsageTableFormat          = "table {{.Type}}\t{{.TotalCount}}\t{{.Active}}\t{{.Size}}\t{{.Reclaimable}}"

	typeHeader        = "TYPE"
	totalHeader       = "TOTAL"
	activeHeader      = "ACTIVE"
	reclaimableHeader = "RECLAIMABLE"
	containersHeader  = "CONTAINERS"
	sharedSizeHeader  = "SHARED SIZE"
	uniqueSizeHeader  = "UNIQUE SiZE"
)

// DiskUsageContext contains disk usage specific information required by the formatter, encapsulate a Context struct.
type DiskUsageContext struct {
	Context
	Verbose    bool
	LayersSize int64
	Images     []*types.ImageSummary
	Containers []*types.Container
	Volumes    []*types.Volume
}

func (ctx *DiskUsageContext) startSubsection(format string) (*template.Template, error) {
	ctx.buffer = bytes.NewBufferString("")
	ctx.header = ""
	ctx.Format = Format(format)
	ctx.preFormat()

	return ctx.parseFormat()
}

//
// NewDiskUsageFormat returns a format for rendering an DiskUsageContext
func NewDiskUsageFormat(source string) Format {
	switch source {
	case TableFormatKey:
		format := defaultDiskUsageTableFormat
		return Format(format)
	case RawFormatKey:
		format := `type: {{.Type}}
total: {{.TotalCount}}
active: {{.Active}}
size: {{.Size}}
reclaimable: {{.Reclaimable}}
`
		return Format(format)
	}
	return Format(source)
}

func (ctx *DiskUsageContext) Write() (err error) {
	if !ctx.Verbose {
		ctx.buffer = bytes.NewBufferString("")
		ctx.preFormat()

		tmpl, err := ctx.parseFormat()
		if err != nil {
			return err
		}

		err = ctx.contextFormat(tmpl, &diskUsageImagesContext{
			totalSize: ctx.LayersSize,
			images:    ctx.Images,
		})
		if err != nil {
			return err
		}
		err = ctx.contextFormat(tmpl, &diskUsageContainersContext{
			containers: ctx.Containers,
		})
		if err != nil {
			return err
		}

		err = ctx.contextFormat(tmpl, &diskUsageVolumesContext{
			volumes: ctx.Volumes,
		})
		if err != nil {
			return err
		}

		diskUsageContainersCtx := diskUsageContainersContext{containers: []*types.Container{}}
		diskUsageContainersCtx.header = map[string]string{
			"Type":        typeHeader,
			"TotalCount":  totalHeader,
			"Active":      activeHeader,
			"Size":        sizeHeader,
			"Reclaimable": reclaimableHeader,
		}
		ctx.postFormat(tmpl, &diskUsageContainersCtx)

		return err
	}

	// First images
	tmpl, err := ctx.startSubsection(defaultDiskUsageImageTableFormat)
	if err != nil {
		return
	}

	ctx.Output.Write([]byte("Images space usage:\n\n"))
	for _, i := range ctx.Images {
		repo := "<none>"
		tag := "<none>"
		if len(i.RepoTags) > 0 && !isDangling(*i) {
			// Only show the first tag
			ref, err := reference.ParseNormalizedNamed(i.RepoTags[0])
			if err != nil {
				continue
			}
			if nt, ok := ref.(reference.NamedTagged); ok {
				repo = reference.FamiliarName(ref)
				tag = nt.Tag()
			}
		}

		err = ctx.contextFormat(tmpl, &imageContext{
			repo:  repo,
			tag:   tag,
			trunc: true,
			i:     *i,
		})
		if err != nil {
			return
		}
	}
	ctx.postFormat(tmpl, newImageContext())

	// Now containers
	ctx.Output.Write([]byte("\nContainers space usage:\n\n"))
	tmpl, err = ctx.startSubsection(defaultDiskUsageContainerTableFormat)
	if err != nil {
		return
	}
	for _, c := range ctx.Containers {
		// Don't display the virtual size
		c.SizeRootFs = 0
		err = ctx.contextFormat(tmpl, &containerContext{
			trunc: true,
			c:     *c,
		})
		if err != nil {
			return
		}
	}
	ctx.postFormat(tmpl, newContainerContext())

	// And volumes
	ctx.Output.Write([]byte("\nLocal Volumes space usage:\n\n"))
	tmpl, err = ctx.startSubsection(defaultDiskUsageVolumeTableFormat)
	if err != nil {
		return
	}
	for _, v := range ctx.Volumes {
		err = ctx.contextFormat(tmpl, &volumeContext{
			v: *v,
		})
		if err != nil {
			return
		}
	}
	ctx.postFormat(tmpl, newVolumeContext())
	return
}

type diskUsageImagesContext struct {
	HeaderContext
	totalSize int64
	images    []*types.ImageSummary
}

func (c *diskUsageImagesContext) MarshalJSON() ([]byte, error) {
	return marshalJSON(c)
}

func (c *diskUsageImagesContext) Type() string {
	return "Images"
}

func (c *diskUsageImagesContext) TotalCount() string {
	return fmt.Sprintf("%d", len(c.images))
}

func (c *diskUsageImagesContext) Active() string {
	used := 0
	for _, i := range c.images {
		if i.Containers > 0 {
			used++
		}
	}

	return fmt.Sprintf("%d", used)
}

func (c *diskUsageImagesContext) Size() string {
	return units.HumanSize(float64(c.totalSize))

}

func (c *diskUsageImagesContext) Reclaimable() string {
	var used int64

	for _, i := range c.images {
		if i.Containers != 0 {
			if i.VirtualSize == -1 || i.SharedSize == -1 {
				continue
			}
			used += i.VirtualSize - i.SharedSize
		}
	}

	reclaimable := c.totalSize - used
	if c.totalSize > 0 {
		return fmt.Sprintf("%s (%v%%)", units.HumanSize(float64(reclaimable)), (reclaimable*100)/c.totalSize)
	}
	return fmt.Sprintf("%s", units.HumanSize(float64(reclaimable)))
}

type diskUsageContainersContext struct {
	HeaderContext
	containers []*types.Container
}

func (c *diskUsageContainersContext) MarshalJSON() ([]byte, error) {
	return marshalJSON(c)
}

func (c *diskUsageContainersContext) Type() string {
	return "Containers"
}

func (c *diskUsageContainersContext) TotalCount() string {
	return fmt.Sprintf("%d", len(c.containers))
}

func (c *diskUsageContainersContext) isActive(container types.Container) bool {
	return strings.Contains(container.State, "running") ||
		strings.Contains(container.State, "paused") ||
		strings.Contains(container.State, "restarting")
}

func (c *diskUsageContainersContext) Active() string {
	used := 0
	for _, container := range c.containers {
		if c.isActive(*container) {
			used++
		}
	}

	return fmt.Sprintf("%d", used)
}

func (c *diskUsageContainersContext) Size() string {
	var size int64

	for _, container := range c.containers {
		size += container.SizeRw
	}

	return units.HumanSize(float64(size))
}

func (c *diskUsageContainersContext) Reclaimable() string {
	var reclaimable int64
	var totalSize int64

	for _, container := range c.containers {
		if !c.isActive(*container) {
			reclaimable += container.SizeRw
		}
		totalSize += container.SizeRw
	}

	if totalSize > 0 {
		return fmt.Sprintf("%s (%v%%)", units.HumanSize(float64(reclaimable)), (reclaimable*100)/totalSize)
	}

	return fmt.Sprintf("%s", units.HumanSize(float64(reclaimable)))
}

type diskUsageVolumesContext struct {
	HeaderContext
	volumes []*types.Volume
}

func (c *diskUsageVolumesContext) MarshalJSON() ([]byte, error) {
	return marshalJSON(c)
}

func (c *diskUsageVolumesContext) Type() string {
	return "Local Volumes"
}

func (c *diskUsageVolumesContext) TotalCount() string {
	return fmt.Sprintf("%d", len(c.volumes))
}

func (c *diskUsageVolumesContext) Active() string {

	used := 0
	for _, v := range c.volumes {
		if v.UsageData.RefCount > 0 {
			used++
		}
	}

	return fmt.Sprintf("%d", used)
}

func (c *diskUsageVolumesContext) Size() string {
	var size int64

	for _, v := range c.volumes {
		if v.UsageData.Size != -1 {
			size += v.UsageData.Size
		}
	}

	return units.HumanSize(float64(size))
}

func (c *diskUsageVolumesContext) Reclaimable() string {
	var reclaimable int64
	var totalSize int64

	for _, v := range c.volumes {
		if v.UsageData.Size != -1 {
			if v.UsageData.RefCount == 0 {
				reclaimable += v.UsageData.Size
			}
			totalSize += v.UsageData.Size
		}
	}

	if totalSize > 0 {
		return fmt.Sprintf("%s (%v%%)", units.HumanSize(float64(reclaimable)), (reclaimable*100)/totalSize)
	}

	return fmt.Sprintf("%s", units.HumanSize(float64(reclaimable)))
}
<commit_msg>Fix cyclomatic complexity of two formatters<commit_after>package formatter

import (
	"bytes"
	"fmt"
	"strings"
	"text/template"

	"github.com/docker/distribution/reference"
	"github.com/docker/docker/api/types"
	units "github.com/docker/go-units"
)

const (
	defaultDiskUsageImageTableFormat     = "table {{.Repository}}\t{{.Tag}}\t{{.ID}}\t{{.CreatedSince}} ago\t{{.VirtualSize}}\t{{.SharedSize}}\t{{.UniqueSize}}\t{{.Containers}}"
	defaultDiskUsageContainerTableFormat = "table {{.ID}}\t{{.Image}}\t{{.Command}}\t{{.LocalVolumes}}\t{{.Size}}\t{{.RunningFor}} ago\t{{.Status}}\t{{.Names}}"
	defaultDiskUsageVolumeTableFormat    = "table {{.Name}}\t{{.Links}}\t{{.Size}}"
	defaultDiskUsageTableFormat          = "table {{.Type}}\t{{.TotalCount}}\t{{.Active}}\t{{.Size}}\t{{.Reclaimable}}"

	typeHeader        = "TYPE"
	totalHeader       = "TOTAL"
	activeHeader      = "ACTIVE"
	reclaimableHeader = "RECLAIMABLE"
	containersHeader  = "CONTAINERS"
	sharedSizeHeader  = "SHARED SIZE"
	uniqueSizeHeader  = "UNIQUE SiZE"
)

// DiskUsageContext contains disk usage specific information required by the formatter, encapsulate a Context struct.
type DiskUsageContext struct {
	Context
	Verbose    bool
	LayersSize int64
	Images     []*types.ImageSummary
	Containers []*types.Container
	Volumes    []*types.Volume
}

func (ctx *DiskUsageContext) startSubsection(format string) (*template.Template, error) {
	ctx.buffer = bytes.NewBufferString("")
	ctx.header = ""
	ctx.Format = Format(format)
	ctx.preFormat()

	return ctx.parseFormat()
}

//
// NewDiskUsageFormat returns a format for rendering an DiskUsageContext
func NewDiskUsageFormat(source string) Format {
	switch source {
	case TableFormatKey:
		format := defaultDiskUsageTableFormat
		return Format(format)
	case RawFormatKey:
		format := `type: {{.Type}}
total: {{.TotalCount}}
active: {{.Active}}
size: {{.Size}}
reclaimable: {{.Reclaimable}}
`
		return Format(format)
	}
	return Format(source)
}

func (ctx *DiskUsageContext) Write() (err error) {
	if ctx.Verbose {
		return ctx.verboseWrite()
	}
	ctx.buffer = bytes.NewBufferString("")
	ctx.preFormat()

	tmpl, err := ctx.parseFormat()
	if err != nil {
		return err
	}

	err = ctx.contextFormat(tmpl, &diskUsageImagesContext{
		totalSize: ctx.LayersSize,
		images:    ctx.Images,
	})
	if err != nil {
		return err
	}
	err = ctx.contextFormat(tmpl, &diskUsageContainersContext{
		containers: ctx.Containers,
	})
	if err != nil {
		return err
	}

	err = ctx.contextFormat(tmpl, &diskUsageVolumesContext{
		volumes: ctx.Volumes,
	})
	if err != nil {
		return err
	}

	diskUsageContainersCtx := diskUsageContainersContext{containers: []*types.Container{}}
	diskUsageContainersCtx.header = map[string]string{
		"Type":        typeHeader,
		"TotalCount":  totalHeader,
		"Active":      activeHeader,
		"Size":        sizeHeader,
		"Reclaimable": reclaimableHeader,
	}
	ctx.postFormat(tmpl, &diskUsageContainersCtx)

	return err
}

func (ctx *DiskUsageContext) verboseWrite() (err error) {
	// First images
	tmpl, err := ctx.startSubsection(defaultDiskUsageImageTableFormat)
	if err != nil {
		return
	}

	ctx.Output.Write([]byte("Images space usage:\n\n"))
	for _, i := range ctx.Images {
		repo := "<none>"
		tag := "<none>"
		if len(i.RepoTags) > 0 && !isDangling(*i) {
			// Only show the first tag
			ref, err := reference.ParseNormalizedNamed(i.RepoTags[0])
			if err != nil {
				continue
			}
			if nt, ok := ref.(reference.NamedTagged); ok {
				repo = reference.FamiliarName(ref)
				tag = nt.Tag()
			}
		}

		err = ctx.contextFormat(tmpl, &imageContext{
			repo:  repo,
			tag:   tag,
			trunc: true,
			i:     *i,
		})
		if err != nil {
			return
		}
	}
	ctx.postFormat(tmpl, newImageContext())

	// Now containers
	ctx.Output.Write([]byte("\nContainers space usage:\n\n"))
	tmpl, err = ctx.startSubsection(defaultDiskUsageContainerTableFormat)
	if err != nil {
		return
	}
	for _, c := range ctx.Containers {
		// Don't display the virtual size
		c.SizeRootFs = 0
		err = ctx.contextFormat(tmpl, &containerContext{
			trunc: true,
			c:     *c,
		})
		if err != nil {
			return
		}
	}
	ctx.postFormat(tmpl, newContainerContext())

	// And volumes
	ctx.Output.Write([]byte("\nLocal Volumes space usage:\n\n"))
	tmpl, err = ctx.startSubsection(defaultDiskUsageVolumeTableFormat)
	if err != nil {
		return
	}
	for _, v := range ctx.Volumes {
		err = ctx.contextFormat(tmpl, &volumeContext{
			v: *v,
		})
		if err != nil {
			return
		}
	}
	ctx.postFormat(tmpl, newVolumeContext())
	return
}

type diskUsageImagesContext struct {
	HeaderContext
	totalSize int64
	images    []*types.ImageSummary
}

func (c *diskUsageImagesContext) MarshalJSON() ([]byte, error) {
	return marshalJSON(c)
}

func (c *diskUsageImagesContext) Type() string {
	return "Images"
}

func (c *diskUsageImagesContext) TotalCount() string {
	return fmt.Sprintf("%d", len(c.images))
}

func (c *diskUsageImagesContext) Active() string {
	used := 0
	for _, i := range c.images {
		if i.Containers > 0 {
			used++
		}
	}

	return fmt.Sprintf("%d", used)
}

func (c *diskUsageImagesContext) Size() string {
	return units.HumanSize(float64(c.totalSize))

}

func (c *diskUsageImagesContext) Reclaimable() string {
	var used int64

	for _, i := range c.images {
		if i.Containers != 0 {
			if i.VirtualSize == -1 || i.SharedSize == -1 {
				continue
			}
			used += i.VirtualSize - i.SharedSize
		}
	}

	reclaimable := c.totalSize - used
	if c.totalSize > 0 {
		return fmt.Sprintf("%s (%v%%)", units.HumanSize(float64(reclaimable)), (reclaimable*100)/c.totalSize)
	}
	return fmt.Sprintf("%s", units.HumanSize(float64(reclaimable)))
}

type diskUsageContainersContext struct {
	HeaderContext
	containers []*types.Container
}

func (c *diskUsageContainersContext) MarshalJSON() ([]byte, error) {
	return marshalJSON(c)
}

func (c *diskUsageContainersContext) Type() string {
	return "Containers"
}

func (c *diskUsageContainersContext) TotalCount() string {
	return fmt.Sprintf("%d", len(c.containers))
}

func (c *diskUsageContainersContext) isActive(container types.Container) bool {
	return strings.Contains(container.State, "running") ||
		strings.Contains(container.State, "paused") ||
		strings.Contains(container.State, "restarting")
}

func (c *diskUsageContainersContext) Active() string {
	used := 0
	for _, container := range c.containers {
		if c.isActive(*container) {
			used++
		}
	}

	return fmt.Sprintf("%d", used)
}

func (c *diskUsageContainersContext) Size() string {
	var size int64

	for _, container := range c.containers {
		size += container.SizeRw
	}

	return units.HumanSize(float64(size))
}

func (c *diskUsageContainersContext) Reclaimable() string {
	var reclaimable int64
	var totalSize int64

	for _, container := range c.containers {
		if !c.isActive(*container) {
			reclaimable += container.SizeRw
		}
		totalSize += container.SizeRw
	}

	if totalSize > 0 {
		return fmt.Sprintf("%s (%v%%)", units.HumanSize(float64(reclaimable)), (reclaimable*100)/totalSize)
	}

	return fmt.Sprintf("%s", units.HumanSize(float64(reclaimable)))
}

type diskUsageVolumesContext struct {
	HeaderContext
	volumes []*types.Volume
}

func (c *diskUsageVolumesContext) MarshalJSON() ([]byte, error) {
	return marshalJSON(c)
}

func (c *diskUsageVolumesContext) Type() string {
	return "Local Volumes"
}

func (c *diskUsageVolumesContext) TotalCount() string {
	return fmt.Sprintf("%d", len(c.volumes))
}

func (c *diskUsageVolumesContext) Active() string {

	used := 0
	for _, v := range c.volumes {
		if v.UsageData.RefCount > 0 {
			used++
		}
	}

	return fmt.Sprintf("%d", used)
}

func (c *diskUsageVolumesContext) Size() string {
	var size int64

	for _, v := range c.volumes {
		if v.UsageData.Size != -1 {
			size += v.UsageData.Size
		}
	}

	return units.HumanSize(float64(size))
}

func (c *diskUsageVolumesContext) Reclaimable() string {
	var reclaimable int64
	var totalSize int64

	for _, v := range c.volumes {
		if v.UsageData.Size != -1 {
			if v.UsageData.RefCount == 0 {
				reclaimable += v.UsageData.Size
			}
			totalSize += v.UsageData.Size
		}
	}

	if totalSize > 0 {
		return fmt.Sprintf("%s (%v%%)", units.HumanSize(float64(reclaimable)), (reclaimable*100)/totalSize)
	}

	return fmt.Sprintf("%s", units.HumanSize(float64(reclaimable)))
}
<|endoftext|>


<commit_before>// Copyright 2018 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package hugo

// CurrentVersion represents the current build version.
// This should be the only one.
var CurrentVersion = Version{
	Major:      0,
	Minor:      107,
	PatchLevel: 0,
	Suffix:     "-DEV",
}
<commit_msg>releaser: Bump versions for release of 0.107.0<commit_after>// Copyright 2018 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package hugo

// CurrentVersion represents the current build version.
// This should be the only one.
var CurrentVersion = Version{
	Major:      0,
	Minor:      107,
	PatchLevel: 0,
	Suffix:     "",
}
<|endoftext|>


<commit_before>package main

import (
	"github.com/Shopify/sarama"
	"github.com/cihub/seelog"
)

type UnboundArbiter struct {
	*StartStopControl

	// input/output
	offsets  chan int64
	messages chan *sarama.ConsumerMessage

	// parent
	manager *PartitionManager

	// config
	config        *CallbackItemConfig
	arbiterConfig ArbiterConfig
}

func NewUnboundArbiter() Arbiter {
	return &UnboundArbiter{
		StartStopControl: &StartStopControl{},
	}
}

func (ua *UnboundArbiter) OffsetChannel() chan<- int64 {
	return ua.offsets
}

func (ua *UnboundArbiter) MessageChannel() <-chan *sarama.ConsumerMessage {
	return ua.messages
}

func (ua *UnboundArbiter) Init(config *CallbackItemConfig, arbiterConfig ArbiterConfig, manager *PartitionManager) error {
	ua.manager = manager
	ua.config = config
	ua.arbiterConfig = arbiterConfig

	return nil
}

func (ua *UnboundArbiter) Run() error {
	if err := ua.ensureStart(); err != nil {
		return err
	}
	defer ua.markStop()

	consumer := ua.manager.GetConsumer()

	ua.messages = make(chan *sarama.ConsumerMessage)
	ua.offsets = make(chan int64)
	offsetBase := int64(-1)
	offsetWindow := make([]bool, 0, 10)

	ua.markReady()

arbiterLoop:
	for {
		select {
		case <-ua.WaitForCloseChannel():
			seelog.Debugf("Stop event triggered [url:%s]", ua.config.Url)
			break arbiterLoop
		case offset := <-ua.offsets:
			seelog.Debugf("Read offset from Transporter [topic:%s][partition:%d][url:%s][offset:%d]",
				ua.manager.Topic, ua.manager.Partition, ua.config.Url, offset)
			if offset >= 0 {
				if offset-offsetBase >= int64(len(offsetWindow)) {
					// extend offsetWindow
					newOffsetWindow := make([]bool, (offset-offsetBase)*2)
					copy(newOffsetWindow, offsetWindow)
					offsetWindow = newOffsetWindow
				}
				offsetWindow[offset-offsetBase] = true
				if offset == offsetBase {
					// rebase
					advanceCount := 0
					for advanceCount, _ = range offsetWindow {
						if !offsetWindow[advanceCount] {
							break
						}
					}

					// trigger offset commit
					ua.manager.GetCallbackManager().GetOffsetManager().CommitOffset(
						ua.manager.Topic, ua.manager.Partition, offsetBase+int64(advanceCount)-1)

					// rebase to idx + offsetBase
					// TODO, use ring buffer instead of array slicing
					offsetBase += int64(advanceCount)
					offsetWindow = offsetWindow[advanceCount:]
				}
			}
		case message := <-consumer.Messages():
			seelog.Debugf("Read message from PartitionConsumer [topic:%s][partition:%d][url:%s][offset:%d]",
				message.Topic, message.Partition, ua.config.Url, message.Offset)
			// feed message to transporter
			select {
			case <-ua.WaitForCloseChannel():
				seelog.Debugf("Stop event triggered [url:%s]", ua.config.Url)
				break arbiterLoop
			case ua.messages <- message:
			}

			// set base
			if -1 == offsetBase {
				offsetBase = message.Offset
			}
		}
	}

	close(ua.messages)

	return nil
}

func init() {
	RegisterArbiter("Unbound", NewUnboundArbiter)
}
<commit_msg>Use two routine to handle offset commit and message deliver in unbound_arbiter.go<commit_after>package main

import (
	"sync"

	"github.com/Shopify/sarama"
	"github.com/cihub/seelog"
)

type UnboundArbiter struct {
	*StartStopControl

	// input/output
	offsets  chan int64
	messages chan *sarama.ConsumerMessage

	// parent
	manager *PartitionManager

	// config
	config        *CallbackItemConfig
	arbiterConfig ArbiterConfig
}

func NewUnboundArbiter() Arbiter {
	return &UnboundArbiter{
		StartStopControl: &StartStopControl{},
	}
}

func (ua *UnboundArbiter) OffsetChannel() chan<- int64 {
	return ua.offsets
}

func (ua *UnboundArbiter) MessageChannel() <-chan *sarama.ConsumerMessage {
	return ua.messages
}

func (ua *UnboundArbiter) Init(config *CallbackItemConfig, arbiterConfig ArbiterConfig, manager *PartitionManager) error {
	ua.manager = manager
	ua.config = config
	ua.arbiterConfig = arbiterConfig

	return nil
}

func (ua *UnboundArbiter) Run() error {
	if err := ua.ensureStart(); err != nil {
		return err
	}
	defer ua.markStop()

	consumer := ua.manager.GetConsumer()

	ua.messages = make(chan *sarama.ConsumerMessage, 256)
	ua.offsets = make(chan int64, 256)
	offsetBase := int64(-1)
	offsetWindow := make([]bool, 0, 10)

	ua.markReady()

	wg := sync.WaitGroup{}

	wg.Add(1)
	go func() {
		defer wg.Done()
	arbiterOffsetLoop:
		for {
			select {
			case <-ua.WaitForCloseChannel():
				seelog.Debugf("Stop event triggered [url:%s]", ua.config.Url)
				break arbiterOffsetLoop
			case offset := <-ua.offsets:
				seelog.Debugf("Read offset from Transporter [topic:%s][partition:%d][url:%s][offset:%d]",
					ua.manager.Topic, ua.manager.Partition, ua.config.Url, offset)
				if offset >= 0 {
					if offset-offsetBase >= int64(len(offsetWindow)) {
						seelog.Debugf("Extend offsetWindow [offset:%d][offsetBase:%d][offsetWindowSize:%d]",
							offset, offsetBase, len(offsetWindow))
						// extend offsetWindow
						newOffsetWindow := make([]bool, (offset-offsetBase+1)*2)
						copy(newOffsetWindow, offsetWindow)
						offsetWindow = newOffsetWindow
					}
					offsetWindow[offset-offsetBase] = true
					if offset == offsetBase {
						// rebase
						advanceCount := 0
						if len(offsetWindow) > 1 {
							for advanceCount, _ = range offsetWindow {
								if !offsetWindow[advanceCount] {
									break
								}
							}
						} else {
							advanceCount = 1
						}

						// trigger offset commit
						ua.manager.GetCallbackManager().GetOffsetManager().CommitOffset(
							ua.manager.Topic, ua.manager.Partition, offsetBase+int64(advanceCount)-1)

						// rebase to idx + offsetBase
						// TODO, use ring buffer instead of array slicing
						offsetBase += int64(advanceCount)
						offsetWindow = offsetWindow[advanceCount:]
						seelog.Debugf("Fast-forward offsetBase [originOffsetBase:%d][offsetBase:%d][offsetWindowSize:%d][advance:%d]",
							offsetBase-int64(advanceCount), offsetBase, len(offsetWindow), advanceCount)
					}
				}
			}
		}
	}()

	wg.Add(1)
	go func() {
		defer wg.Done()
	arbiterMessageLoop:
		for {
			select {
			case <-ua.WaitForCloseChannel():
				seelog.Debugf("Stop event triggered [url:%s]", ua.config.Url)
				break arbiterMessageLoop
			case message := <-consumer.Messages():
				seelog.Debugf("Read message from PartitionConsumer [topic:%s][partition:%d][url:%s][offset:%d]",
					message.Topic, message.Partition, ua.config.Url, message.Offset)

				// set base
				if -1 == offsetBase {
					offsetBase = message.Offset
				}

				// feed message to transporter
				select {
				case <-ua.WaitForCloseChannel():
					seelog.Debugf("Stop event triggered [url:%s]", ua.config.Url)
					break arbiterMessageLoop
				case ua.messages <- message:
				}
			}
		}
	}()

	wg.Wait()

	close(ua.messages)

	return nil
}

func init() {
	RegisterArbiter("Unbound", NewUnboundArbiter)
}
<|endoftext|>


<commit_before>// Copyright 2019 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package updates expands an update item from IDispatch to a struct.
package updates

import (
	"time"

	"github.com/go-ole/go-ole"
)

// Identity represents the unique identifier of an update.
type Identity struct {
	RevisionNumber int
	UpdateID       string
}

// Category is information about a single Category.
type Category struct {
	Name       string
	Type       string
	CategoryID string
}

// Update contains the update interface and properties that are available to an update.
type Update struct {
	Item                     *ole.IDispatch
	Title                    string
	CanRequireSource         bool
	Categories               []Category
	Deadline                 time.Time
	Description              string
	EulaAccepted             bool
	Identity                 Identity
	IsBeta                   bool
	IsDownloaded             bool
	IsHidden                 bool
	IsInstalled              bool
	IsMandatory              bool
	IsUninstallable          bool
	LastDeploymentChangeTime time.Time
	MaxDownloadSize          int
	MinDownloadSize          int
	MsrcSeverity             string
	RecommendedCPUSpeed      int
	RecommendedHardDiskSpace int
	RecommendedMemory        int
	SecurityBulletinIDs      []string
	SupersededUpdateIDs      []string
	SupportURL               string
	Type                     string
	KBArticleIDs             []string
	RebootRequired           bool
	IsPresent                bool
	CveIDs                   []string
	BrowseOnly               bool
	PerUser                  bool
	AutoSelection            int
	AutoDownload             int
}
<commit_msg>Sort `Update` struct members.<commit_after>// Copyright 2019 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package updates expands an update item from IDispatch to a struct.
package updates

import (
	"time"

	"github.com/go-ole/go-ole"
)

// Identity represents the unique identifier of an update.
type Identity struct {
	RevisionNumber int
	UpdateID       string
}

// Category is information about a single Category.
type Category struct {
	Name       string
	Type       string
	CategoryID string
}

// Update contains the update interface and properties that are available to an update.
type Update struct {
	Item  *ole.IDispatch
	Title string

	AutoDownload             int
	AutoSelection            int
	BrowseOnly               bool
	CanRequireSource         bool
	Categories               []Category
	CveIDs                   []string
	Deadline                 time.Time
	Description              string
	EulaAccepted             bool
	Identity                 Identity
	IsBeta                   bool
	IsDownloaded             bool
	IsHidden                 bool
	IsInstalled              bool
	IsMandatory              bool
	IsPresent                bool
	IsUninstallable          bool
	KBArticleIDs             []string
	LastDeploymentChangeTime time.Time
	MaxDownloadSize          int
	MinDownloadSize          int
	MsrcSeverity             string
	PerUser                  bool
	RebootRequired           bool
	RecommendedCPUSpeed      int
	RecommendedHardDiskSpace int
	RecommendedMemory        int
	SecurityBulletinIDs      []string
	SupersededUpdateIDs      []string
	SupportURL               string
	Type                     string
}
<|endoftext|>


<commit_before>package user

import (
	"fmt"
	"strings"

	"github.com/pivotalservices/cf-mgmt/ldap"
	"github.com/pivotalservices/cf-mgmt/uaa"
	"github.com/pkg/errors"
	"github.com/xchapter7x/lo"
)

func (m *DefaultManager) SyncLdapUsers(roleUsers *RoleUsers, uaaUsers *uaa.Users, usersInput UsersInput) error {
	origin := m.LdapConfig.Origin
	if m.LdapConfig.Enabled {
		ldapUsers, err := m.GetLDAPUsers(uaaUsers, usersInput)
		if err != nil {
			return err
		}
		lo.G.Debugf("LdapUsers: %+v", ldapUsers)
		for _, inputUser := range ldapUsers {
			userToUse := m.UpdateUserInfo(inputUser)
			userID := userToUse.UserID
			userList := uaaUsers.GetByName(userID)
			if len(userList) == 0 {
				lo.G.Debug("User", userID, "doesn't exist in cloud foundry, so creating user")
				if userGUID, err := m.UAAMgr.CreateExternalUser(userID, userToUse.Email, userToUse.UserDN, m.LdapConfig.Origin); err != nil {
					lo.G.Errorf("Unable to create user %s with error %s", userID, err.Error())
					continue
				} else {
					uaaUsers.Add(uaa.User{
						Username:   userID,
						ExternalID: userToUse.UserDN,
						Origin:     m.LdapConfig.Origin,
						Email:      userToUse.Email,
						GUID:       userGUID,
					})
				}
			}
			if !roleUsers.HasUserForOrigin(userID, origin) {
				user := uaaUsers.GetByNameAndOrigin(userID, origin)
				if user == nil {
					return fmt.Errorf("Unabled to find user %s for origin %s", userID, origin)
				}
				if err := usersInput.AddUser(usersInput, user.Username, user.GUID); err != nil {
					return errors.Wrap(err, fmt.Sprintf("User %s with origin %s", user.Username, user.Origin))
				}
			} else {
				lo.G.Debugf("User[%s] found in role", userID)
				roleUsers.RemoveUserForOrigin(userID, origin)
			}
		}
	} else {
		lo.G.Debug("Skipping LDAP sync as LDAP is disabled (enable by updating config/ldap.yml)")
	}
	return nil
}

func (m *DefaultManager) GetLDAPUsers(uaaUsers *uaa.Users, usersInput UsersInput) ([]ldap.User, error) {
	var ldapUsers []ldap.User
	for _, groupName := range usersInput.UniqueLdapGroupNames() {
		userDNList, err := m.LdapMgr.GetUserDNs(groupName)
		if err != nil {
			return nil, err
		}
		for _, userDN := range userDNList {
			lo.G.Debugf("Checking for userDN %s", userDN)
			uaaUser := uaaUsers.GetByExternalID(userDN)
			if uaaUser != nil {
				lo.G.Debugf("UserDN [%s] found in UAA as [%s], skipping ldap lookup", userDN, uaaUser.Username)
				ldapUsers = append(ldapUsers, ldap.User{
					UserID: uaaUser.Username,
					UserDN: userDN,
					Email:  uaaUser.Email,
				})
			} else {
				user, err := m.LdapMgr.GetUserByDN(userDN)
				if err != nil {
					return nil, err
				}
				if user != nil {
					ldapUsers = append(ldapUsers, *user)
				}
			}
		}
	}
	for _, userID := range usersInput.LdapUsers {
		userList := uaaUsers.GetByName(userID)
		if len(userList) > 0 {
			lo.G.Debugf("UserID [%s] found in UAA, skipping ldap lookup", userID)
			for _, uaaUser := range userList {
				if strings.EqualFold(uaaUser.Origin, m.LdapConfig.Origin) {
					ldapUsers = append(ldapUsers, ldap.User{
						UserID: userID,
						UserDN: uaaUser.ExternalID,
						Email:  uaaUser.Email,
					})
				}
			}
		} else {
			user, err := m.LdapMgr.GetUserByID(userID)
			if err != nil {
				return nil, err
			}
			if user != nil {
				ldapUsers = append(ldapUsers, *user)
			}
		}
	}
	return ldapUsers, nil
}

func (m *DefaultManager) UpdateUserInfo(user ldap.User) ldap.User {
	userID := strings.ToLower(user.UserID)
	externalID := user.UserDN
	email := user.Email
	if email == "" {
		email = fmt.Sprintf("%s@user.from.ldap.cf", userID)
	}

	return ldap.User{
		UserID: userID,
		UserDN: externalID,
		Email:  email,
	}
}
<commit_msg>adding additional logging<commit_after>package user

import (
	"fmt"
	"strings"

	"github.com/pivotalservices/cf-mgmt/ldap"
	"github.com/pivotalservices/cf-mgmt/uaa"
	"github.com/pkg/errors"
	"github.com/xchapter7x/lo"
)

func (m *DefaultManager) SyncLdapUsers(roleUsers *RoleUsers, uaaUsers *uaa.Users, usersInput UsersInput) error {
	origin := m.LdapConfig.Origin
	if m.LdapConfig.Enabled {
		ldapUsers, err := m.GetLDAPUsers(uaaUsers, usersInput)
		if err != nil {
			return err
		}
		lo.G.Debugf("LdapUsers: %+v", ldapUsers)
		for _, inputUser := range ldapUsers {
			userToUse := m.UpdateUserInfo(inputUser)
			userID := userToUse.UserID
			userList := uaaUsers.GetByName(userID)
			if len(userList) == 0 {
				lo.G.Debug("User", userID, "doesn't exist in cloud foundry, so creating user")
				if userGUID, err := m.UAAMgr.CreateExternalUser(userID, userToUse.Email, userToUse.UserDN, m.LdapConfig.Origin); err != nil {
					lo.G.Errorf("Unable to create user %s with error %s", userID, err.Error())
					continue
				} else {
					uaaUsers.Add(uaa.User{
						Username:   userID,
						ExternalID: userToUse.UserDN,
						Origin:     m.LdapConfig.Origin,
						Email:      userToUse.Email,
						GUID:       userGUID,
					})
				}
			}
			if !roleUsers.HasUserForOrigin(userID, origin) {
				user := uaaUsers.GetByNameAndOrigin(userID, origin)
				if user == nil {
					return fmt.Errorf("Unabled to find user %s for origin %s", userID, origin)
				}
				if err := usersInput.AddUser(usersInput, user.Username, user.GUID); err != nil {
					return errors.Wrap(err, fmt.Sprintf("User %s with origin %s", user.Username, user.Origin))
				}
			} else {
				lo.G.Debugf("User[%s] found in role", userID)
				roleUsers.RemoveUserForOrigin(userID, origin)
			}
		}
	} else {
		lo.G.Debug("Skipping LDAP sync as LDAP is disabled (enable by updating config/ldap.yml)")
	}
	return nil
}

func (m *DefaultManager) GetLDAPUsers(uaaUsers *uaa.Users, usersInput UsersInput) ([]ldap.User, error) {
	var ldapUsers []ldap.User
	for _, groupName := range usersInput.UniqueLdapGroupNames() {
		userDNList, err := m.LdapMgr.GetUserDNs(groupName)
		if err != nil {
			return nil, err
		}
		for _, userDN := range userDNList {
			lo.G.Debugf("Checking for userDN %s", userDN)
			uaaUser := uaaUsers.GetByExternalID(userDN)
			if uaaUser != nil {
				lo.G.Debugf("UserDN [%s] found in UAA as [%s], skipping ldap lookup", userDN, uaaUser.Username)
				ldapUsers = append(ldapUsers, ldap.User{
					UserID: uaaUser.Username,
					UserDN: userDN,
					Email:  uaaUser.Email,
				})
			} else {
				lo.G.Infof("UserDN [%s] not found in UAA, executing ldap lookup", userDN)
				user, err := m.LdapMgr.GetUserByDN(userDN)
				if err != nil {
					return nil, err
				}
				if user != nil {
					ldapUsers = append(ldapUsers, *user)
				} else {
					lo.G.Infof("UserDN %s not found in ldap", userDN)
				}
			}
		}
	}
	for _, userID := range usersInput.LdapUsers {
		userList := uaaUsers.GetByName(userID)
		if len(userList) > 0 {
			lo.G.Debugf("UserID [%s] found in UAA, skipping ldap lookup", userID)
			for _, uaaUser := range userList {
				if strings.EqualFold(uaaUser.Origin, m.LdapConfig.Origin) {
					ldapUsers = append(ldapUsers, ldap.User{
						UserID: userID,
						UserDN: uaaUser.ExternalID,
						Email:  uaaUser.Email,
					})
				}
			}
		} else {
			lo.G.Infof("User [%s] not found in UAA, executing ldap lookup", userID)
			user, err := m.LdapMgr.GetUserByID(userID)
			if err != nil {
				return nil, err
			}
			if user != nil {
				ldapUsers = append(ldapUsers, *user)
			} else {
				lo.G.Infof("User %s not found in ldap", userID)
			}
		}
	}
	return ldapUsers, nil
}

func (m *DefaultManager) UpdateUserInfo(user ldap.User) ldap.User {
	userID := strings.ToLower(user.UserID)
	externalID := user.UserDN
	email := user.Email
	if email == "" {
		email = fmt.Sprintf("%s@user.from.ldap.cf", userID)
	}

	return ldap.User{
		UserID: userID,
		UserDN: externalID,
		Email:  email,
	}
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"flag"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"sort"
	"strings"
	"sync"
	"time"

	"github.com/MichaelTJones/walk"
	"github.com/briandowns/spinner"
)

var (
	mutex    = sync.Mutex{}
	spinSet  = []string{"| ", "/ ", "- ", "\\ "}
	spin     = spinner.New(spinSet, 100*time.Millisecond)
	repodirs = make(sort.StringSlice, 0)
	startdir = "."
)

// init, makes sure we have a start directory
func init() {
	flag.Parse()
	var dir = flag.Arg(0)
	if dir != "" {
		startdir = dir
	}
}

// foreachEntry is called for each entry in the given directory
func foreachEntry(entryName string, f os.FileInfo, err error) error {
	if f == nil || !f.IsDir() || f.Name() != ".git" {
		return nil
	}

	// Getting the repo dir
	var dir = strings.Replace(entryName, string(os.PathSeparator)+".git", string(os.PathSeparator), 1)

	// edge case for current directory
	if dir == ".git" {
		dir = ""
	}

	mutex.Lock()
	repodirs = append(repodirs, dir)
	mutex.Unlock()

	return nil
}

// ensureDir makes sure that the given directory is not empty
// if empty it is the current directory
func ensureDir(dir string) string {
	if dir == "" {
		return "."
	}

	return dir
}

// printStatus runs "git status -s" for the given directory
func printStatus(workdir string) {
	var gitdir = workdir + ".git"

	var cmd = exec.Command("git", "--git-dir="+gitdir, "--work-tree="+ensureDir(workdir), "status", "-s")

	var r, w, _ = os.Pipe()
	cmd.Stdout = w
	cmd.Stderr = os.Stderr

	cmd.Run()

	w.Close()
	var status, _ = ioutil.ReadAll(r)

	if len(bytes.TrimSpace(status)) > 0 {
		var workpath, err = filepath.Abs(ensureDir(workdir))
		if err != nil {
			workpath = workdir
		}

		fmt.Printf("\n--- %s ---\n", workpath)
		fmt.Printf("%v", string(status))
	}
}

// main
func main() {
	spin.Start()
	walk.Walk(startdir, foreachEntry)
	spin.Restart()
	spin.Stop()

	sort.Sort(repodirs)
	for _, wdir := range repodirs {
		printStatus(wdir)
	}
}
<commit_msg>better path handling<commit_after>package main

import (
	"bytes"
	"flag"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"sort"
	"strings"
	"sync"
	"time"

	"github.com/MichaelTJones/walk"
	"github.com/briandowns/spinner"
)

var (
	mutex    = sync.Mutex{}
	spinSet  = []string{"| ", "/ ", "- ", "\\ "}
	spin     = spinner.New(spinSet, 100*time.Millisecond)
	repodirs = make(sort.StringSlice, 0)
	startdir = "."
)

// init, makes sure we have a start directory
func init() {
	flag.Parse()
	var dir = flag.Arg(0)
	if dir != "" {
		startdir = dir
	}
}

// foreachEntry is called for each entry in the given directory
func foreachEntry(entryName string, f os.FileInfo, err error) error {
	if f == nil || !f.IsDir() || f.Name() != ".git" {
		return nil
	}

	// Getting the repo dir
	var dir = strings.Replace(entryName, string(os.PathSeparator)+".git", string(os.PathSeparator), 1)

	// edge case for current directory
	if dir == ".git" {
		dir = ""
	}

	var dirpath, patherr = filepath.Abs(ensureDir(dir))
	if patherr != nil {
		return nil
	}

	mutex.Lock()
	repodirs = append(repodirs, dirpath)
	mutex.Unlock()

	return nil
}

// ensureDir makes sure that the given directory is not empty
// if empty it is the current directory
func ensureDir(dir string) string {
	if dir == "" {
		return "."
	}

	return dir
}

// printStatus runs "git status -s" for the given directory
func printStatus(workdir string) {
	var gitdir = workdir + string(os.PathSeparator) + ".git"

	var cmd = exec.Command("git", "--git-dir="+gitdir, "--work-tree="+ensureDir(workdir), "status", "-s")

	var r, w, _ = os.Pipe()
	cmd.Stdout = w
	cmd.Stderr = os.Stderr

	cmd.Run()

	w.Close()
	var status, _ = ioutil.ReadAll(r)

	if len(bytes.TrimSpace(status)) > 0 {
		fmt.Printf("\n--- %s ---\n", workdir)
		fmt.Printf("%v", string(status))
	}
}

// main
func main() {
	spin.Start()
	walk.Walk(startdir, foreachEntry)
	spin.Restart()
	spin.Stop()

	sort.Sort(repodirs)
	for _, wdir := range repodirs {
		printStatus(wdir)
	}
}
<|endoftext|>


<commit_before>// Tideland Go Library - Generic JSON Parser
//
// Copyright (C) 2017 Frank Mueller / Tideland / Oldenburg / Germany
//
// All rights reserved. Use of this source code is governed
// by the new BSD license.

package gjp

//--------------------
// IMPORTS
//--------------------

import (
	"encoding/json"

	"github.com/tideland/golib/errors"
)

//--------------------
// DOCUMENT
//--------------------

// ValueProcessor describes a function for the processing of
// values while iterating over a document.
type ValueProcessor func(path string, value Value) error

// Document represents one JSON document.
type Document interface {
	json.Marshaler

	// Length returns the number of elements for the given path.
	Length(path string) int

	// SetValueAt sets the value at the given path.
	SetValueAt(path string, value interface{}) error

	// ValueAt returns the addressed value.
	ValueAt(path string) Value

	// Clear removes the so far build document data.
	Clear()

	// Process iterates over a document and processes its values.
	// There's no order, so nesting into an embedded document or
	// list may come earlier than higher level paths.
	Process(processor ValueProcessor) error
}

// document implements Document.
type document struct {
	root *root
}

// Parse reads a raw document and returns it as
// accessible document.
func Parse(data []byte, separator string) (Document, error) {
	var raw interface{}
	err := json.Unmarshal(data, &raw)
	if err != nil {
		return nil, errors.Annotate(err, ErrUnmarshalling, errorMessages)
	}
	return &document{
		root: newRoot(separator, rawToNoder(raw)),
	}, nil
}

// NewDocument creates a new empty document.
func NewDocument(separator string) Document {
	return &document{
		root: newRoot(separator, nil),
	}
}

// Length implements Document.
func (d *document) Length(path string) int {
	nr, ok := d.root.noderAt(path)
	if !ok {
		// Noder not found.
		return -1
	}
	// Check if node or value.
	n, ok := nr.isNode()
	if ok {
		return len(n)
	}
	return 1
}

// SetValueAt implements Document.
func (d *document) SetValueAt(path string, value interface{}) error {
	return d.root.setValueAt(path, value)
}

// ValueAt implements Document.
func (d *document) ValueAt(path string) Value {
	raw, ok := d.root.valueAt(path)
	return &value{raw, ok && raw != nil}
}

// Clear implements Document.
func (d *document) Clear() {
	d.root = newRoot(d.root.separator, nil)
}

// Process implements Document.
func (d *document) Process(processor ValueProcessor) error {
	return d.root.process([]string{}, processor)
}

// MarshalJSON implements json.Marshaler.
func (d *document) MarshalJSON() ([]byte, error) {
	raw := noderToRaw(d.root.data)
	return json.Marshal(raw)
}

// EOF
<commit_msg>Stated adding query<commit_after>// Tideland Go Library - Generic JSON Parser
//
// Copyright (C) 2017 Frank Mueller / Tideland / Oldenburg / Germany
//
// All rights reserved. Use of this source code is governed
// by the new BSD license.

package gjp

//--------------------
// IMPORTS
//--------------------

import (
	"encoding/json"

	"github.com/tideland/golib/errors"
)

//--------------------
// DOCUMENT
//--------------------

// PathValue is the combination of path and value.
type PathValue struct {
	Path  string
	Value Value
}

// PathValues contains a number of path/value combinations.
type PathValues []PathValue

// ValueProcessor describes a function for the processing of
// values while iterating over a document.
type ValueProcessor func(path string, value Value) error

// Document represents one JSON document.
type Document interface {
	json.Marshaler

	// Length returns the number of elements for the given path.
	Length(path string) int

	// SetValueAt sets the value at the given path.
	SetValueAt(path string, value interface{}) error

	// ValueAt returns the addressed value.
	ValueAt(path string) Value

	// Clear removes the so far build document data.
	Clear()

	// Query allows to find pathes matching a given pattern.
	Query(pattern string) (PathValues, error)

	// Process iterates over a document and processes its values.
	// There's no order, so nesting into an embedded document or
	// list may come earlier than higher level paths.
	Process(processor ValueProcessor) error
}

// document implements Document.
type document struct {
	root *root
}

// Parse reads a raw document and returns it as
// accessible document.
func Parse(data []byte, separator string) (Document, error) {
	var raw interface{}
	err := json.Unmarshal(data, &raw)
	if err != nil {
		return nil, errors.Annotate(err, ErrUnmarshalling, errorMessages)
	}
	return &document{
		root: newRoot(separator, rawToNoder(raw)),
	}, nil
}

// NewDocument creates a new empty document.
func NewDocument(separator string) Document {
	return &document{
		root: newRoot(separator, nil),
	}
}

// Length implements Document.
func (d *document) Length(path string) int {
	nr, ok := d.root.noderAt(path)
	if !ok {
		// Noder not found.
		return -1
	}
	// Check if node or value.
	n, ok := nr.isNode()
	if ok {
		return len(n)
	}
	return 1
}

// SetValueAt implements Document.
func (d *document) SetValueAt(path string, value interface{}) error {
	return d.root.setValueAt(path, value)
}

// ValueAt implements Document.
func (d *document) ValueAt(path string) Value {
	raw, ok := d.root.valueAt(path)
	return &value{raw, ok && raw != nil}
}

// Clear implements Document.
func (d *document) Clear() {
	d.root = newRoot(d.root.separator, nil)
}

// Query implements Document.
func (d *document) Query(pattern string) (PathValues, error) {
	return nil, nil
}

// Process implements Document.
func (d *document) Process(processor ValueProcessor) error {
	return d.root.process([]string{}, processor)
}

// MarshalJSON implements json.Marshaler.
func (d *document) MarshalJSON() ([]byte, error) {
	raw := noderToRaw(d.root.data)
	return json.Marshal(raw)
}

// EOF
<|endoftext|>


<commit_before>// Copyright 2015 go-swagger maintainers
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package validate

import (
	"log"
	"reflect"
	"regexp"

	"github.com/go-openapi/errors"
	"github.com/go-openapi/spec"
	"github.com/go-openapi/strfmt"
)

type objectValidator struct {
	Path                 string
	In                   string
	MaxProperties        *int64
	MinProperties        *int64
	Required             []string
	Properties           map[string]spec.Schema
	AdditionalProperties *spec.SchemaOrBool
	PatternProperties    map[string]spec.Schema
	Root                 interface{}
	KnownFormats         strfmt.Registry
}

func (o *objectValidator) SetPath(path string) {
	o.Path = path
}

func (o *objectValidator) Applies(source interface{}, kind reflect.Kind) bool {
	// TODO: this should also work for structs
	// there is a problem in the type validator where it will be unhappy about null values
	// so that requires more testing
	r := reflect.TypeOf(source) == specSchemaType && (kind == reflect.Map || kind == reflect.Struct)
	if Debug {
		log.Printf("object validator for %q applies %t for %T (kind: %v)\n", o.Path, r, source, kind)
	}
	return r
}

func (o *objectValidator) checkArrayMustHaveItems(res *Result, val map[string]interface{}) {
	if t, typeFound := val["type"]; typeFound {
		if tpe, ok := t.(string); ok && tpe == "array" {
			if _, itemsKeyFound := val["items"]; !itemsKeyFound {
				res.AddErrors(errors.Required("items", o.Path))
			}
		}
	}
}

func (o *objectValidator) checkItemsMustBeTypeArray(res *Result, val map[string]interface{}) {
	if _, itemsKeyFound := val["items"]; itemsKeyFound {
		t, typeFound := val["type"]
		if typeFound {
			if tpe, ok := t.(string); !ok || tpe != "array" {
				res.AddErrors(errors.InvalidType(o.Path, o.In, "array", nil))
			}
		} else {
			// there is no type
			// shouldnt validate
			res.AddErrors(errors.Required("type", o.Path))
		}
	}
}
func (o *objectValidator) precheck(res *Result, val map[string]interface{}) {
	o.checkArrayMustHaveItems(res, val)
	o.checkItemsMustBeTypeArray(res, val)
}
func (o *objectValidator) Validate(data interface{}) *Result {
	val := data.(map[string]interface{})
	numKeys := int64(len(val))

	if o.MinProperties != nil && numKeys < *o.MinProperties {
		return sErr(errors.TooFewProperties(o.Path, o.In, *o.MinProperties))
	}
	if o.MaxProperties != nil && numKeys > *o.MaxProperties {
		return sErr(errors.TooManyProperties(o.Path, o.In, *o.MaxProperties))
	}

	res := new(Result)
	o.precheck(res, val)
	if o.AdditionalProperties != nil && !o.AdditionalProperties.Allows {
		for k := range val {
			_, regularProperty := o.Properties[k]
			matched := false

			for pk := range o.PatternProperties {
				if matches, _ := regexp.MatchString(pk, k); matches {
					matched = true
					break
				}
			}
			if !regularProperty && k != "$schema" && k != "id" && !matched {
				res.AddErrors(errors.PropertyNotAllowed(o.Path, o.In, k))
			}
		}
	} else {
		for key, value := range val {
			_, regularProperty := o.Properties[key]
			matched, succeededOnce, _ := o.validatePatternProperty(key, value, res)
			if !(regularProperty || matched || succeededOnce) {
				if o.AdditionalProperties != nil && o.AdditionalProperties.Schema != nil {
					res.Merge(NewSchemaValidator(o.AdditionalProperties.Schema, o.Root, o.Path+"."+key, o.KnownFormats).Validate(value))
				} else if regularProperty && !(matched || succeededOnce) {
					res.AddErrors(errors.FailedAllPatternProperties(o.Path, o.In, key))
				}
			}
		}
	}

	createdFromDefaults := map[string]bool{}

	for pName, pSchema := range o.Properties {
		rName := pName
		if o.Path != "" {
			rName = o.Path + "." + pName
		}

		if v, ok := val[pName]; ok {
			r := NewSchemaValidator(&pSchema, o.Root, rName, o.KnownFormats).Validate(v)
			res.Merge(r)
		} else if pSchema.Default != nil {
			createdFromDefaults[pName] = true
			pName := pName // shaddow
			def := pSchema.Default
			res.Defaulters = append(res.Defaulters, DefaulterFunc(func() {
				val[pName] = def
			}))
		}
	}

	if len(o.Required) > 0 {
		for _, k := range o.Required {
			if _, ok := val[k]; !ok && !createdFromDefaults[k] {
				res.AddErrors(errors.Required(o.Path+"."+k, o.In))
				continue
			}
		}
	}

	for key, value := range val {
		_, regularProperty := o.Properties[key]
		matched, succeededOnce, patterns := o.validatePatternProperty(key, value, res)
		if !regularProperty && (matched || succeededOnce) {
			for _, pName := range patterns {
				if v, ok := o.PatternProperties[pName]; ok {
					res.Merge(NewSchemaValidator(&v, o.Root, o.Path+"."+key, o.KnownFormats).Validate(value))
				}
			}
		}
	}
	return res
}

func (o *objectValidator) validatePatternProperty(key string, value interface{}, result *Result) (bool, bool, []string) {
	matched := false
	succeededOnce := false
	var patterns []string

	for k, schema := range o.PatternProperties {
		if match, _ := regexp.MatchString(k, key); match {
			patterns = append(patterns, k)
			matched = true
			validator := NewSchemaValidator(&schema, o.Root, o.Path+"."+key, o.KnownFormats)

			res := validator.Validate(value)
			result.Merge(res)
		}
	}

	if succeededOnce {
		result.Inc()
	}

	return matched, succeededOnce, patterns
}
<commit_msg>allow a property name to be items<commit_after>// Copyright 2015 go-swagger maintainers
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package validate

import (
	"log"
	"reflect"
	"regexp"
	"strings"

	"github.com/go-openapi/errors"
	"github.com/go-openapi/spec"
	"github.com/go-openapi/strfmt"
)

type objectValidator struct {
	Path                 string
	In                   string
	MaxProperties        *int64
	MinProperties        *int64
	Required             []string
	Properties           map[string]spec.Schema
	AdditionalProperties *spec.SchemaOrBool
	PatternProperties    map[string]spec.Schema
	Root                 interface{}
	KnownFormats         strfmt.Registry
}

func (o *objectValidator) SetPath(path string) {
	o.Path = path
}

func (o *objectValidator) Applies(source interface{}, kind reflect.Kind) bool {
	// TODO: this should also work for structs
	// there is a problem in the type validator where it will be unhappy about null values
	// so that requires more testing
	r := reflect.TypeOf(source) == specSchemaType && (kind == reflect.Map || kind == reflect.Struct)
	if Debug {
		log.Printf("object validator for %q applies %t for %T (kind: %v)\n", o.Path, r, source, kind)
	}
	return r
}

func (o *objectValidator) isPropertyName() bool {
	log.Printf(o.Path)
	p := strings.Split(o.Path, ".")
	return p[len(p)-1] == "properties"
}
func (o *objectValidator) checkArrayMustHaveItems(res *Result, val map[string]interface{}) {
	if t, typeFound := val["type"]; typeFound {
		if tpe, ok := t.(string); ok && tpe == "array" {
			if _, itemsKeyFound := val["items"]; !itemsKeyFound {
				res.AddErrors(errors.Required("items", o.Path))
			}
		}
	}
}

func (o *objectValidator) checkItemsMustBeTypeArray(res *Result, val map[string]interface{}) {
	if !o.isPropertyName() {
		if _, itemsKeyFound := val["items"]; itemsKeyFound {
			t, typeFound := val["type"]
			if typeFound {
				if tpe, ok := t.(string); !ok || tpe != "array" {
					res.AddErrors(errors.InvalidType(o.Path, o.In, "array", nil))
				}
			} else {
				// there is no type
				res.AddErrors(errors.Required("type", o.Path))
			}
		}
	}
}
func (o *objectValidator) precheck(res *Result, val map[string]interface{}) {
	o.checkArrayMustHaveItems(res, val)
	o.checkItemsMustBeTypeArray(res, val)
}
func (o *objectValidator) Validate(data interface{}) *Result {
	val := data.(map[string]interface{})
	numKeys := int64(len(val))

	if o.MinProperties != nil && numKeys < *o.MinProperties {
		return sErr(errors.TooFewProperties(o.Path, o.In, *o.MinProperties))
	}
	if o.MaxProperties != nil && numKeys > *o.MaxProperties {
		return sErr(errors.TooManyProperties(o.Path, o.In, *o.MaxProperties))
	}

	res := new(Result)

	o.precheck(res, val)

	if o.AdditionalProperties != nil && !o.AdditionalProperties.Allows {
		for k := range val {
			_, regularProperty := o.Properties[k]
			matched := false

			for pk := range o.PatternProperties {
				if matches, _ := regexp.MatchString(pk, k); matches {
					matched = true
					break
				}
			}
			if !regularProperty && k != "$schema" && k != "id" && !matched {
				res.AddErrors(errors.PropertyNotAllowed(o.Path, o.In, k))
			}
		}
	} else {
		for key, value := range val {
			_, regularProperty := o.Properties[key]
			matched, succeededOnce, _ := o.validatePatternProperty(key, value, res)
			if !(regularProperty || matched || succeededOnce) {
				if o.AdditionalProperties != nil && o.AdditionalProperties.Schema != nil {
					res.Merge(NewSchemaValidator(o.AdditionalProperties.Schema, o.Root, o.Path+"."+key, o.KnownFormats).Validate(value))
				} else if regularProperty && !(matched || succeededOnce) {
					res.AddErrors(errors.FailedAllPatternProperties(o.Path, o.In, key))
				}
			}
		}
	}

	createdFromDefaults := map[string]bool{}

	for pName, pSchema := range o.Properties {
		rName := pName
		if o.Path != "" {
			rName = o.Path + "." + pName
		}

		if v, ok := val[pName]; ok {
			r := NewSchemaValidator(&pSchema, o.Root, rName, o.KnownFormats).Validate(v)
			res.Merge(r)
		} else if pSchema.Default != nil {
			createdFromDefaults[pName] = true
			pName := pName // shaddow
			def := pSchema.Default
			res.Defaulters = append(res.Defaulters, DefaulterFunc(func() {
				val[pName] = def
			}))
		}
	}

	if len(o.Required) > 0 {
		for _, k := range o.Required {
			if _, ok := val[k]; !ok && !createdFromDefaults[k] {
				res.AddErrors(errors.Required(o.Path+"."+k, o.In))
				continue
			}
		}
	}

	for key, value := range val {
		_, regularProperty := o.Properties[key]
		matched, succeededOnce, patterns := o.validatePatternProperty(key, value, res)
		if !regularProperty && (matched || succeededOnce) {
			for _, pName := range patterns {
				if v, ok := o.PatternProperties[pName]; ok {
					res.Merge(NewSchemaValidator(&v, o.Root, o.Path+"."+key, o.KnownFormats).Validate(value))
				}
			}
		}
	}
	return res
}

func (o *objectValidator) validatePatternProperty(key string, value interface{}, result *Result) (bool, bool, []string) {
	matched := false
	succeededOnce := false
	var patterns []string

	for k, schema := range o.PatternProperties {
		if match, _ := regexp.MatchString(k, key); match {
			patterns = append(patterns, k)
			matched = true
			validator := NewSchemaValidator(&schema, o.Root, o.Path+"."+key, o.KnownFormats)

			res := validator.Validate(value)
			result.Merge(res)
		}
	}

	if succeededOnce {
		result.Inc()
	}

	return matched, succeededOnce, patterns
}
<|endoftext|>


<commit_before><commit_msg>Add timestamp field to datastore entries<commit_after><|endoftext|>


<commit_before>// Copyright 2013 tsuru authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package db encapsulates tsuru connection with MongoDB.
//
// The function Open dials to MongoDB and returns a connection (represented by
// the Storage type). It manages an internal pool of connections, and
// reconnects in case of failures. That means that you should not store
// references to the connection, but always call Open.
package db

import (
	"fmt"
	"github.com/globocom/config"
	"labix.org/v2/mgo"
	"sync"
	"time"
)

var (
	conn   = make(map[string]*session) // pool of connections
	mut    sync.RWMutex                // for pool thread safety
	ticker *time.Ticker                // for garbage collection
)

const period time.Duration = 7 * 24 * time.Hour

type session struct {
	s    *mgo.Session
	used time.Time
}

// Storage holds the connection with the database.
type Storage struct {
	session *mgo.Session
	dbname  string
}

func open(addr, dbname string) (*Storage, error) {
	sess, err := mgo.DialWithTimeout(addr, 1e9)
	if err != nil {
		return nil, err
	}
	storage := &Storage{session: sess, dbname: dbname}
	mut.Lock()
	conn[addr] = &session{s: sess, used: time.Now()}
	mut.Unlock()
	return storage, nil
}

// Open dials to the MongoDB database, and return the connection (represented
// by the type Storage).
//
// addr is a MongoDB connection URI, and dbname is the name of the database.
//
// This function returns a pointer to a Storage, or a non-nil error in case of
// any failure.
func Open(addr, dbname string) (storage *Storage, err error) {
	defer func() {
		if r := recover(); r != nil {
			storage, err = open(addr, dbname)
		}
	}()
	mut.RLock()
	if session, ok := conn[addr]; ok {
		mut.RUnlock()
		if err = session.s.Ping(); err == nil {
			mut.Lock()
			session.used = time.Now()
			conn[addr] = session
			mut.Unlock()
			return &Storage{session.s, dbname}, nil
		}
		return open(addr, dbname)
	}
	mut.RUnlock()
	return open(addr, dbname)
}

// Conn reads the tsuru config and calls Open to get a database connection.
//
// Most tsuru packages should probably use this function. Open is intended for
// use when supporting more than one database.
func Conn() (storage *Storage, err error) {
	url, err := config.GetString("database:url")
	if err != nil {
		return nil, fmt.Errorf("configuration error: %s", err)
	}
	dbname, err := config.GetString("database:name")
	if err != nil {
		return nil, fmt.Errorf("configuration error: %s", err)
	}
	return Open(url, dbname)
}

// Collection returns a collection by its name.
//
// If the collection does not exist, MongoDB will create it.
func (s *Storage) Collection(name string) *mgo.Collection {
	return s.session.DB(s.dbname).C(name)
}

// Apps returns the apps collection from MongoDB.
func (s *Storage) Apps() *mgo.Collection {
	nameIndex := mgo.Index{Key: []string{"name"}, Unique: true}
	c := s.Collection("apps")
	c.EnsureIndex(nameIndex)
	return c
}

// Services returns the services collection from MongoDB.
func (s *Storage) Services() *mgo.Collection {
	c := s.Collection("services")
	return c
}

// ServiceInstances returns the services_instances collection from MongoDB.
func (s *Storage) ServiceInstances() *mgo.Collection {
	return s.Collection("service_instances")
}

// Users returns the users collection from MongoDB.
func (s *Storage) Users() *mgo.Collection {
	emailIndex := mgo.Index{Key: []string{"email"}, Unique: true}
	c := s.Collection("users")
	c.EnsureIndex(emailIndex)
	return c
}

// Teams returns the teams collection from MongoDB.
func (s *Storage) Teams() *mgo.Collection {
	return s.Collection("teams")
}

func init() {
	ticker = time.NewTicker(time.Hour)
	go retire(ticker)
}

// retire retires old connections :-)
func retire(t *time.Ticker) {
	for _ = range t.C {
		now := time.Now()
		var old []string
		mut.RLock()
		for k, v := range conn {
			if now.Sub(v.used) >= period {
				old = append(old, k)
			}
		}
		mut.RUnlock()
		mut.Lock()
		for _, c := range old {
			conn[c].s.Close()
			delete(conn, c)
		}
		mut.Unlock()
	}
}
<commit_msg>db: remove unnecessary named result<commit_after>// Copyright 2013 tsuru authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package db encapsulates tsuru connection with MongoDB.
//
// The function Open dials to MongoDB and returns a connection (represented by
// the Storage type). It manages an internal pool of connections, and
// reconnects in case of failures. That means that you should not store
// references to the connection, but always call Open.
package db

import (
	"fmt"
	"github.com/globocom/config"
	"labix.org/v2/mgo"
	"sync"
	"time"
)

var (
	conn   = make(map[string]*session) // pool of connections
	mut    sync.RWMutex                // for pool thread safety
	ticker *time.Ticker                // for garbage collection
)

const period time.Duration = 7 * 24 * time.Hour

type session struct {
	s    *mgo.Session
	used time.Time
}

// Storage holds the connection with the database.
type Storage struct {
	session *mgo.Session
	dbname  string
}

func open(addr, dbname string) (*Storage, error) {
	sess, err := mgo.DialWithTimeout(addr, 1e9)
	if err != nil {
		return nil, err
	}
	storage := &Storage{session: sess, dbname: dbname}
	mut.Lock()
	conn[addr] = &session{s: sess, used: time.Now()}
	mut.Unlock()
	return storage, nil
}

// Open dials to the MongoDB database, and return the connection (represented
// by the type Storage).
//
// addr is a MongoDB connection URI, and dbname is the name of the database.
//
// This function returns a pointer to a Storage, or a non-nil error in case of
// any failure.
func Open(addr, dbname string) (storage *Storage, err error) {
	defer func() {
		if r := recover(); r != nil {
			storage, err = open(addr, dbname)
		}
	}()
	mut.RLock()
	if session, ok := conn[addr]; ok {
		mut.RUnlock()
		if err = session.s.Ping(); err == nil {
			mut.Lock()
			session.used = time.Now()
			conn[addr] = session
			mut.Unlock()
			return &Storage{session.s, dbname}, nil
		}
		return open(addr, dbname)
	}
	mut.RUnlock()
	return open(addr, dbname)
}

// Conn reads the tsuru config and calls Open to get a database connection.
//
// Most tsuru packages should probably use this function. Open is intended for
// use when supporting more than one database.
func Conn() (*Storage, error) {
	url, err := config.GetString("database:url")
	if err != nil {
		return nil, fmt.Errorf("configuration error: %s", err)
	}
	dbname, err := config.GetString("database:name")
	if err != nil {
		return nil, fmt.Errorf("configuration error: %s", err)
	}
	return Open(url, dbname)
}

// Collection returns a collection by its name.
//
// If the collection does not exist, MongoDB will create it.
func (s *Storage) Collection(name string) *mgo.Collection {
	return s.session.DB(s.dbname).C(name)
}

// Apps returns the apps collection from MongoDB.
func (s *Storage) Apps() *mgo.Collection {
	nameIndex := mgo.Index{Key: []string{"name"}, Unique: true}
	c := s.Collection("apps")
	c.EnsureIndex(nameIndex)
	return c
}

// Services returns the services collection from MongoDB.
func (s *Storage) Services() *mgo.Collection {
	c := s.Collection("services")
	return c
}

// ServiceInstances returns the services_instances collection from MongoDB.
func (s *Storage) ServiceInstances() *mgo.Collection {
	return s.Collection("service_instances")
}

// Users returns the users collection from MongoDB.
func (s *Storage) Users() *mgo.Collection {
	emailIndex := mgo.Index{Key: []string{"email"}, Unique: true}
	c := s.Collection("users")
	c.EnsureIndex(emailIndex)
	return c
}

// Teams returns the teams collection from MongoDB.
func (s *Storage) Teams() *mgo.Collection {
	return s.Collection("teams")
}

func init() {
	ticker = time.NewTicker(time.Hour)
	go retire(ticker)
}

// retire retires old connections :-)
func retire(t *time.Ticker) {
	for _ = range t.C {
		now := time.Now()
		var old []string
		mut.RLock()
		for k, v := range conn {
			if now.Sub(v.used) >= period {
				old = append(old, k)
			}
		}
		mut.RUnlock()
		mut.Lock()
		for _, c := range old {
			conn[c].s.Close()
			delete(conn, c)
		}
		mut.Unlock()
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"os"
	"io/ioutil"
	"flag"
	"log"
)

var (
	elim = flag.String("e", "", "Eliminate this (one-character) function.")
)

func main() {
	flag.Parse()
	var input []byte
	if flag.NArg() == 0 {
		var err os.Error
		input, err = ioutil.ReadAll(os.Stdin)
		if err != nil {
			log.Fatalln(err)
		}
	} else {
		file, err := os.Open(flag.Arg(0), os.O_RDONLY, 0)
		if err != nil {
			log.Fatalln(err)
		}
		defer file.Close()
		input, err = ioutil.ReadAll(file)
		if err != nil {
			log.Fatalln(err)
		}
	}
	input = strip(input)
	if !valid(input) {
		log.Fatalln("Syntax error.")
	}
	if len(*elim) == 0 {
		fmt.Println(parse(input))
	} else {
		input = []byte(parse(input).String())  //First we simplify.
		for i := len(*elim) - 1; i >= 0; i-- { //Iterate backwards.
			input = []byte(eliminate(dumbParse(input), char((*elim)[i])).String())
			input = []byte(parse(input).String()) //Simplify after every step.
		}
		fmt.Println(string(input))
	}
}
<commit_msg>Modified for weekly.2011-04-03<commit_after>package main

import (
	"fmt"
	"os"
	"io/ioutil"
	"flag"
	"log"
)

var (
	elim = flag.String("e", "", "Eliminate this (one-character) function.")
)

func main() {
	flag.Parse()
	var input []byte
	if flag.NArg() == 0 {
		var err os.Error
		input, err = ioutil.ReadAll(os.Stdin)
		if err != nil {
			log.Fatalln(err)
		}
	} else {
		file, err := os.Open(flag.Arg(0))
		if err != nil {
			log.Fatalln(err)
		}
		defer file.Close()
		input, err = ioutil.ReadAll(file)
		if err != nil {
			log.Fatalln(err)
		}
	}
	input = strip(input)
	if !valid(input) {
		log.Fatalln("Syntax error.")
	}
	if len(*elim) == 0 {
		fmt.Println(parse(input))
	} else {
		input = []byte(parse(input).String())  //First we simplify.
		for i := len(*elim) - 1; i >= 0; i-- { //Iterate backwards.
			input = []byte(eliminate(dumbParse(input), char((*elim)[i])).String())
			input = []byte(parse(input).String()) //Simplify after every step.
		}
		fmt.Println(string(input))
	}
}
<|endoftext|>


<commit_before><commit_msg>fixed syntax<commit_after><|endoftext|>


<commit_before>package gocache

import(
    "fmt"
    "net/http"
    "errors"
    "os"
    "strings"
    "log"
    "encoding/json"
    "bytes"
    "io/ioutil"
    "path"
    "io"
)

type Entry struct{
    Response *http.Response
    Content string
}

func main(){
    fmt.Println("Testing web cache");
    url := "http://blog.golang.org/error-handling-and-go"
    Get(url)
}

func Get(url string) (resp *http.Response, err error) {

    resp, err = getFromCache(url)

    if err != nil{
        log.Println(url,"could not be served out of cache.", err)
        return getFromWeb(url)
    }

    log.Print("Request to ", url, " served out of cache")
    return
}

// Fetch contents of url from web and write to cache
func getFromWeb(url string) (resp *http.Response, err error){
    resp, err = http.Get(url)

    //log.Println("From web got response:" , resp)

    if err != nil{
        return
    }

    writeToCache(url, resp)

    return
}

func writeToCache(url string, resp *http.Response){

    cacheEntry := generateCacheEntry(resp)

    b, err := json.Marshal(cacheEntry)

    if err != nil {
        log.Println("Could not marshal response ", err)
        return
    }

    filename := getFilePath(url)

    // Set up any directory needed to write the file.
    // e.g. vg.no/first/second/file will be stored as
    // cache/first/second/file

    err = createDirectories(filename)
    if err != nil{
        pe, _ := err.(*os.PathError)

        if ! strings.Contains(pe.Error(),"file exists") {
            log.Println("Could not create directories ", err)
            return
        }
    }

    // Create file
    file, err := os.Create(filename)
    if err != nil {
        log.Println("could not create cache file ", err)
        return
    }

    // Close and check for error on exit
    defer func() {
        if err := file.Close(); err != nil {

            log.Println("Could not close file ", err)
        }
    }()

    // Write file
    _, err = file.Write(b)
    if err != nil{
        log.Println("Could not write to cache file ", err)
    }

    log.Println(url, "successfully written to cache");
}

func generateCacheEntry(resp *http.Response) Entry {

    body, err := ioutil.ReadAll(resp.Body)

    if err != nil {
        log.Print("Reading response body went bad. ", err);

    }

    // Note the assignment. Since it's a pointer we need a new copy
    // This stuff is really ugly and should be changed later.
    var resp2 http.Response = http.Response{}
    resp2 = *resp

    Response := &resp2
    Content := string(body)
    entry := Entry{Response, Content}

    // Cannot marshal the body from get go
    entry.Response.Body = nil

    return entry

}

// Try to fetch contents of url from cache
func getFromCache(url string) (resp *http.Response, err error) {

    filename := getFilePath(url)

    file, err := os.Open(filename)
    if err != nil{
        err = errors.New("File '"+filename+"' not Found")
        return
    }

    entry, err := readFromFile(file)

    if err != nil{
        return
    }

    resp = entry.GenerateHttpResponse()

    return

}

// Read cache entry from file
func readFromFile(file *os.File) (entry *Entry, err error) {
    fileinfo, err := file.Stat()
    var size int
    size = int(fileinfo.Size())

    buf := make([]byte, size)

    n, err := file.Read(buf)

    if err != nil{
        log.Println("Reading file got:", err, "after",n,"bytes");
        return entry, errors.New("Reading file went wrong")
    }

    // Must trim buffer before unmarshaling it. This is because of
    // the unmarshaling failing if entire buffer is returned
    buf = bytes.Trim(buf[0:], string(0))

    entry = new(Entry)
    err = json.Unmarshal(buf, entry)

    if err != nil {
        log.Print("Unmarshaling gone wrong: ", err)
        return entry, errors.New("Unmarshaling gone wrong")
    }

    return
}

func (entry *Entry) Print () {

    //n := bytes.Index(entry.Content, []byte{0})
    //content := string(entry.Content[:n])
    log.Print("Content: ", entry.Content)

}

func (entry *Entry) GenerateHttpResponse() (resp *http.Response){

    resp = entry.Response
    resp.Body = nopCloser{bytes.NewBufferString(entry.Content)}

    return resp

}

func getFilePath(url string) (dir string) {
	urlTokens := strings.Split(url, "/")
    // 3 because we need to strip away 'http:', ' ', and hostname
    strippedUrl := urlTokens[3:]
	dir = "cache/"+strings.Join(strippedUrl,"/")
	return
}

func createDirectories(filename string) error {

    filepath := path.Dir(filename)
    directories := strings.Split(filepath, "/")

    p := ""
    for i := range directories {
        p += directories[i] + "/"
        err := os.Mkdir(p, 0755)

        if err != nil {
            pe, _ := err.(*os.PathError)

            if ! strings.Contains(pe.Error(),"file exists") {
                log.Println("Mkdir failed miserably: ", err)
                return err
            }

        }
    }

    return nil
}

/* Below are from:
    https://groups.google.com/forum/#!topic/golang-nuts/J-Y4LtdGNSw
*/

type nopCloser struct {
    io.Reader
}

func (nopCloser) Close() error {
    return nil
}
<commit_msg>it fucking works<commit_after>package gocache

import(
    "fmt"
    "net/http"
    "errors"
    "os"
    "strings"
    "log"
    "encoding/json"
    "bytes"
    "io/ioutil"
    "path"
    "io"
)

type Entry struct{
    Response *http.Response
    Content string
}

func main(){
    fmt.Println("Testing web cache");
    url := "http://blog.golang.org/error-handling-and-go"
    Get(url)
}

func Get(url string) (resp *http.Response, err error) {

    resp, err = getFromCache(url)

    if err != nil{
        log.Println(url,"could not be served out of cache.", err)
        return getFromWeb(url)
    }

    log.Print("Request to ", url, " served out of cache")
    return
}

// Fetch contents of url from web and write to cache
func getFromWeb(url string) (resp *http.Response, err error){
    resp, err = http.Get(url)

    if err != nil{
        return
    }
    var resp2 http.Response
    resp2 = *resp

    // Draining the body

    body, err := ioutil.ReadAll(resp.Body)

    if err != nil {
        log.Print("Reading response body went bad. ", err);

    }

    writeToCache(url, body, &resp2)

    resp.Body = nopCloser{bytes.NewBufferString(string(body))}

    return
}

func writeToCache(url string, body []byte, resp *http.Response){

    cacheEntry := generateCacheEntry(resp, body)

    b, err := json.Marshal(cacheEntry)

    if err != nil {
        log.Println("Could not marshal response ", err)
        return
    }

    filename := getFilePath(url)

    // Set up any directory needed to write the file.
    // e.g. vg.no/first/second/file will be stored as
    // cache/first/second/file

    err = createDirectories(filename)
    if err != nil{
        pe, _ := err.(*os.PathError)

        if ! strings.Contains(pe.Error(),"file exists") {
            log.Println("Could not create directories ", err)
            return
        }
    }

    // Create file
    file, err := os.Create(filename)
    if err != nil {
        log.Println("could not create cache file ", err)
        return
    }

    // Close and check for error on exit
    defer func() {
        if err := file.Close(); err != nil {

            log.Println("Could not close file ", err)
        }
    }()

    // Write file
    _, err = file.Write(b)
    if err != nil{
        log.Println("Could not write to cache file ", err)
    }

    log.Println(url, "successfully written to cache");
}

func generateCacheEntry(resp *http.Response, body []byte ) Entry {

    /*
    body, err := ioutil.ReadAll(resp.Body)

    if err != nil {
        log.Print("Reading response body went bad. ", err);

    }
    */

    Response := resp
    Content := string(body)
    entry := Entry{Response, Content}

    // Cannot marshal the body from get go
    entry.Response.Body = nil

    return entry

}

// Try to fetch contents of url from cache
func getFromCache(url string) (resp *http.Response, err error) {

    filename := getFilePath(url)

    file, err := os.Open(filename)
    if err != nil{
        err = errors.New("File '"+filename+"' not Found")
        return
    }

    entry, err := readFromFile(file)

    if err != nil{
        return
    }

    resp = entry.GenerateHttpResponse()

    return

}

// Read cache entry from file
func readFromFile(file *os.File) (entry *Entry, err error) {
    fileinfo, err := file.Stat()
    var size int
    size = int(fileinfo.Size())

    buf := make([]byte, size)

    n, err := file.Read(buf)

    if err != nil{
        log.Println("Reading file got:", err, "after",n,"bytes");
        return entry, errors.New("Reading file went wrong")
    }

    // Must trim buffer before unmarshaling it. This is because of
    // the unmarshaling failing if entire buffer is returned
    buf = bytes.Trim(buf[0:], string(0))

    entry = new(Entry)
    err = json.Unmarshal(buf, entry)

    if err != nil {
        log.Print("Unmarshaling gone wrong: ", err)
        return entry, errors.New("Unmarshaling gone wrong")
    }

    return
}

func (entry *Entry) Print () {

    //n := bytes.Index(entry.Content, []byte{0})
    //content := string(entry.Content[:n])
    log.Print("Content: ", entry.Content)

}

func (entry *Entry) GenerateHttpResponse() (resp *http.Response){

    resp = entry.Response
    resp.Body = nopCloser{bytes.NewBufferString(entry.Content)}

    return resp

}

func getFilePath(url string) (dir string) {
	urlTokens := strings.Split(url, "/")
    // 3 because we need to strip away 'http:', ' ', and hostname
    strippedUrl := urlTokens[3:]
	dir = "cache/"+strings.Join(strippedUrl,"/")
	return
}

func createDirectories(filename string) error {

    filepath := path.Dir(filename)
    directories := strings.Split(filepath, "/")

    p := ""
    for i := range directories {
        p += directories[i] + "/"
        err := os.Mkdir(p, 0755)

        if err != nil {
            pe, _ := err.(*os.PathError)

            // if folder exists, continue to the next one
            if ! strings.Contains(pe.Error(),"file exists") {
                log.Println("Mkdir failed miserably: ", err)
                return err
            }
        }
    }

    return nil
}

/* Below are from:
    https://groups.google.com/forum/#!topic/golang-nuts/J-Y4LtdGNSw
*/

type nopCloser struct {
    io.Reader
}

func (nopCloser) Close() error {
    return nil
}
<|endoftext|>


<commit_before><commit_msg>Add first test file<commit_after><|endoftext|>


<commit_before>package ofsserver

import (
	"errors"
	"fmt"
)

var (
	// ErrAuthRequired occurs after a client connection if the first 4 bytes received is not 0xFFFFFFnn
	// and the server requires authorization
	ErrAuthRequired = errors.New("Client authentication is required")
	// ErrInvalidAuth occurs when the client authentication is incorrect
	ErrInvalidAuth = errors.New("Invalid Credentials")
	// ErrShortWrite occurs when the server writes less bytes than it expected to write and it is
	// considered an error
	ErrShortWrite = errors.New("Expecting to write more bytes than were actually written")
)

// CreateArgumentNilError creates a custom form ErrArgumentNil with the argument
// name included in the error message
func CreateArgumentNilError(paramName string) error {
	return fmt.Errorf("The argument '%s' cannot be nil", paramName)
}

// CreateBadArgumentError creates a custom form ErrArgumentNil with the argument
// name included in the error message
func CreateBadArgumentError(paramName, message string, args ...interface{}) error {
	return fmt.Errorf("The value of the argument '%s' is invalid: %s", paramName, fmt.Sprintf(message, args))
}
<commit_msg>added ErrInvalidReauthentication<commit_after>package ofsserver

import (
	"errors"
	"fmt"
)

var (
	// ErrAuthRequired occurs after a client connection if the first 4 bytes received is not 0xFFFFFFnn
	// and the server requires authorization
	ErrAuthRequired = errors.New("Client authentication is required")
	// ErrInvalidAuth occurs when the client authentication is incorrect
	ErrInvalidAuth = errors.New("Invalid Credentials")
	// ErrInvalidReauthentication occurs when the client sends a authentication header after
	// the client has already authenticated
	ErrInvalidReauthentication = errors.New("Client is attempting unexpected re-authentication")
	// ErrShortWrite occurs when the server writes less bytes than it expected to write and it is
	// considered an error
	ErrShortWrite = errors.New("Expecting to write more bytes than were actually written")
)

// CreateArgumentNilError creates a custom form ErrArgumentNil with the argument
// name included in the error message
func CreateArgumentNilError(paramName string) error {
	return fmt.Errorf("The argument '%s' cannot be nil", paramName)
}

// CreateBadArgumentError creates a custom form ErrArgumentNil with the argument
// name included in the error message
func CreateBadArgumentError(paramName, message string, args ...interface{}) error {
	return fmt.Errorf("The value of the argument '%s' is invalid: %s", paramName, fmt.Sprintf(message, args))
}
<|endoftext|>


<commit_before>// Godless is a peer-to-peer database running over IPFS.
//
// Godless uses a Consistent Replicated Data Type called a Namespace to share schemaless data with peers.
//
// This package is a facade to Godless internals.
//
// Godless is in alpha, and should be considered experimental software.
package godless

import (
	"fmt"
	"sync"
	"time"

	gohttp "net/http"

	"github.com/johnny-morrice/godless/api"
	"github.com/johnny-morrice/godless/cache"
	"github.com/johnny-morrice/godless/crdt"
	"github.com/johnny-morrice/godless/internal/http"
	"github.com/johnny-morrice/godless/internal/ipfs"
	"github.com/johnny-morrice/godless/internal/service"
	"github.com/johnny-morrice/godless/log"
	"github.com/johnny-morrice/godless/query"
	"github.com/pkg/errors"
)

// Godless options.
type Options struct {
	// IpfsServiceUrl is required.
	IpfsServiceUrl string
	// KeyStore is required. A private Key store.
	KeyStore api.KeyStore
	// WebServiceAddr is optional.  If not set, the webservice will be disabled.
	WebServiceAddr string
	// IndexHash is optional.  Set to load an existing index from IPFS.
	IndexHash string
	// FailEarly will cause the godless process to crash if it cannot contact IPFS on startup.
	FailEarly bool
	// ReplicateInterval is optional.  The duration between peer-to-peer replications.
	ReplicateInterval time.Duration
	// Topics is optional.  Two godless servers which share a topic will replicate indices. An empty topics slice will disable replication.
	Topics []string
	// IpfsClient is optional.  Specify a HTTP client for IPFS.
	IpfsClient *gohttp.Client
	// IpfsPingTimeout is optional.  Specify a lower timeout for "Am I Connected?" checks.
	IpfsPingTimeout time.Duration
	// HeadCache is optional.  Build a 12-factor app by supplying your own remote cache.
	HeadCache api.HeadCache
	// IndexCache is optional.  Build a 12-factor app by supplying your own remote cache.
	IndexCache api.IndexCache
	// PriorityQueue is optional. Build a 12-factor app by supplying your own remote cache.
	PriorityQueue api.RequestPriorityQueue
	// APIQueryLimit is optional.  Tune performance by setting the number of simultaneous queries.
	APIQueryLimit int
}

// Godless is a peer-to-peer database.  It shares structured data between peers, using IPFS as a backing store.
// The core datastructure is a CRDT namespace which resembles a relational scheme in that it has tables, rows, and entries.
type Godless struct {
	Options
	errch    chan error
	errwg    sync.WaitGroup
	stopch   chan struct{}
	stoppers []chan<- struct{}
	store    api.RemoteStore
	remote   api.RemoteNamespace
	api      api.APIService
}

// New creates a godless instance, connecting to any services, and providing any services, specified in the options.
func New(options Options) (*Godless, error) {
	godless := &Godless{Options: options}

	missing := godless.findMissingParameters()

	if missing != nil {
		return nil, missing
	}

	setupFuncs := []func() error{
		godless.connectIpfs,
		godless.setupNamespace,
		godless.launchAPI,
		godless.serveWeb,
		godless.replicate,
	}

	err := breakOnError(setupFuncs)

	if err != nil {
		return nil, err
	}

	return godless, nil
}

func (godless *Godless) findMissingParameters() error {
	var missing error
	if godless.IpfsServiceUrl == "" {
		msg := godless.missingParameterText("IpfsServiceUrl")
		missing = errors.New(msg)
	}

	if godless.KeyStore == nil {
		msg := godless.missingParameterText("KeyStore")
		if missing == nil {
			missing = errors.New(msg)
		} else {
			missing = errors.Wrap(missing, msg)
		}
	}

	return missing
}

func (godless *Godless) missingParameterText(param string) string {
	return fmt.Sprintf("Missing required parameter '%v'", param)
}

// Errors provides a stream of errors from godless.  Godless will attempt to handle any errors it can.  Any errors received here indicate that bad things have happened.
func (godless *Godless) Errors() <-chan error {
	return godless.errch
}

// Shutdown stops all godless processes.  It does not wait for those goroutines to stop.
func (godless *Godless) Shutdown() {
	godless.stopch <- struct{}{}
}

func (godless *Godless) connectIpfs() error {
	client := godless.IpfsClient
	pingTimeout := godless.IpfsPingTimeout

	peer := &ipfs.IPFSPeer{
		Url:         godless.IpfsServiceUrl,
		Client:      client,
		PingTimeout: pingTimeout,
	}

	if godless.FailEarly {
		err := peer.Connect()

		if err != nil {
			return err
		}
	}

	godless.store = peer

	return nil
}

// MakeRemoteNamespace creates a data store representing p2p data.
func (godless *Godless) setupNamespace() error {
	headCache := godless.HeadCache

	if headCache == nil {
		headCache = cache.MakeResidentHeadCache()
	}

	indexCache := godless.IndexCache

	if indexCache == nil {
		indexCache = cache.MakeResidentIndexCache(__BUFFER_SIZE)
	}

	if godless.IndexHash != "" {
		head := crdt.IPFSPath(godless.IndexHash)

		err := headCache.SetHead(head)

		if err != nil {
			return err
		}

	}

	namespaceOptions := service.RemoteNamespaceOptions{
		Store:      godless.store,
		HeadCache:  godless.HeadCache,
		IndexCache: godless.IndexCache,
		KeyStore:   godless.KeyStore,
	}

	godless.remote = service.MakeRemoteNamespace(namespaceOptions)
	return nil
}

func (godless *Godless) launchAPI() error {
	limit := godless.APIQueryLimit

	if limit == 0 {
		limit = 1
	}

	queue := godless.PriorityQueue

	if queue == nil {
		queue = cache.MakeResidentBufferQueue(__BUFFER_SIZE)
	}

	api, errch := service.LaunchKeyValueStore(godless.remote, queue, limit)

	godless.addErrors(errch)
	godless.api = api

	return nil
}

// Serve serves the Godless webservice.
func (godless *Godless) serveWeb() error {
	addr := godless.WebServiceAddr

	if addr == "" {
		return nil
	}

	webService := &service.WebService{API: godless.api}
	stopch, err := http.Serve(addr, webService.Handler())

	if err != nil {
		return err
	}

	godless.addStopper(stopch)
	return nil
}

// Replicate shares data via the IPFS pubsub mechanism.
func (godless *Godless) replicate() error {
	topics := godless.Topics
	interval := godless.ReplicateInterval

	if len(topics) == 0 {
		return nil
	}

	ipfsTopics := make([]crdt.IPFSPath, len(topics))

	for i, t := range topics {
		ipfsTopics[i] = crdt.IPFSPath(t)
	}

	stopch, errch := service.Replicate(godless.api, godless.store, interval, ipfsTopics)
	godless.addStopper(stopch)
	godless.addErrors(errch)
	return nil
}

func (godless *Godless) addStopper(stopch chan<- struct{}) {
	if godless.stopch == nil {
		godless.stopch = make(chan struct{})
		go func() {
			godless.handleShutdown()
		}()
	}

	godless.stoppers = append(godless.stoppers, stopch)
}

func (godless *Godless) handleShutdown() {
	<-godless.stopch
	log.Info("Shutting down")
	for _, stopper := range godless.stoppers {
		go close(stopper)
	}

}

func (godless *Godless) addErrors(errch <-chan error) {
	godless.errwg.Add(1)

	if godless.errch == nil {
		godless.errch = make(chan error)
		go func() {
			godless.errwg.Wait()
			close(godless.errch)
		}()
	}

	go func() {
		for err := range errch {
			godless.errch <- err
		}
		godless.errwg.Done()
	}()
}

// Client is a Godless HTTP client.
type Client interface {
	SendQuery(*query.Query) (api.APIResponse, error)
	SendReflection(api.APIReflectionType) (api.APIResponse, error)
}

// MakeClient creates a Godless HTTP Client.
func MakeClient(serviceAddr string) Client {
	return service.MakeClient(serviceAddr)
}

func MakeClientWithHttp(serviceAddr string, webClient *gohttp.Client) Client {
	return service.MakeClientWithHttp(serviceAddr, webClient)
}

func breakOnError(pipeline []func() error) error {
	for _, f := range pipeline {
		err := f()

		if err != nil {
			return err
		}
	}

	return nil
}

// We don't know the right buffer size here, so let the cache package handle it.
const __BUFFER_SIZE = -1
<commit_msg>Root lib updated for new replicator<commit_after>// Godless is a peer-to-peer database running over IPFS.
//
// Godless uses a Consistent Replicated Data Type called a Namespace to share schemaless data with peers.
//
// This package is a facade to Godless internals.
//
// Godless is in alpha, and should be considered experimental software.
package godless

import (
	"fmt"
	"sync"
	"time"

	gohttp "net/http"

	"github.com/johnny-morrice/godless/api"
	"github.com/johnny-morrice/godless/cache"
	"github.com/johnny-morrice/godless/crdt"
	"github.com/johnny-morrice/godless/internal/http"
	"github.com/johnny-morrice/godless/internal/ipfs"
	"github.com/johnny-morrice/godless/internal/service"
	"github.com/johnny-morrice/godless/log"
	"github.com/johnny-morrice/godless/query"
	"github.com/pkg/errors"
)

// Godless options.
type Options struct {
	// IpfsServiceUrl is required.
	IpfsServiceUrl string
	// KeyStore is required. A private Key store.
	KeyStore api.KeyStore
	// WebServiceAddr is optional.  If not set, the webservice will be disabled.
	WebServiceAddr string
	// IndexHash is optional.  Set to load an existing index from IPFS.
	IndexHash string
	// FailEarly will cause the godless process to crash if it cannot contact IPFS on startup.
	FailEarly bool
	// ReplicateInterval is optional.  The duration between peer-to-peer replications.
	ReplicateInterval time.Duration
	// Topics is optional.  Two godless servers which share a topic will replicate indices. An empty topics slice will disable replication.
	Topics []string
	// IpfsClient is optional.  Specify a HTTP client for IPFS.
	IpfsClient *gohttp.Client
	// IpfsPingTimeout is optional.  Specify a lower timeout for "Am I Connected?" checks.
	IpfsPingTimeout time.Duration
	// HeadCache is optional.  Build a 12-factor app by supplying your own remote cache.
	HeadCache api.HeadCache
	// IndexCache is optional.  Build a 12-factor app by supplying your own remote cache.
	IndexCache api.IndexCache
	// PriorityQueue is optional. Build a 12-factor app by supplying your own remote cache.
	PriorityQueue api.RequestPriorityQueue
	// APIQueryLimit is optional.  Tune performance by setting the number of simultaneous queries.
	APIQueryLimit int
}

// Godless is a peer-to-peer database.  It shares structured data between peers, using IPFS as a backing store.
// The core datastructure is a CRDT namespace which resembles a relational scheme in that it has tables, rows, and entries.
type Godless struct {
	Options
	errch    chan error
	errwg    sync.WaitGroup
	stopch   chan struct{}
	stoppers []chan<- struct{}
	store    api.RemoteStore
	remote   api.RemoteNamespace
	api      api.APIService
}

// New creates a godless instance, connecting to any services, and providing any services, specified in the options.
func New(options Options) (*Godless, error) {
	godless := &Godless{Options: options}

	missing := godless.findMissingParameters()

	if missing != nil {
		return nil, missing
	}

	setupFuncs := []func() error{
		godless.connectIpfs,
		godless.setupNamespace,
		godless.launchAPI,
		godless.serveWeb,
		godless.replicate,
	}

	err := breakOnError(setupFuncs)

	if err != nil {
		return nil, err
	}

	return godless, nil
}

func (godless *Godless) findMissingParameters() error {
	var missing error
	if godless.IpfsServiceUrl == "" {
		msg := godless.missingParameterText("IpfsServiceUrl")
		missing = errors.New(msg)
	}

	if godless.KeyStore == nil {
		msg := godless.missingParameterText("KeyStore")
		if missing == nil {
			missing = errors.New(msg)
		} else {
			missing = errors.Wrap(missing, msg)
		}
	}

	return missing
}

func (godless *Godless) missingParameterText(param string) string {
	return fmt.Sprintf("Missing required parameter '%v'", param)
}

// Errors provides a stream of errors from godless.  Godless will attempt to handle any errors it can.  Any errors received here indicate that bad things have happened.
func (godless *Godless) Errors() <-chan error {
	return godless.errch
}

// Shutdown stops all godless processes.  It does not wait for those goroutines to stop.
func (godless *Godless) Shutdown() {
	godless.stopch <- struct{}{}
}

func (godless *Godless) connectIpfs() error {
	client := godless.IpfsClient
	pingTimeout := godless.IpfsPingTimeout

	peer := &ipfs.IPFSPeer{
		Url:         godless.IpfsServiceUrl,
		Client:      client,
		PingTimeout: pingTimeout,
	}

	if godless.FailEarly {
		err := peer.Connect()

		if err != nil {
			return err
		}
	}

	godless.store = peer

	return nil
}

// MakeRemoteNamespace creates a data store representing p2p data.
func (godless *Godless) setupNamespace() error {
	headCache := godless.HeadCache

	if headCache == nil {
		headCache = cache.MakeResidentHeadCache()
	}

	indexCache := godless.IndexCache

	if indexCache == nil {
		indexCache = cache.MakeResidentIndexCache(__BUFFER_SIZE)
	}

	if godless.IndexHash != "" {
		head := crdt.IPFSPath(godless.IndexHash)

		err := headCache.SetHead(head)

		if err != nil {
			return err
		}

	}

	namespaceOptions := service.RemoteNamespaceOptions{
		Store:      godless.store,
		HeadCache:  godless.HeadCache,
		IndexCache: godless.IndexCache,
		KeyStore:   godless.KeyStore,
	}

	godless.remote = service.MakeRemoteNamespace(namespaceOptions)
	return nil
}

func (godless *Godless) launchAPI() error {
	limit := godless.APIQueryLimit

	if limit == 0 {
		limit = 1
	}

	queue := godless.PriorityQueue

	if queue == nil {
		queue = cache.MakeResidentBufferQueue(__BUFFER_SIZE)
	}

	api, errch := service.LaunchKeyValueStore(godless.remote, queue, limit)

	godless.addErrors(errch)
	godless.api = api

	return nil
}

// Serve serves the Godless webservice.
func (godless *Godless) serveWeb() error {
	addr := godless.WebServiceAddr

	if addr == "" {
		return nil
	}

	webService := &service.WebService{API: godless.api}
	stopch, err := http.Serve(addr, webService.Handler())

	if err != nil {
		return err
	}

	godless.addStopper(stopch)
	return nil
}

// Replicate shares data via the IPFS pubsub mechanism.
func (godless *Godless) replicate() error {
	topics := godless.Topics
	interval := godless.ReplicateInterval

	if len(topics) == 0 {
		return nil
	}

	pubsubTopics := make([]api.PubSubTopic, len(topics))

	for i, t := range topics {
		pubsubTopics[i] = api.PubSubTopic(t)
	}

	options := service.ReplicateOptions{
		API:         godless.api,
		RemoteStore: godless.store,
		Interval:    interval,
		Topics:      pubsubTopics,
		KeyStore:    godless.KeyStore,
	}
	stopch, errch := service.Replicate(options)
	godless.addStopper(stopch)
	godless.addErrors(errch)
	return nil
}

func (godless *Godless) addStopper(stopch chan<- struct{}) {
	if godless.stopch == nil {
		godless.stopch = make(chan struct{})
		go func() {
			godless.handleShutdown()
		}()
	}

	godless.stoppers = append(godless.stoppers, stopch)
}

func (godless *Godless) handleShutdown() {
	<-godless.stopch
	log.Info("Shutting down")
	for _, stopper := range godless.stoppers {
		go close(stopper)
	}

}

func (godless *Godless) addErrors(errch <-chan error) {
	godless.errwg.Add(1)

	if godless.errch == nil {
		godless.errch = make(chan error)
		go func() {
			godless.errwg.Wait()
			close(godless.errch)
		}()
	}

	go func() {
		for err := range errch {
			godless.errch <- err
		}
		godless.errwg.Done()
	}()
}

// Client is a Godless HTTP client.
type Client interface {
	SendQuery(*query.Query) (api.APIResponse, error)
	SendReflection(api.APIReflectionType) (api.APIResponse, error)
}

// MakeClient creates a Godless HTTP Client.
func MakeClient(serviceAddr string) Client {
	return service.MakeClient(serviceAddr)
}

func MakeClientWithHttp(serviceAddr string, webClient *gohttp.Client) Client {
	return service.MakeClientWithHttp(serviceAddr, webClient)
}

func breakOnError(pipeline []func() error) error {
	for _, f := range pipeline {
		err := f()

		if err != nil {
			return err
		}
	}

	return nil
}

// We don't know the right buffer size here, so let the cache package handle it.
const __BUFFER_SIZE = -1
<|endoftext|>


<commit_before>package main

import (
    "fmt"
    "io/ioutil"
    "os"
)

func main() {
    var (
        cpu Cpu
        rom Rom
        ram Ram
    )

    fmt.Println("GoMaybe")
    fmt.Println("Jahn Veach <v64@v64.net>")
    fmt.Println("https://github.com/v64/gomaybe")

    if len(os.Args) < 2 {
        fmt.Println("Usage: gomaybe rom.gb")
        return
    }

    file := os.Args[1];

    if romData, err := ioutil.ReadFile(file); err == nil {
        fmt.Println("Loading ROM: " + file)
        ram.Init()
        rom.Init(romData, &ram)
        cpu.Init(&ram)
    } else {
        fmt.Println("Error loading ROM: " + err.Error())
        return
    }

    for {
        cycleCount := cpu.Step()
        if cycleCount == 0 {
            fmt.Println("Unknown opcode encountered, exiting")
            break
        }
    }
}
<commit_msg>move vars out<commit_after>package main

import (
    "fmt"
    "io/ioutil"
    "os"
)

var (
    cpu Cpu
    rom Rom
    ram Ram
)

func main() {
    fmt.Println("GoMaybe")
    fmt.Println("Jahn Veach <v64@v64.net>")
    fmt.Println("https://github.com/v64/gomaybe")

    if len(os.Args) < 2 {
        fmt.Println("Usage: gomaybe rom.gb")
        return
    }

    file := os.Args[1];

    if romData, err := ioutil.ReadFile(file); err == nil {
        fmt.Println("Loading ROM: " + file)
        ram.Init()
        rom.Init(romData, &ram)
        cpu.Init(&ram)
    } else {
        fmt.Println("Error loading ROM: " + err.Error())
        return
    }

    for {
        cycleCount := cpu.Step()
        if cycleCount == 0 {
            fmt.Println("Unknown opcode encountered, exiting")
            break
        }
    }
}
<|endoftext|>