<commit_before>// Copyright 2014 Hajime Hoshi
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package ebiten

import (
	"github.com/hajimehoshi/ebiten/internal/clock"
	"github.com/hajimehoshi/ebiten/internal/graphics"
	"github.com/hajimehoshi/ebiten/internal/hooks"
	"github.com/hajimehoshi/ebiten/internal/restorable"
	"github.com/hajimehoshi/ebiten/internal/ui"
	"github.com/hajimehoshi/ebiten/internal/web"
)

func newGraphicsContext(f func(*Image) error) *graphicsContext {
	return &graphicsContext{
		f: f,
	}
}

type graphicsContext struct {
	f           func(*Image) error
	offscreen   *Image
	screen      *Image
	initialized bool
	invalidated bool // browser only
	offsetX     float64
	offsetY     float64
}

func (c *graphicsContext) Invalidate() {
	// Note that this is called only on browsers so far.
	// TODO: On mobiles, this function is not called and instead IsTexture is called
	// to detect if the context is lost. This is simple but might not work on some platforms.
	// Should Invalidate be called explicitly?
	c.invalidated = true
}

func (c *graphicsContext) SetSize(screenWidth, screenHeight int, screenScale float64) {
	if c.screen != nil {
		_ = c.screen.Dispose()
	}
	if c.offscreen != nil {
		_ = c.offscreen.Dispose()
	}
	c.offscreen = newVolatileImage(screenWidth, screenHeight, FilterDefault)

	w := int(float64(screenWidth) * screenScale)
	h := int(float64(screenHeight) * screenScale)
	px0, py0, _, _ := ui.ScreenPadding()
	c.screen = newImageWithScreenFramebuffer(w, h)

	c.offsetX = px0
	c.offsetY = py0
}

func (c *graphicsContext) initializeIfNeeded() error {
	if !c.initialized {
		if err := restorable.InitializeGLState(); err != nil {
			return err
		}
		c.initialized = true
	}
	if err := c.restoreIfNeeded(); err != nil {
		return err
	}
	return nil
}

func (c *graphicsContext) Update(afterFrameUpdate func()) error {
	updateCount := clock.Update()

	if err := c.initializeIfNeeded(); err != nil {
		return err
	}
	for i := 0; i < updateCount; i++ {
		c.offscreen.fill(0, 0, 0, 0)

		setRunningSlowly(i < updateCount-1)
		if err := hooks.Run(); err != nil {
			return err
		}
		if err := c.f(c.offscreen); err != nil {
			return err
		}
		afterFrameUpdate()
	}

	// Clear the screen framebuffer by DrawImage instad of Fill
	// to clear the whole region including fullscreen's padding.
	// TODO: This clear is needed only when the screen size is changed.
	if c.offsetX > 0 || c.offsetY > 0 {
		op := &DrawImageOptions{}
		w, h := emptyImage.Size()
		// graphics.MaxImageSize should be the maximum size of framebuffer.
		op.GeoM.Scale(graphics.MaxImageSize/float64(w), graphics.MaxImageSize/float64(h))
		op.CompositeMode = CompositeModeCopy
		op.Filter = filterScreen // any filter is fine: just use the same filter as below.
		c.screen.DrawImage(emptyImage, op)
	}

	dw, dh := c.screen.Size()
	sw, _ := c.offscreen.Size()
	scale := float64(dw) / float64(sw)

	op := &DrawImageOptions{}
	// c.screen is special: its Y axis is down to up,
	// and the origin point is lower left.
	op.GeoM.Scale(scale, -scale)
	op.GeoM.Translate(0, float64(dh))
	op.GeoM.Translate(c.offsetX, c.offsetY)

	op.CompositeMode = CompositeModeCopy
	op.Filter = filterScreen
	_ = c.screen.DrawImage(c.offscreen, op)

	if err := restorable.ResolveStaleImages(); err != nil {
		return err
	}
	return nil
}

func (c *graphicsContext) needsRestoring() (bool, error) {
	if web.IsBrowser() {
		return c.invalidated, nil
	}
	return c.offscreen.restorable.IsInvalidated()
}

func (c *graphicsContext) restoreIfNeeded() error {
	if !restorable.IsRestoringEnabled() {
		return nil
	}
	r, err := c.needsRestoring()
	if err != nil {
		return err
	}
	if !r {
		return nil
	}
	if err := restorable.Restore(); err != nil {
		return err
	}
	c.invalidated = false
	return nil
}
<commit_msg>graphics: The screen filter might be heavy<commit_after>// Copyright 2014 Hajime Hoshi
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package ebiten

import (
	"github.com/hajimehoshi/ebiten/internal/clock"
	"github.com/hajimehoshi/ebiten/internal/graphics"
	"github.com/hajimehoshi/ebiten/internal/hooks"
	"github.com/hajimehoshi/ebiten/internal/restorable"
	"github.com/hajimehoshi/ebiten/internal/ui"
	"github.com/hajimehoshi/ebiten/internal/web"
)

func newGraphicsContext(f func(*Image) error) *graphicsContext {
	return &graphicsContext{
		f: f,
	}
}

type graphicsContext struct {
	f           func(*Image) error
	offscreen   *Image
	screen      *Image
	initialized bool
	invalidated bool // browser only
	offsetX     float64
	offsetY     float64
}

func (c *graphicsContext) Invalidate() {
	// Note that this is called only on browsers so far.
	// TODO: On mobiles, this function is not called and instead IsTexture is called
	// to detect if the context is lost. This is simple but might not work on some platforms.
	// Should Invalidate be called explicitly?
	c.invalidated = true
}

func (c *graphicsContext) SetSize(screenWidth, screenHeight int, screenScale float64) {
	if c.screen != nil {
		_ = c.screen.Dispose()
	}
	if c.offscreen != nil {
		_ = c.offscreen.Dispose()
	}
	c.offscreen = newVolatileImage(screenWidth, screenHeight, FilterDefault)

	w := int(float64(screenWidth) * screenScale)
	h := int(float64(screenHeight) * screenScale)
	px0, py0, _, _ := ui.ScreenPadding()
	c.screen = newImageWithScreenFramebuffer(w, h)

	c.offsetX = px0
	c.offsetY = py0
}

func (c *graphicsContext) initializeIfNeeded() error {
	if !c.initialized {
		if err := restorable.InitializeGLState(); err != nil {
			return err
		}
		c.initialized = true
	}
	if err := c.restoreIfNeeded(); err != nil {
		return err
	}
	return nil
}

func (c *graphicsContext) Update(afterFrameUpdate func()) error {
	updateCount := clock.Update()

	if err := c.initializeIfNeeded(); err != nil {
		return err
	}
	for i := 0; i < updateCount; i++ {
		c.offscreen.fill(0, 0, 0, 0)

		setRunningSlowly(i < updateCount-1)
		if err := hooks.Run(); err != nil {
			return err
		}
		if err := c.f(c.offscreen); err != nil {
			return err
		}
		afterFrameUpdate()
	}

	// Clear the screen framebuffer by DrawImage instad of Fill
	// to clear the whole region including fullscreen's padding.
	// TODO: This clear is needed only when the screen size is changed.
	if c.offsetX > 0 || c.offsetY > 0 {
		op := &DrawImageOptions{}
		w, h := emptyImage.Size()
		// graphics.MaxImageSize should be the maximum size of framebuffer.
		op.GeoM.Scale(graphics.MaxImageSize/float64(w), graphics.MaxImageSize/float64(h))
		op.CompositeMode = CompositeModeCopy
		c.screen.DrawImage(emptyImage, op)
	}

	dw, dh := c.screen.Size()
	sw, _ := c.offscreen.Size()
	scale := float64(dw) / float64(sw)

	op := &DrawImageOptions{}
	// c.screen is special: its Y axis is down to up,
	// and the origin point is lower left.
	op.GeoM.Scale(scale, -scale)
	op.GeoM.Translate(0, float64(dh))
	op.GeoM.Translate(c.offsetX, c.offsetY)

	op.CompositeMode = CompositeModeCopy
	op.Filter = filterScreen
	_ = c.screen.DrawImage(c.offscreen, op)

	if err := restorable.ResolveStaleImages(); err != nil {
		return err
	}
	return nil
}

func (c *graphicsContext) needsRestoring() (bool, error) {
	if web.IsBrowser() {
		return c.invalidated, nil
	}
	return c.offscreen.restorable.IsInvalidated()
}

func (c *graphicsContext) restoreIfNeeded() error {
	if !restorable.IsRestoringEnabled() {
		return nil
	}
	r, err := c.needsRestoring()
	if err != nil {
		return err
	}
	if !r {
		return nil
	}
	if err := restorable.Restore(); err != nil {
		return err
	}
	c.invalidated = false
	return nil
}
<|endoftext|>


<commit_before>package pig

import (
	"errors"
	"github.com/jkomoros/boardgame"
)

//+autoreader readsetter
type moveRollDice struct {
	boardgame.DefaultMove
	TargetPlayerIndex boardgame.PlayerIndex
}

//+autoreader readsetter
type moveDoneTurn struct {
	boardgame.DefaultMove
	TargetPlayerIndex boardgame.PlayerIndex
}

//+autoreader readsetter
type moveCountDie struct {
	boardgame.DefaultMove
	TargetPlayerIndex boardgame.PlayerIndex
}

//+autoreader readsetter
type moveAdvanceNextPlayer struct {
	boardgame.DefaultMove
	TargetPlayerIndex boardgame.PlayerIndex
}

/**************************************************
 *
 * MoveRollDice Implementation
 *
 **************************************************/

func MoveRollDiceFactory(state boardgame.State) boardgame.Move {
	result := &moveRollDice{
		boardgame.DefaultMove{
			"Roll Dice",
			"Rolls the dice for the current player",
		},
		0,
	}

	if state != nil {
		result.TargetPlayerIndex = state.CurrentPlayer().PlayerIndex()
	}

	return result
}

func (m *moveRollDice) Legal(state boardgame.State, proposer boardgame.PlayerIndex) error {
	game, players := concreteStates(state)

	if !proposer.Equivalent(game.CurrentPlayer) {
		return errors.New("You are not the current player!")
	}

	if !m.TargetPlayerIndex.Equivalent(proposer) {
		return errors.New("You are not the current player!")
	}

	p := players[game.CurrentPlayer]

	if !p.DieCounted {
		return errors.New("Your most recent roll has not yet been counted")
	}

	return nil
}

func (m *moveRollDice) Apply(state boardgame.MutableState) error {
	game, players := concreteStates(state)

	p := players[game.CurrentPlayer]

	die := game.Die.ComponentAt(0)

	if err := die.DynamicValues(state).(*dieDynamicValue).Roll(die); err != nil {
		return errors.New("Couldn't roll die: " + err.Error())
	}

	p.DieCounted = false

	return nil
}

/**************************************************
 *
 * MoveDoneTurn Implementation
 *
 **************************************************/

func MoveDoneTurnFactory(state boardgame.State) boardgame.Move {
	result := &moveDoneTurn{
		boardgame.DefaultMove{
			"Done Turn",
			"Played when a player is done with their turn and wants to keep their score.",
		},
		0,
	}

	if state != nil {
		result.TargetPlayerIndex = state.CurrentPlayer().PlayerIndex()
	}

	return result
}

func (m *moveDoneTurn) Legal(state boardgame.State, proposer boardgame.PlayerIndex) error {
	game, players := concreteStates(state)

	if !proposer.Equivalent(game.CurrentPlayer) {
		return errors.New("You are not the current player!")
	}

	if !m.TargetPlayerIndex.Equivalent(proposer) {
		return errors.New("You are not the current player!")
	}

	p := players[game.CurrentPlayer]

	if !p.DieCounted {
		return errors.New("Your most recent roll has not yet been counted")
	}

	if p.Done {
		return errors.New("You already signaled that you are done!")
	}

	return nil
}

func (m *moveDoneTurn) Apply(state boardgame.MutableState) error {
	game, players := concreteStates(state)

	p := players[game.CurrentPlayer]

	p.Done = true

	return nil
}

/**************************************************
 *
 * MoveCountDie Implementation
 *
 **************************************************/

func MoveCountDieFactory(state boardgame.State) boardgame.Move {
	result := &moveCountDie{
		boardgame.DefaultMove{
			"Count Die",
			"After a die has been rolled, tabulating its impact",
		},
		0,
	}

	if state != nil {
		result.TargetPlayerIndex = state.CurrentPlayer().PlayerIndex()
	}

	return result
}

func (m *moveCountDie) Legal(state boardgame.State, proposer boardgame.PlayerIndex) error {
	game, players := concreteStates(state)

	if !proposer.Equivalent(game.CurrentPlayer) {
		return errors.New("You are not the current player!")
	}

	if !m.TargetPlayerIndex.Equivalent(proposer) {
		return errors.New("You are not the current player!")
	}

	p := players[game.CurrentPlayer]

	if p.DieCounted {
		return errors.New("The most recent die roll has already been counted.")
	}

	return nil
}

func (m *moveCountDie) Apply(state boardgame.MutableState) error {
	game, players := concreteStates(state)

	p := players[game.CurrentPlayer]

	value := game.Die.ComponentAt(0).DynamicValues(state).(*dieDynamicValue).Value

	if value == 1 {
		//Bust!
		p.Busted = true
	} else {
		p.RoundScore += value
	}

	return nil
}

/**************************************************
 *
 * MoveAdvanceNextPlayer Implementation
 *
 **************************************************/

func MoveAdvanceNextPlayerFactory(state boardgame.State) boardgame.Move {
	result := &moveAdvanceNextPlayer{
		boardgame.DefaultMove{
			"Advance Next Player",
			"Advance to the next player when the current player has busted or said they are done.",
		},
		0,
	}

	if state != nil {
		result.TargetPlayerIndex = state.CurrentPlayer().PlayerIndex()
	}

	return result
}

func (m *moveAdvanceNextPlayer) Legal(state boardgame.State, proposer boardgame.PlayerIndex) error {
	game, players := concreteStates(state)

	if !proposer.Equivalent(game.CurrentPlayer) {
		return errors.New("You are not the current player!")
	}

	if !m.TargetPlayerIndex.Equivalent(proposer) {
		return errors.New("You are not the current player!")
	}

	p := players[game.CurrentPlayer]

	if !p.DieCounted {
		return errors.New("The most recent die roll has not been counted!")
	}

	if !p.Busted && !p.Done {
		return errors.New("The player has not either busted or signaled that they are done.")
	}

	return nil
}

func (m *moveAdvanceNextPlayer) Apply(state boardgame.MutableState) error {
	game, players := concreteStates(state)

	p := players[game.CurrentPlayer]

	if p.Done {
		p.TotalScore += p.RoundScore
	}
	p.ResetForTurn()

	game.CurrentPlayer = game.CurrentPlayer.Next(state)

	p = players[game.CurrentPlayer]

	p.ResetForTurn()

	return nil
}
<commit_msg>Fixed a bug where dies were not marked as counted. Part of #372.<commit_after>package pig

import (
	"errors"
	"github.com/jkomoros/boardgame"
)

//+autoreader readsetter
type moveRollDice struct {
	boardgame.DefaultMove
	TargetPlayerIndex boardgame.PlayerIndex
}

//+autoreader readsetter
type moveDoneTurn struct {
	boardgame.DefaultMove
	TargetPlayerIndex boardgame.PlayerIndex
}

//+autoreader readsetter
type moveCountDie struct {
	boardgame.DefaultMove
	TargetPlayerIndex boardgame.PlayerIndex
}

//+autoreader readsetter
type moveAdvanceNextPlayer struct {
	boardgame.DefaultMove
	TargetPlayerIndex boardgame.PlayerIndex
}

/**************************************************
 *
 * MoveRollDice Implementation
 *
 **************************************************/

func MoveRollDiceFactory(state boardgame.State) boardgame.Move {
	result := &moveRollDice{
		boardgame.DefaultMove{
			"Roll Dice",
			"Rolls the dice for the current player",
		},
		0,
	}

	if state != nil {
		result.TargetPlayerIndex = state.CurrentPlayer().PlayerIndex()
	}

	return result
}

func (m *moveRollDice) Legal(state boardgame.State, proposer boardgame.PlayerIndex) error {
	game, players := concreteStates(state)

	if !proposer.Equivalent(game.CurrentPlayer) {
		return errors.New("You are not the current player!")
	}

	if !m.TargetPlayerIndex.Equivalent(proposer) {
		return errors.New("You are not the current player!")
	}

	p := players[game.CurrentPlayer]

	if !p.DieCounted {
		return errors.New("Your most recent roll has not yet been counted")
	}

	return nil
}

func (m *moveRollDice) Apply(state boardgame.MutableState) error {
	game, players := concreteStates(state)

	p := players[game.CurrentPlayer]

	die := game.Die.ComponentAt(0)

	if err := die.DynamicValues(state).(*dieDynamicValue).Roll(die); err != nil {
		return errors.New("Couldn't roll die: " + err.Error())
	}

	p.DieCounted = false

	return nil
}

/**************************************************
 *
 * MoveDoneTurn Implementation
 *
 **************************************************/

func MoveDoneTurnFactory(state boardgame.State) boardgame.Move {
	result := &moveDoneTurn{
		boardgame.DefaultMove{
			"Done Turn",
			"Played when a player is done with their turn and wants to keep their score.",
		},
		0,
	}

	if state != nil {
		result.TargetPlayerIndex = state.CurrentPlayer().PlayerIndex()
	}

	return result
}

func (m *moveDoneTurn) Legal(state boardgame.State, proposer boardgame.PlayerIndex) error {
	game, players := concreteStates(state)

	if !proposer.Equivalent(game.CurrentPlayer) {
		return errors.New("You are not the current player!")
	}

	if !m.TargetPlayerIndex.Equivalent(proposer) {
		return errors.New("You are not the current player!")
	}

	p := players[game.CurrentPlayer]

	if !p.DieCounted {
		return errors.New("Your most recent roll has not yet been counted")
	}

	if p.Done {
		return errors.New("You already signaled that you are done!")
	}

	return nil
}

func (m *moveDoneTurn) Apply(state boardgame.MutableState) error {
	game, players := concreteStates(state)

	p := players[game.CurrentPlayer]

	p.Done = true

	return nil
}

/**************************************************
 *
 * MoveCountDie Implementation
 *
 **************************************************/

func MoveCountDieFactory(state boardgame.State) boardgame.Move {
	result := &moveCountDie{
		boardgame.DefaultMove{
			"Count Die",
			"After a die has been rolled, tabulating its impact",
		},
		0,
	}

	if state != nil {
		result.TargetPlayerIndex = state.CurrentPlayer().PlayerIndex()
	}

	return result
}

func (m *moveCountDie) Legal(state boardgame.State, proposer boardgame.PlayerIndex) error {
	game, players := concreteStates(state)

	if !proposer.Equivalent(game.CurrentPlayer) {
		return errors.New("You are not the current player!")
	}

	if !m.TargetPlayerIndex.Equivalent(proposer) {
		return errors.New("You are not the current player!")
	}

	p := players[game.CurrentPlayer]

	if p.DieCounted {
		return errors.New("The most recent die roll has already been counted.")
	}

	return nil
}

func (m *moveCountDie) Apply(state boardgame.MutableState) error {
	game, players := concreteStates(state)

	p := players[game.CurrentPlayer]

	value := game.Die.ComponentAt(0).DynamicValues(state).(*dieDynamicValue).Value

	if value == 1 {
		//Bust!
		p.Busted = true
	} else {
		p.RoundScore += value
	}

	p.DieCounted = true

	return nil
}

/**************************************************
 *
 * MoveAdvanceNextPlayer Implementation
 *
 **************************************************/

func MoveAdvanceNextPlayerFactory(state boardgame.State) boardgame.Move {
	result := &moveAdvanceNextPlayer{
		boardgame.DefaultMove{
			"Advance Next Player",
			"Advance to the next player when the current player has busted or said they are done.",
		},
		0,
	}

	if state != nil {
		result.TargetPlayerIndex = state.CurrentPlayer().PlayerIndex()
	}

	return result
}

func (m *moveAdvanceNextPlayer) Legal(state boardgame.State, proposer boardgame.PlayerIndex) error {
	game, players := concreteStates(state)

	if !proposer.Equivalent(game.CurrentPlayer) {
		return errors.New("You are not the current player!")
	}

	if !m.TargetPlayerIndex.Equivalent(proposer) {
		return errors.New("You are not the current player!")
	}

	p := players[game.CurrentPlayer]

	if !p.DieCounted {
		return errors.New("The most recent die roll has not been counted!")
	}

	if !p.Busted && !p.Done {
		return errors.New("The player has not either busted or signaled that they are done.")
	}

	return nil
}

func (m *moveAdvanceNextPlayer) Apply(state boardgame.MutableState) error {
	game, players := concreteStates(state)

	p := players[game.CurrentPlayer]

	if p.Done {
		p.TotalScore += p.RoundScore
	}
	p.ResetForTurn()

	game.CurrentPlayer = game.CurrentPlayer.Next(state)

	p = players[game.CurrentPlayer]

	p.ResetForTurn()

	return nil
}
<|endoftext|>


<commit_before>package aws

import (
	"fmt"
	"log"
	"time"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/awserr"
	"github.com/aws/aws-sdk-go/service/ec2"
	"github.com/hashicorp/terraform/helper/resource"
	"github.com/hashicorp/terraform/helper/schema"
)

func resourceAwsSpotInstanceRequest() *schema.Resource {
	return &schema.Resource{
		Create: resourceAwsSpotInstanceRequestCreate,
		Read:   resourceAwsSpotInstanceRequestRead,
		Delete: resourceAwsSpotInstanceRequestDelete,
		Update: resourceAwsSpotInstanceRequestUpdate,

		Schema: func() map[string]*schema.Schema {
			// The Spot Instance Request Schema is based on the AWS Instance schema.
			s := resourceAwsInstance().Schema

			// Everything on a spot instance is ForceNew except tags
			for k, v := range s {
				if k == "tags" {
					continue
				}
				v.ForceNew = true
			}

			s["spot_price"] = &schema.Schema{
				Type:     schema.TypeString,
				Required: true,
				ForceNew: true,
			}
			s["spot_type"] = &schema.Schema{
				Type:     schema.TypeString,
				Optional: true,
				Default:  "persistent",
			}
			s["wait_for_fulfillment"] = &schema.Schema{
				Type:     schema.TypeBool,
				Optional: true,
				Default:  false,
			}
			s["spot_bid_status"] = &schema.Schema{
				Type:     schema.TypeString,
				Computed: true,
			}
			s["spot_request_state"] = &schema.Schema{
				Type:     schema.TypeString,
				Computed: true,
			}
			s["spot_instance_id"] = &schema.Schema{
				Type:     schema.TypeString,
				Computed: true,
			}

			return s
		}(),
	}
}

func resourceAwsSpotInstanceRequestCreate(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).ec2conn

	instanceOpts, err := buildAwsInstanceOpts(d, meta)
	if err != nil {
		return err
	}

	spotOpts := &ec2.RequestSpotInstancesInput{
		SpotPrice: aws.String(d.Get("spot_price").(string)),
		Type:      aws.String(d.Get("spot_type").(string)),

		// Though the AWS API supports creating spot instance requests for multiple
		// instances, for TF purposes we fix this to one instance per request.
		// Users can get equivalent behavior out of TF's "count" meta-parameter.
		InstanceCount: aws.Int64(1),

		LaunchSpecification: &ec2.RequestSpotLaunchSpecification{
			BlockDeviceMappings: instanceOpts.BlockDeviceMappings,
			EbsOptimized:        instanceOpts.EBSOptimized,
			Monitoring:          instanceOpts.Monitoring,
			IamInstanceProfile:  instanceOpts.IAMInstanceProfile,
			ImageId:             instanceOpts.ImageID,
			InstanceType:        instanceOpts.InstanceType,
			KeyName:             instanceOpts.KeyName,
			Placement:           instanceOpts.SpotPlacement,
			SecurityGroupIds:    instanceOpts.SecurityGroupIDs,
			SecurityGroups:      instanceOpts.SecurityGroups,
			SubnetId:            instanceOpts.SubnetID,
			UserData:            instanceOpts.UserData64,
		},
	}

	// If the instance is configured with a Network Interface (a subnet, has
	// public IP, etc), then the instanceOpts.SecurityGroupIds and SubnetId will
	// be nil
	if len(instanceOpts.NetworkInterfaces) > 0 {
		spotOpts.LaunchSpecification.SecurityGroupIds = instanceOpts.NetworkInterfaces[0].Groups
		spotOpts.LaunchSpecification.SubnetId = instanceOpts.NetworkInterfaces[0].SubnetId
	}

	// Make the spot instance request
	log.Printf("[DEBUG] Requesting spot bid opts: %s", spotOpts)
	resp, err := conn.RequestSpotInstances(spotOpts)
	if err != nil {
		return fmt.Errorf("Error requesting spot instances: %s", err)
	}
	if len(resp.SpotInstanceRequests) != 1 {
		return fmt.Errorf(
			"Expected response with length 1, got: %s", resp)
	}

	sir := *resp.SpotInstanceRequests[0]
	d.SetId(*sir.SpotInstanceRequestId)

	if d.Get("wait_for_fulfillment").(bool) {
		spotStateConf := &resource.StateChangeConf{
			// http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/spot-bid-status.html
			Pending:    []string{"start", "pending-evaluation", "pending-fulfillment"},
			Target:     "fulfilled",
			Refresh:    SpotInstanceStateRefreshFunc(conn, sir),
			Timeout:    10 * time.Minute,
			Delay:      10 * time.Second,
			MinTimeout: 3 * time.Second,
		}

		log.Printf("[DEBUG] waiting for spot bid to resolve... this may take several minutes.")
		_, err = spotStateConf.WaitForState()

		if err != nil {
			return fmt.Errorf("Error while waiting for spot request (%s) to resolve: %s", sir, err)
		}
	}

	return resourceAwsSpotInstanceRequestUpdate(d, meta)
}

// Update spot state, etc
func resourceAwsSpotInstanceRequestRead(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).ec2conn

	req := &ec2.DescribeSpotInstanceRequestsInput{
		SpotInstanceRequestIds: []*string{aws.String(d.Id())},
	}
	resp, err := conn.DescribeSpotInstanceRequests(req)

	if err != nil {
		// If the spot request was not found, return nil so that we can show
		// that it is gone.
		if ec2err, ok := err.(awserr.Error); ok && ec2err.Code() == "InvalidSpotInstanceRequestID.NotFound" {
			d.SetId("")
			return nil
		}

		// Some other error, report it
		return err
	}

	// If nothing was found, then return no state
	if len(resp.SpotInstanceRequests) == 0 {
		d.SetId("")
		return nil
	}

	request := resp.SpotInstanceRequests[0]

	// if the request is cancelled, then it is gone
	if *request.State == "cancelled" {
		d.SetId("")
		return nil
	}

	d.Set("spot_bid_status", *request.Status.Code)
	// Instance ID is not set if the request is still pending
	if request.InstanceId != nil {
		d.Set("spot_instance_id", *request.InstanceId)
		if err := readInstance(d, meta); err != nil {
			return fmt.Errorf("[ERR] Error reading Spot Instance Data: %s", err)
		}
	}
	d.Set("spot_request_state", *request.State)
	d.Set("tags", tagsToMap(request.Tags))

	// return nil
	// let's read the instance data...
	return readInstance(d, meta)
}

func readInstance(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).ec2conn

	resp, err := conn.DescribeInstances(&ec2.DescribeInstancesInput{
		InstanceIds: []*string{aws.String(d.Get("spot_instance_id").(string))},
	})
	if err != nil {
		// If the instance was not found, return nil so that we can show
		// that the instance is gone.
		if ec2err, ok := err.(awserr.Error); ok && ec2err.Code() == "InvalidInstanceID.NotFound" {
			return fmt.Errorf("no instance found")
		}

		// Some other error, report it
		return err
	}

	// If nothing was found, then return no state
	if len(resp.Reservations) == 0 {
		return fmt.Errorf("no instances found")
	}

	instance := resp.Reservations[0].Instances[0]

	// Set these fields for connection information
	if instance != nil {
		d.Set("public_dns", instance.PublicDnsName)
		d.Set("public_ip", instance.PublicIpAddress)
		d.Set("private_dns", instance.PrivateDnsName)
		d.Set("private_ip", instance.PrivateIpAddress)
	}

	// set connection information
	if instance.PublicIpAddress != nil {
		d.SetConnInfo(map[string]string{
			"type": "ssh",
			"host": *instance.PublicIpAddress,
		})
	} else if instance.PrivateIpAddress != nil {
		d.SetConnInfo(map[string]string{
			"type": "ssh",
			"host": *instance.PrivateIpAddress,
		})
	}

	return nil
}

func resourceAwsSpotInstanceRequestUpdate(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).ec2conn

	d.Partial(true)
	if err := setTags(conn, d); err != nil {
		return err
	} else {
		d.SetPartial("tags")
	}

	d.Partial(false)

	return resourceAwsSpotInstanceRequestRead(d, meta)
}

func resourceAwsSpotInstanceRequestDelete(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).ec2conn

	log.Printf("[INFO] Cancelling spot request: %s", d.Id())
	_, err := conn.CancelSpotInstanceRequests(&ec2.CancelSpotInstanceRequestsInput{
		SpotInstanceRequestIds: []*string{aws.String(d.Id())},
	})

	if err != nil {
		return fmt.Errorf("Error cancelling spot request (%s): %s", d.Id(), err)
	}

	if instanceId := d.Get("spot_instance_id").(string); instanceId != "" {
		log.Printf("[INFO] Terminating instance: %s", instanceId)
		if err := awsTerminateInstance(conn, instanceId); err != nil {
			return fmt.Errorf("Error terminating spot instance: %s", err)
		}
	}

	return nil
}

// SpotInstanceStateRefreshFunc returns a resource.StateRefreshFunc that is used to watch
// an EC2 spot instance request
func SpotInstanceStateRefreshFunc(
	conn *ec2.EC2, sir ec2.SpotInstanceRequest) resource.StateRefreshFunc {

	return func() (interface{}, string, error) {
		resp, err := conn.DescribeSpotInstanceRequests(&ec2.DescribeSpotInstanceRequestsInput{
			SpotInstanceRequestIds: []*string{sir.SpotInstanceRequestId},
		})

		if err != nil {
			if ec2err, ok := err.(awserr.Error); ok && ec2err.Code() == "InvalidSpotInstanceRequestID.NotFound" {
				// Set this to nil as if we didn't find anything.
				resp = nil
			} else {
				log.Printf("Error on StateRefresh: %s", err)
				return nil, "", err
			}
		}

		if resp == nil || len(resp.SpotInstanceRequests) == 0 {
			// Sometimes AWS just has consistency issues and doesn't see
			// our request yet. Return an empty state.
			return nil, "", nil
		}

		req := resp.SpotInstanceRequests[0]
		return req, *req.Status.Code, nil
	}
}
<commit_msg>minor tweaks to connection info setup<commit_after>package aws

import (
	"fmt"
	"log"
	"time"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/awserr"
	"github.com/aws/aws-sdk-go/service/ec2"
	"github.com/hashicorp/terraform/helper/resource"
	"github.com/hashicorp/terraform/helper/schema"
)

func resourceAwsSpotInstanceRequest() *schema.Resource {
	return &schema.Resource{
		Create: resourceAwsSpotInstanceRequestCreate,
		Read:   resourceAwsSpotInstanceRequestRead,
		Delete: resourceAwsSpotInstanceRequestDelete,
		Update: resourceAwsSpotInstanceRequestUpdate,

		Schema: func() map[string]*schema.Schema {
			// The Spot Instance Request Schema is based on the AWS Instance schema.
			s := resourceAwsInstance().Schema

			// Everything on a spot instance is ForceNew except tags
			for k, v := range s {
				if k == "tags" {
					continue
				}
				v.ForceNew = true
			}

			s["spot_price"] = &schema.Schema{
				Type:     schema.TypeString,
				Required: true,
				ForceNew: true,
			}
			s["spot_type"] = &schema.Schema{
				Type:     schema.TypeString,
				Optional: true,
				Default:  "persistent",
			}
			s["wait_for_fulfillment"] = &schema.Schema{
				Type:     schema.TypeBool,
				Optional: true,
				Default:  false,
			}
			s["spot_bid_status"] = &schema.Schema{
				Type:     schema.TypeString,
				Computed: true,
			}
			s["spot_request_state"] = &schema.Schema{
				Type:     schema.TypeString,
				Computed: true,
			}
			s["spot_instance_id"] = &schema.Schema{
				Type:     schema.TypeString,
				Computed: true,
			}

			return s
		}(),
	}
}

func resourceAwsSpotInstanceRequestCreate(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).ec2conn

	instanceOpts, err := buildAwsInstanceOpts(d, meta)
	if err != nil {
		return err
	}

	spotOpts := &ec2.RequestSpotInstancesInput{
		SpotPrice: aws.String(d.Get("spot_price").(string)),
		Type:      aws.String(d.Get("spot_type").(string)),

		// Though the AWS API supports creating spot instance requests for multiple
		// instances, for TF purposes we fix this to one instance per request.
		// Users can get equivalent behavior out of TF's "count" meta-parameter.
		InstanceCount: aws.Int64(1),

		LaunchSpecification: &ec2.RequestSpotLaunchSpecification{
			BlockDeviceMappings: instanceOpts.BlockDeviceMappings,
			EbsOptimized:        instanceOpts.EBSOptimized,
			Monitoring:          instanceOpts.Monitoring,
			IamInstanceProfile:  instanceOpts.IAMInstanceProfile,
			ImageId:             instanceOpts.ImageID,
			InstanceType:        instanceOpts.InstanceType,
			KeyName:             instanceOpts.KeyName,
			Placement:           instanceOpts.SpotPlacement,
			SecurityGroupIds:    instanceOpts.SecurityGroupIDs,
			SecurityGroups:      instanceOpts.SecurityGroups,
			SubnetId:            instanceOpts.SubnetID,
			UserData:            instanceOpts.UserData64,
		},
	}

	// If the instance is configured with a Network Interface (a subnet, has
	// public IP, etc), then the instanceOpts.SecurityGroupIds and SubnetId will
	// be nil
	if len(instanceOpts.NetworkInterfaces) > 0 {
		spotOpts.LaunchSpecification.SecurityGroupIds = instanceOpts.NetworkInterfaces[0].Groups
		spotOpts.LaunchSpecification.SubnetId = instanceOpts.NetworkInterfaces[0].SubnetId
	}

	// Make the spot instance request
	log.Printf("[DEBUG] Requesting spot bid opts: %s", spotOpts)
	resp, err := conn.RequestSpotInstances(spotOpts)
	if err != nil {
		return fmt.Errorf("Error requesting spot instances: %s", err)
	}
	if len(resp.SpotInstanceRequests) != 1 {
		return fmt.Errorf(
			"Expected response with length 1, got: %s", resp)
	}

	sir := *resp.SpotInstanceRequests[0]
	d.SetId(*sir.SpotInstanceRequestId)

	if d.Get("wait_for_fulfillment").(bool) {
		spotStateConf := &resource.StateChangeConf{
			// http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/spot-bid-status.html
			Pending:    []string{"start", "pending-evaluation", "pending-fulfillment"},
			Target:     "fulfilled",
			Refresh:    SpotInstanceStateRefreshFunc(conn, sir),
			Timeout:    10 * time.Minute,
			Delay:      10 * time.Second,
			MinTimeout: 3 * time.Second,
		}

		log.Printf("[DEBUG] waiting for spot bid to resolve... this may take several minutes.")
		_, err = spotStateConf.WaitForState()

		if err != nil {
			return fmt.Errorf("Error while waiting for spot request (%s) to resolve: %s", sir, err)
		}
	}

	return resourceAwsSpotInstanceRequestUpdate(d, meta)
}

// Update spot state, etc
func resourceAwsSpotInstanceRequestRead(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).ec2conn

	req := &ec2.DescribeSpotInstanceRequestsInput{
		SpotInstanceRequestIds: []*string{aws.String(d.Id())},
	}
	resp, err := conn.DescribeSpotInstanceRequests(req)

	if err != nil {
		// If the spot request was not found, return nil so that we can show
		// that it is gone.
		if ec2err, ok := err.(awserr.Error); ok && ec2err.Code() == "InvalidSpotInstanceRequestID.NotFound" {
			d.SetId("")
			return nil
		}

		// Some other error, report it
		return err
	}

	// If nothing was found, then return no state
	if len(resp.SpotInstanceRequests) == 0 {
		d.SetId("")
		return nil
	}

	request := resp.SpotInstanceRequests[0]

	// if the request is cancelled, then it is gone
	if *request.State == "cancelled" {
		d.SetId("")
		return nil
	}

	d.Set("spot_bid_status", *request.Status.Code)
	// Instance ID is not set if the request is still pending
	if request.InstanceId != nil {
		d.Set("spot_instance_id", *request.InstanceId)
		if err := readInstance(d, meta); err != nil {
			return fmt.Errorf("[ERR] Error reading Spot Instance Data: %s", err)
		}
	}
	d.Set("spot_request_state", *request.State)
	d.Set("tags", tagsToMap(request.Tags))

	// Read the instance data, setting up connection information
	return readInstance(d, meta)
}

func readInstance(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).ec2conn

	resp, err := conn.DescribeInstances(&ec2.DescribeInstancesInput{
		InstanceIds: []*string{aws.String(d.Get("spot_instance_id").(string))},
	})
	if err != nil {
		// If the instance was not found, return nil so that we can show
		// that the instance is gone.
		if ec2err, ok := err.(awserr.Error); ok && ec2err.Code() == "InvalidInstanceID.NotFound" {
			return fmt.Errorf("no instance found")
		}

		// Some other error, report it
		return err
	}

	// If nothing was found, then return no state
	if len(resp.Reservations) == 0 {
		return fmt.Errorf("no instances found")
	}

	instance := resp.Reservations[0].Instances[0]

	// Set these fields for connection information
	if instance != nil {
		d.Set("public_dns", instance.PublicDnsName)
		d.Set("public_ip", instance.PublicIpAddress)
		d.Set("private_dns", instance.PrivateDnsName)
		d.Set("private_ip", instance.PrivateIpAddress)

		// set connection information
		if instance.PublicIpAddress != nil {
			d.SetConnInfo(map[string]string{
				"type": "ssh",
				"host": *instance.PublicIpAddress,
			})
		} else if instance.PrivateIpAddress != nil {
			d.SetConnInfo(map[string]string{
				"type": "ssh",
				"host": *instance.PrivateIpAddress,
			})
		}
	}

	return nil
}

func resourceAwsSpotInstanceRequestUpdate(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).ec2conn

	d.Partial(true)
	if err := setTags(conn, d); err != nil {
		return err
	} else {
		d.SetPartial("tags")
	}

	d.Partial(false)

	return resourceAwsSpotInstanceRequestRead(d, meta)
}

func resourceAwsSpotInstanceRequestDelete(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).ec2conn

	log.Printf("[INFO] Cancelling spot request: %s", d.Id())
	_, err := conn.CancelSpotInstanceRequests(&ec2.CancelSpotInstanceRequestsInput{
		SpotInstanceRequestIds: []*string{aws.String(d.Id())},
	})

	if err != nil {
		return fmt.Errorf("Error cancelling spot request (%s): %s", d.Id(), err)
	}

	if instanceId := d.Get("spot_instance_id").(string); instanceId != "" {
		log.Printf("[INFO] Terminating instance: %s", instanceId)
		if err := awsTerminateInstance(conn, instanceId); err != nil {
			return fmt.Errorf("Error terminating spot instance: %s", err)
		}
	}

	return nil
}

// SpotInstanceStateRefreshFunc returns a resource.StateRefreshFunc that is used to watch
// an EC2 spot instance request
func SpotInstanceStateRefreshFunc(
	conn *ec2.EC2, sir ec2.SpotInstanceRequest) resource.StateRefreshFunc {

	return func() (interface{}, string, error) {
		resp, err := conn.DescribeSpotInstanceRequests(&ec2.DescribeSpotInstanceRequestsInput{
			SpotInstanceRequestIds: []*string{sir.SpotInstanceRequestId},
		})

		if err != nil {
			if ec2err, ok := err.(awserr.Error); ok && ec2err.Code() == "InvalidSpotInstanceRequestID.NotFound" {
				// Set this to nil as if we didn't find anything.
				resp = nil
			} else {
				log.Printf("Error on StateRefresh: %s", err)
				return nil, "", err
			}
		}

		if resp == nil || len(resp.SpotInstanceRequests) == 0 {
			// Sometimes AWS just has consistency issues and doesn't see
			// our request yet. Return an empty state.
			return nil, "", nil
		}

		req := resp.SpotInstanceRequests[0]
		return req, *req.Status.Code, nil
	}
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"flag"
	"fmt"
	"log"
	"net/http"
	"net/url"
	"runtime"
	"strings"
	"sync"
	"time"

	"github.com/PuerkitoBio/fetchbot"
	"github.com/PuerkitoBio/goquery"
)

var (
	// Protect access to dup
	mu sync.Mutex
	// Duplicates table
	dup = map[string]bool{}

	// Command-line flags
	seed      = flag.String("seed", "http://golang.org", "seed URL")
	stopAfter = flag.Duration("stopafter", 0, "automatically stop the fetchbot after a given time")
	stopAtUrl = flag.String("stopat", "", "automatically stop the fetchbot at a given URL")
	memStats  = flag.Duration("memstats", 0, "display memory statistics at a given interval")
)

func main() {
	flag.Parse()

	// Parse the provided seed
	u, err := url.Parse(*seed)
	if err != nil {
		log.Fatal(err)
	}

	// Create the muxer
	mux := fetchbot.NewMux()

	// Handle all errors the same
	mux.HandleErrors(fetchbot.HandlerFunc(func(ctx *fetchbot.Context, res *http.Response, err error) {
		fmt.Printf("[ERR] %s %s - %s\n", ctx.Cmd.Method(), ctx.Cmd.URL(), err)
	}))

	// Handle GET requests for html responses, to parse the body and enqueue all links as HEAD
	// requests.
	mux.Response().Method("GET").ContentType("text/html").Handler(fetchbot.HandlerFunc(
		func(ctx *fetchbot.Context, res *http.Response, err error) {
			// Process the body to find the links
			doc, err := goquery.NewDocumentFromResponse(res)
			if err != nil {
				fmt.Printf("[ERR] %s %s - %s\n", ctx.Cmd.Method(), ctx.Cmd.URL(), err)
				return
			}
			// Enqueue all links as HEAD requests
			enqueueLinks(ctx, doc)
		}))

	// Handle HEAD requests for html responses coming from the source host - we don't want
	// to crawl links from other hosts.
	mux.Response().Method("HEAD").Host(u.Host).ContentType("text/html").Handler(fetchbot.HandlerFunc(
		func(ctx *fetchbot.Context, res *http.Response, err error) {
			if _, err := ctx.Q.SendStringGet(ctx.Cmd.URL().String()); err != nil {
				fmt.Printf("[ERR] %s %s - %s\n", ctx.Cmd.Method(), ctx.Cmd.URL(), err)
			}
		}))

	// Create the Fetcher, handle the logging first, then dispatch to the Muxer
	h := logHandler(mux)
	if *stopAtUrl != "" {
		h = stopHandler(*stopAtUrl, logHandler(mux))
	}
	f := fetchbot.New(h)
	// First mem stat print must be right after creating the fetchbot
	if *memStats > 0 {
		// Print starting stats
		printMemStats(nil)
		// Run at regular intervals
		runMemStats(f, *memStats)
		// On exit, print ending stats after a GC
		defer func() {
			runtime.GC()
			printMemStats(nil)
		}()
	}
	// Start processing
	q := f.Start()
	if *stopAfter > 0 {
		go func() {
			c := time.After(*stopAfter)
			<-c
			q.Close()
		}()
	}
	// Enqueue the seed, which is the first entry in the dup map
	dup[*seed] = true
	_, err = q.SendStringGet(*seed)
	if err != nil {
		fmt.Printf("[ERR] GET %s - %s\n", *seed, err)
	}
	q.Block()
}

func runMemStats(f *fetchbot.Fetcher, tick time.Duration) {
	var mu sync.Mutex
	var di *fetchbot.DebugInfo

	// Start goroutine to collect fetchbot debug info
	go func() {
		for v := range f.Debug() {
			mu.Lock()
			di = v
			mu.Unlock()
		}
	}()
	// Start ticker goroutine to print mem stats at regular intervals
	go func() {
		c := time.Tick(tick)
		for _ = range c {
			mu.Lock()
			printMemStats(di)
			mu.Unlock()
		}
	}()
}

func printMemStats(di *fetchbot.DebugInfo) {
	var mem runtime.MemStats
	runtime.ReadMemStats(&mem)
	buf := bytes.NewBuffer(nil)
	buf.WriteString(strings.Repeat("=", 72) + "\n")
	buf.WriteString("Memory Profile:\n")
	buf.WriteString(fmt.Sprintf("\tAlloc: %d Kb\n", mem.Alloc/1024))
	buf.WriteString(fmt.Sprintf("\tTotalAlloc: %d Kb\n", mem.TotalAlloc/1024))
	buf.WriteString(fmt.Sprintf("\tNumGC: %d\n", mem.NumGC))
	buf.WriteString(fmt.Sprintf("\tGoroutines: %d\n", runtime.NumGoroutine()))
	if di != nil {
		buf.WriteString(fmt.Sprintf("\tNumHosts: %d\n", di.NumHosts))
	}
	buf.WriteString(strings.Repeat("=", 72))
	fmt.Println(buf.String())
}

// stopHandler stops the fetcher if the stopurl is reached. Otherwise it dispatches
// the call to the wrapped Handler.
func stopHandler(stopurl string, wrapped fetchbot.Handler) fetchbot.Handler {
	return fetchbot.HandlerFunc(func(ctx *fetchbot.Context, res *http.Response, err error) {
		if ctx.Cmd.URL().String() == stopurl {
			ctx.Q.Close()
			return
		}
		wrapped.Handle(ctx, res, err)
	})
}

// logHandler prints the fetch information and dispatches the call to the wrapped Handler.
func logHandler(wrapped fetchbot.Handler) fetchbot.Handler {
	return fetchbot.HandlerFunc(func(ctx *fetchbot.Context, res *http.Response, err error) {
		if err == nil {
			fmt.Printf("[%d] %s %s - %s\n", res.StatusCode, ctx.Cmd.Method(), ctx.Cmd.URL(), res.Header.Get("Content-Type"))
		}
		wrapped.Handle(ctx, res, err)
	})
}

func enqueueLinks(ctx *fetchbot.Context, doc *goquery.Document) {
	mu.Lock()
	doc.Find("a[href]").Each(func(i int, s *goquery.Selection) {
		val, _ := s.Attr("href")
		// Resolve address
		u, err := ctx.Cmd.URL().Parse(val)
		if err != nil {
			fmt.Printf("error: resolve URL %s - %s\n", val, err)
			return
		}
		if !dup[u.String()] {
			if _, err := ctx.Q.SendStringHead(u.String()); err != nil {
				fmt.Printf("error: enqueue head %s - %s\n", u, err)
			} else {
				dup[u.String()] = true
			}
		}
	})
	mu.Unlock()
}
<commit_msg>fix lint messages<commit_after>package main

import (
	"bytes"
	"flag"
	"fmt"
	"log"
	"net/http"
	"net/url"
	"runtime"
	"strings"
	"sync"
	"time"

	"github.com/PuerkitoBio/fetchbot"
	"github.com/PuerkitoBio/goquery"
)

var (
	// Protect access to dup
	mu sync.Mutex
	// Duplicates table
	dup = map[string]bool{}

	// Command-line flags
	seed      = flag.String("seed", "http://golang.org", "seed URL")
	stopAfter = flag.Duration("stopafter", 0, "automatically stop the fetchbot after a given time")
	stopAtURL = flag.String("stopat", "", "automatically stop the fetchbot at a given URL")
	memStats  = flag.Duration("memstats", 0, "display memory statistics at a given interval")
)

func main() {
	flag.Parse()

	// Parse the provided seed
	u, err := url.Parse(*seed)
	if err != nil {
		log.Fatal(err)
	}

	// Create the muxer
	mux := fetchbot.NewMux()

	// Handle all errors the same
	mux.HandleErrors(fetchbot.HandlerFunc(func(ctx *fetchbot.Context, res *http.Response, err error) {
		fmt.Printf("[ERR] %s %s - %s\n", ctx.Cmd.Method(), ctx.Cmd.URL(), err)
	}))

	// Handle GET requests for html responses, to parse the body and enqueue all links as HEAD
	// requests.
	mux.Response().Method("GET").ContentType("text/html").Handler(fetchbot.HandlerFunc(
		func(ctx *fetchbot.Context, res *http.Response, err error) {
			// Process the body to find the links
			doc, err := goquery.NewDocumentFromResponse(res)
			if err != nil {
				fmt.Printf("[ERR] %s %s - %s\n", ctx.Cmd.Method(), ctx.Cmd.URL(), err)
				return
			}
			// Enqueue all links as HEAD requests
			enqueueLinks(ctx, doc)
		}))

	// Handle HEAD requests for html responses coming from the source host - we don't want
	// to crawl links from other hosts.
	mux.Response().Method("HEAD").Host(u.Host).ContentType("text/html").Handler(fetchbot.HandlerFunc(
		func(ctx *fetchbot.Context, res *http.Response, err error) {
			if _, err := ctx.Q.SendStringGet(ctx.Cmd.URL().String()); err != nil {
				fmt.Printf("[ERR] %s %s - %s\n", ctx.Cmd.Method(), ctx.Cmd.URL(), err)
			}
		}))

	// Create the Fetcher, handle the logging first, then dispatch to the Muxer
	h := logHandler(mux)
	if *stopAtURL != "" {
		h = stopHandler(*stopAtURL, logHandler(mux))
	}
	f := fetchbot.New(h)
	// First mem stat print must be right after creating the fetchbot
	if *memStats > 0 {
		// Print starting stats
		printMemStats(nil)
		// Run at regular intervals
		runMemStats(f, *memStats)
		// On exit, print ending stats after a GC
		defer func() {
			runtime.GC()
			printMemStats(nil)
		}()
	}
	// Start processing
	q := f.Start()
	if *stopAfter > 0 {
		go func() {
			c := time.After(*stopAfter)
			<-c
			q.Close()
		}()
	}
	// Enqueue the seed, which is the first entry in the dup map
	dup[*seed] = true
	_, err = q.SendStringGet(*seed)
	if err != nil {
		fmt.Printf("[ERR] GET %s - %s\n", *seed, err)
	}
	q.Block()
}

func runMemStats(f *fetchbot.Fetcher, tick time.Duration) {
	var mu sync.Mutex
	var di *fetchbot.DebugInfo

	// Start goroutine to collect fetchbot debug info
	go func() {
		for v := range f.Debug() {
			mu.Lock()
			di = v
			mu.Unlock()
		}
	}()
	// Start ticker goroutine to print mem stats at regular intervals
	go func() {
		c := time.Tick(tick)
		for _ = range c {
			mu.Lock()
			printMemStats(di)
			mu.Unlock()
		}
	}()
}

func printMemStats(di *fetchbot.DebugInfo) {
	var mem runtime.MemStats
	runtime.ReadMemStats(&mem)
	buf := bytes.NewBuffer(nil)
	buf.WriteString(strings.Repeat("=", 72) + "\n")
	buf.WriteString("Memory Profile:\n")
	buf.WriteString(fmt.Sprintf("\tAlloc: %d Kb\n", mem.Alloc/1024))
	buf.WriteString(fmt.Sprintf("\tTotalAlloc: %d Kb\n", mem.TotalAlloc/1024))
	buf.WriteString(fmt.Sprintf("\tNumGC: %d\n", mem.NumGC))
	buf.WriteString(fmt.Sprintf("\tGoroutines: %d\n", runtime.NumGoroutine()))
	if di != nil {
		buf.WriteString(fmt.Sprintf("\tNumHosts: %d\n", di.NumHosts))
	}
	buf.WriteString(strings.Repeat("=", 72))
	fmt.Println(buf.String())
}

// stopHandler stops the fetcher if the stopurl is reached. Otherwise it dispatches
// the call to the wrapped Handler.
func stopHandler(stopurl string, wrapped fetchbot.Handler) fetchbot.Handler {
	return fetchbot.HandlerFunc(func(ctx *fetchbot.Context, res *http.Response, err error) {
		if ctx.Cmd.URL().String() == stopurl {
			ctx.Q.Close()
			return
		}
		wrapped.Handle(ctx, res, err)
	})
}

// logHandler prints the fetch information and dispatches the call to the wrapped Handler.
func logHandler(wrapped fetchbot.Handler) fetchbot.Handler {
	return fetchbot.HandlerFunc(func(ctx *fetchbot.Context, res *http.Response, err error) {
		if err == nil {
			fmt.Printf("[%d] %s %s - %s\n", res.StatusCode, ctx.Cmd.Method(), ctx.Cmd.URL(), res.Header.Get("Content-Type"))
		}
		wrapped.Handle(ctx, res, err)
	})
}

func enqueueLinks(ctx *fetchbot.Context, doc *goquery.Document) {
	mu.Lock()
	doc.Find("a[href]").Each(func(i int, s *goquery.Selection) {
		val, _ := s.Attr("href")
		// Resolve address
		u, err := ctx.Cmd.URL().Parse(val)
		if err != nil {
			fmt.Printf("error: resolve URL %s - %s\n", val, err)
			return
		}
		if !dup[u.String()] {
			if _, err := ctx.Q.SendStringHead(u.String()); err != nil {
				fmt.Printf("error: enqueue head %s - %s\n", u, err)
			} else {
				dup[u.String()] = true
			}
		}
	})
	mu.Unlock()
}
<|endoftext|>


<commit_before>package smtpapi

import (
	"bytes"
	"encoding/json"
	"fmt"
	"unicode/utf16"
)

// Version represents the current version of the smtpapi-go library
const Version = "0.6.1"

// SMTPAPIHeader will be used to set up X-SMTPAPI params
type SMTPAPIHeader struct {
	To          []string               `json:"to,omitempty"`
	Sub         map[string][]string    `json:"sub,omitempty"`
	Section     map[string]string      `json:"section,omitempty"`
	Category    []string               `json:"category,omitempty"`
	UniqueArgs  map[string]string      `json:"unique_args,omitempty"`
	Filters     map[string]Filter      `json:"filters,omitempty"`
	ASMGroupID  int                    `json:"asm_group_id,omitempty"`
	ASMGroups   []int                  `json:"asm_groups_to_display,omitempty"`
	SendAt      int64                  `json:"send_at,omitempty"`
	SendEachAt  []int64                `json:"send_each_at,omitempty"`
	IpPool      string                 `json:"ip_pool,omitempty"`
	BatchID     string                 `json:"batch_id,omitempty"`
	DynamicData map[string]interface{} `json:"dynamic_template_data,omitempty"`
}

// Filter represents an App/Filter and its settings
type Filter struct {
	Settings map[string]interface{} `json:"settings,omitempty"`
}

// NewSMTPAPIHeader creates a new header struct
func NewSMTPAPIHeader() *SMTPAPIHeader {
	return &SMTPAPIHeader{}
}

// AddTo appends a single email to the To header
func (h *SMTPAPIHeader) AddTo(email string) {
	h.To = append(h.To, email)
}

// AddTos appends multiple emails to the To header
func (h *SMTPAPIHeader) AddTos(emails []string) {
	for i := 0; i < len(emails); i++ {
		h.AddTo(emails[i])
	}
}

// SetTos sets the value of the To header
func (h *SMTPAPIHeader) SetTos(emails []string) {
	h.To = emails
}

// AddSubstitution adds a new substitution to a specific key
func (h *SMTPAPIHeader) AddSubstitution(key, sub string) {
	if h.Sub == nil {
		h.Sub = make(map[string][]string)
	}
	h.Sub[key] = append(h.Sub[key], sub)
}

// AddSubstitutions adds a multiple substitutions to a specific key
func (h *SMTPAPIHeader) AddSubstitutions(key string, subs []string) {
	for i := 0; i < len(subs); i++ {
		h.AddSubstitution(key, subs[i])
	}
}

// SetSubstitutions sets the value of the substitutions on the Sub header
func (h *SMTPAPIHeader) SetSubstitutions(sub map[string][]string) {
	h.Sub = sub
}

// AddSection sets the value for a specific section
func (h *SMTPAPIHeader) AddSection(section, value string) {
	if h.Section == nil {
		h.Section = make(map[string]string)
	}
	h.Section[section] = value
}

// SetSections sets the value for the Section header
func (h *SMTPAPIHeader) SetSections(sections map[string]string) {
	h.Section = sections
}

// AddCategory adds a new category to the Category header
func (h *SMTPAPIHeader) AddCategory(category string) {
	h.Category = append(h.Category, category)
}

// AddCategories adds multiple categories to the Category header
func (h *SMTPAPIHeader) AddCategories(categories []string) {
	for i := 0; i < len(categories); i++ {
		h.AddCategory(categories[i])
	}
}

// SetCategories will set the value of the Categories field
func (h *SMTPAPIHeader) SetCategories(categories []string) {
	h.Category = categories
}

// SetASMGroupID will set the value of the ASMGroupID field
func (h *SMTPAPIHeader) SetASMGroupID(groupID int) {
	h.ASMGroupID = groupID
}

// AddASMGroupToDisplay adds a new ASM group ID to be displayed
func (h *SMTPAPIHeader) AddASMGroupToDisplay(groupID int) {
	h.ASMGroups = append(h.ASMGroups, groupID)
}

// AddASMGroupsToDisplay adds multiple ASM group IDs to be displayed
func (h *SMTPAPIHeader) AddASMGroupsToDisplay(groupIDs []int) {
	for i := 0; i < len(groupIDs); i++ {
		h.AddASMGroupToDisplay(groupIDs[i])
	}
}

// SetASMGroupsToDisplay will set the value of the ASMGroups field
func (h *SMTPAPIHeader) SetASMGroupsToDisplay(groups []int) {
	h.ASMGroups = groups
}

// AddUniqueArg will set the value of a specific argument
func (h *SMTPAPIHeader) AddUniqueArg(arg, value string) {
	if h.UniqueArgs == nil {
		h.UniqueArgs = make(map[string]string)
	}
	h.UniqueArgs[arg] = value
}

// SetUniqueArgs will set the value of the Unique_args header
func (h *SMTPAPIHeader) SetUniqueArgs(args map[string]string) {
	h.UniqueArgs = args
}

// AddFilter will set the specific setting for a filter
func (h *SMTPAPIHeader) AddFilter(filter, setting string, value interface{}) {
	if h.Filters == nil {
		h.Filters = make(map[string]Filter)
	}
	if _, ok := h.Filters[filter]; !ok {
		h.Filters[filter] = Filter{
			Settings: make(map[string]interface{}),
		}
	}
	h.Filters[filter].Settings[setting] = value
}

// SetFilter takes in a Filter struct with predetermined settings and sets it for such Filter key
func (h *SMTPAPIHeader) SetFilter(filter string, value *Filter) {
	if h.Filters == nil {
		h.Filters = make(map[string]Filter)
	}
	h.Filters[filter] = *value
}

// SetSendAt takes in a timestamp which determines when the email will be sent
func (h *SMTPAPIHeader) SetSendAt(sendAt int64) {
	h.SendAt = sendAt
}

// AddSendEachAt takes in a timestamp and pushes it into a list Must match length of To emails
func (h *SMTPAPIHeader) AddSendEachAt(sendEachAt int64) {
	h.SendEachAt = append(h.SendEachAt, sendEachAt)
}

// SetSendEachAt takes an array of timestamps. Must match length of To emails
func (h *SMTPAPIHeader) SetSendEachAt(sendEachAt []int64) {
	h.SendEachAt = sendEachAt
}

// SetIpPool takes a strings and sets the IpPool field
func (h *SMTPAPIHeader) SetIpPool(ipPool string) {
	h.IpPool = ipPool
}

// Unicode escape
func escapeUnicode(input string) string {
	//var buffer bytes.Buffer
	buffer := bytes.NewBufferString("")
	for _, r := range input {
		if r > 65535 {
			// surrogate pair
			var r1, r2 = utf16.EncodeRune(r)
			var s = fmt.Sprintf("\\u%x\\u%x", r1, r2)
			// error always nil https://golang.org/pkg/bytes/#Buffer.WriteString
			buffer.WriteString(s) // nolint: gas, gosec
		} else if r > 127 {
			var s = fmt.Sprintf("\\u%04x", r)
			// error always nil https://golang.org/pkg/bytes/#Buffer.WriteString
			buffer.WriteString(s) // nolint: gas, gosec
		} else {
			var s = fmt.Sprintf("%c", r)
			// error always nil https://golang.org/pkg/bytes/#Buffer.WriteString
			buffer.WriteString(s) // nolint: gas, gosec
		}
	}
	return buffer.String()
}

// JSONString returns the representation of the Header
func (h *SMTPAPIHeader) JSONString() (string, error) {
	headers, e := json.Marshal(h)
	return escapeUnicode(string(headers)), e
}

// Load allows you to load a pre-formed x-smtpapi header
func (h *SMTPAPIHeader) Load(b []byte) error {
	return json.Unmarshal(b, h)
}
<commit_msg>Release 0.6.2<commit_after>package smtpapi

import (
	"bytes"
	"encoding/json"
	"fmt"
	"unicode/utf16"
)

// Version represents the current version of the smtpapi-go library
const Version = "0.6.2"

// SMTPAPIHeader will be used to set up X-SMTPAPI params
type SMTPAPIHeader struct {
	To          []string               `json:"to,omitempty"`
	Sub         map[string][]string    `json:"sub,omitempty"`
	Section     map[string]string      `json:"section,omitempty"`
	Category    []string               `json:"category,omitempty"`
	UniqueArgs  map[string]string      `json:"unique_args,omitempty"`
	Filters     map[string]Filter      `json:"filters,omitempty"`
	ASMGroupID  int                    `json:"asm_group_id,omitempty"`
	ASMGroups   []int                  `json:"asm_groups_to_display,omitempty"`
	SendAt      int64                  `json:"send_at,omitempty"`
	SendEachAt  []int64                `json:"send_each_at,omitempty"`
	IpPool      string                 `json:"ip_pool,omitempty"`
	BatchID     string                 `json:"batch_id,omitempty"`
	DynamicData map[string]interface{} `json:"dynamic_template_data,omitempty"`
}

// Filter represents an App/Filter and its settings
type Filter struct {
	Settings map[string]interface{} `json:"settings,omitempty"`
}

// NewSMTPAPIHeader creates a new header struct
func NewSMTPAPIHeader() *SMTPAPIHeader {
	return &SMTPAPIHeader{}
}

// AddTo appends a single email to the To header
func (h *SMTPAPIHeader) AddTo(email string) {
	h.To = append(h.To, email)
}

// AddTos appends multiple emails to the To header
func (h *SMTPAPIHeader) AddTos(emails []string) {
	for i := 0; i < len(emails); i++ {
		h.AddTo(emails[i])
	}
}

// SetTos sets the value of the To header
func (h *SMTPAPIHeader) SetTos(emails []string) {
	h.To = emails
}

// AddSubstitution adds a new substitution to a specific key
func (h *SMTPAPIHeader) AddSubstitution(key, sub string) {
	if h.Sub == nil {
		h.Sub = make(map[string][]string)
	}
	h.Sub[key] = append(h.Sub[key], sub)
}

// AddSubstitutions adds a multiple substitutions to a specific key
func (h *SMTPAPIHeader) AddSubstitutions(key string, subs []string) {
	for i := 0; i < len(subs); i++ {
		h.AddSubstitution(key, subs[i])
	}
}

// SetSubstitutions sets the value of the substitutions on the Sub header
func (h *SMTPAPIHeader) SetSubstitutions(sub map[string][]string) {
	h.Sub = sub
}

// AddSection sets the value for a specific section
func (h *SMTPAPIHeader) AddSection(section, value string) {
	if h.Section == nil {
		h.Section = make(map[string]string)
	}
	h.Section[section] = value
}

// SetSections sets the value for the Section header
func (h *SMTPAPIHeader) SetSections(sections map[string]string) {
	h.Section = sections
}

// AddCategory adds a new category to the Category header
func (h *SMTPAPIHeader) AddCategory(category string) {
	h.Category = append(h.Category, category)
}

// AddCategories adds multiple categories to the Category header
func (h *SMTPAPIHeader) AddCategories(categories []string) {
	for i := 0; i < len(categories); i++ {
		h.AddCategory(categories[i])
	}
}

// SetCategories will set the value of the Categories field
func (h *SMTPAPIHeader) SetCategories(categories []string) {
	h.Category = categories
}

// SetASMGroupID will set the value of the ASMGroupID field
func (h *SMTPAPIHeader) SetASMGroupID(groupID int) {
	h.ASMGroupID = groupID
}

// AddASMGroupToDisplay adds a new ASM group ID to be displayed
func (h *SMTPAPIHeader) AddASMGroupToDisplay(groupID int) {
	h.ASMGroups = append(h.ASMGroups, groupID)
}

// AddASMGroupsToDisplay adds multiple ASM group IDs to be displayed
func (h *SMTPAPIHeader) AddASMGroupsToDisplay(groupIDs []int) {
	for i := 0; i < len(groupIDs); i++ {
		h.AddASMGroupToDisplay(groupIDs[i])
	}
}

// SetASMGroupsToDisplay will set the value of the ASMGroups field
func (h *SMTPAPIHeader) SetASMGroupsToDisplay(groups []int) {
	h.ASMGroups = groups
}

// AddUniqueArg will set the value of a specific argument
func (h *SMTPAPIHeader) AddUniqueArg(arg, value string) {
	if h.UniqueArgs == nil {
		h.UniqueArgs = make(map[string]string)
	}
	h.UniqueArgs[arg] = value
}

// SetUniqueArgs will set the value of the Unique_args header
func (h *SMTPAPIHeader) SetUniqueArgs(args map[string]string) {
	h.UniqueArgs = args
}

// AddFilter will set the specific setting for a filter
func (h *SMTPAPIHeader) AddFilter(filter, setting string, value interface{}) {
	if h.Filters == nil {
		h.Filters = make(map[string]Filter)
	}
	if _, ok := h.Filters[filter]; !ok {
		h.Filters[filter] = Filter{
			Settings: make(map[string]interface{}),
		}
	}
	h.Filters[filter].Settings[setting] = value
}

// SetFilter takes in a Filter struct with predetermined settings and sets it for such Filter key
func (h *SMTPAPIHeader) SetFilter(filter string, value *Filter) {
	if h.Filters == nil {
		h.Filters = make(map[string]Filter)
	}
	h.Filters[filter] = *value
}

// SetSendAt takes in a timestamp which determines when the email will be sent
func (h *SMTPAPIHeader) SetSendAt(sendAt int64) {
	h.SendAt = sendAt
}

// AddSendEachAt takes in a timestamp and pushes it into a list Must match length of To emails
func (h *SMTPAPIHeader) AddSendEachAt(sendEachAt int64) {
	h.SendEachAt = append(h.SendEachAt, sendEachAt)
}

// SetSendEachAt takes an array of timestamps. Must match length of To emails
func (h *SMTPAPIHeader) SetSendEachAt(sendEachAt []int64) {
	h.SendEachAt = sendEachAt
}

// SetIpPool takes a strings and sets the IpPool field
func (h *SMTPAPIHeader) SetIpPool(ipPool string) {
	h.IpPool = ipPool
}

// Unicode escape
func escapeUnicode(input string) string {
	//var buffer bytes.Buffer
	buffer := bytes.NewBufferString("")
	for _, r := range input {
		if r > 65535 {
			// surrogate pair
			var r1, r2 = utf16.EncodeRune(r)
			var s = fmt.Sprintf("\\u%x\\u%x", r1, r2)
			// error always nil https://golang.org/pkg/bytes/#Buffer.WriteString
			buffer.WriteString(s) // nolint: gas, gosec
		} else if r > 127 {
			var s = fmt.Sprintf("\\u%04x", r)
			// error always nil https://golang.org/pkg/bytes/#Buffer.WriteString
			buffer.WriteString(s) // nolint: gas, gosec
		} else {
			var s = fmt.Sprintf("%c", r)
			// error always nil https://golang.org/pkg/bytes/#Buffer.WriteString
			buffer.WriteString(s) // nolint: gas, gosec
		}
	}
	return buffer.String()
}

// JSONString returns the representation of the Header
func (h *SMTPAPIHeader) JSONString() (string, error) {
	headers, e := json.Marshal(h)
	return escapeUnicode(string(headers)), e
}

// Load allows you to load a pre-formed x-smtpapi header
func (h *SMTPAPIHeader) Load(b []byte) error {
	return json.Unmarshal(b, h)
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	//"strings"
)

var cmdSobject = &Command{
	Run:   runSobject,
	Usage: "sobject",
	Short: "Manage sobjects",
	Long: `
Manage sobjects

Usage:

  force sobject list

  force sobject create <object> [<field>:<type>]...

  force sobject delete <object>

Examples:

  force sobject list

  force sobject create Todo Description:string

  force sobject delete Todo
`,
}

func runSobject(cmd *Command, args []string) {
	if len(args) == 0 {
		cmd.printUsage()
	} else {
		switch args[0] {
		case "list":
			runSobjectList(args[1:])
		case "create", "add":
			runSobjectCreate(args[1:])
		case "delete", "remove":
			runSobjectDelete(args[1:])
		default:
			ErrorAndExit("no such command: %s", args[0])
		}
	}
}

func runSobjectList(args []string) {
	force, _ := ActiveForce()
	sobjects, err := force.ListSobjects()
	if err != nil {
		ErrorAndExit(fmt.Sprintf("ERROR: %s\n", err))
	} else {
		DisplayForceSobjects(sobjects)
	}
}

func runSobjectCreate(args []string) {
	if len(args) < 2 {
		ErrorAndExit("must specify object and at least one field")
	}
	force, _ := ActiveForce()
	if err := force.Metadata.CreateCustomObject(args[0]); err != nil {
		ErrorAndExit(err.Error())
	}
	args[0] = fmt.Sprintf("%s__c", args[0]);

	runFieldCreate(args)
	fmt.Println("Custom object created")
}

func runSobjectDelete(args []string) {
	if len(args) < 1 {
		ErrorAndExit("must specify object")
	}
	force, _ := ActiveForce()
	if err := force.Metadata.DeleteCustomObject(args[0]); err != nil {
		ErrorAndExit(err.Error())
	}
	fmt.Println("Custom object deleted")
}
<commit_msg>clean up small typo<commit_after>package main

import (
	"fmt"
)

var cmdSobject = &Command{
	Run:   runSobject,
	Usage: "sobject",
	Short: "Manage sobjects",
	Long: `
Manage sobjects

Usage:

  force sobject list

  force sobject create <object> [<field>:<type>]...

  force sobject delete <object>

Examples:

  force sobject list

  force sobject create Todo Description:string

  force sobject delete Todo
`,
}

func runSobject(cmd *Command, args []string) {
	if len(args) == 0 {
		cmd.printUsage()
	} else {
		switch args[0] {
		case "list":
			runSobjectList(args[1:])
		case "create", "add":
			runSobjectCreate(args[1:])
		case "delete", "remove":
			runSobjectDelete(args[1:])
		default:
			ErrorAndExit("no such command: %s", args[0])
		}
	}
}

func runSobjectList(args []string) {
	force, _ := ActiveForce()
	sobjects, err := force.ListSobjects()
	if err != nil {
		ErrorAndExit(fmt.Sprintf("ERROR: %s\n", err))
	} else {
		DisplayForceSobjects(sobjects)
	}
}

func runSobjectCreate(args []string) {
	if len(args) < 2 {
		ErrorAndExit("must specify object and at least one field")
	}
	force, _ := ActiveForce()
	if err := force.Metadata.CreateCustomObject(args[0]); err != nil {
		ErrorAndExit(err.Error())
	}
	args[0] = fmt.Sprintf("%s__c", args[0]);

	runFieldCreate(args)
	fmt.Println("Custom object created")
}

func runSobjectDelete(args []string) {
	if len(args) < 1 {
		ErrorAndExit("must specify object")
	}
	force, _ := ActiveForce()
	if err := force.Metadata.DeleteCustomObject(args[0]); err != nil {
		ErrorAndExit(err.Error())
	}
	fmt.Println("Custom object deleted")
}
<|endoftext|>


<commit_before>package som

import (
	"fmt"
	"io"
	"math/rand"
	"runtime"
	"sync"
	"time"

	"github.com/gonum/matrix/mat64"
	"github.com/milosgajdos83/gosom/pkg/utils"
)

// CodebookInitFunc defines SOM codebook initialization function
type CodebookInitFunc func(*mat64.Dense, []int) (*mat64.Dense, error)

// CoordsInitFunc defines SOM grid coordinates initialization function
type CoordsInitFunc func(string, []int) (*mat64.Dense, error)

// NeighbFunc defines SOM neighbourhood function
type NeighbFunc func(float64, float64) float64

// Map is a Self Organizing Map (SOM)
type Map struct {
	// codebook is a matrix which contains SOM codebook vectors
	// codebook dimensions: SOM units x data features
	codebook *mat64.Dense
	// unitDist is a symmetric hollow matrix that maps distances between SOM units
	// unitDist dimesions: SOM units x SOM units
	unitDist *mat64.Dense
	// bmus stores codebook row indices of Best Match Units (BMU) over the training
	// bmus will give us an indication of how many clusters are there in the data
	bmus map[int]int
}

// NewMap creates new SOM based on the provided configuration and input data
// NewMap allows you to pass in SOM codebook init function that is used to initialize
// SOM codebook vectors to initial values. If codebook InitFunc is nil, random initialization
// is used. NewMap returns error if the provided configuration is not valid or if the data matrix
// is nil or if the codebook matrix could not be initialized.
func NewMap(c *MapConfig, data *mat64.Dense) (*Map, error) {
	// if input data is empty throw error
	if data == nil {
		return nil, fmt.Errorf("Invalid input data: %v\n", data)
	}
	// validate the map configuration
	if err := validateMapConfig(c); err != nil {
		return nil, err
	}
	// compute the number of map units
	mUnits := utils.IntProduct(c.Dims)
	if mUnits <= 1 {
		return nil, fmt.Errorf("Incorrect map size dimensions: %v\n", c.Dims)
	}
	// initialize codebook
	codebook, err := c.InitFunc(data, c.Dims)
	if err != nil {
		return nil, err
	}
	// grid coordinates matrix
	gridCoords, err := GridCoords(c.UShape, c.Dims)
	if err != nil {
		return nil, err
	}
	// unit distance matrix
	unitDist, err := DistanceMx("euclidean", gridCoords)
	if err != nil {
		return nil, err
	}
	bmus := make(map[int]int)
	// return pointer to new map
	return &Map{
		codebook: codebook,
		unitDist: unitDist,
		bmus:     bmus,
	}, nil
}

// Codebook returns a matrix which contains SOM codebook vectors
func (m Map) Codebook() *mat64.Dense {
	return m.codebook
}

// UnitDist returns a matrix which contains Euclidean distances between SOM units
func (m Map) UnitDist() *mat64.Dense {
	return m.unitDist
}

// BMUs returns a slice which contains indices of Best Match Units (BMUs) of each input vector
func (m Map) BMUs() map[int]int {
	return m.bmus
}

// MarshalTo serializes SOM codebook in a given format to writer w.
// At the moment only the native gonum binary format is supported.
// It returns the number of bytes written to w or fails with error.
func (m *Map) MarshalTo(format string, w io.Writer) (int, error) {
	switch format {
	case "gonum":
		return m.codebook.MarshalBinaryTo(w)
	}
	// marshal binary to file path
	return 0, fmt.Errorf("Unsupported format: %s\n", format)
}

// UMatrixOut generates SOM u-matrix in a given format and writes the output to w.
// At the moment only SVG format is supported. It fails with error if the write to w fails.
func (m Map) UMatrixOut(format, title string, w io.Writer) error {
	// TODO: needs some UMatrixSVG modifications first
	return nil
}

// Train runs a SOM training for a given data set and training configuration parameters.
// It modifies the map codebook vectors based on the chosen training algorithm.
// It returns error if the supplied training configuration is invalid or training fails
func (m *Map) Train(c *TrainConfig, data *mat64.Dense, iters int) error {
	// number of iterations must be a positive integer
	if iters <= 0 {
		return fmt.Errorf("Invalid number of iterations: %d\n", iters)
	}
	// nil data passed in
	if data == nil {
		return fmt.Errorf("Invalid data supplied: %v\n", data)
	}
	// validate the training configuration
	if err := validateTrainConfig(c); err != nil {
		return err
	}
	// run the training
	switch c.Method {
	case "seq":
		return m.seqTrain(c, data, iters)
	case "batch":
		return m.batchTrain(c, data, iters)
	}

	return nil
}

// seqTrain runs sequential SOM training algorithm on a given data set
func (m *Map) seqTrain(tc *TrainConfig, data *mat64.Dense, iters int) error {
	rows, _ := data.Dims()
	// create random number generator
	rSrc := rand.NewSource(time.Now().UnixNano())
	r := rand.New(rSrc)
	// retrieve Neighbourhood function
	neighbFn := Neighb[tc.NeighbFn]
	// perform iters number of learning iterations
	for i := 0; i < iters; i++ {
		// pick a random sample from dataset
		sample := data.RowView(r.Intn(rows))
		// no need to check for error here:
		// sample and codebook are not nil and have the same dimension
		bmu, _ := ClosestVec("euclidean", sample, m.codebook)
		// no need to check for errors:
		// LRate and Radius are checked by config validation
		lRate, _ := LRate(i, iters, tc.LDecay, tc.LRate)
		radius, _ := Radius(i, iters, tc.RDecay, tc.Radius)
		// pick the bmu unit distance row
		bmuDists := m.unitDist.RowView(bmu)
		// find units which are within the radius
		for i := 0; i < bmuDists.Len(); i++ {
			// bmu distance to i-th map unit
			dist := bmuDists.At(i, 0)
			// we are within BMU radius
			if dist < radius {
				// update particular codebook vector
				m.seqUpdateCbVec(i, sample, lRate, radius, dist, neighbFn)
			}
		}
	}

	return nil
}

// seqUpdateCbVec updates codebook vector on row cbIdx given the learning rate l,
// radius r, distance d and neihgbourhood function nFn
func (m *Map) seqUpdateCbVec(cbIdx int, vec *mat64.Vector, l, r, d float64, nFn NeighbFunc) {
	// pick codebook vector that should be updated
	cbVec := m.codebook.RowView(cbIdx)
	// update codebook vector according to the algorithm
	diff := mat64.NewVector(cbVec.Len(), nil)
	diff.AddScaledVec(vec, -1.0, cbVec)
	mul := l
	// nFn returns 1 for d == 0; skipping this case will save us some CPU time
	if d > 0.0 {
		mul *= nFn(d, r)
	}
	cbVec.AddScaledVec(cbVec, mul, diff)
}

// batchConfig holds batch training configuration
type batchConfig struct {
	// tc is a training configuration
	tc *TrainConfig
	// iters is a number of training iterations
	iters int
}

// batchRow holds data vectoar, ts position in data matri and current training iteration
type batchRow struct {
	// vec is an input data vector
	vec *mat64.Vector
	// idx is index of vec in data matrix
	idx int
	// iter is a current training iteration
	iter int
}

// batchResult holds scaled data vector, neighbourhood of data input and index of codebook vector
type batchResult struct {
	// vec is a scaled data vector
	vec *mat64.Vector
	// nghb is particular input neighbourhood
	nghb float64
	// idx is an index of codebook vector to be updated
	idx int
}

// batchTrain runs batch SOM training on a given data set
func (m *Map) batchTrain(tc *TrainConfig, data *mat64.Dense, iters int) error {
	cbRows, _ := m.codebook.Dims()
	rows, cols := data.Dims()
	// batchSize set to min(cbRows,rows)
	batchSize := cbRows
	if rows < cbRows {
		batchSize = rows
	}
	// number of batchIters in one learning iteration
	batchIters := rows / batchSize
	if rows%batchSize != 0 {
		batchIters++
	}
	// batchConfig: training config and number of iterations
	bc := &batchConfig{
		tc:    tc,
		iters: batchIters,
	}
	// number of worker goroutines
	workers := runtime.NumCPU()
	// train for a number of iterations
	for i := 0; i < iters; i++ {
		batchSamples := batchSize
		batchIter := 1
		// Iterate over the input data in batchSize batchIters
		for j := 0; j < rows; j += batchSize {
			// data and results channels are buffered
			rowChan := make(chan *batchRow, workers*4)
			resChan := make(chan *batchResult, workers*4)
			// adjust batchSize in case it goes over rows
			if j+batchSize > rows {
				batchSamples = rows - j
			}
			// batch matrix is a submatrix of data matrix
			batch := data.View(j, 0, batchSamples, cols)
			// goroutine which feeds worker goroutines
			go readDataRows(batch, batchIter, j, rowChan)
			// start worker goroutines
			wg := &sync.WaitGroup{}
			for j := 0; j < workers; j++ {
				wg.Add(1)
				go m.processRow(resChan, wg, bc, rowChan)
			}
			// wait for workers to finish and close the result channel
			go func() {
				wg.Wait()
				close(resChan)
			}()
			// collect batch results from all workers
			cbVecs := make([]*mat64.Vector, cbRows)
			nghbs := make([]float64, cbRows)
			for res := range resChan {
				if cbVecs[res.idx] != nil {
					cbVecs[res.idx].AddVec(cbVecs[res.idx], res.vec)
				} else {
					cbVecs[res.idx] = res.vec
				}
				nghbs[res.idx] += res.nghb
			}
			// update codebook vectors
			for i := 0; i < cbRows; i++ {
				if cbVecs[i] != nil {
					cbVecs[i].ScaleVec(1.0/nghbs[i], cbVecs[i])
					m.codebook.SetRow(i, cbVecs[i].RawVector().Data)
				}
			}
			// increment batch iteration
			batchIter++
		}
	}

	return nil
}

// processRow processes data rows and sends tehm down the results channel
func (m Map) processRow(res chan<- *batchResult, wg *sync.WaitGroup, bc *batchConfig,
	rows <-chan *batchRow) {
	// retrieve Neighbourhood function
	neighbFn := Neighb[bc.tc.NeighbFn]
	for row := range rows {
		// find codebook BMU for this data row
		bmu, _ := ClosestVec("euclidean", row.vec, m.codebook)
		// calculate radius for this iteration
		radius, _ := Radius(row.iter, bc.iters, bc.tc.RDecay, bc.tc.Radius)
		// pick the BMU's distance row
		bmuDists := m.unitDist.RowView(bmu)
		for i := 0; i < bmuDists.Len(); i++ {
			// bmu distance to i-th map unit
			dist := bmuDists.At(i, 0)
			// when in BMU radius, scale and add to all neighbourhood vecs
			if dist < radius {
				// calculate neighbourhood function
				nghb := neighbFn(dist, radius)
				// allocate new vector
				vec := new(mat64.Vector)
				vec.CloneVec(row.vec)
				vec.ScaleVec(nghb, vec)
				// send batchResult down results channel
				res <- &batchResult{vec: vec, nghb: nghb, idx: i}
			}
		}
	}
	wg.Done()
}

// readDataRows reads data rows and sends them down rowChan channel
func readDataRows(batch mat64.Matrix, iter, idx int, rowChan chan<- *batchRow) {
	// create batchIters of data
	rows, _ := batch.Dims()
	// iterate through all batch rows
	for i := 0; i < rows; i++ {
		rowChan <- &batchRow{
			iter: iter,
			vec:  (batch.(*mat64.Dense)).RowView(i),
			idx:  i + idx,
		}
	}
	// close channel when done
	close(rowChan)
}
<commit_msg>Refactored batch training algorithm<commit_after>package som

import (
	"fmt"
	"io"
	"math/rand"
	"runtime"
	"sync"
	"time"

	"github.com/gonum/matrix/mat64"
	"github.com/milosgajdos83/gosom/pkg/utils"
)

// CodebookInitFunc defines SOM codebook initialization function
type CodebookInitFunc func(*mat64.Dense, []int) (*mat64.Dense, error)

// CoordsInitFunc defines SOM grid coordinates initialization function
type CoordsInitFunc func(string, []int) (*mat64.Dense, error)

// NeighbFunc defines SOM neighbourhood function
type NeighbFunc func(float64, float64) float64

// Map is a Self Organizing Map (SOM)
type Map struct {
	// codebook is a matrix which contains SOM codebook vectors
	// codebook dimensions: SOM units x data features
	codebook *mat64.Dense
	// unitDist is a symmetric hollow matrix that maps distances between SOM units
	// unitDist dimesions: SOM units x SOM units
	unitDist *mat64.Dense
	// bmus stores codebook row indices of Best Match Units (BMU) over the training
	// bmus will give us an indication of how many clusters are there in the data
	bmus map[int]int
}

// NewMap creates new SOM based on the provided configuration and input data
// NewMap allows you to pass in SOM codebook init function that is used to initialize
// SOM codebook vectors to initial values. If codebook InitFunc is nil, random initialization
// is used. NewMap returns error if the provided configuration is not valid or if the data matrix
// is nil or if the codebook matrix could not be initialized.
func NewMap(c *MapConfig, data *mat64.Dense) (*Map, error) {
	// if input data is empty throw error
	if data == nil {
		return nil, fmt.Errorf("Invalid input data: %v\n", data)
	}
	// validate the map configuration
	if err := validateMapConfig(c); err != nil {
		return nil, err
	}
	// compute the number of map units
	mUnits := utils.IntProduct(c.Dims)
	if mUnits <= 1 {
		return nil, fmt.Errorf("Incorrect map size dimensions: %v\n", c.Dims)
	}
	// initialize codebook
	codebook, err := c.InitFunc(data, c.Dims)
	if err != nil {
		return nil, err
	}
	// grid coordinates matrix
	gridCoords, err := GridCoords(c.UShape, c.Dims)
	if err != nil {
		return nil, err
	}
	// unit distance matrix
	unitDist, err := DistanceMx("euclidean", gridCoords)
	if err != nil {
		return nil, err
	}
	bmus := make(map[int]int)
	// return pointer to new map
	return &Map{
		codebook: codebook,
		unitDist: unitDist,
		bmus:     bmus,
	}, nil
}

// Codebook returns a matrix which contains SOM codebook vectors
func (m Map) Codebook() *mat64.Dense {
	return m.codebook
}

// UnitDist returns a matrix which contains Euclidean distances between SOM units
func (m Map) UnitDist() *mat64.Dense {
	return m.unitDist
}

// BMUs returns a slice which contains indices of Best Match Units (BMUs) of each input vector
func (m Map) BMUs() map[int]int {
	return m.bmus
}

// MarshalTo serializes SOM codebook in a given format to writer w.
// At the moment only the native gonum binary format is supported.
// It returns the number of bytes written to w or fails with error.
func (m *Map) MarshalTo(format string, w io.Writer) (int, error) {
	switch format {
	case "gonum":
		return m.codebook.MarshalBinaryTo(w)
	}
	// marshal binary to file path
	return 0, fmt.Errorf("Unsupported format: %s\n", format)
}

// UMatrixOut generates SOM u-matrix in a given format and writes the output to w.
// At the moment only SVG format is supported. It fails with error if the write to w fails.
func (m Map) UMatrixOut(format, title string, w io.Writer) error {
	// TODO: needs some UMatrixSVG modifications first
	return nil
}

// Train runs a SOM training for a given data set and training configuration parameters.
// It modifies the map codebook vectors based on the chosen training algorithm.
// It returns error if the supplied training configuration is invalid or training fails
func (m *Map) Train(c *TrainConfig, data *mat64.Dense, iters int) error {
	// number of iterations must be a positive integer
	if iters <= 0 {
		return fmt.Errorf("Invalid number of iterations: %d\n", iters)
	}
	// nil data passed in
	if data == nil {
		return fmt.Errorf("Invalid data supplied: %v\n", data)
	}
	// validate the training configuration
	if err := validateTrainConfig(c); err != nil {
		return err
	}
	// run the training
	switch c.Method {
	case "seq":
		return m.seqTrain(c, data, iters)
	case "batch":
		return m.batchTrain(c, data, iters)
	}

	return nil
}

// seqTrain runs sequential SOM training algorithm on a given data set
func (m *Map) seqTrain(tc *TrainConfig, data *mat64.Dense, iters int) error {
	rows, _ := data.Dims()
	// create random number generator
	rSrc := rand.NewSource(time.Now().UnixNano())
	r := rand.New(rSrc)
	// retrieve Neighbourhood function
	neighbFn := Neighb[tc.NeighbFn]
	// perform iters number of learning iterations
	for i := 0; i < iters; i++ {
		// pick a random sample from dataset
		sample := data.RowView(r.Intn(rows))
		// no need to check for error here:
		// sample and codebook are not nil and have the same dimension
		bmu, _ := ClosestVec("euclidean", sample, m.codebook)
		// no need to check for errors:
		// LRate and Radius are checked by config validation
		lRate, _ := LRate(i, iters, tc.LDecay, tc.LRate)
		radius, _ := Radius(i, iters, tc.RDecay, tc.Radius)
		// pick the bmu unit distance row
		bmuDists := m.unitDist.RowView(bmu)
		// find units which are within the radius
		for i := 0; i < bmuDists.Len(); i++ {
			// bmu distance to i-th map unit
			dist := bmuDists.At(i, 0)
			// we are within BMU radius
			if dist < radius {
				// update particular codebook vector
				m.seqUpdateCbVec(i, sample, lRate, radius, dist, neighbFn)
			}
		}
	}

	return nil
}

// seqUpdateCbVec updates codebook vector on row cbIdx given the learning rate l,
// radius r, distance d and neihgbourhood function nFn
func (m *Map) seqUpdateCbVec(cbIdx int, vec *mat64.Vector, l, r, d float64, nFn NeighbFunc) {
	// pick codebook vector that should be updated
	cbVec := m.codebook.RowView(cbIdx)
	// update codebook vector according to the algorithm
	diff := mat64.NewVector(cbVec.Len(), nil)
	diff.AddScaledVec(vec, -1.0, cbVec)
	mul := l
	// nFn returns 1 for d == 0; skipping this case will save us some CPU time
	if d > 0.0 {
		mul *= nFn(d, r)
	}
	cbVec.AddScaledVec(cbVec, mul, diff)
}

// batchConfig holds batch training configuration
type batchConfig struct {
	// tc is a training configuration
	tc *TrainConfig
	// iters is a number of batch iterations
	iters int
}

// batchResult holds scaled data vector, neighbourhood of data input and index of codebook vector
type batchResult struct {
	// vec is a scaled data vector
	vec *mat64.Vector
	// nghb is vec BMU neighbourhood
	nghb float64
	// idx is an index of codebook vector to update
	idx int
}

// batchTrain runs batch SOM training on a given data set
func (m *Map) batchTrain(tc *TrainConfig, data *mat64.Dense, iters int) error {
	cbRows, _ := m.codebook.Dims()
	rows, _ := data.Dims()
	// bSize set to min(cbRows,rows)
	bSize := cbRows
	if rows < cbRows {
		bSize = rows
	}
	// number of batchIters in one learning iteration
	bIters := rows / bSize
	if rows%bSize != 0 {
		bIters++
	}
	// batchConfig: training config and number of iterations
	bc := &batchConfig{
		tc:    tc,
		iters: bIters,
	}
	// number of worker goroutines
	workers := runtime.NumCPU()
	// evenly distribute batch work between workers
	workerBatch := bSize / workers
	if workerBatch == 0 {
		workerBatch = 1
	}
	iter := 0
	// train for a number of iterations
	for i := 0; i < iters; i++ {
		count := workerBatch
		// Iterate over the input data in batches of size bSize
		for j := 0; j < rows; j += bSize {
			// create batch results channel
			results := make(chan *batchResult, workers*4)
			wg := &sync.WaitGroup{}
			// start worker goroutines
			for k := 0; k < workers; k++ {
				// from is data matrix row pointer
				from := j + k*workerBatch
				// last worker will work through the batch reminder
				if k == workers-1 {
					count += bSize % workers
				}
				// if we go over the number of rows adjust bSamples
				if from+count > rows {
					count = rows - from
				}
				wg.Add(1)
				go m.processBatch(results, wg, bc, data, from, count, iter)
			}
			// wait for workers to finish and close the result channel
			go func() {
				wg.Wait()
				close(results)
			}()
			// collect batch results from all workers
			cbVecs := make([]*mat64.Vector, cbRows)
			nghbs := make([]float64, cbRows)
			for result := range results {
				if cbVecs[result.idx] != nil {
					cbVecs[result.idx].AddVec(cbVecs[result.idx], result.vec)
				} else {
					cbVecs[result.idx] = result.vec
				}
				nghbs[result.idx] += result.nghb
			}
			// update codebook vectors
			for k := 0; k < cbRows; k++ {
				if cbVecs[k] != nil {
					cbVecs[k].ScaleVec(1.0/nghbs[k], cbVecs[k])
					m.codebook.SetRow(k, cbVecs[k].RawVector().Data)
				}
			}
			iter++
		}
	}

	return nil
}

// processRow processes data rows and sends tehm down the results channel
func (m Map) processBatch(res chan<- *batchResult, wg *sync.WaitGroup,
	bc *batchConfig, data *mat64.Dense, from, count, iter int) {
	// retrieve Neighbourhood function
	neighbFn := Neighb[bc.tc.NeighbFn]
	for i := from; i < count+from; i++ {
		row := data.RowView(i)
		// find codebook BMU for this data row
		bmu, _ := ClosestVec("euclidean", row, m.codebook)
		// calculate radius for this iteration
		radius, _ := Radius(iter, bc.iters, bc.tc.RDecay, bc.tc.Radius)
		// pick the BMU's distance row
		bmuDists := m.unitDist.RowView(bmu)
		for j := 0; j < bmuDists.Len(); j++ {
			// bmu distance to i-th map unit
			dist := bmuDists.At(j, 0)
			// when in BMU radius, scale and add to all neighbourhood vecs
			if dist < radius {
				// calculate neighbourhood function
				nghb := neighbFn(dist, radius)
				// allocate new vector and copy row data to it
				vec := new(mat64.Vector)
				vec.CloneVec(row)
				vec.ScaleVec(nghb, vec)
				// send batchResult down results channel
				res <- &batchResult{vec: vec, nghb: nghb, idx: j}
			}
		}
	}
	wg.Done()
}
<|endoftext|>


<commit_before>/* {{{ Copyright (c) Paul R. Tagliamonte <paultag@debian.org>, 2015
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to deal
 * in the Software without restriction, including without limitation the rights
 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 * copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 * THE SOFTWARE. }}} */

package archive

import (
	"io"
	"os"

	"pault.ag/go/debian/control"
	// "pault.ag/go/debian/deb"
	"pault.ag/go/debian/dependency"
	"pault.ag/go/debian/version"
)

// Source {{{

// The files dists/$DIST/$COMP/source/Sources are called Sources indices. They
// consist of multiple paragraphs, where each paragraph has the format defined
// in Policy 5.5 (5.4 Debian source control files -- .dsc), with the following
// changes and additional fields. The changes are:
//
//  - The "Source" field is renamed to "Package"
//  - A new mandatory field "Directory"
//  - A new optional field "Priority"
//  - A new optional field "Section"
//  - (Note that any fields present in .dsc files can end here as well, even if
//  - they are not documented by Debian policy, or not yet documented yet).
//
// Each paragraph shall begin with a "Package" field. Clients may also accept
// files where this is not the case.
type Source struct {
	control.Paragraph

	Package string

	Directory string `required:"true"`
	Priority  string
	Section   string

	Format           string
	Binaries         []string          `control:"Binary" delim:","`
	Architectures    []dependency.Arch `control:"Architecture"`
	Version          version.Version
	Origin           string
	Maintainer       string
	Uploaders        []string
	Homepage         string
	StandardsVersion string                `control:"Standards-Version"`
	BuildDepends     dependency.Dependency `control:"Build-Depends"`

	ChecksumsSha1   []control.SHA1DebianFileHash   `control:"Checksums-Sha1" delim:"\n" strip:"\n\r\t "`
	ChecksumsSha256 []control.SHA256DebianFileHash `control:"Checksums-Sha256" delim:"\n" strip:"\n\r\t "`
	Files           []control.FileListDSCFileHash  `control:"Files" delim:"\n" strip:"\n\r\t "`
}

// }}}

// Sources {{{

type Sources struct {
	decoder *control.Decoder
}

// Next {{{

// Get the next Source entry in the Sources list. This will return an
// io.EOF at the last entry.
func (p *Sources) Next() (*Source, error) {
	next := Source{}
	return &next, p.decoder.Decode(&next)
}

// }}}

// LoadSourcesFile {{{

// Given a path, create a Sources iterator. Note that the Sources
// file is not OpenPGP signed, so one will need to verify the integrety
// of this file from the InRelease file before trusting any output.
func LoadSourcesFile(path string) (*Sources, error) {
	fd, err := os.Open(path)
	if err != nil {
		return nil, err
	}
	defer fd.Close()
	return LoadSources(fd)
}

// }}}

// LoadSources {{{

// Given an io.Reader, create a Sources iterator. Note that the Sources
// file is not OpenPGP signed, so one will need to verify the integrety
// of this file from the InRelease file before trusting any output.
func LoadSources(in io.Reader) (*Sources, error) {
	decoder, err := control.NewDecoder(in, nil)
	if err != nil {
		return nil, err
	}
	return &Sources{decoder: decoder}, nil
}

// }}}

// }}}

// vim: foldmethod=marker
<commit_msg>remove defer<commit_after>/* {{{ Copyright (c) Paul R. Tagliamonte <paultag@debian.org>, 2015
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to deal
 * in the Software without restriction, including without limitation the rights
 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 * copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 * THE SOFTWARE. }}} */

package archive

import (
	"io"
	"os"

	"pault.ag/go/debian/control"
	// "pault.ag/go/debian/deb"
	"pault.ag/go/debian/dependency"
	"pault.ag/go/debian/version"
)

// Source {{{

// The files dists/$DIST/$COMP/source/Sources are called Sources indices. They
// consist of multiple paragraphs, where each paragraph has the format defined
// in Policy 5.5 (5.4 Debian source control files -- .dsc), with the following
// changes and additional fields. The changes are:
//
//  - The "Source" field is renamed to "Package"
//  - A new mandatory field "Directory"
//  - A new optional field "Priority"
//  - A new optional field "Section"
//  - (Note that any fields present in .dsc files can end here as well, even if
//  - they are not documented by Debian policy, or not yet documented yet).
//
// Each paragraph shall begin with a "Package" field. Clients may also accept
// files where this is not the case.
type Source struct {
	control.Paragraph

	Package string

	Directory string `required:"true"`
	Priority  string
	Section   string

	Format           string
	Binaries         []string          `control:"Binary" delim:","`
	Architectures    []dependency.Arch `control:"Architecture"`
	Version          version.Version
	Origin           string
	Maintainer       string
	Uploaders        []string
	Homepage         string
	StandardsVersion string                `control:"Standards-Version"`
	BuildDepends     dependency.Dependency `control:"Build-Depends"`

	ChecksumsSha1   []control.SHA1DebianFileHash   `control:"Checksums-Sha1" delim:"\n" strip:"\n\r\t "`
	ChecksumsSha256 []control.SHA256DebianFileHash `control:"Checksums-Sha256" delim:"\n" strip:"\n\r\t "`
	Files           []control.FileListDSCFileHash  `control:"Files" delim:"\n" strip:"\n\r\t "`
}

// }}}

// Sources {{{

type Sources struct {
	decoder *control.Decoder
}

// Next {{{

// Get the next Source entry in the Sources list. This will return an
// io.EOF at the last entry.
func (p *Sources) Next() (*Source, error) {
	next := Source{}
	return &next, p.decoder.Decode(&next)
}

// }}}

// LoadSourcesFile {{{

// Given a path, create a Sources iterator. Note that the Sources
// file is not OpenPGP signed, so one will need to verify the integrety
// of this file from the InRelease file before trusting any output.
func LoadSourcesFile(path string) (*Sources, error) {
	fd, err := os.Open(path)
	if err != nil {
		return nil, err
	}
	return LoadSources(fd)
}

// }}}

// LoadSources {{{

// Given an io.Reader, create a Sources iterator. Note that the Sources
// file is not OpenPGP signed, so one will need to verify the integrety
// of this file from the InRelease file before trusting any output.
func LoadSources(in io.Reader) (*Sources, error) {
	decoder, err := control.NewDecoder(in, nil)
	if err != nil {
		return nil, err
	}
	return &Sources{decoder: decoder}, nil
}

// }}}

// }}}

// vim: foldmethod=marker
<|endoftext|>