<commit_before>package digitalocean

import (
	"fmt"
	"testing"

	"github.com/digitalocean/godo"
	"github.com/hashicorp/terraform/helper/acctest"
	"github.com/hashicorp/terraform/helper/resource"
	"github.com/hashicorp/terraform/terraform"
)

func TestAccDigitalOceanVolume_Basic(t *testing.T) {
	name := fmt.Sprintf("volume-%s", acctest.RandString(10))

	volume := godo.Volume{
		Name: name,
	}

	resource.Test(t, resource.TestCase{
		PreCheck:     func() { testAccPreCheck(t) },
		Providers:    testAccProviders,
		CheckDestroy: testAccCheckDigitalOceanVolumeDestroy,
		Steps: []resource.TestStep{
			{
				Config: fmt.Sprintf(testAccCheckDigitalOceanVolumeConfig_basic, name),
				Check: resource.ComposeTestCheckFunc(
					testAccCheckDigitalOceanVolumeExists("digitalocean_volume.foobar", &volume),
					resource.TestCheckResourceAttr(
						"digitalocean_volume.foobar", "name", name),
					resource.TestCheckResourceAttr(
						"digitalocean_volume.foobar", "size", "100"),
					resource.TestCheckResourceAttr(
						"digitalocean_volume.foobar", "region", "nyc1"),
					resource.TestCheckResourceAttr(
						"digitalocean_volume.foobar", "description", "peace makes plenty"),
				),
			},
		},
	})
}

const testAccCheckDigitalOceanVolumeConfig_basic = `
resource "digitalocean_volume" "foobar" {
	region      = "nyc1"
	name        = "%s"
	size        = 100
	description = "peace makes plenty"
}`

func testAccCheckDigitalOceanVolumeExists(rn string, volume *godo.Volume) resource.TestCheckFunc {
	return func(s *terraform.State) error {
		rs, ok := s.RootModule().Resources[rn]
		if !ok {
			return fmt.Errorf("not found: %s", rn)
		}
		if rs.Primary.ID == "" {
			return fmt.Errorf("no volume ID is set")
		}

		client := testAccProvider.Meta().(*godo.Client)

		got, _, err := client.Storage.GetVolume(rs.Primary.ID)
		if err != nil {
			return err
		}
		if got.Name != volume.Name {
			return fmt.Errorf("wrong volume found, want %q got %q", volume.Name, got.Name)
		}
		// get the computed volume details
		*volume = *got
		return nil
	}
}

func testAccCheckDigitalOceanVolumeDestroy(s *terraform.State) error {
	client := testAccProvider.Meta().(*godo.Client)

	for _, rs := range s.RootModule().Resources {
		if rs.Type != "digitalocean_volume" {
			continue
		}

		// Try to find the volume
		_, _, err := client.Storage.GetVolume(rs.Primary.ID)

		if err == nil {
			return fmt.Errorf("Volume still exists")
		}
	}

	return nil
}

func TestAccDigitalOceanVolume_Droplet(t *testing.T) {
	var (
		volume  = godo.Volume{Name: fmt.Sprintf("volume-%s", acctest.RandString(10))}
		droplet godo.Droplet
	)
	rInt := acctest.RandInt()

	resource.Test(t, resource.TestCase{
		PreCheck:     func() { testAccPreCheck(t) },
		Providers:    testAccProviders,
		CheckDestroy: testAccCheckDigitalOceanVolumeDestroy,
		Steps: []resource.TestStep{
			{
				Config: fmt.Sprintf(
					testAccCheckDigitalOceanVolumeConfig_droplet(rInt, volume.Name),
					testAccValidPublicKey, volume.Name,
				),
				Check: resource.ComposeTestCheckFunc(
					testAccCheckDigitalOceanVolumeExists("digitalocean_volume.foobar", &volume),
					testAccCheckDigitalOceanDropletExists("digitalocean_droplet.foobar", &droplet),
					// the droplet should see an attached volume
					resource.TestCheckResourceAttr(
						"digitalocean_droplet.foobar", "volume_ids.#", "1"),
				),
			},
		},
	})
}

func testAccCheckDigitalOceanVolumeConfig_droplet(rInt int, vName string) string {
	return fmt.Sprintf(`
resource "digitalocean_volume" "foobar" {
	region      = "nyc1"
	name        = "%s"
	size        = 100
	description = "peace makes plenty"
}

resource "digitalocean_droplet" "foobar" {
  name               = "baz-%d"
  size               = "1gb"
  image              = "coreos-stable"
  region             = "nyc1"
  ipv6               = true
  private_networking = true
  ssh_keys           = ["${digitalocean_ssh_key.foobar.id}"]
  volume_ids         = ["${digitalocean_volume.foobar.id}"]
}`, vName, rInt)
}
<commit_msg>provider/digitalocean: Fix faililng acceptance test (#11887)<commit_after>package digitalocean

import (
	"fmt"
	"testing"

	"github.com/digitalocean/godo"
	"github.com/hashicorp/terraform/helper/acctest"
	"github.com/hashicorp/terraform/helper/resource"
	"github.com/hashicorp/terraform/terraform"
)

func TestAccDigitalOceanVolume_Basic(t *testing.T) {
	name := fmt.Sprintf("volume-%s", acctest.RandString(10))

	volume := godo.Volume{
		Name: name,
	}

	resource.Test(t, resource.TestCase{
		PreCheck:     func() { testAccPreCheck(t) },
		Providers:    testAccProviders,
		CheckDestroy: testAccCheckDigitalOceanVolumeDestroy,
		Steps: []resource.TestStep{
			{
				Config: fmt.Sprintf(testAccCheckDigitalOceanVolumeConfig_basic, name),
				Check: resource.ComposeTestCheckFunc(
					testAccCheckDigitalOceanVolumeExists("digitalocean_volume.foobar", &volume),
					resource.TestCheckResourceAttr(
						"digitalocean_volume.foobar", "name", name),
					resource.TestCheckResourceAttr(
						"digitalocean_volume.foobar", "size", "100"),
					resource.TestCheckResourceAttr(
						"digitalocean_volume.foobar", "region", "nyc1"),
					resource.TestCheckResourceAttr(
						"digitalocean_volume.foobar", "description", "peace makes plenty"),
				),
			},
		},
	})
}

const testAccCheckDigitalOceanVolumeConfig_basic = `
resource "digitalocean_volume" "foobar" {
	region      = "nyc1"
	name        = "%s"
	size        = 100
	description = "peace makes plenty"
}`

func testAccCheckDigitalOceanVolumeExists(rn string, volume *godo.Volume) resource.TestCheckFunc {
	return func(s *terraform.State) error {
		rs, ok := s.RootModule().Resources[rn]
		if !ok {
			return fmt.Errorf("not found: %s", rn)
		}
		if rs.Primary.ID == "" {
			return fmt.Errorf("no volume ID is set")
		}

		client := testAccProvider.Meta().(*godo.Client)

		got, _, err := client.Storage.GetVolume(rs.Primary.ID)
		if err != nil {
			return err
		}
		if got.Name != volume.Name {
			return fmt.Errorf("wrong volume found, want %q got %q", volume.Name, got.Name)
		}
		// get the computed volume details
		*volume = *got
		return nil
	}
}

func testAccCheckDigitalOceanVolumeDestroy(s *terraform.State) error {
	client := testAccProvider.Meta().(*godo.Client)

	for _, rs := range s.RootModule().Resources {
		if rs.Type != "digitalocean_volume" {
			continue
		}

		// Try to find the volume
		_, _, err := client.Storage.GetVolume(rs.Primary.ID)

		if err == nil {
			return fmt.Errorf("Volume still exists")
		}
	}

	return nil
}

func TestAccDigitalOceanVolume_Droplet(t *testing.T) {
	var (
		volume  = godo.Volume{Name: fmt.Sprintf("volume-%s", acctest.RandString(10))}
		droplet godo.Droplet
	)
	rInt := acctest.RandInt()

	resource.Test(t, resource.TestCase{
		PreCheck:     func() { testAccPreCheck(t) },
		Providers:    testAccProviders,
		CheckDestroy: testAccCheckDigitalOceanVolumeDestroy,
		Steps: []resource.TestStep{
			{
				Config: testAccCheckDigitalOceanVolumeConfig_droplet(rInt, volume.Name),
				Check: resource.ComposeTestCheckFunc(
					testAccCheckDigitalOceanVolumeExists("digitalocean_volume.foobar", &volume),
					testAccCheckDigitalOceanDropletExists("digitalocean_droplet.foobar", &droplet),
					// the droplet should see an attached volume
					resource.TestCheckResourceAttr(
						"digitalocean_droplet.foobar", "volume_ids.#", "1"),
				),
			},
		},
	})
}

func testAccCheckDigitalOceanVolumeConfig_droplet(rInt int, vName string) string {
	return fmt.Sprintf(`
resource "digitalocean_volume" "foobar" {
	region      = "nyc1"
	name        = "%s"
	size        = 100
	description = "peace makes plenty"
}

resource "digitalocean_droplet" "foobar" {
  name               = "baz-%d"
  size               = "1gb"
  image              = "centos-7-x64"
  region             = "nyc1"
  ipv6               = true
  private_networking = true
  volume_ids         = ["${digitalocean_volume.foobar.id}"]
}`, vName, rInt)
}
<|endoftext|>


<commit_before>/*
Copyright 2014 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package wait

import (
	"errors"
	"math/rand"
	"time"

	"k8s.io/apimachinery/pkg/util/runtime"
)

// For any test of the style:
//   ...
//   <- time.After(timeout):
//      t.Errorf("Timed out")
// The value for timeout should effectively be "forever." Obviously we don't want our tests to truly lock up forever, but 30s
// is long enough that it is effectively forever for the things that can slow down a run on a heavily contended machine
// (GC, seeks, etc), but not so long as to make a developer ctrl-c a test run if they do happen to break that test.
var ForeverTestTimeout = time.Second * 30

// NeverStop may be passed to Until to make it never stop.
var NeverStop <-chan struct{} = make(chan struct{})

// Forever calls f every period for ever.
//
// Forever is syntactic sugar on top of Until.
func Forever(f func(), period time.Duration) {
	Until(f, period, NeverStop)
}

// Until loops until stop channel is closed, running f every period.
//
// Until is syntactic sugar on top of JitterUntil with zero jitter factor and
// with sliding = true (which means the timer for period starts after the f
// completes).
func Until(f func(), period time.Duration, stopCh <-chan struct{}) {
	JitterUntil(f, period, 0.0, true, stopCh)
}

// NonSlidingUntil loops until stop channel is closed, running f every
// period.
//
// NonSlidingUntil is syntactic sugar on top of JitterUntil with zero jitter
// factor, with sliding = false (meaning the timer for period starts at the same
// time as the function starts).
func NonSlidingUntil(f func(), period time.Duration, stopCh <-chan struct{}) {
	JitterUntil(f, period, 0.0, false, stopCh)
}

// JitterUntil loops until stop channel is closed, running f every period.
//
// If jitterFactor is positive, the period is jittered before every run of f.
// If jitterFactor is not positive, the period is unchanged and not jitterd.
//
// If slidingis true, the period is computed after f runs. If it is false then
// period includes the runtime for f.
//
// Close stopCh to stop. f may not be invoked if stop channel is already
// closed. Pass NeverStop to if you don't want it stop.
func JitterUntil(f func(), period time.Duration, jitterFactor float64, sliding bool, stopCh <-chan struct{}) {
	for {

		select {
		case <-stopCh:
			return
		default:
		}

		jitteredPeriod := period
		if jitterFactor > 0.0 {
			jitteredPeriod = Jitter(period, jitterFactor)
		}

		var t *time.Timer
		if !sliding {
			t = time.NewTimer(jitteredPeriod)
		}

		func() {
			defer runtime.HandleCrash()
			f()
		}()

		if sliding {
			t = time.NewTimer(jitteredPeriod)
		}

		// NOTE: b/c there is no priority selection in golang
		// it is possible for this to race, meaning we could
		// trigger t.C and stopCh, and t.C select falls through.
		// In order to mitigate we re-check stopCh at the beginning
		// of every loop to prevent extra executions of f().
		select {
		case <-stopCh:
			return
		case <-t.C:
		}
	}
}

// Jitter returns a time.Duration between duration and duration + maxFactor *
// duration.
//
// This allows clients to avoid converging on periodic behavior. If maxFactor
// is 0.0, a suggested default value will be chosen.
func Jitter(duration time.Duration, maxFactor float64) time.Duration {
	if maxFactor <= 0.0 {
		maxFactor = 1.0
	}
	wait := duration + time.Duration(rand.Float64()*maxFactor*float64(duration))
	return wait
}

// ErrWaitTimeout is returned when the condition exited without success.
var ErrWaitTimeout = errors.New("timed out waiting for the condition")

// ConditionFunc returns true if the condition is satisfied, or an error
// if the loop should be aborted.
type ConditionFunc func() (done bool, err error)

// Backoff holds parameters applied to a Backoff function.
type Backoff struct {
	Duration time.Duration // the base duration
	Factor   float64       // Duration is multipled by factor each iteration
	Jitter   float64       // The amount of jitter applied each iteration
	Steps    int           // Exit with error after this many steps
}

// ExponentialBackoff repeats a condition check with exponential backoff.
//
// It checks the condition up to Steps times, increasing the wait by multipling
// the previous duration by Factor.
//
// If Jitter is greater than zero, a random amount of each duration is added
// (between duration and duration*(1+jitter)).
//
// If the condition never returns true, ErrWaitTimeout is returned. All other
// errors terminate immediately.
func ExponentialBackoff(backoff Backoff, condition ConditionFunc) error {
	duration := backoff.Duration
	for i := 0; i < backoff.Steps; i++ {
		if i != 0 {
			adjusted := duration
			if backoff.Jitter > 0.0 {
				adjusted = Jitter(duration, backoff.Jitter)
			}
			time.Sleep(adjusted)
			duration = time.Duration(float64(duration) * backoff.Factor)
		}
		if ok, err := condition(); err != nil || ok {
			return err
		}
	}
	return ErrWaitTimeout
}

// Poll tries a condition func until it returns true, an error, or the timeout
// is reached.
//
// Poll always waits the interval before the run of 'condition'.
// 'condition' will always be invoked at least once.
//
// Some intervals may be missed if the condition takes too long or the time
// window is too short.
//
// If you want to Poll something forever, see PollInfinite.
func Poll(interval, timeout time.Duration, condition ConditionFunc) error {
	return pollInternal(poller(interval, timeout), condition)
}

func pollInternal(wait WaitFunc, condition ConditionFunc) error {
	return WaitFor(wait, condition, NeverStop)
}

// PollImmediate tries a condition func until it returns true, an error, or the timeout
// is reached.
//
// Poll always checks 'condition' before waiting for the interval. 'condition'
// will always be invoked at least once.
//
// Some intervals may be missed if the condition takes too long or the time
// window is too short.
//
// If you want to Poll something forever, see PollInfinite.
func PollImmediate(interval, timeout time.Duration, condition ConditionFunc) error {
	return pollImmediateInternal(poller(interval, timeout), condition)
}

func pollImmediateInternal(wait WaitFunc, condition ConditionFunc) error {
	done, err := condition()
	if err != nil {
		return err
	}
	if done {
		return nil
	}
	return pollInternal(wait, condition)
}

// PollInfinite tries a condition func until it returns true or an error
//
// PollInfinite always waits the interval before the run of 'condition'.
//
// Some intervals may be missed if the condition takes too long or the time
// window is too short.
func PollInfinite(interval time.Duration, condition ConditionFunc) error {
	done := make(chan struct{})
	defer close(done)
	return PollUntil(interval, condition, done)
}

// PollImmediateInfinite tries a condition func until it returns true or an error
//
// PollImmediateInfinite runs the 'condition' before waiting for the interval.
//
// Some intervals may be missed if the condition takes too long or the time
// window is too short.
func PollImmediateInfinite(interval time.Duration, condition ConditionFunc) error {
	done, err := condition()
	if err != nil {
		return err
	}
	if done {
		return nil
	}
	return PollInfinite(interval, condition)
}

// PollUntil tries a condition func until it returns true, an error or stopCh is
// closed.
//
// PolUntil always waits interval before the first run of 'condition'.
// 'condition' will always be invoked at least once.
func PollUntil(interval time.Duration, condition ConditionFunc, stopCh <-chan struct{}) error {
	return WaitFor(poller(interval, 0), condition, stopCh)
}

// WaitFunc creates a channel that receives an item every time a test
// should be executed and is closed when the last test should be invoked.
type WaitFunc func(done <-chan struct{}) <-chan struct{}

// WaitFor continually checks 'fn' as driven by 'wait'.
//
// WaitFor gets a channel from 'wait()'', and then invokes 'fn' once for every value
// placed on the channel and once more when the channel is closed.
//
// If 'fn' returns an error the loop ends and that error is returned, and if
// 'fn' returns true the loop ends and nil is returned.
//
// ErrWaitTimeout will be returned if the channel is closed without fn ever
// returning true.
func WaitFor(wait WaitFunc, fn ConditionFunc, done <-chan struct{}) error {
	c := wait(done)
	for {
		_, open := <-c
		ok, err := fn()
		if err != nil {
			return err
		}
		if ok {
			return nil
		}
		if !open {
			break
		}
	}
	return ErrWaitTimeout
}

// poller returns a WaitFunc that will send to the channel every interval until
// timeout has elapsed and then closes the channel.
//
// Over very short intervals you may receive no ticks before the channel is
// closed. A timeout of 0 is interpreted as an infinity.
//
// Output ticks are not buffered. If the channel is not ready to receive an
// item, the tick is skipped.
func poller(interval, timeout time.Duration) WaitFunc {
	return WaitFunc(func(done <-chan struct{}) <-chan struct{} {
		ch := make(chan struct{})

		go func() {
			defer close(ch)

			tick := time.NewTicker(interval)
			defer tick.Stop()

			var after <-chan time.Time
			if timeout != 0 {
				// time.After is more convenient, but it
				// potentially leaves timers around much longer
				// than necessary if we exit early.
				timer := time.NewTimer(timeout)
				after = timer.C
				defer timer.Stop()
			}

			for {
				select {
				case <-tick.C:
					// If the consumer isn't ready for this signal drop it and
					// check the other channels.
					select {
					case ch <- struct{}{}:
					default:
					}
				case <-after:
					return
				case <-done:
					return
				}
			}
		}()

		return ch
	})
}
<commit_msg>JitterUntil should reuse Timer instead of allocating<commit_after>/*
Copyright 2014 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package wait

import (
	"errors"
	"math/rand"
	"time"

	"k8s.io/apimachinery/pkg/util/runtime"
)

// For any test of the style:
//   ...
//   <- time.After(timeout):
//      t.Errorf("Timed out")
// The value for timeout should effectively be "forever." Obviously we don't want our tests to truly lock up forever, but 30s
// is long enough that it is effectively forever for the things that can slow down a run on a heavily contended machine
// (GC, seeks, etc), but not so long as to make a developer ctrl-c a test run if they do happen to break that test.
var ForeverTestTimeout = time.Second * 30

// NeverStop may be passed to Until to make it never stop.
var NeverStop <-chan struct{} = make(chan struct{})

// Forever calls f every period for ever.
//
// Forever is syntactic sugar on top of Until.
func Forever(f func(), period time.Duration) {
	Until(f, period, NeverStop)
}

// Until loops until stop channel is closed, running f every period.
//
// Until is syntactic sugar on top of JitterUntil with zero jitter factor and
// with sliding = true (which means the timer for period starts after the f
// completes).
func Until(f func(), period time.Duration, stopCh <-chan struct{}) {
	JitterUntil(f, period, 0.0, true, stopCh)
}

// NonSlidingUntil loops until stop channel is closed, running f every
// period.
//
// NonSlidingUntil is syntactic sugar on top of JitterUntil with zero jitter
// factor, with sliding = false (meaning the timer for period starts at the same
// time as the function starts).
func NonSlidingUntil(f func(), period time.Duration, stopCh <-chan struct{}) {
	JitterUntil(f, period, 0.0, false, stopCh)
}

// JitterUntil loops until stop channel is closed, running f every period.
//
// If jitterFactor is positive, the period is jittered before every run of f.
// If jitterFactor is not positive, the period is unchanged and not jitterd.
//
// If slidingis true, the period is computed after f runs. If it is false then
// period includes the runtime for f.
//
// Close stopCh to stop. f may not be invoked if stop channel is already
// closed. Pass NeverStop to if you don't want it stop.
func JitterUntil(f func(), period time.Duration, jitterFactor float64, sliding bool, stopCh <-chan struct{}) {
	var t *time.Timer
	var sawTimeout bool

	for {
		select {
		case <-stopCh:
			return
		default:
		}

		jitteredPeriod := period
		if jitterFactor > 0.0 {
			jitteredPeriod = Jitter(period, jitterFactor)
		}

		if !sliding {
			t = resetOrReuseTimer(t, jitteredPeriod, sawTimeout)
		}

		func() {
			defer runtime.HandleCrash()
			f()
		}()

		if sliding {
			t = resetOrReuseTimer(t, jitteredPeriod, sawTimeout)
		}

		// NOTE: b/c there is no priority selection in golang
		// it is possible for this to race, meaning we could
		// trigger t.C and stopCh, and t.C select falls through.
		// In order to mitigate we re-check stopCh at the beginning
		// of every loop to prevent extra executions of f().
		select {
		case <-stopCh:
			return
		case <-t.C:
			sawTimeout = true
		}
	}
}

// Jitter returns a time.Duration between duration and duration + maxFactor *
// duration.
//
// This allows clients to avoid converging on periodic behavior. If maxFactor
// is 0.0, a suggested default value will be chosen.
func Jitter(duration time.Duration, maxFactor float64) time.Duration {
	if maxFactor <= 0.0 {
		maxFactor = 1.0
	}
	wait := duration + time.Duration(rand.Float64()*maxFactor*float64(duration))
	return wait
}

// ErrWaitTimeout is returned when the condition exited without success.
var ErrWaitTimeout = errors.New("timed out waiting for the condition")

// ConditionFunc returns true if the condition is satisfied, or an error
// if the loop should be aborted.
type ConditionFunc func() (done bool, err error)

// Backoff holds parameters applied to a Backoff function.
type Backoff struct {
	Duration time.Duration // the base duration
	Factor   float64       // Duration is multipled by factor each iteration
	Jitter   float64       // The amount of jitter applied each iteration
	Steps    int           // Exit with error after this many steps
}

// ExponentialBackoff repeats a condition check with exponential backoff.
//
// It checks the condition up to Steps times, increasing the wait by multipling
// the previous duration by Factor.
//
// If Jitter is greater than zero, a random amount of each duration is added
// (between duration and duration*(1+jitter)).
//
// If the condition never returns true, ErrWaitTimeout is returned. All other
// errors terminate immediately.
func ExponentialBackoff(backoff Backoff, condition ConditionFunc) error {
	duration := backoff.Duration
	for i := 0; i < backoff.Steps; i++ {
		if i != 0 {
			adjusted := duration
			if backoff.Jitter > 0.0 {
				adjusted = Jitter(duration, backoff.Jitter)
			}
			time.Sleep(adjusted)
			duration = time.Duration(float64(duration) * backoff.Factor)
		}
		if ok, err := condition(); err != nil || ok {
			return err
		}
	}
	return ErrWaitTimeout
}

// Poll tries a condition func until it returns true, an error, or the timeout
// is reached.
//
// Poll always waits the interval before the run of 'condition'.
// 'condition' will always be invoked at least once.
//
// Some intervals may be missed if the condition takes too long or the time
// window is too short.
//
// If you want to Poll something forever, see PollInfinite.
func Poll(interval, timeout time.Duration, condition ConditionFunc) error {
	return pollInternal(poller(interval, timeout), condition)
}

func pollInternal(wait WaitFunc, condition ConditionFunc) error {
	return WaitFor(wait, condition, NeverStop)
}

// PollImmediate tries a condition func until it returns true, an error, or the timeout
// is reached.
//
// Poll always checks 'condition' before waiting for the interval. 'condition'
// will always be invoked at least once.
//
// Some intervals may be missed if the condition takes too long or the time
// window is too short.
//
// If you want to Poll something forever, see PollInfinite.
func PollImmediate(interval, timeout time.Duration, condition ConditionFunc) error {
	return pollImmediateInternal(poller(interval, timeout), condition)
}

func pollImmediateInternal(wait WaitFunc, condition ConditionFunc) error {
	done, err := condition()
	if err != nil {
		return err
	}
	if done {
		return nil
	}
	return pollInternal(wait, condition)
}

// PollInfinite tries a condition func until it returns true or an error
//
// PollInfinite always waits the interval before the run of 'condition'.
//
// Some intervals may be missed if the condition takes too long or the time
// window is too short.
func PollInfinite(interval time.Duration, condition ConditionFunc) error {
	done := make(chan struct{})
	defer close(done)
	return PollUntil(interval, condition, done)
}

// PollImmediateInfinite tries a condition func until it returns true or an error
//
// PollImmediateInfinite runs the 'condition' before waiting for the interval.
//
// Some intervals may be missed if the condition takes too long or the time
// window is too short.
func PollImmediateInfinite(interval time.Duration, condition ConditionFunc) error {
	done, err := condition()
	if err != nil {
		return err
	}
	if done {
		return nil
	}
	return PollInfinite(interval, condition)
}

// PollUntil tries a condition func until it returns true, an error or stopCh is
// closed.
//
// PolUntil always waits interval before the first run of 'condition'.
// 'condition' will always be invoked at least once.
func PollUntil(interval time.Duration, condition ConditionFunc, stopCh <-chan struct{}) error {
	return WaitFor(poller(interval, 0), condition, stopCh)
}

// WaitFunc creates a channel that receives an item every time a test
// should be executed and is closed when the last test should be invoked.
type WaitFunc func(done <-chan struct{}) <-chan struct{}

// WaitFor continually checks 'fn' as driven by 'wait'.
//
// WaitFor gets a channel from 'wait()'', and then invokes 'fn' once for every value
// placed on the channel and once more when the channel is closed.
//
// If 'fn' returns an error the loop ends and that error is returned, and if
// 'fn' returns true the loop ends and nil is returned.
//
// ErrWaitTimeout will be returned if the channel is closed without fn ever
// returning true.
func WaitFor(wait WaitFunc, fn ConditionFunc, done <-chan struct{}) error {
	c := wait(done)
	for {
		_, open := <-c
		ok, err := fn()
		if err != nil {
			return err
		}
		if ok {
			return nil
		}
		if !open {
			break
		}
	}
	return ErrWaitTimeout
}

// poller returns a WaitFunc that will send to the channel every interval until
// timeout has elapsed and then closes the channel.
//
// Over very short intervals you may receive no ticks before the channel is
// closed. A timeout of 0 is interpreted as an infinity.
//
// Output ticks are not buffered. If the channel is not ready to receive an
// item, the tick is skipped.
func poller(interval, timeout time.Duration) WaitFunc {
	return WaitFunc(func(done <-chan struct{}) <-chan struct{} {
		ch := make(chan struct{})

		go func() {
			defer close(ch)

			tick := time.NewTicker(interval)
			defer tick.Stop()

			var after <-chan time.Time
			if timeout != 0 {
				// time.After is more convenient, but it
				// potentially leaves timers around much longer
				// than necessary if we exit early.
				timer := time.NewTimer(timeout)
				after = timer.C
				defer timer.Stop()
			}

			for {
				select {
				case <-tick.C:
					// If the consumer isn't ready for this signal drop it and
					// check the other channels.
					select {
					case ch <- struct{}{}:
					default:
					}
				case <-after:
					return
				case <-done:
					return
				}
			}
		}()

		return ch
	})
}

// resetOrReuseTimer avoids allocating a new timer if one is already in use.
// Not safe for multiple threads.
func resetOrReuseTimer(t *time.Timer, d time.Duration, sawTimeout bool) *time.Timer {
	if t == nil {
		return time.NewTimer(d)
	}
	if !t.Stop() && !sawTimeout {
		<-t.C
	}
	t.Reset(d)
	return t
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"log"
	"os"
	"strings"
	"io/ioutil"
	"encoding/base64"

	"github.com/sendgrid/sendgrid-go"
	"github.com/sendgrid/sendgrid-go/helpers/mail"
	"github.com/joho/godotenv"
)

func main() {

	err_read := godotenv.Load()
	if err_read != nil {
	  log.Fatalf("error: %v", err_read)
	}

	// .envから環境変数読み込み
    API_KEY := os.Getenv("API_KEY")
    TOS := strings.Split(os.Getenv("TOS"), ",")
    FROM := os.Getenv("FROM")

	message := mail.NewV3Mail()
	// 送信元
	from := mail.NewEmail("", FROM)
	message.SetFrom(from)

	// 宛先
	p := mail.NewPersonalization()
	to := mail.NewEmail("", TOS[0])
	p.AddTos(to)
	p.SetSubstitution("%fullname%", "田中 太郎")
	p.SetSubstitution("%familyname%", "田中")
	p.SetSubstitution("%place%", "中野")
	message.AddPersonalizations(p)

	p2 := mail.NewPersonalization()
	to2 := mail.NewEmail("", TOS[1])
	p2.AddTos(to2)
	p2.SetSubstitution("%fullname%", "佐藤 次郎")
	p2.SetSubstitution("%familyname%", "佐藤")
	p2.SetSubstitution("%place%", "目黒")
	message.AddPersonalizations(p2)

	p3 := mail.NewPersonalization()
	to3 := mail.NewEmail("", TOS[2])
	p3.AddTos(to3)
	p3.SetSubstitution("%fullname%", "鈴木 三郎")
	p3.SetSubstitution("%familyname%", "鈴木")
	p3.SetSubstitution("%place%", "中野")
	message.AddPersonalizations(p3)
	// 件名
	message.Subject = "[sendgrid-go-example] フクロウのお名前は%fullname%さん"
	// テキストパート
	c := mail.NewContent("text/plain", "%familyname% さんは何をしていますか？\r\n 彼は%place%にいます。")
	message.AddContent(c)
	// HTMLパート
	c = mail.NewContent("text/html", " %familyname% さんは何をしていますか？
彼は%place%にいます。")
	message.AddContent(c)
	// カテゴリ
	message.AddCategories("category1")
	// カスタムヘッダ
	message.SetHeader("X-Sent-Using", "SendGrid-API")
	// 添付ファイル
	a := mail.NewAttachment()
	file, _ := os.OpenFile("./gif.gif", os.O_RDONLY, 0600)
	defer file.Close()
	data, _ := ioutil.ReadAll(file)
	data_enc := base64.StdEncoding.EncodeToString(data)
	a.SetContent(data_enc)
	a.SetType("image/gif")
	a.SetFilename("owl.gif")
	a.SetDisposition("attachment")
	message.AddAttachment(a)

	client := sendgrid.NewSendClient(API_KEY)
	response, err := client.Send(message)
	if err != nil {
		log.Println(err)
	} else {
		fmt.Println(response.StatusCode)
		fmt.Println(response.Body)
		fmt.Println(response.Headers)
	}
}
<commit_msg>Modify descriptions<commit_after>package main

import (
	"fmt"
	"log"
	"os"
	"strings"
	"io/ioutil"
	"encoding/base64"

	"github.com/sendgrid/sendgrid-go"
	"github.com/sendgrid/sendgrid-go/helpers/mail"
	"github.com/joho/godotenv"
)

func main() {

	err_read := godotenv.Load()
	if err_read != nil {
		log.Fatalf("error: %v", err_read)
	}

	// .envから環境変数読み込み
    API_KEY := os.Getenv("API_KEY")
    TOS := strings.Split(os.Getenv("TOS"), ",")
    FROM := os.Getenv("FROM")

	// メッセージの構築
	message := mail.NewV3Mail()
	// 送信元を設定
	from := mail.NewEmail("", FROM)
	message.SetFrom(from)
	// 1つ目の宛先と、対応するSubstitutionタグを指定
	p := mail.NewPersonalization()
	to := mail.NewEmail("", TOS[0])
	p.AddTos(to)
	p.SetSubstitution("%fullname%", "田中 太郎")
	p.SetSubstitution("%familyname%", "田中")
	p.SetSubstitution("%place%", "中野")
	message.AddPersonalizations(p)
	// 2つ目の宛先と、対応するSubstitutionタグを指定
	p2 := mail.NewPersonalization()
	to2 := mail.NewEmail("", TOS[1])
	p2.AddTos(to2)
	p2.SetSubstitution("%fullname%", "佐藤 次郎")
	p2.SetSubstitution("%familyname%", "佐藤")
	p2.SetSubstitution("%place%", "目黒")
	message.AddPersonalizations(p2)
	// 3つ目の宛先と、対応するSubstitutionタグを指定
	p3 := mail.NewPersonalization()
	to3 := mail.NewEmail("", TOS[2])
	p3.AddTos(to3)
	p3.SetSubstitution("%fullname%", "鈴木 三郎")
	p3.SetSubstitution("%familyname%", "鈴木")
	p3.SetSubstitution("%place%", "中野")
	message.AddPersonalizations(p3)
	// 件名を設定
	message.Subject = "[sendgrid-go-example] フクロウのお名前は%fullname%さん"
	// テキストパートを設定
	c := mail.NewContent("text/plain", "%familyname% さんは何をしていますか？\r\n 彼は%place%にいます。")
	message.AddContent(c)
	// HTMLパートを設定
	c = mail.NewContent("text/html", " %familyname% さんは何をしていますか？
彼は%place%にいます。")
	message.AddContent(c)
	// カテゴリ情報を付加
	message.AddCategories("category1")
	// カスタムヘッダを指定
	message.SetHeader("X-Sent-Using", "SendGrid-API")
	// 画像ファイルを添付
	a := mail.NewAttachment()
	file, _ := os.OpenFile("./gif.gif", os.O_RDONLY, 0600)
	defer file.Close()
	data, _ := ioutil.ReadAll(file)
	data_enc := base64.StdEncoding.EncodeToString(data)
	a.SetContent(data_enc)
	a.SetType("image/gif")
	a.SetFilename("owl.gif")
	a.SetDisposition("attachment")
	message.AddAttachment(a)

	// メール送信を行い、レスポンスを表示
	client := sendgrid.NewSendClient(API_KEY)
	response, err := client.Send(message)
	if err != nil {
		log.Println(err)
	} else {
		fmt.Println(response.StatusCode)
		fmt.Println(response.Body)
		fmt.Println(response.Headers)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"http_util"
	"log"
	"os"
)

func init() {
	log.SetFlags(log.Lshortfile | log.LstdFlags)
}

func main() {
	if len(os.Args) != 2 {
		log.Fatal("Usage: mcurl curl-cmd-file")
	}
	cmds := []string{}
	fn, err := os.Open(os.Args[1])
	if err != nil {
		log.Fatal(err)
	}
	scanner := bufio.NewScanner(fn)
	for scanner.Scan() {
		cmd := scanner.Text()
		if cmd == "" {
			continue
		}
		// url := curl_cmd.ParseCmdStr(cmd)[1]
		// header := curl_cmd.GetHeadersFromCurlCmd(cmd)
		// fmt.Println(http_util.GetResourceInfo(url, header))
		cmds = append(cmds, cmd)
	}
	http_util.Run(cmds, 2)
}
<commit_msg>can change num_of_workers at runtime<commit_after>package main

import (
	"bufio"
	"flag"
	"fmt"
	"http_util"
	"log"
	"os"
)

func init() {
	log.SetFlags(log.Lshortfile | log.LstdFlags)
}

func main() {
	flag.Usage = func() {
		fmt.Fprintf(os.Stderr, "Usage: %s [options] curl-cmd-file:\n", os.Args[0])
		flag.PrintDefaults()
	}
	num_of_workers := flag.Int("workers", 2, "num of workers")
	flag.Parse()
	if len(flag.Args()) != 1 {
		flag.Usage()
		os.Exit(2)
	}
	cmds := []string{}
	fn, err := os.Open(flag.Arg(0))
	if err != nil {
		log.Fatal(err)
	}
	scanner := bufio.NewScanner(fn)
	for scanner.Scan() {
		cmd := scanner.Text()
		if cmd == "" {
			continue
		}
		cmds = append(cmds, cmd)
	}

	http_util.Run(cmds, *num_of_workers)
}
<|endoftext|>


<commit_before>// Copyright (c) 2016 Pulcy.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package jobs_test

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"sort"
	"strings"
	"testing"

	"github.com/juju/errgo"
	"github.com/kr/pretty"
	"github.com/op/go-logging"

	"github.com/pulcy/j2/cluster"
	fg "github.com/pulcy/j2/flags"
	"github.com/pulcy/j2/jobs"
	"github.com/pulcy/j2/units"
	"github.com/pulcy/j2/vault"
)

const (
	fixtureDir = "./test-fixtures"
)

var (
	maskAny = errgo.MaskFunc(errgo.Any)
)

func TestParse(t *testing.T) {
	cases := []struct {
		Name                    string
		ErrorExpected           bool
		ExpectedUnitNamesCount1 []string
		ExpectedUnitNamesCount3 []string
	}{
		{
			"simple.hcl",
			false,
			[]string{
				"test-couchdb-couchdb-mn@1.service",
				"test-db-db-mn@1.service",
				"test-dummy-dummy-mn@1.service",
				"test-dummy-dummy-mn@2.service",
				"test-dummy-dummy-mn@3.service",
				"test-global-global-mn.service",
				"test-registrator-registrator-mn.service",
				"test-some_domain_proxy-some_domain_proxy-mn@1.service",
				"test-some_proxy-some_proxy-mn@1.service",
				"test-web-backup-mn@1.service",
				"test-web-backup-ti@1.timer",
				"test-web-backup-mn@2.service",
				"test-web-backup-ti@2.timer",
				"test-web-nginx-mn@1.service",
				"test-web-nginx-mn@2.service",
				"test-web-storage-mn@1.service",
				"test-web-storage-mn@2.service",
				"test-web-storage-pr0@1.service",
				"test-web-storage-pr0@2.service",
			},
			[]string{
				"test-couchdb-couchdb-mn@1.service",
				"test-db-db-mn@1.service",
				"test-dummy-dummy-mn@1.service",
				"test-dummy-dummy-mn@2.service",
				"test-dummy-dummy-mn@3.service",
				"test-global-global-mn.service",
				"test-registrator-registrator-mn.service",
				"test-some_domain_proxy-some_domain_proxy-mn@1.service",
				"test-some_proxy-some_proxy-mn@1.service",
				"test-web-backup-mn@1.service",
				"test-web-backup-ti@1.timer",
				"test-web-backup-mn@2.service",
				"test-web-backup-ti@2.timer",
				"test-web-nginx-mn@1.service",
				"test-web-nginx-mn@2.service",
				"test-web-storage-mn@1.service",
				"test-web-storage-mn@2.service",
				"test-web-storage-pr0@1.service",
				"test-web-storage-pr0@2.service",
			},
		},
		{
			"restart-all.hcl",
			false,
			[]string{
				"restartall-lb1-ta-mn.service",
				"restartall-lb1-tb-mn.service",
				"restartall-lb2-ta-mn.service",
				"restartall-lb2-tb-mn.service",
			},
			[]string{
				"restartall-lb1-ta-mn.service",
				"restartall-lb1-tb-mn.service",
				"restartall-lb2-ta-mn@1.service",
				"restartall-lb2-ta-mn@2.service",
				"restartall-lb2-tb-mn@1.service",
				"restartall-lb2-tb-mn@2.service",
			},
		},
		{
			"secret.hcl",
			false,
			[]string{
				"secrets-env_secrets-env_secrets-mn@1.service",
			},
			[]string{
				"secrets-env_secrets-env_secrets-mn@1.service",
			},
		},
		{
			"extra-fields.hcl",
			true,
			[]string{},
			[]string{},
		},
		{
			"variables.hcl",
			false,
			[]string{
				"test-couchdb-couchdb-mn@1.service",
				"test-db-db-mn@1.service",
				"test-dummy-dummy-mn@1.service",
				"test-dummy-dummy-mn@2.service",
				"test-dummy-dummy-mn@3.service",
				"test-global-global-mn.service",
				"test-registrator-registrator-mn.service",
				"test-web-backup-mn@1.service",
				"test-web-backup-ti@1.timer",
				"test-web-backup-mn@2.service",
				"test-web-backup-ti@2.timer",
				"test-web-nginx-mn@1.service",
				"test-web-nginx-mn@2.service",
				"test-web-storage-mn@1.service",
				"test-web-storage-mn@2.service",
				"test-web-storage-pr0@1.service",
				"test-web-storage-pr0@2.service",
			},
			[]string{
				"test-couchdb-couchdb-mn@1.service",
				"test-db-db-mn@1.service",
				"test-dummy-dummy-mn@1.service",
				"test-dummy-dummy-mn@2.service",
				"test-dummy-dummy-mn@3.service",
				"test-global-global-mn.service",
				"test-registrator-registrator-mn.service",
				"test-web-backup-mn@1.service",
				"test-web-backup-ti@1.timer",
				"test-web-backup-mn@2.service",
				"test-web-backup-ti@2.timer",
				"test-web-nginx-mn@1.service",
				"test-web-nginx-mn@2.service",
				"test-web-storage-mn@1.service",
				"test-web-storage-mn@2.service",
				"test-web-storage-pr0@1.service",
				"test-web-storage-pr0@2.service",
			},
		},
		{
			"volumes.hcl",
			false,
			[]string{
				"volumes-v1-backup-mn@1.service",
				"volumes-v1-backup-ti@1.timer",
				"volumes-v1-storage-mn@1.service",
				"volumes-v1-storage-vl0@1.service",
				"volumes-v2-v2-mn@1.service",
				"volumes-v3-v3-mn@1.service",
				"volumes-v3-v3-vl0@1.service",
				"volumes-v4global-v4global-mn.service",
				"volumes-v4global-v4global-vl0.service",
			},
			[]string{
				"volumes-v1-backup-mn@1.service",
				"volumes-v1-backup-ti@1.timer",
				"volumes-v1-storage-mn@1.service",
				"volumes-v1-storage-vl0@1.service",
				"volumes-v2-v2-mn@1.service",
				"volumes-v3-v3-mn@1.service",
				"volumes-v3-v3-vl0@1.service",
				"volumes-v4global-v4global-mn.service",
				"volumes-v4global-v4global-vl0.service",
			},
		},
	}

	for _, tc := range cases {
		t.Logf("testing %s", tc.Name)
		options := fg.Options{}
		options.Set("option1=value1")
		options.Set("option2=value2")
		cluster3 := cluster.New("test.com", "stack", 3)

		log := logging.MustGetLogger("test")
		vaultConfig := vault.VaultConfig{}
		ghLoginData := vault.GithubLoginData{}
		job, err := jobs.ParseJobFromFile(filepath.Join(fixtureDir, tc.Name), cluster3, options, log, vaultConfig, ghLoginData)
		if tc.ErrorExpected {
			if err == nil {
				t.Fatalf("Expected error in %s", tc.Name)
			}
		} else {
			if err != nil {
				t.Fatalf("Got error in %s: %#v", tc.Name, maskAny(err))
			}
			json, err := job.Json()
			if err != nil {
				t.Fatalf("Cannot convert %s to json: %#v", tc.Name, maskAny(err))
			}
			expectedJson, err := ioutil.ReadFile(filepath.Join(fixtureDir, tc.Name+".json"))
			if err != nil {
				t.Fatalf("Cannot read expected json for %s: %#v", tc.Name, maskAny(err))
			}
			if diffs, err := compareJson(json, expectedJson); err != nil {
				t.Fatalf("Cannot comparse json: %#v", maskAny(err))
			} else if len(diffs) > 0 {
				t.Fatalf("JSON diffs in %s\n%s\nGot: %s", tc.Name, strings.Join(diffs, "\n"), json)
			}

			// Now generate units
			testUnits(t, job, cluster3, tc.ExpectedUnitNamesCount3, tc.Name)
		}

		cluster1 := cluster.New("test.com", "stack", 1)
		job1, err := jobs.ParseJobFromFile(filepath.Join(fixtureDir, tc.Name), cluster1, options, log, vaultConfig, ghLoginData)
		if tc.ErrorExpected {
			if err == nil {
				t.Fatalf("Expected error in %s", tc.Name)
			}
		} else {
			if err != nil {
				t.Fatalf("Got error in %s: %#v", tc.Name, maskAny(err))
			}

			// Now generate units
			testUnits(t, job1, cluster1, tc.ExpectedUnitNamesCount1, tc.Name)
		}
	}
}

func testUnits(t *testing.T, job *jobs.Job, cl cluster.Cluster, expectedUnitNames []string, testName string) {
	jobs.FixedPwhashSalt = "test-salt"
	config := jobs.GeneratorConfig{
		Groups:              nil,
		CurrentScalingGroup: 0,
		DockerOptions: cluster.DockerOptions{
			LoggingArgs: []string{"--log-driver=test"},
		},
		FleetOptions: cl.FleetOptions,
	}
	generator := job.Generate(config)
	generator.NewTmpDir()
	ctx := units.RenderContext{
		ProjectName:    "testproject",
		ProjectVersion: "test-version",
		ProjectBuild:   "test-build",
	}
	defer generator.RemoveTmpFiles()
	images := jobs.Images{
		VaultMonkey: "pulcy/vault-monkey:latest",
	}
	if err := generator.WriteTmpFiles(ctx, images, cl.InstanceCount); err != nil {
		t.Fatalf("WriteTmpFiles failed for instance-count %d: %#v", cl.InstanceCount, maskAny(err))
	}
	compareUnitNames(t, expectedUnitNames, generator.UnitNames())
	compareUnitFiles(t, generator.FileNames(), filepath.Join(fixtureDir, "units", fmt.Sprintf("instance-count-%d", cl.InstanceCount), testName))
}

func compareJson(a, b []byte) ([]string, error) {
	oa := make(map[string]interface{})
	if err := json.Unmarshal(a, &oa); err != nil {
		return nil, maskAny(err)
	}

	ob := make(map[string]interface{})
	if err := json.Unmarshal(b, &ob); err != nil {
		return nil, maskAny(err)
	}

	diffs := pretty.Diff(oa, ob)
	return diffs, nil
}

func compareUnitNames(t *testing.T, expected, found []string) {
	sort.Strings(expected)
	sort.Strings(found)
	expectedStr := strings.Join(expected, "\n- ")
	foundStr := strings.Join(found, "\n- ")
	if expectedStr != foundStr {
		t.Fatalf("Unexpected unit names. Expected \n- %s\ngot \n- %s", expectedStr, foundStr)
	}
}

func compareUnitFiles(t *testing.T, fileNames []string, fixtureDir string) {
	errors := []string{}
	for _, fn := range fileNames {
		fixturePath := filepath.Join(fixtureDir, filepath.Base(fn))
		if _, err := os.Stat(fixturePath); os.IsNotExist(err) || os.Getenv("UPDATE-FIXTURES") == "1" {
			// Fixture does not yet exist, create it
			os.MkdirAll(fixtureDir, 0755)
			data, err := ioutil.ReadFile(fn)
			if err != nil {
				errors = append(errors, fmt.Sprintf("Failed to read '%s': %#v", fn, maskAny(err)))
			} else {
				if err := ioutil.WriteFile(fixturePath, data, 0755); err != nil {
					errors = append(errors, fmt.Sprintf("Failed to create fixture: %#v", maskAny(err)))
				}
			}
		} else {
			// Compare
			fixtureRaw, err := ioutil.ReadFile(fixturePath)
			if err != nil {
				errors = append(errors, fmt.Sprintf("Failed to read fixture: %#v", maskAny(err)))
				continue
			}
			fnRaw, err := ioutil.ReadFile(fn)
			if err != nil {
				errors = append(errors, fmt.Sprintf("Failed to read test: %#v", maskAny(err)))
				continue
			}

			fixtureContent := strings.TrimSpace(string(fixtureRaw))
			fnContent := strings.TrimSpace(string(fnRaw))

			if fixtureContent != fnContent {
				cmd := exec.Command("diff", fixturePath, fn)
				if output, err := cmd.Output(); err != nil {
					errors = append(errors, fmt.Sprintf("File '%s' is different:\n%s", fixturePath, string(output)))
				}
			}
		}
	}
	if len(errors) > 0 {
		t.Fatal(strings.Join(errors, "\n"))
	}
}
<commit_msg>Fixed unit tests<commit_after>// Copyright (c) 2016 Pulcy.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package jobs_test

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"sort"
	"strings"
	"testing"

	"github.com/juju/errgo"
	"github.com/kr/pretty"
	"github.com/op/go-logging"

	"github.com/pulcy/j2/cluster"
	fg "github.com/pulcy/j2/flags"
	"github.com/pulcy/j2/jobs"
	"github.com/pulcy/j2/units"
	"github.com/pulcy/j2/vault"
)

const (
	fixtureDir = "./test-fixtures"
)

var (
	maskAny = errgo.MaskFunc(errgo.Any)
)

func TestParse(t *testing.T) {
	cases := []struct {
		Name                    string
		ErrorExpected           bool
		ExpectedUnitNamesCount1 []string
		ExpectedUnitNamesCount3 []string
	}{
		{
			"simple.hcl",
			false,
			[]string{
				"test-couchdb-couchdb-mn@1.service",
				"test-db-db-mn@1.service",
				"test-dummy-dummy-mn@1.service",
				"test-dummy-dummy-mn@2.service",
				"test-dummy-dummy-mn@3.service",
				"test-global-global-mn.service",
				"test-registrator-registrator-mn.service",
				"test-some_domain_proxy-some_domain_proxy-mn@1.service",
				"test-some_proxy-some_proxy-mn@1.service",
				"test-web-backup-mn@1.service",
				"test-web-backup-ti@1.timer",
				"test-web-backup-mn@2.service",
				"test-web-backup-ti@2.timer",
				"test-web-nginx-mn@1.service",
				"test-web-nginx-mn@2.service",
				"test-web-storage-mn@1.service",
				"test-web-storage-mn@2.service",
				"test-web-storage-pr0@1.service",
				"test-web-storage-pr0@2.service",
			},
			[]string{
				"test-couchdb-couchdb-mn@1.service",
				"test-db-db-mn@1.service",
				"test-dummy-dummy-mn@1.service",
				"test-dummy-dummy-mn@2.service",
				"test-dummy-dummy-mn@3.service",
				"test-global-global-mn.service",
				"test-registrator-registrator-mn.service",
				"test-some_domain_proxy-some_domain_proxy-mn@1.service",
				"test-some_proxy-some_proxy-mn@1.service",
				"test-web-backup-mn@1.service",
				"test-web-backup-ti@1.timer",
				"test-web-backup-mn@2.service",
				"test-web-backup-ti@2.timer",
				"test-web-nginx-mn@1.service",
				"test-web-nginx-mn@2.service",
				"test-web-storage-mn@1.service",
				"test-web-storage-mn@2.service",
				"test-web-storage-pr0@1.service",
				"test-web-storage-pr0@2.service",
			},
		},
		{
			"restart-all.hcl",
			false,
			[]string{
				"restartall-lb1-ta-mn.service",
				"restartall-lb1-tb-mn.service",
				"restartall-lb2-ta-mn.service",
				"restartall-lb2-tb-mn.service",
			},
			[]string{
				"restartall-lb1-ta-mn.service",
				"restartall-lb1-tb-mn.service",
				"restartall-lb2-ta-mn@1.service",
				"restartall-lb2-ta-mn@2.service",
				"restartall-lb2-tb-mn@1.service",
				"restartall-lb2-tb-mn@2.service",
			},
		},
		{
			"secret.hcl",
			false,
			[]string{
				"secrets-env_secrets-env_secrets-mn@1.service",
			},
			[]string{
				"secrets-env_secrets-env_secrets-mn@1.service",
			},
		},
		{
			"extra-fields.hcl",
			true,
			[]string{},
			[]string{},
		},
		{
			"variables.hcl",
			false,
			[]string{
				"test-couchdb-couchdb-mn@1.service",
				"test-db-db-mn@1.service",
				"test-dummy-dummy-mn@1.service",
				"test-dummy-dummy-mn@2.service",
				"test-dummy-dummy-mn@3.service",
				"test-global-global-mn.service",
				"test-registrator-registrator-mn.service",
				"test-web-backup-mn@1.service",
				"test-web-backup-ti@1.timer",
				"test-web-backup-mn@2.service",
				"test-web-backup-ti@2.timer",
				"test-web-nginx-mn@1.service",
				"test-web-nginx-mn@2.service",
				"test-web-storage-mn@1.service",
				"test-web-storage-mn@2.service",
				"test-web-storage-pr0@1.service",
				"test-web-storage-pr0@2.service",
			},
			[]string{
				"test-couchdb-couchdb-mn@1.service",
				"test-db-db-mn@1.service",
				"test-dummy-dummy-mn@1.service",
				"test-dummy-dummy-mn@2.service",
				"test-dummy-dummy-mn@3.service",
				"test-global-global-mn.service",
				"test-registrator-registrator-mn.service",
				"test-web-backup-mn@1.service",
				"test-web-backup-ti@1.timer",
				"test-web-backup-mn@2.service",
				"test-web-backup-ti@2.timer",
				"test-web-nginx-mn@1.service",
				"test-web-nginx-mn@2.service",
				"test-web-storage-mn@1.service",
				"test-web-storage-mn@2.service",
				"test-web-storage-pr0@1.service",
				"test-web-storage-pr0@2.service",
			},
		},
		{
			"volumes.hcl",
			false,
			[]string{
				"volumes-v1-backup-mn@1.service",
				"volumes-v1-backup-ti@1.timer",
				"volumes-v1-storage-mn@1.service",
				"volumes-v1-storage-vl0@1.service",
				"volumes-v2-v2-mn@1.service",
				"volumes-v3-v3-mn@1.service",
				"volumes-v3-v3-vl0@1.service",
				"volumes-v4global-v4global-mn.service",
				"volumes-v4global-v4global-vl0.service",
			},
			[]string{
				"volumes-v1-backup-mn@1.service",
				"volumes-v1-backup-ti@1.timer",
				"volumes-v1-storage-mn@1.service",
				"volumes-v1-storage-vl0@1.service",
				"volumes-v2-v2-mn@1.service",
				"volumes-v3-v3-mn@1.service",
				"volumes-v3-v3-vl0@1.service",
				"volumes-v4global-v4global-mn.service",
				"volumes-v4global-v4global-vl0.service",
			},
		},
	}

	for _, tc := range cases {
		t.Logf("testing %s", tc.Name)
		options := fg.Options{}
		options.Set("option1=value1")
		options.Set("option2=value2")
		cluster3 := cluster.New("test.com", "stack", 3)

		log := logging.MustGetLogger("test")
		vaultConfig := vault.VaultConfig{}
		ghLoginData := vault.GithubLoginData{}
		job, err := jobs.ParseJobFromFile(filepath.Join(fixtureDir, tc.Name), cluster3, options, log, vaultConfig, ghLoginData)
		if tc.ErrorExpected {
			if err == nil {
				t.Fatalf("Expected error in %s", tc.Name)
			}
		} else {
			if err != nil {
				t.Fatalf("Got error in %s: %#v", tc.Name, maskAny(err))
			}
			json, err := job.Json()
			if err != nil {
				t.Fatalf("Cannot convert %s to json: %#v", tc.Name, maskAny(err))
			}
			expectedJson, err := ioutil.ReadFile(filepath.Join(fixtureDir, tc.Name+".json"))
			if err != nil {
				t.Fatalf("Cannot read expected json for %s: %#v", tc.Name, maskAny(err))
			}
			if diffs, err := compareJson(json, expectedJson); err != nil {
				t.Fatalf("Cannot comparse json: %#v", maskAny(err))
			} else if len(diffs) > 0 {
				t.Fatalf("JSON diffs in %s\n%s\nGot: %s", tc.Name, strings.Join(diffs, "\n"), json)
			}

			// Now generate units
			testUnits(t, job, cluster3, tc.ExpectedUnitNamesCount3, tc.Name)
		}

		cluster1 := cluster.New("test.com", "stack", 1)
		job1, err := jobs.ParseJobFromFile(filepath.Join(fixtureDir, tc.Name), cluster1, options, log, vaultConfig, ghLoginData)
		if tc.ErrorExpected {
			if err == nil {
				t.Fatalf("Expected error in %s", tc.Name)
			}
		} else {
			if err != nil {
				t.Fatalf("Got error in %s: %#v", tc.Name, maskAny(err))
			}

			// Now generate units
			testUnits(t, job1, cluster1, tc.ExpectedUnitNamesCount1, tc.Name)
		}
	}
}

func testUnits(t *testing.T, job *jobs.Job, cl cluster.Cluster, expectedUnitNames []string, testName string) {
	jobs.FixedPwhashSalt = "test-salt"
	config := jobs.GeneratorConfig{
		Groups:              nil,
		CurrentScalingGroup: 0,
		DockerOptions: cluster.DockerOptions{
			LoggingArgs: []string{"--log-driver=test"},
		},
		FleetOptions: cl.FleetOptions,
	}
	generator := job.Generate(config)
	ctx := units.RenderContext{
		ProjectName:    "testproject",
		ProjectVersion: "test-version",
		ProjectBuild:   "test-build",
	}
	images := jobs.Images{
		VaultMonkey: "pulcy/vault-monkey:latest",
	}
	units, err := generator.GenerateUnits(ctx, images, cl.InstanceCount)
	if err != nil {
		t.Fatalf("GenerateUnits failed for instance-count %d: %#v", cl.InstanceCount, maskAny(err))
	}
	compareUnitNames(t, expectedUnitNames, units)
	compareUnitFiles(t, units, filepath.Join(fixtureDir, "units", fmt.Sprintf("instance-count-%d", cl.InstanceCount), testName))
}

func compareJson(a, b []byte) ([]string, error) {
	oa := make(map[string]interface{})
	if err := json.Unmarshal(a, &oa); err != nil {
		return nil, maskAny(err)
	}

	ob := make(map[string]interface{})
	if err := json.Unmarshal(b, &ob); err != nil {
		return nil, maskAny(err)
	}

	diffs := pretty.Diff(oa, ob)
	return diffs, nil
}

func compareUnitNames(t *testing.T, expected []string, generated []jobs.UnitData) {
	var found []string
	for _, u := range generated {
		found = append(found, u.Name())
	}
	sort.Strings(expected)
	sort.Strings(found)
	expectedStr := strings.Join(expected, "\n- ")
	foundStr := strings.Join(found, "\n- ")
	if expectedStr != foundStr {
		t.Fatalf("Unexpected unit names. Expected \n- %s\ngot \n- %s", expectedStr, foundStr)
	}
}

func compareUnitFiles(t *testing.T, units []jobs.UnitData, fixtureDir string) {
	errors := []string{}
	tmpDir, err := ioutil.TempDir("", "j2-test")
	if err != nil {
		t.Fatalf("Cannot create temp dir: %#v", err)
		return
	}
	defer os.RemoveAll(tmpDir)
	for _, u := range units {
		fixturePath := filepath.Join(fixtureDir, u.Name())
		if _, err := os.Stat(fixturePath); os.IsNotExist(err) || os.Getenv("UPDATE-FIXTURES") == "1" {
			// Fixture does not yet exist, create it
			os.MkdirAll(fixtureDir, 0755)
			if err := ioutil.WriteFile(fixturePath, []byte(u.Content()), 0755); err != nil {
				errors = append(errors, fmt.Sprintf("Failed to create fixture: %#v", maskAny(err)))
			}
		} else {
			// Compare
			fixtureRaw, err := ioutil.ReadFile(fixturePath)
			if err != nil {
				errors = append(errors, fmt.Sprintf("Failed to read fixture: %#v", maskAny(err)))
				continue
			}

			fixtureContent := strings.TrimSpace(string(fixtureRaw))
			fnContent := strings.TrimSpace(u.Content())

			if fixtureContent != fnContent {
				fn := filepath.Join(tmpDir, u.Name())
				if err := ioutil.WriteFile(fn, []byte(u.Content()), 0755); err != nil {
					errors = append(errors, fmt.Sprintf("Failed to create fn: %#v", maskAny(err)))
				} else {
					cmd := exec.Command("diff", fixturePath, fn)
					if output, err := cmd.Output(); err != nil {
						errors = append(errors, fmt.Sprintf("File '%s' is different:\n%s", fixturePath, string(output)))
					}
				}
			}
		}
	}
	if len(errors) > 0 {
		t.Fatal(strings.Join(errors, "\n"))
	}
}
<|endoftext|>


<commit_before>// Copyright 2015 RedHat, Inc.
// Copyright 2015 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Provides a low-level Go interface to the systemd journal C API.
//
// All public methods map closely to the sd-journal API functions. See the
// sd-journal.h documentation[1] for information about each function.
//
// [1] http://www.freedesktop.org/software/systemd/man/sd-journal.html
package journal

/*
#cgo pkg-config: libsystemd-journal
#include <systemd/sd-journal.h>
#include <stdlib.h>
#include <syslog.h>
*/
import "C"
import (
	"fmt"
	"sync"
	"time"
	"unsafe"
)

// Journal entry field strings which correspond to:
// http://www.freedesktop.org/software/systemd/man/systemd.journal-fields.html
const (
	SD_JOURNAL_FIELD_SYSTEMD_UNIT = "_SYSTEMD_UNIT"
	SD_JOURNAL_FIELD_MESSAGE      = "MESSAGE"
	SD_JOURNAL_FIELD_PID          = "_PID"
	SD_JOURNAL_FIELD_UID          = "_UID"
	SD_JOURNAL_FIELD_GID          = "_GID"
	SD_JOURNAL_FIELD_HOSTNAME     = "_HOSTNAME"
	SD_JOURNAL_FIELD_MACHINE_ID   = "_MACHINE_ID"
)

// Journal event constants
const (
	SD_JOURNAL_NOP        = int(C.SD_JOURNAL_NOP)
	SD_JOURNAL_APPEND     = int(C.SD_JOURNAL_APPEND)
	SD_JOURNAL_INVALIDATE = int(C.SD_JOURNAL_INVALIDATE)
)

// A Journal is a Go wrapper of an sd_journal structure.
type Journal struct {
	cjournal *C.sd_journal
	mu       sync.Mutex
}

// A Match is a convenience wrapper to describe filters supplied to AddMatch.
type Match struct {
	Field string
	Value string
}

// String returns a string representation of a Match suitable for use with AddMatch.
func (m *Match) String() string {
	return m.Field + "=" + m.Value
}

// NewJournal returns a new Journal instance pointing to the local journal
func NewJournal() (*Journal, error) {
	j := &Journal{}
	err := C.sd_journal_open(&j.cjournal, C.SD_JOURNAL_LOCAL_ONLY)

	if err < 0 {
		return nil, fmt.Errorf("failed to open journal: %s", err)
	}

	return j, nil
}

// Close closes a journal opened with NewJournal.
func (j *Journal) Close() error {
	j.mu.Lock()
	C.sd_journal_close(j.cjournal)
	j.mu.Unlock()

	return nil
}

// AddMatch adds a match by which to filter the entries of the journal.
func (j *Journal) AddMatch(match string) error {
	m := C.CString(match)
	defer C.free(unsafe.Pointer(m))

	j.mu.Lock()
	C.sd_journal_add_match(j.cjournal, unsafe.Pointer(m), C.size_t(len(match)))
	j.mu.Unlock()

	return nil
}

// Next advances the read pointer into the journal by one entry.
func (j *Journal) Next() (int, error) {
	j.mu.Lock()
	r := C.sd_journal_next(j.cjournal)
	j.mu.Unlock()

	if r < 0 {
		return int(r), fmt.Errorf("failed to iterate journal: %d", r)
	}

	return int(r), nil
}

// Previous sets the read pointer into the journal back by one entry.
func (j *Journal) Previous() (uint64, error) {
	j.mu.Lock()
	r := C.sd_journal_previous(j.cjournal)
	j.mu.Unlock()

	if r < 0 {
		return uint64(r), fmt.Errorf("failed to iterate journal: %d", r)
	}

	return uint64(r), nil
}

// PreviousSkip sets back the read pointer by multiple entries at once,
// as specified by the skip parameter.
func (j *Journal) PreviousSkip(skip uint64) (uint64, error) {
	j.mu.Lock()
	r := C.sd_journal_previous_skip(j.cjournal, C.uint64_t(skip))
	j.mu.Unlock()

	if r < 0 {
		return uint64(r), fmt.Errorf("failed to iterate journal: %d", r)
	}

	return uint64(r), nil
}

// GetData gets the data object associated with a specific field from the
// current journal entry.
func (j *Journal) GetData(field string) (string, error) {
	f := C.CString(field)
	defer C.free(unsafe.Pointer(f))

	var d unsafe.Pointer
	var l C.size_t

	j.mu.Lock()
	err := C.sd_journal_get_data(j.cjournal, f, &d, &l)
	j.mu.Unlock()

	if err < 0 {
		return "", fmt.Errorf("failed to read message: %d", err)
	}

	msg := C.GoStringN((*C.char)(d), C.int(l))

	return msg, nil
}

// GetRealtimeUsec gets the realtime (wallclock) timestamp of the current
// journal entry.
func (j *Journal) GetRealtimeUsec() (uint64, error) {
	var usec C.uint64_t

	j.mu.Lock()
	r := C.sd_journal_get_realtime_usec(j.cjournal, &usec)
	j.mu.Unlock()

	if r < 0 {
		return 0, fmt.Errorf("error getting timestamp for entry: %d", r)
	}

	return uint64(usec), nil
}

// SeekTail may be used to seek to the end of the journal, i.e. the most recent
// available entry.
func (j *Journal) SeekTail() error {
	j.mu.Lock()
	err := C.sd_journal_seek_tail(j.cjournal)
	j.mu.Unlock()

	if err != 0 {
		return fmt.Errorf("failed to seek to tail of journal: %s", err)
	}

	return nil
}

// SeekRealtimeUsec seeks to the entry with the specified realtime (wallclock)
// timestamp, i.e. CLOCK_REALTIME.
func (j *Journal) SeekRealtimeUsec(usec uint64) error {
	j.mu.Lock()
	err := C.sd_journal_seek_realtime_usec(j.cjournal, C.uint64_t(usec))
	j.mu.Unlock()

	if err != 0 {
		return fmt.Errorf("failed to seek to %d: %d", usec, int(err))
	}

	return nil
}

// Wait will synchronously wait until the journal gets changed. The maximum time
// this call sleeps may be controlled with the timeout parameter.
func (j *Journal) Wait(timeout time.Duration) int {
	to := uint64(time.Now().Add(timeout).Unix() / 1000)
	j.mu.Lock()
	r := C.sd_journal_wait(j.cjournal, C.uint64_t(to))
	j.mu.Unlock()

	return int(r)
}
<commit_msg>journal: implement Journal.NextSkip() (sd_journal_next_skip()).<commit_after>// Copyright 2015 RedHat, Inc.
// Copyright 2015 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Provides a low-level Go interface to the systemd journal C API.
//
// All public methods map closely to the sd-journal API functions. See the
// sd-journal.h documentation[1] for information about each function.
//
// [1] http://www.freedesktop.org/software/systemd/man/sd-journal.html
package journal

/*
#cgo pkg-config: libsystemd-journal
#include <systemd/sd-journal.h>
#include <stdlib.h>
#include <syslog.h>
*/
import "C"
import (
	"fmt"
	"sync"
	"time"
	"unsafe"
)

// Journal entry field strings which correspond to:
// http://www.freedesktop.org/software/systemd/man/systemd.journal-fields.html
const (
	SD_JOURNAL_FIELD_SYSTEMD_UNIT = "_SYSTEMD_UNIT"
	SD_JOURNAL_FIELD_MESSAGE      = "MESSAGE"
	SD_JOURNAL_FIELD_PID          = "_PID"
	SD_JOURNAL_FIELD_UID          = "_UID"
	SD_JOURNAL_FIELD_GID          = "_GID"
	SD_JOURNAL_FIELD_HOSTNAME     = "_HOSTNAME"
	SD_JOURNAL_FIELD_MACHINE_ID   = "_MACHINE_ID"
)

// Journal event constants
const (
	SD_JOURNAL_NOP        = int(C.SD_JOURNAL_NOP)
	SD_JOURNAL_APPEND     = int(C.SD_JOURNAL_APPEND)
	SD_JOURNAL_INVALIDATE = int(C.SD_JOURNAL_INVALIDATE)
)

// A Journal is a Go wrapper of an sd_journal structure.
type Journal struct {
	cjournal *C.sd_journal
	mu       sync.Mutex
}

// A Match is a convenience wrapper to describe filters supplied to AddMatch.
type Match struct {
	Field string
	Value string
}

// String returns a string representation of a Match suitable for use with AddMatch.
func (m *Match) String() string {
	return m.Field + "=" + m.Value
}

// NewJournal returns a new Journal instance pointing to the local journal
func NewJournal() (*Journal, error) {
	j := &Journal{}
	err := C.sd_journal_open(&j.cjournal, C.SD_JOURNAL_LOCAL_ONLY)

	if err < 0 {
		return nil, fmt.Errorf("failed to open journal: %s", err)
	}

	return j, nil
}

// Close closes a journal opened with NewJournal.
func (j *Journal) Close() error {
	j.mu.Lock()
	C.sd_journal_close(j.cjournal)
	j.mu.Unlock()

	return nil
}

// AddMatch adds a match by which to filter the entries of the journal.
func (j *Journal) AddMatch(match string) error {
	m := C.CString(match)
	defer C.free(unsafe.Pointer(m))

	j.mu.Lock()
	C.sd_journal_add_match(j.cjournal, unsafe.Pointer(m), C.size_t(len(match)))
	j.mu.Unlock()

	return nil
}

// Next advances the read pointer into the journal by one entry.
func (j *Journal) Next() (int, error) {
	j.mu.Lock()
	r := C.sd_journal_next(j.cjournal)
	j.mu.Unlock()

	if r < 0 {
		return int(r), fmt.Errorf("failed to iterate journal: %d", r)
	}

	return int(r), nil
}

// NextSkip advances the read pointer by multiple entries at once,
// as specified by the skip parameter.
func (j *Journal) NextSkip(skip uint64) (uint64, error) {
	j.mu.Lock()
	r := C.sd_journal_next_skip(j.cjournal, C.uint64_t(skip))
	j.mu.Unlock()

	if r < 0 {
		return uint64(r), fmt.Errorf("failed to iterate journal: %d", r)
	}

	return uint64(r), nil
}

// Previous sets the read pointer into the journal back by one entry.
func (j *Journal) Previous() (uint64, error) {
	j.mu.Lock()
	r := C.sd_journal_previous(j.cjournal)
	j.mu.Unlock()

	if r < 0 {
		return uint64(r), fmt.Errorf("failed to iterate journal: %d", r)
	}

	return uint64(r), nil
}

// PreviousSkip sets back the read pointer by multiple entries at once,
// as specified by the skip parameter.
func (j *Journal) PreviousSkip(skip uint64) (uint64, error) {
	j.mu.Lock()
	r := C.sd_journal_previous_skip(j.cjournal, C.uint64_t(skip))
	j.mu.Unlock()

	if r < 0 {
		return uint64(r), fmt.Errorf("failed to iterate journal: %d", r)
	}

	return uint64(r), nil
}

// GetData gets the data object associated with a specific field from the
// current journal entry.
func (j *Journal) GetData(field string) (string, error) {
	f := C.CString(field)
	defer C.free(unsafe.Pointer(f))

	var d unsafe.Pointer
	var l C.size_t

	j.mu.Lock()
	err := C.sd_journal_get_data(j.cjournal, f, &d, &l)
	j.mu.Unlock()

	if err < 0 {
		return "", fmt.Errorf("failed to read message: %d", err)
	}

	msg := C.GoStringN((*C.char)(d), C.int(l))

	return msg, nil
}

// GetRealtimeUsec gets the realtime (wallclock) timestamp of the current
// journal entry.
func (j *Journal) GetRealtimeUsec() (uint64, error) {
	var usec C.uint64_t

	j.mu.Lock()
	r := C.sd_journal_get_realtime_usec(j.cjournal, &usec)
	j.mu.Unlock()

	if r < 0 {
		return 0, fmt.Errorf("error getting timestamp for entry: %d", r)
	}

	return uint64(usec), nil
}

// SeekTail may be used to seek to the end of the journal, i.e. the most recent
// available entry.
func (j *Journal) SeekTail() error {
	j.mu.Lock()
	err := C.sd_journal_seek_tail(j.cjournal)
	j.mu.Unlock()

	if err != 0 {
		return fmt.Errorf("failed to seek to tail of journal: %s", err)
	}

	return nil
}

// SeekRealtimeUsec seeks to the entry with the specified realtime (wallclock)
// timestamp, i.e. CLOCK_REALTIME.
func (j *Journal) SeekRealtimeUsec(usec uint64) error {
	j.mu.Lock()
	err := C.sd_journal_seek_realtime_usec(j.cjournal, C.uint64_t(usec))
	j.mu.Unlock()

	if err != 0 {
		return fmt.Errorf("failed to seek to %d: %d", usec, int(err))
	}

	return nil
}

// Wait will synchronously wait until the journal gets changed. The maximum time
// this call sleeps may be controlled with the timeout parameter.
func (j *Journal) Wait(timeout time.Duration) int {
	to := uint64(time.Now().Add(timeout).Unix() / 1000)
	j.mu.Lock()
	r := C.sd_journal_wait(j.cjournal, C.uint64_t(to))
	j.mu.Unlock()

	return int(r)
}
<|endoftext|>


<commit_before>// +build noos

package sync

import (
	"sync/atomic"
	"syscall"
)

type waitgroup struct {
	e   syscall.Event
	cnt int32
}

func add(wg *WaitGroup, delta int) {
	cnt := atomic.AddInt32(&wg.cnt, int32(delta))
	if cnt < 0 {
		panic("sync: negative WaitGroup counter")
	}
	if delta < 0 && cnt == 0 {
		e := syscall.AtomicLoadEvent(&wg.e)
		if e != 0 {
			// Waiter should check cnt.
			e.Send()
		}
	}
}

func wait(wg *WaitGroup) {
	e := syscall.AssignEvent()
	syscall.AtomicStoreEvent(&wg.e, e)
	for atomic.LoadInt32(&wg.cnt) != 0 {
		e.Wait()
	}
}
<commit_msg>sync: Fix WaitGroup.Wait.<commit_after>// +build noos

package sync

import (
	"sync/atomic"
	"syscall"
)

type waitgroup struct {
	e   syscall.Event
	cnt int32
}

func add(wg *WaitGroup, delta int) {
	cnt := atomic.AddInt32(&wg.cnt, int32(delta))
	if cnt < 0 {
		panic("sync: negative WaitGroup counter")
	}
	if delta < 0 && cnt == 0 {
		if e := syscall.AtomicLoadEvent(&wg.e); e != 0 {
			e.Send()
		}
	}
}

func wait(wg *WaitGroup) {
	if wg.e == 0 {
		syscall.AtomicStoreEvent(&wg.e, syscall.AssignEvent())
	}
	for atomic.LoadInt32(&wg.cnt) != 0 {
		wg.e.Wait()
	}
}
<|endoftext|>


<commit_before>package commands

import (
	"bytes"
	"errors"
	"fmt"
	"io"
	"os"
	"path"
	"strings"

	cmds "github.com/jbenet/go-ipfs/commands"
	files "github.com/jbenet/go-ipfs/commands/files"
	core "github.com/jbenet/go-ipfs/core"
	importer "github.com/jbenet/go-ipfs/importer"
	"github.com/jbenet/go-ipfs/importer/chunk"
	dag "github.com/jbenet/go-ipfs/merkledag"
	pinning "github.com/jbenet/go-ipfs/pin"
	ft "github.com/jbenet/go-ipfs/unixfs"
	u "github.com/jbenet/go-ipfs/util"

	"github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/cheggaaa/pb"
)

// Error indicating the max depth has been exceded.
var ErrDepthLimitExceeded = fmt.Errorf("depth limit exceeded")

// how many bytes of progress to wait before sending a progress update message
const progressReaderIncrement = 1024 * 256

const progressOptionName = "progress"

type AddedObject struct {
	Name  string
	Hash  string `json:",omitempty"`
	Bytes int64  `json:",omitempty"`
}

var AddCmd = &cmds.Command{
	Helptext: cmds.HelpText{
		Tagline: "Add an object to ipfs.",
		ShortDescription: `
Adds contents of <path> to ipfs. Use -r to add directories.
Note that directories are added recursively, to form the ipfs
MerkleDAG. A smarter partial add with a staging area (like git)
remains to be implemented.
`,
	},

	Arguments: []cmds.Argument{
		cmds.FileArg("path", true, true, "The path to a file to be added to IPFS").EnableRecursive().EnableStdin(),
	},
	Options: []cmds.Option{
		cmds.OptionRecursivePath, // a builtin option that allows recursive paths (-r, --recursive)
		cmds.BoolOption("quiet", "q", "Write minimal output"),
		cmds.BoolOption(progressOptionName, "p", "Stream progress data"),
	},
	PreRun: func(req cmds.Request) error {
		req.SetOption(progressOptionName, true)

		sizeFile, ok := req.Files().(files.SizeFile)
		if !ok {
			// we don't need to error, the progress bar just won't know how big the files are
			return nil
		}

		size, err := sizeFile.Size()
		if err != nil {
			// see comment above
			return nil
		}
		log.Debugf("Total size of file being added: %v\n", size)
		req.Values()["size"] = size

		return nil
	},
	Run: func(req cmds.Request, res cmds.Response) {
		n, err := req.Context().GetNode()
		if err != nil {
			res.SetError(err, cmds.ErrNormal)
			return
		}

		progress, _, _ := req.Option(progressOptionName).Bool()

		outChan := make(chan interface{})
		res.SetOutput((<-chan interface{})(outChan))

		go func() {
			defer close(outChan)

			for {
				file, err := req.Files().NextFile()
				if (err != nil && err != io.EOF) || file == nil {
					return
				}

				_, err = addFile(n, file, outChan, progress)
				if err != nil {
					return
				}
			}
		}()
	},
	PostRun: func(res cmds.Response) {
		outChan, ok := res.Output().(<-chan interface{})
		if !ok {
			res.SetError(u.ErrCast(), cmds.ErrNormal)
			return
		}

		wrapperChan := make(chan interface{})
		res.SetOutput((<-chan interface{})(wrapperChan))

		size := int64(0)
		s, found := res.Request().Values()["size"]
		if found {
			size = s.(int64)
		}
		showProgressBar := size >= progressBarMinSize

		var bar *pb.ProgressBar
		var terminalWidth int
		if showProgressBar {
			bar = pb.New64(size).SetUnits(pb.U_BYTES)
			bar.ManualUpdate = true
			bar.Start()

			// the progress bar lib doesn't give us a way to get the width of the output,
			// so as a hack we just use a callback to measure the output, then git rid of it
			terminalWidth = 0
			bar.Callback = func(line string) {
				terminalWidth = len(line)
				bar.Callback = nil
				bar.Output = os.Stderr
				log.Infof("terminal width: %v\n", terminalWidth)
			}
			bar.Update()
		}

		go func() {
			lastFile := ""
			var totalProgress, prevFiles, lastBytes int64

			for out := range outChan {
				output := out.(*AddedObject)
				if len(output.Hash) > 0 {
					if showProgressBar {
						// clear progress bar line before we print "added x" output
						fmt.Fprintf(os.Stderr, "\r%s\r", strings.Repeat(" ", terminalWidth))
					}
					wrapperChan <- output

				} else {
					log.Debugf("add progress: %v %v\n", output.Name, output.Bytes)

					if !showProgressBar {
						continue
					}

					if len(lastFile) == 0 {
						lastFile = output.Name
					}
					if output.Name != lastFile || output.Bytes < lastBytes {
						prevFiles += lastBytes
						lastFile = output.Name
					}
					lastBytes = output.Bytes
					delta := prevFiles + lastBytes - totalProgress
					totalProgress = bar.Add64(delta)

					bar.Update()
				}
			}

			close(wrapperChan)
		}()
	},
	Marshalers: cmds.MarshalerMap{
		cmds.Text: func(res cmds.Response) (io.Reader, error) {
			outChan, ok := res.Output().(<-chan interface{})
			if !ok {
				return nil, u.ErrCast()
			}

			quiet, _, err := res.Request().Option("quiet").Bool()
			if err != nil {
				return nil, err
			}

			marshal := func(v interface{}) (io.Reader, error) {
				obj, ok := v.(*AddedObject)
				if !ok {
					return nil, u.ErrCast()
				}

				var buf bytes.Buffer
				if quiet {
					buf.WriteString(fmt.Sprintf("%s\n", obj.Hash))
				} else {
					buf.WriteString(fmt.Sprintf("added %s %s\n", obj.Hash, obj.Name))
				}
				return &buf, nil
			}

			return &cmds.ChannelMarshaler{
				Channel:   outChan,
				Marshaler: marshal,
			}, nil
		},
	},
	Type: AddedObject{},
}

func add(n *core.IpfsNode, readers []io.Reader) ([]*dag.Node, error) {
	mp, ok := n.Pinning.(pinning.ManualPinner)
	if !ok {
		return nil, errors.New("invalid pinner type! expected manual pinner")
	}

	dagnodes := make([]*dag.Node, 0)

	for _, reader := range readers {
		node, err := importer.BuildDagFromReader(reader, n.DAG, mp, chunk.DefaultSplitter)
		if err != nil {
			return nil, err
		}
		dagnodes = append(dagnodes, node)
	}

	err := n.Pinning.Flush()
	if err != nil {
		return nil, err
	}

	return dagnodes, nil
}

func addNode(n *core.IpfsNode, node *dag.Node) error {
	err := n.DAG.AddRecursive(node) // add the file to the graph + local storage
	if err != nil {
		return err
	}

	err = n.Pinning.Pin(node, true) // ensure we keep it
	if err != nil {
		return err
	}

	return nil
}

func addFile(n *core.IpfsNode, file files.File, out chan interface{}, progress bool) (*dag.Node, error) {
	if file.IsDirectory() {
		return addDir(n, file, out, progress)
	}

	// if the progress flag was specified, wrap the file so that we can send
	// progress updates to the client (over the output channel)
	var reader io.Reader = file
	if progress {
		reader = &progressReader{file: file, out: out}
	}

	dns, err := add(n, []io.Reader{reader})
	if err != nil {
		return nil, err
	}

	log.Infof("adding file: %s", file.FileName())
	if err := outputDagnode(out, file.FileName(), dns[len(dns)-1]); err != nil {
		return nil, err
	}
	return dns[len(dns)-1], nil // last dag node is the file.
}

func addDir(n *core.IpfsNode, dir files.File, out chan interface{}, progress bool) (*dag.Node, error) {
	log.Infof("adding directory: %s", dir.FileName())

	tree := &dag.Node{Data: ft.FolderPBData()}

	for {
		file, err := dir.NextFile()
		if err != nil && err != io.EOF {
			return nil, err
		}
		if file == nil {
			break
		}

		node, err := addFile(n, file, out, progress)
		if err != nil {
			return nil, err
		}

		_, name := path.Split(file.FileName())

		err = tree.AddNodeLink(name, node)
		if err != nil {
			return nil, err
		}
	}

	err := outputDagnode(out, dir.FileName(), tree)
	if err != nil {
		return nil, err
	}

	err = addNode(n, tree)
	if err != nil {
		return nil, err
	}

	return tree, nil
}

// outputDagnode sends dagnode info over the output channel
func outputDagnode(out chan interface{}, name string, dn *dag.Node) error {
	o, err := getOutput(dn)
	if err != nil {
		return err
	}

	out <- &AddedObject{
		Hash: o.Hash,
		Name: name,
	}

	return nil
}

type progressReader struct {
	file         files.File
	out          chan interface{}
	bytes        int64
	lastProgress int64
}

func (i *progressReader) Read(p []byte) (int, error) {
	n, err := i.file.Read(p)

	i.bytes += int64(n)
	if i.bytes-i.lastProgress >= progressReaderIncrement || err == io.EOF {
		i.lastProgress = i.bytes
		i.out <- &AddedObject{
			Name:  i.file.FileName(),
			Bytes: i.bytes,
		}
	}

	return n, err
}
<commit_msg>core/commands: Fixed 'add' progress bar sometimes not clearing progress bar output line<commit_after>package commands

import (
	"bytes"
	"errors"
	"fmt"
	"io"
	"os"
	"path"
	"strings"

	cmds "github.com/jbenet/go-ipfs/commands"
	files "github.com/jbenet/go-ipfs/commands/files"
	core "github.com/jbenet/go-ipfs/core"
	importer "github.com/jbenet/go-ipfs/importer"
	"github.com/jbenet/go-ipfs/importer/chunk"
	dag "github.com/jbenet/go-ipfs/merkledag"
	pinning "github.com/jbenet/go-ipfs/pin"
	ft "github.com/jbenet/go-ipfs/unixfs"
	u "github.com/jbenet/go-ipfs/util"

	"github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/cheggaaa/pb"
)

// Error indicating the max depth has been exceded.
var ErrDepthLimitExceeded = fmt.Errorf("depth limit exceeded")

// how many bytes of progress to wait before sending a progress update message
const progressReaderIncrement = 1024 * 256

const progressOptionName = "progress"

type AddedObject struct {
	Name  string
	Hash  string `json:",omitempty"`
	Bytes int64  `json:",omitempty"`
}

var AddCmd = &cmds.Command{
	Helptext: cmds.HelpText{
		Tagline: "Add an object to ipfs.",
		ShortDescription: `
Adds contents of <path> to ipfs. Use -r to add directories.
Note that directories are added recursively, to form the ipfs
MerkleDAG. A smarter partial add with a staging area (like git)
remains to be implemented.
`,
	},

	Arguments: []cmds.Argument{
		cmds.FileArg("path", true, true, "The path to a file to be added to IPFS").EnableRecursive().EnableStdin(),
	},
	Options: []cmds.Option{
		cmds.OptionRecursivePath, // a builtin option that allows recursive paths (-r, --recursive)
		cmds.BoolOption("quiet", "q", "Write minimal output"),
		cmds.BoolOption(progressOptionName, "p", "Stream progress data"),
	},
	PreRun: func(req cmds.Request) error {
		req.SetOption(progressOptionName, true)

		sizeFile, ok := req.Files().(files.SizeFile)
		if !ok {
			// we don't need to error, the progress bar just won't know how big the files are
			return nil
		}

		size, err := sizeFile.Size()
		if err != nil {
			// see comment above
			return nil
		}
		log.Debugf("Total size of file being added: %v\n", size)
		req.Values()["size"] = size

		return nil
	},
	Run: func(req cmds.Request, res cmds.Response) {
		n, err := req.Context().GetNode()
		if err != nil {
			res.SetError(err, cmds.ErrNormal)
			return
		}

		progress, _, _ := req.Option(progressOptionName).Bool()

		outChan := make(chan interface{})
		res.SetOutput((<-chan interface{})(outChan))

		go func() {
			defer close(outChan)

			for {
				file, err := req.Files().NextFile()
				if (err != nil && err != io.EOF) || file == nil {
					return
				}

				_, err = addFile(n, file, outChan, progress)
				if err != nil {
					return
				}
			}
		}()
	},
	PostRun: func(res cmds.Response) {
		outChan, ok := res.Output().(<-chan interface{})
		if !ok {
			res.SetError(u.ErrCast(), cmds.ErrNormal)
			return
		}

		wrapperChan := make(chan interface{})
		res.SetOutput((<-chan interface{})(wrapperChan))

		size := int64(0)
		s, found := res.Request().Values()["size"]
		if found {
			size = s.(int64)
		}
		showProgressBar := size >= progressBarMinSize

		var bar *pb.ProgressBar
		var terminalWidth int
		if showProgressBar {
			bar = pb.New64(size).SetUnits(pb.U_BYTES)
			bar.ManualUpdate = true
			bar.Start()

			// the progress bar lib doesn't give us a way to get the width of the output,
			// so as a hack we just use a callback to measure the output, then git rid of it
			terminalWidth = 0
			bar.Callback = func(line string) {
				terminalWidth = len(line)
				bar.Callback = nil
				bar.Output = os.Stderr
				log.Infof("terminal width: %v\n", terminalWidth)
			}
			bar.Update()
		}

		go func() {
			lastFile := ""
			var totalProgress, prevFiles, lastBytes int64

			for out := range outChan {
				output := out.(*AddedObject)
				if len(output.Hash) > 0 {
					if showProgressBar {
						// clear progress bar line before we print "added x" output
						fmt.Fprintf(os.Stderr, "\r%s\r", strings.Repeat(" ", terminalWidth))
					}
					wrapperChan <- output

				} else {
					log.Debugf("add progress: %v %v\n", output.Name, output.Bytes)

					if !showProgressBar {
						continue
					}

					if len(lastFile) == 0 {
						lastFile = output.Name
					}
					if output.Name != lastFile || output.Bytes < lastBytes {
						prevFiles += lastBytes
						lastFile = output.Name
					}
					lastBytes = output.Bytes
					delta := prevFiles + lastBytes - totalProgress
					totalProgress = bar.Add64(delta)
				}

				if showProgressBar {
					bar.Update()
				}
			}

			close(wrapperChan)
		}()
	},
	Marshalers: cmds.MarshalerMap{
		cmds.Text: func(res cmds.Response) (io.Reader, error) {
			outChan, ok := res.Output().(<-chan interface{})
			if !ok {
				return nil, u.ErrCast()
			}

			quiet, _, err := res.Request().Option("quiet").Bool()
			if err != nil {
				return nil, err
			}

			marshal := func(v interface{}) (io.Reader, error) {
				obj, ok := v.(*AddedObject)
				if !ok {
					return nil, u.ErrCast()
				}

				var buf bytes.Buffer
				if quiet {
					buf.WriteString(fmt.Sprintf("%s\n", obj.Hash))
				} else {
					buf.WriteString(fmt.Sprintf("added %s %s\n", obj.Hash, obj.Name))
				}
				return &buf, nil
			}

			return &cmds.ChannelMarshaler{
				Channel:   outChan,
				Marshaler: marshal,
			}, nil
		},
	},
	Type: AddedObject{},
}

func add(n *core.IpfsNode, readers []io.Reader) ([]*dag.Node, error) {
	mp, ok := n.Pinning.(pinning.ManualPinner)
	if !ok {
		return nil, errors.New("invalid pinner type! expected manual pinner")
	}

	dagnodes := make([]*dag.Node, 0)

	for _, reader := range readers {
		node, err := importer.BuildDagFromReader(reader, n.DAG, mp, chunk.DefaultSplitter)
		if err != nil {
			return nil, err
		}
		dagnodes = append(dagnodes, node)
	}

	err := n.Pinning.Flush()
	if err != nil {
		return nil, err
	}

	return dagnodes, nil
}

func addNode(n *core.IpfsNode, node *dag.Node) error {
	err := n.DAG.AddRecursive(node) // add the file to the graph + local storage
	if err != nil {
		return err
	}

	err = n.Pinning.Pin(node, true) // ensure we keep it
	if err != nil {
		return err
	}

	return nil
}

func addFile(n *core.IpfsNode, file files.File, out chan interface{}, progress bool) (*dag.Node, error) {
	if file.IsDirectory() {
		return addDir(n, file, out, progress)
	}

	// if the progress flag was specified, wrap the file so that we can send
	// progress updates to the client (over the output channel)
	var reader io.Reader = file
	if progress {
		reader = &progressReader{file: file, out: out}
	}

	dns, err := add(n, []io.Reader{reader})
	if err != nil {
		return nil, err
	}

	log.Infof("adding file: %s", file.FileName())
	if err := outputDagnode(out, file.FileName(), dns[len(dns)-1]); err != nil {
		return nil, err
	}
	return dns[len(dns)-1], nil // last dag node is the file.
}

func addDir(n *core.IpfsNode, dir files.File, out chan interface{}, progress bool) (*dag.Node, error) {
	log.Infof("adding directory: %s", dir.FileName())

	tree := &dag.Node{Data: ft.FolderPBData()}

	for {
		file, err := dir.NextFile()
		if err != nil && err != io.EOF {
			return nil, err
		}
		if file == nil {
			break
		}

		node, err := addFile(n, file, out, progress)
		if err != nil {
			return nil, err
		}

		_, name := path.Split(file.FileName())

		err = tree.AddNodeLink(name, node)
		if err != nil {
			return nil, err
		}
	}

	err := outputDagnode(out, dir.FileName(), tree)
	if err != nil {
		return nil, err
	}

	err = addNode(n, tree)
	if err != nil {
		return nil, err
	}

	return tree, nil
}

// outputDagnode sends dagnode info over the output channel
func outputDagnode(out chan interface{}, name string, dn *dag.Node) error {
	o, err := getOutput(dn)
	if err != nil {
		return err
	}

	out <- &AddedObject{
		Hash: o.Hash,
		Name: name,
	}

	return nil
}

type progressReader struct {
	file         files.File
	out          chan interface{}
	bytes        int64
	lastProgress int64
}

func (i *progressReader) Read(p []byte) (int, error) {
	n, err := i.file.Read(p)

	i.bytes += int64(n)
	if i.bytes-i.lastProgress >= progressReaderIncrement || err == io.EOF {
		i.lastProgress = i.bytes
		i.out <- &AddedObject{
			Name:  i.file.FileName(),
			Bytes: i.bytes,
		}
	}

	return n, err
}
<|endoftext|>