<commit_before>package db

//go:generate msgp

import (
	"time"

	blake2b "github.com/minio/blake2b-simd"
)

// StringHeapID maps to a stored string identifier.
//
// This creates a layer of indirection when rebuilding items but
// saves on space for ids
type StringHeapID uint32

// StringHeapIDFromBytes generats the corresponding heap id
// from the provided bytes
func StringHeapIDFromBytes(bytes []byte) StringHeapID {
	return StringHeapID(btoi32(bytes))
}

// ToBytes returns the byte-wise represenation of a StringHeapID
func (id StringHeapID) ToBytes() []byte {
	return i32tob(uint32(id))
}

// LeagueHeapID maps to a stored string identifier specific to league
//
// This is basically StringHeapID but specialised for leagues
type LeagueHeapID uint16

// LeagueHeapIDFromSequence transforms a 64 bit bucket sequence number
// into a LeagueHeapID
func LeagueHeapIDFromSequence(seq uint64) LeagueHeapID {
	return LeagueHeapID(int16(seq))
}

// LeagueHeapIDFromBytes generats the corresponding heap id
// from the provided bytes
func LeagueHeapIDFromBytes(bytes []byte) LeagueHeapID {
	return LeagueHeapID(btoi16(bytes))
}

// ToBytes returns the byte-wise represenation of a LeagueHeapID
func (id LeagueHeapID) ToBytes() []byte {
	return i16tob(uint16(id))
}

// TimestampSize is the number of bytes used by Timestamp
//
// This is sized to minimize waste while
const TimestampSize = 4

// Timestamp is a compact represenation of a unix timestamp
type Timestamp [TimestampSize]byte

// NewTimestamp returns a Timestamp at the current time
func NewTimestamp() Timestamp {
	now := time.Now().Unix()

	nowBytes := i64tob(uint64(now))
	nowTrunc := nowBytes[TimestampSize:]

	var ts Timestamp
	copy(ts[:], nowTrunc)
	return ts
}

// ToTime converts a compact Timestamp to a time.Time
func (ts Timestamp) ToTime() time.Time {
	// Size the initial array with preceding zeroes
	fatBytes := make([]byte, 8-TimestampSize)
	// Jam the compact portion on
	fatBytes = append(fatBytes, ts[:]...)
	fatUint := btoi64(fatBytes)
	return time.Unix(int64(fatUint), 0)
}

// GGGIDSize is the size in bytes a derived ID can be
const GGGIDSize = 10

// GGGID is an Identifier derived from per-item/stash tab UID
//
// A PID is 80 bits = 10 bytes,
// this allows 2^40 items to be represented taking into birthdays
// and represents significant savings relative to the GGG api provided id
type GGGID [GGGIDSize]byte

// GGGIDFromUID generates an ID for internal use from a UID string
func GGGIDFromUID(uid string) GGGID {

	var id [GGGIDSize]byte

	hash := blake2b.Sum512([]byte(uid))

	copy(id[:], hash[:])

	return id
}

// IDSize is the size in bytes an internal identifier can be
const IDSize = 8

// ID is an Identifier calculated internally
//
// This is effectively just a 64 bit uint
type ID [IDSize]byte

// IDFromSequence converts a sequence number into an identifier
func IDFromSequence(seq uint64) ID {
	var id [IDSize]byte
	bin := i64tob(seq)
	copy(id[:], bin)

	return id
}

// ItemModAverageScaleFactor is the multiplier applied to the average
// of multi-mod items. This allows a fixed precision of averages.
//
// Any more than 10 and we could risk overflowing our uint16
const ItemModAverageScaleFactor = 10

// ItemMod represents a compact explicit or implicit modifier on an item
//msgp:tuple ItemMod
type ItemMod struct {
	Mod   StringHeapID
	Value uint16
}

// Item represents a compact record of an item.
//msgp:tuple Item
type Item struct {
	ID             ID
	GGGID          GGGID        // Allows mapping from simple ID to UUID
	Stash          GGGID        // Allows access to stash and corresponding metadata
	Name           StringHeapID // On StringHeap
	TypeLine       StringHeapID // On StringHeap
	Note           StringHeapID // On StringHeap
	RootType       StringHeapID // On StringHeap
	RootFlavor     StringHeapID // On StringHeap
	League         LeagueHeapID // On LeagueHeap
	Corrupted      bool
	Identified     bool
	Mods           []ItemMod
	When           Timestamp // When this stash update was processed
	UpdateSequence uint16    // The sequence number associated with this item
}

// Stash represents a compact record of a stash.
//msgp:tuple Stash
type Stash struct {
	ID          GGGID        // Reference value for this Stash
	AccountName string       // Account-wide name, we need nothing else to PM
	Items       []GGGID      // GGGIDs for all items stored in that Stash
	League      LeagueHeapID // LeagueHeapID as stashes are single-league
}

// Diff takes an older version of a Stash and determines which items,
// in terms of GGGID, need to be added and which need to be removed.
func (s Stash) Diff(old Stash) (add, remove []GGGID) {
	// Keep track of which items existed previously
	existing := make(map[GGGID]struct{})
	for _, id := range old.Items {
		existing[id] = struct{}{}
	}

	// Intersect the new Stash from the old
	add = make([]GGGID, 0)
	for _, id := range s.Items {
		// Check if item already exists, if it doesn't, we need to add it
		if _, ok := existing[id]; ok {
			// Remove it from the existing if it exist
			//
			// This will allow us to take the remaining items in existing
			// as those that are not found or shared in the new update and
			// then remove them.
			delete(existing, id)
		} else {
			// We need to add any items not found
			add = append(add, id)
		}
	}

	// Pull out all the remaining keys in existing to find the items that
	// are no longer present in the new update and need to be removed.
	remove = make([]GGGID, len(existing))[:0]
	for id := range existing {
		remove = append(remove, id)
	}

	return
}
<commit_msg>db introduce TimeToTimestamp<commit_after>package db

//go:generate msgp

import (
	"time"

	blake2b "github.com/minio/blake2b-simd"
)

// StringHeapID maps to a stored string identifier.
//
// This creates a layer of indirection when rebuilding items but
// saves on space for ids
type StringHeapID uint32

// StringHeapIDFromBytes generats the corresponding heap id
// from the provided bytes
func StringHeapIDFromBytes(bytes []byte) StringHeapID {
	return StringHeapID(btoi32(bytes))
}

// ToBytes returns the byte-wise represenation of a StringHeapID
func (id StringHeapID) ToBytes() []byte {
	return i32tob(uint32(id))
}

// LeagueHeapID maps to a stored string identifier specific to league
//
// This is basically StringHeapID but specialised for leagues
type LeagueHeapID uint16

// LeagueHeapIDFromSequence transforms a 64 bit bucket sequence number
// into a LeagueHeapID
func LeagueHeapIDFromSequence(seq uint64) LeagueHeapID {
	return LeagueHeapID(int16(seq))
}

// LeagueHeapIDFromBytes generats the corresponding heap id
// from the provided bytes
func LeagueHeapIDFromBytes(bytes []byte) LeagueHeapID {
	return LeagueHeapID(btoi16(bytes))
}

// ToBytes returns the byte-wise represenation of a LeagueHeapID
func (id LeagueHeapID) ToBytes() []byte {
	return i16tob(uint16(id))
}

// TimestampSize is the number of bytes used by Timestamp
//
// This is sized to minimize waste while
const TimestampSize = 4

// Timestamp is a compact represenation of a unix timestamp
type Timestamp [TimestampSize]byte

// NewTimestamp returns a Timestamp at the current time
func NewTimestamp() Timestamp {
	return TimeToTimestamp(time.Now())
}

// TimeToTimestamp returns a Timestamp representing the passed time.Time
func TimeToTimestamp(when time.Time) Timestamp {
	secs := when.Unix()

	nowBytes := i64tob(uint64(secs))
	nowTrunc := nowBytes[TimestampSize:]

	var ts Timestamp
	copy(ts[:], nowTrunc)
	return ts
}

// ToTime converts a compact Timestamp to a time.Time
func (ts Timestamp) ToTime() time.Time {
	// Size the initial array with preceding zeroes
	fatBytes := make([]byte, 8-TimestampSize)
	// Jam the compact portion on
	fatBytes = append(fatBytes, ts[:]...)
	fatUint := btoi64(fatBytes)
	return time.Unix(int64(fatUint), 0)
}

// GGGIDSize is the size in bytes a derived ID can be
const GGGIDSize = 10

// GGGID is an Identifier derived from per-item/stash tab UID
//
// A PID is 80 bits = 10 bytes,
// this allows 2^40 items to be represented taking into birthdays
// and represents significant savings relative to the GGG api provided id
type GGGID [GGGIDSize]byte

// GGGIDFromUID generates an ID for internal use from a UID string
func GGGIDFromUID(uid string) GGGID {

	var id [GGGIDSize]byte

	hash := blake2b.Sum512([]byte(uid))

	copy(id[:], hash[:])

	return id
}

// IDSize is the size in bytes an internal identifier can be
const IDSize = 8

// ID is an Identifier calculated internally
//
// This is effectively just a 64 bit uint
type ID [IDSize]byte

// IDFromSequence converts a sequence number into an identifier
func IDFromSequence(seq uint64) ID {
	var id [IDSize]byte
	bin := i64tob(seq)
	copy(id[:], bin)

	return id
}

// ItemModAverageScaleFactor is the multiplier applied to the average
// of multi-mod items. This allows a fixed precision of averages.
//
// Any more than 10 and we could risk overflowing our uint16
const ItemModAverageScaleFactor = 10

// ItemMod represents a compact explicit or implicit modifier on an item
//msgp:tuple ItemMod
type ItemMod struct {
	Mod   StringHeapID
	Value uint16
}

// Item represents a compact record of an item.
//msgp:tuple Item
type Item struct {
	ID             ID
	GGGID          GGGID        // Allows mapping from simple ID to UUID
	Stash          GGGID        // Allows access to stash and corresponding metadata
	Name           StringHeapID // On StringHeap
	TypeLine       StringHeapID // On StringHeap
	Note           StringHeapID // On StringHeap
	RootType       StringHeapID // On StringHeap
	RootFlavor     StringHeapID // On StringHeap
	League         LeagueHeapID // On LeagueHeap
	Corrupted      bool
	Identified     bool
	Mods           []ItemMod
	When           Timestamp // When this stash update was processed
	UpdateSequence uint16    // The sequence number associated with this item
}

// Stash represents a compact record of a stash.
//msgp:tuple Stash
type Stash struct {
	ID          GGGID        // Reference value for this Stash
	AccountName string       // Account-wide name, we need nothing else to PM
	Items       []GGGID      // GGGIDs for all items stored in that Stash
	League      LeagueHeapID // LeagueHeapID as stashes are single-league
}

// Diff takes an older version of a Stash and determines which items,
// in terms of GGGID, need to be added and which need to be removed.
func (s Stash) Diff(old Stash) (add, remove []GGGID) {
	// Keep track of which items existed previously
	existing := make(map[GGGID]struct{})
	for _, id := range old.Items {
		existing[id] = struct{}{}
	}

	// Intersect the new Stash from the old
	add = make([]GGGID, 0)
	for _, id := range s.Items {
		// Check if item already exists, if it doesn't, we need to add it
		if _, ok := existing[id]; ok {
			// Remove it from the existing if it exist
			//
			// This will allow us to take the remaining items in existing
			// as those that are not found or shared in the new update and
			// then remove them.
			delete(existing, id)
		} else {
			// We need to add any items not found
			add = append(add, id)
		}
	}

	// Pull out all the remaining keys in existing to find the items that
	// are no longer present in the new update and need to be removed.
	remove = make([]GGGID, len(existing))[:0]
	for id := range existing {
		remove = append(remove, id)
	}

	return
}
<|endoftext|>


<commit_before>package command

import (
	"bytes"
	"fmt"
	"log"
	"os"
	"os/signal"
	"strconv"
	"strings"
	"sync"

	"github.com/hashicorp/packer/helper/enumflag"
	"github.com/hashicorp/packer/packer"
	"github.com/hashicorp/packer/template"
)

type BuildCommand struct {
	Meta
}

func (c BuildCommand) Run(args []string) int {
	var cfgColor, cfgDebug, cfgForce, cfgParallel bool
	var cfgOnError string
	flags := c.Meta.FlagSet("build", FlagSetBuildFilter|FlagSetVars)
	flags.Usage = func() { c.Ui.Say(c.Help()) }
	flags.BoolVar(&cfgColor, "color", true, "")
	flags.BoolVar(&cfgDebug, "debug", false, "")
	flags.BoolVar(&cfgForce, "force", false, "")
	flagOnError := enumflag.New(&cfgOnError, "cleanup", "abort", "ask")
	flags.Var(flagOnError, "on-error", "")
	flags.BoolVar(&cfgParallel, "parallel", true, "")
	if err := flags.Parse(args); err != nil {
		return 1
	}

	args = flags.Args()
	if len(args) != 1 {
		flags.Usage()
		return 1
	}

	// Parse the template
	var tpl *template.Template
	var err error
	tpl, err = template.ParseFile(args[0])
	if err != nil {
		c.Ui.Error(fmt.Sprintf("Failed to parse template: %s", err))
		return 1
	}

	// Get the core
	core, err := c.Meta.Core(tpl)
	if err != nil {
		c.Ui.Error(err.Error())
		return 1
	}

	// Get the builds we care about
	buildNames := c.Meta.BuildNames(core)
	builds := make([]packer.Build, 0, len(buildNames))
	for _, n := range buildNames {
		b, err := core.Build(n)
		if err != nil {
			c.Ui.Error(fmt.Sprintf(
				"Failed to initialize build '%s': %s",
				n, err))
			continue
		}

		builds = append(builds, b)
	}

	if cfgDebug {
		c.Ui.Say("Debug mode enabled. Builds will not be parallelized.")
	}

	// Compile all the UIs for the builds
	colors := [5]packer.UiColor{
		packer.UiColorGreen,
		packer.UiColorCyan,
		packer.UiColorMagenta,
		packer.UiColorYellow,
		packer.UiColorBlue,
	}
	buildUis := make(map[string]packer.Ui)
	for i, b := range buildNames {
		var ui packer.Ui
		ui = c.Ui
		if cfgColor {
			ui = &packer.ColoredUi{
				Color: colors[i%len(colors)],
				Ui:    ui,
			}
			if _, ok := c.Ui.(*packer.MachineReadableUi); !ok {
				ui.Say(fmt.Sprintf("%s output will be in this color.", b))
				if i+1 == len(buildNames) {
					// Add a newline between the color output and the actual output
					c.Ui.Say("")
				}
			}
		}

		buildUis[b] = ui
	}

	log.Printf("Build debug mode: %v", cfgDebug)
	log.Printf("Force build: %v", cfgForce)
	log.Printf("On error: %v", cfgOnError)

	// Set the debug and force mode and prepare all the builds
	for _, b := range builds {
		log.Printf("Preparing build: %s", b.Name())
		b.SetDebug(cfgDebug)
		b.SetForce(cfgForce)
		b.SetOnError(cfgOnError)

		warnings, err := b.Prepare()
		if err != nil {
			c.Ui.Error(err.Error())
			return 1
		}
		if len(warnings) > 0 {
			ui := buildUis[b.Name()]
			ui.Say(fmt.Sprintf("Warnings for build '%s':\n", b.Name()))
			for _, warning := range warnings {
				ui.Say(fmt.Sprintf("* %s", warning))
			}
			ui.Say("")
		}
	}

	// Run all the builds in parallel and wait for them to complete
	var interruptWg, wg sync.WaitGroup
	interrupted := false
	var artifacts = struct {
		sync.RWMutex
		m map[string][]packer.Artifact
	}{m: make(map[string][]packer.Artifact)}
	errors := make(map[string]error)
	for _, b := range builds {
		// Increment the waitgroup so we wait for this item to finish properly
		wg.Add(1)

		// Handle interrupts for this build
		sigCh := make(chan os.Signal, 1)
		signal.Notify(sigCh, os.Interrupt)
		defer signal.Stop(sigCh)
		go func(b packer.Build) {
			<-sigCh
			interruptWg.Add(1)
			defer interruptWg.Done()
			interrupted = true

			log.Printf("Stopping build: %s", b.Name())
			b.Cancel()
			log.Printf("Build cancelled: %s", b.Name())
		}(b)

		// Run the build in a goroutine
		go func(b packer.Build) {
			defer wg.Done()

			name := b.Name()
			log.Printf("Starting build run: %s", name)
			ui := buildUis[name]
			runArtifacts, err := b.Run(ui, c.Cache)

			if err != nil {
				ui.Error(fmt.Sprintf("Build '%s' errored: %s", name, err))
				errors[name] = err
			} else {
				ui.Say(fmt.Sprintf("Build '%s' finished.", name))
				artifacts.Lock()
				artifacts.m[name] = runArtifacts
				artifacts.Unlock()
			}
		}(b)

		if cfgDebug {
			log.Printf("Debug enabled, so waiting for build to finish: %s", b.Name())
			wg.Wait()
		}

		if !cfgParallel {
			log.Printf("Parallelization disabled, waiting for build to finish: %s", b.Name())
			wg.Wait()
		}

		if interrupted {
			log.Println("Interrupted, not going to start any more builds.")
			break
		}
	}

	// Wait for both the builds to complete and the interrupt handler,
	// if it is interrupted.
	log.Printf("Waiting on builds to complete...")
	wg.Wait()

	log.Printf("Builds completed. Waiting on interrupt barrier...")
	interruptWg.Wait()

	if interrupted {
		c.Ui.Say("Cleanly cancelled builds after being interrupted.")
		return 1
	}

	if len(errors) > 0 {
		c.Ui.Machine("error-count", strconv.FormatInt(int64(len(errors)), 10))

		c.Ui.Error("\n==> Some builds didn't complete successfully and had errors:")
		for name, err := range errors {
			// Create a UI for the machine readable stuff to be targeted
			ui := &packer.TargetedUI{
				Target: name,
				Ui:     c.Ui,
			}

			ui.Machine("error", err.Error())

			c.Ui.Error(fmt.Sprintf("--> %s: %s", name, err))
		}
	}

	if len(artifacts.m) > 0 {
		c.Ui.Say("\n==> Builds finished. The artifacts of successful builds are:")
		for name, buildArtifacts := range artifacts.m {
			// Create a UI for the machine readable stuff to be targeted
			ui := &packer.TargetedUI{
				Target: name,
				Ui:     c.Ui,
			}

			// Machine-readable helpful
			ui.Machine("artifact-count", strconv.FormatInt(int64(len(buildArtifacts)), 10))

			for i, artifact := range buildArtifacts {
				var message bytes.Buffer
				fmt.Fprintf(&message, "--> %s: ", name)

				if artifact != nil {
					fmt.Fprint(&message, artifact.String())
				} else {
					fmt.Fprint(&message, "<nothing>")
				}

				iStr := strconv.FormatInt(int64(i), 10)
				if artifact != nil {
					ui.Machine("artifact", iStr, "builder-id", artifact.BuilderId())
					ui.Machine("artifact", iStr, "id", artifact.Id())
					ui.Machine("artifact", iStr, "string", artifact.String())

					files := artifact.Files()
					ui.Machine("artifact",
						iStr,
						"files-count", strconv.FormatInt(int64(len(files)), 10))
					for fi, file := range files {
						fiStr := strconv.FormatInt(int64(fi), 10)
						ui.Machine("artifact", iStr, "file", fiStr, file)
					}
				} else {
					ui.Machine("artifact", iStr, "nil")
				}

				ui.Machine("artifact", iStr, "end")
				c.Ui.Say(message.String())
			}
		}
	} else {
		c.Ui.Say("\n==> Builds finished but no artifacts were created.")
	}

	if len(errors) > 0 {
		// If any errors occurred, exit with a non-zero exit status
		return 1
	}

	return 0
}

func (BuildCommand) Help() string {
	helpText := `
Usage: packer build [options] TEMPLATE

  Will execute multiple builds in parallel as defined in the template.
  The various artifacts created by the template will be outputted.

Options:

  -color=false               Disable color output (on by default)
  -debug                     Debug mode enabled for builds
  -except=foo,bar,baz        Build all builds other than these
  -only=foo,bar,baz          Build only the specified builds
  -force                     Force a build to continue if artifacts exist, deletes existing artifacts
  -machine-readable          Machine-readable output
  -on-error=[cleanup|abort|ask] If the build fails do: clean up (default), abort, or ask
  -parallel=false            Disable parallelization (on by default)
  -var 'key=value'           Variable for templates, can be used multiple times.
  -var-file=path             JSON file containing user variables.
`

	return strings.TrimSpace(helpText)
}

func (BuildCommand) Synopsis() string {
	return "build image(s) from template"
}
<commit_msg>Define methods on *BuildCommand (consistency)<commit_after>package command

import (
	"bytes"
	"fmt"
	"log"
	"os"
	"os/signal"
	"strconv"
	"strings"
	"sync"

	"github.com/hashicorp/packer/helper/enumflag"
	"github.com/hashicorp/packer/packer"
	"github.com/hashicorp/packer/template"
)

type BuildCommand struct {
	Meta
}

func (c *BuildCommand) Run(args []string) int {
	var cfgColor, cfgDebug, cfgForce, cfgParallel bool
	var cfgOnError string
	flags := c.Meta.FlagSet("build", FlagSetBuildFilter|FlagSetVars)
	flags.Usage = func() { c.Ui.Say(c.Help()) }
	flags.BoolVar(&cfgColor, "color", true, "")
	flags.BoolVar(&cfgDebug, "debug", false, "")
	flags.BoolVar(&cfgForce, "force", false, "")
	flagOnError := enumflag.New(&cfgOnError, "cleanup", "abort", "ask")
	flags.Var(flagOnError, "on-error", "")
	flags.BoolVar(&cfgParallel, "parallel", true, "")
	if err := flags.Parse(args); err != nil {
		return 1
	}

	args = flags.Args()
	if len(args) != 1 {
		flags.Usage()
		return 1
	}

	// Parse the template
	var tpl *template.Template
	var err error
	tpl, err = template.ParseFile(args[0])
	if err != nil {
		c.Ui.Error(fmt.Sprintf("Failed to parse template: %s", err))
		return 1
	}

	// Get the core
	core, err := c.Meta.Core(tpl)
	if err != nil {
		c.Ui.Error(err.Error())
		return 1
	}

	// Get the builds we care about
	buildNames := c.Meta.BuildNames(core)
	builds := make([]packer.Build, 0, len(buildNames))
	for _, n := range buildNames {
		b, err := core.Build(n)
		if err != nil {
			c.Ui.Error(fmt.Sprintf(
				"Failed to initialize build '%s': %s",
				n, err))
			continue
		}

		builds = append(builds, b)
	}

	if cfgDebug {
		c.Ui.Say("Debug mode enabled. Builds will not be parallelized.")
	}

	// Compile all the UIs for the builds
	colors := [5]packer.UiColor{
		packer.UiColorGreen,
		packer.UiColorCyan,
		packer.UiColorMagenta,
		packer.UiColorYellow,
		packer.UiColorBlue,
	}
	buildUis := make(map[string]packer.Ui)
	for i, b := range buildNames {
		var ui packer.Ui
		ui = c.Ui
		if cfgColor {
			ui = &packer.ColoredUi{
				Color: colors[i%len(colors)],
				Ui:    ui,
			}
			if _, ok := c.Ui.(*packer.MachineReadableUi); !ok {
				ui.Say(fmt.Sprintf("%s output will be in this color.", b))
				if i+1 == len(buildNames) {
					// Add a newline between the color output and the actual output
					c.Ui.Say("")
				}
			}
		}

		buildUis[b] = ui
	}

	log.Printf("Build debug mode: %v", cfgDebug)
	log.Printf("Force build: %v", cfgForce)
	log.Printf("On error: %v", cfgOnError)

	// Set the debug and force mode and prepare all the builds
	for _, b := range builds {
		log.Printf("Preparing build: %s", b.Name())
		b.SetDebug(cfgDebug)
		b.SetForce(cfgForce)
		b.SetOnError(cfgOnError)

		warnings, err := b.Prepare()
		if err != nil {
			c.Ui.Error(err.Error())
			return 1
		}
		if len(warnings) > 0 {
			ui := buildUis[b.Name()]
			ui.Say(fmt.Sprintf("Warnings for build '%s':\n", b.Name()))
			for _, warning := range warnings {
				ui.Say(fmt.Sprintf("* %s", warning))
			}
			ui.Say("")
		}
	}

	// Run all the builds in parallel and wait for them to complete
	var interruptWg, wg sync.WaitGroup
	interrupted := false
	var artifacts = struct {
		sync.RWMutex
		m map[string][]packer.Artifact
	}{m: make(map[string][]packer.Artifact)}
	errors := make(map[string]error)
	for _, b := range builds {
		// Increment the waitgroup so we wait for this item to finish properly
		wg.Add(1)

		// Handle interrupts for this build
		sigCh := make(chan os.Signal, 1)
		signal.Notify(sigCh, os.Interrupt)
		defer signal.Stop(sigCh)
		go func(b packer.Build) {
			<-sigCh
			interruptWg.Add(1)
			defer interruptWg.Done()
			interrupted = true

			log.Printf("Stopping build: %s", b.Name())
			b.Cancel()
			log.Printf("Build cancelled: %s", b.Name())
		}(b)

		// Run the build in a goroutine
		go func(b packer.Build) {
			defer wg.Done()

			name := b.Name()
			log.Printf("Starting build run: %s", name)
			ui := buildUis[name]
			runArtifacts, err := b.Run(ui, c.Cache)

			if err != nil {
				ui.Error(fmt.Sprintf("Build '%s' errored: %s", name, err))
				errors[name] = err
			} else {
				ui.Say(fmt.Sprintf("Build '%s' finished.", name))
				artifacts.Lock()
				artifacts.m[name] = runArtifacts
				artifacts.Unlock()
			}
		}(b)

		if cfgDebug {
			log.Printf("Debug enabled, so waiting for build to finish: %s", b.Name())
			wg.Wait()
		}

		if !cfgParallel {
			log.Printf("Parallelization disabled, waiting for build to finish: %s", b.Name())
			wg.Wait()
		}

		if interrupted {
			log.Println("Interrupted, not going to start any more builds.")
			break
		}
	}

	// Wait for both the builds to complete and the interrupt handler,
	// if it is interrupted.
	log.Printf("Waiting on builds to complete...")
	wg.Wait()

	log.Printf("Builds completed. Waiting on interrupt barrier...")
	interruptWg.Wait()

	if interrupted {
		c.Ui.Say("Cleanly cancelled builds after being interrupted.")
		return 1
	}

	if len(errors) > 0 {
		c.Ui.Machine("error-count", strconv.FormatInt(int64(len(errors)), 10))

		c.Ui.Error("\n==> Some builds didn't complete successfully and had errors:")
		for name, err := range errors {
			// Create a UI for the machine readable stuff to be targeted
			ui := &packer.TargetedUI{
				Target: name,
				Ui:     c.Ui,
			}

			ui.Machine("error", err.Error())

			c.Ui.Error(fmt.Sprintf("--> %s: %s", name, err))
		}
	}

	if len(artifacts.m) > 0 {
		c.Ui.Say("\n==> Builds finished. The artifacts of successful builds are:")
		for name, buildArtifacts := range artifacts.m {
			// Create a UI for the machine readable stuff to be targeted
			ui := &packer.TargetedUI{
				Target: name,
				Ui:     c.Ui,
			}

			// Machine-readable helpful
			ui.Machine("artifact-count", strconv.FormatInt(int64(len(buildArtifacts)), 10))

			for i, artifact := range buildArtifacts {
				var message bytes.Buffer
				fmt.Fprintf(&message, "--> %s: ", name)

				if artifact != nil {
					fmt.Fprint(&message, artifact.String())
				} else {
					fmt.Fprint(&message, "<nothing>")
				}

				iStr := strconv.FormatInt(int64(i), 10)
				if artifact != nil {
					ui.Machine("artifact", iStr, "builder-id", artifact.BuilderId())
					ui.Machine("artifact", iStr, "id", artifact.Id())
					ui.Machine("artifact", iStr, "string", artifact.String())

					files := artifact.Files()
					ui.Machine("artifact",
						iStr,
						"files-count", strconv.FormatInt(int64(len(files)), 10))
					for fi, file := range files {
						fiStr := strconv.FormatInt(int64(fi), 10)
						ui.Machine("artifact", iStr, "file", fiStr, file)
					}
				} else {
					ui.Machine("artifact", iStr, "nil")
				}

				ui.Machine("artifact", iStr, "end")
				c.Ui.Say(message.String())
			}
		}
	} else {
		c.Ui.Say("\n==> Builds finished but no artifacts were created.")
	}

	if len(errors) > 0 {
		// If any errors occurred, exit with a non-zero exit status
		return 1
	}

	return 0
}

func (*BuildCommand) Help() string {
	helpText := `
Usage: packer build [options] TEMPLATE

  Will execute multiple builds in parallel as defined in the template.
  The various artifacts created by the template will be outputted.

Options:

  -color=false               Disable color output (on by default)
  -debug                     Debug mode enabled for builds
  -except=foo,bar,baz        Build all builds other than these
  -only=foo,bar,baz          Build only the specified builds
  -force                     Force a build to continue if artifacts exist, deletes existing artifacts
  -machine-readable          Machine-readable output
  -on-error=[cleanup|abort|ask] If the build fails do: clean up (default), abort, or ask
  -parallel=false            Disable parallelization (on by default)
  -var 'key=value'           Variable for templates, can be used multiple times.
  -var-file=path             JSON file containing user variables.
`

	return strings.TrimSpace(helpText)
}

func (*BuildCommand) Synopsis() string {
	return "build image(s) from template"
}
<|endoftext|>


<commit_before>// Copyright 2016 e-Xpert Solutions SA. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package diff

import (
	"reflect"
	"testing"
	"time"
)

type Foo struct {
	IntVal    int
	FloatVal  float32
	StringVal string
	Bar       Bar
	FooPtr    *Foo
	IntList   []int
	BarList   []Bar
}

type Bar struct {
	StringVal string
}

func TestCompute(t *testing.T) {
	f1 := Foo{
		IntVal:    42,
		FloatVal:  53.032,
		StringVal: "bar",
		Bar: Bar{
			StringVal: "ok",
		},
		FooPtr:  nil,
		IntList: []int{1, 3, 4},
		BarList: []Bar{{StringVal: "aaa"}, {StringVal: "bbb"}},
	}
	f2 := Foo{
		IntVal:    42,
		FloatVal:  53.042,
		StringVal: "baraca",
		Bar: Bar{
			StringVal: "ok",
		},
		FooPtr: &Foo{
			IntVal: 42,
		},
		IntList: []int{1, 2, 4, 5},
		BarList: []Bar{{StringVal: "ccc"}, {StringVal: "ddd"}},
	}
	delta, err := Compute(f1, f2)
	if err != nil {
		t.Fatal("Failed to compute diff: ", err)
	}
	t.Log(string(delta.PrettyJSON()))
}

func TestIsEqual(t *testing.T) {
	d1 := time.Date(2016, time.Month(6), 22, 10, 58, 52, 42, time.Local)
	d2 := time.Date(2016, time.Month(6), 22, 10, 58, 52, 42, time.Local)
	x, y := reflect.ValueOf(d1), reflect.ValueOf(d2)
	if equal := isEqual(x, y); !equal {
		t.Errorf("isEqual('%v', '%v'): found 'false', expected 'true'", d1, d2)
	}

	d1 = time.Date(2016, time.Month(6), 22, 10, 58, 52, 42, time.Local)
	d2 = time.Date(2016, time.Month(6), 22, 10, 58, 52, 24, time.Local)
	x, y = reflect.ValueOf(d1), reflect.ValueOf(d2)
	if equal := isEqual(x, y); equal {
		t.Errorf("isEqual('%v', '%v'): found 'true', expected 'false'", d1, d2)
	}
}
<commit_msg>diff: add test case for the isFullyNonExportedStruct function<commit_after>// Copyright 2016 e-Xpert Solutions SA. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package diff

import (
	"reflect"
	"testing"
	"time"
)

type Foo struct {
	IntVal    int
	FloatVal  float32
	StringVal string
	Bar       Bar
	FooPtr    *Foo
	IntList   []int
	BarList   []Bar
}

type Bar struct {
	StringVal string
}

func TestCompute(t *testing.T) {
	f1 := Foo{
		IntVal:    42,
		FloatVal:  53.032,
		StringVal: "bar",
		Bar: Bar{
			StringVal: "ok",
		},
		FooPtr:  nil,
		IntList: []int{1, 3, 4},
		BarList: []Bar{{StringVal: "aaa"}, {StringVal: "bbb"}},
	}
	f2 := Foo{
		IntVal:    42,
		FloatVal:  53.042,
		StringVal: "baraca",
		Bar: Bar{
			StringVal: "ok",
		},
		FooPtr: &Foo{
			IntVal: 42,
		},
		IntList: []int{1, 2, 4, 5},
		BarList: []Bar{{StringVal: "ccc"}, {StringVal: "ddd"}},
	}
	delta, err := Compute(f1, f2)
	if err != nil {
		t.Fatal("Failed to compute diff: ", err)
	}
	t.Log(string(delta.PrettyJSON()))
}

func TestIsFullyNonExportedStruct(t *testing.T) {
	type Foo struct {
		a int
		b int
	}
	f := Foo{a: 42, b: 42}
	if !isFullyNonExportedStruct(reflect.ValueOf(f)) {
		t.Errorf("isFullyNonExportedStruct(Foo{a: 42, b:42}): found 'false', expected 'true'")
	}

	type Bar struct {
		A int
		b int
	}
	b := Bar{A: 42, b: 42}
	if isFullyNonExportedStruct(reflect.ValueOf(b)) {
		t.Errorf("isFullyNonExportedStruct(Bar{A: 42, b:42}): found 'true', expected 'false'")
	}
}

func TestIsEqual(t *testing.T) {
	d1 := time.Date(2016, time.Month(6), 22, 10, 58, 52, 42, time.Local)
	d2 := time.Date(2016, time.Month(6), 22, 10, 58, 52, 42, time.Local)
	x, y := reflect.ValueOf(d1), reflect.ValueOf(d2)
	if equal := isEqual(x, y); !equal {
		t.Errorf("isEqual('%v', '%v'): found 'false', expected 'true'", d1, d2)
	}

	d1 = time.Date(2016, time.Month(6), 22, 10, 58, 52, 42, time.Local)
	d2 = time.Date(2016, time.Month(6), 22, 10, 58, 52, 24, time.Local)
	x, y = reflect.ValueOf(d1), reflect.ValueOf(d2)
	if equal := isEqual(x, y); equal {
		t.Errorf("isEqual('%v', '%v'): found 'true', expected 'false'", d1, d2)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"log"
	"os/exec"
	"strings"
	"time"
)

const (
	waitTime  = 100 * time.Millisecond
	pauseTime = 10 * time.Millisecond
)

// Runner is the server that runs commands
type Runner struct {
	commands    []*runnerCommand
	runner      func(*runnerCommand)
	gopath      string
	running     bool
	lameDuck    bool
	commandsRun int
}

type runnerCommand struct {
	command    *exec.Cmd
	discard    bool
	output     string
	complete   bool
	background bool
}

func (r *Runner) run() {
	r.running = true

	for r.running {
		time.Sleep(pauseTime)
		if len(r.commands) > 0 {
			r.runner(r.commands[0])
			r.commands = r.commands[1:]
			r.commandsRun++
		}
	}
}

// BlockUntil blocks on this until the command has run
func (r *Runner) BlockUntil(command *runnerCommand) {
	for !command.complete {
		time.Sleep(waitTime)
	}
}

// LameDuck the server
func (r *Runner) LameDuck(shutdown bool) {
	r.lameDuck = true

	if shutdown {
		r.running = false
	}
}

func (r *Runner) addCommand(command *runnerCommand) {
	if !r.lameDuck {
		r.commands = append(r.commands, command)
	}
}

// Checkout a repo - returns the repo version
func (r *Runner) Checkout(repo string) string {
	log.Printf("CHECKOUT = %v", repo)
	r.addCommand(&runnerCommand{command: exec.Command("go", "get", "-u", repo)})
	readCommand := &runnerCommand{command: exec.Command("cat", "$GOPATH/src/"+repo+"/.git/refs/heads/master"), discard: false}
	r.addCommand(readCommand)
	r.BlockUntil(readCommand)
	return readCommand.output
}

// Run the specified server specified in the repo
func (r *Runner) Run(repo string) {
	elems := strings.Split(repo, "/")
	command := elems[len(elems)-1]
	com := &runnerCommand{command: exec.Command("$GOPATH/bin/" + command), background: true}
	r.addCommand(com)
	r.BlockUntil(com)
}
<commit_msg>Fixed cat command<commit_after>package main

import (
	"log"
	"os/exec"
	"strings"
	"time"
)

const (
	waitTime  = 100 * time.Millisecond
	pauseTime = 10 * time.Millisecond
)

// Runner is the server that runs commands
type Runner struct {
	commands    []*runnerCommand
	runner      func(*runnerCommand)
	gopath      string
	running     bool
	lameDuck    bool
	commandsRun int
}

type runnerCommand struct {
	command    *exec.Cmd
	discard    bool
	output     string
	complete   bool
	background bool
}

func (r *Runner) run() {
	r.running = true

	for r.running {
		time.Sleep(pauseTime)
		if len(r.commands) > 0 {
			r.runner(r.commands[0])
			r.commands = r.commands[1:]
			r.commandsRun++
		}
	}
}

// BlockUntil blocks on this until the command has run
func (r *Runner) BlockUntil(command *runnerCommand) {
	for !command.complete {
		time.Sleep(waitTime)
	}
}

// LameDuck the server
func (r *Runner) LameDuck(shutdown bool) {
	r.lameDuck = true

	if shutdown {
		r.running = false
	}
}

func (r *Runner) addCommand(command *runnerCommand) {
	if !r.lameDuck {
		r.commands = append(r.commands, command)
	}
}

// Checkout a repo - returns the repo version
func (r *Runner) Checkout(repo string) string {
	log.Printf("CHECKOUT = %v", repo)
	r.addCommand(&runnerCommand{command: exec.Command("go", "get", "-u", repo)})
	readCommand := &runnerCommand{command: exec.Command("cat", "$GOPATH/src/"+repo+"/.git/refs/heads/master"), discard: false}
	r.addCommand(readCommand)
	r.BlockUntil(readCommand)
	return readCommand.output
}

// Run the specified server specified in the repo
func (r *Runner) Run(repo string) {
	elems := strings.Split(repo, "/")
	command := elems[len(elems)-1]
	com := &runnerCommand{command: exec.Command("$GOPATH/bin/" + command), background: true}
	r.addCommand(com)
}
<|endoftext|>


<commit_before>package commands

import (
	"fmt"
	"os"
	"path/filepath"
	"sort"
	"strings"

	"github.com/github/hub/cmd"
	"github.com/github/hub/ui"
	"github.com/github/hub/utils"
)

var cmdHelp = &Command{
	Run:          runHelp,
	GitExtension: true,
	Usage: `
help hub
help <COMMAND>
help hub-<COMMAND> [--plain-text]
`,
	Long: `Show the help page for a command.

## Options:
	hub-<COMMAND>
		Use this format to view help for hub extensions to an existing git command.

	--plain-text
		Skip man page lookup mechanism and display plain help text.

## Lookup mechanism:

On systems that have 'man', help pages are looked up in these directories
relative to the hub install prefix:

* man/<command>.1
* share/man/man1/<command>.1

On systems without 'man', help pages are looked up using the ".txt" extension.

## See also:

hub(1), git-help(1)
`,
}

var cmdListCmds = &Command{
	Key:          "--list-cmds",
	Run:          runListCmds,
	GitExtension: true,
}

func init() {
	CmdRunner.Use(cmdHelp, "--help")
	CmdRunner.Use(cmdListCmds)
}

func runHelp(helpCmd *Command, args *Args) {
	if args.IsParamsEmpty() {
		args.AfterFn(func() error {
			ui.Println(helpText)
			return nil
		})
		return
	}

	p := utils.NewArgsParser()
	p.RegisterBool("--all", "-a")
	p.RegisterBool("--plain-text")
	p.Parse(args.Params)

	if p.Bool("--all") {
		args.AfterFn(func() error {
			ui.Printf("\nhub custom commands\n\n  %s\n", strings.Join(customCommands(), "  "))
			return nil
		})
		return
	}

	command := args.FirstParam()

	if command == "hub" {
		err := displayManPage("hub.1", args)
		if err != nil {
			utils.Check(err)
		}
	}

	if c := lookupCmd(command); c != nil {
		if !p.Bool("--plain-text") {
			manPage := fmt.Sprintf("hub-%s.1", c.Name())
			err := displayManPage(manPage, args)
			if err == nil {
				return
			}
		}

		ui.Println(c.HelpText())
		args.NoForward()
	}
}

func runListCmds(cmd *Command, args *Args) {
	listOthers := false
	parts := strings.SplitN(args.Command, "=", 2)
	for _, kind := range strings.Split(parts[1], ",") {
		if kind == "others" {
			listOthers = true
			break
		}
	}

	if listOthers {
		args.AfterFn(func() error {
			ui.Println(strings.Join(customCommands(), "\n"))
			return nil
		})
	}
}

func displayManPage(manPage string, args *Args) error {
	manProgram, _ := utils.CommandPath("man")
	if manProgram == "" {
		manPage += ".txt"
		manProgram = os.Getenv("PAGER")
		if manProgram == "" {
			manProgram = "less -R"
		}
	}

	programPath, err := utils.CommandPath(args.ProgramPath)
	if err != nil {
		return err
	}

	installPrefix := filepath.Join(filepath.Dir(programPath), "..")
	manFile, err := localManPage(manPage, installPrefix)
	if err != nil {
		return err
	}

	man := cmd.New(manProgram)
	man.WithArg(manFile)
	if err = man.Run(); err == nil {
		os.Exit(0)
	} else {
		os.Exit(1)
	}
	return nil
}

func localManPage(name, installPrefix string) (string, error) {
	manPath := filepath.Join(installPrefix, "man", name)
	_, err := os.Stat(manPath)
	if err == nil {
		return manPath, nil
	}

	manPath = filepath.Join(installPrefix, "share", "man", "man1", name)
	_, err = os.Stat(manPath)
	if err == nil {
		return manPath, nil
	} else {
		return "", err
	}
}

func lookupCmd(name string) *Command {
	if strings.HasPrefix(name, "hub-") {
		return CmdRunner.Lookup(strings.TrimPrefix(name, "hub-"))
	} else {
		cmd := CmdRunner.Lookup(name)
		if cmd != nil && !cmd.GitExtension {
			return cmd
		} else {
			return nil
		}
	}
}

func customCommands() []string {
	cmds := []string{}
	for n, c := range CmdRunner.All() {
		if !c.GitExtension && !strings.HasPrefix(n, "--") {
			cmds = append(cmds, n)
		}
	}

	sort.Sort(sort.StringSlice(cmds))

	return cmds
}

var helpText = `
These GitHub commands are provided by hub:

   browse         Open a GitHub page in the default browser
   ci-status      Show the status of GitHub checks for a commit
   compare        Open a compare page on GitHub
   create         Create this repository on GitHub and add GitHub as origin
   delete         Delete a repository on GitHub
   fork           Make a fork of a remote repository on GitHub and add as remote
   issue          List or create GitHub issues
   pr             List or checkout GitHub pull requests
   pull-request   Open a pull request on GitHub
   release        List or create GitHub releases
   sync           Fetch git objects from upstream and update branches
`
<commit_msg>[help] List `api` among custom hub commands<commit_after>package commands

import (
	"fmt"
	"os"
	"path/filepath"
	"sort"
	"strings"

	"github.com/github/hub/cmd"
	"github.com/github/hub/ui"
	"github.com/github/hub/utils"
)

var cmdHelp = &Command{
	Run:          runHelp,
	GitExtension: true,
	Usage: `
help hub
help <COMMAND>
help hub-<COMMAND> [--plain-text]
`,
	Long: `Show the help page for a command.

## Options:
	hub-<COMMAND>
		Use this format to view help for hub extensions to an existing git command.

	--plain-text
		Skip man page lookup mechanism and display plain help text.

## Lookup mechanism:

On systems that have 'man', help pages are looked up in these directories
relative to the hub install prefix:

* man/<command>.1
* share/man/man1/<command>.1

On systems without 'man', help pages are looked up using the ".txt" extension.

## See also:

hub(1), git-help(1)
`,
}

var cmdListCmds = &Command{
	Key:          "--list-cmds",
	Run:          runListCmds,
	GitExtension: true,
}

func init() {
	CmdRunner.Use(cmdHelp, "--help")
	CmdRunner.Use(cmdListCmds)
}

func runHelp(helpCmd *Command, args *Args) {
	if args.IsParamsEmpty() {
		args.AfterFn(func() error {
			ui.Println(helpText)
			return nil
		})
		return
	}

	p := utils.NewArgsParser()
	p.RegisterBool("--all", "-a")
	p.RegisterBool("--plain-text")
	p.Parse(args.Params)

	if p.Bool("--all") {
		args.AfterFn(func() error {
			ui.Printf("\nhub custom commands\n\n  %s\n", strings.Join(customCommands(), "  "))
			return nil
		})
		return
	}

	command := args.FirstParam()

	if command == "hub" {
		err := displayManPage("hub.1", args)
		if err != nil {
			utils.Check(err)
		}
	}

	if c := lookupCmd(command); c != nil {
		if !p.Bool("--plain-text") {
			manPage := fmt.Sprintf("hub-%s.1", c.Name())
			err := displayManPage(manPage, args)
			if err == nil {
				return
			}
		}

		ui.Println(c.HelpText())
		args.NoForward()
	}
}

func runListCmds(cmd *Command, args *Args) {
	listOthers := false
	parts := strings.SplitN(args.Command, "=", 2)
	for _, kind := range strings.Split(parts[1], ",") {
		if kind == "others" {
			listOthers = true
			break
		}
	}

	if listOthers {
		args.AfterFn(func() error {
			ui.Println(strings.Join(customCommands(), "\n"))
			return nil
		})
	}
}

func displayManPage(manPage string, args *Args) error {
	manProgram, _ := utils.CommandPath("man")
	if manProgram == "" {
		manPage += ".txt"
		manProgram = os.Getenv("PAGER")
		if manProgram == "" {
			manProgram = "less -R"
		}
	}

	programPath, err := utils.CommandPath(args.ProgramPath)
	if err != nil {
		return err
	}

	installPrefix := filepath.Join(filepath.Dir(programPath), "..")
	manFile, err := localManPage(manPage, installPrefix)
	if err != nil {
		return err
	}

	man := cmd.New(manProgram)
	man.WithArg(manFile)
	if err = man.Run(); err == nil {
		os.Exit(0)
	} else {
		os.Exit(1)
	}
	return nil
}

func localManPage(name, installPrefix string) (string, error) {
	manPath := filepath.Join(installPrefix, "man", name)
	_, err := os.Stat(manPath)
	if err == nil {
		return manPath, nil
	}

	manPath = filepath.Join(installPrefix, "share", "man", "man1", name)
	_, err = os.Stat(manPath)
	if err == nil {
		return manPath, nil
	} else {
		return "", err
	}
}

func lookupCmd(name string) *Command {
	if strings.HasPrefix(name, "hub-") {
		return CmdRunner.Lookup(strings.TrimPrefix(name, "hub-"))
	} else {
		cmd := CmdRunner.Lookup(name)
		if cmd != nil && !cmd.GitExtension {
			return cmd
		} else {
			return nil
		}
	}
}

func customCommands() []string {
	cmds := []string{}
	for n, c := range CmdRunner.All() {
		if !c.GitExtension && !strings.HasPrefix(n, "--") {
			cmds = append(cmds, n)
		}
	}

	sort.Sort(sort.StringSlice(cmds))

	return cmds
}

var helpText = `
These GitHub commands are provided by hub:

   api            Low-level GitHub API request interface
   browse         Open a GitHub page in the default browser
   ci-status      Show the status of GitHub checks for a commit
   compare        Open a compare page on GitHub
   create         Create this repository on GitHub and add GitHub as origin
   delete         Delete a repository on GitHub
   fork           Make a fork of a remote repository on GitHub and add as remote
   issue          List or create GitHub issues
   pr             List or checkout GitHub pull requests
   pull-request   Open a pull request on GitHub
   release        List or create GitHub releases
   sync           Fetch git objects from upstream and update branches
`
<|endoftext|>


<commit_before>// Copyright © 2013 Steve Francia <spf@spf13.com>.
//
// Licensed under the Simple Public License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://opensource.org/licenses/Simple-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package commands

import (
	"fmt"
	"github.com/howeyc/fsnotify"
	"github.com/mostafah/fsync"
	"github.com/spf13/cobra"
	"github.com/spf13/hugo/hugolib"
	"log"
	"os"
	"path/filepath"
	"strings"
	"sync"
	"time"
)

var Config *hugolib.Config
var HugoCmd = &cobra.Command{
	Use:   "hugo",
	Short: "Hugo is a very fast static site generator",
	Long: `A Fast and Flexible Static Site Generator built with
love by spf13 and friends in Go.

Complete documentation is available at http://hugo.spf13.com`,
	Run: build,
}

var Hugo *cobra.Commander
var BuildWatch, Draft, UglyUrls, Verbose bool
var Source, Destination, BaseUrl, CfgFile string

func Execute() {
	AddCommands()
	Hugo := HugoCmd.ToCommander()
	Hugo.Execute()
}

func AddCommands() {
	HugoCmd.AddCommand(serverCmd)
	HugoCmd.AddCommand(version)
	HugoCmd.AddCommand(check)
	HugoCmd.AddCommand(benchmark)
}

func init() {
	HugoCmd.PersistentFlags().BoolVarP(&Draft, "build-drafts", "D", false, "include content marked as draft")
	HugoCmd.PersistentFlags().StringVarP(&Source, "source", "s", "", "filesystem path to read files relative from")
	HugoCmd.PersistentFlags().StringVarP(&Destination, "destination", "d", "", "filesystem path to write files to")
	HugoCmd.PersistentFlags().BoolVarP(&Verbose, "verbose", "v", false, "verbose output")
	HugoCmd.PersistentFlags().BoolVar(&UglyUrls, "uglyurls", false, "if true, use /filename.html instead of /filename/")
	HugoCmd.PersistentFlags().StringVarP(&BaseUrl, "base-url", "b", "", "hostname (and path) to the root eg. http://spf13.com/")
	HugoCmd.PersistentFlags().StringVar(&CfgFile, "config", "", "config file (default is path/config.yaml|json|toml)")
	HugoCmd.Flags().BoolVarP(&BuildWatch, "watch", "w", false, "watch filesystem for changes and recreate as needed")
}

func InitializeConfig() {
	Config = hugolib.SetupConfig(&CfgFile, &Source)
	Config.BuildDrafts = Draft
	Config.UglyUrls = UglyUrls
	Config.Verbose = Verbose
	if BaseUrl != "" {
		Config.BaseUrl = BaseUrl
	}
	if Destination != "" {
		Config.PublishDir = Destination
	}
}

func build(cmd *cobra.Command, args []string) {
	InitializeConfig()

	err := copyStatic()
	if err != nil {
		log.Fatalf("Error copying static files to %s: %v", Config.GetAbsPath(Config.PublishDir), err)
	}
	if _, err := buildSite(); err != nil {
		fmt.Println(err)
		os.Exit(-1)
	}

	if BuildWatch {
		fmt.Println("Watching for changes in", Config.GetAbsPath(Config.ContentDir))
		fmt.Println("Press ctrl+c to stop")
		err := NewWatcher(0)
		if err != nil {
			fmt.Println(err)
		}
	}
}

func copyStatic() error {
	// Copy Static to Destination
	return fsync.Sync(Config.GetAbsPath(Config.PublishDir+"/"), Config.GetAbsPath(Config.StaticDir+"/"))
}

func getDirList() []string {
	var a []string
	walker := func(path string, fi os.FileInfo, err error) error {
		if err != nil {
			fmt.Println("Walker: ", err)
			return nil
		}

		if fi.IsDir() {
			a = append(a, path)
		}
		return nil
	}

	filepath.Walk(Config.GetAbsPath(Config.ContentDir), walker)
	filepath.Walk(Config.GetAbsPath(Config.LayoutDir), walker)
	filepath.Walk(Config.GetAbsPath(Config.StaticDir), walker)

	return a
}

func buildSite() (site *hugolib.Site, err error) {
	startTime := time.Now()
	site = &hugolib.Site{Config: *Config}
	err = site.Build()
	if err != nil {
		return
	}
	site.Stats()
	fmt.Printf("in %v ms\n", int(1000*time.Since(startTime).Seconds()))
	return site, nil
}

func NewWatcher(port int) error {
	watcher, err := fsnotify.NewWatcher()
	var wg sync.WaitGroup

	if err != nil {
		fmt.Println(err)
		return err
	}

	defer watcher.Close()

	wg.Add(1)
	go func() {
		for {
			select {
			case ev := <-watcher.Event:
				if Verbose {
					fmt.Println(ev)
				}
				watchChange(ev)
				// TODO add newly created directories to the watch list
			case err := <-watcher.Error:
				if err != nil {
					fmt.Println("error:", err)
				}
			}
		}
	}()

	for _, d := range getDirList() {
		if d != "" {
			_ = watcher.Watch(d)
		}
	}

	if port > 0 {
		go serve(port)
	}

	wg.Wait()
	return nil
}

func watchChange(ev *fsnotify.FileEvent) {
	if strings.HasPrefix(ev.Name, Config.GetAbsPath(Config.StaticDir)) {
		fmt.Println("Static file changed, syncing\n")
		copyStatic()
	} else {
		fmt.Println("Change detected, rebuilding site\n")
		buildSite()
	}
}
<commit_msg>fixed #85<commit_after>// Copyright © 2013 Steve Francia <spf@spf13.com>.
//
// Licensed under the Simple Public License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://opensource.org/licenses/Simple-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package commands

import (
	"fmt"
	"github.com/howeyc/fsnotify"
	"github.com/mostafah/fsync"
	"github.com/spf13/cobra"
	"github.com/spf13/hugo/hugolib"
	"log"
	"os"
	"path/filepath"
	"strings"
	"sync"
	"time"
)

var Config *hugolib.Config
var HugoCmd = &cobra.Command{
	Use:   "hugo",
	Short: "Hugo is a very fast static site generator",
	Long: `A Fast and Flexible Static Site Generator built with
love by spf13 and friends in Go.

Complete documentation is available at http://hugo.spf13.com`,
	Run: build,
}

var Hugo *cobra.Commander
var BuildWatch, Draft, UglyUrls, Verbose bool
var Source, Destination, BaseUrl, CfgFile string

func Execute() {
	AddCommands()
	Hugo := HugoCmd.ToCommander()
	err := Hugo.Execute()
	if err != nil {
		os.Exit(-1)
	}
}

func AddCommands() {
	HugoCmd.AddCommand(serverCmd)
	HugoCmd.AddCommand(version)
	HugoCmd.AddCommand(check)
	HugoCmd.AddCommand(benchmark)
}

func init() {
	HugoCmd.PersistentFlags().BoolVarP(&Draft, "build-drafts", "D", false, "include content marked as draft")
	HugoCmd.PersistentFlags().StringVarP(&Source, "source", "s", "", "filesystem path to read files relative from")
	HugoCmd.PersistentFlags().StringVarP(&Destination, "destination", "d", "", "filesystem path to write files to")
	HugoCmd.PersistentFlags().BoolVarP(&Verbose, "verbose", "v", false, "verbose output")
	HugoCmd.PersistentFlags().BoolVar(&UglyUrls, "uglyurls", false, "if true, use /filename.html instead of /filename/")
	HugoCmd.PersistentFlags().StringVarP(&BaseUrl, "base-url", "b", "", "hostname (and path) to the root eg. http://spf13.com/")
	HugoCmd.PersistentFlags().StringVar(&CfgFile, "config", "", "config file (default is path/config.yaml|json|toml)")
	HugoCmd.Flags().BoolVarP(&BuildWatch, "watch", "w", false, "watch filesystem for changes and recreate as needed")
}

func InitializeConfig() {
	Config = hugolib.SetupConfig(&CfgFile, &Source)
	Config.BuildDrafts = Draft
	Config.UglyUrls = UglyUrls
	Config.Verbose = Verbose
	if BaseUrl != "" {
		Config.BaseUrl = BaseUrl
	}
	if Destination != "" {
		Config.PublishDir = Destination
	}
}

func build(cmd *cobra.Command, args []string) {
	InitializeConfig()

	err := copyStatic()
	if err != nil {
		log.Fatalf("Error copying static files to %s: %v", Config.GetAbsPath(Config.PublishDir), err)
	}
	if _, err := buildSite(); err != nil {
		fmt.Println(err)
		os.Exit(-1)
	}

	if BuildWatch {
		fmt.Println("Watching for changes in", Config.GetAbsPath(Config.ContentDir))
		fmt.Println("Press ctrl+c to stop")
		err := NewWatcher(0)
		if err != nil {
			fmt.Println(err)
		}
	}
}

func copyStatic() error {
	// Copy Static to Destination
	return fsync.Sync(Config.GetAbsPath(Config.PublishDir+"/"), Config.GetAbsPath(Config.StaticDir+"/"))
}

func getDirList() []string {
	var a []string
	walker := func(path string, fi os.FileInfo, err error) error {
		if err != nil {
			fmt.Println("Walker: ", err)
			return nil
		}

		if fi.IsDir() {
			a = append(a, path)
		}
		return nil
	}

	filepath.Walk(Config.GetAbsPath(Config.ContentDir), walker)
	filepath.Walk(Config.GetAbsPath(Config.LayoutDir), walker)
	filepath.Walk(Config.GetAbsPath(Config.StaticDir), walker)

	return a
}

func buildSite() (site *hugolib.Site, err error) {
	startTime := time.Now()
	site = &hugolib.Site{Config: *Config}
	err = site.Build()
	if err != nil {
		return
	}
	site.Stats()
	fmt.Printf("in %v ms\n", int(1000*time.Since(startTime).Seconds()))
	return site, nil
}

func NewWatcher(port int) error {
	watcher, err := fsnotify.NewWatcher()
	var wg sync.WaitGroup

	if err != nil {
		fmt.Println(err)
		return err
	}

	defer watcher.Close()

	wg.Add(1)
	go func() {
		for {
			select {
			case ev := <-watcher.Event:
				if Verbose {
					fmt.Println(ev)
				}
				watchChange(ev)
				// TODO add newly created directories to the watch list
			case err := <-watcher.Error:
				if err != nil {
					fmt.Println("error:", err)
				}
			}
		}
	}()

	for _, d := range getDirList() {
		if d != "" {
			_ = watcher.Watch(d)
		}
	}

	if port > 0 {
		go serve(port)
	}

	wg.Wait()
	return nil
}

func watchChange(ev *fsnotify.FileEvent) {
	if strings.HasPrefix(ev.Name, Config.GetAbsPath(Config.StaticDir)) {
		fmt.Println("Static file changed, syncing\n")
		copyStatic()
	} else {
		fmt.Println("Change detected, rebuilding site\n")
		buildSite()
	}
}
<|endoftext|>


<commit_before>package dbf

import (
	"bytes"
	"crypto/tls"
	"encoding/json"
	"io"
	"net/http"
	"os"
)

var (
	client    *http.Client
	serverUrl string
)

func initServer() {
	client = &http.Client{
		Transport: &http.Transport{
			DisableCompression: false,
			TLSClientConfig:    &tls.Config{InsecureSkipVerify: (confDbf.Server.Ssl_verify == 0)},
		},
	}

	serverUrl = confDbf.Server.Url_api + "/" + confDbf.Server.Version + "/"
}

func setDefaultHeader(req *http.Request) {
	req.Header.Set("Content-Type", "application/json")
}

func getVendor(vendorType string, vendorName *string, platformId *string, vendorPath *string) bool {
	reqJsonStr := "{\"vendor_type\":\"" + vendorType + "\",\"name\":\"" + *vendorName + "\",\"platform_id\":\"" + *platformId + "\"}"
	reqJsonByte, err := json.Marshal(reqJsonStr)
	if err != nil {
		Log.Printf("%s\n", err)
		return false
	}

	req, err := http.NewRequest("POST", serverUrl+_URL_GET_VENDOR, bytes.NewBuffer(reqJsonByte))
	if err != nil {
		Log.Printf("%s\n", err)
		return false
	}

	setDefaultHeader(req)
	req.Header.Set("Accept", "application/octet-stream")

	resp, err := client.Do(req)
	if err != nil {
		Log.Printf("%s\n", err)
		return false
	}
	defer resp.Body.Close()

	if resp.StatusCode != http.StatusOK {
		Log.Printf("Bad response from server:\nStatus: %s\n Headers: %s\n", resp.Status, resp.Header)
		return false
	}

	vendorFile, err := os.OpenFile(*vendorPath+".tmp", os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0774)
	if err != nil {
		Log.Printf("%s\n", err)
		return false
	}
	defer vendorFile.Close()

	_, err = io.Copy(vendorFile, resp.Body)
	if err != nil {
		Log.Printf("%s\n", err)
		return false
	}

	os.Rename(*vendorPath+".tmp", *vendorPath)

	return true
}
<commit_msg>Do not use json.Marshal() for post body!<commit_after>package dbf

import (
	"bytes"
	"crypto/tls"
	"io"
	"net/http"
	"os"
)

var (
	client    *http.Client
	serverUrl string
)

func initServer() {
	client = &http.Client{
		Transport: &http.Transport{
			DisableCompression: false,
			TLSClientConfig:    &tls.Config{InsecureSkipVerify: (confDbf.Server.Ssl_verify == 0)},
		},
	}

	serverUrl = confDbf.Server.Url_api + "/" + confDbf.Server.Version + "/"
}

func setDefaultHeader(req *http.Request) {
	req.Header.Set("Content-Type", "application/json")
}

func getVendor(vendorType string, vendorName *string, platformId *string, vendorPath *string) bool {
	reqJsonByte := []byte(`{"vendor_type":"` + vendorType + `","name":"` + *vendorName + `","platform_id":"` + *platformId + `"}`)

	req, err := http.NewRequest("POST", serverUrl+_URL_GET_VENDOR, bytes.NewBuffer(reqJsonByte))
	if err != nil {
		Log.Printf("%s\n", err)
		return false
	}

	setDefaultHeader(req)
	req.Header.Set("Accept", "application/octet-stream")

	resp, err := client.Do(req)
	if err != nil {
		Log.Printf("%s\n", err)
		return false
	}
	defer resp.Body.Close()

	if resp.StatusCode != http.StatusOK {
		Log.Printf("Bad response from server:\nStatus: %s\n Headers: %s\n", resp.Status, resp.Header)
		return false
	}

	vendorFile, err := os.OpenFile(*vendorPath+".tmp", os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0774)
	if err != nil {
		Log.Printf("%s\n", err)
		return false
	}
	defer vendorFile.Close()

	_, err = io.Copy(vendorFile, resp.Body)
	if err != nil {
		Log.Printf("%s\n", err)
		return false
	}

	os.Rename(*vendorPath+".tmp", *vendorPath)

	return true
}
<|endoftext|>


<commit_before>// Copyright 2012 Aaron Jacobs. All Rights Reserved.
// Author: aaronjjacobs@gmail.com (Aaron Jacobs)
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"flag"
	"fmt"
	"github.com/jacobsa/comeback/backup"
	"github.com/jacobsa/comeback/blob"
	"github.com/jacobsa/comeback/config"
	"github.com/jacobsa/comeback/fs"
	"github.com/jacobsa/comeback/kv/disk"
	"github.com/jacobsa/comeback/sys"
	"io/ioutil"
	"log"
	"os"
)

var g_configFile = flag.String("config", "", "Path to config file.")
var g_jobName = flag.String("job", "", "Job name within the config file.")

func main() {
	var err error
	flag.Parse()

	// Attempt to read the user's config data.
	if *g_configFile == "" {
		fmt.Println("You must set -config.")
		os.Exit(1)
	}

	configData, err := ioutil.ReadFile(*g_configFile)
	if err != nil {
		fmt.Println("Error reading config file:", err)
		os.Exit(1)
	}

	// Parse the config file.
	cfg, err := config.Parse(configData)
	if err != nil {
		fmt.Println("Parsing config file:", err)
		os.Exit(1)
	}

	// Look for the specified job.
	if *g_jobName == "" {
		fmt.Println("You must set -job.")
		os.Exit(1)
	}

	job, ok := cfg.Jobs[*g_jobName]
	if !ok {
		fmt.Println("Unknown job:", *g_jobName)
		os.Exit(1)
	}

	// Create a user registry.
	userRegistry, err := sys.NewUserRegistry()
	if err != nil {
		log.Fatalf("Creating user registry: %v", err)
	}

	// Create a group registry.
	groupRegistry, err := sys.NewGroupRegistry()
	if err != nil {
		log.Fatalf("Creating group registry: %v", err)
	}

	// Create a file system.
	fileSystem, err := fs.NewFileSystem(userRegistry, groupRegistry)
	if err != nil {
		log.Fatalf("Creating file system: %v", err)
	}

	// Create the kv store.
	kvStore, err := disk.NewDiskKvStore("/tmp/blobs", fileSystem)
	if err != nil {
		log.Fatalf("Creating kv store: %v", err)
	}

	// Create the blob store.
	blobStore := blob.NewKvBasedBlobStore(kvStore)

	// Create the file saver.
	fileSaver, err := backup.NewFileSaver(blobStore, 1<<24)
	if err != nil {
		log.Fatalf("Creating file saver: %v", err)
	}

	// Create a directory saver.
	dirSaver, err := backup.NewDirectorySaver(
		blobStore,
		fileSystem,
		fileSaver)

	if err != nil {
		log.Fatalf("Creating directory saver: %v", err)
	}

	// Run the job.
	score, err := dirSaver.Save(job.BasePath, "", job.Excludes)
	if err != nil {
		log.Fatalf("Saving: %v", err)
	}

	// Print the score.
	fmt.Printf("Score: %s\n", score.Hex())
}
<commit_msg>Added config file validation.<commit_after>// Copyright 2012 Aaron Jacobs. All Rights Reserved.
// Author: aaronjjacobs@gmail.com (Aaron Jacobs)
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"flag"
	"fmt"
	"github.com/jacobsa/comeback/backup"
	"github.com/jacobsa/comeback/blob"
	"github.com/jacobsa/comeback/config"
	"github.com/jacobsa/comeback/fs"
	"github.com/jacobsa/comeback/kv/disk"
	"github.com/jacobsa/comeback/sys"
	"io/ioutil"
	"log"
	"os"
)

var g_configFile = flag.String("config", "", "Path to config file.")
var g_jobName = flag.String("job", "", "Job name within the config file.")

func main() {
	var err error
	flag.Parse()

	// Attempt to read the user's config data.
	if *g_configFile == "" {
		fmt.Println("You must set -config.")
		os.Exit(1)
	}

	configData, err := ioutil.ReadFile(*g_configFile)
	if err != nil {
		fmt.Println("Error reading config file:", err)
		os.Exit(1)
	}

	// Parse the config file.
	cfg, err := config.Parse(configData)
	if err != nil {
		fmt.Println("Parsing config file:", err)
		os.Exit(1)
	}

	// Validate the config file.
	if err := config.Validate(cfg); err != nil {
		fmt.Printf("Config file invalid: %v", err)
		os.Exit(1)
	}

	// Look for the specified job.
	if *g_jobName == "" {
		fmt.Println("You must set -job.")
		os.Exit(1)
	}

	job, ok := cfg.Jobs[*g_jobName]
	if !ok {
		fmt.Println("Unknown job:", *g_jobName)
		os.Exit(1)
	}

	// Create a user registry.
	userRegistry, err := sys.NewUserRegistry()
	if err != nil {
		log.Fatalf("Creating user registry: %v", err)
	}

	// Create a group registry.
	groupRegistry, err := sys.NewGroupRegistry()
	if err != nil {
		log.Fatalf("Creating group registry: %v", err)
	}

	// Create a file system.
	fileSystem, err := fs.NewFileSystem(userRegistry, groupRegistry)
	if err != nil {
		log.Fatalf("Creating file system: %v", err)
	}

	// Create the kv store.
	kvStore, err := disk.NewDiskKvStore("/tmp/blobs", fileSystem)
	if err != nil {
		log.Fatalf("Creating kv store: %v", err)
	}

	// Create the blob store.
	blobStore := blob.NewKvBasedBlobStore(kvStore)

	// Create the file saver.
	fileSaver, err := backup.NewFileSaver(blobStore, 1<<24)
	if err != nil {
		log.Fatalf("Creating file saver: %v", err)
	}

	// Create a directory saver.
	dirSaver, err := backup.NewDirectorySaver(
		blobStore,
		fileSystem,
		fileSaver)

	if err != nil {
		log.Fatalf("Creating directory saver: %v", err)
	}

	// Run the job.
	score, err := dirSaver.Save(job.BasePath, "", job.Excludes)
	if err != nil {
		log.Fatalf("Saving: %v", err)
	}

	// Print the score.
	fmt.Printf("Score: %s\n", score.Hex())
}
<|endoftext|>


<commit_before>package id

import (
	"fmt"

	"github.com/Telmate/proxmox-api-go/cli"
	"github.com/spf13/cobra"
)

var id_checkCmd = &cobra.Command{
	Use:   "check ID",
	Short: "Checks if a ID is availible",
	RunE: func(cmd *cobra.Command, args []string) (err error) {
		id := cli.ValidateIntIDset(args, "ID")
		c := cli.NewClient()
		exixst, err := c.VMIdExists(id)
		if err != nil {
			return
		}
		if exixst {
			fmt.Fprintf(idCmd.OutOrStdout(), "Selected ID is in use: %d\n", id)
		} else {
			fmt.Fprintf(idCmd.OutOrStdout(), "Selected ID is free: %d\n", id)
		}
		return
	},
}

func init() {
	idCmd.AddCommand(id_checkCmd)
}
<commit_msg>fix variable name<commit_after>package id

import (
	"fmt"

	"github.com/Telmate/proxmox-api-go/cli"
	"github.com/spf13/cobra"
)

var id_checkCmd = &cobra.Command{
	Use:   "check ID",
	Short: "Checks if a ID is availible",
	RunE: func(cmd *cobra.Command, args []string) (err error) {
		id := cli.ValidateIntIDset(args, "ID")
		c := cli.NewClient()
		exists, err := c.VMIdExists(id)
		if err != nil {
			return
		}
		if exists {
			fmt.Fprintf(idCmd.OutOrStdout(), "Selected ID is in use: %d\n", id)
		} else {
			fmt.Fprintf(idCmd.OutOrStdout(), "Selected ID is free: %d\n", id)
		}
		return
	},
}

func init() {
	idCmd.AddCommand(id_checkCmd)
}
<|endoftext|>


<commit_before>// Copyright 2015 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package lib

import (
	"context"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"sort"
	"time"

	"github.com/bazelbuild/remote-apis-sdks/go/pkg/chunker"
	"github.com/bazelbuild/remote-apis-sdks/go/pkg/command"
	"github.com/bazelbuild/remote-apis-sdks/go/pkg/filemetadata"
	"github.com/bazelbuild/remote-apis-sdks/go/pkg/tree"
	"github.com/maruel/subcommands"

	"go.chromium.org/luci/client/archiver"
	"go.chromium.org/luci/client/isolated"
	"go.chromium.org/luci/common/data/text/units"
	"go.chromium.org/luci/common/errors"
	isol "go.chromium.org/luci/common/isolated"
	"go.chromium.org/luci/common/system/signals"
)

// CmdArchive returns an object for the `archive` subcommand.
func CmdArchive(options CommandOptions) *subcommands.Command {
	return &subcommands.Command{
		UsageLine: "archive <options>...",
		ShortDesc: "creates a .isolated file and uploads the tree to an isolate server",
		LongDesc: `Given a list of files and directories, creates a .isolated file and uploads the
tree to to an isolate server.

When specifying directories and files, you must also specify a current working
directory for that file or directory. The current working directory will not
be included in the archived path. For example, to isolate './usr/foo/bar' and
have it appear as 'foo/bar' in the .isolated, specify '-files ./usr:foo/bar' or
'-files usr:foo/bar'. When the .isolated is then downloaded, it will then appear
under 'foo/bar' in the desired directory.

Note that '.' may be omitted in general, so to upload 'foo' from the current
working directory, '-files :foo' is sufficient.`,
		CommandRun: func() subcommands.CommandRun {
			c := archiveRun{
				CommandOptions: options,
			}
			c.commonFlags.Init(options.DefaultAuthOpts)
			c.Flags.Var(&c.dirs, "dirs", "Directory(ies) to archive. Specify as <working directory>:<relative path to dir>")
			c.Flags.Var(&c.files, "files", "Individual file(s) to archive. Specify as <working directory>:<relative path to file>")
			c.Flags.StringVar(&c.dumpHash, "dump-hash", "",
				"Write the composite isolated hash to a file")
			c.Flags.StringVar(&c.isolated, "isolated", "",
				"Write the composite isolated to a file")
			c.Flags.StringVar(&c.dumpStatsJSON, "dump-stats-json", "",
				"Write the upload stats to this file as JSON")
			return &c
		},
	}
}

type archiveRun struct {
	commonFlags
	CommandOptions
	dirs          isolated.ScatterGather
	files         isolated.ScatterGather
	dumpHash      string
	isolated      string
	dumpStatsJSON string
}

func (c *archiveRun) Parse(a subcommands.Application, args []string) error {
	if err := c.commonFlags.Parse(); err != nil {
		return err
	}
	if len(args) != 0 {
		return errors.Reason("position arguments not expected").Err()
	}
	return nil
}

// getRoot returns root directory if there is only one working directory.
func getRoot(dirs, files isolated.ScatterGather) (string, error) {
	var rel0, wd0 string
	pickedOne := false
	for rel, wd := range dirs {
		if !pickedOne {
			rel0 = rel
			wd0 = wd
			pickedOne = true
			continue
		}

		if wd0 != wd {
			return "", errors.Reason("different root (working) directory is not supported: %s:%s vs %s:%s", wd0, rel0, wd, rel).Err()
		}
	}

	for rel, wd := range files {
		if !pickedOne {
			rel0 = rel
			wd0 = wd
			pickedOne = true
			continue
		}

		if wd0 != wd {
			return "", errors.Reason("different root (working) directory is not supported: %s:%s vs %s:%s", wd0, rel0, wd, rel).Err()
		}
	}

	if !pickedOne {
		return "", errors.Reason("-dirs or -files should be specified at least once").Err()
	}

	return wd0, nil
}

func (c *archiveRun) doCASAarchive(ctx context.Context) error {
	root, err := getRoot(c.dirs, c.files)
	if err != nil {
		return err
	}

	is := command.InputSpec{}
	for dir := range c.dirs {
		is.Inputs = append(is.Inputs, dir)
	}
	for file := range c.files {
		is.Inputs = append(is.Inputs, file)
	}

	rootDg, chunkers, _, err := tree.ComputeMerkleTree(root, &is, chunker.DefaultChunkSize, filemetadata.NewNoopCache())
	if err != nil {
		return errors.Annotate(err, "failed to call ComputeMerkleTree").Err()
	}

	client, err := c.casFlags.NewClient(ctx)
	if err != nil {
		return errors.Annotate(err, "failed to create cas client").Err()
	}
	defer client.Close()

	if err := client.UploadIfMissing(ctx, chunkers...); err != nil {
		return errors.Annotate(err, "failed to call UploadIfMissing").Err()
	}

	if c.dumpHash != "" {
		if err := ioutil.WriteFile(c.dumpHash, []byte(rootDg.String()), 0644); err != nil {
			return err
		}
	}

	return nil
}

func (c *archiveRun) doIsolatedArchive(ctx context.Context) (stats *archiver.Stats, err error) {
	isolatedClient, isolErr := c.createIsolatedClient(ctx, c.CommandOptions)
	if isolErr != nil {
		err = errors.Annotate(isolErr, "failed to create isolated client").Err()
		return
	}
	var out io.Writer = os.Stdout
	if c.defaultFlags.Quiet {
		out = ioutil.Discard
	}
	arch := archiver.New(ctx, isolatedClient, out)
	defer func() {
		// This waits for all uploads.
		if cerr := arch.Close(); err == nil {
			err = cerr
		}
		// We must take the stats until after all the uploads have finished
		if err == nil {
			stats = arch.Stats()
		}
	}()

	opts := isolated.ArchiveOptions{
		Files:    c.files,
		Dirs:     c.dirs,
		Isolated: c.isolated,
	}
	if len(c.isolated) != 0 {
		var dumpIsolated *os.File
		dumpIsolated, err = os.Create(c.isolated)
		if err != nil {
			return
		}
		// This is OK to close before arch because isolated.Archive
		// does the writing (it's not handed off elsewhere).
		defer dumpIsolated.Close()
		opts.LeakIsolated = dumpIsolated
	}
	item := isolated.Archive(ctx, arch, &opts)
	if err = item.Error(); err != nil {
		return
	}

	item.WaitForHashed()
	if len(c.dumpHash) != 0 {
		if err = ioutil.WriteFile(c.dumpHash, []byte(item.Digest()), 0644); err != nil {
			return
		}
	}
	return
}

// Does the archive by uploading to isolate-server, then return the archive stats and error.
func (c *archiveRun) doArchive(a subcommands.Application, args []string) (stats *archiver.Stats, err error) {
	ctx, cancel := context.WithCancel(c.defaultFlags.MakeLoggingContext(os.Stderr))
	signals.HandleInterrupt(cancel)

	if c.casFlags.Instance != "" {
		// TODO(crbug.com/1110569): get stats
		return &archiver.Stats{}, c.doCASAarchive(ctx)
	}

	return c.doIsolatedArchive(ctx)
}

func (c *archiveRun) postprocessStats(stats *archiver.Stats, start time.Time) error {
	if !c.defaultFlags.Quiet {
		duration := time.Since(start)
		fmt.Fprintf(os.Stderr, "Hits    : %5d (%s)\n", stats.TotalHits(), stats.TotalBytesHits())
		fmt.Fprintf(os.Stderr, "Misses  : %5d (%s)\n", stats.TotalMisses(), stats.TotalBytesPushed())
		fmt.Fprintf(os.Stderr, "Duration: %s\n", units.Round(duration, time.Millisecond))
	}
	if c.dumpStatsJSON != "" {
		return dumpStatsJSON(c.dumpStatsJSON, stats)
	}
	return nil
}

func (c *archiveRun) Run(a subcommands.Application, args []string, _ subcommands.Env) int {
	if err := c.Parse(a, args); err != nil {
		fmt.Fprintf(a.GetErr(), "%s: %s\n", a.GetName(), err)
		return 1
	}
	cl, err := c.defaultFlags.StartTracing()
	if err != nil {
		fmt.Fprintf(a.GetErr(), "%s: %s\n", a.GetName(), err)
		return 1
	}
	defer cl.Close()
	defer c.profilerFlags.Stop()
	start := time.Now()
	stats, err := c.doArchive(a, args)
	if err != nil {
		fmt.Fprintf(a.GetErr(), "%s: %s\n", a.GetName(), err)
		return 1
	}
	if err := c.postprocessStats(stats, start); err != nil {
		fmt.Fprintf(a.GetErr(), "%s: %s\n", a.GetName(), err)
		return 1
	}
	return 0
}

func dumpStatsJSON(jsonPath string, stats *archiver.Stats) error {
	hits := make([]int64, len(stats.Hits))
	for i, h := range stats.Hits {
		hits[i] = int64(h)
	}
	sort.Slice(hits, func(i, j int) bool { return hits[i] < hits[j] })
	itemsHot, err := isol.Pack(hits)
	if err != nil {
		return errors.Annotate(err, "failed to pack itemsHot").Err()
	}

	pushed := make([]int64, len(stats.Pushed))
	for i, p := range stats.Pushed {
		pushed[i] = int64(p.Size)
	}
	sort.Slice(pushed, func(i, j int) bool { return pushed[i] < pushed[j] })
	itemsCold, err := isol.Pack(pushed)
	if err != nil {
		return errors.Annotate(err, "failed to pack itemsCold").Err()
	}

	statsJSON, err := json.Marshal(struct {
		ItemsCold []byte `json:"items_cold"`
		ItemsHot  []byte `json:"items_hot"`
	}{
		ItemsCold: itemsCold,
		ItemsHot:  itemsHot,
	})
	if err != nil {
		return errors.Annotate(err, "failed to marshal result json").Err()
	}
	if err := ioutil.WriteFile(jsonPath, statsJSON, 0664); err != nil {
		return errors.Annotate(err, "failed to write stats json to %s", jsonPath).Err()
	}
	return nil
}
<commit_msg>isolated: fix typo Aarchive -> Archive<commit_after>// Copyright 2015 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package lib

import (
	"context"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"sort"
	"time"

	"github.com/bazelbuild/remote-apis-sdks/go/pkg/chunker"
	"github.com/bazelbuild/remote-apis-sdks/go/pkg/command"
	"github.com/bazelbuild/remote-apis-sdks/go/pkg/filemetadata"
	"github.com/bazelbuild/remote-apis-sdks/go/pkg/tree"
	"github.com/maruel/subcommands"

	"go.chromium.org/luci/client/archiver"
	"go.chromium.org/luci/client/isolated"
	"go.chromium.org/luci/common/data/text/units"
	"go.chromium.org/luci/common/errors"
	isol "go.chromium.org/luci/common/isolated"
	"go.chromium.org/luci/common/system/signals"
)

// CmdArchive returns an object for the `archive` subcommand.
func CmdArchive(options CommandOptions) *subcommands.Command {
	return &subcommands.Command{
		UsageLine: "archive <options>...",
		ShortDesc: "creates a .isolated file and uploads the tree to an isolate server",
		LongDesc: `Given a list of files and directories, creates a .isolated file and uploads the
tree to to an isolate server.

When specifying directories and files, you must also specify a current working
directory for that file or directory. The current working directory will not
be included in the archived path. For example, to isolate './usr/foo/bar' and
have it appear as 'foo/bar' in the .isolated, specify '-files ./usr:foo/bar' or
'-files usr:foo/bar'. When the .isolated is then downloaded, it will then appear
under 'foo/bar' in the desired directory.

Note that '.' may be omitted in general, so to upload 'foo' from the current
working directory, '-files :foo' is sufficient.`,
		CommandRun: func() subcommands.CommandRun {
			c := archiveRun{
				CommandOptions: options,
			}
			c.commonFlags.Init(options.DefaultAuthOpts)
			c.Flags.Var(&c.dirs, "dirs", "Directory(ies) to archive. Specify as <working directory>:<relative path to dir>")
			c.Flags.Var(&c.files, "files", "Individual file(s) to archive. Specify as <working directory>:<relative path to file>")
			c.Flags.StringVar(&c.dumpHash, "dump-hash", "",
				"Write the composite isolated hash to a file")
			c.Flags.StringVar(&c.isolated, "isolated", "",
				"Write the composite isolated to a file")
			c.Flags.StringVar(&c.dumpStatsJSON, "dump-stats-json", "",
				"Write the upload stats to this file as JSON")
			return &c
		},
	}
}

type archiveRun struct {
	commonFlags
	CommandOptions
	dirs          isolated.ScatterGather
	files         isolated.ScatterGather
	dumpHash      string
	isolated      string
	dumpStatsJSON string
}

func (c *archiveRun) Parse(a subcommands.Application, args []string) error {
	if err := c.commonFlags.Parse(); err != nil {
		return err
	}
	if len(args) != 0 {
		return errors.Reason("position arguments not expected").Err()
	}
	return nil
}

// getRoot returns root directory if there is only one working directory.
func getRoot(dirs, files isolated.ScatterGather) (string, error) {
	var rel0, wd0 string
	pickedOne := false
	for rel, wd := range dirs {
		if !pickedOne {
			rel0 = rel
			wd0 = wd
			pickedOne = true
			continue
		}

		if wd0 != wd {
			return "", errors.Reason("different root (working) directory is not supported: %s:%s vs %s:%s", wd0, rel0, wd, rel).Err()
		}
	}

	for rel, wd := range files {
		if !pickedOne {
			rel0 = rel
			wd0 = wd
			pickedOne = true
			continue
		}

		if wd0 != wd {
			return "", errors.Reason("different root (working) directory is not supported: %s:%s vs %s:%s", wd0, rel0, wd, rel).Err()
		}
	}

	if !pickedOne {
		return "", errors.Reason("-dirs or -files should be specified at least once").Err()
	}

	return wd0, nil
}

func (c *archiveRun) doCASArchive(ctx context.Context) error {
	root, err := getRoot(c.dirs, c.files)
	if err != nil {
		return err
	}

	is := command.InputSpec{}
	for dir := range c.dirs {
		is.Inputs = append(is.Inputs, dir)
	}
	for file := range c.files {
		is.Inputs = append(is.Inputs, file)
	}

	rootDg, chunkers, _, err := tree.ComputeMerkleTree(root, &is, chunker.DefaultChunkSize, filemetadata.NewNoopCache())
	if err != nil {
		return errors.Annotate(err, "failed to call ComputeMerkleTree").Err()
	}

	client, err := c.casFlags.NewClient(ctx)
	if err != nil {
		return errors.Annotate(err, "failed to create cas client").Err()
	}
	defer client.Close()

	if err := client.UploadIfMissing(ctx, chunkers...); err != nil {
		return errors.Annotate(err, "failed to call UploadIfMissing").Err()
	}

	if c.dumpHash != "" {
		if err := ioutil.WriteFile(c.dumpHash, []byte(rootDg.String()), 0644); err != nil {
			return err
		}
	}

	return nil
}

func (c *archiveRun) doIsolatedArchive(ctx context.Context) (stats *archiver.Stats, err error) {
	isolatedClient, isolErr := c.createIsolatedClient(ctx, c.CommandOptions)
	if isolErr != nil {
		err = errors.Annotate(isolErr, "failed to create isolated client").Err()
		return
	}
	var out io.Writer = os.Stdout
	if c.defaultFlags.Quiet {
		out = ioutil.Discard
	}
	arch := archiver.New(ctx, isolatedClient, out)
	defer func() {
		// This waits for all uploads.
		if cerr := arch.Close(); err == nil {
			err = cerr
		}
		// We must take the stats until after all the uploads have finished
		if err == nil {
			stats = arch.Stats()
		}
	}()

	opts := isolated.ArchiveOptions{
		Files:    c.files,
		Dirs:     c.dirs,
		Isolated: c.isolated,
	}
	if len(c.isolated) != 0 {
		var dumpIsolated *os.File
		dumpIsolated, err = os.Create(c.isolated)
		if err != nil {
			return
		}
		// This is OK to close before arch because isolated.Archive
		// does the writing (it's not handed off elsewhere).
		defer dumpIsolated.Close()
		opts.LeakIsolated = dumpIsolated
	}
	item := isolated.Archive(ctx, arch, &opts)
	if err = item.Error(); err != nil {
		return
	}

	item.WaitForHashed()
	if len(c.dumpHash) != 0 {
		if err = ioutil.WriteFile(c.dumpHash, []byte(item.Digest()), 0644); err != nil {
			return
		}
	}
	return
}

// Does the archive by uploading to isolate-server, then return the archive stats and error.
func (c *archiveRun) doArchive(a subcommands.Application, args []string) (stats *archiver.Stats, err error) {
	ctx, cancel := context.WithCancel(c.defaultFlags.MakeLoggingContext(os.Stderr))
	signals.HandleInterrupt(cancel)

	if c.casFlags.Instance != "" {
		// TODO(crbug.com/1110569): get stats
		return &archiver.Stats{}, c.doCASArchive(ctx)
	}

	return c.doIsolatedArchive(ctx)
}

func (c *archiveRun) postprocessStats(stats *archiver.Stats, start time.Time) error {
	if !c.defaultFlags.Quiet {
		duration := time.Since(start)
		fmt.Fprintf(os.Stderr, "Hits    : %5d (%s)\n", stats.TotalHits(), stats.TotalBytesHits())
		fmt.Fprintf(os.Stderr, "Misses  : %5d (%s)\n", stats.TotalMisses(), stats.TotalBytesPushed())
		fmt.Fprintf(os.Stderr, "Duration: %s\n", units.Round(duration, time.Millisecond))
	}
	if c.dumpStatsJSON != "" {
		return dumpStatsJSON(c.dumpStatsJSON, stats)
	}
	return nil
}

func (c *archiveRun) Run(a subcommands.Application, args []string, _ subcommands.Env) int {
	if err := c.Parse(a, args); err != nil {
		fmt.Fprintf(a.GetErr(), "%s: %s\n", a.GetName(), err)
		return 1
	}
	cl, err := c.defaultFlags.StartTracing()
	if err != nil {
		fmt.Fprintf(a.GetErr(), "%s: %s\n", a.GetName(), err)
		return 1
	}
	defer cl.Close()
	defer c.profilerFlags.Stop()
	start := time.Now()
	stats, err := c.doArchive(a, args)
	if err != nil {
		fmt.Fprintf(a.GetErr(), "%s: %s\n", a.GetName(), err)
		return 1
	}
	if err := c.postprocessStats(stats, start); err != nil {
		fmt.Fprintf(a.GetErr(), "%s: %s\n", a.GetName(), err)
		return 1
	}
	return 0
}

func dumpStatsJSON(jsonPath string, stats *archiver.Stats) error {
	hits := make([]int64, len(stats.Hits))
	for i, h := range stats.Hits {
		hits[i] = int64(h)
	}
	sort.Slice(hits, func(i, j int) bool { return hits[i] < hits[j] })
	itemsHot, err := isol.Pack(hits)
	if err != nil {
		return errors.Annotate(err, "failed to pack itemsHot").Err()
	}

	pushed := make([]int64, len(stats.Pushed))
	for i, p := range stats.Pushed {
		pushed[i] = int64(p.Size)
	}
	sort.Slice(pushed, func(i, j int) bool { return pushed[i] < pushed[j] })
	itemsCold, err := isol.Pack(pushed)
	if err != nil {
		return errors.Annotate(err, "failed to pack itemsCold").Err()
	}

	statsJSON, err := json.Marshal(struct {
		ItemsCold []byte `json:"items_cold"`
		ItemsHot  []byte `json:"items_hot"`
	}{
		ItemsCold: itemsCold,
		ItemsHot:  itemsHot,
	})
	if err != nil {
		return errors.Annotate(err, "failed to marshal result json").Err()
	}
	if err := ioutil.WriteFile(jsonPath, statsJSON, 0664); err != nil {
		return errors.Annotate(err, "failed to write stats json to %s", jsonPath).Err()
	}
	return nil
}
<|endoftext|>


<commit_before>package scp

import (
	"bytes"
	"errors"
	"fmt"
	"github.com/viant/toolbox"
	"github.com/viant/toolbox/ssh"
	"github.com/viant/toolbox/storage"
	"io"
	"io/ioutil"
	"strings"
	"github.com/viant/toolbox/cred"
	"net/url"
	"github.com/lunixbochs/vtclean"
	"path"
	"sync"
)

const defaultSSHPort = 22
const verificationSizeThreshold = 1024 * 1024

//NoSuchFileOrDirectoryError represents no such file or directory error
var NoSuchFileOrDirectoryError = errors.New("No Such File Or Directory")

type service struct {
	config   *cred.Config
	services map[string]ssh.Service
	multiSessions map[string]ssh.MultiCommandSession
	mutex    *sync.Mutex
}

func (s *service) runCommand(session ssh.MultiCommandSession, URL string, command string) (string, error) {
	output, _ := session.Run(command, 1000)
	var stdout = s.stdout(output)
	return stdout, nil
}

func (s *service) stdout(output string) string {
	return vtclean.Clean(string(output), false)
}

func (s *service) getMultiSession(parsedURL *url.URL) ssh.MultiCommandSession {
	s.mutex.Lock()
	defer s.mutex.Unlock()
	return s.multiSessions[parsedURL.Host]
}

func (s *service) getService(parsedURL *url.URL) (ssh.Service, error) {
	port := toolbox.AsInt(parsedURL.Port())
	if port == 0 {
		port = 22
	}
	key := parsedURL.Host
	s.mutex.Lock()
	defer s.mutex.Unlock()
	if service, ok := s.services[key]; ok {
		return service, nil
	}
	service, err := ssh.NewService(parsedURL.Hostname(), toolbox.AsInt(port), s.config)
	if err != nil {
		return nil, err
	}
	s.services[key] = service
	s.multiSessions[key], err = service.OpenMultiCommandSession(nil)
	if err != nil {
		return nil, err
	}
	return service, nil
}

//List returns a list of object for supplied URL
func (s *service) List(URL string) ([]storage.Object, error) {
	parsedURL, err := url.Parse(URL)
	if err != nil {
		return nil, err
	}
	_, err = s.getService(parsedURL)
	if err != nil {
		return nil, err
	}
	commandSession := s.getMultiSession(parsedURL)
	canListWithTimeStyle := commandSession.System() != "darwin"
	var parser = &Parser{IsoTimeStyle: canListWithTimeStyle}
	var URLPath = parsedURL.Path
	var result = make([]storage.Object, 0)
	var lsCommand = "ls -dltr"
	if canListWithTimeStyle {
		lsCommand += " --time-style=full-iso"
	} else {
		lsCommand += "T"
	}
	output, _ := s.runCommand(commandSession, URL, lsCommand+" "+URLPath)
	var stdout = vtclean.Clean(string(output), false)
	if strings.Contains(stdout, "No such file or directory") {
		return result, NoSuchFileOrDirectoryError
	}
	objects, err := parser.Parse(parsedURL, stdout, false)
	if err != nil {
		return nil, err
	}
	if len(objects) == 1 && objects[0].FileInfo().IsDir() {
		output, _ = s.runCommand(commandSession, URL, lsCommand+" "+path.Join(URLPath, "*"))
		stdout = vtclean.Clean(string(output), false)
		directoryObjects, err := parser.Parse(parsedURL, stdout, true)
		if err != nil {
			return nil, err
		}
		if len(directoryObjects) > 0 {
			objects = append(objects, directoryObjects...)
		}
	}
	return objects, nil
}

func (s *service) Exists(URL string) (bool, error) {
	parsedURL, err := url.Parse(URL)
	if err != nil {
		return false, err
	}

	_, err = s.getService(parsedURL)
	if err != nil {
		return false, err
	}
	commandSession := s.getMultiSession(parsedURL)
	output, _ := s.runCommand(commandSession, URL, "ls -dltr "+parsedURL.Path)
	if strings.Contains(string(output), "No such file or directory") {
		return false, nil
	}
	return true, nil

}

func (s *service) StorageObject(URL string) (storage.Object, error) {
	objects, err := s.List(URL)
	if err != nil {
		return nil, err
	}
	if len(objects) == 0 {
		return nil, NoSuchFileOrDirectoryError
	}
	return objects[0], nil
}

//Download returns reader for downloaded storage object
func (s *service) Download(object storage.Object) (io.Reader, error) {
	if object == nil {
		return nil, fmt.Errorf("Object was nil")
	}
	parsedUrl, err := url.Parse(object.URL())
	if err != nil {
		return nil, err
	}

	port := toolbox.AsInt(parsedUrl.Port())
	if port == 0 {
		port = defaultSSHPort
	}

	service, err := s.getService(parsedUrl)
	content, err := service.Download(parsedUrl.Path)
	if err != nil {
		return nil, err
	}

	if  verificationSizeThreshold < len(content) {
		//download verification (as sometimes scp failed) with one retry
		if int(object.FileInfo().Size()) != len(content) {
			content, err = service.Download(parsedUrl.Path)
			if err != nil {
				return nil, err
			}
			if int(object.FileInfo().Size()) != len(content) {
				return nil, fmt.Errorf("Faled to download from %v,  object size was: %v, but scp download was %v", object.URL(), object.FileInfo().Size(), len(content))
			}
		}
	}

	return bytes.NewReader(content), nil
}

//Upload uploads provided reader content for supplied URL.
func (s *service) Upload(URL string, reader io.Reader) error {
	parsedUrl, err := url.Parse(URL)
	if err != nil {
		return err
	}

	port := toolbox.AsInt(parsedUrl.Port())
	if port == 0 {
		port = defaultSSHPort
	}
	service, err := ssh.NewService(parsedUrl.Hostname(), toolbox.AsInt(port), s.config)
	if err != nil {
		return err
	}

	//defer service.Close()
	content, err := ioutil.ReadAll(reader)
	if err != nil {
		return fmt.Errorf("Failed to upload - unable read: %v", err)
	}

	err = service.Upload(parsedUrl.Path, content)
	if err != nil {
		return fmt.Errorf("Failed to upload: %v %v", URL, err)
	}

	if verificationSizeThreshold < len(content) {
		object, err := s.StorageObject(URL)
		if err != nil {
			return fmt.Errorf("Failed to get upload object  %v for verification: %v", URL, err)
		}
		if int(object.FileInfo().Size()) != len(content) {
			err = service.Upload(parsedUrl.Path, content)
			object, err = s.StorageObject(URL)
			if err != nil {
				return err
			}
			if int(object.FileInfo().Size()) != len(content) {
				return fmt.Errorf("Failed to upload to %v, actual size was:%v,  but uploaded size was %v", URL, len(content), int(object.FileInfo().Size()))
			}
		}
	}

	return err
}

func (s *service) Register(schema string, service storage.Service) error {
	return errors.New("unsupported")
}

func (s *service) Close() error {
	for _, service := range s.services {
		service.Close()
	}
	for _, session := range s.multiSessions {
		session.Close()
	}
	return nil
}

//Delete removes passed in storage object
func (s *service) Delete(object storage.Object) error {
	parsedUrl, err := url.Parse(object.URL())
	if err != nil {
		return err
	}

	port := toolbox.AsInt(parsedUrl.Port())
	if port == 0 {
		port = defaultSSHPort
	}
	service, err := ssh.NewService(parsedUrl.Hostname(), toolbox.AsInt(port), s.config)
	if err != nil {
		return err
	}

	//defer service.Close()
	session, err := service.NewSession()
	if err != nil {
		return err
	}
	defer session.Close()

	if parsedUrl.Path == "/" {
		return fmt.Errorf("Invalid removal path: %v", parsedUrl.Path)
	}
	_, err = session.Output("rm -rf " + parsedUrl.Path)
	return err
}

//NewService create a new gc storage service
func NewService(config *cred.Config) *service {
	return &service{
		services: make(map[string]ssh.Service),
		config:   config,
		multiSessions: make(map[string]ssh.MultiCommandSession),
		mutex:    &sync.Mutex{},
	}
}
<commit_msg>added sync on run command<commit_after>package scp

import (
	"bytes"
	"errors"
	"fmt"
	"github.com/viant/toolbox"
	"github.com/viant/toolbox/ssh"
	"github.com/viant/toolbox/storage"
	"io"
	"io/ioutil"
	"strings"
	"github.com/viant/toolbox/cred"
	"net/url"
	"github.com/lunixbochs/vtclean"
	"path"
	"sync"
)

const defaultSSHPort = 22
const verificationSizeThreshold = 1024 * 1024

//NoSuchFileOrDirectoryError represents no such file or directory error
var NoSuchFileOrDirectoryError = errors.New("No Such File Or Directory")

type service struct {
	config   *cred.Config
	services map[string]ssh.Service
	multiSessions map[string]ssh.MultiCommandSession
	mutex    *sync.Mutex
}

func (s *service) runCommand(session ssh.MultiCommandSession, URL string, command string) (string, error) {
	s.mutex.Lock()
	defer s.mutex.Unlock()
	output, _ := session.Run(command, 5000)
	var stdout = s.stdout(output)
	return stdout, nil
}

func (s *service) stdout(output string) string {
	return vtclean.Clean(string(output), false)
}

func (s *service) getMultiSession(parsedURL *url.URL) ssh.MultiCommandSession {
	s.mutex.Lock()
	defer s.mutex.Unlock()
	return s.multiSessions[parsedURL.Host]
}

func (s *service) getService(parsedURL *url.URL) (ssh.Service, error) {
	port := toolbox.AsInt(parsedURL.Port())
	if port == 0 {
		port = 22
	}
	key := parsedURL.Host
	s.mutex.Lock()
	defer s.mutex.Unlock()
	if service, ok := s.services[key]; ok {
		return service, nil
	}
	service, err := ssh.NewService(parsedURL.Hostname(), toolbox.AsInt(port), s.config)
	if err != nil {
		return nil, err
	}
	s.services[key] = service
	s.multiSessions[key], err = service.OpenMultiCommandSession(nil)
	if err != nil {
		return nil, err
	}
	return service, nil
}

//List returns a list of object for supplied URL
func (s *service) List(URL string) ([]storage.Object, error) {
	parsedURL, err := url.Parse(URL)
	if err != nil {
		return nil, err
	}
	_, err = s.getService(parsedURL)
	if err != nil {
		return nil, err
	}
	commandSession := s.getMultiSession(parsedURL)
	canListWithTimeStyle := commandSession.System() != "darwin"
	var parser = &Parser{IsoTimeStyle: canListWithTimeStyle}
	var URLPath = parsedURL.Path
	var result = make([]storage.Object, 0)
	var lsCommand = "ls -dltr"
	if canListWithTimeStyle {
		lsCommand += " --time-style=full-iso"
	} else {
		lsCommand += "T"
	}
	output, _ := s.runCommand(commandSession, URL, lsCommand+" "+URLPath)
	var stdout = vtclean.Clean(string(output), false)
	if strings.Contains(stdout, "No such file or directory") {
		return result, NoSuchFileOrDirectoryError
	}
	objects, err := parser.Parse(parsedURL, stdout, false)
	if err != nil {
		return nil, err
	}
	if len(objects) == 1 && objects[0].FileInfo().IsDir() {
		output, _ = s.runCommand(commandSession, URL, lsCommand+" "+path.Join(URLPath, "*"))
		stdout = vtclean.Clean(string(output), false)
		directoryObjects, err := parser.Parse(parsedURL, stdout, true)
		if err != nil {
			return nil, err
		}
		if len(directoryObjects) > 0 {
			objects = append(objects, directoryObjects...)
		}
	}
	return objects, nil
}

func (s *service) Exists(URL string) (bool, error) {
	parsedURL, err := url.Parse(URL)
	if err != nil {
		return false, err
	}

	_, err = s.getService(parsedURL)
	if err != nil {
		return false, err
	}
	commandSession := s.getMultiSession(parsedURL)
	output, _ := s.runCommand(commandSession, URL, "ls -dltr "+parsedURL.Path)
	if strings.Contains(string(output), "No such file or directory") {
		return false, nil
	}
	return true, nil

}

func (s *service) StorageObject(URL string) (storage.Object, error) {
	objects, err := s.List(URL)
	if err != nil {
		return nil, err
	}
	if len(objects) == 0 {
		return nil, NoSuchFileOrDirectoryError
	}
	return objects[0], nil
}

//Download returns reader for downloaded storage object
func (s *service) Download(object storage.Object) (io.Reader, error) {
	if object == nil {
		return nil, fmt.Errorf("Object was nil")
	}
	parsedUrl, err := url.Parse(object.URL())
	if err != nil {
		return nil, err
	}

	port := toolbox.AsInt(parsedUrl.Port())
	if port == 0 {
		port = defaultSSHPort
	}

	service, err := s.getService(parsedUrl)
	content, err := service.Download(parsedUrl.Path)
	if err != nil {
		return nil, err
	}

	if  verificationSizeThreshold < len(content) {
		//download verification (as sometimes scp failed) with one retry
		if int(object.FileInfo().Size()) != len(content) {
			content, err = service.Download(parsedUrl.Path)
			if err != nil {
				return nil, err
			}
			if int(object.FileInfo().Size()) != len(content) {
				return nil, fmt.Errorf("Faled to download from %v,  object size was: %v, but scp download was %v", object.URL(), object.FileInfo().Size(), len(content))
			}
		}
	}

	return bytes.NewReader(content), nil
}

//Upload uploads provided reader content for supplied URL.
func (s *service) Upload(URL string, reader io.Reader) error {
	parsedUrl, err := url.Parse(URL)
	if err != nil {
		return err
	}

	port := toolbox.AsInt(parsedUrl.Port())
	if port == 0 {
		port = defaultSSHPort
	}
	service, err := ssh.NewService(parsedUrl.Hostname(), toolbox.AsInt(port), s.config)
	if err != nil {
		return err
	}

	//defer service.Close()
	content, err := ioutil.ReadAll(reader)
	if err != nil {
		return fmt.Errorf("Failed to upload - unable read: %v", err)
	}

	err = service.Upload(parsedUrl.Path, content)
	if err != nil {
		return fmt.Errorf("Failed to upload: %v %v", URL, err)
	}

	if verificationSizeThreshold < len(content) {
		object, err := s.StorageObject(URL)
		if err != nil {
			return fmt.Errorf("Failed to get upload object  %v for verification: %v", URL, err)
		}
		if int(object.FileInfo().Size()) != len(content) {
			err = service.Upload(parsedUrl.Path, content)
			object, err = s.StorageObject(URL)
			if err != nil {
				return err
			}
			if int(object.FileInfo().Size()) != len(content) {
				return fmt.Errorf("Failed to upload to %v, actual size was:%v,  but uploaded size was %v", URL, len(content), int(object.FileInfo().Size()))
			}
		}
	}

	return err
}

func (s *service) Register(schema string, service storage.Service) error {
	return errors.New("unsupported")
}

func (s *service) Close() error {
	for _, service := range s.services {
		service.Close()
	}
	for _, session := range s.multiSessions {
		session.Close()
	}
	return nil
}

//Delete removes passed in storage object
func (s *service) Delete(object storage.Object) error {
	parsedUrl, err := url.Parse(object.URL())
	if err != nil {
		return err
	}

	port := toolbox.AsInt(parsedUrl.Port())
	if port == 0 {
		port = defaultSSHPort
	}
	service, err := ssh.NewService(parsedUrl.Hostname(), toolbox.AsInt(port), s.config)
	if err != nil {
		return err
	}

	//defer service.Close()
	session, err := service.NewSession()
	if err != nil {
		return err
	}
	defer session.Close()

	if parsedUrl.Path == "/" {
		return fmt.Errorf("Invalid removal path: %v", parsedUrl.Path)
	}
	_, err = session.Output("rm -rf " + parsedUrl.Path)
	return err
}

//NewService create a new gc storage service
func NewService(config *cred.Config) *service {
	return &service{
		services: make(map[string]ssh.Service),
		config:   config,
		multiSessions: make(map[string]ssh.MultiCommandSession),
		mutex:    &sync.Mutex{},
	}
}
<|endoftext|>