<commit_before>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Deep equality test via reflection

package reflect

import "unsafe"

// During deepValueEqual, must keep track of checks that are
// in progress. The comparison algorithm assumes that all
// checks in progress are true when it reencounters them.
// Visited comparisons are stored in a map indexed by visit.
type visit struct {
	a1  unsafe.Pointer
	a2  unsafe.Pointer
	typ Type
}

// Tests for deep equality using reflected types. The map argument tracks
// comparisons that have already been seen, which allows short circuiting on
// recursive types.
func deepValueEqual(v1, v2 Value, visited map[visit]bool, depth int) bool {
	if !v1.IsValid() || !v2.IsValid() {
		return v1.IsValid() == v2.IsValid()
	}
	if v1.Type() != v2.Type() {
		return false
	}

	// if depth > 10 { panic("deepValueEqual") }	// for debugging

	// We want to avoid putting more in the visited map than we need to.
	// For any possible reference cycle that might be encountered,
	// hard(t) needs to return true for at least one of the types in the cycle.
	hard := func(k Kind) bool {
		switch k {
		case Map, Slice, Ptr, Interface:
			return true
		}
		return false
	}

	if v1.CanAddr() && v2.CanAddr() && hard(v1.Kind()) {
		addr1 := unsafe.Pointer(v1.UnsafeAddr())
		addr2 := unsafe.Pointer(v2.UnsafeAddr())
		if uintptr(addr1) > uintptr(addr2) {
			// Canonicalize order to reduce number of entries in visited.
			// Assumes non-moving garbage collector.
			addr1, addr2 = addr2, addr1
		}

		// Short circuit if references are already seen.
		typ := v1.Type()
		v := visit{addr1, addr2, typ}
		if visited[v] {
			return true
		}

		// Remember for later.
		visited[v] = true
	}

	switch v1.Kind() {
	case Array:
		for i := 0; i < v1.Len(); i++ {
			if !deepValueEqual(v1.Index(i), v2.Index(i), visited, depth+1) {
				return false
			}
		}
		return true
	case Slice:
		if v1.IsNil() != v2.IsNil() {
			return false
		}
		if v1.Len() != v2.Len() {
			return false
		}
		if v1.Pointer() == v2.Pointer() {
			return true
		}
		for i := 0; i < v1.Len(); i++ {
			if !deepValueEqual(v1.Index(i), v2.Index(i), visited, depth+1) {
				return false
			}
		}
		return true
	case Interface:
		if v1.IsNil() || v2.IsNil() {
			return v1.IsNil() == v2.IsNil()
		}
		return deepValueEqual(v1.Elem(), v2.Elem(), visited, depth+1)
	case Ptr:
		if v1.Pointer() == v2.Pointer() {
			return true
		}
		return deepValueEqual(v1.Elem(), v2.Elem(), visited, depth+1)
	case Struct:
		for i, n := 0, v1.NumField(); i < n; i++ {
			if !deepValueEqual(v1.Field(i), v2.Field(i), visited, depth+1) {
				return false
			}
		}
		return true
	case Map:
		if v1.IsNil() != v2.IsNil() {
			return false
		}
		if v1.Len() != v2.Len() {
			return false
		}
		if v1.Pointer() == v2.Pointer() {
			return true
		}
		for _, k := range v1.MapKeys() {
			val1 := v1.MapIndex(k)
			val2 := v2.MapIndex(k)
			if !val1.IsValid() || !val2.IsValid() || !deepValueEqual(v1.MapIndex(k), v2.MapIndex(k), visited, depth+1) {
				return false
			}
		}
		return true
	case Func:
		if v1.IsNil() && v2.IsNil() {
			return true
		}
		// Can't do better than this:
		return false
	default:
		// Normal equality suffices
		return valueInterface(v1, false) == valueInterface(v2, false)
	}
}

// DeepEqual reports whether x and y are ``deeply equal,'' defined as follows.
// Two values of identical type are deeply equal if one of the following cases applies.
// Values of distinct types are never deeply equal.
//
// Array values are deeply equal when their corresponding elements are deeply equal.
//
// Struct values are deeply equal if their corresponding fields,
// both exported and unexported, are deeply equal.
//
// Func values are deeply equal if both are nil; otherwise they are not deeply equal.
//
// Interface values are deeply equal if they hold deeply equal concrete values.
//
// Map values are deeply equal when all of the following are true:
// they are both nil or both non-nil, they have the same length,
// and either they are the same map object or their corresponding keys
// (matched using Go equality) map to deeply equal values.
//
// Pointer values are deeply equal if they are equal using Go's == operator
// or if they point to deeply equal values.
//
// Slice values are deeply equal when all of the following are true:
// they are both nil or both non-nil, they have the same length,
// and either they point to the same initial entry of the same underlying array
// (that is, &x[0] == &y[0]) or their corresponding elements (up to length) are deeply equal.
// Note that a non-nil empty slice and a nil slice (for example, []byte{} and []byte(nil))
// are not deeply equal.
//
// Other values - numbers, bools, strings, and channels - are deeply equal
// if they are equal using Go's == operator.
//
// In general DeepEqual is a recursive relaxation of Go's == operator.
// However, this idea is impossible to implement without some inconsistency.
// Specifically, it is possible for a value to be unequal to itself,
// either because it is of func type (uncomparable in general)
// or because it is a floating-point NaN value (not equal to itself in floating-point comparison),
// or because it is an array, struct, or interface containing
// such a value.
// On the other hand, pointer values are always equal to themselves,
// even if they point at or contain such problematic values,
// because they compare equal using Go's == operator, and that
// is a sufficient condition to be deeply equal, regardless of content.
// DeepEqual has been defined so that the same short-cut applies
// to slices and maps: if x and y are the same slice or the same map,
// they are deeply equal regardless of content.
func DeepEqual(x, y interface{}) bool {
	if x == nil || y == nil {
		return x == y
	}
	v1 := ValueOf(x)
	v2 := ValueOf(y)
	if v1.Type() != v2.Type() {
		return false
	}
	return deepValueEqual(v1, v2, make(map[visit]bool), 0)
}
<commit_msg>reflect: document how DeepEqual handles cycles<commit_after>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Deep equality test via reflection

package reflect

import "unsafe"

// During deepValueEqual, must keep track of checks that are
// in progress. The comparison algorithm assumes that all
// checks in progress are true when it reencounters them.
// Visited comparisons are stored in a map indexed by visit.
type visit struct {
	a1  unsafe.Pointer
	a2  unsafe.Pointer
	typ Type
}

// Tests for deep equality using reflected types. The map argument tracks
// comparisons that have already been seen, which allows short circuiting on
// recursive types.
func deepValueEqual(v1, v2 Value, visited map[visit]bool, depth int) bool {
	if !v1.IsValid() || !v2.IsValid() {
		return v1.IsValid() == v2.IsValid()
	}
	if v1.Type() != v2.Type() {
		return false
	}

	// if depth > 10 { panic("deepValueEqual") }	// for debugging

	// We want to avoid putting more in the visited map than we need to.
	// For any possible reference cycle that might be encountered,
	// hard(t) needs to return true for at least one of the types in the cycle.
	hard := func(k Kind) bool {
		switch k {
		case Map, Slice, Ptr, Interface:
			return true
		}
		return false
	}

	if v1.CanAddr() && v2.CanAddr() && hard(v1.Kind()) {
		addr1 := unsafe.Pointer(v1.UnsafeAddr())
		addr2 := unsafe.Pointer(v2.UnsafeAddr())
		if uintptr(addr1) > uintptr(addr2) {
			// Canonicalize order to reduce number of entries in visited.
			// Assumes non-moving garbage collector.
			addr1, addr2 = addr2, addr1
		}

		// Short circuit if references are already seen.
		typ := v1.Type()
		v := visit{addr1, addr2, typ}
		if visited[v] {
			return true
		}

		// Remember for later.
		visited[v] = true
	}

	switch v1.Kind() {
	case Array:
		for i := 0; i < v1.Len(); i++ {
			if !deepValueEqual(v1.Index(i), v2.Index(i), visited, depth+1) {
				return false
			}
		}
		return true
	case Slice:
		if v1.IsNil() != v2.IsNil() {
			return false
		}
		if v1.Len() != v2.Len() {
			return false
		}
		if v1.Pointer() == v2.Pointer() {
			return true
		}
		for i := 0; i < v1.Len(); i++ {
			if !deepValueEqual(v1.Index(i), v2.Index(i), visited, depth+1) {
				return false
			}
		}
		return true
	case Interface:
		if v1.IsNil() || v2.IsNil() {
			return v1.IsNil() == v2.IsNil()
		}
		return deepValueEqual(v1.Elem(), v2.Elem(), visited, depth+1)
	case Ptr:
		if v1.Pointer() == v2.Pointer() {
			return true
		}
		return deepValueEqual(v1.Elem(), v2.Elem(), visited, depth+1)
	case Struct:
		for i, n := 0, v1.NumField(); i < n; i++ {
			if !deepValueEqual(v1.Field(i), v2.Field(i), visited, depth+1) {
				return false
			}
		}
		return true
	case Map:
		if v1.IsNil() != v2.IsNil() {
			return false
		}
		if v1.Len() != v2.Len() {
			return false
		}
		if v1.Pointer() == v2.Pointer() {
			return true
		}
		for _, k := range v1.MapKeys() {
			val1 := v1.MapIndex(k)
			val2 := v2.MapIndex(k)
			if !val1.IsValid() || !val2.IsValid() || !deepValueEqual(v1.MapIndex(k), v2.MapIndex(k), visited, depth+1) {
				return false
			}
		}
		return true
	case Func:
		if v1.IsNil() && v2.IsNil() {
			return true
		}
		// Can't do better than this:
		return false
	default:
		// Normal equality suffices
		return valueInterface(v1, false) == valueInterface(v2, false)
	}
}

// DeepEqual reports whether x and y are ``deeply equal,'' defined as follows.
// Two values of identical type are deeply equal if one of the following cases applies.
// Values of distinct types are never deeply equal.
//
// Array values are deeply equal when their corresponding elements are deeply equal.
//
// Struct values are deeply equal if their corresponding fields,
// both exported and unexported, are deeply equal.
//
// Func values are deeply equal if both are nil; otherwise they are not deeply equal.
//
// Interface values are deeply equal if they hold deeply equal concrete values.
//
// Map values are deeply equal when all of the following are true:
// they are both nil or both non-nil, they have the same length,
// and either they are the same map object or their corresponding keys
// (matched using Go equality) map to deeply equal values.
//
// Pointer values are deeply equal if they are equal using Go's == operator
// or if they point to deeply equal values.
//
// Slice values are deeply equal when all of the following are true:
// they are both nil or both non-nil, they have the same length,
// and either they point to the same initial entry of the same underlying array
// (that is, &x[0] == &y[0]) or their corresponding elements (up to length) are deeply equal.
// Note that a non-nil empty slice and a nil slice (for example, []byte{} and []byte(nil))
// are not deeply equal.
//
// Other values - numbers, bools, strings, and channels - are deeply equal
// if they are equal using Go's == operator.
//
// In general DeepEqual is a recursive relaxation of Go's == operator.
// However, this idea is impossible to implement without some inconsistency.
// Specifically, it is possible for a value to be unequal to itself,
// either because it is of func type (uncomparable in general)
// or because it is a floating-point NaN value (not equal to itself in floating-point comparison),
// or because it is an array, struct, or interface containing
// such a value.
// On the other hand, pointer values are always equal to themselves,
// even if they point at or contain such problematic values,
// because they compare equal using Go's == operator, and that
// is a sufficient condition to be deeply equal, regardless of content.
// DeepEqual has been defined so that the same short-cut applies
// to slices and maps: if x and y are the same slice or the same map,
// they are deeply equal regardless of content.
//
// As DeepEqual traverses the data values it may find a cycle. The
// second and subsequent times that DeepEqual compares two pointer
// values that have been compared before, it treats the values as
// equal rather than examining the values to which they point.
// This ensures that DeepEqual terminates.
func DeepEqual(x, y interface{}) bool {
	if x == nil || y == nil {
		return x == y
	}
	v1 := ValueOf(x)
	v2 := ValueOf(y)
	if v1.Type() != v2.Type() {
		return false
	}
	return deepValueEqual(v1, v2, make(map[visit]bool), 0)
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"compress/gzip"
	"errors"
	"fmt"
	"log"
	"math"
	"os"
	"path/filepath"
	"regexp"
	"sort"
	"strconv"
	"strings"
	"time"

	"github.com/brunetto/goutils/readfile"
	"github.com/brunetto/goutils/debug"
)

var (
	reg          *regexp.Regexp = regexp.MustCompile(`[rva]\s*=\s*(\S*)\s*(\S*)\s*(\S*)`)
	res          []string
	idxReg       *regexp.Regexp = regexp.MustCompile(`i\s*=\s*(\S*)`)
	idxRes       []string
	nameReg      *regexp.Regexp = regexp.MustCompile(`name\s*=\s*(\S*)`)
	nameRes      []string
	nReg         *regexp.Regexp = regexp.MustCompile(`N\s*=\s*(\S*)`)
	nRes         []string
	combReg      *regexp.Regexp = regexp.MustCompile(`out-comb(\d+)-\S+-run(\d+)-all.txt\S*`)
	combRes      []string
	particleChan chan *Particle
	linesChan    chan string
	dbug         bool = false
)

// TODO: intercetto la size_scale
// then size scale = ceil ( 6.955e5 / (size_scale * 3.08...e13) ) = ceil ( 1 Rsun in km / (size_scale * 1 pc in km) )

func main() {
	if true {defer debug.TimeMe(time.Now())}

	var (
		err                     error
		inFileName, outFileName string
		inFile, outFile         *os.File
		nReader                 *bufio.Reader
		nWriter                 *bufio.Writer
		done                    = make(chan string, 3)
		ext string
		fZip *gzip.Reader
		binaryPrefix string
	)

	if len(os.Args) < 3 {
		log.Fatal("Provide a STDOUT and a outfile")
	}
	inFileName = os.Args[1]
	ext = filepath.Ext(inFileName)

	outFileName = os.Args[2]

	if combRes = combReg.FindStringSubmatch(inFileName); combRes == nil {
		log.Fatal("Can't reg: ", inFileName)
	}

	binaryPrefix = "c" + combRes[1] + "n" + combRes[2]

	fmt.Println("Reading from ", inFileName)
	fmt.Println("Writing to ", outFileName)

	log.Println("Creating files")
	if inFile, err = os.Open(inFileName); err != nil {
		log.Fatal(err)
	}
	defer inFile.Close()
	switch ext {
	case ".txt":
		{
			nReader = bufio.NewReader(inFile)
		}
	case ".gz":
		{
			fZip, err = gzip.NewReader(inFile)
			if err != nil {
				log.Fatal("Can't open %s: error: %s\n", inFile, err)
			}
			nReader = bufio.NewReader(fZip)
		}
	default:
		{
			log.Println("Unrecognized file type", inFileName)
			log.Fatal("with extension ", ext)
		}
	}

	if outFile, err = os.Create(outFileName); err != nil {
		log.Fatal(err)
	}
	defer outFile.Close()
	nWriter = bufio.NewWriter(outFile)
	defer nWriter.Flush()

	particleChan = make(chan *Particle, 1000)
	linesChan = make(chan string, 1000)

	log.Println("Launching goroutines")
	go Readline(nReader, linesChan, done)
	go ClusterParsing(linesChan, particleChan, done)
	go ClusterPrinting(nWriter, binaryPrefix, particleChan, done)

	// Wait goroutines to finish
	for i := 0; i < 3; i++ {
		fmt.Print(<-done)
	}

	fmt.Println()
}

type Particle struct {
	Time       int64
	PhysTime   float64
	Idx        string
	Name       string
	X, Y, Z    float64
	Dist       float64
/*	Vx, Vy, Vz string
	Ax, Ay, Az string
	Mass*/       string
// 	Multiple bool
// 	Sons []&Particle
// 	HasParent bool
// 	Parent &Particle{}
}

func Readline(nReader *bufio.Reader, linesChan chan string, done chan string) {
	if dbug {defer debug.TimeMe(time.Now())}
	var (
		err  error
		line string
		msg  string
	)

	for {
		if line, err = readfile.Readln(nReader); err != nil {
			if dbug {
				fmt.Println()
				log.Println(err)
			}
			break
		}
		linesChan <- line
	}
	close(linesChan)
	if dbug {
		msg = "Done from Readline\n"
	} else {
		msg = ""
	}
	done <- msg
}

func timeAndScale (linesChan chan string) (timeStp int64, timeScale, sizeScale float64) {

	var (
		err error
		regSysTime = regexp.MustCompile(`system_time\s*=\s*(\d+)`)
		resSysTime []string
		regTimeScale = regexp.MustCompile(`time_scale\s+=\s+(\d\.\d*e*[\+-]*\d*)`)
		resTimeScale []string
		regSizeScale = regexp.MustCompile(`size_scale\s+=\s+(\d\.\d*e*[\+-]*\d*)`)
		resSizeScale []string
	)

	timeStp = 0
	timeScale = 0
	sizeScale = 0

	for line := range linesChan {
		if resSysTime = regSysTime.FindStringSubmatch(line); resSysTime != nil {
			if timeStp, err = strconv.ParseInt(resSysTime[1], 10, 64); err != nil {
				log.Fatal("Error parsing timestep: ", err)
			}
		}
		if resTimeScale = regTimeScale.FindStringSubmatch(line); resTimeScale != nil {
			if timeScale, err = strconv.ParseFloat(resTimeScale[1], 64); err != nil {
				log.Fatal("Error parsing time scale: ", err)
			}
		}
		if resSizeScale = regSizeScale.FindStringSubmatch(line); resSizeScale != nil {
			if sizeScale, err = strconv.ParseFloat(resSizeScale[1], 64); err != nil {
				log.Fatal("Error parsing size scale: ", err)
			}
		}
		if float64(timeStp) * timeScale * sizeScale > 0 {
			break
		}
	}
	return timeStp, 1./timeScale, math.Ceil( 6.955e5 / (sizeScale * 3.08567758e13))
}

func ClusterParsing(linesChan chan string, particleChan chan *Particle, done chan string) {
	if dbug {
		defer debug.TimeMe(time.Now())
	}
	var (
		sentParticles int = 0
		filled        bool
		particle      *Particle
		err           error
		msg           string
		nestingLevel int = 0
		timeStp int64 = 0
		sizeScale, timeScale float64
	)

	log.Println("Start reading loop")
	for line := range linesChan {
		// Start reading a particle
		if strings.Contains(line, "(Particle") {
			nestingLevel++
			if nestingLevel == 1 {
				// Root particle
				timeStp, timeScale, sizeScale = timeAndScale(linesChan)
			}
			if nestingLevel == 2 {
				if particle, filled, err = ParseParticle(linesChan, timeStp, timeScale, sizeScale ); err != nil {
					log.Println("Error in filling the particles detected in ClusterParsing ", err)
					break
				}
				if filled == true {
					if dbug {log.Println("Send particle: ", particle.Idx)}
					sentParticles++
					particleChan <- particle
					if sentParticles % 100 == 0 {
						fmt.Printf("\rProcessed %v particles", sentParticles)
					}
				}
			}
		} else if strings.Contains(line, ")Particle") {
			nestingLevel--
		}
	}
	close(particleChan)
	if dbug {
		msg = "Done from ClusterParsing\n"
	} else {
		msg = ""
	}
	done <- msg
}

func ClusterPrinting(nWriter *bufio.Writer, binaryPrefix string, particleChan chan *Particle, done chan string) {
	if dbug {
		defer debug.TimeMe(time.Now())
	}
	fmt.Fprint(nWriter, "# time, physTime, name, x, y, z, r\n")//, vx, vy, vz, ax, ay, az, mass\n")
	for particle := range particleChan {

		orderedNames := strings.Split(strings.Trim(particle.Name, "()"), ",")
		sort.Strings(orderedNames)

		fmt.Fprintf(nWriter, "%v,%v,%v,%v,%v,%v,%v\n", //%v,%v,%v,%v,%v,%v,%v\n",
			particle.Time,
			particle.PhysTime,
// 			particle.Name,
			binaryPrefix + "a" + orderedNames[0] + "b" + orderedNames[1],
			particle.X,
			particle.Y,
			particle.Z,
			particle.Dist,
// 			particle.Vx,
// 			particle.Vy,
// 			particle.Vz,
// 			particle.Ax,
// 			particle.Ay,
// 			particle.Az,
// 			particle.Mass,
		)
	}
	var msg string
	if dbug {
		msg = "Done from ClusterPrinting\n"
	} else {
		msg = ""
	}
	done <- msg
}

func ParseParticle(linesChan chan string, timeStp int64, timeScale, sizeScale float64) (*Particle, bool, error) {
	if dbug {defer debug.TimeMe(time.Now())}
	var (
		n, name, idx, /*m,*/ x, y, z/*, vx, vy, vz, ax, ay, az*/ string
		dist                                             float64
// 		line                                             string
		err                                              error
	)
	for {
		// TODO: Fare un check su n=2
		if n, name, idx = ReadPBasics(linesChan); n != "2" {    //> indagare, sto cercando di pescare solo le binarie
			return &Particle{}, false, nil                      //> indagare
		}

		ReadPLog(linesChan)
		if /*m,*/ x, y, z, /*vx, vy, vz, ax, ay, az,*/ err = ReadPDynamics(linesChan); err != nil {
			return &Particle{}, false, err
		}
		ReadPHydro(linesChan)
		ReadPStar(linesChan)

		xf, _ := strconv.ParseFloat(x, 64)
		yf, _ := strconv.ParseFloat(y, 64)
		zf, _ := strconv.ParseFloat(z, 64)
		dist = math.Sqrt(xf*xf + yf*yf + zf*zf)

		// Insert a loop here to consume lines until )Particle is reached???
// 		line = <-linesChan
// 		if strings.Contains(line, ")Particle") {
		return &Particle{
			Idx:  idx,
			Name: name,
			Time: timeStp,
			PhysTime: float64(timeStp) * timeScale,
			X:    xf * sizeScale,
			Y:    yf * sizeScale,
			Z:    zf * sizeScale,
			Dist: dist * sizeScale,
// 			Vx:   vx,
// 			Vy:   vy,
// 			Vz:   vz,
// 			Ax:   ax,
// 			Ay:   ay,
// 			Az:   az,
// 			Mass: m,
		}, true, nil
// 		}
	}
}

func ReadPBasics(linesChan chan string) (n, name, idx string) {
	if dbug {
		defer debug.TimeMe(time.Now())
	}
	name = "--" // may be the name is empty
	for line := range linesChan {
		if idxRes = idxReg.FindStringSubmatch(line); idxRes != nil {
			idx = idxRes[1]
		}
		if nameRes = nameReg.FindStringSubmatch(line); nameRes != nil {
			name = nameRes[1]
		}
		if nRes = nReg.FindStringSubmatch(line); nRes != nil {
			n = nRes[1]
		}
		if strings.Contains(line, "(Log") {
			return n, name, idx
		}
	}
	return "-1", "", "" // error while reading
}

func ReadPLog(linesChan chan string) {
	if dbug {defer debug.TimeMe(time.Now())}
	for line := range linesChan {
		if strings.Contains(line, ")Log") {
			break
		}
	}
}

func ReadPDynamics(linesChan chan string) (/*m, */x, y, z/*, vx, vy, vz, ax, ay, az*/ string, err error) {
	if dbug {defer debug.TimeMe(time.Now())}
	for line := range linesChan {
// 		if strings.Contains(line, "m  =") {
// 			m = strings.Split(line, "  = ")[1]
// 		}
		if strings.Contains(line, "r  =") {
			if res = reg.FindStringSubmatch(line); res != nil {
				x = res[1]
				y = res[2]
				z = res[3]
			}
		}
// 		if strings.Contains(line, "v  =") {
// 			if res = reg.FindStringSubmatch(line); res != nil {
// 				vx = res[1]
// 				vy = res[2]
// 				vz = res[3]
// 			}
// 		}
// 		if strings.Contains(line, "a  =") {
// 			if res = reg.FindStringSubmatch(line); res != nil {
// 				ax = res[1]
// 				ay = res[2]
// 				az = res[3]
// 			}
// 		}
		if strings.Contains(line, "(Hydro") {
			return /*m,*/ x, y, z, /*vx, vy, vz, ax, ay, az,*/ nil
		}
	}
	err = errors.New("Stop reading")
	return "", "", "", /*"", "", "", "", "", "", "",*/ err
}

func ReadPHydro(linesChan chan string) {
	if dbug {defer debug.TimeMe(time.Now())}
	for line := range linesChan {
		if strings.Contains(line, ")Hydro") {
			break
		}
	}
}

func ReadPStar(linesChan chan string) {
	if dbug {defer debug.TimeMe(time.Now())}
	for line := range linesChan {
		if strings.Contains(line, ")Star") {
			break
		}
	}
}
<commit_msg>before generalizing binary positions for many files<commit_after>package main

import (
	"bufio"
	"compress/gzip"
	"errors"
	"fmt"
	"log"
	"math"
	"os"
	"path/filepath"
	"regexp"
	"sort"
	"strconv"
	"strings"
	"time"

	"github.com/brunetto/goutils/readfile"
	"github.com/brunetto/goutils/debug"
)

var (
	reg          *regexp.Regexp = regexp.MustCompile(`[rva]\s*=\s*(\S*)\s*(\S*)\s*(\S*)`)
	res          []string
	idxReg       *regexp.Regexp = regexp.MustCompile(`i\s*=\s*(\S*)`)
	idxRes       []string
	nameReg      *regexp.Regexp = regexp.MustCompile(`name\s*=\s*(\S*)`)
	nameRes      []string
	nReg         *regexp.Regexp = regexp.MustCompile(`N\s*=\s*(\S*)`)
	nRes         []string
	combReg      *regexp.Regexp = regexp.MustCompile(`out-comb(\d+)-\S+-run(\d+)-all.txt\S*`)
	combRes      []string
	particleChan chan *Particle
	linesChan    chan string
	dbug         bool = false
)

func main() {
	if true {defer debug.TimeMe(time.Now())}

	var (
		err                     error
		inFileName, outFileName string
		inFile, outFile         *os.File
		nReader                 *bufio.Reader
		nWriter                 *bufio.Writer
		done                    = make(chan string, 3)
		ext string
		fZip *gzip.Reader
		binaryPrefix string
		folders []string
		folder string
	)

	if folders, err = filepath.Glob("comb*"); err != nil {
		log.Fatal("Error globbing: ", err)
	}

	if len(folders) == 0 {
		log.Fatal("No folders found")
	}

	// Loop over found folders
	for _, folder = range folders {
		if info, err := os.Stat(folder); err == nil && info.IsDir() {
			if err = os.Chdir(folder); err != nil {
				log.Fatalf("Can't change dir to %v with err: %v\n", folder, err)
			}
		} else if err != nil {
			log.Fatalf("Error checking if %v is dir: %v\n", folder, err)
		}
	}

	if len(os.Args) < 3 {
		log.Fatal("Provide a STDOUT and a outfile")
	}
	inFileName = os.Args[1]
	ext = filepath.Ext(inFileName)

	outFileName = os.Args[2]

	if combRes = combReg.FindStringSubmatch(inFileName); combRes == nil {
		log.Fatal("Can't reg: ", inFileName)
	}

	binaryPrefix = "c" + combRes[1] + "n" + combRes[2]

	fmt.Println("Reading from ", inFileName)
	fmt.Println("Writing to ", outFileName)

	log.Println("Creating files")
	if inFile, err = os.Open(inFileName); err != nil {
		log.Fatal(err)
	}
	defer inFile.Close()
	switch ext {
	case ".txt":
		{
			nReader = bufio.NewReader(inFile)
		}
	case ".gz":
		{
			fZip, err = gzip.NewReader(inFile)
			if err != nil {
				log.Fatal("Can't open %s: error: %s\n", inFile, err)
			}
			nReader = bufio.NewReader(fZip)
		}
	default:
		{
			log.Println("Unrecognized file type", inFileName)
			log.Fatal("with extension ", ext)
		}
	}

	if outFile, err = os.Create(outFileName); err != nil {
		log.Fatal(err)
	}
	defer outFile.Close()
	nWriter = bufio.NewWriter(outFile)
	defer nWriter.Flush()

	particleChan = make(chan *Particle, 1000)
	linesChan = make(chan string, 1000)

	log.Println("Launching goroutines")
	go Readline(nReader, linesChan, done)
	go ClusterParsing(linesChan, particleChan, done)
	go ClusterPrinting(nWriter, binaryPrefix, particleChan, done)

	// Wait goroutines to finish
	for i := 0; i < 3; i++ {
		fmt.Print(<-done)
	}

	fmt.Println()
}

type Particle struct {
	Time       int64
	PhysTime   float64
	Idx        string
	Name       string
	X, Y, Z    float64
	Dist       float64
/*	Vx, Vy, Vz string
	Ax, Ay, Az string
	Mass*/       string
// 	Multiple bool
// 	Sons []&Particle
// 	HasParent bool
// 	Parent &Particle{}
}

func Readline(nReader *bufio.Reader, linesChan chan string, done chan string) {
	if dbug {defer debug.TimeMe(time.Now())}
	var (
		err  error
		line string
		msg  string
	)

	for {
		if line, err = readfile.Readln(nReader); err != nil {
			if dbug {
				fmt.Println()
				log.Println(err)
			}
			break
		}
		linesChan <- line
	}
	close(linesChan)
	if dbug {
		msg = "Done from Readline\n"
	} else {
		msg = ""
	}
	done <- msg
}

func timeAndScale (linesChan chan string) (timeStp int64, timeScale, sizeScale float64) {

	var (
		err error
		regSysTime = regexp.MustCompile(`system_time\s*=\s*(\d+)`)
		resSysTime []string
		regTimeScale = regexp.MustCompile(`time_scale\s+=\s+(\d\.\d*e*[\+-]*\d*)`)
		resTimeScale []string
		regSizeScale = regexp.MustCompile(`size_scale\s+=\s+(\d\.\d*e*[\+-]*\d*)`)
		resSizeScale []string
	)

	timeStp = 0
	timeScale = 0
	sizeScale = 0

	for line := range linesChan {
		if resSysTime = regSysTime.FindStringSubmatch(line); resSysTime != nil {
			if timeStp, err = strconv.ParseInt(resSysTime[1], 10, 64); err != nil {
				log.Fatal("Error parsing timestep: ", err)
			}
		}
		if resTimeScale = regTimeScale.FindStringSubmatch(line); resTimeScale != nil {
			if timeScale, err = strconv.ParseFloat(resTimeScale[1], 64); err != nil {
				log.Fatal("Error parsing time scale: ", err)
			}
		}
		if resSizeScale = regSizeScale.FindStringSubmatch(line); resSizeScale != nil {
			if sizeScale, err = strconv.ParseFloat(resSizeScale[1], 64); err != nil {
				log.Fatal("Error parsing size scale: ", err)
			}
		}
		if float64(timeStp) * timeScale * sizeScale > 0 {
			break
		}
	}
	return timeStp, 1./timeScale, math.Ceil( 6.955e5 / (sizeScale * 3.08567758e13))
}

func ClusterParsing(linesChan chan string, particleChan chan *Particle, done chan string) {
	if dbug {
		defer debug.TimeMe(time.Now())
	}
	var (
		sentParticles int = 0
		filled        bool
		particle      *Particle
		err           error
		msg           string
		nestingLevel int = 0
		timeStp int64 = 0
		sizeScale, timeScale float64
	)

	log.Println("Start reading loop")
	for line := range linesChan {
		// Start reading a particle
		if strings.Contains(line, "(Particle") {
			nestingLevel++
			if nestingLevel == 1 {
				// Root particle
				timeStp, timeScale, sizeScale = timeAndScale(linesChan)
			}
			if nestingLevel == 2 {
				if particle, filled, err = ParseParticle(linesChan, timeStp, timeScale, sizeScale ); err != nil {
					log.Println("Error in filling the particles detected in ClusterParsing ", err)
					break
				}
				if filled == true {
					if dbug {log.Println("Send particle: ", particle.Idx)}
					sentParticles++
					particleChan <- particle
					if sentParticles % 100 == 0 {
						fmt.Printf("\rProcessed %v particles", sentParticles)
					}
				}
			}
		} else if strings.Contains(line, ")Particle") {
			nestingLevel--
		}
	}
	close(particleChan)
	if dbug {
		msg = "Done from ClusterParsing\n"
	} else {
		msg = ""
	}
	done <- msg
}

func ClusterPrinting(nWriter *bufio.Writer, binaryPrefix string, particleChan chan *Particle, done chan string) {
	if dbug {
		defer debug.TimeMe(time.Now())
	}
	fmt.Fprint(nWriter, "# time, physTime, name, x, y, z, r\n")//, vx, vy, vz, ax, ay, az, mass\n")
	for particle := range particleChan {

		orderedNames := strings.Split(strings.Trim(particle.Name, "()"), ",")
		sort.Strings(orderedNames)

		fmt.Fprintf(nWriter, "%v,%v,%v,%v,%v,%v,%v\n", //%v,%v,%v,%v,%v,%v,%v\n",
			particle.Time,
			particle.PhysTime,
// 			particle.Name,
			binaryPrefix + "a" + orderedNames[0] + "b" + orderedNames[1],
			particle.X,
			particle.Y,
			particle.Z,
			particle.Dist,
// 			particle.Vx,
// 			particle.Vy,
// 			particle.Vz,
// 			particle.Ax,
// 			particle.Ay,
// 			particle.Az,
// 			particle.Mass,
		)
	}
	var msg string
	if dbug {
		msg = "Done from ClusterPrinting\n"
	} else {
		msg = ""
	}
	done <- msg
}

func ParseParticle(linesChan chan string, timeStp int64, timeScale, sizeScale float64) (*Particle, bool, error) {
	if dbug {defer debug.TimeMe(time.Now())}
	var (
		n, name, idx, /*m,*/ x, y, z/*, vx, vy, vz, ax, ay, az*/ string
		dist                                             float64
// 		line                                             string
		err                                              error
	)
	for {
		// TODO: Fare un check su n=2
		if n, name, idx = ReadPBasics(linesChan); n != "2" {    //> indagare, sto cercando di pescare solo le binarie
			return &Particle{}, false, nil                      //> indagare
		}

		ReadPLog(linesChan)
		if /*m,*/ x, y, z, /*vx, vy, vz, ax, ay, az,*/ err = ReadPDynamics(linesChan); err != nil {
			return &Particle{}, false, err
		}
		ReadPHydro(linesChan)
		ReadPStar(linesChan)

		xf, _ := strconv.ParseFloat(x, 64)
		yf, _ := strconv.ParseFloat(y, 64)
		zf, _ := strconv.ParseFloat(z, 64)
		dist = math.Sqrt(xf*xf + yf*yf + zf*zf)

		// Insert a loop here to consume lines until )Particle is reached???
// 		line = <-linesChan
// 		if strings.Contains(line, ")Particle") {
		return &Particle{
			Idx:  idx,
			Name: name,
			Time: timeStp,
			PhysTime: float64(timeStp) * timeScale,
			X:    xf * sizeScale,
			Y:    yf * sizeScale,
			Z:    zf * sizeScale,
			Dist: dist * sizeScale,
// 			Vx:   vx,
// 			Vy:   vy,
// 			Vz:   vz,
// 			Ax:   ax,
// 			Ay:   ay,
// 			Az:   az,
// 			Mass: m,
		}, true, nil
// 		}
	}
}

func ReadPBasics(linesChan chan string) (n, name, idx string) {
	if dbug {
		defer debug.TimeMe(time.Now())
	}
	name = "--" // may be the name is empty
	for line := range linesChan {
		if idxRes = idxReg.FindStringSubmatch(line); idxRes != nil {
			idx = idxRes[1]
		}
		if nameRes = nameReg.FindStringSubmatch(line); nameRes != nil {
			name = nameRes[1]
		}
		if nRes = nReg.FindStringSubmatch(line); nRes != nil {
			n = nRes[1]
		}
		if strings.Contains(line, "(Log") {
			return n, name, idx
		}
	}
	return "-1", "", "" // error while reading
}

func ReadPLog(linesChan chan string) {
	if dbug {defer debug.TimeMe(time.Now())}
	for line := range linesChan {
		if strings.Contains(line, ")Log") {
			break
		}
	}
}

func ReadPDynamics(linesChan chan string) (/*m, */x, y, z/*, vx, vy, vz, ax, ay, az*/ string, err error) {
	if dbug {defer debug.TimeMe(time.Now())}
	for line := range linesChan {
// 		if strings.Contains(line, "m  =") {
// 			m = strings.Split(line, "  = ")[1]
// 		}
		if strings.Contains(line, "r  =") {
			if res = reg.FindStringSubmatch(line); res != nil {
				x = res[1]
				y = res[2]
				z = res[3]
			}
		}
// 		if strings.Contains(line, "v  =") {
// 			if res = reg.FindStringSubmatch(line); res != nil {
// 				vx = res[1]
// 				vy = res[2]
// 				vz = res[3]
// 			}
// 		}
// 		if strings.Contains(line, "a  =") {
// 			if res = reg.FindStringSubmatch(line); res != nil {
// 				ax = res[1]
// 				ay = res[2]
// 				az = res[3]
// 			}
// 		}
		if strings.Contains(line, "(Hydro") {
			return /*m,*/ x, y, z, /*vx, vy, vz, ax, ay, az,*/ nil
		}
	}
	err = errors.New("Stop reading")
	return "", "", "", /*"", "", "", "", "", "", "",*/ err
}

func ReadPHydro(linesChan chan string) {
	if dbug {defer debug.TimeMe(time.Now())}
	for line := range linesChan {
		if strings.Contains(line, ")Hydro") {
			break
		}
	}
}

func ReadPStar(linesChan chan string) {
	if dbug {defer debug.TimeMe(time.Now())}
	for line := range linesChan {
		if strings.Contains(line, ")Star") {
			break
		}
	}
}
<|endoftext|>


<commit_before>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package options

import (
	"fmt"

	"github.com/spf13/pflag"
)

// ServerOption is the main context object for the controller manager.
type ServerOption struct {
	Master               string
	Kubeconfig           string
	SchedulerName        string
	SchedulerConf        string
	NamespaceAsQueue     bool
	EnableLeaderElection bool
	LockObjectNamespace  string
}

// NewServerOption creates a new CMServer with a default config.
func NewServerOption() *ServerOption {
	s := ServerOption{}
	return &s
}

// AddFlags adds flags for a specific CMServer to the specified FlagSet
func (s *ServerOption) AddFlags(fs *pflag.FlagSet) {
	fs.StringVar(&s.Master, "master", s.Master, "The address of the Kubernetes API server (overrides any value in kubeconfig)")
	fs.StringVar(&s.Kubeconfig, "kubeconfig", s.Kubeconfig, "Path to kubeconfig file with authorization and master location information")
	// kube-arbitrator will ignore pods with scheduler names other than specified with the option
	fs.StringVar(&s.SchedulerName, "scheduler-name", "kube-batchd", "kube-batchd will handle pods with the scheduler-name")
	fs.StringVar(&s.SchedulerConf, "scheduler-conf", "", "The namespace and name of ConfigMap for scheduler configuration")
	fs.BoolVar(&s.EnableLeaderElection, "leader-elect", s.EnableLeaderElection, "Start a leader election client and gain leadership before "+
		"executing the main loop. Enable this when running replicated kar-scheduler for high availability")
	fs.BoolVar(&s.NamespaceAsQueue, "enable-namespace-as-queue", s.NamespaceAsQueue, "Make Namespace as Queue with weight one, "+
		"but kube-batchd will not handle Queue CRD anymore")
	fs.StringVar(&s.LockObjectNamespace, "lock-object-namespace", s.LockObjectNamespace, "Define the namespace of the lock object")
}

func (s *ServerOption) CheckOptionOrDie() error {
	if s.EnableLeaderElection && s.LockObjectNamespace == "" {
		return fmt.Errorf("lock-object-namespace must not be nil when LeaderElection is enabled")
	}
	return nil
}
<commit_msg>Enabled Namespace as Queue by default.<commit_after>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package options

import (
	"fmt"

	"github.com/spf13/pflag"
)

// ServerOption is the main context object for the controller manager.
type ServerOption struct {
	Master               string
	Kubeconfig           string
	SchedulerName        string
	SchedulerConf        string
	NamespaceAsQueue     bool
	EnableLeaderElection bool
	LockObjectNamespace  string
}

// NewServerOption creates a new CMServer with a default config.
func NewServerOption() *ServerOption {
	s := ServerOption{}
	return &s
}

// AddFlags adds flags for a specific CMServer to the specified FlagSet
func (s *ServerOption) AddFlags(fs *pflag.FlagSet) {
	fs.StringVar(&s.Master, "master", s.Master, "The address of the Kubernetes API server (overrides any value in kubeconfig)")
	fs.StringVar(&s.Kubeconfig, "kubeconfig", s.Kubeconfig, "Path to kubeconfig file with authorization and master location information")
	// kube-arbitrator will ignore pods with scheduler names other than specified with the option
	fs.StringVar(&s.SchedulerName, "scheduler-name", "kube-batchd", "kube-batchd will handle pods with the scheduler-name")
	fs.StringVar(&s.SchedulerConf, "scheduler-conf", "", "The namespace and name of ConfigMap for scheduler configuration")
	fs.BoolVar(&s.EnableLeaderElection, "leader-elect", s.EnableLeaderElection, "Start a leader election client and gain leadership before "+
		"executing the main loop. Enable this when running replicated kar-scheduler for high availability")
	fs.BoolVar(&s.NamespaceAsQueue, "enable-namespace-as-queue", true, "Make Namespace as Queue with weight one, "+
		"but kube-batchd will not handle Queue CRD anymore")
	fs.StringVar(&s.LockObjectNamespace, "lock-object-namespace", s.LockObjectNamespace, "Define the namespace of the lock object")
}

func (s *ServerOption) CheckOptionOrDie() error {
	if s.EnableLeaderElection && s.LockObjectNamespace == "" {
		return fmt.Errorf("lock-object-namespace must not be nil when LeaderElection is enabled")
	}
	return nil
}
<|endoftext|>


<commit_before>//  Copyright 2020 Google Inc. All Rights Reserved.
//
//  Licensed under the Apache License, Version 2.0 (the "License");
//  you may not use this file except in compliance with the License.
//  You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
//  Unless required by applicable law or agreed to in writing, software
//  distributed under the License is distributed on an "AS IS" BASIS,
//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//  See the License for the specific language governing permissions and
//  limitations under the License.

package importtestsuites

import (
	"bufio"
	"bytes"
	"context"
	"fmt"
	"io"
	"log"
	"os/exec"
	"regexp"
	"strings"
	"sync"

	"github.com/GoogleCloudPlatform/compute-image-tools/cli_tools/common/utils/path"
	"github.com/GoogleCloudPlatform/compute-image-tools/daisy"
	"github.com/GoogleCloudPlatform/compute-image-tools/go/e2e_test_utils/junitxml"
	testconfig "github.com/GoogleCloudPlatform/compute-image-tools/go/e2e_test_utils/test_config"

	"github.com/GoogleCloudPlatform/compute-image-tools/cli_tools_e2e_test/common/logging"
	"github.com/GoogleCloudPlatform/compute-image-tools/cli_tools_e2e_test/common/utils"
)

const (
	suite = "ImageImport"
)

type testCase struct {
	caseName string

	// Specify either image for file.
	source string

	// This is passed directly to the `--os` flag of gce_vm_image_import.
	os string

	// When empty, the test is expected to pass. When non-empty, the
	// actual error message must contain this for the test to pass.
	expectedError string

	// Additional args passed to gce_vm_image_import.
	extraArgs []string
}

var cases = []testCase{
	{
		caseName: "debian-9",
		source:   "projects/compute-image-tools-test/global/images/debian-9-translate",
		os:       "debian-9",
	}, {
		caseName:      "incorrect OS specified",
		source:        "projects/compute-image-tools-test/global/images/debian-9-translate",
		os:            "opensuse-15",
		expectedError: "\"debian-9\" was detected on your disk, but \"opensuse-15\" was specified",
	}, {
		caseName:  "rhel-7-uefi",
		source:    "projects/compute-image-tools-test/global/images/linux-uefi-no-guestosfeature-rhel7",
		os:        "rhel-7",
		extraArgs: []string{"-uefi_compatible=true"},
	}, {
		caseName:  "windows-2019-uefi",
		source:    "projects/compute-image-tools-test/global/images/windows-2019-uefi-nodrivers",
		os:        "windows-2019",
		extraArgs: []string{"-uefi_compatible=true"},
	}, {
		caseName: "windows-10-x86-byol",
		source:   "projects/compute-image-tools-test/global/images/windows-10-1909-ent-x86-nodrivers",
		os:       "windows-10-x86-byol",
	},
}

func (t testCase) run(ctx context.Context, junit *junitxml.TestCase, logger *log.Logger,
	testProjectConfig *testconfig.Project, testType utils.CLITestType) {
	logger = t.createTestScopedLogger(junit, logger)
	imageName := "e2e-test-image-import" + path.RandString(5)
	imagePath := fmt.Sprintf("projects/%s/global/images/%s", testProjectConfig.TestProjectID, imageName)

	importLogs, err := t.runImport(junit, logger, testProjectConfig, imageName)

	if t.expectedError != "" {
		t.verifyExpectedError(junit, err, importLogs)
	} else if err != nil {
		t.writeImportFailed(junit, importLogs)
	} else {
		err = t.runPostTranslateTest(ctx, imagePath, testProjectConfig, logger)
		if err != nil {
			junit.WriteFailure("Failed post translate test: %v", err)
		}
	}
}

// createTestScopedLogger returns a new logger that is prefixed with the name of the test.
func (t testCase) createTestScopedLogger(junit *junitxml.TestCase, logger *log.Logger) *log.Logger {
	return log.New(logger.Writer(), junit.Name+" ", logger.Flags())
}

// runImport runs an image import workflow, and returns its console output and error.
func (t testCase) runImport(junit *junitxml.TestCase, logger *log.Logger,
	testProjectConfig *testconfig.Project, imageName string) (*bytes.Buffer, error) {
	args := []string{
		"-client_id", "e2e",
		"-inspect",
		"-os", t.os,
		"-project", testProjectConfig.TestProjectID,
		"-zone", testProjectConfig.TestZone,
		"-image_name", imageName,
	}
	if strings.Contains(t.source, "gs://") {
		args = append(args, "-source_file", t.source)
	} else {
		args = append(args, "-source_image", t.source)
	}
	if t.extraArgs != nil {
		args = append(args, t.extraArgs...)
	}
	cmd := exec.Command("./gce_vm_image_import", args...)
	cmdOutput := &bytes.Buffer{}
	cmd.Stdout = io.MultiWriter(cmdOutput, logging.AsWriter(logger))
	cmd.Stderr = io.MultiWriter(cmdOutput, logging.AsWriter(logger))
	err := cmd.Start()
	if err != nil {
		return cmdOutput, err
	}
	return cmdOutput, cmd.Wait()
}

func (t testCase) writeImportFailed(junit *junitxml.TestCase, importLog *bytes.Buffer) {
	// The Logf messages are shown in the test framework UI.
	junit.Logf(importLog.String())
	junit.WriteFailure("Import failed: %q", getLastLine(importLog))
}

func (t testCase) verifyExpectedError(junit *junitxml.TestCase, actualErr error, importLog *bytes.Buffer) {
	if actualErr == nil {
		junit.WriteFailure("Expected failed import: %q.", t.expectedError)
		return
	}

	lastLine := getLastLine(importLog)
	if !strings.Contains(lastLine, t.expectedError) {
		junit.WriteFailure("Message shown to user %q did not contain %q.",
			lastLine, t.expectedError)
	}
}

// runPostTranslateTest boots the instance and executes a startup script containing tests.
func (t testCase) runPostTranslateTest(ctx context.Context, imagePath string,
	testProjectConfig *testconfig.Project, logger *log.Logger) error {
	wf, err := daisy.NewFromFile("post_translate_test.wf.json")
	if err != nil {
		return err
	}

	varMap := map[string]string{
		"image_under_test":            imagePath,
		"path_to_post_translate_test": t.testScript(),
	}

	for k, v := range varMap {
		wf.AddVar(k, v)
	}

	wf.Logger = logging.AsDaisyLogger(logger)
	wf.Project = testProjectConfig.TestProjectID
	wf.Zone = testProjectConfig.TestZone
	err = wf.Run(ctx)
	return err
}

func (t testCase) testScript() string {
	if strings.Contains(t.os, "windows") {
		return "post_translate_test.ps1"
	}
	return "post_translate_test.sh"
}

// ImageImportSuite performs image imports, and verifies that the results are bootable and are
// are able to perform basic GCP operations. The suite includes support for negative test cases,
// where error messages are validated against expected error messages.
func ImageImportSuite(
	ctx context.Context, tswg *sync.WaitGroup, testSuites chan *junitxml.TestSuite,
	logger *log.Logger, testSuiteRegex, testCaseRegex *regexp.Regexp,
	testProjectConfig *testconfig.Project) {

	junits := map[*junitxml.TestCase]func(
		context.Context, *junitxml.TestCase, *log.Logger, *testconfig.Project, utils.CLITestType){}

	for _, testCase := range cases {
		junit := junitxml.NewTestCase(
			suite, fmt.Sprintf("[%v]", testCase.caseName))
		junits[junit] = testCase.run
	}

	testsMap := map[utils.CLITestType]map[*junitxml.TestCase]func(
		context.Context, *junitxml.TestCase, *log.Logger, *testconfig.Project, utils.CLITestType){
		utils.Wrapper: junits,
	}

	utils.CLITestSuite(ctx, tswg, testSuites, logger, testSuiteRegex, testCaseRegex,
		testProjectConfig, suite, testsMap)
}

// getLastLine consumes a buffer, and returns its last line.
func getLastLine(buffer *bytes.Buffer) string {
	var lastLine string
	scanner := bufio.NewScanner(buffer)
	for scanner.Scan() {
		lastLine = scanner.Text()
	}
	return lastLine
}
<commit_msg>Test RHEL 8 import with e2e (#1323)<commit_after>//  Copyright 2020 Google Inc. All Rights Reserved.
//
//  Licensed under the Apache License, Version 2.0 (the "License");
//  you may not use this file except in compliance with the License.
//  You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
//  Unless required by applicable law or agreed to in writing, software
//  distributed under the License is distributed on an "AS IS" BASIS,
//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//  See the License for the specific language governing permissions and
//  limitations under the License.

package importtestsuites

import (
	"bufio"
	"bytes"
	"context"
	"fmt"
	"io"
	"log"
	"os/exec"
	"regexp"
	"strings"
	"sync"

	"github.com/GoogleCloudPlatform/compute-image-tools/cli_tools/common/utils/path"
	"github.com/GoogleCloudPlatform/compute-image-tools/daisy"
	"github.com/GoogleCloudPlatform/compute-image-tools/go/e2e_test_utils/junitxml"
	testconfig "github.com/GoogleCloudPlatform/compute-image-tools/go/e2e_test_utils/test_config"

	"github.com/GoogleCloudPlatform/compute-image-tools/cli_tools_e2e_test/common/logging"
	"github.com/GoogleCloudPlatform/compute-image-tools/cli_tools_e2e_test/common/utils"
)

const (
	suite = "ImageImport"
)

type testCase struct {
	caseName string

	// Specify either image for file.
	source string

	// This is passed directly to the `--os` flag of gce_vm_image_import.
	os string

	// When empty, the test is expected to pass. When non-empty, the
	// actual error message must contain this for the test to pass.
	expectedError string

	// Additional args passed to gce_vm_image_import.
	extraArgs []string
}

var cases = []testCase{
	{
		caseName: "debian-9",
		source:   "projects/compute-image-tools-test/global/images/debian-9-translate",
		os:       "debian-9",
	}, {
		caseName:      "incorrect OS specified",
		source:        "projects/compute-image-tools-test/global/images/debian-9-translate",
		os:            "opensuse-15",
		expectedError: "\"debian-9\" was detected on your disk, but \"opensuse-15\" was specified",
	}, {
		caseName:  "rhel-7-uefi",
		source:    "projects/compute-image-tools-test/global/images/linux-uefi-no-guestosfeature-rhel7",
		os:        "rhel-7",
		extraArgs: []string{"-uefi_compatible=true"},
	}, {
		caseName: "rhel-8",
		source:   "projects/compute-image-tools-test/global/images/rhel-8-import",
		os:       "rhel-8",
	}, {
		caseName:  "windows-2019-uefi",
		source:    "projects/compute-image-tools-test/global/images/windows-2019-uefi-nodrivers",
		os:        "windows-2019",
		extraArgs: []string{"-uefi_compatible=true"},
	}, {
		caseName: "windows-10-x86-byol",
		source:   "projects/compute-image-tools-test/global/images/windows-10-1909-ent-x86-nodrivers",
		os:       "windows-10-x86-byol",
	},
}

func (t testCase) run(ctx context.Context, junit *junitxml.TestCase, logger *log.Logger,
	testProjectConfig *testconfig.Project, testType utils.CLITestType) {
	logger = t.createTestScopedLogger(junit, logger)
	imageName := "e2e-test-image-import" + path.RandString(5)
	imagePath := fmt.Sprintf("projects/%s/global/images/%s", testProjectConfig.TestProjectID, imageName)

	importLogs, err := t.runImport(junit, logger, testProjectConfig, imageName)

	if t.expectedError != "" {
		t.verifyExpectedError(junit, err, importLogs)
	} else if err != nil {
		t.writeImportFailed(junit, importLogs)
	} else {
		err = t.runPostTranslateTest(ctx, imagePath, testProjectConfig, logger)
		if err != nil {
			junit.WriteFailure("Failed post translate test: %v", err)
		}
	}
}

// createTestScopedLogger returns a new logger that is prefixed with the name of the test.
func (t testCase) createTestScopedLogger(junit *junitxml.TestCase, logger *log.Logger) *log.Logger {
	return log.New(logger.Writer(), junit.Name+" ", logger.Flags())
}

// runImport runs an image import workflow, and returns its console output and error.
func (t testCase) runImport(junit *junitxml.TestCase, logger *log.Logger,
	testProjectConfig *testconfig.Project, imageName string) (*bytes.Buffer, error) {
	args := []string{
		"-client_id", "e2e",
		"-inspect",
		"-os", t.os,
		"-project", testProjectConfig.TestProjectID,
		"-zone", testProjectConfig.TestZone,
		"-image_name", imageName,
	}
	if strings.Contains(t.source, "gs://") {
		args = append(args, "-source_file", t.source)
	} else {
		args = append(args, "-source_image", t.source)
	}
	if t.extraArgs != nil {
		args = append(args, t.extraArgs...)
	}
	cmd := exec.Command("./gce_vm_image_import", args...)
	cmdOutput := &bytes.Buffer{}
	cmd.Stdout = io.MultiWriter(cmdOutput, logging.AsWriter(logger))
	cmd.Stderr = io.MultiWriter(cmdOutput, logging.AsWriter(logger))
	err := cmd.Start()
	if err != nil {
		return cmdOutput, err
	}
	return cmdOutput, cmd.Wait()
}

func (t testCase) writeImportFailed(junit *junitxml.TestCase, importLog *bytes.Buffer) {
	// The Logf messages are shown in the test framework UI.
	junit.Logf(importLog.String())
	junit.WriteFailure("Import failed: %q", getLastLine(importLog))
}

func (t testCase) verifyExpectedError(junit *junitxml.TestCase, actualErr error, importLog *bytes.Buffer) {
	if actualErr == nil {
		junit.WriteFailure("Expected failed import: %q.", t.expectedError)
		return
	}

	lastLine := getLastLine(importLog)
	if !strings.Contains(lastLine, t.expectedError) {
		junit.WriteFailure("Message shown to user %q did not contain %q.",
			lastLine, t.expectedError)
	}
}

// runPostTranslateTest boots the instance and executes a startup script containing tests.
func (t testCase) runPostTranslateTest(ctx context.Context, imagePath string,
	testProjectConfig *testconfig.Project, logger *log.Logger) error {
	wf, err := daisy.NewFromFile("post_translate_test.wf.json")
	if err != nil {
		return err
	}

	varMap := map[string]string{
		"image_under_test":            imagePath,
		"path_to_post_translate_test": t.testScript(),
	}

	for k, v := range varMap {
		wf.AddVar(k, v)
	}

	wf.Logger = logging.AsDaisyLogger(logger)
	wf.Project = testProjectConfig.TestProjectID
	wf.Zone = testProjectConfig.TestZone
	err = wf.Run(ctx)
	return err
}

func (t testCase) testScript() string {
	if strings.Contains(t.os, "windows") {
		return "post_translate_test.ps1"
	}
	return "post_translate_test.sh"
}

// ImageImportSuite performs image imports, and verifies that the results are bootable and are
// are able to perform basic GCP operations. The suite includes support for negative test cases,
// where error messages are validated against expected error messages.
func ImageImportSuite(
	ctx context.Context, tswg *sync.WaitGroup, testSuites chan *junitxml.TestSuite,
	logger *log.Logger, testSuiteRegex, testCaseRegex *regexp.Regexp,
	testProjectConfig *testconfig.Project) {

	junits := map[*junitxml.TestCase]func(
		context.Context, *junitxml.TestCase, *log.Logger, *testconfig.Project, utils.CLITestType){}

	for _, testCase := range cases {
		junit := junitxml.NewTestCase(
			suite, fmt.Sprintf("[%v]", testCase.caseName))
		junits[junit] = testCase.run
	}

	testsMap := map[utils.CLITestType]map[*junitxml.TestCase]func(
		context.Context, *junitxml.TestCase, *log.Logger, *testconfig.Project, utils.CLITestType){
		utils.Wrapper: junits,
	}

	utils.CLITestSuite(ctx, tswg, testSuites, logger, testSuiteRegex, testCaseRegex,
		testProjectConfig, suite, testsMap)
}

// getLastLine consumes a buffer, and returns its last line.
func getLastLine(buffer *bytes.Buffer) string {
	var lastLine string
	scanner := bufio.NewScanner(buffer)
	for scanner.Scan() {
		lastLine = scanner.Text()
	}
	return lastLine
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"io"
	"math/rand"
	"os"
	"time"

	"github.com/onsi/gomega"

	"github.com/onsi/ginkgo"
	"github.com/spf13/cobra"
	"github.com/spf13/pflag"
	"k8s.io/klog"

	"k8s.io/apiserver/pkg/util/logs"
	"k8s.io/kubernetes/pkg/kubectl/util/templates"
	e2e "k8s.io/kubernetes/test/e2e/framework"

	"github.com/openshift/library-go/pkg/serviceability"
	"github.com/openshift/origin/pkg/cmd/flagtypes"
	"github.com/openshift/origin/pkg/monitor"
	testginkgo "github.com/openshift/origin/pkg/test/ginkgo"
	exutil "github.com/openshift/origin/test/extended/util"
)

func main() {
	logs.InitLogs()
	defer logs.FlushLogs()

	rand.Seed(time.Now().UTC().UnixNano())

	root := &cobra.Command{
		Long: templates.LongDesc(`
		OpenShift Tests

		This command verifies behavior of an OpenShift cluster by running remote tests against
		the cluster API that exercise functionality. In general these tests may be disruptive
		or require elevated privileges - see the descriptions of each test suite.
		`),
	}
	flagtypes.GLog(root.PersistentFlags())

	root.AddCommand(
		newRunCommand(),
		newRunUpgradeCommand(),
		newRunTestCommand(),
		newRunMonitorCommand(),
	)

	pflag.CommandLine = pflag.NewFlagSet("empty", pflag.ExitOnError)
	flag.CommandLine = flag.NewFlagSet("empty", flag.ExitOnError)
	exutil.InitStandardFlags()

	if err := func() error {
		defer serviceability.Profile(os.Getenv("OPENSHIFT_PROFILE")).Stop()
		return root.Execute()
	}(); err != nil {
		if ex, ok := err.(testginkgo.ExitError); ok {
			os.Exit(ex.Code)
		}
		fmt.Fprintf(os.Stderr, "error: %v\n", err)
		os.Exit(1)
	}
}

func newRunMonitorCommand() *cobra.Command {
	monitorOpt := &monitor.Options{
		Out:    os.Stdout,
		ErrOut: os.Stderr,
	}
	cmd := &cobra.Command{
		Use:   "run-monitor",
		Short: "Continuously verify the cluster is functional",
		Long: templates.LongDesc(`
		Run a continuous verification process

		`),

		SilenceUsage:  true,
		SilenceErrors: true,
		RunE: func(cmd *cobra.Command, args []string) error {
			return monitorOpt.Run()
		},
	}
	return cmd
}

func newRunCommand() *cobra.Command {
	opt := &testginkgo.Options{
		Suites: staticSuites,
	}
	cmd := &cobra.Command{
		Use:   "run SUITE",
		Short: "Run a test suite",
		Long: templates.LongDesc(`
		Run a test suite against an OpenShift server

		This command will run one of the following suites against a cluster identified by the current
		KUBECONFIG file. See the suite description for more on what actions the suite will take.

		If you specify the --dry-run argument, the names of each individual test that is part of the
		suite will be printed, one per line. You may filter this list and pass it back to the run
		command with the --file argument. You may also pipe a list of test names, one per line, on
		standard input by passing "-f -".

		`) + testginkgo.SuitesString(opt.Suites, "\n\nAvailable test suites:\n\n"),

		SilenceUsage:  true,
		SilenceErrors: true,
		RunE: func(cmd *cobra.Command, args []string) error {
			return mirrorToFile(opt, func() error {
				if err := initProvider(opt.Provider); err != nil {
					return err
				}
				e2e.AfterReadingAllFlags(exutil.TestContext)
				exutil.TestContext.DumpLogsOnFailure = true
				return opt.Run(args)
			})
		},
	}
	bindOptions(opt, cmd.Flags())
	return cmd
}

func newRunUpgradeCommand() *cobra.Command {
	opt := &testginkgo.Options{Suites: upgradeSuites}
	upgradeOpt := &UpgradeOptions{}

	cmd := &cobra.Command{
		Use:   "run-upgrade SUITE",
		Short: "Run an upgrade suite",
		Long: templates.LongDesc(`
		Run an upgrade test suite against an OpenShift server

		This command will run one of the following suites against a cluster identified by the current
		KUBECONFIG file. See the suite description for more on what actions the suite will take.

		If you specify the --dry-run argument, the actions the suite will take will be printed to the
		output.

		Supported options:

		* abort-at=NUMBER - Set to a number between 0 and 100 to control the percent of operators
		at which to stop the current upgrade and roll back to the current version.
		* disrupt-reboot=POLICY - During upgrades, periodically reboot master nodes. If set to 'graceful'
		the reboot will allow the node to shut down services in an orderly fashion. If set to 'force' the
		machine will terminate immediately without clean shutdown.

		`) + testginkgo.SuitesString(opt.Suites, "\n\nAvailable upgrade suites:\n\n"),

		SilenceUsage:  true,
		SilenceErrors: true,
		RunE: func(cmd *cobra.Command, args []string) error {
			return mirrorToFile(opt, func() error {
				if len(upgradeOpt.ToImage) == 0 {
					return fmt.Errorf("--to-image must be specified to run an upgrade test")
				}

				if len(args) > 0 {
					for _, suite := range opt.Suites {
						if suite.Name == args[0] {
							upgradeOpt.Suite = suite.Name
							upgradeOpt.JUnitDir = opt.JUnitDir
							value := upgradeOpt.ToEnv()
							if err := initUpgrade(value); err != nil {
								return err
							}
							opt.SuiteOptions = value
							break
						}
					}
				}

				if err := initProvider(opt.Provider); err != nil {
					return err
				}
				e2e.AfterReadingAllFlags(exutil.TestContext)
				exutil.TestContext.DumpLogsOnFailure = true
				return opt.Run(args)
			})
		},
	}

	bindOptions(opt, cmd.Flags())
	bindUpgradeOptions(upgradeOpt, cmd.Flags())
	return cmd
}

func newRunTestCommand() *cobra.Command {
	testOpt := &testginkgo.TestOptions{
		Out:    os.Stdout,
		ErrOut: os.Stderr,
	}
	cmd := &cobra.Command{
		Use:   "run-test NAME",
		Short: "Run a single test by name",
		Long: templates.LongDesc(`
		Execute a single test

		This executes a single test by name. It is used by the run command during suite execution but may also
		be used to test in isolation while developing new tests.
		`),

		SilenceUsage:  true,
		SilenceErrors: true,
		RunE: func(cmd *cobra.Command, args []string) error {
			if err := initProvider(os.Getenv("TEST_PROVIDER")); err != nil {
				return err
			}
			if err := initUpgrade(os.Getenv("TEST_SUITE_OPTIONS")); err != nil {
				return err
			}
			e2e.AfterReadingAllFlags(exutil.TestContext)
			exutil.TestContext.DumpLogsOnFailure = true
			return testOpt.Run(args)
		},
	}
	cmd.Flags().BoolVar(&testOpt.DryRun, "dry-run", testOpt.DryRun, "Print the test to run without executing them.")
	return cmd
}

// mirrorToFile ensures a copy of all output goes to the provided OutFile, including
// any error returned from fn. The function returns fn() or any error encountered while
// attempting to open the file.
func mirrorToFile(opt *testginkgo.Options, fn func() error) error {
	if len(opt.OutFile) == 0 {
		opt.Out, opt.ErrOut = os.Stdout, os.Stderr
		return fn()
	}

	f, err := os.OpenFile(opt.OutFile, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0640)
	if err != nil {
		return err
	}
	opt.Out = io.MultiWriter(os.Stdout, f)
	opt.ErrOut = io.MultiWriter(os.Stderr, f)
	exitErr := fn()
	if exitErr != nil {
		fmt.Fprintf(f, "error: %s", exitErr)
	}
	if err := f.Close(); err != nil {
		fmt.Fprintf(os.Stderr, "error: Unable to close output file\n")
	}
	return exitErr
}

func bindOptions(opt *testginkgo.Options, flags *pflag.FlagSet) {
	flags.BoolVar(&opt.DryRun, "dry-run", opt.DryRun, "Print the tests to run without executing them.")
	flags.BoolVar(&opt.PrintCommands, "print-commands", opt.PrintCommands, "Print the sub-commands that would be executed instead.")
	flags.StringVar(&opt.JUnitDir, "junit-dir", opt.JUnitDir, "The directory to write test reports to.")
	flags.StringVar(&opt.Provider, "provider", opt.Provider, "The cluster infrastructure provider. Will automatically default to the correct value.")
	flags.StringVarP(&opt.TestFile, "file", "f", opt.TestFile, "Create a suite from the newline-delimited test names in this file.")
	flags.StringVarP(&opt.OutFile, "output-file", "o", opt.OutFile, "Write all test output to this file.")
	flags.DurationVar(&opt.Timeout, "timeout", opt.Timeout, "Set the maximum time a test can run before being aborted. This is read from the suite by default, but will be 10 minutes otherwise.")
	flags.BoolVar(&opt.IncludeSuccessOutput, "include-success", opt.IncludeSuccessOutput, "Print output from successful tests.")
}

func initProvider(provider string) error {
	// record the exit error to the output file
	if err := decodeProviderTo(provider, exutil.TestContext); err != nil {
		return err
	}
	exutil.TestContext.AllowedNotReadyNodes = 100
	exutil.TestContext.MaxNodesToGather = 0
	exutil.TestContext.Viper = os.Getenv("VIPERCONFIG")

	// set defaults so these tests don't log
	exutil.TestContext.LoggingSoak.Scale = 1
	exutil.TestContext.LoggingSoak.MilliSecondsBetweenWaves = 5000

	exutil.AnnotateTestSuite()
	exutil.InitTest()
	gomega.RegisterFailHandler(ginkgo.Fail)

	// TODO: infer SSH keys from the cluster
	return nil
}

func decodeProviderTo(provider string, testContext *e2e.TestContextType) error {
	switch provider {
	case "":
		if _, ok := os.LookupEnv("KUBE_SSH_USER"); ok {
			if _, ok := os.LookupEnv("LOCAL_SSH_KEY"); ok {
				testContext.Provider = "local"
			}
		}
		// TODO: detect which provider the cluster is running and use that as a default.
	default:
		var providerInfo struct{ Type string }
		if err := json.Unmarshal([]byte(provider), &providerInfo); err != nil {
			return fmt.Errorf("provider must be a JSON object with the 'type' key at a minimum: %v", err)
		}
		if len(providerInfo.Type) == 0 {
			return fmt.Errorf("provider must be a JSON object with the 'type' key")
		}
		testContext.Provider = providerInfo.Type
		if err := json.Unmarshal([]byte(provider), &testContext.CloudConfig); err != nil {
			return fmt.Errorf("provider must decode into the cloud config object: %v", err)
		}
	}
	if len(testContext.Provider) == 0 {
		testContext.Provider = "skeleton"
	}
	klog.V(2).Infof("Provider %s: %#v", testContext.Provider, testContext.CloudConfig)
	return nil
}
<commit_msg>add more globals to try to dump logs on failure<commit_after>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"io"
	"math/rand"
	"os"
	"time"

	"github.com/onsi/gomega"

	"github.com/onsi/ginkgo"
	"github.com/spf13/cobra"
	"github.com/spf13/pflag"
	"k8s.io/klog"

	"k8s.io/apiserver/pkg/util/logs"
	"k8s.io/kubernetes/pkg/kubectl/util/templates"
	e2e "k8s.io/kubernetes/test/e2e/framework"

	"github.com/openshift/library-go/pkg/serviceability"
	"github.com/openshift/origin/pkg/cmd/flagtypes"
	"github.com/openshift/origin/pkg/monitor"
	testginkgo "github.com/openshift/origin/pkg/test/ginkgo"
	exutil "github.com/openshift/origin/test/extended/util"
)

func main() {
	logs.InitLogs()
	defer logs.FlushLogs()

	rand.Seed(time.Now().UTC().UnixNano())

	root := &cobra.Command{
		Long: templates.LongDesc(`
		OpenShift Tests

		This command verifies behavior of an OpenShift cluster by running remote tests against
		the cluster API that exercise functionality. In general these tests may be disruptive
		or require elevated privileges - see the descriptions of each test suite.
		`),
	}
	flagtypes.GLog(root.PersistentFlags())

	root.AddCommand(
		newRunCommand(),
		newRunUpgradeCommand(),
		newRunTestCommand(),
		newRunMonitorCommand(),
	)

	pflag.CommandLine = pflag.NewFlagSet("empty", pflag.ExitOnError)
	flag.CommandLine = flag.NewFlagSet("empty", flag.ExitOnError)
	exutil.InitStandardFlags()

	if err := func() error {
		defer serviceability.Profile(os.Getenv("OPENSHIFT_PROFILE")).Stop()
		return root.Execute()
	}(); err != nil {
		if ex, ok := err.(testginkgo.ExitError); ok {
			os.Exit(ex.Code)
		}
		fmt.Fprintf(os.Stderr, "error: %v\n", err)
		os.Exit(1)
	}
}

func newRunMonitorCommand() *cobra.Command {
	monitorOpt := &monitor.Options{
		Out:    os.Stdout,
		ErrOut: os.Stderr,
	}
	cmd := &cobra.Command{
		Use:   "run-monitor",
		Short: "Continuously verify the cluster is functional",
		Long: templates.LongDesc(`
		Run a continuous verification process

		`),

		SilenceUsage:  true,
		SilenceErrors: true,
		RunE: func(cmd *cobra.Command, args []string) error {
			return monitorOpt.Run()
		},
	}
	return cmd
}

func newRunCommand() *cobra.Command {
	opt := &testginkgo.Options{
		Suites: staticSuites,
	}
	cmd := &cobra.Command{
		Use:   "run SUITE",
		Short: "Run a test suite",
		Long: templates.LongDesc(`
		Run a test suite against an OpenShift server

		This command will run one of the following suites against a cluster identified by the current
		KUBECONFIG file. See the suite description for more on what actions the suite will take.

		If you specify the --dry-run argument, the names of each individual test that is part of the
		suite will be printed, one per line. You may filter this list and pass it back to the run
		command with the --file argument. You may also pipe a list of test names, one per line, on
		standard input by passing "-f -".

		`) + testginkgo.SuitesString(opt.Suites, "\n\nAvailable test suites:\n\n"),

		SilenceUsage:  true,
		SilenceErrors: true,
		RunE: func(cmd *cobra.Command, args []string) error {
			return mirrorToFile(opt, func() error {
				if err := initProvider(opt.Provider); err != nil {
					return err
				}
				e2e.AfterReadingAllFlags(exutil.TestContext)
				e2e.TestContext.DumpLogsOnFailure = true
				exutil.TestContext.DumpLogsOnFailure = true
				return opt.Run(args)
			})
		},
	}
	bindOptions(opt, cmd.Flags())
	return cmd
}

func newRunUpgradeCommand() *cobra.Command {
	opt := &testginkgo.Options{Suites: upgradeSuites}
	upgradeOpt := &UpgradeOptions{}

	cmd := &cobra.Command{
		Use:   "run-upgrade SUITE",
		Short: "Run an upgrade suite",
		Long: templates.LongDesc(`
		Run an upgrade test suite against an OpenShift server

		This command will run one of the following suites against a cluster identified by the current
		KUBECONFIG file. See the suite description for more on what actions the suite will take.

		If you specify the --dry-run argument, the actions the suite will take will be printed to the
		output.

		Supported options:

		* abort-at=NUMBER - Set to a number between 0 and 100 to control the percent of operators
		at which to stop the current upgrade and roll back to the current version.
		* disrupt-reboot=POLICY - During upgrades, periodically reboot master nodes. If set to 'graceful'
		the reboot will allow the node to shut down services in an orderly fashion. If set to 'force' the
		machine will terminate immediately without clean shutdown.

		`) + testginkgo.SuitesString(opt.Suites, "\n\nAvailable upgrade suites:\n\n"),

		SilenceUsage:  true,
		SilenceErrors: true,
		RunE: func(cmd *cobra.Command, args []string) error {
			return mirrorToFile(opt, func() error {
				if len(upgradeOpt.ToImage) == 0 {
					return fmt.Errorf("--to-image must be specified to run an upgrade test")
				}

				if len(args) > 0 {
					for _, suite := range opt.Suites {
						if suite.Name == args[0] {
							upgradeOpt.Suite = suite.Name
							upgradeOpt.JUnitDir = opt.JUnitDir
							value := upgradeOpt.ToEnv()
							if err := initUpgrade(value); err != nil {
								return err
							}
							opt.SuiteOptions = value
							break
						}
					}
				}

				if err := initProvider(opt.Provider); err != nil {
					return err
				}
				e2e.AfterReadingAllFlags(exutil.TestContext)
				e2e.TestContext.DumpLogsOnFailure = true
				exutil.TestContext.DumpLogsOnFailure = true
				return opt.Run(args)
			})
		},
	}

	bindOptions(opt, cmd.Flags())
	bindUpgradeOptions(upgradeOpt, cmd.Flags())
	return cmd
}

func newRunTestCommand() *cobra.Command {
	testOpt := &testginkgo.TestOptions{
		Out:    os.Stdout,
		ErrOut: os.Stderr,
	}
	cmd := &cobra.Command{
		Use:   "run-test NAME",
		Short: "Run a single test by name",
		Long: templates.LongDesc(`
		Execute a single test

		This executes a single test by name. It is used by the run command during suite execution but may also
		be used to test in isolation while developing new tests.
		`),

		SilenceUsage:  true,
		SilenceErrors: true,
		RunE: func(cmd *cobra.Command, args []string) error {
			if err := initProvider(os.Getenv("TEST_PROVIDER")); err != nil {
				return err
			}
			if err := initUpgrade(os.Getenv("TEST_SUITE_OPTIONS")); err != nil {
				return err
			}
			e2e.AfterReadingAllFlags(exutil.TestContext)
			e2e.TestContext.DumpLogsOnFailure = true
			exutil.TestContext.DumpLogsOnFailure = true
			return testOpt.Run(args)
		},
	}
	cmd.Flags().BoolVar(&testOpt.DryRun, "dry-run", testOpt.DryRun, "Print the test to run without executing them.")
	return cmd
}

// mirrorToFile ensures a copy of all output goes to the provided OutFile, including
// any error returned from fn. The function returns fn() or any error encountered while
// attempting to open the file.
func mirrorToFile(opt *testginkgo.Options, fn func() error) error {
	if len(opt.OutFile) == 0 {
		opt.Out, opt.ErrOut = os.Stdout, os.Stderr
		return fn()
	}

	f, err := os.OpenFile(opt.OutFile, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0640)
	if err != nil {
		return err
	}
	opt.Out = io.MultiWriter(os.Stdout, f)
	opt.ErrOut = io.MultiWriter(os.Stderr, f)
	exitErr := fn()
	if exitErr != nil {
		fmt.Fprintf(f, "error: %s", exitErr)
	}
	if err := f.Close(); err != nil {
		fmt.Fprintf(os.Stderr, "error: Unable to close output file\n")
	}
	return exitErr
}

func bindOptions(opt *testginkgo.Options, flags *pflag.FlagSet) {
	flags.BoolVar(&opt.DryRun, "dry-run", opt.DryRun, "Print the tests to run without executing them.")
	flags.BoolVar(&opt.PrintCommands, "print-commands", opt.PrintCommands, "Print the sub-commands that would be executed instead.")
	flags.StringVar(&opt.JUnitDir, "junit-dir", opt.JUnitDir, "The directory to write test reports to.")
	flags.StringVar(&opt.Provider, "provider", opt.Provider, "The cluster infrastructure provider. Will automatically default to the correct value.")
	flags.StringVarP(&opt.TestFile, "file", "f", opt.TestFile, "Create a suite from the newline-delimited test names in this file.")
	flags.StringVarP(&opt.OutFile, "output-file", "o", opt.OutFile, "Write all test output to this file.")
	flags.DurationVar(&opt.Timeout, "timeout", opt.Timeout, "Set the maximum time a test can run before being aborted. This is read from the suite by default, but will be 10 minutes otherwise.")
	flags.BoolVar(&opt.IncludeSuccessOutput, "include-success", opt.IncludeSuccessOutput, "Print output from successful tests.")
}

func initProvider(provider string) error {
	// record the exit error to the output file
	if err := decodeProviderTo(provider, exutil.TestContext); err != nil {
		return err
	}
	exutil.TestContext.AllowedNotReadyNodes = 100
	exutil.TestContext.MaxNodesToGather = 0
	exutil.TestContext.Viper = os.Getenv("VIPERCONFIG")

	// set defaults so these tests don't log
	exutil.TestContext.LoggingSoak.Scale = 1
	exutil.TestContext.LoggingSoak.MilliSecondsBetweenWaves = 5000

	exutil.AnnotateTestSuite()
	exutil.InitTest()
	gomega.RegisterFailHandler(ginkgo.Fail)

	// TODO: infer SSH keys from the cluster
	return nil
}

func decodeProviderTo(provider string, testContext *e2e.TestContextType) error {
	switch provider {
	case "":
		if _, ok := os.LookupEnv("KUBE_SSH_USER"); ok {
			if _, ok := os.LookupEnv("LOCAL_SSH_KEY"); ok {
				testContext.Provider = "local"
			}
		}
		// TODO: detect which provider the cluster is running and use that as a default.
	default:
		var providerInfo struct{ Type string }
		if err := json.Unmarshal([]byte(provider), &providerInfo); err != nil {
			return fmt.Errorf("provider must be a JSON object with the 'type' key at a minimum: %v", err)
		}
		if len(providerInfo.Type) == 0 {
			return fmt.Errorf("provider must be a JSON object with the 'type' key")
		}
		testContext.Provider = providerInfo.Type
		if err := json.Unmarshal([]byte(provider), &testContext.CloudConfig); err != nil {
			return fmt.Errorf("provider must decode into the cloud config object: %v", err)
		}
	}
	if len(testContext.Provider) == 0 {
		testContext.Provider = "skeleton"
	}
	klog.V(2).Infof("Provider %s: %#v", testContext.Provider, testContext.CloudConfig)
	return nil
}
<|endoftext|>


<commit_before>//+build !windows

package daemon // import "github.com/docker/docker/daemon"

import (
	"context"
	"fmt"
	"os/exec"
	"regexp"
	"strconv"
	"strings"

	"github.com/docker/docker/api/types/container"
)

func validatePSArgs(psArgs string) error {
	// NOTE: \\s does not detect unicode whitespaces.
	// So we use fieldsASCII instead of strings.Fields in parsePSOutput.
	// See https://github.com/docker/docker/pull/24358
	// nolint: gosimple
	re := regexp.MustCompile("\\s+([^\\s]*)=\\s*(PID[^\\s]*)")
	for _, group := range re.FindAllStringSubmatch(psArgs, -1) {
		if len(group) >= 3 {
			k := group[1]
			v := group[2]
			if k != "pid" {
				return fmt.Errorf("specifying \"%s=%s\" is not allowed", k, v)
			}
		}
	}
	return nil
}

// fieldsASCII is similar to strings.Fields but only allows ASCII whitespaces
func fieldsASCII(s string) []string {
	fn := func(r rune) bool {
		switch r {
		case '\t', '\n', '\f', '\r', ' ':
			return true
		}
		return false
	}
	return strings.FieldsFunc(s, fn)
}

func appendProcess2ProcList(procList *container.ContainerTopOKBody, fields []string) {
	// Make sure number of fields equals number of header titles
	// merging "overhanging" fields
	process := fields[:len(procList.Titles)-1]
	process = append(process, strings.Join(fields[len(procList.Titles)-1:], " "))
	procList.Processes = append(procList.Processes, process)
}

func hasPid(procs []uint32, pid int) bool {
	for _, p := range procs {
		if int(p) == pid {
			return true
		}
	}
	return false
}

func parsePSOutput(output []byte, procs []uint32) (*container.ContainerTopOKBody, error) {
	procList := &container.ContainerTopOKBody{}

	lines := strings.Split(string(output), "\n")
	procList.Titles = fieldsASCII(lines[0])

	pidIndex := -1
	for i, name := range procList.Titles {
		if name == "PID" {
			pidIndex = i
		}
	}
	if pidIndex == -1 {
		return nil, fmt.Errorf("Couldn't find PID field in ps output")
	}

	// loop through the output and extract the PID from each line
	// fixing #30580, be able to display thread line also when "m" option used
	// in "docker top" client command
	preContainedPidFlag := false
	for _, line := range lines[1:] {
		if len(line) == 0 {
			continue
		}
		fields := fieldsASCII(line)

		var (
			p   int
			err error
		)

		if fields[pidIndex] == "-" {
			if preContainedPidFlag {
				appendProcess2ProcList(procList, fields)
			}
			continue
		}
		p, err = strconv.Atoi(fields[pidIndex])
		if err != nil {
			return nil, fmt.Errorf("Unexpected pid '%s': %s", fields[pidIndex], err)
		}

		if hasPid(procs, p) {
			preContainedPidFlag = true
			appendProcess2ProcList(procList, fields)
			continue
		}
		preContainedPidFlag = false
	}
	return procList, nil
}

// ContainerTop lists the processes running inside of the given
// container by calling ps with the given args, or with the flags
// "-ef" if no args are given.  An error is returned if the container
// is not found, or is not running, or if there are any problems
// running ps, or parsing the output.
func (daemon *Daemon) ContainerTop(name string, psArgs string) (*container.ContainerTopOKBody, error) {
	if psArgs == "" {
		psArgs = "-ef"
	}

	if err := validatePSArgs(psArgs); err != nil {
		return nil, err
	}

	container, err := daemon.GetContainer(name)
	if err != nil {
		return nil, err
	}

	if !container.IsRunning() {
		return nil, errNotRunning(container.ID)
	}

	if container.IsRestarting() {
		return nil, errContainerIsRestarting(container.ID)
	}

	procs, err := daemon.containerd.ListPids(context.Background(), container.ID)
	if err != nil {
		return nil, err
	}

	output, err := exec.Command("ps", strings.Split(psArgs, " ")...).Output()
	if err != nil {
		return nil, fmt.Errorf("Error running ps: %v", err)
	}
	procList, err := parsePSOutput(output, procs)
	if err != nil {
		return nil, err
	}
	daemon.LogContainerEvent(container, "top")
	return procList, nil
}
<commit_msg>daemon/parsePsOutput: minor optimisation<commit_after>//+build !windows

package daemon // import "github.com/docker/docker/daemon"

import (
	"context"
	"fmt"
	"os/exec"
	"regexp"
	"strconv"
	"strings"

	"github.com/docker/docker/api/types/container"
)

func validatePSArgs(psArgs string) error {
	// NOTE: \\s does not detect unicode whitespaces.
	// So we use fieldsASCII instead of strings.Fields in parsePSOutput.
	// See https://github.com/docker/docker/pull/24358
	// nolint: gosimple
	re := regexp.MustCompile("\\s+([^\\s]*)=\\s*(PID[^\\s]*)")
	for _, group := range re.FindAllStringSubmatch(psArgs, -1) {
		if len(group) >= 3 {
			k := group[1]
			v := group[2]
			if k != "pid" {
				return fmt.Errorf("specifying \"%s=%s\" is not allowed", k, v)
			}
		}
	}
	return nil
}

// fieldsASCII is similar to strings.Fields but only allows ASCII whitespaces
func fieldsASCII(s string) []string {
	fn := func(r rune) bool {
		switch r {
		case '\t', '\n', '\f', '\r', ' ':
			return true
		}
		return false
	}
	return strings.FieldsFunc(s, fn)
}

func appendProcess2ProcList(procList *container.ContainerTopOKBody, fields []string) {
	// Make sure number of fields equals number of header titles
	// merging "overhanging" fields
	process := fields[:len(procList.Titles)-1]
	process = append(process, strings.Join(fields[len(procList.Titles)-1:], " "))
	procList.Processes = append(procList.Processes, process)
}

func hasPid(procs []uint32, pid int) bool {
	for _, p := range procs {
		if int(p) == pid {
			return true
		}
	}
	return false
}

func parsePSOutput(output []byte, procs []uint32) (*container.ContainerTopOKBody, error) {
	procList := &container.ContainerTopOKBody{}

	lines := strings.Split(string(output), "\n")
	procList.Titles = fieldsASCII(lines[0])

	pidIndex := -1
	for i, name := range procList.Titles {
		if name == "PID" {
			pidIndex = i
			break
		}
	}
	if pidIndex == -1 {
		return nil, fmt.Errorf("Couldn't find PID field in ps output")
	}

	// loop through the output and extract the PID from each line
	// fixing #30580, be able to display thread line also when "m" option used
	// in "docker top" client command
	preContainedPidFlag := false
	for _, line := range lines[1:] {
		if len(line) == 0 {
			continue
		}
		fields := fieldsASCII(line)

		var (
			p   int
			err error
		)

		if fields[pidIndex] == "-" {
			if preContainedPidFlag {
				appendProcess2ProcList(procList, fields)
			}
			continue
		}
		p, err = strconv.Atoi(fields[pidIndex])
		if err != nil {
			return nil, fmt.Errorf("Unexpected pid '%s': %s", fields[pidIndex], err)
		}

		if hasPid(procs, p) {
			preContainedPidFlag = true
			appendProcess2ProcList(procList, fields)
			continue
		}
		preContainedPidFlag = false
	}
	return procList, nil
}

// ContainerTop lists the processes running inside of the given
// container by calling ps with the given args, or with the flags
// "-ef" if no args are given.  An error is returned if the container
// is not found, or is not running, or if there are any problems
// running ps, or parsing the output.
func (daemon *Daemon) ContainerTop(name string, psArgs string) (*container.ContainerTopOKBody, error) {
	if psArgs == "" {
		psArgs = "-ef"
	}

	if err := validatePSArgs(psArgs); err != nil {
		return nil, err
	}

	container, err := daemon.GetContainer(name)
	if err != nil {
		return nil, err
	}

	if !container.IsRunning() {
		return nil, errNotRunning(container.ID)
	}

	if container.IsRestarting() {
		return nil, errContainerIsRestarting(container.ID)
	}

	procs, err := daemon.containerd.ListPids(context.Background(), container.ID)
	if err != nil {
		return nil, err
	}

	output, err := exec.Command("ps", strings.Split(psArgs, " ")...).Output()
	if err != nil {
		return nil, fmt.Errorf("Error running ps: %v", err)
	}
	procList, err := parsePSOutput(output, procs)
	if err != nil {
		return nil, err
	}
	daemon.LogContainerEvent(container, "top")
	return procList, nil
}
<|endoftext|>


<commit_before>package main

import (
	"log"

	"github.com/Cox-Automotive/alks-go"
	"github.com/hashicorp/terraform/helper/schema"
)

func resourceAlksSession() *schema.Resource {
	return &schema.Resource{
		Create: resourceAlksSessionCreate,
		Read:   resourceAlksSessionRead,
		Exists: nil,
		Delete: resourceAlksSessionDelete,

		Schema: map[string]*schema.Schema{
			"name": &schema.Schema{
				Type:     schema.TypeString,
				Required: true,
				ForceNew: true,
			},
			"use_iam": &schema.Schema{
				Type:     schema.TypeBool,
				Required: true,
				ForceNew: true,
			},
			"access_key": &schema.Schema{
				Type:     schema.TypeString,
				Computed: true,
			},
			"secret_key": &schema.Schema{
				Type:     schema.TypeString,
				Computed: true,
			},
			"session_token": &schema.Schema{
				Type:     schema.TypeString,
				Computed: true,
			},
		},
	}
}

func resourceAlksSessionCreate(d *schema.ResourceData, meta interface{}) error {
	log.Printf("[INFO] ALKS Session Create")

	var useIam = d.Get("use_iam").(bool)
	var name = d.Get("name").(string)
	var duration int = 2
	if useIam {
		duration = 1
	}

	client := meta.(*alks.Client)
	resp, err := client.CreateSession(duration, useIam)

	if err != nil {
		return err
	}

	d.SetId(name)
	d.Set("access_key", resp.AccessKey)
	d.Set("secret_key", resp.SecretKey)
	d.Set("session_token", resp.SessionToken)

	log.Printf("[INFO] alks_session.id: %v", d.Id())

	return nil
}

func resourceAlksSessionRead(d *schema.ResourceData, meta interface{}) error {
	log.Printf("[INFO] ALKS Session Read")

	return nil
}

func resourceAlksSessionDelete(d *schema.ResourceData, meta interface{}) error {
	log.Printf("[INFO] ALKS Session Delete")

	return nil
}
<commit_msg>a lil more work..<commit_after>package main

import (
	"log"

	"github.com/Cox-Automotive/alks-go"
	"github.com/hashicorp/terraform/helper/schema"
)

func resourceAlksSession() *schema.Resource {
	return &schema.Resource{
		Create: resourceAlksSessionCreate,
		Read:   resourceAlksSessionRead,
		Update: resourceAlksSessionUpdate,
		Delete: resourceAlksSessionDelete,

		Schema: map[string]*schema.Schema{
			"use_iam": &schema.Schema{
				Type:     schema.TypeBool,
				Required: true,
			},
			"access_key": &schema.Schema{
				Type:     schema.TypeString,
				Computed: true,
			},
			"secret_key": &schema.Schema{
				Type:     schema.TypeString,
				Computed: true,
			},
			"session_token": &schema.Schema{
				Type:     schema.TypeString,
				Computed: true,
			},
		},
	}
}

// No-op: ALKS Session is currently read-only
func resourceAlksSessionCreate(d *schema.ResourceData, meta interface{}) error {
	return resourceAlksSessionRead(d, meta)
}

func resourceAlksSessionRead(d *schema.ResourceData, meta interface{}) error {
	log.Printf("[INFO] ALKS Session Read")

	var useIam = d.Get("use_iam").(bool)
	var name = d.Get("name").(string)
	var duration int = 2
	if useIam {
		duration = 1
	}

	client := meta.(*alks.Client)
	resp, err := client.CreateSession(duration, useIam)

	if err != nil {
		return err
	}

	d.SetId(name)
	d.Set("access_key", resp.AccessKey)
	d.Set("secret_key", resp.SecretKey)
	d.Set("session_token", resp.SessionToken)

	log.Printf("[INFO] alks_session.id: %v", d.Id())

	return nil
}

// No-op: ALKS Session is currently read-only
func resourceAlksSessionUpdate(d *schema.ResourceData, meta interface{}) error {
	return resourceAlksSessionRead(d, meta)
}

// No-op: ALKS Session is currently read-only
func resourceAlksSessionDelete(d *schema.ResourceData, meta interface{}) error {
	d.SetId("")
	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"os"
	"strconv"
	"strings"

	"github.com/mitchellh/colorstring"

	"github.com/yuuki1/gokc/log"
	"github.com/yuuki1/gokc/parser"
)

func setDebugOutputLevel() {
	for _, f := range os.Args {
		if f == "-D" || f == "--debug" || f == "-debug" {
			log.IsDebug = true
		}
	}

	debugEnv := os.Getenv("GOKC_DEBUG")
	if debugEnv != "" {
		showDebug, err := strconv.ParseBool(debugEnv)
		if err != nil {
			fmt.Fprintf(os.Stderr, "Error parsing boolean value from GRABENI_DEBUG: %s\n", err)
			os.Exit(1)
		}
		log.IsDebug = showDebug
	}
}

func init() {
	setDebugOutputLevel()
}

func main() {
	var filepath string
	flag.StringVar(&filepath, "f", "", "keepalived.conf file path")
	flag.Parse()

	if filepath == "" {
		log.Error("filepath required")
	}

	if _, err := os.Stat(filepath); err != nil {
		log.Error(err)
	}

	file, err := os.Open(filepath)
	if err != nil {
		log.Error(err)
	}

	p := parser.NewParser(file)
	if err := p.Parse(); err != nil {
		var msgs []string
		for _, e := range p.Errors() {
			msg := colorstring.Color(fmt.Sprintf("[white]%s:%d:%d: [red]%s[reset]", filepath, e.Line, e.Column, e.Message))
			msgs = append(msgs, msg)
		}
		log.Errorf("%s", strings.Join(msgs, "\n"))
	}

	log.Infof("gokc: the configuration file %s syntax is ok", filepath)
}
<commit_msg>Revert "Add file exist check"<commit_after>package main

import (
	"flag"
	"fmt"
	"os"
	"strconv"
	"strings"

	"github.com/mitchellh/colorstring"

	"github.com/yuuki1/gokc/log"
	"github.com/yuuki1/gokc/parser"
)

func setDebugOutputLevel() {
	for _, f := range os.Args {
		if f == "-D" || f == "--debug" || f == "-debug" {
			log.IsDebug = true
		}
	}

	debugEnv := os.Getenv("GOKC_DEBUG")
	if debugEnv != "" {
		showDebug, err := strconv.ParseBool(debugEnv)
		if err != nil {
			fmt.Fprintf(os.Stderr, "Error parsing boolean value from GRABENI_DEBUG: %s\n", err)
			os.Exit(1)
		}
		log.IsDebug = showDebug
	}
}

func init() {
	setDebugOutputLevel()
}

func main() {
	var filepath string
	flag.StringVar(&filepath, "f", "", "keepalived.conf file path")
	flag.Parse()

	if filepath == "" {
		log.Error("filepath required")
	}

	file, err := os.Open(filepath)
	if err != nil {
		log.Error(err)
	}

	p := parser.NewParser(file)
	if err := p.Parse(); err != nil {
		var msgs []string
		for _, e := range p.Errors() {
			msg := colorstring.Color(fmt.Sprintf("[white]%s:%d:%d: [red]%s[reset]", filepath, e.Line, e.Column, e.Message))
			msgs = append(msgs, msg)
		}
		log.Errorf("%s", strings.Join(msgs, "\n"))
	}

	log.Infof("gokc: the configuration file %s syntax is ok", filepath)
}
<|endoftext|>


<commit_before>package cmd

import (
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"

	"github.com/koding/kite/kitekey"
)

type List struct{}

func NewList() *List {
	return &List{}
}

func (*List) Definition() string {
	return "List installed kites"
}

func (*List) Exec(args []string) error {
	kites, err := getInstalledKites("")
	if err != nil {
		return err
	}

	for _, k := range kites {
		fmt.Println(k)
	}

	return nil
}

// getIntalledKites returns installed kites in .kd/kites folder.
// an empty argument returns all kites.
func getInstalledKites(kiteName string) ([]string, error) {
	kiteHome, err := kitekey.KiteHome()
	if err != nil {
		return nil, err
	}
	kitesPath := filepath.Join(kiteHome, "kites")

	kites, err := ioutil.ReadDir(kitesPath)
	if err != nil {
		if os.IsNotExist(err) {
			return []string{}, nil
		}

		return nil, err
	}

	installedKites := []string{} // to be returned
	for _, fi := range kites {
		name := fi.Name()
		if !strings.HasSuffix(name, ".kite") {
			continue
		}

		fullName := strings.TrimSuffix(name, ".kite")
		name, _, err := splitVersion(fullName, false)
		if err != nil {
			continue
		}

		if kiteName == "" || kiteName == name {
			installedKites = append(installedKites, fullName)
		}
	}

	return installedKites, nil
}
<commit_msg>update "kite list" command<commit_after>package cmd

import (
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"

	"github.com/koding/kite/kitekey"
)

type List struct{}

func NewList() *List {
	return &List{}
}

func (*List) Definition() string {
	return "List installed kites"
}

func (*List) Exec(args []string) error {
	kites, err := getInstalledKites("")
	if err != nil {
		return err
	}

	for _, k := range kites {
		fmt.Println(k)
	}

	return nil
}

// getIntalledKites returns installed kites in .kd/kites folder.
// an empty argument returns all kites.
func getInstalledKites(kiteName string) ([]string, error) {
	kiteHome, err := kitekey.KiteHome()
	if err != nil {
		return nil, err
	}
	kitesPath := filepath.Join(kiteHome, "kites")

	domains, err := ioutil.ReadDir(kitesPath)
	if err != nil {
		if os.IsNotExist(err) {
			return []string{}, nil
		}

		return nil, err
	}

	installedKites := []string{} // to be returned

	for _, domain := range domains {
		usernames, err := ioutil.ReadDir(filepath.Join(kitesPath, domain.Name()))
		if err != nil {
			fmt.Println(err)
			continue
		}

		for _, username := range usernames {
			repos, err := ioutil.ReadDir(filepath.Join(kitesPath, domain.Name(), username.Name()))
			if err != nil {
				fmt.Println(err)
				continue
			}

			for _, repo := range repos {
				versions, err := ioutil.ReadDir(filepath.Join(kitesPath, domain.Name(), username.Name(), repo.Name()))
				if err != nil {
					fmt.Println(err)
					continue
				}

				for _, version := range versions {
					_, err := os.Stat(filepath.Join(kitesPath, domain.Name(), username.Name(), repo.Name(), version.Name(), "bin", strings.TrimSuffix(repo.Name(), ".kite")))
					if err != nil {
						fmt.Println(err)
						continue
					}

					fullName := filepath.Join(domain.Name(), username.Name(), repo.Name(), version.Name())
					installedKites = append(installedKites, fullName)
				}
			}
		}
	}

	return installedKites, nil
}
<|endoftext|>


<commit_before>package cmd

import (
	"fmt"
	"sync"

	"github.com/skratchdot/open-golang/open"
	"github.com/spf13/cobra"

	"github.com/Shopify/themekit/kit"
)

var openCmd = &cobra.Command{
	Use:   "open",
	Short: "Open the preview for your store.",
	Long: `Open will open the preview page in your browser as well as print out
url for your reference`,
	RunE: forEachClient(preview),
}

func preview(client kit.ThemeClient, filenames []string, wg *sync.WaitGroup) {
	defer wg.Done()
	url := fmt.Sprintf("https://%s?preview_theme_id=%s",
		client.Config.Domain,
		client.Config.ThemeID)

	if openEdit {
		url = fmt.Sprintf("https://%s/admin/themes/%s",
			client.Config.Domain,
			client.Config.ThemeID)
	}

	openURL(client.Config.Environment, url)
}

func openURL(env, url string) {
	if verbose {
		kit.Printf("[%s] opening %s", kit.GreenText(env), kit.GreenText(url))
	}

	err := open.Run(url)
	if err != nil {
		kit.LogErrorf("[%s] %s", kit.GreenText(env), kit.RedText(err))
	}
}
<commit_msg>Changing the url to the editor<commit_after>package cmd

import (
	"fmt"
	"sync"

	"github.com/skratchdot/open-golang/open"
	"github.com/spf13/cobra"

	"github.com/Shopify/themekit/kit"
)

var openCmd = &cobra.Command{
	Use:   "open",
	Short: "Open the preview for your store.",
	Long: `Open will open the preview page in your browser as well as print out
url for your reference`,
	RunE: forEachClient(preview),
}

func preview(client kit.ThemeClient, filenames []string, wg *sync.WaitGroup) {
	defer wg.Done()
	url := fmt.Sprintf("https://%s?preview_theme_id=%s",
		client.Config.Domain,
		client.Config.ThemeID)

	if openEdit {
		url = fmt.Sprintf("https://%s/admin/themes/%s/editor",
			client.Config.Domain,
			client.Config.ThemeID)
	}

	openURL(client.Config.Environment, url)
}

func openURL(env, url string) {
	if verbose {
		kit.Printf("[%s] opening %s", kit.GreenText(env), kit.GreenText(url))
	}

	err := open.Run(url)
	if err != nil {
		kit.LogErrorf("[%s] %s", kit.GreenText(env), kit.RedText(err))
	}
}
<|endoftext|>


<commit_before>package cmd

import (
	"fmt"
	"os"
	"path/filepath"

	"github.com/spf13/cobra"
	"github.com/voidint/gbb/config"
	"github.com/voidint/gbb/tool"
	"github.com/voidint/gbb/util"
)

var confFile, wd string
var debug bool

// RootCmd represents the base command when called without any subcommands
var RootCmd = &cobra.Command{
	Use:   "gbb",
	Short: "",
	Long:  ``,

	Run: func(cmd *cobra.Command, args []string) {
		if !util.FileExist("./gbb.json") {
			genConfigFile(confFile)
			return
		}
		conf, err := config.Load(confFile)
		if err != nil {
			fmt.Fprintf(os.Stderr, err.Error())
			os.Exit(-1)
		}
		conf.Debug = debug

		if conf.Version != Version {
			fmt.Printf("The gbb.json file needs to be upgraded.\n\n")
			genConfigFile(confFile)
			return
		}

		if err := tool.Build(conf, wd); err != nil {
			fmt.Fprintf(os.Stderr, err.Error())
			os.Exit(-1)
		}
	},
}

// Execute adds all child commands to the root command sets flags appropriately.
// This is called by main.main(). It only needs to happen once to the rootCmd.
func Execute() {
	if err := RootCmd.Execute(); err != nil {
		fmt.Println(err)
		os.Exit(-1)
	}
}

func init() {
	cobra.OnInitialize(initConfig)
	RootCmd.PersistentFlags().BoolVar(&debug, "debug", false, "Enable debug mode")
}

// initConfig reads in config file and ENV variables if set.
func initConfig() {
	var err error
	wd, err = os.Getwd()
	if err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(-1)
	}
	confFile = filepath.Join(wd, "gbb.json")
}
<commit_msg>Add command line option `--config`<commit_after>package cmd

import (
	"fmt"
	"os"
	"path/filepath"

	"github.com/spf13/cobra"
	"github.com/voidint/gbb/config"
	"github.com/voidint/gbb/tool"
	"github.com/voidint/gbb/util"
)

const (
	// DefaultConfFile 默认配置文件路径（./gbb.json）
	DefaultConfFile = "gbb.json"
)

var (
	wd       string // 当前工作目录
	confFile string // 配置文件路径
	debug    bool   // 是否开启debug模式
)

// RootCmd represents the base command when called without any subcommands
var RootCmd = &cobra.Command{
	Use:   "gbb",
	Short: "Compile assistant",
	Long:  ``,

	Run: func(cmd *cobra.Command, args []string) {
		if confFile == DefaultConfFile {
			confFile = filepath.Join(wd, "gbb.json")
		}

		if !util.FileExist(confFile) {
			genConfigFile(confFile)
			return
		}
		conf, err := config.Load(confFile)
		if err != nil {
			fmt.Fprintf(os.Stderr, err.Error())
			os.Exit(-1)
		}
		conf.Debug = debug

		if conf.Version != Version {
			fmt.Printf("The gbb.json file needs to be upgraded.\n\n")
			genConfigFile(confFile)
			return
		}

		if err := tool.Build(conf, wd); err != nil {
			fmt.Fprintf(os.Stderr, err.Error())
			os.Exit(-1)
		}
	},
}

// Execute adds all child commands to the root command sets flags appropriately.
// This is called by main.main(). It only needs to happen once to the rootCmd.
func Execute() {
	if err := RootCmd.Execute(); err != nil {
		fmt.Println(err)
		os.Exit(-1)
	}
}

func init() {
	cobra.OnInitialize(initConfig)
	RootCmd.PersistentFlags().BoolVar(&debug, "debug", false, "Enable debug mode")
	RootCmd.PersistentFlags().StringVar(&confFile, "config", DefaultConfFile, "Configuration file")
}

// initConfig reads in config file and ENV variables if set.
func initConfig() {
	var err error
	wd, err = os.Getwd()
	if err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(-1)
	}
}
<|endoftext|>