<commit_before>// Copyright 2013 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package curio

import "io"

// Match provides a simple sequential ASCII text matcher.
// It is specialized for processing well formed, structured printable ASCII.
// s yields input bytes and p yields pattern bytes.
// n and o correspond to the number of bytes read from s and p, respectively.
// If m is non-nil, len(m) should be greater than or equal to the number of
// capture-bytes in the pattern.
//
// Patterns:
//  * Printable ASCII bytes will be matched literally.
//  * All groups (specified by non-printable bytes in the pattern stream)
//    are non-greedy, and match zero or more characters.
//  * \x00: ASCII whitespace bytes.
//  * \x01: non-whitespace ASCII bytes.
//  * \xd0: decimal digits.
//  * \xd6: hexadecimal digits.
//  * \xd8: octal digits.
//  * \xd3: base-36 digits.
//  * \xfe: printable ASCII bytes.
//  * \xff: 8-bit bytes.
//  * All groups are capturing beside \x00.
//  * Use of other non-printable or non-ASCII bytes is undefined.
func Match(s, p io.ByteScanner, m []string) (n, o int, err error) {
	var (
		quit = false
		buf  = make([]byte, 0, 1024)
		a, b byte
		c    byte
		v    int
	)
	for !quit {
		a, err = p.ReadByte()
		switch {
		case err == io.EOF:
			err = nil
			fallthrough
		case err != nil:
			return
		}
		o++
		if tab[a]&prg == 0 {
			c, err = s.ReadByte()
			if err != nil {
				return
			}
			n++
			if a == c {
				continue
			} else if err = s.UnreadByte(); err == nil {
				err = ErrByteMismatch
				n--
			}
			return
		}
		a = tab[a]
		b, err = p.ReadByte()
		if err == io.EOF {
			b = nop
		} else if err != nil {
			return
		} else if err = p.UnreadByte(); err != nil {
			o++
			return
		}
		for {
			c, err = s.ReadByte()
			if err != nil {
				quit = true
				goto fill
			}
			n++
			if tab[b]&prg == 0 && c == b ||
				tab[b]&prg != 0 && tab[c]&tab[b] != 0 ||
				a != any && tab[c]&a == 0 {
				break
			} else if a&^prg != ws {
				buf = append(buf, c)
			}
		}
		if err = s.UnreadByte(); err == nil {
			err = io.EOF
			n--
		}
	fill:
		if a&^prg != ws {
			if _, ok := s.(*rb); ok {
				revbytes(buf)
			}
			m[v] = string(buf)
			v++
			buf = buf[:0]
		}
	}
	return
}

func revbytes(p []byte) {
	j := len(p)
	m := j / 2
	for i := 0; i < m; i++ {
		j--
		p[i], p[j] = p[j], p[i]
	}
}

const (
	prg = 1 << iota
	ws
	pc
	ll
	lu
	do
	dd
	dx
	lx  = ll | lu
	wrd = lx | dd
	nws = wrd | pc
	prn = nws | ws
	any = 0xff
	nop = 0x80
)

var tab = [1 << 8]byte{
	' ': ws, '\n': ws, '\t': ws, '\r': ws, '\f': ws, '\v': ws,
	'0': do | dd | dx, '1': do | dd | dx, '2': do | dd | dx, '3': do | dd | dx,
	'4': do | dd | dx, '5': do | dd | dx, '6': do | dd | dx, '7': do | dd | dx,
	'8': dd | dx, '9': dd | dx,

	'a': ll | dx, 'b': ll | dx, 'c': ll | dx,
	'd': ll | dx, 'e': ll | dx, 'f': ll | dx,
	'g': ll, 'h': ll, 'i': ll, 'j': ll, 'k': ll,
	'l': ll, 'm': ll, 'n': ll, 'o': ll, 'p': ll,
	'q': ll, 'r': ll, 's': ll, 't': ll, 'u': ll,
	'v': ll, 'w': ll, 'x': ll, 'y': ll, 'z': ll,

	'A': lu | dx, 'B': lu | dx, 'C': lu | dx,
	'D': lu | dx, 'E': lu | dx, 'F': lu | dx,
	'G': lu, 'H': lu, 'I': lu, 'J': lu, 'K': lu,
	'L': lu, 'M': lu, 'N': lu, 'O': lu, 'P': lu,
	'Q': lu, 'R': lu, 'S': lu, 'T': lu, 'U': lu,
	'V': lu, 'W': lu, 'X': lu, 'Y': lu, 'Z': lu,

	'!': pc, '"': pc, '#': pc, '$': pc, '%': pc, '&': pc, '\'': pc, '(': pc,
	')': pc, '*': pc, '+': pc, ',': pc, '-': pc, '.': pc, '/': pc, ':': pc,
	';': pc, '<': pc, '=': pc, '>': pc, '?': pc, '@': pc, '[': pc, '\\': pc,
	']': pc, '^': pc, '_': pc, '`': pc, '{': pc, '|': pc, '}': pc, '~': pc,

	'\x00': prg | ws,
	'\x01': prg | nws,
	'\xd8': prg | do,
	'\xd0': prg | dd,
	'\xd6': prg | dx,
	'\xd3': prg | dd | lx,
	'\xfe': prg | prn,
	'\xff': any,
}
<commit_msg>Added letter groups.<commit_after>// Copyright 2013 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package curio

import "io"

// Match provides a simple sequential ASCII text matcher.
// It is specialized for processing well formed, structured printable ASCII.
// s yields input bytes and p yields pattern bytes.
// n and o correspond to the number of bytes read from s and p, respectively.
// If m is non-nil, len(m) should be greater than or equal to the number of
// capture-bytes in the pattern.
//
// Patterns:
//  * Printable ASCII bytes will be matched literally.
//  * All groups (specified by non-printable bytes in the pattern stream)
//    are non-greedy, and match zero or more characters.
//  * \x00: ASCII whitespace bytes.
//  * \x01: non-whitespace ASCII bytes.
//  * \xa0: lowercase letters.
//  * \xa1: uppercase letters.
//  * \xaf: letters.
//  * \xd0: decimal digits.
//  * \xd6: hexadecimal digits.
//  * \xd8: octal digits.
//  * \xd3: base-36 digits.
//  * \xfe: printable ASCII bytes.
//  * \xff: 8-bit bytes.
//  * All groups are capturing beside \x00.
//  * Use of other non-printable or non-ASCII bytes is undefined.
func Match(s, p io.ByteScanner, m []string) (n, o int, err error) {
	var (
		quit = false
		buf  = make([]byte, 0, 1024)
		a, b byte
		c    byte
		v    int
	)
	for !quit {
		a, err = p.ReadByte()
		switch {
		case err == io.EOF:
			err = nil
			fallthrough
		case err != nil:
			return
		}
		o++
		if tab[a]&prg == 0 {
			c, err = s.ReadByte()
			if err != nil {
				return
			}
			n++
			if a == c {
				continue
			} else if err = s.UnreadByte(); err == nil {
				err = ErrByteMismatch
				n--
			}
			return
		}
		a = tab[a]
		b, err = p.ReadByte()
		if err == io.EOF {
			b = nop
		} else if err != nil {
			return
		} else if err = p.UnreadByte(); err != nil {
			o++
			return
		}
		for {
			c, err = s.ReadByte()
			if err != nil {
				quit = true
				goto fill
			}
			n++
			if tab[b]&prg == 0 && c == b ||
				tab[b]&prg != 0 && tab[c]&tab[b] != 0 ||
				a != any && tab[c]&a == 0 {
				break
			} else if a&^prg != ws {
				buf = append(buf, c)
			}
		}
		if err = s.UnreadByte(); err == nil {
			err = io.EOF
			n--
		}
	fill:
		if a&^prg != ws {
			if _, ok := s.(*rb); ok {
				revbytes(buf)
			}
			m[v] = string(buf)
			v++
			buf = buf[:0]
		}
	}
	return
}

func revbytes(p []byte) {
	j := len(p)
	m := j / 2
	for i := 0; i < m; i++ {
		j--
		p[i], p[j] = p[j], p[i]
	}
}

const (
	prg = 1 << iota
	ws
	pc
	ll
	lu
	do
	dd
	dx
	lx  = ll | lu
	wrd = lx | dd
	nws = wrd | pc
	prn = nws | ws
	any = 0xff
	nop = 0x80
)

var tab = [1 << 8]byte{
	' ': ws, '\n': ws, '\t': ws, '\r': ws, '\f': ws, '\v': ws,
	'0': do | dd | dx, '1': do | dd | dx, '2': do | dd | dx, '3': do | dd | dx,
	'4': do | dd | dx, '5': do | dd | dx, '6': do | dd | dx, '7': do | dd | dx,
	'8': dd | dx, '9': dd | dx,

	'a': ll | dx, 'b': ll | dx, 'c': ll | dx,
	'd': ll | dx, 'e': ll | dx, 'f': ll | dx,
	'g': ll, 'h': ll, 'i': ll, 'j': ll, 'k': ll,
	'l': ll, 'm': ll, 'n': ll, 'o': ll, 'p': ll,
	'q': ll, 'r': ll, 's': ll, 't': ll, 'u': ll,
	'v': ll, 'w': ll, 'x': ll, 'y': ll, 'z': ll,

	'A': lu | dx, 'B': lu | dx, 'C': lu | dx,
	'D': lu | dx, 'E': lu | dx, 'F': lu | dx,
	'G': lu, 'H': lu, 'I': lu, 'J': lu, 'K': lu,
	'L': lu, 'M': lu, 'N': lu, 'O': lu, 'P': lu,
	'Q': lu, 'R': lu, 'S': lu, 'T': lu, 'U': lu,
	'V': lu, 'W': lu, 'X': lu, 'Y': lu, 'Z': lu,

	'!': pc, '"': pc, '#': pc, '$': pc, '%': pc, '&': pc, '\'': pc, '(': pc,
	')': pc, '*': pc, '+': pc, ',': pc, '-': pc, '.': pc, '/': pc, ':': pc,
	';': pc, '<': pc, '=': pc, '>': pc, '?': pc, '@': pc, '[': pc, '\\': pc,
	']': pc, '^': pc, '_': pc, '`': pc, '{': pc, '|': pc, '}': pc, '~': pc,

	'\x00': prg | ws,
	'\x01': prg | nws,
	'\xa0': prg | ll,
	'\xa1': prg | lu,
	'\xaf': prg | lx,
	'\xd8': prg | do,
	'\xd0': prg | dd,
	'\xd6': prg | dx,
	'\xd3': prg | dd | lx,
	'\xfe': prg | prn,
	'\xff': any,
}
<|endoftext|>


<commit_before>package filters

import d "github.com/appPlant/alpinepass/src/data"

// PasswordFilter removes passwords from a Config.
type PasswordFilter struct {
	strings []string
}

func (p PasswordFilter) filter(data d.Config) d.Config {
	data.Password = ""
	return data
}
<commit_msg>Remove package alias<commit_after>package filters

import "github.com/appPlant/alpinepass/src/data"

// PasswordFilter removes passwords from a Config.
type PasswordFilter struct {
	strings []string
}

func (p PasswordFilter) filter(config data.Config) data.Config {
	config.Password = ""
	return config
}
<|endoftext|>


<commit_before>package fetcher

import (
	"bytes"
	"context"
	"crypto/rand"
	"errors"
	"io"
	"sync"
	"testing"

	"github.com/stretchr/testify/require"
	"github.com/taskcluster/taskcluster-worker/runtime/client"
)

type fakeContext struct {
	context.Context
	queue           client.Queue
	m               sync.Mutex
	progressReports []float64
}

func (c *fakeContext) Queue() client.Queue {
	return c.queue
}

func (c *fakeContext) Progress(description string, percent float64) {
	c.m.Lock()
	defer c.m.Unlock()
	debug("Progress: %s - %.02f %%", description, percent*100)
	c.progressReports = append(c.progressReports, percent)
}

func (c *fakeContext) ProgressReports() []float64 {
	c.m.Lock()
	defer c.m.Unlock()
	return c.progressReports
}

type fakeWriteReseter struct {
	offset int64
	buffer []byte
}

func (w *fakeWriteReseter) Write(p []byte) (int, error) {
	offset := w.offset + int64(len(p))
	if int64(len(w.buffer)) < offset {
		w.buffer = append(w.buffer, make([]byte, offset-int64(len(w.buffer)))...)
	}
	copy(w.buffer[w.offset:], p)
	w.offset = offset
	return len(p), nil
}

func (w *fakeWriteReseter) Reset() error {
	w.offset = 0
	w.buffer = nil
	return nil
}

func (w *fakeWriteReseter) String() string {
	return string(w.buffer)
}

func TestFakeWriteSeekReseter(t *testing.T) {
	w := &fakeWriteReseter{}
	_, err := io.Copy(w, bytes.NewBufferString("test"))
	require.NoError(t, err)
	require.Equal(t, w.String(), "test")

	err = w.Reset()
	require.NoError(t, err)
	_, err = io.Copy(w, bytes.NewBufferString("test again"))
	require.NoError(t, err)
	require.Equal(t, w.String(), "test again")

	_, err = io.Copy(w, bytes.NewBufferString(" test again"))
	require.NoError(t, err)
	require.Equal(t, w.String(), "test again test again")
}

type fakeReference struct {
	ScopesValue  [][]string
	HashKeyValue string
	Data         []byte
	Reset        bool
	Err          error
}

func (r *fakeReference) HashKey() string {
	return r.HashKeyValue
}

func (r *fakeReference) Scopes() [][]string {
	return r.ScopesValue
}

func (r *fakeReference) Fetch(context Context, target WriteReseter) error {
	i := 0
	// Write a few small blocks
	if i+1 < len(r.Data) {
		n, err := target.Write(r.Data[i : i+1])
		if err != nil {
			return err
		}
		if n != 1 {
			panic(errors.New("Expected n = 1 when no error was returned"))
		}
		i += n
	}
	if i+7 < len(r.Data) {
		n, err := target.Write(r.Data[i : i+7])
		if err != nil {
			return err
		}
		if n != 7 {
			panic(errors.New("Expected n = 7 when no error was returned"))
		}
		i += n
	}
	if i+3 < len(r.Data) {
		n, err := target.Write(r.Data[i : i+3])
		if err != nil {
			return err
		}
		if n != 3 {
			panic(errors.New("Expected n = 3 when no error was returned"))
		}
		i += n
	}
	if i < len(r.Data) {
		n, err := target.Write(r.Data[i:])
		if err != nil {
			return err
		}
		if n != len(r.Data)-i {
			panic(errors.New("Expected n = len(r.Data)-i when no error was returned"))
		}
	}
	if r.Reset {
		r.Reset = false
		if err := target.Reset(); err != nil {
			return nil
		}
		return r.Fetch(context, target)
	}
	return r.Err
}

func TestFakeReference(t *testing.T) {
	ctx := &fakeContext{Context: context.Background()}
	// Create a random blob
	blob := make([]byte, 16*1024+27)
	_, rerr := rand.Read(blob)
	require.NoError(t, rerr, "failed to created random data for testing")

	t.Run("Empty String without Reset", func(t *testing.T) {
		r := &fakeReference{
			Data:  []byte(""),
			Reset: false,
			Err:   nil,
		}
		w := &fakeWriteReseter{}
		err := r.Fetch(ctx, w)
		require.NoError(t, err)
		require.Equal(t, "", w.String())
	})

	t.Run("hello-world without Reset", func(t *testing.T) {
		r := &fakeReference{
			Data:  []byte("hello-world"),
			Reset: false,
			Err:   nil,
		}
		w := &fakeWriteReseter{}
		err := r.Fetch(ctx, w)
		require.NoError(t, err)
		require.Equal(t, "hello-world", w.String())
	})

	t.Run("blob without Reset", func(t *testing.T) {
		r := &fakeReference{
			Data:  blob,
			Reset: false,
			Err:   nil,
		}
		w := &fakeWriteReseter{}
		err := r.Fetch(ctx, w)
		require.NoError(t, err)
		require.True(t, bytes.Equal(blob, w.buffer), "expected blob == w.buffer")
	})

	t.Run("Empty String with Reset", func(t *testing.T) {
		r := &fakeReference{
			Data:  []byte(""),
			Reset: true,
			Err:   nil,
		}
		w := &fakeWriteReseter{}
		err := r.Fetch(ctx, w)
		require.NoError(t, err)
		require.Equal(t, "", w.String())
	})

	t.Run("hello-world with Reset", func(t *testing.T) {
		r := &fakeReference{
			Data:  []byte("hello-world"),
			Reset: true,
			Err:   nil,
		}
		w := &fakeWriteReseter{}
		err := r.Fetch(ctx, w)
		require.NoError(t, err)
		require.Equal(t, "hello-world", w.String())
	})

	t.Run("blob with Reset", func(t *testing.T) {
		r := &fakeReference{
			Data:  blob,
			Reset: true,
			Err:   nil,
		}
		w := &fakeWriteReseter{}
		err := r.Fetch(ctx, w)
		require.NoError(t, err)
		require.True(t, bytes.Equal(blob, w.buffer), "expected blob == w.buffer")
	})
}
<commit_msg>Removed unneeded offset from fakeWriteReseter<commit_after>package fetcher

import (
	"bytes"
	"context"
	"crypto/rand"
	"errors"
	"io"
	"sync"
	"testing"

	"github.com/stretchr/testify/require"
	"github.com/taskcluster/taskcluster-worker/runtime/client"
)

type fakeContext struct {
	context.Context
	queue           client.Queue
	m               sync.Mutex
	progressReports []float64
}

func (c *fakeContext) Queue() client.Queue {
	return c.queue
}

func (c *fakeContext) Progress(description string, percent float64) {
	c.m.Lock()
	defer c.m.Unlock()
	debug("Progress: %s - %.02f %%", description, percent*100)
	c.progressReports = append(c.progressReports, percent)
}

func (c *fakeContext) ProgressReports() []float64 {
	c.m.Lock()
	defer c.m.Unlock()
	return c.progressReports
}

type fakeWriteReseter struct {
	buffer []byte
}

func (w *fakeWriteReseter) Write(p []byte) (int, error) {
	w.buffer = append(w.buffer, p...)
	return len(p), nil
}

func (w *fakeWriteReseter) Reset() error {
	w.buffer = nil
	return nil
}

func (w *fakeWriteReseter) String() string {
	return string(w.buffer)
}

func TestFakeWriteSeekReseter(t *testing.T) {
	w := &fakeWriteReseter{}
	_, err := io.Copy(w, bytes.NewBufferString("test"))
	require.NoError(t, err)
	require.Equal(t, w.String(), "test")

	err = w.Reset()
	require.NoError(t, err)
	_, err = io.Copy(w, bytes.NewBufferString("test again"))
	require.NoError(t, err)
	require.Equal(t, w.String(), "test again")

	_, err = io.Copy(w, bytes.NewBufferString(" test again"))
	require.NoError(t, err)
	require.Equal(t, w.String(), "test again test again")
}

type fakeReference struct {
	ScopesValue  [][]string
	HashKeyValue string
	Data         []byte
	Reset        bool
	Err          error
}

func (r *fakeReference) HashKey() string {
	return r.HashKeyValue
}

func (r *fakeReference) Scopes() [][]string {
	return r.ScopesValue
}

func (r *fakeReference) Fetch(context Context, target WriteReseter) error {
	i := 0
	// Write a few small blocks
	if i+1 < len(r.Data) {
		n, err := target.Write(r.Data[i : i+1])
		if err != nil {
			return err
		}
		if n != 1 {
			panic(errors.New("Expected n = 1 when no error was returned"))
		}
		i += n
	}
	if i+7 < len(r.Data) {
		n, err := target.Write(r.Data[i : i+7])
		if err != nil {
			return err
		}
		if n != 7 {
			panic(errors.New("Expected n = 7 when no error was returned"))
		}
		i += n
	}
	if i+3 < len(r.Data) {
		n, err := target.Write(r.Data[i : i+3])
		if err != nil {
			return err
		}
		if n != 3 {
			panic(errors.New("Expected n = 3 when no error was returned"))
		}
		i += n
	}
	if i < len(r.Data) {
		n, err := target.Write(r.Data[i:])
		if err != nil {
			return err
		}
		if n != len(r.Data)-i {
			panic(errors.New("Expected n = len(r.Data)-i when no error was returned"))
		}
	}
	if r.Reset {
		r.Reset = false
		if err := target.Reset(); err != nil {
			return nil
		}
		return r.Fetch(context, target)
	}
	return r.Err
}

func TestFakeReference(t *testing.T) {
	ctx := &fakeContext{Context: context.Background()}
	// Create a random blob
	blob := make([]byte, 16*1024+27)
	_, rerr := rand.Read(blob)
	require.NoError(t, rerr, "failed to created random data for testing")

	t.Run("Empty String without Reset", func(t *testing.T) {
		r := &fakeReference{
			Data:  []byte(""),
			Reset: false,
			Err:   nil,
		}
		w := &fakeWriteReseter{}
		err := r.Fetch(ctx, w)
		require.NoError(t, err)
		require.Equal(t, "", w.String())
	})

	t.Run("hello-world without Reset", func(t *testing.T) {
		r := &fakeReference{
			Data:  []byte("hello-world"),
			Reset: false,
			Err:   nil,
		}
		w := &fakeWriteReseter{}
		err := r.Fetch(ctx, w)
		require.NoError(t, err)
		require.Equal(t, "hello-world", w.String())
	})

	t.Run("blob without Reset", func(t *testing.T) {
		r := &fakeReference{
			Data:  blob,
			Reset: false,
			Err:   nil,
		}
		w := &fakeWriteReseter{}
		err := r.Fetch(ctx, w)
		require.NoError(t, err)
		require.True(t, bytes.Equal(blob, w.buffer), "expected blob == w.buffer")
	})

	t.Run("Empty String with Reset", func(t *testing.T) {
		r := &fakeReference{
			Data:  []byte(""),
			Reset: true,
			Err:   nil,
		}
		w := &fakeWriteReseter{}
		err := r.Fetch(ctx, w)
		require.NoError(t, err)
		require.Equal(t, "", w.String())
	})

	t.Run("hello-world with Reset", func(t *testing.T) {
		r := &fakeReference{
			Data:  []byte("hello-world"),
			Reset: true,
			Err:   nil,
		}
		w := &fakeWriteReseter{}
		err := r.Fetch(ctx, w)
		require.NoError(t, err)
		require.Equal(t, "hello-world", w.String())
	})

	t.Run("blob with Reset", func(t *testing.T) {
		r := &fakeReference{
			Data:  blob,
			Reset: true,
			Err:   nil,
		}
		w := &fakeWriteReseter{}
		err := r.Fetch(ctx, w)
		require.NoError(t, err)
		require.True(t, bytes.Equal(blob, w.buffer), "expected blob == w.buffer")
	})
}
<|endoftext|>


<commit_before>package cwl

import (
	"fmt"
	"sort"
	"strings"

	"github.com/robertkrimen/otto"
)

// Input represents "CommandInputParameter".
// @see http://www.commonwl.org/v1.0/CommandLineTool.html#CommandInputParameter
type Input struct {
	ID             string          `json:"id"`
	Label          string          `json:"label"`
	Doc            string          `json:"doc"`
	Format         string          `json:"format"`
	Binding        *Binding        `json:"inputBinding"`
	Default        *InputDefault   `json:"default"`
	Types          []Type          `json:"type"`
	SecondaryFiles []SecondaryFile `json:"secondary_files"`
	// Input.Provided is what provided by parameters.(json|yaml)
	Provided *Provided `json:"-"`
	// Requirement ..
	RequiredType *Type
	Requirements Requirements
}

// New constructs "Input" struct from interface{}.
func (input Input) New(i interface{}) *Input {
	dest := &Input{}
	switch x := i.(type) {
	case map[string]interface{}:
		for key, v := range x {
			switch key {
			case "id":
				dest.ID = v.(string)
			case "type":
				dest.Types = Type{}.NewList(v)
			case "label":
				dest.Label = v.(string)
			case "doc":
				dest.Doc = v.(string)
			case "inputBinding":
				dest.Binding = Binding{}.New(v)
			case "default":
				dest.Default = InputDefault{}.New(v)
			case "format":
				dest.Format = v.(string)
			case "secondaryFiles":
				dest.SecondaryFiles = SecondaryFile{}.NewList(v)
			}
		}
		if dest.Default != nil {
			dest.Default.ID = dest.ID
		}
	case string:
		dest.Types = Type{}.NewList(x)
	case []interface{}:
		for _, v := range x {
			dest.Types = append(dest.Types, Type{}.New(v))
		}
	}
	return dest
}

// flatten
func (input *Input) flatten(typ Type, binding *Binding, prov interface{}) []string {
	flattened := []string{}
	switch typ.Type {
	case "int": // Array of Int
		tobejoined := []string{}
		for _, e := range input.Provided.Raw.([]interface{}) {
			tobejoined = append(tobejoined, fmt.Sprintf("%v", e))
		}
		flattened = append(flattened, strings.Join(tobejoined, input.Binding.Separator))
	case "File": // Array of Files
		switch arr := input.Provided.Raw.(type) {
		case []string:
			// TODO:
		case []interface{}:
			separated := []string{}
			for _, e := range arr {
				switch v := e.(type) {
				case map[interface{}]interface{}:
					if binding != nil && binding.Prefix != "" {
						separated = append(separated, binding.Prefix)
					}
					separated = append(separated, fmt.Sprintf("%v", v["location"]))
				default:
					// TODO:
				}
			}
			// In case it's Array of Files, unlike array of int,
			// it's NOT gonna be joined with .Binding.Separator.
			flattened = append(flattened, separated...)
		}
	case "string": // Array of string
		switch arr := prov.(type) {
		case []interface{}:
			separated := []string{}
			for _, e := range arr {
				switch v := e.(type) {
				case string:
					if binding != nil && binding.Prefix != "" {
						separated = append(separated, binding.Prefix)
					}
					separated = append(separated, fmt.Sprintf("%v", v))
				default:
					// TODO
				}
			}
			flattened = append(flattened, separated...)
		default:
			// TODO
		}
	case "array":
		switch arr := prov.(type) {
		case []interface{}:
			flattened = append(flattened, input.flatten(typ.Items[0], typ.Binding, arr[0])...)
		default:
			// TODO
		}
	default:
		if input.RequiredType != nil {
			flattened = append(flattened, input.flattenWithRequiredType()...)
		} else {
			// TODO
		}
	}
	return flattened
}

func (input *Input) flattenWithRequiredType() []string {
	flattened := []string{}
	key, needed := input.Types[0].NeedRequirement()
	if !needed {
		return flattened
	}
	if input.RequiredType.Name != key {
		return flattened
	}
	switch provided := input.Provided.Raw.(type) {
	case []interface{}:
		for _, e := range provided {
			switch v := e.(type) {
			case map[interface{}]interface{}:
				for _, field := range input.RequiredType.Fields {
					val, ok := v[field.Name]
					if !ok {
						continue
					}
					if field.Binding == nil {
						// Without thinking anything, just append it!!!
						flattened = append(flattened, fmt.Sprintf("%v", val))
					} else {
						if field.Binding.Prefix != "" {
							if field.Binding.Separate {
								flattened = append(flattened, field.Binding.Prefix, fmt.Sprintf("%v", val))
							} else {
								// TODO: Join if .Separator is given
								flattened = append(flattened, fmt.Sprintf("%s%v", field.Binding.Prefix, val))
							}
						} else {
							switch v2 := val.(type) {
							case []interface{}:
								for _, val2 := range v2 {
									switch v3 := val2.(type) {
									case []interface{}:
									case map[interface{}]interface{}:
										for _, types := range input.Requirements[0].SchemaDefRequirement.Types {
											val3array := []string{}
											var val3count int = 0
											sort.Sort(types.Fields)
											for _, fields := range types.Fields {
												for key3, val3 := range v3 {
													if fields.Name == key3 {
														for _, val3type := range fields.Types {
															if val3type.Type == "" {
															} else {
																switch val3type.Type {
																case "enum":
																	for _, symbol := range val3type.Symbols {
																		if symbol == val3 {
																			val3array = append(val3array, fmt.Sprintf("%v", val3))
																			val3count = val3count + 1
																		}
																	}
																case "int":
																	if fields.Binding.Prefix != "" {
																		val3array = append(val3array, fields.Binding.Prefix, fmt.Sprintf("%v", val3))
																		val3count = val3count + 1
																	} else {
																		val3array = append(val3array, fmt.Sprintf("%v", val3))
																		val3count = val3count + 1
																	}
																}
															}
														}
													}
												}
											}
											if len(v3) == val3count {
												flattened = append(flattened, val3array...)
											}
										}
									}
								}
							}
						}
					}
				}
			}
		}
	}
	return flattened
}

// Flatten ...
func (input *Input) Flatten() []string {
	if input.Provided == nil {
		// In case "input.Default == nil" should be validated by usage layer.
		if input.Default != nil {
			return input.Default.Flatten(input.Binding)
		} else {
			return []string{}
		}
	}
	flattened := []string{}
	if repr := input.Types[0]; len(input.Types) == 1 {
		switch repr.Type {
		case "array":
			flattened = append(flattened, input.flatten(repr.Items[0], repr.Binding, input.Provided.Raw)...)
		case "int":
			flattened = append(flattened, fmt.Sprintf("%v", input.Provided.Int))
		case "File":
			flattened = append(flattened, input.Provided.Entry.Location)
		default:
			flattened = append(flattened, fmt.Sprintf("%v", input.Provided))
		}
	}
	if input.Binding != nil && input.Binding.Prefix != "" {
		flattened = append([]string{input.Binding.Prefix}, flattened...)
	}

	return flattened
}

// Inputs represents "inputs" field in CWL.
type Inputs []*Input

// New constructs new "Inputs" struct.
func (ins Inputs) New(i interface{}) Inputs {
	dest := Inputs{}
	switch x := i.(type) {
	case []interface{}:
		for _, v := range x {
			dest = append(dest, Input{}.New(v))
		}
	case map[string]interface{}:
		for key, v := range x {
			input := Input{}.New(v)
			input.ID = key
			dest = append(dest, input)
		}
	}
	return dest
}

// Len for sorting.
func (ins Inputs) Len() int {
	return len(ins)
}

// Less for sorting.
func (ins Inputs) Less(i, j int) bool {
	prev, next := ins[i].Binding, ins[j].Binding
	switch [2]bool{prev == nil, next == nil} {
	case [2]bool{true, true}:
		return true
	case [2]bool{false, true}:
		return prev.Position < 0
	case [2]bool{true, false}:
		return next.Position > 0
	default:
		return prev.Position <= next.Position
	}
}

// Swap for sorting.
func (ins Inputs) Swap(i, j int) {
	ins[i], ins[j] = ins[j], ins[i]
}

// ToJavaScriptVM ...
func (ins Inputs) ToJavaScriptVM() (*otto.Otto, error) {
	self := map[string]interface{}{}
	for _, i := range ins {
		if i.Provided != nil {
			if i.Provided.Entry != nil {
				self[i.ID] = map[string]interface{}{
					"path": i.Provided.Entry.Location,
				}
				continue
			}
			if i.Types[0].Type == "string" {
				self[i.ID] = i.Provided.Raw
				continue
			}
		}
		if i.Default != nil && i.Default.Entry != nil {
			self[i.ID] = map[string]interface{}{
				"path": i.Default.Entry.Location,
			}
			continue
		}
	}

	// No contents to load
	if len(self) == 0 {
		return nil, nil
	}

	vm := otto.New()
	if err := vm.Set("inputs", self); err != nil {
		return nil, err
	}
	return vm, nil
}
<commit_msg>Support input type Any.<commit_after>package cwl

import (
	"fmt"
	"sort"
	"strings"

	"github.com/robertkrimen/otto"
)

// Input represents "CommandInputParameter".
// @see http://www.commonwl.org/v1.0/CommandLineTool.html#CommandInputParameter
type Input struct {
	ID             string          `json:"id"`
	Label          string          `json:"label"`
	Doc            string          `json:"doc"`
	Format         string          `json:"format"`
	Binding        *Binding        `json:"inputBinding"`
	Default        *InputDefault   `json:"default"`
	Types          []Type          `json:"type"`
	SecondaryFiles []SecondaryFile `json:"secondary_files"`
	// Input.Provided is what provided by parameters.(json|yaml)
	Provided *Provided `json:"-"`
	// Requirement ..
	RequiredType *Type
	Requirements Requirements
}

// New constructs "Input" struct from interface{}.
func (input Input) New(i interface{}) *Input {
	dest := &Input{}
	switch x := i.(type) {
	case map[string]interface{}:
		for key, v := range x {
			switch key {
			case "id":
				dest.ID = v.(string)
			case "type":
				dest.Types = Type{}.NewList(v)
			case "label":
				dest.Label = v.(string)
			case "doc":
				dest.Doc = v.(string)
			case "inputBinding":
				dest.Binding = Binding{}.New(v)
			case "default":
				dest.Default = InputDefault{}.New(v)
			case "format":
				dest.Format = v.(string)
			case "secondaryFiles":
				dest.SecondaryFiles = SecondaryFile{}.NewList(v)
			}
		}
		if dest.Default != nil {
			dest.Default.ID = dest.ID
		}
	case string:
		dest.Types = Type{}.NewList(x)
	case []interface{}:
		for _, v := range x {
			dest.Types = append(dest.Types, Type{}.New(v))
		}
	}
	return dest
}

// flatten
func (input *Input) flatten(typ Type, binding *Binding, prov interface{}) []string {
	flattened := []string{}
	switch typ.Type {
	case "int": // Array of Int
		tobejoined := []string{}
		for _, e := range input.Provided.Raw.([]interface{}) {
			tobejoined = append(tobejoined, fmt.Sprintf("%v", e))
		}
		flattened = append(flattened, strings.Join(tobejoined, input.Binding.Separator))
	case "File": // Array of Files
		switch arr := input.Provided.Raw.(type) {
		case []string:
			// TODO:
		case []interface{}:
			separated := []string{}
			for _, e := range arr {
				switch v := e.(type) {
				case map[interface{}]interface{}:
					if binding != nil && binding.Prefix != "" {
						separated = append(separated, binding.Prefix)
					}
					separated = append(separated, fmt.Sprintf("%v", v["location"]))
				default:
					// TODO:
				}
			}
			// In case it's Array of Files, unlike array of int,
			// it's NOT gonna be joined with .Binding.Separator.
			flattened = append(flattened, separated...)
		}
	case "string": // Array of string
		switch arr := prov.(type) {
		case []interface{}:
			separated := []string{}
			for _, e := range arr {
				switch v := e.(type) {
				case string:
					if binding != nil && binding.Prefix != "" {
						separated = append(separated, binding.Prefix)
					}
					separated = append(separated, fmt.Sprintf("%v", v))
				default:
					// TODO
				}
			}
			flattened = append(flattened, separated...)
		default:
			// TODO
		}
	case "array":
		switch arr := prov.(type) {
		case []interface{}:
			flattened = append(flattened, input.flatten(typ.Items[0], typ.Binding, arr[0])...)
		default:
			// TODO
		}
	default:
		if input.RequiredType != nil {
			flattened = append(flattened, input.flattenWithRequiredType()...)
		} else {
			// TODO
		}
	}
	return flattened
}

func (input *Input) flattenWithRequiredType() []string {
	flattened := []string{}
	key, needed := input.Types[0].NeedRequirement()
	if !needed {
		return flattened
	}
	if input.RequiredType.Name != key {
		return flattened
	}
	switch provided := input.Provided.Raw.(type) {
	case []interface{}:
		for _, e := range provided {
			switch v := e.(type) {
			case map[interface{}]interface{}:
				for _, field := range input.RequiredType.Fields {
					val, ok := v[field.Name]
					if !ok {
						continue
					}
					if field.Binding == nil {
						// Without thinking anything, just append it!!!
						flattened = append(flattened, fmt.Sprintf("%v", val))
					} else {
						if field.Binding.Prefix != "" {
							if field.Binding.Separate {
								flattened = append(flattened, field.Binding.Prefix, fmt.Sprintf("%v", val))
							} else {
								// TODO: Join if .Separator is given
								flattened = append(flattened, fmt.Sprintf("%s%v", field.Binding.Prefix, val))
							}
						} else {
							switch v2 := val.(type) {
							case []interface{}:
								for _, val2 := range v2 {
									switch v3 := val2.(type) {
									case []interface{}:
									case map[interface{}]interface{}:
										for _, types := range input.Requirements[0].SchemaDefRequirement.Types {
											val3array := []string{}
											var val3count int = 0
											sort.Sort(types.Fields)
											for _, fields := range types.Fields {
												for key3, val3 := range v3 {
													if fields.Name == key3 {
														for _, val3type := range fields.Types {
															if val3type.Type == "" {
															} else {
																switch val3type.Type {
																case "enum":
																	for _, symbol := range val3type.Symbols {
																		if symbol == val3 {
																			val3array = append(val3array, fmt.Sprintf("%v", val3))
																			val3count = val3count + 1
																		}
																	}
																case "int":
																	if fields.Binding.Prefix != "" {
																		val3array = append(val3array, fields.Binding.Prefix, fmt.Sprintf("%v", val3))
																		val3count = val3count + 1
																	} else {
																		val3array = append(val3array, fmt.Sprintf("%v", val3))
																		val3count = val3count + 1
																	}
																}
															}
														}
													}
												}
											}
											if len(v3) == val3count {
												flattened = append(flattened, val3array...)
											}
										}
									}
								}
							}
						}
					}
				}
			}
		}
	}
	return flattened
}

// Flatten ...
func (input *Input) Flatten() []string {
	if input.Provided == nil {
		// In case "input.Default == nil" should be validated by usage layer.
		if input.Default != nil {
			return input.Default.Flatten(input.Binding)
		} else {
			return []string{}
		}
	}
	flattened := []string{}
	if repr := input.Types[0]; len(input.Types) == 1 {
		switch repr.Type {
		case "array":
			flattened = append(flattened, input.flatten(repr.Items[0], repr.Binding, input.Provided.Raw)...)
		case "int":
			flattened = append(flattened, fmt.Sprintf("%v", input.Provided.Int))
		case "File":
			flattened = append(flattened, input.Provided.Entry.Location)
		case "Any":
			switch v := input.Provided.Raw.(type) {
			case string:
				flattened = append(flattened, v)
			default:
				flattened = append(flattened, fmt.Sprintf("%v", v))
			}
		default:
			flattened = append(flattened, fmt.Sprintf("%v", input.Provided))
		}
	}
	if input.Binding != nil && input.Binding.Prefix != "" {
		flattened = append([]string{input.Binding.Prefix}, flattened...)
	}

	return flattened
}

// Inputs represents "inputs" field in CWL.
type Inputs []*Input

// New constructs new "Inputs" struct.
func (ins Inputs) New(i interface{}) Inputs {
	dest := Inputs{}
	switch x := i.(type) {
	case []interface{}:
		for _, v := range x {
			dest = append(dest, Input{}.New(v))
		}
	case map[string]interface{}:
		for key, v := range x {
			input := Input{}.New(v)
			input.ID = key
			dest = append(dest, input)
		}
	}
	return dest
}

// Len for sorting.
func (ins Inputs) Len() int {
	return len(ins)
}

// Less for sorting.
func (ins Inputs) Less(i, j int) bool {
	prev, next := ins[i].Binding, ins[j].Binding
	switch [2]bool{prev == nil, next == nil} {
	case [2]bool{true, true}:
		return true
	case [2]bool{false, true}:
		return prev.Position < 0
	case [2]bool{true, false}:
		return next.Position > 0
	default:
		return prev.Position <= next.Position
	}
}

// Swap for sorting.
func (ins Inputs) Swap(i, j int) {
	ins[i], ins[j] = ins[j], ins[i]
}

// ToJavaScriptVM ...
func (ins Inputs) ToJavaScriptVM() (*otto.Otto, error) {
	self := map[string]interface{}{}
	for _, i := range ins {
		if i.Provided != nil {
			if i.Provided.Entry != nil {
				self[i.ID] = map[string]interface{}{
					"path": i.Provided.Entry.Location,
				}
				continue
			}
			if i.Types[0].Type == "string" {
				self[i.ID] = i.Provided.Raw
				continue
			}
		}
		if i.Default != nil && i.Default.Entry != nil {
			self[i.ID] = map[string]interface{}{
				"path": i.Default.Entry.Location,
			}
			continue
		}
	}

	// No contents to load
	if len(self) == 0 {
		return nil, nil
	}

	vm := otto.New()
	if err := vm.Set("inputs", self); err != nil {
		return nil, err
	}
	return vm, nil
}
<|endoftext|>


<commit_before>package readline

import "io"

type inputHandler func(*Instance) (byte, bool)

var inputMap = map[byte]inputHandler{
	CharEOF:       eofHandler,
	CharTab:       tabHandler,
	CharEnter:     enterHandler,
	CharBackspace: backspaceHandler,
	CharInterrupt: interruptHandler,
}

func interruptHandler(inst *Instance) (byte, bool) {
	inst.Printf("\ngot Interrupt(Ctrl+C)\n")
	inst.clearLine()
	inst.printPrompt()
	return CharInterrupt, false
}

func backspaceHandler(inst *Instance) (byte, bool) {
	if len(inst.line) == 0 {
		return CharBackspace, false
	}
	inst.lineDel()
	return CharBackspace, false
}

func enterHandler(inst *Instance) (byte, bool) {
	end := inst.execute(string(inst.line), inst.data)
	if !end {
		inst.clearLine()
		inst.printPrompt()
	}
	return CharEnter, end
}

func tabHandler(inst *Instance) (byte, bool) {
	key := byte(CharTab)
	if inst.lastKey != CharTab { // First tab
		_, candidates, end, err := getCandidates(inst)
		if err != nil {
			inst.Log("1st tab error: %v\n", err)
			return key, false
		}
		switch len(candidates) {
		case 0:
			inst.Log("TODO: can not happen\n")
		case 1:
			completeWhole(inst, candidates[0])
			if end {
				inst.lineAdd(' ')
				key = ' '
			}
		default:
			completePartial(inst, candidates)
		}
	} else { // Second Tab
		cp, candidates, _, err := getCandidates(inst)
		if err != nil {
			inst.Log("2nd tab error: %v\n", err)
			return key, false
		}
		switch len(candidates) {
		case 0, 1:
			inst.Log("TODO: can not happen\n")
		default:
			inst.Log("multi candidates\n")
			printCandidates(inst, cp, candidates)
			inst.Printf("%s%s", inst.Prompt, string(inst.line))
		}
	}

	return key, false
}

func eofHandler(inst *Instance) (byte, bool) {
	if len(inst.line) == 0 {
		inst.Printf("\ngot EOF(Ctrl+D)\n")
		return CharEOF, true
	}
	inst.clearLine()
	inst.printPrompt()
	return CharEOF, false
}

func InputLoop(inst *Instance) {
	inst.printPrompt()
	end := false
	for !end {
		c, err := inst.r.ReadByte()
		if err != nil {
			if err == io.EOF {
				inst.Printf("got EOF\n")
			} else {
				inst.Printf("error: %v\n", err)
			}
			end = true
		}

		//inst.Log("[%d]", c)
		handler, ok := inputMap[c]
		key := c
		if ok {
			key, end = handler(inst)
		} else {
			inst.lineAdd(c)
		}
		inst.lastKey = key
	}
}
<commit_msg>output <Enter> charactor when press enter<commit_after>package readline

import "io"

type inputHandler func(*Instance) (byte, bool)

var inputMap = map[byte]inputHandler{
	CharEOF:       eofHandler,
	CharTab:       tabHandler,
	CharEnter:     enterHandler,
	CharBackspace: backspaceHandler,
	CharInterrupt: interruptHandler,
}

func interruptHandler(inst *Instance) (byte, bool) {
	inst.Printf("\ngot Interrupt(Ctrl+C)\n")
	inst.clearLine()
	inst.printPrompt()
	return CharInterrupt, false
}

func backspaceHandler(inst *Instance) (byte, bool) {
	if len(inst.line) == 0 {
		return CharBackspace, false
	}
	inst.lineDel()
	return CharBackspace, false
}

func enterHandler(inst *Instance) (byte, bool) {
	inst.Print("\n")
	end := inst.execute(string(inst.line), inst.data)
	if !end {
		inst.clearLine()
		inst.printPrompt()
	}
	return CharEnter, end
}

func tabHandler(inst *Instance) (byte, bool) {
	key := byte(CharTab)
	if inst.lastKey != CharTab { // First tab
		_, candidates, end, err := getCandidates(inst)
		if err != nil {
			inst.Log("1st tab error: %v\n", err)
			return key, false
		}
		switch len(candidates) {
		case 0:
			inst.Log("TODO: can not happen\n")
		case 1:
			completeWhole(inst, candidates[0])
			if end {
				inst.lineAdd(' ')
				key = ' '
			}
		default:
			completePartial(inst, candidates)
		}
	} else { // Second Tab
		cp, candidates, _, err := getCandidates(inst)
		if err != nil {
			inst.Log("2nd tab error: %v\n", err)
			return key, false
		}
		switch len(candidates) {
		case 0, 1:
			inst.Log("TODO: can not happen\n")
		default:
			inst.Log("multi candidates\n")
			printCandidates(inst, cp, candidates)
			inst.Printf("%s%s", inst.Prompt, string(inst.line))
		}
	}

	return key, false
}

func eofHandler(inst *Instance) (byte, bool) {
	if len(inst.line) == 0 {
		inst.Printf("\ngot EOF(Ctrl+D)\n")
		return CharEOF, true
	}
	inst.clearLine()
	inst.printPrompt()
	return CharEOF, false
}

func InputLoop(inst *Instance) {
	inst.printPrompt()
	end := false
	for !end {
		c, err := inst.r.ReadByte()
		if err != nil {
			if err == io.EOF {
				inst.Printf("got EOF\n")
			} else {
				inst.Printf("error: %v\n", err)
			}
			end = true
		}

		//inst.Log("[%d]", c)
		handler, ok := inputMap[c]
		key := c
		if ok {
			key, end = handler(inst)
		} else {
			inst.lineAdd(c)
		}
		inst.lastKey = key
	}
}
<|endoftext|>


<commit_before>package main

import (
	"github.com/rkoesters/goblin/lib"
	"io"
	"log"
	"os"
)

func main() {
	lib.InitLog()

	if len(os.Args) < 2 {
		cat(os.Stdin)
		return
	}

	for i := 1; i < len(os.Args); i++ {
		f, err := os.Open(os.Args[i])
		if err != nil {
			log.Fatal(err)
			continue
		}

		cat(f)
		f.Close()
	}
}

func cat(r io.Reader) {
	_, err := io.Copy(os.Stdout, r)
	if err != nil {
		log.Fatal(err)
	}
}
<commit_msg>Updated cmd/cat for mango-doc.<commit_after>// Concatenate files.
//
// cat reads each file in order and writes to stdout. If no file is
// given, then cat reads from stdin.
package main

import (
	"github.com/rkoesters/goblin/lib"
	"io"
	"log"
	"os"
)

// Usage: %name %flags [file...]
func main() {
	lib.InitLog()

	if len(os.Args) < 2 {
		cat(os.Stdin)
		return
	}

	for i := 1; i < len(os.Args); i++ {
		f, err := os.Open(os.Args[i])
		if err != nil {
			log.Fatal(err)
			continue
		}

		cat(f)
		f.Close()
	}
}

func cat(r io.Reader) {
	_, err := io.Copy(os.Stdout, r)
	if err != nil {
		log.Fatal(err)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"bytes"
	"fmt"
	"io"
	"os"
	"path/filepath"
	"strings"

	"github.com/codegangsta/cli"
	"github.com/dustin/go-humanize"

	"github.intel.com/hpdd/logging/applog"
	"github.intel.com/hpdd/lustre"
	"github.intel.com/hpdd/lustre/fs"
	"github.intel.com/hpdd/lustre/hsm"
)

func init() {
	hsmStateFlags := strings.Join(hsm.GetStateFlagNames(), ",")

	hsmCommand := cli.Command{
		Name:  "hsm",
		Usage: "HSM-related data movement actions",
		Subcommands: []cli.Command{
			{
				Name:      "archive",
				Usage:     "Initiate HSM archive of specified paths",
				ArgsUsage: "[path [path...]]",
				Flags: []cli.Flag{
					cli.IntFlag{
						Name:  "id, i",
						Usage: "Numeric ID of archive backend",
					},
					cli.BoolFlag{
						Name:  "null, 0",
						Usage: "Null-separated paths are read from stdin (e.g. piped from find -print0)",
					},
				},
				Action: hsmRequestAction(hsm.RequestArchive),
			},
			{
				Name:      "release",
				Usage:     "Release local data of HSM-archived paths",
				ArgsUsage: "[path [path...]]",
				Flags: []cli.Flag{
					cli.BoolFlag{
						Name:  "null, 0",
						Usage: "Null-separated paths are read from stdin (e.g. piped from find -print0)",
					},
				},
				Action: hsmRequestAction(hsm.RequestRelease),
			},
			{
				Name:      "restore",
				Usage:     "Explicitly restore local data of HSM-archived paths",
				ArgsUsage: "[path [path...]]",
				Flags: []cli.Flag{
					cli.BoolFlag{
						Name:  "null, 0",
						Usage: "Null-separated paths are read from stdin (e.g. piped from find -print0)",
					},
				},
				Action: hsmRequestAction(hsm.RequestRestore),
			},
			{
				Name:      "remove",
				Usage:     "Remove HSM-archived data of specified paths (local data is not removed)",
				ArgsUsage: "[path [path...]]",
				Flags: []cli.Flag{
					cli.BoolFlag{
						Name:  "null, 0",
						Usage: "Null-separated paths are read from stdin (e.g. piped from find -print0)",
					},
				},
				Action: hsmRequestAction(hsm.RequestRemove),
			},
			{
				Name:      "cancel",
				Usage:     "Cancel HSM operations being performed on specified paths",
				ArgsUsage: "[path [path...]]",
				Flags: []cli.Flag{
					cli.BoolFlag{
						Name:  "null, 0",
						Usage: "Null-separated paths are read from stdin (e.g. piped from find -print0)",
					},
				},
				Action: hsmRequestAction(hsm.RequestCancel),
			},
			{
				Name:      "set",
				Usage:     "Set HSM flags or archive ID for specified paths",
				ArgsUsage: "[path [path...]]",
				Flags: []cli.Flag{
					cli.BoolFlag{
						Name:  "null, 0",
						Usage: "Null-separated paths are read from stdin (e.g. piped from find -print0)",
					},
					cli.IntFlag{
						Name:  "id, i",
						Usage: "Numeric ID of archive backend",
					},
					cli.StringSliceFlag{
						Name:  "flag, f",
						Usage: fmt.Sprintf("HSM flag to set (%s)", hsmStateFlags),
						Value: &cli.StringSlice{},
					},
					cli.StringSliceFlag{
						Name:  "clear, F",
						Usage: fmt.Sprintf("HSM flag to clear (%s)", hsmStateFlags),
						Value: &cli.StringSlice{},
					},
				},
				Action: hsmSetAction,
			},
			{
				Name:      "status",
				Usage:     "Display HSM status for specified paths",
				ArgsUsage: "[path [path...]]",
				Flags: []cli.Flag{
					cli.BoolFlag{
						Name:  "action, a",
						Usage: "Include current HSM action",
					},
					cli.BoolFlag{
						Name:  "hide-path, H",
						Usage: "Hide pathname in output",
					},
					cli.BoolFlag{
						Name:  "long, l",
						Usage: "Show long-form states",
					},
					cli.BoolFlag{
						Name:  "progress, p",
						Usage: "Show copy progress for archive/restore actions",
					},
					cli.BoolFlag{
						Name:  "null, 0",
						Usage: "Null-separated paths are read from stdin (e.g. piped from find -print0)",
					},
				},
				Action: hsmStatusAction,
			},
		},
	}
	commands = append(commands, hsmCommand)
}

func getFilePaths(c *cli.Context) ([]string, error) {
	var paths []string

	if c.Bool("null") {
		reader := bufio.NewReader(os.Stdin)
		path, err := reader.ReadBytes('\000')
		for err == nil {
			paths = append(paths, string(path[:len(path)-1]))
			path, err = reader.ReadBytes('\000')
		}
		if err != io.EOF {
			return nil, err
		}
	} else {
		paths = c.Args()
	}

	return paths, nil
}

func getPathStatus(c *cli.Context, filePath string) (string, error) {
	var buf bytes.Buffer

	s, err := hsm.GetFileStatus(filePath)
	if err != nil {
		return "", err
	}

	if !c.Bool("hide-path") {
		fmt.Fprintf(&buf, "%s ", filePath)
	}
	fmt.Fprintf(&buf, hsm.FileStatusString(s, !c.Bool("long")))

	if s.Exists() && c.Bool("action") {
		a, err := hsm.GetFileAction(filePath)
		if err != nil {
			return "", err
		}
		if a.IsNone() {
			fmt.Fprintf(&buf, " -")
		} else {
			fmt.Fprintf(&buf, " [%s:%s]", a.Action(), a.State())
			if c.Bool("progress") && (a.IsArchive() || a.IsRestore()) {
				st, err := os.Stat(filePath)
				if err != nil {
					applog.Fail(err)
				}
				fmt.Fprintf(&buf, "(%s/%s)",
					humanize.IBytes(a.BytesCopied),
					humanize.IBytes(uint64(st.Size())))
			}
		}
	} else {
		fmt.Fprintf(&buf, " -")
	}

	// TODO: Display xattrs, once we've standardized them?

	return buf.String(), nil
}

func hsmSetAction(c *cli.Context) {
	logContext(c)

	paths, err := getFilePaths(c)
	if err != nil {
		applog.Fail(err)
	}

	if len(paths) < 1 {
		applog.Fail(fmt.Errorf("HSM set request must be made with at least 1 path"))
	}

	setFlags, err := hsm.GetStatusMask(c.StringSlice("flag"))
	if err != nil {
		applog.Fail(err)
	}
	clearFlags, err := hsm.GetStatusMask(c.StringSlice("clear"))
	if err != nil {
		applog.Fail(err)
	}
	archiveID := uint32(c.Int("id"))

	if setFlags == 0 && clearFlags == 0 && archiveID == 0 {
		applog.Fail(fmt.Errorf("HSM set request made with no flags to set or clear, and no new archive ID supplied"))
	}

	// TODO: Parallelize this?
	for _, path := range paths {
		if err := hsm.SetFileStatus(path, setFlags, clearFlags, archiveID); err != nil {
			applog.Fail(err)
		}
	}
}

func hsmStatusAction(c *cli.Context) {
	logContext(c)

	paths, err := getFilePaths(c)
	if err != nil {
		applog.Fail(err)
	}

	if len(paths) < 1 {
		applog.Fail(fmt.Errorf("HSM status request must be made with at least 1 path"))
	}

	for _, path := range paths {
		status, err := getPathStatus(c, path)
		if err != nil {
			applog.Fail("%s: %v", path, err)
		}
		fmt.Println(status)
	}
}

type hsmRequestFn func(fs.RootDir, uint, []*lustre.Fid) error

func hsmRequestAction(requestFn func(fs.RootDir, uint, []*lustre.Fid) error) func(*cli.Context) {
	return func(c *cli.Context) {
		logContext(c)

		paths, err := getFilePaths(c)
		if err != nil {
			applog.Fail(err)
		}
		err = submitHsmRequest(c.Command.Name, hsmRequestFn(requestFn), uint(c.Int("id")), paths...)
		if err != nil {
			applog.Fail(err)
		}
	}
}

func submitHsmRequest(actionName string, requestFn hsmRequestFn, archiveID uint, paths ...string) error {
	var fids []*lustre.Fid

	if len(paths) < 1 {
		return fmt.Errorf("HSM %s request must be made with at least 1 path", actionName)
	}

	fsRoot, err := fs.MountRoot(paths[0])
	if err != nil {
		return fmt.Errorf("Error getting fs root from %s: %s", paths[0], err)
	}

	// TODO: Occurs to me that it might be better to break up a large
	// batch into multiple batches, each serviced by its own goroutine.
	for _, path := range paths {
		absPath, err := filepath.Abs(path)
		if !strings.HasPrefix(absPath, string(fsRoot)) {
			return fmt.Errorf("All files in HSM request must be in the same filesystem (%s is not in %s)",
				path, fsRoot)
		}

		fid, err := fs.LookupFid(path)
		if err != nil {
			return fmt.Errorf("Cannot resolve Fid for %s: %s", path, err)
		}
		fids = append(fids, fid)
	}

	if requestFn != nil {
		err = requestFn(fsRoot, archiveID, fids)
	} else {
		err = fmt.Errorf("Unhandled HSM action: %s", actionName)
	}

	return err
}
<commit_msg>Fix stringer problem for fsroot in ldmc<commit_after>package main

import (
	"bufio"
	"bytes"
	"fmt"
	"io"
	"os"
	"path/filepath"
	"strings"

	"github.com/codegangsta/cli"
	"github.com/dustin/go-humanize"

	"github.intel.com/hpdd/logging/applog"
	"github.intel.com/hpdd/lustre"
	"github.intel.com/hpdd/lustre/fs"
	"github.intel.com/hpdd/lustre/hsm"
)

func init() {
	hsmStateFlags := strings.Join(hsm.GetStateFlagNames(), ",")

	hsmCommand := cli.Command{
		Name:  "hsm",
		Usage: "HSM-related data movement actions",
		Subcommands: []cli.Command{
			{
				Name:      "archive",
				Usage:     "Initiate HSM archive of specified paths",
				ArgsUsage: "[path [path...]]",
				Flags: []cli.Flag{
					cli.IntFlag{
						Name:  "id, i",
						Usage: "Numeric ID of archive backend",
					},
					cli.BoolFlag{
						Name:  "null, 0",
						Usage: "Null-separated paths are read from stdin (e.g. piped from find -print0)",
					},
				},
				Action: hsmRequestAction(hsm.RequestArchive),
			},
			{
				Name:      "release",
				Usage:     "Release local data of HSM-archived paths",
				ArgsUsage: "[path [path...]]",
				Flags: []cli.Flag{
					cli.BoolFlag{
						Name:  "null, 0",
						Usage: "Null-separated paths are read from stdin (e.g. piped from find -print0)",
					},
				},
				Action: hsmRequestAction(hsm.RequestRelease),
			},
			{
				Name:      "restore",
				Usage:     "Explicitly restore local data of HSM-archived paths",
				ArgsUsage: "[path [path...]]",
				Flags: []cli.Flag{
					cli.BoolFlag{
						Name:  "null, 0",
						Usage: "Null-separated paths are read from stdin (e.g. piped from find -print0)",
					},
				},
				Action: hsmRequestAction(hsm.RequestRestore),
			},
			{
				Name:      "remove",
				Usage:     "Remove HSM-archived data of specified paths (local data is not removed)",
				ArgsUsage: "[path [path...]]",
				Flags: []cli.Flag{
					cli.BoolFlag{
						Name:  "null, 0",
						Usage: "Null-separated paths are read from stdin (e.g. piped from find -print0)",
					},
				},
				Action: hsmRequestAction(hsm.RequestRemove),
			},
			{
				Name:      "cancel",
				Usage:     "Cancel HSM operations being performed on specified paths",
				ArgsUsage: "[path [path...]]",
				Flags: []cli.Flag{
					cli.BoolFlag{
						Name:  "null, 0",
						Usage: "Null-separated paths are read from stdin (e.g. piped from find -print0)",
					},
				},
				Action: hsmRequestAction(hsm.RequestCancel),
			},
			{
				Name:      "set",
				Usage:     "Set HSM flags or archive ID for specified paths",
				ArgsUsage: "[path [path...]]",
				Flags: []cli.Flag{
					cli.BoolFlag{
						Name:  "null, 0",
						Usage: "Null-separated paths are read from stdin (e.g. piped from find -print0)",
					},
					cli.IntFlag{
						Name:  "id, i",
						Usage: "Numeric ID of archive backend",
					},
					cli.StringSliceFlag{
						Name:  "flag, f",
						Usage: fmt.Sprintf("HSM flag to set (%s)", hsmStateFlags),
						Value: &cli.StringSlice{},
					},
					cli.StringSliceFlag{
						Name:  "clear, F",
						Usage: fmt.Sprintf("HSM flag to clear (%s)", hsmStateFlags),
						Value: &cli.StringSlice{},
					},
				},
				Action: hsmSetAction,
			},
			{
				Name:      "status",
				Usage:     "Display HSM status for specified paths",
				ArgsUsage: "[path [path...]]",
				Flags: []cli.Flag{
					cli.BoolFlag{
						Name:  "action, a",
						Usage: "Include current HSM action",
					},
					cli.BoolFlag{
						Name:  "hide-path, H",
						Usage: "Hide pathname in output",
					},
					cli.BoolFlag{
						Name:  "long, l",
						Usage: "Show long-form states",
					},
					cli.BoolFlag{
						Name:  "progress, p",
						Usage: "Show copy progress for archive/restore actions",
					},
					cli.BoolFlag{
						Name:  "null, 0",
						Usage: "Null-separated paths are read from stdin (e.g. piped from find -print0)",
					},
				},
				Action: hsmStatusAction,
			},
		},
	}
	commands = append(commands, hsmCommand)
}

func getFilePaths(c *cli.Context) ([]string, error) {
	var paths []string

	if c.Bool("null") {
		reader := bufio.NewReader(os.Stdin)
		path, err := reader.ReadBytes('\000')
		for err == nil {
			paths = append(paths, string(path[:len(path)-1]))
			path, err = reader.ReadBytes('\000')
		}
		if err != io.EOF {
			return nil, err
		}
	} else {
		paths = c.Args()
	}

	return paths, nil
}

func getPathStatus(c *cli.Context, filePath string) (string, error) {
	var buf bytes.Buffer

	s, err := hsm.GetFileStatus(filePath)
	if err != nil {
		return "", err
	}

	if !c.Bool("hide-path") {
		fmt.Fprintf(&buf, "%s ", filePath)
	}
	fmt.Fprintf(&buf, hsm.FileStatusString(s, !c.Bool("long")))

	if s.Exists() && c.Bool("action") {
		a, err := hsm.GetFileAction(filePath)
		if err != nil {
			return "", err
		}
		if a.IsNone() {
			fmt.Fprintf(&buf, " -")
		} else {
			fmt.Fprintf(&buf, " [%s:%s]", a.Action(), a.State())
			if c.Bool("progress") && (a.IsArchive() || a.IsRestore()) {
				st, err := os.Stat(filePath)
				if err != nil {
					applog.Fail(err)
				}
				fmt.Fprintf(&buf, "(%s/%s)",
					humanize.IBytes(a.BytesCopied),
					humanize.IBytes(uint64(st.Size())))
			}
		}
	} else {
		fmt.Fprintf(&buf, " -")
	}

	// TODO: Display xattrs, once we've standardized them?

	return buf.String(), nil
}

func hsmSetAction(c *cli.Context) {
	logContext(c)

	paths, err := getFilePaths(c)
	if err != nil {
		applog.Fail(err)
	}

	if len(paths) < 1 {
		applog.Fail(fmt.Errorf("HSM set request must be made with at least 1 path"))
	}

	setFlags, err := hsm.GetStatusMask(c.StringSlice("flag"))
	if err != nil {
		applog.Fail(err)
	}
	clearFlags, err := hsm.GetStatusMask(c.StringSlice("clear"))
	if err != nil {
		applog.Fail(err)
	}
	archiveID := uint32(c.Int("id"))

	if setFlags == 0 && clearFlags == 0 && archiveID == 0 {
		applog.Fail(fmt.Errorf("HSM set request made with no flags to set or clear, and no new archive ID supplied"))
	}

	// TODO: Parallelize this?
	for _, path := range paths {
		if err := hsm.SetFileStatus(path, setFlags, clearFlags, archiveID); err != nil {
			applog.Fail(err)
		}
	}
}

func hsmStatusAction(c *cli.Context) {
	logContext(c)

	paths, err := getFilePaths(c)
	if err != nil {
		applog.Fail(err)
	}

	if len(paths) < 1 {
		applog.Fail(fmt.Errorf("HSM status request must be made with at least 1 path"))
	}

	for _, path := range paths {
		status, err := getPathStatus(c, path)
		if err != nil {
			applog.Fail("%s: %v", path, err)
		}
		fmt.Println(status)
	}
}

type hsmRequestFn func(fs.RootDir, uint, []*lustre.Fid) error

func hsmRequestAction(requestFn func(fs.RootDir, uint, []*lustre.Fid) error) func(*cli.Context) {
	return func(c *cli.Context) {
		logContext(c)

		paths, err := getFilePaths(c)
		if err != nil {
			applog.Fail(err)
		}
		err = submitHsmRequest(c.Command.Name, hsmRequestFn(requestFn), uint(c.Int("id")), paths...)
		if err != nil {
			applog.Fail(err)
		}
	}
}

func submitHsmRequest(actionName string, requestFn hsmRequestFn, archiveID uint, paths ...string) error {
	var fids []*lustre.Fid

	if len(paths) < 1 {
		return fmt.Errorf("HSM %s request must be made with at least 1 path", actionName)
	}

	fsRoot, err := fs.MountRoot(paths[0])
	if err != nil {
		return fmt.Errorf("Error getting fs root from %s: %s", paths[0], err)
	}

	// TODO: Occurs to me that it might be better to break up a large
	// batch into multiple batches, each serviced by its own goroutine.
	for _, path := range paths {
		absPath, err := filepath.Abs(path)
		if !strings.HasPrefix(absPath, fsRoot.String()) {
			return fmt.Errorf("All files in HSM request must be in the same filesystem (%s is not in %s)",
				path, fsRoot)
		}

		fid, err := fs.LookupFid(path)
		if err != nil {
			return fmt.Errorf("Cannot resolve Fid for %s: %s", path, err)
		}
		fids = append(fids, fid)
	}

	if requestFn != nil {
		err = requestFn(fsRoot, archiveID, fids)
	} else {
		err = fmt.Errorf("Unhandled HSM action: %s", actionName)
	}

	return err
}
<|endoftext|>


<commit_before>package main

import (
	"github.com/google/go-github/github"
	"strconv"
)

type Issue struct {
	ID       string `json:"issue"`
	Status   string `json:"status"`
	Title    string `json:"title,omitempty"`
	Body     string `json:"body,omitempty"`
	Assignee string `json:"assignee,omitempty"`
	Comments int    `json:"comments,omitempty"`
	URL      string `json:"url,omitempty"`
}

func (i *Issue) githubImport(gi *github.Issue) {
}
<commit_msg>Cleanupt<commit_after>package main

type Issue struct {
	ID       string `json:"issue"`
	Status   string `json:"status"`
	Title    string `json:"title,omitempty"`
	Body     string `json:"body,omitempty"`
	Assignee string `json:"assignee,omitempty"`
	Comments int    `json:"comments,omitempty"`
	URL      string `json:"url,omitempty"`
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"fmt"
	"os"
	"path/filepath"

	"github.com/alexflint/go-arg"

	"github.com/walle/lll"
)

var args struct {
	MaxLength int      `arg:"-l,help:max line length to check for"`
	GoOnly    bool     `arg:"-g,help:only check .go files"`
	Input     string   `arg:"positional"`
	SkipList  []string `arg:"-s,help:list of dirs to skip [default: .git vendor]"`
	Vendor    bool     `arg:"help:check files in vendor directory"`
	Files     bool     `arg:"help:read file names from stdin one at each line"`
}

func main() {
	args.Input = "./"
	args.MaxLength = 80
	args.SkipList = []string{".git", "vendor"}
	arg.MustParse(&args)

	// If we should include the vendor dir, attempt to remove it from the skip list
	if args.Vendor {
		for i, p := range args.SkipList {
			if p == "vendor" {
				args.SkipList = append(args.SkipList[:i], args.SkipList[:i]...)
			}
		}
	}

	// If we should read files from stdin, read each line and process the file
	if args.Files {
		s := bufio.NewScanner(os.Stdin)
		for s.Scan() {
			err := lll.ProcessFile(os.Stdout, s.Text(), args.MaxLength)
			if err != nil {
				fmt.Fprintf(os.Stderr, "Error processing file: %s\n", err)
			}
		}
		os.Exit(0)
	}

	// Otherwise, walk the input dir recursively
	err := filepath.Walk(args.Input, func(p string, i os.FileInfo, e error) error {
		skip, ret := lll.ShouldSkip(p, i.IsDir(), e, args.SkipList, args.GoOnly)
		if skip {
			return ret
		}

		err := lll.ProcessFile(os.Stdout, p, args.MaxLength)
		return err
	})

	if err != nil {
		fmt.Fprintf(os.Stderr, "Error walking the file system: %s\n", err)
		os.Exit(1)
	}
}
<commit_msg>Add support for specifying flags with ENV<commit_after>package main

import (
	"bufio"
	"fmt"
	"os"
	"path/filepath"

	"github.com/alexflint/go-arg"

	"github.com/walle/lll"
)

var args struct {
	MaxLength int      `arg:"-l,env,help:max line length to check for"`
	GoOnly    bool     `arg:"-g,env,help:only check .go files"`
	Input     string   `arg:"positional"`
	SkipList  []string `arg:"-s,env,help:list of dirs to skip [default: .git vendor]"`
	Vendor    bool     `arg:"env,help:check files in vendor directory"`
	Files     bool     `arg:"help:read file names from stdin one at each line"`
}

func main() {
	args.Input = "./"
	args.MaxLength = 80
	args.SkipList = []string{".git", "vendor"}
	arg.MustParse(&args)

	// If we should include the vendor dir, attempt to remove it from the skip list
	if args.Vendor {
		for i, p := range args.SkipList {
			if p == "vendor" {
				args.SkipList = append(args.SkipList[:i], args.SkipList[:i]...)
			}
		}
	}

	// If we should read files from stdin, read each line and process the file
	if args.Files {
		s := bufio.NewScanner(os.Stdin)
		for s.Scan() {
			err := lll.ProcessFile(os.Stdout, s.Text(), args.MaxLength)
			if err != nil {
				fmt.Fprintf(os.Stderr, "Error processing file: %s\n", err)
			}
		}
		os.Exit(0)
	}

	// Otherwise, walk the input dir recursively
	err := filepath.Walk(args.Input, func(p string, i os.FileInfo, e error) error {
		skip, ret := lll.ShouldSkip(p, i.IsDir(), e, args.SkipList, args.GoOnly)
		if skip {
			return ret
		}

		err := lll.ProcessFile(os.Stdout, p, args.MaxLength)
		return err
	})

	if err != nil {
		fmt.Fprintf(os.Stderr, "Error walking the file system: %s\n", err)
		os.Exit(1)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"bytes"
	"flag"
	"fmt"
	"io"
	"log"
	"os"
	"os/exec"
	"strings"
	"sync"
	"time"

	"git.apache.org/thrift.git/lib/go/thrift"
	"github.com/daviddengcn/go-colortext"
	"github.com/korovkin/limiter"
	"github.com/korovkin/parallel"
)

type logger struct {
	ticket   int
	hostname string
	buf      *bytes.Buffer
}

var (
	loggerMutex     = new(sync.Mutex)
	loggerIndex     = int(0)
	loggerStartTime = time.Now()
	loggerHostname  = ""
)

var loggerColors = []ct.Color{
	ct.Green,
	ct.Cyan,
	ct.Magenta,
	ct.Yellow,
	ct.Blue,
	ct.Red,
}

func (l *logger) Write(p []byte) (int, error) {
	buf := bytes.NewBuffer(p)
	wrote := 0
	for {
		line, err := buf.ReadBytes('\n')
		if len(line) > 1 {
			now := time.Now().Format("15:04:05")
			s := string(line)
			ts := time.Since(loggerStartTime).String()

			loggerMutex.Lock()
			ct.ChangeColor(loggerColors[l.ticket%len(loggerColors)], false, ct.None, false)
			fmt.Printf("[%16s %s %s %d] ", ts, l.hostname, now, l.ticket)
			ct.ResetColor()

			if l.buf != nil {
				l.buf.Write([]byte(s))
			}

			fmt.Print(s)
			loggerMutex.Unlock()

			wrote += len(line)
		}
		if err != nil {
			break
		}
	}
	if len(p) > 0 && p[len(p)-1] != '\n' {
		fmt.Println()
	}

	return len(p), nil
}

func newLogger(ticket int, collectLines bool) *logger {
	l := &logger{ticket: ticket, buf: nil}
	if collectLines {
		l.buf = &bytes.Buffer{}
	}
	return l
}

func executeCommand(p *Parallel, ticket int, cmdLine string) (*parallel.Output, error) {
	T_START := time.Now()
	var err error
	output := &parallel.Output{}

	loggerOut := newLogger(ticket, true)
	loggerErr := newLogger(ticket, true)

	defer func() {
		fmt.Fprintf(
			loggerOut,
			"execute: done: dt: "+time.Since(T_START).String()+"\n",
		)
	}()

	// execute remotely:
	if len(p.Slaves) > 0 {
		slave := p.Slaves[ticket%len(p.Slaves)]

		var transport thrift.TTransport
		transport, err = thrift.NewTSocket(slave.Address)
		if err != nil {
			log.Fatalln("failed to dial slave:", err.Error())
		}

		transport, err = p.transportFactory.GetTransport(transport)
		if err != nil {
			log.Fatalln("failed to GetTransport:", err.Error())
		}

		err = transport.Open()
		if err != nil {
			log.Fatalln("failed to open:", err.Error())
		}

		defer transport.Close()
		slave.Client = parallel.NewParallelClientFactory(transport, p.protocolFactory)

		output, err = slave.Client.Execute(&parallel.Cmd{
			CmdLine: cmdLine,
			Ticket:  int64(ticket),
		})
		if err != nil {
			log.Fatalln("failed to execute:", err.Error())
		}

		hostname := output.Tags["hostname"]
		loggerOut.hostname = hostname
		loggerErr.hostname = hostname

		fmt.Fprintf(loggerOut, "execute: remotely: host: %s stdout: [%s]\n", hostname, output.Stdout)
		fmt.Fprintf(loggerErr, "execute: remotely: host: %s stderr: [%s]\n", hostname, output.Stderr)

		return output, err
	}

	// execute locally:
	cs := []string{"/bin/sh", "-c", cmdLine}
	cmd := exec.Command(cs[0], cs[1:]...)
	cmd.Stdin = nil
	cmd.Stdout = loggerOut
	cmd.Stderr = loggerErr
	cmd.Env = append(
		os.Environ(),
		fmt.Sprintf("PARALLEL_TICKER=%d", ticket),
	)

	fmt.Fprintf(loggerOut, "run: '"+cmdLine+"'\n")

	err = cmd.Start()
	if err != nil {
		log.Fatalln("failed to start:", err)
		return output, err
	}

	if err == nil {
		err = cmd.Wait()
	}

	output.Tags = map[string]string{"hostname": loggerHostname}

	if loggerOut.buf != nil {
		output.Stdout = string(loggerOut.buf.Bytes())
	}

	if loggerErr.buf != nil {
		output.Stderr = string(loggerErr.buf.Bytes())
	}

	return output, err
}

type Slave struct {
	Address string `json:"address"`
	Client  *parallel.ParallelClient
}

type Parallel struct {
	jobs   int
	logger *logger
	worker *limiter.ConcurrencyLimiter

	// master / slave
	protocolFactory  thrift.TProtocolFactory
	transportFactory thrift.TTransportFactory

	// master:
	Slaves []*Slave

	// slave:
	handler         *ParallelSlaveHandler
	serverTransport thrift.TServerTransport
	address         string
}

func (p *Parallel) Close() {
	p.worker.Wait()
}

func mainMaster(p *Parallel) {
	var err error

	// connect to slaves:
	for _, slave := range p.Slaves {
		var transport thrift.TTransport
		transport, err = thrift.NewTSocket(slave.Address)
		if err != nil {
			log.Fatalln("failed to dial slave:", err.Error())
		}

		transport, err = p.transportFactory.GetTransport(transport)
		if err != nil {
			log.Fatalln("failed to open:", err.Error())
		}

		err = transport.Open()
		if err != nil {
			log.Fatalln("failed to open:", err.Error())
		}

		defer transport.Close()

		slave.Client = parallel.NewParallelClientFactory(transport, p.protocolFactory)
		ok, err := slave.Client.Ping()
		if err != nil {
			log.Fatalln("failed to ping client:", err.Error())
		}

		fmt.Fprintf(p.logger, fmt.Sprintf("adding slave: %s ok: %s", slave.Address, ok))

	}

	r := bufio.NewReaderSize(os.Stdin, 1*1024*1024)
	fmt.Fprintf(p.logger, "reading from stdin...\n")
	for {
		line, err := r.ReadString('\n')
		if err == io.EOF {
			break
		}
		line = strings.TrimSpace(line)

		p.worker.ExecuteWithTicket(func(ticket int) {
			executeCommand(p, ticket, line)
		})
	}
}

type ParallelSlaveHandler struct {
	p *Parallel
}

func NewParallelSlaveHandler() *ParallelSlaveHandler {
	return &ParallelSlaveHandler{}
}

func (p *ParallelSlaveHandler) Execute(command *parallel.Cmd) (output *parallel.Output, err error) {
	log.Println("ParallelSlaveHandler: Execute: ", command.CmdLine)
	output, err = executeCommand(p.p, int(command.Ticket), command.CmdLine)
	// TODO:: recover, handle panics
	return output, err
}

func (p *ParallelSlaveHandler) Ping() (r string, err error) {
	log.Println("ParallelSlaveHandler: Ping")
	return "ping:ok", nil
}

func mainSlave(p *Parallel) {
	var err error

	p.serverTransport, err = thrift.NewTServerSocket(p.address)

	if err != nil {
		log.Fatalln("failed to start server:", err.Error())
		return
	}

	p.handler = NewParallelSlaveHandler()
	p.handler.p = p

	server := thrift.NewTSimpleServer4(
		parallel.NewParallelProcessor(p.handler),
		p.serverTransport,
		p.transportFactory,
		p.protocolFactory)

	err = server.Serve()
	if err != nil {
		log.Fatalln("failed to run slave:", err.Error())
	}
}

func main() {
	T_START := time.Now()
	logger := newLogger(0, false)
	defer func() {
		fmt.Fprintf(logger, "all done: dt: "+time.Since(T_START).String()+"\n")
	}()

	flag_jobs := flag.Int(
		"j",
		2,
		"num of concurrent jobs")

	flag_slave := flag.Bool(
		"slave",
		false,
		"run as slave")

	slaves := flag.String(
		"slaves",
		"",
		"CSV list of slave addresses")

	address := flag.String(
		"address",
		"localhost:9010",
		"slave address")

	loggerHostname, _ = os.Hostname()

	flag.Parse()
	fmt.Fprintf(logger, fmt.Sprintf("concurrency limit: %d", *flag_jobs))
	fmt.Fprintf(logger, fmt.Sprintf("slaves: %s", *slaves))

	p := Parallel{}
	p.jobs = *flag_jobs
	p.logger = logger
	p.worker = limiter.NewConcurrencyLimiter(p.jobs)
	p.address = *address
	p.Slaves = []*Slave{}
	for _, slaveAddr := range strings.Split(*slaves, ",") {
		if slaveAddr == "" {
			continue
		}
		slave := Slave{Address: slaveAddr}
		p.Slaves = append(p.Slaves, &slave)
	}

	defer p.Close()

	// thrift protocol
	p.protocolFactory = thrift.NewTBinaryProtocolFactoryDefault()

	// thrift transport
	p.transportFactory = thrift.NewTTransportFactory()
	p.transportFactory = thrift.NewTFramedTransportFactory(p.transportFactory)

	if *flag_slave == false {
		loggerHostname = p.address
		logger.hostname = loggerHostname

		fmt.Fprintf(logger, fmt.Sprintf("running as master\n"))
		mainMaster(&p)
	} else {
		loggerHostname = p.address
		logger.hostname = loggerHostname

		fmt.Fprintf(logger, fmt.Sprintf("running as slave on: %s\n", p.address))
		mainSlave(&p)
	}
}
<commit_msg>bug fix: Client must not be 'global'<commit_after>package main

import (
	"bufio"
	"bytes"
	"flag"
	"fmt"
	"io"
	"log"
	"os"
	"os/exec"
	"strings"
	"sync"
	"time"

	"git.apache.org/thrift.git/lib/go/thrift"
	"github.com/daviddengcn/go-colortext"
	"github.com/korovkin/limiter"
	"github.com/korovkin/parallel"
)

type logger struct {
	ticket   int
	hostname string
	buf      *bytes.Buffer
}

var (
	loggerMutex     = new(sync.Mutex)
	loggerIndex     = int(0)
	loggerStartTime = time.Now()
	loggerHostname  = ""
)

var loggerColors = []ct.Color{
	ct.Green,
	ct.Cyan,
	ct.Magenta,
	ct.Yellow,
	ct.Blue,
	ct.Red,
}

func (l *logger) Write(p []byte) (int, error) {
	buf := bytes.NewBuffer(p)
	wrote := 0
	for {
		line, err := buf.ReadBytes('\n')
		if len(line) > 1 {
			now := time.Now().Format("15:04:05")
			s := string(line)
			ts := time.Since(loggerStartTime).String()

			loggerMutex.Lock()
			ct.ChangeColor(loggerColors[l.ticket%len(loggerColors)], false, ct.None, false)
			fmt.Printf("[%16s %s %s %d] ", ts, l.hostname, now, l.ticket)
			ct.ResetColor()

			if l.buf != nil {
				l.buf.Write([]byte(s))
			}

			fmt.Print(s)
			loggerMutex.Unlock()

			wrote += len(line)
		}
		if err != nil {
			break
		}
	}
	if len(p) > 0 && p[len(p)-1] != '\n' {
		fmt.Println()
	}

	return len(p), nil
}

func newLogger(ticket int, collectLines bool) *logger {
	l := &logger{ticket: ticket, buf: nil}
	if collectLines {
		l.buf = &bytes.Buffer{}
	}
	return l
}

func executeCommand(p *Parallel, ticket int, cmdLine string) (*parallel.Output, error) {
	T_START := time.Now()
	var err error
	output := &parallel.Output{}
	loggerOut := newLogger(ticket, true)
	loggerErr := newLogger(ticket, true)

	defer func() {
		fmt.Fprintf(
			loggerOut,
			"execute: done: dt: "+time.Since(T_START).String()+"\n",
		)
	}()

	// execute remotely:
	if len(p.Slaves) > 0 {
		slave := p.Slaves[ticket%len(p.Slaves)]

		var transport thrift.TTransport
		transport, err = thrift.NewTSocket(slave.Address)
		if err != nil {
			log.Fatalln("failed to dial slave:", err.Error())
		}

		transport, err = p.transportFactory.GetTransport(transport)
		if err != nil {
			log.Fatalln("failed to GetTransport:", err.Error())
		}

		err = transport.Open()
		if err != nil {
			log.Fatalln("failed to open:", err.Error())
		}

		defer transport.Close()
		client := parallel.NewParallelClientFactory(transport, p.protocolFactory)

		cmd := parallel.Cmd{
			CmdLine: cmdLine,
			Ticket:  int64(ticket),
		}

		output, err = client.Execute(&cmd)
		if err != nil {
			log.Fatalln("failed to execute:", err.Error())
		}

		hostname := output.Tags["hostname"]
		loggerOut.hostname = hostname
		loggerErr.hostname = hostname

		fmt.Fprintf(loggerOut, "execute: remotely: host: %s stdout: [%s]\n", hostname, output.Stdout)
		fmt.Fprintf(loggerErr, "execute: remotely: host: %s stderr: [%s]\n", hostname, output.Stderr)

		return output, err
	}

	// execute locally:
	cs := []string{"/bin/sh", "-c", cmdLine}
	cmd := exec.Command(cs[0], cs[1:]...)
	cmd.Stdin = nil
	cmd.Stdout = loggerOut
	cmd.Stderr = loggerErr
	cmd.Env = append(
		os.Environ(),
		fmt.Sprintf("PARALLEL_TICKER=%d", ticket),
	)

	fmt.Fprintf(loggerOut, "run: '"+cmdLine+"'\n")

	err = cmd.Start()
	if err != nil {
		log.Fatalln("failed to start:", err)
		return output, err
	}

	if err == nil {
		err = cmd.Wait()
	}

	output.Tags = map[string]string{"hostname": loggerHostname}

	if loggerOut.buf != nil {
		output.Stdout = string(loggerOut.buf.Bytes())
	}

	if loggerErr.buf != nil {
		output.Stderr = string(loggerErr.buf.Bytes())
	}

	return output, err
}

type Slave struct {
	Address string `json:"address"`
}

type Parallel struct {
	jobs   int
	logger *logger
	worker *limiter.ConcurrencyLimiter

	// master / slave
	protocolFactory  thrift.TProtocolFactory
	transportFactory thrift.TTransportFactory

	// master:
	Slaves []*Slave

	// slave:
	handler         *ParallelSlaveHandler
	serverTransport thrift.TServerTransport
	address         string
}

func (p *Parallel) Close() {
	p.worker.Wait()
}

func mainMaster(p *Parallel) {
	var err error

	// connect to slaves:
	for _, slave := range p.Slaves {
		var transport thrift.TTransport
		transport, err = thrift.NewTSocket(slave.Address)
		if err != nil {
			log.Fatalln("failed to dial slave:", err.Error())
		}

		transport, err = p.transportFactory.GetTransport(transport)
		if err != nil {
			log.Fatalln("failed to open:", err.Error())
		}

		err = transport.Open()
		if err != nil {
			log.Fatalln("failed to open:", err.Error())
		}

		defer transport.Close()

		client := parallel.NewParallelClientFactory(transport, p.protocolFactory)
		ok, err := client.Ping()
		if err != nil {
			log.Fatalln("failed to ping client:", err.Error())
		}

		fmt.Fprintf(p.logger, fmt.Sprintf("adding slave: %s ok: %s", slave.Address, ok))

	}

	r := bufio.NewReaderSize(os.Stdin, 1*1024*1024)
	fmt.Fprintf(p.logger, "reading from stdin...\n")
	for {
		line, err := r.ReadString('\n')
		if err == io.EOF {
			break
		}
		line = strings.TrimSpace(line)

		p.worker.ExecuteWithTicket(func(ticket int) {
			executeCommand(p, ticket, line)
		})
	}
}

type ParallelSlaveHandler struct {
	p *Parallel
}

func NewParallelSlaveHandler() *ParallelSlaveHandler {
	return &ParallelSlaveHandler{}
}

func (p *ParallelSlaveHandler) Execute(command *parallel.Cmd) (output *parallel.Output, err error) {
	output = nil
	err = nil
	output, err = executeCommand(p.p, int(command.Ticket), command.CmdLine)

	// TODO:: recover, handle panics

	return output, err
}

func (p *ParallelSlaveHandler) Ping() (r string, err error) {
	log.Println("ParallelSlaveHandler: Ping")
	return "ping:ok", nil
}

func mainSlave(p *Parallel) {
	var err error

	p.serverTransport, err = thrift.NewTServerSocket(p.address)

	if err != nil {
		log.Fatalln("failed to start server:", err.Error())
		return
	}

	p.handler = NewParallelSlaveHandler()
	p.handler.p = p

	server := thrift.NewTSimpleServer4(
		parallel.NewParallelProcessor(p.handler),
		p.serverTransport,
		p.transportFactory,
		p.protocolFactory)

	err = server.Serve()
	if err != nil {
		log.Fatalln("failed to run slave:", err.Error())
	}
}

func main() {
	T_START := time.Now()
	logger := newLogger(0, false)
	defer func() {
		fmt.Fprintf(logger, "all done: dt: "+time.Since(T_START).String()+"\n")
	}()

	flag_jobs := flag.Int(
		"j",
		2,
		"num of concurrent jobs")

	flag_slave := flag.Bool(
		"slave",
		false,
		"run as slave")

	slaves := flag.String(
		"slaves",
		"",
		"CSV list of slave addresses")

	address := flag.String(
		"address",
		"localhost:9010",
		"slave address")

	loggerHostname, _ = os.Hostname()

	flag.Parse()
	fmt.Fprintf(logger, fmt.Sprintf("concurrency limit: %d", *flag_jobs))
	fmt.Fprintf(logger, fmt.Sprintf("slaves: %s", *slaves))

	p := Parallel{}
	p.jobs = *flag_jobs
	p.logger = logger
	p.worker = limiter.NewConcurrencyLimiter(p.jobs)
	p.address = *address
	p.Slaves = []*Slave{}
	for _, slaveAddr := range strings.Split(*slaves, ",") {
		if slaveAddr == "" {
			continue
		}
		slave := Slave{Address: slaveAddr}
		p.Slaves = append(p.Slaves, &slave)
	}

	defer p.Close()

	// thrift protocol
	p.protocolFactory = thrift.NewTBinaryProtocolFactoryDefault()

	// thrift transport
	p.transportFactory = thrift.NewTTransportFactory()
	p.transportFactory = thrift.NewTFramedTransportFactory(p.transportFactory)

	if *flag_slave == false {
		loggerHostname = p.address
		logger.hostname = loggerHostname

		fmt.Fprintf(logger, fmt.Sprintf("running as master\n"))
		mainMaster(&p)
	} else {
		loggerHostname = p.address
		logger.hostname = loggerHostname

		fmt.Fprintf(logger, fmt.Sprintf("running as slave on: %s\n", p.address))
		mainSlave(&p)
	}
}
<|endoftext|>


<commit_before>package cmd

import (
	"log"
	"io/ioutil"
	"os"
	"fmt"
	"archive/zip"
	"github.com/gosuri/uilive"
	"time"
	"strings"
	"github.com/fatih/color"
	"path/filepath"
	"gopkg.in/yaml.v2"
)

var allResFiles = map[string]bool{"LICENSE.txt":true, "NOT_A_CONTRIBUTION.txt":true, "README.txt":true,
	"update-descriptor.yaml":true, "instructions.txt":true}

var (
	patchFilesMap map[string]bool
	distFileMap map[string]bool
	addedFilesMap map[string]bool
)

func Validate(patchLocation, distributionLocation string, logsEnabled bool) {

	patchFilesMap = make(map[string]bool)
	distFileMap = make(map[string]bool)
	addedFilesMap = make(map[string]bool)

	if (!logsEnabled) {
		log.SetOutput(ioutil.Discard)
	} else {
		log.Println("Logs enabled")
	}
	log.Println("validate command called")

	log.Println("Patch Loc: " + patchLocation)
	if !isAZipFile(patchLocation) {
		fmt.Println("Patch file should be a zip file.")
		os.Exit(1)
	}
	patchLocationExists := checkFile(patchLocation)
	if patchLocationExists {
		log.Println("Patch location exists.")
	} else {
		color.Set(color.FgRed)
		fmt.Println("Patch location does not exist. Enter a valid file location.")
		color.Unset()
		os.Exit(1)
	}

	log.Println("Reading patch zip...")
	readPatchZip(patchLocation, logsEnabled)
	log.Println("Patch zip successfully read.")
	log.Println("Entries in patch zip: ", patchFilesMap)

	log.Println("Distribution Loc: " + distributionLocation)
	if isAZipFile(distributionLocation) {
		zipFileExists := checkFile(distributionLocation)
		if zipFileExists {
			log.Println("Distribution location exists.")
			readDistZip(distributionLocation, logsEnabled)
		} else {
			color.Set(color.FgRed)
			fmt.Println("Distribution zip does not exist. Enter a valid location.")
			color.Unset()
			os.Exit(1)
		}
	} else {
		distributionLocationExists := checkDir(distributionLocation)
		if distributionLocationExists {
			log.Println("Distribution location exists.")
			readDistDir(distributionLocation, logsEnabled)
		} else {
			color.Set(color.FgRed)
			fmt.Println("Distribution location does not exist. Enter a valid location.")
			color.Unset()
			os.Exit(1)
		}
	}
	validate()
}

func validate() {
	for patchLoc, _ := range patchFilesMap {
		log.Println("Checking location: ", patchLoc)
		_, found := distFileMap[patchLoc]
		if found {
			log.Println(patchLoc, "found in distFileMap")
		} else {
			log.Println(patchLoc, "not found in distFileMap")
			_, found := addedFilesMap[patchLoc]
			if found {
				log.Println(patchLoc, "found in addedFilesMap")
			} else {
				log.Println(patchLoc, "not found in addedFilesMap")
				log.Println("addedFilesMap: ", addedFilesMap)
				color.Set(color.FgRed)
				fmt.Println(patchLoc, "not found in distribution and it is not a newly added file.")
				fmt.Println("If it is a new file, please check the entry in", _DESCRIPTOR_YAML_NAME,
					"file")
				fmt.Println("\nValidation FAILED\n")
				color.Unset()
				os.Exit(1)
			}
		}
	}
	fmt.Println("\nValidation SUCCESSFUL\n")
}

//This function reads the files of the given patch zip
func readPatchZip(zipLocation string, logsEnabled bool) (bool, error) {
	log.Println("Zip file reading started: ", zipLocation)

	patchName := strings.TrimSuffix(zipLocation, ".zip")
	if lastIndex := strings.LastIndex(patchName, string(os.PathSeparator)); lastIndex > -1 {
		patchName = patchName[lastIndex + 1:]
	}

	log.Println("Patch name: ", patchName)

	if !strings.HasPrefix(patchName, _PATCH_NAME_PREFIX) {
		fmt.Println("Patch file does not have", _PATCH_NAME_PREFIX, "prefix")
		os.Exit(0)
	} else {
		log.Println("Patch file does have", _PATCH_NAME_PREFIX, "prefix")
	}

	// Create a reader out of the zip archive
	zipReader, err := zip.OpenReader(zipLocation)

	if err != nil {
		fmt.Println(err)
		log.Fatal(err)
	}
	defer zipReader.Close()

	totalFiles := len(zipReader.Reader.File)
	log.Println("File count in zip: ", totalFiles)

	filesRead := 0

	writer := uilive.New()
	//start listening for updates and render
	writer.Start()

	// Iterate through each file/dir found in
	for _, file := range zipReader.Reader.File {
		filesRead++
		if (!logsEnabled) {
			fmt.Fprintf(writer, "Reading files from patch zip: (%d/%d)\n", filesRead, totalFiles)
			time.Sleep(time.Millisecond * 5)
		}

		log.Println("Checking file: ", file.Name)
		_, found := allResFiles[file.FileInfo().Name()]

		if !found {
			containsCarbonHome := strings.Contains(file.Name, _CARBON_HOME)
			if (!containsCarbonHome) {
				fmt.Println("Does not have a", _CARBON_HOME, "folder")
				os.Exit(1)
			}
			log.Println("Have a", _CARBON_HOME, "folder")

			temp := strings.TrimPrefix(file.Name, patchName + string(os.PathSeparator) + _CARBON_HOME)
			log.Println("Entry: ", temp)
			patchFilesMap[temp] = true
		} else {
			log.Println(file.FileInfo().Name(), "was found in resource map")
			delete(allResFiles, file.FileInfo().Name())
			log.Println(file.FileInfo().Name(), "was removed from the map")

			if file.FileInfo().Name() == _DESCRIPTOR_YAML_NAME {
				yamlFile, err := file.Open()
				if err != nil {
					color.Set(color.FgRed)
					fmt.Println("Error occurred while reading the", _DESCRIPTOR_YAML_NAME, "file:",
						err)
					color.Unset()
				}
				data, err := ioutil.ReadAll(yamlFile)
				if err != nil {
					log.Fatal(err)
				}
				descriptor := update_descriptor{}

				err = yaml.Unmarshal(data, &descriptor)
				if err != nil {
					color.Set(color.FgRed)
					fmt.Println("Error occurred while unmarshalling the yaml:", err)
					color.Unset()
				}

				log.Println("descriptor:", descriptor)

				for _, addedFile := range descriptor.File_changes.Added_files {

					addedFilesMap[addedFile] = true
					//_, found := allResFiles[addedFile]
					//if found {
					//	log.Println(file, "found in patch")
					//
					//} else {
					//	fmt.Println(file, "not found in patch. But it is added as an entry in" +
					//	"'added_file' section in", _DESCRIPTOR_YAML_NAME)
					//	os.Exit(1)
					//}
				}
			}
		}
	}

	log.Println("Resource map:", allResFiles)
	log.Println(patchFilesMap)
	if (len(allResFiles) != 0) {
		writer.Stop()
		fmt.Println("Following resource file(s) were not found in the patch: ")
		for key, _ := range allResFiles {
			fmt.Println("\t", key)
		}
		os.Exit(1)
	}

	writer.Stop()
	log.Println("Zip file reading finished")
	log.Println("Total files read: ", filesRead)
	if totalFiles == filesRead {
		log.Println("All files read")
		return true, nil
	} else {
		color.Set(color.FgRed)
		fmt.Println("All files not read from zip file")
		color.Unset()
		os.Exit(1)
		return false, nil
	}
}

//This function reads the files of the given patch zip
func readDistZip(zipLocation string, logsEnabled bool) (bool, error) {
	log.Println("Zip file reading started: ", zipLocation)

	distName := strings.TrimSuffix(zipLocation, ".zip")
	if lastIndex := strings.LastIndex(distName, string(os.PathSeparator)); lastIndex > -1 {
		distName = distName[lastIndex + 1:]
	}
	//
	//log.Println("Patch name: ", patchName)
	//
	//if !strings.HasPrefix(patchName, _PATCH_NAME_PREFIX) {
	//	fmt.Println("Patch file does not have", _PATCH_NAME_PREFIX, "prefix")
	//	os.Exit(0)
	//} else {
	//	log.Println("Patch file does have", _PATCH_NAME_PREFIX, "prefix")
	//}

	// Create a reader out of the zip archive
	zipReader, err := zip.OpenReader(zipLocation)

	if err != nil {
		fmt.Println(err)
		log.Fatal(err)
	}
	defer zipReader.Close()

	totalFiles := len(zipReader.Reader.File)
	log.Println("File count in zip: ", totalFiles)

	filesRead := 0

	writer := uilive.New()
	//start listening for updates and render
	writer.Start()

	// Iterate through each file/dir found in
	for _, file := range zipReader.Reader.File {
		filesRead++
		if (!logsEnabled) {
			fmt.Fprintf(writer, "Reading files from distribution zip: (%d/%d)\n", filesRead, totalFiles)
			time.Sleep(time.Millisecond * 2)
		}

		log.Println("Checking file: ", file.Name)
		//_, found := allResFiles[file.FileInfo().Name()]

		//if !found {
		//	containsCarbonHome := strings.Contains(file.Name, _CARBON_HOME)
		//	if (!containsCarbonHome) {
		//		fmt.Println("Does not have a", _CARBON_HOME, "folder")
		//		os.Exit(1)
		//	}
		//	log.Println("Have a", _CARBON_HOME, "folder")

		temp := strings.TrimPrefix(file.Name, distName)
		log.Println("Entry: ", temp)
		distFileMap[temp] = true
		//} else {
		//	log.Println(file.FileInfo().Name(), "was found in resource map")
		//	delete(allResFiles, file.FileInfo().Name())
		//	log.Println(file.FileInfo().Name(), "was removed from the map")
		//}
	}

	//log.Println("Resource map:", allResFiles)
	//log.Println(patchFilesMap)
	//if (len(allResFiles) != 0) {
	//	writer.Stop()
	//	fmt.Println("Following resource file(s) were not found in the patch: ")
	//	for key, _ := range allResFiles {
	//		fmt.Println("\t", key)
	//	}
	//	os.Exit(1)
	//}

	writer.Stop()
	log.Println("Zip file reading finished")
	log.Println("Total files read: ", filesRead)
	if totalFiles == filesRead {
		log.Println("All files read")
		return true, nil
	} else {
		color.Set(color.FgRed)
		fmt.Println("All files not read from zip file")
		color.Unset()
		os.Exit(1)
		return false, nil
	}
}
func readDistDir(distributionLocation string, logsEnabled bool) {

	writer := uilive.New()
	//start listening for updates and render
	writer.Start()
	filesRead := 0
	err := filepath.Walk(distributionLocation, func(path string, fileInfo os.FileInfo, err error) error {
		filesRead++;
		if (!logsEnabled) {
			fmt.Fprintf(writer, "Reading files from distribution zip: %d files read\n", filesRead)
			time.Sleep(time.Millisecond * 2)
		}

		log.Println("Walking: ", path)

		//if !fileInfo.IsDir() {
		// Create and write files to the archive, which in turn
		// are getting written to the underlying writer to the
		// .zip file we created at the beginning

		//header, err := zip.FileInfoHeader(fileInfo)
		//if err != nil {
		//	color.Set(color.FgRed)
		//	fmt.Println("Error occurred while creating the zip file: ", err)
		//	return err
		//	color.Unset()
		//}
		//
		//header.Name = _PATCH_NAME + string(os.PathSeparator) + strings.TrimPrefix(path, _TEMP_DIR_NAME + string(os.PathSeparator))

		temp := strings.TrimPrefix(path, distributionLocation)
		log.Println("Entry: ", temp)
		distFileMap[temp] = true

		//}
		//log.Printf("Visited: %s\n", path)
		return nil
	})

	if err != nil {
		color.Set(color.FgRed)
		fmt.Println("Error occurred while reading the zip file: ", err)
		os.Exit(1)
		color.Unset()
	}
	log.Println("Total files read:", filesRead)
	writer.Stop()
}
<commit_msg>instructions.txt file added as optional<commit_after>package cmd

import (
	"log"
	"io/ioutil"
	"os"
	"fmt"
	"archive/zip"
	"github.com/gosuri/uilive"
	"time"
	"strings"
	"github.com/fatih/color"
	"path/filepath"
	"gopkg.in/yaml.v2"
)

var (
	allResFiles  map[string]bool
	patchFilesMap map[string]bool
	distFileMap map[string]bool
	addedFilesMap map[string]bool
)

func Validate(patchLocation, distributionLocation string, logsEnabled bool) {

	if (!logsEnabled) {
		log.SetOutput(ioutil.Discard)
	} else {
		log.Println("Logs enabled")
	}
	log.Println("validate command called")

	initialize()

	log.Println("Patch Loc: " + patchLocation)
	if !isAZipFile(patchLocation) {
		color.Set(color.FgRed)
		fmt.Println("Patch file should be a zip file.")
		color.Unset()
		os.Exit(1)
	}
	patchLocationExists := checkFile(patchLocation)
	if patchLocationExists {
		log.Println("Patch location exists.")
	} else {
		color.Set(color.FgRed)
		fmt.Println("Patch location does not exist. Enter a valid file location.")
		color.Unset()
		os.Exit(1)
	}

	log.Println("Reading patch zip...")
	readPatchZip(patchLocation, logsEnabled)
	log.Println("Patch zip successfully read.")
	log.Println("Entries in patch zip: ", patchFilesMap)

	log.Println("Distribution Loc: " + distributionLocation)
	if isAZipFile(distributionLocation) {
		zipFileExists := checkFile(distributionLocation)
		if zipFileExists {
			log.Println("Distribution location exists.")
			readDistZip(distributionLocation, logsEnabled)
		} else {
			color.Set(color.FgRed)
			fmt.Println("Distribution zip does not exist. Enter a valid location.")
			color.Unset()
			os.Exit(1)
		}
	} else {
		distributionLocationExists := checkDir(distributionLocation)
		if distributionLocationExists {
			log.Println("Distribution location exists.")
			readDistDir(distributionLocation, logsEnabled)
		} else {
			color.Set(color.FgRed)
			fmt.Println("Distribution location does not exist. Enter a valid location.")
			color.Unset()
			os.Exit(1)
		}
	}
	validate()
}

func initialize() {
	allResFiles = make(map[string]bool)
	allResFiles[_LICENSE_FILE_NAME] = true
	allResFiles[_NOT_A_CONTRIBUTION_FILE_NAME] = true
	allResFiles[_README_FILE_NAME] = true
	allResFiles[_UPDATE_DESCRIPTOR_FILE_NAME] = true
	allResFiles[_INSTRUCTIONS_FILE_NAME] = true

	patchFilesMap = make(map[string]bool)
	distFileMap = make(map[string]bool)
	addedFilesMap = make(map[string]bool)
}

func validate() {
	for patchLoc, _ := range patchFilesMap {
		log.Println("Checking location: ", patchLoc)
		_, found := distFileMap[patchLoc]
		if found {
			log.Println(patchLoc, "found in distFileMap")
		} else {
			log.Println(patchLoc, "not found in distFileMap")
			_, found := addedFilesMap[patchLoc]
			if found {
				log.Println(patchLoc, "found in addedFilesMap")
			} else {
				log.Println(patchLoc, "not found in addedFilesMap")
				log.Println("addedFilesMap: ", addedFilesMap)
				color.Set(color.FgRed)
				fmt.Println(patchLoc, "not found in distribution and it is not a newly added file.")
				fmt.Println("If it is a new file, please add an entry in", _UPDATE_DESCRIPTOR_FILE_NAME,
					"file")
				fmt.Println("\nValidation FAILED\n")
				color.Unset()
				os.Exit(1)
			}
		}
	}
	fmt.Println("\nValidation SUCCESSFUL\n")
}

//This function reads the files of the given patch zip
func readPatchZip(zipLocation string, logsEnabled bool) (bool, error) {
	log.Println("Zip file reading started: ", zipLocation)

	patchName := strings.TrimSuffix(zipLocation, ".zip")
	if lastIndex := strings.LastIndex(patchName, string(os.PathSeparator)); lastIndex > -1 {
		patchName = patchName[lastIndex + 1:]
	}

	log.Println("Patch name: ", patchName)

	if !strings.HasPrefix(patchName, _UPDATE_NAME_PREFIX) {
		color.Set(color.FgRed)
		fmt.Println("Patch file does not have", _UPDATE_NAME_PREFIX, "prefix")
		color.Unset()
		os.Exit(1)
	} else {
		log.Println("Patch file does have", _UPDATE_NAME_PREFIX, "prefix")
	}

	// Create a reader out of the zip archive
	zipReader, err := zip.OpenReader(zipLocation)

	if err != nil {
		color.Set(color.FgRed)
		fmt.Println("Error occurred while reading zip:", err)
		color.Unset()
		log.Fatal(err)
	}
	defer zipReader.Close()

	totalFiles := len(zipReader.Reader.File)
	log.Println("File count in zip: ", totalFiles)

	filesRead := 0

	writer := uilive.New()
	//start listening for updates and render
	writer.Start()

	// Iterate through each file/dir found in
	for _, file := range zipReader.Reader.File {
		filesRead++
		if (!logsEnabled) {
			fmt.Fprintf(writer, "Reading files from patch zip: (%d/%d)\n", filesRead, totalFiles)
			time.Sleep(time.Millisecond * 5)
		}

		log.Println("Checking file: ", file.Name)

		index := strings.Index(file.Name, string(os.PathSeparator))

		if index == -1 {
			color.Set(color.FgRed)
			fmt.Println("Patch zip file should have a root folder called", patchName)
			color.Unset()
			os.Exit(1)
		} else {
			rootFolder := file.Name[:index]
			log.Println("RootFolder:", rootFolder)
			if rootFolder != patchName {
				color.Set(color.FgRed)
				fmt.Println(file.Name, "should be in", patchName, "root folder. But it is in ",
					rootFolder, "folder")
				color.Unset()
				os.Exit(1)
			}
		}

		_, found := allResFiles[file.FileInfo().Name()]

		if !found {
			containsCarbonHome := strings.Contains(file.Name, _CARBON_HOME)
			if (!containsCarbonHome) {
				color.Set(color.FgRed)
				fmt.Println("Does not have a '" + _CARBON_HOME + "' folder in the root folder '" +
				patchName + "'")
				color.Unset()
				os.Exit(1)
			}
			log.Println("Have a", _CARBON_HOME, "folder")

			temp := strings.TrimPrefix(file.Name, patchName + string(os.PathSeparator) + _CARBON_HOME)
			log.Println("Entry: ", temp)
			patchFilesMap[temp] = true
		} else {
			log.Println(file.FileInfo().Name(), "was found in resource map")
			delete(allResFiles, file.FileInfo().Name())
			log.Println(file.FileInfo().Name(), "was removed from the map")

			if file.FileInfo().Name() == _UPDATE_DESCRIPTOR_FILE_NAME {
				yamlFile, err := file.Open()
				if err != nil {
					color.Set(color.FgRed)
					fmt.Println("Error occurred while reading the", _UPDATE_DESCRIPTOR_FILE_NAME, "file:",
						err)
					color.Unset()
				}
				data, err := ioutil.ReadAll(yamlFile)
				if err != nil {
					log.Fatal(err)
				}
				descriptor := update_descriptor{}

				err = yaml.Unmarshal(data, &descriptor)
				if err != nil {
					color.Set(color.FgRed)
					fmt.Println("Error occurred while unmarshalling the yaml:", err)
					color.Unset()
				}

				log.Println("descriptor:", descriptor)

				for _, addedFile := range descriptor.File_changes.Added_files {

					addedFilesMap[addedFile] = true
					//_, found := allResFiles[addedFile]
					//if found {
					//	log.Println(file, "found in patch")
					//
					//} else {
					//	fmt.Println(file, "not found in patch. But it is added as an entry in" +
					//	"'added_file' section in", _DESCRIPTOR_YAML_NAME)
					//	os.Exit(1)
					//}
				}
			}
		}
	}

	//Delete instructions.txt file if it is left in the map
	delete(allResFiles,_INSTRUCTIONS_FILE_NAME)

	log.Println("Resource map:", allResFiles)
	log.Println(patchFilesMap)
	if (len(allResFiles) != 0) {
		writer.Stop()
		color.Set(color.FgRed)
		fmt.Println("Following resource file(s) were not found in the patch: ")
		for key, _ := range allResFiles {
			fmt.Println("\t", "-", key)
		}
		color.Unset()
		os.Exit(1)
	}

	writer.Stop()
	log.Println("Zip file reading finished")
	log.Println("Total files read: ", filesRead)
	if totalFiles == filesRead {
		log.Println("All files read")
		return true, nil
	} else {
		color.Set(color.FgRed)
		fmt.Println("All files not read from zip file")
		color.Unset()
		os.Exit(1)
		return false, nil
	}
}

//This function reads the files of the given distribution zip
func readDistZip(zipLocation string, logsEnabled bool) (bool, error) {
	log.Println("Zip file reading started: ", zipLocation)

	distName := strings.TrimSuffix(zipLocation, ".zip")
	if lastIndex := strings.LastIndex(distName, string(os.PathSeparator)); lastIndex > -1 {
		distName = distName[lastIndex + 1:]
	}

	// Create a reader out of the zip archive
	zipReader, err := zip.OpenReader(zipLocation)

	if err != nil {
		color.Set(color.FgRed)
		fmt.Println("Error occurred while reading zip:", err)
		color.Unset()
		log.Fatal(err)
	}
	defer zipReader.Close()

	totalFiles := len(zipReader.Reader.File)
	log.Println("File count in zip: ", totalFiles)

	filesRead := 0

	writer := uilive.New()
	//start listening for updates and render
	writer.Start()

	// Iterate through each file/dir found in
	for _, file := range zipReader.Reader.File {
		filesRead++
		if (!logsEnabled) {
			fmt.Fprintf(writer, "Reading files from distribution zip: (%d/%d)\n", filesRead, totalFiles)
			time.Sleep(time.Millisecond * 2)
		}

		log.Println("Checking file: ", file.Name)

		temp := strings.TrimPrefix(file.Name, distName)
		log.Println("Entry: ", temp)
		distFileMap[temp] = true

	}

	writer.Stop()
	log.Println("Zip file reading finished")
	log.Println("Total files read: ", filesRead)
	if totalFiles == filesRead {
		log.Println("All files read")
		return true, nil
	} else {
		color.Set(color.FgRed)
		fmt.Println("All files not read from zip file")
		color.Unset()
		os.Exit(1)
		return false, nil
	}
}

func readDistDir(distributionLocation string, logsEnabled bool) {

	writer := uilive.New()
	//start listening for updates and render
	writer.Start()
	filesRead := 0
	err := filepath.Walk(distributionLocation, func(path string, fileInfo os.FileInfo, err error) error {
		filesRead++;
		if (!logsEnabled) {
			fmt.Fprintf(writer, "Reading files from distribution directory: %d files read\n", filesRead)
			time.Sleep(time.Millisecond * 2)
		}

		log.Println("Walking: ", path)

		//if !fileInfo.IsDir() {
		// Create and write files to the archive, which in turn
		// are getting written to the underlying writer to the
		// .zip file we created at the beginning

		//header, err := zip.FileInfoHeader(fileInfo)
		//if err != nil {
		//	color.Set(color.FgRed)
		//	fmt.Println("Error occurred while creating the zip file: ", err)
		//	return err
		//	color.Unset()
		//}
		//
		//header.Name = _PATCH_NAME + string(os.PathSeparator) + strings.TrimPrefix(path, _TEMP_DIR_NAME + string(os.PathSeparator))

		temp := strings.TrimPrefix(path, distributionLocation)
		log.Println("Entry: ", temp)
		distFileMap[temp] = true

		//}
		//log.Printf("Visited: %s\n", path)
		return nil
	})

	if err != nil {
		color.Set(color.FgRed)
		fmt.Println("Error occurred while reading the zip file: ", err)
		os.Exit(1)
		color.Unset()
	}
	log.Println("Total files read:", filesRead)
	writer.Stop()
}
<|endoftext|>