<commit_before>package main

import (
	"expvar"
	"flag"
	"fmt"
	"log"
	"net/http"
	_ "net/http/pprof"
	"os"
	"os/signal"
	"time"

	"github.com/thecodearchive/gitarchive/camli"
	"github.com/thecodearchive/gitarchive/github"
	"github.com/thecodearchive/gitarchive/queue"
)

func main() {
	go func() {
		log.Println(http.ListenAndServe("localhost:6060", nil))
	}()

	queuePath := flag.String("queue", "./queue.db", "clone queue path")
	cachePath := flag.String("cache", "./cache.json", "startracker cache path")
	camli.AddFlags()
	flag.Parse()
	if flag.NArg() < 1 {
		log.Fatal("usage: drink 2016-01-02-15")
	}

	if os.Getenv("GITHUB_TOKEN") == "" {
		log.Fatal("Please set the env var GITHUB_TOKEN")
	}
	st := github.NewStarTracker(1000000000, os.Getenv("GITHUB_TOKEN"))

	if f, err := os.Open(*cachePath); err != nil {
		log.Println("[ ] Can't load StarTracker cache, starting empty")
	} else {
		log.Println("[+] Loaded StarTracker cache")
		st.LoadCache(f)
		f.Close()
	}

	log.Println("[ ] Opening queue...")
	q, err := queue.Open(*queuePath)
	fatalIfErr(err)

	defer func() {
		log.Println("[ ] Closing queue...")
		fatalIfErr(q.Close())

		f, err := os.Create(*cachePath)
		fatalIfErr(err)
		log.Println("[ ] Writing StarTracker cache...")
		st.SaveCache(f)
		fatalIfErr(f.Close())
	}()

	exp := expvar.NewMap("drink")
	expEvents := new(expvar.Map).Init()
	expLatest := new(expvar.String)
	exp.Set("latestevent", expLatest)
	exp.Set("events", expEvents)
	exp.Set("github", st.Expvar())

	d := &Drinker{
		q: q, st: st, u: camli.NewUploader(),
		exp: exp, expEvents: expEvents, expLatest: expLatest,
	}

	c := make(chan os.Signal, 1)
	signal.Notify(c, os.Interrupt)
	go func() {
		<-c
		log.Println("[ ] Stopping gracefully...")
		d.Stop()
	}()

	t, err := time.Parse(github.HourFormat, flag.Arg(0))
	fatalIfErr(err)

	startTime := t.Add(time.Hour).Add(2 * time.Minute)
	for {
		if time.Now().Before(startTime) {
			log.Printf("[ ] Waiting for the %s archive until %s...",
				t.Format(github.HourFormat), startTime)
			if !interruptableSleep(startTime.Sub(time.Now())) {
				break
			}
		}
		log.Println("[ ] Opening archive download...")
		a, err := github.DownloadArchive(t)
		fatalIfErr(err) // TODO: make more graceful
		if a == nil {
			exp.Add("archives404", 1)
			startTime = startTime.Add(2 * time.Minute)
			continue
		}

		log.Printf("[+] Archive %s found, consuming...", t.Format(github.HourFormat))
		err = d.DrinkArchive(a)
		a.Close()
		if err == StoppedError {
			break
		}
		fatalIfErr(err) // TODO: make more graceful

		exp.Add("archivesfinished", 1)
		t = t.Add(time.Hour)
		startTime = t.Add(time.Hour).Add(2 * time.Minute)
	}

	log.Println("[+] Processed events until", expLatest)
	fmt.Print(exp.String())
}

func fatalIfErr(err error) {
	if err != nil {
		log.Panic(err) // panic to let the defer run
	}
}
<commit_msg>drink: on 404s don't assume we had just finished waiting<commit_after>package main

import (
	"expvar"
	"flag"
	"fmt"
	"log"
	"net/http"
	_ "net/http/pprof"
	"os"
	"os/signal"
	"time"

	"github.com/thecodearchive/gitarchive/camli"
	"github.com/thecodearchive/gitarchive/github"
	"github.com/thecodearchive/gitarchive/queue"
)

func main() {
	go func() {
		log.Println(http.ListenAndServe("localhost:6060", nil))
	}()

	queuePath := flag.String("queue", "./queue.db", "clone queue path")
	cachePath := flag.String("cache", "./cache.json", "startracker cache path")
	camli.AddFlags()
	flag.Parse()
	if flag.NArg() < 1 {
		log.Fatal("usage: drink 2016-01-02-15")
	}

	if os.Getenv("GITHUB_TOKEN") == "" {
		log.Fatal("Please set the env var GITHUB_TOKEN")
	}
	st := github.NewStarTracker(1000000000, os.Getenv("GITHUB_TOKEN"))

	if f, err := os.Open(*cachePath); err != nil {
		log.Println("[ ] Can't load StarTracker cache, starting empty")
	} else {
		log.Println("[+] Loaded StarTracker cache")
		st.LoadCache(f)
		f.Close()
	}

	log.Println("[ ] Opening queue...")
	q, err := queue.Open(*queuePath)
	fatalIfErr(err)

	defer func() {
		log.Println("[ ] Closing queue...")
		fatalIfErr(q.Close())

		f, err := os.Create(*cachePath)
		fatalIfErr(err)
		log.Println("[ ] Writing StarTracker cache...")
		st.SaveCache(f)
		fatalIfErr(f.Close())
	}()

	exp := expvar.NewMap("drink")
	expEvents := new(expvar.Map).Init()
	expLatest := new(expvar.String)
	exp.Set("latestevent", expLatest)
	exp.Set("events", expEvents)
	exp.Set("github", st.Expvar())

	d := &Drinker{
		q: q, st: st, u: camli.NewUploader(),
		exp: exp, expEvents: expEvents, expLatest: expLatest,
	}

	c := make(chan os.Signal, 1)
	signal.Notify(c, os.Interrupt)
	go func() {
		<-c
		log.Println("[ ] Stopping gracefully...")
		d.Stop()
	}()

	t, err := time.Parse(github.HourFormat, flag.Arg(0))
	fatalIfErr(err)

	startTime := t.Add(time.Hour).Add(2 * time.Minute)
	for {
		if time.Now().Before(startTime) {
			log.Printf("[ ] Waiting for the %s archive until %s...",
				t.Format(github.HourFormat), startTime)
			if !interruptableSleep(startTime.Sub(time.Now())) {
				break
			}
		}
		log.Printf("[ ] Opening %s archive download...", t.Format(github.HourFormat))
		a, err := github.DownloadArchive(t)
		fatalIfErr(err) // TODO: make more graceful
		if a == nil {
			exp.Add("archives404", 1)
			startTime = time.Now().Add(2 * time.Minute)
			continue
		}

		log.Printf("[+] Archive %s found, consuming...", t.Format(github.HourFormat))
		err = d.DrinkArchive(a)
		a.Close()
		if err == StoppedError {
			break
		}
		fatalIfErr(err) // TODO: make more graceful

		exp.Add("archivesfinished", 1)
		t = t.Add(time.Hour)
		startTime = t.Add(time.Hour).Add(2 * time.Minute)
	}

	log.Println("[+] Processed events until", expLatest)
	fmt.Print(exp.String())
}

func fatalIfErr(err error) {
	if err != nil {
		log.Panic(err) // panic to let the defer run
	}
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"io/ioutil"
)

// Flags
var (
	inPath  = flag.String("i", "", "input file path")
	outPath = flag.String("o", "", "output file path")
)

func init() {
	flag.Parse()
}

func main() {
	in, err := ioutil.ReadFile(*inPath)
	if err != nil {
		panic(err)
	}

	fmt.Println(string(in))
}
<commit_msg>Update cmd/esgen/main.go<commit_after>package main

import (
	"encoding/json"
	"flag"
	"io/ioutil"
	"os"
)

// property represents property for each field.
type property struct {
	Type   string
	Length int
	Prefix string
	Value  string
}

// config represents configuration for the processing.
type config struct {
	Action string
	Index  string
	Type   string
	Num    int
	Props  map[string]property
}

// Flags
var (
	inPath  = flag.String("i", "", "input file path")
	outPath = flag.String("o", "", "output file path")
)

func init() {
	flag.Parse()
}

func main() {
	in, err := ioutil.ReadFile(*inPath)
	if err != nil {
		panic(err)
	}

	var conf config
	if err := json.Unmarshal(in, &conf); err != nil {
		panic(err)
	}

	f, err := os.Create(*outPath)
	if err != nil {
		panic(err)
	}

	defer f.Close()

	for i := 0; i < conf.Num; i++ {
		meta := make(map[string]string)

		meta["_index"] = conf.Index
		meta["_type"] = conf.Type
		meta["_id"] = conf.Props["_id"].Value

		action := map[string]map[string]string{
			conf.Action: meta,
		}

		out, err := json.Marshal(action)
		if err != nil {
			panic(err)
		}

		if _, err := f.Write(out); err != nil {
			panic(err)
		}

		f.WriteString("\n")
	}
}
<|endoftext|>


<commit_before>// Copyright 2014 Volker Dobler.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"flag"
	"fmt"
)

var cmdVersion = &Command{
	RunArgs:     runVersion,
	Usage:       "version",
	Description: "print version information",
	Flag:        flag.NewFlagSet("version", flag.ContinueOnError),
	Help: `Version prints version information about ht.
`,
}

var (
	version = "5.2.0-beta"
)

func runVersion(cmd *Command, _ []string) {
	fmt.Printf("ht version %s\n", version)
}
<commit_msg>all: start development of version 5.3<commit_after>// Copyright 2014 Volker Dobler.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"flag"
	"fmt"
)

var cmdVersion = &Command{
	RunArgs:     runVersion,
	Usage:       "version",
	Description: "print version information",
	Flag:        flag.NewFlagSet("version", flag.ContinueOnError),
	Help: `Version prints version information about ht.
`,
}

var (
	version = "5.3.0-beta"
)

func runVersion(cmd *Command, _ []string) {
	fmt.Printf("ht version %s\n", version)
}
<|endoftext|>


<commit_before>package dbus

import (
	"bufio"
	"bytes"
	"errors"
	"io"
	"os"
	"strconv"
)

// AuthStatus represents the Status of an authentication mechanism.
type AuthStatus byte

const (
	// AuthOk signals that authentication is finished; the next command
	// from the server should be an OK.
	AuthOk AuthStatus = iota

	// AuthContinue signals that additional data is needed; the next command
	// from the server should be a DATA.
	AuthContinue

	// AuthError signals an error; the server sent invalid data or some
	// other unexpected thing happened and the current authentication
	// process should be aborted.
	AuthError
)

type authState byte

const (
	waitingForData authState = iota
	waitingForOk
	waitingForReject
)

// Auth defines the behaviour of an authentication mechanism.
type Auth interface {
	// Return the name of the mechanism, the argument to the first AUTH command
	// and the next status.
	FirstData() (name, resp []byte, status AuthStatus)

	// Process the given DATA command, and return the argument to the DATA
	// command and the next status. If len(resp) == 0, no DATA command is sent.
	HandleData(data []byte) (resp []byte, status AuthStatus)
}

// Auth authenticates the connection, trying the given list of authentication
// mechanisms (in that order). If nil is passed, the EXTERNAL and
// DBUS_COOKIE_SHA1 mechanisms are tried for the current user. For private
// connections, this method must be called before sending any messages to the
// bus. Auth must not be called on shared connections.
func (conn *Conn) Auth(methods []Auth) error {
	if methods == nil {
		uid := strconv.Itoa(os.Geteuid())
		methods = []Auth{AuthExternal(uid), AuthCookieSha1(uid, getHomeDir())}
	}
	in := bufio.NewReader(conn.transport)
	err := conn.transport.SendNullByte()
	if err != nil {
		return err
	}
	err = authWriteLine(conn.transport, []byte("AUTH"))
	if err != nil {
		return err
	}
	s, err := authReadLine(in)
	if err != nil {
		return err
	}
	if len(s) < 2 || !bytes.Equal(s[0], []byte("REJECTED")) {
		return errors.New("dbus: authentication protocol error")
	}
	s = s[1:]
	for _, v := range s {
		for _, m := range methods {
			if name, _, status := m.FirstData(); bytes.Equal(v, name) {
				var ok bool
				err = authWriteLine(conn.transport, []byte("AUTH"), v)
				if err != nil {
					return err
				}
				switch status {
				case AuthOk:
					err, ok = conn.tryAuth(m, waitingForOk, in)
				case AuthContinue:
					err, ok = conn.tryAuth(m, waitingForData, in)
				default:
					panic("dbus: invalid authentication status")
				}
				if err != nil {
					return err
				}
				if ok {
					if conn.transport.SupportsUnixFDs() {
						err = authWriteLine(conn, []byte("NEGOTIATE_UNIX_FD"))
						if err != nil {
							return err
						}
						line, err := authReadLine(in)
						if err != nil {
							return err
						}
						switch {
						case bytes.Equal(line[0], []byte("AGREE_UNIX_FD")):
							conn.EnableUnixFDs()
							conn.unixFD = true
						case bytes.Equal(line[0], []byte("ERROR")):
						default:
							return errors.New("dbus: authentication protocol error")
						}
					}
					err = authWriteLine(conn.transport, []byte("BEGIN"))
					if err != nil {
						return err
					}
					go conn.inWorker()
					return nil
				}
			}
		}
	}
	return errors.New("dbus: authentication failed")
}

// tryAuth tries to authenticate with m as the mechanism, using state as the
// initial authState and in for reading input. It returns (nil, true) on
// success, (nil, false) on a REJECTED and (someErr, false) if some other
// error occurred.
func (conn *Conn) tryAuth(m Auth, state authState, in *bufio.Reader) (error, bool) {
	for {
		s, err := authReadLine(in)
		if err != nil {
			return err, false
		}
		switch {
		case state == waitingForData && string(s[0]) == "DATA":
			if len(s) != 2 {
				err = authWriteLine(conn.transport, []byte("ERROR"))
				if err != nil {
					return err, false
				}
				continue
			}
			data, status := m.HandleData(s[1])
			switch status {
			case AuthOk, AuthContinue:
				if len(data) != 0 {
					err = authWriteLine(conn.transport, []byte("DATA"), data)
					if err != nil {
						return err, false
					}
				}
				if status == AuthOk {
					state = waitingForOk
				}
			case AuthError:
				err = authWriteLine(conn.transport, []byte("ERROR"))
				if err != nil {
					return err, false
				}
			}
		case state == waitingForData && string(s[0]) == "REJECTED":
			return nil, false
		case state == waitingForData && string(s[0]) == "ERROR":
			err = authWriteLine(conn.transport, []byte("CANCEL"))
			if err != nil {
				return err, false
			}
			state = waitingForReject
		case state == waitingForData && string(s[0]) == "OK":
			if len(s) != 2 {
				err = authWriteLine(conn.transport, []byte("CANCEL"))
				if err != nil {
					return err, false
				}
				state = waitingForReject
			}
			conn.uuid = string(s[1])
			return nil, true
		case state == waitingForData:
			err = authWriteLine(conn.transport, []byte("ERROR"))
			if err != nil {
				return err, false
			}
		case state == waitingForOk && string(s[0]) == "OK":
			if len(s) != 2 {
				err = authWriteLine(conn.transport, []byte("CANCEL"))
				if err != nil {
					return err, false
				}
				state = waitingForReject
			}
			conn.uuid = string(s[1])
			return nil, true
		case state == waitingForOk && string(s[0]) == "DATA":
			err = authWriteLine(conn.transport, []byte("DATA"))
			if err != nil {
				return err, false
			}
		case state == waitingForOk && string(s[0]) == "REJECTED":
			return nil, false
		case state == waitingForOk && string(s[0]) == "ERROR":
			err = authWriteLine(conn.transport, []byte("CANCEL"))
			if err != nil {
				return err, false
			}
			state = waitingForReject
		case state == waitingForOk:
			err = authWriteLine(conn.transport, []byte("ERROR"))
			if err != nil {
				return err, false
			}
		case state == waitingForReject && string(s[0]) == "REJECTED":
			return nil, false
		case state == waitingForReject:
			return errors.New("dbus: authentication protocol error"), false
		default:
			panic("dbus: invalid auth state")
		}
	}
}

// authReadLine reads a line and separates it into its fields.
func authReadLine(in *bufio.Reader) ([][]byte, error) {
	data, err := in.ReadBytes('\n')
	if err != nil {
		return nil, err
	}
	data = bytes.TrimSuffix(data, []byte("\r\n"))
	return bytes.Split(data, []byte{' '}), nil
}

// authWriteLine writes the given line in the authentication protocol format
// (elements of data separated by a " " and terminated by "\r\n").
func authWriteLine(out io.Writer, data ...[]byte) error {
	buf := make([]byte, 0)
	for i, v := range data {
		buf = append(buf, v...)
		if i != len(data)-1 {
			buf = append(buf, ' ')
		}
	}
	buf = append(buf, '\r')
	buf = append(buf, '\n')
	n, err := out.Write(buf)
	if err != nil {
		return err
	}
	if n != len(buf) {
		return io.ErrUnexpectedEOF
	}
	return nil
}
<commit_msg>auth: handle improperly formatted OK correctly<commit_after>package dbus

import (
	"bufio"
	"bytes"
	"errors"
	"io"
	"os"
	"strconv"
)

// AuthStatus represents the Status of an authentication mechanism.
type AuthStatus byte

const (
	// AuthOk signals that authentication is finished; the next command
	// from the server should be an OK.
	AuthOk AuthStatus = iota

	// AuthContinue signals that additional data is needed; the next command
	// from the server should be a DATA.
	AuthContinue

	// AuthError signals an error; the server sent invalid data or some
	// other unexpected thing happened and the current authentication
	// process should be aborted.
	AuthError
)

type authState byte

const (
	waitingForData authState = iota
	waitingForOk
	waitingForReject
)

// Auth defines the behaviour of an authentication mechanism.
type Auth interface {
	// Return the name of the mechanism, the argument to the first AUTH command
	// and the next status.
	FirstData() (name, resp []byte, status AuthStatus)

	// Process the given DATA command, and return the argument to the DATA
	// command and the next status. If len(resp) == 0, no DATA command is sent.
	HandleData(data []byte) (resp []byte, status AuthStatus)
}

// Auth authenticates the connection, trying the given list of authentication
// mechanisms (in that order). If nil is passed, the EXTERNAL and
// DBUS_COOKIE_SHA1 mechanisms are tried for the current user. For private
// connections, this method must be called before sending any messages to the
// bus. Auth must not be called on shared connections.
func (conn *Conn) Auth(methods []Auth) error {
	if methods == nil {
		uid := strconv.Itoa(os.Geteuid())
		methods = []Auth{AuthExternal(uid), AuthCookieSha1(uid, getHomeDir())}
	}
	in := bufio.NewReader(conn.transport)
	err := conn.transport.SendNullByte()
	if err != nil {
		return err
	}
	err = authWriteLine(conn.transport, []byte("AUTH"))
	if err != nil {
		return err
	}
	s, err := authReadLine(in)
	if err != nil {
		return err
	}
	if len(s) < 2 || !bytes.Equal(s[0], []byte("REJECTED")) {
		return errors.New("dbus: authentication protocol error")
	}
	s = s[1:]
	for _, v := range s {
		for _, m := range methods {
			if name, _, status := m.FirstData(); bytes.Equal(v, name) {
				var ok bool
				err = authWriteLine(conn.transport, []byte("AUTH"), v)
				if err != nil {
					return err
				}
				switch status {
				case AuthOk:
					err, ok = conn.tryAuth(m, waitingForOk, in)
				case AuthContinue:
					err, ok = conn.tryAuth(m, waitingForData, in)
				default:
					panic("dbus: invalid authentication status")
				}
				if err != nil {
					return err
				}
				if ok {
					if conn.transport.SupportsUnixFDs() {
						err = authWriteLine(conn, []byte("NEGOTIATE_UNIX_FD"))
						if err != nil {
							return err
						}
						line, err := authReadLine(in)
						if err != nil {
							return err
						}
						switch {
						case bytes.Equal(line[0], []byte("AGREE_UNIX_FD")):
							conn.EnableUnixFDs()
							conn.unixFD = true
						case bytes.Equal(line[0], []byte("ERROR")):
						default:
							return errors.New("dbus: authentication protocol error")
						}
					}
					err = authWriteLine(conn.transport, []byte("BEGIN"))
					if err != nil {
						return err
					}
					go conn.inWorker()
					return nil
				}
			}
		}
	}
	return errors.New("dbus: authentication failed")
}

// tryAuth tries to authenticate with m as the mechanism, using state as the
// initial authState and in for reading input. It returns (nil, true) on
// success, (nil, false) on a REJECTED and (someErr, false) if some other
// error occurred.
func (conn *Conn) tryAuth(m Auth, state authState, in *bufio.Reader) (error, bool) {
	for {
		s, err := authReadLine(in)
		if err != nil {
			return err, false
		}
		switch {
		case state == waitingForData && string(s[0]) == "DATA":
			if len(s) != 2 {
				err = authWriteLine(conn.transport, []byte("ERROR"))
				if err != nil {
					return err, false
				}
				continue
			}
			data, status := m.HandleData(s[1])
			switch status {
			case AuthOk, AuthContinue:
				if len(data) != 0 {
					err = authWriteLine(conn.transport, []byte("DATA"), data)
					if err != nil {
						return err, false
					}
				}
				if status == AuthOk {
					state = waitingForOk
				}
			case AuthError:
				err = authWriteLine(conn.transport, []byte("ERROR"))
				if err != nil {
					return err, false
				}
			}
		case state == waitingForData && string(s[0]) == "REJECTED":
			return nil, false
		case state == waitingForData && string(s[0]) == "ERROR":
			err = authWriteLine(conn.transport, []byte("CANCEL"))
			if err != nil {
				return err, false
			}
			state = waitingForReject
		case state == waitingForData && string(s[0]) == "OK":
			if len(s) != 2 {
				err = authWriteLine(conn.transport, []byte("CANCEL"))
				if err != nil {
					return err, false
				}
				state = waitingForReject
			} else {
				conn.uuid = string(s[1])
				return nil, true
			}
		case state == waitingForData:
			err = authWriteLine(conn.transport, []byte("ERROR"))
			if err != nil {
				return err, false
			}
		case state == waitingForOk && string(s[0]) == "OK":
			if len(s) != 2 {
				err = authWriteLine(conn.transport, []byte("CANCEL"))
				if err != nil {
					return err, false
				}
				state = waitingForReject
			} else {
				conn.uuid = string(s[1])
				return nil, true
			}
		case state == waitingForOk && string(s[0]) == "DATA":
			err = authWriteLine(conn.transport, []byte("DATA"))
			if err != nil {
				return err, false
			}
		case state == waitingForOk && string(s[0]) == "REJECTED":
			return nil, false
		case state == waitingForOk && string(s[0]) == "ERROR":
			err = authWriteLine(conn.transport, []byte("CANCEL"))
			if err != nil {
				return err, false
			}
			state = waitingForReject
		case state == waitingForOk:
			err = authWriteLine(conn.transport, []byte("ERROR"))
			if err != nil {
				return err, false
			}
		case state == waitingForReject && string(s[0]) == "REJECTED":
			return nil, false
		case state == waitingForReject:
			return errors.New("dbus: authentication protocol error"), false
		default:
			panic("dbus: invalid auth state")
		}
	}
}

// authReadLine reads a line and separates it into its fields.
func authReadLine(in *bufio.Reader) ([][]byte, error) {
	data, err := in.ReadBytes('\n')
	if err != nil {
		return nil, err
	}
	data = bytes.TrimSuffix(data, []byte("\r\n"))
	return bytes.Split(data, []byte{' '}), nil
}

// authWriteLine writes the given line in the authentication protocol format
// (elements of data separated by a " " and terminated by "\r\n").
func authWriteLine(out io.Writer, data ...[]byte) error {
	buf := make([]byte, 0)
	for i, v := range data {
		buf = append(buf, v...)
		if i != len(data)-1 {
			buf = append(buf, ' ')
		}
	}
	buf = append(buf, '\r')
	buf = append(buf, '\n')
	n, err := out.Write(buf)
	if err != nil {
		return err
	}
	if n != len(buf) {
		return io.ErrUnexpectedEOF
	}
	return nil
}
<|endoftext|>


<commit_before>package main

import  (
        "fmt"
        "github.com/spf13/cobra"
        "github.com/rakyll/globalconf"
        "../go-notetxt"
        "flag"
        "os/user"
        "time"
        "strings"
        "strconv"
)

func main() {

        conf, _ := globalconf.New("gonote")

        var flagNotedir = flag.String("dir", "", "Location of the note.txt directory.")
        var dir string

        var today bool

        var cmdAdd = &cobra.Command{
            Use:   "add [title] [tag]",
            Short: "Add a note.",
            Long:  `Add a note and tag it.`,
            Run: func(cmd *cobra.Command, args []string) {
                if len(args) < 1 && !today {
                        fmt.Println("I need something to add.")
                        return
                }

                var text string
                t := time.Now().Local()

                if today {
                        text = fmt.Sprintf("Daily journal, date %s", t.Format("02. 01. 2006"))
                } else {
                        text = strings.Join(args, " ")
                }

                file, err := notetxt.CreateNote(text, t.Format("2006/01/"), dir)
                if err != nil {
                        panic(err);
                }

                notetxt.OpenFileInEditor(file)
            },
        }
        cmdAdd.Flags().BoolVarP(&today, "today", "T", false,
                                 "Add today's journal entry.")

        var cmdList = &cobra.Command{
            Use:   "ls",
            Short: "List notes.",
            Long:  `List all valid note files in the directory.`,
            Run: func(cmd *cobra.Command, args []string) {
                notes, err := notetxt.ParseDir(dir)
                if err != nil {
                    panic(err)
                }

                needle := strings.Join(args, " ")

                for i, note := range notes {
                    if note.Matches(needle) {
                        fmt.Printf("%d %s - %v\n", i, note.Name, note.Tags)
                    }
                }
            },
        }

        var cmdEdit = &cobra.Command{
            Use:   "edit <id>|<selector>",
            Short: "Edit notes.",
            Long:  `Edit a note identified by either an ID or a selector.`,
            Run: func(cmd *cobra.Command, args []string) {
                if len(args) < 1 {
                        fmt.Println("Either a note ID or a selector is required.")
                        return
                }

                notes, err := notetxt.ParseDir(dir)
                if err != nil {
                    panic(err)
                }

                noteid, err := strconv.Atoi(args[0])
                if err != nil {
                        needle := strings.Join(args, " ")
                        filtered_notes := notes.FilterBy(needle)

                        if len(filtered_notes) == 1 {
                                notetxt.OpenFileInEditor(filtered_notes[0].Filename)
                        } else if len (filtered_notes) == 0 {
                                fmt.Printf("No notes matched your selector.")
                        } else {
                                fmt.Printf("Notes matching your selector:\n")
                                filtered_notes.Print()
                        }

                        return
                }

                if noteid > len(notes) || noteid < 0 {
                        fmt.Printf("Invalid note ID (%v)\n", noteid)
                        return
                }

                notetxt.OpenFileInEditor(notes[noteid].Filename)

            },
        }

        var cmdTag = &cobra.Command{
            Use:   "tag <noteid> <tag-name>",
            Short: "Attaches a tag to a note.",
            Long:  `Tags a note with a one or more tags.`,
            Run: func(cmd *cobra.Command, args []string) {
                if len(args) < 2 {
                        fmt.Printf("Too few arguments.")
                }

                notes, err := notetxt.ParseDir(dir)
                if err != nil {
                    panic(err)
                }

                noteid, err := strconv.Atoi(args[0])
                if err != nil {
                        fmt.Printf("Do you really consider that a number? %v\n", err)
                        return
                }

                if noteid > len(notes) || noteid < 0 {
                        fmt.Printf("Invalid note ID (%v)\n", noteid)
                        return
                }

                file := notes[noteid].Filename
                tag := args[1]
                err = notetxt.TagNote(file, tag, dir)
                if err != nil {
                        panic(err)
                }

            },
        }

        var GonoterCmd = &cobra.Command{
            Use:   "gonote",
            Short: "gonote is a go implementation of note.txt specification.",
            Long: `A small, fast and fun implementation of note.txt`,
            Run: func(cmd *cobra.Command, args []string) {
                cmdList.Run(cmd, args)
            },
        }

        GonoterCmd.PersistentFlags().StringVarP(&dir, "directory", "", "",
                                     "Location of the note.txt directory.")

        conf.ParseAll()
        if dir == "" {
                if *flagNotedir == "" {
                        usr, err := user.Current()
                        if err != nil {
                                panic(err)
                        }

                        dir = usr.HomeDir + "/notes"
                } else {
                        dir = *flagNotedir
                }
        }

        GonoterCmd.AddCommand(cmdAdd)
        GonoterCmd.AddCommand(cmdList)
        GonoterCmd.AddCommand(cmdTag)
        GonoterCmd.AddCommand(cmdEdit)
        GonoterCmd.Execute()
}
<commit_msg>update: Fixed help messages<commit_after>package main

import  (
        "fmt"
        "github.com/spf13/cobra"
        "github.com/rakyll/globalconf"
        "../go-notetxt"
        "flag"
        "os/user"
        "time"
        "strings"
        "strconv"
)

func main() {

        conf, _ := globalconf.New("gonote")

        var flagNotedir = flag.String("dir", "", "Location of the note.txt directory.")
        var dir string

        var today bool

        var cmdAdd = &cobra.Command{
            Use:   "add <title> [tag]",
            Short: "Add a note.",
            Long:  `Add a note and tag it.`,
            Run: func(cmd *cobra.Command, args []string) {
                if len(args) < 1 && !today {
                        fmt.Println("I need something to add.")
                        return
                }

                var text string
                t := time.Now().Local()

                if today {
                        text = fmt.Sprintf("Daily journal, date %s", t.Format("02. 01. 2006"))
                } else {
                        text = strings.Join(args, " ")
                }

                file, err := notetxt.CreateNote(text, t.Format("2006/01/"), dir)
                if err != nil {
                        panic(err);
                }

                notetxt.OpenFileInEditor(file)
            },
        }
        cmdAdd.Flags().BoolVarP(&today, "today", "T", false,
                                 "Add today's journal entry.")

        var cmdList = &cobra.Command{
            Use:   "ls <query>",
            Short: "List notes.",
            Long:  `List all valid note files in the directory.`,
            Run: func(cmd *cobra.Command, args []string) {
                notes, err := notetxt.ParseDir(dir)
                if err != nil {
                    panic(err)
                }

                needle := strings.Join(args, " ")

                for i, note := range notes {
                    if note.Matches(needle) {
                        fmt.Printf("%d %s - %v\n", i, note.Name, note.Tags)
                    }
                }
            },
        }

        var cmdEdit = &cobra.Command{
            Use:   "edit <id>|<selector>",
            Short: "Edit notes.",
            Long:  `Edit a note identified by either an ID or a selector.`,
            Run: func(cmd *cobra.Command, args []string) {
                if len(args) < 1 {
                        fmt.Println("Either a note ID or a selector is required.")
                        return
                }

                notes, err := notetxt.ParseDir(dir)
                if err != nil {
                    panic(err)
                }

                noteid, err := strconv.Atoi(args[0])
                if err != nil {
                        needle := strings.Join(args, " ")
                        filtered_notes := notes.FilterBy(needle)

                        if len(filtered_notes) == 1 {
                                notetxt.OpenFileInEditor(filtered_notes[0].Filename)
                        } else if len (filtered_notes) == 0 {
                                fmt.Printf("No notes matched your selector.")
                        } else {
                                fmt.Printf("Notes matching your selector:\n")
                                filtered_notes.Print()
                        }

                        return
                }

                if noteid > len(notes) || noteid < 0 {
                        fmt.Printf("Invalid note ID (%v)\n", noteid)
                        return
                }

                notetxt.OpenFileInEditor(notes[noteid].Filename)

            },
        }

        var cmdTag = &cobra.Command{
            Use:   "tag <noteid> <tag-name>",
            Short: "Attaches a tag to a note.",
            Long:  `Tags a note with a one or more tags.`,
            Run: func(cmd *cobra.Command, args []string) {
                if len(args) < 2 {
                        fmt.Printf("Too few arguments.")
                }

                notes, err := notetxt.ParseDir(dir)
                if err != nil {
                    panic(err)
                }

                noteid, err := strconv.Atoi(args[0])
                if err != nil {
                        fmt.Printf("Do you really consider that a number? %v\n", err)
                        return
                }

                if noteid > len(notes) || noteid < 0 {
                        fmt.Printf("Invalid note ID (%v)\n", noteid)
                        return
                }

                file := notes[noteid].Filename
                tag := args[1]
                err = notetxt.TagNote(file, tag, dir)
                if err != nil {
                        panic(err)
                }

            },
        }

        var GonoterCmd = &cobra.Command{
            Use:   "gonote",
            Short: "gonote is a go implementation of note.txt specification.",
            Long: `A small, fast and fun implementation of note.txt`,
            Run: func(cmd *cobra.Command, args []string) {
                cmdList.Run(cmd, args)
            },
        }

        GonoterCmd.PersistentFlags().StringVarP(&dir, "directory", "", "",
                                     "Location of the note.txt directory.")

        conf.ParseAll()
        if dir == "" {
                if *flagNotedir == "" {
                        usr, err := user.Current()
                        if err != nil {
                                panic(err)
                        }

                        dir = usr.HomeDir + "/notes"
                } else {
                        dir = *flagNotedir
                }
        }

        GonoterCmd.AddCommand(cmdAdd)
        GonoterCmd.AddCommand(cmdList)
        GonoterCmd.AddCommand(cmdTag)
        GonoterCmd.AddCommand(cmdEdit)
        GonoterCmd.Execute()
}
<|endoftext|>


<commit_before>package gunfish_test

import (
	"bytes"
	"encoding/json"
	"fmt"
	"sync"
	"testing"
	"time"

	gunfish "github.com/kayac/Gunfish"
	"github.com/kayac/Gunfish/apns"
	"github.com/kayac/Gunfish/config"
	"github.com/sirupsen/logrus"
)

var (
	conf, _ = config.LoadConfig("./test/gunfish_test.toml")
	mu      sync.Mutex
)

type TestResponseHandler struct {
	scoreboard map[string]*int
	wg         *sync.WaitGroup
	hook       string
}

func (tr *TestResponseHandler) Done(token string) {
	tr.wg.Done()
}

func (tr *TestResponseHandler) Countup(name string) {
	mu.Lock()
	defer mu.Unlock()
	*(tr.scoreboard[name])++
}

func (tr TestResponseHandler) OnResponse(result gunfish.Result) {
	tr.wg.Add(1)
	if err := result.Err(); err != nil {
		logrus.Warnf(err.Error())
		tr.Countup(err.Error())
	} else {
		tr.Countup("success")
	}
	tr.Done(result.RecipientIdentifier())
}

func (tr TestResponseHandler) HookCmd() string {
	return tr.hook
}

func init() {
	logrus.SetLevel(logrus.WarnLevel)
	conf.Apns.Host = gunfish.MockServer
}

func TestEnqueuRequestToSupervisor(t *testing.T) {
	// Prepare
	wg := sync.WaitGroup{}
	score := make(map[string]*int, 5)
	boardList := []string{
		apns.MissingTopic.String(),
		apns.BadDeviceToken.String(),
		apns.Unregistered.String(),
		apns.ExpiredProviderToken.String(),
		"success",
	}
	for _, v := range boardList {
		x := 0
		score[v] = &x
	}

	etr := TestResponseHandler{
		wg:         &wg,
		scoreboard: score,
		hook:       conf.Provider.ErrorHook,
	}
	str := TestResponseHandler{
		wg:         &wg,
		scoreboard: score,
	}
	gunfish.InitErrorResponseHandler(etr)
	gunfish.InitSuccessResponseHandler(str)

	sup, err := gunfish.StartSupervisor(&conf)
	if err != nil {
		t.Errorf("cannot start supervisor: %s", err.Error())
	}
	defer sup.Shutdown()

	// test success requests
	reqs := repeatRequestData("1122334455667788112233445566778811223344556677881122334455667788", 10)
	for range []int{0, 1, 2, 3, 4, 5, 6} {
		sup.EnqueueClientRequest(&reqs)
	}
	time.Sleep(time.Millisecond * 500)
	wg.Wait()
	if g, w := *(score["success"]), 70; g != w {
		t.Errorf("not match success count: got %d want %d", g, w)
	}

	// test error requests
	testTable := []struct {
		errToken string
		num      int
		msleep   time.Duration
		errCode  apns.ErrorResponseCode
		expect   int
	}{
		{
			errToken: "missingtopic",
			num:      1,
			msleep:   300,
			errCode:  apns.MissingTopic,
			expect:   1,
		},
		{
			errToken: "unregistered",
			num:      1,
			msleep:   300,
			errCode:  apns.Unregistered,
			expect:   1,
		},
		{
			errToken: "baddevicetoken",
			num:      1,
			msleep:   300,
			errCode:  apns.BadDeviceToken,
			expect:   1,
		},
		{
			errToken: "expiredprovidertoken",
			num:      1,
			msleep:   5000,
			errCode:  apns.ExpiredProviderToken,
			expect:   1 * gunfish.SendRetryCount,
		},
	}

	for _, tt := range testTable {
		reqs := repeatRequestData(tt.errToken, tt.num)
		sup.EnqueueClientRequest(&reqs)
		time.Sleep(time.Millisecond * tt.msleep)
		wg.Wait()

		errReason := tt.errCode.String()
		if g, w := *(score[errReason]), tt.expect; g != w {
			t.Errorf("not match %s count: got %d want %d", errReason, g, w)
		}
	}
}

func repeatRequestData(token string, num int) []gunfish.Request {
	var reqs []gunfish.Request
	for i := 0; i < num; i++ {
		// Create request
		aps := &apns.APS{
			Alert: &apns.Alert{
				Title: "test",
				Body:  "message",
			},
			Sound: "default",
		}
		payload := apns.Payload{}
		payload.APS = aps

		req := gunfish.Request{
			Notification: apns.Notification{
				Token:   token,
				Payload: payload,
			},
			Tries: 0,
		}

		reqs = append(reqs, req)
	}
	return reqs
}

func TestSuccessOrFailureInvoke(t *testing.T) {
	// prepare SenderResponse
	token := "invalid token"
	sre := fmt.Errorf(apns.Unregistered.String())
	aps := &apns.APS{
		Alert: apns.Alert{
			Title: "test",
			Body:  "hoge message",
		},
		Badge: 1,
		Sound: "default",
	}
	payload := apns.Payload{}
	payload.APS = aps
	sr := gunfish.SenderResponse{
		Req: gunfish.Request{
			Notification: apns.Notification{
				Token:   token,
				Payload: payload,
			},
			Tries: 0,
		},
		RespTime: 0.0,
		Err:      sre,
	}
	j, err := json.Marshal(sr)
	if err != nil {
		t.Errorf(err.Error())
	}

	// Succeed to invoke
	src := bytes.NewBuffer(j)
	out, err := gunfish.InvokePipe(`cat`, src)
	if err != nil {
		t.Errorf("result: %s, err: %s", string(out), err.Error())
	}

	// checks Unmarshaled result
	if string(out) == `{}` {
		t.Errorf("output of result is empty: %s", string(out))
	}
	if string(out) != string(j) {
		t.Errorf("Expected result %s but got %s", j, string(out))
	}

	// Failure to invoke
	src = bytes.NewBuffer(j)
	out, err = gunfish.InvokePipe(`expr 1 1`, src)
	if err == nil {
		t.Errorf("Expected failure to invoke command: %s", string(out))
	}

	// tests command including Pipe '|'
	src = bytes.NewBuffer(j)
	out, err = gunfish.InvokePipe(`cat | head -n 10 | tail -n 10`, src)
	if err != nil {
		t.Errorf("result: %s, err: %s", string(out), err.Error())
	}
	if string(out) != string(j) {
		t.Errorf("Expected result '%s' but got %s", j, string(out))
	}

	// Must fail
	src = bytes.NewBuffer(j)
	out, err = gunfish.InvokePipe(`echo 'Failure test'; false`, src)
	if err == nil {
		t.Errorf("result: %s, err: %s", string(out), err.Error())
	}
	if fmt.Sprintf("%s", err.Error()) != `exit status 1` {
		t.Errorf("invalid err message: %s", err.Error())
	}

	// stdout be not captured
	gunfish.OutputHookStdout = true
	src = bytes.NewBuffer(j)
	out, err = gunfish.InvokePipe(`cat; echo 'this is error.' 1>&2`, src)
	if len(out) != 15 {
		t.Errorf("hooks stdout must not be captured: %s", out)
	}

	// stderr
	gunfish.OutputHookStderr = true
	src = bytes.NewBuffer(j)
	out, err = gunfish.InvokePipe(`cat; echo 'this is error.' 1>&2`, src)
	if len(out) != 0 {
		t.Errorf("hooks stderr must not be captured: %s", out)
	}
}
<commit_msg>fix race<commit_after>package gunfish_test

import (
	"bytes"
	"encoding/json"
	"fmt"
	"sync"
	"testing"
	"time"

	gunfish "github.com/kayac/Gunfish"
	"github.com/kayac/Gunfish/apns"
	"github.com/kayac/Gunfish/config"
	"github.com/sirupsen/logrus"
)

var (
	conf, _ = config.LoadConfig("./test/gunfish_test.toml")
	mu      sync.Mutex
)

type TestResponseHandler struct {
	scoreboard map[string]*int
	wg         *sync.WaitGroup
	hook       string
}

func (tr *TestResponseHandler) Done(token string) {
	tr.wg.Done()
}

func (tr *TestResponseHandler) Countup(name string) {
	mu.Lock()
	defer mu.Unlock()
	*(tr.scoreboard[name])++
}

func (tr *TestResponseHandler) Get(name string) int {
	mu.Lock()
	defer mu.Unlock()
	return *(tr.scoreboard[name])
}

func (tr TestResponseHandler) OnResponse(result gunfish.Result) {
	tr.wg.Add(1)
	if err := result.Err(); err != nil {
		tr.Countup(err.Error())
	} else {
		tr.Countup("success")
	}
	tr.Done(result.RecipientIdentifier())
}

func (tr TestResponseHandler) HookCmd() string {
	return tr.hook
}

func init() {
	logrus.SetLevel(logrus.WarnLevel)
	conf.Apns.Host = gunfish.MockServer
}

func TestEnqueuRequestToSupervisor(t *testing.T) {
	// Prepare
	wg := sync.WaitGroup{}
	score := make(map[string]*int, 5)
	boardList := []string{
		apns.MissingTopic.String(),
		apns.BadDeviceToken.String(),
		apns.Unregistered.String(),
		apns.ExpiredProviderToken.String(),
		"success",
	}
	for _, v := range boardList {
		x := 0
		score[v] = &x
	}

	etr := TestResponseHandler{
		wg:         &wg,
		scoreboard: score,
		hook:       conf.Provider.ErrorHook,
	}
	str := TestResponseHandler{
		wg:         &wg,
		scoreboard: score,
	}
	gunfish.InitErrorResponseHandler(etr)
	gunfish.InitSuccessResponseHandler(str)

	sup, err := gunfish.StartSupervisor(&conf)
	if err != nil {
		t.Errorf("cannot start supervisor: %s", err.Error())
	}
	defer sup.Shutdown()

	// test success requests
	reqs := repeatRequestData("1122334455667788112233445566778811223344556677881122334455667788", 10)
	for range []int{0, 1, 2, 3, 4, 5, 6} {
		sup.EnqueueClientRequest(&reqs)
	}
	time.Sleep(time.Millisecond * 500)
	wg.Wait()
	if g, w := str.Get("success"), 70; g != w {
		t.Errorf("not match success count: got %d want %d", g, w)
	}

	// test error requests
	testTable := []struct {
		errToken string
		num      int
		msleep   time.Duration
		errCode  apns.ErrorResponseCode
		expect   int
	}{
		{
			errToken: "missingtopic",
			num:      1,
			msleep:   300,
			errCode:  apns.MissingTopic,
			expect:   1,
		},
		{
			errToken: "unregistered",
			num:      1,
			msleep:   300,
			errCode:  apns.Unregistered,
			expect:   1,
		},
		{
			errToken: "baddevicetoken",
			num:      1,
			msleep:   300,
			errCode:  apns.BadDeviceToken,
			expect:   1,
		},
		{
			errToken: "expiredprovidertoken",
			num:      1,
			msleep:   5000,
			errCode:  apns.ExpiredProviderToken,
			expect:   1 * gunfish.SendRetryCount,
		},
	}

	for _, tt := range testTable {
		reqs := repeatRequestData(tt.errToken, tt.num)
		sup.EnqueueClientRequest(&reqs)
		time.Sleep(time.Millisecond * tt.msleep)
		wg.Wait()

		errReason := tt.errCode.String()
		if g, w := str.Get(errReason), tt.expect; g != w {
			t.Errorf("not match %s count: got %d want %d", errReason, g, w)
		}
	}
}

func repeatRequestData(token string, num int) []gunfish.Request {
	var reqs []gunfish.Request
	for i := 0; i < num; i++ {
		// Create request
		aps := &apns.APS{
			Alert: &apns.Alert{
				Title: "test",
				Body:  "message",
			},
			Sound: "default",
		}
		payload := apns.Payload{}
		payload.APS = aps

		req := gunfish.Request{
			Notification: apns.Notification{
				Token:   token,
				Payload: payload,
			},
			Tries: 0,
		}

		reqs = append(reqs, req)
	}
	return reqs
}

func TestSuccessOrFailureInvoke(t *testing.T) {
	// prepare SenderResponse
	token := "invalid token"
	sre := fmt.Errorf(apns.Unregistered.String())
	aps := &apns.APS{
		Alert: apns.Alert{
			Title: "test",
			Body:  "hoge message",
		},
		Badge: 1,
		Sound: "default",
	}
	payload := apns.Payload{}
	payload.APS = aps
	sr := gunfish.SenderResponse{
		Req: gunfish.Request{
			Notification: apns.Notification{
				Token:   token,
				Payload: payload,
			},
			Tries: 0,
		},
		RespTime: 0.0,
		Err:      sre,
	}
	j, err := json.Marshal(sr)
	if err != nil {
		t.Errorf(err.Error())
	}

	// Succeed to invoke
	src := bytes.NewBuffer(j)
	out, err := gunfish.InvokePipe(`cat`, src)
	if err != nil {
		t.Errorf("result: %s, err: %s", string(out), err.Error())
	}

	// checks Unmarshaled result
	if string(out) == `{}` {
		t.Errorf("output of result is empty: %s", string(out))
	}
	if string(out) != string(j) {
		t.Errorf("Expected result %s but got %s", j, string(out))
	}

	// Failure to invoke
	src = bytes.NewBuffer(j)
	out, err = gunfish.InvokePipe(`expr 1 1`, src)
	if err == nil {
		t.Errorf("Expected failure to invoke command: %s", string(out))
	}

	// tests command including Pipe '|'
	src = bytes.NewBuffer(j)
	out, err = gunfish.InvokePipe(`cat | head -n 10 | tail -n 10`, src)
	if err != nil {
		t.Errorf("result: %s, err: %s", string(out), err.Error())
	}
	if string(out) != string(j) {
		t.Errorf("Expected result '%s' but got %s", j, string(out))
	}

	// Must fail
	src = bytes.NewBuffer(j)
	out, err = gunfish.InvokePipe(`echo 'Failure test'; false`, src)
	if err == nil {
		t.Errorf("result: %s, err: %s", string(out), err.Error())
	}
	if fmt.Sprintf("%s", err.Error()) != `exit status 1` {
		t.Errorf("invalid err message: %s", err.Error())
	}

	// stdout be not captured
	gunfish.OutputHookStdout = true
	src = bytes.NewBuffer(j)
	out, err = gunfish.InvokePipe(`cat; echo 'this is error.' 1>&2`, src)
	if len(out) != 15 {
		t.Errorf("hooks stdout must not be captured: %s", out)
	}

	// stderr
	gunfish.OutputHookStderr = true
	src = bytes.NewBuffer(j)
	out, err = gunfish.InvokePipe(`cat; echo 'this is error.' 1>&2`, src)
	if len(out) != 0 {
		t.Errorf("hooks stderr must not be captured: %s", out)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"fmt"
	"os"

	"github.com/reusee/ccg"
)

var (
	pt = fmt.Printf
)

func main() {
	if len(os.Args) < 2 {
		pt("usage: %s [command] [args...]\n", os.Args[0])
		return
	}

	getArg := func(index int, usage string) string {
		if len(os.Args) < index+1 {
			pt("usage: %s %s\n", os.Args[0], usage)
			os.Exit(-1)
		}
		return os.Args[index]
	}

	command := os.Args[1]
	switch command {
	case "sorter":
		t := getArg(2, "sorter [type]")
		buf := new(bytes.Buffer)
		ccg.Copy(ccg.Config{
			From: "github.com/reusee/ccg/sorter",
			Params: map[string]string{
				"T": t,
			},
			Renames: map[string]string{
				"Sorter": t + "Sorter",
			},
			Writer: buf,
		})
		pt("%s\n", buf.Bytes())
	default:
		pt("unknown command: %s\n", command)
		return
	}
}
<commit_msg>myccg: add sorter type argument to sorter<commit_after>package main

import (
	"bytes"
	"fmt"
	"os"

	"github.com/reusee/ccg"
)

var (
	pt = fmt.Printf
)

func main() {
	if len(os.Args) < 2 {
		pt("usage: %s [command] [args...]\n", os.Args[0])
		return
	}

	getArg := func(index int, usage string) string {
		if len(os.Args) < index+1 {
			pt("usage: %s %s\n", os.Args[0], usage)
			os.Exit(-1)
		}
		return os.Args[index]
	}

	command := os.Args[1]
	switch command {
	case "sorter":
		usage := "sorter [type] [sorter type]"
		t := getArg(2, usage)
		sorterType := getArg(3, usage)
		buf := new(bytes.Buffer)
		ccg.Copy(ccg.Config{
			From: "github.com/reusee/ccg/sorter",
			Params: map[string]string{
				"T": t,
			},
			Renames: map[string]string{
				"Sorter": sorterType,
			},
			Writer: buf,
		})
		pt("%s\n", buf.Bytes())
	default:
		pt("unknown command: %s\n", command)
		return
	}
}
<|endoftext|>


<commit_before>package gothub

import (
	"encoding/base64"
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"net/http"
	"strconv"
)

// The HTTP host that we will hit to use the GitHub API.
const (
	GitHubUrl string = "https://api.github.com"
)

var (
	ErrRateLimitReached = errors.New("Rate limit reached")
	ErrNoJSON           = errors.New("GitHub did not return a JSON response")
)

// The GitHub struct represents an active session to the GitHub API.
type GitHub struct {
	httpClient         *http.Client
	Authorization      string
	RateLimit          int
	RateLimitRemaining int
}

func hashAuth(u, p string) string {
	var a = fmt.Sprintf("%s:%s", u, p)
	return base64.StdEncoding.EncodeToString([]byte(a))
}

// Log in to GitHub using basic, username/password authentication.
func BasicLogin(username, password string) (*GitHub, error) {
	// Format and Base64-encode the provided username and password, in preparation for basic
	// HTTP auth.
	authorization := fmt.Sprintf("Basic %s", hashAuth(username, password))
	request, err := http.NewRequest("GET", GitHubUrl, nil)
	if err != nil {
		return nil, err
	}

	// Set the Authorization header.
	request.Header.Set("Authorization", authorization)

	// Create a new HTTP client (which we will eventually provide to the GitHub struct), for
	// issuing the above HTTP request, and future requests.
	client := &http.Client{}
	response, err := client.Do(request)
	if err != nil {
		return nil, err
	}

	if response.StatusCode == http.StatusOK {
		// Yaaaaaaay!
		ratelimit, err := strconv.Atoi(response.Header.Get("X-RateLimit-Limit"))
		if err != nil {
			return nil, err
		}

		remaining, err := strconv.Atoi(response.Header.Get("X-RateLimit-Remaining"))
		if err != nil {
			return nil, err
		}

		return &GitHub{httpClient: client, Authorization: authorization,
			RateLimit: ratelimit, RateLimitRemaining: remaining}, nil
	}

	// Should we get here, the basic authentication request failed.
	e := "Authorization failed with HTTP code: %d"
	return nil, errors.New(fmt.Sprintf(e, response.StatusCode))
}

// Updates the call limit rates in the GitHub struct.
func (g *GitHub) updateRates(r *http.Response) (err error) {
	limit, err := strconv.Atoi(r.Header.Get("X-RateLimit-Limit"))
	if err != nil {
		return
	}
	g.RateLimit = limit

	remaining, err := strconv.Atoi(r.Header.Get("X-RateLimit-Remaining"))
	if err != nil {
		return
	}
	g.RateLimitRemaining = remaining
	return
}

// Calls the GitHub API and returns the raw, HTTP response body.
func call(g *GitHub, method, uri string) (response *http.Response, err error) {
	if g.RateLimitRemaining == 0 {
		err = ErrRateLimitReached
		return
	}

	url := fmt.Sprintf("%s%s", GitHubUrl, uri)
	request, err := http.NewRequest(method, url, nil)
	if err != nil {
		return
	}

	request.Header.Set("Authorization", g.Authorization)

	// Fire off the request.
	response, err = g.httpClient.Do(request)
	if err != nil {
		return
	}

	// Update the call rates
	g.updateRates(response)

	// Check to make sure the API came back with an HTTP 200 OK
	if response.StatusCode != http.StatusOK {
		e := "GitHub API responded with HTTP %d"
		err = errors.New(fmt.Sprintf(e, response.StatusCode))
	}

	return
}

// Calls the GitHub API, but will unmarshal a JSON response to the struct
// provided to `rs`.
func (g *GitHub) callGithubApi(method, uri string, rs interface{}) error {
	response, err := call(g, method, uri)
	if err != nil {
		return err
	}

	// Check to make sure we actually got JSON back.
	switch response.Header.Get("Content-Type") {
	case "application/json":
		fallthrough
	case "application/json; charset=utf-8":
		var js []byte
		js, err := ioutil.ReadAll(response.Body)
		if err != nil {
			return err
		} else {
			err = json.Unmarshal(js, rs)
		}
	default:
		err = ErrNoJSON
	}

	return err
}
<commit_msg>Changed the mechanism for what's considered a "successful" HTTP response.<commit_after>package gothub

import (
	"encoding/base64"
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"net/http"
	"strconv"
)

// The HTTP host that we will hit to use the GitHub API.
const (
	GitHubUrl string = "https://api.github.com"
)

var (
	ErrRateLimitReached = errors.New("Rate limit reached")
	ErrNoJSON           = errors.New("GitHub did not return a JSON response")
)

// The GitHub struct represents an active session to the GitHub API.
type GitHub struct {
	httpClient         *http.Client
	Authorization      string
	RateLimit          int
	RateLimitRemaining int
}

func hashAuth(u, p string) string {
	var a = fmt.Sprintf("%s:%s", u, p)
	return base64.StdEncoding.EncodeToString([]byte(a))
}

// Log in to GitHub using basic, username/password authentication.
func BasicLogin(username, password string) (*GitHub, error) {
	// Format and Base64-encode the provided username and password, in preparation for basic
	// HTTP auth.
	authorization := fmt.Sprintf("Basic %s", hashAuth(username, password))
	request, err := http.NewRequest("GET", GitHubUrl, nil)
	if err != nil {
		return nil, err
	}

	// Set the Authorization header.
	request.Header.Set("Authorization", authorization)

	// Create a new HTTP client (which we will eventually provide to the GitHub struct), for
	// issuing the above HTTP request, and future requests.
	client := &http.Client{}
	response, err := client.Do(request)
	if err != nil {
		return nil, err
	}

	if response.StatusCode == http.StatusOK {
		// Yaaaaaaay!
		ratelimit, err := strconv.Atoi(response.Header.Get("X-RateLimit-Limit"))
		if err != nil {
			return nil, err
		}

		remaining, err := strconv.Atoi(response.Header.Get("X-RateLimit-Remaining"))
		if err != nil {
			return nil, err
		}

		return &GitHub{httpClient: client, Authorization: authorization,
			RateLimit: ratelimit, RateLimitRemaining: remaining}, nil
	}

	// Should we get here, the basic authentication request failed.
	e := "Authorization failed with HTTP code: %d"
	return nil, errors.New(fmt.Sprintf(e, response.StatusCode))
}

// Updates the call limit rates in the GitHub struct.
func (g *GitHub) updateRates(r *http.Response) (err error) {
	limit, err := strconv.Atoi(r.Header.Get("X-RateLimit-Limit"))
	if err != nil {
		return
	}
	g.RateLimit = limit

	remaining, err := strconv.Atoi(r.Header.Get("X-RateLimit-Remaining"))
	if err != nil {
		return
	}
	g.RateLimitRemaining = remaining
	return
}

// Calls the GitHub API and returns the raw, HTTP response body.
func call(g *GitHub, method, uri string) (response *http.Response, err error) {
	if g.RateLimitRemaining == 0 {
		err = ErrRateLimitReached
		return
	}

	url := fmt.Sprintf("%s%s", GitHubUrl, uri)
	request, err := http.NewRequest(method, url, nil)
	if err != nil {
		return
	}

	request.Header.Set("Authorization", g.Authorization)

	// Fire off the request.
	response, err = g.httpClient.Do(request)
	if err != nil {
		return
	}

	// Update the call rates
	g.updateRates(response)

	// Check to make sure the API came back with an appropriate HTTP status
	// code, depending on the request method
	switch method {
	case "GET":
		if response.StatusCode != http.StatusOK {
			e := "GitHub API responded with HTTP %d"
			err = errors.New(fmt.Sprintf(e, response.StatusCode))
		}

	case "POST":
		switch response.StatusCode {
		case http.StatusCreated:
			return
		}

	case "DELETE":
		switch response.StatusCode {
		case http.StatusNoContent:
			return
		}
	}

	return
}

// Calls the GitHub API, but will unmarshal a JSON response to the struct
// provided to `rs`.
func (g *GitHub) callGithubApi(method, uri string, rs interface{}) error {
	response, err := call(g, method, uri)
	if err != nil {
		return err
	}

	// Check to make sure we actually got JSON back.
	switch response.Header.Get("Content-Type") {
	case "application/json":
		fallthrough
	case "application/json; charset=utf-8":
		var js []byte
		js, err := ioutil.ReadAll(response.Body)
		if err != nil {
			return err
		} else {
			err = json.Unmarshal(js, rs)
		}
	default:
		err = ErrNoJSON
	}

	return err
}
<|endoftext|>


<commit_before>// goupnp is an implementation of a client for various UPnP services.
//
// For most uses, it is recommended to use the code-generated packages under
// github.com/huin/goupnp/dcps. Example use is shown at
// http://godoc.org/github.com/huin/goupnp/example
//
// A commonly used client is internetgateway1.WANPPPConnection1:
// http://godoc.org/github.com/huin/goupnp/dcps/internetgateway1#WANPPPConnection1
//
// Currently only a couple of schemas have code generated for them from the
// UPnP example XML specifications. Not all methods will work on these clients,
// because the generated stubs contain the full set of specified methods from
// the XML specifications, and the discovered services will likely support a
// subset of those methods.
package goupnp

import (
	"encoding/xml"
	"fmt"
	"net/http"
	"net/url"
	"time"

	"golang.org/x/net/html/charset"

	"github.com/huin/goupnp/httpu"
	"github.com/huin/goupnp/ssdp"
)

// ContextError is an error that wraps an error with some context information.
type ContextError struct {
	Context string
	Err     error
}

func (err ContextError) Error() string {
	return fmt.Sprintf("%s: %v", err.Context, err.Err)
}

// MaybeRootDevice contains either a RootDevice (and URL) or an error.
type MaybeRootDevice struct {
	// Set iff Err == nil.
	Root *RootDevice

	// The location the device was discovered at. This can be used with
	// DeviceByURL, assuming the device is still present. A location represents
	// the discovery of a device, regardless of if there was an error probing it.
	Location *url.URL

	// Any error encountered probing a discovered device.
	Err error
}

// DiscoverDevices attempts to find targets of the given type. This is
// typically the entry-point for this package. searchTarget is typically a URN
// in the form "urn:schemas-upnp-org:device:..." or
// "urn:schemas-upnp-org:service:...". A single error is returned for errors
// while attempting to send the query. An error or RootDevice is returned for
// each discovered RootDevice.
func DiscoverDevices(searchTarget string) ([]MaybeRootDevice, error) {
	httpu, err := httpu.NewHTTPUClient()
	if err != nil {
		return nil, err
	}
	defer httpu.Close()
	responses, err := ssdp.SSDPRawSearch(httpu, string(searchTarget), 2, 3)
	if err != nil {
		return nil, err
	}

	results := make([]MaybeRootDevice, len(responses))
	for i, response := range responses {
		maybe := &results[i]
		loc, err := response.Location()
		if err != nil {
			maybe.Err = ContextError{"unexpected bad location from search", err}
			continue
		}
		maybe.Location = loc
		if root, err := DeviceByURL(loc); err != nil {
			maybe.Err = err
		} else {
			maybe.Root = root
		}
	}

	return results, nil
}

func DeviceByURL(loc *url.URL) (*RootDevice, error) {
	locStr := loc.String()
	root := new(RootDevice)
	if err := requestXml(locStr, DeviceXMLNamespace, root); err != nil {
		return nil, ContextError{fmt.Sprintf("error requesting root device details from %q", locStr), err}
	}
	var urlBaseStr string
	if root.URLBaseStr != "" {
		urlBaseStr = root.URLBaseStr
	} else {
		urlBaseStr = locStr
	}
	urlBase, err := url.Parse(urlBaseStr)
	if err != nil {
		return nil, ContextError{fmt.Sprintf("error parsing location URL %q", locStr), err}
	}
	root.SetURLBase(urlBase)
	return root, nil
}

func requestXml(url string, defaultSpace string, doc interface{}) error {
	timeout := time.Duration(3 * time.Second)
	client := http.Client{
		Timeout: timeout,
	}
	resp, err := client.Get(url)
	if err != nil {
		return err
	}
	defer resp.Body.Close()

	if resp.StatusCode != 200 {
		return fmt.Errorf("goupnp: got response status %s from %q",
			resp.Status, url)
	}

	decoder := xml.NewDecoder(resp.Body)
	decoder.DefaultSpace = defaultSpace
	decoder.CharsetReader = charset.NewReaderLabel

	return decoder.Decode(doc)
}
<commit_msg>Minor comment fix that could mislead.<commit_after>// goupnp is an implementation of a client for various UPnP services.
//
// For most uses, it is recommended to use the code-generated packages under
// github.com/huin/goupnp/dcps. Example use is shown at
// http://godoc.org/github.com/huin/goupnp/example
//
// A commonly used client is internetgateway1.WANPPPConnection1:
// http://godoc.org/github.com/huin/goupnp/dcps/internetgateway1#WANPPPConnection1
//
// Currently only a couple of schemas have code generated for them from the
// UPnP example XML specifications. Not all methods will work on these clients,
// because the generated stubs contain the full set of specified methods from
// the XML specifications, and the discovered services will likely support a
// subset of those methods.
package goupnp

import (
	"encoding/xml"
	"fmt"
	"net/http"
	"net/url"
	"time"

	"golang.org/x/net/html/charset"

	"github.com/huin/goupnp/httpu"
	"github.com/huin/goupnp/ssdp"
)

// ContextError is an error that wraps an error with some context information.
type ContextError struct {
	Context string
	Err     error
}

func (err ContextError) Error() string {
	return fmt.Sprintf("%s: %v", err.Context, err.Err)
}

// MaybeRootDevice contains either a RootDevice or an error.
type MaybeRootDevice struct {
	// Set iff Err == nil.
	Root *RootDevice

	// The location the device was discovered at. This can be used with
	// DeviceByURL, assuming the device is still present. A location represents
	// the discovery of a device, regardless of if there was an error probing it.
	Location *url.URL

	// Any error encountered probing a discovered device.
	Err error
}

// DiscoverDevices attempts to find targets of the given type. This is
// typically the entry-point for this package. searchTarget is typically a URN
// in the form "urn:schemas-upnp-org:device:..." or
// "urn:schemas-upnp-org:service:...". A single error is returned for errors
// while attempting to send the query. An error or RootDevice is returned for
// each discovered RootDevice.
func DiscoverDevices(searchTarget string) ([]MaybeRootDevice, error) {
	httpu, err := httpu.NewHTTPUClient()
	if err != nil {
		return nil, err
	}
	defer httpu.Close()
	responses, err := ssdp.SSDPRawSearch(httpu, string(searchTarget), 2, 3)
	if err != nil {
		return nil, err
	}

	results := make([]MaybeRootDevice, len(responses))
	for i, response := range responses {
		maybe := &results[i]
		loc, err := response.Location()
		if err != nil {
			maybe.Err = ContextError{"unexpected bad location from search", err}
			continue
		}
		maybe.Location = loc
		if root, err := DeviceByURL(loc); err != nil {
			maybe.Err = err
		} else {
			maybe.Root = root
		}
	}

	return results, nil
}

func DeviceByURL(loc *url.URL) (*RootDevice, error) {
	locStr := loc.String()
	root := new(RootDevice)
	if err := requestXml(locStr, DeviceXMLNamespace, root); err != nil {
		return nil, ContextError{fmt.Sprintf("error requesting root device details from %q", locStr), err}
	}
	var urlBaseStr string
	if root.URLBaseStr != "" {
		urlBaseStr = root.URLBaseStr
	} else {
		urlBaseStr = locStr
	}
	urlBase, err := url.Parse(urlBaseStr)
	if err != nil {
		return nil, ContextError{fmt.Sprintf("error parsing location URL %q", locStr), err}
	}
	root.SetURLBase(urlBase)
	return root, nil
}

func requestXml(url string, defaultSpace string, doc interface{}) error {
	timeout := time.Duration(3 * time.Second)
	client := http.Client{
		Timeout: timeout,
	}
	resp, err := client.Get(url)
	if err != nil {
		return err
	}
	defer resp.Body.Close()

	if resp.StatusCode != 200 {
		return fmt.Errorf("goupnp: got response status %s from %q",
			resp.Status, url)
	}

	decoder := xml.NewDecoder(resp.Body)
	decoder.DefaultSpace = defaultSpace
	decoder.CharsetReader = charset.NewReaderLabel

	return decoder.Decode(doc)
}
<|endoftext|>


<commit_before>// Copyright 2021 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package org

import (
	"encoding/json"
	"fmt"
	"os"

	"github.com/apigee/apigeecli/apiclient"
	"github.com/apigee/apigeecli/client/apis"
	"github.com/apigee/apigeecli/client/apps"
	"github.com/apigee/apigeecli/client/datacollectors"
	"github.com/apigee/apigeecli/client/developers"
	"github.com/apigee/apigeecli/client/env"
	"github.com/apigee/apigeecli/client/envgroups"
	"github.com/apigee/apigeecli/client/keystores"
	"github.com/apigee/apigeecli/client/kvm"
	"github.com/apigee/apigeecli/client/orgs"
	"github.com/apigee/apigeecli/client/products"
	"github.com/apigee/apigeecli/client/references"
	"github.com/apigee/apigeecli/client/sharedflows"
	"github.com/apigee/apigeecli/client/sync"
	"github.com/apigee/apigeecli/client/targetservers"
	"github.com/apigee/apigeecli/clilog"
	"github.com/spf13/cobra"
)

//ExportCmd to get org details
var ExportCmd = &cobra.Command{
	Use:   "export",
	Short: "Export Apigee Configuration",
	Long:  "Export Apigee Configuration",
	Args: func(cmd *cobra.Command, args []string) (err error) {
		return apiclient.SetApigeeOrg(org)
	},
	RunE: func(cmd *cobra.Command, args []string) (err error) {

		var productResponse, appsResponse, targetServerResponse, referencesResponse [][]byte
		var respBody []byte

		runtimeType, _ := orgs.GetOrgField("runtimeType")

		if err = createFolders(); err != nil {
			return err
		}

		clilog.Warning.Println("Calls to Apigee APIs have a quota of 6000 per min. Running this tool against large list of entities can exhaust that quota and impact the usage of the platform.")

		fmt.Println("Exporting API Proxies...")
		if err = apis.ExportProxies(conn, proxiesFolderName, allRevisions); err != nil {
			return err
		}

		fmt.Println("Exporting Sharedflows...")
		if err = sharedflows.Export(conn, sharedFlowsFolderName, allRevisions); err != nil {
			return err
		}

		fmt.Println("Exporting API Products...")
		if productResponse, err = products.Export(conn); err != nil {
			return err
		}
		if err = apiclient.WriteArrayByteArrayToFile(productsFileName, false, productResponse); err != nil {
			return err
		}

		fmt.Printf("\tExporting KV Map names for org %s\n", org)
		if respBody, err = kvm.List(""); err != nil {
			return err
		}
		if err = apiclient.WriteByteArrayToFile(org+"_"+kVMFileName, false, respBody); err != nil {
			return err
		}

		fmt.Println("Exporting Developers...")
		if respBody, err = developers.Export(); err != nil {
			return err
		}
		if err = apiclient.WriteByteArrayToFile(developersFileName, false, respBody); err != nil {
			return err
		}

		fmt.Println("Exporting Developer Apps...")
		if appsResponse, err = apps.Export(conn); err != nil {
			return err
		}
		if err = apiclient.WriteArrayByteArrayToFile(appsFileName, false, appsResponse); err != nil {
			return err
		}

		fmt.Println("Exporting Environment Group Configuration...")
		apiclient.SetPrintOutput(false)
		if respBody, err = envgroups.List(); err != nil {
			return err
		}
		if err = apiclient.WriteByteArrayToFile(envGroupsFileName, false, respBody); err != nil {
			return err
		}

		fmt.Println("Exporting Data collectors Configuration...")
		if respBody, err = datacollectors.List(); err != nil {
			return err
		}
		if err = apiclient.WriteByteArrayToFile(dataCollFileName, false, respBody); err != nil {
			return err
		}

		if runtimeType == "HYBRID" {
			fmt.Println("Exporting Sync Authorization Identities...")
			if respBody, err = sync.Get(); err != nil {
				return err
			}
			if err = apiclient.WriteByteArrayToFile(syncAuthFileName, false, respBody); err != nil {
				return err
			}
		}

		var envRespBody []byte
		if envRespBody, err = env.List(); err != nil {
			return err
		}

		environments := []string{}
		if err = json.Unmarshal(envRespBody, &environments); err != nil {
			return err

		}

		for _, environment := range environments {
			fmt.Println("Exporting configuration for environment " + environment)
			apiclient.SetApigeeEnv(environment)
			fmt.Println("\tExporting Target servers...")
			if targetServerResponse, err = targetservers.Export(conn); err != nil {
				return err
			}
			if err = apiclient.WriteArrayByteArrayToFile(environment+"_"+targetServerFileName, false, targetServerResponse); err != nil {
				return err
			}

			fmt.Printf("\tExporting KV Map names for environment %s...\n", environment)
			if respBody, err = kvm.List(""); err != nil {
				return err
			}
			if err = apiclient.WriteByteArrayToFile(environment+"_"+kVMFileName, false, respBody); err != nil {
				return err
			}

			fmt.Println("\tExporting Key store names...")
			if respBody, err = keystores.List(); err != nil {
				return err
			}
			if err = apiclient.WriteByteArrayToFile(environment+"_"+keyStoresFileName, false, respBody); err != nil {
				return err
			}

			fmt.Println("\tExporting debugmask configuration...")
			if respBody, err = env.GetDebug(); err != nil {
				return err
			}
			if err = apiclient.WriteByteArrayToFile(environment+debugmaskFileName, false, respBody); err != nil {
				return err
			}

			fmt.Println("\tExporting traceconfig...")
			if respBody, err = env.GetTraceConfig(); err != nil {
				return err
			}
			if err = apiclient.WriteByteArrayToFile(environment+tracecfgFileName, false, respBody); err != nil {
				return err
			}

			fmt.Println("\t Exporting references...")
			if referencesResponse, err = references.Export(conn); err != nil {
				return err
			}
			if err = apiclient.WriteArrayByteArrayToFile(environment+"_"+referencesFileName, false, referencesResponse); err != nil {
				return err
			}

		}

		return
	},
}

var allRevisions bool

func init() {

	ExportCmd.Flags().StringVarP(&org, "org", "o",
		"", "Apigee organization name")
	ExportCmd.Flags().IntVarP(&conn, "conn", "c",
		4, "Number of connections")
	ExportCmd.Flags().BoolVarP(&allRevisions, "all", "",
		false, "Export all revisions, default=false. Exports the latest revision")
}

func createFolders() (err error) {
	if err = os.Mkdir(proxiesFolderName, 0755); err != nil {
		return err
	}
	if err = os.Mkdir(sharedFlowsFolderName, 0755); err != nil {
		return err
	}
	return nil
}
<commit_msg>hide kvm export behind flag #62<commit_after>// Copyright 2021 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package org

import (
	"encoding/json"
	"fmt"
	"os"
	"strconv"
	"strings"

	"github.com/apigee/apigeecli/apiclient"
	"github.com/apigee/apigeecli/client/apis"
	"github.com/apigee/apigeecli/client/apps"
	"github.com/apigee/apigeecli/client/datacollectors"
	"github.com/apigee/apigeecli/client/developers"
	"github.com/apigee/apigeecli/client/env"
	"github.com/apigee/apigeecli/client/envgroups"
	"github.com/apigee/apigeecli/client/keystores"
	"github.com/apigee/apigeecli/client/kvm"
	"github.com/apigee/apigeecli/client/orgs"
	"github.com/apigee/apigeecli/client/products"
	"github.com/apigee/apigeecli/client/references"
	"github.com/apigee/apigeecli/client/sharedflows"
	"github.com/apigee/apigeecli/client/sync"
	"github.com/apigee/apigeecli/client/targetservers"
	"github.com/apigee/apigeecli/clilog"
	"github.com/spf13/cobra"
)

// ExportCmd to get org details
var ExportCmd = &cobra.Command{
	Use:   "export",
	Short: "Export Apigee Configuration",
	Long:  "Export Apigee Configuration",
	Args: func(cmd *cobra.Command, args []string) (err error) {
		return apiclient.SetApigeeOrg(org)
	},
	RunE: func(cmd *cobra.Command, args []string) (err error) {

		var productResponse, appsResponse, targetServerResponse, referencesResponse [][]byte
		var respBody, listKVMBytes []byte

		runtimeType, _ := orgs.GetOrgField("runtimeType")

		if err = createFolders(); err != nil {
			return err
		}

		clilog.Warning.Println("Calls to Apigee APIs have a quota of 6000 per min. Running this tool against large list of entities can exhaust that quota and impact the usage of the platform.")

		fmt.Println("Exporting API Proxies...")
		if err = apis.ExportProxies(conn, proxiesFolderName, allRevisions); err != nil {
			return err
		}

		fmt.Println("Exporting Sharedflows...")
		if err = sharedflows.Export(conn, sharedFlowsFolderName, allRevisions); err != nil {
			return err
		}

		fmt.Println("Exporting API Products...")
		if productResponse, err = products.Export(conn); err != nil {
			return err
		}
		if err = apiclient.WriteArrayByteArrayToFile(productsFileName, false, productResponse); err != nil {
			return err
		}

		fmt.Printf("\tExporting KV Map names for org %s\n", org)
		if listKVMBytes, err = kvm.List(""); err != nil {
			return err
		}
		if err = apiclient.WriteByteArrayToFile(org+"_"+kVMFileName, false, respBody); err != nil {
			return err
		}

		if err = exportKVMEntries("org", "", listKVMBytes); err != nil {
			return err
		}

		fmt.Println("Exporting Developers...")
		if respBody, err = developers.Export(); err != nil {
			return err
		}
		if err = apiclient.WriteByteArrayToFile(developersFileName, false, respBody); err != nil {
			return err
		}

		fmt.Println("Exporting Developer Apps...")
		if appsResponse, err = apps.Export(conn); err != nil {
			return err
		}
		if err = apiclient.WriteArrayByteArrayToFile(appsFileName, false, appsResponse); err != nil {
			return err
		}

		fmt.Println("Exporting Environment Group Configuration...")
		apiclient.SetPrintOutput(false)
		if respBody, err = envgroups.List(); err != nil {
			return err
		}
		if err = apiclient.WriteByteArrayToFile(envGroupsFileName, false, respBody); err != nil {
			return err
		}

		fmt.Println("Exporting Data collectors Configuration...")
		if respBody, err = datacollectors.List(); err != nil {
			return err
		}
		if err = apiclient.WriteByteArrayToFile(dataCollFileName, false, respBody); err != nil {
			return err
		}

		if runtimeType == "HYBRID" {
			fmt.Println("Exporting Sync Authorization Identities...")
			if respBody, err = sync.Get(); err != nil {
				return err
			}
			if err = apiclient.WriteByteArrayToFile(syncAuthFileName, false, respBody); err != nil {
				return err
			}
		}

		var envRespBody []byte
		if envRespBody, err = env.List(); err != nil {
			return err
		}

		environments := []string{}
		if err = json.Unmarshal(envRespBody, &environments); err != nil {
			return err

		}

		for _, environment := range environments {
			fmt.Println("Exporting configuration for environment " + environment)
			apiclient.SetApigeeEnv(environment)
			fmt.Println("\tExporting Target servers...")
			if targetServerResponse, err = targetservers.Export(conn); err != nil {
				return err
			}
			if err = apiclient.WriteArrayByteArrayToFile(environment+"_"+targetServerFileName, false, targetServerResponse); err != nil {
				return err
			}

			fmt.Printf("\tExporting KV Map names for environment %s...\n", environment)
			if listKVMBytes, err = kvm.List(""); err != nil {
				return err
			}
			if err = apiclient.WriteByteArrayToFile(environment+"_"+kVMFileName, false, respBody); err != nil {
				return err
			}

			if err = exportKVMEntries("env", environment, listKVMBytes); err != nil {
				return err
			}

			fmt.Println("\tExporting Key store names...")
			if respBody, err = keystores.List(); err != nil {
				return err
			}
			if err = apiclient.WriteByteArrayToFile(environment+"_"+keyStoresFileName, false, respBody); err != nil {
				return err
			}

			fmt.Println("\tExporting debugmask configuration...")
			if respBody, err = env.GetDebug(); err != nil {
				return err
			}
			if err = apiclient.WriteByteArrayToFile(environment+debugmaskFileName, false, respBody); err != nil {
				return err
			}

			fmt.Println("\tExporting traceconfig...")
			if respBody, err = env.GetTraceConfig(); err != nil {
				return err
			}
			if err = apiclient.WriteByteArrayToFile(environment+tracecfgFileName, false, respBody); err != nil {
				return err
			}

			fmt.Println("\t Exporting references...")
			if referencesResponse, err = references.Export(conn); err != nil {
				return err
			}
			if err = apiclient.WriteArrayByteArrayToFile(environment+"_"+referencesFileName, false, referencesResponse); err != nil {
				return err
			}

		}

		return
	},
}

var allRevisions bool

func init() {

	ExportCmd.Flags().StringVarP(&org, "org", "o",
		"", "Apigee organization name")
	ExportCmd.Flags().IntVarP(&conn, "conn", "c",
		4, "Number of connections")
	ExportCmd.Flags().BoolVarP(&allRevisions, "all", "",
		false, "Export all revisions, default=false. Exports the latest revision")
}

func createFolders() (err error) {
	if err = os.Mkdir(proxiesFolderName, 0755); err != nil {
		return err
	}
	if err = os.Mkdir(sharedFlowsFolderName, 0755); err != nil {
		return err
	}
	return nil
}

func exportKVMEntries(scope string, env string, listKVMBytes []byte) (err error) {

	var kvmEntries [][]byte
	var listKVM []string
	var fileName string

	//hide kvm exports behind a feature flag
	if os.Getenv("APIGEECLI_KVM_EXPORT") == "" {
		return nil
	}

	if err = json.Unmarshal(listKVMBytes, &listKVM); err != nil {
		return err
	}

	for _, mapName := range listKVM {

		fmt.Printf("\tExporting KVM entries for %s in org %s\n", org, mapName)
		if kvmEntries, err = kvm.ExportEntries("", mapName); err != nil {
			return err
		}

		if scope == "org" {
			fileName = strings.Join([]string{scope, mapName, "kvmfile"}, "_")
		} else if scope == "env" {
			fileName = strings.Join([]string{scope, env, mapName, "kvmfile"}, "_")
		}

		if len(kvmEntries) > 0 {
			for i := range kvmEntries {
				if err = apiclient.WriteByteArrayToFile(fileName+"_"+strconv.Itoa(i)+".json", false, kvmEntries[i]); err != nil {
					return err
				}
			}
		}
	}

	return nil
}
<|endoftext|>


<commit_before>package main // import "github.com/smotes/purse/cmd/purse"

import (
	"errors"
	"flag"
	"log"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"text/template"

	"github.com/smotes/purse"
)

const (
	envar = "GOPACKAGE"
)

var (
	in, out, file, name, pack string
)

func init() {
	flag.StringVar(&in, "in", "", "directory of the input SQL file(s)")
	flag.StringVar(&out, "out", "./", "directory of the output source file")
	flag.StringVar(&file, "file", "out.go", "name of the output source file")
	flag.StringVar(&name, "name", "gen", "variable name of the generated Purse struct")
	flag.StringVar(&pack, "pack", "", "name of the go package for the generated source file")
	flag.Parse()
}

func main() {
	validate(in, errors.New("must provide directory of input SQL file(s)"))
	if pack == "" {
		pack = os.Getenv(envar)
		validate(pack, errors.New("must provide the name of the go package for the generated source file"))
	}

	mp, err := purse.New(in)
	handle(err)

	data := make(map[string]string, len(mp.Files()))
	for _, name := range mp.Files() {
		s, ok := mp.Get(name)
		if !ok {
			log.Printf("Unable to get file %s", name)
			continue
		}
		data[name] = strconv.Quote(s)
	}

	ctx := &context{
		Varname: name,
		Package: pack,
		Files:   data,
	}

	cntnts := contentsHead + contentsBodyStruct + "\n" + contentsBodyVar

	if out != "./" {
		ctx.Varname = strings.Title(name)
		cntnts = contentsHead + contentsBodyVar

		tmplCommon, err := template.New(name).Parse(
				contentsHead + contentsBodyStruct)
		handle(err)

		fCommon, err := os.Create(filepath.Join(out, pack+".go"))
		handle(err)

		err = tmplCommon.Execute(fCommon, ctx)
		handle(err)
	}

	tmpl, err := template.New(name).Parse(cntnts)
	handle(err)

	f, err := os.Create(filepath.Join(out, file))
	handle(err)

	err = tmpl.Execute(f, ctx)
	handle(err)
}

func validate(s string, err error) {
	if s == "" {
		handle(err)
	}
}

func handle(err error) {
	if err != nil {
		log.Fatalf("%v", err)
	}
}

type context struct {
	Varname string
	Package string
	Files   map[string]string
}

const (
	contentsHead = `package {{.Package}}

`

	contentsBodyVar = `var {{.Varname}} = &GenPurse{
	files: map[string]string{
		{{range $key, $val := .Files}}
			"{{$key}}": {{$val}},
		{{end}}
	},
}
`

	contentsBodyStruct = `import (
	"sync"
)

// GenPurse is an literal implementation of a Purse that is programmatically generated
// from SQL file contents within a directory via go generate.
type GenPurse struct {
	mu sync.RWMutex
	files map[string]string
}

func (p *GenPurse) Get(filename string) (v string, ok bool) {
	p.mu.RLock()
	v, ok = p.files[filename]
	p.mu.RUnlock()
	return
}
`
)
<commit_msg>Updated template comments to satisfy golint, and to correct minor typo.<commit_after>package main // import "github.com/smotes/purse/cmd/purse"

import (
	"errors"
	"flag"
	"log"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"text/template"

	"github.com/smotes/purse"
)

const (
	envar = "GOPACKAGE"
)

var (
	in, out, file, name, pack string
)

func init() {
	flag.StringVar(&in, "in", "", "directory of the input SQL file(s)")
	flag.StringVar(&out, "out", "./", "directory of the output source file")
	flag.StringVar(&file, "file", "out.go", "name of the output source file")
	flag.StringVar(&name, "name", "gen", "variable name of the generated Purse struct")
	flag.StringVar(&pack, "pack", "", "name of the go package for the generated source file")
	flag.Parse()
}

func main() {
	validate(in, errors.New("must provide directory of input SQL file(s)"))
	if pack == "" {
		pack = os.Getenv(envar)
		validate(pack, errors.New("must provide the name of the go package for the generated source file"))
	}

	mp, err := purse.New(in)
	handle(err)

	data := make(map[string]string, len(mp.Files()))
	for _, name := range mp.Files() {
		s, ok := mp.Get(name)
		if !ok {
			log.Printf("Unable to get file %s", name)
			continue
		}
		data[name] = strconv.Quote(s)
	}

	ctx := &context{
		Varname: name,
		Package: pack,
		Files:   data,
	}

	cntnts := contentsHead + contentsBodyStruct + "\n" + contentsBodyVar

	if out != "./" {
		ctx.Varname = strings.Title(name)
		cntnts = contentsHead + contentsBodyVar

		tmplCommon, err := template.New(name).Parse(
				contentsHead + contentsBodyStruct)
		handle(err)

		fCommon, err := os.Create(filepath.Join(out, pack+".go"))
		handle(err)

		err = tmplCommon.Execute(fCommon, ctx)
		handle(err)
	}

	tmpl, err := template.New(name).Parse(cntnts)
	handle(err)

	f, err := os.Create(filepath.Join(out, file))
	handle(err)

	err = tmpl.Execute(f, ctx)
	handle(err)
}

func validate(s string, err error) {
	if s == "" {
		handle(err)
	}
}

func handle(err error) {
	if err != nil {
		log.Fatalf("%v", err)
	}
}

type context struct {
	Varname string
	Package string
	Files   map[string]string
}

const (
	contentsHead = `package {{.Package}}

`

	contentsBodyVar = `// {{.Varname}} is a *GenPurse.
	var {{.Varname}} = &GenPurse{
	files: map[string]string{
		{{range $key, $val := .Files}}
			"{{$key}}": {{$val}},
		{{end}}
	},
}
`

	contentsBodyStruct = `import (
	"sync"
)

// GenPurse is a literal implementation of a Purse that is programmatically
// generated from SQL file contents within a directory via go generate.
type GenPurse struct {
	mu sync.RWMutex
	files map[string]string
}

// Get takes a filename and returns a query if it is found within the relevant
// map.  If filename is not found, ok will return false.
func (p *GenPurse) Get(filename string) (v string, ok bool) {
	p.mu.RLock()
	v, ok = p.files[filename]
	p.mu.RUnlock()
	return
}
`
)
<|endoftext|>


<commit_before>package main

import (
	"os"
	"syscall"

	"github.com/Microsoft/hcsshim/internal/appargs"
	"github.com/urfave/cli"
)

// default action is to start a container
var runCommand = cli.Command{
	Name:  "run",
	Usage: "create and run a container",
	ArgsUsage: `<container-id>

Where "<container-id>" is your name for the instance of the container that you
are starting. The name you provide for the container instance must be unique on
your host.`,
	Description: `The run command creates an instance of a container for a bundle. The bundle
is a directory with a specification file named "` + specConfig + `" and a root
filesystem.

The specification file includes an args parameter. The args parameter is used
to specify command(s) that get run when the container is started. To change the
command(s) that get executed on start, edit the args parameter of the spec. See
"runc spec --help" for more explanation.`,
	Flags: append(createRunFlags,
		cli.BoolFlag{
			Name:  "detach, d",
			Usage: "detach from the container's process",
		},
	),
	Before: appargs.Validate(argID),
	Action: func(context *cli.Context) error {
		cfg, err := containerConfigFromContext(context)
		if err != nil {
			return err
		}
		c, err := createContainer(cfg)
		if err != nil {
			return err
		}
		if err != nil {
			return err
		}
		p, err := os.FindProcess(c.ShimPid)
		if err != nil {
			return err
		}
		err = c.Exec()
		if err != nil {
			return err
		}
		if !context.Bool("detach") {
			state, err := p.Wait()
			if err != nil {
				return err
			}
			c.Remove()
			os.Exit(int(state.Sys().(syscall.WaitStatus).ExitCode))
		}
		return nil
	},
}
<commit_msg>removed runc reference from run - modified to remove reference to runc spec (unimplemented in runhcs)<commit_after>package main

import (
	"os"
	"syscall"

	"github.com/Microsoft/hcsshim/internal/appargs"
	"github.com/urfave/cli"
)

// default action is to start a container
var runCommand = cli.Command{
	Name:  "run",
	Usage: "create and run a container",
	ArgsUsage: `<container-id>

Where "<container-id>" is your name for the instance of the container that you
are starting. The name you provide for the container instance must be unique on
your host.`,
	Description: `The run command creates an instance of a container for a bundle. The bundle
is a directory with a specification file named "` + specConfig + `" and a root
filesystem.

The specification file includes an args parameter. The args parameter is used
to specify command(s) that get run when the container is started. To change the
command(s) that get executed on start, edit the args parameter of the spec.`,
	Flags: append(createRunFlags,
		cli.BoolFlag{
			Name:  "detach, d",
			Usage: "detach from the container's process",
		},
	),
	Before: appargs.Validate(argID),
	Action: func(context *cli.Context) error {
		cfg, err := containerConfigFromContext(context)
		if err != nil {
			return err
		}
		c, err := createContainer(cfg)
		if err != nil {
			return err
		}
		if err != nil {
			return err
		}
		p, err := os.FindProcess(c.ShimPid)
		if err != nil {
			return err
		}
		err = c.Exec()
		if err != nil {
			return err
		}
		if !context.Bool("detach") {
			state, err := p.Wait()
			if err != nil {
				return err
			}
			c.Remove()
			os.Exit(int(state.Sys().(syscall.WaitStatus).ExitCode))
		}
		return nil
	},
}
<|endoftext|>