<commit_before>package main

import (
	"compress/gzip"
	"crypto/sha256"
	"encoding/hex"
	"io"
	"net/http"
	"strings"
	"sync"
	"time"

	"github.com/superp00t/etc"
	"github.com/superp00t/etc/yo"
)

type diskStatus struct {
	All  uint64 `json:"all"`
	Used uint64 `json:"used"`
	Free uint64 `json:"free"`
}

type cacher struct {
	Handler http.Handler

	sync.Mutex
}

func hashString(name string) string {
	s := sha256.New()
	s.Write([]byte(name))
	return strings.ToUpper(hex.EncodeToString(s.Sum(nil)))
}

func (c *cacher) Available() uint64 {
	return directory.Concat("c").Free()
}

type cachedResponseWriter struct {
	http.ResponseWriter

	gz *gzip.Writer
}

func (c *cachedResponseWriter) Write(b []byte) (int, error) {
	return c.gz.Write(b)
}

func (c *cacher) serveContent(rw http.ResponseWriter, r *http.Request, path string) {
	if strings.Contains(r.Header.Get("Accept-Ranges"), "-") {
		// Cannot serve compressed in this fashion
		http.ServeFile(rw, r, path)
		return
	}

	if strings.Contains(r.Header.Get("Accept-Encoding"), "gzip") {
		rw.Header().Set("Content-Encoding", "gzip")
		crw := &cachedResponseWriter{ResponseWriter: rw, gz: gzip.NewWriter(rw)}
		http.ServeFile(crw, r, path)
		crw.gz.Close()
		return
	}

	http.ServeFile(rw, r, path)
}

func (c *cacher) ServeHTTP(rw http.ResponseWriter, r *http.Request) {
	pth := r.URL.Path[1:]
	hash := hashString(pth)
	pCachePath := directory.Concat("c").Concat(hash)
	pSrcPath := directory.Concat("i").GetSub(etc.ParseUnixPath(pth))

	if pCachePath.IsExtant() && time.Since(pCachePath.Time()) < Config.CacheDuration.Duration {
		// cached file exists.
		c.serveContent(rw, r, pCachePath.Render())
		return
	}

	// Backend may be down. serve cached file in its place.
	if !pSrcPath.IsExtant() && pCachePath.IsExtant() {
		c.serveContent(rw, r, pCachePath.Render())
		return
	}

	if pSrcPath.IsExtant() == false {
		http.Error(rw, "file not found", 404)
		return
	}

	cacheDir := directory.Concat("c")

	// delete oldest item in cache if we have not enough space.
	for cacheDir.Free() < pSrcPath.Size() || cacheDir.Size() > Config.MaxCacheBytes {
		yo.Ok("erasing until bytes free are more than", cacheDir.Free())

		lru, err := cacheDir.LRU()
		if err != nil {
			yo.Warn(err)
			break
		}

		cacheDir.Concat(lru).Remove()
	}

	pCachePath.Remove()

	f, err := etc.FileController(pCachePath.Render())
	if err != nil {
		yo.Fatal(err)
	}

	if err = f.Flush(); err != nil {
		yo.Fatal(err)
	}

	s, err := etc.FileController(pSrcPath.Render(), true)
	if err != nil {
		yo.Fatal(err)
	}

	if _, err = io.Copy(f, s); err != nil {
		yo.Fatal(err)
	}

	f.Close()
	s.Close()

	c.serveContent(rw, r, pCachePath.Render())
}
<commit_msg>simplify encoding<commit_after>package main

import (
	"compress/gzip"
	"crypto/sha256"
	"encoding/hex"
	"io"
	"net/http"
	"strings"
	"sync"
	"time"

	"github.com/superp00t/etc"
	"github.com/superp00t/etc/yo"
)

type diskStatus struct {
	All  uint64 `json:"all"`
	Used uint64 `json:"used"`
	Free uint64 `json:"free"`
}

type cacher struct {
	Handler http.Handler

	sync.Mutex
}

func hashString(name string) string {
	s := sha256.New()
	s.Write([]byte(name))
	return strings.ToUpper(hex.EncodeToString(s.Sum(nil)))
}

func (c *cacher) Available() uint64 {
	return directory.Concat("c").Free()
}

func (c *cacher) serveContent(rw http.ResponseWriter, r *http.Request, path string) {
	if strings.Contains(r.Header.Get("Accept-Ranges"), "-") {
		// Cannot serve compressed in this fashion
		http.ServeFile(rw, r, path)
		return
	}

	if strings.Contains(r.Header.Get("Accept-Encoding"), "gzip") {
		rw.Header().Set("Content-Encoding", "gzip")

		file, err := etc.FileController(path, true)
		if err != nil {
			return
		}

		tp := http.DetectContentType(file.ReadBytes(512))

		rw.Header().Set("Content-Type", tp)

		file.SeekR(0)

		gz := gzip.NewWriter(rw)
		io.Copy(gz, file)
		gz.Close()
		file.Close()
		return
	}

	http.ServeFile(rw, r, path)
}

func (c *cacher) ServeHTTP(rw http.ResponseWriter, r *http.Request) {
	pth := r.URL.Path[1:]
	hash := hashString(pth)
	pCachePath := directory.Concat("c").Concat(hash)
	pSrcPath := directory.Concat("i").GetSub(etc.ParseUnixPath(pth))

	if pCachePath.IsExtant() && time.Since(pCachePath.Time()) < Config.CacheDuration.Duration {
		// cached file exists.
		c.serveContent(rw, r, pCachePath.Render())
		return
	}

	// Backend may be down. serve cached file in its place.
	if !pSrcPath.IsExtant() && pCachePath.IsExtant() {
		c.serveContent(rw, r, pCachePath.Render())
		return
	}

	if pSrcPath.IsExtant() == false {
		http.Error(rw, "file not found", 404)
		return
	}

	cacheDir := directory.Concat("c")

	// delete oldest item in cache if we have not enough space.
	for cacheDir.Free() < pSrcPath.Size() || cacheDir.Size() > Config.MaxCacheBytes {
		yo.Ok("erasing until bytes free are more than", cacheDir.Free())

		lru, err := cacheDir.LRU()
		if err != nil {
			yo.Warn(err)
			break
		}

		cacheDir.Concat(lru).Remove()
	}

	pCachePath.Remove()

	f, err := etc.FileController(pCachePath.Render())
	if err != nil {
		yo.Fatal(err)
	}

	if err = f.Flush(); err != nil {
		yo.Fatal(err)
	}

	s, err := etc.FileController(pSrcPath.Render(), true)
	if err != nil {
		yo.Fatal(err)
	}

	if _, err = io.Copy(f, s); err != nil {
		yo.Fatal(err)
	}

	f.Close()
	s.Close()

	c.serveContent(rw, r, pCachePath.Render())
}
<|endoftext|>


<commit_before>package fastentity

import (
	"bufio"
	"errors"
	"fmt"
	"io/ioutil"
	"os"
	"regexp"
	"strings"
	"sync"
	"unicode"
)

var (
	MAX_ENTITY_LEN     = 30
	DEFAULT_GROUP_SIZE = 1000
)

const (
	LEFT  = 0
	RIGHT = 1
)

type Pair [2]int

type Store struct {
	Lookup map[string]*Group
	sync.RWMutex
}

type Group struct {
	Name     string
	Entities map[string][][]rune
	Max_len  int
	sync.RWMutex
}

// Pops the last element and adds the new element
// to the front of stack.
func shift(n Pair, s []Pair) (Pair, []Pair) {
	if len(s) == 0 {
		return Pair{}, append(s, n)
	}
	if len(s) == cap(s) {
		return s[0], append(s[1:], n)
	}
	return s[0], append(s, n)
}

// Create a new entity group structure
func Init(groups ...string) *Store {
	store := new(Store)
	store.Lookup = make(map[string]*Group, len(groups))
	for _, name := range groups {
		group := &Group{
			Name:     name,
			Entities: make(map[string][][]rune, DEFAULT_GROUP_SIZE),
		}
		store.Lookup[name] = group
	}
	return store
}

// Add a new entity to a particular group
func (store *Store) Add(name string, entities ...[]rune) {
	if store.Lookup == nil {
		panic("You need to initialize the store before adding to it...")
	}

	store.Lock()
	group, ok := store.Lookup[name]
	if !ok {
		group = &Group{
			Name:     name,
			Entities: make(map[string][][]rune, DEFAULT_GROUP_SIZE),
		}
		store.Lookup[name] = group
	}
	store.Unlock()

	group.Lock()
	for _, e := range entities {
		h := hash([]rune(e))
		group.Entities[h] = append(group.Entities[h], e)
		if len(e) > group.Max_len {
			group.Max_len = len(e)
		}
	}
	group.Unlock()
}

// Take the string and turn it into a hash
func hash(rs []rune) string {
	if len(rs) > 2 {
		return fmt.Sprintf("%s%s%s%03d", string(unicode.ToLower(rs[0])), string(unicode.ToLower(rs[1])), string(unicode.ToLower(rs[2])), len(rs))
	}
	if len(rs) > 1 {
		return fmt.Sprintf("%s%s%03d", string(unicode.ToLower(rs[0])), string(unicode.ToLower(rs[1])), len(rs))
	}
	return fmt.Sprintf("%s%03d", string(unicode.ToLower(rs[0])), len(rs))
}

// Find all entities for all type keys
func (store *Store) FindAll(rs []rune) map[string][][]rune {
	result := make(map[string][][]rune, len(store.Lookup))
	for name, group := range store.Lookup {
		result[name] = group.Find(rs)
	}
	return result
}

/*
// Find all entities for all type keys in parallel (slower)
func (store *Store) FindAll(rs []rune) map[string][][]rune {
	result := make(map[string][][]rune, len(store.Lookup))
	var wg sync.WaitGroup
	for name, group := range store.Lookup {
		wg.Add(1)
		go func(name string, group *Group) {
			result[name] = group.Find(rs)
			wg.Done()
		}(name, group)
	}
	wg.Wait()
	return result
}
*/

/*
// Find all together (slower)
func (store *Store) FindAll(rs []rune) map[string][][]rune {
	groups := make([]*Group, 0, len(store.Lookup))
	for _, group := range store.Lookup {
		groups = append(groups, group)
	}
	store.RLock()
	results := _find(rs, groups...)
	store.RUnlock()
	return results
}
*/

// Find only the entities of a given type = "key"
func (group *Group) Find(rs []rune) [][]rune {
	group.RLock()
	ents := _find(rs, group)
	group.RUnlock()
	return ents[group.Name]
}

// Lock free find for use internally
func _find(rs []rune, groups ...*Group) map[string][][]rune {

	results := make(map[string][][]rune, len(groups))

	pairs := make([]Pair, 0, 20)
	start := 0
	prevspace := true // First char of sequence is legit
	thisspace := false
	var p1, p2 Pair
	for off, r := range rs {

		// What are we looking at?
		thisspace = unicode.IsPunct(r) || unicode.IsSpace(r)

		if prevspace && !thisspace {
			// Word is beginning at this rune
			start = off
		} else if thisspace && !prevspace {
			// Word is ending, shift the pairs stack
			_, pairs = shift(Pair{start, off}, pairs)

			// Run the stack, check for entities working backwards from the current position
			if len(pairs) > 1 {
				p2 = pairs[len(pairs)-1]
				for i := len(pairs) - 1; i >= 0; i-- {
					p1 = pairs[i]
					if p2[RIGHT]-p1[LEFT] > MAX_ENTITY_LEN {
						break // Too long or short, can ignore it
					}
					for _, group := range groups {
						if p2[RIGHT]-p1[LEFT] > group.Max_len {
							continue
						}
						if ents, ok := group.Entities[hash(rs[p1[LEFT]:p2[RIGHT]])]; ok {
							// We have at least one entity with this key
							for _, ent := range ents {
								if len(ent) != p2[RIGHT]-p1[LEFT] {
									break
								}
								match := true
								for i, r := range ent {
									if unicode.ToLower(r) != unicode.ToLower(rs[p1[LEFT]+i]) {
										match = false
										break
									}
								}
								if match {
									results[group.Name] = append(results[group.Name], rs[p1[LEFT]:p2[RIGHT]])
								}
							}
						}
					}
				}
			}

		}

		// Mark prevspace for the next loop
		if thisspace {
			prevspace = true
		} else {
			prevspace = false
		}
	}

	return results
}

type incr struct {
	sync.Mutex
	n int
}

func (i *incr) incr() {
	i.Lock()
	i.n++
	i.Unlock()
}

// Create a new store by loading entity files from a given directory. The format
// expected has the format "<GROUP>.entities.csv"
func Load(dir string) (*Store, error) {
	dir = strings.TrimRight(dir, "/")
	store := Init()
	reGroupFile, _ := regexp.Compile("^(.+).entities.csv")
	var wg sync.WaitGroup
	fileCount := &incr{}
	if files, err := ioutil.ReadDir(dir); err == nil {
		for _, fileInfo := range files {
			if m := reGroupFile.FindStringSubmatch(fileInfo.Name()); m != nil {
				wg.Add(1)
				go func(filename string, group string) {
					if file, err := os.Open(filename); err == nil {
						fileCount.incr()
						defer file.Close()
						reader := bufio.NewScanner(file)
						for reader.Scan() {
							store.Add(group, []rune(reader.Text()))
						}
					} else {
						fmt.Printf("Unable to load \"%s\" entity file: %s: %s\n", group, filename, err.Error())
					}
					wg.Done()
				}(fmt.Sprintf("%s/%s", dir, fileInfo.Name()), m[1])
			}
		}
	}
	wg.Wait()
	if fileCount.n == 0 {
		return store, errors.New("There are no entity files")
	}
	return store, nil
}

// Save the existing entities to disk. Each group becomes a file with the format
// the format "<GROUP>.entities.csv" in the dir specified.
func (store *Store) Save(dir string) error {
	store.RLock()
	defer store.RUnlock()
	dir = strings.TrimRight(dir, "/")
	for name, group := range store.Lookup {
		filename := fmt.Sprintf("%s/%s", dir, strings.Replace(name, "/", "_", -1)+".entities.csv")
		if file, err := os.Create(filename); err != nil {
			// Failed
			return err
		} else {
			w := bufio.NewWriter(file)
			for _, entities := range group.Entities {
				for _, entity := range entities {
					w.WriteString(string(entity) + "\n")
				}
			}
			w.Flush()
			file.Close()
		}
	}
	return nil
}
<commit_msg>Refactor variable names to remove underscores and ALL_CAPS.<commit_after>package fastentity

import (
	"bufio"
	"errors"
	"fmt"
	"io/ioutil"
	"os"
	"regexp"
	"strings"
	"sync"
	"unicode"
)

var (
	MaxEntityLen = 30
	DefaultGroupSize = 1000
)

const (
	left  = 0
	right = 1
)

type Pair [2]int

type Store struct {
	Lookup map[string]*Group
	sync.RWMutex
}

type Group struct {
	Name     string
	Entities map[string][][]rune
	MaxLen   int
	sync.RWMutex
}

// Pops the last element and adds the new element
// to the front of stack.
func shift(n Pair, s []Pair) (Pair, []Pair) {
	if len(s) == 0 {
		return Pair{}, append(s, n)
	}
	if len(s) == cap(s) {
		return s[0], append(s[1:], n)
	}
	return s[0], append(s, n)
}

// Create a new entity group structure
func Init(groups ...string) *Store {
	store := new(Store)
	store.Lookup = make(map[string]*Group, len(groups))
	for _, name := range groups {
		group := &Group{
			Name:     name,
			Entities: make(map[string][][]rune, DefaultGroupSize),
		}
		store.Lookup[name] = group
	}
	return store
}

// Add a new entity to a particular group
func (store *Store) Add(name string, entities ...[]rune) {
	if store.Lookup == nil {
		panic("You need to initialize the store before adding to it...")
	}

	store.Lock()
	group, ok := store.Lookup[name]
	if !ok {
		group = &Group{
			Name:     name,
			Entities: make(map[string][][]rune, DefaultGroupSize),
		}
		store.Lookup[name] = group
	}
	store.Unlock()

	group.Lock()
	for _, e := range entities {
		h := hash([]rune(e))
		group.Entities[h] = append(group.Entities[h], e)
		if len(e) > group.MaxLen {
			group.MaxLen = len(e)
		}
	}
	group.Unlock()
}

// Take the string and turn it into a hash
func hash(rs []rune) string {
	if len(rs) > 2 {
		return fmt.Sprintf("%s%s%s%03d", string(unicode.ToLower(rs[0])), string(unicode.ToLower(rs[1])), string(unicode.ToLower(rs[2])), len(rs))
	}
	if len(rs) > 1 {
		return fmt.Sprintf("%s%s%03d", string(unicode.ToLower(rs[0])), string(unicode.ToLower(rs[1])), len(rs))
	}
	return fmt.Sprintf("%s%03d", string(unicode.ToLower(rs[0])), len(rs))
}

// Find all entities for all type keys
func (store *Store) FindAll(rs []rune) map[string][][]rune {
	result := make(map[string][][]rune, len(store.Lookup))
	for name, group := range store.Lookup {
		result[name] = group.Find(rs)
	}
	return result
}

/*
// Find all entities for all type keys in parallel (slower)
func (store *Store) FindAll(rs []rune) map[string][][]rune {
	result := make(map[string][][]rune, len(store.Lookup))
	var wg sync.WaitGroup
	for name, group := range store.Lookup {
		wg.Add(1)
		go func(name string, group *Group) {
			result[name] = group.Find(rs)
			wg.Done()
		}(name, group)
	}
	wg.Wait()
	return result
}
*/

/*
// Find all together (slower)
func (store *Store) FindAll(rs []rune) map[string][][]rune {
	groups := make([]*Group, 0, len(store.Lookup))
	for _, group := range store.Lookup {
		groups = append(groups, group)
	}
	store.RLock()
	results := find(rs, groups...)
	store.RUnlock()
	return results
}
*/

// Find only the entities of a given type = "key"
func (group *Group) Find(rs []rune) [][]rune {
	group.RLock()
	ents := find(rs, group)
	group.RUnlock()
	return ents[group.Name]
}

// Lock free find for use internally
func find(rs []rune, groups ...*Group) map[string][][]rune {

	results := make(map[string][][]rune, len(groups))

	pairs := make([]Pair, 0, 20)
	start := 0
	prevspace := true // First char of sequence is legit
	thisspace := false
	var p1, p2 Pair
	for off, r := range rs {

		// What are we looking at?
		thisspace = unicode.IsPunct(r) || unicode.IsSpace(r)

		if prevspace && !thisspace {
			// Word is beginning at this rune
			start = off
		} else if thisspace && !prevspace {
			// Word is ending, shift the pairs stack
			_, pairs = shift(Pair{start, off}, pairs)

			// Run the stack, check for entities working backwards from the current position
			if len(pairs) > 1 {
				p2 = pairs[len(pairs)-1]
				for i := len(pairs) - 1; i >= 0; i-- {
					p1 = pairs[i]
					if p2[right]-p1[left] > MaxEntityLen {
						break // Too long or short, can ignore it
					}
					for _, group := range groups {
						if p2[right]-p1[left] > group.MaxLen {
							continue
						}
						if ents, ok := group.Entities[hash(rs[p1[left]:p2[right]])]; ok {
							// We have at least one entity with this key
							for _, ent := range ents {
								if len(ent) != p2[right]-p1[left] {
									break
								}
								match := true
								for i, r := range ent {
									if unicode.ToLower(r) != unicode.ToLower(rs[p1[left]+i]) {
										match = false
										break
									}
								}
								if match {
									results[group.Name] = append(results[group.Name], rs[p1[left]:p2[right]])
								}
							}
						}
					}
				}
			}

		}

		// Mark prevspace for the next loop
		if thisspace {
			prevspace = true
		} else {
			prevspace = false
		}
	}

	return results
}

type incr struct {
	sync.Mutex
	n int
}

func (i *incr) incr() {
	i.Lock()
	i.n++
	i.Unlock()
}

// Create a new store by loading entity files from a given directory. The format
// expected has the format "<GROUP>.entities.csv"
func Load(dir string) (*Store, error) {
	dir = strings.TrimRight(dir, "/")
	store := Init()
	reGroupFile, _ := regexp.Compile("^(.+).entities.csv")
	var wg sync.WaitGroup
	fileCount := &incr{}
	if files, err := ioutil.ReadDir(dir); err == nil {
		for _, fileInfo := range files {
			if m := reGroupFile.FindStringSubmatch(fileInfo.Name()); m != nil {
				wg.Add(1)
				go func(filename string, group string) {
					if file, err := os.Open(filename); err == nil {
						fileCount.incr()
						defer file.Close()
						reader := bufio.NewScanner(file)
						for reader.Scan() {
							store.Add(group, []rune(reader.Text()))
						}
					} else {
						fmt.Printf("Unable to load \"%s\" entity file: %s: %s\n", group, filename, err.Error())
					}
					wg.Done()
				}(fmt.Sprintf("%s/%s", dir, fileInfo.Name()), m[1])
			}
		}
	}
	wg.Wait()
	if fileCount.n == 0 {
		return store, errors.New("There are no entity files")
	}
	return store, nil
}

// Save the existing entities to disk. Each group becomes a file with the format
// the format "<GROUP>.entities.csv" in the dir specified.
func (store *Store) Save(dir string) error {
	store.RLock()
	defer store.RUnlock()
	dir = strings.TrimRight(dir, "/")
	for name, group := range store.Lookup {
		filename := fmt.Sprintf("%s/%s", dir, strings.Replace(name, "/", "_", -1)+".entities.csv")
		if file, err := os.Create(filename); err != nil {
			// Failed
			return err
		} else {
			w := bufio.NewWriter(file)
			for _, entities := range group.Entities {
				for _, entity := range entities {
					w.WriteString(string(entity) + "\n")
				}
			}
			w.Flush()
			file.Close()
		}
	}
	return nil
}
<|endoftext|>


<commit_before>package stack

import (
	"fmt"
	"testing"
)

func TestStack(t *testing.T) {
	if (push())
}

func ExampleStack() {

}
<commit_msg>del stack_test.go<commit_after><|endoftext|>


<commit_before>package main

import (
	"encoding/xml"
	"flag"
	"fmt"
	"log"
	"os"
	"net/http"
	"net/url"
	"sort"
	"strconv"
	"strings"
	"time"
)

func usage() {
	fmt.Fprintf(os.Stderr, `usage: issue [-p project] query

If query is a single number, prints the full history for the issue.
Otherwise, prints a table of matching results.
The special query 'go1' is shorthand for 'Priority-Go1'.
`)
}

type Feed struct {
	Entry Entries `xml:"entry"`
}

type Entry struct {
	ID string `xml:"id"`
	Title string `xml:"title"`
	Published time.Time `xml:"published"`
	Content string `xml:"content"`
	Updates []Update `xml:"updates"`
}

type Update struct {
	Summary string `xml:"summary"`
	Owner string `xml:"ownerUpdate"`
	Label string `xml:"label"`
	Status string `xml:"status"`
}

type Entries []Entry

func (e Entries) Len() int { return len(e) }
func (e Entries) Swap(i, j int) { e[i], e[j] = e[j], e[i] }
func (e Entries) Less(i, j int) bool { return e[i].Title < e[j].Title }

var project = flag.String("p", "go", "code.google.com project identifier")
var v = flag.Bool("v", false, "verbose")

func main() {
	flag.Usage = usage
	flag.Parse()
	if flag.NArg() != 1 {
		usage()
	}

	full := false
	q := flag.Arg(0)
	n, _ := strconv.Atoi(q)
	if n != 0 {
		q = "id:"+q
		full = true
	}
	if q == "go1" {
		q = "label:Priority-Go1"
	}

	log.SetFlags(0)

	query := url.Values{
		"q": {q},
		"max-results": {"400"},
	}
	if !full {
		query["can"] = []string{"open"}
	}
	u := "https://code.google.com/feeds/issues/p/"+*project+"/issues/full?" + query.Encode()
	if *v {
		log.Print(u)
	}
	r, err := http.Get(u)
	if err != nil {
		log.Fatal(err)
	}

	var feed Feed
	if err := xml.NewDecoder(r.Body).Decode(&feed); err != nil {
		log.Fatal(err)
	}
	r.Body.Close()

	sort.Sort(feed.Entry)
	for _, e := range feed.Entry {
		id := e.ID
		if i := strings.Index(id, "id="); i >= 0 {
			id = id[:i+len("id=")]
		}
		fmt.Printf("%s\t%s\n", id, e.Title)
		if full {
			u := "https://code.google.com/feeds/issues/p/"+*project+"/issues/"+id+"/comments/full"
			r, err := http.Get(u)
			if err != nil {
				log.Fatal(err)
			}

			var feed Feed
			if err := xml.NewDecoder(r.Body).Decode(&feed); err != nil {
				log.Fatal(err)
			}
			r.Body.Close()

			for _, e := range feed.Entry {
				fmt.Printf("\n%s (%s)\n", e.Title, e.Published.Format("2006-01-02 15:04:05"))
				for _, up := range e.Updates {
					switch {
					case up.Summary != "":
						fmt.Printf("\tSummary: %s\n", up.Summary)
					case up.Owner != "":
						fmt.Printf("\tOwner: %s\n", up.Owner)
					case up.Status != "":
						fmt.Printf("\tStatus: %s\n", up.Status)
					case up.Label != "":
						fmt.Printf("\tLabel: %s\n", up.Label)
					}
				}
				if e.Content != "" {
					fmt.Printf("\n\t%s\n", wrap(e.Content, "\t"))
				}
			}
		}
	}
}

func wrap(t string, prefix string) string {
	out := ""
	s := t
	for len(s) > 70 {
		i := strings.LastIndex(s[:70], " ")
		if i < 0 {
			i = 69
		}
		i++
		out += s[:i] + "\n" + prefix
		s = s[i:]
	}
	return out + s
}
<commit_msg>issue: handle multiple lines in wrap<commit_after>package main

import (
	"encoding/xml"
	"flag"
	"fmt"
	"log"
	"net/http"
	"net/url"
	"os"
	"sort"
	"strconv"
	"strings"
	"time"
)

func usage() {
	fmt.Fprintf(os.Stderr, `usage: issue [-p project] query

If query is a single number, prints the full history for the issue.
Otherwise, prints a table of matching results.
The special query 'go1' is shorthand for 'Priority-Go1'.
`)
	os.Exit(2)
}

type Feed struct {
	Entry Entries `xml:"entry"`
}

type Entry struct {
	ID        string    `xml:"id"`
	Title     string    `xml:"title"`
	Published time.Time `xml:"published"`
	Content   string    `xml:"content"`
	Updates   []Update  `xml:"updates"`
}

type Update struct {
	Summary string `xml:"summary"`
	Owner   string `xml:"ownerUpdate"`
	Label   string `xml:"label"`
	Status  string `xml:"status"`
}

type Entries []Entry

func (e Entries) Len() int           { return len(e) }
func (e Entries) Swap(i, j int)      { e[i], e[j] = e[j], e[i] }
func (e Entries) Less(i, j int) bool { return e[i].Title < e[j].Title }

var project = flag.String("p", "go", "code.google.com project identifier")
var v = flag.Bool("v", false, "verbose")

func main() {
	flag.Usage = usage
	flag.Parse()
	if flag.NArg() != 1 {
		usage()
	}

	full := false
	q := flag.Arg(0)
	n, _ := strconv.Atoi(q)
	if n != 0 {
		q = "id:" + q
		full = true
	}
	if q == "go1" {
		q = "label:Priority-Go1"
	}

	log.SetFlags(0)

	query := url.Values{
		"q":           {q},
		"max-results": {"400"},
	}
	if !full {
		query["can"] = []string{"open"}
	}
	u := "https://code.google.com/feeds/issues/p/" + *project + "/issues/full?" + query.Encode()
	if *v {
		log.Print(u)
	}
	r, err := http.Get(u)
	if err != nil {
		log.Fatal(err)
	}

	var feed Feed
	if err := xml.NewDecoder(r.Body).Decode(&feed); err != nil {
		log.Fatal(err)
	}
	r.Body.Close()

	sort.Sort(feed.Entry)
	for _, e := range feed.Entry {
		id := e.ID
		if i := strings.Index(id, "id="); i >= 0 {
			id = id[:i+len("id=")]
		}
		fmt.Printf("%s\t%s\n", id, e.Title)
		if full {
			u := "https://code.google.com/feeds/issues/p/" + *project + "/issues/" + id + "/comments/full"
			if *v {
				log.Print(u)
			}
			r, err := http.Get(u)
			if err != nil {
				log.Fatal(err)
			}

			var feed Feed
			if err := xml.NewDecoder(r.Body).Decode(&feed); err != nil {
				log.Fatal(err)
			}
			r.Body.Close()

			for _, e := range feed.Entry {
				fmt.Printf("\n%s (%s)\n", e.Title, e.Published.Format("2006-01-02 15:04:05"))
				for _, up := range e.Updates {
					switch {
					case up.Summary != "":
						fmt.Printf("\tSummary: %s\n", up.Summary)
					case up.Owner != "":
						fmt.Printf("\tOwner: %s\n", up.Owner)
					case up.Status != "":
						fmt.Printf("\tStatus: %s\n", up.Status)
					case up.Label != "":
						fmt.Printf("\tLabel: %s\n", up.Label)
					}
				}
				if e.Content != "" {
					fmt.Printf("\n\t%s\n", wrap(e.Content, "\t"))
				}
			}
		}
	}
}

func wrap(t string, prefix string) string {
	out := ""
	t = strings.Replace(t, "\r\n", "\n", -1)
	lines := strings.Split(t, "\n")
	for i, line := range lines {
		if i > 0 {
			out += "\n" + prefix
		}
		s := line
		for len(s) > 70 {
			i := strings.LastIndex(s[:70], " ")
			if i < 0 {
				i = 69
			}
			i++
			out += s[:i] + "\n" + prefix
			s = s[i:]
		}
		out += s
	}
	return out
}
<|endoftext|>


<commit_before>package rabbithole

import (
	"net/http"
)

// Federation definition: additional arguments
// added to the entities (queues, exchanges or both)
// that match a policy.
type FederationDefinition struct {
	Uri            string `json:"uri"`
	Expires        int    `json:"expires"`
	MessageTTL     int32  `json:"message-ttl"`
	MaxHops        int    `json:"max-hops"`
	PrefetchCount  int    `json:"prefetch-count"`
	ReconnectDelay int    `json:"reconnect-delay"`
	AckMode        string `json:"ack-mode,omitempty"`
	TrustUserId    bool   `json:"trust-user-id"`
	Exchange       string `json:"exchange"`
	Queue          string `json:"queue"`
}

// Represents a configured Federation upstream.
type FederationUpstream struct {
	Name       string               `json:"name"`
	Vhost      string               `json:"vhost"`
	Component  string               `json:"component"`
	Definition FederationDefinition `json:"value"`
}

//
// GET /api/parameters/federation-upstream
//

// ListFederationUpstreams returns all federation upstreams
func (c *Client) ListFederationUpstreams() (ups []FederationUpstream, err error) {
	params, err := c.ListRuntimeParametersFor("federation-upstream")
	if err != nil {
		return nil, err
	}

	ups = []FederationUpstream{}
	for _, param := range params {
		up := paramToUpstream(&param)
		ups = append(ups, *up)
	}
	return ups, nil
}

//
// GET /api/parameters/federation-upstream/{vhost}
//

// ListFederationUpstreamsIn returns all federation upstreams in a vhost
func (c *Client) ListFederationUpstreamsIn(vhost string) (ups []FederationUpstream, err error) {
	params, err := c.ListRuntimeParametersIn("federation-upstream", vhost)
	if err != nil {
		return nil, err
	}

	ups = []FederationUpstream{}
	for _, param := range params {
		up := paramToUpstream(&param)
		ups = append(ups, *up)
	}
	return ups, nil
}

//
// GET /api/parameters/federation-upstream/{vhost}/{upstream}
//

// GetFederationUpstream returns a federation upstream
func (c *Client) GetFederationUpstream(vhost, name string) (up *FederationUpstream, err error) {
	param, err := c.GetRuntimeParameter("federation-upstream", vhost, name)
	if err != nil {
		return nil, err
	}
	return paramToUpstream(param), nil
}

//
// PUT /api/parameters/federation-upstream/{vhost}/{upstream}
//

// Updates a federation upstream
func (c *Client) PutFederationUpstream(vhost string, name string, def FederationDefinition) (res *http.Response, err error) {
	return c.PutRuntimeParameter("federation-upstream", vhost, name, def)
}

//
// DELETE /api/parameters/federation-upstream/{vhost}/{name}
//

// Deletes a federation upstream.
func (c *Client) DeleteFederationUpstream(vhost, name string) (res *http.Response, err error) {
	return c.DeleteRuntimeParameter("federation-upstream", vhost, name)
}

// paramToUpstream maps from a RuntimeParameter structure to a FederationUpstream structure.
func paramToUpstream(p *RuntimeParameter) (up *FederationUpstream) {
	up = &FederationUpstream{
		Name:      p.Name,
		Vhost:     p.Vhost,
		Component: p.Component,
	}

	def := FederationDefinition{}
	m := p.Value.(map[string]interface{})

	if v, ok := m["uri"].(string); ok {
		def.Uri = v
	}

	if v, ok := m["expires"].(float64); ok {
		def.Expires = int(v)
	}

	if v, ok := m["message-ttl"].(float64); ok {
		def.MessageTTL = int32(v)
	}

	if v, ok := m["max-hops"].(float64); ok {
		def.MaxHops = int(v)
	}

	if v, ok := m["prefetch-count"].(float64); ok {
		def.PrefetchCount = int(v)
	}

	if v, ok := m["reconnect-delay"].(float64); ok {
		def.ReconnectDelay = int(v)
	}

	if v, ok := m["ack-mode"].(string); ok {
		def.AckMode = v
	}

	if v, ok := m["trust-user-id"].(bool); ok {
		def.TrustUserId = v
	}

	if v, ok := m["exchange"].(string); ok {
		def.Exchange = v
	}

	if v, ok := m["queue"].(string); ok {
		def.Queue = v
	}

	up.Definition = def
	return up
}
<commit_msg>Remove redundant initialization.<commit_after>package rabbithole

import (
	"net/http"
)

// Federation definition: additional arguments
// added to the entities (queues, exchanges or both)
// that match a policy.
type FederationDefinition struct {
	Uri            string `json:"uri"`
	Expires        int    `json:"expires"`
	MessageTTL     int32  `json:"message-ttl"`
	MaxHops        int    `json:"max-hops"`
	PrefetchCount  int    `json:"prefetch-count"`
	ReconnectDelay int    `json:"reconnect-delay"`
	AckMode        string `json:"ack-mode,omitempty"`
	TrustUserId    bool   `json:"trust-user-id"`
	Exchange       string `json:"exchange"`
	Queue          string `json:"queue"`
}

// Represents a configured Federation upstream.
type FederationUpstream struct {
	Name       string               `json:"name"`
	Vhost      string               `json:"vhost"`
	Component  string               `json:"component"`
	Definition FederationDefinition `json:"value"`
}

//
// GET /api/parameters/federation-upstream
//

// ListFederationUpstreams returns all federation upstreams
func (c *Client) ListFederationUpstreams() (ups []FederationUpstream, err error) {
	params, err := c.ListRuntimeParametersFor("federation-upstream")
	if err != nil {
		return nil, err
	}

	for _, param := range params {
		up := paramToUpstream(&param)
		ups = append(ups, *up)
	}
	return ups, nil
}

//
// GET /api/parameters/federation-upstream/{vhost}
//

// ListFederationUpstreamsIn returns all federation upstreams in a vhost
func (c *Client) ListFederationUpstreamsIn(vhost string) (ups []FederationUpstream, err error) {
	params, err := c.ListRuntimeParametersIn("federation-upstream", vhost)
	if err != nil {
		return nil, err
	}

	for _, param := range params {
		up := paramToUpstream(&param)
		ups = append(ups, *up)
	}
	return ups, nil
}

//
// GET /api/parameters/federation-upstream/{vhost}/{upstream}
//

// GetFederationUpstream returns a federation upstream
func (c *Client) GetFederationUpstream(vhost, name string) (up *FederationUpstream, err error) {
	param, err := c.GetRuntimeParameter("federation-upstream", vhost, name)
	if err != nil {
		return nil, err
	}
	return paramToUpstream(param), nil
}

//
// PUT /api/parameters/federation-upstream/{vhost}/{upstream}
//

// Updates a federation upstream
func (c *Client) PutFederationUpstream(vhost string, name string, def FederationDefinition) (res *http.Response, err error) {
	return c.PutRuntimeParameter("federation-upstream", vhost, name, def)
}

//
// DELETE /api/parameters/federation-upstream/{vhost}/{name}
//

// Deletes a federation upstream.
func (c *Client) DeleteFederationUpstream(vhost, name string) (res *http.Response, err error) {
	return c.DeleteRuntimeParameter("federation-upstream", vhost, name)
}

// paramToUpstream maps from a RuntimeParameter structure to a FederationUpstream structure.
func paramToUpstream(p *RuntimeParameter) (up *FederationUpstream) {
	up = &FederationUpstream{
		Name:      p.Name,
		Vhost:     p.Vhost,
		Component: p.Component,
	}

	def := FederationDefinition{}
	m := p.Value.(map[string]interface{})

	if v, ok := m["uri"].(string); ok {
		def.Uri = v
	}

	if v, ok := m["expires"].(float64); ok {
		def.Expires = int(v)
	}

	if v, ok := m["message-ttl"].(float64); ok {
		def.MessageTTL = int32(v)
	}

	if v, ok := m["max-hops"].(float64); ok {
		def.MaxHops = int(v)
	}

	if v, ok := m["prefetch-count"].(float64); ok {
		def.PrefetchCount = int(v)
	}

	if v, ok := m["reconnect-delay"].(float64); ok {
		def.ReconnectDelay = int(v)
	}

	if v, ok := m["ack-mode"].(string); ok {
		def.AckMode = v
	}

	if v, ok := m["trust-user-id"].(bool); ok {
		def.TrustUserId = v
	}

	if v, ok := m["exchange"].(string); ok {
		def.Exchange = v
	}

	if v, ok := m["queue"].(string); ok {
		def.Queue = v
	}

	up.Definition = def
	return up
}
<|endoftext|>


<commit_before>// Copyright 2016 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"net/url"
	"path"
	"strings"

	"github.com/coreos/mantle/Godeps/_workspace/src/github.com/spf13/pflag"

	"github.com/coreos/mantle/lang/maps"
	"github.com/coreos/mantle/sdk"
)

type storageSpec struct {
	BaseURL       string
	Title         string // Replace the bucket name in index page titles
	NamedPath     string // Copy to $BaseURL/$Board/$NamedPath
	VersionPath   bool   // Copy to $BaseURL/$Board/$Version
	DirectoryHTML bool
	IndexHTML     bool
}

type gceSpec struct {
	Project     string   // GCE project name
	Family      string   // A group name, also used as name prefix
	Description string   // Human readable-ish description
	Licenses    []string // Identifiers for tracking usage
	Image       string   // File name of image source
	Publish     string   // Write published image name to given file
	Limit       int      // Limit on # of old images to keep
}

type channelSpec struct {
	BaseURL      string // Copy from $BaseURL/$Board/$Version
	Destinations []storageSpec
	GCE          gceSpec
}

var (
	specBoard   string
	specChannel string
	specVersion string
	boards      = []string{"amd64-usr", "arm64-usr"}
	specs       = map[string]channelSpec{
		"alpha": channelSpec{
			BaseURL: "gs://builds.release.core-os.net/alpha/boards",
			Destinations: []storageSpec{storageSpec{
				BaseURL:     "gs://alpha.release.core-os.net",
				NamedPath:   "current",
				VersionPath: true,
				IndexHTML:   true,
			}, storageSpec{
				BaseURL:       "gs://coreos-alpha",
				Title:         "alpha.release.core-os.net",
				NamedPath:     "current",
				VersionPath:   true,
				DirectoryHTML: true,
				IndexHTML:     true,
			}, storageSpec{
				BaseURL:     "gs://storage.core-os.net/coreos",
				NamedPath:   "alpha",
				VersionPath: true,
				IndexHTML:   true,
			}, storageSpec{
				BaseURL:       "gs://coreos-net-storage/coreos",
				Title:         "storage.core-os.net",
				NamedPath:     "alpha",
				VersionPath:   true,
				DirectoryHTML: true,
				IndexHTML:     true,
			}},
			GCE: gceSpec{
				Project:     "coreos-cloud",
				Family:      "coreos-alpha",
				Description: "CoreOS, CoreOS alpha",
				Licenses:    []string{"coreos-alpha"},
				Image:       "coreos_production_gce.tar.gz",
				Publish:     "coreos_production_gce.txt",
				Limit:       25,
			},
		},
		"beta": channelSpec{
			BaseURL: "gs://builds.release.core-os.net/beta/boards",
			Destinations: []storageSpec{storageSpec{
				BaseURL:     "gs://beta.release.core-os.net",
				NamedPath:   "current",
				VersionPath: true,
				IndexHTML:   true,
			}, storageSpec{
				BaseURL:       "gs://coreos-beta",
				Title:         "beta.release.core-os.net",
				NamedPath:     "current",
				VersionPath:   true,
				DirectoryHTML: true,
				IndexHTML:     true,
			}, storageSpec{
				BaseURL:   "gs://storage.core-os.net/coreos",
				NamedPath: "beta",
				IndexHTML: true,
			}, storageSpec{
				BaseURL:       "gs://coreos-net-storage/coreos",
				Title:         "storage.core-os.net",
				NamedPath:     "beta",
				DirectoryHTML: true,
				IndexHTML:     true,
			}},
			GCE: gceSpec{
				Project:     "coreos-cloud",
				Family:      "coreos-beta",
				Description: "CoreOS, CoreOS beta",
				Licenses:    []string{"coreos-beta"},
				Image:       "coreos_production_gce.tar.gz",
				Publish:     "coreos_production_gce.txt",
				Limit:       25,
			},
		},
		"stable": channelSpec{
			BaseURL: "gs://builds.release.core-os.net/stable/boards",
			Destinations: []storageSpec{storageSpec{
				BaseURL:     "gs://stable.release.core-os.net",
				NamedPath:   "current",
				VersionPath: true,
				IndexHTML:   true,
			}, storageSpec{
				BaseURL:       "gs://coreos-stable",
				Title:         "stable.release.core-os.net",
				NamedPath:     "current",
				VersionPath:   true,
				DirectoryHTML: true,
				IndexHTML:     true,
			}},
			GCE: gceSpec{
				Project:     "coreos-cloud",
				Family:      "coreos-stable",
				Description: "CoreOS, CoreOS stable",
				Licenses:    []string{"coreos-stable"},
				Image:       "coreos_production_gce.tar.gz",
				Publish:     "coreos_production_gce.txt",
				Limit:       25,
			},
		},
	}
)

func AddSpecFlags(flags *pflag.FlagSet) {
	board := sdk.DefaultBoard()
	channels := strings.Join(maps.SortedKeys(specs), " ")
	versions, _ := sdk.VersionsFromManifest()
	flags.StringVarP(&specBoard, "board", "B",
		board, "target board")
	flags.StringVarP(&specChannel, "channel", "C",
		"alpha", "channels: "+channels)
	flags.StringVarP(&specVersion, "version", "V",
		versions.VersionID, "release version")
}

func ChannelSpec() channelSpec {
	if specBoard == "" {
		plog.Fatal("--board is required")
	}
	if specChannel == "" {
		plog.Fatal("--channel is required")
	}
	if specVersion == "" {
		plog.Fatal("--version is required")
	}

	spec, ok := specs[specChannel]
	if !ok {
		plog.Fatalf("Unknown channel: %s", specChannel)
	}

	boardOk := false
	for _, board := range boards {
		if specBoard == board {
			boardOk = true
			break
		}
	}
	if !boardOk {
		plog.Fatalf("Unknown board: %s", specBoard)
	}

	return spec
}

func (cs channelSpec) SourceURL() string {
	u, err := url.Parse(cs.BaseURL)
	if err != nil {
		panic(err)
	}
	u.Path = path.Join(u.Path, specBoard, specVersion)
	return u.String()
}

func (ss storageSpec) ParentPrefixes() []string {
	u, err := url.Parse(ss.BaseURL)
	if err != nil {
		panic(err)
	}
	return []string{u.Path, path.Join(u.Path, specBoard)}
}

func (ss storageSpec) FinalPrefixes() []string {
	u, err := url.Parse(ss.BaseURL)
	if err != nil {
		plog.Panic(err)
	}

	prefixes := []string{}
	if ss.VersionPath {
		prefixes = append(prefixes,
			path.Join(u.Path, specBoard, specVersion))
	}
	if ss.NamedPath != "" {
		prefixes = append(prefixes,
			path.Join(u.Path, specBoard, ss.NamedPath))
	}
	if len(prefixes) == 0 {
		plog.Panicf("Invalid destination: %#v", ss)
	}

	return prefixes
}
<commit_msg>plume: disable GCE image creation on arm64<commit_after>// Copyright 2016 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"net/url"
	"path"
	"strings"

	"github.com/coreos/mantle/Godeps/_workspace/src/github.com/spf13/pflag"

	"github.com/coreos/mantle/lang/maps"
	"github.com/coreos/mantle/sdk"
)

type storageSpec struct {
	BaseURL       string
	Title         string // Replace the bucket name in index page titles
	NamedPath     string // Copy to $BaseURL/$Board/$NamedPath
	VersionPath   bool   // Copy to $BaseURL/$Board/$Version
	DirectoryHTML bool
	IndexHTML     bool
}

type gceSpec struct {
	Project     string   // GCE project name
	Family      string   // A group name, also used as name prefix
	Description string   // Human readable-ish description
	Licenses    []string // Identifiers for tracking usage
	Image       string   // File name of image source
	Publish     string   // Write published image name to given file
	Limit       int      // Limit on # of old images to keep
}

type channelSpec struct {
	BaseURL      string // Copy from $BaseURL/$Board/$Version
	Destinations []storageSpec
	GCE          gceSpec
}

var (
	specBoard   string
	specChannel string
	specVersion string
	boards      = []string{"amd64-usr", "arm64-usr"}
	gceBoards   = []string{"amd64-usr"}
	specs       = map[string]channelSpec{
		"alpha": channelSpec{
			BaseURL: "gs://builds.release.core-os.net/alpha/boards",
			Destinations: []storageSpec{storageSpec{
				BaseURL:     "gs://alpha.release.core-os.net",
				NamedPath:   "current",
				VersionPath: true,
				IndexHTML:   true,
			}, storageSpec{
				BaseURL:       "gs://coreos-alpha",
				Title:         "alpha.release.core-os.net",
				NamedPath:     "current",
				VersionPath:   true,
				DirectoryHTML: true,
				IndexHTML:     true,
			}, storageSpec{
				BaseURL:     "gs://storage.core-os.net/coreos",
				NamedPath:   "alpha",
				VersionPath: true,
				IndexHTML:   true,
			}, storageSpec{
				BaseURL:       "gs://coreos-net-storage/coreos",
				Title:         "storage.core-os.net",
				NamedPath:     "alpha",
				VersionPath:   true,
				DirectoryHTML: true,
				IndexHTML:     true,
			}},
			GCE: gceSpec{
				Project:     "coreos-cloud",
				Family:      "coreos-alpha",
				Description: "CoreOS, CoreOS alpha",
				Licenses:    []string{"coreos-alpha"},
				Image:       "coreos_production_gce.tar.gz",
				Publish:     "coreos_production_gce.txt",
				Limit:       25,
			},
		},
		"beta": channelSpec{
			BaseURL: "gs://builds.release.core-os.net/beta/boards",
			Destinations: []storageSpec{storageSpec{
				BaseURL:     "gs://beta.release.core-os.net",
				NamedPath:   "current",
				VersionPath: true,
				IndexHTML:   true,
			}, storageSpec{
				BaseURL:       "gs://coreos-beta",
				Title:         "beta.release.core-os.net",
				NamedPath:     "current",
				VersionPath:   true,
				DirectoryHTML: true,
				IndexHTML:     true,
			}, storageSpec{
				BaseURL:   "gs://storage.core-os.net/coreos",
				NamedPath: "beta",
				IndexHTML: true,
			}, storageSpec{
				BaseURL:       "gs://coreos-net-storage/coreos",
				Title:         "storage.core-os.net",
				NamedPath:     "beta",
				DirectoryHTML: true,
				IndexHTML:     true,
			}},
			GCE: gceSpec{
				Project:     "coreos-cloud",
				Family:      "coreos-beta",
				Description: "CoreOS, CoreOS beta",
				Licenses:    []string{"coreos-beta"},
				Image:       "coreos_production_gce.tar.gz",
				Publish:     "coreos_production_gce.txt",
				Limit:       25,
			},
		},
		"stable": channelSpec{
			BaseURL: "gs://builds.release.core-os.net/stable/boards",
			Destinations: []storageSpec{storageSpec{
				BaseURL:     "gs://stable.release.core-os.net",
				NamedPath:   "current",
				VersionPath: true,
				IndexHTML:   true,
			}, storageSpec{
				BaseURL:       "gs://coreos-stable",
				Title:         "stable.release.core-os.net",
				NamedPath:     "current",
				VersionPath:   true,
				DirectoryHTML: true,
				IndexHTML:     true,
			}},
			GCE: gceSpec{
				Project:     "coreos-cloud",
				Family:      "coreos-stable",
				Description: "CoreOS, CoreOS stable",
				Licenses:    []string{"coreos-stable"},
				Image:       "coreos_production_gce.tar.gz",
				Publish:     "coreos_production_gce.txt",
				Limit:       25,
			},
		},
	}
)

func AddSpecFlags(flags *pflag.FlagSet) {
	board := sdk.DefaultBoard()
	channels := strings.Join(maps.SortedKeys(specs), " ")
	versions, _ := sdk.VersionsFromManifest()
	flags.StringVarP(&specBoard, "board", "B",
		board, "target board")
	flags.StringVarP(&specChannel, "channel", "C",
		"alpha", "channels: "+channels)
	flags.StringVarP(&specVersion, "version", "V",
		versions.VersionID, "release version")
}

func ChannelSpec() channelSpec {
	if specBoard == "" {
		plog.Fatal("--board is required")
	}
	if specChannel == "" {
		plog.Fatal("--channel is required")
	}
	if specVersion == "" {
		plog.Fatal("--version is required")
	}

	spec, ok := specs[specChannel]
	if !ok {
		plog.Fatalf("Unknown channel: %s", specChannel)
	}

	boardOk := false
	for _, board := range boards {
		if specBoard == board {
			boardOk = true
			break
		}
	}
	if !boardOk {
		plog.Fatalf("Unknown board: %s", specBoard)
	}

	gceOk := false
	for _, board := range gceBoards {
		if specBoard == board {
			gceOk = true
			break
		}
	}
	if !gceOk {
		spec.GCE = gceSpec{}
	}

	return spec
}

func (cs channelSpec) SourceURL() string {
	u, err := url.Parse(cs.BaseURL)
	if err != nil {
		panic(err)
	}
	u.Path = path.Join(u.Path, specBoard, specVersion)
	return u.String()
}

func (ss storageSpec) ParentPrefixes() []string {
	u, err := url.Parse(ss.BaseURL)
	if err != nil {
		panic(err)
	}
	return []string{u.Path, path.Join(u.Path, specBoard)}
}

func (ss storageSpec) FinalPrefixes() []string {
	u, err := url.Parse(ss.BaseURL)
	if err != nil {
		plog.Panic(err)
	}

	prefixes := []string{}
	if ss.VersionPath {
		prefixes = append(prefixes,
			path.Join(u.Path, specBoard, specVersion))
	}
	if ss.NamedPath != "" {
		prefixes = append(prefixes,
			path.Join(u.Path, specBoard, ss.NamedPath))
	}
	if len(prefixes) == 0 {
		plog.Panicf("Invalid destination: %#v", ss)
	}

	return prefixes
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"log"
	"os"

	"github.com/alecthomas/chroma/quick"
	"github.com/spf13/cobra"
	"golang.org/x/crypto/ssh/terminal"
)

func applyCmd() *cobra.Command {
	cmd := &cobra.Command{
		Use:   "apply",
		Short: "apply the configuration to the cluster",
	}
	cmd.Run = func(cmd *cobra.Command, args []string) {
		raw, err := evalDict()
		if err != nil {
			log.Fatalln("evaluating jsonnet:", err)
		}

		desired, err := kube.Reconcile(raw)
		if err != nil {
			log.Fatalln("reconciling:", err)
		}

		if err := kube.Apply(desired); err != nil {
			log.Fatalln("applying:", err)
		}
	}
	return cmd
}

func diffCmd() *cobra.Command {
	cmd := &cobra.Command{
		Use:   "diff",
		Short: "differences between the configuration and the cluster",
	}
	cmd.Run = func(cmd *cobra.Command, args []string) {
		raw, err := evalDict()
		if err != nil {
			log.Fatalln("evaluating jsonnet:", err)
		}

		desired, err := kube.Reconcile(raw)
		if err != nil {
			log.Fatalln("reconciling:", err)
		}

		changes, err := kube.Diff(desired)
		if err != nil {
			log.Fatalln("diffing:", err)
		}

		if terminal.IsTerminal(int(os.Stdout.Fd())) {
			if err := quick.Highlight(os.Stdout, changes, "diff", "terminal", "vim"); err != nil {
				log.Fatalln("highlighting:", err)
			}
		} else {
			fmt.Println(changes)
		}
	}
	return cmd
}

func showCmd() *cobra.Command {
	cmd := &cobra.Command{
		Use:   "show",
		Short: "jsonnet as yaml",
	}
	cmd.Run = func(cmd *cobra.Command, args []string) {
		raw, err := evalDict()
		if err != nil {
			log.Fatalln("evaluating jsonnet:", err)
		}

		state, err := kube.Reconcile(raw)
		if err != nil {
			log.Fatalln("reconciling:", err)
		}

		pretty, err := kube.Fmt(state)
		if err != nil {
			log.Fatalln("pretty printing state:", err)
		}
		fmt.Println(pretty)
	}
	return cmd
}
<commit_msg>fix(kubernetes): consistent capitalization of errors<commit_after>package main

import (
	"fmt"
	"log"
	"os"

	"github.com/alecthomas/chroma/quick"
	"github.com/spf13/cobra"
	"golang.org/x/crypto/ssh/terminal"
)

func applyCmd() *cobra.Command {
	cmd := &cobra.Command{
		Use:   "apply",
		Short: "apply the configuration to the cluster",
	}
	cmd.Run = func(cmd *cobra.Command, args []string) {
		raw, err := evalDict()
		if err != nil {
			log.Fatalln("Evaluating jsonnet:", err)
		}

		desired, err := kube.Reconcile(raw)
		if err != nil {
			log.Fatalln("Reconciling:", err)
		}

		if err := kube.Apply(desired); err != nil {
			log.Fatalln("Applying:", err)
		}
	}
	return cmd
}

func diffCmd() *cobra.Command {
	cmd := &cobra.Command{
		Use:   "diff",
		Short: "differences between the configuration and the cluster",
	}
	cmd.Run = func(cmd *cobra.Command, args []string) {
		raw, err := evalDict()
		if err != nil {
			log.Fatalln("Evaluating jsonnet:", err)
		}

		desired, err := kube.Reconcile(raw)
		if err != nil {
			log.Fatalln("Reconciling:", err)
		}

		changes, err := kube.Diff(desired)
		if err != nil {
			log.Fatalln("Diffing:", err)
		}

		if terminal.IsTerminal(int(os.Stdout.Fd())) {
			if err := quick.Highlight(os.Stdout, changes, "diff", "terminal", "vim"); err != nil {
				log.Fatalln("Highlighting:", err)
			}
		} else {
			fmt.Println(changes)
		}
	}
	return cmd
}

func showCmd() *cobra.Command {
	cmd := &cobra.Command{
		Use:   "show",
		Short: "jsonnet as yaml",
	}
	cmd.Run = func(cmd *cobra.Command, args []string) {
		raw, err := evalDict()
		if err != nil {
			log.Fatalln("Evaluating jsonnet:", err)
		}

		state, err := kube.Reconcile(raw)
		if err != nil {
			log.Fatalln("Reconciling:", err)
		}

		pretty, err := kube.Fmt(state)
		if err != nil {
			log.Fatalln("Pretty printing state:", err)
		}
		fmt.Println(pretty)
	}
	return cmd
}
<|endoftext|>


<commit_before>package compose

import (
	log "github.com/Sirupsen/logrus"
	"github.com/docker/libcompose/cli/logger"
	"github.com/docker/libcompose/docker"
	"github.com/docker/libcompose/project"
	"github.com/rancherio/os/config"
	rosDocker "github.com/rancherio/os/docker"
	"github.com/rancherio/os/util"
)

func CreateService(cfg *config.CloudConfig, name string, serviceConfig *project.ServiceConfig) (project.Service, error) {
	if cfg == nil {
		var err error
		cfg, err = config.LoadConfig()
		if err != nil {
			return nil, err
		}
	}

	p, err := RunServiceSet("once", cfg, map[string]*project.ServiceConfig{
		name: serviceConfig,
	})
	if err != nil {
		return nil, err
	}

	return p.CreateService(name)
}

func RunServiceSet(name string, cfg *config.CloudConfig, configs map[string]*project.ServiceConfig) (*project.Project, error) {
	p, err := newProject(name, cfg)
	if err != nil {
		return nil, err
	}

	addServices(p, cfg, map[string]string{}, configs)

	return p, p.Up()
}

func RunServices(cfg *config.CloudConfig) error {
	p, err := newCoreServiceProject(cfg)
	if err != nil {
		return err
	}

	return p.Up()
}

func GetProject(cfg *config.CloudConfig) (*project.Project, error) {
	return newCoreServiceProject(cfg)
}

func newProject(name string, cfg *config.CloudConfig) (*project.Project, error) {
	clientFactory, err := rosDocker.NewClientFactory(docker.ClientOpts{})
	if err != nil {
		return nil, err
	}

	serviceFactory := &rosDocker.ServiceFactory{
		Deps: map[string][]string{},
	}
	context := &docker.Context{
		ClientFactory: clientFactory,
		Context: project.Context{
			ProjectName:       name,
			EnvironmentLookup: rosDocker.NewConfigEnvironment(cfg),
			ServiceFactory:    serviceFactory,
			Rebuild:           true,
			Log:               cfg.Rancher.Log,
			LoggerFactory:     logger.NewColorLoggerFactory(),
		},
	}
	serviceFactory.Context = context

	return docker.NewProject(context)
}

func addServices(p *project.Project, cfg *config.CloudConfig, enabled map[string]string, configs map[string]*project.ServiceConfig) {
	// Note: we ignore errors while loading services
	for name, serviceConfig := range configs {
		hash := project.GetServiceHash(name, *serviceConfig)

		if enabled[name] == hash {
			continue
		}

		if err := p.AddConfig(name, serviceConfig); err != nil {
			log.Infof("Failed loading service %s", name)
			continue
		}

		enabled[name] = hash
	}
}

func newCoreServiceProject(cfg *config.CloudConfig) (*project.Project, error) {
	network := false
	projectEvents := make(chan project.ProjectEvent)
	enabled := make(map[string]string)

	p, err := newProject("os", cfg)
	if err != nil {
		return nil, err
	}

	p.AddListener(project.NewDefaultListener(p))
	p.AddListener(projectEvents)

	p.ReloadCallback = func() error {
		err := cfg.Reload()
		if err != nil {
			return err
		}

		for service, serviceEnabled := range cfg.Rancher.ServicesInclude {
			if enabled[service] != "" || !serviceEnabled {
				continue
			}

			bytes, err := LoadServiceResource(service, network, cfg)
			if err != nil {
				if err == util.ErrNoNetwork {
					log.Debugf("Can not load %s, networking not enabled", service)
				} else {
					log.Errorf("Failed to load %s : %v", service, err)
				}
				continue
			}

			err = p.Load(bytes)
			if err != nil {
				log.Errorf("Failed to load %s : %v", service, err)
				continue
			}

			enabled[service] = service
		}

		addServices(p, cfg, enabled, cfg.Rancher.Services)

		return nil
	}

	go func() {
		for event := range projectEvents {
			if event.Event == project.CONTAINER_STARTED && event.ServiceName == "network" {
				network = true
			}
		}
	}()

	err = p.ReloadCallback()
	if err != nil {
		log.Errorf("Failed to reload os: %v", err)
		return nil, err
	}

	return p, nil
}

func LoadServiceResource(name string, network bool, cfg *config.CloudConfig) ([]byte, error) {
	return util.LoadResource(name, network, cfg.Rancher.Repositories.ToArray())
}
<commit_msg>Make sure services_include overrides services<commit_after>package compose

import (
	"fmt"

	log "github.com/Sirupsen/logrus"
	"github.com/docker/libcompose/cli/logger"
	"github.com/docker/libcompose/docker"
	"github.com/docker/libcompose/project"
	"github.com/rancherio/os/config"
	rosDocker "github.com/rancherio/os/docker"
	"github.com/rancherio/os/util"
)

func CreateService(cfg *config.CloudConfig, name string, serviceConfig *project.ServiceConfig) (project.Service, error) {
	if cfg == nil {
		var err error
		cfg, err = config.LoadConfig()
		if err != nil {
			return nil, err
		}
	}

	p, err := RunServiceSet("once", cfg, map[string]*project.ServiceConfig{
		name: serviceConfig,
	})
	if err != nil {
		return nil, err
	}

	return p.CreateService(name)
}

func RunServiceSet(name string, cfg *config.CloudConfig, configs map[string]*project.ServiceConfig) (*project.Project, error) {
	p, err := newProject(name, cfg)
	if err != nil {
		return nil, err
	}

	addServices(p, cfg, map[string]string{}, configs)

	return p, p.Up()
}

func RunServices(cfg *config.CloudConfig) error {
	p, err := newCoreServiceProject(cfg)
	if err != nil {
		return err
	}

	return p.Up()
}

func GetProject(cfg *config.CloudConfig) (*project.Project, error) {
	return newCoreServiceProject(cfg)
}

func newProject(name string, cfg *config.CloudConfig) (*project.Project, error) {
	clientFactory, err := rosDocker.NewClientFactory(docker.ClientOpts{})
	if err != nil {
		return nil, err
	}

	serviceFactory := &rosDocker.ServiceFactory{
		Deps: map[string][]string{},
	}
	context := &docker.Context{
		ClientFactory: clientFactory,
		Context: project.Context{
			ProjectName:       name,
			EnvironmentLookup: rosDocker.NewConfigEnvironment(cfg),
			ServiceFactory:    serviceFactory,
			Rebuild:           true,
			Log:               cfg.Rancher.Log,
			LoggerFactory:     logger.NewColorLoggerFactory(),
		},
	}
	serviceFactory.Context = context

	return docker.NewProject(context)
}

func addServices(p *project.Project, cfg *config.CloudConfig, enabled map[string]string, configs map[string]*project.ServiceConfig) {
	// Note: we ignore errors while loading services
	for name, serviceConfig := range configs {
		hash := project.GetServiceHash(name, *serviceConfig)

		if enabled[name] == hash {
			continue
		}

		if err := p.AddConfig(name, serviceConfig); err != nil {
			log.Infof("Failed loading service %s", name)
			continue
		}

		enabled[name] = hash
	}
}

func newCoreServiceProject(cfg *config.CloudConfig) (*project.Project, error) {
	network := false
	projectEvents := make(chan project.ProjectEvent)
	enabled := make(map[string]string)

	p, err := newProject("os", cfg)
	if err != nil {
		return nil, err
	}

	p.AddListener(project.NewDefaultListener(p))
	p.AddListener(projectEvents)

	p.ReloadCallback = func() error {
		err := cfg.Reload()
		if err != nil {
			return err
		}

		addServices(p, cfg, enabled, cfg.Rancher.Services)

		for service, serviceEnabled := range cfg.Rancher.ServicesInclude {
			if enabled[service] != "" || !serviceEnabled {
				continue
			}

			bytes, err := LoadServiceResource(service, network, cfg)
			if err != nil {
				if err == util.ErrNoNetwork {
					log.Debugf("Can not load %s, networking not enabled", service)
				} else {
					log.Errorf("Failed to load %s : %v", service, err)
				}
				continue
			}

			fmt.Println("Loading config: %s", string(bytes))
			err = p.Load(bytes)
			if err != nil {
				log.Errorf("Failed to load %s : %v", service, err)
				continue
			}

			enabled[service] = service
		}

		return nil
	}

	go func() {
		for event := range projectEvents {
			if event.Event == project.CONTAINER_STARTED && event.ServiceName == "network" {
				network = true
			}
		}
	}()

	err = p.ReloadCallback()
	if err != nil {
		log.Errorf("Failed to reload os: %v", err)
		return nil, err
	}

	return p, nil
}

func LoadServiceResource(name string, network bool, cfg *config.CloudConfig) ([]byte, error) {
	return util.LoadResource(name, network, cfg.Rancher.Repositories.ToArray())
}
<|endoftext|>


<commit_before>package compute

import (
	"errors"
	"fmt"
	"subuk/vmango/util"
)

var ErrArchNotsupported = errors.New("requested arch not supported")

type Event interface {
	Name() string
	Plain() map[string]string
}

type EventPublisher interface {
	Publish(event Event) error
}

type Service struct {
	virt    VirtualMachineRepository
	vol     VolumeRepository
	volpool VolumePoolRepository
	host    HostInfoRepository
	key     KeyRepository
	net     NetworkRepository
	epub    EventPublisher
}

func New(epub EventPublisher, virt VirtualMachineRepository, vol VolumeRepository, volpool VolumePoolRepository, host HostInfoRepository, key KeyRepository, net NetworkRepository) *Service {
	return &Service{epub: epub, virt: virt, vol: vol, volpool: volpool, host: host, key: key, net: net}
}

func (service *Service) VirtualMachineList() ([]*VirtualMachine, error) {
	return service.virt.List()
}

func (service *Service) VirtualMachineDetail(id string) (*VirtualMachine, error) {
	return service.virt.Get(id)
}

type VirtualMachineCreateParamsConfig struct {
	Hostname        string
	UserData        string
	KeyFingerprints []string
}

type VirtualMachineCreateParamsVolume struct {
	CloneFrom  string
	Name       string
	Pool       string
	Format     string
	DeviceType string
	SizeMb     uint64
}

type VirtualMachineCreateParamsInterface struct {
	Network    string
	Mac        string
	Model      string
	AccessVlan uint
}

type VirtualMachineCreateParams struct {
	Id         string
	VCpus      int
	Arch       string
	MemoryKb   uint // KiB
	Volumes    []VirtualMachineCreateParamsVolume
	Interfaces []VirtualMachineCreateParamsInterface
	Config     VirtualMachineCreateParamsConfig
	Start      bool
}

func (service *Service) VirtualMachineCreate(params VirtualMachineCreateParams) (*VirtualMachine, error) {
	volumes := []*VirtualMachineAttachedVolume{}
	for _, volumeParams := range params.Volumes {
		volume, _ := service.vol.GetByName(volumeParams.Pool, volumeParams.Name)
		if volume == nil {
			if volumeParams.CloneFrom != "" {
				clonedVolume, err := service.VolumeClone(volumeParams.CloneFrom, volumeParams.Name, volumeParams.Pool, volumeParams.Format, volumeParams.SizeMb)
				if err != nil {
					return nil, err
				}
				volume = clonedVolume
			} else {
				createdVolume, err := service.VolumeCreate(volumeParams.Name, volumeParams.Pool, volumeParams.Format, volumeParams.SizeMb)
				if err != nil {
					return nil, err
				}
				volume = createdVolume
			}
		}
		if volume.AttachedTo != "" {
			return nil, fmt.Errorf("volume %s already exists and attached to %s as %s", volume.Path, volume.AttachedTo, volume.AttachedAs)
		}
		volumes = append(volumes, &VirtualMachineAttachedVolume{
			Type:   volume.Type,
			Path:   volume.Path,
			Format: volume.Format,
			Device: NewDeviceType(volumeParams.DeviceType),
		})
	}

	interfaces := []*VirtualMachineAttachedInterface{}
	for _, ifaceParams := range params.Interfaces {
		network, err := service.net.Get(ifaceParams.Network)
		if err != nil {
			return nil, util.NewError(err, "network get failed")
		}
		iface := &VirtualMachineAttachedInterface{
			NetworkType: network.Type,
			NetworkName: ifaceParams.Network,
			Mac:         ifaceParams.Mac,
			AccessVlan:  ifaceParams.AccessVlan,
		}
		interfaces = append(interfaces, iface)
	}
	config := &VirtualMachineConfig{
		Hostname: params.Config.Hostname,
		Userdata: []byte(params.Config.UserData),
	}
	for _, fingerprint := range params.Config.KeyFingerprints {
		key, err := service.key.Get(fingerprint)
		if err != nil {
			return nil, util.NewError(err, "cannot load key")
		}
		config.Keys = append(config.Keys, key)
	}

	vm, err := service.virt.Create(params.Id, NewArch(params.Arch), params.VCpus, params.MemoryKb, volumes, interfaces, config)
	if err != nil {
		return nil, util.NewError(err, "cannot create virtual machine")
	}
	if err := service.epub.Publish(NewEventVirtualMachineCreated(vm)); err != nil {
		service.virt.Delete(vm.Id) // Ignore error
		return nil, util.NewError(err, "cannot publish event virtual machine created")
	}
	if params.Start {
		if err := service.virt.Start(vm.Id); err != nil {
			return nil, util.NewError(err, "cannot start vm")
		}
	}
	return vm, nil
}

func (service *Service) VirtualMachineDelete(id string, deleteVolumes bool) error {
	volumesToDelete := []*VirtualMachineAttachedVolume{}
	if deleteVolumes {
		vm, err := service.virt.Get(id)
		if err != nil {
			return util.NewError(err, "cannot fetch vm info")
		}
		for _, volume := range vm.Volumes {
			volumesToDelete = append(volumesToDelete, volume)
		}
	}
	if err := service.virt.Delete(id); err != nil {
		return util.NewError(err, "cannot delete vm")
	}
	for _, volume := range volumesToDelete {
		if err := service.vol.Delete(volume.Path); err != nil {
			return util.NewError(err, "cannot delete volume")
		}
	}
	return nil
}

func (service *Service) VirtualMachineAttachVolume(id, path string, deviceType DeviceType) (*VirtualMachineAttachedVolume, error) {
	vol, err := service.vol.Get(path)
	if err != nil {
		return nil, util.NewError(err, "cannot lookup volume")
	}
	return service.virt.AttachVolume(id, path, vol.Type, vol.Format, deviceType)
}

func (service *Service) VirtualMachineDetachVolume(id, path string) error {
	return service.virt.DetachVolume(id, path)
}

type VirtualMachineAttachInterfaceParams struct {
	NetworkName string
	NetworkType NetworkType
	Mac         string
	Model       string
	AccessVlan  uint
}

func (service *Service) VirtualMachineAttachInterface(id string, iface *VirtualMachineAttachedInterface) error {
	return service.virt.AttachInterface(id, iface)
}

type VirtualMachineUpdateParams struct {
	Vcpus      *int
	MemoryKb   *uint
	Autostart  *bool
	GuestAgent *bool
}

func (service *Service) VirtualMachineUpdate(id string, params VirtualMachineUpdateParams) error {
	return service.virt.Update(id, params)
}

func (service *Service) VirtualMachineDetachInterface(id, mac string) error {
	return service.virt.DetachInterface(id, mac)
}

func (service *Service) VirtualMachineGetConsoleStream(id string) (VirtualMachineConsoleStream, error) {
	return service.virt.GetConsoleStream(id)
}

func (service *Service) VolumeList() ([]*Volume, error) {
	return service.vol.List()
}

func (service *Service) ImageList() ([]*Volume, error) {
	volumes, err := service.vol.List()
	if err != nil {
		return nil, util.NewError(err, "cannot list volume")
	}
	annotatedVolumes := []*Volume{}
	detachedVolumes := []*Volume{}
	for _, volume := range volumes {
		if volume.Format == FormatIso {
			continue
		}
		if volume.AttachedTo != "" {
			continue
		}
		if volume.Metadata.OsName != "" {
			annotatedVolumes = append(annotatedVolumes, volume)
			continue
		}
		detachedVolumes = append(detachedVolumes, volume)
	}
	if len(annotatedVolumes) > 0 {
		return annotatedVolumes, nil
	}
	return detachedVolumes, nil
}

func (service *Service) VolumeGet(path string) (*Volume, error) {
	return service.vol.Get(path)
}

func (service *Service) VolumeClone(originalPath, volumeName, poolName, volumeFormatName string, newSizeMb uint64) (*Volume, error) {
	return service.vol.Clone(originalPath, volumeName, poolName, NewVolumeFormat(volumeFormatName), newSizeMb)
}

func (service *Service) VolumeResize(path string, size uint64) error {
	return service.vol.Resize(path, size)
}

func (service *Service) VolumePoolList() ([]*VolumePool, error) {
	return service.volpool.List()
}

func (service *Service) VolumeCreate(poolName, volumeName, volumeFormatName string, size uint64) (*Volume, error) {
	return service.vol.Create(poolName, volumeName, NewVolumeFormat(volumeFormatName), size)
}

func (service *Service) VolumeDelete(path string) error {
	return service.vol.Delete(path)
}

func (service *Service) HostInfo() (*HostInfo, error) {
	return service.host.Get()
}

func (service *Service) VirtualMachineAction(id string, action string) error {
	switch action {
	default:
		return fmt.Errorf("unknown action %s", action)
	case "reboot":
		return service.virt.Reboot(id)
	case "poweroff":
		return service.virt.Poweroff(id)
	case "start":
		return service.virt.Start(id)
	}
}

func (service *Service) KeyList() ([]*Key, error) {
	return service.key.List()
}

func (service *Service) KeyDetail(fingerprint string) (*Key, error) {
	return service.key.Get(fingerprint)
}

func (service *Service) KeyDelete(fingerprint string) error {
	return service.key.Delete(fingerprint)
}

func (service *Service) KeyAdd(input string) error {
	return service.key.Add([]byte(input))
}

func (service *Service) NetworkList() ([]*Network, error) {
	return service.net.List()
}

func (service *Service) NetworkGet(id string) (*Network, error) {
	return service.net.Get(id)
}
<commit_msg>Remove unused struct<commit_after>package compute

import (
	"errors"
	"fmt"
	"subuk/vmango/util"
)

var ErrArchNotsupported = errors.New("requested arch not supported")

type Event interface {
	Name() string
	Plain() map[string]string
}

type EventPublisher interface {
	Publish(event Event) error
}

type Service struct {
	virt    VirtualMachineRepository
	vol     VolumeRepository
	volpool VolumePoolRepository
	host    HostInfoRepository
	key     KeyRepository
	net     NetworkRepository
	epub    EventPublisher
}

func New(epub EventPublisher, virt VirtualMachineRepository, vol VolumeRepository, volpool VolumePoolRepository, host HostInfoRepository, key KeyRepository, net NetworkRepository) *Service {
	return &Service{epub: epub, virt: virt, vol: vol, volpool: volpool, host: host, key: key, net: net}
}

func (service *Service) VirtualMachineList() ([]*VirtualMachine, error) {
	return service.virt.List()
}

func (service *Service) VirtualMachineDetail(id string) (*VirtualMachine, error) {
	return service.virt.Get(id)
}

type VirtualMachineCreateParamsConfig struct {
	Hostname        string
	UserData        string
	KeyFingerprints []string
}

type VirtualMachineCreateParamsVolume struct {
	CloneFrom  string
	Name       string
	Pool       string
	Format     string
	DeviceType string
	SizeMb     uint64
}

type VirtualMachineCreateParamsInterface struct {
	Network    string
	Mac        string
	Model      string
	AccessVlan uint
}

type VirtualMachineCreateParams struct {
	Id         string
	VCpus      int
	Arch       string
	MemoryKb   uint // KiB
	Volumes    []VirtualMachineCreateParamsVolume
	Interfaces []VirtualMachineCreateParamsInterface
	Config     VirtualMachineCreateParamsConfig
	Start      bool
}

func (service *Service) VirtualMachineCreate(params VirtualMachineCreateParams) (*VirtualMachine, error) {
	volumes := []*VirtualMachineAttachedVolume{}
	for _, volumeParams := range params.Volumes {
		volume, _ := service.vol.GetByName(volumeParams.Pool, volumeParams.Name)
		if volume == nil {
			if volumeParams.CloneFrom != "" {
				clonedVolume, err := service.VolumeClone(volumeParams.CloneFrom, volumeParams.Name, volumeParams.Pool, volumeParams.Format, volumeParams.SizeMb)
				if err != nil {
					return nil, err
				}
				volume = clonedVolume
			} else {
				createdVolume, err := service.VolumeCreate(volumeParams.Name, volumeParams.Pool, volumeParams.Format, volumeParams.SizeMb)
				if err != nil {
					return nil, err
				}
				volume = createdVolume
			}
		}
		if volume.AttachedTo != "" {
			return nil, fmt.Errorf("volume %s already exists and attached to %s as %s", volume.Path, volume.AttachedTo, volume.AttachedAs)
		}
		volumes = append(volumes, &VirtualMachineAttachedVolume{
			Type:   volume.Type,
			Path:   volume.Path,
			Format: volume.Format,
			Device: NewDeviceType(volumeParams.DeviceType),
		})
	}

	interfaces := []*VirtualMachineAttachedInterface{}
	for _, ifaceParams := range params.Interfaces {
		network, err := service.net.Get(ifaceParams.Network)
		if err != nil {
			return nil, util.NewError(err, "network get failed")
		}
		iface := &VirtualMachineAttachedInterface{
			NetworkType: network.Type,
			NetworkName: ifaceParams.Network,
			Mac:         ifaceParams.Mac,
			AccessVlan:  ifaceParams.AccessVlan,
		}
		interfaces = append(interfaces, iface)
	}
	config := &VirtualMachineConfig{
		Hostname: params.Config.Hostname,
		Userdata: []byte(params.Config.UserData),
	}
	for _, fingerprint := range params.Config.KeyFingerprints {
		key, err := service.key.Get(fingerprint)
		if err != nil {
			return nil, util.NewError(err, "cannot load key")
		}
		config.Keys = append(config.Keys, key)
	}

	vm, err := service.virt.Create(params.Id, NewArch(params.Arch), params.VCpus, params.MemoryKb, volumes, interfaces, config)
	if err != nil {
		return nil, util.NewError(err, "cannot create virtual machine")
	}
	if err := service.epub.Publish(NewEventVirtualMachineCreated(vm)); err != nil {
		service.virt.Delete(vm.Id) // Ignore error
		return nil, util.NewError(err, "cannot publish event virtual machine created")
	}
	if params.Start {
		if err := service.virt.Start(vm.Id); err != nil {
			return nil, util.NewError(err, "cannot start vm")
		}
	}
	return vm, nil
}

func (service *Service) VirtualMachineDelete(id string, deleteVolumes bool) error {
	volumesToDelete := []*VirtualMachineAttachedVolume{}
	if deleteVolumes {
		vm, err := service.virt.Get(id)
		if err != nil {
			return util.NewError(err, "cannot fetch vm info")
		}
		for _, volume := range vm.Volumes {
			volumesToDelete = append(volumesToDelete, volume)
		}
	}
	if err := service.virt.Delete(id); err != nil {
		return util.NewError(err, "cannot delete vm")
	}
	for _, volume := range volumesToDelete {
		if err := service.vol.Delete(volume.Path); err != nil {
			return util.NewError(err, "cannot delete volume")
		}
	}
	return nil
}

func (service *Service) VirtualMachineAttachVolume(id, path string, deviceType DeviceType) (*VirtualMachineAttachedVolume, error) {
	vol, err := service.vol.Get(path)
	if err != nil {
		return nil, util.NewError(err, "cannot lookup volume")
	}
	return service.virt.AttachVolume(id, path, vol.Type, vol.Format, deviceType)
}

func (service *Service) VirtualMachineDetachVolume(id, path string) error {
	return service.virt.DetachVolume(id, path)
}

func (service *Service) VirtualMachineAttachInterface(id string, iface *VirtualMachineAttachedInterface) error {
	return service.virt.AttachInterface(id, iface)
}

type VirtualMachineUpdateParams struct {
	Vcpus      *int
	MemoryKb   *uint
	Autostart  *bool
	GuestAgent *bool
}

func (service *Service) VirtualMachineUpdate(id string, params VirtualMachineUpdateParams) error {
	return service.virt.Update(id, params)
}

func (service *Service) VirtualMachineDetachInterface(id, mac string) error {
	return service.virt.DetachInterface(id, mac)
}

func (service *Service) VirtualMachineGetConsoleStream(id string) (VirtualMachineConsoleStream, error) {
	return service.virt.GetConsoleStream(id)
}

func (service *Service) VolumeList() ([]*Volume, error) {
	return service.vol.List()
}

func (service *Service) ImageList() ([]*Volume, error) {
	volumes, err := service.vol.List()
	if err != nil {
		return nil, util.NewError(err, "cannot list volume")
	}
	annotatedVolumes := []*Volume{}
	detachedVolumes := []*Volume{}
	for _, volume := range volumes {
		if volume.Format == FormatIso {
			continue
		}
		if volume.AttachedTo != "" {
			continue
		}
		if volume.Metadata.OsName != "" {
			annotatedVolumes = append(annotatedVolumes, volume)
			continue
		}
		detachedVolumes = append(detachedVolumes, volume)
	}
	if len(annotatedVolumes) > 0 {
		return annotatedVolumes, nil
	}
	return detachedVolumes, nil
}

func (service *Service) VolumeGet(path string) (*Volume, error) {
	return service.vol.Get(path)
}

func (service *Service) VolumeClone(originalPath, volumeName, poolName, volumeFormatName string, newSizeMb uint64) (*Volume, error) {
	return service.vol.Clone(originalPath, volumeName, poolName, NewVolumeFormat(volumeFormatName), newSizeMb)
}

func (service *Service) VolumeResize(path string, size uint64) error {
	return service.vol.Resize(path, size)
}

func (service *Service) VolumePoolList() ([]*VolumePool, error) {
	return service.volpool.List()
}

func (service *Service) VolumeCreate(poolName, volumeName, volumeFormatName string, size uint64) (*Volume, error) {
	return service.vol.Create(poolName, volumeName, NewVolumeFormat(volumeFormatName), size)
}

func (service *Service) VolumeDelete(path string) error {
	return service.vol.Delete(path)
}

func (service *Service) HostInfo() (*HostInfo, error) {
	return service.host.Get()
}

func (service *Service) VirtualMachineAction(id string, action string) error {
	switch action {
	default:
		return fmt.Errorf("unknown action %s", action)
	case "reboot":
		return service.virt.Reboot(id)
	case "poweroff":
		return service.virt.Poweroff(id)
	case "start":
		return service.virt.Start(id)
	}
}

func (service *Service) KeyList() ([]*Key, error) {
	return service.key.List()
}

func (service *Service) KeyDetail(fingerprint string) (*Key, error) {
	return service.key.Get(fingerprint)
}

func (service *Service) KeyDelete(fingerprint string) error {
	return service.key.Delete(fingerprint)
}

func (service *Service) KeyAdd(input string) error {
	return service.key.Add([]byte(input))
}

func (service *Service) NetworkList() ([]*Network, error) {
	return service.net.List()
}

func (service *Service) NetworkGet(id string) (*Network, error) {
	return service.net.Get(id)
}
<|endoftext|>


<commit_before>/*
Copyright 2019 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package awstasks

import (
	"testing"

	"k8s.io/kops/upup/pkg/fi"
)

func TestLaunchTemplateTerraformRender(t *testing.T) {
	cases := []*renderTest{
		{
			Resource: &LaunchTemplate{
				Name:              fi.String("test"),
				AssociatePublicIP: fi.Bool(true),
				IAMInstanceProfile: &IAMInstanceProfile{
					Name: fi.String("nodes"),
				},
				ID:                           fi.String("test-11"),
				InstanceMonitoring:           fi.Bool(true),
				InstanceType:                 fi.String("t2.medium"),
				SpotPrice:                    "0.1",
				SpotDurationInMinutes:        fi.Int64(60),
				InstanceInterruptionBehavior: fi.String("hibernate"),
				RootVolumeOptimization:       fi.Bool(true),
				RootVolumeIops:               fi.Int64(100),
				RootVolumeSize:               fi.Int64(64),
				SSHKey: &SSHKey{
					Name:      fi.String("newkey"),
					PublicKey: fi.WrapResource(fi.NewStringResource("newkey")),
				},
				SecurityGroups: []*SecurityGroup{
					{Name: fi.String("nodes-1"), ID: fi.String("1111")},
					{Name: fi.String("nodes-2"), ID: fi.String("2222")},
				},
				Tenancy: fi.String("dedicated"),
			},
			Expected: `provider "aws" {
  region = "eu-west-2"
}

resource "aws_launch_template" "test" {
  ebs_optimized = true
  iam_instance_profile {
    name = aws_iam_instance_profile.nodes.id
  }
  instance_market_options {
    market_type = "spot"
    spot_options {
      block_duration_minutes         = 60
      instance_interruption_behavior = "hibernate"
      max_price                      = "0.1"
    }
  }
  instance_type = "t2.medium"
  key_name      = aws_key_pair.newkey.id
  lifecycle {
    create_before_destroy = true
  }
  name_prefix = "test-"
  network_interfaces {
    associate_public_ip_address = true
    delete_on_termination       = true
    security_groups             = [aws_security_group.nodes-1.id, aws_security_group.nodes-2.id]
  }
  placement {
    tenancy = "dedicated"
  }
}

terraform {
  required_version = ">= 0.12.0"
}
`,
		},
		{
			Resource: &LaunchTemplate{
				Name:              fi.String("test"),
				AssociatePublicIP: fi.Bool(true),
				IAMInstanceProfile: &IAMInstanceProfile{
					Name: fi.String("nodes"),
				},
				BlockDeviceMappings: []*BlockDeviceMapping{
					{
						DeviceName:             fi.String("/dev/xvdd"),
						EbsVolumeType:          fi.String("gp2"),
						EbsVolumeSize:          fi.Int64(100),
						EbsDeleteOnTermination: fi.Bool(true),
						EbsEncrypted:           fi.Bool(true),
					},
				},
				ID:                     fi.String("test-11"),
				InstanceMonitoring:     fi.Bool(true),
				InstanceType:           fi.String("t2.medium"),
				RootVolumeOptimization: fi.Bool(true),
				RootVolumeIops:         fi.Int64(100),
				RootVolumeSize:         fi.Int64(64),
				SSHKey: &SSHKey{
					Name: fi.String("mykey"),
				},
				SecurityGroups: []*SecurityGroup{
					{Name: fi.String("nodes-1"), ID: fi.String("1111")},
					{Name: fi.String("nodes-2"), ID: fi.String("2222")},
				},
				Tenancy: fi.String("dedicated"),
			},
			Expected: `provider "aws" {
  region = "eu-west-2"
}

resource "aws_launch_template" "test" {
  block_device_mappings {
    device_name = "/dev/xvdd"
    ebs {
      delete_on_termination = true
      encrypted             = true
      volume_size           = 100
      volume_type           = "gp2"
    }
  }
  ebs_optimized = true
  iam_instance_profile {
    name = aws_iam_instance_profile.nodes.id
  }
  instance_type = "t2.medium"
  key_name      = "mykey"
  lifecycle {
    create_before_destroy = true
  }
  name_prefix = "test-"
  network_interfaces {
    associate_public_ip_address = true
    delete_on_termination       = true
    security_groups             = [aws_security_group.nodes-1.id, aws_security_group.nodes-2.id]
  }
  placement {
    tenancy = "dedicated"
  }
}

terraform {
  required_version = ">= 0.12.0"
}
`,
		},
	}
	doRenderTests(t, "RenderTerraform", cases)
}
<commit_msg>Uodate terraform test<commit_after>/*
Copyright 2019 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package awstasks

import (
	"testing"

	"k8s.io/kops/upup/pkg/fi"
)

func TestLaunchTemplateTerraformRender(t *testing.T) {
	cases := []*renderTest{
		{
			Resource: &LaunchTemplate{
				Name:              fi.String("test"),
				AssociatePublicIP: fi.Bool(true),
				IAMInstanceProfile: &IAMInstanceProfile{
					Name: fi.String("nodes"),
				},
				ID:                           fi.String("test-11"),
				InstanceMonitoring:           fi.Bool(true),
				InstanceType:                 fi.String("t2.medium"),
				SpotPrice:                    "0.1",
				SpotDurationInMinutes:        fi.Int64(60),
				InstanceInterruptionBehavior: fi.String("hibernate"),
				RootVolumeOptimization:       fi.Bool(true),
				RootVolumeIops:               fi.Int64(100),
				RootVolumeSize:               fi.Int64(64),
				SSHKey: &SSHKey{
					Name:      fi.String("newkey"),
					PublicKey: fi.WrapResource(fi.NewStringResource("newkey")),
				},
				SecurityGroups: []*SecurityGroup{
					{Name: fi.String("nodes-1"), ID: fi.String("1111")},
					{Name: fi.String("nodes-2"), ID: fi.String("2222")},
				},
				Tenancy: fi.String("dedicated"),
			},
			Expected: `provider "aws" {
  region = "eu-west-2"
}

resource "aws_launch_template" "test" {
  ebs_optimized = true
  iam_instance_profile {
    name = aws_iam_instance_profile.nodes.id
  }
  instance_market_options {
    market_type = "spot"
    spot_options {
      block_duration_minutes         = 60
      instance_interruption_behavior = "hibernate"
      max_price                      = "0.1"
    }
  }
  instance_type = "t2.medium"
  key_name      = aws_key_pair.newkey.id
  lifecycle {
    create_before_destroy = true
  }
  name_prefix = "test-"
  network_interfaces {
    associate_public_ip_address = true
    delete_on_termination       = true
    security_groups             = [aws_security_group.nodes-1.id, aws_security_group.nodes-2.id]
  }
  placement {
    tenancy = "dedicated"
  }
}

terraform {
  required_version = ">= 0.12.0"
}
`,
		},
		{
			Resource: &LaunchTemplate{
				Name:              fi.String("test"),
				AssociatePublicIP: fi.Bool(true),
				IAMInstanceProfile: &IAMInstanceProfile{
					Name: fi.String("nodes"),
				},
				BlockDeviceMappings: []*BlockDeviceMapping{
					{
						DeviceName:             fi.String("/dev/xvdd"),
						EbsVolumeType:          fi.String("gp2"),
						EbsVolumeSize:          fi.Int64(100),
						EbsDeleteOnTermination: fi.Bool(true),
						EbsEncrypted:           fi.Bool(true),
					},
				},
				ID:                     fi.String("test-11"),
				InstanceMonitoring:     fi.Bool(true),
				InstanceType:           fi.String("t2.medium"),
				RootVolumeOptimization: fi.Bool(true),
				RootVolumeIops:         fi.Int64(100),
				RootVolumeSize:         fi.Int64(64),
				SSHKey: &SSHKey{
					Name: fi.String("mykey"),
				},
				SecurityGroups: []*SecurityGroup{
					{Name: fi.String("nodes-1"), ID: fi.String("1111")},
					{Name: fi.String("nodes-2"), ID: fi.String("2222")},
				},
				Tenancy: fi.String("dedicated"),
			},
			Expected: `provider "aws" {
  region = "eu-west-2"
}

resource "aws_launch_template" "test" {
  block_device_mappings {
    device_name = "/dev/xvdd"
    ebs {
      delete_on_termination = true
      encrypted             = true
      volume_size           = 100
      volume_type           = "gp2"
    }
  }
  ebs_optimized = true
  iam_instance_profile {
    name = aws_iam_instance_profile.nodes.id
  }
  instance_type = "t2.medium"
  key_name      = "mykey"
  lifecycle {
    create_before_destroy = true
  }
  name_prefix = "test-"
  network_interfaces {
    associate_public_ip_address = true
    delete_on_termination       = true
    security_groups             = [aws_security_group.nodes-1.id, aws_security_group.nodes-2.id]
  }
  placement {
    tenancy = "dedicated"
  }
}

terraform {
  required_version = ">= 0.12.0"
}
`,
		},
	}
	doRenderTests(t, "RenderTerraform", cases)
}
<|endoftext|>


<commit_before>package command

import (
	"encoding/json"
	"fmt"
	"math"
	"strconv"
	"strings"
	"time"
)

const (
	opSum   = "sum"
	opCount = "count"
	opMin   = "min"
	opMax   = "max"
	opMean  = "mean"
)

const (
	timeFmtSec    = "sec"
	timeFmtMsec   = "msec"
	timeFmtUsec   = "usec"
	timeFmtStruct = "timeval"
)

const (
	outputJson = "json"
	outputCsv  = "csv"
)

const max_duration_str = "24h"

var ops = []string{opSum, opCount, opMin, opMax, opMean}
var timeFmts = []string{timeFmtSec, timeFmtMsec, timeFmtUsec, timeFmtStruct}
var outputs = []string{outputJson, outputCsv}

type exportData struct {
	projectId uint64
	deviceId  string
	series    string
	timeFmt   string
	output    string

	limit       uint64
	from        uint64
	to          uint64
	lessThan    int64
	greaterThan int64
	equal       string

	operator string
	groupBy  string
}

func isInList(item string, list []string) bool {
	for _, i := range list {
		if i == item {
			return true
		}
	}
	return false
}

func (e *exportData) IsValid() bool {
	pidOk := e.projectId > 0
	limitOk := e.limit > 0
	rangeOk := e.from <= e.to
	valRangeOk := e.greaterThan <= e.greaterThan

	equalOk := len(e.equal) == 0
	if !equalOk {
		_, err := strconv.ParseInt(e.equal, 0, 64)
		equalOk = err == nil
	}

	opOk := len(e.operator) == 0 || isInList(e.operator, ops)

	groupOk := len(e.groupBy) == 0
	if !groupOk {
		_, err := time.ParseDuration(e.groupBy)
		groupOk = err == nil && len(e.operator) > 0
	}

	timeOk := isInList(e.timeFmt, timeFmts)
	outputOk := isInList(e.output, outputs)

	return pidOk && limitOk && rangeOk && valRangeOk && equalOk && opOk && groupOk && timeOk && outputOk
}

// NewExportCommand returns the base 'export' command.
func NewExportCommand(ctx *Context) *Command {
	e := new(exportData)
	pid := ctx.Profile.ActiveProject

	cmd := &Command{
		Name:    "query",
		ApiPath: "/v1/exports",
		Usage:   "Get data for projects, devices, and series",
		Data:    e,
		Action:  getExport,
	}

	flags := cmd.NewFlagSet("iobeam query")
	max_duration, _ := time.ParseDuration(max_duration_str)
	maxTime := time.Now().Add(max_duration).UnixNano() / int64(time.Millisecond)
	flags.Uint64Var(&e.projectId, "projectId", pid, "Project ID (if omitted, defaults to active project)")
	flags.StringVar(&e.deviceId, "deviceId", "", "Device ID")
	flags.StringVar(&e.series, "series", "", "Series name")

	flags.Uint64Var(&e.limit, "limit", 10, "Max number of results")
	flags.Uint64Var(&e.from, "from", 0, "Min timestamp (unix time in milliseconds)")
	flags.Uint64Var(&e.to, "to", uint64(maxTime), "Max timestamp (unix time in milliseconds, default is now + a day)")
	flags.Int64Var(&e.lessThan, "lessThan", math.MaxInt64, "Max value for datapoints")
	flags.Int64Var(&e.greaterThan, "greaterThan", math.MinInt64, "Min value for datapoints")
	flags.StringVar(&e.equal, "equalTo", "", "Datapoints with this value")
	flags.StringVar(&e.operator, "operator", "", "Aggregation function to apply to datapoints: "+strings.Join(ops, ", "))
	flags.StringVar(&e.groupBy, "groupBy", "", "Group data by [number][period], where the time period can be ms, s, m, or h (e.g., 30s, 15m, 6h). Requires a valid operator.")
	flags.StringVar(&e.timeFmt, "timeFmt", "msec", "Time unit to display timestamps: "+strings.Join(timeFmts, ", "))
	flags.StringVar(&e.output, "output", "json", "Output format of the results. Valid outputs: json, csv")
	return cmd
}

// getExport fetches the requested data from the iobeam Cloud based on
// the provided projectID, deviceID, and series name.
func getExport(c *Command, ctx *Context) error {
	e := c.Data.(*exportData)

	reqPath := c.ApiPath + "/" + strconv.FormatUint(e.projectId, 10)
	device := "all"
	if len(e.deviceId) > 0 {
		device = e.deviceId
	}
	reqPath += "/" + device
	if len(e.series) > 0 {
		reqPath += "/" + e.series
	}

	req := ctx.Client.Get(reqPath).Expect(200).
		ProjectToken(ctx.Profile, e.projectId).
		ParamUint64("limit", e.limit).
		Param("timefmt", e.timeFmt).
		Param("output", e.output)

	// Only add params if actually set / necessary, i.e.:
	// - "to" is less than current time
	// - "from" is something other than 0
	// - "lessThan" is something other than MAX INT
	// - "greaterThan" is something other than MIN INT
	// etc
	maxTime := uint64(time.Now().UnixNano() / int64(time.Millisecond))
	if e.to < maxTime {
		req = req.ParamUint64("to", e.to)
	}

	if e.from > 0 {
		req = req.ParamUint64("from", e.from)
	}

	if e.lessThan < math.MaxInt64 {
		req = req.ParamInt64("less_than", e.lessThan)
	}

	if e.greaterThan > math.MinInt64 {
		req = req.ParamInt64("greater_than", e.greaterThan)
	}

	if len(e.equal) > 0 {
		temp, _ := strconv.ParseInt(e.equal, 0, 64)
		req = req.ParamInt64("equals", temp)
	}

	if len(e.operator) > 0 {
		req = req.Param("operator", e.operator)

		if len(e.groupBy) > 0 {
			req = req.Param("group_by", e.groupBy)
		}
	}

	x := make(map[string]interface{})
	_, err := req.ResponseBody(&x).
		ResponseBodyHandler(func(token interface{}) error {
		fmt.Println("Results: ")
		if e.output == outputJson {
			output, err := json.MarshalIndent(token, "", "  ")
			fmt.Println(string(output))
			return err
		} else {
			fmt.Println(token)
			return nil
		}
	}).Execute()

	return err
}
<commit_msg>Make query output pure JSON<commit_after>package command

import (
	"encoding/json"
	"fmt"
	"math"
	"strconv"
	"strings"
	"time"
)

const (
	opSum   = "sum"
	opCount = "count"
	opMin   = "min"
	opMax   = "max"
	opMean  = "mean"
)

const (
	timeFmtSec    = "sec"
	timeFmtMsec   = "msec"
	timeFmtUsec   = "usec"
	timeFmtStruct = "timeval"
)

const (
	outputJson = "json"
	outputCsv  = "csv"
)

const max_duration_str = "24h"

var ops = []string{opSum, opCount, opMin, opMax, opMean}
var timeFmts = []string{timeFmtSec, timeFmtMsec, timeFmtUsec, timeFmtStruct}
var outputs = []string{outputJson, outputCsv}

type exportData struct {
	projectId uint64
	deviceId  string
	series    string
	timeFmt   string
	output    string

	limit       uint64
	from        uint64
	to          uint64
	lessThan    int64
	greaterThan int64
	equal       string

	operator string
	groupBy  string
}

func isInList(item string, list []string) bool {
	for _, i := range list {
		if i == item {
			return true
		}
	}
	return false
}

func (e *exportData) IsValid() bool {
	pidOk := e.projectId > 0
	limitOk := e.limit > 0
	rangeOk := e.from <= e.to
	valRangeOk := e.greaterThan <= e.greaterThan

	equalOk := len(e.equal) == 0
	if !equalOk {
		_, err := strconv.ParseInt(e.equal, 0, 64)
		equalOk = err == nil
	}

	opOk := len(e.operator) == 0 || isInList(e.operator, ops)

	groupOk := len(e.groupBy) == 0
	if !groupOk {
		_, err := time.ParseDuration(e.groupBy)
		groupOk = err == nil && len(e.operator) > 0
	}

	timeOk := isInList(e.timeFmt, timeFmts)
	outputOk := isInList(e.output, outputs)

	return pidOk && limitOk && rangeOk && valRangeOk && equalOk && opOk && groupOk && timeOk && outputOk
}

// NewExportCommand returns the base 'export' command.
func NewExportCommand(ctx *Context) *Command {
	e := new(exportData)
	pid := ctx.Profile.ActiveProject

	cmd := &Command{
		Name:    "query",
		ApiPath: "/v1/exports",
		Usage:   "Get data for projects, devices, and series",
		Data:    e,
		Action:  getExport,
	}

	flags := cmd.NewFlagSet("iobeam query")
	max_duration, _ := time.ParseDuration(max_duration_str)
	maxTime := time.Now().Add(max_duration).UnixNano() / int64(time.Millisecond)
	flags.Uint64Var(&e.projectId, "projectId", pid, "Project ID (if omitted, defaults to active project)")
	flags.StringVar(&e.deviceId, "deviceId", "", "Device ID")
	flags.StringVar(&e.series, "series", "", "Series name")

	flags.Uint64Var(&e.limit, "limit", 10, "Max number of results")
	flags.Uint64Var(&e.from, "from", 0, "Min timestamp (unix time in milliseconds)")
	flags.Uint64Var(&e.to, "to", uint64(maxTime), "Max timestamp (unix time in milliseconds, default is now + a day)")
	flags.Int64Var(&e.lessThan, "lessThan", math.MaxInt64, "Max value for datapoints")
	flags.Int64Var(&e.greaterThan, "greaterThan", math.MinInt64, "Min value for datapoints")
	flags.StringVar(&e.equal, "equalTo", "", "Datapoints with this value")
	flags.StringVar(&e.operator, "operator", "", "Aggregation function to apply to datapoints: "+strings.Join(ops, ", "))
	flags.StringVar(&e.groupBy, "groupBy", "", "Group data by [number][period], where the time period can be ms, s, m, or h (e.g., 30s, 15m, 6h). Requires a valid operator.")
	flags.StringVar(&e.timeFmt, "timeFmt", "msec", "Time unit to display timestamps: "+strings.Join(timeFmts, ", "))
	flags.StringVar(&e.output, "output", "json", "Output format of the results. Valid outputs: json, csv")
	return cmd
}

// getExport fetches the requested data from the iobeam Cloud based on
// the provided projectID, deviceID, and series name.
func getExport(c *Command, ctx *Context) error {
	e := c.Data.(*exportData)

	reqPath := c.ApiPath + "/" + strconv.FormatUint(e.projectId, 10)
	device := "all"
	if len(e.deviceId) > 0 {
		device = e.deviceId
	}
	reqPath += "/" + device
	if len(e.series) > 0 {
		reqPath += "/" + e.series
	}

	req := ctx.Client.Get(reqPath).Expect(200).
		ProjectToken(ctx.Profile, e.projectId).
		ParamUint64("limit", e.limit).
		Param("timefmt", e.timeFmt).
		Param("output", e.output)

	// Only add params if actually set / necessary, i.e.:
	// - "to" is less than current time
	// - "from" is something other than 0
	// - "lessThan" is something other than MAX INT
	// - "greaterThan" is something other than MIN INT
	// etc
	maxTime := uint64(time.Now().UnixNano() / int64(time.Millisecond))
	if e.to < maxTime {
		req = req.ParamUint64("to", e.to)
	}

	if e.from > 0 {
		req = req.ParamUint64("from", e.from)
	}

	if e.lessThan < math.MaxInt64 {
		req = req.ParamInt64("less_than", e.lessThan)
	}

	if e.greaterThan > math.MinInt64 {
		req = req.ParamInt64("greater_than", e.greaterThan)
	}

	if len(e.equal) > 0 {
		temp, _ := strconv.ParseInt(e.equal, 0, 64)
		req = req.ParamInt64("equals", temp)
	}

	if len(e.operator) > 0 {
		req = req.Param("operator", e.operator)

		if len(e.groupBy) > 0 {
			req = req.Param("group_by", e.groupBy)
		}
	}

	x := make(map[string]interface{})
	_, err := req.ResponseBody(&x).
		ResponseBodyHandler(func(token interface{}) error {

		if e.output == outputJson {
			output, err := json.MarshalIndent(token, "", "  ")
			fmt.Println(string(output))
			return err
		} else {
			fmt.Println(token)
			return nil
		}
	}).Execute()

	return err
}
<|endoftext|>