<commit_before>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"github.com/edmonds/golang-mtbl"
	"github.com/hdm/inetdata-parsers/utils"
	"github.com/peterbourgon/mergemap"
	"io/ioutil"
	"math"
	"net"
	"os"
	"runtime"
	"strings"
)

var key_only *bool
var val_only *bool
var prefix *string
var rev_prefix *string
var rev_key *bool
var no_quotes *bool
var as_json *bool
var version *bool
var domain *string
var cidr *string

func usage() {
	fmt.Println("Usage: " + os.Args[0] + " [options] <mtbl> ... <mtbl>")
	fmt.Println("")
	fmt.Println("Queries one or more MTBL databases")
	fmt.Println("")
	fmt.Println("Options:")
	flag.PrintDefaults()
}

func findPaths(args []string) []string {
	var paths []string
	for i := range args {
		path := args[i]
		info, e := os.Stat(path)
		if e != nil {
			fmt.Fprintf(os.Stderr, "Error: Path %s : %v\n", path, e)
			os.Exit(1)
		}

		if info.Mode().IsRegular() {
			paths = append(paths, path)
			continue
		}

		if info.Mode().IsDir() {
			if files, e := ioutil.ReadDir(path); e == nil {
				for _, f := range files {
					if f.Mode().IsRegular() {
						npath := path + string(os.PathSeparator) + f.Name()
						paths = append(paths, npath)
					}
				}
			}
		}
	}
	return paths
}

// TODO: Rework to handle [][]string merges
func mergeFunc(key []byte, val0 []byte, val1 []byte) (mergedVal []byte) {
	var v0, v1 map[string]interface{}

	if e := json.Unmarshal(val0, &v0); e != nil {
		return val1
	}

	if e := json.Unmarshal(val1, &v1); e != nil {
		return val0
	}

	m := mergemap.Merge(v0, v1)
	d, e := json.Marshal(m)
	if e != nil {
		fmt.Fprintf(os.Stderr, "JSON merge error: %v -> %v + %v\n", e, val0, val1)
		return val0
	}

	return d
}

func writeOutput(key_bytes []byte, val_bytes []byte) {

	key := string(key_bytes)
	val := string(val_bytes)

	if *rev_key {
		key = utils.ReverseKey(key)
	}

	if *as_json {
		o := make(map[string]interface{})
		v := make([][]string, 1)

		if de := json.Unmarshal([]byte(val), &v); de != nil {
			fmt.Fprintf(os.Stderr, "Could not unmarshal %s -> %s as json: %s\n", key, val, de)
			return
		}

		o["key"] = string(key)
		o["val"] = v

		b, je := json.Marshal(o)
		if je != nil {
			fmt.Fprintf(os.Stderr, "Could not marshal %s -> %s as json: %s\n", key, val, je)
			return
		}
		fmt.Println(string(b))

	} else if *key_only {
		fmt.Printf("%s\n", key)
	} else if *val_only {
		if *no_quotes {
			fmt.Printf("%s\n", val)
		} else {
			fmt.Printf("%q\n", val)
		}
	} else {
		if *no_quotes {
			fmt.Printf("%s\t%s\n", key, val)
		} else {
			fmt.Printf("%s\t%q\n", key, val)
		}
	}
}

func searchPrefix(m *mtbl.Merger, prefix string) {
	it := mtbl.IterPrefix(m, []byte(prefix))
	for {
		key_bytes, val_bytes, ok := it.Next()
		if !ok {
			break
		}
		writeOutput(key_bytes, val_bytes)
	}
}

func searchAll(m *mtbl.Merger) {
	it := mtbl.IterAll(m)
	for {
		key_bytes, val_bytes, ok := it.Next()
		if !ok {
			break
		}
		writeOutput(key_bytes, val_bytes)
	}
}

func searchDomain(m *mtbl.Merger, domain string) {
	rdomain := []byte(utils.ReverseKey(domain))

	// Domain searches always use reversed keys
	*rev_key = true

	// Exact match: "example.com"
	exact, found := mtbl.Get(m, rdomain)
	if found {
		writeOutput([]byte(rdomain), exact)
	}

	// Subdomain matches: ".example.com"
	dot_domain := append(rdomain, '.')
	it := mtbl.IterPrefix(m, dot_domain)
	for {
		key_bytes, val_bytes, ok := it.Next()
		if !ok {
			break
		}
		writeOutput(key_bytes, val_bytes)
	}
}

func searchCIDR(m *mtbl.Merger, cidr string) {

	// Parse CIDR into base address + mask
	ip, net, err := net.ParseCIDR(cidr)
	if err != nil {
		fmt.Fprintf(os.Stderr, "Invalid CIDR %s: %s", cidr, err.Error())
		return
	}

	// Verify IPv4 for now
	ip4 := net.IP.To4()
	if ip4 == nil {
		fmt.Fprintf(os.Stderr, "Invalid IPv4 CIDR %s: %s", cidr, err.Error())
		return
	}

	net_base, ip_err := utils.IPv4_to_UInt(net.IP.String())
	if ip_err != nil {
		fmt.Fprintf(os.Stderr, "Invalid IPv4 Address %s: %s", ip.String(), err.Error())
		return
	}

	mask_ones, mask_total := net.Mask.Size()

	// Does not work for IPv6 due to cast to uint32
	net_size := uint32(math.Pow(2, float64(mask_total-mask_ones)))

	cur_base := net_base
	end_base := net_base + net_size

	// TODO: Special case /16s to speed up
	if mask_ones == 16 {
		ip_prefix := strings.Join(strings.SplitN(utils.UInt_to_IPv4(cur_base), ".", 4)[0:2], ".") + "."
		searchPrefix(m, ip_prefix)
		return
	}

	for ; end_base-cur_base >= 256; cur_base += 256 {
		ip_prefix := strings.Join(strings.SplitN(utils.UInt_to_IPv4(cur_base), ".", 4)[0:3], ".") + "."
		searchPrefix(m, ip_prefix)
	}

	if end_base-cur_base == 0 {
		return
	}

	// One final prefix search
	ip_prefix := strings.Join(strings.SplitN(utils.UInt_to_IPv4(cur_base), ".", 4)[0:3], ".") + "."

	it := mtbl.IterPrefix(m, []byte(ip_prefix))
	for {
		key_bytes, val_bytes, ok := it.Next()
		if !ok {
			break
		}

		// Only print values in our CIDR
		cur_val, _ := utils.IPv4_to_UInt(string(key_bytes))
		if cur_val >= cur_base && cur_val <= end_base {
			writeOutput(key_bytes, val_bytes)
		}
	}

	fmt.Fprintf(os.Stderr, "Final: [%s] %s\n", cidr, ip_prefix)
}

func main() {

	runtime.GOMAXPROCS(runtime.NumCPU())
	os.Setenv("LC_ALL", "C")

	flag.Usage = func() { usage() }

	key_only = flag.Bool("k", false, "Display key names only")
	val_only = flag.Bool("v", false, "Display values only")
	prefix = flag.String("p", "", "Only return keys with this prefix")
	rev_prefix = flag.String("r", "", "Only return keys with this prefix in reverse form")
	rev_key = flag.Bool("R", false, "Display matches with the key in reverse form")
	no_quotes = flag.Bool("n", false, "Print raw values, not quoted values")
	as_json = flag.Bool("j", false, "Print each record as a single line of JSON")
	version = flag.Bool("version", false, "Show the version and build timestamp")
	domain = flag.String("domain", "", "Search for all matches for a specified domain")
	cidr = flag.String("cidr", "", "Search for all matches for the specified CIDR")

	flag.Parse()

	if *version {
		utils.PrintVersion()
		os.Exit(0)
	}

	if len(flag.Args()) == 0 {
		usage()
		os.Exit(1)
	}

	if *key_only && *val_only {
		fmt.Fprintf(os.Stderr, "Error: Only one of -k or -v can be specified\n")
		usage()
		os.Exit(1)
	}

	if len(*prefix) > 0 && len(*rev_prefix) > 0 {
		fmt.Fprintf(os.Stderr, "Error: Only one of -p or -r can be specified\n")
		usage()
		os.Exit(1)
	}

	if len(*domain) > 0 && (len(*prefix) > 0 || len(*rev_prefix) > 0 || len(*cidr) > 0) {
		fmt.Fprintf(os.Stderr, "Error: Only one of -p, -r, -domain, or -cidr can be specified\n")
		usage()
		os.Exit(1)
	}

	if len(*cidr) > 0 && (len(*prefix) > 0 || len(*rev_prefix) > 0 || len(*domain) > 0) {
		fmt.Fprintf(os.Stderr, "Error: Only one of -p, -r, -domain, or -cidr can be specified\n")
		usage()
		os.Exit(1)
	}

	paths := findPaths(flag.Args())

	m := mtbl.MergerInit(&mtbl.MergerOptions{Merge: mergeFunc})
	defer m.Destroy()

	for i := range paths {
		path := paths[i]

		r, e := mtbl.ReaderInit(path, &mtbl.ReaderOptions{VerifyChecksums: true})
		defer r.Destroy()
		if e != nil {
			fmt.Fprintf(os.Stderr, "Error reading %s: %s\n", path, e)
			os.Exit(1)
		}
		m.AddSource(r)
	}

	if len(*domain) > 0 {
		searchDomain(m, *domain)
		return
	}

	if len(*cidr) > 0 {
		searchCIDR(m, *cidr)
		return
	}

	if len(*prefix) > 0 {
		searchPrefix(m, *prefix)
		return
	}

	if len(*rev_prefix) > 0 {
		p := utils.ReverseKey(*rev_prefix)
		searchPrefix(m, p)
		return
	}

	searchAll(m)
}
<commit_msg>Fast large-block CIDRs and better merging<commit_after>package main

import (
	"bytes"
	"encoding/json"
	"flag"
	"fmt"
	"github.com/edmonds/golang-mtbl"
	"github.com/hdm/inetdata-parsers/utils"
	"github.com/peterbourgon/mergemap"
	"io/ioutil"
	"math"
	"net"
	"os"
	"runtime"
	"strings"
)

var key_only *bool
var val_only *bool
var prefix *string
var rev_prefix *string
var rev_key *bool
var no_quotes *bool
var as_json *bool
var version *bool
var domain *string
var cidr *string

func usage() {
	fmt.Println("Usage: " + os.Args[0] + " [options] <mtbl> ... <mtbl>")
	fmt.Println("")
	fmt.Println("Queries one or more MTBL databases")
	fmt.Println("")
	fmt.Println("Options:")
	flag.PrintDefaults()
}

func findPaths(args []string) []string {
	var paths []string
	for i := range args {
		path := args[i]
		info, e := os.Stat(path)
		if e != nil {
			fmt.Fprintf(os.Stderr, "Error: Path %s : %v\n", path, e)
			os.Exit(1)
		}

		if info.Mode().IsRegular() {
			paths = append(paths, path)
			continue
		}

		if info.Mode().IsDir() {
			if files, e := ioutil.ReadDir(path); e == nil {
				for _, f := range files {
					if f.Mode().IsRegular() {
						npath := path + string(os.PathSeparator) + f.Name()
						paths = append(paths, npath)
					}
				}
			}
		}
	}
	return paths
}

// Handles json formats like: {"k1": "val1"} and [ ['k1', 'v1'] ]
func mergeFunc(key []byte, val0 []byte, val1 []byte) (mergedVal []byte) {

	if bytes.Compare(val0, val1) == 0 {
		return val0
	}

	// Try to merge as a map[string]interface{}
	var v0, v1 map[string]interface{}

	if e := json.Unmarshal(val0, &v0); e == nil {
		// Looks like a map[string]interface{}

		// Try to unmarshal the second value the same way
		if e := json.Unmarshal(val1, &v1); e != nil {
			// Second value was not equivalent, return first value
			return val0
		}

		m := mergemap.Merge(v0, v1)
		d, e := json.Marshal(m)
		if e != nil {
			fmt.Fprintf(os.Stderr, "JSON merge error: %s -> %s + %s: %s\n", string(key), string(val0), string(val1), e.Error())
			return val0
		}

		return d
	}

	// Try to merge as a [][]string
	var a0, a1 [][]string

	if e := json.Unmarshal(val0, &a0); e == nil {
		// Looks like a [][]string

		// Try to unmarshal the second value the same way
		if e := json.Unmarshal(val1, &a1); e != nil {
			// Couldn't unmarshal the second value, return val0
			return val0
		}

		unique := map[string]bool{}
		m := [][]string{}

		for i := range a0 {
			unique[strings.Join(a0[i], "\x00")] = true
		}
		for i := range a1 {
			unique[strings.Join(a1[i], "\x00")] = true
		}
		for i := range unique {
			m = append(m, strings.SplitN(i, "\x00", 2))
		}

		d, e := json.Marshal(m)
		if e != nil {
			fmt.Fprintf(os.Stderr, "JSON merge error: %s -> %s + %s: %s\n", string(key), string(val0), string(val1), e.Error())
			return val0
		}

		return d
	}

	// Give up and return the first value
	return val0
}

func writeOutput(key_bytes []byte, val_bytes []byte) {

	key := string(key_bytes)
	val := string(val_bytes)

	if *rev_key {
		key = utils.ReverseKey(key)
	}

	if *as_json {
		o := make(map[string]interface{})
		v := make([][]string, 1)

		if de := json.Unmarshal([]byte(val), &v); de != nil {
			fmt.Fprintf(os.Stderr, "Could not unmarshal %s -> %s as json: %s\n", key, val, de)
			return
		}

		o["key"] = string(key)
		o["val"] = v

		b, je := json.Marshal(o)
		if je != nil {
			fmt.Fprintf(os.Stderr, "Could not marshal %s -> %s as json: %s\n", key, val, je)
			return
		}
		fmt.Println(string(b))

	} else if *key_only {
		fmt.Printf("%s\n", key)
	} else if *val_only {
		if *no_quotes {
			fmt.Printf("%s\n", val)
		} else {
			fmt.Printf("%q\n", val)
		}
	} else {
		if *no_quotes {
			fmt.Printf("%s\t%s\n", key, val)
		} else {
			fmt.Printf("%s\t%q\n", key, val)
		}
	}
}

func searchPrefix(m *mtbl.Merger, prefix string) {
	it := mtbl.IterPrefix(m, []byte(prefix))
	for {
		key_bytes, val_bytes, ok := it.Next()
		if !ok {
			break
		}
		writeOutput(key_bytes, val_bytes)
	}
}

func searchAll(m *mtbl.Merger) {
	it := mtbl.IterAll(m)
	for {
		key_bytes, val_bytes, ok := it.Next()
		if !ok {
			break
		}
		writeOutput(key_bytes, val_bytes)
	}
}

func searchDomain(m *mtbl.Merger, domain string) {
	rdomain := []byte(utils.ReverseKey(domain))

	// Domain searches always use reversed keys
	*rev_key = true

	// Exact match: "example.com"
	exact, found := mtbl.Get(m, rdomain)
	if found {
		writeOutput([]byte(rdomain), exact)
	}

	// Subdomain matches: ".example.com"
	dot_domain := append(rdomain, '.')
	it := mtbl.IterPrefix(m, dot_domain)
	for {
		key_bytes, val_bytes, ok := it.Next()
		if !ok {
			break
		}
		writeOutput(key_bytes, val_bytes)
	}
}

func searchPrefixIPv4(m *mtbl.Merger, prefix string) {
	it := mtbl.IterPrefix(m, []byte(prefix))
	for {
		key_bytes, val_bytes, ok := it.Next()
		if !ok {
			break
		}

		if utils.Match_IPv4.Match(key_bytes) {
			writeOutput(key_bytes, val_bytes)
		}
	}
}

func searchCIDR(m *mtbl.Merger, cidr string) {

	// Parse CIDR into base address + mask
	ip, net, err := net.ParseCIDR(cidr)
	if err != nil {
		fmt.Fprintf(os.Stderr, "Invalid CIDR %s: %s", cidr, err.Error())
		return
	}

	// Verify IPv4 for now
	ip4 := net.IP.To4()
	if ip4 == nil {
		fmt.Fprintf(os.Stderr, "Invalid IPv4 CIDR %s: %s", cidr, err.Error())
		return
	}

	net_base, ip_err := utils.IPv4_to_UInt(net.IP.String())
	if ip_err != nil {
		fmt.Fprintf(os.Stderr, "Invalid IPv4 Address %s: %s", ip.String(), err.Error())
		return
	}

	mask_ones, mask_total := net.Mask.Size()

	// Does not work for IPv6 due to cast to uint32
	net_size := uint32(math.Pow(2, float64(mask_total-mask_ones)))

	cur_base := net_base
	end_base := net_base + net_size

	var ndots uint32 = 3
	var block_size uint32 = 256

	// Handle massive network blocks
	if mask_ones <= 8 {
		ndots = 1
		block_size = 256 * 256 * 256
	} else if mask_ones <= 16 {
		ndots = 2
		block_size = 256 * 256
	}

	// Iterate by block size
	for ; end_base-cur_base >= block_size; cur_base += block_size {
		ip_prefix := strings.Join(strings.SplitN(utils.UInt_to_IPv4(cur_base), ".", 4)[0:ndots], ".") + "."
		searchPrefixIPv4(m, ip_prefix)
	}

	if end_base-cur_base == 0 {
		return
	}

	// Handle any leftovers by looking up a full /24 and ignoring stuff outside our range
	ip_prefix := strings.Join(strings.SplitN(utils.UInt_to_IPv4(cur_base), ".", 4)[0:3], ".") + "."

	it := mtbl.IterPrefix(m, []byte(ip_prefix))
	for {
		key_bytes, val_bytes, ok := it.Next()
		if !ok {
			break
		}

		// Only print results that are valid IPV4 addresses within our CIDR range
		cur_val, _ := utils.IPv4_to_UInt(string(key_bytes))
		if cur_val >= cur_base && cur_val <= end_base {
			if utils.Match_IPv4.Match(key_bytes) {
				writeOutput(key_bytes, val_bytes)
			}
		}
	}

	fmt.Fprintf(os.Stderr, "Final: [%s] %s\n", cidr, ip_prefix)
}

func main() {

	runtime.GOMAXPROCS(runtime.NumCPU())
	os.Setenv("LC_ALL", "C")

	flag.Usage = func() { usage() }

	key_only = flag.Bool("k", false, "Display key names only")
	val_only = flag.Bool("v", false, "Display values only")
	prefix = flag.String("p", "", "Only return keys with this prefix")
	rev_prefix = flag.String("r", "", "Only return keys with this prefix in reverse form")
	rev_key = flag.Bool("R", false, "Display matches with the key in reverse form")
	no_quotes = flag.Bool("n", false, "Print raw values, not quoted values")
	as_json = flag.Bool("j", false, "Print each record as a single line of JSON")
	version = flag.Bool("version", false, "Show the version and build timestamp")
	domain = flag.String("domain", "", "Search for all matches for a specified domain")
	cidr = flag.String("cidr", "", "Search for all matches for the specified CIDR")

	flag.Parse()

	if *version {
		utils.PrintVersion()
		os.Exit(0)
	}

	if len(flag.Args()) == 0 {
		usage()
		os.Exit(1)
	}

	if *key_only && *val_only {
		fmt.Fprintf(os.Stderr, "Error: Only one of -k or -v can be specified\n")
		usage()
		os.Exit(1)
	}

	if len(*prefix) > 0 && len(*rev_prefix) > 0 {
		fmt.Fprintf(os.Stderr, "Error: Only one of -p or -r can be specified\n")
		usage()
		os.Exit(1)
	}

	if len(*domain) > 0 && (len(*prefix) > 0 || len(*rev_prefix) > 0 || len(*cidr) > 0) {
		fmt.Fprintf(os.Stderr, "Error: Only one of -p, -r, -domain, or -cidr can be specified\n")
		usage()
		os.Exit(1)
	}

	if len(*cidr) > 0 && (len(*prefix) > 0 || len(*rev_prefix) > 0 || len(*domain) > 0) {
		fmt.Fprintf(os.Stderr, "Error: Only one of -p, -r, -domain, or -cidr can be specified\n")
		usage()
		os.Exit(1)
	}

	paths := findPaths(flag.Args())

	m := mtbl.MergerInit(&mtbl.MergerOptions{Merge: mergeFunc})
	defer m.Destroy()

	for i := range paths {
		path := paths[i]

		r, e := mtbl.ReaderInit(path, &mtbl.ReaderOptions{VerifyChecksums: true})
		defer r.Destroy()
		if e != nil {
			fmt.Fprintf(os.Stderr, "Error reading %s: %s\n", path, e)
			os.Exit(1)
		}
		m.AddSource(r)
	}

	if len(*domain) > 0 {
		searchDomain(m, *domain)
		return
	}

	if len(*cidr) > 0 {
		searchCIDR(m, *cidr)
		return
	}

	if len(*prefix) > 0 {
		searchPrefix(m, *prefix)
		return
	}

	if len(*rev_prefix) > 0 {
		p := utils.ReverseKey(*rev_prefix)
		searchPrefix(m, p)
		return
	}

	searchAll(m)
}
<|endoftext|>


<commit_before>/*
Copyright 2019 The Knative Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// This file contains functions which get actual resources given the meta resource.

package duck

import (
	"context"
	"fmt"
	"strings"

	"k8s.io/apimachinery/pkg/api/meta"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/client-go/dynamic"
	"k8s.io/client-go/util/retry"
	"knative.dev/pkg/apis/duck"

	"knative.dev/eventing/test/lib/resources"
)

// This is a workaround for https://github.com/knative/pkg/issues/1509
// Because tests currently fail immediately on any creation failure, this
// is problematic. On the reconcilers it's not an issue because they recover,
// but tests need this retry.
//
// https://github.com/knative/eventing/issues/3681
func isWebhookError(err error) bool {
	return strings.Contains(err.Error(), "eventing-webhook.knative-eventing")
}

func RetryWebhookErrors(updater func(int) error) error {
	attempts := 0
	return retry.OnError(retry.DefaultRetry, isWebhookError, func() error {
		err := updater(attempts)
		attempts++
		return err
	})
}

// GetGenericObject returns a generic object representing a Kubernetes resource.
// Callers can cast this returned object to other objects that implement the corresponding duck-type.
func GetGenericObject(
	dynamicClient dynamic.Interface,
	obj *resources.MetaResource,
	rtype runtime.Object,
) (runtime.Object, error) {
	// get the resource's namespace and gvr
	gvr, _ := meta.UnsafeGuessKindToResource(obj.GroupVersionKind())
	var u *unstructured.Unstructured
	err := RetryWebhookErrors(func(attempts int) (err error) {
		var e error
		u, e = dynamicClient.Resource(gvr).Namespace(obj.Namespace).Get(context.Background(), obj.Name, metav1.GetOptions{})
		if e != nil {
			// TODO: Plumb some sort of logging here
			fmt.Printf("Failed to get %s/%s: %v", obj.Namespace, obj.Name, e)
		}
		return e
	})

	if err != nil {
		return nil, err
	}

	res := rtype.DeepCopyObject()
	if err := duck.FromUnstructured(u, res); err != nil {
		return nil, err
	}

	return res, nil
}

// GetGenericObjectList returns a generic object list representing a list of Kubernetes resource.
func GetGenericObjectList(
	dynamicClient dynamic.Interface,
	objList *resources.MetaResourceList,
	rtype runtime.Object,
) ([]runtime.Object, error) {
	// get the resource's namespace and gvr
	gvr, _ := meta.UnsafeGuessKindToResource(objList.GroupVersionKind())
	ul, err := dynamicClient.Resource(gvr).Namespace(objList.Namespace).List(context.Background(), metav1.ListOptions{})

	if err != nil {
		return nil, err
	}

	objs := make([]runtime.Object, 0, len(ul.Items))
	for _, u := range ul.Items {
		res := rtype.DeepCopyObject()
		if err := duck.FromUnstructured(&u, res); err != nil {
			return nil, err
		}
		objs = append(objs, res)
	}

	return objs, nil
}
<commit_msg>Add newline to GetGenericObject log line in testing (#4636)<commit_after>/*
Copyright 2019 The Knative Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// This file contains functions which get actual resources given the meta resource.

package duck

import (
	"context"
	"fmt"
	"strings"

	"k8s.io/apimachinery/pkg/api/meta"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/client-go/dynamic"
	"k8s.io/client-go/util/retry"
	"knative.dev/pkg/apis/duck"

	"knative.dev/eventing/test/lib/resources"
)

// This is a workaround for https://github.com/knative/pkg/issues/1509
// Because tests currently fail immediately on any creation failure, this
// is problematic. On the reconcilers it's not an issue because they recover,
// but tests need this retry.
//
// https://github.com/knative/eventing/issues/3681
func isWebhookError(err error) bool {
	return strings.Contains(err.Error(), "eventing-webhook.knative-eventing")
}

func RetryWebhookErrors(updater func(int) error) error {
	attempts := 0
	return retry.OnError(retry.DefaultRetry, isWebhookError, func() error {
		err := updater(attempts)
		attempts++
		return err
	})
}

// GetGenericObject returns a generic object representing a Kubernetes resource.
// Callers can cast this returned object to other objects that implement the corresponding duck-type.
func GetGenericObject(
	dynamicClient dynamic.Interface,
	obj *resources.MetaResource,
	rtype runtime.Object,
) (runtime.Object, error) {
	// get the resource's namespace and gvr
	gvr, _ := meta.UnsafeGuessKindToResource(obj.GroupVersionKind())
	var u *unstructured.Unstructured
	err := RetryWebhookErrors(func(attempts int) (err error) {
		var e error
		u, e = dynamicClient.Resource(gvr).Namespace(obj.Namespace).Get(context.Background(), obj.Name, metav1.GetOptions{})
		if e != nil {
			// TODO: Plumb some sort of logging here
			fmt.Printf("Failed to get %s/%s: %v\n", obj.Namespace, obj.Name, e)
		}
		return e
	})

	if err != nil {
		return nil, err
	}

	res := rtype.DeepCopyObject()
	if err := duck.FromUnstructured(u, res); err != nil {
		return nil, err
	}

	return res, nil
}

// GetGenericObjectList returns a generic object list representing a list of Kubernetes resource.
func GetGenericObjectList(
	dynamicClient dynamic.Interface,
	objList *resources.MetaResourceList,
	rtype runtime.Object,
) ([]runtime.Object, error) {
	// get the resource's namespace and gvr
	gvr, _ := meta.UnsafeGuessKindToResource(objList.GroupVersionKind())
	ul, err := dynamicClient.Resource(gvr).Namespace(objList.Namespace).List(context.Background(), metav1.ListOptions{})

	if err != nil {
		return nil, err
	}

	objs := make([]runtime.Object, 0, len(ul.Items))
	for _, u := range ul.Items {
		res := rtype.DeepCopyObject()
		if err := duck.FromUnstructured(&u, res); err != nil {
			return nil, err
		}
		objs = append(objs, res)
	}

	return objs, nil
}
<|endoftext|>


<commit_before>package cmd

import (
	"fmt"
	log "github.com/Sirupsen/logrus"
	"github.com/sabakaio/k8s-updater/pkg/updater"
)

func update() {
	list, err := updater.NewList(k, namespace)
	if err != nil {
		log.Fatalln("Can't get deployments", err)
	}
	if len(list.Items) == 0 {
		log.Warningln("No autoupdate deployments found")
	}
	for _, c := range list.Items {
		version, err := c.GetImageVersion()
		if err != nil {
			msg := fmt.Sprintf("could not get container image version for '%s'", c.GetName())
			log.Warningln(msg, err)
		}
		latest, err := c.GetLatestVersion()
		if err != nil {
			log.Error(err)
		}
		// Update container deployment if greater image version found
		if latest.Semver.GT(version.Semver) {
			log.Debugln(c.GetName(), version.Semver.String(), "=>", latest.Semver.String())
			c.UpdateDeployment(k, namespace, latest)
		}
	}
}
<commit_msg>pass version by value<commit_after>package cmd

import (
	"fmt"
	log "github.com/Sirupsen/logrus"
	"github.com/sabakaio/k8s-updater/pkg/updater"
)

func update() {
	list, err := updater.NewList(k, namespace)
	if err != nil {
		log.Fatalln("Can't get deployments", err)
	}
	if len(list.Items) == 0 {
		log.Warningln("No autoupdate deployments found")
	}
	for _, c := range list.Items {
		version, err := c.GetImageVersion()
		if err != nil {
			msg := fmt.Sprintf("could not get container image version for '%s'", c.GetName())
			log.Warningln(msg, err)
		}
		latest, err := c.GetLatestVersion()
		if err != nil {
			log.Error(err)
		}
		// Update container deployment if greater image version found
		if latest.Semver.GT(version.Semver) {
			log.Debugln(c.GetName(), version.Semver.String(), "=>", latest.Semver.String())
			c.UpdateDeployment(k, namespace, *latest)
		}
	}
}
<|endoftext|>


<commit_before>package cmd

import (
	"fmt"

	"github.com/spf13/cobra"
)

const version = "v0.8.1"

var versionCmd = &cobra.Command{
	Use:   "version",
	Short: "Print the version number of Terraformer",
	Run: func(cmd *cobra.Command, args []string) {
		fmt.Println("Terraformer " + version)
	},
}
<commit_msg>version 0.8.2<commit_after>package cmd

import (
	"fmt"

	"github.com/spf13/cobra"
)

const version = "v0.8.2"

var versionCmd = &cobra.Command{
	Use:   "version",
	Short: "Print the version number of Terraformer",
	Run: func(cmd *cobra.Command, args []string) {
		fmt.Println("Terraformer " + version)
	},
}
<|endoftext|>


<commit_before>/*----------------------------------------------------------------
 *  Copyright (c) ThoughtWorks, Inc.
 *  Licensed under the Apache License, Version 2.0
 *  See LICENSE in the project root for license information.
 *----------------------------------------------------------------*/

package cmd

import (
	"encoding/json"
	"fmt"
	"os"
	"path/filepath"

	"github.com/getgauge/gauge/plugin/pluginInfo"
	"github.com/getgauge/gauge/version"
	"github.com/spf13/cobra"
)

var (
	versionCmd = &cobra.Command{
		Use:   "version [flags]",
		Short: "Print Gauge and plugin versions",
		Long:  `Print Gauge and plugin versions.`,
		Example: `  gauge version
  gauge version -m`,
		Run: func(cmd *cobra.Command, args []string) {
			printVersion()
		},
		PersistentPostRun: func(cmd *cobra.Command, args []string) { /* noop */ },
		DisableAutoGenTag: true,
	}
)

func init() {
	GaugeCmd.AddCommand(versionCmd)
}

func printVersion() {
	if machineReadable {
		printJSONVersion()
		return
	}
	printTextVersion()
}

func printJSONVersion() {
	type pluginJSON struct {
		Name    string `json:"name"`
		Version string `json:"version"`
	}
	type versionJSON struct {
		Version    string        `json:"version"`
		CommitHash string        `json:"commitHash"`
		Plugins    []*pluginJSON `json:"plugins"`
	}
	gaugeVersion := versionJSON{version.FullVersion(), version.CommitHash, make([]*pluginJSON, 0)}
	allPluginsWithVersion, err := pluginInfo.GetAllInstalledPluginsWithVersion()
	if err != nil {
		fmt.Println("error:", err.Error())
	}
	for _, pluginInfo := range allPluginsWithVersion {
		gaugeVersion.Plugins = append(gaugeVersion.Plugins, &pluginJSON{pluginInfo.Name, filepath.Base(pluginInfo.Path)})
	}
	b, err := json.MarshalIndent(gaugeVersion, "", "    ")
	if err != nil {
		fmt.Println("error:", err.Error())
	}
	fmt.Println(fmt.Sprintf("%s\n", string(b)))
}

func printTextVersion() {
	fmt.Printf("Gauge version: %s\n", version.FullVersion())
	v := version.CommitHash
	if v != "" {
		fmt.Printf("Commit Hash: %s\n", v)

	}
	fmt.Printf("\nPlugins\n-------\n")
	allPluginsWithVersion, err := pluginInfo.GetAllInstalledPluginsWithVersion()
	if err != nil {
		fmt.Println("No plugins found")
		fmt.Println("Plugins can be installed with `gauge install {plugin-name}`")
		os.Exit(0)
	}
	for _, pluginInfo := range allPluginsWithVersion {
		fmt.Printf("%s (%s)\n", pluginInfo.Name, filepath.Base(pluginInfo.Path))
	}
}
<commit_msg>do not write to stdout for --machinereadable, fixes getgauge/gauge-vscode#431 <commit_after>/*----------------------------------------------------------------
 *  Copyright (c) ThoughtWorks, Inc.
 *  Licensed under the Apache License, Version 2.0
 *  See LICENSE in the project root for license information.
 *----------------------------------------------------------------*/

package cmd

import (
	"encoding/json"
	"fmt"
	"os"
	"path/filepath"

	"github.com/getgauge/gauge/logger"
	"github.com/getgauge/gauge/plugin/pluginInfo"
	"github.com/getgauge/gauge/version"
	"github.com/spf13/cobra"
)

var (
	versionCmd = &cobra.Command{
		Use:   "version [flags]",
		Short: "Print Gauge and plugin versions",
		Long:  `Print Gauge and plugin versions.`,
		Example: `  gauge version
  gauge version -m`,
		Run: func(cmd *cobra.Command, args []string) {
			printVersion()
		},
		PersistentPostRun: func(cmd *cobra.Command, args []string) { /* noop */ },
		DisableAutoGenTag: true,
	}
)

func init() {
	GaugeCmd.AddCommand(versionCmd)
}

func printVersion() {
	if machineReadable {
		printJSONVersion()
		return
	}
	printTextVersion()
}

func printJSONVersion() {
	type pluginJSON struct {
		Name    string `json:"name"`
		Version string `json:"version"`
	}
	type versionJSON struct {
		Version    string        `json:"version"`
		CommitHash string        `json:"commitHash"`
		Plugins    []*pluginJSON `json:"plugins"`
	}
	gaugeVersion := versionJSON{version.FullVersion(), version.CommitHash, make([]*pluginJSON, 0)}
	allPluginsWithVersion, err := pluginInfo.GetAllInstalledPluginsWithVersion()
	if err != nil {
		logger.Errorf(false, "Error fetching plugins info: %s", err.Error())
	}
	for _, pluginInfo := range allPluginsWithVersion {
		gaugeVersion.Plugins = append(gaugeVersion.Plugins, &pluginJSON{pluginInfo.Name, filepath.Base(pluginInfo.Path)})
	}
	b, err := json.MarshalIndent(gaugeVersion, "", "    ")
	if err != nil {
		fmt.Println("Error fetching version info as JSON:", err.Error())
		return
	}
	fmt.Println(fmt.Sprintf("%s\n", string(b)))
}

func printTextVersion() {
	fmt.Printf("Gauge version: %s\n", version.FullVersion())
	v := version.CommitHash
	if v != "" {
		fmt.Printf("Commit Hash: %s\n", v)

	}
	fmt.Printf("\nPlugins\n-------\n")
	allPluginsWithVersion, err := pluginInfo.GetAllInstalledPluginsWithVersion()
	if err != nil {
		fmt.Println("No plugins found")
		fmt.Println("Plugins can be installed with `gauge install {plugin-name}`")
		os.Exit(0)
	}
	for _, pluginInfo := range allPluginsWithVersion {
		fmt.Printf("%s (%s)\n", pluginInfo.Name, filepath.Base(pluginInfo.Path))
	}
}
<|endoftext|>


<commit_before>package cmd

import (
	"fmt"

	"github.com/spf13/cobra"
)

func init() {
	root.AddCommand(versionCmd)
}

var versionCmd = &cobra.Command{
	Use:   "version",
	Short: "Print version",
	Run:   version,
}

func version(cmd *cobra.Command, args []string) {
	fmt.Println(verboseVersion())
}

// Populated at build time by `make build`, plumbed through
// main using SetMeta()
var (
	date string // datestamp
	vers string // verstionof git commit or `tip`
	hash string // git hash built from
)

// SetMeta from the build process, used for verbose version tagging.
func SetMeta(buildTimestamp, commitVersionTag, commitHash string) {
	date = buildTimestamp
	vers = commitVersionTag
	hash = commitHash
}

func verboseVersion() string {
	// If building from source (i.e. from `go install` or `go build` directly,
	// simply print 'v0.0.0-source`, a semver-valid version indicating no version
	// number.  Otherwise print the verbose version populated during `make build`.
	if vers == "" { // not statically populatd
		return "v0.0.0-source"
	}
	return fmt.Sprintf("%s-%s-%s", date, vers, hash)
}
<commit_msg>feat: version prints semver first<commit_after>package cmd

import (
	"fmt"

	"github.com/spf13/cobra"
)

func init() {
	root.AddCommand(versionCmd)
}

var versionCmd = &cobra.Command{
	Use:   "version",
	Short: "Print version",
	Run:   version,
}

func version(cmd *cobra.Command, args []string) {
	fmt.Println(verboseVersion())
}

// Populated at build time by `make build`, plumbed through
// main using SetMeta()
var (
	date string // datestamp
	vers string // verstionof git commit or `tip`
	hash string // git hash built from
)

// SetMeta from the build process, used for verbose version tagging.
func SetMeta(buildTimestamp, commitVersionTag, commitHash string) {
	date = buildTimestamp
	vers = commitVersionTag
	hash = commitHash
}

func verboseVersion() string {
	// If building from source (i.e. from `go install` or `go build` directly,
	// simply print 'v0.0.0-source`, a semver-valid version indicating no version
	// number.  Otherwise print the verbose version populated during `make build`.
	if vers == "" { // not statically populatd
		return "v0.0.0-source"
	}
	return fmt.Sprintf("%s-%s-%s", vers, date, hash)
}
<|endoftext|>


<commit_before>// Copyright 2022 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package e2e

import (
	"fmt"
	"testing"

	appsv1 "k8s.io/api/apps/v1"
	"k8s.io/apimachinery/pkg/types"
	"kpt.dev/configsync/e2e/nomostest"
	"kpt.dev/configsync/e2e/nomostest/gitproviders"
	"kpt.dev/configsync/e2e/nomostest/ntopts"
	"kpt.dev/configsync/pkg/api/configsync"
	"kpt.dev/configsync/pkg/api/configsync/v1beta1"
	"kpt.dev/configsync/pkg/testing/fake"
)

const (
	privateARHelmRegistry   = "oci://us-docker.pkg.dev/stolos-dev/configsync-ci-ar-helm"
	privateHelmChartVersion = "1.13.3"
	privateHelmChart        = "coredns"
)

// TestPublicHelm can run on both Kind and GKE clusters.
// It tests Config Sync can pull from public Helm repo without any authentication.
func TestPublicHelm(t *testing.T) {
	publicHelmRepo := "https://kubernetes.github.io/ingress-nginx"
	nt := nomostest.New(t, ntopts.SkipMonoRepo, ntopts.Unstructured)
	origRepoURL := nt.GitProvider.SyncURL(nt.RootRepos[configsync.RootSyncName].RemoteRepoName)

	rs := fake.RootSyncObjectV1Beta1(configsync.RootSyncName)
	nt.T.Log("Update RootSync to sync from a public Helm Chart")
	nt.MustMergePatch(rs, fmt.Sprintf(`{"spec": {"sourceType": "%s", "helm": {"repo": "%s", "chart": "ingress-nginx", "auth": "none", "version": "4.0.5", "releaseName": "my-ingress-nginx", "namespace": "ingress-nginx"}, "git": null}}`,
		v1beta1.HelmSource, publicHelmRepo))
	nt.T.Cleanup(func() {
		// Change the rs back so that it works in the shared test environment.
		nt.MustMergePatch(rs, fmt.Sprintf(`{"spec": {"sourceType": "%s", "helm": null, "git": {"dir": "acme", "branch": "main", "repo": "%s", "auth": "ssh","gcpServiceAccountEmail": "", "secretRef": {"name": "git-creds"}}}}`,
			v1beta1.GitSource, origRepoURL))
	})
	nt.WaitForRepoSyncs(nomostest.WithRootSha1Func(helmChartVersion("ingress-nginx:4.0.5")),
		nomostest.WithSyncDirectoryMap(map[types.NamespacedName]string{nomostest.DefaultRootRepoNamespacedName: "ingress-nginx"}))
	if err := nt.Validate("my-ingress-nginx-controller", "ingress-nginx", &appsv1.Deployment{}); err != nil {
		nt.T.Error(err)
	}
}

// TestHelmARFleetWISameProject tests the `gcpserviceaccount` auth type with Fleet Workload Identity (in-project).
//
//	The test will run on a GKE cluster only with following pre-requisites
//
// 1. Workload Identity is enabled.
// 2. The Google service account `e2e-test-ar-reader@stolos-dev.iam.gserviceaccount.com` is created with `roles/artifactregistry.reader` for access image in Artifact Registry.
// 3. An IAM policy binding is created between the Google service account and the Kubernetes service accounts with the `roles/iam.workloadIdentityUser` role.
//
//	gcloud iam service-accounts add-iam-policy-binding --project=stolos-dev \
//	   --role roles/iam.workloadIdentityUser \
//	   --member "serviceAccount:stolos-dev.svc.id.goog[config-management-system/root-reconciler]" \
//	   e2e-test-ar-reader@stolos-dev.iam.gserviceaccount.com
//
// 4. The following environment variables are set: GCP_PROJECT, GCP_CLUSTER, GCP_REGION|GCP_ZONE.
func TestHelmARFleetWISameProject(t *testing.T) {
	testWorkloadIdentity(t, workloadIdentityTestSpec{
		fleetWITest:   true,
		crossProject:  false,
		sourceRepo:    privateARHelmRegistry,
		sourceVersion: privateHelmChartVersion,
		sourceChart:   privateHelmChart,
		sourceType:    v1beta1.HelmSource,
		gsaEmail:      gsaARReaderEmail,
		rootCommitFn:  helmChartVersion(privateHelmChart + ":" + privateHelmChartVersion),
	})
}

// TestHelmARTokenAuth verifies Config Sync can pull Helm chart from private Artifact Registry with Token auth type.
// This test will work only with following pre-requisites:
// Google service account `e2e-test-ar-reader@stolos-dev.iam.gserviceaccount.com` is created with `roles/artifactregistry.reader` for accessing images in Artifact Registry.
// A JSON key file is generated for this service account and stored in Secret Manager
func TestHelmARTokenAuth(t *testing.T) {
	nt := nomostest.New(t,
		ntopts.SkipMonoRepo,
		ntopts.Unstructured,
		ntopts.RequireGKE(t),
	)

	rs := fake.RootSyncObjectV1Beta1(configsync.RootSyncName)
	nt.T.Log("Fetch password from Secret Manager")
	key, err := gitproviders.FetchCloudSecret("config-sync-ci-ar-key")
	if err != nil {
		nt.T.Fatal(err)
	}
	nt.T.Log("Create secret for authentication")
	_, err = nt.Kubectl("create", "secret", "generic", "foo", "--namespace=config-management-system", "--from-literal=username=_json_key", fmt.Sprintf("--from-literal=password=%s", key))
	if err != nil {
		nt.T.Fatalf("failed to create secret, err: %v", err)
	}
	nt.T.Log("Update RootSync to sync from a private Artifact Registry")
	nt.MustMergePatch(rs, fmt.Sprintf(`{"spec": {"sourceType": "%s", "helm": {"repo": "%s", "chart": "%s", "auth": "token", "version": "%s", "releaseName": "my-coredns", "namespace": "coredns", "secretRef": {"name" : "foo"}}, "git": null}}`,
		v1beta1.HelmSource, privateARHelmRegistry, privateHelmChart, privateHelmChartVersion))
	nt.WaitForRepoSyncs(nomostest.WithRootSha1Func(helmChartVersion(privateHelmChart+":"+privateHelmChartVersion)),
		nomostest.WithSyncDirectoryMap(map[types.NamespacedName]string{nomostest.DefaultRootRepoNamespacedName: privateHelmChart}))
	if err := nt.Validate("my-coredns-coredns", "coredns", &appsv1.Deployment{},
		containerImagePullPolicy("IfNotPresent")); err != nil {
		nt.T.Error(err)
	}
}

func helmChartVersion(chartVersion string) nomostest.Sha1Func {
	return func(*nomostest.NT, types.NamespacedName) (string, error) {
		return chartVersion, nil
	}
}
<commit_msg>Clean up the share test env after TestHelmARTokenAuth finish. (#40)<commit_after>// Copyright 2022 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package e2e

import (
	"fmt"
	"testing"

	appsv1 "k8s.io/api/apps/v1"
	"k8s.io/apimachinery/pkg/types"
	"kpt.dev/configsync/e2e/nomostest"
	"kpt.dev/configsync/e2e/nomostest/gitproviders"
	"kpt.dev/configsync/e2e/nomostest/ntopts"
	"kpt.dev/configsync/pkg/api/configsync"
	"kpt.dev/configsync/pkg/api/configsync/v1beta1"
	"kpt.dev/configsync/pkg/testing/fake"
)

const (
	privateARHelmRegistry   = "oci://us-docker.pkg.dev/stolos-dev/configsync-ci-ar-helm"
	privateHelmChartVersion = "1.13.3"
	privateHelmChart        = "coredns"
)

// TestPublicHelm can run on both Kind and GKE clusters.
// It tests Config Sync can pull from public Helm repo without any authentication.
func TestPublicHelm(t *testing.T) {
	publicHelmRepo := "https://kubernetes.github.io/ingress-nginx"
	nt := nomostest.New(t, ntopts.SkipMonoRepo, ntopts.Unstructured)
	origRepoURL := nt.GitProvider.SyncURL(nt.RootRepos[configsync.RootSyncName].RemoteRepoName)

	rs := fake.RootSyncObjectV1Beta1(configsync.RootSyncName)
	nt.T.Log("Update RootSync to sync from a public Helm Chart")
	nt.MustMergePatch(rs, fmt.Sprintf(`{"spec": {"sourceType": "%s", "helm": {"repo": "%s", "chart": "ingress-nginx", "auth": "none", "version": "4.0.5", "releaseName": "my-ingress-nginx", "namespace": "ingress-nginx"}, "git": null}}`,
		v1beta1.HelmSource, publicHelmRepo))
	nt.T.Cleanup(func() {
		// Change the rs back so that it works in the shared test environment.
		nt.MustMergePatch(rs, fmt.Sprintf(`{"spec": {"sourceType": "%s", "helm": null, "git": {"dir": "acme", "branch": "main", "repo": "%s", "auth": "ssh","gcpServiceAccountEmail": "", "secretRef": {"name": "git-creds"}}}}`,
			v1beta1.GitSource, origRepoURL))
	})
	nt.WaitForRepoSyncs(nomostest.WithRootSha1Func(helmChartVersion("ingress-nginx:4.0.5")),
		nomostest.WithSyncDirectoryMap(map[types.NamespacedName]string{nomostest.DefaultRootRepoNamespacedName: "ingress-nginx"}))
	if err := nt.Validate("my-ingress-nginx-controller", "ingress-nginx", &appsv1.Deployment{}); err != nil {
		nt.T.Error(err)
	}
}

// TestHelmARFleetWISameProject tests the `gcpserviceaccount` auth type with Fleet Workload Identity (in-project).
//
//	The test will run on a GKE cluster only with following pre-requisites
//
// 1. Workload Identity is enabled.
// 2. The Google service account `e2e-test-ar-reader@stolos-dev.iam.gserviceaccount.com` is created with `roles/artifactregistry.reader` for access image in Artifact Registry.
// 3. An IAM policy binding is created between the Google service account and the Kubernetes service accounts with the `roles/iam.workloadIdentityUser` role.
//
//	gcloud iam service-accounts add-iam-policy-binding --project=stolos-dev \
//	   --role roles/iam.workloadIdentityUser \
//	   --member "serviceAccount:stolos-dev.svc.id.goog[config-management-system/root-reconciler]" \
//	   e2e-test-ar-reader@stolos-dev.iam.gserviceaccount.com
//
// 4. The following environment variables are set: GCP_PROJECT, GCP_CLUSTER, GCP_REGION|GCP_ZONE.
func TestHelmARFleetWISameProject(t *testing.T) {
	testWorkloadIdentity(t, workloadIdentityTestSpec{
		fleetWITest:   true,
		crossProject:  false,
		sourceRepo:    privateARHelmRegistry,
		sourceVersion: privateHelmChartVersion,
		sourceChart:   privateHelmChart,
		sourceType:    v1beta1.HelmSource,
		gsaEmail:      gsaARReaderEmail,
		rootCommitFn:  helmChartVersion(privateHelmChart + ":" + privateHelmChartVersion),
	})
}

// TestHelmARTokenAuth verifies Config Sync can pull Helm chart from private Artifact Registry with Token auth type.
// This test will work only with following pre-requisites:
// Google service account `e2e-test-ar-reader@stolos-dev.iam.gserviceaccount.com` is created with `roles/artifactregistry.reader` for accessing images in Artifact Registry.
// A JSON key file is generated for this service account and stored in Secret Manager
func TestHelmARTokenAuth(t *testing.T) {
	nt := nomostest.New(t,
		ntopts.SkipMonoRepo,
		ntopts.Unstructured,
		ntopts.RequireGKE(t),
	)
	origRepoURL := nt.GitProvider.SyncURL(nt.RootRepos[configsync.RootSyncName].RemoteRepoName)

	rs := fake.RootSyncObjectV1Beta1(configsync.RootSyncName)
	nt.T.Log("Fetch password from Secret Manager")
	key, err := gitproviders.FetchCloudSecret("config-sync-ci-ar-key")
	if err != nil {
		nt.T.Fatal(err)
	}
	nt.T.Log("Create secret for authentication")
	_, err = nt.Kubectl("create", "secret", "generic", "foo", "--namespace=config-management-system", "--from-literal=username=_json_key", fmt.Sprintf("--from-literal=password=%s", key))
	if err != nil {
		nt.T.Fatalf("failed to create secret, err: %v", err)
	}
	nt.T.Log("Update RootSync to sync from a private Artifact Registry")
	nt.MustMergePatch(rs, fmt.Sprintf(`{"spec": {"sourceType": "%s", "helm": {"repo": "%s", "chart": "%s", "auth": "token", "version": "%s", "releaseName": "my-coredns", "namespace": "coredns", "secretRef": {"name" : "foo"}}, "git": null}}`,
		v1beta1.HelmSource, privateARHelmRegistry, privateHelmChart, privateHelmChartVersion))
	nt.T.Cleanup(func() {
		// Change the rs back so that it works in the shared test environment.
		nt.MustMergePatch(rs, fmt.Sprintf(`{"spec": {"sourceType": "%s", "helm": null, "git": {"dir": "acme", "branch": "main", "repo": "%s", "auth": "ssh","gcpServiceAccountEmail": "", "secretRef": {"name": "git-creds"}}}}`,
			v1beta1.GitSource, origRepoURL))
	})
	nt.WaitForRepoSyncs(nomostest.WithRootSha1Func(helmChartVersion(privateHelmChart+":"+privateHelmChartVersion)),
		nomostest.WithSyncDirectoryMap(map[types.NamespacedName]string{nomostest.DefaultRootRepoNamespacedName: privateHelmChart}))
	if err := nt.Validate("my-coredns-coredns", "coredns", &appsv1.Deployment{},
		containerImagePullPolicy("IfNotPresent")); err != nil {
		nt.T.Error(err)
	}
}

func helmChartVersion(chartVersion string) nomostest.Sha1Func {
	return func(*nomostest.NT, types.NamespacedName) (string, error) {
		return chartVersion, nil
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"math/rand"
	"time"
)

const (
	numWorkers = 20
)

func init() {
	rand.Seed(time.Now().UnixNano())
}

type workItem struct {
	reply chan int
	add   int
	dur   time.Duration
}

func wrk(ch <-chan workItem) {
	item := <-ch                        // get some work to do
	time.Sleep(item.dur)                // do the "work"
	item.reply <- rand.Int() + item.add // return the result of the "work"
}

func main() {
	// start up workers, each waiting for a submission
	submitCh := make(chan workItem)
	for i := 0; i < numWorkers; i++ {
		go wrk(submitCh)
	}

	for i := 0; i < numWorkers; i++ {
		wi := workItem{
			reply: make(chan int),
			add:   rand.Int(),
			dur:   time.Duration(rand.Intn(150)) * time.Millisecond,
		}
		submitCh <- wi
		fmt.Println(<-wi.reply)
	}

	// Note: we don't have a mechanism to shut down the wrk goroutines in a
	// clean way. Use the context example in ./ctx.go to do that!
}
<commit_msg>streaming pool results<commit_after>package main

import (
	"fmt"
	"math/rand"
	"sync"
	"time"
)

const (
	numWorkers = 20
)

func init() {
	rand.Seed(time.Now().UnixNano())
}

type workItem struct {
	reply chan int
	add   int
	dur   time.Duration
}

func wrk(ch <-chan workItem) {
	item := <-ch                        // get some work to do
	time.Sleep(item.dur)                // do the work
	item.reply <- rand.Int() + item.add // return the result of the work
}

func main() {
	// start up workers, each waiting for a submission
	submitCh := make(chan workItem)
	for i := 0; i < numWorkers; i++ {
		go wrk(submitCh)
	}

	// submit work to each worker
	workItems := make([]workItem, numWorkers)
	for i := 0; i < numWorkers; i++ {
		wItem := workItem{
			reply: make(chan int),
			add:   rand.Int(),
			dur:   time.Duration(rand.Intn(10)) * time.Second,
		}
		submitCh <- wItem
		workItems[i] = wItem
	}

	// receive work from all the workers. results will receive as they are
	// completed by workers
	var wg sync.WaitGroup
	for _, wi := range workItems {
		wg.Add(1)
		go func(repl <-chan int) {
			defer wg.Done()
			fmt.Println(<-repl)
		}(wi.reply)
	}

	wg.Wait()

	// Note: we didn't build in a mechanism to shut down the wrk goroutines.
	// Hint: use the context package from ./ctx.go to do that!
}
<|endoftext|>


<commit_before>// Copyright © 2016 Nagy Károly Gábriel <karasz@jpi.io>
// This file, part of genet, is free and unencumbered
// software released into the public domain.
// For more information, please refer to <http://unlicense.org/>
//

package cmd

import (
	"fmt"

	"github.com/spf13/cobra"
)

var GVersion = "0.0.1"
var name = "Skilled Haystack"

// versionCmd represents the version command
var versionCmd = &cobra.Command{
	Use:   "version",
	Short: "Print the version number of genet",
	Long:  `All software has versions. This is genet's.`,
	Run: func(cmd *cobra.Command, args []string) {
		fmt.Printf("Genet's version is %s\n", GVersion)
	},
}

func init() {
	versionCmd.SetUsageTemplate("Usage: \n\tgenet version\n\n")
	RootCmd.AddCommand(versionCmd)

}
<commit_msg>version 0.0.2<commit_after>// Copyright © 2016 Nagy Károly Gábriel <karasz@jpi.io>
// This file, part of genet, is free and unencumbered
// software released into the public domain.
// For more information, please refer to <http://unlicense.org/>
//

package cmd

import (
	"fmt"

	"github.com/spf13/cobra"
)

var GVersion = "0.0.2"
var name = "Bobbled Wormhole"

// versionCmd represents the version command
var versionCmd = &cobra.Command{
	Use:   "version",
	Short: "Print the version number of genet",
	Long:  `All software has versions. This is genet's.`,
	Run: func(cmd *cobra.Command, args []string) {
		fmt.Printf("Genet's version is %s\n", GVersion)
	},
}

func init() {
	versionCmd.SetUsageTemplate("Usage: \n\tgenet version\n\n")
	RootCmd.AddCommand(versionCmd)

}
<|endoftext|>


<commit_before>// Copyright © 2020 Karim Radhouani <medkarimrdi@gmail.com>
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package cmd

import (
	"fmt"

	"github.com/spf13/cobra"
)

var (
	version = "dev"
	commit  = "none"
	date    = "unknown"
	gitURL  = ""
)

// versionCmd represents the version command
var versionCmd = &cobra.Command{
	Use:   "version",
	Short: "show gnmiClient version",

	Run: func(cmd *cobra.Command, args []string) {
		fmt.Printf("version : %s\n", version)
		fmt.Printf(" commit : %s\n", commit)
		fmt.Printf("   date : %s\n", date)
		fmt.Printf(" gitURL : %s\n", gitURL)
		return
	},
}

func init() {
	rootCmd.AddCommand(versionCmd)
}
<commit_msg>add doc link<commit_after>// Copyright © 2020 Karim Radhouani <medkarimrdi@gmail.com>
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package cmd

import (
	"fmt"

	"github.com/spf13/cobra"
)

var (
	version = "dev"
	commit  = "none"
	date    = "unknown"
	gitURL  = ""
)

// versionCmd represents the version command
var versionCmd = &cobra.Command{
	Use:   "version",
	Short: "show gnmiClient version",

	Run: func(cmd *cobra.Command, args []string) {
		fmt.Printf("version : %s\n", version)
		fmt.Printf(" commit : %s\n", commit)
		fmt.Printf("   date : %s\n", date)
		fmt.Printf(" gitURL : %s\n", gitURL)
		fmt.Printf("   docs : https://gnmiclient.kmrd.dev\n")
		return
	},
}

func init() {
	rootCmd.AddCommand(versionCmd)
}
<|endoftext|>


<commit_before>package main

import (
	"crypto/tls"
	"flag"
	"fmt"
	"log"
	"math/rand"
	"net/http"
	"os"
	"regexp"
	"strconv"
	"strings"
	"time"
)

var (
	urlRE           = regexp.MustCompile(`https?:\/\/(www\.)?[-a-zA-Z0-9@:%._\+~#=]{1,256}\.[a-zA-Z0-9]{1,6}\b([-a-zA-Z0-9!@:%_\+.~#?&\/\/=$]*)`)
	skipStatus      = flag.String("a", "", "-a 500,400")
	timeout         = flag.Duration("t", 10*time.Second, "-t 10s or -t 1h")
	whitelist       = flag.String("w", "", "-w server1.com,server2.com")
	s               = rand.NewSource(time.Now().Unix())
	backoffSchedule = []time.Duration{
		1 * time.Second,
		3 * time.Second,
		10 * time.Second,
	}
)

var (
	errorColor    = "\033[1;31m%d\033[0m"
	errorStrColor = "\033[1;31m%s\033[0m"
	okColor       = "\033[1;32m%d\033[0m"
	debugColor    = "\033[1;36m%d\033[0m"
)

type response struct {
	URL      string
	Response *http.Response
	Err      error
}

func main() {
	flag.Parse()

	args := flag.Args()
	if len(args) == 0 {
		log.Fatal("filename is required")
	}

	// read file
	file, err := os.ReadFile(args[0])
	if err != nil {
		log.Fatalf("error on reading file: %v", err)
	}

	// validate skipStatus
	var skipped []int
	if len(*skipStatus) > 0 {
		splitted := strings.Split(*skipStatus, ",")
		for _, item := range splitted {
			val, err := strconv.Atoi(item)
			if err != nil {
				log.Fatalf("could not parse skip status value: %v \n", err)
			}
			skipped = append(skipped, val)
		}
	}

	// validate whitelist
	var whitelisted []string
	if len(*whitelist) > 0 {
		whitelisted = strings.Split(*whitelist, ",")
	}

	matches := urlRE.FindAllString(string(file), -1)
	client := &http.Client{
		Timeout: *timeout,
		Transport: &http.Transport{
			TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
		},
	}

	results := make(chan *response)

	// producer
	counter := 0
	for _, url := range matches {
		u := url
		if isIn(url, whitelisted) {
			continue
		}
		counter++
		go worker(u, results, client)
	}
	fmt.Printf("Found %d URIs\n", len(matches))

	totalErrors := 0
	for counter > 0 {
		resp := <-results
		counter--
		if resp.Err != nil && resp.Response == nil {
			fmt.Printf("[%s] %s\n", fmt.Sprintf(errorStrColor, "ERROR"), resp.Err.Error())
			totalErrors++
			continue
		}

		shouldSkipURL := len(skipped) > 0 && isIn(resp.Response.StatusCode, skipped)
		statusColor := okColor
		if resp.Response.StatusCode > http.StatusBadRequest && !shouldSkipURL {
			statusColor = errorColor
			totalErrors++
		} else if shouldSkipURL {
			statusColor = debugColor
		}

		fmt.Printf("[%s] %s \n", fmt.Sprintf(statusColor, resp.Response.StatusCode), resp.URL)
	}

	if totalErrors > 0 {
		fmt.Printf("Total Errors: %s \n", fmt.Sprintf(errorColor, totalErrors))
		os.Exit(1)
	}
}

func worker(url string, results chan<- *response, client *http.Client) {
	var err error

	response := &response{
		URL: url,
	}

	req, err := http.NewRequest("GET", url, nil)
	if err != nil {
		response.Err = err
		return
	}

	userAgents := []string{
		"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_5) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.1.1 Safari/605.1.15",
		"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:77.0) Gecko/20100101 Firefox/77.0",
		"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.97 Safari/537.36",
		"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:77.0) Gecko/20100101 Firefox/77.0",
		"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.97 Safari/537.36",
	}

	userAgent := userAgents[rand.Intn(len(userAgents))]

	req.Header.Add("User-Agent", userAgent)

	for _, backoff := range backoffSchedule {
		resp, err := client.Do(req)
		if err != nil {
			time.Sleep(backoff)
		} else {
			response.Response = resp
			results <- response
			break
		}
	}

	if err != nil {
		response.Err = err
		results <- response
	}
}

func isIn[item int | string](val item, values []item) bool {
	for _, i := range values {
		if i == val {
			return true
		}
	}
	return false
}
<commit_msg>fix backoff<commit_after>package main

import (
	"crypto/tls"
	"flag"
	"fmt"
	"log"
	"math/rand"
	"net/http"
	"os"
	"regexp"
	"strconv"
	"strings"
	"time"
)

var (
	urlRE           = regexp.MustCompile(`https?:\/\/(www\.)?[-a-zA-Z0-9@:%._\+~#=]{1,256}\.[a-zA-Z0-9]{1,6}\b([-a-zA-Z0-9!@:%_\+.~#?&\/\/=$]*)`)
	skipStatus      = flag.String("a", "", "-a 500,400")
	timeout         = flag.Duration("t", 10*time.Second, "-t 10s or -t 1h")
	whitelist       = flag.String("w", "", "-w server1.com,server2.com")
	s               = rand.NewSource(time.Now().Unix())
	backoffSchedule = []time.Duration{
		1 * time.Second,
		3 * time.Second,
		5 * time.Second,
	}
)

var (
	errorColor    = "\033[1;31m%d\033[0m"
	errorStrColor = "\033[1;31m%s\033[0m"
	okColor       = "\033[1;32m%d\033[0m"
	debugColor    = "\033[1;36m%d\033[0m"
)

type response struct {
	URL      string
	Response *http.Response
	Err      error
}

func main() {
	flag.Parse()

	args := flag.Args()
	if len(args) == 0 {
		log.Fatal("filename is required")
	}

	// read file
	file, err := os.ReadFile(args[0])
	if err != nil {
		log.Fatalf("error on reading file: %v", err)
	}

	// validate skipStatus
	var skipped []int
	if len(*skipStatus) > 0 {
		splitted := strings.Split(*skipStatus, ",")
		for _, item := range splitted {
			val, err := strconv.Atoi(item)
			if err != nil {
				log.Fatalf("could not parse skip status value: %v \n", err)
			}
			skipped = append(skipped, val)
		}
	}

	// validate whitelist
	var whitelisted []string
	if len(*whitelist) > 0 {
		whitelisted = strings.Split(*whitelist, ",")
	}

	matches := urlRE.FindAllString(string(file), -1)
	client := &http.Client{
		Timeout: *timeout,
		Transport: &http.Transport{
			TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
		},
	}

	results := make(chan *response)

	// producer
	counter := 0
	for _, url := range matches {
		u := url
		if isIn(u, whitelisted) {
			continue
		}
		counter++
		go worker(u, results, client)
	}
	fmt.Printf("Found %d URIs\n", len(matches))

	totalErrors := 0
	for counter > 0 {
		resp := <-results
		counter--
		if resp.Err != nil && resp.Response == nil {
			fmt.Printf("[%s] %s\n", fmt.Sprintf(errorStrColor, "ERROR"), resp.Err.Error())
			totalErrors++
			continue
		}

		shouldSkipURL := len(skipped) > 0 && isIn(resp.Response.StatusCode, skipped)
		statusColor := okColor
		if resp.Response.StatusCode > http.StatusBadRequest && !shouldSkipURL {
			statusColor = errorColor
			totalErrors++
		} else if shouldSkipURL {
			statusColor = debugColor
		}

		fmt.Printf("[%s] %s \n", fmt.Sprintf(statusColor, resp.Response.StatusCode), resp.URL)
	}

	if totalErrors > 0 {
		fmt.Printf("Total Errors: %s \n", fmt.Sprintf(errorColor, totalErrors))
		os.Exit(1)
	}
}

func newRequest(url string) (*http.Request, error) {
	userAgents := []string{
		"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_5) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/13.1.1 Safari/605.1.15",
		"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:77.0) Gecko/20100101 Firefox/77.0",
		"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.97 Safari/537.36",
		"Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:77.0) Gecko/20100101 Firefox/77.0",
		"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.97 Safari/537.36",
	}

	req, err := http.NewRequest("GET", url, nil)
	if err != nil {
		return nil, err
	}

	userAgent := userAgents[rand.Intn(len(userAgents))]

	req.Header.Add("User-Agent", userAgent)

	return req, err
}

func worker(url string, results chan<- *response, client *http.Client) {
	var err error

	response := &response{
		URL: url,
	}

	req, err := newRequest(url)
	if err != nil {
		response.Err = err
		return
	}

	resp, err := client.Do(req)
	if err != nil {
		for _, backoff := range backoffSchedule {
			time.Sleep(backoff)

			// trying a new request with a different user-agent
			req, err := newRequest(url)
			if err != nil {
				response.Err = err
				break
			}

			resp, err := client.Do(req)
			if err == nil {
				response.Response = resp
				break
			}
		}
	}

	response.Response = resp
	response.Err = err
	results <- response
}

func isIn[item int | string](val item, values []item) bool {
	for _, i := range values {
		if i == val {
			return true
		}
	}
	return false
}
<|endoftext|>


<commit_before>/*
	ZX file server.

	Export zx trees
*/
package main

import (
	"clive/cmd"
	"clive/cmd/opt"
	"clive/dbg"
	"clive/net/auth"
	"clive/zx"
	"clive/zx/zxc"
	"clive/zx/rzx"
	"clive/zx/zux"
	fpath "path"
	"path/filepath"
	"strings"
)

var (
	noauth bool
	Zdebug bool
	dprintf = cmd.Dprintf
	vprintf = cmd.VWarn

	opts  = opt.New("{spec}")
	port, addr  string
)

func main() {
	cmd.UnixIO()
	opts.AddUsage("\tspec is name | name!file | name!file!flags \n")
	opts.AddUsage("\tspec flags are ro | rw | ncro | ncrw \n")
	port = "8002"
	addr = "*!*!zx"
	opts.NewFlag("p", "port: tcp server port (8002 by default)", &port)
	opts.NewFlag("a", "addr: service address (*!*!zx by default)", &addr)
	c := cmd.AppCtx()
	opts.NewFlag("D", "debug", &c.Debug)
	opts.NewFlag("v", "report users logged in/out (verbose)", &c.Verb)
	opts.NewFlag("Z", "verbose debug", &Zdebug)
	opts.NewFlag("n", "no auth", &noauth)
	args, err := opts.Parse()
	if err != nil {
		cmd.Warn("%s", err)
		opts.Usage()
	}
	if len(args) == 0 {
		cmd.Warn("missing arguments")
		opts.Usage()
	}
	c.Debug = c.Debug || Zdebug
	auth.Debug = c.Debug

	trs := map[string]zx.Fs{}
	ros := map[bool]string{false: "rw", true: "ro"}
	cs := map[bool]string{false: "uncached", true: "cached"}
	rotrs := map[string]bool{}
	var mainfs zx.Fs
	for i := 0; i < len(args); i++ {
		al := strings.Split(args[i], "!")
		if len(al) == 1 {
			al = append(al, al[0])
			al[0] = fpath.Base(al[0])
		}
		if _, ok := trs[al[0]]; ok {
			cmd.Warn("dup tree name %s", al[0])
			continue
		}
		ronly := false
		caching := true
		if len(al) == 3 && strings.Contains(al[2], "ro") {
			ronly = true
		}
		if len(al) == 3 && strings.Contains(al[2], "nc") {
			caching = false
		}
		fp, _ := filepath.Abs(al[1])
		t, err := zux.NewZX(fp)
		if err != nil {
			cmd.Warn("%s: %s", al[0], err)
			continue
		}
		t.Tag = al[0]
		cmd.Warn("%s %s %s", al[0], ros[ronly], cs[caching])
		var x zx.Fs = t
		if caching {
			x, err = zxc.New(t)
			if err != nil {
				dbg.Warn("%s: zxc: %s", al[0], err)
				continue
			}
			if Zdebug {
				x.(*zxc.Fs).Debug = true
			}
		} else if Zdebug {
			x.(*zux.Fs).Debug = true
		}
		trs[t.Tag] = x
		if i == 0 {
			mainfs = x
		}
		rotrs[t.Tag] = ronly
	}
	if len(trs) == 0 {
		cmd.Fatal("no trees to serve")
	}
	if _, ok := trs["main"]; !ok {
		trs["main"] = mainfs
	}
	vprintf("serve %s...", addr)
	srv, err := rzx.NewServer(addr, auth.TLSserver)
	if err != nil {
		cmd.Fatal("serve: %s", err)
	}
	if noauth {
		srv.NoAuth()
	}
	if c.Debug {
		srv.Debug = true
	}
	for nm, fs := range trs {
		if cfs, ok := fs.(*zxc.Fs); ok {
			cfs.Flags.Add("debug", &srv.Debug)
			cfs.Flags.Add("zdebug", &cfs.Debug)
		} else if lfs, ok := fs.(*zux.Fs); ok {
			lfs.Flags.Add("debug", &srv.Debug)
			lfs.Flags.Add("zdebug", &lfs.Debug)
		}
		if rotrs[nm] {
			fs = zx.MakeRO(fs)
			trs[nm] = fs
		}
		if err := srv.Serve(nm, fs); err != nil {
			cmd.Fatal("serve: %s: %s", nm, err)
		}
	}
	if err := srv.Wait(); err != nil {
		cmd.Fatal("srv: %s", err)
	}
}
<commit_msg>zxc flag<commit_after>/*
	ZX file server.

	Export zx trees
*/
package main

import (
	"clive/cmd"
	"clive/cmd/opt"
	"clive/dbg"
	"clive/net/auth"
	"clive/zx"
	"clive/zx/zxc"
	"clive/zx/rzx"
	"clive/zx/zux"
	fpath "path"
	"path/filepath"
	"strings"
)

var (
	noauth, wsync bool
	Zdebug bool
	dprintf = cmd.Dprintf
	vprintf = cmd.VWarn

	opts  = opt.New("{spec}")
	port, addr  string
)

func main() {
	cmd.UnixIO()
	opts.AddUsage("\tspec is name | name!file | name!file!flags \n")
	opts.AddUsage("\tspec flags are ro | rw | ncro | ncrw \n")
	port = "8002"
	addr = "*!*!zx"
	opts.NewFlag("p", "port: tcp server port (8002 by default)", &port)
	opts.NewFlag("a", "addr: service address (*!*!zx by default)", &addr)
	opts.NewFlag("s", "use writesync for caches", &wsync)
	c := cmd.AppCtx()
	opts.NewFlag("D", "debug", &c.Debug)
	opts.NewFlag("v", "report users logged in/out (verbose)", &c.Verb)
	opts.NewFlag("Z", "verbose debug", &Zdebug)
	opts.NewFlag("n", "no auth", &noauth)
	args, err := opts.Parse()
	if err != nil {
		cmd.Warn("%s", err)
		opts.Usage()
	}
	if len(args) == 0 {
		cmd.Warn("missing arguments")
		opts.Usage()
	}
	c.Debug = c.Debug || Zdebug
	auth.Debug = c.Debug

	trs := map[string]zx.Fs{}
	ros := map[bool]string{false: "rw", true: "ro"}
	cs := map[bool]string{false: "uncached", true: "cached"}
	rotrs := map[string]bool{}
	var mainfs zx.Fs
	for i := 0; i < len(args); i++ {
		al := strings.Split(args[i], "!")
		if len(al) == 1 {
			al = append(al, al[0])
			al[0] = fpath.Base(al[0])
		}
		if _, ok := trs[al[0]]; ok {
			cmd.Warn("dup tree name %s", al[0])
			continue
		}
		ronly := false
		caching := true
		if len(al) == 3 && strings.Contains(al[2], "ro") {
			ronly = true
		}
		if len(al) == 3 && strings.Contains(al[2], "nc") {
			caching = false
		}
		fp, _ := filepath.Abs(al[1])
		t, err := zux.NewZX(fp)
		if err != nil {
			cmd.Warn("%s: %s", al[0], err)
			continue
		}
		t.Tag = al[0]
		cmd.Warn("%s %s %s", al[0], ros[ronly], cs[caching])
		var x zx.Fs = t
		if caching {
			x, err = zxc.New(t)
			if err != nil {
				dbg.Warn("%s: zxc: %s", al[0], err)
				continue
			}
			if Zdebug {
				x.(*zxc.Fs).Debug = true
			}
			if wsync {
				x.(*zxc.Fs).Flags.Set("writesync", true)
			}
		} else if Zdebug {
			x.(*zux.Fs).Debug = true
		}
		trs[t.Tag] = x
		if i == 0 {
			mainfs = x
		}
		rotrs[t.Tag] = ronly
	}
	if len(trs) == 0 {
		cmd.Fatal("no trees to serve")
	}
	if _, ok := trs["main"]; !ok {
		trs["main"] = mainfs
	}
	vprintf("serve %s...", addr)
	srv, err := rzx.NewServer(addr, auth.TLSserver)
	if err != nil {
		cmd.Fatal("serve: %s", err)
	}
	if noauth {
		srv.NoAuth()
	}
	if c.Debug {
		srv.Debug = true
	}
	for nm, fs := range trs {
		if cfs, ok := fs.(*zxc.Fs); ok {
			cfs.Flags.Add("debug", &srv.Debug)
			cfs.Flags.Add("zdebug", &cfs.Debug)
		} else if lfs, ok := fs.(*zux.Fs); ok {
			lfs.Flags.Add("debug", &srv.Debug)
			lfs.Flags.Add("zdebug", &lfs.Debug)
		}
		if rotrs[nm] {
			fs = zx.MakeRO(fs)
			trs[nm] = fs
		}
		if err := srv.Serve(nm, fs); err != nil {
			cmd.Fatal("serve: %s: %s", nm, err)
		}
	}
	if err := srv.Wait(); err != nil {
		cmd.Fatal("srv: %s", err)
	}
}
<|endoftext|>


<commit_before>package dockerignore // import "github.com/docker/docker/builder/dockerignore"

import (
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"testing"
)

func TestReadAll(t *testing.T) {
	tmpDir, err := ioutil.TempDir("", "dockerignore-test")
	if err != nil {
		t.Fatal(err)
	}
	defer os.RemoveAll(tmpDir)

	di, err := ReadAll(nil)
	if err != nil {
		t.Fatalf("Expected not to have error, got %v", err)
	}

	if diLen := len(di); diLen != 0 {
		t.Fatalf("Expected to have zero dockerignore entry, got %d", diLen)
	}

	diName := filepath.Join(tmpDir, ".dockerignore")
	content := fmt.Sprintf("test1\n/test2\n/a/file/here\n\nlastfile\n# this is a comment\n! /inverted/abs/path\n!\n! \n")
	err = ioutil.WriteFile(diName, []byte(content), 0777)
	if err != nil {
		t.Fatal(err)
	}

	diFd, err := os.Open(diName)
	if err != nil {
		t.Fatal(err)
	}
	defer diFd.Close()

	di, err = ReadAll(diFd)
	if err != nil {
		t.Fatal(err)
	}

	if len(di) != 7 {
		t.Fatalf("Expected 5 entries, got %v", len(di))
	}
	if di[0] != "test1" {
		t.Fatal("First element is not test1")
	}
	if di[1] != "test2" { // according to https://docs.docker.com/engine/reference/builder/#dockerignore-file, /foo/bar should be treated as foo/bar
		t.Fatal("Second element is not test2")
	}
	if di[2] != "a/file/here" { // according to https://docs.docker.com/engine/reference/builder/#dockerignore-file, /foo/bar should be treated as foo/bar
		t.Fatal("Third element is not a/file/here")
	}
	if di[3] != "lastfile" {
		t.Fatal("Fourth element is not lastfile")
	}
	if di[4] != "!inverted/abs/path" {
		t.Fatal("Fifth element is not !inverted/abs/path")
	}
	if di[5] != "!" {
		t.Fatalf("Sixth element is not !, but %s", di[5])
	}
	if di[6] != "!" {
		t.Fatalf("Sixth element is not !, but %s", di[6])
	}
}
<commit_msg>Fix typos<commit_after>package dockerignore // import "github.com/docker/docker/builder/dockerignore"

import (
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"testing"
)

func TestReadAll(t *testing.T) {
	tmpDir, err := ioutil.TempDir("", "dockerignore-test")
	if err != nil {
		t.Fatal(err)
	}
	defer os.RemoveAll(tmpDir)

	di, err := ReadAll(nil)
	if err != nil {
		t.Fatalf("Expected not to have error, got %v", err)
	}

	if diLen := len(di); diLen != 0 {
		t.Fatalf("Expected to have zero dockerignore entry, got %d", diLen)
	}

	diName := filepath.Join(tmpDir, ".dockerignore")
	content := fmt.Sprintf("test1\n/test2\n/a/file/here\n\nlastfile\n# this is a comment\n! /inverted/abs/path\n!\n! \n")
	err = ioutil.WriteFile(diName, []byte(content), 0777)
	if err != nil {
		t.Fatal(err)
	}

	diFd, err := os.Open(diName)
	if err != nil {
		t.Fatal(err)
	}
	defer diFd.Close()

	di, err = ReadAll(diFd)
	if err != nil {
		t.Fatal(err)
	}

	if len(di) != 7 {
		t.Fatalf("Expected 7 entries, got %v", len(di))
	}
	if di[0] != "test1" {
		t.Fatal("First element is not test1")
	}
	if di[1] != "test2" { // according to https://docs.docker.com/engine/reference/builder/#dockerignore-file, /foo/bar should be treated as foo/bar
		t.Fatal("Second element is not test2")
	}
	if di[2] != "a/file/here" { // according to https://docs.docker.com/engine/reference/builder/#dockerignore-file, /foo/bar should be treated as foo/bar
		t.Fatal("Third element is not a/file/here")
	}
	if di[3] != "lastfile" {
		t.Fatal("Fourth element is not lastfile")
	}
	if di[4] != "!inverted/abs/path" {
		t.Fatal("Fifth element is not !inverted/abs/path")
	}
	if di[5] != "!" {
		t.Fatalf("Sixth element is not !, but %s", di[5])
	}
	if di[6] != "!" {
		t.Fatalf("Seventh element is not !, but %s", di[6])
	}
}
<|endoftext|>


<commit_before>// Copyright 2014 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package backups

import (
	"compress/gzip"
	"fmt"
	"io/ioutil"
	"os"
	"path"
	"path/filepath"
	"time"

	"github.com/juju/errors"
	"github.com/juju/utils/hash"
	"github.com/juju/utils/tar"
)

// Backup creates a tar.gz file named juju-backup_<date YYYYMMDDHHMMSS>.tar.gz
// in the specified outputFolder.
// The backup contents look like this:
//   juju-backup/dump/ - the files generated from dumping the database
//   juju-backup/root.tar - contains all the files needed by juju
// Between the two, this is all that is necessary to later restore the
// juju agent on another machine.
func Backup(password string, username string, outputFolder string, addr string) (filename string, sha1sum string, err error) {
	// YYYYMMDDHHMMSS
	formattedDate := time.Now().Format("20060102150405")
	bkpFile := fmt.Sprintf("juju-backup_%s.tar.gz", formattedDate)

	// Prepare the temp dirs.
	root, contentdir, dumpdir, err := prepareTemp()
	if err != nil {
		return "", "", errors.Trace(err)
	}
	defer os.RemoveAll(root)

	// Dump the files.
	logger.Debugf("dumping state-related files")
	err = dumpFiles(contentdir)
	if err != nil {
		return "", "", errors.Trace(err)
	}

	// Dump the database.
	logger.Debugf("dumping database")
	dbinfo := NewDBConnInfo(addr, username, password)
	err = dumpDatabase(dbinfo, dumpdir)
	if err != nil {
		return "", "", errors.Trace(err)
	}

	// Bundle it all into a tarball.
	logger.Debugf("building archive file")
	shaSum, err := createBundle(bkpFile, outputFolder, contentdir, root+sep)
	if err != nil {
		return "", "", errors.Trace(err)
	}

	return bkpFile, shaSum, nil
}

func prepareTemp() (root, contentdir, dumpdir string, err error) {
	root, err = ioutil.TempDir("", "jujuBackup")
	contentdir = filepath.Join(root, "juju-backup")
	dumpdir = filepath.Join(contentdir, "dump")
	err = os.MkdirAll(dumpdir, os.FileMode(0755))
	if err != nil {
		err = errors.Annotate(err, "error creating temporary directories")
	}
	return
}

func createBundle(name, outdir, contentdir, root string) (string, error) {
	archive, err := os.Create(filepath.Join(outdir, name))
	if err != nil {
		return "", errors.Annotate(err, "error opening archive file")
	}
	defer archive.Close()
	hasher := hash.NewSHA1Proxy(archive)
	tarball := gzip.NewWriter(hasher)

	_, err = tar.TarFiles([]string{contentdir}, tarball, root)
	tarball.Close()
	if err != nil {
		return "", errors.Annotate(err, "error bundling final archive")
	}

	return hasher.Base64Sum(), nil
}

// StorageName returns the path in environment storage where a backup
// should be stored.  That name is derived from the provided filename.
func StorageName(filename string) string {
	// Use of path.Join instead of filepath.Join is intentional - this
	// is an environment storage path not a filesystem path.
	return path.Join("/backups", filepath.Base(filename))
}
<commit_msg>Escalate log levels in backups and add more logged info.<commit_after>// Copyright 2014 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package backups

import (
	"compress/gzip"
	"fmt"
	"io/ioutil"
	"os"
	"path"
	"path/filepath"
	"time"

	"github.com/juju/errors"
	"github.com/juju/utils/hash"
	"github.com/juju/utils/tar"
)

// Backup creates a tar.gz file named juju-backup_<date YYYYMMDDHHMMSS>.tar.gz
// in the specified outputFolder.
// The backup contents look like this:
//   juju-backup/dump/ - the files generated from dumping the database
//   juju-backup/root.tar - contains all the files needed by juju
// Between the two, this is all that is necessary to later restore the
// juju agent on another machine.
func Backup(password string, username string, outputFolder string, addr string) (filename string, sha1sum string, err error) {
	// YYYYMMDDHHMMSS
	formattedDate := time.Now().Format("20060102150405")
	bkpFile := fmt.Sprintf("juju-backup_%s.tar.gz", formattedDate)

	// Prepare the temp dirs.
	root, contentdir, dumpdir, err := prepareTemp()
	if err != nil {
		return "", "", errors.Trace(err)
	}
	defer os.RemoveAll(root)

	// Dump the files.
	logger.Infof("dumping state-related files")
	err = dumpFiles(contentdir)
	if err != nil {
		return "", "", errors.Trace(err)
	}

	// Dump the database.
	logger.Infof("dumping database")
	dbinfo := NewDBConnInfo(addr, username, password)
	err = dumpDatabase(dbinfo, dumpdir)
	if err != nil {
		return "", "", errors.Trace(err)
	}

	// Bundle it all into a tarball.
	logger.Infof("building archive file (%s)", bkpFile)
	shaSum, err := createBundle(bkpFile, outputFolder, contentdir, root+sep)
	if err != nil {
		return "", "", errors.Trace(err)
	}

	return bkpFile, shaSum, nil
}

func prepareTemp() (root, contentdir, dumpdir string, err error) {
	root, err = ioutil.TempDir("", "jujuBackup")
	contentdir = filepath.Join(root, "juju-backup")
	dumpdir = filepath.Join(contentdir, "dump")
	err = os.MkdirAll(dumpdir, os.FileMode(0755))
	if err != nil {
		err = errors.Annotate(err, "error creating temporary directories")
	}
	return
}

func createBundle(name, outdir, contentdir, root string) (string, error) {
	archive, err := os.Create(filepath.Join(outdir, name))
	if err != nil {
		return "", errors.Annotate(err, "error opening archive file")
	}
	defer archive.Close()
	hasher := hash.NewSHA1Proxy(archive)
	tarball := gzip.NewWriter(hasher)

	_, err = tar.TarFiles([]string{contentdir}, tarball, root)
	tarball.Close()
	if err != nil {
		return "", errors.Annotate(err, "error bundling final archive")
	}

	return hasher.Base64Sum(), nil
}

// StorageName returns the path in environment storage where a backup
// should be stored.  That name is derived from the provided filename.
func StorageName(filename string) string {
	// Use of path.Join instead of filepath.Join is intentional - this
	// is an environment storage path not a filesystem path.
	return path.Join("/backups", filepath.Base(filename))
}
<|endoftext|>


<commit_before>// Copyright 2014 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package storage

import (
	"io"

	"github.com/juju/errors"
	"github.com/juju/loggo"
	"labix.org/v2/mgo"
)

var logger = loggo.GetLogger("juju.storage")

type gridFSStorage struct {
	namespace string
	session   *mgo.Session
}

var _ ResourceStorage = (*gridFSStorage)(nil)

// NewGridFS returns a ResourceStorage instance backed by a mongo GridFS.
// namespace is used to segregate different sets of data.
func NewGridFS(namespace string, session *mgo.Session) ResourceStorage {
	return &gridFSStorage{
		namespace: namespace,
		session:   session,
	}
}

func (g *gridFSStorage) db() *mgo.Database {
	return g.session.DB("juju")
}

func (g *gridFSStorage) gridFS() *mgo.GridFS {
	return g.db().GridFS(g.namespace)
}

// Get is defined on ResourceStorage.
func (g *gridFSStorage) Get(path string) (io.ReadCloser, error) {
	file, err := g.gridFS().Open(path)
	if err != nil {
		return nil, errors.Annotatef(err, "failed to open GridFS file %q", path)
	}
	return file, nil
}

// Put is defined on ResourceStorage.
func (g *gridFSStorage) Put(path string, r io.Reader, length int64) (checksum string, error error) {
	file, err := g.gridFS().Create(path)
	if err != nil {
		return "", errors.Annotatef(err, "failed to create GridFS file %q", path)
	}
	defer func() {
		if err != nil {
			file.Close()
			if err := g.Remove(path); err != nil {
				logger.Warningf("error cleaning up after failed write: %v", err)
			}
		}
	}()
	num, err := io.Copy(file, r)
	if err != nil {
		return "", errors.Annotatef(err, "failed to write data")
	}
	if num != length {
		return "", errors.Errorf("expected to write %d bytes, only wrote %d", length, num)
	}
	err = file.Close()
	if err != nil {
		return "", errors.Annotatef(err, "failed to flush data")
	}
	return file.MD5(), nil
}

// Remove is defined on ResourceStorage.
func (g *gridFSStorage) Remove(path string) error {
	return g.gridFS().Remove(path)
}
<commit_msg>Code review fixes<commit_after>// Copyright 2014 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package storage

import (
	"io"

	"github.com/juju/errors"
	"github.com/juju/loggo"
	"labix.org/v2/mgo"
)

var logger = loggo.GetLogger("juju.storage")

type gridFSStorage struct {
	namespace string
	session   *mgo.Session
}

var _ ResourceStorage = (*gridFSStorage)(nil)

// NewGridFS returns a ResourceStorage instance backed by a mongo GridFS.
// namespace is used to segregate different sets of data.
func NewGridFS(namespace string, session *mgo.Session) ResourceStorage {
	return &gridFSStorage{
		namespace: namespace,
		session:   session,
	}
}

func (g *gridFSStorage) db() *mgo.Database {
	return g.session.DB("juju")
}

func (g *gridFSStorage) gridFS() *mgo.GridFS {
	return g.db().GridFS(g.namespace)
}

// Get is defined on ResourceStorage.
func (g *gridFSStorage) Get(path string) (io.ReadCloser, error) {
	file, err := g.gridFS().Open(path)
	if err != nil {
		return nil, errors.Annotatef(err, "failed to open GridFS file %q", path)
	}
	return file, nil
}

// Put is defined on ResourceStorage.
func (g *gridFSStorage) Put(path string, r io.Reader, length int64) (checksum string, err error) {
	file, err := g.gridFS().Create(path)
	if err != nil {
		return "", errors.Annotatef(err, "failed to create GridFS file %q", path)
	}
	defer func() {
		if err != nil {
			file.Close()
			if removeErr := g.Remove(path); removeErr != nil {
				logger.Warningf("error cleaning up after failed write: %v", removeErr)
			}
		}
	}()
	if _, err = io.CopyN(file, r, length); err != nil {
		return "", errors.Annotatef(err, "failed to write data")
	}
	if err = file.Close(); err != nil {
		return "", errors.Annotatef(err, "failed to flush data")
	}
	return file.MD5(), nil
}

// Remove is defined on ResourceStorage.
func (g *gridFSStorage) Remove(path string) error {
	return g.gridFS().Remove(path)
}
<|endoftext|>


<commit_before>package core

import (
	"sync"

	"github.com/intelsdilabs/pulse/pkg/ctree"
)

type ConfigDataTree struct {
	cTree *ctree.ConfigTree
}

func NewConfigDataTree() *ConfigDataTree {
	return &ConfigDataTree{
		cTree: ctree.New(),
	}
}

func (c *ConfigDataTree) Add(ns []string, cdn *ConfigDataNode) {
	c.cTree.Add(ns, cdn)
}

func (c *ConfigDataTree) Get(ns []string) *ConfigDataNode {
	// Automatically freeze on first Get
	if !c.cTree.Frozen() {
		c.cTree.Freeze()
	}

	n := c.cTree.Get(ns)
	if n == nil {
		return nil
	} else {
		cd := n.(ConfigDataNode)
		return &cd
	}
}

func (c *ConfigDataTree) Freeze() {
	c.cTree.Freeze()
}

type ConfigDataNode struct {
	mutex sync.Mutex
	table map[string]ConfigValue
}

func NewConfigDataNode() *ConfigDataNode {
	return &ConfigDataNode{
		table: make(map[string]ConfigValue),
	}
}

func (c *ConfigDataNode) Table() map[string]ConfigValue {
	c.mutex.Lock()
	defer c.mutex.Unlock()
	return c.table
}

func (c *ConfigDataNode) AddItem(k string, v ConfigValue) {
	// And empty is a noop
	if k == "" {
		return
	}
	c.mutex.Lock()
	defer c.mutex.Unlock()
	c.table[k] = v
}

func (c ConfigDataNode) Merge(n ctree.Node) ctree.Node {
	// Because Add only allows the ConfigDataNode type we
	// are safe to convert ctree.Node interface to ConfigDataNode
	cd := n.(*ConfigDataNode)
	t := cd.Table()
	// For the table in the passed ConfigDataNode(converted) add each item to
	// this ConfigDataNode overwritting where needed.
	for k, v := range t {
		c.AddItem(k, v)
	}
	// Return modified version of ConfigDataNode(as ctree.Node)
	return c
}
<commit_msg>Switched to pointer to sync.Mutex from value in ConfigDataNode (to prevent passing Locks by value elsewhere)<commit_after>package core

import (
	"sync"

	"github.com/intelsdilabs/pulse/pkg/ctree"
)

type ConfigDataTree struct {
	cTree *ctree.ConfigTree
}

func NewConfigDataTree() *ConfigDataTree {
	return &ConfigDataTree{
		cTree: ctree.New(),
	}
}

func (c *ConfigDataTree) Add(ns []string, cdn *ConfigDataNode) {
	c.cTree.Add(ns, cdn)
}

func (c *ConfigDataTree) Get(ns []string) *ConfigDataNode {
	// Automatically freeze on first Get
	if !c.cTree.Frozen() {
		c.cTree.Freeze()
	}

	n := c.cTree.Get(ns)
	if n == nil {
		return nil
	} else {
		cd := n.(ConfigDataNode)
		return &cd
	}
}

func (c *ConfigDataTree) Freeze() {
	c.cTree.Freeze()
}

type ConfigDataNode struct {
	mutex *sync.Mutex
	table map[string]ConfigValue
}

func NewConfigDataNode() *ConfigDataNode {
	return &ConfigDataNode{
		mutex: new(sync.Mutex),
		table: make(map[string]ConfigValue),
	}
}

func (c *ConfigDataNode) Table() map[string]ConfigValue {
	c.mutex.Lock()
	defer c.mutex.Unlock()
	return c.table
}

func (c *ConfigDataNode) AddItem(k string, v ConfigValue) {
	// And empty is a noop
	if k == "" {
		return
	}
	c.mutex.Lock()
	defer c.mutex.Unlock()
	c.table[k] = v
}

func (c ConfigDataNode) Merge(n ctree.Node) ctree.Node {
	// Because Add only allows the ConfigDataNode type we
	// are safe to convert ctree.Node interface to ConfigDataNode
	cd := n.(*ConfigDataNode)
	t := cd.Table()
	// For the table in the passed ConfigDataNode(converted) add each item to
	// this ConfigDataNode overwritting where needed.
	for k, v := range t {
		c.AddItem(k, v)
	}
	// Return modified version of ConfigDataNode(as ctree.Node)
	return c
}
<|endoftext|>


<commit_before>package arena

import (
	"encoding/json"
	"fmt"
	"io"
	"math/rand"
	"time"

	"github.com/asciimoo/solary/arena/board"
	"github.com/asciimoo/solary/arena/coord"
	"github.com/asciimoo/solary/player"
)

const (
	ROUND_TIMEOUT = 2
	MAX_ROUNDS    = 500
)

var battle_id uint = 0
var player_id uint = 0

type Arena struct {
	Id      uint
	Round   uint
	Players []*player.Player
	Board   *board.Board
}

func Create() *Arena {
	a := &Arena{
		battle_id,
		0,
		make([]*player.Player, 0),
		board.Create(),
	}
	battle_id += 1
	return a
}

func (a *Arena) Play() {
	ch := make(chan *player.Move)
	a.setSpawnPos()
	for _, p := range a.Players {
		defer p.Conn.Close()
		go p.Read(ch)
	}
	for {
		a.Round += 1
		a.broadcastStatus()
		if a.Round == MAX_ROUNDS || a.getActivePlayersNum() == 0 {
			fmt.Println("Game", a.Id, "finished")
			return
		}
		if a.Round%100 == 0 {
			a.Board.PopulateRandomLoot()
		}
		// collect moves
		moves := a.getMoves(ch)
		// activate laser beams and traps
		for _, move := range moves {
			if move.Item == "" {
				continue
			}
			c, ok := move.Player.Inventory[move.Item]
			if !ok || c <= 0 {
				continue
			}
			move.Player.Inventory[move.Item] -= 1
			switch move.Item {
			case "trap":
				a.Board.FieldByCoord(move.Player.Position).AddTrap()
			case "laser beam":
				laser_x := 0
				laser_y := 0
				switch move.Direction {
				case "up":
					laser_y -= 1
				case "down":
					laser_y += 1
				case "left":
					laser_x -= 1
				case "right":
					laser_x += 1
				}
				for i := 1; i <= 2; i++ {
					laser_coord := coord.Coord{
						uint(laser_x*i) + move.Player.Position.X,
						uint(laser_y*i) + move.Player.Position.Y,
					}
					for _, p := range a.Players {
						if p.Position.X == laser_coord.X && p.Position.Y == laser_coord.Y {
							p.Life -= 25
						}
					}
				}
			case "oil":
				if move.Player.Life > 80 {
					move.Player.Life = 100
				} else {
					move.Player.Life += 20
				}
			}
		}

		a.movePlayers(&moves)

		a.checkDeath()

		// helper coord->user map
		player_coords := make(map[coord.Coord][]*player.Player)
		for _, p := range a.Players {
			player_coords[p.Position] = append(player_coords[p.Position], p)
		}

		// activate destination traps && collect loot
		for c, players := range player_coords {
			field := a.Board.FieldByCoord(c)
			if len(players) == 1 {
				for _, l := range a.Board.FieldByCoord(c).Loot {
					players[0].Inventory[l] += 1
				}
				field.ClearLoot()
			}
			if field.Traps > 0 {
				for _, p := range players {
					p.Life -= 50 * a.Board.FieldByCoord(c).Traps
				}
				field.ClearTraps()
			}
		}

		a.checkDeath()

		// trigger inventory items
		for _, p := range a.Players {
			p.Score += p.Inventory["solar panel"]
		}
	}
}

func (a *Arena) checkDeath() {
	for _, p := range a.Players {
		if p.Life <= 0 {
			field := a.Board.FieldByCoord(p.Position)
			for loot, loot_count := range p.Inventory {
				p.Inventory[loot] = 0
				for i := 0; i < loot_count; i++ {
					field.Loot = append(field.Loot, loot)
				}
			}
			p.Life = 100
			p.Position = p.SpawnPosition
		}
	}
}

func (a *Arena) setSpawnPos() {
	board_size := len(a.Board.Fields)
	spawn_x := 0
	spawn_y := 0
	for {
		spawn_x = rand.Intn(board_size / 2)
		spawn_y = rand.Intn(board_size / 2)
		if a.Board.IsValidLocation(spawn_x, spawn_y) {
			break
		}
	}
	for i, p := range a.Players {
		if i%2 == 0 {
			p.SpawnPosition.X = uint(spawn_x)
		} else {
			p.SpawnPosition.X = uint(board_size - spawn_x - 1)
		}
		if i%4 < 2 {
			p.SpawnPosition.Y = uint(spawn_y)
		} else {
			p.SpawnPosition.Y = uint(board_size - spawn_y - 1)
		}
		p.Position = p.SpawnPosition
	}

}

func (a *Arena) movePlayers(moves *[]*player.Move) {
	for _, move := range *moves {
		a.movePlayer(move)
	}
}

func (a *Arena) movePlayer(m *player.Move) {
	new_y := m.Player.Position.Y
	new_x := m.Player.Position.X
	distance := uint(1)
	if m.Item == "pogo stick" {
		distance = 2
	}
	switch m.Direction {
	case "up":
		new_y -= distance
	case "down":
		new_y += distance
	case "left":
		new_x -= distance
	case "right":
		new_x += distance
	}
	if a.Board.IsValidLocation(int(new_x), int(new_y)) {
		m.Player.Position.X = new_x
		m.Player.Position.Y = new_y
	}
}

func (a *Arena) getMoves(ch chan *player.Move) []*player.Move {
	moves := []*player.Move{}
	timeout := make(chan bool)
	go func() {
		time.Sleep(ROUND_TIMEOUT * time.Second)
		timeout <- true
	}()
	recv_break := false
	for !recv_break && len(moves) < a.getActivePlayersNum() {
		select {
		case move := <-ch:
			if move.Error != nil {
				fmt.Println("user error:", move.Error)
				break
			}
			can_move := true
			for _, m := range moves {
				if m.Player == move.Player {
					can_move = false
					break
				}
			}
			if can_move {
				moves = append(moves, move)
			} else {
				fmt.Println("Error: already moved")
			}
		case <-timeout:
			fmt.Println("Timeout")
			recv_break = true
		}
	}
	return moves
}

func (a *Arena) getActivePlayersNum() int {
	active_players := 0
	for _, player := range a.Players {
		if !player.Disconnected {
			active_players += 1
		}
	}
	return active_players
}

func (a *Arena) broadcastStatus() {
	b, err := json.Marshal(a)
	if err != nil {
		fmt.Println(err)
	}
	for _, p := range a.Players {
		p.Write(b)
	}
}

func (a *Arena) AddPlayer(conn io.ReadWriteCloser) {
	p := player.Create(player_id, conn)
	msg, _ := json.Marshal(p)
	p.Write(msg)
	player_id += 1
	a.Players = append(a.Players, p)
}
<commit_msg>[mod] spawn position order<commit_after>package arena

import (
	"encoding/json"
	"fmt"
	"io"
	"math/rand"
	"time"

	"github.com/asciimoo/solary/arena/board"
	"github.com/asciimoo/solary/arena/coord"
	"github.com/asciimoo/solary/player"
)

const (
	ROUND_TIMEOUT = 2
	MAX_ROUNDS    = 500
)

var battle_id uint = 0
var player_id uint = 0

type Arena struct {
	Id      uint
	Round   uint
	Players []*player.Player
	Board   *board.Board
}

func Create() *Arena {
	a := &Arena{
		battle_id,
		0,
		make([]*player.Player, 0),
		board.Create(),
	}
	battle_id += 1
	return a
}

func (a *Arena) Play() {
	ch := make(chan *player.Move)
	a.setSpawnPos()
	for _, p := range a.Players {
		defer p.Conn.Close()
		go p.Read(ch)
	}
	for {
		a.Round += 1
		a.broadcastStatus()
		if a.Round == MAX_ROUNDS || a.getActivePlayersNum() == 0 {
			fmt.Println("Game", a.Id, "finished")
			return
		}
		if a.Round%100 == 0 {
			a.Board.PopulateRandomLoot()
		}
		// collect moves
		moves := a.getMoves(ch)
		// activate laser beams and traps
		for _, move := range moves {
			if move.Item == "" {
				continue
			}
			c, ok := move.Player.Inventory[move.Item]
			if !ok || c <= 0 {
				continue
			}
			move.Player.Inventory[move.Item] -= 1
			switch move.Item {
			case "trap":
				a.Board.FieldByCoord(move.Player.Position).AddTrap()
			case "laser beam":
				laser_x := 0
				laser_y := 0
				switch move.Direction {
				case "up":
					laser_y -= 1
				case "down":
					laser_y += 1
				case "left":
					laser_x -= 1
				case "right":
					laser_x += 1
				}
				for i := 1; i <= 2; i++ {
					laser_coord := coord.Coord{
						uint(laser_x*i) + move.Player.Position.X,
						uint(laser_y*i) + move.Player.Position.Y,
					}
					for _, p := range a.Players {
						if p.Position.X == laser_coord.X && p.Position.Y == laser_coord.Y {
							p.Life -= 25
						}
					}
				}
			case "oil":
				if move.Player.Life > 80 {
					move.Player.Life = 100
				} else {
					move.Player.Life += 20
				}
			}
		}

		a.movePlayers(&moves)

		a.checkDeath()

		// helper coord->user map
		player_coords := make(map[coord.Coord][]*player.Player)
		for _, p := range a.Players {
			player_coords[p.Position] = append(player_coords[p.Position], p)
		}

		// activate destination traps && collect loot
		for c, players := range player_coords {
			field := a.Board.FieldByCoord(c)
			if len(players) == 1 {
				for _, l := range a.Board.FieldByCoord(c).Loot {
					players[0].Inventory[l] += 1
				}
				field.ClearLoot()
			}
			if field.Traps > 0 {
				for _, p := range players {
					p.Life -= 50 * a.Board.FieldByCoord(c).Traps
				}
				field.ClearTraps()
			}
		}

		a.checkDeath()

		// trigger inventory items
		for _, p := range a.Players {
			p.Score += p.Inventory["solar panel"]
		}
	}
}

func (a *Arena) checkDeath() {
	for _, p := range a.Players {
		if p.Life <= 0 {
			field := a.Board.FieldByCoord(p.Position)
			for loot, loot_count := range p.Inventory {
				p.Inventory[loot] = 0
				for i := 0; i < loot_count; i++ {
					field.Loot = append(field.Loot, loot)
				}
			}
			p.Life = 100
			p.Position = p.SpawnPosition
		}
	}
}

func (a *Arena) setSpawnPos() {
	board_size := len(a.Board.Fields)
	spawn_x := 0
	spawn_y := 0
	for {
		spawn_x = rand.Intn(board_size / 2)
		spawn_y = rand.Intn(board_size / 2)
		if a.Board.IsValidLocation(spawn_x, spawn_y) {
			break
		}
	}
	for i, p := range a.Players {
		if i%2 == 0 {
			p.SpawnPosition.X = uint(spawn_x)
		} else {
			p.SpawnPosition.X = uint(board_size - spawn_x - 1)
		}
		if (i+1)%4 > 1 {
			p.SpawnPosition.Y = uint(spawn_y)
		} else {
			p.SpawnPosition.Y = uint(board_size - spawn_y - 1)
		}
		p.Position = p.SpawnPosition
	}

}

func (a *Arena) movePlayers(moves *[]*player.Move) {
	for _, move := range *moves {
		a.movePlayer(move)
	}
}

func (a *Arena) movePlayer(m *player.Move) {
	new_y := m.Player.Position.Y
	new_x := m.Player.Position.X
	distance := uint(1)
	if m.Item == "pogo stick" {
		distance = 2
	}
	switch m.Direction {
	case "up":
		new_y -= distance
	case "down":
		new_y += distance
	case "left":
		new_x -= distance
	case "right":
		new_x += distance
	}
	if a.Board.IsValidLocation(int(new_x), int(new_y)) {
		m.Player.Position.X = new_x
		m.Player.Position.Y = new_y
	}
}

func (a *Arena) getMoves(ch chan *player.Move) []*player.Move {
	moves := []*player.Move{}
	timeout := make(chan bool)
	go func() {
		time.Sleep(ROUND_TIMEOUT * time.Second)
		timeout <- true
	}()
	recv_break := false
	for !recv_break && len(moves) < a.getActivePlayersNum() {
		select {
		case move := <-ch:
			if move.Error != nil {
				fmt.Println("user error:", move.Error)
				break
			}
			can_move := true
			for _, m := range moves {
				if m.Player == move.Player {
					can_move = false
					break
				}
			}
			if can_move {
				moves = append(moves, move)
			} else {
				fmt.Println("Error: already moved")
			}
		case <-timeout:
			fmt.Println("Timeout")
			recv_break = true
		}
	}
	return moves
}

func (a *Arena) getActivePlayersNum() int {
	active_players := 0
	for _, player := range a.Players {
		if !player.Disconnected {
			active_players += 1
		}
	}
	return active_players
}

func (a *Arena) broadcastStatus() {
	b, err := json.Marshal(a)
	if err != nil {
		fmt.Println(err)
	}
	for _, p := range a.Players {
		p.Write(b)
	}
}

func (a *Arena) AddPlayer(conn io.ReadWriteCloser) {
	p := player.Create(player_id, conn)
	msg, _ := json.Marshal(p)
	p.Write(msg)
	player_id += 1
	a.Players = append(a.Players, p)
}
<|endoftext|>


<commit_before>package modelhelper

import (
	"koding/db/models"
	"koding/db/mongodb"

	"labix.org/v2/mgo"
)

func GetAllRelationships(selector Selector) ([]models.Relationship, error) {
	relationships := make([]models.Relationship, 0)

	query := func(c *mgo.Collection) error {
		return c.Find(selector).Sort("timestamp").All(&relationships)
	}

	err := mongodb.Run("relationships", query)

	return relationships, err
}

func GetSomeRelationships(selector Selector, limit int) ([]models.Relationship, error) {
	relationships := make([]models.Relationship, 0)

	query := func(c *mgo.Collection) error {
		return c.Find(selector).Limit(limit).All(&relationships)
	}

	err := mongodb.Run("relationships", query)

	return relationships, err
}

func GetRelationship(selector Selector) (models.Relationship, error) {
	relationship := models.Relationship{}

	query := func(c *mgo.Collection) error {
		return c.Find(selector).One(&relationship)
	}

	err := mongodb.Run("relationships", query)

	return relationship, err
}

func DeleteRelationship(selector Selector) error {
	query := func(c *mgo.Collection) error {
		_, err := c.RemoveAll(selector)
		return err
	}

	return mongodb.Run("relationships", query)
}

func AddRelationship(r *models.Relationship) error {
	query := insertQuery(r)

	return mongodb.Run("relationships", query)
}

func UpdateRelationship(r *models.Relationship) error {
	query := updateByIdQuery(r.Id.Hex(), r)
	return mongodb.Run("relationships", query)
}

func UpdateRelationships(selector, options Selector) error {
	query := func(c *mgo.Collection) error {
		_, err := c.UpdateAll(selector, options)
		return err
	}
	return mongodb.Run("relationships", query)
}
<commit_msg>Go Model: add relationship count function<commit_after>package modelhelper

import (
	"koding/db/models"
	"koding/db/mongodb"

	"labix.org/v2/mgo"
)

func GetAllRelationships(selector Selector) ([]models.Relationship, error) {
	relationships := make([]models.Relationship, 0)

	query := func(c *mgo.Collection) error {
		return c.Find(selector).Sort("timestamp").All(&relationships)
	}

	err := mongodb.Run("relationships", query)

	return relationships, err
}

func GetSomeRelationships(selector Selector, limit int) ([]models.Relationship, error) {
	relationships := make([]models.Relationship, 0)

	query := func(c *mgo.Collection) error {
		return c.Find(selector).Limit(limit).All(&relationships)
	}

	err := mongodb.Run("relationships", query)

	return relationships, err
}

func GetRelationship(selector Selector) (models.Relationship, error) {
	relationship := models.Relationship{}

	query := func(c *mgo.Collection) error {
		return c.Find(selector).One(&relationship)
	}

	err := mongodb.Run("relationships", query)

	return relationship, err
}

func DeleteRelationship(selector Selector) error {
	query := func(c *mgo.Collection) error {
		_, err := c.RemoveAll(selector)
		return err
	}

	return mongodb.Run("relationships", query)
}

func AddRelationship(r *models.Relationship) error {
	query := insertQuery(r)

	return mongodb.Run("relationships", query)
}

func UpdateRelationship(r *models.Relationship) error {
	query := updateByIdQuery(r.Id.Hex(), r)
	return mongodb.Run("relationships", query)
}

func UpdateRelationships(selector, options Selector) error {
	query := func(c *mgo.Collection) error {
		_, err := c.UpdateAll(selector, options)
		return err
	}
	return mongodb.Run("relationships", query)
}

func RelationshipCount(selector Selector) (int, error) {
	var count int
	var err error
	query := func(c *mgo.Collection) error {
		count, err = c.Find(selector).Count()
		return err
	}
	return count, mongodb.Run("relationships", query)
}
<|endoftext|>