<commit_before>package main

import (
	"fmt"
	"io/ioutil"
	"net/http"
	"sync"
)

type webResponse struct {
	requestUrl, responseText string
	status                   int
	error
}

// WaitGroup is used to wait for the program to finish goroutines.
var wg sync.WaitGroup

func main() {
	urls := []string{
		"https://golang.org",
		"https://www.google.com",
		"https://twitter.com",
	}

	wg.Add(2)
	webResponses := make(chan webResponse, len(urls))
	go getWebResponses(urls, webResponses)
	go printWebResponses(webResponses)
	wg.Wait()
}

func getWebResponses(urls []string, webResponses chan<- webResponse) {
	defer wg.Done()
	defer close(webResponses)
	var wgUrlReq sync.WaitGroup
	wgUrlReq.Add(len(urls))
	for _, v := range urls {
		go func(url string) {
			defer wgUrlReq.Done()
			responseVal := webResponse{}
			responseVal.requestUrl = url
			response, err := http.Get(url)
			if err != nil {
				responseVal.error = err
				responseVal.status = http.StatusInternalServerError
			} else {
				defer response.Body.Close()
				body, err := ioutil.ReadAll(response.Body)
				if err == nil {
					responseVal.responseText = string(body)
				}
				responseVal.error = err
				responseVal.status = response.StatusCode
			}
			webResponses <- responseVal
		}(v)
	}
	wgUrlReq.Wait()
}

func printWebResponses(webResponses <-chan webResponse) {
	defer wg.Done()
	for v := range webResponses {
		fmt.Println(v.requestUrl)
		if v.error != nil {
			fmt.Println("Error:", v.error)
		}
		fmt.Println("HTTP Status Code:", v.status)
		fmt.Println("Response Text:\n", v.responseText)

	}
}
<commit_msg>Minor modifications<commit_after>package main

import (
	"fmt"
	"io/ioutil"
	"net/http"
	"sync"
)

type webResponse struct {
	requestUrl, responseText string
	status                   int
	error
}

// WaitGroup is used to wait for the program to finish goroutines.
var wg sync.WaitGroup

func main() {
	urls := []string{
		"https://github.com/shijuvar",
		"https://www.google.com",
		"https://twitter.com",
	}

	wg.Add(2)
	webResponses := make(chan webResponse, len(urls))
	go getWebResponses(urls, webResponses)
	go printWebResponses(webResponses)
	wg.Wait()
}

func getWebResponses(urls []string, webResponses chan<- webResponse) {
	defer wg.Done()
	defer close(webResponses)
	var wgUrlReq sync.WaitGroup
	wgUrlReq.Add(len(urls))
	for _, v := range urls {
		go func(url string) {
			defer wgUrlReq.Done()
			responseVal := webResponse{}
			responseVal.requestUrl = url
			response, err := http.Get(url)
			if err != nil {
				responseVal.error = err
				responseVal.status = http.StatusInternalServerError
			} else {
				defer response.Body.Close()
				body, err := ioutil.ReadAll(response.Body)
				if err == nil {
					responseVal.responseText = string(body)
				}
				responseVal.error = err
				responseVal.status = response.StatusCode
			}
			webResponses <- responseVal
		}(v)
	}
	wgUrlReq.Wait()
}

func printWebResponses(webResponses <-chan webResponse) {
	defer wg.Done()
	for v := range webResponses {
		fmt.Println(v.requestUrl)
		if v.error != nil {
			fmt.Println("Error:", v.error)
		}
		fmt.Println("HTTP Status Code:", v.status)
		fmt.Println("Response Text:\n", v.responseText)

	}
}
<|endoftext|>


<commit_before>package json

import "github.com/gopherjs/gopherjs/js"

func Marshal(v interface{}) ([]byte, error) {
	return MarshalIndent(v, "", "")
}

func MarshalIndent(v interface{}, prefix, indent string) ([]byte, error) {
	s, err := MarshalIndentString(v, prefix, indent)
	return []byte(s), err
}

func MarshalString(v interface{}) (string, error) {
	return MarshalIndentString(v, "", "")
}

func MarshalIndentString(v interface{}, prefix, indent string) (string, error) {
	str := js.Global.Get("JSON").Call("stringify", js.InternalObject(filterIt).Invoke(js.InternalObject(v), v), nil, indent)
	if len(prefix) > 0 {
		str = str.Call("replace", js.Global.Get("RegExp").New("\n", "g"), "\n"+prefix)
	}
	return str.String(), nil
}

const (
	arrayKind     = 17
	interfaceKind = 20
	mapKind       = 21
	ptrKind       = 22
	sliceKind     = 23
	structKind    = 25
)

func filterIt(v, value *js.Object) *js.Object {
	filter(v, v.Get("constructor"), value)
	return value
}

func filter(v, t, value *js.Object) {
	switch t.Get("kind").Int() {
	case arrayKind, sliceKind:
		len := value.Length()
		for i := 0; i < len; i++ {
			filter(v.Get("$array").Index(i), t.Get("elem"), value.Index(i))
		}
	case mapKind:
		keys := js.Global.Get("Object").Call("keys", value)
		len := keys.Length()
		for i := 0; i < len; i++ {
			name := keys.Index(i).String()
			filter(v.Get("$map").Get(name), t.Get("elem"), value.Get(name))
		}
	case ptrKind:
		filter(v, t.Get("elem"), value)
	case structKind:
		nf := t.Get("fields").Length()
		for i := 0; i < nf; i++ {
			field := t.Get("fields").Index(i)
			tag, tagOptions := parseTag(getJSONTag(field.Get("tag")))
			if tag == "-" {
				value.Delete(field.Get("name").String())
				continue
			}
			name := field.Get("name").String()
			if tagOptions.Contains("omitempty") {
				if isEmpty(v.Get(name)) {
					value.Delete(name)
					continue
				}
			}
			//if tagOptions.Contains("string") {

			//} else {
			filter(v.Get(name), field.Get("typ"), value.Get(name))
			//}
			if len(tag) != 0 && tag != name {
				value.Set(tag, value.Get(name))
				value.Delete(name)
			}
		}
	case interfaceKind:
		filter(v.Get("$val"), v.Get("constructor"), value)
	}
}

func isEmpty(v *js.Object) bool {
	return false
}
<commit_msg>implemented isEmpty<commit_after>package json

import "github.com/gopherjs/gopherjs/js"

func Marshal(v interface{}) ([]byte, error) {
	return MarshalIndent(v, "", "")
}

func MarshalIndent(v interface{}, prefix, indent string) ([]byte, error) {
	s, err := MarshalIndentString(v, prefix, indent)
	return []byte(s), err
}

func MarshalString(v interface{}) (string, error) {
	return MarshalIndentString(v, "", "")
}

func MarshalIndentString(v interface{}, prefix, indent string) (string, error) {
	str := js.Global.Get("JSON").Call("stringify", js.InternalObject(filterIt).Invoke(js.InternalObject(v), v), nil, indent)
	if len(prefix) > 0 {
		str = str.Call("replace", js.Global.Get("RegExp").New("\n", "g"), "\n"+prefix)
	}
	return str.String(), nil
}

const (
	boolKind       = 1
	intKind        = 2
	int8Kind       = 3
	int16Kind      = 4
	int32Kind      = 5
	int64Kind      = 6
	uintKind       = 7
	uint8Kind      = 8
	uint16Kind     = 9
	uint32Kind     = 10
	uint64Kind     = 11
	uintptrKind    = 12
	float32Kind    = 13
	float64Kind    = 14
	complex64Kind  = 15
	complex128Kind = 16
	arrayKind      = 17
	interfaceKind  = 20
	mapKind        = 21
	ptrKind        = 22
	sliceKind      = 23
	stringKind     = 24
	structKind     = 25
)

func filterIt(v, value *js.Object) *js.Object {
	filter(v, v.Get("constructor"), value)
	return value
}

func filter(v, t, value *js.Object) {
	switch t.Get("kind").Int() {
	case arrayKind, sliceKind:
		len := value.Length()
		for i := 0; i < len; i++ {
			filter(v.Get("$array").Index(i), t.Get("elem"), value.Index(i))
		}
	case mapKind:
		keys := js.Global.Get("Object").Call("keys", v)
		len := keys.Length()
		for i := 0; i < len; i++ {
			o := v.Get(keys.Index(i).String())
			filter(o.Get("v"), t.Get("elem"), value.Get(o.Get("k").String()))
		}
	case ptrKind:
		filter(v, t.Get("elem"), value)
	case structKind:
		nf := t.Get("fields").Length()
		for i := 0; i < nf; i++ {
			field := t.Get("fields").Index(i)
			tag, tagOptions := parseTag(getJSONTag(field.Get("tag")))
			if tag == "-" {
				value.Delete(field.Get("name").String())
				continue
			}
			name := field.Get("name").String()
			if tagOptions.Contains("omitempty") {
				if isEmpty(v.Get(name), field.Get("typ")) {
					value.Delete(name)
					continue
				}
			}
			if false && tagOptions.Contains("string") {

			} else {
				filter(v.Get(name), field.Get("typ"), value.Get(name))
			}
			if len(tag) != 0 && tag != name {
				value.Set(tag, value.Get(name))
				value.Delete(name)
			}
		}
	case interfaceKind:
		filter(v.Get("$val"), v.Get("constructor"), value)
	}
}

func isEmpty(v, t *js.Object) bool {
	switch t.Get("kind").Int() {
	case boolKind:
		return !v.Bool()
	case int8Kind, int16Kind, int32Kind, intKind, uint8Kind, uint16Kind, uint32Kind, uintKind:
		return v.Int() == 0
	case int64Kind, uint64Kind:
		return v.Int64() == 0
	case float32Kind, float64Kind:
		return v.Float() == 0
	case arrayKind, sliceKind, stringKind:
		return v.Length() == 0
	case ptrKind:
		if v == t.Get("nil") {
			return true
		}
		return isEmpty(v.Call("$get"), t.Get("elem"))
	case interfaceKind:
		if v.Get("$val") == js.Undefined {
			return true
		}
		return isEmpty(v.Get("$val"), v.Get("constructor"))
	case mapKind:
		return js.Global.Get("Object").Call("keys", v).Length() == 0
	}
	return false
}
<|endoftext|>


<commit_before>/*
Copyright 2019 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package model

import (
	"fmt"

	"k8s.io/kops/nodeup/pkg/distros"
	"k8s.io/kops/upup/pkg/fi"
	"k8s.io/kops/upup/pkg/fi/nodeup/nodetasks"

	"k8s.io/klog"
)

// KubectlBuilder install kubectl
type KubectlBuilder struct {
	*NodeupModelContext
}

var _ fi.ModelBuilder = &KubectlBuilder{}

// Build is responsible for managing the kubectl on the nodes
func (b *KubectlBuilder) Build(c *fi.ModelBuilderContext) error {
	if !b.IsMaster {
		return nil
	}

	{
		// TODO: Extract to common function?
		assetName := "kubectl"
		assetPath := ""
		asset, err := b.Assets.Find(assetName, assetPath)
		if err != nil {
			return fmt.Errorf("error trying to locate asset %q: %v", assetName, err)
		}
		if asset == nil {
			return fmt.Errorf("unable to locate asset %q", assetName)
		}

		c.AddTask(&nodetasks.File{
			Path:     b.KubectlPath() + "/" + assetName,
			Contents: asset,
			Type:     nodetasks.FileType_File,
			Mode:     s("0755"),
		})
	}

	{
		kubeconfig, err := b.BuildPKIKubeconfig("kubecfg")
		if err != nil {
			return err
		}

		c.AddTask(&nodetasks.File{
			Path:     "/var/lib/kubectl/kubeconfig",
			Contents: fi.NewStringResource(kubeconfig),
			Type:     nodetasks.FileType_File,
			Mode:     s("0400"),
		})

		adminUser, adminGroup, err := b.findKubeconfigUser()
		if err != nil {
			return err
		}

		if adminUser != nil && adminUser.Home != "" {
			c.AddTask(&nodetasks.File{
				Path:  adminUser.Home + "/.kube/",
				Type:  nodetasks.FileType_Directory,
				Mode:  s("0700"),
				Owner: s(adminUser.Name),
				Group: s(adminGroup.Name),
			})

			c.AddTask(&nodetasks.File{
				Path:     adminUser.Home + "/.kube/config",
				Contents: fi.NewStringResource(kubeconfig),
				Type:     nodetasks.FileType_File,
				Mode:     s("0400"),
				Owner:    s(adminUser.Name),
				Group:    s(adminGroup.Name),
			})
		}
	}

	return nil
}

// findKubeconfigUser finds the default user for whom we should create a kubeconfig
func (b *KubectlBuilder) findKubeconfigUser() (*fi.User, *fi.Group, error) {
	var users []string
	switch b.Distribution {
	case distros.DistributionJessie, distros.DistributionDebian9, distros.DistributionDebian10:
		users = []string{"admin", "root"}
	case distros.DistributionXenial, distros.DistributionBionic, distros.DistributionFocal:
		users = []string{"ubuntu"}
	case distros.DistributionCentos7, distros.DistributionCentos8:
		users = []string{"centos"}
	case distros.DistributionAmazonLinux2, distros.DistributionRhel7, distros.DistributionRhel8:
		users = []string{"ec2-user"}
	default:
		klog.Warningf("Unknown distro; won't write kubeconfig to homedir %s", b.Distribution)
		return nil, nil, nil
	}

	for _, s := range users {
		user, err := fi.LookupUser(s)
		if err != nil {
			klog.Warningf("error looking up user %q: %v", s, err)
			continue
		}
		if user == nil {
			continue
		}
		group, err := fi.LookupGroupById(user.Gid)
		if err != nil {
			klog.Warningf("unable to find group %d for user %q", user.Gid, s)
			continue
		}
		if group == nil {
			continue
		}
		return user, group, nil
	}

	return nil, nil, nil
}
<commit_msg>Update default user for CoreOS, Flatcar for kubecfg<commit_after>/*
Copyright 2019 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package model

import (
	"fmt"

	"k8s.io/kops/nodeup/pkg/distros"
	"k8s.io/kops/upup/pkg/fi"
	"k8s.io/kops/upup/pkg/fi/nodeup/nodetasks"

	"k8s.io/klog"
)

// KubectlBuilder install kubectl
type KubectlBuilder struct {
	*NodeupModelContext
}

var _ fi.ModelBuilder = &KubectlBuilder{}

// Build is responsible for managing the kubectl on the nodes
func (b *KubectlBuilder) Build(c *fi.ModelBuilderContext) error {
	if !b.IsMaster {
		return nil
	}

	{
		// TODO: Extract to common function?
		assetName := "kubectl"
		assetPath := ""
		asset, err := b.Assets.Find(assetName, assetPath)
		if err != nil {
			return fmt.Errorf("error trying to locate asset %q: %v", assetName, err)
		}
		if asset == nil {
			return fmt.Errorf("unable to locate asset %q", assetName)
		}

		c.AddTask(&nodetasks.File{
			Path:     b.KubectlPath() + "/" + assetName,
			Contents: asset,
			Type:     nodetasks.FileType_File,
			Mode:     s("0755"),
		})
	}

	{
		kubeconfig, err := b.BuildPKIKubeconfig("kubecfg")
		if err != nil {
			return err
		}

		c.AddTask(&nodetasks.File{
			Path:     "/var/lib/kubectl/kubeconfig",
			Contents: fi.NewStringResource(kubeconfig),
			Type:     nodetasks.FileType_File,
			Mode:     s("0400"),
		})

		adminUser, adminGroup, err := b.findKubeconfigUser()
		if err != nil {
			return err
		}

		if adminUser != nil && adminUser.Home != "" {
			c.AddTask(&nodetasks.File{
				Path:  adminUser.Home + "/.kube/",
				Type:  nodetasks.FileType_Directory,
				Mode:  s("0700"),
				Owner: s(adminUser.Name),
				Group: s(adminGroup.Name),
			})

			c.AddTask(&nodetasks.File{
				Path:     adminUser.Home + "/.kube/config",
				Contents: fi.NewStringResource(kubeconfig),
				Type:     nodetasks.FileType_File,
				Mode:     s("0400"),
				Owner:    s(adminUser.Name),
				Group:    s(adminGroup.Name),
			})
		}
	}

	return nil
}

// findKubeconfigUser finds the default user for whom we should create a kubeconfig
func (b *KubectlBuilder) findKubeconfigUser() (*fi.User, *fi.Group, error) {
	var users []string
	switch b.Distribution {
	case distros.DistributionJessie, distros.DistributionDebian9, distros.DistributionDebian10:
		users = []string{"admin", "root"}
	case distros.DistributionXenial, distros.DistributionBionic, distros.DistributionFocal:
		users = []string{"ubuntu"}
	case distros.DistributionCentos7, distros.DistributionCentos8:
		users = []string{"centos"}
	case distros.DistributionAmazonLinux2, distros.DistributionRhel7, distros.DistributionRhel8:
		users = []string{"ec2-user"}
	case distros.DistributionCoreOS, distros.DistributionFlatcar:
		users = []string{"core"}
	default:
		klog.Warningf("Unknown distro; won't write kubeconfig to homedir %s", b.Distribution)
		return nil, nil, nil
	}

	for _, s := range users {
		user, err := fi.LookupUser(s)
		if err != nil {
			klog.Warningf("error looking up user %q: %v", s, err)
			continue
		}
		if user == nil {
			continue
		}
		group, err := fi.LookupGroupById(user.Gid)
		if err != nil {
			klog.Warningf("unable to find group %d for user %q", user.Gid, s)
			continue
		}
		if group == nil {
			continue
		}
		return user, group, nil
	}

	return nil, nil, nil
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"github.com/codegangsta/cli"
	"github.com/coreos/go-etcd/etcd"
	"os"
	"testing"

	. "github.com/smartystreets/goconvey/convey"
)

// TODO(jrubin) test recursive keys
func TestEtcd(t *testing.T) {
	Convey("When getting keys from etcd", t, func() {

		os.Setenv("ENVETCD_NO_SANITIZE", "true")
		os.Setenv("ENVETCD_NO_UPCASE", "true")

		appTest.Name = "testApp"
		appTest.Author = "Karl Dominguez"
		appTest.Email = "kdominguez@zvelo.com"
		appTest.Version = "0.0.4"
		appTest.Usage = "get environment variables from etcd"
		appTest.Flags = []cli.Flag{
			cli.StringSliceFlag{
				Name:   "peers, C",
				EnvVar: "ENVETCD_PEERS",
				Value:  &cli.StringSlice{werckerAdd()},
				Usage:  "a comma-delimited list of machine addresses in the cluster (default: \"127.0.0.1:4001\")",
			},
			cli.StringFlag{
				Name:   "ca-file",
				EnvVar: "ENVETCD_CA_FILE",
				Usage:  "certificate authority file",
			},
			cli.StringFlag{
				Name:   "cert-file",
				EnvVar: "ENVETCD_CERT_FILE",
				Usage:  "tls client certificate file",
			},
			cli.StringFlag{
				Name:   "key-file",
				EnvVar: "ENVETCD_KEY_FILE",
				Usage:  "tls client key file",
			},
			cli.StringFlag{
				Name:   "hostname",
				EnvVar: "HOSTNAME",
				Value:  "env",
				Usage:  "computer hostname for host specific configuration",
			},
			cli.StringFlag{
				Name:   "system",
				EnvVar: "ENVETCD_SYSTEM",
				Value:  "systemtest",
				Usage:  "system name for system specific configuration",
			},
			cli.StringFlag{
				Name:   "service",
				EnvVar: "ENVETCD_SERVICE",
				Value:  "servicetest",
				Usage:  "service name for service specific configuration",
			},
			cli.StringFlag{
				Name:   "prefix",
				EnvVar: "ENVETCD_PREFIX",
				Value:  "/config",
				Usage:  "etcd prefix for all keys",
			},
			cli.StringFlag{
				Name:   "log-level, l",
				EnvVar: "ENVETCD_LOG_LEVEL",
				Value:  "DEBUG",
				Usage:  "set log level (DEBUG, INFO, WARN, ERR)",
			},
			cli.StringFlag{
				Name:   "output, o",
				Value:  "testOut.txt",
				EnvVar: "ENVETCD_OUTPUT",
				Usage:  "write stdout from the command to this file",
			},
			cli.BoolFlag{
				Name:   "no-sync",
				EnvVar: "ENVETCD_NO_SYNC",
				Usage:  "don't synchronize cluster information before sending request",
			},
			cli.BoolFlag{
				Name:   "clean-env, c",
				EnvVar: "ENVETCD_CLEAN_ENV",
				Usage:  "don't inherit any environment variables other than those pulled from etcd",
			},
			cli.BoolFlag{
				Name:   "no-sanitize",
				EnvVar: "ENVETCD_NO_SANITIZE",
				Usage:  "don't remove bad characters from environment keys",
			},
			cli.BoolFlag{
				Name:   "no-upcase",
				EnvVar: "ENVETCD_NO_UPCASE",
				Usage:  "don't convert all environment keys to uppercase",
			},
		}
		appTest.Action = run

		set := flagSet(appTest.Name, appTest.Flags)
		ctx := cli.NewContext(appTest, set, set)

		etcdConf := newEtcdConfig(ctx)

		etcdAddress := fmt.Sprintf("%s%s", "http://", werckerPeer)
		etcdPeer := etcd.NewClient([]string{etcdAddress})
		etcdPeer.SetDir("/config/system/systemtest", 0)
		etcdPeer.SetDir("/config/service/servicetest", 0)
		etcdPeer.Set("/config/global/systemtest/testKey", "globaltestVal", 0)
		etcdPeer.Set("/config/host/env", "", 0)

		Convey("newEtcdConfig should return an etcd config", func() {
			So(etcdConf.Key.Prefix, ShouldEqual, "/config")
			So(etcdConf.Key.Hostname, ShouldEqual, "env")
			So(etcdConf.Sync, ShouldBeTrue)
			So(etcdConf.Key.System, ShouldEqual, "systemtest")
			So(etcdConf.Key.Service, ShouldEqual, "servicetest")

			Convey("getEndpoints should return an array of end points", func() {
				endPoints, err := getEndpoints(etcdConf)
				So(err, ShouldBeNil)
				So(endPoints, ShouldNotBeEmpty)
			})

			Convey("getTransport should return an http transport object", func() {
				transObj, err := getTransport(etcdConf)
				So(err, ShouldBeNil)
				So(transObj.Proxy, ShouldBeNil)
				So(transObj.DisableKeepAlives, ShouldBeFalse)
				So(transObj.TLSHandshakeTimeout, ShouldEqual, 10000000000)
				So(transObj.MaxIdleConnsPerHost, ShouldBeZeroValue)
			})

			Convey("getClient should return an etcd client based on a given config", func() {
				etcdClient, err := getClient(etcdConf)
				So(err, ShouldBeNil)
				So(etcdClient, ShouldNotBeEmpty)
				So(etcdClient.CheckRetry, ShouldBeNil)

				Convey("getKeyPairs returns keypairs", func() {
					keyPairs := getKeyPairs(etcdConf, etcdPeer)
					So(keyPairs, ShouldNotBeEmpty)

				})
				Convey("Testing override keys", func() {

					Convey("Setting /config/global testkey only", func() {
						etcdPeer.Set("/config/global/systemtest/testKey", "globaltestVal", 0)
						keyPairs := getKeyPairs(etcdConf, etcdPeer)

						_, isExisting := keyPairs["systemtest_testKey"]
						So(isExisting, ShouldBeTrue)
						So(keyPairs["systemtest_testKey"], ShouldEqual, "globaltestVal")

					})

					Convey("Setting /config/global/testKey", func() {
						etcdPeer.Set("/config/global/systemtest/testKey", "globaltestVal", 0)
						etcdPeer.Set("/config/global/testKey", "testGlobalVal2", 0)
						keyPairs := getKeyPairs(etcdConf, etcdPeer)

						_, isExisting := keyPairs["testKey"]
						So(isExisting, ShouldBeTrue)
						So(keyPairs["systemtest_testKey"], ShouldEqual, "globaltestVal")
						So(keyPairs["testKey"], ShouldEqual, "testGlobalVal2")
					})
					Convey("Setting /config/system/systemtest/testKey should override the global testKey", func() {
						etcdPeer.Set("/config/global/systemtest/testKey", "globaltestVal", 0)
						etcdPeer.Set("/config/global/testKey", "testGlobalVal2", 0)
						etcdPeer.Set("/config/system/systemtest/testKey", "testsystemVal", 0)
						keyPairs := getKeyPairs(etcdConf, etcdPeer)
						_, isExisting := keyPairs["testKey"]

						So(isExisting, ShouldBeTrue)
						So(keyPairs["systemtest_testKey"], ShouldEqual, "globaltestVal")
						So(keyPairs["testKey"], ShouldEqual, "testsystemVal")
					})
				})

				Convey("Setting /config/service/systemtest/testserviceKey should not be in the keypair", func() {
					etcdPeer.Set("/config/service/systemtest/testserviceKey", "testserviceVal", 0)
					keyPairs := getKeyPairs(etcdConf, etcdPeer)

					_, isExisting := keyPairs["testserviceKey"]
					So(isExisting, ShouldBeFalse)
					So(keyPairs["systemtest_testKey"], ShouldEqual, "globaltestVal")
					So(keyPairs["testserviceKey"], ShouldBeEmpty)

					fmt.Println("\nCurrent Directories\n")
					resp2, err := etcdPeer.Get("/config/", false, true)
					if err != nil {
						fmt.Println(err)
					}
					printLs(resp2)
					etcdPeer.Delete("/config", true)
				})

				Convey("Testing nested keys", func() {
					Convey("Adding key-value pairs in systemtest root /config/system/systemtest/", func() {
						etcdPeer.Set("/config/system/systemtest/nestkey1", "nestval1", 0)
						etcdPeer.Set("/config/system/systemtest/nestkey2", "nestval2", 0)
						keyPairs := getKeyPairs(etcdConf, etcdPeer)

						_, isExisting := keyPairs["nestkey1"]
						So(isExisting, ShouldBeTrue)
						So(keyPairs["nestkey1"], ShouldEqual, "nestval1")

						_, isExisting = keyPairs["nestkey2"]
						So(isExisting, ShouldBeTrue)
						So(keyPairs["nestkey2"], ShouldEqual, "nestval2")

						Convey("Adding key-value pairs in systemtest first nest directory /config/system/systemtest/nest1/", func() {
							etcdPeer.Set("/config/system/systemtest/nest1/nest1key1", "nest1val1", 0)
							etcdPeer.Set("/config/system/systemtest/nest1/nest1key2", "nest1val2", 0)
							keyPairs = getKeyPairs(etcdConf, etcdPeer)

							_, isExisting = keyPairs["nest1_nest1key1"]
							So(isExisting, ShouldBeTrue)
							So(keyPairs["nest1_nest1key1"], ShouldEqual, "nest1val1")

							_, isExisting = keyPairs["nest1_nest1key2"]
							So(isExisting, ShouldBeTrue)
							So(keyPairs["nest1_nest1key2"], ShouldEqual, "nest1val2")

							Convey("Adding key-value pairs in systemtest second nest directory /config/system/systemtest/nest1/nest2", func() {
								etcdPeer.Set("/config/system/systemtest/nest1/nest2/nest2key1", "nest2val1", 0)
								etcdPeer.Set("/config/system/systemtest/nest1/nest2/nest2key2", "nest2val2", 0)
								keyPairs = getKeyPairs(etcdConf, etcdPeer)

								_, isExisting = keyPairs["nest1_nest2_nest2key1"]
								So(isExisting, ShouldBeTrue)
								So(keyPairs["nest1_nest2_nest2key1"], ShouldEqual, "nest2val1")

								_, isExisting = keyPairs["nest1_nest2_nest2key1"]
								So(isExisting, ShouldBeTrue)
								So(keyPairs["nest1_nest2_nest2key2"], ShouldEqual, "nest2val2")

								etcdPeer.Delete("/config", true)
							})
						})
					})
				})
			})
		})
	})
}

func printLs(resp *etcd.Response) {
	if !resp.Node.Dir {
		fmt.Println(resp.Node.Key)
	}
	for _, node := range resp.Node.Nodes {
		rPrint(node)
	}
}

// rPrint recursively prints out the nodes in the node structure.
func rPrint(n *etcd.Node) {

	if n.Dir {
		fmt.Println(n.Key)
	}

	for _, node := range n.Nodes {
		rPrint(node)
	}
}
<commit_msg>Addition directory nest<commit_after>package main

import (
	"fmt"
	"github.com/codegangsta/cli"
	"github.com/coreos/go-etcd/etcd"
	"os"
	"testing"

	. "github.com/smartystreets/goconvey/convey"
)

// TODO(jrubin) test recursive keys
func TestEtcd(t *testing.T) {
	Convey("When getting keys from etcd", t, func() {

		os.Setenv("ENVETCD_NO_SANITIZE", "true")
		os.Setenv("ENVETCD_NO_UPCASE", "true")

		appTest.Name = "testApp"
		appTest.Author = "Karl Dominguez"
		appTest.Email = "kdominguez@zvelo.com"
		appTest.Version = "0.0.4"
		appTest.Usage = "get environment variables from etcd"
		appTest.Flags = []cli.Flag{
			cli.StringSliceFlag{
				Name:   "peers, C",
				EnvVar: "ENVETCD_PEERS",
				Value:  &cli.StringSlice{werckerAdd()},
				Usage:  "a comma-delimited list of machine addresses in the cluster (default: \"127.0.0.1:4001\")",
			},
			cli.StringFlag{
				Name:   "ca-file",
				EnvVar: "ENVETCD_CA_FILE",
				Usage:  "certificate authority file",
			},
			cli.StringFlag{
				Name:   "cert-file",
				EnvVar: "ENVETCD_CERT_FILE",
				Usage:  "tls client certificate file",
			},
			cli.StringFlag{
				Name:   "key-file",
				EnvVar: "ENVETCD_KEY_FILE",
				Usage:  "tls client key file",
			},
			cli.StringFlag{
				Name:   "hostname",
				EnvVar: "HOSTNAME",
				Value:  "env",
				Usage:  "computer hostname for host specific configuration",
			},
			cli.StringFlag{
				Name:   "system",
				EnvVar: "ENVETCD_SYSTEM",
				Value:  "systemtest",
				Usage:  "system name for system specific configuration",
			},
			cli.StringFlag{
				Name:   "service",
				EnvVar: "ENVETCD_SERVICE",
				Value:  "servicetest",
				Usage:  "service name for service specific configuration",
			},
			cli.StringFlag{
				Name:   "prefix",
				EnvVar: "ENVETCD_PREFIX",
				Value:  "/config",
				Usage:  "etcd prefix for all keys",
			},
			cli.StringFlag{
				Name:   "log-level, l",
				EnvVar: "ENVETCD_LOG_LEVEL",
				Value:  "DEBUG",
				Usage:  "set log level (DEBUG, INFO, WARN, ERR)",
			},
			cli.StringFlag{
				Name:   "output, o",
				Value:  "testOut.txt",
				EnvVar: "ENVETCD_OUTPUT",
				Usage:  "write stdout from the command to this file",
			},
			cli.BoolFlag{
				Name:   "no-sync",
				EnvVar: "ENVETCD_NO_SYNC",
				Usage:  "don't synchronize cluster information before sending request",
			},
			cli.BoolFlag{
				Name:   "clean-env, c",
				EnvVar: "ENVETCD_CLEAN_ENV",
				Usage:  "don't inherit any environment variables other than those pulled from etcd",
			},
			cli.BoolFlag{
				Name:   "no-sanitize",
				EnvVar: "ENVETCD_NO_SANITIZE",
				Usage:  "don't remove bad characters from environment keys",
			},
			cli.BoolFlag{
				Name:   "no-upcase",
				EnvVar: "ENVETCD_NO_UPCASE",
				Usage:  "don't convert all environment keys to uppercase",
			},
		}
		appTest.Action = run

		set := flagSet(appTest.Name, appTest.Flags)
		ctx := cli.NewContext(appTest, set, set)

		etcdConf := newEtcdConfig(ctx)

		etcdAddress := fmt.Sprintf("%s%s", "http://", werckerPeer)
		etcdPeer := etcd.NewClient([]string{etcdAddress})
		etcdPeer.SetDir("/config/system/systemtest", 0)
		etcdPeer.SetDir("/config/service/servicetest", 0)
		etcdPeer.Set("/config/global/systemtest/testKey", "globaltestVal", 0)
		etcdPeer.Set("/config/host/env", "", 0)

		Convey("newEtcdConfig should return an etcd config", func() {
			So(etcdConf.Key.Prefix, ShouldEqual, "/config")
			So(etcdConf.Key.Hostname, ShouldEqual, "env")
			So(etcdConf.Sync, ShouldBeTrue)
			So(etcdConf.Key.System, ShouldEqual, "systemtest")
			So(etcdConf.Key.Service, ShouldEqual, "servicetest")

			Convey("getEndpoints should return an array of end points", func() {
				endPoints, err := getEndpoints(etcdConf)
				So(err, ShouldBeNil)
				So(endPoints, ShouldNotBeEmpty)
			})

			Convey("getTransport should return an http transport object", func() {
				transObj, err := getTransport(etcdConf)
				So(err, ShouldBeNil)
				So(transObj.Proxy, ShouldBeNil)
				So(transObj.DisableKeepAlives, ShouldBeFalse)
				So(transObj.TLSHandshakeTimeout, ShouldEqual, 10000000000)
				So(transObj.MaxIdleConnsPerHost, ShouldBeZeroValue)
			})

			Convey("getClient should return an etcd client based on a given config", func() {
				etcdClient, err := getClient(etcdConf)
				So(err, ShouldBeNil)
				So(etcdClient, ShouldNotBeEmpty)
				So(etcdClient.CheckRetry, ShouldBeNil)

				Convey("getKeyPairs returns keypairs", func() {
					keyPairs := getKeyPairs(etcdConf, etcdPeer)
					So(keyPairs, ShouldNotBeEmpty)

				})
				Convey("Testing override keys", func() {

					Convey("Setting /config/global testkey only", func() {
						etcdPeer.Set("/config/global/systemtest/testKey", "globaltestVal", 0)
						keyPairs := getKeyPairs(etcdConf, etcdPeer)

						_, isExisting := keyPairs["systemtest_testKey"]
						So(isExisting, ShouldBeTrue)
						So(keyPairs["systemtest_testKey"], ShouldEqual, "globaltestVal")

					})

					Convey("Setting /config/global/testKey", func() {
						etcdPeer.Set("/config/global/systemtest/testKey", "globaltestVal", 0)
						etcdPeer.Set("/config/global/testKey", "testGlobalVal2", 0)
						keyPairs := getKeyPairs(etcdConf, etcdPeer)

						_, isExisting := keyPairs["testKey"]
						So(isExisting, ShouldBeTrue)
						So(keyPairs["systemtest_testKey"], ShouldEqual, "globaltestVal")
						So(keyPairs["testKey"], ShouldEqual, "testGlobalVal2")
					})
					Convey("Setting /config/system/systemtest/testKey should override the global testKey", func() {
						etcdPeer.Set("/config/global/systemtest/testKey", "globaltestVal", 0)
						etcdPeer.Set("/config/global/testKey", "testGlobalVal2", 0)
						etcdPeer.Set("/config/system/systemtest/testKey", "testsystemVal", 0)
						keyPairs := getKeyPairs(etcdConf, etcdPeer)
						_, isExisting := keyPairs["testKey"]

						So(isExisting, ShouldBeTrue)
						So(keyPairs["systemtest_testKey"], ShouldEqual, "globaltestVal")
						So(keyPairs["testKey"], ShouldEqual, "testsystemVal")
					})
				})

				Convey("Setting /config/service/systemtest/testserviceKey should not be in the keypair", func() {
					etcdPeer.Set("/config/service/systemtest/testserviceKey", "testserviceVal", 0)
					keyPairs := getKeyPairs(etcdConf, etcdPeer)

					_, isExisting := keyPairs["testserviceKey"]
					So(isExisting, ShouldBeFalse)
					So(keyPairs["systemtest_testKey"], ShouldEqual, "globaltestVal")
					So(keyPairs["testserviceKey"], ShouldBeEmpty)

					fmt.Println("\nCurrent Directories\n")
					resp2, err := etcdPeer.Get("/config/", false, true)
					if err != nil {
						fmt.Println(err)
					}
					printLs(resp2)
					etcdPeer.Delete("/config", true)
				})

				Convey("Testing nested keys", func() {
					Convey("Adding key-value pairs in systemtest root /config/system/systemtest/", func() {
						etcdPeer.Set("/config/system/systemtest/nestkey1", "nestval1", 0)
						etcdPeer.Set("/config/system/systemtest/nestkey2", "nestval2", 0)
						keyPairs := getKeyPairs(etcdConf, etcdPeer)

						_, isExisting := keyPairs["nestkey1"]
						So(isExisting, ShouldBeTrue)
						So(keyPairs["nestkey1"], ShouldEqual, "nestval1")

						_, isExisting = keyPairs["nestkey2"]
						So(isExisting, ShouldBeTrue)
						So(keyPairs["nestkey2"], ShouldEqual, "nestval2")

						Convey("Adding key-value pairs in systemtest first nest directory /config/system/systemtest/nest1/", func() {
							etcdPeer.Set("/config/system/systemtest/nest1/nest1key1", "nest1val1", 0)
							etcdPeer.Set("/config/system/systemtest/nest1/nest1key2", "nest1val2", 0)
							etcdPeer.Set("/config/system/systemtest/nest2/nest2key1", "nest2val1", 0)
							etcdPeer.Set("/config/system/systemtest/nest2/nest2key2", "nest2val2", 0)
							keyPairs = getKeyPairs(etcdConf, etcdPeer)

							_, isExisting = keyPairs["nest1_nest1key1"]
							So(isExisting, ShouldBeTrue)
							So(keyPairs["nest1_nest1key1"], ShouldEqual, "nest1val1")

							_, isExisting = keyPairs["nest1_nest1key2"]
							So(isExisting, ShouldBeTrue)
							So(keyPairs["nest1_nest1key2"], ShouldEqual, "nest1val2")

							_, isExisting = keyPairs["nest2_nest2key1"]
							So(isExisting, ShouldBeTrue)
							So(keyPairs["nest2_nest2key1"], ShouldEqual, "nest2val1")

							_, isExisting = keyPairs["nest1_nest1key2"]
							So(isExisting, ShouldBeTrue)
							So(keyPairs["nest2_nest2key2"], ShouldEqual, "nest2val2")

							Convey("Adding key-value pairs in systemtest second nest directory /config/system/systemtest/nest1/nest2", func() {
								etcdPeer.Set("/config/system/systemtest/nest1/nest2/nest2key1", "nest2val1", 0)
								etcdPeer.Set("/config/system/systemtest/nest1/nest2/nest2key2", "nest2val2", 0)
								keyPairs = getKeyPairs(etcdConf, etcdPeer)

								_, isExisting = keyPairs["nest1_nest2_nest2key1"]
								So(isExisting, ShouldBeTrue)
								So(keyPairs["nest1_nest2_nest2key1"], ShouldEqual, "nest2val1")

								_, isExisting = keyPairs["nest1_nest2_nest2key1"]
								So(isExisting, ShouldBeTrue)
								So(keyPairs["nest1_nest2_nest2key2"], ShouldEqual, "nest2val2")

								fmt.Println("\nCurrent Directories\n")
								resp2, err := etcdPeer.Get("/config/", false, true)
								if err != nil {
									fmt.Println(err)
								}
								printLs(resp2)
								etcdPeer.Delete("/config", true)
							})
						})
					})
				})
			})
		})
	})
}

func printLs(resp *etcd.Response) {
	if !resp.Node.Dir {
		fmt.Println(resp.Node.Key)
	}
	for _, node := range resp.Node.Nodes {
		rPrint(node)
	}
}

// rPrint recursively prints out the nodes in the node structure.
func rPrint(n *etcd.Node) {

	if n.Dir {
		fmt.Println(n.Key)
	}

	for _, node := range n.Nodes {
		rPrint(node)
	}
}
<|endoftext|>


<commit_before>package learn

import (
	cic "github.com/tsinghua-io/api-server/adapter/tsinghua.edu.cn/cic/learn"
	old "github.com/tsinghua-io/api-server/adapter/tsinghua.edu.cn/learn"
	"github.com/tsinghua-io/api-server/model"
	"github.com/tsinghua-io/api-server/util"
	"net/http"
	"strings"
)

type Adapter struct {
	*cic.Adapter
	Old *old.Adapter
}

func New(userId, password string) (ada *Adapter, status int, errMsg error) {
	ada = new(Adapter)

	sg := util.NewStatusGroup()
	sg.Go(func(status *int, err *error) {
		ada.Adapter, *status, *err = cic.New(userId, password)
	})
	sg.Go(func(status *int, err *error) {
		ada.Old, *status, *err = old.New(userId, password)
	})
	status, errMsg = sg.Wait()

	return
}

func (ada *Adapter) Announcements(courseId string) (announcements []*model.Announcement, status int, errMsg error) {
	if strings.Contains(courseId, "-") {
		return ada.Adapter.Announcements(courseId)
	} else {
		return ada.Old.Announcements(courseId)
	}
}

func (ada *Adapter) Assignments(courseId string) (assignments []*model.Assignment, status int, errMsg error) {
	if strings.Contains(courseId, "-") {
		return ada.Adapter.Assignments(courseId)
	} else {
		return ada.Old.Assignments(courseId)
	}
}

type fatName struct {
	name string
	seq  string
	sem  string
}

func newFatName(course *model.Course) fatName {
	return fatName{course.Name, course.CourseSequence, course.Semester}
}

func (ada *Adapter) nameIdMap() (m map[fatName]string, status int, errMsg error) {
	var courses []*model.Course
	if courses, status, errMsg = ada.Old.AllAttendedList(); errMsg == nil {
		m := make(map[fatName]string)
		for _, course := range courses {
			if strings.Contains(course.Id, "-") {
				m[newFatName(course)] = course.Id
			}
		}
	}
}

func replaceCourseIds(courses []*model.Course, m map[fatName]string) {
	for _, course := range courses {
		if oldId, ok := m[newFatName(course)]; ok {
			course.Id = oldId
		}
	}
}

func (ada *Adapter) Attended(semesterID string, english bool) (courses []*model.Course, status int, errMsg error) {
	var m map[fatName]string

	sg := util.NewStatusGroup()
	sg.Go(func(status *int, err *error) {
		m, *status, *err = ada.nameIdMap()
	})
	sg.Go(func(status *int, err *error) {
		courses, *status, *err = ada.Adapter.Attended(semesterID, english)
	})
	if status, errMsg = sg.Wait(); errMsg == nil {
		replaceCourseIds(courses, m)
	}
}

func (ada *Adapter) NowAttended(english bool) (thisCourses []*model.Course, nextCourses []*model.Course, status int, errMsg error) {
	var m map[fatName]string

	sg := util.NewStatusGroup()
	sg.Go(func(status *int, err *error) {
		m, *status, *err = ada.nameIdMap()
	})
	sg.Go(func(status *int, err *error) {
		courses, *status, *err = ada.Adapter.NowAttended(english)
	})
	if status, errMsg = sg.Wait(); errMsg == nil {
		replaceCourseIds(courses, m)
	}
}

func (ada *Adapter) PastAttended(english bool) (courses []*model.Course, status int, errMsg error) {
	var m map[fatName]string

	sg := util.NewStatusGroup()
	sg.Go(func(status *int, err *error) {
		m, *status, *err = ada.nameIdMap()
	})
	sg.Go(func(status *int, err *error) {
		courses, *status, *err = ada.Adapter.PastAttended(english)
	})
	if status, errMsg = sg.Wait(); errMsg == nil {
		replaceCourseIds(courses, m)
	}
}

func (ada *Adapter) AllAttended(english bool) (courses []*model.Course, status int, errMsg error) {
	var m map[fatName]string

	sg := util.NewStatusGroup()
	sg.Go(func(status *int, err *error) {
		m, *status, *err = ada.nameIdMap()
	})
	sg.Go(func(status *int, err *error) {
		courses, *status, *err = ada.Adapter.AllAttended(english)
	})
	if status, errMsg = sg.Wait(); errMsg == nil {
		replaceCourseIds(courses, m)
	}
}

func (ada *Adapter) Files(courseId string) (files []*model.File, status int, errMsg error) {
	if strings.Contains(courseId, "-") {
		return ada.Adapter.Files(courseId)
	} else {
		return ada.Old.Files(courseId)
	}
}

func (ada *Adapter) Profile() (profile *model.User, status int, errMsg error) {
	return ada.Adapter.Profile()
}

func HandlerFunc(f func(http.ResponseWriter, *http.Request, *Adapter)) http.Handler {
	return http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) {
		userId, password, _ := req.BasicAuth()
		if ada, status, err := New(userId, password); err != nil {
			util.Error(rw, err.Error(), status)
		} else {
			f(rw, req, ada)
		}
	})
}
<commit_msg>Small bug fixes in mixed.<commit_after>package learn

import (
	cic "github.com/tsinghua-io/api-server/adapter/tsinghua.edu.cn/cic/learn"
	old "github.com/tsinghua-io/api-server/adapter/tsinghua.edu.cn/learn"
	"github.com/tsinghua-io/api-server/model"
	"github.com/tsinghua-io/api-server/util"
	"net/http"
	"strings"
)

type Adapter struct {
	*cic.Adapter
	Old *old.Adapter
}

func New(userId, password string) (ada *Adapter, status int, errMsg error) {
	ada = new(Adapter)

	sg := util.NewStatusGroup()
	sg.Go(func(status *int, err *error) {
		ada.Adapter, *status, *err = cic.New(userId, password)
	})
	sg.Go(func(status *int, err *error) {
		ada.Old, *status, *err = old.New(userId, password)
	})
	status, errMsg = sg.Wait()

	return
}

func (ada *Adapter) Announcements(courseId string) (announcements []*model.Announcement, status int, errMsg error) {
	if strings.Contains(courseId, "-") {
		return ada.Adapter.Announcements(courseId)
	} else {
		return ada.Old.Announcements(courseId)
	}
}

func (ada *Adapter) Assignments(courseId string) (assignments []*model.Assignment, status int, errMsg error) {
	if strings.Contains(courseId, "-") {
		return ada.Adapter.Assignments(courseId)
	} else {
		return ada.Old.Assignments(courseId)
	}
}

type fatName struct {
	name string
	seq  string
	sem  string
}

func newFatName(course *model.Course) fatName {
	return fatName{course.Name, course.CourseSequence, course.Semester}
}

func (ada *Adapter) nameIdMap() (m map[fatName]string, status int, errMsg error) {
	var courses []*model.Course
	if courses, status, errMsg = ada.Old.AllAttendedList(); errMsg == nil {
		m := make(map[fatName]string)
		for _, course := range courses {
			if strings.Contains(course.Id, "-") {
				m[newFatName(course)] = course.Id
			}
		}
	}
	return
}

func replaceCourseIds(courses []*model.Course, m map[fatName]string) {
	for _, course := range courses {
		if oldId, ok := m[newFatName(course)]; ok {
			course.Id = oldId
		}
	}
}

func (ada *Adapter) Attended(semesterID string, english bool) (courses []*model.Course, status int, errMsg error) {
	var m map[fatName]string

	sg := util.NewStatusGroup()
	sg.Go(func(status *int, err *error) {
		m, *status, *err = ada.nameIdMap()
	})
	sg.Go(func(status *int, err *error) {
		courses, *status, *err = ada.Adapter.Attended(semesterID, english)
	})
	if status, errMsg = sg.Wait(); errMsg == nil {
		replaceCourseIds(courses, m)
	}

	return
}

func (ada *Adapter) NowAttended(english bool) (thisCourses []*model.Course, nextCourses []*model.Course, status int, errMsg error) {
	var m map[fatName]string

	sg := util.NewStatusGroup()
	sg.Go(func(status *int, err *error) {
		m, *status, *err = ada.nameIdMap()
	})
	sg.Go(func(status *int, err *error) {
		thisCourses, nextCourses, *status, *err = ada.Adapter.NowAttended(english)
	})
	if status, errMsg = sg.Wait(); errMsg == nil {
		replaceCourseIds(thisCourses, m)
		replaceCourseIds(nextCourses, m)
	}

	return
}

func (ada *Adapter) PastAttended(english bool) (courses []*model.Course, status int, errMsg error) {
	var m map[fatName]string

	sg := util.NewStatusGroup()
	sg.Go(func(status *int, err *error) {
		m, *status, *err = ada.nameIdMap()
	})
	sg.Go(func(status *int, err *error) {
		courses, *status, *err = ada.Adapter.PastAttended(english)
	})
	if status, errMsg = sg.Wait(); errMsg == nil {
		replaceCourseIds(courses, m)
	}

	return
}

func (ada *Adapter) AllAttended(english bool) (courses []*model.Course, status int, errMsg error) {
	var m map[fatName]string

	sg := util.NewStatusGroup()
	sg.Go(func(status *int, err *error) {
		m, *status, *err = ada.nameIdMap()
	})
	sg.Go(func(status *int, err *error) {
		courses, *status, *err = ada.Adapter.AllAttended(english)
	})
	if status, errMsg = sg.Wait(); errMsg == nil {
		replaceCourseIds(courses, m)
	}

	return
}

func (ada *Adapter) Files(courseId string) (files []*model.File, status int, errMsg error) {
	if strings.Contains(courseId, "-") {
		return ada.Adapter.Files(courseId)
	} else {
		return ada.Old.Files(courseId)
	}
}

func (ada *Adapter) Profile() (profile *model.User, status int, errMsg error) {
	return ada.Adapter.Profile()
}

func HandlerFunc(f func(http.ResponseWriter, *http.Request, *Adapter)) http.Handler {
	return http.HandlerFunc(func(rw http.ResponseWriter, req *http.Request) {
		userId, password, _ := req.BasicAuth()
		if ada, status, err := New(userId, password); err != nil {
			util.Error(rw, err.Error(), status)
		} else {
			f(rw, req, ada)
		}
	})
}
<|endoftext|>


<commit_before>package hvsc

import (
	"bufio"
	"log"
	"os"
)

const SongInfoFile = "DOCUMENTS/STIL.txt"

func readSTIL() {
	file, err := os.Open(hvscPathTo(SongInfoFile))
	if err != nil {
		log.Fatal(err)
	}
	defer file.Close()

	var tune *SidTune
	var info []string
	var tuneIndex = -1

	scanner := bufio.NewScanner(file)
	for scanner.Scan() {
		line := scanner.Text()
		if len(line) == 0 || line[0] == '#' {
			continue
		}
		if line[0] == '/' && line[len(line)-4:] == ".sid" {
			tuneIndex = tuneIndexByPath(line) //, tuneIndex+1)
			if tuneIndex < 0 {
				log.Fatalf("Unknown path in file %s: %s", SongInfoFile, line)
			}
			if tune != nil {
				tune.Info = info
			}
			tune = &Tunes[tuneIndex]
			info = make([]string, 0)
		} else {
			info = append(info, line)
		}
	}
}

func tuneIndexByPath(path string) int {
	for i, tune := range Tunes {
		if tune.Path == path {
			return i
		}
	}
	return -1
}

func tuneIndexByPathIndex(path string, index int) int {
	for i, tune := range Tunes[index:] {
		if tune.Path == path {
			log.Printf("Found path at index %d from index %d: %s [%s]", index+i, index, path, Tunes[index].Path)
			return index + i
		}
	}
	log.Fatalf("Unknown path from index %d: %s [%s]", index, path, Tunes[index].Path)
	return -1
}
<commit_msg>Fixed STIL parsing to skip folder info.<commit_after>package hvsc

import (
	"bufio"
	"log"
	"os"
)

const SongInfoFile = "DOCUMENTS/STIL.txt"

func readSTIL() {
	file, err := os.Open(hvscPathTo(SongInfoFile))
	if err != nil {
		log.Fatal(err)
	}
	defer file.Close()

	var tune *SidTune
	var info []string
	var tuneIndex = -1

	scanner := bufio.NewScanner(file)
	for scanner.Scan() {
		line := scanner.Text()
		if len(line) == 0 || line[0] == '#' {
			continue
		}
		if line[0] == '/' {
			if tune != nil {
				tune.Info = info
				tune = nil
			}
			if line[len(line)-4:] == ".sid" {
				tuneIndex = tuneIndexByPath(line) //, tuneIndex+1)
				if tuneIndex < 0 {
					log.Fatalf("Unknown path in file %s: %s", SongInfoFile, line)
				}
				tune = &Tunes[tuneIndex]
				info = make([]string, 0)
			}
		} else if tune != nil {
			info = append(info, line)
		}
	}
}

func tuneIndexByPath(path string) int {
	for i, tune := range Tunes {
		if tune.Path == path {
			return i
		}
	}
	return -1
}

func tuneIndexByPathIndex(path string, index int) int {
	for i, tune := range Tunes[index:] {
		if tune.Path == path {
			log.Printf("Found path at index %d from index %d: %s [%s]", index+i, index, path, Tunes[index].Path)
			return index + i
		}
	}
	log.Fatalf("Unknown path from index %d: %s [%s]", index, path, Tunes[index].Path)
	return -1
}
<|endoftext|>


<commit_before>package oracle

import (
	"fmt"
	"os"

	"github.com/juju/errors"
	"github.com/juju/version"

	"github.com/juju/juju/constraints"
	"github.com/juju/juju/environs"
	"github.com/juju/juju/environs/config"
	"github.com/juju/juju/environs/imagemetadata"
	envinstance "github.com/juju/juju/environs/instances"
	"github.com/juju/juju/environs/simplestreams"
	"github.com/juju/juju/instance"
	"github.com/juju/juju/network"
	"github.com/juju/juju/storage"

	oci "github.com/hoenirvili/go-oracle-cloud/api"
)

// oracleEnviron implements the environs.Environ interface
// and has behaviour specific that the interface provides.
type oracleEnviron struct {
	p    *environProvider
	spec environs.CloudSpec
	cfg  *config.Config
}

func newOracleEnviron(p *environProvider, args environs.OpenParams) *oracleEnviron {
	env := &oracleEnviron{
		p:    p,
		spec: args.Cloud,
		cfg:  args.Config,
	}

	return env
}

func (o oracleEnviron) PrepareForBootstrap(ctx environs.BootstrapContext) error {
	return nil
}

func (o oracleEnviron) Validate(cfg, old *config.Config) (valid *config.Config, _ error) {
	return nil, nil
}

func (o oracleEnviron) StorageProviderTypes() ([]storage.ProviderType, error) {
	return nil, nil
}

func (o oracleEnviron) StorageProvider(t storage.ProviderType) (storage.Provider, error) {
	return nil, nil
}

func (o oracleEnviron) StartInstance(args environs.StartInstanceParams) (*environs.StartInstanceResult, error) {
	return nil, nil
}

func (o oracleEnviron) StopInstances(...instance.Id) error {
	return nil
}

func (o oracleEnviron) AllInstances() ([]instance.Instance, error) {
	return nil, nil
}

func (o oracleEnviron) MaintainInstance(args environs.StartInstanceParams) error {
	return nil
}

func (o oracleEnviron) Config() *config.Config {
	return o.cfg
}

// Bootstrap creates a new environment, and an instance inside the
// oracle cloud infrastracture to host the controller for that
// environment. The instnace will have have the series and architecture
// of the Environ's choice, constrained to those of the available tools.
// Bootstrap will return the instance's
// architecture, series, and a function that must be called to finalize
// the bootstrap process by transferring the tools and installing the
// initial Juju controller.
//
// Bootstrap will use just one specific architecture because the oracle
// cloud only supports amd64.
func (o oracleEnviron) Bootstrap(
	ctx environs.BootstrapContext,
	params environs.BootstrapParams,
) (*environs.BootstrapResult, error) {
	// in order too make the entire bootstrap process prossible
	// we must take into accounting some things:
	// validate if we have a shape based on the bootstrap constraints
	// and pick the right one
	// validate if we have in the imagelist a image correspoding to the
	// image tools specified.

	logger.Infof("Loging into the oracle cloud infrastructure")
	if err := o.p.client.Authenticate(); err != nil {
		return nil, errors.Trace(err)
	}

	shapes, err := o.p.client.AllShapeDetails()
	if err != nil {
		return nil, errors.Trace(err)
	}

	shape, err := findShape(shapes.Result, params.BootstrapConstraints)
	if err != nil {
		return nil, errors.Trace(err)
	}

	logger.Infof(
		"Choosing the %s with %d cores and %d MB ram",
		shape.name, shape.cpus, shape.ram,
	)

	imagelist, err := checkImageList(o.p.client, params.ImageMetadata)
	if err != nil {
		return nil, errors.Trace(err)
	}

	os.Exit(1)

	//TODO
	instance, err := launchBootstrapConstroller(o.p.client, []oci.InstanceParams{
		{
			Shape:     shape.name,
			Imagelist: imagelist,
			Label:     "",
			SSHKeys:   nil,
			Name:      "Bootstrap Juju Controller",
		},
	})
	if err != nil {
		return nil, err
	}

	_ = instance
	return nil, nil
}

// here we should check with the client if the we have already a ubuntu/centos/what ever image is specified int he tools metadata
// if there already exists then we should return it
func checkImageList(c *oci.Client, tools []*imagemetadata.ImageMetadata) (string, error) {
	var imageVersion string

	if c == nil {
		return "", errors.NotFoundf("Cannot use nil client")
	}

	if tools == nil {
		return "", errors.NotFoundf("No tools imagemedatada provided")
	}

	for _, val := range tools {
		if len(val.Version) > 0 {
			imageVersion = val.Version
			logger.Infof("Found tools %s and searching tghourgh the oracle imagelist", val)
			break
		}
	}

	if imageVersion == "" {
		return "", errors.NotFoundf("No version found in the tools")
	}

	resp, err := c.AllImageList()
	if err != nil {
		return "", errors.Trace(err)
	}

	for _, val := range resp.Result {
		//TODO
	}
	fmt.Printf("%+v", resp)
	return imageVersion, nil
}

// launchBootstrapController creates a new vm inside
// the oracle infrastracuture and parses the response into a instance.Instance
func launchBootstrapConstroller(c *oci.Client, params []oci.InstanceParams) (instance.Instance, error) {
	if c == nil {
		return nil, errors.NotFoundf("Cannot use nil client")
	}

	resp, err := c.CreateInstance(params)
	if err != nil {
		return nil, errors.Trace(err)
	}

	instance, err := newInstance(resp)
	if err != nil {
		return nil, errors.Trace(err)
	}

	return instance, nil
}

func (o oracleEnviron) BootstrapMessage() string {
	return "SomeBootstrapMessage"
}

func (o oracleEnviron) Create(params environs.CreateParams) error {
	return nil
}

// ConstraintsValidator returns a constraints.Validator instance which
// is used to validate and merge constraints.
//
// Validator defines operations on constraints attributes which are
// used to ensure a constraints value is valid, as well as being able
// to handle overridden attributes.
//
// This will use the default validator implementation from the constraints package.
func (o oracleEnviron) ConstraintsValidator() (constraints.Validator, error) {
	// list of unsupported oracle provider constraints
	unsupportedConstraints := []string{
		constraints.Container,
		constraints.CpuPower,
		constraints.RootDisk,
		constraints.Arch,
		constraints.InstanceType,
		constraints.VirtType,
		constraints.Spaces,
	}

	// we choose to use the default validator implementation
	validator := constraints.NewValidator()

	// we must feed the validator that the oracle cloud
	// provider does not support these constraints
	validator.RegisterUnsupported(unsupportedConstraints)

	return newConstraintsAdaptor(validator), nil
}

func (o oracleEnviron) SetConfig(cfg *config.Config) error {
	return nil
}

func (o oracleEnviron) Instances(ids []instance.Id) ([]instance.Instance, error) {
	return nil, nil
}

func (o oracleEnviron) ControllerInstances(controllerUUID string) ([]instance.Id, error) {
	return nil, nil
}

func (o oracleEnviron) Destroy() error {
	return nil
}

func (o oracleEnviron) DestroyController(controllerUUID string) error {
	return nil
}

func (o oracleEnviron) Provider() environs.EnvironProvider {
	return o.p
}

func (e oracleEnviron) AdoptResources(controllerUUID string, fromVersion version.Number) error {

	return nil
}

func (o oracleEnviron) PrecheckInstance(series string, cons constraints.Value, placement string) error {
	return nil
}

func (o oracleEnviron) OpenPorts(rules []network.IngressRule) error {
	return nil
}

func (o oracleEnviron) ClosePorts(rules []network.IngressRule) error {
	return nil
}

func (o oracleEnviron) IngressRules() ([]network.IngressRule, error) {
	return nil, nil
}

func (o oracleEnviron) InstanceTypes(constraints.Value) (envinstance.InstanceTypesWithCostMetadata, error) {
	var i envinstance.InstanceTypesWithCostMetadata
	return i, nil
}

// Providing this methods oracleEnviron implements also the simplestreams.HasRegion
// interface
//
// Region returns the necessary attributes to uniquely identify this cloud instance.
func (o oracleEnviron) Region() (simplestreams.CloudSpec, error) {
	return simplestreams.CloudSpec{
		Region:   o.spec.Region,
		Endpoint: o.spec.Endpoint,
	}, nil
}
<commit_msg>Add support for searching through image list Add support for uploading the ssh key Add support for launching the controller<commit_after>package oracle

import (
	"fmt"
	"os"
	"strings"

	oci "github.com/hoenirvili/go-oracle-cloud/api"
	"github.com/hoenirvili/go-oracle-cloud/response"
	"github.com/juju/errors"
	"github.com/juju/version"

	"github.com/juju/juju/constraints"
	"github.com/juju/juju/environs"
	"github.com/juju/juju/environs/config"
	"github.com/juju/juju/environs/imagemetadata"
	envinstance "github.com/juju/juju/environs/instances"
	"github.com/juju/juju/environs/simplestreams"
	"github.com/juju/juju/instance"
	"github.com/juju/juju/network"
	"github.com/juju/juju/storage"
)

// oracleEnviron implements the environs.Environ interface
// and has behaviour specific that the interface provides.
type oracleEnviron struct {
	p    *environProvider
	spec environs.CloudSpec
	cfg  *config.Config
}

func newOracleEnviron(p *environProvider, args environs.OpenParams) *oracleEnviron {
	env := &oracleEnviron{
		p:    p,
		spec: args.Cloud,
		cfg:  args.Config,
	}

	return env
}

func (o oracleEnviron) PrepareForBootstrap(ctx environs.BootstrapContext) error {
	return nil
}

func (o oracleEnviron) Validate(cfg, old *config.Config) (valid *config.Config, _ error) {
	return nil, nil
}

func (o oracleEnviron) StorageProviderTypes() ([]storage.ProviderType, error) {
	return nil, nil
}

func (o oracleEnviron) StorageProvider(t storage.ProviderType) (storage.Provider, error) {
	return nil, nil
}

func (o oracleEnviron) StartInstance(args environs.StartInstanceParams) (*environs.StartInstanceResult, error) {
	return nil, nil
}

func (o oracleEnviron) StopInstances(...instance.Id) error {
	return nil
}

func (o oracleEnviron) AllInstances() ([]instance.Instance, error) {
	return nil, nil
}

func (o oracleEnviron) MaintainInstance(args environs.StartInstanceParams) error {
	return nil
}

func (o oracleEnviron) Config() *config.Config {
	return o.cfg
}

// Bootstrap creates a new environment, and an instance inside the
// oracle cloud infrastracture to host the controller for that
// environment. The instnace will have have the series and architecture
// of the Environ's choice, constrained to those of the available tools.
// Bootstrap will return the instance's
// architecture, series, and a function that must be called to finalize
// the bootstrap process by transferring the tools and installing the
// initial Juju controller.
//
// Bootstrap will use just one specific architecture because the oracle
// cloud only supports amd64.
func (o oracleEnviron) Bootstrap(
	ctx environs.BootstrapContext,
	params environs.BootstrapParams,
) (*environs.BootstrapResult, error) {
	// in order too make the entire bootstrap process prossible
	// we must take into accounting some things:
	// validate if we have a shape based on the bootstrap constraints
	// and pick the right one
	// validate if we have in the imagelist a image correspoding to the
	// image tools specified.

	logger.Infof("Loging into the oracle cloud infrastructure")
	if err := o.p.client.Authenticate(); err != nil {
		return nil, errors.Trace(err)
	}

	shapes, err := o.p.client.AllShapeDetails()
	if err != nil {
		return nil, errors.Trace(err)
	}

	shape, err := findShape(shapes.Result, params.BootstrapConstraints)
	if err != nil {
		return nil, errors.Trace(err)
	}

	logger.Infof(
		"Choosing the %s with %d cores and %d MB ram",
		shape.name, shape.cpus, shape.ram,
	)

	imagelist, err := checkImageList(o.p.client, params.ImageMetadata, shape)
	if err != nil {
		return nil, errors.Trace(err)
	}

	// we are using just the juju ssh keys, no other keys
	keys := strings.Split(o.cfg.AuthorizedKeys(), "\n")
	// before bootstraping we must upload or check
	// if the ssh keys from juju their are already uploaded
	nameKey, err := uploadSSHControllerKeys(o.p.client, keys[0])
	if err != nil {
		return nil, errors.Trace(err)
	}
	instance, err := launchBootstrapConstroller(o.p.client, []oci.InstanceParams{
		{
			Shape:     shape.name,
			Imagelist: imagelist,
			Label:     o.cfg.UUID(),
			SSHKeys:   []string{nameKey},
			Name:      o.cfg.Name(),
		},
	})
	if err != nil {
		return nil, err
	}

	_ = instance
	return nil, nil
}

func uploadSSHControllerKeys(c *oci.Client, key string) (string, error) {
	if c == nil {
		return "", errors.Errorf("cannot use nil error client to upload ssh keys")
	}

	if len(key) == 0 {
		return "", errors.Errorf("cannot use empty key in order to upload it")
	}

	resp, err := c.SSHKeyDetails("juju-client-key")
	if err != nil {
		if strings.Contains(err.Error(), "does not exist") {
			// the keys does not exist, we should upload them
			resp, err = c.AddSHHKey("juju-client-key", key, true)
			if err != nil {
				return "", errors.Errorf("cannot upload the juju client keys")
			}
		} else {
			return "", errors.Errorf("cannot get details of the juju-client-key")
		}
	}

	if !resp.Enabled {
		// if it's not true we should enable the key
		resp, err = c.UpdateSSHKey("juju-client-key", key, true)
		if err != nil {
			return "", errors.Errorf("cannot enable the ssh juju-client-key")
		}
		if !resp.Enabled {
			return "", errors.Errorf("juju client key is not enabled after previously tried to enable")
		}
	}

	return resp.Name, nil
}

// here we should check with the client if the we have already a ubuntu/centos/ whatever image is specified in the tools metadata
// if there already exists then we should return it
func checkImageList(
	c *oci.Client,
	tools []*imagemetadata.ImageMetadata,
	shape *shape,
) (string, error) {

	var (
		imageVersion string
		imageArch    string
	)

	if c == nil {
		return "", errors.NotFoundf("Cannot use nil client")
	}

	if tools == nil {
		return "", errors.NotFoundf("No tools imagemedatada provided")
	}

	// take a list of all images that are in the oracle cloud account
	resp, err := c.AllImageList()
	if err != nil {
		return "", errors.Trace(err)
	}

	// filter through all the image list and if we found one complaint with
	// the tools just return it
	for _, val := range tools {
		if len(val.Version) > 0 && len(val.Arch) > 0 {
			imageVersion = val.Version
			imageVersion = "16.10" // TODO REMOVE THIS
			imageArch = val.Arch
		}

		if imageVersion == "" {
			continue
		}

		if imageArch == "" {
			continue
		}

		for _, val := range resp.Result {
			if isComplaint(val, shape.name, imageVersion, imageArch) {
				return theName(val.Name), nil
			}
		}
	}

	return "", errors.Errorf(
		"Cannot find any image in the oracle cloud that is complaint with the image tools",
	)

}

// extracts the name of a imagelist name
// the imagelist names contains the indenity, the username of the client and after
// the actual name of the imagelist all are separted with "/"
func theName(name string) string {
	names := strings.Split(name, "/")
	return names[len(names)-1]
}

func isComplaint(
	list response.ImageList,
	shapeName, imageVersion, imageArch string,
) bool {

	i := 0

	for _, val := range list.Entries {
		if strings.Contains(val.Attributes.SupportedShapes, shapeName) {
			i++
			break
		}
	}

	if strings.Contains(list.Name, imageVersion) {
		i++
	}

	if strings.Contains(list.Name, imageArch) {
		i++
	}

	if i == 3 {
		return true
	}

	return false
}

// launchBootstrapController creates a new vm inside
// the oracle infrastracuture and parses the response into a instance.Instance
func launchBootstrapConstroller(c *oci.Client, params []oci.InstanceParams) (instance.Instance, error) {
	if c == nil {
		return nil, errors.NotFoundf("client")
	}

	resp, err := c.CreateInstance(params)
	if err != nil {
		return nil, errors.Trace(err)
	}

	fmt.Printf("%v\n", resp)
	os.Exit(1)
	instance, err := newInstance(resp)
	if err != nil {
		return nil, errors.Trace(err)
	}

	return instance, nil
}

func (o oracleEnviron) BootstrapMessage() string {
	return "SomeBootstrapMessage"
}

func (o oracleEnviron) Create(params environs.CreateParams) error {
	return nil
}

// ConstraintsValidator returns a constraints.Validator instance which
// is used to validate and merge constraints.
//
// Validator defines operations on constraints attributes which are
// used to ensure a constraints value is valid, as well as being able
// to handle overridden attributes.
//
// This will use the default validator implementation from the constraints package.
func (o oracleEnviron) ConstraintsValidator() (constraints.Validator, error) {
	// list of unsupported oracle provider constraints
	unsupportedConstraints := []string{
		constraints.Container,
		constraints.CpuPower,
		constraints.RootDisk,
		constraints.Arch,
		constraints.InstanceType,
		constraints.VirtType,
		constraints.Spaces,
	}

	// we choose to use the default validator implementation
	validator := constraints.NewValidator()

	// we must feed the validator that the oracle cloud
	// provider does not support these constraints
	validator.RegisterUnsupported(unsupportedConstraints)

	return newConstraintsAdaptor(validator), nil
}

func (o oracleEnviron) SetConfig(cfg *config.Config) error {
	return nil
}

func (o oracleEnviron) Instances(ids []instance.Id) ([]instance.Instance, error) {
	return nil, nil
}

func (o oracleEnviron) ControllerInstances(controllerUUID string) ([]instance.Id, error) {
	return nil, nil
}

func (o oracleEnviron) Destroy() error {
	return nil
}

func (o oracleEnviron) DestroyController(controllerUUID string) error {
	return nil
}

func (o oracleEnviron) Provider() environs.EnvironProvider {
	return o.p
}

func (e oracleEnviron) AdoptResources(controllerUUID string, fromVersion version.Number) error {

	return nil
}

func (o oracleEnviron) PrecheckInstance(series string, cons constraints.Value, placement string) error {
	return nil
}

func (o oracleEnviron) OpenPorts(rules []network.IngressRule) error {
	return nil
}

func (o oracleEnviron) ClosePorts(rules []network.IngressRule) error {
	return nil
}

func (o oracleEnviron) IngressRules() ([]network.IngressRule, error) {
	return nil, nil
}

func (o oracleEnviron) InstanceTypes(constraints.Value) (envinstance.InstanceTypesWithCostMetadata, error) {
	var i envinstance.InstanceTypesWithCostMetadata
	return i, nil
}

// Providing this methods oracleEnviron implements also the simplestreams.HasRegion
// interface
//
// Region returns the necessary attributes to uniquely identify this cloud instance.
func (o oracleEnviron) Region() (simplestreams.CloudSpec, error) {
	return simplestreams.CloudSpec{
		Region:   o.spec.Region,
		Endpoint: o.spec.Endpoint,
	}, nil
}
<|endoftext|>


<commit_before>package coretest

import (
	"encoding/hex"
	"fmt"
	"crypto/rand"
	"strings"
	"testing"
)

const (
	targetAddress = "http://127.0.0.1:4001/v2/keys/"
	helloStr      = "Hello world"
	newHelloStr   = "Hello etcd"
	keyNotFound   = "Key not found"
)

// generateKey generate's a 16 byte random string.
func generateKey() string {
	b := make([]byte, 16)
	_, err := rand.Read(b)
	if err != nil {
		return ""
	}

	return hex.EncodeToString(b)
}

// TestEtcdUpdateValue tests to update value of a key.
// The test coverage includes setting, getting, updating, deleting.
func TestEtcdUpdateValue(t *testing.T) {
	// Use a random key name so members of a cluster don't step on each other.
	target := targetAddress + generateKey()

	stdout, stderr, err := Run("curl", "-L", target, "-XPUT", "-d", fmt.Sprintf("value=\"%s\"", helloStr))
	if err != nil {
		t.Fatalf("curl set failed with error: %v\nstdout: %s\nstderr: %s", err, stdout, stderr)
	}
	if !strings.Contains(stdout, helloStr) {
		t.Fatalf("Failed getting value %v\nstdout: %v", helloStr, stdout)
	}

	stdout, stderr, err = Run("curl", "-L", target, "-XPUT", "-d", fmt.Sprintf("value=\"%s\"", newHelloStr))
	if err != nil {
		t.Fatalf("curl update failed with error: %v\nstdout: %s\nstderr: %s", err, stdout, stderr)
	}

	stdout, stderr, err = Run("curl", "-L", target)
	if err != nil {
		t.Fatalf("curl get failed with error: %v\nstdout: %s\nstderr: %s", err, stdout, stderr)
	}
	if !strings.Contains(stdout, newHelloStr) {
		t.Fatalf("Failed getting value %v\nstdout: %v", newHelloStr, stdout)
	}

	stdout, stderr, err = Run("curl", "-L", target, "-XDELETE")
	if err != nil {
		t.Fatalf("curl delete failed with error: %v\nstdout: %s\nstderr: %s", err, stdout, stderr)
	}

	stdout, stderr, err = Run("curl", "-L", target)
	if err != nil {
		t.Fatalf("curl get failed with error: %v\nstdout: %s\nstderr: %s", err, stdout, stderr)
	}
	if !strings.Contains(stdout, keyNotFound) {
		t.Fatalf("Failed getting value %v\nstdout: %v", keyNotFound, stdout)
	}
}
<commit_msg>fix(etcd_test): retry on curl for etcd<commit_after>package coretest

import (
	"encoding/hex"
	"fmt"
	"crypto/rand"
	"strings"
	"testing"
)

const (
	targetAddress = "http://127.0.0.1:4001/v2/keys/"
	helloStr      = "Hello world"
	newHelloStr   = "Hello etcd"
	keyNotFound   = "Key not found"
)

var (
	// retry is used to avoid getting server error when leader election
	retry = []string{"--retry", "5", "--retry-delay", "2", "--silent"}
)

// generateKey generate's a 16 byte random string.
func generateKey() string {
	b := make([]byte, 16)
	_, err := rand.Read(b)
	if err != nil {
		return ""
	}

	return hex.EncodeToString(b)
}

// TestEtcdUpdateValue tests to update value of a key.
// The test coverage includes setting, getting, updating, deleting.
func TestEtcdUpdateValue(t *testing.T) {
	// Use a random key name so members of a cluster don't step on each other.
	target := targetAddress + generateKey()

	stdout, stderr, err := Run("curl", append(retry, "-L", target, "-XPUT", "-d", fmt.Sprintf("value=\"%s\"", helloStr))...)
	if err != nil {
		t.Fatalf("curl set failed with error: %v\nstdout: %s\nstderr: %s", err, stdout, stderr)
	}
	if !strings.Contains(stdout, helloStr) {
		t.Fatalf("Failed getting value %v\nstdout: %v", helloStr, stdout)
	}

	stdout, stderr, err = Run("curl", append(retry, "-L", target, "-XPUT", "-d", fmt.Sprintf("value=\"%s\"", newHelloStr))...)
	if err != nil {
		t.Fatalf("curl update failed with error: %v\nstdout: %s\nstderr: %s", err, stdout, stderr)
	}

	stdout, stderr, err = Run("curl", append(retry, "-L", target)...)
	if err != nil {
		t.Fatalf("curl get failed with error: %v\nstdout: %s\nstderr: %s", err, stdout, stderr)
	}
	if !strings.Contains(stdout, newHelloStr) {
		t.Fatalf("Failed getting value %v\nstdout: %v", newHelloStr, stdout)
	}

	stdout, stderr, err = Run("curl", append(retry, "-L", target, "-XDELETE")...)
	if err != nil {
		t.Fatalf("curl delete failed with error: %v\nstdout: %s\nstderr: %s", err, stdout, stderr)
	}

	stdout, stderr, err = Run("curl", append(retry, "-L", target)...)
	if err != nil {
		t.Fatalf("curl get failed with error: %v\nstdout: %s\nstderr: %s", err, stdout, stderr)
	}
	if !strings.Contains(stdout, keyNotFound) {
		t.Fatalf("Failed getting value %v\nstdout: %v", keyNotFound, stdout)
	}
}
<|endoftext|>


<commit_before>package pgo_cli_test

/*
 Copyright 2020 Crunchy Data Solutions, Inc.
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
 You may obtain a copy of the License at

      http://www.apache.org/licenses/LICENSE-2.0

 Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
*/

import (
	"context"
	"net"
	"os"
	"strings"
	"sync"
	"testing"
	"time"

	"github.com/crunchydata/postgres-operator/testing/kubeapi"
	"github.com/jackc/pgx/v4/pgxpool"
	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
	core_v1 "k8s.io/api/core/v1"
	"k8s.io/apiserver/pkg/storage/names"
)

type Pool struct {
	*kubeapi.Proxy
	*pgxpool.Pool
}

func (p *Pool) Close() error { p.Pool.Close(); return p.Proxy.Close() }

// clusterConnection opens a PostgreSQL connection to a database pod. Any error
// will cause t to FailNow.
func clusterConnection(t testing.TB, namespace, cluster, dsn string) *Pool {
	t.Helper()

	pods, err := TestContext.Kubernetes.ListPods(namespace, map[string]string{
		"pg-cluster":      cluster,
		"pgo-pg-database": "true",
	})
	require.NoError(t, err)
	require.NotEmpty(t, pods)

	proxy, err := TestContext.Kubernetes.PodPortForward(pods[0].Namespace, pods[0].Name, "5432")
	require.NoError(t, err)

	host, port, err := net.SplitHostPort(proxy.LocalAddr())
	if err != nil {
		proxy.Close()
		require.NoError(t, err)
	}

	pool, err := pgxpool.Connect(context.Background(), dsn+" host="+host+" port="+port)
	if err != nil {
		proxy.Close()
		require.NoError(t, err)
	}

	return &Pool{proxy, pool}
}

// clusterDatabases returns the names of all non-template databases in cluster.
// Any error will cause t to FailNow.
func clusterDatabases(t testing.TB, namespace, cluster string) []string {
	stdout, stderr := clusterPSQL(t, namespace, cluster, `
		\set QUIET yes
		\pset format unaligned
		\pset tuples_only yes
		SELECT datname FROM pg_database WHERE NOT datistemplate;
	`)
	require.Empty(t, stderr)
	return strings.FieldsFunc(stdout, func(c rune) bool { return c == '\r' || c == '\n' })
}

// clusterPSQL executes psql commands and/or SQL on a database pod. Any error
// will cause t to FailNow.
func clusterPSQL(t testing.TB, namespace, cluster, psql string) (string, string) {
	t.Helper()

	pods, err := TestContext.Kubernetes.ListPods(namespace, map[string]string{
		"pg-cluster":      cluster,
		"pgo-pg-database": "true",
	})
	require.NoError(t, err)
	require.NotEmpty(t, pods)

	stdout, stderr, err := TestContext.Kubernetes.PodExec(
		pods[0].Namespace, pods[0].Name,
		strings.NewReader(psql), "psql", "-U", "postgres", "-f-")
	require.NoError(t, err)

	return stdout, stderr
}

// clusterPVCs returns a list of persistent volume claims for the cluster. Any
// error will cause t to FailNow.
func clusterPVCs(t testing.TB, namespace, cluster string) []core_v1.PersistentVolumeClaim {
	t.Helper()

	pvcs, err := TestContext.Kubernetes.ListPVCs(namespace, map[string]string{
		"pg-cluster": cluster,
	})
	require.NoError(t, err)

	return pvcs
}

// primaryPods returns a list of PostgreSQL primary pods for the cluster. Any
// error will cause t to FailNow.
func primaryPods(t testing.TB, namespace, cluster string) []core_v1.Pod {
	t.Helper()

	pods, err := TestContext.Kubernetes.ListPods(namespace, map[string]string{
		"pg-cluster": cluster,
		"role":       "master",
	})
	require.NoError(t, err)

	return pods
}

// replicaPods returns a list of PostgreSQL replica pods for the cluster. Any
// error will cause t to FailNow.
func replicaPods(t testing.TB, namespace, cluster string) []core_v1.Pod {
	t.Helper()

	pods, err := TestContext.Kubernetes.ListPods(namespace, map[string]string{
		"pg-cluster": cluster,
		"role":       "replica",
	})
	require.NoError(t, err)

	return pods
}

// requireClusterReady waits until all deployments of cluster are ready. If
// timeout elapses or any error occurs, t will FailNow.
func requireClusterReady(t testing.TB, namespace, cluster string, timeout time.Duration) {
	t.Helper()

	// Give up now if some part of setting up the cluster failed.
	if t.Failed() || cluster == "" {
		t.FailNow()
	}

	ready := func() bool {
		deployments, err := TestContext.Kubernetes.ListDeployments(namespace, map[string]string{
			"pg-cluster": cluster,
		})
		require.NoError(t, err)

		if len(deployments) == 0 {
			return false
		}

		var database bool
		for _, deployment := range deployments {
			if deployment.Status.Replicas != deployment.Status.ReadyReplicas {
				return false
			}
			if deployment.Labels["pgo-pg-database"] == "true" {
				database = true
			}
		}
		return database
	}

	if !ready() {
		requireWaitFor(t, ready, timeout, time.Second,
			"timeout waiting for %q in %q", cluster, namespace)
	}
}

// requirePgBouncerReady waits until all PgBouncer deployments for cluster are
// ready. If timeout elapses or any error occurs, t will FailNow.
func requirePgBouncerReady(t testing.TB, namespace, cluster string, timeout time.Duration) {
	t.Helper()

	ready := func() bool {
		deployments, err := TestContext.Kubernetes.ListDeployments(namespace, map[string]string{
			"pg-cluster":        cluster,
			"crunchy-pgbouncer": "true",
		})
		require.NoError(t, err)

		if len(deployments) == 0 {
			return false
		}
		for _, deployment := range deployments {
			if deployment.Status.Replicas != deployment.Status.ReadyReplicas {
				return false
			}
		}
		return true
	}

	if !ready() {
		requireWaitFor(t, ready, timeout, time.Second,
			"timeout waiting for PgBouncer of %q in %q", cluster, namespace)
	}
}

// requireReplicasReady waits until all replicas of cluster are ready. If
// timeout elapses or any error occurs, t will FailNow.
func requireReplicasReady(t testing.TB, namespace, cluster string, timeout time.Duration) {
	t.Helper()

	ready := func() bool {
		pods := replicaPods(t, namespace, cluster)

		if len(pods) == 0 {
			return false
		}
		for _, pod := range pods {
			if !kubeapi.IsPodReady(pod) {
				return false
			}
		}
		return true
	}

	if !ready() {
		requireWaitFor(t, ready, timeout, time.Second,
			"timeout waiting for replicas of %q in %q", cluster, namespace)
	}
}

// requireStanzaExists waits until pgBackRest reports the stanza is ok. If
// timeout elapses, t will FailNow.
func requireStanzaExists(t testing.TB, namespace, cluster string, timeout time.Duration) {
	t.Helper()

	var err error
	var output string

	ready := func() bool {
		output, err = pgo("show", "backup", cluster, "-n", namespace).Exec(t)
		return err == nil && strings.Contains(output, "status: ok")
	}

	if !ready() {
		requireWaitFor(t, ready, timeout, time.Second,
			"timeout waiting for stanza of %q in %q:\n%s", cluster, namespace, output)
	}
}

// requireWaitFor calls condition every tick until it returns true. If timeout
// elapses, t will Logf message and args then FailNow. Condition runs in the
// current goroutine so that it may also call t.FailNow.
func requireWaitFor(t testing.TB,
	condition func() bool, timeout, tick time.Duration,
	message string, args ...interface{},
) {
	t.Helper()

	if !waitFor(t, condition, timeout, tick) {
		t.Fatalf(message, args...)
	}
}

// teardownCluster deletes a cluster. It waits sufficiently long after created
// for the delete to go well.
func teardownCluster(t testing.TB, namespace, cluster string, created time.Time) {
	minimum := TestContext.Scale(10 * time.Second)

	if elapsed := time.Since(created); elapsed < minimum {
		time.Sleep(minimum - elapsed)
	}

	_, err := pgo("delete", "cluster", cluster, "-n", namespace, "--no-prompt").Exec(t)
	assert.NoError(t, err, "unable to tear down cluster %q in %q", cluster, namespace)
}

// waitFor calls condition once every tick until it returns true. If timeout
// elapses or t Failed, waitFor returns false. Condition runs in the current
// goroutine so that it may also call t.FailNow.
func waitFor(t testing.TB, condition func() bool, timeout, tick time.Duration) bool {
	t.Helper()

	ticker := time.NewTicker(tick)
	defer ticker.Stop()

	timer := time.NewTimer(TestContext.Scale(timeout))
	defer timer.Stop()

	for {
		select {
		case <-timer.C:
			return false
		case <-ticker.C:
			if condition() {
				return true
			}
			if t.Failed() {
				return false
			}
		}
	}
}

// withCluster calls during with a function that returns the name of an existing
// cluster. The cluster may not exist until that function is called. When during
// returns, the cluster might be destroyed.
func withCluster(t testing.TB, namespace func() string, during func(func() string)) {
	t.Helper()

	var created time.Time
	var name string
	var once sync.Once

	defer func() {
		if name != "" {
			teardownCluster(t, namespace(), name, created)
		}
	}()

	during(func() string {
		once.Do(func() {
			generated := names.SimpleNameGenerator.GenerateName("pgo-test-")
			_, err := pgo("create", "cluster", generated, "-n", namespace()).Exec(t)

			if assert.NoError(t, err) {
				t.Logf("created cluster %q in %q", generated, namespace())
				created = time.Now()
				name = generated
			}
		})
		return name
	})
}

// withNamespace calls during with a function that returns the name of an
// existing namespace. The namespace may not exist until that function is
// called. When during returns, the namespace and all its contents are destroyed.
func withNamespace(t testing.TB, during func(func() string)) {
	t.Helper()

	// Use the namespace specified in the environment.
	if name := os.Getenv("PGO_NAMESPACE"); name != "" {
		during(func() string { return name })
		return
	}

	// Prepare to cleanup a namespace that might be created.
	var namespace *core_v1.Namespace
	var once sync.Once

	defer func() {
		if namespace != nil {
			err := TestContext.Kubernetes.DeleteNamespace(namespace.Name)
			assert.NoErrorf(t, err, "unable to tear down namespace %q", namespace.Name)
		}
	}()

	during(func() string {
		once.Do(func() {
			ns, err := TestContext.Kubernetes.GenerateNamespace(
				"pgo-test-", map[string]string{"pgo-test": kubeapi.SanitizeLabelValue(t.Name())})

			if assert.NoError(t, err) {
				namespace = ns
				_, err = pgo("update", "namespace", namespace.Name).Exec(t)
				assert.NoErrorf(t, err, "unable to take ownership of namespace %q", namespace.Name)
			}
		})

		return namespace.Name
	})
}
<commit_msg>Compare spec to status while waiting in tests<commit_after>package pgo_cli_test

/*
 Copyright 2020 Crunchy Data Solutions, Inc.
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
 You may obtain a copy of the License at

      http://www.apache.org/licenses/LICENSE-2.0

 Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
*/

import (
	"context"
	"net"
	"os"
	"strings"
	"sync"
	"testing"
	"time"

	"github.com/crunchydata/postgres-operator/testing/kubeapi"
	"github.com/jackc/pgx/v4/pgxpool"
	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
	core_v1 "k8s.io/api/core/v1"
	"k8s.io/apiserver/pkg/storage/names"
)

type Pool struct {
	*kubeapi.Proxy
	*pgxpool.Pool
}

func (p *Pool) Close() error { p.Pool.Close(); return p.Proxy.Close() }

// clusterConnection opens a PostgreSQL connection to a database pod. Any error
// will cause t to FailNow.
func clusterConnection(t testing.TB, namespace, cluster, dsn string) *Pool {
	t.Helper()

	pods, err := TestContext.Kubernetes.ListPods(namespace, map[string]string{
		"pg-cluster":      cluster,
		"pgo-pg-database": "true",
	})
	require.NoError(t, err)
	require.NotEmpty(t, pods)

	proxy, err := TestContext.Kubernetes.PodPortForward(pods[0].Namespace, pods[0].Name, "5432")
	require.NoError(t, err)

	host, port, err := net.SplitHostPort(proxy.LocalAddr())
	if err != nil {
		proxy.Close()
		require.NoError(t, err)
	}

	pool, err := pgxpool.Connect(context.Background(), dsn+" host="+host+" port="+port)
	if err != nil {
		proxy.Close()
		require.NoError(t, err)
	}

	return &Pool{proxy, pool}
}

// clusterDatabases returns the names of all non-template databases in cluster.
// Any error will cause t to FailNow.
func clusterDatabases(t testing.TB, namespace, cluster string) []string {
	stdout, stderr := clusterPSQL(t, namespace, cluster, `
		\set QUIET yes
		\pset format unaligned
		\pset tuples_only yes
		SELECT datname FROM pg_database WHERE NOT datistemplate;
	`)
	require.Empty(t, stderr)
	return strings.FieldsFunc(stdout, func(c rune) bool { return c == '\r' || c == '\n' })
}

// clusterPSQL executes psql commands and/or SQL on a database pod. Any error
// will cause t to FailNow.
func clusterPSQL(t testing.TB, namespace, cluster, psql string) (string, string) {
	t.Helper()

	pods, err := TestContext.Kubernetes.ListPods(namespace, map[string]string{
		"pg-cluster":      cluster,
		"pgo-pg-database": "true",
	})
	require.NoError(t, err)
	require.NotEmpty(t, pods)

	stdout, stderr, err := TestContext.Kubernetes.PodExec(
		pods[0].Namespace, pods[0].Name,
		strings.NewReader(psql), "psql", "-U", "postgres", "-f-")
	require.NoError(t, err)

	return stdout, stderr
}

// clusterPVCs returns a list of persistent volume claims for the cluster. Any
// error will cause t to FailNow.
func clusterPVCs(t testing.TB, namespace, cluster string) []core_v1.PersistentVolumeClaim {
	t.Helper()

	pvcs, err := TestContext.Kubernetes.ListPVCs(namespace, map[string]string{
		"pg-cluster": cluster,
	})
	require.NoError(t, err)

	return pvcs
}

// primaryPods returns a list of PostgreSQL primary pods for the cluster. Any
// error will cause t to FailNow.
func primaryPods(t testing.TB, namespace, cluster string) []core_v1.Pod {
	t.Helper()

	pods, err := TestContext.Kubernetes.ListPods(namespace, map[string]string{
		"pg-cluster": cluster,
		"role":       "master",
	})
	require.NoError(t, err)

	return pods
}

// replicaPods returns a list of PostgreSQL replica pods for the cluster. Any
// error will cause t to FailNow.
func replicaPods(t testing.TB, namespace, cluster string) []core_v1.Pod {
	t.Helper()

	pods, err := TestContext.Kubernetes.ListPods(namespace, map[string]string{
		"pg-cluster": cluster,
		"role":       "replica",
	})
	require.NoError(t, err)

	return pods
}

// requireClusterReady waits until all deployments of cluster are ready. If
// timeout elapses or any error occurs, t will FailNow.
func requireClusterReady(t testing.TB, namespace, cluster string, timeout time.Duration) {
	t.Helper()

	// Give up now if some part of setting up the cluster failed.
	if t.Failed() || cluster == "" {
		t.FailNow()
	}

	ready := func() bool {
		deployments, err := TestContext.Kubernetes.ListDeployments(namespace, map[string]string{
			"pg-cluster": cluster,
		})
		require.NoError(t, err)

		if len(deployments) == 0 {
			return false
		}

		var database bool
		for _, deployment := range deployments {
			if *deployment.Spec.Replicas < 1 ||
				deployment.Status.ReadyReplicas != *deployment.Spec.Replicas ||
				deployment.Status.UpdatedReplicas != *deployment.Spec.Replicas {
				return false
			}
			if deployment.Labels["pgo-pg-database"] == "true" {
				database = true
			}
		}
		return database
	}

	if !ready() {
		requireWaitFor(t, ready, timeout, time.Second,
			"timeout waiting for %q in %q", cluster, namespace)
	}
}

// requirePgBouncerReady waits until all PgBouncer deployments for cluster are
// ready. If timeout elapses or any error occurs, t will FailNow.
func requirePgBouncerReady(t testing.TB, namespace, cluster string, timeout time.Duration) {
	t.Helper()

	ready := func() bool {
		deployments, err := TestContext.Kubernetes.ListDeployments(namespace, map[string]string{
			"pg-cluster":        cluster,
			"crunchy-pgbouncer": "true",
		})
		require.NoError(t, err)

		if len(deployments) == 0 {
			return false
		}
		for _, deployment := range deployments {
			if *deployment.Spec.Replicas < 1 ||
				deployment.Status.ReadyReplicas != *deployment.Spec.Replicas ||
				deployment.Status.UpdatedReplicas != *deployment.Spec.Replicas {
				return false
			}
		}
		return true
	}

	if !ready() {
		requireWaitFor(t, ready, timeout, time.Second,
			"timeout waiting for PgBouncer of %q in %q", cluster, namespace)
	}
}

// requireReplicasReady waits until all replicas of cluster are ready. If
// timeout elapses or any error occurs, t will FailNow.
func requireReplicasReady(t testing.TB, namespace, cluster string, timeout time.Duration) {
	t.Helper()

	ready := func() bool {
		pods := replicaPods(t, namespace, cluster)

		if len(pods) == 0 {
			return false
		}
		for _, pod := range pods {
			if !kubeapi.IsPodReady(pod) {
				return false
			}
		}
		return true
	}

	if !ready() {
		requireWaitFor(t, ready, timeout, time.Second,
			"timeout waiting for replicas of %q in %q", cluster, namespace)
	}
}

// requireStanzaExists waits until pgBackRest reports the stanza is ok. If
// timeout elapses, t will FailNow.
func requireStanzaExists(t testing.TB, namespace, cluster string, timeout time.Duration) {
	t.Helper()

	var err error
	var output string

	ready := func() bool {
		output, err = pgo("show", "backup", cluster, "-n", namespace).Exec(t)
		return err == nil && strings.Contains(output, "status: ok")
	}

	if !ready() {
		requireWaitFor(t, ready, timeout, time.Second,
			"timeout waiting for stanza of %q in %q:\n%s", cluster, namespace, output)
	}
}

// requireWaitFor calls condition every tick until it returns true. If timeout
// elapses, t will Logf message and args then FailNow. Condition runs in the
// current goroutine so that it may also call t.FailNow.
func requireWaitFor(t testing.TB,
	condition func() bool, timeout, tick time.Duration,
	message string, args ...interface{},
) {
	t.Helper()

	if !waitFor(t, condition, timeout, tick) {
		t.Fatalf(message, args...)
	}
}

// teardownCluster deletes a cluster. It waits sufficiently long after created
// for the delete to go well.
func teardownCluster(t testing.TB, namespace, cluster string, created time.Time) {
	minimum := TestContext.Scale(10 * time.Second)

	if elapsed := time.Since(created); elapsed < minimum {
		time.Sleep(minimum - elapsed)
	}

	_, err := pgo("delete", "cluster", cluster, "-n", namespace, "--no-prompt").Exec(t)
	assert.NoError(t, err, "unable to tear down cluster %q in %q", cluster, namespace)
}

// waitFor calls condition once every tick until it returns true. If timeout
// elapses or t Failed, waitFor returns false. Condition runs in the current
// goroutine so that it may also call t.FailNow.
func waitFor(t testing.TB, condition func() bool, timeout, tick time.Duration) bool {
	t.Helper()

	ticker := time.NewTicker(tick)
	defer ticker.Stop()

	timer := time.NewTimer(TestContext.Scale(timeout))
	defer timer.Stop()

	for {
		select {
		case <-timer.C:
			return false
		case <-ticker.C:
			if condition() {
				return true
			}
			if t.Failed() {
				return false
			}
		}
	}
}

// withCluster calls during with a function that returns the name of an existing
// cluster. The cluster may not exist until that function is called. When during
// returns, the cluster might be destroyed.
func withCluster(t testing.TB, namespace func() string, during func(func() string)) {
	t.Helper()

	var created time.Time
	var name string
	var once sync.Once

	defer func() {
		if name != "" {
			teardownCluster(t, namespace(), name, created)
		}
	}()

	during(func() string {
		once.Do(func() {
			generated := names.SimpleNameGenerator.GenerateName("pgo-test-")
			_, err := pgo("create", "cluster", generated, "-n", namespace()).Exec(t)

			if assert.NoError(t, err) {
				t.Logf("created cluster %q in %q", generated, namespace())
				created = time.Now()
				name = generated
			}
		})
		return name
	})
}

// withNamespace calls during with a function that returns the name of an
// existing namespace. The namespace may not exist until that function is
// called. When during returns, the namespace and all its contents are destroyed.
func withNamespace(t testing.TB, during func(func() string)) {
	t.Helper()

	// Use the namespace specified in the environment.
	if name := os.Getenv("PGO_NAMESPACE"); name != "" {
		during(func() string { return name })
		return
	}

	// Prepare to cleanup a namespace that might be created.
	var namespace *core_v1.Namespace
	var once sync.Once

	defer func() {
		if namespace != nil {
			err := TestContext.Kubernetes.DeleteNamespace(namespace.Name)
			assert.NoErrorf(t, err, "unable to tear down namespace %q", namespace.Name)
		}
	}()

	during(func() string {
		once.Do(func() {
			ns, err := TestContext.Kubernetes.GenerateNamespace(
				"pgo-test-", map[string]string{"pgo-test": kubeapi.SanitizeLabelValue(t.Name())})

			if assert.NoError(t, err) {
				namespace = ns
				_, err = pgo("update", "namespace", namespace.Name).Exec(t)
				assert.NoErrorf(t, err, "unable to take ownership of namespace %q", namespace.Name)
			}
		})

		return namespace.Name
	})
}
<|endoftext|>


<commit_before>package main

import (
	"crypto/sha256"
	"encoding/base64"
	"flag"
	"fmt"
	"image"
	"log"
	"net/http"

	"github.com/koyachi/go-nude"
)

var (
	// in memory cache
	memCache = map[string]bool{}

	// command line flags
	listenTo = flag.String("listen-to", "localhost:8000", "bind address and port")
)

// fetchLink download image by link and returns it
func fetchLink(link string) (img image.Image, err error) {
	r, err := http.DefaultClient.Get(link)
	if err != nil {
		return
	}

	defer r.Body.Close()

	img, _, err = image.Decode(r.Body)
	return
}

// checkLinknudity checks that passed link is nude
func checkLinknudity(link string) (isNude bool) {
	img, err := fetchLink(link)
	if err != nil {
		log.Printf("fetch url error %s: %s", link, err)
		return
	}

	isNude, err = nude.IsImageNude(img)
	if err != nil {
		log.Printf("check nudity error: %s", err)
	}
	return
}

func main() {
	flag.Parse()

	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		var isNude, ok bool

		w.Header().Set("Content-type", "application/json")

		u := r.URL.Query().Get("u")
		if u != "" {
			link, err := base64.URLEncoding.DecodeString(u)
			if err == nil {
				key := fmt.Sprintf("%x", sha256.Sum256(link))
				isNude, ok = memCache[key]
				if !ok {
					isNude = checkLinknudity(string(link))
					memCache[key] = isNude
				}
			} else {
				log.Printf("url decoding error %s: %s", u, err)
			}
		}

		fmt.Fprintf(w, "{\"isNude\": \"%v\"}", isNude)
	})
	http.ListenAndServe(*listenTo, nil)
}
<commit_msg>cache fix<commit_after>package main

import (
	"crypto/sha256"
	"encoding/base64"
	"encoding/json"
	"flag"
	"fmt"
	"image"
	"log"
	"net/http"
	"sync"

	"github.com/koyachi/go-nude"
)

type memCacheType struct {
	cache map[string]bool
	m     sync.RWMutex
}

func (mc *memCacheType) Get(key string) (isNude bool, ok bool) {
	mc.m.RLock()
	isNude, ok = mc.cache[key]
	mc.m.RUnlock()
	return
}

func (mc *memCacheType) Set(key string, value bool) {
	mc.m.Lock()
	mc.cache[key] = value
	mc.m.Unlock()
}

var (
	// in memory cache
	memCache = &memCacheType{cache: make(map[string]bool)}

	// command line flags
	listenTo = flag.String("listen-to", "localhost:8000", "bind address and port")
)

// fetchLink download image by link and returns it
func fetchLink(link string) (img image.Image, err error) {
	r, err := http.DefaultClient.Get(link)
	if err != nil {
		return
	}

	defer r.Body.Close()

	img, _, err = image.Decode(r.Body)
	return
}

// checkLinknudity checks that passed link is nude
func checkLinknudity(link string) (isNude bool) {
	img, err := fetchLink(link)
	if err != nil {
		log.Printf("fetch url error %s: %s", link, err)
		return
	}

	isNude, err = nude.IsImageNude(img)
	if err != nil {
		log.Printf("check nudity error: %s", err)
	}
	return
}

type resultType struct {
	IsNude bool `json:"isNude"`
}

func main() {
	flag.Parse()

	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		var isNude, ok bool

		w.Header().Set("Content-type", "application/json")

		u := r.URL.Query().Get("u")
		if u != "" {
			link, err := base64.URLEncoding.DecodeString(u)
			if err == nil {
				key := fmt.Sprintf("%x", sha256.Sum256(link))
				isNude, ok = memCache.Get(key)
				if !ok {
					isNude = checkLinknudity(string(link))
					memCache.Set(key, isNude)
				}
			} else {
				log.Printf("url decoding error %s: %s", u, err)
			}
		}

		buf, err := json.Marshal(&resultType{isNude})
		if err != nil {
			log.Printf("json encode error: %s", err)
		}

		w.Write(buf)
	})
	http.ListenAndServe(*listenTo, nil)
}
<|endoftext|>