<commit_before>package k8s

import (
	"fmt"
	"log"
	"strings"
	"time"

	"github.com/datawire/teleproxy/pkg/tpu"
)

type Waiter struct {
	watcher *Watcher
	kinds   map[string]map[string]bool
}

// NewWaiter constructs a Waiter object based on the suppliec Watcher.
func NewWaiter(watcher *Watcher) (w *Waiter, err error) {
	if watcher == nil {
		cli, err := NewClient(nil)
		if err != nil {
			return nil, err
		}
		watcher = cli.Watcher()
	}
	return &Waiter{
		watcher: watcher,
		kinds:   make(map[string]map[string]bool),
	}, nil
}

// canonical returns the canonical form of either a resource name or a
// resource type name:
//
//   ResourceName: TYPE/NAME[.NAMESPACE]
//   ResourceType: TYPE
//
func (w *Waiter) canonical(name string) string {
	parts := strings.Split(name, "/")

	var kind string
	switch len(parts) {
	case 1:
		kind = parts[0]
		name = ""
	case 2:
		kind = parts[0]
		name = parts[1]
	default:
		return ""
	}

	ri, err := w.watcher.client.ResolveResourceType(kind)
	if err != nil {
		panic(fmt.Sprintf("%s: %v", kind, err))
	}
	kind = strings.ToLower(ri.String())

	if name == "" {
		return kind
	}

	if ri.Namespaced {
		var namespace string

		parts = strings.Split(name, ".")
		switch len(parts) {
		case 1:
			namespace = "default"
		case 2:
			name = parts[0]
			namespace = parts[1]
		default:
			return ""
		}

		return fmt.Sprintf("%s/%s.%s", kind, name, namespace)
	}

	return fmt.Sprintf("%s/%s", kind, name)
}

func (w *Waiter) Add(resource string) error {
	cresource := w.canonical(resource)

	parts := strings.Split(cresource, "/")
	if len(parts) != 2 {
		return fmt.Errorf("expecting <kind>/<name>[.<namespace>], got %s", resource)
	}

	kind := parts[0]
	name := parts[1]

	resources, ok := w.kinds[kind]
	if !ok {
		resources = make(map[string]bool)
		w.kinds[kind] = resources
	}
	resources[name] = false
	return nil
}

func (w *Waiter) Scan(path string) (err error) {
	resources, err := LoadResources(path)
	for _, res := range resources {
		err = w.Add(fmt.Sprintf("%s/%s", res.QKind(), res.QName()))
		if err != nil {
			return
		}
	}
	return
}

func (w *Waiter) ScanPaths(files []string) (err error) {
	resources, err := WalkResources(tpu.IsYaml, files...)
	for _, res := range resources {
		err = w.Add(fmt.Sprintf("%s/%s", res.QKind(), res.QName()))
		if err != nil {
			return
		}
	}
	return
}

func (w *Waiter) remove(kind, name string) {
	delete(w.kinds[kind], name)
}

func (w *Waiter) isEmpty() bool {
	for _, names := range w.kinds {
		if len(names) > 0 {
			return false
		}
	}

	return true
}

func (w *Waiter) Wait(timeout time.Duration) bool {
	start := time.Now()
	printed := make(map[string]bool)
	w.watcher.Watch("events", func(watcher *Watcher) {
		for _, r := range watcher.List("events") {
			if lastStr, ok := r["lastTimestamp"].(string); ok {
				last, err := time.Parse("2006-01-02T15:04:05Z", lastStr)
				if err != nil {
					log.Println(err)
					continue
				}
				if last.Before(start) {
					continue
				}
			}
			if !printed[r.QName()] {
				var name string
				if obj, ok := r["involvedObject"].(map[string]interface{}); ok {
					name = w.canonical(fmt.Sprintf("%s/%v.%v", obj["kind"], obj["name"], obj["namespace"]))
				} else {
					name = r.QName()
				}
				fmt.Printf("event: %s %s\n", name, r["message"])
				printed[r.QName()] = true
			}
		}
	})

	listener := func(watcher *Watcher) {
		for kind, names := range w.kinds {
			for name := range names {
				r := watcher.Get(kind, name)
				if r.Ready() {
					if r.ReadyImplemented() {
						fmt.Printf("ready: %s/%s\n", w.canonical(r.QKind()), r.QName())
					} else {
						fmt.Printf("ready: %s/%s (UNIMPLEMENTED)\n",
							w.canonical(r.QKind()), r.QName())
					}
					w.remove(kind, name)
				}
			}
		}

		if w.isEmpty() {
			watcher.Stop()
		}
	}

	for k := range w.kinds {
		err := w.watcher.Watch(k, listener)
		if err != nil {
			panic(err)
		}
	}

	w.watcher.Start()

	go func() {
		time.Sleep(timeout)
		w.watcher.Stop()
	}()

	w.watcher.Wait()

	result := true

	for kind, names := range w.kinds {
		for name := range names {
			fmt.Printf("not ready: %s/%s\n", kind, name)
			result = false
		}
	}

	return result
}
<commit_msg>pkg/k8s.Waiter.Wait: Use the qualified kind in canonicalized names in UI<commit_after>package k8s

import (
	"fmt"
	"log"
	"strings"
	"time"

	"github.com/datawire/teleproxy/pkg/tpu"
)

type Waiter struct {
	watcher *Watcher
	kinds   map[string]map[string]bool
}

// NewWaiter constructs a Waiter object based on the suppliec Watcher.
func NewWaiter(watcher *Watcher) (w *Waiter, err error) {
	if watcher == nil {
		cli, err := NewClient(nil)
		if err != nil {
			return nil, err
		}
		watcher = cli.Watcher()
	}
	return &Waiter{
		watcher: watcher,
		kinds:   make(map[string]map[string]bool),
	}, nil
}

// canonical returns the canonical form of either a resource name or a
// resource type name:
//
//   ResourceName: TYPE/NAME[.NAMESPACE]
//   ResourceType: TYPE
//
func (w *Waiter) canonical(name string) string {
	parts := strings.Split(name, "/")

	var kind string
	switch len(parts) {
	case 1:
		kind = parts[0]
		name = ""
	case 2:
		kind = parts[0]
		name = parts[1]
	default:
		return ""
	}

	ri, err := w.watcher.client.ResolveResourceType(kind)
	if err != nil {
		panic(fmt.Sprintf("%s: %v", kind, err))
	}
	kind = strings.ToLower(ri.String())

	if name == "" {
		return kind
	}

	if ri.Namespaced {
		var namespace string

		parts = strings.Split(name, ".")
		switch len(parts) {
		case 1:
			namespace = "default"
		case 2:
			name = parts[0]
			namespace = parts[1]
		default:
			return ""
		}

		return fmt.Sprintf("%s/%s.%s", kind, name, namespace)
	}

	return fmt.Sprintf("%s/%s", kind, name)
}

func (w *Waiter) Add(resource string) error {
	cresource := w.canonical(resource)

	parts := strings.Split(cresource, "/")
	if len(parts) != 2 {
		return fmt.Errorf("expecting <kind>/<name>[.<namespace>], got %s", resource)
	}

	kind := parts[0]
	name := parts[1]

	resources, ok := w.kinds[kind]
	if !ok {
		resources = make(map[string]bool)
		w.kinds[kind] = resources
	}
	resources[name] = false
	return nil
}

func (w *Waiter) Scan(path string) (err error) {
	resources, err := LoadResources(path)
	for _, res := range resources {
		err = w.Add(fmt.Sprintf("%s/%s", res.QKind(), res.QName()))
		if err != nil {
			return
		}
	}
	return
}

func (w *Waiter) ScanPaths(files []string) (err error) {
	resources, err := WalkResources(tpu.IsYaml, files...)
	for _, res := range resources {
		err = w.Add(fmt.Sprintf("%s/%s", res.QKind(), res.QName()))
		if err != nil {
			return
		}
	}
	return
}

func (w *Waiter) remove(kind, name string) {
	delete(w.kinds[kind], name)
}

func (w *Waiter) isEmpty() bool {
	for _, names := range w.kinds {
		if len(names) > 0 {
			return false
		}
	}

	return true
}

func (w *Waiter) Wait(timeout time.Duration) bool {
	start := time.Now()
	printed := make(map[string]bool)
	w.watcher.Watch("events", func(watcher *Watcher) {
		for _, r := range watcher.List("events") {
			if lastStr, ok := r["lastTimestamp"].(string); ok {
				last, err := time.Parse("2006-01-02T15:04:05Z", lastStr)
				if err != nil {
					log.Println(err)
					continue
				}
				if last.Before(start) {
					continue
				}
			}
			if !printed[r.QName()] {
				var name string
				if obj, ok := r["involvedObject"].(map[string]interface{}); ok {
					name = w.canonical(fmt.Sprintf("%s/%v.%v", Resource(obj).QKind(), obj["name"], obj["namespace"]))
				} else {
					name = r.QName()
				}
				fmt.Printf("event: %s %s\n", name, r["message"])
				printed[r.QName()] = true
			}
		}
	})

	listener := func(watcher *Watcher) {
		for kind, names := range w.kinds {
			for name := range names {
				r := watcher.Get(kind, name)
				if r.Ready() {
					if r.ReadyImplemented() {
						fmt.Printf("ready: %s/%s\n", w.canonical(r.QKind()), r.QName())
					} else {
						fmt.Printf("ready: %s/%s (UNIMPLEMENTED)\n",
							w.canonical(r.QKind()), r.QName())
					}
					w.remove(kind, name)
				}
			}
		}

		if w.isEmpty() {
			watcher.Stop()
		}
	}

	for k := range w.kinds {
		err := w.watcher.Watch(k, listener)
		if err != nil {
			panic(err)
		}
	}

	w.watcher.Start()

	go func() {
		time.Sleep(timeout)
		w.watcher.Stop()
	}()

	w.watcher.Wait()

	result := true

	for kind, names := range w.kinds {
		for name := range names {
			fmt.Printf("not ready: %s/%s\n", kind, name)
			result = false
		}
	}

	return result
}
<|endoftext|>


<commit_before>package k8s

import (
	"encoding/base64"
	"fmt"
	"os"

	"github.com/banzaicloud/bank-vaults/pkg/kv"
	"k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/api/errors"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/client-go/kubernetes"
	"k8s.io/client-go/rest"
	"k8s.io/client-go/tools/clientcmd"
)

type s3Storage struct {
	cl        *kubernetes.Clientset
	namespace string
	secret    string
}

func New(namespace, secret string) (service kv.Service, err error) {
	kubeconfig := os.Getenv("KUBECONFIG")
	var config *rest.Config

	if kubeconfig != "" {
		config, err = clientcmd.BuildConfigFromFlags("", kubeconfig)
	} else {
		config, err = rest.InClusterConfig()
	}

	if err != nil {
		return nil, fmt.Errorf("error creating k8s config: %s", err.Error())
	}

	client, err := kubernetes.NewForConfig(config)
	if err != nil {
		return nil, fmt.Errorf("error creating k8s client: %s", err.Error())
	}

	service = &s3Storage{client, namespace, secret}

	return
}

func (g *s3Storage) Set(key string, val []byte) error {
	secret, err := g.cl.Core().Secrets(g.namespace).Get(g.secret, metav1.GetOptions{})

	b64val := make([]byte, base64.StdEncoding.EncodedLen(len(val)))
	base64.StdEncoding.Encode(b64val, val)

	if errors.IsNotFound(err) {
		secret := &v1.Secret{
			ObjectMeta: metav1.ObjectMeta{
				Namespace: g.namespace,
				Name:      g.secret,
			},
			Data: map[string][]byte{key: b64val},
		}
		secret, err = g.cl.Core().Secrets(g.namespace).Create(secret)
	} else if err == nil {
		secret.Data[key] = b64val
		secret, err = g.cl.Core().Secrets(g.namespace).Update(secret)
		//reflect.DeepEqual()
	} else {
		return fmt.Errorf("error checking if '%s' secret exists: '%s'", g.secret, err.Error())
	}

	if err != nil {
		return fmt.Errorf("error writing secret key '%s' into secret '%s': '%s'", key, g.secret, err.Error())
	}
	return nil
}

func (g *s3Storage) Get(key string) ([]byte, error) {
	secret, err := g.cl.Core().Secrets(g.namespace).Get(g.secret, metav1.GetOptions{})

	if err != nil {
		if errors.IsNotFound(err) {
			return nil, kv.NewNotFoundError("error getting secret for key '%s': %s", key, err.Error())
		}
		return nil, fmt.Errorf("error getting secret for key '%s': %s", key, err.Error())
	}

	val := secret.Data[key]
	if val == nil {
		return nil, kv.NewNotFoundError("key '%s' is not present in secret: %s", key)
	}

	return base64.StdEncoding.DecodeString(string(val))
}

func (g *s3Storage) Test(key string) error {
	// TODO: Implement me properly
	return nil
}
<commit_msg>remove double encoded secret<commit_after>package k8s

import (
	"fmt"
	"os"

	"github.com/banzaicloud/bank-vaults/pkg/kv"
	"k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/api/errors"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/client-go/kubernetes"
	"k8s.io/client-go/rest"
	"k8s.io/client-go/tools/clientcmd"
)

type s3Storage struct {
	cl        *kubernetes.Clientset
	namespace string
	secret    string
}

func New(namespace, secret string) (service kv.Service, err error) {
	kubeconfig := os.Getenv("KUBECONFIG")
	var config *rest.Config

	if kubeconfig != "" {
		config, err = clientcmd.BuildConfigFromFlags("", kubeconfig)
	} else {
		config, err = rest.InClusterConfig()
	}

	if err != nil {
		return nil, fmt.Errorf("error creating k8s config: %s", err.Error())
	}

	client, err := kubernetes.NewForConfig(config)
	if err != nil {
		return nil, fmt.Errorf("error creating k8s client: %s", err.Error())
	}

	service = &s3Storage{client, namespace, secret}

	return
}

func (g *s3Storage) Set(key string, val []byte) error {
	secret, err := g.cl.CoreV1().Secrets(g.namespace).Get(g.secret, metav1.GetOptions{})

	if errors.IsNotFound(err) {
		secret := &v1.Secret{
			ObjectMeta: metav1.ObjectMeta{
				Namespace: g.namespace,
				Name:      g.secret,
			},
			Data: map[string][]byte{key: val},
		}
		secret, err = g.cl.CoreV1().Secrets(g.namespace).Create(secret)
	} else if err == nil {
		secret.Data[key] = val
		secret, err = g.cl.CoreV1().Secrets(g.namespace).Update(secret)
		//reflect.DeepEqual()
	} else {
		return fmt.Errorf("error checking if '%s' secret exists: '%s'", g.secret, err.Error())
	}

	if err != nil {
		return fmt.Errorf("error writing secret key '%s' into secret '%s': '%s'", key, g.secret, err.Error())
	}
	return nil
}

func (g *s3Storage) Get(key string) ([]byte, error) {
	secret, err := g.cl.CoreV1().Secrets(g.namespace).Get(g.secret, metav1.GetOptions{})

	if err != nil {
		if errors.IsNotFound(err) {
			return nil, kv.NewNotFoundError("error getting secret for key '%s': %s", key, err.Error())
		}
		return nil, fmt.Errorf("error getting secret for key '%s': %s", key, err.Error())
	}

	val := secret.Data[key]
	if val == nil {
		return nil, kv.NewNotFoundError("key '%s' is not present in secret: %s", key)
	}

	return val, nil
}

func (g *s3Storage) Test(key string) error {
	// TODO: Implement me properly
	return nil
}
<|endoftext|>


<commit_before>package plugin

import (
	"crypto/rand"
	"encoding/json"
	"fmt"
	"net"
	"os"
	"strings"
	"syscall"

	"github.com/containernetworking/cni/pkg/ipam"
	"github.com/containernetworking/cni/pkg/skel"
	"github.com/containernetworking/cni/pkg/types"
	"github.com/containernetworking/cni/pkg/types/current"
	"github.com/vishvananda/netlink"
	"github.com/vishvananda/netns"
	weaveapi "github.com/weaveworks/weave/api"
	weavenet "github.com/weaveworks/weave/net"
	ipamplugin "github.com/weaveworks/weave/plugin/ipam"
)

var (
	zeroNetwork = net.IPNet{IP: net.IPv4zero, Mask: net.IPv4Mask(0, 0, 0, 0)}
	mask32      = net.IPv4Mask(0xff, 0xff, 0xff, 0xff)
)

type CNIPlugin struct {
	weave *weaveapi.Client
}

func NewCNIPlugin(weave *weaveapi.Client) *CNIPlugin {
	return &CNIPlugin{weave: weave}
}

func loadNetConf(bytes []byte) (*NetConf, error) {
	n := &NetConf{
		BrName: weavenet.WeaveBridgeName,
	}
	if err := json.Unmarshal(bytes, n); err != nil {
		return nil, fmt.Errorf("failed to load netconf: %v", err)
	}
	return n, nil
}

func (c *CNIPlugin) getIP(ipamType string, args *skel.CmdArgs) (newResult *current.Result, err error) {
	var result types.Result
	// Default IPAM is Weave's own
	if ipamType == "" {
		result, err = ipamplugin.NewIpam(c.weave).Allocate(args)
	} else {
		result, err = ipam.ExecAdd(ipamType, args.StdinData)
	}
	if err != nil {
		return nil, err
	}
	newResult, err = current.NewResultFromResult(result)
	// Check if ipam returned no results without error
	if err == nil && len(newResult.IPs) == 0 {
		return nil, fmt.Errorf("IPAM plugin failed to allocate IP address")
	}
	return newResult, err
}

func (c *CNIPlugin) CmdAdd(args *skel.CmdArgs) error {
	conf, err := loadNetConf(args.StdinData)
	if err != nil {
		return err
	}

	if conf.IsGW {
		return fmt.Errorf("Gateway functionality not supported")
	}
	if conf.IPMasq {
		return fmt.Errorf("IP Masquerading functionality not supported")
	}

	result, err := c.getIP(conf.IPAM.Type, args)
	if err != nil {
		return fmt.Errorf("unable to allocate IP address: %s", err)
	}
	// Only expecting one address
	ip := result.IPs[0]

	// If config says nothing about routes or gateway, default one will be via the bridge
	if len(result.Routes) == 0 && ip.Gateway == nil {
		bridgeIP, err := weavenet.FindBridgeIP(conf.BrName, &ip.Address)
		if err == weavenet.ErrBridgeNoIP {
			bridgeArgs := *args
			bridgeArgs.ContainerID = "weave:expose"
			// It would be better if libcni let us send just the desired parameters,
			// but there is a bug: https://github.com/containernetworking/cni/issues/410
			// so just blank out the one we want to change
			os.Setenv("CNI_CONTAINERID", bridgeArgs.ContainerID)
			bridgeIPResult, err := c.getIP(conf.IPAM.Type, &bridgeArgs)
			if err != nil {
				return fmt.Errorf("unable to allocate IP address for bridge: %s", err)
			}
			bridgeCIDR := bridgeIPResult.IPs[0].Address
			if err := assignBridgeIP(conf.BrName, bridgeCIDR); err != nil {
				return fmt.Errorf("unable to assign IP address to bridge: %s", err)
			}
			if err := weavenet.ExposeNAT(bridgeCIDR); err != nil {
				return fmt.Errorf("unable to create NAT rules: %s", err)
			}
			bridgeIP = bridgeCIDR.IP
		} else if err != nil {
			return err
		}
		result.IPs[0].Gateway = bridgeIP
	}

	ns, err := netns.GetFromPath(args.Netns)
	if err != nil {
		return fmt.Errorf("error accessing namespace %q: %s", args.Netns, err)
	}
	defer ns.Close()

	id := args.ContainerID
	if len(id) < 5 {
		data := make([]byte, 5)
		_, err := rand.Reader.Read(data)
		if err != nil {
			return err
		}
		id = fmt.Sprintf("%x", data)
	}

	if err := weavenet.AttachContainer(args.Netns, id, args.IfName, conf.BrName, conf.MTU, false, []*net.IPNet{&ip.Address}, false); err != nil {
		return err
	}
	if err := weavenet.WithNetNSLinkUnsafe(ns, args.IfName, func(link netlink.Link) error {
		return setupRoutes(link, args.IfName, ip.Address, ip.Gateway, result.Routes)
	}); err != nil {
		return fmt.Errorf("error setting up routes: %s", err)
	}

	result.DNS = conf.DNS
	return types.PrintResult(result, conf.CNIVersion)
}

func setupRoutes(link netlink.Link, name string, ipnet net.IPNet, gw net.IP, routes []*types.Route) error {
	var err error
	if routes == nil { // If config says nothing about routes, add a default one
		if !ipnet.Contains(gw) {
			// The bridge IP is not on the same subnet; add a specific route to it
			gw32 := &net.IPNet{IP: gw, Mask: mask32}
			if err = weavenet.AddRoute(link, netlink.SCOPE_LINK, gw32, nil); err != nil {
				return err
			}
		}
		routes = []*types.Route{{Dst: zeroNetwork}}
	}
	for _, r := range routes {
		if r.GW != nil {
			err = weavenet.AddRoute(link, netlink.SCOPE_UNIVERSE, &r.Dst, r.GW)
		} else {
			err = weavenet.AddRoute(link, netlink.SCOPE_UNIVERSE, &r.Dst, gw)
		}
		if err != nil {
			return fmt.Errorf("failed to add route '%v via %v dev %v': %v", r.Dst, gw, name, err)
		}
	}
	return nil
}

func assignBridgeIP(bridgeName string, ipnet net.IPNet) error {
	link, err := netlink.LinkByName(bridgeName)
	if err != nil {
		return err
	}
	if err := netlink.AddrAdd(link, &netlink.Addr{IPNet: &ipnet}); err != nil {
		// Treat as non-error if this address is already there
		// - maybe another copy of this program just added it
		if err == syscall.Errno(syscall.EEXIST) {
			return nil
		}
		return fmt.Errorf("failed to add IP address to %q: %v", bridgeName, err)
	}
	return nil
}

func (c *CNIPlugin) CmdDel(args *skel.CmdArgs) error {
	conf, err := loadNetConf(args.StdinData)
	if err != nil {
		return err
	}

	// As of CNI 0.3 spec, runtimes can send blank if they just want the address deallocated
	if args.Netns != "" {
		if _, err = weavenet.WithNetNS(args.Netns, "del-iface", args.IfName); err != nil {
			return fmt.Errorf("error removing interface %q: %s", args.IfName, err)
		}
	}

	// Default IPAM is Weave's own
	if conf.IPAM.Type == "" {
		err = ipamplugin.NewIpam(c.weave).Release(args)
	} else {
		err = ipam.ExecDel(conf.IPAM.Type, args.StdinData)
	}
	// Hack - don't know how we should detect this situation properly
	if args.Netns == "" && strings.Contains(err.Error(), "no addresses") {
		err = nil
	}
	if err != nil {
		return fmt.Errorf("unable to release IP address: %s", err)
	}
	return nil
}

type NetConf struct {
	types.NetConf
	BrName string `json:"bridge"`
	IsGW   bool   `json:"isGateway"`
	IPMasq bool   `json:"ipMasq"`
	MTU    int    `json:"mtu"`
}
<commit_msg>Extra checks on return value from IPAM plugin<commit_after>package plugin

import (
	"crypto/rand"
	"encoding/json"
	"fmt"
	"net"
	"os"
	"strings"
	"syscall"

	"github.com/containernetworking/cni/pkg/ipam"
	"github.com/containernetworking/cni/pkg/skel"
	"github.com/containernetworking/cni/pkg/types"
	"github.com/containernetworking/cni/pkg/types/current"
	"github.com/vishvananda/netlink"
	"github.com/vishvananda/netns"
	weaveapi "github.com/weaveworks/weave/api"
	weavenet "github.com/weaveworks/weave/net"
	ipamplugin "github.com/weaveworks/weave/plugin/ipam"
)

var (
	zeroNetwork = net.IPNet{IP: net.IPv4zero, Mask: net.IPv4Mask(0, 0, 0, 0)}
	mask32      = net.IPv4Mask(0xff, 0xff, 0xff, 0xff)
)

type CNIPlugin struct {
	weave *weaveapi.Client
}

func NewCNIPlugin(weave *weaveapi.Client) *CNIPlugin {
	return &CNIPlugin{weave: weave}
}

func loadNetConf(bytes []byte) (*NetConf, error) {
	n := &NetConf{
		BrName: weavenet.WeaveBridgeName,
	}
	if err := json.Unmarshal(bytes, n); err != nil {
		return nil, fmt.Errorf("failed to load netconf: %v", err)
	}
	return n, nil
}

func (c *CNIPlugin) getIP(ipamType string, args *skel.CmdArgs) (newResult *current.Result, err error) {
	var result types.Result
	// Default IPAM is Weave's own
	if ipamType == "" {
		result, err = ipamplugin.NewIpam(c.weave).Allocate(args)
	} else {
		result, err = ipam.ExecAdd(ipamType, args.StdinData)
	}
	if err != nil {
		return nil, err
	}
	if result == nil {
		return nil, fmt.Errorf("Received no usable result from IPAM plugin")
	}
	newResult, err = current.NewResultFromResult(result)
	// Check if ipam returned no results without error
	if err == nil && len(newResult.IPs) == 0 {
		return nil, fmt.Errorf("IPAM plugin failed to allocate IP address")
	}
	return newResult, err
}

func (c *CNIPlugin) CmdAdd(args *skel.CmdArgs) error {
	conf, err := loadNetConf(args.StdinData)
	if err != nil {
		return err
	}

	if conf.IsGW {
		return fmt.Errorf("Gateway functionality not supported")
	}
	if conf.IPMasq {
		return fmt.Errorf("IP Masquerading functionality not supported")
	}

	result, err := c.getIP(conf.IPAM.Type, args)
	if err != nil {
		return fmt.Errorf("unable to allocate IP address: %s", err)
	}
	// Only expecting one address
	ip := result.IPs[0]

	// If config says nothing about routes or gateway, default one will be via the bridge
	if len(result.Routes) == 0 && ip.Gateway == nil {
		bridgeIP, err := weavenet.FindBridgeIP(conf.BrName, &ip.Address)
		if err == weavenet.ErrBridgeNoIP {
			bridgeArgs := *args
			bridgeArgs.ContainerID = "weave:expose"
			// It would be better if libcni let us send just the desired parameters,
			// but there is a bug: https://github.com/containernetworking/cni/issues/410
			// so just blank out the one we want to change
			os.Setenv("CNI_CONTAINERID", bridgeArgs.ContainerID)
			bridgeIPResult, err := c.getIP(conf.IPAM.Type, &bridgeArgs)
			if err != nil {
				return fmt.Errorf("unable to allocate IP address for bridge: %s", err)
			}
			bridgeCIDR := bridgeIPResult.IPs[0].Address
			if err := assignBridgeIP(conf.BrName, bridgeCIDR); err != nil {
				return fmt.Errorf("unable to assign IP address to bridge: %s", err)
			}
			if err := weavenet.ExposeNAT(bridgeCIDR); err != nil {
				return fmt.Errorf("unable to create NAT rules: %s", err)
			}
			bridgeIP = bridgeCIDR.IP
		} else if err != nil {
			return err
		}
		result.IPs[0].Gateway = bridgeIP
	}

	ns, err := netns.GetFromPath(args.Netns)
	if err != nil {
		return fmt.Errorf("error accessing namespace %q: %s", args.Netns, err)
	}
	defer ns.Close()

	id := args.ContainerID
	if len(id) < 5 {
		data := make([]byte, 5)
		_, err := rand.Reader.Read(data)
		if err != nil {
			return err
		}
		id = fmt.Sprintf("%x", data)
	}

	if err := weavenet.AttachContainer(args.Netns, id, args.IfName, conf.BrName, conf.MTU, false, []*net.IPNet{&ip.Address}, false); err != nil {
		return err
	}
	if err := weavenet.WithNetNSLinkUnsafe(ns, args.IfName, func(link netlink.Link) error {
		return setupRoutes(link, args.IfName, ip.Address, ip.Gateway, result.Routes)
	}); err != nil {
		return fmt.Errorf("error setting up routes: %s", err)
	}

	result.DNS = conf.DNS
	return types.PrintResult(result, conf.CNIVersion)
}

func setupRoutes(link netlink.Link, name string, ipnet net.IPNet, gw net.IP, routes []*types.Route) error {
	var err error
	if routes == nil { // If config says nothing about routes, add a default one
		if !ipnet.Contains(gw) {
			// The bridge IP is not on the same subnet; add a specific route to it
			gw32 := &net.IPNet{IP: gw, Mask: mask32}
			if err = weavenet.AddRoute(link, netlink.SCOPE_LINK, gw32, nil); err != nil {
				return err
			}
		}
		routes = []*types.Route{{Dst: zeroNetwork}}
	}
	for _, r := range routes {
		if r.GW != nil {
			err = weavenet.AddRoute(link, netlink.SCOPE_UNIVERSE, &r.Dst, r.GW)
		} else {
			err = weavenet.AddRoute(link, netlink.SCOPE_UNIVERSE, &r.Dst, gw)
		}
		if err != nil {
			return fmt.Errorf("failed to add route '%v via %v dev %v': %v", r.Dst, gw, name, err)
		}
	}
	return nil
}

func assignBridgeIP(bridgeName string, ipnet net.IPNet) error {
	link, err := netlink.LinkByName(bridgeName)
	if err != nil {
		return err
	}
	if err := netlink.AddrAdd(link, &netlink.Addr{IPNet: &ipnet}); err != nil {
		// Treat as non-error if this address is already there
		// - maybe another copy of this program just added it
		if err == syscall.Errno(syscall.EEXIST) {
			return nil
		}
		return fmt.Errorf("failed to add IP address to %q: %v", bridgeName, err)
	}
	return nil
}

func (c *CNIPlugin) CmdDel(args *skel.CmdArgs) error {
	conf, err := loadNetConf(args.StdinData)
	if err != nil {
		return err
	}

	// As of CNI 0.3 spec, runtimes can send blank if they just want the address deallocated
	if args.Netns != "" {
		if _, err = weavenet.WithNetNS(args.Netns, "del-iface", args.IfName); err != nil {
			return fmt.Errorf("error removing interface %q: %s", args.IfName, err)
		}
	}

	// Default IPAM is Weave's own
	if conf.IPAM.Type == "" {
		err = ipamplugin.NewIpam(c.weave).Release(args)
	} else {
		err = ipam.ExecDel(conf.IPAM.Type, args.StdinData)
	}
	// Hack - don't know how we should detect this situation properly
	if args.Netns == "" && strings.Contains(err.Error(), "no addresses") {
		err = nil
	}
	if err != nil {
		return fmt.Errorf("unable to release IP address: %s", err)
	}
	return nil
}

type NetConf struct {
	types.NetConf
	BrName string `json:"bridge"`
	IsGW   bool   `json:"isGateway"`
	IPMasq bool   `json:"ipMasq"`
	MTU    int    `json:"mtu"`
}
<|endoftext|>


<commit_before>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"bufio"
	"bytes"
	"encoding/json"
	"errors"
	"flag"
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
	"os/exec"
	"path"
	"regexp"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/google/cadvisor/integration/common"

	cadvisorApi "github.com/google/cadvisor/info/v2"

	"github.com/golang/glog"
)

const cadvisorBinary = "cadvisor"

var cadvisorTimeout = flag.Duration("cadvisor_timeout", 15*time.Second, "Time to wait for cAdvisor to come up on the remote host")
var port = flag.Int("port", 8080, "Port in which to start cAdvisor in the remote host")
var testRetryCount = flag.Int("test-retry-count", 3, "Number of times to retry failed tests before failing.")
var testRetryWhitelist = flag.String("test-retry-whitelist", "", "Path to newline separated list of regexexp for test failures that should be retried.  If empty, no tests are retried.")
var retryRegex *regexp.Regexp

func getAttributes(ipAddress, portStr string) (*cadvisorApi.Attributes, error) {
	// Get host attributes and log attributes if the tests fail.
	var attributes cadvisorApi.Attributes
	resp, err := http.Get(fmt.Sprintf("http://%s:%s/api/v2.1/attributes", ipAddress, portStr))
	if err != nil {
		return nil, fmt.Errorf("failed to get attributes - %v", err)
	}
	if resp.StatusCode != http.StatusOK {
		return nil, fmt.Errorf("failed to get attributes. Status code - %v", resp.StatusCode)
	}
	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, fmt.Errorf("unable to read attributes response body - %v", err)
	}
	if err := json.Unmarshal(body, &attributes); err != nil {
		return nil, fmt.Errorf("failed to unmarshal attributes - %v", err)
	}
	return &attributes, nil
}

func RunCommand(cmd string, args ...string) error {
	output, err := exec.Command(cmd, args...).CombinedOutput()
	if err != nil {
		return fmt.Errorf("command %q %q failed with error: %v and output: %q", cmd, args, err, output)
	}

	return nil
}

func PushAndRunTests(host, testDir string) error {
	// Push binary.
	glog.Infof("Pushing cAdvisor binary to %q...", host)
	args := common.GetGCComputeArgs("ssh", host, "--", "mkdir", "-p", testDir)
	err := RunCommand("gcloud", args...)
	if err != nil {
		return fmt.Errorf("failed to make remote testing directory: %v", err)
	}
	defer func() {
		args := common.GetGCComputeArgs("ssh", host, "--", "rm", "-rf", testDir)
		err := RunCommand("gcloud", args...)
		if err != nil {
			glog.Errorf("Failed to cleanup test directory: %v", err)
		}
	}()
	args = common.GetGCComputeArgs("copy-files", cadvisorBinary, fmt.Sprintf("%s:%s", host, testDir))
	err = RunCommand("gcloud", args...)
	if err != nil {
		return fmt.Errorf("failed to copy binary: %v", err)
	}

	// Start cAdvisor.
	glog.Infof("Running cAdvisor on %q...", host)
	portStr := strconv.Itoa(*port)
	errChan := make(chan error)
	go func() {
		args = common.GetGCComputeArgs("ssh", host, "--", fmt.Sprintf("sudo %s --port %s --logtostderr &> %s/log.txt", path.Join(testDir, cadvisorBinary), portStr, testDir))
		err = RunCommand("gcloud", args...)
		if err != nil {
			errChan <- fmt.Errorf("error running cAdvisor: %v", err)
		}
	}()
	defer func() {
		args = common.GetGCComputeArgs("ssh", host, "--", "sudo", "pkill", cadvisorBinary)
		err := RunCommand("gcloud", args...)
		if err != nil {
			glog.Errorf("Failed to cleanup: %v", err)
		}
	}()

	ipAddress, err := common.GetGceIp(host)
	if err != nil {
		return fmt.Errorf("failed to get GCE IP: %v", err)
	}

	// Wait for cAdvisor to come up.
	endTime := time.Now().Add(*cadvisorTimeout)
	done := false
	for endTime.After(time.Now()) && !done {
		select {
		case err := <-errChan:
			// Quit early if there was an error.
			return err
		case <-time.After(500 * time.Millisecond):
			// Stop waiting when cAdvisor is healthy..
			resp, err := http.Get(fmt.Sprintf("http://%s:%s/healthz", ipAddress, portStr))
			if err == nil && resp.StatusCode == http.StatusOK {
				done = true
				break
			}
		}
	}
	if !done {
		return fmt.Errorf("timed out waiting for cAdvisor to come up at host %q", host)
	}

	// Get attributes for debugging purposes.
	attributes, err := getAttributes(ipAddress, portStr)
	if err != nil {
		return fmt.Errorf("%v - %q", err, host)
	}
	// Run the tests in a retry loop.
	glog.Infof("Running integration tests targeting %q...", host)
	for i := 0; i <= *testRetryCount; i++ {
		// Check if this is a retry
		if i > 0 {
			time.Sleep(time.Second * 15) // Wait 15 seconds before retrying
			glog.Warningf("Retrying (%d of %d) tests on host %s due to error %v", i, *testRetryCount, host, err)
		}
		// Run the command
		err = RunCommand("godep", "go", "test", "github.com/google/cadvisor/integration/tests/...", "--host", host, "--port", portStr)
		if err == nil {
			// On success, break out of retry loop
			break
		}

		// Only retry on test failures caused by these known flaky failure conditions
		if retryRegex == nil || !retryRegex.Match([]byte(err.Error())) {
			glog.Warningf("Skipping retry for tests on host %s because error is not whitelisted: %s", host, err.Error())
			break
		}
	}
	if err != nil {
		// Copy logs from the host
		args = common.GetGCComputeArgs("copy-files", fmt.Sprintf("%s:%s/log.txt", host, testDir), "./")
		// Declare new error or it will get shadowed by logs, err := <>  and we won't be able to unset it from nil
		err2 := RunCommand("gcloud", args...)
		if err2 != nil {
			return fmt.Errorf("error fetching logs: %v for %v", err2, err)
		}
		defer os.Remove("./log.txt")
		logs, err2 := ioutil.ReadFile("./log.txt")
		if err2 != nil {
			return fmt.Errorf("error reading local log file: %v for %v", err2, err)
		}
		glog.Errorf("----------------------\nLogs from Host: %q\n%v\n", host, string(logs))
		err = fmt.Errorf("error on host %s: %v\n%+v", host, err, attributes)
	}
	return err
}

func Run() error {
	start := time.Now()
	defer func() {
		glog.Infof("Execution time %v", time.Since(start))
	}()
	defer glog.Flush()

	hosts := flag.Args()
	testDir := fmt.Sprintf("/tmp/cadvisor-%d", os.Getpid())
	glog.Infof("Running integration tests on host(s) %q", strings.Join(hosts, ","))

	// Build cAdvisor.
	glog.Infof("Building cAdvisor...")
	err := RunCommand("godep", "go", "build", "github.com/google/cadvisor")
	if err != nil {
		return err
	}
	defer func() {
		err := RunCommand("rm", cadvisorBinary)
		if err != nil {
			glog.Error(err)
		}
	}()

	// Run test on all hosts in parallel.
	var wg sync.WaitGroup
	allErrors := make([]error, 0)
	var allErrorsLock sync.Mutex
	for _, host := range hosts {
		wg.Add(1)
		go func(host string) {
			defer wg.Done()
			err := PushAndRunTests(host, testDir)
			if err != nil {
				func() {
					allErrorsLock.Lock()
					defer allErrorsLock.Unlock()
					allErrors = append(allErrors, err)
				}()
			}
		}(host)
	}
	wg.Wait()

	if len(allErrors) != 0 {
		var buffer bytes.Buffer
		for i, err := range allErrors {
			buffer.WriteString(fmt.Sprintf("Error %d: ", i))
			buffer.WriteString(err.Error())
			buffer.WriteString("\n")
		}
		return errors.New(buffer.String())
	}

	glog.Infof("All tests pass!")
	return nil
}

// initRetryWhitelist initializes the whitelist of test failures that can be retried.
func initRetryWhitelist() {
	if *testRetryWhitelist == "" {
		return
	}

	file, err := os.Open(*testRetryWhitelist)
	if err != nil {
		glog.Fatal(err)
	}
	defer file.Close()

	retryStrings := []string{}
	scanner := bufio.NewScanner(file)
	for scanner.Scan() {
		text := scanner.Text()
		if text != "" {
			retryStrings = append(retryStrings, text)
		}
	}
	if err := scanner.Err(); err != nil {
		glog.Fatal(err)
	}
	retryRegex = regexp.MustCompile(strings.Join(retryStrings, "|"))
}

func main() {
	flag.Parse()

	// Check usage.
	if len(flag.Args()) == 0 {
		glog.Fatalf("USAGE: runner <hosts to test>")
	}
	initRetryWhitelist()

	// Run the tests.
	err := Run()
	if err != nil {
		glog.Fatal(err)
	}
}
<commit_msg>Add ssh-options flag so it can be tested by the pr builder in followup #1108<commit_after>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"bufio"
	"bytes"
	"encoding/json"
	"errors"
	"flag"
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
	"os/exec"
	"path"
	"regexp"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/google/cadvisor/integration/common"

	cadvisorApi "github.com/google/cadvisor/info/v2"

	"github.com/golang/glog"
)

const cadvisorBinary = "cadvisor"

var cadvisorTimeout = flag.Duration("cadvisor_timeout", 15*time.Second, "Time to wait for cAdvisor to come up on the remote host")
var port = flag.Int("port", 8080, "Port in which to start cAdvisor in the remote host")
var testRetryCount = flag.Int("test-retry-count", 3, "Number of times to retry failed tests before failing.")
var testRetryWhitelist = flag.String("test-retry-whitelist", "", "Path to newline separated list of regexexp for test failures that should be retried.  If empty, no tests are retried.")
var sshOptions = flag.String("ssh-options", "", "Commandline options passed to ssh.") // Used in a follow up
var retryRegex *regexp.Regexp

func getAttributes(ipAddress, portStr string) (*cadvisorApi.Attributes, error) {
	// Get host attributes and log attributes if the tests fail.
	var attributes cadvisorApi.Attributes
	resp, err := http.Get(fmt.Sprintf("http://%s:%s/api/v2.1/attributes", ipAddress, portStr))
	if err != nil {
		return nil, fmt.Errorf("failed to get attributes - %v", err)
	}
	if resp.StatusCode != http.StatusOK {
		return nil, fmt.Errorf("failed to get attributes. Status code - %v", resp.StatusCode)
	}
	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, fmt.Errorf("unable to read attributes response body - %v", err)
	}
	if err := json.Unmarshal(body, &attributes); err != nil {
		return nil, fmt.Errorf("failed to unmarshal attributes - %v", err)
	}
	return &attributes, nil
}

func RunCommand(cmd string, args ...string) error {
	output, err := exec.Command(cmd, args...).CombinedOutput()
	if err != nil {
		return fmt.Errorf("command %q %q failed with error: %v and output: %q", cmd, args, err, output)
	}

	return nil
}

func PushAndRunTests(host, testDir string) error {
	// Push binary.
	glog.Infof("Pushing cAdvisor binary to %q...", host)
	args := common.GetGCComputeArgs("ssh", host, "--", "mkdir", "-p", testDir)
	err := RunCommand("gcloud", args...)
	if err != nil {
		return fmt.Errorf("failed to make remote testing directory: %v", err)
	}
	defer func() {
		args := common.GetGCComputeArgs("ssh", host, "--", "rm", "-rf", testDir)
		err := RunCommand("gcloud", args...)
		if err != nil {
			glog.Errorf("Failed to cleanup test directory: %v", err)
		}
	}()
	args = common.GetGCComputeArgs("copy-files", cadvisorBinary, fmt.Sprintf("%s:%s", host, testDir))
	err = RunCommand("gcloud", args...)
	if err != nil {
		return fmt.Errorf("failed to copy binary: %v", err)
	}

	// Start cAdvisor.
	glog.Infof("Running cAdvisor on %q...", host)
	portStr := strconv.Itoa(*port)
	errChan := make(chan error)
	go func() {
		args = common.GetGCComputeArgs("ssh", host, "--", fmt.Sprintf("sudo %s --port %s --logtostderr &> %s/log.txt", path.Join(testDir, cadvisorBinary), portStr, testDir))
		err = RunCommand("gcloud", args...)
		if err != nil {
			errChan <- fmt.Errorf("error running cAdvisor: %v", err)
		}
	}()
	defer func() {
		args = common.GetGCComputeArgs("ssh", host, "--", "sudo", "pkill", cadvisorBinary)
		err := RunCommand("gcloud", args...)
		if err != nil {
			glog.Errorf("Failed to cleanup: %v", err)
		}
	}()

	ipAddress, err := common.GetGceIp(host)
	if err != nil {
		return fmt.Errorf("failed to get GCE IP: %v", err)
	}

	// Wait for cAdvisor to come up.
	endTime := time.Now().Add(*cadvisorTimeout)
	done := false
	for endTime.After(time.Now()) && !done {
		select {
		case err := <-errChan:
			// Quit early if there was an error.
			return err
		case <-time.After(500 * time.Millisecond):
			// Stop waiting when cAdvisor is healthy..
			resp, err := http.Get(fmt.Sprintf("http://%s:%s/healthz", ipAddress, portStr))
			if err == nil && resp.StatusCode == http.StatusOK {
				done = true
				break
			}
		}
	}
	if !done {
		return fmt.Errorf("timed out waiting for cAdvisor to come up at host %q", host)
	}

	// Get attributes for debugging purposes.
	attributes, err := getAttributes(ipAddress, portStr)
	if err != nil {
		return fmt.Errorf("%v - %q", err, host)
	}
	// Run the tests in a retry loop.
	glog.Infof("Running integration tests targeting %q...", host)
	for i := 0; i <= *testRetryCount; i++ {
		// Check if this is a retry
		if i > 0 {
			time.Sleep(time.Second * 15) // Wait 15 seconds before retrying
			glog.Warningf("Retrying (%d of %d) tests on host %s due to error %v", i, *testRetryCount, host, err)
		}
		// Run the command
		err = RunCommand("godep", "go", "test", "github.com/google/cadvisor/integration/tests/...", "--host", host, "--port", portStr)
		if err == nil {
			// On success, break out of retry loop
			break
		}

		// Only retry on test failures caused by these known flaky failure conditions
		if retryRegex == nil || !retryRegex.Match([]byte(err.Error())) {
			glog.Warningf("Skipping retry for tests on host %s because error is not whitelisted: %s", host, err.Error())
			break
		}
	}
	if err != nil {
		// Copy logs from the host
		args = common.GetGCComputeArgs("copy-files", fmt.Sprintf("%s:%s/log.txt", host, testDir), "./")
		// Declare new error or it will get shadowed by logs, err := <>  and we won't be able to unset it from nil
		err2 := RunCommand("gcloud", args...)
		if err2 != nil {
			return fmt.Errorf("error fetching logs: %v for %v", err2, err)
		}
		defer os.Remove("./log.txt")
		logs, err2 := ioutil.ReadFile("./log.txt")
		if err2 != nil {
			return fmt.Errorf("error reading local log file: %v for %v", err2, err)
		}
		glog.Errorf("----------------------\nLogs from Host: %q\n%v\n", host, string(logs))
		err = fmt.Errorf("error on host %s: %v\n%+v", host, err, attributes)
	}
	return err
}

func Run() error {
	start := time.Now()
	defer func() {
		glog.Infof("Execution time %v", time.Since(start))
	}()
	defer glog.Flush()

	hosts := flag.Args()
	testDir := fmt.Sprintf("/tmp/cadvisor-%d", os.Getpid())
	glog.Infof("Running integration tests on host(s) %q", strings.Join(hosts, ","))

	// Build cAdvisor.
	glog.Infof("Building cAdvisor...")
	err := RunCommand("godep", "go", "build", "github.com/google/cadvisor")
	if err != nil {
		return err
	}
	defer func() {
		err := RunCommand("rm", cadvisorBinary)
		if err != nil {
			glog.Error(err)
		}
	}()

	// Run test on all hosts in parallel.
	var wg sync.WaitGroup
	allErrors := make([]error, 0)
	var allErrorsLock sync.Mutex
	for _, host := range hosts {
		wg.Add(1)
		go func(host string) {
			defer wg.Done()
			err := PushAndRunTests(host, testDir)
			if err != nil {
				func() {
					allErrorsLock.Lock()
					defer allErrorsLock.Unlock()
					allErrors = append(allErrors, err)
				}()
			}
		}(host)
	}
	wg.Wait()

	if len(allErrors) != 0 {
		var buffer bytes.Buffer
		for i, err := range allErrors {
			buffer.WriteString(fmt.Sprintf("Error %d: ", i))
			buffer.WriteString(err.Error())
			buffer.WriteString("\n")
		}
		return errors.New(buffer.String())
	}

	glog.Infof("All tests pass!")
	return nil
}

// initRetryWhitelist initializes the whitelist of test failures that can be retried.
func initRetryWhitelist() {
	if *testRetryWhitelist == "" {
		return
	}

	file, err := os.Open(*testRetryWhitelist)
	if err != nil {
		glog.Fatal(err)
	}
	defer file.Close()

	retryStrings := []string{}
	scanner := bufio.NewScanner(file)
	for scanner.Scan() {
		text := scanner.Text()
		if text != "" {
			retryStrings = append(retryStrings, text)
		}
	}
	if err := scanner.Err(); err != nil {
		glog.Fatal(err)
	}
	retryRegex = regexp.MustCompile(strings.Join(retryStrings, "|"))
}

func main() {
	flag.Parse()

	// Check usage.
	if len(flag.Args()) == 0 {
		glog.Fatalf("USAGE: runner <hosts to test>")
	}
	initRetryWhitelist()

	// Run the tests.
	err := Run()
	if err != nil {
		glog.Fatal(err)
	}
}
<|endoftext|>


<commit_before>package epub

import (
	"encoding/xml"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"time"
)

const (
	// TODO
	temp = `<?xml version="1.0" encoding="UTF-8"?>
<package version="3.0" unique-identifier="pub-id" xmlns="http://www.idpf.org/2007/opf">
  <metadata xmlns:dc="http://purl.org/dc/elements/1.1/">
    <dc:identifier id="pub-id">urn:uuid:fe93046f-af57-475a-a0cb-a0d4bc99ba6d</dc:identifier>
    <dc:title>Your title here</dc:title>
    <dc:language>en</dc:language>
    <meta refines="#creator" property="role" scheme="marc:relators" id="role">aut</meta>
    <meta property="dcterms:modified">2011-01-01T12:00:00Z</meta>
  </metadata>
  <manifest>
    <item id="nav" href="nav.xhtml" media-type="application/xhtml+xml" properties="nav" />
    <item id="ncx" href="toc.ncx" media-type="application/x-dtbncx+xml" />
    <item id="section0001.xhtml" href="xhtml/section0001.xhtml" media-type="application/xhtml+xml" />
  </manifest>
  <spine toc="ncx">
    <itemref idref="section0001.xhtml" />
  </spine>
</package>
`
	pkgAuthorId       = "role"
	pkgAuthorData     = "aut"
	pkgAuthorProperty = "role"
	pkgAuthorRefines  = "#creator"
	pkgAuthorScheme   = "marc:relators"
	pkgCreatorId      = "creator"
	pkgFileTemplate   = `<?xml version="1.0" encoding="UTF-8"?>
<package version="3.0" unique-identifier="pub-id" xmlns="http://www.idpf.org/2007/opf">
  <metadata xmlns:dc="http://purl.org/dc/elements/1.1/">
    <dc:identifier id="pub-id"></dc:identifier>
    <dc:title></dc:title>
    <dc:language></dc:language>
  </metadata>
  <manifest>
  </manifest>
  <spine toc="ncx">
  </spine>
</package>
`
	pkgModifiedProperty = "dcterms:modified"
	pkgUniqueIdentifier = "pub-id"

	xmlnsDc = "http://purl.org/dc/elements/1.1/"
)

type pkg struct {
	xml          *pkgRoot
	authorMeta   *pkgMeta
	modifiedMeta *pkgMeta
}

type pkgRoot struct {
	XMLName          xml.Name    `xml:"http://www.idpf.org/2007/opf package"`
	UniqueIdentifier string      `xml:"unique-identifier,attr"`
	Version          string      `xml:"version,attr"`
	Metadata         pkgMetadata `xml:"metadata"`
	Item             []pkgItem   `xml:"manifest>item"`
	Spine            pkgSpine    `xml:"spine"`
}

type pkgCreator struct {
	XMLName xml.Name `xml:"dc:creator"`
	Id      string   `xml:"id,attr"`
	Data    string   `xml:",chardata"`
}

type pkgIdentifier struct {
	Id   string `xml:"id,attr"`
	Data string `xml:",chardata"`
}

type pkgItem struct {
	Href       string `xml:"href,attr"`
	Id         string `xml:"id,attr"`
	MediaType  string `xml:"media-type,attr"`
	Properties string `xml:"properties,attr"`
}

type pkgItemref struct {
	Idref string `xml:"idref,attr"`
}

type pkgMeta struct {
	Refines  string `xml:"refines,attr,omitempty"`
	Property string `xml:"property,attr"`
	Scheme   string `xml:"scheme,attr,omitempty"`
	Id       string `xml:"id,attr,omitempty"`
	Data     string `xml:",chardata"`
}

type pkgMetadata struct {
	XmlnsDc    string        `xml:"xmlns:dc,attr"`
	Identifier pkgIdentifier `xml:"dc:identifier"`
	Title      string        `xml:"dc:title"`
	Language   string        `xml:"dc:language"`
	Creator    *pkgCreator
	Meta       []pkgMeta `xml:"meta"`
}

type pkgSpine struct {
	Itemref []pkgItemref `xml:"itemref"`
}

func newPackage() *pkg {
	p := &pkg{
		xml: &pkgRoot{
			Metadata: pkgMetadata{
				XmlnsDc: xmlnsDc,
				Identifier: pkgIdentifier{
					Id: pkgUniqueIdentifier,
				},
			},
		},
	}

	err := xml.Unmarshal([]byte(pkgFileTemplate), &p.xml)
	if err != nil {
		log.Fatalf("xml.Unmarshal error: %s", err)
	}

	return p
}

func (p *pkg) setAuthor(author string) {
	p.xml.Metadata.Creator = &pkgCreator{
		Data: author,
		Id:   pkgCreatorId,
	}
	p.authorMeta = &pkgMeta{
		Data:     pkgAuthorData,
		Id:       pkgAuthorId,
		Property: pkgAuthorProperty,
		Refines:  pkgAuthorRefines,
		Scheme:   pkgAuthorScheme,
	}
}

func (p *pkg) setLang(lang string) {
	p.xml.Metadata.Language = lang
}

func (p *pkg) setModified(timestamp string) {
	var indexToReplace int

	p.modifiedMeta = &pkgMeta{
		Data:     timestamp,
		Property: pkgModifiedProperty,
	}

	if len(p.xml.Metadata.Meta) > 0 {
		// If we've already added the modified meta element to the meta array
		for i, meta := range p.xml.Metadata.Meta {
			if &meta == p.modifiedMeta {
				indexToReplace = i
				break
			}
		}
		// Replace it
		p.xml.Metadata.Meta[indexToReplace] = *p.modifiedMeta
	} else {
		p.xml.Metadata.Meta = append(p.xml.Metadata.Meta, *p.modifiedMeta)
	}
}

func (p *pkg) setTitle(title string) {
	p.xml.Metadata.Title = title
}

func (p *pkg) setUUID(uuid string) {
	p.xml.Metadata.Identifier.Data = uuid
}

func (p *pkg) write(tempDir string) error {
	now := time.Now().UTC().Format("2006-01-02T15:04:05Z")
	p.setModified(now)

	if p.xml.Metadata.Creator != nil {
		p.xml.Metadata.Meta = append(p.xml.Metadata.Meta, *p.authorMeta)
	}

	contentFolderPath := filepath.Join(tempDir, contentFolderName)
	if err := os.Mkdir(contentFolderPath, dirPermissions); err != nil {
		return err
	}

	pkgFilePath := filepath.Join(contentFolderPath, pkgFilename)

	output, err := xml.MarshalIndent(p.xml, "", "  ")
	if err != nil {
		return err
	}
	// Add the xml header to the output
	pkgFileContent := append([]byte(xml.Header), output...)
	// It's generally nice to have files end with a newline
	pkgFileContent = append(pkgFileContent, "\n"...)

	if err := ioutil.WriteFile(pkgFilePath, []byte(pkgFileContent), filePermissions); err != nil {
		return err
	}

	return nil
}
<commit_msg>Replace author meta element also<commit_after>package epub

import (
	"encoding/xml"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"time"
)

const (
	// TODO
	temp = `<?xml version="1.0" encoding="UTF-8"?>
<package version="3.0" unique-identifier="pub-id" xmlns="http://www.idpf.org/2007/opf">
  <metadata xmlns:dc="http://purl.org/dc/elements/1.1/">
    <dc:identifier id="pub-id">urn:uuid:fe93046f-af57-475a-a0cb-a0d4bc99ba6d</dc:identifier>
    <dc:title>Your title here</dc:title>
    <dc:language>en</dc:language>
    <meta refines="#creator" property="role" scheme="marc:relators" id="role">aut</meta>
    <meta property="dcterms:modified">2011-01-01T12:00:00Z</meta>
  </metadata>
  <manifest>
    <item id="nav" href="nav.xhtml" media-type="application/xhtml+xml" properties="nav" />
    <item id="ncx" href="toc.ncx" media-type="application/x-dtbncx+xml" />
    <item id="section0001.xhtml" href="xhtml/section0001.xhtml" media-type="application/xhtml+xml" />
  </manifest>
  <spine toc="ncx">
    <itemref idref="section0001.xhtml" />
  </spine>
</package>
`
	pkgAuthorId       = "role"
	pkgAuthorData     = "aut"
	pkgAuthorProperty = "role"
	pkgAuthorRefines  = "#creator"
	pkgAuthorScheme   = "marc:relators"
	pkgCreatorId      = "creator"
	pkgFileTemplate   = `<?xml version="1.0" encoding="UTF-8"?>
<package version="3.0" unique-identifier="pub-id" xmlns="http://www.idpf.org/2007/opf">
  <metadata xmlns:dc="http://purl.org/dc/elements/1.1/">
    <dc:identifier id="pub-id"></dc:identifier>
    <dc:title></dc:title>
    <dc:language></dc:language>
  </metadata>
  <manifest>
  </manifest>
  <spine toc="ncx">
  </spine>
</package>
`
	pkgModifiedProperty = "dcterms:modified"
	pkgUniqueIdentifier = "pub-id"

	xmlnsDc = "http://purl.org/dc/elements/1.1/"
)

type pkg struct {
	xml          *pkgRoot
	authorMeta   *pkgMeta
	modifiedMeta *pkgMeta
}

type pkgRoot struct {
	XMLName          xml.Name    `xml:"http://www.idpf.org/2007/opf package"`
	UniqueIdentifier string      `xml:"unique-identifier,attr"`
	Version          string      `xml:"version,attr"`
	Metadata         pkgMetadata `xml:"metadata"`
	Item             []pkgItem   `xml:"manifest>item"`
	Spine            pkgSpine    `xml:"spine"`
}

type pkgCreator struct {
	XMLName xml.Name `xml:"dc:creator"`
	Id      string   `xml:"id,attr"`
	Data    string   `xml:",chardata"`
}

type pkgIdentifier struct {
	Id   string `xml:"id,attr"`
	Data string `xml:",chardata"`
}

type pkgItem struct {
	Href       string `xml:"href,attr"`
	Id         string `xml:"id,attr"`
	MediaType  string `xml:"media-type,attr"`
	Properties string `xml:"properties,attr"`
}

type pkgItemref struct {
	Idref string `xml:"idref,attr"`
}

type pkgMeta struct {
	Refines  string `xml:"refines,attr,omitempty"`
	Property string `xml:"property,attr"`
	Scheme   string `xml:"scheme,attr,omitempty"`
	Id       string `xml:"id,attr,omitempty"`
	Data     string `xml:",chardata"`
}

type pkgMetadata struct {
	XmlnsDc    string        `xml:"xmlns:dc,attr"`
	Identifier pkgIdentifier `xml:"dc:identifier"`
	Title      string        `xml:"dc:title"`
	Language   string        `xml:"dc:language"`
	Creator    *pkgCreator
	Meta       []pkgMeta `xml:"meta"`
}

type pkgSpine struct {
	Itemref []pkgItemref `xml:"itemref"`
}

func newPackage() *pkg {
	p := &pkg{
		xml: &pkgRoot{
			Metadata: pkgMetadata{
				XmlnsDc: xmlnsDc,
				Identifier: pkgIdentifier{
					Id: pkgUniqueIdentifier,
				},
			},
		},
	}

	err := xml.Unmarshal([]byte(pkgFileTemplate), &p.xml)
	if err != nil {
		log.Fatalf("xml.Unmarshal error: %s", err)
	}

	return p
}

func replaceOrAppendMeta(a []pkgMeta, m *pkgMeta) []pkgMeta {
	indexToReplace := -1

	if len(a) > 0 {
		// If we've already added the modified meta element to the meta array
		for i, meta := range a {
			if meta == *m {
				indexToReplace = i
				break
			}
		}
	}

	// If the array is empty or the meta element isn't in it
	if indexToReplace == -1 {
		// Add the meta element to the array of meta elements
		a = append(a, *m)

		// If the meta element is found
	} else {
		// Replace it
		a[indexToReplace] = *m
	}

	return a
}

func (p *pkg) setAuthor(author string) {
	p.xml.Metadata.Creator = &pkgCreator{
		Data: author,
		Id:   pkgCreatorId,
	}
	p.authorMeta = &pkgMeta{
		Data:     pkgAuthorData,
		Id:       pkgAuthorId,
		Property: pkgAuthorProperty,
		Refines:  pkgAuthorRefines,
		Scheme:   pkgAuthorScheme,
	}

	p.xml.Metadata.Meta = replaceOrAppendMeta(p.xml.Metadata.Meta, p.authorMeta)
}

func (p *pkg) setLang(lang string) {
	p.xml.Metadata.Language = lang
}

func (p *pkg) setModified(timestamp string) {
	//	var indexToReplace int

	p.modifiedMeta = &pkgMeta{
		Data:     timestamp,
		Property: pkgModifiedProperty,
	}

	p.xml.Metadata.Meta = replaceOrAppendMeta(p.xml.Metadata.Meta, p.modifiedMeta)
}

func (p *pkg) setTitle(title string) {
	p.xml.Metadata.Title = title
}

func (p *pkg) setUUID(uuid string) {
	p.xml.Metadata.Identifier.Data = uuid
}

func (p *pkg) write(tempDir string) error {
	now := time.Now().UTC().Format("2006-01-02T15:04:05Z")
	p.setModified(now)

	contentFolderPath := filepath.Join(tempDir, contentFolderName)
	if err := os.Mkdir(contentFolderPath, dirPermissions); err != nil {
		return err
	}

	pkgFilePath := filepath.Join(contentFolderPath, pkgFilename)

	output, err := xml.MarshalIndent(p.xml, "", "  ")
	if err != nil {
		return err
	}
	// Add the xml header to the output
	pkgFileContent := append([]byte(xml.Header), output...)
	// It's generally nice to have files end with a newline
	pkgFileContent = append(pkgFileContent, "\n"...)

	if err := ioutil.WriteFile(pkgFilePath, []byte(pkgFileContent), filePermissions); err != nil {
		return err
	}

	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"image"
	"image/png"
	"net/http"
	"strconv"
	"time"

	pb "github.com/dgryski/carbonzipper/carbonzipperpb"

	"code.google.com/p/plotinum/plot"
	"code.google.com/p/plotinum/plotter"
	"code.google.com/p/plotinum/plotutil"
	"code.google.com/p/plotinum/vg/vgimg"
)

var linesColors = `blue,green,red,purple,brown,yellow,aqua,grey,magenta,pink,gold,rose`

func marshalPNG(r *http.Request, results []*pb.FetchResponse) []byte {

	p, err := plot.New()
	if err != nil {
		panic(err)
	}

	// need different timeMarker's based on step size
	p.Title.Text = r.FormValue("title")
	p.X.Tick.Marker = timeMarker

	p.Add(plotter.NewGrid())

	var lines []plot.Plotter
	for i, r := range results {

		t := resultXYs(r)

		l, _ := plotter.NewLine(t)
		l.Color = plotutil.Color(i)

		lines = append(lines, l)
	}
	p.Add(lines...)

	height := getInt(r.FormValue("height"), 250)
	width := getInt(r.FormValue("width"), 330)

	// Draw the plot to an in-memory image.
	img := image.NewRGBA(image.Rect(0, 0, width, height))
	da := plot.MakeDrawArea(vgimg.NewImage(img))
	p.Draw(da)

	var b bytes.Buffer
	if err := png.Encode(&b, img); err != nil {
		panic(err)
	}

	return b.Bytes()
}

func timeMarker(min, max float64) []plot.Tick {
	ticks := plot.DefaultTicks(min, max)

	for i, t := range ticks {
		if !t.IsMinor() {
			t0 := time.Unix(int64(t.Value), 0)
			ticks[i].Label = t0.Format("15:04:05")
		}
	}

	return ticks
}

type xy struct {
	X, Y float64
}

func resultXYs(r *pb.FetchResponse) plotter.XYs {
	pts := make(plotter.XYs, 0, len(r.GetValues()))
	start := float64(r.GetStartTime())
	step := float64(r.GetStepTime())
	absent := r.GetIsAbsent()
	for i, v := range r.GetValues() {
		if absent[i] {
			continue
		}
		pts = append(pts, xy{start + float64(i)*step, v})
	}
	return pts
}

func getInt(s string, def int) int {

	if s == "" {
		return def
	}

	n, err := strconv.Atoi(s)
	if err != nil {
		return def
	}

	return n

}
<commit_msg>add extra padding on Y-axis<commit_after>package main

import (
	"bytes"
	"image"
	"image/png"
	"net/http"
	"strconv"
	"time"

	pb "github.com/dgryski/carbonzipper/carbonzipperpb"

	"code.google.com/p/plotinum/plot"
	"code.google.com/p/plotinum/plotter"
	"code.google.com/p/plotinum/plotutil"
	"code.google.com/p/plotinum/vg/vgimg"
)

var linesColors = `blue,green,red,purple,brown,yellow,aqua,grey,magenta,pink,gold,rose`

func marshalPNG(r *http.Request, results []*pb.FetchResponse) []byte {

	p, err := plot.New()
	if err != nil {
		panic(err)
	}

	// need different timeMarker's based on step size
	p.Title.Text = r.FormValue("title")
	p.X.Tick.Marker = timeMarker

	p.Add(plotter.NewGrid())

	var lines []plot.Plotter
	for i, r := range results {

		t := resultXYs(r)

		l, _ := plotter.NewLine(t)
		l.Color = plotutil.Color(i)

		lines = append(lines, l)
	}
	p.Add(lines...)

	height := getInt(r.FormValue("height"), 250)
	width := getInt(r.FormValue("width"), 330)

	p.Y.Max *= 1.05
	p.Y.Min *= 0.95

	// Draw the plot to an in-memory image.
	img := image.NewRGBA(image.Rect(0, 0, width, height))
	da := plot.MakeDrawArea(vgimg.NewImage(img))
	p.Draw(da)

	var b bytes.Buffer
	if err := png.Encode(&b, img); err != nil {
		panic(err)
	}

	return b.Bytes()
}

func timeMarker(min, max float64) []plot.Tick {
	ticks := plot.DefaultTicks(min, max)

	for i, t := range ticks {
		if !t.IsMinor() {
			t0 := time.Unix(int64(t.Value), 0)
			ticks[i].Label = t0.Format("15:04:05")
		}
	}

	return ticks
}

type xy struct {
	X, Y float64
}

func resultXYs(r *pb.FetchResponse) plotter.XYs {
	pts := make(plotter.XYs, 0, len(r.GetValues()))
	start := float64(r.GetStartTime())
	step := float64(r.GetStepTime())
	absent := r.GetIsAbsent()
	for i, v := range r.GetValues() {
		if absent[i] {
			continue
		}
		pts = append(pts, xy{start + float64(i)*step, v})
	}
	return pts
}

func getInt(s string, def int) int {

	if s == "" {
		return def
	}

	n, err := strconv.Atoi(s)
	if err != nil {
		return def
	}

	return n

}
<|endoftext|>


<commit_before>// Copyright 2016 Hajime Hoshi
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package graphics

import (
	"errors"
	"fmt"
	"image"
	"image/color"
	"math"

	"github.com/hajimehoshi/ebiten/internal/affine"
	emath "github.com/hajimehoshi/ebiten/internal/math"
	"github.com/hajimehoshi/ebiten/internal/opengl"
	"github.com/hajimehoshi/ebiten/internal/sync"
)

// command represents a drawing command.
//
// A command for drawing that is created when Image functions are called like DrawImage,
// or Fill.
// A command is not immediately executed after created. Instaed, it is queued after created,
// and executed only when necessary.
type command interface {
	Exec(indexOffsetInBytes int) error
}

// commandQueue is a command queue for drawing commands.
type commandQueue struct {
	// commands is a queue of drawing commands.
	commands []command

	// vertices represents a vertices data in OpenGL's array buffer.
	vertices []float32

	// verticesNum represents the current length of vertices.
	// verticesNum must <= len(vertices).
	// vertices is never shrunk since re-extending a vertices buffer is heavy.
	verticesNum int

	m sync.Mutex
}

// theCommandQueue is the command queue for the current process.
var theCommandQueue = &commandQueue{}

// appendVertices appends vertices to the queue.
func (q *commandQueue) appendVertices(vertices []float32) {
	if len(q.vertices) < q.verticesNum+len(vertices) {
		n := q.verticesNum + len(vertices) - len(q.vertices)
		q.vertices = append(q.vertices, make([]float32, n)...)
	}
	// for-loop might be faster than copy:
	// On GopherJS, copy might cause subarray calls.
	for i := 0; i < len(vertices); i++ {
		q.vertices[q.verticesNum+i] = vertices[i]
	}
	q.verticesNum += len(vertices)
}

// EnqueueDrawImageCommand enqueues a drawing-image command.
func (q *commandQueue) EnqueueDrawImageCommand(dst, src *Image, vertices []float32, clr *affine.ColorM, mode opengl.CompositeMode) {
	// Avoid defer for performance
	q.m.Lock()
	q.appendVertices(vertices)
	if 0 < len(q.commands) {
		if c, ok := q.commands[len(q.commands)-1].(*drawImageCommand); ok {
			if c.canMerge(dst, src, clr, mode) {
				c.verticesNum += len(vertices)
				q.m.Unlock()
				return
			}
		}
	}
	c := &drawImageCommand{
		dst:         dst,
		src:         src,
		verticesNum: len(vertices),
		color:       *clr,
		mode:        mode,
	}
	q.commands = append(q.commands, c)
	q.m.Unlock()
}

// Enqueue enqueues a drawing command other than a draw-image command.
//
// For a draw-image command, use EnqueueDrawImageCommand.
func (q *commandQueue) Enqueue(command command) {
	q.m.Lock()
	q.commands = append(q.commands, command)
	q.m.Unlock()
}

// commandGroups separates q.commands into some groups.
// The number of quads of drawImageCommand in one groups must be equal to or less than
// its limit (maxQuads).
func (q *commandQueue) commandGroups() [][]command {
	cs := q.commands
	var gs [][]command
	quads := 0
	for 0 < len(cs) {
		if len(gs) == 0 {
			gs = append(gs, []command{})
		}
		c := cs[0]
		switch c := c.(type) {
		case *drawImageCommand:
			if maxQuads >= quads+c.quadsNum() {
				quads += c.quadsNum()
				break
			}
			cc := c.split(maxQuads - quads)
			gs[len(gs)-1] = append(gs[len(gs)-1], cc[0])
			cs[0] = cc[1]
			quads = 0
			gs = append(gs, []command{})
			continue
		}
		gs[len(gs)-1] = append(gs[len(gs)-1], c)
		cs = cs[1:]
	}
	return gs
}

// Flush flushes the command queue.
func (q *commandQueue) Flush() error {
	q.m.Lock()
	defer q.m.Unlock()
	// glViewport must be called at least at every frame on iOS.
	opengl.GetContext().ResetViewportSize()
	n := 0
	lastN := 0
	for _, g := range q.commandGroups() {
		for _, c := range g {
			switch c := c.(type) {
			case *drawImageCommand:
				n += c.verticesNum
			}
		}
		if 0 < n-lastN {
			opengl.GetContext().BufferSubData(opengl.ArrayBuffer, q.vertices[lastN:n])
		}
		// NOTE: WebGL doesn't seem to have Check gl.MAX_ELEMENTS_VERTICES or gl.MAX_ELEMENTS_INDICES so far.
		// Let's use them to compare to len(quads) in the future.
		if maxQuads < (n-lastN)*opengl.Float.SizeInBytes()/QuadVertexSizeInBytes() {
			return fmt.Errorf("len(quads) must be equal to or less than %d", maxQuads)
		}
		numc := len(g)
		indexOffsetInBytes := 0
		for _, c := range g {
			if err := c.Exec(indexOffsetInBytes); err != nil {
				return err
			}
			if c, ok := c.(*drawImageCommand); ok {
				n := c.verticesNum * opengl.Float.SizeInBytes() / QuadVertexSizeInBytes()
				indexOffsetInBytes += 6 * n * 2
			}
		}
		if 0 < numc {
			// Call glFlush to prevent black flicking (especially on Android (#226) and iOS).
			opengl.GetContext().Flush()
		}
		lastN = n
	}
	q.commands = nil
	q.verticesNum = 0
	return nil
}

// FlushCommands flushes the command queue.
func FlushCommands() error {
	return theCommandQueue.Flush()
}

// fillCommand represents a drawing command to fill an image with a solid color.
type fillCommand struct {
	dst   *Image
	color color.RGBA
}

// Exec executes the fillCommand.
func (c *fillCommand) Exec(indexOffsetInBytes int) error {
	f, err := c.dst.createFramebufferIfNeeded()
	if err != nil {
		return err
	}
	f.setAsViewport()

	cr, cg, cb, ca := c.color.R, c.color.G, c.color.B, c.color.A
	const max = math.MaxUint8
	r := float32(cr) / max
	g := float32(cg) / max
	b := float32(cb) / max
	a := float32(ca) / max
	if err := opengl.GetContext().FillFramebuffer(r, g, b, a); err != nil {
		return err
	}

	// Flush is needed after filling (#419)
	opengl.GetContext().Flush()
	return nil
}

// drawImageCommand represents a drawing command to draw an image on another image.
type drawImageCommand struct {
	dst         *Image
	src         *Image
	verticesNum int
	color       affine.ColorM
	mode        opengl.CompositeMode
}

// QuadVertexSizeInBytes returns the size in bytes of vertices for a quadrangle.
func QuadVertexSizeInBytes() int {
	return 4 * theArrayBufferLayout.totalBytes()
}

// Exec executes the drawImageCommand.
func (c *drawImageCommand) Exec(indexOffsetInBytes int) error {
	f, err := c.dst.createFramebufferIfNeeded()
	if err != nil {
		return err
	}
	f.setAsViewport()

	opengl.GetContext().BlendFunc(c.mode)

	n := c.quadsNum()
	if n == 0 {
		return nil
	}
	_, h := c.dst.Size()
	proj := f.projectionMatrix(h)
	theOpenGLState.useProgram(proj, c.src.texture.native, c.color)
	// TODO: We should call glBindBuffer here?
	// The buffer is already bound at begin() but it is counterintuitive.
	opengl.GetContext().DrawElements(opengl.Triangles, 6*n, indexOffsetInBytes)
	return nil
}

// split splits the drawImageCommand c into two drawImageCommands.
//
// split is called when the number of vertices reaches of the maximum and
// a command is needed to be executed as another draw call.
func (c *drawImageCommand) split(quadsNum int) [2]*drawImageCommand {
	c1 := *c
	c2 := *c
	s := opengl.Float.SizeInBytes()
	n := quadsNum * QuadVertexSizeInBytes() / s
	c1.verticesNum = n
	c2.verticesNum -= n
	return [2]*drawImageCommand{&c1, &c2}
}

// canMerge returns a boolean value indicating whether the other drawImageCommand can be merged
// with the drawImageCommand c.
func (c *drawImageCommand) canMerge(dst, src *Image, clr *affine.ColorM, mode opengl.CompositeMode) bool {
	if c.dst != dst {
		return false
	}
	if c.src != src {
		return false
	}
	if !c.color.Equals(clr) {
		return false
	}
	if c.mode != mode {
		return false
	}
	return true
}

// quadsNum returns the number of quadrangles.
func (c *drawImageCommand) quadsNum() int {
	return c.verticesNum * opengl.Float.SizeInBytes() / QuadVertexSizeInBytes()
}

// replacePixelsCommand represents a command to replace pixels of an image.
type replacePixelsCommand struct {
	dst    *Image
	pixels []uint8
}

// Exec executes the replacePixelsCommand.
func (c *replacePixelsCommand) Exec(indexOffsetInBytes int) error {
	f, err := c.dst.createFramebufferIfNeeded()
	if err != nil {
		return err
	}
	f.setAsViewport()

	// Filling with non black or white color is required here for glTexSubImage2D.
	// Very mysterious but this actually works (Issue #186).
	// This is needed even after fixing a shader bug at f537378f2a6a8ef56e1acf1c03034967b77c7b51.
	if err := opengl.GetContext().FillFramebuffer(0, 0, 0.5, 1); err != nil {
		return err
	}
	// This is necessary on Android. We can't call glClear just before glTexSubImage2D without
	// glFlush. glTexSubImage2D didn't work without this hack at least on Nexus 5x (#211).
	// This also happens when a fillCommand precedes a replacePixelsCommand.
	// TODO: Can we have a better way like optimizing commands?
	opengl.GetContext().Flush()
	opengl.GetContext().BindTexture(c.dst.texture.native)
	opengl.GetContext().TexSubImage2D(c.pixels, emath.NextPowerOf2Int(c.dst.width), emath.NextPowerOf2Int(c.dst.height))
	return nil
}

// disposeCommand represents a command to dispose an image.
type disposeCommand struct {
	target *Image
}

// Exec executes the disposeCommand.
func (c *disposeCommand) Exec(indexOffsetInBytes int) error {
	if c.target.framebuffer != nil {
		opengl.GetContext().DeleteFramebuffer(c.target.framebuffer.native)
	}
	if c.target.texture != nil {
		opengl.GetContext().DeleteTexture(c.target.texture.native)
	}
	return nil
}

// newImageFromImageCommand represents a command to create an image from an image.RGBA.
type newImageFromImageCommand struct {
	result *Image
	img    *image.RGBA
	filter opengl.Filter
}

// Exec executes the newImageFromImageCommand.
func (c *newImageFromImageCommand) Exec(indexOffsetInBytes int) error {
	origSize := c.img.Bounds().Size()
	if origSize.X < 1 {
		return errors.New("graphics: width must be equal or more than 1.")
	}
	if origSize.Y < 1 {
		return errors.New("graphics: height must be equal or more than 1.")
	}
	w, h := c.img.Bounds().Size().X, c.img.Bounds().Size().Y
	if c.img.Bounds() != image.Rect(0, 0, emath.NextPowerOf2Int(w), emath.NextPowerOf2Int(h)) {
		panic(fmt.Sprintf("graphics: invalid image bounds: %v", c.img.Bounds()))
	}
	native, err := opengl.GetContext().NewTexture(w, h, c.img.Pix, c.filter)
	if err != nil {
		return err
	}
	c.result.texture = &texture{
		native: native,
	}
	return nil
}

// newImageCommand represents a command to create an empty image with given width and height.
type newImageCommand struct {
	result *Image
	width  int
	height int
	filter opengl.Filter
}

// Exec executes a newImageCommand.
func (c *newImageCommand) Exec(indexOffsetInBytes int) error {
	w := emath.NextPowerOf2Int(c.width)
	h := emath.NextPowerOf2Int(c.height)
	if w < 1 {
		return errors.New("graphics: width must be equal or more than 1.")
	}
	if h < 1 {
		return errors.New("graphics: height must be equal or more than 1.")
	}
	native, err := opengl.GetContext().NewTexture(w, h, nil, c.filter)
	if err != nil {
		return err
	}
	c.result.texture = &texture{
		native: native,
	}
	return nil
}

// newScreenFramebufferImageCommand is a command to create a special image for the screen.
type newScreenFramebufferImageCommand struct {
	result  *Image
	width   int
	height  int
	offsetX float64
	offsetY float64
}

// Exec executes a newScreenFramebufferImageCommand.
func (c *newScreenFramebufferImageCommand) Exec(indexOffsetInBytes int) error {
	if c.width < 1 {
		return errors.New("graphics: width must be equal or more than 1.")
	}
	if c.height < 1 {
		return errors.New("graphics: height must be equal or more than 1.")
	}
	c.result.framebuffer = newScreenFramebuffer(c.width, c.height, c.offsetX, c.offsetY)
	return nil
}
<commit_msg>graphics: Use power-of-2 size for the default framebuffer<commit_after>// Copyright 2016 Hajime Hoshi
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package graphics

import (
	"errors"
	"fmt"
	"image"
	"image/color"
	"math"

	"github.com/hajimehoshi/ebiten/internal/affine"
	emath "github.com/hajimehoshi/ebiten/internal/math"
	"github.com/hajimehoshi/ebiten/internal/opengl"
	"github.com/hajimehoshi/ebiten/internal/sync"
)

// command represents a drawing command.
//
// A command for drawing that is created when Image functions are called like DrawImage,
// or Fill.
// A command is not immediately executed after created. Instaed, it is queued after created,
// and executed only when necessary.
type command interface {
	Exec(indexOffsetInBytes int) error
}

// commandQueue is a command queue for drawing commands.
type commandQueue struct {
	// commands is a queue of drawing commands.
	commands []command

	// vertices represents a vertices data in OpenGL's array buffer.
	vertices []float32

	// verticesNum represents the current length of vertices.
	// verticesNum must <= len(vertices).
	// vertices is never shrunk since re-extending a vertices buffer is heavy.
	verticesNum int

	m sync.Mutex
}

// theCommandQueue is the command queue for the current process.
var theCommandQueue = &commandQueue{}

// appendVertices appends vertices to the queue.
func (q *commandQueue) appendVertices(vertices []float32) {
	if len(q.vertices) < q.verticesNum+len(vertices) {
		n := q.verticesNum + len(vertices) - len(q.vertices)
		q.vertices = append(q.vertices, make([]float32, n)...)
	}
	// for-loop might be faster than copy:
	// On GopherJS, copy might cause subarray calls.
	for i := 0; i < len(vertices); i++ {
		q.vertices[q.verticesNum+i] = vertices[i]
	}
	q.verticesNum += len(vertices)
}

// EnqueueDrawImageCommand enqueues a drawing-image command.
func (q *commandQueue) EnqueueDrawImageCommand(dst, src *Image, vertices []float32, clr *affine.ColorM, mode opengl.CompositeMode) {
	// Avoid defer for performance
	q.m.Lock()
	q.appendVertices(vertices)
	if 0 < len(q.commands) {
		if c, ok := q.commands[len(q.commands)-1].(*drawImageCommand); ok {
			if c.canMerge(dst, src, clr, mode) {
				c.verticesNum += len(vertices)
				q.m.Unlock()
				return
			}
		}
	}
	c := &drawImageCommand{
		dst:         dst,
		src:         src,
		verticesNum: len(vertices),
		color:       *clr,
		mode:        mode,
	}
	q.commands = append(q.commands, c)
	q.m.Unlock()
}

// Enqueue enqueues a drawing command other than a draw-image command.
//
// For a draw-image command, use EnqueueDrawImageCommand.
func (q *commandQueue) Enqueue(command command) {
	q.m.Lock()
	q.commands = append(q.commands, command)
	q.m.Unlock()
}

// commandGroups separates q.commands into some groups.
// The number of quads of drawImageCommand in one groups must be equal to or less than
// its limit (maxQuads).
func (q *commandQueue) commandGroups() [][]command {
	cs := q.commands
	var gs [][]command
	quads := 0
	for 0 < len(cs) {
		if len(gs) == 0 {
			gs = append(gs, []command{})
		}
		c := cs[0]
		switch c := c.(type) {
		case *drawImageCommand:
			if maxQuads >= quads+c.quadsNum() {
				quads += c.quadsNum()
				break
			}
			cc := c.split(maxQuads - quads)
			gs[len(gs)-1] = append(gs[len(gs)-1], cc[0])
			cs[0] = cc[1]
			quads = 0
			gs = append(gs, []command{})
			continue
		}
		gs[len(gs)-1] = append(gs[len(gs)-1], c)
		cs = cs[1:]
	}
	return gs
}

// Flush flushes the command queue.
func (q *commandQueue) Flush() error {
	q.m.Lock()
	defer q.m.Unlock()
	// glViewport must be called at least at every frame on iOS.
	opengl.GetContext().ResetViewportSize()
	n := 0
	lastN := 0
	for _, g := range q.commandGroups() {
		for _, c := range g {
			switch c := c.(type) {
			case *drawImageCommand:
				n += c.verticesNum
			}
		}
		if 0 < n-lastN {
			opengl.GetContext().BufferSubData(opengl.ArrayBuffer, q.vertices[lastN:n])
		}
		// NOTE: WebGL doesn't seem to have Check gl.MAX_ELEMENTS_VERTICES or gl.MAX_ELEMENTS_INDICES so far.
		// Let's use them to compare to len(quads) in the future.
		if maxQuads < (n-lastN)*opengl.Float.SizeInBytes()/QuadVertexSizeInBytes() {
			return fmt.Errorf("len(quads) must be equal to or less than %d", maxQuads)
		}
		numc := len(g)
		indexOffsetInBytes := 0
		for _, c := range g {
			if err := c.Exec(indexOffsetInBytes); err != nil {
				return err
			}
			if c, ok := c.(*drawImageCommand); ok {
				n := c.verticesNum * opengl.Float.SizeInBytes() / QuadVertexSizeInBytes()
				indexOffsetInBytes += 6 * n * 2
			}
		}
		if 0 < numc {
			// Call glFlush to prevent black flicking (especially on Android (#226) and iOS).
			opengl.GetContext().Flush()
		}
		lastN = n
	}
	q.commands = nil
	q.verticesNum = 0
	return nil
}

// FlushCommands flushes the command queue.
func FlushCommands() error {
	return theCommandQueue.Flush()
}

// fillCommand represents a drawing command to fill an image with a solid color.
type fillCommand struct {
	dst   *Image
	color color.RGBA
}

// Exec executes the fillCommand.
func (c *fillCommand) Exec(indexOffsetInBytes int) error {
	f, err := c.dst.createFramebufferIfNeeded()
	if err != nil {
		return err
	}
	f.setAsViewport()

	cr, cg, cb, ca := c.color.R, c.color.G, c.color.B, c.color.A
	const max = math.MaxUint8
	r := float32(cr) / max
	g := float32(cg) / max
	b := float32(cb) / max
	a := float32(ca) / max
	if err := opengl.GetContext().FillFramebuffer(r, g, b, a); err != nil {
		return err
	}

	// Flush is needed after filling (#419)
	opengl.GetContext().Flush()
	return nil
}

// drawImageCommand represents a drawing command to draw an image on another image.
type drawImageCommand struct {
	dst         *Image
	src         *Image
	verticesNum int
	color       affine.ColorM
	mode        opengl.CompositeMode
}

// QuadVertexSizeInBytes returns the size in bytes of vertices for a quadrangle.
func QuadVertexSizeInBytes() int {
	return 4 * theArrayBufferLayout.totalBytes()
}

// Exec executes the drawImageCommand.
func (c *drawImageCommand) Exec(indexOffsetInBytes int) error {
	f, err := c.dst.createFramebufferIfNeeded()
	if err != nil {
		return err
	}
	f.setAsViewport()

	opengl.GetContext().BlendFunc(c.mode)

	n := c.quadsNum()
	if n == 0 {
		return nil
	}
	_, h := c.dst.Size()
	proj := f.projectionMatrix(h)
	theOpenGLState.useProgram(proj, c.src.texture.native, c.color)
	// TODO: We should call glBindBuffer here?
	// The buffer is already bound at begin() but it is counterintuitive.
	opengl.GetContext().DrawElements(opengl.Triangles, 6*n, indexOffsetInBytes)
	return nil
}

// split splits the drawImageCommand c into two drawImageCommands.
//
// split is called when the number of vertices reaches of the maximum and
// a command is needed to be executed as another draw call.
func (c *drawImageCommand) split(quadsNum int) [2]*drawImageCommand {
	c1 := *c
	c2 := *c
	s := opengl.Float.SizeInBytes()
	n := quadsNum * QuadVertexSizeInBytes() / s
	c1.verticesNum = n
	c2.verticesNum -= n
	return [2]*drawImageCommand{&c1, &c2}
}

// canMerge returns a boolean value indicating whether the other drawImageCommand can be merged
// with the drawImageCommand c.
func (c *drawImageCommand) canMerge(dst, src *Image, clr *affine.ColorM, mode opengl.CompositeMode) bool {
	if c.dst != dst {
		return false
	}
	if c.src != src {
		return false
	}
	if !c.color.Equals(clr) {
		return false
	}
	if c.mode != mode {
		return false
	}
	return true
}

// quadsNum returns the number of quadrangles.
func (c *drawImageCommand) quadsNum() int {
	return c.verticesNum * opengl.Float.SizeInBytes() / QuadVertexSizeInBytes()
}

// replacePixelsCommand represents a command to replace pixels of an image.
type replacePixelsCommand struct {
	dst    *Image
	pixels []uint8
}

// Exec executes the replacePixelsCommand.
func (c *replacePixelsCommand) Exec(indexOffsetInBytes int) error {
	f, err := c.dst.createFramebufferIfNeeded()
	if err != nil {
		return err
	}
	f.setAsViewport()

	// Filling with non black or white color is required here for glTexSubImage2D.
	// Very mysterious but this actually works (Issue #186).
	// This is needed even after fixing a shader bug at f537378f2a6a8ef56e1acf1c03034967b77c7b51.
	if err := opengl.GetContext().FillFramebuffer(0, 0, 0.5, 1); err != nil {
		return err
	}
	// This is necessary on Android. We can't call glClear just before glTexSubImage2D without
	// glFlush. glTexSubImage2D didn't work without this hack at least on Nexus 5x (#211).
	// This also happens when a fillCommand precedes a replacePixelsCommand.
	// TODO: Can we have a better way like optimizing commands?
	opengl.GetContext().Flush()
	opengl.GetContext().BindTexture(c.dst.texture.native)
	opengl.GetContext().TexSubImage2D(c.pixels, emath.NextPowerOf2Int(c.dst.width), emath.NextPowerOf2Int(c.dst.height))
	return nil
}

// disposeCommand represents a command to dispose an image.
type disposeCommand struct {
	target *Image
}

// Exec executes the disposeCommand.
func (c *disposeCommand) Exec(indexOffsetInBytes int) error {
	if c.target.framebuffer != nil {
		opengl.GetContext().DeleteFramebuffer(c.target.framebuffer.native)
	}
	if c.target.texture != nil {
		opengl.GetContext().DeleteTexture(c.target.texture.native)
	}
	return nil
}

// newImageFromImageCommand represents a command to create an image from an image.RGBA.
type newImageFromImageCommand struct {
	result *Image
	img    *image.RGBA
	filter opengl.Filter
}

// Exec executes the newImageFromImageCommand.
func (c *newImageFromImageCommand) Exec(indexOffsetInBytes int) error {
	origSize := c.img.Bounds().Size()
	if origSize.X < 1 {
		return errors.New("graphics: width must be equal or more than 1.")
	}
	if origSize.Y < 1 {
		return errors.New("graphics: height must be equal or more than 1.")
	}
	w, h := c.img.Bounds().Size().X, c.img.Bounds().Size().Y
	if c.img.Bounds() != image.Rect(0, 0, emath.NextPowerOf2Int(w), emath.NextPowerOf2Int(h)) {
		panic(fmt.Sprintf("graphics: invalid image bounds: %v", c.img.Bounds()))
	}
	native, err := opengl.GetContext().NewTexture(w, h, c.img.Pix, c.filter)
	if err != nil {
		return err
	}
	c.result.texture = &texture{
		native: native,
	}
	return nil
}

// newImageCommand represents a command to create an empty image with given width and height.
type newImageCommand struct {
	result *Image
	width  int
	height int
	filter opengl.Filter
}

// Exec executes a newImageCommand.
func (c *newImageCommand) Exec(indexOffsetInBytes int) error {
	w := emath.NextPowerOf2Int(c.width)
	h := emath.NextPowerOf2Int(c.height)
	if w < 1 {
		return errors.New("graphics: width must be equal or more than 1.")
	}
	if h < 1 {
		return errors.New("graphics: height must be equal or more than 1.")
	}
	native, err := opengl.GetContext().NewTexture(w, h, nil, c.filter)
	if err != nil {
		return err
	}
	c.result.texture = &texture{
		native: native,
	}
	return nil
}

// newScreenFramebufferImageCommand is a command to create a special image for the screen.
type newScreenFramebufferImageCommand struct {
	result  *Image
	width   int
	height  int
	offsetX float64
	offsetY float64
}

// Exec executes a newScreenFramebufferImageCommand.
func (c *newScreenFramebufferImageCommand) Exec(indexOffsetInBytes int) error {
	if c.width < 1 {
		return errors.New("graphics: width must be equal or more than 1.")
	}
	if c.height < 1 {
		return errors.New("graphics: height must be equal or more than 1.")
	}
	w := emath.NextPowerOf2Int(c.width)
	h := emath.NextPowerOf2Int(c.height)
	c.result.framebuffer = newScreenFramebuffer(w, h, c.offsetX, c.offsetY)
	return nil
}
<|endoftext|>


<commit_before>package photomgr

import (
	"fmt"
	"log"
	"net/http"
	"os"
	"path/filepath"
	"regexp"
	"strconv"
	"strings"
	"sync"

	"github.com/PuerkitoBio/goquery"
)

type PTT struct {
	//Inherit
	baseCrawler

	//Handle base folder address to store images
	BaseDir string

	//To store current PTT post result
	storedPostURLList   []string
	storedPostTitleList []string
	storedStarList      []int
}

func NewPTT() *PTT {

	p := new(PTT)
	p.baseAddress = "https://www.ptt.cc"
	p.entryAddress = "https://www.ptt.cc/bbs/Beauty/index.html"
	return p
}

func (p *PTT) GetUrlPhotos(target string) []string {
	var resultSlice []string
	// Get https response with setting cookie over18=1
	resp := getResponseWithCookie(target)
	doc, err := goquery.NewDocumentFromResponse(resp)
	if err != nil {
		log.Println(err)
		return nil
	}

	//Parse Image, currently support <IMG SRC> only
	doc.Find(".richcontent").Each(func(i int, s *goquery.Selection) {
		imgLink, exist := s.Find("img").Attr("src")
		if exist {
			resultSlice = append(resultSlice, "http:"+imgLink)
		}
	})
	return resultSlice
}

func (p *PTT) Crawler(target string, workerNum int) {
	// Get https response with setting cookie over18=1
	resp := getResponseWithCookie(target)
	doc, err := goquery.NewDocumentFromResponse(resp)
	if err != nil {
		log.Println(err)
		return
	}

	//Title and folder
	articleTitle := ""
	doc.Find(".article-metaline").Each(func(i int, s *goquery.Selection) {
		if strings.Contains(s.Find(".article-meta-tag").Text(), "標題") {
			articleTitle = s.Find(".article-meta-value").Text()
		}
	})
	dir := fmt.Sprintf("%v/%v - %v", p.BaseDir, "PTT", articleTitle)
	if exist, _ := exists(dir); exist {
		//fmt.Println("Already download")
		return
	}
	os.MkdirAll(filepath.FromSlash(dir), 0755)

	//Concurrecny
	linkChan := make(chan string)
	wg := new(sync.WaitGroup)
	for i := 0; i < workerNum; i++ {
		wg.Add(1)
		go p.worker(filepath.FromSlash(dir), linkChan, wg)
	}

	//Parse Image, currently support <IMG SRC> only
	foundImage := false
	doc.Find("a").Each(func(i int, s *goquery.Selection) {
		imgLink, _ := s.Attr("href")
		switch {
		case strings.Contains(imgLink, "https://i.imgur.com/"):
			linkChan <- imgLink
			foundImage = true
		case strings.Contains(imgLink, "http://i.imgur.com/"):
			linkChan <- imgLink
			foundImage = true
		case strings.Contains(imgLink, "https://pbs.twimg.com/"):
			linkChan <- imgLink
			foundImage = true
		case strings.Contains(imgLink, "https://imgur.com/"):
			imgLink = imgLink + ".jpg"
			linkChan <- imgLink
			foundImage = true
		}
	})

	if !foundImage {
		log.Println("Don't have any image in this article.")
	}

	close(linkChan)
	wg.Wait()
}

// GetAllImageAddress: return all image address in current page.
func (p *PTT) GetAllImageAddress(target string) []string {
	var ret []string
	// Get https response with setting cookie over18=1
	resp := getResponseWithCookie(target)
	doc, err := goquery.NewDocumentFromResponse(resp)
	if err != nil {
		log.Println(err)
		return nil
	}

	//Parse Image, currently support <IMG SRC> only
	foundImage := false
	doc.Find("a").Each(func(i int, s *goquery.Selection) {
		imgLink, _ := s.Attr("href")
		switch {
		case strings.Contains(imgLink, "https://i.imgur.com/"):
			ret = append(ret, imgLink)
			foundImage = true
		case strings.Contains(imgLink, "http://i.imgur.com/"):
			ret = append(ret, imgLink)
			foundImage = true
		case strings.Contains(imgLink, "https://pbs.twimg.com/"):
			ret = append(ret, imgLink)
			foundImage = true
		case strings.Contains(imgLink, "https://imgur.com/"):
			imgLink = imgLink + ".jpg"
			ret = append(ret, imgLink)
			foundImage = true
		}
	})

	if !foundImage {
		log.Println("Don't have any image in this article.")
	}

	return ret
}

// Return parse page result count, it will be 0 if you still not parse any page
func (p *PTT) GetCurrentPageResultCount() int {
	return len(p.storedPostTitleList)
}

// Get post title by index in current parsed page
func (p *PTT) GetPostTitleByIndex(postIndex int) string {
	if postIndex >= len(p.storedPostTitleList) {
		return ""
	}
	return p.storedPostTitleList[postIndex]
}

// Get post URL by index in current parsed page
func (p *PTT) GetPostUrlByIndex(postIndex int) string {
	if postIndex >= len(p.storedPostURLList) {
		return ""
	}

	return p.storedPostURLList[postIndex]
}

// Get post like count by index in current parsed page
func (p *PTT) GetPostStarByIndex(postIndex int) int {
	if postIndex >= len(p.storedStarList) {
		return 0
	}
	return p.storedStarList[postIndex]
}

//Set Ptt board page index, fetch all post and return article count back
func (p *PTT) ParsePttPageByIndex(page int) int {
	// Get https response with setting cookie over18=1
	resp := getResponseWithCookie(p.entryAddress)
	doc, err := goquery.NewDocumentFromResponse(resp)
	if err != nil {
		log.Fatal(err)
	}

	urlList := make([]string, 0)
	postList := make([]string, 0)
	starList := make([]int, 0)

	maxPageNumberString := ""
	var PageWebSide string
	if page > 0 {
		// Find page result
		doc.Find(".btn-group a").Each(func(i int, s *goquery.Selection) {
			if strings.Contains(s.Text(), "上頁") {
				href, exist := s.Attr("href")
				if exist {
					targetString := strings.Split(href, "index")[1]
					targetString = strings.Split(targetString, ".html")[0]
					log.Println("total page:", targetString)
					maxPageNumberString = targetString
				}
			}
		})
		pageNum, _ := strconv.Atoi(maxPageNumberString)
		pageNum = pageNum - page + 1
		PageWebSide = fmt.Sprintf("https://www.ptt.cc/bbs/Beauty/index%d.html", pageNum)
	} else {
		PageWebSide = p.entryAddress
	}

	// Get https response with setting cookie over18=1
	resp = getResponseWithCookie(PageWebSide)
	doc, err = goquery.NewDocumentFromResponse(resp)
	if err != nil {
		log.Fatal(err)
	}

	doc.Find(".r-ent").Each(func(i int, s *goquery.Selection) {
		title := strings.TrimSpace(s.Find(".title").Text())
		likeCount, _ := strconv.Atoi(s.Find(".nrec span").Text())
		href, _ := s.Find(".title a").Attr("href")
		link := p.baseAddress + href
		urlList = append(urlList, link)
		log.Printf("%d:[%d★]%s\n", i, likeCount, title)
		starList = append(starList, likeCount)
		postList = append(postList, title)
	})

	// Print pages
	log.Printf("Pages: ")
	for i := page - 3; i <= page+2; i++ {
		if i >= 0 {
			if i == page {
				log.Printf("[%v] ", i)
			} else {
				log.Printf("%v ", i)
			}
		}
	}

	p.storedPostURLList = urlList
	p.storedStarList = starList
	p.storedPostTitleList = postList

	return len(p.storedPostTitleList)
}

func getResponseWithCookie(url string) *http.Response {
	req, err := http.NewRequest("GET", url, nil)
	if err != nil {
		log.Fatal("http failed:", err)
	}

	req.AddCookie(&http.Cookie{Name: "over18", Value: "1"})

	client := &http.Client{}
	resp, err := client.Do(req)
	if err != nil {
		log.Fatal("client failed:", err)
	}
	return resp
}

func (p *PTT) GetPostLikeDis(target string) (int, int) {
	// Get https response with setting cookie over18=1
	resp := getResponseWithCookie(target)
	doc, err := goquery.NewDocumentFromResponse(resp)
	if err != nil {
		log.Println(err)
		return 0, 0
	}

	var likeCount int
	var disLikeCount int
	doc.Find(".push-tag").Each(func(i int, s *goquery.Selection) {
		if strings.Contains(s.Text(), "推") {
			likeCount++
		} else if strings.Contains(s.Text(), "噓") {
			disLikeCount++
		}
	})
	// fmt.Println("like:", likeCount, " dislike:", disLikeCount)
	return likeCount, disLikeCount
}

func CheckTitleWithBeauty(title string) bool {
	d, _ := regexp.MatchString("^\\[正妹\\].*", title)
	return d
}
<commit_msg>Only girls will get from package.<commit_after>package photomgr

import (
	"fmt"
	"log"
	"net/http"
	"os"
	"path/filepath"
	"regexp"
	"strconv"
	"strings"
	"sync"

	"github.com/PuerkitoBio/goquery"
)

type PTT struct {
	//Inherit
	baseCrawler

	//Handle base folder address to store images
	BaseDir string

	//To store current PTT post result
	storedPostURLList   []string
	storedPostTitleList []string
	storedStarList      []int
}

func NewPTT() *PTT {

	p := new(PTT)
	p.baseAddress = "https://www.ptt.cc"
	p.entryAddress = "https://www.ptt.cc/bbs/Beauty/index.html"
	return p
}

func (p *PTT) GetUrlPhotos(target string) []string {
	var resultSlice []string
	// Get https response with setting cookie over18=1
	resp := getResponseWithCookie(target)
	doc, err := goquery.NewDocumentFromResponse(resp)
	if err != nil {
		log.Println(err)
		return nil
	}

	//Parse Image, currently support <IMG SRC> only
	doc.Find(".richcontent").Each(func(i int, s *goquery.Selection) {
		imgLink, exist := s.Find("img").Attr("src")
		if exist {
			resultSlice = append(resultSlice, "http:"+imgLink)
		}
	})
	return resultSlice
}

func (p *PTT) Crawler(target string, workerNum int) {
	// Get https response with setting cookie over18=1
	resp := getResponseWithCookie(target)
	doc, err := goquery.NewDocumentFromResponse(resp)
	if err != nil {
		log.Println(err)
		return
	}

	//Title and folder
	articleTitle := ""
	doc.Find(".article-metaline").Each(func(i int, s *goquery.Selection) {
		if strings.Contains(s.Find(".article-meta-tag").Text(), "標題") {
			articleTitle = s.Find(".article-meta-value").Text()
		}
	})
	dir := fmt.Sprintf("%v/%v - %v", p.BaseDir, "PTT", articleTitle)
	if exist, _ := exists(dir); exist {
		//fmt.Println("Already download")
		return
	}
	os.MkdirAll(filepath.FromSlash(dir), 0755)

	//Concurrecny
	linkChan := make(chan string)
	wg := new(sync.WaitGroup)
	for i := 0; i < workerNum; i++ {
		wg.Add(1)
		go p.worker(filepath.FromSlash(dir), linkChan, wg)
	}

	//Parse Image, currently support <IMG SRC> only
	foundImage := false
	doc.Find("a").Each(func(i int, s *goquery.Selection) {
		imgLink, _ := s.Attr("href")
		switch {
		case strings.Contains(imgLink, "https://i.imgur.com/"):
			linkChan <- imgLink
			foundImage = true
		case strings.Contains(imgLink, "http://i.imgur.com/"):
			linkChan <- imgLink
			foundImage = true
		case strings.Contains(imgLink, "https://pbs.twimg.com/"):
			linkChan <- imgLink
			foundImage = true
		case strings.Contains(imgLink, "https://imgur.com/"):
			imgLink = imgLink + ".jpg"
			linkChan <- imgLink
			foundImage = true
		}
	})

	if !foundImage {
		log.Println("Don't have any image in this article.")
	}

	close(linkChan)
	wg.Wait()
}

// GetAllImageAddress: return all image address in current page.
func (p *PTT) GetAllImageAddress(target string) []string {
	var ret []string
	// Get https response with setting cookie over18=1
	resp := getResponseWithCookie(target)
	doc, err := goquery.NewDocumentFromResponse(resp)
	if err != nil {
		log.Println(err)
		return nil
	}

	//Parse Image, currently support <IMG SRC> only
	foundImage := false
	doc.Find("a").Each(func(i int, s *goquery.Selection) {
		imgLink, _ := s.Attr("href")
		switch {
		case strings.Contains(imgLink, "https://i.imgur.com/"):
			ret = append(ret, imgLink)
			foundImage = true
		case strings.Contains(imgLink, "http://i.imgur.com/"):
			ret = append(ret, imgLink)
			foundImage = true
		case strings.Contains(imgLink, "https://pbs.twimg.com/"):
			ret = append(ret, imgLink)
			foundImage = true
		case strings.Contains(imgLink, "https://imgur.com/"):
			imgLink = imgLink + ".jpg"
			ret = append(ret, imgLink)
			foundImage = true
		}
	})

	if !foundImage {
		log.Println("Don't have any image in this article.")
	}

	return ret
}

// Return parse page result count, it will be 0 if you still not parse any page
func (p *PTT) GetCurrentPageResultCount() int {
	return len(p.storedPostTitleList)
}

// Get post title by index in current parsed page
func (p *PTT) GetPostTitleByIndex(postIndex int) string {
	if postIndex >= len(p.storedPostTitleList) {
		return ""
	}
	return p.storedPostTitleList[postIndex]
}

// Get post URL by index in current parsed page
func (p *PTT) GetPostUrlByIndex(postIndex int) string {
	if postIndex >= len(p.storedPostURLList) {
		return ""
	}

	return p.storedPostURLList[postIndex]
}

// Get post like count by index in current parsed page
func (p *PTT) GetPostStarByIndex(postIndex int) int {
	if postIndex >= len(p.storedStarList) {
		return 0
	}
	return p.storedStarList[postIndex]
}

//Set Ptt board page index, fetch all post and return article count back
func (p *PTT) ParsePttPageByIndex(page int) int {
	// Get https response with setting cookie over18=1
	resp := getResponseWithCookie(p.entryAddress)
	doc, err := goquery.NewDocumentFromResponse(resp)
	if err != nil {
		log.Fatal(err)
	}

	urlList := make([]string, 0)
	postList := make([]string, 0)
	starList := make([]int, 0)

	maxPageNumberString := ""
	var PageWebSide string
	if page > 0 {
		// Find page result
		doc.Find(".btn-group a").Each(func(i int, s *goquery.Selection) {
			if strings.Contains(s.Text(), "上頁") {
				href, exist := s.Attr("href")
				if exist {
					targetString := strings.Split(href, "index")[1]
					targetString = strings.Split(targetString, ".html")[0]
					log.Println("total page:", targetString)
					maxPageNumberString = targetString
				}
			}
		})
		pageNum, _ := strconv.Atoi(maxPageNumberString)
		pageNum = pageNum - page + 1
		PageWebSide = fmt.Sprintf("https://www.ptt.cc/bbs/Beauty/index%d.html", pageNum)
	} else {
		PageWebSide = p.entryAddress
	}

	// Get https response with setting cookie over18=1
	resp = getResponseWithCookie(PageWebSide)
	doc, err = goquery.NewDocumentFromResponse(resp)
	if err != nil {
		log.Fatal(err)
	}

	doc.Find(".r-ent").Each(func(i int, s *goquery.Selection) {
		title := strings.TrimSpace(s.Find(".title").Text())
		if CheckTitleWithBeauty(title) {
			likeCount, _ := strconv.Atoi(s.Find(".nrec span").Text())
			href, _ := s.Find(".title a").Attr("href")
			link := p.baseAddress + href
			urlList = append(urlList, link)
			log.Printf("%d:[%d★]%s\n", i, likeCount, title)
			starList = append(starList, likeCount)
			postList = append(postList, title)
		}
	})

	// Print pages
	log.Printf("Pages: ")
	for i := page - 3; i <= page+2; i++ {
		if i >= 0 {
			if i == page {
				log.Printf("[%v] ", i)
			} else {
				log.Printf("%v ", i)
			}
		}
	}

	p.storedPostURLList = urlList
	p.storedStarList = starList
	p.storedPostTitleList = postList

	return len(p.storedPostTitleList)
}

func getResponseWithCookie(url string) *http.Response {
	req, err := http.NewRequest("GET", url, nil)
	if err != nil {
		log.Fatal("http failed:", err)
	}

	req.AddCookie(&http.Cookie{Name: "over18", Value: "1"})

	client := &http.Client{}
	resp, err := client.Do(req)
	if err != nil {
		log.Fatal("client failed:", err)
	}
	return resp
}

func (p *PTT) GetPostLikeDis(target string) (int, int) {
	// Get https response with setting cookie over18=1
	resp := getResponseWithCookie(target)
	doc, err := goquery.NewDocumentFromResponse(resp)
	if err != nil {
		log.Println(err)
		return 0, 0
	}

	var likeCount int
	var disLikeCount int
	doc.Find(".push-tag").Each(func(i int, s *goquery.Selection) {
		if strings.Contains(s.Text(), "推") {
			likeCount++
		} else if strings.Contains(s.Text(), "噓") {
			disLikeCount++
		}
	})
	// fmt.Println("like:", likeCount, " dislike:", disLikeCount)
	return likeCount, disLikeCount
}

func CheckTitleWithBeauty(title string) bool {
	d, _ := regexp.MatchString("^\\[正妹\\].*", title)
	return d
}
<|endoftext|>