<commit_before>package main

import (
	"fmt"
	"github.com/t3rm1n4l/go-mega"
	"html"
	"io"
	"log"
	"net/http"
	"os"
	"path"
	"strings"
)

const CACHEDIR = "cache"
var megaSession *mega.Mega

func handle(w http.ResponseWriter, r *http.Request) {
	switch r.Method {
	case "GET":
		node, err := node(r.URL.Path)
		if err != nil {
			// XXX 404
			fmt.Print(err)
			return
		}
		switch node.GetType() {
		case mega.FOLDER, mega.ROOT:
			list(w, r, node)
		default:
			get(w, r, node)
		}

	case "PUT":
		put(w, r)
	}
}

func list(w http.ResponseWriter, r *http.Request, node *mega.Node) {
	children, err := megaSession.FS.GetChildren(node)
	if err != nil {
		fmt.Print(err)
		// XXX 500
		return
	}

	fmt.Fprint(w, "")
	fmt.Fprintf(w, "
%s
", html.EscapeString(r.URL.Path))
	fmt.Fprint(w, "
")
	if node != megaSession.FS.GetRoot() {
		fmt.Fprintf(w, "
..")
	}
	for _, child := range children {
		var folder string
		if child.GetType() == mega.FOLDER {
			folder = "/"
		}
		fmt.Fprintf(w, "
%s%s",
			html.EscapeString(child.GetName()), folder,
			html.EscapeString(child.GetName()), folder)
	}
	fmt.Fprint(w, "
")
}

func get(w http.ResponseWriter, r *http.Request, node *mega.Node) {
	hash := node.GetHash()
	tempfile := fmt.Sprintf("%s/%s", CACHEDIR, hash)
	err := megaSession.DownloadFile(node, tempfile, nil)
	if err != nil {
		log.Printf("download failed: %s", err)
		os.Remove(tempfile)
		return
	}
	file, err := os.Open(tempfile) // For read access.
	if err != nil {
		log.Printf("error opening %s: %s", tempfile, err)
		return
	}
	defer file.Close()
	_, err = io.Copy(w, file)
	if err != nil {
		log.Print(err)
		return
	}
}

func put(w http.ResponseWriter, r *http.Request) {
	cachefile := CACHEDIR + r.URL.Path
	dir, name := path.Split(cachefile)

	// Create local file
	if err := os.MkdirAll(dir, 0700); err != nil && !os.IsExist(err) {
		log.Print(err)
		return
	}
	fp, err := os.Create(cachefile)
	if err != nil {
		log.Print(err)
		return
	}
	_, err = io.Copy(fp, r.Body)
	if err != nil {
		log.Print(err)
		return
	}

	// Create Mega path
	dirarray := strings.Split(r.URL.Path, "/")
	root := megaSession.FS.GetRoot()
	n, err := mkpath(dirarray[1:len(dirarray)-1], root)
	if err != nil {
		log.Print("mkpath", err)
		return
	}

	// Lookup Mega file (if it exists)
	paths, err := megaSession.FS.PathLookup(root, dirarray[1:])
	// Log unexpected errors
	if err != nil && err.Error() != "Object (typically, node or user) not found" {
		log.Print(err)
		return
	}
	// File exists, so delete it before uploading new file
	if err == nil {
		// We only care about the last node.
		lastnode := paths[len(paths)-1]
		// File exists, delete! (aka overwrite)
		if err = megaSession.Delete(lastnode, false); err != nil {
			log.Print(err)
			return
		}
	}
	// Finally, upload file
	_, err = megaSession.UploadFile(cachefile, n, name, nil)
	if err != nil {
		log.Print("upload", err)
		return
	}
}

func mkpath(p []string, parent *mega.Node) (*mega.Node, error) {
	var n *mega.Node
	var err error

	root := megaSession.FS.GetRoot()
	// Root path is an empty array.
	if p[0] == "" {
		return root, nil
	}

	paths, err := megaSession.FS.PathLookup(root, p)
	if err == nil {
		// We only care about the last path.
		return paths[len(paths)-1], nil
	} else if err.Error() != "Object (typically, node or user) not found" {
		log.Printf("not exist: %#v\n", err)
		return nil, err
	}

	l := len(p)
	if l == 1{
		n = parent
	} else {
		// if a/b/c then parent = mkpath(a/b)
		n, err = mkpath(p[:l-1], parent)
		if err != nil {
			return n, err
		}
	}
	return megaSession.CreateDir(p[l-1], n)
}

func node(url string) (*mega.Node, error) {
	trimmedPath := strings.Trim(url, "/")
	path := strings.Split(trimmedPath, "/")
	root := megaSession.FS.GetRoot()
	// Root path is an empty array.
	if path[0] == "" {
		return root, nil
	} else {
		paths, err := megaSession.FS.PathLookup(root, path)
		if err != nil {
			// XXX should be 404
			return nil, err
		}
		// We only care about the last path.
		return paths[len(paths)-1], nil
	}
}

func main() {
	user := os.Getenv("MEGA_USER")
	pass := os.Getenv("MEGA_PASSWD")
	megaSession = mega.New()
	if err := megaSession.Login(user, pass); err != nil {
		log.Fatal(err)
	}
	if err := os.Mkdir(CACHEDIR, 0700); err != nil && !os.IsExist(err) {
		log.Fatal(err)
	}
	http.HandleFunc("/", handle)
	log.Fatal(http.ListenAndServe("localhost:8080", nil))
}
<commit_msg>Clean up error logging.<commit_after>package main

import (
	"fmt"
	"github.com/t3rm1n4l/go-mega"
	"html"
	"io"
	"log"
	"net/http"
	"os"
	"path"
	"strings"
)

const CACHEDIR = "cache"
var megaSession *mega.Mega

func handle(w http.ResponseWriter, r *http.Request) {
	switch r.Method {
	case "GET":
		node, err := node(r.URL.Path)
		if err != nil {
			// XXX 404
			log.Print(err)
			return
		}
		switch node.GetType() {
		case mega.FOLDER, mega.ROOT:
			list(w, r, node)
		default:
			get(w, r, node)
		}

	case "PUT":
		put(w, r)
	}
}

func list(w http.ResponseWriter, r *http.Request, node *mega.Node) {
	children, err := megaSession.FS.GetChildren(node)
	if err != nil {
		log.Print(err)
		// XXX 500
		return
	}

	fmt.Fprint(w, "")
	fmt.Fprintf(w, "
%s
", html.EscapeString(r.URL.Path))
	fmt.Fprint(w, "
")
	if node != megaSession.FS.GetRoot() {
		fmt.Fprintf(w, "
..")
	}
	for _, child := range children {
		var folder string
		if child.GetType() == mega.FOLDER {
			folder = "/"
		}
		fmt.Fprintf(w, "
%s%s",
			html.EscapeString(child.GetName()), folder,
			html.EscapeString(child.GetName()), folder)
	}
	fmt.Fprint(w, "
")
}

func get(w http.ResponseWriter, r *http.Request, node *mega.Node) {
	hash := node.GetHash()
	tempfile := fmt.Sprintf("%s/%s", CACHEDIR, hash)
	err := megaSession.DownloadFile(node, tempfile, nil)
	if err != nil {
		log.Print(err)
		os.Remove(tempfile)
		return
	}
	file, err := os.Open(tempfile) // For read access.
	if err != nil {
		log.Print(err)
		return
	}
	defer file.Close()
	_, err = io.Copy(w, file)
	if err != nil {
		log.Print(err)
		return
	}
}

func put(w http.ResponseWriter, r *http.Request) {
	cachefile := CACHEDIR + r.URL.Path
	dir, name := path.Split(cachefile)

	// Create local file
	if err := os.MkdirAll(dir, 0700); err != nil && !os.IsExist(err) {
		log.Print(err)
		return
	}
	fp, err := os.Create(cachefile)
	if err != nil {
		log.Print(err)
		return
	}
	_, err = io.Copy(fp, r.Body)
	if err != nil {
		log.Print(err)
		return
	}

	// Create Mega path
	dirarray := strings.Split(r.URL.Path, "/")
	root := megaSession.FS.GetRoot()
	n, err := mkpath(dirarray[1:len(dirarray)-1], root)
	if err != nil {
		log.Print(err)
		return
	}

	// Lookup Mega file (if it exists)
	paths, err := megaSession.FS.PathLookup(root, dirarray[1:])
	// Log unexpected errors
	if err != nil && err.Error() != "Object (typically, node or user) not found" {
		log.Print(err)
		return
	}
	// File exists, so delete it before uploading new file
	if err == nil {
		// We only care about the last node.
		lastnode := paths[len(paths)-1]
		// File exists, delete! (aka overwrite)
		if err = megaSession.Delete(lastnode, false); err != nil {
			log.Print(err)
			return
		}
	}
	// Finally, upload file
	_, err = megaSession.UploadFile(cachefile, n, name, nil)
	if err != nil {
		log.Print(err)
		return
	}
}

func mkpath(p []string, parent *mega.Node) (*mega.Node, error) {
	var n *mega.Node
	var err error

	root := megaSession.FS.GetRoot()
	// Root path is an empty array.
	if p[0] == "" {
		return root, nil
	}

	paths, err := megaSession.FS.PathLookup(root, p)
	if err == nil {
		// We only care about the last path.
		return paths[len(paths)-1], nil
	} else if err.Error() != "Object (typically, node or user) not found" {
		log.Printf("not exist: %#v\n", err)
		return nil, err
	}

	l := len(p)
	if l == 1{
		n = parent
	} else {
		// if a/b/c then parent = mkpath(a/b)
		n, err = mkpath(p[:l-1], parent)
		if err != nil {
			return n, err
		}
	}
	return megaSession.CreateDir(p[l-1], n)
}

func node(url string) (*mega.Node, error) {
	trimmedPath := strings.Trim(url, "/")
	path := strings.Split(trimmedPath, "/")
	root := megaSession.FS.GetRoot()
	// Root path is an empty array.
	if path[0] == "" {
		return root, nil
	} else {
		paths, err := megaSession.FS.PathLookup(root, path)
		if err != nil {
			// XXX should be 404
			return nil, err
		}
		// We only care about the last path.
		return paths[len(paths)-1], nil
	}
}

func main() {
	user := os.Getenv("MEGA_USER")
	pass := os.Getenv("MEGA_PASSWD")
	megaSession = mega.New()
	if err := megaSession.Login(user, pass); err != nil {
		log.Fatal(err)
	}
	if err := os.Mkdir(CACHEDIR, 0700); err != nil && !os.IsExist(err) {
		log.Fatal(err)
	}
	http.HandleFunc("/", handle)
	log.Fatal(http.ListenAndServe("localhost:8080", nil))
}
<|endoftext|>


<commit_before>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package eviction

import (
	"fmt"
	"sync"
	"time"

	"golang.org/x/sys/unix"
	"k8s.io/klog"
)

const (
	// eventSize is the number of bytes returned by a successful read from an eventfd
	// see http://man7.org/linux/man-pages/man2/eventfd.2.html for more information
	eventSize = 8
	// numFdEvents is the number of events we can record at once.
	// If EpollWait finds more than this, they will be missed.
	numFdEvents = 6
)

type linuxCgroupNotifier struct {
	eventfd  int
	epfd     int
	stop     chan struct{}
	stopLock sync.Mutex
}

var _ CgroupNotifier = &linuxCgroupNotifier{}

// NewCgroupNotifier returns a linuxCgroupNotifier, which performs cgroup control operations required
// to receive notifications from the cgroup when the threshold is crossed in either direction.
func NewCgroupNotifier(path, attribute string, threshold int64) (CgroupNotifier, error) {
	var watchfd, eventfd, epfd, controlfd int
	var err error
	watchfd, err = unix.Open(fmt.Sprintf("%s/%s", path, attribute), unix.O_RDONLY, 0)
	if err != nil {
		return nil, err
	}
	defer unix.Close(watchfd)
	controlfd, err = unix.Open(fmt.Sprintf("%s/cgroup.event_control", path), unix.O_WRONLY, 0)
	if err != nil {
		return nil, err
	}
	defer unix.Close(controlfd)
	eventfd, err = unix.Eventfd(0, unix.EFD_CLOEXEC)
	if err != nil {
		return nil, err
	}
	if eventfd < 0 {
		err = fmt.Errorf("eventfd call failed")
		return nil, err
	}
	defer func() {
		// Close eventfd if we get an error later in initialization
		if err != nil {
			unix.Close(eventfd)
		}
	}()
	epfd, err = unix.EpollCreate1(0)
	if err != nil {
		return nil, err
	}
	if epfd < 0 {
		err = fmt.Errorf("EpollCreate1 call failed")
		return nil, err
	}
	defer func() {
		// Close epfd if we get an error later in initialization
		if err != nil {
			unix.Close(epfd)
		}
	}()
	config := fmt.Sprintf("%d %d %d", eventfd, watchfd, threshold)
	_, err = unix.Write(controlfd, []byte(config))
	if err != nil {
		return nil, err
	}
	return &linuxCgroupNotifier{
		eventfd: eventfd,
		epfd:    epfd,
		stop:    make(chan struct{}),
	}, nil
}

func (n *linuxCgroupNotifier) Start(eventCh chan<- struct{}) {
	err := unix.EpollCtl(n.epfd, unix.EPOLL_CTL_ADD, n.eventfd, &unix.EpollEvent{
		Fd:     int32(n.eventfd),
		Events: unix.EPOLLIN,
	})
	if err != nil {
		klog.Warningf("eviction manager: error adding epoll eventfd: %v", err)
		return
	}
	for {
		select {
		case <-n.stop:
			return
		default:
		}
		event, err := wait(n.epfd, n.eventfd, notifierRefreshInterval)
		if err != nil {
			klog.Warningf("eviction manager: error while waiting for memcg events: %v", err)
			return
		} else if !event {
			// Timeout on wait.  This is expected if the threshold was not crossed
			continue
		}
		// Consume the event from the eventfd
		buf := make([]byte, eventSize)
		_, err = unix.Read(n.eventfd, buf)
		if err != nil {
			klog.Warningf("eviction manager: error reading memcg events: %v", err)
			return
		}
		eventCh <- struct{}{}
	}
}

// wait waits up to notifierRefreshInterval for an event on the Epoll FD for the
// eventfd we are concerned about.  It returns an error if one occurrs, and true
// if the consumer should read from the eventfd.
func wait(epfd, eventfd int, timeout time.Duration) (bool, error) {
	events := make([]unix.EpollEvent, numFdEvents+1)
	timeoutMS := int(timeout / time.Millisecond)
	n, err := unix.EpollWait(epfd, events, timeoutMS)
	if n == -1 {
		if err == unix.EINTR {
			// Interrupt, ignore the error
			return false, nil
		}
		return false, err
	}
	if n == 0 {
		// Timeout
		return false, nil
	}
	if n > numFdEvents {
		return false, fmt.Errorf("epoll_wait returned more events than we know what to do with")
	}
	for _, event := range events[:n] {
		if event.Fd == int32(eventfd) {
			if event.Events&unix.EPOLLHUP != 0 || event.Events&unix.EPOLLERR != 0 || event.Events&unix.EPOLLIN != 0 {
				// EPOLLHUP: should not happen, but if it does, treat it as a wakeup.

				// EPOLLERR: If an error is waiting on the file descriptor, we should pretend
				// something is ready to read, and let unix.Read pick up the error.

				// EPOLLIN: There is data to read.
				return true, nil
			}
		}
	}
	// An event occurred that we don't care about.
	return false, nil
}

func (n *linuxCgroupNotifier) Stop() {
	n.stopLock.Lock()
	defer n.stopLock.Unlock()
	select {
	case <-n.stop:
		// the linuxCgroupNotifier is already stopped
		return
	default:
	}
	unix.Close(n.eventfd)
	unix.Close(n.epfd)
	close(n.stop)
}
<commit_msg>Use EPOLL/O_CLOEXEC in evicition notifier<commit_after>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package eviction

import (
	"fmt"
	"sync"
	"time"

	"golang.org/x/sys/unix"
	"k8s.io/klog"
)

const (
	// eventSize is the number of bytes returned by a successful read from an eventfd
	// see http://man7.org/linux/man-pages/man2/eventfd.2.html for more information
	eventSize = 8
	// numFdEvents is the number of events we can record at once.
	// If EpollWait finds more than this, they will be missed.
	numFdEvents = 6
)

type linuxCgroupNotifier struct {
	eventfd  int
	epfd     int
	stop     chan struct{}
	stopLock sync.Mutex
}

var _ CgroupNotifier = &linuxCgroupNotifier{}

// NewCgroupNotifier returns a linuxCgroupNotifier, which performs cgroup control operations required
// to receive notifications from the cgroup when the threshold is crossed in either direction.
func NewCgroupNotifier(path, attribute string, threshold int64) (CgroupNotifier, error) {
	var watchfd, eventfd, epfd, controlfd int
	var err error
	watchfd, err = unix.Open(fmt.Sprintf("%s/%s", path, attribute), unix.O_RDONLY|unix.O_CLOEXEC, 0)
	if err != nil {
		return nil, err
	}
	defer unix.Close(watchfd)
	controlfd, err = unix.Open(fmt.Sprintf("%s/cgroup.event_control", path), unix.O_WRONLY|unix.O_CLOEXEC, 0)
	if err != nil {
		return nil, err
	}
	defer unix.Close(controlfd)
	eventfd, err = unix.Eventfd(0, unix.EFD_CLOEXEC)
	if err != nil {
		return nil, err
	}
	if eventfd < 0 {
		err = fmt.Errorf("eventfd call failed")
		return nil, err
	}
	defer func() {
		// Close eventfd if we get an error later in initialization
		if err != nil {
			unix.Close(eventfd)
		}
	}()
	epfd, err = unix.EpollCreate1(unix.EPOLL_CLOEXEC)
	if err != nil {
		return nil, err
	}
	if epfd < 0 {
		err = fmt.Errorf("EpollCreate1 call failed")
		return nil, err
	}
	defer func() {
		// Close epfd if we get an error later in initialization
		if err != nil {
			unix.Close(epfd)
		}
	}()
	config := fmt.Sprintf("%d %d %d", eventfd, watchfd, threshold)
	_, err = unix.Write(controlfd, []byte(config))
	if err != nil {
		return nil, err
	}
	return &linuxCgroupNotifier{
		eventfd: eventfd,
		epfd:    epfd,
		stop:    make(chan struct{}),
	}, nil
}

func (n *linuxCgroupNotifier) Start(eventCh chan<- struct{}) {
	err := unix.EpollCtl(n.epfd, unix.EPOLL_CTL_ADD, n.eventfd, &unix.EpollEvent{
		Fd:     int32(n.eventfd),
		Events: unix.EPOLLIN,
	})
	if err != nil {
		klog.Warningf("eviction manager: error adding epoll eventfd: %v", err)
		return
	}
	for {
		select {
		case <-n.stop:
			return
		default:
		}
		event, err := wait(n.epfd, n.eventfd, notifierRefreshInterval)
		if err != nil {
			klog.Warningf("eviction manager: error while waiting for memcg events: %v", err)
			return
		} else if !event {
			// Timeout on wait.  This is expected if the threshold was not crossed
			continue
		}
		// Consume the event from the eventfd
		buf := make([]byte, eventSize)
		_, err = unix.Read(n.eventfd, buf)
		if err != nil {
			klog.Warningf("eviction manager: error reading memcg events: %v", err)
			return
		}
		eventCh <- struct{}{}
	}
}

// wait waits up to notifierRefreshInterval for an event on the Epoll FD for the
// eventfd we are concerned about.  It returns an error if one occurrs, and true
// if the consumer should read from the eventfd.
func wait(epfd, eventfd int, timeout time.Duration) (bool, error) {
	events := make([]unix.EpollEvent, numFdEvents+1)
	timeoutMS := int(timeout / time.Millisecond)
	n, err := unix.EpollWait(epfd, events, timeoutMS)
	if n == -1 {
		if err == unix.EINTR {
			// Interrupt, ignore the error
			return false, nil
		}
		return false, err
	}
	if n == 0 {
		// Timeout
		return false, nil
	}
	if n > numFdEvents {
		return false, fmt.Errorf("epoll_wait returned more events than we know what to do with")
	}
	for _, event := range events[:n] {
		if event.Fd == int32(eventfd) {
			if event.Events&unix.EPOLLHUP != 0 || event.Events&unix.EPOLLERR != 0 || event.Events&unix.EPOLLIN != 0 {
				// EPOLLHUP: should not happen, but if it does, treat it as a wakeup.

				// EPOLLERR: If an error is waiting on the file descriptor, we should pretend
				// something is ready to read, and let unix.Read pick up the error.

				// EPOLLIN: There is data to read.
				return true, nil
			}
		}
	}
	// An event occurred that we don't care about.
	return false, nil
}

func (n *linuxCgroupNotifier) Stop() {
	n.stopLock.Lock()
	defer n.stopLock.Unlock()
	select {
	case <-n.stop:
		// the linuxCgroupNotifier is already stopped
		return
	default:
	}
	unix.Close(n.eventfd)
	unix.Close(n.epfd)
	close(n.stop)
}
<|endoftext|>


<commit_before>/*
Copyright 2020 The Rook Authors. All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

	http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package controller

import (
	"context"
	"fmt"
	"reflect"
	"time"

	cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
	"github.com/rook/rook/pkg/clusterd"
	"github.com/rook/rook/pkg/operator/k8sutil"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/types"
	"sigs.k8s.io/controller-runtime/pkg/client"
	"sigs.k8s.io/controller-runtime/pkg/reconcile"
)

// OperatorSettingConfigMapName refers to ConfigMap that configures rook ceph operator
const OperatorSettingConfigMapName string = "rook-ceph-operator-config"

var (
	// ImmediateRetryResult Return this for a immediate retry of the reconciliation loop with the same request object.
	ImmediateRetryResult = reconcile.Result{Requeue: true}

	// WaitForRequeueIfCephClusterNotReady waits for the CephCluster to be ready
	WaitForRequeueIfCephClusterNotReady = reconcile.Result{Requeue: true, RequeueAfter: 10 * time.Second}

	// WaitForRequeueIfFinalizerBlocked waits for resources to be cleaned up before the finalizer can be removed
	WaitForRequeueIfFinalizerBlocked = reconcile.Result{Requeue: true, RequeueAfter: 10 * time.Second}

	// OperatorCephBaseImageVersion is the ceph version in the operator image
	OperatorCephBaseImageVersion string
)

// IsReadyToReconcile determines if a controller is ready to reconcile or not
func IsReadyToReconcile(c client.Client, clustercontext *clusterd.Context, namespacedName types.NamespacedName, controllerName string) (cephv1.CephCluster, bool, bool, reconcile.Result) {
	cephClusterExists := false

	// Running ceph commands won't work and the controller will keep re-queuing so I believe it's fine not to check
	// Make sure a CephCluster exists before doing anything
	var cephCluster cephv1.CephCluster
	clusterList := &cephv1.CephClusterList{}
	err := c.List(context.TODO(), clusterList, client.InNamespace(namespacedName.Namespace))
	if err != nil {
		logger.Errorf("%q:failed to fetch CephCluster %v", controllerName, err)
		return cephCluster, false, cephClusterExists, ImmediateRetryResult
	}
	if len(clusterList.Items) == 0 {
		logger.Errorf("%q: no CephCluster resource found in namespace %q", controllerName, namespacedName.Namespace)
		return cephCluster, false, cephClusterExists, WaitForRequeueIfCephClusterNotReady
	}
	cephClusterExists = true
	cephCluster = clusterList.Items[0]

	logger.Debugf("%q: CephCluster resource %q found in namespace %q", controllerName, cephCluster.Name, namespacedName.Namespace)

	// read the CR status of the cluster
	if cephCluster.Status.CephStatus != nil {
		if cephCluster.Status.CephStatus.Health == "HEALTH_OK" || cephCluster.Status.CephStatus.Health == "HEALTH_WARN" {
			logger.Debugf("%q: ceph status is %q, operator is ready to run ceph command, reconciling", controllerName, cephCluster.Status.CephStatus.Health)
			return cephCluster, true, cephClusterExists, WaitForRequeueIfCephClusterNotReady
		}
		logger.Infof("%s: CephCluster %q found but skipping reconcile since ceph health is %q", controllerName, cephCluster.Name, cephCluster.Status.CephStatus)
	}

	return cephCluster, false, cephClusterExists, WaitForRequeueIfCephClusterNotReady
}

// ClusterOwnerRef represents the owner reference of the CephCluster CR
func ClusterOwnerRef(clusterName, clusterID string) metav1.OwnerReference {
	blockOwner := true
	return metav1.OwnerReference{
		APIVersion:         fmt.Sprintf("%s/%s", ClusterResource.Group, ClusterResource.Version),
		Kind:               ClusterResource.Kind,
		Name:               clusterName,
		UID:                types.UID(clusterID),
		BlockOwnerDeletion: &blockOwner,
	}
}

// ClusterResource operator-kit Custom Resource Definition
var ClusterResource = k8sutil.CustomResource{
	Name:       "cephcluster",
	Plural:     "cephclusters",
	Group:      cephv1.CustomResourceGroup,
	Version:    cephv1.Version,
	Kind:       reflect.TypeOf(cephv1.CephCluster{}).Name(),
	APIVersion: fmt.Sprintf("%s/%s", cephv1.CustomResourceGroup, cephv1.Version),
}
<commit_msg>ceph: do not log an error if no clusters<commit_after>/*
Copyright 2020 The Rook Authors. All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

	http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package controller

import (
	"context"
	"fmt"
	"reflect"
	"time"

	cephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
	"github.com/rook/rook/pkg/clusterd"
	"github.com/rook/rook/pkg/operator/k8sutil"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/types"
	"sigs.k8s.io/controller-runtime/pkg/client"
	"sigs.k8s.io/controller-runtime/pkg/reconcile"
)

// OperatorSettingConfigMapName refers to ConfigMap that configures rook ceph operator
const OperatorSettingConfigMapName string = "rook-ceph-operator-config"

var (
	// ImmediateRetryResult Return this for a immediate retry of the reconciliation loop with the same request object.
	ImmediateRetryResult = reconcile.Result{Requeue: true}

	// WaitForRequeueIfCephClusterNotReady waits for the CephCluster to be ready
	WaitForRequeueIfCephClusterNotReady = reconcile.Result{Requeue: true, RequeueAfter: 10 * time.Second}

	// WaitForRequeueIfFinalizerBlocked waits for resources to be cleaned up before the finalizer can be removed
	WaitForRequeueIfFinalizerBlocked = reconcile.Result{Requeue: true, RequeueAfter: 10 * time.Second}

	// OperatorCephBaseImageVersion is the ceph version in the operator image
	OperatorCephBaseImageVersion string
)

// IsReadyToReconcile determines if a controller is ready to reconcile or not
func IsReadyToReconcile(c client.Client, clustercontext *clusterd.Context, namespacedName types.NamespacedName, controllerName string) (cephv1.CephCluster, bool, bool, reconcile.Result) {
	cephClusterExists := false

	// Running ceph commands won't work and the controller will keep re-queuing so I believe it's fine not to check
	// Make sure a CephCluster exists before doing anything
	var cephCluster cephv1.CephCluster
	clusterList := &cephv1.CephClusterList{}
	err := c.List(context.TODO(), clusterList, client.InNamespace(namespacedName.Namespace))
	if err != nil {
		logger.Errorf("%q: failed to fetch CephCluster %v", controllerName, err)
		return cephCluster, false, cephClusterExists, ImmediateRetryResult
	}
	if len(clusterList.Items) == 0 {
		logger.Debugf("%q: no CephCluster resource found in namespace %q", controllerName, namespacedName.Namespace)
		return cephCluster, false, cephClusterExists, WaitForRequeueIfCephClusterNotReady
	}
	cephClusterExists = true
	cephCluster = clusterList.Items[0]

	logger.Debugf("%q: CephCluster resource %q found in namespace %q", controllerName, cephCluster.Name, namespacedName.Namespace)

	// read the CR status of the cluster
	if cephCluster.Status.CephStatus != nil {
		if cephCluster.Status.CephStatus.Health == "HEALTH_OK" || cephCluster.Status.CephStatus.Health == "HEALTH_WARN" {
			logger.Debugf("%q: ceph status is %q, operator is ready to run ceph command, reconciling", controllerName, cephCluster.Status.CephStatus.Health)
			return cephCluster, true, cephClusterExists, WaitForRequeueIfCephClusterNotReady
		}
		logger.Infof("%s: CephCluster %q found but skipping reconcile since ceph health is %q", controllerName, cephCluster.Name, cephCluster.Status.CephStatus)
	}

	return cephCluster, false, cephClusterExists, WaitForRequeueIfCephClusterNotReady
}

// ClusterOwnerRef represents the owner reference of the CephCluster CR
func ClusterOwnerRef(clusterName, clusterID string) metav1.OwnerReference {
	blockOwner := true
	return metav1.OwnerReference{
		APIVersion:         fmt.Sprintf("%s/%s", ClusterResource.Group, ClusterResource.Version),
		Kind:               ClusterResource.Kind,
		Name:               clusterName,
		UID:                types.UID(clusterID),
		BlockOwnerDeletion: &blockOwner,
	}
}

// ClusterResource operator-kit Custom Resource Definition
var ClusterResource = k8sutil.CustomResource{
	Name:       "cephcluster",
	Plural:     "cephclusters",
	Group:      cephv1.CustomResourceGroup,
	Version:    cephv1.Version,
	Kind:       reflect.TypeOf(cephv1.CephCluster{}).Name(),
	APIVersion: fmt.Sprintf("%s/%s", cephv1.CustomResourceGroup, cephv1.Version),
}
<|endoftext|>


<commit_before>package main

import (
	"log"
	"net/http"
	"os"
	"time"

	"github.com/asamy/steam"
)

func main() {
	log.SetFlags(log.LstdFlags | log.Lshortfile)

	timeTip, err := steam.GetTimeTip()
	if err != nil {
		log.Fatal(err)
	}
	log.Printf("Time tip: %#v\n", timeTip)
	timeDiff := time.Duration(timeTip.Time - time.Now().Unix())

	session := steam.NewSession(&http.Client{}, "")
	if err := session.Login(os.Getenv("steamAccount"), os.Getenv("steamPassword"), os.Getenv("steamSharedSecret"), timeDiff); err != nil {
		log.Fatal(err)
	}
	log.Print("Login successful")

	sid := steam.SteamID(76561198078821986)
	apps, err := session.GetInventoryAppStats(sid)
	if err != nil {
		log.Fatal(err)
	}

	for _, v := range apps {
		log.Printf("-- AppID total asset count: %d\n", v.AssetCount)
		for _, context := range v.Contexts {
			log.Printf("-- Items on %d %d (count %d)\n", v.AppID, context.ID, context.AssetCount)
			inven, err := session.GetInventory(sid, v.AppID, context.ID, false)
			if err != nil {
				log.Fatal(err)
			}

			for _, item := range inven {
				log.Printf("Item: %s = %d\n", item.Desc.MarketHashName, item.AssetID)
			}
		}
	}

	log.Println("Bye!")
}
<commit_msg>Change inventory example<commit_after>package main

import (
	"log"
	"net/http"
	"os"
	"time"

	"github.com/asamy/steam"
)

func main() {
	log.SetFlags(log.LstdFlags | log.Lshortfile)

	timeTip, err := steam.GetTimeTip()
	if err != nil {
		log.Fatal(err)
	}
	log.Printf("Time tip: %#v\n", timeTip)
	timeDiff := time.Duration(timeTip.Time - time.Now().Unix())

	session := steam.NewSession(&http.Client{}, "")
	if err := session.Login(os.Getenv("steamAccount"), os.Getenv("steamPassword"), os.Getenv("steamSharedSecret"), timeDiff); err != nil {
		log.Fatal(err)
	}
	log.Print("Login successful")

	sid := steam.SteamID(76561198078821986)
	apps, err := session.GetInventoryAppStats(sid)
	if err != nil {
		log.Fatal(err)
	}

	for _, v := range apps {
		log.Printf("-- AppID total asset count: %d\n", v.AssetCount)
		for _, context := range v.Contexts {
			log.Printf("-- Items on %d %d (count %d)\n", v.AppID, context.ID, context.AssetCount)
			inven, err := session.GetInventory(sid, v.AppID, context.ID, steam.LangEng)
			if err != nil {
				log.Fatal(err)
			}

			for _, item := range inven {
				log.Printf("Item: %s = %d\n", item.Name.MarketHash, item.AssetID)
			}
		}
	}

	log.Println("Bye!")
}
<|endoftext|>


<commit_before>package pop

import (
	"fmt"
	"os/exec"

	_ "github.com/go-sql-driver/mysql"
	"github.com/markbates/going/clam"
	. "github.com/markbates/pop/columns"
	"github.com/markbates/pop/fizz"
	"github.com/markbates/pop/fizz/translators"
	"github.com/pkg/errors"
)

type mysql struct {
	ConnectionDetails *ConnectionDetails
}

func (m *mysql) Details() *ConnectionDetails {
	return m.ConnectionDetails
}

func (m *mysql) URL() string {
	c := m.ConnectionDetails
	if c.URL != "" {
		return c.URL
	}

	s := "%s:%s@(%s:%s)/%s?parseTime=true&multiStatements=true&readTimeout=1s"
	return fmt.Sprintf(s, c.User, c.Password, c.Host, c.Port, c.Database)
}

func (m *mysql) MigrationURL() string {
	return m.URL()
}

func (m *mysql) Create(s store, model *Model, cols Columns) error {
	return errors.Wrap(genericCreate(s, model, cols), "mysql create")
}

func (m *mysql) Update(s store, model *Model, cols Columns) error {
	return errors.Wrap(genericUpdate(s, model, cols), "mysql update")
}

func (m *mysql) Destroy(s store, model *Model) error {
	return errors.Wrap(genericDestroy(s, model), "mysql destroy")
}

func (m *mysql) SelectOne(s store, model *Model, query Query) error {
	return errors.Wrap(genericSelectOne(s, model, query), "mysql select one")
}

func (m *mysql) SelectMany(s store, models *Model, query Query) error {
	return errors.Wrap(genericSelectMany(s, models, query), "mysql select many")
}

func (m *mysql) CreateDB() error {
	c := m.ConnectionDetails
	cmd := exec.Command("mysql", "-u", c.User, "-p"+c.Password, "-e", fmt.Sprintf("create database %s", c.Database))
	err := clam.RunAndListen(cmd, func(s string) {
		fmt.Println(s)
	})
	return errors.Wrapf(err, "error creating MySQL database %s", c.Database)
}

func (m *mysql) DropDB() error {
	c := m.ConnectionDetails
	cmd := exec.Command("mysql", "-u", c.User, "-p"+c.Password, "-e", fmt.Sprintf("drop database %s", c.Database))
	err := clam.RunAndListen(cmd, func(s string) {
		fmt.Println(s)
	})
	return errors.Wrapf(err, "error dropping MySQL database %s", c.Database)
}

func (m *mysql) TranslateSQL(sql string) string {
	return sql
}

func (m *mysql) FizzTranslator() fizz.Translator {
	t := translators.NewMySQL(m.URL(), m.Details().Database)
	return t
}

func (m *mysql) Lock(fn func() error) error {
	return fn()
}

func newMySQL(deets *ConnectionDetails) dialect {
	cd := &mysql{
		ConnectionDetails: deets,
	}

	return cd
}
<commit_msg>specify host and port for mysql dialect creation and drop<commit_after>package pop

import (
	"fmt"
	"os/exec"

	_ "github.com/go-sql-driver/mysql"
	"github.com/markbates/going/clam"
	. "github.com/markbates/pop/columns"
	"github.com/markbates/pop/fizz"
	"github.com/markbates/pop/fizz/translators"
	"github.com/pkg/errors"
)

type mysql struct {
	ConnectionDetails *ConnectionDetails
}

func (m *mysql) Details() *ConnectionDetails {
	return m.ConnectionDetails
}

func (m *mysql) URL() string {
	c := m.ConnectionDetails
	if c.URL != "" {
		return c.URL
	}

	s := "%s:%s@(%s:%s)/%s?parseTime=true&multiStatements=true&readTimeout=1s"
	return fmt.Sprintf(s, c.User, c.Password, c.Host, c.Port, c.Database)
}

func (m *mysql) MigrationURL() string {
	return m.URL()
}

func (m *mysql) Create(s store, model *Model, cols Columns) error {
	return errors.Wrap(genericCreate(s, model, cols), "mysql create")
}

func (m *mysql) Update(s store, model *Model, cols Columns) error {
	return errors.Wrap(genericUpdate(s, model, cols), "mysql update")
}

func (m *mysql) Destroy(s store, model *Model) error {
	return errors.Wrap(genericDestroy(s, model), "mysql destroy")
}

func (m *mysql) SelectOne(s store, model *Model, query Query) error {
	return errors.Wrap(genericSelectOne(s, model, query), "mysql select one")
}

func (m *mysql) SelectMany(s store, models *Model, query Query) error {
	return errors.Wrap(genericSelectMany(s, models, query), "mysql select many")
}

func (m *mysql) CreateDB() error {
	c := m.ConnectionDetails
	cmd := exec.Command("mysql", "-u", c.User, "-p"+c.Password, "-h", c.Host, "-P", c.Port, "-e", fmt.Sprintf("create database %s", c.Database))
	err := clam.RunAndListen(cmd, func(s string) {
		fmt.Println(s)
	})
	return errors.Wrapf(err, "error creating MySQL database %s", c.Database)
}

func (m *mysql) DropDB() error {
	c := m.ConnectionDetails
	cmd := exec.Command("mysql", "-u", c.User, "-p"+c.Password, "-h", c.Host, "-P", c.Port, "-e", fmt.Sprintf("drop database %s", c.Database))
	err := clam.RunAndListen(cmd, func(s string) {
		fmt.Println(s)
	})
	return errors.Wrapf(err, "error dropping MySQL database %s", c.Database)
}

func (m *mysql) TranslateSQL(sql string) string {
	return sql
}

func (m *mysql) FizzTranslator() fizz.Translator {
	t := translators.NewMySQL(m.URL(), m.Details().Database)
	return t
}

func (m *mysql) Lock(fn func() error) error {
	return fn()
}

func newMySQL(deets *ConnectionDetails) dialect {
	cd := &mysql{
		ConnectionDetails: deets,
	}

	return cd
}
<|endoftext|>


<commit_before>package bittorrent

import (
	"fmt"
	"net"
	"testing"

	"github.com/stretchr/testify/require"
)

var (
	b        = []byte{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20}
	expected = "0102030405060708090a0b0c0d0e0f1011121314"
)

var peerStringTestCases = []struct {
	input    Peer
	expected string
}{
	{
		input: Peer{
			ID: PeerIDFromBytes(b),
			IP: IP{
				IP:            net.IPv4(10, 11, 12, 1),
				AddressFamily: IPv4,
			},
			Port: 1234,
		},
		expected: fmt.Sprintf("%s@[10.11.12.1]:1234", expected),
	},
	{
		input: Peer{
			ID: PeerIDFromBytes(b),
			IP: IP{
				IP:            net.ParseIP("2001:db8::ff00:42:8329"),
				AddressFamily: IPv6,
			},
			Port: 1234,
		},
		expected: fmt.Sprintf("%s@[2001:db8::ff00:42:8329]:1234", expected),
	},
}

func TestPeerID_String(t *testing.T) {
	s := PeerIDFromBytes(b).String()
	require.Equal(t, expected, s)
}

func TestInfoHash_String(t *testing.T) {
	s := InfoHashFromBytes(b).String()
	require.Equal(t, expected, s)
}

func TestPeer_String(t *testing.T) {
	for _, c := range peerStringTestCases {
		got := c.input.String()
		require.Equal(t, c.expected, got)
	}
}
<commit_msg>bittorrent: anonymous fields for IPs in test table<commit_after>package bittorrent

import (
	"fmt"
	"net"
	"testing"

	"github.com/stretchr/testify/require"
)

var (
	b        = []byte{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20}
	expected = "0102030405060708090a0b0c0d0e0f1011121314"
)

var peerStringTestCases = []struct {
	input    Peer
	expected string
}{
	{
		input: Peer{
			ID:   PeerIDFromBytes(b),
			IP:   IP{net.IPv4(10, 11, 12, 1), IPv4},
			Port: 1234,
		},
		expected: fmt.Sprintf("%s@[10.11.12.1]:1234", expected),
	},
	{
		input: Peer{
			ID:   PeerIDFromBytes(b),
			IP:   IP{net.ParseIP("2001:db8::ff00:42:8329"), IPv6},
			Port: 1234,
		},
		expected: fmt.Sprintf("%s@[2001:db8::ff00:42:8329]:1234", expected),
	},
}

func TestPeerID_String(t *testing.T) {
	s := PeerIDFromBytes(b).String()
	require.Equal(t, expected, s)
}

func TestInfoHash_String(t *testing.T) {
	s := InfoHashFromBytes(b).String()
	require.Equal(t, expected, s)
}

func TestPeer_String(t *testing.T) {
	for _, c := range peerStringTestCases {
		got := c.input.String()
		require.Equal(t, c.expected, got)
	}
}
<|endoftext|>


<commit_before>package help

import (
	//	"fmt"
	"github.com/gamelost/bot3server/server"
)

type HelpService struct{}

func (svc *HelpService) NewService() server.BotHandler {
	return &HelpService{}
}

func (svc *HelpService) Handle(botRequest *server.BotRequest, botResponse *server.BotResponse) {

	botResponse.SetSingleLineResponse("i wont make you coffee and give a reach-around but you can ask the following: !remindme !fight !cah !inconceivable !slap !sleep !weather !forecast !nextwedding !zed")
}
<commit_msg>removing !sleep from text<commit_after>package help

import (
	//	"fmt"
	"github.com/gamelost/bot3server/server"
)

type HelpService struct{}

func (svc *HelpService) NewService() server.BotHandler {
	return &HelpService{}
}

func (svc *HelpService) Handle(botRequest *server.BotRequest, botResponse *server.BotResponse) {

	botResponse.SetSingleLineResponse("i wont make you coffee and give a reach-around but you can ask the following: !remindme !fight !cah !inconceivable !slap !weather !forecast !nextwedding !zed")
}
<|endoftext|>


<commit_before>package Mongo

import (
	"time"

	"github.com/chadit/GoShare"

	"gopkg.in/mgo.v2/bson"
)

// Tenant model
type Tenant struct {
	// (ReadOnly) Id of the document, created by system utilizing Mongo Bson Id
	ID string `json:"Id"  bson:"_id" binding:"required"`
	// (ReadOnly) Date the document was created (UTC)
	DateCreated time.Time `json:"DateCreated" bson:"DateCreated" binding:"required"`
	// (ReadOnly) Date the document was modified (UTC)
	DateModified time.Time `json:"DateModified" bson:"DateModified"`
	// (ReadOnly) What user modified the document
	UserModified string `json:"UserModified" bson:"UserModified,omitempty"`
	// (ReadOnly) Has the record been deleted
	IsDeleted bool `json:"IsDeleted" bson:"IsDeleted,omitempty"`
	// (Migration/Sync only)ExternalId is used to sync with external systems (Not used for anything internally)
	ExternalID string `json:"ExternalId" bson:"ExternalId,omitempty"`
	// (Migration/Sync only)ExternalNumber is used to sync with external systems (Not used for anything internally)
	ExternalNumber string `json:"ExternalNumber" bson:"ExternalNumber,omitempty"`
	// TenantID pertains to the tenant the document belongs to.  this is not serialized out
	TenantID string `json:"-" bson:"TenantId"`

	APIKey           string      `json:"ApiKey" bson:"ApiKey" binding:"required"`
	Name             string      `json:"Name"  bson:"Name" binding:"required"`
	FullName         string      `json:"FullName"  bson:"FullName" binding:"required"`
	SubDomain        string      `json:"SubDomain"  bson:"SubDomain" binding:"required"`
	Comments         string      `json:"Comments"  bson:"Comments"`
	DatabaseNode     string      `json:"DatabaseNode"  bson:"DatabaseNode"`
	DatabaseName     string      `json:"DatabaseName"  bson:"DatabaseName"`
	IsEnabled        bool        `json:"IsEnabled"  bson:"IsEnabled"`
	PrimaryContactID string      `json:"PrimaryContactId"  bson:"PrimaryContactId"`
	Status           string      `json:"Status"  bson:"Status"`
	CustomerCode     string      `json:"CustomerCode"  bson:"CustomerCode"`
	Address          interface{} `json:"Address"  bson:"Address"`
	PhoneNumber      interface{} `json:"PhoneNumber"  bson:"PhoneNumber"`
	DispatchSystem   string      `json:"DispatchSystem"  bson:"DispatchSystem"`
}

// NewTenant gets a new object
func NewTenant(tenantID string) *Tenant {
	newItem := new(Tenant)
	newItem.Init(tenantID)
	return newItem
}

// Init sets the defaults
func (p *Tenant) Init(tenantID string) {
	eventTime := time.Now().UTC()
	p.ID = GetNewBsonIDString()
	if p.DateCreated.IsZero() {
		p.DateCreated = eventTime
	}
	p.DateModified = eventTime
	p.IsDeleted = false
	p.TenantID = tenantID
}

// SetupSave updates the object
func (p *Tenant) SetupSave(tenantID string) {
	eventTime := time.Now().UTC()
	if p.ID == "" {
		p.ID = bson.NewObjectId().Hex()
	}

	if p.DateCreated.IsZero() {
		p.DateCreated = eventTime
	}

	p.DateModified = eventTime
	p.TenantID = tenantID
}

// TenantConnectionInfo model
type TenantConnectionInfo struct {
	Tenant           Tenant `json:"-" bson:"-"`
	ConnectionString string `json:"-"  bson:"-"`
	DatabaseNode     string `json:"-"  bson:"-"`
	DatabaseName     string `json:"-"  bson:"-"`
	UserModified     string `json:"-" bson:"-"`
	CollectionName   string `json:"-" bson:"-"`
}

// InitBaseTenantConnectionInfo sets the defaults for a new userlogin
func (p *TenantConnectionInfo) InitBaseTenantConnectionInfo() {
	p.ConnectionString = Shares.MongoConnection
}

// Node model used to hold tenant connection information
type Node struct {
	// (ReadOnly) Id of the document, created by system utilizing Mongo Bson Id
	ID string `json:"Id"  bson:"_id" binding:"required"`
	// (ReadOnly) Date the document was created (UTC)
	DateCreated time.Time `json:"DateCreated" bson:"DateCreated" binding:"required"`
	// (ReadOnly) Date the document was modified (UTC)
	DateModified time.Time `json:"DateModified" bson:"DateModified"`
	// (ReadOnly) What user modified the document
	UserModified string `json:"UserModified" bson:"UserModified,omitempty"`
	// (ReadOnly) Has the record been deleted
	IsDeleted bool `json:"IsDeleted" bson:"IsDeleted,omitempty"`
	// (Migration/Sync only)ExternalId is used to sync with external systems (Not used for anything internally)
	ExternalID string `json:"ExternalId" bson:"ExternalId,omitempty"`
	// (Migration/Sync only)ExternalNumber is used to sync with external systems (Not used for anything internally)
	ExternalNumber string `json:"ExternalNumber" bson:"ExternalNumber,omitempty"`
	// TenantID pertains to the tenant the document belongs to.  this is not serialized out
	TenantID   string `json:"-" bson:"TenantId"`
	Name       string `json:"Name" bson:"Name" binding:"required"`
	Connection string `json:"Connection"  bson:"Connection" binding:"required"`
}

// InitNode sets the defaults for a new node
func (p *Node) InitNode() {
	eventTime := time.Now().UTC()
	p.ID = bson.NewObjectId().Hex()
	p.DateCreated = eventTime
	p.DateModified = eventTime
	p.IsDeleted = false
}

// GetBaseConnectionInformation will attempt to locate the tenant by provided informaiton
func GetBaseConnectionInformation(collectionName string) (*TenantConnectionInfo, error) {
	item := new(Tenant)
	item.ID = "global"
	item.Name = "global"
	baseConnectionInfo := new(TenantConnectionInfo)
	baseConnectionInfo.InitBaseTenantConnectionInfo()
	baseConnectionInfo.Tenant = *item
	baseConnectionInfo.CollectionName = collectionName

	return baseConnectionInfo, nil
}
<commit_msg>changed the connection string<commit_after>package Mongo

import (
	"time"

	"github.com/chadit/GoShare"

	"gopkg.in/mgo.v2/bson"
)

// Tenant model
type Tenant struct {
	// (ReadOnly) Id of the document, created by system utilizing Mongo Bson Id
	ID string `json:"Id"  bson:"_id" binding:"required"`
	// (ReadOnly) Date the document was created (UTC)
	DateCreated time.Time `json:"DateCreated" bson:"DateCreated" binding:"required"`
	// (ReadOnly) Date the document was modified (UTC)
	DateModified time.Time `json:"DateModified" bson:"DateModified"`
	// (ReadOnly) What user modified the document
	UserModified string `json:"UserModified" bson:"UserModified,omitempty"`
	// (ReadOnly) Has the record been deleted
	IsDeleted bool `json:"IsDeleted" bson:"IsDeleted,omitempty"`
	// (Migration/Sync only)ExternalId is used to sync with external systems (Not used for anything internally)
	ExternalID string `json:"ExternalId" bson:"ExternalId,omitempty"`
	// (Migration/Sync only)ExternalNumber is used to sync with external systems (Not used for anything internally)
	ExternalNumber string `json:"ExternalNumber" bson:"ExternalNumber,omitempty"`
	// TenantID pertains to the tenant the document belongs to.  this is not serialized out
	TenantID string `json:"-" bson:"TenantId"`

	APIKey           string      `json:"ApiKey" bson:"ApiKey" binding:"required"`
	Name             string      `json:"Name"  bson:"Name" binding:"required"`
	FullName         string      `json:"FullName"  bson:"FullName" binding:"required"`
	SubDomain        string      `json:"SubDomain"  bson:"SubDomain" binding:"required"`
	Comments         string      `json:"Comments"  bson:"Comments"`
	DatabaseNode     string      `json:"DatabaseNode"  bson:"DatabaseNode"`
	DatabaseName     string      `json:"DatabaseName"  bson:"DatabaseName"`
	IsEnabled        bool        `json:"IsEnabled"  bson:"IsEnabled"`
	PrimaryContactID string      `json:"PrimaryContactId"  bson:"PrimaryContactId"`
	Status           string      `json:"Status"  bson:"Status"`
	CustomerCode     string      `json:"CustomerCode"  bson:"CustomerCode"`
	Address          interface{} `json:"Address"  bson:"Address"`
	PhoneNumber      interface{} `json:"PhoneNumber"  bson:"PhoneNumber"`
	DispatchSystem   string      `json:"DispatchSystem"  bson:"DispatchSystem"`
}

// NewTenant gets a new object
func NewTenant(tenantID string) Tenant {
	newItem := Tenant{}
	newItem.Init(tenantID)
	return newItem
}

// Init sets the defaults
func (p *Tenant) Init(tenantID string) {
	eventTime := time.Now().UTC()
	p.ID = GetNewBsonIDString()
	if p.DateCreated.IsZero() {
		p.DateCreated = eventTime
	}
	p.DateModified = eventTime
	p.IsDeleted = false
	p.TenantID = tenantID
}

// SetupSave updates the object
func (p *Tenant) SetupSave(tenantID string) {
	eventTime := time.Now().UTC()
	if p.ID == "" {
		p.ID = bson.NewObjectId().Hex()
	}

	if p.DateCreated.IsZero() {
		p.DateCreated = eventTime
	}

	p.DateModified = eventTime
	p.TenantID = tenantID
}

// TenantConnectionInfo model
type TenantConnectionInfo struct {
	Tenant           Tenant `json:"-" bson:"-"`
	ConnectionString string `json:"-"  bson:"-"`
	DatabaseNode     string `json:"-"  bson:"-"`
	DatabaseName     string `json:"-"  bson:"-"`
	UserModified     string `json:"-" bson:"-"`
	CollectionName   string `json:"-" bson:"-"`
}

// InitBaseTenantConnectionInfo sets the defaults for a new userlogin
func (p *TenantConnectionInfo) InitBaseTenantConnectionInfo() {
	p.ConnectionString = Shares.MongoConnection
}

// Node model used to hold tenant connection information
type Node struct {
	// (ReadOnly) Id of the document, created by system utilizing Mongo Bson Id
	ID string `json:"Id"  bson:"_id" binding:"required"`
	// (ReadOnly) Date the document was created (UTC)
	DateCreated time.Time `json:"DateCreated" bson:"DateCreated" binding:"required"`
	// (ReadOnly) Date the document was modified (UTC)
	DateModified time.Time `json:"DateModified" bson:"DateModified"`
	// (ReadOnly) What user modified the document
	UserModified string `json:"UserModified" bson:"UserModified,omitempty"`
	// (ReadOnly) Has the record been deleted
	IsDeleted bool `json:"IsDeleted" bson:"IsDeleted,omitempty"`
	// (Migration/Sync only)ExternalId is used to sync with external systems (Not used for anything internally)
	ExternalID string `json:"ExternalId" bson:"ExternalId,omitempty"`
	// (Migration/Sync only)ExternalNumber is used to sync with external systems (Not used for anything internally)
	ExternalNumber string `json:"ExternalNumber" bson:"ExternalNumber,omitempty"`
	// TenantID pertains to the tenant the document belongs to.  this is not serialized out
	TenantID   string `json:"-" bson:"TenantId"`
	Name       string `json:"Name" bson:"Name" binding:"required"`
	Connection string `json:"Connection"  bson:"Connection" binding:"required"`
}

// InitNode sets the defaults for a new node
func (p *Node) InitNode() {
	eventTime := time.Now().UTC()
	p.ID = bson.NewObjectId().Hex()
	p.DateCreated = eventTime
	p.DateModified = eventTime
	p.IsDeleted = false
}

// GetBaseConnectionInformation will attempt to locate the tenant by provided informaiton
func GetBaseConnectionInformation(collectionName string) (TenantConnectionInfo, error) {
	item := NewTenant("global")
	item.ID = "global"
	item.Name = "global"
	baseConnectionInfo := TenantConnectionInfo{}
	baseConnectionInfo.InitBaseTenantConnectionInfo()
	baseConnectionInfo.Tenant = item
	baseConnectionInfo.CollectionName = collectionName

	return baseConnectionInfo, nil
}
<|endoftext|>


<commit_before>package main

import (
	"github.com/hashicorp/terraform/helper/schema"
	"github.com/iron-io/iron_go3/config"
	"github.com/iron-io/iron_go3/mq"
)

func queueSchema() *schema.Resource {
	return &schema.Resource{
		Create: createQueue,
		Read:   readQueue,
		//Update: updateQueue,
		Delete: deleteQueue,
		Schema: map[string]*schema.Schema{
			"name": {
				Type:        schema.TypeString,
				Required:    true,
				ForceNew:    true,
				Description: "name of the queue",
			},
		},
	}
}

func createQueue(data *schema.ResourceData, meta interface{}) error {
	cfg := meta.(config.Settings)
	name := data.Get("name").(string)
	_, err := mq.ConfigCreateQueue(mq.QueueInfo{
		Name: name,
	}, &cfg)
	return err
}

func updateQueue(data *schema.ResourceData, meta interface{}) error {
	return nil
}

func readQueue(data *schema.ResourceData, ironcfg interface{}) error {
	return nil
}

func deleteQueue(data *schema.ResourceData, ironcfg interface{}) error {
	return nil
}
<commit_msg>Keep states<commit_after>package main

import (
	"fmt"

	"github.com/hashicorp/terraform/helper/schema"
	"github.com/iron-io/iron_go3/config"
	"github.com/iron-io/iron_go3/mq"
)

func queueSchema() *schema.Resource {
	return &schema.Resource{
		Create: createQueue,
		Read:   readQueue,
		//Update: updateQueue,
		Delete: deleteQueue,
		Schema: map[string]*schema.Schema{
			"name": {
				Type:        schema.TypeString,
				Required:    true,
				ForceNew:    true,
				Description: "name of the queue",
			},
		},
	}
}

func createQueue(data *schema.ResourceData, meta interface{}) error {
	cfg := meta.(config.Settings)
	name := data.Get("name").(string)
	_, err := mq.ConfigCreateQueue(mq.QueueInfo{
		Name: name,
	}, &cfg)
	if err != nil {
		return err
	}
	if err := readQueue(data, meta); err != nil {
		return err
	}
	return nil
}

func updateQueue(data *schema.ResourceData, meta interface{}) error {
	return nil
}

func readQueue(data *schema.ResourceData, meta interface{}) error {
	cfg := meta.(config.Settings)
	name := data.Get("name").(string)
	data.SetId(fmt.Sprintf("%s/%s", cfg.ProjectId, name))
	return nil
}

func deleteQueue(data *schema.ResourceData, ironcfg interface{}) error {
	return nil
}
<|endoftext|>


<commit_before>package pigae

import (
	"fmt"
	"os"
	"sort"
	"strings"

	"github.com/Deleplace/programming-idioms/pig"
)

//
// Language names exist in 3 forms : nice, standard, lowercase
// Ex : "C++", "Cpp", "cpp"
//

var mainStreamLangs = [...]string{"C", "Cpp", "Csharp", "Go", "Java", "JS", "PHP", "Python"}

// Return alpha codes for each language (no encoding problems).
// See printNiceLang to display them more fancy.
func mainStreamLanguages() []string {
	return mainStreamLangs[:]
}

var moreLangs = [...]string{"Ada", "Caml", "Clojure", "Cobol", "D", "Dart", "Delphi", "Erlang", "Fortran", "Groovy", "Haskell", "Lua", "Lisp", "Pascal", "Perl", "Prolog", "Ruby", "Rust", "Scala", "Scheme", "Smalltalk", "Obj-C", "VB"}

func moreLanguages() []string {
	// These do *not* include the mainStreamLanguages()
	return moreLangs[:]
}

var synonymLangs = map[string]string{
	"Javascript":   "JS",
	"Objective C":  "Obj-C",
	"Visual Basic": "VB",
}

var allLangs []string
var allNiceLangs []string

func allLanguages() []string {
	if allLangs == nil {
		allLangs = append(mainStreamLanguages(), moreLanguages()...)
		sort.Strings(allLangs)
		allNiceLangs = make([]string, len(allLangs))
		for i, lg := range allLangs {
			allNiceLangs[i] = printNiceLang(lg)
		}
	}
	return allLangs
}

// autocompletions is a map[string][]string
var autocompletions = precomputeAutocompletions()

func languageAutoComplete(fragment string) []string {
	fragment = strings.ToLower(fragment)

	// Dynamic search (slow)
	// options := []string{}
	// for _, lg := range allLanguages() {
	// 	if strings.Contains(strings.ToLower(lg), fragment) || strings.Contains(strings.ToLower(printNiceLang(lg)), fragment) {
	// 		options = append(options, printNiceLang(lg))
	// 	}
	// }
	// return options

	// Precomputed search (fast)
	return autocompletions[fragment]
}

func printNiceLang(lang string) string {
	switch strings.TrimSpace(strings.ToLower(lang)) {
	case "cpp":
		return "C++"
	case "csharp":
		return "C#"
	default:
		return lang
	}
}

func indexByLowerCase(langs []string) map[string]string {
	m := map[string]string{}
	for _, lg := range langs {
		m[strings.ToLower(lg)] = lg
	}
	return m
}

var langLowerCaseIndex = indexByLowerCase(allLanguages())

func normLang(lang string) string {
	lg := strings.TrimSpace(strings.ToLower(lang))
	switch lg {
	case "c++":
		return "Cpp"
	case "c#":
		return "Csharp"
	case "javascript":
		return "JS"
	case "golang":
		return "Go"
	case "objective c":
		return "Obj-C"
	default:
		return langLowerCaseIndex[lg]
	}
}

func precomputeAutocompletions() map[string][]string {
	m := make(map[string][]string, 100)

	// Crazy shadowing of variable "lg" is allowed in go...
	for _, trueLg := range allLanguages() {
		niceLg := printNiceLang(trueLg)
		for _, lg := range []string{trueLg, niceLg} {
			lg := strings.ToLower(lg)
			fragments := substrings(lg)
			for _, frag := range fragments {
				if !pig.StringSliceContains(m[frag], niceLg) {
					m[frag] = append(m[frag], niceLg)
				}
			}
		}
	}

	for lg, trueLg := range synonymLangs {
		niceLg := printNiceLang(trueLg)
		lg := strings.ToLower(lg)
		fragments := substrings(lg)
		for _, frag := range fragments {
			if !pig.StringSliceContains(m[frag], niceLg) {
				m[frag] = append(m[frag], niceLg)
			}
		}
	}

	fmt.Fprintf(os.Stderr, "---\n")
	return m
}

func substrings(s string) []string {
	L := len(s)
	seen := make(map[string]bool, L*L)
	fragments := make([]string, L*L)
	// This works well for language names with only 1-byte chars, not for any string
	for i := 0; i < L; i++ {
		for j := i + 1; j <= L; j++ {
			frag := s[i:j]
			if seen[frag] {
				continue
			}
			seen[frag] = true
			fragments = append(fragments, frag)
		}
	}
	return fragments
}
<commit_msg>Delphi not a specific lang, let's merge with Pascal.<commit_after>package pigae

import (
	"fmt"
	"os"
	"sort"
	"strings"

	"github.com/Deleplace/programming-idioms/pig"
)

//
// Language names exist in 3 forms : nice, standard, lowercase
// Ex : "C++", "Cpp", "cpp"
//

var mainStreamLangs = [...]string{"C", "Cpp", "Csharp", "Go", "Java", "JS", "PHP", "Python"}

// Return alpha codes for each language (no encoding problems).
// See printNiceLang to display them more fancy.
func mainStreamLanguages() []string {
	return mainStreamLangs[:]
}

var moreLangs = [...]string{"Ada", "Caml", "Clojure", "Cobol", "D", "Dart", "Erlang", "Fortran", "Groovy", "Haskell", "Lua", "Lisp", "Pascal", "Perl", "Prolog", "Ruby", "Rust", "Scala", "Scheme", "Smalltalk", "Obj-C", "VB"}

func moreLanguages() []string {
	// These do *not* include the mainStreamLanguages()
	return moreLangs[:]
}

var synonymLangs = map[string]string{
	"Javascript":   "JS",
	"Objective C":  "Obj-C",
	"Visual Basic": "VB",
}

var allLangs []string
var allNiceLangs []string

func allLanguages() []string {
	if allLangs == nil {
		allLangs = append(mainStreamLanguages(), moreLanguages()...)
		sort.Strings(allLangs)
		allNiceLangs = make([]string, len(allLangs))
		for i, lg := range allLangs {
			allNiceLangs[i] = printNiceLang(lg)
		}
	}
	return allLangs
}

// autocompletions is a map[string][]string
var autocompletions = precomputeAutocompletions()

func languageAutoComplete(fragment string) []string {
	fragment = strings.ToLower(fragment)

	// Dynamic search (slow)
	// options := []string{}
	// for _, lg := range allLanguages() {
	// 	if strings.Contains(strings.ToLower(lg), fragment) || strings.Contains(strings.ToLower(printNiceLang(lg)), fragment) {
	// 		options = append(options, printNiceLang(lg))
	// 	}
	// }
	// return options

	// Precomputed search (fast)
	return autocompletions[fragment]
}

func printNiceLang(lang string) string {
	switch strings.TrimSpace(strings.ToLower(lang)) {
	case "cpp":
		return "C++"
	case "csharp":
		return "C#"
	default:
		return lang
	}
}

func indexByLowerCase(langs []string) map[string]string {
	m := map[string]string{}
	for _, lg := range langs {
		m[strings.ToLower(lg)] = lg
	}
	return m
}

var langLowerCaseIndex = indexByLowerCase(allLanguages())

func normLang(lang string) string {
	lg := strings.TrimSpace(strings.ToLower(lang))
	switch lg {
	case "c++":
		return "Cpp"
	case "c#":
		return "Csharp"
	case "javascript":
		return "JS"
	case "golang":
		return "Go"
	case "objective c":
		return "Obj-C"
	default:
		return langLowerCaseIndex[lg]
	}
}

func precomputeAutocompletions() map[string][]string {
	m := make(map[string][]string, 100)

	// Crazy shadowing of variable "lg" is allowed in go...
	for _, trueLg := range allLanguages() {
		niceLg := printNiceLang(trueLg)
		for _, lg := range []string{trueLg, niceLg} {
			lg := strings.ToLower(lg)
			fragments := substrings(lg)
			for _, frag := range fragments {
				if !pig.StringSliceContains(m[frag], niceLg) {
					m[frag] = append(m[frag], niceLg)
				}
			}
		}
	}

	for lg, trueLg := range synonymLangs {
		niceLg := printNiceLang(trueLg)
		lg := strings.ToLower(lg)
		fragments := substrings(lg)
		for _, frag := range fragments {
			if !pig.StringSliceContains(m[frag], niceLg) {
				m[frag] = append(m[frag], niceLg)
			}
		}
	}

	fmt.Fprintf(os.Stderr, "---\n")
	return m
}

func substrings(s string) []string {
	L := len(s)
	seen := make(map[string]bool, L*L)
	fragments := make([]string, L*L)
	// This works well for language names with only 1-byte chars, not for any string
	for i := 0; i < L; i++ {
		for j := i + 1; j <= L; j++ {
			frag := s[i:j]
			if seen[frag] {
				continue
			}
			seen[frag] = true
			fragments = append(fragments, frag)
		}
	}
	return fragments
}
<|endoftext|>


<commit_before>package registry

import (
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"net/http"
	"net/url"
	"strings"
	"time"
)

func NewRef(s string) (*Ref, error) {
	if !strings.Contains(s, "://") {
		s = "https://" + s
	}
	u, err := url.Parse(s)
	if err != nil {
		return nil, err
	}

	q := u.Query()
	if q.Get("tag") != "" && q.Get("id") != "" {
		return nil, fmt.Errorf("registry: only one of id or tag may be provided")
	}

	ref := &Ref{
		tag:     q.Get("tag"),
		repo:    strings.TrimPrefix(u.Path, "/"),
		imageID: q.Get("id"),
		index:   fmt.Sprintf("%s://%s/v1", u.Scheme, u.Host),
		scheme:  u.Scheme,
	}
	if u.User != nil {
		ref.username = u.User.Username()
		ref.password, _ = u.User.Password()
	}
	if ref.tag == "" && ref.imageID == "" {
		ref.tag = "latest"
	}

	return ref, nil
}

type Ref struct {
	username  string
	password  string
	token     string
	index     string
	endpoints []string
	repo      string
	tag       string
	imageID   string
	scheme    string
}

func (r *Ref) ImageID() string {
	return r.imageID
}

func (r *Ref) Get() (*Image, error) {
	req, err := http.NewRequest("GET", fmt.Sprintf("%s/repositories/%s/images", r.index, r.repo), nil)
	if err != nil {
		return nil, err
	}
	req.Header.Set("Accept", "application/json")
	req.Header.Set("X-Docker-Token", "true")
	if r.username != "" {
		req.SetBasicAuth(r.username, r.password)
	}

	res, err := http.DefaultClient.Do(req)
	if err != nil {
		return nil, err
	}
	res.Body.Close()
	if res.StatusCode == 404 {
		return nil, fmt.Errorf("registry: repo not found")
	}
	if res.StatusCode != 200 {
		return nil, fmt.Errorf("registry: unexpected status %d", res.StatusCode)
	}

	if res.Header.Get("X-Docker-Token") != "" {
		r.token = strings.Join(res.Header["X-Docker-Token"], ",")
	}
	if res.Header.Get("X-Docker-Endpoints") != "" {
		r.endpoints = parseEndpoints(res.Header["X-Docker-Endpoints"], r.scheme)
	} else {
		r.endpoints = []string{r.index}
	}

	if r.tag != "" {
		tags, err := r.tags()
		if err != nil {
			return nil, err
		}
		if r.imageID = tags[r.tag]; r.imageID == "" {
			return nil, fmt.Errorf("registry: tag %q not found", r.tag)
		}
	}

	return r.image(r.imageID)
}

func (r *Ref) tags() (map[string]string, error) {
	repo := r.repo
	if !strings.Contains(repo, "/") {
		// silly docker hack: https://github.com/dotcloud/docker/blob/1310243d488cfede2f5765e79b01ab20efd46cc0/registry/registry.go#L278-L282
		repo = "library/" + repo
	}
	var tags map[string]string
	_, err := r.registryGet(fmt.Sprintf("/repositories/%s/tags", repo), &tags)
	return tags, err
}

func (r *Ref) setToken(req *http.Request) {
	if r.token != "" {
		req.Header.Set("Authorization", "Token "+r.token)
	}
}

func (r *Ref) layer(id string) (io.ReadCloser, error) {
	res, err := r.registryGet(fmt.Sprintf("/images/%s/layer", id), nil)
	if err != nil {
		return nil, err
	}
	return res.Body, nil
}

func (r *Ref) image(id string) (*Image, error) {
	img := &Image{ref: r}
	_, err := r.registryGet(fmt.Sprintf("/images/%s/json", id), img)
	return img, err
}

func (r *Ref) registryGet(path string, out interface{}) (*http.Response, error) {
	var err error
	for _, endpoint := range r.endpoints {
		var req *http.Request
		req, err = http.NewRequest("GET", endpoint+path, nil)
		if err != nil {
			continue
		}
		r.setToken(req)
		if out != nil {
			req.Header.Set("Accept", "application/json")
		}

		var res *http.Response
		res, err = http.DefaultClient.Do(req)
		if err != nil {
			continue
		}
		if out != nil {
			defer res.Body.Close()
		}
		if res.StatusCode != 200 {
			err = fmt.Errorf("registry: unexpected status %d", res.StatusCode)
			continue
		}

		if out != nil {
			if err = json.NewDecoder(res.Body).Decode(out); err != nil {
				continue
			}
		}
		return res, nil
	}
	return nil, err
}

func parseEndpoints(headers []string, scheme string) []string {
	var res []string
	for _, h := range headers {
		endpoints := strings.Split(h, ",")
		for _, e := range endpoints {
			res = append(res, fmt.Sprintf("%s://%s/v1", scheme, e))
		}
	}
	return res
}

type Image struct {
	ID              string           `json:"id"`
	ParentID        string           `json:"parent,omitempty"`
	Comment         string           `json:"comment,omitempty"`
	Created         time.Time        `json:"created"`
	Container       string           `json:"container,omitempty"`
	ContainerConfig *json.RawMessage `json:"container_config,omitempty"`
	Config          *json.RawMessage `json:"config,omitempty"`
	DockerVersion   string           `json:"docker_version,omitempty"`
	Author          string           `json:"author,omitempty"`
	Architecture    string           `json:"architecture,omitempty"`
	OS              string           `json:"os,omitempty"`
	Size            int64            `json:"size,omitempty"`

	ref   *Ref
	layer io.ReadCloser
}

func (i *Image) Read(p []byte) (int, error) {
	if i.ref == nil {
		return 0, errors.New("registry: improperly initialized Image")
	}
	if i.layer == nil {
		var err error
		i.layer, err = i.ref.layer(i.ID)
		if err != nil {
			return 0, err
		}
	}
	return i.layer.Read(p)
}

func (i *Image) Fetch() error {
	if i.ref == nil {
		return errors.New("registry: improperly initialized Image")
	}
	_, err := i.ref.registryGet(fmt.Sprintf("/images/%s/json", i.ID), i)
	return err
}

func (i *Image) Close() error {
	if i.layer == nil {
		return nil
	}
	return i.layer.Close()
}

var ErrNoParent = errors.New("registry: image has no parent")

func (img *Image) Ancestors() ([]*Image, error) {
	if img.ParentID == "" {
		return nil, ErrNoParent
	}
	var ids []string
	_, err := img.ref.registryGet(fmt.Sprintf("/images/%s/ancestry", img.ID), &ids)
	if err != nil {
		return nil, err
	}

	res := make([]*Image, len(ids))
	for i, id := range ids {
		res[i] = &Image{ID: id, ref: img.ref}
	}
	return res, nil
}
<commit_msg>pinkerton: Support BasicAuth with Docker endpoints<commit_after>package registry

import (
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"net/http"
	"net/url"
	"strings"
	"time"
)

func NewRef(s string) (*Ref, error) {
	if !strings.Contains(s, "://") {
		s = "https://" + s
	}
	u, err := url.Parse(s)
	if err != nil {
		return nil, err
	}

	q := u.Query()
	if q.Get("tag") != "" && q.Get("id") != "" {
		return nil, fmt.Errorf("registry: only one of id or tag may be provided")
	}

	ref := &Ref{
		tag:     q.Get("tag"),
		repo:    strings.TrimPrefix(u.Path, "/"),
		imageID: q.Get("id"),
		index:   fmt.Sprintf("%s://%s/v1", u.Scheme, u.Host),
		scheme:  u.Scheme,
	}
	if u.User != nil {
		ref.username = u.User.Username()
		ref.password, _ = u.User.Password()
	}
	if ref.tag == "" && ref.imageID == "" {
		ref.tag = "latest"
	}

	return ref, nil
}

type Ref struct {
	username  string
	password  string
	token     string
	index     string
	endpoints []string
	repo      string
	tag       string
	imageID   string
	scheme    string
}

func (r *Ref) ImageID() string {
	return r.imageID
}

func (r *Ref) Get() (*Image, error) {
	req, err := http.NewRequest("GET", fmt.Sprintf("%s/repositories/%s/images", r.index, r.repo), nil)
	if err != nil {
		return nil, err
	}
	req.Header.Set("Accept", "application/json")
	req.Header.Set("X-Docker-Token", "true")
	if r.username != "" {
		req.SetBasicAuth(r.username, r.password)
	}

	res, err := http.DefaultClient.Do(req)
	if err != nil {
		return nil, err
	}
	res.Body.Close()
	if res.StatusCode == 404 {
		return nil, fmt.Errorf("registry: repo not found")
	}
	if res.StatusCode != 200 {
		return nil, fmt.Errorf("registry: unexpected status %d", res.StatusCode)
	}

	if res.Header.Get("X-Docker-Token") != "" {
		r.token = strings.Join(res.Header["X-Docker-Token"], ",")
	}
	if res.Header.Get("X-Docker-Endpoints") != "" {
		r.endpoints = parseEndpoints(res.Header["X-Docker-Endpoints"], r)
	} else {
		r.endpoints = []string{r.index}
	}

	if r.tag != "" {
		tags, err := r.tags()
		if err != nil {
			return nil, err
		}
		if r.imageID = tags[r.tag]; r.imageID == "" {
			return nil, fmt.Errorf("registry: tag %q not found", r.tag)
		}
	}

	return r.image(r.imageID)
}

func (r *Ref) tags() (map[string]string, error) {
	repo := r.repo
	if !strings.Contains(repo, "/") {
		// silly docker hack: https://github.com/dotcloud/docker/blob/1310243d488cfede2f5765e79b01ab20efd46cc0/registry/registry.go#L278-L282
		repo = "library/" + repo
	}
	var tags map[string]string
	_, err := r.registryGet(fmt.Sprintf("/repositories/%s/tags", repo), &tags)
	return tags, err
}

func (r *Ref) setToken(req *http.Request) {
	if r.token != "" {
		req.Header.Set("Authorization", "Token "+r.token)
	}
}

func (r *Ref) layer(id string) (io.ReadCloser, error) {
	res, err := r.registryGet(fmt.Sprintf("/images/%s/layer", id), nil)
	if err != nil {
		return nil, err
	}
	return res.Body, nil
}

func (r *Ref) image(id string) (*Image, error) {
	img := &Image{ref: r}
	_, err := r.registryGet(fmt.Sprintf("/images/%s/json", id), img)
	return img, err
}

func (r *Ref) registryGet(path string, out interface{}) (*http.Response, error) {
	var err error
	for _, endpoint := range r.endpoints {
		var req *http.Request
		req, err = http.NewRequest("GET", endpoint+path, nil)
		if err != nil {
			continue
		}
		r.setToken(req)
		if out != nil {
			req.Header.Set("Accept", "application/json")
		}

		var res *http.Response
		res, err = http.DefaultClient.Do(req)
		if err != nil {
			continue
		}
		if out != nil {
			defer res.Body.Close()
		}
		if res.StatusCode != 200 {
			err = fmt.Errorf("registry: unexpected status %d", res.StatusCode)
			continue
		}

		if out != nil {
			if err = json.NewDecoder(res.Body).Decode(out); err != nil {
				continue
			}
		}
		return res, nil
	}
	return nil, err
}

func parseEndpoints(headers []string, ref *Ref) []string {
	var res []string
	for _, h := range headers {
		endpoints := strings.Split(h, ",")
		for _, e := range endpoints {
			u := &url.URL{Scheme: ref.scheme, Host: e, Path: "/v1"}
			if ref.username != "" {
				u.User = url.UserPassword(ref.username, ref.password)
			}
			res = append(res, u.String())
		}
	}
	return res
}

type Image struct {
	ID              string           `json:"id"`
	ParentID        string           `json:"parent,omitempty"`
	Comment         string           `json:"comment,omitempty"`
	Created         time.Time        `json:"created"`
	Container       string           `json:"container,omitempty"`
	ContainerConfig *json.RawMessage `json:"container_config,omitempty"`
	Config          *json.RawMessage `json:"config,omitempty"`
	DockerVersion   string           `json:"docker_version,omitempty"`
	Author          string           `json:"author,omitempty"`
	Architecture    string           `json:"architecture,omitempty"`
	OS              string           `json:"os,omitempty"`
	Size            int64            `json:"size,omitempty"`

	ref   *Ref
	layer io.ReadCloser
}

func (i *Image) Read(p []byte) (int, error) {
	if i.ref == nil {
		return 0, errors.New("registry: improperly initialized Image")
	}
	if i.layer == nil {
		var err error
		i.layer, err = i.ref.layer(i.ID)
		if err != nil {
			return 0, err
		}
	}
	return i.layer.Read(p)
}

func (i *Image) Fetch() error {
	if i.ref == nil {
		return errors.New("registry: improperly initialized Image")
	}
	_, err := i.ref.registryGet(fmt.Sprintf("/images/%s/json", i.ID), i)
	return err
}

func (i *Image) Close() error {
	if i.layer == nil {
		return nil
	}
	return i.layer.Close()
}

var ErrNoParent = errors.New("registry: image has no parent")

func (img *Image) Ancestors() ([]*Image, error) {
	if img.ParentID == "" {
		return nil, ErrNoParent
	}
	var ids []string
	_, err := img.ref.registryGet(fmt.Sprintf("/images/%s/ancestry", img.ID), &ids)
	if err != nil {
		return nil, err
	}

	res := make([]*Image, len(ids))
	for i, id := range ids {
		res[i] = &Image{ID: id, ref: img.ref}
	}
	return res, nil
}
<|endoftext|>


<commit_before>// Copyright 2019 Authors of Cilium
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package informer

import (
	"fmt"
	"time"

	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/client-go/tools/cache"
)

type ConvertFunc func(obj interface{}) interface{}

// NewInformer is a copy of k8s.io/client-go/tools/cache/NewInformer with a new
// argument which converts an object into another object that can be stored in
// the local cache.
func NewInformer(
	lw cache.ListerWatcher,
	objType runtime.Object,
	resyncPeriod time.Duration,
	h cache.ResourceEventHandler,
	convertFunc ConvertFunc,
) (cache.Store, cache.Controller) {
	// This will hold the client state, as we know it.
	clientState := cache.NewStore(cache.DeletionHandlingMetaNamespaceKeyFunc)

	return clientState, NewInformerWithStore(lw, objType, resyncPeriod, h, convertFunc, clientState)
}

// NewInformerWithStore uses the same arguments as NewInformer for which a
// caller can also set a cache.Store.
func NewInformerWithStore(
	lw cache.ListerWatcher,
	objType runtime.Object,
	resyncPeriod time.Duration,
	h cache.ResourceEventHandler,
	convertFunc ConvertFunc,
	clientState cache.Store,
) cache.Controller {

	// This will hold incoming changes. Note how we pass clientState in as a
	// KeyLister, that way resync operations will result in the correct set
	// of update/delete deltas.
	fifo := cache.NewDeltaFIFO(cache.MetaNamespaceKeyFunc, clientState)

	cacheMutationDetector := cache.NewCacheMutationDetector(fmt.Sprintf("%T", objType))

	cfg := &cache.Config{
		Queue:            fifo,
		ListerWatcher:    lw,
		ObjectType:       objType,
		FullResyncPeriod: resyncPeriod,
		RetryOnError:     false,

		Process: func(obj interface{}) error {
			// from oldest to newest
			for _, d := range obj.(cache.Deltas) {

				obj := convertFunc(d.Object)

				// In CI we detect if the objects were modified and panic
				// this is a no-op in production environments.
				cacheMutationDetector.AddObject(obj)

				switch d.Type {
				case cache.Sync, cache.Added, cache.Updated:
					if old, exists, err := clientState.Get(obj); err == nil && exists {
						if err := clientState.Update(obj); err != nil {
							return err
						}
						h.OnUpdate(old, obj)
					} else {
						if err := clientState.Add(obj); err != nil {
							return err
						}
						h.OnAdd(obj)
					}
				case cache.Deleted:
					if err := clientState.Delete(obj); err != nil {
						return err
					}
					h.OnDelete(obj)
				}
			}
			return nil
		},
	}
	return cache.New(cfg)
}
<commit_msg>k8s/informer: panic Cilium if k8s watcher panic<commit_after>// Copyright 2019 Authors of Cilium
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package informer

import (
	"fmt"
	"net/http"
	"time"

	"github.com/cilium/cilium/pkg/logging"
	"github.com/cilium/cilium/pkg/logging/logfields"

	k8sRuntime "k8s.io/apimachinery/pkg/runtime"
	utilRuntime "k8s.io/apimachinery/pkg/util/runtime"
	"k8s.io/client-go/tools/cache"
)

var log = logging.DefaultLogger.WithField(logfields.LogSubsys, "k8s")

func init() {
	utilRuntime.PanicHandlers = append(
		utilRuntime.PanicHandlers,
		func(r interface{}) {
			// from k8s library
			if r == http.ErrAbortHandler {
				// honor the http.ErrAbortHandler sentinel panic value:
				//   ErrAbortHandler is a sentinel panic value to abort a handler.
				//   While any panic from ServeHTTP aborts the response to the client,
				//   panicking with ErrAbortHandler also suppresses logging of a stack trace to the server's error log.
				return
			}
			log.Fatal("Panic in Kubernetes runtime handler")
		},
	)
}

type ConvertFunc func(obj interface{}) interface{}

// NewInformer is a copy of k8s.io/client-go/tools/cache/NewInformer with a new
// argument which converts an object into another object that can be stored in
// the local cache.
func NewInformer(
	lw cache.ListerWatcher,
	objType k8sRuntime.Object,
	resyncPeriod time.Duration,
	h cache.ResourceEventHandler,
	convertFunc ConvertFunc,
) (cache.Store, cache.Controller) {
	// This will hold the client state, as we know it.
	clientState := cache.NewStore(cache.DeletionHandlingMetaNamespaceKeyFunc)

	return clientState, NewInformerWithStore(lw, objType, resyncPeriod, h, convertFunc, clientState)
}

// NewInformerWithStore uses the same arguments as NewInformer for which a
// caller can also set a cache.Store.
func NewInformerWithStore(
	lw cache.ListerWatcher,
	objType k8sRuntime.Object,
	resyncPeriod time.Duration,
	h cache.ResourceEventHandler,
	convertFunc ConvertFunc,
	clientState cache.Store,
) cache.Controller {

	// This will hold incoming changes. Note how we pass clientState in as a
	// KeyLister, that way resync operations will result in the correct set
	// of update/delete deltas.
	fifo := cache.NewDeltaFIFO(cache.MetaNamespaceKeyFunc, clientState)

	cacheMutationDetector := cache.NewCacheMutationDetector(fmt.Sprintf("%T", objType))

	cfg := &cache.Config{
		Queue:            fifo,
		ListerWatcher:    lw,
		ObjectType:       objType,
		FullResyncPeriod: resyncPeriod,
		RetryOnError:     false,

		Process: func(obj interface{}) error {
			// from oldest to newest
			for _, d := range obj.(cache.Deltas) {

				obj := convertFunc(d.Object)

				// In CI we detect if the objects were modified and panic
				// this is a no-op in production environments.
				cacheMutationDetector.AddObject(obj)

				switch d.Type {
				case cache.Sync, cache.Added, cache.Updated:
					if old, exists, err := clientState.Get(obj); err == nil && exists {
						if err := clientState.Update(obj); err != nil {
							return err
						}
						h.OnUpdate(old, obj)
					} else {
						if err := clientState.Add(obj); err != nil {
							return err
						}
						h.OnAdd(obj)
					}
				case cache.Deleted:
					if err := clientState.Delete(obj); err != nil {
						return err
					}
					h.OnDelete(obj)
				}
			}
			return nil
		},
	}
	return cache.New(cfg)
}
<|endoftext|>


<commit_before>/*
Copyright 2019 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package out

import (
	"fmt"
	"os"
	"strconv"
	"testing"

	"k8s.io/minikube/pkg/minikube/tests"
	"k8s.io/minikube/pkg/minikube/translate"
)

func TestOutT(t *testing.T) {
	// Set the system locale to Arabic and define a dummy translation file.
	if err := translate.SetPreferredLanguage("ar"); err != nil {
		t.Fatalf("SetPreferredLanguage: %v", err)
	}
	translate.Translations = map[string]interface{}{
		"Installing Kubernetes version {{.version}} ...": "... {{.version}} تثبيت Kubernetes الإصدار",
	}

	var testCases = []struct {
		style     StyleEnum
		message   string
		params    V
		want      string
		wantASCII string
	}{
		{Happy, "Happy", nil, "😄  Happy\n", "* Happy\n"},
		{Option, "Option", nil, "    ▪ Option\n", "  - Option\n"},
		{Warning, "Warning", nil, "❗  Warning\n", "! Warning\n"},
		{FatalType, "Fatal: {{.error}}", V{"error": "ugh"}, "💣  Fatal: ugh\n", "X Fatal: ugh\n"},
		{Issue, "http://i/{{.number}}", V{"number": 10000}, "    ▪ http://i/10000\n", "  - http://i/10000\n"},
		{Usage, "raw: {{.one}} {{.two}}", V{"one": "'%'", "two": "%d"}, "💡  raw: '%' %d\n", "* raw: '%' %d\n"},
		{Running, "Installing Kubernetes version {{.version}} ...", V{"version": "v1.13"}, "🏃  ... v1.13 تثبيت Kubernetes الإصدار\n", "* ... v1.13 تثبيت Kubernetes الإصدار\n"},
	}
	for _, tc := range testCases {
		for _, override := range []bool{true, false} {
			t.Run(fmt.Sprintf("%s-override-%v", tc.message, override), func(t *testing.T) {
				// Set MINIKUBE_IN_STYLE=<override>
				os.Setenv(OverrideEnv, strconv.FormatBool(override))
				f := tests.NewFakeFile()
				SetOutFile(f)
				T(tc.style, tc.message, tc.params)
				got := f.String()
				want := tc.wantASCII
				if override {
					want = tc.want
				}
				if got != want {
					t.Errorf("OutStyle() = %q (%d runes), want %q (%d runes)", got, len(got), want, len(want))
				}
			})
		}
	}
}

func TestOut(t *testing.T) {
	os.Setenv(OverrideEnv, "")

	var testCases = []struct {
		format string
		arg    interface{}
		want   string
	}{
		{format: "xyz123", want: "xyz123"},
		{format: "Installing Kubernetes version %s ...", arg: "v1.13", want: "Installing Kubernetes version v1.13 ..."},
		{format: "Parameter encoding: %s", arg: "%s%%%d", want: "Parameter encoding: %s%%%d"},
	}
	for _, tc := range testCases {
		t.Run(tc.format, func(t *testing.T) {
			f := tests.NewFakeFile()
			SetOutFile(f)
			ErrLn("unrelated message")
			if tc.arg == nil {
				String(tc.format)
			} else {
				String(tc.format, tc.arg)
			}
			got := f.String()
			if got != tc.want {
				t.Errorf("Out(%s, %s) = %q, want %q", tc.format, tc.arg, got, tc.want)
			}
		})
	}
}

func TestErr(t *testing.T) {
	os.Setenv(OverrideEnv, "0")
	f := tests.NewFakeFile()
	SetErrFile(f)
	Err("xyz123 %s\n", "%s%%%d")
	Ln("unrelated message")
	got := f.String()
	want := "xyz123 %s%%%d\n"

	if got != want {
		t.Errorf("Err() = %q, want %q", got, want)
	}
}

func TestInfof(t *testing.T) {

}
<commit_msg>remove empty test<commit_after>/*
Copyright 2019 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package out

import (
	"fmt"
	"os"
	"strconv"
	"testing"

	"k8s.io/minikube/pkg/minikube/tests"
	"k8s.io/minikube/pkg/minikube/translate"
)

func TestOutT(t *testing.T) {
	// Set the system locale to Arabic and define a dummy translation file.
	if err := translate.SetPreferredLanguage("ar"); err != nil {
		t.Fatalf("SetPreferredLanguage: %v", err)
	}
	translate.Translations = map[string]interface{}{
		"Installing Kubernetes version {{.version}} ...": "... {{.version}} تثبيت Kubernetes الإصدار",
	}

	var testCases = []struct {
		style     StyleEnum
		message   string
		params    V
		want      string
		wantASCII string
	}{
		{Happy, "Happy", nil, "😄  Happy\n", "* Happy\n"},
		{Option, "Option", nil, "    ▪ Option\n", "  - Option\n"},
		{Warning, "Warning", nil, "❗  Warning\n", "! Warning\n"},
		{FatalType, "Fatal: {{.error}}", V{"error": "ugh"}, "💣  Fatal: ugh\n", "X Fatal: ugh\n"},
		{Issue, "http://i/{{.number}}", V{"number": 10000}, "    ▪ http://i/10000\n", "  - http://i/10000\n"},
		{Usage, "raw: {{.one}} {{.two}}", V{"one": "'%'", "two": "%d"}, "💡  raw: '%' %d\n", "* raw: '%' %d\n"},
		{Running, "Installing Kubernetes version {{.version}} ...", V{"version": "v1.13"}, "🏃  ... v1.13 تثبيت Kubernetes الإصدار\n", "* ... v1.13 تثبيت Kubernetes الإصدار\n"},
	}
	for _, tc := range testCases {
		for _, override := range []bool{true, false} {
			t.Run(fmt.Sprintf("%s-override-%v", tc.message, override), func(t *testing.T) {
				// Set MINIKUBE_IN_STYLE=<override>
				os.Setenv(OverrideEnv, strconv.FormatBool(override))
				f := tests.NewFakeFile()
				SetOutFile(f)
				T(tc.style, tc.message, tc.params)
				got := f.String()
				want := tc.wantASCII
				if override {
					want = tc.want
				}
				if got != want {
					t.Errorf("OutStyle() = %q (%d runes), want %q (%d runes)", got, len(got), want, len(want))
				}
			})
		}
	}
}

func TestOut(t *testing.T) {
	os.Setenv(OverrideEnv, "")

	var testCases = []struct {
		format string
		arg    interface{}
		want   string
	}{
		{format: "xyz123", want: "xyz123"},
		{format: "Installing Kubernetes version %s ...", arg: "v1.13", want: "Installing Kubernetes version v1.13 ..."},
		{format: "Parameter encoding: %s", arg: "%s%%%d", want: "Parameter encoding: %s%%%d"},
	}
	for _, tc := range testCases {
		t.Run(tc.format, func(t *testing.T) {
			f := tests.NewFakeFile()
			SetOutFile(f)
			ErrLn("unrelated message")
			if tc.arg == nil {
				String(tc.format)
			} else {
				String(tc.format, tc.arg)
			}
			got := f.String()
			if got != tc.want {
				t.Errorf("Out(%s, %s) = %q, want %q", tc.format, tc.arg, got, tc.want)
			}
		})
	}
}

func TestErr(t *testing.T) {
	os.Setenv(OverrideEnv, "0")
	f := tests.NewFakeFile()
	SetErrFile(f)
	Err("xyz123 %s\n", "%s%%%d")
	Ln("unrelated message")
	got := f.String()
	want := "xyz123 %s%%%d\n"

	if got != want {
		t.Errorf("Err() = %q, want %q", got, want)
	}
}
<|endoftext|>


<commit_before>/*
Copyright 2018 Google LLC

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package docker

import (
	"encoding/json"
	"fmt"
	"io"
	"path"
	"path/filepath"
	"strings"

	"github.com/containers/image/types"

	"github.com/containers/image/manifest"

	"github.com/containers/image/docker"

	"github.com/GoogleCloudPlatform/skaffold/pkg/skaffold/util"
	"github.com/moby/moby/builder/dockerfile/parser"
	"github.com/moby/moby/builder/dockerfile/shell"
	"github.com/pkg/errors"
	"github.com/sirupsen/logrus"
)

const (
	add  = "add"
	copy = "copy"
	env  = "env"
	from = "from"
)

// For testing.
var RetrieveConfig = retrieveImageConfig

// GetDockerfileDependencies parses a dockerfile and returns the full paths
// of all the source files that the resulting docker image depends on.
func GetDockerfileDependencies(workspace string, r io.Reader) ([]string, error) {
	res, err := parser.Parse(r)
	if err != nil {
		return nil, errors.Wrap(err, "parsing dockerfile")
	}
	envs := map[string]string{}
	depMap := map[string]struct{}{}
	// First process onbuilds, if present.
	onbuilds := []string{}
	for _, value := range res.AST.Children {
		switch value.Value {
		case from:
			onbuilds, err = processBaseImage(value)
			if err != nil {
				logrus.Warnf("Error processing base image for onbuild triggers: %s. Dependencies may be incomplete.", err)
			}
		}
	}

	var dispatchInstructions = func(r *parser.Result) {
		for _, value := range r.AST.Children {
			switch value.Value {
			case add, copy:
				processCopy(workspace, value, depMap, envs)
			case env:
				envs[value.Next.Value] = value.Next.Next.Value
			}
		}
	}
	for _, ob := range onbuilds {
		obRes, err := parser.Parse(strings.NewReader(ob))
		if err != nil {
			return nil, err
		}
		dispatchInstructions(obRes)
	}

	dispatchInstructions(res)

	deps := []string{}
	for dep := range depMap {
		deps = append(deps, dep)
	}
	logrus.Infof("Found dependencies for dockerfile %s", deps)
	expandedDeps, err := util.ExpandPaths(workspace, deps)
	if err != nil {
		return nil, errors.Wrap(err, "expanding dockerfile paths")
	}

	// Look for .dockerignore.
	ignorePath := filepath.Join(workspace, ".dockerignore")

	filteredDeps, err := util.ApplyDockerIgnore(expandedDeps, ignorePath)
	if err != nil {
		return nil, errors.Wrap(err, "applying dockerignore")
	}
	return filteredDeps, nil
}

func processBaseImage(value *parser.Node) ([]string, error) {
	base := value.Next.Value
	logrus.Debugf("Checking base image %s for ONBUILD triggers.", base)
	if strings.ToLower(base) == "scratch" {
		logrus.Debugf("SCRATCH base image found, skipping check: %s", base)
		return nil, nil
	}
	cfg, err := RetrieveConfig(base)
	if err != nil {
		return nil, err
	}
	logrus.Debugf("Found onbuild triggers %v in image %s", cfg.Config.OnBuild, base)
	return cfg.Config.OnBuild, nil
}

func retrieveImageConfig(image string) (*manifest.Schema2Image, error) {
	ref, err := docker.ParseReference("//" + image)
	if err != nil {
		return nil, err
	}

	// Hardcode the OS and Arch in case the image returns a manifest list.
	context := &types.SystemContext{
		OSChoice:           "linux",
		ArchitectureChoice: "amd64",
	}
	img, err := ref.NewImage(context)
	if err != nil {
		return nil, err
	}
	defer img.Close()

	cfgBytes, err := img.ConfigBlob()
	if err != nil {
		return nil, err
	}
	cfg := &manifest.Schema2Image{}
	if err := json.Unmarshal(cfgBytes, cfg); err != nil {
		return nil, err
	}
	return cfg, nil
}

func processCopy(workspace string, value *parser.Node, paths map[string]struct{}, envs map[string]string) error {
	slex := shell.NewLex('\\')
	for {
		// Skip last node, since it is the destination, and stop if we arrive at a comment
		if value.Next.Next == nil || strings.HasPrefix(value.Next.Next.Value, "#") {
			break
		}
		src, err := processShellWord(slex, value.Next.Value, envs)
		if err != nil {
			return errors.Wrap(err, "processing word")
		}
		// If the --from flag is provided, we are dealing with a multi-stage dockerfile
		// Adding a dependency from a different stage does not imply a source dependency
		if hasMultiStageFlag(value.Flags) {
			return nil
		}
		dep := path.Join(workspace, src)
		paths[dep] = struct{}{}
		value = value.Next
	}
	return nil
}

func processShellWord(lex *shell.Lex, word string, envs map[string]string) (string, error) {
	envSlice := []string{}
	for envKey, envVal := range envs {
		envSlice = append(envSlice, fmt.Sprintf("%s=%s", envKey, envVal))
	}
	return lex.ProcessWord(word, envSlice)
}

func hasMultiStageFlag(flags []string) bool {
	for _, f := range flags {
		if strings.HasPrefix(f, "--from=") {
			return true
		}
	}
	return false
}
<commit_msg>Cache Image configs by name<commit_after>/*
Copyright 2018 Google LLC

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package docker

import (
	"encoding/json"
	"fmt"
	"io"
	"path"
	"path/filepath"
	"strings"
	"sync"

	"github.com/GoogleCloudPlatform/skaffold/pkg/skaffold/util"
	"github.com/containers/image/docker"
	"github.com/containers/image/manifest"
	"github.com/containers/image/types"
	"github.com/moby/moby/builder/dockerfile/parser"
	"github.com/moby/moby/builder/dockerfile/shell"
	"github.com/pkg/errors"
	"github.com/sirupsen/logrus"
)

const (
	add  = "add"
	copy = "copy"
	env  = "env"
	from = "from"
)

// For testing.
var RetrieveConfig = retrieveImageConfig

// GetDockerfileDependencies parses a dockerfile and returns the full paths
// of all the source files that the resulting docker image depends on.
func GetDockerfileDependencies(workspace string, r io.Reader) ([]string, error) {
	res, err := parser.Parse(r)
	if err != nil {
		return nil, errors.Wrap(err, "parsing dockerfile")
	}
	envs := map[string]string{}
	depMap := map[string]struct{}{}
	// First process onbuilds, if present.
	onbuilds := []string{}
	for _, value := range res.AST.Children {
		switch value.Value {
		case from:
			onbuilds, err = processBaseImage(value)
			if err != nil {
				logrus.Warnf("Error processing base image for onbuild triggers: %s. Dependencies may be incomplete.", err)
			}
		}
	}

	var dispatchInstructions = func(r *parser.Result) {
		for _, value := range r.AST.Children {
			switch value.Value {
			case add, copy:
				processCopy(workspace, value, depMap, envs)
			case env:
				envs[value.Next.Value] = value.Next.Next.Value
			}
		}
	}
	for _, ob := range onbuilds {
		obRes, err := parser.Parse(strings.NewReader(ob))
		if err != nil {
			return nil, err
		}
		dispatchInstructions(obRes)
	}

	dispatchInstructions(res)

	deps := []string{}
	for dep := range depMap {
		deps = append(deps, dep)
	}
	logrus.Infof("Found dependencies for dockerfile %s", deps)
	expandedDeps, err := util.ExpandPaths(workspace, deps)
	if err != nil {
		return nil, errors.Wrap(err, "expanding dockerfile paths")
	}

	// Look for .dockerignore.
	ignorePath := filepath.Join(workspace, ".dockerignore")

	filteredDeps, err := util.ApplyDockerIgnore(expandedDeps, ignorePath)
	if err != nil {
		return nil, errors.Wrap(err, "applying dockerignore")
	}
	return filteredDeps, nil
}

func processBaseImage(value *parser.Node) ([]string, error) {
	base := value.Next.Value
	logrus.Debugf("Checking base image %s for ONBUILD triggers.", base)
	if strings.ToLower(base) == "scratch" {
		logrus.Debugf("SCRATCH base image found, skipping check: %s", base)
		return nil, nil
	}
	cfg, err := RetrieveConfig(base)
	if err != nil {
		return nil, err
	}
	logrus.Debugf("Found onbuild triggers %v in image %s", cfg.Config.OnBuild, base)
	return cfg.Config.OnBuild, nil
}

var imageToConfigCache sync.Map

func retrieveImageConfig(image string) (*manifest.Schema2Image, error) {
	cachedCfg, present := imageToConfigCache.Load(image)
	if present {
		return cachedCfg.(*manifest.Schema2Image), nil
	}

	ref, err := docker.ParseReference("//" + image)
	if err != nil {
		return nil, err
	}

	// Hardcode the OS and Arch in case the image returns a manifest list.
	context := &types.SystemContext{
		OSChoice:           "linux",
		ArchitectureChoice: "amd64",
	}
	img, err := ref.NewImage(context)
	if err != nil {
		return nil, err
	}
	defer img.Close()

	cfgBytes, err := img.ConfigBlob()
	if err != nil {
		return nil, err
	}
	cfg := &manifest.Schema2Image{}
	if err := json.Unmarshal(cfgBytes, cfg); err != nil {
		return nil, err
	}

	imageToConfigCache.Store(image, cfg)

	return cfg, nil
}

func processCopy(workspace string, value *parser.Node, paths map[string]struct{}, envs map[string]string) error {
	slex := shell.NewLex('\\')
	for {
		// Skip last node, since it is the destination, and stop if we arrive at a comment
		if value.Next.Next == nil || strings.HasPrefix(value.Next.Next.Value, "#") {
			break
		}
		src, err := processShellWord(slex, value.Next.Value, envs)
		if err != nil {
			return errors.Wrap(err, "processing word")
		}
		// If the --from flag is provided, we are dealing with a multi-stage dockerfile
		// Adding a dependency from a different stage does not imply a source dependency
		if hasMultiStageFlag(value.Flags) {
			return nil
		}
		dep := path.Join(workspace, src)
		paths[dep] = struct{}{}
		value = value.Next
	}
	return nil
}

func processShellWord(lex *shell.Lex, word string, envs map[string]string) (string, error) {
	envSlice := []string{}
	for envKey, envVal := range envs {
		envSlice = append(envSlice, fmt.Sprintf("%s=%s", envKey, envVal))
	}
	return lex.ProcessWord(word, envSlice)
}

func hasMultiStageFlag(flags []string) bool {
	for _, f := range flags {
		if strings.HasPrefix(f, "--from=") {
			return true
		}
	}
	return false
}
<|endoftext|>