<commit_before>package handlers

import (
	"encoding/hex"
	"net/http"
	"os"
	"path"
	"testing"
	"time"

	log "github.com/repbin/repbin/deferconsole"
)

func TestGenPostHandler(t *testing.T) {
	pubKey, _ := hex.DecodeString("39d8913ab046428e409cf1fa7cee6f63c1f6bf701356a44a8c8c2559bdb2526f")
	privKey, _ := hex.DecodeString("20a2633e422090a4f4a102f8e3d112f2b4378dbd9957e8c892067fc09239d36c39d8913ab046428e409cf1fa7cee6f63c1f6bf701356a44a8c8c2559bdb2526f")

	log.SetMinLevel(log.LevelDebug)
	ms, err := New(path.Join(os.TempDir(), "repbin")+string(os.PathSeparator), pubKey, privKey)
	if err != nil {
		t.Fatalf("New: %s", err)
	}
	enforceTimeOuts = false
	debug = true
	ms.NotifyDuration = 0
	ms.FetchDuration = 0
	ms.LoadPeers()
	ms.NotifyPeers()
	ms.FetchPeers()
	http.HandleFunc("/id", ms.ServeID)
	http.HandleFunc("/keyindex", ms.GetKeyIndex)
	http.HandleFunc("/globalindex", ms.GetGlobalIndex)
	http.HandleFunc("/post", ms.GenPostHandler(false))
	http.HandleFunc("/local/post", ms.GenPostHandler(true))
	http.HandleFunc("/fetch", ms.Fetch)
	http.HandleFunc("/notify", ms.GetNotify)
	http.HandleFunc("/delete", ms.Delete)
	go http.ListenAndServe(":8080", nil)
	time.Sleep(time.Second / 100)
	time.Sleep(time.Second * 10)
}
<commit_msg>create test directory<commit_after>package handlers

import (
	"encoding/hex"
	"net/http"
	"os"
	"path"
	"testing"
	"time"

	log "github.com/repbin/repbin/deferconsole"
)

func TestGenPostHandler(t *testing.T) {
	pubKey, _ := hex.DecodeString("39d8913ab046428e409cf1fa7cee6f63c1f6bf701356a44a8c8c2559bdb2526f")
	privKey, _ := hex.DecodeString("20a2633e422090a4f4a102f8e3d112f2b4378dbd9957e8c892067fc09239d36c39d8913ab046428e409cf1fa7cee6f63c1f6bf701356a44a8c8c2559bdb2526f")

	log.SetMinLevel(log.LevelDebug)
	testDir := path.Join(os.TempDir(), "repbin")
	if err := os.MkdirAll(testDir, 0700); err != nil {
		t.Fatal(err)
	}
	ms, err := New(testDir+string(os.PathSeparator), pubKey, privKey)
	if err != nil {
		t.Fatalf("New: %s", err)
	}
	enforceTimeOuts = false
	debug = true
	ms.NotifyDuration = 0
	ms.FetchDuration = 0
	ms.LoadPeers()
	ms.NotifyPeers()
	ms.FetchPeers()
	http.HandleFunc("/id", ms.ServeID)
	http.HandleFunc("/keyindex", ms.GetKeyIndex)
	http.HandleFunc("/globalindex", ms.GetGlobalIndex)
	http.HandleFunc("/post", ms.GenPostHandler(false))
	http.HandleFunc("/local/post", ms.GenPostHandler(true))
	http.HandleFunc("/fetch", ms.Fetch)
	http.HandleFunc("/notify", ms.GetNotify)
	http.HandleFunc("/delete", ms.Delete)
	go http.ListenAndServe(":8080", nil)
	time.Sleep(time.Second / 100)
	time.Sleep(time.Second * 10)
}
<|endoftext|>


<commit_before>package web

import "net/http"

var ruleController = &RuleController{}

type RuleController struct {
}

func (r *RuleController) Testing(w http.ResponseWriter, req *http.Request) {
	q := req.URL.Query()
	name := q.Get(`name`)
	w.Write([]byte(`Hello:` + name))
	w.WriteHeader(200)
}
<commit_msg>update<commit_after>package web

import "net/http"

var ruleController = &RuleController{}

type RuleController struct {
}

func (r *RuleController) Testing(w http.ResponseWriter, req *http.Request) {
	q := req.URL.Query()
	name := q.Get(`name`)
	w.Write([]byte(`Hello:` + name))
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"log"
	"logyard"
	"logyard/stackato/events"
	"time"
)

// Make relevant cloud events available in application logs. Heroku style.
func MonitorCloudEvents() {
	// TODO: add more events; will require modifying the log
	// invokation to include the required app id
	filters := []string{
		"event.dea_start",
		"event.dea_ready",
		"event.dea_stop",
		"event.stager_start",
		"event.stager_end",
		"event.cc_app_update",
	}

	c, err := logyard.NewClientGlobal()
	if err != nil {
		log.Fatal(err)
	}
	ss, err := c.Recv(filters)
	if err != nil {
		log.Fatal(err)
	}

	log.Println("Listening for app relevant cloud events...")
	for msg := range ss.Ch {
		fmt.Println(msg.Key, msg.Value)
		var event events.Event
		err := json.Unmarshal([]byte(msg.Value), &event)
		if err != nil {
			log.Fatal(err) // not expected at all
		}

		switch msg.Key {
		case "event.dea_start", "event.dea_ready", "event.dea_stop":
			appid := int(event.Info["app_id"].(float64))
			index := int(event.Info["instance"].(float64))
			source := "stackato.dea"
			PublishAppLog(c, appid, index, source, &event)
		case "event.stager_start", "event.stager_end":
			appid := int(event.Info["app_id"].(float64))
			PublishAppLog(c, appid, -1, "stackato.stager", &event)
		case "event.cc_app_update":
			appid := int(event.Info["app_id"].(float64))
			PublishAppLog(c, appid, -1, "stackato.controller", &event)
		}
	}

	err = ss.Wait()
	if err != nil {
		log.Fatal(err)
	}
}

func PublishAppLog(client *logyard.Client, appid int, index int, source string, event *events.Event) {
	m := AppLogMessage{
		Text:          event.Desc,
		LogFilename:   "",
		UnixTime:      event.UnixTime,
		HumanTime:     time.Unix(event.UnixTime, 0).Format("2006-01-02T15:04:05-07:00"), // heroku-format
		InstanceIndex: index,
		Source:        source}
	data, err := json.Marshal(m)
	if err != nil {
		log.Printf("Error encoding %+v into JSON; %s. Skipping this message", m, err)
		return
	}
	key := fmt.Sprintf("apptail.%d", appid)
	err = client.Send(key, string(data))
	if err != nil {
		log.Fatal(err)
	}
}
<commit_msg>apptail: remove debug logging<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"log"
	"logyard"
	"logyard/stackato/events"
	"time"
)

// Make relevant cloud events available in application logs. Heroku style.
func MonitorCloudEvents() {
	// TODO: add more events; will require modifying the log
	// invokation to include the required app id
	filters := []string{
		"event.dea_start",
		"event.dea_ready",
		"event.dea_stop",
		"event.stager_start",
		"event.stager_end",
		"event.cc_app_update",
	}

	c, err := logyard.NewClientGlobal()
	if err != nil {
		log.Fatal(err)
	}
	ss, err := c.Recv(filters)
	if err != nil {
		log.Fatal(err)
	}

	log.Println("Listening for app relevant cloud events...")
	for msg := range ss.Ch {
		var event events.Event
		err := json.Unmarshal([]byte(msg.Value), &event)
		if err != nil {
			log.Fatal(err) // not expected at all
		}

		switch msg.Key {
		case "event.dea_start", "event.dea_ready", "event.dea_stop":
			appid := int(event.Info["app_id"].(float64))
			index := int(event.Info["instance"].(float64))
			source := "stackato.dea"
			PublishAppLog(c, appid, index, source, &event)
		case "event.stager_start", "event.stager_end":
			appid := int(event.Info["app_id"].(float64))
			PublishAppLog(c, appid, -1, "stackato.stager", &event)
		case "event.cc_app_update":
			appid := int(event.Info["app_id"].(float64))
			PublishAppLog(c, appid, -1, "stackato.controller", &event)
		}
	}

	err = ss.Wait()
	if err != nil {
		log.Fatal(err)
	}
}

func PublishAppLog(client *logyard.Client, appid int, index int, source string, event *events.Event) {
	m := AppLogMessage{
		Text:          event.Desc,
		LogFilename:   "",
		UnixTime:      event.UnixTime,
		HumanTime:     time.Unix(event.UnixTime, 0).Format("2006-01-02T15:04:05-07:00"), // heroku-format
		InstanceIndex: index,
		Source:        source}
	data, err := json.Marshal(m)
	if err != nil {
		log.Printf("Error encoding %+v into JSON; %s. Skipping this message", m, err)
		return
	}
	key := fmt.Sprintf("apptail.%d", appid)
	err = client.Send(key, string(data))
	if err != nil {
		log.Fatal(err)
	}
}
<|endoftext|>


<commit_before>package szah

import (
	"bytes"
	"os"
	"path/filepath"
	"runtime"
	"strings"

	humanize "github.com/dustin/go-humanize"
	"github.com/itchio/wharf/archiver"

	"github.com/go-errors/errors"
	"github.com/itchio/butler/archive"
	"github.com/itchio/sevenzip-go/sz"
)

type ExtractState struct {
	HasListedItems        bool
	ItemCount             int64
	TotalDoneSize         int64
	TotalUncompressedSize int64
	CurrentIndex          int64
	Contents              *archive.Contents

	NumFiles    int64
	NumDirs     int64
	NumSymlinks int64
}

type ech struct {
	params          *archive.ExtractParams
	initialProgress float64
	state           *ExtractState
	save            archive.ThrottledSaveFunc
}

func (h *Handler) Extract(params *archive.ExtractParams) (*archive.Contents, error) {
	save := archive.ThrottledSave(params)
	consumer := params.Consumer
	state := &ExtractState{
		Contents: &archive.Contents{},
	}

	err := withArchive(params.Consumer, params.File, func(a *sz.Archive) error {
		err := params.Load(state)
		if err != nil {
			consumer.Infof("szah: could not load state: %s", err.Error())
			consumer.Infof("szah: ...starting from beginning!")
		}

		if !state.HasListedItems {
			consumer.Infof("Listing items...")
			itemCount, err := a.GetItemCount()
			if err != nil {
				return errors.Wrap(err, 0)
			}
			state.ItemCount = itemCount

			var totalUncompressedSize int64
			for i := int64(0); i < itemCount; i++ {
				func() {
					item := a.GetItem(i)
					if item == nil {
						return
					}
					defer item.Free()

					ei := decodeEntryInfo(item)
					if ei.kind == entryKindFile {
						if itemSize, ok := item.GetUInt64Property(sz.PidSize); ok {
							// if we can't get the item size well.. that's not great
							// but it shouldn't impede anything.
							totalUncompressedSize += int64(itemSize)
						}
					}
				}()
			}
			state.TotalUncompressedSize = totalUncompressedSize

			state.HasListedItems = true
			save(state, true)
		} else {
			consumer.Infof("Using cached item listing")
		}

		if params.OnUncompressedSizeKnown != nil {
			params.OnUncompressedSizeKnown(state.TotalUncompressedSize)
		}

		ec, err := sz.NewExtractCallback(&ech{
			params:          params,
			state:           state,
			initialProgress: float64(state.TotalDoneSize) / float64(state.TotalUncompressedSize),
			save:            save,
		})
		if err != nil {
			return errors.Wrap(err, 0)
		}
		defer ec.Free()

		var indices []int64
		for i := state.CurrentIndex; i < state.ItemCount; i++ {
			indices = append(indices, i)
		}
		if len(indices) == 0 {
			consumer.Infof("nothing (0 items) to extract!")
			return nil
		}

		consumer.Infof("Queued %d / %d items for extraction", len(indices), state.ItemCount)

		err = a.ExtractSeveral(indices, ec)
		if err != nil {
			return errors.Wrap(err, 0)
		}

		return nil
	})
	if err != nil {
		return nil, errors.Wrap(err, 0)
	}

	consumer.Statf("extracted %d items successfully", state.ItemCount)
	consumer.Statf("%d files, %d dirs, %d symlinks", state.NumFiles, state.NumDirs, state.NumSymlinks)

	return state.Contents, nil
}

func (e *ech) GetStream(item *sz.Item) (*sz.OutStream, error) {
	consumer := e.params.Consumer

	itemPath, ok := item.GetStringProperty(sz.PidPath)
	if !ok {
		return nil, errors.New("can't get item path")
	}

	sanePath := sanitizePath(itemPath)
	outPath := filepath.Join(e.params.OutputPath, sanePath)

	ei := decodeEntryInfo(item)

	contents := e.state.Contents
	finish := func(totalBytes int64, createEntry bool) {
		if createEntry {
			contents.Entries = append(contents.Entries, &archive.Entry{
				Name:             sanePath,
				UncompressedSize: totalBytes,
			})
		}

		e.state.CurrentIndex = item.GetArchiveIndex() + 1
		e.state.TotalDoneSize += totalBytes
		e.save(e.state, false)
	}

	windows := runtime.GOOS == "windows"

	if ei.kind == entryKindDir {
		e.state.NumDirs++

		err := os.MkdirAll(outPath, 0755)
		if err != nil {
			return nil, errors.Wrap(err, 0)
		}
		finish(0, false)

		// giving 7-zip a null stream will make it skip the entry
		return nil, nil
	}

	if ei.kind == entryKindSymlink && !windows {
		e.state.NumSymlinks++

		// is the link name stored as a property?
		if linkname, ok := item.GetStringProperty(sz.PidSymLink); ok {
			// cool!
			err := archiver.Symlink(linkname, outPath, consumer)
			if err != nil {
				return nil, errors.Wrap(err, 0)
			}

			finish(0, false)
			return nil, nil
		}

		// the link name is stored as the file contents, so
		// we extract to an in-memory buffer
		buf := new(bytes.Buffer)
		nc := &notifyCloser{
			Writer: buf,
			OnClose: func(totalBytes int64) error {
				linkname := buf.Bytes()

				err := archiver.Symlink(string(linkname), outPath, consumer)
				if err != nil {
					return errors.Wrap(err, 0)
				}

				finish(totalBytes, false)
				return nil
			},
		}

		return sz.NewOutStream(nc)
	}

	// if we end up here, it's a regular file
	e.state.NumFiles++

	uncompressedSize, _ := item.GetUInt64Property(sz.PidSize)
	consumer.Infof(`→ %s (%s)`, sanePath, humanize.IBytes(uncompressedSize))

	err := os.MkdirAll(filepath.Dir(outPath), 0755)
	if err != nil {
		return nil, errors.Wrap(err, 0)
	}

	flag := os.O_CREATE | os.O_TRUNC | os.O_WRONLY
	f, err := os.OpenFile(outPath, flag, ei.mode)
	if err != nil {
		return nil, errors.Wrap(err, 0)
	}

	nc := &notifyCloser{
		Writer: f,
		OnClose: func(totalBytes int64) error {
			finish(totalBytes, true)
			return nil
		},
	}
	return sz.NewOutStream(nc)
}

func (e *ech) SetProgress(complete int64, total int64) {
	if total > 0 {
		thisRunProgress := float64(complete) / float64(total)
		actualProgress := e.initialProgress + (1.0-e.initialProgress)*thisRunProgress
		e.params.Consumer.Progress(actualProgress)
	}
	// TODO: some formats don't have 'total' value, should we do
	// something smart there?
}

func sanitizePath(inPath string) string {
	outPath := filepath.ToSlash(inPath)

	if runtime.GOOS == "windows" {
		// Replace illegal character for windows paths with underscores, see
		// https://msdn.microsoft.com/en-us/library/windows/desktop/aa365247(v=vs.85).aspx
		// (N.B: that's what the 7-zip CLI seems to do)
		for i := byte(0); i <= 31; i++ {
			outPath = strings.Replace(outPath, string([]byte{i}), "_", -1)
		}
	}

	return outPath
}
<commit_msg>we can't call GetArchiveIndex in the Close callback, item might be already freed<commit_after>package szah

import (
	"bytes"
	"os"
	"path/filepath"
	"runtime"
	"strings"

	humanize "github.com/dustin/go-humanize"
	"github.com/itchio/wharf/archiver"

	"github.com/go-errors/errors"
	"github.com/itchio/butler/archive"
	"github.com/itchio/sevenzip-go/sz"
)

type ExtractState struct {
	HasListedItems        bool
	ItemCount             int64
	TotalDoneSize         int64
	TotalUncompressedSize int64
	CurrentIndex          int64
	Contents              *archive.Contents

	NumFiles    int64
	NumDirs     int64
	NumSymlinks int64
}

type ech struct {
	params          *archive.ExtractParams
	initialProgress float64
	state           *ExtractState
	save            archive.ThrottledSaveFunc
}

func (h *Handler) Extract(params *archive.ExtractParams) (*archive.Contents, error) {
	save := archive.ThrottledSave(params)
	consumer := params.Consumer
	state := &ExtractState{
		Contents: &archive.Contents{},
	}

	err := withArchive(params.Consumer, params.File, func(a *sz.Archive) error {
		err := params.Load(state)
		if err != nil {
			consumer.Infof("szah: could not load state: %s", err.Error())
			consumer.Infof("szah: ...starting from beginning!")
		}

		if !state.HasListedItems {
			consumer.Infof("Listing items...")
			itemCount, err := a.GetItemCount()
			if err != nil {
				return errors.Wrap(err, 0)
			}
			state.ItemCount = itemCount

			var totalUncompressedSize int64
			for i := int64(0); i < itemCount; i++ {
				func() {
					item := a.GetItem(i)
					if item == nil {
						return
					}
					defer item.Free()

					ei := decodeEntryInfo(item)
					if ei.kind == entryKindFile {
						if itemSize, ok := item.GetUInt64Property(sz.PidSize); ok {
							// if we can't get the item size well.. that's not great
							// but it shouldn't impede anything.
							totalUncompressedSize += int64(itemSize)
						}
					}
				}()
			}
			state.TotalUncompressedSize = totalUncompressedSize

			state.HasListedItems = true
			save(state, true)
		} else {
			consumer.Infof("Using cached item listing")
		}

		if params.OnUncompressedSizeKnown != nil {
			params.OnUncompressedSizeKnown(state.TotalUncompressedSize)
		}

		ec, err := sz.NewExtractCallback(&ech{
			params:          params,
			state:           state,
			initialProgress: float64(state.TotalDoneSize) / float64(state.TotalUncompressedSize),
			save:            save,
		})
		if err != nil {
			return errors.Wrap(err, 0)
		}
		defer ec.Free()

		var indices []int64
		for i := state.CurrentIndex; i < state.ItemCount; i++ {
			indices = append(indices, i)
		}
		if len(indices) == 0 {
			consumer.Infof("nothing (0 items) to extract!")
			return nil
		}

		consumer.Infof("Queued %d / %d items for extraction", len(indices), state.ItemCount)

		err = a.ExtractSeveral(indices, ec)
		if err != nil {
			return errors.Wrap(err, 0)
		}

		return nil
	})
	if err != nil {
		return nil, errors.Wrap(err, 0)
	}

	consumer.Statf("extracted %d items successfully", state.ItemCount)
	consumer.Statf("%d files, %d dirs, %d symlinks", state.NumFiles, state.NumDirs, state.NumSymlinks)

	return state.Contents, nil
}

func (e *ech) GetStream(item *sz.Item) (*sz.OutStream, error) {
	consumer := e.params.Consumer
	itemIndex := item.GetArchiveIndex()

	itemPath, ok := item.GetStringProperty(sz.PidPath)
	if !ok {
		return nil, errors.New("can't get item path")
	}

	sanePath := sanitizePath(itemPath)
	outPath := filepath.Join(e.params.OutputPath, sanePath)

	ei := decodeEntryInfo(item)

	contents := e.state.Contents
	finish := func(totalBytes int64, createEntry bool) {
		if createEntry {
			contents.Entries = append(contents.Entries, &archive.Entry{
				Name:             sanePath,
				UncompressedSize: totalBytes,
			})
		}

		e.state.CurrentIndex = itemIndex + 1
		e.state.TotalDoneSize += totalBytes
		e.save(e.state, false)
	}

	windows := runtime.GOOS == "windows"

	if ei.kind == entryKindDir {
		e.state.NumDirs++

		err := os.MkdirAll(outPath, 0755)
		if err != nil {
			return nil, errors.Wrap(err, 0)
		}
		finish(0, false)

		// giving 7-zip a null stream will make it skip the entry
		return nil, nil
	}

	if ei.kind == entryKindSymlink && !windows {
		e.state.NumSymlinks++

		// is the link name stored as a property?
		if linkname, ok := item.GetStringProperty(sz.PidSymLink); ok {
			// cool!
			err := archiver.Symlink(linkname, outPath, consumer)
			if err != nil {
				return nil, errors.Wrap(err, 0)
			}

			finish(0, false)
			return nil, nil
		}

		// the link name is stored as the file contents, so
		// we extract to an in-memory buffer
		buf := new(bytes.Buffer)
		nc := &notifyCloser{
			Writer: buf,
			OnClose: func(totalBytes int64) error {
				linkname := buf.Bytes()

				err := archiver.Symlink(string(linkname), outPath, consumer)
				if err != nil {
					return errors.Wrap(err, 0)
				}

				finish(totalBytes, false)
				return nil
			},
		}

		return sz.NewOutStream(nc)
	}

	// if we end up here, it's a regular file
	e.state.NumFiles++

	uncompressedSize, _ := item.GetUInt64Property(sz.PidSize)
	consumer.Infof(`→ %s (%s)`, sanePath, humanize.IBytes(uncompressedSize))

	err := os.MkdirAll(filepath.Dir(outPath), 0755)
	if err != nil {
		return nil, errors.Wrap(err, 0)
	}

	flag := os.O_CREATE | os.O_TRUNC | os.O_WRONLY
	f, err := os.OpenFile(outPath, flag, ei.mode)
	if err != nil {
		return nil, errors.Wrap(err, 0)
	}

	nc := &notifyCloser{
		Writer: f,
		OnClose: func(totalBytes int64) error {
			finish(totalBytes, true)
			return nil
		},
	}
	return sz.NewOutStream(nc)
}

func (e *ech) SetProgress(complete int64, total int64) {
	if total > 0 {
		thisRunProgress := float64(complete) / float64(total)
		actualProgress := e.initialProgress + (1.0-e.initialProgress)*thisRunProgress
		e.params.Consumer.Progress(actualProgress)
	}
	// TODO: some formats don't have 'total' value, should we do
	// something smart there?
}

func sanitizePath(inPath string) string {
	outPath := filepath.ToSlash(inPath)

	if runtime.GOOS == "windows" {
		// Replace illegal character for windows paths with underscores, see
		// https://msdn.microsoft.com/en-us/library/windows/desktop/aa365247(v=vs.85).aspx
		// (N.B: that's what the 7-zip CLI seems to do)
		for i := byte(0); i <= 31; i++ {
			outPath = strings.Replace(outPath, string([]byte{i}), "_", -1)
		}
	}

	return outPath
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/xml"
	"io/ioutil"
	"os"
	"path/filepath"
	"regexp"
	"strings"
)

// Structs for XML building
type Package struct {
	Xmlns   string     `xml:"xmlns,attr"`
	Types   []MetaType `xml:"types"`
	Version string     `xml:"version"`
}

type MetaType struct {
	Members []string `xml:"members"`
	Name    string   `xml:"name"`
}

func createPackage() Package {
	return Package{
		Version: strings.TrimPrefix(apiVersion, "v"),
		Xmlns:   "http://soap.sforce.com/2006/04/metadata",
	}
}

type metapath struct {
	path       string
	name       string
	hasFolder  bool
	onlyFolder bool
	extension  string
}

var metapaths = []metapath{
	metapath{path: "actionLinkGroupTemplates", name: "ActionLinkGroupTemplate"},
	metapath{path: "analyticSnapshots", name: "AnalyticSnapshot"},
	metapath{path: "applications", name: "CustomApplication"},
	metapath{path: "appMenus", name: "AppMenu"},
	metapath{path: "approvalProcesses", name: "ApprovalProcess"},
	metapath{path: "assignmentRules", name: "AssignmentRules"},
	metapath{path: "authproviders", name: "AuthProvider"},
	metapath{path: "aura", name: "AuraDefinitionBundle", hasFolder: true, onlyFolder: true},
	metapath{path: "autoResponseRules", name: "AutoResponseRules"},
	metapath{path: "callCenters", name: "CallCenter"},
	metapath{path: "cachePartitions", name: "PlatformCachePartition"},
	metapath{path: "certs", name: "Certificate"},
	metapath{path: "channelLayouts", name: "ChannelLayout"},
	metapath{path: "classes", name: "ApexClass"},
	metapath{path: "communities", name: "Community"},
	metapath{path: "components", name: "ApexComponent"},
	metapath{path: "connectedApps", name: "ConnectedApp"},
	metapath{path: "corsWhitelistOrigins", name: "CorsWhitelistOrigin"},
	metapath{path: "customApplicationComponents", name: "CustomApplicationComponent"},
	metapath{path: "customMetadata", name: "CustomMetadata"},
	metapath{path: "customPermissions", name: "CustomPermission"},
	metapath{path: "dashboards", name: "Dashboard", hasFolder: true},
	metapath{path: "dataSources", name: "ExternalDataSource"},
	metapath{path: "datacategorygroups", name: "DataCategoryGroup"},
	metapath{path: "delegateGroups", name: "DelegateGroup"},
	metapath{path: "documents", name: "Document", hasFolder: true},
	metapath{path: "EmbeddedServiceConfig", name: "EmbeddedServiceConfig"},
	metapath{path: "email", name: "EmailTemplate", hasFolder: true},
	metapath{path: "escalationRules", name: "EscalationRules"},
	metapath{path: "feedFilters", name: "CustomFeedFilter"},
	metapath{path: "flexipages", name: "FlexiPage"},
	metapath{path: "flowDefinitions", name: "FlowDefinition"},
	metapath{path: "flows", name: "Flow"},
	metapath{path: "globalPicklists", name: "GlobalPicklist"},
	metapath{path: "groups", name: "Group"},
	metapath{path: "homePageComponents", name: "HomePageComponent"},
	metapath{path: "homePageLayouts", name: "HomePageLayout"},
	metapath{path: "installedPackages", name: "InstalledPackage"},
	metapath{path: "labels", name: "CustomLabels"},
	metapath{path: "layouts", name: "Layout"},
	metapath{path: "LeadConvertSettings", name: "LeadConvertSettings"},
	metapath{path: "letterhead", name: "Letterhead"},
	metapath{path: "matchingRules", name: "MatchingRules"},
	metapath{path: "namedCredentials", name: "NamedCredential"},
	metapath{path: "objects", name: "CustomObject"},
	metapath{path: "objectTranslations", name: "CustomObjectTranslation"},
	metapath{path: "pages", name: "ApexPage"},
	metapath{path: "pathAssistants", name: "PathAssistant"},
	metapath{path: "permissionsets", name: "PermissionSet"},
	metapath{path: "postTemplates", name: "PostTemplate"},
	metapath{path: "profiles", name: "Profile", extension: ".profile"},
	metapath{path: "postTemplates", name: "PostTemplate"},
	metapath{path: "postTemplates", name: "PostTemplate"},
	metapath{path: "profiles", name: "Profile"},
	metapath{path: "queues", name: "Queue"},
	metapath{path: "quickActions", name: "QuickAction"},
	metapath{path: "remoteSiteSettings", name: "RemoteSiteSetting"},
	metapath{path: "reports", name: "Report", hasFolder: true},
	metapath{path: "reportTypes", name: "ReportType"},
	metapath{path: "roles", name: "Role"},
	metapath{path: "scontrols", name: "Scontrol"},
	metapath{path: "settings", name: "Settings"},
	metapath{path: "sharingRules", name: "SharingRules"},
	metapath{path: "siteDotComSites", name: "SiteDotCom"},
	metapath{path: "sites", name: "CustomSite"},
	metapath{path: "staticresources", name: "StaticResource"},
	metapath{path: "synonymDictionaries", name: "SynonymDictionary"},
	metapath{path: "tabs", name: "CustomTab"},
	metapath{path: "triggers", name: "ApexTrigger"},
	metapath{path: "weblinks", name: "CustomPageWebLink"},
	metapath{path: "workflows", name: "Workflow"},
	metapath{path: "cspTrustedSites", name: "CspTrustedSite"},
}

type PackageBuilder struct {
	IsPush   bool
	Metadata map[string]MetaType
	Files    ForceMetadataFiles
}

func NewPushBuilder() PackageBuilder {
	pb := PackageBuilder{IsPush: true}
	pb.Metadata = make(map[string]MetaType)
	pb.Files = make(ForceMetadataFiles)

	return pb
}

func NewFetchBuilder() PackageBuilder {
	pb := PackageBuilder{IsPush: false}
	pb.Metadata = make(map[string]MetaType)
	pb.Files = make(ForceMetadataFiles)

	return pb
}

// Build and return package.xml
func (pb PackageBuilder) PackageXml() []byte {
	p := createPackage()

	for _, metaType := range pb.Metadata {
		p.Types = append(p.Types, metaType)
	}

	byteXml, _ := xml.MarshalIndent(p, "", "    ")
	byteXml = append([]byte(xml.Header), byteXml...)
	//if err := ioutil.WriteFile("mypackage.xml", byteXml, 0644); err != nil {
	//ErrorAndExit(err.Error())
	//}
	return byteXml
}

// Returns the full ForceMetadataFiles container
func (pb *PackageBuilder) ForceMetadataFiles() ForceMetadataFiles {
	pb.Files["package.xml"] = pb.PackageXml()
	return pb.Files
}

// Returns the source file path for a given metadata file path.
func MetaPathToSourcePath(mpath string) (spath string) {
	spath = strings.TrimSuffix(mpath, "-meta.xml")
	if spath == mpath {
		return
	}

	_, err := os.Stat(spath)
	if err != nil {
		spath = mpath
	}
	return
}

// Add a file to the builder
func (pb *PackageBuilder) AddFile(fpath string) (fname string, err error) {
	fpath, err = filepath.Abs(fpath)
	if err != nil {
		return
	}
	_, err = os.Stat(fpath)
	if err != nil {
		return
	}

	isDestructiveChanges, err := regexp.MatchString("destructiveChanges(Pre|Post)?"+regexp.QuoteMeta(".")+"xml", fpath)
	if err != nil {
		return
	}

	fpath = MetaPathToSourcePath(fpath)
	metaName, fname := getMetaTypeFromPath(fpath)
	if !isDestructiveChanges && !strings.HasSuffix(fpath, "-meta.xml") {
		pb.AddMetaToPackage(metaName, fname)
	}

	// If it's a push, we want to actually add the files
	if pb.IsPush {
		if isDestructiveChanges {
			err = pb.addDestructiveChanges(fpath)
		} else {
			err = pb.addFileToWorkingDir(metaName, fpath)
		}
	}

	return
}

// Adds the file to a temp directory for deploy
func (pb *PackageBuilder) addFileToWorkingDir(metaName string, fpath string) (err error) {
	// Get relative dir from source
	srcDir := filepath.Dir(filepath.Dir(fpath))
	for _, mp := range metapaths {
		if metaName == mp.name && mp.hasFolder {
			srcDir = filepath.Dir(srcDir)
		}
	}
	frel, _ := filepath.Rel(srcDir, fpath)

	// Try to find meta file
	hasMeta := true
	fmeta := fpath + "-meta.xml"
	fmetarel := ""
	if _, err = os.Stat(fmeta); err != nil {
		if os.IsNotExist(err) {
			hasMeta = false
		} else {
			// Has error
			return
		}
	} else {
		// Should be present since we worked back to srcDir
		fmetarel, _ = filepath.Rel(srcDir, fmeta)
	}

	fdata, err := ioutil.ReadFile(fpath)
	if err != nil {
		return
	}

	pb.Files[frel] = fdata
	if hasMeta {
		fdata, err = ioutil.ReadFile(fmeta)
		pb.Files[fmetarel] = fdata
		return
	}

	return
}

func (pb *PackageBuilder) addDestructiveChanges(fpath string) (err error) {
	fdata, err := ioutil.ReadFile(fpath)
	if err != nil {
		return
	}

	frel, _ := filepath.Rel(filepath.Dir(fpath), fpath)
	pb.Files[frel] = fdata

	return
}

func (pb *PackageBuilder) contains(members []string, name string) bool {
	for _, a := range members {
		if a == name {
			return true
		}
	}
	return false
}

// Adds a metadata name to the pending package
func (pb *PackageBuilder) AddMetaToPackage(metaName string, name string) {
	mt := pb.Metadata[metaName]
	if mt.Name == "" {
		mt.Name = metaName
	}

	if !pb.contains(mt.Members, name) {
		mt.Members = append(mt.Members, name)
		pb.Metadata[metaName] = mt
	}
}

// Gets metadata type name and target name from a file path
func getMetaTypeFromPath(fpath string) (metaName string, name string) {
	fpath, err := filepath.Abs(fpath)
	if err != nil {
		ErrorAndExit("Cound not find " + fpath)
	}
	if _, err := os.Stat(fpath); err != nil {
		ErrorAndExit("Cound not open " + fpath)
	}

	// Get the metadata type and name for the file
	metaName, fileName := getMetaForPath(fpath)
	name = strings.TrimSuffix(fileName, filepath.Ext(fileName))
	//name = strings.TrimSuffix(name, filepath.Ext(name))
	return
}

// Gets partial path based on a meta type name
func getPathForMeta(metaname string) string {
	for _, mp := range metapaths {
		if strings.EqualFold(mp.name, metaname) {
			return mp.path
		}
	}

	// Unknown, so use metaname
	return metaname
}

func findMetapathForFile(file string) (path metapath) {
	parentDir := filepath.Dir(file)
	parentName := filepath.Base(parentDir)
	grandparentName := filepath.Base(filepath.Dir(parentDir))
	fileExtension := filepath.Ext(file)

	for _, mp := range metapaths {
		if mp.hasFolder && grandparentName == mp.path {
			return mp
		}
		if mp.path == parentName {
			return mp
		}
	}

	// Hmm, maybe we can use the extension to determine the type
	for _, mp := range metapaths {
		if mp.extension == fileExtension {
			return mp
		}
	}
	return
}

// Gets meta type and name based on a path
func getMetaForPath(path string) (metaName string, objectName string) {
	parentDir := filepath.Dir(path)
	parentName := filepath.Base(parentDir)
	grandparentName := filepath.Base(filepath.Dir(parentDir))
	fileName := filepath.Base(path)

	for _, mp := range metapaths {
		if mp.hasFolder && grandparentName == mp.path {
			metaName = mp.name
			if mp.onlyFolder {
				objectName = parentName
			} else {
				objectName = parentName + "/" + fileName
			}
			return
		}
		if mp.path == parentName {
			metaName = mp.name
			objectName = fileName
			return
		}
	}

	// Unknown, so use path
	metaName = parentName
	objectName = fileName
	return
}
<commit_msg>Support StandardValueSet and GlobalValueSet<commit_after>package main

import (
	"encoding/xml"
	"io/ioutil"
	"os"
	"path/filepath"
	"regexp"
	"strings"
)

// Structs for XML building
type Package struct {
	Xmlns   string     `xml:"xmlns,attr"`
	Types   []MetaType `xml:"types"`
	Version string     `xml:"version"`
}

type MetaType struct {
	Members []string `xml:"members"`
	Name    string   `xml:"name"`
}

func createPackage() Package {
	return Package{
		Version: strings.TrimPrefix(apiVersion, "v"),
		Xmlns:   "http://soap.sforce.com/2006/04/metadata",
	}
}

type metapath struct {
	path       string
	name       string
	hasFolder  bool
	onlyFolder bool
	extension  string
}

var metapaths = []metapath{
	metapath{path: "actionLinkGroupTemplates", name: "ActionLinkGroupTemplate"},
	metapath{path: "analyticSnapshots", name: "AnalyticSnapshot"},
	metapath{path: "applications", name: "CustomApplication"},
	metapath{path: "appMenus", name: "AppMenu"},
	metapath{path: "approvalProcesses", name: "ApprovalProcess"},
	metapath{path: "assignmentRules", name: "AssignmentRules"},
	metapath{path: "authproviders", name: "AuthProvider"},
	metapath{path: "aura", name: "AuraDefinitionBundle", hasFolder: true, onlyFolder: true},
	metapath{path: "autoResponseRules", name: "AutoResponseRules"},
	metapath{path: "callCenters", name: "CallCenter"},
	metapath{path: "cachePartitions", name: "PlatformCachePartition"},
	metapath{path: "certs", name: "Certificate"},
	metapath{path: "channelLayouts", name: "ChannelLayout"},
	metapath{path: "classes", name: "ApexClass"},
	metapath{path: "communities", name: "Community"},
	metapath{path: "components", name: "ApexComponent"},
	metapath{path: "connectedApps", name: "ConnectedApp"},
	metapath{path: "corsWhitelistOrigins", name: "CorsWhitelistOrigin"},
	metapath{path: "customApplicationComponents", name: "CustomApplicationComponent"},
	metapath{path: "customMetadata", name: "CustomMetadata"},
	metapath{path: "customPermissions", name: "CustomPermission"},
	metapath{path: "dashboards", name: "Dashboard", hasFolder: true},
	metapath{path: "dataSources", name: "ExternalDataSource"},
	metapath{path: "datacategorygroups", name: "DataCategoryGroup"},
	metapath{path: "delegateGroups", name: "DelegateGroup"},
	metapath{path: "documents", name: "Document", hasFolder: true},
	metapath{path: "EmbeddedServiceConfig", name: "EmbeddedServiceConfig"},
	metapath{path: "email", name: "EmailTemplate", hasFolder: true},
	metapath{path: "escalationRules", name: "EscalationRules"},
	metapath{path: "feedFilters", name: "CustomFeedFilter"},
	metapath{path: "flexipages", name: "FlexiPage"},
	metapath{path: "flowDefinitions", name: "FlowDefinition"},
	metapath{path: "flows", name: "Flow"},
	metapath{path: "globalPicklists", name: "GlobalPicklist"},
	metapath{path: "globalValueSets", name: "GlobalValueSet"},
	metapath{path: "groups", name: "Group"},
	metapath{path: "homePageComponents", name: "HomePageComponent"},
	metapath{path: "homePageLayouts", name: "HomePageLayout"},
	metapath{path: "installedPackages", name: "InstalledPackage"},
	metapath{path: "labels", name: "CustomLabels"},
	metapath{path: "layouts", name: "Layout"},
	metapath{path: "LeadConvertSettings", name: "LeadConvertSettings"},
	metapath{path: "letterhead", name: "Letterhead"},
	metapath{path: "matchingRules", name: "MatchingRules"},
	metapath{path: "namedCredentials", name: "NamedCredential"},
	metapath{path: "objects", name: "CustomObject"},
	metapath{path: "objectTranslations", name: "CustomObjectTranslation"},
	metapath{path: "pages", name: "ApexPage"},
	metapath{path: "pathAssistants", name: "PathAssistant"},
	metapath{path: "permissionsets", name: "PermissionSet"},
	metapath{path: "postTemplates", name: "PostTemplate"},
	metapath{path: "profiles", name: "Profile", extension: ".profile"},
	metapath{path: "postTemplates", name: "PostTemplate"},
	metapath{path: "postTemplates", name: "PostTemplate"},
	metapath{path: "profiles", name: "Profile"},
	metapath{path: "queues", name: "Queue"},
	metapath{path: "quickActions", name: "QuickAction"},
	metapath{path: "remoteSiteSettings", name: "RemoteSiteSetting"},
	metapath{path: "reports", name: "Report", hasFolder: true},
	metapath{path: "reportTypes", name: "ReportType"},
	metapath{path: "roles", name: "Role"},
	metapath{path: "scontrols", name: "Scontrol"},
	metapath{path: "settings", name: "Settings"},
	metapath{path: "sharingRules", name: "SharingRules"},
	metapath{path: "siteDotComSites", name: "SiteDotCom"},
	metapath{path: "sites", name: "CustomSite"},
	metapath{path: "standardValueSets", name: "StandardValueSet"},
	metapath{path: "staticresources", name: "StaticResource"},
	metapath{path: "synonymDictionaries", name: "SynonymDictionary"},
	metapath{path: "tabs", name: "CustomTab"},
	metapath{path: "triggers", name: "ApexTrigger"},
	metapath{path: "weblinks", name: "CustomPageWebLink"},
	metapath{path: "workflows", name: "Workflow"},
	metapath{path: "cspTrustedSites", name: "CspTrustedSite"},
}

type PackageBuilder struct {
	IsPush   bool
	Metadata map[string]MetaType
	Files    ForceMetadataFiles
}

func NewPushBuilder() PackageBuilder {
	pb := PackageBuilder{IsPush: true}
	pb.Metadata = make(map[string]MetaType)
	pb.Files = make(ForceMetadataFiles)

	return pb
}

func NewFetchBuilder() PackageBuilder {
	pb := PackageBuilder{IsPush: false}
	pb.Metadata = make(map[string]MetaType)
	pb.Files = make(ForceMetadataFiles)

	return pb
}

// Build and return package.xml
func (pb PackageBuilder) PackageXml() []byte {
	p := createPackage()

	for _, metaType := range pb.Metadata {
		p.Types = append(p.Types, metaType)
	}

	byteXml, _ := xml.MarshalIndent(p, "", "    ")
	byteXml = append([]byte(xml.Header), byteXml...)
	//if err := ioutil.WriteFile("mypackage.xml", byteXml, 0644); err != nil {
	//ErrorAndExit(err.Error())
	//}
	return byteXml
}

// Returns the full ForceMetadataFiles container
func (pb *PackageBuilder) ForceMetadataFiles() ForceMetadataFiles {
	pb.Files["package.xml"] = pb.PackageXml()
	return pb.Files
}

// Returns the source file path for a given metadata file path.
func MetaPathToSourcePath(mpath string) (spath string) {
	spath = strings.TrimSuffix(mpath, "-meta.xml")
	if spath == mpath {
		return
	}

	_, err := os.Stat(spath)
	if err != nil {
		spath = mpath
	}
	return
}

// Add a file to the builder
func (pb *PackageBuilder) AddFile(fpath string) (fname string, err error) {
	fpath, err = filepath.Abs(fpath)
	if err != nil {
		return
	}
	_, err = os.Stat(fpath)
	if err != nil {
		return
	}

	isDestructiveChanges, err := regexp.MatchString("destructiveChanges(Pre|Post)?"+regexp.QuoteMeta(".")+"xml", fpath)
	if err != nil {
		return
	}

	fpath = MetaPathToSourcePath(fpath)
	metaName, fname := getMetaTypeFromPath(fpath)
	if !isDestructiveChanges && !strings.HasSuffix(fpath, "-meta.xml") {
		pb.AddMetaToPackage(metaName, fname)
	}

	// If it's a push, we want to actually add the files
	if pb.IsPush {
		if isDestructiveChanges {
			err = pb.addDestructiveChanges(fpath)
		} else {
			err = pb.addFileToWorkingDir(metaName, fpath)
		}
	}

	return
}

// Adds the file to a temp directory for deploy
func (pb *PackageBuilder) addFileToWorkingDir(metaName string, fpath string) (err error) {
	// Get relative dir from source
	srcDir := filepath.Dir(filepath.Dir(fpath))
	for _, mp := range metapaths {
		if metaName == mp.name && mp.hasFolder {
			srcDir = filepath.Dir(srcDir)
		}
	}
	frel, _ := filepath.Rel(srcDir, fpath)

	// Try to find meta file
	hasMeta := true
	fmeta := fpath + "-meta.xml"
	fmetarel := ""
	if _, err = os.Stat(fmeta); err != nil {
		if os.IsNotExist(err) {
			hasMeta = false
		} else {
			// Has error
			return
		}
	} else {
		// Should be present since we worked back to srcDir
		fmetarel, _ = filepath.Rel(srcDir, fmeta)
	}

	fdata, err := ioutil.ReadFile(fpath)
	if err != nil {
		return
	}

	pb.Files[frel] = fdata
	if hasMeta {
		fdata, err = ioutil.ReadFile(fmeta)
		pb.Files[fmetarel] = fdata
		return
	}

	return
}

func (pb *PackageBuilder) addDestructiveChanges(fpath string) (err error) {
	fdata, err := ioutil.ReadFile(fpath)
	if err != nil {
		return
	}

	frel, _ := filepath.Rel(filepath.Dir(fpath), fpath)
	pb.Files[frel] = fdata

	return
}

func (pb *PackageBuilder) contains(members []string, name string) bool {
	for _, a := range members {
		if a == name {
			return true
		}
	}
	return false
}

// Adds a metadata name to the pending package
func (pb *PackageBuilder) AddMetaToPackage(metaName string, name string) {
	mt := pb.Metadata[metaName]
	if mt.Name == "" {
		mt.Name = metaName
	}

	if !pb.contains(mt.Members, name) {
		mt.Members = append(mt.Members, name)
		pb.Metadata[metaName] = mt
	}
}

// Gets metadata type name and target name from a file path
func getMetaTypeFromPath(fpath string) (metaName string, name string) {
	fpath, err := filepath.Abs(fpath)
	if err != nil {
		ErrorAndExit("Cound not find " + fpath)
	}
	if _, err := os.Stat(fpath); err != nil {
		ErrorAndExit("Cound not open " + fpath)
	}

	// Get the metadata type and name for the file
	metaName, fileName := getMetaForPath(fpath)
	name = strings.TrimSuffix(fileName, filepath.Ext(fileName))
	//name = strings.TrimSuffix(name, filepath.Ext(name))
	return
}

// Gets partial path based on a meta type name
func getPathForMeta(metaname string) string {
	for _, mp := range metapaths {
		if strings.EqualFold(mp.name, metaname) {
			return mp.path
		}
	}

	// Unknown, so use metaname
	return metaname
}

func findMetapathForFile(file string) (path metapath) {
	parentDir := filepath.Dir(file)
	parentName := filepath.Base(parentDir)
	grandparentName := filepath.Base(filepath.Dir(parentDir))
	fileExtension := filepath.Ext(file)

	for _, mp := range metapaths {
		if mp.hasFolder && grandparentName == mp.path {
			return mp
		}
		if mp.path == parentName {
			return mp
		}
	}

	// Hmm, maybe we can use the extension to determine the type
	for _, mp := range metapaths {
		if mp.extension == fileExtension {
			return mp
		}
	}
	return
}

// Gets meta type and name based on a path
func getMetaForPath(path string) (metaName string, objectName string) {
	parentDir := filepath.Dir(path)
	parentName := filepath.Base(parentDir)
	grandparentName := filepath.Base(filepath.Dir(parentDir))
	fileName := filepath.Base(path)

	for _, mp := range metapaths {
		if mp.hasFolder && grandparentName == mp.path {
			metaName = mp.name
			if mp.onlyFolder {
				objectName = parentName
			} else {
				objectName = parentName + "/" + fileName
			}
			return
		}
		if mp.path == parentName {
			metaName = mp.name
			objectName = fileName
			return
		}
	}

	// Unknown, so use path
	metaName = parentName
	objectName = fileName
	return
}
<|endoftext|>


<commit_before>// Copyright 2016 Palantir Technologies, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package config

import (
	"strings"

	"github.com/pkg/errors"

	"github.com/palantir/godel/framework/artifactresolver"
	"github.com/palantir/godel/framework/godel/config/internal/v0"
	"github.com/palantir/godel/pkg/osarch"
)

type LocatorWithResolverConfig v0.LocatorWithResolverConfig

func ToLocatorWithResolverConfig(in LocatorWithResolverConfig) v0.LocatorWithResolverConfig {
	return v0.LocatorWithResolverConfig(in)
}

func ToLocatorWithResolverConfigs(in []LocatorWithResolverConfig) []v0.LocatorWithResolverConfig {
	if in == nil {
		return nil
	}
	out := make([]v0.LocatorWithResolverConfig, len(in))
	for i, v := range in {
		out[i] = ToLocatorWithResolverConfig(v)
	}
	return out
}

func (c *LocatorWithResolverConfig) ToParam() (artifactresolver.LocatorWithResolverParam, error) {
	locatorCfg := LocatorConfig(c.Locator)
	locator, err := locatorCfg.ToParam()
	if err != nil {
		return artifactresolver.LocatorWithResolverParam{}, errors.Wrapf(err, "invalid locator")
	}
	var resolver artifactresolver.Resolver
	if c.Resolver != "" {
		resolverVal, err := artifactresolver.NewTemplateResolver(c.Resolver)
		if err != nil {
			return artifactresolver.LocatorWithResolverParam{}, errors.Wrapf(err, "invalid resolver")
		}
		resolver = resolverVal
	}
	return artifactresolver.LocatorWithResolverParam{
		LocatorWithChecksums: locator,
		Resolver:             resolver,
	}, nil
}

// ConfigProviderLocatorWithResolverConfig is the configuration for a locator with resolver for a configuration
// provider. It differs from a LocatorWithResolverConfig in that the locator is a ConfigProviderLocatorConfig rather
// than a LocatorConfig.
type ConfigProviderLocatorWithResolverConfig v0.ConfigProviderLocatorWithResolverConfig

func ToConfigProviderLocatorWithResolverConfig(in ConfigProviderLocatorWithResolverConfig) v0.ConfigProviderLocatorWithResolverConfig {
	return v0.ConfigProviderLocatorWithResolverConfig(in)
}

// ToParam converts the configuration into a LocatorWithResolverParam. Any checksums that exist are put in a map where
// the key is the current OS/Arch.
func (c *ConfigProviderLocatorWithResolverConfig) ToParam() (artifactresolver.LocatorWithResolverParam, error) {
	providerLocatorCfg := ConfigProviderLocatorConfig(c.Locator)
	locatorCfg, err := providerLocatorCfg.ToLocatorConfig()
	if err != nil {
		return artifactresolver.LocatorWithResolverParam{}, err
	}
	cfg := LocatorWithResolverConfig{
		Locator:  v0.LocatorConfig(locatorCfg),
		Resolver: c.Resolver,
	}
	return cfg.ToParam()
}

type LocatorConfig v0.LocatorConfig

func ToLocatorConfig(in LocatorConfig) v0.LocatorConfig {
	return v0.LocatorConfig(in)
}

func (c *LocatorConfig) ToParam() (artifactresolver.LocatorParam, error) {
	parts := strings.Split(c.ID, ":")
	if len(parts) != 3 {
		return artifactresolver.LocatorParam{}, errors.Errorf("locator ID must consist of 3 colon-delimited components ([group]:[product]:[version]), but had %d: %q", len(parts), c.ID)
	}
	var checksums map[osarch.OSArch]string
	if c.Checksums != nil {
		checksums = make(map[osarch.OSArch]string)
		for k, v := range c.Checksums {
			osArchKey, err := osarch.New(k)
			if err != nil {
				return artifactresolver.LocatorParam{}, errors.Wrapf(err, "invalid OSArch specified in checksum key for %s", c.ID)
			}
			checksums[osArchKey] = v
		}
	}
	param := artifactresolver.LocatorParam{
		Locator: artifactresolver.Locator{
			Group:   parts[0],
			Product: parts[1],
			Version: parts[2],
		},
		Checksums: checksums,
	}
	return param, nil
}

// placeholder OS/Arch used for config provider checksums
var configProviderOSArch = osarch.Current()

// ConfigProviderLocatorConfig is the configuration for a locator for a configuration provider. It differs from a
// LocatorConfig in that only a single checksum can be specified.
type ConfigProviderLocatorConfig v0.ConfigProviderLocatorConfig

// ToLocatorConfig translates the ConfigProviderLocatorConfig into a LocatorConfig where the checksum (if any exists) is
// keyed as the current OS/Arch.
func (c *ConfigProviderLocatorConfig) ToLocatorConfig() (LocatorConfig, error) {
	var checksums map[string]string
	if c.Checksum != "" {
		checksums = map[string]string{
			configProviderOSArch.String(): c.Checksum,
		}
	}
	return LocatorConfig{
		ID:        c.ID,
		Checksums: checksums,
	}, nil
}
<commit_msg>Add ConfigProviderLocatorConfig function (#389)<commit_after>// Copyright 2016 Palantir Technologies, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package config

import (
	"strings"

	"github.com/pkg/errors"

	"github.com/palantir/godel/framework/artifactresolver"
	"github.com/palantir/godel/framework/godel/config/internal/v0"
	"github.com/palantir/godel/pkg/osarch"
)

type LocatorWithResolverConfig v0.LocatorWithResolverConfig

func ToLocatorWithResolverConfig(in LocatorWithResolverConfig) v0.LocatorWithResolverConfig {
	return v0.LocatorWithResolverConfig(in)
}

func ToLocatorWithResolverConfigs(in []LocatorWithResolverConfig) []v0.LocatorWithResolverConfig {
	if in == nil {
		return nil
	}
	out := make([]v0.LocatorWithResolverConfig, len(in))
	for i, v := range in {
		out[i] = ToLocatorWithResolverConfig(v)
	}
	return out
}

func (c *LocatorWithResolverConfig) ToParam() (artifactresolver.LocatorWithResolverParam, error) {
	locatorCfg := LocatorConfig(c.Locator)
	locator, err := locatorCfg.ToParam()
	if err != nil {
		return artifactresolver.LocatorWithResolverParam{}, errors.Wrapf(err, "invalid locator")
	}
	var resolver artifactresolver.Resolver
	if c.Resolver != "" {
		resolverVal, err := artifactresolver.NewTemplateResolver(c.Resolver)
		if err != nil {
			return artifactresolver.LocatorWithResolverParam{}, errors.Wrapf(err, "invalid resolver")
		}
		resolver = resolverVal
	}
	return artifactresolver.LocatorWithResolverParam{
		LocatorWithChecksums: locator,
		Resolver:             resolver,
	}, nil
}

// ConfigProviderLocatorWithResolverConfig is the configuration for a locator with resolver for a configuration
// provider. It differs from a LocatorWithResolverConfig in that the locator is a ConfigProviderLocatorConfig rather
// than a LocatorConfig.
type ConfigProviderLocatorWithResolverConfig v0.ConfigProviderLocatorWithResolverConfig

func ToConfigProviderLocatorWithResolverConfig(in ConfigProviderLocatorWithResolverConfig) v0.ConfigProviderLocatorWithResolverConfig {
	return v0.ConfigProviderLocatorWithResolverConfig(in)
}

// ToParam converts the configuration into a LocatorWithResolverParam. Any checksums that exist are put in a map where
// the key is the current OS/Arch.
func (c *ConfigProviderLocatorWithResolverConfig) ToParam() (artifactresolver.LocatorWithResolverParam, error) {
	providerLocatorCfg := ConfigProviderLocatorConfig(c.Locator)
	locatorCfg, err := providerLocatorCfg.ToLocatorConfig()
	if err != nil {
		return artifactresolver.LocatorWithResolverParam{}, err
	}
	cfg := LocatorWithResolverConfig{
		Locator:  v0.LocatorConfig(locatorCfg),
		Resolver: c.Resolver,
	}
	return cfg.ToParam()
}

type LocatorConfig v0.LocatorConfig

func ToLocatorConfig(in LocatorConfig) v0.LocatorConfig {
	return v0.LocatorConfig(in)
}

func (c *LocatorConfig) ToParam() (artifactresolver.LocatorParam, error) {
	parts := strings.Split(c.ID, ":")
	if len(parts) != 3 {
		return artifactresolver.LocatorParam{}, errors.Errorf("locator ID must consist of 3 colon-delimited components ([group]:[product]:[version]), but had %d: %q", len(parts), c.ID)
	}
	var checksums map[osarch.OSArch]string
	if c.Checksums != nil {
		checksums = make(map[osarch.OSArch]string)
		for k, v := range c.Checksums {
			osArchKey, err := osarch.New(k)
			if err != nil {
				return artifactresolver.LocatorParam{}, errors.Wrapf(err, "invalid OSArch specified in checksum key for %s", c.ID)
			}
			checksums[osArchKey] = v
		}
	}
	param := artifactresolver.LocatorParam{
		Locator: artifactresolver.Locator{
			Group:   parts[0],
			Product: parts[1],
			Version: parts[2],
		},
		Checksums: checksums,
	}
	return param, nil
}

// placeholder OS/Arch used for config provider checksums
var configProviderOSArch = osarch.Current()

// ConfigProviderLocatorConfig is the configuration for a locator for a configuration provider. It differs from a
// LocatorConfig in that only a single checksum can be specified.
type ConfigProviderLocatorConfig v0.ConfigProviderLocatorConfig

func ToConfigProviderLocatorConfig(in ConfigProviderLocatorConfig) v0.ConfigProviderLocatorConfig {
	return v0.ConfigProviderLocatorConfig(in)
}

// ToLocatorConfig translates the ConfigProviderLocatorConfig into a LocatorConfig where the checksum (if any exists) is
// keyed as the current OS/Arch.
func (c *ConfigProviderLocatorConfig) ToLocatorConfig() (LocatorConfig, error) {
	var checksums map[string]string
	if c.Checksum != "" {
		checksums = map[string]string{
			configProviderOSArch.String(): c.Checksum,
		}
	}
	return LocatorConfig{
		ID:        c.ID,
		Checksums: checksums,
	}, nil
}
<|endoftext|>


<commit_before>/*
Copyright 2015 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package framework_test

import (
	"fmt"
	"math/rand"
	"sync"
	"testing"
	"time"

	"k8s.io/kubernetes/pkg/api"
	"k8s.io/kubernetes/pkg/client/cache"
	"k8s.io/kubernetes/pkg/controller/framework"
	"k8s.io/kubernetes/pkg/runtime"
	"k8s.io/kubernetes/pkg/util/sets"

	"github.com/google/gofuzz"
)

func Example() {
	// source simulates an apiserver object endpoint.
	source := framework.NewFakeControllerSource()

	// This will hold the downstream state, as we know it.
	downstream := cache.NewStore(framework.DeletionHandlingMetaNamespaceKeyFunc)

	// This will hold incoming changes. Note how we pass downstream in as a
	// KeyLister, that way resync operations will result in the correct set
	// of update/delete deltas.
	fifo := cache.NewDeltaFIFO(cache.MetaNamespaceKeyFunc, nil, downstream)

	// Let's do threadsafe output to get predictable test results.
	deletionCounter := make(chan string, 1000)

	cfg := &framework.Config{
		Queue:            fifo,
		ListerWatcher:    source,
		ObjectType:       &api.Pod{},
		FullResyncPeriod: time.Millisecond * 100,
		RetryOnError:     false,

		// Let's implement a simple controller that just deletes
		// everything that comes in.
		Process: func(obj interface{}) error {
			// Obj is from the Pop method of the Queue we make above.
			newest := obj.(cache.Deltas).Newest()

			if newest.Type != cache.Deleted {
				// Update our downstream store.
				err := downstream.Add(newest.Object)
				if err != nil {
					return err
				}

				// Delete this object.
				source.Delete(newest.Object.(runtime.Object))
			} else {
				// Update our downstream store.
				err := downstream.Delete(newest.Object)
				if err != nil {
					return err
				}

				// fifo's KeyOf is easiest, because it handles
				// DeletedFinalStateUnknown markers.
				key, err := fifo.KeyOf(newest.Object)
				if err != nil {
					return err
				}

				// Report this deletion.
				deletionCounter <- key
			}
			return nil
		},
	}

	// Create the controller and run it until we close stop.
	stop := make(chan struct{})
	defer close(stop)
	go framework.New(cfg).Run(stop)

	// Let's add a few objects to the source.
	testIDs := []string{"a-hello", "b-controller", "c-framework"}
	for _, name := range testIDs {
		// Note that these pods are not valid-- the fake source doesn't
		// call validation or anything.
		source.Add(&api.Pod{ObjectMeta: api.ObjectMeta{Name: name}})
	}

	// Let's wait for the controller to process the things we just added.
	outputSet := sets.String{}
	for i := 0; i < len(testIDs); i++ {
		outputSet.Insert(<-deletionCounter)
	}

	for _, key := range outputSet.List() {
		fmt.Println(key)
	}
	// Output:
	// a-hello
	// b-controller
	// c-framework
}

func ExampleInformer() {
	// source simulates an apiserver object endpoint.
	source := framework.NewFakeControllerSource()

	// Let's do threadsafe output to get predictable test results.
	deletionCounter := make(chan string, 1000)

	// Make a controller that immediately deletes anything added to it, and
	// logs anything deleted.
	_, controller := framework.NewInformer(
		source,
		&api.Pod{},
		time.Millisecond*100,
		framework.ResourceEventHandlerFuncs{
			AddFunc: func(obj interface{}) {
				source.Delete(obj.(runtime.Object))
			},
			DeleteFunc: func(obj interface{}) {
				key, err := framework.DeletionHandlingMetaNamespaceKeyFunc(obj)
				if err != nil {
					key = "oops something went wrong with the key"
				}

				// Report this deletion.
				deletionCounter <- key
			},
		},
	)

	// Run the controller and run it until we close stop.
	stop := make(chan struct{})
	defer close(stop)
	go controller.Run(stop)

	// Let's add a few objects to the source.
	testIDs := []string{"a-hello", "b-controller", "c-framework"}
	for _, name := range testIDs {
		// Note that these pods are not valid-- the fake source doesn't
		// call validation or anything.
		source.Add(&api.Pod{ObjectMeta: api.ObjectMeta{Name: name}})
	}

	// Let's wait for the controller to process the things we just added.
	outputSet := sets.String{}
	for i := 0; i < len(testIDs); i++ {
		outputSet.Insert(<-deletionCounter)
	}

	for _, key := range outputSet.List() {
		fmt.Println(key)
	}
	// Output:
	// a-hello
	// b-controller
	// c-framework
}

func TestHammerController(t *testing.T) {
	// This test executes a bunch of requests through the fake source and
	// controller framework to make sure there's no locking/threading
	// errors. If an error happens, it should hang forever or trigger the
	// race detector.

	// source simulates an apiserver object endpoint.
	source := framework.NewFakeControllerSource()

	// Let's do threadsafe output to get predictable test results.
	outputSetLock := sync.Mutex{}
	// map of key to operations done on the key
	outputSet := map[string][]string{}

	recordFunc := func(eventType string, obj interface{}) {
		key, err := framework.DeletionHandlingMetaNamespaceKeyFunc(obj)
		if err != nil {
			t.Errorf("something wrong with key: %v", err)
			key = "oops something went wrong with the key"
		}

		// Record some output when items are deleted.
		outputSetLock.Lock()
		defer outputSetLock.Unlock()
		outputSet[key] = append(outputSet[key], eventType)
	}

	// Make a controller which just logs all the changes it gets.
	_, controller := framework.NewInformer(
		source,
		&api.Pod{},
		time.Millisecond*100,
		framework.ResourceEventHandlerFuncs{
			AddFunc:    func(obj interface{}) { recordFunc("add", obj) },
			UpdateFunc: func(oldObj, newObj interface{}) { recordFunc("update", newObj) },
			DeleteFunc: func(obj interface{}) { recordFunc("delete", obj) },
		},
	)

	if controller.HasSynced() {
		t.Errorf("Expected HasSynced() to return false before we started the controller")
	}

	// Run the controller and run it until we close stop.
	stop := make(chan struct{})
	go controller.Run(stop)

	// Let's wait for the controller to do its initial sync
	time.Sleep(100 * time.Millisecond)
	if !controller.HasSynced() {
		t.Errorf("Expected HasSynced() to return true after the initial sync")
	}

	wg := sync.WaitGroup{}
	const threads = 3
	wg.Add(threads)
	for i := 0; i < threads; i++ {
		go func() {
			defer wg.Done()
			// Let's add a few objects to the source.
			currentNames := sets.String{}
			rs := rand.NewSource(rand.Int63())
			f := fuzz.New().NilChance(.5).NumElements(0, 2).RandSource(rs)
			r := rand.New(rs) // Mustn't use r and f concurrently!
			for i := 0; i < 100; i++ {
				var name string
				var isNew bool
				if currentNames.Len() == 0 || r.Intn(3) == 1 {
					f.Fuzz(&name)
					isNew = true
				} else {
					l := currentNames.List()
					name = l[r.Intn(len(l))]
				}

				pod := &api.Pod{}
				f.Fuzz(pod)
				pod.ObjectMeta.Name = name
				pod.ObjectMeta.Namespace = "default"
				// Add, update, or delete randomly.
				// Note that these pods are not valid-- the fake source doesn't
				// call validation or perform any other checking.
				if isNew {
					currentNames.Insert(name)
					source.Add(pod)
					continue
				}
				switch r.Intn(2) {
				case 0:
					currentNames.Insert(name)
					source.Modify(pod)
				case 1:
					currentNames.Delete(name)
					source.Delete(pod)
				}
			}
		}()
	}
	wg.Wait()

	// Let's wait for the controller to finish processing the things we just added.
	time.Sleep(100 * time.Millisecond)
	close(stop)

	outputSetLock.Lock()
	t.Logf("got: %#v", outputSet)
}

func TestUpdate(t *testing.T) {
	// This test is going to exercise the various paths that result in a
	// call to update.

	// source simulates an apiserver object endpoint.
	source := framework.NewFakeControllerSource()

	const (
		FROM       = "from"
		ADD_MISSED = "missed the add event"
		TO         = "to"
	)

	// These are the transitions we expect to see; because this is
	// asynchronous, there are a lot of valid possibilities.
	type pair struct{ from, to string }
	allowedTransitions := map[pair]bool{
		pair{FROM, TO}:         true,
		pair{FROM, ADD_MISSED}: true,
		pair{ADD_MISSED, TO}:   true,

		// Because a resync can happen when we've already observed one
		// of the above but before the item is deleted.
		pair{TO, TO}: true,
		// Because a resync could happen before we observe an update.
		pair{FROM, FROM}: true,
	}

	var testDoneWG sync.WaitGroup

	// Make a controller that deletes things once it observes an update.
	// It calls Done() on the wait group on deletions so we can tell when
	// everything we've added has been deleted.
	_, controller := framework.NewInformer(
		source,
		&api.Pod{},
		time.Millisecond*1,
		framework.ResourceEventHandlerFuncs{
			UpdateFunc: func(oldObj, newObj interface{}) {
				o, n := oldObj.(*api.Pod), newObj.(*api.Pod)
				from, to := o.Labels["check"], n.Labels["check"]
				if !allowedTransitions[pair{from, to}] {
					t.Errorf("observed transition %q -> %q for %v", from, to, n.Name)
				}
				source.Delete(n)
			},
			DeleteFunc: func(obj interface{}) {
				testDoneWG.Done()
			},
		},
	)

	pod := func(name, check string) *api.Pod {
		return &api.Pod{
			ObjectMeta: api.ObjectMeta{
				Name:   name,
				Labels: map[string]string{"check": check},
			},
		}
	}

	tests := []func(string){
		func(name string) {
			name = "a-" + name
			source.Add(pod(name, FROM))
			source.Modify(pod(name, TO))
		},
		func(name string) {
			name = "b-" + name
			source.Add(pod(name, FROM))
			source.ModifyDropWatch(pod(name, TO))
		},
		func(name string) {
			name = "c-" + name
			source.AddDropWatch(pod(name, FROM))
			source.Modify(pod(name, ADD_MISSED))
			source.Modify(pod(name, TO))
		},
		func(name string) {
			name = "d-" + name
			source.Add(pod(name, FROM))
		},
	}

	const threads = 3
	testDoneWG.Add(threads * len(tests))

	// Run the controller and run it until we close stop.
	// Once Run() is called, calls to testDoneWG.Done() might start, so
	// all testDoneWG.Add() calls must happen before this point
	stop := make(chan struct{})
	go controller.Run(stop)

	// run every test a few times, in parallel
	var wg sync.WaitGroup
	wg.Add(threads * len(tests))
	for i := 0; i < threads; i++ {
		for j, f := range tests {
			go func(name string, f func(string)) {
				defer wg.Done()
				f(name)
			}(fmt.Sprintf("%v-%v", i, j), f)
		}
	}
	wg.Wait()

	// Let's wait for the controller to process the things we just added.
	testDoneWG.Wait()
	close(stop)
}
<commit_msg>Proper format string for ints<commit_after>/*
Copyright 2015 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package framework_test

import (
	"fmt"
	"math/rand"
	"sync"
	"testing"
	"time"

	"k8s.io/kubernetes/pkg/api"
	"k8s.io/kubernetes/pkg/client/cache"
	"k8s.io/kubernetes/pkg/controller/framework"
	"k8s.io/kubernetes/pkg/runtime"
	"k8s.io/kubernetes/pkg/util/sets"

	"github.com/google/gofuzz"
)

func Example() {
	// source simulates an apiserver object endpoint.
	source := framework.NewFakeControllerSource()

	// This will hold the downstream state, as we know it.
	downstream := cache.NewStore(framework.DeletionHandlingMetaNamespaceKeyFunc)

	// This will hold incoming changes. Note how we pass downstream in as a
	// KeyLister, that way resync operations will result in the correct set
	// of update/delete deltas.
	fifo := cache.NewDeltaFIFO(cache.MetaNamespaceKeyFunc, nil, downstream)

	// Let's do threadsafe output to get predictable test results.
	deletionCounter := make(chan string, 1000)

	cfg := &framework.Config{
		Queue:            fifo,
		ListerWatcher:    source,
		ObjectType:       &api.Pod{},
		FullResyncPeriod: time.Millisecond * 100,
		RetryOnError:     false,

		// Let's implement a simple controller that just deletes
		// everything that comes in.
		Process: func(obj interface{}) error {
			// Obj is from the Pop method of the Queue we make above.
			newest := obj.(cache.Deltas).Newest()

			if newest.Type != cache.Deleted {
				// Update our downstream store.
				err := downstream.Add(newest.Object)
				if err != nil {
					return err
				}

				// Delete this object.
				source.Delete(newest.Object.(runtime.Object))
			} else {
				// Update our downstream store.
				err := downstream.Delete(newest.Object)
				if err != nil {
					return err
				}

				// fifo's KeyOf is easiest, because it handles
				// DeletedFinalStateUnknown markers.
				key, err := fifo.KeyOf(newest.Object)
				if err != nil {
					return err
				}

				// Report this deletion.
				deletionCounter <- key
			}
			return nil
		},
	}

	// Create the controller and run it until we close stop.
	stop := make(chan struct{})
	defer close(stop)
	go framework.New(cfg).Run(stop)

	// Let's add a few objects to the source.
	testIDs := []string{"a-hello", "b-controller", "c-framework"}
	for _, name := range testIDs {
		// Note that these pods are not valid-- the fake source doesn't
		// call validation or anything.
		source.Add(&api.Pod{ObjectMeta: api.ObjectMeta{Name: name}})
	}

	// Let's wait for the controller to process the things we just added.
	outputSet := sets.String{}
	for i := 0; i < len(testIDs); i++ {
		outputSet.Insert(<-deletionCounter)
	}

	for _, key := range outputSet.List() {
		fmt.Println(key)
	}
	// Output:
	// a-hello
	// b-controller
	// c-framework
}

func ExampleInformer() {
	// source simulates an apiserver object endpoint.
	source := framework.NewFakeControllerSource()

	// Let's do threadsafe output to get predictable test results.
	deletionCounter := make(chan string, 1000)

	// Make a controller that immediately deletes anything added to it, and
	// logs anything deleted.
	_, controller := framework.NewInformer(
		source,
		&api.Pod{},
		time.Millisecond*100,
		framework.ResourceEventHandlerFuncs{
			AddFunc: func(obj interface{}) {
				source.Delete(obj.(runtime.Object))
			},
			DeleteFunc: func(obj interface{}) {
				key, err := framework.DeletionHandlingMetaNamespaceKeyFunc(obj)
				if err != nil {
					key = "oops something went wrong with the key"
				}

				// Report this deletion.
				deletionCounter <- key
			},
		},
	)

	// Run the controller and run it until we close stop.
	stop := make(chan struct{})
	defer close(stop)
	go controller.Run(stop)

	// Let's add a few objects to the source.
	testIDs := []string{"a-hello", "b-controller", "c-framework"}
	for _, name := range testIDs {
		// Note that these pods are not valid-- the fake source doesn't
		// call validation or anything.
		source.Add(&api.Pod{ObjectMeta: api.ObjectMeta{Name: name}})
	}

	// Let's wait for the controller to process the things we just added.
	outputSet := sets.String{}
	for i := 0; i < len(testIDs); i++ {
		outputSet.Insert(<-deletionCounter)
	}

	for _, key := range outputSet.List() {
		fmt.Println(key)
	}
	// Output:
	// a-hello
	// b-controller
	// c-framework
}

func TestHammerController(t *testing.T) {
	// This test executes a bunch of requests through the fake source and
	// controller framework to make sure there's no locking/threading
	// errors. If an error happens, it should hang forever or trigger the
	// race detector.

	// source simulates an apiserver object endpoint.
	source := framework.NewFakeControllerSource()

	// Let's do threadsafe output to get predictable test results.
	outputSetLock := sync.Mutex{}
	// map of key to operations done on the key
	outputSet := map[string][]string{}

	recordFunc := func(eventType string, obj interface{}) {
		key, err := framework.DeletionHandlingMetaNamespaceKeyFunc(obj)
		if err != nil {
			t.Errorf("something wrong with key: %v", err)
			key = "oops something went wrong with the key"
		}

		// Record some output when items are deleted.
		outputSetLock.Lock()
		defer outputSetLock.Unlock()
		outputSet[key] = append(outputSet[key], eventType)
	}

	// Make a controller which just logs all the changes it gets.
	_, controller := framework.NewInformer(
		source,
		&api.Pod{},
		time.Millisecond*100,
		framework.ResourceEventHandlerFuncs{
			AddFunc:    func(obj interface{}) { recordFunc("add", obj) },
			UpdateFunc: func(oldObj, newObj interface{}) { recordFunc("update", newObj) },
			DeleteFunc: func(obj interface{}) { recordFunc("delete", obj) },
		},
	)

	if controller.HasSynced() {
		t.Errorf("Expected HasSynced() to return false before we started the controller")
	}

	// Run the controller and run it until we close stop.
	stop := make(chan struct{})
	go controller.Run(stop)

	// Let's wait for the controller to do its initial sync
	time.Sleep(100 * time.Millisecond)
	if !controller.HasSynced() {
		t.Errorf("Expected HasSynced() to return true after the initial sync")
	}

	wg := sync.WaitGroup{}
	const threads = 3
	wg.Add(threads)
	for i := 0; i < threads; i++ {
		go func() {
			defer wg.Done()
			// Let's add a few objects to the source.
			currentNames := sets.String{}
			rs := rand.NewSource(rand.Int63())
			f := fuzz.New().NilChance(.5).NumElements(0, 2).RandSource(rs)
			r := rand.New(rs) // Mustn't use r and f concurrently!
			for i := 0; i < 100; i++ {
				var name string
				var isNew bool
				if currentNames.Len() == 0 || r.Intn(3) == 1 {
					f.Fuzz(&name)
					isNew = true
				} else {
					l := currentNames.List()
					name = l[r.Intn(len(l))]
				}

				pod := &api.Pod{}
				f.Fuzz(pod)
				pod.ObjectMeta.Name = name
				pod.ObjectMeta.Namespace = "default"
				// Add, update, or delete randomly.
				// Note that these pods are not valid-- the fake source doesn't
				// call validation or perform any other checking.
				if isNew {
					currentNames.Insert(name)
					source.Add(pod)
					continue
				}
				switch r.Intn(2) {
				case 0:
					currentNames.Insert(name)
					source.Modify(pod)
				case 1:
					currentNames.Delete(name)
					source.Delete(pod)
				}
			}
		}()
	}
	wg.Wait()

	// Let's wait for the controller to finish processing the things we just added.
	time.Sleep(100 * time.Millisecond)
	close(stop)

	outputSetLock.Lock()
	t.Logf("got: %#v", outputSet)
}

func TestUpdate(t *testing.T) {
	// This test is going to exercise the various paths that result in a
	// call to update.

	// source simulates an apiserver object endpoint.
	source := framework.NewFakeControllerSource()

	const (
		FROM       = "from"
		ADD_MISSED = "missed the add event"
		TO         = "to"
	)

	// These are the transitions we expect to see; because this is
	// asynchronous, there are a lot of valid possibilities.
	type pair struct{ from, to string }
	allowedTransitions := map[pair]bool{
		pair{FROM, TO}:         true,
		pair{FROM, ADD_MISSED}: true,
		pair{ADD_MISSED, TO}:   true,

		// Because a resync can happen when we've already observed one
		// of the above but before the item is deleted.
		pair{TO, TO}: true,
		// Because a resync could happen before we observe an update.
		pair{FROM, FROM}: true,
	}

	var testDoneWG sync.WaitGroup

	// Make a controller that deletes things once it observes an update.
	// It calls Done() on the wait group on deletions so we can tell when
	// everything we've added has been deleted.
	_, controller := framework.NewInformer(
		source,
		&api.Pod{},
		time.Millisecond*1,
		framework.ResourceEventHandlerFuncs{
			UpdateFunc: func(oldObj, newObj interface{}) {
				o, n := oldObj.(*api.Pod), newObj.(*api.Pod)
				from, to := o.Labels["check"], n.Labels["check"]
				if !allowedTransitions[pair{from, to}] {
					t.Errorf("observed transition %q -> %q for %v", from, to, n.Name)
				}
				source.Delete(n)
			},
			DeleteFunc: func(obj interface{}) {
				testDoneWG.Done()
			},
		},
	)

	pod := func(name, check string) *api.Pod {
		return &api.Pod{
			ObjectMeta: api.ObjectMeta{
				Name:   name,
				Labels: map[string]string{"check": check},
			},
		}
	}

	tests := []func(string){
		func(name string) {
			name = "a-" + name
			source.Add(pod(name, FROM))
			source.Modify(pod(name, TO))
		},
		func(name string) {
			name = "b-" + name
			source.Add(pod(name, FROM))
			source.ModifyDropWatch(pod(name, TO))
		},
		func(name string) {
			name = "c-" + name
			source.AddDropWatch(pod(name, FROM))
			source.Modify(pod(name, ADD_MISSED))
			source.Modify(pod(name, TO))
		},
		func(name string) {
			name = "d-" + name
			source.Add(pod(name, FROM))
		},
	}

	const threads = 3
	testDoneWG.Add(threads * len(tests))

	// Run the controller and run it until we close stop.
	// Once Run() is called, calls to testDoneWG.Done() might start, so
	// all testDoneWG.Add() calls must happen before this point
	stop := make(chan struct{})
	go controller.Run(stop)

	// run every test a few times, in parallel
	var wg sync.WaitGroup
	wg.Add(threads * len(tests))
	for i := 0; i < threads; i++ {
		for j, f := range tests {
			go func(name string, f func(string)) {
				defer wg.Done()
				f(name)
			}(fmt.Sprintf("%d-%d", i, j), f)
		}
	}
	wg.Wait()

	// Let's wait for the controller to process the things we just added.
	testDoneWG.Wait()
	close(stop)
}
<|endoftext|>


<commit_before>package aggregate

import (
	"github.com/loadimpact/speedboat/runner"
)

func Aggregate(stats *Stats, in <-chan runner.Result) <-chan runner.Result {
	ch := make(chan runner.Result)

	go func() {
		defer close(ch)

		for res := range in {
			if res.Abort {
				continue
			}
			stats.Ingest(&res)
			ch <- res
		}

		stats.End()
	}()

	return ch
}
<commit_msg>defer is my friend<commit_after>package aggregate

import (
	"github.com/loadimpact/speedboat/runner"
)

func Aggregate(stats *Stats, in <-chan runner.Result) <-chan runner.Result {
	ch := make(chan runner.Result)

	go func() {
		defer close(ch)

		defer stats.End()
		for res := range in {
			if res.Abort {
				continue
			}
			stats.Ingest(&res)
			ch <- res
		}
	}()

	return ch
}
<|endoftext|>


<commit_before>package presence

import (
	"fmt"
	zk "launchpad.net/gozk/zookeeper"
	"time"
)

// changeNode wraps a zookeeper node and can induce watches on that node to fire.
type changeNode struct {
	conn *zk.Conn
	path string
	data string
}

// Change sets the zookeeper node's data (creating it if it doesn't exist) and
// returns the node's new MTime. This allows it to act as an ad-hoc remote clock
// in addition to its primary purpose of triggering watches on the node.
func (n *changeNode) Change() (mtime time.Time, err error) {
	stat, err := n.conn.Set(n.path, n.data, -1)
	if err == zk.ZNONODE {
		_, err = n.conn.Create(n.path, n.data, 0, zk.WorldACL(zk.PERM_ALL))
		if err == nil || err == zk.ZNODEEXISTS {
			// *Someone* created the node anyway; just try again.
			return n.Change()
		}
	}
	if err != nil {
		return
	}
	return stat.MTime(), nil
}

// Pinger continually updates a target node in zookeeper when run.
type Pinger struct {
	conn    *zk.Conn
	target  changeNode
	period  time.Duration
	closing chan bool
	closed  chan bool
}

// run calls Change on p.target every p.period nanoseconds until p is closed.
func (p *Pinger) run() {
	t := time.NewTicker(p.period)
	defer t.Stop()
	for {
		select {
		case <-p.closing:
			close(p.closed)
			return
		case <-t.C:
			_, err := p.target.Change()
			if err != nil {
				close(p.closed)
				<-p.closing
				return
			}
		}
	}
}

// Close just stops updating the target node; AliveW watches will not notice
// any change until they time out.
func (p *Pinger) Close() {
	p.closing <- true
	<-p.closed
}

// Kill stops updating and deletes the target node, causing any AliveW watches
// to observe its departure (almost) immediately.
func (p *Pinger) Kill() {
	p.Close()
	p.conn.Delete(p.target.path, -1)
}

// StartPing creates and returns an active Pinger, refreshing the contents of
// path every period nanoseconds.
func StartPing(conn *zk.Conn, path string, period time.Duration) (*Pinger, error) {
	target := changeNode{conn, path, period.String()}
	_, err := target.Change()
	if err != nil {
		return nil, err
	}
	p := &Pinger{conn, target, period, make(chan bool), make(chan bool)}
	go p.run()
	return p, nil
}

// pstate holds information about a remote Pinger's state.
type pstate struct {
	path    string
	alive   bool
	timeout time.Duration
}

// getPstate gets the latest known state of a remote Pinger, given the stat and
// content of its target node. path is present only for convenience's sake; this
// function is *not* responsible for acquiring stat and data itself, because its
// clients may or may not require a watch on the data; however, conn is still
// required, so that a clock node can be created and used to check staleness.
func getPstate(conn *zk.Conn, path string, stat *zk.Stat, data string) (pstate, error) {
	clock := changeNode{conn, "/clock", ""}
	now, err := clock.Change()
	if err != nil {
		return pstate{}, err
	}
	delay := now.Sub(stat.MTime())
	period, err := time.ParseDuration(data)
	if err != nil {
		err := fmt.Errorf("%s is not a valid presence node: %s", path, err)
		return pstate{}, err
	}
	timeout := period * 2
	alive := delay < timeout
	return pstate{path, alive, timeout}, nil
}

// Alive returns whether a remote Pinger targeting path is alive.
func Alive(conn *zk.Conn, path string) (bool, error) {
	data, stat, err := conn.Get(path)
	if err == zk.ZNONODE {
		return false, nil
	}
	if err != nil {
		return false, err
	}
	p, err := getPstate(conn, path, stat, data)
	if err != nil {
		return false, err
	}
	return p.alive, err
}

// getPstateW gets the latest known state of a remote Pinger targeting path, and
// also returns a zookeeper watch which will fire on changes to the target node.
func getPstateW(conn *zk.Conn, path string) (p pstate, zkWatch <-chan zk.Event, err error) {
	data, stat, zkWatch, err := conn.GetW(path)
	if err == zk.ZNONODE {
		stat, zkWatch, err = conn.ExistsW(path)
		if err != nil {
			return
		}
		if stat != nil {
			// Whoops, node *just* appeared. Try again.
			return getPstateW(conn, path)
		}
		return
	} else if err != nil {
		return
	}
	p, err = getPstate(conn, path, stat, data)
	return
}

// awaitDead sends false to watch when the target node is deleted, or when it has
// not been updated recently enough to still qualify as alive.
func awaitDead(conn *zk.Conn, p pstate, zkWatch <-chan zk.Event, watch chan bool) {
	dead := time.After(p.timeout)
	select {
	case <-dead:
		watch <- false
	case event := <-zkWatch:
		if !event.Ok() {
			close(watch)
			return
		}
		switch event.Type {
		case zk.EVENT_DELETED:
			watch <- false
		case zk.EVENT_CHANGED:
			p, zkWatch, err := getPstateW(conn, p.path)
			if err != nil {
				close(watch)
				return
			}
			if p.alive {
				go awaitDead(conn, p, zkWatch, watch)
			} else {
				watch <- false
			}
		}
	}
}

// awaitAlive send true to watch when the target node is changed or created.
func awaitAlive(conn *zk.Conn, p pstate, zkWatch <-chan zk.Event, watch chan bool) {
	event := <-zkWatch
	if !event.Ok() {
		close(watch)
		return
	}
	switch event.Type {
	case zk.EVENT_CREATED, zk.EVENT_CHANGED:
		watch <- true
	case zk.EVENT_DELETED:
		// The pinger is still dead (just differently dead); start a new watch.
		p, zkWatch, err := getPstateW(conn, p.path)
		if err != nil {
			close(watch)
			return
		}
		if p.alive {
			watch <- true
		} else {
			go awaitAlive(conn, p, zkWatch, watch)
		}
	}
}

// AliveW returns the latest known liveness of a remote Pinger targeting path,
// and a one-shot channel by which the caller will be notified of the first
// liveness change to be detected.
func AliveW(conn *zk.Conn, path string) (bool, <-chan bool, error) {
	p, zkWatch, err := getPstateW(conn, path)
	if err != nil {
		return false, nil, err
	}
	watch := make(chan bool)
	if p.alive {
		go awaitDead(conn, p, zkWatch, watch)
	} else {
		go awaitAlive(conn, p, zkWatch, watch)
	}
	return p.alive, watch, nil
}
<commit_msg>use repeated time.Afters instead of time.NewTicker to avoid what would other wise be an inelegant consequence of unexpected Pinger death<commit_after>package presence

import (
	"fmt"
	zk "launchpad.net/gozk/zookeeper"
	"time"
)

// changeNode wraps a zookeeper node and can induce watches on that node to fire.
type changeNode struct {
	conn *zk.Conn
	path string
	data string
}

// Change sets the zookeeper node's data (creating it if it doesn't exist) and
// returns the node's new MTime. This allows it to act as an ad-hoc remote clock
// in addition to its primary purpose of triggering watches on the node.
func (n *changeNode) Change() (mtime time.Time, err error) {
	stat, err := n.conn.Set(n.path, n.data, -1)
	if err == zk.ZNONODE {
		_, err = n.conn.Create(n.path, n.data, 0, zk.WorldACL(zk.PERM_ALL))
		if err == nil || err == zk.ZNODEEXISTS {
			// *Someone* created the node anyway; just try again.
			return n.Change()
		}
	}
	if err != nil {
		return
	}
	return stat.MTime(), nil
}

// Pinger continually updates a target node in zookeeper when run.
type Pinger struct {
	conn    *zk.Conn
	target  changeNode
	period  time.Duration
	closing chan bool
	closed  chan bool
}

// run calls Change on p.target every p.period nanoseconds until p is closed.
func (p *Pinger) run() {
	for {
		tick := time.After(p.period)
		select {
		case <-p.closing:
			close(p.closed)
			return
		case <-tick:
			_, err := p.target.Change()
			if err != nil {
				close(p.closed)
				<-p.closing
				return
			}
		}
	}
}

// Close just stops updating the target node; AliveW watches will not notice
// any change until they time out.
func (p *Pinger) Close() {
	p.closing <- true
	<-p.closed
}

// Kill stops updating and deletes the target node, causing any AliveW watches
// to observe its departure (almost) immediately.
func (p *Pinger) Kill() {
	p.Close()
	p.conn.Delete(p.target.path, -1)
}

// StartPing creates and returns an active Pinger, refreshing the contents of
// path every period nanoseconds.
func StartPing(conn *zk.Conn, path string, period time.Duration) (*Pinger, error) {
	target := changeNode{conn, path, period.String()}
	_, err := target.Change()
	if err != nil {
		return nil, err
	}
	p := &Pinger{conn, target, period, make(chan bool), make(chan bool)}
	go p.run()
	return p, nil
}

// pstate holds information about a remote Pinger's state.
type pstate struct {
	path    string
	alive   bool
	timeout time.Duration
}

// getPstate gets the latest known state of a remote Pinger, given the stat and
// content of its target node. path is present only for convenience's sake; this
// function is *not* responsible for acquiring stat and data itself, because its
// clients may or may not require a watch on the data; however, conn is still
// required, so that a clock node can be created and used to check staleness.
func getPstate(conn *zk.Conn, path string, stat *zk.Stat, data string) (pstate, error) {
	clock := changeNode{conn, "/clock", ""}
	now, err := clock.Change()
	if err != nil {
		return pstate{}, err
	}
	delay := now.Sub(stat.MTime())
	period, err := time.ParseDuration(data)
	if err != nil {
		err := fmt.Errorf("%s is not a valid presence node: %s", path, err)
		return pstate{}, err
	}
	timeout := period * 2
	alive := delay < timeout
	return pstate{path, alive, timeout}, nil
}

// Alive returns whether a remote Pinger targeting path is alive.
func Alive(conn *zk.Conn, path string) (bool, error) {
	data, stat, err := conn.Get(path)
	if err == zk.ZNONODE {
		return false, nil
	}
	if err != nil {
		return false, err
	}
	p, err := getPstate(conn, path, stat, data)
	if err != nil {
		return false, err
	}
	return p.alive, err
}

// getPstateW gets the latest known state of a remote Pinger targeting path, and
// also returns a zookeeper watch which will fire on changes to the target node.
func getPstateW(conn *zk.Conn, path string) (p pstate, zkWatch <-chan zk.Event, err error) {
	data, stat, zkWatch, err := conn.GetW(path)
	if err == zk.ZNONODE {
		stat, zkWatch, err = conn.ExistsW(path)
		if err != nil {
			return
		}
		if stat != nil {
			// Whoops, node *just* appeared. Try again.
			return getPstateW(conn, path)
		}
		return
	} else if err != nil {
		return
	}
	p, err = getPstate(conn, path, stat, data)
	return
}

// awaitDead sends false to watch when the target node is deleted, or when it has
// not been updated recently enough to still qualify as alive.
func awaitDead(conn *zk.Conn, p pstate, zkWatch <-chan zk.Event, watch chan bool) {
	dead := time.After(p.timeout)
	select {
	case <-dead:
		watch <- false
	case event := <-zkWatch:
		if !event.Ok() {
			close(watch)
			return
		}
		switch event.Type {
		case zk.EVENT_DELETED:
			watch <- false
		case zk.EVENT_CHANGED:
			p, zkWatch, err := getPstateW(conn, p.path)
			if err != nil {
				close(watch)
				return
			}
			if p.alive {
				go awaitDead(conn, p, zkWatch, watch)
			} else {
				watch <- false
			}
		}
	}
}

// awaitAlive send true to watch when the target node is changed or created.
func awaitAlive(conn *zk.Conn, p pstate, zkWatch <-chan zk.Event, watch chan bool) {
	event := <-zkWatch
	if !event.Ok() {
		close(watch)
		return
	}
	switch event.Type {
	case zk.EVENT_CREATED, zk.EVENT_CHANGED:
		watch <- true
	case zk.EVENT_DELETED:
		// The pinger is still dead (just differently dead); start a new watch.
		p, zkWatch, err := getPstateW(conn, p.path)
		if err != nil {
			close(watch)
			return
		}
		if p.alive {
			watch <- true
		} else {
			go awaitAlive(conn, p, zkWatch, watch)
		}
	}
}

// AliveW returns the latest known liveness of a remote Pinger targeting path,
// and a one-shot channel by which the caller will be notified of the first
// liveness change to be detected.
func AliveW(conn *zk.Conn, path string) (bool, <-chan bool, error) {
	p, zkWatch, err := getPstateW(conn, path)
	if err != nil {
		return false, nil, err
	}
	watch := make(chan bool)
	if p.alive {
		go awaitDead(conn, p, zkWatch, watch)
	} else {
		go awaitAlive(conn, p, zkWatch, watch)
	}
	return p.alive, watch, nil
}
<|endoftext|>