<commit_before>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Utility functions.

package io

import (
	"bytes";
	"io";
	"os";
)

// ReadAll reads from r until an error or EOF and returns the data it read.
func ReadAll(r Reader) ([]byte, os.Error) {
	var buf bytes.Buffer;
	n, err := Copy(r, &buf);
	return buf.Data(), err;
}

// ReadFile reads the file named by filename and returns the contents.
func ReadFile(filename string) ([]byte, os.Error) {
	f, err := os.Open(filename, os.O_RDONLY, 0);
	if err != nil {
		return nil, err;
	}
	defer f.Close();
	return ReadAll(f);
}

// WriteFile writes data to a file named by filename.
// If the file does not exist, WriteFile creates it with permissions perm.
//
func WriteFile(filename string, data []byte, perm int) os.Error {
	f, err := os.Open(filename, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, perm);
	if err != nil {
		return err;
	}
	n, err := f.Write(data);
	if err == nil && n < len(data) {
		err = ErrShortWrite;
	}
	f.Close();
	return err;
}
<commit_msg>comment change<commit_after>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Utility functions.

package io

import (
	"bytes";
	"io";
	"os";
)

// ReadAll reads from r until an error or EOF and returns the data it read.
func ReadAll(r Reader) ([]byte, os.Error) {
	var buf bytes.Buffer;
	n, err := Copy(r, &buf);
	return buf.Data(), err;
}

// ReadFile reads the file named by filename and returns the contents.
func ReadFile(filename string) ([]byte, os.Error) {
	f, err := os.Open(filename, os.O_RDONLY, 0);
	if err != nil {
		return nil, err;
	}
	defer f.Close();
	return ReadAll(f);
}

// WriteFile writes data to a file named by filename.
// If the file does not exist, WriteFile creates it with permissions perm;
// otherwise WriteFile truncates it before writing.
func WriteFile(filename string, data []byte, perm int) os.Error {
	f, err := os.Open(filename, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, perm);
	if err != nil {
		return err;
	}
	n, err := f.Write(data);
	if err == nil && n < len(data) {
		err = ErrShortWrite;
	}
	f.Close();
	return err;
}
<|endoftext|>


<commit_before>// Copyright 2017-present The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package hugolib

import (
	"fmt"
	"math/rand"
	"path/filepath"
	"strings"
	"testing"

	"github.com/spf13/afero"
)

type siteBuildingBenchmarkConfig struct {
	NumPages     int
	RootSections int
	Render       bool
	Shortcodes   bool
	TagsPerPage  int
}

func (s siteBuildingBenchmarkConfig) String() string {
	return fmt.Sprintf("num_root_sections=%d|num_pages=%d|tags_per_page=%d|shortcodes=%t|render=%t", s.RootSections, s.NumPages, s.TagsPerPage, s.Shortcodes, s.Render)
}

func BenchmarkSiteBuilding(b *testing.B) {
	var conf siteBuildingBenchmarkConfig
	for _, rootSections := range []int{1, 5} {
		conf.RootSections = rootSections
		for _, tagsPerPage := range []int{0, 1, 5, 20} {
			conf.TagsPerPage = tagsPerPage
			for _, numPages := range []int{10, 100, 500, 1000, 5000} {
				conf.NumPages = numPages
				for _, render := range []bool{false, true} {
					conf.Render = render
					for _, shortcodes := range []bool{false, true} {
						conf.Shortcodes = shortcodes
						doBenchMarkSiteBuilding(conf, b)
					}
				}
			}
		}
	}
}

func doBenchMarkSiteBuilding(conf siteBuildingBenchmarkConfig, b *testing.B) {
	b.Run(conf.String(), func(b *testing.B) {
		sites := createHugoBenchmarkSites(b, b.N, conf)
		b.ResetTimer()
		for i := 0; i < b.N; i++ {
			h := sites[0]

			err := h.Build(BuildCfg{SkipRender: !conf.Render})
			if err != nil {
				b.Fatal(err)
			}

			// Try to help the GC
			sites[0] = nil
			sites = sites[1:len(sites)]
		}
	})
}

func createHugoBenchmarkSites(b *testing.B, count int, cfg siteBuildingBenchmarkConfig) []*HugoSites {
	someMarkdown := `
An h1 header
============

Paragraphs are separated by a blank line.

2nd paragraph. *Italic* and **bold**. Itemized lists
look like:

  * this one
  * that one
  * the other one

Note that --- not considering the asterisk --- the actual text
content starts at 4-columns in.

> Block quotes are
> written like so.
>
> They can span multiple paragraphs,
> if you like.

Use 3 dashes for an em-dash. Use 2 dashes for ranges (ex., "it's all
in chapters 12--14"). Three dots ... will be converted to an ellipsis.
Unicode is supported. ☺
`

	someMarkdownWithShortCode := someMarkdown + `

{{< myShortcode >}}

`

	pageTemplate := `+++
title = "%s"
tags = %s
+++
%s

`

	siteConfig := `
baseURL = "http://example.com/blog"

paginate = 10
defaultContentLanguage = "en"

[Taxonomies]
tag = "tags"
category = "categories"
`
	var (
		contentPagesContent [3]string
		tags                = make([]string, cfg.TagsPerPage)
	)

	for i := 0; i < len(tags); i++ {
		tags[i] = fmt.Sprintf("Hugo %d", i)
	}

	tagsStr := "[]"
	if cfg.TagsPerPage > 0 {
		tagsStr = strings.Replace(fmt.Sprintf("%q", tags[0:cfg.TagsPerPage]), " ", ", ", -1)
	}

	if cfg.Shortcodes {
		contentPagesContent = [3]string{
			someMarkdownWithShortCode,
			strings.Repeat(someMarkdownWithShortCode, 2),
			strings.Repeat(someMarkdownWithShortCode, 3),
		}
	} else {
		contentPagesContent = [3]string{
			someMarkdown,
			strings.Repeat(someMarkdown, 2),
			strings.Repeat(someMarkdown, 3),
		}
	}

	sites := make([]*HugoSites, count)
	for i := 0; i < count; i++ {
		// Maybe consider reusing the Source fs
		mf := afero.NewMemMapFs()
		th, h := newTestSitesFromConfig(b, mf, siteConfig,
			"layouts/_default/single.html", `Single HTML|{{ .Title }}|{{ .Content }}`,
			"layouts/_default/list.html", `List HTML|{{ .Title }}|{{ .Content }}`,
			"layouts/shortcodes/myShortcode.html", `<p>MyShortcode</p>`)

		fs := th.Fs

		pagesPerSection := cfg.NumPages / cfg.RootSections

		for i := 0; i < cfg.RootSections; i++ {
			for j := 0; j < pagesPerSection; j++ {
				content := fmt.Sprintf(pageTemplate, fmt.Sprintf("Title%d_%d", i, j), tagsStr, contentPagesContent[rand.Intn(3)])

				writeSource(b, fs, filepath.Join("content", fmt.Sprintf("sect%d", i), fmt.Sprintf("page%d.md", j)), content)
			}
		}

		sites[i] = h
	}

	return sites
}
<commit_msg>hugolib: Add slightly more realistic tag distribution to benchmark<commit_after>// Copyright 2017-present The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package hugolib

import (
	"fmt"
	"math/rand"
	"path/filepath"
	"strings"
	"testing"

	"github.com/spf13/afero"
)

type siteBuildingBenchmarkConfig struct {
	NumPages     int
	RootSections int
	Render       bool
	Shortcodes   bool
	TagsPerPage  int
}

func (s siteBuildingBenchmarkConfig) String() string {
	return fmt.Sprintf("num_root_sections=%d|num_pages=%d|tags_per_page=%d|shortcodes=%t|render=%t", s.RootSections, s.NumPages, s.TagsPerPage, s.Shortcodes, s.Render)
}

func BenchmarkSiteBuilding(b *testing.B) {
	var conf siteBuildingBenchmarkConfig
	for _, rootSections := range []int{1, 5} {
		conf.RootSections = rootSections
		for _, tagsPerPage := range []int{0, 1, 5, 20} {
			conf.TagsPerPage = tagsPerPage
			for _, numPages := range []int{10, 100, 500, 1000, 5000} {
				conf.NumPages = numPages
				for _, render := range []bool{false, true} {
					conf.Render = render
					for _, shortcodes := range []bool{false, true} {
						conf.Shortcodes = shortcodes
						doBenchMarkSiteBuilding(conf, b)
					}
				}
			}
		}
	}
}

func doBenchMarkSiteBuilding(conf siteBuildingBenchmarkConfig, b *testing.B) {
	b.Run(conf.String(), func(b *testing.B) {
		sites := createHugoBenchmarkSites(b, b.N, conf)
		b.ResetTimer()
		for i := 0; i < b.N; i++ {
			h := sites[0]

			err := h.Build(BuildCfg{SkipRender: !conf.Render})
			if err != nil {
				b.Fatal(err)
			}

			// Try to help the GC
			sites[0] = nil
			sites = sites[1:len(sites)]
		}
	})
}

func createHugoBenchmarkSites(b *testing.B, count int, cfg siteBuildingBenchmarkConfig) []*HugoSites {
	someMarkdown := `
An h1 header
============

Paragraphs are separated by a blank line.

2nd paragraph. *Italic* and **bold**. Itemized lists
look like:

  * this one
  * that one
  * the other one

Note that --- not considering the asterisk --- the actual text
content starts at 4-columns in.

> Block quotes are
> written like so.
>
> They can span multiple paragraphs,
> if you like.

Use 3 dashes for an em-dash. Use 2 dashes for ranges (ex., "it's all
in chapters 12--14"). Three dots ... will be converted to an ellipsis.
Unicode is supported. ☺
`

	someMarkdownWithShortCode := someMarkdown + `

{{< myShortcode >}}

`

	pageTemplate := `+++
title = "%s"
tags = %s
+++
%s

`

	siteConfig := `
baseURL = "http://example.com/blog"

paginate = 10
defaultContentLanguage = "en"

[Taxonomies]
tag = "tags"
category = "categories"
`
	var (
		contentPagesContent [3]string
		tags                = make([]string, cfg.TagsPerPage)
	)

	tagOffset := rand.Intn(10)

	for i := 0; i < len(tags); i++ {
		tags[i] = fmt.Sprintf("Hugo %d", i+tagOffset)
	}

	tagsStr := "[]"
	if cfg.TagsPerPage > 0 {
		tagsStr = strings.Replace(fmt.Sprintf("%q", tags[0:cfg.TagsPerPage]), " ", ", ", -1)
	}

	if cfg.Shortcodes {
		contentPagesContent = [3]string{
			someMarkdownWithShortCode,
			strings.Repeat(someMarkdownWithShortCode, 2),
			strings.Repeat(someMarkdownWithShortCode, 3),
		}
	} else {
		contentPagesContent = [3]string{
			someMarkdown,
			strings.Repeat(someMarkdown, 2),
			strings.Repeat(someMarkdown, 3),
		}
	}

	sites := make([]*HugoSites, count)
	for i := 0; i < count; i++ {
		// Maybe consider reusing the Source fs
		mf := afero.NewMemMapFs()
		th, h := newTestSitesFromConfig(b, mf, siteConfig,
			"layouts/_default/single.html", `Single HTML|{{ .Title }}|{{ .Content }}`,
			"layouts/_default/list.html", `List HTML|{{ .Title }}|{{ .Content }}`,
			"layouts/shortcodes/myShortcode.html", `<p>MyShortcode</p>`)

		fs := th.Fs

		pagesPerSection := cfg.NumPages / cfg.RootSections

		for i := 0; i < cfg.RootSections; i++ {
			for j := 0; j < pagesPerSection; j++ {
				content := fmt.Sprintf(pageTemplate, fmt.Sprintf("Title%d_%d", i, j), tagsStr, contentPagesContent[rand.Intn(3)])

				writeSource(b, fs, filepath.Join("content", fmt.Sprintf("sect%d", i), fmt.Sprintf("page%d.md", j)), content)
			}
		}

		sites[i] = h
	}

	return sites
}
<|endoftext|>


<commit_before>// Google Image Search functionality
package gis

import (
	"fmt"
	"github.com/sdstrowes/gesture/core"
	"github.com/sdstrowes/gesture/util"
	"math/rand"
	neturl "net/url"
	"strings"
	"time"
)

func Create(bot *core.Gobot, config map[string]interface{}) {
	defaultUrl, useDefault := config["default"].(string)
	exclusions := getExclusions(config)
	bot.ListenFor("^gis (.*)", func(msg core.Message, matches []string) core.Response {
		for _, ex := range(exclusions) {
			if ex == msg.Channel {
				return bot.Stop()
			}
		}
		link, err := search(matches[1])
		if err != nil {
			if useDefault {
				link = defaultUrl
			} else {
				return bot.Error(err)
			}
		}
		msg.Ftfy(link)
		return bot.Stop()
	})
	bot.ListenFor("^gie's (.*)", func(msg core.Message, matches []string) core.Response {
		for _, ex := range(exclusions) {
			if ex == msg.Channel {
				return bot.Stop()
			}
		}
		link, err := search("gie's "+matches[1])
		if err != nil {
			if useDefault {
				link = defaultUrl
			} else {
				return bot.Error(err)
			}
		}
		msg.Ftfy(link)
		return bot.Stop()
	})
}

func getExclusions(config map[string]interface{}) []string {
	result := make([]string, 0)
	exclude, ok := config["exclude"].([]interface{})
	if (!ok) {
		return result
	}
	for _, ex := range(exclude) {
		result = append(result, ex.(string))
	}
	return result
}

// these structs really tie the room together, man
type gisResult struct {
	Url string
}
type gisResponse struct {
	ResponseData struct {
		Results *[]gisResult // use a pointer here b/c sometimes the results are null :(
	}
}

// Search queries google for some images, and then randomly selects one
func search(search string) (string, error) {
	searchUrl := "http://ajax.googleapis.com/ajax/services/search/images?v=1.0&q=" + neturl.QueryEscape(search)
	var gisResponse gisResponse

	// NB: i am a terrible programmer
	for i := 0; i < 5; i++ {
		err := util.UnmarshalUrl(searchUrl, &gisResponse)
		if err == nil {
			break
		}
		if i == 4 {
			return "", err
		}
	}

	if gisResponse.ResponseData.Results == nil {
		return "", fmt.Errorf("No results were returned for query %s", search)
	}

	results := *gisResponse.ResponseData.Results

	if len(results) > 0 {

		// start a goroutine to determine image info for each response result
		// we have to use buffered channels so that the senders don't hang on send after the main method exits
		imageUrlCh := make(chan string, len(results))
		errorsCh := make(chan error, len(results))
		for _, resultUrl := range results {
			go getImageInfo(resultUrl.Url, imageUrlCh, errorsCh)
		}

		// until a timeout is met, build a collection of urls
		totalResults := len(results)
		remainingResults := totalResults
		urls := make([]string, 0, totalResults)
		errors := make([]error, 0, totalResults)
		timeout := time.After(2 * time.Second)

	SEARCH:
		for remainingResults > 0 {
			select {
			case url := <-imageUrlCh:
				urls = append(urls, url)
				remainingResults--
			case err := <-errorsCh:
				errors = append(errors, err)
				remainingResults--
			case <-timeout:
				break SEARCH
			}
		}
		if len(urls) == 0 {
			return "", fmt.Errorf("No image could be found for \"%s\"", search)
		}
		return urls[rand.Intn(len(urls))], nil

	}
	return "", fmt.Errorf("No image could be found for \"%s\"", search)
}

// getImageInfo looks at the header info for the url, and if it is an image, it sends an imageInfo on the channel
func getImageInfo(url string, ch chan<- string, failures chan<- error) {
	imageUrl, contentType, err := util.ResponseHeaderContentType(url)
	if err == nil && strings.HasPrefix(contentType, "image/") {
		url, err := ensureSuffix(imageUrl, "."+contentType[len("image/"):])
		if err != nil {
			failures <- err
		} else {
			ch <- url
		}
	} else {
		failures <- fmt.Errorf("Not an image: %s", url)
	}
}

// ensureSuffix ensures a url ends with suffixes like .jpg, .png, etc
func ensureSuffix(url, suffix string) (string, error) {
	var err error
	unescapedUrl, err := neturl.QueryUnescape(url)
	if err != nil {
		return "", err
	}
	lowerSuffix := strings.ToLower(suffix)
	lowerUrl := strings.ToLower(unescapedUrl)
	if lowerSuffix == ".jpeg" && strings.HasSuffix(lowerUrl, ".jpg") {
		return url, nil
	}
	if lowerSuffix == ".jpg" && strings.HasSuffix(lowerUrl, ".jpeg") {
		return url, nil
	}
	if strings.HasSuffix(lowerUrl, lowerSuffix) {
		return url, nil
	}
	if strings.Contains(url, "?") {
		return url + "&lol=lol" + suffix, nil
	}
	return url + "?lol=lol" + suffix, nil
}
<commit_msg>update gis api<commit_after>// Google Image Search functionality
package gis

import (
	"fmt"
	"github.com/sdstrowes/gesture/core"
	"github.com/sdstrowes/gesture/util"
	"math/rand"
	neturl "net/url"
	"strings"
	"time"
)

const developerKey = "nope"
const customcx     = "nope"

func Create(bot *core.Gobot, config map[string]interface{}) {
	defaultUrl, useDefault := config["default"].(string)
	exclusions := getExclusions(config)
	bot.ListenFor("^gis (.*)", func(msg core.Message, matches []string) core.Response {
		for _, ex := range(exclusions) {
			if ex == msg.Channel {
				return bot.Stop()
			}
		}
		link, err := search(matches[1])
		if err != nil {
			if useDefault {
				link = defaultUrl
			} else {
				return bot.Error(err)
			}
		}
		msg.Ftfy(link)
		return bot.Stop()
	})
	bot.ListenFor("^gie's (.*)", func(msg core.Message, matches []string) core.Response {
		for _, ex := range(exclusions) {
			if ex == msg.Channel {
				return bot.Stop()
			}
		}
		link, err := search("gie's "+matches[1])
		if err != nil {
			if useDefault {
				link = defaultUrl
			} else {
				return bot.Error(err)
			}
		}
		msg.Ftfy(link)
		return bot.Stop()
	})
}

func getExclusions(config map[string]interface{}) []string {
	result := make([]string, 0)
	exclude, ok := config["exclude"].([]interface{})
	if (!ok) {
		return result
	}
	for _, ex := range(exclude) {
		result = append(result, ex.(string))
	}
	return result
}

type gisItems struct {
	Link string
}

type gisResponse struct {
	Items *[]gisItems
}

// Search queries google for some images, and then randomly selects one
func search(search string) (string, error) {
	searchUrl := "https://www.googleapis.com/customsearch/v1?cx="+customcx+"&key="+developerKey+"&searchType=image&q=" + neturl.QueryEscape(search)
	var gisResponse gisResponse

	// NB: i am a terrible programmer
	for i := 0; i < 5; i++ {
		err := util.UnmarshalUrl(searchUrl, &gisResponse)

		if err == nil {
			break
		}
		if i == 4 {
			return "", err
		}
	}

log.Println(gisResponse)

	if gisResponse.Items == nil {
		return "", fmt.Errorf("No results were returned for query %s", search)
	}

	results := *gisResponse.Items

	if len(results) > 0 {

		// start a goroutine to determine image info for each response result
		// we have to use buffered channels so that the senders don't hang on send after the main method exits
		imageUrlCh := make(chan string, len(results))
		errorsCh := make(chan error, len(results))
		for _, resultUrl := range results {
			go getImageInfo(resultUrl.Link, imageUrlCh, errorsCh)
		}

		// until a timeout is met, build a collection of urls
		totalResults := len(results)
		remainingResults := totalResults
		urls := make([]string, 0, totalResults)
		errors := make([]error, 0, totalResults)
		timeout := time.After(2 * time.Second)

	SEARCH:
		for remainingResults > 0 {
			select {
			case url := <-imageUrlCh:
				urls = append(urls, url)
				remainingResults--
			case err := <-errorsCh:
				errors = append(errors, err)
				remainingResults--
			case <-timeout:
				break SEARCH
			}
		}
		if len(urls) == 0 {
			return "", fmt.Errorf("No image could be found for \"%s\"", search)
		}
		return urls[rand.Intn(len(urls))], nil

	}
	return "", fmt.Errorf("No image could be found for \"%s\"", search)
}

// getImageInfo looks at the header info for the url, and if it is an image, it sends an imageInfo on the channel
func getImageInfo(url string, ch chan<- string, failures chan<- error) {
	imageUrl, contentType, err := util.ResponseHeaderContentType(url)
	if err == nil && strings.HasPrefix(contentType, "image/") {
		url, err := ensureSuffix(imageUrl, "."+contentType[len("image/"):])
		if err != nil {
			failures <- err
		} else {
			ch <- url
		}
	} else {
		failures <- fmt.Errorf("Not an image: %s", url)
	}
}

// ensureSuffix ensures a url ends with suffixes like .jpg, .png, etc
func ensureSuffix(url, suffix string) (string, error) {
	var err error
	unescapedUrl, err := neturl.QueryUnescape(url)
	if err != nil {
		return "", err
	}
	lowerSuffix := strings.ToLower(suffix)
	lowerUrl := strings.ToLower(unescapedUrl)
	if lowerSuffix == ".jpeg" && strings.HasSuffix(lowerUrl, ".jpg") {
		return url, nil
	}
	if lowerSuffix == ".jpg" && strings.HasSuffix(lowerUrl, ".jpeg") {
		return url, nil
	}
	if strings.HasSuffix(lowerUrl, lowerSuffix) {
		return url, nil
	}
	if strings.Contains(url, "?") {
		return url + "&lol=lol" + suffix, nil
	}
	return url + "?lol=lol" + suffix, nil
}
<|endoftext|>


<commit_before>package main

import "testing"

func TestFileName(t *testing.T) {
	fileNameTests := []struct {
		path   string
		newExt string
		want   string
	}{
		{"/usr/local/foo.old", "new", "/usr/local/foo.new"},
		{"./bar/foo.old", "new", "./bar/foo.new"},
		{"./foo.old", "new", "./foo.new"},
		{"~/foo.old", "new", "~/foo.new"},
		{"foo.old", "new", "foo.new"},
	}

	for _, tt := range fileNameTests {
		got := outPath(tt.path, tt.newExt)
		if got != tt.want {
			t.Errorf("got = %s, but want = %s", got, tt.want)
		}
	}
}
<commit_msg>06/assembler/main: add convert test<commit_after>package main

import (
	"io/ioutil"
	"strings"
	"testing"
)

func TestConvert(t *testing.T) {
	convert("../max/Max.asm")

	gotb, _ := ioutil.ReadFile("../max/Max.hack")
	wantb, _ := ioutil.ReadFile("../max/MaxL.hack")
	got := strings.Split(string(gotb), "\n")
	want := strings.Split(string(wantb), "\n")

	if len(got) != len(want) {
		t.Fatalf("the number of lines should be %d, but got %d", len(want), len(got))
	}
	for i := range got {
		if got[i] != want[i] {
			t.Errorf("line %2d: got %s != want %s", i+1, got[i], want[i])
		}
	}
}

func TestFileName(t *testing.T) {
	fileNameTests := []struct {
		path   string
		newExt string
		want   string
	}{
		{"/usr/local/foo.old", "new", "/usr/local/foo.new"},
		{"./bar/foo.old", "new", "./bar/foo.new"},
		{"./foo.old", "new", "./foo.new"},
		{"~/foo.old", "new", "~/foo.new"},
		{"foo.old", "new", "foo.new"},
	}

	for _, tt := range fileNameTests {
		got := outPath(tt.path, tt.newExt)
		if got != tt.want {
			t.Errorf("got = %s, but want = %s", got, tt.want)
		}
	}
}
<|endoftext|>


<commit_before>package memory

import (
	"fmt"
	"os"
	"strconv"
	"sync"
	"testing"

	"github.com/grafana/metrictank/idx"
	"gopkg.in/raintank/schema.v1"
)

var (
	ix         idx.MetricIndex
	queries    []query
	tagQueries []tagQuery
)

type query struct {
	Pattern         string
	ExpectedResults int
}

type tagQuery struct {
	Expressions     []string
	ExpectedResults int
}

type metric struct {
	Name string
	Tags []string
}

func cpuMetrics(dcCount, hostCount, hostOffset, cpuCount int, prefix string) []metric {
	var series []metric
	for dc := 0; dc < dcCount; dc++ {
		for host := hostOffset; host < hostCount+hostOffset; host++ {
			for cpu := 0; cpu < cpuCount; cpu++ {
				p := prefix + ".dc" + strconv.Itoa(dc) + ".host" + strconv.Itoa(host) + ".cpu." + strconv.Itoa(cpu)
				for _, m := range []string{"idle", "interrupt", "nice", "softirq", "steal", "system", "user", "wait"} {
					series = append(series, metric{
						Name: p + "." + m,
						Tags: []string{
							"dc=dc" + strconv.Itoa(dc),
							"host=host" + strconv.Itoa(host),
							"device=cpu",
							"cpu=cpu" + strconv.Itoa(cpu),
							"metric=" + m,
						},
					})
				}
			}
		}
	}
	return series
}

func diskMetrics(dcCount, hostCount, hostOffset, diskCount int, prefix string) []metric {
	var series []metric
	for dc := 0; dc < dcCount; dc++ {
		for host := hostOffset; host < hostCount+hostOffset; host++ {
			for disk := 0; disk < diskCount; disk++ {
				p := prefix + ".dc" + strconv.Itoa(dc) + ".host" + strconv.Itoa(host) + ".disk.disk" + strconv.Itoa(disk)
				for _, m := range []string{"disk_merged", "disk_octets", "disk_ops", "disk_time"} {
					series = append(series, metric{
						Name: p + "." + m + ".read",
						Tags: []string{
							"dc=dc" + strconv.Itoa(dc),
							"host=host" + strconv.Itoa(host),
							"device=disk",
							"disk=disk" + strconv.Itoa(disk),
							"metric=" + m,
							"direction=read",
						},
					})
					series = append(series, metric{
						Name: p + "." + m + ".write",
						Tags: []string{
							"dc=dc" + strconv.Itoa(dc),
							"host=host" + strconv.Itoa(host),
							"device=disk",
							"disk=disk" + strconv.Itoa(disk),
							"metric=" + m,
							"direction=write",
						},
					})
				}
			}
		}
	}
	return series
}

func TestMain(m *testing.M) {
	_tagSupport := tagSupport
	defer func() { tagSupport = _tagSupport }()
	tagSupport = true

	Init()
	os.Exit(m.Run())
}

func Init() {
	ix = New()
	ix.Init()

	var data *schema.MetricData

	for _, series := range cpuMetrics(5, 1000, 0, 32, "collectd") {
		data = &schema.MetricData{
			Name:     series.Name,
			Metric:   series.Name,
			Tags:     series.Tags,
			Interval: 10,
			OrgId:    1,
		}
		data.SetId()
		ix.AddOrUpdate(data, 1)
	}
	for _, series := range diskMetrics(5, 1000, 0, 10, "collectd") {
		data = &schema.MetricData{
			Name:     series.Name,
			Metric:   series.Name,
			Tags:     series.Tags,
			Interval: 10,
			OrgId:    1,
		}
		data.SetId()
		ix.AddOrUpdate(data, 1)
	}
	// orgId has 1,680,000 series

	for _, series := range cpuMetrics(5, 100, 950, 32, "collectd") {
		data = &schema.MetricData{
			Name:     series.Name,
			Metric:   series.Name,
			Tags:     series.Tags,
			Interval: 10,
			OrgId:    2,
		}
		data.SetId()
		ix.AddOrUpdate(data, 1)
	}
	for _, series := range diskMetrics(5, 100, 950, 10, "collectd") {
		data = &schema.MetricData{
			Name:     series.Name,
			Metric:   series.Name,
			Tags:     series.Tags,
			Interval: 10,
			OrgId:    2,
		}
		data.SetId()
		ix.AddOrUpdate(data, 1)
	}
	//orgId 2 has 168,000 mertics

	queries = []query{
		//LEAF queries
		{Pattern: "collectd.dc1.host960.disk.disk1.disk_ops.read", ExpectedResults: 1},
		{Pattern: "collectd.dc1.host960.disk.disk1.disk_ops.*", ExpectedResults: 2},
		{Pattern: "collectd.*.host960.disk.disk1.disk_ops.read", ExpectedResults: 5},
		{Pattern: "collectd.*.host960.disk.disk1.disk_ops.*", ExpectedResults: 10},
		{Pattern: "collectd.d*.host960.disk.disk1.disk_ops.*", ExpectedResults: 10},
		{Pattern: "collectd.[abcd]*.host960.disk.disk1.disk_ops.*", ExpectedResults: 10},
		{Pattern: "collectd.{dc1,dc50}.host960.disk.disk1.disk_ops.*", ExpectedResults: 2},

		{Pattern: "collectd.dc3.host960.cpu.1.idle", ExpectedResults: 1},
		{Pattern: "collectd.dc30.host960.cpu.1.idle", ExpectedResults: 0},
		{Pattern: "collectd.dc3.host960.*.*.idle", ExpectedResults: 32},
		{Pattern: "collectd.dc3.host960.*.*.idle", ExpectedResults: 32},

		{Pattern: "collectd.dc3.host96[0-9].cpu.1.idle", ExpectedResults: 10},
		{Pattern: "collectd.dc30.host96[0-9].cpu.1.idle", ExpectedResults: 0},
		{Pattern: "collectd.dc3.host96[0-9].*.*.idle", ExpectedResults: 320},
		{Pattern: "collectd.dc3.host96[0-9].*.*.idle", ExpectedResults: 320},

		{Pattern: "collectd.{dc1,dc2,dc3}.host960.cpu.1.idle", ExpectedResults: 3},
		{Pattern: "collectd.{dc*, a*}.host960.cpu.1.idle", ExpectedResults: 5},

		//Branch queries
		{Pattern: "collectd.dc1.host960.*", ExpectedResults: 2},
		{Pattern: "collectd.*.host960.disk.disk1.*", ExpectedResults: 20},
		{Pattern: "collectd.[abcd]*.host960.disk.disk1.*", ExpectedResults: 20},

		{Pattern: "collectd.*.host960.disk.*.*", ExpectedResults: 200},
		{Pattern: "*.dc3.host960.cpu.1.*", ExpectedResults: 8},
		{Pattern: "*.dc3.host96{1,3}.cpu.1.*", ExpectedResults: 16},
		{Pattern: "*.dc3.{host,server}96{1,3}.cpu.1.*", ExpectedResults: 16},

		{Pattern: "*.dc3.{host,server}9[6-9]{1,3}.cpu.1.*", ExpectedResults: 64},
	}

	tagQueries = []tagQuery{
		// simple matching
		{Expressions: []string{"dc=dc1", "host=host960", "disk=disk1", "metric=disk_ops"}, ExpectedResults: 2},
		{Expressions: []string{"dc=dc3", "host=host960", "disk=disk2", "direction=read"}, ExpectedResults: 4},

		// regular expressions
		{Expressions: []string{"dc=~dc[1-3]", "host=~host3[5-9]{2}", "metric=disk_ops"}, ExpectedResults: 1500},
		{Expressions: []string{"dc=~dc[0-9]", "host=~host97[0-9]", "disk=disk2", "metric=disk_ops"}, ExpectedResults: 100},

		// matching and filtering
		{Expressions: []string{"dc=dc1", "host=host666", "cpu=cpu12", "device=cpu", "metric!=softirq"}, ExpectedResults: 7},
		{Expressions: []string{"dc=dc1", "host=host966", "cpu!=cpu12", "device!=disk", "metric!=softirq"}, ExpectedResults: 217},

		// matching and filtering by regular expressions
		{Expressions: []string{"dc=dc1", "host=host666", "cpu!=~cpu[0-9]{2}", "device!=~d.*"}, ExpectedResults: 16},
		{Expressions: []string{"dc=dc1", "host!=~host10[0-9]{2}", "device!=~c.*"}, ExpectedResults: 1500},
	}
}

func ixFind(org, q int) {
	nodes, err := ix.Find(org, queries[q].Pattern, 0)
	if err != nil {
		panic(err)
	}
	if len(nodes) != queries[q].ExpectedResults {
		for _, n := range nodes {
			fmt.Println(n.Path)
		}
		panic(fmt.Sprintf("%s expected %d got %d results instead", queries[q].Pattern, queries[q].ExpectedResults, len(nodes)))
	}
}

func BenchmarkFind(b *testing.B) {
	if ix == nil {
		Init()
	}
	queryCount := len(queries)
	b.ReportAllocs()
	b.ResetTimer()
	for n := 0; n < b.N; n++ {
		q := n % queryCount
		org := (n % 2) + 1
		ixFind(org, q)
	}
}

type testQ struct {
	q   int
	org int
}

func BenchmarkConcurrent4Find(b *testing.B) {
	if ix == nil {
		Init()
	}

	queryCount := len(queries)
	if ix == nil {
		Init()
	}

	ch := make(chan testQ)
	for i := 0; i < 4; i++ {
		go func() {
			for q := range ch {
				ixFind(q.org, q.q)
			}
		}()
	}
	b.ReportAllocs()
	b.ResetTimer()
	for n := 0; n < b.N; n++ {
		q := n % queryCount
		org := (n % 2) + 1
		ch <- testQ{q: q, org: org}
	}
	close(ch)
}

func BenchmarkConcurrent8Find(b *testing.B) {
	if ix == nil {
		Init()
	}

	queryCount := len(queries)
	if ix == nil {
		Init()
	}

	ch := make(chan testQ)
	for i := 0; i < 8; i++ {
		go func() {
			for q := range ch {
				ixFind(q.org, q.q)
			}
		}()
	}
	b.ReportAllocs()
	b.ResetTimer()
	for n := 0; n < b.N; n++ {
		q := n % queryCount
		org := (n % 2) + 1
		ch <- testQ{q: q, org: org}
	}
	close(ch)
}

func ixFindByTag(org, q int) {
	series, err := ix.IdsByTagExpressions(org, tagQueries[q].Expressions)
	if err != nil {
		panic(err)
	}
	if len(series) != tagQueries[q].ExpectedResults {
		for _, s := range series {
			fmt.Println(s)
		}
		panic(fmt.Sprintf("%+v expected %d got %d results instead", tagQueries[q].Expressions, tagQueries[q].ExpectedResults, len(series)))
	}
}

func BenchmarkTagFindSimpleIntersect(b *testing.B) {
	b.ReportAllocs()
	b.ResetTimer()
	fmt.Println("bench count ", b.N)
	for n := 0; n < b.N; n++ {
		q := n % 2
		org := (n % 2) + 1
		fmt.Println("running ", q)
		ixFindByTag(org, q)
	}
}

func BenchmarkTagFindRegexIntersect(b *testing.B) {
	b.ReportAllocs()
	b.ResetTimer()
	fmt.Println("bench count ", b.N)
	for n := 0; n < b.N; n++ {
		q := (n % 2) + 2
		org := (n % 2) + 1
		fmt.Println("running ", q)
		ixFindByTag(org, q)
	}
}

func BenchmarkConcurrent8TagFind(b *testing.B) {
	var wg sync.WaitGroup
	ch := make(chan testQ)
	for i := 0; i < 8; i++ {
		wg.Add(1)
		go func() {
			for q := range ch {
				fmt.Println("running ", q.q)
				ixFindByTag(q.org, q.q)
			}
			wg.Done()
		}()
	}

	queryCount := len(tagQueries)
	b.ReportAllocs()
	b.ResetTimer()
	for n := 0; n < b.N; n++ {
		q := n % queryCount
		org := (n % 2) + 1
		ch <- testQ{q: q, org: org}
	}
	close(ch)
	wg.Wait()
}
<commit_msg>remove prints<commit_after>package memory

import (
	"fmt"
	"os"
	"strconv"
	"sync"
	"testing"

	"github.com/grafana/metrictank/idx"
	"gopkg.in/raintank/schema.v1"
)

var (
	ix         idx.MetricIndex
	queries    []query
	tagQueries []tagQuery
)

type query struct {
	Pattern         string
	ExpectedResults int
}

type tagQuery struct {
	Expressions     []string
	ExpectedResults int
}

type metric struct {
	Name string
	Tags []string
}

func cpuMetrics(dcCount, hostCount, hostOffset, cpuCount int, prefix string) []metric {
	var series []metric
	for dc := 0; dc < dcCount; dc++ {
		for host := hostOffset; host < hostCount+hostOffset; host++ {
			for cpu := 0; cpu < cpuCount; cpu++ {
				p := prefix + ".dc" + strconv.Itoa(dc) + ".host" + strconv.Itoa(host) + ".cpu." + strconv.Itoa(cpu)
				for _, m := range []string{"idle", "interrupt", "nice", "softirq", "steal", "system", "user", "wait"} {
					series = append(series, metric{
						Name: p + "." + m,
						Tags: []string{
							"dc=dc" + strconv.Itoa(dc),
							"host=host" + strconv.Itoa(host),
							"device=cpu",
							"cpu=cpu" + strconv.Itoa(cpu),
							"metric=" + m,
						},
					})
				}
			}
		}
	}
	return series
}

func diskMetrics(dcCount, hostCount, hostOffset, diskCount int, prefix string) []metric {
	var series []metric
	for dc := 0; dc < dcCount; dc++ {
		for host := hostOffset; host < hostCount+hostOffset; host++ {
			for disk := 0; disk < diskCount; disk++ {
				p := prefix + ".dc" + strconv.Itoa(dc) + ".host" + strconv.Itoa(host) + ".disk.disk" + strconv.Itoa(disk)
				for _, m := range []string{"disk_merged", "disk_octets", "disk_ops", "disk_time"} {
					series = append(series, metric{
						Name: p + "." + m + ".read",
						Tags: []string{
							"dc=dc" + strconv.Itoa(dc),
							"host=host" + strconv.Itoa(host),
							"device=disk",
							"disk=disk" + strconv.Itoa(disk),
							"metric=" + m,
							"direction=read",
						},
					})
					series = append(series, metric{
						Name: p + "." + m + ".write",
						Tags: []string{
							"dc=dc" + strconv.Itoa(dc),
							"host=host" + strconv.Itoa(host),
							"device=disk",
							"disk=disk" + strconv.Itoa(disk),
							"metric=" + m,
							"direction=write",
						},
					})
				}
			}
		}
	}
	return series
}

func TestMain(m *testing.M) {
	_tagSupport := tagSupport
	defer func() { tagSupport = _tagSupport }()
	tagSupport = true

	Init()
	os.Exit(m.Run())
}

func Init() {
	ix = New()
	ix.Init()

	var data *schema.MetricData

	for _, series := range cpuMetrics(5, 1000, 0, 32, "collectd") {
		data = &schema.MetricData{
			Name:     series.Name,
			Metric:   series.Name,
			Tags:     series.Tags,
			Interval: 10,
			OrgId:    1,
		}
		data.SetId()
		ix.AddOrUpdate(data, 1)
	}
	for _, series := range diskMetrics(5, 1000, 0, 10, "collectd") {
		data = &schema.MetricData{
			Name:     series.Name,
			Metric:   series.Name,
			Tags:     series.Tags,
			Interval: 10,
			OrgId:    1,
		}
		data.SetId()
		ix.AddOrUpdate(data, 1)
	}
	// orgId has 1,680,000 series

	for _, series := range cpuMetrics(5, 100, 950, 32, "collectd") {
		data = &schema.MetricData{
			Name:     series.Name,
			Metric:   series.Name,
			Tags:     series.Tags,
			Interval: 10,
			OrgId:    2,
		}
		data.SetId()
		ix.AddOrUpdate(data, 1)
	}
	for _, series := range diskMetrics(5, 100, 950, 10, "collectd") {
		data = &schema.MetricData{
			Name:     series.Name,
			Metric:   series.Name,
			Tags:     series.Tags,
			Interval: 10,
			OrgId:    2,
		}
		data.SetId()
		ix.AddOrUpdate(data, 1)
	}
	//orgId 2 has 168,000 mertics

	queries = []query{
		//LEAF queries
		{Pattern: "collectd.dc1.host960.disk.disk1.disk_ops.read", ExpectedResults: 1},
		{Pattern: "collectd.dc1.host960.disk.disk1.disk_ops.*", ExpectedResults: 2},
		{Pattern: "collectd.*.host960.disk.disk1.disk_ops.read", ExpectedResults: 5},
		{Pattern: "collectd.*.host960.disk.disk1.disk_ops.*", ExpectedResults: 10},
		{Pattern: "collectd.d*.host960.disk.disk1.disk_ops.*", ExpectedResults: 10},
		{Pattern: "collectd.[abcd]*.host960.disk.disk1.disk_ops.*", ExpectedResults: 10},
		{Pattern: "collectd.{dc1,dc50}.host960.disk.disk1.disk_ops.*", ExpectedResults: 2},

		{Pattern: "collectd.dc3.host960.cpu.1.idle", ExpectedResults: 1},
		{Pattern: "collectd.dc30.host960.cpu.1.idle", ExpectedResults: 0},
		{Pattern: "collectd.dc3.host960.*.*.idle", ExpectedResults: 32},
		{Pattern: "collectd.dc3.host960.*.*.idle", ExpectedResults: 32},

		{Pattern: "collectd.dc3.host96[0-9].cpu.1.idle", ExpectedResults: 10},
		{Pattern: "collectd.dc30.host96[0-9].cpu.1.idle", ExpectedResults: 0},
		{Pattern: "collectd.dc3.host96[0-9].*.*.idle", ExpectedResults: 320},
		{Pattern: "collectd.dc3.host96[0-9].*.*.idle", ExpectedResults: 320},

		{Pattern: "collectd.{dc1,dc2,dc3}.host960.cpu.1.idle", ExpectedResults: 3},
		{Pattern: "collectd.{dc*, a*}.host960.cpu.1.idle", ExpectedResults: 5},

		//Branch queries
		{Pattern: "collectd.dc1.host960.*", ExpectedResults: 2},
		{Pattern: "collectd.*.host960.disk.disk1.*", ExpectedResults: 20},
		{Pattern: "collectd.[abcd]*.host960.disk.disk1.*", ExpectedResults: 20},

		{Pattern: "collectd.*.host960.disk.*.*", ExpectedResults: 200},
		{Pattern: "*.dc3.host960.cpu.1.*", ExpectedResults: 8},
		{Pattern: "*.dc3.host96{1,3}.cpu.1.*", ExpectedResults: 16},
		{Pattern: "*.dc3.{host,server}96{1,3}.cpu.1.*", ExpectedResults: 16},

		{Pattern: "*.dc3.{host,server}9[6-9]{1,3}.cpu.1.*", ExpectedResults: 64},
	}

	tagQueries = []tagQuery{
		// simple matching
		{Expressions: []string{"dc=dc1", "host=host960", "disk=disk1", "metric=disk_ops"}, ExpectedResults: 2},
		{Expressions: []string{"dc=dc3", "host=host960", "disk=disk2", "direction=read"}, ExpectedResults: 4},

		// regular expressions
		{Expressions: []string{"dc=~dc[1-3]", "host=~host3[5-9]{2}", "metric=disk_ops"}, ExpectedResults: 1500},
		{Expressions: []string{"dc=~dc[0-9]", "host=~host97[0-9]", "disk=disk2", "metric=disk_ops"}, ExpectedResults: 100},

		// matching and filtering
		{Expressions: []string{"dc=dc1", "host=host666", "cpu=cpu12", "device=cpu", "metric!=softirq"}, ExpectedResults: 7},
		{Expressions: []string{"dc=dc1", "host=host966", "cpu!=cpu12", "device!=disk", "metric!=softirq"}, ExpectedResults: 217},

		// matching and filtering by regular expressions
		{Expressions: []string{"dc=dc1", "host=host666", "cpu!=~cpu[0-9]{2}", "device!=~d.*"}, ExpectedResults: 16},
		{Expressions: []string{"dc=dc1", "host!=~host10[0-9]{2}", "device!=~c.*"}, ExpectedResults: 1500},
	}
}

func ixFind(org, q int) {
	nodes, err := ix.Find(org, queries[q].Pattern, 0)
	if err != nil {
		panic(err)
	}
	if len(nodes) != queries[q].ExpectedResults {
		for _, n := range nodes {
			fmt.Println(n.Path)
		}
		panic(fmt.Sprintf("%s expected %d got %d results instead", queries[q].Pattern, queries[q].ExpectedResults, len(nodes)))
	}
}

func BenchmarkFind(b *testing.B) {
	if ix == nil {
		Init()
	}
	queryCount := len(queries)
	b.ReportAllocs()
	b.ResetTimer()
	for n := 0; n < b.N; n++ {
		q := n % queryCount
		org := (n % 2) + 1
		ixFind(org, q)
	}
}

type testQ struct {
	q   int
	org int
}

func BenchmarkConcurrent4Find(b *testing.B) {
	if ix == nil {
		Init()
	}

	queryCount := len(queries)
	if ix == nil {
		Init()
	}

	ch := make(chan testQ)
	for i := 0; i < 4; i++ {
		go func() {
			for q := range ch {
				ixFind(q.org, q.q)
			}
		}()
	}
	b.ReportAllocs()
	b.ResetTimer()
	for n := 0; n < b.N; n++ {
		q := n % queryCount
		org := (n % 2) + 1
		ch <- testQ{q: q, org: org}
	}
	close(ch)
}

func BenchmarkConcurrent8Find(b *testing.B) {
	if ix == nil {
		Init()
	}

	queryCount := len(queries)
	if ix == nil {
		Init()
	}

	ch := make(chan testQ)
	for i := 0; i < 8; i++ {
		go func() {
			for q := range ch {
				ixFind(q.org, q.q)
			}
		}()
	}
	b.ReportAllocs()
	b.ResetTimer()
	for n := 0; n < b.N; n++ {
		q := n % queryCount
		org := (n % 2) + 1
		ch <- testQ{q: q, org: org}
	}
	close(ch)
}

func ixFindByTag(org, q int) {
	series, err := ix.IdsByTagExpressions(org, tagQueries[q].Expressions)
	if err != nil {
		panic(err)
	}
	if len(series) != tagQueries[q].ExpectedResults {
		for _, s := range series {
			fmt.Println(s)
		}
		panic(fmt.Sprintf("%+v expected %d got %d results instead", tagQueries[q].Expressions, tagQueries[q].ExpectedResults, len(series)))
	}
}

func BenchmarkTagFindSimpleIntersect(b *testing.B) {
	b.ReportAllocs()
	b.ResetTimer()
	for n := 0; n < b.N; n++ {
		q := n % 2
		org := (n % 2) + 1
		ixFindByTag(org, q)
	}
}

func BenchmarkTagFindRegexIntersect(b *testing.B) {
	b.ReportAllocs()
	b.ResetTimer()
	for n := 0; n < b.N; n++ {
		q := (n % 2) + 2
		org := (n % 2) + 1
		ixFindByTag(org, q)
	}
}

func BenchmarkConcurrent8TagFind(b *testing.B) {
	var wg sync.WaitGroup
	ch := make(chan testQ)
	for i := 0; i < 8; i++ {
		wg.Add(1)
		go func() {
			for q := range ch {
				ixFindByTag(q.org, q.q)
			}
			wg.Done()
		}()
	}

	queryCount := len(tagQueries)
	b.ReportAllocs()
	b.ResetTimer()
	for n := 0; n < b.N; n++ {
		q := n % queryCount
		org := (n % 2) + 1
		ch <- testQ{q: q, org: org}
	}
	close(ch)
	wg.Wait()
}
<|endoftext|>


<commit_before>package rpcd

import (
	"encoding/gob"
	"errors"
	"io"
	"time"

	imgclient "github.com/Symantec/Dominator/imageserver/client"
	"github.com/Symantec/Dominator/lib/format"
	"github.com/Symantec/Dominator/lib/hash"
	"github.com/Symantec/Dominator/lib/image"
	objectclient "github.com/Symantec/Dominator/lib/objectserver/client"
	"github.com/Symantec/Dominator/lib/srpc"
	"github.com/Symantec/Dominator/proto/imageserver"
)

func (t *srpcType) replicator() {
	initialTimeout := time.Second * 15
	timeout := initialTimeout
	var nextSleepStopTime time.Time
	for {
		nextSleepStopTime = time.Now().Add(timeout)
		if client, err := srpc.DialHTTP("tcp", t.replicationMaster,
			timeout); err != nil {
			t.logger.Printf("Error dialling: %s %s\n", t.replicationMaster, err)
		} else {
			if conn, err := client.Call(
				"ImageServer.GetImageUpdates"); err != nil {
				t.logger.Println(err)
			} else {
				if err := t.getUpdates(conn); err != nil {
					if err == io.EOF {
						t.logger.Println(
							"Connection to image replicator closed")
						if nextSleepStopTime.Sub(time.Now()) < 1 {
							timeout = initialTimeout
						}
					} else {
						t.logger.Println(err)
					}
				}
				conn.Close()
			}
			client.Close()
		}
		time.Sleep(nextSleepStopTime.Sub(time.Now()))
		if timeout < time.Minute {
			timeout *= 2
		}
	}
}

func (t *srpcType) getUpdates(conn *srpc.Conn) error {
	t.logger.Printf("Image replicator: connected to: %s\n", t.replicationMaster)
	replicationStartTime := time.Now()
	decoder := gob.NewDecoder(conn)
	initialImages := make(map[string]struct{})
	if t.archiveMode {
		initialImages = nil
	}
	for {
		var imageUpdate imageserver.ImageUpdate
		if err := decoder.Decode(&imageUpdate); err != nil {
			if err == io.EOF {
				return err
			}
			return errors.New("decode err: " + err.Error())
		}
		switch imageUpdate.Operation {
		case imageserver.OperationAddImage:
			if imageUpdate.Name == "" {
				if initialImages != nil {
					t.deleteMissingImages(initialImages)
					initialImages = nil
				}
				t.logger.Printf("Replicated all current images in %s\n",
					format.Duration(time.Since(replicationStartTime)))
				continue
			}
			if initialImages != nil {
				initialImages[imageUpdate.Name] = struct{}{}
			}
			if err := t.addImage(imageUpdate.Name); err != nil {
				return errors.New("error adding image: " + imageUpdate.Name +
					": " + err.Error())
			}
		case imageserver.OperationDeleteImage:
			if t.archiveMode {
				continue
			}
			t.logger.Printf("Replicator(%s): delete image\n", imageUpdate.Name)
			if err := t.imageDataBase.DeleteImage(imageUpdate.Name,
				nil); err != nil {
				return err
			}
		case imageserver.OperationMakeDirectory:
			directory := imageUpdate.Directory
			if directory == nil {
				return errors.New("nil imageUpdate.Directory")
			}
			if err := t.imageDataBase.UpdateDirectory(*directory); err != nil {
				return err
			}
		}
	}
}

func (t *srpcType) deleteMissingImages(imagesToKeep map[string]struct{}) {
	missingImages := make([]string, 0)
	for _, imageName := range t.imageDataBase.ListImages() {
		if _, ok := imagesToKeep[imageName]; !ok {
			missingImages = append(missingImages, imageName)
		}
	}
	for _, imageName := range missingImages {
		t.logger.Printf("Replicator(%s): delete missing image\n", imageName)
		if err := t.imageDataBase.DeleteImage(imageName, nil); err != nil {
			t.logger.Println(err)
		}
	}
}

func (t *srpcType) addImage(name string) error {
	timeout := time.Second * 60
	if t.imageDataBase.CheckImage(name) || t.checkImageBeingInjected(name) {
		return nil
	}
	t.logger.Printf("Replicator(%s): add image\n", name)
	client, err := t.imageserverResource.GetHTTP(nil, timeout)
	if err != nil {
		return err
	}
	defer client.Put()
	img, err := imgclient.GetImage(client, name)
	if err != nil {
		client.Close()
		return err
	}
	if img == nil {
		return errors.New(name + ": not found")
	}
	t.logger.Printf("Replicator(%s): downloaded image\n", name)
	if t.archiveMode && !img.ExpiresAt.IsZero() && !*archiveExpiringImages {
		t.logger.Printf(
			"Replicator(%s): ignoring expiring image in archiver mode\n",
			name)
		return nil
	}
	img.FileSystem.RebuildInodePointers()
	err = t.imageDataBase.DoWithPendingImage(img, func() error {
		if err := t.getMissingObjects(img, client); err != nil {
			client.Close()
			return err
		}
		if err := t.imageDataBase.AddImage(img, name, nil); err != nil {
			return err
		}
		return nil
	})
	if err != nil {
		return err
	}
	t.logger.Printf("Replicator(%s): added image\n", name)
	return nil
}

func (t *srpcType) checkImageBeingInjected(name string) bool {
	t.imagesBeingInjectedLock.Lock()
	defer t.imagesBeingInjectedLock.Unlock()
	_, ok := t.imagesBeingInjected[name]
	return ok
}

func (t *srpcType) getMissingObjects(img *image.Image,
	client *srpc.Client) error {
	var numObjects uint64
	img.ForEachObject(func(hashVal hash.Hash) error {
		numObjects++
		return nil
	})
	missingObjects, err := img.ListMissingObjects(t.objSrv)
	if err != nil {
		return err
	}
	if len(missingObjects) < 1 {
		return nil
	}
	t.logger.Printf("Replicator: downloading %d of %d objects\n",
		len(missingObjects), numObjects)
	startTime := time.Now()
	objClient := objectclient.AttachObjectClient(client)
	defer objClient.Close()
	objectsReader, err := objClient.GetObjects(missingObjects)
	if err != nil {
		return errors.New("error downloading objects: " + err.Error())
	}
	defer objectsReader.Close()
	var totalBytes uint64
	for _, hash := range missingObjects {
		length, reader, err := objectsReader.NextObject()
		if err != nil {
			return err
		}
		_, _, err = t.objSrv.AddObject(reader, length, &hash)
		reader.Close()
		if err != nil {
			return err
		}
		totalBytes += length
	}
	timeTaken := time.Since(startTime)
	t.logger.Printf("Replicator: downloaded %d objects, %s in %s (%s/s)\n",
		len(missingObjects), format.FormatBytes(totalBytes), timeTaken,
		format.FormatBytes(uint64(float64(totalBytes)/timeTaken.Seconds())))
	return nil
}
<commit_msg>Use new lib/image.Image.GetMissingObjects() in imageserver replicator.<commit_after>package rpcd

import (
	"encoding/gob"
	"errors"
	"fmt"
	"io"
	"time"

	imgclient "github.com/Symantec/Dominator/imageserver/client"
	"github.com/Symantec/Dominator/lib/format"
	"github.com/Symantec/Dominator/lib/image"
	"github.com/Symantec/Dominator/lib/log"
	"github.com/Symantec/Dominator/lib/log/prefixlogger"
	objectclient "github.com/Symantec/Dominator/lib/objectserver/client"
	"github.com/Symantec/Dominator/lib/srpc"
	"github.com/Symantec/Dominator/proto/imageserver"
)

func (t *srpcType) replicator() {
	initialTimeout := time.Second * 15
	timeout := initialTimeout
	var nextSleepStopTime time.Time
	for {
		nextSleepStopTime = time.Now().Add(timeout)
		if client, err := srpc.DialHTTP("tcp", t.replicationMaster,
			timeout); err != nil {
			t.logger.Printf("Error dialling: %s %s\n", t.replicationMaster, err)
		} else {
			if conn, err := client.Call(
				"ImageServer.GetImageUpdates"); err != nil {
				t.logger.Println(err)
			} else {
				if err := t.getUpdates(conn); err != nil {
					if err == io.EOF {
						t.logger.Println(
							"Connection to image replicator closed")
						if nextSleepStopTime.Sub(time.Now()) < 1 {
							timeout = initialTimeout
						}
					} else {
						t.logger.Println(err)
					}
				}
				conn.Close()
			}
			client.Close()
		}
		time.Sleep(nextSleepStopTime.Sub(time.Now()))
		if timeout < time.Minute {
			timeout *= 2
		}
	}
}

func (t *srpcType) getUpdates(conn *srpc.Conn) error {
	t.logger.Printf("Image replicator: connected to: %s\n", t.replicationMaster)
	replicationStartTime := time.Now()
	decoder := gob.NewDecoder(conn)
	initialImages := make(map[string]struct{})
	if t.archiveMode {
		initialImages = nil
	}
	for {
		var imageUpdate imageserver.ImageUpdate
		if err := decoder.Decode(&imageUpdate); err != nil {
			if err == io.EOF {
				return err
			}
			return errors.New("decode err: " + err.Error())
		}
		switch imageUpdate.Operation {
		case imageserver.OperationAddImage:
			if imageUpdate.Name == "" {
				if initialImages != nil {
					t.deleteMissingImages(initialImages)
					initialImages = nil
				}
				t.logger.Printf("Replicated all current images in %s\n",
					format.Duration(time.Since(replicationStartTime)))
				continue
			}
			if initialImages != nil {
				initialImages[imageUpdate.Name] = struct{}{}
			}
			if err := t.addImage(imageUpdate.Name); err != nil {
				return errors.New("error adding image: " + imageUpdate.Name +
					": " + err.Error())
			}
		case imageserver.OperationDeleteImage:
			if t.archiveMode {
				continue
			}
			t.logger.Printf("Replicator(%s): delete image\n", imageUpdate.Name)
			if err := t.imageDataBase.DeleteImage(imageUpdate.Name,
				nil); err != nil {
				return err
			}
		case imageserver.OperationMakeDirectory:
			directory := imageUpdate.Directory
			if directory == nil {
				return errors.New("nil imageUpdate.Directory")
			}
			if err := t.imageDataBase.UpdateDirectory(*directory); err != nil {
				return err
			}
		}
	}
}

func (t *srpcType) deleteMissingImages(imagesToKeep map[string]struct{}) {
	missingImages := make([]string, 0)
	for _, imageName := range t.imageDataBase.ListImages() {
		if _, ok := imagesToKeep[imageName]; !ok {
			missingImages = append(missingImages, imageName)
		}
	}
	for _, imageName := range missingImages {
		t.logger.Printf("Replicator(%s): delete missing image\n", imageName)
		if err := t.imageDataBase.DeleteImage(imageName, nil); err != nil {
			t.logger.Println(err)
		}
	}
}

func (t *srpcType) addImage(name string) error {
	timeout := time.Second * 60
	if t.imageDataBase.CheckImage(name) || t.checkImageBeingInjected(name) {
		return nil
	}
	logger := prefixlogger.New(fmt.Sprintf("Replicator(%s): ", name), t.logger)
	logger.Println("add image")
	client, err := t.imageserverResource.GetHTTP(nil, timeout)
	if err != nil {
		return err
	}
	defer client.Put()
	img, err := imgclient.GetImage(client, name)
	if err != nil {
		client.Close()
		return err
	}
	if img == nil {
		return errors.New(name + ": not found")
	}
	logger.Println("downloaded image")
	if t.archiveMode && !img.ExpiresAt.IsZero() && !*archiveExpiringImages {
		logger.Println("ignoring expiring image in archiver mode")
		return nil
	}
	img.FileSystem.RebuildInodePointers()
	err = t.imageDataBase.DoWithPendingImage(img, func() error {
		if err := t.getMissingObjects(img, client, logger); err != nil {
			client.Close()
			return err
		}
		if err := t.imageDataBase.AddImage(img, name, nil); err != nil {
			return err
		}
		return nil
	})
	if err != nil {
		return err
	}
	logger.Println("added image")
	return nil
}

func (t *srpcType) checkImageBeingInjected(name string) bool {
	t.imagesBeingInjectedLock.Lock()
	defer t.imagesBeingInjectedLock.Unlock()
	_, ok := t.imagesBeingInjected[name]
	return ok
}

func (t *srpcType) getMissingObjects(img *image.Image, client *srpc.Client,
	logger log.DebugLogger) error {
	objClient := objectclient.AttachObjectClient(client)
	defer objClient.Close()
	return img.GetMissingObjects(t.objSrv, objClient, logger)
}
<|endoftext|>


<commit_before>package actors

import (
	"os"

	. "github.com/onsi/gomega"
	"github.com/pivotal-cf-experimental/bosh-bootloader/application"
	"github.com/pivotal-cf-experimental/bosh-bootloader/aws"
	"github.com/pivotal-cf-experimental/bosh-bootloader/aws/cloudformation"
	"github.com/pivotal-cf-experimental/bosh-bootloader/aws/iam"
	"github.com/pivotal-cf-experimental/bosh-bootloader/integration-test"
)

type AWS struct {
	stackManager         cloudformation.StackManager
	cloudFormationClient cloudformation.Client
	certificateDescriber iam.CertificateDescriber
	iamClient            iam.Client
}

func NewAWS(configuration integration.Config) AWS {
	stackManager := cloudformation.NewStackManager(application.NewLogger(os.Stdout))
	certificateDescriber := iam.NewCertificateDescriber()
	cloudFormationClient := aws.NewClientProvider().CloudFormationClient(aws.Config{
		AccessKeyID:     configuration.AWSAccessKeyID,
		SecretAccessKey: configuration.AWSSecretAccessKey,
		Region:          configuration.AWSRegion,
	})

	iamClient := aws.NewClientProvider().IAMClient(aws.Config{
		AccessKeyID:     configuration.AWSAccessKeyID,
		SecretAccessKey: configuration.AWSSecretAccessKey,
		Region:          configuration.AWSRegion,
	})

	return AWS{
		stackManager:         stackManager,
		certificateDescriber: certificateDescriber,
		cloudFormationClient: cloudFormationClient,
		iamClient:            iamClient,
	}
}

func (a AWS) StackExists(stackName string) bool {
	_, err := a.stackManager.Describe(a.cloudFormationClient, stackName)

	if err == cloudformation.StackNotFound {
		return false
	}

	Expect(err).NotTo(HaveOccurred())
	return true
}

func (a AWS) LoadBalancers(stackName string) []string {
	stack, err := a.stackManager.Describe(a.cloudFormationClient, stackName)
	Expect(err).NotTo(HaveOccurred())

	loadBalancers := []string{}

	for _, loadBalancer := range []string{"CFRouterLoadBalancer", "CFSSHProxyLoadBalancer", "ConcourseLoadBalancer"} {
		if stack.Outputs[loadBalancer] != "" {
			loadBalancers = append(loadBalancers, loadBalancer)
		}
	}

	return loadBalancers
}

func (a AWS) DescribeCertificate(certificateName string) iam.Certificate {
	certificate, err := a.certificateDescriber.Describe(certificateName, a.iamClient)
	Expect(err).NotTo(HaveOccurred())

	return certificate
}
<commit_msg>Update integration tests to pass aws clients into constructors<commit_after>package actors

import (
	"os"

	. "github.com/onsi/gomega"
	"github.com/pivotal-cf-experimental/bosh-bootloader/application"
	"github.com/pivotal-cf-experimental/bosh-bootloader/aws"
	"github.com/pivotal-cf-experimental/bosh-bootloader/aws/cloudformation"
	"github.com/pivotal-cf-experimental/bosh-bootloader/aws/iam"
	"github.com/pivotal-cf-experimental/bosh-bootloader/integration-test"
)

type AWS struct {
	stackManager         cloudformation.StackManager
	certificateDescriber iam.CertificateDescriber
}

func NewAWS(configuration integration.Config) AWS {
	cloudFormationClient := aws.NewClientProvider().CloudFormationClient(aws.Config{
		AccessKeyID:     configuration.AWSAccessKeyID,
		SecretAccessKey: configuration.AWSSecretAccessKey,
		Region:          configuration.AWSRegion,
	})

	iamClient := aws.NewClientProvider().IAMClient(aws.Config{
		AccessKeyID:     configuration.AWSAccessKeyID,
		SecretAccessKey: configuration.AWSSecretAccessKey,
		Region:          configuration.AWSRegion,
	})

	stackManager := cloudformation.NewStackManager(cloudFormationClient, application.NewLogger(os.Stdout))
	certificateDescriber := iam.NewCertificateDescriber(iamClient)

	return AWS{
		stackManager:         stackManager,
		certificateDescriber: certificateDescriber,
	}
}

func (a AWS) StackExists(stackName string) bool {
	_, err := a.stackManager.Describe(stackName)

	if err == cloudformation.StackNotFound {
		return false
	}

	Expect(err).NotTo(HaveOccurred())
	return true
}

func (a AWS) LoadBalancers(stackName string) []string {
	stack, err := a.stackManager.Describe(stackName)
	Expect(err).NotTo(HaveOccurred())

	loadBalancers := []string{}

	for _, loadBalancer := range []string{"CFRouterLoadBalancer", "CFSSHProxyLoadBalancer", "ConcourseLoadBalancer"} {
		if stack.Outputs[loadBalancer] != "" {
			loadBalancers = append(loadBalancers, loadBalancer)
		}
	}

	return loadBalancers
}

func (a AWS) DescribeCertificate(certificateName string) iam.Certificate {
	certificate, err := a.certificateDescriber.Describe(certificateName)
	Expect(err).NotTo(HaveOccurred())

	return certificate
}
<|endoftext|>


<commit_before>// Copyright (C) 2014 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at http://mozilla.org/MPL/2.0/.

package osutil_test

import (
	"os"
	"runtime"
	"testing"

	"github.com/syncthing/syncthing/internal/osutil"
)

func TestInWriteableDir(t *testing.T) {
	err := os.RemoveAll("testdata")
	if err != nil {
		t.Fatal(err)
	}
	defer os.RemoveAll("testdata")

	os.Mkdir("testdata", 0700)
	os.Mkdir("testdata/rw", 0700)
	os.Mkdir("testdata/ro", 0500)

	create := func(name string) error {
		fd, err := os.Create(name)
		if err != nil {
			return err
		}
		fd.Close()
		return nil
	}

	// These should succeed

	err = osutil.InWritableDir(create, "testdata/file")
	if err != nil {
		t.Error("testdata/file:", err)
	}
	err = osutil.InWritableDir(create, "testdata/rw/foo")
	if err != nil {
		t.Error("testdata/rw/foo:", err)
	}
	err = osutil.InWritableDir(os.Remove, "testdata/rw/foo")
	if err != nil {
		t.Error("testdata/rw/foo:", err)
	}

	err = osutil.InWritableDir(create, "testdata/ro/foo")
	if err != nil {
		t.Error("testdata/ro/foo:", err)
	}
	err = osutil.InWritableDir(os.Remove, "testdata/ro/foo")
	if err != nil {
		t.Error("testdata/ro/foo:", err)
	}

	// These should not

	err = osutil.InWritableDir(create, "testdata/nonexistent/foo")
	if err == nil {
		t.Error("testdata/nonexistent/foo returned nil error")
	}
	err = osutil.InWritableDir(create, "testdata/file/foo")
	if err == nil {
		t.Error("testdata/file/foo returned nil error")
	}
}

func TestInWritableDirWindowsRemove(t *testing.T) {
	if runtime.GOOS != "windows" {
		t.Skipf("Tests not required")
		return
	}

	err := os.RemoveAll("testdata")
	if err != nil {
		t.Fatal(err)
	}
	defer os.RemoveAll("testdata")

	create := func(name string) error {
		fd, err := os.Create(name)
		if err != nil {
			return err
		}
		fd.Close()
		return nil
	}

	os.Mkdir("testdata", 0700)

	os.Mkdir("testdata/windows", 0500)
	os.Mkdir("testdata/windows/ro", 0500)
	create("testdata/windows/ro/readonly")
	os.Chmod("testdata/windows/ro/readonly", 0500)

	for _, path := range []string{"testdata/windows/ro/readonly", "testdata/windows/ro", "testdata/windows"} {
		err := os.Remove(path)
		if err == nil {
			t.Errorf("Expected error %s", path)
		}
	}

	for _, path := range []string{"testdata/windows/ro/readonly", "testdata/windows/ro", "testdata/windows"} {
		err := osutil.InWritableDir(osutil.Remove, path)
		if err != nil {
			t.Errorf("Unexpected error %s: %s", path, err)
		}
	}
}

func TestInWritableDirWindowsRename(t *testing.T) {
	if runtime.GOOS != "windows" {
		t.Skipf("Tests not required")
		return
	}

	err := os.RemoveAll("testdata")
	if err != nil {
		t.Fatal(err)
	}
	defer os.RemoveAll("testdata")

	create := func(name string) error {
		fd, err := os.Create(name)
		if err != nil {
			return err
		}
		fd.Close()
		return nil
	}

	os.Mkdir("testdata", 0700)

	os.Mkdir("testdata/windows", 0500)
	os.Mkdir("testdata/windows/ro", 0500)
	create("testdata/windows/ro/readonly")
	os.Chmod("testdata/windows/ro/readonly", 0500)

	for _, path := range []string{"testdata/windows/ro/readonly", "testdata/windows/ro", "testdata/windows"} {
		err := os.Rename(path, path+"new")
		if err == nil {
			t.Errorf("Expected error %s", path)
		}
	}

	rename := func(path string) error {
		return osutil.Rename(path, path+"new")
	}

	for _, path := range []string{"testdata/windows/ro/readonly", "testdata/windows/ro", "testdata/windows"} {
		err := osutil.InWritableDir(rename, path)
		if err != nil {
			t.Errorf("Unexpected error %s: %s", path, err)
		}
		_, err = os.Stat(path + "new")
		if err != nil {
			t.Errorf("Unexpected error %s: %s", path, err)
		}
	}
}
<commit_msg>If rename works we are happy (fixes #1767)<commit_after>// Copyright (C) 2014 The Syncthing Authors.
//
// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at http://mozilla.org/MPL/2.0/.

package osutil_test

import (
	"os"
	"runtime"
	"testing"

	"github.com/syncthing/syncthing/internal/osutil"
)

func TestInWriteableDir(t *testing.T) {
	err := os.RemoveAll("testdata")
	if err != nil {
		t.Fatal(err)
	}
	defer os.RemoveAll("testdata")

	os.Mkdir("testdata", 0700)
	os.Mkdir("testdata/rw", 0700)
	os.Mkdir("testdata/ro", 0500)

	create := func(name string) error {
		fd, err := os.Create(name)
		if err != nil {
			return err
		}
		fd.Close()
		return nil
	}

	// These should succeed

	err = osutil.InWritableDir(create, "testdata/file")
	if err != nil {
		t.Error("testdata/file:", err)
	}
	err = osutil.InWritableDir(create, "testdata/rw/foo")
	if err != nil {
		t.Error("testdata/rw/foo:", err)
	}
	err = osutil.InWritableDir(os.Remove, "testdata/rw/foo")
	if err != nil {
		t.Error("testdata/rw/foo:", err)
	}

	err = osutil.InWritableDir(create, "testdata/ro/foo")
	if err != nil {
		t.Error("testdata/ro/foo:", err)
	}
	err = osutil.InWritableDir(os.Remove, "testdata/ro/foo")
	if err != nil {
		t.Error("testdata/ro/foo:", err)
	}

	// These should not

	err = osutil.InWritableDir(create, "testdata/nonexistent/foo")
	if err == nil {
		t.Error("testdata/nonexistent/foo returned nil error")
	}
	err = osutil.InWritableDir(create, "testdata/file/foo")
	if err == nil {
		t.Error("testdata/file/foo returned nil error")
	}
}

func TestInWritableDirWindowsRemove(t *testing.T) {
	if runtime.GOOS != "windows" {
		t.Skipf("Tests not required")
		return
	}

	err := os.RemoveAll("testdata")
	if err != nil {
		t.Fatal(err)
	}
	defer os.RemoveAll("testdata")

	create := func(name string) error {
		fd, err := os.Create(name)
		if err != nil {
			return err
		}
		fd.Close()
		return nil
	}

	os.Mkdir("testdata", 0700)

	os.Mkdir("testdata/windows", 0500)
	os.Mkdir("testdata/windows/ro", 0500)
	create("testdata/windows/ro/readonly")
	os.Chmod("testdata/windows/ro/readonly", 0500)

	for _, path := range []string{"testdata/windows/ro/readonly", "testdata/windows/ro", "testdata/windows"} {
		err := os.Remove(path)
		if err == nil {
			t.Errorf("Expected error %s", path)
		}
	}

	for _, path := range []string{"testdata/windows/ro/readonly", "testdata/windows/ro", "testdata/windows"} {
		err := osutil.InWritableDir(osutil.Remove, path)
		if err != nil {
			t.Errorf("Unexpected error %s: %s", path, err)
		}
	}
}

func TestInWritableDirWindowsRename(t *testing.T) {
	if runtime.GOOS != "windows" {
		t.Skipf("Tests not required")
		return
	}

	err := os.RemoveAll("testdata")
	if err != nil {
		t.Fatal(err)
	}
	defer os.RemoveAll("testdata")

	create := func(name string) error {
		fd, err := os.Create(name)
		if err != nil {
			return err
		}
		fd.Close()
		return nil
	}

	os.Mkdir("testdata", 0700)

	os.Mkdir("testdata/windows", 0500)
	os.Mkdir("testdata/windows/ro", 0500)
	create("testdata/windows/ro/readonly")
	os.Chmod("testdata/windows/ro/readonly", 0500)

	for _, path := range []string{"testdata/windows/ro/readonly", "testdata/windows/ro", "testdata/windows"} {
		err := os.Rename(path, path+"new")
		if err == nil {
			t.Skipf("seem like this test doesn't work here")
			return
		}
	}

	rename := func(path string) error {
		return osutil.Rename(path, path+"new")
	}

	for _, path := range []string{"testdata/windows/ro/readonly", "testdata/windows/ro", "testdata/windows"} {
		err := osutil.InWritableDir(rename, path)
		if err != nil {
			t.Errorf("Unexpected error %s: %s", path, err)
		}
		_, err = os.Stat(path + "new")
		if err != nil {
			t.Errorf("Unexpected error %s: %s", path, err)
		}
	}
}
<|endoftext|>


<commit_before>// Copyright 2016 tsuru authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package dockercommon

import (
	"fmt"
	"strings"

	"github.com/tsuru/config"
	"github.com/tsuru/tsuru/app/image"
	"github.com/tsuru/tsuru/provision"
)

// provisioner deploys a unit using the archive method.
func ArchiveDeployCmds(app provision.App, archiveURL string) []string {
	return DeployCmds(app, "archive", archiveURL)
}

func DeployCmds(app provision.App, params ...string) []string {
	deployCmd, err := config.GetString("docker:deploy-cmd")
	if err != nil {
		deployCmd = "/var/lib/tsuru/deploy"
	}
	cmds := append([]string{deployCmd}, params...)
	host, _ := config.GetString("host")
	token := app.Envs()["TSURU_APP_TOKEN"].Value
	unitAgentCmds := []string{"tsuru_unit_agent", host, token, app.GetName(), `"` + strings.Join(cmds, " ") + `"`, "deploy"}
	finalCmd := strings.Join(unitAgentCmds, " ")
	return []string{"/bin/sh", "-lc", finalCmd}
}

// runWithAgentCmds returns the list of commands that should be passed when the
// provisioner will run a unit using tsuru_unit_agent to start.
//
// This will only be called for legacy containers that have not been re-
// deployed since the introduction of independent units per 'process' in
// 0.12.0.
func runWithAgentCmds(app provision.App) ([]string, error) {
	runCmd, err := config.GetString("docker:run-cmd:bin")
	if err != nil {
		return nil, err
	}
	host, _ := config.GetString("host")
	token := app.Envs()["TSURU_APP_TOKEN"].Value
	return []string{"tsuru_unit_agent", host, token, app.GetName(), runCmd}, nil
}

func ProcessCmdForImage(processName, imageId string) (string, string, error) {
	data, err := image.GetImageCustomData(imageId)
	if err != nil {
		return "", "", err
	}
	if processName == "" {
		if len(data.Processes) == 0 {
			return "", "", nil
		}
		if len(data.Processes) > 1 {
			return "", "", provision.InvalidProcessError{Msg: "no process name specified and more than one declared in Procfile"}
		}
		for name := range data.Processes {
			processName = name
		}
	}
	processCmd := data.Processes[processName]
	if processCmd == "" {
		return "", "", provision.InvalidProcessError{Msg: fmt.Sprintf("no command declared in Procfile for process %q", processName)}
	}
	return processCmd, processName, nil
}

func LeanContainerCmds(processName, imageId string, app provision.App) ([]string, string, error) {
	processCmd, processName, err := ProcessCmdForImage(processName, imageId)
	if err != nil {
		return nil, "", err
	}
	if processCmd == "" {
		// Legacy support, no processes are yet registered for this app's
		// containers.
		var cmds []string
		cmds, err = runWithAgentCmds(app)
		return cmds, "", err
	}
	yamlData, err := image.GetImageTsuruYamlData(imageId)
	if err != nil {
		return nil, "", err
	}
	before := strings.Join(yamlData.Hooks.Restart.Before, " && ")
	if before != "" {
		before += " && "
	}
	if processName == "" {
		processName = "web"
	}
	return []string{
		"/bin/sh",
		"-lc",
		"[ -d /home/application/current ] && cd /home/application/current; " + before + "exec " + processCmd,
	}, processName, nil
}
<commit_msg>provision/dockercommon: allow extra cmds for container startup<commit_after>// Copyright 2016 tsuru authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package dockercommon

import (
	"fmt"
	"strings"

	"github.com/tsuru/config"
	"github.com/tsuru/tsuru/app/image"
	"github.com/tsuru/tsuru/provision"
)

// provisioner deploys a unit using the archive method.
func ArchiveDeployCmds(app provision.App, archiveURL string) []string {
	return DeployCmds(app, "archive", archiveURL)
}

func DeployCmds(app provision.App, params ...string) []string {
	deployCmd, err := config.GetString("docker:deploy-cmd")
	if err != nil {
		deployCmd = "/var/lib/tsuru/deploy"
	}
	cmds := append([]string{deployCmd}, params...)
	host, _ := config.GetString("host")
	token := app.Envs()["TSURU_APP_TOKEN"].Value
	unitAgentCmds := []string{"tsuru_unit_agent", host, token, app.GetName(), `"` + strings.Join(cmds, " ") + `"`, "deploy"}
	finalCmd := strings.Join(unitAgentCmds, " ")
	return []string{"/bin/sh", "-lc", finalCmd}
}

// runWithAgentCmds returns the list of commands that should be passed when the
// provisioner will run a unit using tsuru_unit_agent to start.
//
// This will only be called for legacy containers that have not been re-
// deployed since the introduction of independent units per 'process' in
// 0.12.0.
func runWithAgentCmds(app provision.App) ([]string, error) {
	runCmd, err := config.GetString("docker:run-cmd:bin")
	if err != nil {
		return nil, err
	}
	host, _ := config.GetString("host")
	token := app.Envs()["TSURU_APP_TOKEN"].Value
	return []string{"tsuru_unit_agent", host, token, app.GetName(), runCmd}, nil
}

func ProcessCmdForImage(processName, imageId string) (string, string, error) {
	data, err := image.GetImageCustomData(imageId)
	if err != nil {
		return "", "", err
	}
	if processName == "" {
		if len(data.Processes) == 0 {
			return "", "", nil
		}
		if len(data.Processes) > 1 {
			return "", "", provision.InvalidProcessError{Msg: "no process name specified and more than one declared in Procfile"}
		}
		for name := range data.Processes {
			processName = name
		}
	}
	processCmd := data.Processes[processName]
	if processCmd == "" {
		return "", "", provision.InvalidProcessError{Msg: fmt.Sprintf("no command declared in Procfile for process %q", processName)}
	}
	return processCmd, processName, nil
}

func LeanContainerCmds(processName, imageId string, app provision.App) ([]string, string, error) {
	return LeanContainerCmdsWithExtra(processName, imageId, app, nil)
}

func LeanContainerCmdsWithExtra(processName, imageId string, app provision.App, extraCmds []string) ([]string, string, error) {
	processCmd, processName, err := ProcessCmdForImage(processName, imageId)
	if err != nil {
		return nil, "", err
	}
	if processCmd == "" {
		// Legacy support, no processes are yet registered for this app's
		// containers.
		var cmds []string
		cmds, err = runWithAgentCmds(app)
		return cmds, "", err
	}
	yamlData, err := image.GetImageTsuruYamlData(imageId)
	if err != nil {
		return nil, "", err
	}
	extraCmds = append(extraCmds, yamlData.Hooks.Restart.Before...)
	before := strings.Join(extraCmds, " && ")
	if before != "" {
		before += " && "
	}
	if processName == "" {
		processName = "web"
	}
	return []string{
		"/bin/sh",
		"-lc",
		"[ -d /home/application/current ] && cd /home/application/current; " + before + "exec " + processCmd,
	}, processName, nil
}
<|endoftext|>


<commit_before>package raw_socket

import (
	"log"
	"sort"
	"time"
)

const MSG_EXPIRE = 2000 * time.Millisecond

// TCPMessage ensure that all TCP packets for given request is received, and processed in right sequence
// Its needed because all TCP message can be fragmented or re-transmitted
//
// Each TCP Packet have 2 ids: acknowledgment - message_id, and sequence - packet_id
// Message can be compiled from unique packets with same message_id which sorted by sequence
// Message is received if we didn't receive any packets for 2000ms
type TCPMessage struct {
	ID      string // Message ID
	packets []*TCPPacket

	timer *time.Timer // Used for expire check

	c_packets chan *TCPPacket

	c_del_message chan *TCPMessage
}

// NewTCPMessage pointer created from a Acknowledgment number and a channel of messages readuy to be deleted
func NewTCPMessage(ID string, c_del chan *TCPMessage) (msg *TCPMessage) {
	msg = &TCPMessage{ID: ID}

	msg.c_packets = make(chan *TCPPacket)
	msg.c_del_message = c_del // used for notifying that message completed or expired

	// Every time we receive packet we reset this timer
	msg.timer = time.AfterFunc(MSG_EXPIRE, msg.Timeout)

	go msg.listen()

	return
}

func (t *TCPMessage) listen() {
	for {
		select {
		case packet, more := <-t.c_packets:
			if more {
				t.AddPacket(packet)
			} else {
				// Stop loop if channel closed
				t.c_del_message <- t
				return
			}
		}
	}
}

// Timeout notifies message to stop listening, close channel and message ready to be sent
func (t *TCPMessage) Timeout() {
	close(t.c_packets)   // Notify to stop listen loop and close channel
	t.c_del_message <- t // Notify RAWListener that message is ready to be send to replay server
}

// Bytes sorts packets in right orders and return message content
func (t *TCPMessage) Bytes() (output []byte) {
	sort.Sort(BySeq(t.packets))

	for _, v := range t.packets {
		output = append(output, v.Data...)
	}

	return
}

// AddPacket to the message and ensure packet uniqueness
// TCP allows that packet can be re-send multiple times
func (t *TCPMessage) AddPacket(packet *TCPPacket) {
	packetFound := false

	for _, pkt := range t.packets {
		if packet.Seq == pkt.Seq {
			packetFound = true
			break
		}
	}

	if packetFound {
		log.Println("Received packet with same sequence")
	} else {
		t.packets = append(t.packets, packet)
	}

	// Reset message timeout timer
	t.timer.Reset(MSG_EXPIRE)
}
<commit_msg>revert accidental channel call<commit_after>package raw_socket

import (
	"log"
	"sort"
	"time"
)

const MSG_EXPIRE = 2000 * time.Millisecond

// TCPMessage ensure that all TCP packets for given request is received, and processed in right sequence
// Its needed because all TCP message can be fragmented or re-transmitted
//
// Each TCP Packet have 2 ids: acknowledgment - message_id, and sequence - packet_id
// Message can be compiled from unique packets with same message_id which sorted by sequence
// Message is received if we didn't receive any packets for 2000ms
type TCPMessage struct {
	ID      string // Message ID
	packets []*TCPPacket

	timer *time.Timer // Used for expire check

	c_packets chan *TCPPacket

	c_del_message chan *TCPMessage
}

// NewTCPMessage pointer created from a Acknowledgment number and a channel of messages readuy to be deleted
func NewTCPMessage(ID string, c_del chan *TCPMessage) (msg *TCPMessage) {
	msg = &TCPMessage{ID: ID}

	msg.c_packets = make(chan *TCPPacket)
	msg.c_del_message = c_del // used for notifying that message completed or expired

	// Every time we receive packet we reset this timer
	msg.timer = time.AfterFunc(MSG_EXPIRE, msg.Timeout)

	go msg.listen()

	return
}

func (t *TCPMessage) listen() {
	for {
		select {
		case packet, more := <-t.c_packets:
			if more {
				t.AddPacket(packet)
			} else {
				// Stop loop if channel closed
				return
			}
		}
	}
}

// Timeout notifies message to stop listening, close channel and message ready to be sent
func (t *TCPMessage) Timeout() {
	close(t.c_packets)   // Notify to stop listen loop and close channel
	t.c_del_message <- t // Notify RAWListener that message is ready to be send to replay server
}

// Bytes sorts packets in right orders and return message content
func (t *TCPMessage) Bytes() (output []byte) {
	sort.Sort(BySeq(t.packets))

	for _, v := range t.packets {
		output = append(output, v.Data...)
	}

	return
}

// AddPacket to the message and ensure packet uniqueness
// TCP allows that packet can be re-send multiple times
func (t *TCPMessage) AddPacket(packet *TCPPacket) {
	packetFound := false

	for _, pkt := range t.packets {
		if packet.Seq == pkt.Seq {
			packetFound = true
			break
		}
	}

	if packetFound {
		log.Println("Received packet with same sequence")
	} else {
		t.packets = append(t.packets, packet)
	}

	// Reset message timeout timer
	t.timer.Reset(MSG_EXPIRE)
}
<|endoftext|>


<commit_before>package zmq

import (
	"encoding/json"
	"fmt"

	log "github.com/Sirupsen/logrus"
	odplugin "github.com/oursky/skygear/plugin"
	"github.com/oursky/skygear/plugin/common"
	"github.com/oursky/skygear/skydb"
	"github.com/zeromq/goczmq"
)

type zmqTransport struct {
	name   string
	iaddr  string // the internal addr used by goroutines to make request to plugin
	eaddr  string // the addr exposed for plugin to connect to with REP.
	broker *Broker
}

type request struct {
	Kind  string
	Name  string
	Param interface{}
}

type hookRequest struct {
	Record   interface{} `json:"record"`
	Original interface{} `json:"original"`
}

// type-safe constructors for request.Param assignment

func newLambdaRequest(name string, args json.RawMessage) *request {
	return &request{Kind: "op", Name: name, Param: args}
}

func newHandlerRequest(name string, input json.RawMessage) *request {
	return &request{Kind: "handler", Name: name, Param: input}
}

func newHookRequest(trigger string, record *skydb.Record, originalRecord *skydb.Record) *request {
	param := hookRequest{
		Record:   (*common.JSONRecord)(record),
		Original: (*common.JSONRecord)(originalRecord),
	}
	return &request{Kind: "hook", Name: trigger, Param: param}
}

func newAuthRequest(authReq *odplugin.AuthRequest) *request {
	return &request{
		Kind: "provider",
		Name: authReq.ProviderName,
		Param: struct {
			Action   string                 `json:"action"`
			AuthData map[string]interface{} `json:"auth_data"`
		}{authReq.Action, authReq.AuthData},
	}
}

// TODO(limouren): reduce copying of this method
func (req *request) MarshalJSON() ([]byte, error) {
	if rawParam, ok := req.Param.(json.RawMessage); ok {
		rawParamReq := struct {
			Kind  string          `json:"kind"`
			Name  string          `json:"name,omitempty"`
			Param json.RawMessage `json:"param,omitempty"`
		}{req.Kind, req.Name, rawParam}
		return json.Marshal(&rawParamReq)
	}

	paramReq := struct {
		Kind  string      `json:"kind"`
		Name  string      `json:"name,omitempty"`
		Param interface{} `json:"param,omitempty"`
	}{req.Kind, req.Name, req.Param}

	return json.Marshal(&paramReq)
}

func (p zmqTransport) RunInit() (out []byte, err error) {
	req := request{Kind: "init"}
	out, err = p.ipc(&req)
	return
}

func (p zmqTransport) RunLambda(name string, in []byte) (out []byte, err error) {
	out, err = p.rpc(newLambdaRequest(name, in))
	return
}

func (p zmqTransport) RunHandler(name string, in []byte) (out []byte, err error) {
	out, err = p.rpc(newHandlerRequest(name, in))
	return
}

func (p zmqTransport) RunHook(recordType string, trigger string, record *skydb.Record, originalRecord *skydb.Record) (*skydb.Record, error) {
	out, err := p.rpc(newHookRequest(trigger, record, originalRecord))
	if err != nil {
		return nil, err
	}

	var recordout skydb.Record
	if err := json.Unmarshal(out, (*common.JSONRecord)(&recordout)); err != nil {
		log.WithField("data", string(out)).Error("failed to unmarshal record")
		return nil, fmt.Errorf("failed to unmarshal record: %v", err)
	}
	recordout.OwnerID = record.OwnerID
	recordout.CreatedAt = record.CreatedAt
	recordout.CreatorID = record.CreatorID
	recordout.UpdatedAt = record.UpdatedAt
	recordout.UpdaterID = record.UpdaterID

	return &recordout, nil
}

func (p zmqTransport) RunTimer(name string, in []byte) (out []byte, err error) {
	req := request{Kind: "timer", Name: name}
	out, err = p.rpc(&req)
	return
}

func (p zmqTransport) RunProvider(request *odplugin.AuthRequest) (resp *odplugin.AuthResponse, err error) {
	req := newAuthRequest(request)
	out, err := p.rpc(req)
	if err != nil {
		return
	}

	err = json.Unmarshal(out, &resp)
	return
}

func (p *zmqTransport) rpc(req *request) (out []byte, err error) {
	var rawResp []byte

	rawResp, err = p.ipc(req)
	if err != nil {
		return
	}

	var resp struct {
		Result json.RawMessage   `json:"result"`
		Err    *common.ExecError `json:"error"`
	}

	if err = json.Unmarshal(rawResp, &resp); err != nil {
		return
	}
	if resp.Err != nil {
		err = resp.Err
		return
	}

	out = resp.Result
	return
}

func (p *zmqTransport) ipc(req *request) (out []byte, err error) {
	var (
		in      []byte
		reqSock *goczmq.Sock
	)

	in, err = json.Marshal(req)
	if err != nil {
		return
	}

	reqSock, err = goczmq.NewReq(p.iaddr)
	if err != nil {
		return
	}
	err = reqSock.SendMessage([][]byte{in})
	if err != nil {
		return
	}

	msg, err := reqSock.RecvMessage()
	if err != nil {
		return
	}

	if len(msg) != 1 {
		err = fmt.Errorf("malformed resp msg = %s", msg)
	} else {
		out = msg[0]
	}

	return
}

type zmqTransportFactory struct {
}

func (f zmqTransportFactory) Open(name string, args []string) (transport odplugin.Transport) {
	const internalAddrFmt = `inproc://%s`

	internalAddr := fmt.Sprintf(internalAddrFmt, name)
	externalAddr := args[0]

	broker, err := NewBroker(internalAddr, externalAddr)
	if err != nil {
		log.Panicf("Failed to init broker for zmq transport: %v", err)
	}

	p := zmqTransport{
		name:   name,
		iaddr:  internalAddr,
		eaddr:  externalAddr,
		broker: broker,
	}

	go func() {
		log.Infof("Running zmq broker:\niaddr = %s\neaddr = %s", internalAddr, externalAddr)
		broker.Run()
	}()

	return p
}

func init() {
	odplugin.RegisterTransport("zmq", zmqTransportFactory{})
}
<commit_msg>Retry plugin init request rather than waiting indefinitely<commit_after>package zmq

import (
	"encoding/json"
	"fmt"

	log "github.com/Sirupsen/logrus"
	odplugin "github.com/oursky/skygear/plugin"
	"github.com/oursky/skygear/plugin/common"
	"github.com/oursky/skygear/skydb"
	"github.com/zeromq/goczmq"
)

const initRequestTimeout = 2000

type zmqTransport struct {
	name   string
	iaddr  string // the internal addr used by goroutines to make request to plugin
	eaddr  string // the addr exposed for plugin to connect to with REP.
	broker *Broker
}

type request struct {
	Kind    string
	Name    string
	Param   interface{}
	Timeout int // timeout in millisecond
}

type hookRequest struct {
	Record   interface{} `json:"record"`
	Original interface{} `json:"original"`
}

// type-safe constructors for request.Param assignment

func newLambdaRequest(name string, args json.RawMessage) *request {
	return &request{Kind: "op", Name: name, Param: args}
}

func newHandlerRequest(name string, input json.RawMessage) *request {
	return &request{Kind: "handler", Name: name, Param: input}
}

func newHookRequest(trigger string, record *skydb.Record, originalRecord *skydb.Record) *request {
	param := hookRequest{
		Record:   (*common.JSONRecord)(record),
		Original: (*common.JSONRecord)(originalRecord),
	}
	return &request{Kind: "hook", Name: trigger, Param: param}
}

func newAuthRequest(authReq *odplugin.AuthRequest) *request {
	return &request{
		Kind: "provider",
		Name: authReq.ProviderName,
		Param: struct {
			Action   string                 `json:"action"`
			AuthData map[string]interface{} `json:"auth_data"`
		}{authReq.Action, authReq.AuthData},
	}
}

// TODO(limouren): reduce copying of this method
func (req *request) MarshalJSON() ([]byte, error) {
	if rawParam, ok := req.Param.(json.RawMessage); ok {
		rawParamReq := struct {
			Kind  string          `json:"kind"`
			Name  string          `json:"name,omitempty"`
			Param json.RawMessage `json:"param,omitempty"`
		}{req.Kind, req.Name, rawParam}
		return json.Marshal(&rawParamReq)
	}

	paramReq := struct {
		Kind  string      `json:"kind"`
		Name  string      `json:"name,omitempty"`
		Param interface{} `json:"param,omitempty"`
	}{req.Kind, req.Name, req.Param}

	return json.Marshal(&paramReq)
}

func (p zmqTransport) RunInit() (out []byte, err error) {
	req := request{Kind: "init", Timeout: initRequestTimeout}
	for {
		out, err = p.ipc(&req)
		if err == nil {
			break
		}
		log.Warn("Unable to send init request to plugin. Retrying...")
	}
	return
}

func (p zmqTransport) RunLambda(name string, in []byte) (out []byte, err error) {
	out, err = p.rpc(newLambdaRequest(name, in))
	return
}

func (p zmqTransport) RunHandler(name string, in []byte) (out []byte, err error) {
	out, err = p.rpc(newHandlerRequest(name, in))
	return
}

func (p zmqTransport) RunHook(recordType string, trigger string, record *skydb.Record, originalRecord *skydb.Record) (*skydb.Record, error) {
	out, err := p.rpc(newHookRequest(trigger, record, originalRecord))
	if err != nil {
		return nil, err
	}

	var recordout skydb.Record
	if err := json.Unmarshal(out, (*common.JSONRecord)(&recordout)); err != nil {
		log.WithField("data", string(out)).Error("failed to unmarshal record")
		return nil, fmt.Errorf("failed to unmarshal record: %v", err)
	}
	recordout.OwnerID = record.OwnerID
	recordout.CreatedAt = record.CreatedAt
	recordout.CreatorID = record.CreatorID
	recordout.UpdatedAt = record.UpdatedAt
	recordout.UpdaterID = record.UpdaterID

	return &recordout, nil
}

func (p zmqTransport) RunTimer(name string, in []byte) (out []byte, err error) {
	req := request{Kind: "timer", Name: name}
	out, err = p.rpc(&req)
	return
}

func (p zmqTransport) RunProvider(request *odplugin.AuthRequest) (resp *odplugin.AuthResponse, err error) {
	req := newAuthRequest(request)
	out, err := p.rpc(req)
	if err != nil {
		return
	}

	err = json.Unmarshal(out, &resp)
	return
}

func (p *zmqTransport) rpc(req *request) (out []byte, err error) {
	var rawResp []byte

	rawResp, err = p.ipc(req)
	if err != nil {
		return
	}

	var resp struct {
		Result json.RawMessage   `json:"result"`
		Err    *common.ExecError `json:"error"`
	}

	if err = json.Unmarshal(rawResp, &resp); err != nil {
		return
	}
	if resp.Err != nil {
		err = resp.Err
		return
	}

	out = resp.Result
	return
}

func (p *zmqTransport) ipc(req *request) (out []byte, err error) {
	var (
		in      []byte
		reqSock *goczmq.Sock
	)

	in, err = json.Marshal(req)
	if err != nil {
		return
	}

	reqSock, err = goczmq.NewReq(p.iaddr)
	if err != nil {
		return
	}
	if req.Timeout > 0 {
		reqSock.SetRcvtimeo(req.Timeout)
	}
	err = reqSock.SendMessage([][]byte{in})
	if err != nil {
		return
	}

	msg, err := reqSock.RecvMessage()
	if err != nil {
		return
	}

	if len(msg) != 1 {
		err = fmt.Errorf("malformed resp msg = %s", msg)
	} else {
		out = msg[0]
	}

	return
}

type zmqTransportFactory struct {
}

func (f zmqTransportFactory) Open(name string, args []string) (transport odplugin.Transport) {
	const internalAddrFmt = `inproc://%s`

	internalAddr := fmt.Sprintf(internalAddrFmt, name)
	externalAddr := args[0]

	broker, err := NewBroker(internalAddr, externalAddr)
	if err != nil {
		log.Panicf("Failed to init broker for zmq transport: %v", err)
	}

	p := zmqTransport{
		name:   name,
		iaddr:  internalAddr,
		eaddr:  externalAddr,
		broker: broker,
	}

	go func() {
		log.Infof("Running zmq broker:\niaddr = %s\neaddr = %s", internalAddr, externalAddr)
		broker.Run()
	}()

	return p
}

func init() {
	odplugin.RegisterTransport("zmq", zmqTransportFactory{})
}
<|endoftext|>


<commit_before>package router

import (
	mbus "github.com/cloudfoundry/go_cfmessagebus"
	"github.com/cloudfoundry/gorouter/test"
	. "launchpad.net/gocheck"
	"time"
)

type IntegrationSuite struct {
	Config     *Config
	mbusClient mbus.MessageBus
	router     *Router
}

var _ = Suite(&IntegrationSuite{})

func (s *IntegrationSuite) TestNatsConnectivity(c *C) {
	natsPort := nextAvailPort()
	cmd := StartNats(int(natsPort))

	proxyPort := nextAvailPort()
	statusPort := nextAvailPort()

	s.Config = SpecConfig(natsPort, statusPort, proxyPort)
	s.Config.PruneStaleDropletsInterval = 1 * time.Second

	s.router = NewRouter(s.Config)
	go s.router.Run()

	natsConnected := make(chan bool, 1)
	go func() {
		for {
			if s.router.mbusClient.Publish("Ping", []byte("data")) == nil {
				break
			}
			time.Sleep(500 * time.Millisecond)
		}
		natsConnected <- true
	}()

	<-natsConnected
	s.mbusClient = s.router.mbusClient

	heartbeatInterval := 200 * time.Millisecond
	staleThreshold := 1 * time.Second
	staleCheckInterval := s.router.registry.pruneStaleDropletsInterval

	s.router.registry.dropletStaleThreshold = staleThreshold

	zombieApp := test.NewGreetApp([]string{"test.nats.dying.and.app.dies.while.nats.is.down.vcap.me"}, proxyPort, s.mbusClient, nil)
	zombieApp.Listen()

	runningApp := test.NewGreetApp([]string{"test.nats.dying.and.app.stays.alive.vcap.me"}, proxyPort, s.mbusClient, nil)
	runningApp.Listen()

	c.Assert(s.waitAppRegistered(zombieApp, time.Second*2), Equals, true)
	c.Assert(s.waitAppRegistered(runningApp, time.Second*2), Equals, true)

	zombieTicker := time.NewTicker(heartbeatInterval)
	runningTicker := time.NewTicker(heartbeatInterval)

	go func() {
		for {
			select {
			case <-zombieTicker.C:
				zombieApp.Register()
			case <-runningTicker.C:
				runningApp.Register()
			}
		}
	}()

	zombieApp.VerifyAppStatus(200, c)

	// kill registration ticker => kill app (must be before stopping NATS since app.Register is fake and queues messages in memory)
	zombieTicker.Stop()

	StopNats(cmd)

	time.Sleep(staleCheckInterval + staleThreshold + 250*time.Millisecond)

	// While NATS is down no routes should go down
	zombieApp.VerifyAppStatus(200, c)
	runningApp.VerifyAppStatus(200, c)

	cmd = StartNats(int(natsPort))

	// Right after NATS starts up all routes should stay up
	zombieApp.VerifyAppStatus(200, c)
	runningApp.VerifyAppStatus(200, c)

	// Wait the regular stale timeout
	time.Sleep(staleCheckInterval + staleThreshold + 250*time.Millisecond)

	// Finally the zombie is cleaned up. Maybe proactively enqueue Unregister events in DEA's.
	zombieApp.VerifyAppStatus(404, c)
	runningApp.VerifyAppStatus(200, c)
}

func (s *IntegrationSuite) waitMsgReceived(a *test.TestApp, r bool, t time.Duration) bool {
	i := time.Millisecond * 50
	m := int(t / i)

	for j := 0; j < m; j++ {
		received := true
		for _, v := range a.Urls() {
			_, ok := s.router.registry.Lookup(v)
			if ok != r {
				received = false
				break
			}
		}
		if received {
			return true
		}
		time.Sleep(i)
	}

	return false
}

func (s *IntegrationSuite) waitAppRegistered(app *test.TestApp, timeout time.Duration) bool {
	return s.waitMsgReceived(app, true, timeout)
}
<commit_msg>fix some flakiness in integration spec<commit_after>package router

import (
	mbus "github.com/cloudfoundry/go_cfmessagebus"
	"github.com/cloudfoundry/gorouter/test"
	. "launchpad.net/gocheck"
	"time"
)

type IntegrationSuite struct {
	Config     *Config
	mbusClient mbus.MessageBus
	router     *Router
}

var _ = Suite(&IntegrationSuite{})

func (s *IntegrationSuite) TestNatsConnectivity(c *C) {
	natsPort := nextAvailPort()
	cmd := StartNats(int(natsPort))

	proxyPort := nextAvailPort()
	statusPort := nextAvailPort()

	s.Config = SpecConfig(natsPort, statusPort, proxyPort)
	s.Config.PruneStaleDropletsInterval = 1 * time.Second

	s.router = NewRouter(s.Config)
	go s.router.Run()

	natsConnected := make(chan bool, 1)
	go func() {
		for {
			if s.router.mbusClient.Publish("Ping", []byte("data")) == nil {
				break
			}
			time.Sleep(500 * time.Millisecond)
		}
		natsConnected <- true
	}()

	<-natsConnected
	s.mbusClient = s.router.mbusClient

	heartbeatInterval := 200 * time.Millisecond

	// ensure the threshold is longer than the interval that we check;
	// because we set the route's timestamp to time.Now() on the interval
	// as part of pausing
	staleCheckInterval := s.router.registry.pruneStaleDropletsInterval
	staleThreshold := 2 * staleCheckInterval

	s.router.registry.dropletStaleThreshold = staleThreshold

	zombieApp := test.NewGreetApp([]string{"test.nats.dying.and.app.dies.while.nats.is.down.vcap.me"}, proxyPort, s.mbusClient, nil)
	zombieApp.Listen()

	runningApp := test.NewGreetApp([]string{"test.nats.dying.and.app.stays.alive.vcap.me"}, proxyPort, s.mbusClient, nil)
	runningApp.Listen()

	c.Assert(s.waitAppRegistered(zombieApp, 2*time.Second), Equals, true)
	c.Assert(s.waitAppRegistered(runningApp, 2*time.Second), Equals, true)

	zombieTicker := time.NewTicker(heartbeatInterval)
	runningTicker := time.NewTicker(heartbeatInterval)

	go func() {
		for {
			select {
			case <-zombieTicker.C:
				zombieApp.Register()
			case <-runningTicker.C:
				runningApp.Register()
			}
		}
	}()

	zombieApp.VerifyAppStatus(200, c)

	// kill registration ticker => kill app (must be before stopping NATS since app.Register is fake and queues messages in memory)
	zombieTicker.Stop()

	StopNats(cmd)

	time.Sleep(staleCheckInterval + staleThreshold + 250*time.Millisecond)

	// While NATS is down no routes should go down
	zombieApp.VerifyAppStatus(200, c)
	runningApp.VerifyAppStatus(200, c)

	cmd = StartNats(int(natsPort))

	// Right after NATS starts up all routes should stay up
	zombieApp.VerifyAppStatus(200, c)
	runningApp.VerifyAppStatus(200, c)

	zombieGone := make(chan bool)

	go func() {
		for {
			// Finally the zombie is cleaned up. Maybe proactively enqueue Unregister events in DEA's.
			err := zombieApp.CheckAppStatus(404)
			if err != nil {
				time.Sleep(100 * time.Millisecond)
				continue
			}

			err = runningApp.CheckAppStatus(200)
			if err != nil {
				time.Sleep(100 * time.Millisecond)
				continue
			}

			zombieGone <- true

			break
		}
	}()

	select {
	case <-zombieGone:
	case <-time.After(staleCheckInterval + staleThreshold + 5*time.Second):
		c.Error("Zombie app was not pruned.")
	}
}

func (s *IntegrationSuite) waitMsgReceived(a *test.TestApp, shouldBeRegistered bool, t time.Duration) bool {
	registeredOrUnregistered := make(chan bool)

	go func() {
		for {
			received := true
			for _, v := range a.Urls() {
				_, ok := s.router.registry.Lookup(v)
				if ok != shouldBeRegistered {
					received = false
					break
				}
			}

			if received {
				registeredOrUnregistered <- true
				break
			}

			time.Sleep(50 * time.Millisecond)
		}
	}()

	select {
	case <-registeredOrUnregistered:
		return true
	case <-time.After(t):
		return false
	}
}

func (s *IntegrationSuite) waitAppRegistered(app *test.TestApp, timeout time.Duration) bool {
	return s.waitMsgReceived(app, true, timeout)
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"github.com/HewlettPackard/oneview-golang/ov"
	"os"
)

func main() {
	var (
		clientOV *ov.OVClient
	)
	ovc := clientOV.NewOVClient(
		os.Getenv("ONEVIEW_OV_USER"),
		os.Getenv("ONEVIEW_OV_PASSWORD"),
		os.Getenv("ONEVIEW_OV_DOMAIN"),
		os.Getenv("ONEVIEW_OV_ENDPOINT"),
		false,
		600)

	interconnect_list, err := ovc.GetInterconnects("", "", "", "")
	if err != nil {
		fmt.Println(err)
	} else {

		fmt.Println("#----------------Interconnect List---------------#")

		for i := 0; i < len(interconnect_list.Members); i++ {
			fmt.Println(interconnect_list.Members[i].Name)
		}

		interconnect, err := ovc.GetInterconnectByName(interconnect_list.Members[0].Name)
		if err != nil {
			fmt.Println(err)
		} else {

			fmt.Println("#-------------Interconnect by Name----------------#")
			fmt.Println(interconnect.Name)

			uri := interconnect.URI
			interconnect, err = ovc.GetInterconnectByUri(uri)
			if err != nil {
				fmt.Println(err)
			} else {

				fmt.Println("#----------------Interconnect by URI--------------#")
				fmt.Println(interconnect.Name)
			}
		}
	}
}
<commit_msg>Example updated.<commit_after>package main

import (
	"fmt"
	"github.com/HewlettPackard/oneview-golang/ov"
	"os"
)

func main() {
	var (
		clientOV *ov.OVClient
	)
	ovc := clientOV.NewOVClient(
		os.Getenv("ONEVIEW_OV_USER"),
		os.Getenv("ONEVIEW_OV_PASSWORD"),
		os.Getenv("ONEVIEW_OV_DOMAIN"),
		os.Getenv("ONEVIEW_OV_ENDPOINT"),
		false,
		600)

	sort := "name:desc"
	interconnect_list, err := ovc.GetInterconnects("", "", "", sort)
	if err != nil {
		fmt.Println(err)
	} else {

		fmt.Println("#----------------Interconnect List---------------#")

		for i := 0; i < len(interconnect_list.Members); i++ {
			fmt.Println(interconnect_list.Members[i].Name)
		}

		interconnect, err := ovc.GetInterconnectByName(interconnect_list.Members[0].Name)
		if err != nil {
			fmt.Println(err)
		} else {

			fmt.Println("#-------------Interconnect by Name----------------#")
			fmt.Println(interconnect.Name)

			uri := interconnect.URI
			interconnect, err = ovc.GetInterconnectByUri(uri)
			if err != nil {
				fmt.Println(err)
			} else {

				fmt.Println("#----------------Interconnect by URI--------------#")
				fmt.Println(interconnect.Name)
			}
		}
	}
}
<|endoftext|>