<commit_before>// Copyright 2017 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package arm64asm

import (
	"strings"
	"testing"
)

func TestObjdumpARM64TestDecodeGNUSyntaxdata(t *testing.T) {
	testObjdumpARM64(t, testdataCases(t, "gnu"))
}
func TestObjdumpARM64TestDecodeGoSyntaxdata(t *testing.T) {
	testObjdumpARM64(t, testdataCases(t, "plan9"))
}
func TestObjdumpARM64Manual(t *testing.T) { testObjdumpARM64(t, hexCases(t, objdumpManualTests)) }
func TestObjdumpARM64Cond(t *testing.T)   { testObjdumpARM64(t, condCases(t)) }
func TestObjdumpARM64(t *testing.T)       { testObjdumpARM64(t, JSONCases(t)) }

// objdumpManualTests holds test cases that will be run by TestObjdumpARMManual.
// If you are debugging a few cases that turned up in a longer run, it can be useful
// to list them here and then use -run=Manual, particularly with tracing enabled.
// Note that these are byte sequences, so they must be reversed from the usual
// word presentation.
var objdumpManualTests = `
bf2003d5
9f2003d5
7f2003d5
5f2003d5
3f2003d5
1f2003d5
df4d03d5
ff4d03d5
28d91b14
da6cb530
15e5e514
ff4603d5
df4803d5
bf4100d5
9f3f03d5
9f3e03d5
9f3d03d5
9f3b03d5
9f3a03d5
9f3903d5
9f3703d5
9f3603d5
9f3503d5
9f3303d5
9f3203d5
9f3103d5
ff4603d5
df4803d5
bf4100d5
a3681b53
47dc78d3
0500a012
0500e092
0500a052
0500a0d2
cd5a206e
cd5a202e
743d050e
743d0a0e
743d0c0e
743d084e
`

// allowedMismatchObjdump reports whether the mismatch between text and dec
// should be allowed by the test.
func allowedMismatchObjdump(text string, inst *Inst, dec ExtInst) bool {
	// Skip unsupported instructions
	if hasPrefix(dec.text, todo...) {
		return true
	}
	// GNU objdump has incorrect alias conditions for following instructions
	if inst.Enc&0x000003ff == 0x000003ff && hasPrefix(dec.text, "negs") && hasPrefix(text, "cmp") {
		return true
	}
	// GNU objdump "NV" is equal to our "AL"
	if strings.HasSuffix(dec.text, " nv") && strings.HasSuffix(text, " al") {
		return true
	}
	if strings.HasPrefix(dec.text, "b.nv") && strings.HasPrefix(text, "b.al") {
		return true
	}
	// GNU objdump recognizes invalid binaries as following instructions
	if hasPrefix(dec.text, "hint", "mrs", "msr", "bfc", "orr", "mov") {
		return true
	}
	if strings.HasPrefix(text, "hint") {
		return true
	}
	// GNU objdump recognizes reserved valuse as valid ones
	if strings.Contains(text, "unknown instruction") && hasPrefix(dec.text, "fmla", "fmul", "fmulx", "fcvtzs", "fcvtzu", "fmls", "fmov", "scvtf", "ucvtf") {
		return true
	}
	// Some old objdump recognizes ldur*/stur*/prfum as ldr*/str*/prfm
	for k, v := range oldObjdumpMismatch {
		if strings.HasPrefix(dec.text, k) && strings.Replace(dec.text, k, v, 1) == text {
			return true
		}
	}
	// GNU objdump misses spaces between operands for some instructions (e.g., "ld1 {v10.2s, v11.2s}, [x23],#16")
	if strings.Replace(text, " ", "", -1) == strings.Replace(dec.text, " ", "", -1) {
		return true
	}
	return false
}

// TODO: system instruction.
var todo = strings.Fields(`
	sys
	dc
	at
	tlbi
	ic
	hvc
	smc
`)

// Following instructions can't be covered because they are just aliases to another instructions which are always preferred
var Ncover = strings.Fields(`
	sbfm
	asrv
	bfm
	ubfm
	lslv
	lsrv
	rorv
	ins
	dup
`)

// Some old objdump wrongly decodes following instructions and allow their mismatches to avoid false alarm
var oldObjdumpMismatch = map[string]string{
	//oldObjValue	correctValue
	"ldr":   "ldur",
	"ldrb":  "ldurb",
	"ldrh":  "ldurh",
	"ldrsb": "ldursb",
	"ldrsh": "ldursh",
	"ldrsw": "ldursw",
	"str":   "stur",
	"strb":  "sturb",
	"strh":  "sturh",
	"prfm":  "prfum",
}
<commit_msg>arm64/arm64asm: recognise new ssbb/pssbb mnemonics from objdump<commit_after>// Copyright 2017 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package arm64asm

import (
	"strings"
	"testing"
)

func TestObjdumpARM64TestDecodeGNUSyntaxdata(t *testing.T) {
	testObjdumpARM64(t, testdataCases(t, "gnu"))
}
func TestObjdumpARM64TestDecodeGoSyntaxdata(t *testing.T) {
	testObjdumpARM64(t, testdataCases(t, "plan9"))
}
func TestObjdumpARM64Manual(t *testing.T) { testObjdumpARM64(t, hexCases(t, objdumpManualTests)) }
func TestObjdumpARM64Cond(t *testing.T)   { testObjdumpARM64(t, condCases(t)) }
func TestObjdumpARM64(t *testing.T)       { testObjdumpARM64(t, JSONCases(t)) }

// objdumpManualTests holds test cases that will be run by TestObjdumpARMManual.
// If you are debugging a few cases that turned up in a longer run, it can be useful
// to list them here and then use -run=Manual, particularly with tracing enabled.
// Note that these are byte sequences, so they must be reversed from the usual
// word presentation.
var objdumpManualTests = `
bf2003d5
9f2003d5
7f2003d5
5f2003d5
3f2003d5
1f2003d5
df4d03d5
ff4d03d5
28d91b14
da6cb530
15e5e514
ff4603d5
df4803d5
bf4100d5
9f3f03d5
9f3e03d5
9f3d03d5
9f3b03d5
9f3a03d5
9f3903d5
9f3703d5
9f3603d5
9f3503d5
9f3303d5
9f3203d5
9f3103d5
ff4603d5
df4803d5
bf4100d5
a3681b53
47dc78d3
0500a012
0500e092
0500a052
0500a0d2
cd5a206e
cd5a202e
743d050e
743d0a0e
743d0c0e
743d084e
`

// allowedMismatchObjdump reports whether the mismatch between text and dec
// should be allowed by the test.
func allowedMismatchObjdump(text string, inst *Inst, dec ExtInst) bool {
	// Skip unsupported instructions
	if hasPrefix(dec.text, todo...) {
		return true
	}
	// GNU objdump has incorrect alias conditions for following instructions
	if inst.Enc&0x000003ff == 0x000003ff && hasPrefix(dec.text, "negs") && hasPrefix(text, "cmp") {
		return true
	}
	// GNU objdump "NV" is equal to our "AL"
	if strings.HasSuffix(dec.text, " nv") && strings.HasSuffix(text, " al") {
		return true
	}
	if strings.HasPrefix(dec.text, "b.nv") && strings.HasPrefix(text, "b.al") {
		return true
	}
	// GNU objdump recognizes invalid binaries as following instructions
	if hasPrefix(dec.text, "hint", "mrs", "msr", "bfc", "orr", "mov") {
		return true
	}
	if strings.HasPrefix(text, "hint") {
		return true
	}
	// GNU objdump recognizes reserved valuse as valid ones
	if strings.Contains(text, "unknown instruction") && hasPrefix(dec.text, "fmla", "fmul", "fmulx", "fcvtzs", "fcvtzu", "fmls", "fmov", "scvtf", "ucvtf") {
		return true
	}
	// Some old objdump recognizes ldur*/stur*/prfum as ldr*/str*/prfm
	for k, v := range oldObjdumpMismatch {
		if strings.HasPrefix(dec.text, k) && strings.Replace(dec.text, k, v, 1) == text {
			return true
		}
	}
	// New objdump supports some newer mnemonics than this package. This
	// package should be updated to support the new mnemonics and the sense
	// of this reversed to continue passing with older objdumps but that
	// requires internal ARM tooling.
	if newForm, ok := newMnemonics[text]; ok && newForm == dec.text {
		return true
	}
	// GNU objdump misses spaces between operands for some instructions (e.g., "ld1 {v10.2s, v11.2s}, [x23],#16")
	if strings.Replace(text, " ", "", -1) == strings.Replace(dec.text, " ", "", -1) {
		return true
	}
	return false
}

// TODO: system instruction.
var todo = strings.Fields(`
	sys
	dc
	at
	tlbi
	ic
	hvc
	smc
`)

// Following instructions can't be covered because they are just aliases to another instructions which are always preferred
var Ncover = strings.Fields(`
	sbfm
	asrv
	bfm
	ubfm
	lslv
	lsrv
	rorv
	ins
	dup
`)

// Some old objdump wrongly decodes following instructions and allow their mismatches to avoid false alarm
var oldObjdumpMismatch = map[string]string{
	//oldObjValue	correctValue
	"ldr":   "ldur",
	"ldrb":  "ldurb",
	"ldrh":  "ldurh",
	"ldrsb": "ldursb",
	"ldrsh": "ldursh",
	"ldrsw": "ldursw",
	"str":   "stur",
	"strb":  "sturb",
	"strh":  "sturh",
	"prfm":  "prfum",
}

var newMnemonics = map[string]string{
	"dsb #0x00": "ssbb",
	"dsb #0x04": "pssbb",
}
<|endoftext|>


<commit_before>// Copyright 2016-2019 Google LLC. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package fetch

import (
	"encoding/json"
	"fmt"
	"hash/crc64"
	"io"
	"io/ioutil"
	"math"
	"math/rand"
	"net/http"
	"net/url"
	"os"
	"path/filepath"
	"strings"
	"time"

	"github.com/googlecodelabs/tools/claat/fetch/drive/auth"
	"github.com/googlecodelabs/tools/claat/parser"
	"github.com/googlecodelabs/tools/claat/types"
	"github.com/googlecodelabs/tools/claat/util"
)

const (
	// supported codelab source types must be registered parsers
	// TODO: define these in claat/parser/..., e.g. in parser/gdoc
	SrcInvalid   srcType = ""
	SrcGoogleDoc srcType = "gdoc" // Google Docs doc
	SrcMarkdown  srcType = "md"   // Markdown text

	// driveAPI is a base URL for Drive API
	driveAPI = "https://www.googleapis.com/drive/v3"
)

// srcType is codelab source type
type srcType string

// resource is a codelab resource, loaded from local file
// or fetched from remote location.
type resource struct {
	typ  srcType       // source type
	body io.ReadCloser // resource body
	mod  time.Time     // last update of content
}

// codelab wraps types.Codelab, while adding source type
// and modified timestamp fields.
type codelab struct {
	*types.Codelab
	Typ srcType   //  source type
	Mod time.Time // last modified timestamp
}

type Fetcher struct {
	authHelper   *auth.Helper
	authToken    string
	crcTable     *crc64.Table
	passMetadata map[string]bool
}

func NewFetcher(at string, pm map[string]bool) (*Fetcher, error) {
	h, err := auth.NewHelper(at, auth.ProviderGoogle, nil)
	if err != nil {
		return nil, err
	}

	return &Fetcher{
		authHelper:   h,
		authToken:    at,
		crcTable:     crc64.MakeTable(crc64.ECMA),
		passMetadata: pm,
	}, nil
}

// SlurpCodelab retrieves and parses codelab source.
// It takes the source, plus an auth token and a set of extra metadata to pass along.
// It returns parsed codelab and its source type.
//
// The function will also fetch and parse fragments included
// with types.ImportNode.
func (f *Fetcher) SlurpCodelab(src string) (*codelab, error) {
	res, err := fetch(src, f.authToken)
	if err != nil {
		return nil, err
	}
	defer res.body.Close()

	opts := *parser.NewOptions()
	opts.PassMetadata = f.passMetadata

	clab, err := parser.Parse(string(res.typ), res.body, opts)
	if err != nil {
		return nil, err
	}

	// fetch imports and parse them as fragments
	var imports []*types.ImportNode
	for _, st := range clab.Steps {
		imports = append(imports, types.ImportNodes(st.Content.Nodes)...)
	}
	ch := make(chan error, len(imports))
	defer close(ch)
	for _, imp := range imports {
		go func(n *types.ImportNode) {
			frag, err := slurpFragment(n.URL, f.authToken)
			if err != nil {
				ch <- fmt.Errorf("%s: %v", n.URL, err)
				return
			}
			n.Content.Nodes = frag
			ch <- nil
		}(imp)
	}
	for range imports {
		if err := <-ch; err != nil {
			return nil, err
		}
	}

	v := &codelab{
		Codelab: clab,
		Typ:     res.typ,
		Mod:     res.mod,
	}
	return v, nil
}

func (f *Fetcher) SlurpImages(src, dir string, steps []*types.Step) (map[string]string, error) {
	// make sure img dir exists
	if err := os.MkdirAll(dir, 0755); err != nil {
		return nil, err
	}

	type res struct {
		url, file string
		err       error
	}

	ch := make(chan *res, 100)
	defer close(ch)
	var count int
	for _, st := range steps {
		nodes := types.ImageNodes(st.Content.Nodes)
		count += len(nodes)
		for _, n := range nodes {
			go func(n *types.ImageNode) {
				url := n.Src
				file, err := f.slurpBytes(src, dir, url)
				if err == nil {
					n.Src = filepath.Join(util.ImgDirname, file)
				}
				ch <- &res{url, file, err}
			}(n)
		}
	}

	imap := make(map[string]string, count)
	var err error
	for i := 0; i < count; i++ {
		r := <-ch
		imap[r.file] = r.url
		if r.err != nil && err == nil {
			// record first error
			err = fmt.Errorf("%s => %s: %v", r.url, r.file, r.err)
		}
	}

	return imap, err
}

func (f *Fetcher) slurpBytes(codelabSrc, dir, imgURL string) (string, error) {
	// images can be local in Markdown cases or remote.
	// Only proceed a simple copy on local reference.
	var b []byte
	var ext string
	u, err := url.Parse(imgURL)
	if err != nil {
		return "", err
	}

	// If the codelab source is being downloaded from the network, then we should interpret
	// the image URL in the same way.
	srcUrl, err := url.Parse(codelabSrc)
	if err == nil && srcUrl.Host != "" {
		u = srcUrl.ResolveReference(u)
	}

	if u.Host == "" {
		if imgURL, err = restrictPathToParent(imgURL, filepath.Dir(codelabSrc)); err != nil {
			return "", err
		}
		b, err = ioutil.ReadFile(imgURL)
		ext = filepath.Ext(imgURL)
	} else {
		b, err = slurpRemoteBytes(f.authHelper.DriveClient(), u.String(), 5)
		if string(b[6:10]) == "JFIF" {
			ext = ".jpeg"
		} else if string(b[0:3]) == "GIF" {
			ext = ".gif"
		} else {
			ext = ".png"
		}
	}
	if err != nil {
		return "", err
	}

	crc := crc64.Checksum(b, f.crcTable)
	file := fmt.Sprintf("%x%s", crc, ext)
	dst := filepath.Join(dir, file)
	return file, ioutil.WriteFile(dst, b, 0644)
}

func slurpFragment(url, authToken string) ([]types.Node, error) {
	res, err := fetchRemote(url, authToken, true)
	if err != nil {
		return nil, err
	}
	defer res.body.Close()
	return parser.ParseFragment(string(res.typ), res.body)
}

// fetch retrieves codelab doc either from local disk
// or a remote location.
// The caller is responsible for closing returned stream.
func fetch(name, authToken string) (*resource, error) {
	fi, err := os.Stat(name)
	if os.IsNotExist(err) {
		return fetchRemote(name, authToken, false)
	}
	r, err := os.Open(name)
	if err != nil {
		return nil, err
	}
	return &resource{
		body: r,
		typ:  SrcMarkdown,
		mod:  fi.ModTime(),
	}, nil
}

// fetchRemote retrieves resource r from the network.
//
// If urlStr is not a URL, i.e. does not have the host part, it is considered to be
// a Google Doc ID and fetched accordingly. Otherwise, a simple GET request
// is used to retrieve the contents.
//
// The caller is responsible for closing returned stream.
// If nometa is true, resource.mod may have zero value.
func fetchRemote(urlStr, authToken string, nometa bool) (*resource, error) {
	u, err := url.Parse(urlStr)
	if err != nil {
		return nil, err
	}
	if u.Host == "" || u.Host == "docs.google.com" {
		return fetchDriveFile(urlStr, authToken, nometa)
	}
	return fetchRemoteFile(urlStr)
}

// fetchRemoteFile retrieves codelab resource from url.
// It is a special case of fetchRemote function.
func fetchRemoteFile(url string) (*resource, error) {
	res, err := retryGet(nil, url, 3)
	if err != nil {
		return nil, err
	}
	t, err := http.ParseTime(res.Header.Get("last-modified"))
	if err != nil {
		t = time.Now()
	}
	return &resource{
		body: res.Body,
		mod:  t,
		typ:  SrcMarkdown,
	}, nil
}

// fetchDriveFile uses Drive API to retrieve HTML representation of a Google Doc.
// See https://developers.google.com/drive/web/manage-downloads#downloading_google_documents
// for more details.
//
// If nometa is true, resource.mod will have zero value.
func fetchDriveFile(id, authToken string, nometa bool) (*resource, error) {
	id = gdocID(id)
	exportURL := gdocExportURL(id)
	h, err := auth.NewHelper(authToken, auth.ProviderGoogle, nil)
	if err != nil {
		return nil, err
	}
	client := h.DriveClient()

	if nometa {
		res, err := retryGet(client, exportURL, 7)
		if err != nil {
			return nil, err
		}
		return &resource{body: res.Body, typ: SrcGoogleDoc}, nil
	}

	q := url.Values{
		"fields":             {"id,mimeType,modifiedTime"},
		"supportsTeamDrives": {"true"},
	}
	u := fmt.Sprintf("%s/files/%s?%s", driveAPI, id, q.Encode())
	res, err := retryGet(client, u, 7)
	if err != nil {
		return nil, err
	}
	defer res.Body.Close()
	meta := &struct {
		ID       string    `json:"id"`
		MimeType string    `json:"mimeType"`
		Modified time.Time `json:"modifiedTime"`
	}{}
	if err := json.NewDecoder(res.Body).Decode(meta); err != nil {
		return nil, err
	}
	if meta.MimeType != "application/vnd.google-apps.document" {
		return nil, fmt.Errorf("%s: invalid mime type: %s", id, meta.MimeType)
	}

	if res, err = retryGet(client, exportURL, 7); err != nil {
		return nil, err
	}
	return &resource{
		body: res.Body,
		mod:  meta.Modified,
		typ:  SrcGoogleDoc,
	}, nil
}

func slurpRemoteBytes(client *http.Client, url string, n int) ([]byte, error) {
	res, err := retryGet(client, url, n)
	if err != nil {
		return nil, err
	}
	defer res.Body.Close()
	return ioutil.ReadAll(res.Body)
}

// retryGet tries to GET specified url up to n times.
// Default client will be used if not provided.
func retryGet(client *http.Client, url string, n int) (*http.Response, error) {
	if client == nil {
		client = http.DefaultClient
	}
	for i := 0; i <= n; i++ {
		if i > 0 {
			t := time.Duration((math.Pow(2, float64(i)) + rand.Float64()) * float64(time.Second))
			time.Sleep(t)
		}
		res, err := client.Get(url)
		// return early with a good response
		// the rest is error handling
		if err == nil && res.StatusCode == http.StatusOK {
			return res, nil
		}

		// sometimes Drive API wouldn't even start a response,
		// we get net/http: TLS handshake timeout instead:
		// consider this a temporary failure and retry again
		if err != nil {
			continue
		}
		// otherwise, decode error response and check for "rate limit"
		defer res.Body.Close()
		var erres struct {
			Error struct {
				Errors []struct{ Reason string }
			}
		}
		b, _ := ioutil.ReadAll(res.Body)
		json.Unmarshal(b, &erres)
		var rateLimit bool
		for _, e := range erres.Error.Errors {
			if e.Reason == "rateLimitExceeded" || e.Reason == "userRateLimitExceeded" {
				rateLimit = true
				break
			}
		}
		// this is neither a rate limit error, nor a server error:
		// retrying is useless
		if !rateLimit && res.StatusCode < http.StatusInternalServerError {
			return nil, fmt.Errorf("fetch %s: %s; %s", url, res.Status, b)
		}
	}
	return nil, fmt.Errorf("%s: failed after %d retries", url, n)
}

func gdocID(url string) string {
	const s = "/document/d/"
	if i := strings.Index(url, s); i >= 0 {
		url = url[i+len(s):]
	}
	if i := strings.IndexRune(url, '/'); i > 0 {
		url = url[:i]
	}
	return url
}

func gdocExportURL(id string) string {
	return fmt.Sprintf("%s/files/%s/export?mimeType=text/html", driveAPI, id)
}

// restrictPathToParent will ensure that assetPath is in parent.
// It will thus return an absolute path to the asset.
func restrictPathToParent(assetPath, parent string) (string, error) {
	parent, err := filepath.Abs(parent)
	if err != nil {
		return "", err
	}
	if !filepath.IsAbs(assetPath) {
		assetPath = filepath.Join(parent, assetPath)
	}
	if !strings.HasPrefix(assetPath, parent) {
		return "", fmt.Errorf("%s isn't a subdirectory of %s", assetPath, parent)
	}
	return assetPath, nil
}
<commit_msg>Make slurpFragment a method of Fetcher.<commit_after>// Copyright 2016-2019 Google LLC. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package fetch

import (
	"encoding/json"
	"fmt"
	"hash/crc64"
	"io"
	"io/ioutil"
	"math"
	"math/rand"
	"net/http"
	"net/url"
	"os"
	"path/filepath"
	"strings"
	"time"

	"github.com/googlecodelabs/tools/claat/fetch/drive/auth"
	"github.com/googlecodelabs/tools/claat/parser"
	"github.com/googlecodelabs/tools/claat/types"
	"github.com/googlecodelabs/tools/claat/util"
)

const (
	// supported codelab source types must be registered parsers
	// TODO: define these in claat/parser/..., e.g. in parser/gdoc
	SrcInvalid   srcType = ""
	SrcGoogleDoc srcType = "gdoc" // Google Docs doc
	SrcMarkdown  srcType = "md"   // Markdown text

	// driveAPI is a base URL for Drive API
	driveAPI = "https://www.googleapis.com/drive/v3"
)

// srcType is codelab source type
type srcType string

// resource is a codelab resource, loaded from local file
// or fetched from remote location.
type resource struct {
	typ  srcType       // source type
	body io.ReadCloser // resource body
	mod  time.Time     // last update of content
}

// codelab wraps types.Codelab, while adding source type
// and modified timestamp fields.
type codelab struct {
	*types.Codelab
	Typ srcType   //  source type
	Mod time.Time // last modified timestamp
}

type Fetcher struct {
	authHelper   *auth.Helper
	authToken    string
	crcTable     *crc64.Table
	passMetadata map[string]bool
}

func NewFetcher(at string, pm map[string]bool) (*Fetcher, error) {
	h, err := auth.NewHelper(at, auth.ProviderGoogle, nil)
	if err != nil {
		return nil, err
	}

	return &Fetcher{
		authHelper:   h,
		authToken:    at,
		crcTable:     crc64.MakeTable(crc64.ECMA),
		passMetadata: pm,
	}, nil
}

// SlurpCodelab retrieves and parses codelab source.
// It takes the source, plus an auth token and a set of extra metadata to pass along.
// It returns parsed codelab and its source type.
//
// The function will also fetch and parse fragments included
// with types.ImportNode.
func (f *Fetcher) SlurpCodelab(src string) (*codelab, error) {
	res, err := fetch(src, f.authToken)
	if err != nil {
		return nil, err
	}
	defer res.body.Close()

	opts := *parser.NewOptions()
	opts.PassMetadata = f.passMetadata

	clab, err := parser.Parse(string(res.typ), res.body, opts)
	if err != nil {
		return nil, err
	}

	// fetch imports and parse them as fragments
	var imports []*types.ImportNode
	for _, st := range clab.Steps {
		imports = append(imports, types.ImportNodes(st.Content.Nodes)...)
	}
	ch := make(chan error, len(imports))
	defer close(ch)
	for _, imp := range imports {
		go func(n *types.ImportNode) {
			frag, err := f.slurpFragment(n.URL)
			if err != nil {
				ch <- fmt.Errorf("%s: %v", n.URL, err)
				return
			}
			n.Content.Nodes = frag
			ch <- nil
		}(imp)
	}
	for range imports {
		if err := <-ch; err != nil {
			return nil, err
		}
	}

	v := &codelab{
		Codelab: clab,
		Typ:     res.typ,
		Mod:     res.mod,
	}
	return v, nil
}

func (f *Fetcher) SlurpImages(src, dir string, steps []*types.Step) (map[string]string, error) {
	// make sure img dir exists
	if err := os.MkdirAll(dir, 0755); err != nil {
		return nil, err
	}

	type res struct {
		url, file string
		err       error
	}

	ch := make(chan *res, 100)
	defer close(ch)
	var count int
	for _, st := range steps {
		nodes := types.ImageNodes(st.Content.Nodes)
		count += len(nodes)
		for _, n := range nodes {
			go func(n *types.ImageNode) {
				url := n.Src
				file, err := f.slurpBytes(src, dir, url)
				if err == nil {
					n.Src = filepath.Join(util.ImgDirname, file)
				}
				ch <- &res{url, file, err}
			}(n)
		}
	}

	imap := make(map[string]string, count)
	var err error
	for i := 0; i < count; i++ {
		r := <-ch
		imap[r.file] = r.url
		if r.err != nil && err == nil {
			// record first error
			err = fmt.Errorf("%s => %s: %v", r.url, r.file, r.err)
		}
	}

	return imap, err
}

func (f *Fetcher) slurpBytes(codelabSrc, dir, imgURL string) (string, error) {
	// images can be local in Markdown cases or remote.
	// Only proceed a simple copy on local reference.
	var b []byte
	var ext string
	u, err := url.Parse(imgURL)
	if err != nil {
		return "", err
	}

	// If the codelab source is being downloaded from the network, then we should interpret
	// the image URL in the same way.
	srcUrl, err := url.Parse(codelabSrc)
	if err == nil && srcUrl.Host != "" {
		u = srcUrl.ResolveReference(u)
	}

	if u.Host == "" {
		if imgURL, err = restrictPathToParent(imgURL, filepath.Dir(codelabSrc)); err != nil {
			return "", err
		}
		b, err = ioutil.ReadFile(imgURL)
		ext = filepath.Ext(imgURL)
	} else {
		b, err = slurpRemoteBytes(f.authHelper.DriveClient(), u.String(), 5)
		if string(b[6:10]) == "JFIF" {
			ext = ".jpeg"
		} else if string(b[0:3]) == "GIF" {
			ext = ".gif"
		} else {
			ext = ".png"
		}
	}
	if err != nil {
		return "", err
	}

	crc := crc64.Checksum(b, f.crcTable)
	file := fmt.Sprintf("%x%s", crc, ext)
	dst := filepath.Join(dir, file)
	return file, ioutil.WriteFile(dst, b, 0644)
}

func (f *Fetcher) slurpFragment(url string) ([]types.Node, error) {
	res, err := fetchRemote(url, f.authToken, true)
	if err != nil {
		return nil, err
	}
	defer res.body.Close()
	return parser.ParseFragment(string(res.typ), res.body)
}

// fetch retrieves codelab doc either from local disk
// or a remote location.
// The caller is responsible for closing returned stream.
func fetch(name, authToken string) (*resource, error) {
	fi, err := os.Stat(name)
	if os.IsNotExist(err) {
		return fetchRemote(name, authToken, false)
	}
	r, err := os.Open(name)
	if err != nil {
		return nil, err
	}
	return &resource{
		body: r,
		typ:  SrcMarkdown,
		mod:  fi.ModTime(),
	}, nil
}

// fetchRemote retrieves resource r from the network.
//
// If urlStr is not a URL, i.e. does not have the host part, it is considered to be
// a Google Doc ID and fetched accordingly. Otherwise, a simple GET request
// is used to retrieve the contents.
//
// The caller is responsible for closing returned stream.
// If nometa is true, resource.mod may have zero value.
func fetchRemote(urlStr, authToken string, nometa bool) (*resource, error) {
	u, err := url.Parse(urlStr)
	if err != nil {
		return nil, err
	}
	if u.Host == "" || u.Host == "docs.google.com" {
		return fetchDriveFile(urlStr, authToken, nometa)
	}
	return fetchRemoteFile(urlStr)
}

// fetchRemoteFile retrieves codelab resource from url.
// It is a special case of fetchRemote function.
func fetchRemoteFile(url string) (*resource, error) {
	res, err := retryGet(nil, url, 3)
	if err != nil {
		return nil, err
	}
	t, err := http.ParseTime(res.Header.Get("last-modified"))
	if err != nil {
		t = time.Now()
	}
	return &resource{
		body: res.Body,
		mod:  t,
		typ:  SrcMarkdown,
	}, nil
}

// fetchDriveFile uses Drive API to retrieve HTML representation of a Google Doc.
// See https://developers.google.com/drive/web/manage-downloads#downloading_google_documents
// for more details.
//
// If nometa is true, resource.mod will have zero value.
func fetchDriveFile(id, authToken string, nometa bool) (*resource, error) {
	id = gdocID(id)
	exportURL := gdocExportURL(id)
	h, err := auth.NewHelper(authToken, auth.ProviderGoogle, nil)
	if err != nil {
		return nil, err
	}
	client := h.DriveClient()

	if nometa {
		res, err := retryGet(client, exportURL, 7)
		if err != nil {
			return nil, err
		}
		return &resource{body: res.Body, typ: SrcGoogleDoc}, nil
	}

	q := url.Values{
		"fields":             {"id,mimeType,modifiedTime"},
		"supportsTeamDrives": {"true"},
	}
	u := fmt.Sprintf("%s/files/%s?%s", driveAPI, id, q.Encode())
	res, err := retryGet(client, u, 7)
	if err != nil {
		return nil, err
	}
	defer res.Body.Close()
	meta := &struct {
		ID       string    `json:"id"`
		MimeType string    `json:"mimeType"`
		Modified time.Time `json:"modifiedTime"`
	}{}
	if err := json.NewDecoder(res.Body).Decode(meta); err != nil {
		return nil, err
	}
	if meta.MimeType != "application/vnd.google-apps.document" {
		return nil, fmt.Errorf("%s: invalid mime type: %s", id, meta.MimeType)
	}

	if res, err = retryGet(client, exportURL, 7); err != nil {
		return nil, err
	}
	return &resource{
		body: res.Body,
		mod:  meta.Modified,
		typ:  SrcGoogleDoc,
	}, nil
}

func slurpRemoteBytes(client *http.Client, url string, n int) ([]byte, error) {
	res, err := retryGet(client, url, n)
	if err != nil {
		return nil, err
	}
	defer res.Body.Close()
	return ioutil.ReadAll(res.Body)
}

// retryGet tries to GET specified url up to n times.
// Default client will be used if not provided.
func retryGet(client *http.Client, url string, n int) (*http.Response, error) {
	if client == nil {
		client = http.DefaultClient
	}
	for i := 0; i <= n; i++ {
		if i > 0 {
			t := time.Duration((math.Pow(2, float64(i)) + rand.Float64()) * float64(time.Second))
			time.Sleep(t)
		}
		res, err := client.Get(url)
		// return early with a good response
		// the rest is error handling
		if err == nil && res.StatusCode == http.StatusOK {
			return res, nil
		}

		// sometimes Drive API wouldn't even start a response,
		// we get net/http: TLS handshake timeout instead:
		// consider this a temporary failure and retry again
		if err != nil {
			continue
		}
		// otherwise, decode error response and check for "rate limit"
		defer res.Body.Close()
		var erres struct {
			Error struct {
				Errors []struct{ Reason string }
			}
		}
		b, _ := ioutil.ReadAll(res.Body)
		json.Unmarshal(b, &erres)
		var rateLimit bool
		for _, e := range erres.Error.Errors {
			if e.Reason == "rateLimitExceeded" || e.Reason == "userRateLimitExceeded" {
				rateLimit = true
				break
			}
		}
		// this is neither a rate limit error, nor a server error:
		// retrying is useless
		if !rateLimit && res.StatusCode < http.StatusInternalServerError {
			return nil, fmt.Errorf("fetch %s: %s; %s", url, res.Status, b)
		}
	}
	return nil, fmt.Errorf("%s: failed after %d retries", url, n)
}

func gdocID(url string) string {
	const s = "/document/d/"
	if i := strings.Index(url, s); i >= 0 {
		url = url[i+len(s):]
	}
	if i := strings.IndexRune(url, '/'); i > 0 {
		url = url[:i]
	}
	return url
}

func gdocExportURL(id string) string {
	return fmt.Sprintf("%s/files/%s/export?mimeType=text/html", driveAPI, id)
}

// restrictPathToParent will ensure that assetPath is in parent.
// It will thus return an absolute path to the asset.
func restrictPathToParent(assetPath, parent string) (string, error) {
	parent, err := filepath.Abs(parent)
	if err != nil {
		return "", err
	}
	if !filepath.IsAbs(assetPath) {
		assetPath = filepath.Join(parent, assetPath)
	}
	if !strings.HasPrefix(assetPath, parent) {
		return "", fmt.Errorf("%s isn't a subdirectory of %s", assetPath, parent)
	}
	return assetPath, nil
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"strings"
	"testing"

	"github.com/BytemarkHosting/bytemark-client/cmd/bytemark/testutil"
	"github.com/BytemarkHosting/bytemark-client/lib"
	"github.com/BytemarkHosting/bytemark-client/lib/brain"
	"github.com/BytemarkHosting/bytemark-client/mocks"
	"github.com/cheekybits/is"
)

func TestResetCommand(t *testing.T) {
	is := is.New(t)
	config, c, app := testutil.BaseTestAuthSetup(t, false, commands)
	vmn := lib.VirtualMachineName{VirtualMachine: "test-server", Group: "test-group", Account: "test-account"}

	config.When("GetVirtualMachine").Return(defVM)

	c.When("ResetVirtualMachine", vmn).Times(1)

	err := app.Run(strings.Split("bytemark reset test-server.test-group.test-account", " "))
	is.Nil(err)
	if ok, err := c.Verify(); !ok {
		t.Fatal(err)
	}
}
func TestRestartCommand(t *testing.T) {
	tests := []struct {
		name          string
		input         string
		vmname        lib.VirtualMachineName
		shouldErr     bool
		applianceBoot bool
	}{
		{
			name:  "RestartWithoutAppliance",
			input: "test-server.test-group.test-account",
			vmname: lib.VirtualMachineName{
				VirtualMachine: "test-server",
				Group:          "test-group",
				Account:        "test-account",
			},
		}, {
			name:  "RestartWithoutApplianceWithDefaultAccount",
			input: "test-server.test-group",
			vmname: lib.VirtualMachineName{
				VirtualMachine: "test-server",
				Group:          "test-group",
				Account:        "default-account",
			},
		}, {
			name:  "RestartWithoutApplianceWithDefaultGroup",
			input: "test-server",
			vmname: lib.VirtualMachineName{
				VirtualMachine: "test-server",
				Group:          "default",
				Account:        "default-account",
			},
		}, {
			name:  "RestartWithApplianceFlagWithoutAppliance",
			input: "test-server --appliance",
			vmname: lib.VirtualMachineName{
				VirtualMachine: "test-server",
				Group:          "default",
				Account:        "default-account",
			},
			shouldErr: true,
		}, {
			name:  "RestartWithApplianceFlag",
			input: "test-server --appliance rescue",
			vmname: lib.VirtualMachineName{
				VirtualMachine: "test-server",
				Group:          "default",
				Account:        "default-account",
			},
			applianceBoot: true,
		}, {
			name:  "RestartWithRescueFlag",
			input: "test-server --rescue",
			vmname: lib.VirtualMachineName{
				VirtualMachine: "test-server",
				Group:          "default",
				Account:        "default-account",
			},
			applianceBoot: true,
		}, {
			name:  "RestartWithApplianceAndRescueFlag",
			input: "test-server --appliance a --rescue",
			vmname: lib.VirtualMachineName{
				VirtualMachine: "test-server",
				Group:          "default",
				Account:        "default-account",
			},
			shouldErr: true,
		},
	}
	for _, test := range tests {
		t.Run(test.name, func(t *testing.T) {
			config, client, app := testutil.BaseTestAuthSetup(t, false, commands)
			config.When("GetVirtualMachine").Return(defVM)

			client.When("ShutdownVirtualMachine", test.vmname, true).Times(1)
			client.When("GetVirtualMachine", test.vmname).Return(brain.VirtualMachine{PowerOn: false})

			if test.applianceBoot {
				client.When("BuildRequest", "PUT", lib.Endpoint(1),
					"/accounts/%s/groups/%s/virtual_machines/%s",
					[]string{
						"default-account",
						"default",
						"test-server"}).Return(&mocks.Request{
					T:          t,
					StatusCode: 200,
				}).Times(1)
			} else {
				client.When("StartVirtualMachine", test.vmname).Times(1)
			}

			args := fmt.Sprintf("bytemark restart %s", test.input)
			err := app.Run(strings.Split(args, " "))
			if !test.shouldErr && err != nil {
				t.Errorf("shouldn't err, but did: %T{%s}", err, err.Error())
			} else if test.shouldErr && err == nil {
				t.Errorf("should err, but didn't")
			}
			if !test.shouldErr {
				if ok, err := client.Verify(); !ok {
					t.Fatal(err)
				}
			}
		})
	}
}
func TestShutdownCommand(t *testing.T) {
	is := is.New(t)
	config, c, app := testutil.BaseTestAuthSetup(t, false, commands)
	vmn := lib.VirtualMachineName{VirtualMachine: "test-server", Group: "test-group", Account: "test-account"}

	config.When("GetVirtualMachine").Return(defVM)

	c.When("ShutdownVirtualMachine", vmn, true).Times(1)
	c.When("GetVirtualMachine", vmn).Return(brain.VirtualMachine{PowerOn: false})

	err := app.Run(strings.Split("bytemark shutdown test-server.test-group.test-account", " "))
	is.Nil(err)
	if ok, err := c.Verify(); !ok {
		t.Fatal(err)
	}
}
func TestStartCommand(t *testing.T) {
	is := is.New(t)
	config, c, app := testutil.BaseTestAuthSetup(t, false, commands)
	vmn := lib.VirtualMachineName{VirtualMachine: "test-server", Group: "test-group", Account: "test-account"}

	config.When("GetVirtualMachine").Return(defVM)

	c.When("StartVirtualMachine", vmn).Times(1)

	err := app.Run(strings.Split("bytemark start test-server.test-group.test-account", " "))
	is.Nil(err)
	if ok, err := c.Verify(); !ok {
		t.Fatal(err)
	}
}
func TestStopCommand(t *testing.T) {
	is := is.New(t)
	config, c, app := testutil.BaseTestAuthSetup(t, false, commands)

	vmn := lib.VirtualMachineName{VirtualMachine: "test-server", Group: "test-group", Account: "test-account"}

	config.When("GetVirtualMachine").Return(defVM)

	c.When("StopVirtualMachine", vmn).Times(1)

	err := app.Run(strings.Split("bytemark stop test-server.test-group.test-account", " "))
	is.Nil(err)

	if ok, err := c.Verify(); !ok {
		t.Fatal(err)
	}
}
<commit_msg>throw in the mock call for retreiving the panel url<commit_after>package main

import (
	"fmt"
	"strings"
	"testing"

	"github.com/BytemarkHosting/bytemark-client/cmd/bytemark/testutil"
	"github.com/BytemarkHosting/bytemark-client/lib"
	"github.com/BytemarkHosting/bytemark-client/lib/brain"
	"github.com/BytemarkHosting/bytemark-client/mocks"
	"github.com/cheekybits/is"
)

func TestResetCommand(t *testing.T) {
	is := is.New(t)
	config, c, app := testutil.BaseTestAuthSetup(t, false, commands)
	vmn := lib.VirtualMachineName{VirtualMachine: "test-server", Group: "test-group", Account: "test-account"}

	config.When("GetVirtualMachine").Return(defVM)

	c.When("ResetVirtualMachine", vmn).Times(1)

	err := app.Run(strings.Split("bytemark reset test-server.test-group.test-account", " "))
	is.Nil(err)
	if ok, err := c.Verify(); !ok {
		t.Fatal(err)
	}
}
func TestRestartCommand(t *testing.T) {
	tests := []struct {
		name          string
		input         string
		vmname        lib.VirtualMachineName
		shouldErr     bool
		applianceBoot bool
	}{
		{
			name:  "RestartWithoutAppliance",
			input: "test-server.test-group.test-account",
			vmname: lib.VirtualMachineName{
				VirtualMachine: "test-server",
				Group:          "test-group",
				Account:        "test-account",
			},
		}, {
			name:  "RestartWithoutApplianceWithDefaultAccount",
			input: "test-server.test-group",
			vmname: lib.VirtualMachineName{
				VirtualMachine: "test-server",
				Group:          "test-group",
				Account:        "default-account",
			},
		}, {
			name:  "RestartWithoutApplianceWithDefaultGroup",
			input: "test-server",
			vmname: lib.VirtualMachineName{
				VirtualMachine: "test-server",
				Group:          "default",
				Account:        "default-account",
			},
		}, {
			name:  "RestartWithApplianceFlagWithoutAppliance",
			input: "test-server --appliance",
			vmname: lib.VirtualMachineName{
				VirtualMachine: "test-server",
				Group:          "default",
				Account:        "default-account",
			},
			shouldErr: true,
		}, {
			name:  "RestartWithApplianceFlag",
			input: "test-server --appliance rescue",
			vmname: lib.VirtualMachineName{
				VirtualMachine: "test-server",
				Group:          "default",
				Account:        "default-account",
			},
			applianceBoot: true,
		}, {
			name:  "RestartWithRescueFlag",
			input: "test-server --rescue",
			vmname: lib.VirtualMachineName{
				VirtualMachine: "test-server",
				Group:          "default",
				Account:        "default-account",
			},
			applianceBoot: true,
		}, {
			name:  "RestartWithApplianceAndRescueFlag",
			input: "test-server --appliance a --rescue",
			vmname: lib.VirtualMachineName{
				VirtualMachine: "test-server",
				Group:          "default",
				Account:        "default-account",
			},
			shouldErr: true,
		},
	}
	for _, test := range tests {
		t.Run(test.name, func(t *testing.T) {
			config, client, app := testutil.BaseTestAuthSetup(t, false, commands)
			config.When("GetVirtualMachine").Return(defVM)
			config.When("PanelURL").Return("something.com")

			client.When("ShutdownVirtualMachine", test.vmname, true).Times(1)
			client.When("GetVirtualMachine", test.vmname).Return(brain.VirtualMachine{PowerOn: false})

			if test.applianceBoot {
				client.When("BuildRequest", "PUT", lib.Endpoint(1),
					"/accounts/%s/groups/%s/virtual_machines/%s",
					[]string{
						"default-account",
						"default",
						"test-server"}).Return(&mocks.Request{
					T:          t,
					StatusCode: 200,
				}).Times(1)
			} else {
				client.When("StartVirtualMachine", test.vmname).Times(1)
			}

			args := fmt.Sprintf("bytemark restart %s", test.input)
			err := app.Run(strings.Split(args, " "))
			if !test.shouldErr && err != nil {
				t.Errorf("shouldn't err, but did: %T{%s}", err, err.Error())
			} else if test.shouldErr && err == nil {
				t.Errorf("should err, but didn't")
			}
			if !test.shouldErr {
				if ok, err := client.Verify(); !ok {
					t.Fatal(err)
				}
			}
		})
	}
}
func TestShutdownCommand(t *testing.T) {
	is := is.New(t)
	config, c, app := testutil.BaseTestAuthSetup(t, false, commands)
	vmn := lib.VirtualMachineName{VirtualMachine: "test-server", Group: "test-group", Account: "test-account"}

	config.When("GetVirtualMachine").Return(defVM)

	c.When("ShutdownVirtualMachine", vmn, true).Times(1)
	c.When("GetVirtualMachine", vmn).Return(brain.VirtualMachine{PowerOn: false})

	err := app.Run(strings.Split("bytemark shutdown test-server.test-group.test-account", " "))
	is.Nil(err)
	if ok, err := c.Verify(); !ok {
		t.Fatal(err)
	}
}
func TestStartCommand(t *testing.T) {
	is := is.New(t)
	config, c, app := testutil.BaseTestAuthSetup(t, false, commands)
	vmn := lib.VirtualMachineName{VirtualMachine: "test-server", Group: "test-group", Account: "test-account"}

	config.When("GetVirtualMachine").Return(defVM)

	c.When("StartVirtualMachine", vmn).Times(1)

	err := app.Run(strings.Split("bytemark start test-server.test-group.test-account", " "))
	is.Nil(err)
	if ok, err := c.Verify(); !ok {
		t.Fatal(err)
	}
}
func TestStopCommand(t *testing.T) {
	is := is.New(t)
	config, c, app := testutil.BaseTestAuthSetup(t, false, commands)

	vmn := lib.VirtualMachineName{VirtualMachine: "test-server", Group: "test-group", Account: "test-account"}

	config.When("GetVirtualMachine").Return(defVM)

	c.When("StopVirtualMachine", vmn).Times(1)

	err := app.Run(strings.Split("bytemark stop test-server.test-group.test-account", " "))
	is.Nil(err)

	if ok, err := c.Verify(); !ok {
		t.Fatal(err)
	}
}
<|endoftext|>


<commit_before>package main_test

import (
	"encoding/json"
	"fmt"
	"net/url"
	"syscall"
	"time"

	"github.com/cloudfoundry/storeadapter"
	"github.com/cloudfoundry/storeadapter/storerunner/etcdstorerunner"
	. "github.com/onsi/ginkgo"
	"github.com/onsi/ginkgo/config"
	. "github.com/onsi/gomega"
	"github.com/onsi/gomega/gbytes"
	. "github.com/onsi/gomega/gexec"
	"github.com/pivotal-golang/lager"
	"github.com/pivotal-golang/lager/lagertest"
	"github.com/tedsuo/ifrit"
	"github.com/tedsuo/ifrit/ginkgomon"

	"github.com/cloudfoundry-incubator/bbs"
	bbsrunner "github.com/cloudfoundry-incubator/bbs/cmd/bbs/testrunner"
	"github.com/cloudfoundry-incubator/bbs/models"
	"github.com/cloudfoundry-incubator/bbs/models/test/model_helpers"
	"github.com/cloudfoundry-incubator/consuladapter"
	"github.com/cloudfoundry-incubator/consuladapter/consulrunner"
	convergerrunner "github.com/cloudfoundry-incubator/converger/cmd/converger/testrunner"
	"github.com/cloudfoundry-incubator/locket"
	oldmodels "github.com/cloudfoundry-incubator/runtime-schema/models"
)

type BinPaths struct {
	Converger string
	Bbs       string
}

var _ = Describe("Converger", func() {
	const (
		exitDuration                = 4 * time.Second
		convergeRepeatInterval      = 500 * time.Millisecond
		taskKickInterval            = convergeRepeatInterval
		expireCompletedTaskDuration = 3 * convergeRepeatInterval
		expirePendingTaskDuration   = 30 * time.Minute
	)

	var (
		binPaths         BinPaths
		etcdRunner       *etcdstorerunner.ETCDClusterRunner
		bbsArgs          bbsrunner.Args
		bbsProcess       ifrit.Process
		bbsClient        bbs.Client
		convergerConfig  *convergerrunner.Config
		convergerProcess ifrit.Process
		runner           *ginkgomon.Runner

		consulRunner  *consulrunner.ClusterRunner
		consulSession *consuladapter.Session

		etcdClient storeadapter.StoreAdapter

		logger lager.Logger
	)

	SynchronizedBeforeSuite(func() []byte {
		convergerBinPath, err := Build("github.com/cloudfoundry-incubator/converger/cmd/converger", "-race")
		Expect(err).NotTo(HaveOccurred())
		bbsBinPath, err := Build("github.com/cloudfoundry-incubator/bbs/cmd/bbs", "-race")
		Expect(err).NotTo(HaveOccurred())
		bytes, err := json.Marshal(BinPaths{
			Converger: convergerBinPath,
			Bbs:       bbsBinPath,
		})
		Expect(err).NotTo(HaveOccurred())
		return bytes
	}, func(bytes []byte) {
		binPaths = BinPaths{}
		err := json.Unmarshal(bytes, &binPaths)
		Expect(err).NotTo(HaveOccurred())

		etcdPort := 5001 + config.GinkgoConfig.ParallelNode
		etcdCluster := fmt.Sprintf("http://127.0.0.1:%d", etcdPort)
		etcdRunner = etcdstorerunner.NewETCDClusterRunner(etcdPort, 1, nil)

		etcdClient = etcdRunner.Adapter(nil)

		consulRunner = consulrunner.NewClusterRunner(
			9001+config.GinkgoConfig.ParallelNode*consulrunner.PortOffsetLength,
			1,
			"http",
		)

		logger = lagertest.NewTestLogger("test")

		bbsAddress := fmt.Sprintf("127.0.0.1:%d", 13000+GinkgoParallelNode())

		bbsURL := &url.URL{
			Scheme: "http",
			Host:   bbsAddress,
		}

		bbsArgs = bbsrunner.Args{
			Address:           bbsAddress,
			AdvertiseURL:      bbsURL.String(),
			AuctioneerAddress: "some-address",
			EtcdCluster:       etcdCluster,
			ConsulCluster:     consulRunner.ConsulCluster(),
		}

		convergerConfig = &convergerrunner.Config{
			BinPath:                     binPaths.Converger,
			ConvergeRepeatInterval:      convergeRepeatInterval.String(),
			KickTaskDuration:            taskKickInterval.String(),
			ExpirePendingTaskDuration:   expirePendingTaskDuration.String(),
			ExpireCompletedTaskDuration: expireCompletedTaskDuration.String(),
			ConsulCluster:               consulRunner.ConsulCluster(),
			LogLevel:                    "info",
			BBSAddress:                  bbsURL.String(),
		}
	})

	SynchronizedAfterSuite(func() {
	}, func() {
		CleanupBuildArtifacts()
	})

	BeforeEach(func() {
		etcdRunner.Start()
		consulRunner.Start()
		consulRunner.WaitUntilReady()

		bbsProcess = ginkgomon.Invoke(bbsrunner.New(binPaths.Bbs, bbsArgs))
		bbsClient = bbs.NewClient(fmt.Sprint("http://", bbsArgs.Address))

		consulSession = consulRunner.NewSession("a-session")

		capacity := oldmodels.NewCellCapacity(512, 1024, 124)
		cellPresence := oldmodels.NewCellPresence("the-cell-id", "1.2.3.4", "the-zone", capacity, []string{}, []string{})

		value, err := oldmodels.ToJSON(cellPresence)
		Expect(err).NotTo(HaveOccurred())

		_, err = consulSession.SetPresence(locket.CellSchemaPath(cellPresence.CellID), value)
		Expect(err).NotTo(HaveOccurred())

	})

	AfterEach(func() {
		ginkgomon.Kill(bbsProcess)
		ginkgomon.Kill(convergerProcess)
		consulRunner.Stop()
		etcdRunner.Stop()
	})

	startConverger := func() {
		runner = convergerrunner.New(convergerConfig)
		convergerProcess = ginkgomon.Invoke(runner)
		time.Sleep(convergeRepeatInterval)
	}

	createRunningTaskWithDeadCell := func() {
		task := model_helpers.NewValidTask("task-guid")

		err := bbsClient.DesireTask(task.TaskGuid, task.Domain, task.TaskDefinition)
		Expect(err).NotTo(HaveOccurred())

		_, err = bbsClient.StartTask(task.TaskGuid, "dead-cell")
		Expect(err).NotTo(HaveOccurred())
	}

	itIsInactive := func() {
		Describe("when a task is desired but its cell is dead", func() {
			JustBeforeEach(createRunningTaskWithDeadCell)

			It("does not converge the task", func() {
				Consistently(func() []*models.Task {
					return getTasksByState(bbsClient, models.Task_Completed)
				}, 10*convergeRepeatInterval).Should(BeEmpty())
			})
		})
	}

	Context("when the converger has the lock", func() {
		Describe("when a task is desired but its cell is dead", func() {
			JustBeforeEach(createRunningTaskWithDeadCell)

			It("marks the task as completed and failed", func() {
				Consistently(func() []*models.Task {
					return getTasksByState(bbsClient, models.Task_Completed)
				}, 0.5).Should(BeEmpty())

				startConverger()

				Eventually(func() []*models.Task {
					return getTasksByState(bbsClient, models.Task_Completed)
				}, 10*convergeRepeatInterval).Should(HaveLen(1))
			})
		})
	})

	Context("when the converger loses the lock", func() {
		BeforeEach(func() {
			startConverger()
			Eventually(runner, 5*time.Second).Should(gbytes.Say("acquire-lock-succeeded"))

			consulRunner.DestroySession("converger")
		})

		It("exits with an error", func() {
			Eventually(runner, exitDuration).Should(Exit(1))
		})
	})

	Context("when the converger initially does not have the lock", func() {
		var otherSession *consuladapter.Session

		BeforeEach(func() {
			otherSession = consulRunner.NewSession("other-session")
			err := otherSession.AcquireLock(locket.LockSchemaPath("converge_lock"), []byte("something-else"))
			Expect(err).NotTo(HaveOccurred())

			startConverger()
		})

		itIsInactive()

		Describe("when the lock becomes available", func() {
			BeforeEach(func() {
				otherSession.Destroy()
				time.Sleep(convergeRepeatInterval + 10*time.Millisecond)
			})

			Describe("when a running task with a dead cell is present", func() {
				JustBeforeEach(createRunningTaskWithDeadCell)

				It("eventually marks the task as failed", func() {
					Eventually(func() []*models.Task {
						completedTasks := getTasksByState(bbsClient, models.Task_Completed)
						return failedTasks(completedTasks)
					}, 10*convergeRepeatInterval).Should(HaveLen(1))
				})
			})
		})
	})

	Describe("signal handling", func() {
		BeforeEach(func() {
			startConverger()
		})

		Describe("when it receives SIGINT", func() {
			It("exits successfully", func() {
				convergerProcess.Signal(syscall.SIGINT)
				Eventually(runner, exitDuration).Should(Exit(0))
			})
		})

		Describe("when it receives SIGTERM", func() {
			It("exits successfully", func() {
				convergerProcess.Signal(syscall.SIGTERM)
				Eventually(runner, exitDuration).Should(Exit(0))
			})
		})
	})

	Context("when etcd is down", func() {
		BeforeEach(func() {
			etcdRunner.Stop()
			startConverger()
		})

		It("starts", func() {
			Consistently(runner).ShouldNot(Exit())
		})
	})
})

func getTasksByState(client bbs.Client, state models.Task_State) []*models.Task {
	tasks, err := client.Tasks()
	Expect(err).NotTo(HaveOccurred())

	filteredTasks := make([]*models.Task, 0)
	for _, task := range tasks {
		if task.State == state {
			filteredTasks = append(filteredTasks, task)
		}
	}
	return filteredTasks
}

func failedTasks(tasks []*models.Task) []*models.Task {
	failedTasks := make([]*models.Task, 0)

	for _, task := range tasks {
		if task.Failed {
			failedTasks = append(failedTasks, task)
		}
	}

	return failedTasks
}
<commit_msg>Remove references to runtime-schema/models<commit_after>package main_test

import (
	"encoding/json"
	"fmt"
	"net/url"
	"syscall"
	"time"

	"github.com/cloudfoundry/storeadapter"
	"github.com/cloudfoundry/storeadapter/storerunner/etcdstorerunner"
	. "github.com/onsi/ginkgo"
	"github.com/onsi/ginkgo/config"
	. "github.com/onsi/gomega"
	"github.com/onsi/gomega/gbytes"
	. "github.com/onsi/gomega/gexec"
	"github.com/pivotal-golang/lager"
	"github.com/pivotal-golang/lager/lagertest"
	"github.com/tedsuo/ifrit"
	"github.com/tedsuo/ifrit/ginkgomon"

	"github.com/cloudfoundry-incubator/bbs"
	bbsrunner "github.com/cloudfoundry-incubator/bbs/cmd/bbs/testrunner"
	"github.com/cloudfoundry-incubator/bbs/models"
	"github.com/cloudfoundry-incubator/bbs/models/test/model_helpers"
	"github.com/cloudfoundry-incubator/consuladapter"
	"github.com/cloudfoundry-incubator/consuladapter/consulrunner"
	convergerrunner "github.com/cloudfoundry-incubator/converger/cmd/converger/testrunner"
	"github.com/cloudfoundry-incubator/locket"
	"github.com/cloudfoundry-incubator/locket/presence"
)

type BinPaths struct {
	Converger string
	Bbs       string
}

var _ = Describe("Converger", func() {
	const (
		exitDuration                = 4 * time.Second
		convergeRepeatInterval      = 500 * time.Millisecond
		taskKickInterval            = convergeRepeatInterval
		expireCompletedTaskDuration = 3 * convergeRepeatInterval
		expirePendingTaskDuration   = 30 * time.Minute
	)

	var (
		binPaths         BinPaths
		etcdRunner       *etcdstorerunner.ETCDClusterRunner
		bbsArgs          bbsrunner.Args
		bbsProcess       ifrit.Process
		bbsClient        bbs.Client
		convergerConfig  *convergerrunner.Config
		convergerProcess ifrit.Process
		runner           *ginkgomon.Runner

		consulRunner  *consulrunner.ClusterRunner
		consulSession *consuladapter.Session

		etcdClient storeadapter.StoreAdapter

		logger lager.Logger
	)

	SynchronizedBeforeSuite(func() []byte {
		convergerBinPath, err := Build("github.com/cloudfoundry-incubator/converger/cmd/converger", "-race")
		Expect(err).NotTo(HaveOccurred())
		bbsBinPath, err := Build("github.com/cloudfoundry-incubator/bbs/cmd/bbs", "-race")
		Expect(err).NotTo(HaveOccurred())
		bytes, err := json.Marshal(BinPaths{
			Converger: convergerBinPath,
			Bbs:       bbsBinPath,
		})
		Expect(err).NotTo(HaveOccurred())
		return bytes
	}, func(bytes []byte) {
		binPaths = BinPaths{}
		err := json.Unmarshal(bytes, &binPaths)
		Expect(err).NotTo(HaveOccurred())

		etcdPort := 5001 + config.GinkgoConfig.ParallelNode
		etcdCluster := fmt.Sprintf("http://127.0.0.1:%d", etcdPort)
		etcdRunner = etcdstorerunner.NewETCDClusterRunner(etcdPort, 1, nil)

		etcdClient = etcdRunner.Adapter(nil)

		consulRunner = consulrunner.NewClusterRunner(
			9001+config.GinkgoConfig.ParallelNode*consulrunner.PortOffsetLength,
			1,
			"http",
		)

		logger = lagertest.NewTestLogger("test")

		bbsAddress := fmt.Sprintf("127.0.0.1:%d", 13000+GinkgoParallelNode())

		bbsURL := &url.URL{
			Scheme: "http",
			Host:   bbsAddress,
		}

		bbsArgs = bbsrunner.Args{
			Address:           bbsAddress,
			AdvertiseURL:      bbsURL.String(),
			AuctioneerAddress: "some-address",
			EtcdCluster:       etcdCluster,
			ConsulCluster:     consulRunner.ConsulCluster(),
		}

		convergerConfig = &convergerrunner.Config{
			BinPath:                     binPaths.Converger,
			ConvergeRepeatInterval:      convergeRepeatInterval.String(),
			KickTaskDuration:            taskKickInterval.String(),
			ExpirePendingTaskDuration:   expirePendingTaskDuration.String(),
			ExpireCompletedTaskDuration: expireCompletedTaskDuration.String(),
			ConsulCluster:               consulRunner.ConsulCluster(),
			LogLevel:                    "info",
			BBSAddress:                  bbsURL.String(),
		}
	})

	SynchronizedAfterSuite(func() {
	}, func() {
		CleanupBuildArtifacts()
	})

	BeforeEach(func() {
		etcdRunner.Start()
		consulRunner.Start()
		consulRunner.WaitUntilReady()

		bbsProcess = ginkgomon.Invoke(bbsrunner.New(binPaths.Bbs, bbsArgs))
		bbsClient = bbs.NewClient(fmt.Sprint("http://", bbsArgs.Address))

		consulSession = consulRunner.NewSession("a-session")

		capacity := presence.NewCellCapacity(512, 1024, 124)
		cellPresence := presence.NewCellPresence("the-cell-id", "1.2.3.4", "the-zone", capacity, []string{}, []string{})

		value, err := presence.ToJSON(cellPresence)
		Expect(err).NotTo(HaveOccurred())

		_, err = consulSession.SetPresence(locket.CellSchemaPath(cellPresence.CellID), value)
		Expect(err).NotTo(HaveOccurred())

	})

	AfterEach(func() {
		ginkgomon.Kill(bbsProcess)
		ginkgomon.Kill(convergerProcess)
		consulRunner.Stop()
		etcdRunner.Stop()
	})

	startConverger := func() {
		runner = convergerrunner.New(convergerConfig)
		convergerProcess = ginkgomon.Invoke(runner)
		time.Sleep(convergeRepeatInterval)
	}

	createRunningTaskWithDeadCell := func() {
		task := model_helpers.NewValidTask("task-guid")

		err := bbsClient.DesireTask(task.TaskGuid, task.Domain, task.TaskDefinition)
		Expect(err).NotTo(HaveOccurred())

		_, err = bbsClient.StartTask(task.TaskGuid, "dead-cell")
		Expect(err).NotTo(HaveOccurred())
	}

	itIsInactive := func() {
		Describe("when a task is desired but its cell is dead", func() {
			JustBeforeEach(createRunningTaskWithDeadCell)

			It("does not converge the task", func() {
				Consistently(func() []*models.Task {
					return getTasksByState(bbsClient, models.Task_Completed)
				}, 10*convergeRepeatInterval).Should(BeEmpty())
			})
		})
	}

	Context("when the converger has the lock", func() {
		Describe("when a task is desired but its cell is dead", func() {
			JustBeforeEach(createRunningTaskWithDeadCell)

			It("marks the task as completed and failed", func() {
				Consistently(func() []*models.Task {
					return getTasksByState(bbsClient, models.Task_Completed)
				}, 0.5).Should(BeEmpty())

				startConverger()

				Eventually(func() []*models.Task {
					return getTasksByState(bbsClient, models.Task_Completed)
				}, 10*convergeRepeatInterval).Should(HaveLen(1))
			})
		})
	})

	Context("when the converger loses the lock", func() {
		BeforeEach(func() {
			startConverger()
			Eventually(runner, 5*time.Second).Should(gbytes.Say("acquire-lock-succeeded"))

			consulRunner.DestroySession("converger")
		})

		It("exits with an error", func() {
			Eventually(runner, exitDuration).Should(Exit(1))
		})
	})

	Context("when the converger initially does not have the lock", func() {
		var otherSession *consuladapter.Session

		BeforeEach(func() {
			otherSession = consulRunner.NewSession("other-session")
			err := otherSession.AcquireLock(locket.LockSchemaPath("converge_lock"), []byte("something-else"))
			Expect(err).NotTo(HaveOccurred())

			startConverger()
		})

		itIsInactive()

		Describe("when the lock becomes available", func() {
			BeforeEach(func() {
				otherSession.Destroy()
				time.Sleep(convergeRepeatInterval + 10*time.Millisecond)
			})

			Describe("when a running task with a dead cell is present", func() {
				JustBeforeEach(createRunningTaskWithDeadCell)

				It("eventually marks the task as failed", func() {
					Eventually(func() []*models.Task {
						completedTasks := getTasksByState(bbsClient, models.Task_Completed)
						return failedTasks(completedTasks)
					}, 10*convergeRepeatInterval).Should(HaveLen(1))
				})
			})
		})
	})

	Describe("signal handling", func() {
		BeforeEach(func() {
			startConverger()
		})

		Describe("when it receives SIGINT", func() {
			It("exits successfully", func() {
				convergerProcess.Signal(syscall.SIGINT)
				Eventually(runner, exitDuration).Should(Exit(0))
			})
		})

		Describe("when it receives SIGTERM", func() {
			It("exits successfully", func() {
				convergerProcess.Signal(syscall.SIGTERM)
				Eventually(runner, exitDuration).Should(Exit(0))
			})
		})
	})

	Context("when etcd is down", func() {
		BeforeEach(func() {
			etcdRunner.Stop()
			startConverger()
		})

		It("starts", func() {
			Consistently(runner).ShouldNot(Exit())
		})
	})
})

func getTasksByState(client bbs.Client, state models.Task_State) []*models.Task {
	tasks, err := client.Tasks()
	Expect(err).NotTo(HaveOccurred())

	filteredTasks := make([]*models.Task, 0)
	for _, task := range tasks {
		if task.State == state {
			filteredTasks = append(filteredTasks, task)
		}
	}
	return filteredTasks
}

func failedTasks(tasks []*models.Task) []*models.Task {
	failedTasks := make([]*models.Task, 0)

	for _, task := range tasks {
		if task.Failed {
			failedTasks = append(failedTasks, task)
		}
	}

	return failedTasks
}
<|endoftext|>


<commit_before>// mailbox_server - to receive open click.
/*
Usage:

	mailbox_server [flags]

The flags are:

	`-p`: Http port, default is `:8801`

*/
package main

import (
	"encoding/hex"
	"flag"
	"fmt"
	"log"
	"net/http"
	"net/url"
	"regexp"

	_ "github.com/go-sql-driver/mysql"
	"github.com/toomore/mailbox/campaign"
	"github.com/toomore/mailbox/reader"
	"github.com/toomore/mailbox/utils"
)

var (
	cExpr      = regexp.MustCompile(`/(read|door)/([0-9a-z]+)`)
	httpPort   = flag.String("p", ":8801", "Http port")
	linksCache = make(map[string]string)
)

func read(w http.ResponseWriter, r *http.Request) {
	v, _ := url.ParseQuery(r.Header.Get("X-Args"))

	var hm string
	match := cExpr.FindStringSubmatch(r.Header.Get("X-Uri"))
	if len(match) >= 3 {
		hm = match[2]
	}
	w.WriteHeader(http.StatusNotFound)
	hmbyte, _ := hex.DecodeString(hm)
	if campaign.CheckMac(hmbyte, v.Get("c"), v) {
		log.Println("[read] Pass")
		reader.Save(v.Get("c"), v.Get("u"), r.Header.Get("X-Real-Ip"), r.Header.Get("User-Agent"))
	} else {
		log.Println("[read] Hash Fail!!!")
	}
	log.Printf("%+v", r)
	log.Println(r.Header.Get("X-Real-Ip"))
	log.Println(r.Header.Get("User-Agent"))
}

func door(w http.ResponseWriter, r *http.Request) {
	v, _ := url.ParseQuery(r.Header.Get("X-Args"))
	if v.Get("t") != "a" {
		log.Println("No `t`", v.Get("t"))
		return
	}

	var hm string
	match := cExpr.FindStringSubmatch(r.Header.Get("X-Uri"))
	log.Println(match)
	if len(match) >= 3 {
		hm = match[2]
	}
	hmbyte, _ := hex.DecodeString(hm)
	if campaign.CheckMac(hmbyte, v.Get("c"), v) {
		utils.GetConn().Query(`INSERT INTO doors(cid,uid,linkid,ip,agent) VALUES(?,?,?,?,?)`,
			v.Get("c"), v.Get("u"), v.Get("l"), r.Header.Get("X-Real-Ip"), r.Header.Get("User-Agent"))
		log.Println("[door] Pass")
	} else {
		log.Println("[door] Hash Fail!!!")
	}
	log.Printf("%+v", r)
	log.Println(r.Header.Get("X-Real-Ip"))
	log.Println(r.Header.Get("User-Agent"))

	linksCacheKey := fmt.Sprintf("%s%s", v.Get("c"), v.Get("l"))
	if url, ok := linksCache[linksCacheKey]; ok {
		log.Println("Using door cache", linksCacheKey)
		http.Redirect(w, r, url, http.StatusSeeOther)
		return
	}

	rows, err := utils.GetConn().Query(`SELECT url FROM links WHERE cid=? AND id=?`, v.Get("c"), v.Get("l"))
	if err == nil {
		for rows.Next() {
			var url string
			rows.Scan(&url)
			linksCache[linksCacheKey] = url
			log.Println("Find door", linksCacheKey)
			http.Redirect(w, r, url, http.StatusSeeOther)
			return
		}
	}
}

func main() {
	flag.Parse()
	http.HandleFunc("/read/", read)
	http.HandleFunc("/door/", door)
	log.Println("HTTP Port:", *httpPort)
	log.Println(http.ListenAndServe(*httpPort, nil))
}
<commit_msg>Fixed door check<commit_after>// mailbox_server - to receive open click.
/*
Usage:

	mailbox_server [flags]

The flags are:

	`-p`: Http port, default is `:8801`

*/
package main

import (
	"encoding/hex"
	"flag"
	"fmt"
	"log"
	"net/http"
	"net/url"
	"regexp"

	_ "github.com/go-sql-driver/mysql"
	"github.com/toomore/mailbox/campaign"
	"github.com/toomore/mailbox/reader"
	"github.com/toomore/mailbox/utils"
)

var (
	cExpr      = regexp.MustCompile(`/(read|door)/([0-9a-z]+)`)
	httpPort   = flag.String("p", ":8801", "Http port")
	linksCache = make(map[string]string)
)

func read(w http.ResponseWriter, r *http.Request) {
	v, _ := url.ParseQuery(r.Header.Get("X-Args"))

	var hm string
	match := cExpr.FindStringSubmatch(r.Header.Get("X-Uri"))
	if len(match) >= 3 {
		hm = match[2]
	}
	w.WriteHeader(http.StatusNotFound)
	hmbyte, _ := hex.DecodeString(hm)
	if campaign.CheckMac(hmbyte, v.Get("c"), v) {
		log.Println("[read] Pass")
		reader.Save(v.Get("c"), v.Get("u"), r.Header.Get("X-Real-Ip"), r.Header.Get("User-Agent"))
	} else {
		log.Println("[read] Hash Fail!!!")
	}
	log.Printf("%+v", r)
	log.Println(r.Header.Get("X-Real-Ip"))
	log.Println(r.Header.Get("User-Agent"))
}

func door(w http.ResponseWriter, r *http.Request) {
	v, _ := url.ParseQuery(r.Header.Get("X-Args"))
	if v.Get("t") != "a" {
		log.Println("No `t`", v.Get("t"))
		return
	}

	var hm string
	match := cExpr.FindStringSubmatch(r.Header.Get("X-Uri"))
	log.Println(match)
	if len(match) >= 3 {
		hm = match[2]
	}

	log.Printf("%+v", r)
	log.Println(r.Header.Get("X-Real-Ip"))
	log.Println(r.Header.Get("User-Agent"))

	hmbyte, _ := hex.DecodeString(hm)
	if campaign.CheckMac(hmbyte, v.Get("c"), v) {
		utils.GetConn().Query(`INSERT INTO doors(cid,uid,linkid,ip,agent) VALUES(?,?,?,?,?)`,
			v.Get("c"), v.Get("u"), v.Get("l"), r.Header.Get("X-Real-Ip"), r.Header.Get("User-Agent"))
		log.Println("[door] Pass")

		linksCacheKey := fmt.Sprintf("%s%s", v.Get("c"), v.Get("l"))
		if url, ok := linksCache[linksCacheKey]; ok {
			log.Println("Using door cache", linksCacheKey)
			http.Redirect(w, r, url, http.StatusSeeOther)
			return
		}

		rows, err := utils.GetConn().Query(`SELECT url FROM links WHERE cid=? AND id=?`, v.Get("c"), v.Get("l"))
		if err == nil {
			for rows.Next() {
				var url string
				rows.Scan(&url)
				linksCache[linksCacheKey] = url
				log.Println("Find door", linksCacheKey)
				http.Redirect(w, r, url, http.StatusSeeOther)
				return
			}
		}
	} else {
		log.Println("[door] Hash Fail!!!")
	}
	w.WriteHeader(http.StatusNotFound)
}

func main() {
	flag.Parse()
	http.HandleFunc("/read/", read)
	http.HandleFunc("/door/", door)
	log.Println("HTTP Port:", *httpPort)
	log.Println(http.ListenAndServe(*httpPort, nil))
}
<|endoftext|>


<commit_before>// Copyright 2016 Lars Wiegman. All rights reserved. Use of this source code is
// governed by a BSD-style license that can be found in the LICENSE file.

package main

import (
	"errors"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"runtime"
	"strconv"
	"strings"

	lumberjack "gopkg.in/natefinch/lumberjack.v2"

	"github.com/xenolf/lego/acme"

	"github.com/mholt/caddy"
	// plug in the HTTP server type
	_ "github.com/mholt/caddy/caddyhttp"
	"github.com/mholt/caddy/caddytls"

	// This is where other plugins get plugged in (imported)
	_ "github.com/mholt/caddy/caddyhttp/proxy"
	_ "github.com/namsral/multipass/caddy"
)

func init() {
	caddy.TrapSignals()
	setVersion()

	flag.BoolVar(&caddytls.Agreed, "agree", false, "Agree to the CA's Subscriber Agreement")
	flag.StringVar(&caddytls.DefaultCAUrl, "ca", "https://acme-v01.api.letsencrypt.org/directory", "URL to certificate authority's ACME server directory")
	flag.StringVar(&conf, "conf", "", "Caddyfile to load (default \""+caddy.DefaultConfigFile+"\")")
	flag.StringVar(&cpu, "cpu", "100%", "CPU cap")
	flag.BoolVar(&plugins, "plugins", false, "List installed plugins")
	flag.StringVar(&caddytls.DefaultEmail, "email", "", "Default ACME CA account email address")
	flag.StringVar(&logfile, "log", "stdout", "Process log file")
	flag.StringVar(&caddy.PidFile, "pidfile", "", "Path to write pid file")
	flag.BoolVar(&caddy.Quiet, "quiet", false, "Quiet mode (no initialization output)")
	flag.StringVar(&revoke, "revoke", "", "Hostname for which to revoke the certificate")
	flag.StringVar(&serverType, "type", "http", "Type of server to run")
	flag.BoolVar(&version, "version", false, "Show version")

	caddy.RegisterCaddyfileLoader("flag", caddy.LoaderFunc(confLoader))
	caddy.SetDefaultCaddyfileLoader("default", caddy.LoaderFunc(defaultLoader))
}

// Run is Caddy's main() function.
func main() {
	flag.Parse()

	caddy.AppName = appName
	caddy.AppVersion = appVersion
	acme.UserAgent = appName + "/" + appVersion

	// Set up process log before anything bad happens
	switch logfile {
	case "stdout":
		log.SetOutput(os.Stdout)
	case "stderr":
		log.SetOutput(os.Stderr)
	case "":
		log.SetOutput(ioutil.Discard)
	default:
		log.SetOutput(&lumberjack.Logger{
			Filename:   logfile,
			MaxSize:    100,
			MaxAge:     14,
			MaxBackups: 10,
		})
	}

	// Check for one-time actions
	if revoke != "" {
		err := caddytls.Revoke(revoke)
		if err != nil {
			log.Fatal(err)
		}
		fmt.Printf("Revoked certificate for %s\n", revoke)
		os.Exit(0)
	}
	if version {
		fmt.Printf("%s %s\n", appName, appVersion)
		if devBuild && gitShortStat != "" {
			fmt.Printf("%s\n%s\n", gitShortStat, gitFilesModified)
		}
		os.Exit(0)
	}
	if plugins {
		fmt.Println(caddy.DescribePlugins())
		os.Exit(0)
	}

	// Set CPU cap
	err := setCPU(cpu)
	if err != nil {
		mustLogFatal(err)
	}

	// Get Caddyfile input
	caddyfile, err := caddy.LoadCaddyfile(serverType)
	if err != nil {
		mustLogFatal(err)
	}

	// Start your engines
	instance, err := caddy.Start(caddyfile)
	if err != nil {
		mustLogFatal(err)
	}

	// Twiddle your thumbs
	instance.Wait()
}

// mustLogFatal wraps log.Fatal() in a way that ensures the
// output is always printed to stderr so the user can see it
// if the user is still there, even if the process log was not
// enabled. If this process is an upgrade, however, and the user
// might not be there anymore, this just logs to the process
// log and exits.
func mustLogFatal(args ...interface{}) {
	if !caddy.IsUpgrade() {
		log.SetOutput(os.Stderr)
	}
	log.Fatal(args...)
}

// confLoader loads the Caddyfile using the -conf flag.
func confLoader(serverType string) (caddy.Input, error) {
	if conf == "" {
		return nil, nil
	}

	if conf == "stdin" {
		return caddy.CaddyfileFromPipe(os.Stdin)
	}

	contents, err := ioutil.ReadFile(conf)
	if err != nil {
		return nil, err
	}
	return caddy.CaddyfileInput{
		Contents:       contents,
		Filepath:       conf,
		ServerTypeName: serverType,
	}, nil
}

// defaultLoader loads the Caddyfile from the current working directory.
func defaultLoader(serverType string) (caddy.Input, error) {
	contents, err := ioutil.ReadFile(caddy.DefaultConfigFile)
	if err != nil {
		if os.IsNotExist(err) {
			return nil, nil
		}
		return nil, err
	}
	return caddy.CaddyfileInput{
		Contents:       contents,
		Filepath:       caddy.DefaultConfigFile,
		ServerTypeName: serverType,
	}, nil
}

// setVersion figures out the version information
// based on variables set by -ldflags.
func setVersion() {
	// A development build is one that's not at a tag or has uncommitted changes
	devBuild = gitTag == "" || gitShortStat != ""

	// Only set the appVersion if -ldflags was used
	if gitNearestTag != "" || gitTag != "" {
		if devBuild && gitNearestTag != "" {
			appVersion = fmt.Sprintf("%s (+%s %s)",
				strings.TrimPrefix(gitNearestTag, "v"), gitCommit, buildDate)
		} else if gitTag != "" {
			appVersion = strings.TrimPrefix(gitTag, "v")
		}
	}
}

// setCPU parses string cpu and sets GOMAXPROCS
// according to its value. It accepts either
// a number (e.g. 3) or a percent (e.g. 50%).
func setCPU(cpu string) error {
	var numCPU int

	availCPU := runtime.NumCPU()

	if strings.HasSuffix(cpu, "%") {
		// Percent
		var percent float32
		pctStr := cpu[:len(cpu)-1]
		pctInt, err := strconv.Atoi(pctStr)
		if err != nil || pctInt < 1 || pctInt > 100 {
			return errors.New("invalid CPU value: percentage must be between 1-100")
		}
		percent = float32(pctInt) / 100
		numCPU = int(float32(availCPU) * percent)
	} else {
		// Number
		num, err := strconv.Atoi(cpu)
		if err != nil || num < 1 {
			return errors.New("invalid CPU value: provide a number or percent greater than 0")
		}
		numCPU = num
	}

	if numCPU > availCPU {
		numCPU = availCPU
	}

	runtime.GOMAXPROCS(numCPU)
	return nil
}

const appName = "Multipass"

// Flags that control program flow or startup
var (
	serverType string
	conf       string
	cpu        string
	logfile    string
	revoke     string
	version    bool
	plugins    bool
)

// Build information obtained with the help of -ldflags
var (
	appVersion = "(untracked dev build)" // inferred at startup
	devBuild   = true                    // inferred at startup

	buildDate        string // date -u
	gitTag           string // git describe --exact-match HEAD 2> /dev/null
	gitNearestTag    string // git describe --abbrev=0 --tags HEAD
	gitCommit        string // git rev-parse HEAD
	gitShortStat     string // git diff-index --shortstat
	gitFilesModified string // git diff-index --name-only HEAD
)
<commit_msg>Support Caddy's CaddyFile server type association<commit_after>// Copyright 2016 Lars Wiegman. All rights reserved. Use of this source code is
// governed by a BSD-style license that can be found in the LICENSE file.

package main

import (
	"errors"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"runtime"
	"strconv"
	"strings"

	lumberjack "gopkg.in/natefinch/lumberjack.v2"

	"github.com/xenolf/lego/acme"

	"github.com/mholt/caddy"
	// plug in the HTTP server type
	_ "github.com/mholt/caddy/caddyhttp"
	"github.com/mholt/caddy/caddytls"

	// This is where other plugins get plugged in (imported)
	_ "github.com/mholt/caddy/caddyhttp/proxy"
	_ "github.com/namsral/multipass/caddy"
)

func init() {
	caddy.TrapSignals()
	setVersion()

	flag.BoolVar(&caddytls.Agreed, "agree", false, "Agree to the CA's Subscriber Agreement")
	flag.StringVar(&caddytls.DefaultCAUrl, "ca", "https://acme-v01.api.letsencrypt.org/directory", "URL to certificate authority's ACME server directory")
	flag.StringVar(&conf, "conf", "", "Caddyfile to load (default \""+caddy.DefaultConfigFile+"\")")
	flag.StringVar(&cpu, "cpu", "100%", "CPU cap")
	flag.BoolVar(&plugins, "plugins", false, "List installed plugins")
	flag.StringVar(&caddytls.DefaultEmail, "email", "", "Default ACME CA account email address")
	flag.StringVar(&logfile, "log", "stdout", "Process log file")
	flag.StringVar(&caddy.PidFile, "pidfile", "", "Path to write pid file")
	flag.BoolVar(&caddy.Quiet, "quiet", false, "Quiet mode (no initialization output)")
	flag.StringVar(&revoke, "revoke", "", "Hostname for which to revoke the certificate")
	flag.StringVar(&serverType, "type", "http", "Type of server to run")
	flag.BoolVar(&version, "version", false, "Show version")

	caddy.RegisterCaddyfileLoader("flag", caddy.LoaderFunc(confLoader))
	caddy.SetDefaultCaddyfileLoader("default", caddy.LoaderFunc(defaultLoader))
}

// Run is Caddy's main() function.
func main() {
	flag.Parse()

	caddy.AppName = appName
	caddy.AppVersion = appVersion
	acme.UserAgent = appName + "/" + appVersion

	// Set up process log before anything bad happens
	switch logfile {
	case "stdout":
		log.SetOutput(os.Stdout)
	case "stderr":
		log.SetOutput(os.Stderr)
	case "":
		log.SetOutput(ioutil.Discard)
	default:
		log.SetOutput(&lumberjack.Logger{
			Filename:   logfile,
			MaxSize:    100,
			MaxAge:     14,
			MaxBackups: 10,
		})
	}

	// Check for one-time actions
	if revoke != "" {
		err := caddytls.Revoke(revoke)
		if err != nil {
			log.Fatal(err)
		}
		fmt.Printf("Revoked certificate for %s\n", revoke)
		os.Exit(0)
	}
	if version {
		fmt.Printf("%s %s\n", appName, appVersion)
		if devBuild && gitShortStat != "" {
			fmt.Printf("%s\n%s\n", gitShortStat, gitFilesModified)
		}
		os.Exit(0)
	}
	if plugins {
		fmt.Println(caddy.DescribePlugins())
		os.Exit(0)
	}

	// Set CPU cap
	err := setCPU(cpu)
	if err != nil {
		mustLogFatal(err)
	}

	// Get Caddyfile input
	caddyfile, err := caddy.LoadCaddyfile(serverType)
	if err != nil {
		mustLogFatal(err)
	}

	// Start your engines
	instance, err := caddy.Start(caddyfile)
	if err != nil {
		mustLogFatal(err)
	}

	// Twiddle your thumbs
	instance.Wait()
}

// mustLogFatal wraps log.Fatal() in a way that ensures the
// output is always printed to stderr so the user can see it
// if the user is still there, even if the process log was not
// enabled. If this process is an upgrade, however, and the user
// might not be there anymore, this just logs to the process
// log and exits.
func mustLogFatal(args ...interface{}) {
	if !caddy.IsUpgrade() {
		log.SetOutput(os.Stderr)
	}
	log.Fatal(args...)
}

// confLoader loads the Caddyfile using the -conf flag.
func confLoader(serverType string) (caddy.Input, error) {
	if conf == "" {
		return nil, nil
	}

	if conf == "stdin" {
		return caddy.CaddyfileFromPipe(os.Stdin, serverType)
	}

	contents, err := ioutil.ReadFile(conf)
	if err != nil {
		return nil, err
	}
	return caddy.CaddyfileInput{
		Contents:       contents,
		Filepath:       conf,
		ServerTypeName: serverType,
	}, nil
}

// defaultLoader loads the Caddyfile from the current working directory.
func defaultLoader(serverType string) (caddy.Input, error) {
	contents, err := ioutil.ReadFile(caddy.DefaultConfigFile)
	if err != nil {
		if os.IsNotExist(err) {
			return nil, nil
		}
		return nil, err
	}
	return caddy.CaddyfileInput{
		Contents:       contents,
		Filepath:       caddy.DefaultConfigFile,
		ServerTypeName: serverType,
	}, nil
}

// setVersion figures out the version information
// based on variables set by -ldflags.
func setVersion() {
	// A development build is one that's not at a tag or has uncommitted changes
	devBuild = gitTag == "" || gitShortStat != ""

	// Only set the appVersion if -ldflags was used
	if gitNearestTag != "" || gitTag != "" {
		if devBuild && gitNearestTag != "" {
			appVersion = fmt.Sprintf("%s (+%s %s)",
				strings.TrimPrefix(gitNearestTag, "v"), gitCommit, buildDate)
		} else if gitTag != "" {
			appVersion = strings.TrimPrefix(gitTag, "v")
		}
	}
}

// setCPU parses string cpu and sets GOMAXPROCS
// according to its value. It accepts either
// a number (e.g. 3) or a percent (e.g. 50%).
func setCPU(cpu string) error {
	var numCPU int

	availCPU := runtime.NumCPU()

	if strings.HasSuffix(cpu, "%") {
		// Percent
		var percent float32
		pctStr := cpu[:len(cpu)-1]
		pctInt, err := strconv.Atoi(pctStr)
		if err != nil || pctInt < 1 || pctInt > 100 {
			return errors.New("invalid CPU value: percentage must be between 1-100")
		}
		percent = float32(pctInt) / 100
		numCPU = int(float32(availCPU) * percent)
	} else {
		// Number
		num, err := strconv.Atoi(cpu)
		if err != nil || num < 1 {
			return errors.New("invalid CPU value: provide a number or percent greater than 0")
		}
		numCPU = num
	}

	if numCPU > availCPU {
		numCPU = availCPU
	}

	runtime.GOMAXPROCS(numCPU)
	return nil
}

const appName = "Multipass"

// Flags that control program flow or startup
var (
	serverType string
	conf       string
	cpu        string
	logfile    string
	revoke     string
	version    bool
	plugins    bool
)

// Build information obtained with the help of -ldflags
var (
	appVersion = "(untracked dev build)" // inferred at startup
	devBuild   = true                    // inferred at startup

	buildDate        string // date -u
	gitTag           string // git describe --exact-match HEAD 2> /dev/null
	gitNearestTag    string // git describe --abbrev=0 --tags HEAD
	gitCommit        string // git rev-parse HEAD
	gitShortStat     string // git diff-index --shortstat
	gitFilesModified string // git diff-index --name-only HEAD
)
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"os"
	"strings"
	"syscall"

	"github.com/concourse/docker-image-resource/cmd/print-metadata/passwd"
)

type imageMetadata struct {
	User string   `json:"user"`
	Env  []string `json:"env"`
}

var blacklistedEnv = map[string]bool{
	"HOSTNAME": true,
}

func main() {
	err := json.NewEncoder(os.Stdout).Encode(imageMetadata{
		User: username(),
		Env:  env(),
	})
	if err != nil {
		panic(err)
	}
}

func username() string {
	users, err := passwd.ReadUsers("/etc/passwd")
	if err != nil {
		panic(err)
	}

	name, found := users.NameForID(syscall.Getuid())
	if !found {
		panic("could not find user in /etc/passwd")
	}

	return name
}

func env() []string {
	var envVars []string
	for _, e := range os.Environ() {
		parts := strings.SplitN(e, "=", 2)
		name := parts[0]

		if !blacklistedEnv[name] {
			envVars = append(envVars, e)
		}
	}

	return envVars
}
<commit_msg>Handle when /etc/passwd is missing because the image is built using scratch or another base layer that does not provide that file.<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"os"
	"strings"
	"syscall"

	"github.com/concourse/docker-image-resource/cmd/print-metadata/passwd"
)

type imageMetadata struct {
	User string   `json:"user,omitempty"`
	Env  []string `json:"env"`
}

var blacklistedEnv = map[string]bool{
	"HOSTNAME": true,
}

func main() {
	username, err := username()
	if err != nil {
		fmt.Fprintf(os.Stderr, "Unable to determine username, will not be included in metadata")
	}

	err = json.NewEncoder(os.Stdout).Encode(imageMetadata{
		User: username,
		Env:  env(),
	})
	if err != nil {
		panic(err)
	}
}

func username() (string, error) {
	users, err := passwd.ReadUsers("/etc/passwd")
	if err != nil {
		return "", err
	}

	name, found := users.NameForID(syscall.Getuid())
	if !found {
		return "", fmt.Errorf("could not find user in /etc/passwd")
	}

	return name, nil
}

func env() []string {
	var envVars []string
	for _, e := range os.Environ() {
		parts := strings.SplitN(e, "=", 2)
		name := parts[0]

		if !blacklistedEnv[name] {
			envVars = append(envVars, e)
		}
	}

	return envVars
}
<|endoftext|>


<commit_before>package main

import (
	"github.com/ligato/cn-infra/datasync"
	"github.com/ligato/cn-infra/datasync/kvdbsync"
	"github.com/ligato/cn-infra/datasync/kvdbsync/local"
	"github.com/ligato/cn-infra/datasync/msgsync"
	"github.com/ligato/cn-infra/datasync/resync"
	"github.com/ligato/cn-infra/db/keyval/etcd"
	"github.com/ligato/cn-infra/rpc/rest"
	"github.com/ligato/vpp-agent/plugins/govppmux"
	"github.com/ligato/vpp-agent/plugins/linux"
	"github.com/ligato/vpp-agent/plugins/telemetry"
	"github.com/ligato/vpp-agent/plugins/vpp"
	"github.com/ligato/vpp-agent/plugins/vpp/rpc"
)

type VPPAgent struct {
	GoVPP *govppmux.Plugin
	Linux *linux.Plugin
	VPP   *vpp.Plugin

	IfStatePub      *msgsync.Plugin
	GRPCSvcPlugin   *rpc.GRPCSvcPlugin
	RESTAPIPlugin   *rest.Plugin
	TelemetryPlugin *telemetry.Plugin
}

func NewVppAgent() *VPPAgent {
	etcdDataSync := kvdbsync.NewPlugin(
		kvdbsync.UseDeps(func(deps *kvdbsync.Deps) {
			deps.KvPlugin = &etcd.DefaultPlugin
			deps.ResyncOrch = &resync.DefaultPlugin
		}),
	)
	watcher := &datasync.CompositeKVProtoWatcher{
		Adapters: []datasync.KeyValProtoWatcher{
			local.Get(),
			etcdDataSync,
		}}

	/*govpp := govppmux.NewPlugin(
		govppmux.UseDeps(govppmux.Deps{
			Resync: resync.DefaultPlugin,
		}),
	)*/
	govppPlugin := &govppmux.DefaultPlugin

	var linuxAPI vpp.LinuxpluginAPI
	vppPlugin := vpp.NewPlugin(
		vpp.UseDeps(func(deps *vpp.Deps) {
			deps.Linux = linuxAPI
			deps.GoVppmux = govppPlugin
			deps.Publish = etcdDataSync
			deps.Watch = watcher
			deps.DataSyncs = map[string]datasync.KeyProtoValWriter{
				"etcd": etcdDataSync,
			}
		}),
	)

	linuxPlugin := linux.NewPlugin(
		linux.UseDeps(func(deps *linux.Deps) {
			deps.VPP = vppPlugin
			deps.Watcher = watcher
		}),
	)
	linuxAPI = linuxPlugin

	return &VPPAgent{
		GoVPP: govppPlugin,
		Linux: linuxPlugin,
		VPP:   vppPlugin,
	}
}

func (VPPAgent) String() string {
	return "vpp-agent"
}

func (VPPAgent) Init() error {
	return nil
}

func (VPPAgent) AfterInit() error {
	// Manually run resync at the very end
	resync.DefaultPlugin.DoResync()
	return nil
}

func (VPPAgent) Close() error {
	return nil
}
<commit_msg>Add logmanager and msgsync to VPPAgent plugin and fix linux dep<commit_after>package main

import (
	"sync"

	"github.com/ligato/cn-infra/datasync"
	"github.com/ligato/cn-infra/datasync/kvdbsync"
	"github.com/ligato/cn-infra/datasync/kvdbsync/local"
	"github.com/ligato/cn-infra/datasync/msgsync"
	"github.com/ligato/cn-infra/datasync/resync"
	"github.com/ligato/cn-infra/db/keyval/etcd"
	"github.com/ligato/cn-infra/logging/logmanager"
	"github.com/ligato/cn-infra/messaging/kafka"
	"github.com/ligato/cn-infra/rpc/rest"
	"github.com/ligato/vpp-agent/plugins/govppmux"
	"github.com/ligato/vpp-agent/plugins/linux"
	"github.com/ligato/vpp-agent/plugins/telemetry"
	"github.com/ligato/vpp-agent/plugins/vpp"
	"github.com/ligato/vpp-agent/plugins/vpp/rpc"
)

type VPPAgent struct {
	LogManager *logmanager.Plugin

	GoVPP *govppmux.Plugin
	Linux *linux.Plugin
	VPP   *vpp.Plugin

	IfStatePub *msgsync.Plugin

	GRPCSvcPlugin   *rpc.GRPCSvcPlugin
	RESTAPIPlugin   *rest.Plugin
	TelemetryPlugin *telemetry.Plugin
}

func NewVppAgent() *VPPAgent {
	a := &VPPAgent{
		LogManager: &logmanager.DefaultPlugin,
		GoVPP:      &govppmux.DefaultPlugin,
	}

	ifStatePub := msgsync.NewPlugin(
		msgsync.UseDeps(func(deps *msgsync.Deps) {
			deps.Messaging = &kafka.DefaultPlugin
		}), msgsync.UseConf(msgsync.Cfg{
			Topic: "if_state",
		}),
	)
	a.IfStatePub = ifStatePub

	dataSync := kvdbsync.NewPlugin(kvdbsync.UseDeps(func(deps *kvdbsync.Deps) {
		deps.KvPlugin = &etcd.DefaultPlugin
		deps.ResyncOrch = &resync.DefaultPlugin
	}),
	)
	watcher := &datasync.CompositeKVProtoWatcher{Adapters: []datasync.KeyValProtoWatcher{
		local.Get(),
		dataSync,
	}}

	var watchEventsMutex sync.Mutex
	a.VPP = vpp.NewPlugin(vpp.UseDeps(func(deps *vpp.Deps) {
		deps.GoVppmux = a.GoVPP
		deps.Publish = dataSync
		deps.Watch = watcher
		deps.DataSyncs = map[string]datasync.KeyProtoValWriter{
			"etcd": dataSync,
		}
		deps.WatchEventsMutex = &watchEventsMutex
		deps.IfStatePub = a.IfStatePub
	}))
	a.Linux = linux.NewPlugin(linux.UseDeps(func(deps *linux.Deps) {
		deps.VPP = a.VPP
		deps.Watcher = watcher
		deps.WatchEventsMutex = &watchEventsMutex
	}))
	a.VPP.Deps.Linux = a.Linux

	return a
}

func (VPPAgent) String() string {
	return "VPPAgent"
}

func (VPPAgent) Init() error {
	return nil
}

func (VPPAgent) AfterInit() error {
	// manually start resync after all plugins started
	resync.DefaultPlugin.DoResync()
	return nil
}

func (VPPAgent) Close() error {
	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"crypto/rand"
	"encoding/binary"
	"flag"
	"fmt"
	"os"
	"path/filepath"
	"runtime"

	"code.google.com/p/go.crypto/scrypt"
)

func main() {
	stateFile := flag.String("state-file", "", "File in which to save persistent state")
	pandaScrypt := flag.Bool("panda-scrypt", false, "Run in subprocess mode to process passphrase")
	cliFlag := flag.Bool("cli", false, "If true, the CLI will be used, even if the GUI is available")
	devFlag := flag.Bool("dev", false, "Is this a development environment?")
	flag.Parse()

	if *pandaScrypt {
		var numBytes uint32
		if err := binary.Read(os.Stdin, binary.LittleEndian, &numBytes); err != nil {
			panic(err)
		}
		if numBytes > 1024*1024 {
			panic("passphrase too large")
		}
		passphrase := make([]byte, int(numBytes))
		if _, err := os.Stdin.Read(passphrase); err != nil {
			panic(err)
		}
		data, err := scrypt.Key(passphrase, nil, 1<<17, 16, 4, 32*3)
		if err != nil {
			panic(err)
		}
		os.Stdout.Write(data)
		os.Exit(0)
	}

	dev := os.Getenv("POND") == "dev" || *devFlag
	runtime.GOMAXPROCS(4)

	if len(*stateFile) == 0 && dev {
		*stateFile = "state"
	}

	if len(*stateFile) == 0 {
		home := os.Getenv("HOME")
		if len(home) == 0 {
			fmt.Fprintf(os.Stderr, "$HOME not set. Please either export $HOME or use --state-file to set the location of the state file explicitly.\n")
			os.Exit(1)
		}
		configDir := filepath.Join(home, ".config")
		os.Mkdir(configDir, 0700)
		*stateFile = filepath.Join(configDir, "pond")
	}

	if !haveGUI || *cliFlag || len(os.Getenv("PONDCLI")) > 0 {
		client := NewCLIClient(*stateFile, rand.Reader, false /* testing */, true /* autoFetch */)
		client.disableV2Ratchet = true
		client.dev = dev
		client.Start()
	} else {
		ui := NewGTKUI()
		client := NewGUIClient(*stateFile, ui, rand.Reader, false /* testing */, true /* autoFetch */)
		client.disableV2Ratchet = true
		client.dev = dev
		client.Start()
		ui.Run()
	}
}
<commit_msg>Store statefile in Persistent directory on Tails.<commit_after>package main

import (
	"crypto/rand"
	"encoding/binary"
	"flag"
	"fmt"
	"os"
	"path/filepath"
	"runtime"

	"code.google.com/p/go.crypto/scrypt"
)

func main() {
	stateFile := flag.String("state-file", "", "File in which to save persistent state")
	pandaScrypt := flag.Bool("panda-scrypt", false, "Run in subprocess mode to process passphrase")
	cliFlag := flag.Bool("cli", false, "If true, the CLI will be used, even if the GUI is available")
	devFlag := flag.Bool("dev", false, "Is this a development environment?")
	flag.Parse()

	if *pandaScrypt {
		var numBytes uint32
		if err := binary.Read(os.Stdin, binary.LittleEndian, &numBytes); err != nil {
			panic(err)
		}
		if numBytes > 1024*1024 {
			panic("passphrase too large")
		}
		passphrase := make([]byte, int(numBytes))
		if _, err := os.Stdin.Read(passphrase); err != nil {
			panic(err)
		}
		data, err := scrypt.Key(passphrase, nil, 1<<17, 16, 4, 32*3)
		if err != nil {
			panic(err)
		}
		os.Stdout.Write(data)
		os.Exit(0)
	}

	dev := os.Getenv("POND") == "dev" || *devFlag
	runtime.GOMAXPROCS(4)

	if len(*stateFile) == 0 && dev {
		*stateFile = "state"
	}

	if len(*stateFile) == 0 {
		home := os.Getenv("HOME")
		if len(home) == 0 {
			fmt.Fprintf(os.Stderr, "$HOME not set. Please either export $HOME or use --state-file to set the location of the state file explicitly.\n")
			os.Exit(1)
		}
		persistentDir := filepath.Join(home, "Persistent")
		if stat, err := os.Lstat(persistentDir); err == nil && stat.IsDir() {
			// Looks like Tails.
			*stateFile = filepath.Join(persistentDir, ".pond")
		} else {
			configDir := filepath.Join(home, ".config")
			os.Mkdir(configDir, 0700)
			*stateFile = filepath.Join(configDir, "pond")
		}
	}

	if !haveGUI || *cliFlag || len(os.Getenv("PONDCLI")) > 0 {
		client := NewCLIClient(*stateFile, rand.Reader, false /* testing */, true /* autoFetch */)
		client.disableV2Ratchet = true
		client.dev = dev
		client.Start()
	} else {
		ui := NewGTKUI()
		client := NewGUIClient(*stateFile, ui, rand.Reader, false /* testing */, true /* autoFetch */)
		client.disableV2Ratchet = true
		client.dev = dev
		client.Start()
		ui.Run()
	}
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"fmt"
	"os"
	"path/filepath"
)

func main() {
	base := "/sys/bus/w1/devices"
	matches, err := filepath.Glob(fmt.Sprintf("%s/28*", base))
	handleErr(err)

	deviceFile := filepath.Join(matches[0], "w1_slave")

	_, err = os.Stat(deviceFile)
	handleErr(err)

	file, err := os.Open(deviceFile)
	handleErr(err)
	defer file.Close()

	scanner := bufio.NewScanner(file)
	for scanner.Scan() {
		fmt.Println("Line: ", scanner.Text())
	}

	if err := scanner.Err(); err != nil {
		panic(err)
	}
}

func handleErr(err error) {
	if err != nil {
		panic(err)
	}
}
<commit_msg>Adding a regex to match the text<commit_after>package main

import (
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"regexp"
)

func main() {
	base := "/sys/bus/w1/devices"
	matches, err := filepath.Glob(fmt.Sprintf("%s/28*", base))
	handleErr(err)

	deviceFile := filepath.Join(matches[0], "w1_slave")

	_, err = os.Stat(deviceFile)
	handleErr(err)

	buf, err := ioutil.ReadFile(deviceFile)
	handleErr(err)

	s := string(buf)

	match, _ := regexp.MatchString("t=([0-9]{5-6})", s)
	fmt.Println(match)
}

func handleErr(err error) {
	if err != nil {
		panic(err)
	}
}
<|endoftext|>


<commit_before>/*
Package cloudapi interacts with the Cloud API (http://apidocs.joyent.com/cloudapi/).

Licensed under the Mozilla Public License version 2.0

Copyright (c) Joyent Inc.
*/
package cloudapi

import (
	"net/http"
	"net/url"
	"path"

	"github.com/joyent/gocommon/client"
	jh "github.com/joyent/gocommon/http"
	"github.com/juju/loggo"
)

const (
	// DefaultAPIVersion defines the default version of the Cloud API to use
	DefaultAPIVersion = "~7.3"

	// CloudAPI URL parts
	apiKeys                    = "keys"
	apiPackages                = "packages"
	apiImages                  = "images"
	apiDatacenters             = "datacenters"
	apiMachines                = "machines"
	apiMetadata                = "metadata"
	apiSnapshots               = "snapshots"
	apiTags                    = "tags"
	apiAnalytics               = "analytics"
	apiInstrumentations        = "instrumentations"
	apiInstrumentationsValue   = "value"
	apiInstrumentationsRaw     = "raw"
	apiInstrumentationsHeatmap = "heatmap"
	apiInstrumentationsImage   = "image"
	apiInstrumentationsDetails = "details"
	apiUsage                   = "usage"
	apiAudit                   = "audit"
	apiFirewallRules           = "fwrules"
	apiFirewallRulesEnable     = "enable"
	apiFirewallRulesDisable    = "disable"
	apiNetworks                = "networks"
	apiFabricVLANs             = "fabrics/default/vlans"
	apiFabricNetworks          = "networks"
	apiNICs                    = "nics"

	// CloudAPI actions
	actionExport    = "export"
	actionStop      = "stop"
	actionStart     = "start"
	actionReboot    = "reboot"
	actionResize    = "resize"
	actionRename    = "rename"
	actionEnableFw  = "enable_firewall"
	actionDisableFw = "disable_firewall"
)

// Logger for this package
var Logger = loggo.GetLogger("gosdc.cloudapi")

// Client provides a means to access the Joyent CloudAPI
type Client struct {
	client client.Client
}

// New creates a new Client.
func New(client client.Client) *Client {
	return &Client{client}
}

// Filter represents a filter that can be applied to an API request.
type Filter struct {
	v url.Values
}

// NewFilter creates a new Filter.
func NewFilter() *Filter {
	return &Filter{make(url.Values)}
}

// Set a value for the specified filter.
func (f *Filter) Set(filter, value string) {
	f.v.Set(filter, value)
}

// Add a value for the specified filter.
func (f *Filter) Add(filter, value string) {
	f.v.Add(filter, value)
}

// request represents an API request
type request struct {
	method         string
	url            string
	filter         *Filter
	reqValue       interface{}
	reqHeader      http.Header
	resp           interface{}
	respHeader     *http.Header
	expectedStatus int
}

// Helper method to send an API request
func (c *Client) sendRequest(req request) (*jh.ResponseData, error) {
	request := jh.RequestData{
		ReqValue:   req.reqValue,
		ReqHeaders: req.reqHeader,
	}
	if req.filter != nil {
		request.Params = &req.filter.v
	}
	if req.expectedStatus == 0 {
		req.expectedStatus = http.StatusOK
	}
	respData := jh.ResponseData{
		RespValue:      req.resp,
		RespHeaders:    req.respHeader,
		ExpectedStatus: []int{req.expectedStatus},
	}
	err := c.client.SendRequest(req.method, req.url, "", &request, &respData)
	return &respData, err
}

// Helper method to create the API URL
func makeURL(parts ...string) string {
	return path.Join(parts...)
}
<commit_msg>added services to api url parts<commit_after>/*
Package cloudapi interacts with the Cloud API (http://apidocs.joyent.com/cloudapi/).

Licensed under the Mozilla Public License version 2.0

Copyright (c) Joyent Inc.
*/
package cloudapi

import (
	"net/http"
	"net/url"
	"path"

	"github.com/joyent/gocommon/client"
	jh "github.com/joyent/gocommon/http"
	"github.com/juju/loggo"
)

const (
	// DefaultAPIVersion defines the default version of the Cloud API to use
	DefaultAPIVersion = "~7.3"

	// CloudAPI URL parts
	apiKeys                    = "keys"
	apiPackages                = "packages"
	apiImages                  = "images"
	apiDatacenters             = "datacenters"
	apiMachines                = "machines"
	apiMetadata                = "metadata"
	apiSnapshots               = "snapshots"
	apiTags                    = "tags"
	apiAnalytics               = "analytics"
	apiInstrumentations        = "instrumentations"
	apiInstrumentationsValue   = "value"
	apiInstrumentationsRaw     = "raw"
	apiInstrumentationsHeatmap = "heatmap"
	apiInstrumentationsImage   = "image"
	apiInstrumentationsDetails = "details"
	apiUsage                   = "usage"
	apiAudit                   = "audit"
	apiFirewallRules           = "fwrules"
	apiFirewallRulesEnable     = "enable"
	apiFirewallRulesDisable    = "disable"
	apiNetworks                = "networks"
	apiFabricVLANs             = "fabrics/default/vlans"
	apiFabricNetworks          = "networks"
	apiNICs                    = "nics"
	apiServices                = "services"

	// CloudAPI actions
	actionExport    = "export"
	actionStop      = "stop"
	actionStart     = "start"
	actionReboot    = "reboot"
	actionResize    = "resize"
	actionRename    = "rename"
	actionEnableFw  = "enable_firewall"
	actionDisableFw = "disable_firewall"
)

// Logger for this package
var Logger = loggo.GetLogger("gosdc.cloudapi")

// Client provides a means to access the Joyent CloudAPI
type Client struct {
	client client.Client
}

// New creates a new Client.
func New(client client.Client) *Client {
	return &Client{client}
}

// Filter represents a filter that can be applied to an API request.
type Filter struct {
	v url.Values
}

// NewFilter creates a new Filter.
func NewFilter() *Filter {
	return &Filter{make(url.Values)}
}

// Set a value for the specified filter.
func (f *Filter) Set(filter, value string) {
	f.v.Set(filter, value)
}

// Add a value for the specified filter.
func (f *Filter) Add(filter, value string) {
	f.v.Add(filter, value)
}

// request represents an API request
type request struct {
	method         string
	url            string
	filter         *Filter
	reqValue       interface{}
	reqHeader      http.Header
	resp           interface{}
	respHeader     *http.Header
	expectedStatus int
}

// Helper method to send an API request
func (c *Client) sendRequest(req request) (*jh.ResponseData, error) {
	request := jh.RequestData{
		ReqValue:   req.reqValue,
		ReqHeaders: req.reqHeader,
	}
	if req.filter != nil {
		request.Params = &req.filter.v
	}
	if req.expectedStatus == 0 {
		req.expectedStatus = http.StatusOK
	}
	respData := jh.ResponseData{
		RespValue:      req.resp,
		RespHeaders:    req.respHeader,
		ExpectedStatus: []int{req.expectedStatus},
	}
	err := c.client.SendRequest(req.method, req.url, "", &request, &respData)
	return &respData, err
}

// Helper method to create the API URL
func makeURL(parts ...string) string {
	return path.Join(parts...)
}
<|endoftext|>