<commit_before>package exec

import (
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"os"
	"path"

	"github.com/cozy/cozy-stack/pkg/apps"
	"github.com/cozy/cozy-stack/pkg/config"
	"github.com/cozy/cozy-stack/pkg/instance"
	"github.com/cozy/cozy-stack/pkg/jobs"
	"github.com/sirupsen/logrus"
	"github.com/spf13/afero"
)

// ServiceOptions contains the options to execute a service.
type ServiceOptions struct {
	Slug        string          `json:"slug"`
	Type        string          `json:"type"`
	ServiceFile string          `json:"service_file"`
	Message     *ServiceOptions `json:"message"`
}

type serviceWorker struct {
	opts *ServiceOptions
	man  *apps.WebappManifest
}

func (w *serviceWorker) PrepareWorkDir(ctx *jobs.WorkerContext, i *instance.Instance) (workDir string, err error) {
	opts := &ServiceOptions{}
	if err = ctx.UnmarshalMessage(&opts); err != nil {
		return
	}
	if opts.Message != nil {
		opts = opts.Message
	}

	slug := opts.Slug

	man, err := apps.GetWebappBySlug(i, slug)
	if err != nil {
		return
	}
	if man.State() != apps.Ready {
		err = errors.New("Application is not ready")
		return
	}

	w.opts = opts
	w.man = man

	osFS := afero.NewOsFs()
	workDir, err = afero.TempDir(osFS, "", "service-"+slug)
	if err != nil {
		return
	}
	workFS := afero.NewBasePathFs(osFS, workDir)

	fs := i.AppsFileServer()
	src, err := fs.Open(man.Slug(), man.Version(), path.Join("/", opts.ServiceFile))
	if err != nil {
		return
	}
	defer src.Close()

	dst, err := workFS.OpenFile("index.js", os.O_CREATE|os.O_WRONLY, 0640)
	if err != nil {
		return
	}
	defer dst.Close()

	_, err = io.Copy(dst, src)
	if err != nil {
		return
	}

	return workDir, nil
}

func (w *serviceWorker) Slug() string {
	return w.opts.Slug
}

func (w *serviceWorker) PrepareCmdEnv(ctx *jobs.WorkerContext, i *instance.Instance) (cmd string, env []string, err error) {
	token := i.BuildAppToken(w.man, "")
	cmd = config.GetConfig().Konnectors.Cmd
	env = []string{
		"COZY_URL=" + i.PageURL("/", nil),
		"COZY_CREDENTIALS=" + token,
		"COZY_TYPE=" + w.opts.Type,
		"COZY_LOCALE=" + i.Locale,
		"COZY_JOB_ID=" + ctx.ID(),
	}
	return
}

func (w *serviceWorker) Logger(ctx *jobs.WorkerContext) *logrus.Entry {
	return ctx.Logger().WithField("slug", w.Slug())
}

func (w *serviceWorker) ScanOutput(ctx *jobs.WorkerContext, i *instance.Instance, line []byte) error {
	var msg struct {
		Type    string `json:"type"`
		Message string `json:"message"`
	}
	if err := json.Unmarshal(line, &msg); err != nil {
		return fmt.Errorf("Could not parse stdout as JSON: %q", string(line))
	}
	log := w.Logger(ctx)
	switch msg.Type {
	case konnectorMsgTypeDebug:
		log.Debug(msg.Message)
	case konnectorMsgTypeInfo:
		log.Debug(msg.Message)
	case konnectorMsgTypeWarning:
		log.Warn(msg.Message)
	case konnectorMsgTypeError:
		log.Error(msg.Message)
	case konnectorMsgTypeCritical:
		log.Error(msg.Message)
	}
	return nil
}

func (w *serviceWorker) Error(i *instance.Instance, err error) error {
	return err
}

func (w *serviceWorker) Commit(ctx *jobs.WorkerContext, errjob error) error {
	return nil
}
<commit_msg>Also support "warn" as a warning level for services<commit_after>package exec

import (
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"os"
	"path"

	"github.com/cozy/cozy-stack/pkg/apps"
	"github.com/cozy/cozy-stack/pkg/config"
	"github.com/cozy/cozy-stack/pkg/instance"
	"github.com/cozy/cozy-stack/pkg/jobs"
	"github.com/sirupsen/logrus"
	"github.com/spf13/afero"
)

// ServiceOptions contains the options to execute a service.
type ServiceOptions struct {
	Slug        string          `json:"slug"`
	Type        string          `json:"type"`
	ServiceFile string          `json:"service_file"`
	Message     *ServiceOptions `json:"message"`
}

type serviceWorker struct {
	opts *ServiceOptions
	man  *apps.WebappManifest
}

func (w *serviceWorker) PrepareWorkDir(ctx *jobs.WorkerContext, i *instance.Instance) (workDir string, err error) {
	opts := &ServiceOptions{}
	if err = ctx.UnmarshalMessage(&opts); err != nil {
		return
	}
	if opts.Message != nil {
		opts = opts.Message
	}

	slug := opts.Slug

	man, err := apps.GetWebappBySlug(i, slug)
	if err != nil {
		return
	}
	if man.State() != apps.Ready {
		err = errors.New("Application is not ready")
		return
	}

	w.opts = opts
	w.man = man

	osFS := afero.NewOsFs()
	workDir, err = afero.TempDir(osFS, "", "service-"+slug)
	if err != nil {
		return
	}
	workFS := afero.NewBasePathFs(osFS, workDir)

	fs := i.AppsFileServer()
	src, err := fs.Open(man.Slug(), man.Version(), path.Join("/", opts.ServiceFile))
	if err != nil {
		return
	}
	defer src.Close()

	dst, err := workFS.OpenFile("index.js", os.O_CREATE|os.O_WRONLY, 0640)
	if err != nil {
		return
	}
	defer dst.Close()

	_, err = io.Copy(dst, src)
	if err != nil {
		return
	}

	return workDir, nil
}

func (w *serviceWorker) Slug() string {
	return w.opts.Slug
}

func (w *serviceWorker) PrepareCmdEnv(ctx *jobs.WorkerContext, i *instance.Instance) (cmd string, env []string, err error) {
	token := i.BuildAppToken(w.man, "")
	cmd = config.GetConfig().Konnectors.Cmd
	env = []string{
		"COZY_URL=" + i.PageURL("/", nil),
		"COZY_CREDENTIALS=" + token,
		"COZY_TYPE=" + w.opts.Type,
		"COZY_LOCALE=" + i.Locale,
		"COZY_JOB_ID=" + ctx.ID(),
	}
	return
}

func (w *serviceWorker) Logger(ctx *jobs.WorkerContext) *logrus.Entry {
	return ctx.Logger().WithField("slug", w.Slug())
}

func (w *serviceWorker) ScanOutput(ctx *jobs.WorkerContext, i *instance.Instance, line []byte) error {
	var msg struct {
		Type    string `json:"type"`
		Message string `json:"message"`
	}
	if err := json.Unmarshal(line, &msg); err != nil {
		return fmt.Errorf("Could not parse stdout as JSON: %q", string(line))
	}
	log := w.Logger(ctx)
	switch msg.Type {
	case konnectorMsgTypeDebug, konnectorMsgTypeInfo:
		log.Debug(msg.Message)
	case konnectorMsgTypeWarning, "warn":
		log.Warn(msg.Message)
	case konnectorMsgTypeError:
		log.Error(msg.Message)
	case konnectorMsgTypeCritical:
		log.Error(msg.Message)
	}
	return nil
}

func (w *serviceWorker) Error(i *instance.Instance, err error) error {
	return err
}

func (w *serviceWorker) Commit(ctx *jobs.WorkerContext, errjob error) error {
	return nil
}
<|endoftext|>


<commit_before>package adhier

import (
	"fmt"
	"testing"

	"github.com/ready-steady/numeric/basis/linhat"
	"github.com/ready-steady/numeric/grid/newcot"
	"github.com/ready-steady/support/assert"
)

func TestComputeStep(t *testing.T) {
	interpolator := prepare(&fixtureStep)

	surrogate := interpolator.Compute(step)

	assert.Equal(surrogate, fixtureStep.surrogate, t)
}

func TestEvaluateStep(t *testing.T) {
	interpolator := prepare(&fixtureStep)

	values := interpolator.Evaluate(fixtureStep.surrogate, fixtureStep.points)

	assert.Equal(values, fixtureStep.values, t)
}

func TestComputeHat(t *testing.T) {
	interpolator := prepare(&fixtureHat)

	surrogate := interpolator.Compute(hat)

	assert.Equal(surrogate, fixtureHat.surrogate, t)
}

func TestEvaluateHat(t *testing.T) {
	interpolator := prepare(&fixtureHat)

	values := interpolator.Evaluate(fixtureHat.surrogate, fixtureHat.points)

	assert.AlmostEqual(values, fixtureHat.values, t)
}

func TestComputeCube(t *testing.T) {
	interpolator := prepare(&fixtureCube)

	surrogate := interpolator.Compute(cube)

	assert.Equal(surrogate, fixtureCube.surrogate, t)
}

func TestComputeBox(t *testing.T) {
	interpolator := prepare(&fixtureBox)

	surrogate := interpolator.Compute(box)

	assert.Equal(surrogate, fixtureBox.surrogate, t)
}

func TestEvaluateBox(t *testing.T) {
	interpolator := prepare(&fixtureBox)

	values := interpolator.Evaluate(fixtureBox.surrogate, fixtureBox.points)

	assert.AlmostEqual(values, fixtureBox.values, t)
}

func BenchmarkHat(b *testing.B) {
	interpolator := prepare(&fixtureHat)

	for i := 0; i < b.N; i++ {
		interpolator.Compute(hat)
	}
}

func BenchmarkCube(b *testing.B) {
	interpolator := prepare(&fixtureCube, func(config *Config) {
		config.MaxLevel = 9
	})

	for i := 0; i < b.N; i++ {
		interpolator.Compute(cube)
	}
}

func BenchmarkBox(b *testing.B) {
	interpolator := prepare(&fixtureCube, func(config *Config) {
		config.MaxLevel = 9
	})

	for i := 0; i < b.N; i++ {
		interpolator.Compute(box)
	}
}

func BenchmarkMany(b *testing.B) {
	interpolator := prepare(&fixture{
		surrogate: &Surrogate{
			level: 9,
			ic:    2,
			oc:    1000,
		},
	})

	function := many(2, 1000)

	for i := 0; i < b.N; i++ {
		interpolator.Compute(function)
	}
}

// A one-input-one-output scenario with a non-smooth function.
func ExampleInterpolator_step() {
	const (
		inputs  = 1
		outputs = 1
	)

	grid := newcot.NewClosed(inputs)
	basis := linhat.NewClosed(inputs, outputs)

	config := DefaultConfig()
	config.MaxLevel = 19
	interpolator, _ := New(grid, basis, config, outputs)

	surrogate := interpolator.Compute(step)
	fmt.Println(surrogate)

	// Output:
	// Surrogate{inputs: 1, outputs: 1, level: 19, nodes: 38}
}

// A one-input-one-output scenario with a smooth function.
func ExampleInterpolator_hat() {
	const (
		inputs  = 1
		outputs = 1
	)

	grid := newcot.NewClosed(inputs)
	basis := linhat.NewClosed(inputs, outputs)

	config := DefaultConfig()
	config.MaxLevel = 9
	interpolator, _ := New(grid, basis, config, outputs)

	surrogate := interpolator.Compute(hat)
	fmt.Println(surrogate)

	// Output:
	// Surrogate{inputs: 1, outputs: 1, level: 9, nodes: 305}
}

// A multiple-input-one-output scenario with a non-smooth function.
func ExampleInterpolator_cube() {
	const (
		inputs  = 2
		outputs = 1
	)

	grid := newcot.NewClosed(inputs)
	basis := linhat.NewClosed(inputs, outputs)

	config := DefaultConfig()
	config.MaxLevel = 9
	interpolator, _ := New(grid, basis, config, outputs)

	surrogate := interpolator.Compute(cube)
	fmt.Println(surrogate)

	// Output:
	// Surrogate{inputs: 2, outputs: 1, level: 9, nodes: 377}
}

// A multiple-input-many-output scenario with a non-smooth function.
func ExampleInterpolator_many() {
	const (
		inputs  = 2
		outputs = 1000
	)

	grid := newcot.NewClosed(inputs)
	basis := linhat.NewClosed(inputs, outputs)
	config := DefaultConfig()
	config.MaxNodes = 300

	interpolator, _ := New(grid, basis, config, outputs)

	surrogate := interpolator.Compute(many(inputs, outputs))
	fmt.Println(surrogate)

	// Output:
	// Surrogate{inputs: 2, outputs: 1000, level: 9, nodes: 300}
}

func prepare(fixture *fixture, arguments ...interface{}) *Interpolator {
	surrogate := fixture.surrogate

	ic, oc := uint16(surrogate.ic), uint16(surrogate.oc)

	config := DefaultConfig()
	config.MaxLevel = surrogate.level

	if len(arguments) > 0 {
		process, _ := arguments[0].(func(*Config))
		process(&config)
	}

	interpolator, _ := New(newcot.NewClosed(ic), linhat.NewClosed(ic, oc), config, oc)

	return interpolator
}
<commit_msg>A minor adjustment and a fixed typo<commit_after>package adhier

import (
	"fmt"
	"testing"

	"github.com/ready-steady/numeric/basis/linhat"
	"github.com/ready-steady/numeric/grid/newcot"
	"github.com/ready-steady/support/assert"
)

func TestComputeStep(t *testing.T) {
	interpolator := prepare(&fixtureStep)

	surrogate := interpolator.Compute(step)

	assert.Equal(surrogate, fixtureStep.surrogate, t)
}

func TestEvaluateStep(t *testing.T) {
	interpolator := prepare(&fixtureStep)

	values := interpolator.Evaluate(fixtureStep.surrogate, fixtureStep.points)

	assert.Equal(values, fixtureStep.values, t)
}

func TestComputeHat(t *testing.T) {
	interpolator := prepare(&fixtureHat)

	surrogate := interpolator.Compute(hat)

	assert.Equal(surrogate, fixtureHat.surrogate, t)
}

func TestEvaluateHat(t *testing.T) {
	interpolator := prepare(&fixtureHat)

	values := interpolator.Evaluate(fixtureHat.surrogate, fixtureHat.points)

	assert.AlmostEqual(values, fixtureHat.values, t)
}

func TestComputeCube(t *testing.T) {
	interpolator := prepare(&fixtureCube)

	surrogate := interpolator.Compute(cube)

	assert.Equal(surrogate, fixtureCube.surrogate, t)
}

func TestComputeBox(t *testing.T) {
	interpolator := prepare(&fixtureBox)

	surrogate := interpolator.Compute(box)

	assert.Equal(surrogate, fixtureBox.surrogate, t)
}

func TestEvaluateBox(t *testing.T) {
	interpolator := prepare(&fixtureBox)

	values := interpolator.Evaluate(fixtureBox.surrogate, fixtureBox.points)

	assert.AlmostEqual(values, fixtureBox.values, t)
}

func BenchmarkHat(b *testing.B) {
	interpolator := prepare(&fixtureHat)

	for i := 0; i < b.N; i++ {
		interpolator.Compute(hat)
	}
}

func BenchmarkCube(b *testing.B) {
	interpolator := prepare(&fixtureCube, func(config *Config) {
		config.MaxLevel = 9
	})

	for i := 0; i < b.N; i++ {
		interpolator.Compute(cube)
	}
}

func BenchmarkBox(b *testing.B) {
	interpolator := prepare(&fixtureBox, func(config *Config) {
		config.MaxLevel = 9
	})

	for i := 0; i < b.N; i++ {
		interpolator.Compute(box)
	}
}

func BenchmarkMany(b *testing.B) {
	const (
		inputs  = 2
		outputs = 1000
	)

	interpolator := prepare(&fixture{
		surrogate: &Surrogate{
			level: 9,
			ic:    inputs,
			oc:    outputs,
		},
	})

	function := many(inputs, outputs)

	for i := 0; i < b.N; i++ {
		interpolator.Compute(function)
	}
}

// A one-input-one-output scenario with a non-smooth function.
func ExampleInterpolator_step() {
	const (
		inputs  = 1
		outputs = 1
	)

	grid := newcot.NewClosed(inputs)
	basis := linhat.NewClosed(inputs, outputs)

	config := DefaultConfig()
	config.MaxLevel = 19
	interpolator, _ := New(grid, basis, config, outputs)

	surrogate := interpolator.Compute(step)
	fmt.Println(surrogate)

	// Output:
	// Surrogate{inputs: 1, outputs: 1, level: 19, nodes: 38}
}

// A one-input-one-output scenario with a smooth function.
func ExampleInterpolator_hat() {
	const (
		inputs  = 1
		outputs = 1
	)

	grid := newcot.NewClosed(inputs)
	basis := linhat.NewClosed(inputs, outputs)

	config := DefaultConfig()
	config.MaxLevel = 9
	interpolator, _ := New(grid, basis, config, outputs)

	surrogate := interpolator.Compute(hat)
	fmt.Println(surrogate)

	// Output:
	// Surrogate{inputs: 1, outputs: 1, level: 9, nodes: 305}
}

// A multiple-input-one-output scenario with a non-smooth function.
func ExampleInterpolator_cube() {
	const (
		inputs  = 2
		outputs = 1
	)

	grid := newcot.NewClosed(inputs)
	basis := linhat.NewClosed(inputs, outputs)

	config := DefaultConfig()
	config.MaxLevel = 9
	interpolator, _ := New(grid, basis, config, outputs)

	surrogate := interpolator.Compute(cube)
	fmt.Println(surrogate)

	// Output:
	// Surrogate{inputs: 2, outputs: 1, level: 9, nodes: 377}
}

// A multiple-input-many-output scenario with a non-smooth function.
func ExampleInterpolator_many() {
	const (
		inputs  = 2
		outputs = 1000
	)

	grid := newcot.NewClosed(inputs)
	basis := linhat.NewClosed(inputs, outputs)
	config := DefaultConfig()
	config.MaxNodes = 300

	interpolator, _ := New(grid, basis, config, outputs)

	surrogate := interpolator.Compute(many(inputs, outputs))
	fmt.Println(surrogate)

	// Output:
	// Surrogate{inputs: 2, outputs: 1000, level: 9, nodes: 300}
}

func prepare(fixture *fixture, arguments ...interface{}) *Interpolator {
	surrogate := fixture.surrogate

	ic, oc := uint16(surrogate.ic), uint16(surrogate.oc)

	config := DefaultConfig()
	config.MaxLevel = surrogate.level

	if len(arguments) > 0 {
		process, _ := arguments[0].(func(*Config))
		process(&config)
	}

	interpolator, _ := New(newcot.NewClosed(ic), linhat.NewClosed(ic, oc), config, oc)

	return interpolator
}
<|endoftext|>


<commit_before>// Copyright 2016 The Upspin Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package postgres

import (
	"bytes"
	"database/sql"
	"fmt"

	// Required when importing this package.
	_ "github.com/lib/pq"

	"upspin.io/cloud/storage"
	"upspin.io/errors"
	"upspin.io/log"
)

// postgresImpl is a Storage that connects to a Postgres backend.
// It likely won't work with other SQL databases because of a few
// Postgres-ism such as the text index and how "upsert" is handled.
type postgresImpl struct {
	db *sql.DB
}

var _ storage.Storage = (*postgresImpl)(nil)

// PutLocalFile implements storage.Storage.
func (p *postgresImpl) PutLocalFile(srcLocalFilename string, ref string) (refLink string, error error) {
	// TODO: implement. Only relevant if we want to store blobs though.
	return "", errors.E("Postgres.PutLocalFile", errors.Syntax, errors.Str("putlocalfile not implemented for postgres"))
}

// Get implements storage.Storage.
func (p *postgresImpl) Get(ref string) (link string, error error) {
	// TODO: implement. Only relevant if we want to store blobs though.
	return "", errors.E("Postgres.Get", errors.Syntax, errors.Str("get not implemented for postgres"))
}

// Download implements storage.Storage.
func (p *postgresImpl) Download(ref string) ([]byte, error) {
	const Download = "Postgres.Download"
	var data string
	// QueryRow with $1 parameters ensures we don't have SQL escape problems.
	err := p.db.QueryRow("SELECT data FROM directory WHERE ref = $1;", ref).Scan(&data)
	if err == sql.ErrNoRows {
		return nil, errors.E(Download, errors.NotExist, err)
	}
	if err != nil {
		return nil, errors.E(Download, errors.IO, err)
	}
	return []byte(data), nil
}

// Put implements storage.Storage.
func (p *postgresImpl) Put(ref string, contents []byte) (refLink string, error error) {
	const Put = "Postgres.Put"
	res, err := p.db.Exec(
		`INSERT INTO directory (ref, data) values ($1, $2) ON CONFLICT (ref) DO UPDATE SET data = $2;`,
		ref, string(contents))
	if err != nil {
		return "", errors.E(Put, errors.IO, err)
	}
	n, err := res.RowsAffected()
	if err != nil {
		// No information. Assume success.
		return "", nil
	}
	if n != 1 {
		// Something went wrong.
		return "", errors.E(Put, errors.IO, errors.Str("spurious updates in SQL DB"))
	}
	return "", nil
}

// ListPrefix implements storage.Storage.
func (p *postgresImpl) ListPrefix(prefix string, depth int) ([]string, error) {
	const ListPrefix = "Postgres.ListPrefix"
	query := "SELECT ref FROM directory WHERE ref LIKE $1"
	arg := fmt.Sprintf("%s%%", prefix) // a left-prefix-match.
	// TODO: check depth and enforce it.
	return p.commonListDir(ListPrefix, query, arg)
}

// commonListDir implements common functionality shared between ListPrefix and ListDir.
func (p *postgresImpl) commonListDir(op string, query string, args ...interface{}) ([]string, error) {
	rows, err := p.db.Query(query, args...)
	if err == sql.ErrNoRows {
		return nil, nil
	}
	if err != nil {
		return nil, errors.E(op, errors.IO, err)
	}
	defer rows.Close()
	res := make([]string, 0, 16) // We don't know the size ahead of time without doing a SELECT COUNT.
	var firstErr error
	saveErr := func(err error) {
		if firstErr != nil {
			firstErr = err
		}
	}
	for rows.Next() {
		var name string
		if err := rows.Scan(&name); err != nil {
			saveErr(err)
			continue
		}
		res = append(res, name)
	}
	if err := rows.Err(); err != nil {
		saveErr(err)
	}
	if firstErr != nil {
		return res, errors.E(op, errors.IO, err)
	}
	return res, nil
}

// ListDir implements storage.Storage.
func (p *postgresImpl) ListDir(dir string) ([]string, error) {
	const ListDir = "Postgres.ListDir"
	topDir := fmt.Sprintf("%s%%", dir)
	notSubDir := fmt.Sprintf("%s[^/]+/%%", dir)
	// Usage of LIKE and NOT SIMILAR is necessary here to trigger the use of the index.
	// Using posix-regex (i.e. using the operator "~") does not trigger the index.
	query := "SELECT ref FROM directory WHERE ref LIKE $1 AND ref NOT SIMILAR TO $2"
	return p.commonListDir(ListDir, query, topDir, notSubDir)
}

// Delete implements storage.Storage.
func (p *postgresImpl) Delete(ref string) error {
	const Delete = "Postgres.Delete"
	_, err := p.db.Exec("DELETE FROM directory WHERE ref = $1", ref)
	if err != nil {
		return errors.E(Delete, errors.IO, err)
	}
	return nil
}

// Dial implements storage.Storage.
func (p *postgresImpl) Dial(opts *storage.Opts) error {
	const Dial = "Postgres.Dial"
	optStr := buildOptStr(opts)
	log.Printf("Connecting and creating table with options [%s]", optStr)
	db, err := sql.Open("postgres", optStr)
	if err != nil {
		return errors.E(Dial, errors.IO, err)
	}
	// We need a dummy primary key so that we can build an index on ref.
	_, err = db.Exec(
		`CREATE TABLE IF NOT EXISTS directory (
	             id SERIAL PRIMARY KEY,
	             ref varchar(8000) UNIQUE NOT NULL,
	             data text NOT NULL
	         )`)
	if err != nil {
		return errors.E(Dial, errors.IO, err)
	}
	// Build a text index on ref to speed up regex pattern matching queries.
	_, err = db.Exec("CREATE INDEX IF NOT EXISTS directory_ref_index ON directory (ref text_pattern_ops);")
	if err != nil {
		return errors.E(Dial, errors.IO, err)
	}
	log.Printf("No errors found!")

	// We're ready to have fun with the db.
	p.db = db

	return nil
}

func buildOptStr(opts *storage.Opts) string {
	var b bytes.Buffer
	first := true
	for k, v := range opts.Opts {
		if !first {
			fmt.Fprintf(&b, " %s=%s", k, v)
		} else {
			fmt.Fprintf(&b, "%s=%s", k, v)
			first = false
		}
	}
	return b.String()
}

// Close implements storage.Storage.
func (p *postgresImpl) Close() {
	p.db.Close()
	p.db = nil
}

func init() {
	err := storage.Register("Postgres", &postgresImpl{})
	if err != nil {
		log.Fatal(err)
	}
}
<commit_msg>storage/postgres: re-apply accidentally-reverted changes<commit_after>// Copyright 2016 The Upspin Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package postgres implements a storage backend for interfacing with a Postgres database.
package postgres

import (
	"bytes"
	"database/sql"
	"fmt"

	// Required when importing this package.
	_ "github.com/lib/pq"

	"upspin.io/cloud/storage"
	"upspin.io/errors"
	"upspin.io/log"
)

// postgres is a Storage that connects to a Postgres backend.
// It likely won't work with other SQL databases because of a few
// Postgres-ism such as the text index and how "upsert" is handled.
type postgres struct {
	db *sql.DB
}

var _ storage.Storage = (*postgres)(nil)

// PutLocalFile implements storage.Storage.
func (p *postgres) PutLocalFile(srcLocalFilename string, ref string) (refLink string, error error) {
	// TODO: implement. Only relevant if we want to store blobs though.
	return "", errors.E("Postgres.PutLocalFile", errors.Syntax, errors.Str("putlocalfile not implemented for postgres"))
}

// Get implements storage.Storage.
func (p *postgres) Get(ref string) (link string, error error) {
	// TODO: implement. Only relevant if we want to store blobs though.
	return "", errors.E("Postgres.Get", errors.Syntax, errors.Str("get not implemented for postgres"))
}

// Download implements storage.Storage.
func (p *postgres) Download(ref string) ([]byte, error) {
	const Download = "Postgres.Download"
	var data string
	// QueryRow with $1 parameters ensures we don't have SQL escape problems.
	err := p.db.QueryRow("SELECT data FROM directory WHERE ref = $1;", ref).Scan(&data)
	if err == sql.ErrNoRows {
		return nil, errors.E(Download, errors.NotExist, err)
	}
	if err != nil {
		return nil, errors.E(Download, errors.IO, err)
	}
	return []byte(data), nil
}

// Put implements storage.Storage.
func (p *postgres) Put(ref string, contents []byte) (refLink string, error error) {
	const Put = "Postgres.Put"
	res, err := p.db.Exec(
		`INSERT INTO directory (ref, data) values ($1, $2) ON CONFLICT (ref) DO UPDATE SET data = $2;`,
		ref, string(contents))
	if err != nil {
		return "", errors.E(Put, errors.IO, err)
	}
	n, err := res.RowsAffected()
	if err != nil {
		// No information. Assume success.
		return "", nil
	}
	if n != 1 {
		// Something went wrong.
		return "", errors.E(Put, errors.IO, errors.Errorf("spurious updates in SQL DB, expected 1, got %d", n))
	}
	return "", nil
}

// ListPrefix implements storage.Storage.
func (p *postgres) ListPrefix(prefix string, depth int) ([]string, error) {
	const ListPrefix = "Postgres.ListPrefix"
	query := "SELECT ref FROM directory WHERE ref LIKE $1"
	arg := prefix + "%" // a left-prefix-match.
	// TODO: check depth and enforce it.
	return p.commonListDir(ListPrefix, query, arg)
}

// commonListDir implements common functionality shared between ListPrefix and ListDir.
func (p *postgres) commonListDir(op string, query string, args ...interface{}) ([]string, error) {
	rows, err := p.db.Query(query, args...)
	if err == sql.ErrNoRows {
		return nil, nil
	}
	if err != nil {
		return nil, errors.E(op, errors.IO, err)
	}
	defer rows.Close()
	var res []string // We don't know the size ahead of time without doing a SELECT COUNT.
	var firstErr error
	saveErr := func(err error) {
		if firstErr != nil {
			firstErr = err
		}
	}
	for rows.Next() {
		var name string
		if err := rows.Scan(&name); err != nil {
			saveErr(err)
			continue
		}
		res = append(res, name)
	}
	if err := rows.Err(); err != nil {
		saveErr(err)
	}
	if firstErr != nil {
		return res, errors.E(op, errors.IO, err)
	}
	return res, nil
}

// ListDir implements storage.Storage.
func (p *postgres) ListDir(dir string) ([]string, error) {
	const ListDir = "Postgres.ListDir"
	topDir := dir + "%"
	notSubDir := dir + "[^/]+/%"
	// Usage of LIKE and NOT SIMILAR is necessary here to trigger the use of the index.
	// Using posix-regex (i.e. using the operator "~") does not trigger the index.
	query := "SELECT ref FROM directory WHERE ref LIKE $1 AND ref NOT SIMILAR TO $2"
	return p.commonListDir(ListDir, query, topDir, notSubDir)
}

// Delete implements storage.Storage.
func (p *postgres) Delete(ref string) error {
	const Delete = "Postgres.Delete"
	_, err := p.db.Exec("DELETE FROM directory WHERE ref = $1", ref)
	if err != nil {
		return errors.E(Delete, errors.IO, err)
	}
	return nil
}

// Dial implements storage.Storage.
func (p *postgres) Dial(opts *storage.Opts) error {
	const Dial = "Postgres.Dial"
	optStr := buildOptStr(opts)
	log.Printf("Connecting and creating table with options [%s]", optStr)
	db, err := sql.Open("postgres", optStr)
	if err != nil {
		return errors.E(Dial, errors.IO, err)
	}
	// We need a dummy primary key so that we can build an index on ref.
	_, err = db.Exec(
		`CREATE TABLE IF NOT EXISTS directory (
	             id SERIAL PRIMARY KEY,
	             ref varchar(8000) UNIQUE NOT NULL,
	             data text NOT NULL
	         )`)
	if err != nil {
		return errors.E(Dial, errors.IO, err)
	}
	// Build a text index on ref to speed up regex pattern matching queries.
	_, err = db.Exec("CREATE INDEX IF NOT EXISTS directory_ref_index ON directory (ref text_pattern_ops);")
	if err != nil {
		return errors.E(Dial, errors.IO, err)
	}

	// We're ready to have fun with the db.
	p.db = db

	return nil
}

func buildOptStr(opts *storage.Opts) string {
	var b bytes.Buffer
	first := true
	for k, v := range opts.Opts {
		if !first {
			fmt.Fprintf(&b, " %s=%s", k, v)
		} else {
			fmt.Fprintf(&b, "%s=%s", k, v)
			first = false
		}
	}
	return b.String()
}

// Close implements storage.Storage.
func (p *postgres) Close() {
	p.db.Close()
	p.db = nil
}

func init() {
	err := storage.Register("Postgres", &postgres{})
	if err != nil {
		log.Fatal(err)
	}
}
<|endoftext|>


<commit_before>// Copyright 2018 Istio Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package kubernetes

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"strings"

	v1 "k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/runtime/serializer"
	"k8s.io/client-go/kubernetes/scheme"
	"k8s.io/client-go/rest"
	"k8s.io/client-go/tools/remotecommand"

	"istio.io/istio/pkg/kube"
	"istio.io/istio/pkg/version"
)

var (
	proxyContainer     = "istio-proxy"
	discoveryContainer = "discovery"
	pilotDiscoveryPath = "/usr/local/bin/pilot-discovery"
	pilotAgentPath     = "/usr/local/bin/pilot-agent"
)

// Client is a helper wrapper around the Kube RESTClient for istioctl -> Pilot/Envoy/Mesh related things
type Client struct {
	Config *rest.Config
	*rest.RESTClient
}

// ExecClient is an interface for remote execution
type ExecClient interface {
	EnvoyDo(podName, podNamespace, method, path string, body []byte) ([]byte, error)
	AllPilotsDiscoveryDo(pilotNamespace, method, path string, body []byte) (map[string][]byte, error)
	GetIstioVersions(namespace string) (*version.MeshInfo, error)
}

// NewClient is the constructor for the client wrapper
func NewClient(kubeconfig, configContext string) (*Client, error) {
	config, err := defaultRestConfig(kubeconfig, configContext)
	if err != nil {
		return nil, err
	}
	restClient, err := rest.RESTClientFor(config)
	if err != nil {
		return nil, err
	}
	return &Client{config, restClient}, nil
}

func defaultRestConfig(kubeconfig, configContext string) (*rest.Config, error) {
	config, err := kube.BuildClientConfig(kubeconfig, configContext)
	if err != nil {
		return nil, err
	}
	config.APIPath = "/api"
	config.GroupVersion = &v1.SchemeGroupVersion
	config.NegotiatedSerializer = serializer.DirectCodecFactory{CodecFactory: scheme.Codecs}
	return config, nil
}

// PodExec takes a command and the pod data to run the command in the specified pod
func (client *Client) PodExec(podName, podNamespace, container string, command []string) (*bytes.Buffer, *bytes.Buffer, error) {
	req := client.Post().
		Resource("pods").
		Name(podName).
		Namespace(podNamespace).
		SubResource("exec").
		Param("container", container).
		VersionedParams(&v1.PodExecOptions{
			Container: container,
			Command:   command,
			Stdin:     false,
			Stdout:    true,
			Stderr:    true,
			TTY:       false,
		}, scheme.ParameterCodec)

	exec, err := remotecommand.NewSPDYExecutor(client.Config, "POST", req.URL())
	if err != nil {
		return nil, nil, err
	}

	var stdout, stderr bytes.Buffer
	err = exec.Stream(remotecommand.StreamOptions{
		Stdin:  nil,
		Stdout: &stdout,
		Stderr: &stderr,
		Tty:    false,
	})

	return &stdout, &stderr, err
}

// AllPilotsDiscoveryDo makes an http request to each Pilot discovery instance
func (client *Client) AllPilotsDiscoveryDo(pilotNamespace, method, path string, body []byte) (map[string][]byte, error) {
	pilots, err := client.GetIstioPods(pilotNamespace, map[string]string{"labelSelector": "istio=pilot"})
	if err != nil {
		return nil, err
	}
	if len(pilots) == 0 {
		return nil, errors.New("unable to find any Pilot instances")
	}
	cmd := []string{pilotDiscoveryPath, "request", method, path, string(body)}
	result := map[string][]byte{}
	for _, pilot := range pilots {
		res, err := client.ExtractExecResult(pilot.Name, pilot.Namespace, discoveryContainer, cmd)
		if err != nil {
			return nil, err
		}
		if len(res) > 0 {
			result[pilot.Name] = res
		}
	}
	return result, err
}

// PilotDiscoveryDo makes an http request to a single Pilot discovery instance
func (client *Client) PilotDiscoveryDo(pilotNamespace, method, path string, body []byte) ([]byte, error) {
	pilots, err := client.GetIstioPods(pilotNamespace, map[string]string{"labelSelector": "istio=pilot"})
	if err != nil {
		return nil, err
	}
	if len(pilots) == 0 {
		return nil, errors.New("unable to find any Pilot instances")
	}
	cmd := []string{pilotDiscoveryPath, "request", method, path, string(body)}
	return client.ExtractExecResult(pilots[0].Name, pilots[0].Namespace, discoveryContainer, cmd)
}

// EnvoyDo makes an http request to the Envoy in the specified pod
func (client *Client) EnvoyDo(podName, podNamespace, method, path string, body []byte) ([]byte, error) {
	container, err := client.GetPilotAgentContainer(podName, podNamespace)
	if err != nil {
		return nil, fmt.Errorf("unable to retrieve proxy container name: %v", err)
	}
	cmd := []string{pilotAgentPath, "request", method, path, string(body)}
	return client.ExtractExecResult(podName, podNamespace, container, cmd)
}

// ExtractExecResult wraps PodExec and return the execution result and error if has any.
func (client *Client) ExtractExecResult(podName, podNamespace, container string, cmd []string) ([]byte, error) {
	stdout, stderr, err := client.PodExec(podName, podNamespace, container, cmd)
	if err != nil {
		return nil, fmt.Errorf("error execing into %v/%v %v container: %v", podName, podNamespace, container, err)
	}
	if stderr.String() != "" {
		fmt.Printf("Warning! error execing into %v/%v %v container: %v\n", podName, podNamespace, container, stderr.String())
	}
	return stdout.Bytes(), nil
}

// GetIstioPods retrieves the pod objects for Istio deployments
func (client *Client) GetIstioPods(namespace string, params map[string]string) ([]v1.Pod, error) {
	req := client.Get().
		Resource("pods").
		Namespace(namespace)
	for k, v := range params {
		req.Param(k, v)
	}

	res := req.Do()
	if res.Error() != nil {
		return nil, fmt.Errorf("unable to retrieve Pods: %v", res.Error())
	}
	list := &v1.PodList{}
	if err := res.Into(list); err != nil {
		return nil, fmt.Errorf("unable to parse PodList: %v", res.Error())
	}
	return list.Items, nil
}

// GetPilotAgentContainer retrieves the pilot-agent container name for the specified pod
func (client *Client) GetPilotAgentContainer(podName, podNamespace string) (string, error) {
	req := client.Get().
		Resource("pods").
		Namespace(podNamespace).
		Name(podName)

	res := req.Do()
	if res.Error() != nil {
		return "", fmt.Errorf("unable to retrieve Pod: %v", res.Error())
	}
	pod := &v1.Pod{}
	if err := res.Into(pod); err != nil {
		return "", fmt.Errorf("unable to parse Pod: %v", res.Error())
	}
	for _, c := range pod.Spec.Containers {
		switch c.Name {
		case "egressgateway", "ingress", "ingressgateway":
			return c.Name, nil
		}
	}
	return proxyContainer, nil
}

type podDetail struct {
	binary    string
	container string
}

// GetIstioVersions gets the version for each Istio component
func (client *Client) GetIstioVersions(namespace string) (*version.MeshInfo, error) {
	pods, err := client.GetIstioPods(namespace, map[string]string{"labelSelector": "istio"})
	if err != nil {
		return nil, err
	}
	if len(pods) == 0 {
		return nil, errors.New("unable to find any Istio pod in namespace " + namespace)
	}

	labelToPodDetail := map[string]podDetail{
		"pilot":            {"/usr/local/bin/pilot-discovery", "discovery"},
		"citadel":          {"/usr/local/bin/istio_ca", "citadel"},
		"egressgateway":    {"/usr/local/bin/pilot-agent", "istio-proxy"},
		"galley":           {"/usr/local/bin/galley", "galley"},
		"ingressgateway":   {"/usr/local/bin/pilot-agent", "istio-proxy"},
		"telemetry":        {"/usr/local/bin/mixs", "mixer"},
		"policy":           {"/usr/local/bin/mixs", "mixer"},
		"sidecar-injector": {"/usr/local/bin/sidecar-injector", "sidecar-injector-webhook"},
	}

	res := version.MeshInfo{}
	for _, pod := range pods {
		component := pod.Labels["istio"]

		// Special cases
		switch component {
		case "statsd-prom-bridge":
			continue
		case "mixer":
			component = pod.Labels["istio-mixer-type"]
		}

		server := version.ServerInfo{Component: component}

		if detail, ok := labelToPodDetail[component]; ok {
			cmd := []string{detail.binary, "version"}
			cmdJSON := append(cmd, "-o", "json")

			var info version.BuildInfo
			var v version.Version

			stdout, stderr, err := client.PodExec(pod.Name, pod.Namespace, detail.container, cmdJSON)

			if err != nil {
				return nil, fmt.Errorf("error execing into %v %v container: %v", pod.Name, detail.container, err)
			}

			// At first try parsing stdout
			err = json.Unmarshal(stdout.Bytes(), &v)
			if err == nil && v.ClientVersion.Version != "" {
				info = *v.ClientVersion
			} else {
				// If stdout fails, try the old behavior
				if strings.HasPrefix(stderr.String(), "Error: unknown shorthand flag") {
					stdout, err := client.ExtractExecResult(pod.Name, pod.Namespace, detail.container, cmd)
					if err != nil {
						return nil, fmt.Errorf("error execing into %v %v container: %v", pod.Name, detail.container, err)
					}

					info, err = version.NewBuildInfoFromOldString(string(stdout))
					if err != nil {
						return nil, fmt.Errorf("error converting server info from JSON: %v", err)
					}
				} else {
					return nil, fmt.Errorf("error execing into %v %v container: %v", pod.Name, detail.container, stderr.String())
				}
			}

			server.Info = info
		}
		res = append(res, server)
	}
	return &res, nil
}
<commit_msg>istioctl proxy-status should only exec into running pilot pods (#11539)<commit_after>// Copyright 2018 Istio Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package kubernetes

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"strings"

	v1 "k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/runtime/serializer"
	"k8s.io/client-go/kubernetes/scheme"
	"k8s.io/client-go/rest"
	"k8s.io/client-go/tools/remotecommand"

	"istio.io/istio/pkg/kube"
	"istio.io/istio/pkg/version"
)

var (
	proxyContainer     = "istio-proxy"
	discoveryContainer = "discovery"
	pilotDiscoveryPath = "/usr/local/bin/pilot-discovery"
	pilotAgentPath     = "/usr/local/bin/pilot-agent"
)

// Client is a helper wrapper around the Kube RESTClient for istioctl -> Pilot/Envoy/Mesh related things
type Client struct {
	Config *rest.Config
	*rest.RESTClient
}

// ExecClient is an interface for remote execution
type ExecClient interface {
	EnvoyDo(podName, podNamespace, method, path string, body []byte) ([]byte, error)
	AllPilotsDiscoveryDo(pilotNamespace, method, path string, body []byte) (map[string][]byte, error)
	GetIstioVersions(namespace string) (*version.MeshInfo, error)
}

// NewClient is the constructor for the client wrapper
func NewClient(kubeconfig, configContext string) (*Client, error) {
	config, err := defaultRestConfig(kubeconfig, configContext)
	if err != nil {
		return nil, err
	}
	restClient, err := rest.RESTClientFor(config)
	if err != nil {
		return nil, err
	}
	return &Client{config, restClient}, nil
}

func defaultRestConfig(kubeconfig, configContext string) (*rest.Config, error) {
	config, err := kube.BuildClientConfig(kubeconfig, configContext)
	if err != nil {
		return nil, err
	}
	config.APIPath = "/api"
	config.GroupVersion = &v1.SchemeGroupVersion
	config.NegotiatedSerializer = serializer.DirectCodecFactory{CodecFactory: scheme.Codecs}
	return config, nil
}

// PodExec takes a command and the pod data to run the command in the specified pod
func (client *Client) PodExec(podName, podNamespace, container string, command []string) (*bytes.Buffer, *bytes.Buffer, error) {
	req := client.Post().
		Resource("pods").
		Name(podName).
		Namespace(podNamespace).
		SubResource("exec").
		Param("container", container).
		VersionedParams(&v1.PodExecOptions{
			Container: container,
			Command:   command,
			Stdin:     false,
			Stdout:    true,
			Stderr:    true,
			TTY:       false,
		}, scheme.ParameterCodec)

	exec, err := remotecommand.NewSPDYExecutor(client.Config, "POST", req.URL())
	if err != nil {
		return nil, nil, err
	}

	var stdout, stderr bytes.Buffer
	err = exec.Stream(remotecommand.StreamOptions{
		Stdin:  nil,
		Stdout: &stdout,
		Stderr: &stderr,
		Tty:    false,
	})

	return &stdout, &stderr, err
}

// AllPilotsDiscoveryDo makes an http request to each Pilot discovery instance
func (client *Client) AllPilotsDiscoveryDo(pilotNamespace, method, path string, body []byte) (map[string][]byte, error) {
	pilots, err := client.GetIstioPods(pilotNamespace, map[string]string{
		"labelSelector": "istio=pilot",
		"fieldSelector": "status.phase=Running",
	})
	if err != nil {
		return nil, err
	}
	if len(pilots) == 0 {
		return nil, errors.New("unable to find any Pilot instances")
	}
	cmd := []string{pilotDiscoveryPath, "request", method, path, string(body)}
	result := map[string][]byte{}
	for _, pilot := range pilots {
		res, err := client.ExtractExecResult(pilot.Name, pilot.Namespace, discoveryContainer, cmd)
		if err != nil {
			return nil, err
		}
		if len(res) > 0 {
			result[pilot.Name] = res
		}
	}
	return result, err
}

// PilotDiscoveryDo makes an http request to a single Pilot discovery instance
func (client *Client) PilotDiscoveryDo(pilotNamespace, method, path string, body []byte) ([]byte, error) {
	pilots, err := client.GetIstioPods(pilotNamespace, map[string]string{
		"labelSelector": "istio=pilot",
		"fieldSelector": "status.phase=Running",
	})
	if err != nil {
		return nil, err
	}
	if len(pilots) == 0 {
		return nil, errors.New("unable to find any Pilot instances")
	}
	cmd := []string{pilotDiscoveryPath, "request", method, path, string(body)}
	return client.ExtractExecResult(pilots[0].Name, pilots[0].Namespace, discoveryContainer, cmd)
}

// EnvoyDo makes an http request to the Envoy in the specified pod
func (client *Client) EnvoyDo(podName, podNamespace, method, path string, body []byte) ([]byte, error) {
	container, err := client.GetPilotAgentContainer(podName, podNamespace)
	if err != nil {
		return nil, fmt.Errorf("unable to retrieve proxy container name: %v", err)
	}
	cmd := []string{pilotAgentPath, "request", method, path, string(body)}
	return client.ExtractExecResult(podName, podNamespace, container, cmd)
}

// ExtractExecResult wraps PodExec and return the execution result and error if has any.
func (client *Client) ExtractExecResult(podName, podNamespace, container string, cmd []string) ([]byte, error) {
	stdout, stderr, err := client.PodExec(podName, podNamespace, container, cmd)
	if err != nil {
		return nil, fmt.Errorf("error execing into %v/%v %v container: %v", podName, podNamespace, container, err)
	}
	if stderr.String() != "" {
		fmt.Printf("Warning! error execing into %v/%v %v container: %v\n", podName, podNamespace, container, stderr.String())
	}
	return stdout.Bytes(), nil
}

// GetIstioPods retrieves the pod objects for Istio deployments
func (client *Client) GetIstioPods(namespace string, params map[string]string) ([]v1.Pod, error) {
	req := client.Get().
		Resource("pods").
		Namespace(namespace)
	for k, v := range params {
		req.Param(k, v)
	}

	res := req.Do()
	if res.Error() != nil {
		return nil, fmt.Errorf("unable to retrieve Pods: %v", res.Error())
	}
	list := &v1.PodList{}
	if err := res.Into(list); err != nil {
		return nil, fmt.Errorf("unable to parse PodList: %v", res.Error())
	}
	return list.Items, nil
}

// GetPilotAgentContainer retrieves the pilot-agent container name for the specified pod
func (client *Client) GetPilotAgentContainer(podName, podNamespace string) (string, error) {
	req := client.Get().
		Resource("pods").
		Namespace(podNamespace).
		Name(podName)

	res := req.Do()
	if res.Error() != nil {
		return "", fmt.Errorf("unable to retrieve Pod: %v", res.Error())
	}
	pod := &v1.Pod{}
	if err := res.Into(pod); err != nil {
		return "", fmt.Errorf("unable to parse Pod: %v", res.Error())
	}
	for _, c := range pod.Spec.Containers {
		switch c.Name {
		case "egressgateway", "ingress", "ingressgateway":
			return c.Name, nil
		}
	}
	return proxyContainer, nil
}

type podDetail struct {
	binary    string
	container string
}

// GetIstioVersions gets the version for each Istio component
func (client *Client) GetIstioVersions(namespace string) (*version.MeshInfo, error) {
	pods, err := client.GetIstioPods(namespace, map[string]string{
		"labelSelector": "istio",
		"fieldSelector": "status.phase=Running",
	})
	if err != nil {
		return nil, err
	}
	if len(pods) == 0 {
		return nil, errors.New("unable to find any Istio pod in namespace " + namespace)
	}

	labelToPodDetail := map[string]podDetail{
		"pilot":            {"/usr/local/bin/pilot-discovery", "discovery"},
		"citadel":          {"/usr/local/bin/istio_ca", "citadel"},
		"egressgateway":    {"/usr/local/bin/pilot-agent", "istio-proxy"},
		"galley":           {"/usr/local/bin/galley", "galley"},
		"ingressgateway":   {"/usr/local/bin/pilot-agent", "istio-proxy"},
		"telemetry":        {"/usr/local/bin/mixs", "mixer"},
		"policy":           {"/usr/local/bin/mixs", "mixer"},
		"sidecar-injector": {"/usr/local/bin/sidecar-injector", "sidecar-injector-webhook"},
	}

	res := version.MeshInfo{}
	for _, pod := range pods {
		component := pod.Labels["istio"]

		// Special cases
		switch component {
		case "statsd-prom-bridge":
			continue
		case "mixer":
			component = pod.Labels["istio-mixer-type"]
		}

		server := version.ServerInfo{Component: component}

		if detail, ok := labelToPodDetail[component]; ok {
			cmd := []string{detail.binary, "version"}
			cmdJSON := append(cmd, "-o", "json")

			var info version.BuildInfo
			var v version.Version

			stdout, stderr, err := client.PodExec(pod.Name, pod.Namespace, detail.container, cmdJSON)

			if err != nil {
				return nil, fmt.Errorf("error execing into %v %v container: %v", pod.Name, detail.container, err)
			}

			// At first try parsing stdout
			err = json.Unmarshal(stdout.Bytes(), &v)
			if err == nil && v.ClientVersion.Version != "" {
				info = *v.ClientVersion
			} else {
				// If stdout fails, try the old behavior
				if strings.HasPrefix(stderr.String(), "Error: unknown shorthand flag") {
					stdout, err := client.ExtractExecResult(pod.Name, pod.Namespace, detail.container, cmd)
					if err != nil {
						return nil, fmt.Errorf("error execing into %v %v container: %v", pod.Name, detail.container, err)
					}

					info, err = version.NewBuildInfoFromOldString(string(stdout))
					if err != nil {
						return nil, fmt.Errorf("error converting server info from JSON: %v", err)
					}
				} else {
					return nil, fmt.Errorf("error execing into %v %v container: %v", pod.Name, detail.container, stderr.String())
				}
			}

			server.Info = info
		}
		res = append(res, server)
	}
	return &res, nil
}
<|endoftext|>


<commit_before>// +build linux darwin

package fingerprint

import (
	"errors"
	"fmt"
	"io/ioutil"
	"log"
	"net"
	"os/exec"
	"regexp"
	"strconv"
	"strings"

	"github.com/hashicorp/nomad/client/config"
	"github.com/hashicorp/nomad/nomad/structs"
)

// NetworkFingerprint is used to fingerprint the Network capabilities of a node
type NetworkFingerprint struct {
	logger *log.Logger
}

// NewNetworkFingerprinter returns a new NetworkFingerprinter with the given
// logger
func NewNetworkFingerprinter(logger *log.Logger) Fingerprint {
	f := &NetworkFingerprint{logger: logger}
	return f
}

func (f *NetworkFingerprint) Fingerprint(cfg *config.Config, node *structs.Node) (bool, error) {
	// newNetwork is populated and addded to the Nodes resources
	newNetwork := &structs.NetworkResource{}
	var ip string

	interfaces, err := f.findInterfaces(cfg.NetworkInterface)
	if err != nil {
		return false, fmt.Errorf("Error while detecting network interface during fingerprinting: %s", err.Error())
	}

	if len(interfaces) == 0 {
		return false, errors.New("No network interfaces were detected")
	}

	// Use the first interface that we have detected.
	intf := interfaces[0]
	if ip, err = f.ipAddress(intf); err != nil {
		return false, fmt.Errorf("Unable to find IP address of interface: %s, err: %s", intf.Name, err.Error())
	}

	newNetwork.Device = intf.Name
	node.Attributes["network.ip-address"] = ip
	newNetwork.IP = ip
	newNetwork.CIDR = newNetwork.IP + "/32"

	f.logger.Println("[DEBUG] fingerprint.network Detected interface ", intf.Name, " with IP ", ip, " during fingerprinting")

	if throughput := f.linkSpeed(intf.Name); throughput > 0 {
		newNetwork.MBits = throughput
	} else {
		f.logger.Printf("[DEBUG] fingerprint.network: Unable to read link speed; setting to default %v", cfg.NetworkSpeed)
		newNetwork.MBits = cfg.NetworkSpeed
	}

	if node.Resources == nil {
		node.Resources = &structs.Resources{}
	}

	node.Resources.Networks = append(node.Resources.Networks, newNetwork)

	// return true, because we have a network connection
	return true, nil
}

// linkSpeed returns link speed in Mb/s, or 0 when unable to determine it.
func (f *NetworkFingerprint) linkSpeed(device string) int {
	// Use LookPath to find the ethtool in the systems $PATH
	// If it's not found or otherwise errors, LookPath returns and empty string
	// and an error we can ignore for our purposes
	ethtoolPath, _ := exec.LookPath("ethtool")
	if ethtoolPath != "" {
		if speed := f.linkSpeedEthtool(ethtoolPath, device); speed > 0 {
			return speed
		}
	}

	// Fall back on checking a system file for link speed.
	return f.linkSpeedSys(device)
}

// linkSpeedSys parses link speed in Mb/s from /sys.
func (f *NetworkFingerprint) linkSpeedSys(device string) int {
	path := fmt.Sprintf("/sys/class/net/%s/speed", device)

	// Read contents of the device/speed file
	content, err := ioutil.ReadFile(path)
	if err != nil {
		f.logger.Printf("[WARN] fingerprint.network: Unable to read link speed from %s", path)
		return 0
	}

	lines := strings.Split(string(content), "\n")
	mbs, err := strconv.Atoi(lines[0])
	if err != nil || mbs <= 0 {
		f.logger.Printf("[WARN] fingerprint.network: Unable to parse link speed from %s", path)
		return 0
	}

	return mbs
}

// linkSpeedEthtool determines link speed in Mb/s with 'ethtool'.
func (f *NetworkFingerprint) linkSpeedEthtool(path, device string) int {
	outBytes, err := exec.Command(path, device).Output()
	if err != nil {
		f.logger.Printf("[WARN] fingerprint.network: Error calling ethtool (%s %s): %v", path, device, err)
		return 0
	}

	output := strings.TrimSpace(string(outBytes))
	re := regexp.MustCompile("Speed: [0-9]+[a-zA-Z]+/s")
	m := re.FindString(output)
	if m == "" {
		// no matches found, output may be in a different format
		f.logger.Printf("[WARN] fingerprint.network: Unable to parse Speed in output of '%s %s'", path, device)
		return 0
	}

	// Split and trim the Mb/s unit from the string output
	args := strings.Split(m, ": ")
	raw := strings.TrimSuffix(args[1], "Mb/s")

	// convert to Mb/s
	mbs, err := strconv.Atoi(raw)
	if err != nil || mbs <= 0 {
		f.logger.Printf("[WARN] fingerprint.network: Unable to parse Mb/s in output of '%s %s'", path, device)
		return 0
	}

	return mbs
}

// Gets the ipv4 addr for a network interface
func (f *NetworkFingerprint) ipAddress(intf *net.Interface) (string, error) {
	var (
		addrs []net.Addr
		err   error
	)
	if addrs, err = intf.Addrs(); err != nil {
		return "", err
	}

	if len(addrs) == 0 {
		return "", errors.New(fmt.Sprintf("Interface %s has no IP address", intf.Name))
	}
	var ipV4 net.IP
	for _, addr := range addrs {
		var ip net.IP
		switch v := (addr).(type) {
		case *net.IPNet:
			ip = v.IP
		case *net.IPAddr:
			ip = v.IP
		}
		if ip.To4() != nil {
			ipV4 = ip
			break
		}
	}

	if ipV4 == nil {
		return "", errors.New(fmt.Sprintf("Couldn't parse IP address for interface %s with addr %s", intf.Name))
	}
	return ipV4.String(), nil

}

// Checks if the device is marked UP by the operator
func (f *NetworkFingerprint) isDeviceEnabled(intf *net.Interface) bool {
	return intf.Flags&net.FlagUp != 0
}

// Checks if the device has any IP address configured
func (f *NetworkFingerprint) deviceHasIpAddress(intf *net.Interface) bool {
	if _, err := f.ipAddress(intf); err == nil {
		return true
	}
	return false
}

func (n *NetworkFingerprint) isDeviceLoopBackOrPointToPoint(intf *net.Interface) bool {
	return intf.Flags&(net.FlagLoopback|net.FlagPointToPoint) == 0
}

// Returns interfaces which are routable and marked as UP
// Tries to get the specific interface if the user has specified name
func (f *NetworkFingerprint) findInterfaces(deviceName string) ([]*net.Interface, error) {
	var (
		interfaces []*net.Interface
		err        error
	)
	if deviceName != "" {
		if intf, err := net.InterfaceByName(deviceName); err == nil {
			interfaces = append(interfaces, intf)
		}
		return interfaces, err
	}

	var intfs []net.Interface

	if intfs, err = net.Interfaces(); err != nil {
		return nil, err
	}

	for _, intf := range intfs {
		if f.isDeviceEnabled(&intf) && !f.isDeviceLoopBackOrPointToPoint(&intf) && f.deviceHasIpAddress(&intf) {
			interfaces = append(interfaces, &intf)
		}
	}
	return interfaces, nil
}
<commit_msg>Some coding style changes<commit_after>// +build linux darwin

package fingerprint

import (
	"errors"
	"fmt"
	"io/ioutil"
	"log"
	"net"
	"os/exec"
	"regexp"
	"strconv"
	"strings"

	"github.com/hashicorp/nomad/client/config"
	"github.com/hashicorp/nomad/nomad/structs"
)

// NetworkFingerprint is used to fingerprint the Network capabilities of a node
type NetworkFingerprint struct {
	logger *log.Logger
}

// NewNetworkFingerprinter returns a new NetworkFingerprinter with the given
// logger
func NewNetworkFingerprinter(logger *log.Logger) Fingerprint {
	f := &NetworkFingerprint{logger: logger}
	return f
}

func (f *NetworkFingerprint) Fingerprint(cfg *config.Config, node *structs.Node) (bool, error) {
	// newNetwork is populated and addded to the Nodes resources
	newNetwork := &structs.NetworkResource{}
	var ip string

	interfaces, err := f.findInterfaces(cfg.NetworkInterface)
	if err != nil {
		return false, fmt.Errorf("Error while detecting network interface during fingerprinting: %v", err)
	}

	if len(interfaces) == 0 {
		return false, errors.New("No network interfaces were detected")
	}

	// Use the first interface that we have detected.
	intf := interfaces[0]
	if ip, err = f.ipAddress(intf); err != nil {
		return false, fmt.Errorf("Unable to find IP address of interface: %s, err: %v", intf.Name, err)
	}

	newNetwork.Device = intf.Name
	node.Attributes["network.ip-address"] = ip
	newNetwork.IP = ip
	newNetwork.CIDR = newNetwork.IP + "/32"

	f.logger.Println("[DEBUG] fingerprint.network Detected interface ", intf.Name, " with IP ", ip, " during fingerprinting")

	if throughput := f.linkSpeed(intf.Name); throughput > 0 {
		newNetwork.MBits = throughput
	} else {
		f.logger.Printf("[DEBUG] fingerprint.network: Unable to read link speed; setting to default %v", cfg.NetworkSpeed)
		newNetwork.MBits = cfg.NetworkSpeed
	}

	if node.Resources == nil {
		node.Resources = &structs.Resources{}
	}

	node.Resources.Networks = append(node.Resources.Networks, newNetwork)

	// return true, because we have a network connection
	return true, nil
}

// linkSpeed returns link speed in Mb/s, or 0 when unable to determine it.
func (f *NetworkFingerprint) linkSpeed(device string) int {
	// Use LookPath to find the ethtool in the systems $PATH
	// If it's not found or otherwise errors, LookPath returns and empty string
	// and an error we can ignore for our purposes
	ethtoolPath, _ := exec.LookPath("ethtool")
	if ethtoolPath != "" {
		if speed := f.linkSpeedEthtool(ethtoolPath, device); speed > 0 {
			return speed
		}
	}

	// Fall back on checking a system file for link speed.
	return f.linkSpeedSys(device)
}

// linkSpeedSys parses link speed in Mb/s from /sys.
func (f *NetworkFingerprint) linkSpeedSys(device string) int {
	path := fmt.Sprintf("/sys/class/net/%s/speed", device)

	// Read contents of the device/speed file
	content, err := ioutil.ReadFile(path)
	if err != nil {
		f.logger.Printf("[WARN] fingerprint.network: Unable to read link speed from %s", path)
		return 0
	}

	lines := strings.Split(string(content), "\n")
	mbs, err := strconv.Atoi(lines[0])
	if err != nil || mbs <= 0 {
		f.logger.Printf("[WARN] fingerprint.network: Unable to parse link speed from %s", path)
		return 0
	}

	return mbs
}

// linkSpeedEthtool determines link speed in Mb/s with 'ethtool'.
func (f *NetworkFingerprint) linkSpeedEthtool(path, device string) int {
	outBytes, err := exec.Command(path, device).Output()
	if err != nil {
		f.logger.Printf("[WARN] fingerprint.network: Error calling ethtool (%s %s): %v", path, device, err)
		return 0
	}

	output := strings.TrimSpace(string(outBytes))
	re := regexp.MustCompile("Speed: [0-9]+[a-zA-Z]+/s")
	m := re.FindString(output)
	if m == "" {
		// no matches found, output may be in a different format
		f.logger.Printf("[WARN] fingerprint.network: Unable to parse Speed in output of '%s %s'", path, device)
		return 0
	}

	// Split and trim the Mb/s unit from the string output
	args := strings.Split(m, ": ")
	raw := strings.TrimSuffix(args[1], "Mb/s")

	// convert to Mb/s
	mbs, err := strconv.Atoi(raw)
	if err != nil || mbs <= 0 {
		f.logger.Printf("[WARN] fingerprint.network: Unable to parse Mb/s in output of '%s %s'", path, device)
		return 0
	}

	return mbs
}

// Gets the ipv4 addr for a network interface
func (f *NetworkFingerprint) ipAddress(intf *net.Interface) (string, error) {
	var addrs []net.Addr
	var err error

	if addrs, err = intf.Addrs(); err != nil {
		return "", err
	}

	if len(addrs) == 0 {
		return "", errors.New(fmt.Sprintf("Interface %s has no IP address", intf.Name))
	}
	var ipV4 net.IP
	for _, addr := range addrs {
		var ip net.IP
		switch v := (addr).(type) {
		case *net.IPNet:
			ip = v.IP
		case *net.IPAddr:
			ip = v.IP
		}
		if ip.To4() != nil {
			ipV4 = ip
			break
		}
	}

	if ipV4 == nil {
		return "", fmt.Errorf("Couldn't parse IP address for interface %s with addr %s", intf.Name)
	}
	return ipV4.String(), nil

}

// Checks if the device is marked UP by the operator
func (f *NetworkFingerprint) isDeviceEnabled(intf *net.Interface) bool {
	return intf.Flags&net.FlagUp != 0
}

// Checks if the device has any IP address configured
func (f *NetworkFingerprint) deviceHasIpAddress(intf *net.Interface) bool {
	if _, err := f.ipAddress(intf); err == nil {
		return true
	}
	return false
}

func (n *NetworkFingerprint) isDeviceLoopBackOrPointToPoint(intf *net.Interface) bool {
	return intf.Flags&(net.FlagLoopback|net.FlagPointToPoint) == 0
}

// Returns interfaces which are routable and marked as UP
// Tries to get the specific interface if the user has specified name
func (f *NetworkFingerprint) findInterfaces(deviceName string) ([]*net.Interface, error) {
	var interfaces []*net.Interface
	var err error

	if deviceName != "" {
		if intf, err := net.InterfaceByName(deviceName); err == nil {
			interfaces = append(interfaces, intf)
		}
		return interfaces, err
	}

	var intfs []net.Interface

	if intfs, err = net.Interfaces(); err != nil {
		return nil, err
	}

	for _, intf := range intfs {
		if f.isDeviceEnabled(&intf) && !f.isDeviceLoopBackOrPointToPoint(&intf) && f.deviceHasIpAddress(&intf) {
			interfaces = append(interfaces, &intf)
		}
	}
	return interfaces, nil
}
<|endoftext|>


<commit_before>// Copyright 2015 CoreOS, Inc.
// Copyright 2015-2017 Rancher Labs, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package cloudinitsave

import (
	"bytes"
	"errors"
	"os"
	"path"
	"strings"
	"sync"
	"time"

	yaml "github.com/cloudfoundry-incubator/candiedyaml"

	"github.com/rancher/os/cmd/control"
	"github.com/rancher/os/cmd/network"
	rancherConfig "github.com/rancher/os/config"
	"github.com/rancher/os/config/cloudinit/config"
	"github.com/rancher/os/config/cloudinit/datasource"
	"github.com/rancher/os/config/cloudinit/datasource/configdrive"
	"github.com/rancher/os/config/cloudinit/datasource/file"
	"github.com/rancher/os/config/cloudinit/datasource/metadata/digitalocean"
	"github.com/rancher/os/config/cloudinit/datasource/metadata/ec2"
	"github.com/rancher/os/config/cloudinit/datasource/metadata/gce"
	"github.com/rancher/os/config/cloudinit/datasource/metadata/packet"
	"github.com/rancher/os/config/cloudinit/datasource/proccmdline"
	"github.com/rancher/os/config/cloudinit/datasource/url"
	"github.com/rancher/os/config/cloudinit/pkg"
	"github.com/rancher/os/log"
	"github.com/rancher/os/netconf"
	"github.com/rancher/os/util"
)

const (
	datasourceInterval    = 100 * time.Millisecond
	datasourceMaxInterval = 30 * time.Second
	datasourceTimeout     = 5 * time.Minute
)

func Main() {
	log.InitLogger()
	log.Info("Running cloud-init-save")

	if err := control.UdevSettle(); err != nil {
		log.Errorf("Failed to run udev settle: %v", err)
	}

	if err := saveCloudConfig(); err != nil {
		log.Errorf("Failed to save cloud-config: %v", err)
	}
}

func saveCloudConfig() error {
	log.Debugf("SaveCloudConfig")

	cfg := rancherConfig.LoadConfig()
	log.Debugf("init: SaveCloudConfig(pre ApplyNetworkConfig): %#v", cfg.Rancher.Network)
	network.ApplyNetworkConfig(cfg)

	log.Debugf("datasources that will be consided: %#v", cfg.Rancher.CloudInit.Datasources)
	dss := getDatasources(cfg)
	if len(dss) == 0 {
		log.Errorf("currentDatasource - none found")
		return nil
	}

	selectDatasource(dss)

	// TODO: can't run these here, but it needs to be triggered from here :()
	// Apply any newly detected network config.
	cfg = rancherConfig.LoadConfig()
	log.Debugf("init: SaveCloudConfig(post ApplyNetworkConfig): %#v", cfg.Rancher.Network)
	network.ApplyNetworkConfig(cfg)

	return nil
}

func RequiresNetwork(datasource string) bool {
	// TODO: move into the datasources (and metadatasources)
	// and then we can enable that platforms defaults..
	parts := strings.SplitN(datasource, ":", 2)
	requiresNetwork, ok := map[string]bool{
		"ec2":          true,
		"file":         false,
		"url":          true,
		"cmdline":      true,
		"configdrive":  false,
		"digitalocean": true,
		"gce":          true,
		"packet":       true,
	}[parts[0]]
	return ok && requiresNetwork
}

func saveFiles(cloudConfigBytes, scriptBytes []byte, metadata datasource.Metadata) error {
	os.MkdirAll(rancherConfig.CloudConfigDir, os.ModeDir|0600)

	if len(scriptBytes) > 0 {
		log.Infof("Writing to %s", rancherConfig.CloudConfigScriptFile)
		if err := util.WriteFileAtomic(rancherConfig.CloudConfigScriptFile, scriptBytes, 500); err != nil {
			log.Errorf("Error while writing file %s: %v", rancherConfig.CloudConfigScriptFile, err)
			return err
		}
	}

	if len(cloudConfigBytes) > 0 {
		if err := util.WriteFileAtomic(rancherConfig.CloudConfigBootFile, cloudConfigBytes, 400); err != nil {
			return err
		}
		// TODO: Don't put secrets into the log
		log.Infof("Written to %s:\n%s", rancherConfig.CloudConfigBootFile, string(cloudConfigBytes))
	}

	metaDataBytes, err := yaml.Marshal(metadata)
	if err != nil {
		return err
	}

	if err = util.WriteFileAtomic(rancherConfig.MetaDataFile, metaDataBytes, 400); err != nil {
		return err
	}
	// TODO: Don't put secrets into the log
	log.Infof("Written to %s:\n%s", rancherConfig.MetaDataFile, string(metaDataBytes))

	// if we write the empty meta yml, the merge fails.
	// TODO: the problem is that a partially filled one will still have merge issues, so that needs fixing - presumably by making merge more clever, and making more fields optional
	emptyMeta, err := yaml.Marshal(datasource.Metadata{})
	if err != nil {
		return err
	}
	if bytes.Compare(metaDataBytes, emptyMeta) == 0 {
		log.Infof("not writing %s: its all defaults.", rancherConfig.CloudConfigNetworkFile)
		return nil
	}

	type nonRancherCfg struct {
		Network netconf.NetworkConfig `yaml:"network,omitempty"`
	}
	type nonCfg struct {
		Rancher nonRancherCfg `yaml:"rancher,omitempty"`
	}
	// write the network.yml file from metadata
	cc := nonCfg{
		Rancher: nonRancherCfg{
			Network: metadata.NetworkConfig,
		},
	}

	if err := os.MkdirAll(path.Dir(rancherConfig.CloudConfigNetworkFile), 0700); err != nil {
		log.Errorf("Failed to create directory for file %s: %v", rancherConfig.CloudConfigNetworkFile, err)
	}

	if err := rancherConfig.WriteToFile(cc, rancherConfig.CloudConfigNetworkFile); err != nil {
		log.Errorf("Failed to save config file %s: %v", rancherConfig.CloudConfigNetworkFile, err)
	}
	log.Infof("Written to %s:", rancherConfig.CloudConfigNetworkFile)

	return nil
}

func fetchAndSave(ds datasource.Datasource) error {
	var metadata datasource.Metadata

	log.Infof("Fetching user-data from datasource %s", ds)
	userDataBytes, err := ds.FetchUserdata()
	if err != nil {
		log.Errorf("Failed fetching user-data from datasource: %v", err)
		return err
	}
	log.Infof("Fetching meta-data from datasource of type %v", ds.Type())
	metadata, err = ds.FetchMetadata()
	if err != nil {
		log.Errorf("Failed fetching meta-data from datasource: %v", err)
		return err
	}

	userData := string(userDataBytes)
	scriptBytes := []byte{}

	if config.IsScript(userData) {
		scriptBytes = userDataBytes
		userDataBytes = []byte{}
	} else if isCompose(userData) {
		if userDataBytes, err = composeToCloudConfig(userDataBytes); err != nil {
			log.Errorf("Failed to convert compose to cloud-config syntax: %v", err)
			return err
		}
	} else if config.IsCloudConfig(userData) {
		if _, err := rancherConfig.ReadConfig(userDataBytes, false); err != nil {
			log.WithFields(log.Fields{"cloud-config": userData, "err": err}).Warn("Failed to parse cloud-config, not saving.")
			userDataBytes = []byte{}
		}
	} else {
		log.Errorf("Unrecognized user-data\n(%s)", userData)
		userDataBytes = []byte{}
	}

	if _, err := rancherConfig.ReadConfig(userDataBytes, false); err != nil {
		log.WithFields(log.Fields{"cloud-config": userData, "err": err}).Warn("Failed to parse cloud-config")
		return errors.New("Failed to parse cloud-config")
	}

	return saveFiles(userDataBytes, scriptBytes, metadata)
}

// getDatasources creates a slice of possible Datasources for cloudinit based
// on the different source command-line flags.
func getDatasources(cfg *rancherConfig.CloudConfig) []datasource.Datasource {
	dss := make([]datasource.Datasource, 0, 5)

	for _, ds := range cfg.Rancher.CloudInit.Datasources {
		parts := strings.SplitN(ds, ":", 2)

		root := ""
		if len(parts) > 1 {
			root = parts[1]
		}

		switch parts[0] {
		case "ec2":
			dss = append(dss, ec2.NewDatasource(root))
		case "file":
			if root != "" {
				dss = append(dss, file.NewDatasource(root))
			}
		case "url":
			if root != "" {
				dss = append(dss, url.NewDatasource(root))
			}
		case "cmdline":
			if len(parts) == 1 {
				dss = append(dss, proccmdline.NewDatasource())
			}
		case "configdrive":
			if root != "" {
				dss = append(dss, configdrive.NewDatasource(root))
			}
		case "digitalocean":
			// TODO: should we enableDoLinkLocal() - to avoid the need for the other kernel/oem options?
			dss = append(dss, digitalocean.NewDatasource(root))
		case "gce":
			dss = append(dss, gce.NewDatasource(root))
		case "packet":
			dss = append(dss, packet.NewDatasource(root))
		}
	}

	return dss
}

func enableDoLinkLocal() {
	err := netconf.ApplyNetworkConfigs(&netconf.NetworkConfig{
		Interfaces: map[string]netconf.InterfaceConfig{
			"eth0": {
				IPV4LL: true,
			},
		},
	})
	if err != nil {
		log.Errorf("Failed to apply link local on eth0: %v", err)
	}
}

// selectDatasource attempts to choose a valid Datasource to use based on its
// current availability. The first Datasource to report to be available is
// returned. Datasources will be retried if possible if they are not
// immediately available. If all Datasources are permanently unavailable or
// datasourceTimeout is reached before one becomes available, nil is returned.
func selectDatasource(sources []datasource.Datasource) datasource.Datasource {
	ds := make(chan datasource.Datasource)
	stop := make(chan struct{})
	var wg sync.WaitGroup

	for _, s := range sources {
		wg.Add(1)
		go func(s datasource.Datasource) {
			defer wg.Done()

			duration := datasourceInterval
			for {
				log.Infof("cloud-init: Checking availability of %q\n", s.Type())
				if s.IsAvailable() {
					log.Infof("cloud-init: Datasource available: %s", s)
					ds <- s
					return
				}
				if !s.AvailabilityChanges() {
					log.Infof("cloud-init: Datasource unavailable, skipping: %s", s)
					return
				}
				log.Errorf("cloud-init: Datasource not ready, will retry: %s", s)
				select {
				case <-stop:
					return
				case <-time.After(duration):
					duration = pkg.ExpBackoff(duration, datasourceMaxInterval)
				}
			}
		}(s)
	}

	done := make(chan struct{})
	go func() {
		wg.Wait()
		close(done)
	}()

	var s datasource.Datasource
	select {
	case s = <-ds:
		err := fetchAndSave(s)
		if err != nil {
			log.Errorf("Error fetching cloud-init datasource(%s): %s", s, err)
		}
	case <-done:
	case <-time.After(datasourceTimeout):
	}

	close(stop)
	return s
}

func isCompose(content string) bool {
	return strings.HasPrefix(content, "#compose\n")
}

func composeToCloudConfig(bytes []byte) ([]byte, error) {
	compose := make(map[interface{}]interface{})
	err := yaml.Unmarshal(bytes, &compose)
	if err != nil {
		return nil, err
	}

	return yaml.Marshal(map[interface{}]interface{}{
		"rancher": map[interface{}]interface{}{
			"services": compose,
		},
	})
}
<commit_msg>Don't log the cloud-init metadata to the dmesg log - it will contain some secrets<commit_after>// Copyright 2015 CoreOS, Inc.
// Copyright 2015-2017 Rancher Labs, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package cloudinitsave

import (
	"bytes"
	"errors"
	"os"
	"path"
	"strings"
	"sync"
	"time"

	yaml "github.com/cloudfoundry-incubator/candiedyaml"

	"github.com/rancher/os/cmd/control"
	"github.com/rancher/os/cmd/network"
	rancherConfig "github.com/rancher/os/config"
	"github.com/rancher/os/config/cloudinit/config"
	"github.com/rancher/os/config/cloudinit/datasource"
	"github.com/rancher/os/config/cloudinit/datasource/configdrive"
	"github.com/rancher/os/config/cloudinit/datasource/file"
	"github.com/rancher/os/config/cloudinit/datasource/metadata/digitalocean"
	"github.com/rancher/os/config/cloudinit/datasource/metadata/ec2"
	"github.com/rancher/os/config/cloudinit/datasource/metadata/gce"
	"github.com/rancher/os/config/cloudinit/datasource/metadata/packet"
	"github.com/rancher/os/config/cloudinit/datasource/proccmdline"
	"github.com/rancher/os/config/cloudinit/datasource/url"
	"github.com/rancher/os/config/cloudinit/pkg"
	"github.com/rancher/os/log"
	"github.com/rancher/os/netconf"
	"github.com/rancher/os/util"
)

const (
	datasourceInterval    = 100 * time.Millisecond
	datasourceMaxInterval = 30 * time.Second
	datasourceTimeout     = 5 * time.Minute
)

func Main() {
	log.InitLogger()
	log.Info("Running cloud-init-save")

	if err := control.UdevSettle(); err != nil {
		log.Errorf("Failed to run udev settle: %v", err)
	}

	if err := saveCloudConfig(); err != nil {
		log.Errorf("Failed to save cloud-config: %v", err)
	}
}

func saveCloudConfig() error {
	log.Debugf("SaveCloudConfig")

	cfg := rancherConfig.LoadConfig()
	log.Debugf("init: SaveCloudConfig(pre ApplyNetworkConfig): %#v", cfg.Rancher.Network)
	network.ApplyNetworkConfig(cfg)

	log.Debugf("datasources that will be consided: %#v", cfg.Rancher.CloudInit.Datasources)
	dss := getDatasources(cfg)
	if len(dss) == 0 {
		log.Errorf("currentDatasource - none found")
		return nil
	}

	selectDatasource(dss)

	// Apply any newly detected network config.
	cfg = rancherConfig.LoadConfig()
	log.Debugf("init: SaveCloudConfig(post ApplyNetworkConfig): %#v", cfg.Rancher.Network)
	network.ApplyNetworkConfig(cfg)

	return nil
}

func RequiresNetwork(datasource string) bool {
	// TODO: move into the datasources (and metadatasources)
	// and then we can enable that platforms defaults..
	parts := strings.SplitN(datasource, ":", 2)
	requiresNetwork, ok := map[string]bool{
		"ec2":          true,
		"file":         false,
		"url":          true,
		"cmdline":      true,
		"configdrive":  false,
		"digitalocean": true,
		"gce":          true,
		"packet":       true,
	}[parts[0]]
	return ok && requiresNetwork
}

func saveFiles(cloudConfigBytes, scriptBytes []byte, metadata datasource.Metadata) error {
	os.MkdirAll(rancherConfig.CloudConfigDir, os.ModeDir|0600)

	if len(scriptBytes) > 0 {
		log.Infof("Writing to %s", rancherConfig.CloudConfigScriptFile)
		if err := util.WriteFileAtomic(rancherConfig.CloudConfigScriptFile, scriptBytes, 500); err != nil {
			log.Errorf("Error while writing file %s: %v", rancherConfig.CloudConfigScriptFile, err)
			return err
		}
	}

	if len(cloudConfigBytes) > 0 {
		if err := util.WriteFileAtomic(rancherConfig.CloudConfigBootFile, cloudConfigBytes, 400); err != nil {
			return err
		}
		log.Infof("Wrote to %s", rancherConfig.CloudConfigBootFile)
	}

	metaDataBytes, err := yaml.Marshal(metadata)
	if err != nil {
		return err
	}

	if err = util.WriteFileAtomic(rancherConfig.MetaDataFile, metaDataBytes, 400); err != nil {
		return err
	}
	log.Infof("Wrote to %s", rancherConfig.MetaDataFile)

	// if we write the empty meta yml, the merge fails.
	// TODO: the problem is that a partially filled one will still have merge issues, so that needs fixing - presumably by making merge more clever, and making more fields optional
	emptyMeta, err := yaml.Marshal(datasource.Metadata{})
	if err != nil {
		return err
	}
	if bytes.Compare(metaDataBytes, emptyMeta) == 0 {
		log.Infof("not writing %s: its all defaults.", rancherConfig.CloudConfigNetworkFile)
		return nil
	}

	type nonRancherCfg struct {
		Network netconf.NetworkConfig `yaml:"network,omitempty"`
	}
	type nonCfg struct {
		Rancher nonRancherCfg `yaml:"rancher,omitempty"`
	}
	// write the network.yml file from metadata
	cc := nonCfg{
		Rancher: nonRancherCfg{
			Network: metadata.NetworkConfig,
		},
	}

	if err := os.MkdirAll(path.Dir(rancherConfig.CloudConfigNetworkFile), 0700); err != nil {
		log.Errorf("Failed to create directory for file %s: %v", rancherConfig.CloudConfigNetworkFile, err)
	}

	if err := rancherConfig.WriteToFile(cc, rancherConfig.CloudConfigNetworkFile); err != nil {
		log.Errorf("Failed to save config file %s: %v", rancherConfig.CloudConfigNetworkFile, err)
	}
	log.Infof("Wrote to %s", rancherConfig.CloudConfigNetworkFile)

	return nil
}

func fetchAndSave(ds datasource.Datasource) error {
	var metadata datasource.Metadata

	log.Infof("Fetching user-data from datasource %s", ds)
	userDataBytes, err := ds.FetchUserdata()
	if err != nil {
		log.Errorf("Failed fetching user-data from datasource: %v", err)
		return err
	}
	log.Infof("Fetching meta-data from datasource of type %v", ds.Type())
	metadata, err = ds.FetchMetadata()
	if err != nil {
		log.Errorf("Failed fetching meta-data from datasource: %v", err)
		return err
	}

	userData := string(userDataBytes)
	scriptBytes := []byte{}

	if config.IsScript(userData) {
		scriptBytes = userDataBytes
		userDataBytes = []byte{}
	} else if isCompose(userData) {
		if userDataBytes, err = composeToCloudConfig(userDataBytes); err != nil {
			log.Errorf("Failed to convert compose to cloud-config syntax: %v", err)
			return err
		}
	} else if config.IsCloudConfig(userData) {
		if _, err := rancherConfig.ReadConfig(userDataBytes, false); err != nil {
			log.WithFields(log.Fields{"cloud-config": userData, "err": err}).Warn("Failed to parse cloud-config, not saving.")
			userDataBytes = []byte{}
		}
	} else {
		log.Errorf("Unrecognized user-data\n(%s)", userData)
		userDataBytes = []byte{}
	}

	if _, err := rancherConfig.ReadConfig(userDataBytes, false); err != nil {
		log.WithFields(log.Fields{"cloud-config": userData, "err": err}).Warn("Failed to parse cloud-config")
		return errors.New("Failed to parse cloud-config")
	}

	return saveFiles(userDataBytes, scriptBytes, metadata)
}

// getDatasources creates a slice of possible Datasources for cloudinit based
// on the different source command-line flags.
func getDatasources(cfg *rancherConfig.CloudConfig) []datasource.Datasource {
	dss := make([]datasource.Datasource, 0, 5)

	for _, ds := range cfg.Rancher.CloudInit.Datasources {
		parts := strings.SplitN(ds, ":", 2)

		root := ""
		if len(parts) > 1 {
			root = parts[1]
		}

		switch parts[0] {
		case "ec2":
			dss = append(dss, ec2.NewDatasource(root))
		case "file":
			if root != "" {
				dss = append(dss, file.NewDatasource(root))
			}
		case "url":
			if root != "" {
				dss = append(dss, url.NewDatasource(root))
			}
		case "cmdline":
			if len(parts) == 1 {
				dss = append(dss, proccmdline.NewDatasource())
			}
		case "configdrive":
			if root != "" {
				dss = append(dss, configdrive.NewDatasource(root))
			}
		case "digitalocean":
			// TODO: should we enableDoLinkLocal() - to avoid the need for the other kernel/oem options?
			dss = append(dss, digitalocean.NewDatasource(root))
		case "gce":
			dss = append(dss, gce.NewDatasource(root))
		case "packet":
			dss = append(dss, packet.NewDatasource(root))
		}
	}

	return dss
}

func enableDoLinkLocal() {
	err := netconf.ApplyNetworkConfigs(&netconf.NetworkConfig{
		Interfaces: map[string]netconf.InterfaceConfig{
			"eth0": {
				IPV4LL: true,
			},
		},
	})
	if err != nil {
		log.Errorf("Failed to apply link local on eth0: %v", err)
	}
}

// selectDatasource attempts to choose a valid Datasource to use based on its
// current availability. The first Datasource to report to be available is
// returned. Datasources will be retried if possible if they are not
// immediately available. If all Datasources are permanently unavailable or
// datasourceTimeout is reached before one becomes available, nil is returned.
func selectDatasource(sources []datasource.Datasource) datasource.Datasource {
	ds := make(chan datasource.Datasource)
	stop := make(chan struct{})
	var wg sync.WaitGroup

	for _, s := range sources {
		wg.Add(1)
		go func(s datasource.Datasource) {
			defer wg.Done()

			duration := datasourceInterval
			for {
				log.Infof("cloud-init: Checking availability of %q\n", s.Type())
				if s.IsAvailable() {
					log.Infof("cloud-init: Datasource available: %s", s)
					ds <- s
					return
				}
				if !s.AvailabilityChanges() {
					log.Infof("cloud-init: Datasource unavailable, skipping: %s", s)
					return
				}
				log.Errorf("cloud-init: Datasource not ready, will retry: %s", s)
				select {
				case <-stop:
					return
				case <-time.After(duration):
					duration = pkg.ExpBackoff(duration, datasourceMaxInterval)
				}
			}
		}(s)
	}

	done := make(chan struct{})
	go func() {
		wg.Wait()
		close(done)
	}()

	var s datasource.Datasource
	select {
	case s = <-ds:
		err := fetchAndSave(s)
		if err != nil {
			log.Errorf("Error fetching cloud-init datasource(%s): %s", s, err)
		}
	case <-done:
	case <-time.After(datasourceTimeout):
	}

	close(stop)
	return s
}

func isCompose(content string) bool {
	return strings.HasPrefix(content, "#compose\n")
}

func composeToCloudConfig(bytes []byte) ([]byte, error) {
	compose := make(map[interface{}]interface{})
	err := yaml.Unmarshal(bytes, &compose)
	if err != nil {
		return nil, err
	}

	return yaml.Marshal(map[interface{}]interface{}{
		"rancher": map[interface{}]interface{}{
			"services": compose,
		},
	})
}
<|endoftext|>


<commit_before>package network

import (
	"fmt"
	"os"
	"strings"
	"unicode/utf8"

	"github.com/dokku/dokku/plugins/common"
	"github.com/dokku/dokku/plugins/config"
	"github.com/dokku/dokku/plugins/proxy"
)

// TriggerInstall runs the install step for the network plugin
func TriggerInstall() {
	if err := common.PropertySetup("network"); err != nil {
		common.LogFail(fmt.Sprintf("Unable to install the network plugin: %s", err.Error()))
	}

	apps, err := common.DokkuApps()
	if err != nil {
		return
	}
	for _, appName := range apps {
		if common.PropertyExists("network", appName, "bind-all-interfaces") {
			continue
		}
		if proxy.IsAppProxyEnabled(appName) {
			common.LogVerboseQuiet("Setting network property 'bind-all-interfaces' to false")
			if err := common.PropertyWrite("network", appName, "bind-all-interfaces", "false"); err != nil {
				common.LogWarn(err.Error())
			}
		} else {
			common.LogVerboseQuiet("Setting network property 'bind-all-interfaces' to true")
			if err := common.PropertyWrite("network", appName, "bind-all-interfaces", "true"); err != nil {
				common.LogWarn(err.Error())
			}
		}
	}
}

// TriggerNetworkComputePorts computes the ports for a given app container
func TriggerNetworkComputePorts(appName string, processType string, isHerokuishContainer bool) {
	if processType != "web" {
		return
	}

	var dockerfilePorts []string
	if !isHerokuishContainer {
		dokkuDockerfilePorts := strings.Trim(config.GetWithDefault(appName, "DOKKU_DOCKERFILE_PORTS", ""), " ")
		if utf8.RuneCountInString(dokkuDockerfilePorts) > 0 {
			dockerfilePorts = strings.Split(dokkuDockerfilePorts, " ")
		}
	}

	var ports []string
	if len(dockerfilePorts) == 0 {
		ports = append(ports, "5000")
	} else {
		for _, port := range dockerfilePorts {
			port = strings.TrimSuffix(strings.TrimSpace(port), "/tcp")
			if port == "" || strings.HasSuffix(port, "/udp") {
				continue
			}
			ports = append(ports, port)
		}
	}
	fmt.Fprint(os.Stdout, strings.Join(ports, " "))
}

// TriggerNetworkConfigExists writes true or false to stdout whether a given app has network config
func TriggerNetworkConfigExists(appName string) {
	if HasNetworkConfig(appName) {
		fmt.Fprintln(os.Stdout, "true")
		return
	}

	fmt.Fprintln(os.Stdout, "false")
}

// TriggerNetworkGetIppaddr writes the ipaddress to stdout for a given app container
func TriggerNetworkGetIppaddr(appName string, processType string, containerID string) {
	ipAddress := GetContainerIpaddress(appName, processType, containerID)
	fmt.Fprintln(os.Stdout, ipAddress)
}

// TriggerNetworkGetListeners returns the listeners (host:port combinations) for a given app container
func TriggerNetworkGetListeners(appName string) {
	listeners := GetListeners(appName)
	fmt.Fprint(os.Stdout, strings.Join(listeners, " "))
}

// TriggerNetworkGetPort writes the port to stdout for a given app container
func TriggerNetworkGetPort(appName string, processType string, isHerokuishContainer bool, containerID string) {
	port := GetContainerPort(appName, processType, isHerokuishContainer, containerID)
	fmt.Fprintln(os.Stdout, port)
}

// TriggerNetworkGetProperty writes the network property to stdout for a given app container
func TriggerNetworkGetProperty(appName string, property string) {
	defaultValue := GetDefaultValue(property)
	value := common.PropertyGetDefault("network", appName, property, defaultValue)
	fmt.Fprintln(os.Stdout, value)
}

// TriggerNetworkWriteIpaddr writes the ip to disk
func TriggerNetworkWriteIpaddr(appName string, processType string, containerIndex string, ip string) {
	if appName == "" {
		common.LogFail("Please specify an app to run the command on")
	}
	err := common.VerifyAppName(appName)
	if err != nil {
		common.LogFail(err.Error())
	}

	appRoot := strings.Join([]string{common.MustGetEnv("DOKKU_ROOT"), appName}, "/")
	filename := fmt.Sprintf("%v/IP.%v.%v", appRoot, processType, containerIndex)
	f, err := os.Create(filename)
	if err != nil {
		common.LogFail(err.Error())
	}
	defer f.Close()

	ipBytes := []byte(ip)
	_, err = f.Write(ipBytes)
	if err != nil {
		common.LogFail(err.Error())
	}
}

// TriggerNetworkWritePort writes the port to disk
func TriggerNetworkWritePort(appName string, processType string, containerIndex string, port string) {
	if appName == "" {
		common.LogFail("Please specify an app to run the command on")
	}
	err := common.VerifyAppName(appName)
	if err != nil {
		common.LogFail(err.Error())
	}

	appRoot := strings.Join([]string{common.MustGetEnv("DOKKU_ROOT"), appName}, "/")
	filename := fmt.Sprintf("%v/PORT.%v.%v", appRoot, processType, containerIndex)
	f, err := os.Create(filename)
	if err != nil {
		common.LogFail(err.Error())
	}
	defer f.Close()

	portBytes := []byte(port)
	_, err = f.Write(portBytes)
	if err != nil {
		common.LogFail(err.Error())
	}
}

// TriggerPostAppCloneSetup cleans up network files for a new app clone
func TriggerPostAppCloneSetup(appName string) {
	success := PostAppCloneSetup(appName)
	if !success {
		os.Exit(1)
	}
}

// TriggerPostContainerCreate associates the container with a specified network
func TriggerPostContainerCreate(containerType string, containerID string, appName string, phase string, processType string) {
	if containerType != "app" {
		return
	}

	property := "attach-post-create"
	defaultValue := GetDefaultValue(property)
	networkName := common.PropertyGetDefault("network", appName, property, defaultValue)
	if networkName == "" {
		return
	}

	attachAppToNetwork(containerID, networkName, appName, phase, processType)
}

// TriggerPostCreate sets bind-all-interfaces to false by default
func TriggerPostCreate(appName string) {
	err := common.PropertyWrite("network", appName, "bind-all-interfaces", "false")
	if err != nil {
		common.LogWarn(err.Error())
	}
}

// TriggerPostDelete destroys the network property for a given app container
func TriggerPostDelete(appName string) {
	err := common.PropertyDestroy("network", appName)
	if err != nil {
		common.LogFail(err.Error())
	}
}

// TriggerCorePostDeploy associates the container with a specified network
func TriggerCorePostDeploy(appName string) {
	property := "attach-post-deploy"
	defaultValue := GetDefaultValue(property)
	networkName := common.PropertyGetDefault("network", appName, property, defaultValue)
	if networkName == "" {
		return
	}

	common.LogInfo1Quiet(fmt.Sprintf("Associating app with network %s", networkName))
	containerIDs, err := common.GetAppRunningContainerIDs(appName, "")
	if err != nil {
		common.LogFail(err.Error())
	}

	for _, containerID := range containerIDs {
		processType, err := common.DockerInspect(containerID, "{{ index .Config.Labels \"com.dokku.process-type\"}}")
		if err != nil {
			common.LogFail(err.Error())
		}
		attachAppToNetwork(containerID, networkName, appName, "deploy", processType)
	}
}
<commit_msg>feat: ensure network exists before attempting to attach it to a container<commit_after>package network

import (
	"fmt"
	"os"
	"strings"
	"unicode/utf8"

	"github.com/dokku/dokku/plugins/common"
	"github.com/dokku/dokku/plugins/config"
	"github.com/dokku/dokku/plugins/proxy"
)

// TriggerInstall runs the install step for the network plugin
func TriggerInstall() {
	if err := common.PropertySetup("network"); err != nil {
		common.LogFail(fmt.Sprintf("Unable to install the network plugin: %s", err.Error()))
	}

	apps, err := common.DokkuApps()
	if err != nil {
		return
	}
	for _, appName := range apps {
		if common.PropertyExists("network", appName, "bind-all-interfaces") {
			continue
		}
		if proxy.IsAppProxyEnabled(appName) {
			common.LogVerboseQuiet("Setting network property 'bind-all-interfaces' to false")
			if err := common.PropertyWrite("network", appName, "bind-all-interfaces", "false"); err != nil {
				common.LogWarn(err.Error())
			}
		} else {
			common.LogVerboseQuiet("Setting network property 'bind-all-interfaces' to true")
			if err := common.PropertyWrite("network", appName, "bind-all-interfaces", "true"); err != nil {
				common.LogWarn(err.Error())
			}
		}
	}
}

// TriggerNetworkComputePorts computes the ports for a given app container
func TriggerNetworkComputePorts(appName string, processType string, isHerokuishContainer bool) {
	if processType != "web" {
		return
	}

	var dockerfilePorts []string
	if !isHerokuishContainer {
		dokkuDockerfilePorts := strings.Trim(config.GetWithDefault(appName, "DOKKU_DOCKERFILE_PORTS", ""), " ")
		if utf8.RuneCountInString(dokkuDockerfilePorts) > 0 {
			dockerfilePorts = strings.Split(dokkuDockerfilePorts, " ")
		}
	}

	var ports []string
	if len(dockerfilePorts) == 0 {
		ports = append(ports, "5000")
	} else {
		for _, port := range dockerfilePorts {
			port = strings.TrimSuffix(strings.TrimSpace(port), "/tcp")
			if port == "" || strings.HasSuffix(port, "/udp") {
				continue
			}
			ports = append(ports, port)
		}
	}
	fmt.Fprint(os.Stdout, strings.Join(ports, " "))
}

// TriggerNetworkConfigExists writes true or false to stdout whether a given app has network config
func TriggerNetworkConfigExists(appName string) {
	if HasNetworkConfig(appName) {
		fmt.Fprintln(os.Stdout, "true")
		return
	}

	fmt.Fprintln(os.Stdout, "false")
}

// TriggerNetworkGetIppaddr writes the ipaddress to stdout for a given app container
func TriggerNetworkGetIppaddr(appName string, processType string, containerID string) {
	ipAddress := GetContainerIpaddress(appName, processType, containerID)
	fmt.Fprintln(os.Stdout, ipAddress)
}

// TriggerNetworkGetListeners returns the listeners (host:port combinations) for a given app container
func TriggerNetworkGetListeners(appName string) {
	listeners := GetListeners(appName)
	fmt.Fprint(os.Stdout, strings.Join(listeners, " "))
}

// TriggerNetworkGetPort writes the port to stdout for a given app container
func TriggerNetworkGetPort(appName string, processType string, isHerokuishContainer bool, containerID string) {
	port := GetContainerPort(appName, processType, isHerokuishContainer, containerID)
	fmt.Fprintln(os.Stdout, port)
}

// TriggerNetworkGetProperty writes the network property to stdout for a given app container
func TriggerNetworkGetProperty(appName string, property string) {
	defaultValue := GetDefaultValue(property)
	value := common.PropertyGetDefault("network", appName, property, defaultValue)
	fmt.Fprintln(os.Stdout, value)
}

// TriggerNetworkWriteIpaddr writes the ip to disk
func TriggerNetworkWriteIpaddr(appName string, processType string, containerIndex string, ip string) {
	if appName == "" {
		common.LogFail("Please specify an app to run the command on")
	}
	err := common.VerifyAppName(appName)
	if err != nil {
		common.LogFail(err.Error())
	}

	appRoot := strings.Join([]string{common.MustGetEnv("DOKKU_ROOT"), appName}, "/")
	filename := fmt.Sprintf("%v/IP.%v.%v", appRoot, processType, containerIndex)
	f, err := os.Create(filename)
	if err != nil {
		common.LogFail(err.Error())
	}
	defer f.Close()

	ipBytes := []byte(ip)
	_, err = f.Write(ipBytes)
	if err != nil {
		common.LogFail(err.Error())
	}
}

// TriggerNetworkWritePort writes the port to disk
func TriggerNetworkWritePort(appName string, processType string, containerIndex string, port string) {
	if appName == "" {
		common.LogFail("Please specify an app to run the command on")
	}
	err := common.VerifyAppName(appName)
	if err != nil {
		common.LogFail(err.Error())
	}

	appRoot := strings.Join([]string{common.MustGetEnv("DOKKU_ROOT"), appName}, "/")
	filename := fmt.Sprintf("%v/PORT.%v.%v", appRoot, processType, containerIndex)
	f, err := os.Create(filename)
	if err != nil {
		common.LogFail(err.Error())
	}
	defer f.Close()

	portBytes := []byte(port)
	_, err = f.Write(portBytes)
	if err != nil {
		common.LogFail(err.Error())
	}
}

// TriggerPostAppCloneSetup cleans up network files for a new app clone
func TriggerPostAppCloneSetup(appName string) {
	success := PostAppCloneSetup(appName)
	if !success {
		os.Exit(1)
	}
}

// TriggerPostContainerCreate associates the container with a specified network
func TriggerPostContainerCreate(containerType string, containerID string, appName string, phase string, processType string) {
	if containerType != "app" {
		return
	}

	property := "attach-post-create"
	defaultValue := GetDefaultValue(property)
	networkName := common.PropertyGetDefault("network", appName, property, defaultValue)
	if networkName == "" {
		return
	}

	exists, err := networkExists(networkName)
	if err != nil {
		common.LogFail(err.Error())
	}

	if !exists {
		common.LogFail(fmt.Sprintf("Network %v does not exist", networkName))
	}

	attachAppToNetwork(containerID, networkName, appName, phase, processType)
}

// TriggerPostCreate sets bind-all-interfaces to false by default
func TriggerPostCreate(appName string) {
	err := common.PropertyWrite("network", appName, "bind-all-interfaces", "false")
	if err != nil {
		common.LogWarn(err.Error())
	}
}

// TriggerPostDelete destroys the network property for a given app container
func TriggerPostDelete(appName string) {
	err := common.PropertyDestroy("network", appName)
	if err != nil {
		common.LogFail(err.Error())
	}
}

// TriggerCorePostDeploy associates the container with a specified network
func TriggerCorePostDeploy(appName string) {
	property := "attach-post-deploy"
	defaultValue := GetDefaultValue(property)
	networkName := common.PropertyGetDefault("network", appName, property, defaultValue)
	if networkName == "" {
		return
	}

	common.LogInfo1Quiet(fmt.Sprintf("Associating app with network %s", networkName))
	containerIDs, err := common.GetAppRunningContainerIDs(appName, "")
	if err != nil {
		common.LogFail(err.Error())
	}

	exists, err := networkExists(networkName)
	if err != nil {
		common.LogFail(err.Error())
	}

	if !exists {
		common.LogFail(fmt.Sprintf("Network %v does not exist", networkName))
	}

	for _, containerID := range containerIDs {
		processType, err := common.DockerInspect(containerID, "{{ index .Config.Labels \"com.dokku.process-type\"}}")
		if err != nil {
			common.LogFail(err.Error())
		}
		attachAppToNetwork(containerID, networkName, appName, "deploy", processType)
	}
}
<|endoftext|>


<commit_before>package plugins

import (
	"fmt"
	"testing"
)

func TestMatch(t *testing.T) {
	testStrings := make(map[string]string)
	testStrings["man: pledge"] = "https://man.openbsd.org/pledge"
	testStrings["man: 2 pledge"] = "https://man.openbsd.org/pledge.2"
	testStrings["man: unveil"] = "https://man.openbsd.org/unveil"
	testStrings["man: 3p vars"] = "https://man.openbsd.org/man3p/vars.3p"

	om := &OpenBSDMan{}
	for msg, resp := range testStrings {
		matched := fmt.Sprintf("https://man.openbsd.org/%s", om.fix(msg))
		if matched != resp {
			t.Errorf("OpenBSDMan expected %q; got %q (%q)", resp, matched, msg)
		}
	}
}
<commit_msg>better name for man test<commit_after>package plugins

import (
	"fmt"
	"testing"
)

func TestOpenBSDManfix(t *testing.T) {
	testStrings := make(map[string]string)
	testStrings["man: pledge"] = "https://man.openbsd.org/pledge"
	testStrings["man: 2 pledge"] = "https://man.openbsd.org/pledge.2"
	testStrings["man: unveil"] = "https://man.openbsd.org/unveil"
	testStrings["man: 3p vars"] = "https://man.openbsd.org/man3p/vars.3p"

	om := &OpenBSDMan{}
	for msg, resp := range testStrings {
		matched := fmt.Sprintf("https://man.openbsd.org/%s", om.fix(msg))
		if matched != resp {
			t.Errorf("OpenBSDMan expected %q; got %q (%q)", resp, matched, msg)
		}
	}
}
<|endoftext|>