<commit_before>package fabric

import (
	"bufio"
	"encoding/json"

	"github.com/jhunt/go-log"
	"golang.org/x/crypto/ssh"

	"github.com/shieldproject/shield/core/scheduler"
	"github.com/shieldproject/shield/core/vault"
	"github.com/shieldproject/shield/db"
)

func Legacy(ip string, config *ssh.ClientConfig, db *db.DB) LegacyFabric {
	return LegacyFabric{
		ip:  ip,
		ssh: config,
		db:  db,
	}
}

type LegacyFabric struct {
	ip  string
	ssh *ssh.ClientConfig
	db  *db.DB
}

type Command struct {
	Op string `json:"operation"`

	TargetPlugin   string `json:"target_plugin,omitempty"`
	TargetEndpoint string `json:"target_endpoint,omitempty"`

	StorePlugin   string `json:"store_plugin,omitempty"`
	StoreEndpoint string `json:"store_endpoint,omitempty"`

	RestoreKey string `json:"restore_key,omitempty"`

	EncryptType string `json:"encrypt_type,omitempty"`
	EncryptKey  string `json:"encrypt_key,omitempty"`
	EncryptIV   string `json:"encrypt_iv,omitempty"`

	Compression string `json:"compression,omitempty"`
}

func (f LegacyFabric) Backup(task *db.Task, encryption vault.Parameters) scheduler.Chore {
	op := "backup"

	chore := f.Execute(op, task.UUID, Command{
		Op: op,

		TargetPlugin:   task.TargetPlugin,
		TargetEndpoint: task.TargetEndpoint,

		StorePlugin:   task.StorePlugin,
		StoreEndpoint: task.StoreEndpoint,

		Compression: task.Compression,

		EncryptType: encryption.Type,
		EncryptKey:  encryption.Key,
		EncryptIV:   encryption.IV,
	})

	chore.Encryption = encryption.Type
	return chore
}

func (f LegacyFabric) Restore(task *db.Task, encryption vault.Parameters) scheduler.Chore {
	op := "restore"

	return f.Execute(op, task.UUID, Command{
		Op: op,

		RestoreKey:     task.RestoreKey,
		TargetPlugin:   task.TargetPlugin,
		TargetEndpoint: task.TargetEndpoint,

		StorePlugin:   task.StorePlugin,
		StoreEndpoint: task.StoreEndpoint,

		Compression: task.Compression,

		EncryptType: encryption.Type,
		EncryptKey:  encryption.Key,
		EncryptIV:   encryption.IV,
	})
}

func (f LegacyFabric) Status(task *db.Task) scheduler.Chore {
	return f.Execute("agent status", task.UUID, Command{
		Op: "status",
	})
}

func (f LegacyFabric) Purge(task *db.Task) scheduler.Chore {
	return f.Execute("archive purge", task.UUID, Command{
		Op: "purge",

		RestoreKey:    task.RestoreKey,
		StorePlugin:   task.StorePlugin,
		StoreEndpoint: task.StoreEndpoint,
	})
}

func (f LegacyFabric) TestStore(task *db.Task) scheduler.Chore {
	op := "storage test"

	return f.Execute(op, task.UUID, Command{
		Op: "test-store",

		StorePlugin:   task.StorePlugin,
		StoreEndpoint: task.StoreEndpoint,
	})
}

func (f LegacyFabric) Execute(op, id string, command Command) scheduler.Chore {
	return scheduler.NewChore(
		id,
		func(chore scheduler.Chore) {
			log.Debugf("starting up legacy agent execution...")
			log.Debugf("checking that we have a SHIELD agent...")
			if f.ip == "" {
				chore.Errorf("ERR> unable to determine SHIELD agent to connect to")
				chore.UnixExit(2)
				return
			}

			log.Debugf("marshaling command into JSON for transport across SSH (legacy) fabric...")
			b, err := json.Marshal(command)
			if err != nil {
				chore.Errorf("ERR> unable to marshal %s task payload: %s", op, err)
				chore.UnixExit(2)
				return
			}
			payload := string(b)

			chore.Errorf("connecting to %s (tcp/ipv4)", f.ip)
			conn, err := ssh.Dial("tcp4", f.ip, f.ssh)
			if err != nil {
				chore.Errorf("ERR> unable to connect to %s: %s", err)
				chore.UnixExit(2)
				return
			}
			defer conn.Close()

			chore.Errorf("connected to %s...", f.ip)
			sess, err := conn.NewSession()
			if err != nil {
				chore.Errorf("ERR> unable to create a new execution session against %s: %s", f.ip, err)
				return
			}
			defer sess.Close()

			/* set up an output sink on ssh output pipe */
			pipe, err := sess.StdoutPipe()
			if err != nil {
				chore.Errorf("ERR> unable to redirect standard output from remote execution session: %s", err)
				return
			}

			/* we do this in a goroutine so that we can
			   exec the payload in the main thread. */
			wait := make(chan bool)
			go func() {
				/* on the other side of the ssh session,
				   the shield-agent process combines standard
				   output and standard error into a single
				   stream, prefixing each line with either
				   "O:" (stdout) or "E:" (stderr). */
				b := bufio.NewScanner(pipe)
				for b.Scan() {
					s := b.Text()
					switch s[:2] {
					case "O:":
						chore.Infof("%s", s[2:])
					case "E:":
						chore.Errorf("%s", s[2:])
					}
				}

				wait <- true
			}()

			/* execute the payload remotely */
			chore.Errorf("executing %s task on remote agent.", op)
			err = sess.Run(payload)
			<-wait
			if err != nil {
				chore.Errorf("ERR> remote execution failed: %s", err)
				chore.UnixExit(1)
				return
			}

			chore.UnixExit(0)
		})
}
<commit_msg>Fix !%s(MISSING) in agent error messaging<commit_after>package fabric

import (
	"bufio"
	"encoding/json"

	"github.com/jhunt/go-log"
	"golang.org/x/crypto/ssh"

	"github.com/shieldproject/shield/core/scheduler"
	"github.com/shieldproject/shield/core/vault"
	"github.com/shieldproject/shield/db"
)

func Legacy(ip string, config *ssh.ClientConfig, db *db.DB) LegacyFabric {
	return LegacyFabric{
		ip:  ip,
		ssh: config,
		db:  db,
	}
}

type LegacyFabric struct {
	ip  string
	ssh *ssh.ClientConfig
	db  *db.DB
}

type Command struct {
	Op string `json:"operation"`

	TargetPlugin   string `json:"target_plugin,omitempty"`
	TargetEndpoint string `json:"target_endpoint,omitempty"`

	StorePlugin   string `json:"store_plugin,omitempty"`
	StoreEndpoint string `json:"store_endpoint,omitempty"`

	RestoreKey string `json:"restore_key,omitempty"`

	EncryptType string `json:"encrypt_type,omitempty"`
	EncryptKey  string `json:"encrypt_key,omitempty"`
	EncryptIV   string `json:"encrypt_iv,omitempty"`

	Compression string `json:"compression,omitempty"`
}

func (f LegacyFabric) Backup(task *db.Task, encryption vault.Parameters) scheduler.Chore {
	op := "backup"

	chore := f.Execute(op, task.UUID, Command{
		Op: op,

		TargetPlugin:   task.TargetPlugin,
		TargetEndpoint: task.TargetEndpoint,

		StorePlugin:   task.StorePlugin,
		StoreEndpoint: task.StoreEndpoint,

		Compression: task.Compression,

		EncryptType: encryption.Type,
		EncryptKey:  encryption.Key,
		EncryptIV:   encryption.IV,
	})

	chore.Encryption = encryption.Type
	return chore
}

func (f LegacyFabric) Restore(task *db.Task, encryption vault.Parameters) scheduler.Chore {
	op := "restore"

	return f.Execute(op, task.UUID, Command{
		Op: op,

		RestoreKey:     task.RestoreKey,
		TargetPlugin:   task.TargetPlugin,
		TargetEndpoint: task.TargetEndpoint,

		StorePlugin:   task.StorePlugin,
		StoreEndpoint: task.StoreEndpoint,

		Compression: task.Compression,

		EncryptType: encryption.Type,
		EncryptKey:  encryption.Key,
		EncryptIV:   encryption.IV,
	})
}

func (f LegacyFabric) Status(task *db.Task) scheduler.Chore {
	return f.Execute("agent status", task.UUID, Command{
		Op: "status",
	})
}

func (f LegacyFabric) Purge(task *db.Task) scheduler.Chore {
	return f.Execute("archive purge", task.UUID, Command{
		Op: "purge",

		RestoreKey:    task.RestoreKey,
		StorePlugin:   task.StorePlugin,
		StoreEndpoint: task.StoreEndpoint,
	})
}

func (f LegacyFabric) TestStore(task *db.Task) scheduler.Chore {
	op := "storage test"

	return f.Execute(op, task.UUID, Command{
		Op: "test-store",

		StorePlugin:   task.StorePlugin,
		StoreEndpoint: task.StoreEndpoint,
	})
}

func (f LegacyFabric) Execute(op, id string, command Command) scheduler.Chore {
	return scheduler.NewChore(
		id,
		func(chore scheduler.Chore) {
			log.Debugf("starting up legacy agent execution...")
			log.Debugf("checking that we have a SHIELD agent...")
			if f.ip == "" {
				chore.Errorf("ERR> unable to determine SHIELD agent to connect to")
				chore.UnixExit(2)
				return
			}

			log.Debugf("marshaling command into JSON for transport across SSH (legacy) fabric...")
			b, err := json.Marshal(command)
			if err != nil {
				chore.Errorf("ERR> unable to marshal %s task payload: %s", op, err)
				chore.UnixExit(2)
				return
			}
			payload := string(b)

			chore.Errorf("connecting to %s (tcp/ipv4)", f.ip)
			conn, err := ssh.Dial("tcp4", f.ip, f.ssh)
			if err != nil {
				chore.Errorf("ERR> unable to connect to %s: %s", f.ip, err)
				chore.UnixExit(2)
				return
			}
			defer conn.Close()

			chore.Errorf("connected to %s...", f.ip)
			sess, err := conn.NewSession()
			if err != nil {
				chore.Errorf("ERR> unable to create a new execution session against %s: %s", f.ip, err)
				return
			}
			defer sess.Close()

			/* set up an output sink on ssh output pipe */
			pipe, err := sess.StdoutPipe()
			if err != nil {
				chore.Errorf("ERR> unable to redirect standard output from remote execution session: %s", err)
				return
			}

			/* we do this in a goroutine so that we can
			   exec the payload in the main thread. */
			wait := make(chan bool)
			go func() {
				/* on the other side of the ssh session,
				   the shield-agent process combines standard
				   output and standard error into a single
				   stream, prefixing each line with either
				   "O:" (stdout) or "E:" (stderr). */
				b := bufio.NewScanner(pipe)
				for b.Scan() {
					s := b.Text()
					switch s[:2] {
					case "O:":
						chore.Infof("%s", s[2:])
					case "E:":
						chore.Errorf("%s", s[2:])
					}
				}

				wait <- true
			}()

			/* execute the payload remotely */
			chore.Errorf("executing %s task on remote agent.", op)
			err = sess.Run(payload)
			<-wait
			if err != nil {
				chore.Errorf("ERR> remote execution failed: %s", err)
				chore.UnixExit(1)
				return
			}

			chore.UnixExit(0)
		})
}
<|endoftext|>


<commit_before>// Copyright 2015 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package oem

import (
	"github.com/coreos/ignition/src/registry"
)

// Config represents a set of command line flags that map to a particular OEM.
type Config struct {
	name  string
	flags map[string]string
}

func (c Config) Name() string {
	return c.name
}

func (c Config) Flags() map[string]string {
	return c.flags
}

var configs = registry.Create("oem configs")

func init() {
	configs.Register(Config{
		name: "azure",
		flags: map[string]string{
			"provider": "noop",
		},
	})
	configs.Register(Config{
		name: "cloudsigma",
		flags: map[string]string{
			"provider": "noop",
		},
	})
	configs.Register(Config{
		name: "cloudstack",
		flags: map[string]string{
			"provider": "noop",
		},
	})
	configs.Register(Config{
		name: "digitalocean",
		flags: map[string]string{
			"provider": "noop",
		},
	})
	configs.Register(Config{
		name: "brightbox",
		flags: map[string]string{
			"provider": "noop",
		},
	})
	configs.Register(Config{
		name: "openstack",
		flags: map[string]string{
			"provider": "noop",
		},
	})
	configs.Register(Config{
		name: "ec2",
		flags: map[string]string{
			"provider": "noop",
		},
	})
	configs.Register(Config{
		name: "exoscale",
		flags: map[string]string{
			"provider": "noop",
		},
	})
	configs.Register(Config{
		name: "gce",
		flags: map[string]string{
			"provider": "noop",
		},
	})
	configs.Register(Config{
		name: "hyperv",
		flags: map[string]string{
			"provider": "noop",
		},
	})
	configs.Register(Config{
		name: "niftycloud",
		flags: map[string]string{
			"provider": "noop",
		},
	})
	configs.Register(Config{
		name: "packet",
		flags: map[string]string{
			"provider": "noop",
		},
	})
	configs.Register(Config{
		name: "pxe",
		flags: map[string]string{
			"provider": "cmdline",
		},
	})
	configs.Register(Config{
		name: "rackspace",
		flags: map[string]string{
			"provider": "noop",
		},
	})
	configs.Register(Config{
		name: "rackspace-onmetal",
		flags: map[string]string{
			"provider": "noop",
		},
	})
	configs.Register(Config{
		name: "vagrant",
		flags: map[string]string{
			"provider": "noop",
		},
	})
	configs.Register(Config{
		name: "vmware",
		flags: map[string]string{
			"provider": "noop",
		},
	})
	configs.Register(Config{
		name: "xendom0",
		flags: map[string]string{
			"provider": "noop",
		},
	})
}

func Get(name string) (config Config, ok bool) {
	config, ok = configs.Get(name).(Config)
	return
}

func Names() (names []string) {
	return configs.Names()
}
<commit_msg>Revert "oem: disable ec2"<commit_after>// Copyright 2015 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package oem

import (
	"github.com/coreos/ignition/src/registry"
)

// Config represents a set of command line flags that map to a particular OEM.
type Config struct {
	name  string
	flags map[string]string
}

func (c Config) Name() string {
	return c.name
}

func (c Config) Flags() map[string]string {
	return c.flags
}

var configs = registry.Create("oem configs")

func init() {
	configs.Register(Config{
		name: "azure",
		flags: map[string]string{
			"provider": "noop",
		},
	})
	configs.Register(Config{
		name: "cloudsigma",
		flags: map[string]string{
			"provider": "noop",
		},
	})
	configs.Register(Config{
		name: "cloudstack",
		flags: map[string]string{
			"provider": "noop",
		},
	})
	configs.Register(Config{
		name: "digitalocean",
		flags: map[string]string{
			"provider": "noop",
		},
	})
	configs.Register(Config{
		name: "brightbox",
		flags: map[string]string{
			"provider": "noop",
		},
	})
	configs.Register(Config{
		name: "openstack",
		flags: map[string]string{
			"provider": "noop",
		},
	})
	configs.Register(Config{
		name: "ec2",
		flags: map[string]string{
			"provider": "ec2",
		},
	})
	configs.Register(Config{
		name: "exoscale",
		flags: map[string]string{
			"provider": "noop",
		},
	})
	configs.Register(Config{
		name: "gce",
		flags: map[string]string{
			"provider": "noop",
		},
	})
	configs.Register(Config{
		name: "hyperv",
		flags: map[string]string{
			"provider": "noop",
		},
	})
	configs.Register(Config{
		name: "niftycloud",
		flags: map[string]string{
			"provider": "noop",
		},
	})
	configs.Register(Config{
		name: "packet",
		flags: map[string]string{
			"provider": "noop",
		},
	})
	configs.Register(Config{
		name: "pxe",
		flags: map[string]string{
			"provider": "cmdline",
		},
	})
	configs.Register(Config{
		name: "rackspace",
		flags: map[string]string{
			"provider": "noop",
		},
	})
	configs.Register(Config{
		name: "rackspace-onmetal",
		flags: map[string]string{
			"provider": "noop",
		},
	})
	configs.Register(Config{
		name: "vagrant",
		flags: map[string]string{
			"provider": "noop",
		},
	})
	configs.Register(Config{
		name: "vmware",
		flags: map[string]string{
			"provider": "noop",
		},
	})
	configs.Register(Config{
		name: "xendom0",
		flags: map[string]string{
			"provider": "noop",
		},
	})
}

func Get(name string) (config Config, ok bool) {
	config, ok = configs.Get(name).(Config)
	return
}

func Names() (names []string) {
	return configs.Names()
}
<|endoftext|>


<commit_before>package core

import (
	. "github.com/smartystreets/goconvey/convey"
	"pfi/sensorbee/sensorbee/core/tuple"
	"testing"
)

type DummyTopology struct{}

func (this *DummyTopology) Run() {}

type DummyTopologyBuilder struct{}

func (this *DummyTopologyBuilder) AddSource(name string, source Source) SourceDeclarer {
	return &DummySourceDeclarer{}
}
func (this *DummyTopologyBuilder) AddBox(name string, box Box) BoxDeclarer {
	return &DummyBoxDeclarer{}
}
func (this *DummyTopologyBuilder) AddSink(name string, sink Sink) SinkDeclarer {
	return &DummySinkDeclarer{}
}
func (this *DummyTopologyBuilder) Build() Topology {
	return &DummyTopology{}
}

type DummySourceDeclarer struct{}

func (this *DummySourceDeclarer) Err() error {
	return nil
}

type DummyBoxDeclarer struct{}

func (this *DummyBoxDeclarer) Input(name string, schema *Schema) BoxDeclarer {
	return &DummyBoxDeclarer{}
}

func (this *DummyBoxDeclarer) Err() error {
	return nil
}

type DummySinkDeclarer struct{}

func (this *DummySinkDeclarer) Input(name string) SinkDeclarer {
	return &DummySinkDeclarer{}
}

func (this *DummySinkDeclarer) Err() error {
	return nil
}

type DummySource struct{}

func (this *DummySource) GenerateStream(w Writer) error {
	return nil
}
func (this *DummySource) Schema() *Schema {
	var s Schema = Schema("test")
	return &s
}

type DummyBox struct{}

func (this *DummyBox) Init(ctx *Context) error {
	return nil
}
func (this *DummyBox) Process(t *tuple.Tuple, s Writer) error {
	return nil
}
func (this *DummyBox) RequiredInputSchema() ([]*Schema, error) {
	return []*Schema{nil}, nil
}
func (this *DummyBox) OutputSchema(s []*Schema) (*Schema, error) {
	return nil, nil
}

type DummySink struct{}

func (this *DummySink) Write(t *tuple.Tuple) error {
	return nil
}

func TestTopology(t *testing.T) {
	Convey("Given dummy topology builder, to build topology", t, func() {

		source := &DummySource{}
		tb := DummyTopologyBuilder{}
		tb.AddSource("test_source1", source)

		box := &DummyBox{}
		inputSchema, _ := box.RequiredInputSchema()
		tb.AddBox("test_box1", box).
			Input("test_input_schema", inputSchema[0])

		t := tb.Build()

		Convey("It should satisfy Topology interface", func() {
			So(t, ShouldNotBeNil)
		})
	})

	Convey("Given dummy topology builder, to build complex topology", t, func() {

		so1 := &DummySource{}
		so2 := &DummySource{}
		b1 := &DummyBox{}
		b2 := &DummyBox{}
		b3 := &DummyBox{}
		si1 := &DummySink{}
		si2 := &DummySink{}

		tb := DummyTopologyBuilder{}

		var err DeclarerError
		err = tb.AddSource("so1", so1)
		So(err.Err(), ShouldBeNil)
		err = tb.AddSource("so2", so2)
		So(err.Err(), ShouldBeNil)
		err = tb.AddBox("b1", b1).
			Input("so1", nil).
			Input("so2", nil)
		So(err.Err(), ShouldBeNil)
		err = tb.AddBox("b2", b2).
			Input("so1", nil)
		So(err.Err(), ShouldBeNil)
		err = tb.AddBox("b3", b3).
			Input("b1", nil)
		So(err.Err(), ShouldBeNil)
		err = tb.AddSink("si1", si1).
			Input("b2").
			Input("b1").
			Input("so1")
		So(err.Err(), ShouldBeNil)
		err = tb.AddSink("si2", si2).
			Input("b2").
			Input("b3")
		So(err.Err(), ShouldBeNil)

	})
}
<commit_msg>replace 'this' by appropriate replacements in topology test<commit_after>package core

import (
	. "github.com/smartystreets/goconvey/convey"
	"pfi/sensorbee/sensorbee/core/tuple"
	"testing"
)

type DummyTopology struct{}

func (dt *DummyTopology) Run() {}

type DummyTopologyBuilder struct{}

func (dtb *DummyTopologyBuilder) AddSource(name string, source Source) SourceDeclarer {
	return &DummySourceDeclarer{}
}
func (dtb *DummyTopologyBuilder) AddBox(name string, box Box) BoxDeclarer {
	return &DummyBoxDeclarer{}
}
func (dtb *DummyTopologyBuilder) AddSink(name string, sink Sink) SinkDeclarer {
	return &DummySinkDeclarer{}
}
func (dtb *DummyTopologyBuilder) Build() Topology {
	return &DummyTopology{}
}

type DummySourceDeclarer struct{}

func (dsd *DummySourceDeclarer) Err() error {
	return nil
}

type DummyBoxDeclarer struct{}

func (dbd *DummyBoxDeclarer) Input(name string, schema *Schema) BoxDeclarer {
	return &DummyBoxDeclarer{}
}

func (dbd *DummyBoxDeclarer) Err() error {
	return nil
}

type DummySinkDeclarer struct{}

func (dsd *DummySinkDeclarer) Input(name string) SinkDeclarer {
	return &DummySinkDeclarer{}
}

func (dsd *DummySinkDeclarer) Err() error {
	return nil
}

type DummySource struct{}

func (ds *DummySource) GenerateStream(w Writer) error {
	return nil
}
func (ds *DummySource) Schema() *Schema {
	var s Schema = Schema("test")
	return &s
}

type DummyBox struct{}

func (db *DummyBox) Init(ctx *Context) error {
	return nil
}
func (db *DummyBox) Process(t *tuple.Tuple, s Writer) error {
	return nil
}
func (db *DummyBox) RequiredInputSchema() ([]*Schema, error) {
	return []*Schema{nil}, nil
}
func (db *DummyBox) OutputSchema(s []*Schema) (*Schema, error) {
	return nil, nil
}

type DummySink struct{}

func (ds *DummySink) Write(t *tuple.Tuple) error {
	return nil
}

func TestTopology(t *testing.T) {
	Convey("Given dummy topology builder, to build topology", t, func() {

		source := &DummySource{}
		tb := DummyTopologyBuilder{}
		tb.AddSource("test_source1", source)

		box := &DummyBox{}
		inputSchema, _ := box.RequiredInputSchema()
		tb.AddBox("test_box1", box).
			Input("test_input_schema", inputSchema[0])

		t := tb.Build()

		Convey("It should satisfy Topology interface", func() {
			So(t, ShouldNotBeNil)
		})
	})

	Convey("Given dummy topology builder, to build complex topology", t, func() {

		so1 := &DummySource{}
		so2 := &DummySource{}
		b1 := &DummyBox{}
		b2 := &DummyBox{}
		b3 := &DummyBox{}
		si1 := &DummySink{}
		si2 := &DummySink{}

		tb := DummyTopologyBuilder{}

		var err DeclarerError
		err = tb.AddSource("so1", so1)
		So(err.Err(), ShouldBeNil)
		err = tb.AddSource("so2", so2)
		So(err.Err(), ShouldBeNil)
		err = tb.AddBox("b1", b1).
			Input("so1", nil).
			Input("so2", nil)
		So(err.Err(), ShouldBeNil)
		err = tb.AddBox("b2", b2).
			Input("so1", nil)
		So(err.Err(), ShouldBeNil)
		err = tb.AddBox("b3", b3).
			Input("b1", nil)
		So(err.Err(), ShouldBeNil)
		err = tb.AddSink("si1", si1).
			Input("b2").
			Input("b1").
			Input("so1")
		So(err.Err(), ShouldBeNil)
		err = tb.AddSink("si2", si2).
			Input("b2").
			Input("b3")
		So(err.Err(), ShouldBeNil)

	})
}
<|endoftext|>


<commit_before>// Copyright 2015 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package oem

import (
	"github.com/coreos/ignition/src/registry"
)

// Config represents a set of command line flags that map to a particular OEM.
type Config struct {
	name  string
	flags map[string]string
}

func (c Config) Name() string {
	return c.name
}

func (c Config) Flags() map[string]string {
	return c.flags
}

var configs = registry.Create("oem configs")

func init() {
	configs.Register(Config{
		name: "pxe",
		flags: map[string]string{
			"provider": "cmdline",
		},
	})

	configs.Register(Config{
		name: "ec2",
		flags: map[string]string{
			"provider": "ec2",
		},
	})
}

func Get(name string) (config Config, ok bool) {
	config, ok = configs.Get(name).(Config)
	return
}

func Names() (names []string) {
	return configs.Names()
}
<commit_msg>oem: add all of the OEMS supported by CoreOS<commit_after>// Copyright 2015 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package oem

import (
	"github.com/coreos/ignition/src/registry"
)

// Config represents a set of command line flags that map to a particular OEM.
type Config struct {
	name  string
	flags map[string]string
}

func (c Config) Name() string {
	return c.name
}

func (c Config) Flags() map[string]string {
	return c.flags
}

var configs = registry.Create("oem configs")

func init() {
	configs.Register(Config{
		name: "azure",
		flags: map[string]string{
			"provider": "noop",
		},
	})
	configs.Register(Config{
		name: "cloudsigma",
		flags: map[string]string{
			"provider": "noop",
		},
	})
	configs.Register(Config{
		name: "cloudstack",
		flags: map[string]string{
			"provider": "noop",
		},
	})
	configs.Register(Config{
		name: "digitalocean",
		flags: map[string]string{
			"provider": "noop",
		},
	})
	configs.Register(Config{
		name: "brightbox",
		flags: map[string]string{
			"provider": "noop",
		},
	})
	configs.Register(Config{
		name: "openstack",
		flags: map[string]string{
			"provider": "noop",
		},
	})
	configs.Register(Config{
		name: "ec2",
		flags: map[string]string{
			"provider": "ec2",
		},
	})
	configs.Register(Config{
		name: "exoscale",
		flags: map[string]string{
			"provider": "noop",
		},
	})
	configs.Register(Config{
		name: "gce",
		flags: map[string]string{
			"provider": "noop",
		},
	})
	configs.Register(Config{
		name: "hyperv",
		flags: map[string]string{
			"provider": "noop",
		},
	})
	configs.Register(Config{
		name: "niftycloud",
		flags: map[string]string{
			"provider": "noop",
		},
	})
	configs.Register(Config{
		name: "packet",
		flags: map[string]string{
			"provider": "noop",
		},
	})
	configs.Register(Config{
		name: "pxe",
		flags: map[string]string{
			"provider": "cmdline",
		},
	})
	configs.Register(Config{
		name: "rackspace",
		flags: map[string]string{
			"provider": "noop",
		},
	})
	configs.Register(Config{
		name: "rackspace-onmetal",
		flags: map[string]string{
			"provider": "noop",
		},
	})
	configs.Register(Config{
		name: "vagrant",
		flags: map[string]string{
			"provider": "noop",
		},
	})
	configs.Register(Config{
		name: "vmware",
		flags: map[string]string{
			"provider": "noop",
		},
	})
	configs.Register(Config{
		name: "xendom0",
		flags: map[string]string{
			"provider": "noop",
		},
	})
}

func Get(name string) (config Config, ok bool) {
	config, ok = configs.Get(name).(Config)
	return
}

func Names() (names []string) {
	return configs.Names()
}
<|endoftext|>


<commit_before>package criteria

import (
	"github.com/viant/assertly"
	"github.com/viant/toolbox"
	"github.com/viant/toolbox/data"
)

//Criterion represent evaluation criterion
type Criterion struct {
	*Predicate
	LeftOperand  interface{}
	Operator     string
	RightOperand interface{}
}

func (c *Criterion) expandOperand(opperand interface{}, state data.Map) interface{} {
	if opperand == nil {
		return nil
	}
	return state.Expand(opperand)
}

//Apply evaluates criterion with supplied context and state map . Dolar prefixed $expression will be expanded before evaluation.
func (c *Criterion) Apply(state data.Map) (bool, error) {
	if c.Predicate != nil && len(c.Predicate.Criteria) > 0 {
		return c.Predicate.Apply(state)
	}
	leftOperand := c.expandOperand(c.LeftOperand, state)
	rightOperand := c.expandOperand(c.RightOperand, state)
	var err error
	var leftNumber, rightNumber float64
	var rootPath = assertly.NewDataPath("/")
	var context = assertly.NewDefaultContext()

	if text, ok := leftOperand.(string); ok {
		switch text {
		case "t", "T", "true", "TRUE", "True":
			leftOperand = true
		case "f", "F", "false", "FALSE", "False":
			leftOperand = false
		}
	}

	if rightOperand == nil {
		switch leftOperand.(type) {
		case bool:
			rightOperand = false
		case string:
			rightOperand = ""
		}
	}

	switch c.Operator {
	case "=", ":":
		validation, err := assertly.AssertWithContext(rightOperand, leftOperand, rootPath, context)
		if err != nil {
			return false, err
		}
		return validation.FailedCount == 0, nil
	case "!=", "":
		validation, err := assertly.AssertWithContext(leftOperand, rightOperand, rootPath, context)
		if err != nil {
			return false, err
		}
		return validation.FailedCount > 0, nil
	case ">=":
		if leftNumber, err = toolbox.ToFloat(leftOperand); err == nil {
			if rightNumber, err = toolbox.ToFloat(rightOperand); err == nil {
				return leftNumber >= rightNumber, nil
			}
		}
	case "<=":
		if leftNumber, err = toolbox.ToFloat(leftOperand); err == nil {
			if rightNumber, err = toolbox.ToFloat(rightOperand); err == nil {
				return leftNumber <= rightNumber, nil
			}
		}

	case ">":
		if leftNumber, err = toolbox.ToFloat(leftOperand); err == nil {
			if rightNumber, err = toolbox.ToFloat(rightOperand); err == nil {
				return leftNumber > rightNumber, nil
			}
		}
	case "<":
		if leftNumber, err = toolbox.ToFloat(leftOperand); err == nil {
			if rightNumber, err = toolbox.ToFloat(rightOperand); err == nil {
				return leftNumber < rightNumber, nil
			}
		}
	}
	return false, err
}

//NewCriterion creates a new criterion
func NewCriterion(leftOperand interface{}, operator string, rightOperand interface{}) *Criterion {
	return &Criterion{
		LeftOperand:  leftOperand,
		Operator:     operator,
		RightOperand: rightOperand,
	}
}
<commit_msg>updated error message<commit_after>package criteria

import (
	"fmt"
	"github.com/viant/assertly"
	"github.com/viant/toolbox"
	"github.com/viant/toolbox/data"
)

//Criterion represent evaluation criterion
type Criterion struct {
	*Predicate
	LeftOperand  interface{}
	Operator     string
	RightOperand interface{}
}

func (c *Criterion) expandOperand(opperand interface{}, state data.Map) interface{} {
	if opperand == nil {
		return nil
	}
	return state.Expand(opperand)
}

func checkUndefined(err error, left, right interface{}, operator string) error {
	if err != nil {
		if text, ok := left.(string); ok {
			return fmt.Errorf("undefined %v  in expression: %v %s %v", text, left, operator, right)
		}
		if text, ok := right.(string); ok {
			return fmt.Errorf("undefined %v  in expression: %v %s %v", text, left, operator, right)
		}

	}
	return err
}

//Apply evaluates criterion with supplied context and state map . Dolar prefixed $expression will be expanded before evaluation.
func (c *Criterion) Apply(state data.Map) (bool, error) {
	if c.Predicate != nil && len(c.Predicate.Criteria) > 0 {
		return c.Predicate.Apply(state)
	}
	leftOperand := c.expandOperand(c.LeftOperand, state)
	rightOperand := c.expandOperand(c.RightOperand, state)
	var err error
	var leftNumber, rightNumber float64
	var rootPath = assertly.NewDataPath("/")
	var context = assertly.NewDefaultContext()

	if text, ok := leftOperand.(string); ok {
		switch text {
		case "t", "T", "true", "TRUE", "True":
			leftOperand = true
		case "f", "F", "false", "FALSE", "False":
			leftOperand = false
		}
	}

	if rightOperand == nil {
		switch leftOperand.(type) {
		case bool:
			rightOperand = false
		case string:
			rightOperand = ""
		}
	}
	switch c.Operator {
	case "=", ":":
		validation, err := assertly.AssertWithContext(rightOperand, leftOperand, rootPath, context)
		if err != nil {
			return false, err
		}
		return validation.FailedCount == 0, nil
	case "!=", "":
		validation, err := assertly.AssertWithContext(leftOperand, rightOperand, rootPath, context)
		if err != nil {
			return false, err
		}
		return validation.FailedCount > 0, nil
	case ">=":
		if leftNumber, err = toolbox.ToFloat(leftOperand); err == nil {
			if rightNumber, err = toolbox.ToFloat(rightOperand); err == nil {
				return leftNumber >= rightNumber, nil
			}
		}
	case "<=":
		if leftNumber, err = toolbox.ToFloat(leftOperand); err == nil {
			if rightNumber, err = toolbox.ToFloat(rightOperand); err == nil {
				return leftNumber <= rightNumber, nil
			}
		}

	case ">":
		if leftNumber, err = toolbox.ToFloat(leftOperand); err == nil {
			if rightNumber, err = toolbox.ToFloat(rightOperand); err == nil {
				return leftNumber > rightNumber, nil
			}
		}
	case "<":
		if leftNumber, err = toolbox.ToFloat(leftOperand); err == nil {
			if rightNumber, err = toolbox.ToFloat(rightOperand); err == nil {
				return leftNumber < rightNumber, nil
			}
		}
	}
	err = checkUndefined(err, leftOperand, rightNumber, c.Operator)
	return false, err
}

//NewCriterion creates a new criterion
func NewCriterion(leftOperand interface{}, operator string, rightOperand interface{}) *Criterion {
	return &Criterion{
		LeftOperand:  leftOperand,
		Operator:     operator,
		RightOperand: rightOperand,
	}
}
<|endoftext|>


<commit_before>package statsd

import (
	"bytes"
	"fmt"
	"io"
	"log"
	"net"
	"strconv"
)

// DefaultMetricsAddr is the default address on which a MetricReceiver will listen
const DefaultMetricsAddr = ":8125"

// Objects implementing the Handler interface can be used to handle metrics for a MetricReceiver
type Handler interface {
	HandleMetric(m Metric)
}

// The HandlerFunc type is an adapter to allow the use of ordinary functions as metric handlers
type HandlerFunc func(Metric)

// HandleMetric calls f(m)
func (f HandlerFunc) HandleMetric(m Metric) {
	f(m)
}

// MetricReceiver receives data on its listening port and converts lines in to Metrics.
// For each Metric it calls r.Handler.HandleMetric()
type MetricReceiver struct {
	Addr    string  // UDP address on which to listen for metrics
	Handler Handler // handler to invoke
}

// ListenAndReceive listens on the UDP network address of srv.Addr and then calls
// Receive to handle the incoming datagrams. If Addr is blank then DefaultMetricsAddr is used.
func (r *MetricReceiver) ListenAndReceive() error {
	addr := r.Addr
	if addr == "" {
		addr = DefaultMetricsAddr
	}
	c, err := net.ListenPacket("udp", addr)
	if err != nil {
		return err
	}
	return r.Receive(c)
}

// Receive accepts incoming datagrams on c and calls r.Handler.HandleMetric() for each line in the
// datagram that successfully parses in to a Metric
func (r *MetricReceiver) Receive(c net.PacketConn) error {
	defer c.Close()

	msg := make([]byte, 1024)
	for {
		nbytes, addr, err := c.ReadFrom(msg)
		if err != nil {
			log.Printf("%s", err)
			continue
		}
		buf := make([]byte, nbytes)
		copy(buf, msg[:nbytes])
		go r.handleMessage(addr, buf)
	}
	panic("not reached")
}

// handleMessage handles the contents of a datagram and attempts to parse a Metric from each line
func (srv *MetricReceiver) handleMessage(addr net.Addr, msg []byte) {
	buf := bytes.NewBuffer(msg)
	for {
		line, readerr := buf.ReadBytes('\n')

		// protocol does not require line to end in \n, if EOF use received line if valid
		if readerr != nil && readerr != io.EOF {
			log.Printf("error reading message from %s: %s", addr, readerr)
			return
		} else if readerr != io.EOF {
			// remove newline, only if not EOF
			if len(line) > 0 {
				line = line[:len(line)-1]
			}
		}

		// Only process lines with more than one character
		if len(line) > 1 {
			metric, err := parseLine(line)
			if err != nil {
				log.Printf("error parsing line %q from %s: %s", line, addr, err)
				continue
			}
			go srv.Handler.HandleMetric(metric)
		}

		if readerr != nil && readerr == io.EOF {
			// if was EOF, finished handling
			return
		}
	}
}

func parseLine(line []byte) (Metric, error) {
	var metric Metric

	buf := bytes.NewBuffer(line)
	bucket, err := buf.ReadBytes(':')
	if err != nil {
		return metric, fmt.Errorf("error parsing metric name: %s", err)
	}
	metric.Bucket = string(bucket[:len(bucket)-1])

	value, err := buf.ReadBytes('|')
	if err != nil {
		return metric, fmt.Errorf("error parsing metric value: %s", err)
	}
	metric.Value, err = strconv.ParseFloat(string(value[:len(value)-1]), 64)
	if err != nil {
		return metric, fmt.Errorf("error converting metric value: %s", err)
	}

	metricType := buf.Bytes()
	if err != nil && err != io.EOF {
		return metric, fmt.Errorf("error parsing metric type: %s", err)
	}

	switch string(metricType[:len(metricType)]) {
	case "ms":
		// Timer
		metric.Type = TIMER
	case "g":
		// Gauge
		metric.Type = GAUGE
	case "c":
		metric.Type = COUNTER
	default:
		err = fmt.Errorf("invalid metric type: %q", metricType)
		return metric, err
	}

	return metric, nil
}
<commit_msg>* Remove redundant code<commit_after>package statsd

import (
	"bytes"
	"fmt"
	"io"
	"log"
	"net"
	"strconv"
)

// DefaultMetricsAddr is the default address on which a MetricReceiver will listen
const DefaultMetricsAddr = ":8125"

// Objects implementing the Handler interface can be used to handle metrics for a MetricReceiver
type Handler interface {
	HandleMetric(m Metric)
}

// The HandlerFunc type is an adapter to allow the use of ordinary functions as metric handlers
type HandlerFunc func(Metric)

// HandleMetric calls f(m)
func (f HandlerFunc) HandleMetric(m Metric) {
	f(m)
}

// MetricReceiver receives data on its listening port and converts lines in to Metrics.
// For each Metric it calls r.Handler.HandleMetric()
type MetricReceiver struct {
	Addr    string  // UDP address on which to listen for metrics
	Handler Handler // handler to invoke
}

// ListenAndReceive listens on the UDP network address of srv.Addr and then calls
// Receive to handle the incoming datagrams. If Addr is blank then DefaultMetricsAddr is used.
func (r *MetricReceiver) ListenAndReceive() error {
	addr := r.Addr
	if addr == "" {
		addr = DefaultMetricsAddr
	}
	c, err := net.ListenPacket("udp", addr)
	if err != nil {
		return err
	}
	return r.Receive(c)
}

// Receive accepts incoming datagrams on c and calls r.Handler.HandleMetric() for each line in the
// datagram that successfully parses in to a Metric
func (r *MetricReceiver) Receive(c net.PacketConn) error {
	defer c.Close()

	msg := make([]byte, 1024)
	for {
		nbytes, addr, err := c.ReadFrom(msg)
		if err != nil {
			log.Printf("%s", err)
			continue
		}
		buf := make([]byte, nbytes)
		copy(buf, msg[:nbytes])
		go r.handleMessage(addr, buf)
	}
	panic("not reached")
}

// handleMessage handles the contents of a datagram and attempts to parse a Metric from each line
func (srv *MetricReceiver) handleMessage(addr net.Addr, msg []byte) {
	buf := bytes.NewBuffer(msg)
	for {
		line, readerr := buf.ReadBytes('\n')

		// protocol does not require line to end in \n, if EOF use received line if valid
		if readerr != nil && readerr != io.EOF {
			log.Printf("error reading message from %s: %s", addr, readerr)
			return
		} else if readerr != io.EOF {
			// remove newline, only if not EOF
			if len(line) > 0 {
				line = line[:len(line)-1]
			}
		}

		// Only process lines with more than one character
		if len(line) > 1 {
			metric, err := parseLine(line)
			if err != nil {
				log.Printf("error parsing line %q from %s: %s", line, addr, err)
				continue
			}
			go srv.Handler.HandleMetric(metric)
		}

		if readerr == io.EOF {
			// if was EOF, finished handling
			return
		}
	}
}

func parseLine(line []byte) (Metric, error) {
	var metric Metric

	buf := bytes.NewBuffer(line)
	bucket, err := buf.ReadBytes(':')
	if err != nil {
		return metric, fmt.Errorf("error parsing metric name: %s", err)
	}
	metric.Bucket = string(bucket[:len(bucket)-1])

	value, err := buf.ReadBytes('|')
	if err != nil {
		return metric, fmt.Errorf("error parsing metric value: %s", err)
	}
	metric.Value, err = strconv.ParseFloat(string(value[:len(value)-1]), 64)
	if err != nil {
		return metric, fmt.Errorf("error converting metric value: %s", err)
	}

	metricType := buf.Bytes()
	if err != nil && err != io.EOF {
		return metric, fmt.Errorf("error parsing metric type: %s", err)
	}

	switch string(metricType[:len(metricType)]) {
	case "ms":
		// Timer
		metric.Type = TIMER
	case "g":
		// Gauge
		metric.Type = GAUGE
	case "c":
		metric.Type = COUNTER
	default:
		err = fmt.Errorf("invalid metric type: %q", metricType)
		return metric, err
	}

	return metric, nil
}
<|endoftext|>


<commit_before>// GCS related utility functions to fetch and wrap objects with tar.Reader.
//
// Testing:
//   This has been manually tested, but test automation is probably not
//   worthwhile until there is an emulator for GCS.

package storage

import (
	"archive/tar"
	"compress/gzip"
	"encoding/base64"
	"errors"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"strconv"
	"strings"
	"time"

	"github.com/m-lab/etl/metrics"

	"golang.org/x/net/context"
	"golang.org/x/oauth2/google"
	storage "google.golang.org/api/storage/v1"
)

type TarReader interface {
	Next() (*tar.Header, error)
	Read(b []byte) (int, error)
}

type ETLSource struct {
	TarReader
	io.Closer
}

// Retrieve next file header.
// Lots of error handling because of common faults in underlying GCS.
func (rr *ETLSource) nextHeader(trial int) (*tar.Header, bool, error) {
	h, err := rr.Next()
	if err != nil {
		if err == io.EOF {
			return nil, false, err
		} else if strings.Contains(err.Error(), "unexpected EOF") {
			metrics.GCSRetryCount.WithLabelValues(
				"next", strconv.Itoa(trial), "unexpected EOF").Inc()
			// TODO: These are likely unrecoverable, so we should
			// just return.
		} else {
			// Quite a few of these now, and they seem to be
			// unrecoverable.
			metrics.GCSRetryCount.WithLabelValues(
				"next", strconv.Itoa(trial), "other").Inc()
		}
		log.Printf("Next: %v\n", err)
	}
	return h, true, err
}

// Retrieve the data for a single file.
// Lots of error handling because of common faults in underlying GCS.
// Returns data in byte array, error and boolean regarding whether to retry.
func (rr *ETLSource) nextData(h *tar.Header, trial int) ([]byte, bool, error) {
	var data []byte
	var err error
	var phase string
	if strings.HasSuffix(strings.ToLower(h.Name), "gz") {
		// TODO add unit test
		var zipReader *gzip.Reader
		zipReader, err = gzip.NewReader(rr)
		if err != nil {
			metrics.GCSRetryCount.WithLabelValues(
				"open zip", strconv.Itoa(trial), "zipReaderError").Inc()
			log.Printf("zipReaderError: %v in file %s\n", err, h.Name)

			return nil, true, err
		}
		defer zipReader.Close()
		phase = "read zip"
		data, err = ioutil.ReadAll(zipReader)
	} else {
		phase = "read"
		data, err = ioutil.ReadAll(rr)
	}
	if err != nil {
		if strings.Contains(err.Error(), "stream error") {
			// We are seeing these very rarely, maybe 1 per hour.
			// They are non-deterministic, so probably related to GCS problems.
			metrics.GCSRetryCount.WithLabelValues(
				phase, strconv.Itoa(trial), "stream error").Inc()
		} else {
			// We haven't seen any of these so far (as of May 9)
			metrics.GCSRetryCount.WithLabelValues(
				phase, strconv.Itoa(trial), "other error").Inc()
		}
		return nil, true, err
	}

	return data, false, nil
}

// Next reads the next test object from the tar file.
// Returns io.EOF when there are no more tests.
func (rr *ETLSource) NextTest() (string, []byte, error) {
	metrics.WorkerState.WithLabelValues("read").Inc()
	defer metrics.WorkerState.WithLabelValues("read").Dec()

	// Try to get the next file.  We retry multiple times, because sometimes
	// GCS stalls and produces stream errors.
	// TODO - keep track of elapsed time instead of trials ??
	var err error
	var data []byte
	var h *tar.Header

	trial := 0
	delay := 5 * time.Millisecond
	for {
		trial++
		var retry bool
		h, retry, err = rr.nextHeader(trial)
		if err == nil {
			break
		}
		if !retry || trial > 10 {
			return "", nil, err
		}
		// For each trial, increase backoff delay by 2x.
		delay *= 2
		time.Sleep(delay)
	}

	// Only process regular files.
	if h.Typeflag == tar.TypeReg {
		trial = 0
		delay = 5 * time.Millisecond
		for {
			trial++
			var retry bool
			data, retry, err = rr.nextData(h, trial)
			if err == nil {
				break
			}
			if !retry || trial > 10 {
				break
			}
			// For each trial, increase backoff delay by 2x.
			delay *= 2
			time.Sleep(delay)

		}
	}

	return h.Name, data, nil
}

// Compound closer, for use with gzip files.
type Closer struct {
	zipper io.Closer // Must be non-null
	body   io.Closer // Must be non-null
}

func (t *Closer) Close() error {
	err := t.zipper.Close()
	t.body.Close()
	return err
}

var errNoClient = errors.New("client should be non-null")

// Create a ETLSource suitable for injecting into Task.
// Caller is responsible for calling Close on the returned object.
//
// uri should be of form gs://bucket/filename.tar or gs://bucket/filename.tgz
// FYI Using a persistent client saves about 80 msec, and 220 allocs, totalling 70kB.
// TODO(now) rename
func NewETLSource(client *http.Client, uri string) (*ETLSource, error) {
	if client == nil {
		return nil, errNoClient
	}
	// For now only handle gcs paths.
	if !strings.HasPrefix(uri, "gs://") {
		return nil, errors.New("invalid file path: " + uri)
	}
	parts := strings.SplitN(uri, "/", 4)
	if len(parts) != 4 {
		return nil, errors.New("invalid file path: " + uri)
	}
	bucket := parts[2]
	fn := parts[3]

	// TODO - consider just always testing for valid gzip file.
	if !(strings.HasSuffix(fn, ".tgz") || strings.HasSuffix(fn, ".tar") ||
		strings.HasSuffix(fn, ".tar.gz")) {
		return nil, errors.New("not tar or tgz: " + uri)
	}

	obj, err := getObject(client, bucket, fn, 10*time.Minute)
	if err != nil {
		return nil, err
	}

	rdr := obj.Body
	var closer io.Closer = obj.Body
	// Handle .tar.gz, .tgz files.
	if strings.HasSuffix(strings.ToLower(fn), "gz") {
		// TODO add unit test
		// NB: This must not be :=, or it creates local rdr.
		// TODO - add retries with backoff.
		rdr, err = gzip.NewReader(obj.Body)
		if err != nil {
			obj.Body.Close()
			return nil, err
		}

		closer = &Closer{rdr, obj.Body}
	}
	tarReader := tar.NewReader(rdr)

	return &ETLSource{tarReader, closer}, nil
}

// Create a storage reader client.
func GetStorageClient(writeAccess bool) (*http.Client, error) {
	var scope string
	if writeAccess {
		scope = storage.DevstorageReadWriteScope
	} else {
		scope = storage.DevstorageReadOnlyScope
	}

	// Use a short timeout, so we get an error quickly if there is a problem.
	ctx, _ := context.WithTimeout(context.Background(), 10*time.Second)
	client, err := google.DefaultClient(ctx, scope)
	if err != nil {
		return nil, err
	}
	return client, nil
}

// Turn the bytes received from the queue into a filename
// TODO(dev) Add unit test
func GetFilename(filename string) (string, error) {
	if strings.HasPrefix(filename, "gs://") {
		return filename, nil
	}

	decode, err := base64.StdEncoding.DecodeString(filename)
	if err != nil {
		return "", errors.New("invalid file path: " + filename)
	}
	fn := string(decode[:])
	if strings.HasPrefix(fn, "gs://") {
		return fn, nil
	}

	return "", errors.New("invalid base64 encoded file path: " + fn)
}

//---------------------------------------------------------------------------------
//          Local functions
//---------------------------------------------------------------------------------

// Caller is responsible for closing response body.
func getObject(client *http.Client, bucket string, fn string, timeout time.Duration) (*http.Response, error) {
	// Lightweight, error only if client is nil.
	service, err := storage.New(client)
	if err != nil {
		return nil, err
	}

	// Lightweight - only setting up the local object.
	call := service.Objects.Get(bucket, fn)
	ctx, _ := context.WithTimeout(context.Background(), timeout)
	call = call.Context(ctx)

	// Heavyweight.
	// Doesn't look like any googleapi.CallOptions are useful here.
	contentResponse, err := call.Download()
	if err != nil {
		return nil, err
	}
	return contentResponse, err
}
<commit_msg>tweak storage log, delays<commit_after>// GCS related utility functions to fetch and wrap objects with tar.Reader.
//
// Testing:
//   This has been manually tested, but test automation is probably not
//   worthwhile until there is an emulator for GCS.

package storage

import (
	"archive/tar"
	"compress/gzip"
	"encoding/base64"
	"errors"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"strconv"
	"strings"
	"time"

	"github.com/m-lab/etl/metrics"

	"golang.org/x/net/context"
	"golang.org/x/oauth2/google"
	storage "google.golang.org/api/storage/v1"
)

type TarReader interface {
	Next() (*tar.Header, error)
	Read(b []byte) (int, error)
}

type ETLSource struct {
	TarReader
	io.Closer
}

// Retrieve next file header.
// Lots of error handling because of common faults in underlying GCS.
func (rr *ETLSource) nextHeader(trial int) (*tar.Header, bool, error) {
	h, err := rr.Next()
	if err != nil {
		if err == io.EOF {
			return nil, false, err
		} else if strings.Contains(err.Error(), "unexpected EOF") {
			metrics.GCSRetryCount.WithLabelValues(
				"next", strconv.Itoa(trial), "unexpected EOF").Inc()
			// TODO: These are likely unrecoverable, so we should
			// just return.
		} else {
			// Quite a few of these now, and they seem to be
			// unrecoverable.
			metrics.GCSRetryCount.WithLabelValues(
				"next", strconv.Itoa(trial), "other").Inc()
		}
		log.Printf("nextHeader: %v\n", err)
	}
	return h, true, err
}

// Retrieve the data for a single file.
// Lots of error handling because of common faults in underlying GCS.
// Returns data in byte array, error and boolean regarding whether to retry.
func (rr *ETLSource) nextData(h *tar.Header, trial int) ([]byte, bool, error) {
	var data []byte
	var err error
	var phase string
	if strings.HasSuffix(strings.ToLower(h.Name), "gz") {
		// TODO add unit test
		var zipReader *gzip.Reader
		zipReader, err = gzip.NewReader(rr)
		if err != nil {
			metrics.GCSRetryCount.WithLabelValues(
				"open zip", strconv.Itoa(trial), "zipReaderError").Inc()
			log.Printf("zipReaderError: %v in file %s\n", err, h.Name)

			return nil, true, err
		}
		defer zipReader.Close()
		phase = "read zip"
		data, err = ioutil.ReadAll(zipReader)
	} else {
		phase = "read"
		data, err = ioutil.ReadAll(rr)
	}
	if err != nil {
		if strings.Contains(err.Error(), "stream error") {
			// We are seeing these very rarely, maybe 1 per hour.
			// They are non-deterministic, so probably related to GCS problems.
			metrics.GCSRetryCount.WithLabelValues(
				phase, strconv.Itoa(trial), "stream error").Inc()
		} else {
			// We haven't seen any of these so far (as of May 9)
			metrics.GCSRetryCount.WithLabelValues(
				phase, strconv.Itoa(trial), "other error").Inc()
		}
		log.Printf("nextData: %v\n", err)
		return nil, true, err
	}

	return data, false, nil
}

// Next reads the next test object from the tar file.
// Returns io.EOF when there are no more tests.
func (rr *ETLSource) NextTest() (string, []byte, error) {
	metrics.WorkerState.WithLabelValues("read").Inc()
	defer metrics.WorkerState.WithLabelValues("read").Dec()

	// Try to get the next file.  We retry multiple times, because sometimes
	// GCS stalls and produces stream errors.
	// TODO - keep track of elapsed time instead of trials ??
	var err error
	var data []byte
	var h *tar.Header

	// Last trial will be after total delay of 16ms + 32ms + ... + 8192ms,
    // or about 15 seconds.
	trial := 0
	delay := 16 * time.Millisecond
	for {
		trial++
		var retry bool
		h, retry, err = rr.nextHeader(trial)
		if err == nil {
			break
		}
		if !retry || trial >= 10 {
			return "", nil, err
		}
		// For each trial, increase backoff delay by 2x.
		delay *= 2
		time.Sleep(delay)
	}

	// Only process regular files.
	if h.Typeflag == tar.TypeReg {
		trial = 0
		delay = 16 * time.Millisecond
		for {
			trial++
			var retry bool
			data, retry, err = rr.nextData(h, trial)
			if err == nil {
				break
			}
			if !retry || trial >= 10 {
				break
			}
			// For each trial, increase backoff delay by 2x.
			delay *= 2
			time.Sleep(delay)

		}
	}

	return h.Name, data, nil
}

// Compound closer, for use with gzip files.
type Closer struct {
	zipper io.Closer // Must be non-null
	body   io.Closer // Must be non-null
}

func (t *Closer) Close() error {
	err := t.zipper.Close()
	t.body.Close()
	return err
}

var errNoClient = errors.New("client should be non-null")

// Create a ETLSource suitable for injecting into Task.
// Caller is responsible for calling Close on the returned object.
//
// uri should be of form gs://bucket/filename.tar or gs://bucket/filename.tgz
// FYI Using a persistent client saves about 80 msec, and 220 allocs, totalling 70kB.
// TODO(now) rename
func NewETLSource(client *http.Client, uri string) (*ETLSource, error) {
	if client == nil {
		return nil, errNoClient
	}
	// For now only handle gcs paths.
	if !strings.HasPrefix(uri, "gs://") {
		return nil, errors.New("invalid file path: " + uri)
	}
	parts := strings.SplitN(uri, "/", 4)
	if len(parts) != 4 {
		return nil, errors.New("invalid file path: " + uri)
	}
	bucket := parts[2]
	fn := parts[3]

	// TODO - consider just always testing for valid gzip file.
	if !(strings.HasSuffix(fn, ".tgz") || strings.HasSuffix(fn, ".tar") ||
		strings.HasSuffix(fn, ".tar.gz")) {
		return nil, errors.New("not tar or tgz: " + uri)
	}

	obj, err := getObject(client, bucket, fn, 10*time.Minute)
	if err != nil {
		return nil, err
	}

	rdr := obj.Body
	var closer io.Closer = obj.Body
	// Handle .tar.gz, .tgz files.
	if strings.HasSuffix(strings.ToLower(fn), "gz") {
		// TODO add unit test
		// NB: This must not be :=, or it creates local rdr.
		// TODO - add retries with backoff.
		rdr, err = gzip.NewReader(obj.Body)
		if err != nil {
			obj.Body.Close()
			return nil, err
		}

		closer = &Closer{rdr, obj.Body}
	}
	tarReader := tar.NewReader(rdr)

	return &ETLSource{tarReader, closer}, nil
}

// Create a storage reader client.
func GetStorageClient(writeAccess bool) (*http.Client, error) {
	var scope string
	if writeAccess {
		scope = storage.DevstorageReadWriteScope
	} else {
		scope = storage.DevstorageReadOnlyScope
	}

	// Use a short timeout, so we get an error quickly if there is a problem.
	ctx, _ := context.WithTimeout(context.Background(), 10*time.Second)
	client, err := google.DefaultClient(ctx, scope)
	if err != nil {
		return nil, err
	}
	return client, nil
}

// Turn the bytes received from the queue into a filename
// TODO(dev) Add unit test
func GetFilename(filename string) (string, error) {
	if strings.HasPrefix(filename, "gs://") {
		return filename, nil
	}

	decode, err := base64.StdEncoding.DecodeString(filename)
	if err != nil {
		return "", errors.New("invalid file path: " + filename)
	}
	fn := string(decode[:])
	if strings.HasPrefix(fn, "gs://") {
		return fn, nil
	}

	return "", errors.New("invalid base64 encoded file path: " + fn)
}

//---------------------------------------------------------------------------------
//          Local functions
//---------------------------------------------------------------------------------

// Caller is responsible for closing response body.
func getObject(client *http.Client, bucket string, fn string, timeout time.Duration) (*http.Response, error) {
	// Lightweight, error only if client is nil.
	service, err := storage.New(client)
	if err != nil {
		return nil, err
	}

	// Lightweight - only setting up the local object.
	call := service.Objects.Get(bucket, fn)
	ctx, _ := context.WithTimeout(context.Background(), timeout)
	call = call.Context(ctx)

	// Heavyweight.
	// Doesn't look like any googleapi.CallOptions are useful here.
	contentResponse, err := call.Download()
	if err != nil {
		return nil, err
	}
	return contentResponse, err
}
<|endoftext|>


<commit_before>package store

import (
	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/dynamodb"
	"github.com/wantedly/developers-account-mapper/models"
)

const (
	githubUsersTable = "GitHubUsers"
)

type DynamoDB struct {
	db *dynamodb.DynamoDB
}

func NewDynamoDB() *DynamoDB {
	db := dynamodb.New(session.New(&aws.Config{}))

	return &DynamoDB{
		db: db,
	}
}

func (d *DynamoDB) ListUsers() ([]*models.User, error) {
	resp, err := d.db.Scan(&dynamodb.ScanInput{
		TableName: aws.String(githubUsersTable),
	})
	if err != nil {
		return nil, err
	}

	var users []*models.User

	for _, item := range resp.Items {
		users = append(users, models.NewUser(*item["LoginName"].S, *item["GitHubUSername"].S))
	}

	return users, nil
}

func (d *DynamoDB) AddUser(user *models.User) (error) {
	_, err := d.db.PutItem(&dynamodb.PutItemInput{
		TableName: aws.String(githubUsersTable),
		Item: map[string]*dynamodb.AttributeValue{
			"LoginName": {
				S: aws.String(user.LoginName),
			},
			"GitHubUsername": {
				S: aws.String(user.GitHubUsername),
			},
		},
	})
	if err != nil {
		return err
	}

	return nil
}
<commit_msg>Rename var name to one more suitable for developers-account-mapper<commit_after>package store

import (
	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/dynamodb"
	"github.com/wantedly/developers-account-mapper/models"
)

const (
	accountMapTable = "GitHubUsers"
)

type DynamoDB struct {
	db *dynamodb.DynamoDB
}

func NewDynamoDB() *DynamoDB {
	db := dynamodb.New(session.New(&aws.Config{}))

	return &DynamoDB{
		db: db,
	}
}

func (d *DynamoDB) ListUsers() ([]*models.User, error) {
	resp, err := d.db.Scan(&dynamodb.ScanInput{
		TableName: aws.String(accountMapTable),
	})
	if err != nil {
		return nil, err
	}

	var users []*models.User

	for _, item := range resp.Items {
		users = append(users, models.NewUser(*item["LoginName"].S, *item["GitHubUSername"].S))
	}

	return users, nil
}

func (d *DynamoDB) AddUser(user *models.User) (error) {
	_, err := d.db.PutItem(&dynamodb.PutItemInput{
		TableName: aws.String(accountMapTable),
		Item: map[string]*dynamodb.AttributeValue{
			"LoginName": {
				S: aws.String(user.LoginName),
			},
			"GitHubUsername": {
				S: aws.String(user.GitHubUsername),
			},
		},
	})
	if err != nil {
		return err
	}

	return nil
}
<|endoftext|>


<commit_before>package rpcd

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"github.com/Symantec/Dominator/lib/hash"
	"github.com/Symantec/Dominator/lib/triggers"
	"github.com/Symantec/Dominator/proto/sub"
	"os"
	"os/exec"
	"path"
	"strings"
	"time"
)

func (t *rpcType) Update(request sub.UpdateRequest,
	reply *sub.UpdateResponse) error {
	rwLock.Lock()
	defer rwLock.Unlock()
	fs := fileSystemHistory.FileSystem()
	if fs == nil {
		return errors.New("No file-system history yet")
	}
	logger.Printf("Update()\n")
	if fetchInProgress {
		logger.Println("Error: fetch already in progress")
		return errors.New("fetch already in progress")
	}
	if updateInProgress {
		logger.Println("Error: update progress")
		return errors.New("update in progress")
	}
	updateInProgress = true
	go doUpdate(request, fs.RootDirectoryName())
	return nil
}

func doUpdate(request sub.UpdateRequest, rootDirectoryName string) {
	defer clearUpdateInProgress()
	var oldTriggers triggers.Triggers
	file, err := os.Open(oldTriggersFilename)
	if err == nil {
		decoder := json.NewDecoder(file)
		var trig triggers.Triggers
		err = decoder.Decode(&trig.Triggers)
		file.Close()
		if err == nil {
			oldTriggers = trig
		} else {
			logger.Printf("Error decoding old triggers: %s", err.Error())
		}
	}
	if len(oldTriggers.Triggers) > 0 {
		processMakeInodes(request.InodesToMake, rootDirectoryName,
			request.MultiplyUsedObjects, &oldTriggers, false)
		processHardlinksToMake(request.HardlinksToMake, rootDirectoryName,
			&oldTriggers, false)
		processDeletes(request.PathsToDelete, rootDirectoryName, &oldTriggers,
			false)
		processMakeDirectories(request.DirectoriesToMake, rootDirectoryName,
			&oldTriggers, false)
		processChangeDirectories(request.DirectoriesToChange, rootDirectoryName,
			&oldTriggers, false)
		processChangeInodes(request.InodesToChange, rootDirectoryName,
			&oldTriggers, false)
		matchedOldTriggers := oldTriggers.GetMatchedTriggers()
		runTriggers(matchedOldTriggers, "stop")
	}
	processFilesToCopyToCache(request.FilesToCopyToCache, rootDirectoryName)
	processMakeInodes(request.InodesToMake, rootDirectoryName,
		request.MultiplyUsedObjects, request.Triggers, true)
	processHardlinksToMake(request.HardlinksToMake, rootDirectoryName,
		request.Triggers, true)
	processDeletes(request.PathsToDelete, rootDirectoryName, request.Triggers,
		true)
	processMakeDirectories(request.DirectoriesToMake, rootDirectoryName,
		request.Triggers, true)
	processChangeDirectories(request.DirectoriesToChange, rootDirectoryName,
		request.Triggers, true)
	processChangeInodes(request.InodesToChange, rootDirectoryName,
		request.Triggers, true)
	matchedNewTriggers := request.Triggers.GetMatchedTriggers()
	file, err = os.Create(oldTriggersFilename)
	if err == nil {
		b, err := json.Marshal(request.Triggers.Triggers)
		if err == nil {
			var out bytes.Buffer
			json.Indent(&out, b, "", "    ")
			out.WriteTo(file)
		} else {
			logger.Printf("Error marshaling triggers: %s", err.Error())
		}
		file.Close()
	}
	runTriggers(matchedNewTriggers, "start")
	// TODO(rgooch): Remove debugging hack and implement.
	time.Sleep(time.Second * 15)
	logger.Printf("Update() complete\n")
}

func clearUpdateInProgress() {
	rwLock.Lock()
	defer rwLock.Unlock()
	updateInProgress = false
}

func processFilesToCopyToCache(filesToCopyToCache []sub.FileToCopyToCache,
	rootDirectoryName string) {
	for _, fileToCopy := range filesToCopyToCache {
		// TODO(rgooch): Remove debugging.
		fmt.Printf("Copy: %s to cache\n", fileToCopy.Name)
		// TODO(rgooch): Implement.
	}
}

func processMakeInodes(inodesToMake []sub.Inode, rootDirectoryName string,
	multiplyUsedObjects map[hash.Hash]uint64, triggers *triggers.Triggers,
	takeAction bool) {
	for _, inode := range inodesToMake {
		// TODO(rgooch): Remove debugging.
		fmt.Printf("Make inode: %s\n", inode.Name)
		// TODO(rgooch): Implement.
	}
}

func processHardlinksToMake(hardlinksToMake []sub.Hardlink,
	rootDirectoryName string, triggers *triggers.Triggers, takeAction bool) {
	for _, hardlink := range hardlinksToMake {
		fmt.Printf("Link: %s => %s\n", hardlink.NewLink, hardlink.Target)
		// TODO(rgooch): Implement.
	}
}

func processDeletes(pathsToDelete []string, rootDirectoryName string,
	triggers *triggers.Triggers, takeAction bool) {
	for _, pathname := range pathsToDelete {
		fullPathname := path.Join(rootDirectoryName, pathname)
		triggers.Match(pathname)
		if takeAction {
			// TODO(rgooch): Remove debugging.
			fmt.Printf("Delete: %s\n", fullPathname)
			// TODO(rgooch): Implement.
		}
	}
}

func processMakeDirectories(directoriesToMake []sub.Directory,
	rootDirectoryName string, triggers *triggers.Triggers, takeAction bool) {
	for _, newdir := range directoriesToMake {
		if scannerConfiguration.ScanFilter.Match(newdir.Name) {
			continue
		}
		if newdir.Name == "/.subd" {
			continue
		}
		if strings.HasPrefix(newdir.Name, "/.subd/") {
			continue
		}
		fullPathname := path.Join(rootDirectoryName, newdir.Name)
		triggers.Match(newdir.Name)
		if takeAction {
			// TODO(rgooch): Remove debugging.
			fmt.Printf("Mkdir: %s\n", fullPathname)
			// TODO(rgooch): Implement.
		}
	}
}

func processChangeDirectories(directoriesToChange []sub.Directory,
	rootDirectoryName string, triggers *triggers.Triggers, takeAction bool) {
	for _, directory := range directoriesToChange {
		// TODO(rgooch): Remove debugging.
		fmt.Printf("Change directory: %s\n", directory.Name)
		// TODO(rgooch): Implement.
	}
}

func processChangeInodes(inodesToChange []sub.Inode,
	rootDirectoryName string, triggers *triggers.Triggers, takeAction bool) {
	for _, inode := range inodesToChange {
		// TODO(rgooch): Remove debugging.
		fmt.Printf("Change inode: %s\n", inode.Name)
		// TODO(rgooch): Implement.
	}
}

func runTriggers(triggers []*triggers.Trigger, action string) {
	// For "start" action, if there is a reboot trigger, just do that one.
	if action == "start" {
		for _, trigger := range triggers {
			if trigger.Service == "reboot" {
				logger.Print("Rebooting")
				// TODO(rgooch): Remove debugging output.
				cmd := exec.Command("echo", "reboot")
				cmd.Stdout = os.Stdout
				if err := cmd.Run(); err != nil {
					logger.Print(err)
				}
				return
			}
		}
	}
	ppid := fmt.Sprint(os.Getppid())
	for _, trigger := range triggers {
		if trigger.Service == "reboot" && action == "stop" {
			continue
		}
		logger.Printf("Action: service %s %s\n", trigger.Service, action)
		// TODO(rgooch): Remove debugging output.
		cmd := exec.Command("run-in-mntns", ppid, "echo", "service", action,
			trigger.Service)
		cmd.Stdout = os.Stdout
		if err := cmd.Run(); err != nil {
			logger.Print(err)
		}
		// TODO(rgooch): Implement.
	}
}
<commit_msg>Incremental work on subd.Update() RPC.<commit_after>package rpcd

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"github.com/Symantec/Dominator/lib/hash"
	"github.com/Symantec/Dominator/lib/triggers"
	"github.com/Symantec/Dominator/proto/sub"
	"os"
	"os/exec"
	"path"
	"strings"
	"time"
)

func (t *rpcType) Update(request sub.UpdateRequest,
	reply *sub.UpdateResponse) error {
	rwLock.Lock()
	defer rwLock.Unlock()
	fs := fileSystemHistory.FileSystem()
	if fs == nil {
		return errors.New("No file-system history yet")
	}
	logger.Printf("Update()\n")
	if fetchInProgress {
		logger.Println("Error: fetch already in progress")
		return errors.New("fetch already in progress")
	}
	if updateInProgress {
		logger.Println("Error: update progress")
		return errors.New("update in progress")
	}
	updateInProgress = true
	go doUpdate(request, fs.RootDirectoryName())
	return nil
}

func doUpdate(request sub.UpdateRequest, rootDirectoryName string) {
	defer clearUpdateInProgress()
	var oldTriggers triggers.Triggers
	file, err := os.Open(oldTriggersFilename)
	if err == nil {
		decoder := json.NewDecoder(file)
		var trig triggers.Triggers
		err = decoder.Decode(&trig.Triggers)
		file.Close()
		if err == nil {
			oldTriggers = trig
		} else {
			logger.Printf("Error decoding old triggers: %s", err.Error())
		}
	}
	if len(oldTriggers.Triggers) > 0 {
		processMakeInodes(request.InodesToMake, rootDirectoryName,
			request.MultiplyUsedObjects, &oldTriggers, false)
		processHardlinksToMake(request.HardlinksToMake, rootDirectoryName,
			&oldTriggers, false)
		processDeletes(request.PathsToDelete, rootDirectoryName, &oldTriggers,
			false)
		processMakeDirectories(request.DirectoriesToMake, rootDirectoryName,
			&oldTriggers, false)
		processChangeDirectories(request.DirectoriesToChange, rootDirectoryName,
			&oldTriggers, false)
		processChangeInodes(request.InodesToChange, rootDirectoryName,
			&oldTriggers, false)
		matchedOldTriggers := oldTriggers.GetMatchedTriggers()
		runTriggers(matchedOldTriggers, "stop")
	}
	processFilesToCopyToCache(request.FilesToCopyToCache, rootDirectoryName)
	processMakeInodes(request.InodesToMake, rootDirectoryName,
		request.MultiplyUsedObjects, request.Triggers, true)
	processHardlinksToMake(request.HardlinksToMake, rootDirectoryName,
		request.Triggers, true)
	processDeletes(request.PathsToDelete, rootDirectoryName, request.Triggers,
		true)
	processMakeDirectories(request.DirectoriesToMake, rootDirectoryName,
		request.Triggers, true)
	processChangeDirectories(request.DirectoriesToChange, rootDirectoryName,
		request.Triggers, true)
	processChangeInodes(request.InodesToChange, rootDirectoryName,
		request.Triggers, true)
	matchedNewTriggers := request.Triggers.GetMatchedTriggers()
	file, err = os.Create(oldTriggersFilename)
	if err == nil {
		b, err := json.Marshal(request.Triggers.Triggers)
		if err == nil {
			var out bytes.Buffer
			json.Indent(&out, b, "", "    ")
			out.WriteTo(file)
		} else {
			logger.Printf("Error marshaling triggers: %s", err.Error())
		}
		file.Close()
	}
	runTriggers(matchedNewTriggers, "start")
	// TODO(rgooch): Remove debugging hack and implement.
	time.Sleep(time.Second * 15)
	logger.Printf("Update() complete\n")
}

func clearUpdateInProgress() {
	rwLock.Lock()
	defer rwLock.Unlock()
	updateInProgress = false
}

func processFilesToCopyToCache(filesToCopyToCache []sub.FileToCopyToCache,
	rootDirectoryName string) {
	for _, fileToCopy := range filesToCopyToCache {
		// TODO(rgooch): Remove debugging.
		fmt.Printf("Copy: %s to cache\n", fileToCopy.Name)
		// TODO(rgooch): Implement.
	}
}

func processMakeInodes(inodesToMake []sub.Inode, rootDirectoryName string,
	multiplyUsedObjects map[hash.Hash]uint64, triggers *triggers.Triggers,
	takeAction bool) {
	for _, inode := range inodesToMake {
		fullPathname := path.Join(rootDirectoryName, inode.Name)
		triggers.Match(inode.Name)
		// TODO(rgooch): Remove debugging.
		fmt.Printf("Make inode: %s\n", fullPathname)
		// TODO(rgooch): Implement.
	}
}

func processHardlinksToMake(hardlinksToMake []sub.Hardlink,
	rootDirectoryName string, triggers *triggers.Triggers, takeAction bool) {
	for _, hardlink := range hardlinksToMake {
		triggers.Match(hardlink.NewLink)
		fmt.Printf("Link: %s => %s\n", hardlink.NewLink, hardlink.Target)
		// TODO(rgooch): Implement.
		// err := os.Link(path.Join(rootDirectoryName, hardlink.Target),
		//	path.Join(rootDirectoryName, hardlink.NewLink))
	}
}

func processDeletes(pathsToDelete []string, rootDirectoryName string,
	triggers *triggers.Triggers, takeAction bool) {
	for _, pathname := range pathsToDelete {
		fullPathname := path.Join(rootDirectoryName, pathname)
		triggers.Match(pathname)
		if takeAction {
			// TODO(rgooch): Remove debugging.
			fmt.Printf("Delete: %s\n", fullPathname)
			// TODO(rgooch): Implement.
		}
	}
}

func processMakeDirectories(directoriesToMake []sub.Directory,
	rootDirectoryName string, triggers *triggers.Triggers, takeAction bool) {
	for _, newdir := range directoriesToMake {
		if skipPath(newdir.Name) {
			continue
		}
		fullPathname := path.Join(rootDirectoryName, newdir.Name)
		triggers.Match(newdir.Name)
		if takeAction {
			// TODO(rgooch): Remove debugging.
			fmt.Printf("Mkdir: %s\n", fullPathname)
			// TODO(rgooch): Implement.
		}
	}
}

func processChangeDirectories(directoriesToChange []sub.Directory,
	rootDirectoryName string, triggers *triggers.Triggers, takeAction bool) {
	for _, directory := range directoriesToChange {
		// TODO(rgooch): Remove debugging.
		fmt.Printf("Change directory: %s\n", directory.Name)
		// TODO(rgooch): Implement.
	}
}

func processChangeInodes(inodesToChange []sub.Inode,
	rootDirectoryName string, triggers *triggers.Triggers, takeAction bool) {
	for _, inode := range inodesToChange {
		// TODO(rgooch): Remove debugging.
		fmt.Printf("Change inode: %s\n", inode.Name)
		// TODO(rgooch): Implement.
	}
}

func skipPath(pathname string) bool {
	if scannerConfiguration.ScanFilter.Match(pathname) {
		return true
	}
	if pathname == "/.subd" {
		return true
	}
	if strings.HasPrefix(pathname, "/.subd/") {
		return true
	}
	return false
}

func runTriggers(triggers []*triggers.Trigger, action string) {
	// For "start" action, if there is a reboot trigger, just do that one.
	if action == "start" {
		for _, trigger := range triggers {
			if trigger.Service == "reboot" {
				logger.Print("Rebooting")
				// TODO(rgooch): Remove debugging output.
				cmd := exec.Command("echo", "reboot")
				cmd.Stdout = os.Stdout
				if err := cmd.Run(); err != nil {
					logger.Print(err)
				}
				return
			}
		}
	}
	ppid := fmt.Sprint(os.Getppid())
	for _, trigger := range triggers {
		if trigger.Service == "reboot" && action == "stop" {
			continue
		}
		logger.Printf("Action: service %s %s\n", trigger.Service, action)
		// TODO(rgooch): Remove debugging output.
		cmd := exec.Command("run-in-mntns", ppid, "echo", "service", action,
			trigger.Service)
		cmd.Stdout = os.Stdout
		if err := cmd.Run(); err != nil {
			logger.Print(err)
		}
		// TODO(rgooch): Implement.
	}
}
<|endoftext|>


<commit_before>// +build linux

package fs

import (
	"bufio"
	"fmt"
	"math"
	"os"
	"path"
	"path/filepath"
	"strconv"
	"strings"

	"github.com/opencontainers/runc/libcontainer/cgroups"
	"github.com/opencontainers/runc/libcontainer/cgroups/fscommon"
	"github.com/opencontainers/runc/libcontainer/configs"
)

const (
	numaNodeSymbol            = "N"
	numaStatColumnSeparator   = " "
	numaStatKeyValueSeparator = "="
	numaStatMaxColumns        = math.MaxUint8 + 1
	numaStatValueIndex        = 1
	numaStatTypeIndex         = 0
	numaStatColumnSliceLength = 2
	cgroupMemorySwapLimit     = "memory.memsw.limit_in_bytes"
	cgroupMemoryLimit         = "memory.limit_in_bytes"
	cgroupMemoryPagesByNuma   = "memory.numa_stat"
)

type MemoryGroup struct {
}

func (s *MemoryGroup) Name() string {
	return "memory"
}

func (s *MemoryGroup) Apply(d *cgroupData) (err error) {
	path, err := d.path("memory")
	if err != nil && !cgroups.IsNotFound(err) {
		return err
	} else if path == "" {
		return nil
	}
	if memoryAssigned(d.config) {
		if _, err := os.Stat(path); os.IsNotExist(err) {
			if err := os.MkdirAll(path, 0755); err != nil {
				return err
			}
			// Only enable kernel memory accouting when this cgroup
			// is created by libcontainer, otherwise we might get
			// error when people use `cgroupsPath` to join an existed
			// cgroup whose kernel memory is not initialized.
			if err := EnableKernelMemoryAccounting(path); err != nil {
				return err
			}
		}
	}
	defer func() {
		if err != nil {
			os.RemoveAll(path)
		}
	}()

	// We need to join memory cgroup after set memory limits, because
	// kmem.limit_in_bytes can only be set when the cgroup is empty.
	_, err = d.join("memory")
	if err != nil && !cgroups.IsNotFound(err) {
		return err
	}
	return nil
}

func setMemoryAndSwap(path string, cgroup *configs.Cgroup) error {
	// If the memory update is set to -1 we should also
	// set swap to -1, it means unlimited memory.
	if cgroup.Resources.Memory == -1 {
		// Only set swap if it's enabled in kernel
		if cgroups.PathExists(filepath.Join(path, cgroupMemorySwapLimit)) {
			cgroup.Resources.MemorySwap = -1
		}
	}

	// When memory and swap memory are both set, we need to handle the cases
	// for updating container.
	if cgroup.Resources.Memory != 0 && cgroup.Resources.MemorySwap != 0 {
		memoryUsage, err := getMemoryData(path, "")
		if err != nil {
			return err
		}

		// When update memory limit, we should adapt the write sequence
		// for memory and swap memory, so it won't fail because the new
		// value and the old value don't fit kernel's validation.
		if cgroup.Resources.MemorySwap == -1 || memoryUsage.Limit < uint64(cgroup.Resources.MemorySwap) {
			if err := fscommon.WriteFile(path, cgroupMemorySwapLimit, strconv.FormatInt(cgroup.Resources.MemorySwap, 10)); err != nil {
				return err
			}
			if err := fscommon.WriteFile(path, cgroupMemoryLimit, strconv.FormatInt(cgroup.Resources.Memory, 10)); err != nil {
				return err
			}
		} else {
			if err := fscommon.WriteFile(path, cgroupMemoryLimit, strconv.FormatInt(cgroup.Resources.Memory, 10)); err != nil {
				return err
			}
			if err := fscommon.WriteFile(path, cgroupMemorySwapLimit, strconv.FormatInt(cgroup.Resources.MemorySwap, 10)); err != nil {
				return err
			}
		}
	} else {
		if cgroup.Resources.Memory != 0 {
			if err := fscommon.WriteFile(path, cgroupMemoryLimit, strconv.FormatInt(cgroup.Resources.Memory, 10)); err != nil {
				return err
			}
		}
		if cgroup.Resources.MemorySwap != 0 {
			if err := fscommon.WriteFile(path, cgroupMemorySwapLimit, strconv.FormatInt(cgroup.Resources.MemorySwap, 10)); err != nil {
				return err
			}
		}
	}

	return nil
}

func (s *MemoryGroup) Set(path string, cgroup *configs.Cgroup) error {
	if err := setMemoryAndSwap(path, cgroup); err != nil {
		return err
	}

	if cgroup.Resources.KernelMemory != 0 {
		if err := setKernelMemory(path, cgroup.Resources.KernelMemory); err != nil {
			return err
		}
	}

	if cgroup.Resources.MemoryReservation != 0 {
		if err := fscommon.WriteFile(path, "memory.soft_limit_in_bytes", strconv.FormatInt(cgroup.Resources.MemoryReservation, 10)); err != nil {
			return err
		}
	}

	if cgroup.Resources.KernelMemoryTCP != 0 {
		if err := fscommon.WriteFile(path, "memory.kmem.tcp.limit_in_bytes", strconv.FormatInt(cgroup.Resources.KernelMemoryTCP, 10)); err != nil {
			return err
		}
	}
	if cgroup.Resources.OomKillDisable {
		if err := fscommon.WriteFile(path, "memory.oom_control", "1"); err != nil {
			return err
		}
	}
	if cgroup.Resources.MemorySwappiness == nil || int64(*cgroup.Resources.MemorySwappiness) == -1 {
		return nil
	} else if *cgroup.Resources.MemorySwappiness <= 100 {
		if err := fscommon.WriteFile(path, "memory.swappiness", strconv.FormatUint(*cgroup.Resources.MemorySwappiness, 10)); err != nil {
			return err
		}
	} else {
		return fmt.Errorf("invalid value:%d. valid memory swappiness range is 0-100", *cgroup.Resources.MemorySwappiness)
	}

	return nil
}

func (s *MemoryGroup) Remove(d *cgroupData) error {
	return removePath(d.path("memory"))
}

func (s *MemoryGroup) GetStats(path string, stats *cgroups.Stats) error {
	// Set stats from memory.stat.
	statsFile, err := os.Open(filepath.Join(path, "memory.stat"))
	if err != nil {
		if os.IsNotExist(err) {
			return nil
		}
		return err
	}
	defer statsFile.Close()

	sc := bufio.NewScanner(statsFile)
	for sc.Scan() {
		t, v, err := fscommon.GetCgroupParamKeyValue(sc.Text())
		if err != nil {
			return fmt.Errorf("failed to parse memory.stat (%q) - %v", sc.Text(), err)
		}
		stats.MemoryStats.Stats[t] = v
	}
	stats.MemoryStats.Cache = stats.MemoryStats.Stats["cache"]

	memoryUsage, err := getMemoryData(path, "")
	if err != nil {
		return err
	}
	stats.MemoryStats.Usage = memoryUsage
	swapUsage, err := getMemoryData(path, "memsw")
	if err != nil {
		return err
	}
	stats.MemoryStats.SwapUsage = swapUsage
	kernelUsage, err := getMemoryData(path, "kmem")
	if err != nil {
		return err
	}
	stats.MemoryStats.KernelUsage = kernelUsage
	kernelTCPUsage, err := getMemoryData(path, "kmem.tcp")
	if err != nil {
		return err
	}
	stats.MemoryStats.KernelTCPUsage = kernelTCPUsage

	useHierarchy := strings.Join([]string{"memory", "use_hierarchy"}, ".")
	value, err := fscommon.GetCgroupParamUint(path, useHierarchy)
	if err != nil {
		return err
	}
	if value == 1 {
		stats.MemoryStats.UseHierarchy = true
	}

	pagesByNUMA, err := getPageUsageByNUMA(path)
	if err != nil {
		return err
	}
	stats.MemoryStats.PageUsageByNUMA = pagesByNUMA

	return nil
}

func memoryAssigned(cgroup *configs.Cgroup) bool {
	return cgroup.Resources.Memory != 0 ||
		cgroup.Resources.MemoryReservation != 0 ||
		cgroup.Resources.MemorySwap > 0 ||
		cgroup.Resources.KernelMemory > 0 ||
		cgroup.Resources.KernelMemoryTCP > 0 ||
		cgroup.Resources.OomKillDisable ||
		(cgroup.Resources.MemorySwappiness != nil && int64(*cgroup.Resources.MemorySwappiness) != -1)
}

func getMemoryData(path, name string) (cgroups.MemoryData, error) {
	memoryData := cgroups.MemoryData{}

	moduleName := "memory"
	if name != "" {
		moduleName = strings.Join([]string{"memory", name}, ".")
	}
	usage := strings.Join([]string{moduleName, "usage_in_bytes"}, ".")
	maxUsage := strings.Join([]string{moduleName, "max_usage_in_bytes"}, ".")
	failcnt := strings.Join([]string{moduleName, "failcnt"}, ".")
	limit := strings.Join([]string{moduleName, "limit_in_bytes"}, ".")

	value, err := fscommon.GetCgroupParamUint(path, usage)
	if err != nil {
		if moduleName != "memory" && os.IsNotExist(err) {
			return cgroups.MemoryData{}, nil
		}
		return cgroups.MemoryData{}, fmt.Errorf("failed to parse %s - %v", usage, err)
	}
	memoryData.Usage = value
	value, err = fscommon.GetCgroupParamUint(path, maxUsage)
	if err != nil {
		if moduleName != "memory" && os.IsNotExist(err) {
			return cgroups.MemoryData{}, nil
		}
		return cgroups.MemoryData{}, fmt.Errorf("failed to parse %s - %v", maxUsage, err)
	}
	memoryData.MaxUsage = value
	value, err = fscommon.GetCgroupParamUint(path, failcnt)
	if err != nil {
		if moduleName != "memory" && os.IsNotExist(err) {
			return cgroups.MemoryData{}, nil
		}
		return cgroups.MemoryData{}, fmt.Errorf("failed to parse %s - %v", failcnt, err)
	}
	memoryData.Failcnt = value
	value, err = fscommon.GetCgroupParamUint(path, limit)
	if err != nil {
		if moduleName != "memory" && os.IsNotExist(err) {
			return cgroups.MemoryData{}, nil
		}
		return cgroups.MemoryData{}, fmt.Errorf("failed to parse %s - %v", limit, err)
	}
	memoryData.Limit = value

	return memoryData, nil
}

func getPageUsageByNUMA(cgroupPath string) (cgroups.PageUsageByNUMA, error) {
	stats := cgroups.PageUsageByNUMA{}

	file, err := os.Open(path.Join(cgroupPath, cgroupMemoryPagesByNuma))
	if err != nil {
		return stats, err
	}

	scanner := bufio.NewScanner(file)
	for scanner.Scan() {
		var statsType string
		statsByType := cgroups.PageStats{Nodes: map[uint8]uint64{}}
		columns := strings.SplitN(scanner.Text(), numaStatColumnSeparator, numaStatMaxColumns)

		for _, column := range columns {
			pagesByNode := strings.SplitN(column, numaStatKeyValueSeparator, numaStatColumnSliceLength)

			if strings.HasPrefix(pagesByNode[numaStatTypeIndex], numaNodeSymbol) {
				nodeID, err := strconv.ParseUint(pagesByNode[numaStatTypeIndex][1:], 10, 8)
				if err != nil {
					return cgroups.PageUsageByNUMA{}, err
				}

				statsByType.Nodes[uint8(nodeID)], err = strconv.ParseUint(pagesByNode[numaStatValueIndex], 0, 64)
				if err != nil {
					return cgroups.PageUsageByNUMA{}, err
				}
			} else {
				statsByType.Total, err = strconv.ParseUint(pagesByNode[numaStatValueIndex], 0, 64)
				if err != nil {
					return cgroups.PageUsageByNUMA{}, err
				}

				statsType = pagesByNode[numaStatTypeIndex]
			}

			err := addNUMAStatsByType(&stats, statsByType, statsType)
			if err != nil {
				return cgroups.PageUsageByNUMA{}, err
			}
		}
	}
	err = scanner.Err()
	if err != nil {
		return cgroups.PageUsageByNUMA{}, err
	}

	return stats, nil
}

func addNUMAStatsByType(stats *cgroups.PageUsageByNUMA, byTypeStats cgroups.PageStats, statsType string) error {
	switch statsType {
	case "total":
		stats.Total = byTypeStats
	case "file":
		stats.File = byTypeStats
	case "anon":
		stats.Anon = byTypeStats
	case "unevictable":
		stats.Unevictable = byTypeStats
	case "hierarchical_total":
		stats.Hierarchical.Total = byTypeStats
	case "hierarchical_file":
		stats.Hierarchical.File = byTypeStats
	case "hierarchical_anon":
		stats.Hierarchical.Anon = byTypeStats
	case "hierarchical_unevictable":
		stats.Hierarchical.Unevictable = byTypeStats
	default:
		return fmt.Errorf("unsupported NUMA page type found: %s", statsType)
	}
	return nil
}
<commit_msg>cgroupv1: don't ignore MemorySwap if Memory==-1<commit_after>// +build linux

package fs

import (
	"bufio"
	"fmt"
	"math"
	"os"
	"path"
	"path/filepath"
	"strconv"
	"strings"

	"github.com/opencontainers/runc/libcontainer/cgroups"
	"github.com/opencontainers/runc/libcontainer/cgroups/fscommon"
	"github.com/opencontainers/runc/libcontainer/configs"
)

const (
	numaNodeSymbol            = "N"
	numaStatColumnSeparator   = " "
	numaStatKeyValueSeparator = "="
	numaStatMaxColumns        = math.MaxUint8 + 1
	numaStatValueIndex        = 1
	numaStatTypeIndex         = 0
	numaStatColumnSliceLength = 2
	cgroupMemorySwapLimit     = "memory.memsw.limit_in_bytes"
	cgroupMemoryLimit         = "memory.limit_in_bytes"
	cgroupMemoryPagesByNuma   = "memory.numa_stat"
)

type MemoryGroup struct {
}

func (s *MemoryGroup) Name() string {
	return "memory"
}

func (s *MemoryGroup) Apply(d *cgroupData) (err error) {
	path, err := d.path("memory")
	if err != nil && !cgroups.IsNotFound(err) {
		return err
	} else if path == "" {
		return nil
	}
	if memoryAssigned(d.config) {
		if _, err := os.Stat(path); os.IsNotExist(err) {
			if err := os.MkdirAll(path, 0755); err != nil {
				return err
			}
			// Only enable kernel memory accouting when this cgroup
			// is created by libcontainer, otherwise we might get
			// error when people use `cgroupsPath` to join an existed
			// cgroup whose kernel memory is not initialized.
			if err := EnableKernelMemoryAccounting(path); err != nil {
				return err
			}
		}
	}
	defer func() {
		if err != nil {
			os.RemoveAll(path)
		}
	}()

	// We need to join memory cgroup after set memory limits, because
	// kmem.limit_in_bytes can only be set when the cgroup is empty.
	_, err = d.join("memory")
	if err != nil && !cgroups.IsNotFound(err) {
		return err
	}
	return nil
}

func setMemoryAndSwap(path string, cgroup *configs.Cgroup) error {
	// If the memory update is set to -1 and the swap is not explicitly
	// set, we should also set swap to -1, it means unlimited memory.
	if cgroup.Resources.Memory == -1 && cgroup.Resources.MemorySwap == 0 {
		// Only set swap if it's enabled in kernel
		if cgroups.PathExists(filepath.Join(path, cgroupMemorySwapLimit)) {
			cgroup.Resources.MemorySwap = -1
		}
	}

	// When memory and swap memory are both set, we need to handle the cases
	// for updating container.
	if cgroup.Resources.Memory != 0 && cgroup.Resources.MemorySwap != 0 {
		memoryUsage, err := getMemoryData(path, "")
		if err != nil {
			return err
		}

		// When update memory limit, we should adapt the write sequence
		// for memory and swap memory, so it won't fail because the new
		// value and the old value don't fit kernel's validation.
		if cgroup.Resources.MemorySwap == -1 || memoryUsage.Limit < uint64(cgroup.Resources.MemorySwap) {
			if err := fscommon.WriteFile(path, cgroupMemorySwapLimit, strconv.FormatInt(cgroup.Resources.MemorySwap, 10)); err != nil {
				return err
			}
			if err := fscommon.WriteFile(path, cgroupMemoryLimit, strconv.FormatInt(cgroup.Resources.Memory, 10)); err != nil {
				return err
			}
		} else {
			if err := fscommon.WriteFile(path, cgroupMemoryLimit, strconv.FormatInt(cgroup.Resources.Memory, 10)); err != nil {
				return err
			}
			if err := fscommon.WriteFile(path, cgroupMemorySwapLimit, strconv.FormatInt(cgroup.Resources.MemorySwap, 10)); err != nil {
				return err
			}
		}
	} else {
		if cgroup.Resources.Memory != 0 {
			if err := fscommon.WriteFile(path, cgroupMemoryLimit, strconv.FormatInt(cgroup.Resources.Memory, 10)); err != nil {
				return err
			}
		}
		if cgroup.Resources.MemorySwap != 0 {
			if err := fscommon.WriteFile(path, cgroupMemorySwapLimit, strconv.FormatInt(cgroup.Resources.MemorySwap, 10)); err != nil {
				return err
			}
		}
	}

	return nil
}

func (s *MemoryGroup) Set(path string, cgroup *configs.Cgroup) error {
	if err := setMemoryAndSwap(path, cgroup); err != nil {
		return err
	}

	if cgroup.Resources.KernelMemory != 0 {
		if err := setKernelMemory(path, cgroup.Resources.KernelMemory); err != nil {
			return err
		}
	}

	if cgroup.Resources.MemoryReservation != 0 {
		if err := fscommon.WriteFile(path, "memory.soft_limit_in_bytes", strconv.FormatInt(cgroup.Resources.MemoryReservation, 10)); err != nil {
			return err
		}
	}

	if cgroup.Resources.KernelMemoryTCP != 0 {
		if err := fscommon.WriteFile(path, "memory.kmem.tcp.limit_in_bytes", strconv.FormatInt(cgroup.Resources.KernelMemoryTCP, 10)); err != nil {
			return err
		}
	}
	if cgroup.Resources.OomKillDisable {
		if err := fscommon.WriteFile(path, "memory.oom_control", "1"); err != nil {
			return err
		}
	}
	if cgroup.Resources.MemorySwappiness == nil || int64(*cgroup.Resources.MemorySwappiness) == -1 {
		return nil
	} else if *cgroup.Resources.MemorySwappiness <= 100 {
		if err := fscommon.WriteFile(path, "memory.swappiness", strconv.FormatUint(*cgroup.Resources.MemorySwappiness, 10)); err != nil {
			return err
		}
	} else {
		return fmt.Errorf("invalid value:%d. valid memory swappiness range is 0-100", *cgroup.Resources.MemorySwappiness)
	}

	return nil
}

func (s *MemoryGroup) Remove(d *cgroupData) error {
	return removePath(d.path("memory"))
}

func (s *MemoryGroup) GetStats(path string, stats *cgroups.Stats) error {
	// Set stats from memory.stat.
	statsFile, err := os.Open(filepath.Join(path, "memory.stat"))
	if err != nil {
		if os.IsNotExist(err) {
			return nil
		}
		return err
	}
	defer statsFile.Close()

	sc := bufio.NewScanner(statsFile)
	for sc.Scan() {
		t, v, err := fscommon.GetCgroupParamKeyValue(sc.Text())
		if err != nil {
			return fmt.Errorf("failed to parse memory.stat (%q) - %v", sc.Text(), err)
		}
		stats.MemoryStats.Stats[t] = v
	}
	stats.MemoryStats.Cache = stats.MemoryStats.Stats["cache"]

	memoryUsage, err := getMemoryData(path, "")
	if err != nil {
		return err
	}
	stats.MemoryStats.Usage = memoryUsage
	swapUsage, err := getMemoryData(path, "memsw")
	if err != nil {
		return err
	}
	stats.MemoryStats.SwapUsage = swapUsage
	kernelUsage, err := getMemoryData(path, "kmem")
	if err != nil {
		return err
	}
	stats.MemoryStats.KernelUsage = kernelUsage
	kernelTCPUsage, err := getMemoryData(path, "kmem.tcp")
	if err != nil {
		return err
	}
	stats.MemoryStats.KernelTCPUsage = kernelTCPUsage

	useHierarchy := strings.Join([]string{"memory", "use_hierarchy"}, ".")
	value, err := fscommon.GetCgroupParamUint(path, useHierarchy)
	if err != nil {
		return err
	}
	if value == 1 {
		stats.MemoryStats.UseHierarchy = true
	}

	pagesByNUMA, err := getPageUsageByNUMA(path)
	if err != nil {
		return err
	}
	stats.MemoryStats.PageUsageByNUMA = pagesByNUMA

	return nil
}

func memoryAssigned(cgroup *configs.Cgroup) bool {
	return cgroup.Resources.Memory != 0 ||
		cgroup.Resources.MemoryReservation != 0 ||
		cgroup.Resources.MemorySwap > 0 ||
		cgroup.Resources.KernelMemory > 0 ||
		cgroup.Resources.KernelMemoryTCP > 0 ||
		cgroup.Resources.OomKillDisable ||
		(cgroup.Resources.MemorySwappiness != nil && int64(*cgroup.Resources.MemorySwappiness) != -1)
}

func getMemoryData(path, name string) (cgroups.MemoryData, error) {
	memoryData := cgroups.MemoryData{}

	moduleName := "memory"
	if name != "" {
		moduleName = strings.Join([]string{"memory", name}, ".")
	}
	usage := strings.Join([]string{moduleName, "usage_in_bytes"}, ".")
	maxUsage := strings.Join([]string{moduleName, "max_usage_in_bytes"}, ".")
	failcnt := strings.Join([]string{moduleName, "failcnt"}, ".")
	limit := strings.Join([]string{moduleName, "limit_in_bytes"}, ".")

	value, err := fscommon.GetCgroupParamUint(path, usage)
	if err != nil {
		if moduleName != "memory" && os.IsNotExist(err) {
			return cgroups.MemoryData{}, nil
		}
		return cgroups.MemoryData{}, fmt.Errorf("failed to parse %s - %v", usage, err)
	}
	memoryData.Usage = value
	value, err = fscommon.GetCgroupParamUint(path, maxUsage)
	if err != nil {
		if moduleName != "memory" && os.IsNotExist(err) {
			return cgroups.MemoryData{}, nil
		}
		return cgroups.MemoryData{}, fmt.Errorf("failed to parse %s - %v", maxUsage, err)
	}
	memoryData.MaxUsage = value
	value, err = fscommon.GetCgroupParamUint(path, failcnt)
	if err != nil {
		if moduleName != "memory" && os.IsNotExist(err) {
			return cgroups.MemoryData{}, nil
		}
		return cgroups.MemoryData{}, fmt.Errorf("failed to parse %s - %v", failcnt, err)
	}
	memoryData.Failcnt = value
	value, err = fscommon.GetCgroupParamUint(path, limit)
	if err != nil {
		if moduleName != "memory" && os.IsNotExist(err) {
			return cgroups.MemoryData{}, nil
		}
		return cgroups.MemoryData{}, fmt.Errorf("failed to parse %s - %v", limit, err)
	}
	memoryData.Limit = value

	return memoryData, nil
}

func getPageUsageByNUMA(cgroupPath string) (cgroups.PageUsageByNUMA, error) {
	stats := cgroups.PageUsageByNUMA{}

	file, err := os.Open(path.Join(cgroupPath, cgroupMemoryPagesByNuma))
	if err != nil {
		return stats, err
	}

	scanner := bufio.NewScanner(file)
	for scanner.Scan() {
		var statsType string
		statsByType := cgroups.PageStats{Nodes: map[uint8]uint64{}}
		columns := strings.SplitN(scanner.Text(), numaStatColumnSeparator, numaStatMaxColumns)

		for _, column := range columns {
			pagesByNode := strings.SplitN(column, numaStatKeyValueSeparator, numaStatColumnSliceLength)

			if strings.HasPrefix(pagesByNode[numaStatTypeIndex], numaNodeSymbol) {
				nodeID, err := strconv.ParseUint(pagesByNode[numaStatTypeIndex][1:], 10, 8)
				if err != nil {
					return cgroups.PageUsageByNUMA{}, err
				}

				statsByType.Nodes[uint8(nodeID)], err = strconv.ParseUint(pagesByNode[numaStatValueIndex], 0, 64)
				if err != nil {
					return cgroups.PageUsageByNUMA{}, err
				}
			} else {
				statsByType.Total, err = strconv.ParseUint(pagesByNode[numaStatValueIndex], 0, 64)
				if err != nil {
					return cgroups.PageUsageByNUMA{}, err
				}

				statsType = pagesByNode[numaStatTypeIndex]
			}

			err := addNUMAStatsByType(&stats, statsByType, statsType)
			if err != nil {
				return cgroups.PageUsageByNUMA{}, err
			}
		}
	}
	err = scanner.Err()
	if err != nil {
		return cgroups.PageUsageByNUMA{}, err
	}

	return stats, nil
}

func addNUMAStatsByType(stats *cgroups.PageUsageByNUMA, byTypeStats cgroups.PageStats, statsType string) error {
	switch statsType {
	case "total":
		stats.Total = byTypeStats
	case "file":
		stats.File = byTypeStats
	case "anon":
		stats.Anon = byTypeStats
	case "unevictable":
		stats.Unevictable = byTypeStats
	case "hierarchical_total":
		stats.Hierarchical.Total = byTypeStats
	case "hierarchical_file":
		stats.Hierarchical.File = byTypeStats
	case "hierarchical_anon":
		stats.Hierarchical.Anon = byTypeStats
	case "hierarchical_unevictable":
		stats.Hierarchical.Unevictable = byTypeStats
	default:
		return fmt.Errorf("unsupported NUMA page type found: %s", statsType)
	}
	return nil
}
<|endoftext|>


<commit_before>package drivers

import (
	"fmt"
	"path/filepath"
	"strings"

	"github.com/lxc/lxd/shared"
)

func (d *zfs) patchStorageCreateVM() error {
	// Create any missing initial dataset.
	for _, dataset := range d.initialDatasets() {
		if d.checkDataset(filepath.Join(d.config["zfs.pool_name"], dataset)) {
			continue
		}

		err := d.createDataset(filepath.Join(d.config["zfs.pool_name"], dataset), "mountpoint=none")
		if err != nil {
			return err
		}
	}

	return nil
}

func (d *zfs) patchStorageZFSMount() error {
	datasets, err := d.getDatasets(d.config["zfs.pool_name"])
	if err != nil {
		return err
	}

	for _, dataset := range datasets {
		// Skip snapshots.
		if strings.Contains(dataset, "@") {
			continue
		}

		// Skip bookmarks.
		if strings.Contains(dataset, "#") {
			continue
		}

		// Skip block devices.
		if strings.HasSuffix(dataset, ".block") {
			continue
		}

		// Skip top level.
		if !strings.Contains(dataset, "/") {
			continue
		}

		// We only care about containers, images and custom volumes.
		if !shared.StringInSlice(strings.SplitN(dataset, "/", 2)[0], []string{"containers", "images", "custom"}) {
			continue
		}

		// Apply mountpoint changes.
		oldMountPoint, err := d.getDatasetProperty(filepath.Join(d.config["zfs.pool_name"], dataset), "mountpoint")
		if err != nil {
			return err
		}
		newMountPoint := filepath.Join(shared.VarPath("storage-pools", d.name, dataset))

		if oldMountPoint != newMountPoint {
			err := d.setDatasetProperties(filepath.Join(d.config["zfs.pool_name"], dataset), fmt.Sprintf("mountpoint=%s", newMountPoint), "canmount=noauto")
			if err != nil {
				return err
			}
		}

		// Apply canmount changes.
		oldCanMount, err := d.getDatasetProperty(filepath.Join(d.config["zfs.pool_name"], dataset), "canmount")
		if err != nil {
			return err
		}

		if oldCanMount != "noauto" {
			err := d.setDatasetProperties(filepath.Join(d.config["zfs.pool_name"], dataset), "canmount=noauto")
			if err != nil {
				return err
			}
		}
	}

	return nil
}

func (d *zfs) patchStorageZFSVolMode() error {
	if len(zfsVersion) >= 3 && zfsVersion[0:3] == "0.6" {
		d.logger.Warn("Unable to set volmode on parent virtual-machines datasets due to ZFS being too old")
		return nil
	}

	// Set volmode=none on the parent virtual-machines directory
	err := d.setDatasetProperties(filepath.Join(d.config["zfs.pool_name"], "virtual-machines"), "volmode=none")
	if err != nil {
		return err
	}

	err = d.setDatasetProperties(filepath.Join(d.config["zfs.pool_name"], "deleted", "virtual-machines"), "volmode=none")
	if err != nil {
		return err
	}

	return nil
}
<commit_msg>lxd/storage/drivers/driver/zfs/patches: Uses zfsBlockVolSuffix in patchStorageZFSMount function<commit_after>package drivers

import (
	"fmt"
	"path/filepath"
	"strings"

	"github.com/lxc/lxd/shared"
)

func (d *zfs) patchStorageCreateVM() error {
	// Create any missing initial dataset.
	for _, dataset := range d.initialDatasets() {
		if d.checkDataset(filepath.Join(d.config["zfs.pool_name"], dataset)) {
			continue
		}

		err := d.createDataset(filepath.Join(d.config["zfs.pool_name"], dataset), "mountpoint=none")
		if err != nil {
			return err
		}
	}

	return nil
}

func (d *zfs) patchStorageZFSMount() error {
	datasets, err := d.getDatasets(d.config["zfs.pool_name"])
	if err != nil {
		return err
	}

	for _, dataset := range datasets {
		// Skip snapshots.
		if strings.Contains(dataset, "@") {
			continue
		}

		// Skip bookmarks.
		if strings.Contains(dataset, "#") {
			continue
		}

		// Skip block devices.
		if strings.HasSuffix(dataset, zfsBlockVolSuffix) {
			continue
		}

		// Skip top level.
		if !strings.Contains(dataset, "/") {
			continue
		}

		// We only care about containers, images and custom volumes.
		if !shared.StringInSlice(strings.SplitN(dataset, "/", 2)[0], []string{"containers", "images", "custom"}) {
			continue
		}

		// Apply mountpoint changes.
		oldMountPoint, err := d.getDatasetProperty(filepath.Join(d.config["zfs.pool_name"], dataset), "mountpoint")
		if err != nil {
			return err
		}
		newMountPoint := filepath.Join(shared.VarPath("storage-pools", d.name, dataset))

		if oldMountPoint != newMountPoint {
			err := d.setDatasetProperties(filepath.Join(d.config["zfs.pool_name"], dataset), fmt.Sprintf("mountpoint=%s", newMountPoint), "canmount=noauto")
			if err != nil {
				return err
			}
		}

		// Apply canmount changes.
		oldCanMount, err := d.getDatasetProperty(filepath.Join(d.config["zfs.pool_name"], dataset), "canmount")
		if err != nil {
			return err
		}

		if oldCanMount != "noauto" {
			err := d.setDatasetProperties(filepath.Join(d.config["zfs.pool_name"], dataset), "canmount=noauto")
			if err != nil {
				return err
			}
		}
	}

	return nil
}

func (d *zfs) patchStorageZFSVolMode() error {
	if len(zfsVersion) >= 3 && zfsVersion[0:3] == "0.6" {
		d.logger.Warn("Unable to set volmode on parent virtual-machines datasets due to ZFS being too old")
		return nil
	}

	// Set volmode=none on the parent virtual-machines directory
	err := d.setDatasetProperties(filepath.Join(d.config["zfs.pool_name"], "virtual-machines"), "volmode=none")
	if err != nil {
		return err
	}

	err = d.setDatasetProperties(filepath.Join(d.config["zfs.pool_name"], "deleted", "virtual-machines"), "volmode=none")
	if err != nil {
		return err
	}

	return nil
}
<|endoftext|>