<commit_before>package git

import (
	"os"
	"path/filepath"
	"strings"
	"time"

	"github.com/inconshreveable/log15"
	"github.com/spacemonkeygo/errors"
	"github.com/spacemonkeygo/errors/try"

	"go.polydawn.net/repeatr/lib/fs"
	"go.polydawn.net/repeatr/rio"
)

const Kind = rio.TransmatKind("git")

var _ rio.Transmat = &GitTransmat{}

type GitTransmat struct {
	workArea workArea
}

var _ rio.TransmatFactory = New

func New(workPath string) rio.Transmat {
	mustDir(workPath)
	workPath, err := filepath.Abs(workPath)
	if err != nil {
		panic(rio.TransmatError.New("Unable to set up workspace: %s", err))
	}
	wa := workArea{
		fullCheckouts:  filepath.Join(workPath, "full"),
		nosubCheckouts: filepath.Join(workPath, "nosub"),
		gitDirs:        filepath.Join(workPath, "gits"),
	}
	mustDir(wa.fullCheckouts)
	mustDir(wa.nosubCheckouts)
	mustDir(wa.gitDirs)
	return &GitTransmat{wa}
}

/*
	Git transmats plonk down the contents of one commit (or tree) as a filesystem.

	A fileset materialized by git does *not* include the `.git` dir by default,
	since those files are not themselves part of what's described by the hash.

	Git effectively "filters" out several attributes -- permissions are only loosely
	respected (execution only), file timestamps are undefined, uid/gid bits
	are not tracked, xattrs are not tracked, etc.  If you desired defined values,
	*you must still configure materialization to use a filter* (particularly for
	file timestamps, since they will otherwise be allowed to vary from one
	materialization to the next(!)).

	Git also allows for several other potential pitfalls with lossless data
	transmission: git cannot transmit empty directories.  This can be a major pain.
	Typical workarounds include creating a ".gitkeep" file in the empty directory.
	Gitignore files may also inadventantly cause trouble.  Transmat.Materialize
	will act *consistently*, but it does not overcome these issues in git
	(doing so would require additional metadata or protocol extensions).

	This transmat is *not* currently well optimized, and should generally be assumed
	to be re-cloning on all materializations -- specifically, it is not smart
	enough to recognize requests for different commits and trees from the
	same repos in order to save reclones.
*/
func (t *GitTransmat) Materialize(
	kind rio.TransmatKind,
	dataHash rio.CommitID,
	siloURIs []rio.SiloURI,
	log log15.Logger,
	options ...rio.MaterializerConfigurer,
) rio.Arena {
	var arena gitArena
	try.Do(func() {
		// Basic validation and config
		//config := rio.EvaluateConfig(options...)
		if kind != Kind {
			panic(errors.ProgrammerError.New("This transmat supports definitions of type %q, not %q", Kind, kind))
		}

		// Short circut out if we have the whole hash cached.
		finalPath := t.workArea.getFullchFinalPath(string(dataHash))
		if _, err := os.Stat(finalPath); err == nil {
			arena.workDirPath = finalPath
			arena.hash = dataHash
			return
		}

		// Emit git version.
		// Until we get a reasonably static version linked&contained, this is going to be an ongoing source of potential trouble.
		gitv := git.Bake("version").CombinedOutput()
		log.Info("using `git version`:", "v", strings.TrimSpace(gitv))

		// Ping silos
		if len(siloURIs) < 1 {
			panic(rio.ConfigError.New("Materialization requires at least one data source!"))
			// Note that it's possible a caching layer will satisfy things even without data sources...
			//  but if that was going to happen, it already would have by now.
		}
		// Our policy is to take the first path that exists.
		//  This lets you specify a series of potential locations,
		//  and if one is unavailable we'll just take the next.
		var warehouse *Warehouse
		for _, uri := range siloURIs {
			wh := NewWarehouse(uri)
			pong := wh.Ping()
			if pong == nil {
				log.Info("git: connected to remote warehouse", "remote", uri)
				warehouse = wh
				break
			} else {
				log.Info("Warehouse unavailable, skipping",
					"remote", uri,
					"reason", pong.Message(),
				)
			}
		}
		if warehouse == nil {
			panic(rio.WarehouseUnavailableError.New("No warehouses were available!"))
		}
		gitDirPath := t.workArea.gitDirPath(warehouse.url)

		// Fetch objects.
		func() {
			started := time.Now()
			yank(
				log,
				gitDirPath,
				warehouse.url,
			)
			log.Info("git: fetch complete",
				"elapsed", time.Now().Sub(started).Seconds(),
			)
		}()

		// Enumerate and fetch submodule objects.
		submodules := listSubmodules(string(dataHash), gitDirPath)
		log.Info("git: submodules found",
			"count", len(submodules),
		)
		submodules = applyGitmodulesUrls(string(dataHash), gitDirPath, submodules)
		func() {
			started := time.Now()
			// TODO ideally this would be smart enough to skip if we have hash cached. -- general problem with yank now actually
			for _, subm := range submodules {
				yank(
					log.New("submhash", subm.hash),
					t.workArea.gitDirPath(subm.url),
					subm.url,
				)
			}
			log.Info("git: fetch submodules complete",
				"elapsed", time.Now().Sub(started).Seconds(),
			)
		}()

		// Checkout.
		// Pick tempdir under full checkouts area.
		// We'll move from this tmpdir to the final one after both of:
		//  - this checkout
		//  - AND getting all submodules in place
		arena.workDirPath = t.workArea.makeFullchTempPath(string(dataHash))
		defer os.RemoveAll(arena.workDirPath)
		func() {
			started := time.Now()
			checkout(
				log,
				arena.workDirPath,
				string(dataHash),
				gitDirPath,
			)
			log.Info("git: checkout main repo complete",
				"elapsed", time.Now().Sub(started).Seconds(),
			)
		}()

		// Checkout submodules.
		// Pick tempdirs under the no-sub checkouts area (because we won't be recursing on these!)
		func() {
			started := time.Now()
			for _, subm := range submodules {
				pth := t.workArea.makeNosubchTempPath(subm.hash)
				defer os.RemoveAll(pth)
				checkout(
					log.New("submhash", subm.hash),
					pth,
					subm.hash,
					t.workArea.gitDirPath(subm.url),
				)
				moveOrShrug(pth, t.workArea.getNosubchFinalPath(subm.hash))
			}
			log.Info("git: checkout submodules complete",
				"elapsed", time.Now().Sub(started).Seconds(),
			)
		}()

		// Copy in submodules.
		func() {
			started := time.Now()
			for _, subm := range submodules {
				if err := fs.CopyR(
					t.workArea.getNosubchFinalPath(subm.hash),
					filepath.Join(arena.workDirPath, subm.path),
				); err != nil {
					panic(Error.New("Unexpected issues copying between local cache layers: %s", err))
				}
			}
			log.Info("git: full work tree assembled",
				"elapsed", time.Now().Sub(started).Seconds(),
			)
		}()

		// Since git doesn't convey permission bits, the default value
		// should be 1000 (consistent with being accessible under the "routine" policy).
		// Chown/chmod everything as such.
		if err := fs.Chownr(arena.workDirPath, git_uid, git_gid); err != nil {
			panic(rio.TransmatError.New("Unable to coerce perms: %s", err))
		}

		// verify total integrity
		// actually this is a nil step; there's no such thing as "acceptHashMismatch", checkout would have simply failed
		arena.hash = dataHash

		// Move the thing into final place!
		pth := t.workArea.getFullchFinalPath(string(dataHash))
		moveOrShrug(arena.workDirPath, pth)
		arena.workDirPath = pth
		log.Info("git: repo materialize complete")
	}).Catch(rio.Error, func(err *errors.Error) {
		panic(err)
	}).CatchAll(func(err error) {
		panic(rio.UnknownError.Wrap(err))
	}).Done()
	return arena
}

func (t GitTransmat) Scan(
	kind rio.TransmatKind,
	subjectPath string,
	siloURIs []rio.SiloURI,
	log log15.Logger,
	options ...rio.MaterializerConfigurer,
) rio.CommitID {
	var commitID rio.CommitID
	try.Do(func() {
		// Basic validation and config
		//config := rio.EvaluateConfig(options...)
		if kind != Kind {
			panic(errors.ProgrammerError.New("This transmat supports definitions of type %q, not %q", Kind, kind))
		}

		// Get off my lawn.
		panic(errors.NotImplementedError.New("The git transmat does not support scan."))
	}).Catch(rio.Error, func(err *errors.Error) {
		panic(err)
	}).CatchAll(func(err error) {
		panic(rio.UnknownError.Wrap(err))
	}).Done()
	return commitID
}

type gitArena struct {
	workDirPath string
	hash        rio.CommitID
}

func (a gitArena) Path() string {
	return a.workDirPath
}

func (a gitArena) Hash() rio.CommitID {
	return a.hash
}

// The git transmat teardown method is a stub.
// Unlike most other transmats, this one does its own caching and does not expect
// to have another dircacher layer wrapped around it.
func (a gitArena) Teardown() {
}
<commit_msg>Skip fetch for submodules if the full hash is already cached.<commit_after>package git

import (
	"os"
	"path/filepath"
	"strings"
	"time"

	"github.com/inconshreveable/log15"
	"github.com/spacemonkeygo/errors"
	"github.com/spacemonkeygo/errors/try"

	"go.polydawn.net/repeatr/lib/fs"
	"go.polydawn.net/repeatr/rio"
)

const Kind = rio.TransmatKind("git")

var _ rio.Transmat = &GitTransmat{}

type GitTransmat struct {
	workArea workArea
}

var _ rio.TransmatFactory = New

func New(workPath string) rio.Transmat {
	mustDir(workPath)
	workPath, err := filepath.Abs(workPath)
	if err != nil {
		panic(rio.TransmatError.New("Unable to set up workspace: %s", err))
	}
	wa := workArea{
		fullCheckouts:  filepath.Join(workPath, "full"),
		nosubCheckouts: filepath.Join(workPath, "nosub"),
		gitDirs:        filepath.Join(workPath, "gits"),
	}
	mustDir(wa.fullCheckouts)
	mustDir(wa.nosubCheckouts)
	mustDir(wa.gitDirs)
	return &GitTransmat{wa}
}

/*
	Git transmats plonk down the contents of one commit (or tree) as a filesystem.

	A fileset materialized by git does *not* include the `.git` dir by default,
	since those files are not themselves part of what's described by the hash.

	Git effectively "filters" out several attributes -- permissions are only loosely
	respected (execution only), file timestamps are undefined, uid/gid bits
	are not tracked, xattrs are not tracked, etc.  If you desired defined values,
	*you must still configure materialization to use a filter* (particularly for
	file timestamps, since they will otherwise be allowed to vary from one
	materialization to the next(!)).

	Git also allows for several other potential pitfalls with lossless data
	transmission: git cannot transmit empty directories.  This can be a major pain.
	Typical workarounds include creating a ".gitkeep" file in the empty directory.
	Gitignore files may also inadventantly cause trouble.  Transmat.Materialize
	will act *consistently*, but it does not overcome these issues in git
	(doing so would require additional metadata or protocol extensions).

	This transmat is *not* currently well optimized, and should generally be assumed
	to be re-cloning on all materializations -- specifically, it is not smart
	enough to recognize requests for different commits and trees from the
	same repos in order to save reclones.
*/
func (t *GitTransmat) Materialize(
	kind rio.TransmatKind,
	dataHash rio.CommitID,
	siloURIs []rio.SiloURI,
	log log15.Logger,
	options ...rio.MaterializerConfigurer,
) rio.Arena {
	var arena gitArena
	try.Do(func() {
		// Basic validation and config
		//config := rio.EvaluateConfig(options...)
		if kind != Kind {
			panic(errors.ProgrammerError.New("This transmat supports definitions of type %q, not %q", Kind, kind))
		}

		// Short circut out if we have the whole hash cached.
		finalPath := t.workArea.getFullchFinalPath(string(dataHash))
		if _, err := os.Stat(finalPath); err == nil {
			arena.workDirPath = finalPath
			arena.hash = dataHash
			return
		}

		// Emit git version.
		// Until we get a reasonably static version linked&contained, this is going to be an ongoing source of potential trouble.
		gitv := git.Bake("version").CombinedOutput()
		log.Info("using `git version`:", "v", strings.TrimSpace(gitv))

		// Ping silos
		if len(siloURIs) < 1 {
			panic(rio.ConfigError.New("Materialization requires at least one data source!"))
			// Note that it's possible a caching layer will satisfy things even without data sources...
			//  but if that was going to happen, it already would have by now.
		}
		// Our policy is to take the first path that exists.
		//  This lets you specify a series of potential locations,
		//  and if one is unavailable we'll just take the next.
		var warehouse *Warehouse
		for _, uri := range siloURIs {
			wh := NewWarehouse(uri)
			pong := wh.Ping()
			if pong == nil {
				log.Info("git: connected to remote warehouse", "remote", uri)
				warehouse = wh
				break
			} else {
				log.Info("Warehouse unavailable, skipping",
					"remote", uri,
					"reason", pong.Message(),
				)
			}
		}
		if warehouse == nil {
			panic(rio.WarehouseUnavailableError.New("No warehouses were available!"))
		}
		gitDirPath := t.workArea.gitDirPath(warehouse.url)

		// Fetch objects.
		func() {
			started := time.Now()
			yank(
				log,
				gitDirPath,
				warehouse.url,
			)
			log.Info("git: fetch complete",
				"elapsed", time.Now().Sub(started).Seconds(),
			)
		}()

		// Enumerate and fetch submodule objects.
		submodules := listSubmodules(string(dataHash), gitDirPath)
		log.Info("git: submodules found",
			"count", len(submodules),
		)
		submodules = applyGitmodulesUrls(string(dataHash), gitDirPath, submodules)
		func() {
			started := time.Now()
			for _, subm := range submodules {
				// Skip yank if we have the full checkout cached already.
				if _, err := os.Stat(t.workArea.getNosubchFinalPath(subm.hash)); err == nil {
					continue
				}
				// Okay, we need more stuff.  Fetch away.
				yank(
					log.New("submhash", subm.hash),
					t.workArea.gitDirPath(subm.url),
					subm.url,
				)
			}
			log.Info("git: fetch submodules complete",
				"elapsed", time.Now().Sub(started).Seconds(),
			)
		}()

		// Checkout.
		// Pick tempdir under full checkouts area.
		// We'll move from this tmpdir to the final one after both of:
		//  - this checkout
		//  - AND getting all submodules in place
		arena.workDirPath = t.workArea.makeFullchTempPath(string(dataHash))
		defer os.RemoveAll(arena.workDirPath)
		func() {
			started := time.Now()
			checkout(
				log,
				arena.workDirPath,
				string(dataHash),
				gitDirPath,
			)
			log.Info("git: checkout main repo complete",
				"elapsed", time.Now().Sub(started).Seconds(),
			)
		}()

		// Checkout submodules.
		// Pick tempdirs under the no-sub checkouts area (because we won't be recursing on these!)
		func() {
			started := time.Now()
			for _, subm := range submodules {
				pth := t.workArea.makeNosubchTempPath(subm.hash)
				defer os.RemoveAll(pth)
				checkout(
					log.New("submhash", subm.hash),
					pth,
					subm.hash,
					t.workArea.gitDirPath(subm.url),
				)
				moveOrShrug(pth, t.workArea.getNosubchFinalPath(subm.hash))
			}
			log.Info("git: checkout submodules complete",
				"elapsed", time.Now().Sub(started).Seconds(),
			)
		}()

		// Copy in submodules.
		func() {
			started := time.Now()
			for _, subm := range submodules {
				if err := fs.CopyR(
					t.workArea.getNosubchFinalPath(subm.hash),
					filepath.Join(arena.workDirPath, subm.path),
				); err != nil {
					panic(Error.New("Unexpected issues copying between local cache layers: %s", err))
				}
			}
			log.Info("git: full work tree assembled",
				"elapsed", time.Now().Sub(started).Seconds(),
			)
		}()

		// Since git doesn't convey permission bits, the default value
		// should be 1000 (consistent with being accessible under the "routine" policy).
		// Chown/chmod everything as such.
		if err := fs.Chownr(arena.workDirPath, git_uid, git_gid); err != nil {
			panic(rio.TransmatError.New("Unable to coerce perms: %s", err))
		}

		// verify total integrity
		// actually this is a nil step; there's no such thing as "acceptHashMismatch", checkout would have simply failed
		arena.hash = dataHash

		// Move the thing into final place!
		pth := t.workArea.getFullchFinalPath(string(dataHash))
		moveOrShrug(arena.workDirPath, pth)
		arena.workDirPath = pth
		log.Info("git: repo materialize complete")
	}).Catch(rio.Error, func(err *errors.Error) {
		panic(err)
	}).CatchAll(func(err error) {
		panic(rio.UnknownError.Wrap(err))
	}).Done()
	return arena
}

func (t GitTransmat) Scan(
	kind rio.TransmatKind,
	subjectPath string,
	siloURIs []rio.SiloURI,
	log log15.Logger,
	options ...rio.MaterializerConfigurer,
) rio.CommitID {
	var commitID rio.CommitID
	try.Do(func() {
		// Basic validation and config
		//config := rio.EvaluateConfig(options...)
		if kind != Kind {
			panic(errors.ProgrammerError.New("This transmat supports definitions of type %q, not %q", Kind, kind))
		}

		// Get off my lawn.
		panic(errors.NotImplementedError.New("The git transmat does not support scan."))
	}).Catch(rio.Error, func(err *errors.Error) {
		panic(err)
	}).CatchAll(func(err error) {
		panic(rio.UnknownError.Wrap(err))
	}).Done()
	return commitID
}

type gitArena struct {
	workDirPath string
	hash        rio.CommitID
}

func (a gitArena) Path() string {
	return a.workDirPath
}

func (a gitArena) Hash() rio.CommitID {
	return a.hash
}

// The git transmat teardown method is a stub.
// Unlike most other transmats, this one does its own caching and does not expect
// to have another dircacher layer wrapped around it.
func (a gitArena) Teardown() {
}
<|endoftext|>


<commit_before>// Copyright 2016 Joel Scoble and The JoeFriday authors.
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package diskstats handles processing of IO statistics of each block device:
// /proc/diskstats.
package diskstats

import (
	"fmt"
	"io"
	"sync"
	"time"

	"github.com/SermoDigital/helpers"
	joe "github.com/mohae/joefriday"
	"github.com/mohae/joefriday/disk/structs"
)

const procFile = "/proc/diskstats"

// Profiler is used to process the /proc/diskstats file.
type Profiler struct {
	*joe.Proc
}

// Returns an initialized Profiler; ready to use.
func NewProfiler() (prof *Profiler, err error) {
	proc, err := joe.New(procFile)
	if err != nil {
		return nil, err
	}
	return &Profiler{Proc: proc}, nil
}

// Get returns information about current IO statistics of the block devices.
func (prof *Profiler) Get() (stats *structs.DiskStats, err error) {
	err = prof.Reset()
	if err != nil {
		return nil, err
	}
	var (
		i, priorPos, pos, line, fieldNum int
		n                                uint64
		v                                byte
		dev                              structs.Device
	)

	stats = &structs.DiskStats{Timestamp: time.Now().UTC().UnixNano(), Device: make([]structs.Device, 0, 2)}

	// read each line until eof
	for {
		prof.Line, err = prof.Buf.ReadSlice('\n')
		if err != nil {
			if err == io.EOF {
				break
			}
			return nil, &joe.ReadError{Err: err}
		}
		line++
		pos = 0
		fieldNum = 0
		// process the fields in the line
		for {
			// ignore spaces on the first two fields
			if fieldNum < 2 {
				for i, v = range prof.Line[pos:] {
					if v != 0x20 {
						break
					}
				}
				pos += i
			}
			fieldNum++
			for i, v = range prof.Line[pos:] {
				if v == 0x20 || v == '\n' {
					break
				}
			}
			if fieldNum != 3 {
				n, err = helpers.ParseUint(prof.Line[pos : pos+i])
				if err != nil {
					return stats, &joe.ParseError{Info: fmt.Sprintf("line %d: field %d", line, fieldNum), Err: err}
				}
			}
			priorPos, pos = pos, pos+i+1
			if fieldNum < 8 {
				if fieldNum < 4 {
					if fieldNum < 2 {
						if fieldNum == 1 {
							dev.Major = uint32(n)
							continue
						}
						dev.Minor = uint32(n)
						continue
					}
					dev.Name = string(prof.Line[priorPos:pos])
					continue
				}
				if fieldNum < 6 {
					if fieldNum == 4 {
						dev.ReadsCompleted = n
						continue
					}
					dev.ReadsMerged = n
					continue
				}
				if fieldNum == 6 {
					dev.ReadSectors = n
					continue
				}
				dev.ReadingTime = n
				continue
			}
			if fieldNum < 12 {
				if fieldNum < 10 {
					if fieldNum == 8 {
						dev.WritesCompleted = n
						continue
					}
					dev.WritesMerged = n
					continue
				}
				if fieldNum == 10 {
					dev.WrittenSectors = n
					continue
				}
				dev.WritingTime = n
				continue
			}
			if fieldNum == 12 {
				dev.IOInProgress = int32(n)
				continue
			}
			if fieldNum == 13 {
				dev.IOTime = n
				continue
			}
			dev.WeightedIOTime = n
			break
		}
		stats.Device = append(stats.Device, dev)
	}
	return stats, nil
}

var std *Profiler
var stdMu sync.Mutex

// Get returns the current IO statistics of the block devices using the
// package's global Profiler.
func Get() (stat *structs.DiskStats, err error) {
	stdMu.Lock()
	defer stdMu.Unlock()
	if std == nil {
		std, err = NewProfiler()
		if err != nil {
			return nil, err
		}
	}
	return std.Get()
}

// Ticker delivers the system's IO statistics of the block devices at
// intervals.
type Ticker struct {
	*joe.Ticker
	Data chan *structs.DiskStats
	*Profiler
}

// NewTicker returns a new Ticker containing a Data channel that delivers the
// data at intervals and an error channel that delivers any errors encountered.
// Stop the ticker to signal the ticker to stop running. Stopping the ticker
// does not close the Data channel; call Close to close both the ticker and the
// data channel.
func NewTicker(d time.Duration) (joe.Tocker, error) {
	p, err := NewProfiler()
	if err != nil {
		return nil, err
	}
	t := Ticker{Ticker: joe.NewTicker(d), Data: make(chan *structs.DiskStats), Profiler: p}
	go t.Run()
	return &t, nil
}

// Run runs the ticker.
func (t *Ticker) Run() {
	// ticker
	for {
		select {
		case <-t.Done:
			return
		case <-t.C:
			s, err := t.Get()
			if err != nil {
				t.Errs <- err
				continue
			}
			t.Data <- s
		}
	}
}

// Close closes the ticker resources.
func (t *Ticker) Close() {
	t.Ticker.Close()
	close(t.Data)
}
<commit_msg>refactor Profiler to use joe.Procer and joe.Buffer; add Reset method<commit_after>// Copyright 2016 Joel Scoble and The JoeFriday authors.
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package diskstats handles processing of IO statistics of each block device:
// /proc/diskstats.
package diskstats

import (
	"fmt"
	"io"
	"sync"
	"time"

	"github.com/SermoDigital/helpers"
	joe "github.com/mohae/joefriday"
	"github.com/mohae/joefriday/disk/structs"
)

const procFile = "/proc/diskstats"

// Profiler is used to process the /proc/diskstats file.
type Profiler struct {
	joe.Procer
	*joe.Buffer
}

// Returns an initialized Profiler; ready to use.
func NewProfiler() (prof *Profiler, err error) {
	proc, err := joe.New(procFile)
	if err != nil {
		return nil, err
	}
	return &Profiler{Procer: proc, Buffer: joe.NewBuffer()}, nil
}

// Reset resources: after reset, the profiler is ready to be used again.
func (prof *Profiler) Reset() error {
	prof.Buffer.Reset()
	return prof.Procer.Reset()
}

// Get returns information about current IO statistics of the block devices.
func (prof *Profiler) Get() (stats *structs.DiskStats, err error) {
	err = prof.Reset()
	if err != nil {
		return nil, err
	}
	var (
		i, priorPos, pos, line, fieldNum int
		n                                uint64
		v                                byte
		dev                              structs.Device
	)

	stats = &structs.DiskStats{Timestamp: time.Now().UTC().UnixNano(), Device: make([]structs.Device, 0, 2)}

	// read each line until eof
	for {
		prof.Line, err = prof.ReadSlice('\n')
		if err != nil {
			if err == io.EOF {
				break
			}
			return nil, &joe.ReadError{Err: err}
		}
		line++
		pos = 0
		fieldNum = 0
		// process the fields in the line
		for {
			// ignore spaces on the first two fields
			if fieldNum < 2 {
				for i, v = range prof.Line[pos:] {
					if v != 0x20 {
						break
					}
				}
				pos += i
			}
			fieldNum++
			for i, v = range prof.Line[pos:] {
				if v == 0x20 || v == '\n' {
					break
				}
			}
			if fieldNum != 3 {
				n, err = helpers.ParseUint(prof.Line[pos : pos+i])
				if err != nil {
					return stats, &joe.ParseError{Info: fmt.Sprintf("line %d: field %d", line, fieldNum), Err: err}
				}
			}
			priorPos, pos = pos, pos+i+1
			if fieldNum < 8 {
				if fieldNum < 4 {
					if fieldNum < 2 {
						if fieldNum == 1 {
							dev.Major = uint32(n)
							continue
						}
						dev.Minor = uint32(n)
						continue
					}
					dev.Name = string(prof.Line[priorPos:pos])
					continue
				}
				if fieldNum < 6 {
					if fieldNum == 4 {
						dev.ReadsCompleted = n
						continue
					}
					dev.ReadsMerged = n
					continue
				}
				if fieldNum == 6 {
					dev.ReadSectors = n
					continue
				}
				dev.ReadingTime = n
				continue
			}
			if fieldNum < 12 {
				if fieldNum < 10 {
					if fieldNum == 8 {
						dev.WritesCompleted = n
						continue
					}
					dev.WritesMerged = n
					continue
				}
				if fieldNum == 10 {
					dev.WrittenSectors = n
					continue
				}
				dev.WritingTime = n
				continue
			}
			if fieldNum == 12 {
				dev.IOInProgress = int32(n)
				continue
			}
			if fieldNum == 13 {
				dev.IOTime = n
				continue
			}
			dev.WeightedIOTime = n
			break
		}
		stats.Device = append(stats.Device, dev)
	}
	return stats, nil
}

var std *Profiler
var stdMu sync.Mutex

// Get returns the current IO statistics of the block devices using the
// package's global Profiler.
func Get() (stat *structs.DiskStats, err error) {
	stdMu.Lock()
	defer stdMu.Unlock()
	if std == nil {
		std, err = NewProfiler()
		if err != nil {
			return nil, err
		}
	}
	return std.Get()
}

// Ticker delivers the system's IO statistics of the block devices at
// intervals.
type Ticker struct {
	*joe.Ticker
	Data chan *structs.DiskStats
	*Profiler
}

// NewTicker returns a new Ticker containing a Data channel that delivers the
// data at intervals and an error channel that delivers any errors encountered.
// Stop the ticker to signal the ticker to stop running. Stopping the ticker
// does not close the Data channel; call Close to close both the ticker and the
// data channel.
func NewTicker(d time.Duration) (joe.Tocker, error) {
	p, err := NewProfiler()
	if err != nil {
		return nil, err
	}
	t := Ticker{Ticker: joe.NewTicker(d), Data: make(chan *structs.DiskStats), Profiler: p}
	go t.Run()
	return &t, nil
}

// Run runs the ticker.
func (t *Ticker) Run() {
	// ticker
	for {
		select {
		case <-t.Done:
			return
		case <-t.C:
			s, err := t.Get()
			if err != nil {
				t.Errs <- err
				continue
			}
			t.Data <- s
		}
	}
}

// Close closes the ticker resources.
func (t *Ticker) Close() {
	t.Ticker.Close()
	close(t.Data)
}
<|endoftext|>


<commit_before>package realms

import (
	"crypto/rand"
	"encoding/hex"
	"encoding/json"
	log "github.com/Sirupsen/logrus"
	"github.com/matrix-org/go-neb/database"
	"github.com/matrix-org/go-neb/types"
	"net/http"
	"net/url"
)

type githubRealm struct {
	id              string
	ClientSecret    string
	ClientID        string
	RedirectBaseURI string
}

type githubSession struct {
	State   string
	id      string
	userID  string
	realmID string
}

func (s *githubSession) UserID() string {
	return s.userID
}

func (s *githubSession) RealmID() string {
	return s.realmID
}

func (s *githubSession) ID() string {
	return s.id
}

func (r *githubRealm) ID() string {
	return r.id
}

func (r *githubRealm) Type() string {
	return "github"
}

func (r *githubRealm) RequestAuthSession(userID string, req json.RawMessage) interface{} {
	state, err := randomString(10)
	if err != nil {
		log.WithError(err).Print("Failed to generate state param")
		return nil
	}
	u, _ := url.Parse("https://github.com/login/oauth/authorize")
	q := u.Query()
	q.Set("client_id", r.ClientID)
	q.Set("client_secret", r.ClientSecret)
	q.Set("state", state)
	// TODO: Path is from goneb.go - we should probably factor it out.
	q.Set("redirect_uri", r.RedirectBaseURI+"/realms/redirects/"+r.ID())
	u.RawQuery = q.Encode()
	session := &githubSession{
		id:      state, // key off the state for redirects
		userID:  userID,
		realmID: r.ID(),
	}
	_, err = database.GetServiceDB().StoreAuthSession(session)
	if err != nil {
		log.WithError(err).Print("Failed to store new auth session")
		return nil
	}

	return &struct {
		URL string
	}{u.String()}
}

func (r *githubRealm) OnReceiveRedirect(w http.ResponseWriter, req *http.Request) {
	code := req.URL.Query().Get("code")
	state := req.URL.Query().Get("state")
	logger := log.WithFields(log.Fields{
		"state": state,
	})
	logger.WithField("code", code).Print("GithubRealm: OnReceiveRedirect")
	if code == "" || state == "" {
		w.WriteHeader(400)
		w.Write([]byte("code and state are required"))
		return
	}
	// load the session (we keyed off the state param)
	session, err := database.GetServiceDB().LoadAuthSessionByID(r.ID(), state)
	if err != nil {
		logger.WithError(err).Print("Failed to load session")
		w.WriteHeader(400)
		w.Write([]byte("Provided ?state= param is not recognised.")) // most likely cause
		return
	}
	logger.WithField("user_id", session.UserID()).Print("Mapped redirect to user")
}

func (r *githubRealm) AuthSession(id, userID, realmID string) types.AuthSession {
	return &githubSession{
		id:      id,
		userID:  userID,
		realmID: realmID,
	}
}

// Generate a cryptographically secure pseudorandom string with the given number of bytes (length).
// Returns a hex string of the bytes.
func randomString(length int) (string, error) {
	b := make([]byte, length)
	_, err := rand.Read(b)
	if err != nil {
		return "", err
	}
	return hex.EncodeToString(b), nil
}

func init() {
	types.RegisterAuthRealm(func(realmID string) types.AuthRealm {
		return &githubRealm{id: realmID}
	})
}
<commit_msg>Store access_tokens in the DB<commit_after>package realms

import (
	"crypto/rand"
	"encoding/hex"
	"encoding/json"
	log "github.com/Sirupsen/logrus"
	"github.com/matrix-org/go-neb/database"
	"github.com/matrix-org/go-neb/types"
	"io/ioutil"
	"net/http"
	"net/url"
)

type githubRealm struct {
	id              string
	ClientSecret    string
	ClientID        string
	RedirectBaseURI string
}

type githubSession struct {
	AccessToken string
	Scopes      string
	id          string
	userID      string
	realmID     string
}

func (s *githubSession) UserID() string {
	return s.userID
}

func (s *githubSession) RealmID() string {
	return s.realmID
}

func (s *githubSession) ID() string {
	return s.id
}

func (r *githubRealm) ID() string {
	return r.id
}

func (r *githubRealm) Type() string {
	return "github"
}

func (r *githubRealm) RequestAuthSession(userID string, req json.RawMessage) interface{} {
	state, err := randomString(10)
	if err != nil {
		log.WithError(err).Print("Failed to generate state param")
		return nil
	}
	u, _ := url.Parse("https://github.com/login/oauth/authorize")
	q := u.Query()
	q.Set("client_id", r.ClientID)
	q.Set("client_secret", r.ClientSecret)
	q.Set("state", state)
	// TODO: Path is from goneb.go - we should probably factor it out.
	q.Set("redirect_uri", r.RedirectBaseURI+"/realms/redirects/"+r.ID())
	u.RawQuery = q.Encode()
	session := &githubSession{
		id:      state, // key off the state for redirects
		userID:  userID,
		realmID: r.ID(),
	}
	_, err = database.GetServiceDB().StoreAuthSession(session)
	if err != nil {
		log.WithError(err).Print("Failed to store new auth session")
		return nil
	}

	return &struct {
		URL string
	}{u.String()}
}

func (r *githubRealm) OnReceiveRedirect(w http.ResponseWriter, req *http.Request) {
	// parse out params from the request
	code := req.URL.Query().Get("code")
	state := req.URL.Query().Get("state")
	logger := log.WithFields(log.Fields{
		"state": state,
	})
	logger.WithField("code", code).Print("GithubRealm: OnReceiveRedirect")
	if code == "" || state == "" {
		failWith(logger, w, 400, "code and state are required", nil)
		return
	}
	// load the session (we keyed off the state param)
	session, err := database.GetServiceDB().LoadAuthSessionByID(r.ID(), state)
	if err != nil {
		// most likely cause
		failWith(logger, w, 400, "Provided ?state= param is not recognised.", err)
		return
	}
	ghSession := session.(*githubSession)
	logger.WithField("user_id", ghSession.UserID()).Print("Mapped redirect to user")

	// exchange code for access_token
	res, err := http.PostForm("https://github.com/login/oauth/access_token",
		url.Values{"client_id": {r.ClientID}, "client_secret": {r.ClientSecret}, "code": {code}})
	if err != nil {
		failWith(logger, w, 502, "Failed to exchange code for token", err)
		return
	}
	defer res.Body.Close()
	body, err := ioutil.ReadAll(res.Body)
	if err != nil {
		failWith(logger, w, 502, "Failed to read token response", err)
		return
	}
	vals, err := url.ParseQuery(string(body))
	if err != nil {
		failWith(logger, w, 502, "Failed to parse token response", err)
		return
	}

	// update database and return
	ghSession.AccessToken = vals.Get("access_token")
	ghSession.Scopes = vals.Get("scope")
	logger.WithField("scope", ghSession.Scopes).Print("Scopes granted.")
	_, err = database.GetServiceDB().StoreAuthSession(ghSession)
	if err != nil {
		failWith(logger, w, 500, "Failed to persist session", err)
		return
	}
	w.WriteHeader(200)
	w.Write([]byte("OK!"))
}

func (r *githubRealm) AuthSession(id, userID, realmID string) types.AuthSession {
	return &githubSession{
		id:      id,
		userID:  userID,
		realmID: realmID,
	}
}

func failWith(logger *log.Entry, w http.ResponseWriter, code int, msg string, err error) {
	logger.WithError(err).Print(msg)
	w.WriteHeader(code)
	w.Write([]byte(msg))
}

// Generate a cryptographically secure pseudorandom string with the given number of bytes (length).
// Returns a hex string of the bytes.
func randomString(length int) (string, error) {
	b := make([]byte, length)
	_, err := rand.Read(b)
	if err != nil {
		return "", err
	}
	return hex.EncodeToString(b), nil
}

func init() {
	types.RegisterAuthRealm(func(realmID string) types.AuthRealm {
		return &githubRealm{id: realmID}
	})
}
<|endoftext|>


<commit_before>package main

import (
	"context"
	"crypto/tls"
	"crypto/x509"
	"encoding/json"
	"google.golang.org/grpc"
	"google.golang.org/grpc/credentials"
	"log"
	"notification.com/model"
	"notification.com/proto"
	"os"
	"strconv"
	"time"
)

var domain = os.Getenv("DOMAIN")
var port = os.Getenv("PORT")
var insecureStr = os.Getenv("INSECURE")

type notificationClient struct {
	client    proto.NotificationClient
	conn      *grpc.ClientConn
	eventName string
}

func makeNotificationClient(eventName string) *notificationClient {
	conn, err := makeConnection()
	if err != nil {
		log.Fatalf("could not make connection")
	}
	return &notificationClient{
		client:    proto.NewNotificationClient(conn),
		conn:      conn,
		eventName: eventName,
	}
}

func (c *notificationClient) getNotification() (proto.Notification_GetNotificationClient, error) {
	return c.client.GetNotification(context.Background(), &proto.NotificationRequest{EventName: c.eventName})
}

func (c *notificationClient) startSubscription() {
	var err error
	var stream proto.Notification_GetNotificationClient
	log.Println("Subscribing...")
	for {
		if stream == nil {
			if stream, err = c.getNotification(); err != nil {
				log.Printf("Failed to have stream: %v", err)
				c.sleep()
				continue
			}
		}
		msg, err := stream.Recv()
		if err != nil {
			log.Printf("Failed to receive message: %v", err)
			stream = nil
			c.sleep()
			continue
		}
		event := model.Event{
			EventName: msg.GetEventName(),
			Purchaser: msg.GetPurchaser(),
			OrderID:   uint(msg.GetOrderId()),
			ItemID:    uint(msg.GetItemId()),
		}
		bytes, _ := json.Marshal(&event)
		log.Println(string(bytes))
	}
}

func (c *notificationClient) sleep() {
	time.Sleep(time.Second * 5)
}

func makeConnection() (*grpc.ClientConn, error) {
	var opts []grpc.DialOption
	host := domain + ":" + port
	insecure, _ := strconv.ParseBool(insecureStr)
	if insecure {
		opts = append(opts, grpc.WithInsecure())
	}else {
		systemRoots, err := x509.SystemCertPool()
		if err != nil {
			log.Fatalf("Could not make cert pool: %v", err)
		}
		cred := credentials.NewTLS(&tls.Config{
			RootCAs: systemRoots,
		})
		opts = append(opts, grpc.WithTransportCredentials(cred))
	}
	return grpc.Dial(host, opts...)
}

func main() {
	client := makeNotificationClient("all")
	client.startSubscription()
}
<commit_msg>fix a gosec warning 'G402: TLS MinVersion too low.'<commit_after>package main

import (
	"context"
	"crypto/tls"
	"crypto/x509"
	"encoding/json"
	"google.golang.org/grpc"
	"google.golang.org/grpc/credentials"
	"log"
	"notification.com/model"
	"notification.com/proto"
	"os"
	"strconv"
	"time"
)

var domain = os.Getenv("DOMAIN")
var port = os.Getenv("PORT")
var insecureStr = os.Getenv("INSECURE")

type notificationClient struct {
	client    proto.NotificationClient
	conn      *grpc.ClientConn
	eventName string
}

func makeNotificationClient(eventName string) *notificationClient {
	conn, err := makeConnection()
	if err != nil {
		log.Fatalf("could not make connection")
	}
	return &notificationClient{
		client:    proto.NewNotificationClient(conn),
		conn:      conn,
		eventName: eventName,
	}
}

func (c *notificationClient) getNotification() (proto.Notification_GetNotificationClient, error) {
	return c.client.GetNotification(context.Background(), &proto.NotificationRequest{EventName: c.eventName})
}

func (c *notificationClient) startSubscription() {
	var err error
	var stream proto.Notification_GetNotificationClient
	log.Println("Subscribing...")
	for {
		if stream == nil {
			if stream, err = c.getNotification(); err != nil {
				log.Printf("Failed to have stream: %v", err)
				c.sleep()
				continue
			}
		}
		msg, err := stream.Recv()
		if err != nil {
			log.Printf("Failed to receive message: %v", err)
			stream = nil
			c.sleep()
			continue
		}
		event := model.Event{
			EventName: msg.GetEventName(),
			Purchaser: msg.GetPurchaser(),
			OrderID:   uint(msg.GetOrderId()),
			ItemID:    uint(msg.GetItemId()),
		}
		bytes, _ := json.Marshal(&event)
		log.Println(string(bytes))
	}
}

func (c *notificationClient) sleep() {
	time.Sleep(time.Second * 5)
}

func makeConnection() (*grpc.ClientConn, error) {
	var opts []grpc.DialOption
	host := domain + ":" + port
	insecure, _ := strconv.ParseBool(insecureStr)
	if insecure {
		opts = append(opts, grpc.WithInsecure())
	}else {
		systemRoots, err := x509.SystemCertPool()
		if err != nil {
			log.Fatalf("Could not make cert pool: %v", err)
		}
		cred := credentials.NewTLS(&tls.Config{
			RootCAs:    systemRoots,
			MinVersion: tls.VersionTLS12,
		})
		opts = append(opts, grpc.WithTransportCredentials(cred))
	}
	return grpc.Dial(host, opts...)
}

func main() {
	client := makeNotificationClient("all")
	client.startSubscription()
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/gob"
	"errors"
	"fmt"
	"github.com/hanwen/go-fuse/fuse"
	"github.com/hanwen/go-fuse/fuse/nodefs"
	"io"
	"io/ioutil"
)

const (
	kMaxEntryNumberInFile = 1024
)

type NodeInfo struct {
}

type Entry struct {
	Id       int
	ParentId int
	Dir      bool
	Key      string
	Name     string
	FsName   string
	fs       *FileSystem
	nodefs.Node
}

func NewDir(id int, name string, parent *Entry) (*Entry, error) {
	return newEntry(id, name, "", "", parent, true)
}

func NewFile(id int, name string, fsName string, key string, parent *Entry) (*Entry, error) {
	return newEntry(id, name, fsName, key, parent, false)
}

func newEntry(id int, name string, fsName string, key string, parent *Entry, dir bool) (*Entry, error) {
	if parent != nil && parent.Dir == false {
		return nil, errors.New("parent entry can't be file type")
	}
	entry := Entry{id, 0, dir, key, name, fsName, nil, nil}

	entry.Id = id
	if parent == nil {
		entry.ParentId = 0
	} else {
		entry.ParentId = parent.Id
	}
	entry.Key = key
	entry.Dir = dir
	entry.Name = name
	entry.FsName = fsName
	return &entry, nil
}

func ReadEntries(reader io.Reader, buff []*Entry) error {
	dec := gob.NewDecoder(reader)
	return dec.Decode(&buff)
}

func WriteEntries(writer io.Writer, buff []*Entry) error {
	enc := gob.NewEncoder(writer)
	return enc.Encode(buff)
}

func (e *Entry) Print() {
	if e.Dir {
		fmt.Printf("[Dir]: Id: %d, ParentId: %d, Name: %s, FsName: %s\n",
			e.Id, e.ParentId, e.Name, e.FsName)
	} else {
		fmt.Printf("Id: %d, ParentId: %d, Name: %s, FsName: %s, Key %s\n",
			e.Id, e.ParentId, e.Name, e.FsName, e.Key)
	}
}

func (e *Entry) Stat(out *fuse.Attr) {
	out.Mode = fuse.S_IFREG | 0444
	out.Size = e.fs.oss.Size(e.FsName)
}

func (e *Entry) Data() (data []byte) {
	fp, err := e.fs.oss.Open(e.FsName)
	if err != nil {
		return nil
	}
	fpd := MakeDecryptor(e.fs.config.Enc, fp)
	data, _ = ioutil.ReadAll(fpd)
	fpd.Close()
	fp.Close()
	return
}

func (n *Entry) OnMount(c *nodefs.FileSystemConnector) {
	n.fs.onMount()
}

func (n *Entry) OpenDir(context *fuse.Context) (stream []fuse.DirEntry, code fuse.Status) {
	children := n.Inode().Children()
	stream = make([]fuse.DirEntry, 0, len(children))
	for k, v := range children {
		mode := fuse.S_IFREG | 0666
		if v.IsDir() {
			mode = fuse.S_IFDIR | 0777
		}
		stream = append(stream, fuse.DirEntry{
			Name: k,
			Mode: uint32(mode),
		})
	}
	return stream, fuse.OK
}

func (n *Entry) Open(flags uint32, context *fuse.Context) (fuseFile nodefs.File, code fuse.Status) {
	if flags&fuse.O_ANYWRITE != 0 {
		return nil, fuse.EPERM
	}

	return nodefs.NewDataFile(n.Data()), fuse.OK
}

func (n *Entry) Deletable() bool {
	return false
}

func (n *Entry) GetAttr(out *fuse.Attr, file nodefs.File, context *fuse.Context) fuse.Status {
	if n.Inode().IsDir() {
		out.Mode = fuse.S_IFDIR | 0777
		return fuse.OK
	}
	n.Stat(out)
	return fuse.OK
}
<commit_msg>fix data size issue<commit_after>package main

import (
	"encoding/gob"
	"errors"
	"fmt"
	"github.com/hanwen/go-fuse/fuse"
	"github.com/hanwen/go-fuse/fuse/nodefs"
	"io"
	"io/ioutil"
)

const (
	kMaxEntryNumberInFile = 1024
)

type Entry struct {
	Id       int
	ParentId int
	Dir      bool
	Key      string
	Name     string
	FsName   string
	fs       *FileSystem
	nodefs.Node
	size uint64
}

func NewDir(id int, name string, parent *Entry) (*Entry, error) {
	return newEntry(id, name, "", "", parent, true)
}

func NewFile(id int, name string, fsName string, key string, parent *Entry) (*Entry, error) {
	return newEntry(id, name, fsName, key, parent, false)
}

func newEntry(id int, name string, fsName string, key string, parent *Entry, dir bool) (*Entry, error) {
	if parent != nil && parent.Dir == false {
		return nil, errors.New("parent entry can't be file type")
	}
	entry := Entry{id, 0, dir, key, name, fsName, nil, nil, 0}

	entry.Id = id
	if parent == nil {
		entry.ParentId = 0
	} else {
		entry.ParentId = parent.Id
	}
	entry.Key = key
	entry.Dir = dir
	entry.Name = name
	entry.FsName = fsName
	return &entry, nil
}

func ReadEntries(reader io.Reader, buff []*Entry) error {
	dec := gob.NewDecoder(reader)
	return dec.Decode(&buff)
}

func WriteEntries(writer io.Writer, buff []*Entry) error {
	enc := gob.NewEncoder(writer)
	return enc.Encode(buff)
}

func (e *Entry) Print() {
	if e.Dir {
		fmt.Printf("[Dir]: Id: %d, ParentId: %d, Name: %s, FsName: %s\n",
			e.Id, e.ParentId, e.Name, e.FsName)
	} else {
		fmt.Printf("Id: %d, ParentId: %d, Name: %s, FsName: %s, Key %s\n",
			e.Id, e.ParentId, e.Name, e.FsName, e.Key)
	}
}

func (e *Entry) Stat(out *fuse.Attr) {
	out.Mode = fuse.S_IFREG | 0444
	if e.size != 0 {
		out.Size = e.fs.oss.Size(e.FsName)
		e.size = out.Size
	} else {
		out.Size = e.size
	}
}

func (e *Entry) Data() (data []byte) {
	fp, err := e.fs.oss.Open(e.FsName)
	if err != nil {
		return nil
	}
	fpd := MakeDecryptor(e.fs.config.Enc, fp)
	data, _ = ioutil.ReadAll(fpd)

	out := fuse.Attr{}
	e.Stat(&out)
	size := int(out.Size)
	if len(data) < size {
		data = append(data, make([]byte, size-len(data))...)
	}

	fpd.Close()
	fp.Close()
	return
}

func (n *Entry) OnMount(c *nodefs.FileSystemConnector) {
	n.fs.onMount()
}

func (n *Entry) OpenDir(context *fuse.Context) (stream []fuse.DirEntry, code fuse.Status) {
	children := n.Inode().Children()
	stream = make([]fuse.DirEntry, 0, len(children))
	for k, v := range children {
		mode := fuse.S_IFREG | 0666
		if v.IsDir() {
			mode = fuse.S_IFDIR | 0777
		}
		stream = append(stream, fuse.DirEntry{
			Name: k,
			Mode: uint32(mode),
		})
	}
	return stream, fuse.OK
}

func (n *Entry) Open(flags uint32, context *fuse.Context) (fuseFile nodefs.File, code fuse.Status) {
	if flags&fuse.O_ANYWRITE != 0 {
		return nil, fuse.EPERM
	}

	return nodefs.NewDataFile(n.Data()), fuse.OK
}

func (n *Entry) Deletable() bool {
	return false
}

func (n *Entry) GetAttr(out *fuse.Attr, file nodefs.File, context *fuse.Context) fuse.Status {
	if n.Inode().IsDir() {
		out.Mode = fuse.S_IFDIR | 0777
		return fuse.OK
	}
	n.Stat(out)
	return fuse.OK
}
<|endoftext|>


<commit_before>// Copyright ©2015 The gonum Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package mat64

import (
	"math/rand"

	"gopkg.in/check.v1"
)

type dims struct{ r, c int }

var productTests = []struct {
	n       int
	factors []dims
	product dims
	panics  bool
}{
	{
		n:       1,
		factors: []dims{{3, 4}},
		product: dims{3, 4},
		panics:  false,
	},
	{
		n:       1,
		factors: []dims{{2, 4}},
		product: dims{3, 4},
		panics:  true,
	},
	{
		n:       3,
		factors: []dims{{10, 30}, {30, 5}, {5, 60}},
		product: dims{10, 60},
		panics:  false,
	},
	{
		n:       3,
		factors: []dims{{100, 30}, {30, 5}, {5, 60}},
		product: dims{10, 60},
		panics:  true,
	},
	{
		n:       7,
		factors: []dims{{60, 5}, {5, 5}, {5, 4}, {4, 10}, {10, 22}, {22, 45}, {45, 10}},
		product: dims{60, 10},
		panics:  false,
	},
	{
		n:       7,
		factors: []dims{{60, 5}, {5, 5}, {5, 400}, {4, 10}, {10, 22}, {22, 45}, {45, 10}},
		product: dims{60, 10},
		panics:  true,
	},
	{
		n:       3,
		factors: []dims{{1, 1000}, {1000, 2}, {2, 2}},
		product: dims{1, 2},
		panics:  false,
	},

	// Random chains.
	{
		n:       0,
		product: dims{0, 0},
		panics:  false,
	},
	{
		n:       2,
		product: dims{60, 10},
		panics:  false,
	},
	{
		n:       3,
		product: dims{60, 10},
		panics:  false,
	},
	{
		n:       4,
		product: dims{60, 10},
		panics:  false,
	},
	{
		n:       10,
		product: dims{60, 10},
		panics:  false,
	},
}

func (s *S) TestProduct(c *check.C) {
	for _, test := range productTests {
		dimensions := test.factors
		if dimensions == nil && test.n > 0 {
			dimensions = make([]dims, test.n)
			for i := range dimensions {
				if i != 0 {
					dimensions[i].r = dimensions[i-1].c
				}
				dimensions[i].c = rand.Intn(50) + 1
			}
			dimensions[0].r = test.product.r
			dimensions[test.n-1].c = test.product.c
		}
		factors := make([]Matrix, test.n)
		for i, d := range dimensions {
			data := make([]float64, d.r*d.c)
			for i := range data {
				data[i] = rand.Float64()
			}
			factors[i] = NewDense(d.r, d.c, data)
		}

		want := &Dense{}
		if !test.panics {
			a := &Dense{}
			for i, b := range factors {
				if i == 0 {
					want.Clone(b)
					continue
				}
				a, want = want, &Dense{}
				want.Mul(a, b)
			}
		}

		got := NewDense(test.product.r, test.product.c, nil)
		panicked, message := panics(func() {
			got.Product(factors...)
		})
		if test.panics {
			if !panicked {
				c.Errorf("fail to panic with product chain dimentions: %+v result dimension: %+v",
					dimensions, test.product)
			}
			continue
		} else if panicked {
			c.Errorf("unexpected panic %q with product chain dimentions: %+v result dimension: %+v",
				message, dimensions, test.product)
			continue
		}

		if !EqualApprox(got, want, 1e-14) {
			c.Errorf("unexpected result from product chain dimensions: %+v", dimensions)
		}
	}
}
<commit_msg>mat64: improve product tests<commit_after>// Copyright ©2015 The gonum Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package mat64

import (
	"fmt"
	"math/rand"

	"gopkg.in/check.v1"
)

type dims struct{ r, c int }

var productTests = []struct {
	n       int
	factors []dims
	product dims
	panics  bool
}{
	{
		n:       1,
		factors: []dims{{3, 4}},
		product: dims{3, 4},
		panics:  false,
	},
	{
		n:       1,
		factors: []dims{{2, 4}},
		product: dims{3, 4},
		panics:  true,
	},
	{
		n:       3,
		factors: []dims{{10, 30}, {30, 5}, {5, 60}},
		product: dims{10, 60},
		panics:  false,
	},
	{
		n:       3,
		factors: []dims{{100, 30}, {30, 5}, {5, 60}},
		product: dims{10, 60},
		panics:  true,
	},
	{
		n:       7,
		factors: []dims{{60, 5}, {5, 5}, {5, 4}, {4, 10}, {10, 22}, {22, 45}, {45, 10}},
		product: dims{60, 10},
		panics:  false,
	},
	{
		n:       7,
		factors: []dims{{60, 5}, {5, 5}, {5, 400}, {4, 10}, {10, 22}, {22, 45}, {45, 10}},
		product: dims{60, 10},
		panics:  true,
	},
	{
		n:       3,
		factors: []dims{{1, 1000}, {1000, 2}, {2, 2}},
		product: dims{1, 2},
		panics:  false,
	},

	// Random chains.
	{
		n:       0,
		product: dims{0, 0},
		panics:  false,
	},
	{
		n:       2,
		product: dims{60, 10},
		panics:  false,
	},
	{
		n:       3,
		product: dims{60, 10},
		panics:  false,
	},
	{
		n:       4,
		product: dims{60, 10},
		panics:  false,
	},
	{
		n:       10,
		product: dims{60, 10},
		panics:  false,
	},
}

func (s *S) TestProduct(c *check.C) {
	for _, test := range productTests {
		dimensions := test.factors
		if dimensions == nil && test.n > 0 {
			dimensions = make([]dims, test.n)
			for i := range dimensions {
				if i != 0 {
					dimensions[i].r = dimensions[i-1].c
				}
				dimensions[i].c = rand.Intn(50) + 1
			}
			dimensions[0].r = test.product.r
			dimensions[test.n-1].c = test.product.c
		}
		factors := make([]Matrix, test.n)
		for i, d := range dimensions {
			data := make([]float64, d.r*d.c)
			for i := range data {
				data[i] = rand.Float64()
			}
			factors[i] = NewDense(d.r, d.c, data)
		}

		want := &Dense{}
		if !test.panics {
			a := &Dense{}
			for i, b := range factors {
				if i == 0 {
					want.Clone(b)
					continue
				}
				a, want = want, &Dense{}
				want.Mul(a, b)
			}
		}

		got := NewDense(test.product.r, test.product.c, nil)
		panicked, message := panics(func() {
			got.Product(factors...)
		})
		if test.panics {
			if !panicked {
				c.Errorf("fail to panic with product chain dimensions: %+v result dimension: %+v",
					dimensions, test.product)
			}
			continue
		} else if panicked {
			c.Errorf("unexpected panic %q with product chain dimensions: %+v result dimension: %+v",
				message, dimensions, test.product)
			continue
		}

		if len(factors) > 0 {
			p := newMultiplier(NewDense(test.product.r, test.product.c, nil), factors)
			p.optimize()
			gotCost := p.table.at(0, len(factors)-1).cost
			expr, wantCost, ok := bestExpressionFor(dimensions)
			if !ok {
				c.Fatal("unexpected number of expressions in brute force expression search")
			}
			if gotCost != wantCost {
				c.Errorf("unexpected cost for chain dimensions: %+v got: %d want: %d\n%s",
					dimensions, got, want, expr)
			}
		}

		if !EqualApprox(got, want, 1e-14) {
			c.Errorf("unexpected result from product chain dimensions: %+v", dimensions)
		}
	}
}

// node is a subexpression node.
type node struct {
	dims
	left, right *node
}

func (n *node) String() string {
	if n.left == nil || n.right == nil {
		rows, cols := n.shape()
		return fmt.Sprintf("[%d×%d]", rows, cols)
	}
	rows, cols := n.shape()
	return fmt.Sprintf("(%s * %s):[%d×%d]", n.left, n.right, rows, cols)
}

// shape returns the dimensions of the result of the subexpression.
func (n *node) shape() (rows, cols int) {
	if n.left == nil || n.right == nil {
		return n.r, n.c
	}
	rows, _ = n.left.shape()
	_, cols = n.right.shape()
	return rows, cols
}

// cost returns the cost to evaluate the subexpression.
func (n *node) cost() int {
	if n.left == nil || n.right == nil {
		return 0
	}
	lr, lc := n.left.shape()
	_, rc := n.right.shape()
	return lr*lc*rc + n.left.cost() + n.right.cost()
}

// expressionsFor returns a channel that can be used to iterate over all
// expressions of the given factor dimensions.
func expressionsFor(factors []dims) chan *node {
	if len(factors) == 1 {
		c := make(chan *node, 1)
		c <- &node{dims: factors[0]}
		close(c)
		return c
	}
	c := make(chan *node)
	go func() {
		for i := 1; i < len(factors); i++ {
			for left := range expressionsFor(factors[:i]) {
				for right := range expressionsFor(factors[i:]) {
					c <- &node{left: left, right: right}
				}
			}
		}
		close(c)
	}()
	return c
}

// catalan returns the nth 0-based Catalan number.
func catalan(n int) int {
	p := 1
	for k := n + 1; k < 2*n+1; k++ {
		p *= k
	}
	for k := 2; k < n+2; k++ {
		p /= k
	}
	return p
}

// bestExpressonFor returns the lowest cost expression for the given expression
// factor dimensions, the cost of the expression and whether the number of
// expressions searched matches the Catalan number for the number of factors.
func bestExpressionFor(factors []dims) (exp *node, cost int, ok bool) {
	const maxInt = int(^uint(0) >> 1)
	min := maxInt
	var best *node
	var n int
	for exp := range expressionsFor(factors) {
		n++
		cost := exp.cost()
		if cost < min {
			min = cost
			best = exp
		}
	}
	return best, min, n == catalan(len(factors)-1)
}
<|endoftext|>


<commit_before>package main

import . "github.com/strickyak/basic_basic"
import "github.com/strickyak/basic_basic/draw"

import (
	"flag"
	. "fmt"
	//"io/ioutil"
	"log"
	"net/http"
	//"os"
)

func main() {
	flag.BoolVar(&Debug, "d", false, "debug bit")
  flag.Parse()

	http.HandleFunc("/", Render)
	err := http.ListenAndServe(":12345", nil)
	if err != nil {
		log.Fatal("ListenAndServe: ", err)
	}
}

func Render(w http.ResponseWriter, req *http.Request) {
  defer func() {
    r := recover()
    if r != nil {
      w.Header().Set("Content-Type", "text/plain")
      Fprintf(w, "%v", r)
    }
  }()

  var code string
	req.ParseForm()
  if x, ok := req.Form["code"]; ok {
    code = x[0]

    terp := NewTerp(code)
    d := draw.Register(terp)
    terp.Run()
    Printf("\n")
    w.Header().Set("Content-Type", "image/png")
    if d.HasImage() {
      d.WritePng(w)
    }
  }
}
<commit_msg>gofmt<commit_after>package main

import . "github.com/strickyak/basic_basic"
import "github.com/strickyak/basic_basic/draw"

import (
	"flag"
	. "fmt"
	//"io/ioutil"
	"log"
	"net/http"
	//"os"
)

func main() {
	flag.BoolVar(&Debug, "d", false, "debug bit")
	flag.Parse()

	http.HandleFunc("/", Render)
	err := http.ListenAndServe(":12345", nil)
	if err != nil {
		log.Fatal("ListenAndServe: ", err)
	}
}

func Render(w http.ResponseWriter, req *http.Request) {
	defer func() {
		r := recover()
		if r != nil {
			w.Header().Set("Content-Type", "text/plain")
			Fprintf(w, "%v", r)
		}
	}()

	var code string
	req.ParseForm()
	if x, ok := req.Form["code"]; ok {
		code = x[0]

		terp := NewTerp(code)
		d := draw.Register(terp)
		terp.Run()
		Printf("\n")
		w.Header().Set("Content-Type", "image/png")
		if d.HasImage() {
			d.WritePng(w)
		}
	}
}
<|endoftext|>


<commit_before>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package rpc

import (
	"http";
	"log";
	"net";
	"once";
	"os";
	"strings";
	"testing";
)

var serverAddr string
var httpServerAddr string

const second = 1e9

type Args struct {
	A, B int;
}

type Reply struct {
	C int;
}

type Arith int

func (t *Arith) Add(args *Args, reply *Reply) os.Error {
	reply.C = args.A + args.B;
	return nil;
}

func (t *Arith) Mul(args *Args, reply *Reply) os.Error {
	reply.C = args.A * args.B;
	return nil;
}

func (t *Arith) Div(args *Args, reply *Reply) os.Error {
	if args.B == 0 {
		return os.ErrorString("divide by zero")
	}
	reply.C = args.A / args.B;
	return nil;
}

func (t *Arith) Error(args *Args, reply *Reply) os.Error {
	panicln("ERROR")
}

func startServer() {
	Register(new(Arith));

	l, e := net.Listen("tcp", ":0");	// any available address
	if e != nil {
		log.Exitf("net.Listen tcp :0: %v", e)
	}
	serverAddr = l.Addr().String();
	log.Stderr("Test RPC server listening on ", serverAddr);
	go Accept(l);

	HandleHTTP();
	l, e = net.Listen("tcp", ":0");	// any available address
	if e != nil {
		log.Stderrf("net.Listen tcp :0: %v", e);
		os.Exit(1);
	}
	httpServerAddr = l.Addr().String();
	log.Stderr("Test HTTP RPC server listening on ", httpServerAddr);
	go http.Serve(l, nil);
}

func TestRPC(t *testing.T) {
	once.Do(startServer);

	client, err := Dial("tcp", serverAddr);
	if err != nil {
		t.Fatal("dialing", err)
	}

	// Synchronous calls
	args := &Args{7, 8};
	reply := new(Reply);
	err = client.Call("Arith.Add", args, reply);
	if reply.C != args.A+args.B {
		t.Errorf("Add: expected %d got %d", reply.C, args.A+args.B)
	}

	args = &Args{7, 8};
	reply = new(Reply);
	err = client.Call("Arith.Mul", args, reply);
	if reply.C != args.A*args.B {
		t.Errorf("Mul: expected %d got %d", reply.C, args.A*args.B)
	}

	// Out of order.
	args = &Args{7, 8};
	mulReply := new(Reply);
	mulCall := client.Go("Arith.Mul", args, mulReply, nil);
	addReply := new(Reply);
	addCall := client.Go("Arith.Add", args, addReply, nil);

	<-addCall.Done;
	if addReply.C != args.A+args.B {
		t.Errorf("Add: expected %d got %d", addReply.C, args.A+args.B)
	}

	<-mulCall.Done;
	if mulReply.C != args.A*args.B {
		t.Errorf("Mul: expected %d got %d", mulReply.C, args.A*args.B)
	}

	// Error test
	args = &Args{7, 0};
	reply = new(Reply);
	err = client.Call("Arith.Div", args, reply);
	// expect an error: zero divide
	if err == nil {
		t.Error("Div: expected error")
	} else if err.String() != "divide by zero" {
		t.Error("Div: expected divide by zero error; got", err)
	}
}

func TestHTTPRPC(t *testing.T) {
	once.Do(startServer);

	client, err := DialHTTP("tcp", httpServerAddr);
	if err != nil {
		t.Fatal("dialing", err)
	}

	// Synchronous calls
	args := &Args{7, 8};
	reply := new(Reply);
	err = client.Call("Arith.Add", args, reply);
	if reply.C != args.A+args.B {
		t.Errorf("Add: expected %d got %d", reply.C, args.A+args.B)
	}
}

func TestCheckUnknownService(t *testing.T) {
	once.Do(startServer);

	conn, err := net.Dial("tcp", "", serverAddr);
	if err != nil {
		t.Fatal("dialing:", err)
	}

	client := NewClient(conn);

	args := &Args{7, 8};
	reply := new(Reply);
	err = client.Call("Unknown.Add", args, reply);
	if err == nil {
		t.Error("expected error calling unknown service")
	} else if strings.Index(err.String(), "service") < 0 {
		t.Error("expected error about service; got", err)
	}
}

func TestCheckUnknownMethod(t *testing.T) {
	once.Do(startServer);

	conn, err := net.Dial("tcp", "", serverAddr);
	if err != nil {
		t.Fatal("dialing:", err)
	}

	client := NewClient(conn);

	args := &Args{7, 8};
	reply := new(Reply);
	err = client.Call("Arith.Unknown", args, reply);
	if err == nil {
		t.Error("expected error calling unknown service")
	} else if strings.Index(err.String(), "method") < 0 {
		t.Error("expected error about method; got", err)
	}
}

func TestCheckBadType(t *testing.T) {
	once.Do(startServer);

	conn, err := net.Dial("tcp", "", serverAddr);
	if err != nil {
		t.Fatal("dialing:", err)
	}

	client := NewClient(conn);

	reply := new(Reply);
	err = client.Call("Arith.Add", reply, reply);	// args, reply would be the correct thing to use
	if err == nil {
		t.Error("expected error calling Arith.Add with wrong arg type")
	} else if strings.Index(err.String(), "type") < 0 {
		t.Error("expected error about type; got", err)
	}
}
<commit_msg>Make non-errored RPC calls return 'nil' error to caller.<commit_after>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package rpc

import (
	"http";
	"log";
	"net";
	"once";
	"os";
	"strings";
	"testing";
)

var serverAddr string
var httpServerAddr string

const second = 1e9

type Args struct {
	A, B int;
}

type Reply struct {
	C int;
}

type Arith int

func (t *Arith) Add(args *Args, reply *Reply) os.Error {
	reply.C = args.A + args.B;
	return nil;
}

func (t *Arith) Mul(args *Args, reply *Reply) os.Error {
	reply.C = args.A * args.B;
	return nil;
}

func (t *Arith) Div(args *Args, reply *Reply) os.Error {
	if args.B == 0 {
		return os.ErrorString("divide by zero")
	}
	reply.C = args.A / args.B;
	return nil;
}

func (t *Arith) Error(args *Args, reply *Reply) os.Error {
	panicln("ERROR")
}

func startServer() {
	Register(new(Arith));

	l, e := net.Listen("tcp", ":0");	// any available address
	if e != nil {
		log.Exitf("net.Listen tcp :0: %v", e)
	}
	serverAddr = l.Addr().String();
	log.Stderr("Test RPC server listening on ", serverAddr);
	go Accept(l);

	HandleHTTP();
	l, e = net.Listen("tcp", ":0");	// any available address
	if e != nil {
		log.Stderrf("net.Listen tcp :0: %v", e);
		os.Exit(1);
	}
	httpServerAddr = l.Addr().String();
	log.Stderr("Test HTTP RPC server listening on ", httpServerAddr);
	go http.Serve(l, nil);
}

func TestRPC(t *testing.T) {
	once.Do(startServer);

	client, err := Dial("tcp", serverAddr);
	if err != nil {
		t.Fatal("dialing", err)
	}

	// Synchronous calls
	args := &Args{7, 8};
	reply := new(Reply);
	err = client.Call("Arith.Add", args, reply);
	if err != nil {
		t.Errorf("Add: expected no error but got string %q", err.String())
	}
	if reply.C != args.A+args.B {
		t.Errorf("Add: expected %d got %d", reply.C, args.A+args.B)
	}

	args = &Args{7, 8};
	reply = new(Reply);
	err = client.Call("Arith.Mul", args, reply);
	if err != nil {
		t.Errorf("Mul: expected no error but got string %q", err.String())
	}
	if reply.C != args.A*args.B {
		t.Errorf("Mul: expected %d got %d", reply.C, args.A*args.B)
	}

	// Out of order.
	args = &Args{7, 8};
	mulReply := new(Reply);
	mulCall := client.Go("Arith.Mul", args, mulReply, nil);
	addReply := new(Reply);
	addCall := client.Go("Arith.Add", args, addReply, nil);

	addCall = <-addCall.Done;
	if addCall.Error != nil {
		t.Errorf("Add: expected no error but got string %q", addCall.Error.String())
	}
	if addReply.C != args.A+args.B {
		t.Errorf("Add: expected %d got %d", addReply.C, args.A+args.B)
	}

	mulCall = <-mulCall.Done;
	if mulCall.Error != nil {
		t.Errorf("Mul: expected no error but got string %q", mulCall.Error.String())
	}
	if mulReply.C != args.A*args.B {
		t.Errorf("Mul: expected %d got %d", mulReply.C, args.A*args.B)
	}

	// Error test
	args = &Args{7, 0};
	reply = new(Reply);
	err = client.Call("Arith.Div", args, reply);
	// expect an error: zero divide
	if err == nil {
		t.Error("Div: expected error")
	} else if err.String() != "divide by zero" {
		t.Error("Div: expected divide by zero error; got", err)
	}
}

func TestHTTPRPC(t *testing.T) {
	once.Do(startServer);

	client, err := DialHTTP("tcp", httpServerAddr);
	if err != nil {
		t.Fatal("dialing", err)
	}

	// Synchronous calls
	args := &Args{7, 8};
	reply := new(Reply);
	err = client.Call("Arith.Add", args, reply);
	if err != nil {
		t.Errorf("Add: expected no error but got string %q", err.String())
	}
	if reply.C != args.A+args.B {
		t.Errorf("Add: expected %d got %d", reply.C, args.A+args.B)
	}
}

func TestCheckUnknownService(t *testing.T) {
	once.Do(startServer);

	conn, err := net.Dial("tcp", "", serverAddr);
	if err != nil {
		t.Fatal("dialing:", err)
	}

	client := NewClient(conn);

	args := &Args{7, 8};
	reply := new(Reply);
	err = client.Call("Unknown.Add", args, reply);
	if err == nil {
		t.Error("expected error calling unknown service")
	} else if strings.Index(err.String(), "service") < 0 {
		t.Error("expected error about service; got", err)
	}
}

func TestCheckUnknownMethod(t *testing.T) {
	once.Do(startServer);

	conn, err := net.Dial("tcp", "", serverAddr);
	if err != nil {
		t.Fatal("dialing:", err)
	}

	client := NewClient(conn);

	args := &Args{7, 8};
	reply := new(Reply);
	err = client.Call("Arith.Unknown", args, reply);
	if err == nil {
		t.Error("expected error calling unknown service")
	} else if strings.Index(err.String(), "method") < 0 {
		t.Error("expected error about method; got", err)
	}
}

func TestCheckBadType(t *testing.T) {
	once.Do(startServer);

	conn, err := net.Dial("tcp", "", serverAddr);
	if err != nil {
		t.Fatal("dialing:", err)
	}

	client := NewClient(conn);

	reply := new(Reply);
	err = client.Call("Arith.Add", reply, reply);	// args, reply would be the correct thing to use
	if err == nil {
		t.Error("expected error calling Arith.Add with wrong arg type")
	} else if strings.Index(err.String(), "type") < 0 {
		t.Error("expected error about type; got", err)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"github.com/docker/docker/daemon/graphdriver/devmapper"
	"os"
	"path"
	"sort"
	"strconv"
	"strings"
)

func usage() {
	fmt.Fprintf(os.Stderr, "Usage: %s <flags>  [status] | [list] | [device id]  | [resize new-pool-size] | [snap new-id base-id] | [remove id] | [mount id mountpoint]\n", os.Args[0])
	flag.PrintDefaults()
	os.Exit(1)
}

func byteSizeFromString(arg string) (int64, error) {
	digits := ""
	rest := ""
	last := strings.LastIndexAny(arg, "0123456789")
	if last >= 0 {
		digits = arg[:last+1]
		rest = arg[last+1:]
	}

	val, err := strconv.ParseInt(digits, 10, 64)
	if err != nil {
		return val, err
	}

	rest = strings.ToLower(strings.TrimSpace(rest))

	var multiplier int64 = 1
	switch rest {
	case "":
		multiplier = 1
	case "k", "kb":
		multiplier = 1024
	case "m", "mb":
		multiplier = 1024 * 1024
	case "g", "gb":
		multiplier = 1024 * 1024 * 1024
	case "t", "tb":
		multiplier = 1024 * 1024 * 1024 * 1024
	default:
		return 0, fmt.Errorf("Unknown size unit: %s", rest)
	}

	return val * multiplier, nil
}

func main() {
	root := flag.String("r", "/var/lib/docker", "Docker root dir")
	flDebug := flag.Bool("D", false, "Debug mode")

	flag.Parse()

	if *flDebug {
		os.Setenv("DEBUG", "1")
	}

	if flag.NArg() < 1 {
		usage()
	}

	args := flag.Args()

	home := path.Join(*root, "devicemapper")
	devices, err := devmapper.NewDeviceSet(home, false)
	if err != nil {
		fmt.Println("Can't initialize device mapper: ", err)
		os.Exit(1)
	}

	switch args[0] {
	case "status":
		status := devices.Status()
		fmt.Printf("Pool name: %s\n", status.PoolName)
		fmt.Printf("Data Loopback file: %s\n", status.DataLoopback)
		fmt.Printf("Metadata Loopback file: %s\n", status.MetadataLoopback)
		fmt.Printf("Sector size: %d\n", status.SectorSize)
		fmt.Printf("Data use: %d of %d (%.1f %%)\n", status.Data.Used, status.Data.Total, 100.0*float64(status.Data.Used)/float64(status.Data.Total))
		fmt.Printf("Metadata use: %d of %d (%.1f %%)\n", status.Metadata.Used, status.Metadata.Total, 100.0*float64(status.Metadata.Used)/float64(status.Metadata.Total))
		break
	case "list":
		ids := devices.List()
		sort.Strings(ids)
		for _, id := range ids {
			fmt.Println(id)
		}
		break
	case "device":
		if flag.NArg() < 2 {
			usage()
		}
		status, err := devices.GetDeviceStatus(args[1])
		if err != nil {
			fmt.Println("Can't get device info: ", err)
			os.Exit(1)
		}
		fmt.Printf("Id: %d\n", status.DeviceId)
		fmt.Printf("Size: %d\n", status.Size)
		fmt.Printf("Transaction Id: %d\n", status.TransactionId)
		fmt.Printf("Size in Sectors: %d\n", status.SizeInSectors)
		fmt.Printf("Mapped Sectors: %d\n", status.MappedSectors)
		fmt.Printf("Highest Mapped Sector: %d\n", status.HighestMappedSector)
		break
	case "resize":
		if flag.NArg() < 2 {
			usage()
		}

		size, err := byteSizeFromString(args[1])
		if err != nil {
			fmt.Println("Invalid size: ", err)
			os.Exit(1)
		}

		err = devices.ResizePool(size)
		if err != nil {
			fmt.Println("Error resizeing pool: ", err)
			os.Exit(1)
		}

		break
	case "snap":
		if flag.NArg() < 3 {
			usage()
		}

		err := devices.AddDevice(args[1], args[2])
		if err != nil {
			fmt.Println("Can't create snap device: ", err)
			os.Exit(1)
		}
		break
	case "remove":
		if flag.NArg() < 2 {
			usage()
		}

		err := devices.RemoveDevice(args[1])
		if err != nil {
			fmt.Println("Can't remove device: ", err)
			os.Exit(1)
		}
		break
	case "mount":
		if flag.NArg() < 3 {
			usage()
		}

		err := devices.MountDevice(args[1], args[2], false)
		if err != nil {
			fmt.Println("Can't create snap device: ", err)
			os.Exit(1)
		}
		break
	default:
		fmt.Printf("Unknown command %s\n", args[0])
		usage()

		os.Exit(1)
	}

	return
}
<commit_msg>fix device-tool compile error<commit_after>package main

import (
	"flag"
	"fmt"
	"github.com/docker/docker/daemon/graphdriver/devmapper"
	"os"
	"path"
	"sort"
	"strconv"
	"strings"
)

func usage() {
	fmt.Fprintf(os.Stderr, "Usage: %s <flags>  [status] | [list] | [device id]  | [resize new-pool-size] | [snap new-id base-id] | [remove id] | [mount id mountpoint]\n", os.Args[0])
	flag.PrintDefaults()
	os.Exit(1)
}

func byteSizeFromString(arg string) (int64, error) {
	digits := ""
	rest := ""
	last := strings.LastIndexAny(arg, "0123456789")
	if last >= 0 {
		digits = arg[:last+1]
		rest = arg[last+1:]
	}

	val, err := strconv.ParseInt(digits, 10, 64)
	if err != nil {
		return val, err
	}

	rest = strings.ToLower(strings.TrimSpace(rest))

	var multiplier int64 = 1
	switch rest {
	case "":
		multiplier = 1
	case "k", "kb":
		multiplier = 1024
	case "m", "mb":
		multiplier = 1024 * 1024
	case "g", "gb":
		multiplier = 1024 * 1024 * 1024
	case "t", "tb":
		multiplier = 1024 * 1024 * 1024 * 1024
	default:
		return 0, fmt.Errorf("Unknown size unit: %s", rest)
	}

	return val * multiplier, nil
}

func main() {
	root := flag.String("r", "/var/lib/docker", "Docker root dir")
	flDebug := flag.Bool("D", false, "Debug mode")

	flag.Parse()

	if *flDebug {
		os.Setenv("DEBUG", "1")
	}

	if flag.NArg() < 1 {
		usage()
	}

	args := flag.Args()

	home := path.Join(*root, "devicemapper")
	devices, err := devmapper.NewDeviceSet(home, false, nil)
	if err != nil {
		fmt.Println("Can't initialize device mapper: ", err)
		os.Exit(1)
	}

	switch args[0] {
	case "status":
		status := devices.Status()
		fmt.Printf("Pool name: %s\n", status.PoolName)
		fmt.Printf("Data Loopback file: %s\n", status.DataLoopback)
		fmt.Printf("Metadata Loopback file: %s\n", status.MetadataLoopback)
		fmt.Printf("Sector size: %d\n", status.SectorSize)
		fmt.Printf("Data use: %d of %d (%.1f %%)\n", status.Data.Used, status.Data.Total, 100.0*float64(status.Data.Used)/float64(status.Data.Total))
		fmt.Printf("Metadata use: %d of %d (%.1f %%)\n", status.Metadata.Used, status.Metadata.Total, 100.0*float64(status.Metadata.Used)/float64(status.Metadata.Total))
		break
	case "list":
		ids := devices.List()
		sort.Strings(ids)
		for _, id := range ids {
			fmt.Println(id)
		}
		break
	case "device":
		if flag.NArg() < 2 {
			usage()
		}
		status, err := devices.GetDeviceStatus(args[1])
		if err != nil {
			fmt.Println("Can't get device info: ", err)
			os.Exit(1)
		}
		fmt.Printf("Id: %d\n", status.DeviceId)
		fmt.Printf("Size: %d\n", status.Size)
		fmt.Printf("Transaction Id: %d\n", status.TransactionId)
		fmt.Printf("Size in Sectors: %d\n", status.SizeInSectors)
		fmt.Printf("Mapped Sectors: %d\n", status.MappedSectors)
		fmt.Printf("Highest Mapped Sector: %d\n", status.HighestMappedSector)
		break
	case "resize":
		if flag.NArg() < 2 {
			usage()
		}

		size, err := byteSizeFromString(args[1])
		if err != nil {
			fmt.Println("Invalid size: ", err)
			os.Exit(1)
		}

		err = devices.ResizePool(size)
		if err != nil {
			fmt.Println("Error resizeing pool: ", err)
			os.Exit(1)
		}

		break
	case "snap":
		if flag.NArg() < 3 {
			usage()
		}

		err := devices.AddDevice(args[1], args[2])
		if err != nil {
			fmt.Println("Can't create snap device: ", err)
			os.Exit(1)
		}
		break
	case "remove":
		if flag.NArg() < 2 {
			usage()
		}

		err := devmapper.RemoveDevice(args[1])
		if err != nil {
			fmt.Println("Can't remove device: ", err)
			os.Exit(1)
		}
		break
	case "mount":
		if flag.NArg() < 3 {
			usage()
		}

		err := devices.MountDevice(args[1], args[2], "")
		if err != nil {
			fmt.Println("Can't create snap device: ", err)
			os.Exit(1)
		}
		break
	default:
		fmt.Printf("Unknown command %s\n", args[0])
		usage()

		os.Exit(1)
	}

	return
}
<|endoftext|>


<commit_before>// Package kubernetes provides abstractions for the Kubernetes platform. At the
// moment, Kubernetes is the only supported platform, so we are directly
// returning Kubernetes objects. As we add more platforms, we will create
// abstractions and common data types in package platform.
package kubernetes

import (
	"bytes"
	"fmt"

	k8syaml "github.com/ghodss/yaml"
	"github.com/go-kit/kit/log"

	"github.com/pkg/errors"
	"gopkg.in/yaml.v2"
	meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/client-go/discovery"
	k8sclient "k8s.io/client-go/kubernetes"
	v1beta1apps "k8s.io/client-go/kubernetes/typed/apps/v1beta1"
	v2alpha1batch "k8s.io/client-go/kubernetes/typed/batch/v2alpha1"
	v1core "k8s.io/client-go/kubernetes/typed/core/v1"
	v1beta1extensions "k8s.io/client-go/kubernetes/typed/extensions/v1beta1"
	"k8s.io/client-go/pkg/api"
	apiv1 "k8s.io/client-go/pkg/api/v1"

	"github.com/weaveworks/flux"
	"github.com/weaveworks/flux/cluster"
	"github.com/weaveworks/flux/registry"
	"github.com/weaveworks/flux/ssh"
)

const (
	StatusUnknown  = "unknown"
	StatusReady    = "ready"
	StatusUpdating = "updating"
)

type extendedClient struct {
	discovery.DiscoveryInterface
	v1core.CoreV1Interface
	v1beta1extensions.ExtensionsV1beta1Interface
	v1beta1apps.StatefulSetsGetter
	v2alpha1batch.CronJobsGetter
}

type apiObject struct {
	bytes    []byte
	Version  string `yaml:"apiVersion"`
	Kind     string `yaml:"kind"`
	Metadata struct {
		Name      string `yaml:"name"`
		Namespace string `yaml:"namespace"`
	} `yaml:"metadata"`
}

func (obj *apiObject) namespaceOrDefault() string {
	if obj.Metadata.Namespace == "" {
		return "default"
	}
	return obj.Metadata.Namespace
}

// --- add-ons

// Kubernetes has a mechanism of "Add-ons", whereby manifest files
// left in a particular directory on the Kubernetes master will be
// applied. We can recognise these, because they:
//  1. Must be in the namespace `kube-system`; and,
//  2. Must have one of the labels below set, else the addon manager will ignore them.
//
// We want to ignore add-ons, since they are managed by the add-on
// manager, and attempts to control them via other means will fail.

type namespacedLabeled interface {
	GetNamespace() string
	GetLabels() map[string]string
}

func isAddon(obj namespacedLabeled) bool {
	if obj.GetNamespace() != "kube-system" {
		return false
	}
	labels := obj.GetLabels()
	if labels["kubernetes.io/cluster-service"] == "true" ||
		labels["addonmanager.kubernetes.io/mode"] == "EnsureExists" ||
		labels["addonmanager.kubernetes.io/mode"] == "Reconcile" {
		return true
	}
	return false
}

// --- /add ons

type Applier interface {
	Delete(logger log.Logger, def *apiObject) error
	Apply(logger log.Logger, def *apiObject) error
}

// Cluster is a handle to a Kubernetes API server.
// (Typically, this code is deployed into the same cluster.)
type Cluster struct {
	client     extendedClient
	applier    Applier
	actionc    chan func()
	version    string // string response for the version command.
	logger     log.Logger
	sshKeyRing ssh.KeyRing
}

// NewCluster returns a usable cluster. Host should be of the form
// "http://hostname:8080".
func NewCluster(clientset k8sclient.Interface,
	applier Applier,
	sshKeyRing ssh.KeyRing,
	logger log.Logger) (*Cluster, error) {

	c := &Cluster{
		client: extendedClient{
			clientset.Discovery(),
			clientset.Core(),
			clientset.Extensions(),
			clientset.AppsV1beta1(),
			clientset.BatchV2alpha1()},
		applier:    applier,
		actionc:    make(chan func()),
		logger:     logger,
		sshKeyRing: sshKeyRing,
	}

	go c.loop()
	return c, nil
}

// Stop terminates the goroutine that serializes and executes requests against
// the cluster. A stopped cluster cannot be restarted.
func (c *Cluster) Stop() {
	close(c.actionc)
}

func (c *Cluster) loop() {
	for f := range c.actionc {
		f()
	}
}

// --- cluster.Cluster

// SomeControllers returns the controllers named, missing out any that don't
// exist in the cluster. They do not necessarily have to be returned
// in the order requested.
func (c *Cluster) SomeControllers(ids []flux.ResourceID) (res []cluster.Controller, err error) {
	var controllers []cluster.Controller
	for _, id := range ids {
		ns, kind, name := id.Components()

		resourceKind, ok := resourceKinds[kind]
		if !ok {
			return nil, fmt.Errorf("Unsupported kind %v", kind)
		}

		podController, err := resourceKind.getPodController(c, ns, name)
		if err != nil {
			return nil, err
		}

		if !isAddon(podController) {
			controllers = append(controllers, podController.toClusterController(id))
		}
	}
	return controllers, nil
}

// AllControllers returns all controllers matching the criteria; that is, in
// the namespace (or any namespace if that argument is empty)
func (c *Cluster) AllControllers(namespace string) (res []cluster.Controller, err error) {
	namespaces, err := c.client.Namespaces().List(meta_v1.ListOptions{})
	if err != nil {
		return nil, errors.Wrap(err, "getting namespaces")
	}

	var allControllers []cluster.Controller
	for _, ns := range namespaces.Items {
		if namespace != "" && ns.Name != namespace {
			continue
		}

		for kind, resourceKind := range resourceKinds {
			podControllers, err := resourceKind.getPodControllers(c, ns.Name)
			if err != nil {
				return nil, err
			}

			for _, podController := range podControllers {
				if !isAddon(podController) {
					id := flux.MakeResourceID(ns.Name, kind, podController.name)
					allControllers = append(allControllers, podController.toClusterController(id))
				}
			}
		}
	}

	return allControllers, nil
}

// Sync performs the given actions on resources. Operations are
// asynchronous, but serialised.
func (c *Cluster) Sync(spec cluster.SyncDef) error {
	errc := make(chan error)
	logger := log.NewContext(c.logger).With("method", "Sync")
	c.actionc <- func() {
		errs := cluster.SyncError{}
		for _, action := range spec.Actions {
			logger := log.NewContext(logger).With("resource", action.ResourceID)
			if len(action.Delete) > 0 {
				obj, err := definitionObj(action.Delete)
				if err == nil {
					err = c.applier.Delete(logger, obj)
				}
				if err != nil {
					errs[action.ResourceID] = err
					continue
				}
			}
			if len(action.Apply) > 0 {
				obj, err := definitionObj(action.Apply)
				if err == nil {
					err = c.applier.Apply(logger, obj)
				}
				if err != nil {
					errs[action.ResourceID] = err
					continue
				}
			}
		}
		if len(errs) > 0 {
			errc <- errs
		} else {
			errc <- nil
		}
	}
	return <-errc
}

func (c *Cluster) Ping() error {
	_, err := c.client.ServerVersion()
	return err
}

func (c *Cluster) Export() ([]byte, error) {
	var config bytes.Buffer
	list, err := c.client.Namespaces().List(meta_v1.ListOptions{})
	if err != nil {
		return nil, errors.Wrap(err, "getting namespaces")
	}
	for _, ns := range list.Items {
		err := appendYAML(&config, "v1", "Namespace", ns)
		if err != nil {
			return nil, errors.Wrap(err, "marshalling namespace to YAML")
		}

		for _, resourceKind := range resourceKinds {
			podControllers, err := resourceKind.getPodControllers(c, ns.Name)
			if err != nil {
				return nil, err
			}

			for _, pc := range podControllers {
				if !isAddon(pc) {
					if err := appendYAML(&config, pc.apiVersion, pc.kind, pc.apiObject); err != nil {
						return nil, err
					}
				}
			}
		}
	}
	return config.Bytes(), nil
}

// kind & apiVersion must be passed separately as the object's TypeMeta is not populated
func appendYAML(buffer *bytes.Buffer, apiVersion, kind string, object interface{}) error {
	yamlBytes, err := k8syaml.Marshal(object)
	if err != nil {
		return err
	}
	buffer.WriteString("---\n")
	buffer.WriteString("apiVersion: ")
	buffer.WriteString(apiVersion)
	buffer.WriteString("\nkind: ")
	buffer.WriteString(kind)
	buffer.WriteString("\n")
	buffer.Write(yamlBytes)
	return nil
}

func (c *Cluster) PublicSSHKey(regenerate bool) (ssh.PublicKey, error) {
	if regenerate {
		if err := c.sshKeyRing.Regenerate(); err != nil {
			return ssh.PublicKey{}, err
		}
	}
	publicKey, _ := c.sshKeyRing.KeyPair()
	return publicKey, nil
}

func mergeCredentials(c *Cluster, namespace string, podTemplate apiv1.PodTemplateSpec, imageCreds registry.ImageCreds) {
	creds := registry.NoCredentials()
	for _, imagePullSecret := range podTemplate.Spec.ImagePullSecrets {
		secret, err := c.client.Secrets(namespace).Get(imagePullSecret.Name, meta_v1.GetOptions{})
		if err != nil {
			c.logger.Log("err", errors.Wrapf(err, "getting secret %q from namespace %q", secret.Name, namespace))
			continue
		}

		var decoded []byte
		var ok bool
		// These differ in format; but, ParseCredentials will
		// handle either.
		switch api.SecretType(secret.Type) {
		case api.SecretTypeDockercfg:
			decoded, ok = secret.Data[api.DockerConfigKey]
		case api.SecretTypeDockerConfigJson:
			decoded, ok = secret.Data[api.DockerConfigJsonKey]
		default:
			c.logger.Log("skip", "unknown type", "secret", namespace+"/"+secret.Name, "type", secret.Type)
			continue
		}

		if !ok {
			c.logger.Log("err", errors.Wrapf(err, "retrieving pod secret %q", secret.Name))
			continue
		}

		// Parse secret
		crd, err := registry.ParseCredentials(decoded)
		if err != nil {
			c.logger.Log("err", err.Error())
			continue
		}

		// Merge into the credentials for this PodSpec
		creds.Merge(crd)
	}

	// Now create the service and attach the credentials
	for _, container := range podTemplate.Spec.Containers {
		r, err := flux.ParseImageID(container.Image)
		if err != nil {
			c.logger.Log("err", err.Error())
			continue
		}
		imageCreds[r] = creds
	}
}

// ImagesToFetch is a k8s specific method to get a list of images to update along with their credentials
func (c *Cluster) ImagesToFetch() registry.ImageCreds {
	allImageCreds := make(registry.ImageCreds)

	namespaces, err := c.client.Namespaces().List(meta_v1.ListOptions{})
	if err != nil {
		c.logger.Log("err", errors.Wrap(err, "getting namespaces"))
		return allImageCreds
	}

	for _, ns := range namespaces.Items {
		for _, resourceKind := range resourceKinds {
			podControllers, err := resourceKind.getPodControllers(c, ns.Name)
			if err != nil {
				c.logger.Log("err", errors.Wrapf(err, "getting kind %s for namespace %s", resourceKind, ns.Name))
				continue
			}

			imageCreds := make(registry.ImageCreds)
			for _, podController := range podControllers {
				mergeCredentials(c, ns.Name, podController.podTemplate, imageCreds)
			}

			// Merge creds
			for imageID, creds := range imageCreds {
				existingCreds, ok := allImageCreds[imageID]
				if ok {
					existingCreds.Merge(creds)
				} else {
					allImageCreds[imageID] = creds
				}
			}
		}
	}

	return allImageCreds
}

// --- end cluster.Cluster

// A convenience for getting an minimal object from some bytes.
func definitionObj(bytes []byte) (*apiObject, error) {
	obj := apiObject{bytes: bytes}
	return &obj, yaml.Unmarshal(bytes, &obj)
}
<commit_msg>Skip unsupported kinds<commit_after>// Package kubernetes provides abstractions for the Kubernetes platform. At the
// moment, Kubernetes is the only supported platform, so we are directly
// returning Kubernetes objects. As we add more platforms, we will create
// abstractions and common data types in package platform.
package kubernetes

import (
	"bytes"
	"fmt"

	k8syaml "github.com/ghodss/yaml"
	"github.com/go-kit/kit/log"
	"github.com/pkg/errors"
	"gopkg.in/yaml.v2"
	apierrors "k8s.io/apimachinery/pkg/api/errors"
	meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/client-go/discovery"
	k8sclient "k8s.io/client-go/kubernetes"
	v1beta1apps "k8s.io/client-go/kubernetes/typed/apps/v1beta1"
	v2alpha1batch "k8s.io/client-go/kubernetes/typed/batch/v2alpha1"
	v1core "k8s.io/client-go/kubernetes/typed/core/v1"
	v1beta1extensions "k8s.io/client-go/kubernetes/typed/extensions/v1beta1"
	"k8s.io/client-go/pkg/api"
	apiv1 "k8s.io/client-go/pkg/api/v1"

	"github.com/weaveworks/flux"
	"github.com/weaveworks/flux/cluster"
	"github.com/weaveworks/flux/registry"
	"github.com/weaveworks/flux/ssh"
)

const (
	StatusUnknown  = "unknown"
	StatusReady    = "ready"
	StatusUpdating = "updating"
)

type extendedClient struct {
	discovery.DiscoveryInterface
	v1core.CoreV1Interface
	v1beta1extensions.ExtensionsV1beta1Interface
	v1beta1apps.StatefulSetsGetter
	v2alpha1batch.CronJobsGetter
}

type apiObject struct {
	bytes    []byte
	Version  string `yaml:"apiVersion"`
	Kind     string `yaml:"kind"`
	Metadata struct {
		Name      string `yaml:"name"`
		Namespace string `yaml:"namespace"`
	} `yaml:"metadata"`
}

func (obj *apiObject) namespaceOrDefault() string {
	if obj.Metadata.Namespace == "" {
		return "default"
	}
	return obj.Metadata.Namespace
}

// --- add-ons

// Kubernetes has a mechanism of "Add-ons", whereby manifest files
// left in a particular directory on the Kubernetes master will be
// applied. We can recognise these, because they:
//  1. Must be in the namespace `kube-system`; and,
//  2. Must have one of the labels below set, else the addon manager will ignore them.
//
// We want to ignore add-ons, since they are managed by the add-on
// manager, and attempts to control them via other means will fail.

type namespacedLabeled interface {
	GetNamespace() string
	GetLabels() map[string]string
}

func isAddon(obj namespacedLabeled) bool {
	if obj.GetNamespace() != "kube-system" {
		return false
	}
	labels := obj.GetLabels()
	if labels["kubernetes.io/cluster-service"] == "true" ||
		labels["addonmanager.kubernetes.io/mode"] == "EnsureExists" ||
		labels["addonmanager.kubernetes.io/mode"] == "Reconcile" {
		return true
	}
	return false
}

// --- /add ons

type Applier interface {
	Delete(logger log.Logger, def *apiObject) error
	Apply(logger log.Logger, def *apiObject) error
}

// Cluster is a handle to a Kubernetes API server.
// (Typically, this code is deployed into the same cluster.)
type Cluster struct {
	client     extendedClient
	applier    Applier
	actionc    chan func()
	version    string // string response for the version command.
	logger     log.Logger
	sshKeyRing ssh.KeyRing
}

// NewCluster returns a usable cluster. Host should be of the form
// "http://hostname:8080".
func NewCluster(clientset k8sclient.Interface,
	applier Applier,
	sshKeyRing ssh.KeyRing,
	logger log.Logger) (*Cluster, error) {

	c := &Cluster{
		client: extendedClient{
			clientset.Discovery(),
			clientset.Core(),
			clientset.Extensions(),
			clientset.AppsV1beta1(),
			clientset.BatchV2alpha1()},
		applier:    applier,
		actionc:    make(chan func()),
		logger:     logger,
		sshKeyRing: sshKeyRing,
	}

	go c.loop()
	return c, nil
}

// Stop terminates the goroutine that serializes and executes requests against
// the cluster. A stopped cluster cannot be restarted.
func (c *Cluster) Stop() {
	close(c.actionc)
}

func (c *Cluster) loop() {
	for f := range c.actionc {
		f()
	}
}

// --- cluster.Cluster

// SomeControllers returns the controllers named, missing out any that don't
// exist in the cluster. They do not necessarily have to be returned
// in the order requested.
func (c *Cluster) SomeControllers(ids []flux.ResourceID) (res []cluster.Controller, err error) {
	var controllers []cluster.Controller
	for _, id := range ids {
		ns, kind, name := id.Components()

		resourceKind, ok := resourceKinds[kind]
		if !ok {
			return nil, fmt.Errorf("Unsupported kind %v", kind)
		}

		podController, err := resourceKind.getPodController(c, ns, name)
		if err != nil {
			return nil, err
		}

		if !isAddon(podController) {
			controllers = append(controllers, podController.toClusterController(id))
		}
	}
	return controllers, nil
}

// AllControllers returns all controllers matching the criteria; that is, in
// the namespace (or any namespace if that argument is empty)
func (c *Cluster) AllControllers(namespace string) (res []cluster.Controller, err error) {
	namespaces, err := c.client.Namespaces().List(meta_v1.ListOptions{})
	if err != nil {
		return nil, errors.Wrap(err, "getting namespaces")
	}

	var allControllers []cluster.Controller
	for _, ns := range namespaces.Items {
		if namespace != "" && ns.Name != namespace {
			continue
		}

		for kind, resourceKind := range resourceKinds {
			podControllers, err := resourceKind.getPodControllers(c, ns.Name)
			if err != nil {
				if se, ok := err.(*apierrors.StatusError); ok && se.ErrStatus.Reason == meta_v1.StatusReasonNotFound {
					// Kind not supported by API server, skip
					continue
				} else {
					return nil, err
				}
			}

			for _, podController := range podControllers {
				if !isAddon(podController) {
					id := flux.MakeResourceID(ns.Name, kind, podController.name)
					allControllers = append(allControllers, podController.toClusterController(id))
				}
			}
		}
	}

	return allControllers, nil
}

// Sync performs the given actions on resources. Operations are
// asynchronous, but serialised.
func (c *Cluster) Sync(spec cluster.SyncDef) error {
	errc := make(chan error)
	logger := log.NewContext(c.logger).With("method", "Sync")
	c.actionc <- func() {
		errs := cluster.SyncError{}
		for _, action := range spec.Actions {
			logger := log.NewContext(logger).With("resource", action.ResourceID)
			if len(action.Delete) > 0 {
				obj, err := definitionObj(action.Delete)
				if err == nil {
					err = c.applier.Delete(logger, obj)
				}
				if err != nil {
					errs[action.ResourceID] = err
					continue
				}
			}
			if len(action.Apply) > 0 {
				obj, err := definitionObj(action.Apply)
				if err == nil {
					err = c.applier.Apply(logger, obj)
				}
				if err != nil {
					errs[action.ResourceID] = err
					continue
				}
			}
		}
		if len(errs) > 0 {
			errc <- errs
		} else {
			errc <- nil
		}
	}
	return <-errc
}

func (c *Cluster) Ping() error {
	_, err := c.client.ServerVersion()
	return err
}

func (c *Cluster) Export() ([]byte, error) {
	var config bytes.Buffer
	list, err := c.client.Namespaces().List(meta_v1.ListOptions{})
	if err != nil {
		return nil, errors.Wrap(err, "getting namespaces")
	}
	for _, ns := range list.Items {
		err := appendYAML(&config, "v1", "Namespace", ns)
		if err != nil {
			return nil, errors.Wrap(err, "marshalling namespace to YAML")
		}

		for _, resourceKind := range resourceKinds {
			podControllers, err := resourceKind.getPodControllers(c, ns.Name)
			if err != nil {
				if se, ok := err.(*apierrors.StatusError); ok && se.ErrStatus.Reason == meta_v1.StatusReasonNotFound {
					// Kind not supported by API server, skip
					continue
				} else {
					return nil, err
				}
			}

			for _, pc := range podControllers {
				if !isAddon(pc) {
					if err := appendYAML(&config, pc.apiVersion, pc.kind, pc.apiObject); err != nil {
						return nil, err
					}
				}
			}
		}
	}
	return config.Bytes(), nil
}

// kind & apiVersion must be passed separately as the object's TypeMeta is not populated
func appendYAML(buffer *bytes.Buffer, apiVersion, kind string, object interface{}) error {
	yamlBytes, err := k8syaml.Marshal(object)
	if err != nil {
		return err
	}
	buffer.WriteString("---\n")
	buffer.WriteString("apiVersion: ")
	buffer.WriteString(apiVersion)
	buffer.WriteString("\nkind: ")
	buffer.WriteString(kind)
	buffer.WriteString("\n")
	buffer.Write(yamlBytes)
	return nil
}

func (c *Cluster) PublicSSHKey(regenerate bool) (ssh.PublicKey, error) {
	if regenerate {
		if err := c.sshKeyRing.Regenerate(); err != nil {
			return ssh.PublicKey{}, err
		}
	}
	publicKey, _ := c.sshKeyRing.KeyPair()
	return publicKey, nil
}

func mergeCredentials(c *Cluster, namespace string, podTemplate apiv1.PodTemplateSpec, imageCreds registry.ImageCreds) {
	creds := registry.NoCredentials()
	for _, imagePullSecret := range podTemplate.Spec.ImagePullSecrets {
		secret, err := c.client.Secrets(namespace).Get(imagePullSecret.Name, meta_v1.GetOptions{})
		if err != nil {
			c.logger.Log("err", errors.Wrapf(err, "getting secret %q from namespace %q", secret.Name, namespace))
			continue
		}

		var decoded []byte
		var ok bool
		// These differ in format; but, ParseCredentials will
		// handle either.
		switch api.SecretType(secret.Type) {
		case api.SecretTypeDockercfg:
			decoded, ok = secret.Data[api.DockerConfigKey]
		case api.SecretTypeDockerConfigJson:
			decoded, ok = secret.Data[api.DockerConfigJsonKey]
		default:
			c.logger.Log("skip", "unknown type", "secret", namespace+"/"+secret.Name, "type", secret.Type)
			continue
		}

		if !ok {
			c.logger.Log("err", errors.Wrapf(err, "retrieving pod secret %q", secret.Name))
			continue
		}

		// Parse secret
		crd, err := registry.ParseCredentials(decoded)
		if err != nil {
			c.logger.Log("err", err.Error())
			continue
		}

		// Merge into the credentials for this PodSpec
		creds.Merge(crd)
	}

	// Now create the service and attach the credentials
	for _, container := range podTemplate.Spec.Containers {
		r, err := flux.ParseImageID(container.Image)
		if err != nil {
			c.logger.Log("err", err.Error())
			continue
		}
		imageCreds[r] = creds
	}
}

// ImagesToFetch is a k8s specific method to get a list of images to update along with their credentials
func (c *Cluster) ImagesToFetch() registry.ImageCreds {
	allImageCreds := make(registry.ImageCreds)

	namespaces, err := c.client.Namespaces().List(meta_v1.ListOptions{})
	if err != nil {
		c.logger.Log("err", errors.Wrap(err, "getting namespaces"))
		return allImageCreds
	}

	for _, ns := range namespaces.Items {
		for kind, resourceKind := range resourceKinds {
			podControllers, err := resourceKind.getPodControllers(c, ns.Name)
			if err != nil {
				if se, ok := err.(*apierrors.StatusError); ok && se.ErrStatus.Reason == meta_v1.StatusReasonNotFound {
					// Kind not supported by API server, skip
				} else {
					c.logger.Log("err", errors.Wrapf(err, "getting kind %s for namespace %s", kind, ns.Name))
				}
				continue
			}

			imageCreds := make(registry.ImageCreds)
			for _, podController := range podControllers {
				mergeCredentials(c, ns.Name, podController.podTemplate, imageCreds)
			}

			// Merge creds
			for imageID, creds := range imageCreds {
				existingCreds, ok := allImageCreds[imageID]
				if ok {
					existingCreds.Merge(creds)
				} else {
					allImageCreds[imageID] = creds
				}
			}
		}
	}

	return allImageCreds
}

// --- end cluster.Cluster

// A convenience for getting an minimal object from some bytes.
func definitionObj(bytes []byte) (*apiObject, error) {
	obj := apiObject{bytes: bytes}
	return &obj, yaml.Unmarshal(bytes, &obj)
}
<|endoftext|>