<commit_before>// Copyright 2016 Joel Scoble and The JoeFriday authors.
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package cpux provides information about each cpuX on the system, where X is
// the integer of each CPU on the system, e.g. cpu0, cpu1, etc. On linux
// systems this comes from /sys/devices/system/cpu. Not all paths are available
// on all systems, e.g. /sys/devices/system/cpu/cpuX/cpufreq and its children
// may not exist on some systems. If the system doesn't have a particular file
// within this path, the field's value will be the type's zero value.
//
// This package does not currently have a ticker implementation.
package cpux

import (
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"runtime"
	"sort"
	"strconv"
	"strings"
)

const (
	SystemCPUPath = "/sys/devices/system/cpu"
	CPUFreq       = "cpufreq"
)

type CPUs struct {
	Sockets int32 `json:"sockets"`
	CPU     []CPU `json:"cpu"`
}

type CPU struct {
	PhysicalPackageID int32             `json:"physical_package_id"`
	CoreID            int32             `json:"core_id"`
	MHzMin            float32           `json:"mhz_min"`
	MHzMax            float32           `json:"mhz_max"`
	Cache             map[string]string `json:"cache:`
	// a sorted list of caches so that the cache info can be pulled out in order.
	CacheIDs []string
}

// GetCPU returns the cpu information for the provided physical_package_id
// (pID) and core_id (coreID). A false will be returned if an entry matching
// the physical_package_id and core_id is not found.
func (c *CPUs) GetCPU(pID, coreID int32) (cpu CPU, found bool) {
	for i := 0; i < len(c.CPU); i++ {
		if c.CPU[i].PhysicalPackageID == pID && c.CPU[i].CoreID == coreID {
			return c.CPU[i], true
		}
	}
	return CPU{}, false
}

// Profiler is used to process the system's cpuX information.
type Profiler struct {
	// this is an exported fied for testing purposes. It should not be set in
	// non-test usage
	NumCPU int
	// this is an exported fied for testing purposes. It should not be set in
	// non-test usage
	SystemCPUPath string
}

// Returns an initialized Profiler; ready to use.
func NewProfiler() (prof *Profiler, err error) {
	// NumCPU provides the number of logical cpus usable by the current process.
	// Is this sufficient, or will there ever be a delta between that and either
	// what /proc/cpuinfo reports or what is available on /sys/devices/system/cpu/
	return &Profiler{NumCPU: runtime.NumCPU(), SystemCPUPath: SystemCPUPath}, nil
}

// Reset resources: this does nothing for this implemenation.
func (prof *Profiler) Reset() error {
	return nil
}

// Get the cpuX info for each cpu. Currently only min and max frequency are
// implemented.
func (prof *Profiler) Get() (*CPUs, error) {
	cpus := &CPUs{CPU: make([]CPU, prof.NumCPU)}
	var err error
	var pids []int32 // the physical ids encountered

	hasFreq := prof.hasCPUFreq()
	for x := 0; x < prof.NumCPU; x++ {
		var cpu CPU
		var found bool

		cpu.PhysicalPackageID, err = prof.physicalPackageID(x)
		if err != nil {
			return nil, err
		}
		// see if this is a new physical id; if so, add it to the inventory
		for _, v := range pids {
			if v == cpu.PhysicalPackageID {
				found = true
				break
			}
		}
		if !found {
			pids = append(pids, cpu.PhysicalPackageID)
		}
		cpu.CoreID, err = prof.coreID(x)
		if err != nil {
			return nil, err
		}
		err := prof.cache(x, &cpu)
		if err != nil {
			return nil, err
		}
		if hasFreq {
			cpu.MHzMin, err = prof.cpuMHzMin(x)
			if err != nil {
				return nil, err
			}
			cpu.MHzMax, err = prof.cpuMHzMax(x)
			if err != nil {
				return nil, err
			}
		}
		cpus.CPU[x] = cpu
	}
	cpus.Sockets = int32(len(pids))
	return cpus, nil
}

// cpuXPath returns the system's cpuX path for a given cpu number.
func (prof *Profiler) cpuXPath(x int) string {
	return fmt.Sprintf("%s/cpu%d", prof.SystemCPUPath, x)
}

// coreIDPath returns the path of the core_id file for the given cpuX.
func (prof *Profiler) coreIDPath(x int) string {
	return fmt.Sprintf("%s/topology/core_id", prof.cpuXPath(x))
}

// physicalPackageIDPath returns the path of the physical_package_id file for
// the given cpuX.
func (prof *Profiler) physicalPackageIDPath(x int) string {
	return fmt.Sprintf("%s/topology/physical_package_id", prof.cpuXPath(x))
}

// cpuInfoFreqMaxPath returns the path for the cpuinfo_max_freq file of the
// given cpuX.
func (prof *Profiler) cpuInfoFreqMaxPath(x int) string {
	return fmt.Sprintf("%s/cpufreq/cpuinfo_max_freq", prof.cpuXPath(x))
}

// cpuInfoFreqMinPath returns the path for the cpuinfo_min_freq file of the
// given cpuX.
func (prof *Profiler) cpuInfoFreqMinPath(x int) string {
	return fmt.Sprintf("%s/cpufreq/cpuinfo_min_freq", prof.cpuXPath(x))
}

// cachePath returns the path for the cache dir
func (prof *Profiler) cachePath(x int) string {
	return fmt.Sprintf("%s/cache", prof.cpuXPath(x))
}

// hasCPUFreq returns if the system has cpufreq information:
func (prof *Profiler) hasCPUFreq() bool {
	_, err := os.Stat(filepath.Join(prof.SystemCPUPath, CPUFreq))
	if err == nil {
		return true
	}
	return false
}

// gets the core_id of cpuX
func (prof *Profiler) coreID(x int) (int32, error) {
	v, err := ioutil.ReadFile(prof.coreIDPath(x))
	if err != nil {
		return 0, err
	}
	id, err := strconv.Atoi(string(v[:len(v)-1]))
	if err != nil {
		return 0, fmt.Errorf("cpu%d core_id: conversion error: %s", x, err)
	}
	return int32(id), nil
}

// gets the physical_package_id of cpuX
func (prof *Profiler) physicalPackageID(x int) (int32, error) {
	v, err := ioutil.ReadFile(prof.physicalPackageIDPath(x))
	if err != nil {
		return 0, err
	}
	id, err := strconv.Atoi(string(v[:len(v)-1]))
	if err != nil {
		return 0, fmt.Errorf("cpu%d physical_package_id: conversion error: %s", x, err)
	}
	return int32(id), nil
}

// gets the cpu_mhz_min information
func (prof *Profiler) cpuMHzMin(x int) (float32, error) {
	v, err := ioutil.ReadFile(prof.cpuInfoFreqMinPath(x))
	if err != nil {
		return 0, err
	}
	// insert the . in the appropriate spot
	v = append(v[:len(v)-4], append([]byte{'.'}, v[len(v)-4:len(v)-1]...)...)
	m, err := strconv.ParseFloat(string(v[:len(v)-1]), 32)
	if err != nil {
		return 0, fmt.Errorf("cpu%d MHz min: conversion error: %s", x, err)
	}
	return float32(m), nil
}

// gets the cpu_mhz_max information
func (prof *Profiler) cpuMHzMax(x int) (float32, error) {
	v, err := ioutil.ReadFile(prof.cpuInfoFreqMaxPath(x))
	if err != nil {
		return 0, err
	}
	// insert the . in the appropriate spot
	v = append(v[:len(v)-4], append([]byte{'.'}, v[len(v)-4:len(v)-1]...)...)
	m, err := strconv.ParseFloat(string(v[:len(v)-1]), 32)
	if err != nil {
		return 0, fmt.Errorf("cpu%d MHz max: conversion error: %s", x, err)
	}
	return float32(m), nil
}

// Get the cache info for the given cpuX entry
func (prof *Profiler) cache(x int, cpu *CPU) error {
	cpu.Cache = map[string]string{}
	//go through all the entries in cpuX/cache
	p := prof.cachePath(x)
	dirs, err := ioutil.ReadDir(p)
	if err != nil {
		return err
	}
	var cacheID string
	// all the entries should be dirs with their contents holding the cache info
	for _, d := range dirs {
		if !d.IsDir() {
			continue // this shouldn't happen but if it does we just skip the entry
		}
		// cache level
		l, err := ioutil.ReadFile(filepath.Join(p, d.Name(), "level"))
		if err != nil {
			return err
		}

		t, err := ioutil.ReadFile(filepath.Join(p, d.Name(), "type"))
		if err != nil {
			return err
		}

		// cache type: unified entries aren't decorated, otherwise the first letter is used
		// like what lscpu does.
		if t[0] != 'U' && t[0] != 'u' {
			cacheID = fmt.Sprintf("L%s%s cache", string(l[:len(l)-1]), strings.ToLower(string(t[0])))
		} else {
			cacheID = fmt.Sprintf("L%s cache", string(l[:len(l)-1]))
		}

		// cache size
		s, err := ioutil.ReadFile(filepath.Join(p, d.Name(), "size"))
		if err != nil {
			return err
		}

		// add the info
		cpu.Cache[cacheID] = string(s[:len(s)-1])
		cpu.CacheIDs = append(cpu.CacheIDs, cacheID)
	}
	// sort the cache names
	sort.Strings(cpu.CacheIDs)

	return nil
}
<commit_msg>add json tag to omit id field<commit_after>// Copyright 2016 Joel Scoble and The JoeFriday authors.
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package cpux provides information about each cpuX on the system, where X is
// the integer of each CPU on the system, e.g. cpu0, cpu1, etc. On linux
// systems this comes from /sys/devices/system/cpu. Not all paths are available
// on all systems, e.g. /sys/devices/system/cpu/cpuX/cpufreq and its children
// may not exist on some systems. If the system doesn't have a particular file
// within this path, the field's value will be the type's zero value.
//
// This package does not currently have a ticker implementation.
package cpux

import (
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"runtime"
	"sort"
	"strconv"
	"strings"
)

const (
	SystemCPUPath = "/sys/devices/system/cpu"
	CPUFreq       = "cpufreq"
)

type CPUs struct {
	Sockets int32 `json:"sockets"`
	CPU     []CPU `json:"cpu"`
}

type CPU struct {
	PhysicalPackageID int32             `json:"physical_package_id"`
	CoreID            int32             `json:"core_id"`
	MHzMin            float32           `json:"mhz_min"`
	MHzMax            float32           `json:"mhz_max"`
	Cache             map[string]string `json:"cache:`
	// a sorted list of caches so that the cache info can be pulled out in order.
	CacheIDs []string `json:"-"`
}

// GetCPU returns the cpu information for the provided physical_package_id
// (pID) and core_id (coreID). A false will be returned if an entry matching
// the physical_package_id and core_id is not found.
func (c *CPUs) GetCPU(pID, coreID int32) (cpu CPU, found bool) {
	for i := 0; i < len(c.CPU); i++ {
		if c.CPU[i].PhysicalPackageID == pID && c.CPU[i].CoreID == coreID {
			return c.CPU[i], true
		}
	}
	return CPU{}, false
}

// Profiler is used to process the system's cpuX information.
type Profiler struct {
	// this is an exported fied for testing purposes. It should not be set in
	// non-test usage
	NumCPU int
	// this is an exported fied for testing purposes. It should not be set in
	// non-test usage
	SystemCPUPath string
}

// Returns an initialized Profiler; ready to use.
func NewProfiler() (prof *Profiler, err error) {
	// NumCPU provides the number of logical cpus usable by the current process.
	// Is this sufficient, or will there ever be a delta between that and either
	// what /proc/cpuinfo reports or what is available on /sys/devices/system/cpu/
	return &Profiler{NumCPU: runtime.NumCPU(), SystemCPUPath: SystemCPUPath}, nil
}

// Reset resources: this does nothing for this implemenation.
func (prof *Profiler) Reset() error {
	return nil
}

// Get the cpuX info for each cpu. Currently only min and max frequency are
// implemented.
func (prof *Profiler) Get() (*CPUs, error) {
	cpus := &CPUs{CPU: make([]CPU, prof.NumCPU)}
	var err error
	var pids []int32 // the physical ids encountered

	hasFreq := prof.hasCPUFreq()
	for x := 0; x < prof.NumCPU; x++ {
		var cpu CPU
		var found bool

		cpu.PhysicalPackageID, err = prof.physicalPackageID(x)
		if err != nil {
			return nil, err
		}
		// see if this is a new physical id; if so, add it to the inventory
		for _, v := range pids {
			if v == cpu.PhysicalPackageID {
				found = true
				break
			}
		}
		if !found {
			pids = append(pids, cpu.PhysicalPackageID)
		}
		cpu.CoreID, err = prof.coreID(x)
		if err != nil {
			return nil, err
		}
		err := prof.cache(x, &cpu)
		if err != nil {
			return nil, err
		}
		if hasFreq {
			cpu.MHzMin, err = prof.cpuMHzMin(x)
			if err != nil {
				return nil, err
			}
			cpu.MHzMax, err = prof.cpuMHzMax(x)
			if err != nil {
				return nil, err
			}
		}
		cpus.CPU[x] = cpu
	}
	cpus.Sockets = int32(len(pids))
	return cpus, nil
}

// cpuXPath returns the system's cpuX path for a given cpu number.
func (prof *Profiler) cpuXPath(x int) string {
	return fmt.Sprintf("%s/cpu%d", prof.SystemCPUPath, x)
}

// coreIDPath returns the path of the core_id file for the given cpuX.
func (prof *Profiler) coreIDPath(x int) string {
	return fmt.Sprintf("%s/topology/core_id", prof.cpuXPath(x))
}

// physicalPackageIDPath returns the path of the physical_package_id file for
// the given cpuX.
func (prof *Profiler) physicalPackageIDPath(x int) string {
	return fmt.Sprintf("%s/topology/physical_package_id", prof.cpuXPath(x))
}

// cpuInfoFreqMaxPath returns the path for the cpuinfo_max_freq file of the
// given cpuX.
func (prof *Profiler) cpuInfoFreqMaxPath(x int) string {
	return fmt.Sprintf("%s/cpufreq/cpuinfo_max_freq", prof.cpuXPath(x))
}

// cpuInfoFreqMinPath returns the path for the cpuinfo_min_freq file of the
// given cpuX.
func (prof *Profiler) cpuInfoFreqMinPath(x int) string {
	return fmt.Sprintf("%s/cpufreq/cpuinfo_min_freq", prof.cpuXPath(x))
}

// cachePath returns the path for the cache dir
func (prof *Profiler) cachePath(x int) string {
	return fmt.Sprintf("%s/cache", prof.cpuXPath(x))
}

// hasCPUFreq returns if the system has cpufreq information:
func (prof *Profiler) hasCPUFreq() bool {
	_, err := os.Stat(filepath.Join(prof.SystemCPUPath, CPUFreq))
	if err == nil {
		return true
	}
	return false
}

// gets the core_id of cpuX
func (prof *Profiler) coreID(x int) (int32, error) {
	v, err := ioutil.ReadFile(prof.coreIDPath(x))
	if err != nil {
		return 0, err
	}
	id, err := strconv.Atoi(string(v[:len(v)-1]))
	if err != nil {
		return 0, fmt.Errorf("cpu%d core_id: conversion error: %s", x, err)
	}
	return int32(id), nil
}

// gets the physical_package_id of cpuX
func (prof *Profiler) physicalPackageID(x int) (int32, error) {
	v, err := ioutil.ReadFile(prof.physicalPackageIDPath(x))
	if err != nil {
		return 0, err
	}
	id, err := strconv.Atoi(string(v[:len(v)-1]))
	if err != nil {
		return 0, fmt.Errorf("cpu%d physical_package_id: conversion error: %s", x, err)
	}
	return int32(id), nil
}

// gets the cpu_mhz_min information
func (prof *Profiler) cpuMHzMin(x int) (float32, error) {
	v, err := ioutil.ReadFile(prof.cpuInfoFreqMinPath(x))
	if err != nil {
		return 0, err
	}
	// insert the . in the appropriate spot
	v = append(v[:len(v)-4], append([]byte{'.'}, v[len(v)-4:len(v)-1]...)...)
	m, err := strconv.ParseFloat(string(v[:len(v)-1]), 32)
	if err != nil {
		return 0, fmt.Errorf("cpu%d MHz min: conversion error: %s", x, err)
	}
	return float32(m), nil
}

// gets the cpu_mhz_max information
func (prof *Profiler) cpuMHzMax(x int) (float32, error) {
	v, err := ioutil.ReadFile(prof.cpuInfoFreqMaxPath(x))
	if err != nil {
		return 0, err
	}
	// insert the . in the appropriate spot
	v = append(v[:len(v)-4], append([]byte{'.'}, v[len(v)-4:len(v)-1]...)...)
	m, err := strconv.ParseFloat(string(v[:len(v)-1]), 32)
	if err != nil {
		return 0, fmt.Errorf("cpu%d MHz max: conversion error: %s", x, err)
	}
	return float32(m), nil
}

// Get the cache info for the given cpuX entry
func (prof *Profiler) cache(x int, cpu *CPU) error {
	cpu.Cache = map[string]string{}
	//go through all the entries in cpuX/cache
	p := prof.cachePath(x)
	dirs, err := ioutil.ReadDir(p)
	if err != nil {
		return err
	}
	var cacheID string
	// all the entries should be dirs with their contents holding the cache info
	for _, d := range dirs {
		if !d.IsDir() {
			continue // this shouldn't happen but if it does we just skip the entry
		}
		// cache level
		l, err := ioutil.ReadFile(filepath.Join(p, d.Name(), "level"))
		if err != nil {
			return err
		}

		t, err := ioutil.ReadFile(filepath.Join(p, d.Name(), "type"))
		if err != nil {
			return err
		}

		// cache type: unified entries aren't decorated, otherwise the first letter is used
		// like what lscpu does.
		if t[0] != 'U' && t[0] != 'u' {
			cacheID = fmt.Sprintf("L%s%s cache", string(l[:len(l)-1]), strings.ToLower(string(t[0])))
		} else {
			cacheID = fmt.Sprintf("L%s cache", string(l[:len(l)-1]))
		}

		// cache size
		s, err := ioutil.ReadFile(filepath.Join(p, d.Name(), "size"))
		if err != nil {
			return err
		}

		// add the info
		cpu.Cache[cacheID] = string(s[:len(s)-1])
		cpu.CacheIDs = append(cpu.CacheIDs, cacheID)
	}
	// sort the cache names
	sort.Strings(cpu.CacheIDs)

	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"strings"

	"github.com/julian-klode/lingolang/permission"
	"github.com/julian-klode/lingolang/permission/parser"
)

func main() {
	fmt.Printf("Owned = %v:%T\n", permission.Owned, permission.Owned)
	fmt.Printf("Read = %v:%T\n", permission.Read, permission.Read)
	fmt.Printf("Write = %v:%T\n", permission.Write, permission.Write)
	fmt.Printf("ExclRead = %v:%T\n", permission.ExclRead, permission.ExclRead)
	fmt.Printf("ExclWrite = %v:%T\n", permission.ExclWrite, permission.ExclWrite)

	sc := parser.NewScanner(strings.NewReader("of (or) func (oa, ob) oR"))
	for tok := sc.Scan(); tok.Type != parser.EndOfFile; tok = sc.Scan() {
		fmt.Printf("Token %#v \n", tok)
	}

	p := parser.NewParser(strings.NewReader("om map [ov] ol"))
	perm, err := p.Parse()
	fmt.Printf("Parsed %v with error %v", perm, err)

}
<commit_msg>lingolint: Make compile again<commit_after>package main

import (
	"fmt"

	"github.com/julian-klode/lingolang/permission"
	"github.com/julian-klode/lingolang/permission/parser"
)

func main() {
	fmt.Printf("Owned = %v:%T\n", permission.Owned, permission.Owned)
	fmt.Printf("Read = %v:%T\n", permission.Read, permission.Read)
	fmt.Printf("Write = %v:%T\n", permission.Write, permission.Write)
	fmt.Printf("ExclRead = %v:%T\n", permission.ExclRead, permission.ExclRead)
	fmt.Printf("ExclWrite = %v:%T\n", permission.ExclWrite, permission.ExclWrite)

	sc := parser.NewScanner("of (or) func (oa, ob) oR")
	for tok := sc.Scan(); tok.Type != parser.EndOfFile; tok = sc.Scan() {
		fmt.Printf("Token %#v \n", tok)
	}

	p := parser.NewParser("om map [ov] ol")
	perm, err := p.Parse()
	fmt.Printf("Parsed %v with error %v", perm, err)

}
<|endoftext|>


<commit_before>package logger

import (
	"context"
	"fmt"
	"os"
	"path/filepath"
	"runtime"
	"sort"
	"strings"
	"sync"
	"time"

	dlog "github.com/micro/go-micro/v2/debug/log"
)

func init() {
	lvl, err := GetLevel(os.Getenv("MICRO_LOG_LEVEL"))
	if err != nil {
		lvl = InfoLevel
	}

	DefaultLogger = NewHelper(NewLogger(WithLevel(lvl)))
}

type defaultLogger struct {
	sync.RWMutex
	opts Options
}

// Init(opts...) should only overwrite provided options
func (l *defaultLogger) Init(opts ...Option) error {
	for _, o := range opts {
		o(&l.opts)
	}
	return nil
}

func (l *defaultLogger) String() string {
	return "default"
}

func (l *defaultLogger) Fields(fields map[string]interface{}) Logger {
	l.Lock()
	l.opts.Fields = copyFields(fields)
	l.Unlock()
	return l
}

func copyFields(src map[string]interface{}) map[string]interface{} {
	dst := make(map[string]interface{}, len(src))
	for k, v := range src {
		dst[k] = v
	}
	return dst
}

func logCallerfilePath(loggingFilePath string) string {
	parts := strings.Split(loggingFilePath, string(filepath.Separator))
	return parts[len(parts)-1]
}

func (l *defaultLogger) Log(level Level, v ...interface{}) {
	// TODO decide does we need to write message if log level not used?
	if !l.opts.Level.Enabled(level) {
		return
	}

	l.RLock()
	fields := copyFields(l.opts.Fields)
	l.RUnlock()

	fields["level"] = level.String()

	if _, file, line, ok := runtime.Caller(l.opts.CallerSkipCount); ok {
		fields["file"] = fmt.Sprintf("%s:%d", logCallerfilePath(file), line)
	}

	rec := dlog.Record{
		Timestamp: time.Now(),
		Message:   fmt.Sprint(v...),
		Metadata:  make(map[string]string, len(fields)),
	}

	keys := make([]string, 0, len(fields))
	for k, v := range fields {
		keys = append(keys, k)
		rec.Metadata[k] = fmt.Sprintf("%v", v)
	}

	sort.Strings(keys)
	metadata := ""

	for _, k := range keys {
		metadata += fmt.Sprintf(" %s=%v", k, fields[k])
	}

	dlog.DefaultLog.Write(rec)

	t := rec.Timestamp.Format("2006-01-02 15:04:05")
	fmt.Printf("%s %s %v\n", t, metadata, rec.Message)
}

func (l *defaultLogger) Logf(level Level, format string, v ...interface{}) {
	//	 TODO decide does we need to write message if log level not used?
	if level < l.opts.Level {
		return
	}

	l.RLock()
	fields := copyFields(l.opts.Fields)
	l.RUnlock()

	fields["level"] = level.String()

	if _, file, line, ok := runtime.Caller(l.opts.CallerSkipCount); ok {
		fields["file"] = fmt.Sprintf("%s:%d", logCallerfilePath(file), line)
	}

	rec := dlog.Record{
		Timestamp: time.Now(),
		Message:   fmt.Sprintf(format, v...),
		Metadata:  make(map[string]string, len(fields)),
	}

	keys := make([]string, 0, len(fields))
	for k, v := range fields {
		keys = append(keys, k)
		rec.Metadata[k] = fmt.Sprintf("%v", v)
	}

	sort.Strings(keys)
	metadata := ""

	for _, k := range keys {
		metadata += fmt.Sprintf(" %s=%v", k, fields[k])
	}

	dlog.DefaultLog.Write(rec)

	t := rec.Timestamp.Format("2006-01-02 15:04:05")
	fmt.Printf("%s %s %v\n", t, metadata, rec.Message)
}

func (n *defaultLogger) Options() Options {
	// not guard against options Context values
	n.RLock()
	opts := n.opts
	opts.Fields = copyFields(n.opts.Fields)
	n.RUnlock()
	return opts
}

// NewLogger builds a new logger based on options
func NewLogger(opts ...Option) Logger {
	// Default options
	options := Options{
		Level:           InfoLevel,
		Fields:          make(map[string]interface{}),
		Out:             os.Stderr,
		CallerSkipCount: 2,
		Context:         context.Background(),
	}

	l := &defaultLogger{opts: options}
	if err := l.Init(opts...); err != nil {
		l.Log(FatalLevel, err)
	}

	return l
}
<commit_msg>make caller filepath package/file style this code is from zap https://github.com/uber-go/zap/blob/9a9fa7d4b5f07a9b634983678a65b5525f81e58b/zapcore/entry.go#L101<commit_after>package logger

import (
	"context"
	"fmt"
	"os"
	"runtime"
	"sort"
	"strings"
	"sync"
	"time"

	dlog "github.com/micro/go-micro/v2/debug/log"
)

func init() {
	lvl, err := GetLevel(os.Getenv("MICRO_LOG_LEVEL"))
	if err != nil {
		lvl = InfoLevel
	}

	DefaultLogger = NewHelper(NewLogger(WithLevel(lvl)))
}

type defaultLogger struct {
	sync.RWMutex
	opts Options
}

// Init(opts...) should only overwrite provided options
func (l *defaultLogger) Init(opts ...Option) error {
	for _, o := range opts {
		o(&l.opts)
	}
	return nil
}

func (l *defaultLogger) String() string {
	return "default"
}

func (l *defaultLogger) Fields(fields map[string]interface{}) Logger {
	l.Lock()
	l.opts.Fields = copyFields(fields)
	l.Unlock()
	return l
}

func copyFields(src map[string]interface{}) map[string]interface{} {
	dst := make(map[string]interface{}, len(src))
	for k, v := range src {
		dst[k] = v
	}
	return dst
}

// logCallerfilePath returns a package/file:line description of the caller,
// preserving only the leaf directory name and file name.
func logCallerfilePath(loggingFilePath string) string {
	// To make sure we trim the path correctly on Windows too, we
	// counter-intuitively need to use '/' and *not* os.PathSeparator here,
	// because the path given originates from Go stdlib, specifically
	// runtime.Caller() which (as of Mar/17) returns forward slashes even on
	// Windows.
	//
	// See https://github.com/golang/go/issues/3335
	// and https://github.com/golang/go/issues/18151
	//
	// for discussion on the issue on Go side.
	idx := strings.LastIndexByte(loggingFilePath, '/')
	if idx == -1 {
		return loggingFilePath
	}
	idx = strings.LastIndexByte(loggingFilePath[:idx], '/')
	if idx == -1 {
		return loggingFilePath
	}
	return loggingFilePath[idx+1:]
}

func (l *defaultLogger) Log(level Level, v ...interface{}) {
	// TODO decide does we need to write message if log level not used?
	if !l.opts.Level.Enabled(level) {
		return
	}

	l.RLock()
	fields := copyFields(l.opts.Fields)
	l.RUnlock()

	fields["level"] = level.String()

	if _, file, line, ok := runtime.Caller(l.opts.CallerSkipCount); ok {
		fields["file"] = fmt.Sprintf("%s:%d", logCallerfilePath(file), line)
	}

	rec := dlog.Record{
		Timestamp: time.Now(),
		Message:   fmt.Sprint(v...),
		Metadata:  make(map[string]string, len(fields)),
	}

	keys := make([]string, 0, len(fields))
	for k, v := range fields {
		keys = append(keys, k)
		rec.Metadata[k] = fmt.Sprintf("%v", v)
	}

	sort.Strings(keys)
	metadata := ""

	for _, k := range keys {
		metadata += fmt.Sprintf(" %s=%v", k, fields[k])
	}

	dlog.DefaultLog.Write(rec)

	t := rec.Timestamp.Format("2006-01-02 15:04:05")
	fmt.Printf("%s %s %v\n", t, metadata, rec.Message)
}

func (l *defaultLogger) Logf(level Level, format string, v ...interface{}) {
	//	 TODO decide does we need to write message if log level not used?
	if level < l.opts.Level {
		return
	}

	l.RLock()
	fields := copyFields(l.opts.Fields)
	l.RUnlock()

	fields["level"] = level.String()

	if _, file, line, ok := runtime.Caller(l.opts.CallerSkipCount); ok {
		fields["file"] = fmt.Sprintf("%s:%d", logCallerfilePath(file), line)
	}

	rec := dlog.Record{
		Timestamp: time.Now(),
		Message:   fmt.Sprintf(format, v...),
		Metadata:  make(map[string]string, len(fields)),
	}

	keys := make([]string, 0, len(fields))
	for k, v := range fields {
		keys = append(keys, k)
		rec.Metadata[k] = fmt.Sprintf("%v", v)
	}

	sort.Strings(keys)
	metadata := ""

	for _, k := range keys {
		metadata += fmt.Sprintf(" %s=%v", k, fields[k])
	}

	dlog.DefaultLog.Write(rec)

	t := rec.Timestamp.Format("2006-01-02 15:04:05")
	fmt.Printf("%s %s %v\n", t, metadata, rec.Message)
}

func (n *defaultLogger) Options() Options {
	// not guard against options Context values
	n.RLock()
	opts := n.opts
	opts.Fields = copyFields(n.opts.Fields)
	n.RUnlock()
	return opts
}

// NewLogger builds a new logger based on options
func NewLogger(opts ...Option) Logger {
	// Default options
	options := Options{
		Level:           InfoLevel,
		Fields:          make(map[string]interface{}),
		Out:             os.Stderr,
		CallerSkipCount: 2,
		Context:         context.Background(),
	}

	l := &defaultLogger{opts: options}
	if err := l.Init(opts...); err != nil {
		l.Log(FatalLevel, err)
	}

	return l
}
<|endoftext|>


<commit_before>// Copyright (c) 2018 Cisco and/or its affiliates.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at:
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package cmdimpl

import (
	"encoding/json"
	"fmt"
	"github.com/contiv/vpp/plugins/contiv/model/node"
	"github.com/contiv/vpp/plugins/crd/cache/telemetrymodel"
	"github.com/contiv/vpp/plugins/netctl/http"
	"github.com/coreos/etcd/clientv3"
	"github.com/ligato/cn-infra/db/keyval/etcd"
	"github.com/ligato/cn-infra/logging"
	"github.com/ligato/cn-infra/logging/logrus"
	"os"
	"text/tabwriter"
	"time"
)

func PrintAllIpams() {
	etcdCfg := etcd.ClientConfig{
		Config: &clientv3.Config{
			Endpoints: []string{"127.0.0.1:32379"},
		},
		OpTimeout: 1 * time.Second,
	}

	logger := logrus.DefaultLogger()
	logger.SetLevel(logging.ErrorLevel)

	// Create connection to etcd.
	var err error
	var db *etcd.BytesConnectionEtcd
	if db, err = etcd.NewEtcdConnectionWithBytes(etcdCfg, logger); err != nil {
		fmt.Println(err)
		os.Exit(1)
	}

	itr, err := db.ListValues("/vnf-agent/contiv-ksr/allocatedIDs/")
	if err != nil {
		fmt.Printf("Failed to discover nodes in Contiv cluster")
		return
	}

	w := getTabWriterAndPrintHeader()
	for {
		kv, stop := itr.GetNext()
		if stop {
			fmt.Println()
			break
		}
		buf := kv.GetValue()
		nodeInfo := &node.NodeInfo{}
		err = json.Unmarshal(buf, nodeInfo)
		nodeIpamCmd(w, nodeInfo.Name)
	}
	w.Flush()
	db.Close()
}

//NodeIPamCmd prints out the ipam information of a specific node
func NodeIPamCmd(nodeName string) {
	w := getTabWriterAndPrintHeader()
	nodeIpamCmd(w, nodeName)
	w.Flush()
}

func nodeIpamCmd(w *tabwriter.Writer, nodeName string) {
	ip := resolveNodeOrIP(nodeName)
	b := http.GetNodeInfo(ip, "contiv/v1/ipam")
	ipam := telemetrymodel.IPamEntry{}
	err := json.Unmarshal(b, &ipam)
	if err != nil {
		fmt.Println(err)
	}

	fmt.Fprintf(w, "%d\t%s\t%s\t%s\t%s\t%s\t%s\n",
		ipam.NodeID,
		ipam.NodeName,
		ipam.NodeIP,
		ipam.PodNetwork,
		ipam.VppHostNetwork,
		ipam.Config.PodIfIPCIDR,
		ipam.Config.PodSubnetCIRDR)
}

func getTabWriterAndPrintHeader() *tabwriter.Writer {
	w := tabwriter.NewWriter(os.Stdout, 0, 8, 2, ' ', 0)
	fmt.Fprintf(w, "ID\tNODE-NAME\tNODE-IP\tPOD-NET-IP\tVPP-HOST-IP\tPOD-IFIP-CIDR\tPOD-SUBNET-CIDR\n")
	return w
}
<commit_msg>netctl: fix golint travis issue<commit_after>// Copyright (c) 2018 Cisco and/or its affiliates.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at:
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package cmdimpl

import (
	"encoding/json"
	"fmt"
	"github.com/contiv/vpp/plugins/contiv/model/node"
	"github.com/contiv/vpp/plugins/crd/cache/telemetrymodel"
	"github.com/contiv/vpp/plugins/netctl/http"
	"github.com/coreos/etcd/clientv3"
	"github.com/ligato/cn-infra/db/keyval/etcd"
	"github.com/ligato/cn-infra/logging"
	"github.com/ligato/cn-infra/logging/logrus"
	"os"
	"text/tabwriter"
	"time"
)

// PrintAllIpams prints out the ipam information for all the nodes
func PrintAllIpams() {
	etcdCfg := etcd.ClientConfig{
		Config: &clientv3.Config{
			Endpoints: []string{"127.0.0.1:32379"},
		},
		OpTimeout: 1 * time.Second,
	}

	logger := logrus.DefaultLogger()
	logger.SetLevel(logging.ErrorLevel)

	// Create connection to etcd.
	var err error
	var db *etcd.BytesConnectionEtcd
	if db, err = etcd.NewEtcdConnectionWithBytes(etcdCfg, logger); err != nil {
		fmt.Println(err)
		os.Exit(1)
	}

	itr, err := db.ListValues("/vnf-agent/contiv-ksr/allocatedIDs/")
	if err != nil {
		fmt.Printf("Failed to discover nodes in Contiv cluster")
		return
	}

	w := getTabWriterAndPrintHeader()
	for {
		kv, stop := itr.GetNext()
		if stop {
			fmt.Println()
			break
		}
		buf := kv.GetValue()
		nodeInfo := &node.NodeInfo{}
		err = json.Unmarshal(buf, nodeInfo)
		nodeIpamCmd(w, nodeInfo.Name)
	}
	w.Flush()
	db.Close()
}

//NodeIPamCmd prints out the ipam information of a specific node
func NodeIPamCmd(nodeName string) {
	w := getTabWriterAndPrintHeader()
	nodeIpamCmd(w, nodeName)
	w.Flush()
}

func nodeIpamCmd(w *tabwriter.Writer, nodeName string) {
	ip := resolveNodeOrIP(nodeName)
	b := http.GetNodeInfo(ip, "contiv/v1/ipam")
	ipam := telemetrymodel.IPamEntry{}
	err := json.Unmarshal(b, &ipam)
	if err != nil {
		fmt.Println(err)
	}

	fmt.Fprintf(w, "%d\t%s\t%s\t%s\t%s\t%s\t%s\n",
		ipam.NodeID,
		ipam.NodeName,
		ipam.NodeIP,
		ipam.PodNetwork,
		ipam.VppHostNetwork,
		ipam.Config.PodIfIPCIDR,
		ipam.Config.PodSubnetCIRDR)
}

func getTabWriterAndPrintHeader() *tabwriter.Writer {
	w := tabwriter.NewWriter(os.Stdout, 0, 8, 2, ' ', 0)
	fmt.Fprintf(w, "ID\tNODE-NAME\tNODE-IP\tPOD-NET-IP\tVPP-HOST-IP\tPOD-IFIP-CIDR\tPOD-SUBNET-CIDR\n")
	return w
}
<|endoftext|>


<commit_before>/*
   Copyright The containerd Authors.

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
*/

package tasks

import (
	gocontext "context"
	"time"

	"github.com/containerd/console"
	"github.com/containerd/containerd"
	"github.com/containerd/containerd/cio"
	"github.com/containerd/containerd/log"
	"github.com/pkg/errors"
	"github.com/urfave/cli"
)

// HandleConsoleResize resizes the console
func HandleConsoleResize(ctx gocontext.Context, task resizer, con console.Console) error {
	// do an initial resize of the console
	size, err := con.Size()
	if err != nil {
		return err
	}
	go func() {
		prevSize := size
		for {
			time.Sleep(time.Millisecond * 250)

			size, err := con.Size()
			if err != nil {
				log.G(ctx).WithError(err).Error("get pty size")
				continue
			}

			if size.Width != prevSize.Width || size.Height != prevSize.Height {
				if err := task.Resize(ctx, uint32(size.Width), uint32(size.Height)); err != nil {
					log.G(ctx).WithError(err).Error("resize pty")
				}
				prevSize = size
			}
		}
	}()
	return nil
}

// NewTask creates a new task
func NewTask(ctx gocontext.Context, client *containerd.Client, container containerd.Container, _ string, con console.Console, nullIO bool, ioOpts []cio.Opt, opts ...containerd.NewTaskOpts) (containerd.Task, error) {
	var ioCreator cio.Creator
	if con != nil {
		if nullIO {
			return nil, errors.New("tty and null-io cannot be used together")
		}
		ioCreator = cio.NewCreator(append([]cio.Opt{cio.WithStreams(con, con, con), cio.WithTerminal}, ioOpts...)...)
	} else if nullIO {
		ioCreator = cio.NullIO
	} else {
		ioCreator = cio.NewCreator(append([]cio.Opt{cio.WithStdio}, ioOpts...)...)
	}
	return container.NewTask(ctx, ioCreator)
}

func getNewTaskOpts(_ *cli.Context) []containerd.NewTaskOpts {
	return nil
}
<commit_msg>Stop sending stderr with TTY on Windows<commit_after>/*
   Copyright The containerd Authors.

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
*/

package tasks

import (
	gocontext "context"
	"time"

	"github.com/containerd/console"
	"github.com/containerd/containerd"
	"github.com/containerd/containerd/cio"
	"github.com/containerd/containerd/log"
	"github.com/pkg/errors"
	"github.com/urfave/cli"
)

// HandleConsoleResize resizes the console
func HandleConsoleResize(ctx gocontext.Context, task resizer, con console.Console) error {
	// do an initial resize of the console
	size, err := con.Size()
	if err != nil {
		return err
	}
	go func() {
		prevSize := size
		for {
			time.Sleep(time.Millisecond * 250)

			size, err := con.Size()
			if err != nil {
				log.G(ctx).WithError(err).Error("get pty size")
				continue
			}

			if size.Width != prevSize.Width || size.Height != prevSize.Height {
				if err := task.Resize(ctx, uint32(size.Width), uint32(size.Height)); err != nil {
					log.G(ctx).WithError(err).Error("resize pty")
				}
				prevSize = size
			}
		}
	}()
	return nil
}

// NewTask creates a new task
func NewTask(ctx gocontext.Context, client *containerd.Client, container containerd.Container, _ string, con console.Console, nullIO bool, ioOpts []cio.Opt, opts ...containerd.NewTaskOpts) (containerd.Task, error) {
	var ioCreator cio.Creator
	if con != nil {
		if nullIO {
			return nil, errors.New("tty and null-io cannot be used together")
		}
		ioCreator = cio.NewCreator(append([]cio.Opt{cio.WithStreams(con, con, nil), cio.WithTerminal}, ioOpts...)...)
	} else if nullIO {
		ioCreator = cio.NullIO
	} else {
		ioCreator = cio.NewCreator(append([]cio.Opt{cio.WithStdio}, ioOpts...)...)
	}
	return container.NewTask(ctx, ioCreator)
}

func getNewTaskOpts(_ *cli.Context) []containerd.NewTaskOpts {
	return nil
}
<|endoftext|>


<commit_before>// Copyright 2019 The Kubeflow Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package app

import (
	"context"
	"fmt"
	"os"
	"time"

	"github.com/golang/glog"
	kubeflowScheme "github.com/kubeflow/mpi-operator/pkg/client/clientset/versioned/scheme"
	kubebatchclient "github.com/kubernetes-sigs/kube-batch/pkg/client/clientset/versioned"
	kubebatchinformers "github.com/kubernetes-sigs/kube-batch/pkg/client/informers/externalversions"
	podgroupsinformer "github.com/kubernetes-sigs/kube-batch/pkg/client/informers/externalversions/scheduling/v1alpha1"
	corev1 "k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/api/errors"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/util/uuid"
	kubeinformers "k8s.io/client-go/informers"
	kubeclientset "k8s.io/client-go/kubernetes"
	clientgokubescheme "k8s.io/client-go/kubernetes/scheme"
	v1core "k8s.io/client-go/kubernetes/typed/core/v1"
	restclientset "k8s.io/client-go/rest"
	"k8s.io/client-go/tools/clientcmd"
	election "k8s.io/client-go/tools/leaderelection"
	"k8s.io/client-go/tools/leaderelection/resourcelock"
	"k8s.io/client-go/tools/record"
	"k8s.io/sample-controller/pkg/signals"

	"github.com/kubeflow/mpi-operator/cmd/mpi-operator.v1alpha2/app/options"
	"github.com/kubeflow/mpi-operator/pkg/apis/kubeflow/v1alpha2"
	mpijobclientset "github.com/kubeflow/mpi-operator/pkg/client/clientset/versioned"
	informers "github.com/kubeflow/mpi-operator/pkg/client/informers/externalversions"
	controllersv1alpha2 "github.com/kubeflow/mpi-operator/pkg/controllers/v1alpha2"
	"github.com/kubeflow/mpi-operator/pkg/version"
)

const (
	apiVersion                   = "v1alpha2"
	RecommendedKubeConfigPathEnv = "KUBECONFIG"
)

var (
	// leader election config
	leaseDuration = 15 * time.Second
	renewDuration = 5 * time.Second
	retryPeriod   = 3 * time.Second
)

func Run(opt *options.ServerOption) error {
	// Check if the -version flag was passed and, if so, print the version and exit.
	if opt.PrintVersion {
		version.PrintVersionAndExit(apiVersion)
	}

	namespace := os.Getenv(v1alpha2.EnvKubeflowNamespace)
	if len(namespace) == 0 {
		glog.Infof("%s not set, use default namespace", v1alpha2.EnvKubeflowNamespace)
		namespace = metav1.NamespaceDefault
	}

	if opt.Namespace == corev1.NamespaceAll {
		glog.Info("Using cluster scoped operator")
	} else {
		glog.Infof("Scoping operator to namespace %s", opt.Namespace)
	}

	// To help debugging, immediately log version.
	glog.Infof("%+v", version.Info(apiVersion))

	// To help debugging, immediately log opts.
	glog.Infof("Server options: %+v", opt)

	// set up signals so we handle the first shutdown signal gracefully
	stopCh := signals.SetupSignalHandler()

	// Note: ENV KUBECONFIG will overwrite user defined Kubeconfig option.
	if len(os.Getenv(RecommendedKubeConfigPathEnv)) > 0 {
		// use the current context in kubeconfig
		// This is very useful for running locally.
		opt.Kubeconfig = os.Getenv(RecommendedKubeConfigPathEnv)
	}

	cfg, err := clientcmd.BuildConfigFromFlags(opt.MasterURL, opt.Kubeconfig)
	if err != nil {
		glog.Fatalf("Error building kubeConfig: %s", err.Error())
	}

	// Create clients.
	kubeClient, leaderElectionClientSet, mpiJobClientSet, kubeBatchClientSet, err := createClientSets(cfg)
	if err != nil {
		return err
	}
	if !checkCRDExists(mpiJobClientSet, opt.Namespace) {
		glog.Info("CRD doesn't exist. Exiting")
		os.Exit(1)
	}

	// Add mpi-job-controller types to the default Kubernetes Scheme so Events
	// can be logged for mpi-job-controller types.
	err = kubeflowScheme.AddToScheme(clientgokubescheme.Scheme)
	if err != nil {
		return fmt.Errorf("CoreV1 Add Scheme failed: %v", err)
	}

	// Set leader election start function.
	run := func(ctx context.Context) {
		var kubeInformerFactory kubeinformers.SharedInformerFactory
		var kubeflowInformerFactory informers.SharedInformerFactory
		var kubebatchInformerFactory kubebatchinformers.SharedInformerFactory
		if opt.Namespace == "" {
			kubeInformerFactory = kubeinformers.NewSharedInformerFactory(kubeClient, 0)
			kubeflowInformerFactory = informers.NewSharedInformerFactory(mpiJobClientSet, 0)
			kubebatchInformerFactory = kubebatchinformers.NewSharedInformerFactory(kubeBatchClientSet, 0)
		} else {
			kubeInformerFactory = kubeinformers.NewSharedInformerFactoryWithOptions(kubeClient, 0, kubeinformers.WithNamespace(opt.Namespace))
			kubeflowInformerFactory = informers.NewSharedInformerFactoryWithOptions(mpiJobClientSet, 0, informers.WithNamespace(opt.Namespace))
			kubebatchInformerFactory = kubebatchinformers.NewSharedInformerFactoryWithOptions(kubeBatchClientSet, 0, kubebatchinformers.WithNamespace(opt.Namespace))
		}

		var podgroupsInformer podgroupsinformer.PodGroupInformer
		if opt.EnableGangScheduling {
			podgroupsInformer = kubebatchInformerFactory.Scheduling().V1alpha1().PodGroups()
		}
		controller := controllersv1alpha2.NewMPIJobController(
			kubeClient,
			mpiJobClientSet,
			kubeBatchClientSet,
			kubeInformerFactory.Core().V1().ConfigMaps(),
			kubeInformerFactory.Core().V1().ServiceAccounts(),
			kubeInformerFactory.Rbac().V1().Roles(),
			kubeInformerFactory.Rbac().V1().RoleBindings(),
			kubeInformerFactory.Apps().V1().StatefulSets(),
			kubeInformerFactory.Batch().V1().Jobs(),
			podgroupsInformer,
			kubeflowInformerFactory.Kubeflow().V1alpha2().MPIJobs(),
			opt.KubectlDeliveryImage,
			opt.EnableGangScheduling)

		go kubeInformerFactory.Start(ctx.Done())
		go kubeflowInformerFactory.Start(ctx.Done())
		if opt.EnableGangScheduling {
			go kubebatchInformerFactory.Start(ctx.Done())
		}

		if err = controller.Run(opt.Threadiness, stopCh); err != nil {
			glog.Fatalf("Error running controller: %s", err.Error())
		}
	}

	id, err := os.Hostname()
	if err != nil {
		return fmt.Errorf("failed to get hostname: %v", err)
	}
	// add a uniquifier so that two processes on the same host don't accidentally both become active
	id = id + "_" + string(uuid.NewUUID())

	// Prepare event clients.
	eventBroadcaster := record.NewBroadcaster()
	eventBroadcaster.StartLogging(glog.Infof)
	eventBroadcaster.StartRecordingToSink(&v1core.EventSinkImpl{Interface: kubeClient.CoreV1().Events("")})
	recorder := eventBroadcaster.NewRecorder(clientgokubescheme.Scheme, corev1.EventSource{Component: "mpi-operator"})

	rl := &resourcelock.EndpointsLock{
		EndpointsMeta: metav1.ObjectMeta{
			Namespace: namespace,
			Name:      "mpi-operator",
		},
		Client: leaderElectionClientSet.CoreV1(),
		LockConfig: resourcelock.ResourceLockConfig{
			Identity:      id,
			EventRecorder: recorder,
		},
	}

	ctx, cancel := context.WithCancel(context.TODO())
	defer cancel()

	go func() {
		select {
		case <-stopCh:
			cancel()
		case <-ctx.Done():
		}
	}()

	// Start leader election.
	election.RunOrDie(ctx, election.LeaderElectionConfig{
		Lock:          rl,
		LeaseDuration: leaseDuration,
		RenewDeadline: renewDuration,
		RetryPeriod:   retryPeriod,
		Callbacks: election.LeaderCallbacks{
			OnStartedLeading: run,
			OnStoppedLeading: func() {
				glog.Fatalf("leader election lost")
			},
		},
		Name: "mpi-operator",
	})

	return fmt.Errorf("finished without leader elect")
}

func createClientSets(config *restclientset.Config) (kubeclientset.Interface, kubeclientset.Interface, mpijobclientset.Interface, kubebatchclient.Interface, error) {

	kubeClientSet, err := kubeclientset.NewForConfig(restclientset.AddUserAgent(config, "mpi-operator"))
	if err != nil {
		return nil, nil, nil, nil, err
	}

	leaderElectionClientSet, err := kubeclientset.NewForConfig(restclientset.AddUserAgent(config, "leader-election"))
	if err != nil {
		return nil, nil, nil, nil, err
	}

	mpiJobClientSet, err := mpijobclientset.NewForConfig(config)
	if err != nil {
		return nil, nil, nil, nil, err
	}

	kubeBatchClientSet, err := kubebatchclient.NewForConfig(restclientset.AddUserAgent(config, "kube-batch"))
	if err != nil {
		return nil, nil, nil, nil, err
	}

	return kubeClientSet, leaderElectionClientSet, mpiJobClientSet, kubeBatchClientSet, nil
}

func checkCRDExists(clientset mpijobclientset.Interface, namespace string) bool {
	_, err := clientset.KubeflowV1alpha2().MPIJobs(namespace).List(metav1.ListOptions{})

	if err != nil {
		glog.Error(err)
		if _, ok := err.(*errors.StatusError); ok {
			if errors.IsNotFound(err) {
				return false
			}
		}
	}
	return true
}
<commit_msg>Add callbacks around additional leader election phases (#173)<commit_after>// Copyright 2019 The Kubeflow Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package app

import (
	"context"
	"fmt"
	"os"
	"time"

	"github.com/golang/glog"
	kubeflowScheme "github.com/kubeflow/mpi-operator/pkg/client/clientset/versioned/scheme"
	kubebatchclient "github.com/kubernetes-sigs/kube-batch/pkg/client/clientset/versioned"
	kubebatchinformers "github.com/kubernetes-sigs/kube-batch/pkg/client/informers/externalversions"
	podgroupsinformer "github.com/kubernetes-sigs/kube-batch/pkg/client/informers/externalversions/scheduling/v1alpha1"
	corev1 "k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/api/errors"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/util/uuid"
	kubeinformers "k8s.io/client-go/informers"
	kubeclientset "k8s.io/client-go/kubernetes"
	clientgokubescheme "k8s.io/client-go/kubernetes/scheme"
	v1core "k8s.io/client-go/kubernetes/typed/core/v1"
	restclientset "k8s.io/client-go/rest"
	"k8s.io/client-go/tools/clientcmd"
	election "k8s.io/client-go/tools/leaderelection"
	"k8s.io/client-go/tools/leaderelection/resourcelock"
	"k8s.io/client-go/tools/record"
	"k8s.io/sample-controller/pkg/signals"

	"github.com/kubeflow/mpi-operator/cmd/mpi-operator.v1alpha2/app/options"
	"github.com/kubeflow/mpi-operator/pkg/apis/kubeflow/v1alpha2"
	mpijobclientset "github.com/kubeflow/mpi-operator/pkg/client/clientset/versioned"
	informers "github.com/kubeflow/mpi-operator/pkg/client/informers/externalversions"
	controllersv1alpha2 "github.com/kubeflow/mpi-operator/pkg/controllers/v1alpha2"
	"github.com/kubeflow/mpi-operator/pkg/version"
)

const (
	apiVersion                   = "v1alpha2"
	RecommendedKubeConfigPathEnv = "KUBECONFIG"
)

var (
	// leader election config
	leaseDuration = 15 * time.Second
	renewDuration = 5 * time.Second
	retryPeriod   = 3 * time.Second
)

func Run(opt *options.ServerOption) error {
	// Check if the -version flag was passed and, if so, print the version and exit.
	if opt.PrintVersion {
		version.PrintVersionAndExit(apiVersion)
	}

	namespace := os.Getenv(v1alpha2.EnvKubeflowNamespace)
	if len(namespace) == 0 {
		glog.Infof("%s not set, use default namespace", v1alpha2.EnvKubeflowNamespace)
		namespace = metav1.NamespaceDefault
	}

	if opt.Namespace == corev1.NamespaceAll {
		glog.Info("Using cluster scoped operator")
	} else {
		glog.Infof("Scoping operator to namespace %s", opt.Namespace)
	}

	// To help debugging, immediately log version.
	glog.Infof("%+v", version.Info(apiVersion))

	// To help debugging, immediately log opts.
	glog.Infof("Server options: %+v", opt)

	// set up signals so we handle the first shutdown signal gracefully
	stopCh := signals.SetupSignalHandler()

	// Note: ENV KUBECONFIG will overwrite user defined Kubeconfig option.
	if len(os.Getenv(RecommendedKubeConfigPathEnv)) > 0 {
		// use the current context in kubeconfig
		// This is very useful for running locally.
		opt.Kubeconfig = os.Getenv(RecommendedKubeConfigPathEnv)
	}

	cfg, err := clientcmd.BuildConfigFromFlags(opt.MasterURL, opt.Kubeconfig)
	if err != nil {
		glog.Fatalf("Error building kubeConfig: %s", err.Error())
	}

	// Create clients.
	kubeClient, leaderElectionClientSet, mpiJobClientSet, kubeBatchClientSet, err := createClientSets(cfg)
	if err != nil {
		return err
	}
	if !checkCRDExists(mpiJobClientSet, opt.Namespace) {
		glog.Info("CRD doesn't exist. Exiting")
		os.Exit(1)
	}

	// Add mpi-job-controller types to the default Kubernetes Scheme so Events
	// can be logged for mpi-job-controller types.
	err = kubeflowScheme.AddToScheme(clientgokubescheme.Scheme)
	if err != nil {
		return fmt.Errorf("CoreV1 Add Scheme failed: %v", err)
	}

	// Set leader election start function.
	run := func(ctx context.Context) {
		var kubeInformerFactory kubeinformers.SharedInformerFactory
		var kubeflowInformerFactory informers.SharedInformerFactory
		var kubebatchInformerFactory kubebatchinformers.SharedInformerFactory
		if opt.Namespace == "" {
			kubeInformerFactory = kubeinformers.NewSharedInformerFactory(kubeClient, 0)
			kubeflowInformerFactory = informers.NewSharedInformerFactory(mpiJobClientSet, 0)
			kubebatchInformerFactory = kubebatchinformers.NewSharedInformerFactory(kubeBatchClientSet, 0)
		} else {
			kubeInformerFactory = kubeinformers.NewSharedInformerFactoryWithOptions(kubeClient, 0, kubeinformers.WithNamespace(opt.Namespace))
			kubeflowInformerFactory = informers.NewSharedInformerFactoryWithOptions(mpiJobClientSet, 0, informers.WithNamespace(opt.Namespace))
			kubebatchInformerFactory = kubebatchinformers.NewSharedInformerFactoryWithOptions(kubeBatchClientSet, 0, kubebatchinformers.WithNamespace(opt.Namespace))
		}

		var podgroupsInformer podgroupsinformer.PodGroupInformer
		if opt.EnableGangScheduling {
			podgroupsInformer = kubebatchInformerFactory.Scheduling().V1alpha1().PodGroups()
		}
		controller := controllersv1alpha2.NewMPIJobController(
			kubeClient,
			mpiJobClientSet,
			kubeBatchClientSet,
			kubeInformerFactory.Core().V1().ConfigMaps(),
			kubeInformerFactory.Core().V1().ServiceAccounts(),
			kubeInformerFactory.Rbac().V1().Roles(),
			kubeInformerFactory.Rbac().V1().RoleBindings(),
			kubeInformerFactory.Apps().V1().StatefulSets(),
			kubeInformerFactory.Batch().V1().Jobs(),
			podgroupsInformer,
			kubeflowInformerFactory.Kubeflow().V1alpha2().MPIJobs(),
			opt.KubectlDeliveryImage,
			opt.EnableGangScheduling)

		go kubeInformerFactory.Start(ctx.Done())
		go kubeflowInformerFactory.Start(ctx.Done())
		if opt.EnableGangScheduling {
			go kubebatchInformerFactory.Start(ctx.Done())
		}

		if err = controller.Run(opt.Threadiness, stopCh); err != nil {
			glog.Fatalf("Error running controller: %s", err.Error())
		}
	}

	id, err := os.Hostname()
	if err != nil {
		return fmt.Errorf("failed to get hostname: %v", err)
	}
	// add a uniquifier so that two processes on the same host don't accidentally both become active
	id = id + "_" + string(uuid.NewUUID())

	// Prepare event clients.
	eventBroadcaster := record.NewBroadcaster()
	eventBroadcaster.StartLogging(glog.Infof)
	eventBroadcaster.StartRecordingToSink(&v1core.EventSinkImpl{Interface: kubeClient.CoreV1().Events("")})
	recorder := eventBroadcaster.NewRecorder(clientgokubescheme.Scheme, corev1.EventSource{Component: "mpi-operator"})

	rl := &resourcelock.EndpointsLock{
		EndpointsMeta: metav1.ObjectMeta{
			Namespace: namespace,
			Name:      "mpi-operator",
		},
		Client: leaderElectionClientSet.CoreV1(),
		LockConfig: resourcelock.ResourceLockConfig{
			Identity:      id,
			EventRecorder: recorder,
		},
	}

	ctx, cancel := context.WithCancel(context.TODO())
	defer cancel()

	go func() {
		select {
		case <-stopCh:
			cancel()
		case <-ctx.Done():
		}
	}()

	// Start leader election.
	election.RunOrDie(ctx, election.LeaderElectionConfig{
		Lock:          rl,
		LeaseDuration: leaseDuration,
		RenewDeadline: renewDuration,
		RetryPeriod:   retryPeriod,
		Callbacks: election.LeaderCallbacks{
			OnStartedLeading: func(ctx context.Context) {
				glog.Infof("Leading started")
				run(ctx)
			},
			OnStoppedLeading: func() {
				glog.Fatalf("Leader election stopped")
			},
			OnNewLeader: func(identity string) {
				if identity == id {
					return
				}
				glog.Infof("New leader has been elected: %s", identity)
			},
		},
		Name: "mpi-operator",
	})

	return fmt.Errorf("finished without leader elect")
}

func createClientSets(config *restclientset.Config) (kubeclientset.Interface, kubeclientset.Interface, mpijobclientset.Interface, kubebatchclient.Interface, error) {

	kubeClientSet, err := kubeclientset.NewForConfig(restclientset.AddUserAgent(config, "mpi-operator"))
	if err != nil {
		return nil, nil, nil, nil, err
	}

	leaderElectionClientSet, err := kubeclientset.NewForConfig(restclientset.AddUserAgent(config, "leader-election"))
	if err != nil {
		return nil, nil, nil, nil, err
	}

	mpiJobClientSet, err := mpijobclientset.NewForConfig(config)
	if err != nil {
		return nil, nil, nil, nil, err
	}

	kubeBatchClientSet, err := kubebatchclient.NewForConfig(restclientset.AddUserAgent(config, "kube-batch"))
	if err != nil {
		return nil, nil, nil, nil, err
	}

	return kubeClientSet, leaderElectionClientSet, mpiJobClientSet, kubeBatchClientSet, nil
}

func checkCRDExists(clientset mpijobclientset.Interface, namespace string) bool {
	_, err := clientset.KubeflowV1alpha2().MPIJobs(namespace).List(metav1.ListOptions{})

	if err != nil {
		glog.Error(err)
		if _, ok := err.(*errors.StatusError); ok {
			if errors.IsNotFound(err) {
				return false
			}
		}
	}
	return true
}
<|endoftext|>


<commit_before>// Copyright 2013 Joyent Inc.
// Licensed under the AGPLv3, see LICENCE file for details.

package joyent

import (
	"bytes"
	"fmt"
	"text/template"
	"time"

	"github.com/joyent/gosign/auth"
	jc "github.com/juju/testing/checkers"
	"github.com/juju/utils"
	gc "gopkg.in/check.v1"

	"github.com/juju/juju/constraints"
	"github.com/juju/juju/environs"
	"github.com/juju/juju/environs/config"
	"github.com/juju/juju/environs/configstore"
	"github.com/juju/juju/environs/imagemetadata"
	"github.com/juju/juju/environs/instances"
	"github.com/juju/juju/environs/jujutest"
	"github.com/juju/juju/environs/storage"
	envtesting "github.com/juju/juju/environs/testing"
	"github.com/juju/juju/testing"
)

// Use ShortAttempt to poll for short-term events.
var ShortAttempt = utils.AttemptStrategy{
	Total: 5 * time.Second,
	Delay: 200 * time.Millisecond,
}

var Provider environs.EnvironProvider = GetProviderInstance()
var EnvironmentVariables = environmentVariables

var indexData = `
		{
		 "index": {
		  "com.ubuntu.cloud:released:joyent": {
		   "updated": "Fri, 14 Feb 2014 13:39:35 +0000",
		   "clouds": [
			{
			 "region": "{{.Region}}",
			 "endpoint": "{{.SdcEndpoint.URL}}"
			}
		   ],
		   "cloudname": "joyent",
		   "datatype": "image-ids",
		   "format": "products:1.0",
		   "products": [
			"com.ubuntu.cloud:server:14.04:amd64",
			"com.ubuntu.cloud:server:12.10:amd64",
			"com.ubuntu.cloud:server:13.04:amd64"
		   ],
		   "path": "streams/v1/com.ubuntu.cloud:released:joyent.json"
		  }
		 },
		 "updated": "Fri, 14 Feb 2014 13:39:35 +0000",
		 "format": "index:1.0"
		}
`

var imagesData = `
{
  "content_id": "com.ubuntu.cloud:released:joyent",
  "format": "products:1.0",
  "updated": "Fri, 14 Feb 2014 13:39:35 +0000",
  "datatype": "image-ids",
  "products": {
    "com.ubuntu.cloud:server:14.04:amd64": {
      "release": "trusty",
      "version": "14.04",
      "arch": "amd64",
      "versions": {
        "20140214": {
          "items": {
            "11223344-0a0a-ff99-11bb-0a1b2c3d4e5f": {
              "region": "some-region",
              "id": "11223344-0a0a-ff99-11bb-0a1b2c3d4e5f",
              "virt": "kvm"
            }
          },
          "pubname": "ubuntu-trusty-14.04-amd64-server-20140214",
          "label": "release"
        }
      }
    },
    "com.ubuntu.cloud:server:12.10:amd64": {
      "release": "quantal",
      "version": "12.10",
      "arch": "amd64",
      "versions": {
        "20140214": {
          "items": {
            "11223344-0a0a-ee88-22ab-00aa11bb22cc": {
              "region": "some-region",
              "id": "11223344-0a0a-ee88-22ab-00aa11bb22cc",
              "virt": "kvm"
            }
          },
          "pubname": "ubuntu-quantal-12.10-amd64-server-20140214",
          "label": "release"
        }
      }
    },
    "com.ubuntu.cloud:server:13.04:amd64": {
      "release": "raring",
      "version": "13.04",
      "arch": "amd64",
      "versions": {
        "20140214": {
          "items": {
            "11223344-0a0a-dd77-33cd-abcd1234e5f6": {
              "region": "some-region",
              "id": "11223344-0a0a-dd77-33cd-abcd1234e5f6",
              "virt": "kvm"
            }
          },
          "pubname": "ubuntu-raring-13.04-amd64-server-20140214",
          "label": "release"
        }
      }
    }
  }
}
`

func parseIndexData(creds *auth.Credentials) bytes.Buffer {
	var metadata bytes.Buffer

	t := template.Must(template.New("").Parse(indexData))
	if err := t.Execute(&metadata, creds); err != nil {
		panic(fmt.Errorf("cannot generate index metdata: %v", err))
	}

	return metadata
}

// This provides the content for code accessing test://host/... URLs. This allows
// us to set the responses for things like the Metadata server, by pointing
// metadata requests at test://host/...
var testRoundTripper = &jujutest.ProxyRoundTripper{}

func init() {
	testRoundTripper.RegisterForScheme("test")
}

var origImagesUrl = imagemetadata.DefaultBaseURL

// Set Metadata requests to be served by the filecontent supplied.
func UseExternalTestImageMetadata(creds *auth.Credentials) {
	metadata := parseIndexData(creds)
	files := map[string]string{
		"/streams/v1/index.json":                            metadata.String(),
		"/streams/v1/com.ubuntu.cloud:released:joyent.json": imagesData,
	}
	testRoundTripper.Sub = jujutest.NewCannedRoundTripper(files, nil)
	imagemetadata.DefaultBaseURL = "test://host"
}

func UnregisterExternalTestImageMetadata() {
	testRoundTripper.Sub = nil
	imagemetadata.DefaultBaseURL = origImagesUrl
}

// RegisterMachinesEndpoint creates a fake endpoint so that
// machines api calls succeed.
func RegisterMachinesEndpoint() {
	files := map[string]string{
		"/test/machines": "",
	}
	testRoundTripper.Sub = jujutest.NewCannedRoundTripper(files, nil)
}

// UnregisterMachinesEndpoint resets the machines endpoint.
func UnregisterMachinesEndpoint() {
	testRoundTripper.Sub = nil
}

func FindInstanceSpec(e environs.Environ, series, arch, cons string) (spec *instances.InstanceSpec, err error) {
	env := e.(*joyentEnviron)
	spec, err = env.FindInstanceSpec(&instances.InstanceConstraint{
		Series:      series,
		Arches:      []string{arch},
		Region:      env.Ecfg().Region(),
		Constraints: constraints.MustParse(cons),
	})
	return
}

func ControlBucketName(e environs.Environ) string {
	env := e.(*joyentEnviron)
	return env.Storage().(*JoyentStorage).GetContainerName()
}

func CreateContainer(s *JoyentStorage) error {
	return s.createContainer()
}

// MakeEnviron creates a functional Environ for a test.
func MakeEnviron(c *gc.C, attrs testing.Attrs) environs.Environ {
	cfg, err := config.New(config.NoDefaults, attrs)
	c.Assert(err, jc.ErrorIsNil)
	env, err := environs.Prepare(cfg, envtesting.BootstrapContext(c), configstore.NewMem())
	c.Assert(err, jc.ErrorIsNil)
	return env
}

// MakeConfig creates a functional environConfig for a test.
func MakeConfig(c *gc.C, attrs testing.Attrs) *environConfig {
	return MakeEnviron(c, attrs).(*joyentEnviron).Ecfg()
}

// MakeCredentials creates credentials for a test.
func MakeCredentials(c *gc.C, attrs testing.Attrs) *auth.Credentials {
	creds, err := credentials(MakeConfig(c, attrs))
	c.Assert(err, jc.ErrorIsNil)
	return creds
}

// MakeStorage creates an env storage for a test.
func MakeStorage(c *gc.C, attrs testing.Attrs) storage.Storage {
	stor, err := newStorage(MakeConfig(c, attrs), "")
	c.Assert(err, jc.ErrorIsNil)
	return stor
}

var GetPorts = getPorts

var CreateFirewallRuleAll = createFirewallRuleAll

var CreateFirewallRuleVm = createFirewallRuleVm
<commit_msg>Revert unnecessary refactoring.<commit_after>// Copyright 2013 Joyent Inc.
// Licensed under the AGPLv3, see LICENCE file for details.

package joyent

import (
	"bytes"
	"fmt"
	"text/template"
	"time"

	"github.com/joyent/gosign/auth"
	jc "github.com/juju/testing/checkers"
	"github.com/juju/utils"
	gc "gopkg.in/check.v1"

	"github.com/juju/juju/constraints"
	"github.com/juju/juju/environs"
	"github.com/juju/juju/environs/config"
	"github.com/juju/juju/environs/configstore"
	"github.com/juju/juju/environs/imagemetadata"
	"github.com/juju/juju/environs/instances"
	"github.com/juju/juju/environs/jujutest"
	"github.com/juju/juju/environs/storage"
	envtesting "github.com/juju/juju/environs/testing"
	"github.com/juju/juju/testing"
)

// Use ShortAttempt to poll for short-term events.
var ShortAttempt = utils.AttemptStrategy{
	Total: 5 * time.Second,
	Delay: 200 * time.Millisecond,
}

var Provider environs.EnvironProvider = GetProviderInstance()
var EnvironmentVariables = environmentVariables

var indexData = `
		{
		 "index": {
		  "com.ubuntu.cloud:released:joyent": {
		   "updated": "Fri, 14 Feb 2014 13:39:35 +0000",
		   "clouds": [
			{
			 "region": "{{.Region}}",
			 "endpoint": "{{.SdcEndpoint.URL}}"
			}
		   ],
		   "cloudname": "joyent",
		   "datatype": "image-ids",
		   "format": "products:1.0",
		   "products": [
			"com.ubuntu.cloud:server:14.04:amd64",
			"com.ubuntu.cloud:server:12.10:amd64",
			"com.ubuntu.cloud:server:13.04:amd64"
		   ],
		   "path": "streams/v1/com.ubuntu.cloud:released:joyent.json"
		  }
		 },
		 "updated": "Fri, 14 Feb 2014 13:39:35 +0000",
		 "format": "index:1.0"
		}
`

var imagesData = `
{
  "content_id": "com.ubuntu.cloud:released:joyent",
  "format": "products:1.0",
  "updated": "Fri, 14 Feb 2014 13:39:35 +0000",
  "datatype": "image-ids",
  "products": {
    "com.ubuntu.cloud:server:14.04:amd64": {
      "release": "trusty",
      "version": "14.04",
      "arch": "amd64",
      "versions": {
        "20140214": {
          "items": {
            "11223344-0a0a-ff99-11bb-0a1b2c3d4e5f": {
              "region": "some-region",
              "id": "11223344-0a0a-ff99-11bb-0a1b2c3d4e5f",
              "virt": "kvm"
            }
          },
          "pubname": "ubuntu-trusty-14.04-amd64-server-20140214",
          "label": "release"
        }
      }
    },
    "com.ubuntu.cloud:server:12.10:amd64": {
      "release": "quantal",
      "version": "12.10",
      "arch": "amd64",
      "versions": {
        "20140214": {
          "items": {
            "11223344-0a0a-ee88-22ab-00aa11bb22cc": {
              "region": "some-region",
              "id": "11223344-0a0a-ee88-22ab-00aa11bb22cc",
              "virt": "kvm"
            }
          },
          "pubname": "ubuntu-quantal-12.10-amd64-server-20140214",
          "label": "release"
        }
      }
    },
    "com.ubuntu.cloud:server:13.04:amd64": {
      "release": "raring",
      "version": "13.04",
      "arch": "amd64",
      "versions": {
        "20140214": {
          "items": {
            "11223344-0a0a-dd77-33cd-abcd1234e5f6": {
              "region": "some-region",
              "id": "11223344-0a0a-dd77-33cd-abcd1234e5f6",
              "virt": "kvm"
            }
          },
          "pubname": "ubuntu-raring-13.04-amd64-server-20140214",
          "label": "release"
        }
      }
    }
  }
}
`

func parseIndexData(creds *auth.Credentials) bytes.Buffer {
	var metadata bytes.Buffer

	t := template.Must(template.New("").Parse(indexData))
	if err := t.Execute(&metadata, creds); err != nil {
		panic(fmt.Errorf("cannot generate index metdata: %v", err))
	}

	return metadata
}

// This provides the content for code accessing test://host/... URLs. This allows
// us to set the responses for things like the Metadata server, by pointing
// metadata requests at test://host/...
var testRoundTripper = &jujutest.ProxyRoundTripper{}

func init() {
	testRoundTripper.RegisterForScheme("test")
}

var origImagesUrl = imagemetadata.DefaultBaseURL

// Set Metadata requests to be served by the filecontent supplied.
func UseExternalTestImageMetadata(creds *auth.Credentials) {
	metadata := parseIndexData(creds)
	files := map[string]string{
		"/streams/v1/index.json":                            metadata.String(),
		"/streams/v1/com.ubuntu.cloud:released:joyent.json": imagesData,
	}
	testRoundTripper.Sub = jujutest.NewCannedRoundTripper(files, nil)
	imagemetadata.DefaultBaseURL = "test://host"
}

func UnregisterExternalTestImageMetadata() {
	testRoundTripper.Sub = nil
	imagemetadata.DefaultBaseURL = origImagesUrl
}

// RegisterMachinesEndpoint creates a fake endpoint so that
// machines api calls succeed.
func RegisterMachinesEndpoint() {
	files := map[string]string{
		"/test/machines": "",
	}
	testRoundTripper.Sub = jujutest.NewCannedRoundTripper(files, nil)
}

// UnregisterMachinesEndpoint resets the machines endpoint.
func UnregisterMachinesEndpoint() {
	testRoundTripper.Sub = nil
}

func FindInstanceSpec(e environs.Environ, series, arch, cons string) (spec *instances.InstanceSpec, err error) {
	env := e.(*joyentEnviron)
	spec, err = env.FindInstanceSpec(&instances.InstanceConstraint{
		Series:      series,
		Arches:      []string{arch},
		Region:      env.Ecfg().Region(),
		Constraints: constraints.MustParse(cons),
	})
	return
}

func ControlBucketName(e environs.Environ) string {
	env := e.(*joyentEnviron)
	return env.Storage().(*JoyentStorage).GetContainerName()
}

func CreateContainer(s *JoyentStorage) error {
	return s.createContainer()
}

// MakeConfig creates a functional environConfig for a test.
func MakeConfig(c *gc.C, attrs testing.Attrs) *environConfig {
	cfg, err := config.New(config.NoDefaults, attrs)
	c.Assert(err, jc.ErrorIsNil)
	env, err := environs.Prepare(cfg, envtesting.BootstrapContext(c), configstore.NewMem())
	c.Assert(err, jc.ErrorIsNil)
	return env.(*joyentEnviron).Ecfg()
}

// MakeCredentials creates credentials for a test.
func MakeCredentials(c *gc.C, attrs testing.Attrs) *auth.Credentials {
	creds, err := credentials(MakeConfig(c, attrs))
	c.Assert(err, jc.ErrorIsNil)
	return creds
}

// MakeStorage creates an env storage for a test.
func MakeStorage(c *gc.C, attrs testing.Attrs) storage.Storage {
	stor, err := newStorage(MakeConfig(c, attrs), "")
	c.Assert(err, jc.ErrorIsNil)
	return stor
}

var GetPorts = getPorts

var CreateFirewallRuleAll = createFirewallRuleAll

var CreateFirewallRuleVm = createFirewallRuleVm
<|endoftext|>


<commit_before>package twitchtv

import (
	"fmt"
	"net/url"

	"github.com/zquestz/s/providers"
)

func init() {
	providers.AddProvider("twitchtv", &Provider{})
}

// Provider merely implements the Provider interface.
type Provider struct{}

// BuildURI generates a search URL for Twitchtv.
func (p *Provider) BuildURI(q string) string {
	return fmt.Sprintf("https://www.twitch.tv/search?query=%s", url.QueryEscape(q))
}

// Tags returns the tags relevant to this provider.
func (p *Provider) Tags() []string {
	return []string{}
}
<commit_msg>TwitchTv: Fix search query parameter (#157)<commit_after>package twitchtv

import (
	"fmt"
	"net/url"

	"github.com/zquestz/s/providers"
)

func init() {
	providers.AddProvider("twitchtv", &Provider{})
}

// Provider merely implements the Provider interface.
type Provider struct{}

// BuildURI generates a search URL for Twitchtv.
func (p *Provider) BuildURI(q string) string {
	return fmt.Sprintf("https://www.twitch.tv/search?term=%s", url.QueryEscape(q))
}

// Tags returns the tags relevant to this provider.
func (p *Provider) Tags() []string {
	return []string{}
}
<|endoftext|>


<commit_before>// Copyright 2013 tsuru authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package docker

import (
	"github.com/globocom/config"
	"github.com/globocom/tsuru/db"
	"launchpad.net/gocheck"
	"testing"
)

func Test(t *testing.T) { gocheck.TestingT(t) }

type S struct {
	collName      string
	imageCollName string
	conn          *db.Storage
	gitHost       string
	repoNamespace string
}

var _ = gocheck.Suite(&S{})

func (s *S) SetUpSuite(c *gocheck.C) {
	s.collName = "docker_unit"
	s.imageCollName = "docker_image"
	s.gitHost = "my.gandalf.com"
	s.repoNamespace = "tsuru"
	config.Set("git:host", s.gitHost)
	config.Set("docker:repository-namespace", s.repoNamespace)
	config.Set("docker:binary", "docker")
	config.Set("router", "fake")
	config.Set("docker:collection", s.collName)
	config.Set("database:url", "127.0.0.1:27017")
	config.Set("database:name", "juju_provision_tests_s")
	config.Set("docker:authorized-key-path", "somepath")
	config.Set("docker:image", "base")
	config.Set("docker:deploy-cmd", "/var/lib/tsuru/deploy")
	var err error
	s.conn, err = db.Conn()
	c.Assert(err, gocheck.IsNil)
}

func (s *S) TearDownSuite(c *gocheck.C) {
	s.conn.Collection(s.collName).Database.DropDatabase()
}
<commit_msg>provision/docker/suite_test: added needed configurations to run a container in SetUpSuite<commit_after>// Copyright 2013 tsuru authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package docker

import (
	"github.com/globocom/config"
	"github.com/globocom/tsuru/db"
	"launchpad.net/gocheck"
	"testing"
)

func Test(t *testing.T) { gocheck.TestingT(t) }

type S struct {
	collName      string
	imageCollName string
	conn          *db.Storage
	gitHost       string
	repoNamespace string
}

var _ = gocheck.Suite(&S{})

func (s *S) SetUpSuite(c *gocheck.C) {
	s.collName = "docker_unit"
	s.imageCollName = "docker_image"
	s.gitHost = "my.gandalf.com"
	s.repoNamespace = "tsuru"
	config.Set("git:host", s.gitHost)
	config.Set("docker:repository-namespace", s.repoNamespace)
	config.Set("docker:binary", "docker")
	config.Set("router", "fake")
	config.Set("docker:collection", s.collName)
	config.Set("database:url", "127.0.0.1:27017")
	config.Set("database:name", "juju_provision_tests_s")
	config.Set("docker:authorized-key-path", "somepath")
	config.Set("docker:image", "base")
	config.Set("docker:deploy-cmd", "/var/lib/tsuru/deploy")
	config.Set("docker:run-cmd:bin", "/usr/local/bin/circusd")
	config.Set("docker:run-cmd:args", "/etc/circus/circus.ini")
	config.Set("docker:run-cmd:port", "8888")
	var err error
	s.conn, err = db.Conn()
	c.Assert(err, gocheck.IsNil)
}

func (s *S) TearDownSuite(c *gocheck.C) {
	s.conn.Collection(s.collName).Database.DropDatabase()
}
<|endoftext|>


<commit_before>package shared_test

import (
	"errors"
	"time"

	"code.cloudfoundry.org/cli/actor/v3action"
	"code.cloudfoundry.org/cli/api/cloudcontroller/ccv3"
	"code.cloudfoundry.org/cli/command/translatableerror"
	. "code.cloudfoundry.org/cli/command/v3/shared"
	"code.cloudfoundry.org/cli/util/ui"
	"code.cloudfoundry.org/clock/fakeclock"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	. "github.com/onsi/gomega/gbytes"
)

var _ = FDescribe("V3PollStage", func() {
	var (
		appName             string
		returnedDropletGUID string
		executeErr          error
		testUI              *ui.UI
		buildStream         chan v3action.Build
		warningsStream      chan v3action.Warnings
		errStream           chan error
		logStream           chan *v3action.LogMessage
		logErrStream        chan error
		fakeClock           *fakeclock.FakeClock
		// done                chan struct{}
		closeStreams     func()
		executePollStage func(func())
		blockOnExecute   chan bool
		blockOnChannels  chan bool
	)

	BeforeEach(func() {
		executeErr = nil
		returnedDropletGUID = ""

		testUI = ui.NewTestUI(nil, NewBuffer(), NewBuffer())
		fakeClock = fakeclock.NewFakeClock(time.Now())
		buildStream = make(chan v3action.Build, 1)
		warningsStream = make(chan v3action.Warnings, 1)
		errStream = make(chan error, 1)
		logStream = make(chan *v3action.LogMessage, 1)
		logErrStream = make(chan error, 1)
		appName = "some-app"
		blockOnExecute = make(chan bool)
		blockOnChannels = make(chan bool)

		closeStreams = func() {
			close(buildStream)
			close(warningsStream)
			close(errStream)
			close(logStream)
			close(logErrStream)
		}

		executePollStage = func(codeAssertions func()) {
			// done = make(chan struct{}, 1)

			go func() {
				returnedDropletGUID, executeErr = PollStage(
					appName,
					buildStream,
					warningsStream,
					errStream,
					logStream,
					logErrStream,
					testUI,
					fakeClock,
					15*time.Minute)
				codeAssertions()
				close(blockOnExecute)
				// done <- struct{}{}
			}()
		}
	})

	JustBeforeEach(func() {
		go func() {
			closeStreams()
			close(blockOnChannels)
		}()
	})

	AfterEach(func() {
		Eventually(blockOnExecute).Should(BeClosed())
		Eventually(blockOnChannels).Should(BeClosed())
	})

	Context("when the build stream contains a droplet GUID", func() {
		BeforeEach(func() {
			buildStream <- v3action.Build{Droplet: ccv3.Droplet{GUID: "droplet-guid"}}
		})

		It("returns the droplet GUID", func() {
			executePollStage(func() {
				Expect(executeErr).ToNot(HaveOccurred())
				Expect(returnedDropletGUID).To(Equal("droplet-guid"))
			})

			Eventually(testUI.Out).Should(Say("droplet: droplet-guid"))
		})
	})

	Context("when the warnings stream contains warnings", func() {
		BeforeEach(func() {
			warningsStream <- v3action.Warnings{"warning-1", "warning-2"}
		})

		It("displays the warnings", func() {
			executePollStage(func() {
				Expect(executeErr).ToNot(HaveOccurred())
				Expect(returnedDropletGUID).To(BeEmpty())
			})

			Eventually(testUI.Err).Should(Say("warning-1"))
			Eventually(testUI.Err).Should(Say("warning-2"))
		})
	})

	Context("when the log stream contains a log message", func() {
		Context("and the message is a staging message", func() {
			BeforeEach(func() {
				logStream <- v3action.NewLogMessage("some-log-message", 1, time.Now(), v3action.StagingLog, "1")
			})

			It("prints the log message", func() {
				executePollStage(func() {
					Expect(executeErr).ToNot(HaveOccurred())
					Expect(returnedDropletGUID).To(BeEmpty())
				})
				Eventually(testUI.Out).Should(Say("some-log-message"))
			})
		})

		Context("and the message is not a staging message", func() {
			BeforeEach(func() {
				logStream <- v3action.NewLogMessage("some-log-message", 1, time.Now(), "RUN", "1")
			})

			It("ignores the log message", func() {
				executePollStage(func() {
					Expect(executeErr).ToNot(HaveOccurred())
					Expect(returnedDropletGUID).To(BeEmpty())
				})
				Consistently(testUI.Out).ShouldNot(Say("some-log-message"))
			})
		})
	})

	Context("when the error stream contains an error", func() {
		BeforeEach(func() {
			errStream <- errors.New("some error")
		})

		It("returns the error without waiting for streams to be closed", func() {
			executePollStage(func() {
				Expect(executeErr).To(MatchError("some error"))
				Expect(returnedDropletGUID).To(BeEmpty())
			})
			Consistently(testUI.Out).ShouldNot(Say("droplet: droplet-guid"))
		})
	})

	Context("when the log error stream contains errors", func() {
		BeforeEach(func() {
			logErrStream <- errors.New("some-log-error")
		})

		It("displays the log errors as warnings", func() {
			executePollStage(func() {
				Expect(executeErr).ToNot(HaveOccurred())
				Expect(returnedDropletGUID).To(BeEmpty())
			})
			Eventually(testUI.Err).Should(Say("some-log-error"))
		})
	})

	XContext("when the staging timeout has expired", func() {
		It("exits with an error", func() {
			// Consistently(done).ShouldNot(Receive())
			fakeClock.WaitForWatcherAndIncrement(15*time.Minute - time.Nanosecond)
			// Consistently(done).ShouldNot(Receive())

			fakeClock.Increment(time.Duration(time.Nanosecond))
			// Eventually(done).Should(Receive())

			executePollStage(func() {
				Expect(executeErr).To(MatchError(translatableerror.StagingTimeoutError{
					AppName: "some-app",
					Timeout: 15 * time.Minute}))
				Expect(returnedDropletGUID).To(BeEmpty())
			})
		})
	})
})
<commit_msg>removed race conditions in v3_poll_stage_test<commit_after>package shared_test

import (
	"errors"
	"time"

	"code.cloudfoundry.org/cli/actor/v3action"
	"code.cloudfoundry.org/cli/api/cloudcontroller/ccv3"
	"code.cloudfoundry.org/cli/command/translatableerror"
	. "code.cloudfoundry.org/cli/command/v3/shared"
	"code.cloudfoundry.org/cli/util/ui"
	"code.cloudfoundry.org/clock/fakeclock"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	. "github.com/onsi/gomega/gbytes"
)

var _ = FDescribe("V3PollStage", func() {
	var (
		appName               string
		returnedDropletGUID   string
		executeErr            error
		testUI                *ui.UI
		buildStream           chan v3action.Build
		warningsStream        chan v3action.Warnings
		errStream             chan error
		logStream             chan *v3action.LogMessage
		logErrStream          chan error
		fakeClock             *fakeclock.FakeClock
		closeStreams          func()
		executePollStage      func(func())
		finishedWritingEvents chan bool
		finishedClosing       chan bool
	)

	closeStreams = func() {
		close(errStream)
		close(warningsStream)
		close(buildStream)
		finishedClosing <- true
	}

	executePollStage = func(codeAssertions func()) {
		returnedDropletGUID, executeErr = PollStage(
			appName,
			buildStream,
			warningsStream,
			errStream,
			logStream,
			logErrStream,
			testUI,
			fakeClock,
			15*time.Minute)
		codeAssertions()
		Eventually(finishedClosing).Should(Receive(Equal(true)))
	}

	BeforeEach(func() {
		// reset assertion variables
		executeErr = nil
		returnedDropletGUID = ""

		// create new channels
		testUI = ui.NewTestUI(nil, NewBuffer(), NewBuffer())
		fakeClock = fakeclock.NewFakeClock(time.Now())
		buildStream = make(chan v3action.Build)
		warningsStream = make(chan v3action.Warnings)
		errStream = make(chan error)
		logStream = make(chan *v3action.LogMessage)
		logErrStream = make(chan error)
		appName = "some-app"

		finishedWritingEvents = make(chan bool)
		finishedClosing = make(chan bool)

		// wait for all events to be written before closing channels
		go func() {
			defer GinkgoRecover()

			Eventually(finishedWritingEvents).Should(Receive(Equal(true)))
			closeStreams()
		}()
	})

	Context("when the build stream contains a droplet GUID", func() {
		BeforeEach(func() {
			go func() {
				defer GinkgoRecover()

				buildStream <- v3action.Build{Droplet: ccv3.Droplet{GUID: "droplet-guid"}}
				finishedWritingEvents <- true
			}()
		})

		It("returns the droplet GUID", func() {
			executePollStage(func() {
				Expect(executeErr).ToNot(HaveOccurred())
				Expect(returnedDropletGUID).To(Equal("droplet-guid"))
			})

			Eventually(testUI.Out).Should(Say("droplet: droplet-guid"))
		})
	})

	Context("when the warnings stream contains warnings", func() {
		BeforeEach(func() {
			go func() {
				defer GinkgoRecover()

				warningsStream <- v3action.Warnings{"warning-1", "warning-2"}
				finishedWritingEvents <- true
			}()
		})

		It("displays the warnings", func() {
			executePollStage(func() {
				Expect(executeErr).ToNot(HaveOccurred())
				Expect(returnedDropletGUID).To(BeEmpty())
			})

			Eventually(testUI.Err).Should(Say("warning-1"))
			Eventually(testUI.Err).Should(Say("warning-2"))
		})
	})

	Context("when the log stream contains a log message", func() {
		Context("and the message is a staging message", func() {
			BeforeEach(func() {
				go func() {
					defer GinkgoRecover()

					logStream <- v3action.NewLogMessage("some-log-message", 1, time.Now(), v3action.StagingLog, "1")
					finishedWritingEvents <- true
				}()
			})

			It("prints the log message", func() {
				executePollStage(func() {
					Expect(executeErr).ToNot(HaveOccurred())
					Expect(returnedDropletGUID).To(BeEmpty())
				})
				Eventually(testUI.Out).Should(Say("some-log-message"))
			})
		})

		Context("and the message is not a staging message", func() {
			BeforeEach(func() {
				go func() {
					defer GinkgoRecover()

					logStream <- v3action.NewLogMessage("some-log-message", 1, time.Now(), "RUN", "1")
					finishedWritingEvents <- true
				}()
			})

			It("ignores the log message", func() {
				executePollStage(func() {
					Expect(executeErr).ToNot(HaveOccurred())
					Expect(returnedDropletGUID).To(BeEmpty())
				})
				Consistently(testUI.Out).ShouldNot(Say("some-log-message"))
			})
		})
	})

	Context("when the error stream contains an error", func() {
		BeforeEach(func() {
			go func() {
				defer GinkgoRecover()

				errStream <- errors.New("some error")
				finishedWritingEvents <- true
			}()
		})

		It("returns the error without waiting for streams to be closed", func() {
			executePollStage(func() {
				Expect(executeErr).To(MatchError("some error"))
				Expect(returnedDropletGUID).To(BeEmpty())
			})
			Consistently(testUI.Out).ShouldNot(Say("droplet: droplet-guid"))
		})
	})

	Context("when the log error stream contains errors", func() {
		BeforeEach(func() {
			go func() {
				defer GinkgoRecover()

				logErrStream <- errors.New("some-log-error")
				finishedWritingEvents <- true
			}()
		})

		It("displays the log errors as warnings", func() {
			executePollStage(func() {
				Expect(executeErr).ToNot(HaveOccurred())
				Expect(returnedDropletGUID).To(BeEmpty())
			})
			Eventually(testUI.Err).Should(Say("some-log-error"))
		})
	})

	XContext("when the staging timeout has expired", func() {
		It("exits with an error", func() {
			// Consistently(done).ShouldNot(Receive())
			fakeClock.WaitForWatcherAndIncrement(15*time.Minute - time.Nanosecond)
			// Consistently(done).ShouldNot(Receive())

			fakeClock.Increment(time.Duration(time.Nanosecond))
			// Eventually(done).Should(Receive())

			executePollStage(func() {
				Expect(executeErr).To(MatchError(translatableerror.StagingTimeoutError{
					AppName: "some-app",
					Timeout: 15 * time.Minute}))
				Expect(returnedDropletGUID).To(BeEmpty())
			})
		})
	})
})
<|endoftext|>


<commit_before>package daemon

import "testing"

func TestService(t *testing.T) {
	// TODO(xiaq): Add tests.
}
<commit_msg>daemon: Add a test.<commit_after>package daemon

import (
	"testing"
	"time"

	"github.com/elves/elvish/util"
)

func TestDaemon(t *testing.T) {
	util.InTempDir(func(string) {
		serverDone := make(chan struct{})
		go func() {
			Serve("sock", "db")
			close(serverDone)
		}()

		client := NewClient("sock")
		for i := 0; i < 10; i++ {
			client.ResetConn()
			_, err := client.Version()
			if err == nil {
				break
			} else if i == 9 {
				t.Fatal("Failed to connect after 100ms")
			}
			time.Sleep(10 * time.Millisecond)
		}
		_, err := client.AddCmd("test cmd")
		if err != nil {
			t.Errorf("client.AddCmd -> error %v", err)
		}
		client.Close()
		// Wait for server to quit before returning
		<-serverDone
	})
}
<|endoftext|>


<commit_before>package job

import (
	"context"
	"github.com/pkg/errors"
	"github.com/prometheus/client_golang/prometheus"
	"github.com/zrepl/zrepl/config"
	"github.com/zrepl/zrepl/daemon/filters"
	"github.com/zrepl/zrepl/daemon/job/wakeup"
	"github.com/zrepl/zrepl/daemon/logging"
	"github.com/zrepl/zrepl/daemon/pruner"
	"github.com/zrepl/zrepl/daemon/snapper"
	"github.com/zrepl/zrepl/endpoint"
	"github.com/zrepl/zrepl/zfs"
)

type SnapJob struct {
	name          string
    fsfilter         zfs.DatasetFilter
    snapper *snapper.PeriodicOrManual

	prunerFactory *pruner.SinglePrunerFactory

	promPruneSecs *prometheus.HistogramVec // no labels!

	pruner *pruner.Pruner
}

func (j *SnapJob) Name() string { return j.name }

func (j *SnapJob) getPruner(ctx context.Context, sender *endpoint.Sender) (*pruner.Pruner) {
    p := j.prunerFactory.BuildSinglePruner(ctx,sender,sender)
    return p
}

func (j *SnapJob) Type() Type { return TypeSnap }

func (j *SnapJob) RunPeriodic(ctx context.Context, wakeUpCommon chan <- struct{}) {
    j.snapper.Run(ctx, wakeUpCommon)
}

func (j *SnapJob) FSFilter() zfs.DatasetFilter {
	return j.fsfilter
}

func snapJob(g *config.Global, in *config.SnapJob) (j *SnapJob, err error) {
	j = &SnapJob{}
    fsf, err := filters.DatasetMapFilterFromConfig(in.Filesystems)
    if err != nil {
        return nil, errors.Wrap(err, "cannnot build filesystem filter")
    }
    j.fsfilter = fsf

    if j.snapper, err = snapper.FromConfig(g, fsf, in.Snapshotting); err != nil {
        return nil, errors.Wrap(err, "cannot build snapper")
    }
	j.name = in.Name
	j.promPruneSecs = prometheus.NewHistogramVec(prometheus.HistogramOpts{
		Namespace:   "zrepl",
		Subsystem:   "pruning",
		Name:        "time",
		Help:        "seconds spent in pruner",
		ConstLabels: prometheus.Labels{"zrepl_job":j.name},
	}, []string{"prune_side"})
	j.prunerFactory, err = pruner.NewSinglePrunerFactory(in.Pruning, j.promPruneSecs)
	if err != nil {
		return nil, errors.Wrap(err, "cannot build snapjob pruning rules")
	}
	return j, nil
}

func (j *SnapJob) RegisterMetrics(registerer prometheus.Registerer) {
	registerer.MustRegister(j.promPruneSecs)
}

type SnapJobStatus struct {
	Pruning *pruner.Report
}

func (j *SnapJob) Status() *Status {
	s := &SnapJobStatus{}
	t := j.Type()
	if j.pruner != nil {
		s.Pruning = j.pruner.Report()
	}
	return &Status{Type: t, JobSpecific: s}
}

func (j *SnapJob) Run(ctx context.Context) {
	log := GetLogger(ctx)
	ctx = logging.WithSubsystemLoggers(ctx, log)

	defer log.Info("job exiting")

	periodicDone := make(chan struct{})
	ctx, cancel := context.WithCancel(ctx)
	defer cancel()
	go j.RunPeriodic(ctx, periodicDone)

	invocationCount := 0
outer:
	for {
		log.Info("wait for wakeups")
		select {
		case <-ctx.Done():
			log.WithError(ctx.Err()).Info("context")
			break outer

		case <-wakeup.Wait(ctx):
		case <-periodicDone:
		}
		invocationCount++
		invLog := log.WithField("invocation", invocationCount)
		j.doPrune(WithLogger(ctx, invLog))
	}
}

func (j *SnapJob) doPrune(ctx context.Context) {
	log := GetLogger(ctx)
	ctx = logging.WithSubsystemLoggers(ctx, log)
	sender := endpoint.NewSender(j.FSFilter())
	j.pruner = j.getPruner(ctx, sender)
	log.Info("start pruning")
	j.pruner.Prune()
	log.Info("finished pruning")
}

<commit_msg>Gofmt snapjob.go<commit_after>package job

import (
	"context"
	"github.com/pkg/errors"
	"github.com/prometheus/client_golang/prometheus"
	"github.com/zrepl/zrepl/config"
	"github.com/zrepl/zrepl/daemon/filters"
	"github.com/zrepl/zrepl/daemon/job/wakeup"
	"github.com/zrepl/zrepl/daemon/logging"
	"github.com/zrepl/zrepl/daemon/pruner"
	"github.com/zrepl/zrepl/daemon/snapper"
	"github.com/zrepl/zrepl/endpoint"
	"github.com/zrepl/zrepl/zfs"
)

type SnapJob struct {
	name     string
	fsfilter zfs.DatasetFilter
	snapper  *snapper.PeriodicOrManual

	prunerFactory *pruner.SinglePrunerFactory

	promPruneSecs *prometheus.HistogramVec // no labels!

	pruner *pruner.Pruner
}

func (j *SnapJob) Name() string { return j.name }

func (j *SnapJob) getPruner(ctx context.Context, sender *endpoint.Sender) *pruner.Pruner {
	p := j.prunerFactory.BuildSinglePruner(ctx, sender, sender)
	return p
}

func (j *SnapJob) Type() Type { return TypeSnap }

func (j *SnapJob) RunPeriodic(ctx context.Context, wakeUpCommon chan<- struct{}) {
	j.snapper.Run(ctx, wakeUpCommon)
}

func (j *SnapJob) FSFilter() zfs.DatasetFilter {
	return j.fsfilter
}

func snapJob(g *config.Global, in *config.SnapJob) (j *SnapJob, err error) {
	j = &SnapJob{}
	fsf, err := filters.DatasetMapFilterFromConfig(in.Filesystems)
	if err != nil {
		return nil, errors.Wrap(err, "cannnot build filesystem filter")
	}
	j.fsfilter = fsf

	if j.snapper, err = snapper.FromConfig(g, fsf, in.Snapshotting); err != nil {
		return nil, errors.Wrap(err, "cannot build snapper")
	}
	j.name = in.Name
	j.promPruneSecs = prometheus.NewHistogramVec(prometheus.HistogramOpts{
		Namespace:   "zrepl",
		Subsystem:   "pruning",
		Name:        "time",
		Help:        "seconds spent in pruner",
		ConstLabels: prometheus.Labels{"zrepl_job": j.name},
	}, []string{"prune_side"})
	j.prunerFactory, err = pruner.NewSinglePrunerFactory(in.Pruning, j.promPruneSecs)
	if err != nil {
		return nil, errors.Wrap(err, "cannot build snapjob pruning rules")
	}
	return j, nil
}

func (j *SnapJob) RegisterMetrics(registerer prometheus.Registerer) {
	registerer.MustRegister(j.promPruneSecs)
}

type SnapJobStatus struct {
	Pruning *pruner.Report
}

func (j *SnapJob) Status() *Status {
	s := &SnapJobStatus{}
	t := j.Type()
	if j.pruner != nil {
		s.Pruning = j.pruner.Report()
	}
	return &Status{Type: t, JobSpecific: s}
}

func (j *SnapJob) Run(ctx context.Context) {
	log := GetLogger(ctx)
	ctx = logging.WithSubsystemLoggers(ctx, log)

	defer log.Info("job exiting")

	periodicDone := make(chan struct{})
	ctx, cancel := context.WithCancel(ctx)
	defer cancel()
	go j.RunPeriodic(ctx, periodicDone)

	invocationCount := 0
outer:
	for {
		log.Info("wait for wakeups")
		select {
		case <-ctx.Done():
			log.WithError(ctx.Err()).Info("context")
			break outer

		case <-wakeup.Wait(ctx):
		case <-periodicDone:
		}
		invocationCount++
		invLog := log.WithField("invocation", invocationCount)
		j.doPrune(WithLogger(ctx, invLog))
	}
}

func (j *SnapJob) doPrune(ctx context.Context) {
	log := GetLogger(ctx)
	ctx = logging.WithSubsystemLoggers(ctx, log)
	sender := endpoint.NewSender(j.FSFilter())
	j.pruner = j.getPruner(ctx, sender)
	log.Info("start pruning")
	j.pruner.Prune()
	log.Info("finished pruning")
}
<|endoftext|>