<commit_before>// Copyright 2012, Google Inc. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package tabletserver

import (
	"net/http"
	"os"
	"os/exec"
	"strconv"
	"strings"
	"sync"
	"time"

	log "github.com/golang/glog"
	"github.com/youtube/vitess/go/acl"
	"github.com/youtube/vitess/go/memcache"
	"github.com/youtube/vitess/go/pools"
	"github.com/youtube/vitess/go/stats"
	"github.com/youtube/vitess/go/sync2"
)

const statsURL = "/debug/memcache/"

type CreateCacheFunc func() (*memcache.Connection, error)

// CachePool re-exposes ResourcePool as a pool of Memcache connection objects.
type CachePool struct {
	name           string
	pool           *pools.ResourcePool
	maxPrefix      sync2.AtomicInt64
	cmd            *exec.Cmd
	rowCacheConfig RowCacheConfig
	capacity       int
	port           string
	idleTimeout    time.Duration
	DeleteExpiry   uint64
	memcacheStats  *MemcacheStats
	mu             sync.Mutex
}

func NewCachePool(name string, rowCacheConfig RowCacheConfig, queryTimeout time.Duration, idleTimeout time.Duration) *CachePool {
	cp := &CachePool{name: name, idleTimeout: idleTimeout}
	if name != "" {
		cp.memcacheStats = NewMemcacheStats(cp, true, false, false)
		stats.Publish(name+"ConnPoolCapacity", stats.IntFunc(cp.Capacity))
		stats.Publish(name+"ConnPoolAvailable", stats.IntFunc(cp.Available))
		stats.Publish(name+"ConnPoolMaxCap", stats.IntFunc(cp.MaxCap))
		stats.Publish(name+"ConnPoolWaitCount", stats.IntFunc(cp.WaitCount))
		stats.Publish(name+"ConnPoolWaitTime", stats.DurationFunc(cp.WaitTime))
		stats.Publish(name+"ConnPoolIdleTimeout", stats.DurationFunc(cp.IdleTimeout))
	}
	http.Handle(statsURL, cp)

	if rowCacheConfig.Binary == "" {
		return cp
	}
	cp.rowCacheConfig = rowCacheConfig

	// Start with memcached defaults
	cp.capacity = 1024 - 50
	cp.port = "11211"
	if rowCacheConfig.Socket != "" {
		cp.port = rowCacheConfig.Socket
	}
	if rowCacheConfig.TcpPort > 0 {
		cp.port = strconv.Itoa(rowCacheConfig.TcpPort)
	}
	if rowCacheConfig.Connections > 0 {
		if rowCacheConfig.Connections <= 50 {
			log.Fatalf("insufficient capacity: %d", rowCacheConfig.Connections)
		}
		cp.capacity = rowCacheConfig.Connections - 50
	}

	seconds := uint64(queryTimeout / time.Second)
	// Add an additional grace period for
	// memcache expiry of deleted items
	if seconds != 0 {
		cp.DeleteExpiry = 2*seconds + 15
	}
	return cp
}

func (cp *CachePool) Open() {
	cp.mu.Lock()
	defer cp.mu.Unlock()
	if cp.pool != nil {
		panic(NewTabletError(FATAL, "rowcache is already open"))
	}
	if cp.rowCacheConfig.Binary == "" {
		panic(NewTabletError(FATAL, "rowcache binary not specified"))
	}
	cp.startMemcache()
	log.Infof("rowcache is enabled")
	f := func() (pools.Resource, error) {
		return memcache.Connect(cp.port, 10*time.Second)
	}
	cp.pool = pools.NewResourcePool(f, cp.capacity, cp.capacity, cp.idleTimeout)
	if cp.memcacheStats != nil {
		cp.memcacheStats.Open()
	}
}

func (cp *CachePool) startMemcache() {
	if strings.Contains(cp.port, "/") {
		_ = os.Remove(cp.port)
	}
	commandLine := cp.rowCacheConfig.GetSubprocessFlags()
	cp.cmd = exec.Command(commandLine[0], commandLine[1:]...)
	if err := cp.cmd.Start(); err != nil {
		panic(NewTabletError(FATAL, "can't start memcache: %v", err))
	}
	attempts := 0
	for {
		time.Sleep(100 * time.Millisecond)
		c, err := memcache.Connect(cp.port, 30*time.Millisecond)
		if err != nil {
			attempts++
			if attempts >= 50 {
				cp.cmd.Process.Kill()
				// Avoid zombies
				go cp.cmd.Wait()
				// FIXME(sougou): Throw proper error if we can recover
				log.Fatal("Can't connect to memcache")
			}
			continue
		}
		if _, err = c.Set("health", 0, 0, []byte("ok")); err != nil {
			panic(NewTabletError(FATAL, "can't communicate with memcache: %v", err))
		}
		c.Close()
		break
	}
}

func (cp *CachePool) Close() {
	// Close the underlying pool first.
	// You cannot close the pool while holding the
	// lock because we have to still allow Put to
	// return outstanding connections, if any.
	pool := cp.getPool()
	if pool == nil {
		return
	}
	pool.Close()

	// No new operations will be allowed now.
	// Safe to cleanup.
	cp.mu.Lock()
	defer cp.mu.Unlock()
	if cp.pool == nil {
		return
	}
	if cp.memcacheStats != nil {
		cp.memcacheStats.Close()
	}
	cp.cmd.Process.Kill()
	// Avoid zombies
	go cp.cmd.Wait()
	if strings.Contains(cp.port, "/") {
		_ = os.Remove(cp.port)
	}
	cp.pool = nil
}

func (cp *CachePool) IsClosed() bool {
	cp.mu.Lock()
	defer cp.mu.Unlock()
	return cp.pool == nil
}

func (cp *CachePool) getPool() *pools.ResourcePool {
	cp.mu.Lock()
	defer cp.mu.Unlock()
	return cp.pool
}

// You must call Put after Get.
func (cp *CachePool) Get(timeout time.Duration) *memcache.Connection {
	pool := cp.getPool()
	if pool == nil {
		panic(NewTabletError(FATAL, "cache pool is not open"))
	}
	r, err := pool.Get(timeout)
	if err != nil {
		panic(NewTabletErrorSql(FATAL, err))
	}
	return r.(*memcache.Connection)
}

func (cp *CachePool) Put(conn *memcache.Connection) {
	pool := cp.getPool()
	if pool == nil {
		return
	}
	if conn == nil {
		pool.Put(nil)
	} else {
		pool.Put(conn)
	}
}

func (cp *CachePool) StatsJSON() string {
	pool := cp.getPool()
	if pool == nil {
		return "{}"
	}
	return pool.StatsJSON()
}

func (cp *CachePool) Capacity() int64 {
	pool := cp.getPool()
	if pool == nil {
		return 0
	}
	return pool.Capacity()
}

func (cp *CachePool) Available() int64 {
	pool := cp.getPool()
	if pool == nil {
		return 0
	}
	return pool.Available()
}

func (cp *CachePool) MaxCap() int64 {
	pool := cp.getPool()
	if pool == nil {
		return 0
	}
	return pool.MaxCap()
}

func (cp *CachePool) WaitCount() int64 {
	pool := cp.getPool()
	if pool == nil {
		return 0
	}
	return pool.WaitCount()
}

func (cp *CachePool) WaitTime() time.Duration {
	pool := cp.getPool()
	if pool == nil {
		return 0
	}
	return pool.WaitTime()
}

func (cp *CachePool) IdleTimeout() time.Duration {
	pool := cp.getPool()
	if pool == nil {
		return 0
	}
	return pool.IdleTimeout()
}

func (cp *CachePool) ServeHTTP(response http.ResponseWriter, request *http.Request) {
	if err := acl.CheckAccessHTTP(request, acl.MONITORING); err != nil {
		acl.SendError(response, err)
		return
	}
	defer func() {
		if x := recover(); x != nil {
			response.Write(([]byte)(x.(error).Error()))
		}
	}()
	response.Header().Set("Content-Type", "text/plain")
	pool := cp.getPool()
	if pool == nil {
		response.Write(([]byte)("closed"))
		return
	}
	command := request.URL.Path[len(statsURL):]
	if command == "stats" {
		command = ""
	}
	conn := cp.Get(0)
	// This is not the same as defer rc.cachePool.Put(conn)
	defer func() { cp.Put(conn) }()
	r, err := conn.Stats(command)
	if err != nil {
		conn.Close()
		conn = nil
		response.Write(([]byte)(err.Error()))
	} else {
		response.Write(r)
	}
}
<commit_msg>fix memcached local tcp address handling<commit_after>// Copyright 2012, Google Inc. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package tabletserver

import (
	"net/http"
	"os"
	"os/exec"
	"strconv"
	"strings"
	"sync"
	"time"

	log "github.com/golang/glog"
	"github.com/youtube/vitess/go/acl"
	"github.com/youtube/vitess/go/memcache"
	"github.com/youtube/vitess/go/pools"
	"github.com/youtube/vitess/go/stats"
	"github.com/youtube/vitess/go/sync2"
)

const statsURL = "/debug/memcache/"

type CreateCacheFunc func() (*memcache.Connection, error)

// CachePool re-exposes ResourcePool as a pool of Memcache connection objects.
type CachePool struct {
	name           string
	pool           *pools.ResourcePool
	maxPrefix      sync2.AtomicInt64
	cmd            *exec.Cmd
	rowCacheConfig RowCacheConfig
	capacity       int
	port           string
	idleTimeout    time.Duration
	DeleteExpiry   uint64
	memcacheStats  *MemcacheStats
	mu             sync.Mutex
}

func NewCachePool(name string, rowCacheConfig RowCacheConfig, queryTimeout time.Duration, idleTimeout time.Duration) *CachePool {
	cp := &CachePool{name: name, idleTimeout: idleTimeout}
	if name != "" {
		cp.memcacheStats = NewMemcacheStats(cp, true, false, false)
		stats.Publish(name+"ConnPoolCapacity", stats.IntFunc(cp.Capacity))
		stats.Publish(name+"ConnPoolAvailable", stats.IntFunc(cp.Available))
		stats.Publish(name+"ConnPoolMaxCap", stats.IntFunc(cp.MaxCap))
		stats.Publish(name+"ConnPoolWaitCount", stats.IntFunc(cp.WaitCount))
		stats.Publish(name+"ConnPoolWaitTime", stats.DurationFunc(cp.WaitTime))
		stats.Publish(name+"ConnPoolIdleTimeout", stats.DurationFunc(cp.IdleTimeout))
	}
	http.Handle(statsURL, cp)

	if rowCacheConfig.Binary == "" {
		return cp
	}
	cp.rowCacheConfig = rowCacheConfig

	// Start with memcached defaults
	cp.capacity = 1024 - 50
	cp.port = "11211"
	if rowCacheConfig.Socket != "" {
		cp.port = rowCacheConfig.Socket
	}
	if rowCacheConfig.TcpPort > 0 {
		//address: ":11211"
		cp.port = ":" + strconv.Itoa(rowCacheConfig.TcpPort)
	}
	if rowCacheConfig.Connections > 0 {
		if rowCacheConfig.Connections <= 50 {
			log.Fatalf("insufficient capacity: %d", rowCacheConfig.Connections)
		}
		cp.capacity = rowCacheConfig.Connections - 50
	}

	seconds := uint64(queryTimeout / time.Second)
	// Add an additional grace period for
	// memcache expiry of deleted items
	if seconds != 0 {
		cp.DeleteExpiry = 2*seconds + 15
	}
	return cp
}

func (cp *CachePool) Open() {
	cp.mu.Lock()
	defer cp.mu.Unlock()
	if cp.pool != nil {
		panic(NewTabletError(FATAL, "rowcache is already open"))
	}
	if cp.rowCacheConfig.Binary == "" {
		panic(NewTabletError(FATAL, "rowcache binary not specified"))
	}
	cp.startMemcache()
	log.Infof("rowcache is enabled")
	f := func() (pools.Resource, error) {
		return memcache.Connect(cp.port, 10*time.Second)
	}
	cp.pool = pools.NewResourcePool(f, cp.capacity, cp.capacity, cp.idleTimeout)
	if cp.memcacheStats != nil {
		cp.memcacheStats.Open()
	}
}

func (cp *CachePool) startMemcache() {
	if strings.Contains(cp.port, "/") {
		_ = os.Remove(cp.port)
	}
	commandLine := cp.rowCacheConfig.GetSubprocessFlags()
	cp.cmd = exec.Command(commandLine[0], commandLine[1:]...)
	if err := cp.cmd.Start(); err != nil {
		panic(NewTabletError(FATAL, "can't start memcache: %v", err))
	}
	attempts := 0
	for {
		time.Sleep(100 * time.Millisecond)
		c, err := memcache.Connect(cp.port, 30*time.Millisecond)
		if err != nil {
			attempts++
			if attempts >= 50 {
				cp.cmd.Process.Kill()
				// Avoid zombies
				go cp.cmd.Wait()
				// FIXME(sougou): Throw proper error if we can recover
				log.Fatal("Can't connect to memcache")
			}
			continue
		}
		if _, err = c.Set("health", 0, 0, []byte("ok")); err != nil {
			panic(NewTabletError(FATAL, "can't communicate with memcache: %v", err))
		}
		c.Close()
		break
	}
}

func (cp *CachePool) Close() {
	// Close the underlying pool first.
	// You cannot close the pool while holding the
	// lock because we have to still allow Put to
	// return outstanding connections, if any.
	pool := cp.getPool()
	if pool == nil {
		return
	}
	pool.Close()

	// No new operations will be allowed now.
	// Safe to cleanup.
	cp.mu.Lock()
	defer cp.mu.Unlock()
	if cp.pool == nil {
		return
	}
	if cp.memcacheStats != nil {
		cp.memcacheStats.Close()
	}
	cp.cmd.Process.Kill()
	// Avoid zombies
	go cp.cmd.Wait()
	if strings.Contains(cp.port, "/") {
		_ = os.Remove(cp.port)
	}
	cp.pool = nil
}

func (cp *CachePool) IsClosed() bool {
	cp.mu.Lock()
	defer cp.mu.Unlock()
	return cp.pool == nil
}

func (cp *CachePool) getPool() *pools.ResourcePool {
	cp.mu.Lock()
	defer cp.mu.Unlock()
	return cp.pool
}

// You must call Put after Get.
func (cp *CachePool) Get(timeout time.Duration) *memcache.Connection {
	pool := cp.getPool()
	if pool == nil {
		panic(NewTabletError(FATAL, "cache pool is not open"))
	}
	r, err := pool.Get(timeout)
	if err != nil {
		panic(NewTabletErrorSql(FATAL, err))
	}
	return r.(*memcache.Connection)
}

func (cp *CachePool) Put(conn *memcache.Connection) {
	pool := cp.getPool()
	if pool == nil {
		return
	}
	if conn == nil {
		pool.Put(nil)
	} else {
		pool.Put(conn)
	}
}

func (cp *CachePool) StatsJSON() string {
	pool := cp.getPool()
	if pool == nil {
		return "{}"
	}
	return pool.StatsJSON()
}

func (cp *CachePool) Capacity() int64 {
	pool := cp.getPool()
	if pool == nil {
		return 0
	}
	return pool.Capacity()
}

func (cp *CachePool) Available() int64 {
	pool := cp.getPool()
	if pool == nil {
		return 0
	}
	return pool.Available()
}

func (cp *CachePool) MaxCap() int64 {
	pool := cp.getPool()
	if pool == nil {
		return 0
	}
	return pool.MaxCap()
}

func (cp *CachePool) WaitCount() int64 {
	pool := cp.getPool()
	if pool == nil {
		return 0
	}
	return pool.WaitCount()
}

func (cp *CachePool) WaitTime() time.Duration {
	pool := cp.getPool()
	if pool == nil {
		return 0
	}
	return pool.WaitTime()
}

func (cp *CachePool) IdleTimeout() time.Duration {
	pool := cp.getPool()
	if pool == nil {
		return 0
	}
	return pool.IdleTimeout()
}

func (cp *CachePool) ServeHTTP(response http.ResponseWriter, request *http.Request) {
	if err := acl.CheckAccessHTTP(request, acl.MONITORING); err != nil {
		acl.SendError(response, err)
		return
	}
	defer func() {
		if x := recover(); x != nil {
			response.Write(([]byte)(x.(error).Error()))
		}
	}()
	response.Header().Set("Content-Type", "text/plain")
	pool := cp.getPool()
	if pool == nil {
		response.Write(([]byte)("closed"))
		return
	}
	command := request.URL.Path[len(statsURL):]
	if command == "stats" {
		command = ""
	}
	conn := cp.Get(0)
	// This is not the same as defer rc.cachePool.Put(conn)
	defer func() { cp.Put(conn) }()
	r, err := conn.Stats(command)
	if err != nil {
		conn.Close()
		conn = nil
		response.Write(([]byte)(err.Error()))
	} else {
		response.Write(r)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"net/url"

	"github.com/intelsdi-x/snap/mgmt/rest/client"
	"github.com/intelsdi-x/snap/mgmt/rest/rbody"
	"github.com/intelsdi-x/snap/scheduler/wmap"
	"github.com/raintank/raintank-apps/server/model"
)

var SnapClient *client.Client

func InitSnapClient(u *url.URL) {
	SnapClient = client.New(u.String(), "v1", false)
}

func GetSnapMetrics() ([]*rbody.Metric, error) {
	resp := SnapClient.GetMetricCatalog()
	return resp.Catalog, resp.Err
}

func GetSnapTasks() ([]*rbody.ScheduledTask, error) {
	resp := SnapClient.GetTasks()
	var tasks []*rbody.ScheduledTask
	if resp.Err == nil {
		tasks = make([]*rbody.ScheduledTask, len(resp.ScheduledTasks))
		for i, t := range resp.ScheduledTasks {
			tasks[i] = &t
		}
	}
	return tasks, resp.Err
}

func RemoveSnapTask(task *rbody.ScheduledTask) error {
	stopResp := SnapClient.StopTask(task.ID)
	if stopResp.Err != nil {
		return stopResp.Err
	}
	removeResp := SnapClient.RemoveTask(task.ID)
	return removeResp.Err
}

func CreateSnapTask(t *model.TaskDTO, name string) (*rbody.ScheduledTask, error) {
	s := &client.Schedule{
		Type:     "simple",
		Interval: fmt.Sprintf("%ds", t.Interval),
	}
	wf := wmap.NewWorkflowMap()
	for ns, ver := range t.Metrics {
		if err := wf.CollectNode.AddMetric(ns, int(ver)); err != nil {
			return nil, err
		}
	}
	token := ""
	for ns, conf := range t.Config {
		for key, value := range conf {
			wf.CollectNode.AddConfigItem(ns, key, value)
			if key == "token" {
				token = value.(string)
			}
		}
	}
	publisher := getPublisher(
		1, //TODO: replace with actual orgId
		t.Interval,
		token,
	)
	if err := wf.CollectNode.Add(publisher); err != nil {
		return nil, err
	}

	resp := SnapClient.CreateTask(s, wf, name, "10s", true)
	newTask := rbody.ScheduledTask(*resp.AddScheduledTask)
	return &newTask, resp.Err
}

func getPublisher(orgId, interval int64, token string) *wmap.PublishWorkflowMapNode {
	return &wmap.PublishWorkflowMapNode{
		Name: "rt-hostedtsdb",
		Config: map[string]interface{}{
			"interval": interval,
			"url":      "http://localhost:8081",
			"orgId":    orgId,
			"token":    token,
		},
	}
}
<commit_msg>remove token and url from per task publisher config<commit_after>package main

import (
	"fmt"
	"net/url"

	"github.com/intelsdi-x/snap/mgmt/rest/client"
	"github.com/intelsdi-x/snap/mgmt/rest/rbody"
	"github.com/intelsdi-x/snap/scheduler/wmap"
	"github.com/raintank/raintank-apps/server/model"
)

var SnapClient *client.Client

func InitSnapClient(u *url.URL) {
	SnapClient = client.New(u.String(), "v1", false)
}

func GetSnapMetrics() ([]*rbody.Metric, error) {
	resp := SnapClient.GetMetricCatalog()
	return resp.Catalog, resp.Err
}

func GetSnapTasks() ([]*rbody.ScheduledTask, error) {
	resp := SnapClient.GetTasks()
	var tasks []*rbody.ScheduledTask
	if resp.Err == nil {
		tasks = make([]*rbody.ScheduledTask, len(resp.ScheduledTasks))
		for i, t := range resp.ScheduledTasks {
			tasks[i] = &t
		}
	}
	return tasks, resp.Err
}

func RemoveSnapTask(task *rbody.ScheduledTask) error {
	stopResp := SnapClient.StopTask(task.ID)
	if stopResp.Err != nil {
		return stopResp.Err
	}
	removeResp := SnapClient.RemoveTask(task.ID)
	return removeResp.Err
}

func CreateSnapTask(t *model.TaskDTO, name string) (*rbody.ScheduledTask, error) {
	s := &client.Schedule{
		Type:     "simple",
		Interval: fmt.Sprintf("%ds", t.Interval),
	}
	wf := wmap.NewWorkflowMap()
	for ns, ver := range t.Metrics {
		if err := wf.CollectNode.AddMetric(ns, int(ver)); err != nil {
			return nil, err
		}
	}
	token := ""
	for ns, conf := range t.Config {
		for key, value := range conf {
			wf.CollectNode.AddConfigItem(ns, key, value)
			if key == "token" {
				token = value.(string)
			}
		}
	}
	publisher := getPublisher(
		1, //TODO: replace with actual orgId
		t.Interval,
		token,
	)
	if err := wf.CollectNode.Add(publisher); err != nil {
		return nil, err
	}

	resp := SnapClient.CreateTask(s, wf, name, "10s", true)
	newTask := rbody.ScheduledTask(*resp.AddScheduledTask)
	return &newTask, resp.Err
}

func getPublisher(orgId, interval int64, token string) *wmap.PublishWorkflowMapNode {
	return &wmap.PublishWorkflowMapNode{
		Name: "rt-hostedtsdb",
		Config: map[string]interface{}{
			"interval": interval,
			"orgId":    orgId,
		},
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"io"
	"log"
	"os"
	"syscall"
	"unsafe"
)

/*** data ***/

type Termios struct {
	Iflag  uint32
	Oflag  uint32
	Cflag  uint32
	Lflag  uint32
	Cc     [20]byte
	Ispeed uint32
	Ospeed uint32
}

type editorConfig struct {
	screenRows  int
	screenCols  int
	origTermios *Termios
}

type WinSize struct {
	Row    uint16
	Col    uint16
	Xpixel uint16
	Ypixel uint16
}

var E editorConfig

/*** terminal ***/

func die(err error) {
	disableRawMode()
	io.WriteString(os.Stdout, "\x1b[2J")
	io.WriteString(os.Stdout, "\x1b[H")
	log.Fatal(err)
}

func TcSetAttr(fd uintptr, termios *Termios) error {
	// TCSETS+1 == TCSETSW, because TCSAFLUSH doesn't exist
	if _, _, err := syscall.Syscall(syscall.SYS_IOCTL, fd, uintptr(syscall.TCSETS+1), uintptr(unsafe.Pointer(termios))); err != 0 {
		return err
	}
	return nil
}

func TcGetAttr(fd uintptr) *Termios {
	var termios = &Termios{}
	if _, _, err := syscall.Syscall(syscall.SYS_IOCTL, fd, syscall.TCGETS, uintptr(unsafe.Pointer(termios))); err != 0 {
		log.Fatalf("Problem getting terminal attributes: %s\n", err)
	}
	return termios
}

func enableRawMode() {
	E.origTermios = TcGetAttr(os.Stdin.Fd())
	var raw Termios
	raw = *E.origTermios
	raw.Iflag &^= syscall.BRKINT | syscall.ICRNL | syscall.INPCK | syscall.ISTRIP | syscall.IXON
	raw.Oflag &^= syscall.OPOST
	raw.Cflag |= syscall.CS8
	raw.Lflag &^= syscall.ECHO | syscall.ICANON | syscall.IEXTEN | syscall.ISIG
	raw.Cc[syscall.VMIN+1] = 0
	raw.Cc[syscall.VTIME+1] = 1
	if e := TcSetAttr(os.Stdin.Fd(), &raw); e != nil {
		log.Fatalf("Problem enabling raw mode: %s\n", e)
	}
}

func disableRawMode() {
	if e := TcSetAttr(os.Stdin.Fd(), E.origTermios); e != nil {
		log.Fatalf("Problem disabling raw mode: %s\n", e)
	}
}

func editorReadKey() byte {
	var buffer [1]byte
	var cc int
	var err error
	for cc, err = os.Stdin.Read(buffer[:]); cc != 1; cc, err = os.Stdin.Read(buffer[:]) {
	}
	if err != nil {
		die(err)
	}
	return buffer[0]
}

func getCursorPosition(rows *int, cols *int) int {
	io.WriteString(os.Stdout, "\x1b[6n")
	var buffer [1]byte
	var buf []byte
	var cc int
	for cc, _ = os.Stdin.Read(buffer[:]); cc == 1; cc, _ = os.Stdin.Read(buffer[:]) {
		if buffer[0] == 'R' {
			break
		}
		buf = append(buf, buffer[0])
	}
	if string(buf[0:2]) != "\x1b[" {
		log.Printf("Failed to read rows;cols from tty\n")
		return -1
	}
	if n, e := fmt.Sscanf(string(buf[2:]), "%d;%d", rows, cols); n != 2 || e != nil {
		if e != nil {
			log.Printf("getCursorPosition: fmt.Sscanf() failed: %s\n", e)
		}
		if n != 2 {
			log.Printf("getCursorPosition: got %d items, wanted 2\n", n)
		}
		return -1
	}
	return 0
}

func getWindowSize(rows *int, cols *int) int {
	var w WinSize
	_, _, err := syscall.Syscall(syscall.SYS_IOCTL,
		os.Stdout.Fd(),
		syscall.TIOCGWINSZ,
		uintptr(unsafe.Pointer(&w)),
	)
	if err != 0 { // type syscall.Errno
		io.WriteString(os.Stdout, "\x1b[999C\x1b[999B")
		return getCursorPosition(rows, cols)
	} else {
		*rows = int(w.Row)
		*cols = int(w.Col)
		return 0
	}
	return -1
}

/*** input ***/

func editorProcessKeypress() {
	c := editorReadKey()
	switch c {
	case ('q' & 0x1f):
		io.WriteString(os.Stdout, "\x1b[2J")
		io.WriteString(os.Stdout, "\x1b[H")
		disableRawMode()
		os.Exit(0)
	}
}

/*** append buffer ***/

type abuf struct {
	buf []byte
}

func (p abuf) String() string {
	return string(p.buf)
}

func (p *abuf) abAppend(s string) {
	p.buf = append(p.buf, []byte(s)...)
}

/*** output ***/

func editorRefreshScreen() {
	var ab abuf
	ab.abAppend("\x1b[2J")
	ab.abAppend("\x1b[H")
	editorDrawRows(&ab)
	ab.abAppend("\x1b[H")
	io.WriteString(os.Stdout, ab.String())
}

func editorDrawRows(ab *abuf) {
	for y := 0; y < E.screenRows-1; y++ {
		ab.abAppend("~\r\n")
	}
	ab.abAppend("~")
}

/*** init ***/

func initEditor() {
	if getWindowSize(&E.screenRows, &E.screenCols) == -1 {
		die(fmt.Errorf("couldn't get screen size"))
	}
}

func main() {
	enableRawMode()
	defer disableRawMode()
	initEditor()

	for {
		editorRefreshScreen()
		editorProcessKeypress()
	}
}
<commit_msg>Step 39<commit_after>package main

import (
	"fmt"
	"io"
	"log"
	"os"
	"syscall"
	"unsafe"
)

/*** data ***/

type Termios struct {
	Iflag  uint32
	Oflag  uint32
	Cflag  uint32
	Lflag  uint32
	Cc     [20]byte
	Ispeed uint32
	Ospeed uint32
}

type editorConfig struct {
	screenRows  int
	screenCols  int
	origTermios *Termios
}

type WinSize struct {
	Row    uint16
	Col    uint16
	Xpixel uint16
	Ypixel uint16
}

var E editorConfig

/*** terminal ***/

func die(err error) {
	disableRawMode()
	io.WriteString(os.Stdout, "\x1b[2J")
	io.WriteString(os.Stdout, "\x1b[H")
	log.Fatal(err)
}

func TcSetAttr(fd uintptr, termios *Termios) error {
	// TCSETS+1 == TCSETSW, because TCSAFLUSH doesn't exist
	if _, _, err := syscall.Syscall(syscall.SYS_IOCTL, fd, uintptr(syscall.TCSETS+1), uintptr(unsafe.Pointer(termios))); err != 0 {
		return err
	}
	return nil
}

func TcGetAttr(fd uintptr) *Termios {
	var termios = &Termios{}
	if _, _, err := syscall.Syscall(syscall.SYS_IOCTL, fd, syscall.TCGETS, uintptr(unsafe.Pointer(termios))); err != 0 {
		log.Fatalf("Problem getting terminal attributes: %s\n", err)
	}
	return termios
}

func enableRawMode() {
	E.origTermios = TcGetAttr(os.Stdin.Fd())
	var raw Termios
	raw = *E.origTermios
	raw.Iflag &^= syscall.BRKINT | syscall.ICRNL | syscall.INPCK | syscall.ISTRIP | syscall.IXON
	raw.Oflag &^= syscall.OPOST
	raw.Cflag |= syscall.CS8
	raw.Lflag &^= syscall.ECHO | syscall.ICANON | syscall.IEXTEN | syscall.ISIG
	raw.Cc[syscall.VMIN+1] = 0
	raw.Cc[syscall.VTIME+1] = 1
	if e := TcSetAttr(os.Stdin.Fd(), &raw); e != nil {
		log.Fatalf("Problem enabling raw mode: %s\n", e)
	}
}

func disableRawMode() {
	if e := TcSetAttr(os.Stdin.Fd(), E.origTermios); e != nil {
		log.Fatalf("Problem disabling raw mode: %s\n", e)
	}
}

func editorReadKey() byte {
	var buffer [1]byte
	var cc int
	var err error
	for cc, err = os.Stdin.Read(buffer[:]); cc != 1; cc, err = os.Stdin.Read(buffer[:]) {
	}
	if err != nil {
		die(err)
	}
	return buffer[0]
}

func getCursorPosition(rows *int, cols *int) int {
	io.WriteString(os.Stdout, "\x1b[6n")
	var buffer [1]byte
	var buf []byte
	var cc int
	for cc, _ = os.Stdin.Read(buffer[:]); cc == 1; cc, _ = os.Stdin.Read(buffer[:]) {
		if buffer[0] == 'R' {
			break
		}
		buf = append(buf, buffer[0])
	}
	if string(buf[0:2]) != "\x1b[" {
		log.Printf("Failed to read rows;cols from tty\n")
		return -1
	}
	if n, e := fmt.Sscanf(string(buf[2:]), "%d;%d", rows, cols); n != 2 || e != nil {
		if e != nil {
			log.Printf("getCursorPosition: fmt.Sscanf() failed: %s\n", e)
		}
		if n != 2 {
			log.Printf("getCursorPosition: got %d items, wanted 2\n", n)
		}
		return -1
	}
	return 0
}

func getWindowSize(rows *int, cols *int) int {
	var w WinSize
	_, _, err := syscall.Syscall(syscall.SYS_IOCTL,
		os.Stdout.Fd(),
		syscall.TIOCGWINSZ,
		uintptr(unsafe.Pointer(&w)),
	)
	if err != 0 { // type syscall.Errno
		io.WriteString(os.Stdout, "\x1b[999C\x1b[999B")
		return getCursorPosition(rows, cols)
	} else {
		*rows = int(w.Row)
		*cols = int(w.Col)
		return 0
	}
	return -1
}

/*** input ***/

func editorProcessKeypress() {
	c := editorReadKey()
	switch c {
	case ('q' & 0x1f):
		io.WriteString(os.Stdout, "\x1b[2J")
		io.WriteString(os.Stdout, "\x1b[H")
		disableRawMode()
		os.Exit(0)
	}
}

/*** append buffer ***/

type abuf struct {
	buf []byte
}

func (p abuf) String() string {
	return string(p.buf)
}

func (p *abuf) abAppend(s string) {
	p.buf = append(p.buf, []byte(s)...)
}

/*** output ***/

func editorRefreshScreen() {
	var ab abuf
	ab.abAppend("\x1b[25l")
	ab.abAppend("\x1b[2J")
	ab.abAppend("\x1b[H")
	editorDrawRows(&ab)
	ab.abAppend("\x1b[H")
	ab.abAppend("\x1b[25h")
	io.WriteString(os.Stdout, ab.String())
}

func editorDrawRows(ab *abuf) {
	for y := 0; y < E.screenRows-1; y++ {
		ab.abAppend("~\r\n")
	}
	ab.abAppend("~")
}

/*** init ***/

func initEditor() {
	if getWindowSize(&E.screenRows, &E.screenCols) == -1 {
		die(fmt.Errorf("couldn't get screen size"))
	}
}

func main() {
	enableRawMode()
	defer disableRawMode()
	initEditor()

	for {
		editorRefreshScreen()
		editorProcessKeypress()
	}
}
<|endoftext|>


<commit_before>// Copyright 2017 Jeff Foley. All rights reserved.
// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.

package format

import (
	"fmt"
	"io"
	"net"
	"strconv"
	"strings"

	amassnet "github.com/OWASP/Amass/v3/net"
	"github.com/OWASP/Amass/v3/requests"
	"github.com/fatih/color"
)

// Banner is the ASCII art logo used within help output.
const Banner = `
        .+++:.            :                             .+++.
      +W@@@@@@8        &+W@#               o8W8:      +W@@@@@@#.   oW@@@W#+
     &@#+   .o@##.    .@@@o@W.o@@o       :@@#&W8o    .@#:  .:oW+  .@#+++&#&
    +@&        &@&     #@8 +@W@&8@+     :@W.   +@8   +@:          .@8
    8@          @@     8@o  8@8  WW    .@W      W@+  .@W.          o@#:
    WW          &@o    &@:  o@+  o@+   #@.      8@o   +W@#+.        +W@8:
    #@          :@W    &@+  &@+   @8  :@o       o@o     oW@@W+        oW@8
    o@+          @@&   &@+  &@+   #@  &@.      .W@W       .+#@&         o@W.
     WW         +@W@8. &@+  :&    o@+ #@      :@W&@&         &@:  ..     :@o
     :@W:      o@# +Wo &@+        :W: +@W&o++o@W. &@&  8@#o+&@W.  #@:    o@+
      :W@@WWWW@@8       +              :&W@@@@&    &W  .o#@@W&.   :W@WWW@@&
        +o&&&&+.                                                    +oooo.
`

const (
	// Version is used to display the current version of Amass.
	Version = "v3.11.6"

	// Author is used to display the Amass Project Team.
	Author = "OWASP Amass Project - @owaspamass"

	// Description is the slogan for the Amass Project.
	Description = "In-depth Attack Surface Mapping and Asset Discovery"
)

var (
	// Colors used to ease the reading of program output
	g      = color.New(color.FgHiGreen)
	b      = color.New(color.FgHiBlue)
	yellow = color.New(color.FgHiYellow).SprintFunc()
	green  = color.New(color.FgHiGreen).SprintFunc()
	blue   = color.New(color.FgHiBlue).SprintFunc()
)

// ASNSummaryData stores information related to discovered ASs and netblocks.
type ASNSummaryData struct {
	Name      string
	Netblocks map[string]int
}

// UpdateSummaryData updates the summary maps using the provided requests.Output data.
func UpdateSummaryData(output *requests.Output, tags map[string]int, asns map[int]*ASNSummaryData) {
	tags[output.Tag]++

	for _, addr := range output.Addresses {
		if addr.CIDRStr == "" {
			continue
		}

		data, found := asns[addr.ASN]
		if !found {
			asns[addr.ASN] = &ASNSummaryData{
				Name:      addr.Description,
				Netblocks: make(map[string]int),
			}
			data = asns[addr.ASN]
		}
		// Increment how many IPs were in this netblock
		data.Netblocks[addr.CIDRStr]++
	}
}

// PrintEnumerationSummary outputs the summary information utilized by the command-line tools.
func PrintEnumerationSummary(total int, tags map[string]int, asns map[int]*ASNSummaryData, demo bool) {
	FprintEnumerationSummary(color.Error, total, tags, asns, demo)
}

// FprintEnumerationSummary outputs the summary information utilized by the command-line tools.
func FprintEnumerationSummary(out io.Writer, total int, tags map[string]int, asns map[int]*ASNSummaryData, demo bool) {
	pad := func(num int, chr string) {
		for i := 0; i < num; i++ {
			b.Fprint(out, chr)
		}
	}

	fmt.Fprintln(out)
	// Print the header information
	title := "OWASP Amass "
	site := "https://github.com/OWASP/Amass"
	b.Fprint(out, title+Version)
	num := 80 - (len(title) + len(Version) + len(site))
	pad(num, " ")
	b.Fprintf(out, "%s\n", site)
	pad(8, "----------")
	fmt.Fprintf(out, "\n%s%s", yellow(strconv.Itoa(total)), green(" names discovered - "))
	// Print the stats using tag information
	num, length := 1, len(tags)
	for k, v := range tags {
		fmt.Fprintf(out, "%s: %s", green(k), yellow(strconv.Itoa(v)))
		if num < length {
			g.Fprint(out, ", ")
		}
		num++
	}
	fmt.Fprintln(out)

	if len(asns) == 0 {
		return
	}
	// Another line gets printed
	pad(8, "----------")
	fmt.Fprintln(out)
	// Print the ASN and netblock information
	for asn, data := range asns {
		asnstr := strconv.Itoa(asn)
		datastr := data.Name

		if demo && asn > 0 {
			asnstr = censorString(asnstr, 0, len(asnstr))
			datastr = censorString(datastr, 0, len(datastr))
		}
		fmt.Fprintf(out, "%s%s %s %s\n", blue("ASN: "), yellow(asnstr), green("-"), green(datastr))

		for cidr, ips := range data.Netblocks {
			countstr := strconv.Itoa(ips)
			cidrstr := cidr

			if demo {
				cidrstr = censorNetBlock(cidrstr)
			}

			countstr = fmt.Sprintf("\t%-4s", countstr)
			cidrstr = fmt.Sprintf("\t%-18s", cidrstr)
			fmt.Fprintf(out, "%s%s %s\n", yellow(cidrstr), yellow(countstr), blue("Subdomain Name(s)"))
		}
	}
}

// PrintBanner outputs the Amass banner the same for all tools.
func PrintBanner() {
	FprintBanner(color.Error)
}

// FprintBanner outputs the Amass banner the same for all tools.
func FprintBanner(out io.Writer) {
	y := color.New(color.FgHiYellow)
	r := color.New(color.FgHiRed)
	rightmost := 76

	pad := func(num int) {
		for i := 0; i < num; i++ {
			fmt.Fprint(out, " ")
		}
	}
	r.Fprintln(out, Banner)
	pad(rightmost - len(Version))
	y.Fprintln(out, Version)
	pad(rightmost - len(Author))
	y.Fprintln(out, Author)
	pad(rightmost - len(Description))
	y.Fprintf(out, "%s\n\n\n", Description)
}

func censorDomain(input string) string {
	return censorString(input, strings.Index(input, "."), len(input))
}

func censorIP(input string) string {
	return censorString(input, 0, strings.LastIndex(input, "."))
}

func censorNetBlock(input string) string {
	return censorString(input, 0, strings.Index(input, "/"))
}

func censorString(input string, start, end int) string {
	runes := []rune(input)
	for i := start; i < end; i++ {
		if runes[i] == '.' ||
			runes[i] == '/' ||
			runes[i] == '-' ||
			runes[i] == ' ' {
			continue
		}
		runes[i] = 'x'
	}
	return string(runes)
}

// OutputLineParts returns the parts of a line to be printed for a requests.Output.
func OutputLineParts(out *requests.Output, src, addrs, demo bool) (source, name, ips string) {
	if src {
		source = fmt.Sprintf("%-18s", "["+out.Sources[0]+"] ")
	}
	if addrs {
		for i, a := range out.Addresses {
			if i != 0 {
				ips += ","
			}
			if demo {
				ips += censorIP(a.Address.String())
			} else {
				ips += a.Address.String()
			}
		}
		if ips == "" {
			ips = "N/A"
		}
	}
	name = out.Name
	if demo {
		name = censorDomain(name)
	}
	return
}

// DesiredAddrTypes removes undesired address types from the AddressInfo slice.
func DesiredAddrTypes(addrs []requests.AddressInfo, ipv4, ipv6 bool) []requests.AddressInfo {
	if !ipv4 && !ipv6 {
		return addrs
	}

	var keep []requests.AddressInfo
	for _, addr := range addrs {
		if amassnet.IsIPv4(addr.Address) && !ipv4 {
			continue
		} else if amassnet.IsIPv6(addr.Address) && !ipv6 {
			continue
		}
		keep = append(keep, addr)
	}
	return keep
}

// InterfaceInfo returns network interface information specific to the current host.
func InterfaceInfo() string {
	var output string

	if ifaces, err := net.Interfaces(); err == nil {
		for _, i := range ifaces {
			addrs, err := i.Addrs()
			if err != nil {
				continue
			}
			output += fmt.Sprintf("%s%s%s\n", blue(i.Name+": "), green("flags="), yellow("<"+strings.ToUpper(i.Flags.String()+">")))
			if i.HardwareAddr.String() != "" {
				output += fmt.Sprintf("\t%s%s\n", green("ether: "), yellow(i.HardwareAddr.String()))
			}
			for _, addr := range addrs {
				inet := "inet"
				if a, ok := addr.(*net.IPNet); ok && amassnet.IsIPv6(a.IP) {
					inet += "6"
				}
				inet += ": "
				output += fmt.Sprintf("\t%s%s\n", green(inet), yellow(addr.String()))
			}
		}
	}

	return output
}
<commit_msg>v3.11.7 release<commit_after>// Copyright 2017 Jeff Foley. All rights reserved.
// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.

package format

import (
	"fmt"
	"io"
	"net"
	"strconv"
	"strings"

	amassnet "github.com/OWASP/Amass/v3/net"
	"github.com/OWASP/Amass/v3/requests"
	"github.com/fatih/color"
)

// Banner is the ASCII art logo used within help output.
const Banner = `
        .+++:.            :                             .+++.
      +W@@@@@@8        &+W@#               o8W8:      +W@@@@@@#.   oW@@@W#+
     &@#+   .o@##.    .@@@o@W.o@@o       :@@#&W8o    .@#:  .:oW+  .@#+++&#&
    +@&        &@&     #@8 +@W@&8@+     :@W.   +@8   +@:          .@8
    8@          @@     8@o  8@8  WW    .@W      W@+  .@W.          o@#:
    WW          &@o    &@:  o@+  o@+   #@.      8@o   +W@#+.        +W@8:
    #@          :@W    &@+  &@+   @8  :@o       o@o     oW@@W+        oW@8
    o@+          @@&   &@+  &@+   #@  &@.      .W@W       .+#@&         o@W.
     WW         +@W@8. &@+  :&    o@+ #@      :@W&@&         &@:  ..     :@o
     :@W:      o@# +Wo &@+        :W: +@W&o++o@W. &@&  8@#o+&@W.  #@:    o@+
      :W@@WWWW@@8       +              :&W@@@@&    &W  .o#@@W&.   :W@WWW@@&
        +o&&&&+.                                                    +oooo.
`

const (
	// Version is used to display the current version of Amass.
	Version = "v3.11.7"

	// Author is used to display the Amass Project Team.
	Author = "OWASP Amass Project - @owaspamass"

	// Description is the slogan for the Amass Project.
	Description = "In-depth Attack Surface Mapping and Asset Discovery"
)

var (
	// Colors used to ease the reading of program output
	g      = color.New(color.FgHiGreen)
	b      = color.New(color.FgHiBlue)
	yellow = color.New(color.FgHiYellow).SprintFunc()
	green  = color.New(color.FgHiGreen).SprintFunc()
	blue   = color.New(color.FgHiBlue).SprintFunc()
)

// ASNSummaryData stores information related to discovered ASs and netblocks.
type ASNSummaryData struct {
	Name      string
	Netblocks map[string]int
}

// UpdateSummaryData updates the summary maps using the provided requests.Output data.
func UpdateSummaryData(output *requests.Output, tags map[string]int, asns map[int]*ASNSummaryData) {
	tags[output.Tag]++

	for _, addr := range output.Addresses {
		if addr.CIDRStr == "" {
			continue
		}

		data, found := asns[addr.ASN]
		if !found {
			asns[addr.ASN] = &ASNSummaryData{
				Name:      addr.Description,
				Netblocks: make(map[string]int),
			}
			data = asns[addr.ASN]
		}
		// Increment how many IPs were in this netblock
		data.Netblocks[addr.CIDRStr]++
	}
}

// PrintEnumerationSummary outputs the summary information utilized by the command-line tools.
func PrintEnumerationSummary(total int, tags map[string]int, asns map[int]*ASNSummaryData, demo bool) {
	FprintEnumerationSummary(color.Error, total, tags, asns, demo)
}

// FprintEnumerationSummary outputs the summary information utilized by the command-line tools.
func FprintEnumerationSummary(out io.Writer, total int, tags map[string]int, asns map[int]*ASNSummaryData, demo bool) {
	pad := func(num int, chr string) {
		for i := 0; i < num; i++ {
			b.Fprint(out, chr)
		}
	}

	fmt.Fprintln(out)
	// Print the header information
	title := "OWASP Amass "
	site := "https://github.com/OWASP/Amass"
	b.Fprint(out, title+Version)
	num := 80 - (len(title) + len(Version) + len(site))
	pad(num, " ")
	b.Fprintf(out, "%s\n", site)
	pad(8, "----------")
	fmt.Fprintf(out, "\n%s%s", yellow(strconv.Itoa(total)), green(" names discovered - "))
	// Print the stats using tag information
	num, length := 1, len(tags)
	for k, v := range tags {
		fmt.Fprintf(out, "%s: %s", green(k), yellow(strconv.Itoa(v)))
		if num < length {
			g.Fprint(out, ", ")
		}
		num++
	}
	fmt.Fprintln(out)

	if len(asns) == 0 {
		return
	}
	// Another line gets printed
	pad(8, "----------")
	fmt.Fprintln(out)
	// Print the ASN and netblock information
	for asn, data := range asns {
		asnstr := strconv.Itoa(asn)
		datastr := data.Name

		if demo && asn > 0 {
			asnstr = censorString(asnstr, 0, len(asnstr))
			datastr = censorString(datastr, 0, len(datastr))
		}
		fmt.Fprintf(out, "%s%s %s %s\n", blue("ASN: "), yellow(asnstr), green("-"), green(datastr))

		for cidr, ips := range data.Netblocks {
			countstr := strconv.Itoa(ips)
			cidrstr := cidr

			if demo {
				cidrstr = censorNetBlock(cidrstr)
			}

			countstr = fmt.Sprintf("\t%-4s", countstr)
			cidrstr = fmt.Sprintf("\t%-18s", cidrstr)
			fmt.Fprintf(out, "%s%s %s\n", yellow(cidrstr), yellow(countstr), blue("Subdomain Name(s)"))
		}
	}
}

// PrintBanner outputs the Amass banner the same for all tools.
func PrintBanner() {
	FprintBanner(color.Error)
}

// FprintBanner outputs the Amass banner the same for all tools.
func FprintBanner(out io.Writer) {
	y := color.New(color.FgHiYellow)
	r := color.New(color.FgHiRed)
	rightmost := 76

	pad := func(num int) {
		for i := 0; i < num; i++ {
			fmt.Fprint(out, " ")
		}
	}
	r.Fprintln(out, Banner)
	pad(rightmost - len(Version))
	y.Fprintln(out, Version)
	pad(rightmost - len(Author))
	y.Fprintln(out, Author)
	pad(rightmost - len(Description))
	y.Fprintf(out, "%s\n\n\n", Description)
}

func censorDomain(input string) string {
	return censorString(input, strings.Index(input, "."), len(input))
}

func censorIP(input string) string {
	return censorString(input, 0, strings.LastIndex(input, "."))
}

func censorNetBlock(input string) string {
	return censorString(input, 0, strings.Index(input, "/"))
}

func censorString(input string, start, end int) string {
	runes := []rune(input)
	for i := start; i < end; i++ {
		if runes[i] == '.' ||
			runes[i] == '/' ||
			runes[i] == '-' ||
			runes[i] == ' ' {
			continue
		}
		runes[i] = 'x'
	}
	return string(runes)
}

// OutputLineParts returns the parts of a line to be printed for a requests.Output.
func OutputLineParts(out *requests.Output, src, addrs, demo bool) (source, name, ips string) {
	if src {
		source = fmt.Sprintf("%-18s", "["+out.Sources[0]+"] ")
	}
	if addrs {
		for i, a := range out.Addresses {
			if i != 0 {
				ips += ","
			}
			if demo {
				ips += censorIP(a.Address.String())
			} else {
				ips += a.Address.String()
			}
		}
		if ips == "" {
			ips = "N/A"
		}
	}
	name = out.Name
	if demo {
		name = censorDomain(name)
	}
	return
}

// DesiredAddrTypes removes undesired address types from the AddressInfo slice.
func DesiredAddrTypes(addrs []requests.AddressInfo, ipv4, ipv6 bool) []requests.AddressInfo {
	if !ipv4 && !ipv6 {
		return addrs
	}

	var keep []requests.AddressInfo
	for _, addr := range addrs {
		if amassnet.IsIPv4(addr.Address) && !ipv4 {
			continue
		} else if amassnet.IsIPv6(addr.Address) && !ipv6 {
			continue
		}
		keep = append(keep, addr)
	}
	return keep
}

// InterfaceInfo returns network interface information specific to the current host.
func InterfaceInfo() string {
	var output string

	if ifaces, err := net.Interfaces(); err == nil {
		for _, i := range ifaces {
			addrs, err := i.Addrs()
			if err != nil {
				continue
			}
			output += fmt.Sprintf("%s%s%s\n", blue(i.Name+": "), green("flags="), yellow("<"+strings.ToUpper(i.Flags.String()+">")))
			if i.HardwareAddr.String() != "" {
				output += fmt.Sprintf("\t%s%s\n", green("ether: "), yellow(i.HardwareAddr.String()))
			}
			for _, addr := range addrs {
				inet := "inet"
				if a, ok := addr.(*net.IPNet); ok && amassnet.IsIPv6(a.IP) {
					inet += "6"
				}
				inet += ": "
				output += fmt.Sprintf("\t%s%s\n", green(inet), yellow(addr.String()))
			}
		}
	}

	return output
}
<|endoftext|>


<commit_before>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package cpumanager

import (
	"fmt"
	"sort"

	"k8s.io/klog"

	"k8s.io/kubernetes/pkg/kubelet/cm/cpumanager/topology"
	"k8s.io/kubernetes/pkg/kubelet/cm/cpuset"
)

type cpuAccumulator struct {
	topo          *topology.CPUTopology
	details       topology.CPUDetails
	numCPUsNeeded int
	result        cpuset.CPUSet
}

func newCPUAccumulator(topo *topology.CPUTopology, availableCPUs cpuset.CPUSet, numCPUs int) *cpuAccumulator {
	return &cpuAccumulator{
		topo:          topo,
		details:       topo.CPUDetails.KeepOnly(availableCPUs),
		numCPUsNeeded: numCPUs,
		result:        cpuset.NewCPUSet(),
	}
}

func (a *cpuAccumulator) take(cpus cpuset.CPUSet) {
	a.result = a.result.Union(cpus)
	a.details = a.details.KeepOnly(a.details.CPUs().Difference(a.result))
	a.numCPUsNeeded -= cpus.Size()
}

// Returns true if the supplied socket is fully available in `topoDetails`.
func (a *cpuAccumulator) isSocketFree(socketID int) bool {
	return a.details.CPUsInSocket(socketID).Size() == a.topo.CPUsPerSocket()
}

// Returns true if the supplied core is fully available in `topoDetails`.
func (a *cpuAccumulator) isCoreFree(coreID int) bool {
	return a.details.CPUsInCore(coreID).Size() == a.topo.CPUsPerCore()
}

// Returns free socket IDs as a slice sorted by:
// - socket ID, ascending.
func (a *cpuAccumulator) freeSockets() []int {
	return a.details.Sockets().Filter(a.isSocketFree).ToSlice()
}

// Returns core IDs as a slice sorted by:
// - the number of whole available cores on the socket, ascending
// - socket ID, ascending
// - core ID, ascending
func (a *cpuAccumulator) freeCores() []int {
	socketIDs := a.details.Sockets().ToSlice()
	sort.Slice(socketIDs,
		func(i, j int) bool {
			iCores := a.details.CoresInSocket(socketIDs[i]).Filter(a.isCoreFree)
			jCores := a.details.CoresInSocket(socketIDs[j]).Filter(a.isCoreFree)
			return iCores.Size() < jCores.Size() || socketIDs[i] < socketIDs[j]
		})

	coreIDs := []int{}
	for _, s := range socketIDs {
		coreIDs = append(coreIDs, a.details.CoresInSocket(s).Filter(a.isCoreFree).ToSlice()...)
	}
	return coreIDs
}

// Returns CPU IDs as a slice sorted by:
// - socket affinity with result
// - number of CPUs available on the same socket
// - number of CPUs available on the same core
// - socket ID.
// - core ID.
func (a *cpuAccumulator) freeCPUs() []int {
	result := []int{}
	cores := a.details.Cores().ToSlice()

	sort.Slice(
		cores,
		func(i, j int) bool {
			iCore := cores[i]
			jCore := cores[j]

			iCPUs := a.topo.CPUDetails.CPUsInCore(iCore).ToSlice()
			jCPUs := a.topo.CPUDetails.CPUsInCore(jCore).ToSlice()

			iSocket := a.topo.CPUDetails[iCPUs[0]].SocketID
			jSocket := a.topo.CPUDetails[jCPUs[0]].SocketID

			// Compute the number of CPUs in the result reside on the same socket
			// as each core.
			iSocketColoScore := a.topo.CPUDetails.CPUsInSocket(iSocket).Intersection(a.result).Size()
			jSocketColoScore := a.topo.CPUDetails.CPUsInSocket(jSocket).Intersection(a.result).Size()

			// Compute the number of available CPUs available on the same socket
			// as each core.
			iSocketFreeScore := a.details.CPUsInSocket(iSocket).Size()
			jSocketFreeScore := a.details.CPUsInSocket(jSocket).Size()

			// Compute the number of available CPUs on each core.
			iCoreFreeScore := a.details.CPUsInCore(iCore).Size()
			jCoreFreeScore := a.details.CPUsInCore(jCore).Size()

			return iSocketColoScore > jSocketColoScore ||
				iSocketFreeScore < jSocketFreeScore ||
				iCoreFreeScore < jCoreFreeScore ||
				iSocket < jSocket ||
				iCore < jCore
		})

	// For each core, append sorted CPU IDs to result.
	for _, core := range cores {
		result = append(result, a.details.CPUsInCore(core).ToSlice()...)
	}
	return result
}

func (a *cpuAccumulator) needs(n int) bool {
	return a.numCPUsNeeded >= n
}

func (a *cpuAccumulator) isSatisfied() bool {
	return a.numCPUsNeeded < 1
}

func (a *cpuAccumulator) isFailed() bool {
	return a.numCPUsNeeded > a.details.CPUs().Size()
}

func takeByTopology(topo *topology.CPUTopology, availableCPUs cpuset.CPUSet, numCPUs int) (cpuset.CPUSet, error) {
	acc := newCPUAccumulator(topo, availableCPUs, numCPUs)
	if acc.isSatisfied() {
		return acc.result, nil
	}
	if acc.isFailed() {
		return cpuset.NewCPUSet(), fmt.Errorf("not enough cpus available to satisfy request")
	}

	// Algorithm: topology-aware best-fit
	// 1. Acquire whole sockets, if available and the container requires at
	//    least a socket's-worth of CPUs.
	for _, s := range acc.freeSockets() {
		if acc.needs(acc.topo.CPUsPerSocket()) {
			klog.V(4).Infof("[cpumanager] takeByTopology: claiming socket [%d]", s)
			acc.take(acc.details.CPUsInSocket(s))
			if acc.isSatisfied() {
				return acc.result, nil
			}
		}
	}

	// 2. Acquire whole cores, if available and the container requires at least
	//    a core's-worth of CPUs.
	for _, c := range acc.freeCores() {
		if acc.needs(acc.topo.CPUsPerCore()) {
			klog.V(4).Infof("[cpumanager] takeByTopology: claiming core [%d]", c)
			acc.take(acc.details.CPUsInCore(c))
			if acc.isSatisfied() {
				return acc.result, nil
			}
		}
	}

	// 3. Acquire single threads, preferring to fill partially-allocated cores
	//    on the same sockets as the whole cores we have already taken in this
	//    allocation.
	for _, c := range acc.freeCPUs() {
		klog.V(4).Infof("[cpumanager] takeByTopology: claiming CPU [%d]", c)
		if acc.needs(1) {
			acc.take(cpuset.NewCPUSet(c))
		}
		if acc.isSatisfied() {
			return acc.result, nil
		}
	}

	return cpuset.NewCPUSet(), fmt.Errorf("failed to allocate cpus")
}
<commit_msg>kubelet/cm: code optimization for the static policy<commit_after>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package cpumanager

import (
	"fmt"
	"sort"

	"k8s.io/klog"

	"k8s.io/kubernetes/pkg/kubelet/cm/cpumanager/topology"
	"k8s.io/kubernetes/pkg/kubelet/cm/cpuset"
)

type cpuAccumulator struct {
	topo          *topology.CPUTopology
	details       topology.CPUDetails
	numCPUsNeeded int
	result        cpuset.CPUSet
}

func newCPUAccumulator(topo *topology.CPUTopology, availableCPUs cpuset.CPUSet, numCPUs int) *cpuAccumulator {
	return &cpuAccumulator{
		topo:          topo,
		details:       topo.CPUDetails.KeepOnly(availableCPUs),
		numCPUsNeeded: numCPUs,
		result:        cpuset.NewCPUSet(),
	}
}

func (a *cpuAccumulator) take(cpus cpuset.CPUSet) {
	a.result = a.result.Union(cpus)
	a.details = a.details.KeepOnly(a.details.CPUs().Difference(a.result))
	a.numCPUsNeeded -= cpus.Size()
}

// Returns true if the supplied socket is fully available in `topoDetails`.
func (a *cpuAccumulator) isSocketFree(socketID int) bool {
	return a.details.CPUsInSocket(socketID).Size() == a.topo.CPUsPerSocket()
}

// Returns true if the supplied core is fully available in `topoDetails`.
func (a *cpuAccumulator) isCoreFree(coreID int) bool {
	return a.details.CPUsInCore(coreID).Size() == a.topo.CPUsPerCore()
}

// Returns free socket IDs as a slice sorted by:
// - socket ID, ascending.
func (a *cpuAccumulator) freeSockets() []int {
	return a.details.Sockets().Filter(a.isSocketFree).ToSlice()
}

// Returns core IDs as a slice sorted by:
// - the number of whole available cores on the socket, ascending
// - socket ID, ascending
// - core ID, ascending
func (a *cpuAccumulator) freeCores() []int {
	socketIDs := a.details.Sockets().ToSlice()
	sort.Slice(socketIDs,
		func(i, j int) bool {
			iCores := a.details.CoresInSocket(socketIDs[i]).Filter(a.isCoreFree)
			jCores := a.details.CoresInSocket(socketIDs[j]).Filter(a.isCoreFree)
			return iCores.Size() < jCores.Size() || socketIDs[i] < socketIDs[j]
		})

	coreIDs := []int{}
	for _, s := range socketIDs {
		coreIDs = append(coreIDs, a.details.CoresInSocket(s).Filter(a.isCoreFree).ToSlice()...)
	}
	return coreIDs
}

// Returns CPU IDs as a slice sorted by:
// - socket affinity with result
// - number of CPUs available on the same socket
// - number of CPUs available on the same core
// - socket ID.
// - core ID.
func (a *cpuAccumulator) freeCPUs() []int {
	result := []int{}
	cores := a.details.Cores().ToSlice()

	sort.Slice(
		cores,
		func(i, j int) bool {
			iCore := cores[i]
			jCore := cores[j]

			iCPUs := a.topo.CPUDetails.CPUsInCore(iCore).ToSlice()
			jCPUs := a.topo.CPUDetails.CPUsInCore(jCore).ToSlice()

			iSocket := a.topo.CPUDetails[iCPUs[0]].SocketID
			jSocket := a.topo.CPUDetails[jCPUs[0]].SocketID

			// Compute the number of CPUs in the result reside on the same socket
			// as each core.
			iSocketColoScore := a.topo.CPUDetails.CPUsInSocket(iSocket).Intersection(a.result).Size()
			jSocketColoScore := a.topo.CPUDetails.CPUsInSocket(jSocket).Intersection(a.result).Size()

			// Compute the number of available CPUs available on the same socket
			// as each core.
			iSocketFreeScore := a.details.CPUsInSocket(iSocket).Size()
			jSocketFreeScore := a.details.CPUsInSocket(jSocket).Size()

			// Compute the number of available CPUs on each core.
			iCoreFreeScore := a.details.CPUsInCore(iCore).Size()
			jCoreFreeScore := a.details.CPUsInCore(jCore).Size()

			return iSocketColoScore > jSocketColoScore ||
				iSocketFreeScore < jSocketFreeScore ||
				iCoreFreeScore < jCoreFreeScore ||
				iSocket < jSocket ||
				iCore < jCore
		})

	// For each core, append sorted CPU IDs to result.
	for _, core := range cores {
		result = append(result, a.details.CPUsInCore(core).ToSlice()...)
	}
	return result
}

func (a *cpuAccumulator) needs(n int) bool {
	return a.numCPUsNeeded >= n
}

func (a *cpuAccumulator) isSatisfied() bool {
	return a.numCPUsNeeded < 1
}

func (a *cpuAccumulator) isFailed() bool {
	return a.numCPUsNeeded > a.details.CPUs().Size()
}

func takeByTopology(topo *topology.CPUTopology, availableCPUs cpuset.CPUSet, numCPUs int) (cpuset.CPUSet, error) {
	acc := newCPUAccumulator(topo, availableCPUs, numCPUs)
	if acc.isSatisfied() {
		return acc.result, nil
	}
	if acc.isFailed() {
		return cpuset.NewCPUSet(), fmt.Errorf("not enough cpus available to satisfy request")
	}

	// Algorithm: topology-aware best-fit
	// 1. Acquire whole sockets, if available and the container requires at
	//    least a socket's-worth of CPUs.
	if acc.needs(acc.topo.CPUsPerSocket()) {
		for _, s := range acc.freeSockets() {
			klog.V(4).Infof("[cpumanager] takeByTopology: claiming socket [%d]", s)
			acc.take(acc.details.CPUsInSocket(s))
			if acc.isSatisfied() {
				return acc.result, nil
			}
			if !acc.needs(acc.topo.CPUsPerSocket()) {
				break
			}
		}
	}

	// 2. Acquire whole cores, if available and the container requires at least
	//    a core's-worth of CPUs.
	if acc.needs(acc.topo.CPUsPerCore()) {
		for _, c := range acc.freeCores() {
			klog.V(4).Infof("[cpumanager] takeByTopology: claiming core [%d]", c)
			acc.take(acc.details.CPUsInCore(c))
			if acc.isSatisfied() {
				return acc.result, nil
			}
			if !acc.needs(acc.topo.CPUsPerCore()) {
				break
			}
		}
	}

	// 3. Acquire single threads, preferring to fill partially-allocated cores
	//    on the same sockets as the whole cores we have already taken in this
	//    allocation.
	for _, c := range acc.freeCPUs() {
		klog.V(4).Infof("[cpumanager] takeByTopology: claiming CPU [%d]", c)
		if acc.needs(1) {
			acc.take(cpuset.NewCPUSet(c))
		}
		if acc.isSatisfied() {
			return acc.result, nil
		}
	}

	return cpuset.NewCPUSet(), fmt.Errorf("failed to allocate cpus")
}
<|endoftext|>


<commit_before>package v1

import (
	"fmt"
	log "github.com/sirupsen/logrus"
	"runtime"
	"strings"
	"time"

	"github.com/ivanilves/lstags/api/v1/collection"
	dockerclient "github.com/ivanilves/lstags/docker/client"
	dockerconfig "github.com/ivanilves/lstags/docker/config"
	"github.com/ivanilves/lstags/repository"
	"github.com/ivanilves/lstags/tag"
	"github.com/ivanilves/lstags/tag/local"
	"github.com/ivanilves/lstags/tag/remote"
	"github.com/ivanilves/lstags/util/wait"
)

// Config holds API instance configuration
type Config struct {
	DockerJSONConfigFile string
	ConcurrentRequests   int
	TraceRequests        bool
	RetryRequests        int
	RetryDelay           time.Duration
	InsecureRegistryEx   string
	VerboseLogging       bool
}

// PushConfig holds push-specific configuration
type PushConfig struct {
	Prefix        string
	Registry      string
	UpdateChanged bool
}

// API represents application API instance
type API struct {
	config       Config
	dockerClient *dockerclient.DockerClient
}

// fn gives the name of the calling function (e.g. enriches log.Debugf() output)
// + optionally attaches free form string labels (mainly to identify goroutines)
func fn(labels ...string) string {
	function, _, _, _ := runtime.Caller(1)

	longname := runtime.FuncForPC(function).Name()

	nameparts := strings.Split(longname, ".")
	shortname := nameparts[len(nameparts)-1]

	if labels == nil {
		return fmt.Sprintf("[%s()]", shortname)
	}

	return fmt.Sprintf("[%s():%s]", shortname, strings.Join(labels, ":"))
}

// CollectTags collects information on tags present in remote registry and [local] Docker daemon,
// makes required comparisons between them and spits organized info back as collection.Collection
func (api *API) CollectTags(refs []string) (*collection.Collection, error) {
	if len(refs) == 0 {
		return nil, fmt.Errorf("no image references passed")
	}

	log.Debugf("%s references: %+v", fn(), refs)

	repos, err := repository.ParseRefs(refs)
	if err != nil {
		return nil, err
	}
	for _, repo := range repos {
		log.Debugf("%s repository: %+v", fn(), repo)
	}

	done := make(chan error, len(repos))
	tags := make(map[string][]*tag.Tag)

	for _, repo := range repos {
		go func(repo *repository.Repository, done chan error) {
			log.Infof("ANALYZE %s", repo.Ref())

			username, password, _ := api.dockerClient.Config().GetCredentials(repo.Registry())

			remoteTags, err := remote.FetchTags(repo, username, password)
			if err != nil {
				done <- err
				return
			}
			log.Debugf("%s remote tags: %+v", fn(repo.Ref()), remoteTags)

			localTags, _ := local.FetchTags(repo, api.dockerClient)

			log.Debugf("%s local tags: %+v", fn(repo.Ref()), localTags)

			sortedKeys, tagNames, joinedTags := tag.Join(
				remoteTags,
				localTags,
				repo.Tags(),
			)
			log.Debugf("%s joined tags: %+v", fn(repo.Ref()), joinedTags)

			tags[repo.Ref()] = tag.Collect(sortedKeys, tagNames, joinedTags)

			done <- nil

			log.Infof("FETCHED %s", repo.Ref())

			return
		}(repo, done)
	}

	if err := wait.Until(done); err != nil {
		return nil, err
	}
	log.Debugf("%s tags: %+v", fn(), tags)

	return collection.New(refs, tags)
}

// CollectPushTags blends passed collection with information fetched from [local] "push" registry,
// makes required comparisons between them and spits organized info back as collection.Collection
func (api *API) CollectPushTags(cn *collection.Collection, push PushConfig) (*collection.Collection, error) {
	log.Debugf(
		"%s collection: %+v (%d repos / %d tags)",
		fn(), cn, cn.RepoCount(), cn.TagCount(),
	)
	log.Debugf("%s push config: %+v", fn(), push)

	refs := make([]string, len(cn.Refs()))
	done := make(chan error, len(cn.Refs()))
	tags := make(map[string][]*tag.Tag)

	for i, repo := range cn.Repos() {
		go func(repo *repository.Repository, i int, done chan error) {
			refs[i] = repo.Ref()

			pushPrefix := push.Prefix
			if pushPrefix == "" {
				pushPrefix = repo.PushPrefix()
			}

			var pushRepoPath string
			pushRepoPath = pushPrefix + "/" + repo.Path()
			pushRepoPath = pushRepoPath[1:] // Leading "/" in prefix should be removed!

			pushRef := fmt.Sprintf("%s/%s~/.*/", push.Registry, pushRepoPath)

			log.Debugf("%s 'push' reference: %+v", fn(repo.Ref()), pushRef)

			pushRepo, _ := repository.ParseRef(pushRef)

			log.Infof("[PULL/PUSH] ANALYZE %s => %s", repo.Ref(), pushRef)

			username, password, _ := api.dockerClient.Config().GetCredentials(push.Registry)

			pushedTags, err := remote.FetchTags(pushRepo, username, password)
			if err != nil {
				if !strings.Contains(err.Error(), "404 Not Found") {
					done <- err
					return
				}

				log.Warnf("%s repo not found: %+s", fn(repo.Ref()), pushRef)

				pushedTags = make(map[string]*tag.Tag)
			}
			log.Debugf("%s pushed tags: %+v", fn(repo.Ref()), pushedTags)

			remoteTags := cn.TagMap(repo.Ref())
			log.Debugf("%s remote tags: %+v", fn(repo.Ref()), remoteTags)

			sortedKeys, tagNames, joinedTags := tag.Join(
				remoteTags,
				pushedTags,
				repo.Tags(),
			)
			log.Debugf("%s joined tags: %+v", fn(repo.Ref()), joinedTags)

			tagsToPush := make([]*tag.Tag, 0)
			for _, key := range sortedKeys {
				name := tagNames[key]
				tg := joinedTags[name]

				if tg.NeedsPush(push.UpdateChanged) {
					tagsToPush = append(tagsToPush, tg)
				}
			}
			log.Debugf("%s tags to push: %+v", fn(repo.Ref()), tagsToPush)

			tags[repo.Ref()] = tagsToPush

			done <- nil

			return
		}(repo, i, done)
	}

	if err := wait.Until(done); err != nil {
		return nil, err
	}
	log.Debugf("%s 'push' tags: %+v", fn(), tags)

	return collection.New(refs, tags)
}

// PullTags compares images from remote registry and Docker daemon and pulls
// images that match tag spec passed and are not present in Docker daemon.
func (api *API) PullTags(cn *collection.Collection) error {
	log.Debugf(
		"%s collection: %+v (%d repos / %d tags)",
		fn(), cn, cn.RepoCount(), cn.TagCount(),
	)

	done := make(chan error, cn.TagCount())

	for _, ref := range cn.Refs() {
		repo := cn.Repo(ref)
		tags := cn.Tags(ref)

		log.Debugf("%s repository: %+v", fn(), repo)
		for _, tg := range tags {
			log.Debugf("%s tag: %+v", fn(), tg)
		}

		go func(repo *repository.Repository, tags []*tag.Tag, done chan error) {
			for _, tg := range tags {
				if !tg.NeedsPull() {
					done <- nil
					continue
				}

				ref := repo.Name() + ":" + tg.Name()

				log.Infof("PULLING %s", ref)

				done <- api.dockerClient.Pull(ref)
			}
		}(repo, tags, done)
	}

	return wait.Until(done)
}

// PushTags compares images from remote and "push" (usually local) registries,
// pulls images that are present in remote registry, but are not in "push" one
// and then [re-]pushes them to the "push" registry.
func (api *API) PushTags(cn *collection.Collection, push PushConfig) error {
	log.Debugf(
		"%s 'push' collection: %+v (%d repos / %d tags)",
		fn(), cn, cn.RepoCount(), cn.TagCount(),
	)
	log.Debugf("%s push config: %+v", fn(), push)

	done := make(chan error, cn.TagCount())

	if cn.TagCount() == 0 {
		log.Infof("%s No tags to push", fn())
		return nil
	}

	for _, ref := range cn.Refs() {
		repo := cn.Repo(ref)
		tags := cn.Tags(ref)

		log.Debugf("%s repository: %+v", fn(), repo)
		for _, tg := range tags {
			log.Debugf("%s tag: %+v", fn(), tg)
		}

		go func(repo *repository.Repository, tags []*tag.Tag, done chan error) {
			for _, tg := range tags {
				srcRef := repo.Name() + ":" + tg.Name()
				dstRef := push.Registry + push.Prefix + "/" + repo.Path() + ":" + tg.Name()

				log.Infof("[PULL/PUSH] PUSHING %s => %s", srcRef, dstRef)
				if err := api.dockerClient.RePush(srcRef, dstRef); err != nil {
					done <- err
					return
				}

				done <- nil
			}
		}(repo, tags, done)
	}

	return wait.Until(done)
}

// New creates new instance of application API
func New(config Config) (*API, error) {
	if config.VerboseLogging {
		log.SetLevel(log.DebugLevel)
	}
	log.Debugf("%s API config: %+v", fn(), config)

	if config.ConcurrentRequests == 0 {
		config.ConcurrentRequests = 1
	}
	remote.ConcurrentRequests = config.ConcurrentRequests
	remote.TraceRequests = config.TraceRequests
	remote.RetryRequests = config.RetryRequests
	remote.RetryDelay = config.RetryDelay

	dockerclient.RetryPulls = config.RetryRequests
	dockerclient.RetryDelay = config.RetryDelay

	if config.InsecureRegistryEx != "" {
		repository.InsecureRegistryEx = config.InsecureRegistryEx
	}

	if config.DockerJSONConfigFile == "" {
		config.DockerJSONConfigFile = dockerconfig.DefaultDockerJSON
	}
	dockerConfig, err := dockerconfig.Load(config.DockerJSONConfigFile)
	if err != nil {
		return nil, err
	}
	dockerClient, err := dockerclient.New(dockerConfig)
	if err != nil {
		return nil, err
	}

	return &API{
		config:       config,
		dockerClient: dockerClient,
	}, nil
}
<commit_msg>NORELEASE: Low hanging fruit of RePush<commit_after>package v1

import (
	"fmt"
	log "github.com/sirupsen/logrus"
	"runtime"
	"strings"
	"time"

	"github.com/ivanilves/lstags/api/v1/collection"
	dockerclient "github.com/ivanilves/lstags/docker/client"
	dockerconfig "github.com/ivanilves/lstags/docker/config"
	"github.com/ivanilves/lstags/repository"
	"github.com/ivanilves/lstags/tag"
	"github.com/ivanilves/lstags/tag/local"
	"github.com/ivanilves/lstags/tag/remote"
	"github.com/ivanilves/lstags/util/wait"
)

// Config holds API instance configuration
type Config struct {
	DockerJSONConfigFile string
	ConcurrentRequests   int
	TraceRequests        bool
	RetryRequests        int
	RetryDelay           time.Duration
	InsecureRegistryEx   string
	VerboseLogging       bool
}

// PushConfig holds push-specific configuration
type PushConfig struct {
	Prefix        string
	Registry      string
	UpdateChanged bool
}

// API represents application API instance
type API struct {
	config       Config
	dockerClient *dockerclient.DockerClient
}

// fn gives the name of the calling function (e.g. enriches log.Debugf() output)
// + optionally attaches free form string labels (mainly to identify goroutines)
func fn(labels ...string) string {
	function, _, _, _ := runtime.Caller(1)

	longname := runtime.FuncForPC(function).Name()

	nameparts := strings.Split(longname, ".")
	shortname := nameparts[len(nameparts)-1]

	if labels == nil {
		return fmt.Sprintf("[%s()]", shortname)
	}

	return fmt.Sprintf("[%s():%s]", shortname, strings.Join(labels, ":"))
}

// CollectTags collects information on tags present in remote registry and [local] Docker daemon,
// makes required comparisons between them and spits organized info back as collection.Collection
func (api *API) CollectTags(refs []string) (*collection.Collection, error) {
	if len(refs) == 0 {
		return nil, fmt.Errorf("no image references passed")
	}

	log.Debugf("%s references: %+v", fn(), refs)

	repos, err := repository.ParseRefs(refs)
	if err != nil {
		return nil, err
	}
	for _, repo := range repos {
		log.Debugf("%s repository: %+v", fn(), repo)
	}

	done := make(chan error, len(repos))
	tags := make(map[string][]*tag.Tag)

	for _, repo := range repos {
		go func(repo *repository.Repository, done chan error) {
			log.Infof("ANALYZE %s", repo.Ref())

			username, password, _ := api.dockerClient.Config().GetCredentials(repo.Registry())

			remoteTags, err := remote.FetchTags(repo, username, password)
			if err != nil {
				done <- err
				return
			}
			log.Debugf("%s remote tags: %+v", fn(repo.Ref()), remoteTags)

			localTags, _ := local.FetchTags(repo, api.dockerClient)

			log.Debugf("%s local tags: %+v", fn(repo.Ref()), localTags)

			sortedKeys, tagNames, joinedTags := tag.Join(
				remoteTags,
				localTags,
				repo.Tags(),
			)
			log.Debugf("%s joined tags: %+v", fn(repo.Ref()), joinedTags)

			tags[repo.Ref()] = tag.Collect(sortedKeys, tagNames, joinedTags)

			done <- nil

			log.Infof("FETCHED %s", repo.Ref())

			return
		}(repo, done)
	}

	if err := wait.Until(done); err != nil {
		return nil, err
	}
	log.Debugf("%s tags: %+v", fn(), tags)

	return collection.New(refs, tags)
}

// CollectPushTags blends passed collection with information fetched from [local] "push" registry,
// makes required comparisons between them and spits organized info back as collection.Collection
func (api *API) CollectPushTags(cn *collection.Collection, push PushConfig) (*collection.Collection, error) {
	log.Debugf(
		"%s collection: %+v (%d repos / %d tags)",
		fn(), cn, cn.RepoCount(), cn.TagCount(),
	)
	log.Debugf("%s push config: %+v", fn(), push)

	refs := make([]string, len(cn.Refs()))
	done := make(chan error, len(cn.Refs()))
	tags := make(map[string][]*tag.Tag)

	for i, repo := range cn.Repos() {
		go func(repo *repository.Repository, i int, done chan error) {
			refs[i] = repo.Ref()

			pushPrefix := push.Prefix
			if pushPrefix == "" {
				pushPrefix = repo.PushPrefix()
			}

			var pushRepoPath string
			pushRepoPath = pushPrefix + "/" + repo.Path()
			pushRepoPath = pushRepoPath[1:] // Leading "/" in prefix should be removed!

			pushRef := fmt.Sprintf("%s/%s~/.*/", push.Registry, pushRepoPath)

			log.Debugf("%s 'push' reference: %+v", fn(repo.Ref()), pushRef)

			pushRepo, _ := repository.ParseRef(pushRef)

			log.Infof("[PULL/PUSH] ANALYZE %s => %s", repo.Ref(), pushRef)

			username, password, _ := api.dockerClient.Config().GetCredentials(push.Registry)

			pushedTags, err := remote.FetchTags(pushRepo, username, password)
			if err != nil {
				if !strings.Contains(err.Error(), "404 Not Found") {
					done <- err
					return
				}

				log.Warnf("%s repo not found: %+s", fn(repo.Ref()), pushRef)

				pushedTags = make(map[string]*tag.Tag)
			}
			log.Debugf("%s pushed tags: %+v", fn(repo.Ref()), pushedTags)

			remoteTags := cn.TagMap(repo.Ref())
			log.Debugf("%s remote tags: %+v", fn(repo.Ref()), remoteTags)

			sortedKeys, tagNames, joinedTags := tag.Join(
				remoteTags,
				pushedTags,
				repo.Tags(),
			)
			log.Debugf("%s joined tags: %+v", fn(repo.Ref()), joinedTags)

			tagsToPush := make([]*tag.Tag, 0)
			for _, key := range sortedKeys {
				name := tagNames[key]
				tg := joinedTags[name]

				if tg.NeedsPush(push.UpdateChanged) {
					tagsToPush = append(tagsToPush, tg)
				}
			}
			log.Debugf("%s tags to push: %+v", fn(repo.Ref()), tagsToPush)

			tags[repo.Ref()] = tagsToPush

			done <- nil

			return
		}(repo, i, done)
	}

	if err := wait.Until(done); err != nil {
		return nil, err
	}
	log.Debugf("%s 'push' tags: %+v", fn(), tags)

	return collection.New(refs, tags)
}

// PullTags compares images from remote registry and Docker daemon and pulls
// images that match tag spec passed and are not present in Docker daemon.
func (api *API) PullTags(cn *collection.Collection) error {
	log.Debugf(
		"%s collection: %+v (%d repos / %d tags)",
		fn(), cn, cn.RepoCount(), cn.TagCount(),
	)

	done := make(chan error, cn.TagCount())

	for _, ref := range cn.Refs() {
		repo := cn.Repo(ref)
		tags := cn.Tags(ref)

		log.Debugf("%s repository: %+v", fn(), repo)
		for _, tg := range tags {
			log.Debugf("%s tag: %+v", fn(), tg)
		}

		go func(repo *repository.Repository, tags []*tag.Tag, done chan error) {
			for _, tg := range tags {
				if !tg.NeedsPull() {
					done <- nil
					continue
				}

				ref := repo.Name() + ":" + tg.Name()

				log.Infof("PULLING %s", ref)

				done <- api.dockerClient.Pull(ref)
			}
		}(repo, tags, done)
	}

	return wait.Until(done)
}

// PushTags compares images from remote and "push" (usually local) registries,
// pulls images that are present in remote registry, but are not in "push" one
// and then [re-]pushes them to the "push" registry.
func (api *API) PushTags(cn *collection.Collection, push PushConfig) error {
	log.Debugf(
		"%s 'push' collection: %+v (%d repos / %d tags)",
		fn(), cn, cn.RepoCount(), cn.TagCount(),
	)
	log.Debugf("%s push config: %+v", fn(), push)

	done := make(chan error, cn.TagCount())

	if cn.TagCount() == 0 {
		log.Infof("%s No tags to push", fn())
		return nil
	}

	for _, ref := range cn.Refs() {
		repo := cn.Repo(ref)
		tags := cn.Tags(ref)

		log.Debugf("%s repository: %+v", fn(), repo)
		for _, tg := range tags {
			log.Debugf("%s tag: %+v", fn(), tg)
		}

		go func(repo *repository.Repository, tags []*tag.Tag, done chan error) {
			for _, tg := range tags {
				srcRef := repo.Name() + ":" + tg.Name()
				dstRef := push.Registry + push.Prefix + "/" + repo.Path() + ":" + tg.Name()

				log.Infof("[PULL/PUSH] PUSHING %s => %s", srcRef, dstRef)

				done <- api.dockerClient.RePush(srcRef, dstRef)
			}
		}(repo, tags, done)
	}

	return wait.Until(done)
}

// New creates new instance of application API
func New(config Config) (*API, error) {
	if config.VerboseLogging {
		log.SetLevel(log.DebugLevel)
	}
	log.Debugf("%s API config: %+v", fn(), config)

	if config.ConcurrentRequests == 0 {
		config.ConcurrentRequests = 1
	}
	remote.ConcurrentRequests = config.ConcurrentRequests
	remote.TraceRequests = config.TraceRequests
	remote.RetryRequests = config.RetryRequests
	remote.RetryDelay = config.RetryDelay

	dockerclient.RetryPulls = config.RetryRequests
	dockerclient.RetryDelay = config.RetryDelay

	if config.InsecureRegistryEx != "" {
		repository.InsecureRegistryEx = config.InsecureRegistryEx
	}

	if config.DockerJSONConfigFile == "" {
		config.DockerJSONConfigFile = dockerconfig.DefaultDockerJSON
	}
	dockerConfig, err := dockerconfig.Load(config.DockerJSONConfigFile)
	if err != nil {
		return nil, err
	}
	dockerClient, err := dockerclient.New(dockerConfig)
	if err != nil {
		return nil, err
	}

	return &API{
		config:       config,
		dockerClient: dockerClient,
	}, nil
}
<|endoftext|>


<commit_before>// Commands are the abstraction of the actual commands being executed on the
// target. Such an abstraction is helpful as they provide the possibility to
// add shortcuts for common tasks and more complex commands. There are some
// interfaces that must or can be implemented, depending on the required
// features.
//
// For further information see http://urknall.dynport.de/docs/library/#commands.
package cmd

import "io"

// All commands must implement this interface. The Shell method returns the
// command actually executed on the target.
type Command interface {
	Shell() string
}

// If not implemented by a command the string returned by the Shell method
// will be used for logging. If this method is implemented the returned string
// will be used instead.
type Logger interface {
	Logging() string
}

type StdinConsumer interface {
	Input() io.ReadCloser
}

// Interface that allows for rendering template content into a structure. Implement this interface for commands that
// should have the ability for templating. For example the ShellCommand provided by `urknall init` implements this,
// allowing for substitution of a package's values in the command.
type Renderer interface {
	Render(i interface{})
}

// This interface can be implemented by commands that need to make sure the
// configuration is valid. This helps to let the command fail as early and
// graceful as possible.
type Validator interface {
	Validate() error
}
<commit_msg>improved command interfaces' documentation<commit_after>// The Command Interfaces
//
// This package contains a set of interfaces, commands must or can implement.
package cmd

import "io"

// The Command interface is used to have specialized commands that are used for
// execution and logging (the latter is useful to hide the gory details of more
// complex commands).
type Command interface {
	Shell() string
}

// The Logger interface should be implemented by commands, which hide their
// intent behind a series of complex shell commands. The returned string will
// be printed instead of the raw output of the Shell function.
type Logger interface {
	Logging() string
}

// If a command needs to send something to the remote host (a file for example)
// the content can be made available on standard input of the remote command.
// The command must make sure that changed local content will reissue execution
// of the command (by printing the content's hash to standard output for
// example).
type StdinConsumer interface {
	Input() io.ReadCloser
}

// Often it is convenient to directly use values or methods of the template in
// the commands (using go's templating mechanism).
type Renderer interface {
	Render(i interface{})
}

// Interface used for types that will validate its state. An error is returned
// if the state is invalid.
type Validator interface {
	Validate() error
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"math"
)

func reverse(n int) int {
	divider := 10

	result := 0

	result += n % divider
	n /= divider

	// divider *= 10

	fmt.Println(result)
	fmt.Println(n)
	fmt.Println("-----")

	result += (n % divider) * divider
	n /= divider

	fmt.Println(result)
	fmt.Println(n)

	// result += (result % divider) * divider

	// result += n % 1000

	return result
}

func isPalindrome(n int) bool {
	return n - reverse(n) == 0
}

func findLargestPalindrome(digitNum int) int {
	limit := int(math.Pow(10, float64(digitNum)) - 1);
	for x := limit; x >= 0; x-- {
		for y := limit; y >= 0; y-- {
			if (isPalindrome(x * y)) {
				return x * y
			}
		}
	}

	return 0
}

func main() {
	// fmt.Println(reverse(120))
	reverse(123)
	// fmts.Println(reverse(1))

	// fmt.Printf("The largest palindrome made from the product of two 3-digit numbers is %d.\n", findLargestPalindrome(3))
}<commit_msg>Remove stupid solutions.<commit_after>package main

import (
	"fmt"
	"math"
)

func isPalindrome(n int) bool {
	return true
}

func findLargestPalindrome(digitNum int) int {
	return 0
}

func main() {

	n := 123

	fmt.Println("length", int(math.Floor(math.Log10(float64(n))) + 1))

	// fmt.Printf("The largest palindrome made from the product of two 3-digit numbers is %d.\n", findLargestPalindrome(3))
}<|endoftext|>


<commit_before>package couchdb

// Cursor holds a pointer in a couchdb map reduce results
type Cursor struct {
	Limit     int
	Done      bool
	NextKey   interface{}
	NextDocID string
}

// ApplyTo applies the cursor to a ViewRequest
// the transformed ViewRequest will retrive elements from Cursor to
// Limit or StartKey whichever comes first
// Mutates req
func (c *Cursor) ApplyTo(req *ViewRequest) *ViewRequest {
	if c.NextKey != "" {
		if req.Key != nil && req.StartKey == nil {
			req.StartKey = req.Key
			req.EndKey = req.Key
			req.InclusiveEnd = true
			req.Key = nil
		}

		req.StartKey = c.NextKey
		if c.NextDocID != "" {
			req.StartKeyDocID = c.NextDocID
		}
	}

	if c.Limit != 0 {
		req.Limit = c.Limit + 1
	}

	return req
}

// UpdateFrom change the cursor status depending on information from
// the view's response
func (c *Cursor) UpdateFrom(res *ViewResponse) {
	lrows := len(res.Rows)
	if lrows <= c.Limit {
		c.Done = true
		c.NextKey = nil
		c.NextDocID = ""
	} else {
		c.Done = false
		next := res.Rows[len(res.Rows)-1]
		res.Rows = res.Rows[:len(res.Rows)-1]
		c.NextKey = next.Key
		c.NextDocID = next.ID
	}
}

// GetNextCursor returns a cursor to the end of a ViewResponse
// it removes the last item from the response to create a Cursor
func GetNextCursor(res *ViewResponse) *Cursor {
	if len(res.Rows) == 0 {
		return &Cursor{}
	}
	next := res.Rows[len(res.Rows)-1]
	res.Rows = res.Rows[:len(res.Rows)-1]

	return &Cursor{
		NextKey:   next.Key,
		NextDocID: next.ID,
	}
}
<commit_msg>[misc] code style fix<commit_after>package couchdb

// Cursor holds a pointer in a couchdb map reduce results
type Cursor struct {
	Limit     int
	Done      bool
	NextKey   interface{}
	NextDocID string
}

// ApplyTo applies the cursor to a ViewRequest
// the transformed ViewRequest will retrive elements from Cursor to
// Limit or StartKey whichever comes first
// Mutates req
func (c *Cursor) ApplyTo(req *ViewRequest) *ViewRequest {
	if c.NextKey != "" {
		if req.Key != nil && req.StartKey == nil {
			req.StartKey = req.Key
			req.EndKey = req.Key
			req.InclusiveEnd = true
			req.Key = nil
		}

		req.StartKey = c.NextKey
		if c.NextDocID != "" {
			req.StartKeyDocID = c.NextDocID
		}
	}

	if c.Limit != 0 {
		req.Limit = c.Limit + 1
	}

	return req
}

// UpdateFrom change the cursor status depending on information from
// the view's response
func (c *Cursor) UpdateFrom(res *ViewResponse) {
	lrows := len(res.Rows)
	if lrows <= c.Limit {
		c.Done = true
		c.NextKey = nil
		c.NextDocID = ""
	} else {
		c.Done = false
		next := res.Rows[lrows-1]
		res.Rows = res.Rows[:lrows-1]
		c.NextKey = next.Key
		c.NextDocID = next.ID
	}
}

// GetNextCursor returns a cursor to the end of a ViewResponse
// it removes the last item from the response to create a Cursor
func GetNextCursor(res *ViewResponse) *Cursor {
	if len(res.Rows) == 0 {
		return &Cursor{}
	}
	next := res.Rows[len(res.Rows)-1]
	res.Rows = res.Rows[:len(res.Rows)-1]

	return &Cursor{
		NextKey:   next.Key,
		NextDocID: next.ID,
	}
}
<|endoftext|>


<commit_before>package taskrunner

import (
	"context"
	"fmt"

	log "github.com/hashicorp/go-hclog"
	multierror "github.com/hashicorp/go-multierror"
	"github.com/hashicorp/nomad/client/allocrunner/interfaces"
	"github.com/hashicorp/nomad/nomad/structs"
	"github.com/hashicorp/nomad/plugins/drivers"
)

type volumeHook struct {
	alloc  *structs.Allocation
	runner *TaskRunner
	logger log.Logger
}

func newVolumeHook(runner *TaskRunner, logger log.Logger) *volumeHook {
	h := &volumeHook{
		alloc:  runner.Alloc(),
		runner: runner,
	}
	h.logger = logger.Named(h.Name())
	return h
}

func (*volumeHook) Name() string {
	return "volumes"
}

func validateHostVolumes(requestedByAlias map[string]*structs.VolumeRequest, clientVolumesByName map[string]*structs.ClientHostVolumeConfig) error {
	var result error

	for _, req := range requestedByAlias {
		// This is a defensive check, but this function should only ever receive
		// host-type volumes.
		if req.Type != structs.VolumeTypeHost {
			continue
		}

		_, ok := clientVolumesByName[req.Source]
		if !ok {
			result = multierror.Append(result, fmt.Errorf("missing %s", req.Source))
		}
	}

	return result
}

// hostVolumeMountConfigurations takes the users requested volume mounts,
// volumes, and the client host volume configuration and converts them into a
// format that can be used by drivers.
func (h *volumeHook) hostVolumeMountConfigurations(taskMounts []*structs.VolumeMount, taskVolumesByAlias map[string]*structs.VolumeRequest, clientVolumesByName map[string]*structs.ClientHostVolumeConfig) ([]*drivers.MountConfig, error) {
	var mounts []*drivers.MountConfig
	for _, m := range taskMounts {
		req, ok := taskVolumesByAlias[m.Volume]
		if !ok {
			// This function receives only the task volumes that are of type Host,
			// if we can't find a group volume then we assume the mount is for another
			// type.
			continue
		}

		// This is a defensive check, but this function should only ever receive
		// host-type volumes.
		if req.Type != structs.VolumeTypeHost {
			continue
		}

		hostVolume, ok := clientVolumesByName[req.Source]
		if !ok {
			// Should never happen, but unless the client volumes were mutated during
			// the execution of this hook.
			return nil, fmt.Errorf("No host volume named: %s", req.Source)
		}

		mcfg := &drivers.MountConfig{
			HostPath: hostVolume.Path,
			TaskPath: m.Destination,
			Readonly: hostVolume.ReadOnly || req.ReadOnly || m.ReadOnly,
		}
		mounts = append(mounts, mcfg)
	}

	return mounts, nil
}

// partitionVolumesByType takes a map of volume-alias to volume-request and
// returns them in the form of volume-type:(volume-alias:volume-request)
func partitionVolumesByType(xs map[string]*structs.VolumeRequest) map[string]map[string]*structs.VolumeRequest {
	result := make(map[string]map[string]*structs.VolumeRequest)
	for name, req := range xs {
		txs, ok := result[req.Type]
		if !ok {
			txs = make(map[string]*structs.VolumeRequest)
			result[req.Type] = txs
		}
		txs[name] = req
	}

	return result
}

func (h *volumeHook) prepareHostVolumes(volumes map[string]*structs.VolumeRequest, req *interfaces.TaskPrestartRequest) ([]*drivers.MountConfig, error) {
	hostVolumes := h.runner.clientConfig.Node.HostVolumes

	// Always validate volumes to ensure that we do not allow volumes to be used
	// if a host is restarted and loses the host volume configuration.
	if err := validateHostVolumes(volumes, hostVolumes); err != nil {
		h.logger.Error("Requested Host Volume does not exist", "existing", hostVolumes, "requested", volumes)
		return nil, fmt.Errorf("host volume validation error: %v", err)
	}

	hostVolumeMounts, err := h.hostVolumeMountConfigurations(req.Task.VolumeMounts, volumes, hostVolumes)
	if err != nil {
		h.logger.Error("Failed to generate host volume mounts", "error", err)
		return nil, err
	}

	return hostVolumeMounts, nil
}

// partitionMountsByVolume takes a list of volume mounts and returns them in the
// form of volume-alias:[]volume-mount because one volume may be mounted multiple
// times.
func partitionMountsByVolume(xs []*structs.VolumeMount) map[string][]*structs.VolumeMount {
	result := make(map[string][]*structs.VolumeMount)
	for _, mount := range xs {
		result[mount.Volume] = append(result[mount.Volume], mount)
	}

	return result
}

func (h *volumeHook) prepareCSIVolumes(req *interfaces.TaskPrestartRequest, volumes map[string]*structs.VolumeRequest) ([]*drivers.MountConfig, error) {
	if len(volumes) == 0 {
		return nil, nil
	}

	var mounts []*drivers.MountConfig

	mountRequests := partitionMountsByVolume(req.Task.VolumeMounts)
	csiMountPoints := h.runner.allocHookResources.GetCSIMounts()
	for alias, request := range volumes {
		mountsForAlias, ok := mountRequests[alias]
		if !ok {
			// This task doesn't use the volume
			continue
		}

		csiMountPoint, ok := csiMountPoints[alias]
		if !ok {
			return nil, fmt.Errorf("No CSI Mount Point found for volume: %s", alias)
		}

		for _, m := range mountsForAlias {
			mcfg := &drivers.MountConfig{
				HostPath: csiMountPoint.Source,
				TaskPath: m.Destination,
				Readonly: request.ReadOnly || m.ReadOnly,
			}
			mounts = append(mounts, mcfg)
		}
	}

	return mounts, nil
}

func (h *volumeHook) Prestart(ctx context.Context, req *interfaces.TaskPrestartRequest, resp *interfaces.TaskPrestartResponse) error {
	volumes := partitionVolumesByType(h.alloc.Job.LookupTaskGroup(h.alloc.TaskGroup).Volumes)

	hostVolumeMounts, err := h.prepareHostVolumes(volumes[structs.VolumeTypeHost], req)
	if err != nil {
		return err
	}

	csiVolumeMounts, err := h.prepareCSIVolumes(req, volumes[structs.VolumeTypeCSI])
	if err != nil {
		return err
	}

	// Because this hook is also ran on restores, we only add mounts that do not
	// already exist. Although this loop is somewhat expensive, there are only
	// a small number of mounts that exist within most individual tasks. We may
	// want to revisit this using a `hookdata` param to be "mount only once"
	mounts := h.runner.hookResources.getMounts()
	for _, m := range hostVolumeMounts {
		mounts = ensureMountpointInserted(mounts, m)
	}
	for _, m := range csiVolumeMounts {
		mounts = ensureMountpointInserted(mounts, m)
	}
	h.runner.hookResources.setMounts(mounts)

	return nil
}
<commit_msg>taskrunner/volume_hook: Cleanup arg order of prepareHostVolumes<commit_after>package taskrunner

import (
	"context"
	"fmt"

	log "github.com/hashicorp/go-hclog"
	multierror "github.com/hashicorp/go-multierror"
	"github.com/hashicorp/nomad/client/allocrunner/interfaces"
	"github.com/hashicorp/nomad/nomad/structs"
	"github.com/hashicorp/nomad/plugins/drivers"
)

type volumeHook struct {
	alloc  *structs.Allocation
	runner *TaskRunner
	logger log.Logger
}

func newVolumeHook(runner *TaskRunner, logger log.Logger) *volumeHook {
	h := &volumeHook{
		alloc:  runner.Alloc(),
		runner: runner,
	}
	h.logger = logger.Named(h.Name())
	return h
}

func (*volumeHook) Name() string {
	return "volumes"
}

func validateHostVolumes(requestedByAlias map[string]*structs.VolumeRequest, clientVolumesByName map[string]*structs.ClientHostVolumeConfig) error {
	var result error

	for _, req := range requestedByAlias {
		// This is a defensive check, but this function should only ever receive
		// host-type volumes.
		if req.Type != structs.VolumeTypeHost {
			continue
		}

		_, ok := clientVolumesByName[req.Source]
		if !ok {
			result = multierror.Append(result, fmt.Errorf("missing %s", req.Source))
		}
	}

	return result
}

// hostVolumeMountConfigurations takes the users requested volume mounts,
// volumes, and the client host volume configuration and converts them into a
// format that can be used by drivers.
func (h *volumeHook) hostVolumeMountConfigurations(taskMounts []*structs.VolumeMount, taskVolumesByAlias map[string]*structs.VolumeRequest, clientVolumesByName map[string]*structs.ClientHostVolumeConfig) ([]*drivers.MountConfig, error) {
	var mounts []*drivers.MountConfig
	for _, m := range taskMounts {
		req, ok := taskVolumesByAlias[m.Volume]
		if !ok {
			// This function receives only the task volumes that are of type Host,
			// if we can't find a group volume then we assume the mount is for another
			// type.
			continue
		}

		// This is a defensive check, but this function should only ever receive
		// host-type volumes.
		if req.Type != structs.VolumeTypeHost {
			continue
		}

		hostVolume, ok := clientVolumesByName[req.Source]
		if !ok {
			// Should never happen, but unless the client volumes were mutated during
			// the execution of this hook.
			return nil, fmt.Errorf("No host volume named: %s", req.Source)
		}

		mcfg := &drivers.MountConfig{
			HostPath: hostVolume.Path,
			TaskPath: m.Destination,
			Readonly: hostVolume.ReadOnly || req.ReadOnly || m.ReadOnly,
		}
		mounts = append(mounts, mcfg)
	}

	return mounts, nil
}

// partitionVolumesByType takes a map of volume-alias to volume-request and
// returns them in the form of volume-type:(volume-alias:volume-request)
func partitionVolumesByType(xs map[string]*structs.VolumeRequest) map[string]map[string]*structs.VolumeRequest {
	result := make(map[string]map[string]*structs.VolumeRequest)
	for name, req := range xs {
		txs, ok := result[req.Type]
		if !ok {
			txs = make(map[string]*structs.VolumeRequest)
			result[req.Type] = txs
		}
		txs[name] = req
	}

	return result
}

func (h *volumeHook) prepareHostVolumes(req *interfaces.TaskPrestartRequest, volumes map[string]*structs.VolumeRequest) ([]*drivers.MountConfig, error) {
	hostVolumes := h.runner.clientConfig.Node.HostVolumes

	// Always validate volumes to ensure that we do not allow volumes to be used
	// if a host is restarted and loses the host volume configuration.
	if err := validateHostVolumes(volumes, hostVolumes); err != nil {
		h.logger.Error("Requested Host Volume does not exist", "existing", hostVolumes, "requested", volumes)
		return nil, fmt.Errorf("host volume validation error: %v", err)
	}

	hostVolumeMounts, err := h.hostVolumeMountConfigurations(req.Task.VolumeMounts, volumes, hostVolumes)
	if err != nil {
		h.logger.Error("Failed to generate host volume mounts", "error", err)
		return nil, err
	}

	return hostVolumeMounts, nil
}

// partitionMountsByVolume takes a list of volume mounts and returns them in the
// form of volume-alias:[]volume-mount because one volume may be mounted multiple
// times.
func partitionMountsByVolume(xs []*structs.VolumeMount) map[string][]*structs.VolumeMount {
	result := make(map[string][]*structs.VolumeMount)
	for _, mount := range xs {
		result[mount.Volume] = append(result[mount.Volume], mount)
	}

	return result
}

func (h *volumeHook) prepareCSIVolumes(req *interfaces.TaskPrestartRequest, volumes map[string]*structs.VolumeRequest) ([]*drivers.MountConfig, error) {
	if len(volumes) == 0 {
		return nil, nil
	}

	var mounts []*drivers.MountConfig

	mountRequests := partitionMountsByVolume(req.Task.VolumeMounts)
	csiMountPoints := h.runner.allocHookResources.GetCSIMounts()
	for alias, request := range volumes {
		mountsForAlias, ok := mountRequests[alias]
		if !ok {
			// This task doesn't use the volume
			continue
		}

		csiMountPoint, ok := csiMountPoints[alias]
		if !ok {
			return nil, fmt.Errorf("No CSI Mount Point found for volume: %s", alias)
		}

		for _, m := range mountsForAlias {
			mcfg := &drivers.MountConfig{
				HostPath: csiMountPoint.Source,
				TaskPath: m.Destination,
				Readonly: request.ReadOnly || m.ReadOnly,
			}
			mounts = append(mounts, mcfg)
		}
	}

	return mounts, nil
}

func (h *volumeHook) Prestart(ctx context.Context, req *interfaces.TaskPrestartRequest, resp *interfaces.TaskPrestartResponse) error {
	volumes := partitionVolumesByType(h.alloc.Job.LookupTaskGroup(h.alloc.TaskGroup).Volumes)

	hostVolumeMounts, err := h.prepareHostVolumes(req, volumes[structs.VolumeTypeHost])
	if err != nil {
		return err
	}

	csiVolumeMounts, err := h.prepareCSIVolumes(req, volumes[structs.VolumeTypeCSI])
	if err != nil {
		return err
	}

	// Because this hook is also ran on restores, we only add mounts that do not
	// already exist. Although this loop is somewhat expensive, there are only
	// a small number of mounts that exist within most individual tasks. We may
	// want to revisit this using a `hookdata` param to be "mount only once"
	mounts := h.runner.hookResources.getMounts()
	for _, m := range hostVolumeMounts {
		mounts = ensureMountpointInserted(mounts, m)
	}
	for _, m := range csiVolumeMounts {
		mounts = ensureMountpointInserted(mounts, m)
	}
	h.runner.hookResources.setMounts(mounts)

	return nil
}
<|endoftext|>


<commit_before>package main

import "fmt"
import "golang.org/x/exp/io/i2c"

func main() {
	fmt.Println("hello PI!")
	d, err := i2c.Open(&i2c.Devfs{Dev: "/dev/i2c-1"}, 0x4d)
	if err != nil {
		panic(err)
	}
	b := make([]byte, 3)
	err = d.Read(b)
	if err != nil {
		panic(err)
	}
	temp := b[1]
	celcius := temp / 5 + b[2]
	f := celcius*9/5 + 32

	fmt.Println(temp, celcius, f)
}
<commit_msg>now with 100% more gobot goodness<commit_after>package main

import (

	"fmt"
        "github.com/hybridgroup/gobot/platforms/raspi"
)

func main() {

        r := raspi.NewRaspiAdaptor("raspi")
	errs := r.Connect()
	if errs != nil {
		panic(errs)
	}

	e := r.I2cStart(0x4d)
	if e != nil {
		panic(e)
	}
	b, e := r.I2cRead(0x4d,3)
	if e != nil {
		panic(e)
	}
	c := b[1] / 5
	fmt.Println(c)
}
<|endoftext|>