<commit_before>// Copyright 2010 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Export guts for testing.

package runtime

import (
	"runtime/internal/atomic"
	"runtime/internal/sys"
	"unsafe"
)

var Fadd64 = fadd64
var Fsub64 = fsub64
var Fmul64 = fmul64
var Fdiv64 = fdiv64
var F64to32 = f64to32
var F32to64 = f32to64
var Fcmp64 = fcmp64
var Fintto64 = fintto64
var F64toint = f64toint

var Entersyscall = entersyscall
var Exitsyscall = exitsyscall
var LockedOSThread = lockedOSThread
var Xadduintptr = atomic.Xadduintptr

var FuncPC = funcPC

var Fastlog2 = fastlog2

var Atoi = atoi
var Atoi32 = atoi32

type LFNode struct {
	Next    uint64
	Pushcnt uintptr
}

func LFStackPush(head *uint64, node *LFNode) {
	(*lfstack)(head).push((*lfnode)(unsafe.Pointer(node)))
}

func LFStackPop(head *uint64) *LFNode {
	return (*LFNode)(unsafe.Pointer((*lfstack)(head).pop()))
}

func GCMask(x interface{}) (ret []byte) {
	systemstack(func() {
		ret = getgcmask(x)
	})
	return
}

func RunSchedLocalQueueTest() {
	_p_ := new(p)
	gs := make([]g, len(_p_.runq))
	for i := 0; i < len(_p_.runq); i++ {
		if g, _ := runqget(_p_); g != nil {
			throw("runq is not empty initially")
		}
		for j := 0; j < i; j++ {
			runqput(_p_, &gs[i], false)
		}
		for j := 0; j < i; j++ {
			if g, _ := runqget(_p_); g != &gs[i] {
				print("bad element at iter ", i, "/", j, "\n")
				throw("bad element")
			}
		}
		if g, _ := runqget(_p_); g != nil {
			throw("runq is not empty afterwards")
		}
	}
}

func RunSchedLocalQueueStealTest() {
	p1 := new(p)
	p2 := new(p)
	gs := make([]g, len(p1.runq))
	for i := 0; i < len(p1.runq); i++ {
		for j := 0; j < i; j++ {
			gs[j].sig = 0
			runqput(p1, &gs[j], false)
		}
		gp := runqsteal(p2, p1, true)
		s := 0
		if gp != nil {
			s++
			gp.sig++
		}
		for {
			gp, _ = runqget(p2)
			if gp == nil {
				break
			}
			s++
			gp.sig++
		}
		for {
			gp, _ = runqget(p1)
			if gp == nil {
				break
			}
			gp.sig++
		}
		for j := 0; j < i; j++ {
			if gs[j].sig != 1 {
				print("bad element ", j, "(", gs[j].sig, ") at iter ", i, "\n")
				throw("bad element")
			}
		}
		if s != i/2 && s != i/2+1 {
			print("bad steal ", s, ", want ", i/2, " or ", i/2+1, ", iter ", i, "\n")
			throw("bad steal")
		}
	}
}

func RunSchedLocalQueueEmptyTest(iters int) {
	// Test that runq is not spuriously reported as empty.
	// Runq emptiness affects scheduling decisions and spurious emptiness
	// can lead to underutilization (both runnable Gs and idle Ps coexist
	// for arbitrary long time).
	done := make(chan bool, 1)
	p := new(p)
	gs := make([]g, 2)
	ready := new(uint32)
	for i := 0; i < iters; i++ {
		*ready = 0
		next0 := (i & 1) == 0
		next1 := (i & 2) == 0
		runqput(p, &gs[0], next0)
		go func() {
			for atomic.Xadd(ready, 1); atomic.Load(ready) != 2; {
			}
			if runqempty(p) {
				println("next:", next0, next1)
				throw("queue is empty")
			}
			done <- true
		}()
		for atomic.Xadd(ready, 1); atomic.Load(ready) != 2; {
		}
		runqput(p, &gs[1], next1)
		runqget(p)
		<-done
		runqget(p)
	}
}

var (
	StringHash = stringHash
	BytesHash  = bytesHash
	Int32Hash  = int32Hash
	Int64Hash  = int64Hash
	MemHash    = memhash
	MemHash32  = memhash32
	MemHash64  = memhash64
	EfaceHash  = efaceHash
	IfaceHash  = ifaceHash
)

var UseAeshash = &useAeshash

func MemclrBytes(b []byte) {
	s := (*slice)(unsafe.Pointer(&b))
	memclrNoHeapPointers(s.array, uintptr(s.len))
}

var HashLoad = &hashLoad

// entry point for testing
func GostringW(w []uint16) (s string) {
	systemstack(func() {
		s = gostringw(&w[0])
	})
	return
}

type Uintreg sys.Uintreg

var Open = open
var Close = closefd
var Read = read
var Write = write

func Envs() []string     { return envs }
func SetEnvs(e []string) { envs = e }

var BigEndian = sys.BigEndian

// For benchmarking.

func BenchSetType(n int, x interface{}) {
	e := *efaceOf(&x)
	t := e._type
	var size uintptr
	var p unsafe.Pointer
	switch t.kind & kindMask {
	case kindPtr:
		t = (*ptrtype)(unsafe.Pointer(t)).elem
		size = t.size
		p = e.data
	case kindSlice:
		slice := *(*struct {
			ptr      unsafe.Pointer
			len, cap uintptr
		})(e.data)
		t = (*slicetype)(unsafe.Pointer(t)).elem
		size = t.size * slice.len
		p = slice.ptr
	}
	allocSize := roundupsize(size)
	systemstack(func() {
		for i := 0; i < n; i++ {
			heapBitsSetType(uintptr(p), allocSize, size, t)
		}
	})
}

const PtrSize = sys.PtrSize

var ForceGCPeriod = &forcegcperiod

// SetTracebackEnv is like runtime/debug.SetTraceback, but it raises
// the "environment" traceback level, so later calls to
// debug.SetTraceback (e.g., from testing timeouts) can't lower it.
func SetTracebackEnv(level string) {
	setTraceback(level)
	traceback_env = traceback_cache
}

var ReadUnaligned32 = readUnaligned32
var ReadUnaligned64 = readUnaligned64

func CountPagesInUse() (pagesInUse, counted uintptr) {
	stopTheWorld("CountPagesInUse")

	pagesInUse = uintptr(mheap_.pagesInUse)

	for _, s := range mheap_.allspans {
		if s.state == mSpanInUse {
			counted += s.npages
		}
	}

	startTheWorld()

	return
}

func Fastrand() uint32          { return fastrand() }
func Fastrandn(n uint32) uint32 { return fastrandn(n) }

type ProfBuf profBuf

func NewProfBuf(hdrsize, bufwords, tags int) *ProfBuf {
	return (*ProfBuf)(newProfBuf(hdrsize, bufwords, tags))
}

func (p *ProfBuf) Write(tag *unsafe.Pointer, now int64, hdr []uint64, stk []uintptr) {
	(*profBuf)(p).write(tag, now, hdr, stk)
}

const (
	ProfBufBlocking    = profBufBlocking
	ProfBufNonBlocking = profBufNonBlocking
)

func (p *ProfBuf) Read(mode profBufReadMode) ([]uint64, []unsafe.Pointer, bool) {
	return (*profBuf)(p).read(profBufReadMode(mode))
}

func (p *ProfBuf) Close() {
	(*profBuf)(p).close()
}

// ReadMemStatsSlow returns both the runtime-computed MemStats and
// MemStats accumulated by scanning the heap.
func ReadMemStatsSlow() (base, slow MemStats) {
	stopTheWorld("ReadMemStatsSlow")

	// Run on the system stack to avoid stack growth allocation.
	systemstack(func() {
		// Make sure stats don't change.
		getg().m.mallocing++

		readmemstats_m(&base)

		// Initialize slow from base and zero the fields we're
		// recomputing.
		slow = base
		slow.Alloc = 0
		slow.TotalAlloc = 0
		slow.Mallocs = 0
		slow.Frees = 0
		var bySize [_NumSizeClasses]struct {
			Mallocs, Frees uint64
		}

		// Add up current allocations in spans.
		for _, s := range mheap_.allspans {
			if s.state != mSpanInUse {
				continue
			}
			if sizeclass := s.spanclass.sizeclass(); sizeclass == 0 {
				slow.Mallocs++
				slow.Alloc += uint64(s.elemsize)
			} else {
				slow.Mallocs += uint64(s.allocCount)
				slow.Alloc += uint64(s.allocCount) * uint64(s.elemsize)
				bySize[sizeclass].Mallocs += uint64(s.allocCount)
			}
		}

		// Add in frees. readmemstats_m flushed the cached stats, so
		// these are up-to-date.
		var smallFree uint64
		slow.Frees = mheap_.nlargefree
		for i := range mheap_.nsmallfree {
			slow.Frees += mheap_.nsmallfree[i]
			bySize[i].Frees = mheap_.nsmallfree[i]
			bySize[i].Mallocs += mheap_.nsmallfree[i]
			smallFree += mheap_.nsmallfree[i] * uint64(class_to_size[i])
		}
		slow.Frees += memstats.tinyallocs
		slow.Mallocs += slow.Frees

		slow.TotalAlloc = slow.Alloc + mheap_.largefree + smallFree

		for i := range slow.BySize {
			slow.BySize[i].Mallocs = bySize[i].Mallocs
			slow.BySize[i].Frees = bySize[i].Frees
		}

		getg().m.mallocing--
	})

	startTheWorld()
	return
}

// BlockOnSystemStack switches to the system stack, prints "x\n" to
// stderr, and blocks in a stack containing
// "runtime.blockOnSystemStackInternal".
func BlockOnSystemStack() {
	systemstack(blockOnSystemStackInternal)
}

func blockOnSystemStackInternal() {
	print("x\n")
	lock(&deadlock)
	lock(&deadlock)
}

type RWMutex struct {
	rw rwmutex
}

func (rw *RWMutex) RLock() {
	rw.rw.rlock()
}

func (rw *RWMutex) RUnlock() {
	rw.rw.runlock()
}

func (rw *RWMutex) Lock() {
	rw.rw.lock()
}

func (rw *RWMutex) Unlock() {
	rw.rw.unlock()
}

const RuntimeHmapSize = unsafe.Sizeof(hmap{})

func MapBucketsCount(m map[int]int) int {
	h := *(**hmap)(unsafe.Pointer(&m))
	return 1 << h.B
}

func MapBucketsPointerIsNil(m map[int]int) bool {
	h := *(**hmap)(unsafe.Pointer(&m))
	return h.buckets == nil
}

func LockOSCounts() (external, internal uint32) {
	g := getg()
	if g.m.lockedExt+g.m.lockedInt == 0 {
		if g.lockedm != 0 {
			panic("lockedm on non-locked goroutine")
		}
	} else {
		if g.lockedm == 0 {
			panic("nil lockedm on locked goroutine")
		}
	}
	return g.m.lockedExt, g.m.lockedInt
}

//go:noinline
func TracebackSystemstack(stk []uintptr, i int) int {
	if i == 0 {
		pc, sp := getcallerpc(), getcallersp()
		return gentraceback(pc, sp, 0, getg(), 0, &stk[0], len(stk), nil, nil, _TraceJumpStack)
	}
	n := 0
	systemstack(func() {
		n = TracebackSystemstack(stk, i-1)
	})
	return n
}

func KeepNArenaHints(n int) {
	hint := mheap_.arenaHints
	for i := 1; i < n; i++ {
		hint = hint.next
		if hint == nil {
			return
		}
	}
	hint.next = nil
}

// MapNextArenaHint reserves a page at the next arena growth hint,
// preventing the arena from growing there, and returns the range of
// addresses that are no longer viable.
func MapNextArenaHint() (start, end uintptr) {
	hint := mheap_.arenaHints
	addr := hint.addr
	if hint.down {
		start, end = addr-heapArenaBytes, addr
		addr -= physPageSize
	} else {
		start, end = addr, addr+heapArenaBytes
	}
	sysReserve(unsafe.Pointer(addr), physPageSize)
	return
}

func GetNextArenaHint() uintptr {
	return mheap_.arenaHints.addr
}

type G = g

func Getg() *G {
	return getg()
}

//go:noinline
func PanicForTesting(b []byte, i int) byte {
	return unexportedPanicForTesting(b, i)
}

//go:noinline
func unexportedPanicForTesting(b []byte, i int) byte {
	return b[i]
}

func G0StackOverflow() {
	systemstack(func() {
		stackOverflow(nil)
	})
}

func stackOverflow(x *byte) {
	var buf [256]byte
	stackOverflow(&buf[0])
}
<commit_msg>runtime: extend ReadMemStatsSlow to re-compute HeapReleased<commit_after>// Copyright 2010 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Export guts for testing.

package runtime

import (
	"runtime/internal/atomic"
	"runtime/internal/sys"
	"unsafe"
)

var Fadd64 = fadd64
var Fsub64 = fsub64
var Fmul64 = fmul64
var Fdiv64 = fdiv64
var F64to32 = f64to32
var F32to64 = f32to64
var Fcmp64 = fcmp64
var Fintto64 = fintto64
var F64toint = f64toint

var Entersyscall = entersyscall
var Exitsyscall = exitsyscall
var LockedOSThread = lockedOSThread
var Xadduintptr = atomic.Xadduintptr

var FuncPC = funcPC

var Fastlog2 = fastlog2

var Atoi = atoi
var Atoi32 = atoi32

type LFNode struct {
	Next    uint64
	Pushcnt uintptr
}

func LFStackPush(head *uint64, node *LFNode) {
	(*lfstack)(head).push((*lfnode)(unsafe.Pointer(node)))
}

func LFStackPop(head *uint64) *LFNode {
	return (*LFNode)(unsafe.Pointer((*lfstack)(head).pop()))
}

func GCMask(x interface{}) (ret []byte) {
	systemstack(func() {
		ret = getgcmask(x)
	})
	return
}

func RunSchedLocalQueueTest() {
	_p_ := new(p)
	gs := make([]g, len(_p_.runq))
	for i := 0; i < len(_p_.runq); i++ {
		if g, _ := runqget(_p_); g != nil {
			throw("runq is not empty initially")
		}
		for j := 0; j < i; j++ {
			runqput(_p_, &gs[i], false)
		}
		for j := 0; j < i; j++ {
			if g, _ := runqget(_p_); g != &gs[i] {
				print("bad element at iter ", i, "/", j, "\n")
				throw("bad element")
			}
		}
		if g, _ := runqget(_p_); g != nil {
			throw("runq is not empty afterwards")
		}
	}
}

func RunSchedLocalQueueStealTest() {
	p1 := new(p)
	p2 := new(p)
	gs := make([]g, len(p1.runq))
	for i := 0; i < len(p1.runq); i++ {
		for j := 0; j < i; j++ {
			gs[j].sig = 0
			runqput(p1, &gs[j], false)
		}
		gp := runqsteal(p2, p1, true)
		s := 0
		if gp != nil {
			s++
			gp.sig++
		}
		for {
			gp, _ = runqget(p2)
			if gp == nil {
				break
			}
			s++
			gp.sig++
		}
		for {
			gp, _ = runqget(p1)
			if gp == nil {
				break
			}
			gp.sig++
		}
		for j := 0; j < i; j++ {
			if gs[j].sig != 1 {
				print("bad element ", j, "(", gs[j].sig, ") at iter ", i, "\n")
				throw("bad element")
			}
		}
		if s != i/2 && s != i/2+1 {
			print("bad steal ", s, ", want ", i/2, " or ", i/2+1, ", iter ", i, "\n")
			throw("bad steal")
		}
	}
}

func RunSchedLocalQueueEmptyTest(iters int) {
	// Test that runq is not spuriously reported as empty.
	// Runq emptiness affects scheduling decisions and spurious emptiness
	// can lead to underutilization (both runnable Gs and idle Ps coexist
	// for arbitrary long time).
	done := make(chan bool, 1)
	p := new(p)
	gs := make([]g, 2)
	ready := new(uint32)
	for i := 0; i < iters; i++ {
		*ready = 0
		next0 := (i & 1) == 0
		next1 := (i & 2) == 0
		runqput(p, &gs[0], next0)
		go func() {
			for atomic.Xadd(ready, 1); atomic.Load(ready) != 2; {
			}
			if runqempty(p) {
				println("next:", next0, next1)
				throw("queue is empty")
			}
			done <- true
		}()
		for atomic.Xadd(ready, 1); atomic.Load(ready) != 2; {
		}
		runqput(p, &gs[1], next1)
		runqget(p)
		<-done
		runqget(p)
	}
}

var (
	StringHash = stringHash
	BytesHash  = bytesHash
	Int32Hash  = int32Hash
	Int64Hash  = int64Hash
	MemHash    = memhash
	MemHash32  = memhash32
	MemHash64  = memhash64
	EfaceHash  = efaceHash
	IfaceHash  = ifaceHash
)

var UseAeshash = &useAeshash

func MemclrBytes(b []byte) {
	s := (*slice)(unsafe.Pointer(&b))
	memclrNoHeapPointers(s.array, uintptr(s.len))
}

var HashLoad = &hashLoad

// entry point for testing
func GostringW(w []uint16) (s string) {
	systemstack(func() {
		s = gostringw(&w[0])
	})
	return
}

type Uintreg sys.Uintreg

var Open = open
var Close = closefd
var Read = read
var Write = write

func Envs() []string     { return envs }
func SetEnvs(e []string) { envs = e }

var BigEndian = sys.BigEndian

// For benchmarking.

func BenchSetType(n int, x interface{}) {
	e := *efaceOf(&x)
	t := e._type
	var size uintptr
	var p unsafe.Pointer
	switch t.kind & kindMask {
	case kindPtr:
		t = (*ptrtype)(unsafe.Pointer(t)).elem
		size = t.size
		p = e.data
	case kindSlice:
		slice := *(*struct {
			ptr      unsafe.Pointer
			len, cap uintptr
		})(e.data)
		t = (*slicetype)(unsafe.Pointer(t)).elem
		size = t.size * slice.len
		p = slice.ptr
	}
	allocSize := roundupsize(size)
	systemstack(func() {
		for i := 0; i < n; i++ {
			heapBitsSetType(uintptr(p), allocSize, size, t)
		}
	})
}

const PtrSize = sys.PtrSize

var ForceGCPeriod = &forcegcperiod

// SetTracebackEnv is like runtime/debug.SetTraceback, but it raises
// the "environment" traceback level, so later calls to
// debug.SetTraceback (e.g., from testing timeouts) can't lower it.
func SetTracebackEnv(level string) {
	setTraceback(level)
	traceback_env = traceback_cache
}

var ReadUnaligned32 = readUnaligned32
var ReadUnaligned64 = readUnaligned64

func CountPagesInUse() (pagesInUse, counted uintptr) {
	stopTheWorld("CountPagesInUse")

	pagesInUse = uintptr(mheap_.pagesInUse)

	for _, s := range mheap_.allspans {
		if s.state == mSpanInUse {
			counted += s.npages
		}
	}

	startTheWorld()

	return
}

func Fastrand() uint32          { return fastrand() }
func Fastrandn(n uint32) uint32 { return fastrandn(n) }

type ProfBuf profBuf

func NewProfBuf(hdrsize, bufwords, tags int) *ProfBuf {
	return (*ProfBuf)(newProfBuf(hdrsize, bufwords, tags))
}

func (p *ProfBuf) Write(tag *unsafe.Pointer, now int64, hdr []uint64, stk []uintptr) {
	(*profBuf)(p).write(tag, now, hdr, stk)
}

const (
	ProfBufBlocking    = profBufBlocking
	ProfBufNonBlocking = profBufNonBlocking
)

func (p *ProfBuf) Read(mode profBufReadMode) ([]uint64, []unsafe.Pointer, bool) {
	return (*profBuf)(p).read(profBufReadMode(mode))
}

func (p *ProfBuf) Close() {
	(*profBuf)(p).close()
}

// ReadMemStatsSlow returns both the runtime-computed MemStats and
// MemStats accumulated by scanning the heap.
func ReadMemStatsSlow() (base, slow MemStats) {
	stopTheWorld("ReadMemStatsSlow")

	// Run on the system stack to avoid stack growth allocation.
	systemstack(func() {
		// Make sure stats don't change.
		getg().m.mallocing++

		readmemstats_m(&base)

		// Initialize slow from base and zero the fields we're
		// recomputing.
		slow = base
		slow.Alloc = 0
		slow.TotalAlloc = 0
		slow.Mallocs = 0
		slow.Frees = 0
		slow.HeapReleased = 0
		var bySize [_NumSizeClasses]struct {
			Mallocs, Frees uint64
		}

		// Add up current allocations in spans.
		for _, s := range mheap_.allspans {
			if s.state != mSpanInUse {
				continue
			}
			if sizeclass := s.spanclass.sizeclass(); sizeclass == 0 {
				slow.Mallocs++
				slow.Alloc += uint64(s.elemsize)
			} else {
				slow.Mallocs += uint64(s.allocCount)
				slow.Alloc += uint64(s.allocCount) * uint64(s.elemsize)
				bySize[sizeclass].Mallocs += uint64(s.allocCount)
			}
		}

		// Add in frees. readmemstats_m flushed the cached stats, so
		// these are up-to-date.
		var smallFree uint64
		slow.Frees = mheap_.nlargefree
		for i := range mheap_.nsmallfree {
			slow.Frees += mheap_.nsmallfree[i]
			bySize[i].Frees = mheap_.nsmallfree[i]
			bySize[i].Mallocs += mheap_.nsmallfree[i]
			smallFree += mheap_.nsmallfree[i] * uint64(class_to_size[i])
		}
		slow.Frees += memstats.tinyallocs
		slow.Mallocs += slow.Frees

		slow.TotalAlloc = slow.Alloc + mheap_.largefree + smallFree

		for i := range slow.BySize {
			slow.BySize[i].Mallocs = bySize[i].Mallocs
			slow.BySize[i].Frees = bySize[i].Frees
		}

		mheap_.scav.treap.walkTreap(func(tn *treapNode) {
			slow.HeapReleased += uint64(tn.spanKey.released())
		})

		getg().m.mallocing--
	})

	startTheWorld()
	return
}

// BlockOnSystemStack switches to the system stack, prints "x\n" to
// stderr, and blocks in a stack containing
// "runtime.blockOnSystemStackInternal".
func BlockOnSystemStack() {
	systemstack(blockOnSystemStackInternal)
}

func blockOnSystemStackInternal() {
	print("x\n")
	lock(&deadlock)
	lock(&deadlock)
}

type RWMutex struct {
	rw rwmutex
}

func (rw *RWMutex) RLock() {
	rw.rw.rlock()
}

func (rw *RWMutex) RUnlock() {
	rw.rw.runlock()
}

func (rw *RWMutex) Lock() {
	rw.rw.lock()
}

func (rw *RWMutex) Unlock() {
	rw.rw.unlock()
}

const RuntimeHmapSize = unsafe.Sizeof(hmap{})

func MapBucketsCount(m map[int]int) int {
	h := *(**hmap)(unsafe.Pointer(&m))
	return 1 << h.B
}

func MapBucketsPointerIsNil(m map[int]int) bool {
	h := *(**hmap)(unsafe.Pointer(&m))
	return h.buckets == nil
}

func LockOSCounts() (external, internal uint32) {
	g := getg()
	if g.m.lockedExt+g.m.lockedInt == 0 {
		if g.lockedm != 0 {
			panic("lockedm on non-locked goroutine")
		}
	} else {
		if g.lockedm == 0 {
			panic("nil lockedm on locked goroutine")
		}
	}
	return g.m.lockedExt, g.m.lockedInt
}

//go:noinline
func TracebackSystemstack(stk []uintptr, i int) int {
	if i == 0 {
		pc, sp := getcallerpc(), getcallersp()
		return gentraceback(pc, sp, 0, getg(), 0, &stk[0], len(stk), nil, nil, _TraceJumpStack)
	}
	n := 0
	systemstack(func() {
		n = TracebackSystemstack(stk, i-1)
	})
	return n
}

func KeepNArenaHints(n int) {
	hint := mheap_.arenaHints
	for i := 1; i < n; i++ {
		hint = hint.next
		if hint == nil {
			return
		}
	}
	hint.next = nil
}

// MapNextArenaHint reserves a page at the next arena growth hint,
// preventing the arena from growing there, and returns the range of
// addresses that are no longer viable.
func MapNextArenaHint() (start, end uintptr) {
	hint := mheap_.arenaHints
	addr := hint.addr
	if hint.down {
		start, end = addr-heapArenaBytes, addr
		addr -= physPageSize
	} else {
		start, end = addr, addr+heapArenaBytes
	}
	sysReserve(unsafe.Pointer(addr), physPageSize)
	return
}

func GetNextArenaHint() uintptr {
	return mheap_.arenaHints.addr
}

type G = g

func Getg() *G {
	return getg()
}

//go:noinline
func PanicForTesting(b []byte, i int) byte {
	return unexportedPanicForTesting(b, i)
}

//go:noinline
func unexportedPanicForTesting(b []byte, i int) byte {
	return b[i]
}

func G0StackOverflow() {
	systemstack(func() {
		stackOverflow(nil)
	})
}

func stackOverflow(x *byte) {
	var buf [256]byte
	stackOverflow(&buf[0])
}
<|endoftext|>


<commit_before>package shell

import (
	"context"
	"errors"
	"fmt"
	"io"
	"os"
	"os/exec"
	"reflect"
	"strings"
	"sync"
	"syscall"

	"github.com/zetamatta/go-findfile"

	"github.com/zetamatta/nyagos/defined"
	"github.com/zetamatta/nyagos/dos"
)

var WildCardExpansionAlways = false

type CommandNotFound struct {
	Name string
	Err  error
}

func (this CommandNotFound) Stringer() string {
	return fmt.Sprintf("'%s' is not recognized as an internal or external command,\noperable program or batch file", this.Name)
}

func (this CommandNotFound) Error() string {
	return this.Stringer()
}

type session struct {
	unreadline []string
}

type CloneCloser interface {
	Clone(context.Context) (context.Context, CloneCloser, error)
	Close() error
}

type Shell struct {
	*session
	Stdout       *os.File
	Stderr       *os.File
	Stdin        *os.File
	Console      io.Writer
	tag          CloneCloser
	IsBackGround bool
}

func (sh *Shell) In() io.Reader          { return sh.Stdin }
func (sh *Shell) Out() io.Writer         { return sh.Stdout }
func (sh *Shell) Err() io.Writer         { return sh.Stderr }
func (sh *Shell) Term() io.Writer        { return sh.Console }
func (sh *Shell) Tag() CloneCloser       { return sh.tag }
func (sh *Shell) SetTag(tag CloneCloser) { sh.tag = tag }

type Cmd struct {
	Shell
	args            []string
	rawArgs         []string
	fullPath        string
	UseShellExecute bool
	Closers         []io.Closer
}

func (cmd *Cmd) Arg(n int) string      { return cmd.args[n] }
func (cmd *Cmd) Args() []string        { return cmd.args }
func (cmd *Cmd) SetArgs(s []string)    { cmd.args = s }
func (cmd *Cmd) RawArg(n int) string   { return cmd.rawArgs[n] }
func (cmd *Cmd) RawArgs() []string     { return cmd.rawArgs }
func (cmd *Cmd) SetRawArgs(s []string) { cmd.rawArgs = s }

var LookCurdirOrder = dos.LookCurdirFirst

func (cmd *Cmd) FullPath() string {
	if cmd.args == nil || len(cmd.args) <= 0 {
		return ""
	}
	if cmd.fullPath == "" {
		cmd.fullPath = dos.LookPath(LookCurdirOrder, cmd.args[0], "NYAGOSPATH")
	}
	return cmd.fullPath
}

func (cmd *Cmd) Close() {
	if cmd.Closers != nil {
		for _, c := range cmd.Closers {
			c.Close()
		}
		cmd.Closers = nil
	}
}

func (sh *Shell) Close() {}

func New() *Shell {
	return &Shell{
		Stdin:   os.Stdin,
		Stdout:  os.Stdout,
		Stderr:  os.Stderr,
		session: &session{},
	}
}

func (sh *Shell) Command() *Cmd {
	cmd := &Cmd{
		Shell: Shell{
			Stdin:   sh.Stdin,
			Stdout:  sh.Stdout,
			Stderr:  sh.Stderr,
			Console: sh.Console,
			tag:     sh.tag,
		},
	}
	if sh.session != nil {
		cmd.session = sh.session
	} else {
		cmd.session = &session{}
	}
	return cmd
}

type ArgsHookT func(ctx context.Context, sh *Shell, args []string) ([]string, error)

var argsHook = func(ctx context.Context, sh *Shell, args []string) ([]string, error) {
	return args, nil
}

func SetArgsHook(argsHook_ ArgsHookT) (rv ArgsHookT) {
	rv, argsHook = argsHook, argsHook_
	return
}

type HookT func(context.Context, *Cmd) (int, bool, error)

var hook = func(context.Context, *Cmd) (int, bool, error) {
	return 0, false, nil
}

func SetHook(hook_ HookT) (rv HookT) {
	rv, hook = hook, hook_
	return
}

var OnCommandNotFound = func(ctx context.Context, cmd *Cmd, err error) error {
	err = &CommandNotFound{cmd.args[0], err}
	return err
}

var LastErrorLevel int

func makeCmdline(args, rawargs []string) string {
	var buffer strings.Builder
	for i, s := range args {
		if i > 0 {
			buffer.WriteRune(' ')
		}
		if (len(rawargs) > i && len(rawargs[i]) > 0 && rawargs[i][0] == '"') || strings.ContainsAny(s, " &|<>\t\"") {
			fmt.Fprintf(&buffer, `"%s"`, strings.Replace(s, `"`, `\"`, -1))
		} else {
			buffer.WriteString(s)
		}
	}
	return buffer.String()
}

var UseSourceRunBatch = true

func encloseWithQuote(fullpath string) string {
	if strings.ContainsRune(fullpath, ' ') {
		var f strings.Builder
		f.WriteRune('"')
		f.WriteString(fullpath)
		f.WriteRune('"')
		return f.String()
	} else {
		return fullpath
	}
}

func (cmd *Cmd) spawnvpSilent(ctx context.Context) (int, error) {
	// command is empty.
	if len(cmd.args) <= 0 {
		return 0, nil
	}
	if defined.DBG {
		print("spawnvpSilent('", cmd.args[0], "')\n")
	}

	// aliases and lua-commands
	if errorlevel, done, err := hook(ctx, cmd); done || err != nil {
		return errorlevel, err
	}

	// command not found hook
	fullpath := cmd.FullPath()
	if fullpath == "" {
		return 255, OnCommandNotFound(ctx, cmd, os.ErrNotExist)
	}
	cmd.args[0] = fullpath

	if defined.DBG {
		print("exec.LookPath(", cmd.args[0], ")==", fullpath, "\n")
	}
	if WildCardExpansionAlways {
		cmd.args = findfile.Globs(cmd.args)
	}
	if cmd.UseShellExecute {
		// GUI Application
		cmdline := makeCmdline(cmd.args[1:], cmd.rawArgs[1:])
		return 0, dos.ShellExecute("open", fullpath, cmdline, "")
	}
	if UseSourceRunBatch {
		lowerName := strings.ToLower(cmd.args[0])
		if strings.HasSuffix(lowerName, ".cmd") || strings.HasSuffix(lowerName, ".bat") {
			rawargs := cmd.RawArgs()
			args := make([]string, len(rawargs))
			args[0] = encloseWithQuote(fullpath)
			for i, end := 1, len(rawargs); i < end; i++ {
				args[i] = rawargs[i]
			}
			// Batch files
			return RawSource(args, nil, false, cmd.Stdin, cmd.Stdout, cmd.Stderr)
		}
	}
	// Do not use exec.CommandContext because it cancels background process.
	xcmd := exec.Command(cmd.args[0], cmd.args[1:]...)
	xcmd.Stdin = cmd.Stdin
	xcmd.Stdout = cmd.Stdout
	xcmd.Stderr = cmd.Stderr

	if xcmd.SysProcAttr == nil {
		xcmd.SysProcAttr = new(syscall.SysProcAttr)
	}
	cmdline := makeCmdline(xcmd.Args, cmd.rawArgs)
	if defined.DBG {
		println(cmdline)
	}
	xcmd.SysProcAttr.CmdLine = cmdline
	err := xcmd.Run()
	errorlevel, errorlevelOk := dos.GetErrorLevel(xcmd)
	if errorlevelOk {
		return errorlevel, err
	} else {
		return 255, err
	}
}

type AlreadyReportedError struct {
	Err error
}

func (_ AlreadyReportedError) Error() string {
	return ""
}

func IsAlreadyReported(err error) bool {
	_, ok := err.(AlreadyReportedError)
	return ok
}

func (cmd *Cmd) Spawnvp(ctx context.Context) (int, error) {
	errorlevel, err := cmd.spawnvpSilent(ctx)
	if err != nil && err != io.EOF && !IsAlreadyReported(err) {
		if defined.DBG {
			val := reflect.ValueOf(err)
			fmt.Fprintf(cmd.Stderr, "error-type=%s\n", val.Type())
		}
		fmt.Fprintln(cmd.Stderr, err.Error())
		err = AlreadyReportedError{err}
	}
	return errorlevel, err
}

func (sh *Shell) Spawnlp(ctx context.Context, args, rawargs []string) (int, error) {
	cmd := sh.Command()
	defer cmd.Close()
	cmd.SetArgs(args)
	cmd.SetRawArgs(rawargs)
	return cmd.Spawnvp(ctx)
}

func (sh *Shell) Interpret(ctx context.Context, text string) (errorlevel int, finalerr error) {
	if defined.DBG {
		print("Interpret('", text, "')\n")
	}
	if sh == nil {
		return 255, errors.New("Fatal Error: Interpret: instance is nil")
	}
	errorlevel = 0
	finalerr = nil

	statements, statementsErr := Parse(text)
	if statementsErr != nil {
		if defined.DBG {
			print("Parse Error:", statementsErr.Error(), "\n")
		}
		return 0, statementsErr
	}
	if argsHook != nil {
		if defined.DBG {
			print("call argsHook\n")
		}
		for _, pipeline := range statements {
			for _, state := range pipeline {
				var err error
				state.Args, err = argsHook(ctx, sh, state.Args)
				if err != nil {
					return 255, err
				}
			}
		}
		if defined.DBG {
			print("done argsHook\n")
		}
	}
	for _, pipeline := range statements {
		for i, state := range pipeline {
			if state.Term == "|" && (i+1 >= len(pipeline) || len(pipeline[i+1].Args) <= 0) {
				return 255, errors.New("The syntax of the command is incorrect.")
			}
		}
	}

	for _, pipeline := range statements {

		var pipeIn *os.File = nil
		isBackGround := sh.IsBackGround
		for _, state := range pipeline {
			if state.Term == "&" {
				isBackGround = true
				break
			}
		}
		var wg sync.WaitGroup
		shutdown_immediately := false
		for i, state := range pipeline {
			if defined.DBG {
				print(i, ": pipeline loop(", state.Args[0], ")\n")
			}
			cmd := sh.Command()
			cmd.IsBackGround = isBackGround

			if pipeIn != nil {
				cmd.Stdin = pipeIn
				cmd.Closers = append(cmd.Closers, pipeIn)
				pipeIn = nil
			}

			var err error
			if state.Term[0] == '|' {
				var pipeOut *os.File
				pipeIn, pipeOut, err = os.Pipe()
				cmd.Stdout = pipeOut
				if state.Term == "|&" {
					cmd.Stderr = pipeOut
				}
				cmd.Closers = append(cmd.Closers, pipeOut)
			}

			for _, red := range state.Redirect {
				var fd *os.File
				fd, err = red.OpenOn(cmd)
				if err != nil {
					return 0, err
				}
				cmd.Closers = append(cmd.Closers, fd)
			}

			cmd.args = state.Args
			cmd.rawArgs = state.RawArgs
			if i > 0 {
				cmd.IsBackGround = true
			}
			if len(pipeline) == 1 && dos.IsGui(cmd.FullPath()) {
				cmd.UseShellExecute = true
			}
			if i == len(pipeline)-1 && state.Term != "&" {
				// foreground execution.
				errorlevel, finalerr = cmd.Spawnvp(ctx)
				LastErrorLevel = errorlevel
				cmd.Close()
			} else {
				// background
				if !isBackGround {
					wg.Add(1)
				}
				newctx := ctx
				if tag := cmd.Tag(); tag != nil {
					var newtag CloneCloser
					if newctx, newtag, err = tag.Clone(ctx); err != nil {
						fmt.Fprintln(os.Stderr, err.Error())
						return -1, err
					} else {
						cmd.SetTag(newtag)
					}
				}
				go func(ctx1 context.Context, cmd1 *Cmd) {
					if !isBackGround {
						defer wg.Done()
					}
					cmd1.Spawnvp(ctx1)
					if tag := cmd1.Tag(); tag != nil {
						if err := tag.Close(); err != nil {
							fmt.Fprintln(os.Stderr, err.Error())
						}
					}
					cmd1.Close()
				}(newctx, cmd)
			}
		}
		if !isBackGround {
			wg.Wait()
			if shutdown_immediately {
				return errorlevel, nil
			}
			if len(pipeline) > 0 {
				switch pipeline[len(pipeline)-1].Term {
				case "&&":
					if errorlevel != 0 {
						return errorlevel, nil
					}
				case "||":
					if errorlevel == 0 {
						return errorlevel, nil
					}
				}
			}
		}
	}
	return
}
<commit_msg>Fix #313 (reopened) let Context not to terminate background task<commit_after>package shell

import (
	"context"
	"errors"
	"fmt"
	"io"
	"os"
	"os/exec"
	"reflect"
	"strings"
	"sync"
	"syscall"

	"github.com/zetamatta/go-findfile"

	"github.com/zetamatta/nyagos/defined"
	"github.com/zetamatta/nyagos/dos"
)

var WildCardExpansionAlways = false

type CommandNotFound struct {
	Name string
	Err  error
}

func (this CommandNotFound) Stringer() string {
	return fmt.Sprintf("'%s' is not recognized as an internal or external command,\noperable program or batch file", this.Name)
}

func (this CommandNotFound) Error() string {
	return this.Stringer()
}

type session struct {
	unreadline []string
}

type CloneCloser interface {
	Clone(context.Context) (context.Context, CloneCloser, error)
	Close() error
}

type Shell struct {
	*session
	Stdout       *os.File
	Stderr       *os.File
	Stdin        *os.File
	Console      io.Writer
	tag          CloneCloser
	IsBackGround bool
}

func (sh *Shell) In() io.Reader          { return sh.Stdin }
func (sh *Shell) Out() io.Writer         { return sh.Stdout }
func (sh *Shell) Err() io.Writer         { return sh.Stderr }
func (sh *Shell) Term() io.Writer        { return sh.Console }
func (sh *Shell) Tag() CloneCloser       { return sh.tag }
func (sh *Shell) SetTag(tag CloneCloser) { sh.tag = tag }

type Cmd struct {
	Shell
	args            []string
	rawArgs         []string
	fullPath        string
	UseShellExecute bool
	Closers         []io.Closer
}

func (cmd *Cmd) Arg(n int) string      { return cmd.args[n] }
func (cmd *Cmd) Args() []string        { return cmd.args }
func (cmd *Cmd) SetArgs(s []string)    { cmd.args = s }
func (cmd *Cmd) RawArg(n int) string   { return cmd.rawArgs[n] }
func (cmd *Cmd) RawArgs() []string     { return cmd.rawArgs }
func (cmd *Cmd) SetRawArgs(s []string) { cmd.rawArgs = s }

var LookCurdirOrder = dos.LookCurdirFirst

func (cmd *Cmd) FullPath() string {
	if cmd.args == nil || len(cmd.args) <= 0 {
		return ""
	}
	if cmd.fullPath == "" {
		cmd.fullPath = dos.LookPath(LookCurdirOrder, cmd.args[0], "NYAGOSPATH")
	}
	return cmd.fullPath
}

func (cmd *Cmd) Close() {
	if cmd.Closers != nil {
		for _, c := range cmd.Closers {
			c.Close()
		}
		cmd.Closers = nil
	}
}

func (sh *Shell) Close() {}

func New() *Shell {
	return &Shell{
		Stdin:   os.Stdin,
		Stdout:  os.Stdout,
		Stderr:  os.Stderr,
		session: &session{},
	}
}

func (sh *Shell) Command() *Cmd {
	cmd := &Cmd{
		Shell: Shell{
			Stdin:   sh.Stdin,
			Stdout:  sh.Stdout,
			Stderr:  sh.Stderr,
			Console: sh.Console,
			tag:     sh.tag,
		},
	}
	if sh.session != nil {
		cmd.session = sh.session
	} else {
		cmd.session = &session{}
	}
	return cmd
}

type ArgsHookT func(ctx context.Context, sh *Shell, args []string) ([]string, error)

var argsHook = func(ctx context.Context, sh *Shell, args []string) ([]string, error) {
	return args, nil
}

func SetArgsHook(argsHook_ ArgsHookT) (rv ArgsHookT) {
	rv, argsHook = argsHook, argsHook_
	return
}

type HookT func(context.Context, *Cmd) (int, bool, error)

var hook = func(context.Context, *Cmd) (int, bool, error) {
	return 0, false, nil
}

func SetHook(hook_ HookT) (rv HookT) {
	rv, hook = hook, hook_
	return
}

var OnCommandNotFound = func(ctx context.Context, cmd *Cmd, err error) error {
	err = &CommandNotFound{cmd.args[0], err}
	return err
}

var LastErrorLevel int

func makeCmdline(args, rawargs []string) string {
	var buffer strings.Builder
	for i, s := range args {
		if i > 0 {
			buffer.WriteRune(' ')
		}
		if (len(rawargs) > i && len(rawargs[i]) > 0 && rawargs[i][0] == '"') || strings.ContainsAny(s, " &|<>\t\"") {
			fmt.Fprintf(&buffer, `"%s"`, strings.Replace(s, `"`, `\"`, -1))
		} else {
			buffer.WriteString(s)
		}
	}
	return buffer.String()
}

var UseSourceRunBatch = true

func encloseWithQuote(fullpath string) string {
	if strings.ContainsRune(fullpath, ' ') {
		var f strings.Builder
		f.WriteRune('"')
		f.WriteString(fullpath)
		f.WriteRune('"')
		return f.String()
	} else {
		return fullpath
	}
}

func (cmd *Cmd) spawnvpSilent(ctx context.Context) (int, error) {
	// command is empty.
	if len(cmd.args) <= 0 {
		return 0, nil
	}
	if defined.DBG {
		print("spawnvpSilent('", cmd.args[0], "')\n")
	}

	// aliases and lua-commands
	if errorlevel, done, err := hook(ctx, cmd); done || err != nil {
		return errorlevel, err
	}

	// command not found hook
	fullpath := cmd.FullPath()
	if fullpath == "" {
		return 255, OnCommandNotFound(ctx, cmd, os.ErrNotExist)
	}
	cmd.args[0] = fullpath

	if defined.DBG {
		print("exec.LookPath(", cmd.args[0], ")==", fullpath, "\n")
	}
	if WildCardExpansionAlways {
		cmd.args = findfile.Globs(cmd.args)
	}
	if cmd.UseShellExecute {
		// GUI Application
		cmdline := makeCmdline(cmd.args[1:], cmd.rawArgs[1:])
		return 0, dos.ShellExecute("open", fullpath, cmdline, "")
	}
	if UseSourceRunBatch {
		lowerName := strings.ToLower(cmd.args[0])
		if strings.HasSuffix(lowerName, ".cmd") || strings.HasSuffix(lowerName, ".bat") {
			rawargs := cmd.RawArgs()
			args := make([]string, len(rawargs))
			args[0] = encloseWithQuote(fullpath)
			for i, end := 1, len(rawargs); i < end; i++ {
				args[i] = rawargs[i]
			}
			// Batch files
			return RawSource(args, nil, false, cmd.Stdin, cmd.Stdout, cmd.Stderr)
		}
	}
	// Do not use exec.CommandContext because it cancels background process.
	xcmd := exec.Command(cmd.args[0], cmd.args[1:]...)
	xcmd.Stdin = cmd.Stdin
	xcmd.Stdout = cmd.Stdout
	xcmd.Stderr = cmd.Stderr

	if xcmd.SysProcAttr == nil {
		xcmd.SysProcAttr = new(syscall.SysProcAttr)
	}
	cmdline := makeCmdline(xcmd.Args, cmd.rawArgs)
	if defined.DBG {
		println(cmdline)
	}
	xcmd.SysProcAttr.CmdLine = cmdline
	err := xcmd.Run()
	errorlevel, errorlevelOk := dos.GetErrorLevel(xcmd)
	if errorlevelOk {
		return errorlevel, err
	} else {
		return 255, err
	}
}

type AlreadyReportedError struct {
	Err error
}

func (_ AlreadyReportedError) Error() string {
	return ""
}

func IsAlreadyReported(err error) bool {
	_, ok := err.(AlreadyReportedError)
	return ok
}

func (cmd *Cmd) Spawnvp(ctx context.Context) (int, error) {
	errorlevel, err := cmd.spawnvpSilent(ctx)
	if err != nil && err != io.EOF && !IsAlreadyReported(err) {
		if defined.DBG {
			val := reflect.ValueOf(err)
			fmt.Fprintf(cmd.Stderr, "error-type=%s\n", val.Type())
		}
		fmt.Fprintln(cmd.Stderr, err.Error())
		err = AlreadyReportedError{err}
	}
	return errorlevel, err
}

func (sh *Shell) Spawnlp(ctx context.Context, args, rawargs []string) (int, error) {
	cmd := sh.Command()
	defer cmd.Close()
	cmd.SetArgs(args)
	cmd.SetRawArgs(rawargs)
	return cmd.Spawnvp(ctx)
}

func (sh *Shell) Interpret(ctx context.Context, text string) (errorlevel int, finalerr error) {
	if defined.DBG {
		print("Interpret('", text, "')\n")
	}
	if sh == nil {
		return 255, errors.New("Fatal Error: Interpret: instance is nil")
	}
	errorlevel = 0
	finalerr = nil

	statements, statementsErr := Parse(text)
	if statementsErr != nil {
		if defined.DBG {
			print("Parse Error:", statementsErr.Error(), "\n")
		}
		return 0, statementsErr
	}
	if argsHook != nil {
		if defined.DBG {
			print("call argsHook\n")
		}
		for _, pipeline := range statements {
			for _, state := range pipeline {
				var err error
				state.Args, err = argsHook(ctx, sh, state.Args)
				if err != nil {
					return 255, err
				}
			}
		}
		if defined.DBG {
			print("done argsHook\n")
		}
	}
	for _, pipeline := range statements {
		for i, state := range pipeline {
			if state.Term == "|" && (i+1 >= len(pipeline) || len(pipeline[i+1].Args) <= 0) {
				return 255, errors.New("The syntax of the command is incorrect.")
			}
		}
	}

	for _, pipeline := range statements {

		var pipeIn *os.File = nil
		isBackGround := sh.IsBackGround
		for _, state := range pipeline {
			if state.Term == "&" {
				isBackGround = true
				break
			}
		}
		var wg sync.WaitGroup
		shutdown_immediately := false
		for i, state := range pipeline {
			if defined.DBG {
				print(i, ": pipeline loop(", state.Args[0], ")\n")
			}
			cmd := sh.Command()
			cmd.IsBackGround = isBackGround

			if pipeIn != nil {
				cmd.Stdin = pipeIn
				cmd.Closers = append(cmd.Closers, pipeIn)
				pipeIn = nil
			}

			var err error
			if state.Term[0] == '|' {
				var pipeOut *os.File
				pipeIn, pipeOut, err = os.Pipe()
				cmd.Stdout = pipeOut
				if state.Term == "|&" {
					cmd.Stderr = pipeOut
				}
				cmd.Closers = append(cmd.Closers, pipeOut)
			}

			for _, red := range state.Redirect {
				var fd *os.File
				fd, err = red.OpenOn(cmd)
				if err != nil {
					return 0, err
				}
				cmd.Closers = append(cmd.Closers, fd)
			}

			cmd.args = state.Args
			cmd.rawArgs = state.RawArgs
			if i > 0 {
				cmd.IsBackGround = true
			}
			if len(pipeline) == 1 && dos.IsGui(cmd.FullPath()) {
				cmd.UseShellExecute = true
			}
			if i == len(pipeline)-1 && state.Term != "&" {
				// foreground execution.
				errorlevel, finalerr = cmd.Spawnvp(ctx)
				LastErrorLevel = errorlevel
				cmd.Close()
			} else {
				// background
				var newctx context.Context
				if isBackGround {
					// let Context not terminate background-work (#313's 2nd)
					// for the problem gvim starts with empty buffer
					// executing `git blame FILE | type | gvim - &`.
					newctx = context.Background()
				} else {
					wg.Add(1)
					newctx = ctx
				}
				if tag := cmd.Tag(); tag != nil {
					var newtag CloneCloser
					if newctx, newtag, err = tag.Clone(newctx); err != nil {
						fmt.Fprintln(os.Stderr, err.Error())
						return -1, err
					} else {
						cmd.SetTag(newtag)
					}
				}
				go func(ctx1 context.Context, cmd1 *Cmd) {
					if !isBackGround {
						defer wg.Done()
					}
					cmd1.Spawnvp(ctx1)
					if tag := cmd1.Tag(); tag != nil {
						if err := tag.Close(); err != nil {
							fmt.Fprintln(os.Stderr, err.Error())
						}
					}
					cmd1.Close()
				}(newctx, cmd)
			}
		}
		if !isBackGround {
			wg.Wait()
			if shutdown_immediately {
				return errorlevel, nil
			}
			if len(pipeline) > 0 {
				switch pipeline[len(pipeline)-1].Term {
				case "&&":
					if errorlevel != 0 {
						return errorlevel, nil
					}
				case "||":
					if errorlevel == 0 {
						return errorlevel, nil
					}
				}
			}
		}
	}
	return
}
<|endoftext|>


<commit_before>// Copyright 2014 Bobby Powers. All rights reserved.
// Use of this source code is governed by the MIT
// license that can be found in the LICENSE file.

package signedcookie

import (
	"bytes"
	"compress/zlib"
	"crypto/hmac"
	"crypto/sha1"
	"crypto/subtle"
	"encoding/base64"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"strings"
	"time"

	"github.com/bpowers/go-django/internal/github.com/kisielk/og-rek"
)

// Serializer represents the method used to convert a data structure
// into a string.  Currently two serializers are supported, JSON and Pickle.
type Serializer int

const (
	JSON Serializer = iota
	Pickle
)

// Django's default max_age is defined as 2 weeks.
const DefaultMaxAge = 14 * 24 * time.Hour

// the salt value used by the signed_cookies SessionStore, it is not
// configurable through normal means.
const salt = "django.contrib.sessions.backends.signed_cookies"

var defaultSep = []byte{':'}

// b64Encode encodes a slice of bytes in a Django-compatable way,
// trimming trailing '=' padding specified by the standard.
func b64Encode(b []byte) []byte {
	var buf bytes.Buffer
	encoder := base64.NewEncoder(base64.URLEncoding, &buf)
	encoder.Write(b)
	encoder.Close()
	return bytes.TrimRight(buf.Bytes(), "=")
}

// b64Decode decodes a base64-encoded string that was generated by
// Django - specifically it adds any '=' padding that had previously
// been stripped back to the end of the byte slice to ensure Go's
// base64 decoder reads the entire payload.
func b64Decode(b []byte) ([]byte, error) {
	// Django's signing module strips all '=' padding from its
	// encoded representation of b.  Add them back here.
	pad := 4 - (len(b) % 4)
	for i := 0; i < pad; i++ {
		// append is ideal here, because we can overwrite the
		// timestamp that immediately follows the payload and
		// avoid an allocation.
		b = append(b, '=')
	}
	return ioutil.ReadAll(base64.NewDecoder(base64.URLEncoding, bytes.NewReader(b)))
}

var (
	base62Alphabet = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"
)

// b62decode decodes a base62-encoded string into an int64, using the
// same method as Django's django.utils.baseconv.BaseConverter.
func b62Decode(b []byte) (int64, error) {
	var n int64
	for _, d := range b {
		i := strings.IndexByte(base62Alphabet, d)
		if i < 0 {
			return -1, fmt.Errorf("not base62 encoded")
		}
		n = n*int64(len(base62Alphabet)) + int64(i)
	}
	return n, nil
}

// djangoSignature calculates a HMAC signature in a way that matches
// django.core.signing.Signer.signature().
func djangoSignature(salt string, value []byte, secret string) []byte {
	// explicit make + append instead of
	// []byte(salt+"signer"+secret) avoids an allocation. copy
	// instead of append doesn't change allocation count.
	key := make([]byte, 0, len(salt)+len("signer")+len(secret))
	key = append(key, salt...)
	key = append(key, "signer"...)
	key = append(key, secret...)
	mac := hmac.New(sha1.New, key)
	mac.Write(value)
	return b64Encode(mac.Sum(nil))
}

// unsign returns the cookie payload if the signature matches the
// expected signature using the given secret, or an error otherwise.
func unsign(secret string, cookie []byte) ([]byte, error) {
	i := bytes.LastIndex(cookie, defaultSep)
	if i == -1 {
		return nil, fmt.Errorf("expected : in '%s'", string(cookie))
	}
	val := cookie[:i]
	sig := cookie[i+1:]
	expectedSig := djangoSignature(salt, val, secret)
	if subtle.ConstantTimeCompare([]byte(sig), expectedSig) != 1 {
		return nil, fmt.Errorf("signature mismatch: '%s' != '%s'", sig, string(expectedSig))
	}
	return val, nil
}

var now = time.Now

// timestampUnsign returns the cookie payload if the signature matches
// the expected signature using the given secret, and the timestamp of
// the cookie is still valid.  It wraps the unsign method.
func timestampUnsign(maxAge time.Duration, secret string, cookie []byte) ([]byte, error) {
	val, err := unsign(secret, cookie)
	if err != nil {
		return nil, fmt.Errorf("unsign('%s'): %s", string(cookie), err)
	}
	i := bytes.LastIndex(val, defaultSep)
	if i == -1 {
		return nil, fmt.Errorf("expected : in '%s'", string(cookie))
	}
	ts := val[i+1:]
	val = val[:i]
	stamp, err := b62Decode(ts)
	if err != nil {
		return nil, fmt.Errorf("b62Decode: %s", err)
	}
	if time.Unix(stamp, 0).Add(maxAge).Before(now()) {
		return nil, fmt.Errorf("expired timestamp: %d", stamp)
	}
	return val, nil
}

// signingLoads implements cookie object decoding in a way that is
// compatable with django.core.signing.loads.  It returns a map
// representing the encoded object, or an error if one occured.
func signingLoads(s Serializer, maxAge time.Duration, secret, cookie string) (map[string]interface{}, error) {
	c := []byte(cookie) // XXX: does this escape?
	payload, err := timestampUnsign(maxAge, secret, c)
	if err != nil {
		return nil, fmt.Errorf("timestampUnsign: %s", err)
	}
	decompress := false
	if payload[0] == '.' {
		decompress = true
		payload = payload[1:]
	}
	payload, err = b64Decode(payload)
	if err != nil {
		return nil, fmt.Errorf("base64Decode('%s'): %s", string(payload), err)
	}
	if decompress {
		r, err := zlib.NewReader(bytes.NewReader(payload))
		if err != nil {
			return nil, fmt.Errorf("zlib.NewReader: %s", err)
		}
		payload, err = ioutil.ReadAll(r)
		r.Close()
		if err != nil {
			return nil, fmt.Errorf("ReadAll(zlib): %s", err)
		}
	}
	o := make(map[string]interface{})
	if s == JSON {
		json.Unmarshal(payload, &o)
	} else {
		d := ogórek.NewDecoder(bytes.NewReader(payload))
		val, err := d.Decode()
		if err != nil {
			return nil, fmt.Errorf("Decode: %s", err)
		}
		mapI, ok := val.(map[interface{}]interface{})
		if !ok {
			return nil, fmt.Errorf("mapI not an object: %#v", mapI)
		}
		for ki, v := range mapI {
			k, ok := ki.(string)
			if !ok {
				return nil, fmt.Errorf("non-string key in map: %#v", ki)
			}
			o[k] = v
		}
	}
	return o, nil
}

// Decode returns a map representing an object that was encoded and
// signed by the django.contrib.sessions.backends.signed_cookies
// SessionStore, or an error if the cookie could not be decoded or if
// signature validation failed.
func Decode(s Serializer, maxAge time.Duration, secret, cookie string) (map[string]interface{}, error) {
	return signingLoads(s, maxAge, secret, cookie)
}
<commit_msg>doc: add pkgdoc<commit_after>// Copyright 2014 Bobby Powers. All rights reserved.
// Use of this source code is governed by the MIT
// license that can be found in the LICENSE file.

/*
	The signedcookie package implements read-only access to
	cookies generated by Django's signed_cookies session backend.
*/
package signedcookie

import (
	"bytes"
	"compress/zlib"
	"crypto/hmac"
	"crypto/sha1"
	"crypto/subtle"
	"encoding/base64"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"strings"
	"time"

	"github.com/bpowers/go-django/internal/github.com/kisielk/og-rek"
)

// Serializer represents the method used to convert a data structure
// into a string.  Currently two serializers are supported, JSON and Pickle.
type Serializer int

const (
	JSON Serializer = iota
	Pickle
)

// Django's default max_age is defined as 2 weeks.
const DefaultMaxAge = 14 * 24 * time.Hour

// the salt value used by the signed_cookies SessionStore, it is not
// configurable through normal means.
const salt = "django.contrib.sessions.backends.signed_cookies"

var defaultSep = []byte{':'}

// b64Encode encodes a slice of bytes in a Django-compatable way,
// trimming trailing '=' padding specified by the standard.
func b64Encode(b []byte) []byte {
	var buf bytes.Buffer
	encoder := base64.NewEncoder(base64.URLEncoding, &buf)
	encoder.Write(b)
	encoder.Close()
	return bytes.TrimRight(buf.Bytes(), "=")
}

// b64Decode decodes a base64-encoded string that was generated by
// Django - specifically it adds any '=' padding that had previously
// been stripped back to the end of the byte slice to ensure Go's
// base64 decoder reads the entire payload.
func b64Decode(b []byte) ([]byte, error) {
	// Django's signing module strips all '=' padding from its
	// encoded representation of b.  Add them back here.
	pad := 4 - (len(b) % 4)
	for i := 0; i < pad; i++ {
		// append is ideal here, because we can overwrite the
		// timestamp that immediately follows the payload and
		// avoid an allocation.
		b = append(b, '=')
	}
	return ioutil.ReadAll(base64.NewDecoder(base64.URLEncoding, bytes.NewReader(b)))
}

var (
	base62Alphabet = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"
)

// b62decode decodes a base62-encoded string into an int64, using the
// same method as Django's django.utils.baseconv.BaseConverter.
func b62Decode(b []byte) (int64, error) {
	var n int64
	for _, d := range b {
		i := strings.IndexByte(base62Alphabet, d)
		if i < 0 {
			return -1, fmt.Errorf("not base62 encoded")
		}
		n = n*int64(len(base62Alphabet)) + int64(i)
	}
	return n, nil
}

// djangoSignature calculates a HMAC signature in a way that matches
// django.core.signing.Signer.signature().
func djangoSignature(salt string, value []byte, secret string) []byte {
	// explicit make + append instead of
	// []byte(salt+"signer"+secret) avoids an allocation. copy
	// instead of append doesn't change allocation count.
	key := make([]byte, 0, len(salt)+len("signer")+len(secret))
	key = append(key, salt...)
	key = append(key, "signer"...)
	key = append(key, secret...)
	mac := hmac.New(sha1.New, key)
	mac.Write(value)
	return b64Encode(mac.Sum(nil))
}

// unsign returns the cookie payload if the signature matches the
// expected signature using the given secret, or an error otherwise.
func unsign(secret string, cookie []byte) ([]byte, error) {
	i := bytes.LastIndex(cookie, defaultSep)
	if i == -1 {
		return nil, fmt.Errorf("expected : in '%s'", string(cookie))
	}
	val := cookie[:i]
	sig := cookie[i+1:]
	expectedSig := djangoSignature(salt, val, secret)
	if subtle.ConstantTimeCompare([]byte(sig), expectedSig) != 1 {
		return nil, fmt.Errorf("signature mismatch: '%s' != '%s'", sig, string(expectedSig))
	}
	return val, nil
}

var now = time.Now

// timestampUnsign returns the cookie payload if the signature matches
// the expected signature using the given secret, and the timestamp of
// the cookie is still valid.  It wraps the unsign method.
func timestampUnsign(maxAge time.Duration, secret string, cookie []byte) ([]byte, error) {
	val, err := unsign(secret, cookie)
	if err != nil {
		return nil, fmt.Errorf("unsign('%s'): %s", string(cookie), err)
	}
	i := bytes.LastIndex(val, defaultSep)
	if i == -1 {
		return nil, fmt.Errorf("expected : in '%s'", string(cookie))
	}
	ts := val[i+1:]
	val = val[:i]
	stamp, err := b62Decode(ts)
	if err != nil {
		return nil, fmt.Errorf("b62Decode: %s", err)
	}
	if time.Unix(stamp, 0).Add(maxAge).Before(now()) {
		return nil, fmt.Errorf("expired timestamp: %d", stamp)
	}
	return val, nil
}

// signingLoads implements cookie object decoding in a way that is
// compatable with django.core.signing.loads.  It returns a map
// representing the encoded object, or an error if one occured.
func signingLoads(s Serializer, maxAge time.Duration, secret, cookie string) (map[string]interface{}, error) {
	c := []byte(cookie) // XXX: does this escape?
	payload, err := timestampUnsign(maxAge, secret, c)
	if err != nil {
		return nil, fmt.Errorf("timestampUnsign: %s", err)
	}
	decompress := false
	if payload[0] == '.' {
		decompress = true
		payload = payload[1:]
	}
	payload, err = b64Decode(payload)
	if err != nil {
		return nil, fmt.Errorf("base64Decode('%s'): %s", string(payload), err)
	}
	if decompress {
		r, err := zlib.NewReader(bytes.NewReader(payload))
		if err != nil {
			return nil, fmt.Errorf("zlib.NewReader: %s", err)
		}
		payload, err = ioutil.ReadAll(r)
		r.Close()
		if err != nil {
			return nil, fmt.Errorf("ReadAll(zlib): %s", err)
		}
	}
	o := make(map[string]interface{})
	if s == JSON {
		json.Unmarshal(payload, &o)
	} else {
		d := ogórek.NewDecoder(bytes.NewReader(payload))
		val, err := d.Decode()
		if err != nil {
			return nil, fmt.Errorf("Decode: %s", err)
		}
		mapI, ok := val.(map[interface{}]interface{})
		if !ok {
			return nil, fmt.Errorf("mapI not an object: %#v", mapI)
		}
		for ki, v := range mapI {
			k, ok := ki.(string)
			if !ok {
				return nil, fmt.Errorf("non-string key in map: %#v", ki)
			}
			o[k] = v
		}
	}
	return o, nil
}

// Decode returns a map corresponding to the object encoded and signed
// by the django.contrib.sessions.backends.signed_cookies
// SessionStore, or an error if the cookie could not be decoded or if
// signature validation failed.
func Decode(s Serializer, maxAge time.Duration, secret, cookie string) (map[string]interface{}, error) {
	return signingLoads(s, maxAge, secret, cookie)
}
<|endoftext|>


<commit_before>package compression

import (
	"compress/flate"
	"compress/gzip"
	"github.com/TShadwell/fweight"
	"io"
	"net/http"
	"strings"
	"sync"
)

type (
	Compressor interface {
		io.Writer
		Close() error
	}

	Compression func(io.Writer) Compressor
)

var compressions = map[string]Compression{
	"gzip":  Gzip,
	"flate": Flate,
}

var rwm sync.RWMutex

// bodyAllowedForStatus reports whether a given response status code
// permits a body.  See RFC2616, section 4.4.
func bodyAllowedForStatus(status int) bool {
	switch {
	case status >= 100 && status <= 199:
		return false
	case status == 204:
		return false
	case status == 304:
		return false
	}
	return true
}

/*
	Registers a new compression with this package. Gzip and Flate are already registered.
	Name should be canonicalised to lower case.
*/
func Register(name string, c Compression) {
	rwm.Lock()
	compressions[name] = c
	rwm.Unlock()
}

func Gzip(r io.Writer) Compressor {
	return gzip.NewWriter(r)
}

func Flate(r io.Writer) (c Compressor) {
	var err error
	if c, err = flate.NewWriter(r, -1); err != nil {
		panic(err)
	}
	return
}

//writer is a replacement http.ResponseWriter
//used to intercept and compress written bytes.
type writer struct {
	rw http.ResponseWriter
	Compression
	c Compressor
}

func (w *writer) Header() http.Header {
	return w.rw.Header()
}

func (w *writer) Write(b []byte) (int, error) {
	//only load compression on first write.
	//this should precent errors when status disallows body.
	if w.c == nil {
		w.c = w.Compression(w.rw)
	}
	return w.c.Write(b)
}

func (w *writer) WriteHeader(i int) {
	w.rw.WriteHeader(i)
}

func (w *writer) flush() {
	if w.c != nil {
		if err := w.c.Close(); err != nil {
			panic(err)
		}
	}
}

var Middleware = fweight.MiddlewareFunc(func(h http.Handler) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		var flush func()

		defer func() {
			h.ServeHTTP(w, r)
			if flush != nil {
				flush()
			}
		}()

		encs := r.Header.Get("Accept-Encoding")
		if encs == "" {
			return
		}

		var compression Compression
		var encoding string

		rwm.RLock()
		for _, encoding = range strings.Split(strings.ToLower(encs), ",") {
			if compression = compressions[encoding]; compression != nil {
				break
			}
		}
		rwm.RUnlock()

		if compression == nil {
			return
		}

		//copy the old writer
		ow := w

		//create our writer, which compresses.
		uw := writer{
			rw: ow,
			Compression: func(iw io.Writer) Compressor {
				w.Header().Set("Content-Encoding", encoding)
				return compression(iw)
			},
		}

		//replace the writer (for the deferred Handler)
		w = &uw

		//once it's written to our writer, flush it
		flush = uw.flush
	})
})

func init() {

}
<commit_msg>Fix spelling mistake, remove vestigial code<commit_after>package compression

import (
	"compress/flate"
	"compress/gzip"
	"github.com/TShadwell/fweight"
	"io"
	"net/http"
	"strings"
	"sync"
)

type (
	Compressor interface {
		io.Writer
		Close() error
	}

	Compression func(io.Writer) Compressor
)

var compressions = map[string]Compression{
	"gzip":  Gzip,
	"flate": Flate,
}

var rwm sync.RWMutex

/*
	Registers a new compression with this package. Gzip and Flate are already registered.
	Name should be canonicalised to lower case.
*/
func Register(name string, c Compression) {
	rwm.Lock()
	compressions[name] = c
	rwm.Unlock()
}

func Gzip(r io.Writer) Compressor {
	return gzip.NewWriter(r)
}

func Flate(r io.Writer) (c Compressor) {
	var err error
	if c, err = flate.NewWriter(r, -1); err != nil {
		panic(err)
	}
	return
}

//writer is a replacement http.ResponseWriter
//used to intercept and compress written bytes.
type writer struct {
	rw http.ResponseWriter
	Compression
	c Compressor
}

func (w *writer) Header() http.Header {
	return w.rw.Header()
}

func (w *writer) Write(b []byte) (int, error) {
	//only load compression on first write.
	//this should prevent errors when status disallows body.
	if w.c == nil {
		w.c = w.Compression(w.rw)
	}
	return w.c.Write(b)
}

func (w *writer) WriteHeader(i int) {
	w.rw.WriteHeader(i)
}

func (w *writer) flush() {
	if w.c != nil {
		if err := w.c.Close(); err != nil {
			panic(err)
		}
	}
}

var Middleware = fweight.MiddlewareFunc(func(h http.Handler) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		var flush func()

		defer func() {
			h.ServeHTTP(w, r)
			if flush != nil {
				flush()
			}
		}()

		encs := r.Header.Get("Accept-Encoding")
		if encs == "" {
			return
		}

		var compression Compression
		var encoding string

		rwm.RLock()
		for _, encoding = range strings.Split(strings.ToLower(encs), ",") {
			if compression = compressions[encoding]; compression != nil {
				break
			}
		}
		rwm.RUnlock()

		if compression == nil {
			return
		}

		//copy the old writer
		ow := w

		//create our writer, which compresses.
		uw := writer{
			rw: ow,
			Compression: func(iw io.Writer) Compressor {
				w.Header().Set("Content-Encoding", encoding)
				return compression(iw)
			},
		}

		//replace the writer (for the deferred Handler)
		w = &uw

		//once it's written to our writer, flush it
		flush = uw.flush
	})
})

func init() {

}
<|endoftext|>


<commit_before>// Copyright 2018 The Terraformer Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package alicloud

import (
	"strings"

	"github.com/GoogleCloudPlatform/terraformer/terraform_utils"
	"github.com/aliyun/alibaba-cloud-sdk-go/services/ram"
	"github.com/GoogleCloudPlatform/terraformer/providers/alicloud/connectivity"
)

// RAMGenerator Struct for generating AliCloud Elastic Compute Service
type RAMGenerator struct {
	AliCloudService
}

func resourceFromRAMRole(role ram.Role) terraform_utils.Resource {
	return terraform_utils.NewResource(
		role.RoleName,                  // id
		role.RoleId+"__"+role.RoleName, // name
		"alicloud_ram_role",
		"alicloud",
		map[string]string{},
		[]string{},
		map[string]interface{}{},
	)
}

func resourceFromRAMPolicy(policy ram.Policy, roleName string) terraform_utils.Resource {
	// https://github.com/terraform-providers/terraform-provider-alicloud/blob/master/alicloud/resource_alicloud_ram_role_policy_attachment.go#L93
	id := strings.Join([]string{"role", policy.PolicyName, policy.PolicyType, roleName}, ":")

	return terraform_utils.NewResource(
		id, // id
		id+"__"+roleName+"_"+policy.PolicyName, // name
		"alicloud_ram_role_policy_attachment",
		"alicloud",
		map[string]string{},
		[]string{},
		map[string]interface{}{},
	)
}

func initRoles(client *connectivity.AliyunClient) ([]ram.Role, error) {
	allRoles := make([]ram.Role, 0)

	raw, err := client.WithRamClient(func(ramClient *ram.Client) (interface{}, error) {
		request := ram.CreateListRolesRequest()
		request.RegionId = client.RegionId
		return ramClient.ListRoles(request)
	})
	if err != nil {
		return nil, err
	}

	response := raw.(*ram.ListRolesResponse)
	for _, role := range response.Roles.Role {
		allRoles = append(allRoles, role)

	}

	return allRoles, nil
}

func initRAMPolicyAttachment(client *connectivity.AliyunClient, allRoles []ram.Role) ([]ram.Policy, []string, error) {
	allRAMPolicies := make([]ram.Policy, 0)
	roleNames := make([]string, 0)

	for _, role := range allRoles {
		raw, err := client.WithRamClient(func(ramClient *ram.Client) (interface{}, error) {
			request := ram.CreateListPoliciesForRoleRequest()
			request.RegionId = client.RegionId
			request.RoleName = role.RoleName
			return ramClient.ListPoliciesForRole(request)
		})
		if err != nil {
			return nil, nil, err
		}

		response := raw.(*ram.ListPoliciesForRoleResponse)
		for _, policy := range response.Policies.Policy {
			allRAMPolicies = append(allRAMPolicies, policy)
			roleNames = append(roleNames, role.RoleName)
		}
	}

	return allRAMPolicies, roleNames, nil
}

// InitResources Gets the list of all ram role ids and generates resources
func (g *RAMGenerator) InitResources() error {
	client, err := g.LoadClientFromProfile()
	if err != nil {
		return err
	}

	allRoles, err := initRoles(client)
	if err != nil {
		return err
	}

	allRAMPolicyAttachment, roleNames, err := initRAMPolicyAttachment(client, allRoles)
	if err != nil {
		return err
	}

	for _, role := range allRoles {
		resource := resourceFromRAMRole(role)
		g.Resources = append(g.Resources, resource)
	}

	for i, ramPolicy := range allRAMPolicyAttachment {
		resource := resourceFromRAMPolicy(ramPolicy, roleNames[i])
		g.Resources = append(g.Resources, resource)
	}

	return nil
}
<commit_msg>AliCloud: RAM remove deprecated fields<commit_after>// Copyright 2018 The Terraformer Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package alicloud

import (
	"strings"

	"github.com/GoogleCloudPlatform/terraformer/providers/alicloud/connectivity"
	"github.com/GoogleCloudPlatform/terraformer/terraform_utils"
	"github.com/aliyun/alibaba-cloud-sdk-go/services/ram"
)

// RAMGenerator Struct for generating AliCloud Elastic Compute Service
type RAMGenerator struct {
	AliCloudService
}

func resourceFromRAMRole(role ram.Role) terraform_utils.Resource {
	return terraform_utils.NewResource(
		role.RoleName,                  // id
		role.RoleId+"__"+role.RoleName, // name
		"alicloud_ram_role",
		"alicloud",
		map[string]string{},
		[]string{},
		map[string]interface{}{},
	)
}

func resourceFromRAMPolicy(policy ram.Policy, roleName string) terraform_utils.Resource {
	// https://github.com/terraform-providers/terraform-provider-alicloud/blob/master/alicloud/resource_alicloud_ram_role_policy_attachment.go#L93
	id := strings.Join([]string{"role", policy.PolicyName, policy.PolicyType, roleName}, ":")

	return terraform_utils.NewResource(
		id, // id
		id+"__"+roleName+"_"+policy.PolicyName, // name
		"alicloud_ram_role_policy_attachment",
		"alicloud",
		map[string]string{},
		[]string{},
		map[string]interface{}{},
	)
}

func initRoles(client *connectivity.AliyunClient) ([]ram.Role, error) {
	allRoles := make([]ram.Role, 0)

	raw, err := client.WithRamClient(func(ramClient *ram.Client) (interface{}, error) {
		request := ram.CreateListRolesRequest()
		request.RegionId = client.RegionId
		return ramClient.ListRoles(request)
	})
	if err != nil {
		return nil, err
	}

	response := raw.(*ram.ListRolesResponse)
	for _, role := range response.Roles.Role {
		allRoles = append(allRoles, role)

	}

	return allRoles, nil
}

func initRAMPolicyAttachment(client *connectivity.AliyunClient, allRoles []ram.Role) ([]ram.Policy, []string, error) {
	allRAMPolicies := make([]ram.Policy, 0)
	roleNames := make([]string, 0)

	for _, role := range allRoles {
		raw, err := client.WithRamClient(func(ramClient *ram.Client) (interface{}, error) {
			request := ram.CreateListPoliciesForRoleRequest()
			request.RegionId = client.RegionId
			request.RoleName = role.RoleName
			return ramClient.ListPoliciesForRole(request)
		})
		if err != nil {
			return nil, nil, err
		}

		response := raw.(*ram.ListPoliciesForRoleResponse)
		for _, policy := range response.Policies.Policy {
			allRAMPolicies = append(allRAMPolicies, policy)
			roleNames = append(roleNames, role.RoleName)
		}
	}

	return allRAMPolicies, roleNames, nil
}

// InitResources Gets the list of all ram role ids and generates resources
func (g *RAMGenerator) InitResources() error {
	client, err := g.LoadClientFromProfile()
	if err != nil {
		return err
	}

	allRoles, err := initRoles(client)
	if err != nil {
		return err
	}

	allRAMPolicyAttachment, roleNames, err := initRAMPolicyAttachment(client, allRoles)
	if err != nil {
		return err
	}

	for _, role := range allRoles {
		resource := resourceFromRAMRole(role)
		g.Resources = append(g.Resources, resource)
	}

	for i, ramPolicy := range allRAMPolicyAttachment {
		resource := resourceFromRAMPolicy(ramPolicy, roleNames[i])
		g.Resources = append(g.Resources, resource)
	}

	return nil
}

// PostConvertHook Runs before HCL files are generated
func (g *RAMGenerator) PostConvertHook() error {
	for _, r := range g.Resources {
		if r.InstanceInfo.Type == "alicloud_ram_role" {
			// https://www.terraform.io/docs/providers/alicloud/r/ram_role.html
			delete(r.Item, "services")  // deprecated
			delete(r.Item, "ram_users") // deprecated
			delete(r.Item, "version")   // deprecated
		}
	}

	return nil
}
<|endoftext|>


<commit_before>package slackboard

import (
	"log"
	"net"
	"net/http"
	"os"
	"strconv"
	"strings"

	statsGo "github.com/fukata/golang-stats-api-handler"
)

func init() {
	statsGo.PrettyPrintEnabled()
}

func RegisterAPIs() {
	http.HandleFunc("/notify", NotifyHandler)
	http.HandleFunc("/notify-directly", NotifyDirectlyHandler)
	http.HandleFunc("/app/config", ConfigAppHandler)
	http.HandleFunc("/stat/go", statsGo.Handler)
}

func Run() {
	// Listen TCP Port
	if _, err := strconv.Atoi(ConfSlackboard.Core.Port); err == nil {
		http.ListenAndServe(":"+ConfSlackboard.Core.Port, nil)
	}

	// Listen UNIX Socket
	if strings.HasPrefix(ConfSlackboard.Core.Port, "unix:/") {
		sockPath := ConfSlackboard.Core.Port[5:]
		fi, err := os.Lstat(sockPath)
		if err == nil && (fi.Mode()&os.ModeSocket) == os.ModeSocket {
			err := os.Remove(sockPath)
			if err != nil {
				log.Fatalf("failed to remove %s", sockPath)
			}
		}
		l, err := net.Listen("unix", sockPath)
		if err != nil {
			log.Fatalf("failed to listen: %s", sockPath)
		}
		http.Serve(l, nil)
	}

	log.Fatalf("port parameter is invalid: %s", ConfSlackboard.Core.Port)
}
<commit_msg>bugfix: mistaken API endpoint.<commit_after>package slackboard

import (
	"log"
	"net"
	"net/http"
	"os"
	"strconv"
	"strings"

	statsGo "github.com/fukata/golang-stats-api-handler"
)

func init() {
	statsGo.PrettyPrintEnabled()
}

func RegisterAPIs() {
	http.HandleFunc("/notify", NotifyHandler)
	http.HandleFunc("/notify-directly", NotifyDirectlyHandler)
	http.HandleFunc("/config/app", ConfigAppHandler)
	http.HandleFunc("/stat/go", statsGo.Handler)
}

func Run() {
	// Listen TCP Port
	if _, err := strconv.Atoi(ConfSlackboard.Core.Port); err == nil {
		http.ListenAndServe(":"+ConfSlackboard.Core.Port, nil)
	}

	// Listen UNIX Socket
	if strings.HasPrefix(ConfSlackboard.Core.Port, "unix:/") {
		sockPath := ConfSlackboard.Core.Port[5:]
		fi, err := os.Lstat(sockPath)
		if err == nil && (fi.Mode()&os.ModeSocket) == os.ModeSocket {
			err := os.Remove(sockPath)
			if err != nil {
				log.Fatalf("failed to remove %s", sockPath)
			}
		}
		l, err := net.Listen("unix", sockPath)
		if err != nil {
			log.Fatalf("failed to listen: %s", sockPath)
		}
		http.Serve(l, nil)
	}

	log.Fatalf("port parameter is invalid: %s", ConfSlackboard.Core.Port)
}
<|endoftext|>


<commit_before>//
// Copyright (c) 2018 Red Hat, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//

package broker

import (
	"github.com/openshift/ansible-service-broker/pkg/apb"
)

// DeprovisionWorkSubscriber - Lissten for provision messages
type DeprovisionWorkSubscriber struct {
	dao       SubscriberDAO
	msgBuffer <-chan JobMsg
}

// NewDeprovisionWorkSubscriber - Create a new work subscriber.
func NewDeprovisionWorkSubscriber(dao SubscriberDAO) *DeprovisionWorkSubscriber {
	return &DeprovisionWorkSubscriber{dao: dao}
}

// Subscribe - will start the work subscriber listenning on the message buffer for deprovision messages.
func (d *DeprovisionWorkSubscriber) Subscribe(msgBuffer <-chan JobMsg) {
	d.msgBuffer = msgBuffer
	go func() {
		log.Info("Listening for deprovision messages")
		for msg := range msgBuffer {
			log.Debug("received deprovision message from buffer")

			if _, err := d.dao.SetState(msg.InstanceUUID, msg.State); err != nil {
				log.Errorf("failed to set state after deprovision %v", err)
				continue
			}
			//only want to do this on success
			if msg.State.State == apb.StateSucceeded {
				if err := cleanupDeprovision(msg.InstanceUUID, d.dao); err != nil {
					log.Errorf("Failed cleaning up deprovision after job, error: %v", err)
					// Cleanup is reporting something has gone wrong. Deprovision overall
					// has not completed. Mark the job as failed.
					setFailedDeprovisionJob(d.dao, msg)
					continue
				}
			}
		}
	}()
}

func setFailedDeprovisionJob(dao SubscriberDAO, dmsg JobMsg) {
	// have to set the state here manually as the logic that triggers this is in the subscriber
	dmsg.State.State = apb.StateFailed
	if _, err := dao.SetState(dmsg.InstanceUUID, dmsg.State); err != nil {
		log.Errorf("failed to set state after deprovision %v", err)
	}
}

func cleanupDeprovision(instanceID string, dao SubscriberDAO) error {
	if err := dao.DeleteExtractedCredentials(instanceID); err != nil {
		log.Error("failed to delete extracted credentials - %v", err)
		return err
	}

	if err := dao.DeleteServiceInstance(instanceID); err != nil {
		log.Error("failed to delete service instance - %v", err)
		return err
	}

	return nil
}
<commit_msg>Bug 1548311 - Return success on depro extcred miss (#791)<commit_after>//
// Copyright (c) 2018 Red Hat, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//

package broker

import (
	"github.com/coreos/etcd/client"
	"github.com/openshift/ansible-service-broker/pkg/apb"
)

// DeprovisionWorkSubscriber - Lissten for provision messages
type DeprovisionWorkSubscriber struct {
	dao       SubscriberDAO
	msgBuffer <-chan JobMsg
}

// NewDeprovisionWorkSubscriber - Create a new work subscriber.
func NewDeprovisionWorkSubscriber(dao SubscriberDAO) *DeprovisionWorkSubscriber {
	return &DeprovisionWorkSubscriber{dao: dao}
}

// Subscribe - will start the work subscriber listenning on the message buffer for deprovision messages.
func (d *DeprovisionWorkSubscriber) Subscribe(msgBuffer <-chan JobMsg) {
	d.msgBuffer = msgBuffer
	go func() {
		log.Info("Listening for deprovision messages")
		for msg := range msgBuffer {
			log.Debug("received deprovision message from buffer")

			if _, err := d.dao.SetState(msg.InstanceUUID, msg.State); err != nil {
				log.Errorf("failed to set state after deprovision %v", err)
				continue
			}
			//only want to do this on success
			if msg.State.State == apb.StateSucceeded {
				if err := cleanupDeprovision(msg.InstanceUUID, d.dao); err != nil {
					log.Errorf("Failed cleaning up deprovision after job, error: %v", err)
					// Cleanup is reporting something has gone wrong. Deprovision overall
					// has not completed. Mark the job as failed.
					setFailedDeprovisionJob(d.dao, msg)
					continue
				}
			}
		}
	}()
}

func setFailedDeprovisionJob(dao SubscriberDAO, dmsg JobMsg) {
	// have to set the state here manually as the logic that triggers this is in the subscriber
	dmsg.State.State = apb.StateFailed
	if _, err := dao.SetState(dmsg.InstanceUUID, dmsg.State); err != nil {
		log.Errorf("failed to set state after deprovision %v", err)
	}
}

func cleanupDeprovision(instanceID string, dao SubscriberDAO) error {
	err := dao.DeleteExtractedCredentials(instanceID)

	if err != nil && client.IsKeyNotFound(err) {
		log.Infof("Attempted to delete extracted credentials, but key was not found: [%v]", instanceID)
	} else if err != nil {
		return err
	}

	if err := dao.DeleteServiceInstance(instanceID); err != nil {
		log.Error("failed to delete service instance - %v", err)
		return err
	}

	return nil
}
<|endoftext|>


<commit_before>package extensions

import (
	"context"
	"encoding/json"
	"fmt"
	"strings"

	"github.com/telepresenceio/telepresence/v2/pkg/client"
)

// builtinExtensions is a function instead of a would-be-const var because its result includes the
// CLI version number, which might not be initialized yet at init-time (esp. during `go test`).
func builtinExtensions(ctx context.Context) map[string]ExtensionInfo {
	cfg := client.GetConfig(ctx)
	registry := cfg.Images.Registry
	cloud := cfg.Cloud
	version := strings.TrimPrefix(client.Version(), "v")
	image := fmt.Sprintf("%s/tel2:%s", registry, version)
	// XXX: not using net.JoinHostPort means that setting cloud.SystemaHost to an IPv6 address won't work
	extImage := fmt.Sprintf("grpc+https://%s:%s", cloud.SystemaHost, cloud.SystemaPort)
	return map[string]ExtensionInfo{
		// Real extensions won't have a "/" in the extname, by putting one builtin extension names
		// we can avoid clashes.
		"/builtin/telepresence": {
			Image: image,
			Mechanisms: map[string]MechanismInfo{
				"tcp": {},
			},
		},
		// FIXME(lukeshu): We shouldn't compile in the info about the Ambassador Smart Agent
		// extension, but we don't yet have an installer to install the extension file; so this
		// metadata here is fine in the mean-time.
		"/builtin/ambassador": {
			Image:                   extImage,
			RequiresAPIKeyOrLicense: true,
			Mechanisms: map[string]MechanismInfo{
				"http": {
					Preference: 100,
					Flags: map[string]FlagInfo{
						"match": {
							Type:    "string-array",
							Default: json.RawMessage(`["auto"]`),
							Usage: `` +
								`Rather than intercepting all traffic service, only intercept traffic that matches this "HTTP2_HEADER=REGEXP" specifier. ` +
								`Instead of a "--http-match=HTTP2_HEADER=REGEXP" pair, you may say "--http-match=auto", which will automatically select a unique matcher for your intercept. ` +
								`Alternatively, you may say "--http-match=all", which is a no-op, but will inhibit the default "--http-match=auto" when you are logged in. ` +
								`If this flag is given multiple times, then it will only intercept traffic that matches *all* of the specifiers. ` +
								`(default "auto" if you are logged in with 'telepresence login', default "all" otherwise)`,
						},
					},
				},
			},
		},
	}
}
<commit_msg>Add "--http-protocol" option.<commit_after>package extensions

import (
	"context"
	"encoding/json"
	"fmt"
	"strings"

	"github.com/telepresenceio/telepresence/v2/pkg/client"
)

// builtinExtensions is a function instead of a would-be-const var because its result includes the
// CLI version number, which might not be initialized yet at init-time (esp. during `go test`).
func builtinExtensions(ctx context.Context) map[string]ExtensionInfo {
	cfg := client.GetConfig(ctx)
	registry := cfg.Images.Registry
	cloud := cfg.Cloud
	version := strings.TrimPrefix(client.Version(), "v")
	image := fmt.Sprintf("%s/tel2:%s", registry, version)
	// XXX: not using net.JoinHostPort means that setting cloud.SystemaHost to an IPv6 address won't work
	extImage := fmt.Sprintf("grpc+https://%s:%s", cloud.SystemaHost, cloud.SystemaPort)
	return map[string]ExtensionInfo{
		// Real extensions won't have a "/" in the extname, by putting one builtin extension names
		// we can avoid clashes.
		"/builtin/telepresence": {
			Image: image,
			Mechanisms: map[string]MechanismInfo{
				"tcp": {},
			},
		},
		// FIXME(lukeshu): We shouldn't compile in the info about the Ambassador Smart Agent
		// extension, but we don't yet have an installer to install the extension file; so this
		// metadata here is fine in the mean-time.
		"/builtin/ambassador": {
			Image:                   extImage,
			RequiresAPIKeyOrLicense: true,
			Mechanisms: map[string]MechanismInfo{
				"http": {
					Preference: 100,
					Flags: map[string]FlagInfo{
						"match": {
							Type:    "string-array",
							Default: json.RawMessage(`["auto"]`),
							Usage: `` +
								`Rather than intercepting all traffic service, only intercept traffic that matches this "HTTP2_HEADER=REGEXP" specifier. ` +
								`Instead of a "--http-match=HTTP2_HEADER=REGEXP" pair, you may say "--http-match=auto", which will automatically select a unique matcher for your intercept. ` +
								`Alternatively, you may say "--http-match=all", which is a no-op, but will inhibit the default "--http-match=auto" when you are logged in. ` +
								`If this flag is given multiple times, then it will only intercept traffic that matches *all* of the specifiers. ` +
								`(default "auto" if you are logged in with 'telepresence login', default "all" otherwise)`,
						},
						"protocol": {
							Type: "string",
							Usage: `` +
								`Supported protocols are ` +
								`"http" (Plaintext HTTP/1.1), ` +
								`"http2" (Plaintext HTTP/2), ` +
								`"tcp" (Opaque TCP stream, implies --http_match=all), and ` +
								`"tls" (TLS Encrypted data)`,
						},
					},
				},
			},
		},
	}
}
<|endoftext|>


<commit_before>/*
Copyright 2019 The Skaffold Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package cache

import (
	"context"
	"fmt"
	"io"
	"time"

	"github.com/sirupsen/logrus"

	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/build"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/build/tag"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/color"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/docker"
	sErrors "github.com/GoogleContainerTools/skaffold/pkg/skaffold/errors"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/schema/latest"
)

func (c *cache) Build(ctx context.Context, out io.Writer, tags tag.ImageTags, artifacts []*latest.Artifact, buildAndTest BuildAndTestFn) ([]build.Artifact, error) {
	ctx, cancel := context.WithCancel(ctx)
	defer cancel()

	start := time.Now()

	color.Default.Fprintln(out, "Checking cache...")

	lookup := make(chan []cacheDetails)
	go func() { lookup <- c.lookupArtifacts(ctx, tags, artifacts) }()

	var results []cacheDetails
	select {
	case <-ctx.Done():
		return nil, context.Canceled
	case results = <-lookup:
	}

	hashByName := make(map[string]string)
	var needToBuild []*latest.Artifact
	var alreadyBuilt []build.Artifact
	for i, artifact := range artifacts {
		color.Default.Fprintf(out, " - %s: ", artifact.ImageName)

		result := results[i]
		switch result := result.(type) {
		case failed:
			color.Red.Fprintln(out, "Error checking cache.")
			return nil, result.err

		case needsBuilding:
			color.Yellow.Fprintln(out, "Not found. Building")
			hashByName[artifact.ImageName] = result.Hash()
			needToBuild = append(needToBuild, artifact)
			continue

		case needsTagging:
			color.Green.Fprintln(out, "Found. Tagging")
			if err := result.Tag(ctx, c); err != nil {
				return nil, fmt.Errorf("tagging image: %w", err)
			}

		case needsPushing:
			color.Green.Fprintln(out, "Found. Pushing")
			if err := result.Push(ctx, out, c); err != nil {
				return nil, fmt.Errorf("%s: %w", sErrors.PushImageErrPrefix, err)
			}

		default:
			if c.imagesAreLocal {
				color.Green.Fprintln(out, "Found Locally")
			} else {
				color.Green.Fprintln(out, "Found Remotely")
			}
		}

		// Image is already built
		c.cacheMutex.RLock()
		entry := c.artifactCache[result.Hash()]
		c.cacheMutex.RUnlock()
		tag := tags[artifact.ImageName]

		var uniqueTag string
		if c.imagesAreLocal {
			var err error
			uniqueTag, err = build.TagWithImageID(ctx, tag, entry.ID, c.client)
			if err != nil {
				return nil, err
			}
		} else {
			uniqueTag = build.TagWithDigest(tag, entry.Digest)
		}

		alreadyBuilt = append(alreadyBuilt, build.Artifact{
			ImageName: artifact.ImageName,
			Tag:       uniqueTag,
		})
	}

	logrus.Infoln("Cache check complete in", time.Since(start))

	bRes, err := buildAndTest(ctx, out, tags, needToBuild)
	if err != nil {
		return nil, err
	}

	if err := c.addArtifacts(ctx, bRes, hashByName); err != nil {
		logrus.Warnf("error adding artifacts to cache; caching may not work as expected: %v", err)
		return append(bRes, alreadyBuilt...), nil
	}

	if err := saveArtifactCache(c.cacheFile, c.artifactCache); err != nil {
		logrus.Warnf("error saving cache file; caching may not work as expected: %v", err)
		return append(bRes, alreadyBuilt...), nil
	}

	return maintainArtifactOrder(append(bRes, alreadyBuilt...), artifacts), err
}

func maintainArtifactOrder(built []build.Artifact, artifacts []*latest.Artifact) []build.Artifact {
	byName := make(map[string]build.Artifact)
	for _, build := range built {
		byName[build.ImageName] = build
	}

	var ordered []build.Artifact

	for _, artifact := range artifacts {
		ordered = append(ordered, byName[artifact.ImageName])
	}

	return ordered
}

func (c *cache) addArtifacts(ctx context.Context, bRes []build.Artifact, hashByName map[string]string) error {
	for _, a := range bRes {
		entry := ImageDetails{}

		if !c.imagesAreLocal {
			ref, err := docker.ParseReference(a.Tag)
			if err != nil {
				return fmt.Errorf("parsing reference %q: %w", a.Tag, err)
			}

			entry.Digest = ref.Digest
		}

		imageID, err := c.client.ImageID(ctx, a.Tag)
		if err != nil {
			return err
		}

		if imageID != "" {
			entry.ID = imageID
		}

		c.cacheMutex.Lock()
		c.artifactCache[hashByName[a.ImageName]] = entry
		c.cacheMutex.Unlock()
	}

	return nil
}
<commit_msg>Fix return of error adding artifacts to cache when images are built remotely (#4850)<commit_after>/*
Copyright 2019 The Skaffold Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package cache

import (
	"context"
	"fmt"
	"io"
	"time"

	"github.com/sirupsen/logrus"

	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/build"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/build/tag"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/color"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/docker"
	sErrors "github.com/GoogleContainerTools/skaffold/pkg/skaffold/errors"
	"github.com/GoogleContainerTools/skaffold/pkg/skaffold/schema/latest"
)

func (c *cache) Build(ctx context.Context, out io.Writer, tags tag.ImageTags, artifacts []*latest.Artifact, buildAndTest BuildAndTestFn) ([]build.Artifact, error) {
	ctx, cancel := context.WithCancel(ctx)
	defer cancel()

	start := time.Now()

	color.Default.Fprintln(out, "Checking cache...")

	lookup := make(chan []cacheDetails)
	go func() { lookup <- c.lookupArtifacts(ctx, tags, artifacts) }()

	var results []cacheDetails
	select {
	case <-ctx.Done():
		return nil, context.Canceled
	case results = <-lookup:
	}

	hashByName := make(map[string]string)
	var needToBuild []*latest.Artifact
	var alreadyBuilt []build.Artifact
	for i, artifact := range artifacts {
		color.Default.Fprintf(out, " - %s: ", artifact.ImageName)

		result := results[i]
		switch result := result.(type) {
		case failed:
			color.Red.Fprintln(out, "Error checking cache.")
			return nil, result.err

		case needsBuilding:
			color.Yellow.Fprintln(out, "Not found. Building")
			hashByName[artifact.ImageName] = result.Hash()
			needToBuild = append(needToBuild, artifact)
			continue

		case needsTagging:
			color.Green.Fprintln(out, "Found. Tagging")
			if err := result.Tag(ctx, c); err != nil {
				return nil, fmt.Errorf("tagging image: %w", err)
			}

		case needsPushing:
			color.Green.Fprintln(out, "Found. Pushing")
			if err := result.Push(ctx, out, c); err != nil {
				return nil, fmt.Errorf("%s: %w", sErrors.PushImageErrPrefix, err)
			}

		default:
			if c.imagesAreLocal {
				color.Green.Fprintln(out, "Found Locally")
			} else {
				color.Green.Fprintln(out, "Found Remotely")
			}
		}

		// Image is already built
		c.cacheMutex.RLock()
		entry := c.artifactCache[result.Hash()]
		c.cacheMutex.RUnlock()
		tag := tags[artifact.ImageName]

		var uniqueTag string
		if c.imagesAreLocal {
			var err error
			uniqueTag, err = build.TagWithImageID(ctx, tag, entry.ID, c.client)
			if err != nil {
				return nil, err
			}
		} else {
			uniqueTag = build.TagWithDigest(tag, entry.Digest)
		}

		alreadyBuilt = append(alreadyBuilt, build.Artifact{
			ImageName: artifact.ImageName,
			Tag:       uniqueTag,
		})
	}

	logrus.Infoln("Cache check complete in", time.Since(start))

	bRes, err := buildAndTest(ctx, out, tags, needToBuild)
	if err != nil {
		return nil, err
	}

	if err := c.addArtifacts(ctx, bRes, hashByName); err != nil {
		logrus.Warnf("error adding artifacts to cache; caching may not work as expected: %v", err)
		return append(bRes, alreadyBuilt...), nil
	}

	if err := saveArtifactCache(c.cacheFile, c.artifactCache); err != nil {
		logrus.Warnf("error saving cache file; caching may not work as expected: %v", err)
		return append(bRes, alreadyBuilt...), nil
	}

	return maintainArtifactOrder(append(bRes, alreadyBuilt...), artifacts), err
}

func maintainArtifactOrder(built []build.Artifact, artifacts []*latest.Artifact) []build.Artifact {
	byName := make(map[string]build.Artifact)
	for _, build := range built {
		byName[build.ImageName] = build
	}

	var ordered []build.Artifact

	for _, artifact := range artifacts {
		ordered = append(ordered, byName[artifact.ImageName])
	}

	return ordered
}

func (c *cache) addArtifacts(ctx context.Context, bRes []build.Artifact, hashByName map[string]string) error {
	for _, a := range bRes {
		entry := ImageDetails{}
		if c.imagesAreLocal {
			imageID, err := c.client.ImageID(ctx, a.Tag)
			if err != nil {
				return err
			}

			if imageID != "" {
				entry.ID = imageID
			}
		} else {
			ref, err := docker.ParseReference(a.Tag)
			if err != nil {
				return fmt.Errorf("parsing reference %q: %w", a.Tag, err)
			}
			entry.Digest = ref.Digest
		}
		c.cacheMutex.Lock()
		c.artifactCache[hashByName[a.ImageName]] = entry
		c.cacheMutex.Unlock()
	}
	return nil
}
<|endoftext|>