<commit_before>/*
   Copyright 2014 Outbrain Inc.

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
*/

package inst

import (
	"errors"
	"fmt"
	"github.com/github/orchestrator/go/config"
	"github.com/openark/golib/log"
	"github.com/patrickmn/go-cache"
	"net"
	"regexp"
	"strings"
	"time"
)

type HostnameResolve struct {
	hostname         string
	resolvedHostname string
}

func (this HostnameResolve) String() string {
	return fmt.Sprintf("%s %s", this.hostname, this.resolvedHostname)
}

type HostnameUnresolve struct {
	hostname           string
	unresolvedHostname string
}

func (this HostnameUnresolve) String() string {
	return fmt.Sprintf("%s %s", this.hostname, this.unresolvedHostname)
}

var hostnameResolvesLightweightCache *cache.Cache
var hostnameResolvesLightweightCacheLoadedOnceFromDB bool = false

func init() {
	if config.Config.ExpiryHostnameResolvesMinutes < 1 {
		config.Config.ExpiryHostnameResolvesMinutes = 1
	}
	go initializeResolvePostConfiguration()
}

func initializeResolvePostConfiguration() {
	<-config.ConfigurationLoaded

	hostnameResolvesLightweightCache = cache.New(time.Duration(config.Config.ExpiryHostnameResolvesMinutes)*time.Minute, time.Minute)
}

func HostnameResolveMethodIsNone() bool {
	return strings.ToLower(config.Config.HostnameResolveMethod) == "none"
}

// GetCNAME resolves an IP or hostname into a normalized valid CNAME
func GetCNAME(hostname string) (string, error) {
	res, err := net.LookupCNAME(hostname)
	if err != nil {
		return hostname, err
	}
	res = strings.TrimRight(res, ".")
	return res, nil
}

func resolveHostname(hostname string) (string, error) {
	switch strings.ToLower(config.Config.HostnameResolveMethod) {
	case "none":
		return hostname, nil
	case "default":
		return hostname, nil
	case "cname":
		return GetCNAME(hostname)
	}
	return hostname, nil
}

// Attempt to resolve a hostname. This may return a database cached hostname or otherwise
// it may resolve the hostname via CNAME
func ResolveHostname(hostname string) (string, error) {
	hostname = strings.TrimSpace(hostname)
	if hostname == "" {
		return hostname, errors.New("Will not resolve empty hostname")
	}
	if strings.Contains(hostname, ",") {
		return hostname, fmt.Errorf("Will not resolve multi-hostname: %+v", hostname)
	}
	if (&InstanceKey{Hostname: hostname}).IsDetached() {
		// quietly abort. Nothign to do. The hostname is detached for a reason: it
		// will not be resolved, for sure.
		return hostname, nil
	}

	// First go to lightweight cache
	if resolvedHostname, found := hostnameResolvesLightweightCache.Get(hostname); found {
		return resolvedHostname.(string), nil
	}

	if !hostnameResolvesLightweightCacheLoadedOnceFromDB {
		// A continuous-discovery will first make sure to load all resolves from DB.
		// However cli does not do so.
		// Anyway, it seems like the cache was not loaded from DB. Before doing real resolves,
		// let's try and get the resolved hostname from database.
		if !HostnameResolveMethodIsNone() {
			if resolvedHostname, err := ReadResolvedHostname(hostname); err == nil && resolvedHostname != "" {
				hostnameResolvesLightweightCache.Set(hostname, resolvedHostname, 0)
				return resolvedHostname, nil
			}
		}
	}

	// Unfound: resolve!
	log.Debugf("Hostname unresolved yet: %s", hostname)
	resolvedHostname, err := resolveHostname(hostname)
	if config.Config.RejectHostnameResolvePattern != "" {
		// Reject, don't even cache
		if matched, _ := regexp.MatchString(config.Config.RejectHostnameResolvePattern, resolvedHostname); matched {
			log.Warningf("ResolveHostname: %+v resolved to %+v but rejected due to RejectHostnameResolvePattern '%+v'", hostname, resolvedHostname, config.Config.RejectHostnameResolvePattern)
			return hostname, nil
		}
	}

	if err != nil {
		// Problem. What we'll do is cache the hostname for just one minute, so as to avoid flooding requests
		// on one hand, yet make it refresh shortly on the other hand. Anyway do not write to database.
		hostnameResolvesLightweightCache.Set(hostname, resolvedHostname, time.Minute)
		return hostname, err
	}
	// Good result! Cache it, also to DB
	log.Debugf("Cache hostname resolve %s as %s", hostname, resolvedHostname)
	UpdateResolvedHostname(hostname, resolvedHostname)
	return resolvedHostname, nil
}

// UpdateResolvedHostname will store the given resolved hostname in cache
// Returns false when the key already existed with same resolved value (similar
// to AFFECTED_ROWS() in mysql)
func UpdateResolvedHostname(hostname string, resolvedHostname string) bool {
	if resolvedHostname == "" {
		return false
	}
	if existingResolvedHostname, found := hostnameResolvesLightweightCache.Get(hostname); found && (existingResolvedHostname == resolvedHostname) {
		return false
	}
	hostnameResolvesLightweightCache.Set(hostname, resolvedHostname, 0)
	if !HostnameResolveMethodIsNone() {
		WriteResolvedHostname(hostname, resolvedHostname)
	}
	return true
}

func LoadHostnameResolveCache() error {
	if !HostnameResolveMethodIsNone() {
		return loadHostnameResolveCacheFromDatabase()
	}
	return nil
}

func loadHostnameResolveCacheFromDatabase() error {
	allHostnamesResolves, err := ReadAllHostnameResolves()
	if err != nil {
		return err
	}
	for _, hostnameResolve := range allHostnamesResolves {
		hostnameResolvesLightweightCache.Set(hostnameResolve.hostname, hostnameResolve.resolvedHostname, 0)
	}
	hostnameResolvesLightweightCacheLoadedOnceFromDB = true
	return nil
}

func FlushNontrivialResolveCacheToDatabase() error {
	if !HostnameResolveMethodIsNone() {
		return log.Errorf("FlushNontrivialResolveCacheToDatabase() called, but HostnameResolveMethod is %+v", config.Config.HostnameResolveMethod)
	}
	items, _ := HostnameResolveCache()
	for hostname := range items {
		resolvedHostname, found := hostnameResolvesLightweightCache.Get(hostname)
		if found && (resolvedHostname.(string) != hostname) {
			WriteResolvedHostname(hostname, resolvedHostname.(string))
		}
	}
	return nil
}

func ResetHostnameResolveCache() error {
	err := deleteHostnameResolves()
	hostnameResolvesLightweightCache.Flush()
	hostnameResolvesLightweightCacheLoadedOnceFromDB = false
	return err
}

func HostnameResolveCache() (map[string]cache.Item, error) {
	return hostnameResolvesLightweightCache.Items(), nil
}

func UnresolveHostname(instanceKey *InstanceKey) (InstanceKey, bool, error) {
	if *config.RuntimeCLIFlags.SkipUnresolve {
		return *instanceKey, false, nil
	}
	unresolvedHostname, err := readUnresolvedHostname(instanceKey.Hostname)
	if err != nil {
		return *instanceKey, false, log.Errore(err)
	}
	if unresolvedHostname == instanceKey.Hostname {
		// unchanged. Nothing to do
		return *instanceKey, false, nil
	}
	// We unresovled to a different hostname. We will now re-resolve to double-check!
	unresolvedKey := &InstanceKey{Hostname: unresolvedHostname, Port: instanceKey.Port}

	instance, err := ReadTopologyInstance(unresolvedKey)
	if err != nil {
		return *instanceKey, false, log.Errore(err)
	}
	if instance.IsBinlogServer() && config.Config.SkipBinlogServerUnresolveCheck {
		// Do nothing. Everything is assumed to be fine.
	} else if instance.Key.Hostname != instanceKey.Hostname {
		// Resolve(Unresolve(hostname)) != hostname ==> Bad; reject
		if *config.RuntimeCLIFlags.SkipUnresolveCheck {
			return *instanceKey, false, nil
		}
		return *instanceKey, false, log.Errorf("Error unresolving; hostname=%s, unresolved=%s, re-resolved=%s; mismatch. Skip/ignore with --skip-unresolve-check", instanceKey.Hostname, unresolvedKey.Hostname, instance.Key.Hostname)
	}
	return *unresolvedKey, true, nil
}

func RegisterHostnameUnresolve(instanceKey *InstanceKey, unresolvedHostname string) (err error) {
	return WriteHostnameUnresolve(instanceKey, unresolvedHostname)
}
<commit_msg>resolve is a low level code called implicitly on init; cache now set by mutex protection<commit_after>/*
   Copyright 2014 Outbrain Inc.

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
*/

package inst

import (
	"errors"
	"fmt"
	"github.com/github/orchestrator/go/config"
	"github.com/openark/golib/log"
	"github.com/patrickmn/go-cache"
	"net"
	"regexp"
	"strings"
	"sync"
	"time"
)

type HostnameResolve struct {
	hostname         string
	resolvedHostname string
}

func (this HostnameResolve) String() string {
	return fmt.Sprintf("%s %s", this.hostname, this.resolvedHostname)
}

type HostnameUnresolve struct {
	hostname           string
	unresolvedHostname string
}

func (this HostnameUnresolve) String() string {
	return fmt.Sprintf("%s %s", this.hostname, this.unresolvedHostname)
}

var hostnameResolvesLightweightCache *cache.Cache
var hostnameResolvesLightweightCacheInit = &sync.Mutex{}
var hostnameResolvesLightweightCacheLoadedOnceFromDB bool = false

func init() {
	if config.Config.ExpiryHostnameResolvesMinutes < 1 {
		config.Config.ExpiryHostnameResolvesMinutes = 1
	}
}

func getHostnameResolvesLightweightCache() *cache.Cache {
	hostnameResolvesLightweightCacheInit.Lock()
	defer hostnameResolvesLightweightCacheInit.Unlock()
	if hostnameResolvesLightweightCache == nil {
		hostnameResolvesLightweightCache = cache.New(time.Duration(config.Config.ExpiryHostnameResolvesMinutes)*time.Minute, time.Minute)
	}
	return hostnameResolvesLightweightCache
}

func HostnameResolveMethodIsNone() bool {
	return strings.ToLower(config.Config.HostnameResolveMethod) == "none"
}

// GetCNAME resolves an IP or hostname into a normalized valid CNAME
func GetCNAME(hostname string) (string, error) {
	res, err := net.LookupCNAME(hostname)
	if err != nil {
		return hostname, err
	}
	res = strings.TrimRight(res, ".")
	return res, nil
}

func resolveHostname(hostname string) (string, error) {
	switch strings.ToLower(config.Config.HostnameResolveMethod) {
	case "none":
		return hostname, nil
	case "default":
		return hostname, nil
	case "cname":
		return GetCNAME(hostname)
	}
	return hostname, nil
}

// Attempt to resolve a hostname. This may return a database cached hostname or otherwise
// it may resolve the hostname via CNAME
func ResolveHostname(hostname string) (string, error) {
	hostname = strings.TrimSpace(hostname)
	if hostname == "" {
		return hostname, errors.New("Will not resolve empty hostname")
	}
	if strings.Contains(hostname, ",") {
		return hostname, fmt.Errorf("Will not resolve multi-hostname: %+v", hostname)
	}
	if (&InstanceKey{Hostname: hostname}).IsDetached() {
		// quietly abort. Nothign to do. The hostname is detached for a reason: it
		// will not be resolved, for sure.
		return hostname, nil
	}

	// First go to lightweight cache
	if resolvedHostname, found := getHostnameResolvesLightweightCache().Get(hostname); found {
		return resolvedHostname.(string), nil
	}

	if !hostnameResolvesLightweightCacheLoadedOnceFromDB {
		// A continuous-discovery will first make sure to load all resolves from DB.
		// However cli does not do so.
		// Anyway, it seems like the cache was not loaded from DB. Before doing real resolves,
		// let's try and get the resolved hostname from database.
		if !HostnameResolveMethodIsNone() {
			if resolvedHostname, err := ReadResolvedHostname(hostname); err == nil && resolvedHostname != "" {
				getHostnameResolvesLightweightCache().Set(hostname, resolvedHostname, 0)
				return resolvedHostname, nil
			}
		}
	}

	// Unfound: resolve!
	log.Debugf("Hostname unresolved yet: %s", hostname)
	resolvedHostname, err := resolveHostname(hostname)
	if config.Config.RejectHostnameResolvePattern != "" {
		// Reject, don't even cache
		if matched, _ := regexp.MatchString(config.Config.RejectHostnameResolvePattern, resolvedHostname); matched {
			log.Warningf("ResolveHostname: %+v resolved to %+v but rejected due to RejectHostnameResolvePattern '%+v'", hostname, resolvedHostname, config.Config.RejectHostnameResolvePattern)
			return hostname, nil
		}
	}

	if err != nil {
		// Problem. What we'll do is cache the hostname for just one minute, so as to avoid flooding requests
		// on one hand, yet make it refresh shortly on the other hand. Anyway do not write to database.
		getHostnameResolvesLightweightCache().Set(hostname, resolvedHostname, time.Minute)
		return hostname, err
	}
	// Good result! Cache it, also to DB
	log.Debugf("Cache hostname resolve %s as %s", hostname, resolvedHostname)
	UpdateResolvedHostname(hostname, resolvedHostname)
	return resolvedHostname, nil
}

// UpdateResolvedHostname will store the given resolved hostname in cache
// Returns false when the key already existed with same resolved value (similar
// to AFFECTED_ROWS() in mysql)
func UpdateResolvedHostname(hostname string, resolvedHostname string) bool {
	if resolvedHostname == "" {
		return false
	}
	if existingResolvedHostname, found := getHostnameResolvesLightweightCache().Get(hostname); found && (existingResolvedHostname == resolvedHostname) {
		return false
	}
	getHostnameResolvesLightweightCache().Set(hostname, resolvedHostname, 0)
	if !HostnameResolveMethodIsNone() {
		WriteResolvedHostname(hostname, resolvedHostname)
	}
	return true
}

func LoadHostnameResolveCache() error {
	if !HostnameResolveMethodIsNone() {
		return loadHostnameResolveCacheFromDatabase()
	}
	return nil
}

func loadHostnameResolveCacheFromDatabase() error {
	allHostnamesResolves, err := ReadAllHostnameResolves()
	if err != nil {
		return err
	}
	for _, hostnameResolve := range allHostnamesResolves {
		getHostnameResolvesLightweightCache().Set(hostnameResolve.hostname, hostnameResolve.resolvedHostname, 0)
	}
	hostnameResolvesLightweightCacheLoadedOnceFromDB = true
	return nil
}

func FlushNontrivialResolveCacheToDatabase() error {
	if !HostnameResolveMethodIsNone() {
		return log.Errorf("FlushNontrivialResolveCacheToDatabase() called, but HostnameResolveMethod is %+v", config.Config.HostnameResolveMethod)
	}
	items, _ := HostnameResolveCache()
	for hostname := range items {
		resolvedHostname, found := getHostnameResolvesLightweightCache().Get(hostname)
		if found && (resolvedHostname.(string) != hostname) {
			WriteResolvedHostname(hostname, resolvedHostname.(string))
		}
	}
	return nil
}

func ResetHostnameResolveCache() error {
	err := deleteHostnameResolves()
	getHostnameResolvesLightweightCache().Flush()
	hostnameResolvesLightweightCacheLoadedOnceFromDB = false
	return err
}

func HostnameResolveCache() (map[string]cache.Item, error) {
	return getHostnameResolvesLightweightCache().Items(), nil
}

func UnresolveHostname(instanceKey *InstanceKey) (InstanceKey, bool, error) {
	if *config.RuntimeCLIFlags.SkipUnresolve {
		return *instanceKey, false, nil
	}
	unresolvedHostname, err := readUnresolvedHostname(instanceKey.Hostname)
	if err != nil {
		return *instanceKey, false, log.Errore(err)
	}
	if unresolvedHostname == instanceKey.Hostname {
		// unchanged. Nothing to do
		return *instanceKey, false, nil
	}
	// We unresovled to a different hostname. We will now re-resolve to double-check!
	unresolvedKey := &InstanceKey{Hostname: unresolvedHostname, Port: instanceKey.Port}

	instance, err := ReadTopologyInstance(unresolvedKey)
	if err != nil {
		return *instanceKey, false, log.Errore(err)
	}
	if instance.IsBinlogServer() && config.Config.SkipBinlogServerUnresolveCheck {
		// Do nothing. Everything is assumed to be fine.
	} else if instance.Key.Hostname != instanceKey.Hostname {
		// Resolve(Unresolve(hostname)) != hostname ==> Bad; reject
		if *config.RuntimeCLIFlags.SkipUnresolveCheck {
			return *instanceKey, false, nil
		}
		return *instanceKey, false, log.Errorf("Error unresolving; hostname=%s, unresolved=%s, re-resolved=%s; mismatch. Skip/ignore with --skip-unresolve-check", instanceKey.Hostname, unresolvedKey.Hostname, instance.Key.Hostname)
	}
	return *unresolvedKey, true, nil
}

func RegisterHostnameUnresolve(instanceKey *InstanceKey, unresolvedHostname string) (err error) {
	return WriteHostnameUnresolve(instanceKey, unresolvedHostname)
}
<|endoftext|>


<commit_before>package lua

var cmdRc = `
func _fallback(name, val)
    if type(val) == 'function' then
        val()
    else
        print val
    end
end

func _is_public(name)
    return _builtins[name] != true and name:sub(1, 1) != '_'
end

func help()
    local funcs = {}
    local vars = {}
    local vkeys = {}
    for name, val in pairs(_G) do
        if _is_public(name) then
            if type(val) == 'function' then
                table.insert(funcs, name)
            else
                vars[name] = val
                table.insert(vkeys, name)
            end
        end
    end
    table.sort(funcs)
    print 'Functions:'
    for _, name in ipairs(funcs) do
        print(name)
    end
    print

    table.sort(vkeys)
    print 'Variables:'
    for _, name in ipairs(vkeys) do
        local val = vars[name]
        print name '=' val
    end
end

func dir()
    local ret = {}
    for name, _ in pairs(_G) do
        if _is_public(name) then
            table.insert(ret, name)
        end
    end
    table.sort(ret)
    return ret
end

local _hook_types = {
    -- TODO: hook memory map?
    -- sys and sys_pre are hardcoded to call hook_sys
    code = cpu.HOOK_CODE,
    block = cpu.HOOK_BLOCK,
    intr = cpu.HOOK_INTR,
    read = cpu.HOOK_MEM_READ,
    write = cpu.HOOK_MEM_WRITE,
    fault = cpu.HOOK_MEM_ERR,
}

func _on_hook(name, fn, start, stop)
    if name == 'sys_pre' then
        return u.hook_sys_add(start, fn, nil)
    end
    if name == 'sys' then
        return u.hook_sys_add(start, nil, fn)
    end
    local type = _hook_types[name]
    if type != nil then
        if start == nil then
            return u.hook_add(type, fn)
        else
            if stop == nil then stop = start end
            return u.hook_add(type, fn, start, stop)
        end
    end
    print 'unimplemented hook type %s' % name
    -- u.on_hook_add(name, fn, start, stop)
end

func off()
    if hh then
        u.hook_del(hh)
        u.hook_sys_del(hh)
        hh = nil
    end
end

func read(addr, size)
    if size == nil then size = 16 end
    return u.mem_read(addr, size)
end

func write(addr, s)
    u.mem_write(addr, s)
end

func maps(addr)
    print 'Memory map:'
    local m = us:Mappings()
    for i in m() do
        local map = m[i]
        if addr != nil then
            if map:Contains(addr) then
                print map:String()
                print '%#x: %#x+%#x' % {addr, map.Addr, addr-map.Addr}
                return
            end
        else
            print map:String()
        end
    end
    if addr != nil then
        print '%#x not found.' % addr
    end
end

func map(addr, size, prot)
    if addr == nil then
        maps()
    else
        if prot == nil then prot = cpu.PROT_ALL end
        u.mem_map(addr, size, prot)
    end
end

func asm(src, addr)
    if addr == nil then addr = 0 end
    return u.asm(src, addr)
end

func patch(addr, src)
    local x = asm(src, addr)
    -- autonop
    local total = 0
    for _, ins in pairs(u.dis(addr, x:len() + 16)) do
        total = total + ins.bytes:len()
        if total == x:len() then break end
        if total > x:len() then
            x = x .. asm('nop', 0):rep(total - x:len())
            break
        end
    end
    write(addr, x)
end

func dis(addr, size, indent, count)
    if addr == nil then addr = pc end
    if size == nil then size = 16 end
    if indent == nil then indent = 0 end
    local pad = (' '):rep(indent)
    local d = u.dis(addr, size)
    local width = 0
    for i, ins in ipairs(d) do
        if count != nil and i > count then break end
        local fmt = pad .. '0x%x: %s %s'
        print fmt % {ins.addr, ins.name, ins.op_str}
    end
end

func s(steps)
    if steps == nil then steps = 1 end
    u.step(steps)
end

c = u.continue
func rewind(n)
    if n == nil then n = 1 end
    u.rewind_n(n)
end
rw = rewind
rwaddr = u.rewind_addr

func rwto(insto)
    repeat
        rw 1
    until ins.name == insto
end

func b(baddr)
    local hh = u.hook_add(cpu.HOOK_CODE, func()
        if bskip == baddr then
            bskip = nil
            return
        end
        bskip = baddr
        print 'Breakpoint N hit at 0x%x' % baddr
        u.stop()
    end, baddr, baddr)
    print 'Breakpoint N at 0x%x' % baddr
end

func runto(name)
    on code do
        if ins.name == name then
            print '[-] stopping at "%s"' % name
            dis pc 16 0 1
            u.stop()
            off
        end
    end
    c
end

func regs(name)
    reglist, err = us:RegDump()
    for _, reg in reglist() do
        print reg.Name reg.Val
    end
end
r = regs
`
<commit_msg>add unmap and malloc repl/luaish functions<commit_after>package lua

var cmdRc = `
func _fallback(name, val)
    if type(val) == 'function' then
        val()
    else
        print val
    end
end

func _is_public(name)
    return _builtins[name] != true and name:sub(1, 1) != '_'
end

func help()
    local funcs = {}
    local vars = {}
    local vkeys = {}
    for name, val in pairs(_G) do
        if _is_public(name) then
            if type(val) == 'function' then
                table.insert(funcs, name)
            else
                vars[name] = val
                table.insert(vkeys, name)
            end
        end
    end
    table.sort(funcs)
    print 'Functions:'
    for _, name in ipairs(funcs) do
        print(name)
    end
    print

    table.sort(vkeys)
    print 'Variables:'
    for _, name in ipairs(vkeys) do
        local val = vars[name]
        print name '=' val
    end
end

func dir()
    local ret = {}
    for name, _ in pairs(_G) do
        if _is_public(name) then
            table.insert(ret, name)
        end
    end
    table.sort(ret)
    return ret
end

local _hook_types = {
    -- TODO: hook memory map?
    -- sys and sys_pre are hardcoded to call hook_sys
    code = cpu.HOOK_CODE,
    block = cpu.HOOK_BLOCK,
    intr = cpu.HOOK_INTR,
    read = cpu.HOOK_MEM_READ,
    write = cpu.HOOK_MEM_WRITE,
    fault = cpu.HOOK_MEM_ERR,
}

func _on_hook(name, fn, start, stop)
    if name == 'sys_pre' then
        return u.hook_sys_add(start, fn, nil)
    end
    if name == 'sys' then
        return u.hook_sys_add(start, nil, fn)
    end
    local type = _hook_types[name]
    if type != nil then
        if start == nil then
            return u.hook_add(type, fn)
        else
            if stop == nil then stop = start end
            return u.hook_add(type, fn, start, stop)
        end
    end
    print 'unimplemented hook type %s' % name
    -- u.on_hook_add(name, fn, start, stop)
end

func off()
    if hh then
        u.hook_del(hh)
        u.hook_sys_del(hh)
        hh = nil
    end
end

func read(addr, size)
    if size == nil then size = 16 end
    return u.mem_read(addr, size)
end

func write(addr, s)
    u.mem_write(addr, s)
end

func maps(addr)
    print 'Memory map:'
    local m = us:Mappings()
    for i in m() do
        local map = m[i]
        if addr != nil then
            if map:Contains(addr) then
                print map:String()
                print '%#x: %#x+%#x' % {addr, map.Addr, addr-map.Addr}
                return
            end
        else
            print map:String()
        end
    end
    if addr != nil then
        print '%#x not found.' % addr
    end
end

func malloc(size, desc)
	if desc == nil then desc = "[script]" end
	addr, err = us:Malloc(size, desc)
	if err != nil then
		print err
	else
		return addr
	end
end

func map(addr, size, prot)
    if addr == nil then
        maps()
    else
        if prot == nil then prot = cpu.PROT_ALL end
        u.mem_map(addr, size, prot)
    end
end

func unmap(addr, size)
	u.mem_unmap(addr, size)
end

func asm(src, addr)
    if addr == nil then addr = 0 end
    return u.asm(src, addr)
end

func patch(addr, src)
    local x = asm(src, addr)
    -- autonop
    local total = 0
    for _, ins in pairs(u.dis(addr, x:len() + 16)) do
        total = total + ins.bytes:len()
        if total == x:len() then break end
        if total > x:len() then
            x = x .. asm('nop', 0):rep(total - x:len())
            break
        end
    end
    write(addr, x)
end

func dis(addr, size, indent, count)
    if addr == nil then addr = pc end
    if size == nil then size = 16 end
    if indent == nil then indent = 0 end
    local pad = (' '):rep(indent)
    local d = u.dis(addr, size)
    local width = 0
    for i, ins in ipairs(d) do
        if count != nil and i > count then break end
        local fmt = pad .. '0x%x: %s %s'
        print fmt % {ins.addr, ins.name, ins.op_str}
    end
end

func s(steps)
    if steps == nil then steps = 1 end
    u.step(steps)
end

c = u.continue
func rewind(n)
    if n == nil then n = 1 end
    u.rewind_n(n)
end
rw = rewind
rwaddr = u.rewind_addr

func rwto(insto)
    repeat
        rw 1
    until ins.name == insto
end

func b(baddr)
    local hh = u.hook_add(cpu.HOOK_CODE, func()
        if bskip == baddr then
            bskip = nil
            return
        end
        bskip = baddr
        print 'Breakpoint N hit at 0x%x' % baddr
        u.stop()
    end, baddr, baddr)
    print 'Breakpoint N at 0x%x' % baddr
end

func runto(name)
    on code do
        if ins.name == name then
            print '[-] stopping at "%s"' % name
            dis pc 16 0 1
            u.stop()
            off
        end
    end
    c
end

func regs(name)
    reglist, err = us:RegDump()
    for _, reg in reglist() do
        print reg.Name reg.Val
    end
end
r = regs
`
<|endoftext|>


<commit_before>package cli

import (
	"fmt"
	"log"
	//"strings"

	"command"
)

func Execute(ctx command.ConfContext, line string, isLine, history bool, c *Client) {
	log.Printf("cli.Execute: isLine=%v cmd=[%s]", isLine, line)

	if isLine {
		// full-line command
		executeLine(ctx, line, history, c)
		return
	}

	// single-char command
	executeKey(ctx, line, c)
}

func executeKey(ctx command.ConfContext, line string, c *Client) {
	log.Printf("executeKey(): [%v]", line)

	if line == "q" {
		c.outputQueue = nil // discard output queue
	}

	// key feedback
	// RETURN is empty line (line == "")
	c.Output() <- fmt.Sprintf("%s\r\n", line)

	commandFeedback(c, getHostname(ctx.ConfRootActive()))
}

func getHostname(root *command.ConfNode) string {
	node, err := root.Get("hostname")
	if err != nil {
		return "hostname?"
	}

	return node.Value[0]
}

func isAnyUserDefined(root *command.ConfNode) bool {
	node, err := root.Get("username")
	if err != nil {
		return false
	}

	return len(node.Children) > 0
}

func checkPassword(root *command.ConfNode, username, password string) bool {
	path := fmt.Sprintf("username %s password", username)

	node, err := root.Get(path)
	if err != nil {
		return false
	}

	if len(node.Value) != 1 {
		return false
	}

	return password == node.Value[0]
}

func executeLine(ctx command.ConfContext, line string, history bool, c *Client) {
	log.Printf("executeLine: [%v]", line)

	status := c.Status()

	switch status {
	case command.MOTD:
		c.Sendln("")
		c.Sendln("rib server ready")
		c.Sendln("")
		if isAnyUserDefined(ctx.ConfRootActive()) {
			c.StatusSet(command.USER) // request user/password auth
		} else {
			c.StatusSet(command.EXEC) // login without user/password auth
		}
	case command.USER:
		c.EchoDisable()
		c.StatusSet(command.PASS)
		c.UsernameSet(line)
	case command.PASS:
		c.EchoEnable()
		if checkPassword(ctx.ConfRootActive(), c.Username(), line) {
			c.StatusSet(command.EXEC) // login allowed
		} else {
			c.StatusSet(command.USER) // request user again
		}
	case command.EXEC, command.ENAB, command.CONF:
		if err := command.Dispatch(ctx, line, c, status, history); err != nil {
			c.Sendln(fmt.Sprintf("executeLine: error: %v", err))
		}
	default:
		msg := fmt.Sprintf("unknown state for command: [%s]", line)
		log.Print(msg)
		c.Sendln(msg)
	}

	commandFeedback(c, getHostname(ctx.ConfRootActive()))
}

func commandFeedback(c *Client, hostname string) {
	paging := c.SendQueue()
	c.SetSendEveryChar(paging)
	c.SendPrompt(hostname, paging)
	c.Flush()
}
<commit_msg>Remove daemon specific label from shared helper command.<commit_after>package cli

import (
	"fmt"
	"log"
	//"strings"

	"command"
)

func Execute(ctx command.ConfContext, line string, isLine, history bool, c *Client) {
	log.Printf("cli.Execute: isLine=%v cmd=[%s]", isLine, line)

	if isLine {
		// full-line command
		executeLine(ctx, line, history, c)
		return
	}

	// single-char command
	executeKey(ctx, line, c)
}

func executeKey(ctx command.ConfContext, line string, c *Client) {
	log.Printf("executeKey(): [%v]", line)

	if line == "q" {
		c.outputQueue = nil // discard output queue
	}

	// key feedback
	// RETURN is empty line (line == "")
	c.Output() <- fmt.Sprintf("%s\r\n", line)

	commandFeedback(c, getHostname(ctx.ConfRootActive()))
}

func getHostname(root *command.ConfNode) string {
	node, err := root.Get("hostname")
	if err != nil {
		return "hostname?"
	}

	return node.Value[0]
}

func isAnyUserDefined(root *command.ConfNode) bool {
	node, err := root.Get("username")
	if err != nil {
		return false
	}

	return len(node.Children) > 0
}

func checkPassword(root *command.ConfNode, username, password string) bool {
	path := fmt.Sprintf("username %s password", username)

	node, err := root.Get(path)
	if err != nil {
		return false
	}

	if len(node.Value) != 1 {
		return false
	}

	return password == node.Value[0]
}

func executeLine(ctx command.ConfContext, line string, history bool, c *Client) {
	log.Printf("executeLine: [%v]", line)

	status := c.Status()

	switch status {
	case command.MOTD:
		c.Sendln("")
		c.Sendln("vty server ready")
		c.Sendln("")
		if isAnyUserDefined(ctx.ConfRootActive()) {
			c.StatusSet(command.USER) // request user/password auth
		} else {
			c.StatusSet(command.EXEC) // login without user/password auth
		}
	case command.USER:
		c.EchoDisable()
		c.StatusSet(command.PASS)
		c.UsernameSet(line)
	case command.PASS:
		c.EchoEnable()
		if checkPassword(ctx.ConfRootActive(), c.Username(), line) {
			c.StatusSet(command.EXEC) // login allowed
		} else {
			c.StatusSet(command.USER) // request user again
		}
	case command.EXEC, command.ENAB, command.CONF:
		if err := command.Dispatch(ctx, line, c, status, history); err != nil {
			c.Sendln(fmt.Sprintf("executeLine: error: %v", err))
		}
	default:
		msg := fmt.Sprintf("unknown state for command: [%s]", line)
		log.Print(msg)
		c.Sendln(msg)
	}

	commandFeedback(c, getHostname(ctx.ConfRootActive()))
}

func commandFeedback(c *Client, hostname string) {
	paging := c.SendQueue()
	c.SetSendEveryChar(paging)
	c.SendPrompt(hostname, paging)
	c.Flush()
}
<|endoftext|>


<commit_before>package gocroaring

import (
	"fmt"
	"testing"
)

// go test -run MemoryUsage
func TestMemoryUsage(t *testing.T) {
  bitmap := New()
  for i:= 0; i < 1000000; i++ {
    bitmap.Add(uint32(i)*10)
  }
  sb := bitmap.SerializedSizeInBytes()
  memory_alloc := 8*1024*1024*1024
  howmany := (memory_alloc + sb - 1)/ sb
  fmt.Println("size in MB of one bitmap ", sb/(1024*1024), "; number of copies = ", howmany, "; total alloc: ", howmany * sb / (1024*1024*1024), "GB")
  for i:= 0; i < howmany; i++ {
    y := bitmap.Clone()
    _ = y
  }
}

func TestSimpleCard(t *testing.T) {
	bitmap := New()
	for i := 100; i < 1000; i++ {
		bitmap.Add(uint32(i))
	}
	c := bitmap.Cardinality()
	fmt.Println("cardinality: ", c)
	if c != 900 {
		t.Error("Expected ", 900, ", got ", c)
	}
	bitmap.RunOptimize()
	if c != 900 {
		t.Error("Expected ", 900, ", got ", c)
	}
}

func TestNewWithVals(t *testing.T) {
	vals := []uint32{1, 2, 3, 6, 7, 8, 20, 44444}
	rb := New(vals...)
	for _, v := range vals {
		if !rb.Contains(v) {
			t.Errorf("expected %d from initialized values\n", v)
		}
	}
}

func TestAddMany(t *testing.T) {
	rb1 := New()
	sl := []uint32{1, 2, 3, 6, 7, 8, 20, 44444}
	rb1.Add(sl...)

	if int(rb1.Cardinality()) != len(sl) {
		t.Errorf("cardinality: expected %d, got %d", rb1.Cardinality(), len(sl))
	}
	if rb1.Contains(5) {
		t.Error("didn't expect to contain 5")
	}
	for _, v := range sl {
		if !rb1.Contains(v) {
			t.Errorf("expected to contain %d", v)
		}
	}
}

func TestFancier(t *testing.T) {
	rb1 := New()
	rb1.Add(1)
	rb1.Add(2)
	rb1.Add(3)
	rb1.Add(4)
	rb1.Add(5)
	rb1.Add(100)
	rb1.Add(1000)
	rb1.RunOptimize()
	rb2 := New()
	rb2.Add(3)
	rb2.Add(4)
	rb2.Add(1000)
	rb2.RunOptimize()
	rb3 := New()
	fmt.Println("Cardinality: ", rb1.Cardinality())
	if rb1.Cardinality() != 7 {
		t.Error("Bad card")
	}
	if !rb1.Contains(3) {
		t.Error("should contain it")
	}
	rb1.And(rb2)
	fmt.Println(rb1)
	rb3.Add(5)
	rb3.Or(rb1)
	fmt.Println(rb3.ToArray())
	fmt.Println(rb3)
	rb4 := FastOr(rb1, rb2, rb3)
	fmt.Println(rb4)
	// next we include an example of serialization
	buf := make([]byte, rb1.SerializedSizeInBytes())
	rb1.Write(buf) // we omit error handling
	newrb, _ := Read(buf)
	if rb1.Equals(newrb) {
		fmt.Println("I wrote the content to a byte stream and read it back.")
	} else {
		t.Error("Bad read")
	}
}

func TestStats(t *testing.T) {

	rb := New()
	rb.Add(1, 2, 3, 4, 6, 7)
	rb.Add(999991, 999992, 999993, 999994, 999996, 999997)

	stats := rb.Stats()
	if stats["cardinality"] != rb.Cardinality() {
		t.Errorf("cardinality: expected %d got %d\n", rb.Cardinality(), stats["cardinality"])
	}

	if stats["n_containers"] != 2 {
		t.Errorf("n_containers: expected %d got %d\n", 2, stats["n_containers"], stats)
	}
	if stats["n_array_containers"] != 2 {
		t.Errorf("n_array_containers: expected %d got %d\n", 2, stats["n_array_containers"], stats)
	}
	for _, c := range []string{"n_run_containers", "n_bitmap_containers"} {
		if stats[c] != 0 {
			t.Errorf("%s: expected 0 got %d\n", 2, c, stats[c])
		}
	}
}
<commit_msg>go fmt<commit_after>package gocroaring

import (
	"fmt"
	"testing"
)

// go test -run MemoryUsage
func TestMemoryUsage(t *testing.T) {
	bitmap := New()
	for i := 0; i < 1000000; i++ {
		bitmap.Add(uint32(i) * 10)
	}
	sb := bitmap.SerializedSizeInBytes()
	memory_alloc := 8 * 1024 * 1024 * 1024
	howmany := (memory_alloc + sb - 1) / sb
	fmt.Println("size in MB of one bitmap ", sb/(1024*1024), "; number of copies = ", howmany, "; total alloc: ", howmany*sb/(1024*1024*1024), "GB")
	for i := 0; i < howmany; i++ {
		y := bitmap.Clone()
		_ = y
	}
}

func TestSimpleCard(t *testing.T) {
	bitmap := New()
	for i := 100; i < 1000; i++ {
		bitmap.Add(uint32(i))
	}
	c := bitmap.Cardinality()
	fmt.Println("cardinality: ", c)
	if c != 900 {
		t.Error("Expected ", 900, ", got ", c)
	}
	bitmap.RunOptimize()
	if c != 900 {
		t.Error("Expected ", 900, ", got ", c)
	}
}

func TestNewWithVals(t *testing.T) {
	vals := []uint32{1, 2, 3, 6, 7, 8, 20, 44444}
	rb := New(vals...)
	for _, v := range vals {
		if !rb.Contains(v) {
			t.Errorf("expected %d from initialized values\n", v)
		}
	}
}

func TestAddMany(t *testing.T) {
	rb1 := New()
	sl := []uint32{1, 2, 3, 6, 7, 8, 20, 44444}
	rb1.Add(sl...)

	if int(rb1.Cardinality()) != len(sl) {
		t.Errorf("cardinality: expected %d, got %d", rb1.Cardinality(), len(sl))
	}
	if rb1.Contains(5) {
		t.Error("didn't expect to contain 5")
	}
	for _, v := range sl {
		if !rb1.Contains(v) {
			t.Errorf("expected to contain %d", v)
		}
	}
}

func TestFancier(t *testing.T) {
	rb1 := New()
	rb1.Add(1)
	rb1.Add(2)
	rb1.Add(3)
	rb1.Add(4)
	rb1.Add(5)
	rb1.Add(100)
	rb1.Add(1000)
	rb1.RunOptimize()
	rb2 := New()
	rb2.Add(3)
	rb2.Add(4)
	rb2.Add(1000)
	rb2.RunOptimize()
	rb3 := New()
	fmt.Println("Cardinality: ", rb1.Cardinality())
	if rb1.Cardinality() != 7 {
		t.Error("Bad card")
	}
	if !rb1.Contains(3) {
		t.Error("should contain it")
	}
	rb1.And(rb2)
	fmt.Println(rb1)
	rb3.Add(5)
	rb3.Or(rb1)
	fmt.Println(rb3.ToArray())
	fmt.Println(rb3)
	rb4 := FastOr(rb1, rb2, rb3)
	fmt.Println(rb4)
	// next we include an example of serialization
	buf := make([]byte, rb1.SerializedSizeInBytes())
	rb1.Write(buf) // we omit error handling
	newrb, _ := Read(buf)
	if rb1.Equals(newrb) {
		fmt.Println("I wrote the content to a byte stream and read it back.")
	} else {
		t.Error("Bad read")
	}
}

func TestStats(t *testing.T) {

	rb := New()
	rb.Add(1, 2, 3, 4, 6, 7)
	rb.Add(999991, 999992, 999993, 999994, 999996, 999997)

	stats := rb.Stats()
	if stats["cardinality"] != rb.Cardinality() {
		t.Errorf("cardinality: expected %d got %d\n", rb.Cardinality(), stats["cardinality"])
	}

	if stats["n_containers"] != 2 {
		t.Errorf("n_containers: expected %d got %d\n", 2, stats["n_containers"], stats)
	}
	if stats["n_array_containers"] != 2 {
		t.Errorf("n_array_containers: expected %d got %d\n", 2, stats["n_array_containers"], stats)
	}
	for _, c := range []string{"n_run_containers", "n_bitmap_containers"} {
		if stats[c] != 0 {
			t.Errorf("%s: expected 0 got %d\n", 2, c, stats[c])
		}
	}
}
<|endoftext|>


<commit_before>package cli

import (
	"fmt"
	"github.com/qiniu/api/auth/digest"
	fio "github.com/qiniu/api/io"
	rio "github.com/qiniu/api/resumable/io"
	"github.com/qiniu/api/rs"
	"github.com/qiniu/log"
	"os"
	"sync/atomic"
)

func FormPut(cmd string, params ...string) {
	if len(params) == 3 || len(params) == 4 {
		bucket := params[0]
		key := params[1]
		localFile := params[2]
		mimeType := ""
		if len(params) == 4 {
			mimeType = params[3]
		}
		accountS.Get()
		mac := digest.Mac{accountS.AccessKey, []byte(accountS.SecretKey)}
		policy := rs.PutPolicy{}
		policy.Scope = bucket
		putExtra := fio.PutExtra{}
		if mimeType != "" {
			putExtra.MimeType = mimeType
		}
		uptoken := policy.Token(&mac)
		putRet := fio.PutRet{}
		err := fio.PutFile(nil, &putRet, uptoken, key, localFile, &putExtra)
		if err != nil {
			log.Error("Put file error", err)
		} else {
			fmt.Println("Put file", localFile, "=>", bucket, ":", putRet.Key, "(", putRet.Hash, ")", "success!")
		}
	} else {
		Help(cmd)
	}
}

func ResumablePut(cmd string, params ...string) {
	if len(params) == 3 || len(params) == 4 {
		bucket := params[0]
		key := params[1]
		localFile := params[2]
		mimeType := ""
		if len(params) == 4 {
			mimeType = params[3]
		}
		fi, err := os.Stat(localFile)
		if err != nil {
			log.Error("Stat local file error,", err)
			return
		}
		fileSize := fi.Size()
		accountS.Get()
		mac := digest.Mac{accountS.AccessKey, []byte(accountS.SecretKey)}
		policy := rs.PutPolicy{}
		policy.Scope = bucket
		putExtra := rio.PutExtra{}
		if mimeType != "" {
			putExtra.MimeType = mimeType
		}
		progressHandler := ProgressHandler{
			fileSize,
			0,
		}
		putExtra.Notify = progressHandler.Notify
		uptoken := policy.Token(&mac)
		putRet := rio.PutRet{}
		err = rio.PutFile(nil, &putRet, uptoken, key, localFile, &putExtra)
		if err != nil {
			log.Error("\r\nPut file error", err)
		} else {
			fmt.Println("\r\nPut file", localFile, "=>", bucket, ":", putRet.Key, "(", putRet.Hash, ")", "success!")
		}
	} else {
		Help(cmd)
	}
}

type ProgressHandler struct {
	FileSize int64
	Offset   int64
}

func (this *ProgressHandler) Percent() float32 {
	return float32(this.Offset) / float32(this.FileSize) * 100
}

func (this *ProgressHandler) Notify(blkIdx int, blkSize int, ret *rio.BlkputRet) {
	offset := ret.Offset
	atomic.AddInt64(&this.Offset, int64(offset))
	percent := this.Percent()
	output := fmt.Sprintf("Uploading %.2f%%", percent)
	for i := 0; i < len(output); i++ {
		fmt.Print("\b")
	}
	fmt.Print(output)
}
<commit_msg>Fix the resumable upload progress error.<commit_after>package cli

import (
	"fmt"
	"github.com/qiniu/api/auth/digest"
	fio "github.com/qiniu/api/io"
	rio "github.com/qiniu/api/resumable/io"
	"github.com/qiniu/api/rs"
	"github.com/qiniu/log"
	"os"
	"sync"
)

func FormPut(cmd string, params ...string) {
	if len(params) == 3 || len(params) == 4 {
		bucket := params[0]
		key := params[1]
		localFile := params[2]
		mimeType := ""
		if len(params) == 4 {
			mimeType = params[3]
		}
		accountS.Get()
		mac := digest.Mac{accountS.AccessKey, []byte(accountS.SecretKey)}
		policy := rs.PutPolicy{}
		policy.Scope = bucket
		putExtra := fio.PutExtra{}
		if mimeType != "" {
			putExtra.MimeType = mimeType
		}
		uptoken := policy.Token(&mac)
		putRet := fio.PutRet{}
		err := fio.PutFile(nil, &putRet, uptoken, key, localFile, &putExtra)
		if err != nil {
			log.Error("Put file error", err)
		} else {
			fmt.Println("Put file", localFile, "=>", bucket, ":", putRet.Key, "(", putRet.Hash, ")", "success!")
		}
	} else {
		Help(cmd)
	}
}

func ResumablePut(cmd string, params ...string) {
	if len(params) == 3 || len(params) == 4 {
		bucket := params[0]
		key := params[1]
		localFile := params[2]
		mimeType := ""
		if len(params) == 4 {
			mimeType = params[3]
		}
		fi, err := os.Stat(localFile)
		if err != nil {
			log.Error("Stat local file error,", err)
			return
		}
		fileSize := fi.Size()
		accountS.Get()
		mac := digest.Mac{accountS.AccessKey, []byte(accountS.SecretKey)}
		policy := rs.PutPolicy{}
		policy.Scope = bucket
		putExtra := rio.PutExtra{}
		if mimeType != "" {
			putExtra.MimeType = mimeType
		}
		progressHandler := ProgressHandler{
			FileSize: fileSize,
			Offset:   0,
		}
		putExtra.Notify = progressHandler.Notify
		uptoken := policy.Token(&mac)
		putRet := rio.PutRet{}
		err = rio.PutFile(nil, &putRet, uptoken, key, localFile, &putExtra)
		if err != nil {
			log.Error("\r\nPut file error", err)
		} else {
			fmt.Println("\r\nPut file", localFile, "=>", bucket, ":", putRet.Key, "(", putRet.Hash, ")", "success!")
		}
	} else {
		Help(cmd)
	}
}

type ProgressHandler struct {
	FileSize int64
	Offset   int64
	RWMutex  sync.Mutex
}

func (this *ProgressHandler) Percent() float32 {
	return float32(this.Offset) / float32(this.FileSize) * 100
}

func (this *ProgressHandler) Notify(blkIdx int, blkSize int, ret *rio.BlkputRet) {
	this.RWMutex.Lock()
	defer this.RWMutex.Unlock()
	this.Offset = int64(ret.Offset)
	percent := this.Percent()
	output := fmt.Sprintf("Uploading %.2f%%", percent)
	for i := 0; i < len(output); i++ {
		fmt.Print("\b")
	}
	fmt.Print(output)
}
<|endoftext|>


<commit_before>package main

/*
 * Copyright 2014 Albert P. Tobey <atobey@datastax.com> @AlTobey
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * pcstat.go - page cache stat
 *
 * uses the mincore(2) syscall to find out which pages (almost always 4k)
 * of a file are currently cached in memory
 *
 */

import (
	"encoding/json"
	"flag"
	"fmt"
	"log"
	"os"
	"path"
	"strings"
	"syscall"
	"time"
	"unsafe"
)

// pcStat: page cache status
// Bytes: size of the file (from os.File.Stat())
// Pages: array of booleans: true if cached, false otherwise
type pcStat struct {
	Name      string    `json:"filename"`  // file name as specified on command line
	Size      int64     `json:"size"`      // file size in bytes
	Timestamp time.Time `json:"timestamp"` // time right before calling mincore
	Mtime     time.Time `json:"mtime"`     // last modification time of the file
	Pages     int       `json:"pages"`     // total memory pages
	Cached    int       `json:"cached"`    // number of pages that are cached
	Uncached  int       `json:"uncached"`  // number of pages that are not cached
	Percent   float64   `json:"percent"`   // percentage of pages cached
	PPStat    []bool    `json:"status"`    // per-page status, true if cached, false otherwise
}

type pcStatList []pcStat

var (
	terseFlag, nohdrFlag, jsonFlag, ppsFlag, bnameFlag bool
)

func init() {
	// TODO: error on useless/broken combinations
	flag.BoolVar(&terseFlag, "terse", false, "show terse output")
	flag.BoolVar(&nohdrFlag, "nohdr", false, "omit the header from terse & text output")
	flag.BoolVar(&jsonFlag, "json", false, "return data in JSON format")
	flag.BoolVar(&ppsFlag, "pps", false, "include the per-page status in JSON output")
	flag.BoolVar(&bnameFlag, "bname", false, "convert paths to basename to narrow the output")
}

func main() {
	flag.Parse()

	// all non-flag arguments are considered to be filenames
	// this works well with shell globbing
	// file order is preserved throughout this program
	stats := make(pcStatList, len(flag.Args()))

	for i, fname := range flag.Args() {
		stats[i] = getMincore(fname)
	}

	if jsonFlag {
		stats.formatJson()
	} else if terseFlag {
		stats.formatTerse()
	} else {
		stats.formatText()
	}
}

func (stats pcStatList) formatText() {
	// find the longest filename in the data for calculating whitespace padding
	maxName := 8
	for _, pcs := range stats {
		if len(pcs.Name) > maxName {
			maxName = len(pcs.Name)
		}
	}

	// create horizontal grid line
	pad := strings.Repeat("-", maxName+2)
	hr := fmt.Sprintf("|%s+----------------+------------+-----------+---------|", pad)

	fmt.Println(hr)

	// -nohdr may be chosen to save 2 lines of precious vertical space
	if !nohdrFlag {
		pad = strings.Repeat(" ", maxName-4)
		fmt.Printf("| Name%s | Size           | Pages      | Cached    | Percent |\n", pad)
		fmt.Println(hr)
	}

	for _, pcs := range stats {
		pad = strings.Repeat(" ", maxName-len(pcs.Name))

		// %07.3f was chosen to make it easy to scan the percentages vertically
		// I tried a few different formats only this one kept the decimals aligned
		fmt.Printf("| %s%s | %-15d| %-11d| %-10d| %07.3f |\n",
			pcs.Name, pad, pcs.Size, pcs.Pages, pcs.Cached, pcs.Percent)
	}

	fmt.Println(hr)
}

func (stats pcStatList) formatTerse() {
	if !nohdrFlag {
		fmt.Println("name,size,timestamp,mtime,pages,cached,percent")
	}
	for _, pcs := range stats {
		time := pcs.Timestamp.Unix()
		mtime := pcs.Mtime.Unix()
		fmt.Printf("%s,%d,%d,%d,%d,%d,%g\n",
			pcs.Name, pcs.Size, time, mtime, pcs.Pages, pcs.Cached, pcs.Percent)
	}
}

func (stats pcStatList) formatJson() {
	b, err := json.Marshal(stats)
	if err != nil {
		log.Fatalf("JSON formatting failed: %s\n", err)
	}
	os.Stdout.Write(b)
	fmt.Println("")
}

func getMincore(fname string) pcStat {
	f, err := os.Open(fname)
	if err != nil {
		log.Fatalf("Could not open file '%s' for read: %s\n", fname, err)
	}
	defer f.Close()

	fi, err := f.Stat()
	if err != nil {
		log.Fatalf("Could not stat file %s: %s\n", fname, err)
	}
	if fi.Size() == 0 {
		log.Fatalf("%s appears to be 0 bytes in length\n", fname)
	}

	// []byte slice
	mmap, err := syscall.Mmap(int(f.Fd()), 0, int(fi.Size()), syscall.PROT_NONE, syscall.MAP_SHARED)
	if err != nil {
		log.Fatalf("Could not mmap file '%s': %s\n", fname, err)
	}
	// TODO: check for MAP_FAILED which is ((void *) -1)
	// but maybe unnecessary since it looks like errno is always set when MAP_FAILED

	// one byte per page, only LSB is used, remainder is reserved and clear
	vecsz := (fi.Size() + int64(os.Getpagesize()) - 1) / int64(os.Getpagesize())
	vec := make([]byte, vecsz)

	// get all of the arguments to the mincore syscall converted to uintptr
	mmap_ptr := uintptr(unsafe.Pointer(&mmap[0]))
	size_ptr := uintptr(fi.Size())
	vec_ptr := uintptr(unsafe.Pointer(&vec[0]))

	// get the timestamp right before the syscall
	ts := time.Now()

	// use Go's ASM to submit directly to the kernel, no C wrapper needed
	// mincore(2): int mincore(void *addr, size_t length, unsigned char *vec);
	// 0 on success, takes the pointer to the mmap, a size, which is the
	// size that came from f.Stat(), and the vector, which is a pointer
	// to the memory behind an []byte
	// this writes a snapshot of the data into vec which a list of 8-bit flags
	// with the LSB set if the page in that position is currently in VFS cache
	ret, _, err := syscall.RawSyscall(syscall.SYS_MINCORE, mmap_ptr, size_ptr, vec_ptr)
	if ret != 0 {
		log.Fatalf("syscall SYS_MINCORE failed: %s", err)
	}
	defer syscall.Munmap(mmap)

	pcs := pcStat{fname, fi.Size(), ts, fi.ModTime(), int(vecsz), 0, 0, 0.0, []bool{}}

	// only export the per-page cache mapping if it's explicitly enabled
	// an empty "status": [] field, but NBD.
	if ppsFlag {
		pcs.PPStat = make([]bool, vecsz)

		// there is no bitshift only bool
		for i, b := range vec {
			if b%2 == 1 {
				pcs.PPStat[i] = true
			} else {
				pcs.PPStat[i] = false
			}
		}

	}

	// convert long paths to their basename with the -bname flag
	// this overwrites the original filename in pcs but it doesn't matter since
	// it's not used to access the file again -- and should not be!
	if bnameFlag {
		pcs.Name = path.Base(fname)
	}

	for _, b := range vec {
		if b%2 == 1 {
			pcs.Cached++
		} else {
			pcs.Uncached++
		}
	}

	// convert to float for the occasional sparsely-cached file
	// see the README.md for how to produce one
	pcs.Percent = (float64(pcs.Cached) / float64(pcs.Pages)) * 100.00

	return pcs
}
<commit_msg>Add a test TODO I thought of but don't have time to verify at the moment.<commit_after>package main

/*
 * Copyright 2014 Albert P. Tobey <atobey@datastax.com> @AlTobey
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 * pcstat.go - page cache stat
 *
 * uses the mincore(2) syscall to find out which pages (almost always 4k)
 * of a file are currently cached in memory
 *
 */

import (
	"encoding/json"
	"flag"
	"fmt"
	"log"
	"os"
	"path"
	"strings"
	"syscall"
	"time"
	"unsafe"
)

// pcStat: page cache status
// Bytes: size of the file (from os.File.Stat())
// Pages: array of booleans: true if cached, false otherwise
type pcStat struct {
	Name      string    `json:"filename"`  // file name as specified on command line
	Size      int64     `json:"size"`      // file size in bytes
	Timestamp time.Time `json:"timestamp"` // time right before calling mincore
	Mtime     time.Time `json:"mtime"`     // last modification time of the file
	Pages     int       `json:"pages"`     // total memory pages
	Cached    int       `json:"cached"`    // number of pages that are cached
	Uncached  int       `json:"uncached"`  // number of pages that are not cached
	Percent   float64   `json:"percent"`   // percentage of pages cached
	PPStat    []bool    `json:"status"`    // per-page status, true if cached, false otherwise
}

type pcStatList []pcStat

var (
	terseFlag, nohdrFlag, jsonFlag, ppsFlag, bnameFlag bool
)

func init() {
	// TODO: error on useless/broken combinations
	flag.BoolVar(&terseFlag, "terse", false, "show terse output")
	flag.BoolVar(&nohdrFlag, "nohdr", false, "omit the header from terse & text output")
	flag.BoolVar(&jsonFlag, "json", false, "return data in JSON format")
	flag.BoolVar(&ppsFlag, "pps", false, "include the per-page status in JSON output")
	flag.BoolVar(&bnameFlag, "bname", false, "convert paths to basename to narrow the output")
}

func main() {
	flag.Parse()

	// all non-flag arguments are considered to be filenames
	// this works well with shell globbing
	// file order is preserved throughout this program
	stats := make(pcStatList, len(flag.Args()))

	for i, fname := range flag.Args() {
		stats[i] = getMincore(fname)
	}

	if jsonFlag {
		stats.formatJson()
	} else if terseFlag {
		stats.formatTerse()
	} else {
		stats.formatText()
	}
}

func (stats pcStatList) formatText() {
	// find the longest filename in the data for calculating whitespace padding
	maxName := 8
	for _, pcs := range stats {
		if len(pcs.Name) > maxName {
			maxName = len(pcs.Name)
		}
	}

	// create horizontal grid line
	pad := strings.Repeat("-", maxName+2)
	hr := fmt.Sprintf("|%s+----------------+------------+-----------+---------|", pad)

	fmt.Println(hr)

	// -nohdr may be chosen to save 2 lines of precious vertical space
	if !nohdrFlag {
		pad = strings.Repeat(" ", maxName-4)
		fmt.Printf("| Name%s | Size           | Pages      | Cached    | Percent |\n", pad)
		fmt.Println(hr)
	}

	for _, pcs := range stats {
		pad = strings.Repeat(" ", maxName-len(pcs.Name))

		// %07.3f was chosen to make it easy to scan the percentages vertically
		// I tried a few different formats only this one kept the decimals aligned
		fmt.Printf("| %s%s | %-15d| %-11d| %-10d| %07.3f |\n",
			pcs.Name, pad, pcs.Size, pcs.Pages, pcs.Cached, pcs.Percent)
	}

	fmt.Println(hr)
}

func (stats pcStatList) formatTerse() {
	if !nohdrFlag {
		fmt.Println("name,size,timestamp,mtime,pages,cached,percent")
	}
	for _, pcs := range stats {
		time := pcs.Timestamp.Unix()
		mtime := pcs.Mtime.Unix()
		fmt.Printf("%s,%d,%d,%d,%d,%d,%g\n",
			pcs.Name, pcs.Size, time, mtime, pcs.Pages, pcs.Cached, pcs.Percent)
	}
}

func (stats pcStatList) formatJson() {
	b, err := json.Marshal(stats)
	if err != nil {
		log.Fatalf("JSON formatting failed: %s\n", err)
	}
	os.Stdout.Write(b)
	fmt.Println("")
}

func getMincore(fname string) pcStat {
	f, err := os.Open(fname)
	if err != nil {
		log.Fatalf("Could not open file '%s' for read: %s\n", fname, err)
	}
	defer f.Close()

	// TEST TODO: verify behavior when the file size is changing quickly
	// while this function is running. I assume that the size parameter to
	// mincore will prevent overruns of the output vector, but it's not clear
	// what will be in there when the file is truncated between here and the
	// mincore() call.
	fi, err := f.Stat()
	if err != nil {
		log.Fatalf("Could not stat file %s: %s\n", fname, err)
	}
	if fi.Size() == 0 {
		log.Fatalf("%s appears to be 0 bytes in length\n", fname)
	}

	// []byte slice
	mmap, err := syscall.Mmap(int(f.Fd()), 0, int(fi.Size()), syscall.PROT_NONE, syscall.MAP_SHARED)
	if err != nil {
		log.Fatalf("Could not mmap file '%s': %s\n", fname, err)
	}
	// TODO: check for MAP_FAILED which is ((void *) -1)
	// but maybe unnecessary since it looks like errno is always set when MAP_FAILED

	// one byte per page, only LSB is used, remainder is reserved and clear
	vecsz := (fi.Size() + int64(os.Getpagesize()) - 1) / int64(os.Getpagesize())
	vec := make([]byte, vecsz)

	// get all of the arguments to the mincore syscall converted to uintptr
	mmap_ptr := uintptr(unsafe.Pointer(&mmap[0]))
	size_ptr := uintptr(fi.Size())
	vec_ptr := uintptr(unsafe.Pointer(&vec[0]))

	// get the timestamp right before the syscall
	ts := time.Now()

	// use Go's ASM to submit directly to the kernel, no C wrapper needed
	// mincore(2): int mincore(void *addr, size_t length, unsigned char *vec);
	// 0 on success, takes the pointer to the mmap, a size, which is the
	// size that came from f.Stat(), and the vector, which is a pointer
	// to the memory behind an []byte
	// this writes a snapshot of the data into vec which a list of 8-bit flags
	// with the LSB set if the page in that position is currently in VFS cache
	ret, _, err := syscall.RawSyscall(syscall.SYS_MINCORE, mmap_ptr, size_ptr, vec_ptr)
	if ret != 0 {
		log.Fatalf("syscall SYS_MINCORE failed: %s", err)
	}
	defer syscall.Munmap(mmap)

	pcs := pcStat{fname, fi.Size(), ts, fi.ModTime(), int(vecsz), 0, 0, 0.0, []bool{}}

	// only export the per-page cache mapping if it's explicitly enabled
	// an empty "status": [] field, but NBD.
	if ppsFlag {
		pcs.PPStat = make([]bool, vecsz)

		// there is no bitshift only bool
		for i, b := range vec {
			if b%2 == 1 {
				pcs.PPStat[i] = true
			} else {
				pcs.PPStat[i] = false
			}
		}
	}

	// convert long paths to their basename with the -bname flag
	// this overwrites the original filename in pcs but it doesn't matter since
	// it's not used to access the file again -- and should not be!
	if bnameFlag {
		pcs.Name = path.Base(fname)
	}

	for _, b := range vec {
		if b%2 == 1 {
			pcs.Cached++
		} else {
			pcs.Uncached++
		}
	}

	// convert to float for the occasional sparsely-cached file
	// see the README.md for how to produce one
	pcs.Percent = (float64(pcs.Cached) / float64(pcs.Pages)) * 100.00

	return pcs
}
<|endoftext|>


<commit_before>package goportscan

import(
	"net"
	"reflect"
	"testing"
)

func setupLocalListener(t *testing.T, port int) net.Listener {
	listener, err := net.Listen("tcp4", assembleEndpoint("localhost", port))
	if err != nil {
		t.Fatal(err)
	}
	return listener
}

func TestNewPortScanner(t *testing.T){
	expected := PortScanner{"localhost"}
	actual := *NewPortScanner("localhost")

	if actual != expected {
		t.Errorf("Got: %q; Expected: %q", actual, expected)
	}

	actual = *NewPortScanner("")

	if actual != expected {
		t.Errorf("Got: %q; Expected: %q", actual, expected)
	}
}

func TestScanPorts(t *testing.T){
	ps := NewPortScanner("localhost")

	expected := make(map[int]string)
	actual := ps.ScanPorts([]int{})

	if !reflect.DeepEqual(actual, expected){
		t.Errorf("Got: %q; Expected: %q", actual, expected)
	}

	listener_known := setupLocalListener(t, KNOWN_TEST_PORT)
	defer listener_known.Close()

	expected = map[int]string {
  	    KNOWN_TEST_PORT: "test port",
	}
	actual = ps.ScanPorts([]int{KNOWN_TEST_PORT})

	if !reflect.DeepEqual(actual, expected){
		t.Errorf("Got: %q; Expected: %q", actual, expected)
	}

	listener_unknown := setupLocalListener(t, UNKNOWN_TEST_PORT)
	defer listener_unknown.Close()

	expected = map[int]string {
  	    UNKNOWN_TEST_PORT: "unknown",
	}
	actual = ps.ScanPorts([]int{UNKNOWN_TEST_PORT})

	if !reflect.DeepEqual(actual, expected){
		t.Errorf("Got: %q; Expected: %q", actual, expected)
	}

}

func TestIsOpen(t *testing.T){
	listener := setupLocalListener(t, KNOWN_TEST_PORT)
	defer listener.Close()

	expected := true
	actual := isOpen("localhost", KNOWN_TEST_PORT)

	if actual != expected{
		t.Errorf("Got: %q; Expected: %q", actual, expected)
	}

	expected = false
	// port 0 is wildcard for a random free port
	actual = isOpen("localhost", 0)

	if actual != expected{
		t.Errorf("Got: %q; Expected: %q", actual, expected)
	}
}

func TestAssembleEndpoint(t * testing.T){
	expected := "localhost:80"
	actual := assembleEndpoint("localhost", 80)

	if actual != expected {
		t.Errorf("Got: %q; Expected: %q", actual, expected)
	}
}
<commit_msg>more tests<commit_after>package goportscan

import(
	"net"
	"reflect"
	"testing"
)

func setupLocalListener(t *testing.T, port int) net.Listener {
	listener, err := net.Listen("tcp4", assembleEndpoint("localhost", port))
	if err != nil {
		t.Fatal(err)
	}
	return listener
}

func TestNewPortScanner(t *testing.T){
	expected := PortScanner{"localhost"}
	actual := *NewPortScanner("localhost")

	if actual != expected {
		t.Errorf("Got: %q; Expected: %q", actual, expected)
	}

	actual = *NewPortScanner("")

	if actual != expected {
		t.Errorf("Got: %q; Expected: %q", actual, expected)
	}
}

func TestScanPorts(t *testing.T){
	ps := NewPortScanner("localhost")

	expected := make(map[int]string)
	actual := ps.ScanPorts([]int{})

	if !reflect.DeepEqual(actual, expected){
		t.Errorf("Got: %q; Expected: %q", actual, expected)
	}

	listener_known := setupLocalListener(t, KNOWN_TEST_PORT)
	defer listener_known.Close()

	expected = map[int]string {
  	    KNOWN_TEST_PORT: "test port",
	}
	actual = ps.ScanPorts([]int{KNOWN_TEST_PORT})

	if !reflect.DeepEqual(actual, expected){
		t.Errorf("Got: %q; Expected: %q", actual, expected)
	}

	listener_unknown := setupLocalListener(t, UNKNOWN_TEST_PORT)
	defer listener_unknown.Close()

	expected = map[int]string {
  	    UNKNOWN_TEST_PORT: "unknown",
	}
	actual = ps.ScanPorts([]int{UNKNOWN_TEST_PORT})

	if !reflect.DeepEqual(actual, expected){
		t.Errorf("Got: %q; Expected: %q", actual, expected)
	}

}

func TestScanKnownPorts(t *testing.T){
	listener_known := setupLocalListener(t, KNOWN_TEST_PORT)
	defer listener_known.Close()

	ps := NewPortScanner("localhost")

	expected := map[int]string {
  	    KNOWN_TEST_PORT: "test port",
	}
	actual := ps.ScanKnownPorts()

	// only compare our test port, cause there could be other open ports on localhost
	if actual[KNOWN_TEST_PORT] != expected[KNOWN_TEST_PORT]{
		t.Errorf("Got: %q; Expected: %q", actual, expected)
	}
}

func TestScanPortRange(t *testing.T){
	listener_known := setupLocalListener(t, KNOWN_TEST_PORT)
	defer listener_known.Close()

	listener_unknown := setupLocalListener(t, UNKNOWN_TEST_PORT)
	defer listener_unknown.Close()

	ps := NewPortScanner("localhost")

	expected := map[int]string {
  	    KNOWN_TEST_PORT: "test port",
		UNKNOWN_TEST_PORT: "unknown",
	}
	actual := ps.ScanPortRange(KNOWN_TEST_PORT, UNKNOWN_TEST_PORT)

	if !reflect.DeepEqual(actual, expected){
		t.Errorf("Got: %q; Expected: %q", actual, expected)
	}

	defer func() {
		if err := recover(); err != "start_port has to be smaller than end_port" {
		    t.Errorf("Got: %q; Expected: %q", err, "start_port has to be smaller than end_port")
		}

	}()

	ps.ScanPortRange(1, 0)
}

func TestIsOpen(t *testing.T){
	listener := setupLocalListener(t, KNOWN_TEST_PORT)
	defer listener.Close()

	expected := true
	actual := isOpen("localhost", KNOWN_TEST_PORT)

	if actual != expected{
		t.Errorf("Got: %q; Expected: %q", actual, expected)
	}

	expected = false
	// port 0 is wildcard for a random free port
	actual = isOpen("localhost", 0)

	if actual != expected{
		t.Errorf("Got: %q; Expected: %q", actual, expected)
	}
}

func TestAssembleEndpoint(t * testing.T){
	expected := "localhost:80"
	actual := assembleEndpoint("localhost", 80)

	if actual != expected {
		t.Errorf("Got: %q; Expected: %q", actual, expected)
	}
}
<|endoftext|>


<commit_before>package gqt_test

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"strings"

	"github.com/cloudfoundry-incubator/garden"
	"github.com/cloudfoundry-incubator/guardian/gqt/runner"

	. "github.com/cloudfoundry-incubator/guardian/matchers"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/ginkgo/extensions/table"
	. "github.com/onsi/gomega"
	"github.com/onsi/gomega/gbytes"
	"github.com/onsi/gomega/gexec"
)

var _ = Describe("Creating a Container", func() {
	var (
		client    *runner.RunningGarden
		container garden.Container
	)

	BeforeEach(func() {
		client = startGarden()
	})

	AfterEach(func() {
		Expect(client.DestroyAndStop()).To(Succeed())
	})

	Context("after creating a container without a specified handle", func() {
		var (
			privileged bool

			initProcPid int
		)

		JustBeforeEach(func() {
			var err error
			container, err = client.Create(garden.ContainerSpec{
				Privileged: privileged,
			})
			Expect(err).NotTo(HaveOccurred())

			initProcPid = initProcessPID(container.Handle())
		})

		It("should create a depot subdirectory based on the container handle", func() {
			Expect(container.Handle()).NotTo(BeEmpty())
			Expect(filepath.Join(client.DepotDir, container.Handle())).To(BeADirectory())
			Expect(filepath.Join(client.DepotDir, container.Handle(), "config.json")).To(BeARegularFile())
		})

		It("should lookup the right container", func() {
			lookupContainer, lookupError := client.Lookup(container.Handle())

			Expect(lookupError).NotTo(HaveOccurred())
			Expect(lookupContainer).To(Equal(container))
		})

		DescribeTable("placing the container in to all namespaces", func(ns string) {
			hostNS, err := gexec.Start(exec.Command("ls", "-l", fmt.Sprintf("/proc/1/ns/%s", ns)), GinkgoWriter, GinkgoWriter)
			Expect(err).NotTo(HaveOccurred())

			containerNS, err := gexec.Start(exec.Command("ls", "-l", fmt.Sprintf("/proc/%d/ns/%s", initProcPid, ns)), GinkgoWriter, GinkgoWriter)
			Expect(err).NotTo(HaveOccurred())

			Eventually(containerNS).Should(gexec.Exit(0))
			Eventually(hostNS).Should(gexec.Exit(0))

			hostFD := strings.Split(string(hostNS.Out.Contents()), ">")[1]
			containerFD := strings.Split(string(containerNS.Out.Contents()), ">")[1]

			Expect(hostFD).NotTo(Equal(containerFD))
		},
			Entry("should place the container in to the NET namespace", "net"),
			Entry("should place the container in to the IPC namespace", "ipc"),
			Entry("should place the container in to the UTS namespace", "uts"),
			Entry("should place the container in to the PID namespace", "pid"),
			Entry("should place the container in to the MNT namespace", "mnt"),
			Entry("should place the container in to the USER namespace", "user"),
		)

		Context("which is privileged", func() {
			BeforeEach(func() {
				privileged = true
			})

			It("should not place the container in its own user namespace", func() {
				hostNS, err := gexec.Start(exec.Command("ls", "-l", "/proc/1/ns/user"), GinkgoWriter, GinkgoWriter)
				Expect(err).NotTo(HaveOccurred())

				containerNS, err := gexec.Start(exec.Command("ls", "-l", fmt.Sprintf("/proc/%d/ns/user", initProcPid)), GinkgoWriter, GinkgoWriter)
				Expect(err).NotTo(HaveOccurred())

				Eventually(containerNS).Should(gexec.Exit(0))
				Eventually(hostNS).Should(gexec.Exit(0))

				hostFD := strings.Split(string(hostNS.Out.Contents()), ">")[1]
				containerFD := strings.Split(string(containerNS.Out.Contents()), ">")[1]

				Expect(hostFD).To(Equal(containerFD))
			})
		})
	})

	Context("after creating a container with a specified root filesystem", func() {
		var rootFSPath string

		BeforeEach(func() {
			var err error

			rootFSPath, err = ioutil.TempDir("", "test-rootfs")
			Expect(err).NotTo(HaveOccurred())
			command := fmt.Sprintf("cp -rf %s/* %s", os.Getenv("GARDEN_TEST_ROOTFS"), rootFSPath)
			Expect(exec.Command("sh", "-c", command).Run()).To(Succeed())
			Expect(ioutil.WriteFile(filepath.Join(rootFSPath, "my-file"), []byte("some-content"), 0644)).To(Succeed())

			container, err = client.Create(garden.ContainerSpec{
				RootFSPath: rootFSPath,
			})
			Expect(err).NotTo(HaveOccurred())
		})

		It("provides the containers with the right rootfs", func() {
			Expect(container).To(HaveFile("/my-file"))
		})

		It("isolates the filesystem properly for multiple containers", func() {
			runCommand(container, "touch", []string{"/created-file"})
			Expect(container).To(HaveFile("/created-file"))

			container2, err := client.Create(garden.ContainerSpec{
				RootFSPath: rootFSPath,
			})
			Expect(err).NotTo(HaveOccurred())

			Expect(container2).To(HaveFile("/my-file"))
			Expect(container2).NotTo(HaveFile("/created-file"))
		})
	})

	Context("after creating a container with a specified handle", func() {
		It("should lookup the right container for the handle", func() {
			container, err := client.Create(garden.ContainerSpec{
				Handle: "container-banana",
			})
			Expect(err).NotTo(HaveOccurred())

			lookupContainer, lookupError := client.Lookup("container-banana")
			Expect(lookupError).NotTo(HaveOccurred())
			Expect(lookupContainer).To(Equal(container))
		})

		It("allow the container to be created with the same name after destroying", func() {
			container, err := client.Create(garden.ContainerSpec{
				Handle: "another-banana",
			})
			Expect(err).NotTo(HaveOccurred())

			Expect(client.Destroy(container.Handle())).To(Succeed())

			container, err = client.Create(garden.ContainerSpec{
				Handle: "another-banana",
			})
			Expect(err).NotTo(HaveOccurred())
		})
	})

	Context("when creating a container fails", func() {
		It("should not leak networking configuration", func() {
			_, err := client.Create(garden.ContainerSpec{
				Network:    fmt.Sprintf("172.250.%d.20/24", GinkgoParallelNode()),
				RootFSPath: "/banana/does/not/exist",
			})
			Expect(err).To(HaveOccurred())

			session, err := gexec.Start(
				exec.Command("ifconfig"),
				GinkgoWriter, GinkgoWriter,
			)
			Expect(err).NotTo(HaveOccurred())
			Consistently(session).ShouldNot(gbytes.Say(fmt.Sprintf("172-250-%d-0", GinkgoParallelNode())))
		})
	})
})

func initProcessPID(handle string) int {
	Eventually(fmt.Sprintf("/run/opencontainer/containers/%s/state.json", handle)).Should(BeAnExistingFile())
	stateFile, err := os.Open(fmt.Sprintf("/run/opencontainer/containers/%s/state.json", handle))
	Expect(err).NotTo(HaveOccurred())

	state := struct {
		Pid int `json:"init_process_pid"`
	}{}

	Eventually(func() error {
		// state.json is sometimes empty immediately after creation, so keep
		// trying until it's valid json
		return json.NewDecoder(stateFile).Decode(&state)
	}).Should(Succeed())

	return state.Pid
}

func runCommand(container garden.Container, path string, args []string) {
	proc, err := container.Run(
		garden.ProcessSpec{
			Path: path,
			Args: args,
		},
		ginkgoIO)
	Expect(err).NotTo(HaveOccurred())

	exitCode, err := proc.Wait()
	Expect(err).NotTo(HaveOccurred())
	Expect(exitCode).To(Equal(0))
}
<commit_msg>Make sure state file is reopened on each poll<commit_after>package gqt_test

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"strings"

	"github.com/cloudfoundry-incubator/garden"
	"github.com/cloudfoundry-incubator/guardian/gqt/runner"

	. "github.com/cloudfoundry-incubator/guardian/matchers"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/ginkgo/extensions/table"
	. "github.com/onsi/gomega"
	"github.com/onsi/gomega/gbytes"
	"github.com/onsi/gomega/gexec"
)

var _ = Describe("Creating a Container", func() {
	var (
		client    *runner.RunningGarden
		container garden.Container
	)

	BeforeEach(func() {
		client = startGarden()
	})

	AfterEach(func() {
		Expect(client.DestroyAndStop()).To(Succeed())
	})

	Context("after creating a container without a specified handle", func() {
		var (
			privileged bool

			initProcPid int
		)

		JustBeforeEach(func() {
			var err error
			container, err = client.Create(garden.ContainerSpec{
				Privileged: privileged,
			})
			Expect(err).NotTo(HaveOccurred())

			initProcPid = initProcessPID(container.Handle())
		})

		It("should create a depot subdirectory based on the container handle", func() {
			Expect(container.Handle()).NotTo(BeEmpty())
			Expect(filepath.Join(client.DepotDir, container.Handle())).To(BeADirectory())
			Expect(filepath.Join(client.DepotDir, container.Handle(), "config.json")).To(BeARegularFile())
		})

		It("should lookup the right container", func() {
			lookupContainer, lookupError := client.Lookup(container.Handle())

			Expect(lookupError).NotTo(HaveOccurred())
			Expect(lookupContainer).To(Equal(container))
		})

		DescribeTable("placing the container in to all namespaces", func(ns string) {
			hostNS, err := gexec.Start(exec.Command("ls", "-l", fmt.Sprintf("/proc/1/ns/%s", ns)), GinkgoWriter, GinkgoWriter)
			Expect(err).NotTo(HaveOccurred())

			containerNS, err := gexec.Start(exec.Command("ls", "-l", fmt.Sprintf("/proc/%d/ns/%s", initProcPid, ns)), GinkgoWriter, GinkgoWriter)
			Expect(err).NotTo(HaveOccurred())

			Eventually(containerNS).Should(gexec.Exit(0))
			Eventually(hostNS).Should(gexec.Exit(0))

			hostFD := strings.Split(string(hostNS.Out.Contents()), ">")[1]
			containerFD := strings.Split(string(containerNS.Out.Contents()), ">")[1]

			Expect(hostFD).NotTo(Equal(containerFD))
		},
			Entry("should place the container in to the NET namespace", "net"),
			Entry("should place the container in to the IPC namespace", "ipc"),
			Entry("should place the container in to the UTS namespace", "uts"),
			Entry("should place the container in to the PID namespace", "pid"),
			Entry("should place the container in to the MNT namespace", "mnt"),
			Entry("should place the container in to the USER namespace", "user"),
		)

		Context("which is privileged", func() {
			BeforeEach(func() {
				privileged = true
			})

			It("should not place the container in its own user namespace", func() {
				hostNS, err := gexec.Start(exec.Command("ls", "-l", "/proc/1/ns/user"), GinkgoWriter, GinkgoWriter)
				Expect(err).NotTo(HaveOccurred())

				containerNS, err := gexec.Start(exec.Command("ls", "-l", fmt.Sprintf("/proc/%d/ns/user", initProcPid)), GinkgoWriter, GinkgoWriter)
				Expect(err).NotTo(HaveOccurred())

				Eventually(containerNS).Should(gexec.Exit(0))
				Eventually(hostNS).Should(gexec.Exit(0))

				hostFD := strings.Split(string(hostNS.Out.Contents()), ">")[1]
				containerFD := strings.Split(string(containerNS.Out.Contents()), ">")[1]

				Expect(hostFD).To(Equal(containerFD))
			})
		})
	})

	Context("after creating a container with a specified root filesystem", func() {
		var rootFSPath string

		BeforeEach(func() {
			var err error

			rootFSPath, err = ioutil.TempDir("", "test-rootfs")
			Expect(err).NotTo(HaveOccurred())
			command := fmt.Sprintf("cp -rf %s/* %s", os.Getenv("GARDEN_TEST_ROOTFS"), rootFSPath)
			Expect(exec.Command("sh", "-c", command).Run()).To(Succeed())
			Expect(ioutil.WriteFile(filepath.Join(rootFSPath, "my-file"), []byte("some-content"), 0644)).To(Succeed())

			container, err = client.Create(garden.ContainerSpec{
				RootFSPath: rootFSPath,
			})
			Expect(err).NotTo(HaveOccurred())
		})

		It("provides the containers with the right rootfs", func() {
			Expect(container).To(HaveFile("/my-file"))
		})

		It("isolates the filesystem properly for multiple containers", func() {
			runCommand(container, "touch", []string{"/created-file"})
			Expect(container).To(HaveFile("/created-file"))

			container2, err := client.Create(garden.ContainerSpec{
				RootFSPath: rootFSPath,
			})
			Expect(err).NotTo(HaveOccurred())

			Expect(container2).To(HaveFile("/my-file"))
			Expect(container2).NotTo(HaveFile("/created-file"))
		})
	})

	Context("after creating a container with a specified handle", func() {
		It("should lookup the right container for the handle", func() {
			container, err := client.Create(garden.ContainerSpec{
				Handle: "container-banana",
			})
			Expect(err).NotTo(HaveOccurred())

			lookupContainer, lookupError := client.Lookup("container-banana")
			Expect(lookupError).NotTo(HaveOccurred())
			Expect(lookupContainer).To(Equal(container))
		})

		It("allow the container to be created with the same name after destroying", func() {
			container, err := client.Create(garden.ContainerSpec{
				Handle: "another-banana",
			})
			Expect(err).NotTo(HaveOccurred())

			Expect(client.Destroy(container.Handle())).To(Succeed())

			container, err = client.Create(garden.ContainerSpec{
				Handle: "another-banana",
			})
			Expect(err).NotTo(HaveOccurred())
		})
	})

	Context("when creating a container fails", func() {
		It("should not leak networking configuration", func() {
			_, err := client.Create(garden.ContainerSpec{
				Network:    fmt.Sprintf("172.250.%d.20/24", GinkgoParallelNode()),
				RootFSPath: "/banana/does/not/exist",
			})
			Expect(err).To(HaveOccurred())

			session, err := gexec.Start(
				exec.Command("ifconfig"),
				GinkgoWriter, GinkgoWriter,
			)
			Expect(err).NotTo(HaveOccurred())
			Consistently(session).ShouldNot(gbytes.Say(fmt.Sprintf("172-250-%d-0", GinkgoParallelNode())))
		})
	})
})

func initProcessPID(handle string) int {
	Eventually(fmt.Sprintf("/run/opencontainer/containers/%s/state.json", handle)).Should(BeAnExistingFile())

	state := struct {
		Pid int `json:"init_process_pid"`
	}{}

	Eventually(func() error {
		stateFile, err := os.Open(fmt.Sprintf("/run/opencontainer/containers/%s/state.json", handle))
		Expect(err).NotTo(HaveOccurred())
		defer stateFile.Close()

		// state.json is sometimes empty immediately after creation, so keep
		// trying until it's valid json
		return json.NewDecoder(stateFile).Decode(&state)
	}).Should(Succeed())

	return state.Pid
}

func runCommand(container garden.Container, path string, args []string) {
	proc, err := container.Run(
		garden.ProcessSpec{
			Path: path,
			Args: args,
		},
		ginkgoIO)
	Expect(err).NotTo(HaveOccurred())

	exitCode, err := proc.Wait()
	Expect(err).NotTo(HaveOccurred())
	Expect(exitCode).To(Equal(0))
}
<|endoftext|>


<commit_before>// Copyright 2021 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package main

import (
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"regexp"
	"strings"

	"golang.org/x/mod/semver"
	"gopkg.in/yaml.v2"
)

var (
	// pattern of release branches, e.g. apply-setters/v1.0
	releaseBranchPattern = regexp.MustCompile(`[-\w]*/(v\d*\.\d*)`)
	// pattern of release tags, e.g. functions/go/apply-setters/v1.0.1
	releaseTagPattern = regexp.MustCompile(`.*(go|ts)/[-\w]*/(v\d*\.\d*\.\d*)`)
	// pattern for version tags, e.g. unstable, v0.1.1, v0.1
	versionGroup = `unstable|v\d*\.\d*\.\d*|v\d*\.\d*`
)

func dirExists(path string) bool {
	if stat, err := os.Stat(path); err == nil && stat.IsDir() {
		return true
	}
	return false
}

type functionExample struct {
	ExamplePath string
	ExampleName string
}

type functionExamples []functionExample

// exampleNames returns a list of the functionExample names
func (fe functionExamples) exampleNames() []string {
	var exampleNames []string
	for _, example := range fe {
		exampleNames = append(exampleNames, example.ExampleName)
	}
	return exampleNames
}

type functionRelease struct {
	FunctionName       string
	MinorVersion       string
	Language           string
	LatestPatchVersion string
	FunctionPath       string
	Examples           functionExamples
	IsContrib          bool
}

// newFunctionRelease allocates and initializes a functionRelease
func newFunctionRelease(branch string) (*functionRelease, error) {
	fr := &functionRelease{}
	if !releaseBranchPattern.MatchString(branch) {
		return nil, fmt.Errorf("invalid branch format")
	}
	segments := strings.Split(branch, "/")
	// assume branch format: */<func_name>/<minor_version>
	fr.MinorVersion = segments[len(segments)-1]
	fr.FunctionName = segments[len(segments)-2]
	if err := fr.readLatestPatchVersion(); err != nil {
		return nil, err
	}
	if err := fr.readDocPaths(); err != nil {
		return nil, err
	}
	return fr, nil
}

// readLatestPatchVersion of the release from git tags
func (fr *functionRelease) readLatestPatchVersion() error {
	if fr.FunctionName == "" || fr.MinorVersion == "" {
		return fmt.Errorf("missing function name and/or minor version")
	}
	tags, err := gitTag()
	if err != nil {
		return err
	}
	funcPattern := fmt.Sprintf("%s/%s", fr.FunctionName, fr.MinorVersion)
	var lang, latestPatchVersion string
	for _, tag := range strings.Split(tags, "\n") {
		if !strings.Contains(tag, funcPattern) || !releaseTagPattern.MatchString(tag) {
			continue
		}
		segments := strings.Split(tag, "/")
		patchVersion := segments[len(segments)-1]
		if latestPatchVersion == "" ||
			semver.Compare(patchVersion, latestPatchVersion) == 1 {
			latestPatchVersion = patchVersion
			lang = segments[len(segments)-3]
		}
	}
	if latestPatchVersion == "" || lang == "" {
		return fmt.Errorf("could not find matching tag for release branch")
	}
	fr.Language = lang
	fr.LatestPatchVersion = latestPatchVersion
	return nil
}

// readDocPaths and set documentation paths
func (fr *functionRelease) readDocPaths() error {
	executablePath, err := os.Executable()
	if err != nil {
		return err
	}
	repoBase := filepath.Dir(filepath.Dir(filepath.Dir(executablePath)))
	pathsToTry := []struct {
		functionPath string
		examplesPath string
		isContrib    bool
	}{
		{
			functionPath: filepath.Join(repoBase, "functions", fr.Language, fr.FunctionName),
			examplesPath: filepath.Join(repoBase, "examples"),
			isContrib:    false,
		},
		{
			functionPath: filepath.Join(repoBase, "contrib", "functions", fr.Language, fr.FunctionName),
			examplesPath: filepath.Join(repoBase, "contrib", "examples"),
			isContrib:    true,
		},
	}
	var examplesPath string
	for _, pathToTry := range pathsToTry {
		if dirExists(pathToTry.functionPath) {
			fr.FunctionPath = pathToTry.functionPath
			fr.IsContrib = pathToTry.isContrib
			examplesPath = pathToTry.examplesPath
			break
		}
	}
	if fr.FunctionPath == "" {
		return fmt.Errorf("function doc paths not found from %+v", pathsToTry)
	}
	if err = fr.parseMetadata(examplesPath); err != nil {
		return err
	}
	return nil
}

// parseMetadata from metadata.yaml and set example paths
func (fr *functionRelease) parseMetadata(examplesPath string) error {
	type metadata struct {
		ExamplePackageUrls []string `yaml:"examplePackageURLs"`
	}
	if fr.FunctionPath == "" {
		return fmt.Errorf("expected FunctionPath in parseMetadata")
	}

	metadataPath := filepath.Join(fr.FunctionPath, "metadata.yaml")
	var md metadata
	yamlFile, err := ioutil.ReadFile(metadataPath)
	if err != nil {
		return err
	}

	err = yaml.Unmarshal(yamlFile, &md)
	if err != nil {
		return err
	}
	for _, exampleURL := range md.ExamplePackageUrls {
		segments := strings.Split(exampleURL, "/")
		exampleName := segments[len(segments)-1]
		examplePath := filepath.Join(examplesPath, exampleName)
		if !dirExists(examplePath) {
			return fmt.Errorf("example dir does not exist: %s", examplePath)
		}
		fr.Examples = append(fr.Examples, functionExample{
			ExamplePath: examplePath,
			ExampleName: exampleName,
		})
	}
	return nil
}

// updateDocs updates all the docs for the functionRelease on the filesystem
func (fr *functionRelease) updateDocs() error {
	if err := fr.updateFunctionDoc(); err != nil {
		return err
	}
	if err := fr.updateExampleDocs(); err != nil {
		return err
	}
	return nil
}

// updateFunctionDoc updates the function docs for the functionRelease
func (fr *functionRelease) updateFunctionDoc() error {
	functionReadme := filepath.Join(fr.FunctionPath, "README.md")
	return fr.updateDoc(functionReadme)
}

// updateExampleDocs updates the example docs for the functionRelease
func (fr *functionRelease) updateExampleDocs() error {
	for _, example := range fr.Examples {
		exampleReadme := filepath.Join(example.ExamplePath, "README.md")
		if err := fr.updateDoc(exampleReadme); err != nil {
			return err
		}
	}
	return nil
}

// Perform in place search/replace operations on a documentation file
func (fr *functionRelease) updateDoc(filePath string) error {
	contents, err := ioutil.ReadFile(filePath)
	if err != nil {
		return err
	}
	contents = fr.replaceTags(contents)
	contents = fr.replaceURLs(contents)
	contents = fr.replaceKptPackages(contents)
	if err = os.WriteFile(filePath, contents, 0644); err != nil {
		return err
	}
	return nil
}

// replace tags with patch e.g. apply-setters:v1.0.1, apply-setters/v1.0.1
func (fr *functionRelease) replaceTags(contents []byte) []byte {
	tagPattern := regexp.MustCompile(
		fmt.Sprintf(`(%s)(:|/)(%s)`, fr.FunctionName, versionGroup))
	contents = tagPattern.ReplaceAll(contents,
		[]byte(fmt.Sprintf(`${1}${2}%s`, fr.LatestPatchVersion)))
	return contents
}

// replace url with minor e.g. https://catalog.kpt.dev/apply-setters/v1.0
func (fr *functionRelease) replaceURLs(contents []byte) []byte {
	urlPattern := regexp.MustCompile(
		fmt.Sprintf(`(https://catalog\.kpt\.dev/%s/)(%s)`, fr.FunctionName, versionGroup))
	contents = urlPattern.ReplaceAll(contents,
		[]byte(fmt.Sprintf(`${1}%s`, fr.MinorVersion)))
	return contents
}

// replace kpt package names for all examples, e.g.
// https://github.com/GoogleContainerTools/kpt-functions-catalog.git/examples/apply-setters-simple ->
// https://github.com/GoogleContainerTools/kpt-functions-catalog.git/examples/apply-setters-simple@apply-setters/v1.0.1
func (fr *functionRelease) replaceKptPackages(contents []byte) []byte {
	exampleGroup := strings.Join(fr.Examples.exampleNames(), "|")
	exampleSubPath := "examples"
	if fr.IsContrib {
		exampleSubPath = "contrib/examples"
	}
	kptPkgPattern := regexp.MustCompile(
		fmt.Sprintf(`(https://github\.com/GoogleContainerTools/kpt-functions-catalog\.git/%s/)(%s)(\s+)`,
			exampleSubPath, exampleGroup))
	contents = kptPkgPattern.ReplaceAll(contents,
		[]byte(fmt.Sprintf(`${1}${2}@%s/%s${3}`, fr.FunctionName, fr.LatestPatchVersion)))
	return contents
}
<commit_msg>docs: update-function-docs Kptfile/metadata.yaml (#718)<commit_after>// Copyright 2021 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package main

import (
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"regexp"
	"strings"

	"golang.org/x/mod/semver"
	"gopkg.in/yaml.v2"
)

var (
	// pattern of release branches, e.g. apply-setters/v1.0
	releaseBranchPattern = regexp.MustCompile(`[-\w]*/(v\d*\.\d*)`)
	// pattern of release tags, e.g. functions/go/apply-setters/v1.0.1
	releaseTagPattern = regexp.MustCompile(`.*(go|ts)/[-\w]*/(v\d*\.\d*\.\d*)`)
	// pattern for version tags, e.g. unstable, v0.1.1, v0.1
	versionGroup = `unstable|v\d*\.\d*\.\d*|v\d*\.\d*`
)

func dirExists(path string) bool {
	if stat, err := os.Stat(path); err == nil && stat.IsDir() {
		return true
	}
	return false
}

func fileExists(path string) bool {
	if _, err := os.Stat(path); err == nil {
		return true
	}
	return false
}

type functionExample struct {
	ExamplePath string
	ExampleName string
}

type functionExamples []functionExample

// exampleNames returns a list of the functionExample names
func (fe functionExamples) exampleNames() []string {
	var exampleNames []string
	for _, example := range fe {
		exampleNames = append(exampleNames, example.ExampleName)
	}
	return exampleNames
}

type functionRelease struct {
	FunctionName       string
	MinorVersion       string
	Language           string
	LatestPatchVersion string
	FunctionPath       string
	Examples           functionExamples
	IsContrib          bool
}

// newFunctionRelease allocates and initializes a functionRelease
func newFunctionRelease(branch string) (*functionRelease, error) {
	fr := &functionRelease{}
	if !releaseBranchPattern.MatchString(branch) {
		return nil, fmt.Errorf("invalid branch format")
	}
	segments := strings.Split(branch, "/")
	// assume branch format: */<func_name>/<minor_version>
	fr.MinorVersion = segments[len(segments)-1]
	fr.FunctionName = segments[len(segments)-2]
	if err := fr.readLatestPatchVersion(); err != nil {
		return nil, err
	}
	if err := fr.readDocPaths(); err != nil {
		return nil, err
	}
	return fr, nil
}

// readLatestPatchVersion of the release from git tags
func (fr *functionRelease) readLatestPatchVersion() error {
	if fr.FunctionName == "" || fr.MinorVersion == "" {
		return fmt.Errorf("missing function name and/or minor version")
	}
	tags, err := gitTag()
	if err != nil {
		return err
	}
	funcPattern := fmt.Sprintf("%s/%s", fr.FunctionName, fr.MinorVersion)
	var lang, latestPatchVersion string
	for _, tag := range strings.Split(tags, "\n") {
		if !strings.Contains(tag, funcPattern) || !releaseTagPattern.MatchString(tag) {
			continue
		}
		segments := strings.Split(tag, "/")
		patchVersion := segments[len(segments)-1]
		if latestPatchVersion == "" ||
			semver.Compare(patchVersion, latestPatchVersion) == 1 {
			latestPatchVersion = patchVersion
			lang = segments[len(segments)-3]
		}
	}
	if latestPatchVersion == "" || lang == "" {
		return fmt.Errorf("could not find matching tag for release branch")
	}
	fr.Language = lang
	fr.LatestPatchVersion = latestPatchVersion
	return nil
}

// readDocPaths and set documentation paths
func (fr *functionRelease) readDocPaths() error {
	executablePath, err := os.Executable()
	if err != nil {
		return err
	}
	repoBase := filepath.Dir(filepath.Dir(filepath.Dir(executablePath)))
	pathsToTry := []struct {
		functionPath string
		examplesPath string
		isContrib    bool
	}{
		{
			functionPath: filepath.Join(repoBase, "functions", fr.Language, fr.FunctionName),
			examplesPath: filepath.Join(repoBase, "examples"),
			isContrib:    false,
		},
		{
			functionPath: filepath.Join(repoBase, "contrib", "functions", fr.Language, fr.FunctionName),
			examplesPath: filepath.Join(repoBase, "contrib", "examples"),
			isContrib:    true,
		},
	}
	var examplesPath string
	for _, pathToTry := range pathsToTry {
		if dirExists(pathToTry.functionPath) {
			fr.FunctionPath = pathToTry.functionPath
			fr.IsContrib = pathToTry.isContrib
			examplesPath = pathToTry.examplesPath
			break
		}
	}
	if fr.FunctionPath == "" {
		return fmt.Errorf("function doc paths not found from %+v", pathsToTry)
	}
	if err = fr.parseMetadata(examplesPath); err != nil {
		return err
	}
	return nil
}

// parseMetadata from metadata.yaml and set example paths
func (fr *functionRelease) parseMetadata(examplesPath string) error {
	type metadata struct {
		ExamplePackageUrls []string `yaml:"examplePackageURLs"`
	}
	if fr.FunctionPath == "" {
		return fmt.Errorf("expected FunctionPath in parseMetadata")
	}

	metadataPath := filepath.Join(fr.FunctionPath, "metadata.yaml")
	var md metadata
	yamlFile, err := ioutil.ReadFile(metadataPath)
	if err != nil {
		return err
	}

	err = yaml.Unmarshal(yamlFile, &md)
	if err != nil {
		return err
	}
	for _, exampleURL := range md.ExamplePackageUrls {
		segments := strings.Split(exampleURL, "/")
		exampleName := segments[len(segments)-1]
		examplePath := filepath.Join(examplesPath, exampleName)
		if !dirExists(examplePath) {
			return fmt.Errorf("example dir does not exist: %s", examplePath)
		}
		fr.Examples = append(fr.Examples, functionExample{
			ExamplePath: examplePath,
			ExampleName: exampleName,
		})
	}
	return nil
}

// updateDocs updates all the docs for the functionRelease on the filesystem
func (fr *functionRelease) updateDocs() error {
	if err := fr.updateFunctionDoc(); err != nil {
		return err
	}
	if err := fr.updateExampleDocs(); err != nil {
		return err
	}
	return nil
}

// updateFunctionDoc updates the function docs for the functionRelease
func (fr *functionRelease) updateFunctionDoc() error {
	functionReadme := filepath.Join(fr.FunctionPath, "README.md")
	if err := fr.updateDoc(functionReadme); err != nil {
		return err
	}
	functionMetadata := filepath.Join(fr.FunctionPath, "metadata.yaml")
	if err := fr.updateDoc(functionMetadata); err != nil {
		return err
	}
	return nil
}

// updateExampleDocs updates the example docs for the functionRelease
func (fr *functionRelease) updateExampleDocs() error {
	for _, example := range fr.Examples {
		exampleReadme := filepath.Join(example.ExamplePath, "README.md")
		if err := fr.updateDoc(exampleReadme); err != nil {
			return err
		}
		exampleKptfile := filepath.Join(example.ExamplePath, "Kptfile")
		if fileExists(exampleKptfile) {
			if err := fr.updateDoc(exampleKptfile); err != nil {
				return err
			}
		}
	}
	return nil
}

// Perform in place search/replace operations on a documentation file
func (fr *functionRelease) updateDoc(filePath string) error {
	contents, err := ioutil.ReadFile(filePath)
	if err != nil {
		return err
	}
	contents = fr.replaceTags(contents)
	contents = fr.replaceURLs(contents)
	contents = fr.replaceKptPackages(contents)
	contents = fr.replaceGithubURLs(contents)
	if err = os.WriteFile(filePath, contents, 0644); err != nil {
		return err
	}
	return nil
}

// replace tags with patch e.g. apply-setters:v1.0.1, apply-setters/v1.0.1
func (fr *functionRelease) replaceTags(contents []byte) []byte {
	tagPattern := regexp.MustCompile(
		fmt.Sprintf(`(%s)(:|/)(%s)`, fr.FunctionName, versionGroup))
	contents = tagPattern.ReplaceAll(contents,
		[]byte(fmt.Sprintf(`${1}${2}%s`, fr.LatestPatchVersion)))
	return contents
}

// replace url with minor e.g. https://catalog.kpt.dev/apply-setters/v1.0
func (fr *functionRelease) replaceURLs(contents []byte) []byte {
	urlPattern := regexp.MustCompile(
		fmt.Sprintf(`(https://catalog\.kpt\.dev/%s/)(%s)`, fr.FunctionName, versionGroup))
	contents = urlPattern.ReplaceAll(contents,
		[]byte(fmt.Sprintf(`${1}%s`, fr.MinorVersion)))
	return contents
}

// get sub-path to examples e.g. examples, contrib/examples
func (fr *functionRelease) exampleSubPath() string {
	exampleSubPath := "examples"
	if fr.IsContrib {
		exampleSubPath = "contrib/examples"
	}
	return exampleSubPath
}

// replace kpt package names for all examples, e.g.
// https://github.com/GoogleContainerTools/kpt-functions-catalog.git/examples/apply-setters-simple ->
// https://github.com/GoogleContainerTools/kpt-functions-catalog.git/examples/apply-setters-simple@apply-setters/v1.0.1
func (fr *functionRelease) replaceKptPackages(contents []byte) []byte {
	exampleGroup := strings.Join(fr.Examples.exampleNames(), "|")
	exampleSubPath := fr.exampleSubPath()
	kptPkgPattern := regexp.MustCompile(
		fmt.Sprintf(`(https://github\.com/GoogleContainerTools/kpt-functions-catalog\.git/%s/)(%s)(\s+)`,
			exampleSubPath, exampleGroup))
	contents = kptPkgPattern.ReplaceAll(contents,
		[]byte(fmt.Sprintf(`${1}${2}@%s/%s${3}`, fr.FunctionName, fr.LatestPatchVersion)))
	return contents
}

// replace branch name with release branch for all GitHub URLs, e.g.
// https://github.com/GoogleContainerTools/kpt-functions-catalog/tree/master/examples/set-namespace-simple ->
// https://github.com/GoogleContainerTools/kpt-functions-catalog/tree/set-namespace/v0.2/examples/set-namespace-simple
func (fr *functionRelease) replaceGithubURLs(contents []byte) []byte {
	exampleSubPath := fr.exampleSubPath()
	suffixes := []string{
		fmt.Sprintf(`/functions/%s/%s`, fr.Language, fr.FunctionName),
	}
	for _, ex := range fr.Examples.exampleNames() {
		suffixes = append(suffixes, fmt.Sprintf(`/%s/%s`, exampleSubPath, ex))
	}
	suffixGroup := strings.Join(suffixes, "|")
	refGroup := fmt.Sprintf(`master|%s/v\d*\.\d*`, fr.FunctionName)
	githubURLPattern := regexp.MustCompile(
		fmt.Sprintf(`(https://github\.com/GoogleContainerTools/kpt-functions-catalog/tree/)(%s)(%s)`,
			refGroup, suffixGroup))
	contents = githubURLPattern.ReplaceAll(contents,
		[]byte(fmt.Sprintf(`${1}%s/%s${3}`, fr.FunctionName, fr.MinorVersion)))
	return contents
}
<|endoftext|>


<commit_before>package main

import (
	"github.com/StabbyCutyou/buffstreams"
	"log"
	"strconv"
	"time"
)

func TestCallback(bts []byte) error {
	return nil
}

// This is not a proper test, but it lets me benchmark how it
// performs on a raw level. Run this with the time command
func main() {
	cfg := buffstreams.BuffManagerConfig{
		MaxMessageSize: 256,
	}
	// 100 byte message
	msg := []byte("HeyheyheyHeyheyheyHeyheyheyHeyheyheyHeyheyheyHeyheyheyHeyheyheyHeyheyheyHeyheyheyHeyheyheyHeyheyheyH!")
	startingPort := 5031
	for i := 0; i < 10; i++ {
		go func(n int, port string) {
			bm := buffstreams.New(cfg)
			bm.StartListening(strconv.Itoa(startingPort), TestCallback)
			address := buffstreams.FormatAddress("127.0.0.1", strconv.Itoa(startingPort))
			count := 0
			for {
				_, err := bm.WriteTo(address, msg, true)
				if err != nil {
					log.Print("EEEEEERRRRROOOOOOOORRRRRRRRRRR")
					log.Print(err)
				}
				count = count + 1
				log.Printf("%d - %d", n, count)
			}
		}(i, strconv.Itoa(startingPort))
		startingPort = startingPort + 1
	}
	time.Sleep(time.Minute * 10)
}
<commit_msg>Turning the 'too simple to be very useful, but technically a working implementation' test into a proper concurrency test on writes<commit_after>package main

import (
	"github.com/StabbyCutyou/buffstreams"
	"log"
	"strconv"
	"time"
)

func TestCallback(bts []byte) error {
	return nil
}

// This is not a proper test, but it lets me benchmark how it
// performs on a raw level. Run this with the time command
func main() {
	cfg := buffstreams.BuffManagerConfig{
		MaxMessageSize: 256,
		EnableLogging:  true,
	}
	// 100 byte message
	msg := []byte("HeyheyheyHeyheyheyHeyheyheyHeyheyheyHeyheyheyHeyheyheyHeyheyheyHeyheyheyHeyheyheyHeyheyheyHeyheyheyH!")
	startingPort := 5031
	bm := buffstreams.New(cfg)
	bm.StartListening(strconv.Itoa(startingPort), TestCallback)

	for i := 0; i < 10; i++ {
		go func(n int, port string, bm *buffstreams.BuffManager) {
			address := buffstreams.FormatAddress("127.0.0.1", port)
			count := 0
			for {
				_, err := bm.WriteTo(address, msg, true)
				if err != nil {
					log.Print("EEEEEERRRRROOOOOOOORRRRRRRRRRR")
					log.Print(err)
				}
				count = count + 1
				log.Printf("%d - %d", n, count)
			}
		}(i, strconv.Itoa(startingPort), bm)
		//startingPort = startingPort + 1
	}
	time.Sleep(time.Minute * 10)
}
<|endoftext|>