<commit_before>// Copyright 2018 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package git

import (
	"container/heap"
	"fmt"
	"sync"

	"github.com/golang/protobuf/proto"
	"golang.org/x/net/context"
	"google.golang.org/grpc/codes"
	"google.golang.org/grpc/status"

	"go.chromium.org/gae/service/datastore"
	gitilesapi "go.chromium.org/luci/common/api/gitiles"
	"go.chromium.org/luci/common/errors"
	"go.chromium.org/luci/common/logging"
	gitpb "go.chromium.org/luci/common/proto/git"
	"go.chromium.org/luci/common/proto/google"
	"go.chromium.org/luci/common/sync/parallel"
)

// A structure to keep a list of commits for some ref.
type refCommits struct {
	commits []*gitpb.Commit
}

// The pop method removes and returns first commit. Second return value is true
// if this was the last commit. Caller must ensure refCommits has commits when
// calling the method.
func (rc *refCommits) pop() (commit *gitpb.Commit, empty bool) {
	commit, rc.commits = rc.commits[0], rc.commits[1:]
	return commit, len(rc.commits) == 0
}

// We use commitHeap to merge slices of commits using max-heap algorithm below.
// Only first commit in each slice is used for comparisons.
type commitHeap []refCommits

func (h commitHeap) Len() int {
	return len(h)
}

func (h commitHeap) Swap(i, j int) {
	h[i], h[j] = h[j], h[i]
}

func (h commitHeap) Less(i, j int) bool {
	iTime := google.TimeFromProto(h[i].commits[0].Committer.Time)
	jTime := google.TimeFromProto(h[j].commits[0].Committer.Time)

	// Ensure consistent ordering based on commmit hash when times are identical.
	if iTime == jTime {
		return h[i].commits[0].Id > h[j].commits[0].Id
	}

	// To make heap behave as max-heap, we consider later time to be smaller than
	// earlier timer, i.e. latest commit will be the at the root of the heap.
	return iTime.After(jTime)
}

func (h *commitHeap) Push(x interface{}) {
	*h = append(*h, x.(refCommits))
}

func (h *commitHeap) Pop() interface{} {
	old := *h
	n := len(old)
	x := old[n-1]
	*h = old[0 : n-1]
	return x
}

// logCache stores a cached list of commits (log) for a given ref at a given
// commit position return by Gerrit. The Key decribes the query that was used to
// retrieve the log and follows the following format:
//
//   host|project|ref|exclude_ref|limit
//
// When the ref moves, entity is updated with the new CommitID and updated Log.
// The Log field is an encoded list of commits, which is a created by encoding a
// varint for the number of commits in the list followed by the corresponding
// number of serialized gitpb.Commit messages.
type logCache struct {
	Key      string `gae:"$id"`
	CommitID string `gae:"commit,noindex"`
	Log      []byte `gae:"log,noindex"`

	ref string `gae:"-"`
}

func logCacheFor(host, project, ref, excludeRef string, limit int) logCache {
	return logCache{
		Key: fmt.Sprintf("%s|%s|%s|%s|%d", host, project, ref, excludeRef, limit),
		ref: ref,
	}
}

func loadCacheFromDS(c context.Context, host, project, excludeRef string, limit int, refTips map[string]string) (cachedLogs map[string][]*gitpb.Commit) {
	items := make([]logCache, 0, len(refTips))
	for ref := range refTips {
		items = append(items, logCacheFor(host, project, ref, excludeRef, limit))
	}

	cachedLogs = map[string][]*gitpb.Commit{}
	var merr errors.MultiError
	switch err := datastore.Get(c, items).(type) {
	case errors.MultiError:
		merr = err
	case nil:
		merr = nil
	default:
		return
	}

	for i, item := range items {
		if (merr != nil && merr[i] != nil) || item.CommitID != refTips[item.ref] {
			continue
		}

		buf := proto.NewBuffer(item.Log)
		numCommits, err := buf.DecodeVarint()
		if err != nil {
			continue
		}

		log := make([]*gitpb.Commit, 0, numCommits)
		for j := uint64(0); j < numCommits; j++ {
			var commit gitpb.Commit
			if err = buf.DecodeMessage(&commit); err != nil {
				continue
			}

			log = append(log, &commit)
		}

		cachedLogs[item.ref] = log
	}

	return
}

func saveCacheToDS(c context.Context, host, project, excludeRef string, limit int, refLogs map[string][]*gitpb.Commit, refTips map[string]string) error {
	items := make([]logCache, 0, len(refLogs))
	totalBytes := 0
	for ref, log := range refLogs {
		buf := proto.NewBuffer([]byte{})
		if err := buf.EncodeVarint(uint64(len(log))); err != nil {
			return err
		}

		for _, commit := range log {
			if err := buf.EncodeMessage(commit); err != nil {
				return err
			}
		}

		item := logCacheFor(host, project, ref, excludeRef, limit)
		item.CommitID = refTips[ref]
		item.Log = buf.Bytes()
		items = append(items, item)

		// This logic breaks storing caches into datastore into smaller requests to
		// avoid exceeding 1MB limit on datastore requests set by AppEngine.
		totalBytes += len(item.Log)
		if totalBytes > 512*1024 { // 0.5 MiB
			if err := datastore.Put(c, items); err != nil {
				return err
			}
			totalBytes = 0
			items = items[:0]
		}
	}

	return datastore.Put(c, items)
}

func (impl *implementation) loadLogsForRefs(c context.Context, host, project, excludeRef string, limit int, refTips map[string]string) ([][]*gitpb.Commit, error) {
	cachedLogs := loadCacheFromDS(c, host, project, excludeRef, limit, refTips)
	logging.Infof(c, "Fetched %d logs from cache, will fetch remaining %d logs from Gitiles", len(cachedLogs), len(refTips)-len(cachedLogs))

	// Load missing logs from Gitiles.
	newLogs := make(map[string][]*gitpb.Commit)
	lock := sync.Mutex{} // for concurrent writes to the map
	err := parallel.FanOutIn(func(ch chan<- func() error) {
		for ref := range refTips {
			if _, ok := cachedLogs[ref]; ok {
				continue
			}

			ref := ref
			ch <- func() error {
				log, err := impl.log(c, host, project, refTips[ref], excludeRef, &LogOptions{Limit: limit})
				if err != nil {
					return err
				}

				lock.Lock()
				defer lock.Unlock()
				newLogs[ref] = log
				return nil
			}
		}
	})

	// Try to cache what we've fetched even if some requests failed.
	if derr := saveCacheToDS(c, host, project, excludeRef, limit, newLogs, refTips); derr != nil {
		logging.WithError(derr).Warningf(c, "Failed to cache logs fetched from Gitiles")
	}

	if err != nil {
		return nil, errors.Annotate(err, "failed to fetch %d logs from Gitiles", len(refTips)-len(cachedLogs)-len(newLogs)).Err()
	}

	// Drop ref names and create a list containing all logs.
	logs := make([][]*gitpb.Commit, 0, len(cachedLogs)+len(newLogs))
	for _, log := range cachedLogs {
		logs = append(logs, log)
	}
	for _, log := range newLogs {
		logs = append(logs, log)
	}

	return logs, nil
}

// CombinedLogs implements Client interface.
func (impl *implementation) CombinedLogs(c context.Context, host, project, excludeRef string, refs []string, limit int) (commits []*gitpb.Commit, err error) {
	defer func() { err = errors.Annotate(tagError(c, err), "gitiles.CombinedLogs").Err() }()

	// Check if the user is allowed to access this project.
	allowed, err := impl.acls.IsAllowed(c, host, project)
	switch {
	case err != nil:
		return
	case !allowed:
		err = status.Errorf(codes.NotFound, "not found")
		return
	}

	// Prepare Gitiles client.
	client, err := impl.gitilesClient(c, host)
	if err != nil {
		return
	}

	// Resolve all refs and commits they are pointing at.
	refTips, err := gitilesapi.NewRefSet(refs).Resolve(c, client, project)
	if err != nil {
		return
	}

	var logs [][]*gitpb.Commit
	if logs, err = impl.loadLogsForRefs(c, host, project, excludeRef, limit, refTips); err != nil {
		return
	}

	// We merge commits from all refs sorted by time into a single list up to a
	// limit. We use max-heap based merging algorithm below.
	var h commitHeap
	for _, log := range logs {
		if len(log) > 0 {
			h = append(h, refCommits{log})
		}
	}

	// Keep adding commits to the merged list until we reach the limit or run out
	// of commits on all refs.
	heap.Init(&h)
	commits = make([]*gitpb.Commit, 0, limit)
	for len(commits) < limit && len(h) != 0 {
		commit, empty := h[0].pop()
		// Do not add duplicate commits that come from different refs.
		if len(commits) == 0 || commits[len(commits)-1].Id != commit.Id {
			commits = append(commits, commit)
		}
		if empty {
			heap.Remove(&h, 0)
		} else {
			heap.Fix(&h, 0)
		}
	}

	return
}
<commit_msg>Limit number of concurrent requests to Gitiles to avoid exceeding quota<commit_after>// Copyright 2018 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package git

import (
	"container/heap"
	"fmt"
	"sync"

	"github.com/golang/protobuf/proto"
	"golang.org/x/net/context"
	"google.golang.org/grpc/codes"
	"google.golang.org/grpc/status"

	"go.chromium.org/gae/service/datastore"
	gitilesapi "go.chromium.org/luci/common/api/gitiles"
	"go.chromium.org/luci/common/errors"
	"go.chromium.org/luci/common/logging"
	gitpb "go.chromium.org/luci/common/proto/git"
	"go.chromium.org/luci/common/proto/google"
	"go.chromium.org/luci/common/sync/parallel"
)

// A structure to keep a list of commits for some ref.
type refCommits struct {
	commits []*gitpb.Commit
}

// The pop method removes and returns first commit. Second return value is true
// if this was the last commit. Caller must ensure refCommits has commits when
// calling the method.
func (rc *refCommits) pop() (commit *gitpb.Commit, empty bool) {
	commit, rc.commits = rc.commits[0], rc.commits[1:]
	return commit, len(rc.commits) == 0
}

// We use commitHeap to merge slices of commits using max-heap algorithm below.
// Only first commit in each slice is used for comparisons.
type commitHeap []refCommits

func (h commitHeap) Len() int {
	return len(h)
}

func (h commitHeap) Swap(i, j int) {
	h[i], h[j] = h[j], h[i]
}

func (h commitHeap) Less(i, j int) bool {
	iTime := google.TimeFromProto(h[i].commits[0].Committer.Time)
	jTime := google.TimeFromProto(h[j].commits[0].Committer.Time)

	// Ensure consistent ordering based on commmit hash when times are identical.
	if iTime == jTime {
		return h[i].commits[0].Id > h[j].commits[0].Id
	}

	// To make heap behave as max-heap, we consider later time to be smaller than
	// earlier timer, i.e. latest commit will be the at the root of the heap.
	return iTime.After(jTime)
}

func (h *commitHeap) Push(x interface{}) {
	*h = append(*h, x.(refCommits))
}

func (h *commitHeap) Pop() interface{} {
	old := *h
	n := len(old)
	x := old[n-1]
	*h = old[0 : n-1]
	return x
}

// logCache stores a cached list of commits (log) for a given ref at a given
// commit position return by Gerrit. The Key decribes the query that was used to
// retrieve the log and follows the following format:
//
//   host|project|ref|exclude_ref|limit
//
// When the ref moves, entity is updated with the new CommitID and updated Log.
// The Log field is an encoded list of commits, which is a created by encoding a
// varint for the number of commits in the list followed by the corresponding
// number of serialized gitpb.Commit messages.
type logCache struct {
	Key      string `gae:"$id"`
	CommitID string `gae:"commit,noindex"`
	Log      []byte `gae:"log,noindex"`

	ref string `gae:"-"`
}

func logCacheFor(host, project, ref, excludeRef string, limit int) logCache {
	return logCache{
		Key: fmt.Sprintf("%s|%s|%s|%s|%d", host, project, ref, excludeRef, limit),
		ref: ref,
	}
}

func loadCacheFromDS(c context.Context, host, project, excludeRef string, limit int, refTips map[string]string) (cachedLogs map[string][]*gitpb.Commit) {
	items := make([]logCache, 0, len(refTips))
	for ref := range refTips {
		items = append(items, logCacheFor(host, project, ref, excludeRef, limit))
	}

	cachedLogs = map[string][]*gitpb.Commit{}
	var merr errors.MultiError
	switch err := datastore.Get(c, items).(type) {
	case errors.MultiError:
		merr = err
	case nil:
		merr = nil
	default:
		return
	}

	for i, item := range items {
		if (merr != nil && merr[i] != nil) || item.CommitID != refTips[item.ref] {
			continue
		}

		buf := proto.NewBuffer(item.Log)
		numCommits, err := buf.DecodeVarint()
		if err != nil {
			continue
		}

		log := make([]*gitpb.Commit, 0, numCommits)
		for j := uint64(0); j < numCommits; j++ {
			var commit gitpb.Commit
			if err = buf.DecodeMessage(&commit); err != nil {
				continue
			}

			log = append(log, &commit)
		}

		cachedLogs[item.ref] = log
	}

	return
}

func saveCacheToDS(c context.Context, host, project, excludeRef string, limit int, refLogs map[string][]*gitpb.Commit, refTips map[string]string) error {
	items := make([]logCache, 0, len(refLogs))
	totalBytes := 0
	for ref, log := range refLogs {
		buf := proto.NewBuffer([]byte{})
		if err := buf.EncodeVarint(uint64(len(log))); err != nil {
			return err
		}

		for _, commit := range log {
			if err := buf.EncodeMessage(commit); err != nil {
				return err
			}
		}

		item := logCacheFor(host, project, ref, excludeRef, limit)
		item.CommitID = refTips[ref]
		item.Log = buf.Bytes()
		items = append(items, item)

		// This logic breaks storing caches into datastore into smaller requests to
		// avoid exceeding 1MB limit on datastore requests set by AppEngine.
		totalBytes += len(item.Log)
		if totalBytes > 512*1024 { // 0.5 MiB
			if err := datastore.Put(c, items); err != nil {
				return err
			}
			totalBytes = 0
			items = items[:0]
		}
	}

	return datastore.Put(c, items)
}

// maxGitilesLogRPCsPerRequest is the max number of Gitiles requests allowed per
// user request to avoid exceeding Gitiles quota.
const maxGitilesLogRPCsPerRequest = 50

func (impl *implementation) loadLogsForRefs(c context.Context, host, project, excludeRef string, limit int, refTips map[string]string) ([][]*gitpb.Commit, error) {
	cachedLogs := loadCacheFromDS(c, host, project, excludeRef, limit, refTips)
	logging.Infof(c, "Fetched %d logs from cache, will fetch remaining %d logs from Gitiles", len(cachedLogs), len(refTips)-len(cachedLogs))

	// Load missing logs from Gitiles.
	newLogs := make(map[string][]*gitpb.Commit)
	lock := sync.Mutex{} // for concurrent writes to the map
	err := parallel.FanOutIn(func(ch chan<- func() error) {
		numRequests := 0
		for ref := range refTips {
			if _, ok := cachedLogs[ref]; ok {
				continue
			}

			if numRequests++; numRequests > maxGitilesLogRPCsPerRequest {
				ch <- func() error {
					// TODO(sergiyb,tandrii): if you have genuine need for this many refs
					// at once, implement a cron job that runs this very function
					// continuously to avoid bursts of gitiles traffic that will make Milo
					// not functional for the other projects.
					return errors.Reason("too many refs are new or changed to be "+
						"fetched at once, stopping after %d. Check your config and/or "+
						"reload the page", maxGitilesLogRPCsPerRequest).Err()
				}
				break
			}

			ref := ref
			ch <- func() error {
				log, err := impl.log(c, host, project, refTips[ref], excludeRef, &LogOptions{Limit: limit})
				if err != nil {
					return err
				}

				lock.Lock()
				defer lock.Unlock()
				newLogs[ref] = log
				return nil
			}
		}
	})

	// Try to cache what we've fetched even if some requests failed.
	if derr := saveCacheToDS(c, host, project, excludeRef, limit, newLogs, refTips); derr != nil {
		logging.WithError(derr).Warningf(c, "Failed to cache logs fetched from Gitiles")
	}

	if err != nil {
		return nil, errors.Annotate(err, "failed to fetch %d logs from Gitiles", len(refTips)-len(cachedLogs)-len(newLogs)).Err()
	}

	// Drop ref names and create a list containing all logs.
	logs := make([][]*gitpb.Commit, 0, len(cachedLogs)+len(newLogs))
	for _, log := range cachedLogs {
		logs = append(logs, log)
	}
	for _, log := range newLogs {
		logs = append(logs, log)
	}

	return logs, nil
}

// CombinedLogs implements Client interface.
func (impl *implementation) CombinedLogs(c context.Context, host, project, excludeRef string, refs []string, limit int) (commits []*gitpb.Commit, err error) {
	defer func() { err = errors.Annotate(tagError(c, err), "gitiles.CombinedLogs").Err() }()

	// Check if the user is allowed to access this project.
	allowed, err := impl.acls.IsAllowed(c, host, project)
	switch {
	case err != nil:
		return
	case !allowed:
		err = status.Errorf(codes.NotFound, "not found")
		return
	}

	// Prepare Gitiles client.
	client, err := impl.gitilesClient(c, host)
	if err != nil {
		return
	}

	// Resolve all refs and commits they are pointing at.
	refTips, err := gitilesapi.NewRefSet(refs).Resolve(c, client, project)
	if err != nil {
		return
	}

	var logs [][]*gitpb.Commit
	if logs, err = impl.loadLogsForRefs(c, host, project, excludeRef, limit, refTips); err != nil {
		return
	}

	// We merge commits from all refs sorted by time into a single list up to a
	// limit. We use max-heap based merging algorithm below.
	var h commitHeap
	for _, log := range logs {
		if len(log) > 0 {
			h = append(h, refCommits{log})
		}
	}

	// Keep adding commits to the merged list until we reach the limit or run out
	// of commits on all refs.
	heap.Init(&h)
	commits = make([]*gitpb.Commit, 0, limit)
	for len(commits) < limit && len(h) != 0 {
		commit, empty := h[0].pop()
		// Do not add duplicate commits that come from different refs.
		if len(commits) == 0 || commits[len(commits)-1].Id != commit.Id {
			commits = append(commits, commit)
		}
		if empty {
			heap.Remove(&h, 0)
		} else {
			heap.Fix(&h, 0)
		}
	}

	return
}
<|endoftext|>


<commit_before>package dhcpv4

// MessageType is the type for the various DHCP messages defined in RFC2132.
type MessageType byte

const (
	MessageTypeDHCPDiscover = MessageType(1)
	MessageTypeDHCPOffer    = MessageType(2)
	MessageTypeDHCPRequest  = MessageType(3)
	MessageTypeDHCPDecline  = MessageType(4)
	MessageTypeDHCPAck      = MessageType(5)
	MessageTypeDHCPNak      = MessageType(6)
	MessageTypeDHCPRelease  = MessageType(7)
	MessageTypeDHCPInform   = MessageType(8)
)

// Option is the type for DHCP option tags.
type Option byte

// OptionMap maps DHCP option tags to their values.
type OptionMap map[Option][]byte

// GetOption gets the []byte value of an option.
func (om OptionMap) GetOption(o Option) ([]byte, bool) {
	v, ok := om[o]
	return v, ok
}

// SetOption sets the []byte value of an option.
func (om OptionMap) SetOption(o Option, v []byte) {
	om[o] = v
	return
}

// GetMessageType gets the message type from the DHCPMsgType option field.
func (om OptionMap) GetMessageType() MessageType {
	v, ok := om.GetOption(OptionDHCPMsgType)
	if !ok || len(v) != 1 {
		return MessageType(0)
	}

	return MessageType(v[0])
}

// SetMessageType sets the message type in the DHCPMsgType option field.
func (om OptionMap) SetMessageType(m MessageType) {
	om.SetOption(OptionDHCPMsgType, []byte{byte(m)})
	return
}

// From RFC2132: DHCP Options and BOOTP Vendor Extensions
const (
	// RFC2132 Section 3: RFC 1497 Vendor Extensions
	OptionPad           = Option(0)
	OptionEnd           = Option(255)
	OptionSubnetMask    = Option(1)
	OptionTimeOffset    = Option(2)
	OptionRouter        = Option(3)
	OptionTimeServer    = Option(4)
	OptionNameServer    = Option(5)
	OptionDomainServer  = Option(6)
	OptionLogServer     = Option(7)
	OptionQuotesServer  = Option(8)
	OptionLPRServer     = Option(9)
	OptionImpressServer = Option(10)
	OptionRLPServer     = Option(11)
	OptionHostname      = Option(12)
	OptionBootFileSize  = Option(13)
	OptionMeritDumpFile = Option(14)
	OptionDomainName    = Option(15)
	OptionSwapServer    = Option(16)
	OptionRootPath      = Option(17)
	OptionExtensionFile = Option(18)

	// RFC2132 Section 4: IP Layer Parameters per Host
	OptionForwardOnOff  = Option(19)
	OptionSrcRteOnOff   = Option(20)
	OptionPolicyFilter  = Option(21)
	OptionMaxDGAssembly = Option(22)
	OptionDefaultIPTTL  = Option(23)
	OptionMTUTimeout    = Option(24)
	OptionMTUPlateau    = Option(25)

	// RFC2132 Section 5: IP Layer Parameters per Interface
	OptionMTUInterface     = Option(26)
	OptionMTUSubnet        = Option(27)
	OptionBroadcastAddress = Option(28)
	OptionMaskDiscovery    = Option(29)
	OptionMaskSupplier     = Option(30)
	OptionRouterDiscovery  = Option(31)
	OptionRouterRequest    = Option(32)
	OptionStaticRoute      = Option(33)

	// RFC2132 Section 6: Link Layer Parameters per Interface
	OptionTrailers   = Option(34)
	OptionARPTimeout = Option(35)
	OptionEthernet   = Option(36)

	// RFC2132 Section 7: TCP Parameters
	OptionDefaultTCPTTL = Option(37)
	OptionKeepaliveTime = Option(38)
	OptionKeepaliveData = Option(39)

	// RFC2132 Section 8: Application and Service Parameters
	OptionNISDomain        = Option(40)
	OptionNISServers       = Option(41)
	OptionNTPServers       = Option(42)
	OptionVendorSpecific   = Option(43)
	OptionNETBIOSNameSrv   = Option(44)
	OptionNETBIOSDistSrv   = Option(45)
	OptionNETBIOSNodeType  = Option(46)
	OptionNETBIOSScope     = Option(47)
	OptionXWindowFont      = Option(48)
	OptionXWindowManager   = Option(49)
	OptionNISDomainName    = Option(64)
	OptionNISServerAddr    = Option(65)
	OptionHomeAgentAddrs   = Option(68)
	OptionSMTPServer       = Option(69)
	OptionPOP3Server       = Option(70)
	OptionNNTPServer       = Option(71)
	OptionWWWServer        = Option(72)
	OptionFingerServer     = Option(73)
	OptionIRCServer        = Option(74)
	OptionStreetTalkServer = Option(75)
	OptionSTDAServer       = Option(76)

	// RFC2132 Section 9: DHCP Extensions
	OptionAddressRequest = Option(50)
	OptionAddressTime    = Option(51)
	OptionOverload       = Option(52)
	OptionServerName     = Option(66)
	OptionBootfileName   = Option(67)
	OptionDHCPMsgType    = Option(53)
	OptionDHCPServerID   = Option(54)
	OptionParameterList  = Option(55)
	OptionDHCPMessage    = Option(56)
	OptionDHCPMaxMsgSize = Option(57)
	OptionRenewalTime    = Option(58)
	OptionRebindingTime  = Option(59)
	OptionClassID        = Option(60)
	OptionClientID       = Option(61)
)

// From RFC2241: DHCP Options for Novell Directory Services
const (
	OptionNDSServers  = Option(85)
	OptionNDSTreeName = Option(86)
	OptionNDSContext  = Option(87)
)

// From RFC2242: NetWare/IP Domain Name and Information
const (
	OptionNetWareIPDomain = Option(62)
	OptionNetWareIPOption = Option(63)
)

// From RFC2485: DHCP Option for The Open Group\x27s User Authentication Protocol
const (
	OptionUserAuth = Option(98)
)

// From RFC2563: DHCP Option to Disable Stateless Auto-Configuration in IPv4 Clients
const (
	OptionAutoConfig = Option(116)
)

// From RFC2610: DHCP Options for Service Location Protocol
const (
	OptionDirectoryAgent = Option(78)
	OptionServiceScope   = Option(79)
)

// From RFC2937: The Name Service Search Option for DHCP
const (
	OptionNameServiceSearch = Option(117)
)

// From RFC3004: The User Class Option for DHCP
const (
	OptionUserClass = Option(77)
)

// From RFC3011: The IPv4 Subnet Selection Option for DHCP
const (
	OptionSubnetSelectionOption = Option(118)
)

// From RFC3046: DHCP Relay Agent Information Option
const (
	OptionRelayAgentInformation = Option(82)
)

// From RFC3118: Authentication for DHCP Messages
const (
	OptionAuthentication = Option(90)
)

// From RFC3361: Dynamic Host Configuration Protocol (DHCP-for-IPv4) Option for Session Initiation Protocol (SIP) Servers
const (
	OptionSIPServersDHCPOption = Option(120)
)

// From RFC3397: Dynamic Host Configuration Protocol (DHCP) Domain Search Option
const (
	OptionDomainSearch = Option(119)
)

// From RFC3442: The Classless Static Route Option for Dynamic Host Configuration Protocol (DHCP) version 4
const (
	OptionClasslessStaticRouteOption = Option(121)
)

// From RFC3495: Dynamic Host Configuration Protocol (DHCP) Option for CableLabs Client Configuration
const (
	OptionCCC = Option(122)
)

// From RFC3679: Unused Dynamic Host Configuration Protocol (DHCP) Option Codes
const (
	OptionLDAP           = Option(95)
	OptionNetinfoAddress = Option(112)
	OptionNetinfoTag     = Option(113)
	OptionURL            = Option(114)
)

// From RFC3925: Vendor-Identifying Vendor Options for Dynamic Host Configuration Protocol version 4 (DHCPv4)
const (
	OptionVIVendorClass               = Option(124)
	OptionVIVendorSpecificInformation = Option(125)
)

// From RFC4039: Rapid Commit Option for the Dynamic Host Configuration Protocol version 4 (DHCPv4)
const (
	OptionRapidCommit = Option(80)
)

// From RFC4174: The IPv4 Dynamic Host Configuration Protocol (DHCP) Option for the Internet Storage Name Service
const (
	OptioniSNS = Option(83)
)

// From RFC4280: Dynamic Host Configuration Protocol (DHCP) Options for Broadcast and Multicast Control Servers
const (
	OptionBCMCSControllerDomainNameList    = Option(88)
	OptionBCMCSControllerIPv4AddressOption = Option(89)
)

// From RFC4388: Dynamic Host Configuration Protocol (DHCP) Leasequery
const (
	OptionClientLastTransactionTimeOption = Option(91)
	OptionAssociatedIPOption              = Option(92)
)

// From RFC4578: Dynamic Host Configuration Protocol (DHCP) Options for the Intel Preboot eXecution Environment (PXE)
const (
	OptionClientSystem    = Option(93)
	OptionClientNDI       = Option(94)
	OptionUUIDGUID        = Option(97)
	OptionPXEUndefined128 = Option(128)
	OptionPXEUndefined129 = Option(129)
	OptionPXEUndefined130 = Option(130)
	OptionPXEUndefined131 = Option(131)
	OptionPXEUndefined132 = Option(132)
	OptionPXEUndefined133 = Option(133)
	OptionPXEUndefined134 = Option(134)
	OptionPXEUndefined135 = Option(135)
)

// From RFC4702: The Dynamic Host Configuration Protocol (DHCP) Client Fully Qualified Domain Name (FQDN) Option
const (
	OptionClientFQDN = Option(81)
)

// From RFC4776: Dynamic Host Configuration Protocol (DHCPv4 and DHCPv6) Option for Civic Addresses Configuration Information
const (
	OptionGeoConfCivic = Option(99)
)

// From RFC4833: Timezone Options for DHCP
const (
	OptionPCode = Option(100)
	OptionTCode = Option(101)
)

// From RFC6225: Dynamic Host Configuration Protocol Options for Coordinate-Based Location Configuration Information
const (
	OptionGeoConfOption = Option(123)
	OptionGeoLoc        = Option(144)
)
<commit_msg>Remove superfluous return statements<commit_after>package dhcpv4

// MessageType is the type for the various DHCP messages defined in RFC2132.
type MessageType byte

const (
	MessageTypeDHCPDiscover = MessageType(1)
	MessageTypeDHCPOffer    = MessageType(2)
	MessageTypeDHCPRequest  = MessageType(3)
	MessageTypeDHCPDecline  = MessageType(4)
	MessageTypeDHCPAck      = MessageType(5)
	MessageTypeDHCPNak      = MessageType(6)
	MessageTypeDHCPRelease  = MessageType(7)
	MessageTypeDHCPInform   = MessageType(8)
)

// Option is the type for DHCP option tags.
type Option byte

// OptionMap maps DHCP option tags to their values.
type OptionMap map[Option][]byte

// GetOption gets the []byte value of an option.
func (om OptionMap) GetOption(o Option) ([]byte, bool) {
	v, ok := om[o]
	return v, ok
}

// SetOption sets the []byte value of an option.
func (om OptionMap) SetOption(o Option, v []byte) {
	om[o] = v
}

// GetMessageType gets the message type from the DHCPMsgType option field.
func (om OptionMap) GetMessageType() MessageType {
	v, ok := om.GetOption(OptionDHCPMsgType)
	if !ok || len(v) != 1 {
		return MessageType(0)
	}

	return MessageType(v[0])
}

// SetMessageType sets the message type in the DHCPMsgType option field.
func (om OptionMap) SetMessageType(m MessageType) {
	om.SetOption(OptionDHCPMsgType, []byte{byte(m)})
}

// From RFC2132: DHCP Options and BOOTP Vendor Extensions
const (
	// RFC2132 Section 3: RFC 1497 Vendor Extensions
	OptionPad           = Option(0)
	OptionEnd           = Option(255)
	OptionSubnetMask    = Option(1)
	OptionTimeOffset    = Option(2)
	OptionRouter        = Option(3)
	OptionTimeServer    = Option(4)
	OptionNameServer    = Option(5)
	OptionDomainServer  = Option(6)
	OptionLogServer     = Option(7)
	OptionQuotesServer  = Option(8)
	OptionLPRServer     = Option(9)
	OptionImpressServer = Option(10)
	OptionRLPServer     = Option(11)
	OptionHostname      = Option(12)
	OptionBootFileSize  = Option(13)
	OptionMeritDumpFile = Option(14)
	OptionDomainName    = Option(15)
	OptionSwapServer    = Option(16)
	OptionRootPath      = Option(17)
	OptionExtensionFile = Option(18)

	// RFC2132 Section 4: IP Layer Parameters per Host
	OptionForwardOnOff  = Option(19)
	OptionSrcRteOnOff   = Option(20)
	OptionPolicyFilter  = Option(21)
	OptionMaxDGAssembly = Option(22)
	OptionDefaultIPTTL  = Option(23)
	OptionMTUTimeout    = Option(24)
	OptionMTUPlateau    = Option(25)

	// RFC2132 Section 5: IP Layer Parameters per Interface
	OptionMTUInterface     = Option(26)
	OptionMTUSubnet        = Option(27)
	OptionBroadcastAddress = Option(28)
	OptionMaskDiscovery    = Option(29)
	OptionMaskSupplier     = Option(30)
	OptionRouterDiscovery  = Option(31)
	OptionRouterRequest    = Option(32)
	OptionStaticRoute      = Option(33)

	// RFC2132 Section 6: Link Layer Parameters per Interface
	OptionTrailers   = Option(34)
	OptionARPTimeout = Option(35)
	OptionEthernet   = Option(36)

	// RFC2132 Section 7: TCP Parameters
	OptionDefaultTCPTTL = Option(37)
	OptionKeepaliveTime = Option(38)
	OptionKeepaliveData = Option(39)

	// RFC2132 Section 8: Application and Service Parameters
	OptionNISDomain        = Option(40)
	OptionNISServers       = Option(41)
	OptionNTPServers       = Option(42)
	OptionVendorSpecific   = Option(43)
	OptionNETBIOSNameSrv   = Option(44)
	OptionNETBIOSDistSrv   = Option(45)
	OptionNETBIOSNodeType  = Option(46)
	OptionNETBIOSScope     = Option(47)
	OptionXWindowFont      = Option(48)
	OptionXWindowManager   = Option(49)
	OptionNISDomainName    = Option(64)
	OptionNISServerAddr    = Option(65)
	OptionHomeAgentAddrs   = Option(68)
	OptionSMTPServer       = Option(69)
	OptionPOP3Server       = Option(70)
	OptionNNTPServer       = Option(71)
	OptionWWWServer        = Option(72)
	OptionFingerServer     = Option(73)
	OptionIRCServer        = Option(74)
	OptionStreetTalkServer = Option(75)
	OptionSTDAServer       = Option(76)

	// RFC2132 Section 9: DHCP Extensions
	OptionAddressRequest = Option(50)
	OptionAddressTime    = Option(51)
	OptionOverload       = Option(52)
	OptionServerName     = Option(66)
	OptionBootfileName   = Option(67)
	OptionDHCPMsgType    = Option(53)
	OptionDHCPServerID   = Option(54)
	OptionParameterList  = Option(55)
	OptionDHCPMessage    = Option(56)
	OptionDHCPMaxMsgSize = Option(57)
	OptionRenewalTime    = Option(58)
	OptionRebindingTime  = Option(59)
	OptionClassID        = Option(60)
	OptionClientID       = Option(61)
)

// From RFC2241: DHCP Options for Novell Directory Services
const (
	OptionNDSServers  = Option(85)
	OptionNDSTreeName = Option(86)
	OptionNDSContext  = Option(87)
)

// From RFC2242: NetWare/IP Domain Name and Information
const (
	OptionNetWareIPDomain = Option(62)
	OptionNetWareIPOption = Option(63)
)

// From RFC2485: DHCP Option for The Open Group\x27s User Authentication Protocol
const (
	OptionUserAuth = Option(98)
)

// From RFC2563: DHCP Option to Disable Stateless Auto-Configuration in IPv4 Clients
const (
	OptionAutoConfig = Option(116)
)

// From RFC2610: DHCP Options for Service Location Protocol
const (
	OptionDirectoryAgent = Option(78)
	OptionServiceScope   = Option(79)
)

// From RFC2937: The Name Service Search Option for DHCP
const (
	OptionNameServiceSearch = Option(117)
)

// From RFC3004: The User Class Option for DHCP
const (
	OptionUserClass = Option(77)
)

// From RFC3011: The IPv4 Subnet Selection Option for DHCP
const (
	OptionSubnetSelectionOption = Option(118)
)

// From RFC3046: DHCP Relay Agent Information Option
const (
	OptionRelayAgentInformation = Option(82)
)

// From RFC3118: Authentication for DHCP Messages
const (
	OptionAuthentication = Option(90)
)

// From RFC3361: Dynamic Host Configuration Protocol (DHCP-for-IPv4) Option for Session Initiation Protocol (SIP) Servers
const (
	OptionSIPServersDHCPOption = Option(120)
)

// From RFC3397: Dynamic Host Configuration Protocol (DHCP) Domain Search Option
const (
	OptionDomainSearch = Option(119)
)

// From RFC3442: The Classless Static Route Option for Dynamic Host Configuration Protocol (DHCP) version 4
const (
	OptionClasslessStaticRouteOption = Option(121)
)

// From RFC3495: Dynamic Host Configuration Protocol (DHCP) Option for CableLabs Client Configuration
const (
	OptionCCC = Option(122)
)

// From RFC3679: Unused Dynamic Host Configuration Protocol (DHCP) Option Codes
const (
	OptionLDAP           = Option(95)
	OptionNetinfoAddress = Option(112)
	OptionNetinfoTag     = Option(113)
	OptionURL            = Option(114)
)

// From RFC3925: Vendor-Identifying Vendor Options for Dynamic Host Configuration Protocol version 4 (DHCPv4)
const (
	OptionVIVendorClass               = Option(124)
	OptionVIVendorSpecificInformation = Option(125)
)

// From RFC4039: Rapid Commit Option for the Dynamic Host Configuration Protocol version 4 (DHCPv4)
const (
	OptionRapidCommit = Option(80)
)

// From RFC4174: The IPv4 Dynamic Host Configuration Protocol (DHCP) Option for the Internet Storage Name Service
const (
	OptioniSNS = Option(83)
)

// From RFC4280: Dynamic Host Configuration Protocol (DHCP) Options for Broadcast and Multicast Control Servers
const (
	OptionBCMCSControllerDomainNameList    = Option(88)
	OptionBCMCSControllerIPv4AddressOption = Option(89)
)

// From RFC4388: Dynamic Host Configuration Protocol (DHCP) Leasequery
const (
	OptionClientLastTransactionTimeOption = Option(91)
	OptionAssociatedIPOption              = Option(92)
)

// From RFC4578: Dynamic Host Configuration Protocol (DHCP) Options for the Intel Preboot eXecution Environment (PXE)
const (
	OptionClientSystem    = Option(93)
	OptionClientNDI       = Option(94)
	OptionUUIDGUID        = Option(97)
	OptionPXEUndefined128 = Option(128)
	OptionPXEUndefined129 = Option(129)
	OptionPXEUndefined130 = Option(130)
	OptionPXEUndefined131 = Option(131)
	OptionPXEUndefined132 = Option(132)
	OptionPXEUndefined133 = Option(133)
	OptionPXEUndefined134 = Option(134)
	OptionPXEUndefined135 = Option(135)
)

// From RFC4702: The Dynamic Host Configuration Protocol (DHCP) Client Fully Qualified Domain Name (FQDN) Option
const (
	OptionClientFQDN = Option(81)
)

// From RFC4776: Dynamic Host Configuration Protocol (DHCPv4 and DHCPv6) Option for Civic Addresses Configuration Information
const (
	OptionGeoConfCivic = Option(99)
)

// From RFC4833: Timezone Options for DHCP
const (
	OptionPCode = Option(100)
	OptionTCode = Option(101)
)

// From RFC6225: Dynamic Host Configuration Protocol Options for Coordinate-Based Location Configuration Information
const (
	OptionGeoConfOption = Option(123)
	OptionGeoLoc        = Option(144)
)
<|endoftext|>


<commit_before>package base

import (
	"testing"

	"github.com/go-gitea/gitea/modules/setting"
	"github.com/stretchr/testify/assert"
	"strk.kbt.io/projects/go/libravatar"
)

func TestEncodeMD5(t *testing.T) {
	assert.Equal(t,
		"3858f62230ac3c915f300c664312c63f",
		EncodeMD5("foobar"),
	)
}

func TestEncodeSha1(t *testing.T) {
	assert.Equal(t,
		"8843d7f92416211de9ebb963ff4ce28125932878",
		EncodeSha1("foobar"),
	)
}

func TestShortSha(t *testing.T) {
	assert.Equal(t, "veryverylo", ShortSha("veryverylong"))
}

// TODO: Test DetectEncoding()

func TestBasicAuthDecode(t *testing.T) {
	_, _, err := BasicAuthDecode("?")
	assert.Equal(t, "illegal base64 data at input byte 0", err.Error())

	user, pass, err := BasicAuthDecode("Zm9vOmJhcg==")
	assert.NoError(t, err)
	assert.Equal(t, "foo", user)
	assert.Equal(t, "bar", pass)
}

func TestBasicAuthEncode(t *testing.T) {
	assert.Equal(t, "Zm9vOmJhcg==", BasicAuthEncode("foo", "bar"))
}

func TestGetRandomString(t *testing.T) {
	assert.Len(t, GetRandomString(4), 4)
}

// TODO: Test PBKDF2()
// TODO: Test VerifyTimeLimitCode()
// TODO: Test CreateTimeLimitCode()

func TestHashEmail(t *testing.T) {
	assert.Equal(t,
		"d41d8cd98f00b204e9800998ecf8427e",
		HashEmail(""),
	)
	assert.Equal(t,
		"353cbad9b58e69c96154ad99f92bedc7",
		HashEmail("gitea@example.com"),
	)
}

func TestAvatarLink(t *testing.T) {
	setting.EnableFederatedAvatar = false
	setting.LibravatarService = nil
	setting.DisableGravatar = true

	assert.Equal(t, "/img/avatar_default.png", AvatarLink(""))

	setting.DisableGravatar = false
	assert.Equal(t,
		"353cbad9b58e69c96154ad99f92bedc7",
		AvatarLink("gitea@example.com"),
	)

	setting.EnableFederatedAvatar = true
	assert.Equal(t,
		"353cbad9b58e69c96154ad99f92bedc7",
		AvatarLink("gitea@example.com"),
	)
	setting.LibravatarService = libravatar.New()
	assert.Equal(t,
		"http://cdn.libravatar.org/avatar/353cbad9b58e69c96154ad99f92bedc7",
		AvatarLink("gitea@example.com"),
	)
}

// TODO: computeTimeDiff()
// TODO: TimeSincePro()
// TODO: timeSince()
// TODO: RawTimeSince()
// TODO: TimeSince()

func TestFileSize(t *testing.T) {
	var size int64
	size = 512
	assert.Equal(t, "512B", FileSize(size))
	size = size * 1024
	assert.Equal(t, "512KB", FileSize(size))
	size = size * 1024
	assert.Equal(t, "512MB", FileSize(size))
	size = size * 1024
	assert.Equal(t, "512GB", FileSize(size))
	size = size * 1024
	assert.Equal(t, "512TB", FileSize(size))
	size = size * 1024
	assert.Equal(t, "512PB", FileSize(size))
	//size = size * 1024 TODO: Fix bug for EB
	//assert.Equal(t, "512EB", FileSize(size))
}

// TODO: Subtract()

func TestEllipsisString(t *testing.T) {
	assert.Equal(t, "...", EllipsisString("foobar", 0))
	assert.Equal(t, "...", EllipsisString("foobar", 1))
	assert.Equal(t, "...", EllipsisString("foobar", 2))
	assert.Equal(t, "...", EllipsisString("foobar", 3))
	assert.Equal(t, "f...", EllipsisString("foobar", 4))
	assert.Equal(t, "fo...", EllipsisString("foobar", 5))
	assert.Equal(t, "foobar", EllipsisString("foobar", 6))
	assert.Equal(t, "foobar", EllipsisString("foobar", 10))
}

func TestTruncateString(t *testing.T) {
	assert.Equal(t, "", TruncateString("foobar", 0))
	assert.Equal(t, "f", TruncateString("foobar", 1))
	assert.Equal(t, "fo", TruncateString("foobar", 2))
	assert.Equal(t, "foo", TruncateString("foobar", 3))
	assert.Equal(t, "foob", TruncateString("foobar", 4))
	assert.Equal(t, "fooba", TruncateString("foobar", 5))
	assert.Equal(t, "foobar", TruncateString("foobar", 6))
	assert.Equal(t, "foobar", TruncateString("foobar", 7))
}

func TestStringsToInt64s(t *testing.T) {
	assert.Equal(t, []int64{}, StringsToInt64s([]string{}))
	assert.Equal(t,
		[]int64{1, 4, 16, 64, 256},
		StringsToInt64s([]string{"1", "4", "16", "64", "256"}),
	)

	// TODO: StringsToInt64s should return ([]int64, error)
	assert.Equal(t, []int64{-1, 0, 0}, StringsToInt64s([]string{"-1", "a", "$"}))
}

func TestInt64sToStrings(t *testing.T) {
	assert.Equal(t, []string{}, Int64sToStrings([]int64{}))
	assert.Equal(t,
		[]string{"1", "4", "16", "64", "256"},
		Int64sToStrings([]int64{1, 4, 16, 64, 256}),
	)
}

func TestInt64sToMap(t *testing.T) {
	assert.Equal(t, map[int64]bool{}, Int64sToMap([]int64{}))
	assert.Equal(t,
		map[int64]bool{1: true, 4: true, 16: true},
		Int64sToMap([]int64{1, 4, 16}),
	)
}

// TODO: IsLetter()
// TODO: IsTextFile()
// TODO: IsImageFile()
// TODO: IsPDFFile()
<commit_msg>Add test for IsLetter()<commit_after>package base

import (
	"testing"

	"github.com/go-gitea/gitea/modules/setting"
	"github.com/stretchr/testify/assert"
	"strk.kbt.io/projects/go/libravatar"
)

func TestEncodeMD5(t *testing.T) {
	assert.Equal(t,
		"3858f62230ac3c915f300c664312c63f",
		EncodeMD5("foobar"),
	)
}

func TestEncodeSha1(t *testing.T) {
	assert.Equal(t,
		"8843d7f92416211de9ebb963ff4ce28125932878",
		EncodeSha1("foobar"),
	)
}

func TestShortSha(t *testing.T) {
	assert.Equal(t, "veryverylo", ShortSha("veryverylong"))
}

// TODO: Test DetectEncoding()

func TestBasicAuthDecode(t *testing.T) {
	_, _, err := BasicAuthDecode("?")
	assert.Equal(t, "illegal base64 data at input byte 0", err.Error())

	user, pass, err := BasicAuthDecode("Zm9vOmJhcg==")
	assert.NoError(t, err)
	assert.Equal(t, "foo", user)
	assert.Equal(t, "bar", pass)
}

func TestBasicAuthEncode(t *testing.T) {
	assert.Equal(t, "Zm9vOmJhcg==", BasicAuthEncode("foo", "bar"))
}

func TestGetRandomString(t *testing.T) {
	assert.Len(t, GetRandomString(4), 4)
}

// TODO: Test PBKDF2()
// TODO: Test VerifyTimeLimitCode()
// TODO: Test CreateTimeLimitCode()

func TestHashEmail(t *testing.T) {
	assert.Equal(t,
		"d41d8cd98f00b204e9800998ecf8427e",
		HashEmail(""),
	)
	assert.Equal(t,
		"353cbad9b58e69c96154ad99f92bedc7",
		HashEmail("gitea@example.com"),
	)
}

func TestAvatarLink(t *testing.T) {
	setting.EnableFederatedAvatar = false
	setting.LibravatarService = nil
	setting.DisableGravatar = true

	assert.Equal(t, "/img/avatar_default.png", AvatarLink(""))

	setting.DisableGravatar = false
	assert.Equal(t,
		"353cbad9b58e69c96154ad99f92bedc7",
		AvatarLink("gitea@example.com"),
	)

	setting.EnableFederatedAvatar = true
	assert.Equal(t,
		"353cbad9b58e69c96154ad99f92bedc7",
		AvatarLink("gitea@example.com"),
	)
	setting.LibravatarService = libravatar.New()
	assert.Equal(t,
		"http://cdn.libravatar.org/avatar/353cbad9b58e69c96154ad99f92bedc7",
		AvatarLink("gitea@example.com"),
	)
}

// TODO: computeTimeDiff()
// TODO: TimeSincePro()
// TODO: timeSince()
// TODO: RawTimeSince()
// TODO: TimeSince()

func TestFileSize(t *testing.T) {
	var size int64
	size = 512
	assert.Equal(t, "512B", FileSize(size))
	size = size * 1024
	assert.Equal(t, "512KB", FileSize(size))
	size = size * 1024
	assert.Equal(t, "512MB", FileSize(size))
	size = size * 1024
	assert.Equal(t, "512GB", FileSize(size))
	size = size * 1024
	assert.Equal(t, "512TB", FileSize(size))
	size = size * 1024
	assert.Equal(t, "512PB", FileSize(size))
	//size = size * 1024 TODO: Fix bug for EB
	//assert.Equal(t, "512EB", FileSize(size))
}

// TODO: Subtract()

func TestEllipsisString(t *testing.T) {
	assert.Equal(t, "...", EllipsisString("foobar", 0))
	assert.Equal(t, "...", EllipsisString("foobar", 1))
	assert.Equal(t, "...", EllipsisString("foobar", 2))
	assert.Equal(t, "...", EllipsisString("foobar", 3))
	assert.Equal(t, "f...", EllipsisString("foobar", 4))
	assert.Equal(t, "fo...", EllipsisString("foobar", 5))
	assert.Equal(t, "foobar", EllipsisString("foobar", 6))
	assert.Equal(t, "foobar", EllipsisString("foobar", 10))
}

func TestTruncateString(t *testing.T) {
	assert.Equal(t, "", TruncateString("foobar", 0))
	assert.Equal(t, "f", TruncateString("foobar", 1))
	assert.Equal(t, "fo", TruncateString("foobar", 2))
	assert.Equal(t, "foo", TruncateString("foobar", 3))
	assert.Equal(t, "foob", TruncateString("foobar", 4))
	assert.Equal(t, "fooba", TruncateString("foobar", 5))
	assert.Equal(t, "foobar", TruncateString("foobar", 6))
	assert.Equal(t, "foobar", TruncateString("foobar", 7))
}

func TestStringsToInt64s(t *testing.T) {
	assert.Equal(t, []int64{}, StringsToInt64s([]string{}))
	assert.Equal(t,
		[]int64{1, 4, 16, 64, 256},
		StringsToInt64s([]string{"1", "4", "16", "64", "256"}),
	)

	// TODO: StringsToInt64s should return ([]int64, error)
	assert.Equal(t, []int64{-1, 0, 0}, StringsToInt64s([]string{"-1", "a", "$"}))
}

func TestInt64sToStrings(t *testing.T) {
	assert.Equal(t, []string{}, Int64sToStrings([]int64{}))
	assert.Equal(t,
		[]string{"1", "4", "16", "64", "256"},
		Int64sToStrings([]int64{1, 4, 16, 64, 256}),
	)
}

func TestInt64sToMap(t *testing.T) {
	assert.Equal(t, map[int64]bool{}, Int64sToMap([]int64{}))
	assert.Equal(t,
		map[int64]bool{1: true, 4: true, 16: true},
		Int64sToMap([]int64{1, 4, 16}),
	)
}

func TestIsLetter(t *testing.T) {
	assert.True(t, IsLetter('a'))
	assert.True(t, IsLetter('e'))
	assert.True(t, IsLetter('q'))
	assert.True(t, IsLetter('z'))
	assert.True(t, IsLetter('A'))
	assert.True(t, IsLetter('E'))
	assert.True(t, IsLetter('Q'))
	assert.True(t, IsLetter('Z'))
	assert.True(t, IsLetter('_'))
	assert.False(t, IsLetter('-'))
	assert.False(t, IsLetter('1'))
	assert.False(t, IsLetter('$'))
}

// TODO: IsTextFile()
// TODO: IsImageFile()
// TODO: IsPDFFile()
<|endoftext|>


<commit_before><commit_msg>kola/tests: use FailFast for podman workflow test<commit_after><|endoftext|>


<commit_before>// +build windows

package host

import (
	"context"
	"fmt"
	"os"
	"runtime"
	"strings"
	"sync/atomic"
	"syscall"
	"time"
	"unsafe"

	"github.com/shirou/gopsutil/internal/common"
	process "github.com/shirou/gopsutil/process"
	"golang.org/x/sys/windows"
	"golang.org/x/sys/windows/registry"
)

var (
	procGetSystemTimeAsFileTime = common.Modkernel32.NewProc("GetSystemTimeAsFileTime")
	procGetTickCount32          = common.Modkernel32.NewProc("GetTickCount")
	procGetTickCount64          = common.Modkernel32.NewProc("GetTickCount64")
	procRtlGetVersion           = common.ModNt.NewProc("RtlGetVersion")
)

// https://docs.microsoft.com/en-us/windows-hardware/drivers/ddi/content/wdm/ns-wdm-_osversioninfoexw
type osVersionInfoExW struct {
	dwOSVersionInfoSize uint32
	dwMajorVersion      uint32
	dwMinorVersion      uint32
	dwBuildNumber       uint32
	dwPlatformId        uint32
	szCSDVersion        [128]uint16
	wServicePackMajor   uint16
	wServicePackMinor   uint16
	wSuiteMask          uint16
	wProductType        uint8
	wReserved           uint8
}

func Info() (*InfoStat, error) {
	return InfoWithContext(context.Background())
}

func InfoWithContext(ctx context.Context) (*InfoStat, error) {
	ret := &InfoStat{
		OS: runtime.GOOS,
	}

	{
		hostname, err := os.Hostname()
		if err == nil {
			ret.Hostname = hostname
		}
	}

	{
		platform, family, version, err := PlatformInformation()
		if err == nil {
			ret.Platform = platform
			ret.PlatformFamily = family
			ret.PlatformVersion = version
		} else {
			return ret, err
		}
	}

	{
		boot, err := BootTime()
		if err == nil {
			ret.BootTime = boot
			ret.Uptime, _ = Uptime()
		}
	}

	{
		hostID, err := getMachineGuid()
		if err == nil {
			ret.HostID = strings.ToLower(hostID)
		}
	}

	{
		procs, err := process.Pids()
		if err == nil {
			ret.Procs = uint64(len(procs))
		}
	}

	return ret, nil
}

func getMachineGuid() (string, error) {
	var h windows.Handle
	err := windows.RegOpenKeyEx(windows.HKEY_LOCAL_MACHINE, windows.StringToUTF16Ptr(`SOFTWARE\Microsoft\Cryptography`), 0, windows.KEY_READ|windows.KEY_WOW64_64KEY, &h)
	if err != nil {
		return "", err
	}
	defer windows.RegCloseKey(h)

	const windowsRegBufLen = 74 // len(`{`) + len(`abcdefgh-1234-456789012-123345456671` * 2) + len(`}`) // 2 == bytes/UTF16
	const uuidLen = 36

	var regBuf [windowsRegBufLen]uint16
	bufLen := uint32(windowsRegBufLen)
	var valType uint32
	err = windows.RegQueryValueEx(h, windows.StringToUTF16Ptr(`MachineGuid`), nil, &valType, (*byte)(unsafe.Pointer(&regBuf[0])), &bufLen)
	if err != nil {
		return "", err
	}

	hostID := windows.UTF16ToString(regBuf[:])
	hostIDLen := len(hostID)
	if hostIDLen != uuidLen {
		return "", fmt.Errorf("HostID incorrect: %q\n", hostID)
	}

	return hostID, nil
}

func Uptime() (uint64, error) {
	return UptimeWithContext(context.Background())
}

func UptimeWithContext(ctx context.Context) (uint64, error) {
	procGetTickCount := procGetTickCount64
	err := procGetTickCount64.Find()
	if err != nil {
		procGetTickCount = procGetTickCount32 // handle WinXP, but keep in mind that "the time will wrap around to zero if the system is run continuously for 49.7 days." from MSDN
	}
	r1, _, lastErr := syscall.Syscall(procGetTickCount.Addr(), 0, 0, 0, 0)
	if lastErr != 0 {
		return 0, lastErr
	}
	return uint64((time.Duration(r1) * time.Millisecond).Seconds()), nil
}

func bootTimeFromUptime(up uint64) uint64 {
	return uint64(time.Now().Unix()) - up
}

// cachedBootTime must be accessed via atomic.Load/StoreUint64
var cachedBootTime uint64

func BootTime() (uint64, error) {
	return BootTimeWithContext(context.Background())
}

func BootTimeWithContext(ctx context.Context) (uint64, error) {
	t := atomic.LoadUint64(&cachedBootTime)
	if t != 0 {
		return t, nil
	}
	up, err := Uptime()
	if err != nil {
		return 0, err
	}
	t = bootTimeFromUptime(up)
	atomic.StoreUint64(&cachedBootTime, t)
	return t, nil
}

func PlatformInformation() (platform string, family string, version string, err error) {
	return PlatformInformationWithContext(context.Background())
}

func PlatformInformationWithContext(ctx context.Context) (platform string, family string, version string, err error) {
	// GetVersionEx lies on Windows 8.1 and returns as Windows 8 if we don't declare compatibility in manifest
	// RtlGetVersion bypasses this lying layer and returns the true Windows version
	// https://docs.microsoft.com/en-us/windows-hardware/drivers/ddi/content/wdm/nf-wdm-rtlgetversion
	// https://docs.microsoft.com/en-us/windows-hardware/drivers/ddi/content/wdm/ns-wdm-_osversioninfoexw
	var osInfo osVersionInfoExW
	osInfo.dwOSVersionInfoSize = uint32(unsafe.Sizeof(osInfo))
	ret, _, err := procRtlGetVersion.Call(uintptr(unsafe.Pointer(&osInfo)))
	if ret != 0 {
		return
	}

	// Platform
	k, err := registry.OpenKey(registry.LOCAL_MACHINE, `SOFTWARE\Microsoft\Windows NT\CurrentVersion`, registry.QUERY_VALUE)
	if err != nil {
		return
	}
	defer k.Close()
	platform, _, err = k.GetStringValue("ProductName")
	if err != nil {
		return
	}
	if !strings.HasPrefix(platform, "Microsoft") {
		platform = "Microsoft " + platform
	}
	csd, _, err := k.GetStringValue("CSDVersion")
	if err == nil {
		platform += " " + csd
	}

	// PlatformFamily
	switch osInfo.wProductType {
	case 1:
		family = "Standalone Workstation"
	case 2:
		family = "Server (Domain Controller)"
	case 3:
		family = "Server"
	}

	// Platform Version
	version = fmt.Sprintf("%d.%d.%d Build %d", osInfo.dwMajorVersion, osInfo.dwMinorVersion, osInfo.dwBuildNumber, osInfo.dwBuildNumber)

	return
}

func Users() ([]UserStat, error) {
	return UsersWithContext(context.Background())
}

func UsersWithContext(ctx context.Context) ([]UserStat, error) {
	var ret []UserStat

	return ret, nil
}

func SensorsTemperatures() ([]TemperatureStat, error) {
	return SensorsTemperaturesWithContext(context.Background())
}

func SensorsTemperaturesWithContext(ctx context.Context) ([]TemperatureStat, error) {
	return []TemperatureStat{}, common.ErrNotImplementedError
}

func Virtualization() (string, string, error) {
	return VirtualizationWithContext(context.Background())
}

func VirtualizationWithContext(ctx context.Context) (string, string, error) {
	return "", "", common.ErrNotImplementedError
}

func KernelVersion() (string, error) {
	return KernelVersionWithContext(context.Background())
}

func KernelVersionWithContext(ctx context.Context) (string, error) {
	_, _, version, err := PlatformInformation()
	return version, err
}
<commit_msg>[host][windows] Query registry via golang.org/x/sys/windows instead of golang.org/x/sys/windows/registry<commit_after>// +build windows

package host

import (
	"context"
	"fmt"
	"os"
	"runtime"
	"strings"
	"sync/atomic"
	"syscall"
	"time"
	"unsafe"

	"github.com/shirou/gopsutil/internal/common"
	process "github.com/shirou/gopsutil/process"
	"golang.org/x/sys/windows"
)

var (
	procGetSystemTimeAsFileTime = common.Modkernel32.NewProc("GetSystemTimeAsFileTime")
	procGetTickCount32          = common.Modkernel32.NewProc("GetTickCount")
	procGetTickCount64          = common.Modkernel32.NewProc("GetTickCount64")
	procRtlGetVersion           = common.ModNt.NewProc("RtlGetVersion")
)

// https://docs.microsoft.com/en-us/windows-hardware/drivers/ddi/content/wdm/ns-wdm-_osversioninfoexw
type osVersionInfoExW struct {
	dwOSVersionInfoSize uint32
	dwMajorVersion      uint32
	dwMinorVersion      uint32
	dwBuildNumber       uint32
	dwPlatformId        uint32
	szCSDVersion        [128]uint16
	wServicePackMajor   uint16
	wServicePackMinor   uint16
	wSuiteMask          uint16
	wProductType        uint8
	wReserved           uint8
}

func Info() (*InfoStat, error) {
	return InfoWithContext(context.Background())
}

func InfoWithContext(ctx context.Context) (*InfoStat, error) {
	ret := &InfoStat{
		OS: runtime.GOOS,
	}

	{
		hostname, err := os.Hostname()
		if err == nil {
			ret.Hostname = hostname
		}
	}

	{
		platform, family, version, err := PlatformInformation()
		if err == nil {
			ret.Platform = platform
			ret.PlatformFamily = family
			ret.PlatformVersion = version
		} else {
			return ret, err
		}
	}

	{
		boot, err := BootTime()
		if err == nil {
			ret.BootTime = boot
			ret.Uptime, _ = Uptime()
		}
	}

	{
		hostID, err := getMachineGuid()
		if err == nil {
			ret.HostID = strings.ToLower(hostID)
		}
	}

	{
		procs, err := process.Pids()
		if err == nil {
			ret.Procs = uint64(len(procs))
		}
	}

	return ret, nil
}

func getMachineGuid() (string, error) {
	// there has been reports of issues on 32bit using golang.org/x/sys/windows/registry, see https://github.com/shirou/gopsutil/pull/312#issuecomment-277422612
	// for rationale of using windows.RegOpenKeyEx/RegQueryValueEx instead of registry.OpenKey/GetStringValue
	var h windows.Handle
	err := windows.RegOpenKeyEx(windows.HKEY_LOCAL_MACHINE, windows.StringToUTF16Ptr(`SOFTWARE\Microsoft\Cryptography`), 0, windows.KEY_READ|windows.KEY_WOW64_64KEY, &h)
	if err != nil {
		return "", err
	}
	defer windows.RegCloseKey(h)

	const windowsRegBufLen = 74 // len(`{`) + len(`abcdefgh-1234-456789012-123345456671` * 2) + len(`}`) // 2 == bytes/UTF16
	const uuidLen = 36

	var regBuf [windowsRegBufLen]uint16
	bufLen := uint32(windowsRegBufLen)
	var valType uint32
	err = windows.RegQueryValueEx(h, windows.StringToUTF16Ptr(`MachineGuid`), nil, &valType, (*byte)(unsafe.Pointer(&regBuf[0])), &bufLen)
	if err != nil {
		return "", err
	}

	hostID := windows.UTF16ToString(regBuf[:])
	hostIDLen := len(hostID)
	if hostIDLen != uuidLen {
		return "", fmt.Errorf("HostID incorrect: %q\n", hostID)
	}

	return hostID, nil
}

func Uptime() (uint64, error) {
	return UptimeWithContext(context.Background())
}

func UptimeWithContext(ctx context.Context) (uint64, error) {
	procGetTickCount := procGetTickCount64
	err := procGetTickCount64.Find()
	if err != nil {
		procGetTickCount = procGetTickCount32 // handle WinXP, but keep in mind that "the time will wrap around to zero if the system is run continuously for 49.7 days." from MSDN
	}
	r1, _, lastErr := syscall.Syscall(procGetTickCount.Addr(), 0, 0, 0, 0)
	if lastErr != 0 {
		return 0, lastErr
	}
	return uint64((time.Duration(r1) * time.Millisecond).Seconds()), nil
}

func bootTimeFromUptime(up uint64) uint64 {
	return uint64(time.Now().Unix()) - up
}

// cachedBootTime must be accessed via atomic.Load/StoreUint64
var cachedBootTime uint64

func BootTime() (uint64, error) {
	return BootTimeWithContext(context.Background())
}

func BootTimeWithContext(ctx context.Context) (uint64, error) {
	t := atomic.LoadUint64(&cachedBootTime)
	if t != 0 {
		return t, nil
	}
	up, err := Uptime()
	if err != nil {
		return 0, err
	}
	t = bootTimeFromUptime(up)
	atomic.StoreUint64(&cachedBootTime, t)
	return t, nil
}

func PlatformInformation() (platform string, family string, version string, err error) {
	return PlatformInformationWithContext(context.Background())
}

func PlatformInformationWithContext(ctx context.Context) (platform string, family string, version string, err error) {
	// GetVersionEx lies on Windows 8.1 and returns as Windows 8 if we don't declare compatibility in manifest
	// RtlGetVersion bypasses this lying layer and returns the true Windows version
	// https://docs.microsoft.com/en-us/windows-hardware/drivers/ddi/content/wdm/nf-wdm-rtlgetversion
	// https://docs.microsoft.com/en-us/windows-hardware/drivers/ddi/content/wdm/ns-wdm-_osversioninfoexw
	var osInfo osVersionInfoExW
	osInfo.dwOSVersionInfoSize = uint32(unsafe.Sizeof(osInfo))
	ret, _, err := procRtlGetVersion.Call(uintptr(unsafe.Pointer(&osInfo)))
	if ret != 0 {
		return
	}

	// Platform
	var h windows.Handle // like getMachineGuid(), we query the registry using the raw windows.RegOpenKeyEx/RegQueryValueEx
	err = windows.RegOpenKeyEx(windows.HKEY_LOCAL_MACHINE, windows.StringToUTF16Ptr(`SOFTWARE\Microsoft\Windows NT\CurrentVersion`), 0, windows.KEY_READ|windows.KEY_WOW64_64KEY, &h)
	if err != nil {
		return
	}
	defer windows.RegCloseKey(h)
	var bufLen uint32
	var valType uint32
	err = windows.RegQueryValueEx(h, windows.StringToUTF16Ptr(`ProductName`), nil, &valType, nil, &bufLen)
	if err != nil {
		return
	}
	regBuf := make([]uint16, bufLen/2+1)
	err = windows.RegQueryValueEx(h, windows.StringToUTF16Ptr(`ProductName`), nil, &valType, (*byte)(unsafe.Pointer(&regBuf[0])), &bufLen)
	if err != nil {
		return
	}
	platform = windows.UTF16ToString(regBuf[:])
	if !strings.HasPrefix(platform, "Microsoft") {
		platform = "Microsoft " + platform
	}
	err = windows.RegQueryValueEx(h, windows.StringToUTF16Ptr(`CSDVersion`), nil, &valType, nil, &bufLen) // append Service Pack number, only on success
	if err == nil {                                                                                // don't return an error if only the Service Pack retrieval fails
		regBuf = make([]uint16, bufLen/2+1)
		err = windows.RegQueryValueEx(h, windows.StringToUTF16Ptr(`CSDVersion`), nil, &valType, (*byte)(unsafe.Pointer(&regBuf[0])), &bufLen)
		if err == nil {
			platform += " " + windows.UTF16ToString(regBuf[:])
		}
	}

	// PlatformFamily
	switch osInfo.wProductType {
	case 1:
		family = "Standalone Workstation"
	case 2:
		family = "Server (Domain Controller)"
	case 3:
		family = "Server"
	}

	// Platform Version
	version = fmt.Sprintf("%d.%d.%d Build %d", osInfo.dwMajorVersion, osInfo.dwMinorVersion, osInfo.dwBuildNumber, osInfo.dwBuildNumber)

	return platform, family, version, nil
}

func Users() ([]UserStat, error) {
	return UsersWithContext(context.Background())
}

func UsersWithContext(ctx context.Context) ([]UserStat, error) {
	var ret []UserStat

	return ret, nil
}

func SensorsTemperatures() ([]TemperatureStat, error) {
	return SensorsTemperaturesWithContext(context.Background())
}

func SensorsTemperaturesWithContext(ctx context.Context) ([]TemperatureStat, error) {
	return []TemperatureStat{}, common.ErrNotImplementedError
}

func Virtualization() (string, string, error) {
	return VirtualizationWithContext(context.Background())
}

func VirtualizationWithContext(ctx context.Context) (string, string, error) {
	return "", "", common.ErrNotImplementedError
}

func KernelVersion() (string, error) {
	return KernelVersionWithContext(context.Background())
}

func KernelVersionWithContext(ctx context.Context) (string, error) {
	_, _, version, err := PlatformInformation()
	return version, err
}
<|endoftext|>


<commit_before>package regression

import (
	"context"
	"fmt"
	"math"

	"go.opencensus.io/trace"
	"go.skia.org/infra/go/paramtools"
	"go.skia.org/infra/go/skerr"
	"go.skia.org/infra/go/sklog"
	"go.skia.org/infra/go/vec32"
	"go.skia.org/infra/perf/go/alerts"
	"go.skia.org/infra/perf/go/clustering2"
	"go.skia.org/infra/perf/go/config"
	"go.skia.org/infra/perf/go/dataframe"
	"go.skia.org/infra/perf/go/dfiter"
	perfgit "go.skia.org/infra/perf/go/git"
	"go.skia.org/infra/perf/go/progress"
	"go.skia.org/infra/perf/go/shortcut"
	"go.skia.org/infra/perf/go/types"
)

// ProcessState is the state of a RegressionDetectionProcess.
type ProcessState string

const (
	// ProcessRunning means the process is still running.
	ProcessRunning ProcessState = "Running"

	// ProcessSuccess means the process has finished successfully.
	ProcessSuccess ProcessState = "Success"

	// ProcessError means the process has ended on an error.
	ProcessError ProcessState = "Error"
)

// AllProcessState is a list of all ProcessState possible values.
var AllProcessState = []ProcessState{ProcessRunning, ProcessSuccess, ProcessError}

const (
	// The following limits are just to prevent excessively large or long-running
	// regression detections from being triggered.

	// maxK is the largest K used for clustering.
	maxK = 100
)

// DetectorResponseProcessor is a callback that is called with RegressionDetectionResponses as a RegressionDetectionRequest is being processed.
type DetectorResponseProcessor func(*RegressionDetectionRequest, []*RegressionDetectionResponse, string)

// ParamsetProvider is a function that's called to return the current paramset.
type ParamsetProvider func() paramtools.ParamSet

// RegressionDetectionRequest is all the info needed to start a clustering run,
// an Alert and the Domain over which to run that Alert.
type RegressionDetectionRequest struct {
	Alert  *alerts.Alert `json:"alert"`
	Domain types.Domain  `json:"domain"`

	// Query is the exact query being run. It may be more specific than the one
	// in the Alert if the Alert has a non-empty GroupBy.
	Query string `json:"query"`

	// Step/TotalQueries is the current percent of all the queries that have been processed.
	Step int `json:"step"`

	// TotalQueries is the number of sub-queries to be processed based on the
	// GroupBy setting in the Alert.
	TotalQueries int `json:"total_queries"`

	// Progress of the detection request.
	Progress progress.Progress `json:"-"`
}

// NewRegressionDetectionRequest returns a new RegressionDetectionRequest.
func NewRegressionDetectionRequest() *RegressionDetectionRequest {
	return &RegressionDetectionRequest{
		Progress: progress.New(),
	}
}

// RegressionDetectionResponse is the response from running a RegressionDetectionRequest.
type RegressionDetectionResponse struct {
	Summary *clustering2.ClusterSummaries `json:"summary"`
	Frame   *dataframe.FrameResponse      `json:"frame"`
}

// regressionDetectionProcess handles the processing of a single RegressionDetectionRequest.
type regressionDetectionProcess struct {
	// These members are read-only, should not be modified.
	request                   *RegressionDetectionRequest
	perfGit                   *perfgit.Git
	iter                      dfiter.DataFrameIterator
	detectorResponseProcessor DetectorResponseProcessor
	shortcutStore             shortcut.Store
}

// NewRunningProcess starts a regression detection process.
func NewRunningProcess(ctx context.Context,
	req *RegressionDetectionRequest,
	detectorResponseProcessor DetectorResponseProcessor,
	perfGit *perfgit.Git,
	shortcutStore shortcut.Store,
	dfBuilder dataframe.DataFrameBuilder,
) {
	req.Progress.Message("Stage", "Loading data to analyze")
	// Create a single large dataframe then chop it into 2*radius+1 length sub-dataframes in the iterator.
	iter, err := dfiter.NewDataFrameIterator(ctx, nil, dfBuilder, perfGit, nil, req.Query, req.Domain, req.Alert)
	if err != nil {
		req.Progress.Error(fmt.Sprintf("Failed to create iterator: %s", err))
	}
	ret := &regressionDetectionProcess{
		request:                   req,
		perfGit:                   perfGit,
		detectorResponseProcessor: detectorResponseProcessor,
		shortcutStore:             shortcutStore,
		iter:                      iter,
	}
	ret.iter = iter
	go ret.run(ctx)
}

// reportError records the reason a RegressionDetectionProcess failed.
func (p *regressionDetectionProcess) reportError(err error, message string) {
	sklog.Warningf("RegressionDetectionRequest failed: %#v %s: %s", *(p.request), message, err)
	p.request.Progress.Error(fmt.Sprintf("%s: %s", message, err))
}

// progress records the progress of a RegressionDetectionProcess.
func (p *regressionDetectionProcess) progress(step, totalSteps int) {
	p.request.Progress.Message("Querying", fmt.Sprintf("%d%%", int(float32(100.0)*float32(step)/float32(totalSteps))))
}

// detectionProgress records the progress of a RegressionDetectionProcess.
func (p *regressionDetectionProcess) detectionProgress(totalError float64) {
	p.request.Progress.Message("Regression Total Error", fmt.Sprintf("%0.2f", totalError))
}

// missing returns true if >50% of the trace is vec32.MISSING_DATA_SENTINEL.
func missing(tr types.Trace) bool {
	count := 0
	for _, x := range tr {
		if x == vec32.MissingDataSentinel {
			count++
		}
	}
	return (100*count)/len(tr) > 50
}

// tooMuchMissingData returns true if a trace has too many
// MISSING_DATA_SENTINEL values.
//
// The criteria is if there is >50% missing data on either side of the target
// commit, which sits at the center of the trace.
func tooMuchMissingData(tr types.Trace) bool {
	if len(tr) < 3 {
		return false
	}
	n := len(tr) / 2
	if tr[n] == vec32.MissingDataSentinel {
		return true
	}
	return missing(tr[:n]) || missing(tr[len(tr)-n:])
}

// shortcutFromKeys stores a new shortcut for each regression based on its Keys.
func (p *regressionDetectionProcess) shortcutFromKeys(summary *clustering2.ClusterSummaries) error {
	var err error
	for _, cs := range summary.Clusters {
		if cs.Shortcut, err = p.shortcutStore.InsertShortcut(context.Background(), &shortcut.Shortcut{Keys: cs.Keys}); err != nil {
			return err
		}
	}
	return nil
}

// run does the work in a RegressionDetectionProcess. It does not return until all the
// work is done or the request failed. Should be run as a Go routine.
func (p *regressionDetectionProcess) run(ctx context.Context) {
	ctx, span := trace.StartSpan(ctx, "regressionDetectionProcess.run")
	defer span.End()

	if p.request.Alert.Algo == "" {
		p.request.Alert.Algo = types.KMeansGrouping
	}
	for p.iter.Next() {
		df, err := p.iter.Value(ctx)
		if err != nil {
			p.reportError(err, "Failed to get DataFrame from DataFrameIterator.")
			return
		}
		sklog.Infof("Next dataframe: %d traces", len(df.TraceSet))
		before := len(df.TraceSet)
		// Filter out Traces with insufficient data. I.e. we need 50% or more data
		// on either side of the target commit.
		df.FilterOut(tooMuchMissingData)
		after := len(df.TraceSet)
		message := fmt.Sprintf("Filtered Traces: Num Before: %d Num After: %d Delta: %d", before, after, before-after)
		sklog.Info(message)

		k := p.request.Alert.K
		if k <= 0 || k > maxK {
			n := len(df.TraceSet)
			// We want K to be around 50 when n = 30000, which has been determined via
			// trial and error to be a good value for the Perf data we are working in. We
			// want K to decrease from  there as n gets smaller, but don't want K to go
			// below 10, so we use a simple linear relation:
			//
			//  k = 40/30000 * n + 10
			//
			k = int(math.Floor((40.0/30000.0)*float64(n) + 10))
		}
		sklog.Infof("Clustering with K=%d", k)

		var summary *clustering2.ClusterSummaries
		switch p.request.Alert.Algo {
		case types.KMeansGrouping:
			summary, err = clustering2.CalculateClusterSummaries(df, k, config.MinStdDev, p.detectionProgress, p.request.Alert.Interesting, p.request.Alert.Step)
		case types.StepFitGrouping:
			summary, err = StepFit(df, k, config.MinStdDev, p.detectionProgress, p.request.Alert.Interesting, p.request.Alert.Step)

		default:
			p.reportError(skerr.Fmt("Invalid type of clustering: %s", p.request.Alert.Algo), "Invalid type of clustering.")
		}
		if err != nil {
			p.reportError(err, "Invalid regression detection.")
			return
		}
		if err := p.shortcutFromKeys(summary); err != nil {
			p.reportError(err, "Failed to write shortcut for keys.")
			return
		}

		df.TraceSet = types.TraceSet{}
		frame, err := dataframe.ResponseFromDataFrame(ctx, df, p.perfGit, false, p.request.Progress)
		if err != nil {
			p.reportError(err, "Failed to convert DataFrame to FrameResponse.")
			return
		}

		cr := &RegressionDetectionResponse{
			Summary: summary,
			Frame:   frame,
		}
		p.detectorResponseProcessor(p.request, []*RegressionDetectionResponse{cr}, message)
	}
	// We Finish the process, but record Results. The detectorResponseProcessor
	// callback should add the results to Progress if that's required.
	p.request.Progress.Finished()
}
<commit_msg>[perf] Return error on unknown algorithm<commit_after>package regression

import (
	"context"
	"fmt"
	"math"

	"go.opencensus.io/trace"
	"go.skia.org/infra/go/paramtools"
	"go.skia.org/infra/go/skerr"
	"go.skia.org/infra/go/sklog"
	"go.skia.org/infra/go/vec32"
	"go.skia.org/infra/perf/go/alerts"
	"go.skia.org/infra/perf/go/clustering2"
	"go.skia.org/infra/perf/go/config"
	"go.skia.org/infra/perf/go/dataframe"
	"go.skia.org/infra/perf/go/dfiter"
	perfgit "go.skia.org/infra/perf/go/git"
	"go.skia.org/infra/perf/go/progress"
	"go.skia.org/infra/perf/go/shortcut"
	"go.skia.org/infra/perf/go/types"
)

// ProcessState is the state of a RegressionDetectionProcess.
type ProcessState string

const (
	// ProcessRunning means the process is still running.
	ProcessRunning ProcessState = "Running"

	// ProcessSuccess means the process has finished successfully.
	ProcessSuccess ProcessState = "Success"

	// ProcessError means the process has ended on an error.
	ProcessError ProcessState = "Error"
)

// AllProcessState is a list of all ProcessState possible values.
var AllProcessState = []ProcessState{ProcessRunning, ProcessSuccess, ProcessError}

const (
	// The following limits are just to prevent excessively large or long-running
	// regression detections from being triggered.

	// maxK is the largest K used for clustering.
	maxK = 100
)

// DetectorResponseProcessor is a callback that is called with RegressionDetectionResponses as a RegressionDetectionRequest is being processed.
type DetectorResponseProcessor func(*RegressionDetectionRequest, []*RegressionDetectionResponse, string)

// ParamsetProvider is a function that's called to return the current paramset.
type ParamsetProvider func() paramtools.ParamSet

// RegressionDetectionRequest is all the info needed to start a clustering run,
// an Alert and the Domain over which to run that Alert.
type RegressionDetectionRequest struct {
	Alert  *alerts.Alert `json:"alert"`
	Domain types.Domain  `json:"domain"`

	// Query is the exact query being run. It may be more specific than the one
	// in the Alert if the Alert has a non-empty GroupBy.
	Query string `json:"query"`

	// Step/TotalQueries is the current percent of all the queries that have been processed.
	Step int `json:"step"`

	// TotalQueries is the number of sub-queries to be processed based on the
	// GroupBy setting in the Alert.
	TotalQueries int `json:"total_queries"`

	// Progress of the detection request.
	Progress progress.Progress `json:"-"`
}

// NewRegressionDetectionRequest returns a new RegressionDetectionRequest.
func NewRegressionDetectionRequest() *RegressionDetectionRequest {
	return &RegressionDetectionRequest{
		Progress: progress.New(),
	}
}

// RegressionDetectionResponse is the response from running a RegressionDetectionRequest.
type RegressionDetectionResponse struct {
	Summary *clustering2.ClusterSummaries `json:"summary"`
	Frame   *dataframe.FrameResponse      `json:"frame"`
}

// regressionDetectionProcess handles the processing of a single RegressionDetectionRequest.
type regressionDetectionProcess struct {
	// These members are read-only, should not be modified.
	request                   *RegressionDetectionRequest
	perfGit                   *perfgit.Git
	iter                      dfiter.DataFrameIterator
	detectorResponseProcessor DetectorResponseProcessor
	shortcutStore             shortcut.Store
}

// NewRunningProcess starts a regression detection process.
func NewRunningProcess(ctx context.Context,
	req *RegressionDetectionRequest,
	detectorResponseProcessor DetectorResponseProcessor,
	perfGit *perfgit.Git,
	shortcutStore shortcut.Store,
	dfBuilder dataframe.DataFrameBuilder,
) {
	req.Progress.Message("Stage", "Loading data to analyze")
	// Create a single large dataframe then chop it into 2*radius+1 length sub-dataframes in the iterator.
	iter, err := dfiter.NewDataFrameIterator(ctx, nil, dfBuilder, perfGit, nil, req.Query, req.Domain, req.Alert)
	if err != nil {
		req.Progress.Error(fmt.Sprintf("Failed to create iterator: %s", err))
	}
	ret := &regressionDetectionProcess{
		request:                   req,
		perfGit:                   perfGit,
		detectorResponseProcessor: detectorResponseProcessor,
		shortcutStore:             shortcutStore,
		iter:                      iter,
	}
	ret.iter = iter
	go ret.run(ctx)
}

// reportError records the reason a RegressionDetectionProcess failed.
func (p *regressionDetectionProcess) reportError(err error, message string) {
	sklog.Warningf("RegressionDetectionRequest failed: %#v %s: %s", *(p.request), message, err)
	p.request.Progress.Error(fmt.Sprintf("%s: %s", message, err))
}

// progress records the progress of a RegressionDetectionProcess.
func (p *regressionDetectionProcess) progress(step, totalSteps int) {
	p.request.Progress.Message("Querying", fmt.Sprintf("%d%%", int(float32(100.0)*float32(step)/float32(totalSteps))))
}

// detectionProgress records the progress of a RegressionDetectionProcess.
func (p *regressionDetectionProcess) detectionProgress(totalError float64) {
	p.request.Progress.Message("Regression Total Error", fmt.Sprintf("%0.2f", totalError))
}

// missing returns true if >50% of the trace is vec32.MISSING_DATA_SENTINEL.
func missing(tr types.Trace) bool {
	count := 0
	for _, x := range tr {
		if x == vec32.MissingDataSentinel {
			count++
		}
	}
	return (100*count)/len(tr) > 50
}

// tooMuchMissingData returns true if a trace has too many
// MISSING_DATA_SENTINEL values.
//
// The criteria is if there is >50% missing data on either side of the target
// commit, which sits at the center of the trace.
func tooMuchMissingData(tr types.Trace) bool {
	if len(tr) < 3 {
		return false
	}
	n := len(tr) / 2
	if tr[n] == vec32.MissingDataSentinel {
		return true
	}
	return missing(tr[:n]) || missing(tr[len(tr)-n:])
}

// shortcutFromKeys stores a new shortcut for each regression based on its Keys.
func (p *regressionDetectionProcess) shortcutFromKeys(summary *clustering2.ClusterSummaries) error {
	var err error
	for _, cs := range summary.Clusters {
		if cs.Shortcut, err = p.shortcutStore.InsertShortcut(context.Background(), &shortcut.Shortcut{Keys: cs.Keys}); err != nil {
			return err
		}
	}
	return nil
}

// run does the work in a RegressionDetectionProcess. It does not return until all the
// work is done or the request failed. Should be run as a Go routine.
func (p *regressionDetectionProcess) run(ctx context.Context) {
	ctx, span := trace.StartSpan(ctx, "regressionDetectionProcess.run")
	defer span.End()

	if p.request.Alert.Algo == "" {
		p.request.Alert.Algo = types.KMeansGrouping
	}
	for p.iter.Next() {
		df, err := p.iter.Value(ctx)
		if err != nil {
			p.reportError(err, "Failed to get DataFrame from DataFrameIterator.")
			return
		}
		sklog.Infof("Next dataframe: %d traces", len(df.TraceSet))
		before := len(df.TraceSet)
		// Filter out Traces with insufficient data. I.e. we need 50% or more data
		// on either side of the target commit.
		df.FilterOut(tooMuchMissingData)
		after := len(df.TraceSet)
		message := fmt.Sprintf("Filtered Traces: Num Before: %d Num After: %d Delta: %d", before, after, before-after)
		sklog.Info(message)

		k := p.request.Alert.K
		if k <= 0 || k > maxK {
			n := len(df.TraceSet)
			// We want K to be around 50 when n = 30000, which has been determined via
			// trial and error to be a good value for the Perf data we are working in. We
			// want K to decrease from  there as n gets smaller, but don't want K to go
			// below 10, so we use a simple linear relation:
			//
			//  k = 40/30000 * n + 10
			//
			k = int(math.Floor((40.0/30000.0)*float64(n) + 10))
		}
		sklog.Infof("Clustering with K=%d", k)

		var summary *clustering2.ClusterSummaries
		switch p.request.Alert.Algo {
		case types.KMeansGrouping:
			summary, err = clustering2.CalculateClusterSummaries(df, k, config.MinStdDev, p.detectionProgress, p.request.Alert.Interesting, p.request.Alert.Step)
		case types.StepFitGrouping:
			summary, err = StepFit(df, k, config.MinStdDev, p.detectionProgress, p.request.Alert.Interesting, p.request.Alert.Step)
		default:
			err = skerr.Fmt("Invalid type of clustering: %s", p.request.Alert.Algo)
		}
		if err != nil {
			p.reportError(err, "Invalid regression detection.")
			return
		}
		if err := p.shortcutFromKeys(summary); err != nil {
			p.reportError(err, "Failed to write shortcut for keys.")
			return
		}

		df.TraceSet = types.TraceSet{}
		frame, err := dataframe.ResponseFromDataFrame(ctx, df, p.perfGit, false, p.request.Progress)
		if err != nil {
			p.reportError(err, "Failed to convert DataFrame to FrameResponse.")
			return
		}

		cr := &RegressionDetectionResponse{
			Summary: summary,
			Frame:   frame,
		}
		p.detectorResponseProcessor(p.request, []*RegressionDetectionResponse{cr}, message)
	}
	// We Finish the process, but record Results. The detectorResponseProcessor
	// callback should add the results to Progress if that's required.
	p.request.Progress.Finished()
}
<|endoftext|>


<commit_before>package typhon

import (
	"encoding/json"
	"errors"
	"net/http"
	"strings"

	log "github.com/mondough/slog"
	"github.com/mondough/terrors"
	"github.com/mondough/terrors/proto"
)

var (
	mapTerr2Status = map[string]int{
		terrors.ErrBadRequest:      http.StatusBadRequest,
		terrors.ErrUnauthorized:    http.StatusUnauthorized,
		terrors.ErrForbidden:       http.StatusForbidden,
		terrors.ErrNotFound:        http.StatusNotFound,
		terrors.ErrBadResponse:     http.StatusNotAcceptable,
		terrors.ErrTimeout:         http.StatusGatewayTimeout,
		terrors.ErrInternalService: http.StatusInternalServerError}
	mapStatus2Terr map[int]string
)

func init() {
	mapStatus2Terr = make(map[int]string, len(mapTerr2Status))
	for k, v := range mapTerr2Status {
		mapStatus2Terr[v] = k
	}
}

// ErrorStatusCode returns an HTTP status code for the error
func ErrorStatusCode(err error) int {
	code := terrors.Wrap(err, nil).(*terrors.Error).Code
	if c, ok := mapTerr2Status[strings.SplitN(code, ".", 2)[0]]; ok {
		return c
	}
	return http.StatusInternalServerError
}

// terr2StatusCode converts HTTP status codes to a roughly equivalent terrors' code
func status2TerrCode(code int) string {
	if c, ok := mapStatus2Terr[code]; ok {
		return c
	}
	return terrors.ErrInternalService
}

// ErrorFilter serialises and de-serialises response errors
func ErrorFilter(req Request, svc Service) Response {
	rsp := svc(req)

	if rsp.Response == nil {
		rsp.Response = newHttpResponse(req)
	}
	if rsp.ctx == nil {
		rsp.ctx = req
	}

	if rsp.Error != nil {
		// Serialise the error into the response
		terr := terrors.Wrap(rsp.Error, nil).(*terrors.Error)
		rsp.Encode(terrors.Marshal(terr))
		rsp.StatusCode = ErrorStatusCode(terr)
		rsp.Header.Set("Terror", "1")
	} else {
		if rsp.StatusCode >= 400 && rsp.StatusCode <= 599 {
			b, _ := rsp.BodyBytes(false)
			switch rsp.Header.Get("Terror") {
			case "1":
				tp := &terrorsproto.Error{}
				if err := json.Unmarshal(b, tp); err != nil {
					log.Warn(rsp.ctx, "Failed to unmarshal terror: %v", err)
					rsp.Error = errors.New(string(b))
				} else {
					rsp.Error = terrors.Unmarshal(tp)
				}

			default:
				rsp.Error = errors.New(string(b))
			}
		}
	}

	return rsp
}
<commit_msg>Error for empty body errors<commit_after>package typhon

import (
	"encoding/json"
	"errors"
	"net/http"
	"strings"

	log "github.com/mondough/slog"
	"github.com/mondough/terrors"
	"github.com/mondough/terrors/proto"
)

var (
	mapTerr2Status = map[string]int{
		terrors.ErrBadRequest:      http.StatusBadRequest,
		terrors.ErrUnauthorized:    http.StatusUnauthorized,
		terrors.ErrForbidden:       http.StatusForbidden,
		terrors.ErrNotFound:        http.StatusNotFound,
		terrors.ErrBadResponse:     http.StatusNotAcceptable,
		terrors.ErrTimeout:         http.StatusGatewayTimeout,
		terrors.ErrInternalService: http.StatusInternalServerError}
	mapStatus2Terr map[int]string
)

func init() {
	mapStatus2Terr = make(map[int]string, len(mapTerr2Status))
	for k, v := range mapTerr2Status {
		mapStatus2Terr[v] = k
	}
}

// ErrorStatusCode returns an HTTP status code for the error
func ErrorStatusCode(err error) int {
	code := terrors.Wrap(err, nil).(*terrors.Error).Code
	if c, ok := mapTerr2Status[strings.SplitN(code, ".", 2)[0]]; ok {
		return c
	}
	return http.StatusInternalServerError
}

// terr2StatusCode converts HTTP status codes to a roughly equivalent terrors' code
func status2TerrCode(code int) string {
	if c, ok := mapStatus2Terr[code]; ok {
		return c
	}
	return terrors.ErrInternalService
}

// ErrorFilter serialises and de-serialises response errors
func ErrorFilter(req Request, svc Service) Response {
	rsp := svc(req)

	if rsp.Response == nil {
		rsp.Response = newHttpResponse(req)
	}
	if rsp.ctx == nil {
		rsp.ctx = req
	}

	if rsp.Error != nil {
		// Serialise the error into the response
		terr := terrors.Wrap(rsp.Error, nil).(*terrors.Error)
		rsp.Encode(terrors.Marshal(terr))
		rsp.StatusCode = ErrorStatusCode(terr)
		rsp.Header.Set("Terror", "1")
	} else {
		if rsp.StatusCode >= 400 && rsp.StatusCode <= 599 {
			b, _ := rsp.BodyBytes(false)
			switch rsp.Header.Get("Terror") {
			case "1":
				tp := &terrorsproto.Error{}
				if err := json.Unmarshal(b, tp); err != nil {
					log.Warn(rsp.ctx, "Failed to unmarshal terror: %v", err)
					rsp.Error = errors.New(string(b))
				} else {
					rsp.Error = terrors.Unmarshal(tp)
				}

			default:
				rsp.Error = errors.New(string(b))
			}
		}
	}
	if rsp.Error != nil && rsp.Error.Error() == "" {
		rsp.Error = errors.New("Response error (empty body)")
	}

	return rsp
}
<|endoftext|>


<commit_before>// Copyright 2017 The etcd-operator Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package writer

import (
	"context"
	"fmt"
	"io"

	"github.com/coreos/etcd-operator/pkg/backup/util"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/service/s3"
	"github.com/aws/aws-sdk-go/service/s3/s3manager"
)

type s3Writer struct {
	s3 *s3.S3
}

// NewS3Writer creates a s3 writer.
func NewS3Writer(s3 *s3.S3) Writer {
	return &s3Writer{s3}
}

// Write writes the backup file to the given s3 path, "<s3-bucket-name>/<key>".
func (s3w *s3Writer) Write(ctx context.Context, path string, r io.Reader) (int64, error) {
	bk, key, err := util.ParseBucketAndKey(path)
	if err != nil {
		return 0, err
	}

	_, err = s3manager.NewUploaderWithClient(s3w.s3).UploadWithContext(ctx,
		&s3manager.UploadInput{
			Bucket: aws.String(bk),
			Key:    aws.String(key),
			Body:   r,
		})
	if err != nil {
		return 0, err
	}

	resp, err := s3w.s3.GetObject(&s3.GetObjectInput{
		Bucket: aws.String(bk),
		Key:    aws.String(key),
	})
	if err != nil {
		return 0, err
	}
	if resp.ContentLength == nil {
		return 0, fmt.Errorf("failed to compute s3 object size")
	}
	return *resp.ContentLength, nil
}
<commit_msg>backup-operator: close s3 reader after saving snapshot<commit_after>// Copyright 2017 The etcd-operator Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package writer

import (
	"context"
	"fmt"
	"io"

	"github.com/coreos/etcd-operator/pkg/backup/util"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/service/s3"
	"github.com/aws/aws-sdk-go/service/s3/s3manager"
)

type s3Writer struct {
	s3 *s3.S3
}

// NewS3Writer creates a s3 writer.
func NewS3Writer(s3 *s3.S3) Writer {
	return &s3Writer{s3}
}

// Write writes the backup file to the given s3 path, "<s3-bucket-name>/<key>".
func (s3w *s3Writer) Write(ctx context.Context, path string, r io.Reader) (int64, error) {
	bk, key, err := util.ParseBucketAndKey(path)
	if err != nil {
		return 0, err
	}

	_, err = s3manager.NewUploaderWithClient(s3w.s3).UploadWithContext(ctx,
		&s3manager.UploadInput{
			Bucket: aws.String(bk),
			Key:    aws.String(key),
			Body:   r,
		})
	if err != nil {
		return 0, err
	}

	resp, err := s3w.s3.GetObject(&s3.GetObjectInput{
		Bucket: aws.String(bk),
		Key:    aws.String(key),
	})
	if err != nil {
		return 0, err
	}
	if resp.Body != nil {
		defer resp.Body.Close()
	}
	if resp.ContentLength == nil {
		return 0, fmt.Errorf("failed to compute s3 object size")
	}
	return *resp.ContentLength, nil
}
<|endoftext|>


<commit_before>// Copyright 2017-2018 Authors of Cilium
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package loader

import (
	"context"
	"fmt"

	"github.com/cilium/cilium/pkg/bpf"
	"github.com/cilium/cilium/pkg/command/exec"

	"github.com/vishvananda/netlink"
)

const (
	libbpfFixupMsg = "struct bpf_elf_map fixup performed due to size mismatch!"
)

func replaceQdisc(ifName string) error {
	link, err := netlink.LinkByName(ifName)
	if err != nil {
		return err
	}
	attrs := netlink.QdiscAttrs{
		LinkIndex: link.Attrs().Index,
		Handle:    netlink.MakeHandle(0xffff, 0),
		Parent:    netlink.HANDLE_CLSACT,
	}

	qdisc := &netlink.GenericQdisc{
		QdiscAttrs: attrs,
		QdiscType:  "clsact",
	}

	if err = netlink.QdiscReplace(qdisc); err != nil {
		return fmt.Errorf("netlink: Replacing qdisc for %s failed: %s", ifName, err)
	} else {
		log.Debugf("netlink: Replacing qdisc for %s succeeded", ifName)
	}

	return nil
}

// replaceDatapath the qdisc and BPF program for a endpoint
func (l *Loader) replaceDatapath(ctx context.Context, ifName, objPath, progSec, progDirection string) error {
	err := replaceQdisc(ifName)
	if err != nil {
		return fmt.Errorf("Failed to replace Qdisc for %s: %s", ifName, err)
	}

	// FIXME: Replace cilium-map-migrate with Golang map migration
	cmd := exec.CommandContext(ctx, "cilium-map-migrate", "-s", objPath)
	cmd.Env = bpf.Environment()
	if _, err = cmd.CombinedOutput(log, true); err != nil {
		return err
	}
	defer func() {
		var retCode string
		if err == nil {
			retCode = "0"
		} else {
			retCode = "1"
		}
		args := []string{"-e", objPath, "-r", retCode}
		cmd := exec.CommandContext(ctx, "cilium-map-migrate", args...)
		cmd.Env = bpf.Environment()
		_, _ = cmd.CombinedOutput(log, true) // ignore errors
	}()

	// FIXME: replace exec with native call
	args := []string{"filter", "replace", "dev", ifName, progDirection,
		"prio", "1", "handle", "1", "bpf", "da", "obj", objPath,
		"sec", progSec,
	}
	cmd = exec.CommandContext(ctx, "tc", args...).WithFilters(libbpfFixupMsg)
	_, err = cmd.CombinedOutput(log, true)
	if err != nil {
		return fmt.Errorf("Failed to load tc filter: %s", err)
	}

	return nil
}

// graftDatapath replaces obj in tail call map
func graftDatapath(ctx context.Context, mapPath, objPath, progSec string) error {
	var err error

	// FIXME: Replace cilium-map-migrate with Golang map migration
	cmd := exec.CommandContext(ctx, "cilium-map-migrate", "-s", objPath)
	cmd.Env = bpf.Environment()
	if _, err = cmd.CombinedOutput(log, true); err != nil {
		return err
	}
	defer func() {
		var retCode string
		if err == nil {
			retCode = "0"
		} else {
			retCode = "1"
		}
		args := []string{"-e", objPath, "-r", retCode}
		cmd := exec.CommandContext(ctx, "cilium-map-migrate", args...)
		cmd.Env = bpf.Environment()
		_, _ = cmd.CombinedOutput(log, true) // ignore errors
	}()

	// FIXME: replace exec with native call
	// FIXME: only key 0 right now, could be made more flexible
	args := []string{"exec", "bpf", "graft", mapPath, "key", "0",
		"obj", objPath, "sec", progSec,
	}
	cmd = exec.CommandContext(ctx, "tc", args...).WithFilters(libbpfFixupMsg)
	_, err = cmd.CombinedOutput(log, true)
	if err != nil {
		return fmt.Errorf("Failed to graft tc object: %s", err)
	}

	return nil
}

// DeleteDatapath filter from the given ifName
func (l *Loader) DeleteDatapath(ctx context.Context, ifName, direction string) error {
	args := []string{"filter", "delete", "dev", ifName, direction, "pref", "1", "handle", "1", "bpf"}
	cmd := exec.CommandContext(ctx, "tc", args...).WithFilters(libbpfFixupMsg)
	_, err := cmd.CombinedOutput(log, true)
	if err != nil {
		return fmt.Errorf("Failed to remove tc filter: %s", err)
	}

	return nil
}
<commit_msg>loader: Remove unused arguments in DeleteDatapath<commit_after>// Copyright 2017-2018 Authors of Cilium
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package loader

import (
	"context"
	"fmt"

	"github.com/cilium/cilium/pkg/bpf"
	"github.com/cilium/cilium/pkg/command/exec"

	"github.com/vishvananda/netlink"
)

const (
	libbpfFixupMsg = "struct bpf_elf_map fixup performed due to size mismatch!"
)

func replaceQdisc(ifName string) error {
	link, err := netlink.LinkByName(ifName)
	if err != nil {
		return err
	}
	attrs := netlink.QdiscAttrs{
		LinkIndex: link.Attrs().Index,
		Handle:    netlink.MakeHandle(0xffff, 0),
		Parent:    netlink.HANDLE_CLSACT,
	}

	qdisc := &netlink.GenericQdisc{
		QdiscAttrs: attrs,
		QdiscType:  "clsact",
	}

	if err = netlink.QdiscReplace(qdisc); err != nil {
		return fmt.Errorf("netlink: Replacing qdisc for %s failed: %s", ifName, err)
	} else {
		log.Debugf("netlink: Replacing qdisc for %s succeeded", ifName)
	}

	return nil
}

// replaceDatapath the qdisc and BPF program for a endpoint
func (l *Loader) replaceDatapath(ctx context.Context, ifName, objPath, progSec, progDirection string) error {
	err := replaceQdisc(ifName)
	if err != nil {
		return fmt.Errorf("Failed to replace Qdisc for %s: %s", ifName, err)
	}

	// FIXME: Replace cilium-map-migrate with Golang map migration
	cmd := exec.CommandContext(ctx, "cilium-map-migrate", "-s", objPath)
	cmd.Env = bpf.Environment()
	if _, err = cmd.CombinedOutput(log, true); err != nil {
		return err
	}
	defer func() {
		var retCode string
		if err == nil {
			retCode = "0"
		} else {
			retCode = "1"
		}
		args := []string{"-e", objPath, "-r", retCode}
		cmd := exec.CommandContext(ctx, "cilium-map-migrate", args...)
		cmd.Env = bpf.Environment()
		_, _ = cmd.CombinedOutput(log, true) // ignore errors
	}()

	// FIXME: replace exec with native call
	args := []string{"filter", "replace", "dev", ifName, progDirection,
		"prio", "1", "handle", "1", "bpf", "da", "obj", objPath,
		"sec", progSec,
	}
	cmd = exec.CommandContext(ctx, "tc", args...).WithFilters(libbpfFixupMsg)
	_, err = cmd.CombinedOutput(log, true)
	if err != nil {
		return fmt.Errorf("Failed to load tc filter: %s", err)
	}

	return nil
}

// graftDatapath replaces obj in tail call map
func graftDatapath(ctx context.Context, mapPath, objPath, progSec string) error {
	var err error

	// FIXME: Replace cilium-map-migrate with Golang map migration
	cmd := exec.CommandContext(ctx, "cilium-map-migrate", "-s", objPath)
	cmd.Env = bpf.Environment()
	if _, err = cmd.CombinedOutput(log, true); err != nil {
		return err
	}
	defer func() {
		var retCode string
		if err == nil {
			retCode = "0"
		} else {
			retCode = "1"
		}
		args := []string{"-e", objPath, "-r", retCode}
		cmd := exec.CommandContext(ctx, "cilium-map-migrate", args...)
		cmd.Env = bpf.Environment()
		_, _ = cmd.CombinedOutput(log, true) // ignore errors
	}()

	// FIXME: replace exec with native call
	// FIXME: only key 0 right now, could be made more flexible
	args := []string{"exec", "bpf", "graft", mapPath, "key", "0",
		"obj", objPath, "sec", progSec,
	}
	cmd = exec.CommandContext(ctx, "tc", args...).WithFilters(libbpfFixupMsg)
	_, err = cmd.CombinedOutput(log, true)
	if err != nil {
		return fmt.Errorf("Failed to graft tc object: %s", err)
	}

	return nil
}

// DeleteDatapath filter from the given ifName
func (l *Loader) DeleteDatapath(ctx context.Context, ifName, direction string) error {
	args := []string{"filter", "delete", "dev", ifName, direction}
	cmd := exec.CommandContext(ctx, "tc", args...).WithFilters(libbpfFixupMsg)
	_, err := cmd.CombinedOutput(log, true)
	if err != nil {
		return fmt.Errorf("Failed to remove tc filter: %s", err)
	}

	return nil
}
<|endoftext|>