<commit_before>package cgroups

import (
	"bufio"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"time"

	units "github.com/docker/go-units"
	specs "github.com/opencontainers/runtime-spec/specs-go"
)

var isUserNS = runningInUserNS()

// runningInUserNS detects whether we are currently running in a user namespace.
// Copied from github.com/lxc/lxd/shared/util.go
func runningInUserNS() bool {
	file, err := os.Open("/proc/self/uid_map")
	if err != nil {
		// This kernel-provided file only exists if user namespaces are supported
		return false
	}
	defer file.Close()

	buf := bufio.NewReader(file)
	l, _, err := buf.ReadLine()
	if err != nil {
		return false
	}

	line := string(l)
	var a, b, c int64
	fmt.Sscanf(line, "%d %d %d", &a, &b, &c)
	/*
	 * We assume we are in the initial user namespace if we have a full
	 * range - 4294967295 uids starting at uid 0.
	 */
	if a == 0 && b == 0 && c == 4294967295 {
		return false
	}
	return true
}

// defaults returns all known groups
func defaults(root string) ([]Subsystem, error) {
	h, err := NewHugetlb(root)
	if err != nil && !os.IsNotExist(err) {
		return nil, err
	}
	s := []Subsystem{
		NewNamed(root, "systemd"),
		NewFreezer(root),
		NewPids(root),
		NewNetCls(root),
		NewNetPrio(root),
		NewPerfEvent(root),
		NewCputset(root),
		NewCpu(root),
		NewCpuacct(root),
		NewMemory(root),
		NewBlkio(root),
	}
	// only add the devices cgroup if we are not in a user namespace
	// because modifications are not allowed
	if !isUserNS {
		s = append(s, NewDevices(root))
	}
	// add the hugetlb cgroup if error wasn't due to missing hugetlb
	// cgroup support on the host
	if err == nil {
		s = append(s, h)
	}
	return s, nil
}

// remove will remove a cgroup path handling EAGAIN and EBUSY errors and
// retrying the remove after a exp timeout
func remove(path string) error {
	for i := 0; i < 5; i++ {
		delay := 10 * time.Millisecond
		if i != 0 {
			time.Sleep(delay)
			delay *= 2
		}
		if err := os.RemoveAll(path); err == nil {
			return nil
		}
	}
	return fmt.Errorf("cgroups: unable to remove path %q", path)
}

// readPids will read all the pids in a cgroup by the provided path
func readPids(path string, subsystem Name) ([]Process, error) {
	f, err := os.Open(filepath.Join(path, cgroupProcs))
	if err != nil {
		return nil, err
	}
	defer f.Close()
	var (
		out []Process
		s   = bufio.NewScanner(f)
	)
	for s.Scan() {
		if t := s.Text(); t != "" {
			pid, err := strconv.Atoi(t)
			if err != nil {
				return nil, err
			}
			out = append(out, Process{
				Pid:       pid,
				Subsystem: subsystem,
				Path:      path,
			})
		}
	}
	return out, nil
}

func hugePageSizes() ([]string, error) {
	var (
		pageSizes []string
		sizeList  = []string{"B", "kB", "MB", "GB", "TB", "PB"}
	)
	files, err := ioutil.ReadDir("/sys/kernel/mm/hugepages")
	if err != nil {
		return nil, err
	}
	for _, st := range files {
		nameArray := strings.Split(st.Name(), "-")
		pageSize, err := units.RAMInBytes(nameArray[1])
		if err != nil {
			return nil, err
		}
		pageSizes = append(pageSizes, units.CustomSize("%g%s", float64(pageSize), 1024.0, sizeList))
	}
	return pageSizes, nil
}

func readUint(path string) (uint64, error) {
	v, err := ioutil.ReadFile(path)
	if err != nil {
		return 0, err
	}
	return parseUint(strings.TrimSpace(string(v)), 10, 64)
}

func parseUint(s string, base, bitSize int) (uint64, error) {
	v, err := strconv.ParseUint(s, base, bitSize)
	if err != nil {
		intValue, intErr := strconv.ParseInt(s, base, bitSize)
		// 1. Handle negative values greater than MinInt64 (and)
		// 2. Handle negative values lesser than MinInt64
		if intErr == nil && intValue < 0 {
			return 0, nil
		} else if intErr != nil &&
			intErr.(*strconv.NumError).Err == strconv.ErrRange &&
			intValue < 0 {
			return 0, nil
		}
		return 0, err
	}
	return v, nil
}

func parseKV(raw string) (string, uint64, error) {
	parts := strings.Fields(raw)
	switch len(parts) {
	case 2:
		v, err := parseUint(parts[1], 10, 64)
		if err != nil {
			return "", 0, err
		}
		return parts[0], v, nil
	default:
		return "", 0, ErrInvalidFormat
	}
}

func parseCgroupFile(path string) (map[string]string, error) {
	f, err := os.Open(path)
	if err != nil {
		return nil, err
	}
	defer f.Close()
	return parseCgroupFromReader(f)
}

func parseCgroupFromReader(r io.Reader) (map[string]string, error) {
	var (
		cgroups = make(map[string]string)
		s       = bufio.NewScanner(r)
	)
	for s.Scan() {
		if err := s.Err(); err != nil {
			return nil, err
		}
		var (
			text  = s.Text()
			parts = strings.SplitN(text, ":", 3)
		)
		if len(parts) < 3 {
			return nil, fmt.Errorf("invalid cgroup entry: must contain at least two colons: %v", text)
		}
		for _, subs := range strings.Split(parts[1], ",") {
			if subs != "" {
				cgroups[subs] = parts[2]
			}
		}
	}
	return cgroups, nil
}

func getCgroupDestination(subsystem string) (string, error) {
	f, err := os.Open("/proc/self/mountinfo")
	if err != nil {
		return "", err
	}
	defer f.Close()
	s := bufio.NewScanner(f)
	for s.Scan() {
		if err := s.Err(); err != nil {
			return "", err
		}
		fields := strings.Fields(s.Text())
		for _, opt := range strings.Split(fields[len(fields)-1], ",") {
			if opt == subsystem {
				return fields[3], nil
			}
		}
	}
	return "", ErrNoCgroupMountDestination
}

func pathers(subystems []Subsystem) []pather {
	var out []pather
	for _, s := range subystems {
		if p, ok := s.(pather); ok {
			out = append(out, p)
		}
	}
	return out
}

func initializeSubsystem(s Subsystem, path Path, resources *specs.LinuxResources) error {
	if c, ok := s.(creator); ok {
		p, err := path(s.Name())
		if err != nil {
			return err
		}
		if err := c.Create(p, resources); err != nil {
			return err
		}
	} else if c, ok := s.(pather); ok {
		p, err := path(s.Name())
		if err != nil {
			return err
		}
		// do the default create if the group does not have a custom one
		if err := os.MkdirAll(c.Path(p), defaultDirPerm); err != nil {
			return err
		}
	}
	return nil
}

func cleanPath(path string) string {
	if path == "" {
		return ""
	}
	path = filepath.Clean(path)
	if !filepath.IsAbs(path) {
		path, _ = filepath.Rel(string(os.PathSeparator), filepath.Clean(string(os.PathSeparator)+path))
	}
	return filepath.Clean(path)
}
<commit_msg>fix exp delay in remove<commit_after>package cgroups

import (
	"bufio"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"time"

	units "github.com/docker/go-units"
	specs "github.com/opencontainers/runtime-spec/specs-go"
)

var isUserNS = runningInUserNS()

// runningInUserNS detects whether we are currently running in a user namespace.
// Copied from github.com/lxc/lxd/shared/util.go
func runningInUserNS() bool {
	file, err := os.Open("/proc/self/uid_map")
	if err != nil {
		// This kernel-provided file only exists if user namespaces are supported
		return false
	}
	defer file.Close()

	buf := bufio.NewReader(file)
	l, _, err := buf.ReadLine()
	if err != nil {
		return false
	}

	line := string(l)
	var a, b, c int64
	fmt.Sscanf(line, "%d %d %d", &a, &b, &c)
	/*
	 * We assume we are in the initial user namespace if we have a full
	 * range - 4294967295 uids starting at uid 0.
	 */
	if a == 0 && b == 0 && c == 4294967295 {
		return false
	}
	return true
}

// defaults returns all known groups
func defaults(root string) ([]Subsystem, error) {
	h, err := NewHugetlb(root)
	if err != nil && !os.IsNotExist(err) {
		return nil, err
	}
	s := []Subsystem{
		NewNamed(root, "systemd"),
		NewFreezer(root),
		NewPids(root),
		NewNetCls(root),
		NewNetPrio(root),
		NewPerfEvent(root),
		NewCputset(root),
		NewCpu(root),
		NewCpuacct(root),
		NewMemory(root),
		NewBlkio(root),
	}
	// only add the devices cgroup if we are not in a user namespace
	// because modifications are not allowed
	if !isUserNS {
		s = append(s, NewDevices(root))
	}
	// add the hugetlb cgroup if error wasn't due to missing hugetlb
	// cgroup support on the host
	if err == nil {
		s = append(s, h)
	}
	return s, nil
}

// remove will remove a cgroup path handling EAGAIN and EBUSY errors and
// retrying the remove after a exp timeout
func remove(path string) error {
	delay := 10 * time.Millisecond
	for i := 0; i < 5; i++ {
		if i != 0 {
			time.Sleep(delay)
			delay *= 2
		}
		if err := os.RemoveAll(path); err == nil {
			return nil
		}
	}
	return fmt.Errorf("cgroups: unable to remove path %q", path)
}

// readPids will read all the pids in a cgroup by the provided path
func readPids(path string, subsystem Name) ([]Process, error) {
	f, err := os.Open(filepath.Join(path, cgroupProcs))
	if err != nil {
		return nil, err
	}
	defer f.Close()
	var (
		out []Process
		s   = bufio.NewScanner(f)
	)
	for s.Scan() {
		if t := s.Text(); t != "" {
			pid, err := strconv.Atoi(t)
			if err != nil {
				return nil, err
			}
			out = append(out, Process{
				Pid:       pid,
				Subsystem: subsystem,
				Path:      path,
			})
		}
	}
	return out, nil
}

func hugePageSizes() ([]string, error) {
	var (
		pageSizes []string
		sizeList  = []string{"B", "kB", "MB", "GB", "TB", "PB"}
	)
	files, err := ioutil.ReadDir("/sys/kernel/mm/hugepages")
	if err != nil {
		return nil, err
	}
	for _, st := range files {
		nameArray := strings.Split(st.Name(), "-")
		pageSize, err := units.RAMInBytes(nameArray[1])
		if err != nil {
			return nil, err
		}
		pageSizes = append(pageSizes, units.CustomSize("%g%s", float64(pageSize), 1024.0, sizeList))
	}
	return pageSizes, nil
}

func readUint(path string) (uint64, error) {
	v, err := ioutil.ReadFile(path)
	if err != nil {
		return 0, err
	}
	return parseUint(strings.TrimSpace(string(v)), 10, 64)
}

func parseUint(s string, base, bitSize int) (uint64, error) {
	v, err := strconv.ParseUint(s, base, bitSize)
	if err != nil {
		intValue, intErr := strconv.ParseInt(s, base, bitSize)
		// 1. Handle negative values greater than MinInt64 (and)
		// 2. Handle negative values lesser than MinInt64
		if intErr == nil && intValue < 0 {
			return 0, nil
		} else if intErr != nil &&
			intErr.(*strconv.NumError).Err == strconv.ErrRange &&
			intValue < 0 {
			return 0, nil
		}
		return 0, err
	}
	return v, nil
}

func parseKV(raw string) (string, uint64, error) {
	parts := strings.Fields(raw)
	switch len(parts) {
	case 2:
		v, err := parseUint(parts[1], 10, 64)
		if err != nil {
			return "", 0, err
		}
		return parts[0], v, nil
	default:
		return "", 0, ErrInvalidFormat
	}
}

func parseCgroupFile(path string) (map[string]string, error) {
	f, err := os.Open(path)
	if err != nil {
		return nil, err
	}
	defer f.Close()
	return parseCgroupFromReader(f)
}

func parseCgroupFromReader(r io.Reader) (map[string]string, error) {
	var (
		cgroups = make(map[string]string)
		s       = bufio.NewScanner(r)
	)
	for s.Scan() {
		if err := s.Err(); err != nil {
			return nil, err
		}
		var (
			text  = s.Text()
			parts = strings.SplitN(text, ":", 3)
		)
		if len(parts) < 3 {
			return nil, fmt.Errorf("invalid cgroup entry: must contain at least two colons: %v", text)
		}
		for _, subs := range strings.Split(parts[1], ",") {
			if subs != "" {
				cgroups[subs] = parts[2]
			}
		}
	}
	return cgroups, nil
}

func getCgroupDestination(subsystem string) (string, error) {
	f, err := os.Open("/proc/self/mountinfo")
	if err != nil {
		return "", err
	}
	defer f.Close()
	s := bufio.NewScanner(f)
	for s.Scan() {
		if err := s.Err(); err != nil {
			return "", err
		}
		fields := strings.Fields(s.Text())
		for _, opt := range strings.Split(fields[len(fields)-1], ",") {
			if opt == subsystem {
				return fields[3], nil
			}
		}
	}
	return "", ErrNoCgroupMountDestination
}

func pathers(subystems []Subsystem) []pather {
	var out []pather
	for _, s := range subystems {
		if p, ok := s.(pather); ok {
			out = append(out, p)
		}
	}
	return out
}

func initializeSubsystem(s Subsystem, path Path, resources *specs.LinuxResources) error {
	if c, ok := s.(creator); ok {
		p, err := path(s.Name())
		if err != nil {
			return err
		}
		if err := c.Create(p, resources); err != nil {
			return err
		}
	} else if c, ok := s.(pather); ok {
		p, err := path(s.Name())
		if err != nil {
			return err
		}
		// do the default create if the group does not have a custom one
		if err := os.MkdirAll(c.Path(p), defaultDirPerm); err != nil {
			return err
		}
	}
	return nil
}

func cleanPath(path string) string {
	if path == "" {
		return ""
	}
	path = filepath.Clean(path)
	if !filepath.IsAbs(path) {
		path, _ = filepath.Rel(string(os.PathSeparator), filepath.Clean(string(os.PathSeparator)+path))
	}
	return filepath.Clean(path)
}
<|endoftext|>


<commit_before>/*
 * MinIO Go Library for Amazon S3 Compatible Cloud Storage
 * Copyright 2015-2017 MinIO, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package minio

import (
	"crypto/md5"
	"encoding/base64"
	"encoding/hex"
	"encoding/xml"
	"io"
	"io/ioutil"
	"net"
	"net/http"
	"net/url"
	"regexp"
	"strconv"
	"strings"
	"time"

	"github.com/minio/sha256-simd"

	"github.com/minio/minio-go/v6/pkg/s3utils"
)

func trimEtag(etag string) string {
	etag = strings.TrimPrefix(etag, "\"")
	return strings.TrimSuffix(etag, "\"")
}

// xmlDecoder provide decoded value in xml.
func xmlDecoder(body io.Reader, v interface{}) error {
	d := xml.NewDecoder(body)
	return d.Decode(v)
}

// sum256 calculate sha256sum for an input byte array, returns hex encoded.
func sum256Hex(data []byte) string {
	hash := sha256.New()
	hash.Write(data)
	return hex.EncodeToString(hash.Sum(nil))
}

// sumMD5Base64 calculate md5sum for an input byte array, returns base64 encoded.
func sumMD5Base64(data []byte) string {
	hash := md5.New()
	hash.Write(data)
	return base64.StdEncoding.EncodeToString(hash.Sum(nil))
}

// getEndpointURL - construct a new endpoint.
func getEndpointURL(endpoint string, secure bool) (*url.URL, error) {
	if strings.Contains(endpoint, ":") {
		host, _, err := net.SplitHostPort(endpoint)
		if err != nil {
			return nil, err
		}
		if !s3utils.IsValidIP(host) && !s3utils.IsValidDomain(host) {
			msg := "Endpoint: " + endpoint + " does not follow ip address or domain name standards."
			return nil, ErrInvalidArgument(msg)
		}
	} else {
		if !s3utils.IsValidIP(endpoint) && !s3utils.IsValidDomain(endpoint) {
			msg := "Endpoint: " + endpoint + " does not follow ip address or domain name standards."
			return nil, ErrInvalidArgument(msg)
		}
	}
	// If secure is false, use 'http' scheme.
	scheme := "https"
	if !secure {
		scheme = "http"
	}

	// Construct a secured endpoint URL.
	endpointURLStr := scheme + "://" + endpoint
	endpointURL, err := url.Parse(endpointURLStr)
	if err != nil {
		return nil, err
	}

	// Validate incoming endpoint URL.
	if err := isValidEndpointURL(*endpointURL); err != nil {
		return nil, err
	}
	return endpointURL, nil
}

// closeResponse close non nil response with any response Body.
// convenient wrapper to drain any remaining data on response body.
//
// Subsequently this allows golang http RoundTripper
// to re-use the same connection for future requests.
func closeResponse(resp *http.Response) {
	// Callers should close resp.Body when done reading from it.
	// If resp.Body is not closed, the Client's underlying RoundTripper
	// (typically Transport) may not be able to re-use a persistent TCP
	// connection to the server for a subsequent "keep-alive" request.
	if resp != nil && resp.Body != nil {
		// Drain any remaining Body and then close the connection.
		// Without this closing connection would disallow re-using
		// the same connection for future uses.
		//  - http://stackoverflow.com/a/17961593/4465767
		io.Copy(ioutil.Discard, resp.Body)
		resp.Body.Close()
	}
}

var (
	// Hex encoded string of nil sha256sum bytes.
	emptySHA256Hex = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"

	// Sentinel URL is the default url value which is invalid.
	sentinelURL = url.URL{}
)

// Verify if input endpoint URL is valid.
func isValidEndpointURL(endpointURL url.URL) error {
	if endpointURL == sentinelURL {
		return ErrInvalidArgument("Endpoint url cannot be empty.")
	}
	if endpointURL.Path != "/" && endpointURL.Path != "" {
		return ErrInvalidArgument("Endpoint url cannot have fully qualified paths.")
	}
	if strings.Contains(endpointURL.Host, ".s3.amazonaws.com") {
		if !s3utils.IsAmazonEndpoint(endpointURL) {
			return ErrInvalidArgument("Amazon S3 endpoint should be 's3.amazonaws.com'.")
		}
	}
	if strings.Contains(endpointURL.Host, ".googleapis.com") {
		if !s3utils.IsGoogleEndpoint(endpointURL) {
			return ErrInvalidArgument("Google Cloud Storage endpoint should be 'storage.googleapis.com'.")
		}
	}
	return nil
}

// Verify if input expires value is valid.
func isValidExpiry(expires time.Duration) error {
	expireSeconds := int64(expires / time.Second)
	if expireSeconds < 1 {
		return ErrInvalidArgument("Expires cannot be lesser than 1 second.")
	}
	if expireSeconds > 604800 {
		return ErrInvalidArgument("Expires cannot be greater than 7 days.")
	}
	return nil
}

// Extract only necessary metadata header key/values by
// filtering them out with a list of custom header keys.
func extractObjMetadata(header http.Header) http.Header {
	preserveKeys := []string{
		"Content-Type",
		"Cache-Control",
		"Content-Encoding",
		"Content-Language",
		"Content-Disposition",
		"X-Amz-Storage-Class",
		"X-Amz-Object-Lock-Mode",
		"X-Amz-Object-Lock-Retain-Until-Date",
		"X-Amz-Object-Lock-Legal-Hold",
		"X-Amz-Website-Redirect-Location",
		"X-Amz-Server-Side-Encryption",
		"X-Amz-Meta-",
		// Add new headers to be preserved.
		// if you add new headers here, please extend
		// PutObjectOptions{} to preserve them
		// upon upload as well.
	}
	filteredHeader := make(http.Header)
	for k, v := range header {
		var found bool
		for _, prefix := range preserveKeys {
			if !strings.HasPrefix(k, prefix) {
				continue
			}
			found = true
			break
		}
		if found {
			filteredHeader[k] = v
		}
	}
	return filteredHeader
}

// ToObjectInfo converts http header values into ObjectInfo type,
// extracts metadata and fills in all the necessary fields in ObjectInfo.
func ToObjectInfo(bucketName string, objectName string, h http.Header) (ObjectInfo, error) {
	var err error
	// Trim off the odd double quotes from ETag in the beginning and end.
	etag := trimEtag(h.Get("ETag"))

	// Parse content length is exists
	var size int64 = -1
	contentLengthStr := h.Get("Content-Length")
	if contentLengthStr != "" {
		size, err = strconv.ParseInt(contentLengthStr, 10, 64)
		if err != nil {
			// Content-Length is not valid
			return ObjectInfo{}, ErrorResponse{
				Code:       "InternalError",
				Message:    "Content-Length is invalid. " + reportIssue,
				BucketName: bucketName,
				Key:        objectName,
				RequestID:  h.Get("x-amz-request-id"),
				HostID:     h.Get("x-amz-id-2"),
				Region:     h.Get("x-amz-bucket-region"),
			}
		}
	}

	// Parse Last-Modified has http time format.
	date, err := time.Parse(http.TimeFormat, h.Get("Last-Modified"))
	if err != nil {
		return ObjectInfo{}, ErrorResponse{
			Code:       "InternalError",
			Message:    "Last-Modified time format is invalid. " + reportIssue,
			BucketName: bucketName,
			Key:        objectName,
			RequestID:  h.Get("x-amz-request-id"),
			HostID:     h.Get("x-amz-id-2"),
			Region:     h.Get("x-amz-bucket-region"),
		}
	}

	// Fetch content type if any present.
	contentType := strings.TrimSpace(h.Get("Content-Type"))
	if contentType == "" {
		contentType = "application/octet-stream"
	}

	expiryStr := h.Get("Expires")
	var expTime time.Time
	if t, err := time.Parse(http.TimeFormat, expiryStr); err == nil {
		expTime = t.UTC()
	}

	metadata := extractObjMetadata(h)
	userMetadata := make(map[string]string)
	for k, v := range metadata {
		if strings.HasPrefix(k, "X-Amz-Meta-") {
			userMetadata[strings.TrimPrefix(k, "X-Amz-Meta-")] = v[0]
		}
	}
	userTags := s3utils.TagDecode(h.Get(amzTaggingHeader))

	// Save object metadata info.
	return ObjectInfo{
		ETag:         etag,
		Key:          objectName,
		Size:         size,
		LastModified: date,
		ContentType:  contentType,
		Expires:      expTime,
		// Extract only the relevant header keys describing the object.
		// following function filters out a list of standard set of keys
		// which are not part of object metadata.
		Metadata:     metadata,
		UserMetadata: userMetadata,
		UserTags:     userTags,
	}, nil
}

// regCred matches credential string in HTTP header
var regCred = regexp.MustCompile("Credential=([A-Z0-9]+)/")

// regCred matches signature string in HTTP header
var regSign = regexp.MustCompile("Signature=([[0-9a-f]+)")

// Redact out signature value from authorization string.
func redactSignature(origAuth string) string {
	if !strings.HasPrefix(origAuth, signV4Algorithm) {
		// Set a temporary redacted auth
		return "AWS **REDACTED**:**REDACTED**"
	}

	/// Signature V4 authorization header.

	// Strip out accessKeyID from:
	// Credential=<access-key-id>/<date>/<aws-region>/<aws-service>/aws4_request
	newAuth := regCred.ReplaceAllString(origAuth, "Credential=**REDACTED**/")

	// Strip out 256-bit signature from: Signature=<256-bit signature>
	return regSign.ReplaceAllString(newAuth, "Signature=**REDACTED**")
}

// Get default location returns the location based on the input
// URL `u`, if region override is provided then all location
// defaults to regionOverride.
//
// If no other cases match then the location is set to `us-east-1`
// as a last resort.
func getDefaultLocation(u url.URL, regionOverride string) (location string) {
	if regionOverride != "" {
		return regionOverride
	}
	region := s3utils.GetRegionFromURL(u)
	if region == "" {
		region = "us-east-1"
	}
	return region
}

var supportedHeaders = []string{
	"content-type",
	"cache-control",
	"content-encoding",
	"content-disposition",
	"content-language",
	"x-amz-website-redirect-location",
	"x-amz-object-lock-mode",
	"x-amz-object-lock-retain-until-date",
	"expires",
	// Add more supported headers here.
}

// isStorageClassHeader returns true if the header is a supported storage class header
func isStorageClassHeader(headerKey string) bool {
	return strings.EqualFold(amzStorageClass, headerKey)
}

// isStandardHeader returns true if header is a supported header and not a custom header
func isStandardHeader(headerKey string) bool {
	key := strings.ToLower(headerKey)
	for _, header := range supportedHeaders {
		if strings.ToLower(header) == key {
			return true
		}
	}
	return false
}

// sseHeaders is list of server side encryption headers
var sseHeaders = []string{
	"x-amz-server-side-encryption",
	"x-amz-server-side-encryption-aws-kms-key-id",
	"x-amz-server-side-encryption-context",
	"x-amz-server-side-encryption-customer-algorithm",
	"x-amz-server-side-encryption-customer-key",
	"x-amz-server-side-encryption-customer-key-MD5",
}

// isSSEHeader returns true if header is a server side encryption header.
func isSSEHeader(headerKey string) bool {
	key := strings.ToLower(headerKey)
	for _, h := range sseHeaders {
		if strings.ToLower(h) == key {
			return true
		}
	}
	return false
}

// isAmzHeader returns true if header is a x-amz-meta-* or x-amz-acl header.
func isAmzHeader(headerKey string) bool {
	key := strings.ToLower(headerKey)

	return strings.HasPrefix(key, "x-amz-meta-") || strings.HasPrefix(key, "x-amz-grant-") || key == "x-amz-acl" || isSSEHeader(headerKey)
}
<commit_msg>Add tagging count to known header (#1291)<commit_after>/*
 * MinIO Go Library for Amazon S3 Compatible Cloud Storage
 * Copyright 2015-2017 MinIO, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package minio

import (
	"crypto/md5"
	"encoding/base64"
	"encoding/hex"
	"encoding/xml"
	"io"
	"io/ioutil"
	"net"
	"net/http"
	"net/url"
	"regexp"
	"strconv"
	"strings"
	"time"

	"github.com/minio/sha256-simd"

	"github.com/minio/minio-go/v6/pkg/s3utils"
)

func trimEtag(etag string) string {
	etag = strings.TrimPrefix(etag, "\"")
	return strings.TrimSuffix(etag, "\"")
}

// xmlDecoder provide decoded value in xml.
func xmlDecoder(body io.Reader, v interface{}) error {
	d := xml.NewDecoder(body)
	return d.Decode(v)
}

// sum256 calculate sha256sum for an input byte array, returns hex encoded.
func sum256Hex(data []byte) string {
	hash := sha256.New()
	hash.Write(data)
	return hex.EncodeToString(hash.Sum(nil))
}

// sumMD5Base64 calculate md5sum for an input byte array, returns base64 encoded.
func sumMD5Base64(data []byte) string {
	hash := md5.New()
	hash.Write(data)
	return base64.StdEncoding.EncodeToString(hash.Sum(nil))
}

// getEndpointURL - construct a new endpoint.
func getEndpointURL(endpoint string, secure bool) (*url.URL, error) {
	if strings.Contains(endpoint, ":") {
		host, _, err := net.SplitHostPort(endpoint)
		if err != nil {
			return nil, err
		}
		if !s3utils.IsValidIP(host) && !s3utils.IsValidDomain(host) {
			msg := "Endpoint: " + endpoint + " does not follow ip address or domain name standards."
			return nil, ErrInvalidArgument(msg)
		}
	} else {
		if !s3utils.IsValidIP(endpoint) && !s3utils.IsValidDomain(endpoint) {
			msg := "Endpoint: " + endpoint + " does not follow ip address or domain name standards."
			return nil, ErrInvalidArgument(msg)
		}
	}
	// If secure is false, use 'http' scheme.
	scheme := "https"
	if !secure {
		scheme = "http"
	}

	// Construct a secured endpoint URL.
	endpointURLStr := scheme + "://" + endpoint
	endpointURL, err := url.Parse(endpointURLStr)
	if err != nil {
		return nil, err
	}

	// Validate incoming endpoint URL.
	if err := isValidEndpointURL(*endpointURL); err != nil {
		return nil, err
	}
	return endpointURL, nil
}

// closeResponse close non nil response with any response Body.
// convenient wrapper to drain any remaining data on response body.
//
// Subsequently this allows golang http RoundTripper
// to re-use the same connection for future requests.
func closeResponse(resp *http.Response) {
	// Callers should close resp.Body when done reading from it.
	// If resp.Body is not closed, the Client's underlying RoundTripper
	// (typically Transport) may not be able to re-use a persistent TCP
	// connection to the server for a subsequent "keep-alive" request.
	if resp != nil && resp.Body != nil {
		// Drain any remaining Body and then close the connection.
		// Without this closing connection would disallow re-using
		// the same connection for future uses.
		//  - http://stackoverflow.com/a/17961593/4465767
		io.Copy(ioutil.Discard, resp.Body)
		resp.Body.Close()
	}
}

var (
	// Hex encoded string of nil sha256sum bytes.
	emptySHA256Hex = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"

	// Sentinel URL is the default url value which is invalid.
	sentinelURL = url.URL{}
)

// Verify if input endpoint URL is valid.
func isValidEndpointURL(endpointURL url.URL) error {
	if endpointURL == sentinelURL {
		return ErrInvalidArgument("Endpoint url cannot be empty.")
	}
	if endpointURL.Path != "/" && endpointURL.Path != "" {
		return ErrInvalidArgument("Endpoint url cannot have fully qualified paths.")
	}
	if strings.Contains(endpointURL.Host, ".s3.amazonaws.com") {
		if !s3utils.IsAmazonEndpoint(endpointURL) {
			return ErrInvalidArgument("Amazon S3 endpoint should be 's3.amazonaws.com'.")
		}
	}
	if strings.Contains(endpointURL.Host, ".googleapis.com") {
		if !s3utils.IsGoogleEndpoint(endpointURL) {
			return ErrInvalidArgument("Google Cloud Storage endpoint should be 'storage.googleapis.com'.")
		}
	}
	return nil
}

// Verify if input expires value is valid.
func isValidExpiry(expires time.Duration) error {
	expireSeconds := int64(expires / time.Second)
	if expireSeconds < 1 {
		return ErrInvalidArgument("Expires cannot be lesser than 1 second.")
	}
	if expireSeconds > 604800 {
		return ErrInvalidArgument("Expires cannot be greater than 7 days.")
	}
	return nil
}

// Extract only necessary metadata header key/values by
// filtering them out with a list of custom header keys.
func extractObjMetadata(header http.Header) http.Header {
	preserveKeys := []string{
		"Content-Type",
		"Cache-Control",
		"Content-Encoding",
		"Content-Language",
		"Content-Disposition",
		"X-Amz-Storage-Class",
		"X-Amz-Object-Lock-Mode",
		"X-Amz-Object-Lock-Retain-Until-Date",
		"X-Amz-Object-Lock-Legal-Hold",
		"X-Amz-Website-Redirect-Location",
		"X-Amz-Server-Side-Encryption",
		"X-Amz-Tagging-Count",
		"X-Amz-Meta-",
		// Add new headers to be preserved.
		// if you add new headers here, please extend
		// PutObjectOptions{} to preserve them
		// upon upload as well.
	}
	filteredHeader := make(http.Header)
	for k, v := range header {
		var found bool
		for _, prefix := range preserveKeys {
			if !strings.HasPrefix(k, prefix) {
				continue
			}
			found = true
			break
		}
		if found {
			filteredHeader[k] = v
		}
	}
	return filteredHeader
}

// ToObjectInfo converts http header values into ObjectInfo type,
// extracts metadata and fills in all the necessary fields in ObjectInfo.
func ToObjectInfo(bucketName string, objectName string, h http.Header) (ObjectInfo, error) {
	var err error
	// Trim off the odd double quotes from ETag in the beginning and end.
	etag := trimEtag(h.Get("ETag"))

	// Parse content length is exists
	var size int64 = -1
	contentLengthStr := h.Get("Content-Length")
	if contentLengthStr != "" {
		size, err = strconv.ParseInt(contentLengthStr, 10, 64)
		if err != nil {
			// Content-Length is not valid
			return ObjectInfo{}, ErrorResponse{
				Code:       "InternalError",
				Message:    "Content-Length is invalid. " + reportIssue,
				BucketName: bucketName,
				Key:        objectName,
				RequestID:  h.Get("x-amz-request-id"),
				HostID:     h.Get("x-amz-id-2"),
				Region:     h.Get("x-amz-bucket-region"),
			}
		}
	}

	// Parse Last-Modified has http time format.
	date, err := time.Parse(http.TimeFormat, h.Get("Last-Modified"))
	if err != nil {
		return ObjectInfo{}, ErrorResponse{
			Code:       "InternalError",
			Message:    "Last-Modified time format is invalid. " + reportIssue,
			BucketName: bucketName,
			Key:        objectName,
			RequestID:  h.Get("x-amz-request-id"),
			HostID:     h.Get("x-amz-id-2"),
			Region:     h.Get("x-amz-bucket-region"),
		}
	}

	// Fetch content type if any present.
	contentType := strings.TrimSpace(h.Get("Content-Type"))
	if contentType == "" {
		contentType = "application/octet-stream"
	}

	expiryStr := h.Get("Expires")
	var expTime time.Time
	if t, err := time.Parse(http.TimeFormat, expiryStr); err == nil {
		expTime = t.UTC()
	}

	metadata := extractObjMetadata(h)
	userMetadata := make(map[string]string)
	for k, v := range metadata {
		if strings.HasPrefix(k, "X-Amz-Meta-") {
			userMetadata[strings.TrimPrefix(k, "X-Amz-Meta-")] = v[0]
		}
	}
	userTags := s3utils.TagDecode(h.Get(amzTaggingHeader))

	// Save object metadata info.
	return ObjectInfo{
		ETag:         etag,
		Key:          objectName,
		Size:         size,
		LastModified: date,
		ContentType:  contentType,
		Expires:      expTime,
		// Extract only the relevant header keys describing the object.
		// following function filters out a list of standard set of keys
		// which are not part of object metadata.
		Metadata:     metadata,
		UserMetadata: userMetadata,
		UserTags:     userTags,
	}, nil
}

// regCred matches credential string in HTTP header
var regCred = regexp.MustCompile("Credential=([A-Z0-9]+)/")

// regCred matches signature string in HTTP header
var regSign = regexp.MustCompile("Signature=([[0-9a-f]+)")

// Redact out signature value from authorization string.
func redactSignature(origAuth string) string {
	if !strings.HasPrefix(origAuth, signV4Algorithm) {
		// Set a temporary redacted auth
		return "AWS **REDACTED**:**REDACTED**"
	}

	/// Signature V4 authorization header.

	// Strip out accessKeyID from:
	// Credential=<access-key-id>/<date>/<aws-region>/<aws-service>/aws4_request
	newAuth := regCred.ReplaceAllString(origAuth, "Credential=**REDACTED**/")

	// Strip out 256-bit signature from: Signature=<256-bit signature>
	return regSign.ReplaceAllString(newAuth, "Signature=**REDACTED**")
}

// Get default location returns the location based on the input
// URL `u`, if region override is provided then all location
// defaults to regionOverride.
//
// If no other cases match then the location is set to `us-east-1`
// as a last resort.
func getDefaultLocation(u url.URL, regionOverride string) (location string) {
	if regionOverride != "" {
		return regionOverride
	}
	region := s3utils.GetRegionFromURL(u)
	if region == "" {
		region = "us-east-1"
	}
	return region
}

var supportedHeaders = []string{
	"content-type",
	"cache-control",
	"content-encoding",
	"content-disposition",
	"content-language",
	"x-amz-website-redirect-location",
	"x-amz-object-lock-mode",
	"x-amz-object-lock-retain-until-date",
	"expires",
	// Add more supported headers here.
}

// isStorageClassHeader returns true if the header is a supported storage class header
func isStorageClassHeader(headerKey string) bool {
	return strings.EqualFold(amzStorageClass, headerKey)
}

// isStandardHeader returns true if header is a supported header and not a custom header
func isStandardHeader(headerKey string) bool {
	key := strings.ToLower(headerKey)
	for _, header := range supportedHeaders {
		if strings.ToLower(header) == key {
			return true
		}
	}
	return false
}

// sseHeaders is list of server side encryption headers
var sseHeaders = []string{
	"x-amz-server-side-encryption",
	"x-amz-server-side-encryption-aws-kms-key-id",
	"x-amz-server-side-encryption-context",
	"x-amz-server-side-encryption-customer-algorithm",
	"x-amz-server-side-encryption-customer-key",
	"x-amz-server-side-encryption-customer-key-MD5",
}

// isSSEHeader returns true if header is a server side encryption header.
func isSSEHeader(headerKey string) bool {
	key := strings.ToLower(headerKey)
	for _, h := range sseHeaders {
		if strings.ToLower(h) == key {
			return true
		}
	}
	return false
}

// isAmzHeader returns true if header is a x-amz-meta-* or x-amz-acl header.
func isAmzHeader(headerKey string) bool {
	key := strings.ToLower(headerKey)

	return strings.HasPrefix(key, "x-amz-meta-") || strings.HasPrefix(key, "x-amz-grant-") || key == "x-amz-acl" || isSSEHeader(headerKey)
}
<|endoftext|>


<commit_before>package c3

var (
	// An empty Iterable
	emptyIterable Iterable = &nilIterable{}
	// An empty Iterator.
	emptyIterator Iterator = &nilIterator{}
)

func EmptyIterable() Iterable {
	return emptyIterable
}

func EmptyIterator() Iterator {
	return emptyIterator
}

type nilIterable struct{}

func (x *nilIterable) Iterator() Iterator {
	return emptyIterator
}

type nilIterator struct{}

func (x *nilIterator) MoveNext() bool { return false }

func (x *nilIterator) Value() interface{} { return nil }

// Sort sorts the list with the given Lesser function
func Sort(l List, lesser Lesser) {
	s := &Sorter{l, lesser}
	s.Sort()
}

// ToSlice makes a new slice of the items in an Iterable
func ToSlice(c Iterable) []interface{} {
	var slice []interface{}
	if col, ok := c.(ReadOnlyBag); ok {
		slice = make([]interface{}, 0, col.Len())
	} else {
		slice = make([]interface{}, 0, 4)
	}
	for i := c.Iterator(); i.MoveNext(); {
		slice = append(slice, i.Value())
	}
	return slice
}

// ToList creates a new List of the items in an Iterable
func ToList(c Iterable) List {
	l := NewList()
	for i := c.Iterator(); i.MoveNext(); {
		l.Add(i.Value())
	}
	return l
}

// ToReadOnlyList creates a new ReadOnlyList of the items in an Iterable
func ToReadOnlyList(c Iterable) ReadOnlyList {
	l := NewList()
	for i := c.Iterator(); i.MoveNext(); {
		l.Add(i.Value())
	}
	return l
}

// ToBag creates a new Bag of the items in an Iterable
func ToBag(c Iterable) Bag {
	l := NewList()
	for i := c.Iterator(); i.MoveNext(); {
		l.Add(i.Value())
	}
	return l
}

// ToReadOnlyBag creates a new ReadOnlyBag of the items in an Iterable
func ToReadOnlyBag(c Iterable) ReadOnlyBag {
	l := NewList()
	for i := c.Iterator(); i.MoveNext(); {
		l.Add(i.Value())
	}
	return l
}

// ToSet makes a new Set of the unique items in an Iterable
func ToSet(c Iterable) Set {
	set := NewSet()
	for i := c.Iterator(); i.MoveNext(); {
		set.Add(i.Value())
	}
	return set
}

// ToStack makes a new Stack of the items in an Iterable
func ToStack(c Iterable) Stack {
	s := NewStack()
	for i := c.Iterator(); i.MoveNext(); {
		s.Push(i.Value())
	}
	return s
}

// ToQueue makes a new Queue of the items in an Iterable
func ToQueue(c Iterable) Queue {
	s := NewQueue()
	for i := c.Iterator(); i.MoveNext(); {
		s.Enqueue(i.Value())
	}
	return s
}

// For applies the action to each item in the Iterable
func For(c Iterable, action Action) {
	for i := c.Iterator(); i.MoveNext(); {
		action(i.Value())
	}
}

//  Go applies the action to each item in the Iterable
//  on a separate goroutine using an unbuffered channel
func Go(c Iterable, action Action) {
	ch := make(chan interface{})
	defer close(ch)
	go func() {
		for value := range ch {
			action(value)
		}
	}()
	For(c, func(value interface{}) { ch <- value })
}

// GoBuffered applies the action to each item in the Iterable
// on a separate goroutine using a buffered channel
func GoBuffered(c Iterable, bufferSize int, action Action) {
	ch := make(chan interface{}, bufferSize)
	defer close(ch)
	go func() {
		for value := range ch {
			action(value)
		}
	}()
	For(c, func(value interface{}) { ch <- value })
}

// Repeat repeats the item count times.
//
// e.g.:
//		Repeat(3,42) // returns [42,42,42]
func Repeat(count int, item interface{}) Iterable {
	if count < 0 {
		panic("Count parameter invalid")
	}
	if count == 0 {
		return emptyIterable
	}
	return MakeIterable(func() Generate {
		x := 0
		return func() (interface{}, bool) {
			if x == count {
				return nil, false
			}
			x++
			return item, true
		}
	})
}

// Range creates a range iterable that iterates over the ints from start to end inclusive.
//
// e.g.:
//		 Range(0,9) // returns [0,1,2,3,4,5,6,7,8,9]
//		 Range(9,0) // returns [9,8,7,6,5,4,3,2,1,0]
func Range(start, end int) Iterable {
	inc := 1
	if end < start {
		inc = -1
	}
	return MakeIterable(func() Generate {
		x := start - inc
		y := end
		return func() (interface{}, bool) {
			if x == y {
				return nil, false
			}
			x = x + inc
			return x, true
		}
	})
}

func max(a, b int) int {
	if a > b {
		return a
	} else {
		return b
	}
}

func min(a, b int) int {
	if a < b {
		return a
	} else {
		return b
	}
}
<commit_msg>support go 1.0.3<commit_after>package c3

var (
	// An empty Iterable
	emptyIterable Iterable = &nilIterable{}
	// An empty Iterator.
	emptyIterator Iterator = &nilIterator{}
)

func EmptyIterable() Iterable {
	return emptyIterable
}

func EmptyIterator() Iterator {
	return emptyIterator
}

type nilIterable struct{}

func (x *nilIterable) Iterator() Iterator {
	return emptyIterator
}

type nilIterator struct{}

func (x *nilIterator) MoveNext() bool { return false }

func (x *nilIterator) Value() interface{} { return nil }

// Sort sorts the list with the given Lesser function
func Sort(l List, lesser Lesser) {
	s := &Sorter{l, lesser}
	s.Sort()
}

// ToSlice makes a new slice of the items in an Iterable
func ToSlice(c Iterable) []interface{} {
	var slice []interface{}
	if col, ok := c.(ReadOnlyBag); ok {
		slice = make([]interface{}, 0, col.Len())
	} else {
		slice = make([]interface{}, 0, 4)
	}
	for i := c.Iterator(); i.MoveNext(); {
		slice = append(slice, i.Value())
	}
	return slice
}

// ToList creates a new List of the items in an Iterable
func ToList(c Iterable) List {
	l := NewList()
	for i := c.Iterator(); i.MoveNext(); {
		l.Add(i.Value())
	}
	return l
}

// ToReadOnlyList creates a new ReadOnlyList of the items in an Iterable
func ToReadOnlyList(c Iterable) ReadOnlyList {
	l := NewList()
	for i := c.Iterator(); i.MoveNext(); {
		l.Add(i.Value())
	}
	return l
}

// ToBag creates a new Bag of the items in an Iterable
func ToBag(c Iterable) Bag {
	l := NewList()
	for i := c.Iterator(); i.MoveNext(); {
		l.Add(i.Value())
	}
	return l
}

// ToReadOnlyBag creates a new ReadOnlyBag of the items in an Iterable
func ToReadOnlyBag(c Iterable) ReadOnlyBag {
	l := NewList()
	for i := c.Iterator(); i.MoveNext(); {
		l.Add(i.Value())
	}
	return l
}

// ToSet makes a new Set of the unique items in an Iterable
func ToSet(c Iterable) Set {
	set := NewSet()
	for i := c.Iterator(); i.MoveNext(); {
		set.Add(i.Value())
	}
	return set
}

// ToStack makes a new Stack of the items in an Iterable
func ToStack(c Iterable) Stack {
	s := NewStack()
	for i := c.Iterator(); i.MoveNext(); {
		s.Push(i.Value())
	}
	return s
}

// ToQueue makes a new Queue of the items in an Iterable
func ToQueue(c Iterable) Queue {
	s := NewQueue()
	for i := c.Iterator(); i.MoveNext(); {
		s.Enqueue(i.Value())
	}
	return s
}

// For applies the action to each item in the Iterable
func For(c Iterable, action Action) {
	for i := c.Iterator(); i.MoveNext(); {
		action(i.Value())
	}
}

//  Go applies the action to each item in the Iterable
//  on a separate goroutine using an unbuffered channel
func Go(c Iterable, action Action) {
	ch := make(chan interface{})
	defer close(ch)
	go func() {
		for value := range ch {
			action(value)
		}
	}()
	For(c, func(value interface{}) { ch <- value })
}

// GoBuffered applies the action to each item in the Iterable
// on a separate goroutine using a buffered channel
func GoBuffered(c Iterable, bufferSize int, action Action) {
	ch := make(chan interface{}, bufferSize)
	defer close(ch)
	go func() {
		for value := range ch {
			action(value)
		}
	}()
	For(c, func(value interface{}) { ch <- value })
}

// Repeat repeats the item count times.
//
// e.g.:
//		Repeat(3,42) // returns [42,42,42]
func Repeat(count int, item interface{}) Iterable {
	if count < 0 {
		panic("Count parameter invalid")
	}
	if count == 0 {
		return emptyIterable
	}
	return MakeIterable(func() Generate {
		x := 0
		return func() (interface{}, bool) {
			if x == count {
				return nil, false
			}
			x++
			return item, true
		}
	})
}

// Range creates a range iterable that iterates over the ints from start to end inclusive.
//
// e.g.:
//		 Range(0,9) // returns [0,1,2,3,4,5,6,7,8,9]
//		 Range(9,0) // returns [9,8,7,6,5,4,3,2,1,0]
func Range(start, end int) Iterable {
	inc := 1
	if end < start {
		inc = -1
	}
	return MakeIterable(func() Generate {
		x := start - inc
		y := end
		return func() (interface{}, bool) {
			if x == y {
				return nil, false
			}
			x = x + inc
			return x, true
		}
	})
}

func max(a, b int) int {
	if a > b {
		return a
	}
   return b
}

func min(a, b int) int {
	if a < b {
		return a
	}
   return b
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"fmt"
	log "github.com/sirupsen/logrus"
	"os"
	"strconv"
	"strings"
	"time"
)

const (
	everyInstance = -1 // Used when replacing strings
)

var (
	// A selection of allowed keywords for the HTML meta tag
	metaKeywords = []string{"application-name", "author", "description", "generator", "keywords", "robots", "language", "googlebot", "Slurp", "bingbot", "geo.position", "geo.placename", "geo.region", "ICBM", "viewport"}
)

// Check if a given path is a directory
func isDir(path string) bool {
	fs, err := os.Stat(path)
	if err != nil {
		return false
	}
	return fs.IsDir()
}

// Check if a given path exists
func exists(path string) bool {
	_, err := os.Stat(path)
	return err == nil
}

// Translate a given URL path to a probable full filename
func url2filename(dirname, urlpath string) string {
	if strings.Contains(urlpath, "..") {
		log.Warn("Someone was trying to access a directory with .. in the URL")
		return dirname + pathsep
	}
	if strings.HasPrefix(urlpath, "/") {
		if strings.HasSuffix(dirname, pathsep) {
			return dirname + urlpath[1:]
		}
		return dirname + pathsep + urlpath[1:]
	}
	return dirname + "/" + urlpath
}

// Get a list of filenames from a given directory name (that must exist)
func getFilenames(dirname string) []string {
	dir, err := os.Open(dirname)
	defer dir.Close()
	if err != nil {
		log.WithFields(log.Fields{
			"dirname": dirname,
			"error":   err.Error(),
		}).Error("Could not open directory")
		return []string{}
	}
	filenames, err := dir.Readdirnames(-1)
	if err != nil {
		log.WithFields(log.Fields{
			"dirname": dirname,
			"error":   err.Error(),
		}).Error("Could not read filenames from directory")

		return []string{}
	}
	return filenames
}

// Easy way to output a HTML page
func easyPage(title, body string) string {
	return fmt.Sprintf("<!doctype html><title>%s</title>%s
%s
%s", title, defaultFont, defaultStyle, title, body)
}

// Easy way to build links to directories
func easyLink(text, url string, isDirectory bool) string {
	// Add a final slash, if needed
	if isDirectory {
		text += "/"
		url += "/"
	}
	return "" + text + "
"
}

// Build up a string on the form "functionname(arg1, arg2, arg3)"
func infostring(functionName string, args []string) string {
	s := functionName + "("
	if len(args) > 0 {
		s += "\"" + strings.Join(args, "\", \"") + "\""
	}
	return s + ")"
}

// Find one level of whitespace, given indented data
// and a keyword to extract the whitespace in front of
func oneLevelOfIndentation(data *[]byte, keyword string) string {
	whitespace := ""
	kwb := []byte(keyword)
	// If there is a line that contains the given word, extract the whitespace
	if bytes.Contains(*data, kwb) {
		// Find the line that contains they keyword
		var byteline []byte
		found := false
		// Try finding the line with keyword, using \n as the newline
		for _, byteline = range bytes.Split(*data, []byte("\n")) {
			if bytes.Contains(byteline, kwb) {
				found = true
				break
			}
		}
		if found {
			// Find the whitespace in front of the keyword
			whitespaceBytes := byteline[:bytes.Index(byteline, kwb)]
			// Whitespace for one level of indentation
			whitespace = string(whitespaceBytes)
		}
	}
	// Return an empty string, or whitespace for one level of indentation
	return whitespace
}

// Add a link to a stylesheet in the given Amber code
func linkToStyle(amberdata *[]byte, url string) {
	// If the given url is not already mentioned and the data contains "body"
	if !bytes.Contains(*amberdata, []byte(url)) && bytes.Contains(*amberdata, []byte("html")) && bytes.Contains(*amberdata, []byte("body")) {
		// Extract one level of indendation
		whitespace := oneLevelOfIndentation(amberdata, "body")
		// Check if there already is a head section
		if bytes.Contains(*amberdata, []byte("head")) {
			// Add a link to the stylesheet
			*amberdata = bytes.Replace(*amberdata, []byte("head\n"), []byte("head\n"+whitespace+whitespace+`link[href="`+url+`"][rel="stylesheet"][type="text/css"]`+"\n"), 1)

		} else if bytes.Contains(*amberdata, []byte("body")) {

			// Add a link to the stylesheet
			*amberdata = bytes.Replace(*amberdata, []byte("html\n"), []byte("html\n"+whitespace+"head\n"+whitespace+whitespace+`link[href="`+url+`"][rel="stylesheet"][type="text/css"]`+"\n"), 1)
		}
	}
}

// Filter []byte slices into two groups, depending on the given filter function
func filterIntoGroups(bytelines [][]byte, filterfunc func([]byte) bool) ([][]byte, [][]byte) {
	var special, regular [][]byte
	for _, byteline := range bytelines {
		if filterfunc(byteline) {
			// Special
			special = append(special, byteline)
		} else {
			// Regular
			regular = append(regular, byteline)
		}
	}
	return special, regular
}

// Given a source file, extract keywords and values into the given map.
// The map must be filled with keywords to look for.
// The keywords in the data must be on the form "keyword: value",
// and can be within single-line HTML comments (<-- ... -->).
// Returns the data for the lines that does not contain any of the keywords.
func extractKeywords(data []byte, special map[string]string) []byte {
	bnl := []byte("\n")
	// Find and separate the lines starting with one of the keywords in the special map
	_, regular := filterIntoGroups(bytes.Split(data, bnl), func(byteline []byte) bool {
		// Check if the current line has one of the special keywords
		for keyword := range special {
			// Check for lines starting with the keyword and a ":"
			if bytes.HasPrefix(byteline, []byte(keyword+":")) {
				// Set (possibly overwrite) the value in the map, if the keyword is found.
				// Trim the surrounding whitespace and skip the letters of the keyword itself.
				special[keyword] = strings.TrimSpace(string(byteline)[len(keyword)+1:])
				return true
			}
			// Check for lines that starts with "" and contains the keyword and a ":"
			if bytes.HasPrefix(byteline, []byte("")) {
				// Strip away the comment markers
				stripped := strings.TrimSpace(string(byteline[5 : len(byteline)-3]))
				// Check if one of the relevant keywords are present
				if strings.HasPrefix(stripped, keyword+":") {
					// Set (possibly overwrite) the value in the map, if the keyword is found.
					// Trim the surrounding whitespace and skip the letters of the keyword itself.
					special[keyword] = strings.TrimSpace(stripped[len(keyword)+1:])
					return true
				}
			}

		}
		// Not special
		return false
	})
	// Use the regular lines as the new data (remove the special lines)
	return bytes.Join(regular, bnl)
}

// Add meta tag names to the given map
func addMetaKeywords(keywords map[string]string) {
	for _, keyword := range metaKeywords {
		keywords[keyword] = ""
	}
}

// Fatal exit
func fatalExit(err error) {
	// Log to file, if a log file is used
	if serverLogFile != "" {
		log.Error(err)
	}
	// Then switch to stderr and log the message there as well
	log.SetOutput(os.Stderr)
	// Use the standard formatter
	log.SetFormatter(&log.TextFormatter{})
	// Log and exit
	log.Fatalln(err)
}

// Insert doctype in HTML, if missing
func insertDoctype(htmldata []byte) []byte {
	// If there are more than two lines
	if bytes.Count(htmldata, []byte("\n")) > 2 {
		fields := bytes.SplitN(htmldata, []byte("\n"), 3)
		line1 := strings.ToLower(string(fields[0]))
		line2 := strings.ToLower(string(fields[1]))
		if strings.Contains(line1, "doctype") || strings.Contains(line2, "doctype") {
			return htmldata
		}
		// Doctype is missing from the first two lines, add it
		return []byte("<!doctype html>" + string(htmldata))
	}
	return htmldata
}

// Convert time.Duration to milliseconds, as a string (without "ms")
func durationToMS(d time.Duration, multiplier float64) string {
	return strconv.Itoa(int(d.Seconds() * 1000.0 * multiplier))
}
<commit_msg>Added a touch function<commit_after>package main

import (
	"bytes"
	"fmt"
	log "github.com/sirupsen/logrus"
	"os"
	"strconv"
	"strings"
	"time"
)

const (
	everyInstance = -1 // Used when replacing strings
)

var (
	// A selection of allowed keywords for the HTML meta tag
	metaKeywords = []string{"application-name", "author", "description", "generator", "keywords", "robots", "language", "googlebot", "Slurp", "bingbot", "geo.position", "geo.placename", "geo.region", "ICBM", "viewport"}
)

// Check if a given path is a directory
func isDir(path string) bool {
	fs, err := os.Stat(path)
	if err != nil {
		return false
	}
	return fs.IsDir()
}

// Check if a given path exists
func exists(path string) bool {
	_, err := os.Stat(path)
	return err == nil
}

// Create an empty file if it doesn't exist
func touch(filename string) error {
	if !exists(filename) {
		_, err := os.Create(filename)
		return err
	}
	return nil
}

// Translate a given URL path to a probable full filename
func url2filename(dirname, urlpath string) string {
	if strings.Contains(urlpath, "..") {
		log.Warn("Someone was trying to access a directory with .. in the URL")
		return dirname + pathsep
	}
	if strings.HasPrefix(urlpath, "/") {
		if strings.HasSuffix(dirname, pathsep) {
			return dirname + urlpath[1:]
		}
		return dirname + pathsep + urlpath[1:]
	}
	return dirname + "/" + urlpath
}

// Get a list of filenames from a given directory name (that must exist)
func getFilenames(dirname string) []string {
	dir, err := os.Open(dirname)
	defer dir.Close()
	if err != nil {
		log.WithFields(log.Fields{
			"dirname": dirname,
			"error":   err.Error(),
		}).Error("Could not open directory")
		return []string{}
	}
	filenames, err := dir.Readdirnames(-1)
	if err != nil {
		log.WithFields(log.Fields{
			"dirname": dirname,
			"error":   err.Error(),
		}).Error("Could not read filenames from directory")

		return []string{}
	}
	return filenames
}

// Easy way to output a HTML page
func easyPage(title, body string) string {
	return fmt.Sprintf("<!doctype html><title>%s</title>%s
%s
%s", title, defaultFont, defaultStyle, title, body)
}

// Easy way to build links to directories
func easyLink(text, url string, isDirectory bool) string {
	// Add a final slash, if needed
	if isDirectory {
		text += "/"
		url += "/"
	}
	return "" + text + "
"
}

// Build up a string on the form "functionname(arg1, arg2, arg3)"
func infostring(functionName string, args []string) string {
	s := functionName + "("
	if len(args) > 0 {
		s += "\"" + strings.Join(args, "\", \"") + "\""
	}
	return s + ")"
}

// Find one level of whitespace, given indented data
// and a keyword to extract the whitespace in front of
func oneLevelOfIndentation(data *[]byte, keyword string) string {
	whitespace := ""
	kwb := []byte(keyword)
	// If there is a line that contains the given word, extract the whitespace
	if bytes.Contains(*data, kwb) {
		// Find the line that contains they keyword
		var byteline []byte
		found := false
		// Try finding the line with keyword, using \n as the newline
		for _, byteline = range bytes.Split(*data, []byte("\n")) {
			if bytes.Contains(byteline, kwb) {
				found = true
				break
			}
		}
		if found {
			// Find the whitespace in front of the keyword
			whitespaceBytes := byteline[:bytes.Index(byteline, kwb)]
			// Whitespace for one level of indentation
			whitespace = string(whitespaceBytes)
		}
	}
	// Return an empty string, or whitespace for one level of indentation
	return whitespace
}

// Add a link to a stylesheet in the given Amber code
func linkToStyle(amberdata *[]byte, url string) {
	// If the given url is not already mentioned and the data contains "body"
	if !bytes.Contains(*amberdata, []byte(url)) && bytes.Contains(*amberdata, []byte("html")) && bytes.Contains(*amberdata, []byte("body")) {
		// Extract one level of indendation
		whitespace := oneLevelOfIndentation(amberdata, "body")
		// Check if there already is a head section
		if bytes.Contains(*amberdata, []byte("head")) {
			// Add a link to the stylesheet
			*amberdata = bytes.Replace(*amberdata, []byte("head\n"), []byte("head\n"+whitespace+whitespace+`link[href="`+url+`"][rel="stylesheet"][type="text/css"]`+"\n"), 1)

		} else if bytes.Contains(*amberdata, []byte("body")) {

			// Add a link to the stylesheet
			*amberdata = bytes.Replace(*amberdata, []byte("html\n"), []byte("html\n"+whitespace+"head\n"+whitespace+whitespace+`link[href="`+url+`"][rel="stylesheet"][type="text/css"]`+"\n"), 1)
		}
	}
}

// Filter []byte slices into two groups, depending on the given filter function
func filterIntoGroups(bytelines [][]byte, filterfunc func([]byte) bool) ([][]byte, [][]byte) {
	var special, regular [][]byte
	for _, byteline := range bytelines {
		if filterfunc(byteline) {
			// Special
			special = append(special, byteline)
		} else {
			// Regular
			regular = append(regular, byteline)
		}
	}
	return special, regular
}

// Given a source file, extract keywords and values into the given map.
// The map must be filled with keywords to look for.
// The keywords in the data must be on the form "keyword: value",
// and can be within single-line HTML comments (<-- ... -->).
// Returns the data for the lines that does not contain any of the keywords.
func extractKeywords(data []byte, special map[string]string) []byte {
	bnl := []byte("\n")
	// Find and separate the lines starting with one of the keywords in the special map
	_, regular := filterIntoGroups(bytes.Split(data, bnl), func(byteline []byte) bool {
		// Check if the current line has one of the special keywords
		for keyword := range special {
			// Check for lines starting with the keyword and a ":"
			if bytes.HasPrefix(byteline, []byte(keyword+":")) {
				// Set (possibly overwrite) the value in the map, if the keyword is found.
				// Trim the surrounding whitespace and skip the letters of the keyword itself.
				special[keyword] = strings.TrimSpace(string(byteline)[len(keyword)+1:])
				return true
			}
			// Check for lines that starts with "" and contains the keyword and a ":"
			if bytes.HasPrefix(byteline, []byte("")) {
				// Strip away the comment markers
				stripped := strings.TrimSpace(string(byteline[5 : len(byteline)-3]))
				// Check if one of the relevant keywords are present
				if strings.HasPrefix(stripped, keyword+":") {
					// Set (possibly overwrite) the value in the map, if the keyword is found.
					// Trim the surrounding whitespace and skip the letters of the keyword itself.
					special[keyword] = strings.TrimSpace(stripped[len(keyword)+1:])
					return true
				}
			}

		}
		// Not special
		return false
	})
	// Use the regular lines as the new data (remove the special lines)
	return bytes.Join(regular, bnl)
}

// Add meta tag names to the given map
func addMetaKeywords(keywords map[string]string) {
	for _, keyword := range metaKeywords {
		keywords[keyword] = ""
	}
}

// Fatal exit
func fatalExit(err error) {
	// Log to file, if a log file is used
	if serverLogFile != "" {
		log.Error(err)
	}
	// Then switch to stderr and log the message there as well
	log.SetOutput(os.Stderr)
	// Use the standard formatter
	log.SetFormatter(&log.TextFormatter{})
	// Log and exit
	log.Fatalln(err)
}

// Insert doctype in HTML, if missing
func insertDoctype(htmldata []byte) []byte {
	// If there are more than two lines
	if bytes.Count(htmldata, []byte("\n")) > 2 {
		fields := bytes.SplitN(htmldata, []byte("\n"), 3)
		line1 := strings.ToLower(string(fields[0]))
		line2 := strings.ToLower(string(fields[1]))
		if strings.Contains(line1, "doctype") || strings.Contains(line2, "doctype") {
			return htmldata
		}
		// Doctype is missing from the first two lines, add it
		return []byte("<!doctype html>" + string(htmldata))
	}
	return htmldata
}

// Convert time.Duration to milliseconds, as a string (without "ms")
func durationToMS(d time.Duration, multiplier float64) string {
	return strconv.Itoa(int(d.Seconds() * 1000.0 * multiplier))
}
<|endoftext|>


<commit_before>package tgbot

import (
	"bytes"
	"encoding/json"
	"errors"
	"io/ioutil"
	"mime/multipart"
	"net/http"
	"reflect"
	"strconv"
	"strings"
)

var httpClient = &http.Client{}

func sendResuest(method string, apiKey string, paramsObject interface{}, t interface{}) error {
	return _sendResuest(method, apiKey, paramsObject, t)
}

func _sendResuest(method string, apiKey string, paramsObject interface{}, t interface{}) error {
	var requestBytes bytes.Buffer
	var writer = multipart.NewWriter(&requestBytes)
	var withParameters = false

	var addFileParameter = func(key string, value []byte, filename string) {
		writer, _ := writer.CreateFormFile(key, filename)
		writer.Write(value)
		withParameters = true
	}

	var addStringParameter = func(key string, value string) {
		writer, _ := writer.CreateFormField(key)
		writer.Write([]byte(value))
		withParameters = true
	}

	parameters, err := extractParams(paramsObject)
	for key, value := range parameters {
		if param, ok := value.(string); ok {
			addStringParameter(key, param)
		} else if param, ok := value.(*InputFile); ok {
			fileData := param.getData()
			if stringData, ok := fileData.(string); ok {
				addStringParameter(key, stringData)
			} else if bytesData, ok := fileData.([]byte); ok {
				addFileParameter(key, bytesData, param.getFilename())
			}
		} else if param, ok := value.([]byte); ok {
			addFileParameter(key, param, "file")
		}
	}

	writer.Close()

	request, err := http.NewRequest("POST", "https://api.telegram.org/bot"+apiKey+"/"+method, &requestBytes)
	if err != nil {
		return errors.New("unable to create request with given parameters")
	}

	if withParameters {
		request.Header.Add("Content-Type", writer.FormDataContentType())
	}

	response, err := httpClient.Do(request)
	if err != nil {
		return errors.New("unable to execute request with given parameters")
	}

	defer response.Body.Close()

	body, err := ioutil.ReadAll(response.Body)
	if err != nil {
		return errors.New("unable to read response body")
	}

	var apiResponse = apiResponse{}
	err = json.Unmarshal(body, &apiResponse)
	if err != nil {
		return errors.New("unable to unserialize response from server")
	}

	if apiResponse.Ok != true {
		return errors.New(apiResponse.Description)
	}

	err = json.Unmarshal(apiResponse.Result, &t)
	if err != nil {
		return errors.New("unable to unserialize API result")
	}

	return nil
}

func extractParams(paramsObject interface{}) (map[string]interface{}, error) {
	var result = map[string]interface{}{}

	if paramsObject == nil {
		return result, nil
	}

	reflectType := reflect.TypeOf(paramsObject)
	reflectValue := reflect.ValueOf(paramsObject)

	for i := 0; i < reflectType.NumField(); i++ {
		field := reflectType.Field(i)
		option := field.Tag.Get("option")

		var extractedValue interface{}
		switch v := reflectValue.FieldByName(field.Name).Interface().(type) {
		case bool:
			if v {
				extractedValue = "true"
			}
		case int:
			if v != 0 {
				extractedValue = strconv.FormatInt(int64(v), 10)
			}
		case int64:
			if v != 0 {
				extractedValue = strconv.FormatInt(v, 10)
			}
		case float64:
			if v != 0 {
				extractedValue = strconv.FormatFloat(v, 'f', 6, 64)
			}
		case string:
			if len(v) > 0 {
				extractedValue = v
			}
		case *InputFile:
			if v != nil {
				extractedValue = v
			}
		case stringConfig:
			if !reflect.ValueOf(v).IsNil() {
				extractedValue = v.getString()
			}
		case []InlineQueryResult:
			serializedQueryResults := []string{}
			for _, queryResult := range v {
				queryResultJSON := buildInlineQueryResult(queryResult, queryResult.getType())
				serializedQueryResults = append(serializedQueryResults, queryResultJSON)
			}
			extractedValue = "[" + strings.Join(serializedQueryResults, ",") + "]"
		}

		if extractedValue != nil {
			result[option] = extractedValue
		}
	}

	return result, nil
}

func buildInlineQueryResult(queryResult InlineQueryResult, resultType string) string {
	queryResultMap := map[string]interface{}{"type": resultType}
	reflectType := reflect.TypeOf(queryResult)
	reflectValue := reflect.ValueOf(queryResult)

	for i := 0; i < reflectType.NumField(); i++ {
		field := reflectType.Field(i)
		if jsonAttribute, ok := field.Tag.Lookup("json"); ok {
			val := reflectValue.FieldByName(field.Name).Interface()
			if val != reflect.Zero(reflect.TypeOf(val)).Interface() {
				queryResultMap[jsonAttribute] = val
			}
		}
	}

	queryResultJSON, _ := json.Marshal(queryResultMap)
	return string(queryResultJSON)
}
<commit_msg>Remove sendRequest wrapper<commit_after>package tgbot

import (
	"bytes"
	"encoding/json"
	"errors"
	"io/ioutil"
	"mime/multipart"
	"net/http"
	"reflect"
	"strconv"
	"strings"
)

var httpClient = &http.Client{}

func sendResuest(method string, apiKey string, paramsObject interface{}, t interface{}) error {
	var requestBytes bytes.Buffer
	var writer = multipart.NewWriter(&requestBytes)
	var withParameters = false

	var addFileParameter = func(key string, value []byte, filename string) {
		writer, _ := writer.CreateFormFile(key, filename)
		writer.Write(value)
		withParameters = true
	}

	var addStringParameter = func(key string, value string) {
		writer, _ := writer.CreateFormField(key)
		writer.Write([]byte(value))
		withParameters = true
	}

	parameters, err := extractParams(paramsObject)
	for key, value := range parameters {
		if param, ok := value.(string); ok {
			addStringParameter(key, param)
		} else if param, ok := value.(*InputFile); ok {
			fileData := param.getData()
			if stringData, ok := fileData.(string); ok {
				addStringParameter(key, stringData)
			} else if bytesData, ok := fileData.([]byte); ok {
				addFileParameter(key, bytesData, param.getFilename())
			}
		} else if param, ok := value.([]byte); ok {
			addFileParameter(key, param, "file")
		}
	}

	writer.Close()

	request, err := http.NewRequest("POST", "https://api.telegram.org/bot"+apiKey+"/"+method, &requestBytes)
	if err != nil {
		return errors.New("unable to create request with given parameters")
	}

	if withParameters {
		request.Header.Add("Content-Type", writer.FormDataContentType())
	}

	response, err := httpClient.Do(request)
	if err != nil {
		return errors.New("unable to execute request with given parameters")
	}

	defer response.Body.Close()

	body, err := ioutil.ReadAll(response.Body)
	if err != nil {
		return errors.New("unable to read response body")
	}

	var apiResponse = apiResponse{}
	err = json.Unmarshal(body, &apiResponse)
	if err != nil {
		return errors.New("unable to unserialize response from server")
	}

	if apiResponse.Ok != true {
		return errors.New(apiResponse.Description)
	}

	err = json.Unmarshal(apiResponse.Result, &t)
	if err != nil {
		return errors.New("unable to unserialize API result")
	}

	return nil
}

func extractParams(paramsObject interface{}) (map[string]interface{}, error) {
	var result = map[string]interface{}{}

	if paramsObject == nil {
		return result, nil
	}

	reflectType := reflect.TypeOf(paramsObject)
	reflectValue := reflect.ValueOf(paramsObject)

	for i := 0; i < reflectType.NumField(); i++ {
		field := reflectType.Field(i)
		option := field.Tag.Get("option")

		var extractedValue interface{}
		switch v := reflectValue.FieldByName(field.Name).Interface().(type) {
		case bool:
			if v {
				extractedValue = "true"
			}
		case int:
			if v != 0 {
				extractedValue = strconv.FormatInt(int64(v), 10)
			}
		case int64:
			if v != 0 {
				extractedValue = strconv.FormatInt(v, 10)
			}
		case float64:
			if v != 0 {
				extractedValue = strconv.FormatFloat(v, 'f', 6, 64)
			}
		case string:
			if len(v) > 0 {
				extractedValue = v
			}
		case *InputFile:
			if v != nil {
				extractedValue = v
			}
		case stringConfig:
			if !reflect.ValueOf(v).IsNil() {
				extractedValue = v.getString()
			}
		case []InlineQueryResult:
			serializedQueryResults := []string{}
			for _, queryResult := range v {
				queryResultJSON := buildInlineQueryResult(queryResult, queryResult.getType())
				serializedQueryResults = append(serializedQueryResults, queryResultJSON)
			}
			extractedValue = "[" + strings.Join(serializedQueryResults, ",") + "]"
		}

		if extractedValue != nil {
			result[option] = extractedValue
		}
	}

	return result, nil
}

func buildInlineQueryResult(queryResult InlineQueryResult, resultType string) string {
	queryResultMap := map[string]interface{}{"type": resultType}
	reflectType := reflect.TypeOf(queryResult)
	reflectValue := reflect.ValueOf(queryResult)

	for i := 0; i < reflectType.NumField(); i++ {
		field := reflectType.Field(i)
		if jsonAttribute, ok := field.Tag.Lookup("json"); ok {
			val := reflectValue.FieldByName(field.Name).Interface()
			if val != reflect.Zero(reflect.TypeOf(val)).Interface() {
				queryResultMap[jsonAttribute] = val
			}
		}
	}

	queryResultJSON, _ := json.Marshal(queryResultMap)
	return string(queryResultJSON)
}
<|endoftext|>


<commit_before>package core

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/url"

	"github.com/FrenchBen/oracle-sdk-go/bmc"
	"github.com/Sirupsen/logrus"
)

// SecurityList contains the SecurityList reference from: https://docs.us-phoenix-1.oraclecloud.com/api/#/en/iaas/20160918/SecurityList/
type SecurityList struct {
	// The OCID of the compartment that contains the security list
	CompartmentID string `json:"compartmentId"`
	// A user-friendly name
	DisplayName string `json:"displayName"`
	// The OCID of the SecurityList
	ID string `json:"id,omitempty"`
	// Rules for allowing egress IP packets
	EgressSecurityRules []EgressSecurityRule `json:"egressSecurityRules"`
	// Rules for allowing ingress IP packets
	IngressSecurityRules []IngressSecurityRule `json:"ingressSecurityRules"`
	// The SecurityList's current state
	LifeCycleState string `json:"lifecycleState,omitempty"`
	// The date and time the instance was created (RFC3339)
	TimeCreated string `json:"timeCreated,omitempty"`
	// The OCID of the VCN
	VcnID string `json:"vcnId"`
}

// EgressSecurityRule rule for allowing outbound IP packets
type EgressSecurityRule struct {
	Destination string `json:"destination"`
	IcmpOptions string `json:"icmpOptions,omitempty"`
	IsStateless string `json:"isStateless,omitempty"`
	// Protocol values: all, ICMP ("1"), TCP ("6"), UDP ("17").
	Protocol   string     `json:"protocol"`
	TcpOptions PortConfig `json:"tcpOptions,omitempty"`
	UdpOptions PortConfig `json:"udpOptions,omitempty"`
}

// IngressSecurityRule rule for allowing inbound IP packets
type IngressSecurityRule struct {
	Destination string `json:"destination"`
	IcmpOptions string `json:"icmpOptions,omitempty"`
	IsStateless string `json:"isStateless,omitempty"`
	// Protocol values: all, ICMP ("1"), TCP ("6"), UDP ("17").
	Protocol   string     `json:"protocol"`
	Source     string     `json:"source,omitempty"`
	TcpOptions PortConfig `json:"tcpOptions,omitempty"`
	UdpOptions PortConfig `json:"udpOptions,omitempty"`
}

type PortConfig struct {
	DestinationPortRange PortRange `json:"destinationPortRange"`
	SourcePortRange      PortRange `json:"sourcePortRange"`
}

type PortRange struct {
	Min string `json:"min"`
	Max string `json:"max"`
}

// GetSecurityList returns a struct of a SecurityList request given an SecurityList ID
func (c *Client) GetSecurityList(securityListID string) (SecurityList, *bmc.Error) {
	securityList := SecurityList{}
	queryString := url.QueryEscape(securityListID)
	resp, err := c.Request("GET", "/securityLists/"+queryString, nil)
	if err != nil {
		logrus.Error(err)
		bmcError := bmc.Error{Code: string(resp.StatusCode), Message: err.Error()}
		return securityList, &bmcError
	}
	logrus.Debug("StatusCode: ", resp.StatusCode)
	if resp.StatusCode != 200 {
		return securityList, bmc.NewError(*resp)
	}
	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)
	logrus.Debug("Body: ", string(body))
	if err != nil {
		logrus.Fatalf("Could not read JSON response: %s", err)
	}
	if err = json.Unmarshal(body, &securityList); err != nil {
		logrus.Fatalf("Unmarshal impossible: %s", err)
	}
	return securityList, nil
}

// ListSecurityLists returns a slice struct of all securityList
func (c *Client) ListSecurityLists(vcnID string) ([]SecurityList, *bmc.Error) {
	securityLists := []SecurityList{}
	queryString := url.QueryEscape(c.CompartmentID)
	if vcnID != "" {
		queryString = queryString + "&vcnId=" + url.QueryEscape(vcnID)
	}
	resp, err := c.Request("GET", fmt.Sprintf("/securityLists?compartmentId=%s", queryString), nil)
	if err != nil {
		logrus.Error(err)
		bmcError := bmc.Error{Code: string(resp.StatusCode), Message: err.Error()}
		return securityLists, &bmcError
	}
	logrus.Debug("StatusCode: ", resp.StatusCode)
	if resp.StatusCode != 200 {
		return securityLists, bmc.NewError(*resp)
	}
	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		logrus.Fatalf("Could not read JSON response: %s", err)
	}

	logrus.Debug("Body: ", string(body))
	if err = json.Unmarshal(body, &securityLists); err != nil {
		logrus.Fatalf("Unmarshal impossible: %s", err)
	}
	return securityLists, nil
}

// CreateSecurityList creates a new security list for the specified VCN
func (c *Client) CreateSecurityList(securityList *SecurityList) (bool, *bmc.Error) {
	resp, err := c.Request("POST", fmt.Sprintf("/securityLists/"), *securityList)
	if err != nil {
		logrus.Error(err)
		bmcError := bmc.Error{Code: string(resp.StatusCode), Message: err.Error()}
		return false, &bmcError
	}
	logrus.Debug("StatusCode: ", resp.StatusCode)
	if resp.StatusCode != 204 {
		return false, bmc.NewError(*resp)
	}
	return true, nil
}

// UpdateSecurityList updates a security list
func (c *Client) UpdateSecurityList(listener *SecurityList) {
	// PUT securityLists/{loadBalancerId}/listeners/{listenerName}
	logrus.Warning("Method not yet implemented")
	return
}

// DeleteSecurityList deletes the specified security list, but only if it's not associated with a subnet
func (c *Client) DeleteSecurityList(securityListID string) (bool, *bmc.Error) {
	securityListID = url.PathEscape(securityListID)
	resp, err := c.Request("DELETE", fmt.Sprintf("/securityLists/%s", securityListID), nil)
	if err != nil {
		logrus.Error(err)
		bmcError := bmc.Error{Code: string(resp.StatusCode), Message: err.Error()}
		return false, &bmcError
	}
	logrus.Debug("StatusCode: ", resp.StatusCode)
	if resp.StatusCode != 204 {
		return false, bmc.NewError(*resp)
	}
	return true, nil
}
<commit_msg>Ingress accept source not destination<commit_after>package core

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/url"

	"github.com/FrenchBen/oracle-sdk-go/bmc"
	"github.com/Sirupsen/logrus"
)

// SecurityList contains the SecurityList reference from: https://docs.us-phoenix-1.oraclecloud.com/api/#/en/iaas/20160918/SecurityList/
type SecurityList struct {
	// The OCID of the compartment that contains the security list
	CompartmentID string `json:"compartmentId"`
	// A user-friendly name
	DisplayName string `json:"displayName"`
	// The OCID of the SecurityList
	ID string `json:"id,omitempty"`
	// Rules for allowing egress IP packets
	EgressSecurityRules *[]EgressSecurityRule `json:"egressSecurityRules"`
	// Rules for allowing ingress IP packets
	IngressSecurityRules *[]IngressSecurityRule `json:"ingressSecurityRules"`
	// The SecurityList's current state
	LifeCycleState string `json:"lifecycleState,omitempty"`
	// The date and time the instance was created (RFC3339)
	TimeCreated string `json:"timeCreated,omitempty"`
	// The OCID of the VCN
	VcnID string `json:"vcnId"`
}

// EgressSecurityRule rule for allowing outbound IP packets
type EgressSecurityRule struct {
	Destination string `json:"destination"`
	IcmpOptions string `json:"icmpOptions,omitempty"`
	IsStateless string `json:"isStateless,omitempty"`
	// Protocol values: all, ICMP ("1"), TCP ("6"), UDP ("17").
	Protocol   string      `json:"protocol"`
	TcpOptions *PortConfig `json:"tcpOptions,omitempty"`
	UdpOptions *PortConfig `json:"udpOptions,omitempty"`
}

// IngressSecurityRule rule for allowing inbound IP packets
type IngressSecurityRule struct {
	Source      string `json:"source"`
	IcmpOptions string `json:"icmpOptions,omitempty"`
	IsStateless string `json:"isStateless,omitempty"`
	// Protocol values: all, ICMP ("1"), TCP ("6"), UDP ("17").
	Protocol   string      `json:"protocol"`
	TcpOptions *PortConfig `json:"tcpOptions,omitempty"`
	UdpOptions *PortConfig `json:"udpOptions,omitempty"`
}

type PortConfig struct {
	DestinationPortRange *PortRange `json:"destinationPortRange,omitempty"`
	SourcePortRange      *PortRange `json:"sourcePortRange,omitempty"`
}

type PortRange struct {
	Min string `json:"min,omitempty"`
	Max string `json:"max,omitempty"`
}

// GetSecurityList returns a struct of a SecurityList request given an SecurityList ID
func (c *Client) GetSecurityList(securityListID string) (SecurityList, *bmc.Error) {
	securityList := SecurityList{}
	queryString := url.QueryEscape(securityListID)
	resp, err := c.Request("GET", "/securityLists/"+queryString, nil)
	if err != nil {
		logrus.Error(err)
		bmcError := bmc.Error{Code: string(resp.StatusCode), Message: err.Error()}
		return securityList, &bmcError
	}
	logrus.Debug("StatusCode: ", resp.StatusCode)
	if resp.StatusCode != 200 {
		return securityList, bmc.NewError(*resp)
	}
	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)
	logrus.Debug("Body: ", string(body))
	if err != nil {
		logrus.Fatalf("Could not read JSON response: %s", err)
	}
	if err = json.Unmarshal(body, &securityList); err != nil {
		logrus.Fatalf("Unmarshal impossible: %s", err)
	}
	return securityList, nil
}

// ListSecurityLists returns a slice struct of all securityList
func (c *Client) ListSecurityLists(vcnID string) ([]SecurityList, *bmc.Error) {
	securityLists := []SecurityList{}
	queryString := url.QueryEscape(c.CompartmentID)
	if vcnID != "" {
		queryString = queryString + "&vcnId=" + url.QueryEscape(vcnID)
	}
	resp, err := c.Request("GET", fmt.Sprintf("/securityLists?compartmentId=%s", queryString), nil)
	if err != nil {
		logrus.Error(err)
		bmcError := bmc.Error{Code: string(resp.StatusCode), Message: err.Error()}
		return securityLists, &bmcError
	}
	logrus.Debug("StatusCode: ", resp.StatusCode)
	if resp.StatusCode != 200 {
		return securityLists, bmc.NewError(*resp)
	}
	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		logrus.Fatalf("Could not read JSON response: %s", err)
	}

	logrus.Debug("Body: ", string(body))
	if err = json.Unmarshal(body, &securityLists); err != nil {
		logrus.Fatalf("Unmarshal impossible: %s", err)
	}
	return securityLists, nil
}

// CreateSecurityList creates a new security list for the specified VCN
func (c *Client) CreateSecurityList(securityList *SecurityList) (bool, *bmc.Error) {
	resp, err := c.Request("POST", fmt.Sprintf("/securityLists/"), *securityList)
	if err != nil {
		logrus.Error(err)
		bmcError := bmc.Error{Code: string(resp.StatusCode), Message: err.Error()}
		return false, &bmcError
	}
	logrus.Debug("StatusCode: ", resp.StatusCode)
	if resp.StatusCode != 204 {
		return false, bmc.NewError(*resp)
	}
	return true, nil
}

// UpdateSecurityList updates a security list
func (c *Client) UpdateSecurityList(listener *SecurityList) {
	// PUT securityLists/{loadBalancerId}/listeners/{listenerName}
	logrus.Warning("Method not yet implemented")
	return
}

// DeleteSecurityList deletes the specified security list, but only if it's not associated with a subnet
func (c *Client) DeleteSecurityList(securityListID string) (bool, *bmc.Error) {
	securityListID = url.PathEscape(securityListID)
	resp, err := c.Request("DELETE", fmt.Sprintf("/securityLists/%s", securityListID), nil)
	if err != nil {
		logrus.Error(err)
		bmcError := bmc.Error{Code: string(resp.StatusCode), Message: err.Error()}
		return false, &bmcError
	}
	logrus.Debug("StatusCode: ", resp.StatusCode)
	if resp.StatusCode != 204 {
		return false, bmc.NewError(*resp)
	}
	return true, nil
}
<|endoftext|>


<commit_before>package cmds

import (
	"io/ioutil"
	"os"
	"testing"

	"github.com/pachyderm/pachyderm/src/client/pkg/require"
	tu "github.com/pachyderm/pachyderm/src/server/pkg/testutil"
)

func run(t *testing.T, cmd string) error {
	t.Helper()

	tmpfile, err := ioutil.TempFile("", "test-pach-config-*.json")
	require.NoError(t, err)

	// remove the empty file so that a config can be generated
	require.NoError(t, os.Remove(tmpfile.Name()))

	// remove the config file when done
	defer os.Remove(tmpfile.Name())

	return tu.BashCmd(`
		export PACH_CONFIG={{.config}}
		# ensure setting PACH_CONTEXT has no effect in various tests
		export PACH_CONTEXT=from_env_var

		{{.cmd}}
		`,
		"config", tmpfile.Name(),
		"cmd", cmd,
	).Run()
}

func TestMetrics(t *testing.T) {
	if testing.Short() {
		t.Skip("Skipping integration tests in short mode")
	}

	require.NoError(t, run(t, `
		pachctl config get metrics | match true
		pachctl config set metrics false
		pachctl config get metrics | match false
		pachctl config set metrics true
		pachctl config get metrics | match true
	`))
}

func TestActiveContext(t *testing.T) {
	if testing.Short() {
		t.Skip("Skipping integration tests in short mode")
	}

	require.YesError(t, run(t, `
		pachctl config set active-context foo 2>&1 | match "context does not exist: foo"
	`))

	require.NoError(t, run(t, `
		echo '{}' | pachctl config set context foo --overwrite
		pachctl config set active-context foo
		pachctl config get active-context | match "foo"
	`))
}

func TestSetContext(t *testing.T) {
	if testing.Short() {
		t.Skip("Skipping integration tests in short mode")
	}

	require.YesError(t, run(t, `
		echo 'malformed_json' | pachctl config set context foo
	`))

	require.YesError(t, run(t, `
		echo '{}' | pachctl config set context foo
		echo '{}' | pachctl config set context foo
	`))

	require.NoError(t, run(t, `
		echo '{}' | pachctl config set context foo
		echo '{"pachd_address": "foobar:9000"}' | pachctl config set context foo --overwrite
		pachctl config get context foo | match '"pachd_address": "foobar:9000"'
	`))
}

func TestUpdateContext(t *testing.T) {
	if testing.Short() {
		t.Skip("Skipping integration tests in short mode")
	}

	require.YesError(t, run(t, `
		pachctl config update context foo --pachd-address=bar
	`))

	require.NoError(t, run(t, `
		echo '{}' | pachctl config set context foo
		pachctl config update context foo --pachd-address="foobar:9000"
		pachctl config get context foo | match '"pachd_address": "foobar:9000"'
		pachctl config update context foo --pachd-address=""
		pachctl config get context foo | match -v pachd_address
	`))
}

func TestDeleteContext(t *testing.T) {
	if testing.Short() {
		t.Skip("Skipping integration tests in short mode")
	}

	require.YesError(t, run(t, `
		pachctl config delete context foo
	`))

	require.NoError(t, run(t, `
		echo '{}' | pachctl config set context foo
		pachctl config delete context foo
	`))
}

func TestConfigListContext(t *testing.T) {
	if testing.Short() {
		t.Skip("Skipping integration tests in short mode")
	}

	require.NoError(t, run(t, `
		echo '{}' | pachctl config set context foo
		echo '{}' | pachctl config set context bar
		pachctl config list context | match -v "\*	bar"
		pachctl config list context | match "	bar"
		pachctl config list context | match "	foo"

		pachctl config set active-context bar
		pachctl config list context | match "\*	bar"
		pachctl config list context | match "	foo"
	`))
}
<commit_msg>Fixed tests to use `PACH_CONTEXT` correctly<commit_after>package cmds

import (
	"io/ioutil"
	"os"
	"testing"

	"github.com/pachyderm/pachyderm/src/client/pkg/require"
	tu "github.com/pachyderm/pachyderm/src/server/pkg/testutil"
)

func run(t *testing.T, cmd string) error {
	t.Helper()

	tmpfile, err := ioutil.TempFile("", "test-pach-config-*.json")
	require.NoError(t, err)

	// remove the empty file so that a config can be generated
	require.NoError(t, os.Remove(tmpfile.Name()))

	// remove the config file when done
	defer os.Remove(tmpfile.Name())

	return tu.BashCmd(`
		export PACH_CONFIG={{.config}}
		{{.cmd}}
		`,
		"config", tmpfile.Name(),
		"cmd", cmd,
	).Run()
}

func TestInvalidEnvValue(t *testing.T) {
	if testing.Short() {
		t.Skip("Skipping integration tests in short mode")
	}

	require.YesError(t, run(t, `
		export PACH_CONTEXT=foobar
		pachctl config get active-context
	`))
}

func TestEnvValue(t *testing.T) {
	if testing.Short() {
		t.Skip("Skipping integration tests in short mode")
	}

	require.NoError(t, run(t, `
		echo '{}' | pachctl config set context foo --overwrite
		echo '{}' | pachctl config set context bar --overwrite
		pachctl config set active-context bar
		export PACH_CONTEXT=foo
		pachctl config get active-context | match foo
	`))
}

func TestMetrics(t *testing.T) {
	if testing.Short() {
		t.Skip("Skipping integration tests in short mode")
	}

	require.NoError(t, run(t, `
		pachctl config get metrics | match true
		pachctl config set metrics false
		pachctl config get metrics | match false
		pachctl config set metrics true
		pachctl config get metrics | match true
	`))
}

func TestActiveContext(t *testing.T) {
	if testing.Short() {
		t.Skip("Skipping integration tests in short mode")
	}

	require.YesError(t, run(t, `
		pachctl config set active-context foo 2>&1 | match "context does not exist: foo"
	`))

	require.NoError(t, run(t, `
		echo '{}' | pachctl config set context foo --overwrite
		pachctl config set active-context foo
		pachctl config get active-context | match "foo"
	`))
}

func TestSetContext(t *testing.T) {
	if testing.Short() {
		t.Skip("Skipping integration tests in short mode")
	}

	require.YesError(t, run(t, `
		echo 'malformed_json' | pachctl config set context foo
	`))

	require.YesError(t, run(t, `
		echo '{}' | pachctl config set context foo
		echo '{}' | pachctl config set context foo
	`))

	require.NoError(t, run(t, `
		echo '{}' | pachctl config set context foo
		echo '{"pachd_address": "foobar:9000"}' | pachctl config set context foo --overwrite
		pachctl config get context foo | match '"pachd_address": "foobar:9000"'
	`))
}

func TestUpdateContext(t *testing.T) {
	if testing.Short() {
		t.Skip("Skipping integration tests in short mode")
	}

	require.YesError(t, run(t, `
		pachctl config update context foo --pachd-address=bar
	`))

	require.NoError(t, run(t, `
		echo '{}' | pachctl config set context foo
		pachctl config update context foo --pachd-address="foobar:9000"
		pachctl config get context foo | match '"pachd_address": "foobar:9000"'
		pachctl config update context foo --pachd-address=""
		pachctl config get context foo | match -v pachd_address
	`))
}

func TestDeleteContext(t *testing.T) {
	if testing.Short() {
		t.Skip("Skipping integration tests in short mode")
	}

	require.YesError(t, run(t, `
		pachctl config delete context foo
	`))

	require.NoError(t, run(t, `
		echo '{}' | pachctl config set context foo
		pachctl config delete context foo
	`))
}

func TestConfigListContext(t *testing.T) {
	if testing.Short() {
		t.Skip("Skipping integration tests in short mode")
	}

	require.NoError(t, run(t, `
		echo '{}' | pachctl config set context foo
		echo '{}' | pachctl config set context bar
		pachctl config list context | match -v "\*	bar"
		pachctl config list context | match "	bar"
		pachctl config list context | match "	foo"

		pachctl config set active-context bar
		pachctl config list context | match "\*	bar"
		pachctl config list context | match "	foo"
	`))
}
<|endoftext|>


<commit_before>/*
 FreeBSD Golang sniffer using BPF,
 by david415, the author of bsdbpf
  https://gist.github.com/david415/d38936fd3e93168ae221

 only for *BSD with src/syscall/bpf_bsd.go
  https://golang.org/src/syscall/bpf_bsd.go
*/

package main

import (
	"encoding/hex"
	"fmt"
	"syscall"
	//"syscall/bpf_bsd"
)

func main() {
	enable := 1

	fd, err := syscall.Open("/dev/bpf0", syscall.O_RDWR, syscall.S_IRUSR|syscall.S_IWUSR)
	if err != nil {
		panic(err)
	}
	err = syscall.SetBpfInterface(fd, "vtnet0")
	if err != nil {
		panic(err)
	}
	err = syscall.SetBpfImmediate(fd, enable)
	if err != nil {
		panic(err)
	}
	err = syscall.SetBpfHeadercmpl(fd, enable)
	if err != nil {
		panic(err)
	}

	var bufLen int
	bufLen, err = syscall.BpfBuflen(fd)
	if err != nil {
		panic(err)
	}
	fmt.Printf("buflen %d\n", bufLen)

	err = syscall.SetBpfPromisc(fd, enable)
	if err != nil {
		panic(err)
	}

	var n int
	for {
		buf := make([]byte, bufLen)
		n, err = syscall.Read(fd, buf)
		if err != nil {
			panic(err)
		}
		//fmt.Printf("% X\n", buf[:n])
		fmt.Printf("\npacket of size %d captured\n", n)
		fmt.Print(hex.Dump(buf[:n]))

	}
}
<commit_msg>cleanup<commit_after><|endoftext|>


<commit_before>// +build darwin
// +build cgo

package cpu

/*
#include <stdlib.h>
#include <sys/sysctl.h>
#include <sys/mount.h>
#include <mach/mach_init.h>
#include <mach/mach_host.h>
#include <mach/host_info.h>
#if TARGET_OS_MAC
#include <libproc.h>
#endif
#include <mach/processor_info.h>
#include <mach/vm_map.h>
*/
import "C"

import (
	"bytes"
	"encoding/binary"
	"fmt"
	"unsafe"
)

// these CPU times for darwin is borrowed from influxdb/telegraf.

func perCPUTimes() ([]TimesStat, error) {
	var (
		count   C.mach_msg_type_number_t
		cpuload *C.processor_cpu_load_info_data_t
		ncpu    C.natural_t
	)

	status := C.host_processor_info(C.host_t(C.mach_host_self()),
		C.PROCESSOR_CPU_LOAD_INFO,
		&ncpu,
		(*C.processor_info_array_t)(unsafe.Pointer(&cpuload)),
		&count)

	if status != C.KERN_SUCCESS {
		return nil, fmt.Errorf("host_processor_info error=%d", status)
	}

	// jump through some cgo casting hoops and ensure we properly free
	// the memory that cpuload points to
	target := C.vm_map_t(C.mach_task_self_)
	address := C.vm_address_t(uintptr(unsafe.Pointer(cpuload)))
	defer C.vm_deallocate(target, address, C.vm_size_t(ncpu))

	// the body of struct processor_cpu_load_info
	// aka processor_cpu_load_info_data_t
	var cpu_ticks [C.CPU_STATE_MAX]uint32

	// copy the cpuload array to a []byte buffer
	// where we can binary.Read the data
	size := int(ncpu) * binary.Size(cpu_ticks)
	buf := C.GoBytes(unsafe.Pointer(cpuload), C.int(size))

	bbuf := bytes.NewBuffer(buf)

	var ret []TimesStat

	for i := 0; i < int(ncpu); i++ {
		err := binary.Read(bbuf, binary.LittleEndian, &cpu_ticks)
		if err != nil {
			return nil, err
		}

		c := TimesStat{
			CPU:    fmt.Sprintf("cpu%d", i),
			User:   float64(cpu_ticks[C.CPU_STATE_USER]) / ClocksPerSec,
			System: float64(cpu_ticks[C.CPU_STATE_SYSTEM]) / ClocksPerSec,
			Nice:   float64(cpu_ticks[C.CPU_STATE_NICE]) / ClocksPerSec,
			Idle:   float64(cpu_ticks[C.CPU_STATE_IDLE]) / ClocksPerSec,
		}

		ret = append(ret, c)
	}

	return ret, nil
}

func allCPUTimes() ([]TimesStat, error) {
	var count C.mach_msg_type_number_t
	var cpuload C.host_cpu_load_info_data_t

	count = C.HOST_CPU_LOAD_INFO_COUNT

	status := C.host_statistics(C.host_t(C.mach_host_self()),
		C.HOST_CPU_LOAD_INFO,
		C.host_info_t(unsafe.Pointer(&cpuload)),
		&count)

	if status != C.KERN_SUCCESS {
		return nil, fmt.Errorf("host_statistics error=%d", status)
	}

	c := TimesStat{
		CPU:    "cpu-total",
		User:   float64(cpuload.cpu_ticks[C.CPU_STATE_USER]) / ClocksPerSec,
		System: float64(cpuload.cpu_ticks[C.CPU_STATE_SYSTEM]) / ClocksPerSec,
		Nice:   float64(cpuload.cpu_ticks[C.CPU_STATE_NICE]) / ClocksPerSec,
		Idle:   float64(cpuload.cpu_ticks[C.CPU_STATE_IDLE]) / ClocksPerSec,
	}

	return []TimesStat{c}, nil

}
<commit_msg>cpu[darwin]: fix Go arrays panic in cgo #279<commit_after>// +build darwin
// +build cgo

package cpu

/*
#include <stdlib.h>
#include <sys/sysctl.h>
#include <sys/mount.h>
#include <mach/mach_init.h>
#include <mach/mach_host.h>
#include <mach/host_info.h>
#if TARGET_OS_MAC
#include <libproc.h>
#endif
#include <mach/processor_info.h>
#include <mach/vm_map.h>
*/
import "C"

import (
	"bytes"
	"encoding/binary"
	"fmt"
	"unsafe"
)

// these CPU times for darwin is borrowed from influxdb/telegraf.

func perCPUTimes() ([]TimesStat, error) {
	var (
		count   C.mach_msg_type_number_t
		cpuload *C.processor_cpu_load_info_data_t
		ncpu    C.natural_t
	)

	status := C.host_processor_info(C.host_t(C.mach_host_self()),
		C.PROCESSOR_CPU_LOAD_INFO,
		&ncpu,
		(*C.processor_info_array_t)(unsafe.Pointer(&cpuload)),
		&count)

	if status != C.KERN_SUCCESS {
		return nil, fmt.Errorf("host_processor_info error=%d", status)
	}

	// jump through some cgo casting hoops and ensure we properly free
	// the memory that cpuload points to
	target := C.vm_map_t(C.mach_task_self_)
	address := C.vm_address_t(uintptr(unsafe.Pointer(cpuload)))
	defer C.vm_deallocate(target, address, C.vm_size_t(ncpu))

	// the body of struct processor_cpu_load_info
	// aka processor_cpu_load_info_data_t
	var cpu_ticks [C.CPU_STATE_MAX]uint32

	// copy the cpuload array to a []byte buffer
	// where we can binary.Read the data
	size := int(ncpu) * binary.Size(cpu_ticks)
	buf := (*[1 << 30]byte)(unsafe.Pointer(cpuload))[:size:size]

	bbuf := bytes.NewBuffer(buf)

	var ret []TimesStat

	for i := 0; i < int(ncpu); i++ {
		err := binary.Read(bbuf, binary.LittleEndian, &cpu_ticks)
		if err != nil {
			return nil, err
		}

		c := TimesStat{
			CPU:    fmt.Sprintf("cpu%d", i),
			User:   float64(cpu_ticks[C.CPU_STATE_USER]) / ClocksPerSec,
			System: float64(cpu_ticks[C.CPU_STATE_SYSTEM]) / ClocksPerSec,
			Nice:   float64(cpu_ticks[C.CPU_STATE_NICE]) / ClocksPerSec,
			Idle:   float64(cpu_ticks[C.CPU_STATE_IDLE]) / ClocksPerSec,
		}

		ret = append(ret, c)
	}

	return ret, nil
}

func allCPUTimes() ([]TimesStat, error) {
	var count C.mach_msg_type_number_t
	var cpuload C.host_cpu_load_info_data_t

	count = C.HOST_CPU_LOAD_INFO_COUNT

	status := C.host_statistics(C.host_t(C.mach_host_self()),
		C.HOST_CPU_LOAD_INFO,
		C.host_info_t(unsafe.Pointer(&cpuload)),
		&count)

	if status != C.KERN_SUCCESS {
		return nil, fmt.Errorf("host_statistics error=%d", status)
	}

	c := TimesStat{
		CPU:    "cpu-total",
		User:   float64(cpuload.cpu_ticks[C.CPU_STATE_USER]) / ClocksPerSec,
		System: float64(cpuload.cpu_ticks[C.CPU_STATE_SYSTEM]) / ClocksPerSec,
		Nice:   float64(cpuload.cpu_ticks[C.CPU_STATE_NICE]) / ClocksPerSec,
		Idle:   float64(cpuload.cpu_ticks[C.CPU_STATE_IDLE]) / ClocksPerSec,
	}

	return []TimesStat{c}, nil

}
<|endoftext|>